diff --git a/.asf.yaml b/.asf.yaml index b25937889..2c4346295 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -30,6 +30,7 @@ github: - hive - sql - kubernetes + - hacktoberfest enabled_merge_buttons: squash: true merge: false @@ -39,7 +40,7 @@ github: issues: true discussions: true wiki: false - projects: false + projects: true notifications: commits: commits@kyuubi.apache.org issues: notifications@kyuubi.apache.org diff --git a/.gitattributes b/.gitattributes index b3623c426..0df00fae2 100644 --- a/.gitattributes +++ b/.gitattributes @@ -18,7 +18,6 @@ .github/ export-ignore .idea/ export-ignore .readthedocs.yml export-ignore -.travis.yml export-ignore _config.yml export-ignore codecov.yml export-ignore licenses-binary/ export-ignore @@ -27,6 +26,7 @@ NOTICE-binary export-ignore *.bat text eol=crlf *.cmd text eol=crlf *.java text eol=lf +*.md text eol=lf *.scala text eol=lf *.xml text eol=lf *.py text eol=lf diff --git a/.github/ISSUE_TEMPLATE/code-contrib-task.yml b/.github/ISSUE_TEMPLATE/code-contrib-task.yml new file mode 100644 index 000000000..3191e4fe4 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/code-contrib-task.yml @@ -0,0 +1,115 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# +# This is a dedicated issue template for 2023 Kyuubi Code Contribution Program, all proposed +# tasks will be listed at https://github.com/orgs/apache/projects/296 after approval +# +name: 2023 Kyuubi Code Contribution Task +title: "[TASK][] " +description: Propose a task for 2023 Kyuubi Code Contribution Program +labels: [ "hacktoberfest" ] +body: + - type: markdown + attributes: + value: | + You are very welcome to propose new task for 2023 Kyuubi Code Contribution Program. + Your brilliant ideas keep Apache Kyuubi evolving. + Please replace the placeholder `` in the issue title with one of the following options: + - TRIVIAL - it's usually for new contributors to learn the contributor process, e.g. how to cut branch, + how to use GitHub to send PR, how to response with reviewers, the contributor should not stay at this + stage too long. + - EASY - tasks like minor bugs, or simple features without requirements of knowledge for whole Kyuubi + architecture. + - MEDIUM - tasks typical requires that contributors have knowledge on one or more Kyuubi components, + normally, unit tests and integration tests is also required to verify the implementations. + - CHALLENGE - tasks requires that contributors have deep knowledge on one or more Kyuubi components, + have good logical thinking and the ability to solve complex problems, be proficient in programming + skills or algorithms + + - type: checkboxes + attributes: + label: Code of Conduct + description: The Code of Conduct helps create a safe space for everyone. We require that everyone agrees to it. + options: + - label: > + I agree to follow this project's [Code of Conduct](https://www.apache.org/foundation/policies/conduct) + required: true + + - type: checkboxes + attributes: + label: Search before creating + options: + - label: > + I have searched in the [task list](https://github.com/orgs/apache/projects/296) and found no similar + tasks. + required: true + + - type: checkboxes + attributes: + label: Mentor + description: Mentor is required for MEDIUM and CHALLENGE tasks, to guide contributors to complete the task. + options: + - label: > + I have sufficient knowledge and experience of this task, and I volunteer to be the mentor of this task + to guide contributors to complete the task. + required: false + + - type: textarea + attributes: + label: Skill requirements + description: Which stills are required for contributors who want to take this task? + placeholder: | + e.g. + - Basic knowledge on Scala Programing Language + - Familiar with Apache Maven, Docker and GitHub Action + - Basic knowledge on network programing and Apache Thrift RPC framework + - Familiar with Apache Spark + - ... + validations: + required: true + + - type: textarea + attributes: + label: Background and Goals + description: What's the current problem, and what's the final status should be after the task is completed? + placeholder: > + Please describe the background and your goal for requesting this task. + validations: + required: true + + - type: textarea + attributes: + label: Implementation steps + description: How could it be implemented? + placeholder: > + Please list the implementation steps in as much detail as possible so that contributors who meet + the skill requirements could complete the task quickly and independently. + validations: + required: true + + - type: textarea + attributes: + label: Additional context + placeholder: > + Anything else that related to this task that the contributors need to know. + validations: + required: false + + - type: markdown + attributes: + value: "Thanks for taking the time to fill out this task form!" diff --git a/.github/ISSUE_TEMPLATE/dependency.yml b/.github/ISSUE_TEMPLATE/dependency.yml new file mode 100644 index 000000000..e71c7d1c6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/dependency.yml @@ -0,0 +1,109 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# See https://gh-community.github.io/issue-template-feedback/structured/ + +name: Dependency +title: ":arrow_up: Upgrade from to " +description: Keep upstream dependencies fresh and stable +labels: [ "kind:build, priority:major, good first issue, help wanted" ] +body: + - type: markdown + attributes: + value: | + Thank you for finding the time to report the issue! We really appreciate the community's efforts to improve Kyuubi. + + It doesn't really matter whether what you are reporting is a bug or not, just feel free to share the problem you have + encountered with the community. For best practices, if it is indeed a bug, please try your best to provide the reproducible + steps. If you want to ask questions or share ideas, please [subscribe to our mailing list](mailto:dev-subscribe@kyuubi.apache.org) + and send emails to [our mailing list](mailto:dev@kyuubi.apache.org), you can also head to our + [Discussions](https://github.com/apache/kyuubi/discussions) tab. + + - type: checkboxes + attributes: + label: Code of Conduct + description: The Code of Conduct helps create a safe space for everyone. We require that everyone agrees to it. + options: + - label: > + I agree to follow this project's [Code of Conduct](https://www.apache.org/foundation/policies/conduct) + required: true + + - type: checkboxes + attributes: + label: Search before asking + options: + - label: > + I have searched in the [issues](https://github.com/apache/kyuubi/issues?q=is%3Aissue) and found no similar + issues. + required: true + + - type: dropdown + id: priority + attributes: + label: Why do we need to upgrade this artifact? + options: + - Common Vulnerabilities and Exposures (CVE) + - Bugfixes + - Usage of New Features + - Performance Improvements + - Regular Updates + validations: + required: true + + - type: input + id: artifact + attributes: + label: Artifact Name + description: Which artifact shall be upgraded? + placeholder: e.g. spark-sql + value: https://mvnrepository.com/search?q= + validations: + required: true + + - type: input + id: versions + attributes: + label: Target Version + description: Which version shall be upgraded? + placeholder: e.g. 1.2.1 + validations: + required: true + + - type: textarea + id: changes + attributes: + label: Notable Changes + description: Please provide notable changes, or release notes if any + validations: + required: false + + - type: checkboxes + attributes: + label: Are you willing to submit PR? + description: > + A pull request is optional, but we are glad to help you in the contribution process + especially if you already know a good understanding of how to implement the fix. + Kyuubi is a community-driven project and we love to bring new contributors in. + options: + - label: Yes. I would be willing to submit a PR with guidance from the Kyuubi community to fix. + - label: No. I cannot submit a PR at this time. + + - type: markdown + attributes: + value: > + After changing the corresponding dependency version and before submitting your pull request, + it is necessary to execute `build/dependency.sh --replace` locally to update `dev/dependencyList`. diff --git a/.github/ISSUE_TEMPLATE/doc-improvement-report.yml b/.github/ISSUE_TEMPLATE/documentation.yml similarity index 67% rename from .github/ISSUE_TEMPLATE/doc-improvement-report.yml rename to .github/ISSUE_TEMPLATE/documentation.yml index 668ddb256..87b87a6cd 100644 --- a/.github/ISSUE_TEMPLATE/doc-improvement-report.yml +++ b/.github/ISSUE_TEMPLATE/documentation.yml @@ -15,16 +15,11 @@ # limitations under the License. # -name: Doc Improvement Report -title: "[DOCS] " +name: Documentation fixes or improvement +title: ":memo: Fix/Add for page" description: Fix errors, or improve the content or refactor architecture of online documentation -labels: ["kind:documentation"] +labels: ["kind:documentation,kind:minor,help wanted,good first issue"] body: - - type: markdown - attributes: - value: | - Thank you for finding the time to report the problem! We really appreciate the community efforts to improve Kyuubi. - - type: checkboxes attributes: label: Code of Conduct @@ -43,22 +38,25 @@ body: issues. required: true - - type: textarea + - type: dropdown + id: priority attributes: - label: Which parts of the documentation do you think need improvement? - description: Please describe the details with documentation you have. - placeholder: > - Please include links to the documentation that you want to improve and possibly screenshots showing - the details. Explain why do you think it needs to improve. Make sure you include view of the target - audience of the documentation. Please explain why you think the docs are wrong. + label: What type of changes will we make to the documentation? + options: + - Bugfixes + - Usage of New Feature + - Showcase + - Refactoring + - Typo, layout, grammar, spelling, punctuation errors, etc. + validations: + required: true - type: input id: versions attributes: label: Affects Version(s) description: Which versions of Kyuubi Documentation are affected by this issue? - placeholder: > - e.g. master/1.5.0/1.4.1/... + placeholder: e.g. master/1.5.0/1.4.1/... validations: required: true @@ -67,20 +65,9 @@ body: label: Improving the documentation description: How do you think the documentation can be improved? placeholder: > - Please explain how you think the documentation could be improved. Ideally specify where a new or missing - documentation should be added and what kind of information should be included. Sometimes people - writing the documentation do not realise that some assumptions they have might not be in the heads - of the reader, so try to explain exactly what you would like to see in the docs and why. - - - type: textarea - attributes: - label: Anything else - description: Anything else we need to know? - placeholder: > - How often does this problem occur? (Once? Every time? Only when certain conditions are met?) - Any relevant logs to include? Put them here inside fenced - ``` ``` blocks or inside a foldable details tag if it's long: -
x.log lots of stuff
+ Please include links to the documentation that you want to improve and possibly screenshots showing + the details. Explain why do you think it needs to improve. Make sure you include view of the target + audience of the documentation. Please explain why you think the docs are wrong. - type: checkboxes attributes: diff --git a/.github/PULL_REQUEST_TEMPLATE b/.github/PULL_REQUEST_TEMPLATE index bdb71f30f..3cab99d1f 100644 --- a/.github/PULL_REQUEST_TEMPLATE +++ b/.github/PULL_REQUEST_TEMPLATE @@ -20,4 +20,13 @@ Please clarify why the changes are needed. For instance, - [ ] Add screenshots for manual tests if appropriate -- [ ] [Run test](https://kyuubi.readthedocs.io/en/master/develop_tools/testing.html#running-tests) locally before make a pull request +- [ ] [Run test](https://kyuubi.readthedocs.io/en/master/contributing/code/testing.html#running-tests) locally before make a pull request + + +### _Was this patch authored or co-authored using generative AI tooling?_ + diff --git a/.github/actions/cache-engine-archives/action.yaml b/.github/actions/cache-engine-archives/action.yaml new file mode 100644 index 000000000..86a9ccafb --- /dev/null +++ b/.github/actions/cache-engine-archives/action.yaml @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: cache-engine-archives +description: 'Cache download engine archives from Apache Archives website used by Maven download plugin' +runs: + using: composite + steps: + - name: Cache Engine Archives + uses: actions/cache@v3 + with: + path: /tmp/engine-archives + key: engine-archives diff --git a/.github/actions/setup-maven/action.yaml b/.github/actions/setup-maven/action.yaml new file mode 100644 index 000000000..0cb4b54c2 --- /dev/null +++ b/.github/actions/setup-maven/action.yaml @@ -0,0 +1,31 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: setup-maven +description: 'Install and cache maven' +runs: + using: composite + steps: + - name: Restore cached Maven + uses: actions/cache@v3 + with: + path: build/apache-maven-* + key: setup-maven-${{ hashFiles('pom.xml') }} + restore-keys: setup-maven- + - name: Install Maven + shell: bash + run: build/mvn -v diff --git a/.github/labeler.yml b/.github/labeler.yml index a9f79a537..ecec12532 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -45,7 +45,6 @@ - ".gitattributes" - ".github/**/*" - ".gitignore" - - ".travis.yml" - "LICENSE" - "LICENSE-binary" - "NOTICE" @@ -103,7 +102,8 @@ "module:server": - "bin/kyuubi" - - "kyuubi-server/**/*" + - "kyuubi-server/src/**/*" + - "kyuubi-server/pom.xml" - "extension/server/kyuubi-server-plugin/**/*" "module:spark": @@ -122,3 +122,6 @@ "module:authz": - "extensions/spark/kyuubi-spark-authz/**/*" + +"module:ui": + - "kyuubi-server/web-ui/**/*" diff --git a/.github/workflows/dep.yml b/.github/workflows/dep.yml index 5ea4447cc..f39e5e6a2 100644 --- a/.github/workflows/dep.yml +++ b/.github/workflows/dep.yml @@ -23,11 +23,12 @@ on: - master - branch-* paths: - # dependency check happens only pom changes + # when pom or dependency workflow changes - '**/pom.xml' + - '.github/workflows/dep.yml' concurrency: - group: dep-${{ github.ref }} + group: dep-${{ github.head_ref || github.run_id }} cancel-in-progress: true jobs: @@ -43,9 +44,20 @@ jobs: java-version: 8 cache: 'maven' check-latest: false + - name: Setup Maven + uses: ./.github/actions/setup-maven + - name: Check kyuubi modules available + id: modules-check + run: >- + build/mvn dependency:resolve validate + -DincludeGroupIds="org.apache.kyuubi" -DincludeScope="compile" + -Pfast -Denforcer.skip=false + -pl kyuubi-ctl,kyuubi-server,kyuubi-assembly -am + continue-on-error: true - name: build env: MAVEN_OPTS: -Dorg.slf4j.simpleLogger.defaultLogLevel=error + if: steps.modules-check.conclusion == 'success' && steps.modules-check.outcome == 'failure' run: >- build/mvn clean install -Pflink-provided,spark-provided,hive-provided @@ -57,3 +69,7 @@ jobs: -pl kyuubi-ctl,kyuubi-server,kyuubi-assembly -am - name: Check dependency list run: build/dependency.sh + - name: Dependency Review + uses: actions/dependency-review-action@v3 + with: + fail-on-severity: moderate diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 000000000..55cb6b8b1 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,48 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: Docs + +on: + pull_request: + branches: + - master + +concurrency: + group: docs-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + sphinx: + name: sphinx-build + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.9' + cache: 'pip' + cache-dependency-path: docs/requirements.txt + - run: pip install -r docs/requirements.txt + - name: make html + run: make -d --directory docs html + - name: upload html + uses: actions/upload-artifact@v3 + with: + path: | + docs/_build/html/ + !docs/_build/html/_sources/ diff --git a/.github/workflows/license.yml b/.github/workflows/license.yml index 73ef05864..55ef485f8 100644 --- a/.github/workflows/license.yml +++ b/.github/workflows/license.yml @@ -26,7 +26,7 @@ on: - branch-* concurrency: - group: lincense-${{ github.ref }} + group: license-${{ github.head_ref || github.run_id }} cancel-in-progress: true jobs: @@ -45,7 +45,7 @@ jobs: - run: >- build/mvn org.apache.rat:apache-rat-plugin:check -Ptpcds -Pspark-block-cleaner -Pkubernetes-it - -Pspark-3.1 -Pspark-3.2 -Pspark-3.3 + -Pspark-3.1 -Pspark-3.2 -Pspark-3.3 -Pspark-3.4 -Pspark-3.5 - name: Upload rat report if: failure() uses: actions/upload-artifact@v3 diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 6bb2658ef..74c53ab08 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -28,11 +28,13 @@ on: - branch-* concurrency: - group: test-${{ github.ref }} + group: test-${{ github.head_ref || github.run_id }} cancel-in-progress: true env: - MVN_OPT: -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip -Dorg.slf4j.simpleLogger.defaultLogLevel=warn -Pjdbc-shaded + MVN_OPT: -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip -Dorg.slf4j.simpleLogger.defaultLogLevel=warn -Pjdbc-shaded,gen-policy -Dmaven.plugin.download.cache.path=/tmp/engine-archives + KUBERNETES_VERSION: v1.26.1 + MINIKUBE_VERSION: v1.29.0 jobs: default: @@ -44,24 +46,43 @@ jobs: java: - 8 - 11 + - 17 spark: - '3.1' - '3.2' - '3.3' + - '3.4' + - '3.5' spark-archive: [""] exclude-tags: [""] comment: ["normal"] include: - java: 8 - spark: '3.3' - spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.1.3 -Dspark.archive.name=spark-3.1.3-bin-hadoop3.2.tgz' - exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.IcebergTest' + spark: '3.4' + spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.1.3 -Dspark.archive.name=spark-3.1.3-bin-hadoop3.2.tgz -Pzookeeper-3.6' + exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.SparkLocalClusterTest' comment: 'verify-on-spark-3.1-binary' - java: 8 - spark: '3.3' - spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.2.3 -Dspark.archive.name=spark-3.2.3-bin-hadoop3.2.tgz' - exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.IcebergTest' + spark: '3.4' + spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.2.4 -Dspark.archive.name=spark-3.2.4-bin-hadoop3.2.tgz -Pzookeeper-3.6' + exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.SparkLocalClusterTest' comment: 'verify-on-spark-3.2-binary' + - java: 8 + spark: '3.4' + spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.3.3 -Dspark.archive.name=spark-3.3.3-bin-hadoop3.tgz -Pzookeeper-3.6' + exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.HudiTest,org.apache.kyuubi.tags.SparkLocalClusterTest' + comment: 'verify-on-spark-3.3-binary' + - java: 8 + spark: '3.4' + spark-archive: '-Dspark.archive.mirror=https://archive.apache.org/dist/spark/spark-3.5.0 -Dspark.archive.name=spark-3.5.0-bin-hadoop3.tgz -Pzookeeper-3.6' + exclude-tags: '-Dmaven.plugin.scalatest.exclude.tags=org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.PaimonTest,org.apache.kyuubi.tags.SparkLocalClusterTest' + comment: 'verify-on-spark-3.5-binary' + exclude: + # SPARK-33772: Spark supports JDK 17 since 3.3.0 + - java: 17 + spark: '3.1' + - java: 17 + spark: '3.2' env: SPARK_LOCAL_IP: localhost steps: @@ -75,19 +96,23 @@ jobs: java-version: ${{ matrix.java }} cache: 'maven' check-latest: false + - name: Cache Engine Archives + uses: ./.github/actions/cache-engine-archives + - name: Setup Maven + uses: ./.github/actions/setup-maven - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.9' + python-version: '3.9' - name: Build and test Kyuubi and Spark with maven w/o linters run: | TEST_MODULES="dev/kyuubi-codecov" ./build/mvn clean install ${MVN_OPT} -pl ${TEST_MODULES} -am \ - -Pspark-${{ matrix.spark }} ${{ matrix.spark-archive }} ${{ matrix.exclude-tags }} + -Pjava-${{ matrix.java }} -Pspark-${{ matrix.spark }} -Pspark-authz-hudi-test ${{ matrix.spark-archive }} ${{ matrix.exclude-tags }} - name: Code coverage if: | matrix.java == 8 && - matrix.spark == '3.2' && + matrix.spark == '3.4' && matrix.spark-archive == '' uses: codecov/codecov-action@v3 with: @@ -100,21 +125,20 @@ jobs: path: | **/target/unit-tests.log **/kyuubi-spark-sql-engine.log* + **/kyuubi-spark-batch-submit.log* - authz: - name: Kyuubi-AuthZ and Spark Test + scala-test: + name: Scala Test runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: + scala: + - '2.13' java: - - 8 - - 11 + - '8' spark: - - '3.0.3' - comment: ["normal"] - env: - SPARK_LOCAL_IP: localhost + - '3.4' steps: - uses: actions/checkout@v3 - name: Tune Runner VM @@ -126,19 +150,26 @@ jobs: java-version: ${{ matrix.java }} cache: 'maven' check-latest: false - - name: Build and test Kyuubi AuthZ with supported Spark versions + - name: Setup Maven + uses: ./.github/actions/setup-maven + - name: Cache Engine Archives + uses: ./.github/actions/cache-engine-archives + - name: Build on Scala ${{ matrix.scala }} run: | - TEST_MODULES="extensions/spark/kyuubi-spark-authz" - ./build/mvn clean test ${MVN_OPT} -pl ${TEST_MODULES} -am \ - -Dspark.version=${{ matrix.spark }} + TEST_MODULES="!externals/kyuubi-flink-sql-engine,!integration-tests/kyuubi-flink-it" + ./build/mvn clean install ${MVN_OPT} -pl ${TEST_MODULES} -am \ + -Pscala-${{ matrix.scala }} -Pjava-${{ matrix.java }} -Pspark-${{ matrix.spark }} - name: Upload test logs if: failure() uses: actions/upload-artifact@v3 with: - name: unit-tests-log-java-${{ matrix.java }}-spark-${{ matrix.spark }}-${{ matrix.comment }} + name: unit-tests-log-scala-${{ matrix.scala }}-java-${{ matrix.java }}-spark-${{ matrix.spark }} path: | **/target/unit-tests.log **/kyuubi-spark-sql-engine.log* + **/kyuubi-spark-batch-submit.log* + **/kyuubi-jdbc-engine.log* + **/kyuubi-hive-sql-engine.log* flink-it: name: Flink Test @@ -150,20 +181,20 @@ jobs: - 8 - 11 flink: - - '1.14' - - '1.15' - '1.16' + - '1.17' + - '1.18' flink-archive: [ "" ] comment: [ "normal" ] include: - java: 8 - flink: '1.15' - flink-archive: '-Dflink.archive.mirror=https://archive.apache.org/dist/flink/flink-1.14.5 -Dflink.archive.name=flink-1.14.5-bin-scala_2.12.tgz' - comment: 'verify-on-flink-1.14-binary' - - java: 8 - flink: '1.15' - flink-archive: '-Dflink.archive.mirror=https://archive.apache.org/dist/flink/flink-1.16.0 -Dflink.archive.name=flink-1.16.0-bin-scala_2.12.tgz' + flink: '1.17' + flink-archive: '-Dflink.archive.mirror=https://archive.apache.org/dist/flink/flink-1.16.1 -Dflink.archive.name=flink-1.16.1-bin-scala_2.12.tgz' comment: 'verify-on-flink-1.16-binary' + - java: 8 + flink: '1.17' + flink-archive: '-Dflink.archive.mirror=https://archive.apache.org/dist/flink/flink-1.18.0 -Dflink.archive.name=flink-1.18.0-bin-scala_2.12.tgz' + comment: 'verify-on-flink-1.18-binary' steps: - uses: actions/checkout@v3 - name: Tune Runner VM @@ -175,6 +206,10 @@ jobs: java-version: ${{ matrix.java }} cache: 'maven' check-latest: false + - name: Setup Maven + uses: ./.github/actions/setup-maven + - name: Cache Engine Archives + uses: ./.github/actions/cache-engine-archives - name: Build Flink with maven w/o linters run: | TEST_MODULES="externals/kyuubi-flink-sql-engine,integration-tests/kyuubi-flink-it" @@ -219,6 +254,10 @@ jobs: java-version: ${{ matrix.java }} cache: 'maven' check-latest: false + - name: Setup Maven + uses: ./.github/actions/setup-maven + - name: Cache Engine Archives + uses: ./.github/actions/cache-engine-archives - name: Build and test Hive with maven w/o linters run: | TEST_MODULES="externals/kyuubi-hive-sql-engine,integration-tests/kyuubi-hive-it" @@ -254,6 +293,10 @@ jobs: java-version: ${{ matrix.java }} cache: 'maven' check-latest: false + - name: Setup Maven + uses: ./.github/actions/setup-maven + - name: Cache Engine Archives + uses: ./.github/actions/cache-engine-archives - name: Build and test JDBC with maven w/o linters run: | TEST_MODULES="externals/kyuubi-jdbc-engine,integration-tests/kyuubi-jdbc-it" @@ -289,11 +332,15 @@ jobs: java-version: ${{ matrix.java }} cache: 'maven' check-latest: false + - name: Setup Maven + uses: ./.github/actions/setup-maven + - name: Cache Engine Archives + uses: ./.github/actions/cache-engine-archives - name: Build and test Trino with maven w/o linters run: | - TEST_MODULES="externals/kyuubi-trino-engine,integration-tests/kyuubi-trino-it" - ./build/mvn ${MVN_OPT} -pl ${TEST_MODULES} -am clean install -DskipTests - ./build/mvn ${MVN_OPT} -pl ${TEST_MODULES} test + TEST_MODULES="kyuubi-server,externals/kyuubi-trino-engine,externals/kyuubi-spark-sql-engine,externals/kyuubi-download,integration-tests/kyuubi-trino-it" + ./build/mvn ${MVN_OPT} -pl ${TEST_MODULES} -am -Pflink-provided -Phive-provided clean install -DskipTests + ./build/mvn -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip -pl ${TEST_MODULES} -am -Pflink-provided -Phive-provided test -Dtest=none -DwildcardSuites=org.apache.kyuubi.it.trino.operation.TrinoOperationSuite,org.apache.kyuubi.it.trino.server.TrinoFrontendSuite - name: Upload test logs if: failure() uses: actions/upload-artifact@v3 @@ -319,6 +366,10 @@ jobs: java-version: 8 cache: 'maven' check-latest: false + - name: Setup Maven + uses: ./.github/actions/setup-maven + - name: Cache Engine Archives + uses: ./.github/actions/cache-engine-archives - name: Run TPC-DS Tests run: | TEST_MODULES="kyuubi-server,extensions/spark/kyuubi-spark-connector-tpcds,extensions/spark/kyuubi-spark-connector-tpch" @@ -347,15 +398,22 @@ jobs: file: build/Dockerfile load: true tags: apache/kyuubi:latest - # from https://github.com/marketplace/actions/setup-minikube-kubernetes-cluster + - name: Cache Engine Archives + uses: ./.github/actions/cache-engine-archives - name: Setup Minikube - uses: manusa/actions-setup-minikube@v2.7.2 - with: - minikube version: 'v1.28.0' - kubernetes version: 'v1.25.4' - github token: ${{ secrets.GITHUB_TOKEN }} + run: | + # https://minikube.sigs.k8s.io/docs/start/ + curl -LO https://github.com/kubernetes/minikube/releases/download/${MINIKUBE_VERSION}/minikube-linux-amd64 + sudo install minikube-linux-amd64 /usr/local/bin/minikube + minikube start --cpus 2 --memory 4096 --kubernetes-version=${KUBERNETES_VERSION} --force + # https://minikube.sigs.k8s.io/docs/handbook/pushing/#7-loading-directly-to-in-cluster-container-runtime + minikube image load apache/kyuubi:latest + # pre-install spark into minikube + docker pull apache/spark:3.4.1 + minikube image load apache/spark:3.4.1 - name: kubectl pre-check run: | + kubectl get nodes kubectl get serviceaccount kubectl create serviceaccount kyuubi kubectl create clusterrolebinding kyuubi-role --clusterrole=edit --serviceaccount=default:kyuubi @@ -375,6 +433,9 @@ jobs: - name: Cat kyuubi server log if: failure() run: kubectl logs kyuubi-test + - name: Copy spark engine log from kyuubi pod + if: failure() + run: kubectl cp kyuubi-test:/opt/kyuubi/work ./target/work - name: Cat spark driver log if: failure() run: | @@ -387,6 +448,7 @@ jobs: name: unit-tests-log-kyuubi-on-k8s-it path: | **/target/unit-tests.log + **/target/work/** spark-on-k8s-it: name: Spark Engine On Kubernetes Integration Test @@ -394,14 +456,14 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 - # from https://github.com/marketplace/actions/setup-minikube-kubernetes-cluster + - name: Cache Engine Archives + uses: ./.github/actions/cache-engine-archives - name: Setup Minikube - uses: manusa/actions-setup-minikube@v2.7.2 - with: - minikube version: 'v1.25.2' - kubernetes version: 'v1.23.3' - driver: docker - start args: '--extra-config=kubeadm.ignore-preflight-errors=NumCPU --force --cpus 2 --memory 4096' + run: | + # https://minikube.sigs.k8s.io/docs/start/ + curl -LO https://github.com/kubernetes/minikube/releases/download/${MINIKUBE_VERSION}/minikube-linux-amd64 + sudo install minikube-linux-amd64 /usr/local/bin/minikube + minikube start --cpus 2 --memory 4096 --kubernetes-version=${KUBERNETES_VERSION} --force # in case: https://spark.apache.org/docs/latest/running-on-kubernetes.html#rbac - name: Create Service Account run: | @@ -413,7 +475,6 @@ jobs: run: >- ./build/mvn ${MVN_OPT} clean install -Pflink-provided,hive-provided - -Pspark-3.2 -Pkubernetes-it -Dtest=none -DwildcardSuites=org.apache.kyuubi.kubernetes.test.spark - name: Print Driver Pod logs @@ -451,6 +512,10 @@ jobs: java-version: ${{ matrix.java }} cache: 'maven' check-latest: false + - name: Setup Maven + uses: ./.github/actions/setup-maven + - name: Cache Engine Archives + uses: ./.github/actions/cache-engine-archives - name: zookeeper integration tests run: | export KYUUBI_IT_ZOOKEEPER_VERSION=${{ matrix.zookeeper }} diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 149da6d82..5ff634da6 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -43,6 +43,8 @@ jobs: java-version: 8 cache: 'maven' check-latest: false + - name: Setup Maven + uses: ./.github/actions/setup-maven - name: Build with Maven run: ./build/mvn clean install ${{ matrix.profiles }} -Dmaven.javadoc.skip=true -V - name: Upload test logs diff --git a/.github/workflows/docker-image.yml b/.github/workflows/publish-snapshot-docker.yml similarity index 94% rename from .github/workflows/docker-image.yml rename to .github/workflows/publish-snapshot-docker.yml index fb680cdc9..44197d92b 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/publish-snapshot-docker.yml @@ -15,12 +15,11 @@ # limitations under the License. # -name: Publish Docker image +name: Publish Snapshot Docker Image on: - push: - branches: - - master + schedule: + - cron: '0 0 * * *' jobs: push_to_registry: @@ -62,4 +61,4 @@ jobs: REPO_NAME: ndap/kyuubi run: | aws ecr describe-repositories --repository-names ${REPO_NAME} || aws ecr create-repository --repository-name ${REPO_NAME} - docker push $ECR_REGISTRY/$REPO_NAME:1.7.0-spark${{ matrix.spark-version }} + docker push $ECR_REGISTRY/$REPO_NAME:1.9.0-spark${{ matrix.spark-version }} diff --git a/.github/workflows/publish-snapshot.yml b/.github/workflows/publish-snapshot-nexus.yml similarity index 61% rename from .github/workflows/publish-snapshot.yml rename to .github/workflows/publish-snapshot-nexus.yml index acd04bfab..b4191396b 100644 --- a/.github/workflows/publish-snapshot.yml +++ b/.github/workflows/publish-snapshot-nexus.yml @@ -15,11 +15,11 @@ # limitations under the License. # -name: Publish Snapshot +name: Publish Snapshot Nexus on: schedule: - - cron: '0 0 * * *' + - cron: '0 0 * * *' jobs: publish-snapshot: @@ -30,30 +30,31 @@ jobs: matrix: branch: - master - - branch-1.6 - - branch-1.5 + - branch-1.7 + - branch-1.8 profiles: - -Pflink-provided,spark-provided,hive-provided,spark-3.1 - - -Pflink-provided,spark-provided,hive-provided,spark-3.2,tpcds + - -Pflink-provided,spark-provided,hive-provided,spark-3.2 + - -Pflink-provided,spark-provided,hive-provided,spark-3.3,tpcds include: - branch: master - profiles: -Pflink-provided,spark-provided,hive-provided,spark-3.3 - - branch: branch-1.6 - profiles: -Pflink-provided,spark-provided,hive-provided,spark-3.3 + profiles: -Pflink-provided,spark-provided,hive-provided,spark-3.4 + - branch: branch-1.8 + profiles: -Pflink-provided,spark-provided,hive-provided,spark-3.4 steps: - - name: Checkout repository - uses: actions/checkout@v3 - with: - ref: ${{ matrix.branch }} - - name: Setup JDK 8 - uses: actions/setup-java@v3 - with: - distribution: temurin - java-version: 8 - cache: 'maven' - check-latest: false - - name: Publish snapshot - ${{ matrix.branch }} - env: - ASF_USERNAME: ${{ secrets.NEXUS_USER }} - ASF_PASSWORD: ${{ secrets.NEXUS_PW }} - run: ./build/mvn clean deploy -s ./build/release/asf-settings.xml -DskipTests ${{ matrix.profiles }} + - name: Checkout repository + uses: actions/checkout@v3 + with: + ref: ${{ matrix.branch }} + - name: Setup JDK 8 + uses: actions/setup-java@v3 + with: + distribution: temurin + java-version: 8 + cache: 'maven' + check-latest: false + - name: Publish Snapshot Jar to Nexus - ${{ matrix.branch }} + env: + ASF_USERNAME: ${{ secrets.NEXUS_USER }} + ASF_PASSWORD: ${{ secrets.NEXUS_PW }} + run: build/mvn clean deploy -s build/release/asf-settings.xml -DskipTests ${{ matrix.profiles }} diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml index c848a2f8c..87823ddbd 100644 --- a/.github/workflows/style.yml +++ b/.github/workflows/style.yml @@ -24,7 +24,7 @@ on: - branch-* concurrency: - group: linter-${{ github.ref }} + group: linter-${{ github.head_ref || github.run_id }} cancel-in-progress: true jobs: @@ -34,10 +34,12 @@ jobs: strategy: matrix: profiles: - - '-Pflink-provided,hive-provided,spark-provided,spark-block-cleaner,spark-3.3,spark-3.2,spark-3.1,tpcds' + - '-Pflink-provided,hive-provided,spark-provided,spark-block-cleaner,spark-3.5,spark-3.4,spark-3.3,spark-3.2,tpcds,kubernetes-it' steps: - uses: actions/checkout@v3 + with: + fetch-depth: 0 - name: Setup JDK 8 uses: actions/setup-java@v3 with: @@ -45,6 +47,8 @@ jobs: java-version: 8 cache: 'maven' check-latest: false + - name: Setup Maven + uses: ./.github/actions/setup-maven - name: Setup Python 3 uses: actions/setup-python@v4 with: @@ -63,11 +67,16 @@ jobs: MVN_OPT="-DskipTests -Dorg.slf4j.simpleLogger.defaultLogLevel=warn -Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip" build/mvn clean install ${MVN_OPT} -Pflink-provided,hive-provided,spark-provided,spark-block-cleaner,spark-3.2,tpcds build/mvn clean install ${MVN_OPT} -pl extensions/spark/kyuubi-extension-spark-3-1 -Pspark-3.1 - build/mvn clean install ${MVN_OPT} -pl extensions/spark/kyuubi-extension-spark-3-3,extensions/spark/kyuubi-spark-connector-kudu,extensions/spark/kyuubi-spark-connector-hive -Pspark-3.3 + build/mvn clean install ${MVN_OPT} -pl extensions/spark/kyuubi-extension-spark-3-3,extensions/spark/kyuubi-spark-connector-hive -Pspark-3.3 + build/mvn clean install ${MVN_OPT} -pl extensions/spark/kyuubi-extension-spark-3-4 -Pspark-3.4 + build/mvn clean install ${MVN_OPT} -pl extensions/spark/kyuubi-extension-spark-3-5 -Pspark-3.5 - name: Scalastyle with maven id: scalastyle-check - run: build/mvn scalastyle:check ${{ matrix.profiles }} + # Check with Spark 3.1 profile separately as it use Iceberg 1.3.1 which is not compatible with Spark 3.5+ + run: | + build/mvn scalastyle:check ${{ matrix.profiles }} + build/mvn scalastyle:check -Pflink-provided,hive-provided,spark-provided,spark-3.1 - name: Print scalastyle error report if: failure() && steps.scalastyle-check.outcome != 'success' run: >- @@ -81,15 +90,15 @@ jobs: run: | SPOTLESS_BLACK_VERSION=$(build/mvn help:evaluate -Dexpression=spotless.python.black.version -q -DforceStdout) pip install black==$SPOTLESS_BLACK_VERSION - build/mvn spotless:check ${{ matrix.profiles }} -Pspotless-python + build/mvn spotless:check ${{ matrix.profiles }} -Pspotless-python,spark-3.1 - name: setup npm uses: actions/setup-node@v3 with: - node-version: 16 + node-version: 18 - name: Web UI Style with node run: | cd ./kyuubi-server/web-ui - npm install pnpm -g + npm install pnpm@8 -g pnpm install pnpm run lint echo "---------------------------------------Notice------------------------------------" @@ -102,10 +111,32 @@ jobs: echo "---------------------------------------------------------------------------------" shellcheck: - name: Shellcheck + name: Super Linter and Shellcheck runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v3 + - name: Super Linter Checks + uses: github/super-linter/slim@v5 + env: + CREATE_LOG_FILE: true + ERROR_ON_MISSING_EXEC_BIT: true + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + IGNORE_GENERATED_FILES: true + IGNORE_GITIGNORED_FILES: true + LINTER_RULES_PATH: / + LOG_LEVEL: NOTICE + SUPPRESS_POSSUM: true + VALIDATE_BASH_EXEC: true + VALIDATE_ENV: true + VALIDATE_JSONC: true + VALIDATE_POWERSHELL: true + VALIDATE_XML: true + - name: Upload Super Linter logs + if: failure() + uses: actions/upload-artifact@v3 + with: + name: super-linter-log + path: super-linter.log - name: check bin directory uses: ludeeus/action-shellcheck@1.1.0 with: diff --git a/.github/workflows/web-ui.yml b/.github/workflows/web-ui.yml index 08c97cfc9..9de7a599d 100644 --- a/.github/workflows/web-ui.yml +++ b/.github/workflows/web-ui.yml @@ -11,7 +11,7 @@ on: - branch-* concurrency: - group: web-ui-${{ github.ref }} + group: web-ui-${{ github.head_ref || github.run_id }} cancel-in-progress: true jobs: @@ -21,14 +21,35 @@ jobs: steps: - name: checkout uses: actions/checkout@v3 - - name: setup npm + - name: Setup JDK 8 + uses: actions/setup-java@v3 + with: + distribution: temurin + java-version: 8 + - name: Setup Maven + uses: ./.github/actions/setup-maven + - name: Get NodeJS and PNPM version + run: | + NODEJS_VERSION=$(build/mvn help:evaluate -Dexpression=node.version -q -DforceStdout) + PNPM_VERSION=$(build/mvn help:evaluate -Dexpression=pnpm.version -q -DforceStdout) + echo "NODEJS_VERSION=${NODEJS_VERSION}" >> "$GITHUB_ENV" + echo "PNPM_VERSION=${PNPM_VERSION}" >> "$GITHUB_ENV" + - name: Setup Nodejs and NPM uses: actions/setup-node@v3 with: - node-version: 16 + node-version: ${{env.NODEJS_VERSION}} + cache: npm + cache-dependency-path: ./kyuubi-server/web-ui/package.json + - name: Cache NPM dependencies + uses: actions/cache@v3 + with: + path: ./kyuubi-server/web-ui/node_modules + key: webui-dependencies-${{ hashFiles('kyuubi-server/web-ui/pnpm-lock.yaml') }} + restore-keys: webui-dependencies- - name: npm run coverage & build run: | cd ./kyuubi-server/web-ui - npm install pnpm -g + npm install pnpm@${PNPM_VERSION} -g pnpm install pnpm run coverage pnpm run build diff --git a/.gitignore b/.gitignore index d2c1ba3b7..a2f6fb1ef 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ *#*# *.#* +*.db *.iml *.ipr *.iws @@ -32,11 +33,7 @@ .ensime_lucene .generated-mima* .vscode/ -# The star is required for further !/.idea/ to work, see https://git-scm.com/docs/gitignore -/.idea/* -# Icon for JetBrains Toolbox -!/.idea/icon.png -!/.idea/vcs.xml +.idea/ .idea_modules/ .project .pydevproject @@ -59,10 +56,9 @@ hs_err_pid* spark-warehouse/ metastore_db derby.log -ldap +rest-audit.log **/dependency-reduced-pom.xml -metrics/report.json -metrics/.report.json.crc +metrics/ /kyuubi-ha/embedded_zookeeper/ embedded_zookeeper/ /externals/kyuubi-spark-sql-engine/operation_logs/ diff --git a/.idea/vcs.xml b/.idea/vcs.xml index 740593019..9c45aa8a4 100644 --- a/.idea/vcs.xml +++ b/.idea/vcs.xml @@ -25,7 +25,7 @@ GitHub share the sequence number of issues and pull requests, and it will redirect to the right place when the the sequence number not match kind. --> - diff --git a/.rat-excludes b/.rat-excludes index 86c38ec99..6823fa44e 100644 --- a/.rat-excludes +++ b/.rat-excludes @@ -39,17 +39,14 @@ build/scala-*/** **/*.output.schema **/apache-kyuubi-*-bin*/** **/benchmarks/** -**/jquery-*.min.js -**/semantic.min.js -**/semantic.min.css -**/icon.png -**/icon.min.css -**/org/apache/kyuubi/ui/static/assets/** -**/org/apache/kyuubi/ui/swagger/** **/org.apache.spark.status.AppHistoryServerPlugin **/metadata-store-schema*.sql **/*.derby.sql **/*.mysql.sql +**/*.sqlite.sql +**/node/** +**/web-ui/dist/** +**/web-ui/coverage/** **/pnpm-lock.yaml **/node_modules/** **/gen/* diff --git a/.readthedocs.yml b/.readthedocs.yaml similarity index 76% rename from .readthedocs.yml rename to .readthedocs.yaml index 671f29266..115d9c338 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yaml @@ -16,23 +16,19 @@ # version: 2 +build: + os: ubuntu-22.04 + tools: + python: "3.11" -# Build documentation in the docs/ directory with Sphinx sphinx: builder: html configuration: docs/conf.py -# Build documentation with MkDocs -#mkdocs: -# configuration: mkdocs.yml - -# Optionally build your docs in additional formats such as PDF formats: - pdf - epub -# Optionally set the version of Python and requirements required to build your docs python: - version: 3.7 install: - requirements: docs/requirements.txt diff --git a/.scalafmt.conf b/.scalafmt.conf index e682a17f7..b0e130715 100644 --- a/.scalafmt.conf +++ b/.scalafmt.conf @@ -1,4 +1,4 @@ -version = 3.7.1 +version = 3.7.5 runner.dialect=scala212 project.git=true diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index c09fa9566..000000000 --- a/.travis.yml +++ /dev/null @@ -1,71 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -sudo: required -dist: focal -arch: arm64-graviton2 -group: edge -virt: vm -env: SPARK_LOCAL_IP=localhost - -branches: - only: - - master - -language: java - -matrix: - include: - - name: Build Kyuubi common on Linux ARM64 - script: - - ./build/mvn test $MVN_ARGS -pl kyuubi-common,kyuubi-zookeeper,kyuubi-ha,kyuubi-ctl,kyuubi-metrics,kyuubi-hive-beeline,kyuubi-hive-jdbc,extensions/server/kyuubi-server-plugin -am - - name: Build Kyuubi Flink on Linux ARM64 - script: - - ./build/mvn test $MVN_ARGS -pl externals/kyuubi-flink-sql-engine,integration-tests/kyuubi-flink-it - - name: Build Kyuubi Spark on Linux ARM64 - script: - - ./build/mvn test $MVN_ARGS -pl externals/kyuubi-spark-sql-engine - - ./build/mvn test $MVN_ARGS -pl kyuubi-server -DwildcardSuites=org.apache.kyuubi.operation.KyuubiOperationPerUserSuite - - name: Build Kyuubi Trino on Linux ARM64 - script: - - ./build/mvn test $MVN_ARGS -pl externals/kyuubi-trino-engine,integration-tests/kyuubi-trino-it - - name: Build Kyuubi Hive on Linux ARM64 - script: - - ./build/mvn test $MVN_ARGS -pl externals/kyuubi-hive-sql-engine,integration-tests/kyuubi-hive-it - -cache: - directories: - - $HOME/.m2 - -install: - - sudo apt update - - sudo apt install -y openjdk-8-jdk - - export JAVA_HOME="/usr/lib/jvm/java-8-openjdk-${TRAVIS_CPU_ARCH}" - - export PATH="$JAVA_HOME/bin:/usr/share/maven/bin:$PATH" - - ./build/mvn --version - -before_script: - - export MVN_ARGS="-Dmaven.javadoc.skip=true -Drat.skip=true -Dscalastyle.skip=true -Dspotless.check.skip -V -B -ntp -Dorg.slf4j.simpleLogger.defaultLogLevel=warn -Pjdbc-shaded" - - ./build/mvn clean install -DskipTests $MVN_ARGS - - -after_success: - - echo "Travis exited with ${TRAVIS_TEST_RESULT}" - -after_failure: - - echo "Travis exited with ${TRAVIS_TEST_RESULT}" - - for log in `find * -name "unit-tests.log"`; do echo "=========$log========="; grep "ERROR" $log -A 100 -B 5; done diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ef28d560e..dc9094b8b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -60,4 +60,4 @@ TBD, please be patient for the surprise. ## IDE Setup Guide -[IntelliJ IDEA Setup Guide](https://kyuubi.readthedocs.io/en/master/develop_tools/idea_setup.html) +[IntelliJ IDEA Setup Guide](https://kyuubi.readthedocs.io/en/master/contributing/code/idea_setup.html) diff --git a/LICENSE b/LICENSE index 837c40027..261eeb9e9 100644 --- a/LICENSE +++ b/LICENSE @@ -199,20 +199,3 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - ------------------------------------------------------------------------------------- -This product bundles various third-party components under other open source licenses. -This section summarizes those components and their licenses. See licenses/ -for text of these licenses. - -Apache License Version 2.0 --------------------------- -kyuubi-server/src/main/resources/org/apache/kyuubi/ui/swagger/* - -MIT license ------------ -kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/assets/fonts/* -kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/icon.min.css -kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/semantic.min.css -kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/semantic.min.js -kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/jquery-3.6.0.min.js diff --git a/LICENSE-binary b/LICENSE-binary index 017dd53f8..748842a61 100644 --- a/LICENSE-binary +++ b/LICENSE-binary @@ -212,9 +212,6 @@ com.google.android:annotations commons-lang:commons-lang commons-logging:commons-logging org.apache.commons:commons-lang3 -org.apache.curator:curator-client -org.apache.curator:curator-framework -org.apache.curator:curator-recipes org.apache.derby:derby com.google.errorprone:error_prone_annotations net.jodah:failsafe @@ -264,6 +261,7 @@ io.etcd:jetcd-api io.etcd:jetcd-common io.etcd:jetcd-core io.etcd:jetcd-grpc +org.eclipse.jetty:jetty-client org.eclipse.jetty:jetty-http org.eclipse.jetty:jetty-io org.eclipse.jetty:jetty-security @@ -271,13 +269,13 @@ org.eclipse.jetty:jetty-server org.eclipse.jetty:jetty-servlet org.eclipse.jetty:jetty-util-ajax org.eclipse.jetty:jetty-util +org.eclipse.jetty:jetty-proxy org.apache.thrift:libfb303 org.apache.thrift:libthrift org.apache.logging.log4j:log4j-1.2-api org.apache.logging.log4j:log4j-api org.apache.logging.log4j:log4j-core org.apache.logging.log4j:log4j-slf4j-impl -org.webjars:swagger-ui org.yaml:snakeyaml io.dropwizard.metrics:metrics-core io.dropwizard.metrics:metrics-jmx @@ -318,16 +316,24 @@ io.swagger.core.v3:swagger-jaxrs2 io.swagger.core.v3:swagger-models io.vertx:vertx-core io.vertx:vertx-grpc -org.apache.zookeeper:zookeeper +com.squareup.retrofit2:retrofit +com.squareup.okhttp3:okhttp +org.apache.kafka:kafka-clients +org.lz4:lz4-java +org.xerial.snappy:snappy-java +org.xerial:sqlite-jdbc BSD ------------ +org.antlr:antlr-runtime org.antlr:antlr4-runtime +org.antlr:ST4 jline:jline com.thoughtworks.paranamer:paranamer dk.brics.automaton:automaton com.google.protobuf:protobuf-java-util com.google.protobuf:protobuf-java +com.github.luben:zstd-jni Eclipse Distribution License - v 1.0 ------------------------------------ @@ -354,12 +360,9 @@ org.codehaus.mojo:animal-sniffer-annotations org.slf4j:slf4j-api org.slf4j:jcl-over-slf4j org.slf4j:jul-over-slf4j - -kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/assets/fonts/* -kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/icon.min.css -kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/semantic.min.css -kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/semantic.min.js -kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/jquery-3.6.0.min.js +com.theokanning.openai-gpt3-java:api +com.theokanning.openai-gpt3-java:client +com.theokanning.openai-gpt3-java:service This product also contains the following third-party components, the information is auto-generated by `pnpm licenses list --prod`. @@ -367,14 +370,20 @@ is auto-generated by `pnpm licenses list --prod`. ┌────────────────────────────────────┬──────────────┐ │ Package │ License │ ├────────────────────────────────────┼──────────────┤ +│ swagger-ui-dist │ Apache-2.0 │ +├────────────────────────────────────┼──────────────┤ │ typescript │ Apache-2.0 │ ├────────────────────────────────────┼──────────────┤ +│ moo │ BSD-3-Clause │ +├────────────────────────────────────┼──────────────┤ │ normalize-wheel-es │ BSD-3-Clause │ ├────────────────────────────────────┼──────────────┤ │ source-map │ BSD-3-Clause │ ├────────────────────────────────────┼──────────────┤ │ source-map-js │ BSD-3-Clause │ ├────────────────────────────────────┼──────────────┤ +│ railroad-diagrams │ CC0-1.0 │ +├────────────────────────────────────┼──────────────┤ │ picocolors │ ISC │ ├────────────────────────────────────┼──────────────┤ │ @babel/helper-string-parser │ MIT │ @@ -447,12 +456,18 @@ is auto-generated by `pnpm licenses list --prod`. ├────────────────────────────────────┼──────────────┤ │ combined-stream │ MIT │ ├────────────────────────────────────┼──────────────┤ +│ commander │ MIT │ +├────────────────────────────────────┼──────────────┤ │ csstype │ MIT │ ├────────────────────────────────────┼──────────────┤ +│ date-fns │ MIT │ +├────────────────────────────────────┼──────────────┤ │ dayjs │ MIT │ ├────────────────────────────────────┼──────────────┤ │ delayed-stream │ MIT │ ├────────────────────────────────────┼──────────────┤ +│ discontinuous-range │ MIT │ +├────────────────────────────────────┼──────────────┤ │ element-plus │ MIT │ ├────────────────────────────────────┼──────────────┤ │ escape-html │ MIT │ @@ -463,6 +478,8 @@ is auto-generated by `pnpm licenses list --prod`. ├────────────────────────────────────┼──────────────┤ │ form-data │ MIT │ ├────────────────────────────────────┼──────────────┤ +│ get-stdin │ MIT │ +├────────────────────────────────────┼──────────────┤ │ lodash │ MIT │ ├────────────────────────────────────┼──────────────┤ │ lodash-es │ MIT │ @@ -477,16 +494,26 @@ is auto-generated by `pnpm licenses list --prod`. ├────────────────────────────────────┼──────────────┤ │ mime-types │ MIT │ ├────────────────────────────────────┼──────────────┤ +│ monaco-editor │ MIT │ +├────────────────────────────────────┼──────────────┤ │ nanoid │ MIT │ ├────────────────────────────────────┼──────────────┤ +│ nearley │ MIT │ +├────────────────────────────────────┼──────────────┤ │ pinia │ MIT │ ├────────────────────────────────────┼──────────────┤ │ pinia-plugin-persistedstate │ MIT │ ├────────────────────────────────────┼──────────────┤ │ postcss │ MIT │ ├────────────────────────────────────┼──────────────┤ +│ randexp │ MIT │ +├────────────────────────────────────┼──────────────┤ +│ ret │ MIT │ +├────────────────────────────────────┼──────────────┤ │ sourcemap-codec │ MIT │ ├────────────────────────────────────┼──────────────┤ +│ sql-formatter │ MIT │ +├────────────────────────────────────┼──────────────┤ │ to-fast-properties │ MIT │ ├────────────────────────────────────┼──────────────┤ │ vue │ MIT │ @@ -496,4 +523,6 @@ is auto-generated by `pnpm licenses list --prod`. │ vue-i18n │ MIT │ ├────────────────────────────────────┼──────────────┤ │ vue-router │ MIT │ +├────────────────────────────────────┼──────────────┤ +│ argparse │ Python-2.0 │ └────────────────────────────────────┴──────────────┘ diff --git a/NOTICE-binary b/NOTICE-binary index ef58e21f6..40ec15010 100644 --- a/NOTICE-binary +++ b/NOTICE-binary @@ -92,15 +92,6 @@ Copyright 2001-2020 The Apache Software Foundation Apache Commons Logging Copyright 2003-2013 The Apache Software Foundation -Curator Client -Copyright 2011-2017 The Apache Software Foundation - -Curator Framework -Copyright 2011-2017 The Apache Software Foundation - -Curator Recipes -Copyright 2011-2017 The Apache Software Foundation - ========================================================================= == NOTICE file corresponding to section 4(d) of the Apache License, == Version 2.0, in this case for the Apache Derby distribution. @@ -1236,7 +1227,7 @@ This product optionally depends on 'zstd-jni', a zstd-jni Java compression and decompression library, which can be obtained at: * LICENSE: - * license/LICENSE.zstd-jni.txt (Apache License 2.0) + * license/LICENSE.zstd-jni.txt (BSD License) * HOMEPAGE: * https://github.com/luben/zstd-jni @@ -1370,3 +1361,26 @@ decompression for Java., which can be obtained at: * HOMEPAGE: * https://github.com/hyperxpro/Brotli4j +This product depends on 'kafka-clients', Java clients for Kafka, +which can be obtained at: + + * LICENSE: + * license/LICENSE.kafka.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/apache/kafka + +This product optionally depends on 'snappy-java', Snappy compression and +decompression for Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.snappy-java.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/xerial/snappy-java + +This product optionally depends on 'lz4-java', Lz4 compression and +decompression for Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.lz4-java.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/lz4/lz4-java diff --git a/README.md b/README.md index b38d69334..d87cfabd8 100644 --- a/README.md +++ b/README.md @@ -1,42 +1,56 @@ +- Licensed to the Apache Software Foundation (ASF) under one or more +- contributor license agreements. See the NOTICE file distributed with +- this work for additional information regarding copyright ownership. +- The ASF licenses this file to You under the Apache License, Version 2.0 +- (the "License"); you may not use this file except in compliance with +- the License. You may obtain a copy of the License at +- +- http://www.apache.org/licenses/LICENSE-2.0 +- +- Unless required by applicable law or agreed to in writing, software +- distributed under the License is distributed on an "AS IS" BASIS, +- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +- See the License for the specific language governing permissions and +- limitations under the License. +--> + +

+ Kyuubi logo +

+ +

+ + + + + + + + + + + + + + + +

+

+ Project + - + Documentation + - + Who's using +

# Apache Kyuubi -Kyuubi logo - -[![License](https://img.shields.io/badge/license-Apache%202-blue.svg)](https://www.apache.org/licenses/LICENSE-2.0.html) -[![Release](https://img.shields.io/github/v/release/apache/kyuubi?label=release)](https://github.com/apache/kyuubi/releases) -[![](https://tokei.rs/b1/github.com/apache/kyuubi)](https://github.com/apache/kyuubi) -[![codecov](https://codecov.io/gh/apache/kyuubi/branch/master/graph/badge.svg)](https://codecov.io/gh/apache/kyuubi) -![GitHub Workflow Status](https://img.shields.io/github/workflow/status/apache/kyuubi/Kyuubi/master?style=plastic) -[![Travis](https://api.travis-ci.com/apache/kyuubi.svg?branch=master)](https://travis-ci.com/apache/kyuubi) -[![Documentation Status](https://readthedocs.org/projects/kyuubi/badge/?version=latest)](https://kyuubi.readthedocs.io/en/master/) -![GitHub top language](https://img.shields.io/github/languages/top/apache/kyuubi) -[![Commit activity](https://img.shields.io/github/commit-activity/m/apache/kyuubi)](https://github.com/apache/kyuubi/graphs/commit-activity) -[![Average time to resolve an issue](http://isitmaintained.com/badge/resolution/apache/kyuubi.svg)](http://isitmaintained.com/project/apache/kyuubi "Average time to resolve an issue") -[![Percentage of issues still open](http://isitmaintained.com/badge/open/apache/kyuubi.svg)](http://isitmaintained.com/project/apache/kyuubi "Percentage of issues still open") - - -## What is Kyuubi? - Apache Kyuubi™ is a distributed and multi-tenant gateway to provide serverless SQL on data warehouses and lakehouses. +## What is Kyuubi? + Kyuubi provides a pure SQL gateway through Thrift JDBC/ODBC interface for end-users to manipulate large-scale data with pre-programmed and extensible Spark SQL engines. This "out-of-the-box" model minimizes the barriers and costs for end-users to use Spark at the client side. At the server-side, Kyuubi server and engines' multi-tenant architecture provides the administrators a way to achieve computing resource isolation, data security, high availability, high client concurrency, etc. ![](./docs/imgs/kyuubi_positioning.png) @@ -45,19 +59,16 @@ Kyuubi provides a pure SQL gateway through Thrift JDBC/ODBC interface for end-us - [x] Multi-tenant Spark Support - [x] Running Spark in a serverless way - ### Target Users Kyuubi's goal is to make it easy and efficient for `anyone` to use Spark(maybe other engines soon) and facilitate users to handle big data like ordinary data. Here, `anyone` means that users do not need to have a Spark technical background but a human language, SQL only. Sometimes, SQL skills are unnecessary when integrating Kyuubi with Apache Superset, which supports rich visualizations and dashboards. - In typical big data production environments with Kyuubi, there should be system administrators and end-users. - System administrators: A small group consists of Spark experts responsible for Kyuubi deployment, configuration, and tuning. - End-users: Focus on business data of their own, not where it stores, how it computes. -Additionally, the Kyuubi community will continuously optimize the whole system with various features, such as History-Based Optimizer, Auto-tuning, Materialized View, SQL Dialects, Functions, e.t.c. - +Additionally, the Kyuubi community will continuously optimize the whole system with various features, such as History-Based Optimizer, Auto-tuning, Materialized View, SQL Dialects, Functions, etc. ### Usage scenarios @@ -71,8 +82,7 @@ HiveServer2 can identify and authenticate a caller, and then if the caller also Kyuubi extends the use of STS in a multi-tenant model based on a unified interface and relies on the concept of multi-tenancy to interact with cluster managers to finally gain the ability of resources sharing/isolation and data security. The loosely coupled architecture of the Kyuubi server and engine dramatically improves the client concurrency and service stability of the service itself. - -#### DataLake/LakeHouse Support +#### DataLake/Lakehouse Support The vision of Kyuubi is to unify the portal and become an easy-to-use data lake management platform. Different kinds of workloads, such as ETL processing and BI analytics, can be supported by one platform, using one copy of data, with one SQL interface. @@ -80,30 +90,20 @@ The vision of Kyuubi is to unify the portal and become an easy-to-use data lake - Multiple Catalogs support - SQL Standard Authorization support for DataLake(coming) - #### Cloud Native Support Kyuubi can deploy its engines on different kinds of Cluster Managers, such as, Hadoop YARN, Kubernetes, etc. - ![](./docs/imgs/kyuubi_migrating_yarn_to_k8s.png) - ### The Kyuubi Ecosystem(present and future) - The figure below shows our vision for the Kyuubi Ecosystem. Some of them have been realized, some in development, and others would not be possible without your help. ![](./docs/imgs/kyuubi_ecosystem.drawio.png) - - -## Online Documentation - -Since Kyuubi 1.3.0-incubating, the Kyuubi online documentation is hosted by [https://kyuubi.apache.org/](https://kyuubi.apache.org/). -You can find the latest Kyuubi documentation on [this web page](https://kyuubi.readthedocs.io/en/master/). -For 1.2 and earlier versions, please check the [Readthedocs](https://kyuubi.readthedocs.io/en/v1.2.0/) directly. +## Online Documentation Documentation Status ## Quick Start @@ -111,9 +111,32 @@ Ready? [Getting Started](https://kyuubi.readthedocs.io/en/master/quick_start/) w ## [Contributing](./CONTRIBUTING.md) -## Contributor over time - -[![Contributor over time](https://contributor-graph-api.apiseven.com/contributors-svg?chart=contributorOverTime&repo=apache/kyuubi)](https://api7.ai/contributor-graph?chart=contributorOverTime&repo=apache/kyuubi) +## Project & Community Status + +

+ + + + + + + + + + + + + + + + + + + +

+

+ +

## Aside @@ -121,7 +144,3 @@ The project took its name from a character of a popular Japanese manga - `Naruto The character is named `Kyuubi Kitsune/Kurama`, which is a nine-tailed fox in mythology. `Kyuubi` spread the power and spirit of fire, which is used here to represent the powerful [Apache Spark](http://spark.apache.org). Its nine tails stand for end-to-end multi-tenancy support of this project. - -## License - -This project is licensed under the Apache 2.0 License. See the [LICENSE](./LICENSE) file for details. diff --git a/bin/docker-image-tool.sh b/bin/docker-image-tool.sh index e9e4338b5..14d5fe7b0 100755 --- a/bin/docker-image-tool.sh +++ b/bin/docker-image-tool.sh @@ -27,19 +27,21 @@ function error { if [ -z "${KYUUBI_HOME}" ]; then KYUUBI_HOME="$(cd "`dirname "$0"`"/..; pwd)" fi - -CTX_DIR="$KYUUBI_HOME/target/tmp/docker" +KYUUBI_IMAGE_NAME="kyuubi" function is_dev_build { [ ! -f "$KYUUBI_HOME/RELEASE" ] } -function cleanup_ctx_dir { - if is_dev_build; then - rm -rf "$CTX_DIR" - fi -} -trap cleanup_ctx_dir EXIT +if is_dev_build; then + cat <" exit 1 fi diff --git a/build/Dockerfile b/build/Dockerfile index b53b6716e..8ecc6c8b7 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -29,15 +29,15 @@ # Declare the BASE_IMAGE argument in the first line, for more detail # see: https://github.com/moby/moby/issues/38379 -ARG BASE_IMAGE=openjdk:8-jdk +ARG BASE_IMAGE=eclipse-temurin:8-jdk-focal -FROM maven:3.6-jdk-8 as builder +FROM eclipse-temurin:8-jdk-focal as builder ARG MVN_ARG # Pass the environment variable `CI` into container, for internal use only. # -# Continuous integration(aka. CI) services like GitHub Actions, Travis always provide +# Continuous integration(aka. CI) services like GitHub Actions always provide # an environment variable `CI` in runners, and we detect this variable to run some # specific actions, e.g. run `mvn` in batch mode to suppress noisy logs. ARG CI @@ -48,7 +48,8 @@ WORKDIR /workspace/kyuubi RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive \ - apt-get install -y python3 && \ + apt-get install -y bash python3 && \ + ln -snf /bin/bash /bin/sh && \ ./build/dist ${MVN_ARG} && \ mv /workspace/kyuubi/dist /opt/kyuubi && \ # Removing stuff saves time because docker creates a temporary layer @@ -71,7 +72,8 @@ COPY --from=builder /opt/kyuubi ${KYUUBI_HOME} RUN set -ex && \ apt-get update && \ DEBIAN_FRONTEND=noninteractive \ - apt install -y bash tini libc6 libpam-modules krb5-user libnss3 procps && \ + apt-get install -y bash tini libc6 libpam-modules krb5-user libnss3 procps && \ + ln -snf /bin/bash /bin/sh && \ useradd -u ${kyuubi_uid} -g root kyuubi && \ mkdir -p ${KYUUBI_HOME} ${KYUUBI_LOG_DIR} ${KYUUBI_PID_DIR} ${KYUUBI_WORK_DIR_ROOT} && \ chmod ug+rw -R ${KYUUBI_HOME} && \ diff --git a/build/dist b/build/dist index 7b51886df..df9498008 100755 --- a/build/dist +++ b/build/dist @@ -31,6 +31,7 @@ set -x KYUUBI_HOME="$(cd "`dirname "$0"`/.."; pwd)" DISTDIR="$KYUUBI_HOME/dist" MAKE_TGZ=false +ENABLE_WEBUI=false FLINK_PROVIDED=false SPARK_PROVIDED=false HIVE_PROVIDED=false @@ -42,15 +43,16 @@ function usage { echo "./build/dist - Tool for making binary distributions of Kyuubi" echo "" echo "Usage:" - echo "+------------------------------------------------------------------------------------------------------+" - echo "| ./build/dist [--name ] [--tgz] [--flink-provided] [--spark-provided] [--hive-provided] |" - echo "| [--mvn ] |" - echo "+------------------------------------------------------------------------------------------------------+" + echo "+----------------------------------------------------------------------------------------------+" + echo "| ./build/dist [--name ] [--tgz] [--web-ui] [--flink-provided] [--hive-provided] |" + echo "| [--spark-provided] [--mvn ] |" + echo "+----------------------------------------------------------------------------------------------+" echo "name: - custom binary name, using project version if undefined" echo "tgz: - whether to make a whole bundled package" + echo "web-ui: - whether to include web ui" echo "flink-provided: - whether to make a package without Flink binary" - echo "spark-provided: - whether to make a package without Spark binary" echo "hive-provided: - whether to make a package without Hive binary" + echo "spark-provided: - whether to make a package without Spark binary" echo "mvn: - external maven executable location" echo "" } @@ -67,6 +69,9 @@ while (( "$#" )); do --tgz) MAKE_TGZ=true ;; + --web-ui) + ENABLE_WEBUI=true + ;; --flink-provided) FLINK_PROVIDED=true ;; @@ -210,7 +215,11 @@ else echo "Making distribution for Kyuubi $VERSION in '$DISTDIR'..." fi -MVN_DIST_OPT="-DskipTests" +MVN_DIST_OPT="-DskipTests -Dmaven.javadoc.skip=true -Dmaven.scaladoc.skip=true -Dmaven.source.skip" + +if [[ "$ENABLE_WEBUI" == "true" ]]; then + MVN_DIST_OPT="$MVN_DIST_OPT -Pweb-ui" +fi if [[ "$SPARK_PROVIDED" == "true" ]]; then MVN_DIST_OPT="$MVN_DIST_OPT -Pspark-provided" @@ -238,14 +247,16 @@ echo -e "\$ ${BUILD_COMMAND[@]}\n" rm -rf "$DISTDIR" mkdir -p "$DISTDIR/pid" mkdir -p "$DISTDIR/logs" -mkdir -p "$DISTDIR/jars" mkdir -p "$DISTDIR/work" +mkdir -p "$DISTDIR/jars" +mkdir -p "$DISTDIR/beeline-jars" +mkdir -p "$DISTDIR/web-ui" mkdir -p "$DISTDIR/externals/engines/flink" mkdir -p "$DISTDIR/externals/engines/spark" mkdir -p "$DISTDIR/externals/engines/trino" mkdir -p "$DISTDIR/externals/engines/hive" mkdir -p "$DISTDIR/externals/engines/jdbc" -mkdir -p "$DISTDIR/beeline-jars" +mkdir -p "$DISTDIR/externals/engines/chat" echo "Kyuubi $VERSION $GITREVSTRING built for" > "$DISTDIR/RELEASE" echo "Java $JAVA_VERSION" >> "$DISTDIR/RELEASE" echo "Scala $SCALA_VERSION" >> "$DISTDIR/RELEASE" @@ -303,6 +314,18 @@ for jar in $(ls "$DISTDIR/jars/"); do fi done +# Copy chat engines +cp "$KYUUBI_HOME/externals/kyuubi-chat-engine/target/kyuubi-chat-engine_${SCALA_VERSION}-${VERSION}.jar" "$DISTDIR/externals/engines/chat/" +cp -r "$KYUUBI_HOME"/externals/kyuubi-chat-engine/target/scala-$SCALA_VERSION/jars/*.jar "$DISTDIR/externals/engines/chat/" + +# Share the jars w/ server to reduce binary size +# shellcheck disable=SC2045 +for jar in $(ls "$DISTDIR/jars/"); do + if [[ -f "$DISTDIR/externals/engines/chat/$jar" ]]; then + (cd $DISTDIR/externals/engines/chat; ln -snf "../../../jars/$jar" "$DISTDIR/externals/engines/chat/$jar") + fi +done + # Copy kyuubi tools if [[ -f "$KYUUBI_HOME/tools/spark-block-cleaner/target/spark-block-cleaner_${SCALA_VERSION}-${VERSION}.jar" ]]; then mkdir -p "$DISTDIR/tools/spark-block-cleaner/kubernetes" @@ -312,7 +335,7 @@ if [[ -f "$KYUUBI_HOME/tools/spark-block-cleaner/target/spark-block-cleaner_${SC fi # Copy Kyuubi Spark extension -SPARK_EXTENSION_VERSIONS=('3-1' '3-2' '3-3') +SPARK_EXTENSION_VERSIONS=('3-1' '3-2' '3-3' '3-4' '3-5') # shellcheck disable=SC2068 for SPARK_EXTENSION_VERSION in ${SPARK_EXTENSION_VERSIONS[@]}; do if [[ -f $"$KYUUBI_HOME/extensions/spark/kyuubi-extension-spark-$SPARK_EXTENSION_VERSION/target/kyuubi-extension-spark-${SPARK_EXTENSION_VERSION}_${SCALA_VERSION}-${VERSION}.jar" ]]; then @@ -321,6 +344,11 @@ for SPARK_EXTENSION_VERSION in ${SPARK_EXTENSION_VERSIONS[@]}; do fi done +if [[ "$ENABLE_WEBUI" == "true" ]]; then + # Copy web ui dist + cp -r "$KYUUBI_HOME/kyuubi-server/web-ui/dist" "$DISTDIR/web-ui/" +fi + if [[ "$FLINK_PROVIDED" != "true" ]]; then # Copy flink binary dist FLINK_BUILTIN="$(find "$KYUUBI_HOME/externals/kyuubi-download/target" -name 'flink-*' -type d)" @@ -356,7 +384,11 @@ if [[ "$MAKE_TGZ" == "true" ]]; then TARDIR="$KYUUBI_HOME/$TARDIR_NAME" rm -rf "$TARDIR" cp -R "$DISTDIR" "$TARDIR" - tar czf "$TARDIR_NAME.tgz" -C "$KYUUBI_HOME" "$TARDIR_NAME" + TAR="tar" + if [ "$(uname -s)" = "Darwin" ]; then + TAR="tar --no-mac-metadata --no-xattrs --no-fflags" + fi + $TAR -czf "$TARDIR_NAME.tgz" -C "$KYUUBI_HOME" "$TARDIR_NAME" rm -rf "$TARDIR" echo "The Kyuubi tarball $TARDIR_NAME.tgz is successfully generated in $KYUUBI_HOME." fi diff --git a/build/kyuubi-build-info.cmd b/build/kyuubi-build-info.cmd index 7717b48e4..d9e8e6c6a 100755 --- a/build/kyuubi-build-info.cmd +++ b/build/kyuubi-build-info.cmd @@ -36,6 +36,7 @@ echo kyuubi_trino_version=%~9 echo user=%username% FOR /F %%i IN ('git rev-parse HEAD') DO SET "revision=%%i" +FOR /F "delims=" %%i IN ('git show -s --format^=%%ci HEAD') DO SET "revision_time=%%i" FOR /F %%i IN ('git rev-parse --abbrev-ref HEAD') DO SET "branch=%%i" FOR /F %%i IN ('git config --get remote.origin.url') DO SET "url=%%i" @@ -44,6 +45,7 @@ FOR /f %%i IN ("%TIME%") DO SET current_time=%%i set date=%current_date%_%current_time% echo revision=%revision% +echo revision_time=%revision_time% echo branch=%branch% echo date=%date% echo url=%url% diff --git a/build/mvn b/build/mvn index d67638ba2..cd6c0c796 100755 --- a/build/mvn +++ b/build/mvn @@ -35,7 +35,7 @@ fi ## Arg2 - Tarball Name ## Arg3 - Checkable Binary install_app() { - local remote_tarball="$1/$2" + local remote_tarball="$1/$2$4" local local_tarball="${_DIR}/$2" local binary="${_DIR}/$3" @@ -76,13 +76,26 @@ install_mvn() { fi # See simple version normalization: http://stackoverflow.com/questions/16989598/bash-comparing-version-numbers function version { echo "$@" | awk -F. '{ printf("%03d%03d%03d\n", $1,$2,$3); }'; } - if [ $(version $MVN_DETECTED_VERSION) -lt $(version $MVN_VERSION) ]; then - local APACHE_MIRROR=${APACHE_MIRROR:-'https://archive.apache.org/dist/'} + if [ $(version $MVN_DETECTED_VERSION) -ne $(version $MVN_VERSION) ]; then + local APACHE_MIRROR=${APACHE_MIRROR:-'https://www.apache.org/dyn/closer.lua'} + local MIRROR_URL_QUERY="?action=download" + local MVN_TARBALL="apache-maven-${MVN_VERSION}-bin.tar.gz" + local FILE_PATH="maven/maven-3/${MVN_VERSION}/binaries" + + if [ $(command -v curl) ]; then + if ! curl -L --output /dev/null --silent --head --fail "${APACHE_MIRROR}/${FILE_PATH}/${MVN_TARBALL}${MIRROR_URL_QUERY}" ; then + # Fall back to archive.apache.org for older Maven + echo "Falling back to archive.apache.org to download Maven" + APACHE_MIRROR="https://archive.apache.org/dist" + MIRROR_URL_QUERY="" + fi + fi install_app \ - "${APACHE_MIRROR}/maven/maven-3/${MVN_VERSION}/binaries" \ - "apache-maven-${MVN_VERSION}-bin.tar.gz" \ - "apache-maven-${MVN_VERSION}/bin/mvn" + "${APACHE_MIRROR}/${FILE_PATH}" \ + "${MVN_TARBALL}" \ + "apache-maven-${MVN_VERSION}/bin/mvn" \ + "${MIRROR_URL_QUERY}" MVN_BIN="${_DIR}/apache-maven-${MVN_VERSION}/bin/mvn" fi diff --git a/build/release/create-package.sh b/build/release/create-package.sh index c98e7c0f8..28a89165e 100755 --- a/build/release/create-package.sh +++ b/build/release/create-package.sh @@ -75,7 +75,7 @@ package_binary() { echo "Creating binary release tarball ${BIN_TGZ_FILE}" - ${KYUUBI_DIR}/build/dist --tgz --spark-provided --flink-provided --hive-provided + ${KYUUBI_DIR}/build/dist --tgz --web-ui --spark-provided --flink-provided --hive-provided cp "${BIN_TGZ_FILE}" "${RELEASE_DIR}" diff --git a/build/release/release.sh b/build/release/release.sh index 4afac3865..49fef9f8b 100755 --- a/build/release/release.sh +++ b/build/release/release.sh @@ -52,6 +52,21 @@ if [[ ${RELEASE_VERSION} =~ .*-SNAPSHOT ]]; then exit 1 fi +if [ -n "${JAVA_HOME}" ]; then + JAVA="${JAVA_HOME}/bin/java" +elif [ "$(command -v java)" ]; then + JAVA="java" +else + echo "JAVA_HOME is not set" >&2 + exit 1 +fi + +JAVA_VERSION=$($JAVA -version 2>&1 | awk -F '"' '/version/ {print $2}') +if [[ $JAVA_VERSION != 1.8.* ]]; then + echo "Unexpected Java version: $JAVA_VERSION. Java 8 is required for release." + exit 1 +fi + RELEASE_TAG="v${RELEASE_VERSION}-rc${RELEASE_RC_NO}" SVN_STAGING_REPO="https://dist.apache.org/repos/dist/dev/kyuubi" @@ -85,7 +100,7 @@ upload_svn_staging() { svn add "${SVN_STAGING_DIR}/${RELEASE_TAG}" - echo "Uploading release tarballs to ${SVN_STAGING_DIR}/${RELEASE_TAG}" + echo "Uploading release tarballs to ${SVN_STAGING_REPO}/${RELEASE_TAG}" ( cd "${SVN_STAGING_DIR}" && \ svn commit --username "${ASF_USERNAME}" --password "${ASF_PASSWORD}" --message "Apache Kyuubi ${RELEASE_TAG}" @@ -94,17 +109,34 @@ upload_svn_staging() { } upload_nexus_staging() { - ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,flink-provided,spark-provided,hive-provided \ - -s "${KYUUBI_DIR}/build/release/asf-settings.xml" + # Spark Extension Plugin for Spark 3.1 ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,flink-provided,spark-provided,hive-provided,spark-3.1 \ -s "${KYUUBI_DIR}/build/release/asf-settings.xml" \ -pl extensions/spark/kyuubi-extension-spark-3-1 -am + + # Spark Extension Plugin for Spark 3.2 ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,flink-provided,spark-provided,hive-provided,spark-3.2 \ -s "${KYUUBI_DIR}/build/release/asf-settings.xml" \ -pl extensions/spark/kyuubi-extension-spark-3-2 -am + + # Spark Extension Plugin for Spark 3.3 ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,flink-provided,spark-provided,hive-provided,spark-3.3 \ -s "${KYUUBI_DIR}/build/release/asf-settings.xml" \ -pl extensions/spark/kyuubi-extension-spark-3-3 -am + + # Spark Extension Plugin for Spark 3.5 + ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,flink-provided,spark-provided,hive-provided,spark-3.5 \ + -s "${KYUUBI_DIR}/build/release/asf-settings.xml" \ + -pl extensions/spark/kyuubi-extension-spark-3-5 -am + + # Spark TPC-DS/TPC-H Connector built with default Spark version (3.4) and Scala 2.13 + ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,flink-provided,spark-provided,hive-provided,spark-3.4,scala-2.13 \ + -s "${KYUUBI_DIR}/build/release/asf-settings.xml" \ + -pl extensions/spark/kyuubi-spark-connector-tpcds,extensions/spark/kyuubi-spark-connector-tpch -am + + # All modules including Spark Extension Plugin and Connectors built with default Spark version (3.4) and default Scala version (2.12) + ${KYUUBI_DIR}/build/mvn clean deploy -DskipTests -Papache-release,flink-provided,spark-provided,hive-provided,spark-3.4 \ + -s "${KYUUBI_DIR}/build/release/asf-settings.xml" } finalize_svn() { diff --git a/build/release/script/announce.sh b/build/release/script/announce.sh old mode 100644 new mode 100755 diff --git a/build/release/script/dev_kyuubi_vote.sh b/build/release/script/dev_kyuubi_vote.sh old mode 100644 new mode 100755 diff --git a/charts/kyuubi/Chart.yaml b/charts/kyuubi/Chart.yaml index 6b377ecc5..56abc9edc 100644 --- a/charts/kyuubi/Chart.yaml +++ b/charts/kyuubi/Chart.yaml @@ -20,7 +20,7 @@ name: kyuubi description: A Helm chart for Kyuubi server type: application version: 0.1.0 -appVersion: "master-snapshot" +appVersion: 1.7.3 home: https://kyuubi.apache.org icon: https://raw.githubusercontent.com/apache/kyuubi/master/docs/imgs/logo.png sources: diff --git a/charts/kyuubi/README.md b/charts/kyuubi/README.md new file mode 100644 index 000000000..dfec578dd --- /dev/null +++ b/charts/kyuubi/README.md @@ -0,0 +1,57 @@ + + +# Helm Chart for Apache Kyuubi + +[Apache Kyuubi](https://kyuubi.apache.org) is a distributed and multi-tenant gateway to provide serverless SQL on Data Warehouses and Lakehouses. + + +## Introduction + +This chart will bootstrap an [Kyuubi](https://kyuubi.apache.org) deployment on a [Kubernetes](http://kubernetes.io) +cluster using the [Helm](https://helm.sh) package manager. + +## Requirements + +- Kubernetes cluster +- Helm 3.0+ + +## Template rendering + +When you want to test the template rendering, but not actually install anything. [Debugging templates](https://helm.sh/docs/chart_template_guide/debugging/) provide a quick way of viewing the generated content without YAML parse errors blocking. + +There are two ways to render templates. It will return the rendered template to you so you can see the output. + +- Local rendering chart templates +```shell +helm template --debug ../kyuubi +``` +- Server side rendering chart templates +```shell +helm install --dry-run --debug --generate-name ../kyuubi +``` + + +## Documentation + +Configuration guide documentation for Kyuubi lives [on the website](https://kyuubi.readthedocs.io/en/master/configuration/settings.html#kyuubi-configurations). (Not just for Helm Chart) + +## Contributing + +Want to help build Apache Kyuubi? Check out our [contributing documentation](https://kyuubi.readthedocs.io/en/master/community/CONTRIBUTING.html). \ No newline at end of file diff --git a/charts/kyuubi/templates/NOTES.txt b/charts/kyuubi/templates/NOTES.txt index 44a35b6b7..2693f5ef6 100644 --- a/charts/kyuubi/templates/NOTES.txt +++ b/charts/kyuubi/templates/NOTES.txt @@ -1,21 +1,47 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# +{{/* + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at -Get kyuubi expose URL by running these commands: - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "kyuubi.fullname" . }}-nodeport) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo $NODE_IP:$NODE_PORT \ No newline at end of file + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/}} + +The chart has been installed! + +In order to check the release status, use: + helm status {{ .Release.Name }} -n {{ .Release.Namespace }} + or for more detailed info + helm get all {{ .Release.Name }} -n {{ .Release.Namespace }} + +************************ +******* Services ******* +************************ +{{- range $name, $frontend := .Values.server }} +{{- if $frontend.enabled }} +{{ $name | snakecase | upper }}: +- To access {{ $.Release.Name }}-{{ $name | kebabcase }} service within the cluster, use the following URL: + {{ $.Release.Name }}-{{ $name | kebabcase }}.{{ $.Release.Namespace }}.svc.cluster.local +{{- if $.Values.kyuubiConf.kyuubiDefaults }} +{{- if regexMatch "(^|\\s)kyuubi.frontend.bind.host\\s*=?\\s*(localhost|127\\.0\\.0\\.1)($|\\s)" $.Values.kyuubiConf.kyuubiDefaults }} +- To access {{ $.Release.Name }}-{{ $name | kebabcase }} service from outside the cluster for debugging, run the following command: + kubectl port-forward svc/{{ $.Release.Name }}-{{ $name | kebabcase }} {{ tpl $frontend.service.port $ }}:{{ tpl $frontend.service.port $ }} -n {{ $.Release.Namespace }} + and use 127.0.0.1:{{ tpl $frontend.service.port $ }} +{{- end }} +{{- end }} +{{- if eq $frontend.service.type "NodePort" }} +- To access {{ $.Release.Name }}-{{ $name | kebabcase }} service from outside the cluster through configured NodePort, run the following commands: + export NODE_PORT=$(kubectl get service {{ $.Release.Name }}-{{ $name | kebabcase }} -n {{ $.Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}") + export NODE_IP=$(kubectl get nodes -n {{ $.Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/kyuubi/templates/_helpers.tpl b/charts/kyuubi/templates/_helpers.tpl index 684c1f354..502bf4646 100644 --- a/charts/kyuubi/templates/_helpers.tpl +++ b/charts/kyuubi/templates/_helpers.tpl @@ -16,33 +16,36 @@ */}} {{/* -Expand the name of the chart. +A comma separated string of enabled frontend protocols, e.g. "REST,THRIFT_BINARY". +For details, see 'kyuubi.frontend.protocols': https://kyuubi.readthedocs.io/en/master/configuration/settings.html#frontend */}} -{{- define "kyuubi.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- define "kyuubi.frontend.protocols" -}} + {{- $protocols := list }} + {{- range $name, $frontend := .Values.server }} + {{- if $frontend.enabled }} + {{- $protocols = $name | snakecase | upper | append $protocols }} + {{- end }} + {{- end }} + {{- if not $protocols }} + {{ fail "At least one frontend protocol must be enabled!" }} + {{- end }} + {{- $protocols | join "," }} {{- end }} {{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. +Selector labels */}} -{{- define "kyuubi.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} +{{- define "kyuubi.selectorLabels" -}} +app.kubernetes.io/name: {{ .Chart.Name }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} {{/* -Create chart name and version as used by the chart label. +Common labels */}} -{{- define "kyuubi.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} \ No newline at end of file +{{- define "kyuubi.labels" -}} +helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} +{{ include "kyuubi.selectorLabels" . }} +app.kubernetes.io/version: {{ .Values.image.tag | default .Chart.AppVersion | quote }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} diff --git a/charts/kyuubi/templates/kyuubi-alert.yaml b/charts/kyuubi/templates/kyuubi-alert.yaml new file mode 100644 index 000000000..89fd11dc7 --- /dev/null +++ b/charts/kyuubi/templates/kyuubi-alert.yaml @@ -0,0 +1,28 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/}} + +{{- if and .Values.monitoring.prometheus.enabled (eq .Values.metricsReporters "PROMETHEUS") .Values.prometheusRule.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ .Release.Name }} + labels: + {{- include "kyuubi.labels" . | nindent 4 }} +spec: + groups: + {{- toYaml .Values.prometheusRule.groups | nindent 4 }} +{{- end }} diff --git a/charts/kyuubi/templates/kyuubi-configmap.yaml b/charts/kyuubi/templates/kyuubi-configmap.yaml index ada9e3dc8..62413567d 100644 --- a/charts/kyuubi/templates/kyuubi-configmap.yaml +++ b/charts/kyuubi/templates/kyuubi-configmap.yaml @@ -1,47 +1,51 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# +{{/* + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/}} apiVersion: v1 kind: ConfigMap metadata: name: {{ .Release.Name }} labels: - helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} - app.kubernetes.io/name: {{ .Chart.Name }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/version: {{ .Values.image.tag | default .Chart.AppVersion | quote }} - app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- include "kyuubi.labels" . | nindent 4 }} data: - {{- with .Values.server.conf.kyuubiEnv }} + {{- with .Values.kyuubiConf.kyuubiEnv }} kyuubi-env.sh: | #!/usr/bin/env bash {{- tpl . $ | nindent 4 }} {{- end }} kyuubi-defaults.conf: | ## Helm chart provided Kyuubi configurations - kyuubi.frontend.bind.host={{ .Values.server.bind.host }} - kyuubi.frontend.bind.port={{ .Values.server.bind.port }} kyuubi.kubernetes.namespace={{ .Release.Namespace }} + kyuubi.frontend.connection.url.use.hostname=false + kyuubi.frontend.thrift.binary.bind.port={{ .Values.server.thriftBinary.port }} + kyuubi.frontend.thrift.http.bind.port={{ .Values.server.thriftHttp.port }} + kyuubi.frontend.rest.bind.port={{ .Values.server.rest.port }} + kyuubi.frontend.mysql.bind.port={{ .Values.server.mysql.port }} + kyuubi.frontend.protocols={{ include "kyuubi.frontend.protocols" . }} + + # Kyuubi Metrics + kyuubi.metrics.enabled={{ .Values.monitoring.prometheus.enabled }} + kyuubi.metrics.reporters={{ .Values.metricsReporters }} ## User provided Kyuubi configurations - {{- with .Values.server.conf.kyuubiDefaults }} - {{- tpl . $ | nindent 4 }} + {{- with .Values.kyuubiConf.kyuubiDefaults }} + {{- tpl . $ | nindent 4 }} {{- end }} - {{- with .Values.server.conf.log4j2 }} + {{- with .Values.kyuubiConf.log4j2 }} log4j2.xml: | {{- tpl . $ | nindent 4 }} {{- end }} diff --git a/charts/kyuubi/templates/kyuubi-deployment.yaml b/charts/kyuubi/templates/kyuubi-deployment.yaml deleted file mode 100644 index 941fdf164..000000000 --- a/charts/kyuubi/templates/kyuubi-deployment.yaml +++ /dev/null @@ -1,113 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ .Release.Name }} - labels: - helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} - app.kubernetes.io/name: {{ .Chart.Name }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/version: {{ .Values.image.tag | default .Chart.AppVersion | quote }} - app.kubernetes.io/managed-by: {{ .Release.Service }} -spec: - replicas: {{ .Values.replicaCount }} - selector: - matchLabels: - app.kubernetes.io/name: {{ .Chart.Name }} - app.kubernetes.io/instance: {{ .Release.Name }} - template: - metadata: - labels: - app.kubernetes.io/name: {{ .Chart.Name }} - app.kubernetes.io/instance: {{ .Release.Name }} - annotations: - checksum/conf: {{ include (print $.Template.BasePath "/kyuubi-configmap.yaml") . | sha256sum }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ .Values.serviceAccount.name | default .Release.Name }} - {{- with .Values.initContainers }} - initContainers: {{- tpl (toYaml .) $ | nindent 8 }} - {{- end }} - containers: - - name: kyuubi-server - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - {{- with .Values.env }} - env: {{- tpl (toYaml .) $ | nindent 12 }} - {{- end }} - {{- with .Values.envFrom }} - envFrom: {{- tpl (toYaml .) $ | nindent 12 }} - {{- end }} - ports: - - name: frontend-port - containerPort: {{ .Values.server.bind.port }} - protocol: TCP - {{- if .Values.probe.liveness.enabled }} - livenessProbe: - tcpSocket: - port: {{ .Values.server.bind.port }} - initialDelaySeconds: {{ .Values.probe.liveness.initialDelaySeconds }} - periodSeconds: {{ .Values.probe.liveness.periodSeconds }} - timeoutSeconds: {{ .Values.probe.liveness.timeoutSeconds }} - failureThreshold: {{ .Values.probe.liveness.failureThreshold }} - successThreshold: {{ .Values.probe.liveness.successThreshold }} - {{- end }} - {{- if .Values.probe.readiness.enabled }} - readinessProbe: - tcpSocket: - port: {{ .Values.server.bind.port }} - initialDelaySeconds: {{ .Values.probe.readiness.initialDelaySeconds }} - periodSeconds: {{ .Values.probe.readiness.periodSeconds }} - timeoutSeconds: {{ .Values.probe.readiness.timeoutSeconds }} - failureThreshold: {{ .Values.probe.readiness.failureThreshold }} - successThreshold: {{ .Values.probe.readiness.successThreshold }} - {{- end }} - {{- with .Values.resources }} - resources: {{- toYaml . | nindent 12 }} - {{- end }} - volumeMounts: - - name: conf - mountPath: {{ .Values.server.confDir }} - {{- with .Values.volumeMounts }} - {{- tpl (toYaml .) $ | nindent 12 }} - {{- end }} - {{- with .Values.containers }} - {{- tpl (toYaml .) $ | nindent 8 }} - {{- end }} - volumes: - - name: conf - configMap: - name: {{ .Release.Name }} - {{- with .Values.volumes }} - {{- tpl (toYaml .) $ | nindent 8 }} - {{- end }} - {{- with .Values.nodeSelector }} - nodeSelector: {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.securityContext }} - securityContext: {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/charts/kyuubi/templates/kyuubi-headless-service.yaml b/charts/kyuubi/templates/kyuubi-headless-service.yaml new file mode 100644 index 000000000..fa04ffeef --- /dev/null +++ b/charts/kyuubi/templates/kyuubi-headless-service.yaml @@ -0,0 +1,40 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/}} + +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-headless + labels: + {{- include "kyuubi.labels" $ | nindent 4 }} +spec: + type: ClusterIP + clusterIP: None + ports: + {{- range $name, $frontend := .Values.server }} + - name: {{ $name | kebabcase }} + port: {{ tpl $frontend.service.port $ }} + targetPort: {{ $frontend.port }} + {{- end }} + {{- if .Values.monitoring.prometheus.enabled }} + - name: prometheus + port: {{ .Values.monitoring.prometheus.port }} + targetPort: {{ .Values.monitoring.prometheus.port }} + {{- end }} + selector: + {{- include "kyuubi.selectorLabels" $ | nindent 4 }} + diff --git a/charts/kyuubi/templates/kyuubi-podmonitor.yaml b/charts/kyuubi/templates/kyuubi-podmonitor.yaml new file mode 100644 index 000000000..458ff66ed --- /dev/null +++ b/charts/kyuubi/templates/kyuubi-podmonitor.yaml @@ -0,0 +1,31 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/}} + +{{- if and .Values.monitoring.prometheus.enabled (eq .Values.metricsReporters "PROMETHEUS") .Values.podMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: {{ .Release.Name }} + labels: + {{- include "kyuubi.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + app: {{ .Release.Name }} + podMetricsEndpoints: + {{- toYaml .Values.podMonitor.podMetricsEndpoint | nindent 4 }} +{{- end }} diff --git a/charts/kyuubi/templates/kyuubi-priorityclass.yaml b/charts/kyuubi/templates/kyuubi-priorityclass.yaml new file mode 100644 index 000000000..c756108ae --- /dev/null +++ b/charts/kyuubi/templates/kyuubi-priorityclass.yaml @@ -0,0 +1,26 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/}} + +{{- if .Values.priorityClass.create }} +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: {{ .Values.priorityClass.name | default .Release.Name }} + labels: + {{- include "kyuubi.labels" . | nindent 4 }} +value: {{ .Values.priorityClass.value }} +{{- end }} diff --git a/charts/kyuubi/templates/kyuubi-role.yaml b/charts/kyuubi/templates/kyuubi-role.yaml index fcb5a9f6e..5ee8c1dff 100644 --- a/charts/kyuubi/templates/kyuubi-role.yaml +++ b/charts/kyuubi/templates/kyuubi-role.yaml @@ -1,19 +1,19 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# +{{/* + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/}} {{- if .Values.rbac.create }} apiVersion: rbac.authorization.k8s.io/v1 @@ -21,10 +21,6 @@ kind: Role metadata: name: {{ .Release.Name }} labels: - helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} - app.kubernetes.io/name: {{ .Chart.Name }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/version: {{ .Values.image.tag | default .Chart.AppVersion | quote }} - app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- include "kyuubi.labels" . | nindent 4 }} rules: {{- toYaml .Values.rbac.rules | nindent 2 }} {{- end }} diff --git a/charts/kyuubi/templates/kyuubi-rolebinding.yaml b/charts/kyuubi/templates/kyuubi-rolebinding.yaml index 8f74efc2d..0f9dbd049 100644 --- a/charts/kyuubi/templates/kyuubi-rolebinding.yaml +++ b/charts/kyuubi/templates/kyuubi-rolebinding.yaml @@ -1,19 +1,19 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# +{{/* + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/}} {{- if .Values.rbac.create }} apiVersion: rbac.authorization.k8s.io/v1 @@ -21,11 +21,7 @@ kind: RoleBinding metadata: name: {{ .Release.Name }} labels: - helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} - app.kubernetes.io/name: {{ .Chart.Name }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/version: {{ .Values.image.tag | default .Chart.AppVersion | quote }} - app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- include "kyuubi.labels" . | nindent 4 }} subjects: - kind: ServiceAccount name: {{ .Values.serviceAccount.name | default .Release.Name }} diff --git a/charts/kyuubi/templates/kyuubi-service.yaml b/charts/kyuubi/templates/kyuubi-service.yaml index 0152bd23d..64c8b06ac 100644 --- a/charts/kyuubi/templates/kyuubi-service.yaml +++ b/charts/kyuubi/templates/kyuubi-service.yaml @@ -1,41 +1,42 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# +{{/* + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/}} + +{{- range $name, $frontend := .Values.server }} +{{- if $frontend.enabled }} apiVersion: v1 kind: Service metadata: - name: {{ .Release.Name }} + name: {{ $.Release.Name }}-{{ $name | kebabcase }} labels: - helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} - app.kubernetes.io/name: {{ .Chart.Name }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/version: {{ .Values.image.tag | default .Chart.AppVersion | quote }} - app.kubernetes.io/managed-by: {{ .Release.Service }} - {{- with .Values.service.annotations }} - annotations: - {{- toYaml . | nindent 4 }} + {{- include "kyuubi.labels" $ | nindent 4 }} + {{- with $frontend.service.annotations }} + annotations: {{- toYaml . | nindent 4 }} {{- end }} spec: + type: {{ $frontend.service.type }} ports: - - name: http - nodePort: {{ .Values.service.port }} - port: {{ .Values.server.bind.port }} - protocol: TCP - type: {{ .Values.service.type }} + - name: {{ $name | kebabcase }} + port: {{ tpl $frontend.service.port $ }} + targetPort: {{ $frontend.port }} + {{- if and (eq $frontend.service.type "NodePort") ($frontend.service.nodePort) }} + nodePort: {{ $frontend.service.nodePort }} + {{- end }} selector: - app.kubernetes.io/name: {{ .Chart.Name }} - app.kubernetes.io/instance: {{ .Release.Name }} + {{- include "kyuubi.selectorLabels" $ | nindent 4 }} +--- +{{- end }} +{{- end }} diff --git a/charts/kyuubi/templates/kyuubi-serviceaccount.yaml b/charts/kyuubi/templates/kyuubi-serviceaccount.yaml index 770d50136..a8e282a1f 100644 --- a/charts/kyuubi/templates/kyuubi-serviceaccount.yaml +++ b/charts/kyuubi/templates/kyuubi-serviceaccount.yaml @@ -1,19 +1,19 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# +{{/* + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/}} {{- if .Values.serviceAccount.create }} apiVersion: v1 @@ -21,9 +21,5 @@ kind: ServiceAccount metadata: name: {{ .Values.serviceAccount.name | default .Release.Name }} labels: - helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} - app.kubernetes.io/name: {{ .Chart.Name }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/version: {{ .Values.image.tag | default .Chart.AppVersion | quote }} - app.kubernetes.io/managed-by: {{ .Release.Service }} + {{- include "kyuubi.labels" . | nindent 4 }} {{- end }} diff --git a/charts/kyuubi/templates/kyuubi-servicemonitor.yaml b/charts/kyuubi/templates/kyuubi-servicemonitor.yaml new file mode 100644 index 000000000..11098a0ea --- /dev/null +++ b/charts/kyuubi/templates/kyuubi-servicemonitor.yaml @@ -0,0 +1,31 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/}} + +{{- if and .Values.monitoring.prometheus.enabled (eq .Values.metricsReporters "PROMETHEUS") .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ .Release.Name }} + labels: + {{- include "kyuubi.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + app: {{ .Release.Name }} + endpoints: + {{- toYaml .Values.serviceMonitor.endpoints | nindent 4 }} +{{- end }} diff --git a/charts/kyuubi/templates/kyuubi-statefulset.yaml b/charts/kyuubi/templates/kyuubi-statefulset.yaml new file mode 100644 index 000000000..309ef8ec9 --- /dev/null +++ b/charts/kyuubi/templates/kyuubi-statefulset.yaml @@ -0,0 +1,132 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/}} + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ .Release.Name }} + labels: + {{- include "kyuubi.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "kyuubi.selectorLabels" . | nindent 6 }} + serviceName: {{ .Release.Name }}-headless + minReadySeconds: {{ .Values.minReadySeconds }} + replicas: {{ .Values.replicaCount }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + podManagementPolicy: {{ .Values.podManagementPolicy }} + {{- with .Values.updateStrategy }} + updateStrategy: {{- toYaml . | nindent 4 }} + {{- end }} + template: + metadata: + labels: + {{- include "kyuubi.selectorLabels" . | nindent 8 }} + annotations: + checksum/conf: {{ include (print $.Template.BasePath "/kyuubi-configmap.yaml") . | sha256sum }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: {{- toYaml . | nindent 8 }} + {{- end }} + {{- if or .Values.serviceAccount.name .Values.serviceAccount.create }} + serviceAccountName: {{ .Values.serviceAccount.name | default .Release.Name }} + {{- end }} + {{- if or .Values.priorityClass.name .Values.priorityClass.create }} + priorityClassName: {{ .Values.priorityClass.name | default .Release.Name }} + {{- end }} + {{- with .Values.initContainers }} + initContainers: {{- tpl (toYaml .) $ | nindent 8 }} + {{- end }} + containers: + - name: kyuubi-server + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + {{- with .Values.command }} + command: {{- tpl (toYaml .) $ | nindent 12 }} + {{- end }} + {{- with .Values.args }} + args: {{- tpl (toYaml .) $ | nindent 12 }} + {{- end }} + {{- with .Values.env }} + env: {{- tpl (toYaml .) $ | nindent 12 }} + {{- end }} + {{- with .Values.envFrom }} + envFrom: {{- tpl (toYaml .) $ | nindent 12 }} + {{- end }} + ports: + {{- range $name, $frontend := .Values.server }} + {{- if $frontend.enabled }} + - name: {{ $name | kebabcase }} + containerPort: {{ $frontend.port }} + {{- end }} + {{- end }} + {{- if .Values.monitoring.prometheus.enabled }} + - name: prometheus + containerPort: {{ .Values.monitoring.prometheus.port }} + {{- end }} + {{- if .Values.livenessProbe.enabled }} + livenessProbe: + exec: + command: ["/bin/bash", "-c", "bin/kyuubi status"] + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + {{- end }} + {{- if .Values.readinessProbe.enabled }} + readinessProbe: + exec: + command: ["/bin/bash", "-c", "$KYUUBI_HOME/bin/kyuubi status"] + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + {{- end }} + {{- with .Values.resources }} + resources: {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: conf + mountPath: {{ .Values.kyuubiConfDir }} + {{- with .Values.volumeMounts }} + {{- tpl (toYaml .) $ | nindent 12 }} + {{- end }} + {{- with .Values.containers }} + {{- tpl (toYaml .) $ | nindent 8 }} + {{- end }} + volumes: + - name: conf + configMap: + name: {{ .Release.Name }} + {{- with .Values.volumes }} + {{- tpl (toYaml .) $ | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.securityContext }} + securityContext: {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/charts/kyuubi/values.yaml b/charts/kyuubi/values.yaml index 22ae9d5a9..faa854b10 100644 --- a/charts/kyuubi/values.yaml +++ b/charts/kyuubi/values.yaml @@ -22,61 +22,143 @@ # Kyuubi server numbers replicaCount: 2 +# controls how Kyuubi server pods are created during initial scale up, +# when replacing pods on nodes, or when scaling down. +# The default policy is `OrderedReady`, alternative policy is `Parallel`. +podManagementPolicy: OrderedReady + +# Minimum number of seconds for which a newly created kyuubi server +# should be ready without any of its container crashing for it to be considered available. +minReadySeconds: 30 + +# maximum number of revisions that will be maintained in the StatefulSet's revision history. +revisionHistoryLimit: 10 + +# indicates the StatefulSetUpdateStrategy that will be employed to update Kyuubi server Pods in the StatefulSet +# when a revision is made to Template. +updateStrategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + partition: 0 + image: repository: apache/kyuubi - pullPolicy: Always + pullPolicy: IfNotPresent tag: ~ imagePullSecrets: [] -# ServiceAccount used for Kyuubi create/list/delete pod in kubernetes +# ServiceAccount used for Kyuubi create/list/delete pod in Kubernetes serviceAccount: + # Specifies whether a ServiceAccount should be created create: true + # Specifies ServiceAccount name to be used (created if `create: true`) + name: ~ + +# priorityClass used for Kyuubi server pod +priorityClass: + # Specifies whether a priorityClass should be created + create: false + # Specifies priorityClass name to be used (created if `create: true`) name: ~ + # half of system-cluster-critical by default + value: 1000000000 +# Role-based access control rbac: + # Specifies whether RBAC resources should be created create: true + # RBAC rules rules: - apiGroups: [""] resources: ["pods"] verbs: ["create", "list", "delete"] -probe: - liveness: - enabled: true - initialDelaySeconds: 30 - periodSeconds: 10 - timeoutSeconds: 2 - failureThreshold: 10 - successThreshold: 1 - readiness: - enabled: true - initialDelaySeconds: 30 - periodSeconds: 10 - timeoutSeconds: 2 - failureThreshold: 10 - successThreshold: 1 - server: - bind: - host: 0.0.0.0 + # Thrift Binary protocol (HiveServer2 compatible) + thriftBinary: + enabled: true port: 10009 - confDir: /opt/kyuubi/conf - conf: - # The value (templated string) is used for kyuubi-env.sh file - # See https://kyuubi.apache.org/docs/latest/deployment/settings.html#environments for more details - kyuubiEnv: ~ - - # The value (templated string) is used for kyuubi-defaults.conf file - # See https://kyuubi.apache.org/docs/latest/deployment/settings.html#kyuubi-configurations for more details - kyuubiDefaults: ~ - - # The value (templated string) is used for log4j2.xml file - # See https://kyuubi.apache.org/docs/latest/deployment/settings.html#logging for more details - log4j2: ~ + service: + type: ClusterIP + port: "{{ .Values.server.thriftBinary.port }}" + nodePort: ~ + annotations: {} + + # Thrift HTTP protocol (HiveServer2 compatible) + thriftHttp: + enabled: false + port: 10010 + service: + type: ClusterIP + port: "{{ .Values.server.thriftHttp.port }}" + nodePort: ~ + annotations: {} + + # REST API protocol (experimental) + rest: + enabled: true + port: 10099 + service: + type: ClusterIP + port: "{{ .Values.server.rest.port }}" + nodePort: ~ + annotations: {} + + # MySQL compatible text protocol (experimental) + mysql: + enabled: false + port: 3309 + service: + type: ClusterIP + port: "{{ .Values.server.mysql.port }}" + nodePort: ~ + annotations: {} + +monitoring: + # Exposes metrics in Prometheus format + prometheus: + enabled: true + port: 10019 + +# $KYUUBI_CONF_DIR directory +kyuubiConfDir: /opt/kyuubi/conf +# Kyuubi configurations files +kyuubiConf: + # The value (templated string) is used for kyuubi-env.sh file + # See example at conf/kyuubi-env.sh.template and https://kyuubi.readthedocs.io/en/master/configuration/settings.html#environments for more details + kyuubiEnv: ~ + # kyuubiEnv: | + # export JAVA_HOME=/usr/jdk64/jdk1.8.0_152 + # export SPARK_HOME=/opt/spark + # export FLINK_HOME=/opt/flink + # export HIVE_HOME=/opt/hive + + # The value (templated string) is used for kyuubi-defaults.conf file + # See https://kyuubi.readthedocs.io/en/master/configuration/settings.html#kyuubi-configurations for more details + kyuubiDefaults: ~ + # kyuubiDefaults: | + # kyuubi.authentication=NONE + # kyuubi.frontend.bind.host=10.0.0.1 + # kyuubi.engine.type=SPARK_SQL + # kyuubi.engine.share.level=USER + # kyuubi.session.engine.initialize.timeout=PT3M + # kyuubi.ha.addresses=zk1:2181,zk2:2181,zk3:2181 + # kyuubi.ha.namespace=kyuubi + + # The value (templated string) is used for log4j2.xml file + # See example at conf/log4j2.xml.template https://kyuubi.readthedocs.io/en/master/configuration/settings.html#logging for more details + log4j2: ~ + +# Command to launch Kyuubi server (templated) +command: ~ +# Arguments to launch Kyuubi server (templated) +args: ~ # Environment variables (templated) env: [] +# Environment variables from ConfigMaps and Secrets (templated) envFrom: [] # Additional volumes for Kyuubi pod (templated) @@ -89,30 +171,67 @@ initContainers: [] # Additional containers for Kyuubi pod (templated) containers: [] -service: - type: NodePort - # The default port limit of kubernetes is 30000-32767 - # to change: - # vim kube-apiserver.yaml (usually under path: /etc/kubernetes/manifests/) - # add or change line 'service-node-port-range=1-32767' under kube-apiserver - port: 30009 - annotations: {} - +# Resource requests and limits for Kyuubi pods resources: {} - # Used to specify resource, default unlimited. - # If you do want to specify resources: - # 1. remove the curly braces after 'resources:' - # 2. uncomment the following lines - # limits: - # cpu: 4 - # memory: 10Gi - # requests: - # cpu: 2 - # memory: 4Gi - -# Constrain Kyuubi server pods to specific nodes +# resources: +# requests: +# cpu: 2 +# memory: 4Gi +# limits: +# cpu: 4 +# memory: 10Gi + +# Liveness probe +livenessProbe: + enabled: true + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 2 + failureThreshold: 10 + successThreshold: 1 + +# Readiness probe +readinessProbe: + enabled: true + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 2 + failureThreshold: 10 + successThreshold: 1 + +# Constrain Kyuubi pods to nodes with specific node labels nodeSelector: {} +# Allow to schedule Kyuubi pods on nodes with matching taints tolerations: [] +# Constrain Kyuubi pods to nodes by complex affinity/anti-affinity rules affinity: {} +# Kyuubi pods security context securityContext: {} + +# Monitoring Kyuubi - Server Metrics +# PROMETHEUS - PrometheusReporter which exposes metrics in Prometheus format +metricsReporters: ~ + +# Prometheus pod monitor +podMonitor: + # If enabled, podMonitor for operator's pod will be created + enabled: false + # The podMetricsEndpoint contains metrics information such as port, interval, scheme, and possibly other relevant details. + # This information is used to configure the endpoint from which Prometheus can scrape and collect metrics for a specific Pod in Kubernetes. + podMetricsEndpoint: [] + +# Prometheus service monitor +serviceMonitor: + # If enabled, ServiceMonitor resources for Prometheus Operator are created + enabled: false + # The endpoints section in a ServiceMonitor specifies the metrics information for each target endpoint. + # This allows you to collect metrics from multiple Services across your Kubernetes cluster in a standardized and automated way. + endpoints: [] + +# Rules for the Prometheus Operator +prometheusRule: + # If enabled, a PrometheusRule resource for Prometheus Operator is created + enabled: false + # Contents of Prometheus rules file + groups: [] diff --git a/codecov.yml b/codecov.yml index 6267ea380..1be776f58 100644 --- a/codecov.yml +++ b/codecov.yml @@ -16,4 +16,11 @@ # codecov: - token: b624e642-b0c8-4d45-94a1-a370888435bb + token: 5115fd3e-2ef2-40ed-b012-376a2afdc382 + +coverage: + status: + project: + default: + target: auto # auto compares coverage to the previous base commit + threshold: 2% #this allows a 2% drop from the previous base commit coverage diff --git a/conf/kyuubi-defaults.conf.template b/conf/kyuubi-defaults.conf.template index d3e6026d9..eef36ad10 100644 --- a/conf/kyuubi-defaults.conf.template +++ b/conf/kyuubi-defaults.conf.template @@ -18,9 +18,19 @@ ## Kyuubi Configurations # -# kyuubi.authentication NONE -# kyuubi.frontend.bind.host localhost -# kyuubi.frontend.bind.port 10009 +# kyuubi.authentication NONE +# +# kyuubi.frontend.bind.host 10.0.0.1 +# kyuubi.frontend.protocols THRIFT_BINARY,REST +# kyuubi.frontend.thrift.binary.bind.port 10009 +# kyuubi.frontend.rest.bind.port 10099 +# +# kyuubi.engine.type SPARK_SQL +# kyuubi.engine.share.level USER +# kyuubi.session.engine.initialize.timeout PT3M +# +# kyuubi.ha.addresses zk1:2181,zk2:2181,zk3:2181 +# kyuubi.ha.namespace kyuubi # -# Details in https://kyuubi.readthedocs.io/en/master/deployment/settings.html +# Details in https://kyuubi.readthedocs.io/en/master/configuration/settings.html diff --git a/conf/log4j2.xml.template b/conf/log4j2.xml.template index 37fc8acf0..215fddf47 100644 --- a/conf/log4j2.xml.template +++ b/conf/log4j2.xml.template @@ -21,19 +21,30 @@ Set to debug or trace if log4j initialization is failing. --> + ${env:KYUUBI_LOG_DIR} rest-audit.log rest-audit-%d{yyyy-MM-dd}-%i.log + k8s-audit.log + k8s-audit-%d{yyyy-MM-dd}-%i.log - + - - + + + + + + + + @@ -58,5 +69,8 @@ + + + diff --git a/dev/dependencyList b/dev/dependencyList index 9b8064e42..ede67c961 100644 --- a/dev/dependencyList +++ b/dev/dependencyList @@ -16,38 +16,40 @@ # HikariCP/4.0.3//HikariCP-4.0.3.jar +ST4/4.3.4//ST4-4.3.4.jar animal-sniffer-annotations/1.21//animal-sniffer-annotations-1.21.jar annotations/4.1.1.4//annotations-4.1.1.4.jar +antlr-runtime/3.5.3//antlr-runtime-3.5.3.jar antlr4-runtime/4.9.3//antlr4-runtime-4.9.3.jar aopalliance-repackaged/2.6.1//aopalliance-repackaged-2.6.1.jar -automaton/1.11-8//automaton-1.11-8.jar +arrow-format/12.0.0//arrow-format-12.0.0.jar +arrow-memory-core/12.0.0//arrow-memory-core-12.0.0.jar +arrow-memory-netty/12.0.0//arrow-memory-netty-12.0.0.jar +arrow-vector/12.0.0//arrow-vector-12.0.0.jar classgraph/4.8.138//classgraph-4.8.138.jar commons-codec/1.15//commons-codec-1.15.jar commons-collections/3.2.2//commons-collections-3.2.2.jar commons-lang/2.6//commons-lang-2.6.jar -commons-lang3/3.12.0//commons-lang3-3.12.0.jar +commons-lang3/3.13.0//commons-lang3-3.13.0.jar commons-logging/1.1.3//commons-logging-1.1.3.jar -curator-client/2.12.0//curator-client-2.12.0.jar -curator-framework/2.12.0//curator-framework-2.12.0.jar -curator-recipes/2.12.0//curator-recipes-2.12.0.jar derby/10.14.2.0//derby-10.14.2.0.jar error_prone_annotations/2.14.0//error_prone_annotations-2.14.0.jar failsafe/2.4.4//failsafe-2.4.4.jar failureaccess/1.0.1//failureaccess-1.0.1.jar +flatbuffers-java/1.12.0//flatbuffers-java-1.12.0.jar fliptables/1.0.2//fliptables-1.0.2.jar -generex/1.0.2//generex-1.0.2.jar -grpc-api/1.48.0//grpc-api-1.48.0.jar -grpc-context/1.48.0//grpc-context-1.48.0.jar -grpc-core/1.48.0//grpc-core-1.48.0.jar -grpc-grpclb/1.48.0//grpc-grpclb-1.48.0.jar -grpc-netty/1.48.0//grpc-netty-1.48.0.jar -grpc-protobuf-lite/1.48.0//grpc-protobuf-lite-1.48.0.jar -grpc-protobuf/1.48.0//grpc-protobuf-1.48.0.jar -grpc-stub/1.48.0//grpc-stub-1.48.0.jar +grpc-api/1.53.0//grpc-api-1.53.0.jar +grpc-context/1.53.0//grpc-context-1.53.0.jar +grpc-core/1.53.0//grpc-core-1.53.0.jar +grpc-grpclb/1.53.0//grpc-grpclb-1.53.0.jar +grpc-netty/1.53.0//grpc-netty-1.53.0.jar +grpc-protobuf-lite/1.53.0//grpc-protobuf-lite-1.53.0.jar +grpc-protobuf/1.53.0//grpc-protobuf-1.53.0.jar +grpc-stub/1.53.0//grpc-stub-1.53.0.jar gson/2.9.0//gson-2.9.0.jar -guava/31.1-jre//guava-31.1-jre.jar -hadoop-client-api/3.3.4//hadoop-client-api-3.3.4.jar -hadoop-client-runtime/3.3.4//hadoop-client-runtime-3.3.4.jar +guava/32.0.1-jre//guava-32.0.1-jre.jar +hadoop-client-api/3.3.6//hadoop-client-api-3.3.6.jar +hadoop-client-runtime/3.3.6//hadoop-client-runtime-3.3.6.jar hive-common/3.1.3//hive-common-3.1.3.jar hive-metastore/3.1.3//hive-metastore-3.1.3.jar hive-serde/3.1.3//hive-serde-3.1.3.jar @@ -63,16 +65,16 @@ httpclient/4.5.14//httpclient-4.5.14.jar httpcore/4.4.16//httpcore-4.4.16.jar httpmime/4.5.14//httpmime-4.5.14.jar j2objc-annotations/1.3//j2objc-annotations-1.3.jar -jackson-annotations/2.14.1//jackson-annotations-2.14.1.jar -jackson-core/2.14.1//jackson-core-2.14.1.jar -jackson-databind/2.14.1//jackson-databind-2.14.1.jar -jackson-dataformat-yaml/2.14.1//jackson-dataformat-yaml-2.14.1.jar -jackson-datatype-jdk8/2.12.3//jackson-datatype-jdk8-2.12.3.jar -jackson-datatype-jsr310/2.14.1//jackson-datatype-jsr310-2.14.1.jar -jackson-jaxrs-base/2.14.1//jackson-jaxrs-base-2.14.1.jar -jackson-jaxrs-json-provider/2.14.1//jackson-jaxrs-json-provider-2.14.1.jar -jackson-module-jaxb-annotations/2.14.1//jackson-module-jaxb-annotations-2.14.1.jar -jackson-module-scala_2.12/2.14.1//jackson-module-scala_2.12-2.14.1.jar +jackson-annotations/2.15.0//jackson-annotations-2.15.0.jar +jackson-core/2.15.0//jackson-core-2.15.0.jar +jackson-databind/2.15.0//jackson-databind-2.15.0.jar +jackson-dataformat-yaml/2.15.0//jackson-dataformat-yaml-2.15.0.jar +jackson-datatype-jdk8/2.15.0//jackson-datatype-jdk8-2.15.0.jar +jackson-datatype-jsr310/2.15.0//jackson-datatype-jsr310-2.15.0.jar +jackson-jaxrs-base/2.15.0//jackson-jaxrs-base-2.15.0.jar +jackson-jaxrs-json-provider/2.15.0//jackson-jaxrs-json-provider-2.15.0.jar +jackson-module-jaxb-annotations/2.15.0//jackson-module-jaxb-annotations-2.15.0.jar +jackson-module-scala_2.12/2.15.0//jackson-module-scala_2.12-2.15.0.jar jakarta.annotation-api/1.3.5//jakarta.annotation-api-1.3.5.jar jakarta.inject/2.6.1//jakarta.inject-2.6.1.jar jakarta.servlet-api/4.0.4//jakarta.servlet-api-4.0.4.jar @@ -81,77 +83,85 @@ jakarta.ws.rs-api/2.1.6//jakarta.ws.rs-api-2.1.6.jar jakarta.xml.bind-api/2.3.2//jakarta.xml.bind-api-2.3.2.jar javassist/3.25.0-GA//javassist-3.25.0-GA.jar jcl-over-slf4j/1.7.36//jcl-over-slf4j-1.7.36.jar -jersey-client/2.38//jersey-client-2.38.jar -jersey-common/2.38//jersey-common-2.38.jar -jersey-container-servlet-core/2.38//jersey-container-servlet-core-2.38.jar -jersey-entity-filtering/2.38//jersey-entity-filtering-2.38.jar -jersey-hk2/2.38//jersey-hk2-2.38.jar -jersey-media-json-jackson/2.38//jersey-media-json-jackson-2.38.jar -jersey-media-multipart/2.38//jersey-media-multipart-2.38.jar -jersey-server/2.38//jersey-server-2.38.jar +jersey-client/2.39.1//jersey-client-2.39.1.jar +jersey-common/2.39.1//jersey-common-2.39.1.jar +jersey-container-servlet-core/2.39.1//jersey-container-servlet-core-2.39.1.jar +jersey-entity-filtering/2.39.1//jersey-entity-filtering-2.39.1.jar +jersey-hk2/2.39.1//jersey-hk2-2.39.1.jar +jersey-media-json-jackson/2.39.1//jersey-media-json-jackson-2.39.1.jar +jersey-media-multipart/2.39.1//jersey-media-multipart-2.39.1.jar +jersey-server/2.39.1//jersey-server-2.39.1.jar jetcd-api/0.7.3//jetcd-api-0.7.3.jar jetcd-common/0.7.3//jetcd-common-0.7.3.jar jetcd-core/0.7.3//jetcd-core-0.7.3.jar jetcd-grpc/0.7.3//jetcd-grpc-0.7.3.jar -jetty-http/9.4.50.v20221201//jetty-http-9.4.50.v20221201.jar -jetty-io/9.4.50.v20221201//jetty-io-9.4.50.v20221201.jar -jetty-security/9.4.50.v20221201//jetty-security-9.4.50.v20221201.jar -jetty-server/9.4.50.v20221201//jetty-server-9.4.50.v20221201.jar -jetty-servlet/9.4.50.v20221201//jetty-servlet-9.4.50.v20221201.jar -jetty-util-ajax/9.4.50.v20221201//jetty-util-ajax-9.4.50.v20221201.jar -jetty-util/9.4.50.v20221201//jetty-util-9.4.50.v20221201.jar +jetty-client/9.4.52.v20230823//jetty-client-9.4.52.v20230823.jar +jetty-http/9.4.52.v20230823//jetty-http-9.4.52.v20230823.jar +jetty-io/9.4.52.v20230823//jetty-io-9.4.52.v20230823.jar +jetty-proxy/9.4.52.v20230823//jetty-proxy-9.4.52.v20230823.jar +jetty-security/9.4.52.v20230823//jetty-security-9.4.52.v20230823.jar +jetty-server/9.4.52.v20230823//jetty-server-9.4.52.v20230823.jar +jetty-servlet/9.4.52.v20230823//jetty-servlet-9.4.52.v20230823.jar +jetty-util-ajax/9.4.52.v20230823//jetty-util-ajax-9.4.52.v20230823.jar +jetty-util/9.4.52.v20230823//jetty-util-9.4.52.v20230823.jar jline/0.9.94//jline-0.9.94.jar jul-to-slf4j/1.7.36//jul-to-slf4j-1.7.36.jar -kubernetes-client/5.12.1//kubernetes-client-5.12.1.jar -kubernetes-model-admissionregistration/5.12.1//kubernetes-model-admissionregistration-5.12.1.jar -kubernetes-model-apiextensions/5.12.1//kubernetes-model-apiextensions-5.12.1.jar -kubernetes-model-apps/5.12.1//kubernetes-model-apps-5.12.1.jar -kubernetes-model-autoscaling/5.12.1//kubernetes-model-autoscaling-5.12.1.jar -kubernetes-model-batch/5.12.1//kubernetes-model-batch-5.12.1.jar -kubernetes-model-certificates/5.12.1//kubernetes-model-certificates-5.12.1.jar -kubernetes-model-common/5.12.1//kubernetes-model-common-5.12.1.jar -kubernetes-model-coordination/5.12.1//kubernetes-model-coordination-5.12.1.jar -kubernetes-model-core/5.12.1//kubernetes-model-core-5.12.1.jar -kubernetes-model-discovery/5.12.1//kubernetes-model-discovery-5.12.1.jar -kubernetes-model-events/5.12.1//kubernetes-model-events-5.12.1.jar -kubernetes-model-extensions/5.12.1//kubernetes-model-extensions-5.12.1.jar -kubernetes-model-flowcontrol/5.12.1//kubernetes-model-flowcontrol-5.12.1.jar -kubernetes-model-metrics/5.12.1//kubernetes-model-metrics-5.12.1.jar -kubernetes-model-networking/5.12.1//kubernetes-model-networking-5.12.1.jar -kubernetes-model-node/5.12.1//kubernetes-model-node-5.12.1.jar -kubernetes-model-policy/5.12.1//kubernetes-model-policy-5.12.1.jar -kubernetes-model-rbac/5.12.1//kubernetes-model-rbac-5.12.1.jar -kubernetes-model-scheduling/5.12.1//kubernetes-model-scheduling-5.12.1.jar -kubernetes-model-storageclass/5.12.1//kubernetes-model-storageclass-5.12.1.jar +kafka-clients/3.5.1//kafka-clients-3.5.1.jar +kubernetes-client-api/6.8.1//kubernetes-client-api-6.8.1.jar +kubernetes-client/6.8.1//kubernetes-client-6.8.1.jar +kubernetes-httpclient-okhttp/6.8.1//kubernetes-httpclient-okhttp-6.8.1.jar +kubernetes-model-admissionregistration/6.8.1//kubernetes-model-admissionregistration-6.8.1.jar +kubernetes-model-apiextensions/6.8.1//kubernetes-model-apiextensions-6.8.1.jar +kubernetes-model-apps/6.8.1//kubernetes-model-apps-6.8.1.jar +kubernetes-model-autoscaling/6.8.1//kubernetes-model-autoscaling-6.8.1.jar +kubernetes-model-batch/6.8.1//kubernetes-model-batch-6.8.1.jar +kubernetes-model-certificates/6.8.1//kubernetes-model-certificates-6.8.1.jar +kubernetes-model-common/6.8.1//kubernetes-model-common-6.8.1.jar +kubernetes-model-coordination/6.8.1//kubernetes-model-coordination-6.8.1.jar +kubernetes-model-core/6.8.1//kubernetes-model-core-6.8.1.jar +kubernetes-model-discovery/6.8.1//kubernetes-model-discovery-6.8.1.jar +kubernetes-model-events/6.8.1//kubernetes-model-events-6.8.1.jar +kubernetes-model-extensions/6.8.1//kubernetes-model-extensions-6.8.1.jar +kubernetes-model-flowcontrol/6.8.1//kubernetes-model-flowcontrol-6.8.1.jar +kubernetes-model-gatewayapi/6.8.1//kubernetes-model-gatewayapi-6.8.1.jar +kubernetes-model-metrics/6.8.1//kubernetes-model-metrics-6.8.1.jar +kubernetes-model-networking/6.8.1//kubernetes-model-networking-6.8.1.jar +kubernetes-model-node/6.8.1//kubernetes-model-node-6.8.1.jar +kubernetes-model-policy/6.8.1//kubernetes-model-policy-6.8.1.jar +kubernetes-model-rbac/6.8.1//kubernetes-model-rbac-6.8.1.jar +kubernetes-model-resource/6.8.1//kubernetes-model-resource-6.8.1.jar +kubernetes-model-scheduling/6.8.1//kubernetes-model-scheduling-6.8.1.jar +kubernetes-model-storageclass/6.8.1//kubernetes-model-storageclass-6.8.1.jar libfb303/0.9.3//libfb303-0.9.3.jar libthrift/0.9.3//libthrift-0.9.3.jar -log4j-1.2-api/2.19.0//log4j-1.2-api-2.19.0.jar -log4j-api/2.19.0//log4j-api-2.19.0.jar -log4j-core/2.19.0//log4j-core-2.19.0.jar -log4j-slf4j-impl/2.19.0//log4j-slf4j-impl-2.19.0.jar +log4j-1.2-api/2.20.0//log4j-1.2-api-2.20.0.jar +log4j-api/2.20.0//log4j-api-2.20.0.jar +log4j-core/2.20.0//log4j-core-2.20.0.jar +log4j-slf4j-impl/2.20.0//log4j-slf4j-impl-2.20.0.jar logging-interceptor/3.12.12//logging-interceptor-3.12.12.jar +lz4-java/1.8.0//lz4-java-1.8.0.jar metrics-core/4.2.8//metrics-core-4.2.8.jar metrics-jmx/4.2.8//metrics-jmx-4.2.8.jar metrics-json/4.2.8//metrics-json-4.2.8.jar metrics-jvm/4.2.8//metrics-jvm-4.2.8.jar mimepull/1.9.15//mimepull-1.9.15.jar -netty-all/4.1.87.Final//netty-all-4.1.87.Final.jar -netty-buffer/4.1.87.Final//netty-buffer-4.1.87.Final.jar -netty-codec-dns/4.1.87.Final//netty-codec-dns-4.1.87.Final.jar -netty-codec-http/4.1.87.Final//netty-codec-http-4.1.87.Final.jar -netty-codec-http2/4.1.87.Final//netty-codec-http2-4.1.87.Final.jar -netty-codec-socks/4.1.87.Final//netty-codec-socks-4.1.87.Final.jar -netty-codec/4.1.87.Final//netty-codec-4.1.87.Final.jar -netty-common/4.1.87.Final//netty-common-4.1.87.Final.jar -netty-handler-proxy/4.1.87.Final//netty-handler-proxy-4.1.87.Final.jar -netty-handler/4.1.87.Final//netty-handler-4.1.87.Final.jar -netty-resolver-dns/4.1.87.Final//netty-resolver-dns-4.1.87.Final.jar -netty-resolver/4.1.87.Final//netty-resolver-4.1.87.Final.jar -netty-transport-classes-epoll/4.1.87.Final//netty-transport-classes-epoll-4.1.87.Final.jar -netty-transport-native-epoll/4.1.87.Final/linux-aarch_64/netty-transport-native-epoll-4.1.87.Final-linux-aarch_64.jar -netty-transport-native-epoll/4.1.87.Final/linux-x86_64/netty-transport-native-epoll-4.1.87.Final-linux-x86_64.jar -netty-transport-native-unix-common/4.1.87.Final//netty-transport-native-unix-common-4.1.87.Final.jar -netty-transport/4.1.87.Final//netty-transport-4.1.87.Final.jar +netty-all/4.1.93.Final//netty-all-4.1.93.Final.jar +netty-buffer/4.1.93.Final//netty-buffer-4.1.93.Final.jar +netty-codec-dns/4.1.93.Final//netty-codec-dns-4.1.93.Final.jar +netty-codec-http/4.1.93.Final//netty-codec-http-4.1.93.Final.jar +netty-codec-http2/4.1.93.Final//netty-codec-http2-4.1.93.Final.jar +netty-codec-socks/4.1.93.Final//netty-codec-socks-4.1.93.Final.jar +netty-codec/4.1.93.Final//netty-codec-4.1.93.Final.jar +netty-common/4.1.93.Final//netty-common-4.1.93.Final.jar +netty-handler-proxy/4.1.93.Final//netty-handler-proxy-4.1.93.Final.jar +netty-handler/4.1.93.Final//netty-handler-4.1.93.Final.jar +netty-resolver-dns/4.1.93.Final//netty-resolver-dns-4.1.93.Final.jar +netty-resolver/4.1.93.Final//netty-resolver-4.1.93.Final.jar +netty-transport-classes-epoll/4.1.93.Final//netty-transport-classes-epoll-4.1.93.Final.jar +netty-transport-native-epoll/4.1.93.Final/linux-aarch_64/netty-transport-native-epoll-4.1.93.Final-linux-aarch_64.jar +netty-transport-native-epoll/4.1.93.Final/linux-x86_64/netty-transport-native-epoll-4.1.93.Final-linux-x86_64.jar +netty-transport-native-unix-common/4.1.93.Final//netty-transport-native-unix-common-4.1.93.Final.jar +netty-transport/4.1.93.Final//netty-transport-4.1.93.Final.jar okhttp-urlconnection/3.14.9//okhttp-urlconnection-3.14.9.jar okhttp/3.12.12//okhttp-3.12.12.jar okio/1.15.0//okio-1.15.0.jar @@ -161,7 +171,7 @@ perfmark-api/0.25.0//perfmark-api-0.25.0.jar proto-google-common-protos/2.9.0//proto-google-common-protos-2.9.0.jar protobuf-java-util/3.21.7//protobuf-java-util-3.21.7.jar protobuf-java/3.21.7//protobuf-java-3.21.7.jar -scala-library/2.12.17//scala-library-2.12.17.jar +scala-library/2.12.18//scala-library-2.12.18.jar scopt_2.12/4.1.0//scopt_2.12-4.1.0.jar simpleclient/0.16.0//simpleclient-0.16.0.jar simpleclient_common/0.16.0//simpleclient_common-0.16.0.jar @@ -172,16 +182,18 @@ simpleclient_tracer_common/0.16.0//simpleclient_tracer_common-0.16.0.jar simpleclient_tracer_otel/0.16.0//simpleclient_tracer_otel-0.16.0.jar simpleclient_tracer_otel_agent/0.16.0//simpleclient_tracer_otel_agent-0.16.0.jar slf4j-api/1.7.36//slf4j-api-1.7.36.jar -snakeyaml/1.33//snakeyaml-1.33.jar +snakeyaml-engine/2.6//snakeyaml-engine-2.6.jar +snakeyaml/2.2//snakeyaml-2.2.jar +snappy-java/1.1.10.1//snappy-java-1.1.10.1.jar +sqlite-jdbc/3.42.0.0//sqlite-jdbc-3.42.0.0.jar swagger-annotations/2.2.1//swagger-annotations-2.2.1.jar swagger-core/2.2.1//swagger-core-2.2.1.jar swagger-integration/2.2.1//swagger-integration-2.2.1.jar swagger-jaxrs2/2.2.1//swagger-jaxrs2-2.2.1.jar swagger-models/2.2.1//swagger-models-2.2.1.jar -swagger-ui/4.9.1//swagger-ui-4.9.1.jar trino-client/363//trino-client-363.jar units/1.6//units-1.6.jar vertx-core/4.3.2//vertx-core-4.3.2.jar vertx-grpc/4.3.2//vertx-grpc-4.3.2.jar zjsonpatch/0.3.0//zjsonpatch-0.3.0.jar -zookeeper/3.4.14//zookeeper-3.4.14.jar +zstd-jni/1.5.5-1//zstd-jni-1.5.5-1.jar diff --git a/dev/gen/gen_all_config_docs.sh b/dev/gen/gen_all_config_docs.sh new file mode 100755 index 000000000..2a5dca7f9 --- /dev/null +++ b/dev/gen/gen_all_config_docs.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Golden result file: +# docs/deployment/settings.md + +KYUUBI_UPDATE="${KYUUBI_UPDATE:-1}" \ +build/mvn clean test \ + -pl kyuubi-server -am \ + -Pflink-provided,spark-provided,hive-provided \ + -Dtest=none \ + -DwildcardSuites=org.apache.kyuubi.config.AllKyuubiConfiguration diff --git a/dev/gen/gen_hive_kdf_docs.sh b/dev/gen/gen_hive_kdf_docs.sh new file mode 100755 index 000000000..b670dc3c5 --- /dev/null +++ b/dev/gen/gen_hive_kdf_docs.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Golden result file: +# docs/extensions/engines/hive/functions.md + +KYUUBI_UPDATE="${KYUUBI_UPDATE:-1}" \ +build/mvn clean test \ + -pl externals/kyuubi-hive-sql-engine -am \ + -Pflink-provided,spark-provided,hive-provided \ + -DwildcardSuites=org.apache.kyuubi.engine.hive.udf.KyuubiDefinedFunctionSuite diff --git a/dev/gen/gen_ranger_policy_json.sh b/dev/gen/gen_ranger_policy_json.sh new file mode 100755 index 000000000..1f4193d3e --- /dev/null +++ b/dev/gen/gen_ranger_policy_json.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Golden result file: +# extensions/spark/kyuubi-spark-authz/src/test/resources/sparkSql_hive_jenkins.json + +KYUUBI_UPDATE="${KYUUBI_UPDATE:-1}" \ +build/mvn clean test \ + -pl extensions/spark/kyuubi-spark-authz \ + -Pgen-policy \ + -Dtest=none \ + -DwildcardSuites=org.apache.kyuubi.plugin.spark.authz.gen.PolicyJsonFileGenerator diff --git a/dev/gen/gen_ranger_spec_json.sh b/dev/gen/gen_ranger_spec_json.sh new file mode 100755 index 000000000..e00857f8f --- /dev/null +++ b/dev/gen/gen_ranger_spec_json.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Golden result file: +# extensions/spark/kyuubi-spark-authz/src/main/resources/*_spec.json + +KYUUBI_UPDATE="${KYUUBI_UPDATE:-1}" \ +build/mvn clean test \ + -pl extensions/spark/kyuubi-spark-authz \ + -Pgen-policy \ + -Dtest=none \ + -DwildcardSuites=org.apache.kyuubi.plugin.spark.authz.gen.JsonSpecFileGenerator diff --git a/dev/gen/gen_spark_kdf_docs.sh b/dev/gen/gen_spark_kdf_docs.sh new file mode 100755 index 000000000..ac13082e3 --- /dev/null +++ b/dev/gen/gen_spark_kdf_docs.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Golden result file: +# docs/extensions/engines/spark/functions.md + +KYUUBI_UPDATE="${KYUUBI_UPDATE:-1}" \ +build/mvn clean test \ + -pl externals/kyuubi-spark-sql-engine -am \ + -Pflink-provided,spark-provided,hive-provided \ + -DwildcardSuites=org.apache.kyuubi.engine.spark.udf.KyuubiDefinedFunctionSuite diff --git a/dev/gen/gen_tpcds_output_schema.sh b/dev/gen/gen_tpcds_output_schema.sh new file mode 100755 index 000000000..49f8d7798 --- /dev/null +++ b/dev/gen/gen_tpcds_output_schema.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Golden result file: +# extensions/spark/kyuubi-spark-authz/src/test/resources/*.output.schema + +KYUUBI_UPDATE="${KYUUBI_UPDATE:-1}" \ +build/mvn clean install \ + -pl kyuubi-server -am \ + -Dmaven.plugin.scalatest.exclude.tags="" \ + -Dtest=none \ + -DwildcardSuites=org.apache.kyuubi.operation.tpcds.OutputSchemaTPCDSSuite diff --git a/dev/gen/gen_tpcds_queries.sh b/dev/gen/gen_tpcds_queries.sh new file mode 100755 index 000000000..07f075b7a --- /dev/null +++ b/dev/gen/gen_tpcds_queries.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Golden result file: +# kyuubi-spark-connector-tpcds/src/main/resources/kyuubi/tpcds_*/*.sql + +KYUUBI_UPDATE="${KYUUBI_UPDATE:-1}" \ +build/mvn clean install \ + -pl extensions/spark/kyuubi-spark-connector-tpcds -am \ + -Dmaven.plugin.scalatest.exclude.tags="" \ + -Dtest=none \ + -DwildcardSuites=org.apache.kyuubi.spark.connector.tpcds.TPCDSQuerySuite diff --git a/dev/gen/gen_tpch_queries.sh b/dev/gen/gen_tpch_queries.sh new file mode 100755 index 000000000..d0c65256f --- /dev/null +++ b/dev/gen/gen_tpch_queries.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Golden result file: +# kyuubi-spark-connector-tpcds/src/main/resources/kyuubi/tpcdh_*/*.sql + +KYUUBI_UPDATE="${KYUUBI_UPDATE:-1}" \ +build/mvn clean install \ + -pl extensions/spark/kyuubi-spark-connector-tpch -am \ + -Dmaven.plugin.scalatest.exclude.tags="" \ + -Dtest=none \ + -DwildcardSuites=org.apache.kyuubi.spark.connector.tpch.TPCHQuerySuite diff --git a/dev/kyuubi-codecov/pom.xml b/dev/kyuubi-codecov/pom.xml index 1d1dcb574..0f22c3316 100644 --- a/dev/kyuubi-codecov/pom.xml +++ b/dev/kyuubi-codecov/pom.xml @@ -21,16 +21,28 @@ org.apache.kyuubi kyuubi-parent - 1.7.0-SNAPSHOT + 1.9.0-SNAPSHOT ../../pom.xml - kyuubi-codecov_2.12 + kyuubi-codecov_${scala.binary.version} pom Kyuubi Dev Code Coverage https://kyuubi.apache.org/ + + org.apache.kyuubi + kyuubi-util + ${project.version} + + + + org.apache.kyuubi + kyuubi-util-scala_${scala.binary.version} + ${project.version} + + org.apache.kyuubi kyuubi-common_${scala.binary.version} @@ -199,7 +211,17 @@ org.apache.kyuubi - kyuubi-spark-connector-kudu_${scala.binary.version} + kyuubi-spark-connector-hive_${scala.binary.version} + ${project.version} + + + + + spark-3.4 + + + org.apache.kyuubi + kyuubi-extension-spark-3-4_${scala.binary.version} ${project.version} @@ -209,5 +231,15 @@ + + spark-3.5 + + + org.apache.kyuubi + kyuubi-extension-spark-3-5_${scala.binary.version} + ${project.version} + + + diff --git a/dev/kyuubi-tpcds/README.md b/dev/kyuubi-tpcds/README.md index adffb6726..a9a6487aa 100644 --- a/dev/kyuubi-tpcds/README.md +++ b/dev/kyuubi-tpcds/README.md @@ -1,21 +1,22 @@ +- Licensed to the Apache Software Foundation (ASF) under one or more +- contributor license agreements. See the NOTICE file distributed with +- this work for additional information regarding copyright ownership. +- The ASF licenses this file to You under the Apache License, Version 2.0 +- (the "License"); you may not use this file except in compliance with +- the License. You may obtain a copy of the License at +- +- http://www.apache.org/licenses/LICENSE-2.0 +- +- Unless required by applicable law or agreed to in writing, software +- distributed under the License is distributed on an "AS IS" BASIS, +- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +- See the License for the specific language governing permissions and +- limitations under the License. +--> # Introduction + This module includes TPC-DS data generator and benchmark tool. # How to use @@ -27,12 +28,12 @@ package jar with following command: Support options: -| key | default | description | -|--------------|-----------------|-----------------------------------| -| db | default | the database to write data | -| scaleFactor | 1 | the scale factor of TPC-DS | -| format | parquet | the format of table to store data | -| parallel | scaleFactor * 2 | the parallelism of Spark job | +| key | default | description | +|-------------|-----------------|-----------------------------------| +| db | default | the database to write data | +| scaleFactor | 1 | the scale factor of TPC-DS | +| format | parquet | the format of table to store data | +| parallel | scaleFactor * 2 | the parallelism of Spark job | Example: the following command to generate 10GB data with new database `tpcds_sf10`. @@ -47,7 +48,7 @@ $SPARK_HOME/bin/spark-submit \ Support options: -| key | default | description | +| key | default | description | |-------------|------------------------|---------------------------------------------------------------| | db | none(required) | the TPC-DS database | | benchmark | tpcds-v2.4-benchmark | the name of application | @@ -65,6 +66,7 @@ $SPARK_HOME/bin/spark-submit \ ``` We also support run one of the TPC-DS query: + ```shell $SPARK_HOME/bin/spark-submit \ --class org.apache.kyuubi.tpcds.benchmark.RunBenchmark \ @@ -73,6 +75,7 @@ $SPARK_HOME/bin/spark-submit \ The result of TPC-DS benchmark like: -| name | minTimeMs | maxTimeMs | avgTimeMs | stdDev | stdDevPercent | -|---------|-----------|-------------|------------|----------|----------------| -| q1-v2.4 | 50.522384 | 868.010383 | 323.398267 | 471.6482 | 145.8413108576 | +| name | minTimeMs | maxTimeMs | avgTimeMs | stdDev | stdDevPercent | +|---------|-----------|------------|------------|----------|----------------| +| q1-v2.4 | 50.522384 | 868.010383 | 323.398267 | 471.6482 | 145.8413108576 | + diff --git a/dev/kyuubi-tpcds/pom.xml b/dev/kyuubi-tpcds/pom.xml index 2921cbe8b..b80c1227f 100644 --- a/dev/kyuubi-tpcds/pom.xml +++ b/dev/kyuubi-tpcds/pom.xml @@ -21,11 +21,11 @@ org.apache.kyuubi kyuubi-parent - 1.7.0-SNAPSHOT + 1.9.0-SNAPSHOT ../../pom.xml - kyuubi-tpcds_2.12 + kyuubi-tpcds_${scala.binary.version} jar Kyuubi Dev TPCDS Generator https://kyuubi.apache.org/ diff --git a/dev/merge_kyuubi_pr.py b/dev/merge_kyuubi_pr.py index cb3696d1f..fe8893748 100755 --- a/dev/merge_kyuubi_pr.py +++ b/dev/merge_kyuubi_pr.py @@ -30,9 +30,9 @@ import re import subprocess import sys -from urllib.request import urlopen -from urllib.request import Request from urllib.error import HTTPError +from urllib.request import Request +from urllib.request import urlopen KYUUBI_HOME = os.environ.get("KYUUBI_HOME", os.getcwd()) PR_REMOTE_NAME = os.environ.get("PR_REMOTE_NAME", "apache") @@ -248,6 +248,8 @@ def main(): user_login = pr["user"]["login"] base_ref = pr["head"]["ref"] pr_repo_desc = "%s/%s" % (user_login, base_ref) + assignees = pr["assignees"] + milestone = pr["milestone"] # Merged pull requests don't appear as merged in the GitHub API; # Instead, they're closed by asfgit. @@ -276,6 +278,17 @@ def main(): print("\n=== Pull Request #%s ===" % pr_num) print("title:\t%s\nsource:\t%s\ntarget:\t%s\nurl:\t%s\nbody:\n\n%s" % (title, pr_repo_desc, target_ref, url, body)) + + if assignees is None or len(assignees)==0: + continue_maybe("Assignees have NOT been set. Continue?") + else: + print("assignees: %s" % [assignee["login"] for assignee in assignees]) + + if milestone is None: + continue_maybe("Milestone has NOT been set. Continue?") + else: + print("milestone: %s" % milestone["title"]) + continue_maybe("Proceed with merging pull request #%s?" % pr_num) merged_refs = [target_ref] diff --git a/dev/reformat b/dev/reformat index 7c6ef7124..7ad26ae2e 100755 --- a/dev/reformat +++ b/dev/reformat @@ -20,7 +20,7 @@ set -x KYUUBI_HOME="$(cd "`dirname "$0"`/.."; pwd)" -PROFILES="-Pflink-provided,hive-provided,spark-provided,spark-block-cleaner,spark-3.3,spark-3.2,spark-3.1,tpcds" +PROFILES="-Pflink-provided,hive-provided,spark-provided,spark-block-cleaner,spark-3.5,spark-3.4,spark-3.3,spark-3.2,spark-3.1,tpcds,kubernetes-it" # python style checks rely on `black` in path if ! command -v black &> /dev/null diff --git a/docker/Dockerfile b/docker/Dockerfile index 588f99b1f..0440022de 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -24,7 +24,7 @@ # -t the target repo and tag name # more options can be found with -h -ARG BASE_IMAGE=openjdk:8-jre-slim +ARG BASE_IMAGE=eclipse-temurin:8-jdk-focal ARG spark_provided="spark_builtin" FROM ${BASE_IMAGE} as builder_spark_provided @@ -34,7 +34,7 @@ ONBUILD ENV SPARK_HOME ${spark_home_in_docker} FROM ${BASE_IMAGE} as builder_spark_builtin ONBUILD ENV SPARK_HOME /opt/spark -ONBUILD RUN mkdir -p ${SPARK_HOME} +ONBUILD RUN mkdir -p ${SPARK_HOME} ONBUILD COPY spark-binary ${SPARK_HOME} FROM builder_${spark_provided} @@ -50,7 +50,8 @@ ENV KYUUBI_WORK_DIR_ROOT ${KYUUBI_HOME}/work RUN set -ex && \ sed -i 's/http:\/\/deb.\(.*\)/https:\/\/deb.\1/g' /etc/apt/sources.list && \ apt-get update && \ - apt install -y bash tini libc6 libpam-modules krb5-user libnss3 procps && \ + apt-get install -y bash tini libc6 libpam-modules krb5-user libnss3 procps && \ + ln -snf /bin/bash /bin/sh && \ useradd -u ${kyuubi_uid} -g root kyuubi -d /home/kyuubi -m && \ mkdir -p ${KYUUBI_HOME} ${KYUUBI_LOG_DIR} ${KYUUBI_PID_DIR} ${KYUUBI_WORK_DIR_ROOT} && \ rm -rf /var/cache/apt/* @@ -59,6 +60,7 @@ COPY LICENSE NOTICE RELEASE ${KYUUBI_HOME}/ COPY bin ${KYUUBI_HOME}/bin COPY jars ${KYUUBI_HOME}/jars COPY beeline-jars ${KYUUBI_HOME}/beeline-jars +COPY web-ui ${KYUUBI_HOME}/web-ui COPY externals/engines/spark ${KYUUBI_HOME}/externals/engines/spark WORKDIR ${KYUUBI_HOME} diff --git a/docker/kyuubi-configmap.yaml b/docker/kyuubi-configmap.yaml index 13835493b..6a6d430ce 100644 --- a/docker/kyuubi-configmap.yaml +++ b/docker/kyuubi-configmap.yaml @@ -52,4 +52,4 @@ data: # kyuubi.frontend.bind.port 10009 # - # Details in https://kyuubi.apache.org/docs/latest/deployment/settings.html + # Details in https://kyuubi.readthedocs.io/en/master/configuration/settings.html diff --git a/docker/playground/.env b/docker/playground/.env index d50e964cf..24284bd39 100644 --- a/docker/playground/.env +++ b/docker/playground/.env @@ -15,16 +15,16 @@ # limitations under the License. # -AWS_JAVA_SDK_VERSION=1.12.239 -HADOOP_VERSION=3.3.1 +AWS_JAVA_SDK_VERSION=1.12.367 +HADOOP_VERSION=3.3.6 HIVE_VERSION=2.3.9 -ICEBERG_VERSION=1.1.0 -KYUUBI_VERSION=1.6.1-incubating -KYUUBI_HADOOP_VERSION=3.3.4 +ICEBERG_VERSION=1.3.1 +KYUUBI_VERSION=1.7.3 +KYUUBI_HADOOP_VERSION=3.3.5 POSTGRES_VERSION=12 POSTGRES_JDBC_VERSION=42.3.4 SCALA_BINARY_VERSION=2.12 -SPARK_VERSION=3.3.1 +SPARK_VERSION=3.3.3 SPARK_BINARY_VERSION=3.3 SPARK_HADOOP_VERSION=3.3.2 ZOOKEEPER_VERSION=3.6.3 diff --git a/docker/playground/README.md b/docker/playground/README.md index d9e227c2c..66dca2af0 100644 --- a/docker/playground/README.md +++ b/docker/playground/README.md @@ -1,5 +1,5 @@ Playground -=== +========== ## For Users @@ -45,3 +45,4 @@ Kyuubi supply some built-in dataset, after Kyuubi started, you can run the follo 1. Build images `docker/playground/build-image.sh`; 2. Optional to use `buildx` to build and publish cross-platform images `BUILDX=1 docker/playground/build-image.sh`; + diff --git a/docker/playground/compose.yml b/docker/playground/compose.yml index 069624ee2..362b3505b 100644 --- a/docker/playground/compose.yml +++ b/docker/playground/compose.yml @@ -17,11 +17,11 @@ services: minio: - image: alekcander/bitnami-minio-multiarch:RELEASE.2022-05-26T05-48-41Z + image: bitnami/minio:2023-debian-11 environment: MINIO_ROOT_USER: minio MINIO_ROOT_PASSWORD: minio_minio - MINIO_DEFAULT_BUCKETS: spark-bucket,iceberg-bucket + MINIO_DEFAULT_BUCKETS: spark-bucket container_name: minio hostname: minio ports: @@ -68,6 +68,7 @@ services: ports: - 4040-4050:4040-4050 - 10009:10009 + - 10099:10099 volumes: - ./conf/core-site.xml:/etc/hadoop/conf/core-site.xml - ./conf/hive-site.xml:/etc/hive/conf/hive-site.xml diff --git a/docker/playground/conf/kyuubi-defaults.conf b/docker/playground/conf/kyuubi-defaults.conf index 4906c5de4..e4a674634 100644 --- a/docker/playground/conf/kyuubi-defaults.conf +++ b/docker/playground/conf/kyuubi-defaults.conf @@ -18,8 +18,10 @@ ## Kyuubi Configurations kyuubi.authentication=NONE -kyuubi.frontend.thrift.binary.bind.host=0.0.0.0 +kyuubi.frontend.bind.host=0.0.0.0 +kyuubi.frontend.protocols=THRIFT_BINARY,REST kyuubi.frontend.thrift.binary.bind.port=10009 +kyuubi.frontend.rest.bind.port=10099 kyuubi.ha.addresses=zookeeper:2181 kyuubi.session.engine.idle.timeout=PT5M kyuubi.operation.incremental.collect=true @@ -28,4 +30,4 @@ kyuubi.operation.progress.enabled=true kyuubi.engine.session.initialize.sql \ show namespaces in tpcds; \ show namespaces in tpch; \ - show namespaces in postgres; + show namespaces in postgres diff --git a/docker/playground/conf/kyuubi-log4j2.xml b/docker/playground/conf/kyuubi-log4j2.xml index 6aedf7652..313c121bc 100644 --- a/docker/playground/conf/kyuubi-log4j2.xml +++ b/docker/playground/conf/kyuubi-log4j2.xml @@ -22,7 +22,7 @@ - + diff --git a/docker/playground/conf/spark-defaults.conf b/docker/playground/conf/spark-defaults.conf index 9d1d4a602..7983b5e70 100644 --- a/docker/playground/conf/spark-defaults.conf +++ b/docker/playground/conf/spark-defaults.conf @@ -38,7 +38,3 @@ spark.sql.catalog.postgres.url=jdbc:postgresql://postgres:5432/metastore spark.sql.catalog.postgres.driver=org.postgresql.Driver spark.sql.catalog.postgres.user=postgres spark.sql.catalog.postgres.password=postgres - -spark.sql.catalog.iceberg=org.apache.iceberg.spark.SparkCatalog -spark.sql.catalog.iceberg.type=hadoop -spark.sql.catalog.iceberg.warehouse=s3a://iceberg-bucket/iceberg-warehouse diff --git a/docker/playground/image/kyuubi-playground-base.Dockerfile b/docker/playground/image/kyuubi-playground-base.Dockerfile index 6ee4ed405..e8375eb68 100644 --- a/docker/playground/image/kyuubi-playground-base.Dockerfile +++ b/docker/playground/image/kyuubi-playground-base.Dockerfile @@ -20,4 +20,4 @@ RUN set -x && \ mkdir /opt/busybox && \ busybox --install /opt/busybox -ENV PATH=/opt/java/openjdk/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/busybox +ENV PATH=${PATH}:/opt/busybox diff --git a/docs/appendix/terminology.md b/docs/appendix/terminology.md index 21b8cb1b6..b349d77c7 100644 --- a/docs/appendix/terminology.md +++ b/docs/appendix/terminology.md @@ -129,9 +129,9 @@ As an enterprise service, SLA commitment is essential. Deploying Kyuubi in High

-## DataLake & LakeHouse +## DataLake & Lakehouse -Kyuubi unifies DataLake & LakeHouse access in the simplest pure SQL way, meanwhile it's also the securest way with authentication and SQL standard authorization. +Kyuubi unifies DataLake & Lakehouse access in the simplest pure SQL way, meanwhile it's also the securest way with authentication and SQL standard authorization. ### Apache Iceberg @@ -139,7 +139,7 @@ Kyuubi unifies DataLake & LakeHouse access in the simplest pure SQL way, meanwhi

-http://iceberg.apache.org/ +https://iceberg.apache.org/

diff --git a/docs/client/advanced/kerberos.md b/docs/client/advanced/kerberos.md index 4962dd2c8..a9cb55812 100644 --- a/docs/client/advanced/kerberos.md +++ b/docs/client/advanced/kerberos.md @@ -242,5 +242,5 @@ jdbc:hive2://:/;kyuubiServerPrinc - `principal` is inherited from Hive JDBC Driver and is a little ambiguous, and we could use `kyuubiServerPrincipal` as its alias. - `kyuubi_server_principal` is the value of `kyuubi.kinit.principal` set in `kyuubi-defaults.conf`. - As a command line argument, JDBC URL should be quoted to avoid being split into 2 commands by ";". -- As to DBeaver, `;principal=` should be set as the `Database/Schema` argument. +- As to DBeaver, `;principal=` or `;kyuubiServerPrincipal=` should be set as the `Database/Schema` argument. diff --git a/docs/client/cli/hive_beeline.rst b/docs/client/cli/hive_beeline.rst index fda925aa1..f75e00819 100644 --- a/docs/client/cli/hive_beeline.rst +++ b/docs/client/cli/hive_beeline.rst @@ -17,7 +17,7 @@ Hive Beeline ============ Kyuubi supports Apache Hive beeline that works with Kyuubi server. -Hive beeline is a `SQLLine CLI `_ based on the `Hive JDBC Driver <../jdbc/hive_jdbc.html>`_. +Hive beeline is a `SQLLine CLI `_ based on the `Hive JDBC Driver <../jdbc/hive_jdbc.html>`_. Prerequisites ------------- diff --git a/docs/client/cli/index.rst b/docs/client/cli/index.rst index 61be9ad8c..19122ced4 100644 --- a/docs/client/cli/index.rst +++ b/docs/client/cli/index.rst @@ -21,3 +21,4 @@ Command Line Interface(CLI)s kyuubi_beeline hive_beeline + trino_cli diff --git a/docs/client/cli/trino_cli.md b/docs/client/cli/trino_cli.md new file mode 100644 index 000000000..68ebd8300 --- /dev/null +++ b/docs/client/cli/trino_cli.md @@ -0,0 +1,88 @@ + + +# Trino command line interface + +The Trino CLI provides a terminal-based, interactive shell for running queries. We can use it to connect Kyuubi server now. + +## Start Kyuubi Trino Server + +First we should configure the trino protocol and the service port in the `kyuubi.conf` + +``` +kyuubi.frontend.protocols TRINO +kyuubi.frontend.trino.bind.port 10999 #default port +``` + +## Install + +Download [trino-cli-363-executable.jar](https://repo1.maven.org/maven2/io/trino/trino-jdbc/363/trino-jdbc-363.jar), rename it to `trino`, make it executable with `chmod +x`, and run it to show the version of the CLI: + +``` +wget https://repo1.maven.org/maven2/io/trino/trino-jdbc/363/trino-jdbc-363.jar +mv trino-jdbc-363.jar trino +chmod +x trino +./trino --version +``` + +## Running the CLI + +The minimal command to start the CLI in interactive mode specifies the URL of the kyuubi server with the Trino protocol: + +``` +./trino --server http://localhost:10999 +``` + +If successful, you will get a prompt to execute commands. Use the help command to see a list of supported commands. Use the clear command to clear the terminal. To stop and exit the CLI, run exit or quit.: + +``` +trino> help + +Supported commands: +QUIT +EXIT +CLEAR +EXPLAIN [ ( option [, ...] ) ] + options: FORMAT { TEXT | GRAPHVIZ | JSON } + TYPE { LOGICAL | DISTRIBUTED | VALIDATE | IO } +DESCRIBE +SHOW COLUMNS FROM
+SHOW FUNCTIONS +SHOW CATALOGS [LIKE ] +SHOW SCHEMAS [FROM ] [LIKE ] +SHOW TABLES [FROM ] [LIKE ] +USE [.] +``` + +You can now run SQL statements. After processing, the CLI will show results and statistics. + +``` +trino> select 1; + _col0 +------- + 1 +(1 row) + +Query 20230216_125233_00806_examine_6hxus, FINISHED, 1 node +Splits: 1 total, 1 done (100.00%) +0.29 [0 rows, 0B] [0 rows/s, 0B/s] + +trino> +``` + +Many other options are available to further configure the CLI in interactive mode to +refer https://trino.io/docs/current/client/cli.html#running-the-cli diff --git a/docs/client/jdbc/hive_jdbc.md b/docs/client/jdbc/hive_jdbc.md index 42d2f7b5a..00498dfaa 100644 --- a/docs/client/jdbc/hive_jdbc.md +++ b/docs/client/jdbc/hive_jdbc.md @@ -19,14 +19,18 @@ ## Instructions -Kyuubi does not provide its own JDBC Driver so far, -as it is fully compatible with Hive JDBC and ODBC drivers that let you connect to popular Business Intelligence (BI) tools to query, -analyze and visualize data though Spark SQL engines. +Kyuubi is fully compatible with Hive JDBC and ODBC drivers that let you connect to popular Business Intelligence (BI) +tools to query, analyze and visualize data though Spark SQL engines. + +It's recommended to use [Kyuubi JDBC driver](./kyuubi_jdbc.html) for new applications. ## Install Hive JDBC For programing, the easiest way to get `hive-jdbc` is from [the maven central](https://mvnrepository.com/artifact/org.apache.hive/hive-jdbc). For example, +The following sections demonstrate how to use Hive JDBC driver 2.3.8 to connect Kyuubi Server, actually, any version +less or equals 3.1.x should work fine. + - **maven** ```xml @@ -76,7 +80,3 @@ jdbc:hive2://:/;?#<[spark|hive]Var jdbc:hive2://localhost:10009/default;hive.server2.proxy.user=proxy_user?kyuubi.engine.share.level=CONNECTION;spark.ui.enabled=false#var_x=y ``` -## Unsupported Hive Features - -- Connect to HiveServer2 using HTTP transport. ```transportMode=http``` - diff --git a/docs/client/jdbc/index.rst b/docs/client/jdbc/index.rst index 31871f138..abcd6a452 100644 --- a/docs/client/jdbc/index.rst +++ b/docs/client/jdbc/index.rst @@ -22,4 +22,5 @@ JDBC Drivers kyuubi_jdbc hive_jdbc mysql_jdbc + trino_jdbc diff --git a/docs/client/jdbc/kyuubi_jdbc.rst b/docs/client/jdbc/kyuubi_jdbc.rst index fdc40d599..7a63dbd98 100644 --- a/docs/client/jdbc/kyuubi_jdbc.rst +++ b/docs/client/jdbc/kyuubi_jdbc.rst @@ -17,14 +17,14 @@ Kyuubi Hive JDBC Driver ======================= .. versionadded:: 1.4.0 - Since 1.4.0, kyuubi community maintains a forked hive jdbc driver module and provides both shaded and non-shaded packages. + Kyuubi community maintains a forked Hive JDBC driver module and provides both shaded and non-shaded packages. -This packages aims to support some missing functionalities of the original hive jdbc. -For kyuubi engines that support multiple catalogs, it provides meta APIs for better support. -The behaviors of the original hive jdbc have remained. +This packages aims to support some missing functionalities of the original Hive JDBC driver. +For Kyuubi engines that support multiple catalogs, it provides meta APIs for better support. +The behaviors of the original Hive JDBC driver have remained. -To access a Hive data warehouse or new lakehouse formats, such as Apache Iceberg/Hudi, delta lake using the kyuubi jdbc driver for Apache kyuubi, you need to configure -the following: +To access a Hive data warehouse or new Lakehouse formats, such as Apache Iceberg/Hudi, Delta Lake using the Kyuubi JDBC driver +for Apache kyuubi, you need to configure the following: - The list of driver library files - :ref:`referencing-libraries`. - The Driver or DataSource class - :ref:`registering_class`. @@ -46,28 +46,28 @@ In the code, specify the artifact `kyuubi-hive-jdbc-shaded` from `Maven Central` Maven ^^^^^ -.. code-block:: xml +.. parsed-literal:: org.apache.kyuubi kyuubi-hive-jdbc-shaded - 1.5.2-incubating + \ |release|\ -Sbt +sbt ^^^ -.. code-block:: sbt +.. parsed-literal:: - libraryDependencies += "org.apache.kyuubi" % "kyuubi-hive-jdbc-shaded" % "1.5.2-incubating" + libraryDependencies += "org.apache.kyuubi" % "kyuubi-hive-jdbc-shaded" % "\ |release|\" Gradle ^^^^^^ -.. code-block:: gradle +.. parsed-literal:: - implementation group: 'org.apache.kyuubi', name: 'kyuubi-hive-jdbc-shaded', version: '1.5.2-incubating' + implementation group: 'org.apache.kyuubi', name: 'kyuubi-hive-jdbc-shaded', version: '\ |release|\' Using the Driver in a JDBC Application ************************************** @@ -92,11 +92,9 @@ connection for JDBC: .. code-block:: java - private static Connection connectViaDM() throws Exception - { - Connection connection = null; - connection = DriverManager.getConnection(CONNECTION_URL); - return connection; + private static Connection newKyuubiConnection() throws Exception { + Connection connection = DriverManager.getConnection(CONNECTION_URL); + return connection; } .. _building_url: @@ -112,12 +110,13 @@ accessing. The following is the format of the connection URL for the Kyuubi Hive .. code-block:: jdbc - jdbc:subprotocol://host:port/schema;<[#|?]sessionProperties> + jdbc:subprotocol://host:port[/catalog]/[schema];<[#|?]sessionProperties> - subprotocol: kyuubi or hive2 - host: DNS or IP address of the kyuubi server - port: The number of the TCP port that the server uses to listen for client requests -- dbName: Optional database name to set the current database to run the query against, use `default` if absent. +- catalog: Optional catalog name to set the current catalog to run the query against. +- schema: Optional database name to set the current database to run the query against, use `default` if absent. - clientProperties: Optional `semicolon(;)` separated `key=value` parameters identified and affect the client behavior locally. e.g., user=foo;password=bar. - sessionProperties: Optional `semicolon(;)` separated `key=value` parameters used to configure the session, operation or background engines. For instance, `kyuubi.engine.share.level=CONNECTION` determines the background engine instance is used only by the current connection. `spark.ui.enabled=false` disables the Spark UI of the engine. @@ -127,7 +126,7 @@ accessing. The following is the format of the connection URL for the Kyuubi Hive - Properties are case-sensitive - Do not duplicate properties in the connection URL -Connection URL over Http +Connection URL over HTTP ************************ .. versionadded:: 1.6.0 @@ -145,16 +144,101 @@ Connection URL over Service Discovery jdbc:subprotocol:///;serviceDiscoveryMode=zooKeeper;zooKeeperNamespace=kyuubi -- zookeeper quorum is the corresponding zookeeper cluster configured by `kyuubi.ha.zookeeper.quorum` at the server side. -- zooKeeperNamespace is the corresponding namespace configured by `kyuubi.ha.zookeeper.namespace` at the server side. +- zookeeper quorum is the corresponding zookeeper cluster configured by `kyuubi.ha.addresses` at the server side. +- zooKeeperNamespace is the corresponding namespace configured by `kyuubi.ha.namespace` at the server side. -Authentication --------------- +HiveServer2 Compatibility +************************* +.. versionadded:: 1.8.0 -DataTypes ---------- +JDBC Drivers need to negotiate a protocol version with Kyuubi Server/HiveServer2 when connecting. + +Kyuubi Hive JDBC Driver offers protocol version v10 (`clientProtocolVersion=9`, supported since Hive 2.3.0) +to server by default. + +If you need to connect to HiveServer2 before 2.3.0, +please set client property `clientProtocolVersion` to a lower number. + +.. code-block:: jdbc + + jdbc:subprotocol://host:port[/catalog]/[schema];clientProtocolVersion=9; + + +.. tip:: + All supported protocol versions and corresponding Hive versions can be found in `TProtocolVersion.java`_ + and its git commits. + +Kerberos Authentication +----------------------- +Since 1.6.0, Kyuubi JDBC driver implements the Kerberos authentication based on JAAS framework instead of `Hadoop UserGroupInformation`_, +which means it does not forcibly rely on Hadoop dependencies to connect a kerberized Kyuubi Server. + +Kyuubi JDBC driver supports different approaches to connect a kerberized Kyuubi Server. First of all, please follow +the `krb5.conf instruction`_ to setup ``krb5.conf`` properly. + +Authentication by Principal and Keytab +************************************** + +.. versionadded:: 1.6.0 + +.. tip:: + + It's the simplest way w/ minimal setup requirements for Kerberos authentication. + +It's straightforward to use principal and keytab for Kerberos authentication, just simply configure them in the JDBC URL. + +.. code-block:: + + jdbc:kyuubi://host:port/schema;kyuubiClientPrincipal=;kyuubiClientKeytab=;kyuubiServerPrincipal= + +- kyuubiClientPrincipal: Kerberos ``principal`` for client authentication +- kyuubiClientKeytab: path of Kerberos ``keytab`` file for client authentication +- kyuubiServerPrincipal: Kerberos ``principal`` configured by `kyuubi.kinit.principal` at the server side. ``kyuubiServerPrincipal`` is available + as an alias of ``principal`` since 1.7.0, use ``principal`` for previous versions. + +Authentication by Principal and TGT Cache +***************************************** + +Another typical usage of Kerberos authentication is using `kinit` to generate the TGT cache first, then the application +does Kerberos authentication through the TGT cache. + +.. code-block:: + + jdbc:kyuubi://host:port/schema;kyuubiServerPrincipal= + +Authentication by `Hadoop UserGroupInformation`_ ``doAs`` (programing only) +*************************************************************************** + +.. tip:: + + This approach allows project which already uses `Hadoop UserGroupInformation`_ for Kerberos authentication to easily + connect the kerberized Kyuubi Server. This approach does not work between [1.6.0, 1.7.0], and got fixed in 1.7.1. + +.. code-block:: + + String jdbcUrl = "jdbc:kyuubi://host:port/schema;kyuubiServerPrincipal=" + UserGroupInformation ugi = UserGroupInformation.loginUserFromKeytab(clientPrincipal, clientKeytab); + ugi.doAs((PrivilegedExceptionAction) () -> { + Connection conn = DriverManager.getConnection(jdbcUrl); + ... + }); + +Authentication by Subject (programing only) +******************************************* + +.. code-block:: java + + String jdbcUrl = "jdbc:kyuubi://host:port/schema;kyuubiServerPrincipal=;kerberosAuthType=fromSubject" + Subject kerberizedSubject = ...; + Subject.doAs(kerberizedSubject, (PrivilegedExceptionAction) () -> { + Connection conn = DriverManager.getConnection(jdbcUrl); + ... + }); .. _Maven Central: https://mvnrepository.com/artifact/org.apache.kyuubi/kyuubi-hive-jdbc-shaded .. _JDBC Applications: ../bi_tools/index.html .. _java.sql.DriverManager: https://docs.oracle.com/javase/8/docs/api/java/sql/DriverManager.html +.. _Hadoop UserGroupInformation: https://hadoop.apache.org/docs/stable/api/org/apache/hadoop/security/UserGroupInformation.html +.. _krb5.conf instruction: https://docs.oracle.com/javase/8/docs/technotes/guides/security/jgss/tutorials/KerberosReq.html +.. _TProtocolVersion.java: https://github.com/apache/hive/blob/master/service-rpc/src/gen/thrift/gen-javabean/org/apache/hive/service/rpc/thrift/TProtocolVersion.java \ No newline at end of file diff --git a/docs/client/jdbc/trino_jdbc.md b/docs/client/jdbc/trino_jdbc.md new file mode 100644 index 000000000..0f91c4337 --- /dev/null +++ b/docs/client/jdbc/trino_jdbc.md @@ -0,0 +1,92 @@ + + +# Trino JDBC Driver + +## Instructions + +Kyuubi currently supports the Trino connection protocol, so we can use Trino-JDBC to connect to the kyuubi server +and submit SQL to Spark, Trino and other engines for execution. + +## Start Kyuubi Trino Server + +First we should configure the trino protocol and the service port in the `kyuubi.conf` + +``` +kyuubi.frontend.protocols TRINO +kyuubi.frontend.trino.bind.port 10999 #default port +``` + +## Install Trino JDBC + +Download [trino-jdbc-363.jar](https://repo1.maven.org/maven2/io/trino/trino-jdbc/363/trino-jdbc-363.jar) and add it to the classpath of your Java application. + +The driver is also available from Maven Central: + +```xml + + io.trino + trino-jdbc + 363 + +``` + +## JDBC URL + +When your driver is loaded, registered and configured, you are ready to connect to Trino from your application. The following JDBC URL formats are supported: + +``` +jdbc:trino://host:port +``` + +Trino JDBC example + +```java +String trinoHost = "localhost"; +String trinoPort = "10999"; +String trinoUser = "default"; +String trinoPassword = null; +Connection connection = null; +ResultSet rs = null; + +try { + // Create the connection using the JDBC URL + connection = DriverManager.getConnection("jdbc:trino://" + trinoHost + ":" + trinoPort, trinoUser, trinoPassword); + + // Do whatever you need to do with the connection + Statement stmt = connection.createStatement(); + rs = stmt.executeQuery("SELECT 1"); + + while (rs.next()) { + // retrieve data from the ResultSet + } + +} catch (Exception e) { + e.printStackTrace(); +} finally { + try { + // Close the connection when you're done with it + if (rs != null) rs.close(); + if (connection != null) connection.close(); + } catch (Exception e) { + e.printStackTrace(); + } +} +``` + +The configuration of the connection parameters can be found in the official trino documentation at: https://trino.io/docs/current/client/jdbc.html#connection-parameters + diff --git a/docs/client/python/index.rst b/docs/client/python/index.rst index 70d2bc9e3..5e8ae4228 100644 --- a/docs/client/python/index.rst +++ b/docs/client/python/index.rst @@ -22,4 +22,4 @@ Python pyhive pyspark - + jaydebeapi diff --git a/docs/client/python/jaydebeapi.md b/docs/client/python/jaydebeapi.md new file mode 100644 index 000000000..3d89fd722 --- /dev/null +++ b/docs/client/python/jaydebeapi.md @@ -0,0 +1,87 @@ + + +# Python-JayDeBeApi + +The [JayDeBeApi](https://pypi.org/project/JayDeBeApi/) module allows you to connect from Python code to databases using Java JDBC. +It provides a Python DB-API v2.0 to that database. + +## Requirements + +To install Python-JayDeBeApi, you can use pip, the Python package manager. Open your command-line interface or terminal and run the following command: + +```shell +pip install jaydebeapi +``` + +If you want to install JayDeBeApi in Jython, you'll need to ensure that you have either pip or EasyInstall available for Jython. These tools are used to install Python packages, including JayDeBeApi. +Or you can get a copy of the source by cloning from the [JayDeBeApi GitHub project](https://github.com/baztian/jaydebeapi) and install it. + +```shell +python setup.py install +``` + +or if you are using Jython use + +```shell +jython setup.py install +``` + +## Preparation + +Using the Python-JayDeBeApi package to connect to Kyuubi, you need to install the library and configure the relevant JDBC driver. You can download JDBC driver from maven repository and specify its path in Python. Choose the matching driver `kyuubi-hive-jdbc-*.jar` package based on the Kyuubi server version. +The driver class name is `org.apache.kyuubi.jdbc.KyuubiHiveDriver`. + +| Package | Repo | +|--------------------|-----------------------------------------------------------------------------------------------------| +| kyuubi jdbc driver | [kyuubi-hive-jdbc-*.jar](https://repo1.maven.org/maven2/org/apache/kyuubi/kyuubi-hive-jdbc-shaded/) | + +## Usage + +Below is a simple example demonstrating how to use Python-JayDeBeApi to connect to Kyuubi database and execute a query: + +```python +import jaydebeapi + +# Set JDBC driver path and connection URL +driver = "org.apache.kyuubi.jdbc.KyuubiHiveDriver" +url = "jdbc:kyuubi://host:port/default" +jdbc_driver_path = ["/path/to/kyuubi-hive-jdbc-*.jar"] + +# Connect to the database using JayDeBeApi +conn = jaydebeapi.connect(driver, url, ["user", "password"], jdbc_driver_path) + +# Create a cursor object +cursor = conn.cursor() + +# Execute the SQL query +cursor.execute("SELECT * FROM example_table LIMIT 10") + +# Retrieve query results +result_set = cursor.fetchall() + +# Process the results +for row in result_set: + print(row) + +# Close the cursor and the connection +cursor.close() +conn.close() +``` + +Make sure to replace the placeholders (host, port, user, password) with your actual Kyuubi configuration. +With the above code, you can connect to Kyuubi and execute SQL queries in Python. Please handle exceptions and errors appropriately in real-world applications. diff --git a/docs/client/python/pyhive.md b/docs/client/python/pyhive.md index dbebf684f..b5e57ea2e 100644 --- a/docs/client/python/pyhive.md +++ b/docs/client/python/pyhive.md @@ -64,7 +64,47 @@ If password is provided for connection, make sure the `auth` param set to either ```python # open connection -conn = hive.Connection(host=kyuubi_host,port=10009, -user='user', password='password', auth='CUSTOM') +conn = hive.Connection(host=kyuubi_host, port=10009, + username='user', password='password', auth='CUSTOM') +``` + +Use Kerberos to connect to Kyuubi. + +`kerberos_service_name` must be the name of the service that started the Kyuubi server, usually the prefix of the first slash of `kyuubi.kinit.principal`. + +Note that PyHive does not support passing in `principal`, it splices in part of `principal` with `kerberos_service_name` and `kyuubi_host`. + +```python +# open connection +conn = hive.Connection(host=kyuubi_host, port=10009, auth="KERBEROS", kerberos_service_name="kyuubi") +``` + +If you encounter the following errors, you need to install related packages. + +``` +thrift.transport.TTransport.TTransportException: Could not start SASL: b'Error in sasl_client_start (-4) SASL(-4): no mechanism available: No worthy mechs found' +``` + +```bash +yum install -y cyrus-sasl-plain cyrus-sasl-devel cyrus-sasl-gssapi cyrus-sasl-md5 +``` + +Note that PyHive does not support the connection method based on zookeeper HA, you can connect to zookeeper to get the service address via [Kazoo](https://pypi.org/project/kazoo/). + +Code reference [https://stackoverflow.com/a/73326589](https://stackoverflow.com/a/73326589) + +```python +from pyhive import hive +import random +from kazoo.client import KazooClient +zk = KazooClient(hosts='kyuubi1.xx.com:2181,kyuubi2.xx.com:2181,kyuubi3.xx.com:2181', read_only=True) +zk.start() +servers = [kyuubi_server.split(';')[0].split('=')[1].split(':') + for kyuubi_server + in zk.get_children(path='kyuubi')] +kyuubi_host, kyuubi_port = random.choice(servers) +zk.stop() +print(kyuubi_host, kyuubi_port) +conn = hive.Connection(host=kyuubi_host, port=kyuubi_port, auth="KERBEROS", kerberos_service_name="kyuubi") ``` diff --git a/docs/client/rest/rest_api.md b/docs/client/rest/rest_api.md index f863404a6..fc04857d0 100644 --- a/docs/client/rest/rest_api.md +++ b/docs/client/rest/rest_api.md @@ -89,19 +89,16 @@ Create a session #### Request Parameters -| Name | Description | Type | -|:----------------|:-----------------------------------------|:-------| -| protocolVersion | The protocol version of Hive CLI service | Int | -| user | The user name | String | -| password | The user password | String | -| ipAddr | The user client IP address | String | -| configs | The configuration of the session | Map | +| Name | Description | Type | +|:--------|:---------------------------------|:-----| +| configs | The configuration of the session | Map | #### Response Body -| Name | Description | Type | -|:-----------|:------------------------------|:-------| -| identifier | The session handle identifier | String | +| Name | Description | Type | +|:---------------|:---------------------------------------------------------------------------------------------------|:-------| +| identifier | The session handle identifier | String | +| kyuubiInstance | The Kyuubi instance that holds the session and to call for the following operations in the session | String | ### DELETE /sessions/${sessionHandle} @@ -113,11 +110,12 @@ Create an operation with EXECUTE_STATEMENT type #### Request Body -| Name | Description | Type | -|:-------------|:---------------------------------------------------------------|:--------| -| statement | The SQL statement that you execute | String | -| runAsync | The flag indicates whether the query runs synchronously or not | Boolean | -| queryTimeout | The interval of query time out | Long | +| Name | Description | Type | +|:-------------|:---------------------------------------------------------------|:---------------| +| statement | The SQL statement that you execute | String | +| runAsync | The flag indicates whether the query runs synchronously or not | Boolean | +| queryTimeout | The interval of query time out | Long | +| confOverlay | The conf to overlay only for current operation | Map of key=val | #### Response Body @@ -400,7 +398,7 @@ curl --location --request POST 'http://localhost:10099/api/v1/batches' \ The created [Batch](#batch) object. -### GET /batches/{batchId} +### GET /batches/${batchId} Returns the batch information. @@ -451,7 +449,13 @@ Refresh the Hadoop configurations of the Kyuubi server. ### POST /admin/refresh/user_defaults_conf -Refresh the [user defaults configs](../../deployment/settings.html#user-defaults) with key in format in the form of `___{username}___.{config key}` from default property file. +Refresh the [user defaults configs](../../configuration/settings.html#user-defaults) with key in format in the form of `___{username}___.{config key}` from default property file. + +### POST /admin/refresh/kubernetes_conf + +Refresh the kubernetes configs with key prefixed with `kyuubi.kubernetes` from default property file. + +It is helpful if you need to support multiple kubernetes contexts and namespaces, see [KYUUBI #4843](https://github.com/apache/kyuubi/issues/4843). ### DELETE /admin/engine @@ -493,6 +497,7 @@ The [Engine](#engine) List. | user | The user created the batch | String | | batchType | The batch type | String | | name | The batch name | String | +| appStartTime | The batch application start time | Long | | appId | The batch application Id | String | | appUrl | The batch application tracking url | String | | appState | The batch application state | String | diff --git a/docs/community/release.md b/docs/community/release.md index 5d3a00b03..f2c8541b1 100644 --- a/docs/community/release.md +++ b/docs/community/release.md @@ -43,12 +43,14 @@ The release process consists of several steps: 1. Decide to release 2. Prepare for the release -3. Cut branch off for __major__ release +3. Cut branch off for __feature__ release 4. Build a release candidate 5. Vote on the release candidate 6. If necessary, fix any issues and go back to step 3. 7. Finalize the release 8. Promote the release +9. Remove the dist repo directories for deprecated release candidates +10. Publish docker image ## Decide to release @@ -151,12 +153,12 @@ gpg --keyserver hkp://keyserver.ubuntu.com --send-keys ${PUBLIC_KEY} # send publ gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys ${PUBLIC_KEY} # verify ``` -## Cut branch if for major release +## Cut branch if for feature release Kyuubi use version pattern `{MAJOR_VERSION}.{MINOR_VERSION}.{PATCH_VERSION}[-{OPTIONAL_SUFFIX}]`, e.g. `1.7.0`. -__Major Release__ means `MAJOR_VERSION` or `MINOR_VERSION` changed, and __Patch Release__ means `PATCH_VERSION` changed. +__Feature Release__ means `MAJOR_VERSION` or `MINOR_VERSION` changed, and __Patch Release__ means `PATCH_VERSION` changed. -The main step towards preparing a major release is to create a release branch. This is done via standard Git branching +The main step towards preparing a feature release is to create a release branch. This is done via standard Git branching mechanism and should be announced to the community once the branch is created. > Note: If you are releasing a patch version, you can ignore this step. @@ -169,29 +171,49 @@ After cutting release branch, don't forget bump version in `master` branch. > Don't forget to switch to the release branch! -1. Set environment variables. +- Set environment variables. ```shell export RELEASE_VERSION= export RELEASE_RC_NO= +export NEXT_VERSION= ``` -2. Bump version. +- Bump version, and create a git tag for the release candidate. + +Considering that other committers may merge PRs during your release period, you should accomplish the version change +first, and then come back to the release candidate tag to continue the rest release process. + +The tag pattern is `v${RELEASE_VERSION}-rc${RELEASE_RC_NO}`, e.g. `v1.7.0-rc0` + +> NOTE: After all the voting passed, be sure to create a final tag with the pattern: `v${RELEASE_VERSION}` ```shell +# Bump to the release version build/mvn versions:set -DgenerateBackupPoms=false -DnewVersion="${RELEASE_VERSION}" - +(cd kyuubi-server/web-ui && npm version "${RELEASE_VERSION}") git commit -am "[RELEASE] Bump ${RELEASE_VERSION}" -``` -3. Create a git tag for the release candidate. +# Create tag +git tag v${RELEASE_VERSION}-rc${RELEASE_RC_NO} -The tag pattern is `v${RELEASE_VERSION}-rc${RELEASE_RC_NO}`, e.g. `v1.7.0-rc0` +# Prepare for the next development version +build/mvn versions:set -DgenerateBackupPoms=false -DnewVersion="${NEXT_VERSION}-SNAPSHOT" +(cd kyuubi-server/web-ui && npm version "${NEXT_VERSION}-SNAPSHOT") +git commit -am "[RELEASE] Bump ${NEXT_VERSION}-SNAPSHOT" -> NOTE: After all the voting passed, be sure to create a final tag with the pattern: `v${RELEASE_VERSION}` +# Push branch to apache remote repo +git push apache -4. Package the release binaries & sources, and upload them to the Apache staging SVN repo. Publish jars to the Apache - staging Maven repo. +# Push tag to apache remote repo +git push apache v${RELEASE_VERSION}-rc${RELEASE_RC_NO} + +# Go back to release candidate tag +git checkout v${RELEASE_VERSION}-rc${RELEASE_RC_NO} +``` + +- Package source and binary artifacts, and upload them to the Apache staging SVN repo. Publish jars to the Apache + staging Maven repo. ```shell build/release/release.sh publish @@ -199,7 +221,7 @@ build/release/release.sh publish To make your release available in the staging repository, you must close the staging repo in the [Apache Nexus](https://repository.apache.org/#stagingRepositories). Until you close, you can re-run deploying to staging multiple times. But once closed, it will create a new staging repo. So ensure you close this, so that the next RC (if need be) is on a new repo. Once everything is good, close the staging repository on Apache Nexus. -5. Generate a pre-release note from GitHub for the subsequent voting. +- Generate a pre-release note from GitHub for the subsequent voting. Goto the [release page](https://github.com/apache/kyuubi/releases) and click the "Draft a new release" button, then it would jump to a new page to prepare the release. @@ -255,8 +277,7 @@ Fork and clone [Apache Kyuubi website](https://github.com/apache/kyuubi-website) 1. Add a new markdown file in `src/zh/news/`, `src/en/news/` 2. Add a new markdown file in `src/zh/release/`, `src/en/release/` -3. Follow [Build Document](../develop_tools/build_document.md) to build documents, then copy `apache/kyuubi`'s - folder `docs/_build/html` to `apache/kyuubi-website`'s folder `content/docs/r{RELEASE_VERSION}` +3. Update `releases` defined in `hugo.toml`'s `[params]` part. ### Create an Announcement @@ -280,3 +301,9 @@ svn delete https://dist.apache.org/repos/dist/dev/kyuubi/{RELEASE_TAG} \ --message "Remove deprecated Apache Kyuubi ${RELEASE_TAG}" ``` +## Keep other artifacts up-to-date + +- Docker Image: https://github.com/apache/kyuubi-docker/blob/master/release/release_guide.md +- Helm Charts: https://github.com/apache/kyuubi/blob/master/charts/kyuubi/Chart.yaml +- Playground: https://github.com/apache/kyuubi/blob/master/docker/playground/.env + diff --git a/docs/conf.py b/docs/conf.py index 3df98c6e3..eaac1aced 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -64,7 +64,7 @@ author = 'Apache Kyuubi Community' # The full version, including alpha/beta/rc tags -release = subprocess.getoutput("cd .. && build/mvn help:evaluate -Dexpression=project.version|grep -v Using|grep -v INFO|grep -v WARNING|tail -n 1").split('\n')[-1] +release = subprocess.getoutput("grep 'kyuubi-parent' -C1 ../pom.xml | grep '' | awk -F '[<>]' '{print $3}'") # -- General configuration --------------------------------------------------- @@ -77,9 +77,11 @@ 'sphinx.ext.napoleon', 'sphinx.ext.mathjax', 'recommonmark', + 'sphinx_copybutton', 'sphinx_markdown_tables', 'sphinx_togglebutton', 'notfound.extension', + 'sphinxemoji.sphinxemoji', ] master_doc = 'index' diff --git a/docs/deployment/settings.md b/docs/configuration/settings.md similarity index 62% rename from docs/deployment/settings.md rename to docs/configuration/settings.md index f8beaa83b..5e00d0b75 100644 --- a/docs/deployment/settings.md +++ b/docs/configuration/settings.md @@ -16,151 +16,62 @@ --> -# Introduction to the Kyuubi Configurations System +# Configurations Kyuubi provides several ways to configure the system and corresponding engines. ## Environments -You can configure the environment variables in `$KYUUBI_HOME/conf/kyuubi-env.sh`, e.g, `JAVA_HOME`, then this java runtime will be used both for Kyuubi server instance and the applications it launches. You can also change the variable in the subprocess's env configuration file, e.g.`$SPARK_HOME/conf/spark-env.sh` to use more specific ENV for SQL engine applications. - -```bash -#!/usr/bin/env bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# -# - JAVA_HOME Java runtime to use. By default use "java" from PATH. -# -# -# - KYUUBI_CONF_DIR Directory containing the Kyuubi configurations to use. -# (Default: $KYUUBI_HOME/conf) -# - KYUUBI_LOG_DIR Directory for Kyuubi server-side logs. -# (Default: $KYUUBI_HOME/logs) -# - KYUUBI_PID_DIR Directory stores the Kyuubi instance pid file. -# (Default: $KYUUBI_HOME/pid) -# - KYUUBI_MAX_LOG_FILES Maximum number of Kyuubi server logs can rotate to. -# (Default: 5) -# - KYUUBI_JAVA_OPTS JVM options for the Kyuubi server itself in the form "-Dx=y". -# (Default: none). -# - KYUUBI_CTL_JAVA_OPTS JVM options for the Kyuubi ctl itself in the form "-Dx=y". -# (Default: none). -# - KYUUBI_BEELINE_OPTS JVM options for the Kyuubi BeeLine in the form "-Dx=Y". -# (Default: none) -# - KYUUBI_NICENESS The scheduling priority for Kyuubi server. -# (Default: 0) -# - KYUUBI_WORK_DIR_ROOT Root directory for launching sql engine applications. -# (Default: $KYUUBI_HOME/work) -# - HADOOP_CONF_DIR Directory containing the Hadoop / YARN configuration to use. -# - YARN_CONF_DIR Directory containing the YARN configuration to use. -# -# - SPARK_HOME Spark distribution which you would like to use in Kyuubi. -# - SPARK_CONF_DIR Optional directory where the Spark configuration lives. -# (Default: $SPARK_HOME/conf) -# - FLINK_HOME Flink distribution which you would like to use in Kyuubi. -# - FLINK_CONF_DIR Optional directory where the Flink configuration lives. -# (Default: $FLINK_HOME/conf) -# - FLINK_HADOOP_CLASSPATH Required Hadoop jars when you use the Kyuubi Flink engine. -# - HIVE_HOME Hive distribution which you would like to use in Kyuubi. -# - HIVE_CONF_DIR Optional directory where the Hive configuration lives. -# (Default: $HIVE_HOME/conf) -# - HIVE_HADOOP_CLASSPATH Required Hadoop jars when you use the Kyuubi Hive engine. -# - - -## Examples ## - -# export JAVA_HOME=/usr/jdk64/jdk1.8.0_152 -# export SPARK_HOME=/opt/spark -# export FLINK_HOME=/opt/flink -# export HIVE_HOME=/opt/hive -# export FLINK_HADOOP_CLASSPATH=/path/to/hadoop-client-runtime-3.3.2.jar:/path/to/hadoop-client-api-3.3.2.jar -# export HIVE_HADOOP_CLASSPATH=${HADOOP_HOME}/share/hadoop/common/lib/commons-collections-3.2.2.jar:${HADOOP_HOME}/share/hadoop/client/hadoop-client-runtime-3.1.0.jar:${HADOOP_HOME}/share/hadoop/client/hadoop-client-api-3.1.0.jar:${HADOOP_HOME}/share/hadoop/common/lib/htrace-core4-4.1.0-incubating.jar -# export HADOOP_CONF_DIR=/usr/ndp/current/mapreduce_client/conf -# export YARN_CONF_DIR=/usr/ndp/current/yarn/conf -# export KYUUBI_JAVA_OPTS="-Xmx10g -XX:+UnlockDiagnosticVMOptions -XX:ParGCCardsPerStrideChunk=4096 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -XX:+CMSClassUnloadingEnabled -XX:+CMSParallelRemarkEnabled -XX:+UseCondCardMark -XX:MaxDirectMemorySize=1024m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=./logs -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintTenuringDistribution -Xloggc:./logs/kyuubi-server-gc-%t.log -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=5M -XX:NewRatio=3 -XX:MetaspaceSize=512m" -# export KYUUBI_BEELINE_OPTS="-Xmx2g -XX:+UnlockDiagnosticVMOptions -XX:ParGCCardsPerStrideChunk=4096 -XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSConcurrentMTEnabled -XX:CMSInitiatingOccupancyFraction=70 -XX:+UseCMSInitiatingOccupancyOnly -XX:+CMSClassUnloadingEnabled -XX:+CMSParallelRemarkEnabled -XX:+UseCondCardMark" -``` - +You can configure the environment variables in `$KYUUBI_HOME/conf/kyuubi-env.sh`, e.g, `JAVA_HOME`, then this java runtime will be used both for Kyuubi server instance and the applications it launches. You can also change the variable in the subprocess's env configuration file, e.g.`$SPARK_HOME/conf/spark-env.sh` to use more specific ENV for SQL engine applications. see `$KYUUBI_HOME/conf/kyuubi-env.sh.template` as an example. For the environment variables that only needed to be transferred into engine side, you can set it with a Kyuubi configuration item formatted `kyuubi.engineEnv.VAR_NAME`. For example, with `kyuubi.engineEnv.SPARK_DRIVER_MEMORY=4g`, the environment variable `SPARK_DRIVER_MEMORY` with value `4g` would be transferred into engine side. With `kyuubi.engineEnv.SPARK_CONF_DIR=/apache/confs/spark/conf`, the value of `SPARK_CONF_DIR` on the engine side is set to `/apache/confs/spark/conf`. ## Kyuubi Configurations -You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.conf`. For example: - -```bash -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -## Kyuubi Configurations - -# -# kyuubi.authentication NONE -# kyuubi.frontend.bind.host localhost -# kyuubi.frontend.bind.port 10009 -# - -# Details in https://kyuubi.readthedocs.io/en/master/deployment/settings.html -``` +You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.conf`, see `$KYUUBI_HOME/conf/kyuubi-defaults.conf.template` as an example. ### Authentication -| Key | Default | Meaning | Type | Since | -|-----------------------------------------|-------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------|-------| -| kyuubi.authentication | NONE | A comma-separated list of client authentication types.
  • NOSASL: raw transport.
  • NONE: no authentication check.
  • KERBEROS: Kerberos/GSSAPI authentication.
  • CUSTOM: User-defined authentication.
  • JDBC: JDBC query authentication.
  • LDAP: Lightweight Directory Access Protocol authentication.
The following tree describes the catalog of each option.
  • NOSASL
  • SASL
    • SASL/PLAIN
      • NONE
      • LDAP
      • JDBC
      • CUSTOM
    • SASL/GSSAPI
      • KERBEROS
Note that: for SASL authentication, KERBEROS and PLAIN auth types are supported at the same time, and only the first specified PLAIN auth type is valid. | seq | 1.0.0 | -| kyuubi.authentication.custom.class | <undefined> | User-defined authentication implementation of org.apache.kyuubi.service.authentication.PasswdAuthenticationProvider | string | 1.3.0 | -| kyuubi.authentication.jdbc.driver.class | <undefined> | Driver class name for JDBC Authentication Provider. | string | 1.6.0 | -| kyuubi.authentication.jdbc.password | <undefined> | Database password for JDBC Authentication Provider. | string | 1.6.0 | -| kyuubi.authentication.jdbc.query | <undefined> | Query SQL template with placeholders for JDBC Authentication Provider to execute. Authentication passes if the result set is not empty.The SQL statement must start with the `SELECT` clause. Available placeholders are `${user}` and `${password}`. | string | 1.6.0 | -| kyuubi.authentication.jdbc.url | <undefined> | JDBC URL for JDBC Authentication Provider. | string | 1.6.0 | -| kyuubi.authentication.jdbc.user | <undefined> | Database user for JDBC Authentication Provider. | string | 1.6.0 | -| kyuubi.authentication.ldap.base.dn | <undefined> | LDAP base DN. | string | 1.0.0 | -| kyuubi.authentication.ldap.domain | <undefined> | LDAP domain. | string | 1.0.0 | -| kyuubi.authentication.ldap.guidKey | uid | LDAP attribute name whose values are unique in this LDAP server.For example:uid or cn. | string | 1.2.0 | -| kyuubi.authentication.ldap.url | <undefined> | SPACE character separated LDAP connection URL(s). | string | 1.0.0 | -| kyuubi.authentication.sasl.qop | auth | Sasl QOP enable higher levels of protection for Kyuubi communication with clients.
  • auth - authentication only (default)
  • auth-int - authentication plus integrity protection
  • auth-conf - authentication plus integrity and confidentiality protection. This is applicable only if Kyuubi is configured to use Kerberos authentication.
| string | 1.0.0 | +| Key | Default | Meaning | Type | Since | +|-----------------------------------------------|-------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------|-------| +| kyuubi.authentication | NONE | A comma-separated list of client authentication types.
  • NOSASL: raw transport.
  • NONE: no authentication check.
  • KERBEROS: Kerberos/GSSAPI authentication.
  • CUSTOM: User-defined authentication.
  • JDBC: JDBC query authentication.
  • LDAP: Lightweight Directory Access Protocol authentication.
The following tree describes the catalog of each option.
  • NOSASL
  • SASL
    • SASL/PLAIN
      • NONE
      • LDAP
      • JDBC
      • CUSTOM
    • SASL/GSSAPI
      • KERBEROS
Note that: for SASL authentication, KERBEROS and PLAIN auth types are supported at the same time, and only the first specified PLAIN auth type is valid. | set | 1.0.0 | +| kyuubi.authentication.custom.class | <undefined> | User-defined authentication implementation of org.apache.kyuubi.service.authentication.PasswdAuthenticationProvider | string | 1.3.0 | +| kyuubi.authentication.jdbc.driver.class | <undefined> | Driver class name for JDBC Authentication Provider. | string | 1.6.0 | +| kyuubi.authentication.jdbc.password | <undefined> | Database password for JDBC Authentication Provider. | string | 1.6.0 | +| kyuubi.authentication.jdbc.query | <undefined> | Query SQL template with placeholders for JDBC Authentication Provider to execute. Authentication passes if the result set is not empty.The SQL statement must start with the `SELECT` clause. Available placeholders are `${user}` and `${password}`. | string | 1.6.0 | +| kyuubi.authentication.jdbc.url | <undefined> | JDBC URL for JDBC Authentication Provider. | string | 1.6.0 | +| kyuubi.authentication.jdbc.user | <undefined> | Database user for JDBC Authentication Provider. | string | 1.6.0 | +| kyuubi.authentication.ldap.baseDN | <undefined> | LDAP base DN. | string | 1.7.0 | +| kyuubi.authentication.ldap.binddn | <undefined> | The user with which to bind to the LDAP server, and search for the full domain name of the user being authenticated. This should be the full domain name of the user, and should have search access across all users in the LDAP tree. If not specified, then the user being authenticated will be used as the bind user. For example: CN=bindUser,CN=Users,DC=subdomain,DC=domain,DC=com | string | 1.7.0 | +| kyuubi.authentication.ldap.bindpw | <undefined> | The password for the bind user, to be used to search for the full name of the user being authenticated. If the username is specified, this parameter must also be specified. | string | 1.7.0 | +| kyuubi.authentication.ldap.customLDAPQuery | <undefined> | A full LDAP query that LDAP Atn provider uses to execute against LDAP Server. If this query returns a null resultset, the LDAP Provider fails the Authentication request, succeeds if the user is part of the resultset.For example: `(&(objectClass=group)(objectClass=top)(instanceType=4)(cn=Domain*))`, `(&(objectClass=person)(|(sAMAccountName=admin)(|(memberOf=CN=Domain Admins,CN=Users,DC=domain,DC=com)(memberOf=CN=Administrators,CN=Builtin,DC=domain,DC=com))))` | string | 1.7.0 | +| kyuubi.authentication.ldap.domain | <undefined> | LDAP domain. | string | 1.0.0 | +| kyuubi.authentication.ldap.groupClassKey | groupOfNames | LDAP attribute name on the group entry that is to be used in LDAP group searches. For example: group, groupOfNames or groupOfUniqueNames. | string | 1.7.0 | +| kyuubi.authentication.ldap.groupDNPattern | <undefined> | COLON-separated list of patterns to use to find DNs for group entities in this directory. Use %s where the actual group name is to be substituted for. For example: CN=%s,CN=Groups,DC=subdomain,DC=domain,DC=com. | string | 1.7.0 | +| kyuubi.authentication.ldap.groupFilter || COMMA-separated list of LDAP Group names (short name not full DNs). For example: HiveAdmins,HadoopAdmins,Administrators | set | 1.7.0 | +| kyuubi.authentication.ldap.groupMembershipKey | member | LDAP attribute name on the group object that contains the list of distinguished names for the user, group, and contact objects that are members of the group. For example: member, uniqueMember or memberUid | string | 1.7.0 | +| kyuubi.authentication.ldap.guidKey | uid | LDAP attribute name whose values are unique in this LDAP server. For example: uid or CN. | string | 1.2.0 | +| kyuubi.authentication.ldap.url | <undefined> | SPACE character separated LDAP connection URL(s). | string | 1.0.0 | +| kyuubi.authentication.ldap.userDNPattern | <undefined> | COLON-separated list of patterns to use to find DNs for users in this directory. Use %s where the actual group name is to be substituted for. For example: CN=%s,CN=Users,DC=subdomain,DC=domain,DC=com. | string | 1.7.0 | +| kyuubi.authentication.ldap.userFilter || COMMA-separated list of LDAP usernames (just short names, not full DNs). For example: hiveuser,impalauser,hiveadmin,hadoopadmin | set | 1.7.0 | +| kyuubi.authentication.ldap.userMembershipKey | <undefined> | LDAP attribute name on the user object that contains groups of which the user is a direct member, except for the primary group, which is represented by the primaryGroupId. For example: memberOf | string | 1.7.0 | +| kyuubi.authentication.sasl.qop | auth | Sasl QOP enable higher levels of protection for Kyuubi communication with clients.
  • auth - authentication only (default)
  • auth-int - authentication plus integrity protection
  • auth-conf - authentication plus integrity and confidentiality protection. This is applicable only if Kyuubi is configured to use Kerberos authentication.
| string | 1.0.0 | ### Backend -| Key | Default | Meaning | Type | Since | -|--------------------------------------------------|---------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|-------| -| kyuubi.backend.engine.exec.pool.keepalive.time | PT1M | Time(ms) that an idle async thread of the operation execution thread pool will wait for a new task to arrive before terminating in SQL engine applications | duration | 1.0.0 | -| kyuubi.backend.engine.exec.pool.shutdown.timeout | PT10S | Timeout(ms) for the operation execution thread pool to terminate in SQL engine applications | duration | 1.0.0 | -| kyuubi.backend.engine.exec.pool.size | 100 | Number of threads in the operation execution thread pool of SQL engine applications | int | 1.0.0 | -| kyuubi.backend.engine.exec.pool.wait.queue.size | 100 | Size of the wait queue for the operation execution thread pool in SQL engine applications | int | 1.0.0 | -| kyuubi.backend.server.event.json.log.path | file:///tmp/kyuubi/events | The location of server events go for the built-in JSON logger | string | 1.4.0 | -| kyuubi.backend.server.event.loggers || A comma-separated list of server history loggers, where session/operation etc events go.
  • JSON: the events will be written to the location of kyuubi.backend.server.event.json.log.path
  • JDBC: to be done
  • CUSTOM: User-defined event handlers.
Note that: Kyuubi supports custom event handlers with the Java SPI. To register a custom event handler, the user needs to implement a class which is a child of org.apache.kyuubi.events.handler.CustomEventHandlerProvider which has a zero-arg constructor. | seq | 1.4.0 | -| kyuubi.backend.server.exec.pool.keepalive.time | PT1M | Time(ms) that an idle async thread of the operation execution thread pool will wait for a new task to arrive before terminating in Kyuubi server | duration | 1.0.0 | -| kyuubi.backend.server.exec.pool.shutdown.timeout | PT10S | Timeout(ms) for the operation execution thread pool to terminate in Kyuubi server | duration | 1.0.0 | -| kyuubi.backend.server.exec.pool.size | 100 | Number of threads in the operation execution thread pool of Kyuubi server | int | 1.0.0 | -| kyuubi.backend.server.exec.pool.wait.queue.size | 100 | Size of the wait queue for the operation execution thread pool of Kyuubi server | int | 1.0.0 | +| Key | Default | Meaning | Type | Since | +|--------------------------------------------------|---------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|-------| +| kyuubi.backend.engine.exec.pool.keepalive.time | PT1M | Time(ms) that an idle async thread of the operation execution thread pool will wait for a new task to arrive before terminating in SQL engine applications | duration | 1.0.0 | +| kyuubi.backend.engine.exec.pool.shutdown.timeout | PT10S | Timeout(ms) for the operation execution thread pool to terminate in SQL engine applications | duration | 1.0.0 | +| kyuubi.backend.engine.exec.pool.size | 100 | Number of threads in the operation execution thread pool of SQL engine applications | int | 1.0.0 | +| kyuubi.backend.engine.exec.pool.wait.queue.size | 100 | Size of the wait queue for the operation execution thread pool in SQL engine applications | int | 1.0.0 | +| kyuubi.backend.server.event.json.log.path | file:///tmp/kyuubi/events | The location of server events go for the built-in JSON logger | string | 1.4.0 | +| kyuubi.backend.server.event.kafka.close.timeout | PT5S | Period to wait for Kafka producer of server event handlers to close. | duration | 1.8.0 | +| kyuubi.backend.server.event.kafka.topic | <undefined> | The topic of server events go for the built-in Kafka logger | string | 1.8.0 | +| kyuubi.backend.server.event.loggers || A comma-separated list of server history loggers, where session/operation etc events go.
  • JSON: the events will be written to the location of kyuubi.backend.server.event.json.log.path
  • KAFKA: the events will be serialized in JSON format and sent to topic of `kyuubi.backend.server.event.kafka.topic`. Note: For the configs of Kafka producer, please specify them with the prefix: `kyuubi.backend.server.event.kafka.`. For example, `kyuubi.backend.server.event.kafka.bootstrap.servers=127.0.0.1:9092`
  • JDBC: to be done
  • CUSTOM: User-defined event handlers.
Note that: Kyuubi supports custom event handlers with the Java SPI. To register a custom event handler, the user needs to implement a class which is a child of org.apache.kyuubi.events.handler.CustomEventHandlerProvider which has a zero-arg constructor. | seq | 1.4.0 | +| kyuubi.backend.server.exec.pool.keepalive.time | PT1M | Time(ms) that an idle async thread of the operation execution thread pool will wait for a new task to arrive before terminating in Kyuubi server | duration | 1.0.0 | +| kyuubi.backend.server.exec.pool.shutdown.timeout | PT10S | Timeout(ms) for the operation execution thread pool to terminate in Kyuubi server | duration | 1.0.0 | +| kyuubi.backend.server.exec.pool.size | 100 | Number of threads in the operation execution thread pool of Kyuubi server | int | 1.0.0 | +| kyuubi.backend.server.exec.pool.wait.queue.size | 100 | Size of the wait queue for the operation execution thread pool of Kyuubi server | int | 1.0.0 | ### Batch @@ -168,7 +79,7 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co |---------------------------------------------|---------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|-------| | kyuubi.batch.application.check.interval | PT5S | The interval to check batch job application information. | duration | 1.6.0 | | kyuubi.batch.application.starvation.timeout | PT3M | Threshold above which to warn batch application may be starved. | duration | 1.7.0 | -| kyuubi.batch.conf.ignore.list || A comma-separated list of ignored keys for batch conf. If the batch conf contains any of them, the key and the corresponding value will be removed silently during batch job submission. Note that this rule is for server-side protection defined via administrators to prevent some essential configs from tampering. You can also pre-define some config for batch job submission with the prefix: kyuubi.batchConf.[batchType]. For example, you can pre-define `spark.master` for the Spark batch job with key `kyuubi.batchConf.spark.spark.master`. | seq | 1.6.0 | +| kyuubi.batch.conf.ignore.list || A comma-separated list of ignored keys for batch conf. If the batch conf contains any of them, the key and the corresponding value will be removed silently during batch job submission. Note that this rule is for server-side protection defined via administrators to prevent some essential configs from tampering. You can also pre-define some config for batch job submission with the prefix: kyuubi.batchConf.[batchType]. For example, you can pre-define `spark.master` for the Spark batch job with key `kyuubi.batchConf.spark.spark.master`. | set | 1.6.0 | | kyuubi.batch.session.idle.timeout | PT6H | Batch session idle timeout, it will be closed when it's not accessed for this duration | duration | 1.6.2 | ### Credentials @@ -209,59 +120,82 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co ### Engine -| Key | Default | Meaning | Type | Since | -|----------------------------------------------------------|---------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|-------| -| kyuubi.engine.connection.url.use.hostname | true | (deprecated) When true, the engine registers with hostname to zookeeper. When Spark runs on K8s with cluster mode, set to false to ensure that server can connect to engine | boolean | 1.3.0 | -| kyuubi.engine.deregister.exception.classes || A comma-separated list of exception classes. If there is any exception thrown, whose class matches the specified classes, the engine would deregister itself. | seq | 1.2.0 | -| kyuubi.engine.deregister.exception.messages || A comma-separated list of exception messages. If there is any exception thrown, whose message or stacktrace matches the specified message list, the engine would deregister itself. | seq | 1.2.0 | -| kyuubi.engine.deregister.exception.ttl | PT30M | Time to live(TTL) for exceptions pattern specified in kyuubi.engine.deregister.exception.classes and kyuubi.engine.deregister.exception.messages to deregister engines. Once the total error count hits the kyuubi.engine.deregister.job.max.failures within the TTL, an engine will deregister itself and wait for self-terminated. Otherwise, we suppose that the engine has recovered from temporary failures. | duration | 1.2.0 | -| kyuubi.engine.deregister.job.max.failures | 4 | Number of failures of job before deregistering the engine. | int | 1.2.0 | -| kyuubi.engine.event.json.log.path | file:///tmp/kyuubi/events | The location where all the engine events go for the built-in JSON logger.
  • Local Path: start with 'file://'
  • HDFS Path: start with 'hdfs://'
| string | 1.3.0 | -| kyuubi.engine.event.loggers | SPARK | A comma-separated list of engine history loggers, where engine/session/operation etc events go.
  • SPARK: the events will be written to the Spark listener bus.
  • JSON: the events will be written to the location of kyuubi.engine.event.json.log.path
  • JDBC: to be done
  • CUSTOM: User-defined event handlers.
Note that: Kyuubi supports custom event handlers with the Java SPI. To register a custom event handler, the user needs to implement a subclass of `org.apache.kyuubi.events.handler.CustomEventHandlerProvider` which has a zero-arg constructor. | seq | 1.3.0 | -| kyuubi.engine.flink.extra.classpath | <undefined> | The extra classpath for the Flink SQL engine, for configuring the location of hadoop client jars, etc | string | 1.6.0 | -| kyuubi.engine.flink.java.options | <undefined> | The extra Java options for the Flink SQL engine | string | 1.6.0 | -| kyuubi.engine.flink.memory | 1g | The heap memory for the Flink SQL engine | string | 1.6.0 | -| kyuubi.engine.hive.event.loggers | JSON | A comma-separated list of engine history loggers, where engine/session/operation etc events go.
  • JSON: the events will be written to the location of kyuubi.engine.event.json.log.path
  • JDBC: to be done
  • CUSTOM: to be done.
| seq | 1.7.0 | -| kyuubi.engine.hive.extra.classpath | <undefined> | The extra classpath for the Hive query engine, for configuring location of the hadoop client jars and etc. | string | 1.6.0 | -| kyuubi.engine.hive.java.options | <undefined> | The extra Java options for the Hive query engine | string | 1.6.0 | -| kyuubi.engine.hive.memory | 1g | The heap memory for the Hive query engine | string | 1.6.0 | -| kyuubi.engine.initialize.sql | SHOW DATABASES | SemiColon-separated list of SQL statements to be initialized in the newly created engine before queries. i.e. use `SHOW DATABASES` to eagerly active HiveClient. This configuration can not be used in JDBC url due to the limitation of Beeline/JDBC driver. | seq | 1.2.0 | -| kyuubi.engine.jdbc.connection.password | <undefined> | The password is used for connecting to server | string | 1.6.0 | -| kyuubi.engine.jdbc.connection.properties || The additional properties are used for connecting to server | seq | 1.6.0 | -| kyuubi.engine.jdbc.connection.provider | <undefined> | The connection provider is used for getting a connection from the server | string | 1.6.0 | -| kyuubi.engine.jdbc.connection.url | <undefined> | The server url that engine will connect to | string | 1.6.0 | -| kyuubi.engine.jdbc.connection.user | <undefined> | The user is used for connecting to server | string | 1.6.0 | -| kyuubi.engine.jdbc.driver.class | <undefined> | The driver class for JDBC engine connection | string | 1.6.0 | -| kyuubi.engine.jdbc.extra.classpath | <undefined> | The extra classpath for the JDBC query engine, for configuring the location of the JDBC driver and etc. | string | 1.6.0 | -| kyuubi.engine.jdbc.java.options | <undefined> | The extra Java options for the JDBC query engine | string | 1.6.0 | -| kyuubi.engine.jdbc.memory | 1g | The heap memory for the JDBC query engine | string | 1.6.0 | -| kyuubi.engine.jdbc.type | <undefined> | The short name of JDBC type | string | 1.6.0 | -| kyuubi.engine.operation.convert.catalog.database.enabled | true | When set to true, The engine converts the JDBC methods of set/get Catalog and set/get Schema to the implementation of different engines | boolean | 1.6.0 | -| kyuubi.engine.operation.log.dir.root | engine_operation_logs | Root directory for query operation log at engine-side. | string | 1.4.0 | -| kyuubi.engine.pool.name | engine-pool | The name of the engine pool. | string | 1.5.0 | -| kyuubi.engine.pool.selectPolicy | RANDOM | The select policy of an engine from the corresponding engine pool engine for a session.
  • RANDOM - Randomly use the engine in the pool
  • POLLING - Polling use the engine in the pool
| string | 1.7.0 | -| kyuubi.engine.pool.size | -1 | The size of the engine pool. Note that, if the size is less than 1, the engine pool will not be enabled; otherwise, the size of the engine pool will be min(this, kyuubi.engine.pool.size.threshold). | int | 1.4.0 | -| kyuubi.engine.pool.size.threshold | 9 | This parameter is introduced as a server-side parameter controlling the upper limit of the engine pool. | int | 1.4.0 | -| kyuubi.engine.session.initialize.sql || SemiColon-separated list of SQL statements to be initialized in the newly created engine session before queries. This configuration can not be used in JDBC url due to the limitation of Beeline/JDBC driver. | seq | 1.3.0 | -| kyuubi.engine.share.level | USER | Engines will be shared in different levels, available configs are:
  • CONNECTION: engine will not be shared but only used by the current client connection
  • USER: engine will be shared by all sessions created by a unique username, see also kyuubi.engine.share.level.subdomain
  • GROUP: the engine will be shared by all sessions created by all users belong to the same primary group name. The engine will be launched by the group name as the effective username, so here the group name is in value of special user who is able to visit the computing resources/data of the team. It follows the [Hadoop GroupsMapping](https://reurl.cc/xE61Y5) to map user to a primary group. If the primary group is not found, it fallback to the USER level.
  • SERVER: the App will be shared by Kyuubi servers
| string | 1.2.0 | -| kyuubi.engine.share.level.sub.domain | <undefined> | (deprecated) - Using kyuubi.engine.share.level.subdomain instead | string | 1.2.0 | -| kyuubi.engine.share.level.subdomain | <undefined> | Allow end-users to create a subdomain for the share level of an engine. A subdomain is a case-insensitive string values that must be a valid zookeeper subpath. For example, for the `USER` share level, an end-user can share a certain engine within a subdomain, not for all of its clients. End-users are free to create multiple engines in the `USER` share level. When disable engine pool, use 'default' if absent. | string | 1.4.0 | -| kyuubi.engine.single.spark.session | false | When set to true, this engine is running in a single session mode. All the JDBC/ODBC connections share the temporary views, function registries, SQL configuration and the current database. | boolean | 1.3.0 | -| kyuubi.engine.spark.event.loggers | SPARK | A comma-separated list of engine loggers, where engine/session/operation etc events go.
  • SPARK: the events will be written to the Spark listener bus.
  • JSON: the events will be written to the location of kyuubi.engine.event.json.log.path
  • JDBC: to be done
  • CUSTOM: to be done.
| seq | 1.7.0 | -| kyuubi.engine.spark.python.env.archive | <undefined> | Portable Python env archive used for Spark engine Python language mode. | string | 1.7.0 | -| kyuubi.engine.spark.python.env.archive.exec.path | bin/python | The Python exec path under the Python env archive. | string | 1.7.0 | -| kyuubi.engine.spark.python.home.archive | <undefined> | Spark archive containing $SPARK_HOME/python directory, which is used to init session Python worker for Python language mode. | string | 1.7.0 | -| kyuubi.engine.trino.event.loggers | JSON | A comma-separated list of engine history loggers, where engine/session/operation etc events go.
  • JSON: the events will be written to the location of kyuubi.engine.event.json.log.path
  • JDBC: to be done
  • CUSTOM: to be done.
| seq | 1.7.0 | -| kyuubi.engine.trino.extra.classpath | <undefined> | The extra classpath for the Trino query engine, for configuring other libs which may need by the Trino engine | string | 1.6.0 | -| kyuubi.engine.trino.java.options | <undefined> | The extra Java options for the Trino query engine | string | 1.6.0 | -| kyuubi.engine.trino.memory | 1g | The heap memory for the Trino query engine | string | 1.6.0 | -| kyuubi.engine.type | SPARK_SQL | Specify the detailed engine supported by Kyuubi. The engine type bindings to SESSION scope. This configuration is experimental. Currently, available configs are:
  • SPARK_SQL: specify this engine type will launch a Spark engine which can provide all the capacity of the Apache Spark. Note, it's a default engine type.
  • FLINK_SQL: specify this engine type will launch a Flink engine which can provide all the capacity of the Apache Flink.
  • TRINO: specify this engine type will launch a Trino engine which can provide all the capacity of the Trino.
  • HIVE_SQL: specify this engine type will launch a Hive engine which can provide all the capacity of the Hive Server2.
  • JDBC: specify this engine type will launch a JDBC engine which can provide a MySQL protocol connector, for now we only support Doris dialect.
| string | 1.4.0 | -| kyuubi.engine.ui.retainedSessions | 200 | The number of SQL client sessions kept in the Kyuubi Query Engine web UI. | int | 1.4.0 | -| kyuubi.engine.ui.retainedStatements | 200 | The number of statements kept in the Kyuubi Query Engine web UI. | int | 1.4.0 | -| kyuubi.engine.ui.stop.enabled | true | When true, allows Kyuubi engine to be killed from the Spark Web UI. | boolean | 1.3.0 | -| kyuubi.engine.user.isolated.spark.session | true | When set to false, if the engine is running in a group or server share level, all the JDBC/ODBC connections will be isolated against the user. Including the temporary views, function registries, SQL configuration, and the current database. Note that, it does not affect if the share level is connection or user. | boolean | 1.6.0 | -| kyuubi.engine.user.isolated.spark.session.idle.interval | PT1M | The interval to check if the user-isolated Spark session is timeout. | duration | 1.6.0 | -| kyuubi.engine.user.isolated.spark.session.idle.timeout | PT6H | If kyuubi.engine.user.isolated.spark.session is false, we will release the Spark session if its corresponding user is inactive after this configured timeout. | duration | 1.6.0 | +| Key | Default | Meaning | Type | Since | +|----------------------------------------------------------|---------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|-------| +| kyuubi.engine.chat.extra.classpath | <undefined> | The extra classpath for the Chat engine, for configuring the location of the SDK and etc. | string | 1.8.0 | +| kyuubi.engine.chat.gpt.apiKey | <undefined> | The key to access OpenAI open API, which could be got at https://platform.openai.com/account/api-keys | string | 1.8.0 | +| kyuubi.engine.chat.gpt.http.connect.timeout | PT2M | The timeout[ms] for establishing the connection with the Chat GPT server. A timeout value of zero is interpreted as an infinite timeout. | duration | 1.8.0 | +| kyuubi.engine.chat.gpt.http.proxy | <undefined> | HTTP proxy url for API calling in Chat GPT engine. e.g. http://127.0.0.1:1087 | string | 1.8.0 | +| kyuubi.engine.chat.gpt.http.socket.timeout | PT2M | The timeout[ms] for waiting for data packets after Chat GPT server connection is established. A timeout value of zero is interpreted as an infinite timeout. | duration | 1.8.0 | +| kyuubi.engine.chat.gpt.model | gpt-3.5-turbo | ID of the model used in ChatGPT. Available models refer to OpenAI's [Model overview](https://platform.openai.com/docs/models/overview). | string | 1.8.0 | +| kyuubi.engine.chat.java.options | <undefined> | The extra Java options for the Chat engine | string | 1.8.0 | +| kyuubi.engine.chat.memory | 1g | The heap memory for the Chat engine | string | 1.8.0 | +| kyuubi.engine.chat.provider | ECHO | The provider for the Chat engine. Candidates:
  • ECHO: simply replies a welcome message.
  • GPT: a.k.a ChatGPT, powered by OpenAI.
| string | 1.8.0 | +| kyuubi.engine.connection.url.use.hostname | true | (deprecated) When true, the engine registers with hostname to zookeeper. When Spark runs on K8s with cluster mode, set to false to ensure that server can connect to engine | boolean | 1.3.0 | +| kyuubi.engine.deregister.exception.classes || A comma-separated list of exception classes. If there is any exception thrown, whose class matches the specified classes, the engine would deregister itself. | set | 1.2.0 | +| kyuubi.engine.deregister.exception.messages || A comma-separated list of exception messages. If there is any exception thrown, whose message or stacktrace matches the specified message list, the engine would deregister itself. | set | 1.2.0 | +| kyuubi.engine.deregister.exception.ttl | PT30M | Time to live(TTL) for exceptions pattern specified in kyuubi.engine.deregister.exception.classes and kyuubi.engine.deregister.exception.messages to deregister engines. Once the total error count hits the kyuubi.engine.deregister.job.max.failures within the TTL, an engine will deregister itself and wait for self-terminated. Otherwise, we suppose that the engine has recovered from temporary failures. | duration | 1.2.0 | +| kyuubi.engine.deregister.job.max.failures | 4 | Number of failures of job before deregistering the engine. | int | 1.2.0 | +| kyuubi.engine.event.json.log.path | file:///tmp/kyuubi/events | The location where all the engine events go for the built-in JSON logger.
  • Local Path: start with 'file://'
  • HDFS Path: start with 'hdfs://'
| string | 1.3.0 | +| kyuubi.engine.event.loggers | SPARK | A comma-separated list of engine history loggers, where engine/session/operation etc events go.
  • SPARK: the events will be written to the Spark listener bus.
  • JSON: the events will be written to the location of kyuubi.engine.event.json.log.path
  • JDBC: to be done
  • CUSTOM: User-defined event handlers.
Note that: Kyuubi supports custom event handlers with the Java SPI. To register a custom event handler, the user needs to implement a subclass of `org.apache.kyuubi.events.handler.CustomEventHandlerProvider` which has a zero-arg constructor. | seq | 1.3.0 | +| kyuubi.engine.flink.application.jars | <undefined> | A comma-separated list of the local jars to be shipped with the job to the cluster. For example, SQL UDF jars. Only effective in yarn application mode. | string | 1.8.0 | +| kyuubi.engine.flink.extra.classpath | <undefined> | The extra classpath for the Flink SQL engine, for configuring the location of hadoop client jars, etc. Only effective in yarn session mode. | string | 1.6.0 | +| kyuubi.engine.flink.java.options | <undefined> | The extra Java options for the Flink SQL engine. Only effective in yarn session mode. | string | 1.6.0 | +| kyuubi.engine.flink.memory | 1g | The heap memory for the Flink SQL engine. Only effective in yarn session mode. | string | 1.6.0 | +| kyuubi.engine.hive.event.loggers | JSON | A comma-separated list of engine history loggers, where engine/session/operation etc events go.
  • JSON: the events will be written to the location of kyuubi.engine.event.json.log.path
  • JDBC: to be done
  • CUSTOM: to be done.
| seq | 1.7.0 | +| kyuubi.engine.hive.extra.classpath | <undefined> | The extra classpath for the Hive query engine, for configuring location of the hadoop client jars and etc. | string | 1.6.0 | +| kyuubi.engine.hive.java.options | <undefined> | The extra Java options for the Hive query engine | string | 1.6.0 | +| kyuubi.engine.hive.memory | 1g | The heap memory for the Hive query engine | string | 1.6.0 | +| kyuubi.engine.initialize.sql | SHOW DATABASES | SemiColon-separated list of SQL statements to be initialized in the newly created engine before queries. i.e. use `SHOW DATABASES` to eagerly active HiveClient. This configuration can not be used in JDBC url due to the limitation of Beeline/JDBC driver. | seq | 1.2.0 | +| kyuubi.engine.jdbc.connection.password | <undefined> | The password is used for connecting to server | string | 1.6.0 | +| kyuubi.engine.jdbc.connection.propagateCredential | false | Whether to use the session's user and password to connect to database | boolean | 1.8.0 | +| kyuubi.engine.jdbc.connection.properties || The additional properties are used for connecting to server | seq | 1.6.0 | +| kyuubi.engine.jdbc.connection.provider | <undefined> | The connection provider is used for getting a connection from the server | string | 1.6.0 | +| kyuubi.engine.jdbc.connection.url | <undefined> | The server url that engine will connect to | string | 1.6.0 | +| kyuubi.engine.jdbc.connection.user | <undefined> | The user is used for connecting to server | string | 1.6.0 | +| kyuubi.engine.jdbc.driver.class | <undefined> | The driver class for JDBC engine connection | string | 1.6.0 | +| kyuubi.engine.jdbc.extra.classpath | <undefined> | The extra classpath for the JDBC query engine, for configuring the location of the JDBC driver and etc. | string | 1.6.0 | +| kyuubi.engine.jdbc.initialize.sql | SELECT 1 | SemiColon-separated list of SQL statements to be initialized in the newly created engine before queries. i.e. use `SELECT 1` to eagerly active JDBCClient. | seq | 1.8.0 | +| kyuubi.engine.jdbc.java.options | <undefined> | The extra Java options for the JDBC query engine | string | 1.6.0 | +| kyuubi.engine.jdbc.memory | 1g | The heap memory for the JDBC query engine | string | 1.6.0 | +| kyuubi.engine.jdbc.session.initialize.sql || SemiColon-separated list of SQL statements to be initialized in the newly created engine session before queries. | seq | 1.8.0 | +| kyuubi.engine.jdbc.type | <undefined> | The short name of JDBC type | string | 1.6.0 | +| kyuubi.engine.kubernetes.submit.timeout | PT30S | The engine submit timeout for Kubernetes application. | duration | 1.7.2 | +| kyuubi.engine.operation.convert.catalog.database.enabled | true | When set to true, The engine converts the JDBC methods of set/get Catalog and set/get Schema to the implementation of different engines | boolean | 1.6.0 | +| kyuubi.engine.operation.log.dir.root | engine_operation_logs | Root directory for query operation log at engine-side. | string | 1.4.0 | +| kyuubi.engine.pool.name | engine-pool | The name of the engine pool. | string | 1.5.0 | +| kyuubi.engine.pool.selectPolicy | RANDOM | The select policy of an engine from the corresponding engine pool engine for a session.
  • RANDOM - Randomly use the engine in the pool
  • POLLING - Polling use the engine in the pool
| string | 1.7.0 | +| kyuubi.engine.pool.size | -1 | The size of the engine pool. Note that, if the size is less than 1, the engine pool will not be enabled; otherwise, the size of the engine pool will be min(this, kyuubi.engine.pool.size.threshold). | int | 1.4.0 | +| kyuubi.engine.pool.size.threshold | 9 | This parameter is introduced as a server-side parameter controlling the upper limit of the engine pool. | int | 1.4.0 | +| kyuubi.engine.session.initialize.sql || SemiColon-separated list of SQL statements to be initialized in the newly created engine session before queries. This configuration can not be used in JDBC url due to the limitation of Beeline/JDBC driver. | seq | 1.3.0 | +| kyuubi.engine.share.level | USER | Engines will be shared in different levels, available configs are:
  • CONNECTION: engine will not be shared but only used by the current client connection
  • USER: engine will be shared by all sessions created by a unique username, see also kyuubi.engine.share.level.subdomain
  • GROUP: the engine will be shared by all sessions created by all users belong to the same primary group name. The engine will be launched by the group name as the effective username, so here the group name is in value of special user who is able to visit the computing resources/data of the team. It follows the [Hadoop GroupsMapping](https://reurl.cc/xE61Y5) to map user to a primary group. If the primary group is not found, it fallback to the USER level.
  • SERVER: the App will be shared by Kyuubi servers
| string | 1.2.0 | +| kyuubi.engine.share.level.sub.domain | <undefined> | (deprecated) - Using kyuubi.engine.share.level.subdomain instead | string | 1.2.0 | +| kyuubi.engine.share.level.subdomain | <undefined> | Allow end-users to create a subdomain for the share level of an engine. A subdomain is a case-insensitive string values that must be a valid zookeeper subpath. For example, for the `USER` share level, an end-user can share a certain engine within a subdomain, not for all of its clients. End-users are free to create multiple engines in the `USER` share level. When disable engine pool, use 'default' if absent. | string | 1.4.0 | +| kyuubi.engine.single.spark.session | false | When set to true, this engine is running in a single session mode. All the JDBC/ODBC connections share the temporary views, function registries, SQL configuration and the current database. | boolean | 1.3.0 | +| kyuubi.engine.spark.event.loggers | SPARK | A comma-separated list of engine loggers, where engine/session/operation etc events go.
  • SPARK: the events will be written to the Spark listener bus.
  • JSON: the events will be written to the location of kyuubi.engine.event.json.log.path
  • JDBC: to be done
  • CUSTOM: to be done.
| seq | 1.7.0 | +| kyuubi.engine.spark.python.env.archive | <undefined> | Portable Python env archive used for Spark engine Python language mode. | string | 1.7.0 | +| kyuubi.engine.spark.python.env.archive.exec.path | bin/python | The Python exec path under the Python env archive. | string | 1.7.0 | +| kyuubi.engine.spark.python.home.archive | <undefined> | Spark archive containing $SPARK_HOME/python directory, which is used to init session Python worker for Python language mode. | string | 1.7.0 | +| kyuubi.engine.submit.timeout | PT30S | Period to tolerant Driver Pod ephemerally invisible after submitting. In some Resource Managers, e.g. K8s, the Driver Pod is not visible immediately after `spark-submit` is returned. | duration | 1.7.1 | +| kyuubi.engine.trino.connection.keystore.password | <undefined> | The keystore password used for connecting to trino cluster | string | 1.8.0 | +| kyuubi.engine.trino.connection.keystore.path | <undefined> | The keystore path used for connecting to trino cluster | string | 1.8.0 | +| kyuubi.engine.trino.connection.keystore.type | <undefined> | The keystore type used for connecting to trino cluster | string | 1.8.0 | +| kyuubi.engine.trino.connection.password | <undefined> | The password used for connecting to trino cluster | string | 1.8.0 | +| kyuubi.engine.trino.connection.truststore.password | <undefined> | The truststore password used for connecting to trino cluster | string | 1.8.0 | +| kyuubi.engine.trino.connection.truststore.path | <undefined> | The truststore path used for connecting to trino cluster | string | 1.8.0 | +| kyuubi.engine.trino.connection.truststore.type | <undefined> | The truststore type used for connecting to trino cluster | string | 1.8.0 | +| kyuubi.engine.trino.event.loggers | JSON | A comma-separated list of engine history loggers, where engine/session/operation etc events go.
  • JSON: the events will be written to the location of kyuubi.engine.event.json.log.path
  • JDBC: to be done
  • CUSTOM: to be done.
| seq | 1.7.0 | +| kyuubi.engine.trino.extra.classpath | <undefined> | The extra classpath for the Trino query engine, for configuring other libs which may need by the Trino engine | string | 1.6.0 | +| kyuubi.engine.trino.java.options | <undefined> | The extra Java options for the Trino query engine | string | 1.6.0 | +| kyuubi.engine.trino.memory | 1g | The heap memory for the Trino query engine | string | 1.6.0 | +| kyuubi.engine.type | SPARK_SQL | Specify the detailed engine supported by Kyuubi. The engine type bindings to SESSION scope. This configuration is experimental. Currently, available configs are:
  • SPARK_SQL: specify this engine type will launch a Spark engine which can provide all the capacity of the Apache Spark. Note, it's a default engine type.
  • FLINK_SQL: specify this engine type will launch a Flink engine which can provide all the capacity of the Apache Flink.
  • TRINO: specify this engine type will launch a Trino engine which can provide all the capacity of the Trino.
  • HIVE_SQL: specify this engine type will launch a Hive engine which can provide all the capacity of the Hive Server2.
  • JDBC: specify this engine type will launch a JDBC engine which can forward queries to the database system through the certain JDBC driver, for now, it supports Doris and Phoenix.
  • CHAT: specify this engine type will launch a Chat engine.
| string | 1.4.0 | +| kyuubi.engine.ui.retainedSessions | 200 | The number of SQL client sessions kept in the Kyuubi Query Engine web UI. | int | 1.4.0 | +| kyuubi.engine.ui.retainedStatements | 200 | The number of statements kept in the Kyuubi Query Engine web UI. | int | 1.4.0 | +| kyuubi.engine.ui.stop.enabled | true | When true, allows Kyuubi engine to be killed from the Spark Web UI. | boolean | 1.3.0 | +| kyuubi.engine.user.isolated.spark.session | true | When set to false, if the engine is running in a group or server share level, all the JDBC/ODBC connections will be isolated against the user. Including the temporary views, function registries, SQL configuration, and the current database. Note that, it does not affect if the share level is connection or user. | boolean | 1.6.0 | +| kyuubi.engine.user.isolated.spark.session.idle.interval | PT1M | The interval to check if the user-isolated Spark session is timeout. | duration | 1.6.0 | +| kyuubi.engine.user.isolated.spark.session.idle.timeout | PT6H | If kyuubi.engine.user.isolated.spark.session is false, we will release the Spark session if its corresponding user is inactive after this configured timeout. | duration | 1.6.0 | +| kyuubi.engine.yarn.submit.timeout | PT30S | The engine submit timeout for YARN application. | duration | 1.7.2 | ### Event @@ -273,94 +207,96 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co ### Frontend -| Key | Default | Meaning | Type | Since | -|--------------------------------------------------------|-------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|-------| -| kyuubi.frontend.backoff.slot.length | PT0.1S | (deprecated) Time to back off during login to the thrift frontend service. | duration | 1.0.0 | -| kyuubi.frontend.bind.host | <undefined> | Hostname or IP of the machine on which to run the frontend services. | string | 1.0.0 | -| kyuubi.frontend.bind.port | 10009 | (deprecated) Port of the machine on which to run the thrift frontend service via the binary protocol. | int | 1.0.0 | -| kyuubi.frontend.connection.url.use.hostname | true | When true, frontend services prefer hostname, otherwise, ip address. Note that, the default value is set to `false` when engine running on Kubernetes to prevent potential network issues. | boolean | 1.5.0 | -| kyuubi.frontend.login.timeout | PT20S | (deprecated) Timeout for Thrift clients during login to the thrift frontend service. | duration | 1.0.0 | -| kyuubi.frontend.max.message.size | 104857600 | (deprecated) Maximum message size in bytes a Kyuubi server will accept. | int | 1.0.0 | -| kyuubi.frontend.max.worker.threads | 999 | (deprecated) Maximum number of threads in the frontend worker thread pool for the thrift frontend service | int | 1.0.0 | -| kyuubi.frontend.min.worker.threads | 9 | (deprecated) Minimum number of threads in the frontend worker thread pool for the thrift frontend service | int | 1.0.0 | -| kyuubi.frontend.mysql.bind.host | <undefined> | Hostname or IP of the machine on which to run the MySQL frontend service. | string | 1.4.0 | -| kyuubi.frontend.mysql.bind.port | 3309 | Port of the machine on which to run the MySQL frontend service. | int | 1.4.0 | -| kyuubi.frontend.mysql.max.worker.threads | 999 | Maximum number of threads in the command execution thread pool for the MySQL frontend service | int | 1.4.0 | -| kyuubi.frontend.mysql.min.worker.threads | 9 | Minimum number of threads in the command execution thread pool for the MySQL frontend service | int | 1.4.0 | -| kyuubi.frontend.mysql.netty.worker.threads | <undefined> | Number of thread in the netty worker event loop of MySQL frontend service. Use min(cpu_cores, 8) in default. | int | 1.4.0 | -| kyuubi.frontend.mysql.worker.keepalive.time | PT1M | Time(ms) that an idle async thread of the command execution thread pool will wait for a new task to arrive before terminating in MySQL frontend service | duration | 1.4.0 | -| kyuubi.frontend.protocols | THRIFT_BINARY | A comma-separated list for all frontend protocols
  • THRIFT_BINARY - HiveServer2 compatible thrift binary protocol.
  • THRIFT_HTTP - HiveServer2 compatible thrift http protocol.
  • REST - Kyuubi defined REST API(experimental).
  • MYSQL - MySQL compatible text protocol(experimental).
  • TRINO - Trino compatible http protocol(experimental).
| seq | 1.4.0 | -| kyuubi.frontend.proxy.http.client.ip.header | X-Real-IP | The HTTP header to record the real client IP address. If your server is behind a load balancer or other proxy, the server will see this load balancer or proxy IP address as the client IP address, to get around this common issue, most load balancers or proxies offer the ability to record the real remote IP address in an HTTP header that will be added to the request for other devices to use. Note that, because the header value can be specified to any IP address, so it will not be used for authentication. | string | 1.6.0 | -| kyuubi.frontend.rest.bind.host | <undefined> | Hostname or IP of the machine on which to run the REST frontend service. | string | 1.4.0 | -| kyuubi.frontend.rest.bind.port | 10099 | Port of the machine on which to run the REST frontend service. | int | 1.4.0 | -| kyuubi.frontend.rest.max.worker.threads | 999 | Maximum number of threads in the frontend worker thread pool for the rest frontend service | int | 1.6.2 | -| kyuubi.frontend.ssl.keystore.algorithm | <undefined> | SSL certificate keystore algorithm. | string | 1.7.0 | -| kyuubi.frontend.ssl.keystore.password | <undefined> | SSL certificate keystore password. | string | 1.7.0 | -| kyuubi.frontend.ssl.keystore.path | <undefined> | SSL certificate keystore location. | string | 1.7.0 | -| kyuubi.frontend.ssl.keystore.type | <undefined> | SSL certificate keystore type. | string | 1.7.0 | -| kyuubi.frontend.thrift.backoff.slot.length | PT0.1S | Time to back off during login to the thrift frontend service. | duration | 1.4.0 | -| kyuubi.frontend.thrift.binary.bind.host | <undefined> | Hostname or IP of the machine on which to run the thrift frontend service via the binary protocol. | string | 1.4.0 | -| kyuubi.frontend.thrift.binary.bind.port | 10009 | Port of the machine on which to run the thrift frontend service via the binary protocol. | int | 1.4.0 | -| kyuubi.frontend.thrift.binary.ssl.disallowed.protocols | SSLv2,SSLv3 | SSL versions to disallow for Kyuubi thrift binary frontend. | seq | 1.7.0 | -| kyuubi.frontend.thrift.binary.ssl.enabled | false | Set this to true for using SSL encryption in thrift binary frontend server. | boolean | 1.7.0 | -| kyuubi.frontend.thrift.binary.ssl.include.ciphersuites || A comma-separated list of include SSL cipher suite names for thrift binary frontend. | seq | 1.7.0 | -| kyuubi.frontend.thrift.http.allow.user.substitution | true | Allow alternate user to be specified as part of open connection request when using HTTP transport mode. | boolean | 1.6.0 | -| kyuubi.frontend.thrift.http.bind.host | <undefined> | Hostname or IP of the machine on which to run the thrift frontend service via http protocol. | string | 1.6.0 | -| kyuubi.frontend.thrift.http.bind.port | 10010 | Port of the machine on which to run the thrift frontend service via http protocol. | int | 1.6.0 | -| kyuubi.frontend.thrift.http.compression.enabled | true | Enable thrift http compression via Jetty compression support | boolean | 1.6.0 | -| kyuubi.frontend.thrift.http.cookie.auth.enabled | true | When true, Kyuubi in HTTP transport mode, will use cookie-based authentication mechanism | boolean | 1.6.0 | -| kyuubi.frontend.thrift.http.cookie.domain | <undefined> | Domain for the Kyuubi generated cookies | string | 1.6.0 | -| kyuubi.frontend.thrift.http.cookie.is.httponly | true | HttpOnly attribute of the Kyuubi generated cookie. | boolean | 1.6.0 | -| kyuubi.frontend.thrift.http.cookie.max.age | 86400 | Maximum age in seconds for server side cookie used by Kyuubi in HTTP mode. | int | 1.6.0 | -| kyuubi.frontend.thrift.http.cookie.path | <undefined> | Path for the Kyuubi generated cookies | string | 1.6.0 | -| kyuubi.frontend.thrift.http.max.idle.time | PT30M | Maximum idle time for a connection on the server when in HTTP mode. | duration | 1.6.0 | -| kyuubi.frontend.thrift.http.path | cliservice | Path component of URL endpoint when in HTTP mode. | string | 1.6.0 | -| kyuubi.frontend.thrift.http.request.header.size | 6144 | Request header size in bytes, when using HTTP transport mode. Jetty defaults used. | int | 1.6.0 | -| kyuubi.frontend.thrift.http.response.header.size | 6144 | Response header size in bytes, when using HTTP transport mode. Jetty defaults used. | int | 1.6.0 | -| kyuubi.frontend.thrift.http.ssl.exclude.ciphersuites || A comma-separated list of exclude SSL cipher suite names for thrift http frontend. | seq | 1.7.0 | -| kyuubi.frontend.thrift.http.ssl.keystore.password | <undefined> | SSL certificate keystore password. | string | 1.6.0 | -| kyuubi.frontend.thrift.http.ssl.keystore.path | <undefined> | SSL certificate keystore location. | string | 1.6.0 | -| kyuubi.frontend.thrift.http.ssl.protocol.blacklist | SSLv2,SSLv3 | SSL Versions to disable when using HTTP transport mode. | seq | 1.6.0 | -| kyuubi.frontend.thrift.http.use.SSL | false | Set this to true for using SSL encryption in http mode. | boolean | 1.6.0 | -| kyuubi.frontend.thrift.http.xsrf.filter.enabled | false | If enabled, Kyuubi will block any requests made to it over HTTP if an X-XSRF-HEADER header is not present | boolean | 1.6.0 | -| kyuubi.frontend.thrift.login.timeout | PT20S | Timeout for Thrift clients during login to the thrift frontend service. | duration | 1.4.0 | -| kyuubi.frontend.thrift.max.message.size | 104857600 | Maximum message size in bytes a Kyuubi server will accept. | int | 1.4.0 | -| kyuubi.frontend.thrift.max.worker.threads | 999 | Maximum number of threads in the frontend worker thread pool for the thrift frontend service | int | 1.4.0 | -| kyuubi.frontend.thrift.min.worker.threads | 9 | Minimum number of threads in the frontend worker thread pool for the thrift frontend service | int | 1.4.0 | -| kyuubi.frontend.thrift.worker.keepalive.time | PT1M | Keep-alive time (in milliseconds) for an idle worker thread | duration | 1.4.0 | -| kyuubi.frontend.trino.bind.host | <undefined> | Hostname or IP of the machine on which to run the TRINO frontend service. | string | 1.7.0 | -| kyuubi.frontend.trino.bind.port | 10999 | Port of the machine on which to run the TRINO frontend service. | int | 1.7.0 | -| kyuubi.frontend.trino.max.worker.threads | 999 | Maximum number of threads in the frontend worker thread pool for the Trino frontend service | int | 1.7.0 | -| kyuubi.frontend.worker.keepalive.time | PT1M | (deprecated) Keep-alive time (in milliseconds) for an idle worker thread | duration | 1.0.0 | +| Key | Default | Meaning | Type | Since | +|--------------------------------------------------------|--------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|-------| +| kyuubi.frontend.advertised.host | <undefined> | Hostname or IP of the Kyuubi server's frontend services to publish to external systems such as the service discovery ensemble and metadata store. Use it when you want to advertise a different hostname or IP than the bind host. | string | 1.8.0 | +| kyuubi.frontend.backoff.slot.length | PT0.1S | (deprecated) Time to back off during login to the thrift frontend service. | duration | 1.0.0 | +| kyuubi.frontend.bind.host | <undefined> | Hostname or IP of the machine on which to run the frontend services. | string | 1.0.0 | +| kyuubi.frontend.bind.port | 10009 | (deprecated) Port of the machine on which to run the thrift frontend service via the binary protocol. | int | 1.0.0 | +| kyuubi.frontend.connection.url.use.hostname | true | When true, frontend services prefer hostname, otherwise, ip address. Note that, the default value is set to `false` when engine running on Kubernetes to prevent potential network issues. | boolean | 1.5.0 | +| kyuubi.frontend.login.timeout | PT20S | (deprecated) Timeout for Thrift clients during login to the thrift frontend service. | duration | 1.0.0 | +| kyuubi.frontend.max.message.size | 104857600 | (deprecated) Maximum message size in bytes a Kyuubi server will accept. | int | 1.0.0 | +| kyuubi.frontend.max.worker.threads | 999 | (deprecated) Maximum number of threads in the frontend worker thread pool for the thrift frontend service | int | 1.0.0 | +| kyuubi.frontend.min.worker.threads | 9 | (deprecated) Minimum number of threads in the frontend worker thread pool for the thrift frontend service | int | 1.0.0 | +| kyuubi.frontend.mysql.bind.host | <undefined> | Hostname or IP of the machine on which to run the MySQL frontend service. | string | 1.4.0 | +| kyuubi.frontend.mysql.bind.port | 3309 | Port of the machine on which to run the MySQL frontend service. | int | 1.4.0 | +| kyuubi.frontend.mysql.max.worker.threads | 999 | Maximum number of threads in the command execution thread pool for the MySQL frontend service | int | 1.4.0 | +| kyuubi.frontend.mysql.min.worker.threads | 9 | Minimum number of threads in the command execution thread pool for the MySQL frontend service | int | 1.4.0 | +| kyuubi.frontend.mysql.netty.worker.threads | <undefined> | Number of thread in the netty worker event loop of MySQL frontend service. Use min(cpu_cores, 8) in default. | int | 1.4.0 | +| kyuubi.frontend.mysql.worker.keepalive.time | PT1M | Time(ms) that an idle async thread of the command execution thread pool will wait for a new task to arrive before terminating in MySQL frontend service | duration | 1.4.0 | +| kyuubi.frontend.protocols | THRIFT_BINARY,REST | A comma-separated list for all frontend protocols
  • THRIFT_BINARY - HiveServer2 compatible thrift binary protocol.
  • THRIFT_HTTP - HiveServer2 compatible thrift http protocol.
  • REST - Kyuubi defined REST API(experimental).
  • MYSQL - MySQL compatible text protocol(experimental).
  • TRINO - Trino compatible http protocol(experimental).
| seq | 1.4.0 | +| kyuubi.frontend.proxy.http.client.ip.header | X-Real-IP | The HTTP header to record the real client IP address. If your server is behind a load balancer or other proxy, the server will see this load balancer or proxy IP address as the client IP address, to get around this common issue, most load balancers or proxies offer the ability to record the real remote IP address in an HTTP header that will be added to the request for other devices to use. Note that, because the header value can be specified to any IP address, so it will not be used for authentication. | string | 1.6.0 | +| kyuubi.frontend.rest.bind.host | <undefined> | Hostname or IP of the machine on which to run the REST frontend service. | string | 1.4.0 | +| kyuubi.frontend.rest.bind.port | 10099 | Port of the machine on which to run the REST frontend service. | int | 1.4.0 | +| kyuubi.frontend.rest.max.worker.threads | 999 | Maximum number of threads in the frontend worker thread pool for the rest frontend service | int | 1.6.2 | +| kyuubi.frontend.ssl.keystore.algorithm | <undefined> | SSL certificate keystore algorithm. | string | 1.7.0 | +| kyuubi.frontend.ssl.keystore.password | <undefined> | SSL certificate keystore password. | string | 1.7.0 | +| kyuubi.frontend.ssl.keystore.path | <undefined> | SSL certificate keystore location. | string | 1.7.0 | +| kyuubi.frontend.ssl.keystore.type | <undefined> | SSL certificate keystore type. | string | 1.7.0 | +| kyuubi.frontend.thrift.backoff.slot.length | PT0.1S | Time to back off during login to the thrift frontend service. | duration | 1.4.0 | +| kyuubi.frontend.thrift.binary.bind.host | <undefined> | Hostname or IP of the machine on which to run the thrift frontend service via the binary protocol. | string | 1.4.0 | +| kyuubi.frontend.thrift.binary.bind.port | 10009 | Port of the machine on which to run the thrift frontend service via the binary protocol. | int | 1.4.0 | +| kyuubi.frontend.thrift.binary.ssl.disallowed.protocols | SSLv2,SSLv3 | SSL versions to disallow for Kyuubi thrift binary frontend. | set | 1.7.0 | +| kyuubi.frontend.thrift.binary.ssl.enabled | false | Set this to true for using SSL encryption in thrift binary frontend server. | boolean | 1.7.0 | +| kyuubi.frontend.thrift.binary.ssl.include.ciphersuites || A comma-separated list of include SSL cipher suite names for thrift binary frontend. | seq | 1.7.0 | +| kyuubi.frontend.thrift.http.allow.user.substitution | true | Allow alternate user to be specified as part of open connection request when using HTTP transport mode. | boolean | 1.6.0 | +| kyuubi.frontend.thrift.http.bind.host | <undefined> | Hostname or IP of the machine on which to run the thrift frontend service via http protocol. | string | 1.6.0 | +| kyuubi.frontend.thrift.http.bind.port | 10010 | Port of the machine on which to run the thrift frontend service via http protocol. | int | 1.6.0 | +| kyuubi.frontend.thrift.http.compression.enabled | true | Enable thrift http compression via Jetty compression support | boolean | 1.6.0 | +| kyuubi.frontend.thrift.http.cookie.auth.enabled | true | When true, Kyuubi in HTTP transport mode, will use cookie-based authentication mechanism | boolean | 1.6.0 | +| kyuubi.frontend.thrift.http.cookie.domain | <undefined> | Domain for the Kyuubi generated cookies | string | 1.6.0 | +| kyuubi.frontend.thrift.http.cookie.is.httponly | true | HttpOnly attribute of the Kyuubi generated cookie. | boolean | 1.6.0 | +| kyuubi.frontend.thrift.http.cookie.max.age | 86400 | Maximum age in seconds for server side cookie used by Kyuubi in HTTP mode. | int | 1.6.0 | +| kyuubi.frontend.thrift.http.cookie.path | <undefined> | Path for the Kyuubi generated cookies | string | 1.6.0 | +| kyuubi.frontend.thrift.http.max.idle.time | PT30M | Maximum idle time for a connection on the server when in HTTP mode. | duration | 1.6.0 | +| kyuubi.frontend.thrift.http.path | cliservice | Path component of URL endpoint when in HTTP mode. | string | 1.6.0 | +| kyuubi.frontend.thrift.http.request.header.size | 6144 | Request header size in bytes, when using HTTP transport mode. Jetty defaults used. | int | 1.6.0 | +| kyuubi.frontend.thrift.http.response.header.size | 6144 | Response header size in bytes, when using HTTP transport mode. Jetty defaults used. | int | 1.6.0 | +| kyuubi.frontend.thrift.http.ssl.exclude.ciphersuites || A comma-separated list of exclude SSL cipher suite names for thrift http frontend. | seq | 1.7.0 | +| kyuubi.frontend.thrift.http.ssl.keystore.password | <undefined> | SSL certificate keystore password. | string | 1.6.0 | +| kyuubi.frontend.thrift.http.ssl.keystore.path | <undefined> | SSL certificate keystore location. | string | 1.6.0 | +| kyuubi.frontend.thrift.http.ssl.protocol.blacklist | SSLv2,SSLv3 | SSL Versions to disable when using HTTP transport mode. | seq | 1.6.0 | +| kyuubi.frontend.thrift.http.use.SSL | false | Set this to true for using SSL encryption in http mode. | boolean | 1.6.0 | +| kyuubi.frontend.thrift.http.xsrf.filter.enabled | false | If enabled, Kyuubi will block any requests made to it over HTTP if an X-XSRF-HEADER header is not present | boolean | 1.6.0 | +| kyuubi.frontend.thrift.login.timeout | PT20S | Timeout for Thrift clients during login to the thrift frontend service. | duration | 1.4.0 | +| kyuubi.frontend.thrift.max.message.size | 104857600 | Maximum message size in bytes a Kyuubi server will accept. | int | 1.4.0 | +| kyuubi.frontend.thrift.max.worker.threads | 999 | Maximum number of threads in the frontend worker thread pool for the thrift frontend service | int | 1.4.0 | +| kyuubi.frontend.thrift.min.worker.threads | 9 | Minimum number of threads in the frontend worker thread pool for the thrift frontend service | int | 1.4.0 | +| kyuubi.frontend.thrift.worker.keepalive.time | PT1M | Keep-alive time (in milliseconds) for an idle worker thread | duration | 1.4.0 | +| kyuubi.frontend.trino.bind.host | <undefined> | Hostname or IP of the machine on which to run the TRINO frontend service. | string | 1.7.0 | +| kyuubi.frontend.trino.bind.port | 10999 | Port of the machine on which to run the TRINO frontend service. | int | 1.7.0 | +| kyuubi.frontend.trino.max.worker.threads | 999 | Maximum number of threads in the frontend worker thread pool for the Trino frontend service | int | 1.7.0 | +| kyuubi.frontend.worker.keepalive.time | PT1M | (deprecated) Keep-alive time (in milliseconds) for an idle worker thread | duration | 1.0.0 | ### Ha -| Key | Default | Meaning | Type | Since | -|------------------------------------------------|----------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|-------| -| kyuubi.ha.addresses || The connection string for the discovery ensemble | string | 1.6.0 | -| kyuubi.ha.client.class | org.apache.kyuubi.ha.client.zookeeper.ZookeeperDiscoveryClient | Class name for service discovery client.
  • Zookeeper: org.apache.kyuubi.ha.client.zookeeper.ZookeeperDiscoveryClient
  • Etcd: org.apache.kyuubi.ha.client.etcd.EtcdDiscoveryClient
| string | 1.6.0 | -| kyuubi.ha.etcd.lease.timeout | PT10S | Timeout for etcd keep alive lease. The kyuubi server will know the unexpected loss of engine after up to this seconds. | duration | 1.6.0 | -| kyuubi.ha.etcd.ssl.ca.path | <undefined> | Where the etcd CA certificate file is stored. | string | 1.6.0 | -| kyuubi.ha.etcd.ssl.client.certificate.path | <undefined> | Where the etcd SSL certificate file is stored. | string | 1.6.0 | -| kyuubi.ha.etcd.ssl.client.key.path | <undefined> | Where the etcd SSL key file is stored. | string | 1.6.0 | -| kyuubi.ha.etcd.ssl.enabled | false | When set to true, will build an SSL secured etcd client. | boolean | 1.6.0 | -| kyuubi.ha.namespace | kyuubi | The root directory for the service to deploy its instance uri | string | 1.6.0 | -| kyuubi.ha.zookeeper.acl.enabled | false | Set to true if the ZooKeeper ensemble is kerberized | boolean | 1.0.0 | -| kyuubi.ha.zookeeper.auth.digest | <undefined> | The digest auth string is used for ZooKeeper authentication, like: username:password. | string | 1.3.2 | -| kyuubi.ha.zookeeper.auth.keytab | <undefined> | Location of the Kyuubi server's keytab is used for ZooKeeper authentication. | string | 1.3.2 | -| kyuubi.ha.zookeeper.auth.principal | <undefined> | Name of the Kerberos principal is used for ZooKeeper authentication. | string | 1.3.2 | -| kyuubi.ha.zookeeper.auth.type | NONE | The type of ZooKeeper authentication, all candidates are
  • NONE
  • KERBEROS
  • DIGEST
| string | 1.3.2 | -| kyuubi.ha.zookeeper.connection.base.retry.wait | 1000 | Initial amount of time to wait between retries to the ZooKeeper ensemble | int | 1.0.0 | -| kyuubi.ha.zookeeper.connection.max.retries | 3 | Max retry times for connecting to the ZooKeeper ensemble | int | 1.0.0 | -| kyuubi.ha.zookeeper.connection.max.retry.wait | 30000 | Max amount of time to wait between retries for BOUNDED_EXPONENTIAL_BACKOFF policy can reach, or max time until elapsed for UNTIL_ELAPSED policy to connect the zookeeper ensemble | int | 1.0.0 | -| kyuubi.ha.zookeeper.connection.retry.policy | EXPONENTIAL_BACKOFF | The retry policy for connecting to the ZooKeeper ensemble, all candidates are:
  • ONE_TIME
  • N_TIME
  • EXPONENTIAL_BACKOFF
  • BOUNDED_EXPONENTIAL_BACKOFF
  • UNTIL_ELAPSED
| string | 1.0.0 | -| kyuubi.ha.zookeeper.connection.timeout | 15000 | The timeout(ms) of creating the connection to the ZooKeeper ensemble | int | 1.0.0 | -| kyuubi.ha.zookeeper.engine.auth.type | NONE | The type of ZooKeeper authentication for the engine, all candidates are
  • NONE
  • KERBEROS
  • DIGEST
| string | 1.3.2 | -| kyuubi.ha.zookeeper.namespace | kyuubi | (deprecated) The root directory for the service to deploy its instance uri | string | 1.0.0 | -| kyuubi.ha.zookeeper.node.creation.timeout | PT2M | Timeout for creating ZooKeeper node | duration | 1.2.0 | -| kyuubi.ha.zookeeper.publish.configs | false | When set to true, publish Kerberos configs to Zookeeper. Note that the Hive driver needs to be greater than 1.3 or 2.0 or apply HIVE-11581 patch. | boolean | 1.4.0 | -| kyuubi.ha.zookeeper.quorum || (deprecated) The connection string for the ZooKeeper ensemble | string | 1.0.0 | -| kyuubi.ha.zookeeper.session.timeout | 60000 | The timeout(ms) of a connected session to be idled | int | 1.0.0 | +| Key | Default | Meaning | Type | Since | +|------------------------------------------------|----------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|-------| +| kyuubi.ha.addresses || The connection string for the discovery ensemble | string | 1.6.0 | +| kyuubi.ha.client.class | org.apache.kyuubi.ha.client.zookeeper.ZookeeperDiscoveryClient | Class name for service discovery client.
  • Zookeeper: org.apache.kyuubi.ha.client.zookeeper.ZookeeperDiscoveryClient
  • Etcd: org.apache.kyuubi.ha.client.etcd.EtcdDiscoveryClient
| string | 1.6.0 | +| kyuubi.ha.etcd.lease.timeout | PT10S | Timeout for etcd keep alive lease. The kyuubi server will know the unexpected loss of engine after up to this seconds. | duration | 1.6.0 | +| kyuubi.ha.etcd.ssl.ca.path | <undefined> | Where the etcd CA certificate file is stored. | string | 1.6.0 | +| kyuubi.ha.etcd.ssl.client.certificate.path | <undefined> | Where the etcd SSL certificate file is stored. | string | 1.6.0 | +| kyuubi.ha.etcd.ssl.client.key.path | <undefined> | Where the etcd SSL key file is stored. | string | 1.6.0 | +| kyuubi.ha.etcd.ssl.enabled | false | When set to true, will build an SSL secured etcd client. | boolean | 1.6.0 | +| kyuubi.ha.namespace | kyuubi | The root directory for the service to deploy its instance uri | string | 1.6.0 | +| kyuubi.ha.zookeeper.acl.enabled | false | Set to true if the ZooKeeper ensemble is kerberized | boolean | 1.0.0 | +| kyuubi.ha.zookeeper.auth.digest | <undefined> | The digest auth string is used for ZooKeeper authentication, like: username:password. | string | 1.3.2 | +| kyuubi.ha.zookeeper.auth.keytab | <undefined> | Location of the Kyuubi server's keytab that is used for ZooKeeper authentication. | string | 1.3.2 | +| kyuubi.ha.zookeeper.auth.principal | <undefined> | Kerberos principal name that is used for ZooKeeper authentication. | string | 1.3.2 | +| kyuubi.ha.zookeeper.auth.serverPrincipal | <undefined> | Kerberos principal name of ZooKeeper Server. It only takes effect when Zookeeper client's version at least 3.5.7 or 3.6.0 or applies ZOOKEEPER-1467. To use Zookeeper 3.6 client, compile Kyuubi with `-Pzookeeper-3.6`. | string | 1.8.0 | +| kyuubi.ha.zookeeper.auth.type | NONE | The type of ZooKeeper authentication, all candidates are
  • NONE
  • KERBEROS
  • DIGEST
| string | 1.3.2 | +| kyuubi.ha.zookeeper.connection.base.retry.wait | 1000 | Initial amount of time to wait between retries to the ZooKeeper ensemble | int | 1.0.0 | +| kyuubi.ha.zookeeper.connection.max.retries | 3 | Max retry times for connecting to the ZooKeeper ensemble | int | 1.0.0 | +| kyuubi.ha.zookeeper.connection.max.retry.wait | 30000 | Max amount of time to wait between retries for BOUNDED_EXPONENTIAL_BACKOFF policy can reach, or max time until elapsed for UNTIL_ELAPSED policy to connect the zookeeper ensemble | int | 1.0.0 | +| kyuubi.ha.zookeeper.connection.retry.policy | EXPONENTIAL_BACKOFF | The retry policy for connecting to the ZooKeeper ensemble, all candidates are:
  • ONE_TIME
  • N_TIME
  • EXPONENTIAL_BACKOFF
  • BOUNDED_EXPONENTIAL_BACKOFF
  • UNTIL_ELAPSED
| string | 1.0.0 | +| kyuubi.ha.zookeeper.connection.timeout | 15000 | The timeout(ms) of creating the connection to the ZooKeeper ensemble | int | 1.0.0 | +| kyuubi.ha.zookeeper.engine.auth.type | NONE | The type of ZooKeeper authentication for the engine, all candidates are
  • NONE
  • KERBEROS
  • DIGEST
| string | 1.3.2 | +| kyuubi.ha.zookeeper.namespace | kyuubi | (deprecated) The root directory for the service to deploy its instance uri | string | 1.0.0 | +| kyuubi.ha.zookeeper.node.creation.timeout | PT2M | Timeout for creating ZooKeeper node | duration | 1.2.0 | +| kyuubi.ha.zookeeper.publish.configs | false | When set to true, publish Kerberos configs to Zookeeper. Note that the Hive driver needs to be greater than 1.3 or 2.0 or apply HIVE-11581 patch. | boolean | 1.4.0 | +| kyuubi.ha.zookeeper.quorum || (deprecated) The connection string for the ZooKeeper ensemble | string | 1.0.0 | +| kyuubi.ha.zookeeper.session.timeout | 60000 | The timeout(ms) of a connected session to be idled | int | 1.0.0 | ### Kinit @@ -373,98 +309,118 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co ### Kubernetes -| Key | Default | Meaning | Type | Since | -|-----------------------------------------------|-------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------|-------| -| kyuubi.kubernetes.authenticate.caCertFile | <undefined> | Path to the CA cert file for connecting to the Kubernetes API server over TLS from the kyuubi. Specify this as a path as opposed to a URI (i.e. do not provide a scheme) | string | 1.7.0 | -| kyuubi.kubernetes.authenticate.clientCertFile | <undefined> | Path to the client cert file for connecting to the Kubernetes API server over TLS from the kyuubi. Specify this as a path as opposed to a URI (i.e. do not provide a scheme) | string | 1.7.0 | -| kyuubi.kubernetes.authenticate.clientKeyFile | <undefined> | Path to the client key file for connecting to the Kubernetes API server over TLS from the kyuubi. Specify this as a path as opposed to a URI (i.e. do not provide a scheme) | string | 1.7.0 | -| kyuubi.kubernetes.authenticate.oauthToken | <undefined> | The OAuth token to use when authenticating against the Kubernetes API server. Note that unlike, the other authentication options, this must be the exact string value of the token to use for the authentication. | string | 1.7.0 | -| kyuubi.kubernetes.authenticate.oauthTokenFile | <undefined> | Path to the file containing the OAuth token to use when authenticating against the Kubernetes API server. Specify this as a path as opposed to a URI (i.e. do not provide a scheme) | string | 1.7.0 | -| kyuubi.kubernetes.context | <undefined> | The desired context from your kubernetes config file used to configure the K8s client for interacting with the cluster. | string | 1.6.0 | -| kyuubi.kubernetes.master.address | <undefined> | The internal Kubernetes master (API server) address to be used for kyuubi. | string | 1.7.0 | -| kyuubi.kubernetes.namespace | default | The namespace that will be used for running the kyuubi pods and find engines. | string | 1.7.0 | -| kyuubi.kubernetes.trust.certificates | false | If set to true then client can submit to kubernetes cluster only with token | boolean | 1.7.0 | +| Key | Default | Meaning | Type | Since | +|-----------------------------------------------------|-------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|-------| +| kyuubi.kubernetes.authenticate.caCertFile | <undefined> | Path to the CA cert file for connecting to the Kubernetes API server over TLS from the kyuubi. Specify this as a path as opposed to a URI (i.e. do not provide a scheme) | string | 1.7.0 | +| kyuubi.kubernetes.authenticate.clientCertFile | <undefined> | Path to the client cert file for connecting to the Kubernetes API server over TLS from the kyuubi. Specify this as a path as opposed to a URI (i.e. do not provide a scheme) | string | 1.7.0 | +| kyuubi.kubernetes.authenticate.clientKeyFile | <undefined> | Path to the client key file for connecting to the Kubernetes API server over TLS from the kyuubi. Specify this as a path as opposed to a URI (i.e. do not provide a scheme) | string | 1.7.0 | +| kyuubi.kubernetes.authenticate.oauthToken | <undefined> | The OAuth token to use when authenticating against the Kubernetes API server. Note that unlike, the other authentication options, this must be the exact string value of the token to use for the authentication. | string | 1.7.0 | +| kyuubi.kubernetes.authenticate.oauthTokenFile | <undefined> | Path to the file containing the OAuth token to use when authenticating against the Kubernetes API server. Specify this as a path as opposed to a URI (i.e. do not provide a scheme) | string | 1.7.0 | +| kyuubi.kubernetes.context | <undefined> | The desired context from your kubernetes config file used to configure the K8s client for interacting with the cluster. | string | 1.6.0 | +| kyuubi.kubernetes.context.allow.list || The allowed kubernetes context list, if it is empty, there is no kubernetes context limitation. | set | 1.8.0 | +| kyuubi.kubernetes.master.address | <undefined> | The internal Kubernetes master (API server) address to be used for kyuubi. | string | 1.7.0 | +| kyuubi.kubernetes.namespace | default | The namespace that will be used for running the kyuubi pods and find engines. | string | 1.7.0 | +| kyuubi.kubernetes.namespace.allow.list || The allowed kubernetes namespace list, if it is empty, there is no kubernetes namespace limitation. | set | 1.8.0 | +| kyuubi.kubernetes.terminatedApplicationRetainPeriod | PT5M | The period for which the Kyuubi server retains application information after the application terminates. | duration | 1.7.1 | +| kyuubi.kubernetes.trust.certificates | false | If set to true then client can submit to kubernetes cluster only with token | boolean | 1.7.0 | + +### Lineage + +| Key | Default | Meaning | Type | Since | +|---------------------------------------|--------------------------------------------------------|---------------------------------------------------|--------|-------| +| kyuubi.lineage.parser.plugin.provider | org.apache.kyuubi.plugin.lineage.LineageParserProvider | The provider for the Spark lineage parser plugin. | string | 1.8.0 | ### Metadata -| Key | Default | Meaning | Type | Since | -|-------------------------------------------------|----------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|-------| -| kyuubi.metadata.cleaner.enabled | true | Whether to clean the metadata periodically. If it is enabled, Kyuubi will clean the metadata that is in the terminate state with max age limitation. | boolean | 1.6.0 | -| kyuubi.metadata.cleaner.interval | PT30M | The interval to check and clean expired metadata. | duration | 1.6.0 | -| kyuubi.metadata.max.age | PT72H | The maximum age of metadata, the metadata exceeding the age will be cleaned. | duration | 1.6.0 | -| kyuubi.metadata.recovery.threads | 10 | The number of threads for recovery from the metadata store when the Kyuubi server restarts. | int | 1.6.0 | -| kyuubi.metadata.request.retry.interval | PT5S | The interval to check and trigger the metadata request retry tasks. | duration | 1.6.0 | -| kyuubi.metadata.request.retry.queue.size | 65536 | The maximum queue size for buffering metadata requests in memory when the external metadata storage is down. Requests will be dropped if the queue exceeds. | int | 1.6.0 | -| kyuubi.metadata.request.retry.threads | 10 | Number of threads in the metadata request retry manager thread pool. The metadata store might be unavailable sometimes and the requests will fail, tolerant for this case and unblock the main thread, we support retrying the failed requests in an async way. | int | 1.6.0 | -| kyuubi.metadata.store.class | org.apache.kyuubi.server.metadata.jdbc.JDBCMetadataStore | Fully qualified class name for server metadata store. | string | 1.6.0 | -| kyuubi.metadata.store.jdbc.database.schema.init | true | Whether to init the JDBC metadata store database schema. | boolean | 1.6.0 | -| kyuubi.metadata.store.jdbc.database.type | DERBY | The database type for server jdbc metadata store.
  • DERBY: Apache Derby, JDBC driver `org.apache.derby.jdbc.AutoloadedDriver`.
  • MYSQL: MySQL, JDBC driver `com.mysql.jdbc.Driver`.
  • CUSTOM: User-defined database type, need to specify corresponding JDBC driver.
    • Note that: The JDBC datasource is powered by HiKariCP, for datasource properties, please specify them with the prefix: kyuubi.metadata.store.jdbc.datasource. For example, kyuubi.metadata.store.jdbc.datasource.connectionTimeout=10000. | string | 1.6.0 | -| kyuubi.metadata.store.jdbc.driver | <undefined> | JDBC driver class name for server jdbc metadata store. | string | 1.6.0 | -| kyuubi.metadata.store.jdbc.password || The password for server JDBC metadata store. | string | 1.6.0 | -| kyuubi.metadata.store.jdbc.url | jdbc:derby:memory:kyuubi_state_store_db;create=true | The JDBC url for server JDBC metadata store. By default, it is a DERBY in-memory database url, and the state information is not shared across kyuubi instances. To enable high availability for multiple kyuubi instances, please specify a production JDBC url. | string | 1.6.0 | -| kyuubi.metadata.store.jdbc.user || The username for server JDBC metadata store. | string | 1.6.0 | +| Key | Default | Meaning | Type | Since | +|-------------------------------------------------|----------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|-------| +| kyuubi.metadata.cleaner.enabled | true | Whether to clean the metadata periodically. If it is enabled, Kyuubi will clean the metadata that is in the terminate state with max age limitation. | boolean | 1.6.0 | +| kyuubi.metadata.cleaner.interval | PT30M | The interval to check and clean expired metadata. | duration | 1.6.0 | +| kyuubi.metadata.max.age | PT72H | The maximum age of metadata, the metadata exceeding the age will be cleaned. | duration | 1.6.0 | +| kyuubi.metadata.recovery.threads | 10 | The number of threads for recovery from the metadata store when the Kyuubi server restarts. | int | 1.6.0 | +| kyuubi.metadata.request.async.retry.enabled | true | Whether to retry in async when metadata request failed. When true, return success response immediately even the metadata request failed, and schedule it in background until success, to tolerate long-time metadata store outages w/o blocking the submission request. | boolean | 1.7.0 | +| kyuubi.metadata.request.async.retry.queue.size | 65536 | The maximum queue size for buffering metadata requests in memory when the external metadata storage is down. Requests will be dropped if the queue exceeds. Only take affect when kyuubi.metadata.request.async.retry.enabled is `true`. | int | 1.6.0 | +| kyuubi.metadata.request.async.retry.threads | 10 | Number of threads in the metadata request async retry manager thread pool. Only take affect when kyuubi.metadata.request.async.retry.enabled is `true`. | int | 1.6.0 | +| kyuubi.metadata.request.retry.interval | PT5S | The interval to check and trigger the metadata request retry tasks. | duration | 1.6.0 | +| kyuubi.metadata.store.class | org.apache.kyuubi.server.metadata.jdbc.JDBCMetadataStore | Fully qualified class name for server metadata store. | string | 1.6.0 | +| kyuubi.metadata.store.jdbc.database.schema.init | true | Whether to init the JDBC metadata store database schema. | boolean | 1.6.0 | +| kyuubi.metadata.store.jdbc.database.type | SQLITE | The database type for server jdbc metadata store.
    • (Deprecated) DERBY: Apache Derby, JDBC driver `org.apache.derby.jdbc.AutoloadedDriver`.
    • SQLITE: SQLite3, JDBC driver `org.sqlite.JDBC`.
    • MYSQL: MySQL, JDBC driver `com.mysql.cj.jdbc.Driver` (fallback `com.mysql.jdbc.Driver`).
    • CUSTOM: User-defined database type, need to specify corresponding JDBC driver.
      • Note that: The JDBC datasource is powered by HiKariCP, for datasource properties, please specify them with the prefix: kyuubi.metadata.store.jdbc.datasource. For example, kyuubi.metadata.store.jdbc.datasource.connectionTimeout=10000. | string | 1.6.0 | +| kyuubi.metadata.store.jdbc.driver | <undefined> | JDBC driver class name for server jdbc metadata store. | string | 1.6.0 | +| kyuubi.metadata.store.jdbc.password || The password for server JDBC metadata store. | string | 1.6.0 | +| kyuubi.metadata.store.jdbc.priority.enabled | false | Whether to enable the priority scheduling for batch impl v2. When false, ignore kyuubi.batch.priority and use the FIFO ordering strategy for batch job scheduling. Note: this feature may cause significant performance issues when using MySQL 5.7 as the metastore backend due to the lack of support for mixed order index. See more details at KYUUBI #5329. | boolean | 1.8.0 | +| kyuubi.metadata.store.jdbc.url | jdbc:sqlite:kyuubi_state_store.db | The JDBC url for server JDBC metadata store. By default, it is a SQLite database url, and the state information is not shared across kyuubi instances. To enable high availability for multiple kyuubi instances, please specify a production JDBC url. | string | 1.6.0 | +| kyuubi.metadata.store.jdbc.user || The username for server JDBC metadata store. | string | 1.6.0 | ### Metrics -| Key | Default | Meaning | Type | Since | -|---------------------------------|----------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|-------| -| kyuubi.metrics.console.interval | PT5S | How often should report metrics to console | duration | 1.2.0 | -| kyuubi.metrics.enabled | true | Set to true to enable kyuubi metrics system | boolean | 1.2.0 | -| kyuubi.metrics.json.interval | PT5S | How often should report metrics to JSON file | duration | 1.2.0 | -| kyuubi.metrics.json.location | metrics | Where the JSON metrics file located | string | 1.2.0 | -| kyuubi.metrics.prometheus.path | /metrics | URI context path of prometheus metrics HTTP server | string | 1.2.0 | -| kyuubi.metrics.prometheus.port | 10019 | Prometheus metrics HTTP server port | int | 1.2.0 | -| kyuubi.metrics.reporters | JSON | A comma-separated list for all metrics reporters
      • CONSOLE - ConsoleReporter which outputs measurements to CONSOLE periodically.
      • JMX - JmxReporter which listens for new metrics and exposes them as MBeans.
      • JSON - JsonReporter which outputs measurements to json file periodically.
      • PROMETHEUS - PrometheusReporter which exposes metrics in Prometheus format.
      • SLF4J - Slf4jReporter which outputs measurements to system log periodically.
      | seq | 1.2.0 | -| kyuubi.metrics.slf4j.interval | PT5S | How often should report metrics to SLF4J logger | duration | 1.2.0 | +| Key | Default | Meaning | Type | Since | +|---------------------------------|------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|-------| +| kyuubi.metrics.console.interval | PT5S | How often should report metrics to console | duration | 1.2.0 | +| kyuubi.metrics.enabled | true | Set to true to enable kyuubi metrics system | boolean | 1.2.0 | +| kyuubi.metrics.json.interval | PT5S | How often should report metrics to JSON file | duration | 1.2.0 | +| kyuubi.metrics.json.location | metrics | Where the JSON metrics file located | string | 1.2.0 | +| kyuubi.metrics.prometheus.path | /metrics | URI context path of prometheus metrics HTTP server | string | 1.2.0 | +| kyuubi.metrics.prometheus.port | 10019 | Prometheus metrics HTTP server port | int | 1.2.0 | +| kyuubi.metrics.reporters | PROMETHEUS | A comma-separated list for all metrics reporters
      • CONSOLE - ConsoleReporter which outputs measurements to CONSOLE periodically.
      • JMX - JmxReporter which listens for new metrics and exposes them as MBeans.
      • JSON - JsonReporter which outputs measurements to json file periodically.
      • PROMETHEUS - PrometheusReporter which exposes metrics in Prometheus format.
      • SLF4J - Slf4jReporter which outputs measurements to system log periodically.
      | set | 1.2.0 | +| kyuubi.metrics.slf4j.interval | PT5S | How often should report metrics to SLF4J logger | duration | 1.2.0 | ### Operation -| Key | Default | Meaning | Type | Since | -|-----------------------------------------|---------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|-------| -| kyuubi.operation.idle.timeout | PT3H | Operation will be closed when it's not accessed for this duration of time | duration | 1.0.0 | -| kyuubi.operation.interrupt.on.cancel | true | When true, all running tasks will be interrupted if one cancels a query. When false, all running tasks will remain until finished. | boolean | 1.2.0 | -| kyuubi.operation.language | SQL | Choose a programing language for the following inputs
      • SQL: (Default) Run all following statements as SQL queries.
      • SCALA: Run all following input a scala codes
      | string | 1.5.0 | -| kyuubi.operation.log.dir.root | server_operation_logs | Root directory for query operation log at server-side. | string | 1.4.0 | -| kyuubi.operation.plan.only.excludes | ResetCommand,SetCommand,SetNamespaceCommand,UseStatement,SetCatalogAndNamespace | Comma-separated list of query plan names, in the form of simple class names, i.e, for `SET abc=xyz`, the value will be `SetCommand`. For those auxiliary plans, such as `switch databases`, `set properties`, or `create temporary view` etc., which are used for setup evaluating environments for analyzing actual queries, we can use this config to exclude them and let them take effect. See also kyuubi.operation.plan.only.mode. | seq | 1.5.0 | -| kyuubi.operation.plan.only.mode | none | Configures the statement performed mode, The value can be 'parse', 'analyze', 'optimize', 'optimize_with_stats', 'physical', 'execution', or 'none', when it is 'none', indicate to the statement will be fully executed, otherwise only way without executing the query. different engines currently support different modes, the Spark engine supports all modes, and the Flink engine supports 'parse', 'physical', and 'execution', other engines do not support planOnly currently. | string | 1.4.0 | -| kyuubi.operation.plan.only.output.style | plain | Configures the planOnly output style. The value can be 'plain' or 'json', and the default value is 'plain'. This configuration supports only the output styles of the Spark engine | string | 1.7.0 | -| kyuubi.operation.progress.enabled | false | Whether to enable the operation progress. When true, the operation progress will be returned in `GetOperationStatus`. | boolean | 1.6.0 | -| kyuubi.operation.query.timeout | <undefined> | Timeout for query executions at server-side, take effect with client-side timeout(`java.sql.Statement.setQueryTimeout`) together, a running query will be cancelled automatically if timeout. It's off by default, which means only client-side take full control of whether the query should timeout or not. If set, client-side timeout is capped at this point. To cancel the queries right away without waiting for task to finish, consider enabling kyuubi.operation.interrupt.on.cancel together. | duration | 1.2.0 | -| kyuubi.operation.result.format | thrift | Specify the result format, available configs are:
      • THRIFT: the result will convert to TRow at the engine driver side.
      • ARROW: the result will be encoded as Arrow at the executor side before collecting by the driver, and deserialized at the client side. note that it only takes effect for kyuubi-hive-jdbc clients now.
      | string | 1.7.0 | -| kyuubi.operation.result.max.rows | 0 | Max rows of Spark query results. Rows exceeding the limit would be ignored. By setting this value to 0 to disable the max rows limit. | int | 1.6.0 | -| kyuubi.operation.scheduler.pool | <undefined> | The scheduler pool of job. Note that, this config should be used after changing Spark config spark.scheduler.mode=FAIR. | string | 1.1.1 | -| kyuubi.operation.spark.listener.enabled | true | When set to true, Spark engine registers an SQLOperationListener before executing the statement, logging a few summary statistics when each stage completes. | boolean | 1.6.0 | -| kyuubi.operation.status.polling.timeout | PT5S | Timeout(ms) for long polling asynchronous running sql query's status | duration | 1.0.0 | +| Key | Default | Meaning | Type | Since | +|--------------------------------------------------|---------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|-------| +| kyuubi.operation.getTables.ignoreTableProperties | false | Speed up the `GetTables` operation by returning table identities only. | boolean | 1.8.0 | +| kyuubi.operation.idle.timeout | PT3H | Operation will be closed when it's not accessed for this duration of time | duration | 1.0.0 | +| kyuubi.operation.interrupt.on.cancel | true | When true, all running tasks will be interrupted if one cancels a query. When false, all running tasks will remain until finished. | boolean | 1.2.0 | +| kyuubi.operation.language | SQL | Choose a programing language for the following inputs
      • SQL: (Default) Run all following statements as SQL queries.
      • SCALA: Run all following input as scala codes
      • PYTHON: (Experimental) Run all following input as Python codes with Spark engine
      | string | 1.5.0 | +| kyuubi.operation.log.dir.root | server_operation_logs | Root directory for query operation log at server-side. | string | 1.4.0 | +| kyuubi.operation.plan.only.excludes | SetCatalogAndNamespace,UseStatement,SetNamespaceCommand,SetCommand,ResetCommand | Comma-separated list of query plan names, in the form of simple class names, i.e, for `SET abc=xyz`, the value will be `SetCommand`. For those auxiliary plans, such as `switch databases`, `set properties`, or `create temporary view` etc., which are used for setup evaluating environments for analyzing actual queries, we can use this config to exclude them and let them take effect. See also kyuubi.operation.plan.only.mode. | set | 1.5.0 | +| kyuubi.operation.plan.only.mode | none | Configures the statement performed mode, The value can be 'parse', 'analyze', 'optimize', 'optimize_with_stats', 'physical', 'execution', 'lineage' or 'none', when it is 'none', indicate to the statement will be fully executed, otherwise only way without executing the query. different engines currently support different modes, the Spark engine supports all modes, and the Flink engine supports 'parse', 'physical', and 'execution', other engines do not support planOnly currently. | string | 1.4.0 | +| kyuubi.operation.plan.only.output.style | plain | Configures the planOnly output style. The value can be 'plain' or 'json', and the default value is 'plain'. This configuration supports only the output styles of the Spark engine | string | 1.7.0 | +| kyuubi.operation.progress.enabled | false | Whether to enable the operation progress. When true, the operation progress will be returned in `GetOperationStatus`. | boolean | 1.6.0 | +| kyuubi.operation.query.timeout | <undefined> | Timeout for query executions at server-side, take effect with client-side timeout(`java.sql.Statement.setQueryTimeout`) together, a running query will be cancelled automatically if timeout. It's off by default, which means only client-side take full control of whether the query should timeout or not. If set, client-side timeout is capped at this point. To cancel the queries right away without waiting for task to finish, consider enabling kyuubi.operation.interrupt.on.cancel together. | duration | 1.2.0 | +| kyuubi.operation.result.arrow.timestampAsString | false | When true, arrow-based rowsets will convert columns of type timestamp to strings for transmission. | boolean | 1.7.0 | +| kyuubi.operation.result.format | thrift | Specify the result format, available configs are:
      • THRIFT: the result will convert to TRow at the engine driver side.
      • ARROW: the result will be encoded as Arrow at the executor side before collecting by the driver, and deserialized at the client side. note that it only takes effect for kyuubi-hive-jdbc clients now.
      | string | 1.7.0 | +| kyuubi.operation.result.max.rows | 0 | Max rows of Spark query results. Rows exceeding the limit would be ignored. By setting this value to 0 to disable the max rows limit. | int | 1.6.0 | +| kyuubi.operation.scheduler.pool | <undefined> | The scheduler pool of job. Note that, this config should be used after changing Spark config spark.scheduler.mode=FAIR. | string | 1.1.1 | +| kyuubi.operation.spark.listener.enabled | true | When set to true, Spark engine registers an SQLOperationListener before executing the statement, logging a few summary statistics when each stage completes. | boolean | 1.6.0 | +| kyuubi.operation.status.polling.timeout | PT5S | Timeout(ms) for long polling asynchronous running sql query's status | duration | 1.0.0 | ### Server -| Key | Default | Meaning | Type | Since | -|----------------------------------------------------------|-------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------|-------| -| kyuubi.server.batch.limit.connections.per.ipaddress | <undefined> | Maximum kyuubi server batch connections per ipaddress. Any user exceeding this limit will not be allowed to connect. | int | 1.7.0 | -| kyuubi.server.batch.limit.connections.per.user | <undefined> | Maximum kyuubi server batch connections per user. Any user exceeding this limit will not be allowed to connect. | int | 1.7.0 | -| kyuubi.server.batch.limit.connections.per.user.ipaddress | <undefined> | Maximum kyuubi server batch connections per user:ipaddress combination. Any user-ipaddress exceeding this limit will not be allowed to connect. | int | 1.7.0 | -| kyuubi.server.info.provider | ENGINE | The server information provider name, some clients may rely on this information to check the server compatibilities and functionalities.
    • SERVER: Return Kyuubi server information.
    • ENGINE: Return Kyuubi engine information.
      • | string | 1.6.1 | -| kyuubi.server.limit.connections.per.ipaddress | <undefined> | Maximum kyuubi server connections per ipaddress. Any user exceeding this limit will not be allowed to connect. | int | 1.6.0 | -| kyuubi.server.limit.connections.per.user | <undefined> | Maximum kyuubi server connections per user. Any user exceeding this limit will not be allowed to connect. | int | 1.6.0 | -| kyuubi.server.limit.connections.per.user.ipaddress | <undefined> | Maximum kyuubi server connections per user:ipaddress combination. Any user-ipaddress exceeding this limit will not be allowed to connect. | int | 1.6.0 | -| kyuubi.server.limit.connections.user.unlimited.list || The maximin connections of the user in the white list will not be limited. | seq | 1.7.0 | -| kyuubi.server.name | <undefined> | The name of Kyuubi Server. | string | 1.5.0 | -| kyuubi.server.redaction.regex | <undefined> | Regex to decide which Kyuubi contain sensitive information. When this regex matches a property key or value, the value is redacted from the various logs. || 1.6.0 | +| Key | Default | Meaning | Type | Since | +|----------------------------------------------------------|-------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|-------| +| kyuubi.server.administrators || Comma-separated list of Kyuubi service administrators. We use this config to grant admin permission to any service accounts. | set | 1.8.0 | +| kyuubi.server.info.provider | ENGINE | The server information provider name, some clients may rely on this information to check the server compatibilities and functionalities.
    • SERVER: Return Kyuubi server information.
    • ENGINE: Return Kyuubi engine information.
      • | string | 1.6.1 | +| kyuubi.server.limit.batch.connections.per.ipaddress | <undefined> | Maximum kyuubi server batch connections per ipaddress. Any user exceeding this limit will not be allowed to connect. | int | 1.7.0 | +| kyuubi.server.limit.batch.connections.per.user | <undefined> | Maximum kyuubi server batch connections per user. Any user exceeding this limit will not be allowed to connect. | int | 1.7.0 | +| kyuubi.server.limit.batch.connections.per.user.ipaddress | <undefined> | Maximum kyuubi server batch connections per user:ipaddress combination. Any user-ipaddress exceeding this limit will not be allowed to connect. | int | 1.7.0 | +| kyuubi.server.limit.client.fetch.max.rows | <undefined> | Max rows limit for getting result row set operation. If the max rows specified by client-side is larger than the limit, request will fail directly. | int | 1.8.0 | +| kyuubi.server.limit.connections.per.ipaddress | <undefined> | Maximum kyuubi server connections per ipaddress. Any user exceeding this limit will not be allowed to connect. | int | 1.6.0 | +| kyuubi.server.limit.connections.per.user | <undefined> | Maximum kyuubi server connections per user. Any user exceeding this limit will not be allowed to connect. | int | 1.6.0 | +| kyuubi.server.limit.connections.per.user.ipaddress | <undefined> | Maximum kyuubi server connections per user:ipaddress combination. Any user-ipaddress exceeding this limit will not be allowed to connect. | int | 1.6.0 | +| kyuubi.server.limit.connections.user.deny.list || The user in the deny list will be denied to connect to kyuubi server, if the user has configured both user.unlimited.list and user.deny.list, the priority of the latter is higher. | set | 1.8.0 | +| kyuubi.server.limit.connections.user.unlimited.list || The maximum connections of the user in the white list will not be limited. | set | 1.7.0 | +| kyuubi.server.name | <undefined> | The name of Kyuubi Server. | string | 1.5.0 | +| kyuubi.server.periodicGC.interval | PT30M | How often to trigger a garbage collection. | duration | 1.7.0 | +| kyuubi.server.redaction.regex | <undefined> | Regex to decide which Kyuubi contain sensitive information. When this regex matches a property key or value, the value is redacted from the various logs. || 1.6.0 | ### Session | Key | Default | Meaning | Type | Since | |------------------------------------------------------|-------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|-------| | kyuubi.session.check.interval | PT5M | The check interval for session timeout. | duration | 1.0.0 | -| kyuubi.session.conf.advisor | <undefined> | A config advisor plugin for Kyuubi Server. This plugin can provide some custom configs for different users or session configs and overwrite the session configs before opening a new session. This config value should be a subclass of `org.apache.kyuubi.plugin.SessionConfAdvisor` which has a zero-arg constructor. | string | 1.5.0 | +| kyuubi.session.close.on.disconnect | true | Session will be closed when client disconnects from kyuubi gateway. Set this to false to have session outlive its parent connection. | boolean | 1.8.0 | +| kyuubi.session.conf.advisor | <undefined> | A config advisor plugin for Kyuubi Server. This plugin can provide a list of custom configs for different users or session configs and overwrite the session configs before opening a new session. This config value should be a subclass of `org.apache.kyuubi.plugin.SessionConfAdvisor` which has a zero-arg constructor. | seq | 1.5.0 | | kyuubi.session.conf.file.reload.interval | PT10M | When `FileSessionConfAdvisor` is used, this configuration defines the expired time of `$KYUUBI_CONF_DIR/kyuubi-session-.conf` in the cache. After exceeding this value, the file will be reloaded. | duration | 1.7.0 | -| kyuubi.session.conf.ignore.list || A comma-separated list of ignored keys. If the client connection contains any of them, the key and the corresponding value will be removed silently during engine bootstrap and connection setup. Note that this rule is for server-side protection defined via administrators to prevent some essential configs from tampering but will not forbid users to set dynamic configurations via SET syntax. | seq | 1.2.0 | +| kyuubi.session.conf.ignore.list || A comma-separated list of ignored keys. If the client connection contains any of them, the key and the corresponding value will be removed silently during engine bootstrap and connection setup. Note that this rule is for server-side protection defined via administrators to prevent some essential configs from tampering but will not forbid users to set dynamic configurations via SET syntax. | set | 1.2.0 | | kyuubi.session.conf.profile | <undefined> | Specify a profile to load session-level configurations from `$KYUUBI_CONF_DIR/kyuubi-session-.conf`. This configuration will be ignored if the file does not exist. This configuration only takes effect when `kyuubi.session.conf.advisor` is set as `org.apache.kyuubi.session.FileSessionConfAdvisor`. | string | 1.7.0 | -| kyuubi.session.conf.restrict.list || A comma-separated list of restricted keys. If the client connection contains any of them, the connection will be rejected explicitly during engine bootstrap and connection setup. Note that this rule is for server-side protection defined via administrators to prevent some essential configs from tampering but will not forbid users to set dynamic configurations via SET syntax. | seq | 1.2.0 | +| kyuubi.session.conf.restrict.list || A comma-separated list of restricted keys. If the client connection contains any of them, the connection will be rejected explicitly during engine bootstrap and connection setup. Note that this rule is for server-side protection defined via administrators to prevent some essential configs from tampering but will not forbid users to set dynamic configurations via SET syntax. | set | 1.2.0 | +| kyuubi.session.engine.alive.max.failures | 3 | The maximum number of failures allowed for the engine. | int | 1.8.0 | | kyuubi.session.engine.alive.probe.enabled | false | Whether to enable the engine alive probe, it true, we will create a companion thrift client that keeps sending simple requests to check whether the engine is alive. | boolean | 1.6.0 | | kyuubi.session.engine.alive.probe.interval | PT10S | The interval for engine alive probe. | duration | 1.6.0 | | kyuubi.session.engine.alive.timeout | PT2M | The timeout for engine alive. If there is no alive probe success in the last timeout window, the engine will be marked as no-alive. | duration | 1.6.0 | | kyuubi.session.engine.check.interval | PT1M | The check interval for engine timeout | duration | 1.0.0 | +| kyuubi.session.engine.flink.fetch.timeout | <undefined> | Result fetch timeout for Flink engine. If the timeout is reached, the result fetch would be stopped and the current fetched would be returned. If no data are fetched, a TimeoutException would be thrown. | duration | 1.8.0 | | kyuubi.session.engine.flink.main.resource | <undefined> | The package used to create Flink SQL engine remote job. If it is undefined, Kyuubi will use the default | string | 1.4.0 | | kyuubi.session.engine.flink.max.rows | 1000000 | Max rows of Flink query results. For batch queries, rows exceeding the limit would be ignored. For streaming queries, the query would be canceled if the limit is reached. | int | 1.5.0 | | kyuubi.session.engine.hive.main.resource | <undefined> | The package used to create Hive engine remote job. If it is undefined, Kyuubi will use the default | string | 1.6.0 | @@ -477,10 +433,12 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co | kyuubi.session.engine.open.retry.wait | PT10S | How long to wait before retrying to open the engine after failure. | duration | 1.7.0 | | kyuubi.session.engine.share.level | USER | (deprecated) - Using kyuubi.engine.share.level instead | string | 1.0.0 | | kyuubi.session.engine.spark.main.resource | <undefined> | The package used to create Spark SQL engine remote application. If it is undefined, Kyuubi will use the default | string | 1.0.0 | +| kyuubi.session.engine.spark.max.initial.wait | PT1M | Max wait time for the initial connection to Spark engine. The engine will self-terminate no new incoming connection is established within this time. This setting only applies at the CONNECTION share level. 0 or negative means not to self-terminate. | duration | 1.8.0 | | kyuubi.session.engine.spark.max.lifetime | PT0S | Max lifetime for Spark engine, the engine will self-terminate when it reaches the end of life. 0 or negative means not to self-terminate. | duration | 1.6.0 | | kyuubi.session.engine.spark.progress.timeFormat | yyyy-MM-dd HH:mm:ss.SSS | The time format of the progress bar | string | 1.6.0 | | kyuubi.session.engine.spark.progress.update.interval | PT1S | Update period of progress bar. | duration | 1.6.0 | | kyuubi.session.engine.spark.showProgress | false | When true, show the progress bar in the Spark's engine log. | boolean | 1.6.0 | +| kyuubi.session.engine.startup.destroy.timeout | PT5S | Engine startup process destroy wait time, if the process does not stop after this time, force destroy instead. This configuration only takes effect when `kyuubi.session.engine.startup.waitCompletion=false`. | duration | 1.8.0 | | kyuubi.session.engine.startup.error.max.size | 8192 | During engine bootstrapping, if anderror occurs, using this config to limit the length of error message(characters). | int | 1.1.0 | | kyuubi.session.engine.startup.maxLogLines | 10 | The maximum number of engine log lines when errors occur during the engine startup phase. Note that this config effects on client-side to help track engine startup issues. | int | 1.4.0 | | kyuubi.session.engine.startup.waitCompletion | true | Whether to wait for completion after the engine starts. If false, the startup process will be destroyed after the engine is started. Note that only use it when the driver is not running locally, such as in yarn-cluster mode; Otherwise, the engine will be killed. | boolean | 1.5.0 | @@ -491,7 +449,7 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co | kyuubi.session.engine.trino.showProgress.debug | false | When true, show the progress debug info in the Trino engine log. | boolean | 1.6.0 | | kyuubi.session.group.provider | hadoop | A group provider plugin for Kyuubi Server. This plugin can provide primary group and groups information for different users or session configs. This config value should be a subclass of `org.apache.kyuubi.plugin.GroupProvider` which has a zero-arg constructor. Kyuubi provides the following built-in implementations:
    • hadoop: delegate the user group mapping to hadoop UserGroupInformation.
      • | string | 1.7.0 | | kyuubi.session.idle.timeout | PT6H | session idle timeout, it will be closed when it's not accessed for this duration | duration | 1.2.0 | -| kyuubi.session.local.dir.allow.list || The local dir list that are allowed to access by the kyuubi session application. End-users might set some parameters such as `spark.files` and it will upload some local files when launching the kyuubi engine, if the local dir allow list is defined, kyuubi will check whether the path to upload is in the allow list. Note that, if it is empty, there is no limitation for that. And please use absolute paths. | seq | 1.6.0 | +| kyuubi.session.local.dir.allow.list || The local dir list that are allowed to access by the kyuubi session application. End-users might set some parameters such as `spark.files` and it will upload some local files when launching the kyuubi engine, if the local dir allow list is defined, kyuubi will check whether the path to upload is in the allow list. Note that, if it is empty, there is no limitation for that. And please use absolute paths. | set | 1.6.0 | | kyuubi.session.name | <undefined> | A human readable name of the session and we use empty string by default. This name will be recorded in the event. Note that, we only apply this value from session conf. | string | 1.4.0 | | kyuubi.session.timeout | PT6H | (deprecated)session timeout, it will be closed when it's not accessed for this duration | duration | 1.0.0 | | kyuubi.session.user.sign.enabled | false | Whether to verify the integrity of session user name on the engine side, e.g. Authz plugin in Spark. | boolean | 1.7.0 | @@ -503,26 +461,34 @@ You can configure the Kyuubi properties in `$KYUUBI_HOME/conf/kyuubi-defaults.co | kyuubi.spnego.keytab | <undefined> | Keytab file for SPNego principal | string | 1.6.0 | | kyuubi.spnego.principal | <undefined> | SPNego service principal, typical value would look like HTTP/_HOST@EXAMPLE.COM. SPNego service principal would be used when restful Kerberos security is enabled. This needs to be set only if SPNEGO is to be used in authentication. | string | 1.6.0 | +### Yarn + +| Key | Default | Meaning | Type | Since | +|---------------------------|---------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------|-------| +| kyuubi.yarn.user.admin | yarn | When kyuubi.yarn.user.strategy is set to ADMIN, use this admin user to construct YARN client for application management, e.g. kill application. | string | 1.8.0 | +| kyuubi.yarn.user.strategy | NONE | Determine which user to use to construct YARN client for application management, e.g. kill application. Options:
      • NONE: use Kyuubi server user.
      • ADMIN: use admin user configured in `kyuubi.yarn.user.admin`.
      • OWNER: use session user, typically is application owner.
      | string | 1.8.0 | + ### Zookeeper -| Key | Default | Meaning | Type | Since | -|--------------------------------------------------|--------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------|-------| -| kyuubi.zookeeper.embedded.client.port | 2181 | clientPort for the embedded ZooKeeper server to listen for client connections, a client here could be Kyuubi server, engine, and JDBC client | int | 1.2.0 | -| kyuubi.zookeeper.embedded.client.port.address | <undefined> | clientPortAddress for the embedded ZooKeeper server to | string | 1.2.0 | -| kyuubi.zookeeper.embedded.data.dir | embedded_zookeeper | dataDir for the embedded zookeeper server where stores the in-memory database snapshots and, unless specified otherwise, the transaction log of updates to the database. | string | 1.2.0 | -| kyuubi.zookeeper.embedded.data.log.dir | embedded_zookeeper | dataLogDir for the embedded ZooKeeper server where writes the transaction log . | string | 1.2.0 | -| kyuubi.zookeeper.embedded.directory | embedded_zookeeper | The temporary directory for the embedded ZooKeeper server | string | 1.0.0 | -| kyuubi.zookeeper.embedded.max.client.connections | 120 | maxClientCnxns for the embedded ZooKeeper server to limit the number of concurrent connections of a single client identified by IP address | int | 1.2.0 | -| kyuubi.zookeeper.embedded.max.session.timeout | 60000 | maxSessionTimeout in milliseconds for the embedded ZooKeeper server will allow the client to negotiate. Defaults to 20 times the tickTime | int | 1.2.0 | -| kyuubi.zookeeper.embedded.min.session.timeout | 6000 | minSessionTimeout in milliseconds for the embedded ZooKeeper server will allow the client to negotiate. Defaults to 2 times the tickTime | int | 1.2.0 | -| kyuubi.zookeeper.embedded.port | 2181 | The port of the embedded ZooKeeper server | int | 1.0.0 | -| kyuubi.zookeeper.embedded.tick.time | 3000 | tickTime in milliseconds for the embedded ZooKeeper server | int | 1.2.0 | +| Key | Default | Meaning | Type | Since | +|--------------------------------------------------|--------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------|-------| +| kyuubi.zookeeper.embedded.client.port | 2181 | clientPort for the embedded ZooKeeper server to listen for client connections, a client here could be Kyuubi server, engine, and JDBC client | int | 1.2.0 | +| kyuubi.zookeeper.embedded.client.port.address | <undefined> | clientPortAddress for the embedded ZooKeeper server to | string | 1.2.0 | +| kyuubi.zookeeper.embedded.client.use.hostname | false | When true, embedded Zookeeper prefer to bind hostname, otherwise, ip address. | boolean | 1.7.2 | +| kyuubi.zookeeper.embedded.data.dir | embedded_zookeeper | dataDir for the embedded zookeeper server where stores the in-memory database snapshots and, unless specified otherwise, the transaction log of updates to the database. If it is a relative path, it is resolved relative to KYUUBI_HOME. | string | 1.2.0 | +| kyuubi.zookeeper.embedded.data.log.dir | embedded_zookeeper | dataLogDir for the embedded ZooKeeper server where writes the transaction log. If it is a relative path, it is resolved relative to KYUUBI_HOME. | string | 1.2.0 | +| kyuubi.zookeeper.embedded.directory | embedded_zookeeper | The temporary directory for the embedded ZooKeeper server. If it is a relative path, it is resolved relative to KYUUBI_HOME. | string | 1.0.0 | +| kyuubi.zookeeper.embedded.max.client.connections | 120 | maxClientCnxns for the embedded ZooKeeper server to limit the number of concurrent connections of a single client identified by IP address | int | 1.2.0 | +| kyuubi.zookeeper.embedded.max.session.timeout | 60000 | maxSessionTimeout in milliseconds for the embedded ZooKeeper server will allow the client to negotiate. Defaults to 20 times the tickTime | int | 1.2.0 | +| kyuubi.zookeeper.embedded.min.session.timeout | 6000 | minSessionTimeout in milliseconds for the embedded ZooKeeper server will allow the client to negotiate. Defaults to 2 times the tickTime | int | 1.2.0 | +| kyuubi.zookeeper.embedded.port | 2181 | The port of the embedded ZooKeeper server | int | 1.0.0 | +| kyuubi.zookeeper.embedded.tick.time | 3000 | tickTime in milliseconds for the embedded ZooKeeper server | int | 1.2.0 | ## Spark Configurations ### Via spark-defaults.conf -Setting them in `$SPARK_HOME/conf/spark-defaults.conf` supplies with default values for SQL engine application. Available properties can be found at Spark official online documentation for [Spark Configurations](http://spark.apache.org/docs/latest/configuration.html) +Setting them in `$SPARK_HOME/conf/spark-defaults.conf` supplies with default values for SQL engine application. Available properties can be found at Spark official online documentation for [Spark Configurations](https://spark.apache.org/docs/latest/configuration.html) ### Via kyuubi-defaults.conf @@ -533,13 +499,13 @@ Setting them in `$KYUUBI_HOME/conf/kyuubi-defaults.conf` supplies with default v Setting them in the JDBC Connection URL supplies session-specific for each SQL engine. For example: ```jdbc:hive2://localhost:10009/default;#spark.sql.shuffle.partitions=2;spark.executor.memory=5g``` - **Runtime SQL Configuration** - - For [Runtime SQL Configurations](http://spark.apache.org/docs/latest/configuration.html#runtime-sql-configuration), they will take affect every time + - For [Runtime SQL Configurations](https://spark.apache.org/docs/latest/configuration.html#runtime-sql-configuration), they will take affect every time - **Static SQL and Spark Core Configuration** - - For [Static SQL Configurations](http://spark.apache.org/docs/latest/configuration.html#static-sql-configuration) and other spark core configs, e.g. `spark.executor.memory`, they will take effect if there is no existing SQL engine application. Otherwise, they will just be ignored + - For [Static SQL Configurations](https://spark.apache.org/docs/latest/configuration.html#static-sql-configuration) and other spark core configs, e.g. `spark.executor.memory`, they will take effect if there is no existing SQL engine application. Otherwise, they will just be ignored ### Via SET Syntax -Please refer to the Spark official online documentation for [SET Command](http://spark.apache.org/docs/latest/sql-ref-syntax-aux-conf-mgmt-set.html) +Please refer to the Spark official online documentation for [SET Command](https://spark.apache.org/docs/latest/sql-ref-syntax-aux-conf-mgmt-set.html) ## Flink Configurations @@ -568,80 +534,42 @@ Setting them in the JDBC Connection URL supplies session-specific for each SQL e Please refer to the Flink official online documentation for [SET Statements](https://nightlies.apache.org/flink/flink-docs-stable/docs/dev/table/sql/set/) -## Logging +## Trino Configurations -Kyuubi uses [log4j](https://logging.apache.org/log4j/2.x/) for logging. You can configure it using `$KYUUBI_HOME/conf/log4j2.xml`. +### Via config.properties -```bash - - - - - - - - rest-audit.log - rest-audit-%d{yyyy-MM-dd}-%i.log - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Setting them in `$TRINO_HOME/etc/config.properties` supplies with default values for SQL engine application. Available properties can be found at Trino official online documentation for [Trino Configurations](https://trino.io/docs/current/admin/properties.html) + +### Via kyuubi-defaults.conf + +Setting them in `$KYUUBI_HOME/conf/kyuubi-defaults.conf` supplies with default values for SQL engine application too. You can use properties with the additional prefix `trino.` to override settings in `$TRINO_HOME/etc/config.properties`. + +For example: + +``` +trino.query_max_stage_count 500 +trino.parse_decimal_literals_as_double true ``` +The below options in `kyuubi-defaults.conf` will set `query_max_stage_count: 500` and `parse_decimal_literals_as_double: true` into trino session properties. + +### Via JDBC Connection URL + +Setting them in the JDBC Connection URL supplies session-specific for each SQL engine. For example: ```jdbc:hive2://localhost:10009/default;#trino.query_max_stage_count=500;trino.parse_decimal_literals_as_double=true``` + +### Via SET Statements + +Please refer to the Trino official online documentation for [SET Statements](https://trino.io/docs/current/sql/set-session.html) + +## Logging + +Kyuubi uses [log4j](https://logging.apache.org/log4j/2.x/) for logging. You can configure it using `$KYUUBI_HOME/conf/log4j2.xml`, see `$KYUUBI_HOME/conf/log4j2.xml.template` as an example. + ## Other Configurations ### Hadoop Configurations -Specifying `HADOOP_CONF_DIR` to the directory containing Hadoop configuration files or treating them as Spark properties with a `spark.hadoop.` prefix. Please refer to the Spark official online documentation for [Inheriting Hadoop Cluster Configuration](http://spark.apache.org/docs/latest/configuration.html#inheriting-hadoop-cluster-configuration). Also, please refer to the [Apache Hadoop](http://hadoop.apache.org)'s online documentation for an overview on how to configure Hadoop. +Specifying `HADOOP_CONF_DIR` to the directory containing Hadoop configuration files or treating them as Spark properties with a `spark.hadoop.` prefix. Please refer to the Spark official online documentation for [Inheriting Hadoop Cluster Configuration](https://spark.apache.org/docs/latest/configuration.html#inheriting-hadoop-cluster-configuration). Also, please refer to the [Apache Hadoop](https://hadoop.apache.org)'s online documentation for an overview on how to configure Hadoop. ### Hive Configurations diff --git a/docs/connector/flink/index.rst b/docs/connector/flink/index.rst index c9d91091f..e7d40fd43 100644 --- a/docs/connector/flink/index.rst +++ b/docs/connector/flink/index.rst @@ -19,6 +19,6 @@ Connectors For Flink SQL Query Engine .. toctree:: :maxdepth: 2 - flink_table_store + paimon hudi iceberg diff --git a/docs/connector/flink/flink_table_store.rst b/docs/connector/flink/paimon.rst similarity index 51% rename from docs/connector/flink/flink_table_store.rst rename to docs/connector/flink/paimon.rst index 14c576bf3..b67101488 100644 --- a/docs/connector/flink/flink_table_store.rst +++ b/docs/connector/flink/paimon.rst @@ -13,57 +13,56 @@ See the License for the specific language governing permissions and limitations under the License. -`Flink Table Store`_ -========== +`Apache Paimon (Incubating)`_ +============================= -Flink Table Store is a unified storage to build dynamic tables for both streaming and batch processing in Flink, -supporting high-speed data ingestion and timely data query. +Apache Paimon (Incubating) is a streaming data lake platform that supports high-speed data ingestion, change data tracking, and efficient real-time analytics. .. tip:: - This article assumes that you have mastered the basic knowledge and operation of `Flink Table Store`_. - For the knowledge about Flink Table Store not mentioned in this article, + This article assumes that you have mastered the basic knowledge and operation of `Apache Paimon (Incubating)`_. + For the knowledge not mentioned in this article, you can obtain it from its `Official Documentation`_. -By using kyuubi, we can run SQL queries towards Flink Table Store which is more -convenient, easy to understand, and easy to expand than directly using -flink to manipulate Flink Table Store. +By using kyuubi, we can run SQL queries towards Apache Paimon (Incubating) which is more +convenient, easy to understand, and easy to expand than directly using flink. -Flink Table Store Integration -------------------- +Apache Paimon (Incubating) Integration +-------------------------------------- -To enable the integration of kyuubi flink sql engine and Flink Table Store, you need to: +To enable the integration of kyuubi flink sql engine and Apache Paimon (Incubating), you need to: -- Referencing the Flink Table Store :ref:`dependencies` +- Referencing the Apache Paimon (Incubating) :ref:`dependencies` -.. _flink-table-store-deps: +.. _flink-paimon-deps: Dependencies ************ -The **classpath** of kyuubi flink sql engine with Flink Table Store supported consists of +The **classpath** of kyuubi flink sql engine with Apache Paimon (Incubating) supported consists of 1. kyuubi-flink-sql-engine-\ |release|\ _2.12.jar, the engine jar deployed with Kyuubi distributions 2. a copy of flink distribution -3. flink-table-store-dist-.jar (example: flink-table-store-dist-0.2.jar), which can be found in the `Maven Central`_ +3. paimon-flink-.jar (example: paimon-flink-1.16-0.4-SNAPSHOT.jar), which can be found in the `Apache Paimon (Incubating) Supported Engines Flink`_ +4. flink-shaded-hadoop-2-uber-.jar, which code can be found in the `Pre-bundled Hadoop Jar`_ -In order to make the Flink Table Store packages visible for the runtime classpath of engines, we can use these methods: +In order to make the Apache Paimon (Incubating) packages visible for the runtime classpath of engines, you need to: -1. Put the Flink Table Store packages into ``$FLINK_HOME/lib`` directly +1. Put the Apache Paimon (Incubating) packages into ``$FLINK_HOME/lib`` directly 2. Setting the HADOOP_CLASSPATH environment variable or copy the `Pre-bundled Hadoop Jar`_ to flink/lib. .. warning:: - Please mind the compatibility of different Flink Table Store and Flink versions, which can be confirmed on the page of `Flink Table Store multi engine support`_. + Please mind the compatibility of different Apache Paimon (Incubating) and Flink versions, which can be confirmed on the page of `Apache Paimon (Incubating) multi engine support`_. -Flink Table Store Operations ------------------- +Apache Paimon (Incubating) Operations +------------------------------------- Taking ``CREATE CATALOG`` as a example, .. code-block:: sql CREATE CATALOG my_catalog WITH ( - 'type'='table-store', - 'warehouse'='hdfs://nn:8020/warehouse/path' -- or 'file:///tmp/foo/bar' + 'type'='paimon', + 'warehouse'='file:/tmp/paimon' ); USE CATALOG my_catalog; @@ -104,8 +103,8 @@ Taking ``Rescale Bucket`` as a example, INSERT OVERWRITE my_table PARTITION (dt = '2022-01-01'); -.. _Flink Table Store: https://nightlies.apache.org/flink/flink-table-store-docs-stable/ -.. _Official Documentation: https://nightlies.apache.org/flink/flink-table-store-docs-stable/ -.. _Maven Central: https://mvnrepository.com/artifact/org.apache.flink/flink-table-store-dist -.. _Pre-bundled Hadoop Jar: https://flink.apache.org/downloads.html -.. _Flink Table Store multi engine support: https://nightlies.apache.org/flink/flink-table-store-docs-stable/docs/engines/overview/ +.. _Apache Paimon (Incubating): https://paimon.apache.org/ +.. _Official Documentation: https://paimon.apache.org/docs/master/ +.. _Apache Paimon (Incubating) Supported Engines Flink: https://paimon.apache.org/docs/master/engines/flink/#preparing-paimon-jar-file +.. _Pre-bundled Hadoop Jar: https://flink.apache.org/downloads/#additional-components +.. _Apache Paimon (Incubating) multi engine support: https://paimon.apache.org/docs/master/engines/overview/ diff --git a/docs/connector/hive/index.rst b/docs/connector/hive/index.rst index 2b2b863a6..d96f8b041 100644 --- a/docs/connector/hive/index.rst +++ b/docs/connector/hive/index.rst @@ -19,4 +19,5 @@ Connectors for Hive SQL Query Engine .. toctree:: :maxdepth: 2 + paimon iceberg diff --git a/docs/connector/hive/paimon.rst b/docs/connector/hive/paimon.rst new file mode 100644 index 000000000..000d2d7e8 --- /dev/null +++ b/docs/connector/hive/paimon.rst @@ -0,0 +1,100 @@ +.. Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +`Apache Paimon (Incubating)`_ +========== + +Apache Paimon(incubating) is a streaming data lake platform that supports high-speed data ingestion, change data tracking and efficient real-time analytics. + +.. tip:: + This article assumes that you have mastered the basic knowledge and operation of `Apache Paimon (Incubating)`_. + For the knowledge about Apache Paimon (Incubating) not mentioned in this article, + you can obtain it from its `Official Documentation`_. + +By using Kyuubi, we can run SQL queries towards Apache Paimon (Incubating) which is more +convenient, easy to understand, and easy to expand than directly using +Hive to manipulate Apache Paimon (Incubating). + +Apache Paimon (Incubating) Integration +------------------- + +To enable the integration of kyuubi hive sql engine and Apache Paimon (Incubating), you need to: + +- Referencing the Apache Paimon (Incubating) :ref:`dependencies` +- Setting the environment variable :ref:`configurations` + +.. _hive-paimon-deps: + +Dependencies +************ + +The **classpath** of kyuubi hive sql engine with Iceberg supported consists of + +1. kyuubi-hive-sql-engine-\ |release|\ _2.12.jar, the engine jar deployed with Kyuubi distributions +2. a copy of hive distribution +3. paimon-hive-connector--.jar (example: paimon-hive-connector-3.1-0.4-SNAPSHOT.jar), which can be found in the `Apache Paimon (Incubating) Supported Engines Hive`_ + +In order to make the Hive packages visible for the runtime classpath of engines, we can use one of these methods: + +1. You can create an auxlib folder under the root directory of Hive, and copy paimon-hive-connector-3.1-.jar into auxlib. +2. Execute ADD JAR statement in the Kyuubi to add dependencies to Hive’s auxiliary classpath. For example: + +.. code-block:: sql + + ADD JAR /path/to/paimon-hive-connector-3.1-.jar; + +.. warning:: + The second method is not recommended. If you’re using the MR execution engine and running a join statement, you may be faced with the exception + ``org.apache.hive.com.esotericsoftware.kryo.kryoexception: unable to find class.`` + +.. warning:: + Please mind the compatibility of different Apache Paimon (Incubating) and Hive versions, which can be confirmed on the page of `Apache Paimon (Incubating) multi engine support`_. + +.. _hive-paimon-conf: + +Configurations +************** + +If you are using HDFS, make sure that the environment variable HADOOP_HOME or HADOOP_CONF_DIR is set. + +Apache Paimon (Incubating) Operations +------------------ + +Apache Paimon (Incubating) only supports only reading table store tables through Hive. +A common scenario is to write data with Spark or Flink and read data with Hive. +You can follow this document `Apache Paimon (Incubating) Quick Start with Paimon Hive Catalog`_ to write data to a table which can also be accessed directly from Hive. +and then use Kyuubi Hive SQL engine to query the table with the following SQL ``SELECT`` statement. + +Taking ``Query Data`` as an example, + +.. code-block:: sql + + SELECT a, b FROM test_table ORDER BY a; + +Taking ``Query External Table`` as an example, + +.. code-block:: sql + + CREATE EXTERNAL TABLE external_test_table + STORED BY 'org.apache.paimon.hive.PaimonStorageHandler' + LOCATION '/path/to/table/store/warehouse/default.db/test_table'; + + SELECT a, b FROM test_table ORDER BY a; + +.. _Apache Paimon (Incubating): https://paimon.apache.org/ +.. _Official Documentation: https://paimon.apache.org/docs/master/ +.. _Apache Paimon (Incubating) Quick Start with Paimon Hive Catalog: https://paimon.apache.org/docs/master/engines/hive/#quick-start-with-paimon-hive-catalog +.. _Apache Paimon (Incubating) Supported Engines Hive: https://paimon.apache.org/docs/master/engines/hive/ +.. _Apache Paimon (Incubating) multi engine support: https://paimon.apache.org/docs/master/engines/overview/ diff --git a/docs/connector/spark/flink_table_store.rst b/docs/connector/spark/flink_table_store.rst deleted file mode 100644 index ee4c2b352..000000000 --- a/docs/connector/spark/flink_table_store.rst +++ /dev/null @@ -1,90 +0,0 @@ -.. Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - -.. http://www.apache.org/licenses/LICENSE-2.0 - -.. Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -`Flink Table Store`_ -========== - -Flink Table Store is a unified storage to build dynamic tables for both streaming and batch processing in Flink, -supporting high-speed data ingestion and timely data query. - -.. tip:: - This article assumes that you have mastered the basic knowledge and operation of `Flink Table Store`_. - For the knowledge about Flink Table Store not mentioned in this article, - you can obtain it from its `Official Documentation`_. - -By using kyuubi, we can run SQL queries towards Flink Table Store which is more -convenient, easy to understand, and easy to expand than directly using -spark to manipulate Flink Table Store. - -Flink Table Store Integration -------------------- - -To enable the integration of kyuubi spark sql engine and Flink Table Store through -Apache Spark Datasource V2 and Catalog APIs, you need to: - -- Referencing the Flink Table Store :ref:`dependencies` -- Setting the spark extension and catalog :ref:`configurations` - -.. _spark-flink-table-store-deps: - -Dependencies -************ - -The **classpath** of kyuubi spark sql engine with Flink Table Store supported consists of - -1. kyuubi-spark-sql-engine-\ |release|\ _2.12.jar, the engine jar deployed with Kyuubi distributions -2. a copy of spark distribution -3. flink-table-store-spark-.jar (example: flink-table-store-spark-0.2.jar), which can be found in the `Maven Central`_ - -In order to make the Flink Table Store packages visible for the runtime classpath of engines, we can use one of these methods: - -1. Put the Flink Table Store packages into ``$SPARK_HOME/jars`` directly -2. Set ``spark.jars=/path/to/flink-table-store-spark`` - -.. warning:: - Please mind the compatibility of different Flink Table Store and Spark versions, which can be confirmed on the page of `Flink Table Store multi engine support`_. - -.. _spark-flink-table-store-conf: - -Configurations -************** - -To activate functionality of Flink Table Store, we can set the following configurations: - -.. code-block:: properties - - spark.sql.catalog.tablestore=org.apache.flink.table.store.spark.SparkCatalog - spark.sql.catalog.tablestore.warehouse=file:/tmp/warehouse - -Flink Table Store Operations ------------------- - -Flink Table Store supports reading table store tables through Spark. -A common scenario is to write data with Flink and read data with Spark. -You can follow this document `Flink Table Store Quick Start`_ to write data to a table store table -and then use kyuubi spark sql engine to query the table with the following SQL ``SELECT`` statement. - - -.. code-block:: sql - - select * from table_store.default.word_count; - - - -.. _Flink Table Store: https://nightlies.apache.org/flink/flink-table-store-docs-stable/ -.. _Flink Table Store Quick Start: https://nightlies.apache.org/flink/flink-table-store-docs-stable/docs/try-table-store/quick-start/ -.. _Official Documentation: https://nightlies.apache.org/flink/flink-table-store-docs-stable/ -.. _Maven Central: https://mvnrepository.com/artifact/org.apache.flink -.. _Flink Table Store multi engine support: https://nightlies.apache.org/flink/flink-table-store-docs-stable/docs/engines/overview/ diff --git a/docs/connector/spark/index.rst b/docs/connector/spark/index.rst index 790e804f2..d1503443c 100644 --- a/docs/connector/spark/index.rst +++ b/docs/connector/spark/index.rst @@ -23,7 +23,7 @@ By default, it provides accessibility to hive warehouses with various file forma supported, such as parquet, orc, json, etc. Also，it can easily integrate with other third-party libraries, such as Hudi, -Iceberg, Delta Lake, Kudu, Flink Table Store, HBase，Cassandra, etc. +Iceberg, Delta Lake, Kudu, Apache Paimon (Incubating), HBase，Cassandra, etc. We also provide sample data sources like TDC-DS, TPC-H for testing and benchmarking purpose. @@ -37,7 +37,7 @@ purpose. iceberg kudu hive - flink_table_store + paimon tidb tpcds tpch diff --git a/docs/connector/spark/paimon.rst b/docs/connector/spark/paimon.rst new file mode 100644 index 000000000..14e741955 --- /dev/null +++ b/docs/connector/spark/paimon.rst @@ -0,0 +1,110 @@ +.. Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +`Apache Paimon (Incubating)`_ +========== + +Apache Paimon(incubating) is a streaming data lake platform that supports high-speed data ingestion, change data tracking and efficient real-time analytics. + +.. tip:: + This article assumes that you have mastered the basic knowledge and operation of `Apache Paimon (Incubating)`_. + For the knowledge about Apache Paimon (Incubating) not mentioned in this article, + you can obtain it from its `Official Documentation`_. + +By using kyuubi, we can run SQL queries towards Apache Paimon (Incubating) which is more +convenient, easy to understand, and easy to expand than directly using +spark to manipulate Apache Paimon (Incubating). + +Apache Paimon (Incubating) Integration +------------------- + +To enable the integration of kyuubi spark sql engine and Apache Paimon (Incubating), you need to set the following configurations: + +- Referencing the Apache Paimon (Incubating) :ref:`dependencies` +- Setting the spark extension and catalog :ref:`configurations` + +.. _spark-paimon-deps: + +Dependencies +************ + +The **classpath** of kyuubi spark sql engine with Apache Paimon (Incubating) consists of + +1. kyuubi-spark-sql-engine-\ |release|\ _2.12.jar, the engine jar deployed with Kyuubi distributions +2. a copy of spark distribution +3. paimon-spark-.jar (example: paimon-spark-3.3-0.4-20230323.002035-5.jar), which can be found in the `Apache Paimon (Incubating) Supported Engines Spark3`_ + +In order to make the Apache Paimon (Incubating) packages visible for the runtime classpath of engines, we can use one of these methods: + +1. Put the Apache Paimon (Incubating) packages into ``$SPARK_HOME/jars`` directly +2. Set ``spark.jars=/path/to/paimon-spark-.jar`` + +.. warning:: + Please mind the compatibility of different Apache Paimon (Incubating) and Spark versions, which can be confirmed on the page of `Apache Paimon (Incubating) multi engine support`_. + +.. _spark-paimon-conf: + +Configurations +************** + +To activate functionality of Apache Paimon (Incubating), we can set the following configurations: + +.. code-block:: properties + + spark.sql.catalog.paimon=org.apache.paimon.spark.SparkCatalog + spark.sql.catalog.paimon.warehouse=file:/tmp/paimon + +Apache Paimon (Incubating) Operations +------------------ + + +Taking ``CREATE NAMESPACE`` as a example, + +.. code-block:: sql + + CREATE DATABASE paimon.default; + USE paimon.default; + +Taking ``CREATE TABLE`` as a example, + +.. code-block:: sql + + create table my_table ( + k int, + v string + ) tblproperties ( + 'primary-key' = 'k' + ); + +Taking ``SELECT`` as a example, + +.. code-block:: sql + + SELECT * FROM my_table; + + +Taking ``INSERT`` as a example, + +.. code-block:: sql + + INSERT INTO my_table VALUES (1, 'Hi Again'), (3, 'Test'); + + + + +.. _Apache Paimon (Incubating): https://paimon.apache.org/ +.. _Official Documentation: https://paimon.apache.org/docs/master/ +.. _Apache Paimon (Incubating) Supported Engines Spark3: https://paimon.apache.org/docs/master/engines/spark3/ +.. _Apache Paimon (Incubating) multi engine support: https://paimon.apache.org/docs/master/engines/overview/ diff --git a/docs/connector/trino/flink_table_store.rst b/docs/connector/trino/flink_table_store.rst deleted file mode 100644 index 8dd0c4061..000000000 --- a/docs/connector/trino/flink_table_store.rst +++ /dev/null @@ -1,94 +0,0 @@ -.. Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - -.. http://www.apache.org/licenses/LICENSE-2.0 - -.. Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -`Flink Table Store`_ -========== - -Flink Table Store is a unified storage to build dynamic tables for both streaming and batch processing in Flink, -supporting high-speed data ingestion and timely data query. - -.. tip:: - This article assumes that you have mastered the basic knowledge and operation of `Flink Table Store`_. - For the knowledge about Flink Table Store not mentioned in this article, - you can obtain it from its `Official Documentation`_. - -By using kyuubi, we can run SQL queries towards Flink Table Store which is more -convenient, easy to understand, and easy to expand than directly using -trino to manipulate Flink Table Store. - -Flink Table Store Integration -------------------- - -To enable the integration of kyuubi trino sql engine and Flink Table Store, you need to: - -- Referencing the Flink Table Store :ref:`dependencies` -- Setting the trino extension and catalog :ref:`configurations` - -.. _trino-flink-table-store-deps: - -Dependencies -************ - -The **classpath** of kyuubi trino sql engine with Flink Table Store supported consists of - -1. kyuubi-trino-sql-engine-\ |release|\ _2.12.jar, the engine jar deployed with Kyuubi distributions -2. a copy of trino distribution -3. flink-table-store-trino-.jar (example: flink-table-store-trino-0.2.jar), which code can be found in the `Source Code`_ -4. flink-shaded-hadoop-2-uber-2.8.3-10.0.jar, which code can be found in the `Pre-bundled Hadoop 2.8.3`_ - -In order to make the Flink Table Store packages visible for the runtime classpath of engines, we can use these methods: - -1. Build the flink-table-store-trino-.jar by reference to `Flink Table Store Trino README`_ -2. Put the flink-table-store-trino-.jar and flink-shaded-hadoop-2-uber-2.8.3-10.0.jar packages into ``$TRINO_SERVER_HOME/plugin/tablestore`` directly - -.. warning:: - Please mind the compatibility of different Flink Table Store and Trino versions, which can be confirmed on the page of `Flink Table Store multi engine support`_. - -.. _trino-flink-table-store-conf: - -Configurations -************** - -To activate functionality of Flink Table Store, we can set the following configurations: - -Catalogs are registered by creating a catalog properties file in the $TRINO_SERVER_HOME/etc/catalog directory. -For example, create $TRINO_SERVER_HOME/etc/catalog/tablestore.properties with the following contents to mount the tablestore connector as the tablestore catalog: - -.. code-block:: properties - - connector.name=tablestore - warehouse=file:///tmp/warehouse - -Flink Table Store Operations ------------------- - -Flink Table Store supports reading table store tables through Trino. -A common scenario is to write data with Flink and read data with Trino. -You can follow this document `Flink Table Store Quick Start`_ to write data to a table store table -and then use kyuubi trino sql engine to query the table with the following SQL ``SELECT`` statement. - - -.. code-block:: sql - - SELECT * FROM tablestore.default.t1 - - -.. _Flink Table Store: https://nightlies.apache.org/flink/flink-table-store-docs-stable/ -.. _Flink Table Store Quick Start: https://nightlies.apache.org/flink/flink-table-store-docs-stable/docs/try-table-store/quick-start/ -.. _Official Documentation: https://nightlies.apache.org/flink/flink-table-store-docs-stable/ -.. _Source Code: https://github.com/JingsongLi/flink-table-store-trino -.. _Flink Table Store multi engine support: https://nightlies.apache.org/flink/flink-table-store-docs-stable/docs/engines/overview/ -.. _Pre-bundled Hadoop 2.8.3: https://repo.maven.apache.org/maven2/org/apache/flink/flink-shaded-hadoop-2-uber/2.8.3-10.0/flink-shaded-hadoop-2-uber-2.8.3-10.0.jar -.. _Flink Table Store Trino README: https://github.com/JingsongLi/flink-table-store-trino#readme diff --git a/docs/connector/trino/hudi.rst b/docs/connector/trino/hudi.rst new file mode 100644 index 000000000..5c965a0b6 --- /dev/null +++ b/docs/connector/trino/hudi.rst @@ -0,0 +1,80 @@ +.. Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +`Hudi`_ +======== + +Apache Hudi (pronounced “hoodie”) is the next generation streaming data lake platform. +Apache Hudi brings core warehouse and database functionality directly to a data lake. + +.. tip:: + This article assumes that you have mastered the basic knowledge and operation of `Hudi`_. + For the knowledge about Hudi not mentioned in this article, + you can obtain it from its `Official Documentation`_. + +By using Kyuubi, we can run SQL queries towards Hudi which is more convenient, easy to understand, +and easy to expand than directly using Trino to manipulate Hudi. + +Hudi Integration +---------------- + +To enable the integration of Kyuubi Trino SQL engine and Hudi, you need to: + +- Setting the Trino extension and catalog :ref:`configurations` + +.. _trino-hudi-conf: + +Configurations +************** + +Catalogs are registered by creating a file of catalog properties in the `$TRINO_SERVER_HOME/etc/catalog` directory. +For example, we can create a `$TRINO_SERVER_HOME/etc/catalog/hudi.properties` with the following contents to mount the Hudi connector as a Hudi catalog: + +.. code-block:: properties + + connector.name=hudi + hive.metastore.uri=thrift://example.net:9083 + +Note: You need to replace $TRINO_SERVER_HOME above to your Trino server home path like `/opt/trino-server-406`. + +More configuration properties can be found in the `Hudi connector in Trino document`_. + +.. tip:: + Trino version 398 or higher, it is recommended to use the Hudi connector. + You don't need to install any dependencies in version 398 or higher. + +Hudi Operations +--------------- +The globally available and read operation statements are supported in Trino. +These statements can be found in `Trino SQL Support`_. +Currently, Trino cannot write data to a Hudi table. +A common scenario is to write data with Spark/Flink and read data with Trino. +You can use the Kyuubi Trino SQL engine to query the table with the following SQL ``SELECT`` statement. + +Taking ``Query Data`` as a example, + +.. code-block:: sql + + USE example.example_schema; + + SELECT symbol, max(ts) + FROM stock_ticks_cow + GROUP BY symbol + HAVING symbol = 'GOOG'; + +.. _Hudi: https://hudi.apache.org/ +.. _Official Documentation: https://hudi.apache.org/docs/overview +.. _Hudi connector in Trino document: https://trino.io/docs/current/connector/hudi.html +.. _Trino SQL Support: https://trino.io/docs/current/language/sql-support.html# diff --git a/docs/connector/trino/index.rst b/docs/connector/trino/index.rst index a5c5675ce..290966a5c 100644 --- a/docs/connector/trino/index.rst +++ b/docs/connector/trino/index.rst @@ -19,5 +19,6 @@ Connectors For Trino SQL Engine .. toctree:: :maxdepth: 2 - flink_table_store + paimon + hudi iceberg \ No newline at end of file diff --git a/docs/connector/trino/paimon.rst b/docs/connector/trino/paimon.rst new file mode 100644 index 000000000..5ac892234 --- /dev/null +++ b/docs/connector/trino/paimon.rst @@ -0,0 +1,92 @@ +.. Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +`Apache Paimon (Incubating)`_ +========== + +Apache Paimon(incubating) is a streaming data lake platform that supports high-speed data ingestion, change data tracking and efficient real-time analytics. + +.. tip:: + This article assumes that you have mastered the basic knowledge and operation of `Apache Paimon (Incubating)`_. + For the knowledge about Apache Paimon (Incubating) not mentioned in this article, + you can obtain it from its `Official Documentation`_. + +By using kyuubi, we can run SQL queries towards Apache Paimon (Incubating) which is more +convenient, easy to understand, and easy to expand than directly using +trino to manipulate Apache Paimon (Incubating). + +Apache Paimon (Incubating) Integration +------------------- + +To enable the integration of kyuubi trino sql engine and Apache Paimon (Incubating), you need to: + +- Referencing the Apache Paimon (Incubating) :ref:`dependencies` +- Setting the trino extension and catalog :ref:`configurations` + +.. _trino-paimon-deps: + +Dependencies +************ + +The **classpath** of kyuubi trino sql engine with Apache Paimon (Incubating) supported consists of + +1. kyuubi-trino-sql-engine-\ |release|\ _2.12.jar, the engine jar deployed with Kyuubi distributions +2. a copy of trino distribution +3. paimon-trino-.jar (example: paimon-trino-0.2.jar), which code can be found in the `Source Code`_ +4. flink-shaded-hadoop-2-uber-.jar, which code can be found in the `Pre-bundled Hadoop`_ + +In order to make the Apache Paimon (Incubating) packages visible for the runtime classpath of engines, you need to: + +1. Build the paimon-trino-.jar by reference to `Apache Paimon (Incubating) Trino README`_ +2. Put the paimon-trino-.jar and flink-shaded-hadoop-2-uber-.jar packages into ``$TRINO_SERVER_HOME/plugin/tablestore`` directly + +.. warning:: + Please mind the compatibility of different Apache Paimon (Incubating) and Trino versions, which can be confirmed on the page of `Apache Paimon (Incubating) multi engine support`_. + +.. _trino-paimon-conf: + +Configurations +************** + +To activate functionality of Apache Paimon (Incubating), we can set the following configurations: + +Catalogs are registered by creating a catalog properties file in the $TRINO_SERVER_HOME/etc/catalog directory. +For example, create $TRINO_SERVER_HOME/etc/catalog/tablestore.properties with the following contents to mount the tablestore connector as the tablestore catalog: + +.. code-block:: properties + + connector.name=tablestore + warehouse=file:///tmp/warehouse + +Apache Paimon (Incubating) Operations +------------------ + +Apache Paimon (Incubating) supports reading table store tables through Trino. +A common scenario is to write data with Spark or Flink and read data with Trino. +You can follow this document `Apache Paimon (Incubating) Engines Flink Quick Start`_ to write data to a table store table +and then use kyuubi trino sql engine to query the table with the following SQL ``SELECT`` statement. + + +.. code-block:: sql + + SELECT * FROM tablestore.default.t1 + +.. _Apache Paimon (Incubating): https://paimon.apache.org/ +.. _Apache Paimon (Incubating) multi engine support: https://paimon.apache.org/docs/master/engines/overview/ +.. _Apache Paimon (Incubating) Engines Flink Quick Start: https://paimon.apache.org/docs/master/engines/flink/#quick-start +.. _Official Documentation: https://paimon.apache.org/docs/master/ +.. _Source Code: https://github.com/JingsongLi/paimon-trino +.. _Pre-bundled Hadoop: https://flink.apache.org/downloads/#additional-components +.. _Apache Paimon (Incubating) Trino README: https://github.com/JingsongLi/paimon-trino#readme diff --git a/docs/develop_tools/building.md b/docs/contributing/code/building.md similarity index 90% rename from docs/develop_tools/building.md rename to docs/contributing/code/building.md index 9dfc01f42..8c5c5aeec 100644 --- a/docs/develop_tools/building.md +++ b/docs/contributing/code/building.md @@ -15,11 +15,11 @@ - limitations under the License. --> -# Building Kyuubi +# Building From Source -## Building Kyuubi with Apache Maven +## Building With Maven -**Kyuubi** is built based on [Apache Maven](http://maven.apache.org), +**Kyuubi** is built based on [Apache Maven](https://maven.apache.org), ```bash ./build/mvn clean package -DskipTests @@ -33,7 +33,7 @@ If you want to test it manually, you can start Kyuubi directly from the Kyuubi p bin/kyuubi start ``` -## Building a Submodule Individually +## Building A Submodule Individually For instance, you can build the Kyuubi Common module using: @@ -49,7 +49,7 @@ For instance, you can build the Kyuubi Common module using: build/mvn clean package -pl kyuubi-common,kyuubi-ha -DskipTests ``` -## Skipping Some modules +## Skipping Some Modules For instance, you can build the Kyuubi modules without Kyuubi Codecov and Assembly modules using: @@ -57,7 +57,7 @@ For instance, you can build the Kyuubi modules without Kyuubi Codecov and Assemb mvn clean install -pl '!dev/kyuubi-codecov,!kyuubi-assembly' -DskipTests ``` -## Building Kyuubi against Different Apache Spark versions +## Building Kyuubi Against Different Apache Spark Versions Since v1.1.0, Kyuubi support building with different Spark profiles, @@ -67,7 +67,7 @@ Since v1.1.0, Kyuubi support building with different Spark profiles, | -Pspark-3.2 | No | 1.4.0 | | -Pspark-3.3 | Yes | 1.6.0 | -## Building with Apache dlcdn site +## Building With Apache dlcdn Site By default, we use `https://archive.apache.org/dist/` to download the built-in release packages of engines, such as Spark or Flink. diff --git a/docs/develop_tools/debugging.md b/docs/contributing/code/debugging.md similarity index 98% rename from docs/develop_tools/debugging.md rename to docs/contributing/code/debugging.md index faf7173e4..d3fb6d16f 100644 --- a/docs/develop_tools/debugging.md +++ b/docs/contributing/code/debugging.md @@ -35,7 +35,7 @@ In the IDE, you set the corresponding parameters(host&port) in debug configurati
      -![](../imgs/idea_debug.png) +![](../../imgs/idea_debug.png)
      diff --git a/docs/develop_tools/developer.md b/docs/contributing/code/developer.md similarity index 70% rename from docs/develop_tools/developer.md rename to docs/contributing/code/developer.md index 329e219de..518d71871 100644 --- a/docs/develop_tools/developer.md +++ b/docs/contributing/code/developer.md @@ -24,16 +24,6 @@ build/mvn versions:set -DgenerateBackupPoms=false ``` -## Update Document Version - -Whenever project version updates, please also update the document version at `docs/conf.py` to target the upcoming release. - -For example, - -```python -release = '1.2.0' -``` - ## Update Dependency List Kyuubi uses the `dev/dependencyList` file to indicate what upstream dependencies will actually go to the server-side classpath. @@ -56,5 +46,13 @@ You can run `dev/reformat` to format all Java and Scala code. Kyuubi uses settings.md to explain available configurations. -You can run `KYUUBI_UPDATE=1 build/mvn clean test -pl kyuubi-server -am -Pflink-provided,spark-provided,hive-provided -DwildcardSuites=org.apache.kyuubi.config.AllKyuubiConfiguration` -to append descriptions of new configurations to settings.md. +You can run `dev/gen/gen_all_config_docs.sh` to append and update descriptions of new configurations to `settings.md`. + +## Generative Tooling Usage + +In general, the ASF allows contributions co-authored using generative AI tools. However, there are several considerations when you submit a patch containing generated content. + +Foremost, you are required to disclose usage of such tool. Furthermore, you are responsible for ensuring that the terms and conditions of the tool in question are +compatible with usage in an Open Source project and inclusion of the generated content doesn't pose a risk of copyright violation. + +Please refer to [The ASF Generative Tooling Guidance](https://www.apache.org/legal/generative-tooling.html) for more detailed information. diff --git a/docs/develop_tools/distribution.md b/docs/contributing/code/distribution.md similarity index 86% rename from docs/develop_tools/distribution.md rename to docs/contributing/code/distribution.md index abc2ac91b..23c9c6542 100644 --- a/docs/develop_tools/distribution.md +++ b/docs/contributing/code/distribution.md @@ -15,7 +15,7 @@ - limitations under the License. --> -# Building a Runnable Distribution +# Building A Runnable Distribution To create a Kyuubi distribution like those distributed by [Kyuubi Release Page](https://kyuubi.apache.org/releases.html), and that is laid out to be runnable, use `./build/dist` in the project root directory. @@ -26,15 +26,16 @@ For more information on usage, run `./build/dist --help` ./build/dist - Tool for making binary distributions of Kyuubi Usage: -+------------------------------------------------------------------------------------------------------+ -| ./build/dist [--name ] [--tgz] [--flink-provided] [--spark-provided] [--hive-provided] | -| [--mvn ] | -+------------------------------------------------------------------------------------------------------+ ++----------------------------------------------------------------------------------------------+ +| ./build/dist [--name ] [--tgz] [--web-ui] [--flink-provided] [--hive-provided] | +| [--spark-provided] [--mvn ] | ++----------------------------------------------------------------------------------------------+ name: - custom binary name, using project version if undefined tgz: - whether to make a whole bundled package +web-ui: - whether to include web ui flink-provided: - whether to make a package without Flink binary -spark-provided: - whether to make a package without Spark binary hive-provided: - whether to make a package without Hive binary +spark-provided: - whether to make a package without Spark binary mvn: - external maven executable location ``` diff --git a/docs/contributing/code/get_started.rst b/docs/contributing/code/get_started.rst new file mode 100644 index 000000000..0dcd90304 --- /dev/null +++ b/docs/contributing/code/get_started.rst @@ -0,0 +1,95 @@ +.. Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +Get Started +=========== + +Good First Issues +----------------- + +.. image:: https://img.shields.io/github/issues/apache/kyuubi/good%20first%20issue?color=green&label=Good%20first%20issue&logo=gfi&logoColor=red&style=for-the-badge + :alt: GitHub issues by-label + :target: `Good First Issues`_ + +**Good First Issue** is initiative to curate easy pickings for first-time +contributors. It helps you locate suitable development tasks with beginner's +skills required, and finally make your first contribution to Kyuubi. + +After solving one or more good first issues, you should be able to + +- Find efficient ways to communicate with the community and get help +- Setup `develop environment`_ on your machine +- `Build`_ Kyuubi from source +- `Run tests`_ locally +- `Submit a pull request`_ through Github +- Be listed in `Apache Kyuubi contributors`_ +- And most importantly, you can move to the next level and try some tricky issues + +.. note:: Don't linger too long at this stage. + :class: dropdown, toggle + +Help Wanted Issues +------------------ + +.. image:: https://img.shields.io/github/issues/apache/kyuubi/help%20wanted?color=brightgreen&label=HELP%20WANTED&style=for-the-badge + :alt: GitHub issues by-label + :target: `Help Wanted Issues`_ + +Issues that maintainers labeled as help wanted are mostly + +- sub-tasks of an ongoing shorthanded umbrella +- non-urgent improvements +- bug fixes for corner cases +- feature requests not covered by current technology stack of kyuubi community + +Since these problems are not urgent, you can take your time when fixing them. + +.. note:: Help wanted issues may contain easy pickings and tricky ones. + :class: dropdown, toggle + + +Code Contribution Programs +-------------------------- + +Kyuubi Code Program is a **semi-annual** and **annual** coding program. It's +a 2-month program and the first round will start in October, 2023. + +The program is open to all contributors and newbie-friendly as it will provide +a mentor to help you get through the sub-tasks. + +You will be rewarded with a Kyuubi SWAG, such as a Kyuubi Contributor T-shirt, +after you complete the program. + +.. image:: https://img.shields.io/badge/Kyuubi%20Code%20Program-2024H1-blue?style=for-the-badge + +- Status: Planning +- Duration: 2024.02.01 - 2024.04.01 +- Sponsors: (keeping seats vacant in anticipation) + +.. image:: https://img.shields.io/badge/Kyuubi%20Code%20Program-2023-blue?style=for-the-badge + :target: https://github.com/apache/kyuubi/issues/5357 + +- Status: In Progress +- Duration: 2023.10.01 - 2023.12.01 +- Sponsors: NetEase + +.. _Good First Issues: https://github.com/apache/kyuubi/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22 +.. _develop environment: idea_setup.html +.. _Build: build.html +.. _Run tests: testing.html +.. _Submit a pull request: https://kyuubi.apache.org/pull_request.html +.. _Apache Kyuubi contributors: https://github.com/apache/kyuubi/graphs/contributors +.. _Help Wanted Issues: https://github.com/apache/kyuubi/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22 + diff --git a/docs/develop_tools/idea_setup.md b/docs/contributing/code/idea_setup.md similarity index 100% rename from docs/develop_tools/idea_setup.md rename to docs/contributing/code/idea_setup.md diff --git a/docs/develop_tools/index.rst b/docs/contributing/code/index.rst similarity index 84% rename from docs/develop_tools/index.rst rename to docs/contributing/code/index.rst index c56321cb3..25a6e421b 100644 --- a/docs/develop_tools/index.rst +++ b/docs/contributing/code/index.rst @@ -13,15 +13,19 @@ See the License for the specific language governing permissions and limitations under the License. -Develop Tools -============= +Contributing Code +================= + +These sections explain the process, guidelines, and tools for contributing +code to the Kyuubi project. .. toctree:: :maxdepth: 2 + get_started + style building distribution - build_document testing debugging developer diff --git a/docs/contributing/code/style.rst b/docs/contributing/code/style.rst new file mode 100644 index 000000000..d967e8959 --- /dev/null +++ b/docs/contributing/code/style.rst @@ -0,0 +1,39 @@ +.. Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +Code Style Guide +================ + +Code is written once by its author, but read and modified multiple times by +lots of other engineers. As most bugs actually come from future modification +of the code, we need to optimize our codebase for long-term, global +readability and maintainability. The best way to achieve this is to write +simple code. + +Kyuubi's source code is multilingual, specific code style will be applied to +corresponding language. + +Scala Coding Style Guide +------------------------ + +Kyuubi adopts the `Databricks Scala Coding Style Guide`_ for scala codes. + +Java Coding Style Guide +----------------------- + +Kyuubi adopts the `Google Java style`_ for java codes. + +.. _Databricks Scala Coding Style Guide: https://github.com/databricks/scala-style-guide +.. _Google Java style: https://google.github.io/styleguide/javaguide.html \ No newline at end of file diff --git a/docs/develop_tools/testing.md b/docs/contributing/code/testing.md similarity index 87% rename from docs/develop_tools/testing.md rename to docs/contributing/code/testing.md index 48a2e9787..3e63aa1a2 100644 --- a/docs/develop_tools/testing.md +++ b/docs/contributing/code/testing.md @@ -17,8 +17,8 @@ # Running Tests -**Kyuubi** can be tested based on [Apache Maven](http://maven.apache.org) and the ScalaTest Maven Plugin, -please refer to the [ScalaTest documentation](http://www.scalatest.org/user_guide/using_the_scalatest_maven_plugin), +**Kyuubi** can be tested based on [Apache Maven](https://maven.apache.org) and the ScalaTest Maven Plugin, +please refer to the [ScalaTest documentation](https://www.scalatest.org/user_guide/using_the_scalatest_maven_plugin), ## Running Tests Fully diff --git a/docs/contributing/doc/build.rst b/docs/contributing/doc/build.rst new file mode 100644 index 000000000..4ec2362f3 --- /dev/null +++ b/docs/contributing/doc/build.rst @@ -0,0 +1,96 @@ +.. Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +Building Documentation +====================== + +Follow the steps below and learn how to build the Kyuubi documentation as the +one you are watching now. + +Setup Environment +----------------- + +- Firstly, install ``virtualenv``, this is optional but recommended as it is useful + to create an independent environment to resolve dependency issues for building + the documentation. + +.. code-block:: sh + :caption: Install virtualenv + + $ pip install virtualenv + +- Switch to the ``docs`` root directory. + +.. code-block:: sh + :caption: Switch to docs + + $ cd $KYUUBI_SOURCE_PATH/docs + +- Create a virtual environment named 'kyuubi' or anything you like using ``virtualenv`` + if it's not existing. + +.. code-block:: sh + :caption: New virtual environment + + $ virtualenv kyuubi + +- Activate the virtual environment, + +.. code-block:: sh + :caption: Activate virtual environment + + $ source ./kyuubi/bin/activate + +Install All Dependencies +------------------------ + +Install all dependencies enumerated in the ``requirements.txt``. + +.. code-block:: sh + :caption: Install dependencies + + $ pip install -r requirements.txt + + +Create Documentation +-------------------- + +Make sure you are in the ``$KYUUBI_SOURCE_PATH/docs`` directory. + +Linux & MacOS +~~~~~~~~~~~~~ + +.. code-block:: sh + :caption: Sphinx build on Unix-like OS + + $ make html + +Windows +~~~~~~~ + +.. code-block:: sh + :caption: Sphinx build on Windows + + $ make.bat html + + +If the build process succeed, the HTML pages are in +``$KYUUBI_SOURCE_PATH/docs/_build/html``. + +View Locally +------------ + +Open the `$KYUUBI_SOURCE_PATH/docs/_build/html/index.html` file in your +favorite web browser. diff --git a/docs/contributing/doc/get_started.rst b/docs/contributing/doc/get_started.rst new file mode 100644 index 000000000..f262695b7 --- /dev/null +++ b/docs/contributing/doc/get_started.rst @@ -0,0 +1,117 @@ +.. Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +Get Started +=========== + +.. image:: https://img.shields.io/github/issues/apache/kyuubi/kind:documentation?color=green&logo=gfi&logoColor=red&style=for-the-badge + :alt: GitHub issues by-label + + +Trivial Fixes +------------- + +For typos, layout, grammar, spelling, punctuation errors and other similar issues +or changes that occur within a single file, it is acceptable to make edits directly +on the page being viewed. When viewing a source file on kyuubi's +`Github repository`_, a simple click on the ``edit icon`` or keyboard shortcut +``e`` will activate the editor. Similarly, when viewing files on `Read The Docs`_ +platform, clicking on the ``suggest edit`` button will lead you to the editor. +These methods do not require any local development environment setup and +are convenient for making quick fixes. + +Upon completion of the editing process, opt the ``commit changes`` option, +adhere to the provided instructions to submit a pull request, +and await feedback from the designated reviewer. + +Major Fixes +----------- + +For significant modifications that affect multiple files, it is advisable to +clone the repository to a local development environment, implement the necessary +changes, and conduct thorough testing prior to submitting a pull request. + + +`Fork`_ The Repository +~~~~~~~~~~~~~~~~~~~~~~ + +Clone The Forked Repository +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: + :caption: Clone the repository + + $ git clone https://github.com/your_username/kyuubi.git + +Replace "your_username" with your GitHub username. This will create a local +copy of your forked repository on your machine. You will see the ``master`` +branch if you run ``git branch`` in the ``kyuubi`` folder. + +Create A New Branch +~~~~~~~~~~~~~~~~~~~ + +.. code-block:: + :caption: Create a new branch + + $ git checkout -b guide + Switched to a new branch 'guide' + +Editing And Testing +~~~~~~~~~~~~~~~~~~~ + +Make the necessary changes to the documentation files using a text editor. +`Build and verify`_ the changes you have made to see if they look fine. + +.. note:: + :class: dropdown, toggle + +Create A Pull Request +~~~~~~~~~~~~~~~~~~~~~ + +Once you have made the changes, + +- Commit them with a descriptive commit message using the command: + +.. code-block:: + :caption: commit the changes + + $ git commit -m "Description of changes made" + +- Push the changes to your forked repository using the command + +.. code-block:: + :caption: push the changes + + $ git push origin guide + +- `Create A Pull Request`_ with a descriptive PR title and description. + +- Polishing the PR with comments of reviews addressed + +Report Only +----------- + +If you don't have time to fix the doc issue and submit a pull request on your own, +`reporting a document issue`_ also helps. Please follow some basic rules: + +- Use the title field to clearly describe the issue +- Choose the documentation report template +- Fill out the required field in the documentation report + +.. _Home Page: https://kyuubi.apache.org +.. _Fork: https://github.com/apache/kyuubi/fork +.. _Build and verify: build.html +.. _Create A Pull Request: https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/creating-a-pull-request +.. _reporting a document issue: https://github.com/apache/kyuubi/issues/new/choose \ No newline at end of file diff --git a/docs/contributing/doc/index.rst b/docs/contributing/doc/index.rst new file mode 100644 index 000000000..bf6ae41bd --- /dev/null +++ b/docs/contributing/doc/index.rst @@ -0,0 +1,44 @@ +.. Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +Contributing Documentations +=========================== + +The project documentation is crucial for users and contributors. This guide +outlines the contribution guidelines for Apache Kyuubi documentation. + +Kyuubi's documentation source files are maintained in the same `github repository`_ +as the code base, which ensures updating code and documentation synchronously. +All documentation source files can be found in the sub-folder named ``docs``. + +Kyuubi's documentation is published and hosted on `Read The Docs`_ platform by +version. with each version having its own dedicated page. To access a specific +version of the document, simply navigate to the "Docs" tab on our Home Page. + +We welcome any contributions to the documentation, including but not limited to +writing, translation, report doc issues on Github, reposting. + + +.. toctree:: + :maxdepth: 2 + + get_started + style + build + +.. _Github repository: https://github.com/apache/kyuubi +.. _Restructured Text: https://en.wikipedia.org/wiki/ReStructuredText +.. _Read The Docs: https://kyuubi.rtfd.io +.. _Home Page: https://kyuubi.apache.org \ No newline at end of file diff --git a/docs/contributing/doc/style.rst b/docs/contributing/doc/style.rst new file mode 100644 index 000000000..14cc2b8ac --- /dev/null +++ b/docs/contributing/doc/style.rst @@ -0,0 +1,135 @@ +.. Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +Documentation Style Guide +========================= + +This guide contains guidelines, not rules. While guidelines are important +to follow, they are not hard and fast rules. It's important to use your +own judgement and discretion when creating content, and to depart from the +guidelines when necessary to improve the quality and effectiveness of your +content. Ultimately, the goal is to create content that is clear, concise, +and useful to your audience, and sometimes deviating from the guidelines +may be necessary to achieve that goal. + +Goals +----- + +- Source text files are readable and portable +- Source diagram files are editable +- Source files are maintainable over time and across community + +License Header +-------------- + +All original documents should include the ASF license header. All reproduced +or quoted content should be authorized and attributed to the source. + +If you are about to quote some from commercial materials, please refer to +`ASF 3RD PARTY LICENSE POLICY`_, or consult the Apache Kyuubi PMC to avoid +legality issues. + +General Style +------------- + +- Use `ReStructuredText`_ or `Markdown`_ format for text, avoid HTML hacks +- Use `draw.io`_ for drawing or editing an image, and export it as PNG for + referencing in document. A pull request should commit both of them +- Use Kyuubi for short instead of Apache Kyuubi after the first time in the + same page +- Character line limit: 78, except unbreakable ones +- Prefer lists to tables +- Prefer unordered list than ordered + +ReStructuredText +---------------- + +Headings +~~~~~~~~ + +- Use **Pascal Case**, every word starts with an uppercase letter, + e.g., 'Documentation Style Guide' +- Use a max of **three levels** + - Split into multiple files when there comes an H4 + - Prefer `directive rubric`_ than H4 +- Use underline-only adornment styles, **DO NOT** use overline + - The length of underline characters **SHOULD** match the title + - H1 should be underlined with '=' + - H2 should be underlined with '-' + - H3 should be underlined with '~' + - H4 should be underlined with '^', but it's better to avoid using H4 +- **DO NOT** use numbering for sections +- **DO NOT** use "Kyuubi" in titles if possible + +Links +~~~~~ + +- Define links with short descriptive phrases, group them at the bottom of the file + +.. note:: + :class: dropdown, toggle + + .. code-block:: + :caption: Recommended + + Please refer to `Apache Kyuubi Home Page`_. + + .. _Apache Kyuubi Home Page: https://kyuubi.apache.org/ + + .. code-block:: + :caption: Not recommended + + Please refer to `Apache Kyuubi Home Page `_. + + +Markdown +-------- + +Headings +~~~~~~~~ + +- Use **Pascal Case**, every word starts with an uppercase letter, + e.g., 'Documentation Style Guide' +- Use a max of **three levels** + - Split into multiple files when there comes an H4 +- **DO NOT** use numbering for sections +- **DO NOT** use "Kyuubi" in titles if possible + +Images +------ + +Use images only when they provide helpful visual explanations of information +otherwise difficult to express with words + +Third-party references +---------------------- + +If the preceding references don't provide explicit guidance, then see these +third-party references, depending on the nature of your question: + +- `Google developer documentation style`_ +- `Apple Style Guide`_ +- `Red Hat supplementary style guide for product documentation`_ + +.. References + +.. _ASF 3RD PARTY LICENSE POLICY: https://www.apache.org/legal/resolved.html#asf-3rd-party-license-policy +.. _directive rubric :https://www.sphinx-doc.org/en/master/usage/restructuredtext/directives.html#directive-rubric +.. _ReStructuredText: https://docutils.sourceforge.io/rst.html +.. _Markdown: https://en.wikipedia.org/wiki/Markdown +.. _draw.io: https://www.diagrams.net/ +.. _Google developer documentation style: https://developers.google.com/style +.. _Apple Style Guide: https://help.apple.com/applestyleguide/ +.. _Red Hat supplementary style guide for product documentation: https://redhat-documentation.github.io/supplementary-style-guide/ diff --git a/docs/deployment/engine_on_kubernetes.md b/docs/deployment/engine_on_kubernetes.md index ae8edcb75..a8f7c6ca0 100644 --- a/docs/deployment/engine_on_kubernetes.md +++ b/docs/deployment/engine_on_kubernetes.md @@ -21,7 +21,7 @@ When you want to run Kyuubi's Spark SQL engines on Kubernetes, you'd better have cognition upon the following things. -* Read about [Running Spark On Kubernetes](http://spark.apache.org/docs/latest/running-on-kubernetes.html) +* Read about [Running Spark On Kubernetes](https://spark.apache.org/docs/latest/running-on-kubernetes.html) * An active Kubernetes cluster * [Kubectl](https://kubernetes.io/docs/reference/kubectl/overview/) * KubeConfig of the target cluster @@ -36,6 +36,17 @@ Spark on Kubernetes config master by using a special format. You can use cmd `kubectl cluster-info` to get api-server host and port. +### Deploy Mode + +One of the main advantages of the Kyuubi server compared to other interactive Spark clients is that it supports cluster deploy mode. +It is highly recommended to run Spark in k8s in cluster mode. + +The minimum required configurations are: + +* spark.submit.deployMode (cluster) +* spark.kubernetes.file.upload.path (path on s3 or hdfs) +* spark.kubernetes.authenticate.driver.serviceAccountName ([viz ServiceAccount](#serviceaccount)) + ### Docker Image Spark ships a `./bin/docker-image-tool.sh` script to build and publish the Docker images for running Spark applications on Kubernetes. @@ -97,7 +108,7 @@ As it known to us all, Kubernetes can use configurations to mount volumes into d * persistentVolumeClaim: mounts a PersistentVolume into a pod. Note: Please -see [the Security section of this document](http://spark.apache.org/docs/latest/running-on-kubernetes.html#security) for security issues related to volume mounts. +see [the Security section of this document](https://spark.apache.org/docs/latest/running-on-kubernetes.html#security) for security issues related to volume mounts. ``` spark.kubernetes.driver.volumes...options.path= @@ -107,7 +118,7 @@ spark.kubernetes.executor.volumes...options.path= spark.kubernetes.executor.volumes...mount.path= ``` -Read [Using Kubernetes Volumes](http://spark.apache.org/docs/latest/running-on-kubernetes.html#using-kubernetes-volumes) for more about volumes. +Read [Using Kubernetes Volumes](https://spark.apache.org/docs/latest/running-on-kubernetes.html#using-kubernetes-volumes) for more about volumes. ### PodTemplateFile @@ -117,4 +128,4 @@ To do so, specify the spark properties `spark.kubernetes.driver.podTemplateFile` ### Other -You can read Spark's official documentation for [Running on Kubernetes](http://spark.apache.org/docs/latest/running-on-kubernetes.html) for more information. +You can read Spark's official documentation for [Running on Kubernetes](https://spark.apache.org/docs/latest/running-on-kubernetes.html) for more information. diff --git a/docs/deployment/engine_on_yarn.md b/docs/deployment/engine_on_yarn.md index cb5bdd9e0..1025418d9 100644 --- a/docs/deployment/engine_on_yarn.md +++ b/docs/deployment/engine_on_yarn.md @@ -15,19 +15,19 @@ - limitations under the License. --> -# Deploy Kyuubi engines on Yarn +# Deploy Kyuubi engines on YARN -## Deploy Kyuubi Spark Engine on Yarn +## Deploy Kyuubi Spark Engine on YARN ### Requirements -When you want to deploy Kyuubi's Spark SQL engines on YARN, you'd better have cognition upon the following things. +To deploy Kyuubi's Spark SQL engines on YARN, you'd better have cognition upon the following things. -- Knowing the basics about [Running Spark on YARN](http://spark.apache.org/docs/latest/running-on-yarn.html) +- Knowing the basics about [Running Spark on YARN](https://spark.apache.org/docs/latest/running-on-yarn.html) - A binary distribution of Spark which is built with YARN support - You can use the built-in Spark distribution - You can get it from [Spark official website](https://spark.apache.org/downloads.html) directly - - You can [Build Spark](http://spark.apache.org/docs/latest/building-spark.html#specifying-the-hadoop-version-and-enabling-yarn) with `-Pyarn` maven option + - You can [Build Spark](https://spark.apache.org/docs/latest/building-spark.html#specifying-the-hadoop-version-and-enabling-yarn) with `-Pyarn` maven option - An active [Apache Hadoop YARN](https://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/YARN.html) cluster - An active Apache Hadoop HDFS cluster - Setup Hadoop client configurations at the machine the Kyuubi server locates @@ -92,7 +92,7 @@ and how many cpus and memory will Spark driver, ApplicationMaster and each execu | spark.executor.memory | 1g | Amount of memory to use for the executor process | | spark.executor.memoryOverhead | executorMemory * 0.10, with minimum of 384 | Amount of additional memory to be allocated per executor process. This is memory that accounts for things like VM overheads, interned strings other native overheads, etc | -It is recommended to use [Dynamic Allocation](http://spark.apache.org/docs/3.0.1/configuration.html#dynamic-allocation) with Kyuubi, +It is recommended to use [Dynamic Allocation](https://spark.apache.org/docs/3.0.1/configuration.html#dynamic-allocation) with Kyuubi, since the SQL engine will be long-running for a period, execute user's queries from clients periodically, and the demand for computing resources is not the same for those queries. It is better for Spark to release some executors when either the query is lightweight, or the SQL engine is being idled. @@ -104,20 +104,20 @@ which allows YARN to cache it on nodes so that it doesn't need to be distributed ##### Others -Please refer to [Spark properties](http://spark.apache.org/docs/latest/running-on-yarn.html#spark-properties) to check other acceptable configs. +Please refer to [Spark properties](https://spark.apache.org/docs/latest/running-on-yarn.html#spark-properties) to check other acceptable configs. ### Kerberos -Kyuubi currently does not support Spark's [YARN-specific Kerberos Configuration](http://spark.apache.org/docs/3.0.1/running-on-yarn.html#kerberos), +Kyuubi currently does not support Spark's [YARN-specific Kerberos Configuration](https://spark.apache.org/docs/3.0.1/running-on-yarn.html#kerberos), so `spark.kerberos.keytab` and `spark.kerberos.principal` should not use now. Instead, you can schedule a periodically `kinit` process via `crontab` task on the local machine that hosts Kyuubi server or simply use [Kyuubi Kinit](settings.html#kinit). -## Deploy Kyuubi Flink Engine on Yarn +## Deploy Kyuubi Flink Engine on YARN ### Requirements -When you want to deploy Kyuubi's Flink SQL engines on YARN, you'd better have cognition upon the following things. +To deploy Kyuubi's Flink SQL engines on YARN, you'd better have cognition upon the following things. - Knowing the basics about [Running Flink on YARN](https://nightlies.apache.org/flink/flink-docs-stable/docs/deployment/resource-providers/yarn) - A binary distribution of Flink which is built with YARN support @@ -127,13 +127,59 @@ When you want to deploy Kyuubi's Flink SQL engines on YARN, you'd better have co - An active Object Storage cluster, e.g. [HDFS](https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-hdfs/HdfsDesign.html), S3 and [Minio](https://min.io/) etc. - Setup Hadoop client configurations at the machine the Kyuubi server locates -### Yarn Session Mode +### Flink Deployment Modes + +Currently, Flink supports two deployment modes on YARN: [YARN Application Mode](https://nightlies.apache.org/flink/flink-docs-release-1.17/docs/deployment/resource-providers/yarn/#application-mode) and [YARN Session Mode](https://nightlies.apache.org/flink/flink-docs-release-1.17/docs/deployment/resource-providers/yarn/#application-mode). + +- YARN Application Mode: In this mode, Kyuubi starts a dedicated Flink application cluster and runs the SQL engine on it. +- YARN Session Mode: In this mode, Kyuubi starts the Flink SQL engine locally and connects to a running Flink YARN session cluster. + +As Kyuubi has to know the deployment mode before starting the SQL engine, it's required to specify the deployment mode in Kyuubi configuration. + +```properties +# candidates: yarn-application, yarn-session +flink.execution.target=yarn-application +``` + +### YARN Application Mode + +#### Flink Configurations + +Since the Flink SQL engine runs inside the JobManager, it's recommended to tune the resource configurations of the JobManager based on your workload. + +The related Flink configurations are listed below (see more details at [Flink Configuration](https://nightlies.apache.org/flink/flink-docs-master/docs/deployment/config/#yarn)): + +| Name | Default | Meaning | +|--------------------------------|---------|----------------------------------------------------------------------------------------| +| yarn.appmaster.vcores | 1 | The number of virtual cores (vcores) used by the JobManager (YARN application master). | +| jobmanager.memory.process.size | (none) | Total size of the memory of the JobManager process. | + +Note that Flink application mode doesn't support HA for multiple jobs as for now, this also applies to Kyuubi's Flink SQL engine. If JobManager fails and restarts, the submitted jobs would not be recovered and should be re-submitted. + +#### Environment + +Either `HADOOP_CONF_DIR` or `YARN_CONF_DIR` is configured and points to the Hadoop client configurations directory, usually, `$HADOOP_HOME/etc/hadoop`. + +You could verify your setup by the following command: + +```bash +# we assume to be in the root directory of +# the unzipped Flink distribution + +# (0) export HADOOP_CLASSPATH +export HADOOP_CLASSPATH=`hadoop classpath` + +# (1) submit a Flink job and ensure it runs successfully +./bin/flink run -m yarn-cluster ./examples/streaming/WordCount.jar +``` + +### YARN Session Mode #### Flink Configurations ```bash execution.target: yarn-session -# Yarn Session Cluster application id. +# YARN Session Cluster application id. yarn.application.id: application_00000000XX_00XX ``` @@ -194,23 +240,19 @@ To use Hadoop vanilla jars, please configure $KYUUBI_HOME/conf/kyuubi-env.sh as $ echo "export FLINK_HADOOP_CLASSPATH=`hadoop classpath`" >> $KYUUBI_HOME/conf/kyuubi-env.sh ``` -### Deployment Modes Supported by Flink on YARN - -For experiment use, we recommend deploying Kyuubi Flink SQL engine in [Session Mode](https://nightlies.apache.org/flink/flink-docs-stable/docs/deployment/resource-providers/yarn/#session-mode). -At present, [Application Mode](https://nightlies.apache.org/flink/flink-docs-stable/docs/deployment/resource-providers/yarn/#application-mode) and [Per-Job Mode (deprecated)](https://nightlies.apache.org/flink/flink-docs-stable/docs/deployment/resource-providers/yarn/#per-job-mode-deprecated) are not supported for Flink engine. - ### Kerberos -As Kyuubi Flink SQL engine wraps the Flink SQL client that currently does not support [Flink Kerberos Configuration](https://nightlies.apache.org/flink/flink-docs-stable/docs/deployment/config/#security-kerberos-login-keytab), -so `security.kerberos.login.keytab` and `security.kerberos.login.principal` should not use now. +With regard to YARN application mode, Kerberos is supported natively by Flink, see [Flink Kerberos Configuration](https://nightlies.apache.org/flink/flink-docs-stable/docs/deployment/config/#security-kerberos-login-keytab) for details. -Instead, you can schedule a periodically `kinit` process via `crontab` task on the local machine that hosts Kyuubi server or simply use [Kyuubi Kinit](settings.html#kinit). +With regard to YARN session mode, `security.kerberos.login.keytab` and `security.kerberos.login.principal` are not effective, as Kyuubi Flink SQL engine mainly relies on Flink SQL client which currently does not support [Flink Kerberos Configuration](https://nightlies.apache.org/flink/flink-docs-stable/docs/deployment/config/#security-kerberos-login-keytab), + +As a workaround, you can schedule a periodically `kinit` process via `crontab` task on the local machine that hosts Kyuubi server or simply use [Kyuubi Kinit](settings.html#kinit). -## Deploy Kyuubi Hive Engine on Yarn +## Deploy Kyuubi Hive Engine on YARN ### Requirements -When you want to deploy Kyuubi's Hive SQL engines on YARN, you'd better have cognition upon the following things. +To deploy Kyuubi's Hive SQL engines on YARN, you'd better have cognition upon the following things. - Knowing the basics about [Running Hive on YARN](https://cwiki.apache.org/confluence/display/Hive/GettingStarted) - A binary distribution of Hive @@ -239,7 +281,7 @@ $ $HIVE_HOME/bin/beeline -u 'jdbc:hive2://localhost:10000/default' 0: jdbc:hive2://localhost:10000/default> INSERT INTO TABLE pokes VALUES (1, 'hello'); ``` -If the `Hive SQL` passes and there is a job in Yarn Web UI, It indicates the hive environment is normal. +If the `Hive SQL` passes and there is a job in YARN Web UI, it indicates the hive environment is good. #### Required Environment Variable diff --git a/docs/deployment/high_availability_guide.md b/docs/deployment/high_availability_guide.md index 353e549eb..51c878157 100644 --- a/docs/deployment/high_availability_guide.md +++ b/docs/deployment/high_availability_guide.md @@ -39,7 +39,7 @@ Using multiple Kyuubi service units with load balancing instead of a single unit - High concurrency - By adding or removing Kyuubi server instances can easily scale up or down to meet the need of client requests. - Upgrade smoothly - - Kyuubi server supports stop gracefully. We could delete a `k.i.` but not stop it immediately. + - Kyuubi server supports stopping gracefully. We could delete a `k.i.` but not stop it immediately. In this case, the `k.i.` will not take any new connection request but only operation requests from existing connections. After all connection are released, it stops then. - The dependencies of Kyuubi engines are free to change, such as bump up versions, modify configurations, add external jars, relocate to another engine home. Everything will be reloaded during start and stop. diff --git a/docs/deployment/hive_metastore.md b/docs/deployment/hive_metastore.md index f3a24d897..f60465a1a 100644 --- a/docs/deployment/hive_metastore.md +++ b/docs/deployment/hive_metastore.md @@ -30,7 +30,7 @@ In this section, you will learn how to configure Kyuubi to interact with Hive Me - A Spark binary distribution built with `-Phive` support - Use the built-in one in the Kyuubi distribution - Download from [Spark official website](https://spark.apache.org/downloads.html) - - Build from Spark source, [Building With Hive and JDBC Support](http://spark.apache.org/docs/latest/building-spark.html#building-with-hive-and-jdbc-support) + - Build from Spark source, [Building With Hive and JDBC Support](https://spark.apache.org/docs/latest/building-spark.html#building-with-hive-and-jdbc-support) - A copy of Hive client configuration So the whole thing here is to let Spark applications use this copy of Hive configuration to start a Hive metastore client for their own to talk to the Hive metastore server. @@ -199,13 +199,13 @@ Caused by: org.apache.thrift.TApplicationException: Invalid method name: 'get_ta ... 93 more ``` -To prevent this problem, we can use Spark's [Interacting with Different Versions of Hive Metastore](http://spark.apache.org/docs/latest/sql-data-sources-hive-tables.html#interacting-with-different-versions-of-hive-metastore). +To prevent this problem, we can use Spark's [Interacting with Different Versions of Hive Metastore](https://spark.apache.org/docs/latest/sql-data-sources-hive-tables.html#interacting-with-different-versions-of-hive-metastore). ## Further Readings - Hive Wiki - [Hive Metastore Administration](https://cwiki.apache.org/confluence/display/Hive/AdminManual+Metastore+Administration) - Spark Online Documentation - - [Custom Hadoop/Hive Configuration](http://spark.apache.org/docs/latest/configuration.html#custom-hadoophive-configuration) - - [Hive Tables](http://spark.apache.org/docs/latest/sql-data-sources-hive-tables.html) + - [Custom Hadoop/Hive Configuration](https://spark.apache.org/docs/latest/configuration.html#custom-hadoophive-configuration) + - [Hive Tables](https://spark.apache.org/docs/latest/sql-data-sources-hive-tables.html) diff --git a/docs/deployment/index.rst b/docs/deployment/index.rst index ec3ece951..1b6bf8766 100644 --- a/docs/deployment/index.rst +++ b/docs/deployment/index.rst @@ -31,15 +31,6 @@ Basics high_availability_guide migration-guide -Configurations --------------- - -.. toctree:: - :maxdepth: 2 - :glob: - - settings - Engines ------- diff --git a/docs/deployment/kyuubi_on_kubernetes.md b/docs/deployment/kyuubi_on_kubernetes.md index 8bb1d88c3..11ffe8e48 100644 --- a/docs/deployment/kyuubi_on_kubernetes.md +++ b/docs/deployment/kyuubi_on_kubernetes.md @@ -90,7 +90,7 @@ See more related details in [Using RBAC Authorization](https://kubernetes.io/doc ## Config -You can configure Kyuubi the old-fashioned way by placing kyuubi-default.conf inside the image. Kyuubi do not recommend using this way on Kubernetes. +You can configure Kyuubi the old-fashioned way by placing `kyuubi-defaults.conf` inside the image. Kyuubi does not recommend using this way on Kubernetes. Kyuubi provide `${KYUUBI_HOME}/docker/kyuubi-configmap.yaml` to build Configmap for Kyuubi. diff --git a/docs/deployment/migration-guide.md b/docs/deployment/migration-guide.md index 42905340e..bf5b184cd 100644 --- a/docs/deployment/migration-guide.md +++ b/docs/deployment/migration-guide.md @@ -17,6 +17,31 @@ # Kyuubi Migration Guide +## Upgrading from Kyuubi 1.8 to 1.9 + +* Since Kyuubi 1.9.0, `kyuubi.session.conf.advisor` can be set as a sequence, Kyuubi supported chaining SessionConfAdvisors. + +## Upgrading from Kyuubi 1.7 to 1.8 + +* Since Kyuubi 1.8, SQLite is added and becomes the default database type of Kyuubi metastore, as Derby has been deprecated. + Both Derby and SQLite are mainly for testing purposes, and they're not supposed to be used in production. + To restore previous behavior, set `kyuubi.metadata.store.jdbc.database.type=DERBY` and + `kyuubi.metadata.store.jdbc.url=jdbc:derby:memory:kyuubi_state_store_db;create=true`. +* Since Kyuubi 1.8, if the directory of the embedded zookeeper configuration (`kyuubi.zookeeper.embedded.directory` + & `kyuubi.zookeeper.embedded.data.dir` & `kyuubi.zookeeper.embedded.data.log.dir`) is a relative path, it is resolved + relative to `$KYUUBI_HOME` instead of `$PWD`. +* Since Kyuubi 1.8, PROMETHEUS is changed as the default metrics reporter. To restore previous behavior, + set `kyuubi.metrics.reporters=JSON`. + +## Upgrading from Kyuubi 1.7.1 to 1.7.2 + +* Since Kyuubi 1.7.2, for Kyuubi BeeLine, please use `--python-mode` option to run python code or script. + +## Upgrading from Kyuubi 1.7.0 to 1.7.1 + +* Since Kyuubi 1.7.1, `protocolVersion` is removed from the request parameters of the REST API `Open(create) a session`. All removed or unknown parameters will be silently ignored and affects nothing. +* Since Kyuubi 1.7.1, `confOverlay` is supported in the request parameters of the REST API `Create an operation with EXECUTE_STATEMENT type`. + ## Upgrading from Kyuubi 1.6 to 1.7 * In Kyuubi 1.7, `kyuubi.ha.zookeeper.engine.auth.type` does not fallback to `kyuubi.ha.zookeeper.auth.type`. @@ -24,7 +49,7 @@ * Since Kyuubi 1.7, Kyuubi returns engine's information for `GetInfo` request instead of server. To restore the previous behavior, set `kyuubi.server.info.provider` to `SERVER`. * Since Kyuubi 1.7, Kyuubi session type `SQL` is refactored to `INTERACTIVE`, because Kyuubi supports not only `SQL` session, but also `SCALA` and `PYTHON` sessions. User need to use `INTERACTIVE` sessionType to look up the session event. -* Since Kyuubi 1.7, the REST API of `Open(create) a session` will not contains parameters `user` `password` and `IpAddr`. User and password should be set in `Authorization` of http request if needed. +* Since Kyuubi 1.7, the REST API of `Open(create) a session` will not contain parameters `user` `password` and `IpAddr`. User and password should be set in `Authorization` of http request if needed. ## Upgrading from Kyuubi 1.6.0 to 1.6.1 diff --git a/docs/deployment/spark/aqe.md b/docs/deployment/spark/aqe.md index 90cc5aff8..3682c7f9e 100644 --- a/docs/deployment/spark/aqe.md +++ b/docs/deployment/spark/aqe.md @@ -210,7 +210,7 @@ Kyuubi is a long-running service to make it easier for end-users to use Spark SQ ### Setting Default Configurations -[Configuring by `spark-defaults.conf`](settings.html#via-spark-defaults-conf) at the engine side is the best way to set up Kyuubi with AQE. All engines will be instantiated with AQE enabled. +[Configuring by `spark-defaults.conf`](../settings.html#via-spark-defaults-conf) at the engine side is the best way to set up Kyuubi with AQE. All engines will be instantiated with AQE enabled. Here is a config setting that we use in our platform when deploying Kyuubi. diff --git a/docs/deployment/spark/dynamic_allocation.md b/docs/deployment/spark/dynamic_allocation.md index b177b63c3..1a5057e73 100644 --- a/docs/deployment/spark/dynamic_allocation.md +++ b/docs/deployment/spark/dynamic_allocation.md @@ -170,7 +170,7 @@ Kyuubi is a long-running service to make it easier for end-users to use Spark SQ ### Setting Default Configurations -[Configuring by `spark-defaults.conf`](settings.html#via-spark-defaults-conf) at the engine side is the best way to set up Kyuubi with DRA. All engines will be instantiated with DRA enabled. +[Configuring by `spark-defaults.conf`](../settings.html#via-spark-defaults-conf) at the engine side is the best way to set up Kyuubi with DRA. All engines will be instantiated with DRA enabled. Here is a config setting that we use in our platform when deploying Kyuubi. diff --git a/docs/develop_tools/build_document.md b/docs/develop_tools/build_document.md deleted file mode 100644 index 0be5a1807..000000000 --- a/docs/develop_tools/build_document.md +++ /dev/null @@ -1,76 +0,0 @@ - - -# Building Kyuubi Documentation - -Follow the steps below and learn how to build the Kyuubi documentation as the one you are watching now. - -## Install & Activate `virtualenv` - -Firstly, install `virtualenv`, this is optional but recommended as it is useful to create an independent environment to resolve dependency issues for building the documentation. - -```bash -pip install virtualenv -``` - -Switch to the `docs` root directory. - -```bash -cd $KYUUBI_SOURCE_PATH/docs -``` - -Create a virtual environment named 'kyuubi' or anything you like using `virtualenv` if it's not existing. - -```bash -virtualenv kyuubi -``` - -Activate it, - -```bash -source ./kyuubi/bin/activate -``` - -## Install all dependencies - -Install all dependencies enumerated in the `requirements.txt`. - -```bash -pip install -r requirements.txt -``` - -## Create Documentation - -Make sure you are in the `$KYUUBI_SOURCE_PATH/docs` directory. - -linux & macos - -```bash -make html -``` - -windows - -```bash -make.bat html -``` - -If the build process succeed, the HTML pages are in `$KYUUBI_SOURCE_PATH/docs/_build/html`. - -## View Locally - -Open the `$KYUUBI_SOURCE_PATH/docs/_build/html/index.html` file in your favorite web browser. diff --git a/docs/extensions/engines/flink/functions.md b/docs/extensions/engines/flink/functions.md new file mode 100644 index 000000000..1d047d078 --- /dev/null +++ b/docs/extensions/engines/flink/functions.md @@ -0,0 +1,30 @@ + + +# Auxiliary SQL Functions + +Kyuubi provides several auxiliary SQL functions as supplement to +Flink's [Built-in Functions](https://nightlies.apache.org/flink/flink-docs-release-1.17/docs/dev/table/functions/systemfunctions/) + +| Name | Description | Return Type | Since | +|---------------------|-------------------------------------------------------------|-------------|-------| +| kyuubi_version | Return the version of Kyuubi Server | string | 1.8.0 | +| kyuubi_engine_name | Return the application name for the associated query engine | string | 1.8.0 | +| kyuubi_engine_id | Return the application id for the associated query engine | string | 1.8.0 | +| kyuubi_system_user | Return the system user name for the associated query engine | string | 1.8.0 | +| kyuubi_session_user | Return the session username for the associated query engine | string | 1.8.0 | + diff --git a/docs/extensions/engines/flink/index.rst b/docs/extensions/engines/flink/index.rst index 01bbecf92..58105b0fa 100644 --- a/docs/extensions/engines/flink/index.rst +++ b/docs/extensions/engines/flink/index.rst @@ -20,6 +20,7 @@ Extensions for Flink :maxdepth: 1 ../../../connector/flink/index + functions .. warning:: This page is still in-progress. diff --git a/docs/extensions/engines/hive/functions.md b/docs/extensions/engines/hive/functions.md new file mode 100644 index 000000000..24094ecce --- /dev/null +++ b/docs/extensions/engines/hive/functions.md @@ -0,0 +1,30 @@ + + + +# Auxiliary SQL Functions + +Kyuubi provides several auxiliary SQL functions as supplement to Hive's [Built-in Functions](https://cwiki.apache.org/confluence/display/hive/languagemanual+udf#LanguageManualUDF-Built-inFunctions) + +| Name | Description | Return Type | Since | +|----------------|-------------------------------------|-------------|-------| +| kyuubi_version | Return the version of Kyuubi Server | string | 1.8.0 | +| engine_name | Return the name of engine | string | 1.8.0 | +| engine_id | Return the id of engine | string | 1.8.0 | +| system_user | Return the system user | string | 1.8.0 | +| session_user | Return the session user | string | 1.8.0 | + diff --git a/docs/extensions/engines/hive/index.rst b/docs/extensions/engines/hive/index.rst index 8aeebf1bc..f43ec11e0 100644 --- a/docs/extensions/engines/hive/index.rst +++ b/docs/extensions/engines/hive/index.rst @@ -20,6 +20,7 @@ Extensions for Hive :maxdepth: 2 ../../../connector/hive/index + functions .. warning:: This page is still in-progress. diff --git a/docs/extensions/engines/spark/functions.md b/docs/extensions/engines/spark/functions.md index 66f22aea8..78c269243 100644 --- a/docs/extensions/engines/spark/functions.md +++ b/docs/extensions/engines/spark/functions.md @@ -27,4 +27,5 @@ Kyuubi provides several auxiliary SQL functions as supplement to Spark's [Built- | engine_id | Return the spark application id for the associated query engine | string | 1.4.0 | | system_user | Return the system user name for the associated query engine | string | 1.3.0 | | session_user | Return the session username for the associated query engine | string | 1.4.0 | +| engine_url | Return the engine url for the associated query engine | string | 1.8.0 | diff --git a/docs/extensions/engines/spark/lineage.md b/docs/extensions/engines/spark/lineage.md index 1ef28c173..2dbb2a026 100644 --- a/docs/extensions/engines/spark/lineage.md +++ b/docs/extensions/engines/spark/lineage.md @@ -45,14 +45,14 @@ The lineage of this SQL: ```json { - "inputTables": ["default.test_table0"], + "inputTables": ["spark_catalog.default.test_table0"], "outputTables": [], "columnLineage": [{ "column": "col0", - "originalColumns": ["default.test_table0.a"] + "originalColumns": ["spark_catalog.default.test_table0.a"] }, { "column": "col1", - "originalColumns": ["default.test_table0.b"] + "originalColumns": ["spark_catalog.default.test_table0.b"] }] } ``` @@ -97,17 +97,16 @@ Currently supported column lineage for spark's `Command` and `Query` type: ### Build with Apache Maven -Kyuubi Spark Lineage Listener Extension is built using [Apache Maven](http://maven.apache.org). +Kyuubi Spark Lineage Listener Extension is built using [Apache Maven](https://maven.apache.org). To build it, `cd` to the root direct of kyuubi project and run: ```shell -build/mvn clean package -pl :kyuubi-spark-lineage_2.12 -DskipTests +build/mvn clean package -pl :kyuubi-spark-lineage_2.12 -am -DskipTests ``` After a while, if everything goes well, you will get the plugin finally in two parts: - The main plugin jar, which is under `./extensions/spark/kyuubi-spark-lineage/target/kyuubi-spark-lineage_${scala.binary.version}-${project.version}.jar` -- The least transitive dependencies needed, which are under `./extensions/spark/kyuubi-spark-lineage/target/scala-${scala.binary.version}/jars` ### Build against Different Apache Spark Versions @@ -118,7 +117,7 @@ Sometimes, it may be incompatible with other Spark distributions, then you may n For example, ```shell -build/mvn clean package -pl :kyuubi-spark-lineage_2.12 -DskipTests -Dspark.version=3.1.2 +build/mvn clean package -pl :kyuubi-spark-lineage_2.12 -am -DskipTests -Dspark.version=3.1.2 ``` The available `spark.version`s are shown in the following table. @@ -126,6 +125,7 @@ The available `spark.version`s are shown in the following table. | Spark Version | Supported | Remark | |:-------------:|:---------:|:------:| | master | √ | - | +| 3.4.x | √ | - | | 3.3.x | √ | - | | 3.2.x | √ | - | | 3.1.x | √ | - | @@ -168,13 +168,65 @@ Add `org.apache.kyuubi.plugin.lineage.SparkOperationLineageQueryExecutionListene spark.sql.queryExecutionListeners=org.apache.kyuubi.plugin.lineage.SparkOperationLineageQueryExecutionListener ``` -### Settings for Lineage Logger and Path +### Optional configuration -#### Lineage Logger Path +#### Whether to Skip Permanent View Resolution -The location of all the engine operation lineage events go for the builtin JSON logger. -We first need set `kyuubi.engine.event.loggers` to `JSON`. -All operation lineage events will be written in the unified event json logger path, which be setting with -`kyuubi.engine.event.json.log.path`. We can get the lineage logger from the `operation_lineage` dir in the -`kyuubi.engine.event.json.log.path`. +If enabled, lineage resolution will stop at permanent views and treats them as physical tables. We need +to add one configurations. + +```properties +spark.kyuubi.plugin.lineage.skip.parsing.permanent.view.enabled=true +``` + +### Get Lineage Events + +The lineage dispatchers are used to dispatch lineage events, configured via `spark.kyuubi.plugin.lineage.dispatchers`. + +
        +
      • SPARK_EVENT (by default): send lineage event to spark event bus
        • +
      • KYUUBI_EVENT: send lineage event to kyuubi event bus
        • +
      • ATLAS: send lineage to apache atlas
        • +
      + +#### Get Lineage Events from SparkListener + +When using the `SPARK_EVENT` dispatcher, the lineage events will be sent to the `SparkListenerBus`. To handle lineage events, a new `SparkListener` needs to be added. +Example for Adding `SparkListener`: + +```scala +spark.sparkContext.addSparkListener(new SparkListener { + override def onOtherEvent(event: SparkListenerEvent): Unit = { + event match { + case lineageEvent: OperationLineageEvent => + // Your processing logic + case _ => + } + } + }) +``` + +#### Get Lineage Events from Kyuubi EventHandler + +When using the `KYUUBI_EVENT` dispatcher, the lineage events will be sent to the Kyuubi `EventBus`. Refer to [Kyuubi Event Handler](../../server/events) to handle kyuubi events. + +#### Ingest Lineage Entities to Apache Atlas + +The lineage entities can be ingested into [Apache Atlas](https://atlas.apache.org/) using the `ATLAS` dispatcher. + +Extra works: + ++ The least transitive dependencies needed, which are under `./extensions/spark/kyuubi-spark-lineage/target/scala-${scala.binary.version}/jars` ++ Use `spark.files` to specify the `atlas-application.properties` configuration file for Atlas + +Atlas Client configurations (Configure in `atlas-application.properties` or passed in `spark.atlas.` prefix): + +| Name | Default Value | Description | Since | +|-----------------------------------------|------------------------|-------------------------------------------------------|-------| +| atlas.rest.address | http://localhost:21000 | The rest endpoint url for the Atlas server | 1.8.0 | +| atlas.client.type | rest | The client type (currently only supports rest) | 1.8.0 | +| atlas.client.username | none | The client username | 1.8.0 | +| atlas.client.password | none | The client password | 1.8.0 | +| atlas.cluster.name | primary | The cluster name to use in qualifiedName of entities. | 1.8.0 | +| atlas.hook.spark.column.lineage.enabled | true | Whether to ingest column lineages to Atlas. | 1.8.0 | diff --git a/docs/extensions/engines/spark/rules.md b/docs/extensions/engines/spark/rules.md index 5c8c04869..4614f5244 100644 --- a/docs/extensions/engines/spark/rules.md +++ b/docs/extensions/engines/spark/rules.md @@ -63,24 +63,33 @@ Now, you can enjoy the Kyuubi SQL Extension. Kyuubi provides some configs to make these feature easy to use. -| Name | Default Value | Description | Since | -|---------------------------------------------------------------------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------| -| spark.sql.optimizer.insertRepartitionBeforeWrite.enabled | true | Add repartition node at the top of query plan. An approach of merging small files. | 1.2.0 | -| spark.sql.optimizer.insertRepartitionNum | none | The partition number if `spark.sql.optimizer.insertRepartitionBeforeWrite.enabled` is enabled. If AQE is disabled, the default value is `spark.sql.shuffle.partitions`. If AQE is enabled, the default value is none that means depend on AQE. | 1.2.0 | -| spark.sql.optimizer.dynamicPartitionInsertionRepartitionNum | 100 | The partition number of each dynamic partition if `spark.sql.optimizer.insertRepartitionBeforeWrite.enabled` is enabled. We will repartition by dynamic partition columns to reduce the small file but that can cause data skew. This config is to extend the partition of dynamic partition column to avoid skew but may generate some small files. | 1.2.0 | -| spark.sql.optimizer.forceShuffleBeforeJoin.enabled | false | Ensure shuffle node exists before shuffled join (shj and smj) to make AQE `OptimizeSkewedJoin` works (complex scenario join, multi table join). | 1.2.0 | -| spark.sql.optimizer.finalStageConfigIsolation.enabled | false | If true, the final stage support use different config with previous stage. The prefix of final stage config key should be `spark.sql.finalStage.`. For example, the raw spark config: `spark.sql.adaptive.advisoryPartitionSizeInBytes`, then the final stage config should be: `spark.sql.finalStage.adaptive.advisoryPartitionSizeInBytes`. | 1.2.0 | -| spark.sql.analyzer.classification.enabled | false | When true, allows Kyuubi engine to judge this SQL's classification and set `spark.sql.analyzer.classification` back into sessionConf. Through this configuration item, Spark can optimizing configuration dynamic. | 1.4.0 | -| spark.sql.optimizer.insertZorderBeforeWriting.enabled | true | When true, we will follow target table properties to insert zorder or not. The key properties are: 1) `kyuubi.zorder.enabled`: if this property is true, we will insert zorder before writing data. 2) `kyuubi.zorder.cols`: string split by comma, we will zorder by these cols. | 1.4.0 | -| spark.sql.optimizer.zorderGlobalSort.enabled | true | When true, we do a global sort using zorder. Note that, it can cause data skew issue if the zorder columns have less cardinality. When false, we only do local sort using zorder. | 1.4.0 | -| spark.sql.watchdog.maxPartitions | none | Set the max partition number when spark scans a data source. Enable MaxPartitionStrategy by specifying this configuration. Add maxPartitions Strategy to avoid scan excessive partitions on partitioned table, it's optional that works with defined | 1.4.0 | -| spark.sql.optimizer.dropIgnoreNonExistent | false | When true, do not report an error if DROP DATABASE/TABLE/VIEW/FUNCTION/PARTITION specifies a non-existent database/table/view/function/partition | 1.5.0 | -| spark.sql.optimizer.rebalanceBeforeZorder.enabled | false | When true, we do a rebalance before zorder in case data skew. Note that, if the insertion is dynamic partition we will use the partition columns to rebalance. Note that, this config only affects with Spark 3.3.x. | 1.6.0 | -| spark.sql.optimizer.rebalanceZorderColumns.enabled | false | When true and `spark.sql.optimizer.rebalanceBeforeZorder.enabled` is true, we do rebalance before Z-Order. If it's dynamic partition insert, the rebalance expression will include both partition columns and Z-Order columns. Note that, this config only affects with Spark 3.3.x. | 1.6.0 | -| spark.sql.optimizer.twoPhaseRebalanceBeforeZorder.enabled | false | When true and `spark.sql.optimizer.rebalanceBeforeZorder.enabled` is true, we do two phase rebalance before Z-Order for the dynamic partition write. The first phase rebalance using dynamic partition column; The second phase rebalance using dynamic partition column Z-Order columns. Note that, this config only affects with Spark 3.3.x. | 1.6.0 | -| spark.sql.optimizer.zorderUsingOriginalOrdering.enabled | false | When true and `spark.sql.optimizer.rebalanceBeforeZorder.enabled` is true, we do sort by the original ordering i.e. lexicographical order. Note that, this config only affects with Spark 3.3.x. | 1.6.0 | -| spark.sql.optimizer.inferRebalanceAndSortOrders.enabled | false | When ture, infer columns for rebalance and sort orders from original query, e.g. the join keys from join. It can avoid compression ratio regression. | 1.7.0 | -| spark.sql.optimizer.inferRebalanceAndSortOrdersMaxColumns | 3 | The max columns of inferred columns. | 1.7.0 | -| spark.sql.optimizer.insertRepartitionBeforeWriteIfNoShuffle.enabled | false | When true, add repartition even if the original plan does not have shuffle. | 1.7.0 | -| spark.sql.optimizer.finalStageConfigIsolationWriteOnly.enabled | true | When true, only enable final stage isolation for writing. | 1.7.0 | +| Name | Default Value | Description | Since | +|---------------------------------------------------------------------|----------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------| +| spark.sql.optimizer.insertRepartitionBeforeWrite.enabled | true | Add repartition node at the top of query plan. An approach of merging small files. | 1.2.0 | +| spark.sql.optimizer.insertRepartitionNum | none | The partition number if `spark.sql.optimizer.insertRepartitionBeforeWrite.enabled` is enabled. If AQE is disabled, the default value is `spark.sql.shuffle.partitions`. If AQE is enabled, the default value is none that means depend on AQE. This config is used for Spark 3.1 only. | 1.2.0 | +| spark.sql.optimizer.dynamicPartitionInsertionRepartitionNum | 100 | The partition number of each dynamic partition if `spark.sql.optimizer.insertRepartitionBeforeWrite.enabled` is enabled. We will repartition by dynamic partition columns to reduce the small file but that can cause data skew. This config is to extend the partition of dynamic partition column to avoid skew but may generate some small files. | 1.2.0 | +| spark.sql.optimizer.forceShuffleBeforeJoin.enabled | false | Ensure shuffle node exists before shuffled join (shj and smj) to make AQE `OptimizeSkewedJoin` works (complex scenario join, multi table join). | 1.2.0 | +| spark.sql.optimizer.finalStageConfigIsolation.enabled | false | If true, the final stage support use different config with previous stage. The prefix of final stage config key should be `spark.sql.finalStage.`. For example, the raw spark config: `spark.sql.adaptive.advisoryPartitionSizeInBytes`, then the final stage config should be: `spark.sql.finalStage.adaptive.advisoryPartitionSizeInBytes`. | 1.2.0 | +| spark.sql.analyzer.classification.enabled | false | When true, allows Kyuubi engine to judge this SQL's classification and set `spark.sql.analyzer.classification` back into sessionConf. Through this configuration item, Spark can optimizing configuration dynamic. | 1.4.0 | +| spark.sql.optimizer.insertZorderBeforeWriting.enabled | true | When true, we will follow target table properties to insert zorder or not. The key properties are: 1) `kyuubi.zorder.enabled`: if this property is true, we will insert zorder before writing data. 2) `kyuubi.zorder.cols`: string split by comma, we will zorder by these cols. | 1.4.0 | +| spark.sql.optimizer.zorderGlobalSort.enabled | true | When true, we do a global sort using zorder. Note that, it can cause data skew issue if the zorder columns have less cardinality. When false, we only do local sort using zorder. | 1.4.0 | +| spark.sql.watchdog.maxPartitions | none | Set the max partition number when spark scans a data source. Enable maxPartition Strategy by specifying this configuration. Add maxPartitions Strategy to avoid scan excessive partitions on partitioned table, it's optional that works with defined | 1.4.0 | +| spark.sql.watchdog.maxFileSize | none | Set the maximum size in bytes of files when spark scans a data source. Enable maxFileSize Strategy by specifying this configuration. Add maxFileSize Strategy to avoid scan excessive size of files, it's optional that works with defined | 1.8.0 | +| spark.sql.optimizer.dropIgnoreNonExistent | false | When true, do not report an error if DROP DATABASE/TABLE/VIEW/FUNCTION/PARTITION specifies a non-existent database/table/view/function/partition | 1.5.0 | +| spark.sql.optimizer.rebalanceBeforeZorder.enabled | false | When true, we do a rebalance before zorder in case data skew. Note that, if the insertion is dynamic partition we will use the partition columns to rebalance. Note that, this config only affects with Spark 3.3.x. | 1.6.0 | +| spark.sql.optimizer.rebalanceZorderColumns.enabled | false | When true and `spark.sql.optimizer.rebalanceBeforeZorder.enabled` is true, we do rebalance before Z-Order. If it's dynamic partition insert, the rebalance expression will include both partition columns and Z-Order columns. Note that, this config only affects with Spark 3.3.x. | 1.6.0 | +| spark.sql.optimizer.twoPhaseRebalanceBeforeZorder.enabled | false | When true and `spark.sql.optimizer.rebalanceBeforeZorder.enabled` is true, we do two phase rebalance before Z-Order for the dynamic partition write. The first phase rebalance using dynamic partition column; The second phase rebalance using dynamic partition column Z-Order columns. Note that, this config only affects with Spark 3.3.x. | 1.6.0 | +| spark.sql.optimizer.zorderUsingOriginalOrdering.enabled | false | When true and `spark.sql.optimizer.rebalanceBeforeZorder.enabled` is true, we do sort by the original ordering i.e. lexicographical order. Note that, this config only affects with Spark 3.3.x. | 1.6.0 | +| spark.sql.optimizer.inferRebalanceAndSortOrders.enabled | false | When ture, infer columns for rebalance and sort orders from original query, e.g. the join keys from join. It can avoid compression ratio regression. | 1.7.0 | +| spark.sql.optimizer.inferRebalanceAndSortOrdersMaxColumns | 3 | The max columns of inferred columns. | 1.7.0 | +| spark.sql.optimizer.insertRepartitionBeforeWriteIfNoShuffle.enabled | false | When true, add repartition even if the original plan does not have shuffle. | 1.7.0 | +| spark.sql.optimizer.finalStageConfigIsolationWriteOnly.enabled | true | When true, only enable final stage isolation for writing. | 1.7.0 | +| spark.sql.finalWriteStage.eagerlyKillExecutors.enabled | false | When true, eagerly kill redundant executors before running final write stage. | 1.8.0 | +| spark.sql.finalWriteStage.skipKillingExecutorsForTableCache | true | When true, skip killing executors if the plan has table caches. | 1.8.0 | +| spark.sql.finalWriteStage.retainExecutorsFactor | 1.2 | If the target executors * factor < active executors, and target executors * factor > min executors, then inject kill executors or inject custom resource profile. | 1.8.0 | +| spark.sql.finalWriteStage.resourceIsolation.enabled | false | When true, make final write stage resource isolation using custom RDD resource profile. | 1.8.0 | +| spark.sql.finalWriteStageExecutorCores | fallback spark.executor.cores | Specify the executor core request for final write stage. It would be passed to the RDD resource profile. | 1.8.0 | +| spark.sql.finalWriteStageExecutorMemory | fallback spark.executor.memory | Specify the executor on heap memory request for final write stage. It would be passed to the RDD resource profile. | 1.8.0 | +| spark.sql.finalWriteStageExecutorMemoryOverhead | fallback spark.executor.memoryOverhead | Specify the executor memory overhead request for final write stage. It would be passed to the RDD resource profile. | 1.8.0 | +| spark.sql.finalWriteStageExecutorOffHeapMemory | NONE | Specify the executor off heap memory request for final write stage. It would be passed to the RDD resource profile. | 1.8.0 | diff --git a/docs/extensions/server/authentication.rst b/docs/extensions/server/authentication.rst index ab238040c..7a83b07c2 100644 --- a/docs/extensions/server/authentication.rst +++ b/docs/extensions/server/authentication.rst @@ -49,12 +49,12 @@ To create custom Authenticator class derived from the above interface, we need t - Referencing the library -.. code-block:: xml +.. parsed-literal:: org.apache.kyuubi kyuubi-common_2.12 - 1.5.2-incubating + \ |release|\ provided diff --git a/docs/extensions/server/events.rst b/docs/extensions/server/events.rst index 832c1e5df..aee7d4899 100644 --- a/docs/extensions/server/events.rst +++ b/docs/extensions/server/events.rst @@ -51,12 +51,12 @@ To create custom EventHandlerProvider class derived from the above interface, we - Referencing the library -.. code-block:: xml +.. parsed-literal:: org.apache.kyuubi - kyuubi-event_2.12 - 1.7.0-incubating + kyuubi-events_2.12 + \ |release|\ provided diff --git a/docs/imgs/kyuubi_ecosystem.drawio b/docs/imgs/kyuubi_ecosystem.drawio index 723b306e8..7171491ef 100644 --- a/docs/imgs/kyuubi_ecosystem.drawio +++ b/docs/imgs/kyuubi_ecosystem.drawio @@ -1 +1 @@ -7L3XtqRYki36NfXYNdDiEQ2OdDS83IEGRzvCga8/rB2RVZkZWd3VfSrr9h03Isd2scARy8ymTZu23PMvKNcf0juZan3Mi+4vCJQff0H5vyAIAuHI/QRGzm8jMIIR30aqd5N/H/v7gNNcxfdB6Pvo1uTF8psd13Hs1mb67WA2DkORrb8ZS97v8fPb3cqx++1Zp6QqfhhwsqT7cTRo8rX+Nkoh5N/H5aKp6l/ODBP0ty198svO3+9kqZN8/PxqCBX+gnLvcVy/veoPrujA7P0yL98+J/6DrX+7sHcxrP/MB/Ra4nvI2YIw+kC0cw1vy/kPFP12mD3ptu93/P1q1/OXKXiP25AX4CjQX1D2Uzdr4UxJBrZ+bqvfY/Xad/c7+H7ZJWnRsUnWVl8f48ZufN+bhnG492eX9T22f5vIewrYsum6X3b6C4KWZUFkGRgfh1VM+qYDjsONfZPdF+Ukw3I/6c73Hb67C4ze75OuqYb7TXZPR3Efjc2Tpf66avhvJ/7ViXKSTiFwPz9O4/eZ3Yv3Why/Gvo+rVIx9sX6Pu9dftmK0X/F8W+f+u7nCPrd6p+/Ow0JfR+rf+Uw2C+DyXdHrf52+L/b8n7x3Zx/bNopiAdiL9v/5x2jTb/RSpgQ/4HTf2Baolu/zx0Ihm9GvEfnDXgh++M8/23Tb1zil0FwnP9YvozA3DvAyHT8+hNEBZ755p78Jt3W2xj3oYv3PbHLL1dy39m3i/m273/ie/B/7Xs/elf1TvLmNuxvPCynCeJ3DkTd74G9mzvwme+OtI7TL/5sjUuzNuNv3OuX3bXf7dA3eQ6u/keHrMd3c90nTX653q8r+H6vX9d/31kzVPe7/6B+Fxrf4+df4awE8htPxVH4r/gPvkpQf+Cr1J/lquQPnson9zQlbQG8sFgT8PZ+yVjK/x06/YE/kEVCFNCPSJQnBVVmP3gV+s9g02/R5l9pO+yvCEH//R/1G1Oi1I+g87dE+mtDEn+WIf8ryPmOCLc1tW/GdZ7aL3b97wLCP2Huf6FJ/zPP+bPMjUH4X/H/2sJ/lFb+NAv/4k6/MvE3SAfz9WWI/ysI/4NppskcIsk/sCZeUDn2gzV/zPbfDfJrvMf+GZP/cUL4LWj/K2Ia/SuM/tbK8I9WJkjyryT67zQ0/IOh/yiWnfVd3JN4Z60/O5j/59j9qy1ERhVp+cfs8L+FB/8CyxMY9Nd/AsD/veGN/NtJI3aTxi+2+Dve6A1N2XxxRmZb63ua71j84lh/PnXE/qgwyXKE+gMY+WdQ4h8Q0QSH/sz08Xumh+F/ACt/xPP+NO9Cfqw2f0kevzPeXSxP4GXTf9Xn7Nczs0zfSnwwa8kvb8rmAFb+h4Q8Hdd17P+hpb6fgc8BxUSZb28RcQKIxjU+a9ofSJWqkbn/GY5XC151v/I+9wPbcYwOxu2YGwfwIvFZ3Rd8ORPOuP9v/3e7tsg8pd+8k/+lj9JvzvLtldi0ekCDq1dC23E7naEe7FMwmM9HYZnIZKqM5T6jzj91i2MwiWVGjn22GvvR+onFHQEFn+VCVglCMBuLcz9oQiU7aEhS9+tHDnXC07exwYQzAp/nrVaE2+QsI9peKENxRx/7HefiaenTSjiE2fAN1LVv2jdGh3vnnYcNklx/HiyFpjxEv04BH3JDq0s0XJECUj94KWLyTT7Z0kDzLekcTc0nS5VoUy+n2XPNR0CS5hwGFPBbq3xF2Z0JYnhAyKRoNpyZGB+/p4hdWxep8AeyuN2R0YIWiwsysPiRyy4SsVZqPwaTy3c0zsORhEt8kbcHGsnkZp3Zu2tTevPvKGB15TXkTtY8efuuiMVZ5op0ghBuv9jFeiz6+SRwXlGCIn4M7UcC0fRIoscJ9+AmrJQu3Du2WfXppfamXIaaD5jX9pJRqKiwabdrsqiJRvfT86My4Jl4zyeGpMhTOuMidO116IkE2oeXk5f6ez91fE9JOv341cUxF2LO+C6C+dIRVur9N0rIIZojwCOCj9SgBq9LF2Fx0w3p7IfphIekMw+aghD5thRLTsJTe4iexbdPa4c4x2T5pXXQx/JCWilzn/Hj4dbqVPDJJT0N6QKfMRea2xSSe6xtBxuzvxtroGBgfhisQx/JwnAmrUqXADmyFSVO8llwmmQalXlQRHlDB9v20dMrpfsTvBf05TBTspuFohrc21JOzUIMzEgmeMj7BhHWfTpJIBgyu38Q7zLTpuXVaEDcsnDvrVsJlZLEfi75vWL2fUxB5scTV8GmLuReMs983CXA1CUiN+fjtHuXApWHDdq6FRD0YjeeGmFwyvAkcO+YUzqSK8mZk8YwKavTzwnZtNaEoIvOa01e3+Q7C6iqQqOxf87OboGr1XvdfMfYQMRT8R4W5xXnCJ5EzD3tqvC+J05s+ceLpWeqTEjfsAUVPTKtzz73h4elux2aFQa/Hp7+LLqiGX7NwLBUpuPJuf6CTbJdlk6L+50JXojvnMLeB+iRhMaD2pG6o87AFCvcW8ZzxqTwEiMs8LZagzFS2qm1fQ9N++rV8dgkHne5dc6vIdg3aDPY2d4M+aHig/CmapICXumZmFxlbg15Anym+HXAejWxCt4/JhAWE1ZXs5S6gSas+07Kh4EuIQFcQKIrQT+zMKHELeujN8El7MQOVMwuzFNbzkZsobS2alh/rM1HEesKWDj6PO5HGe6Xk9+36YCNCT4dO3Pb7Gw/yqkHW+4w6+s+gSw7KrdNkqas7oMHQRm81NfZKGphORQrSVcTq/HUZ6+S5dSkVBYP6s0tYXARlxZTar0ouurt4Iyzz8W9K9sPxvT0jLU6pHLXeoe9CB9KPB+nVMRzs+l5sFC8DTzGwmXNd9n8EzNv9pi43ISOG6AAheFji/Dy6WnUNIIeR/T2YSVehRRJ3eOQBvZAJwXhs0FADwNLbdpRIyk6T38MLirhEBmVVvmMjzlUSQ3jbO59xbsY8908CRumWeprj6refc8AV0wMF+ElQ0nsOdKHl/WZ4xDhGvW8q7zGTJc343y6UlHb+hAs0pg6RcHBdXfo3jII9E5StZI+OQszsoV5dJhdOZdqVy5V9VGqqJD5mMgRW3R141ReV/U4WMZFh7U2pR6pC7fCCSW2dbpOr0jeGKBIujW6KzxEvfdJTqoKMpj0hxxDnxSBXv0Tu5PR5Uwa3MIVHNRvxe+DY3ptevaKXKTWXkdD41n7HE5PcgBQaZ+g3doPlCrxYxpqs54kRDeSe5rJoE4FvUfoyPOTrXYA4srC5XJ2Z7fIhhT4YUOt/jEf7fP8nPVeKSd08oh3H3VMrBZacPhZbf6dnbi08IQLEUoI+C3pqLHmXYK1JnDT3VQg0Vvv2MJj7aiDLNOzVNN9LaXewHiOak+m6EL95bl60EurfeeJoyUOwjsu+vOOHb1R+Vh4PfT65dcgIy6EgprLcea2XltZjDtSHafiFT6m4GlMw9NUYOZtO7yEPZeg8BeQCq7ObwMvVBdtUh9kmpjDJot1aGRfnBkmaX9nJRt5zZzMYcUipnHjP46a7MOZXaJYJlaz7bsA9vzPppdi7rA9yGk9fAnpA5cFkwmVVzpLERZ9ghng38TOjHz6H5Gd+GkO9TG7kikFWek+YZjkmXT6bhNajvnY2HOnvDfB6LblToPjusiM9vY6PfG2OjVp2AC8k8UstmfnNigaQLM90AWdU12f1cwL16bQIs9lvpTgIs436TK6u8DlNoTao1MHYzf6z9QQXY04atqvPKxMG9yhQhWfCz4qcUtdjaB9euEQ9aXu8xEx8DFSoruSYN9t38NBLMakVPKoFtuvx41u5oAJnbRvp+NKSCpBsLSN945eevjppJervF5WceN2oB58F1XLMw1DUZmmhzrT5KCzYhGP0ULujB/45is2sEf49rkuMY50peuVzrKIxKZnjfYTuT1O/mHoU7DMAWAFkxi8m5x9i1uBvNeBzo+wS5BIYOKpXV+MjQFKHlikQB6uBxUxq/lbA5z7pTzlh47lyIn1ltDQIK+pQdq9JiS4PfYQ5myNXFOFTRsA6eMOOC15iPT9mRjJumZJEjXGlhfrA+COn/wcODHsJFbaqNLbeCkbjDnni3/JRZ3r4+ZsDRcf2hCZi2LO0yFnLhc39mTt2SrW8ln11jW+7GFI9eSERTSyxOU86GnSa21ImkelRWPOz58E7aXG2eL1QJQAXJpNvfjbl0SLajpMvRKiljhHejCET4+AJAWxebYjRXQB9+nJoDx0wQumgKv69c5IYgbyhJj5ua/a7Ig7xNXeA9BOvl+7MhjHPrZz37TvJ1Te4zUaZryabL6XJR9LtrR7DDuuaFj7edRZ8kn3Jme7FTxGE7uoOcRlH3U16juXVHceqXu4SZbkyF5fyckJYvkRPEfXlD+R8Xzcpq/vowUD4yDzXPLMizXw1GSPYsVut2d5wL/8142+ak2RlKV3raxUGIi+tpoeFTozoZsxiqprQ1kL9zVpMyKE7/HSAd1xuSSheqRdHTv0IPvaGZ5ctOVRGF3S05eRnR3tEOk1nLYw6/dxaR1vjqB4fQghPjYEl14jHDg3y+LCq7BnwMxALcSbt4V3cYVHoTGTQDXZzBQf6kuGn9xBSjcsxVtLDjwrxPAzyl8RH2wp1UGgWLzptikKkI1DaiIaKPsoPUq+PMXgdhN7tbcPjXfQr64Ku6GR1hyKP+JS8/0PORMfdQMOwCwefWUh26ZSDKGvZ9YnBCDAbCIlySFgbiK97Wuo/Vchzqr36NmRJuiUwriMRpKyGl+zMYor7bpmXTHTSziRKrCuGQb31yt7BDwEcDFAMeOTAYxqn+GSluGiPQR1LckYhOL9Z3zSVyJX7mhIHyrnSPtjZKzD4a/d/nwGv8LcXO8DSmTxwsK/yJvMnoOFkje4iJAJTKxebddby12p3Bjnc4AZhzRkRZL1yPJRsqW8H7TxgMf45cIcTpt32pQSiO5syrkSnsu7d9BQzvt0zUipNsBkIhhC21iKNREDpRjjeL5pqzgXKQqopv9IIf2N/JU377uU/lYtL+MGdmLf45p8H/oPGvq9IuYX7zwZkj9BDsVI0EvF/kGHA4eov+LQ37fSPwga2B/oGTAK/xWj/yRNA/ux5/Er5eo/bVj9j3Sp/4kM+nvN6h/q4/8DPfxPEKr+ZsT/14Qq/Mcux6+aVvrt+lXRgxv+84172zb91q/+nSBJZUX2e+NS/4wd/yy9EaXI37Wr4D9QHBGE/nca8scuhuBqP832n69e+WO7/Tu7T7+sG/uV3Zgh6c4bgJaf1vt1EwlF/gr9E9aD/xab/x77ET+V/p9K/0+l/6fS/1Pp/6n0/1T6fyr9P5X+n0r/T6X/p9L/U+n/qfT/VPp/Kv1gmfNfIfTvWj5J/m+X+vEfv6fyU9T4KWr8FDV+iho/RY2fosZPUeOnqPFT1PgpavwUNX6KGj9FjZ+ixk9R4/9/ogYJEb8VNf4XaRp/+ANAf/R93/+PiRngjxEYjlGAqKFSXfu5n5uw/Va2I9uRc2HlxuXN4cX4TGJD6HdDQhNoNINn3x+QoODnKVSKmr2nz3tRZmwcN/UkdvxGXmg/JKPrbI8S/Lz2HVy0zWeAP54cHkyT6kD0KMBpW+NTO3Utvt8n0WkIdUIOUUb1oPYzt27IMvBp30/qIovbJ0TUV5qGY54Ce/8ZBz0u1bfXNcMoHFOBP/Zr4P5jmm8DCmN//wSjVN/3Eu5b/Rp07ldfgw+OYf4XHqtVfn9wsNfz+7G4/+Tgz9u6X4PKk+H+Fx7rb1PzrH5/8D+emh8Pzvwwzf+rjvUP/CHtCSW+J5CTSct7MN2m9YJphrRGvA0SfgCi8rwLEYZgMJlKOssecb654+w+gvSUM80ABCctS7og/LJohpDhR9RZyEPiP0oU8aBAUU08RpN9MMIz7Z3x43XVZD7So3QMcpdTT5dA+iPLBwG/uK7jmVErF1QZm8V9gg3bzf/Erc9BJZTRBiBJOJuBdHZlb5DijYKN0L587wdF2Nub0d8fk5y9mifzGwfZDj6XUaUIcnc3BV0+mRmQ14hHAr6HkKWlIyJW32dWwpYR1PPQssR3WSHih7x3Fx5EjZrf5xUH1KODC/BRDBEPTGL4Rie1wXqx2H04D6YwNY2ZeLMsuKo95tO/4CAxCOxIXY6gcp0NQp5caWR83wfb2W7QxauIoe9GNDn08WFVtJKwy329AQzB26dDro+oPBiBWnu+XMkX3eZaxJBMuVl8lQZvUqtVNXSfzKI/hLe/htageSxjxCvKA/Zyca8bpp7CfmZPQCH4XQrI6Rs14lzqLHT5CQU327vfF6s/fkkVdVW38eDAGYXnlZUJeCIMo07IZ46fMMMejC4DPSXKInq1fdED95PDvcUhhmztGvd0ZIpj3gqvaC/bC1roaL140T6FEQwq/g7ej7vG/nYNhIPfjx7hFc9VkZoEMGo4MReaJyT9at3VmTm/dVRn3TXLVSAgGoJL5fI9NvpLnGlsQM79nsOYyZnM1JurDj86MFfBXskelqhXDd3bQndXKxE2rYimX/gY4wG3XoRXAywOJqRNoyYEFaMC3RVzeDg3zrOGzyYXiY9W+XxGMCJ0ovrdWhK6oqx+UktoWQ+WtIZruLoj7ZvdJ6AGtx0VWFCDtEqwogyrpBVvYlwbJ3qbWkbY6rbUmNMEuURMPlRWopbsXi+kboJWfm0BzAsu0v4SqfXhSSO6l+/HjN0npNnxpFQQGhdrrXTqGS7488NuOvu+7R628ZKjuIxYMM1+6e9HDu53fAIy37DQtvNwiJLIK9+GWKqqkMwXGk/pBo/jjwqoCAKtbVkxZZN/jiLzBg4pSkAjF5RHU8rYY4Z8Iydzre1St5MSdPmnf+hNxVZqQSaj5VTlkgUlzU7mMinPk8kwUOKenfk+jg8T8WKIEK9W6gwISvv57W7CAwipeQOMp6BvINE3Hx6PNs168x37Ddn4SkQ0eqVpQ+KXhBFBTwfoCvKBHSN51BuGD49AifiFyML+3tAkTCRgegfjT/HJLY6Ra3cdV9nDzsU1QBPy3NPD06YP7M48ugPdA5w6uSZiQ0KANL0ALkV68gsSliNZAvhxTeRJ68WU45Iefcdo+ChvC2nzGG4Dkz2jV7A0hrFEmFN9xvjku+rFIA4jFzfdQs/kKVo4Zt/IDHUxwJ3VMsZ2tSfKTfwxP8RdAAAWj9P73dHneqnCtVp+Z4cHOni9W380lS7p88Pc9byowduR7Yk+F09Zab7y3cdKgIfY6nxEcu+lLyOiMZNAAbbJ13oX41449C1V7gfXJuareASvjLJeiqPLGiFQaEXcsV7GBE648IBaD+4Z8Gz7lB4VO7TlMqWH6T9jp9NLe4Tjht39uOOPAymsPEjNaVpls37GULW8QHzhCMMU5rXHZHFDAXFX79+zlszAiWZfYB/tApVrkJuDfB5KagTnaspV+YRFMhEoYouIuomMvbQ2oE4EdBoyryrYr4JwkekZvT+Xts5cHlI7FNGLNYb++ZJf1JDTlvvZ8IFnsLDWlJBoeFXLQHp8FNlrp192VBJUBb+MlLQvj4gNlDyQvCQeCB4ibW2I+NvRya3Lbxx9/JI/y6Y1ZX0n48VEUcDjUyJML6aumflja3jUuxuupTKdOkUZcHTeijE2zkbow6QJ8zwPir6ZocLkVF4SfIF02XBp6Z9FWEtjC7T0BURcmQ0yQV/7DIL5Ldvy1T52/sSVBi6S5ZV05+708zUML9qJ+7QI7sLw8w3jWWbKVAM98z42jCt83z4rklvDHbguhNtMUpieST2NXGp9h1cm7nyngDpxoCUUqBtwaXXuN6QeFRgV7vDDeXYlgx0p9qqECsDvL7+ZTDwhv+2nz+bFJgfJCEp+vBvLAKUt6NoA9AB9wL0ZSorEInSSSzdwACicKt1JJESuWfU+TaE2G6W3yBi476crSQVvdbHZuMNQog2c8CDIyZIZXWGcNjedZaepwhGWxJBfAg0k70X2EAGvvQOct7Qm209ejaQXMpqn7r4rfGANeD9Ir3pHsyYqpDqnvVWWfSdMhJAZFH7jUdUWgjugpc9WFJ9nEk1HROkbulmUgzQ0/37eYAcsYuwaOk1U+pVD9xCgCpRw0eVH7odndjyJvU65CInVoHToiUzP21hANAzcSneMuxOedrOeGOo+ttnwXYLIPEbHPdlCJj7oPiZKgvkzr3fYF29yPyq6mJOzHz/zs7Du+oZ+Bo/MNawizaNuwKM1rLmgPf1djqZ32oI5Lw5liSteu3AgM1xFRH1kjyw3As03ipZUdjY+AovctE7QGnWTX1DvoiRfobP1UatwKhzKlJ0SwiLouVXPch5Cm1/vOQtLQGfGKfMtLX6hlAy9hjiFedkrRqV1MbJctP2A2v2B+R5S+jxyAj9gis1HvMnatNzjJ7S8iQf7yNM+ljZqlbGeLgzrA0JqmbczgEvJgi47OXU9KJGLPgDzI3DPzQS32CjxmvpJfzPSneNEc1of3I5N5MH4MqXfDso6mM4+90cNu4kDDAa/csA6fGZbnui4NgLrQh31qtGPw68FDj2ZMaoDaBQDrqv81+mgE+FmPgJAp8jwxnRqYUxseGztKHpc7tnwC76jk6FJL55ypi4OCrvHTkbQgUByvsk3NzdcNRblAC4RenneDdEupT4ZQxlo2xpZdUNIrffK7+kMsMyL0TH3teWP5+FzB0vND7CJsLBa6cLA8GM/uSTicBZxrF7eFmlJxz9Bp0UJcRZru7GrLc52ENLK8hvqkZWBKkcuxodsMPAp5/0KmxGfe59JogQyAwwb+Io3nDHUl3Lo0Akx7aCvZpSy3XAOJYZwkmST0tBAndvtSH9xV/hk3AIeSqJjSfSKY6EfkdD2x+erPkWFutG65OP41Af3jU0au20yeTM29tQfkVJLKbVCHvMaLhyyASDKeBVS4SYlcLqF+9GXrYcu1ucuYaw7s2v4MGe4OqLytIHYNjfE1CdCekEZZkgu1uegI4mErS6V7xKRfVfgI8B5tiiyBv7o+uFFdgDDGpwkvuaavRkbDlxNtx1H6FoFxPRu8jTiDFwrz3spTdKWdzulOuKcn0TZ1mnFamENpWcXzsPKim5LadsdW2TP6EfV8JmvfoNfcp5IP1e649mrlPjJlN0kPcWt3c9NqYhcdcYhvSMJ0G5z55cn8MpDdkdiMZtwXypsCnODD2WgAMaO9Kq0wZe3k822UdUnanyeFbmQab3iXvelIEugPgnYaMNBxURlRoM/zqQS0bfvC01POS8PvnlQYEFI8TSxTzUv/PwMoYToMEf9oMi7Qy24FyeQfV5DuGNnORxQLmghvCfRVjMkWk59bJIxvS+bq1tk2VgPC13HxiWW0hn6k4IhGLrY/k46NZlQoLlthy11dNHx8SSyZibOFutogzeG43HJZqqgIU3KJz7T2xVyC/k8sGIVeAlti4JIbHYxW8m3vlkrj+7DVXfyNowi8F/N7BI5lO69pI91sp9Dh4MdhEl04Re5c2o7vyH4LnWvvkOgq61O7niI44sxmk/81AVnGJkJohWRrwaPwQbQ1xZkjyGLJPXPdVHc4yNVDpJYPeDPqhkfpMgP/m5X1WOHAnvmbFrOiMTHCZ299pWPA+Bv5Y4+F8ZKTbK9Nr0yP0zaVz4gluyypP1ExN+yZdGXOMihlOR0GiVcxrkNZUBr2wroQJsMxU2il9dHt70hw+lEB9nZF7+I8hHTTuJ49sdfxrN3DnDnbLozE7Oh6AAqNNCKkWYSd+Ao2MVjzW3dJaJTz2+ML9ZUAVgkciFSB8PZv1mJoyaNkhvAOhgFNWA0zWMSf1EqwywASOkyJfDRQIhRjG+CqoYi519h9+jQz1ujjGK22TKwrqgnaJx0Xu5ywQi90UPmbYeUCcy7oln2LQvPVr2mzlTE18B9XptGjXqlRQ1Gacad+jLNjFeQVcp1Ms2mtlv78yr6xZvMPmC8j+Qx0FPEYJxz3m4ibbJj59c20Ef9pUsIxgumiS/14WFeIQriuWb8AAF1aDGgKwF4VPjOlQ/OSkQRrlWIRR+QvU2XRi8yGcTlSDeJk04w0W2nCVVyJxqv666nySGMVeLEQqMkEIBZXHcf6VcN2/qPGZQc3apJIfQ2w9jwow7aPewCpkZodAaBzpZGMt0ZVypslU2Ju1ZmHLOLOvVd6mS9J7mEu8DbRcsi05TWZDR8g29NihF/THKGvAHEvd9BNGO72j86lzhQQ2BsWJriPX1mN6kV6/kOhkcLsSpxp/EPxbweuui0T1WXZ8VQYc8vgvLToFN6aq6xKo9MwmQt1yK/igJthq30vGc0dSsKgzXWA/9XAJYunqTYCxV2k3adMxr10gMy3VHFf9El9knicXuUTvrVs5pBYRu+3Gu31KAGQ2XnhGB9lb1pRQXFm8yz/OPsVVRgPKpLUWwHPHOpFPZcdlMqTeTFkuRy6gE9azvjZeITwyBRuxNEzyiNnlo6EAOsOT+wrs691OcmYwOw66HbaK0TyPAP34EeDh2uWP8F9DBafEkIDU9hZ4hoAs7LkYlADFCqHtZhHa5coyIVKMzhihf5MnHV3hBAGhq0Gvz01cc6k1nbR3GxoF0p9JqglQ5R7dOU4MdJFSbAHSPZqMx8KpVkORc43folxlnsG8NTiDY2ideTadc6cu1RAtukucHCTkQUJrU2dKCksg/5UGN4LN0gslNmJwd8gJCCnb49q5XlJz35CF19CkY8UipI3ZDBeDB/nRkllhLVCqKL4vjGz3agH+OWjD7mbFIKj/vLJKVlxZv3gWEfIrKu4GVNDz0KqVdsY3LwCh2VWMe3QS4xVq64S7PLrnggmwzyI88qhenwVy7QG/zJrUYpBVZkr1NH0JI1ghYsbcpyJufv/KperPlJnte5+pq38E1Q1ejma1bTfFWoMsuPhFwMDx/nPkx2XovIaKiTCuIN2EHvvcb5pg6F7CLXBnpZ16dVP75FkiNcwmZjS/JNRZLa4MXAelyd1vfrrBRANdyG3RLuPGdn9oP3uFgylZdgRZtYdihf3BSzLUCaHCXNXPVIZaxNnVtP/uq83ScS68mGl1yG4etxB2ozk6K1WRpllyelJWh5Cevp2jybVtlTZAUiSaymtGXHqULPj0fNQfFw6hG0yfZrKkqSMP0QQQtGW7/kl0pqw/jNt0Yrl/HjJJ9G++jPRXtlYVfJDGbhdDl7k2B9imd8Mfvx4D6U4kIPa3RYiPEf9RhZflmSPv3ZmppBDQLDv5YUKecCig8qVfNJnCxwLwdTDeBHLNgPPi8UY71klIpVAixJyi8RVCAx217iu2UyfchkVtq44IVndnkRtrsyBhMMSHfSDNt5Jn/lvMFKB56DUs4eujLEWlReI7jr6vgradQ6MytCkDjZi9HcN1yXUHbXdJdVKb05UxbDyHIcnh1JrdsDgKInmJ71+rSx6XyiGJs2c/uWGG9oea+QzB8kMhs2dOS8VLmp8iiP6ZNj2TDNGyR4D3BvomCNSPzRh4/Tpusu2zbyuqZoomkiD9+Gtm5yPg30tgz2WbwwzV4RMiqvF5wUKglYkziZvPXihTtkNw1TvFSovWfESDzd+1bKRDbA+cEoSLCWx9lICNuCL0gHaxNeE0LiJWfzCKodpG1g3PgQQX1ypNMLfjB9zlnWWqHZprWzWj0hB1NRNgnUzIZRriGk7EmUaVUs+h4KFNasZF8KTVMl78W66RHqr3J1Gg8eBUplJsRgYUtgeiWqlUB4da+qRvY36UC7qyOP6FK9lzg+LTlrmCk+rixSK/nNjv5wnXFewLMYrATMnfheq8AvdB8TVQ7PlqWVxahyqzdtWdvO4THEhBugsO4LeWrD7DlPgZbrdLuKTBxzA+6qPfNKkZg/kurKKzCEVLEcU7ydoz0OaFQtHmKJ02Nx8kV42FJsQao/rlx4BFW08AUOFq3xXVpSMml6GH70+Si53UHdrvFGT4Ko1lCPlQ/EpNqAhya8pxrPDBtmuw43MnnOvqtEYz6EDLNFU97VjUdROH3gTwWDZJeDe5QCokR1jJ+KveNpztj+/OZeKAIW+iCAhrHAWDtTMfJ7QpGlfpT+L6oi2OdmEPfjDYWS8859CI5WHAcJzeVeM86FgGZJuQWj7vvVPHvUu3hxzWrbFsaRU6fzS1smgkYMfIgUY+S9f3mReSnx6oQ06+NhqcYPb4/z11hgpExuC+rm0NuG+veV+yJRGINLwTH8+ERk3rYvR41EuGLwB4v7BGwc4a5A8VvPrM+OOE38RRzQNzFTcJW1WFTg2V32NfwoQFJfawwXRmT/GGvl/d5pa4W5Jyj6gCqgcd7CdLGZfapFSwVBZyFuVtUXjDu9QsY2M04OFAXcUVUl710v+eDysjUIZFb5LMRHvS5KK7mSCvEcJX+kOdxijZF87EuzEb9dZxghA6JZ2f2B8coTqj41wzIC755JiBJAar2LoweeFJzzUWxLer3Jx3Z4NrPXcBXg9WDFvgfjQbCTXJ0aInqjrLvfLHPOgq/2+VRGpBsMd9oF6ZIHaXu+MOmM6BDgI0+TsZdaWBahRPWI57m405QD7p7kgKfDRW5SMehHDdurqPyZfiekiTadp5RzSJPdvHI3/y3Md3tP0AVJwg17hyxv9HvQw47R+v16P3GiRX3DBglmOGX/UYRph5Kos+xQJyRPuxXEmdx7Wcy/t2HEcoNmeXtcC6pSfRf0zaau2bmZGgDwsMoA32MVi42zzPlIxCDG76M5ClS6juB6xjH5JGCnl2AT5Yrs5LjcYNX3tdrY4zDr3sw0bapYnTaEGfaCVwfI6krQAvvU82lrNawYSfR11CnQXj+HWy9PrvI7hO/pt4oUl3ig2V6mcHV8sM/D0Bu0UDNdTNoXDlY9Znq0+N6IGvbuHa2cnTQp8RggugAReGRM7edSSBqffVckPx8PYt3oXcDOVh1g5b3SO14IugN7GpRQnynXV/8N1OS7iiSf8MLhjM4l9oITRin7uiAy42tRyzBtcOJUr8qnKaFhuSaaijWQB5DLpVYLp2Z8EWkZlKHf7eFX4F8Hf4JOmFeAcjKamYeU5fGLxPNBeIXfrhF5vAAlLJByC/cTXCPrtXISFPqk0YqdkgSW+HF7ecwV33C95xLilMkDfu8pN53cAs+vCnbZl5+ltLDXH95e5CkVKHj0WEgRHNaW84UNXCYKD1OeWQYDvhCe51F/SMAYj3pvtm8rmNhGz5x9KPPBcR6pWiE5WG11No2uGu8KkgNxbzkmq1T5eQCtBphX+DDmW6RetVq1gg93Tr0Qud5wN2jzO/8Mt48+PsvxUwcBDelj35QF7jZuDqoOBvr4OCVzuP9sP17P3mgOQbEFCP3+5uSitFko2kg41T0MO3KaQIsY2mYoHLIl8MA1g+uzoPdGs1IU3dDMj00E2IFrt8eZzfWSuHFhn3NXSEv3LQZwSYirdimBhAHxhFbQ1jlLL69ymEMiFXd5XfCaLkFN8qsrCXCRvUW1fpkUqHCRPiuZ5bl+idF8DgPHwYvIc7inSmjZwepIoYdh3EzerrkQQVIircKF/5Tr9mjQVc3ebf5IwFLSd3wNW5eLSjV5Thbe/G2Ek8lX4SgTCe+GDbY6PKBVTqH5seYPQFvjQTD0Z+c7/l0BSal57IbvoDS45auzyb6ahSqUmmROkdySUScZjNQTVvSVrXB5Jl+SDQMKFBSpoUk5U3gAXyUQA75iYu8itDwHrHZfjJnh7GVcGxoYA3yqjgto5Yw16Pj6g4NYYnqgUr5f6uF65ToTFG+0pysjA/q1mH9K71q8blgchabJROFdP7lIRbOoCDHQfgNVPislBiY1vW2PYE5G0fIba7rQTjZK9BwgWrBw6mVpKHpa3c5dvh9J4+2WBLazd61JYE+XuZO2ZAG2KxLWfqhFX2FEVHRDSBMQNFbPj0S3nKBkiogt4hKoedNNAhXptkDA5cMNVGgNwf9GQ/Q04xEI/NNrHvpwKFQ0XI8L3qPr4dyVo8glN9/Lz8K5k+ELpUkX2w6KxcNYSy0c7nZScs5zdzpgmJ3pIW015RrwJnUzAyIc1mAjrHQI87o8P64QbZyb0JZcCR2zeU9TypVc0BLIwqsV05HcbLg6Ot0uAUpdBboZBAl4Gb8rEfyAMDYzEYIuLtA70VdQO6rTG+39sxUVAVrL/FgV68G5BSoAzycmZ+vN1B/OeIrP0FgOMlGj3oBcwnkZwtBd17Zic1RDKFiGKTKdznOcUjM3Dy4Oq+IrkPYy6/SeNmUyxlgt0H7EMBLOxkk76NjciXNygeXJnKUU/wI/w8ZCTzoXduBAmzmkiK2NHbXQInvMRgLL1JTAH6c6TmoHTIWeRtLlOw80Rl/KdJFmBMenKOApKQvk43pogv/kXq8ce74Lq6YO745N1rPNilUkeragbaXX3H5T+gY3LLNgG9JIBrYJ3gmDSEpN7xiJsaKIhZXKLNXClK9aQvS8vEw/VgVQmjToMjcUDu0L4BMzqmk9l9JKGq0yLOUXH05fh2r5R9O+L04ZYdNueXfHtfo9sBAwFzs4i2TGMdJ4HyCU1VeCdHAGPz07x1Q7tofd9Wmhska+7JhYQBn1o74+UGg+d+8dueEGchp6XzWqnDBzV43+sS+i/Y0pDi/jgheCzQ9XL9U6D5Y3ik3eFKUNaICRA+GIkIrj6Z6GFsN/MgII7mJIewtKp2eHvKyN+kz1rrsJYWPW5VJM3fHJTbo/G8tBSqsMPRK0CL5Kcdd+lT2KX0Rcfsp91erOs8bOJzo85+SRHGMV8U6ISKcu9VhDO7lx3ITDwTmZN5Tm6tCYnhJ1FLGHKb8gwQnujC9FouSe1b4VVwA3/WFkUM5jZVPiR6HWs7MrqI/bhZlCVGp2mNMxKn+N8hHxzXwzkxefSE+YyskxPORJ7qGefxLSBgVtjBcEJlxxN6z8NPmj9hHM0g762YBiNV8ExiKyRkuz486u3zRe7/0R9w+K3nFHslyFnnpY7UzHO0jwSDqXtFZ5+lhSOL0PtmIwwjzdDHb2LNFshVO0uiYFcyM7aMUWIb8RKyD8IMEi62Wi0cN8yxc2jVb6kTdND9SxNQBiGstIzbo+tnaCaj1JL5j9TF1JnWro5iBrDqyoEnMddhfgkkW3+UnFF8oIQG5yE3897GIs8NK30RXWwVf92CB1b1hcZC/UsMhACnLdLE8ocQIVTPhC+CRPxNAW7Zo0CVthw92Ai8PWYwjXXqpvZ1SiJijnEqrmpQ82F5mQJRtET5jYqOU8cYHEGck4mUB4mONbD4k8jJSKxoey2X1lqSYUxg1NtI9X2yoTWa21o37K8sghj1pOyw8HlDgEC1L2XSXE8oKeOBvz+arMzTVRVNO3KdAUdqxnkPYV4QwEvf0uy7ctvzx5i/JdXZjKT+Y3A2AVe/aEMnTjgMDPRVB53E/c6rMGIY/psxvKGzY3nhFMo3kTn2d7V2UXSDsfWWUNTpjRhgOwVKNP0DUIQn2NR0Rx2ZVMenYmpeulaiWcgtRKV4ZPxbYIz6cxn3vvHjPe63MxnzhqIr4V7Cthem7Zo9YhEbgCs2f+zjcTT4kvNVCLQDzzO0yREXzJJ/2G5/D5GBc3vWO/Htkb2j702VAiLzN289Dg+54wHA8/kPLShm5DmUQ/ZXSQDvsobhRhuPR0n3TXRvZWlE/lAekxEBe2TR8sGqnJfWftmdrzyGM4eDX1SR2LKUPe7AGtTna1jCQl2XseNLp7XiGe4VMEf6OAhzQPpf6AE9Xjpohln0NidoUxTZ5WlVHu6v2kBSVh+2DW6radVuUEGDdme2QiBvhSC0s1wovuas3qStLsw7RMQ+PqjVnCRGafYNIJ3yJtb7Lh+l6Ix96dz2nk2+nBUoCbLzcNbiMpZsygxcjDue0L3cwmC7t4IN+FgoMHDsaD/kvhwb79rlo1IRP4rnN9VgnhQT5AmUnKaAG6GE9LHhR/SDPQFflUypPjwkOv1TqGliKSdJLMka+v8SBGlybQUIByCFVUkMF6ZwaNVGfAX/yTyz2YcaiNsq4PM+pVJnyDkY/O1DNnLvYkvANSgu04oKPw5oelETyoc0eu9qWQFCouw7pkuK4LRjPjY0I67p5hXoFH22zfcH9nfVEOjIu/IQaRGAFiKXMeZ4EfVKlPandBqwRlQ9nbWn7MuHMORUrcUBKmPO+uz2MuRFsFkTK5pDv4zEDKHg2VdSAPl0ZOs2/Ci2Bg1YJ8gYSrlw5HTsTE899uY7Q/810gwMFuyp9TIi+oa1hhu+uyb1855q/PGAq8Y3zaIEmf9dtmldNdYYeOPFqTKRglCo7RQPXnS0e+oWmBFrXfC4Aux2Wu1sOYuT1QeIOuB7lRhPjkGg+5PEgSqr2YhDF+Zo7F396M2980XxtufisebBScFrG1MqYYoO5gySdOretBjcVpPpJtlcG9rgPiBm06iMlQVsGbPQXHAaa0LSKA83iRdLQWjV7/drfJEG1OnslbckxYD7cLWvILXXWJ0csCLDC1N7m8L1sAlCSBQSg086Ls5pOAvdMtKAo/s9WATjabARIBJ8kObeikuVPIBrxyGowAW5MY9AcODjJAH5jE8IfZA1GUJtmZMZ/8OrvSI3NZM6rEJOkE6T6tCJHlHGGFBYDuDR6CY2KS1mBPBI9TegPaUv1kIBRmPAH8KC67J7UTQ8TBsEynkklohAuse8pVzBrvXZNTBAWIU9B7D7tIGBXnjm5u7Kpnsx0hI05ygN6e3RKSzzW7DILxipicp2hLB0AJPOAJ3B3UPJMAbDivz6cM4ka8oqjWSLRL6K2zWXayZ0bja6Ix7IkVGFsdBS8Z9Ip55BJYCSU21AWarFPwJE61teS8TLpx3xzq1U+Om/esJhrTp+I20i2fc0plJ1NlaYhWQa7KR3Z7Vbg7DUq7PSRgEufxah5rjjp0mTKrA1W4kA9wQgS1bqS/NrzFQ358YAjBZvXwqXfrPUEohzoHYvcobQ4uhFKbdMX77rNr/vb7J6k4NFjSASZi1iyPhKiL3xqVqmVbhOSHhPP9C0vc101cumj2WRvGbsRj3QVhRtvoX7IoqCGKPskAVuF4wYbgLr04RVn7J8yzrwbmZ2yLKOk5KBfHYVwoqFnBpj7N29Kk5Ix6ipLo7pW1kx7XynGuQ5OvHxGRmdwTM43jejMmbp+4BzqB8sRRYzWK3pPfWzJHA6qM3z5pIzWrHYGhL1WEj3wlgHJuuK1fmQz6IUEYoHlO7bC3kaZRQpnIf3H1nkx9X1PYExg8ldSw1PpaQphetjXwTZ7aA13TeCxJ8Mysg0vnLfhoA8of0c+dbqTF9c6SkR0+nEIAC6x0pPXDIP6gIwDlna0aiTchhLYPczaNpEFKsX5r2lens+sXi6vioh8Vw3avO/6Fd0ia2nXBXc9YSu3lriPUrfu82bEIh7YrFHj8Bs55HaJpWUP9zXYLW1uDsOTiU3cq+er6VWlJv9AY9ZnX/hWvVDnqxmM8EOg9MMnzaaucLdTV8915qFp0dwz17ROauIrRWy4mYFFUOI8B7b/XvEXqcNXbk6mYZelkriaek+4392VFmaBb0l054uTusVWakRfvha+aYhbtogkPxGEXZAlpH173vLlEmnAcKx7EWpnaIC9DwSYlczHtmpA7YHixohQXrT0aEhTszgp9whzBVj6vzccFuhrilWJKDZu7UbOO1afxp1L320UUBK3lZ/PWyPmkGITvLIt9fjribV8zitucP7YkWsBRJIfSTa4KRxd91keGkohorBDlDjH0zcNtTIIUdSKyzQdf3ut0yoAt/wVDJPIhzpVrMvy1RoFUW7OJacfaug9zI6YF4eO7BmcQFmbwhyf6SXJEjZFAAbXYjRzdbM+33hZNZbmR0/H72+r762NpOJvvHMHtOIS8IaCW7WIDEQT9cO55PXFyCmiKhfBJg+RzhNxCWVdt5QTD4t7tDUJAcTPktI/34Cm5wScqhocHFUdOJjJaa4qbnjJPsYwunnHNv1gGJAr6EzxMMnmD3qJ9uBeFfLXRsyRrcSKuCDe01VefRjtLSIeV0AUSytLz9O3G8Lrsfd9r5xA6xrONHD6KrNzHflH1N/aSgMwPdRTRqd5d3/jKF8UARN9MZZgmGr9p5xMbGYN7ddIjKIwO1qULMOsZuMrzpFKqSVRaLAu96enZkFzk6amfOh2jZyF95aB0nkclD2hVevW1OKu9XjyLw7e8PZOhxUfSTJy6Dss1IjZEtXOAEg00aYESBYz9+p0Fw9qMqkyvEeIxxiaomjnDWmUSpX3BIOfc7ofPrzRXco78GM/xEVy+fZGCc3t6gGj7HK2mU+CsAtIxQM8D026s4ojypFDICLWXEdyV9CfaKMifHgLufjG/R99ACqIsadfmcQ9w5AqUHc4rHMOKTMktDEK89RFoHEEC78uTM3w2lJU6La2a+eYKtwNcFCcIlLG+sFSxGO1JrihGx/4n8jQPJGuUCTyPrz8sjhYK1RIPWOyBXrEaQKSxMj0rXUAQzw0QhOUMIxr6HI3icnlen1NR+uXVoO8GKgynpE8PnfoYx44hEcT2wxBB3562ZrdTOhrWfJ7D3G/SkKU0+yGyglMGIHhRcPqi9id2TcRdirdqZ+zhaOayf3aC8Kq8z2S/uZokaHobvlawkieK05sd4Wr5IenO3Ml9CYM5zMytXvjYikN8w+SKXyV+zwh+DdXRO7KJxvB3ggyTZs8V0g6P3ImgMTJl1x0c31dXPEHULhNDlZSNTySxDtU4mNJtHtD7wPznCAf1o8pMxusZ5ymzoacXeduz11GudROzz7Q0bv2PB3UY5e0MJepFj35VrphjGeZ88s9ZgCgDvT4O0zxXhjNkLcfJYNz9NuYqpVe6tDZ8hR26Z2NnU/1WW7L/pq6otuTUR5sHsOiJol88mqIE0ngTiQ9296xMm/yi7zK/d/Q3gw5qacWnOr6Zw43nimRXxse28SPkW/ZyB5J+NQSZdMF8pLnoYDeHUNHQXRgdKoqjjJ/jqxkSG4XEKSbfx6pyAlN1VK7V4gcTm2Lier7cShZ57bLUNXdher7h/Gj1mke+dOQSFrySnsDyH9AFvKgiDHMWsgU7CancOGb1FcMyxYKGczhlJZObD/tju5qWxXgg3lCtvV33+BQd1dmdsHcI+raWAn3gpLkN/oNSPGgnXsJRC5+glmGnA/J7uE0HNPuetJQ5VURrDaU1alrMVk+vD0sxxZt6PArtFR/BV4OE5Y+FOoHwLtnjHY989FLRDvh6A20z6cri+bLPtVX5DRI1Z4lmKDVLInQ7a8tu+3EbXZmPbPOe+l7Fn/Qw6zDGTPyLCAsMqguY4q6y7YuB6Whu839Yumo16Y0t+DQ3F0MoZubJxDhiGj39Ve/vwIH9eXel1oGqOtCpmTqhA1mf+V6Mb29RGQHiKrKS/p7TDZOyolizzAt8sW4cw0SKhI9z9AqluU+gWWM3idFQ2MY8J0VrfOKohcJrcTXmzeJbZPDVnGkUqkeMogqm6rwImC53lYjjGjc5FpYwnyLVnO96oNzR2Es5uI9sPIrT5J9R1eNouJsZ93Hpq6TTupYY3QEXLWvmN7d9OPoqQ+m8Pn0gZw3d5IetTFKnf217gPJB33az1F43gi/U4yLCV90q5MMWq5petQ0+fX5bfye561Xuy3aIj53UrMBfC/aeTurZjwxxiuWd+gXEFZDi6Qwd5/vL1VEweIvGvg6cWubKw6L/QJO/Nh2kJXBut1sgmiSVY66T2hJaJLCTSpAH/0GyMvpTNZZVwNfwGSbp0WP0xSgpZVYkqDtHnR0TVTj2B/WJ4uNWDxLBkqFeN+zFLPeCMDJvU1RpCc8h0ApxgnIZAtFVMClw97otH8BzgSYvmLmYeuLPwybO+PcRet1pOK89T4W432+moMM86fxlTi4FWP15nJDldUFjJx26Zs91NBHBT4ykyFLgHRrHQr+slCgw68dyC+hssq/A0cHPktRHssg3eoQFFGo6ihBXs2E09bQQkEQbRc6NVPubDQhsGbuWzm5yfw39OuHcj/z3zeJ4s39fTCrgOnDRHdTt5A4fVZmV/ctwdVxvONWUQ5I0v/SG6E940xTDmNLnWxKbDbIwnC7XodPtbUTohEicyM5fplLzokw9nvkbcReuxwnc8cM5JhLKWRjA2pt6U4Fh2gNoQeTZnxjonxrZipzxvXvIqKguEPU2kAm/IICU0SPiBsPGsJIy1LTiWPpXSfiQveTp8hsZz83RzgmPZb6UclW6+XX1PfUih9CcdR7QHp/Jnb2ysIrP0i8hdEWjSx2I7pdD51BsFzgBx+v3I4mXOzzz77OS8nmpygCQbbxaPzIyEGQdc7s4c7H0bppsCq+ZNLJizJ9siwn98rPDSMnPr0fiIdUd746zfPOiNy0A4VKQoEh19P2oPvZZ9Zz30zoCQ4OFnSqeYmj6fARq98oOVM85ryOIF6MB+XyYOFrCgZM967H9V5r8beaZFfnuEZrWl3lvMVq+fulSJUtTrZLPC8EvrPH4pK8QoQM/pWSDUcHUltdgzstOZqOrdJa9W2vUxba53igq7BUBl0RUdWE3Z9+LAwNpbAplcW3YarR+0dvrKdO3HObihk34xgyowwUfqFnND32DFP1jBPpFZ+7BwO25PJYrhGmexAiNt280Fz6RK8bFE2JGzdr8iw1ki3cBXFPHyt6nvPLQdcU/DDGhcfA1vBdATiMPXSERAN3pZ5BRN1mDrR2bj+A2U7iEKfuU5FX8QrgZbRzwDX0sb1+oedQHHNm/D94S5b9iAetUKDnLFX1+rXixGYGqeLR83ybN0+2vHZg+nkCe4MP6gbKkmF0Fc+MYri9oNPxx/nn72iPf4nBg4k0+VETXIMexPYNSUCINawQNKqPnXf+NtG5/BRCm4AfLDEQrL86924bseIkfM4W91HxmhcLQJ722DlBF+wdLbc2mvxfiWY0gjKfqO6kaTajEcqo1IpQQkFJfVXtx8dynCu9MrXgmUzjEdtTW0c8RFrnI/+s12MQCS3taLRbEQt/oqz9+O+/ovkuxxO/nKb94VW8hoYKzJDQGc/nMvPOJTQ05x4wzead6v9znJTTjzhU9zkTz+1KVom+FKXe9x5zfI7sJJxkBuq0rWdYruIqKZ0QBT74HCRja/fkYR2b7K8j21hsius3w0UpuIhr1fCai/Dzuq1hXzBN17BpEV+qatIKRubv2RwKoYkHHEx+YZHUYC164QPHMaGKmeR4/0Tuecoc+kv/0RFm8iJQiDnT90051iC2iKHUOEEMGJufXhovZnyu5cKE0ulyk4LmT/riZkEZj42h2XYeGC4qhRXQsf2R/OMQNytJ6AMocrNL4hTEzarQD/s4OXRmy6RL4MDv7v/B9PMGmYRty82+fEYZMKvjXTrFq7n8XxoAB2/BGKoT6k1aZ2XnOXVMdAYqRSwLh1/l8f/ZEfJvvNEBhXYE8OJ3QEBZiUgi/vtdTaZocaDICzrG+hWAxcJfCW3lbQqKyhF6nSqKkKZz/oBa8swP/uLm9ZwDoheNBViUL80S50syvA8ZcsHBYysSnHIzJUH71v0P94fHrmXfCFGD7NVuRUk/T0rpoTBzLpZtCYPxVOvhmRG0umcqDwsxAKO22mkWqGrTnCrhGFNgUDg4THA0WuHHCsDrVdkdQ/mjkyXIfG7TqMF9geD7EZkXlZMU7gywKqv89tnqx8p5m8PHksgyI65ePhmlN6/HFWKdrz0+ldgyBtIX5Wg8rl5qffGepC6jP61rItVriUsA+M7hbZo6+jmjNOGYAL2McpDC1sukcTThOTwcZWnGlU5VtRBtCrfCHZQ1OtqHW7nyVXzntFAOeyDCZr4vabHfGM4akBDOqgkxaxmUexS4x+48osy+BU1Z/C6ugHY9l6LIetT7UAc0czLggYL2M6fRYytxOBrLn2Op1feXQltPHxpIz1wdj11guEdTND8bMQmXVP64LHMAnPuLnmwh/FU08ytkJt2R6rYorH6GewdM00MrbgZqxVZG7OaMtgaWLPNg34QtQLvv8yzlipWiRQjYgDOI/x+pcdnnl7drwOy+yz8umC18Yb2/ihnTJlTJ9MRI4/nD2ZdvkQPFhRAsCTICydJQJEJZrgDN3GaSDGgJxtYPbRnPni5CHAd+KtGYzeiRF/uz7ZE3b9qUpoP2XVU9kQgEZg9LGCZQBd3mnBWFeJkJqYyheCWGmua06Sl95TPn+dbtaWmMhPj94GooSnKmGj7W79qoYrWaj3HV76n89OJk8Zb0TSbTM00mlqQp3u6qaI2HP8mesch0v/KiycV1mstpJkqmFHD4tBKF54HlLr0F6oCJ44bcJQb0pbGGoIPx+z0UcqRfqVDfAT5Umo5GF8qLqePAyu4e92A0nEMfWeE6cneHC/g3OkF3h9fkn4MX+d1R9+klvEb+CF2GT/khtCcMMj68rVaswmkSrPlqM1Aho9x9SF0KtrusInM9hkCPJhibIgmWahw3+LO95sMgtDSHUt8Mq/vZJeU9GSkvU19N9sxtJuzSzP6SPvHG1V1fe5YBMdFTcZQtyNY1bTTF475sWIpkGtfbP52ulA7IzBciYiaA9bqTkrK1LpV97yqEkgsMY0skXjh/Z411BqB5jGxMunnMCnJWfZ2YlPhrx1otgWCtEQVoT4GmiEhSuZiN8cMr4GWfjNG7jrE9RJkN1QgFUpvoE2gdBn4FoVdlGfBY2Qx1eUyizcNSpfynP7OnMQ2hsvb6IS18nOOiia3lBbnINIXam3aIOo+7wEUp2BqnTYjDOgcGS96xxTXlu31gfzIP2lAUOMMFo5MJCM60k0f3HPjTBfTnvQmARlpr1E+9zf9/TI04KQAZDg79xV4CpvbFC0sqyQdPMeeQYtvmSxX4RT7XVuzC3KPrGTm6u4vNvqcHQ8tj4nlFLDttqUg65BpFM9xImTVr7VzwxBSvN0n13MhSfl3S/jQBWy1XqP0Uutid7e/nYATTxnUSFuryWfmy5vEWD6cnKzCUHVDJGeASVLHcJigs+hsuwflefcBZrMdEZT1BZf3bZk0XBVeW1v1XxUBCvG53dfUMFQh4vHqjVffxrHVS1ddUd8Y0ZDCMVFuf2WI7s6+syDHPFodAqM8BCJ+iM0eX7wPJyvb4grQUffeZqiNF+666CGixaGFoAFSD5I58Jz8a63E3cECTKKponBp2PWIq3tqmGkUfWAlQBwXvIT53ZJ+DkFsMF4S86keFxiqCR4MsRJ+FO4mAVjo/aUiYKxb/x99wekaEbfudD25MHE6TamgqSnl54Vp0UVAbq2uCa3/N9FDHYxn9gAkdrsuT67TvA+KbxC2RY/J7dlznm8YumoC5kp0BrsRmXNeVG4xpQEi7AzaJxx3z+OqxruvClm9eWlQ/kmc+vVm7POaqI7K4jKqi1Vmewgc+O7rt3wWQLbWHOwnhf7WIkmvmChsgjek+j4q9eZzOlupqzmwgDtEF2fGLtRl0rGM8woltcPQR69qSNgoRlgNC/vtFT3onS/DJnjXlBZEfsVF5zxM5HyinuIMDE+jM7tpD5phv2KNZ6L5+wKh0FsU+WIlrHTiYAOEHqVvxRNJ5r3efzV6gfNKpLIkZPLXHycfu77uWGLCZav6R+Ze1hHOKt5/QfjZn1XCtZRLfEH7MD9YzMOIE02YAmWjEH5sLPVfdFfxBcjFrP9Ob3Mz6f9WLhP2A4Z70buGX7q8W8KYAcDH7LDZHjR/CQ7M79N8VPkIDJDJ9F1UFs2PkTN6S0kI2IvxvAmoTpp1zGgwOf1kWznn+oP24+r1mh5S6k2i9hznxFPCJFsqI4VvLaXT5XkYh4eAv6mX/FPUb8wRGHMTiCwJdS/YbtMPMu/WQK5OLB++aMFp1ocVZCikRLoCySqUlHu2PgQjlxdzXjSbGOmvOdL5bjh/An1QQ7cM/CYpyt39MIDbmxcDf8P13YWHNcOFbbQawu0MwMh5fG+uWXKjnC9HszzGCoFX1bw+D2JmdF9jJe9BuJSEPdC6ZamcTOZhaoJmP3qXGcSCGmh3L/KiRofR+8vSZEX8aTNsqe/DjQ+AKDtD0lbhGHXyf4f23ox7CAGM5+XD3vmhkdGkLYmNr3ecabVgQ7DMbtGE6aSHTEp9rHIHHTnJd+0dAVqNtyaMWLFQr8SJAmjbaFj0gZaIfHZquW3fLKB5aeAuwPIc7yYPWnKuDOPiz9E2rz+fs81FVZf9NAsbabP+ISnW4lIVik1aQVcragGzSMCju9UoXbFCQfJOkwpliKoYwGwqP82RacN851cHnme4SKox04QgeApbIfueS6YBXv80J3h5QcSSwOD4WNb8M0t/ZywTOZrpVNnpc5le0Fv6nyWkj7bCWd02llv4RnHw7KrHHQkqvdhb5znDrXGpLzR2mEpfnBLnmLgT0PWPGy4jM3oe+DjWVVY/tlP8ZYdDN2Ayi4tpwBpx+Akmy/mUhG/gJT/eaQ4nXJQaRCYd/dwy/KEeIF69vTjpVfuk9RFcvgaFDKUGa0IaL2mtXXGPvRKIixxvEo6u7cfZ5s3XZRa0jL15skRK3JzSTM2M6lV2fKG+ZjGyq6lKHXC35llLoM3186896f0UXAGNiWUD02b2aHevSUohK+II16vb3r+9TGwL6kE8vuBEr29yv/wOYV3x0K2mo6I66K762E5darhk8fGvOZAuvs6tTgrJq79qRskwtQtHTlP5v7haaCoCg7Fj0gyGAid5tAhqXcGKVoJag3yhqprt6p5PjaFvqh91yiCPLwY4QgnqispcVQ13ai+BeOQ6EwZzO1+1/8h0cons6rQm0764s6I41MmQ16768MvJdHRPhK2z4kqndJu9Pq6Qi82LCRZ/AiFLILBQt/aovo06bWpFtxhDvXasQRpjgQRj7PfmnZeQ0mOa3XTuKrlO+coVrfTdYopYMj4UaUs6mKxg4+wuiW7hQM2+3jR+/vOds3xyVE15A5jT0u07mCYd0lZJoqvxGp8AxsOeXc0z1gvsWdXwVW/G0xgJjH15ZBIncLlKHYgdFNo9bPnIUWRmYTvku/h68JLuh0rDz1SGTQcBGrAlIdmJ7zyxsxGnMmMcwlx1lbxRxBAuh9ovXHyLiOydfKSXRJ7anKBMAYKss08Xs/raOytUt2iuT7HC2oTnaniAKCmXHbZw/9Ij4YvXbE7IRQIH3E54tjkrs4QANCNuElJTe/xdryFeBCq2N4p7lrkR3CiqdKP3rkg6iQzxCXjaORn/HzWpO99y+qOd0H8/ATcVQ/QDS0kzjx7g/o8N8vfzjfaiMh4AGQjbsRoim1N366FdhR+PvthNzX3lFzxk+5DTIqIQvVzoJ6lhjtAQZjMXlUI6G30TbeKX5VnKHlxqGlQKHedlcMqxLBW93KvtKY+9q1YLpbkKr5OJKlj+8iqwpBdsc5wXSY8DTfGm54hW1nlGu6c7cUZ4J4yK37vK3xv2Jgd1+ooWhI6Tc8Try4lQhkQPXwcj5JQ7aTAQZd5V2RKVyi0rJ0vO4mu5wewu7Kvt9NsVNyhARC6unyc61hIJWuMIUoLi6aNtUTUpU5MtfBgrZ8u4WgXOkhm31VBh6NQ/7iX2tOo9f/n+xN8YzdFhzD+LGMe/HEtglEHs0y6p92DutwfK5nx2x0H3QXi18jdgqoVe9DK889M+5dm4jvkA8rPddUfoHkDSxvKRw+N0E4hA49PIWSLAjNJg86/OdWclgO1UNcjaIA3vbGeomXQisO7qyurHAkLj7A1ovB4OFFsa4xwRKZwQv+0jo6COY3iP1ykLfSCjlUnXTQ3xH6/YYWR9AoQcD3x6n96Id+ZNQc+snmHz757iVoP2Rvgk3tJinh+KNJyhEcp2JlCpMsZgI2bHiEeadFztsqTtpEF9msBGnkbiLZvN90GIVwFs6BfE903EDWF4drN8+HIfX8aNXNkeDTlXbJfNAsx7AICj+2CpbNlf+rlZAbKVDPS4LhMiIHlOImuXqDQLzpIKoglgvm3vA4prHZr1/I2yMfFKQ3EDKxkIFBRaC0gwOrJ6V2LwbnWI7vKq9xDrwpizgDH43tRn6JprKh7W9S6UYxQBSEUNi3UOliRy3PAqzheAmWzQQi/IH6RJ2UdVm0dUnFeV5Ga4rRNaN2BBVJPB9eg6yoT4DHZH3mcei8vki4EEaYcvU7j3Jddp5yWzbHCwevZieDIcwzS9yDeFxv4OVGCaZ/YQ6wje6cyduS0GMbyUkDr3WaTfWN/1ZRIYa9A90DOvotm/Oojfap7Bz723duiK5HnyYl5OHTfAHwRPPfEMdY53ID2P9q/q5sNbZ48WCytW9BmiOQaZzvVS3Vq4igupDGNe/SSsShP+KQdjZkRgmaf3+lAJI3mK4bbFsOHhMTGlbm+dxhkE9awD4HegPia3VK+3QMm6IYDsTeOE2z+vix1/NkHoQUNkvU+Rmio1V+69LmtWXppOb+mO30CbH9Ib6aBhT+8wWu6PmractXt4EooGH4wHZIrDrJou8/iNnFsKZ1oJx4ZG+etUsPzOxnJkGtRAiLBKnKs7R1B1qr5OcNgVk+uwqjB4tB8WeA7CA6N9dsk+VV9MXQbn0h4dzyYliQZTP92xGal4YhMTIje0x6xoMtOvQPz1ZNO4Y7/Iz7Mgh14+IjKD48M1Akf8L4rGGE0Xrbo8pLcLIHD3RTSynRBssM3+yjz/XE9Up4A+2MqPc5PcmhEKdiXSN5hW4EqsleW/OqrFBQPwCd0JEj170+aUIL41i4eBjWiNnoQh30ZkTVRv+GFMEbOSLiiiQK2axASYajVoZ5MapKcDidcRpv/tKPw0itDjZfiZ6kLHzxbc2iQjPc4eH2LmlS/hpJbh4MjG7W36a3N6vcAkK+AYeEF18UlSesxPtz/6hsf+bvs9eG0KQFEp0j2/32jGnP398gAAK6AWMoCm9j/VETtx/0mxYMfnJfqvxXxcpLqdDtiVzdM5lVLP+FP2kv6EN2Dsk2Qvm0Uf5k/ngKe3J//WhoOLu/kFLq2GqvLSYVL1EGEdSKQSY+3mynPKDSAsgovJG7eGlDLQ3MJHAjUnXc9hcsCGDRCyHg0YGZkVZbEi9/cc8xh1Af6GfBJfM34dDoFzOucatk94J91BhjVesLRzyrR6oEL0Qr5n+vYRLHao5b085L+ifSFDYJI0ikArLe9PUFj+vn5ena2vSDtj7pNI3BkJCNNQT59YLmT46mLAg4/O10DH5TBPABUFvjGDkv2R/zkm2XKLnEGvfJfb+fJotZEbXm4IshEzRoprosiQwXP0g38dLo1NATKDbLbvjBKrcruiEsoJifrdisVTp5w8U2ZOqPacHuUrGeV8/SnHBskcMosuCL3plAfGB/RQieNMjs6pRsjxyuncXjxLhKfx2BcziKGsVc/H4DTdms+oiCMxzSkcjSvSaGFvfh6PMhK1q/9LEqtbIIGa+GE2eBvik3Acm01/EvFrYgwVnmE/+eKnDIqZZwdbG44q/MvY+H7+3y8/l1sOLMNWQdZBq6m3zoKIOz82u51bjOhyR1+IbrPwNzTUOOP7PtoqIXVsiK/je/8ccnrN7MfU+mS73bxbP6gloFkUvFjv3IHx5YlQPX1mgRxvh3MDkd1QLOu3cvIHVfu5oL31QyhZx33CW6zUx0UPsdAvuDAZdf3rzy0DLqfbZC9NsXQVUgqVm2CR2aMnxF6dKn7tjwVO6O8YScdQDdXumpFAna0BFvnlrsgS1HeHWwMGpMEB1kw1DFoliQadRPf4Wwb+cOFL2a6+85jEOwxMharjF6EUPixXNA0anYuSi9PQ2ZAHxcRRl9LDj6+9tpCDYJ2uSKCuzZpwRfE9OliYZ/JlYtfSljQVAOmphNhKi4KC/KeixwA5O4g9kscY2noQAyJDmKtPc5nI/NIOGoBRNX70DxZlCjpSpviTR2Fsyp6cHZQ8rrbkMEjVK+tc1tmJpafm6mVSlVlI467wk3WnqNd5ZaKQVeqy93MA5aKxTLG/hrDxKqsfPJm70CF8jpPcSxwXAbDAIa66joZItjJxQIgue1a6h1rpgvyt5Fs/4pVVqvftyeb7uJFyR88LYUEFofTdon7UoTEjJp47YbQjObLYsl/tvxmZPhle5Mcb6Q8b67GXkokh3v+fPCYp2rDhJsKcDNn23qk61UT+2/NsoVH1+ecBz7juKKgkOyI4yqe1YQb92o1qA+yuhcBQXoKE+iRmtAqd0RJ5pnLKozuBgWlERiHBD05Rt3tcFdD2LXIPIQcIFF4mmsbEUIRvuC0FfXHwR2/YjSbwF52X4sAlgH88Glyvxeov0r2Plgtzxexm4XbrL4m6VEdiCXTi106tHIKpE/9G52ZL372Dr+AmiQTRC0PMVoDdfQbkdLhRXm592pG/bEtHZrRY0fTGUcUf3xb0pfGeXFjCSLBdm1Ug/5pXY2rY2Ecdpk0DlNfSFGQzPnw3rB0QVcawQLuAJINHVavyeMCV8Yp5L6RDfCIk0CZd7fJuAMu8O9FWkxXIQyuMin7pfo8WyHdx/J2ksxpOeNyFFWefG25/upyN1fLR80zNsyXyyR+rCUhY0LmL1lMQrRenKMKW+xvVi3B/Tj1YuBr/FZWXi60S1ZKAjD+5tEdM/fePvlTjUleqUfdmlpASlqP3Irs8+BDAblhFSmADPhLMZqfyuKHZ/8ruFCVxTVEzz9bVwbvaRVMu2etYvTL+xqs/hyrdgf22P0awDBQE2FBCLHdv7+Ns2GTOe3qy5VuUPniJdnZHhPPTc12qeXnm+iZ6IfaKLTZnnZfl0V9Hq8eLKS0DupGYV3jgq0v4kGnZ6xu6VwcrnqYe/UQg8N2FD4wkE1Kv963jdr1IlQaKaWdyFASC3eQMpoCCNQ04XmjBWG1uwmKm3wmiKF3eqzvZ837rdRntoDBf9Di/rzs54v9jibzb28BMBz4k1C09fsKG1otUUR0ur1yzrtz+doVHeIcxs6oLZzjY8HODs8KY+xvu5FE5gGhNGSUmjISGFgzglNz9X0ZlJK6jvDc+nHoVAv5UhvC1ZdGzMQlQqp2FDQ0zQKuVZ1az9L4PfslY4d59Ubvnhj5GcfG8No9Xqq/zgWe8ZSD4HbDFis1F8C+KVFAix8ApdeiNqcuXlZsQMp2gSdy93d6JJPpHUycLMyFSS9p/bXxjeC9OXqPA3uzVMhe+d0XnLP/DG/qH1i8U1+gVwMx1Wbz2a6CMUq7PlKmW7yomcw3INpz52dyPsHbmrz8P5Q0uk3apN98ge/jw/epc3o/jdeAKMClVrD/BLZhY3MsdreSD5uE0ytF2mfOntRwXZKltJqgfTGUH69ajtjghs0/p8PnnCP4FGnhV2bPSIrOfMtBUpxXbkXodvNP4+u1vfpY9BaIIazyZATXOym1jdMhGSlQGu0++YHdBl9BP6GbrPGHz6BQWMs+wB2qD9mHEbxLAWS0oupNHx+FdHdzxOzFt1BQiq41WHMXWPmRHUchyu0CuouexwddnpHRzCEO6qkksqHYfa5pL8r8eblOBLjjvPJRU0bhy+mY9U2nFDk40VDVqsSAHxu32g+k/ieMR7n2qvUtAPSNMXZngW30s6Osuckhf/K8gbUQYRN4L5agDHDsj14/dN4vVPUz0ugl9e+ZiWjLBA78CkSXwxN8mGuDwRshJ/AXu2BhVNJ/jya4DLM1FaVf1zph7uoE6h+2jl7YDn0++OgjlEABfP6ZS/Fymzp0UmEyADEcMcNsE7sKfe16CE1obt5dsThE1XB18AVTFykeXoO/ooLy+nxN1FNSKulbwAJsYETcC3pvZDBxBgm09xOL3sJde1F72M68G6XDeJvg7khVN1UgoZI/uCa07vsPg0IN1fVsX/NXWqsuusWmXNYIj40fWBfi1/l7Ogv1DXSNycrlTkfXW5UkmxnZzHiF5IVa1n5tMMMe4qPD/bmN7D78veomvt60EiLJ17zfOkauj+M1lqf2DnExDVrYvB9U8Lxg1b9eUHXC/hZfvsbG8tPMqXxRCBy2x4o/Ue/GGcUJ9tXmD/2e6qT/D1HAbO6hp2BS8TicqxS0zDhyxjT3UKrSoPU9nfMC4+TFr/W0TRw/lwaznzUZyYol2B+JD+c26HcNsFh5qlAZMHYC8g9ozvrDDHYJXwSA5WuCjXBsuvN3+yCDUuapVdgSphjvrRrgPdF+yqdkyCkSrx4eRAYVoOIBJXxSl+aj+fxh41OGwC34N0y5pp13/rnxrKv1HUQ2ntv550YhOezpJSNlT+m8fLz05iHRIO4hOdHBKWFHa0wlsnuznPpsvn2JgPUOFLu3x1QU+CmM+uDyrtXFiT6nfYR9gQLdYbBCY4RnbMzrj6DZcibclYPnYjMeadO2Qsgy92U8BRv2ruF342DhwNUJPqUjqgsObS/cQR5EIhjwhDAlu5afjI9xV3VAd9BC2qq03n9TT/DxqPX5ymEO6o4Ky6o2n7TbLnYaRau5pHDiNXO/depxoX48lMWQ4eG1+PKNP1SGY2VoQZXZSDm0NYkabQD8JM6dohw2A2kEWACJ1TCbfYjZMviAD2l7aOJmIFAYtbMU5WVN/f7GYf7ileCxer8r40iGftjEeORfh/wByPlTaprhLoHi5Q7+QLA4ztnchTCweinS7x8L7TShxWSvorVmub1PL9CUnglZcVkK3EPbv803elnBYX9Hkv5dxFDBs5OzDaLdBc4Xd+MVa9/M1OZiaLX2GItrXZTL+RGJBjZDieRHPNoIYtXhQfMKnTP2ARntbeEGLA6F91wWNjYQVhX0pIeSFlsh4lynqCjBXOoPAMiqzUJ+ISx3jUg8Xa/YO+L/uMn1fmhDdzwkMtQwpqQ+Gvj54ZHKxsE9jBGhS2tRvUTR3nV6iRIeFSemgXSpequ2Z/kz7kAlIorENr5buSfD6/xEkCcUc045rlzWzqk5C8VJfnbXWOx7ZMeU5zoxQODfgK1ZYRcNW4mjuDyhGG0hgyJJ35Xmda1Y9Z8QwSuzw0cxykGuWmMIEC5Kd2kj6d097cwm/RB6XL7W576G4gRiCa/DoCtC++ofcihJm1SY8jiUx+QCO7E4aooWnCRtihUqPiENJLtBe2KZeeaaUrk20lRgb/Wz6H/JAHpLUfqtx4xN1BWZTH1mQSwPIGeclKb4THxQnLtW+mR+Jwmy4FIEErSSWFbCvNN59yYGm1rHir8O0gQ5hGdbeDXel5mE8AJYpEdTNkpju1eL478VjXLAQoBKXN+wEt90QYhd1cwZONv05Bbo0UaJEJ9VQEzsT8KwmbdzftiSL5z3TDM5SDKuI/Fmt0FwyeEu8o0Swl7ekJiHjsku3Fc8iB83VztE/ZAyKjO8SA49ha16tfbvy0aIbz4d1+uEvQ6o718rnkjbnUyW/aAbTss7hI/XfeI74hjsULG4ET4RYf9lU80QWrC9QMckiisMdTX3EpZ728yHnymY3+NiGcckO9AC2ZcREIN07fJza03NcZROEhFX8TCM/svIVXlmZDPLrVuifhXO7Csm01wFzBJxOW6z8o31a0vqgAob0rYtIsXATYDGZejOP9OsQuATWc4glKjjMFopgfF6hjuCsNQDaMomaA3i8BZzHcKixKLCA/u2aySWuZRfibXVq2QpSbNWFkcATFAocf6MginfsM4oLOfNMC1ULPqsTLfoIDZIncD7pLFk8DeHYT6lWn8DYHJhJiYkanJqV8c4eKk77OpZcxeXO5/GtyH2r+RQ+OxAhaqebJF7Pq+PrfpyMHNupfAlrziS2O1rxvH2Bhik1/id9L8M12Gn/DDHITXvOBZRunuugsNstbThcHIEj3JyQaG7gDhIFUlqTU9hR7g6wYiSxoGkz0EKrEJM0nFEhbKRWfs48kuTsAawul+HrYWnqy23RwrfRXvpb1gNYMzy2lwsB77ucVhDg1A1F+H2D9s81GwCYovIVq2w3L+9sB/hW1EvAtuPqA3oVTnK/vSGgZJQ/nityhATFK7qvomZt9iBCIm6IhnT/Y7dYG60NHFMLgW6ItymzDpMGCbpHDvq/etnDzAHbbtoAcorQPcbj22atzFJLoxs2cVE5uVRheob/V2YfNr0/FAf8SMxs3ckAlWHiJmUagCJ8wZg1gXOg8GMxlUR0eY99Nrq7EeRNx+OFJ4A/oibvtPuolCtlmdzwKT4dPzaaF/d570nDGOuo2cZbTIkcsYi666DszZ1zBwjilWu9R9QZezDhfOBjHqckwLBZv2zuQT1VjpC6Q3LVn+ukpccHKvrXlfbj9tdAbEvGp0boED0PfHIiiPuhx0NRF2E0vCJka0g/9u7E/xK6qHIw5ixdvgPWC7l76/H/Vv02lQWmBmv/mEO0HsPxmxmCXz+7F0kFwPeIZtrB7mTvk1dTkjWslg1UxV2tsnJLHOpx23uPRWcjoyO5AC0GHDcn0zcujAG7xDd/OAgnpS8l2wW/A/GCYyDx1Y6PtQ6MOrgvxlBOd9IEuiNt6Xv1cmRvgBYXwxGUGoRggVhgGO/xiB4Vk+uZv6yvcqMt2+xX5Wn96fuOpQ3+A0Mgm42k1P4sAr102suLtTP4GLsWDSKx1qNo81R+JYltdfQBm9uKNfEqHVxQXS+Alcj23d7eD3fShvQApLhpkAKXPRr0u9HEazM/GgT8+Q5wMJ4IFkZIt3lBCjp9/WKJzWBS3pYmT9GFWmWLuFRW8UuiCGGT5/Y5Dz6M4ewfCW4KdtomBRxjpyYXMr5XEuIxtfVqb+UMjaNyrPJqHIsMmBVIykprxKIJpqi8aakRRr56zLngjRfUAuINfKs4gSCMrY9scU+aoolu8bSmBssETjdYnnYcxHIbmFYcFFhusC/H0R8F/OhvWabxcN0NaXQD7yP9yJ7q+Pykn+YBsmE09Bh/CM2G1/H6nXYnk0FgF/0kSfqPzCOCPZkUmhyxMvwBsWfZwaLdUsR6qqq74jqfQs1HJInTvZWJZBWVzqBOpA7uEuIssT4UDaWoht27mTExa6hQDxc7GskHeZWlNV8DD7f6j/C8iYJxwtyRBDoScQI/PFUafw5wVkIcVKN9Xgt1/WqP/bk2tW8XV1HfUBvdOJMnYlor14/FAextOnzW3EOmApLwnBMcYHkH+npmP8+HP0+2cj0ir7UMKgMVgMrzi65RWYZgDTeQDBdR3BMnv/Onf6guYIt8B8CozGvMaRqDDqEHjYHDGbCX8JjDfyxbiL3UWvF2OVxSQLjSKWgxU7zhpa+74mN7Ti/HDUzpG3hUMJPMVI9MHITJAzwTP22LR4LTLES9eXFq2w2Xf39r9GGUE+uP62P3fgJJ4JDX83Db1vUxvzLVqjPwc8uobazPUypeacI2mbFypSj8+jv6Dya5vaxP92U3U49PyNczbdrCyrrPtg3WPCJkEXa/ERze7c/u25EF11zDXIoktrwh66oHVfAhpMn5o3s3NGrDOTzfkCGv++R7tcLN3QzXCgcGnUHrU8uS6JXokm6byxQV7P1nf1NNFXmny/ze+LdLHQxnpQ0GYv3mVS0xX4G9ksw2xPGtUSkXsDR8gDGUGTCnDwxGpK/y7K46mYuVJD55p1W3bN+du5dbFM3s8yY+1rRsWNLYJJfqZCeXL3RKCT7q7oospjPT9MsB6q2nX0BBcasNE4E+0qd+zI1n44hef9bciFMNyZ+aVTxjyaQygsFt4RDHGceGTynRyBuKfHWsLzkiMs6yv7Gy/Uv+QnytPwE5WmK74Xr2eeyTHf9P1uK0B5kbl/MkP63KDx/iwIo5vXCOoiCqq0DwweWW2up02hPRAbt3TYYtC46Zxb2EESBrr/vjuNXcfXh69IRsBSHDh64B9zeRR2gjLhVlxX0bel9ZCyh4DeRxs9Uaa546LAwJfVdxTF1QH5pNsQhfieq73ya6QRbKGQApwVZk5+T5paWeWHMAmzyIxhesXiQ7NzT6vemItkMN7ouHyLBJMWLTrvGMnGuXrWP/zvuXsk0MgU7Nb3PY9exiJ/dvI5yDA9q3II3OLmwDAbirSJW5BkZLzckYZJeafGI/vsFmXFlS/Zxz9LVO18DC/Q8q/iFZlpp8ZZbREdX/OLOnEEmjEB9w3VJLv9y3/eYNvWq1+4hi1HeBGLvhZjfzsXCbXt0iz/2h2+mDt/S7q0RMymdKJvSTdUIiYYgmFd/bURN01qaM9Au4h+wnLDKypO8hkjOXS1Fz8UKBwI+OIx5BTMh7wfw62oJ2Z87v4rfqQWwxgventTNnq6hRTLNEbKLkn+7YVvK6xCX7SJenpI6PvQ7L+f09nFoJA3XKmiHQD0zfj/tVaJoGYnCpzhkI0DdE623DS67BsHKMYqnkOyg/6IM37z6fuhaWwY6+BfoD9gtIf0PDYMyBJnGQc2wj3Cm/Vy8d8Bo2gOhEkprsr0GPe/y5lpNNHBa3wjcrrrecP7sOiIeytkbX5U/V7P8RHzQhObuD/iwW2A0pxBDnp9yZyBZ/0SurQ78/dUGSn8WIyjMJUJabqVLcSxBow0NipOCQY613O3U8k2WENyi6f8AMiE8f6U1CU9ADGvFPxiP6TPy3CiTmkPBIwRmT/bj8u+ZUqu8uhPK9wU9Yf//70T8hFJeO7+dnduG5KnNkLHC5UwIV5Fv5trkUT+i97oODBfbbI6MV8EtPi762JHQL/MWB22/aWLyJ5nHLRbel7wfqHgQtkin/Ikq76a70Hrc5vDUrwhlneu5DiIF5TWHPHSruK8yLJzlnYUvi7GsC/CqYmX0r6/aj4phiSgdaJw1CQWPevAcN98yD+RIPwXLaB7la9cymTWhqmnAjWaCpbZYuZwnNCe/80dzCvvJ6VL06JJrEADMNSLaaFqfYSXP3wAoizWyAXHonltwYF+om3UuHmtun6WCBu1i5NCzrsz3ePLF9K0l1MrAYo3OOQQfY19+FXu7YeVwLgUkRBoaLWWFP5YtmYYBwiafquso+EZ9K5hP2ZCXVwD/O9AgWbFHXImpavMTtm6HOuBkE9YTtu9pSMA8BzQGki9EumZWCAy+wyfFFlGf/Qqbz3iI3DFvaw+Oz2FK/7S5c+IZj+rS/WsrBbvdUHatGaiQCt+0gsR3CPFarpO5zjpkVZlU65z/x3132VMUv745Dmo64sTh+TwE0NIwWWjjDjxCteWUX3Dx3jXn9jbPjjtv78XAKZj/K4ZHvuJ8gOOw1qsi1an39IPSvuBqFYvnXKLnf7+bVO6jqcQO+VvMgWU5r5tyftGyQ5NgGGd3uKp53r67x6qxmZE4H7+TMZX+Rgr6IIwl3U9VpTf+8CKtERyqvD9klt4EqDJjf2KIrbD+YWKPo3/kGuEaV4BDkKpvmePAgVdgsEob9B5KZGYRoPSNIyJgYtSRdrurvBLFOv7nd9/VQhzAignj6Yz85NLbOwMrrpxSkM1uw0LqIm+mIPJhZL7JuCeEZGOJIseyubkkit4LQgqqi3CkzL2K/TaNk+fgyPGjSf4NKK5/XT5VnxLW8q+2PDpw4vaGKGI/ncX+7PsZwj+vlp6pXzjqTLfX3l6c0S37lxC2dTxZCJMifMMGvJg2ttJ6ZdwChZ/GjiVfdvLie7q8F8sn77A8faffC98VI9v9fE7TLgYlD+0AWh7/sZjC0U94kNWT1mnuNMPnhwUqYYP210+Q8Zx/II3NuTPKeg9AiDl+zfbH7NgfY7KvjHarXoe1UVhklYBI/iQguBKvM9UgrcA6+XErvHNIzuYW7aNfcq7x8lN4BVP8PyVYv/3d2+jqE02iOXd9ZftK2SUetsj9ZlGuotEwnQbjc5AbQskIgxCYJwIx9XPCmghYhPf9Vl9Iw3WUSsVoL1013rnoOTdFrfsaG+oZUDws1qA1cRhqQWfT1u6yE87uj6yu8frkw4TGZtFbiHJovdf4By9GURofJ9N5aV7o8H283eFvDYg90sBMlQ3zYLxC6pwicgoJwj/wKcEe1jdhgNRbiYRpZzvP+xX9U/po3gy+hUXu2tSLHmSh3VSiSSPuznTaiyN1vwLgDg05l9+hYcuEMqfYV91dYNZojyVUMyZyG2lEdW7LaDENGJTwB+wJAq4zDDnDbiKmzkjjvawmZJ38lMmUJRGC35FRiyCtkq2+wGTMEHTRSzefwiSl4n7TKPT/OokaErT8KrdAYifti+hFysOfi71gp/Mb22ID8JTaGk2+rckwq5cd3Y4tL6BxqNOd26Ykb+GJJpvTC6NFGovEo0TYti+d8XpH/uWwJ+nMZlrn+9z4MdJUd08j/8igxuJb2zIpS0yW1LyycZWWXzK93/Zfh5Aiyu2D6nxhyHkq5NrHdOFCxelKMXzM3RPjqd/1c4GH+mzVjEgshvtIxEFtHEzAPHicM0+sppU3+gWxa0wyz6rp26ol6jmXiX0/LqO45qBmBVzM1u+7lUyC5Ho3/27X6jj3lxD3077c2HYxBuSAfceilUHQZQDlfl4G2X8J3ervROmNm4+9t/SKWSsfwc69m3Dfib2+zGcfzHp0tI1JcLfggwok4eFi9FSFmWfS21p5e+aZWK3/pYB/1KA18Bl9SyKuAiCZ7MMMrNN0G2vAbCiBkHA/NhW6AwpuaUxgUaCth+oCmaJOitZAwLYDSHC11iDUipS0/lrKGThNm+l+w3FR/qdmgmydlV1F22OSjY8qS+20ufek4l3G2fYjc+Izr/ObIftb0sNrNoa5eFf/QlSiTVwGhVEcNm1rnYzMUwByLB7uO2VRh2/sD3q7FcSONls+evRa5ylw+0qT1wRI7Vk6QrYTY5KGwx71lw4f42SzJzvr/fs7eN43j8sI79uy360ZJd6HnE2jmiXZ5/wbg2q1Dug0Ga+QpCV0d/SCzklY0mVUk8rl0qzFzskj8H6ZEurn6U1e4b2pP4cTbBwYDzYQDaaSZelY562Ev2tonn8br7gbhWUN+XyAUIJafUmaod+hGFu/JFmvdpGGprrEhIw+qPNXpFBM6sTBlHx9L6PDda83H4fHTovCjzFXURp1cB+7YYG5nF8x7b5uV9ZLqTelHQ8ScH9K5sq39ocUtIHhXQQ9ZPPRLdb5ORVSWMxCyN3M9dftghYxXpWDe1SbXE7NwjZLopQWaVYs61Dkr5gApdJ79REZXL4AscgcjGgiV7ALZXi8cAXKdNWSWddNwXxeOjz14KHe8GJOL3gC5UjrMr9ebO52Q8BSHRlUu9fcrz78/kCp7gC2GZBEdgZtrz/fMkztUS4luxmUyWbASNpy03pM/W8vlGmfP0h/B+OhA5pI8zSdxcoo2DWxr0xDHzfzJ4kMM/JkiX+s34DF6gfO77J8PTOXtmEnqqRdIujdbWsDGln1LGRH+qhmjzNZpyc1HplKPnNIMjAShu9q5Ry+eErAPPajfr15YTi0sYQ9bOut+Dj390iTSqKNlTbt+E32VLG420Ubw7tUzMOS7yxLEviivNuXGUlzVheDKKTyR6uhU6vSePGhfWmc9CQSaXZSy9f21FdfwGRXc/8Rvg4YMH1AAzYZliIyQVO4Gym9fdh/In/Zpe0A8ZIygr/tqirMKsC4P93Ceuf+vlv9EUVBpic3Sf+bLvZbPg4fIGw+cR0LbKZClJCmOM13avIrOpbckGojhh/zdKuguiiI8U4dpQVUURr0TK391FIcm2fx4JMLVaDpM/KDacwJROkyDJwMBdbPIskuU04DKDUQJWyL8rV3MXfCRCvvLsMEMWV3bXZy2Eioa2qCggJgfM3dpif7krrzw2ubxYVoDkpTapuVY5IO5u6Vk/D5hPAyCf5W8IMQv29zSdf9kPsd1exMOeh3Oj34dAdq2I6CgXqjaf91zqWk3fz8BsUVg7LaZkYBRPr0j3DN1x7Bib+Gj9arShLjcpPz++3aoYQF5pq43g5KHwCGO6kbFpLdN36Oj/DcEQGe+xuE8HVlhlc+h55XqLmNr+CZO2m+sK4NsR4IK+dPe2E+kHP4xfj5RQWSPkg1fhcsDwRb7b+Oqtw5FxtaavVwlN1s0mAvWeFtRNenzfoXl6ej3sKxvfYqP3XuufjJ1/lbtqPW9SKW3wJM0nba+G/MWbbzbQoAoxX24s8n+RzIVqnGPGAMwF9zv8n7r2WXUdytNGnmcvpoBd5Se8leoq8OUHvPSmapz9MrV3V1V01JqJn/tEOrS26ZBok8AGJBM4FpsvdhP3r1iZHYH01+VvEnA01YD7pK+275iRoQoz6eFnPlMeucmf6oeQJ4JydBR5l2Ys99fw3PnotPNiRkT+GoQ5R7YnVs0rfHuxl2abNbNDiJg5rM98vrJk2cOHbII61og0N9G7TaH7z0lMfWUXmQjvTqpSMSIXFoGe1AOR3VVr9mcLdAsSk1xoWmVR+k/dqcJCHs5NO80GzQMY4FxLA28IXugFpl1EAtZSUNHShFMcB+rkh+nBuVyhgdUfFo14dhDPTHe2Cxu8y4CMXuhnjFu6dVLdpct9L1JMTLdID7BLeknHE+JYLsZ2Wrhla+7qkGmCEozZyaoj+cncfgT4515KLT/Yp8ZgfaJGdM1Iy0fyJlfCWnZbEa4f/Bka191O/Oc/gcxBKuemS5rOuuCaDtp7QwhSlAZg8nZl6oq/O/yYMabpBIPA8adnqu/EpKJZ8JCu4YB3/GxS3OUmydGKn44Y42gwCvAZpibbczO+KZPBSmpB/X6qIrOZ7GxWOv4iC4zzsGZoS5/cvLHdvkZnNF0S0H686oYgNdymbzDdQkpfN91o9vPEaEoKymW9K9RxMbcbnrRsavqHMDCN/9kA+CYoEyN3r4b59s/uxn2M/egPxkt+hjcCfz8Sb0XceS7PwTXvBjwnMKxKOAWkD1na4MEvdjXNy+HNVZDKfgjkCRsE9jPy71NxgBVHPZuz/5AcGSmv8eUEI6krvFnjnGUizeE0klst6uelcGo+DBUrELPerv1ei+flsr2xDgdaGbW9Y6NJnuX2x2a5qAkGpb179ZDn/GtEDWTQtGQSJJ9QZ+KXfUkimDRr2Z0VO1Viryov6PFfoit3Qtwy+Kuj7pc6osu7X+ddCjDS+5czwkPTPN2GOEM6xmMMuZX2y8t2FMz7pk+wo79YKdjYGy7K5uqfZj0FKKPden9xSkOeVTOcnSbXRoA8nLuoMK1S5mtalUjmi6ick+W3Sjbp+IX81dHNzPnM+gB6SFdkjdGO3CQ0zvY8IYA//oM+LJIGeFXu4Fc6pxMXBB3ncAiKP53uOWv3K0XjwYr75N6+v2qjp4Tc8qisAZHmJ0WvTkOwB7LWLo1tsmRPnY4uJEs33J+P6I/HECOhrbroFMrRjXxdWMk7rcWEbBP+ED58Cgqwkh9eHNlwaE69o1STk4PMPggc+jmLxN/U0sFRZvQNYQ95/iMe1GimgN3R+v9IE/xhaOT8pDhk6Mm2zLAIoYdhfQ9/rAj09QSxzRgpuXv2xV83+3POf/KzpliUj9Xkw6EEaooFwsBIvD7GdvffNSYkcpsCS/UfUNwZVHkZEJtkLIS4vL5G7vmbBYxk6U/kqLMjNDj4WdWCOS64QdXEY/BBHBSVjYx3egmEiMvfsnsCPqIAX1Iuc1/WZ0PcDzu8S8Bn/RnlCtIjY5vmmSLRKM5yLJaoJyah3GQsIpFN/JMcB1mlviHUlUS2h5zecUUJ3rE7SCRN+iKxYJwPgBlmmY9t/VdW2bdpFB2v4AHwkGx8LZaQ0TBix8TFTMaAoGX1J9fIB87BH77mXWtmqsb2S33DHIj0cRUOvj1fi81n64fOaPw/jgIZVGYZWpp5suaJ6Yo6nggN66l4Kl22mWLB74b9ohpRC3YKT7yLsrdo553vm9YHkrnl2QxA/nKlAyHKB+tw6FdBWOWzAPBj8AriP/mqw/8e/fxDAz4n/09+AAxXObwe/Kvl/+vtPXabT+fwhn/c4/nAxvOjTYKadsxm/d/Kt4DT2ZnYs+2/ojeqRf0OgMZqzfv039H4EgX9OfbJ5zY4/nEL5f0PZ7hCzocvW+dZ7oN+u/v7M+XMCQ9Gf471K1/LXTRjyc67MqqL89arHr+ei5ee4+L3sb76E7xsBRR9s1ra/VeD7G4Gq9OcZvRS5DrI3/31Dasq++tmw/x399bJP1G7Zz33/hhDt/VomrW6Sh5b1bH8uENM2gPP50K//vlTXffbWFiAYAbvjfrt4/yp+/f8tZBmj/n+7lP1XR4Fy+mHuItDq79PQf+stajbH2TzcnJuIuvE+0cfL+Ie3A6HxrcD/ZNP+HdTxX3nBv9pqjaON/9ct/j9s7j83lR2WFQTi/E+r9B+f/k6O384i/1BtZAUO6Cizl9Wa2WOUgNP7HIG3lmt3T08O/tUq+1vpW/P4dSxEXdUC3sAOXXVLdMiO+pssId3+7YFfb0Hv46itiv7+ndwMKZvBDVXbskM7zN9qoPn3c58HDKpKopb+9UBXpSko5X+CpSGPx98AbPojUwMJv/6JqWHY42/I4898DUH+t/ga9hd87Z/GqZgH4D/y3+0DBEpukJYlaxT/VgL0n/YNhSP/0DE4+ueOISjybw/0zx2D4tC/3jOjH/bEJ2/+vzlEq26j5HdE/Dv1557hojVqo/Mmon/uoruH+jQDxUH/NUWn0VJ+7wUHyzoPTeb/ainyZ/LMvp//NuH/minYXxH+PxP4Ooz/NF/+XqM/1ACCCOI/EO3/OU3917Pjn0cYof7++X852NifxTuXtWukRU22/CejDf/Xo/2PAwzGpZijtLq78A9dTD1S6PH48+CneEam2F+M7V8P5W8j3mb5+t+hmH8c6n7o/5rb/auj/OsqilF/IykYf1AwCmEo9iD+YdrDfzHtUZS6iQLD8V9PkH8mChj63yIKAv4TUfwm2UDPAon6M+h/l6J/7uQ/CNj/BiDA/kokA7ZjD9ucAFL8XbL+VOE/EK1LGY3g54+ETYct/n16/6us6h8YA/j8NRP7I+l52ZxGffRPZEz+BcH+I/WTEPj3H1L7/xqhwvDfYJSCH+hvhPr4J0JF/kSoCIH+DSEpnMCR73N/KcT/1yj1N/XoD5Sqb+1asUBgDcX/Cw6W5ylFEH+JrpAk+Y94zb8gsv7bGBD+36MT6G8w8tuIfynlH+jkr2DMg/zbzff+Tid/phKS/Bv0P6DF/rWYg/5rtPcb66i6qABD9P2fXsYb0P3iGdFvB3l1ABr6fZC0KM5aY1iqtRrAYMXDug7dfziK/wkO+mfgT/5WkfsgvYn6Zpc/h4hQj1nxbwjoDwp44WM0bdpNqFgFzQAfJ5tm5B/LBWaygfUux7CgaceFaLX4MXkwJnCHcsEBA/7YBR//8dgt+AEYQX6OgUWEP363jNwff0jH+6W/3x/+sqjQ3zfTP5f470nGgtqny9KnVheb7oBzL3BV+l7k6FN32OIPx4fO3ccmzX3L+eWb/VvpP4Zh/ib/EkolmtBOaktOvAxFqgptvI4BKYvCrl38prMkbXAUlohCHSEeJItKGyLtZtj5Z3uQvSy2jWGn76cL7c6b4UK/HG1pPEPvSTitVWfdWr98szcuTDWkgshEeI99DwpsBov9Y0uuEbuf/3m3M4LjNXxbpXYCvxK2Ps5QDAi3sURZKtdYxK9XrzRhDfWRZEEJN3w0NEXTE0f1E/8kXfLRnQZ/2eSuV+SpVzB4fk3QdktFAdN8/JJ/Si6GwpBASSMfvp910rV7KrafuPq9Xm0mtt1dtyGVrP1VkZ+417fAtz5B524BQq0a4mGRH3x0G9t/1beiC0P8niV8QWkThFrStzXGyNev5pLJX8/8xdV/eNb7T5/909U/Put23hUjB/xtTett0RvYem0fryNu/4SAjuvRvtv8iSUPinxq83x8TDlMlVma/HXdiZEQShHhDB2c+aELRol9oQ/vd999gBss9e2fV/v8hL2FBm+lNe4+lbnj53wDahP64Vu5wDuM73FQeJ13JgjoZ0DL/O9t+s6vNmyT/vkXbbK6o0yQ0oze5R9KDJfg/RxcscVk4fcnCzAav9fuD/f8/ub6H1rK3X31SdrnHvjP1rj7K0CEJfKVM3hbbfj3HvvH+wD1sNSXNi3RuwJUGRPp99EoBlJDv7RlW3zwpeP9k3EjFr+ZL3Xfz5MGWoKy5vBtFqGPg1l0KhdYFQPpcoWY5m8mUfDMz/eu8c+XY0ydo3edY4qbSe2uxBSJxOzB/S0Udm/umwaZ2ROVMweV3Xdws8SamMGZ++t7bPwq/afUb0ngu99PFT9F/ytfUDpLM7xM8wkjmjrDmzTLmzzHm8Av5GaA6ddGXOhfLvvHD1O4Er2bMm3KYDMoad7s7L7DdP90581FOYZ2RdpMwBoINdyMTgc81qQPcFVa9/uvQb+kj0kzUrApihsq4YoficrG7Rov7AvDaxsPyKkazg8S9+rQp8iMZD+WaZrE4ff0SN/DvMlz/DKDMHggTUK9wcrcmN93CEbpie/m+rTep0IXkCVDJRzCkzhoi0sxaewIeTr11pXnQ0GsyVfGmwI/0ylNsd+HypXNRkNsN5xJvPcM9kS/ydgB7kgM1E5LyEjAtWnZYf3KSwWVJzEaC0yYYmps54kKyRttksVLeO/enAk92D2KzxFxM6npBXFEDSKaXe75IqbZCFrEO6xr6dZKsFQfGR/24507df6aeVhvZRuTJuIjIlITWcoYwND5tCZPPYKHZntkzYWN7PgUZx7eoqblnmjA88zOJcKnLrLAGazdL3e8odnbuSbUmSP99IBLvBDJWYCa2ae2P1COHQIe54iN4Oy6OfNlAbd1ZrL8PFaDtCYfz/3VUtI3s2RY56P6tsBia9d7iBItGpxS/bOS0dUUx4PP0ygKAvtB71vXTPFsjGCvc/SeOrqNu48NpeWHeeBeM66cek4zNAtzWFnANbKuF4hBiwupjdQEqhFjj880e66t7dZuZKRp9ONxmHwm9EpTc7bZm3Y8ZEbh9ZrrZ+Yh1WKflzpLi6jNkSYsj2pdrjfp54IZ13kLZ6WHrnum5aawRQ/2KRYjNzCel+FjpqC14RoqNQFQ9o2sfqGE45vAP33e2ofXk+fDb4humRTz8rHwWePTZ2jsquNK3Rb1MyNHnCOsdJ53hNBelTO9IbFTNYgb3lHGnBMSqfsH6h3Foq742t/GtfIZcDJ3eIdQsBAK2tyjtOm7CgP3KxARgV8r5gkcoHYmZdb+QkDMaYZigGc2bFjTBJZ6e+kbO/QB18BRIyDAhgfCh9Fu29c1i9o497GWg6tLXZkSqXtlbqJnuMyNTcetbQP+CH2aCJYRTz9Y6cF2sMowK3Ja78X3KJ5QKmf/VutZum/hLX+Sm97E7lnIu2zqdS9bUhpgjJ9s8ULib76DEbef3dqqHbKOm/587UP1ekd4lXzDlwjDy57rVyq+Ds0Y1vgjxaVb6+X+TxyFZo0HauwIV71wPfOx4Ocszwr2ZOJlMHm/4Fn+w5s0gNm+PJlnGdO8OekvLmYGN2+SbyTGy3wZOALjBjJDW8LJ1zrrHibgkDduLG6uRjOMqZE58Jr/UO/oMILSdcRwYae7QWEjtksB5jVNxRS8juc9hS/rpnQFrfhXrikR2k7aPTYnmP2MM+U90OcY1MNhEMeJ8Z+Tt2127PQP3EeeT1SyctWbNUmbDYmUXOjFzsSgNwit7GCVfMa8/J3Da8B10Uw/cWsT2DljFtR8nL1b8ygL+6EhhnP4URSQVJrBQcBqYFRwIBiJ6vhCOAfXOBgevmvYYDHX0bSZE0+sUn1v3EV3h7XES8YgACzPD0clbRfGURWsc5mRppL7tWSNcxIZvT9c5+Y8RjU62J7MnbhmlExxPAfK46VbPO8c3C6UMzCTOxDbW8E01F5g6C07jt7P4lErk3GcjfvEsQxtymV+hLXIysAFT8As2lqTeYZf95T2i3l6JYq3pCj3dPMAOZYjg0veQRiFepUIyASspZtbKz5RE53ejde6Hdmt/qUJj4cWMnwIXEJP49PkjJCrkaNK2Bt0jRe9QhYmc6lD6ObzfMMKEreOtnb3rMlGhaXmu7lqFXBPv6DHyqJLay6BA0B7KeWDMcuPKSiws3xnZaOieaO8CULM3D0BicUZjOV69BaDwD3SXDbRKXHlWtJDQji+4XwyOB9OTg3tqyBo7nMj9Hz63Ij+neMv0gW7nOWHgLw5DHhR9fAWGB6/9VYlCfVCJWUUfnwjtyTY51GlLX+GeufM1lvaG0zAiSEAlxDaq4l1RiEK1Cg9qGqBrOOA1LWtphfXcmG9JBXKCqOk+XvxfI3rCBy2OimM1d3U5AZd1Wc4TbPjJSbuBMPKIyF1DHVnNIH8OKvVsr3U0hrz+VwfH2k/lRcSxYifEW5IVteaKrqIpEStulGBelJgWW+ilKKxPGtRsH3yGKLTWDE+2iaG9CyzRRXzaJ10YdGiXzttQiEJSN5cSZjpaUrfbdvA7fZEVJRRQEw7oSovlpt0mUjp2X8tsh5saxk6flp7T67hnBTZGOGanUyf3+sNFL5uU+/uu9nhICawKc1+5qDdbmSxKkGqA4HaaRvcWJ4Jr0mJvWj3+KKTt58UKc9gpgb1k8Nc8LjHTBHTOh577hvWqysl6POOuJ4k9NVfqkcBYWHXM581mCMnuAgbOnkle6fP7Bl84IZ2JfiTfbCtyquvYUNYLmP9hFRnI+IwfjdjhE5FQQ2Zv3ButtfrpAH9opT5UxUIFpLdKNEnzkh6WsRdpkERUo3Ka7C8kDJmrwu8Cc+Z5tnpXnDPFk8dIeCKxKmfI/cJ9WPDN5qpD6uGNy3WslcNo9KTStSK3Qhs9U7WyKF7wr3NbSwKukCWpzscQgFp/PMBZZ8wGSprYnnI1zoZOGrRepe/ZXz4VF//H0XnhjrDQwimqQ9vQAkVl1TxTbVmNiQTuZ5wsTiGj9/QQNSqOtcjcvt6Hp5R+wQMq95W9CEehqQKr53NnulzZJxMXRGtz2vRvKc4tr+XDfSImk5O+z6kOfxutBWe0Wx94ykSXl2Dsg8IerUaW2IY+3rEputHwEl/+YBXFo+DcCwQEEhQvpvbD/donqx94wnueURu4vHfDF0hPA1o8Q1NEwzwSxWnogKWYQECRMXTHGQBIHpj6fu7C7c28DuUvs/xjMXbssOL/KkwrqOWzEG7ghUUMqyUSSgewkVf5iDQUCY4xagx5N8l084zxi6RXaOvWGtLENbIeA5ygWOnr7DNk+rdT6Mko+6xIrS2iI2zQTkieuciPZs+JAh4He4v/sClTTfWCvuQuekiIoLZOOivIF5zDIXkLBVnzAmdgvhG9ZbJtLRDZrjk6iSdRCkm0mFGLAO+wwj2+jr/A1uoYGpeBSI5CrJc6w9ehuneoEo6QLQGRH51vUBYTMSfixtU9A0e2FSLO36c84LyVBCPuDRRJnf3uSFBY87hQMZ14lVyWZKXYpEW8tgwx8ceJKM1gEkuS/smpEFZThFxonenh5h9HaJZb88LhSxoRGM7uQBwBagbE9wgb/s2U8Dehsb0BLsYdWF+UTdEsWFG0a+9UaOSEN2ibQ++yb4NrPKB9fNRt78h+QrJe4MNv1FPCFgzPi3EFom3/kZ8gLuKaNRXMGnPcBVgwJWBNDbPVvWkFPYr4ESOd2qIvaDAvp8CxomSg+zRYR0jzd/GoxQvNbzwUxCKCOffBl5p9tC4sy4v+rOcBHUvLsAAgg8kQqlsk9cHEg4BrdB6Zw3hhejPUVF8XfVK/dIVv420kZtfz8xslseHDSYhjj1Zx0HGFPkQuskuKCj7pruTfSIUP41XwIqPtDafb6Q91iMGKnkNhkH+7DO6+Wu18TbqNktlrFGp2bWK8zwcme7nBJsdxDc1M310viJShYHGiFU1ZHen35zwLU443gJpkUZZiw5Zb0JRTR1xKAP/vPAe5hKPfXsG/+aHUOjMTpGhgMOLR0TvAc66ZMlqE3sDettmQjZW7P0cj50sa/nZJlbwbRTIE8nnMiA4X8XcW1Fi/WZ0gqJtmmkzp7Av1P0GaGTbLdsQNQckKy+5xKPjcTCniRSLfUn9DTt3EgPYPJd1Rz7rhxk0O2JEdnK85iWy8VwI5R5+iZYYM7gvSDHawM+LN+JbUovNIp1MdgttoTGO51gSnRDnLtZBm/zCa9JV0ySvasec92aXOLlh0L3YeekbP4m3+zIv+YLtHk6WtBh/y//C/4ZyuAFsd6zqEukXu6yWmw4BGZfWoYK9YaNhtxlkaCNWvfZSePA1R2RybV2MGNlYYZ8+ZjUaasiC5iBeVJ/tx70Er2u36w3xL3UCPGyzmfh5GhmPqLiYoFYnEslDLZ9tRdRQmOCaNEC3irPwdrMJsBA12+vzWC6IjwMiIU4HlT22flndimGDC+9I10snHuWnmtKNg67I1UlrNT7eIt6Y/d7L8ybsO6WlFm2WCoEKr+Eixxepma+HcvpffGXuK4v0C0Peylu72TpDzqyx5EYWKvDHvF9nWD6FDY/mWabYlHn1CaR6tKm7B1Sjpzk989a+rteRGreQr7zAqhf/QTDH0oYqowvT8qi1RllPr3JOZuOzXH8LebablzY92y6e3cc9yXzJSbpJsPtN1oc5sxRWfpp+zTcFzt7yD2sjtmszu+dlNaU0Rd3CErEvcRRxO785q/dJS/r9eLEdCnDDkHH+iwD+2mc5vuRj9ZROtOTGl2uC3V4YPlU4eyuXwhholsLtDkSNHRvXbbleT3vKNqSyOOczpi3DqDd0lS796akaN+DW6R0IJMIa1Yyx1j070bGhMX/DTR88GpksUCk5nu8y5u1HNav6y1AireLmOlx6mMxg9jLoohKvJoEs0ltNYqYLmGX3mFwWK6oG/aNkRiZ0PFxl5Yv/Jjyks2mhYJEJcYlk7GBCRfTGx2xKnh++d4s0nSxCfZRyfjeoGR/64Q11HceRral75dmhoLG79BUR3MPeMJegKS3bsl73wjEKo1ovrvf4sHUuWkp5Im/9RgjxXDaHuXbsMTq/0qeXdeP1NOJjdrOcKLOjR5kj6YLHEXZqLLwMyb7xyjy6UXKyt46mnjXnjMMLZ44d7iuPGENghJeEOFKU7dW21KHYcKh2p/BUWTJjDqFlKFdodwQVTbAJwao1pM5R9p32WZRvXIbEFPoozzewXvTL0gXJ9JS05RvS8Glmk6Y0kX8zL8BXTo0H/56jFFu31hdryw28q0qohDL1NSyHwk6rsMZeyfKlrQD4Cq1q2qvZMgporvKzfHDjjeXGGzx3YwuRLhKB/VnGAEoyDfAGb7q2bAaOXN86r3VjDZuu9U5kSI61hkQCPO8yW+VYUt43eSDckIIG1j6GdkiGAnNwct4GDSGKq5349nlay4WmqdUAzcPsCHQEW+QF3NNv0O+T5/rd7PhRrp4EyiT3wiAkJvA5IfzNNRY2jlpPfLrNq0bVEdnL8HEjaARx53pfl3pF5auXUle8WwgeFzC7s5+Zfjk5vLyRsi6YmI9bO/aD7g1WTEj/6hBH7W+oDNDiB3QU/czkm9NpHdkKXaiaj2nA0I7qUxcTn2N+qbpTC9uszGdSo7mxt5qEe3haffUoaaaiOamPa33M2XPNCMEF/Y24bfZyXGLuFUiddA/lUujy0NRLw2+ORj4ZjapHlvHtR8thJq8Jed9wk9yLLOPvgsA2aOa5AvWO+eqg23RcESv6ebIZC/IN+UDlGq6uXayTW30L53bsPd9vxy4JvbAk15tKutg6d1QMCI7zW2CXFnAtjCXnfdpLEiO4ypZ5Kz7Eqqng5TF0K31rnmaWFyL9frcB5NOI787JU0tvjXyYfCQ6NfSJmf2bGLqzaRNcQNoPOzaatY+Y8Xk5ez+iT2fJcMQqnEHvlv7Jl0HsnJ7+8K3ay7xkInxvWYJwgMiLSuHVWvmzq9GYm9yLf5uBh7ePuXNudQRXfeeZTFUEYamBfrfRAHyGTNHYFM21vKZuDsNxpeR485wKaqCwvAsEewOImeqdHK1sNCb3KdxsXncbfu7f53wuH0PtjterYJ/SE3m3SowsH9Wgrxc77tekTZn6zcv8GAZCfb+00ptX1GnmswGwTRms/RZe7ZSWyZcC7I9i34pOWYD+DbMSGNzOSWend3SrzduYP6Z0aT3tK5Fjd+Z1mmyu7BV2Xqhp8pMFeQOZau9De5yuhNQyYu8ZNSvCSI/AzhPmYWK+inya5MH3awr5rMcpquiroxjOz2+A9EcN3/jbgrvSghJYMK+1/wb/2Vf4QYJEjIyKOlu8Wc7aTqctrJ7DgXgsQk6d9a5+muZkddQ/xevTyMVPAnp8ysJFz0KC0kwUYouOvywZQIe5riaJ0KDlatv3mPbWlLZaPBVlfaMwsImaUQ+iUJuDaeZlbb01oC55Iya1evVheor6ngrOpfdxqhcBMSSPZeo+dQmpca0b6hmZWFhMTIGmsFavhkgeDTG1M4b3eKxFvYo06vhsieqbtQ1lYVe8PMGRjHbD6eylwA9VvaeunRTrN0omgzXxe7nFagtGbztYsGtu68ZQJ1XqaS8f/KVbPfb+UPgnxn1n7Ld7MjpjEAbPwyXwX1sVV6D60YCFsKP+yllGfKPBy1yK4xMGHv923r52+jWXs2XYjrtgQzOL5i9nqloGVmMXGg3W85q1VNg+3TDW7MlZo7EjaZ/YBBA1Ezy09ArObg3z0pdu6ZWksO1rnIb6L0e0h5fnchJPqxGZ3BNWLgvIISI/29B1FYolC1o0DCDisaMrAZF8AoVVK+zeqpT6S4SUOorxeiRfXx4AgCMneV70Tcm+4nP0SYOUN7SDqwiqLUUGFvLnTX4ap7JzfdhPxEEc2jRmrH+TBq0xHXK91CPYcu1+7gCR5KfKSc33NvneeIul2nTHJ3TeGsHV0V76gsdIStqHanXeSUnuJyY9/lTlhB1wZEuwuhsJXwWK181nzlaS1XHyLQvZpudDski12Ykue7xIVS1HDA2RaZjEVyaLlEvVitjWe1l6GsRGiNhL7HQm9hmWBEkZkn4psla6Pg+toh2lnU/M0a2DV7qObewkluUolxyr3ZBEedeNa9/sCXfFt/o+1aS63rqnBwrTJjdaIpQauDl/ShnykNlNmuQgJkAURNsa7sDPupfMLFhNotraXU+IeauX4Mdg8gGl3PlqkgzZ5T3VwIekCIdhPH7W024tH6ynmTJH7wnP7tBvcpn5kcm0wJd6qZuBK9cKc7emTgKx5BkXvmXxWYxidat3hX6ZiIuodqv4zRJrjMK6e0IqtAlZdRi8HCywa/KPlmvR2LVHh6KC1qbCizKAJXfgpzQSn1kLFuWu+PPaD07T6dWTMnF0M6o1iTcVHF1DInF48cFyTLLnq5M6TbMcc/xmX7C75iysXVvfduvq0rF96nmemGZu1MjxDc0FXbCx2jz1+MZLz0m6/UhVPUSS9QjCBreVXoBLz0pxDkWkEG0CpEoGHRd+ksjC0a8g0l7sxPtxxv6kJ4Jj+JHaRum1oGSrEOv+/ATZgMPu7HazniF8iRSSkRoStj2k7TrBpHs7lC8Pa9Jsp6vNTew+/If9jYVE7/HTt15oKuFg47VoyNBC9gfC5u96cBPgciIma/FUbAts8d3fIB3GR+zeb+ebEe/zxkmmML04Y19gHQrvYx1Y/bhbvHhsBzUB3acftkYv/7MtuZ5oIGDlTTNJS8D3eDAvoQ+N0tOnZp3zzK6aTFE+xZFBKW3mub5NDqVTAAWP+9LEcBaRfb21VdJFE1Cst5CDR1AnyklL5I0FN3pXh2qa/ORt2895Qpew7Pw3RHEGA3Q59B4z60XmHnreTCZG429MZ1rJ1IrT5cnvxUy7Ktm0WwhZAjqvBrRNwYZ1p352QwKPhC0h8LQVlLQ6eSStcC2kXbK6SIDjbybyRXPgxaGDIvkTaN+YtTVS430zFPzr5EiygX+2pGbbax6zXFIehrqwinw8TQ8HZEvT0BxZCbXDIOfNkTwKPenfRZfZ7aTvAWsrSK2skvq82AOrKz0vRQCl1xeIMYQWBxqhxtPwAeT7bn2tSeWqqRGWwBoeCI3ECWMCncTzFn0lnWOYGNjeQg8oqigZBvboaqRWIkHb1CW+tqkGhPOsNghKVXKBLgf1TpFAxUPb2o8vAK/HV24+dyqjx0sctmMRKs9VyFelJ1c9hctxyY+bsWXeGGBg+Yf+7hmNv/SX7h8P/sro4iDAQDo71Xog/DMteuPIZV3//JBrjWDEjVpexLAEsG7HjIViQurD7LG/zCbrx6d+YGohUlXOcXilstXJaTDxMW4I8HaibRCKWqaTBoEAo3oxnZc/BY9cIuTZlMHE3a2I4kZ+iXMBwo0L4+ulFkRTTxGRbRPiP07bf78j+BREjMsREH9FnXXyVqvGZBmDY3sw6JJBDUqzNdw8BuKhjceTUTI02FEErZejp6SWfyot4bnilhxXHD2KMl3FEeLi+PHMdlTN2nh0n4GaRX5xqQo2bV3ZaapaCHazItjiwUy9wf4nSHSTUzfbT56mlliNrcaPK8+cD4oemqHdcMX8NG06vCM7lR31iRFGgZA3aGiLt6ngfX/fmyLPM3GX5WyRAUUWoNEk4mJl6Ggs3uALdhe8QgWKR0W2NSkFjcB4v5qjVmK7aGlBambGmlbiJaov0qQHCd6ZrJKvUSUZTOITHbb4BkLRDUzeVjH0D49V+Mvr3jDscJ5yfRxOiZ3MXeEZJbDXc7W+ceq+4xUR9XP0PG8FqwuEkxOL2UbEMAbiexjKZZcOCpCIfcyTEkaAzEt7exvz3sFcDF7osHFloRzd9kFKR/IAowrdB1O9iHKQzQIDe9JeH8A4BlTGgjYCbrJn67FbSJ4005huNC2Q6bPMt2/WFu/BmkeamYhUYvenPbexmRQiuwlzEZosj16wPdkvYoZ6xNWsyMKaOgMrqd2VGJp7yNH9t8N2uTebbt35pi7ABH0uGnL4jqfAZtJR/PPJTag4NOYLOqSiVkGU18wRP8khHFSm1x9Y7fqJyPF3EzLEgBzLiMEJXbaKyx/LiwPsOtKTfVSzVzOmyttin4ZUE7lUl1EliqrmSOoK+8gnslX4oxe1Z2Hxdgl0ua7ce/JI5AJigdLDN22izRS+E/r9RIg4W3pmJK7mOpfmYscpjKYpk7TH8NjQMCmot1ODLKGMnz2+vBqwohKLHCVzKecpalvjybU+YAmj3Ddu/vthvB+Zcfh4mC9UvmIV/7JCtG1VX3qpxGcQVfekRmgAS/gX9vLIbiDPSYZRcaxK8uNmMk0MFFD2n5XvZfZbnZYrG3wSIlsNBqoGN2228CHpd3krFedHD/jp7oYNVk/FAcvtJntDYqT1d9h8CBaZDzl8cJejJvg0uK4yVyNLdMhjOR4FORiwsGSYoXKUFQvfaHDQZkLZdUuDYxff8lcpcnZ58FGM2jPXICpEuvl1exHiM328KN6c6qtnmoGcrm2uXcKGnspMG+n2Mo4Md/KJ2Z85IMbWFh62vm03Z7aLqLx8dmsE6H0juRhYz3Xg23DjfU28otPs0SUqv7kckuT90Jk5ZiUsrmu8RDBon6Ea7uP5+RD81XIUd2V4cgFyWzXGZ8U+saOh31OtqREgxuotuhceuiQ0J2jzentGQsp15kQ+FtuGDaSgYKz7fsMJgaP7KD1VPVT1pRNioV0a2DIMng3ukQqKIZ8WO3zt/dTzXQnn4XPIjs9gJLnEIy4+vl2K6VrdVPgMq9BJRO3kCT/iR8Jtx9pfs1oKQCdk0m8I2Tj/HIBt+xBxHq7wfKmR4W9tF3Z0er5kLOHQtH0E7Rw2bfbSp8EW4u0Ii4GWxl0hj1NrNXMH9qfCMCxG0JL4rWXM3qbWw7X0w4cKaBlFncqirm/b3mee7nM7tPWgpAsoEfQCojaE+bzfULZ7fvQexvwMw7cztnHxYUVfDZkTdDKR6x7kJWQgW1rBaVfqWITK88oJIDHyUkN08LuO8XNO7K8liHNbzYaTWBFDP3KcgOa5D2nVij527sp5nubxw0O0FTZVSNZ3g6qdLGUMS467uRuaUOdcv58tkllaBqMjDKx1/XEhNbJXXKdXXqqp+5ITtRjiFSTryHvfraKJ1+Ek1rxnCYtIkrO8kc3lL3wGZ2oxrhtkU+XdhUt0aQk1KtmrijoPpAgO1oodqluXK4XGJLVp3wyvVGSZaqW3x6/2lDNMBICZ5SUOa2jLYwowKeQ0zmGGa8QaySsnu7GXuru1ezykaYtgk6THBIHYnOddvPlqBpSYc2sXW9dNwkf4AqaOmkoXzh47kKhYeCuf9abwPHFFK4zH6pJaWgmg6W2YowzdJx+Jn91wApiXDA4o89Z2rVrjPBgWJ9z2lbUsStwaFxkH+MefU0aehz0YSLcwrFQUKZ/B+IAL1068rzs7i1Xz2wkqNDqnqbxIIJVoehD7EgvOWwRGdvlUvZHo5sIV4GhQfFo38kHhCbtf2seNJbZtJvx2CCP4urapnv0OHihqWtTIGuGijZesmhMR4H0eCbR286HZlMN3Z3eWT9ObkyHSDa2IpHpEa0v68TfzF0AyNshsKLTkgWNotb62BcTJyNObfmbAj5VblgF9M3QNkjZqNagDAVu/5PxK3ySr2zjlQmABuiMvmH14qX+YSyrTRgmY3EuRlWnWvLZ6tceSejUhHIKze+L7vJxNhkwXJyyDTt9iEMvvtq/xVXyRJ8isIRCLdx2SuQq2+NL2V2/381rYq+25iVJ5lgxpjPbByBeEXfYWZorNcOQDTInh8F9cmixDgLEvHPeBSWmnp7lWCfUzoaPqFwOwCNYVUgvdq9Fi6H1YLuI9u+Q+kYXTDJJSMPEIMIp3HO1JpHrrZOlHclMUSu/h+DAwpxNElBpVtIJIp6P+DQSTRs9p7vZ1lyuxZ6HGp1bblA3+i1UXbEaqbYIoNRjitFwHB26CoACaHjFpABMmrpOSuKiox+SngTCmjr1sTeZFO0begF/UoRk4bGwDXEhnDQxrZ/YRPQslXjaePwEKU5vXuZIAgz4mWXNvMZFaZ5zVE0Y9XKjPY/TsC++FFRjih3M7tOdDrUJ8Ejq5R5oBiTZRstYhLJ7U3N+Mk2ufIlThdQc7EuMsdcSJ5t6ED9IaPTuu96NDPLjLQesvTsOCUVY9Ykdv1T9GA2pi5jlC0/xAIDyzbcdbArmFHadhcRa4mrwL3T7cNuW6XtV0FelnsSjA6Kvn/OQtcT1gz6LBBMZj6rOEsPx8bZnU7cwjHUxgLK9uxUrF6llvWAkgvcR7Wm8P5mwrdNppv5Wx7wKgeq7ztl4knX2YcsCHqFN1E/IMhUfG9+cDfCfEdAnrG2DZ/eh8g5DFSSRQoah/orq1bfXV1zX6TMbQ1G6hTdb4TcYJUeOZC1QRXTwEz93e1uHxCgw0EaSJ7Vz317qhJM9YsDE5/BB5CHyeY+/QIXSDU+WpayZv2T/pW+pFIxkNuIsfGXePJr0+6evh2FK4NTFAZrbyKOuWGVqLGPAC7PMEVBrxjGd+K5sorkexEHRqA1eiJkIxO/fwNhj4LUYOQdr3/CqnZzuMsoA+M7hdPiCLtzDW6mUty2S5p/UZKQxzoyIXKdKG+c1f1xDRPmPbuPytOH9KdepOBLaEj9XdU0H6LA7Qx/X5ZEVPmapotqPRchJFEqBzzhSINLgr6AaiFduiF0aBkT+fz3G3FHG/c+IbLe/D+E85WYfgmsP+1qgPBevFlBzD4F2WVztgBP/S9EA+AgPEKGcMoKo+/DWBU6FTrcc9uvUkFp2XPec6schAEpROePdihAOV4A2qGI2uqLM3Lmq7V2lNT+BnJ2g+1PrJekXlJkeNsgohBw+oc08RfI/5YFrOeRrCvhfekDg5wnII/It3I/TV5s7nBTzYL36LobuQRqR4/NLEwshDn/5gGHXk6Jk/MoVxQsnsIxFHiG64uSgFoEbfnF466ke6Ph292DKMmbmLkzbtZ1DBilF8PMDayQfE50XIMAeatYN7S7Qlj2zs8EPHxuElUI8EBMlNQlx649HVNTM6YvnDUscbc81qeuVMEkTzrfrMySZSHfxcZaOOKYy0bq3oi7lhT1ObHR11Rb5ENBqXAwtQMww10p7A2yc9w9sXsHO0PwYlMsNhue+nmsperdRGWfb1NJqllpj6muTrJidpuAaJuJjUQYmdY8wbQkCS8ImC/sBxg/r8Q1tpkQz7Tj7daFLbpnPFpV9sQaGBQ1BeLRP1GiO7hwrqeuUTF5xvJUK02HUFw3jDrj828Ew9nEyT5t/7Xejr8yX0TnGDHQXU20orPSOhDPicBHhfv89vAMLNu+DeqIknooSCq0IHc9f99ZyBVxsAj9LiffRZadcUj4O0NYAo2gOfRHr8Q7K5NuFiM6rd3nhvlUduVp/FDOJx7kFsr27tvgFbg3Yit3F0pzCgqut0Pmr6fCC35Iu/qx3G17pJD/x3dVFmaeBj+9018OPhxNs8rQN/Jos5ilAsF5tj9Ke0ky73+8ri/aRDfhVZ4BwuWgj/AHKFJ78LigJNFZIPzFCWVZHsyp8Zvw9EXBWsKKvACSWOi6bgbz0QEQPdlwws9ixoOVOrrYzSGoXItZWDcUNnel9cvtcofiSyKAqYwjQnv8Mav3lbzUMuakAeIAaatHHTTccEBDuiRY/WPOy4RQJs3OgfT7iQJIAmsayu1NwMwSQOPd8cRpa+GdzWXBbpyHYbEO9MqOYFDdkxYemm3jX8g6kEIReVYzRo3k3UZXx80IY+zGOVrKEh0GQczB0aT8rPYt3DUodE5D8d3HqU9g4dngAIX5uV/dSWEyx7ZQRQT7z6eTGGm7piZi/CW1446zivwwzEUO8Y1H6QUNhVtOJhJyTOF3bQ7bJAA+ZghbOjLy9yzv76iG9IH4TACzj5AotEU4UADBJyOlvRqz+8ev6xP8QpluiFr7SUV4hFgrwaf7NC0GKaWXjddkuqPQlLkrFb5cNya2eraGHmxYEt+Qpqj74YnqGzAu6Y+lRvoKTgffXlqWdSfIAMDfyMBcqeaJQfk0Tb3RTL9Ba8/FCMph7TI2RuNLxzU7W0yc2FzEbQWoHmlH3wWFROTULDodNvLo9O3nT+YrdzFHu5xeW2rl2BmM6arlKlaz6T14kD8H9qPmi63Jj/ahnL7tVwKO6/Jq4kezu9i5lHegb4PsWQKyvwMyueWrrPM8bgCQbJjh0ean0aYWMnfWPqJs810ZUPmfb4RpKbJOXis3Y79fEW8o34Ju1NlesqTxtjSMSqmNx38l1sQK49unmEwtYrj7lTpvRyVtiBvUk0t1a+Tw7ZE3NCVGXzgqa+wYPpdq/37sBquaALw4aCBWWHMF1Fu2Ff38jPap70vuVX3N2T3xx+BLMDDqGTPXFqotWLCSXsaBGGS26sfOwNLR0fbOq42zVszlRn/lgIKHgSMY6qa6UZxzWV8kY2ULgq06tKDhqGbbZcW1Adf7KG3j6xh0pUshykkfTWwagqp+GaqmLSceJN2KK73kgvyEtu5WV+lylRdY19spDRwyKnsTovIcd4AYgqMct0FmezZ1kQUZcKCZ8EMuSb5pqIJT2zMbOXK6TnRkNTNSmtEcRkUPF6QrdJXXMSxrrmNZmlRremoFHjrgdnyZRtSM4wiRVM880OOEChyu9tlqCA0Ur4XiaV/KkAV8qMxfU/14dBQjhb/Bc81WBLP+No+K6Hkprpmu9k+zfuHU1ihElNFIUZzHvwCL0TZrkdnWAcT/rjYuPNpT+G7JdLufPbc7SHXZNllyfnnVkP7VjdoDvNw+2ucLU6LpF4Psj4LNkgfHtMm9+ubhVsxsAB5+e2Lz5cQ0Ztw5eDUDiWTKusDDGy9moIidXFTOZ5RVek8owGxPOCYoQgACBqmZOL+lnLq/tpP8bIGQ2E1EDcMMvzXS0WRryY6xTCZ/g85W70zuMh+1Noi36H6MCVgyF65gWYV2II6Folotww4uE5dMEt7A229Y5aUXeDX2YL9FCaF4XnUwaQDte5qaUcgTQMqeHfIOY80x2cfYFNLKMy/EjXNhvvtlGh8aMFjaxJtMJLqZKhemvyMkjd+KLh5+GzC0ITZhDD5FHbcWPZ7xZnNm645x/0+tQ5o1mDteYgICraC62WzxO7JrRZcGURy7KMhq+mdujM00v8OWZYo2g8KrbSR3zk0NtZLMRES5/eZSxNg4Nyu/qzqhXWlPKrpnl+J4t550/cda+JbofjYN13q3+GvJZfAYZY78O2KsGgCgMpz8Ki2LA2Fg07xkEOPw4dGaocIdBziOZQFTJJycfKGZYyIjGatf0hqclPrRAsOhzqx1Bfh9Id5/sb0JHoRabhPVwAscUZDz/Nii99M4rOyOgZ47QX+Vlf2m6UYkB/QMhrwaGdqksLIG49CpdLtnVEDu967ilykmtRSEpGjV2WgTXnvEQxFBb5sQd1Io9zkUyGTPU0vk73Zd0nV/P1Yf36STV+WGCqbLtMHLIdP3sMu/MtDa4fL6WEUc4CUvNGRj72KlD16v052Ztm1U0UlYNNiJl48LexaRMFnfind+qtNAAzBkuG15dpiSZgIu4AcEHS1aISO8KrrUe06OshvuixaNOAoG5UCGyg+66du0aGlXyzZh2mbwTDSgsfYicBV2H0nbwMe1kxyXXP8seoIED9KVMyrjSiUrs7o8jwmYRsQ2ku8jKj1ckuDisreioOLnp8pEu/uIMiz92uIJiBvIgu/OLBj4pFv7T39WpO55LNyGMVbfcqA8ngRiU5vhfFD3Y8O865Z0qnNG5zPuVnYe/nZmbJlB8q+45Wt2DhgStOTwfZzhlu5Sxy5gG4s3VtnNYJ2ej4seO3MumGb8P03DWoYZnjcZzGEdb0ZasY3nPTr5W3LxahZDMuc8LQOCKNm4zbTpLdP1gdVR6Q7UssQXMFJQ+LZk5wcckvv96GOTA3BWzXneKeW6dvVnG+xAK8KVWCbtijDtQpS1ul+Fpm5RwK5zyevwtnJO0QvlkZ1zMHSCjnTuBiwbgGoQcgiGh6314l6qicTT5LXQBDeaugQpG/ZHg3sV2fZqmf6GUwx5EM8Vvxivb4MeTNtWe+/8wf33fUMhE35qCkNH33/uFbTUKz5M4hN2hUuGKXRbU6AKu+idcGqjHPEaZt2a59XLe607wk1h/rM4EVoW0lU+beoId/csmBdfSdu7Es+ysYwI1qK/oW5pb5BDvPmMAcaDa6wK379ViErK5nQiSZh5u3z4CJSPUad6Ws9aIrlGkcohJFlZ79Bg8ehHWeHgkaIuws4r2yvOgUOXHAWFtI8hDWpx7yNw7AuNM51KAg3w1Da+Z7+G7NZSVhw0elc/dZNX/bUsCj3w1u5e8RBAA6H3nzi84ZunB5sGvg9/0HDF9+0fn75iOJZPGqeJAsvWO5AJ6m7sYBT0IT2IB4g0iccnXWXeKuOA2GCkLIjxeCZEUDOakq4jmv4O3vYdr6smGVXIZHCuNn6PtUvHm+4E2fzOfoSjbaPzrsLbaoa9waNVgWbbhu9jtnRo/5/BC3/qwBi2tf90ftDQ5FhbkLYAAHVgHM6AVjsq/BcJL3BtKBBQ31c7DLSVWjd2nbja0euzOWlIRX++SAqd0Mr0Yr6tTgR6DYtTVy0I9dhEbi9dG3WF2G5+YjeyI5L6V7GQjn0Xu+YvM+rO4YHwgH6RI+4oLnU1694x/dIxDpwyNzCrmSV7jCW0hQaOJv3vNU+lfkwl8vJEFov7mxUa3IB3Nv38W5+2hfUXjKnaWvv9vt/2fsO7YkN5Itv2b20GIJrWVA7wIyoANafP3Ao0hO97wzPc1DFrMyMyDczc3uNdnIQe/MWjghDcMiSOVtvASaCAAHZvTRKY7NZ2oGXpLkDSvENOtEqs+cCZ80AtPJ9QEL5TNJfKti0MmiV1uytaNwtw0QR7txeJifrpqzc/V6kNlkGj+nqCGf9S7HvBtWddQQRZ3FZOYaqSNX76bwGVl6E/xZ2hEhomPfxg4R5pZNMcl4fSOpMAMENoFLbO5psk4Fq7BDtkHFMqaU41HRwOejUfJ4hD0uq02/iuQhF2z16EbF1+L3c4B1+iVTn3cXDm8ahTBGU2dN3D/T9kGMcF5j7TFGraKch8HCH/I0GVRv2RDByJufgMsLRYpl/WRfnnnspYQvWMuPnTgwTXT6JmVg7kHunQDjHTTL6WI6fcBIdzXPBi3P8qPFndwgZXjQ6ctPNIrpSyteKRGSJ5zCxPnRpg1eoqfw5jkEftNIKokgJBkmNfGS7i92Lfy8lYbwWN3hFgj1AbCrRqL0jX8+640Nv2lA3vN/hoWxsaA2ovhSVwVHU2moPAIxuxq+sdJkYnN+TzgaoC5ivLa6F7S8wqiILqAVYeeqdB/yZc1dKlq8cR59Jj8oYjHaTfPV+KUIENv1b9CrVwxq8LrgpMbhLxf3GwFa86vB4skTu3+ZBaz8mm0Z62ORN13RoXJ7OqxQP/uI67dkCZRC56nImP3fb8vYYH3hHave+KXcMTVfZhGgxAvp8mBfs8jIcX2hv7m7BvsuzytT5EDnFHiLJDlIrFNR5epVrVJLpOszqQm3bX9hlQXvvx+jlPZOYUckHlMcyEDpPjjYjMR3Qd6fXgrxdOZ41AHW5K0c2a8m7mC2kOUsWfwQcYP5z6MVKYXHqzeb7pvjizj5VA8H35FGIhF70D7SLDf7Y6PlHq7ffL5TDE8tYSM7Yxas+8wEN6MR83SypmYnNRPLKIji5MlisAGMAaVreG849sIZe3hP8Otdz3CyiW7OV35J7ouibUvtRLhM0UGAYwoYN1Ar9Oo2JYJOM19r7MT693K6u03UmoXStPoOZIR4FEEyRqQzFS36gV7krdGPRXc/dSNjdXFvh/2WDVgzgr1Elg+fmVAdqIRdh9mnCcpayVz6mga0wyCbYxOZzr/O517VyeTXgtIMtgG2zCXwiMC69OBCHCl+AH8PiLTO0fI3IDu/4ff0rt0B0M/+m3hsfthjWQLHmPgc2cpUnxdJJ8XC34TEEGAUjpi/5nZJTVr9XkbyLlF6U5mFAInMw88rphtQQ+d6N+hEKfnpXn6oBGAEJrCvVrrnTca2HwNNAtxSRtQn/RFrBD5t1h+SqOh2tEukgUYGaz7Vb2bZYwsK5csUipKaFyFX0Q8eyoCsMeKbzLQRyIf3j90b/7Z7D/f4Y/ec/2H33MrlPksOppEz/+aV+uW7g7zQ5gVcedVlCfLZVSIcl8broB8TAIlhvqFFGhD2MJPDRy4lxPo9Ejox+Xsrx+V7w7TqqS5ejSBkUD1gpvZaejmt4ZfvIRkF9nGb25d5sHQv0WVxG5skTH93dwLCW5+HWEcjTMAGRuuMrTeULQVCNkBs5VYpOBhF7MJ9AJt34KyHkfNphdFlD/I7WLy08cH9Js9jjeKSTFgkcNpNxkmgnNBMluOaNtCwqUjssh6DlzCvybfWrXCoDnn5zp/VfhtMn3GruBjuhlpZh6VDoSK/ioaKQ743UIroploV8y2r5XbwDNFZL/uqSA2DnxEzGXexhQ7Jjs3aBEevnnCgICmJmtqVx1Cv66CGkcVCa2WkTJyj4aZxxxWSTbGAhd8E2Ji4L+76hQydB7ppDzX5pv7r9YbZ+jh5kdrRbLo06qwv2z3lHOvdzcLVbEF4QV96LY0/BnMzlkM5YBCXnBiCo3y1+X7OIR49eyRpOq9D6TxGnWwsf+QJkAGDw3lv4Sq63uvOv1VZ33gtTMU312Cr4PtE0Oxb9Oi+eQh5yCV12+3mOymmCe9fULk91qlViO0WblWAxVZ87QZ+63QgFAj56NDH+grtGBl4xsqmcY8dr/WZ05gYpfcCocxvsMglLIPZDjbRNeDkAgA9EvBn3cDimiRh+D2Lcasrv0kireLtV7qshPLP9z4TtruACEapgpbtj3xdWqdrqXC4h1K71Wf9kFoL4d3P0cpBPDd3AdM38imMCDGP/t02NRUmBirCKuJfWACtlgaBviJR5AWrlolu69WnlK25614Ra6i9+rC9KjW7z1vimJMRYJAKUD9ww2c3f/IG+WV+8FqUeJGJQd7l+B6RHa8MLcYeNRb1TL/W8mWIfEyWMOv8Rnb5jGaW/lvkXaCitl0ufXdkmwVjMYn6HB9UnxNXmE45buIGctVGDCG1ECP9DUNIEPjBUvTslr5cJ9OUQTvbpWgyoM5y2c4vWmEP21dFeMuzKOOQ6IaS4AWHtvahHgUyI4NBRmhnbeoHXS+4yQMjjaR9yLCxO83Uu6HYB63kHjNhvkbC3MlmyZxuxNps5kjK41prmFCq8bmvDObLF+flHmHhw+xR0MgRudCBLchalay3obI9sQScnTuY4JXDd91WpWAFhwhq1cX8W+0ry30kGO2kbtXd0R712POzQBORXMrPVcEqn518JJBvvTTMeiFPK/2jkckzi7ybQabF70e5UPHVCHnq0JMIrzlQS6CohEWNMX3132YXqA0pG+M8pe1dhwd9fmOF+GN/ERfCnPLmhkdLpfQMZjWIB+c0APGSARC8VZ+nHbhtQnvSSgeW9LqHe20exEyJ3qYIp2amG/gWs6V5i821kWY7XIcneXg8hQNyymmEJ2i7uMnEHYaPHLrH/7JVxWwfph820YZiyHo4/YutMcbbek51BbcNkwMxedSYB07xwQNzQDmMxoKecYA6Pbr9METmb9bkCowCOoCE4ikokquY8mE8TOj4Vxb0cKY/LAhiXpwOPFRLnddZj+MDIVSsE4z3x5+Ft3HWMO9w34uJBhofyeBnzfi4Yh7CKBxbgpihFccfYebG602Eypqx9YAdnzWqxffhqHxe1zVq7AQtmOP2Ro+ImWff7V+LO1LV3F+x0eqFY6qHJ8ANzCusohGfd5vo/BvYmIewpdbosGtAxbBMGIR1ylQozhSw9Rif2+2DeimF/oWJH9wOZrOJgIO594mCvbPaz5/xuSy/GhTmMaq7jPxLPfqBVlKuCGF1Oq9NzNaSNFJRlZij5+o6A5MTkS+K0gh5kfpGjt3nUWsQR+2fVnAsoa0wgRf599T1dYwuTAUG97BZ1PPmyqyPWGnrCNMYZP1S6CJmO75p9RbnTfnopCj0H4zP5Hi9kOaQOffdbUxdNXXBKOBdCUNe3bCeXIYxDzb21yplNTPRe6qg6RUhTw4CClMrX+E9YnIXNRJC/7qv/AkL8mKbtOpXT0V34fiao18wo4nCh3WwiBx55+4yHnvn/NhfbwRZM4ebfHP75YG/qjPl1pfuPahJle/vL3QoKx/zfdJ5rOnnSEhAO7CnNAoMxoOsofdG9HngQA0eghr8+YRFxzm1lhUCPj0wdTG4aXDG6YQYQZFdHIFe7mRORyCtbYgprEu+KGG+pJ4ntzgOmBCuiI+2OPWhx+yH5hn6gyWbz2luT5X1uXxpt5qyINau3QXgp0uN+EgDqNCzsa2ndWFCWsLIFetazO1rFc7DOzwmgoUkrpdJRZHMXoIOgG0+Wyojqy4trOxrE7XobStMe9S6sxcqbMw5zxnq3TlspK/fCml+xRrykZiRkq1GIAqA+iv+0zPlPfpqSH38MaUMgQPyJoDp9q2Xb4InjpL+q8fRz+fDMYssnKTENScoRA5fOHt9DpPk6tYHNGuaitdxPxQhzBfidJJ9ULVYGZWaM2KHYUcbe7D82oQxWYtQyXCLeuQbO8o4THU4K5r3QGYAm7Y2yVZrPPo0bLFT8QH+Ukgm45gHX5lGpHj+8uhKamwOiUk7TZj8I8hS9aEznvNRGRX3W3bvleE1E4WWaBysgiKLD0njt/8YB6tHzYA+xdz+mqzATaKop7LLwUXN+KDRAGMb/qAGthtQ/S+yqlX+l8EkGEPwRd/yNIXS5phcZ6cqxraEl+wcDcNzfSvr1Q5pIgQC09JtmrPvVwKqJBf2HSF9xNFlGR7L2bxFNl00T8I014jYUJti4TO3l09l/HBlAwNhh19ZUya500amVSOQmLJzsRMFDP6Y3K/wwXsuh5D8qA2X6tmjwaBBlFUmfY/pSRXO3bLO+F14JPllOhhz7R9hqQDcwtkD7z+otqt9iksE7r17XZY+L1CixVJwVWWd+MbbHd9cZqko6+taY85SVeJB7BD7jUHUYpT6Lyc5Cud3uS8/QDKXTu+aomvC7E7jAy732paJWg14/ExTLNZleCh6hrvDGZthdLiMGPOsyn2EVFH0owqVgPNljW+m/gR3+I0rZGJ+YscgYzY2ZJqzkKoqyYK6IXTKjmX30Cr9Xu0iVpLq3WR8KznfqvnsjqfkjbOftCRGoOzlsybh0g21hzEM6fT71FG91jCXH78h4nO61mJjcgbQFF1dBl9BZv6RcwZqPDBZloV0J2XSj/K56wWKTVk/6pdFK52sbbJwACWJPLr3Q7VVb/iCyHk9j13iGWixfGoPRmBYNjwe5cHD3B0fil6HxClgPeI6dhUI6dS539NZOhdufIQhokqqHWHJYB8n6JHy5a/RL0jEsNHG+possaoyFNsXj5nq3DoqFiWaeQelYq8HXEnt5EJqgpXXUcjbKb/4fHFcPCaULx6ApQgyrJ67B/fF0i7WBabUfa21L92vTLHBEEcZn6MgPJuMmkeB/TrdtAYfsbsq8Ac78i6wTKoqFhWHOb5DXzpdDDWqKFXfm1HGPyCSjg00r1VvI3gaxRNWEBpLEDy/+Tq9xFM4YQEYxP1Mm2iJyfXyPZ0cMb4wwsheIk0RMZWrMs0PN/G1Bf7bFXDhKrTujQsu8yjZSaKy9dG2PMIJCF8sbormF0p9Zjn1kJ7tXnXBh6pi6y1P9SCUJTnqLg2fwZW3QjnQ1mvoE2prnvAhkKkvogMc1n4VbsAIdGsGkiBrDBkJNCER/p7CDGOr8rBbk2HRddfwsxwuBWTTOMYVVOpSGrImMngetyNYIRZ2nZXa974j/HApZEuCqzZQkHSEYmIUSR7JrBVefEE13YTQMpsYriFSsXO3v5LstJmZyJzHtJIuxuTH5aovzSlOf0OFFkBjyLSHecoDx514wmCCcgpvUVcdT1oYaeGKF1pxVCXbNn0nPAmDN0xFAbAp7c4c/+ag+pf8lCq2xk4utqlEb9dj+WgAbxJ545xxs9N8N7peM65wxCJYDCflDFUZz7EN/eYhdJ0xqiawFRBU2pI2h8JB2CNj1Iyi1ADDChamB5AuiiZj8Lm+27lIfoU15ExymT9VPbrOB+kqpmuZI4CshCAvO6ZDetRq3ue7hq3mVkUujteYgKt+kSTGD8642iozAd6UNYA4JpseNVdhp+Kd11hn6ci6rUOBJJXNVafaHyHfxXBFabGjH4VCYNWKe6zeR17aUvSJh5eClI2QMpyFDxil44JKA/lsUToofK2y2VcAJoei+rTkOHwyvtbqOQWRzO4b5OeNjVy1ygvXHV+oqm8XKnvDKXHEOR+zRyYRqfH3+R3LPYLZ7FGfmZsVmXr2pSRKCokiVLw85li8mpLzpYqDbogIj27zhkVhRcWSJC+A2tijlZBhP36svdHXObjDutMtfvFWqKtr6jIUzQPlJcln6nQ0gMf4jF3d2bB2JZj0iWa6K7jl+7QPGZIZc/B/oxVFfZU0azJ2Gk/K/TcHMHUnCYJEbdUdizFuZAorFLRlZxOHe8gHK6gKskJVZgI2iAGYC4k2P6+Y0zM8n1S2+mhGFP9gj0LkVKGQZcV0zCDL6MwtoYSQS/N9KP3BLoFoZDKn9OTDUZ8r4Y+5O87NvfV3INN/RX2S/4Wy6XspCOx/IVwdsJZ7QJpUjYArmC//I/igyWAE/uAVDgB7hq8V+zv82ERkvlxIYeYFy4gfw+iOl9jdzxe6cDAMdxosoz4WFjii2PwbiB8oFGDD6s09ff3T3xlLo7/YCX9i1vBZMwnuckmoCgle0sEgCh6q/+pzDNqJEn/6NVO18lf/5lxaHlvxMX2O5VNU7RTe30wOO5Q/nXX/7qsMet/+c82sd3v7pf7TSzlHc1Qfslvv6Su5qNPyWly/mUu/lUuPns/X8F2EOBRH1fp8vvn72v9y/f+7b/OvG28cqnseObRSK//8/t///fN+zzt4PkQr/1X/a3pNn3cH3Yj/7Xr/dKM2Ib8P/u3d/kefaPTzyTjq1Btmz2AXzyR//3VpRoMrRoJXEsaNAnon/9vz/p/+2M9+/ZfP+p+e0/0mffwfr/OsY5/Uf57zX9f7z5o/ciLAptW63bPmSBK6UtbTq/JnL444Uh/89Xu3RzZ+/anhrDe7//s6f671Z82svtsy1P2kz++9/N9Ewmdn9L77Jvx4uYICGb5zm81zWz4+Dd9HHM+BTD8+LD++zds5jabCXO/Yn6uAVSTeIX7nkviseqC6/H+687M7aPCAKBz61zu7/37n+7+/87OOTR7CXTq4/+PONkeDzs+m5wGZVb95H7TuoO6p9/9c4//i6Szv//90f0k64f6np/vnrv/Nblj8f3/X13/Yjd9d+S+f9cEnl+grkOg95f/pNf6AVBpKUXNMUaZyIKMyGuY0X8zohWLz7PLvZ9pf97Ja80pC8fme6qcIvdj/fpaon1w3X/uRW7ACn5++aLDz2RfoHSa91dLXOwxAh/hfP/R/+fzffbz/Xdf8ref+6gAOutPbv57f/9Lf/tnbCKAJGviCErtTXUH0C3NeS6fPZDH84dCjAI6CEkGVyPCEB6s/SE2+Pmm79Vv/bR1RvrSNk8Za6Hq1F1E87ljaAnHZigXBjEA8jxobSb2li0Q0fBFmHCH7LEyqOY2HerYHghsUDZPeHn37beq6jfy+CvLn3R/RljqNvIzWCdATUH+Do3upFc1xQCgqc82dpVA/4nhunQFdZLmcoJiVEgQS6TzvlVnPmU3kNHF6b3hCyeO1YdEMUyBUSAQ/p50RoXNK1MCPFzjAbYdsCvhxltr957NumylbvrfVWRWFMWTnLkaZ+UB3uc6T7A6QLjnLH3ICIzHEvMF8g97PEcTw1tQG7mv8eGvOdJIreAVJkhACM/K98uaXwZQpC5qZsTxjwy9mN3MEIuc+x7XzRoZupNfvhcTocGr37dH33oy+nSdLXYPCXdZk0OrPKqSZSMdDwyqOl5U0DMp8Rv3bvBezBtlnJl8GhvBlbb8CtfAXeuRETRg86qe/skABJYSxvHbPrMgWjXiwcu83Uz1U/4Z3pQZ8kk7IG/5UlYmAv4UsWqdE2TR1T0gFaU73NCxJ/Wugjt4xPJDlOi4Ig7Vy+duHt2KPQyZj9DciydaoyjywwIUwur2Tqzk8BDKFuw5B7HY7IpzD7HtbXjNYLSx7L9W+gGBkQoAPdXaF3F2t37FELqLrcCA1yqQF+DQX28XREtwxMyxTrIhP0r5l8TxFqMx47DmGMf7z/YFF39e1EYgbragA8kksTtO5OgtaIpkySz26QaGXC5q5Dv7rVB/EnLf42ybJjsJ2R2zul16rpr1tvz2FJIvBtw+WknxrlfaaT8dB+9ChUsobRdGXNiLz80yGaSJJx/HRDTbJB/Vc0Pw6SR9ZkpEA4z7EX4N+ZJCmb59I6ng4ImFE8m8VM81OIG8BgTrzU8GQKFzIXY9TwKkVa7+CuX4eLfVucxoA18M0EzMfLHqMcFTZHQSm8h7Rgj5fwzx5kuRDE8otG/uoLqRtxPxKLqVEelY8jlPPPbn7YI8TK8gdk9t3n7rFTyX47RrLJv+j+fgJfYG3RgaRjq9nDH/eER+Tw7MZjzgzSDBcekQqv33fGCjufJacbAPELCQ8VqCL0HLKAE858gunF/17LE6BJSmCnHPaMMKNkcmgrPFleZDyJ1UCeBgcNbj/7HAyMqTSN/2RPW8AYcwGGML85veb+blY5eQ9EeU0IW2IfbTl2eHm404xM2MP7bBR+lZF2whBnjD6fjVv3kNO27Ln7SS+8V7/OQvF29BfgU4fZZfKcc4ado9yojkQBwg4Zc33RlfK+fvkFBfQmCygMADFiwJWNTuayoQP64tBOJ6PGS9Kg3ES7cOPm+NK/OepuonBiQ+ea81sOS9qvBlZwqihfV3dKDtaKiQIyCGcSKIKBm+BPhomZ8wW/qQvPNzdshu386BiZjqnYZ3mCMYaLS/g1HHXVBQDTVG3rKr9k9Dkw0z9tTdt2EnpnZtDLOQW7FsDnU1u2AGkJkZT5rUNtoq9CZT8uiL/0IqZu5AtYa0h5nCj8LSQ0nRjWRoGRXcmjaAeW38RY4Uh2fM0veW69Ips4kdPsU2wVQnPKyA+e1TvBKNBglC0fRkPJWEeDR3rfOyQQ7cpWbxfj9mcO1uc0wSb5jntUCB+hZd8+68hCvjNwHvz3jiDk2EKjsgcg+w7ukHex7Qv8TjXlkqWhonzgIYHglcda6EoQ7OlDDiKpommvBBUPqNd0lbTMyLIL3f/eHBzgegjkYGiJbGnBxLN9Q/rZR68LFrNEI1suk4CNrfZKqf7+lj4LS5YmQo6eFEV77yT8tMcNLrH5a8wPv866RTMO8hsykIM0zdII4oF/HVvS8LaceIYFjmrSXDyljNlHwNHJx7BQqAPFDqsyVGtAUj+Z77rH6KmGwTILFxdml58fK1DyMr4klLwyvloW/kzApBOIV2UJPGlQ/t7XFXzYzYMicpMzJvztIEE1QEDdql7uXmhns5BiluDOz7NCp4H0IHzcby9PEiap1ItzaFXa+EmfBwYEjazzx52QdUCO2qofTQBs7iboiicvZWd72AODu8n4epcPtWkMEeMqcdB9Om/AiJ3ApHX97ERNHKvK4gEiNIZigfwkgwkcJmKfITfrbafNMPv7VipaNmTXThM21erQFJosHilw0zaBI5FnoWP3vKT+TEzHeFYMq982wPlee4zfV0ZHksxivwWxRuGRsprE6UHFUVc8T32KF4Zzmadi6ykL2V3oyINwzh9yYbTAEKwGjJbMNDeJl1EjoV4OR29EkQ6nFDXuDvua2gYbh9p9AWpXTqNhyKLigLYLii6mAB2WLcwdxQ58gdUBtBzHANzpI1UrI9YAFLQoBV6nMGCAh/2ixaOjx2t7TV+kZsNGJTrCPIjCROhbJ8V95AtznvRGSBJjBnhAaL3s/qfMYKInfbIlr+q0xrT/JITwnuAz7czOB3Hah3WTGaYzMFv7Z0otplsyRvgKAmExw5tV3qB3+IqDeLKfNOdmji5fX+wP9Y8ajhO+6XI72OK09yrT4oUXQrO4OVfASvyZae1FhW+0yqRWNuyHKkpR5ziUTKpOkADqGnxsmxlJLKVSN47ZJ/D1G7rPY8Z2tuLaJNtz9IJnxObyUX5Wz9XY3Z0GLWY+jD2CKuHo9Mxawa8YmBqPi1qoEgEq9kkKDYtoDutaEn5WIswwXyLMuyjpO2UDQMp+7xC7rJ57sASqTiTV57jRoI2Dc1/meeMaeskmuXmOYlXRLIFaxm+p6pUrP/Yw5WthgjkBWmZy2joYA6o6Nbmwm4MXOpfUH6x8Bgpk7cWn1vUQJn10BMMi3B13vce+0SbjFKl0v/a/NW2BMqBbsQmFAoV9uh6+ROI2VaxLyQaAEL2vnr6opw3NfnR92YiROLlV4MXAtoAIFeS57CRg/2zUZY3idTnkyIRJLRTmHnqObHagYTCtylGlQne6vTKK+2wwFlzlG5ZjyFSlWftgnvFHvaScoiYvDNGHuB2s5dgVT5UxTqojT2nacLyVdnG6PWWej8ryRXvxt1j4WerO9nuxXZ3UbkhrwFFs/SvvfOew9+UibzhOHSkMc+/xpGRmYVWH5IgVJ9jioEFtfRygGs61iJoo1dUVicPMcSAN7yO0YyV5s560m8D6wA6vwnWHBroQ/LzwSgcAlpdstwnw39d4P2NiVrgUA7fbwiV/1hv5hcxb7XuohU7/RqdzMDyZnGZQUT5ztJZwZYswfiC12bI4j3wEmJiJ+LbULSON2MpzKu793P1P0jvHy8Ft1a6+4if6dZMtSa23/QrgLKIrpkD3fGBs52d4+TbCdI6/tKdQAIDX/rOb/oQkvt6sdrmJ9wA+HfFr8o52LfivhTp+7IEGU1LoEUiHxkewQAzZdUS/zCSlD1SIUescIlwYmxsPBmaNDLw6wWqb0TSbyDBotKe58yMfOicrToiFo5q3Luk9cjsIC/0nBc/XZnqmjqax6M8PrrSfNqQZV6n9yxZnGHvh1dBQWWWjEfeODJB56TxOa4zorfYeUAx0pFd6hvaR8CRHiaSdW/3TKQoglYR8zzi4ehNSR9qInzO5m7yfMM21+eD9h0xQKmWUN/1Rog8FGJYpvGQ+hcorVgQTQ3aIzIeinUCK9PzLrw6+LN3wK5XJojcT74HoSVYUhuxZvwmYPGGnUWzVLy99Q9SX+JF1sFjDOY46rr3iudZlTQTOYUwYV1/5CK7vmG2pLl+XyEc7muu4NL0Gvw3YpkfukJmw3cL6KO+sbSMqC/FvnrVghZKGl5/DHBc1e7XJBnbLuqmgAgTpoVpqqZIPM5b3wE6brB0OJg/HI1wmF9YM+aWL73YacBGw+0EahbesXKgtkWv7+fTiRx1OTXAGTkz+kvVldCDLkykkt49a4WJFYYAUHt2853pi46KhBdoLQCIF10eKAaNl6R94EBF4XgaSTQbHkMEECJvAb4jQyjeDsPnuzoKvUmIk1L294jML4A6J5ET92p6evVG2kDVYkOSn+VcaxjrRRIF+TsOaV+cXqG5zRt4yTZMEaXRDlF5Ergvkn6ztHz4Ek3sDsNE+xoIg8f/QeqPFbe5PDanpZPHowvfRXxa60kv33P3nLMIw1qz4Bzd0doo8+yuN83Mko+/WMrby7MuHxeIO5uEeY4n0JKLny/greXPL011nYwvBTs0RaMDizvBIETD0q1rbYdzaPPUWRCXCf3O0LMTMUrekjSkLApFzCJKqmoaEw+b6FgH197Uk6BDcctGR1/Dqh77axS8mBs7XfpRkw8U9beX6nJ7Oyj9QAzO0sfyKT0Ehl6OsKJ/vewtX4g0J0URM2vS8pba6ChIi/DO7PgmK5BttHe3OwyNC5pWI5gt75sNcVaxSKR+aB4tDEBgHkAH14kiG3OJCoH6gK4d6BPsC11SZ23Ey5n3SlNEhCl8zrfe+cHg/m2odNa/sOTkwiPKlpLSm3GLeTZoXqPifBVi3ySf8EX806qu7joeNExNh9af9ier1psBAWe/5G3IEJojjjUVjh5Y0jwnW7vVtdySUM+/CrOb6YN0UURWyjt4hMbSoqXM0M3Jj7qPEpzjW0HlLdUYZvLT+TS6gTFFLKvDiWpISwcFfIRADL0EXaL4mKyNPMH2xOXlN/2Yrun6eTiYRoK5cp2Y4iyXuq3CtRq9xyhEERolka35SjEuIfPGhPe3aNlUZSrfrDz6xlb684XcWmJyxiQzBVjvTf/ea3Y6X08HeprlG0iZSe5Vi5/hS31cEQQTm/F5uKDYYTItNDi5NTZ3lN1s4BJSNnU79xzI9C0wNNd5mzXb5PV6KLFfvL+cArdXeqD6pDKR5dF+/D0f6vWANTz5hN/vxHwebV5p5FryFvaGvBYEhlw0k/Cwq78bJpQPW5QrupBfdf7BAnvN3uSutSBI9pq2A2Df6PVK3p6Q7K8i6VIpcOhJJN7lWI/BbS2hAexvG38JI+5MV4vdttiFq/JpUQONdOqWH6CbcUdgftE6LAl8UO53IthnJ6keJ57v90jVQawevCoKs/RSN4D8WGqB5ZuPeA8lYrtU/zAu9UJWowDxwWiniTH8Blm6fNh62BlCt3vDPIsMs/4kG4m5kV/kjv6KujZgO1Qmhd+ODNfWs2dM9IZdeSxVkSEQSnD0xelCUPEwv9VgeDnDr6/B4smiUBZUrpGj5X02mqCqNEIoKs5pWSBANfXw60EQLlnj8+KoShpTesBXd3x+2RHPv60ut91Nc9YXYxHbvjKRcLMZHF1xlH6+FvN0QFQTGRncqQ/l9tnQl8WLZTnjnUKVR2pQ0Hg8o5V5C+f9o3inqL+Lhez6Skvi5a3Fb45eCzqsI0t2HLaSieTsK/AOzedRXMB3SJTNWxzGgb1fwMMz/Yo/qVMQfc4YPXaerG/I4FkmeOzOBD9yzdUr81q/jVpsveitJnihaxuwnjKErbvYb3ILl4/u0h6Gs+LAnrNBJzqJUb681uCkO+Yxf8FlTMXxbu6/vZK4X9nzW1tf+vwGo0LFUz+gw5V2ghMwKWYsn8lLO8wnau6o0AxzTw9uLXVJCSBvJ3B9VE9uW0TCRP3WKj27GeMUMEFgmjYW1E9XY+L1ZoDnRrNE59KDB7pw7/E6gxyWfVZ+FVAmb3TsLtKBemPmf2PTxvEE9jGJTl/LO6GbyFAC8StVCSyKKAIqcCto7iY4VrKZFZc4yO3S8I2clGrWth/ofVk5CnyQ2XRwMfFKQQBVgOqru0oCOwz/G1B6+SlcZaqiyvoKm9Mo63TOgZe1dDvJssOEYmvarXW+3kqH4DjSyT3bBmVxEx6ajcPMdZtuHVG6XaMDPJjinAGPzvnFLLLyFiysoqQcQTmoWIsCxmY8SkWMPtfQHY4rcYtRypALv/wmcfa6nL345BzEK0AgjocHvWM8pRsE2zQ3gi+W4DdaZExFyQrElUwkMsXxeClYRiT7nwdB9iV/whjnELgq8yoWx8rxl78qNXpBsJGPUK/p/HVMjLGZ6LZKlDhnMzRZxWiSYx1bv6AkyE9o9of21T8oS5ETVeu3fA/7sXvbjnshlgR78aEHiw9Jq2UUliRd+ATTUgtFlU0Pl33YwJHiqgbTVCg/viq694NVG9Yce1vdSMuhf7bGh1z3DxYybahti4bipon6825tI7IS/EmpvOFI2LkpfegssPsQNOAGeldScv+xV3ylAxG0563CG6k2yUN1grKje/n7Nd5NugxHm6blGfcW0/Dj+4i7M32QKlcooRWaMrSk1dzA30vTPwF3VUjLLCCGqGClgOLKCsxw5w4E8B7WJ4sZzw14BvOvuNggKsZEx2DCPoKmZi5WNqzkKe52xlxkFOhaovw1qq0i4vVoCuCVCRjiM4QPffaVtHmTk/UBJaGszjGNl46Ga29fchZp0QbfbT3/XpgHt9xxRcn5KmMpaWPZITt3RZ3YG53su2Jh5r3kpODzyCuiUTyC6OjAtJAwFsZ3KFJF8HXgO1GsFD7NsrN+HcFqp+6dTQ+oLP2SmLzJ/mk9iylSEBNQgbhgPmMGB2n8Ajp91U1s4UhMtC6znXGS5b2DQEYe5MBeeZK0jmBrOWNHN3huHqScljo9QJYKK/Cie1f6FQyQEgzceD5YE2okM7nkGw+m3kzbmgmGcd5FEsHytjHNCGGIOAOJYCK/fTnydPOrPWoCFDphaXh6DHR1IwHek9C5XRNIYXYz2S9c1UPk5mC/Vse//P4zirC6cSSVSL5a2pfnXQJ7xN60PeSLYZrwDaGVYEknIjXctVuJSL789l3T1SeVTGfFpmxu1rEqBwnRpg0v1ogmT2clvAZye8woMMkHMPxzpjIDVWR3Qe996Bhry/y+nwuinL7b9hyQpuZ17+2plfLFyU4LpT11sz2rIDIBRTBf7j3ABsvPaHimjB6Ba4LI1Ps6GrAHbydC87TSJz9x42sgJqzCk9yGfeB7bvy6Yl5VZWQlE1KgoQzb13js2JNvGUdlQ4kNGIiQmdkMHzRqR5FkPnZieHDV+m5exxxTqEDDugDSWYJANzaZmAqai8V7ugqhIhSKSOlw9Qsc+ulz1SpiCH7TseBic8QF9I4CX9x+25+7lo2VcevKdrH2A994aByP6hsyL5GtzM5g2vYzhN2MmZ52Cz1yaFa3/Ajg/WW5NFTin6T4ZMqV9osLf/DrlfCuG+KUDd34Rup4030Ii+GalC5HUkyaiPnFEZP2c94FzKPlxpZIoX2b+nkz2oBe7jy9Wigr1p2HipDh7INUkgiF7D56q2i9HrbSfvuGfUh6f3hmge0cQtNEKU/tNFqouMEF8tEzhi7x66FF8r6MbagPAy8IfkkV527FoJvk0kwYRzVCdEkf/3yJeug1K80RYyPJsMwEBEoChzyu3BkrxUX5ZUIfFqPyg0oeCdE48AbSHMecF0bePeQZZQ0a0WNbSIF8o1+LZGBIOiDcVvl90+VdhFTnTJA0EEzT7NsGOLKb0Rp+LyFO8sP96yYOfKrA4RvP6LADAw9SAkEwLAU/eVtWiDYZWnWsPdYZ4V0yZAflc/8hiwiCHtd7S4EP9GswKTvEBEzIkLSVzh/nBuvcv8kn35+I2b+2cjMBwlslXlhBQg605x3hlgz7Bkro6pLsbd6OtvLh+r+4biSBZznZQwDuMZXYXXvMSbpGPz/gyCbjMJgpd+Tqewf3+I3AXVfdsCl8qO07TBr7l6StNEAlAGTg0RJw/S+AL3pDyOx12sAbZtLCaNPtI3roQjK5hn3CppjO7WuZr8zmH1D6glOlQzWMGR5FWXiW9V4+ffArwliDcA3wVOIPHqoycCfw2rwN7ownFghbZ6ZnrwWIa7/c3zjZt1rzmHGji4e8mbdcdx39bBh//8oPFltnIQz6XjfYBgqNfrssM69QjdkeJbQfXBWxioPbOBTmo6gK2G7gC9RC/+xrKdLkS4lUpt3WlHEBc/so6iSntAijU7Gj543bjVlaTHWg8GbPX/kWmqr0tpwsNHBxwvP6z4yVR6jEJViya4+A/keBE8P88wCGfNvbb6TuCvM5y1PUVNZl3pRafJFntFcHSQzWBeM6dJY40A6efZuOk+WgjztrVgzGnp8ZkX9TkLlUaaOdb67Pr8UM8F4Q9Tbb30rrcah0XXJLUI5+NO9hAftrDUBIyrxylD4ast5b5sK+gLecrolz2xeLcQ8OPDOkOhLeHBzaFyQMBNBGhbYO6PvHdRax0Mt6mEodiagDxjD9xriOF+KS2GF854LjYYrZsn9KdRQsp3vHjUjkAZCgjYUJrmJyXTx08cIyiyagsIIzrZzLc8uJE7/1R7rbobV55Yt57g36lJAgSPOZZ6SvWLum4hSXZxPIRkLY2Cgl6epgu/GpbGy2P8AENBvwg8U7JdLFe4rrIqAI7OZg2epxL+I/NHhpQImEB8uDycfiprhb8mCkjBddx32bVyTIXk1YrAkd1OiwnFq1S/+IvTHYDvpi0EvrQa6pHIAE/Z6QoNe3c9bpPgtE7TonLVWi/pSNfGwr/tYx1MMJmiK8NztXnMKho+XTDseiuXEo9vxJNO1Lr/AIfSvHMTO+Wh1vJPSo8CuqPbBdAXPXQVinJQAD86I+d3SRkakB84l3+GKnscmANjP3Ew0ToB0UmOoBRX74vTJHwBrxsjLZFhT0njepY3TGSvQBciHsIX5ZqpChFKDHBRbWqfcteqzt+7IsamYhfwiAPoJ4tgdCpt8lA7YEQy3YUkHSMb1DCWon/nPs4HIvC4PVoutKx7XOwqMuxs/XvVBkb+araJaUm6iiF7CioqlkS44AxNmhax8DG+jWtsjry8h6zurrpjDZb8i6uQh2LAUdeCh/BC4HmbFMi/1eddN5Jn1eiblMtkbxX1XnsWXy91RzLPSLMJT1EJiyi6Do2bnIWfw0E/2d1vlY05uVLN8L/ZmHQrvvUj30WwXGISihmmduYm+3G1EhQDJfmyObVV402YUBSBp7KBYMRPKAdISg+M2QSeVNkj4glzN/xri1FdrEt5ayY526KfvERMOvhxXgipWIAWhRUvZQ/8oS1JzVUTugdTtq17L5dbrZj5jU1/403TMxmtqYN4B8De9zAeevhJxj1r9ZjTvgKhwMgBXJ9cWLokC76bTvDdxqTWouy4RLeihtdm2n7VKcbo4DXfTQ4vIL+9EVfM7YljQ1leW6opUfxzyKA30l9m+9o2mpoZMVCyanb9cucIu/kMqYriRbihfEQmpucBZLtK7lA7dBuiqSMT9cjoTLsxI30w8FHweui2/HNhSfk6xu0N7DzI9vFMX05stAI/5mnO3u3hPA9HozAlVMIerwIJjlryu0P3TfhxG+gAWp6jdFU9O2BSyzJh01W2In85LM6acHGvg9v3IkO0B9ZYMe/YA17zxwSS8fz8Eg9rNsQYcJQpVIjMZgQY649iVbwjuVs04GuxRKx9pDyPxVsq7obVgFLVdgUuEoyyW627+h5UEs+XHf9nePEd3mg21MLMi9ALAYmiKRfdKann1hLrtCX/UAhAZPv823p+NYY1StDh8dOIfzvNPhpJHnD/GXTGXwHTVZtr1/3abUY2f7penxhHcPFolNq+ywvjJnYMC1eIlJxPKuMLD0ckgOlMtk7wa+SxFklZO8Sc2ZpWouunafenJ607erd6Rwq8gfVlQ6QJ1fUbSwrQNoczV780zidSirjY7nRYT/Zk58sXCCv7fmVRUp2mYSRoG3x4FDnr/y/2P5+ojE0x64mipKFFMOH7KixLmsHjIrUhp6Vy4c3m9eqBkU4V9mftC5J6W924GIxCr9Bi/Bb7KssKllUm39JTUIgRlbSPpSaRDhgfILgJpj45F4mmUmYvZMkNtPvIR8PTphx5WY3A68VtGGShGJHz9WS5T7C6ApNrgiYnUloOoReIGyZ8c/EGO9eQXf9APRmEVQIcl/WV9HUO+14hhdFb57rUQQV8U9UAp2QebTY7sOqAD+nqnW30Hq2mvZhfh9MXivGhrbStAdfVC/w+zNWEqlQYj8RtmDtUt5LG+vfU0lDHTZrxGSmbPbg+ch+0uINFSXJ7bQzf7BsdeoybnrjoQazOPPBLbwHxN/WnCLqdP7FeVfGEaWrM2VX6PNQrgzTsRvlFFeIL1OPgA+TTVVB6Kuenr5EuBj0yTVNbHgWxUeu/25JOD9K8vl1o0BG335bGYQH40nRqUh4AKliYHMel0erzN0jjdh3rwFn4xCyao1sdH8MGi3GNnhq6DtxaxG+iwNukNg18QvCLRE0C6SGpuOmxIz1FlWIYFaiLckF3CfsPtMym7AvPV0wzvFcRkZYLrURE0HbLx8+hPTHeI4wczMOyzFGKfSKylw2ZGZz6WUyzC3JYWjOb8x1n0sSYtszImkOXhBIGG/YCKP3pQF0MF1uOd3G0za3hEqnCN/fE1fYM+CcSqGU/RGJhQYpvQhsO3dEm5cNmWOPrwIzHxtH0pTH/gHtHa/i36x59eK2O1Bdl5zlq98/AopTl+Bzu4prywKE5EMNIB8Mwun0hmVwPZ7CYWTQOW1fnpIRWa4/uFP19YHVl5vYpN6w0c6lzYtC51dRPAgWByprqqteRbo/kD7CwscAecW1ZdbpBrMuVJ6hBYGmo7+sJZL01twkzTu8x4ZUwy8EUxax6JYVTWAupnoCI4cFXTF9Y7H/FpkOL7PllH0niPpRRNvjKaWWGJ2BIztBbIy03c2SgF0CmB3y8HI5HW/9h2TP0FS/RyDivbN1a/fsDRshcLyYzQuufoKwR2JPZx3xHBn8BBIgPPjL8HD1IfML+Y9V+nRlS+AZU/JFXYti8QRvM2vYOnHeMa2q8LEzj+01S9YmhF0qaNt8wFpETv6/sbZWoZa88gmZ9hzekdXEeX1SnijUP5qsS1YhMF5+qAi8abFO3pE/Pn+FLZTlVFwDx12QNk6dwN9bworq3/tpZo3EC8yEOkGzcUDcptSnvWaKNBJdAWXuxr/W/c6c/pGaRZ405rWa10ZrlAj69hoAKtya0ylg7LRktUVd2n0EizpJ5yZ0cUI5o1xFe0DxTIgL5jtEvSFdQi+vA5dEl0mwcj2dV3wvuyeI8QCkkYyWFkf5KYqNEMReTpLBSEYfsFTsRMTTe/R9J1fELXjZH7ib3zQ9bEydJjvDdNnHNAxRIzJKALwa2Te71y7B0KzMeOLuFCWq7wp4NVof8FOD+VkkEv0EEAE0yquYFRkLxJN724ig7JfryIYJXnE3fAEey0KBGiubZz02UQcejWt486FJ0MRVqIK1A2ovBNmKe56NZN7cb6g9r0X02qXdf4YVJ1YKWtnv3JP1zyOhBazjz4dtivmpQcC3nsubS/N8nwWCGcp6mLN5l+57bdLhxXNqnCS9OhRK9XuQqgR3Kt5HxjxJhFN/LnW9j4BBnk04ym9Ab4TObYM6lPrpkt7Xw6H6REXKIMnGmgkBVW2PfyEsV3nTZs/Kslz1tXM8ZyySR0EYVae2rqdxkFfbDbsBJIszhybV9zyFWECEQs04isE3ClW0UOq2loKG4XOpH3DuoLGrWp/VGiNzQV/xQdSe9F5/OoMI7gHiFEneOcHGBGSw3q5G7awTVQAl+2vW0XvhZX7iAwdjtqN9PX1t7JmoWrqRDyBrQNWGjSE7JXIbKe0yf22UdYNBSTCAXtZD4nhiWohpRPyk1QFDIOHdQUaT3BEx/1HXY1Bgds3A4Rl/bUQvom8R8WSP3x5YtfjEK5cVFcDhVgrdNjozuUpSV+kxX8A0xrpdNNzgh6HGmh3OssnKE/Do1VRT9OzBxBSqsVF1vvLUrM4JqbZ0Zc38BuaRakL5TCi6S/QTgeDVaboErAoXX7Ozhc2SmZPKegx2JQ0+QufvYZ3gA8Hy0JpPxPiXAw0Eby1uc5hVYuwTQ5+SEV72Bp5zc8/qIkw3Sc52ETdILAkS+7XRXrryOUwnQzmvYtEVE3KYj14UScXTgOjC0RgNKhSVaiZw4YednhMB9AcCh3f3P9Q/v2dboyhMMj24eYtolzTtrviJYj3bzjDFjDfOCLc3sjqzG4+QEktVd06Z0uKpyAs8oSDWNfeYM6EcOJbCNdDEVthXjLB8SoNctjx605i1aiM+diKjyFoDSUTjOZvzyF9VSKt+Tb+FWWvVxMENzQFr72uARcFNixeM/nUDdMs93EY3u9wQaiJnsQofBcSadpZtbUPmlAv9lPX5jaihnfmu+XNhT1bI/C0JegXd0rg4XiBIGz1v3m6im3XkSD5NbMXw1KMFpO1E1vM+PWjuq9netPnka+sqsyMiKRJuWogi9W5HX2D1C345PfXt3fi456P+d+Ewkr/cEXUly7/rQX3pNZTFJeGD1SwMmxwogGC/eYog+0+ldebaxlGuDSVkRUIQj4zsQT5mONtPjCFMxu3QZXCYi8dyJ8Q4SYcCdzyByL/VzDwOjX/Zq+lEcJVvrTZ4/e3ZlmK5E0AjqvKvle3ak+XHl83jwiTOuP2CImuYpz4dXua60FjQNx3mFrkbwGqWE7GI2txCIp/4S9mPh2FGFvzGLYWkx7jc+TE1YFRUzP43hfHDo8e2qESqEheA8gDZPcAJuk3XPkPfUVBygCuZem2AkI3UC+wKhHZEHIu4H6biQP/A3jG6RvnEGvqdmsauARRjT5DZFV6RSk2cMSYwGvGqYrVigl0ydam4VcUPczLenr4ya7SMuadwHvQbw8a4DN1pt1+zciYpyx+hnETKOqevRhnlH46tfkUneZjb1pvRcmBJ4mGK+Q2FRZU1v8pdwSo1m6O5dCZfCZ/tVoETvy5gcKYPrdFZykjc8w55NFM+IsdQWemRQ9O/ShQdaL+YQu9OMGl0fOJCO3bqYViZwdLbVeAe+u6S5eo896jkutPZgcktwcEERnlJ/NsrSkCkfZdMZK5YFJddaUsm5F6nDdwnyG0XGD6H9HFCqwP388ZrkUOP5XJdAn8Mgd/wiUcPynVRsI9vJZWyuZGeaKmgj6XnTaADQrrRt/M3Hv6BZ7V0HTcI3Dil3QCOMnbBujOsldZC6uLJdPMju7Ea/BsaYnczIT8c+sZLhtwzuLKmVcJa2Wk6tiT1+IyxVXxl4RW2pizWPnNu8lx3VgN6J/KoY8cnWEY/LeRvBCCSrp4DNntw4n9YZkQoDSiz5MSG7LX1DjqWjAruR8SPLBw6BFQ86DbYcB+sM3CUfDSVsaV2VuNMFYBj61HZndvetMQ9tKf3CVRJZL65N2peC710BTqz48zAIdoorHC26dqPlY+XJQF0z3VX76CTQAdUaRYBdnI6SsNv7StAhYnlc++V5W3MsOtvaChcAr2+ULMa1HgNkvcDP89w17eZuM7W4a9ZKz8K+D1giosdxCjrYB9IZjE2/Rls6yS0Hv8Quxf/Zts7/pbBEWxmo+uPYCJF7GqZq3x+PkNIPIivYtJG76xKVVfIlrPzhebFJbrY7YCUoqiEoa1Qx0P+hvunuzLAbxPfgYw7zm+G57+8NrJ+iG5IYyquQpWsWIAvdd/ZegSZf3sGxsnQ3cl4ucgJ8Jk33uURnXulfeJBZRLHamrvq2QAbeXoP6uJJU5pX3M8JHM/p4sJoQjcl5g9hU5wgN/TfzR29eaFjahgYldvuieQSxmYATXcPMJIeAW9DeFEyCEonPEL6inusLvoMWlUf1eHPONoorUeEeBufmUtn4sSTQ5BgsLIJrLHMJ6Ssna+48AcPUThPPldI75UltZAep5Hqqvf5LZmOAL6JIBDsghnxOfsXNgL3eJ+RZ1P0UmEQx0MmZxhj6Y9ekYRBmP+Jnpoe37+2/Zky1NAd87VoX2xWKEtAteLlz3GEtKhbc9aS6axqflYQJHNFCQaLMu8mFLslX93vlNZOIOdXmupR5/RWauYVyJszGavbWusfABNyl7iQjVDDQGBzVinmPJJO1sUKjf9m60KALIFtnO+uGDsVa5yDrn4CukvqAlGJk/IL2A+KjgwlGcfkR7Ng9h0VEEhQBBK0DEQWPDoQK4FPUPMlWEqW3J2jTzEYrDBTD7T+UPL9MITqRWGdquIbQv4eVxY1Z4hoceAiLz1n33daS8sbNiD4blPQPypZVRfUU6b7ph8oHGP8jBVPsM+X56S+h+GHRjjRwYZENqdVfSxzIRapValgslL8X1OqRsG6eSwrDsfyiSktTXGvKUxwPwpSoKWqs/Q4QUZOvhuxM89mwhVFxt0WWgZHyhGl+fT/U5IBAJYXdzK4RnmPDj400mV0+5ZX/HVUlmb2dq6QVLa+HIdfZ3/lI8oCOiaAvVFXaBpwwjVXVn+QCH0kgVq+HdOCpu3NHpN+3+xhgdAQv/KpWuzm9ENfmIbIJTGD+hgqvuFo1AQ+Vir38KNwltfoq44sKAS9A2QJ4Ks4eFsp2hxptFKOJw7Kh7O/put8udsVOrqGlwjBimgjd0IRbO0Svi850YejI5Hfhu6ZIXQdCPjScx6BsNd3AOGNfyCGfVLMKkhGUOIJx9+JrC3AEQb6WNZMjUidxY6Rvfwm2rg7lpJCaNj7IttigYtE9CxzX2bde+c2imP3OUiRa9UuFT/W8oQGEs4mz314eJCKEes69rF14x7GwRNKfg+sshkRPpx88oPgoWqGG9xbej7SlFXqI9cBojMm94LpMfIf9JhTVaQBJhbfP1VRmvSmgmq1W1SLQQMFKgu9h5RMlrc46+qffCZFNUwWWpF6Ba6H5oK0o3pYECbJC2AmVmCl2Ql7yhQPMT98u7DVNnEjOvuETfgQVVmUXZdUX5GjkFEDpRvxwTlxkvqkD2VDu/axLWPeU7rb0RcgaA+dQ/EMwgZ79/aYQfqPPg2qd08Y+F4eqnLCmCAgkmk4hISX6wfHrQ95pB6RGUeQDUx9f0c+Pxl7Q+cgxHLGhlwBAHtI2sDiXrM77WHMeLtEweAKGGzb2P3mZBcdyewP2MG70GpDA2llaYWJMD3sPoFwDoPq9DoNb16+RjOGxKwzCLgsjnYNbmTwNO3rzjEObDPiGQFpXbZULV9iaead1g35WEjF4PKRVRgb1rc5LMH1Gqv7J2Kq0Y5j/S94fdQAIwXX5/bWGcbImY683fDbhUff6bJv0FMlegwkM0qKVJeR89NNVY5XTJ+1Ff9p+IJdZKuC2IW2ypLkt3Jds0RaVF1q4PZ3N3WdGgzSkNp5/A36Oo+St86axp8/vXA1lhsCZGJH997hPFxEyklD3pRZgvPQ5/TQAIXQ/vPWGgXyGrFIdKUUzZnHVOQAjyOw9+lioJjl1XWzwQ1lkxVDNw7C7i1UnAjb/lbAKhFx0KxBjWmA16+uX++5vwgeAFRG6QC1lfsG9uhB7UQhrCCxB5W8zPQ1bte0mcaKlPmZkA1liEfGviKbDVXxKrOteqV/dMXsIYQu3xZm2af3YVQQXQ28Fl9xITP7BhtkzhsjGq0JJO3Yw/SHD3y2E2IPXfdG0JJwRKxDM9tfjVhe7QCT6DuwnWxiO23fIgJuXnq1HNB8fozsi/QbG+oaq10TJr2xbLSa9mRCii9+J17iB4onPpmvdylTn3Bw8/m/colT6bBJrG0hPjhi311NmTxItaz6+e6kem3VP814sd3GBjpWJy06e/mZsjnXLk92p4YimFGhLCOcTHAV/FHo07rI8yoScG4QdtuKJ5TQl77eCqT8V7C815wg7+0jEiyseXRNFC9uNv25lj8B6o1TzpOHc4DDNygkjLlzR/fxtsXH8OinVxS+InpcEXeg/B7S8+s+AMW/Eryj/0Cd7lb+xrR4qVpt7grdlAWrChul+xEjAW4ssXT2NbfRjdJaZZy2uuJaLVL6zDMFacoHMDeaZjgoNIZ8NTJ9z1VyPok5icNNNruXsMakOoslcWx0ig9+NDyWeEhPwutAE26ZoSLFDEV6qpKcOKr6GeRsWjKONQnzySx5SkcJaaxGarHoBmsTky3tqz5dLcr7uNzD9zFDckjWwUZBw+jziSUXmr39bzxdH+chaPLYY6JkQzlVEf+c1nhujfjgeHmLkOLdrrHxvp/1NDZWZURL36+jd6S23pdFxvK5mvcMMb1f3bjSViOhsrD/e2kOWAB1BBloYGh2xH9/3+DAOoOR+hK6vSbPIqv37EWFk3O+4f3ZwXkF9J7M/0PMYH0is+qAbDz2xUpjqTsSAzed1X4AMdT41I+kOylSLvHI585A//GaBUjp7idt1056knvv6Yc6Y86l65lQEkTTEVZ5TWDq9UFdjbt9y120Xyecm+CCFZJ3KRhywwq3RvpUVMq6dF0vaFt+6ICao2NWuSMujw+20k9GOqODThBOJFda0fMdvyNzKJrpAiimUEGDoAMM9KD9yANA474DJtjd78l6dniN/fvPN74o5d/OYWArEO9HXkr9x6KOr+De9fZOc9mBIq88Vnihc9vKT34OFM+lP5c+DetpgXvO3nCxWN6HEhpnRSF+LgKH5054LofQ4i7W8csyh2V/SHPqtCxaZ5jLd5tUga1NN9GV+Cf5+fqsWQGiZkzo3ICaFI03MGCq9zCDf2OPen7s0nVAvE8enZqPxaiWXr5m9Wf2no98/7/R5NMx/8I+7khMacCJZ+i1ou00PJL5j7nSNTJC3kTP5mXgP/BWS3h/vp7pL8tTSJP55xBMwjuZwSK8pEpOc2OSLhJRxLsGeHIDPTv1SyjrSJF3Yd61j2sY+Em4MCRV+Aj7atFFodEAloD+SbAq+RfX95Xlhic1tX/82NyDfiebDfmB8S6DLRPgT71wvCFbRz9ZI6ferx+qR421ZtALwtK2LLc3bf6Psb2MDgDnNzzUFvbvUSxggpCC/vJNyGWB0zaDueEEXXZea7AXq2dQCbvFB0+jzvBdCeASLLFPaAJySlpjBWeN5LqLLyI3ecckqqyU1Mkzms8ktoKu4uyvNronXJM4GY2NgNj1ZUw7l3tES4r4YtPH4unnw1jV/dxUWLl12tVQLG/bGc/wEzGh9D/Yq+A1tRbxFkon/TuLq1P04CHdFyrq76rTV/BGhCLzA3an9uns3NceL7UqphV/UpKTs0gUkiRsz12Q4JCHlJLi32KB/FwjDUpDhjkyUq5Ktu0iYxX0cuVx1KiOaA8KLZb6+b8fb744OlRZ9I6RQj/uf4S4RStzf9CkMe8UQbADqqhwDa9W7JJKwb+S0mY0d33/8yrgpKgsFEAe2aDSglDaF2I8kNt1HgalwuFgzCN1BzB7b8vvxySroQCXAs+xukWbTpsf3iAztEyx3ZA2og79TzSJzpiFLAtqVVIS3uWzooNRhLPiWqfm8zcnjEy6H7wxQN5rhE/+TLMvpJkrvPc/KBw1/LjS6xnbzN5ZloRC+LJly3femZ5ntjyW+uK42a/rf+j6GXgrfh8zevAQ8Jar92uT4MFSmdXWsvTJGnHvC3X8x3Ezod6SSw4CMFh2kiIfgaGg5DR3J2gUj1gLwOjtAnpDMfJ0O3mN86+1+/QGCSYKPc33SD8jYkSnbbhHcTWCRgeZTtOWC0NqLHnFevh8TkWeWYSp26Ca/buPjeuMuWKRhY/kf+R13jPtVM7gOOtbAVqAz5UR39gb1nBx6Oz3J/rgRxgfDVZ2iqERdXvY0FHlqobe14u3FS0sfhK/MemrJGesIExZOrbA7xMR5g8asIbiV2EVhB/I3TrRN92B3eMVbA7mqXD+Tfr/iyzbx1wjXPdO7F6R6ePFbVJtONOIZALhDoQPqTkQkaeEpo6dH6CJe6Unvdm7w2Wnx6AE5otdnCXqHmFky+x4E6LW3ID2IU83VFEkVD4BSaJhl8oU46CIjxf1UDhO0lSBvGuQlJtkHHaUGex3ii1tGvPVWDKbfiG0QV/b080zUtp0Fhfm1069nBlHV4NtF8hJZDp5UZEfTL/nVhAua9ww0SwAEDqhR12DgwpYWLRVN/TX5Kqr2u8v6Yj6llwlIFm9HjxguRGl2qvWMc04hVycXOzc/VZEpyDoyNBc0UX7mG8kZyHdD96T9D29NRsvsOiq28nZWi/ZJClN0rXk474fFt7NeuR7TfMcWF8KojhkfXr6sMSoehs78yspeRzpZGHsqRuMVrjaiS5Mrn4E4j/Fv7nUmVFn0+DJCC5wSg1oBdNDOg5SeBR/NT4wTJ9SRyD8iFkRubNCuJa6kbm9Ad7NqOE8iFfvkSuW6nu3RYzEhbIW0PsCDnmpCfst/AV/F/y8++zlGW/u+BDuZkbPOlFBfgGMP08rPo9jJo7WsXV6jLhGL78fTFD/OB3ADS6P42J65phxRhAEBFND+TTzGN/iVUbuklb4HhjbNYb1zDI/mQeTKycgnhZtRi3evj3ThbgXWBEYOiiq2j1esp/HSpEDS+DVV9XJWJ18b+HB/PNmVpVpV5imm7QYkOOf68IgZPENULSXdNhR203NCfxy42H0F1a8oq361XnlgGyjxcjo9n9qXAP8BXERx6KB1fiWPgD5tLHftspuSwhi7qZ36Pt4sPf5kXWQ87jKYFlXIXVkmF8CzxgTaJTdrXteHBHiw+67Ql9YI4Q9g1/QJ+hoXQAdse6jFST9DVrD9U5dowuDn0rtRpllD1Dp6+0uonfq5n+fJLdpZXbcoy3WmqzVOjQq7/8OVXyMVYx1Xt1/XH8PWDrpgTH5L3/tKONSf/JhLZL+dyvl7cCRaWyIIVYUnbMWw7sbt1MqUBJHs4b5NuFJCMbRyy6fnfhxV2Iz5sJ3S2yLBhD8TwXPvCZdXw/n79jCOMdljSvrGJa3lRmoAOxfuP4a/X4yBGvD0i8f0eWVS4pJiCusBiAaCKvb+0uIFiizU//2btFDPtYwYxMTFZ0LeySnXbtb/92olKptSQ/B1PwCXR1/FWC23JzG5uHV9g+2eB+jV6YK3ucjZlZZpqzAIqOVmLX+JZSYjJvpM0PqPuPShFmQ7aN0cvr8xvxCejzaJ7rc1uT14+i1ChtR7c/ZNpL2+K+76hEUPmU1B0bJEkjx9KeE3aw+RCZf6by9v3zwXHrmHBA1trL8Qyru36onhETbBwvlR8mHfaT37oen8dclMzvTUcGgr6PjmuaIaCn7b/JWBCmmNgX8cVpaeU4pt2wF+1DjxVW5HF7OpxfG+u4o3oyVm/4XWbBQKGK06apqFNZDI8IzIzmjOq9zgbwzxR48h38wS9ty82+TmpRJwEXrjoLKKyvGyGOtPkDXEllxIHFIm+9eW65qvOu0C0ofORmXnUD30XeriHOSRyJca8cF+cbNwKmT7dKIxdknvpUx4ImnkTPPSCzMX+8/VJxdJN75tzAjFrrsJjlse2E1V3SkseBrAJ6ptlyLtp5qMHhAkMGntN79/AsRfQ8BtXMr6KiApi2Jjg1F2VkvRSpmQMnrfOGBTas6r6eQiyjSAHPjX7pUTAXbkTi7Iaw8Mv0Jbi1sU3FoWj+Oyhn+EDmCgH9pE5Cc01R6CWrSeUcvwjwYvhBeJBeiYbrnBRhK+qg4FPInJzh4KICvC6AtyMGtSyIG/CUlc+IENbhp/jxcUGOwx4xHGqYyuYZozLn6nEz2dtASX6EK7wsubfkKtF13BJ7/obj/4VDm0v9A+ByoK7B6i7kPL6+5Ld4y9N3h7sKETs9yS4Dkg/416IrKkfKsfR/J9CwRjeFqiSZhOndGladYVsiQtpgV0GmKhhiz6yyNC0jjH9N1C/JoVjcmF9cdJZSMm0exRX/FI2a8dcnuNBWK80moi/qfpIo6DAMvzcgw69nBYgwQVuOB8SM87naOtz6FGDytLVDJlFhhzYvsx+Oa0ohO5HS6Wu6V+2lHW2oNzUGnESKn+lUYnpKvvDdmVvcAHBX5jFyCzrWXW0U2nvXxnMgbYBcoZnvuxP0k8SUaoAaYn5PnW4xv9UtJEy+9aPHEqpP1fIEEr52ohe5pnhvUgcQnuUKD55yVY05W848DG8/F5LzaTzNv+p68u6+g+NPoUuuuR6B8YX6p1CdDYgr7nXpzzbs/G7u3SZ4neMP0IQRziHvqy9dp/YGWGkwB8RqS+Q68Dtz+uOTg2GvxJTFis4z7lIekjx8pewNwS4DiKKbdTPESpYWFVwK8Lb87hp/KgqKkydfMxUMSyo2OVEVOgVcWwZx/5wDzU9KkVqrfUiHs1o0UEKTgI31u5Z0nDQFLMpvnpUCk9wg7UjJPTT10XeFaqYYbWczBPv9GtoQ+mz4ky0pdsJMMUDDKeytLtjIqXpKkuUYNZbjHuNnehPRlF/5ZJOTxssOhgkrEF5T5zj3Xif8G+kHqky4ftihntQ7mewBUioacwx0kLB9wmgTPflvEpecG6/+Ent+q4wm0vhPiLsKhieteR+CFFtO7uaXxhSjAzIzEHR9td1l4F4Dwjm0d4mT0Udg+fWLVRWtgfqsFsf46apb1gYTBuf23Q6OCToDer5XxpCLWkRvANPkOCwRdtVRc7sALbvoVDt5mb7/nYAftGuO1aKwTc7e09Nq1UOfa4MeTFf9LcKqfElaVRldDtMc2IOraisu536I5ykjZ1wM0miZjQVdAuzqD/uPovPFaKUmFPwo/zxKaDgcolCPoZOp4bYn6AzyQAPr33yZ/SWxyUDVg3yU4oGf4qKasIq/QLMZYSlPH0GnaxavzHbOjq5vRm52DmqR/7yPfYBFFFLv4HeqEdELPGdgWcumVmNShCHQwIGTCgpITo759sU2Stx+2TLIxWW3OSUT1VuyoIQ8O1k/Wun6aBW8eRCnBItnnbM7hx7cbaWUUCQy/ZRWpVBV2z29PDFDaXtp9HlY6A/SdcMtxVdTlJjo+0Vva/szibGl6Mv04IyxlXWx2gHSZC4oAbF37r/755JLY77cgsNoDnQE7cBIw6ksB3yXKD3T099OU7++3XWT2xjoJTEntfqj8akLQZKuFP7+qrBwWXas6tISFCLjTDolHItkezwC+6pvbSMtZZV8YzmIAGFf+LQ378z1aJJ40t95VNXl1ew2bC0JPf7GTCv11v0jVh/oe1k8eYHfA43dCs0bH6lhvNQ5l4u1A/hYSxK7p1KSgTdknCjU93PdYmKwW2K+br9a/fsg6XIUQlBMgdmuhV/jWYjD78mefebYiMIU/4c/MLiYp0oZofYZD9QxtZn8JaGL5OM4qkggpZiMvExyhc0Sc2WXtQ5eCcZDYAL2aiDHMksISM3XKPxAaaUQoovYwzFLxP79MRfidsvQSvJ2U8dtec0jQO4kGHE/6xmregG3091Ws1Na4uBToyv3/5ZjJUqtRHpOy6VsQhVQfgd7s37u9o+ODKdm13+LH1PTqD4QTukRWoSz6mAqEcqhdQyKTXB+8an6Au/3ds5TxJxDa/135W0suSRzJUoCSxVQ/K+Un4Xs9BCoRT4ACqOcgf+VDwtjDKiEmUstG1J0CgK0a5AyC0fqc+8Ay4CMgXYJgVRVhZXDTQopEqzGGlq/8uztnEKx8LYothAeEsfRlb/jSgpiwHSueB1jIKEXBN6OWOFA50swRQar36cxfMc1/1xVCN+dCP8wra/9M7ZYF5RQn2ECqUV5TQ0i8xwk+0yt3z75CGEVuiaqZj+Xr/PeOQuQ5U8fVgIjPkin0gVXq08XCdYz/CViz/A2RFzxsOpubcjjO9FdZYQ8nHCrEnHeKRitl0STanyJpX5PybEm4jPg+BjpzMKTXR4UkmYkzSp7yFCEksxFec2osxR6NIlQCwBz117QzBA3Y5hkniRktzrdwCRBr/4iTZySD91/MXOunAHj+KITqEd2CsbVZv3qbSPTH5wJAr7w5XJb2il4JZpjxaGWK8qU6Dhlh5xeeLE0qxBc8ouLum5fVUFF749eYwg7PE6MKWvvXQDCQj2r3UPaFdwTYnfK6dbENhc77n1TBHpBwSApMfS1zvl0ahf55goy4Wfa9a6f10RkvN9/270ALyMNI4QmHf9/W5bTKHLS/7f3+0IewhsLW6GQxrg4usSMJKQ1AYxtD5lecodu19D7wUc9kYq/gbv3v3zPHkT8jruCtNc1JOUcId6pt13t+06Vanub1OkMM8zk6EEgUJDTfwcIwMGq0iVa/u1tjee6cowJzYs0zSfrfR69MSPdHCPdq+gWqofTb5/iYKaJdz5xguiv7FnRjdYSSm7VbSBmpCONKL8D4rCplAHpw1Q3F+NVgTwX8/Tix5f5CPqf/09RCnMki64XMUxQEihnLC0yxn3rxDE82NvxGVK6Kuuw/qp52LSEN3s4Bo7S/bGc7ETmUD7flt18bafrWFdq/ltwXuAu+mIEqnT87QgJfQyITbyR5Cg/yFSmzjgXnH9l0CM1+Oqa/z34wiEtbyRo8BZL7EoeY8t4tQJOscIDVQzzivEFEEPtCdQ0DyS+LeeOpcUMIxiYH6iHZyhXd5AnIN17jgudeYrqdu3m36dPUlqWCm6lg1RWztZXJBEd7j0y4rRpTrHH6KLqCp1rIeWERECi2crxv/oFn3+rVTBJNRF4icgk3FwOQKPewNV97sWp8cICbYvvPgsipBCwOPLY9JV24DThVJQKJZZO0jUnV9oEamXmwm2O2RgtxcLtX3rXmOPMOtSD3eAm3Uo6E9KDmGnq76BKIPid3P3iFYUQsGB53t/0LMhqtqj76nR+zxETdNHjYrBgOyDoOpWWcRP2zeJ/ntNTRDL2B19zdG+1U3WS63P1vuc1EFpMzCm+3vj4MJYWYI9CiKryMtK4UulyG2qMU102NBf+AI18jOi0KynJOuLTwH+xRoq1NcDTpkj+CuNr6BSFAsAI0Jz3vIuVdpJ4iX3SRsU/C54iTP5QSo74b7IX9H3Y2D4YeFHb8Egeur2RgV+k8JAYfApdPXrZhLzBypoZGa0dWG/a2dIqR5SxGuxa6pkYSkc93KCOO0qE59q7hLkLO9jn/0YS5czxJRtme59O0fXf9dP+WvvoKcucoqmcf5wkHgbL8rH/iwF3SN0kNJdWZf2X4O3rcb+354x/mbU5YXww90Z2aP77BoAu+qyQENHPzZWyGEyf1FBKtT5fBgHYACk3e9/hQAnh271p/oQBjuWdWqVUr2hwmdfT5ku0iBVulFK7UXv7xZ6blzmzfADF6oYIMQQ3pA6oW+ETcDAY5GEKjaYP+q1i2JQf868Y1B35KyfEdtc4yXguXN3EOeIHPNyeiR9txcMU+/v3uG0mhWtJE1ypK2MFXm/W2uHFEslHG5pMcLIQacdbH+h+dHEOU4qxPzMkaqSu12hzor+7vkK3nhrv3Zor2x3gJ8mauPVWDO9x0VDpJ6U1Z25rhU2IWX2y13CNKgqjlHn8OMnGmUn/wfnNMdT1af4Kc08KTRg6YtDbD+PeIQR6gvtp88VO6zkl8XkfZmjj2wTt99uVIX4y0NZXSIkxYzvF6vE/I0rXVoHyzp5xDJm/iFgs4vvBu9G3zDwyCi21JHo0PXUIcPOMeioK5ToEWlB8/b0LZf5IThZBwq/J7QJm2Nc23HJ0jP70Yf0DAy2wr+4r2laWCe+21Hz4p9z0dC96jjZGxoDK/0rspePpWx94J6yXCsudRcY0VBgbBFKA800cVs/9pO3uuoMxtHc19w3U7TAjnAEMSqZv6tcFi8COhVxfTF/XmbZJjMErz31jv7KWXp290KYm/2rBfLvHrX7KbVGRA4QctlG4UuAI6jkVxs/2nGiGG4hkV2zNN0lvcTUsPv+1J80Xv3eX2LL2La67k4BSqPKQ0oWuT9ZoIImw3Hvv4JQDX2bqScCR71Rkj62ekt7QJhxJbt/Q9Fr0mClj4jl8T4K2sluOqU+aEKk2tSNAvWJmB9HDks4wOD5DYnXJ37P/Pxvs3nGrpEaD3OTiR82i11SZmZu53OR0jnHrg2RlnDyYRfRLsltRYPntWdRCgNXhPhgFgCq6O5am/QbtCf/tdpVv1ET21RJg9UgG9O9DOf+pGIHZveBmF37qjrA9Ja51/KUUDB3SfXGCekmmttknjPh45p7yac+eGoSnzVVYyZqZsz9OQxm8fVTS1S1XcTT4kquA9MNbgeVt7XEzptGJ3VG6rjYKTm7JGogqw77ynJ+EHXS3Xf+owWQiDyhv5kgqwxmDLJIfrvzqZX4LTqDpF66MZhh2VoB3GaBbywkeQr0demfEcnoAHYGFH4fOXtWeb1JYwklDv0SNKRCT2B8ycE5auvb/9SOcsZpy6gvLqxOF5lLcofYWKtF2j9nAGHzlaDSYN9pnsH9eCkd7V2cmDM+dip/6xbJzq4kAX90AA/c89pJz+Ci7K+yoaPdwOrD+CapXVXwXlBnbwHpQ/oBVyXeAoYnZnjWr18s6D0JIqAXqGn9bVAdjhRNxwnNBKyAR4PAOFL4/omUc23hvIyOXPi/sGlxTWFHr7YPiyPhpzjJTRWVj8a5hCCqqxjuH3WvQfuM1vLgaJHtDaQFQpMHtDDj+WW8m4dV3HOdk6PgAUQ8xZRNh7bdl0l1EBOn2YIupYSCeEQIVE45PxfaBCsqu+MTOc37r2tI4xIrY+9YbDfTcjb76nqsDRn5HHcQ7ttxjqPZKgMiZqFmZKhYMbR6+/qtXqogTfJYjkfnvOf5eSDoZkGlXBq9JKoEShV81hCfVrzni5kcc9bSjVNM4nTgQtDlS9q3v5swgHQAgUiWX0sv3OisSab+DexXblSUGvf0+Qmk6BZmAGNzqCY6Q90+wue3tp+laEwM170Yy899qwmjfd9RKwgIBhWTM+ch8gydsTq/8PV1kFlvS1pGGrRW3UMA+9oDGO7GfnCSNHo+6Grm7f8BS3+pDbwPneHcOaQt0uHkWYX1pn0WODf4CpLCBBPRP15YLTVb8Kg9uLmvdH4sLwtJK00AUFDettX+s6QYL7YGLLkpAVRUwQ1sCRGxsRwe+ZIQfI1MbjmJPkZgFihSR7/8Py2A48tHJzxoffAUq915gypt7pDyo28766SRC1nZskJcUDgEI4HMoRsqDvNylYthrIcS/HFeaMAjy5GTBG1Obx2Znx/cLOeHtt4f3YYHcFeONPy6cNTLL4yqnze8fciB29pfrPVgXDy76/el6ym30kRuEKRGafgfuuqZT9UQ1Ll95lvwXZ9yDBGec8lK79/S+CVPTPv3fvmGQP5InoGhGmP4kz8vS4HxY2tkvN5K5/rNUMSXSlJN+nX3Jdwy+/sWS30wk9TYdhkUy7HB9m0QmzOZibAJEuvHzOIm2HHOvx5G7Fg5OYOpT0F2Q/bBsmzd74sw3PBfE/aaCxkoQsf6XsXwuf+aVXNRO+x77y8+5uWof4goT0wLtm/GaZB2cMhk0qKfrcp2tAUWJI8EkvzNBe5cJmkhr+AVNnJ82TIiCLA2owGh4LZ5GEABYSQKA2+xxc0WKNgtNG4g02oAoVtYWZOzox7Ay6Xxh3ZrTP2sxZZW3NRv2zqqs5GEZtfwReGSFf+iqo9GeEtny4RZeaembSyt/mtdbpP5890cOK60ScWBFGBlwt+mbBVO1kiPlliglwOfj1PUe9DeBDw1a4jMzfwp1sLGWAnNVLzSxQfQoRi+K11crz6jiSkWdb1R6SVD+OivHAWiM+vP3+Tww/Kmypr8PUAeK5h6M8wH5du54iksi4XMIB2WrECa2eH6rmuDQEHBx8MmmTZey1CtI1rr56v3wZIE/F0MsJNEJ1T3GQTW+rKFSs3sa8EycZ37Io1D07EyKJzmdI/k83jkUZJQWoK2AjMNfbK6A+ceRKzLjVwqPMxEQlv8qMQYsKtPfSo2yw8JOb8AcV6UDA/FFrj53XfVl94+bcKCxNz6DH+d5JUdk1wL9aIgAm1ilxHVPwiesLMbL7AhzF3rJRVddNoAZITgiuotzKxbiG1dJaCsXLU1cRyit6BF57JV4H7gl4L7W+UGc0qZr0noyJCYYz6us8Hkli2rtDBykWPKDGZbpKs6uRINRnFl7SxigfWnFLA5dsiFt7nh0BwFGiLm31gd2YiPnePnkvXeoIcZhKurT8y/rzpsfj0nV0kxjfGfsrrKeokqzpN1xci7/YJwGe5b++JyZUrG+DVSiRE4opGdO+EJIIREogMbtN3aHSII8zOoCyt0rvc350Qn0RB3fuklOwaNnZLGBwLzNyUfJPLFs2Vaq1CJ7gs+COS/5cWBmQcG8z5YHRxzWjduWkeTNgK6H3Q3g98cG0ck3mAZG23YUuwnx6p97bqaI4u9S5bXBKZQCf1S3XBImSUV3R27rcaZKh7+Nz2mxdEG2IeancxuyL0NRu6wyqybizHDzwcYyMSPXF5M20qqgvdSS3TEQMGb9xVmn5FRZJ1mJm96uA9k4ncAy//MCVbKTPkYr0lFyAV6GMHnyvkqeeop45u8guhSsUm7pEJ4QdMxRIX0t3WVoaVGJmr/3Kptrfou+h1X8DvMwewuPILekHyz74vNgS95KFzGZxvSyAqcL9IzvJjnjxt6L938HGXlVLuEHov3/K1Kqmu43myMBA2cVQpSG6whK6f7tZOWOkSmns2/Nmh+QL8YA4QAvnU4WQzCZfz+zBw1FAbkD6fQkAtTfl9AgWTszCf6pbsaPlaVRRsjrnXExKlfLqKKuwH+dFa8fKNcEqeyOIBIa9ULZfmKUY3/zQlJFtBvBaki7Hr5YO90fmfu6RLagBj8qZip0VFFiG+FTWxmu6h3jNeiuHOxTJ6dl7TYuJxIIygB+/UlsbIuDsBMH0+Qb2h3yuxqeYxxpUdo7HCvjHeydK6ya8MMJ8paVVwszxsDWP4DkP3wea9Nr+osHAqDAEqaHLDAZKS407eR0xHDO590Z8kYj5UwN5J/On5BWUXwJUdG/Eta0aR3P2DoCovvlHlg6fW3/cnYbd8ykpzhX58MHAEDOvZf7j5jyBZfidaw2o86SM7x8VXFDwFtpfkj0auPTdKMOaZb8ZwCSspyxcMu5du3pPYUQ8pwPdfa/wRCGNAXuZvDfnZc9zoo/Y2TH4rExzLAtmTal5OuwLJNUSbSOQ8e3FOp9LI5rHVlGaFP+QFMoWQVv/qQrw/LDgxFBHOYNl+V5Y9wR0BWzdVkmaD3tTo5JbRqMd7zNoB/SBdKO3srOtkiln0aBLW6K0mtb3x44aHgkNPDrsJ/e6uR75fXtDQUsqJoxZNZl2dy1yABieJQEw3MDjKpTBUCuS49KKNZjJpQuL/kvBWMYurPk+QwmOWOsC49Nfa6SgYd5+DFK+oDWxt0jhPIZu3ofOZf00h+LP++AD8supE8UsdhWel9d+u6psvfIkttq6ZK2Dkhx7idsXwT+qqXNfbg9BpOLqqh6LdIfng9QNuchVwoCBDr8oA5bbQufNoQ0byDSWXv8cd6Lk4vmkq7IwauSv/aci1yhxitxTfgET4wH0xaPB/s1ipR4HvZrETfz1dFKwIUNd52nis8RX6gw/d5r+HqmfmSPx5oVn6eY3gUeNsybb+zqnFLx3zzUhZeAbXG2XIu7chanwXoSXjM4E0Z4I0vnLai4n75KShqbxZcfRnai3xVF84NawQnKIrz6OpQfB/kQ1Je6mFH4/JsLuMeOQ+4YJefXQ8J+iUOfOPbROtQH3nAyaVTQmPv0Z94wUF7SSdyauDk9EHB2sOxnU+/JVpmQuBUHwV4o7soLN6j7FaoTFXNX3vOqcl35vWC2USmyX4pSlW8qapVAurbq9TBsLlt2m7NeTqXJ0FWNPnQnho+mQLMizr7ftqmW7Cx0ALobfwbXpFXAXKrqolkCssrrUG6QwIiUhvYfHrKxaEVcDFLn3uR4mEVW1VN3HkanJQrqBDlGnbhF/4sjQ92k3Kzeus2OJoPCa//+6G49TVJYp1IKkJZuiRocmZSgruJ2kMlzmvHql1JEMKMmUnuLmLvdPO0OJCQrJq3D5fCGO/WOUk2lxrKg+maC12UEpk8YRL3EFnKbW74c8vTNOWxv7OTXpa6edWqitV3uewUW8+0fI1AhYBf7o9sp39//aooNFvgXjrP8+160Ml2S3uLFtq4wiIVBTulc7Bmy2TmeFs0+pbdxajep9jh/LbYuyekp6g32gWZor7PQfvHwUiY9sIVllg5G0RkSvuwAXy8wIFU/Q+s87OH/s2OF4rvXoc/xqpRoAZ1OQKIwN88zyR1ttnmj8XieOvaAbkWG1B/LvbloWeTG4ueK2LJwL/QHrgp9bjnn5eyimrks4fvK2PgvkqV5RZ431Lx7iiWnfzz+1sClMU+74tEMkvnx3ydWFDxEKW1PT8mv8/oF66ky5lyjjCV/ayx9nQG3GPvUh1Cqvxu9W60d/T6xrnsjWiIB/o8Nf203fxaNUERUEtipGPrpv7rmvLnxY0DG3VV8SjV+xDGEN/Bx+9X9Fn+Uo1W9IG2C/5VzIhpugFiHVtMqMiE0lDFQTGINbl9AN9ToM3HrPDSV5Mx2MvOJX9s929UWbg6Y4wFgujfpC6+9MPUjwJdwDeP0EVRBhUSg/lGNK+Jbv9XqaBKhxVN5B6lvJO7rH+xseNHeQBXvoz0kUvoledoVrCtWycv7uGRZeYixnh1ctBRVoRC04nf6DH9TWvp/Yo9o7PnBjekNsjRPNj7tNci4lfV5i6ywTqMmLoywx/y9Tf1OE37cffIpTsfLsQ8Lqz9urITegM31LmCv/m5Axrrw8J95+Nr/WRhjQXbp2Lsb7A5jiZq8ZdSpsPmOyrBi3wCo3UO3xJ4RLGtuSyQC1o7/OsyNwLrST4aEf7bbAZdp/t3cM4BaFpIWi2DfcZ68Zak/KF7IdjqFwTKTPY8LgDdmIPNPVz9JyamuBlvTMDCwciQs3ZRET71+GQpzs0SAb6MRzhuWv9IebBzv237WeXHrKMGN+I1LzJo8V+m/n7QJTMDl5C1ydFEknpNTpNYhxMzfh5Vu0w/JdiZ6agqQwOVo/e9FMStUfNsfwCbh6UscJucC7ID6V8Oq4qO7pSxYPzhOxBDBLZ6wasZzNc89SC391fjy/ps8o3NkPYB8Lin8HD5162hza6zFIXIWTXAGbsUdfr9fm2GciB1BrFhDhgkdm6JVZv8AEkmg0Bu5vxu1N3DAhu7zI7Uxc78+BOpo44dZ3f0cIfURlIDrpzi/5bZHxKq+n/Ny08DhfdrJkLuMHZQB6x7Zo4PPHHFrVF0mrn+aFZWRFXe0RX4AiysZjn4rPCKQoeuWpVqstehUh73Ih9yoANG9QeunwZpMti5zJQeR7fEhjlmYNBfJwi5bPPoJXiItJPXjXz+euuxUYzaXeHNHEG0YWKTCcVDQ0V/5TN1l/jygHQT72N/gvHFyVs0SHB1U7Kn+onN0FQVZrJfeDuUl908J7uhWaUViCLIyVnhLLdVqBCl6QSl/DAFqTfNPY/vazj4+JY8gWdY2C+5lWru6ez/Uow2/RCVz/ALbdfAvp08utHRJuPjKMrSLMq6yxd2/GSyY+InCImi43x/K72kcpD2dVsRLoB3NfsVlM3mbZM0QniWwl3CRUq5t79BScKDoIRbueGl4VDslQ1I+TtngKyL+dV3CYgONam5rpt/eW2HkNkQrGP1Pnpmqrg5pcUJSuo8o4ai1AtzgqIvrdJ1UcnVABxjC/Mf4N+8JsZ0rQ0SrLu8NHWFks+gzL+uw8ku97ggiUlfX6Bm8iXh4etVzOMfRhXO87OR50cFIHhCpeGmTIV56FaKutc8gILKZh28cpFI5jq2xx5uFR/W88X890I6u6d6bbnU/W+zq4UVDh1W9vmyytqM9MlUPWPtPWgeJ9wb3JMRTMW9Tw7ImpWkGCzRRcqoa6HBfcy/cUSu8TOd87chjTL8mPTMEYfasJkgGn/1nwCCGS75yXjbjdttBy1xze5v2TxLH7Z2n4PfbuStjjwdbZ1cJ5P2d3NN5wdc2X4wFuRHuzerH4xQ01IYeezIZSr/cbgee32JfxG6ZbXPpKWknZilzXyOZw6GBLhjtUEE0lVgTZMirDMy9eu6QQJUzu/dUEYdoIl+MnmY/tjJ4RChhIV1gt8rsZZAmlWP0pHrML+Y4pfu7EdjlhidRU4UGP8L5ddHU2Q8177T0SG+VVXuGJ/juOVi/xIUP1nJ7jzxyBerPFwgbzyIXXJZnxrnfrR+IG39LU+BIDOu1aCaQSh/IsI8Crg5pHBzDEXv99SrC0BIMesqD5xXCZIlP00Z6nl22OCwIsc7++sU92/K+a16e6aKSfLZhaD/J/jNkOelzmJdvYpk96zxqHS8yCB3cfLcOx8+9YGeRL4exhI94TKWKI2OoMhYndilQXWLwXMJbRkYiJKtebb+zjFfJ8X6jUjx+IbpgN+ogHDPJtzjOSkDCruYHQIretijHRNK1F3PeRGMhsd01yiw4gecgwoI2t8VetIx8XJk/gt9Fpy1YvZTJBx6Gkq3ckXjTtdy65sv0BTjyU2tPAIpeCuz59eWZx6ztzIrFkPn9dphTFzvjlGtdusSJeV7D6jqspuR3YEyZidpqUD5w9EbHJ0/RTbDvfpmKBSGUjjkaYntbd2yt3kgSqCIzoKhmmqz2qrRmAr6wQb63rVadAtV2NWJlywNLnfx9wbtdvIvOfgUsqQXN6x3N04r4zjuX5qVmvOlAgGu30Wfo/vs1DmdWQbwMiI/3zo7nmHDFOOCFsOfDv24CNX+GCgDDqTJ3D3VPjdTo/nwuk6H5NKhSPOp5Fij5LqOXTAHyMof1xaWqGMQaIVJNkP5X8QHNpFNCJuk+E/VqZlmzUdpe5obK5Q84up1gfY0EanrPuwHK7kbSkKcEjO/HAkd86xluitLW3LCvnUq2B3rxRf876uTH+LS4N/pM3qpA7GFf/6N4RYe7G+6XERiv29TTEteOXeBa0Hi8VysGO0YUEL81YLCcRX9D3JVSY+wxEDk/WxfjuEWsg1maJd+reIlCb6YbVxUZSyTrHVImIVQZV+3Vms5akBpvh/IN8tkqcih8GAV4SPgSoFy7P8yd11ZjipBdjXzjzefeCO8hz+sEN4Iu/ohVf1mDdOnT7dUhVCSGRlxb2QYIS/cvXMuhenbB7nUrzrK8pxQiNco4Du09+u3nHS3zgcyuDeFhZD2agro/LW2/Ttd9qoEkjo2h1+KAHuMao/SfK6xCUB4kDPLgbZfQqlLsVm3z9D/GgpDNZP0Cor3+rZQD/I8c55T/25XAj8F27xCHlT4Fm8bfYmz/jniB9Gsr0LR2l3/xtwz8atCy2/rV5FOido9EYfcaX2xS7wd3nOvYZJGxLyAYDqWybWh7Tgnh0VRgbFfniQHdAqiP6wJCl7XOzocx+94UjXmPoE4PYiKcqmUT3A7GUUj7viOaXKFkBQguj7kgNdzH9uYzuDOdRlt40/FyMMzpqYGqT4v8ZINH6/Etj0z9b0BL66yJT/whLdyxIWXiWevWXuYrD8zdpG3YnLFHuWbYFz15rd18l5AtXXJS2KPJjNTGXTDMsSE86LDBWdUIEwLR+Lx5bO+1WVvoEZ+UfDA3ZOk7HDFE5NyAvmrJ2N78zy+UJrgKGkLyzyi/gzkMpqI9RmY+UsNOJjzF7LTtfv+Fc2huf35EBMegeLhHcxgvZT2d9753gitSyjR3RsPnTQFIVqiMWhFaZFE25bneZxCS1vSqSbaQXmSU1bYAj84YNJxEImMnBvF8uLsvbJwrUwWjhujfvhtb0cEcOqmguzZzRZ1XIwmczPHnzfXHt6wLrLsX3ekD6NUEh/itsUjEPD0V5Z/Zgeb4WEoKZzgQc8Nn+HajVYF1otI7GDihrYWb9yRpnxvp+nhJUF0idQFKamIfZh0DddZpvIZNx3NDuKqNsToLyCfifVrfj5FWVeipi0YQmP/KjpU8OUAx2bRR6m+FZW0rHiJRVyC9L0o1Eq97yor91q4vrjr8irozQdqvBnhq9E5GHhlFSDvpXgFIF0QbHVLm2+/74wBexPkrwPI+XkN4uy/6eOoT1iSqjkvIMhr1/1bwO7slOQ959zQO1fL1WPJoIFJiQyiJ1Kxfn/ZrcnoLYJzjf67FQmV0T4k84EO+yroWRpqU1ZT1708ejhiLX0n5iebA+WhPly4Jm4n9VU3rLzocxuTi1Hd2VcegRK9rEuvFAEXc8bAOp4uerfmlmRN3whNzi4qlviFCvu4BMt+fiZNffWUhcRNfXeEyFhqqpPZ+wVvr1PVE2/6xDwtDNxypXOJN42Cl8dD3TlceDUu0XLzgqqlKOwH0cqzxs3Oa7VF/5tm2qMDJ01YFVs0sW87NhRrXjuPk3zfdqkyvj9nZ1eYkgFH/z0rbzNYMIO77kfsbmjslQvmzoG0M1TMd0CKRLngunmvjMtFfTllmbaqIzV7TYTjwA91N70m219f+uW0i2UfzNYAP6L7sRgx4gW41mBrC8kW80gti4AaZNy5xUH1bTR394Cx5dKeCWYatjaDZCsSF7fUykZtjrV6YZ2zKUfv8nGo3DNwiudfvQsAw9zt4Yr8cHxn9Runj3UrYBDZiDlvGFd1BkTr7YZVT9d1sQ8Zf6s1HOgE3B4LLSpNocOxdxUlhawbDmJnhwUkn4uCHLAGj/VH9Iv0/0Iu0AvsVhEfD5MJU2PYMfm6SNoEkCemfUSlqf2Q3J8LyNeSlXJ/UlrUmc7kH3hlfiXkQ58xqrn/8DBKQO+HJk/rHIxWv3+sJFIhNbMrSiTeJxj1Vn7F9ODuQll9uFzfafG16EJzVcH7brhqWIze7My7bJOAjw5ATTFfYIeS1bySvStlFZFO1d+v+IGBwC1s3Bq2xCJcZAbHvXy/2L13m+1ExQIjcyT0VqfSh1+3hb8hr/nYHaGl/VuVClH0rL22iQEVYIZPUJN11mM4sNiTWIItAiI6slzvWDo0SpVXtnuSgtfovi3iJbITJkDIxEovUpXaB0fwwDcqnKM2fEKhgdhLt8iPwt8hJtL2nYhZ2ecW2b+AEu65VryP422npSiekat/NSMyTevzcvZH9hTlSykZq2Z7saEURW5m++zbCWqJAFDZFU6TPGY5kFw/R5j10kSOfWwtEQAXwdIYxNIC/Zgwtj/sQCS5DuxfrgptIroi+sKUdz0TInTufLkC7xY2rhZZ+PKd7LVHbPKaqWLt80EQMh9TY6Pxz3jYsEI8KtXUP2VL8bUVfkc44qaNWzI9sg4+KyeBmOPyo6urU57XeCQqRt6oV8nv3g+euZV/pnFsZRFd7omLvEEI09dZeAJgFWWZ3LYRFdV2f6Svvv5aw2ZCOB9LBkTHzGv1FD3blYQUitf9Dt4fM6anj1wJZIfB6yljYqbGxqqbo6adG1qcQecal0CJPqVFWieg8wo5ptBC0Y7ne7qPe0QZmaKpMfxuDtxK+s2OyVo+P11vYC+DkXsEokaRj0Tu7SNTffLYdLlgtJJe2aVfbms8QrNHWCC4/9ergWUtBqSSIznO2e9o9SzP0IdD2TR6e6F6/Ko6r1N/zZm6dMJm31K3PqEDA48qr4VOntPajmJpHSm3U/UqDkQZf2XnHZb8Mk7bcKgAB73beAq8gcucdZ+51KkV2rRnKmKqOoO6VA06Uz2U97f2kSMsbvCkDbEbG9uFDwDrkAe5Dzj1LdKSUqcNiIEIE/4Lt6mVVU7TAZTalVHrLRMA/VX7hq2RNvRqt1hEEppvhzO0ah+bywp2cUCqkInI/tH6oZYBUBKIWdU9WLAddSLui4Js0PezAp2ULdK9fB7mpU4PEtvdQPfmgVtRFatXhKY8aJqbdkwYOFFv/+uE5xD3oBefyG/PrfzGZzuRRH0/nD8QxK0SEtXxVEV0Z8h5Fz2WciPZN/mp361B3NTGw78yd6iLUQI74h8GKpgG6/toXeMXddP5l0Qg/Zeb8RUyWO5y5c7Y0AmHIPyVagHYlgux+0Q/Fa/2teGZtmxCqj+9JgrzdQekKrOi9DAvn41/WakuLvwAw6q4EG/xJcFcv9YahswFWckl8JKPRnvGZN4s0qCOTJTO0JfhSYophl+LQWqhrNOeusqzvQ57jIL6sp0ccFy1nuOCZsjk4y+H//pMDzQm436TpJDP1LnXySrhh9opWEt2Qn6LaGFsZ8ZhRHpQPM+f3VAC/PXCyjmdUawqk2c/IX5xPFvCu/xQYj5vDSnIgCFtWnaYjw3Htt35EYuRXeMY6TNDwgbjLUlXnVz5kPio1wTFeb/hfKGQqiStaV+GuAI4Kg45ME+e5Xe7MlN05mUXfvgmUeVVisJ7SslD96j0YRfpfk77MKFeiGC524HsQgkmgjANbatXfKzLmJxeCw51r5S+vGgBzmiKJHzegcZoFgsJTelxVcx2DmqtO2SgIC89jEhT8Yn8e5UPLAwl1X0UO66eqvmF4KxkjOCXudIrKTCFS+loSZ7ZhZ2604x+LrO6nBZY5u9PB7KLyJ+U1QJV2x0k5OtB3qhrdqmIOLf6Gtfp4SV4DF0ZJnB9PSrHXrDjqmsbYx15ObQfrnX80aHiyxXGG/eOAHz9MZXD/ULAvhlWKyO9zLXI0Npob+xRbOvdUqxKWgXcD/L7ZUV7Kc5FC47DXnk/9H1if2YeaPudaQqFLhyJPvvNn/taCrS3cMTnd9qV4qUyFESXQK5HqeqRr4SuBsVHSkUhmasoKSCtNWlmDhxQoNTuVuDAMwi4QvPRKNqNL/LOiIuEgOcR/1kkJ1UTaZl34HXb1X5ZcDmWE6yrNdPp3pIocLV8i23DsME2Er+yU8JDk8PLAQpTxmKAxOgOmPM9n3lptGzrl4+mtsKcqvMgLeE7zM9BdB+Wvf4wC6y+fg4MQ7jum34lY1PH9baCCPHs2WCd03XvuxQ/6lKe7Ow5ChcZw1bWcCVQrAybZMwZGVW22PeGWuhapcYZP4VASfikvJXAKUOLsXipE97hG4FgFdogPxBiQt+cV9AyGKX3PeLYgv9xr3CTcUhnEdVLsK+8nb4tTZosLquccmb50PfkQYMFoo3EiH+XQ/We/fth7hGrnJBdsi3aMPh7n+8F5X2Obsved4Hp896xv375UmiUxQihrrnNfiH7r032S7pcG5+G7YPa9lV6K1fJLs+X7XlMvsavnHxJ5y/UC6V1WPlFhMCReGyzU/Rl8CfL37VmpgU55TmhF5ZLBxkfNJrFvyYpOT7LP7RBu8NLeSkzoQUURxL0Fa/MzPgja8LI2f6yHNHBdyRpsddRYbTTwfxBtY03cGQILKSDNUVaXnvY+/uxVEguDTQLV439soipUd9W/GgwC5reze7nVtom3s+h/coIPv6l4J9X/wqS+gOcKG1oDRQHP1tdBiS3TKDV/UYZV36asV6p7dfvVUBpWjJjs3Dm/OSX7FbRhMgFHnRLYyXaC2euFmClFAMcjZCByXRgW8RxHT/vHAeEvBpSBP76bfh5K4c5yVChejnZs/glTazI8t/zYMRGqzCd6fzA/ro0IbAwu4DmPyK0R2SNbBpRKbSiis1wTmME+yus2c+3Z3zxa0QTN2a5+ejIGed7LZgRlqrt1xSB6a2hvIHBTbFilo22Jm5Oau4CxDINmoLPbTQH9N3uhUQH9IuLZzITBqdnHihgJg9q7jzyQD/Fh+iLROk7X8yoaAAWSr84cRw+8D09V9w7g4Re3j7zzNImQFe3BbG4L47kqljeK2cCq0Gj1Nvqek2WVOE868VcURIzeEZV1zcyCigSsL7eGGb/NRMYOXgqb1Hd+WLjTvxaDkDC+hv0qX69mVLdBColXhBVs7hy1jc2m2gG3dv53TZTliJGxWi9E9xIWXuE695+4YaGPmV9VuS6bhj0KGCiHI/LIhnHfYwLYr4k9TwO0AebbcSx2sjo06avNuva93UYjwE1HuLTP1/zzRbQUXnjn68RMYiSKDhWYW1UpIHoIF/a2uLtfH99KCVSJzvBc62r7OOHbKUlu7c5tvMb/984DMwQjZrcJaC3PtLsRUeAeCtu/vrJY5ifpkoRihDwAPGV8jbuUAnMSN7PmZJx1GZILM6ypXHYN7fLldmm6KHz3mESJ7pV9JvdWTAWUZIOhTJ+ShjUi2TT+uO60fx3Hs22cAwx5nRI6K8TONdUNl9urOgaOCJI+9tlqgv4GxLH/NfY+Tc6yE6blT5Q4bG/nmC6CPPoOe8EYSpNpzslZWIGfIBZRd7RG3OKTeqXFntLVFM19IawcGNZKzmkNOjk+8zqzW58J91KYv5y5Ejj4Z/52IbIWtDyhrSZhqpI+8ae0fG/7tE67Sx9peltPoUiPB7lp+QdZVqjOvi5A3Hmvystg2GB65NCPjNQ9q7ZFK3HKmXgymN0MW1xaDsYw/bVcwXp01q5F/2cP+KvviZRfGOy3NvlDwC+XR8oLCjMoZ/M8HxTwOsa8fx/3+bMr/5ob3XTld6gz+P4dx3vFT3QDOcv7MRS2zonahpjwB/XD0znhXOxovwPyv8PymbpWhLPVHCfgDWdA3pJ7xFcaLh+Lfjv59UF3vIpx8TP/2zZuukL/ESKHDGUHS9DEqh4AFpis2wi0Z/EfchgKA5JoHZx6OB53nXWc/3ri6uOIPql8QACH74DsSwBApC8Pl985vgISnXHok/mYhmmnyz+ih3pZ8Izl7TsSZS9ll/fcV8UE8qBkNlu1oMs949TMqFyw45+xvgryYe9Xlo+9ukA6ULIjSmOyRxNE7YavwFJ6N3EUuz2k+D4Sd4LWnmQE7471KzR7z4MP9t+HxNrM/+P/rDHrDM1eGVzwLlB50kegEAqNf6VwHJ+rTGrtaL2FL1RNEzhmO5/JUG6hzMAa6TtSYXW9Kb3jvpbYOYdQVIju+os21KD70IDW2HzYD30VilVfdu/GUi4+ggElDGlq5wPYIdvcbg37lU7TOmyfWkyjPL8ZbFnfWLOOZu3nZo4LjD6x2QdpXSkHBuZ3834uo5xIJSR43+jEXcaDYKTsW/E49Ow7G9QrUSOrcaekRYQZptclBplcelS/Me8I0YQBHCR9n7IggKde9TUynSXqv/lKwhTMElwy+PvGuO5plYQtUWxg9Vda06CGS7boxTeQKiZF4dionAfqSVB0XvjXHXgOOXvsyxD/d9rxo6t10IQ6Tl77iHb8fMk4AZcxdpXGoG+oKLsxoRUeCqIIwie4RCr95s8gzHlqwAixTZ1yjcepNBeWeDas5Tc72vYOlhKMTSezxtoRPd606yv44Ofz27l+AgrTahaQHEM1mLNZv81sGQ3/1ct4zc6nDcwF7yVO5nQ88ZxAzjHiTdSs8czBu7XJ8a3tq2j+opxm6mbaSXr4jx+ZgIIgXUIPN7l2gKV2gvKlu3nGZBh7JkmDgwT0UHWHfnR07U6SYVQlfU3A631mIEw41hqX87ZmOz2TadAL+vmb2h8wgMMM9CFIR5tAeHyWcBbsVvSt2oecQH3JpXYIio/xqFa8NDEsiD/mfDn76v5pZYQpUvQcpOajpT0ySBgv99e4mORlKHI1MlZKpjr/8mqwJPvN3HZhev1gqD8rlXki/nvNWMKPGKvdZUcJLj+GPPvRn1FOKh0+kY3fHvNDVk61AVpvHOp5AP8mOP5LMuijNanxf6xDw34lsaXU8oEMb4dzbhAol4I03aEgmhonPtozhKTomj/PglgklDDw0Gp0vnMngzAUNE8vLfv6NDYwhBel8TkOOm3ns7viOTe5qFFeoqGHsEup+7PifmKhM/QWtxJVIZTRd9fGNWzmgpF3vbf/uFI7gRnbmy/a/jw1TKqUG3TfrCvZqVbXGjAHC8ei+0zlS4eWnNJJ2LDb0foLAhzUnlcbrSp0j3g2MVQqKMd7Y3CKiWw+8RXLbxfr62RUNwcv/0nFW5W/Y38YSiUa35styyC7s8RwU/yC01U7aGws3ZlOn+AMB3LxW3xGapyQlGE+QZhyaC4+h9lffZSGWjpeUOL/9nP7vjFa/dw5ARwioJ+FR9Hbl6JfHrrJNsfDUgL4/10YpanFQQR1QT03q9dXvtaHtEVvw1chbCJ4HhK/har/dVR2QIxyCutqFUiOAvwDFicW1v6peEqIne+9oPhtxiUwtKnw5wMtczI1KH5IBRszxX+R26FZDyJB07YOlhtsD1VFoh68cuLnXyR20luug8eWUQVgk8Szo3D/pN+VawDOIqy60XLp8ph/MaIouegkVAZvoW+VzVgn6URuKYoX9cETy/Sa4dfGsHKvw/X1nabYLkv3mQ+TEdNQISA+dqX8L391eKA9tI+VMYnCgmVxOsdDV71upUIrpbG/ilusNlBNckdOFmIupReFBwCmV3Xji7+mBc49StnkExGxOLHztAYRRGPtvauKIKGhRyrf1PVjP0tIErn9E6RRe378E5qMwkaBbPDL2+Vjv/d8tegkuMYnN6SHSW2GR4ak26tNXjITH+vMG0a0s9+LfkM/3NmfKsbVfzHBDpw1Lkn9qAhivDE6OwpeBn60EAe3kru1Y78pLz4wn9NexHZfYfw/nCam0Toe86tgLaHO4n8+xt9aDDu4Hf8+WujUEVh8WsXxWZ5TBSJ61Em8aU9ol79tZbqHXu1E0joa/Oo5pns9AD/yeCO62M8Eo33u6+fvWjbMoSuM4h5neEOu707LANCui0ncrfla/xaHIN5jpCu29mROr/UK+fDgH5g0yp+swsht/HrgOzL0yasKiQO/kNeBWIMaZSC5UIJV3dFSgP73ezHF3oKVkYPdnMO9May/JFe1k28X5lwfVQUHDBMrUXfUY85PQ0Ag9iW6yXnTTl8viiSp3Q16xmUQ16UQXpZ25E1zZXx8EG6wYf8JHY5d1UYdUEellWuqTmRdrZbR19ZAAtfr2wGEYSNsKZ07guRBgp2sSutcA8+AA3VSK7dNqvi+a++njCPfQLzpBxxVQZasn5Hwc2omjTt1ySRUx+xyJMgpsSPALh1QIdijnkV2gJPj8Gy7GTrn+JOWoUZ3CClSyLsvWjhtWhhD28LyGW1g8mlivo2HPwL6jqyNQ7D6DuiTfXR8897vsO/97GMy42O9OW7IS1HO0mYZNwsSGDVFOuwMeJavmRB5mM2Ust1dMru27dXDg8E71iBEdpGIZKTcCZP8Lweq/J58JHdCuCevgzjEggIkQWaUQkk+X5YNConkJghTi1Cw1iDIfIyOYlW8mfP4bGmk3QdBf2IHqEhJzxlc65gdfiNEIbbdhdqDK0ZYiVCj+z7PGBiY09gK433+oEhJHrABb+FZHdvSVAVcGk4/iXwowrJ6uDny2NFjObVf2F0anwgEwlZ7wt9rhBuSmg/Uk4PrWXm/nKPtF97RMAlYDiJcLlIP2PVJLEin1Dqg19GSLrpx0sZMS1mQd1Nr/W8Udv2Z5Vll6kH3PBnxekdyawXtrffH6pGdUtFNMYurduZYhiCCzn/nvJewNUrQg/4GdiW632xn+dMfMoF5ACM1jRFVSKos4VuMOVtSdIP+dhBHcQqGVSKEMzKj6KP3stnRmLOZBnPbpwPzGCf6TI3CtceuBoNO5dPrwuXAgkX4Gqtqfv7y1snfqZn7G/inJP8j4C+I7Vfmpz7qAhXGrZ/SrpTYcjMuXIziKLwYTlDEpAUiz0n0vLCoHdz3uvt2wZRNn3Kfp4t+rf/acjqC/KgfJHdCZOv485fCJWNtU90286pLc4DxrTXfbRCSqpFYBmodd9Gh88UU0sNY4UDFY1kva2ocmeZcY3oM38A/5ub1MUFaCRS6QRYGgmE8rHENzVD4g2eiqmV3OUDNfVVjF81+VYmbVv5CrFxextM3VRMwfbsmvPagtsRDKNfrzGxKiyyAWFi1Z0PKu2OvAFswkkumgUfKqqYGJLcwl8CO8XneuohZim9xRYoB6GZqWzifLlVUk+Kbb5Ggoku4K4jkSR7cCqwvgb2TAp9jsbrp82E61kLV3EsJ/FfRIPGbyb2jjZJPVTopFP9xPwjQbBcOPJAe8Uw4PK/riZA28/Iuy3IaixWhQVdFkSFZP0h89o070qQZAK0fIc9mH5pmtg/BXR1RsSJ2ThnO+1bwGgJD0aAZmHOmxZe4rNL0d/XUXbgw0Pr9926U/z70Zg5ukv6clSsl3QGzyeW7aCo2XAcb2njKefwGOdXnqzpeIrM5m8wGhVoa1Cc/oz+W5K/Zp8/Y7j0d3Hc1WH2QQP2m4IfNvaWo0/Z0KiYGheiTy5HkFb3BQgpw68x+XvkEZsUoQyGW8KDIkh/P/KDQ/mospnwgxALz2a/OvhXXX0oIWKeZNIC+K6YJeD+KgnTIb8gulyMqCTPmrVwn+cOZJq+4z/xl/nXl7CfXWqFUV+sDqPOwGUYiifgk17S+DVO/dll42jfRCU6ErhgPC9flfepNdDE9gsPwoh6LooePMJ4XABWusD+dyesjUGeD1FZxd7ol62IFZNYv7x8T7kdLVqfWLYfrCjGnl/2zzzc3y/KEMG7IBb2MmFHlToaEOLFMkoeYLvoJ6WJTMi3ucbaIqIOdAP+1D/0zHLFLogGBTSzFqd3+355DLl/43zPDn/bKhiDD5QrLMq4rV4P5X4WDmJu7jb78iwnEElgpCddPJx7x4+fsSKx/Y0GjVE/nA6chIGDR123k0T3qbrEiodybkeowUhL6mJlYCfsGeRbh4oXX1BeREVqjl6dF+Dojscpw4XeZUrFdowIvlgVVNIOjdrylloMrmZQZaE03NBTlG1o887eq8bjIVjh+3ijky9bdvBhnqeKH7o7vFCtejbbaMniO7wjzvPUsYnRuUdeM72gH6P1Shw5qTBfyGnyNeDoBU9ghj92qT74VORTshjBWw9xEe6KxPsuRlO5X1b6bEMNa3Ay69KqQzYCG770Ru6YekBeaYCOSobJ95WwTmaJTHfDTVZJQOsChH9xIOm7uF/C+mhnLuMm+lfWi30HU/krXrNSTFBl9z6n5ztsaQZFLfjwX6LiRKoyULe3tMTI2a3cVQi5T0YuDHx+SftYY7a4ld/f1+Tn0U/bs7912jT9aju9pcAjcNgdEsDV6B+Re13KWn16y/0gb6y6H72TaGRkFjs8rIWBhCA4Et28EPU6GKOIfFtKbGnwKhSNSoPGGMvyfPyG8Nd1sErOBheNSOCh/85lFMKih72izmwepndwHGuRrefLrC35LVNSkYC61MuQvPHOKrEDWuwwGAAmjW1ZOkHUmlnX+MLdQGEXPCZNvYBJvEe1gu5poVp1ucqJWrfCstbZphvN94J/d7QDIJvd+cak+ptWGAo2hwakELVwUVnO1k0PTs1zd+cxfF9lYjcQSqwT99m03CbyMhcCB6wUki90SsntJOR2m28DbvtLEh7Ii21DMnQZ/Kid+iU1WHbEpb875CgkleYmX1zUMo0OB4+gRZ1wbj3UCFYvEyIMSBDq8cWJdPajRAJxHkT6O8peOGJpLaXfp0/sQ3QkdtbxHqh25h2+JlH5xpUH3lh8ZbNV3Po+3k5+ZZ3AS0WXVqGoq3hktsr0XfCxtPK74VwQwbs3Gr+Wd9nc1dNV331KCTeXU5abkj3FXN0tv6ZkLikE1bnDiOXqpMjWACRFJnTTygD0KDkj+aIzVX2xZEBDA+uW6q14UDEgkGsJTh0rtDv5U/pLthfOh2U+lDjsEWNLIGVsxGARZuJ77QzWHW9qW5naVrx6S4dVi8g3rhPYI7HbShIZP4M0EOI46m+XiAUPRTjR7L/aXOqv/xLA8GzZ/mpdMRbP8yvilpxu3qSqB0ux/LwmCZG9EhtJoVcVG1MF29xHetQ7pqWQyLGLhjev1sjYSog3wuLHszUZVzGUKOiSm1BmCpCbyGizoW5aYav7qCWLo/0Gskv3bT5xEjDfM1t0Cj4OCvW6JCkhRPTRNE30RWnTQnTMkIWGfL3vmSGZpVbutXJ8xvoYCCmzxtTEIu20AgV212W4SByLwyUlFxsGw3LIL1RIXoCLpQT9zP+0w0Vq2NyvWTdJDhXq/spKBJ7mPgKPfBvI/mXrbjv1SBDyidqPeQW/lr9LFF3Rg+AfwPqrMIdCr2l8L3QJaTjBP9wb55fKZwijCS8suED9BXGu1siu3ag6Dd1XLbUzZkyRyo+NVv2xph+FnSTp3Ouv97BXwG1lib4lbMc1ayLvb41V28/WERPwQFsNSdAGYZmEsXeHa2WozbR5Veat/IUwgA+zXtwl73It72yNdP8+2nU6WjZmIDgnod6nMUg5L4T2vjpEC/r6MXdCWTSnf63ToVJzfTCnPqKVAM4L0sVr5RVJ4x+bGmJveTd4MVfV4qhoFSDIn3GfrXGZSiyFqX25QnMMRJa82uWbRBVcC0cLFDIP2j2LD3jRzcrjnmkaFo0MseH1wEhFHqyFTImAomSrv4VzYhiNWD+MiS5LDj69r0INzkkCYwntpZlIQL9QWn4NpgbR5VUZBl6UrfDnYAA+jWyAcbGrpADrPncDwEpVh+6uAfO99E1KIZE/otgO/IoivxiZzZ0mk3jv9sF9mLF65di9dV7OaEsqopRXsNZHsncdZBFWhlfwPD5Tn4us7lE9FcDWbeH+/YZ2Ep192pXPdjhycPjMF4N8URTw/vyAGtBawGDMIZEOd/ogqiPlKw6i9wWEIkeLodP8Kt3AiZC8zap4rZKjvmOsiRdGy1dauF1XV0tdCyluCy8tbfo9llszwJJgKCOSOFF6Wr5DkYCi96KFdEWYDrlUI88slZV/5/Q2fzcDuR8pWCeyCk+5sflPPUPdOYMQNV8xfSRpaC7pIprYvcHGOexh2XzfuFBm0Ff6Bg+8fF6hPmPRFYaOGk0rO6m6fuMq7eiLWGSgi4Hf8bw5Mb5CUR8gUL8sawAwAdaL93YLSpa0wnMov/BpBAbdHLLAbuLxujXRfjGyNo/l9AdTDw/7XIJHx3PL6umykhao9cJqTNO/QnNiq5b+QjhdTtpBkeVI/rquLihXNmpSJZQJCGUENH8hoaWLvh9b88wjQJDySmdC4Q+kz7lMKODIBR1BGhy/xh1uvL+ab+PVbf9u3B0Pp7dMDaH5szgBIhX8tf6JnfUxzW083poLF3Af1cettX1A1dBUD9dblb9aQ4UAYLZZFPHStYPX8BFMJOzbPISzavcq2PuNwLkFoen2GZhnhdtFyqbCmcNorSS3SYxy376Is7s12cEFheMaVUsWca0qWO3EgRzCEdT0fFb9q2olPw1N7BmY+sgOgemM0Lei7esywU+Q+xJ6jFatkCPfIaf4aWZeu7aLTYWv4FR1WfRhY+BwQeHrRlYmDfZVUknY7obHNnV7qcZCc/2tDkDF0dXydAbeJdcGf/HBZp55ljqbwfnA2ryTZVPZQS26aaCCuqr5IEsMOtsC6SqcXoxI40IKGZrBGCm4AczhviMzGxJgQ4k5uZk8QafI/LTnRC7KBuzb8NJfXxLwVALnHPz8FUbpCEZQqZF35QccpvlZNltznx3DRTF5mNPHzF8hmofBnt+fDfW4/FvZxPeTvpFz2e3qA4VILjpwBvTJHu7zDHDFr7IHuTxT6hC0RoS4jMlfelIe1dpAiPm2OM6hPJlJW9erkOk6EG2mBPjbPHBhpW0fNOWEWXRASbhwCNtLfv4ti8wKaR26EFmxa7/TAPpeJrd7OPVVc1VRY+vF+JdtvRlkExvU0LQr9vZRTd1iQZNl6yfJRrFfHri0b9LBpzL0HpyZJLVSngh4p75dBfhgtYN9pANPNhNYqnYID5mVgffYqFkFAi48bxFOqVNM+ase3c6cyS52WRxbEgY/FKjK7Jp+lZDXxewnpRXl1zm6V8yHtcQp6cM6tBEXEUI1CMcFLeZZ7kwJ9u2jf/Lho9PxFthEMEXI0/uEeU1twZ7npsy2z6B0XqtwsIh3lCx3CWyh9GOPIa7f2YYQz3PSOkmitm/RFC2hgUx1qAAR5qPpOkJM9NZdcvQTelwaySBu98uOtLumDXH9uUeBYA79G6rHK3BvBSjklTzWiOGbn1OIzWmmtKcRPxHFWN55LB8NbqcNHAVhJ8+zpmUYLalVeF1aVRsKm0bBpNtUR4ESDwh6sLApJTScGQ+GmviiTfyAfvjwDifoUsBOvWTBfH7x4iAWF0aWuWu/9Pw9o2z0Nc5ZOq7LQflE5vvxePtF2qZu3PaqTtfvLKjpW6gZptlSyVrj1O/6uXcHHHrnPvL3cNQajNZgqBOGBiEAA2DFm76b30WJubZrMzzD9S3jba00fc771VutFL/E2z+BpTK6gXmWUtM+rg3c5WwlbuC/r7oVIV/5Blet10XOyWrwws4z0FL0CP3VFO0teq3AcMqnIGSZLk1i4c+2Xn1mTQxcnKUN31MbwgGhX1Ixjh+MrIusS7S8PHefx0pIQbEHN/V+vxmT81lVWszRzXGReedrzkVbBuuvfBhIHF9fwI5QepnZEYFYESR+QUwXqN5+3ND+k7zWZmxBYMTLlSfpNkcnf1TvArYI773KGFEC+wIC6SFo1h3SDIldeYj6cousoj0aSzRlRmhdBjZNnMHXz+IkvqIoHz1LE7LRTUfqLAPM7Z7LTegowXdq9zHXN2fCvHWDTZENz4joTdZgmN5YsHnaCMPFKoV7N6zKCeKA5Ty6v2+IeljlcyNHBlQN39KonQMcIp1zX6u7E2JpKxlFYfi3ETs5zj/zYeZUzdA1huFkU+himExR0U7WQVo9SOAH+3sX/zmz9NryDjwLLffhMLQ8R8EDJAuXqBUd6y7ISSf2K7HFtqdBuswTOLv57jQVakMSEUIv0HojGYvrM4d6hwauRmAOp94iIOO1SOZelHTzK8JM7/O3KKUuAx0hxXPFuOq2QldFY2HL8Eh7xhE40Lpe9JqBnY3mOxPbacaH5XAY9dAK+QeHz+gjHCdLzXLXx/CKZj04i6u3Y7NC8tTcCiYoTNi7UiFNEGTcz2sDFj8ya54cVpysoCPZNqOlVUI/q4FLgFnWSFAyj10Ajlz+Job1qN1bUCDc7+6ovK8UMMvtF3lhyy/iZFYJLa5QHe2VNcxkPY9HKc7fvH/z0YuP1VfzgfhZoluMwvmBksT7XoVxuqWRbos5kYaSv/+QMcA7Nl00SxVTX4nsWFHMx25ZRgbPfBlXp3lJJv5tX4IA2SDMZ+tjpKiaRQkj+bH0Z0xOrmbWZUDA+LC1SPlZZ/rXhMSgwXkU23a9NsyXQ8CBitm2clgWOX+j4Xt+0QWeQlAWEc6+0PO030fbFhdaQMy3ug3rU7jF69H/YJhBsUAJhCGZT5/QV2EKFlfBI2R+21lhVOM+OZ5Xb76a69FhCC5/BMjoIYFSlwZFT890ls+Uby70wgzpLh6LVWggVpX0gGrEBxgc6TzCub6pG2pL5sCJ1zEG1uwOEwfXbvAQKb4t7E+4FKyNnQo3gLgoRhziuwklIfdezttIDQ0x7G33Ez/LJqnkzv02nDW0IKpXd0uDlnsP6bHbA2fkszbNm9p1P2mrXq1swsFjYV44z7G9qzwMcGMw+MrejQJ3bm9sJid/mNdr32g9SSFeCMERMjhXszbUUo5bJx6L9eiJMCFAJT0RIcRq+uw4isvdtw4L8mAGs6QP6fTdj9VvgcgUv0bDLiOipCt2xHQastW4AIZa8RkvqN7ax0wlan83JhWlPflB/3WuBeC4Gu7H2EMPpTe4+TGd7Hqh0EP9+gNy1FW7wPQCDgu99w9jO+QDLnNOs0o6qNbAfTfbLqcu7h3GsBfaLcPopzjwT3Y9AIZAtriHZkT9oJr1SnjGudJtjAhvGL30Ts01LGl0ZMdQwyXpkE2As/pU06FFKb0xQ4mTBJtJOqPLmpLoS1aBgzTbO1jwbNQYXeHuwYx5PXwMfLPmti27rAo2GwyT37O7tBhl0CTerI2DzGNVdWn8XFxu3G6R6PppT+8dL7RrxYwv1AZQhL3XUfrUlTXGdfje1skGsnl9n5VRwZrve/SiDpP5cTkSMDxCjngIaCbm8oNfpwCPgL5LxwNA8fMSDSi05d2tGtxRRFq7GCzJBieGssHDCSkeJXEnwCdO4uJtL9wXM2bzY+vKyXVpxTErx9nqy7eTbukub4q6rsE5AELVZaeKwzZ4VeGynIZhkPZTGph8wMhB8xOboXVfTDOyOqoyFFSjtLbFaaO69/m6J5e/B/pUAX3Vnw7jQkdaGZ/M3IgGXfroBEDYKtm7dJWgz1ZvqEa4Uh/tWjP2THTGlypt6v19L23aMsuwvOda3fPGj19Y0Uk5S9buS9H07wXFZQ+nfcrfnlNcXxZFfPKBKyZN07dldpNpJgachltEG+ufTJYVeb2jpT92ZF7c5Vhe1Avs6mGzsXdSPzs3dlBltYM1P4f1KxhWbuyC+4hrtDsYz/3qo02sNGFIjajG1HVyr8Fw3w2Y9hidhpHE74gOcZd0CTDxduORBP4I/K8aC1hUmJAibchRwiLhnmbpEqlFxF3KrwVHbbo6SLJLapYnGRRjVnnDoF6D+PFXkWX5VlM8v0uQbjQepil6L1BWpaPObSNrs6/noMNeSY6i2wQORQpUmbt6Th4UJess1VxQdghmkAgKAS3AUucnzdwiVIcH4vuZlQbcR6TUT2d3HymiBf5higfiVm7RLskGbwW9c136SJJaRUUgvYdfG7qHyVpoTGfnaupLWKHsxV82ARBdFVAGzWZRZsnZdCL8dIUGY12LqB50Iqc7TUhJC8tXNMNjZxJ205WbJ09cqJ9JRSC2MiOvxH7UEu7zmM0u0ao4g3RD71uEOfg8jNOZVCJbIU155CVhuQ/fb1BjNMBefUogZedJwCBgBXXPlWNc9+VNfN0e6uy4lcUv/BXwkxvxCVpUwD9FmpyL0/gel9b2Xt+1P1VQbXwgeiEeigJ01dUgPy8SYP4fl3mZG61Iynw77GdzeMR3GL4n5aBHP7yNtSXxCNXnDC6kNU9cSbM7kws7tm1wD7qAgbNvrpPy3h65WwliunWMFzKQ+PLNqiRCJQ3j1fd7HfK036wGwRETODr7d/cYBI3AthmoAqDG3mMZk/GidYlt82fINZmhBlJAAr7EZjWoqWH6DbdElyQS8PsxQml4E/DtSDJb676E4e9Emb5Sr76kukQyt1Y5ba6ClTYckm5nnSLIYWSXRE6e7WyJQUXv4eea7YHGcUgU6pH/EgM0wqvZdak32OjEKFvZ3c8sWVzvT3hEj840FLmIQUFZZ9wIhWjJRIx/x6/vUgrAGaYTM5ynsaEB42H47L75SAsjalSocZal+3SiVfGM70EnFEwsSG82gCknUXGLCciJUZqL+eSQRBAeDJ4cBf/oUdeejL9PWDfjbuvxHX7frPe6Nxzp/TRDeQkxy2N2c7EXnYZxIIOz1yJIwioh+KzXKLHn+P5LGsjo9tADfXqyohbWvSp+1C/jSywN1Sr94e+H+dAEdP8Ff+Cyv1bXfLzbT1V9kb01iJIkZFrGW0e3Is9bucswxbU+IREFrqi2aZNp3B+FZby44XMzpQonQBjM3C0I4M5liEL/zLahAkipM9DZH2g4AZ3EGt8VnetXfn3C+sH1zvVjy7uAJHEiIy52T+ttfKaZP7/FdEPCsd6Fai0rLcKYTUQ7Mn28RwfbhaZvWVu1pYQ+kty1TFkZ0vd05vHAyxDGIw44iiQzb6NiGqE7gLoyQZprIwpsRHoADAa6NfUEw3hNIYQyOo05Krql6tamGfBm8F/VfrmL4qNSsclzjD/gLra6/qsN3rPXkWRpVOwxbGtJkhNs9tXcElsDnw+trsaG4n+1gbRhD7z4jWfbTM2r1eiJzJlZ024cLicnNz1fjE/Gw23piLh+DavmxJHYb5SiZS6DKI/hvJ5lniEsqjTIYULXM88XsT62lNBeF0Kg2RyOswb3l4G04xz8juRb6ybpzZhWTEJ6+UZhn5aZZ6NWpnIgjrS/c7qUWTwrupn/9S1H8RvVv5VfUHkOo9V362+/NnFkKArbg9MxxSdiIc0HdKHUVmZTrp4aRqT2zVMgchQrjcSEZerbF2YV3QH2S9lBONH+atkatHabiulUCuzX+1VK/XXF22zX4hkEJ77eURamyWgq/nZZdHvZF8UD3QOMfHq2BmJ+Em873xJyg0LxIhdjGblBWeBlyp3W2ssLmaoo0NfCE940gI4ibGxw5yPAOba/sOPtf6fAT3d3I2WENplX/6uNPNlYLznfCy57DvQB4td3+HARf8f9wx5eZpRTjlms2XlIGeyIcBKVFC26L+l0TMDfbkTiss7JRxT1lF+bI2TJzgys/IvpPywgOBCuLBo7locBfaTmQeRI/0mZWrTYwVJpu1NDg6aSD9vSpfV83O3IEjP8IF54mUJboO7bfFj1Js2pX7l1oEi3MuJkyke8V9t8VA/hLQxTPPiXJvVdkzyT8VEXWhUKeFbHX4nbUfwuSX4uHWDlLTaw6E+poZVL31hqpuhurpvm/1zlfbmKmxNoPPW53+uHRUvwU2xQQxX/7oUuo9KpMi/Vy7BSVwoMcd55JhhK/oCTVX5kYxY76rpt6PWrR1PIY1zVj364FMTzjIht8ASczYp3Pl/bY7B2Ccwb+EnS0AF6QSueCnFKOCeGYQ/Q+nnbGM7gUl4moF+IQDzB+grtjxB8ksEvg8LyS18KRCI27w7bL606aeUFvamyspbPTDbONmeRO+qzGpgAcgg+R7xw94a6zy7cOBMf2riF+BhKGOU9Sv4ACEgjf32nZv9ho8G5BYuzbwbI7BCjK3lReanFWQXq7ao+pQHvcleR5OczP3uoT3e5n2xYT77twRApbluz9hluFnEGKjrexUcdo95OssxxWfhABo3OVZ8x6AkeXVvTr6XvBNbjm8qZeCNgGjz7kED1OmbGxnR4kZ+WoWo7GueNQnqYOdp35GRg1B/pIb23JjaJrkRXqmOd8gVHY3V8OCOrZ/2DXx0D6Agy2pbua90SrpV9OG7+adVkS9GU9tIupbM0BSQHojZa3y8VSyzl5BLvk3xUjKeGr5KFI8ukLvru2pk92g/tA8O2dudYk3XTCI06+FtCQNLOQq9fwZP5F5/g+Q8ihSutbE9YFo0HEIdLQLhh/tnGOH21Jq0AylcOMHB5Lq9rwW02XGW4pISGc6ABYbbBcnERtlbYvlty3J08vJLajfxeJ8rynpuQ1XUH1OFW7t1IMJajl8YHWEi2ZJsjOBZd9XyF3/4lml9bsYJi3a5j2eh2H/c19JSS4Htbkd8z1ryaykaM08LBEhumMU0LTmkJQRNwlMgAbBkpEIoJBLGMoKiGOF0JTYOYumeOadihQfFQdg+HseRmhjdv1aeRlFghgRk/9DN/qevp/Ue97eZcWHCE/yGoiBW/XiYtso4JYldAhge/HxS89c2kJmYdcoLR3wF8jcDERlKhV15eupKBm+8VBYLgYcXoUoSMYlAIwftr+1T7IFVLeGwzBjrf1dR2kfRUxKve2fOsvYS2dDwIqGeEImmtcR78DEh5iYre5VKinAxJqeLUDPGBDro+MLlTCPOcvL9DO2T81mKqbyTpY8gxirmqKEGtv3TIZwO4lLx2ikXw0/ucQAKMMd32cucFskLto+jaTIUuA/BW87XTKDlHeqtnhKdn8y6bPe9Jbg22442+y68LF9wlxK77da0PEsJ2OyS1KvxKR50j0054T+uXKjeKwK2qTK99D85reuANe+GwFk484a8vvh6O8RGCsQhmpBKSUbjSE4/07s0wB/8BGt6YpxlfYLRHGCh/iOH5cJQiYF5bjRRq9Ur7X3XwXjQXbt8wcP/y+ES0CkIbezw+moj88w4SvoEAwE9Z5hsqSd27DueMiHyxdudkNd8xGOi9tnBMALlN4G89aQhaviQ3Y+HmEzOKxRAZ7fl5CRG/cwXzaNONuE+a5IhFVxKIPUXO+Bj8ZhBVljb/DgJgC/VCISk1HNMesRKVO+lCEmbFX+z/8Yb3gLSgwXV/ZSEt54urQ0jgNCoCqab/xS6gBuB34XmnIMmNGsRbdyetOnyu5s2gl8SDYRSB7aPvhWsyMpxpEBRl9/fsboB/twhErG94uEBL/zD6Wz8hYhZCleP+EqWM8wzSs06T4b5jXSS4/2XvvZYkxpEtwa+5j7eMOhiP1FprvowxqLWMoPj6JSKr6nZP986qmbVds4y0yExqAATcjzv8OPRvOpG6hvzloVTP1pIn7Iedw8ilvE3mjTgejlVNC0b4zucxXs/AwEk+M/+M0l9vrRQG/qrIyTeG0LsGm3Ye2sedu2ywbsw/fJkylFWUtW366bnBhSPDTxK8+cd+4AB/CKCxgVMG58xnxghh9dNyWC2rsxC/H7hhmksf0jRXMG+fnrzZN/l9cUPjlgHtl3HDUcJVbzzHA1X6YjX/FFvFqfMcJwoI4z+yMFg/Zc45VWq0pSzkGz3Joky4Hx8FkuU5hY7rA0l+aoVI+kb5yo4f/gZWjpUGhlk7rOFHXj/Xo3G4l7AWnNxhYfVQgPHhHWI+k+wuAXILhQo2CYRA1E+vFFWm8xU+1BemoeIhGrjYPniS42QJ3F4WJ4Qm2xQ3HTkCK8Pz6zLATTSrpgvjiFjnHphraEaLfGDaT31FUmrdT24/pMiar62rMfd6mOCBuLDmtA8m4/588Qf9ZNw+PN7PIVJfsIXOQO1MuAuTItceGiQwf7GSHNFVZvPr60xuq2w+Jmr2Q5DSE/SyNKuZg/zhft1FhwQQF9cKOON8mT/uMwamCeSOwB8AOT80wbKCosRa3Gj+urK84SvJLSfJYvSbZyCycBTm6i/PhqaZcin817sV2ebKEJiLTuH62Ojnu3iEq4WY9MOw0iCrXQZ/vUcVqmLfGdNvrR5hkzvRMCzpMIBGA2Ou9aRvO+vtPYi7dPwUf47c4op/8F6gVl8qbJPHzhLaP/2PrHwG+bw54Lf8ZPNjCsjAKmcfC9xPg502iZZfGCOwX7qIzD45UZV0Qbp4T81/bswK3rH65+pesV5cL1ms/Darkd7ejX0h9bvfUqr6uX7WfKN1IiiO5ZYV1mMVYFkE9rymPt2f0gI/Ob0YO6kG8eeRepHYPPX4Q0c/vchsZcmWqeu9iA5Yz5yGkPXD3jgJIChzRh49vIh7R+RIdGpym+4/jSJLjkbUxcTaQ5dQtliti6wtb10VnzIF+IDf0BlrbKCSu+YD6o3bMD7BcuuO8ie/EeGUYZqwdbYg45Q221ocPny+DjLyXzR4WyczuASLjCA6IcWEubcwFPvqVZ6FKO3Lpcv6L5qgkCltnLR9tDkHDwnyZ38vBYI9heGZcdKhaisf5LyCD0t724byTykYXfJcpjqv0CAf9mokWjFdKjInzA/9i41YRvZhf9zqk3VurOATH39MmfLPvlHlnNJP8MKsWoQJe/TzXIp/0pOyjFzWKyx42YkU/kmp+64bXnOlZCR4pHubhKh12Rs6DaQRJbGXxsg0rjTfFY6o9W53bORMkeVT5ofz+Y2jhxofLDnA24NO8rn/cT/AP83X5fZXzVWWqU1Hg1l17xJHxLBRFi3U0KifgYhFic3ZOoP0Qiusoth1hPEu2ywLR+JHADCSibOSpqDsbXPrYZa/5IFOEys1qTL9aRy9NeR7nBNVN1W4yIJcYLyvqriOukdBWGW2Ypb+7Q0Us2sJo80G1m+Vtv+PmLn/73zo13fs8CTDUmypQgJJlSiV/ByTbQ/nllYuy/JL6Qas7v8AcV7QlCz5sIE9CKJVAttDzjsIox16OtewmM5/otjPiZ982fLj50QY7EK5/0CZ/hDysc+35RZ10J9H788fDxLFHiRCQCTxxP+8w/lzmCD/IFDo7w8M/xzd62yr/rw9Bv3sq/K6rP4sHIr/AdQT2J2sP7vKvx/9nSn/FghgpIPJu+6v8n3/R6A6+7lmCuKB+BTtf1titO7fTylMiP/E/izDJ+ne+c95PzvW7ez+3LFWyQT+rfukvP/S37/UOuUpKCB070n+2ijqI78fR4Mmq9OkU5NX3pnjWm/1ONzHX+O2jf0/nEB1dQkObON0782StQLXs/C9sW7L2ObBn40D9hTjsDn1BcoCk38V5N7Iktv8QqmfzVsDDOUvY/+Xsf/L2P9l7P8y9n8Z+7+M/V/G/i9j/z9+Gfu/jP1fxv5//DL2fxn7v4z9X8b+L2P/l7H/y9j/Zez/MvZ/Gfu/jP1fxv4vY/+Xsf/L2P9l7P8y9n8Z+7+M/V/G/i9j/5ex/8vY/2Xs/zL2fxn7v4z9X8b+L2P/l7H/y9j/Zez/MvZ/Gfu/jP1fxv4vY/+Xsf/L2P9l7P8y9n8Z+7+M/V/G/i9j/5ex/8vY/2Xs/zL2/098/r/D2Efx5x/Y8xbY8AN9wij0z4R9mET+DUWf+AMj/5Wl/xdz/38+RR/7F4q+AP7772n64/vWKEPOjMPwX8x8QJlnxm5cvueg9w8PHk2XS5LV+X8dG8YBUPuLuuv+4XScQiEa/5uL/9+d/DdbH/q/z/hP/txK77LcJjVKV1vf/Ru6P/LX9p/1hf9d1oFprMFduM99s/WvLAJ/Ji/oj7vKU/VHsq/YH+Xdfv8NZA34b+n9qxvLP+/OJ33dgXfPjH19G1qQk4D4e0hz/lf2QQyH/yAwBOSLIEBH/Oc+iP9rlggM+oNA/k0PxP4X9cC/HvZ/JUnE/2F/6MABOknbchnfQ/YPva74fv73u8x//9r/NR/FX4khqm27r6B+kNA45cNe1Wv7RzIlaZX/MS4lkNrgXDDBVKfj8Nff/2yTok3+8z59y/9zW+5e8AfIKvG/MnUIRPxBQDDyJP6URcQ/S6In/se/6QkE8sfz38iif9j9P78z/Ks4+v99xpBmykHKEGDZgTgsQMt32li2y9smsDjKoegbbwDDD7OYyA6rKb61qutBlPJFwPdpFndrL++rv8Avp+Re/7jtldxIUdqf2wC8cMfP35+dwZhNP5z+n/Pj8m9V+T3pe4j7gds21OkeQ51qU741F+wzwFHxe5ClTs1lyn/YPjT23gZxV+DzQ0zh/7r7D0jlbnFXQZlIEer5fKcnXv0kPsGbF+jCtxmrXtxbY0jKZJ9YKvBNgviQJMhgNvNtOsXn/SAHSeha08lC3YN2N6TZOKgmR5zO2NcJt7ObvN8aI7AG88IUUyyJXID3V+BDkUNjr+B4p9eE3df/PNudwPYWh3alfolITHOcsRARXmsLklhtLwG/jEFu4wYaEtGGUnb8qGiGZieOaif+Sfv0o7ktbjjkrtXkqdUwuH5L0e6dCTymBvgl/dy5HEtTBHeauDjUm7Tv9kzoPq/673J1udD1d9nGTLyNgJr8vAbtHQX2J+q9dwQC0BAfS4LooznY/md5a6o0he9eIuDlLkWeaxba0wv4O2j9ksg/r/k3R//pWv9/eO2/HP3Ha73ev17IAX9r0/nvJASRNk6ANwm7f2LQj5vJuev8eYk+lATPtx/gU8ZiisRQ5J/H/yshjov/24Q4JvP8to/R6Z94sNEolDvzblOJPX72t6A0cRCH8gWeYX63o9Lv/TNFQDuDvsz9Xafv+Opum2/Q/02d7P6oUqSykrD6hzvGaxTqoyd0mMT/fWUJ3sbfpfuHc/5+cvNPNWXvtvqknb5Hgd6Zd3tFt1GfBPIZhXYX/1eL/fN5oPcwz2/ftAX/ilB5SsW/30Y5kir67VuOzUXffrx/cnbCXiH97d339aSJVuBeSxxaZRzgYBSd8gU8swzwo70oDpgVHP3zvUv882VpS2OpXWPp8hZSuyfSZSrSe3R/S5nZ2/ukUaL3VGGtUWH2HZwsMhZmstZufLfNP+/+c9fvncB3v68qf279/+QL7s5QNCdRXEoLlkZzFsVwFsdyFvAH3wIQJKm4a6J9pew/GQilJ1K7JVGWJAM7wbrF2X2G5f3LmcBqoylPoKwUuJaf4y3oNCBjLeoAR8UN2DsmZYgfi6LF6C3LXizHG36kCvPqttfKGBjeOHhEzvV4fpDXoIxDhixIPv3cn8ThcH7cptnylpaXYUVx9EDa9BkCMtVU3GfwZuULYXt9Ov9ToyuDmJZCuIQvstD7VQlp6ySI7jbvvjofMmLPgTzdPfAzn+L8CoZYvvLFbIn3japTP1wA6yIkXy5wMtNQN68xLQLnzLrD2lVUMirNQjKVGD+/nlO3zM+YRGCYLA0+3EH2nQGwHvAlIW4hNRsQSzRg0vDyToOYFzPqEP+wr7Xfat5WAmR6OI+wcJvCWDhY6yQHE2fiIyBim9gyYLOfuj37yhE9VMcnGzZuJTd4stbhr0pW7akKPA1OIRLB8yJLnMa6/fKm20II3WtG3SXRTh/MEfCJlEeolX8a5wMV2MHjrwJxEJzZ3u5y2RfwKs12ULyUKGvIh74b3VMEVvgVN8WkhDZwjvWDj8jJqsLZc9BrCd0sYTq4IkuSKHIe1P7u2/m1mBNwIiXh3FPdq/84UFZ96Afut9PGKue8QAu/xLUNJtiaZoVotLyQxswsgAJpZ9KzXN86x2u8xMyy5MfLk35m9Moya3GYu+/4yILC27U0eu4j9eqcl7KIq6Auicqvj3pbr5AMCt56NUUH55WPbnuuFhb/Th6MLpQTO9K+n+NTLqON6ZnKcwagDDh27Qsl3MACnqjl3T38gTwfQUv06yxbV4DFeoPPn7F16p6tNEfQzpyccJaws2XZEUI1ancOIaFXVIgdwySnzxlJlP0DDa5sP6/XtYfmtXE5cAK6nEvIWAxFXeE/1Rl0dQ4eNqAioqCRrRP4e3c6o7fhQiBQuOd3KhQ27XkGjp9BBFE6zQMGAbh0RACGGxHAaP/ety1PulcRYB0L15ey0RXSDPLSJnq8Lq1DvToHZJqioU+bwBLiawcjPpgeVmh6Q047XAP/yRFy7X6dFZxeeSEfSp/07m9Cr5fSLllaM0i2mEUYHaTv10riIdfDiDcsXmM3Ltm82uE09rE2wgSvUxDhPPKj4SyNkQnGoZrj9vqIr8prtOpfPCKM+UDNHWFrA9fyAIt+9nIM78wWXkWz/yc8K35kkwow21cmcwxtWbck/VOKWdEtm6QbiXESV0UuT3uRRFM2f3KNxniHBSTkjRvLW6pRNG2p5NfJ+XmGyWFGlecK8crMd4XiVujWEoxr6vl6wtt03kP4su+eLqM1ZxSqnKDdrN7v5gSjn3bnYkDBlBXq4zAPhl+gz/777bzc4YEHiK6jol0o/qKK6mKKpOhBBrMQo9YilLwDxvKC+UVYwFvE9slC6bj95pklp1fUepyD13AoAwexKcRL/JHlHSg2HDd+nDrubcUkzetCWBdXWRjESfGICtIOuKq6sMKJ1UrgT7vg7bCa+ukURUDkBfEkZ91Ku4qM9R49Uc/0fizZ4KxIJuGH7b2Cw56tBqK72RNXzYouD318+px4q+edhbv16Y707I3EO5QxFXVWGAol19WGRTgaeTaPs/V0HMvRtlqXR9wIjASmFXnMpuwtXRbYuId0UC6zkcr+mqGs7hURcqxHDleci9Dy06gQIjUYNXt7jRwQDdFr/XRt7yO/zb4s5fDYRsYPgYvoaX7aguYLJXEVEQtB0/iJETMwWYg9QrUfPYRl5NW56taDvGCTzDyXu7pKHbF6UFJTbVOVvVQgOqi75OpBW9XH4mXYXb+jslXQopVDghByb08vIKUwhh3QWw0CgrC1vgW3wuVrzQ4RYbmWDcjofLjFc+yMkqDYz43Qi/lzI/qwwA3SA1Mo0oNHQhb7hqbC78j0ufdg1yLfrM+0SuJPYBa2CAccKnfVz6veWavz1+4GE3Bq8iBymPIbYltQ6AlKlB3PeoXs44CUratng+3YuFnTGmX4SVSDvdSNaZsA97UX45eyW6rUopuix/O8uH5q4W40bhwSP4+x6c02kh5nvdmOn9lqa+n69viI+ykbSPJCgpzwYrK+tkzWBCQjGsVLStQXI9sOiUpMpupsBN4JyGNMTnPDuOQ906RvWx0qW0fnZiuDlsPWqzMKiUDzFnJKz7olAnsQjKT3iSgoLS9goq+uLoadNYnIqCUwVkmL3lsVu0HW+Drbsm6GvGn+WtxcW8LtBgpAeb/DPgfi4iBmML3h6CBmkfcSm1EIUhkJ1Mm66MbydHzN8stPdp8re+lNZUAV6NHyHJVPAbPR435nspA1r2lggY+A7isR+oQJO5CEtgVr/SghLO4H+rNFS+JGF+FAJyfnYabnevSBW8oT4U/+wd51UQMj+ca4l7l94mfvIMI4eWBP7NZPqCULA2cXZ7tOCvRf9Gn9FAWC+XQ3K1THaVHLylefq1CC1JNsjLYfP83F7yN/xgu61XvNj+7R4isTBNjKrPI5ioBQPg58o5nmsBv4rb7U3GhgVNSfqVIzbwLb/JMxC+gecKH1nsqSKpFV98aDLyGV0x9Q/onTsbZnhoMCtf/GY1BaX4QSPn7qDESyyBo7NjkeQzD1/HAmlD5f1bME050vqyXpxPP5i8ExHGSio/HnprjXI/GGZhn1pNOBwGreG/oQDlNUeGNncj3TJ9rNlQ1Rh6IRrHuIY3u4vkGLKNnsduEhLnEGXAS8niw20G0a4TcNuPcBQUanMhWGMcbjZXlBAsLC1w94ZPk4CNf2AcyXSVCTwztanXFuPMHqR+KlPgeG1RjD84h+89800QgbijCXtQYmHSDQqTiKhWwARG8sfX93/rYG/obS9z6OtjlHcjmBO2Xac5WKPiiPt6NSguUqjYWDv6jLGnkKynm3nFSa/C/NtHO0uYtk32ob1jkihLUSXphS5WFnIDOt/hy8Tyunk+YzArR1iIMzUTUhWu8hA5M9RAjML+4Gd+DiWzO3GvuQheUhAoI5OGiv6LUVGApJeSYsmBu7JfGl40tkVjkxPV5SfZJuKpcz6dITloOAYAQzFgBOvoxBS/VrH4hJSWq0ByfB1GA+KypC1BYE1nh+xK8WEizlDSqGFo+cZ4e7wavgeFmXEZ+4VEEid09/I1FrLfFIvprUr6WqIi/ZJm3k8cbcAHuQtNoCIbmuXUiIo7yeAuImYa/FmHMdgtW89QuFbGhCX056AeAKUDfGe1HRDV0ug0CB1vJ5p5w0fjGeN0RxYFrWrr1VkooQvLLrDq7NvxWsi5EJiklzQL/IStEPwXRiMhA81k66jTgCEWohArLj0WUyaRsYtGe88YCOzANtbJ2d4osZHNTfSfpeiTEDipz7KuCcqFjImVzGNbMiNB+VcCnxhZ88XyY4F5p4rTpj6y2atGp6NfPKXl5AAEQfSIAyySGvD8QfPFqjzc6YvIFo+iTLgab4lXZpctAl6sQuhp5b7fr4MNHMv16+pOFgKk46+H52yieUO8D5KgVELHxav4TlAOkcrniTztRMGCjkNZomeX5jU275Wr85B/XatTa3pFKdRsE5Dk4s7/NlJgrhc6GH5DQSUoGBxYjVDeT0Z9Ce8K1OWM7u1kWfJDU5JK2NBSVzhbGKgvPCB5hNfSb0TS7kxpjvrV6WoIjFy0dC7RHOeGTFqDNzA3rHoWPmJTv7OR07WTWS3qV29K3Ue3URrpBAhwsUzLsNJSZoJzcqu7ad39YcD6Wy3wCN7Pr1PSbtAUmyIVV4cjwO+rSQcnUucbhh505iAJsXkuZKZ/OwonZHzMRJD2NZEwcv+FgaYEOwhReNB7z4QltYvzjzdWtqoV3Fk85vpc235qFPFdHzr8LDeugtGXhDekqWFnXjWsve7iIrtTS6lzsnxiDnFecMVVFxJdM/3DztMO7W/2VQAOFyA9j+2JQ10S5m3WwvGyPyVdmHAqLXJtPpcshUJ6w29op/cA1L5FJjX7SQOFjpnAFmtypqSrzqIn7SnN3Hu3i/795XCHGGAhKE8G+HfumnmXOIggspavcCkT6USu9qooHiFFfFEbpNnJVz2jcP80n7Nj6P9YK4V0SkxOmiks80ht1vGDZ68I70g3jiSXEqGdW66IZcvbjV0yMU8NYa9kFa3vy+P9XMpqxKJlDeGC9yMkjVMh7yGXzxlbVvDDKsNHkbb93b0WhyYcy1MPNYhj/W/TjTDp7Y+Gj1KsPm3G9OoNWTt7L7wDTSrVkvOue6jCMzbyVf+5HdrMGDoI+1ixVa4+f10aitvJ1+7Z70m8sLLeSLfLcudda7/rV4j3uQBaKb9jPvDG9JG5fclhlJt4KGa0ucufUf1iVM3+XOwElK9lRl5R1XiHMJk4A7xS1Z/U9WUeHDYHoU4IYxZwODAAyls5oM6dh8uRdsqQ2khmDeBobPNc7cxiU/Raots7sLPaeeeTVdtV26M+dvpLZZ9zNlHU0rN3QVL033FZUdcfv0DwQSYPXZTi+113vBdaCpCOF2iB6tRJaomB56WL0451EvimaYcqLW7NLE6wCTOcxcJlXWwtWmkE36m0UsVAkzzP4i19VO6lH7yLmZ8z0H13llcA0wUah8Xp+wQMe4SNJONKMCeuNjJiPPDzd4ZZbNNqE8Kqm4K9ROD+3wx6Z5vRJHVfbad2JeZXbxqyLYh/PGPIJ6qvk7HzQ/npI4abTyCqeHo7HJWkkzeds3fIwXkjUujetMyfnVPoOkmYZuvo7Fywuiyo8BpY+0jx5H3Csv3jBF58Yry+Ql6cncNppyNqw7jQZOHzs81D4xxcAJL/KvRJbfRtc9D9mBY6U/eV1hyJw++I5+eny3I6hggXA9u1GRpkCZMBvypHizOfJ6oo/qDIH3YljXPkpnXVRXGwAu3cpnVW6T4BZeQK6cKgd+9El82bfV91LXG3jXNV/zVRaoWAHFvVpjrbORlaFuAPjynWI5m9XRMqiu/DN9cOON1QZRJje2EKgy5ZmfaQzqG0Zx4w3O8hzJilypuW1e+8YaDtVovUCTLGOPqQhk3mV18rFmXGBxQLkhJQW8fTTlkvQTjMHZDU0KQmRPPfH3R7fXC80yuwWWh9UT6IQCOwX3tRv0B+S5gR6BfuRrIIExyRoYhLwIfEmJ4O2ZK/NKOl/QvdZoUGVC9ip+3AgaQbyl2be12VDpGsTME+4agst5zOkdPdcut4DXEKmakn5xr855BVEfghkTMrh6xFWGGyoDtPjNNkvpuXRLOrUnO76PFesxjxjaP4fMwwR9Ki5Fcxv+vcjLmTZoYe6dKuI+/mVo7qy4PJMlbY5reyy5vuUE74H2RrwuN1yPWAYZUmbNR9kMunw087NYV8GLSCezHpB1CoNkPazUmJHwhpvkXuY5d98IRB/R+gbMO/prg77n40oYISjSt7kiIGaseRYqrmz9SyPfza2cu2nwg6Cb+jT244rc7l7Sv+xzR4WIYNmgA35pHlfjl+iGp7OmLwRXmKrohIdQtzW8PsZ+o27L08qLUqDCsIuggEICb0l1Nbst8nEOkORUUR2zhpAY+7PtUpxHug8ztaq9T5j5Mdx9mFDdXXMcsUt31Pp10Lkqermnrz0Cu/FzP52JwF/XKB4h8npm8GZv3Nk36IudvYsLrcjHu8fSu7c5giuBq6dznUBYZqJv6MfVxiNzMrVle63G3C9xPG1P6fX23Rpqobi6bwgCo4nlObgFWjvoi9zn+O1wmtdyyxCey7l+TKU/DKNkdFFHwk5+IetHManLYKb9mtU5VxJgHzzGkVBCQ638ZUPddjlbANvk0d5v5dXNWZV+e4DzkZ3b0KlK0L5xXgGH2zlrzBwmt9n8norHnK0diBG8NfLLWziNItsrN+Lej1VV0pkSpEKq9yF2pvlKSTUn9oFW8jJOtATEq9IPCwsU5NOmD27YMihgfFZWhECZhHjR7a8rr4Fv/G3DfWVDKcxb1zbwwDG7b/CD/ABjSUHd9+ttu1s3nw6/+S6rfVP1P89mVz5tezIaGoDwv1YqASj9vPE5j1eQLu2pWijElD132RKADktTzyKhQuvVdeGUDfacdeprLqvmRmEPQP1SDqJU2oNul3Xr/C16XtKbmJXaGOLsFLQ9491LG16ZVkbEmD7Wuf80FaS8Gs1UzsTC4nKmSzSD1WYzBfJoiblbMHzAX2oyKEirTHpH1CCaMEYZ2BMun3dFs3vjVG7I8ENR7qHrpOU2fhEB1r7C9VarHXh774MB0aPvfoo1UnnqzvrBDc0esPDzxD8vPHCn4X0PRneK4kg/PECpBLnM+O3LCAYihJk0o2BoIUQjw1rL4xNHPhe6YaCeQcMWTBV308470MKgheHOdUfDysuDJpPx/XarZGbI3hhjDeSiUtiRdjo2A0RNRw81u6Kz3+KiCsRbe6UZ7AQqq6KB4QrOaPgeK3KUkpDpPWClqoRcIgnyN7ptfLnmUYfGEUQ8dnQjIJJLobju+N3f5EozBEhukhfeTKTxlQEAOLKi7ydgOufc8CX5ZFHGmerB1sSzqwQa5gv97n4qqzBLczg64iIuZZkLNoSkSal0j1yGckTvQr2vO+5HO3PtZlb4ngN/utVSY3mTDp23RXD1lJ8Z8JSIafdQ7N4/n6L3eZE+dypSyow48k6xpp+IQAGG1y1nzk6UlGkObBt5z/pDtEml3Yk+fxikolQThsbIPM6CkUvC03s2stA1e1X5KsQkiDCIzHymzhlXBPk0Re2SJbXyAg7aBCfJ+oBYktsGrzUNezOzUFWTVLGMekMSOWxaz7nFE+4JoRKeSlpfoeZrkUx36Y2WCLlZp2T4VBLkI4uXtulBzKBTEF1neiO3aH66MGA26dk13nZCdKhcfPACgw8Y5e7XkqTJvhieLXyIMn+Y5uNnPu228sF8miWx1J5yzA79pZfpH51M8VylVZoVeVIj03dtmjQSKo724FsXn+Uk1Ld5V2qXhXiI4nRy0K4vlZYZb09JmbIgu4kjw8UipyH/0XMtmLv66FGUV7uMN54m8OSO3Jwlgp53YFLuen2M/WBVjdp8MRcmL392FhE+o6NvSeQVX1y0HrPkB8qszPMivVju7VywtxUMrF7voeu3zaNezqkVRWpZhdkgBwK82tAFm5vDPR/AG6AWJNV9xLoZE9F+RHGLO/LAw5VvZziLImKMthFSp6OG8yD80IvhxPiJf/Bf7ms/zlcwaynvmkGidEl2rSjZycS2658oH3HYW7x+0XKEq5BSNDNTxN4P8X2dYNCF7jOQxi1t36enLu3LewQPBwSWkNT+0gPbQDMRB6GzgilBKzkcCFOEzeilIORESLdSlx0bpAjYQ8A7/Ah9GLrAj1t8QpykS8t/5YwB5qHw4aUBrx97qxef6aE2oobswzToFXzea6GlqgF0faikHQHf74M2+CE2K1+b220pcqduc1n+lEcOZZRVFNp7dp/aE6DgaV/bF5wn5NC8uzrtkxkY1u+YhSdQpqebVUiIRTd6V8Z6noM0dBx9mdE1rvoghJ6s+c2Cj97vzDbIwkfPW8i80Ff2tSHlXKlZTZqDQcjVq5Ysp4OQNaKKekS7DDA63UbvxxSeCEdE4PldPsXNLRJxgxs+69PNQyIcD+kkEKyRE8YeSqRPpIIp2KZBGnxox5IzTpYkW5j+ABMnfxvLlBei/DCVlZGlQ7d8HHRbioKWxE6fOzy9cOtIH6WWDmHZ5043a3vEODLSyJuo6BdzYE2tFZUAoPRmfHOnlgeaoKZufvPUgucUDSlfzXOCv+kyo7cbsfyUQieh36qvogoMEyLHX6kRRWU5xwAJUyXVCom6tqnwrcu+SckXpUXQZy2V6Ho8wwyJFDx27P34AvBmMgpL3585BRK8vI+Vr31PJo1aS69mjtfjkh63YMv9KcLA9A8FmqV4fftftn98+Kujy4P4JjTfn53/pcoI/jSxeT/oH3JrEIy4UYtBjGsEa86LtlGMzwKYOXbDavNh0rUDU0rhWRcsi9cKU5+sChMf84YAoZu8R75sJCptEQgIKoPu/ULnfXJNEL2topm9a5G8WskQlpIB7TQZhlISbTMnRP6ekeBxOkEYJvDJCxhbICC2Xlk08jarpnSdouP9oNE1h1qUYhq4fYzEQ50OnZZzNNpRBG3WY3iKHafLHeF7wjs9rlfyKKtsEyaIfb0eer6jSt69Jk+PlDwJSsDem9991auKUvJOuyHY6sN084aDT5RqFqu8nSDVLTW1W0d5Pa4idz8oeqimesMV69N22RgmTia5io4RZomQN2joytCS8WG4z80Q/Uy9dT07ZESRFVg0qbDaOTqZqz8GvNNHRixDr0mWHFXMQCUwLqiXpBOZPlk7DUxN2fNGGIJikBY1ivBO57V0TQpJYyKXarDNtRCKvsHg7WRT+3BYjRt+H8Kwy/ry9XFZ+eXm3gYvKIEZ+manwDXxfV8J0eiT7/sbmF0g3IJYrS4hxikSwnGs1l08nl+Cz7HMcpyAbl4579Bc9h5mX1++IvOqbZSluiHKqEQaYVSmhmhuVkGK8oWnYV/cmwM4x4DJWFJmxM7OYj92GynSdp6yN0XxZKZXxVsAgNd/MNaR5RYiVtj96c731M4ykd8dc+XbvEgM2Jkdg1igAfFUO7GxtsnBTGp/pabqHVJy/+6xXRqstt92rm1KMED1VUWOwPVl2Er7J6fr7IwKY2sZ0CGWjQJyFOeu8EkP/njmWvOBlX6YiQIP25gmRuRYJwxOqaqTPe5YDRaI60RL90nJjXbK5NBmdFNsiEJsqqQWBEV1RWWDA+STOAr80crGt7HX++KpatvYcPZJ5AJq4anFIWWh7RyHKRXqCPHK14GeiKu9zrW9mGmOk3nORfUxPt5onJbP0G1IELAQ5I+vrAaiqMISV869p6sL6rv1pUYbsZSW7xPfQfgww0duHgEeF+uz2LCaM+wY7TolEA2F+IyC4p3PCRrBFP6FGT7Zj+Q5SzAqTHVFfrxcoojxCYx9vQ783AmVeb3yMSAhslNhYGqw89vhPyQVVrdRcX60iJvvZnjDyim7YLrdYm5IjHTBDlsP3iaLsYAP9nKVFJ9Hz5OXemKIHnmsx6MkRxPm1xwzFfZpv/ge3B56W1B+3drg2IVQ+hpF7i6NAYo999wziRoRb3ndXYSgZw/jyVlzcw10O5Lz9V4aj3AgXV4oM3sb5pHjbjHTu16Aztg5/MPR3u9bMjtlUl0B8255KLyR3At4zzUQ23DjfVW4ktMa0DWpAEaI0jR8aPTyYkTs1TR4hWDQvkANPLwW/cEHm+3K3kZz5Ar0tmJOes3o2NFS4dyoPysC1KHgXXjskdCSoq0R+mZKSk3uJgH2ckwHaEHe3Pb9hhM8Sw1JdiparGhrz7/4bm1h2zQ5JrrfVFSOxbw6sbEP88D1FVzE+pgfn9FMC5FDPHwKvSfdd5olczlWo7OAOqkOP16PlH0f23AtSsUDm5DOJACFXsXnAGI7gIjz8HjdUBIzeHd93FPZaUggUVzWPaJuidsuN7R5dPjX+4jLkRJBwt7jVDvV2oH/qTRNm+bV9BWqOb13mf3wbO0IoBJaJ0F75kk/dN0Q0Lqnvw91O54iyK/IUytYdCoulv2Gsr3+0QYYC3IMf58vBxcedvK1kFleI1OpGS5ASYIc8cvg9sSeQZ5FUbsRJCR+ZgoufpfxpS+p8/UEsV6nOnD6koU4SFw3ojj2Q9qNrE29t7G+r/rc+BAcmclkkgm8qO5mW57iimVv6Yamz3NpQr1DclvNYXSCgbduOC6kQfaa7bXaz1RlXwuiEWK8hiQNCffdLtvXNp7EVgwMYRNpelY3srmClcvhXCmn7Q05z+puwjW51PQ5yblRJ70P1taJtpoZ69uWq/jWItV5f5t+JUvSsxNDn9ucuaDpBAAz209dxlTXxxxhYsyqrEuP14S1ol/NTuusTX9b93hMUTbBpOmA8TzxdvX79pbRjiixFPYudJ6Xxo/YAK6O5pmtrDP1LYChofzZ7h5epJ5gx6+pvsSOkiNoDk1rkqB75yMN8htOAPeSyQJj3n5fm9q6D5rBCa8z8o5BidviIl8R/gEJ1aRl3KOR9ErTzgThGdAYF7Hx1gv3cXdnsHoJ3ahGk3Oeq4sEWomiRmGosOi8VWDiVLriT0S/lB4PJ6McUJpZjDJHOMPaPW4s8X4vRNCNcQJf13tulqCHx+dzXpXEnuCye615vaQCwPscEqndO4AWS4rD3untgKLebo6IN7Qi0vqRbB0ZvIBYBCkWDKcH9lZHHjiG1pvxXgGfv8ju/rMAeSzfugzYm7FnkpTZKFET8dj27c5GFpKM5uBPDwIT0D15wczDz4LDWjOJMisg5AxZkudF9bva6I418xuCP3h394XwvNy3BFkeTtgmlYVC9JLCbmjwTTDIE6Tb4YnVvw7R2nhHMNTdGJxh2Upnc3wvlWvfliCVVj8YaUDY5bzjXHZolnyAITEegcFm6TpGGGPgeABcSjs1L41CKJ8ZnZSgHIFHsKmRhu+NVn1B4WF7iK/36b0jj+fFArbIawIYxT+O7iQyrXPz7CN6GQpl9+v40DCrEUSSmXWyAR7opH1XE8wSfV76fdulWhgYqA2em2NJJvfFqiu2IPV7hp5KNL6yahtduI2iElh6xKwCTJh6bkbigqwcc5BF/JS5zvpucz/ZMfIG/IIGLSBg4z3CpXg2wLF25h/Bt1HCcPBCByhMaY1zA7wu/jFLqnericw+X3kzY8+HBw3FCz2H0jewEkOCeOnG7nwodYzPfC8NSDsiyVsQ7W2MS/25DLfgZDtdgGq86WFXpN21SVjB2tv4QdqT77ya/egRH+4LUPuLVbFokhSf2NHb9H+h0XOmlyVBs+JAIDx3HNdfI6mDXbdlcAaEmoSl5hxel7H9oKiaggyLUJbg7SvnonO2sB2wb1NgAOOv52eNYUk33rnY7/QjGy3gLK9vw0rBmkVrGREgvdTX7dCHWceO3W7eb2PsOwGonNvy3i6Syj90NeJj0iuaBfmmzCFT+Pnm7BeyNW5ugOUMkwvSe9KvNOGfsaB9kqZzHMUYmgbV0ym21Ftpkw1+d+OUaPDcA6aIJhy8771D+/A5GQaWCNK+nEILtqZ9ir65YlN6BDHy4LmiwMLYJTSTVaS5b2d/3T9ZKA6CmU4m3L8eOXu/TWrTqevhOmL8bl8AmTnyo2o6euxsYsRLHPqJ3Ew42re+hU1lz38yEHSqI1uhFvKkd/bhv+FvZn6XIJ17fFWz3o2TxKN6DnfrJwK+v6lRLntdZ9s77c/0xDAvKQvhSTow9w62LUbUz9S1Hncbzp9KmfsTgW3+Y/f3UBA/qwvscW05GcGX5zpZnGSy3VQWeehcchkiTfaK+pHohK4c+Imnpc/nc9w1BanK7jFRAMX8oQNdSrcxupZ4uC3qQ8YGISOnOAqr6upGjOAMVYukIzIxUGcTmKqPYEvhjO8V+3G/3WYWyt7P9aVJbTISebnnw0FIcGAShKCIyeQJGnPjoq43KnvWQZwdrwZQF6TblVRvKWnljY9ZeETde4jg+4uL5vVc5jEeBj6EhNnl14PnDM5LUKMr3I/xXfqRe7+g+yat8OTwSxVKs4gD6oNhz6NAz+KRy7Qbi9aQCDhC9OMtRZ8Aagzt6WeTdmSb7mrlO8fohb1Y8a3+vFQwY/Q6HmDu5AOI7ggZF8CydnF/Td7pI596/NCwaTT45yN1cvFIY1wM8eTq2wWdsOJhK9ONuRYluwo6jZLlNn2W9C08e1jfJLN5PTHSvq2iL+aGfVVpd3TSZOkS0GRaDyxCrThWSWcGT5+1HO8M4Of4Se1AkzkOS8MwN8/c6MQuyb9rXtKL2BHz0JBc0xYkBTc8cHJnLkrsLG3dEAIS+U8SDQeOm8/PP9WVEsh46KXTS2ala3tPWIfV4WUKBAQVIB+rMSXOAJXPyyhmNjpDOUHUl+fxphnCXjC18PJ8uLkqLn+3Oz80p8EPbnmDHRmU285qLSehHMScRPjQhGcHYkfe/gUPZkPoiBzzngId9F12Q19AVBsAj+Lqf7RF7rYMf0VZZwJVtEcgiTr+IZlCnXGhnZR+b/1Q4ZBb1OcvGvFZ7yDeRr/1wMKYom4m39PkzXH0rK/T/SiZ/kBuzff6znaYX+8mNXLf2UUJULH/ZA38RDhxDkdpIJ7Jpo8yFqrVYWlNF3fym5foZ2bxvtIlv4YsCA4XbIR7AL3Ckd8JRZ56lmIA3FC2XZPMxp05t4/Eqy4ZQVJAEMrrVbYld9uBiBBpgWhiL5BN+MzsrjYre+ITz5EP2ovdObzYYm9Q/EglQeAxmW5PbodV7u2/Gw7yUBP6ridKkQ5uedmUgkXJKMGnVB87bpUAmzf6x1M2JglgSaybJ7a3QLCIQyveLi2JQKtNWyEJVOJ4rQFcRPWyojEzpQzVNruKfzCFIKSyds0WLfr5eZmfANRhiIuXQjbQGKkSDsYOhafVZ7Xv19LERBLoLm4/KmeHDp8HHV9d5P1U1xNMe+UEME/8Rr9o08s8IXdWPpRW1j7O67AiIdZ6GnUeJBT3NSX72AkJy4UdVLeu0Ii5WOnuqOEn7jlcHyGEtJGP/IiVLjBJNNcIwCAxqzE1tQWjMXCP/SHML5FauVrNOJlYRchv8JDhow5TrdLv37em2tO4Immnkz8Mu/WOgpZWUR7YWoB83DRq0BxN5SXc082p3EBJxof6K1PPtPwAHRoFOQOMPcGsPhaJdrslVNmteLmxnCztRU2Q9abgnZ3rtUtvKWS1vNrxFCvvo8+gUmYRKg6dQXv5VBpShcG8z0kYpA6XuqbxeGI+G6rO5L79zH4vgIWn6PaDZuuN+a+Otp1Bicfy/m3hcrp3c1guHDLQIPbpBXmSDOt5qavZviwYjacYJLlOfCjNacatkw6tpVkc2yZXMebqA0gdahbli8u796lNt5JvhZB03orU1EXWmmMq1OXshel3sgG59uSWETLTbBzmzbk8SHnpRM5bpNitDgJyzHXMjVGFKUrqKQKfBdXtzd4fWCOVVGk6ULSizBhnm+C0jAHKnChFOgR2ULN3S+Lf7D70DiSERg7EqQr2IKRPfkfLOF4Lc+Ne/thRr4PJXO99jW93bvJgKvmf1MbmUfeduOC4qjz9iYlkts61upailmbad6GuqIbrjKl1OvZQiFqSoiwRQw28Vfk0PUuRLeqV+jO2ap4/UStiSJ20LmGVEXXfOicDmQMssCqjcSJyTBeAqCK9zmd5tnueR8nzUiD+k0KmdPe5NmFI32qt3PD47HxT0FzPcmdGLzKqOS2lurRpWBFjPOuarUqlOotXn9OuRWdFV11MLjCJlXT7AF6AEYoVbu/yFAWCVsT3Kq2lTw2kUm6uXvC5PjQSw/kaGPDcAPIy7ar4rsWiAhZccfO9ACHgFIkR1nN+PjGTDkef0Hp+kbrJjabppD4eNt1S+mNKQbVWO/fWJ2fcVUnyOHLZ6e1Qj82L+tM6vP6KN7tnU5HjopzL0zeEvx/zO+g2r47e5siC4OduKD9sSyZdy1UjX7q2RCmMBNGSarSEyGhCLnGcrMlidSYj4vtROUEQABCNxEplozfS5n26jzmxZgshDVA39KqH9WpjhEFfJx/rsX5K/eSfx0MK5tgRgh7RQCgHTQy0AYRXavLoVqeC1NLC4btUya7MDba1/rmh3hs2rA7YoRQn8LouAUiHa+zcPV2eNE2x5cIEmGn9wToXILFM8vijXbt8uuv2jM0fK2hiLKLjDblOxzpUpXUU+8mgYP0ImBWhCCt6weTROK/WdsIOp9/seI8/yPg0Ba3ao70V31WBBr5Ti2VmtpSySrYqX5IkobHRNi6V+1qF61OOtbLKoUInfoRHAYXuaiMWWgXULmFZFh1Pr28+m1JjbSUZDcVxO1kuO3finnfNVDceB+OFnfYZi0YyIgyxw8Oxa958liZSnaX9ZOLGXFXsmEYp/rhUYipSgkD6mCyxwueiXEy1O65VQmIU4wRj2pCfRiYYdDyUj6kYh9wfJ2C40RcxCHTL+TjvgplvHz+tmqsCK0nOxBxo83RWSW8udTcrIaI+MTB0XMqt+6wE6tZ/4lLFdK7A4v3A6gIrevYTycikdaoqspeCE5/0E0uClw/1AoeziUTGdK2b36D7qhnSq/3GsH7jpNogLjFFcjz6FTM9t/g0s3MdBY4fhlzBKGsDrXkjowAzSlS5hmBJ97bdNAtFpejNv+jXGLyntktldOZ0/9Q6cQRuDIaMr6/QEiwgRLwR4IK0bwT55fJG10xoOTTj66Kmsssi4nmjQuAD3Xf13FUyrqVbNGswdSMYRly5GDsJuI6T7+Clmct+kWyvVz9OBR4aTukp4XIryI2307IEn2nMtE/VQwwr2dz8YrGqpubyYJPHR7y0iz2e5Lk7NQTTkJ9QZVA+uEm2KUMNL6M93UuyEp+R1d2vTSSHW4VkuUEQPtih96x7j5Rebr321CW9dPbzbeXpXBwKEyabVzLwyJanrxXAwGY31iYXDoA7R1OneZuRN/V67PhtTHpxaFq+t0UNLLEcjlM4wliBZJdjuLTDVvv7ahNyvuASy4+tK1C4RXvdLDrDg9FQ+QE5gcgQFFs+pXFVrRkuL8kImve4RNZbBnTd+TWw2zyCKVWuwiK8rRSCapmjiZQ5zzq5/HpmpQKKl+K1fCfOSMolAqs2L70ASKhgTxBiQXsmoUUgf352n16nyiSfbbGIfQRDRSejfFkYErxb2K7NizjM1Dpa00TG+G14JfvrMRbttedBoBeP7zMaiXi11ihnFHW3/hHYbUox5M4iN2iU2XKXBKU+gKi+O68DTGOOJSzHdjznuG5zpzVEJpiaM4VlvutES2JD0MJ/LpGRgLx6N5Zl/kwGcKPamrqVuW3pgHlGR9ZIMcn1XYDqeqx83jQLIZD0wys6PaITUrmmXa4arexLeZ7GpEJReWCA/TiP/LbMjxSNEWYR8EFeDSpDThwI1g4SfYQJno+fZdOnnSqgFv2uIk2pVjh+qbmMyL/xSe69fVGsvygFHPoluFV/ZxAA6HzirC86p6nS4wBr4G/+Ac1VX3Qe3nIkFW1OEQ6SoXas4MHVz7tyIJLQAj4gziRSt9rcbRfZ65VFYw0h5MePTTfMR3JWFMR3jSgM9jjrAsm0KzbHE5kOcjQ8ZX9ZLvitzZY+eaKDDo8eC4UO9czbogbToi3bL0HvLuixnB/itp9V4HEdmuFo/NF9PuPCAzDgm0vXSgwYkwIVhtNiMJHvCinK52DW81lP/qW+b2z12N2peop4vc8uGNrtaLRq2WQmB9ZTp7sGOajHLkATYXy090tZR/0dIHsquobcGybC+tRebNiyj5s3vQ6EhTQRn3DeD55+s+MfzScQ8cMhSwZ5ol96fMinKDRzt+zR5cFIPPgbhcTz3fpdClAti9Hau7A89wAd6ieesWcVaGH3bkS/txYlmJGGohGkdN+sAJIIAAdmWKkkQ2cLuXzXWkxgiZgXlXipC6PDxxOBn/FZgYbyqDi6ZN7vRN6tDdH4oHD3HiDmaUfBrldduaTH5vYgsknXvk5RTTzqjxixdlDWYUPkdRo9Ult7WWKZNLlHiUJCsEdhhmC1xb6NLCLIDJOk4vGcQiHXfQTWgUts6Z+P+sUZuRnQDcoXESntt4gGPh+FFMc96HFRbvqNf+xiTpe3bJQ8JUruAaw+HZGski4YkicKYZQiLwr/qeZ3hWjBskXKrYxaSTp2jYarx6FTqNrSAYI9Lva7UC+K5OtWpRNL3fpSwFesZceOH6gmPDyd1DB7f3w6DsY7aBFfq271PiVc5bJoT3ERbyluZdpDhAf1eXqxQlJ9YUQbyUPijJMYv9zStMEL9OAS9n9j7Ku2ZLeyLb+m38XwKGYM8VsIQxxi+PrWjmO7q/qOri4P2ydPZoZgw1pzLpibQ+A3jaTS7/DnMKmJl3R/sWvh5600hMfrDrdAqA+AXTUSpW/881lvbLhBiNp7/mRYGBsLaiOKL3VVcDSVhsojELOr4RsrTSY25/eEowHqIsZrq3tByyuMiugCWhF2rkr3IV/W3KWixRvn0WfygyIWo900X41figCxXf9eQfQ0qMHrgp0ah79a3G8EaM2vB4snT+z+VRaw8mu2ZayPRd50RYfK7emwQv3sI67fkiVQCp2nImP2f78tY4P1hXeseuOXcsfUfJlFgBIvpMuDfc0iI8f1hf7m7hrsuzyvTJH/tHDxFklyUFinosrVq1qllkjXZ1ITbtv+wioL3n8/RintncKOSDyuOPgdsv3gYDMS3wV5f3opxNOZ41EHeJO3cmS/nriD2UKWs2TxQ8QN5j+PVqQUHq/ebLpvji/i5FM9HHxHGolE7EH7SLPc7I+Plnu4fvP5TjE8tYSN7IxZsO4zE9yMRszTyZqandRMLKMgi5Mni8EGMNATYw3vDcdeOGMP7wloELlgONlEN+crvyT3RdG2pXYiXKboIMAxBZwb6BV6dZsSQaeZrzV2Yv17Od3dJmrNQmlafQcyQjyGIBkj0pmKFv1AL/LW6Meju5+6kbG6uLfDfssGrBnBXiLLh89MqA5Uwq7D7NMEZa1kLn1NA9phkM2xiUznX+dzr+pk8mtBaQbbAF/mEnhEYF16cCGOFD+AvwdEWudo+ZP2z2/4Pb1rdwD0s/8mHpsf9liWIDAmPlu2MtXnRdJJsfA3ITHEDjK++Wtul9Sk1e9lJO8SpTeVWQhQyDz8omK6ATV0rneDTpSSn+7lhwIHCYDT5a5WuudNxrYfA00C3FJG1Cf9EWsEPm3WH5Ko6Ha0S6SBRgZrPlUO+NnjCwrlyxSKkpoXIVfRDx7KgKwx4pvMtBGsD+8fvzf+7fdYIO4H/J7zP/yeW7ncZ8mF6iiZf4tK/erdQV1o8zsCtrosQT67SoTj0ngd9OMCIDHMN7RIA8IeZnL4yKWEWL9HQicmf2/luHxvmFY91cWrEaQMqgfM1F5LL6c1/Oo9JKPAPm5z+zL/k4AUXRa3sUnC9Hd3JyC99XmIdTTCBGxgtM7YekPZUiBkA8RWbpWCjVHELtwHsHkHznoYOZ9WGF32oL6DxUsbH9xv8jzWKC7JhEUCp91knATKCc1kOa5pAw2bisQu6zF4CfOafGvdCofqkJfv/Bntt8H0GbeKi+FuqJV1WDoUKvLraKg45HsDo4huqlUx37JabgfPEJ31sq+K1DD4GTGTcRdb6JDs2KxNcPTqCQcKkpKoqV15HPW6DmoYWSy0VkbKxDkabhp3XCHZFAsY+E2AjYn74q5fyNB5oJv2UJNv6r9eb5itj5MXqR3Npkujzvqy3VPOsd7dLFzNFoQX9KXX0vhjMDdjOZRzJtEqJ4bgKF9tvp99iEfPHEmazutQOo9RJxvLn/UEyIDB4by3cBVd73Xn36qsb7wWpuKba7BV8H0iaPYtemzfPIQ85JK67XbznRTThPcvqNwe79QqxHYLtyrAYiu+dgO/dToQCoR8bOjjfYV2jAw8Y2XTuMeO1/rMaUyM0nuBUOY3GOQSBrKlrE10Ddi5AECPBPxZNzC4JkkYfs9i3OrKb5JIq3j7tS4rofyLvc+E7S4gg1E+MPG3vi6t07VUONxDqd3qs35IrYWAPiXriBzEc3MXMH0jn8KIEPPo321TU2FioCKsIv6FBdBqaRDQFYkiL1i1THRbrz6lbM1d94pYQ+3Vh+1Vqdl93hLHnIwAg1KA+oEbPrv5kzfIL/OD16LEi0wM6i7H94jseGVoMfaYsahn+rWWL0PkY7KEWQdUm4s+o5ml/xZ5F5iobZdL3x3ZZsFYTKI+xwfV58QVplOOm7iBXLURQ0gtxEh/wxASBH6wFD27pS/XyTRl0M52KZoMmLNctvOLVtjD9lUR3vIsyjgkuqEkeMGhrX2ox4DMyGCQEdpZm/pB1wtu8sBII2kfMmzsTjP1bij2gYTc4ybM10iYOwmOtepGrM1mjqQ8rrWGCaUan/vKQK63OC/3CAsfZo+CRo7IhQ5sQdaqZL0Nle2JJeDs3D8jZOXwXbdVKVjBIYJedTH/VvvKch8JRjupW3V3tEc99vws0EQkl/JzVbDKZycfCeRbLw2zXsjTSv9YZPLMIu9mkGnx+1EuVHw1Qp469CTCaw70EigqYVFjTF/9t9kFakPKxjhPaXvX4UGf31gh/vhfxIUwp7y54bFSKT2DY7PEg3PAkTEi+VM3XvV52kHYJrQnrXRgSa97uNfmQcyU6G2KcGpmuoFvMVuat9hcG2m2w3V4kofHUzggp5xGeIK2i5tM3GH4yKF7/K9aVcz2YfphE20ohqyH07/YGmO8rWdXV3DbMDlYJo8Z88AuPvjf6ZcOo7FAMw5Qp8e2H4bI/M2aXIFRgAJIKJ6CIrmKKR/Gw4SOf2VBQIb2x4Ig5sXpIEK11Hmd9Tg+EELFOsF4f/xZeBtnDfMO972YaKDxkQx+3oyPK+YhjMKxJYgZWnH8EWZuvN5EqKwZWw/Y8VmjWnwfjsrndV2jxk7Qgjlub/SImHn23f61uCNVzf0VG61eOKZ6eALcwLzCKhrxebeJzv+kzh/Cllqjw64BFcMyYRDWKVOhOFPA12N8brcP6qUU+pcmfnA7B8AO4GDufaJg7qz289cpc/xqUJjHqO4y8i/16AdaSbkihNXpvDYxW0vSSEVVYo6eq+sM3iUY+aIojZAXqW/k2H0eswZx1P5pBccS2goTeJF/T11fx+jCVCQolcuinjdXZn2WlbaOMI1B1q+ELmK245tWb3HelI9OikL/wfhMjtcLaQ6Zc9/dxtRVUxeMAt6VMOTVDevJZRjzYGN/rVJWMxO9pwqaXhHy5CBgMLXyFd4jJndRIyH0T33lT1qQF9ukVb96KroLx9cc/YIZTRQ+rINF5Mg7d5fx2Dvnx/56I8iaOdzkm9uvDvxVnSm3vnTvQU2qfH9/qUNZ+Zjvk85jTT9HQgLWgT2lUWAwHlQNvTeizwMHavAQ9ODPJyw6zqm1rBDw6YGpi8FNgzNOJ8QIiuziCPRyJ3M6AmltQ0xhXfJFCfMl9Ty5xXHAhHBFfLTFqQ89Zj80D44ATTaf09yeKutz+dJuNWVBrF27C8BPlxrxkQZQoWdjW0/rwoS0hJEr1rWY29cqnId3eEwEC0lcL5OKIpm9BB0A23y2VEZWXVpY2dcmatHbVpj2qHVnL1TYmHOePdS7c9hIX78V0vyKNeQjMSMlW41AFAD1V/ynZ8p79NWQ+vhjShkCB9aboNzMq/XyTfDEUdJ//Tj6+Xw4ZpGFk5S45gSFyOELZ6/PYZJc3QIld3aaitdxPxQhzBfidJJ9ULVYGZWaM2KHYUcbe7D82oQxWYtQyXCLeuQbO8o4THU4K5r3QGYAm7Y2yVZrPPo0bLFT8QHxUkgm45gHX5lGpHj+8thKamwOiUk7TZj8I8hS9aEznvNRGRX3W3bvleE1E4WWaBysgiaLD0njt/84B6tHzYA+xdz+mqzATaKop7LLwUXN+EBogLENf1AD2w2o/pdZ1Sr/y2ASjCH4om95mkJpc0yus1MVY1vCS3aOhuG5vpX1aoc0EQKJaek2zdn3KwFVkgv7jpA+4uiyDI/nbN4imy6aJ2Gaa0RsqE2x8Jnby6cyfriygYGww6+sKZPcaSPTqhFITNm52IkCBn9c7lf44D2XQ0h+1IZL9ezRYNAgyiqTvsf0pArnblln/C48kvwqHYy59o+wVABu4eyB9x9U29U+xSUC9969LkufFyjRYim4qrJOfOPtjm8us1SU9XWtMWepKvEgdoj9xiBrMUr9l5MchfO73JcfIJlLp3dN0TVhdqfxAZd7bctErQYifqYpFusyPBQ9w93hjM0wOlxGjHlW5T5Cqij6UYVKwPmyxjdTf4I7sL/+5Zif2DHImI0NmeYspKpKsqBuCJ2yY9k9tEq/V7uIlaR6NxnfSs63aj674yl54+wnLYkRaHv5rEm4dEPtYQxDOv0+dVSvNczlx2+I+JyutdiYnAE0RVeXwVeQmX/knIEaj3XeEgvpTsqkH+Vz1wsUm7J+1C+LVjpZ22ThAEYSeWzvh2qr3vAFkfN6HrvEM9Bi+dQejMCwbHg8xoOHuTs+FL0OiVPAesR17CoQ0qlzv6ezdC7c+AhDRJVUO8KSwT5O0CPly1+jX5CIYaON9TVZYlVlKLYvHjPVuXVULEo08w5KxV4PuJLayYXUBCuvo5C3U37x+eK4eEwoXzwAQxFkWD13D+6LpV2sC0yp+1prX7pfmWKDIY4yPltBeCYZNY8C+yndtAYfsbsq8Ac78i7wTKoqFhWHOb5DXzpdDDWqKFXfm1HGPyCSjg00r1VvI3gaxRNWEBpLEDy/+Tq9xFM4YQEYxP1cm2iJyfXyPZ0cMb4wwsheIk0RMZWrMs0PN/G1Bf7bFXDhKrTujQsu8xjZSaKy9bG2PMIJCF8sbormF0p9Zjn1kJ7tXnXBh6pi6y1P9SCVJTnqLg2fwZW3QjnQ1mvoE2prnvAhUKkvogMc1n4VbsAJdGsGiiBrDBkJNCER/p7CDGOr8rBbk2HRddfwsxwuBVTTOMYVVOpSGrImMngetyMYIRZ2nZXa974j/HApZEuCqzZQkHSEYmIUSR7JrBVefEE13YTQMpsYriFSsXO3v5LstJmZyJzHtJIuxuTH5aovzSlOf0OFFkBjyLSHecoDx514wmCCcgpvUVcdT1oYaeGKF1pxVCXbNn0nPAmDN0xFAbAp7c4c/+ag+lf8lCq2xk4utqlEb9dj+VgAbxJ545xxs9N8N7peM65wxCJYDCflDFUZz7YN/eYhdJ0xqibwFRBU2pI2h8JB2CNj1IyigDNEWcHC9ADSRdFkDD7XdzsXya+whpxJLvOnqkfX+SBdxXQtcwSQlRDkZcd0SI9azft817DV3KrIxfEaE3DVL5PE+MEZV1tlJiCasgYQx2TTY+Yq7FS88xrrLB1Zt3UoUKSyuepU+yPkuxiuKC129KNQCKxacY/X+8hLW4o+8fBSULIRUoaz8AGjdFxQaaCeLUoHha9VNvsKwOVQVJ+WHIdPxtdaPacgktl9g/q8sZGrVnnhuuMLVfXtQmVvOCWOOOdj9sgkIjX+Pr9juUcwmz3mM3OzIlPPvpRESSFRhIqXxx2LV1NyvlRx0A0R4dFt3rAorKhYkuQFUBt7tBIy7MePtTf6Ogd3WHe6xS/eCnV1TV2GonlgvCT5TJ2OBvAYn7GrOxvWrgSTPtFMdwW3fJ/2IUMyYw5+BKZI1FdJsyZjp/Gk3DswxKk7SRAkaqvuWIxxI1NYoSRAG4nDPeSDFVQFWaEq+52rjgGYC4k2P6+Y0zM8n1S2+lhGFP9gj0HkVKGQZcV0zCDL6MwtoYSQS/N9KP3BLoFoZDKn9OTDUZ8r4Y+7O87NvfV3INN/ZX2S/4Wy6XspgJ42Vwes5R6QJlUj4Army/8IPhAZjMD/eIUDwJ7ha8X+Dj82EZkvF1KYecEy4scwuuMldvfzhS4cDMOdBsuoj4cFgSg2/wbiBwoF2LB6c09f/+g7Y2n0FzvhT8waPmsmwV0uCVUhwUs6GETBQ/VfOsdATpT4o9dM1cpf+s25tDy+4mP6HMunqNopvL+ZHHYof5R1/9ZVBtq3/1wz693efqn/aCnnaI7qQ3brPX0lF3VaXovrN3Ppt3Lp0fP5Gr6LEIfiqFqfzzd/X/tfrv9/6zb/1HjjUN3zyKGVWvnn9//+75/3e97B8yFa+a/0r+k1fd4dqBH/2/X+UaM2Ib8P/u3d/odONPr5ZBx16g2zZ7CLZ5K//1Sa0eCKkeCVhHGjAO3kf3ve/6OP/czXf/ms/+k53W/Sx//xOs849kn95zn/dbz/jPmzTgTYtFq3e8YcSUJXynp6Vf7MxRFH6oO/fu/2rI2fPjWc9Wb3f1/nz7X+jJnVd1uGup/0+b2XDxJRYGb0vvsm/Hi5ggIZvnObzXNbPj4N30ccz4FMPz4sP77N2zmNpsJc79ifq4BRJN4hfueS+Ix6oLr8f7rzMzto8IAoHPrXO7v/fuf7v7/zM45NHsJdOrj/4842RwPlZ9PzwJpVv3kftO6g7qn3/xzj/+LpLO///3R/rXTC/U9P989d/5vZsPj//q6v/zAbv7vyXz7rg08u0Vcg0XvK/6M1/oBUGkpRc0xRpnIgozIa5jRfzOiFYvPM8u9n2l/3slrzSkLx+Z7qpwi92P++l6jfum6+9rNuwQh8fvaiwc5nXqB3mPRWS1/vMAAK8T899H/5/N863v9ua/62c38pgAN1evun+f0v+vbP3EYATdAgFpTYneoKol+Y81o6fSaL4Q+HHgUIFJQIqkSGJzxY/UFq8vVJ263f+m/riPKlbZw01kLXq72I4nHH0hbIy1bgQFgxEM+jxkZSb+kiEQ1fhBlHyD4Lk2pO46Ge7YHkBkXDpLdH336bum4jv6+C/EX3R7SlTiMvo3UC9AT03+DoXmpFcxwQispcc2cp1I84nltnQBdZLicoZqUEgUQ6z3tl1nNmEzlNnN4bnlDyeG1YNMMUSBUSwS9oZ0TonBI1iOMFDgjbIZsCfpyldv/5rNtmypbvbXVWRWEM2bmLUWY+0F2u8yS7A6RLzvKHnFDQzJw3mG/Q+zmCHN6a/jlE7XhrznSSK3gFSZIQAjPyvfLml8GUKQvEzFieseEXs5s5ApFzn+PaeSNDN9Lr90JidDi1+/boe29G386Tpa5B4y5rMmj1ZxTSTKTjoWEVx8tKGgZtPqP+bd6LWYPqM5MvA0P4srZfgV74Cz1yoiYMHvXTX1uggBLCWF67Z1Zki0Y8GLn3m6keqn/Du1IDPkkn5A1/qspEwN9CFq1TomyauiekgjSnexqWpP4JqKN3DA9kuY4LwmCtXP7m4a3Y45DJGP2NSLI1qjIPLHAhjG7v5GoOD4FM4a5DkLvdjgjnMPveltcMRgvL3ku1LyAZmRDgQ51dIXdX63cskYvoOhwojTJpAT7NxXZxtAR3zAzLFCvik7RvWTxPESozHnu2YYz/Yn9g0Pd1bQTiRisqgHwSi9N0rs6ClkimzFKPblDo5QIx18F/neqDmPMWf9sk2VHY7ojN/dJr1bS37TenkGQx+PbBUpJvrdJe8+k4aB86VEp5oyj60kZkfp7JME0k6Tg+usEk+aCfC5pfJ+kjSzIS4LgP8SfQjwzS9O0TSR0PRySMSP6NYqbZCeQtIFFnfioYEoULuetxCji1Yu1XMNfPo6XebU4D4HqYZmLmg0WPEY4qu4MQ9jKOaEGfr2GePEnyoQnllo19VBfSNmJ+JZdSIj0jHsep557cfbDHiRXkjsntu0/d4mcS/HaNZZP/0Xz8hL4gWiODTMfXM4Y/74iPyeHZjEecGSQYLj0ild++bww0dz5DTrYBYhYSHivQRWg5ZYCnHPmF04v+PRanwJIUQc45bRjhxshkUNb4sjxI+ZMqATwMjhrcf2Y4GRlS6Zv+yJ43gDBmAwxhfvP7zfxCrHLynohympA2xD7a8sxw83GnmJmxh3bYKH2rom2EoE4Yfb+aN+8hp23Z83YS33iv/+yF4m3or0Cnj7JL5ThnDbtHOdEciAMknLLme6Mr5fy9c4oLWEwWUBiA4kUBq5odTWXCh/XFIBzPx4wXpcE4ifbhx81xJf7zVN3E4MQHz7VmtpwXNd6MLGHU0L6ubpQdLRUSBNQQTiRRBYO3QB8NkzNmC3+rLzzc3bIbt/OgYmY6p2Gd5gjGGi0vENRx11QUA01Rt6yq/ZPQ5MNM/bU3bdhJ6Z2bQyzkFuxbA5tNbtgBVk2MpsxrG2wVexMo+XVF/qEVM3chW8JaQ8zhRuFpIaXpxrI0DIruTBpBPbb+MsYKQ7LnaXrLdekV2cSPnWKbYKsSnldAfvao3glGgwKhaPsyHkrCPBo61vn4IYduU7J4vx63OXe2OKcJNs1z2qFg+RVe8u2/hijgNwPvzXvjDE6GKTgicwyy7+gGdR/TvsTjXFsqWRomzgMaHghedayFogzNljJgK5ommvJCUPmMdklbTc+IIL/c/ePBzQWyj0QGmpbEnh5INNc/rJd58LJoNUM0suk6CZjcZquc7utj4be4YGUq6OBFVbzzTspPc9DoHpe/xvj866RTMO+gsikLMUzfII0oFvDXvS0Ja8eJY1jkrCbBzlvOlH0cHJ14BAsBHSh0WJOjWgNQ/M981z9ETTcIUFm4ujS9+Phah5CV8SWl4JXz0bby5wQgnUK6KEniS4f297iq5sdsGBKVmZg352kDBaoDBvxS93LzQj2dgxS3Bnd8mhU8D6AD5+N4e3mQNE+lWppDr9bCTfg4MCRsZp897IKqBXbUUPtoAmZxN0VROHsrO9/BHBzeT8LVuXyqSWGOGFOPg+jTfwVE7gQir+9jI2jkXleQCRClMxQPECUZSBAyFfkIv1ttP2mG39uxUtGyJ7twmLavVoGi0GDxSoeZtAlsizwLH7vlJ/PjZjrCsWRe+bYHyvPcZ/q6MjyWYhT5LYo3DI2U1yZKDyqKuOJ77FG8MpzNOhdZSV/K7kZFGoZx+pINBw5jFa2GzBYMyNuki8ixEC+no1eCTIcT6hp3x30NDcPtI42+ILVLp/FQZFFRAN8FRRcTwA7rFuaOIkf+gMoAerZjYI60kYr1EQtgFTRohR5nsKAghv2iheNjR2t7jV/kZgMG5TqC/EjCRCjbZ8U9ZIvzXnQGSBJjRniA6P2M/meMIGKnPbLlr+q0xjS/5ITwHuDz7QxOx7FahzWTGSZz8Ft7J4ptJlvyBjhKAumxQ9uVXuC3uEqDuDLfdKcmTm7fH+yPN48ajtN+JfL7mOI09+qTIkWXgjN4+dfAinzZaa1Fhe+0SiTWtixHasoRp3iMTKoO0AB6WrwsWxmJbCWS9w7Z5zC123rPY4b29iLaZNuzdMJnx2ZyUf7Gz9WYHR1GLaY+jD3C6uHodMyaAa8YmJpPixooEsFq9u/Y1gK604qWlI+1CBPMtyjDPkbaTtkwkLLPK+Qum+cOLJGKM3nlOW4kaNPQ/Jd59pi2TqJZbp6TeEUkW7CW4XuqSsX6jz9c2WqIQF2QlrmMhg7mgIpubS7sxsCl/gXtFwuPkTJ5a/G5RQ2UWQ89wbAIV+d977FPtMkoVSr9T+avtiXQDnQjNqFQqLBH18ufQM62in0h0QAQsvfV0xflvKnJj743EyESL78avBDQBgC5kjyHjRzsn4+yvEmkPp8UiSChncLMU8+J1Q4kFL5NMapM8FanV15phwX2mqN0y3oMkao8YxfcK/awl5RDxOSdMfIAt5u9BKvyoSrWQW3s2U0Tlq/KNkavt9T7WUmueDfuHgs/U93Jdi+2u4vKDXkNKJqlf82d92z+pkzkDcehI415/jWOjMwstPqQBKH6HFMMPKillwNc07EWQRu9orI6eYghBrzhdYxmrDR31pN+G1gH0PlNsObQQB+Snw9G4RAgdclynwz/qcD7GxO1IKAcvt8QKv/x3swvY95q3UUrdvo1OpmB5c3iMoOI8p2ls4ItWYLxBa/NkMV74CXExE7Et6FoHW/GUphXd+/n6n+Q3j9eCm6tdPcRP9OtmWpNbL/TrwDKIrpmDnTHB8F2do6TbydI6/grdwIFDHzpO7/Th5Dc14vVNj/hBsC/K35VzsG+FfelSN+XJchoWgItEvnI8AgGmCmrlviHkaTsWRVyxAqXCCfGxsaToUkjA79eoPtGJP0GEiwq7XnOzMiHztmqI2LhqMa9S1rPmh3khZ7z4mcrU11TR/N4jMdHV5pPG7LM6/SeIYsz7P3wKiiozJLxyBtHJuicND7HdUb0FjsPKEY6skt9Q/sIONLDRLLu7Z6JFEXQKmKeRzwcvSnpQ02Ez9ncTZ5v2Ob6fNC+IwYY1RLqu94IkYdCDMs0HlL/Aq0VC6KpQXtExkOxTuBlet6FVwd/5g749coEmfvJ9yC0BENqI9aM3wQs3rCzaJaKt7f+QepLvMg6eJzBHEdd917xPKuSZiKnECas68+6yK5vmC1prt9XCIf7miu4NL0G/41Y5oeukNnw3QL6qG8sLSPqS7GvXrWghZKG1x8HHFe1+zVJxraLuikgwoRpYZqqKRKP89Z3gI4bLB0O5g9HIxzml9aMueVLL3YasNFwO4GahXesHKht0ev7+XQiR11ODXBGzoz+UnUl9KALE6mkd89aYWKFIQDUnt18Z/qioyLhBaQFAPGiywPFoPGStA8cqCgcTyOJZsPjiABC5C3Ad2QIxdth+HxXR6E3CXFSyv4ekfkFUOckcuJeTU+v3kgbqFpsSPIznGsNY71IoqB+xyHti9MrNLd5Ay/ZhimiNNohKk8C90XSb5aWD1+iid1hmGhfA2Hw+D9I/fHiNpfH5rR08nh04buIT2s96eV77p5zFmFYaxacoztaG2We3fWmmVny8RdLeXt51uXjAnFnkzDP9gRWcvHzBby1/PmVqa6T8aVgh6ZodGBxJxiEaFi6da3tcA5tnjoL4jKh3x56ZiJGyVuShpRFoYhZRElVTWPiYRMd6+Dam3oSdChu2ejoa1jVY3+NghdzY6dLP2bygaL+9lJdbm8HpR+IwVn6WD6lh8DQyxFW9E/L3vKFSHNSFDGzJi1vqY2OgrQI78yOb7KCtY327naHoXFB02oEs+V9syHOKhaJ1A/No4UBCMwD6OA6UWRjLlEhUB/QtQN7gn2hS+qsjXg5815piogwhc/51js/GNy/DZXO+heWnFx4RNlSUnozbjHPBs1rVJyvQuyb5BO+iH9a1dVdx4OGqenQ+tP+1qr1ZkDC2S95GzKE5ohjTYWjB5Y0z87WbnUttyTU86/C7Gb6IF0UkZXyDp5FY2nRUmbo5uRH3UcJzvGtoPKWagwz+el8Gt3AMUUsq8OJakhLBwV8hEAMvQRdoviYrI08wfbE5eU3/biu6fpFOJhGgrlynZjiLJe6rcK1Gr3HKUQRGiWRrflKMS4h88aE97do2VRlKt+sPPrGVvrzhdxaYnLGJDMFeO9N/95rdjpfTwd2muUbSJlJ7lWLn+FLfVwRJBOb8Xm4oNhhMi00OLk1NneU3WzgElI2dTv3HKzpW2BorvM2a7bJ6/VQYr94fzkFbq/0QPVJZSLLo/34ez7U6wFrePIJv9+J+TzWvNLIteQt7A15LUgMuWgm4WFXfzdMKB+2KFd0Ib/q/IMF9pq9yV1rQZLsNW0HwL7R65W8PSHZX0XSpVLg0JNIvMuxHoPbWkID+N82/hJG3JmuFrttsQtX5dOiBoR06pYfoJtxR+B+0TosCXxQ7nci2GcnqR4nnu/3SNVBrB68Kgqz9FI3gPxYaoHlm494DyViu1T/MC71QlajAPnBaKeJMfwGWbp82HrYGUK3e8M8iwyz/hQbibmRX+SO/pq6NuA7VCaF344M19YzZ0z0hl15LFWRIRBKcPTF6ULQ8TC/1WB4OcNP12DxZFEoCyrXyNHyPhtNUFUaIRQV57QsEKCbevhpEIRL1vi8OKqSxpQeiNUdn191xPNvq8ttd9Oc9cVYxLavTCTcbAZbVxylX6zFPB2Q1URGBnfqQ7l9NvRl8WJZzninUOWRGhQ0Hs9oZd7Cef8Y3inq72Ihu77Sknh5a/Gbo9eCDuvIkh2HrWQiOfsKvEPzeQwXiB0SZfMWh3Fg7xeI8Ey/5k/qFESfM0aPnSfrGzJ4lgkeuzPBj1xz9cq81m+jFlsveqsJXujaBqynDGHrLvab3MLlo7u0h+GsOLDnbNCJTmKUL681OOmOedxfcBlTcbyb+++oJO5X9vzW1pc+v8FRoeKpH9DhSjvBCZgUM5bP5KUd5hM1d1RohrmnB7eWuqQEkLcTuD6qJ7ctImGifmuVnt2McQqYIDBNGwvqZ6sx8XozIHKjWaJz6cEDXbj3eJ1BDss+K78KKJM3OnYX6UC9MfO/sWnjeAL7mESnr+Wd0E1kKIH4laoEFkUUAR24FTR3Exwr2cyKSxzkdmn4Rk5KNWvbD/S+rBwFMchsOriYeKUggSpA9dVdJYEdhv8NKL38FK4yVVFlfYXNaZR1OufAy1q6nWTZYUKxNe3WOl9vpUNwHOnknm2DsrgJD83GYea6TbeOKN2u0QERTHHOQETn/GIWWXkLFlZRUo6gHVSsRQFjMx6lIkafa+gOx5W4xShlyIVffidx9rqcvfjkHMQrQCCOhwe9YzylGwTbNDeCL5bgd7TImIqSFYgrmUhkiuPxUrCMSPa/CILsS/6EMc4hcFXmVSyOleOvflVq9IJgIx+hXtP5U0yMsZnotkqUOGczNFnFaJJjHVu/oCTIT2j2h/bVPyhLkRNV67d8D/uxe9uOeyGWBHvxoQeLD0mrZRSWJF34BNNSC0WVTQ+XfdggkOKqBtNUKD++Krr3g1Ub1hx7W91Iy6F/tsaHXPcPFjJtqG2LhuKmifrzbm0jshL8Sam84UjYuSl96Cyw+xA0EAZ6V1Jy//FXfKWDJWjPW4U3Um2Sh+oEZUf38vdrvJt0GY42Tcsz7i2m4cf3EXdn+iBVrlBCKzRlaEmruYG/l6Z/Au6qkJZZQA5RwUoBxZUVuOHOHQgQPaxPFjOeG/AM5l9xsUFUjImOwYR9BE3NXKxsWMlT3O2MucgosLVE+ROqrSLi9VgKEJUJGOIzhA999pW0eZOT9QEtoazOMY2XjoZrb19yFmnRBt9tPf9emAe33HFFyfkqYylpY9khO3dFndgbney7YmHmveSk4PPIK6JRPILo6MC0kDAWxncoUkXwdeA7UawUPs2ys34dwWqn7p1ND6gs/ZKYvMn+WT2LKVKQE1DBcsF8xgwO0vgldPqqm9jCkZhoXWY74yTLeweBjDzIgb3yJGkdwdZyxo5u8Nw8KDktdXqALBVW4EX3rvQrGKAkGITxfDAm1Ehmcsk3Hky9mbY1EwzjvIskguVtY5oRwhBxBhLBRH77cuTp5ld71AQodMLS8PQY2OpGArwnoXO7JpDC7GayX7iqh8jNwX5Sx7/6/jOKsLpxJJVIvlral+ddAn/E3rQ95IthmvANoZVgSSciNdy1W4lIvvz2XdPVJ5VMZ8WmbG7WsSoHCdGmDS/WiCZPZyW8BnJ7zCgwyQcw/HOmMgNVZHdB733oGGvL/L6fC6Kcvtv2bJCm5nXv7amV8sXJTgulPXWzPasgMgFNMF/uPcAGy89oeKaMHoFrgszU+zoaMAdvJ0LztNInP3HjayAmrMKT3IZ9EHtu/LpiXlVlZCUTUkBQhu1rPHbsybeMo7KhxAYMRMjMbIYPGrWjSDIfPzE8uGp9N69jjilUoGFdAOUsQaAbm0xMBc3F4j1dhVARCkWkdLj6BQ797LlqFTEEv+lYcLE54gJ6R0Esbr/tz13Lxsq4dWW7WPuBbzw0jsf0DZmXyFZmZzBt+xnCbsZMT7uFHjk0q1t+BPD+slwaKvFPUnwy5Ur7xYU/+PVKeNcNccqGbnwjdbzpPoTFcE1KlyMpJk3E/PKISfs57wLm0XJjS6TQvk39vBltQC93nl4tlBXrzkNFyHD2QSpJhEJ2H71VtF4PW2m/fcM+JL0/PLPAdg6haaKUp3YaLVTc4AL56BlDl/j10CJ5X8Y21IeBFwS/pIpzt2KgJrk0E8ZRjRBd0sc/X6Iees1Kc8TYSDIsMwGBkiAgjyt3xkpxUX6Z0IfFqPygkkdCNA6igTTHMeeFkXcPeUZZAyF6bAspUG/0k0gGjqQDi9sqv2+6vIuQ6pwJkgaCaZp92wBHdjNaw+8lxEl+uH9q4iCmCgK+8YwOO3DwoCQQJMNS8JO3ZYVok6FVx9pjnRHeJUN2UD73H7KIIOhxvbcUxEC/BpOyQ0zAhAxJW+n8CW6wzv07+eT7W2L2T1ZuJkB6q8QLK0jIgfa8I9ySYd9AC11dkr3N29FWPlz/l9eNJPAsJ3sIIDymErtrjzlJ1+jnBxzZZBwGM+WOXH3v4B6/I3DXVTdsCh9q+w6Txv4VaSsNMAkAGXi0BEL/C+CL3hAye5028IaZtDDadPssPXQhmVzDPmFTTOf2tcxXZvMPKH3BqdKhGsYMj6EsPMt6L58++DVhrEG4Bngq8QcPVRm4E3ht3gZ3xhMLpK0z07PXAuS1X+7vONm3WvOYcaOLh7yZt1x3Hf1MGH//2g8WW2chDPpeN5gGCo1+sywzr1CN2R4ltB9cFbGKg9s4FOajqArYbuAL9EL//Gsp0uRLiVSm3daUcQFz+yjqJKe0CKNTsaPnjduNWVpMdaDwZs9f+RaaqvS2nCw0cHHC8/rPjJVHqMQlGLJrj4D9R0EQw/zzAIZ829vvSN0V5nOWp6iprMu8KbX4Is9orw6SGKwLxnXoLHFgHTz7Nh0ny4GOO2tWDMaenxmRf6cgc6nSRjvfXJ+fxAyIXhD1NtvfSutxqHRdcktQjn4s72EB/2sNYJGUeeUofTRkvbfMhX2BaDldE+e2LxbjHhx4Zkh1JLw5OLQvSBgsQBsV2jqg7x/XWcRCL+thKnUkog4Yw/Qb4zpeiEtih/GdC46HKWbL/inVUbCc7h03IpEHYAVtLExwFZPr4qGLF5ZZNAGFFZxp5VyeW06c+K0/q7sdWptXvpjn3kCnhARJms88I33F2jUVp7g8m2BtJISNjVKSrg62G5/Kxmb7A1xAs4E4WLxTIl28p7guAorAbg6WrR73Iv5Dg5cGlEh4sDw4+VjcFHdLHoyU8aLruG/zigTZqwmLNaGDGh2WU6t26Z9lbwy2g74Y9NJ6UGsqB6BAvyck6PXtnHW6zwJRu85JS5WoP2UjH9uKv3UM9XCCpgjvzc4Vp3DoaPm0w7FobhyKPX8STfvSKzxC38pxzIyvVscbCT0q/IpqD2xXwLnrIK3TEoCBeVGfO7rIyNSA+cQ7fLHT2GTAmpn7iYYJsA4KTPWAIj/8Xpkj4I14WZlsCwp6z5vUMTpjJfqAdSHsIX5ZqpChFKDHBRbWqfcteqzt+7IsamYhfwiAPoJ4tgdCpt8lA6YEQy3YUkHRMb1DCWon/rPt4HIvC4PVoutKx7XOwqMuxs/XvVBkb+araJaUm6iiF7CioqlkS44A5Nmhax8DG9jWtsjry8h6zurrpjDZb8i6uQhmLAUKPJQ/gpCDzFimxX6vuuk8kz6vxFwmW6P4r6rz2DL5e6o5FvpFGMp6CEzZRVD0zFzkLH6aif5O63ys6c1Klu+F/sxDod13qR76rQLnEJRQzTM3sbfbjagQIJmvzZHNKi+a7MIAJI09FAsGInlAOkJQ/GbIpPImSR+Qy5k/Y9zaCm3iW0vZsU7dlH1iouGnYQW4YiViAFqUlD3Uv7YENWd11A5o3Y7atWx+Sjf7EZP62p+meyZGUxvzBpCv4X0uEPyVkHPM+jercQdchYMBsCK5vnhRFGg3nfa9gVutSc1lmXBJD6XNru20XYrTzXFgix5aXH5hP7qCzxnbkqamslxXtPLjmEdxoK/E/o13NC01dLJiweT07doFbvEXUhnTlWRL8YJYSM0NzmKJ1rV8EDZIV0Uy5ofLkXB5VuJm+qHg4yB08e3YhuJzktUN2nuY+fGNopjefBlYxN8ZZ7u79wRwvd6MQBVTiDo8CGb5U4X2h+77MMIX8CBV/aZoatq2gGXWpKNmS+xkXpI5/fSAgN/zK0eyA9RXNujRD1jzzgOX9PLxHAxiP8sWKEwQqkRiNAYLcsS1L9kS3qmcdTKYpVA61h5C5q+SdUVvwyqQXIFJhaMsl+hu/4aWB7Hkx33b3z1GdJsPtjGxIPcCwGJoikT2SWt65oW57Ap91QNYNHj6bb49Hccao2p1+NjAOZznnQ4njTx/iL9kKoPvqMmy7f3rNqUeO9uvTI8nvHuwSGxaZYf1lTkDB1yLl5hELO8KA0svh+RAuUz2buC7FEFWOcmb1JxZquaia/epJ6c3fbt6Rwq3ivxhRaUDzPkVRQvbOoA2V7M3zyReh7La6HheRPjvzIkvFk7w99a8qiJF20zCKPD2OHDI89f+fyxfH5F42gNXU0WJYsrhQ1aUOJfVQ2ZFSkPvyoXD+80LNYMi/MvMDzr3pLR3O5CRWKXfwUvwmywrbGqZVFt/RQ1CYMYWkr5UGmR4oPwCoObYeCSeZpmJmD0T5PYTLyFfj07YcSUmtwOvVbShUkTix4/XEuX+AmiKDa6IWF0JmHoEXqDsmfEPxFhvXsE3/UA0ZhFUSPJf1tcR1HutOEZXhe9eKxHEVXEPjIJdkPn0+K4DKkC8Z6r1d5C69lp2IX5fDN6rhsa2EnRHH9TvMHszllJpECK/UfZg7VIey9trX1MJA1v2E0Iyc3Z78DxkfwmRhuryxBa62T849ho1OXfdkVCDefy5wBb+4+JPC24xdXq/ovwLw8iStbnyE9oshDvjRPxGGeUFyuvkA+DTVFN1sNRVTy9fAnxsmqS6JhZ8q8Jjtz+XBLx/ZbncujHgoy+fzQzio/HEqDQEXKA0MZBZr8vjdYbO8SbMm7fgk1EoWbUmNpofBu0WIzt8FbS9mNVIn6FBdwjMmvgFiZYI2kVSY9NxU2KGOssqJFAL8ZbkAuETdp9J2Q2Yt55ueKc4LiMDTJeaqOmAiZdPf2K6QxwnmJl5h6UY41R6JQUhOzLzuZRyGea2pHA05zfGuo8naZGNOZE0By8IVtgvmcijN2UBdHAd7vndBpO2d4QK58gfX9MX+LNgnIrhFL2RCQWGKX0ITHu3hBuXTZmjDy8CM1/bh9LUB/4Bq93vol/s+bUidnuQndec5Ssfv0KK01egs3vKK4vCRCQDDaDezMKpdEYlMP1eQuEkMHmtnx5SkRmuf/jTtfWBldeb2KTe8JHOpU3LQmcXETwIFkeqq2prngW6P9D+woJAwLlF9eUWqQZzrpQeoYUB0dEf1nJpegtuksZ93iNjioE3gknrWBSrqgZQNxMdwZGjgq643vGYn0SG4/tsGUXvOZJeNPHGaGqJJWZHwLG9YK3M9J2NUgCdApjdcjAyed2vfcfkT5BUv8Cgon1z9es3LA1bobD8GI1Lrr5CcEdiD+cdMdwZPAQS4Pz4S/Aw9SHzi3nPVXp05Qtg2VNyhV3LInEEb/NrWPoxnrHtqjCx8w9t9QuWZgRd6mjbfEBZxI6+v3G2lqHWPGuTM+w5vaOriPJ6JbxRKH+92BYswmA/fVCReNPiHT1L/Pn+FLZTlVFwDx12QNk6dwN7bworq3/tpZo3kC8yEOkG4uIBuU0pz3pNFOgkuoLLXY3/rXudOX2jNAu8aU3rta4MV6iRdWw0gFW5NabSQdloyeqKuzR6CYb0E87M6GIE88a4ivaBYRmQF8x2CfrCOgRfXocuiS6TYGT7ui54X3bPEWIBSSMZjKwPalMVmqGIPJ2lghAMv+Cp2ImJpvdo+s4viNpxMj/xNz7o+lgZOsz3hukzDlAMEWMyigD8Gpn3O9fugdBszPgiLpTlKm8KeDXaXzDTQzkZ5BI9BBDBtIorGBXZi0TTu5vIoOynVQSjJI+4G55gr0WBAM21jZM+m4hDr6Z13LnwZCjCSlSBugGVd8IsxV2vZnIvzhfUvvdiWu2yzh+HqhMrZe3sV+7pmseR0GL20afDdsW89EDAe8+l7aVZns8C4SxFXazZ/Gu3/XbpsKJZFU6SHj1mpdpdCDWCezXvAyPeJKKJv9Da3ifAIY9mPKU3wHcix5ZBfWrddGnvy+EwPeICZfBEA42koMq2h58wtuu8afNHJXnOupo5nlM2qYMgzMpTW7fTOOiLzYadQJLFmWPzilu+IkywxAKN+AoBd4pV9JCqtpbCRqEzad+wrqBxq9ofE1pjc8Ff8YHUXnQevz7DCO4BYtQJ3vkBRoTksF7uhi1sExXAZfvrVtF7YeU+IkOHo3YjfX39raxZqJo6EU9g64CVBg0heyUy2yltcr9tlHVDAYlwwF7WQ2J4olpI6YT8JFUBw+BhXYHGE2zRcf9RV2NQ4PbNgMWy/iSEbyLvUbHkD1+e2PU4hCsX1dVAIdYKHTa6c3lK0hdp8R/AtEY63fScoMehBtadzvIJytPwaFXU0/TsAYSUanGR9f6y1CyOiWl29OUN/IZmUepCOYxo+gvI6WCwyhRdAgaly8/Z+cJGyewpBT0Om5Imf+Gz1/AO8OFgWSjtZ0Kci4Emgrc21zmsahG2ycEPqWgPWyOv+fkHNRGm+yQHm6gbBIZkyf26SG8duRymk8F57yIRVZOyWA9e1MmF08DRBSJwGlSpKtTMYUMPOzymA2gOhY5v7n8o//5ON8ZQGGT7cPMWUa5p213xEsT7dzjDFjDfOCLc3sjqzG4+wEgtVd06Z0uKpyAs8oSDXNfeYM6EcOJbCNdDEVthXjLB8SoNctjx605i1aiM+fiKjyFoDSUTjOZvzyZ9VSKt+Tb+FWWvVxMENzQFr72uARcFPixeM/nUDdMs93EY3u9wQaiJnsQofBcSadpZtbUPmlAv9lPX5jaihnfmu+XNhT1bI4i0JegXd0oQ4XiBJGz1Vc4ahMXq3IniIH0V/Pvz69s78HHLx/ynUFjpBldEffni41p4HdRyiOLc8IEKjgwb3GiAYL/Zy2C9DuWx5lqGES+aysgKOCGf+bIEeVvjZd0whTMrt0KVwmIPHcjvEOG+OBK8yg/w/LFg4nVq/bTX0gjhKl9anTH+LFmWInkTgOmqsvjsFu3u0j1+5RFhUUfS7iHRVYybPGZPe3nQGBDXFaY2+ZlBVCwnk5G1OQTFYzjGrLujEHNtbtPREtJjfI78cnVg1tQE3vvk2OHWQydUAhXJawB5QNg9gEn6cVf+TZ9RkDKAa9m6owDXDaIXWPUW2RByT2B+my8H/gB4xu0bdxdr6nrVNDAJohoZQ2RXekUpDjDEmMBr5qGK1YIJdMnWlulXFD1M83J4+MEu0jzmncB70GcLGmAzdaZdP83IWIcsGsP/5ukqEiRHguRr9i6GoxhTTKmbOMWMr19F9ezOrWu6s5ShcHczcxo3gaLu2YtxRumnU5tP0Wk+9qb1VpQceJJouEJuU2FBZf2fckeAau3mWA6dyWfyV6tF4MSfGyiM6XNbdJYyMsecQx7NhL/YEXRmWvTg1I8CVSfqH7bQixNcGj2fiNC+nVoodnaw1HYFuLeuu3SJOu99VXL9yeyA5PaAICKj/GSerTVFINK+K0YyF0yqq66UZTNSj/MG7jOElgtM/yO6WIH14fs5w7XI4acymS6BX+bgT7iE4yel2ki4h9fSStncKE/UVNDnstMGsEFh3eibmXtPv8CzGpqOewRO/JJOAG/ytgG6s+xV1sLqYsk0s6M78Ro8W1oiNzMh/9x6hssGnLO4cuZVwloZqTr25LW4THFV/CWhlTbmLFZ+825yXDdWA/qncugjR2cYBv9tJC+EoJIuHkN2+3Bif1gmBCiN6POkxIbsNTWOuhbMSu6HBA8sHHoE1DzodhiwH2yzcBS8tJVxZfZWI4xVwGPrkdndm940hL30J3dJVImkPnl3Kp5LPTSF+vPjDMAhmmis8Papmo+VDxdlwXRP9ZevYBNARxQpVkE2cvpKwy9tq4DFSeWz71Xlrcxway9oKJyCfb4Q81oUuM0SN8N/z7CXt9n4zpZhLxkr/wp4vaAKyx3EaCtgXwgm8TZ92SyrJPQevxD7V/8m27v+FkFRrOajaw9g4kWsqllrPH5+A4i8SO9i0oZvbErVl4jWs/PFJoXl+pitgJSiqIRh7VDHg/6Guyf7cgDnyc8A5j3Hd8PTH147WT8kN4RRNVfBKlYMoPf6rwxdoqyffWPjZOiuRPwc5ESY7HuP0qjOvfI+sYByqSN11bcVMuD2EtTflaQyp7SPGT6S2d+TxYRwRM4LzL4iR3jgr4k/evta08ImNDCxyxfdM4jFDIzgGm4+IQTcgv6mcAKEUHSO+AX1VFf4HbS4NKrfi2O+UVSRGu8oMDef0taPJYkmx2BhAURzmUNYTylZe/8RAK5+gnC+nM4xX2orK0A9z0P19U8yGxN8AV0ywAE55HPiM3YO7OUuMd+i7qfIJIKBTsYsztAHsz4dgyjjET8zPbR9f/8te7KlKeB7x6rQvliMkHbB4cJ1j7GkVHjbk+aiaXxaHiZwRAMFiTbrIh+2JFvV753fRCbuUJfnWurxV2TmGsaVOBuj2VvrGgsfcJOyl4hQzUBjcFAj5jmWTNLOBoX6be9GiyKAbJHtrB8+GGuVi6xzDr5C6gtagpH5A9ILiI8KLhzF6Ue0Z/MQFh1FUAgQtAJEHDQ2HCqAS1H/IFNFmNqWrE0zH6E4XACz/1T+8DKN4ERqlaHtGkL7El4eN2aFZ3joISAyb913X0fKGzsr9mBY3jMgX1oZ1Vek86YbJh9o/IMcTLXPkO+nt4Tuh0E31siBQTakVnclfSwToVapZblQ8lJcr0PKtnEqKQzL/ociKUl9rSFPeTwAX6qioLX6M0RIQbYevjvBY88WQsXVFl0GSsYXqvH1+VSfAwKREHY3t0J4hgk/Pt5kcvWUW/b3uirJ7O1MLb1gaS0cuc7+zl+KB3REFG2husIu8JRhpKruLB/gpTRSxWp4N46KG3d0+k27vzFGR8DCv0qlq/MbUU0+IpvgFMZPqOCqu0Uj0FC52Oufwk1Cm58irrgw4BK0DZCnwuxhoWxnqPFmEYo4HDvq3o6+2+1yZ+zUKmoaHCOGqeANXYiFc/SK+Hwnhp5MTge+W7rkRRD0Y+NJDPpGwx2cA8a1PMJZNYswKWGZAwhnH76mMHcAxFtpIxkydSI3VvrGt3Db6mBuGolJ46Nsiy0KBu2T0HGNfdu17xya6c8cZaJFr1T4VP8bClAYizjb/fVhIkKox+zr2oVXDDtbBM0puP5ySORE+vEzio+CBWpYb/HtaHtKkZdoD5zGiMwbnsvkR8h/UmGNFpBEWNt8fVXGqxKayWpVLRItBIwU6C52HlHy2pyjb+q9MNkUVXBZ6gWoFrof2orSTWmgABukrUCZmUIX5CVvKND8xP3ybsPUmcTMKy7Rd2BBVWZRdl1RvkZOAYRO1C/HxGXGiyqQPdXO75qEdU/5TmtvhJwBYD71DwQzyNnvXxrhB+o8uPYpXfxjYbj6KUuKoECCySQiUpIfLJ8e9L1mUHoEZR4A9fE1/dx4/CWtjxzDEQtaGTDEAW0jq0PJ+oyvNcfxIi2TB0CoYXPvo7dZUBy3J3A/40avASmMjaUVJtbkgPcw+gUAus/rEKh1/Tr5GA6b0jDMoiDyOZi1+dOAN2/ecQjzYZ8QSIvK7TKhansTz7RusO9KQkavh5SKqMDetTlJ5o8o1V9ZO5VWDPMf6fvDbiABmC6/v7YwTrZEzPXm7wZcqj7/TZP+ApkrUOEhGtTSpLyPHppqrHK65P2oL/tPxBJrJdwWxC22VJelu5JtmqLSImvXh7O5u6xo0OaUhtNP4O9R1PwVvnTWtPn964GsMFgTI5K/PveJYmImUsqe9CLMlx6HvyYAhK6H954w0K+QVYpDpSimbM46JyAE+Z0HP0uVBMeuqy0eCOusGKoZOHYX8eok4MbfcjaB0IsOBWIMa8wGPf1y//0hfCB4AZEb5ELWF+ybG6EHtZCG8AJE3hbz85BV+14SJ1rqU2YmgDUWId+aeAps9ZfEqs616tU9k5cwhlB7vFmb5p9dRVAB9HZw2b3ExA9smC1TuGyMKrSkUzfjDxLc/XKYDUj9N11bwgmBEvFMTy1+daE7dILP4G6CtfGIbbc8iEn5+WpU88ExujPyb1Csb6hqbbTM2rbFctKrGRGK6L14nTsInuhcuua9XGXO/cHDz+Y9SqXPJoGmsfTEuGFLPXX2JPGi1vOrp/qRafcU//ViBzfYWKmY3PTpb+bmSKcc+b0anlhKoYaEcA7xccBXsUfjDuujTOiJQfhBG65oXlPCXju46lPx3kJznrCDv3SMiPLxJVG0kP3423bmGJwDtZonHecOh2FGThBp+ZLm72+DjevPQbEubkn8pDT4Qu8huP3FZxacYSt+RfmHPsFZ/sa+dqRYaeoN3poNpAUbqvsVKwFjIb588TS21YfRXWKatbzmWiJa/cI6DGPFCTo3kGc6JjiIdDY8dcJdfzWCPonJSTO9lrvHoDaEKntlcYwEej8+lHxGSMjvQhtgk64pwQJFfKWamjKs+BrqaVQ8ijIO9ckjeUxJCmepSWy26gFoFpsj4609Wy7N/brbyPwzR3FD0shGQcbh84gjGZW3+m09XxztL2fx2GKoY0I0Uxn1kd98Zoj+7XhwiJnr0KK9/rGR/j81VGZGRdSrr3+jt9SWTsf1tpL5Cje8Ud2/3VgiprOx8nBvC1kOeAAVZGlocMh2dN/vzzCAmvMRurIqzSav8utHjJV1s+P+0c15AfmVxP5Mz2N8IL3ig2ow/MxGZaozGQsyk9d9BT7Q8dSIpD8kWynyzuHIR/7wnwFK5egpbtdNd5564uuPOWfKo+6VWxlA0hRTcUZp7fBKVYG9fctdu10kn5fsixCSdSIXecgCs0r3VlrEtHpaJG1feOuOmKBqU7MmKYMOv99GQj+mikMTTiBeVNf6EbMtfyOT6AopolhGgKEDAPOs9MANSOOwAy7T1ujNf3l6hvj9zTu/J+7YxW9uIRDrQF9H/sqth6Lu3/D+RXbeF1NCZb74TPGih5f0HjycSX8qfw7c2xbzgrf9fKGiET0uxJRO6kIcHMWP7lwQvc9BpP2NYxbF7or+0GdVqNg0j/E2rxZJg3q6L+NL8O/zU7UYUsOEzLkROSEUaXrOQOF1DuHGHuf+1L35hGqBOD49G5VfK7Fs3fzN6i8N/f55v9+jaeaDf8SdnNCYE8HSb1HLZXoo+QVzv3NkiqSFnMnfzGvgv4Ds9nA/3V2Sv5Ym8cczjoB5JJdTYkWZiPTcJkckvIRjCfbsEGRm+pdK1pE28cKuYx3LPvaRcHNQoOgL8NG2lUKrAyIB7YF8U+A1su8vzwtLbG7r6r+5EflGPA/2G/NDAl0m2odg/3pBuIJ2rl5Sp089Xp8Ub9uqDYC3ZUVsec7uG31/AxsY3GFurjnoza1ewhghBeHlnYTbEKtjBm3HE6Lousx8N0DPtg5gkxeKTp/nvQDaM0BkmcIe8ISk1BTGCs97CVVWfuSOU05JNbmJaTKHVX4JTcXdRXl+TbQueSYQExu74dGKajj3jpYI99WwhcfPxZOvpvGru7ho8bKrtUrAuD+W8z9gRuNjqF/Rd2Ar6i2CTPRvGle39sdJoCNaztVVv7XmjwBN6AXmRu3PzbO5OU58X0o17Ko+JWWHJjBJxIi5PtshASEvyaXFHuWjWBiGmhRnbLJEhXzVTdok5uvI5apDCdEcEF40++11M95+f3ywtOgTKZ1ixP8cf4lQ6vamX2HII55oA0BH9RBAu94tmYR1I7/FZOzo7vtfxlVBSTCYKKBdswGlpCHUbiS54TYKXI3LxYJB+AZq7sCW35dfTkkXIgGOZX+DNIs2PbZffGCHaLkje0AN5J16HokzHVEK2La0KqTFfUsHpQZjyadE1e9tRg6PeDl0f5iiwRyX6J98WUY/SXL3eU4+cPhrudEltpO3uTwTjehl0YTrti8903xvLPnNdaVR0//W/zH0UvA2fP7mNeAhQe3XLteHoSKls2vthSny1AP+9ov5bkKnI50EFnyk4GWaSAi+hobD0JGcXSBSPSCvgyP0CenMx8nQLea3zv7XLxCYJNgo9zfdoLwNiZLdNuHdBBYJWB5lew4YrY3oMefV6yExeVY5plKnbsLrNi6+N+6yZQoGlv+R/1HXuE81k/uAYy1sBSpDflRHf2Dv2YGH47PcnytBXCB89RmaasTFVW9jgYcWals73m6clPRx+Mq8h6askZ4wQfHkKptDfIwHWPwqgluJXQRWEH/jdOtEH3aHd4wVsLva5QP59yu+bDNvnXDNM517cbqHJ49Vtcl0I44hkAsEOpD+ZGSCBp4SWnq0PsKlrtRe9yavjRafHoATWm22sFeouQWT73GgTksb8oMYxXxdkUTREHgLTZMMvlAnHQTE+L+qAcL2EqQN49yEJNug47Qgz2M8Uevo156qwZRb8Q2iiv5enumaltOgML82uvXsYMo6PJtoPkLLodPKjAj6Zf+6MAHz3uEGCeCAAVWKOmwcmNLCxaKpvyY/JdVeV3l/zMfUMmGpgs3oceOFSI0u1d4xjmnEquRi5+bnajIlOQfGxoJmiq9cQ3kjuQ7o/vSfoe3pKNl9B8VW3s5K0X5JIcruFS+nnfD4NvZr1yPa75jiQnjVEcOj69dVBqXD0NlfGdnLSGdLIw/lSNzitUZUSXLlc3CnEf6t/c6kSos+HwZIwXMCUGvALpoZ0PKTwKP5qXGC5HoSuQfkwsiNTZqVxLXUjU3oDnZtxwnkQr98iVy30106LGakrZC2B1iQc03IT9lv4Kv4v+VnX+coS//3QAdzMrb5UooLcIxhevlZdHsZtPa1iyvUZUKx/Xj64of5QG4AaXR/mxPXtEOKMACgIpqfyaeYRv8SKrf0krfA8MZZrDeu4ZF8yDwZWbmEcDNqse718W6crcC6wIhBUcXW0er1FH66VAga34aqPq7KxGtjf46PZ5uyNKvKPMW03aBEhxx/XhGDJ4jqhaS7psIOWm7oz2MXm4+gujVlle/WK08sA2UeLsfHM/tS4B/gqwgOPZSOr8Qx8IfNpY59NlNyWEMX9TO/x9vFh7/Mi6yHHUbTgkq5C6ukQniW+ECbxCbt69rwYA8Wn3XaknpBnCHsmn4BP8NC6IBtD/UYqSfoatYfqnJtGNwcelfqNEuoegdPX2n1Ez/Xs3z5JTvLqzZlme401eapUSHXf/jyK+RirOOq9uv6Y/j6QVfMiQ/Je39px5qTfxOJ7JdzOV8v7gQLS2TBirCk7Ri2ndjdOpnSAJI9nLdJNwpIxjYO2fT878MKuxEfthM6W2TYsAdieK594bJqeH+/fsYRRjssad/YxLW8KE1Ah+L9x/DX63EQI94ekfh+jywqXFJMQV1gsQBQxd5fWtxAscWan3+zdoqZ9jGDmJiYLOhbWaW67drffu1EJVNqSP6OJ+CS6Ot4q4W2ZGY3t44vsP2zQP0aPbBWdzmbsjJNNWYBlZysxS/xrCTEZN9JGp9R974oRZkO2jdHL6/Mb8Qno82ie63Nbk9ePotQobUe3P2TaS9vivu+oRFD5lNQdGyRJI8fSnhN2sPkQmX+m8vb988Fx65hwQNbay/EMq7t+qJ4RE2wcL5UfJh32k9+6Hp/HXJTM701HBoK+j45rmiGgp+2/yVgQppjYF/HFaWnlOKbdsBftQ48VVuRxezqcXxvruKN6MlZv+F1mwUChitOmqahTWQyPCMyM5ozqvc4G8M8UePId/MEvbcvNvk5qUScBA5cdBZRWV42Q51p8oa4kkuJA4pE3/pyXfNV510g2tD5yMw86oe+Cz3cwxwSuRJjXrgvTjZuhUyfbhTGLsm99CkPBM28CR56QeZi//n6pGLppvfNOYGYNVfhMctj24mqO6UlDwPYBPXNMuTdNPPRA8IEBo29pvdv4NgLaPiNKxlfRUQFMWxMcOquSkl6KVMyBs9bZwwK7VlV/TwE2UaQA5+a/VIi4K7ciUVZjeHhF2hLceviG4vCUXz20M/wAUyUA/vInITmmiNQy9YTSjn+keBgeIF4kJ7JhitcFOGr6mDgk4jc3KEgogK8rgA3owa1LMibsNSVD8jQluHneHGxwQ4DHnGc6tgKphnj8mcq8fNZW0CJPoQrvKz5N+Rq0TVc0rv+xqN/hUPbC/1DoLLg7gHqLqS8/r5k9/hLk7cHOwoR+z0JrgPSz7gXImvqh8pxNP+nUDCGtwWqpNnEKV2aVl0hW+JCWmCXASZq2KKPLDI0rWNM/w3Ur0nhmFxYX5x0FlIy7R7FFb+Uzdoxl+d4ENYrjSbib6o+0igosAw/96BDL6cFSHCBG86HxIzzOdr6HHrUoLJ0NUNmkSEHti+zX04rCqH70VKpa/qXLWWdLSg3tUachMpfaVRiusr+sF3ZG1xA8BdmMTLLelYd7VTa+1cGc6BtgJzhmS/7k/STRJQqQFpivk8drvE/FW2kzL71I4dS6s8VMoRSvjail3lmeC8Sh9AeJYpPXrIVTfkbDnwML7/XUjPpvM1/6vqyrv5Do0+hiy653oHxhXqnEJ0NyGvu9SnP9mz87i5dpvgd448QxBHOoS9rr90ndkYYKfBHROoL5Dpw+/O6o1OD4a/ElMUK3udcJD2kePlL2BsCXAcRxTbq5wgVLKwquBXh7XncNH5UFRWmTj5mqhgWVOxyIir0iji2jGN/uIeaHpUitdZ6EY9mtOggBSeBG2v3LGk4aIrZFF89KoUnuMHaERL66esi7wpVzLBaTuaJd/o1tKH0WXEm2tLtBJjiAYZTWdrdMZHSdJUlSjDrLca9xk70J6Oov3JJp6cNFh0MEtagvCfO8W68T/g3Uo9UmfA9mOEelPsZbAESahpzjLRQ8H0CKNN9Oa+SF5zbL35Su74rzOZSuI8IuwqGZy25H0JU286u5heGFCMDMnNQtP113WUg3gOCebS3yVNRx+C5dQuVle2BOuzWx7hp6hsWBtPG5zadDg4JeoN6/peGUEtaBO/AEyQ4bNF2VZEzO4DteyhUu7nZvr8dgF+0646VYvDNzt63ptUqhz5XhryYL/pbhdT4kjSqMrodpjkxh1ZU1t1O/RFO0sZOuJkkUTOaCrqFWdQfd5/F5wpRSswp+FH++BRQcLlEIR9Dp1ND7E/QmWSAh9c++TN6y+OSAasG+SlFgz9FRTVhlX4B5jLCUp4+g05Wrd+YbR2d3N6MXOwc1SN/+R77AIqopd9Ab9QjIpb4zsAzl8ysRiWIwyEBAyaUlBCdnfNtiuyVuH2y5ZEKS25yyqcqN2VBCPh2sv6103RQq3hyIU6JFk87ZneOvThbyyggyGX7KK3KoCs2e3r44obS9tPo8jHQn6RrhtuKLiepsdH2it5XdmcT48vRl2lBGeMq62O0gyRIXFCD4m/d/3fPpBbHfbmFBtAc6InbgBEHUtgOeS7Q+39PfTlO/vt11k9sY6CUxJ7X6o/GpC0GSrhT+/qqwcFl2rOrSEhQi40w6JRyLZHs8Avuqb20jLWWVfGM5iABhX/i0N+/M9WiSeNLfeVTV5dXsNmwtCT3+xkwr9db9I1Yf6HtZPHmB3wON3QrNGx+pYbzUOZeLtQP4WEsSu6dSkoE3ZJwo1Pdz3WJisFtivm6/Wv37IOlyFEJQTIHZroVf41mIw+/Jnn3m2IjCFP+HPzC4mKdKGaH2GQ/UMbWZ3BKw5dJRvFUEEFLMZn4GOULmqRmSy/qHLyTjAbAhWzUQY5klpCRG67R+ABTSiHFlzGG4peJfXrir8Ttl6CV5OynjtpzmsYBXMgw4n9Ws1Z0g++nOq3mprXFQCfG12//LMZKldqI9B2XyliEqiD8Dvfm/V1tH7wynZtd/ix9T06g+EE7pEVqEs+pgKhHKoXUMik1wfvGp+gLv93bOU8ScQ2v9d+VtLLkkcyVKAksVUPyHim/i1looVAKfAAVR7kDfyqeFkYZUYkyFtq2JGgUhWhXIOSWj9Rn3gEXAZkCbJOCKCuLqwYaFFKlWYw0tf/lWds4hWNhbFFsILylDyOr/0aUlMUA6VzwOkZBQq4JvZyxwoFOlmAKjVc/zuJ5juv+OKoRP7oRfmHbX3rnbDCvKKE+QoXSinIamkVmuMl2mVu+ffIQQit0zVRMf6/fZzxyl6FKnj4sBMZ8kU+kCq9WHq4TrGf4ysUf4OyIOePh1NzbEcb3ojpLCPk4YdakYzxSMdsuiaZUeZPK/B8T4k3E50HwsdMZhSY6PKkkzEma1PcQIYmlmIpzG1HmKHTpEiCWgOeuvSEYoG7HMEm8SEnu9TuASIM//EQbOaSfOv5iZ124g0dxRKfQDuyVjarN+1TaRyY/OBKF/eHK5De0UnDLtEcLQ6xXlSnQcEuPuDxxYmnWoDllF5f03L6qggvfnjxGEPZ4HZjS1166gQQE+9e6B7QruKbE75XTLQhsrvfceqaI9AMCQNJj6eud8mjUr3NMlOXCzzVr3b+uCMn5vn83egBeRhpHCMy7/n63LabQ5SX/7087wh4CW4ub4ZAGuPi6BIwkJLVBDK1PWZ5yx+7X0HsBh72Rir/Bu3f/PE/ehLyOu8I0F/UkJdyhnmn33W27TlWq+9sUKczzzGQoQaDQUBM/x8iAwSpS5dp+re2NZ7oyzIkNyzTNZyu9Hj3xIx3co90rqJbqR5PvX6KgZgl3vvGC6G/smdENVlLKbhVtoCakI40o/4OisCnUwWkDFPdXoxUB/Nfz9KLHF/mI+l9/D1EKs6QLLldxDBBSKCcs7XLG/SsE8fzYG3GZEvqq67B+6rmYNEQ3O7jGzpK98VzsRCbQvt9WXbztZ2tY12p+W/Ae4G46okTq9DwtSAm9TIiN/BEk6H+I1CYOuFdc/yUQ4/W46hr//ToCYS1v5CjwrpdYlLzHFnHqBJ1jhAaqGecVYoqgB9oTKGgeSfxbT51LChhGMTA/0Q7O0C5vIM7BOncclzrzldTt202/zp4kNawUXcuGqK2dLC5Iojtc+mXF6FKd4w/RRVSVOtZDy4gIgcWzFeN/dIs+/1aqYBLqIvETkMk4uByBx72Bqvtdi9NjhATbF158FkVIIeDx5THpqm3A6UIpKBTLrB0k6s4vtIjUy80E2x0ysNuLhdq+da+xR5h1qYc7wM06FPQnJYew01XfQJRB8bu5e0QrCqHgwPO9P+jZEFXt0ffU6H0eoqbpo0bFYED2QVB1qyzip+2bRP+9piaIZeyOvuZo3+om66XWZ+t9TuqgtBkY0/29cXBhrCzBHgWRVeRlpfClUuQ21ZgmOmzoL3yBGvkZUWjWU5L1xacA/2INFerrAafMEfyVxldQKYoFgBGhOW95lyrtJPGS+6QNCn4KDnEmP0hlJ9wX+Sv6fgwMPyz86C0YRE/d3qjAb1IYKAw+ha5+3Uxi/kAFjcyMti7sd+0MKdVDingtdk2VLCyF415OEKddZeJTzV2CnOV97LMfY+lyhpiyLdO9p3N0/Xf9lL/2DnrqIqdoGucPB4m38aJ87M9S0D1CByndlXVp/zV422rs/+0Z429GXV4IP9ydkT26z64BsKsuCzR09GNjhRwm8xcVpEKdz4dxAAZA2v3+VwhwcuhWf6oPYbBjWadWKdUbKnz29ZTpIg1SpRul1F70/m6h58Zl3gw/cKGKAUIM4Q2pE/pG2AQMPBZJqGKD+aNeuygG9efMOwZ1R876GbHNNV4Cnjt3B3GOyDEvp0fSd3vBMPX+7h1Oq1nRStIkR9rKWJH3u7V2SLFUwuGWFiOMHHTawfYXmh9NnOOkQszPHKkqudsV6qzo756v4I239muH9sp2B/htojZejTXTe1w0ROpJWd2Z61phE1Jmv9wlTIOq4hh1Dj9+olF28n9wTnM8VX2Kn9LMk0IDlr44xPbziEcYob7QfvpcscNKfllM3pc5+sg2cfvtRlWIvzyU1SVCUsz4frFKzN+40qV1sKyTRyxj5h8CNrv4bvBu9A0Dj4xiSx2JDl1PHTLsHIOOukKJHpEWNG9P33KZH4KTdaDwe0KbsDnGtR2XLD2zH31Iz8BgK/yL+5qmhXXiux01L/45Fw3dq46TvaExsNK/Inv5WMrWB+4py7XiUneBEQ0FxhahNNBME7f1Yz95q6vOYBzNfc19M0UL7AhHEKOS+bvKZfEioFMR1xfz52WWbTJD8NpT7+ivnKVndy+Eudm/WiD/7lG7n1JrROQAIZdtFL4EOIJKfrXxox0niuEWEtk1S9Nd0ktMDbvvb/1J49Xv/SW2jG2r6+4UoDSqPKRkkfuTBSpoMhz3/isI1dC3mXoi8Ko3StLHVm9pDwgzrmT3byh6TRqs9BGxPN5HQTvZTafUB02IVJu6UaA+EfPjyGEJBxg8vyHx+sTvmZ//bTbP2DVS42FuMvHDZrFLyszM7XwuUjrn2LUh0hJOPuwi2iW5rWjwvPYsSmHgihAfzAJAFd1da5N+g/bkv1a76jdqYpsqabAaZGO6l+Hcn1TswOw+ELNrX1UHmN4y91qeEgrmLqneOCHdRHObzHMmfFxzL/nUB09N4rOmasxEzYy5P4fBLL5+aomqtot4WlzJdWC6we2g8raW2HnT6KTOSB0XOyVnl0QNZNVhX1nOD6JOuvvOf7QAEpEn9DcTZJXBjEEWyW93PrUSv0VnkNRLNwYzLFsrgNss8I2FJE+Bvi79MyIZHcDOgMLvI2fPKq83aSyhxKFfgoZU6AmMLzk4R219+5/aUc44bRn1xYXV6SJzSe4QG2u1SPvnDCBsvhJUGuw7zTO4Hy+lo72LE3PGx07lb90i2dmVJOCPDuCBe1476RlclP1VNnS0G1h9GN8ktasK3gvq7C0gfUg/4KrEW8DwxAzP+vWLBb0nQQT0AjWtvw2qw5Gi6TihmYAV8GgQGEcK3z+Rcq4tnJfRkQv/FzYtrins6NX2YXEk/BQnuami8tE4lxBEdRXD/aPuNWif0VoevFpkewNpgdDkAS3MeH4Z7+ZhFfdc5+QoeAARTzFl06Ft92VSHcTEabagSymhIB4RApVTzs+FNsGKyu74RE7z/usa0rjEytg7FtvNtJzNvroea0NGPscdhPt2nONotsqAiFmoGRkqVgyt3r5+q5cqSJM8luPROe95fh4IullQKZdGL4kqgVIFnzXEpxXv+WImx5y1dOMUkzgduBB0+ZL27e8mDCAdQCCS5dfSCzc6a5KpfwP7lRsVpcY9fX4CKbqFGcDYHKqJzlC3j/D5re1nKRoTw3UvxvJz32rCaN8zagUBwaBicuY8RJ6hM1bnF76+DjLrbUnLSIPWqnsIYF97AMPd2A9OkkbPB13NvP0/YOkvtYH3oTOcO4e0RTqcPKuw3rTPAucGX0FSmGAi+scLq6VmCx61Bzf3lc6P5WUhaaUJAArK27baf5YU48XWgCU3JYCKKriBLSEiNpbDI18Sgq+RyS0n0ccIzAJF6uiX/6cFcHz56IQHrQ+eYrU7b1ClzR1SfvRtZ500ciErW1aICwqHYCSQOXRDxWFernIxjPVQgj/OCw14ZDlykqDN6a0j8/ODm+X80Nb7q9vwAO7KkYZfF456+YVR9fOGtw85cFv7i7UejItnd/2+dD3lVprIDYLUKA3/Q1c986kagjq3z3wLvutTjiHCcy5Z6f1bGr/kiWn/3i/fEMgfyTMwVGMMf/LnZSkwfmyNjNdb6Vy/GYr4UkmqSb/uvoRbZn9PsdQHM0mNbZdBsRwbbN8GsTmTmQibILF+zCxugh3n/OthxI6VkzOY+hRkN2QfLMvW/b4Iww3/NWGvuZCBInSs71UMn/uvWTUXtcO+9/7hY16O+oeI8sS0YPtmnAZpB4dMJi362apsR1tgQfJIIMnfXODOZZIW8gpeYSPHly0jggBrMxoQCm6bhwEUEEaiMPAWW9xsgYLdQuMGMq0GELqFlTU5O+oBHC6NP7RbY+pnLba04qZ+29ZRnY0kNLuGLwqXrPgXVX00wls6WybMyjs1bWNp9V/rcpvMn+/mwHGlTSoOpAArE/42ZatwskZ6tMQCvRz4fJyi3oP2JuCpWUNkbuZPsRY2xkpopuKVLj6ADsXwXenievUZTUyxqOuNSi8Zwkd/5SgQnVl//iaHH5Y3Vdbk7wHyWMHUm2E+KN/OFU9hWSxkBumwZAXSzA7Xd10bBAoKPh42ybTxWoZqHdFaP1+9D5Yk4O9igJ0kOqG6zyCw1pctVGpmXwuWievcF2kcmo6VQeE0p3skn8cjj5KE0hK0FZhp6JPVHTj3IGJdbuRS4WEmEtriRyXGgF196lOxWX5IyPkFiPOiZHgotsDN776rvvT2aRMWJObWZ/jrJK/smORaqBcFEWgTu4yo/kHwhJ3deIENYe5aL6nootMGICMEV1RvYWbdQmzrKgFl5aqtieMQvQUtOpetAvcDvxTc3yo3mFPKfE1CR4bEHPNxnQ0mt2xZpYWRixxTZjDbIl3VyZVoMIora2cRC6w/pYDNsUMuvM0Nh+Yo0BAx/8bqyEZ87Bw/l6z3Bj3MIFxdfWL+Peqw+fWcXCXFNMZ/yuoq6yWqOE/WFSPv9gvCZbhv7YvLlSkZ49dIJUbgiEZ27oQngBASiQ5s0HZrd4ggzM+gLqzQud7fnBOdREPc+aWX7Bg0dkoaHwjM35R8kMgXz5ZprUIlui/4IJD/lhcHZh4YzPtgdfCa07px0zqatBHQ/aC7Gfzm2Dgi8QbL2GjDlmI/OVbta9fVHFnsXbK8JjCFSuiX6oZDyiyp6O7YbTXOVPHwv+kxLY42wD7U7GR2Q+5tMHKHVWbdXIwZfj7AQCZ+5PJi2lZSFbyXWqIjBgrevK8w+4yMIus0M3nTw30gE78DWP5nTrBSZsrHeE0qQi7Qwwg+V85XyVNPGd/kFUSXik3aJRXCC5qOISqkv62rDC01MlH751Zta9V30e+4gt9hDmZ34RH0huSbfQ82B77koXAZn21IIyvwfpGe4cU8f9zQe+nm5ygrp9ol9Fi8529VUl3D9WZjJGjgrFKQ2mANWTndr5201CEy9Wz+tUHzA/rFGCAE8K3DyWIQLuP3Z+aooTAgfziFhlyY8nsABZKxM5/ol+5q+FhVFm2MuNYRE6d+uYgq7gb401nx8o1ySZzK4gAirVUvlOUrRjX+NyckWUC/FaSKsOvlg73T+Z25p0toA2Lwp2KmRkcVIb4VNrGZ7aLeMV6L4s7FMnl2XtJi43IijaAE7NeXxMq6OAAzfTxBvqHdKbOr5THGlR6hscO9Mt7J0rnKrg0znChrVXGxPG8MYPkPQPbD5702vaqzcCgMAihpcsACk5HiTt9GTkcM73zSnSVjPFbC3Ej+6fgFZRXBlxwZ8S9pRZPe/YChKyy+U+aBpdff9idjt33LSHKGf30ycAQM6Nh/ufuMIVt8JVrDaj/qIDnHx1cVPwS0leaPRK8+Nkkz5phuxXMKKCnLFQ+7lG/fktpTDCnD9Vxr/xMIYUBf5G4O+9lx3eug9DdOfigSH8sA25JpX066Ass2RZlI5zx4cE+l0svmsNaVZYQ+5QcwhZJV/OpDvj4sOzAUEcxh2nxVlj/CHQFZNVeTZYLeY3VySmjVYrznbQD/kC6UdvZWdLJFLPs0CGp1V5Ja3/jwwkPBIaeHXYX/9lYj3y+vaWkoZEXRiiezLs/krkECEsWhJhqYHWRSmSoEcl16UEazGDWhcH/JeSsYxdSfJ8lhMMsdYV16aux1lQw6zsGLV9QHtjboHCeQzdrR+cy/ppH8WP49AD8supE8UsdhWek9u3Vd0+VvkaW2VVMl7JyQY9zOWL4JfdXLGnvw9hpOLqqh6LdIfng9QNuchVwoCBDr8oA5bbQufNoQ0byDSWXv8cd6Lk4vmkq7IwauSv/aci1yhxitxTfgET4wH0xaPB/s1ipR4HvZrETfz1dFKwIUNd52nis8RX6gw/d5r+HqmfmSPx5oVn6eY3gUeNsybb+zqnFLx3zzUhZeAbXG2XIu7chanwXoSXjM4E0Z4I0vnLai4n75KShqbxZcfRnai3xVF84NawRvUBTn0dWh+D7Ih6S81MOOxuXZXMY9ch5wwS4/ux4S9Esc+Ma3idahPvKAk0unhMbeoz/xgoP2kk7k1MCb0wcFaw/Hdj79lmiZCYG3+ijAG91FYfEeZbdCZapq/tpzTk2+M68XzCYyTfZLUariTVWtElDfXqUOhs1t03ZrztO5PAmyosmH9tTwyRRgXtTZ99M23YKNhRZAb+Pf8Iq8CpBbVU0kU1heaQ3SHRIQkdrA5tNTLg6tgItZ+tyLFA+r2Kpq4s7T4KRcQYUo17ALv/BnaXywm5Sb1Vu3wdF8SHj93w/Fra9JEutEUhHK0iVBkzOTEtxN1B4qcV47Vu1KghBmzExydxF7p5unxYGEZNW8fbgUxni3zkmyudRQHkzXXOiilMjkCZO4h8hSbnPDn1uepimP/Z2d9LLUzatWVay+y2Wn2Hqm5WsEKgT8cn9kO/3761dFodkC99J5nm/Xg062W9pbtNDGFRapKNgpnYM1WyYzx9ui0bfsLkb1PsUO57fF3j0hPUW90S7IFPV9Dto/DkbCtBeusMTK2SAiU9qHDeDjBQ6k6n9gnZ899G92vFB89zr8MVaNAjWoyxFABP7meSaps802fywWx1vXDsi12ID6c7EvDz2b3Fj0XBFLBv6F9sBNqcc9/7yUVVQjnz18XxkD91WqLLfA+5aKd0ex7OSf398SoCz2eV8kklk6P+brxIKKhyit7fkx+X1Gv3AlXc6Uc4Sp7GeNtacz4B57l+oQUuV3q3ejvaPXN85lb0RDPNDnqemn7ebXqgmKgFoSIx1bN/Vf15Q/L24c2Kirikep3ocwhvgOPn6/os/yl2q0og+0XfCvYkZM0w0Q69hiQkUmlIYqDopBrMntA/ieAm0+ZoWXvpqMwV52Lvlju3+jysLVGWMsEET/JnXxpR+mfhToAr55hC6KMqiQGMw3onlNdPu/SgVVOqxoIvco5Z3cZf2LjR0/ygO48mWkj1xCrzxHs4Jt3Tp5cQ+PLDMXMcark4OOsiIUmk78Ro/pb1pL71fsGZ09N7ghtUGO5sHep70WEb+qNneRDdZhxNSVGf6Qr7+px2naj7tHLt35cCHmcWHt15Wd0Bu4oc4V/M3PHdBYHxbuOx9f6ycLayzYPhVjf4PNcTRRi7+UMh0231EJXuQTGK1z+JbAI4ptzWWBXNDa4V+XuRFYT/LRiPDfZjPoOt2/g3MOQNNC0moZ7DPWi7ck5Q/dC8FWvyBQZrLncQHoxhxs7uHqPzExxc14YwIWDkaGnLWLivCpxydLcW6WCPBlPMJx0/pHyoOd+23bzyo/Zh01uBGveZFBi/8y9feDLpkZuISsTY4mktRrcprEOpyY8fOo2mX6KcHOTEdVGRqoHL3vpSBujZpn+wPYPCxlgdvkXJAdSP9yWFV0dKeMBeMP34EYIrDVC17NYL7mqQe5vb8aX9Znk29shrQPgMc9hYfLv24NbXadpShEzqoBztilqNPv92szlAOpM4gNc8AgsXNLrNrkB0gyGQRyM+d3o+4eFtjYZXakLnbmx59IHXXsOLujhzukNpIacOUU/7fM/pBQ1f9rXn4aKLxfMxFyh7GDOmDdM3N84Ikrbo2i08z1R7OyIqryjq7AF2BhNcvBZ4VXFDp01apUk70OlfK4F/mQAx0wqj9w/TRIk8HOZab0OLolNswxA4P+OkHIZZtHL8FDpJ28buTz11uPjWLU7gpv5giiDRObTCgeGir6K5+pu8SXB6SbeB/7E4wvTt6iQYKrm5I91U9shqaqMJP9wtuhvOzmOdkNzSqtQBRBTs4KZ7mtQoUoTSco5YcpSL1p7nl8j+Hg41vyBJ5hYb/kVqq5p7P/SzHa9ENUPsMvtF0D+3by6EZHm4yPoyhLsyjrLl/Y8ZPJjomfICSKjvP9rfSSykHa121FuADe1exXUDabt03SCOFZCncJFynl3v4GJQkPghJu5YaXhkOxVzYg5e+cAbIu5lffJSA61KTmum7+5bUdQmZDsI7V++iZqeLmlBYnKKnzjBqKUi/MCYq+tErXRSVXA/AaW5j/AP/mNTGma22QYN3lpakrlHwGZf51HU52uccFSUz6+gI1ky8JD1+vYh7/MKpwnp+NPD8qAMETKg03ZSrMQ7dS1L3mARRUNuvglYtEMtexPfZwq/iwni/mvxfS2T3Va8ul7n+bXS2scOiwss+XVdZmpE+m6hlr70HzOOHe4J6MYCrufXJA1qwkxWCJLlJGXQsN7mP+jSNyjZ/pnL8NaZThx6RnjjjUhs0E0fir/wQQzHDJT8bbbtxuO2iJa3Z/y+ZZ+rC1+xz8diNvdeTpaOvkOpm0v5trOj/gyvaDsSA/2r1Z/WCEmpbCyGNHLlP5j8P12OtL/IvQLat9Ji0l7cQsbeZzPHMwJMAdqw0ikK4Ca5oUYZ2RqV/XDRKgcn7vhjLqAE30k8nD9MdODocIJSysE/xeibUE0qx6lI5ch/nFFL90Zz8as8ToLHKiwPhfKL8+miLjufadjg7xrapyx/gcxy0X+5eg+MlKdueJR75Y5eECeeNB7JLL+tQ496P1A2nrb3kKBJlxrQbVDEL5ExHmUcDNIYWbYyh6v6deXQBCillXeeC8SpAs+WnKUM+zwwaHFTne2V+nuH9Tzm/V2zNVTJLPLgT9P8FvhjwvdRbr6lUku2eNR6XjRQa5i5Pn3vnwqQ/0JPL1MJboCZexRGl0BEXG6sQuDapbDJ5LaMvAQJRszbP1d475OinWb0SKxzdMB/xGBYR7NuEez0kZUNjF7BBY0cMe7ZhQou56zotgNDymu0aBFT/gHFRA0P6u0JOOiZcj81/os+CsFbOfIuHQ01C6lSsad7qWW998gaYYT25q5RFIwVuZPb+2PPOYvZVZsRg6r9cOY+J6d4xqtVuXKCnfe0BVl92M7A6UMTtJSwXKH47e4Oj8KbIZ7tU3Q6EwlMIhT0tsb+uWvc0DUQJFdBYM1VSb1VaNxlTQDzbQ967Voluowq5OvGRpcLmLvzdot5N/ycGnkCW9uGG9u3FaGcdx/9Ks1JwvFQhw/S76HN1np87pzDKAlxH5+dbZ8QwbphgXtBj+dOjHRaj2x0AZcCBN5u6p9rmZGs2H13U6JJcORZpPJccaJdd17II5QFb+uLawRB2DQCtMshnK/yI+sIlsQtgkxX+qTs00az5K29PcWKHkEVevC7SniUhd92E/WMndUBLilJj55UjomGct011Z2pIT9q1Twe5YL77gf1+d/BCXBv9On9FLHYgt/PNvDLfwYH/T5SIS+32bYlryyrkLXAsSj+dixWjHgBLirxYUjqvof5CrSnqEJQYi72f7cgy3kG0wQ7v0axUvSfDFbOOiKmOZZK1DwiyEKvu6tVrLUQNK8/1AvlkmS0UOhQerCB8BVwqUY4Usd4/OuRWmb1/k8tN+UZplhEJoo4Af0NGvWzF93F82kMGzKyyEtHeTQ9ffatt/2WWvjCGpYzNYUwTYY1R7lOZr/ZoAhAcZs5xouxHKrxCbda+H/m+hMPRj4l5B8f6zL9SLPK+M59R/H1cAnYJttJAHE77Fx0Y1cf7U5/dFNKuWK3p7fLYv9x78qtByZf1NpFOi9ojFIXNaX+xi74CPzGuYuBExLyCYjmUyfWg7zslgUVRg7K9PkgM+Bfm8rAkKtLuKTsfxO55UjbmPIe4TRHmxlEodPE5K0Yg7Vl+aXCEkAYiuDzmgeh5j+6VTuHNdRt/5SzGy8PpSU4OUtSbesuHjpdi2V6pWO1BxlT3+A094K0dceJt4qs36y2T9mbHzrBXj++tRvgme67f77S+uFjBtXfLir0eTqakMH8MyxJjzotP9X+auK8tRJciuZv7x5hNvhPfiD4T3RtjVD6nqN2uYPn26pSokkszIiHsjw4AzKhCmhSPv8eWzvtWlJVAjvyh44O6JE3a43hOTcAL5qydje/M8vlCa4ChpC/NPRP0ZyGU0EasemPlLDTiY8xey05Vb/orm0Nz+fIgJj0Dx8A5msF5K+vvT+d4IrUso0V2Jh06SgBAt0Ri0LLdIom3z8zxOoaUt6VRj7aA8yckLbIEfHDDpOIhERs6NYnlx9l5puBYmC78bo3r4bW9HBHDqJoLs2c0Wddwbjedmftcl1x7esC6y7F93pA+jlBM1cdviEQh48ivLP7ODzfAwFGdO8KDnhk9x7UaLDOtF5O1g4oa2Fm/ckaZ8b6fp4SVGdInUBSkuiH2YdA3XWabwGTcZzQ7iijbE6C8gn7H1a34+RWmXo6YtGEJj/yo6FPDlAMdm1keJvmWFtKx4jkVcjPS9KFRKte8qK/dauL646/IKqOQD9b0Z4avRORh4ZRUg77l4BSBdEGx1S5tvv++MASsJ8tcB5Kxfgzj7JX0c1QlLUjF/Mgjy2nX/ZrA7Ozl5zx9u6J2r5aoxZ9DApEQG0WMpW7+/7NZ49BbBuUa/bEVCZbSaZGrosK+MnqWhMmU1cd3Lo4fjrSVlbNbpHCgP9eHCNXY7qS+6YeVFn9uYjxhVnX19IlCil3XplSLgbE4ZWMeTRe/WjyVZ0zdC47OLsuX9QoV9XIJlP+tJU189ZSHvpro7QmQsNdHJtHzB2+tU9dib6jdPCwO3XMmc402j4PnxUHcOF16NS7TcvKBqLgr7QbTyrHGz81pt0f8mqfbowEkTVsUWTezbjg3FmtfO4yTft12ijGV9dnaBKSlw9N+zUprBghncdT9id0Njr1wwdw6knaLiZwekSJQzrpv3wrhc1JcTlmmLKlLT10Q4DvxQd9Nr0v31pV9Ou1j2wWwN8CO6tcWIES/AlQZbW0i2mEdqaQTUIOPOLQ6qb6Mfdw8YW87tmWCmYWtTSLYicXFzLW/U5liLF9Y5m3L0Lv8OlXsGTvHPV+8CwDB3e7giPxzLtCpx+li3DAaRjZhTwriqMyBabzesarqui33IeKlWcKATcHsstKg0mQ6/vSvLKWTdcBA7Oywg+VwU5IA1eKw/ol+k/xdygV5gt4KoPUwmTI1hx/jrIkkTQJ6Y9BGVJPZDcn8uIF+LV8r9SWlWpTrzqeGV+ZWQD33GKOa+5mGUgMqHJk/rHIxWv9dWHKmQmtoFJRLlCUa95V8xObg7U1Yfztcyyb4WnWmuKnjfDVcNi9GbnSnzNg746ADUFPMFdshZzcvZu1BWEelUvXy9HxgI3MLGrWHLW4Sz1OC4l+9nu1e26U4ULDAyR0xvVSLV/Lot/A15TW13hJb0pSplouhZe2UTAyrADB+jJuusx3Bgb09iCTYLiOhIP3rH0qGRq7yy3ZMUvEa3tIiXyE6YACETK71IVWofHMED36hwjtpQh0IDsZdukbXC3yEm0vYdi2nefyyyfwEl3HOteB9HaSe5KJ6Rq381IzJNq345+yN7ivKllJRV0z3bUIoiN7N99u0EtUQAqOwKJ/HnzXIguX6OMOuliRz72FoiAC6CpTGIpQX6MWZsf9iBSHId2L9cEdpEdEX0hSllNRMidO58vgLvFjauFpn58h3vlUds8pqqYuXzQRAytamx0fhnPGxYIR6Vaup13lJ8ZYXfEY64aeOWVI+sg0/zSSDmd17r6urk5zUesYqRN+oVctn7wTO38s80jq0soss9cZE3CGHyOjNPAKwiz+PbNqKs2O5a+urrrzVsKoTzsaRAdMxPpZ6iZ7uSkEDvdb+Dsjbf9FTLhUB2GLyeMiam6ttYdXPUtHNDszPoXOMSKNGntEjrBHReIccUWija8c+e7OMeUUaqaOobLpsDt+J+s99kJZ911xvYy2DkHoGoUeQjkSt9ZKpOHpsuF4xW0gs79/NtfY/Q7BEWCO7/9WpgWYsBqeTIB+fsMlo9yzP04VA2jd5eqP5+FZ3Xqb/mTF0yYbNvqVsf04GBR4XXQifPaW1HsbSO5NupegUHooy/slOGOb+M0zYcKsBBZfueAm/gUmfdZy5xKoU27ZmKmKJKoS5Rg85UD6X8Vj5yhNkNnrQhdmNju/ABYB3yIPcBp75ZklPqtAExEGHCf+E2tbLKaTqAUrsyapUyAdBfsW/YGmlDr3aLRcShWTqcoRX72FxWsIsDUoRMRPaP1g+1FICSQEyL7sGC7agT7z7LyAYtnxXopHSR7qV+mJc6PUhsdwPdmwduRVWsWhGa8qBpbtoxZuBYvf2vE57Duwe9+ER+e77Kb3y2E0nU98O5hiBulZCoek9FRHeG/Omix1JuJFuSdVW2BnFTGw//ytyhLkYJ7IjXDJQxDdb30bq+X9RNf74kAum/3IyvkMJy91HulA2dcAjCX6kWgG25ELtPtC54ta8Mz7RlE1L96TVRmK87IFWZFaWHefns+5eV6uLCDzCsigvxFp8TzPVrrWHIXJDmXAwvn9Fozzf5aRZpUEcmSmboy/AkxWTDr8UgtVDWaU9d4dlehz1GQX3ZzgdwXLWa3xnNkHHtL4f/qqcHGpPvfpOkkE/VudfJIuaHyslYS3ZCfotoYWxnxmFEelA8z5/dUAL89cLyOZlRrMjjZz8hfnY8W8K7/FBi6lJDMjJgSJuWHaa24bdtd37EYmTXOEbyzJCwwXhL0kUnFz4kPuo1RnHebzhfyKQiTiralyEuA46KQw7Mk2f53S7MBJ152YUfvkkUnyJB4T2h5KF7VPqwi3Q/J30YUy9EsNztQHYhBxNBmIa2VSs+VvmbnF4LDnWvhL68aAHOaIokfN6BxmgWMwlN6HFVzHYOKq07ZKAgLz2MSFPxic/3yh9YGEqq+yh2XD1V8wvBac4YwS9zpVcSYAqX3NHiT2pnduJOM1pfZnE5LbDM358OZBeRPymrBaq2O0jI14NPo67ppSLi3Orru0oOL8bf0JViAtdXo3LsGTuuurYx1vHJh7bmWscfHep9ucJ4494RgNsfUz7cLwTsm2G1UtJLXYsMrY32xh7Ftt7NxSKnVcD9IL9fVrSX3h/Rgt9hr5QPfZ/Yn5kH2n5nmkyhM0eiz37z576SAq0Ujvf5nXYle6kMBdE5kOtRKnrkK6GrQfGRUlBI6ipKAkhrRZqpAwcUKLW7ZTjwDAKu0NQaRbvvi7xT4iIh4HnEfxbJSdRYWuYdeN12tV8WXH7LMdZVmul0pSQKXCXfYtswbLCNxK/slPDQ5PBygMKUsTdAYnQHzPn+mXlptGzrl4+mtsKcqPMgLWEZfs5BdB+Wvf4wC6y+fg4MQ7jum37FY1O9q20FEeLps8E6p+vKOxdrdclPdvYchYuMYcsruBAoVoZN8s0ZKZW32PeGWuhapcYZ60ygJHxSSiVw8tBiLF7qhDIsEQhWoQ3yA+FN6JvzCloGo/S+Rxxb8Gv3CjcZh3QWUb0Y+8rb6dvSpMnissoJZ+YPfY8fNJgh2kiM+Hc5VO/ZvzVzj1jhhOySbtGGwd/7LBeU9zm6zXvfBabPK9/++uVzoVEWI4S65jb7hey/NtkvyXJtfBK2D2rbV6lUrpxdnpvtnzf5Gr9y/CWdv1AvlNZh5RcRAkfisc1O1ufBnyx/14qZFuSU55heWC4ZZHzQaBb/mqTk+Cz/0AbtDi/lpcyEFlAcSdDXe2Vmxh9ZE0bO9pfliA6+I0mLvY4Ko50O5g+qbZTAkSGwkA7WFGl57WHv5WOpkI800CxcNPbLIqZGLa33o8EsaCqb3f9YSRt7P4f2KyX49y8F/7z6VxBXNXCitKE1UBz8bHUZkNw8hlb3G6VcXjdjtVLbr9+rgNK0ZL7NzJk/J7+kt4rGxEfgQbc0VqK9cOYqAVZyMcDRCBmYVAe2RRzXsS4/OCDkxZAg8Ndvw7pUDnOSoUz1PmTP4pc0sSLLf8+DERutwHSm8wP769KEwMLsApr/iNAekRWyaUSh0IoqNsM5jRHsr7BmP3dP+ezXiObdmPnmoyNnnOWaMSMsFduvKQLTW0N+A4ObYNksG21F3JzU3BmIZRo0BZ/baA7ou90ziQ7oF/eeyVQYnJ55oIAZP6i588gDrbOa6LNY6TtfTKloABZKvzhxHGr4np4r7p1BQu/TPvPM0iZAV7cFsbgvjuSqWN7rwwRWg0aJt1XVGi+JwnnWi7mi+M3gKVVc38jIoEjA+mpjmP3XTGDk4Cm/RXXns4078Ws5AAnrb9Cn+lUyuboJVEK8IKpiceWsbmw20RS6t/O7baYsRYyK0XonuJGy9gjXlX7mhoY+pX2afXTdMOhRwET5PS6LZBz3MS6I+ZLU8zhAH2y2EcdiI6O6TV5t2rXldRiPATUe4tM/t/mmC+iovPHPbUQMoiQKfquwNirSQHSQL21tVjrfXx9KidTJTvBc68r790O2kpzd2w+28xv/3zgMzBCNitwloLdqafaiI0C8FTd//eQxzE8SJQtFCHiA+EIpjTtUAjOS93OmZBy1GRJ7p+nSOGzJ7XJhtgl66Lx3mMSJbgVdsjsLxiJK0qFQxk8Jg3qRbFLVrhvNf+fRbAu/IcacDgn9dQLnmsLm840VXQNHBGkvXaa4gL8hdsx/jZ1/o4PspFnpAxUe++sJposwj57zThCm0nS6k1MmZsAHmFWkjErMyTapX1qslKimaOgNYeHGslZySGjQyfeZ1Zvd+E66ldj85ciRxsM/P2MbImtGyxvSphqqIm2JPaPjf92jddpZ+kLT288UivB45HXOO8q0RlXwcwfizH9XWgbDAtcnhdQzUPau2WStxyp54MpjdDFtdmg7GMP21T8K0ieVci/6Odfir74mkX3fZL63yx8ALF0fKCwo/EA/meH5JoPXNeL5/+7mzK/+aG9105XeoM/j+Hcd72U90AznL+zEUtvqQ1Q0xoA/rh+Yzgvn3oryPyj/PyibJmtOPFPB1QFrOgf0ksoRXGi4fiX45fPqAm/5hGPez/9s3rrJC/xEihwxlB0vRWIoewBabLNsLNF17D5kMBSHOFC7d+jgn0/XWc/1ry+uOoLo58YDCHz4DsQ8BwhA8vrP4jNHLSjF/RZ98iPmYVKn76/YkX4qPHNJy55E2Wv+9R33RTGhHAip7aY9yHKvnZwJlRt29PONv+LPsFdLy799OkC6EHLfFMekjqYJW4XfgCT0bmwpdlvHOH6S94IWHuSEZYeaFfrdh+Fn2+9jYm3m/9Ef9ph1pgKvbA44N+hP/AlAIJX6/pXAcn6tMYu1oPYEvVE0TOA33f9KgnQPZwDWSNvjAq3oTe8d9bfATBlBUiO76izbUoPvQgNbYfNgPfRWKVUt7d8MxFx1BALKmNKVzweww7c43Bv3qhwmd9k+NxlGef6y2LM+b845m9JOTBwXGL02WUfJHemDjczvy/iqeuNAKCPH/0Yj7jQaBMdj34hH3bDsb1CtRI6txp6RFhBmG1+UGqXv3KX42rwjRhAEcJFWPmRBgc49aiplunPV//IFhCmYJLj58XeN8VxTKYjaotjB6q41x8EM5+2RCyUQaubFoZgo3EdiSVBUbpyrDhyn/H2WZaj/e83Yb+u1EERyzp57yPb7eRLwBVzB2lcSgb6gouy+CSnzVBBHEDzDIVbvN3kGY8pXBkSKbaqEbzxIob08w7VnKbnfbdgqWHIxNJ7PG2hE93rTrK+jxs9nt3J8hOUmVCygOAZrsWaz/xpYspv/q5bxGx3OG5gL3sqdTOifxnED+IMTJVKxxzMG7tcnxre2raP6gnGbqZtpJe3en/czE0AIrEPg8e6jLVCuvaB02X6eARnGnmniwDARHWTdkbWerMVJKoSqrL8ZaK3HDIQpx1L7cs7GZLclnQC9rJu/ofExDzDMQGeGeLQZhMtnBm/ZbknfonnEBXw3qbwtovDfOFQJHhpbFuQ/E/78fTW/1BIidwlabhLTkeI+HgTs99tLfCySMmSpOjlLAXP9P1kVeLIsicvOXK8XBOV3rSJfzH+vGVPgEXutivggwfXH+Plu1FeEg0Knb3TDt9fckLlDXZDGO5dKPsCPOZ7PsizKaH2S7bV9aMC3NL6cXCaIsXQ04wKJeiFM2xEKoqFxrtac5U2Kov37JIBJQgUPB6VK5zN7MgBDWfPw3r6jQ2MLQ3hdYpPjpN96Or8jknubhxbpKRp6BDufuj8n5isS6qG1uJMoDKeIvr8wqmc1FYq87b/9w5HcCc7c2H7X8OGrpVSm2qb9YF/NSrZ3pgFzvHgsts9UsnhoxcWdiA2/HaGzIMxJ5XG50aZC94BjF0Ohjna0EoVVSmD3iS9aeL9eWyOhuDl++zoRblb9jfxhKJRr1rabZ0H354jgJ/mFxqr2UNhZu1KdP0CYjuXitvgMVTmhKMJ8g7BkUFz9j7I+eykPtOS8ocWv97M7fvHaPRw5AZygoF9F7cjNK5ZPb51ku9aAtDDeTyemn6SAIKKYgN77tctrX8sjuuK3gYsQNhEcT8jfYrW/OipbIAafQssqlQjODDwD9v5YW/Kl4SIid77yg+G3GJTC0qfDnAy1zMjUoZ9ByNiey/xaboV4PIkHTtg6WG2wPVUWiHr2y4udfJHbSW66Dx5ZRBWCTxL+GIf9J/2qWAVwFKXXi5ZPlcP4jRFFz0EjoTB8Cy1XNWCfpRG4Jstf1wRPL9Jrh18awcqXh2tru02w3BdvUh+moyYgQsB87Uv43v5qcUB7aTWV8rFCQjnxKqPBK163EsHF0tg/xQ02O6gmuQMnC1Hl0ouCQyCz69rR2R/zAqd++QySyYi3WNsp+kZRxKOtvcuyoGEhx+pLqpixvwVE6Q+9U2RW+T68k9pMgkbB7PDLW6Xf/77y16CS4xic3uIdJbYZHhqTbq01eMhMf68wbRrSz34tnxn+58z4Fjeq+I8JdOCoc0/sQUMU4YnR2VPwMvShgTy8ldyLHflJefaF/5r2IrJbhvD+cJqbROh7/lgBbQ93HPn3N6ppMO7gd/z5a6NQRGH2axfFpp83kcWuR5nEl/aIavXXSqp27NVOIKGv/UQVz6SnB/hPCndc/8Yj0SjLvnr2om3LELrOIOZ1hjvs9u4wDwjptpzI3Zav8WtxDOY5QrpuZ0fq/FKvDx8G9AObVvGbXgi5jV8HZF+eNmEVIXHwNXlliDEkUQKWCyVc3RUpDex3sx9f6ClYKT3YzTnQG8vyR3JZN1G+UuGqVRQcMEytRd9Rjzk9DQCD2ObrJX+afKi/KPJJ6GLWU+gDeVEK6XllR9Y0F8bDB+kGHz4nscsfV4VRF+RhWfmamBNpp7t19IUFsPD1SmcQQdgIa0J/fCHSQMEudqUV7sEHoKEaybXbZhU8/9XXE+axOjBPyhFXZaAl63cU3IyqSdN+RRIfqhazTxy8KbEWALcO6FD8YF6BtsDTY7AsO9l6nd1xqzCDGyR0ToS9Fy28Fi3s4W0Buax2MLlUVt2Gg39BXUe2wmEYLSPaVB89/7znO/x7H8u43OhIX74b0nK0k4RJvpsFCayKYh32jbiWL1mQ+ZiNxHIdnbL7tvTy4YHgHSswQtsoRHwSzuQJntdjxWcefGS3ArinL8O4BAJCZIFmVAKJvzWLRvkEEjPEqUVoGGswRF4mJ9Zy/uw5/K3pJF1FQT+iR2jIMU/ZnCtYHX4jhOG23YUaQ2uGWI7QI1ueB0xs7AlspVGuNQwh0QMu+C0ku3uLgyKDc8PxL4EfVUhWB/+zPFbEaF79F0anxgcyEZPVvtDnCuGmhPYj5fTQmqfuL/dI+7VHBFwChuMIl7OkHosmfivyCSU++GWEJJt+vJQR094sqLvptZ43atv+rLLsMtWAG/6sOL0jmdXC9nZZUxWqWyqiMXZu3c70hiE4kz/fU94zuHhF6AE/A9s+ep/t5zkTdb6AHIDRmqaoiAV1ttANprwtjvvhM3ZQB7FKCuUiBLPyo+ijcqln5M2ZLOPZjVPDDFZPl7lRuPbA1WjYuc/0unApkHABLtaKur+/vHXiZ3rG/ibOOf78EdAyUvul+XC1inC5YfunpDsFhsycKzeDKAo1yxmSgCTY23Mi7ZMZ9G7Oe7V92yBKpzrv59mif/ufhqw+Iw/KF9mdMPnq3fkLobJvrY5u2zm1xXnAmPa6j1ZISDULLAO17tvo8JliKqlhrHCgopGsthVV7jQ1rhF95g/gf3OTuncGGokUOgGWRgKhfCzxTcyQKMFTMZXycflATXwV41dNvpVJ21a+QGzc3gZTNxVTsD274rw243YEw+jXa4ytAotsQJhYdeeDQrsjbwCbcJKzZsGHgsomhiS38JfATvEfPfEQM5dKsQXKQWhmKp04X26VxJPeNl8hwURncNeRSJw+OBVYXwN7JoU+R+P102bC9ayFqziWE/svokHfJfP2jjZOPFTopFOt3/wjQbCcOfJAe9kw4PK/riZA289I2WZkMWarwoIuC6JCsv6Qem3y6XKQZAK0fIc9mH5pmrd/CujqjIjzZt8fttO+GYzm8GAEaBp+eNPCc3x2Kfr7OvIOfHho/b5bd4ovH435QXdJX46C9eLO4PnYsh0UNRuO4y1tPOUPPL4/1ydek/EUmc3fYDTK0NagOP0Z/Tcnf80+f8Zw6e/suIvD7IMG7DcFP2yslKM6b2hUTIwL0SeXI0ir+wKElOLXGP898ohNipAHwy3hQRYkvx/5waHUqmzG/CC8hWezXx38q64+5BAxTzJpAXyXzRJwf+WE6ZBfEF0uRlT8SZs1c5/nDmSavt9/4i/zry9hP7vUCqM+Wx1GnYHLMBRPwCe9uPErnPqzy8bRlkQhOhK4YDwvX5X3qTXQ2PYzD8KIas6yHjzCeFwAVrrA/ncnrI3B5zNEefH2Rj9vRSybxOrlffaE29Gs9Yll+8GKbOz5Za/n4f5+UYYIyoxY2MuEHVXqaECIF8vIeYDtop+UxjIh3+b61hYRdaAb8Kf+oWeWK3ZBNCigmbU4lW358hhy/74/e3r421bAGHygXGZRxm31eij3s3AQc3O36ZdnOYGIAyM56ezh3Dt+/IwVie0lGjRG9XA6cBIGDh513Y5j3aeqHMseyrkdoQYjLamLhYGdsGeQpQ5lLz6jvIiK1A96dV6Aozv+Thgu9C5TyrZjRPDFKqCcdmjUlrfEYnA1hQoLpeGGnqJ0Q5syLVeNx0OwwvdRopMvW3ZQM89TvR+6O7xQrXg222jJYhneEed56ti80blHXjO9oLXRejmOnFT4Wchp8jXg6AVPYIY/dqk++FTkEzIbwVsPcRHuisT7zkZTuV9W8mxDDWtwMu2SokM2Ahu+9EbumHpAXm6AjkqGyfeFsE5mjkx3w01WTkDrAoR/cSDpu7hfwqq18yPjJvpX1ostgyn/Fa9ZKSYo0nufk7MMW5pBUQs+/JeoOJGqDNTtLS0xcnYrdwVC7pPxEQb+c0n7WGG2uOXf320+59FP27O/ddo0/WI7vSXDI3DYHRLA1egfkXtdylrUveXWSIkV96N3Yo2MzGyHhzUzkBAER6KbF6JeB2MU8dmWHFsavAhFo9Cg8Y2ln8/4DeGv62CFnA4uGpHAQ/+d8yiERQ97RZ3ZPEzv4DjWIlvPl1lb8lsmpyIBdamXIXnjnRZiB7TYYTAATBrbsnSCqDWzrvGZu4HCLvibNPUMJvEe1TK6p4Vi1eXiQ1S6FeaVzjbdaJYL/t3RDoBsducbk+pvWmEo2BwakELUwllhOVs3PTj183F3HsP3VSZ2A6HEKnafTcttIi9zIXDASiH5QqeE3E5Cbrf5NuC2vyThgbzYNsRDl8KP2qleUoOlxzv3d4cchbjQ3PiLi1qq0eHgEbSoE86thxrB6nlMhAEJQj2+OJHMfhRLIM6DSH5H2QtHLK2l9PtUv32IjsTOOsqBamfe4SsSlW9ceeCNxRc2W7xb38fbyS+sE3ip6NzKFHUVj9RWmb4LakvLvxvOBRG8e6Pxa3mXzl01XdXdJ5Rwcx/KchOyp5iru+XXFM85haA6dxhvuTgpsjUASZEJ3bRSAD1yzoi/6EwVXywe0NDAuqUoFQ/KBgRyLcGp3grtTv6U/JLthfNhmQ8lDnvE2GJIGRsxWISZ+F47g3VHSW0rU9mKV23JsGoRWeI6gT0Su60kkfIzSAMhjqP6drGY8VCEE83+q82l/vovAQzP5u2v1hVj8Ty/Im7O6eZNqnqwZMvPaxIT6Su2kQR6FW9jKmCbq6VHvWNaAokcu2h482qNlC2E90ZY/Hi2JuMqhhIFXXwTykwBchMZbTpUTStsVR+1ZHa030B26b79TJwEzPfMZp2Cj4NCvS5JigkRfTRNE31R2rQQHTNkoSFf5T0zJLNUyr0Wjs9YtYGQMmtMzVuknVagwO66DBd5v8XhkuKLDYNhOeQXKsQvwMUSgn7mf9rhLDFs7tesmySHAnV/ZSUCT3MfgUe+DWT/snW3nXokCKmjtjav4Nfyd4miK3oQ/ANYfxXmUOg1jeVC55CGE/zDvXF+KXyGMJrwwoIL1F8Q52KN7MqNitPQfdVSO2PGFCmvbbTojzWpFXaSpHOvvt7DXgG3lSX6lrAd16yJvL8VVmw/W0dMwANtNSRBG4RlEsbeHa6VojbTfor808pfCAP4MO3FXfIu1/LO1kj276Ndp6Nl3wwEf0io92kMUs4Lob2vDtGCvtbmTiiL5vSvdTpUaq4O5tRHtBDAeUGyeK28Isn7x6aGt7eUDZ7NRbE4KloECPJn3GdrXKYcS2BqX67QHAORJa92+cZRAVfC0QKFzIN2z+IDXnSz8LhnmoZFI0NseD0wUpEHayETIqAo2epv4ZwYRiPWmjHRZfmAT++rUIFzksBYQntpJhLQL5SWX4OpQXR+FYaBZ3kr/DkYgE8jHWBc7AopwLr6bgBYKarQ3TVgvpe+SSgk8kcU24FfUeQXI7W502Rir2wf3IcZq5ePXanzckpbUhYlvIK1PpKWVZBGWB5ewfP4THUusrpH1ZQBW7eF+/cb2nF09kmXP9vh+IDDZz4b5IuigPfnB9SA1gIGYw6JZLiTB1EdCV9wEL0vIBQ5Wgyd5lfpBk6EuDSL7LVKjlq+sea9MNpnpYXbdXU117WQ4rbw0pKm399yawZYHAx5RBInSk/Ld8hiUPRetJAuC5PhI1XIM0t54d8fepu/m4HcjxSsE1mEp9zYfF3NUHfOIETNV0wfiRuai7uIJnZvsHEOe1g23zculBr0lZTggZf6FeozFl1h6KjRtLKTqus3rtKOvohZCroY+B3PmxPjKxRVA4H6ZVkDgAmw3ntvtyBnSSs8h/wLn0Zg0M0hC+wmHq9bE+0XI2vzmE9/MPXwsPoSPPo9t6yeLCtpgVovrMY0/Ss0J7Zo6S+E0/mkHRSZj+Sv6+qCcnmjxkVMmYBQRkDzZxKau2j52JpnHgGClFc6FTJ/IH3OZUIBRy7oCJLg+DXucN/7q/k2XtX2ZePueDiVMjWE5s/iBIiU8df6J3ZWbZrbeJSaC2dwH1XHrbV9QFXQVA1XqcpfraFCADDbNIp46drBa/gIJhL2bR7CWbV7ZexdIvDHgtBkqwfmWeF2kdIpc+YwWgvJbWIj37cv4uxuRXZwRuG4RlWSRVyrClY7diCHcAQ1OZ9V/6pazk9D8/YMTH1kh8B0Ruhb0fZ1meAnyH0JPUarVsiRZcgpfpKa167tYlPgKzhVXRZ92Bg4XFD4upGVSYJ9lVQStrvhsU3dnqtvobn+Vgeg4uhqeToF7+Jrg7/4YDPPPEudzeB8YG3eybKJ7KAW3TRQRl3FfJA5Bp1thnQFTi9GpHEhhQzNYIwU3ADmcN+RmQ4xsKHEHN/MJ0anyKzbcyIXZQP2bXjpry8JeCqBcw5+/gqjdAQjqNTIu/IDDpPPmTdbc58dw0Vv8jCn2vy8QvQTBvvnrjfU4z7fwia+dVIi57LbRQ2FyEd04BTokz3c5xngil9lD3J5ptQhaI0IcRmTv/SkPKq1gRCztDjOoTyZSVrXK5DpOhBtpgT42zxwYaVtHzTlhFl0QEk4cwjbi3/+LYtMM2kduhBZsWu/kwD6Xia3ezj1VT+qor6tF+NftlUyyCY2qKFp19vbRzVxswWNl62fJBvFfnng0r5JB5/IUDk4M0lquTwR8E59uwLwwWIH+0gHnmwmsFTtEB4yKwPvsVGxCgRceN4inFKnmPJXPbqdOeNd7NL325Iw+KFARWpX9CuHvO7N1gmtKL/O0b1iPqzlnZA+rEMbcREhVIFwXNBinuXOhGBLH/2TDx+djlJgY8EUIU/vY+Y1tRl7npsy2z6D0p9KhYNFvKN4uXNgC6Ufewxx/U43hHiek9ZJErV9i6ZoCQ1kqkMFiDAfTdcRYqy37vJB69DjkkgGcbtfdqTdNWmI6889CgRz6EuoGq/AvRWgkFfyWCOGb35OIfZDM7k9jfiJKMZSft7y0eB20sBREHbyPGtaitGSWoTXpRWVobBJFEy6TXUUKPGAoAcLm1JMw6nxYKiJz9rYD+iHD+9wjC4Z7FRLGsznF88OYnFhZJm79kvP3zNKR1/jnKXjug8on8h8a4+3X6Rt6sZtr+p0/c6Cmr6FmmGaLZWsNE79rvW9O+DQ++Mjfw9HrcFoDYY6YWgQAjAAVrzpu7nMcsy1XZvhGa5vGW9rpak+71dvtdL7Jd7+CSyV0Q3Ms5SaVrs2cJezhbiB/77qloV84RtcsV4XOcerwQs7z0BL1iP0V1O0UvRageGUOiNkmc5NYuHPtlp9Zo0NXJylDd8TG8IBoV8S8f1+MLIusi7R8vLc1Y+VkIJsD26qLEvG5HxWlRZzdD+4yJSf9cNFWwrrr88wkDi+voAdofQ8tSMCsSJI/IKYLlC9/bih/Sd5rc3YgsCIlytP0m2OzudRvQvYIrz3yt+IEtgXEEgPQdPukGZI7PJD1JdbZBXt0ViiKTNC6zKwaeIMvtaLE/uKotR6msRko5uO1FkGmNv9IzehowTfqd3Hj745E+atG2yKbHhGRG+yBsP0xoLN00YYLlYoXNmwKieIA/bh0b28Iephlc8XOTKgaviWRO0c4BDpnPta3J3wlracURSGL42388H5Zz7MD1UxdIVhONlkuhjGU5S1k3WQVg8S+MH+3sV/ziy9srwDT0PLfTgMLc9R8ADJzCUqRce6C3KSif1KbLbtSZAs8wTObr47TYXaEEeE0Au03kjG4vrMod6hgasRmMOptwjIeC2SuWc53fyKMNP7/M1yqUtBR0jxXDGuuK3QVdG3sKV4pD3jCBxoXS96TcHORj8787aTlA/z4TCqoRU+NQ6fUS0cJ0vNcte/4RVNe3AWV23HZoXkqbkFTFCYsHe5QpogyLif1wYsfmRWPDmsOFlAR7xtRkurhH4WAxcDs6yRoGQeuwAcufxNDOtRu7egQLjL7ii8rxQwy+1nn8yWX8TJrBKaXaE62itrmPF6Ho9SnL+fvuSjF/9WX00N8bNEtxiF8wMlife9CuN0SyPdZnMsDTl//yFjgHdsOmuW4k19JbJjRfEzdssyMnjqy7g6zUs88aV9CQJkgzCfrX8jWdEsShjJj6U/3+TkamaVBwSMD1uL5PU6078mJAYNzqPYtuu1Yb4cAg5UzLaVw7LI+RsN3/OLLvAUgrKIcPqFnqf9Pto2u9AMYr7FbVh15mavR/+DYQbZAsUQhqQ+fUJfhclYXAWPkPptZ4VRhfvkeF69+WquR4chuFwLkNFDAqUuDYqenuks9fTZXOiFGdKdPRYr00CsKukB1YgPMDjSeYRzLakbanPmwInXMQbW7A4TB1du8BApvs3sOlwy1sZOhRtAXBQjDu+7CSXh472c0kgMDTHsbfdjP00nKefO/TacNbQgqld3S4OWew/psdsDZ+TTNvk0levWSaterWzCwWNhXjjPsb2rPAxwYzD4SstGgTu3NzaTk2vm9do3Wo8TiBdCcIQMztWsDbWU49aJx2I9eiKMCVBJT0QIsZjqHUdxuftWYUYezGDm9CGdvltb/RaITPZrNOwyIkq6YkdMpyFbjQtgqPU+3wuqt/YxU7Ha341JRUlP1ui/zrUAHBfD/Rh76KH0Bjc/ppNdLxR6qF9/QI66aheYXsBhoXKvGdshH3D54TQrp4NiDdyy2XY5cXHvMIY9024ZRuvswOv0egAMgWzvHpoRtUY16xXzjHMl2xgR3jB6yZ2Ya5jT6MiOoYZL0iGbAGf1iaZDi5J7Y4oSJwk2k3RGlzXF0ZcsAgdptjJY8HTUGF3h7sF883r4GPhm/di27LIq2GwwTH7P7tLeKIPG783aOMg8VlWXxvriPsbtZrGun/ZU7nimXStmfKE2gCKsXEeprgprfFdhua2TDWTz+j4ro4I13/foRR0m8+NyJGB4hBzxENBMzOUHv04BHgF9l44HgOLnJRpQaPt0t2pwRxZp7WKwJBucGMoGDyekeJTEnQCfOIl7b3vmvpgxnR9bl0+uSyuOWTjOVl2+HXdLd3lT1HUNzgEQqi47lR22wasKl35oGAZpP7mByQeMHDQ/sSla9dk0I6ujKkNGNUprW5w2qnv/Wff48vdAnwqgr/rTYVzoSAqjTs2NaNClj04AhK2cvXNXCfp09YZihAv10a4VY89EZ3yp3KbKb7m0Scssw1LOlbp/Gv/9wrJO+rBk5b4UTf9e0Dvv4aRP+NtzsuvLoohPPnDFpGn6tsxuMs3YgJNwi2hj/ZPJvCCvMlr6Y0fmxV2O5UW9wK4eNhsr4+rZuW8HVVY7WD/nsH4Fw/oYu+A+4hrtDsZzv/poEytNGFIhqjF1ndxrMNx3A6Y9RqdhJPE7osO7i7sYmHi78UgCfwT+V40FLCpMSJE2fFDCIuGeZukcqUTEXfKvBUdtsjpIvEtq+olT6I1Z+Q2Deg1i7a8iy/Ktpnh+FyPdaDxMU/ReoKxKR53bRlZmX81Bh73iD4puEzgUyVBl7qo5flCUrLNUc0HpIZhBLCgEtABL/Tlp5hahKjwQ30+tJOBqkVLrzu5qKaIF/mGKB+IWbtYu8QZvGb1zXfJIklpEWSCVw68N3cNkLfRNp+dq6ktYoOzFXzYBEF0RUAbNplFqyel0Ivx0hQZjXYuoHnQsJztNSHELy1c0w2NnEnbT5ZsnT1yon3FBILYyI6/YftQS7vOYzS7RqjiDdEPlLcIcfB7G6Uwqka6QpjzyErNczfcb1BgNsFd1DqTsPAkYBKyg7rlyjOu+vImv2kOdHbew+IW/An5yIz5GswL4p0iTc3Ea39+5tZVrWflTAVVGDdEL8VAUoKuuBvl5kQDzr13mZW60Iinz7bD15vCI7zB8T8pBj9a8jbU58QhVfQYX0ponriTpncqZ/bZt8B10BgNn31zF+b09crcSxHTrGC+kIPHlmxZxhEoaxqtluQ6fpN+sBsEREzg6+7J7DIJGYNsMVAFQY+WYv8n3onWxbfNnyDWpoQZSQAK+xKYVqKlh+g23RJckEnD5GKEkvAn4diSZrXRfwvAyVqav1KsvqcqR1K1UTpuLYKUNh6TbWacIchjZJZbjZztbYlDQe1hfsz3QOA6JQjXyX2KARng1uy7xBhudGGXLu/uZJYvr/QmP6NGZhuwjYlCQVyk3QiGaMxHj3+/Xd8kF4AzTiRn+JG9DA8bD8Nl985EWRtQoU99pmuzTiRbZM74HnVAwsSC92QCmHEfZLcYgJ0ZpLqb+QBJBeDB4chT8o0ddezL+PmHdjLutx3f4fbPe695wpPeTFOUlxMyP2f2Iveg0jAMZnL1mQRwWMcGnvUaJPcf3X9JARreHHujTkwW1sO5V8KN+GV9iaahW6Q9/P8yHJqD7L/gDl/21uOajbOui+CJ7axA5Sci0jLeObkWet3KXYYprdUIiClxRbdPG07g/Cst4cUN9M7kKx0AYzI+bEcCdyxCZXs+2oQJIqTPQ2R9oOAGdxBrfFZ2r1+eqw+rB9c71Y8u7gMTvWEZc7J7W26inmT+/2XRDwrHemWotKy3CmE1EOzLV3qOD7UzTt7Qt2lxCH0nuWiYvDOl7OvN44HkI4xEHHEWS+WmjbBqhO4C6PEaaayMybER6AAwGujX1GMN4TSGEPDqNOcq6pejWphnwZvBfxX65i+KjUrbJ8xt/wN3b6vqvNnjPXkfipVGxx7CtOUlOsNkXc0tsDXw+tLoYG4r/1QbShj3w3iWebjM1r1ajxzJnpk27cbgcn9z03BifjIfb0hFx/RpWzbEjsd8oQfOPDKI8hvN6lnmGsKjQIIcJXc88X8T62FJCe10IgaZzOM4a3F8G0o5z8DuSb62bpDdjWjEJ6eUbhX1aZp6NWpjKgTjSXn7oXGbxNOtm/te3HMVvVP8WfkZ9PjBafLf+9isTR4Yssz04GRN8IhbSfEAXSm15On3UU8OIxL55CkSOYrkRm7BMffvMLKI7wH4pOwgn2l8tXYPWbhMxmXKB/Xq/Sqm/rnib7Vo8g+DE1zvyzDQZTcVLl0W3l31RPNA9wMgnZ2sgZh1721lKyA0KxYvcG0vJDUoDL1XupNJeXsgUWYa+Fp7wpgF0FGHfBnc+AvzB9hd2lP53CvxkdzdSRmiTefW/2siTjfWS873gvOdAHyB+LcOHi/g77h/28DKjD+WY2Zqeh5TCjgjHUU7RovuSTscE/O1GJC7tnM+Iop7ya3OELOmZgpV/MX3NAoID4cqisWN+GFAtNQ8iR/o6YSrRYgdLpe1ODQ2aimu2pXPr+bjbkTlm+MF74WUKbYG6bz/DqjfJh/qVWweKdMsjTqZ8xHu1Ta16CG9hmOLBvzSp7xp/UhkfdaFVoYBndfwVux3F75Lkf6QDrLzFBhZd5xpauPSNJWaC7ua6af7PVd7nq7g5gcZT9V2uNYvm4KfYoIYq/t0zXUalU2Veqpdiua5kGOKUn1QwlM8DTlb5kY1Z7KjrtqHXrx5NJo/vonr0w6UgnmdEbIPH4GxWvD/ztT0Ga5fAvIGfxA0doBe04onwTgjnxDDsAVo/bxvDGVzCywT0CxF4T7C+QvsjBHU8+HmQWX7uS4FIvM27w/ZLK05aeUEllRfWUs9k42xzGrmjPquBCSCH4HPEC3dvqKt34caZ96GNW4iPoYRR3qPkD4CANPLXd2r2HzYanFuwOPtmgMwOMbriF/XJtXdagHq7qk9pwLvcFSRZ1/Ozh/pkl/vJhvX42x4MkeC2NWv1cLOIM1DRUWa1Oka9Haep47LwgQwa/VF9xqAneHRtTb+WvhNYj28KZ+KNgGnwtCaB6nXMlH3T4UXWLUNVdjTOG4X0MHO0ZeSkYNS19JDeWxObWFeiK9GxTvmCo7HqfTgjq6f9g18dA+gIMtqW7mvdEq7lfThu/mlVZEvRlPbSLqWzNAUkB6I2Wt0vFYst5eRir45rFeOp4auk4cgyiYuWXTuzR1vTPjBsa3eOFVk1jdCog7/FBCTtLPT6FTyZf/EJnv8gUrjQ8vaEZdF4AHG4BIQbfuptfCev1qQVQPnyAQYuz+V1LbjNhqsM55TQcA40IMw2WC4uwtYK23dLjrvzCa+4ciO/14k8v+cmZHXdAXW4lXs3Yozl6KXxARaSLdnmCI5FV/2zwqV/iebXVqwgW7frWDa63cd9DT0lJ/jeVuRyxppXU9iIcVo4WGLDNKZpwSktJmgCjmIZgC0jAUIxgSCWERTVEKcrpmkQU/fMMQ07NCgeyu7hMObczPDmrfo0khArJDBjTT/zl7ie3tfqbTfnwoIj/JqgIlb8eqm0yDomiF0GGR5cPih465tJjc0q5ASjvwP4GoGJjaRML7xP7koGbpYrCgTBw7LRpQgZxaAQgvfXVhf7IBVLeGwzBjrfVdR2kfSUvVe9s+dZewlt7ngQUM8IRdJa4zz4GZDyHBW9y6VEOR7iXMWpGeIDHXR9YD5OJsxzXH6Hdkj5rcVU34iTx5BjFHMVUYxaf+mQzwZwKXntFIvgp/KcQAKMMd32cn8yZIXaR9G1qQpdBuCt5munUXKO9FZPCU9P5102e96T3Apsxxst868LZ9wlvF3361o1EsJ2O8SVKvxKR50j0054T+uXKjeKwK2qTK99D85reuANe+GwFk484a8vvhqO8RGCMQtmpBDiUbiSE4/0rmSYg6+BhjfmacYXGO0RBvo8xPB8OEoWMK+tQjK1eCX9rzp4L5oLt28Y+P78qCNaBaGNPf4+moj88w4SvoEAwE9ZZgnlpO5dh3NGxGexdudkNd8xGKhcW/hNALmN4W81aQiavyQ3ZeGmfjOKxRAp7fmfHCJ+5wrm0SYbcZ80yRGLrsQQe4qcURv8ZhBFmjT/DgJgC/VCIc41HNMesRKVO+5CEmbFX+z/UcJ7QFrQ4Lq/spCW88XVISRwGhWBVNP/YhdQA/C78LwTkORGDeKtu5NWHD5X8WbQS+LBMIrA9tH3wjUZGc4kCLK8+3t2N8C/WwQi1jc8XKClfxj9rZ8QMQuhynF/iVLGeQbJWSXxcN9vXSQ441dOpK6hYCFfFd3a6oT9Zedwaql+J+tBHKRrV9OCEYG7k+NNhyZOiZn1L0p/faxSFAbrS01+MYT+PTisS+q7N3fZYD+Yf/hlyjB2UdaOFXyuL1y4KkxTYOXJ48QB/pDAZAOnDC5YdMZJUfU3c1itarMUbyRuWtbSRywrFNwWsJM/B5Z4LF5kPjqg/WXcCIx0119REIEpTXk9uOT25dZ5jhMFhIm7Kg3235hzQVMafSkL9UFPqqwS3h6gQLPQU+R6AdDkl17IVGCWaXb+5W9g5VjpYJu1wxrt6rrfZOMKqbQWgtphUUW+APnwTzmfKf5QQHILg0oOBZTAu5/SD/qarjQitRTTUfmUTVxuSZESBFUBX6/KE8JS7Qe3XPUNOsOL6zLAzXvWLA/GEbnOfXDW0Iw2RWL63/PKlNJ6e+6Qytue729XY95NWuCGuLTm7P9y915LjxtLuujTzOXqAAh/CW9IeM8bBbwhvCee/qD4d2sktdZaitmj2XvOr5BEgiRQlZWV+WVWGg8cxn1f+IOhWKcNjpXqwkcMm8gI1M6AOTAp8a9DhUT2R1aSLTn30fj4OqPLKhuPgR69AJT0BFyWpBV7kF+5X9fQIRHExb1EjLU/mT8O9QSmCeT0wB8A2V9pgkUJhZE5OeH4cWW53UeSm3aUPpFPnYHQxBCYrz55NgzDFlPuxetL4uozvcF8+BbPzUK2T/MIRw1Q+SvDSoXM19R587WrkAf6OTH9zIoI6swOu25Kug4QDey5lyt/6Ky9rk3cJP2Wf9+5+fn8wnv+o/ykwtbZ054C64v/yNJjb9vKA7/llo7E4JO+WYwe6jtbjb4tEik+MEbkPukiCkfx0kPWRPkU3Ef2dWNOdI/Ze8/O+dTyM1ak0nul1a21dn2fSO3iW/rx2M6vnm+Mhvv5MV2ywiRmEVYkYM+rD8r5Gi3wkzOTvpMP/7kRiRtKNaU9Nyb84iLjpciWQp/rJNmgnzkD3eaNu3ASQFDGeCNaeJL2Bs9u4VtVXsn+RRRFtlW8ygfO6pqItqRynhR1WrWHRCk0yAf8hM6YfQ0V/DkeUKtfhvEbtFu379/zG2/8vRsGdB5NSH/Li2VOthBQ8UGGXsyA1XqznYNztx5EJySoOLYmiqAfvSpwEK1+cunS9oMm6NuQ1HbyIl4ZD3fR7Tu/FyLOvcWOSnn5eKiz4GfCHeum12UbKl+jYDXZddjyfQY6SVizHqn5cD5uY8R+pX9xIccqHuz1S/Xm7AsrePjm9QlbfOeNMuPv7QBP7KyGqLiHX8+lBYoZ7lPPp+2dA4sdycH3lLpP3/CKL2Q9wkLNXeTboypaXWOANKJl7lRZhcHu9afDET1fdEd73pA4IWG/cj4/cfRQ7YGWA4LVaaSQeZuzAf+0UBXLj5k/OLYybBXmHnsT2RKK9opkIrpKf21ENIws3tLYWyu+xFmSmgbX1+KVpkGPfwkAVjYwTlbvCHfZ3FqQZrHSMUlkJgZdJF/E0V66cu1zvGyGEpM4UAtM8B4PTEOcI8fNIp1RU/twA83uasSqo462S6nu/yoz93/mj4k/e0cgWY7migckknSB0NHXZ4rlYvz0Uoqi+KR0g6zu/wBxXtAQTVm3gCu3m1qKXAvZqx+EO0TZZzcZ9j8Q9OuLWzYt2fH1RRhcQvj/QNj2ELO+zZbpEnXQj09vt28YAlEwAl3sTZA3/OsO76+Pb/jt6/1epUv5/Sco9HWtzKqi/D4cBPsGFBK4HM1fl4pfH/Y5G/8MAaCig82a5seIPq9vUJV+/Wbwnx2+5a9fpidStSslBxH+DwL5PquoWbOv731dmJd38/3CXEYDeFm1UXH9nwEUqJKoeURx1hj9XC1V312fx/2y9O31hQZ8wETJq5j6tUvZvumnz62Q/PP3m3vQTVWA3y79cF2N5iFLwKzz6siuYTOfR9I/rkI/rlyvy2UZLmLQX5oo64pva5xN35IehKHuwz+SvlvAgl4IaGj6KP00MYY+FaMggE2EKMnivn/9A/42dMXfyQYoQX1DYYqCSZzCcOwHgd9/WPHfcMH1Axz/mQ9Q+Bt5+5v4AMX++/igrdIU/OZf8AF0cQEE/Qkf/Prb/zIrNNWW/eMCwkv0j6r/NkTdUmZ9dw0u+1b1f403vn7dRK/sH01f9P9Y2u888nnif07oay745w8M+fsckuve2fRj/r8hzK8f/G28dnHXN4KAMewSNhDgt9/zGol/A+bFH4UOBn/Dbn8id6BvEPx3yZ2f+e1x0dte+gks6h9Z70NuwAYcfJFvL6/VtIdrB18X9ikCoqNc2ub7x/My9a/M/z5B9LpSTFFaXbT+DQcSWYRngIvyqml+cx1UzcsTcP1iErs6wSNg9J9LrO/vmixfvv9IiNqqARRn+7ZKrnnYEUi0gVT717H9eF7Xd9nfyQ4ISn0jKRgjfuigP2ggBP+G/swPCEJ9I24ohv1QXD+zBgxBfxdj4P9eEPXrZd90FxW77j+lQRrN5YdDoL+iov71av66UX/LVd+lX3tc3DSU36J9Rr6VaT7/kjTr/PX1P/CSgJEY8mf8933df88MQCx+/n5i4dtf4aw/8uvfx1QE9A1HoC9lBhMISfyeqX5s7d9wFIp9w3DqCwN9xBL6J9rtb4M41L/nqJ+0zW8WfuirbvkMCmP+A+P+jE1+aMTfrsHtNxqqaouPZgJZFe2cROAohIuW6Bcg877NW/Fb/v1ZhMF/53LiCPwNpvDrxddqUn9YzZ9BKvInAgH7u+QBCf0/tXrRZU5nYErRMPwyZ9NWJZ/EUPp6a3+9/YXr26jq5n++rn9xx/9emJAQ+Ofv5ATiwpcU+auxAv8eOxDYn2EHHPsTXvi7QCoJ/z/FCz928m/XXu3jqsl+uS79r1v+CzV+o1AS/VVM/16uw9ifSII/keN/nyT4C8jgf4GpmiLUa4wWeB86uMe+JU2/pvl0sdu3Llu+BMufGiYfr1HTrEfVPy6j5Bepn6rz+lnU/MJcBPnFmS4w8MUWf7tFixHfCJj8VZv/AVWS0DcU+s8/+GeDA/kTL8dl3eJ/F+eQ/55zfo/00n7/M1j5k7j4p+brT2hxmvp9vn279nk1ZCkN3oK7A5pdcAq8BMRFwVO6fkk+UgD770GCf5AkeZ6kN/LPraJfmf5/1k65EcQ39DKif0DKP2AQ+E9cJNA3kvzxbWCn/E8Kor8AKP8XCKIvEfNtr15Vm6VV9K0HRdEF8H4A78EBT9+2/adKxFKunzqZn3/jj9KLkjL7Raq27BfgJPmou5sA3yBoOP7xZ5/+3VKJor7hJIaSv+Md/GfWwYAx8j/ILtTtJ3aRu7yf2ujDA9fevUjVghZt//ehzMcoYa//XIv23wxgfmXjfyV2/pps+04i+J9vm79bYGE48Q0i/1P6oL9jOgz5duG1/1SAP2tAFP4TBYh8+3Ei8H/Cg386w78Anf5vGFHpxWmgqO7HggIXf+G+X/nlkpFLMWW2+fhg62z6zo7/esH/q/z5t/ML9Q0mKIT6YVz9wWVycRNO/CvE9GfHQoDLkL+JX4j/dfyivv9/wirAg3+7pMqvzrI/YCHiZyvsT/0x3/D/Bofan8/pL5wVAZIPP9MI/ifUuEHJlzM3in/cAfrXyh5Dv/04u/xOGoT6WdfjP1wWv3Ne3/42Vf8zZaxs7tcpyYACi7qLt9vPyRNkNNECQMBPlPs/OOq4/TPtmlKfA6q/dqTx73Xwf303YP9k/X9e5/+RRfzTQf4FJ9P/k+geCMcL3H+9vQkAbf+LSvXOJ3ajZGkVxEW8NljrwBfeHqN6fPAf34teCOQZry6iOS6k1vL2sKH5bnfEV4TFjGp12CqsXfWRqNchAuw2IQEdThgjgJd4IY15fbCneX7aRjWKpTsuLJrXvSSQCjrgegnusMUtmonHotqvT3OlABqUd/JpAuNA1AZ9okidN7kptFETJZpJPWHMfaSbNGh/oU/jOOJr3IlHEtzhPIzDVQs8z6dAqottjhnhAJ7IU8xo9tc6QiA0VRMzPaK2x/u1GkxWrlU06xTaZqsyZDfk2DQ8C0CcW6o/HRBGl0LNug6bIWSBd+N6fae9VYlLvfB4hlX58AFCTGzNbXjTs9BOh5MqiCq9LH3ftR8+5kXPQOKYCPGn53pLnQmEscJ0MA8C3S+xHvu5SYA6+1TFv6QPxdKUvncBw1hpkqi8ILzNuKcC0BSZeXfK9XPJMAMkzmSizvTN+kQKvjr0jtvdMtDqFjsdhmsbqRJ0YkBmVAyAIK4UGlWXZS8/q4c1nxwQ4j/FUIoMbtqzmd5+GgFqIBwLm48MPRQRl/Ss+lTTN8Bl73Vu2xOntReTEjhBQLBBiqcUOfBMH1STSo9ZBi2NmCwN6ZutIXXp9HeQIwJbCx30WZ5fQyNWBH1gWFtHF88HCXun10kmaVfX7zCDNxOKpiDAuzJNmYvPIK6eb2eRNh8NzVyhy8E+52kjsykDBi2BJp9au7OTKriIgFrCTIA+HO1I7Cw6C1CIkbk1nCUhC8iuQ4vHcLLvrJ32jAqcfNdX4vY6Fj5cE9cpTYPZG/mgoJxq1ORTfa0Dkdlaer+Hn07PMSF91WNiJkKm+taI1q8tMwgB5Cxzhq0nmlJrdIlTgnMA5Qf/ZN71Ta63PFJ0HOvWLvGV9m7EUFi7AjIeiEBtWZYHi4WgbGGR9Y7g3EZhTiOBygiAie1qKyTBPDcvGc4lrlEh6ENcxJFV2xPSAGVIbQKk8mx4Oe/Qlj/aSc+scz/TYEDuElV1oWhnuELxdkaFkT0oZH3cBuHT2MybRZCuIeqoNrPZLdKD7GZEiUwyhBAmzIsPn/mWuPMzNfT5qZElabU9yPsqAm5ic4S/ZZ/o+Hym08l1TQTkkvSGTzbInmV4I1VuqaaVtaY4wZZwNdlNVg5iUT0XuxMtaCC71Wbg9lVcC/1k9/eyFT3hgJvepI3eYT4A9cwqJNm22N+IXsrcdYmioFNi1JyjWFB0ZCl4S0vTpDHGN2WvK7XQKRYIq9/DKlx2SxDsRolXeFho6SfSF2tybnMrYSszydip96N8ji537VhOHnkhYVK+5yM6yrdDOFedyisKDKK8neL9aQew5lOPJ9MFZugiJZ+03jbq3k21FcxLbizYML6KkhL9DIO1xTZLlyaXtvNEojoQjvqMklA2QICuhJ18Cspq1IBb91VTzBVjW0h5RvrynpEnLnebPsIoi0SHKTkpL1yTvuxhwIakQhzPNfTlUs2xwMNRZwXSkbBuvm7Bq+Ee9uOAI6/bGBZ+O1KWGpn6sEFMDJDYI53DW4O/QDsibq3QVEwsOQVeZ330FvyOe3eSgR6dac+Knj46gmNJ7LHGEZOFDSXAvcoulWrSn1LuNNorNNzM5qeWoqq9DwNjrrtRM4lcIGjMlpXlkLRM0xgEgQqj4aRuEahBVN+9GfY58eVVaubJ7Ca/057w+vMNS44IAWLyfq4xB7reFPfGzUpaCEF3STXJwYiIa0xueGOUtppxGyXpQ477hxF18p08b6iLkQyg+JhFg0dxpOMVsSufA2YkygmYhUg5Nod7N9q2ghlS/wlkIzwoXPXUMxWTbyw3emWXhTJdiLb1KQdfULvW3fUtFrZHiirwDRgu/iCaXTyGajG+JlAWN9krfgppoEkc772c/DrTRF1BPGYauE9blhdvWjkcQRCmgTlGREdboMUwiSyvGt9VyA14wikCo7BW9aF79Y0GYfIHu0w+Med+CgwlDsb5mtof00MmU9RQaepNwkUkCnmsJx1/yUMj3B6PG9rmPgXiJDnhXZQuUgcsiT90fcZcz5BdajUZlZmpjBes7EUd+kt5BMEhd/ucxCOSMZOFo8+bsDQmI23v9pJfRmI2morlCXH4g662I1y32YHwrE+4o+d1KcRJSdhNimPoGBSgUcA3Nw9bNpALqdwNAph0611rkh3Nyd56DfjJ93lLDXgKxDqFEu8N5sfu5fXHkkiOMVvRNhBLRpLZcp4bIU9j8OaqnLUpgqO0J4xiSnveYuMBos+tLUgxXTu0bBKH5LWmY67iMKbzF4WVresPRpHTLofIxwgbAql3bgXF0/LpESCSFBcVDzKEpdiLJkryGZlRehsBQf7TJiexSnXP81ZHIeZJ7wYLCvzRPrU5ySVGoJRJB3lVEOnAu4gnSOSlhglru6D2uoC5pMcfj3lgjchKDTKki/N0xlpXqOAEhR1i1++phmT3vX631g0gl6COVDGTt7YsxZ4DLtvHeOtxIh/wA85eL6lttYO+4faOVdca3YI7On6KuO4B9EyebxvIZIbtsLiZRSXl0QETBhKxWCi45iPVvY7cbvxaqzcOPdlQh1RnOyrvGI9ImDC0Wg5f4Z1Qk7kbm1a325tBXafBD07A47ekmil5N4SRgfTcNkkf5W4HKNCFFLHhxAvE7xROd+lIincfF+GH5HRkIYyux+tYkjstc9LpJ1eBJyX4fODCNnWhgL/9m1oXeUliDgjR04pRxfNshSxk6qss1u0LznXUWrTMQS65pFKj5sveXdoT3HSKd0P2J8cMwXkos9o17dPcZ4VryNIfjTwMCrCRrUxuobtss0KSjpng0H79qTAbpe/7osQy9DCn2hD1CIeZZYP76AsGryKtYWUo0MH0nMng7OK4W1Os/vR2X1LYSrPLJHukvVV5WAXEa28qZY2s7uhUufFpqeqI3i194LjQrWT1ipucmR8mTnFxgbeKkp9zOQQyLJI8kH8PUHRmrQnnU+ILqNuBfzCi59zfQjFdi8fJNpAGdx6ErZNpyu9cN7rZeT5nz5sZXSA6dSPDkCmQa9u6tQ7yCD1MQjJUaw1DeVExYhpS87KP3HqcmKmojHo+92cqFgw3TCsBXbZet1Jsw57RsbPc5oWHQp+5wCgKW5R7o3MBkOt2FbVk6b5K7nqSyvCVlKjvyzQxumLj5SCzyY7MWPJNP7PFSpdwKM36vXyysJvNmhNThwoNefOud2i0VmF4rCFR11PlLheZujvCw9t6sHYSMg7By1NkmTtEu2jyVJqDXC1OvXQtCmUtn0Yp+uCZEM83YWspWl8W+CnNoAiEcIu2sKJopC5sRPApTY2ftyRUHxOxoeWNVl9zX+ISDXLlbB3iEt11VTc6+fuFEGyDCBojrsiUCSTZJ0+TNLrWCS8E5+p0340RmUsmfGQBmZ/Law/lwgFIcZX1W84VksgWKWf0zvMUgSzMnwlKVTulSagkySB2ieHKT33IbCFVD/rkfIg71Z1vqniXQk/xZLVdSurYd9MNgIlx2QukSJmb12XAxrhAIEBMfn9MXJ4/yKVDBmw8ORm5Oe8cscDm3JW378LOk38Rr8umWTzLuXAhyMGKdVCMpsjMx81lO35ITFMMGhI3daOZDsVdD+XGswivKCSxO15d1iiGMrPtAbDqiV4N4CQgcn8hIF48dG96bbaghIPoONjRRCTGFUpScbMFcloU/ZB9lEf4kng5fPVY2OgBNHZPIv0TKpOR4gzM3qg+Eak6FmdN2bScJ8xOuGtnx30q53gAaycX7xMs1jTRhVRSjxy8ZKO4KrihF5w/S88pHUs5nt5IyjfRNv21Brspmy85KZhAwtip9cR6BRiO+/Oh9mqfxN2ouWzNtblbALgGBAAT7Wsu6kTPyi+YeMNP5XkfPHWmkPkQUXISU83Ko7LVbwBkycbGKVO9tNY4jDKAXkBKNFTnUkDLsSB4nYfq8VYzNzYQPKkKBrLfYhMygoOyF9R3LjuxkHTBRo0jH9UlXv3t+TDaoIEOrJNnqh23T05krAcVOrTNe+HgPkGLXTFngm2tGaQEC510qFLuDm7zqAzFLYC8sEn8SM/nKmh7qLGLZo30vYnF11t4hEDVkbXtuGiHL/HFBQCgPlNJWJ6461tj/tD0ZiLcXXgPoSPcqZzDzeeadY2czrO63d4hRQMKuLddlFZuIltPMHPxQSOHXHLbTefnlA4J5Xx6TdtCWwALKpwPGZDAdGmLJ46McH+Dx0htQw++DfzwFtyZfK2OejaXjdK47s3YHrdOTUhqmLPOYzsJ5JOtpX8r7xt+OJVaByMyG7H/MJJerojaM+pPURgZtcQg9O1VVi+LrPHi49ACUDYxG7jkUTLwRKKNQo5IZAbK8kKjxLqgsmWkScnj6q227vdrA6Br3Eq6BsxLsWlGBbt7cEi2DyCj426JS64n8V5O7/ylunaGbJcIq6undt+2cEJd6InP5rFO6uwleeCbPoygT6dHX/ULNvCHJNpotW381ASrJu7RIDSf+i43ZHMjWDAe/tu9dNA682KBkIZc8lWpvD3cGOGqe2ltl1RhtnUQJdV59JooIG1sT5T0Qbw24TUOfb2Tmj5Qb1XtDWRbZDBuNmbwu9rdGX2y+vp0vJEIIti/Ecq2D+bbxHYoGMJH6QNjuDg7OpizveyjjSF1hGncl6CwgVm5AUrHqYqRD9q6Bq0KwTi707bVSOntxurHTElGFozK0WDhWh4vxKtPfO8CqofuxxAYXYwbCY1YA3bcIfFVsSS1L4t2WazSyRxEMr7Do1qalScu46S9iay8vxXTmu2nfyny8qku4cgRjV8Fl7qiyWBPoJa2VTvtMSw2M7HGoaFsnwIr81rG1ujawzNpClCoj3JYev4rM3eJy2Fxh9ycl62O2Rld4y/UWB2pX9ByVwbRjeODjJB0uUJaGq8XfBKlh39X2u58FCGKqfVwmfvvwU2Eu5jEgPWc55uhRlQsX/xo02hOLT5ssVRqMucg2cJaMRjfPSESCyamSVaoeM+vqVf5euqR/E48fahkWSClqHxzbFJDhfvdt09jorr3NfbdPyUNC8VGtxxBa5VtxtDieN0pojzfA2kLF59Cot9y871y1bt/lxIO0oGr53g4XjcqRKDxHe14K+hL070znbO0kEkets70x3uWbjhlyMWnspK/0F40+y+sdmmd8LTR20rVhF+Wxrz3UT2hk39aaKzDILVPvJZcqy4liG89qpAdV70OUkmaeYuYJbHol28QeOlcFlIUiD0bFF2Jtt7YvLgK0m1nTCqHuynFM0y5OoMcWLEB4mBiesiUh8Ub+XmwUjAyluwDf1Zf+NF78nTZRGaJdxKXwynTrczKmZCWkWbTol0jqk2qLZtWZepKrkLYChwKQ1/c7dS7J2+idVxNIfo667e43itz9ViU02Y7iEjIt4/z5qjp/FiPV5XPOjqbJRzwtvKOVkF96mWQCS/71px5ebzLhxKHTbpnmWIivYYww+a5rbJ/8vQvmg39eiHCS7Y4LVy0x6dBmxLrEZ0/LXmo7GwwD/ElYPrROGycU3m73PpsorMsmEUAnyGgfoVewtOVk6O83HPDyk+Xq3k5p4JpNjegl2cQ5BEMvozIUhdJJu0+9RSNFetoVGulQ3RhaZkdWJOcLITVLsBBmk/KCYf3SEN30IWhfwSSS0VJZhKj2oSkrC73I+pRCrfMuismtIYShcbje0k/xY0ZMSk+gOfSFC552tNQy9qFzb4eRYIFckZHOtsqKqe5Vmdhg6m47fOy2IsDffGOmSkg4VLjuazSCkg64AfGS/LUc+UxWWx8l3gzfr/K6kYm2cF7rXSx5QswJudb4RjywrmTe3JdfEh079sOOr3mwiDtoJplfbl20nN7lWXAiud7tTlRMWvLaYA7A2KO2WqKQOJlApaYtwO9cGNVIvTC9ojd3lTmaY/Q03nxC3Z/uUSTyfYt2EnJuzuHO07iUaL4aiaaQvNvhpgISF2PlTMUMW7hgbKf4l1gsnC6c60p4tplsTzVk5w5LMGMTrBzs3jXaTnzN82WEyEj7Ah5KJlUkKMWPC4IlFnTYbiwur3RuTCdiV9hsVvdFVXPAwXouBRpMTzfjFduhkfO14yEeqPaPrIe7bj4piLoxNuECGve+saYtY65a3yYIsqmeMXYVM93UrdncFmVXe7TcThp3ihWDQ3T4/E+Nfud2ixvWy2MvLVVswmJgbN62J73CeKdXOhkVhtHb1Qn2g4KqK/VWE979tJksRU75gcdnsbJZzK+8hZNzDdNQzqhbY/Q0C4NlNwGsxafx5u0SwI2NNqFctW+e5gVcsib2EAzRsFQz+31LFb7kkfRmFeHwveVf/jU86boqsOXPkPi3D1YHUoIDJA+bkonqo2bAyq5X3yEInE7nLiE8OBDlq8x7jLKCmd52M1DavvamDzq4JuovIxUyGrOaBpeT49sHm3goqQ6vaK9AnBFp/G367z5YMKeZoua7/Tle70tHdSj9ySuURjaajO+qB+kG1HH+DSfPMJxr62EW7Oxl4fopilHeAk8DPrnqBvVEJFClVVMneNaCubRojdrl3tNQYtTnC/l9Wi2/Mngu3eXdUZKM0cGvr9AUTbSN/HGDIkF4L51NhB9S0lelHNjlFBONYkm96ogekfyGUAC8iAU7hz4KjCBCz1PZve+MAkEtycHHDECoTwP7wb5m99IZ28cZZ0QCpQ3UdsLDsVTZqBPCZFYDCQuJW0gOHZyJQyT620Hij9sD8h6dum1aidhjot3R8l+z9kjwIKDM48s0nwP1N3JTouKFjahqwkB9a78Oi+Vu+b2D+h6LrureWOhNTujRw7WL+cRT+dMsnVZbEdaaEpkW+KUOClwW7xwBC1CQ7BAggVAOMB1PaEe9uLM21fK8k+9nv/7TzR/5FsQ326/ifWAfh9m/yfZoBfxvlHIzyegOPwN+W+ImP3zaf10Bmo7tMbRjwvx/NvT0J+PKH86xfwv5mf91446A/AfTv5qys1VsjF8jjrpQLMtSKanGb0kLrjQ7BccOq8XD/7iCvZQGVoZE9BslmHSwRNKyOdhVW+1Lbax8quBN4bGwffMc+5A9a5cEhFuUpEvMhGe407FMw6qQt/awtbFwfvY96DQJitZKvDrO3sqzpTclJdlyXAxojQy564ai+5yjd6vQReGpLye9WBbfPjrPZPWag1b6VPJ2vWK3FIkRR5dcj5a6v18k4fuvLDHSb8fp/x+BNfvK/jMfAwKg2K5fl//uPdv7s8/A61O2uYaT7PFFfN+iiEe+sqWBiYlV/Kv3//x76/zu+bguBAltyWUSjT+eFNr8v6VPnUMdoco7I+TX1WWWuJr7mz1+3vJUrnEInbBLA1yW+93c7uesX6n3xqCguxIWSYseTxqektgC0tEd7t+c8aI9w5v3oX3w1ou+t/fn2XQ2D/W5BzAev3Fsf6rcVrDsw3/5X0uOrbP6mucv6X3F80vPuFhTX9ZzUXz29O3xKSlFvlrLfYwAIbwZ24XbyhNcqPgpNWaP97n615fNNPbZk0Qq4yv79mgbRsDVubRNsOT698WL0Oqa55afT2WCw/1MpdNx4Q0N9x1Nzy10zzUukAtZ9+uuwAq4pGPnakoXFT3FIv7V0++VgfxlufFYb99svX7J59//ckXHevUh5u4s356ssFS1xoymuMAnlWGtPVeVqdssfNPafwXRqc7/3503zkdt/7V6H596l9ZDZ3760+1/8VqfJ7KDVzSemUqUm9PpLaYw5gvfmTMSKSgGNH6GKELE1ILtaYPzaZ7xxfqa5U/n92/P0t/ae+nL1zXFDe+UbPx+71Efvi6HoyLbwEFyo+8qNHjWhco8p+t/qLeke+t11iv99T6m9+TD+RPZM0POcdSH5njvizx+s313S/5+JGXoDkcOOSkQHPyp9EoFi+4mTYtudkmkuCDs/nXngG7Nb8hcnABPoYqmiWR3mX8Wtu1HV6mIL3vKyv2Fd+0SisgWNgwlA5wTvFpqe0Jx16hPfF4UdlTUF0Bpk0+KWc6vpu1gzjGp0UkScGEswVDu45NsxKDnQEv7YUokBd5qGkeLCPoxwWOrDFky+9Zve8QAhrOnEkMtT2GpfrhUVmSSk8E1WMcvwUPjnPypGW1OjDrMD5X7ElK/XtFgwkmwRks7oF6toIaIFOMV6CTqGeCo4TbKn96ZMRGW5bLumqS7jprlRSBH0JGaqGklnZUkz44gtnAGRAxSSUxggQcIa1RV6W2o8+I8xJrX8Wi9uhujgexgCmIonjDUTXdCmeyVTqPmRx8h6MN2KY3Lb1BxNSm2P2yfLump5bhfQuR7rifp0OdW927Rvqcq2oGUFSjkeKLCnEiUGFXM7LpJDkFHx8rcqijWauAD0njck/lB8ZwC1D45o3sKV7hKoe48Qz60/EIzvf5e3O0gnghAQcoF0V00df7CW9yBar5UU/ihMui0G7gnc8gVYzndV21uJgR2niO3fysTHBAgZwh3BH50s83Gn1J+WcdItnou0RCqSEgiJda5Kmngxuh1Ot8vuvduUEaf1Y+iHBZ9wBjUeNcZ3sC1EKTaC62GRzqPHHwo8YobmdTPc5QJGbBMlkAgzWKhw9tNiwMycETE1XXhAIvn69IEo5DgPKEQ69tGILIEx2wErctS83jJ1KQHuQSaBjHU3FklEjQeRI7VI1AtkVs0t659qHsHpS+sMggiIZEN1OoT/tRKZqxrp81hUSdxtYSjQnupefGko77TrnQrpByhCCIfe9v0zUmVdNuz4blghMskpuCU8zJPgj3Nj97PAPn3KD3N3vrxHFon6LS76aAq4H0oWJyN56QM4MDSq0sYEjg37ez6kePVQrGsL2puoYWO6c2dsDgQu8aqmnUsfdwUBgNdGPe6h7MyPUa5oiDIGiVy9ekb4MqE9cedQspF5/iRfEwjB3rYM+d2Q80IzZUekVtbGUfkeC+llDSuACYwdgBDeDsTAJuj8FRu685Yv1zdwzawY8E4lWL6m+F+4rOT7+5i+TEy7tpmYiFMvTG7ympglH23Mw+sjbqs4NnCBInppRSVX+lJcLLK2yemfiS/bIHd52peOfXCj97mpDbut2TawYQSq/ABTtF3HbSBHCwS89oxPNxvL18tLzP1wrXpTWG9ISyXmEg1KlcVq4PrCMksuuIc26HoRvTeuBDuFVfeyGL1IftPag9b2IpTBnVaBFW0Dp8/7T5rIcTWUjzx87J3kBiMh/DDviYeLSoNySWcBd+zCpuOi6q2uQdxgik9UsrxeTwa1TNSGN4iaX3etJNm+xPWhJRsnvZ76aXzHvMP2+gw9tI4IXXOTNU3lEpoVf/w33+bm26UVuNA2UT3Zg1Y9a711dI/gbuFmuJBeEyZJU1KSr3wO/SrsXu0moGbMbUxk4+6l+G31ABmU2s6A64JkRi2l47Q0EjHCEGS+BMzZvY9219MnoXspiaOXefvD/Uea5pBNnoOIBaFNyieMs0wRyH5szv96MgQCSjxNTeWjw5TgZ11PcieqIgYEII1oF2EALmEN/Uj0sPmdQrJrLIvtTm1BjCFD/RcZriBgHslznPoR1UgcdOGt7qaGVVVoJJOCBSFDLO4NNtd9zmsJ8qXSFyVcM4cGTn8U6xL5ksd/Ua02ArahoSc7xXuPT9La4VNd14yba20oHrT382PEHBWXRLdQSSPkrGSRx4nu8VjdeSZplPsLj1WpjN4KL+kL1hecwozyYLzoyeeVnvFLKFeQa+lg5mPHrTBk54Eh9FHyt0x7MZvN1eOa5vGL53s5RUBNh58xEzl4Kjng7OQJ/eN93y3IvFM9WYpoflKybuoeIg4HOxKGp2saXyIT3hclLGCrO8r/lHCUAP8tYEz2f4flyGZ78oWqnVNIFIdMhp0/jVDg0FeqmxrTRTDnMnhLXGTJdieMcB6MAsTWfLd4LiyPgep5D90jEN3nf05teTy+xGRlY8098RY689erZWWZZZY80b10RNDN4O3Hqw6VgR/BTQ2iP0grId+JvU8HhanfuKU7dzWXwgQ8TDF3Zw0tN9GrUKXICdr/t2UDS3vfpCQfKWaPxuXId7kX4C65zcpMf7CLZFmviX3HKf06VmGtzUJU4eXjvCcWw5DpYE97kQBO4LwWqauuXvVRAvVBSw2bBvQbjQrMGYb6IQB9Joelnsun4ciJoFRScFvSaSGf20wJ4FloE4Ke6dHIT2mf7jzp5hW0Fdd7q3+jHfKouKwy5LgiwDugsK3rQHm4yVaRty29MLVHrQtR09rafUWKj2kAdcUCMFsh/e/OlAaFP8XhrB8nr3w+1kPBphG5woRX7E5bVcMOe2hmkrmB0kCiHNX0D0vKhf9gGEb5RDvLh3ceh9nL6lJ+5cwGdoVPaBodUDvmt0N2qd+zI2PFsn4kWcAEeJIFhhv29yy3NrWMReWGgR1ShPMzXOEv3S5kHNsncFdLTb+hijWLt9ZjEyZ6zKScBddL8NzLhUgsw190LAl1ee9+SY3szsEjKx0kEdcCk6SbLQIvESCc7ZJZdFlWZtHYfuXqcTUBrzOnLTv3ZsImX5h37Wnd6Qrr+HZEkbPazs5oMKGc3jZBVV0nFWPFnEmbvxKU+ZQWdcUKJc6jM/wtwLoZlLSBsx43tiUto++zY4dkefYnY87TTF1CdS1xQ30Nceuy+joOWrYz6dLJB0+J5gW6yI2fKrPlyYogtA2b57YtF3pNM6RLAqbWZWGs4fAw2N1cyhhESc9/BYgxpK9Ms8QdEAU6Zta9EyWCWEzOUWwMtbZXxqhp83A5dJhN+Ct+1+OpwXocs/7wAIGdviPGb5OMnRDYaTDm4iJ9k1lvFIDYBcThzdSnTGR0fpziiQZRnfAoh/jX7iKMfI3Pebzw911iu0FymjnRb3XQd7zZSbedm7QJEv2nnngl7WS8x+AidpqYNfqzF7i1ySBWMiBnrtphFNF3ntAzsSWzfJiQVr+s1h4GupG8lohddmIVJNvDsESeLva+dcm7/On9KKYdAehxxn9z0t0TOlXEYCX5T7GAINqj/yDq6o8B5AK7UgkjI6N1XwONVp6Lu6UOxRjY9TRRuAzk+c0boaKglu2mmZvV1D7hm2TDBgD2ruSgcvUCHVjyIIkb60N4hlpu+ve/OmZCMe1EaiYWnV2UTFg3RjqCRjcganXd55JbfZueAlRIdmwL18Qd8jWpdpuzm3Y3HLW+vutozpC9WUQjmed02p8BUAtg/Kwpt68h6mq4OQoil8Dg0vLr0OnPEg/J/LXZP7NP5L3Ue2GFrprwD8W8KgsCY6FOxAEq4riZBav3Ake0p7ggWgVAGTFHP4wUhicnGFFDD8W4Cf6sqEo3oXexq2beDVFgi3hnidjFuO1RLiMucMxRRQv1fC1iL0i2c7aaamNPvIyvhxV3ptv4RH+ZDr8uUztH04F8nCBI0uuwryCi2nHeLEbiN0jHcuxR604MxG6pG0uCdvJYK2HthIlyWSNJF1PMUggBYBdRz8stHrnNqVJ18e9Vmn6Yqulst5ryiggVDNobZpVf92mRDdPPa72NqgaPd8uyveaw/Uy8Q6gJZpOQteTOxaO6DXCw0cSI6uAyE5IKlx0yfsxGHhhM35rivY63yUt+otvInKu5TBFAZNEy1YmhTPeiRGH8b19xdfJO/BT+Y4fZxvH/a3JZUxcbQ7N7rpWkkVt0l1rQwqlQiN84AcSMZuFR2aSbGzvxRwWFTWoBG0YWRVnUG4BlP8OBZjIOzH+dgAOq7RuNvpLxsNN2mw9kfIzgM1G7HHBN1pekrin6G8I4ZOLdH166cUNCnZwQkx0Q9beci+A71RgXy21lHJdCjTOIDak5VudJs1ZMDbIKhC+pxW7wgK9W/xXsKegsDh2BNI0l2KCCBETgf2jgQh2KvrymExZWoVb2ZMGsMeaAOAOgee4ueiOY8iur085R6qonSRc6lgtBUIBETzmITxZh8FkhqciuVMTWdBHGwQmT49yyaoiKGk3RUpfDNpOtgWj+8c7gupX1rcYNNQG+dG6vfGj7Lw0JeDmodjc8wj8/3qrsMpsiGVmqegzepdS56lO+ty5KRJk/YzxB71k25AroBQzG46g1lLJchPZ5ZRHUjYpEgK6RjM9Do+6OZmWSrDn3yDI48Mf2vQZw9dKxEixCmKXcwgUEDPgqgomjpysIb0lffe6mrkH1D4YoK9rWDlEbpL4Nn0iR4WdYnJC4q6q61Y7Pbq5LbDO3NuQ+kQLwOGmne/oIA5UeouH9zNGLlpSR3np/gK9ozQcedI9uG5AN5GWms9fV99Q+OiepPuDEkXJgVzC5SS4pBM/UQIyRhcPWVJnXKE95QLdG1AnqAD9BYbfcVtc9qKuyzc6MxlXT1KdxpzT1WhktZGnwfr70Ey5+Sj7teQY7za7mVzkPFtFV3cFbDypVgPy3SgbqwbpCpfH17VIxrE07k5Z0AqX+9heFfg4IIl9bWz76ey5OvTf6SDTG9afCFd5CbJ+eldTKPfgzlPkNVM96oNnhjLvXiF0xW1m4iycSlkDUevYJgH/FRUcW4gjwtuEE3NXvOUXVS69xzOtPjbSU/qUl3j++PhoGsRZvNlpLMjn6tX4S9F71xKIQiQ4BkYd1fO+tmnI5SPhuzFxApduFrhUCe6UOUAWZVIp7RGJDLQ3utjOJfkMAfnAeQ0w9WQPBGsXQllN5ClJYADx7q/BudlG0zE2R1+nncmNeVNq+EckldlPbYU8PTJ0xTbOKs+GcTbvkxiN4sGVoZf73hHHqNCB7pDueFwXKbXBdawZ+kPw0iXlzQv7sSSczoaQc4LhFRZSCJiflMNK8rnl7UoFVQm2VVaop6xJBGx3V8gts0e1x1g38C2n5HDPzc7ezax6JnUKOBR3le9d+qzrwL9+woHXA0bzbqH1ivb+HfhUsIdREBXL66DTtrqgfpFKj/HsU4+oydvHI2oOKxwRFFPVl6o7Jwi8JNoKytAfgw5w9LJBZyD4KGRK18Wl/K+LWoGItGCjcJ7f/CSeC6Zqtto/GG0qnZkCar7X1IzVcFBJQLmwK1Adyh0DEemBFf6tWZ0EMGW1OeKQOM3kjcfs9n4IN5gihSvs80OpI+NsyMJfJ6R6Z3odadcKZws4uBGkmFKSTwOEiQ6GkzNn5Pa5YReEe907nzitcs7/YkxZl4P6dWcFKsPKHMzjHci4FYyga0r9OLH16IdJoiavfU0Zla7fLqM70rCm2FYNYqhwiHukFc7HH3P0xectpfgHYP2zGaiaYv7M5yjexix1JJRfhXokmkyhYQ/j7YAc6jLS3AB3yGe15HQ9R1z2sDDM4KhNeTBCy6r9g4zjfrg01iS8A6z0d7HuGarhbaXoVaytRWcRQMTeq8d2pIqvzZvZnie/NtFNnHz/Uk2YcdcoQMZhSCd7cU7qIa+1J/3Vsdsj+rzh1cScwtjiu6L/ZiiN4iuPx47tFvihrM8Koa07tJpbvjpSE4N6Wt+6jy88x5bhAiQt+lZLvJ4noZw85/KUCnUZCW0mcE4jt7vfUZ+ZDUqvCMaeG7uumC+H94FXdiofx9eCksuI9kZlEgrFVqzuCNOn7hDqBkY9oRdVKRie46eVB2osicMYvGEBQG5gTyEApqaEQ7lZGKEOfRSI1ddNSXEijGMC3q/9RQBPshk3NkQt+PpEw9avZt3jqO76g4e+cjLzJLHIij0gV/NWl7GY/Kc5EW9RkkyaV94acZLP+xIbm4Ydmuklnl5eXbiDpL03cQ260Pfg3h99ybwYApTAjw6x4DqROHMqF8Ez7ynP0XHBR5lEg4hA/oxVdDp9wt+CkFMEzM388Ax0j6kxOaeRye8vRvEcnD3aGhHbjre0LQV57L5E4wr9rEg6p6wEE+RiDEsnDOGFoj240GQXNEdUdrcebZInILB0LwHxecEsX5kOBO4N9IeD9BqRwjRCW/WQhBZc1XvkoJSBMuYxuMNPb30gCa3e9nthbJk6anc2zXd/LZvIsO03jddhJ1wf3izC4mLrma6KL6xEabEFxQUBtW9jd0AjhRLUem6QLjeLqjW9ZZ7t6RopDc9Jfnu8VJLYtlK1Kdf/n2d7wimaYg7bfra3xacO0iFU00RPVa59c0Zti4DDbiBokJ8nl/6iisegAWNaS2wWqw0YldML2+oVhoGNarjudtfcZwfYavTNddHe9gc8YVU2Uz2dV+ToDkuphoe3vdH6bHv4vaiZ3CGKKM5j2Dy8ukPb3U48B5WB4Oq1wM4GnXfYbZCZIgKpkr7bQCN9ZQtjF9IY9hstDZLIGVSwPMB+CSLALcvSQG8Mh6Nl51/mc+uHNcRMeplCMb/YOnaiXvVMtaBmARKMMDVl+OeM33hljMsSCldJDQmDDTZJfMsyAONkNE4CwamozkleJe72QGFYAFEBTt693F1pl2TJJQbtnRcIwiFzMVJclT27i1GbJ3JeIHK3M3x0RmNj9TT6SwGZwIKYBfUpTVvJ9TPgU5bNCOTmSIdLPNkJKyoO5HnSbcLOTDv9Pl8mbxxT2kjOMG4ORDKlj+oDtIVWIbnh/OOB14FcYPAjecCmpA9kUg5VzswGdGvl/ZEUdZ5E7g3RwZ6V30Ywg9PxOnAfdmmNJ7cYvR3HvJNP1edRwhkdS0Cu+dJpUaF3zKtmYh2ZosWIlYTtcGDQJ195ggCtKpNUcGfwz1u8+PMgT5iTsro0lnVNPiEkILXxeMm1ux7058CYbuvqKKKMhY1c0HHZKqXvsg78XYfVyxbAoo4zAV3ashqUTVDRRfA8PKIJRoqiOYNRVvX0PqauG07ZXg+Dut6bZC64h5O5CiFPGBEc/fFLbaSLSkg4gn6PQxs1MEqw02If8T04xMvDU6movdegzWIzABJ4+Ixuk8rfHf4iBbYMzVgF/iea7cqaLso1CSnfbIEKUlthYWmMbq6uhcG9DSABcInWjLBO4UYQSBql57oLly1RLW9TyGJ8BT84EF6gOc91FXCx4xiQ+Ec3xlf4DKJx5S/uBkGfeS5omchBEdUyFvoFLAetSHAF7edRnlWkrrQVlUYFvoq4RPz1f0SfV3iPCU9MRKYMtzkxqzqRI2bjuwpNClrunvwZusWBeVY+czKRH7H7WzBJfa2n5xl+RhpQCe2Eg+sbkpcp9k6pvKeEJ71pzMPST1f5XFmMIfkK5PfsvtQV9fMKBWyrWm0X1CSLRsHZT7NGjshPwMEMtogUpBq2Q35NbQ1cxnp7e5oGbqxN4rCc2l8jb2OCCuc3cpHQlM59r7MImmb+5f/6DqO592czI5ND6F5EOd6RFmy5oO3WLqHLTx8p14oFu9rUYIl2sMRAjjkMflMGDHM8oH2XVgI8hIRHQKiQEVIhmJZ+nijxNlCjppXR7jm6OqTHJAd3xv5CA1gbj0fIio/M59szBESO5yu621dgY1sJdQdO2cfI7juBGkgHvCpAodvOCHdBhQ8CIwFh2Ex+CTSdR+pE6RoGKOvEtx5S5Dh5dfzuyTAcapfzjUGPtBBpWOmC3EYlyBxzc0v5wZjnggA18OHxQwgos8JB8dbOZbp3pPoKMfZ/fXZbSuIo61yojU4I1jzy9b/nOsGIhjLwew8cI8p+GYZfUpQFei9Bcb/7LtOi9k9VaINPCMFqSLL8lANEusq4/Sf9bWkIBatBiIBIAOHEj/ZOsBedDqf3qq4hldUo/jeoF4X6yEzQad3tPTrbDzWQdfsxOAuUGrDsdwgd5TuLkGZOboezWXrcSA3YPH8xcNikds5qEjAk8C0OQM8GXvqn3INmmMsGTjXti0bOIcipeJQ9URm5xbRkVQ1DXUtGHeyQJHPxoOBUGh4n2AZSCT4rLJE274SMi2C3z9wVUALFn6FPj/tWZHBRg2/8fgxffRrLlCELQcK/VqXmLaA5VbKyijFlAAjY7Yhx4kZtZbrdLEj8GpMg3TydZE7a0pkd3Bz3HHackLz3ZfDHJDsvQVA/iOf7mFfA1Cl01hrMMYF5lKGI8kxr/K0zu/hmziCrdgJvNPfMPaAjhwD0sExTs00k5T6xBgXNMoc5XSTgI8zYmP5FWxc/S4r9cswoPBqnYyhuLcYlFsWsT4Rlrok764D/at3gEnytDDlNuiS1pmnzHgDbzlV4ce6zTpt7SwYM6SYIlbvLNJmBAwY0ED4V+VR58fWmYXskVfdmD9uAbnDKPo4Ubbh+DDHNxjbWG+/LMVk3spc6XndbKKwFvDUAxy0MjDOFnT6EPaH8EYTncIhv4CTez7lx5riB3Y+Lu5+dS+DkwfUsU4Qk0+AQ5pymm5twRgVGcaYNH06Fz5xA+3FZ7yY6KaWhYFORglUQL0CP1i4kQKVRWNYZR6JoycLS3qLOQFXUmDSwCTiLywPml0Kq2ytzwsjJZxgmVakvQNecipcZzRoJ3uTYZXiNbcX26udYSI2jbzvbXN3LckDYZstLkL20JjLeB7ZTWkaM84VvCrzWtrXBYseKOJgOEXiTsRMBSuzSK+7lMkySKrusjGVz/t9oBa4h4bCNLWEKxbT6fFHkLkF+drRTd5VjgbHOq9PxqwTtKn5EGiJ7FAXj3ybGfsaFPMRtO1A/CeQDjJMtsBEvux7eQqANuIkeTR0yGsdZ1T64AjloAR8wW//H09Xse440ixfSQxLMVpM1k5sMePTX9Xp+W8vZr7jPm3LVZWZEZFQIX6bqpChFKDHBRbWqTcVPdb2fVkWNbOSfwiAPoPvYg2ETCclA7YEQ03YVEGjHn1AMWrF/mt2cHmUxYfVovtOx63OwrMuxt/k3ChyNMtdNGvKzVTRC1hR0VS8x2cA8uzQfYwBuHWcbYu8vj9Zz5l93RQGO4Wsk4tgx1JQJkv5fzcyyYxpmOx0103nGfR1x8Y6WxrFT6rOY+vsH6lmm+iEMJT5Epiyi6Do3bnIXv00E/2D1vmvpjcbWSYr/VuGQnueUj31RwXBISihmmce4mj3B1EhQDLd3ZaNKi+a7MYAJP16KBYMRPyCdISg+P0jk0pCkj4glwt/fXFzL7SZb03lwDp1V46ZiQZQYwHosFSJGIAWJWUNNfA1o5qzOmoFtG5F7VY2xB8EOL+kvvWX4Vzxp6k/yw6Q78f73UD8lZBrzPqE1bgTrsLhA7Aiubm8KAq0k87H0cCt1qTGus64pIfSbtVW2q7F5eQ48EUvLS4n2I/u4Hd9LUlTU1muK1r545hncaJubP2tdzSvNXSxYsHk9ONYBW7yN1J95jvO1sKFWEjNP5zJEq1j+kA2SDdF+iwvlyPh8qrE3fBDwceBdDF1bEPxOcnqH9p7mfk5RdGX3n0ZeETghYfDOXoChF5vQaCKKUQdHgSjBK9A/tBNLyN0/7rL6oSiqXnfA5bZ4o5aTLGTeUnm9MuTLQykxM/4AKivbNCzH7AmyQOH9PLxGj7EcZUtKEAmVInEaAwW5IhrXdkUklTOOhnsUiidWw8hy6RkXdFbsAqGJcCkwlGmQ3SP/0Dri1jy83ms6fgiusUH+xibkPN3VSy4oE32SXN+94W5rQp16wEcGjydmqmnv1+NUbU6fH3gEi7LQYezRl5/iL9kqg/fUbNpWcfkNKX+tfe/Mj2e8J7BJLF5k23WV5ZMAWDtFuOI5R1hYOn1lGwol8neCXyHIsgqJ3mDWjJT1Rx06371bPeGb1VJpHCbyJ9mVNrAnd9RtLKtDWhztXjLQuJ1KKuNjudFhGMA4U9YOMPTo3lVRYqWEYdR4B3fwCavvxvMz3XyEYmnPfBuqihRTDn8yIoSl7J6yaxIaehTOXD4JLxQMyjCu0Z+0rknpb3TgYzEJnXgAMAJWVbY3DKptv0VNQiB8TWR1FVpkOGB8huAmnPnke+8yEzEHJkgt7/vGvL1aIcdV2JyO/BaRX9Uioj97xu1RLm/AZpigzsiNkcCrh6BVyh7d/wHMWbCK/iun4jGrIIKSb5rTragPlvFMboqTEetRBBXff8anayCzOc3dp1QAfSeudaTIHWsrexC/LkZvFc/GttK0BP9UL/DrP2zlkqDEPmDsidrlfJYPl7rziUMfBmoUhKMnN1fPA9ZEyHSUF1e2Eo3xw/H3FGTc8cZCTVYxr8Q2ML/Qvxlwi2mzokb5RMMI2vW5n/X5E6F8GSciD8oo7igvE4+AT5NNVX/u5rY00tXgM9dk1THwIKpKjz2v8lAgPdvLJebDyjzF2+fzT7ET+OJUWkIuEBpYiCzXpfH+wrtMyGMhzfhi1EoWTVnNlpeBu0UIztMCtrezPZJ36VBDwjsmjj9tXJBh0hqbDruypehrrIKCdREvDW+/24qPhZSdgIm0dMd7xTbYWSA6VIDNWyw8fLlz0x3iuMMMwtvsxTzuZReSYFkR2Y+l1IOwzymFI7GkmCs80aSFtmZC0lz8AX/bpIFEYNHH8oE6ABcuDrtg0FbB0KFS+SP7jyBeBaMczFcojcyocAwpQ+62sVuDXcumzNbH1wCM9z9R2nqC/+A1+4P0S+O/N4Qqz3Jzmuu0s3HSUhx+g509kh5ZVWYiGSgAdSbmTiVLqgEtt+LKRwMF2FbPz2lIvs4/unP994HZl7vYpN6w0+61jYtC51dRfAg2DdSHVXb8izQ/YH2VxYIAdce1bdTpBrMOVJ6hiYG+qz/sJZD03vwkDTu8x75pRh4J5i0/opiVdUA6maiLdhyVNAV19sekwNEZfs+W0ZRskSSSxMJRlPrV2IOBMv+nZWFfrJRCqBLALtbDp9M3o77ODD5F8TVnzCoaFOuTn7D0rAZCusfo3HIzVcI7oyt4XoihruCl0ACnP+dCB6mfmR+M8lSpWdXgjEg4iU5wqFlkTiCbwOSCfYf4xnbrgpjK//RZr9iaUbQpY62zQ+URRxoMn2zrQy15j2b3Mda0ie6iyivN8IbhRL4S9qExb82zx8qEgktPtF7xN/X57Cdq4yCe+i0AsrSuQf4e0PYWH2y1mrZQb7og0gPaGAJyH1OedZrokAn0Q283d34U93rzOV/SqPAm9Yw3W1juEKNzHOnAazKzTGVTspCS1ZXnLXRS7Ckv3BhRgcjmATjKtoHjmVAXJjtYtTFOgRf3VOXRIeJMbJ17xs+1sOzha+ApJEMVtYHtakKzVBEni5SQQgfv+Cpr/0lmt6j6Se/IerAyfzCE3zQ9bH66DDffwyfsXkA3L5kFAH4NTJJkmvPQGgW9pkQB8pylTcEvBqtCez0UM4fco1eAohgWsUVjIocRazp3UNkUAawjAejJI84Ox5j7qpAgOZan4u+mohD76a1naXwZCjCSlSBugGVD8IoxUOvFvIoLhdqk6OYN6us8zeg6sRGmQc7yT1d8zgSmswx+nTYbpiXngj43ktpeWmW54tA2GtRF1v2gug/iJEOG5pV4Szp0etWqsOB0E/wbMZzYkRCIpr4J60dfQwC8mh85/QB+E7k2DKoL62bby25bQ7TIy5QBk/8oJEUVNn+8hPGcuyENv6oJM+Zd7N8l5SN6yAIs/LStv36nPTNZsNBIPFqL1/j/rZ8RRjgiAUaMQkBd4lV9JKqtpbCRqEz6dixrqBxszpeF1pjS8Hf3xOpveg6weeYEdwDxKgTvP0HGBGSw3q5G/awjf8mYViTU0XJysp9RIY2Rx2f1J38vaxZqJo7EY9h84SVBg0hayMyyy4t8ngslHVCAYlwwF62U2J4olpJ6YL8OFUBw+BhXYHGC5joePxR18+gwG3CgMOyiX9VN0Teo2LJn748s9t5CncuqtsHhVgztNnoyeU5Tl3S5H+AaY10uus5QY9DDbw7neUzlKfh2aqop+nZCwgp1eQiM5lYahHH2DA6+vYGfkezKHWgHEY03bXBysMqU3Tx3yyW/FrsCf6UzJFS0BuwKWn2Vz5zhyTAh5NlobRfCHEpBpoIEm2pc1jVImyXgz+kor1sjbyX9w9qIEz3i082VncILMma+3WRPjpy20wni8AZEFE1K6v54kWdXDmtArIiCBpUqSrUwmFDD9s8pgNoDoW2bxz/KP+RpDvzURhk/3HLHlGOYVld4QriE4JSnD1gpm9EOP0nqzOr+QEntVZ1a18tKV6CsMozDnJdR4PZM8KJiRBupyK2wrJmgu1VGmSz4+TMYtWojPHGit9H0BpKJhjN318jdSuR1nwLn0TZ69UYwT+agtdeB0YI/RU8fLdMvvSPYZTHOAxJEq4INdOzGIVJIZGGlVV7+6IJ9WZ/dW3sI/rxrvwwvaWwFnP8G7eBTrhdAoXDBUnYalKuGshidW5H3yB1Cz75ReBvTnzc8zHHwa9W+ocror50+W8tuCe1nqK4NHygFvuMDE40QLDfHGWw3afyenMtwwiXpjKyAkHIZyaWIB9zvM0HpnBm4zaoUljspQP5EyLchCOBW/5A5P8KBl6npgjYURohXOVLmz1+f2uWpUjeBGC7qux7dav2dOnxdfOIMKkzbo+Q6CrGiV+3p7keNAbEfYepRf5AGyObk/HIWhyC4l/4i5lPRyHG1jyGrcWkx/gcOXF1YNTUDL73xbHDo4d2qAQqktcA8gDZPYBJ+g1X/kNfUZCC+ROspdsKCN1AvcCqRGRDyLmA+20mDvwP4Bmnb5xDrKnbrWngEkQ1+gyRVekVpdjAEWMCrxmnKlYrJtAlW5uGX1H0MC/r6eEnu0rLmHcC70G//d8YHJ1pt18zMuYpi59h3ASKumcvxhmln05tPkWn+dib1ltRcuBJouEKuU2FBZX1f8odAaq1m2M5dCafyV+tFoETf26gMKbPbdFZysgccw55NBP+YkfQmWnRg1M/cAMKq/5hC704waHR84kI7duphWJnB0ttV4B767pLl6jz3q2S609mByS3BwQRGeUn82ytKQKR9l0xkrlgUl11pSybkXqcN3CfIbRcYPof0cUKrA/fzxmuRQ4/lcl0CfwyB3/CJRw/KdVGwj28llbK5kZ5oqaCPpedNoANCutG38zce/oFntXQdNwjcOKXdALYydsG6M6yV1kLq4sl08yO7sRr8GxpidzMhPxz6xkuG3DO4sqZVwlrZaTq2JPX4jLFVfGXhFbamLNY+c27yXHdWA3on8qhjxydYRi4a0QcCiGopIvHkN0+nNgflgkBSiP6PCmxIXtNjaOuBbOS+yHBAwuHHgE1D7odBuwH2ywcBS9tZVyZvdUIYxXw2HpkdvemNw1hL/3JXRJVIqlP3p2K51IPTaH+/DgDcIgmGiu8farmY+XDRVkw3VP95SvYBNARRYpVkI2cvtLwS9vA6DJWKp99rypvZYZbe0FD4RTs84WY16LAaZa4Gf57hr28zcZ3tgx7yVj5V8DrBVVY7iBGWwH7QjCJt+nLZlkloff4hdi/+jfZ3uUCoYRiNR9dewATL2JVzVrj8fMbQORFeheTNnxjU6q+RLSenS82KSzXx2wFpBRFJQxrhzoe9DfcPdmXA1hPfgYw7zm+G57+8NrJ+iG5IYyquQpWseJvxu1fGbpEWT/7xsbJ0F2J+DnIiTDZ9x6lUZ175X1iAeVSR+qqbytkwO0lqL8rSWVOaR8zfCSzvyeLCeGInBeYfUWO8MCviT96+1rTwiY0MLHLF90ziMUMjE8dbj4hBNyCwA8AKptF54hfUE91hd9Bi0uj+r045htFFanxjgJz8ylt/ViSaHIMFhZANJc5hPWUkrX3HwHg6icI58vpHPOltrIC1PM8VF//JLMxwRfQJQMckEM+Jz5j58Be7hLzLep+ikziS8kZYxZn6INZn45BlPGIn5ke2r6/gd7M2NIU8L1jVWhfLEZIu2Bx4brHWFIqvO1Jc9E0Pi0PEziigYJEm3WRD1uSrer3zm8iE3eoy3Mt9fgrMnMN40qcjdHsrXWNhQ84SdlLRKhmoDE4qBHzHEsmaWeDQv22d6NFEUC2yHbWDx+MtcpF1jkHXyH1BS3ByPwB6QXERwUXjuL0I9qzeQiLjiIoBAhaASIOGhsOFcClqH+QqSJMbUvWppmPUBwugNl/Kn94mUZwIrXK0HYNoX0JL48bs8IzPPQQEJm37ruvI+WNnRV7MCzvGZAvrYzqK9J50w2TDzT+QQ6m2mfI99NbQvfDoBtr5MC0R1Kru5I+lolQq9SyXCh5Ka7XIWXbOJUUhmX/Q5GUpL7WkKc8HoAvVVHQWv0ZIqQgWw/fneCxZwuh4mqLLgMl4wvV+Pp8qs/xN3wadje3QniGCT8+3mRy9ZRb9rddlWT2dqaWXrC0Fo5cZ3/nL8UDOiKKtlBdYRd4yjBSVXeWD7ApjVSxGt6No+LGHZ1+004CExOOgIV/lUpX5zeimnxENsEpjJ9QwVV3i0agoXKx1z+Fm4Q2P0VccWHAJWgbIE+F2cNC2c5Q480iFHE4dtS9HX232+XO2KlV1DQ4RgxTwRu6EAvn6BXx+U4MPZmcDny3dMmLIOjHxpMY9I2GOzgHjGt5hLNqFmFSwjIHEM4+fE1hfzPFRaWNZMjUidxY6Rvfwm2rg7lpJCaNj7IttigYtE9CxzX2bde+c2imP3OUiRa9UuFTlf8x58JYxNnurw8TEUI9Zl/XLrxi2NkiaE7B9ZdDIifSj59RfBQsUMN6i29H21OKvER74DRGZN7wXCY/Qv6TCmu0gCTC2ubrqzJeldBMVqtqkWghYKRAd7HziJLX5hx9U++FyaaogstSL0C10P3QVpRuSgMF2CBtBcrMFLogL3lDgeYn7pd3G6bOJGZecYm+AwuqMouy64ryNXIKIHSifjkmLjNeVIHsqXZ+1ySse8p3Wnsj5AwA86l/IJhBzn7/0gg/UOfBtU/p4h8Lw9VPWVIEBRJMJhGRkvxg+fSg7zGD0iMo8wCoj6/p58bjL2l95BiOWNDKgJvo0DayOpSsz/hacxwv0jJ5AIQaNvc+epsFxXF7AvczbvQakMLYWFphYk0OeA+jXwCg+7wOgVrXr5OP4bApDcMsCiKfg1mbPw3YefOOQ5gP+4RAWlRulwlV25t4pnWDfVcSMno9pFREBfauzUkyf0Sp/sraqbRimP9I3x92AwnAdPn9tYVxsiVirjd/N+BS9Rl0wHRfIHMFKjxEg1qalPfRQ1ONVU6XvB/1Zf+JWGKthNuCuMWW6rJ0V7JNU1RaZO36cDZ3l9XfqL40nH4Cf4+i5q/wpbOmze9fD2SFQ7Af5K/PfaKYmImUsie9CPOlx+GvCQCh6+G9Jwz0K2SV4lApiimbs84JCEF+58HPUiXBsetqiwfCOiuGagaO3UW8Ogm4sQA+JBB60aFAjGGN2aCnX+6/L8IHghcQuUEuZH2h6HJG6EEtpCG8AJG3xfw8ZNW+h8SJlvqUmQlgjUXItyaeAlv9JbGqc616dc/kJYwh1B5v1qb5Z1cRVAC9HRx2LzHxAxtmyxQuG6MKLenUzfiDBHe/HGYDUv9N15ZwQqBEPNNTi19d6A6d4DO4m2BtPGLbLQ9iUn6+GtV8cIzujPwbFOsbqlobLbO2bbGc9GpGhCJ6L17nDoInOpeueS9XmXN/8PCzeY9S6bNJoGksPTFu2FJPnT1JvKj1/OqpfmTaPcV/vdjBTTmNqpjc9Olv5uZIpxz5vRqeWEqhhoRwDvFxwFexR+MO66NM6IlB+EEbrmheU8JeOzjqU/GeQnOesIO/dIyI8vElUbSQ/fjbduYYrAO1micd5w6HYUZOEGn5kubvb4ON689BsS5uSfykNPhC7yE4/cVnFpxhK35F+Yc+wVr+xr52pFhp6g3emg2kBRuq+xUrAWMhvnzxNLbVh9FdYpq1vOZaIlr9wjoMY8UJOjeQZzomOIh0Njx1wl1/NYI+iclJM72Wu8egNoQqe2VxjAR6Pz6UfEZIyO9CG2CTrinBAkV8pZqaMqz4GuppVDyKMg71ySN5TEkKZ6lJbLbqAWgWmyPjrT1bLs39utvI/DNHcUPSyEZBxuHziCMZlbf6bT1fHO0vZ/HYYqhjQjRTGfWR33xmiP7teHCImevQor3+sZH+PzVUZkZF1Kuvf6O31JZOx/W2kvkKN7xR3b/dWCKms7HycG8LWQ54ABVkaWhwyHZ03+/PMICa8xG6sirNJq/y60eMlXWzIximPYOL79jE/kzPY3wgveKDajD8zEZlqjMZCzKT130FPtDx1IikPyRbKfLO4chH/vCfAUrl6Clu1013nnri6485Z8qj7pVbGUDSFFNxRmnt8EpVgb19y127XSSfl+yLEJJ1Ihd5yAKzSvdWWsS0elokbV94646YoGpTsyYpgw6/30ZCP6aKQxNOIF5U1/oRsy1/I5PoCimiWEaAoQMA86z0wA1I47ADLtPW6M1/eXqG+D2Aut4Td+ziN7cQiHWgryN/5dZDUdcHsX+RnXdjSqjMF58pXvTwkt6DhzPpT+XPgXvbYl7wtp8vVDSix4WY0kldiIOj+NGdC6L3OYi0v3HMothd0R/6rAoVm+Yx3ubVIv9GT34ZX4J/n5+qxZAaJmTOjcgJoUjTcwYKr3MIN/Y496fuzSdUC8Tx6dmo/FqJZevmb1Z/aej3z/v9Hk0zH/wj7uSExpz4AcqOlsv0UPIL5n7nyBRJCzkTwvwXI/9Kqx/up7tL8tfSJP54xhEwj+RySqwoE5Ge2+SIhJdwLMGeHYLMTP9SyTrSJl7Ydaxj2cc+Em4OChR9AT7atlJodUAkoD2Qbwq8Rvb95Xlhic1tXf03NyLfiOfBfmN+SKDLRPsQ7F8vCFfQztVL6vSpx+uT4m1btQHwtqyILc/ZfaPvb2ADgzvMzTUHvbnVSxgjpCC8vJNwG2J1zKDteEIUXZeZ7wbo2dYBbPJC0enzvAdAewaILFP47yIFUmoKY4XnvYQqKz9yxymnpJrcxDSZwyq/hKbi7qI8vyZalzwTiImN3fBoRTWce0dLhPtq2MLj5+LJV9P41V1ctHjZ1VolYNwfy3n/01qPoX5F34GtqLcIMtG/aVzd2h8ngY5oOVdX/daaPwI0oReYG7U/N8/m5jjxfSnVsKv6lJQdmsAkESPm+myHBIS8JJcWe5SPYmEYalKcsckSFfJVN2mTmK8jl6sOJURzQHjR7LfXzXj7/fHB0qJPpHSKEf9z/CVCqdubfoUhj3iiDQAd1UMA7Xq3ZBLWjfwWk7Gju++fjKuCkmAwUUC7ZgNKSUOo3Uhyw20UuBqXiwWD8A3U3IEtvy+/nJIuRAIc+7s8pCra9Nh+8YEdouWO7AE1kHfqeSTOdEQphqHsq0Ja3Ld0UGowlnxKVP3eZuTwiJdD94cpGsxxif7Jl2X0kyR3n+fkA4e/lhtdYjt5m8sz0YheFk24bvvSM833xpLfXFcaNV0BEhFDLwVvw+dvXgMeEtR+7XJ9GCpSOrvWXpgiTz3gb7+Y7yZ0OtJJYMFHCjbTRELwNTQcho7k7AKR6gF5HRyhT0hnPk6GbjG/dfa/foHAJFm3U/+mG5S3IVGy2ya8m8AiAcujbM8Bo7URPea8ej0kJs8qx1Tq1E143cbF98Zdtkyf+T/yP+oa96lmch9wrIWtQGXIj+roD+w9O/BwfJb7cyWIC4SvPkNTjbi46m0s8NBCbWvH242Tkj4OX5n30JQ10hMmKJ5cZXOIj/FgAFwEpxK7CKwg2D/En+jD7vCOsQJ2V7t8IP9+xZdt5q0Trnmmcy9O9/Dksao2mW7EMQRygUAH0p+MTNDAU0JLj9ZHuNSV2uve5LXR4tMDcEKrzRb2CjW3YPI9DtRpaUN+EKOYryuSKBoCu9A0yeALddJBQIz/qxogbC9B2jDOTUiyDTpOC/I8xhO1jn7tqXoHisYbRBX9PTzTNS2nQWF+bXTr2cGUdXg20XyElkOnlRkR9Mv+dWEC5r3DDRLAAQOqFHXYODClhYtFU39Nfkqqva7y/piPqWXCUgWb0ePGC5EaXaq9YxzTiFXJxc7Nz9VkSnIOjI0FzRRfuYbyRnId0P3pP0Pb01Gy+w6KrbydlaL9kkKU3SteTjvh8W3s165HtN8xxYXwqiOGR9evqwxKh6GzvzKyl5HOlkYeypG4xWuNqJLkyufgTiMEEe+bSZUWfT4MkILnBKDWgF00M6DlJ4FH81PjBMn1JHIPyIWRG5s0K4lrqRub0B3s2o4TyIV++RK5bqe7dFjMSFshbQ+wIOeakJ+y38BX8SCATF/nKEv/90AHczK2+VKKC3CMYXr5WXR7GbT2tYsr1GVCsf14+uKH+UBuAGl0B9A617RDijAAoCKan8mnmEb/Eiq39JK3wPDGWaw3ruGRfMg8GVm5hHAzarHu9fFunK3AuhKgrKnYOlq9nsJPlwpB49tQ1cdVmXht7M/x8WxTlmZVmaeYthuU6JDjzyti8ARRvZB011TYQcsN/XnsYvMRVLemrPLdeuWJZaDMw+X4eGZfCvwDfBXBoYfS8ZU4Bv6wudSxz2ZKDmvoon7m93i7+PCXeZH1sMNoWlApd2GVVAjPEh9ok9ikfV0bHuzB4rNOW1IviDOEXdMv4GdYCB2w7aEeI/UEXc36Q1WuDYObQ+9KnWYJVe/g6SutfuLnepYvv2RnedWmLNOdpto8NSrk+g9ffoVcjHVc1X5dfwxfP+iKOfEhee8v7Vhz8m8ikf1yLufrxZ1gYYksWBGWtB3DthO7WydTGkCyh/M26UYBydjGIZue/31YYTfiw3ZCZ4sMG/b+LurSvnBZNby/Xz/jCKMdlrRvbOJaXpQmoEPx/mP46/U4iBFvj0h8v0cWFS4ppqAusFgAqGLvLy1uoNhizc+/WTvFTPuYQUxMTBb0raxS3Xbtb792opIpNSR/xxNwSfR1vNVCWzKzm1vHF9j+WaB+jR5Yq7ucTVmZphqzgEpO1uKXeFYSYrLvJI3PqHs3SlGmg/bN0csr8xvxyWiz6F5rs9uTl88iVGitB3f/ZNrLm+K+b2jEkPkUFB1bJMnjhxJek/YwuVCZvwB39/1zwbFrWPDA1toLsYxru74oHlETLJwvFR/mnfaTH7reX4fc1ExvDYeGgr5PjiuaoeCn7X8JmJDmGNjXcUXpKaX4ph3wV60DT9VWZDG7ehzfm6t4I3py1m943WaBgOGKk6ZpaBOZDM+IzIzmjOo9zsYwT9Q48t08Qe/ti01+TioRJ4EFF51FVJaXzVBnmrwhruRS4oAi0be+XNd81XkXiDZ0PjIzj/qh70IP9zCHRK7EmBfui5ONWyHTpxuFsUtyL33KA0Ezb4KHXpC52H++PqlYuul9c04gZs1VeMzy2Hai6k5pycMANkF9swx5N8189IAwgUFjr+n9Gzj2Ahp+40rG4BaJIIaNCU7dVSlJL2VKxuB564xBoT2rqp+HINsIcuBTs19KBNyVO7EoqzE8/AJtKW5dfGNROIrPHvoZPoCJcmAfmZPQXHMEatl6QinHPxIsDC8QD9Iz2XCFiyJ8VR0MfBKRmzsURFSA1xXgZtSglgV5E5a68gEZ2jL8HC8uNthhwCOOUx1bwTRjXP5MJX4+awso0YdwhZc1/4ZcLbqGS3rX33j0r3Boe6F/CFQW3D1A3YWU19+X7B5/afL2YEchYr8nwXVA+hn3QmRN/VA5jub/FArG8LZAlTSbOKVL06orZEtcSAvsMsBEDVv0kUWGpnWMafFP+SKFY3JhfXHSWUjJtHsUV/xSNmvHXJ7jQVivNJqIv6n6SKOgwDL83IMOvZwWIMEFbjgfEjPO52jrc+hRg8rS1QyZRYacBW5Z+nJaUQjdj5ZKXdO/bCnrbEG5qTXiJFT+SqMS01X2h+3K3uACgr8wi5FZ1rPqaKfS3r8ymANtA+QMz3zZn6SfJKJUAdIS833qcI3/qWgjZfatHzmUUn+ukCGU8rURvcwzw3uROIT2KFF88pKtaMrfcOBjePk9lppJ523+U9eXdfUfGn0KXXTJ9Q6ML9Q7hehsQF5zr095tmfjd3fpMsXvGH+EII5wDn1Ze+0+sTPCSIE/IlJfINeB25/XHZ0aDH8lpixWsJ9zkfSQ4uUvYW8IcBxEFNuonyNUsLCq4FSEt+dx0/hRVVSYOvmYqWJYULHLiajQK+LYMo794R5qelSK1FrrRTya0aKDFJwETqzds6ThoClmU3z1qBSe4AZrR0jop6+LvCtUMcNqOZkn3unX0IbSZ8WZaEu3E2CKBxhOZWl3x0RK01WWKMGstxj3GjvRn4yi/solnZ42WHQwSFiD8p44x7vxPuHfSD1SZcJ3YYZ7UO5nsAVIqGnMMdJCwfcJoEz35bxKXnBuv/hJ7fquMJtL4T4i7CoYnrXkfghRbTu7ml8YUowMyMxB0fbXdZeBeA8I5tHeJk9FHYPn1i1UVrYH6rBbH+OmqW9YGEwbn9t0Ojgk6A3q+V8aQi1pEbwDT5DgsEXbVUXO7AC276FQ7eZm+/52AH7RrjtWisE3O3t3TatVDn2uDHkxXwSEqRdgSNKoyuh2mObEHFpRWXc79Uc4SRs74WaSRM1oKugWZlF/3H0WnytEKTGn4Ef541NAweUShXwMnU4NsT9BZ5IBHl775M/oLY9LBqwa5KcUDf4UFdWEVfoFmMsIS3n6DDpZtX5jtnV0cnszcrFzVI/85XvsAyiiln4DvVGPiFjiOwPPXDKzGpUgDocEDJhQUkJ0ds63KbJX4vbJlkcqLLnJKZ+q3JQFIeDbyfrXTtNBreLJhTglWjztmN059uJsLaOAIJfto7Qqg67Y7OnhixtK20+jy8dAf5KuGW4rupykxkbbK3pf2Z1NjC9HX6YFZYyrrI/RDpIgcUENir91/989k1oc9+UWGkBzoCduA0YcSGE75LlA79+e+nKc/PfrrJ/YxkApiT2v1R+NSVvs7xIE+/qqwcFl2rOrSEhQi40w6JRyLZHs8Avuqb20jLWWVfGM5iABhX/i0N+/M9WiSeNLfeVTV5dXNfrppSW538+Aeb3eom/E+gttJ4s3P+BzuKFboWHzKzWchzL3cqF+CA9jUXLvVFIi6JaEG53qfq5LVAxuU8zX7V+7Zx8sRY5KCJI5MNOt+Gs0G3n4Ncm73xQbQZjy5+AXFhfrRDE7xCb7gTK2PoNVGr5MMoqngghaisnExyhf0CQ1W3pR5+CdZDQALmSjDnIks4SM3HCNxufvPjVI8WWMofhlYp+e+Ctx+yVoJTn7qaP2nKZxABcyjPif1awV3eD7qU6ruWltMdCJ8fXbP4uxUqU2In3HpTIWoSoIv8O9eX9H2wdbpnOzy5+l78kJFD9oh7RITeI5FRD1SKWQWialJnjf+BR94bd7O+dJIq7htf67klaWPJK5EiWBpWpI3iXldzELLRRKgQ+g4ih34E/F08IoIypRxkLblgSNohDtCoTc8pH6zODybRZkCrBNCqKsLK4aaFBIlWYx0tT+l2dt4xSOhbFFsYHwlj6MrP4bUVIWA6RzwesYBQm5JvRyxgoHOlmCKTRe/TiL5zmu++OoRvzoRviFbX/pnbPBvKKE+ggVSivKaWgWmeEm22Vu+fbJQwit0DVTMf09fp/xyF2GKnn6sBAY80U+kSq8Wnm4TrCe4SsXf4CzI+aMh1Nzb0cY34vqLCHk44RZk47xSMVsuySaUuVNKvN/TIg3EZ8HwcdOZxSa6PCkkjAnaVLfQ4QklmIqzm1EmaPQpUuAWAKeu/aGYIC6HcMk8SIludfvACINfviJNnJIP3X8xc66cAeP4ohOoZ3s2EZUbd6n0j4y+cGRKOwPVya/oZWCU6Y9WhhivapMgYZbesTliRNLswbNKbu4pOf2VRVc+PbkMYKwx+vAlL720g0kINi/1j2gXcE1JX6vnG5BYHO959YzRaQfEACSHktf75RHo36dY6IsF36uWev+dUVIzvf93egBeBlpHCEw7/r73baYQpeX/L+vdoQ9BLYWN8MhDXDxdQkYSUhqgxhan7I85Y7dr6EEXCPRSMXf4N27f54nb0Jex11hmot6khLuUM+0++62Xacq1YHbQllhnmcmQwkChYaa+DlGBgxWkSrX9mttbzzTlWFObFimaT5b6fXoiR/p4B7tXkG1VD+afP8SBTVLuPONF0R/Y8+M7kxqQtmtog3UhHSkEeV/UBQ2hTo4bYDi/mq0IoD/ep5e9PgiH1H/6+8hSmGWdMHlKo4BQgrlhKVdzrh/hSCeH3sjLlNCX3Ud1k89F5OG6GYH19hZsjeei53IBNr326qLt/1sDetazW8L3gPcTUeUSJ2epwUpoZcJsZE/ggT9D5HaxAHniuu/BGK8Hldd47+PIxDW8kaOAnu9xKLkPbaIUyfoHCM0UM04rxBTBD3QnkBB80ji33rqXFLAMIqB+Yl2cIZ2eQNxDta547jUma+kbt9u+nX2JKlhpehaNkRt7WRxQRLd4dIvK0aX6hx/iC6iqtSxHlpGRAgsnq0Y/6Nb9Amq2VlMQl0kfgIyGQeXI/C4N1B1v2txeoyQYPvCi8+iCCkEPL48Jl21DThdKAWFYpm1g0Td+YUWkXq5mWC7Q+aBhAPU9q17jT3CrEs93AFu1qGgPyk5hJ2u+gaiDIrfzd0jWlEIBQee7/1Bz4aoao++p0bv8xA1TR81KgYDsg+CqltlET9t3yT67zU1QSxjd/Q1R/tWN1kvtT5b73NSB6XNwJju742DA2NlCfYoiKwiLyuFL5Uit6nGNNFhQ3/hC9TIz4hCs56SrC8+BfgXa6hQXw84ZY6/O+3qCipFsQAwIjTnLe9SpZ0kXnKftEHBq2ARZ/KDVHbCfZG/ou/HwPDDwo/egkH01O2NCvwmhYHC4FPo6tfNJOYPVNDIzGjrwn7XzpBSPaSI12LXVMnCUjju5QRx2lUmPtXcJchZ3sc++zGWLmeIKdsy3bs6R9d/10/5a++gpy5yiqZx/nCQeBsvysf+LAXdI3SQ0l1Zl/Zfg7etxj6oUmX5m1GXF8IPd2dkj+6zawDsqssCDR392Fghh8n8RQWpUOfzYRyAAZB2v/8VApwcutWf6kMY7FjWqVVK9YYKn309ZbpIg1TpRim1F72/W+i5cZk3ww9cqGKAEEN4Q+qEvhE2AQOPRRKq2GD+qNcuikH9OfOOQd2Rs35GbHONl4Dnzt1BnCNyzMvpkfTdXjBMvb97h9NqVrSSNMmRtjJW5P1urR1SLJVwuKXFCCMHnXaw/YXmRxPnOKkQ8zNHqkrudoU6K/q75yt446392qG9st0BPk3UxquxZnqPi4ZIPSmrO3NdK2xCyuyXu4RpUFUco87hx080yk7+D85pjqeqT/FTmnlSaMDSF4fYfh644BbqC+2nzxU7rOSXxeR9maOPbBO3325UhfjLQ1ldIiTFjO8Xq8T8jStdWgfLOnnEMmb+IWCzi+8G70bfMPDIKLbUkejQ9dQhw84x6KgrlOgRaUHz9vQtl/khOFkHCr8ntAmbY1zbccnSM/vRh/QMDLbCv7ivaVpYJ77bUfPin3PR0L3qONkbGgMr/Suyl4+lbH3gnrJcKy51FxjRUGBsEUoDzTRxWz/2k7e66gzG0dzX3DdTtMCOcAQxKpm/q1wWLwI6FXF9MX9eZtkmMwSvPfWO/spZenb3Qpib/QvcSc3uUbufUmtE5AAhl20UvgQ4gkp+tfGjHSeK4RYS2TVL013SS0wNu++n/qTx6vf+ElvGttV1dwpQGlUeUrLI/ckCFTQZjnv/FYRq6NtMPRHY6o2S9LHVW9oDwowr2f0bil6TVoABYnm8j4J2sptOqQ+aEKk2daNAfSLmx5HDEg4weH5D4vWJ3zM/98BUgIxdIzUe5iYTP2wWu6TMzNzO5yKlc45dGyIt4eTDLqJdktuKBs9rz6IUBq4I8cEsAFTR3bU26TdoT/5rtat+oya2qZIGq0E2pnsZzv1JxQ7M7gMxu/ZVdYDpLXOv5SmhYO6S6o0T0k00t8k8Z8LHNfeST33w1CQ+a6rGTNTMmPtzGMzi66eWqGq7iKfFlVwHphvcDipva4mdN41O6ozUcbFTcnZJ1EBWHfaV5fwg6qS77/xHCyAReUJ/M0FWGcwYZJH8dudTK/FbdAZJvXRjMMOytQK4zQLfWEjyFOjr0j8jktEB7Awo/D5y9qzyepPGEkoc+iVoSIWewPiSg3PU1rf/qR3ljNOWUV9cWJ0uMpfkDrGxVou0f84AwuYrQaXBvtM8g/vxUjrauzgxZ3zsVCigyJOdXUkC/ugAHrjntZOewUXZX2VDR7uB1YfxTVK7quC9oM7eAtKH9AOOSrwFDE/M8Kxfv1jQexJEQC9Q0/rboDocKZqOE5oJWAGPBoFxpPD9Eynn2sJ5GR258H9h0+Kawo5ebR8WR8JPcZKbKiofjXMJQVRXMdw/6l6D9hmt5cHWItsbSAuEJg9oYcbzy3g3D6u45zonR8EDiHiKKZsObbsvk+ogJk6zBV1KCVwIxBIClVPOz4U2wYrK7vhETvP+6xrSuMTK2DsW2820nM2+uh5rQ0Y+xx2E+3ac42i2yoCIWagZGSpWDK3evn6rlypIkzyW49E573l+Hgi6WVApl0YviSqBUgWfNcSnFe/5YibHnLV04xSTOB24EHT5kvbt7yYMIB1AIJLl19ILNzprkql/A/uVGxWlxj19fgIpuoUZwNgcqonOULeP8Pmt7WcpGhPDdS/G8nPfasJo3zVqBQHBoGJy5jxEnqEzVucXvr4OMuttSctIg9aqewhgX3sAw93YD06SRs8HXc28/R+w9JfawPvQGc6dQ9oiHU6eVVhv2meBc4OvIClMMBH944XVUrMFj9qDm/tK58fyspC00gQABeVtW+0/S4rxYmvAJTclgIoqOIEtISI2lsMjXxKCr5HJLSfRxwjMAkXq6Jf/pwVwfPnohAetD55itTtvUKXNHVJ+9G1nnTRyIStbVogLCodgJJA5dEPFYV6ucjGM9VCCP84LDXhkOXKSoM3prSPz84Ob5fzQ1vvRbXgAd+VIw68LR738wqj6ecPbhxy4rf3FWg/GxbO7fl+6nnIrTeQGQWqUhv+hq575VA1BndtnvgXf9SnHEOE5l6z0/i2NX/LEtH/vl28I5I/kGRiqMYY/+fOyFBg/tkbG6610rt8MRXypJNWkX3dfwi2zv6tY6oOZpMa2y6BYjg22b4PYnMlMhE2QWD9mFjfBjnP+9TBix8rJGUx9CrIbsg+WZet+X4Thhv+asNdcyEAROtb3KobP/desmovaYd97f/iYl6P+IaI8MS3YvhmnQdrBIZNJi362KtvRFliQPBJI8jcXuHOZpIW8glfYyPFly4ggwNqMBoSC2+ZhAAWEkSgMvMUWN1ugYLfQuIFMqwGEbmFlTc6Oevi77R5/aLfG1M9abGnFTf22raM6G0lodg1fFC5Z8S+q+miEt3S2TJiVd2raxtLqv9blNpk/382B40qbVBxIAVYmMMAGVThZIz1aYoFeDnw+TlHvQXsT8NSsITI386dYCxtjJTRT8UoXH0CHYviudHG9+owmpljU9Uallwzho79yFIjOrD9/k8MPy5sqa/L3AHmsYOrNMB+Ub+eKp7AsFjKDdFiyAmlmh+u7rg0CBQVvD5tk2ngtQ7WOaK2fr94HSxLwdzHAThKdUN1nEAMyAYVKzexrwTJxnfsijUPTsTIonOZ0j+TzeORRklBagrYCMw19sroD5x5ErMuNXCo8zERCW/yoxBiwq099KjbLDwk5vwBxXpQMD8UWuPndd9WX3j5twoLE3PoMf53klR2TXAv1oiACbWKXEdU/CJ6wsxsvsCHMXeslFV102n831oEjqrcws24htnWVgLJy1dbEcYjeghady1aB+4FfCu5vlRvMKWW+JqEjQ2KO+bjOBpNbtqzSwshFjikzmG2RrurkSjQYxZW1s4gF1p9SwObYIRfe5oZDcxRoiJh/Y3VkIz52jp9L1nuDHmYQrq4+Mf8uddj8ek6ukmIa4z9ldZX1ElWcJ+uKkXf7BeEy3Lf2xeXKlIzxa6QSI3BEIzt3wvu7HzESHdig7dbuEEGYn0FdWKFzvb85JzqJhrjzSy/ZMWjslDQ+EJi/KfkgkS+eLdNahUp0X/BGIP8tLw7MPDCY98GC677ZtG7ctI4mbQR0P+huBr85No5IvMEyNtqwpdhPjlX72nU1RxZ7lyyvCUyhEvqluuGQMksqujt2W40zVTz8b3pMi6MNsA81O5ndkHsbjNxhlVk3F2OGnw8wkIkfubyYtpVUBe+lluiIgYI37yvMPiOjyDrNTN70cB/IxO8Alv+ZE6yUmfIxXpOKkAv0MIL3lfNV8tRTxjd5BdGlYpN2SYXwgqZjiAoJwLgPQ0uNTNT+uVXbWvVd9Duu4HeYg9ldeAS9Iflm34XNgS95KFzGZxvSyArsL9IzvJjnjxt6L938HGXlVLuEHov3/F2VVNdwvdkYCRo4qxSkNlhDVk73ayctdYhMPZt/bdD8gH4xBggBfOtwshiEy/j9mTlqKAzIH06hIRem/C5AgWTszCf6pbsaPlaVRRsjrnXExKlfLqKKuwH+dFa8fKNcEqeyOIBIa9ULZfmKUY3/zQlJFtBvBaki7Hr5YO90fmfu6RLagBj8qZip0VFFiG+FTWxmu6h3jNeiuHOxTJ6dl7TYuJxIIygB+/UlsbIuDsBMH0+Qb2h3yuxqeYxxpUdo7HCvjHeydK6ya8MMJ8paVVwszxsDWP4DkP3weY9Nr+osHAqDAEqaHHCByUhxp28jpyOGdz7pzpIxHithbiT/dPyCsorgS46M+Je0oknvfsDQFRbfKfPA0uvv9idjt33LSHKGf30ycAQM6Nh/ufuMIVt8JVrDaj/qIDnHx1cVPwS0leaPRK8+Nkkz5phuxXMKKCnLFQ+7lG/fktpTDCnD9Vxr/xMIYUBf5G4O+9lx3eug9DdOfigSH8sA25JpX066qsGbyEQ658GDeyqVXjaHta4sI/QpP4AplKziVx/y9WHZgaGIYA7T5quy/BHuCMiquZosE/Quq5NTQqsW4z1vA/iHdKG0s7eiky1i2adBUKu7ktT6xocXHgoOOT3sKvy3txr5fnlNS0MhK4pWPJl1eSZ3DRKQKA410cDsIJPKVCGQ69KDMprFqAmF+0vOW8Eopv48SQ6DWe4I69JTY6+rZNBxDl68oj6wtUHnOIFs1o7OZ/41jeTH8u8C+GHRjeSROg7LSu/areuaLsA7p9pWTZWwc0KOcTtj+Sb0VS9r7MHuNZxcVEPRb5H88HqAtjkLuVAQINblAXPaaF34tCGieQeTyt7jj/VcnF40lXZHDFyV/rXlWuQOMVqLb8AjfGA+mLR4PtitVaLA97JZib6fr4pWBChqvO08V3iK/ECH7/New9Uz8yV/PNCs/DzH8CjwtmXafuf/sfdeza8iWb7op5nHU4E3jxiBACG8fbmB9054Pv0ltWv3reqqNnOm+5y5EaOK2n+BIDPJXPlbhmWKCtdfmKMd0oeXgK9x8tk/zcDq6gfYk/CQwevcxWvnsRuSjDu5mlHUWn9w+dbQbslXtuD0rQ9gBQVhGqwXFJ4beZGUHdvYVls8mz5xm5x6/GHk6vryCPpWHPjaMYjGpNRnj5OfVvLea4eWwgG7zSHuyK6AlXv1EtZspmGq3RIpiQaKmwqXBNDozDKdtymjeRSaLKf3fk6p0TGn+YDZ6EmT3SfLZeGkikZyqaCTqY1hU0MzrIqzX1wauUlWp32zK/ioPWBeeLF3a8tLh98f+gFiG7/JK9LCRU5Z1pBEYnmpeZNWHwGO1LgGH+/PbFMyOJtE9fyIYT8LjSxH1jT2ZsxllIdyNfvhP/yev1XsJJ/1bM9LbyoO9Ljxr0RxPdBIYh5JykdZOidocmJigjuJykZFzm6GoplJwMLeExOdrc+e8WIroSsiSTEtKhfDGG9VKUnWh+w9e83SPnSWi2R0eVHYQWT+bNK3MzU8TVM2W+6teGupi13MslAEn8OIsXmP83sTyBDA5W5LVrr8xqui0KQDujSvK2g7EMl2imuDZsowwwLluyv14mDFeJKJaS/+4OhGG6KvLsY2s1xC+xyRjqJubucmknyPg3a2jREx5RZXWGLmDMCRKUVlXXi7BQdSdlT4xU82+s0d/8iCtfJKRq9QYA1qUwQoAt98nlFsLpPBbx+d4/VjBcq1UAP/c6HLt1cyWqFgWwIW9fwt2gOYkrdzKu2YleR3Otn4OjNv3JGpPF9cO8gl+/TDp5mq5bcIUBI6vCMQ0STuqnaDmFvwEKU0HT9EpTo4mSW+nom0DzCVlPpQ2S8G0LF9yCYhFk472yfama/qxLnk5miIDeI8ldduWOkxKw/pgeoiI25LO3aBpT3VW27sWb8tsksq7kG8+/B0Vaeb0evzfdWo+yq0HHBZMAOmvN6A17HZiAqMJ/ZF6Ga9UJGLCvQ9CVocTPeO16wxb/YwUtEZmjXwCx2XJ4zRARP9Zuric8eLHd99PfDFJl6C8AQeEr12czS79k+nLGTgpcMKGnIOYto+26S7ZWPT8VMXLpwn0vkW8SpsU9HdZV7a58fabDJPLOQ9HO3TbSndR6Fxx090G7/ZWjqnYHd/77je8qgFMhUbttXm+Aj4UTSphSzwC0a0lzTBKnnjTTWM47qdHXK8TJXzMJvzKqcqjIheAIWah/vNn9uj4av/cMG0BXr5fMzhw3CoEPsmNsfRSM6+r5Rprw4Gyb0lH/fdmJujP3hEMvQpz5ADmls8sJgTgV9ROrx9vFwMBp3Hs9w4cwNqmkfqDYOpQ/WxP1Feomv2MOQAMMrkaducC6Ixe4O7uOprTIxxLVwYl4XdgSEn5aB8fOzwUZfMkyVc/DNs3rAo3SWm7sqVy1LquapVfo2/wznNEujj3Jr63dDxZHouIiuNo4kotuuUJrEWJyZ834rmM5aSuzLjVhRvBXiOnucnI06FmiZDBdo8LCauVaecm2xId+uwsmC+zDx8vL/yHeAhD7a4hVfNnY5p7MC7va+PL+uwURBqHu0AweMcvc3ib1hD6/XFUhTyTIoeTthPVsVBEBgMZULyBHjD5DJIaJ4iK9fpBl4yvQnkZPZgoc4OfrChxaxIla1Mye9I5bfsMFmDjZukMpAKgHKKB2CKbCIqO9/g5auGvPPeJo/UZAy3cllrT0wHIHHBzb6/a+nrUvQk84u0pQvwACwsJyloyzt8z6SLRqbq5AZUyuZuyYfsaZeRnZ7rxl4c3+yUJ1KHo0tkwBzTM2jZPh7p0+DR42Ej4koeJ6J+Y+uxQfCbVeK1FEGUfmSjEcW9t4yW+TW2h3DrAfEinNt6ucMtJy9+L8LFST1t2YkMhqYKL3k6mb1Cad5OU7S+FT3XXUEA7+R0b3o2hScRuWa6+fNiMvJV1+c03NOw8eEp2g+eYWEn52aqPse9+75iNOiLKByG/9BGBfa3mfonOhhkuG1ZnmtZXrXphx3U5Glq+A5YomCaQTnTn/jpxl3VFIQFxLuKDR7SYvCGRr49eBK9VcQFSjqXb6Kkx4WghFVY3qHgUGjnNXjlb+4uMn+04LWKwOhQkYplWWnAKyuETO+Hvs22+ko0GdfGONuBS539riA/tr2UoOhDKV4vQUplFyxjA/MqwDe7DrGX0rgR1h52HFuPnE+gxDmOzUwOazsgkYlvLJCT5yHi3o0q2vZDRn3su7qQuyoDIXhExf6kNIm56Eb023t7AAsqm7TwzPkCmb6wNbRxPVNZ2xHS8hbpjI7qlM8hr9/KrjqWmbRXGPutVVaa/xo12X7PnQ1Nw4jbvbUzD02yzp0DZs1ClN4s0frS8FK8N6dq33RE1rvUzL1ckFrqSybeU8SkFmwiiNqZncuFYIaLyifetMNyGm5DHJNVfhZbf/VLs05uub7T5oVcLa3vXPskjWCxNLMEULZujA45/mpPsooRcpw/Bh7b0ieVlhz+Cu0uxwOEblhFHZWYNCItNxh1uya3jwAcyzXyIC0JVhTRx9p3IgeW5UbAyhmcNfWuXDR67UzqxSU7mhzyyOHHPMI3Scw5MM3KW24+Ky89mKyMV1ZVmE+ITgInPBgngNJDVaQnnirBuLWIoxeFNYT7MCyp0N0KihPNZLvvuO8IRep9IHvYiFW0WIcapm7QS/DaOsj3B0EmXKNAFYNQzkh4qe9yk0fh2uAJdnlVswWEkGx6yTwArxy8LCkVqa+myWTdTfdNe++OXViDmHMa+bQ1GROfe+uB+B+3nCDbjs2PfnQykpyTwqPidksGqYWT+9o68P7q6VHgq37I0R3OQ5FSaB/y37MZWjTwbnnzXETrbwxwyUbbG2flmMCMsW4hYjw8YdrlF8olrL321nCKcpfCDmaFQIkedmuGiBJelm3eEoyCh3RbS7DkuJyJPhC0Owt0p0Pi1pH5AFI/OKuHrJpFHLq/pXbmstoaj8/5WpwHTTH2s66k60E+7JlZ02NJE5tZmycrZH1rd8r2HrnOGvxKbuePH+U3HVDFYdQDuwLLmBHFuQSlF0cvsL+X0lPz1iJIUMjzRK9P4xxbm6phT21DJFcSzA+GKrLBKrNCYzKIB+vpc1UqwcrkxyqPvKgrcL4K5c20m9E5nq6aPcVXdsKv9sRpaRiGNaBZsd5vVcDFX2fWpeg6mVVKJ/oboIzAT+eLHXavZrLhg2b91w59WQjVlAyUAACpE2uNFfVkKjTtb+g0SS7uszgdc45951zbsh/MBGZl1TIeH79lEGiGSTZB+dLnXYNIRoSNYryUX9REs9olNR3NDQVKbmFxQ6AxjkRsWRerYjl3QpGHU0Li5APxwmz9M56FrnxSwjhflLua+i1f8GXwIlXiUOByd5hX/gLGFv76kYb7cWHf7HI+iZVBnY2ftDDPDFfcyOa5UHo3g0s9wkBxM9OSXl+Rq4g6hCV6Iu0m4zDfVvY0QA7t3KlkPCfBgxnvgyren/GptIiXeFBhHKdSKSn6huJ03ZAgSZ5ilkLexkoP9YFLGcqxjyS1ttY8JaZrbsmlVEo/ThJCIpThgW/Q1s1LNqpWmfSke60SCyHNWafQ8S1t++Ptsp2HkNiyCaxID9hmZGMQp2MONCCEuwnz2dFmIaQyE+p5rfruW1AYKpmwk1C8U9cPdUueR8Jz8o/mMmCnYGvF40GGb+EyUEWY1GoPbolmVlLp1WzqEnD3xM8S/Sz0b0Y6yW+2UOgTs3GENrQ3eEvsmglrAbNdgmlZJnn1TcuZCSwIEox94yQ5gCmIemtNkKuchb+bptPypPyeuhDiVNdPs08uVe5lxhSNWEMR0OQMIRGQ6DqPA1bPbWgCOoZby2JeK39I78Q7AmqskbxShPP5dvBcaJojlosVWHGlNfwKT3jz9Dnv1PBYmV63JutMjJEmjRCegU05GhhXuTpNGRYfkG1dtMPApslYk3r1rb+FkLP93QLvqICbFo4Eg+Kwjt7GBYCRrxc8MPeEEdufwchE3IP85pMx7GkaFJQmOEpcvSzxqR8M8jNoiF71zLRQPQ7mXEE2urSKb9IcmtvumxhvdyUbb2EG68Sou5LWsQdo/ngi3Ra4Z0YRcNES3v0rzXSSaJrsOPbj0dC6eMjha6ds0cxy7APfcsCo4sATGTlWiuWFyVZib841Fg7qd3nrt53hE8CoGz2etlGvfssFaDjVU1AVXLPb/fx5Pp3z8tV+EDOiIi5D2N0HHn3T8k9sbzA8DIWp6d7Sc83H+OtC8xTrBCQwMWFFG51/X/5LWi6z7uBPiKgiqT7EMCe2flRfuMoyucNY0aC1EJc3HkYvQPkM9W/x89GP2wzVjMf7URvfjA45fJrAsJl2fqSuaS5+ZjzDfC5Euk54lFK5bTL77F7erHDnaedQwbtysL49pVY5GFhlJUDvmXC6IFwQbHX9NV1O1757rCDIbwWQo1J6YXIKet/LAxbFfEpSCLKbeVtS2JrMjLymhOs782y4csgY1NUogUHUUEzn5RvdGg7252Geg1M0AiEzr4pkKmg3zpSexL7UnnJkWadN93vwiopQq+LJlW7Vh/Pm0GrFLm/7mRccbmUSwS9b40x8kKKXteiZIuB0ihlYxaOP2s6JLurj4qPh0frpJ1DQxzZ83M92VONLVjpKR4K6vFpCYHQ5Usm4UOBVOWQ1tMcq4OlHz33OaMrwupbwbL9Vdw5/KLVFNNz0QeVMeGw70TynFzeZymwIzhLFrxsDx9djlgxBw5ZmqClWOzceJ/muaSNpKKqjNXJMioGh/5qkQnM/2Js7r5vsLmjopBPmjp40YlRINqAUCc+Ua6ctf58W6jwjlmny0pdjZSRME75Vd82u401ZaMVsPrqxM2sN7IhWpTOCzz/g8gXrq0c2mE2+Yh/AIGNNDQ6yb6OJtbmM8cyMiWDGfm1i6Kn7wsfKXlkt1/ucK1hrrtLeWXzgSdcEjOLJorYu0DA3oz99xxuKuCxwep/XFAaejZhZwLisMsBbb3vr5XieJ3sr44Vcwq5KwM3+oQWpTlU4sM80o5B5xYHvbP8BwefC4+mybx7rdv/r6b9AFsAFds2JysaehPZi2CFcLCSqXcgWos6nosi4ldyvCch5hTNlfak0LWOVSSp4Zr4p5D2HeedTV/EwSkDFrSaP8+QOerdVeujLkBwbOSUQxQFGvWaLEO3clUqzA2dzEaWLTqcvS37Yy4rLb51R640psiZ0eX8HqinmPNg+Y192xl65NAtIK6uFEtxiIDALv68X9gkEOI3fHKc4TrrZRRNvRM4CJrOH9FpGYsXP64e/ILuujJZ4RV0hi6kg2PpWGkSPPmCGD1GNNee937HAFlmCTV3C3+NEbVnae2cyL63XKLrKYBU6oQjsiD0gZGRFhZTF5pYjeGAbfRzDq6+8Rw2xp6qTlcRfHibQxhUKcdYlOtkpAIQ7rhGufS+MKBOEw7fU5fX2NU2vFHO7aU+SFkqKWTne0hWlKHLVmnvfjlBDuECVneEoTAKWA8H1k4/pykvg2JvXEi4wEXzqN/FpAD6GjOH0GyBJrgX7l8s9g/BPnz4xqSgnQoCOjc9mYN3ChlknU+d5hVtpE+tzjmWhdHjX9ZhKe7H+8IN5GLBE3JCqqVXWUHype8sA+9y4cp9Y9fWdj7PxQUxBVqnybGbHOeyhjJEXaufPonPce26fX9Y4NE8B/Vwj59v9w4uUI7UfQKvIsvAy3n6ar1clLur8LQ0bP7xp/8SAdLSklA/BNizxEUHBvF1uUWkBPVbP/EG2GDwfT0yI5eA9q9rweh0rmh5ua73PByU41Mt/tQ90miFTezSQv+HJFm3D5lPvWHrJAVzUO66H3WoEZPk8qrZ7Y8qbeXYIRA0C7wtc4SBjefDYeFpgtKKaG5mTrXMwQJNN6MC5/1urgWV1BoSSIwnOGYU/27r9VvtdWl/0qqBqoOSt3crf4kxtNGKTo8trF9LuG/dzu4EOnns1LcXSKpKth2znHPAyXp5m4WX8ZxjXfpeBHFQ0wejaPReb8zZxkVlKtGZMlM/kZQy1key2mrxLxVI6yO6lF3jSmtjeK9t6twDWIrfk3uPUkkYZJY8rIAMBJhwFN6iZlQ7NBCq19UT14kkA6S/fVmz2X30ntx+dCD2tMLn3K9+G+tTdTeiR3GN8srtR33vFQChxhThvb1mwGVQi6NKUrNHiXoFWjD/i9aluzUseb0lss1zVnnpuRmWsnBGasqFxqpshZOBQvpzF9I4+6EAtPoFf76ac2mFbgUQdx5sqCOJmEfHLYMx9un0/k9a/OeVKsgVZlUXzJi5q5eFvmjvUwqgHO+AVA6VMjXWdP8+BQl10spAIpH5jM5ZHDD/bRLpi1jO93vW+qVqAbMt52HWgVc7LXfm2NeOpQbIzKiOFOaoJQpVZQbw1L4cNvlGpFv74CgyzZEG8zmcEc35La7yfnBtnXAh/kuHdHAGZ1B+xlwfGjyZoYXiSYtL+W2KQ+lD6YYxtbht2i91MQVYMMwE6rlxOQUozZFg5n91RqvEWjcmgW0XR42N56lQyD/m+NFNWf5oev/r0Y2gmxmQEupds25ksTwT664llUzShWJ6F935CnHS/t4R9Op7IVMULSUmXIQ36aTKVAQeG0To+i5Ftbb6je4YeK4w3JJ23z9yBhBteQxTnnZpzHqmYh1FJO0+IS4GhYn+62sGz/GbkWoRO/NOCb32TyJM8QuEtop59e0N6vwl0N0WdF1IK8tCtdUe2RwYmgtDer7Wc8aHMAnJUPjjUKhF92v4HGKMpknB4Exr8SUhFNKKHWdKayS1f7f4EAHmqnk9qkkMky5ndYqEnytYN7Lh8yNoCwXHGvN1v5EonRYAVfjLzFSaxkRqRNU5odWr5aTaAMy9fDGQ/An9QegOgtt1JyFHdpJbn+JQRYWrUOSij3Q7xADpj7MF15SDtW8oOs/paGX1Psr6puMZ0BpMKTusxXLi9u6D7fcz6S0HAvulnPSbt2NJJT19pe+hQbO2sTMgzWga6H+R0nxntxCARdDjwOqm41feR/bJ5gPYbU6cSnZoifXSrM3Wl6L6Kxx4cy7hJqSIzFERngK4HMe+QRUTnN8X7Uk4hsSVJEVBaS1KLTdilQKrdNcWBZRDoCnX1omgrOMkrJk4SApZH/MuRzEgOxc+0AavbJnefD/4MniHWli/NbAtReHDl8xKammHddSC+aacet5rsnSYAzCcWAEmMbgE735KJFwfd0L/xaHLzmCJ56sWPV3jJ0QvWrWXPX5kFlpWvAeP9OK+LVsKhLoNynYGHeHxvsNZs2+LKhEr+ZAc72abE+e9+zUo4f1DsE9bIgHvHVNZgywU10DmLtTlU6YMS8VEqJNfMPJ3RebF9FF6BQLAMrZDjPgJCXU3FbRiMUrsOMY2HU1mntz5xSGUR2Q6x5bkejiGOr6fwmZ8Rp2W3+h7e0mCKvAZiwJfPLtv3/q2Ya8By02M/8eqvGLxcR/FBeYejm6xzLMD67CJw5oXPHrX0eXtQW19a9yG7xSC7T/Q5Vz7ymltq22axkM6M/dydbUlAKsPyDBfS/OHqhdIqLH09QmBf2NfJTLvM/UHLy1wy4wc5nlNIf1gu6p94/6JZfNFI0XRY/lYbXpd3Soo0ES+X4kiCPoOZmRhnYDUYOZpvlCPaO6Yofox5kJjXYWJOLxvvAhgyHiykgjVFGv51a+/FzamQROxpFs5rQ9GJsZYLPbgRTIfGot6cRI+a0P4atJWY4INvCP5xdooblhUwojSe3lMcfG/1J1BysxCarcWPuayqh3Km1m+91wdK06IWaKk5JQf/iS8ZDYnkwYNqaaxI297ElQ9YygQXR32kZ2IV8BZhmIeqSHCgkOd9hMCL03hVIe3a+IRS2U7IjsVPcWQFll+OnRHqV46pTOu4xmLRxIOF2Q8o/iNAm0+WyPoicomWZKHuj3HwYWeGX8bde8yn30I0Qa1lq4MO3Pso5pQZYDFfv0URmE7vswsw3AhLp+e7KYmLE+srBb5M/UvCp8afXPpqtlSkXVrhgomMH73ZMbcooIW31Nza5I5WaUV0aSh1rSPElN8DDqWenDD0FXyN9xXXxiCenTT3PLO0BqSrS4dY3BEGcpZ0W0kYV69RP7LXspzDTyRxtq4wpx8GDB5T+bn47xTyH1hXrgyzfYsJDBw8Zpcgb3y6cgd+fnaghHUXqFOtFEwmrw8qIhSIKllcOsoLmzQ0hq71WNZVe4o+I2O02j4sX5o7hGsLJ7W8tzrGXZwmqvp+08MDE57B8PmI7/3ahw+iKaJ87Duog83WwpCvpF81kdLEbVOc+/tmoO9b8enubpb4Ayoqr/zdjYBBlEjBgQy/BknsiRZyxLVJC3P51qEUSZVsH7aln1kX3MpWlLFbk2Abv/I/x/HG3sK7JDcR4FYlTra/u4g949q3njyGOVEkpZ4AAQsQn0vF+/IkV/Of2zFRTxw1GBIL4vhTm2zBbc9cayJ0V3l714gDXXO6YDcWjEUQxV2i3l8QBvki2aisLMuffryPZhs4gBht3EX0Wwmcq3ODz1ZWsN448hC3wmLyE9gbQlP7tbDzd3SQEdUzvaOPm//aD81CmBvn7AO4qdStamaUhr3hHcwqUvgFZqar2H0arBCpOq/pFWHhWtdnso9oUMn3ntWLXflWvKRQ+8bIke9b/0yGxkPmlH6uSBO/UBlpCuweHf+tHq3S5qfLX2qTjJ4AD3tWZbwpjbNful9zIM78vFJ/MywwfVJINQGwt7Q6bWxWylzrOfgn06T7awNjWBc1kZAuKqXrox5TJXzzaxLpEpDZ1nx+CICF5QDAgrwE+tIMz9cpPM8+z//szZyUbm8ueVWl7k0f+/7rdbyddgAZjq/biS43ZUKUNMaAj+W4mqngXCBJ/4Hy/4Gy+dAvVnVl9xGM/Hp869JVe95nuKGrbk4NWVF/oxukWj9vWM72e8d/IDfSQWP0yfoFtIcgainyHWStnh/sEG1d/Ue3/heK/7hwyz5Ldvy48Hsv+vgPlOsOMRu6bAFZ7aFff8UR5BdQQxPcdf44BQoDgcO9Spfyxyma+nGqzKqiXH7e+QuM/jgdzT9OFX9pHuDyj07BZj+4rG1/juH7HYGq9O88B07/+hxRu2Y/rvsPhGjvntm0ugERmn+dmPvstA7Lr9P1v+bvBN90DcEIqPLz88f7W/Hr328j8xj1/+5W9l/nCrTTD58uAk/9vRv6p3qxss9WgTp/EF/NyXAv6T2vRNSN9299PI+/GQiQtr5j+Vc+5f8Cw/2vdPBfnYDnffv9I7NFVRvFVVst/8cn4P/i0//1o76GKL0vZqM26pOqv4U4CKzQr5PEDX2yfm50SM5/MOq/ffq7s36eRX73ZMiNJuD8XlZLZo1RAk7vnwgMrFy69gtR/0WMQ+/jqK1uGRHlb5VoyT7ggqptuaEdPt9hoPn3c58HAFclUcv8ekNXpSlohf0jUML/WUhEKPgXivgdJKJ/gokYRv6CkH/ERQT5N4EiAf3fBMXf0jMGQX9sxi4/Vb78SpbmA1R9+H6VeZb79av2/fo/dPifoEPyd3SIkP8d6BD9Ax3y0VzGQ/RJf11o9dYolnuyfmLUfB+AaQI/3WOoQB1o6KtuQtYSLffv31N/tdifYe3TDAwF+sdrnt5D+F4LDm5+kbVslDTFt42fK9cPPVibefkMTfab5aSQGCWIv/zi/Tq5yB8XPsUzKsXu88UnSqt7bX/zG02mEEn+0+T2K31if0Zu5T1d131R1P768P8CcoJ/L+X9mZiH0b/8vOq3lET+RJ5/PSlhfwJpf0UFv1njuYzSYf91Sv6y4NCfzeDf3Jh3GyNouTvuJRzLX6LPZ9hn5Jd76asxSxlwCFoH0wT9AogiBROIgV76YUm+hIH/Oa38Z9f9D6iSpAj158T1F8T5PfH+StH/AvKgCPgXmiYIGidwBCZR+nfUQlJ/pBYS+oWifl4NoxD2JzrCv41w8P8hnP9fEA6OEb/g/71Ih/pvQzr12o1S/zvKgfGfhIOCBf1e+cwi0CeO/23O9seF/q/RV1uNz18f80/p5k8ITbg/KPpH4qS/n38NraG/52Ew/kdU+glUv6Ul+t9FS+T/wNC/BIZ+0hv8ryGTG5Fg+DeIBP8eklD4vxkkkcT/kNF/QzIi/4qx/Z6MSPq/l0iE/hNEBJjG+CczjoP/fl2Z35z/8fkzjYn4fv5T04xAydD3WbJE8c/hQH9fYfkruzTxJxoLCf0J3GPYv22O/8wE81/RWv+TiipKoDSa/jOKapJk+Hej/Kd245/aPn6nWf9xvf8+Mf7jzfZ/ZDVHL+yJLW/+n0+IVt1KS35E/NlqPo4l69MvhSLQoy+qPvt7dgn4H6/w71fqb5gO8jylv9vpj3YmJEn+Jtz91cr9Ndgvw/iTxPRhrpZq+FPe8PqrC/7CI/5oBfvdK6l/grb+9dTy81fod9DwJ9op8WeCIPxvoyXkzyTBHybN+D/+GdMs8fdMs+Di+6L8B5H9f9f9cc5/08T/bqdOX+VVBkx5zLp8LVHRlzJ+Ps7nT2y2Pwb4h9PxX5/7w4l/15Nhf286/zcf4V8LA39L6vntrqb+5q7+UxCJcOhvamz/vq34C0LQv/n81ZsT8o+iNkHRv5DoH3fnv8LM+Oebk/zHbPunQFx1UQHm6/uXmcdbYPkpXP88yKsDLPvfBNB4WJah+5tL92sPfBot0U2tPw4RYQTv9LjKZTVzhxSxGIAbwdtyyodT3N988A8vcUwA/laSPvZfRwP/bZmQxHxmLCEMcKLdLaG97i+vx84w3KGyjDwl4n2CZdPRFUrIe8Cq1r232MLLUKSr0MKx2Gd+fPgD0/pySUS4TcVHkYnwHPcqkfFQFXjmFnQOAY5jz4UCi6qkZ0Hc1+ypONNSW74djuVjVG4l3lnfHLZLNabcgy70p9yE9WiZj+AvbSadCRLQDenT3LWK2lI0RV99cr06+gxP6tDsBn9dzPm6pPPl3/dX8JV5OBT4xXLfX/9s+zftP0L/XSdde4+n3eKKPUMxIAJP3lLfoKVK+sv1P///y/Pdz2A7EC11JZQ+GeJ10mty/mV+6hhsCVHYX9djVTl6ie9n56rftyU9yyUW8Uvr35DTub97truP9df5WwOEXl5oWSYcdbxqZktgE09EZ7vvuWLUPQPEtUIvqKVi+H37HIvF3rEm1wjW658c698bpzmGXfB327nnsQurH+P87Xz/mPObTh7wW2vM9p5zJPRMMenoRfqxFnvgy1fofp/tpg25TRAaTrp3+9ft/Gjrx5xpXbsmqFnG93WWA8L0wMq8unYM+eE0HxKkOsb1ru9u+eBQHQcxbAN6O8GuOcH1voxDrQvMtPftbgXMIhF5+JWKwj3rrmzyf6/ne3VQdwlvCvttz+bve77++Z7veaxTD27j3vxDzzpH32vIvm0b0Kw8pp3bmL28xfbfnON/YnSa/Y9H9yulE+bfG91fev1nVkPj//lerb+zGt9e+ZFPOrdMRfp0RXqLeZz9QY+sEYk0FKPvIUaZwoDUQq2Z420xg+0J9b3K39+UX/vSmvcZesJ9TnZihJ713+8l6kvX9ajfdAtmoPziRY0d97pAkRd2WkOfkeeu91jvY3r9zf3UC/0TrPmJcxz9xRynMcX7nvvaH/j4xct7bb9p0WjmRupQb2XzITjZ+7PkRpc8BQ+4TTd79s0zgKCSr9oPli7aJXmeZdys3dqNjSE8T2XlxKF6tJ3cCSgetCytgZw/BQvc0V3h2CtsIF8NnYWC6ggwYzyScmZixaht1Na/lUApGibtzR+7dWrblRytjAQBEdmANtShprm/fHOIAW95HN1yJat/eL1x9ZXEUDfgeKodLp0l6TNEMS0mCMR/fT3mOu5d+0YdxNeKh9RzOFfM/8AUcEIn3PobROmjn5ioQCoe1wAerMgqgZ+TWO++/pjvp+bYa5UUvhdAempi1Dvt6TZ98SS73bIbS36eJTmh33I3Neao9HYMGXndsKaDKhH4HinGdJDL+nVvBP6NaroV9sdSmTxmc3ANz+iwxWzvFIHIT5fiynEhfTvQy3giAdofynXZ9LXVg6On4VxV87deG/PTqzJOBDroa1Yy7CSnYZBEeniNdTS/KxAt+uZzV32MrO4UIGPKie4pUREqjzrx/P66IxOPIT83+12QDfrD1zCKmGKo9wvepAq4ZtIhecFlUbwRcOSxaBUTeV1XHSFm5Hu6pn4OKwNEIaBXAPdkvgwzwmDNM/+uQyTpQ588MXr0SbJRizx1vx6fGN1c4VnvoKrQ46o84AC97j7OYfq1ztbn66uaRHOxAZd6NiTATa1eIFdbva5AJGfBNDgTxK3QD/h4z7qJo/nXV1LV3kJBlGETPYXjEKA84bF7G34DvkFCdZbflqV+EBdaUC7k/PB3LY6MFkkmT2KbrlHIMsntufeOdci7C6UNHukk2VLYZgj1Zb0q+a2v63dNIVFj8LXEYpJvtFxf0mnfaQfaZUqKUBS1lAH5JF/fXCRsOd6/wCI5KfDG/lgH6SBzOBAg5gFUcWA5pBensQtFedgNgVD950//zxCyZx48blnAkPA4kasaJpeTC1a33E91Dy22r/fUg0ySmPLG3sBHdID9Qm8hEGuy+zN6f4d58iBJRuXzNRk6v8rEdcCc4pmLoXjPeBDEtnlw187uB5aRG/Zsoi7+hkI0u9MswfPNAydfAT+gEeRNe4IAsNFW+x/PiA/hbuuMTRwJ9FBNekAKp4kuDERr3FNONi7yzkQ8kKCTUFIKeCwfAz9zr6yLhux4sCRFkJ+UVlVvZZ6km1f4PLPxjf2SC/e9IbvXjxUOB4aUurrbk/sJIIxZvzVWI367GBKk5nmG0UTk04Q0HlYqM/BoLs0pYD4Y5xY6Sl+yoKseSNuFRlYd8TZy6Jr+WQ9iDLbqx17IIvVluS96z9v4GaSsqncoJ7x7YgfZLpN6vNCFMn7unOwEiAlSAgskiPR4YEW9ofGTcODXrBKG7WCqRSkwTqKdV5opLgU/RtVODE6UeKrUH82wqOFiniJG9Y0FEjAaSvwIEeCsPZFE4fb2DJUK9kyY1ftSn7ebm6bXZmtD2YdpjZo16t0dKjQ/QQS+ucSC4CqSvCZF5RyE8tzfsbN0bx02YnrjPh7mcTM2VgCzyRXbAdUEaMxYa6/LWESg5GgKvPF2P9yJrCGr9QGHq5mteJTyUue5ZlB0Y2If6jDQRHFKDMkex9uez/NVkHXw9X931yLkeQmET+xFFGI0cFj215GxURLmUc/QjpsPGXQTk1lk3Wzz0+rCJw6x6fOJWxSQX2aHYzeqwgO/GHiro5VTuSdMwT6ZYpB++RcIQZ62ORg+lSaTufrGeRAm6D7sYl8ySerrNWbAVny/0Zh/uIXDKKe4VvQHeTwtcyttuD5B/i0iAQm5hI7uSTR9layd2PA8KxVD1M+3aYRgceu1MNrRwbwxO2FpymjXogreiMK8rHca3YI8A5eloxFP7me7tTw28TDstUIKkc3gcGtyQttwYu/nZ1KRYOfNR8zeDI4ObYKFwm/4yhLuxeIaasww448oL+GlEsDVezFpenbwpfIgLeFzSsILo1TW/MsEoBeFtH4YBufrVjeHRX6X75oh0ScT8O/PtIL0Uz32TR5nmWkmH8ZOCmuNGw7NPmxQvok1SsPe8p2keSpW4hSyGg0HvvcY4tUfh931jKoe7KCg+l67zGyukiRx+pq3joEZOLwdhPni0qkiHx+feb8C1y+78YE82weRVte+EjRyLYsHMEQ8PGEHeUl68psbl/fxq1G2g2b4rRkKGc07svX6aR2VIgXRm7OdG8ykTGBbpIkHohrCz81mWsLQnrw0NjvK81w5jeYTHnLB950GxWuGRvJzFcRbKvK5bNw3P1gYTmeNkyzEkdJbEHHRD9NI1pwCJAStJpMZu/c6G88Cx0L8Mx7sHARPGt5L4a6gq6C+vxykfs1IZdJx0GeJn2XfSAb/ZFzYYM3svaHInt5CpQvd29F9D7QaC9UePAAV1GiB7oc7o69vFoPHXur+0pzDiFysy6BcS5Cl+JgIaS0X3EbWIO0Eo4dEIWAetyB63bNfDj5EbLRNNvxZHNoQp+czJOxb8BlblXvhWPWClTfTT+/eafSNyNYP2ZAXkKPEb7ZrZZO6B78GRewGxTuiWzk0Uv0qsR/c3K85TpFBKP02xDjNWV2YxeiccSr/BHVwFWRkp6USJL5VCoFYmjwfqClFjOwGmVjuoR5k9bSTZGFEshFJ3t6fDofJ7drZNtM3l+3Tb7Y5csO7d2zyzPLv/JkKs6H9oARUyegDLO/Giw7Yt8tLKian0yy7kkiwik6CZFYZdMUFLUqlNj8mmG9Qhr1BWo9ZzxWT0vK4Uwf1LkMxO0IrTXE1ROua5kfm3mPKMgnvfLWN0M78pwYrCb7Fspgtf+GHC1v0PojBVRKTUdD+3aOCWb1ndmXg/DUy0FTNPEY+yUsJjtWvoUS71RMM83H5s20dVvrrE6VyqQPiJVLpIggluRCdkCj0sfmn5UwgOqsInEeoAEFI3xb7NUvHRU2OP16Mj4j806rx7IHWQJDLyaNfyV7/8ijNngSqLGPEhx7N5CW2fEyssiPeY6yzQWbcSJ6stFB2Dew1Q2rnZe99Wbrnzr0W7NZeYg4Rwihhnj3crPrsLlJJFayB6ti9myYsXaR18K1I7JwkJxe8HTabhe+lbp96JzSbiT5r8uxRNIl/XTv73vx1Hj5XHIf2OOB5axiYJzPT8q0kPIpyn76Z4rVX3sMVHSg+tNIL+pQnG1EFl1ftllHUheaOanpdKtYC6fwi2HdfQyXJf3ZG4pB7yAPLlQkO9MG3szJ+A+L5vCiCfkTFZBHImcsojdKetKTHo9o+Gfi5alyiEn66sXSSsTlLMM7DbhJktm/xEmICw+cbT9D2iNEkxmqv7VicEumc3ZJwbaHbUiinS3nLFbECge0rZRFt/XFfhqOBquefIBzbh7gMGkgP5X/DrByDB5OPpM4rW/R36YGMF6wpjDJnYGPBjRTpOE8RUuuGQLPwuSe4D0LD2aSYg6+MJCY3VTx99nEKcKiubDCpijgwsGV9a5eTTg09NCrueO6dkLc6p8uGgHmDHHQmqd002z9n+pNmX6yMX4o8vPcbPMqXVJeNxzLWYd9TFiRYdOtVkFu8c8YmLxyZoGNS+BR/MYI966lLMeKenHIEbQPQkW5NJGkj8whF34cWAbNt4tbR65ze5fBRHvVVp+mKrabDu03kMwBUc6hrO9VDbhWin6dhFzsLpMGeEUV2m91XbxXrAFym4014MfB77QBfL94g8cjk2BCagynVEe2DXwQsXLAxK5qMN9erRKpTOMnKvZnBJ/DbNlrwNCnCeiInDya08wddJOfoJXOcvq7Tg71tSSVcnKzeiRDtXdIF8lEdM4NKOcLi3KdGirU6WYNmSuytHww4KCpzfJOMrmdVnUHEG6Yf01RMvrAf1wtUdxFqLO535oeORhigCo1wBNw80rMeu6zfX4YrJ94VSDuqa/QS3XeHT79NqR5OyA/zsuSX5NnQiQlU2JlHJTGBxBBA1P6Y6cZ0WUv5DwsEZwPFi853FIOGU1RK2JVROJgGEk367BvxB/Ea0HeeEIo3fV+OiyHRq4gYMaWPu/8egahzEClxLW/7VURI48pKoIrPezqXCsY6gURBvKxB6if3KtBU51U8Z2sm82N/g6g0dE2LpCOWfu6OSBObwTD+triP3uZ/SOo3F9e5NHhPc/sc9taLsuDQloOex2OzjSPzvErR4BTd0ErN0+SqVuWdhKUza1Jkp0mbDjPEHXXItCA9nVDMTjp/yzOWGNAMl0kdKdigKRrtWdxw+4ffz+2yVLr38XSeOjLifEPfPXSvRICSlyj2MYtCPjMLoiy/1YmH3+hQuedWV9PjBQUN6+9dBcuvwFl812Iu7DDpGyZvUdRZLdnktqaXup7ojbkLnod4KzD0vHvFNwtwqTkPXzFiFHkndZxfYuPvGakR9pHsY7gA2kY7c708Tz2haVHdj2aPSR8kBYv4cknzaKYCBeYW6OAqlJ7qJ0cfrnwLXRvAE2yETrHVVsIyPluhSALCZA7naFG6M7hzqTKddBYWHpy3+8mcU696WAOedWtrkIxRIrZVdAhHwMtGNl+mYUP9VLdoVTZfWtUiBgQcOzmvQ+qj3oNAkWH/Fkvqe2crl7zka+i90lFitnd8S7oo8pTyy72JRlP8OU/Q1Uj3qvNDnOObh8xrstp/yLJ1aHQNJrdg2Rccyqo4t5DL+wjE0LPbhpKDPZWBJ9iOOO30om/WNZ1fCwdTizCXLxOTHflcNYW3FIN9MwXfR/3Q1xVHyobZYyLsEY1Zw8YyUzjvwqYvbKHLETIrkUmZN5lIgHuvr/FaksMY7RfAaZavIelDclYllP1IleY3kVw93INzsw0m40yBw0thU0Pa3jWcQ9Iqr8eWApq+HgzNtfaqfXTytG6V2MmikZPg5ox39DXJjK/ZtBOMx6163cIaHpbeOE5MeaN5oZBLzmtYBNkNyElgoomIe201rtgjv7XFZ0FnT6tKS8zVlyQiN6UBKW6sad2B7OtbVhjZj3CzsrCNRdegJ4GI8qEa3EubPRXw3yYYCTVo36YSmE22Pc7CoQUFxNlXDd9DF2MOgP2ilZcTeC9dUfjQj1aUbU44omigKjeQd14WHh/Rklcg+bHUDD8v3udtlAj0XP6hccknsqgZSH/rbzQxeKObxHPJVv3GEC+9U99HlmCa9wM1UzU9yQ0Fz8CvgHfITAxHxhOutHvNGD+CzeeQywJDINTDeM1G64GQ3U8ku71l9CC7yDTbT+GRZ1SqkINmlytNUEXsIxQVpPTzQYDE2T0DHs2bk9rhhUEWFSa3ga1uL5VfA42b17NpL5rTRoxFdP1MBMJMQK1xVhjEr63lfRggPxMyMLhR7dLlsJ7zFE6W5dQohgqbVCC3tnlGydMGTrsbeCe/u7KZbLtCCYM5UoKIo5eM9ipfexoGWzyJ8OgK8Ax1eQMXsB0SeR0J/dCzlwUsPBMYWksdD8Hh1MFmP5M2egyeJA+b3Rj3q1xz1cJYy1jL2doJ9vLNKHCuPdZR6mNtT3YMr8fpoJu4ed5HMmDbWKEDnQQ/na3FPeiWudmfe6pTtkf19dMqiTuF/omUxXp9om96l+O1Q7spbgT3wMSA0RwmzXUvnahPS3lvL7Vf7qXEJikCydtwTQd9hZcuIF4oj5VMf8yEMTKYIDBFGTLqi9WYcEYMsNwommCcL/cWXbhoOA83hZ8O+7QyKHmudGDO4o7aQ+KMwVvH8RB2MJGOrTkK6dpXJVcYxSKEBQFFQK6mAvq0ExxIyYcV5sBN9Vx11JQUK1bXb9H71FIU2CCTaecCwgJv24UHVJ3tmRPYrjqjS73yMjOlqfALbXysRi0t0/Fx7aShm+n5NBhPaN56ox1WJLUIjiPts2MbN88uwkaTof9w7frSdj9ez8EAFkzhkwCLzjFiGlnYM+YVfpgPDJAbKuGBsQmPUj7z+lTQ5Q0LcQl+zJAzPz+AYaR7PROLD49eOF0E4ni4f7WMLbX9Q3+/V4LPZhfUXxGHWBA1V1jIUCRjHA/mjGUEsvtaEJ6O6EwYY+wPrkjsgsWxfCCAPV+sXxnB+g5CWdMRg6q+AfYh2rUQRM5YVeUpYzTJsYb+OqHQTQ/o4/SN1d1SlvQMZaVb083rhjbSDfNENBG2g/3lzg4kLpqaaaJ44hNMiw3kFzrdn/quA0OKKatMXaD8YBV057iL0i8pFmntQD8952jUkly2EvOYxlPWWUHx9xt1Ppu2DshC8Acl86ohYscqdZ4xw+atoAEzUFSI4fWDX/HFC5Cg/lkLvBarN7nLhpu3dPccRzWq47nfmzjOj6DTmJofoj1oj/iWVLlM8jTv/YTmuPjU8Hgqr9LlzgJpmBm8Q5Sw/IHi0gLYcGv2BLAeVgeLqXcHPIM5Z5CtEBVggqEyXudDU/3JFtYrnlPQbsx7fqIAa4l8BDbJwiesGymAVcZliLL3bvXZkeI6Iiet/Nayf3FMbceDaurrSH4EWtDB2cZ2rpm55ZYrKKhnujyxmNSxZH8aV0EdWIRO+lWwMBPNKflweMTyaRT3IdrfMcUj1JlxDIqUEXzp+VYQComPk+SorN1d9Ni8kukWKnMnJyZ70r+opzFZDN4JyIBcMId5uzupfl/odEU7sZkhMv4yf/SEEzU7ct0ncksO7JmGYWM8dCVldP8C4+ZBCoj8RfeQJsMSPL/sMx4f6jd/JBAtwJxQA5k8c762YSpimuYdYhhnnyThzpGOKaoHQ8ThigTjO41lPKeLX/RBeUCe4eWq/QoAVtfiN0kZneoVgWTv9kN2M1d0ELkamAU6+ib/O3wfq2pDlIlwVOIuP64c8CP2ovU+ndX3G74gtHho4oGINXduWiiQltNEFV2Usfg2FmxKPvUyFHkvIsq04tni0+RhLIRdQ2aHqRkmOkAML4/4yUAF2Z5QtPUto62J03WfjMin8Zu5o674lx3ZciGNONkqnrjFZrIlBUSG2j2gkYt6WGX5D+odMfPyQZvfGkHnXoM1iAwfTePiNTmhGZw9MWEFHqY67ADbc+1UBWMVhZrkjEeV34oTFR4Y+uRo6l7oUKgDDeSRvJMPvNOo7vvi++YT/S1XLVFt7Z+AQh80/HqALO+u+1LXJzFlNBcI13Rmj4KQKCKmvcXJcOiL57KWBRAc0cHDxD4+59IbCmxx26WXV/VUF8asCt3EmhK+cE/db+jrEzt8aomewLTuJAi7qh962jR0T6GPvKa7C2+WZtJQjpdhVibSGXezCZf4aYW8aXo4pUMXvpIvvG5LQmO4OqbzgRTC2me+7xHDpjyuDObRfGVzJFPGurqfjFYhy/xMVgMl2bLxUOYxnL6TUuijkN75kYxWy65LzdjV7K2kd7v9zrCNQ2iayJ9TMw0aKqxwhpSvhKFz/LzVouc2D4336nv+8XByKjs2LYDmUZzrCeOo+uGfYukclvDy7HqhOWKoxSf8ZFwC/VFgWLoSVgyyfGQ8Bxb8vERFm4Ro4GXM0hzHHKB2VwfZal4dwZpjq0fxADuAORkwEuATKmj5GNH5lXlUa0yQ2BNMXW/rCnRkM6EV/Jo9nOT7C2RScoFNFRh8gw/ab4DBg5o84GXYN1VtpGkeWido0bL6UCWEfT4h3c3v/vvEJwh6WK4VFLplR5WJ2T4gYOIJiWtu/DBusMaFAuF6/JKYDiD6+hDg9VaOZ5obkj1t27u3hv22fotG52Sn87q/5reu/32v64tgLAe7P4B5TCY2Ux9SEtRY/QqObDj0/Tvm9lSONtDHj9TVy0vVKbyv9MsL63tJ73NSDSABSAY2LQLT/wz0Rbv3mK2Ka3jF3vRj0OnmJj10JpkU5DSus+lYR+1tJTp/C6UWHEstqmBMfwNlZmtaNJed+y1Dv7je4uKxyO88VCTf0vHAtKODnvFQA6+tk7etLxl4r22ZFjAORXLFY+qFzjYSMdGzalv6XjD+4gAjn/UXC2HQeIL80iyF+t9VfjKWJwdshxLKV1wVsIKDm8B7fPasyGC9hk8ifn2+/DUXaNKSfJlp1iVmzG+pO0menjEtwOiUbehx4Xr9zjWm2FF41T/j83rURW6vKZkp3yQ0tt2V/y9P17HlOBIjv2bv9OZIb0XvxBu96L39+mVWz24fZl6pqyUqMwFEBJDAgpVnqHxLsGT3EQH/jwIRw/j3AB/5sfYGPOMG8znLU9Rc1mXelNr3Jq/oqE6SGMwbxnXoKnHgHTzrMWw7y2nwFYyKwdjrtyAy0DgTLlXa6OCb+1d//hEDmqj3xZoqrceh0nHIPUY5+vW8pwnirzmAQ1Lmla300ZD13roU1g3Ucromrv1YTcY5OfDMkGpLeHNyaF+QYJydaKFCWwf088d1VrHQy3qYSx2JqBPGMP3BuI4XviVxwPjBBefLFLP1+JXqKJh2l3wbkcgDcIJ2Fia4isl18dTFG8tMmoDCCs60cimvPScu/NHf090OrcUrE+Y5DxggQ4IkzW9ZkL5irZr6pri8GOBsxISFjVKcbjZ2fH6VhS3WD4SAZgc62PegRLpI5m9dBBSBPRwsmz3uRfyPBl8aUCLhxfIsQLO74uzxi5EyXnRsJzHuSJC9mjBZAzqp0WY5tWrX/j32n8GyUZdBb63vNN+R/2YA9oQEuVNnb/NzFYjadXZaqkT9Kxv53Dc80THUwwmaIryEXSpO4dDR9GmbY9H8cyrW8os1baI3eISmyraNjK822xsJPSr8impP7FDOD8+AtE5LAAbmRX1u6yIjUwPmE0nosvPYZH9dSY8LDWPgHRSY6gFFfvm9skQgGvGyMlsmFPSeN6tjdH2V6G+yonCE+G2qQoZSgB4XWFin3lT0WNv3ZVnUzEr+IQD6DL6LNRAynZR/bS4x1IRNFXT/og8oRq3Yf80OLo+y+LBadN/puNVZeNbF+JucG0WOZrmLZk25mSp6ASsqmor3+AxAnh26jzGwgG9ti7y+P1nPmX3dFAY7hayTi2DHUgdYtz8CyUFmTMNkp7tuOs+grzs21tnSKH5SdR5bZ/9INdtEJ4ShzJfAlF0ERe/ORfbqp5noH7TOfzW92cgyWenfMhTa85TqqT8qCA5BCdU88xBHuz+ICgGS6e62bFR50WQ3mCgpfj0UCwYifkE6QlD8/pFJJSFJH5DLhb++uLkX2sy3pnJgnborx8xEA6ixAHRYqkTQRI4tKWuoga8Z1ZzVUSugdStqt7Ih/iDA+SX1rb8M54o/Tf1ZdoB8P97vBuKvhFxj1iesxp1wFQ4fgBXJzeVFUaCddD6OBm61JjXWdcYlPZR2q7bSdi0uJ8eBL3ppcTnBfgR6WH8tSVNTWa4rWvnjmGdxom5s/a13NK81dLFiweT041gFbvI3Un3mO87WwoVYSM0/nMkSrWP6QDZIN0X6LC+XI+HyqsTd8EPBB8MMxaljG4rPSVb/0N7LzM8pir707v/NpQJeeDicoydA6PUWBKqYQtThQTBK8ArkD930MkIXRJCqTiiamvc9YJkt7qjFFDuZl2ROvzzZwkBK/Iz/JkOWDXr2A9YkeeCQXj5ew4c4rrIFdceEKpEYjcGCHHGtK5tCkspZ99e4L5TOrYeQZVKyrugtWAXj9mBS4SjTIbrHf6D1RSz5+TzWdHwR3eKDfYxNyLkBsBiaIpZ90pzffWFuq0LdegCHBk+nZurp71djVK0OXx+4hMty0OGskdcf4i+Z6sN31Gxa1jE5Tal/7f2vTI8nvGcwSWzeZJv1lSVTAFi7xThieUcYWHo9JRvKZbJ3At+hCLLKSd6glsxUNQfdul89273hW1USKdwm8qcZlTZw53cUrey/4cXV4i0LidehrDY6nhcRjgGEP2HhDE+P5lUVKVpGHEaBd3wDm7xAzlM518lHJJ72wLupokQx5fAjK0pcyuolsyKloU/lwOGT8ELNoAjvGvlJ556U9k4HMhKb1IEDACdkWWFzy6Ta9lfUIATG10RSV6VBhgfKQXt68dx55DsvMhMxRybI7e+7hnw92mHHlZjcDrxW0R+VImL/+0YtUe5v62+s/R0RmyMBV4/AK5S9O/6DGDPhFXzXT0RjVkGFJN81J1tQn63iGF0VpqNWIoirvj1wClZB5vMbu06oAHrPXOtJkDrWVnYh/twM3qsfjW0l6Il+qN9h1v5ZS6VBiPxB2ZO1SnksH6915xL+m/sLvqCRs/uL5yFrIkQaqssLW+nm+OGYO2py7jgjoQbL+BcCW/hfiL9MuMXUOXGjfIJhZM3aXAGvT4XwZJyIPyijuKC8Tj4BPk01VQdHXfX00hXgc9ck1TGwYKoKj93/vSXg/RvL5eYDLtOIt89mH+Kn8cSoNARcoDQxkFmvy+N9hfaZEMbDm/DFKJSsmjMbLS+DdoqRHSYFbW9m+6Tv0qAHBHZNnECiJYIOkdTYdNyVL0NdZRUSqIl4a3wD+YQ9FlJ2AibR0x3vFNthQPtwMTVQwwYbL1/+zHSnOM4ws/A2SzGfS+mVFEh2ZOZzKeUwzGNK4WgsCcY6byRpkZ25kDQHXxCcsL9kIo8+lAnQwX0617QPBm0dCBUukT+6M7hcIgbjXAyX6I1MKDBM6UNg27s13Llszmx9cAnMcPcfpakv/ANeuz9Evzjye0Os9iQ7r7lKNx8nIcXpO9DZI+WVVWEikoEGUG9m4lS6oBLYfi+mcBK4vNZPT6nIPo5/+vO994GZ17vYpN7wk661TctCZ1cRPAj2jVRH1bY8C3R/oP2VBULAtUf17RSpBnOOlJ6hiYH2on9Yy6HpPXhIGvd5j/xSDLwTTFp/RbGqagB1M9EWbDkq6IrrbY/JAaKyfZ8toyhZIsmliQSjqfUrMQeCZf/OykI/2SgF0CWA3S2HTyZvx30cmPwL4upPGFS0KVcnv2Fp2AyF9Y/ROOTmKwR3xtZwPRHDXcFLIAHO/04ED1M/Mr+ZZKnSsytdgGUvyREOLYvEv2apIJnwN329HtuuCmMr/9Fmv2JpRtCljrbND5RFHGgyfbOtDLXmPZvcx1rSJ7qLKK83whuFEvhL2oTFv5mEP1QkElp8oveIv6/PYTtXGQX30GkFlKVzD/D3hrCx+mSt1bL/DUZDpAfcWwnIfU551muiQCfRDbzd3fhT3evM5X9Ko8Cb1jDdbWO4Qo3Mc6cBrMrNMZVOykJLVlectdFLsKS/cGFGByOYBOMq+m908oC4MNvFqIt1CL66py6JDhNjZOveN3ysh2cLXwFJIxmsrA9qUxWaoYg8XaSCED5+wVNf+0s0vUfTT35D1IGT+YUn+KDrY/XRYb7/GD5j8wC4fckoAvBrZJIk156B0CzsMyEOlOUqbwh4NVp/TV+Hcv6Qa/QSQATTKq5gVOQoYk3vHiKD/ub1eDBK8oiz4zHmrgoEaK71ueiriTj0blrbWQpPhiKsRBWoG1D5IIxSPPRqIY/icqE2OYp5s8o6fwOqTmyUebCT3NM1jyOhyRyjT4fthnnpiYDvvZSWl2Z5vgiEvRZ1sWUviP6DGOmwoVkVzpIevW6lOhwI/QTPZjwnRiQkool/0trRxyAgj8Z3Th+A70SOLYP60rr51pLb5jA94gJl8MQPGklBle0vP2Esx05o449K8px5N8t3Sdm4DoIwKy9t26/PSd9sNhwEEq/28jXub8tXhAGOWKARkxBwl1hFL6lqaylsFDqTjh3rCho3q+N1oTW2FPz9PZHai64TfI4ZwT1AjDrB23+AESE5rJe7YQ/bWAVw2ZqcKkpWVu4jMrQ56vik7uTvZc1C1dyJeAybJ6w0aAhZG5FZdmmRx2OhrBMKSIQD9rKdEsMT1UpKF+THqQoYBg/rCjRewETH44+6fgYFbhMGHJZN/Ku6IfIeFUv+9OWZ3c5TuHNR3T4oxJqhzUZPLs9x6pIm/wNMa6TTXc8Jehxq4N3pLJ+hPA3PVkU9Tc9eQEipJheZycRSizjGhtHRtzfwO5pFqQPlMKLpLpjThMEqU3QxWJQuvxZ7gj8lc6QU9AZsSpr9lc/cIQnw4WRZKO0XQlyKgSaCRFvqHFa1CNvl4A+paC9bI+/l/YMaCNP94pON1R0CS7Lmfl2kj47cNtPJInAGRFTNymq+eFEnV06rgKwIggZVqgq1cNjQwzaP6QCaQ6HtG8c/yn8k6c58FAbZf9yyR5RjWFZXuIL4hKAUZw+Y6RsRTv/J6sxqfsBJrVXd2ldLipcgrPKMg1zX0WD2jHBiIoTbqYitsKyZYHuVBtnsODmzWDUqY7yx4vcRtIaSCUbz99dI3UqkNd/CJ1H2ejVG8I+m4LXXNeBNQQz7bpl86R/DKI9xGJIkXBFqpmcxCpNCIg0rq/b2RRPqzf7q2thH9ONd+WF6S2Et5giUthidcLsECocLkrDVpFz136jq3I6+QeoWfPKLwN+c+LjnY46DX630D1dEfeny31pwT2o9RXFp+EAt9hkZnGiAYL85ymC7T+X15lqGES5NZWQFgpDPTCxBPuZ4mw9M4czGbVClsNhLB/InRLgJRwK3/IHI/xUMvE5NEbCjNEK4ypc2e/z+1ixLkbwJwHZV2ffqVu3p0uPr5hFhUmfcHiHRVYwTv25Pcz1oDIj7DlOL/IHbi2xOxiNrcQiKf+EvZj4dhRhb8xi2FpMe43PkxNWBUVMz+N4Xxw6PHtqhEqhIXgPIA2T3ACbpN1z5D31FQcoArmXptvI3ehnIE1UisiHkXMD9NhMH/gfwjNM3ziHW1O3WNHAJohp9hsiq9IpSbOCIMYHXjFMVqxUT6JKtTcOvKHqYl/X08JNdpWXMO4H3oN8eNMBn6ky7/ZqRMU9Z/AzjJlDUPXsxzij9dGrzKTrNx9603oqSA08SDVfIbSos6G+6MnDGBKjWbo7l0Jl8Jn+1WgRO/LmBwpg+t0VnKSNzzDnk0Uz4ix1BZ6ZFD079KFB1ov5hC704waHR84kI7duphWJnQQfmFeDeuu7SJeq8d6vk+pPZAcntAUFERvnJPFtrikCkfVeMZC6YVFddKctmpB7nDdxnCC0XmP5HdLEC68P3c4ZrkcNPZTJdAr/MwZ9wCcdPSrWRcA+vpZWyuVGeqKmgz2WnDWCDwrrRNzP3nn6BZzU0HfcInPglnQB28rYBurPsVdbC6mLJNLOjO/EaPFtaIjczIf/ceobLBpyzuHLmVcJaGak69uS1uExxVfwloZU25ixWfvNuclw3VgP6p3LoI0dnGPY3kXsohKCSLh5DdvtwYn9YJgQojejzpMSG7DU1jroWzEruhwQPLBx6BNQ86HYYsB9ss3AUvLSVcWX2ViOMBRPORT0yu3vTm4awl/7kLokqkdQn707Fc6mHplB/fpwBOEQTjRXePlXzsfLhoiyY7qn+8hVsAuiIIsUqyEZOX2n4pW0VsDipfPa9qryVGW7tBQ2FU7DPF2I00EtU3CVuhv+eYS9vs/GdLcNeMlb+FfB6QRWWO4jRVsC+EEzibfqyWVZJ6D1+Ifav/k22d7lAKKFYzUfXHsDEi1hVs9Z4/PwGEHmR3sWkDd/YlKovEa1n54tNCsv1MVuZ/obhhWHtUMeD/oa7J/tyAOvJzwDmPcd3w9MfXjtZPyQ3hFE1V8EqVgzq//wNUmYzibJ+9o2Nk6G7EvFzkBNhsu89SqM698r7xALKpY7UVd9WyIDbS1B/V5LKnNI+ZvhIZn9PFhPCETkvMPuKHOGBXxN/9Pa1poVNaGBily+6ZxCLfxP/hptPCAG3IPADgMpm0TniF9RTXeF30OLSqH4vjvlGUUVqvKPA3HxKWz+WJJocg4UFEM1lDmE9pWTt/UcAuPoJwvlyOsd8qa2s/A1hDtXXP8lsTPAFdMl/o3UgnxOfsXNgL3eJ+RZ1P0Um8aXkjDGLM/TBrE/HIMp4xM9MD23f30BvZmxpCvjesSq0LxYjpF2wuHDdYywpFd72pLloGp+Whwkc0UBBos26yIctyVb1e+c3kYk71OW5lnr8FZm5hnElzsZo9ta6xsIHnKTsJSJUM9AYHNSIeY4lk7SzQaF+27vRogggW2Q764cPxlrlIuucg6+Q+oKWYGT+gPQC4qOCC0dx+hHt2TyERUcRFAIErQARB40NhwrgUtQ/yFQRprYla9PMRygOF8DsP5U/vEwjOJFaZWi7htC+hJfHjVnhGR56CIjMW/fd15Hyxs6KPRiW9wzIl1ZG9RXpvOmGyQca/yAHU+0z5PvpLaH7YdCNNXJgKAup1V1JH8tEqFVqWS6UvBTX65CybZxKCsOy/6FISlJfa8hTHg/Al6ooaK3+DBFSkK2H707wWDCzXFxt0WWgZHyhGl+fT/U5IBAJYXdzK4RnmPDj400mV0+5ZX/bVUlmb2dq6QVLa+HIdfZ3/lI8oCOiaAvVFXaBpwwjVXVn+QCb0kgVq+HdOCpu3NHpN+0kMHzgCFj4V6l0dX4jqslHZBOcwvgJFVx1t2gEGioXe/1TuElo81PEFRcGXIK2AfJUmD0slO0MNd4sQhGHY0fd29F3u13ujJ1aRU2DY8QwFbyhC7Fwjl4Rn+/E0JPJ6cB3S5e8CIJ+bDyJQd9ouINzwLiWRzirZhEmJSxzAOHsw9cU5g6AeCttJEOmTuTGSt/4Fm5bHcxNIzFpfJRtsUXBoH0SOq6xb7v2nUMz/ZmjTLTolQqfqvyPORfGIs52f32YiBDqMfu6duEVw84WQXMKrr8cEjmRfvyM4qNggRrWW3w72p5S5CXaA6cxIvOG5zL5EfKfVFijBSQR1jZfX5XxqoRmslpVi0QLASMFuoudR5S8Nufom3ovTDZFFVyWegGqhe6HtqJ0UxoowAZpK1BmptAFeckbCjQ/cb+82zB1JjHzikv0HVhQlVmUXVeUr5FTAKET9csxcZnxogpkT7XzuyZh3VO+09obIWcAmE/9A8EMcvb7l0b4gToPrn1KF/9YGK5+ypIiKJBgMomIlOQHy6cHfY8ZlB5BmQdAfXxNPzcef0nrI8dwxIJWZgIW2UZWh5L1GV9rjuNFWiYPgFDD5t5Hb7OgOG5P4H7GjV4DUhgbSytMrMkB72H0CwB0n9chUOv6dfIxHDalYZhFQeRzMGvzpwE7b95xCPNhnxBIi8rtMqFqexPPtG6w70pCRq+HlIqowN61OUnmjyjVX1k7lVYM8x/p+8NuIAGYLr+/a2GcbImY683fDbhU/W/eaPcFMlegwkM0qKVJeR89NNVY5XTJ+1F/Ey7AL9dKuC2IW2ypLkt3Jds0RaVF1q4PZ3N3WdHgmlMaTj+Bv0dR81f40lnT5vevB7LCIdgP8tfnPlFMzERK2ZNehPnS4/DXBIDQ9fDeEwb6FbJKcagUxZTNWecEhCC/8+BnqZLg2HW1xQNhnRVDNQPH7iJenQTcWAAfEgi96FAgxrDGbNDTL/ffF+EDwQuI3CAXsr5QdDkj9KAW0hBegMjbYn4esmrfQ+JES33KzASwxiLkWxNPga3+kljVuVa9umfyEsYQao83a9P8s6sIKoDeDg67l5j4gQ2zZQqXjVGFlnTqZvxBgrtfDrMBqf+ma0s4IVAinumpxa8udIdO8BncTbA2HrHtlgcxKT9fjWo+OEZ3Rv4NivUNVa2NllnbtlhOejUjQhG9F69zB8ETnUvXvJerzLk/ePjZvEep9Nkk0DSWnhg3bKmnzp4kXtR6fvVUPzLtnuK/u9jBTTmNqpjc9Olv5uZIpxz5vRqeWEqhhoRwDvFxwFexR+MO66NM6IlB+EEbrmheU8JeOzjqU/GeQnOesIO/dIyI8vElUbSQ/fjbdmYwo5ilVvOk49zhMMzICSItX9L8/W2wcf05KNbFLYmflAZf6D0Ep7/4zIIzbMWv+BvlBUDe9Rv72pFipak3eGs2kBZsqO5XrASMhfjyxdPYVh9Gd4lp1vKaa4lo9QvrMIwVJ+jcQJ7pmOAg0tnw1Al3/dUI+iQmJ830Wu4eg9oQquyVxTESuPvxoeQzQkJ+F9oAm3RNCRYo4ivV1JRhxddQT6PiUZRxqE8eyWNKUjhLTWKzVQ9As9gcGW/t2XJp7tfdRuafOYobkkY2CjIOn0ccyai81W/r+eJofzmLxxZDHROimcqoj/zmM0P0b8eDQ8xchxbt9Y+N9P+poTIzKqJeff0bvaW2dDqut5XMV7jhjer+7cYSMZ2NlYd7W8hywAOoIEtDg0O2o/t+f4YB1JyP0JVVaTZ5lV8/Yqysmx33j27Of2MJE/szPY/xgfSKD6rB8DMblanOZCzITF73FfhAx1Mjkv6QbKXIO4cjH/nDfwYolaOnuF033Xnqia8/5pwpj7pXbmUASVNMxRmltcMrVQX29i137XaRfF6yL0JI1olc5CELzCrdW2kR0+ppkbR94a07YoKqTc2apAw6/H4bCf2YKg5NOIF4UV3rR8y2/I1MoiukiGIZAYYOAMyz0gM3II3DDrhMW6M3/+XpGeL3AOp6T9yxi9/cQiDWgb6O/JVbD0VdH8T+RXbejSmhMl98pnjRw0t6Dx7OpD+VPwfubYt5wdt+vlDRiB4XYkondSEOjuJHdy6I3ucg0v7GMYtid0V/6LMqVGyax3ibV4ukQT3dl/El+Pf5qVoMqWFC5tyIgBlJTc8ZKLzOIdzY49yfujefUC0Qx6dno/JrJZatm79Z/aWh3z/v93s0zXzwj7iTExpz4gcoO1ou00PJL5j7nSNTJC3kTAjzX4z8K61+uJ/uLsnflSbxxzOOgHkkl1NiRZmI9NwmRyS8hGMJ9uwQZGb6l0rWkTbxwq5jHcs+9pFwc1Cg6Avw0baVQqv7m4zugXxT4DWy7y/PC0tsbuvqv74R+UY8D/Yb80Pi/6Y2EX+zxjeuoJ2rl9TpU4/XJ8XbtmoD4G1ZEVues/tG39/ABgZ3mJtrDnpzq5cwRkhBeHkn4TbE6phB2/GEKLouM98N0LOtA9jkhaLT53kPgPYMEFmmsAc8ISk1hbHC815ClZUfueOUU1JNbmKazGGVX0JTcXdRnl8TrUueCcTExm54tKIazr2jJcJ9NWzh8XPx5Ktp/OouLlq87GqtEjDuj+W8/2mtx1C/ou/AVtRbBJno3zSubu2Pk0BHtJyrq35rzf+bsa0XmBu1PzfP5uY48X0p1bCr+pSUHZrAJBEj5vpshwSEvCSXFnuUj2JhGGpSnLHJEhXyVTdpk5ivI5erDiVEc0B40ey31814+/3xwdKiT6R0ihH/c/wlQqnbm36FIY94og0AHdVDAO16t2QS1o38FpOxo7vvn4yrgpJgMFFAu2YDSklDqN1IcsNtFLgal4sFg/AN1NyBLb8vv5ySLkQCHMsAJqiKNj22X3xgh2i5I3tADeSdeh6JMx1RimEo+6qQFvctHZQajCWfElW/txk5POLl0P1higZzXKJ/8mUZ/STJ3ec5+cDhr+VGl9hO3ubyTDSil0UTrtu+9EzzvbHkN9eVRk1XgETE0EvB2/D5m9eAhwS1X7tcH4aKlM6utRemyFMP+Nsv5rsJnY50EljwkYLNNJEQfA0Nh6EjObtApHpAXgdH6BPSmY+ToVvMb539775AYJKs26l/3Q3K25Ao2W0T3k1gkYDlUbbngNHaiB5zXr0eEpNnlWMqdeomvG7j4nvjLlumz/wf+R91jftUM7kPONbCVqAy5Ed19Af2nh14OD7L/bkSxAXCV5+hqUZcXPU2FjASvm3teLtxUtLH4SvzHpqyRnrCBMWTq2wO8TEeDICL4FRiF4EVBPuH+BN92B3eMVbA7mqXD+Tfr/iyzbx1wjXPdO7F6R6ePFbVJtONOIZALhDoQPqTkQkaeEpo6dH6CJe6Unvdm7w2Wnx6AE5otdnCXqHmFky+x4E6LW3ID2IU83VFEkVDYBeaJhl8oU46CIjxf1UDhO0lSBvGuQlJtkHHaUGex3ii1tGvPVXvQNF4g6iiv4dnuqblNCjMr41uPTuYsg7PJpqP0HLotDIjgn7Zv1uYgHnvcIMEcMCAKkUdNg5MaeFi0dRfk5+Saq+rvD/mY2qZsFTBZvS48UKkRpdq7xjHNGJVcrFz83M1mZKcA2NjQTPFV66hvJFcB3R/+s/Q9nSU7L6DYitvZ6Vov6QQZfeKl9NOeHwb+7XrEe13THEhvOqI4dH16yqD0mHo7K+M7GWks6WRh3IkbvFaI6okufI5uNMIQcT7ZlKlRZ8PA6TgOQGoNWAXzQxo+Ung0fzUOEFyPYncA3Jh5MYmzUriWurGJnQHu7bjBHKhX75ErtvpLh0WM9JWSNsDLMi5JuSn7DfwVTwIINPXOcrS/z3QwZyMbb6U4gIcY5hefhbdXgatfe3iCnWZUGw/nr74YT6QG0Aa3QG0zjXtkCIMAKiI5mfyKabRv4TKLb3kLTC8cRbrjWt4JB8yT0ZWLiHcjFqse328G2crsK4EKGsqto5Wr6fw06VC0Pg2VPVxVSZeG/tzfDzblKVZVeYppu0GJTrk+POKGDxBVC8k3TUVdtByQ38eu9h8BNWtKat8t155Yhko83A5Pp7ZlwKDscwigkMPpeMrcQz8YXOpY5/NlBzW0EX9zO/xdvHhL/Mi62GH0bSgUu7CKqkQniU+0CaxSfu6NjzYg8VnnbakXhBnCLumX8DPsBA6YNtDPUbqCbqa9YeqXBsGN4felTrNEqrewdNXWv3Ez/UsX37JzvKqTVmmO021eWpUyPUfvvwKuRjruKr9uv4Yvn7QFXPiQ/LeX9qx5uRfRyL75VzO14s7wcISWbAiLGk7hm0ndrdOpjSAZA/nbdKNApKxjUM2Pf/7sMJuxIfthM4WGTbsgRiea1+4rBre36+fcYTRDkvaNzZxLS9KE9CheP8x/PV6HMSIt0ckvt8jiwqXFFNQF1gsAFSx95cWN1BssebnX6+dYqZ9zCAmJiYL+lZWqW679rdfO1HJlBqSv+MJuCT6Ot5qoS2Z2c2t4wts/yxQv0YPrNVdzqasTFONWUAlJ2vxSzwrCTHZd5LGZ9S9G6Uo00H75ujllfmN+GS0WXSvtdntyctnESq01oO7fzLt5U1x3zc0Ysh8CoqOLZLk8UMJr0l7mFyozH+zuvv+ueDYNSx4YGvthVjGtV1fFI+oCRbOl4oP8077yQ9d769Dbmqmt4ZDQ0HfJ8cVzVDw0/a/BExIcwzs67ii9JRSfNO/8bWtA0/VVmQxu3oc35ureCN6ctZveN1mgYDhipOmaWgTmQzPiMyM5ozqPc7GME/UOPLdPEHv7YtNfk4qESeBBRedRVSWl81QZ5q8Ia7kUuKAItG3vlzXfNV5F4g2dD4yM4/6oe9CD/cwh0SuxJgX7ouTjVsh06cbhbFLci99ygNBM2+Ch16Qudh/vj6pWLrpfXNOIGbNVXjM8th2oupOacnDADZBfbMMeTfNfPSAMIFBY6/p/Rs49gIafuNKxlcRUUEMGxOcuqtSkl7KlIzB89YZg0J7VlU/D0G2EeTAp2a/lAi4K3diUVZjePgF2lLcuvjGonAUnz30M3wAE+XAPjInobnmCNSy9YRSjn8kWBheIB6kZ7LhChdF+Ko6aPgkIjd3KIgIpiCzAtyMGtSyIG/CUlc+IENbhp/jxcUGOwx4xHGqYyuYZozLn6nEz2dtASX6EK7wsubfkKtF13BJ7/obj/4VDm0v9A+ByoK7B6i7kPL6+5Ld4y9N3h7sKETs9yS4Dkg/416IrKkfKsfR/J9CwRjeFqiSZhOndGladYVsiQtpgV0G6Khhiz6yyNC0jjEt/ilfpHBMLqwvTjoLKZl2j+KKX8pm7ZjLczwI65VGE/E3VR9pFBRYhp970KGX0wIkuMAN50NixvkcbX0OPWpQWbqaIbPIkLNSEJk4rSiE7kdLpa7pX7aUdbag3NQacRIqf6VRiekq+8N2ZW9wAcFfmMXILOtZdbRTae9fGcyBtgFyhme+7E/STxJRqgBpifk+dbjG/1S0kTL71o8cSqk/V8gQSvnaiF7mmeG9SBxCe5QoPnnJVjTlbzjwMbz8HkvNpPM2/6nry7r6D40+hS665HoHxhfqnUJ0NiCvudenPNuz8bu7dJnid4w/QhBHOIe+rL12n9gZYaTAHxGpL5DrwO3P645ODYa/ElMWK9jPuUh6SPHyl7A3YJoMK6LYRv0coYKFVQWnIrw9j5vGj6qiwtTJx0wVw4KKXU5EhV4Rx5Zx7A/3UNOjUqTWWi/i0YwWHaTgJHBi7Z4lDQdNMZviq0el8AQ3WDtCQj99XeRdoYoZVsvJPPFOv4Y2lD4rzkRbup0AUzzAcCpLuzsmUpquskQJer3FuNfYif5kFPVXLun0tMGig0HCGpT3xDnejfcJ/1rqkSoTvgsz3INyP4MtQEJNY46RFgq+TwBlui/nVfKCc/vFT2rXd4XZXAr3EWFXwfCsJfdDiGrb2dX8wpBiZEBmDoq2v1t3GYj3gGAe7W3yVNQxeG7dQmVle6AOu/Uxbpr6hoXBtPG5TaeDQ4LeoJ7/pSHUkhbBO/AECQ5btF1V5MwOYPseCtVubrbvbwfgF+26Y6UYfLOzd9e0WuXQ58qQF/NFQJh6AYYkjaqMbodpTsyhFZV1t1N/hJO0sRNuJknUjKaCbmEW9cfdZ/G5QpQScwp+lD8+BRRcLlHIx9Dp1BD7E3QmGeDhtU/+jN7yuGTAqkF+StHgT1FRTVilX4C5jLCUp8+gk1XrN2ZbRye3NyMXO0f1yF++xz6AImrpN9Ab9YiIJb4z8MwlM6vR32z1kIABE0pKiM7O+TZF9krcPtnySIUlNznlU5WbsiAEfDtZ/9ppOqhVPLkQp0SLpx2zO8denK1lFBDksn2UVmXQFZs9PXxxQ2n7aXT5GOhP0jXDbUWXk9TYaHtF7yu7s4nx5ejLtKCMcZX1MdpBEiQuqEHxt+7/b8+kFsd9uYUG0BzoiduAEQdS2A55LtD7t6e+HCf//TrrJ7YxUEpiz2v1R2PSFgMl3Kl9fdXg4DLt2VUkJKjFRhh0SrmWSHb4BffUXlrGWsuqeEZzkIDCP3Ho79+ZatGk8aW+8qmry6sa/fTSktzvZ8C8Xm/RN2L9hbaTxZsf8Dnc0K3QsPmVGs5DmXu5UD+Eh7EouXcqKRF0S8KNTnU/1yUqBrcp5uv2r92zD5YiRyUEyRyY6Vb8XTQbefg1ybvfFBtBmPLn4BcWF+tEMTvEJvuBMrY+g1UavkwyiqeCCFqKycTHKF/QJDVbelHn4J1kNAAuZKMOciSzhIzccI3GB5hSCim+jDEUv0zs0xN/JW6/BK0kZz911J7TNA7gQoYR/7OataIbfD/VaTU3rS0GOjG+fvtnMVaq1Eak77hUxiJUBeF3uDfv72j7YMt0bnb5s/Q9OYHiB+2QFqlJPKcCoh6pFFLLpNQE7xufoi/8dm/nPEnENbzWf1fSypJHMleiJLBUDcm7pPwuZqGFQinwAVQc5Q78qXhaGGVEJcpYaNuSoFEUol2BkFs+Up95B1wEZAqwTQqirCyuGmhQSJVmMdLU/pdnbeMUjoWxRbGB8JY+jKz+a1FSFgOkc8HrGAUJuSb0csYKBzpZgik0Xv04i+c5rvvjqEb86Eb4hW1/6Z2zwbyihPoIFUorymloFpnhJttlbvn2yUMIrdA1UzH9PX6f8chdhip5+rAQGPNFPpEqvFp5uE6wnuErF3+AsyPmjIdTc29HGN+L6iwh5OOEWZOO8UjFbLskmlLlTSrzf0yINxGfB8HHTmcUmujwpJIwJ2lS30OEJJZiKs5tRJmj0KVLgFgCnrv2hmCAuh3DJPEiJbnX7wAiDX74iTZySD91/MXOunAHj+KITqGd7NhGVG3ep9I+MvnBkSjsD1cmv6GVglOmPVoYYr2qTIGGW3rE5YkTS7MGzSm7uKTn9lUVXPj25DGCsMfrwJS+9tINJCDYv6t7QLuCa0r8XjndgsDmes+tZ4pIPyAAJD2Wvt4pj0b9OsdEWS78XLPW/bsVITnf93ejB+BlpHGEwLzr73fbYgpdXvL/vtoR9hDYWtwMhzTAxdclYCQhqQ1iaH3K8pQ7dr+G3gM47I1U/DXevfvnefIm5HXcFaa5qCcp4Q71TLvvbtt1qlJdDjQiYZ5nJkMJAoWGmvg5RgYMVpEq1/ZrbW8805VhTmxYpmk+W+n16Ikf6eAe7V5BtVQ/mnz/EgU1S7jzjRdEf2PPjO5MakLZraIN1IR0pBHlf1AUNoU6OG2A4v5qtCKA/3qeXvT4Ih9R/7vfQ5TCLOmCy1UcA4QUyglLu5xx/wpBPD/2RlymhL7qOqyfei4mDdHNDq6xs2RvPBc7kQm077dVF2/72RrWtZrfFrwHuJuOKJE6PU8LUkIvE2IjfwQJ+h8itYkDzhXXfwnEeD2uusZ/H0cgrOWNHAX2eolFyXtsEadOcHOM0EA147xCTBH0QHsCBc0jiX/rqXNJAcMoBuYn2sEZ2uUNxDlY547jUme+krp9u+nX2ZOkhpWia9kQtbWTxQVJdIdLv6wYXapz/CG6iKpSx3poGREhsHi2YvyPbtEnqGZnMQl1kfgJyGQcXI7A495A1f2uxekxQoLtCy8+iyKkEPD48ph01TbgdKEUFIpl1g4SdecXWkTq5WaC7Q6ZBxIOUNu37jX2CLMu9XAHuFmHgv6k5BB2uuobiDIofjd3j2hFIRQceL73Bz0boqo9+p4avc9D1DR91KgYDMg+CKpulUX8tH2T6L/X1ASxjN3R1xztW91kvdT6bL3PSR2UNgNjur83Dg6MlSXYoyCyirysFL5UitymGtNEhw39hS9QIz8jCs16SrK++BTgX6yhQn094JQ5gr/S+AoqRbEAMCI05y3vUqWdJF5yn7RBwatgEWfyg1R2wn2Rv6Lvx8Dww8KP3oJB9NTtjQr8JoWBwuBT6OrXzSTmD1TQyMxo68J+186QUj2kiNdi11TJwlI47uUEcdpVJj7V3CXIWd7HPvsxli5niCnbMt27OkfXf9dP+WvvoKcucoqmcf5wkHgbL8rH/iwF3SN0kNJdWZf23wVvW419UKXK8jejLi+EH+7OyB7dZ9cA2FWXBRo6+rGxQg6T+YsKUqHO58M4AAMg7X7/KwQ4OXSrP9WHMNixrFOrlOoNFT77esp0kQap0o1Sai96f7fQc+Myb4YfuFDFACGG8IbUCX0jbAIaHoskVLHB/FGvXRSD+nPmHYO6I2f9jNjmGi8Bz527gzhH5JiX0yPpu71gmHp/9w6n1axoJWmSI21lrMj73Vo7pFgq4XBLixFGDjrtYPsLzY8mznFSIeZnjlSV3O0KdVb0d89X8MZb+7VDe2W7A3yaqI1XY830HhcNkXpSVnfmulbYhJTZL3cJ06CqOEadw4+faJSd/B+c0xxPVZ/ipzTzpNCApS8Osf084hFGqC+0nz5X7LCSXxaT92WOPrJN3H67URXiLw9ldYmQFDO+X6wS8zeudGkdLOvkEcuY+YeAzS6+G7wbfcPAI6PYUkeiQ9dThww7x6CjrlCiR6QFzdvTt1zmh+BkHSj8ntAmbI5xbcclS8/sRx/SMzDYCv/ivqZpYZ34bkfNi3/ORUP3quNkb2gMrPSvyF4+lrL1gXvKcq241F1gREOBsUUoDTTTxG392E/e6qozGEdzX3PfTNECO8IRxKhk/q5yWbwI6FTE9cX8eZllm8wQvPbUO/orZ+nZ3QthbvavFsi/e9Tup9QaETlAyGUbhS8BjqCSX238aMeJYriFRHbN0nSX9BJTw+77qT9pvPq9v8SWsW113Z0ClEaVh5Qscn+yQAVNhuPefwWhGvo2U08EtnqjJH1s9Zb2gDDjSnb/hqLXpBVggFge76OgneymU+qDJkSqTd0oUJ+I+XHksIQDDJ7fkHh94vfMzz3QFSBj10iNh7nJxA+bxS4pMzO387lI6Zxj14ZISzj5sItol+S2osHz2rMohYErQnwwCwBVdHetTfoNrif/XbWrfqMmtqmSBqtBNqZ7Gc79ScUO9O4DMbv2VXWA6S1zr+UpoWDukuqNE9JNNLfJPGfCxzX3kk998NQkPmuqxkzUzJj7cxjM4uunlqhqu4inxZVcB7ob3A4qb2uJnTeNTuqM1HGxU3J2SdRAVh32leX8IOqku+/8RwsgEXlCfz1BVhn0GGSR/HbnUyvxW3QGSb10YzDDsrUCuM0C31hI8hTo69I/I5LRAewMKPw+cvas8nqTxhJKHPolaEiFnsD4koNz1Na3/6kd5YzTllFfXFidLjKX5A6xsVaLtH/OAMLmK0Glwb7TPIP78VI62rs4MWd87FQooMiTnV1JAv7oAB6457WTnsFF2V9lQ0e7gdWH8U1Su6rgvaDO3gLSh/QDjkq8BQxPzPCsX79Y0HsSREAvUNP626A6HCmajhOaCVgBjwaBcaTw/RMp59rCeRkdufB/YdPimsKOXm0fFkfCT3GSmyoqH41zCUFUVzHcP+peg+szWsuDrUW2N5AWCE0e0MKM55fxbh5Wcc91To6CBxDxFFM2Hdp2XybVQUycZgu6lBKYA8QSApVTzs+FNsGKyu74RE7z/usa0rjEytg7FtvNtJzNvroea0NGPscdhPt2nONotsqAiFmoGRkqVgyt3r5+q5cqSJM8luPROe95fh4IullQKZdGL4kqgVIFnzXEpxXv+WImx5y1dOMUkzgduBB0+ZL27e8mDCAdQCCS5dfSCzc6a5KpfwP7lRsVpcY9fX4CKbqFGcDYHKqJzlC3j/D5re1nKRoTw3UvxvJz32rCaN81agUBwaBicuY8RJ6hM1bnF76+DjLrbUnLSIPWqnsIYF97AMPd2A9OkkbPB13NvP0/YOkvtYH3oTOcO4e0RTqcPKuw3rTPAucGX0FSmGAi+scLq6VmCx61Bzf3lc6P5WUhaaUJAArK27baf5YU48XWgCE3JYCKKjiBLSEiNpbDI18Sgq+RyS0n0ccIzAJF6uiX/6cFcHz56IQHrQ+eYrU7b1ClzR1SfvRtZ500ciErW1aICwqHYCSQOXRDxWFernIxjPVQgj/OCw14ZDlykqDN6a0j8/ODm+X80Nb70W14AHflSMOvC0e9/MKo+nnD24ccuK39xVoP2sWzu35fup5yK03kBkFqlIb/oaue+VQNQZ3bZ74F3/UpxxDhOZes9P4tjV/yxLR/75dvCOSP5BkYqjGGP/nzshQYP7ZGxuutdK7fDEV8qSTVpF93X8Its7+rWOqDmaTGtsugWI4Ntm+D2JzJTIRNkFg/ZhY3wY5z/t1hxI6VkzOY+hRkN2QfLMvW/b4Iww3/XcJecyEDRehY36sYPvdfs2ouaod97/3hY16O+oeI8sS0YPtmnAZpB4dMJi362apsR1tgQfJIIMlfX+DOZZIW8gpeYSPHly0jggBrMxoQCm6bhwEUEEaiMPAWW9xsgYLdQuMGMq0GELqFlTU5O+oBLC6NP7RbY+pnLba04qZ+29ZRnY0kNLuGLwqXrPgXVX00wls6WybMyjs1bWNp9d/V5TaZP9/NgeNKm1QcSAFWJjDABlU4WSM9WmKBXg58Pk5R78H1JuCpWUNkbuZPsRY2xkpopuKVLj6ADsXwXenievUZTUyxqOuNSi8Zwkd/5SgQnVl//iaHH5Y3Vdbk7wHyWMHUm2E+KN/OFU9hWSxkBumwZAXSzA7Xd10bBAoK3h42ybTxWoZqHdFaP1+9D5Yk4O9igJ0kOqG6zyAGZAIKlZrZ14Jl4jr3RRqHpmNlUDjN6R7J5/HIoyShtARtBWYa+mR1B849iFiXG7lUeJiJhLb4UYkxYFef+lRslh8Scn4B4rwoGR6KLXDzu++qL7192oQFibn1Gf5ukld2THIt1IuCCLSJXUZU/yB4ws5uvMCGMHetl1R00WkDkBGCI6q3MLNuIbZ1lYCyctXWxHGI3oIWnctWgfuBXwrub5UbzCllviahI0Nijvm4zgaTW7as0sLIRY4pM5htka7q5Eo0GMWVtbOIBdafUsDm2CEX3uaGQ3MUaIiYf211ZCM+do6fS9Z7gx5mEK6uPjH/LnXY/HpOrpJiGuM/ZXWV9RJVnCfripF3+wXhMty39sXlypSM8WukEiNwRCM7d8ITQAiJRAc2aLu1O0QQ5mdQF1boXO+vz4lOoiHu/NJLdgwaOyWNDwTmr0s+SOSLZ8u0VqES3Re8Ech/y4sDMw8M+n2wOtjmtG7ctI4mbQR0P+huBr85No5IvMEyNtqwpdhPjlX72nU1RxZ7lyyvCXShEvqluuGQMksqujt2W40zVTz8r3tMi6MNsA81O5ndkHsbtNxhlVk3F2OGnw8wkIkfubyYtpVUBe+lluiIgYI37yvMPiOjyDrNTN70cB/IxO8Alv+ZE6yUmfIxXpOKkAvcYQTvK+er5KmnjG/yCqJLxSbtkgrhBU3HEBUSgHEfhpYamaj9c6u2teq76Hdcwe8wB7O78Ah6Q/LNvgubA1/yULiMzzakkRXYX6RneDHPHzf0Xrr5OcrKqXYJPRbv+RuVVNdwvdkYCS5wVilIbbCGrJzu105a6hCZejb/rkHzA/rFGCAE8K3DyWIQLuP3Z+aooTAgfziFhlyY8rsABZKxM5/ol+5q+FhVFm2MuNYRE6d+uYgq7gb401nx8o1ySZzK4gAirVUvlOUrRjX+1yckWcB9K0gVYdfLB3un8ztzT5fQBsTgT8VMjY4qQnwrbGIz20W9Y7wWxZ2LZfLsvKTFxuVEGkEJ2K8viZV1cQBm+niCfEO7U2ZXy2OMKz1CY4d7ZbyTpXOVXRtmOFHWquJied4YwPIfgOyHz3tselVn/7e791p23EjTRZ+mI865UAW8uYS3JDwB8EYBDxDeA3z6jWSV1JKq1N0zPZqZvZdUa5EJl8j88nf5G9gXOgG4NNmggElPcbtnIbst+mc66PaUMC4rYU4glzp+QElB8DlHBvyltKJR69xA0hUWXyljw+LjU/3pvlqeeY9Shr9oMiAEDIjYv3T3EUOW5xFpL1YrqY3kbA+fVXwT0FoabxI9e9ggjZhtOAXPKcClLFVc7FDCtia1d9bFDNdytfXVQAgD9UVuRr8dbcc5Nkq/+OSNIvE+f2BLNKzTThcVuIlMxGP6eOOuSsWHxWG1I8sIvctvoCnkrOIVN/KiYcmGoYhgdMPiqbJ8E84AmFVTNZoG6BpWO6WEWs36c1w6cCGdKfXozuhgiVhyeyGo2RxRbIbPzfU3BYfsFnYUPmzNl3xeek1NQz4riuZzMKp8j84KbECiOPQKOmYFO6lM4QNzXbxR99d0rwiF+2zOm49ejL1xkGwGM50e1qV3hV2kkkH78XHJK+obNhdo7wewm7Wi456Gxj0qWf4aAM/Pmp7cYttmWekau3me4wlQ51hbiqEQVk5IMW5lTM+AQvUw+xbM3ouTs6LL2iWQ37z+QOuUhRzo8UDMwwXLaaF14Vb7iOZuTCy7b6+vxmx3gyG3GqLjivgTlmuSK8RoNb4AinCD+cegPceNXWoleHhuMipBeAtVtCCAU+NppanCU+QN2jyPd19cNTIhWfLAZuWlKYYHD3eZhqXciwo3dcwzDmXiFeBrnEz7VPeseZuAPQl/Mvgrf+AvT9gtRcW9/JZR1PqacPXS0C7JV3Xg9G72YAZFcewdHXqeG/kmKTd2se3l8Gwq4y45drhg5bdV9wn6Uhz4l2cRtU3d5A4np0bx72uLluIBP+pD2pFdAzOndwpWb7Zl39ol0hIDArP6VgA1OrPM5F3KqoXCUNX0Ws8pNXj2OB8wG8k02U5ZroonVdTKgwpbldoYNrUMy6k4V+fS6JFkr7Srdw0fDAHmRZ297rboJnyfaAHENn6SV6TFAzlV1UASheWV+k46XQQ4Uv2w+HiXs03L4GyUbuckPbtZrFU1csahs2Muo3yUe7ETP/F7fr9hJym/ZndeOlvzIOGifyWKm6FBEvNAUgHK0jlBkyMTE9xJVC4qcW7dF/VMAhZ2H5nobAL2jBdXez4kJCnG5cbFMMY7VUqSr0P15c5wjInOcomM3n70bCEyl+v07o01T9OUy5Z7I11a6uIWsyoW4XRYMTbvcX4tAhUCdLndkpUuP/GqKDSaAJf2+x02LYhkO6W1RjOtn2GRCh4rpXOwZslkYrtL0Hum1TxRvY2xzS6Xp3sOSEtRF7d7JIp69YP2to2RMO0SV1hi5izAkSntxj7g7RIcSNW7wTo/uugnd7yQhWvll4xZocAa1KSgnrP4yecZxfYyWvw2mRxvHitQrsUX8D8X23zTk8F5iq4jYlHHX6I9IFPqdo6lG7OKek9HF19n5o57KpXny8MNc8U9g6dsp7fyUwQoeXq8JxLRKO034yJij4KHKK1u+T4qb72XOZIuJ8rew1RSmn3l6gzAsXuoNiEVXjO7J9raenXiXHJxNMQFcZ6avltOesyaoAioKTHStjRDGzqGfLvkxo4NmiJ7K8XViXv3PB83r53R9/TZajSDG7QccFkwPabpd8Dr2GxARcaXuuL5yDqxIpcb0PcUaPEw0z/02WDu7GGlktfXaxgUJq6OGGMCJvrJ1MXnnh97wUMX8MUldFGUgYdEZ1wczX0Fp1cWKvDSYUUDOXspbeQmaS/Z2PaC9AEXnoy0gUPohWtr5mOZl0aenM0l88RB7v3RyI+GMgMUGnb8RLfhk62l9Qp2D/aW6xyfWiBbc2H3Vh+TiB9FnTrIAuswYujKCN/Ii95U/TCs29kih27fOB9zOb/yqsKK6AUg1D4en/y5HfrUu4kLxy00S1mYn4LlUU/sk9gcRyM1+2wp0/4r7JXHJfk87rW9eabAI4pljnmGHNDc4KHDnAisR2l/D/BysRh0Hs5y4+wNqGk+adYMduuryZ2ivETXTLDUEDDKRHZd7gGiMTuLe3PVx5gY48ZzYR4s/OgZctQOKsCHFh9MxT5Z4oFP/eb3i9a+pfSxcuWylGZ+M6rghd+fc5ol0ORdmvp1o0NmOi4iK4OjiSh2XylNYg1OjPi+FfU0lMpjZYatKO4a8Bw9zykjTo0aR+sGtHlYSh7OK+UeyYa0lw6rirZu50/h/pHvAA8R2OISXo3HeIxDC/b2Pj6+rMdG4dPwaQ8IHufgbw5/kTX0teosRSFyUnRwwk5ZFYdhaDGUDakj4A3jg0Ge9imx6ivdwCbTnUBOZg8X6mxhgX06zIpU2cqU/I5UQcP2o9O7uE1qPakBUk7xgJgim4Sq3id4+f2C/PNaJkJqM9ajerDOntgeoMQFNwfBbqT6WzOTLCjShi7AC7CwmqTgXv4R+DZd1Cr1Si6CSrncJfmQHf1gVK/j2qGThjs75onS4ugSWTDHdAxaNoKQyhaPHoKLSCt5nMjtE1uP9WJQrwpvpAiidQMbDSju31W0zN9Dc4iXHhAv4rmt70d/yclL0ElwcVKyq3qRxdBU4Seyl7krlObNOEbrXTNz8yGKYE/O9Ee5LnyFyA37kctvJiP11+sc+2sYNv55Sq7AMyzs5dxMvc5hbz9bjBb9JgqP4SfaqsD6ttPgRHuLfG5bludGlldNOrH9LZFtA98BSxRtOyxneorlR9xWdUE4QLyr2FBQFou3DPLuw6PkrxIuUsq5fBIlCW8EJZzC8Q8Nh55u/gJb/vb+QObJCPVVAkaHitQcx0lDXlshZLwL5ja7Nz0xVNwY4mwHLnXuvYKC2PVTgqIPrdB1UUnVB5jGGuZvgL65ryema/UjwprDjWNHyPkESrzj2OzkcLYDkpj4ogVqIh8S7l9Uxdi+yqjCvt8Wcr+pQAgeUKk7KUNh3nQtBc21PIAFlU0aeOYCkUx1bH26uJndWNcT0/IS6ayWarXpUNdPZVcTy2zaL6z90iorI9AHQ3Xvc+tCYz/gbufsjGAozrlzwKxZSMqdJZpA6XXNv3M345OOyLmXhr2XC/JSupKJ9xSxqQUbCeLlzd77AcEMF5UyXjf9clqPmjhGp5wW19S7pV7HR7ne01pH3g1t7lwjk1a4OIZdAlK2bowJecHqjuoNI9Q4F3oe21KZSksO159um+MhQtesdhu0mLQiI7eY2/YeH10EyLH6QgTSUWBNkwKsuSdq6DiPCFg5w/NF3asHGuk7k/pxyQ42hwg5LMwDfEFizoFpVt1yW6789GCyMl7Zm8ZMT3QUOVFgvBBKj5umyHiqhcPWIJ5ZFE7/3Pt+ScX2UlC8aCabfccDTyxSf4LcfiNWyWE9qh/b3izBtnWY7wJBJlytQRWDUN5A+Gnw4Eafwo3eF93yXc0OEEKyUVd5QLxysFlSakpXjaPNPjYzsN29PXZxDWPOq9XTNVRMkvfGB/E/j3KEXDe2J/NoVSQ5R41Hpe2SDFIHJ/e18eBd7+hB5Kuuz9Edzp8SpdEBFNxn++nQwLvlznMRbd4xwCVrY6+9lWNCO8bahYjx5wnTD36hHoSzv/z1OUb5g8IOZoVAiR52q/uIEnXHtS8JRsOfdPNSYMV7cDYqIGh7FuhOP4lLR+ZD6DbhrPlkb1nEoftdaWYueznDMZ364gk0xbjyq1LeAim4M7Omx5ImLrPWMitmXeO22nYfuNbpg0pt5imI8gsHVHFYr55dgWXMiuJcgdI3Ry9wsJeKbPhrESYo5PuS36Vxjq11VbOnsSHKQxHtCUM11WK1WaMxFcSDdfS5apXoZKqwqgMvmRqcr2J5Me168A75cctkSc9OWG9OnFb6vl9DmpVe+6UKPHD9zNoUXUe7SunEvAMqI/LjqbP97r+YrJ/QrPvYod8OQtUlAyWAgLwSZ42128lUaNpdpNMmubjL4nTIOfaec03DTpgNzMo3xxKmoGEQaIZJNkH5MuAfFpEMCBvFeKnq1EizxlupW5rrC5TcnsVFAq1hIGLHebM3LOdOKPJxSky8vCd0zDWn4SxMbUoJ69Spx2qbl3zBl6FO3ohDg8vdY/RcB8YW/v01Dbfwxj7Z5QISK8NXNkxpYZ8Zrj0il+eeyr3uH5TwDLVHZjuK/hG5iqhFWKIj0na0DvvuZLIFcmjnXqXiOQlezLofVHGfBllrED/xocI6Tq3SUvQOxem6IWGSyFKWQv7GKsJNwJUM5VghSZ2tsU+FaetLcim1MoiThFAIrRfwDdraecmGm1MmHfl4rwoLIfX5SqHjU9r26+6ymz8hqWETWFME2GVUq5fGYw4NIIQ/Emba0XohlDITX/Nade2noDBUMs9WQfH2tk7UJXkeCc+pX2+XATsF+9J8HmT4Ft8WqonjrdrDS6KZtVTR6+22hNw18LNCy4X5yUinBPX2FLvErj2xebobvCXui3m+RMx9EEzDMone1Q1nJ7AoKjD2iZPkAE1BbpfWBD20swh22/YanlTvY/uEuNsjSLMpV6rH244pGnH6IqTJGUIiING1Pgesnltfh3QMN47D6Ct/KPfEP0JqeCF5pYmnfPfwXKzrI1aLFVhxlfX5EZ7wWg44/zTwWBv1S5P1RsZKk1p8nqFLeQboV7l6dfksJpBtXXKfoUuTsaF0t7t5F5+cG+wO2KMCblo4Evaax3pmExeAjHy84IG55xmx3RkOTMQJ5CefjOWOY6+hNMFR0upnSUB9ZZBTbyBm1THjQnU4GHMN2ejSKT5Jc2huuy5i/P2huHgDM1grRe07aTy3h+bJl+imwH07ioCLlnjv9DQzSaKus+PYD6GmTelQn/pOuZKd5dgEX3LAcMOBJzJyrBTLi6Orxf6cGywcvu7lpd+2VkAAo24kyK71WoOGC9Hn+BrDquDq3e3mSZa98x3cul7KiIp4W+L+EPDok5Z/ZDuL4WHomdqPS3p+8TGuv9E8xVoRCW1MXNHa5O/vQFeWt/1q4emJ3CTyJkjPnNi64abjN5bJPcaJeqOBuLz2MXoByufT/BQ/H4K4yVDDEu7Cy/pkdMjh0waGzbQNotua5tI04xkWcE+kbUWhVMptU1m51f1Z487TzaGCf6jheve1142DgVVWAXjPxPMBwgXBUjf18e21zb3DCoL8VAA5Kq0TR6+g9708YEnKxySFILeetyWFndHOyPeYcF1rnzVX9hmDPgxKZJDbU0rn5RPd+uzdSbDP3itqkVAZvSKZCtqtM6VHqSsNWY0c53Tpbg/1qHgaVTw+lEv14fz56TRSmzfdzIsetzKJGJSNdSYBSNHLOvRMEXA6xgx8w6Pp1syJKZnDEqDPownSKdRQYeunx7Qd1aCrWkuZSPgq3w0hMqYa3ci40OBVO9Tb0x2qkKeFjpvOaMzw10vBs/1S3Tlc0F4OUXPjhKqZKGw7Ucujzo22Nluit0SxftHAQRdmxRINbKn7F8Ua58bjJN/WTaT0RXU0Vo4pMTD0v0elMB4TdufO9wW7N9S3yglzR0daMSomG1CKRDnlmnHL76eDenLEMnVeBmqsDYRtw5fqbriveNMWWrPrybR2Zn0BO6JTmYwY8AJc6rC5+mSNuaQeB4AMMs5Y4yD7Npo424Ox5MwaCWbo1jqGZDMQJyfTs5f62udcwxp7VfbW4UNfeY/AKJ4st+YBNMzN6s7A8/siLguc3uc1hYFnI2YXMK7eGOCtt93NcjjPk72U8UIt4ceNgOt9okXlld7g0D3TjELmFQe+s90Egs9FQX6wdx5r9+Dj6b9ADqAL7JoTlYvJhKEzbP9cHCR6PSBXjNqAiiLrUnI/JiBPf86U80FpWsY3JqngmfmkkPc95p6PbcXDKAEVl5o8zOOjN9utMp+BCqmxlVMiURyg12u2iNHOvVNl9uBsLqJ0MelUd1TBXVZcvZvM7bUxRVY/H3ywA9UU8wS2y1jdzdh3rswi0qi3QgsvMRCYhe9vHZtCEU7jO8dpnpdublHHG5GzgMnsT3otI6ni53Xi35D7qqyG0KO2UKVUFF1zKy2iQwWY4Z+owdrz3u1Y6EoswaYPItjj5NawtH/PVF5Z34P00HqnMAlNZAdMgJCBlTRSlepLjuCBbVQ4er2rfOEFsefNJCuFf/uYSFvvpxhnbWKSrQaIcMvV4nvfCyvKRPEInNui3wPDMCvN3i7sKcpCKTGrxlu6ohRFrkZ9rdsBqokHUGVnOHomIcuB4PoxwExNFzn24rXEA5gIptedmGpAH5+M5XUbgCTXgPXL5b5FBGdAn5hSlCMhQsfGZzOwbmH9bJKpJ7+fW+kSqzzHqlh6/OPhM5Whs0H/lXlYsEJcJNW4VVlN8aXpLz0ccMPKTfEtMHc+zgaBGMOsuqmznR1nvz9VjHyjbi4Xrfe4xlb+sMa+lkV0eg9c4HaCH2lH6gpAq8iy59u6B2m+vitpuc2f0rCx4I/7FAPoGEmpHqJrOZIQQeG8vR9FZYT0UMm5QDYYPB8yJsZqeJ9vRq/rx4qmx6Nx7qdAiR6lB3ojoOMM2YZQQ8GGJ1u09VtA3WNFV0O4eO24+WxXKyRL+aia9o5pd0ZuEYjqRT4QucJDhvLgseF0QG+lW25lXrbOYQ+NLmEC5/5PrQaWNRkQSo4kOGcVweya7v3W7cqq06uG3kItb9xG/RRnaqIBGz1TXdsn/bjjQe7W0MFzet1QLH1DsvVQ3ZwDXsaLbBd+xk/9sHa7CuSgog6Hh9txsT1vIxfZpUIb1kgFTF7GUBOpj8ZQd6VYSg/Z/fQN3vRFbPeVbfxLAGuQS3LvcGpJo4xShxXAQIQJT8MtamaVw7CBSu3IqFnIBJD+8m3F5kDvWrWZTOLpG4XN3fV861+n+djEDsl9JiDbi+r7egyEkocY580lC9b9jQjbNCVfaHHNQCPFk/SeqkvzUodLEtucx80dO25GVaycEZpyoWF81f2TgZ/q21ts/+jCFtTiE/n1upX38thGJFHP88cKgrhZQoIyHPKAbu5y0gQXp1xJtiCrsqjvxJtaefiT5g51MEpge7xioJR5YW0bzHOoUW86WUgEun1iMxYhhuUmUd4x69t+9/A/qVqAbMv52PtAq5xX2/LuGpZsQKo3aAOFeTcbhCqzonRpXh4bfqJSHVz4CAyz4kC8yWcEc35Ka9xl7hFn3BOekv5eHyGZvCapU3smiEZoYXiSYtLuU2KQmijzsIYmdy23wS6moGqWnQAdVy3HMKUZ8ll50+5p1XCJxmTYrpLk87E6tjcyf/JdaaesKds+vwa00NcjYzMi3Smu642OLwH99cSyMRpRLM+e13pCvHS/loR7er7EVIWOpOSDIS1atpnKgkPLaryAxcjmZd+ja4SEFcZrks4bOfcg8SKvTxTnvRfnCamUP6OS9mSIS4GhYpcfxsGz/GblRoSOvOzAl75J5EkeofAWUXLXXCS920S6HaPWf1IaIpjOuiObkIGBIIy7vpYz3pdZSA7ahEONFtGnG0zAGE2RhMfbUB+MYiqhEd3PilGPj1JvdhkQyPPmB6SheESynNklFvqS6lyEHVcP1VggOM6Y++MTudIqEWCFU2brzyS2UityhhGtTiM/7Rpw5uVDA9lJ5A/KrAGpbXYS8m6P5KXO8aki4ljf5rCMdveJh9AZYwLXlr2ybynbzzd9Zcw9ybq64mrb620qPB2hf+Pu/gCP34ese2sIWDfdbMakGzsm6Zsr7fYtiq2tk4l5RqtA94O8dprRVgoT0YRDv1WKS30f2A+bB9R+Y16pQqe2RB/t6o1tKT30QtjDYxk2JdVUhoLoDOC6l/IWWSR0vlN8oOQUEjuKEgGltSSN2IYfFEi1u6Y4sAwCXeFV6RTthCf5jomThIDlEf9wJDtSn9I0bsDqtqntNOFyKD+xptQNuykkUeBK+S3WL4Z9rD3xSTslXGqyf9qAYMpYCCQxugHsfEtGXupNy/zEo6m1MEbq2EmTX/jJ0YnOpWXPH5kFVrWPAeMunO83rT37VxmW6ww8xONrgTV20xTvTKzUKTvY0bUVLrh3a1bCuUCxMmyQIXePqazGljdUQ+csvey+SgVKwgelUB525puMyUuNUPgFAsEqtELeQwiJ22prj5rBqFvbIrYleJVz+quMQzcWUd0ntsjr4VnSoMviNMsRZ2SX+v68pMEU0Xuix5dpV91r/VbMu8dy22eneA1WDF7eRzGhvMfRddZ6DmB9bhF688JnwkuZ7j7UvN5GO5HtYpHtFE3nykd+fUlt2ywVypmx0/WwLQlJrV/k50LaX129UPoGKx+PEDgQ93W00zZ7fMXyMpfMMCGHPD7pieWiTsY7nWbxxSAl22P5S23Q3/6paMpI6A+KIwn6DGdmZLyeNWDkqD9Rjmjn2ZI0WXOvMPphY16nWvcCGDIEFrqBOUVqXr+09+LiVEgidTQL5y9LM4nhpRZmeFEwExqK1+YlZlQ/3Y9BW4sJPvyE4B9nqz2eZQWMKLVvdhQHX0tdBkpu9oRmZwliLqtefTlT66feq4DStGSERmqPycFP8VtFn0Qi8KBaGivRrj9ypQArmfjA0QDpmPgGeIvYz31VJDhQyPMuQuDFq/2qUHZjkKFUdROyZfFTGliR5ZdjZ8SXnmM3pvEe1uLQhMDC7ASK/4jQFpAlsupErtCKKr66Y+gD2Jth3bqeHvPppxBN+DKy1UN77n4Uc8r0sJSvn6IITGt22Rsw3AhLR/lel8Sbk17vFPgydbqCj3UwPuh3vaUS/aA1LhzJWOjslrlEAeN5Sc2NS+5olVZEmz6VtvHEmAo6wKFuJyf2XQW/h+uM98YgvpvU1ziztAGkq7cJsbgn9uSsmK6WMA/zhQaRu5bl/JwihXNNjTmDZ8jgMZWfS3BPoUDA2nJlmO1TTKDn4CF7i+rGpyt34Oe0AyWsfYM61VrBZOoqUBGhQVTJ4spRvrHRQGPovR7LuhqyFDAqRt8awQmUuUW4pvBSx7/fhriN0+R2u9/pXsBEOeynSbrv772fEEOT1GPfQR1s9iX2+UoGVR1pddzUxbnfLwZ6vxSf9nrMEk+govLKX48RMYiSKDhUYb1XpI5oIE9a67Swl08dSom8kY3gOuaZteGlbEUZu9UJtvEr/0s/7thdvJfkJgG6VUmjG+wPxJ1x41NPHsO8KFJSX4SABYjPleL+9pWHEcjbMVIyjloMiYVxPL1stuA2OTfqCN1vvLsbxIGuOV2wGwv6IkrSrlD3DxEG+SLZqKwcJxi/7kezNRxCjDHsEvqpBM69covPVlZ07jgiSFvhMPkJ7A1P2/hW2PnTO8iKXjO9o8LFf13BcBDmonPuAdxUXs3NzigDu8M7GFWkCArMTlepnWqskKhX/qJXhIVfpjmTXUSDSr7XqL7ZlW+kt/I0PjFy5P3SP5O+9pE5peUVqWMdVZG6wK7e8Z/q0Tfantpcv9XJ4Itwv2dVxtvKMAfl42MOxJlfzjTvDAtMnxRSjYDYO8YrrV1WyR6O3AcnU6e7voE+rMstUZA2KpX3dDvGSvzk1yTSJSSzrZ6+CoCF4wGCBfkJ9MEMz79SeJ4Dnv/lafaotXv9Vteb0t7pY9+/nce7aQsow/FxOzHVukyIksYY8ON4D8PWcC5UlL+h/N9Qdl6mvs78Kl3KqwG+WvK+W5zqnYGv1PW9ieKsYfspzSaub/oJXIeg0OfnOly1UZF9PXwd6fruuvB6nYsCQkM0Zd3yOR+5lRLfQs7qB+EO0c67m0znJ5T4euKWTUt2fD0RBk2o8DeUaw8p69tsAdnuoW9HURj9QlL033++3eH8djF2Hf7asn97pesbhX1tKrOqKL/1B8O+gCwLoDmavzYVvz4NkO+vfQA04eCypvmlS5/PCFSlX68Z/GdHbHn98/REq3allSAifkLhb68VNWv29by/IURzPZm95oUols8IfW0Ao32dmkfJ1zOJce1BO9e31SUkQU7UXYwFujl/P3R9nZez+f354D4/zZ9pu9YPBGOgmtDfr/j6UKErKlCAEXKG63nzL30AXOTTjd93DQgnf29DfvdYZOrXLs3Sb5jZy2rJPje9GvYpGq62cmmbb4eLKUqrCwq/wQ9NphBJgo5XTfOb9hTPqBT7ITC/tvxy7jek/RGtZT9V76st+uXZAFtVEjVMU118HuWXfvgrAQojXxCC/nOAQt+hk6Cg7+FJQH8VNvHvsPmIpqpfAcqYNP2p737FBSil9C9ADb+g9kHNH9D272Hm9/OP/BhFeZ7SBPEHGOB/OuvgLDFqqwbMxvcr7NttvnUY/gM4/2rSRv5D4CDYj5BDfI8cGPkvgM4PXwn7U6p23f9ieFBSRtOcLb8Dy7rkP1H/QdoF/SmgfiWiP7hL8nWewB2mIv7/rlG4XhP65c////Xhn0fl3yDAfIau2TIAld8c/3ZvcLjrpxaQEnCsyZYlm36aL8hWXfH98Wuel5+qC+EAGODgL+/x9cgyXUDLr/N/ufIbnKBLOEx/f9dfL4yjpC4+y+anP7weglFf3wzB6G8f8N+95H9gPP8qHvRnz7WzuV8nUCsWUua+iZaqvzQw6KPnQY6lg99L1KXRlF6XRC1Yul08D7/nTf+cZf03oeX7F/5/Akb/W9DCrMuHqf8BJks/XYInYFqc/m9JMmBU/zk7+i2Dgf4VVhJ94zzJNY9ANP4LRQ4a+kcyMQah33EOFP2BzIH+VYwD/p5z3K/Z3MDsCUeWrF9n9g/z8gsL14H+YfZz9TkL5eN+Wfr297PzY3b/3RT8WIL8vQAKQawgAnFjLqMBdKU9LsFjKL9E73XKviR9mv18XQKuHPoK3FjYrvvP33ryVVv6db39QFr95wLIN5ghfylo0EveIH8vYpD090ChfqA6wX8RTn6RXH6DE3HtEjDv83fw6NeluZQZru+6LAE9+2VV/mYur/9E8PzvRMcfTz1CkjBM/BlO0mguP8Ir9BvA/Sk4/0VE/gmFQf4gi4JHRvPw9UXz6gD9+CH8vgPtPmNfmqiN0+jn/NtY/vso/ctQif8IlfgXHPr7D/wdRrEfYfS/QLf/MUb/XLe/eG/3L3E65E843VfRh6+iBkz033nX1xv/Cfv630km57H5ecqG/noekEb+ryGLGEF+If8IQOyfAfAHzBT/q5gpuP4PAPyDhPQHgPyRyFEMJfxLJOw7Zfx3ijf4+dch9aFVn6HA2et/6AuM4Ze8+evfv+E8ED/Bka/ff99G4T8+G/60/vEOv5z97e8f7g3/oe2Xnnx39u/vjfP/IslNqjnpf56jHAgL7QAGNlOSD9mNi68f+Gueq2EG0J+y+Vvbj66Phiiummo5v1RAH7g+/Bz9brL/71lZKEn/IykVIb+32+L/resK/W5dcVFSZt+tp+/48G/Wye9hzv9oJXzs5T8Y+G/tfNWCjdCmAhb8dk4isOn86cjPdpZW88/m1KdrsnyZt39OV79btr9a7b+3rv3A1PU/JwcgMHyR4d+A5Z9Lqj8CC4b8VWD5XqORq/lSScE7XJLahQsEMoalaq8Znr6D0C8k4xcs/FNS/I+n+U9o8B9x+nka80sr9BvMlcsyzB/hBHjO7fv+5VKuk+ZDwP72tbTZkiXAn+MrOpOYhn4a12w6fxqyCRgjoi7JvgwfVv8fITnUX0px8H9sUaW+14sx6Avy36rxfG+N1y+l5mPY+CeY+X42v5vw/6SS8j9LU/6MLqbREl0Q/foVEQHWEK56sIa9Q5pU9GB38+54peAV16cb+MUrHBNef7nF14gMnMDKd855WFd7oeRMWVefTdFmd8TmfX3Qhf06+7ixjDom0tUgyIdqC6KXqe+l85E9eDzwA6XQmnklrKC4k/6wPE2rODkKoTJnhYmDxHFF7g7SAO/8wR/pF+u2bXO3Km8XbTF09nwSozLThY05hmjhbstr0SAxnInFjthHOe2PmyEqlXU9hWMz1CSwuOkcgUT8Nl+2NI8ynAZegp/tcjrzk6vtPcmokYOcK+gTEyyrKMqSZTmOkwRBUBQltKxw1cJoGVlZwKT3lgK/JCgoYmQ67fDlU1HKoDioMIAOx2S9XNPMC3EueMqTzIxLhcL+hLMcJ0/7zc2o5qZy0pvBm/K7SLO0TS1deUzaK3Qfyp1ezloow4k7Dx7WCyFRBajIM6FWJdpkoFJHOVu68wnn7kbAKJYcyHuUAPqzL1im9wyacNu+Ub7C3TWH4Z9lvjicUwlutJM4jWkoAzwX89ti7lvC8sf95gu34Y0y5sePg7HPELjmpB7GLjwNFbXSV6GAeXHBUMS2zUTCyHtMscNurqx36pG8McyjM2kxYDxKkgv4QIFXUwnv942dmRdNuYyxMUP8iPlMeGLsFjoQIzzlsp2Nx1bCWJA+oE/OWG+XHzB82hurh4EmJG5kb7tD1Jl1m5XoiQdqYyWxB8NV5uK7hklnk7SkZfsDWt8mX0hndsF8qH9ppDwaN9+8MxTFdruZSc5x00CMmSDlQv5i8E5FQdTP4Qzu0zg8ihlJBTi5iTLbH/nbDz10MpdIdp3R0bDXu6B2rqN44JzA0cCJdCdR0XzueSljgsyiqr7k1RYCz03pE2OrQdPTuXpeMdMsSAGBceQJVFIFsURKjAvNUNHdAc6L9OGKiuN55yuE/SaHyrjIDabA+jCfa4yJER1l+7ZIDU2SUF5sVPlMVUWjmGlMEw+1poQBb2bEBZWkVlUDF8bTMHCN3tlJoUGXxKmCS+Adklef/G2ldgo5fqtq8RHGsWDtGb2A6WeB58O+BSqGfwLpuYp6SSKf+ZK4Kq/BqMMND/aFogxMKQSzL0GCBUbfc8oCodssykDhjFizW7ajphzjzHd34v5+jh1+wcPaQSKLQH2F/FuK0DM1pkzUTIrN2h7RwwsZXXtD4VlaiyN/kUeyMEoe4Mqm4G/d4XzDMvv1nt3RdY1CJHnuyL50Vrsaj10IGD0BwxtO5otG1DgGrBcQkgoqlSOWGfSW5TQWEKDEkfLJtGhdsyophKzPtse9wprHRIp+d1iNmdh7Dp4vxxBy5k6yhCptd3Q7EQVbGtaFqZfhyag3KxjGHiS/af0zOnIhQm9kVpA0vxVqbMBBawU7qFN2Us4nL4o9a81D9XCHL3KomF41WuvwKOLQ85o5z45IuXqlhRAcRMfgcwqcd6E9YTd/dkj93u4Z7q8GCZxvGLiNm1cISBmPFva9iLJ2SY6g1yhhe8mn3GRVDc3AN1ltY0U90M4E/kUJl6u5XZr71WnfpdOZpjWq3ET8qSZwle/YY1uAdxp8hEPczu+XdSyuOHWURkol7TzHVyEFoYiw/quWF/reZRk2IcRxfhzDpM0TJzEj4EB+3MU2BHhC4LufuWeAvywYZg9eA903HgTAGfdmN/u+i92nnkL9ls1H4OI2ngcBcFQrdf/u4Tec1BP4+ZR1GdkWVOaI1tcrgrnlFQz8MES3dnzyfirv0MZQPaGR0JAjaGwH8mvWOfDueQ2XOcEjtY+vD/YZlbrT6r4ZBY3plhDFop/RkSvDrDpzaXv8MdcXVaiUHM2GPXXHD3k/Zx0XhXZLlpLGPJEaXqJYA0Qb2ftgVVHU1wTjMidVs0+USLkIWSXjSlVfcy0Q+uwenhVK3cehWc35iAGjBtwVzf1+UeThLFfW/FQ5ydn4UyJPpwoQccKy7OyYDEoquYYV5mnPj3Xfw9RKCz7IpHf9Th/UNDJ3+cRivcNAChXD5+61/uosBBNflYnDuH+RbPBACeEZaV5fx7is5Qv71JBHVUSm4ep184Q3TZEvMYU2dlPGNo4reUIRtw3aAUfr1kwSus1MYmLC+OmUo4LeE/mc71GPl+wrUN/hkRHGOBlWPMyliDOGADuieHUiRqtDf8oyg9F1lID8Z906Jw5if5KkvNEbfLy5N6K4ULqd7H0hZADFMBQuPuawTyJV8249fDVW4jDTFAkhFGqiAs6KG6y8y1UG4qmKKbzzet03ApjRnUiNaymSdw8jzeIgtEg5BoxZQpSgejuxWHFVe1WrhaR8rfnM+zYfyqYSwIBGKr1wqA/4NkPLa4UqodKq4ja/DVjxtgfbtPIbJKz+BCdlRDwoKGCfFZpExObbuv0GSebqlFQSHe9uFjmi8jRMmHGjaNupCSfvd3FuT05aGfmEQSiiI4WOSJjZgeYFr18PK1hIplwEVuOnmQURanpIoeKN0AbGul9Cdt3dVWo5RMjpPG/9lMLxbOdNZtBEzAwI7ACe2lxVbx3ixG7yZD2EyYvAN9Dik4aWf71OPzRn3NvOniPLi7FgO7ZgeCy/6SCtjONe1moyuWIoU08GpJwKOG1QelJ20pRA+izAdtJP9LzbugQjxzYyVwQANpBl013BMsjtZi+whxKYYYxsmjI8ac593nzIq4ev7oqP4HgctCKbNveOnTu5JJlVH29IvnqGhhtRJHRZeyWpzwpOoh0Sn6VDsGrH581X6vSAc+h5cs646n1IzEu03hVUQknp/UblcLmjtwC75QkDDAN6SC2PcIyMmb8VlDlZB6At1wFEnamp0ty3fIGQKIvoJt/fa5LZ4KpmuahjSYfKw0qA67kb+kyckOQLTRIF0IawYfGNKyen0SnmXGNIwt81bE2DjDEvMzdWPcNIUQwRTzMwIIXjxSVLiOtwmmWZHbeh40uYuJura/dv4lp5/KhT1khEm2Dat65l7sdGBKRvMquMpUWLIxllOEcKT4yxOqs5Z4yAXqINwO0cedEUSJZhoj1w9xxYGcQe9AKzBhx8eyrrXibYgGkBJaKCHYVYwM6t+fDplfZMrKUZB2vim/xytvPio5ipwSF065N8e060pD3lzVJYnxTjBeJHop0JPwRxxcNt4UV5ebkfVZgajX2HTy3VdoPdSBGj5qzrt9gvs+eOp9n7mhIIFFJgC3sHl+tAu5ogtLcbLnv2dkmTC3AUFWlVCKebtb1q8eweSFjN79Im6NbE4sGLaWZnnub91cwHNJAFTyuBghdxpd8Ityseh4knHEjO81yCa5VunkAcpWXEew9nLx7toEgIi1BDuBOq3V3uYyELybBGriM6yBwp1nss5iFvsrJvcPJY7WDaSk4PSpkY3pSUkXNtHJckxSZ7VotZtC332cpjJEhiUL2VVeF4xNjnxuQv1skHPww3+n0KpptWZ48Qo7MMaVmDEres4Np5VuhxDmOy79V5Cw3Pqs+BS21Bhs4loutOc1Zoc1qDlIKxed3hcIq1DPIMtEQUnX/TRqd2mAt7Q85IJ6fjiH9pAcP01ULBeslTfz67WYLkZhc2kDvncROj6KijLYw/gbKzXBEvRPwQX+N4b0lEaq7bqjc6998DRdLymzdMOmFzdQptIF0GTPDOF9aMyO2wyet8pm0fb1/8JGwluSBClmf1WiosrmryzRFwP7HnWSEyEaiLwogX/m2N05zmli5NLLrNJZcFIrHLAtKYEAkzt+juC8hjSIudJChsA0lX2NLXcLlYuPSUHzxmmuEuTkdzSnoMSkpyW/IxMawX2Ex7hzkD05KAdIkLxHNJfqqSvgz0uPQQdYS6dnYqeSeuV5pdRACBkFFjV5E850bOltkFmJvwYdTM1BLRjpWFDItU+5BYTM2YbU/ulQ4rt4dI2c38SlCI4xc5vEkQjguYfbvNbIL1CeAX75nx1d5S8dSqY/VUj4qXW3kv3ON4DzTO6B6QCvz9EKlbzhEIKRirzaqjE0COJa4cua8PoUxx93XCMEO7Nwcwq1FLLzF3HkLMQtFne8rUeqMQHBbQCJ4dDoo/SRc9y5DOVqjIEJ4T3qhMJn3n7+da7nbfQwCC6VEJo3ogy0sQRgV2OJbLVJYpTaYorFdpWYq91/eYDRRW7PHWuKOX3ExXVtu9Prkvw3yQXsmb3ODsvF6Lj/z+pkBWXwucQP3wbv1NUHpL5nyLSqyA+rh/M2eFvYpwN1lE7u4oxhs7db4uMv6psGym9XMVVTl0TCIjKPS+plYfU1rV2ok6mYC6IrxwvdF9thmFBX700tY/hhAeFxsilvel8HCMuyUqFM/Yuo3ptF/CE+DAwSu0SNzpFuTeGcpozKqsTrVoMCrWuCt0OCAiE4x38VrfoxANKky1ZhkLz5B7QLEBBNK7ohjoyzDYS6IGscmmO0STzTiOOSum/5Xl8aLJIFb3Jo/9al8+onKMp3uvDjJC5bf3dcT6HOncT8Gl/FaVVqLaTGEK9YvMeBvw1hetdEDq6WqBjw4SfnXdr9/A8j3eJO06FxQxIhKyPHFxmsJVlUcSoKHupXcJ28dF7qrX5lYau6gmGvbzA6TUU26JKNcmv5rsXTXIQlhfw7uzPzqWneDu/WZLagdqJbMYawptBpg5lX8by+cAKHtaOkBF0aGPtMGoe34cqC2+okpq7e1dKFkvVksC8loOzYzo5dIqLK/fbiu77JcuKW9a/t71Xcy4BWHCAXoIiNLLWMvJX6lZskM4rKCeTQscQzkVy2RIN0i9X5jmLlFFS1CumTDpbua8Dl/AE9amYNCLZcZpZbE3umJsoGZ55QPGLska6oQbeliCX3NUbyihoDTnHKcje6i7nQzC9f2AchtrCLlQOM/6pIgca+7IzLDNbqnqF1t+9UNmD8zXoujuCsKs8SynCeze/NrHzxLQbVg05oi66JQpIm8ZN29wwhHDACsnF3L22kXawOGH40Th+7Gn97uhclZfTUTkCyr2WVDctaAM8bqbzFm4CEBU9HMtNsECKC9BMMY1fA9mgpvAn+t0l32b2oYFUx0pZixbG5jHrl4KExeJ6W5sTIaPyCD5c/jAsDDwayBCdlNylwRNgPeIb1vOMtgI75Gis3VN4FrXPni+KiMrFWbM0i4ZgOZLSr4bCBOpTyACZzdd8OxPymVocmK414nWfCrkXarO2rUFhHHnF4j/BKYQ2PfspTlXWXegPbxRFu5riSmLpnX0709+yluFjNfiS8aDR4aKy76+fbFbVcl8XR99dA9HUrSrunld4ki0sDf8YN7jvaQOGTADH52VBVOwdiiMRs3c9tnHk1OeqQMCSNaX1JSdAxBm+lm5jGHR/Z5SgXk/NEVgifGYfaLNrZJaEhYEtXwt5sPuUXspKZ/Kk0JGY/cmncfXMLOi+FD4a5Ud7iiB0Mz8hZQkeglS97mGiCx5XGqVXapDaHXV9Uh3VxRBwbxrqhOVH9fXOpIySJbn+dirVeMMTtyYQoyl9SXhbfT+gXrCq1o1sIAOh3q+TZHl1YusRL07n0Xc+XtT9tyx10ftXST4t09xxIeKQl0imGxGht4mY4E4MDPbrj2kSvFNUqQ0WnmnMrzRck/NqPPwEo4kW1U0LdmthNZcZy9NoSh2UOFAjKzEEYAmxXKeg7USGhWJTQF1yapAy8o+sU2qO+ElyY1P1Q93G05UlMmeECMoDwtbXB70AMONSFwTf38gTBcHQXnTBJG8vVVnr2bL0559Xhdx0ScqMsi/9OD78dN5g3GMcb+Jr9LAX4JZrPxgYXxlV/iPxxv2SGIMj+DFWQFX7l0y1mMt3YL64FJbgGbFPa8FYJsesIFFJ6S7fF+2nB1cBKmlI45G7xnCgPqhrM+AEFuQC9OCb2kUL7spHPJwrTEi1AJ9TrgzyJsMCfRtCtP5mKp4X7z9e0s0sE4f7Atoi5AAFky1dTEy6x9jvNAAY8hqtRz3l+4w0vgXCv7jFvQXHP1+v+i3+0P0F+wHPve/af6v3yIif+Tv0S9r979ia9rt+2b+f3szmoDxL/hvN6P/6J/2PUzI/0bHhX8QbZZW27/skPan/t7/Ibe2/+Rd9m8D9RtfeAT62w997H/4lNvaLBcBg5asizpglYP+1H3uv+Klfvp3H/Dvvu8/jqAoowl43f31g/A/OALXgV+jGZsqz5IzuR7zj3v0582fhfJfFh/wi/fuvxxqhv6IGv/Bb+9PPe7aKk2b7K8kgDCCfCF+yxh/TwF/EJmGYX/34/0tEfzFefHfIYKP0OL0XiVO+sGfcBbq4VL+9CMa+Bc53aRZHq0f4PwFfjfRAFy9vvTTh9P2lyIENvo/W0RpP1W//v25vAb1av/mc/OXTT32SyDqrwHW9BcU/o037PcyEgz9yBML/kKif9Hc/+Kx898x+f+C6+t/dvrXoemj9Mte1VWbpVX0DQTg+wC+A7+rvm0/sRDiUq6fGP/PP/CL+QDnZ7nasp8Bbj4iGSLCnyiwn3509K+GDk7RX2gax8gfEg34QgSE/QZI1PcefZfY9QNh+79CkKqMsI4XXi0xRLd/Uhu1Tb3/J2jIcnH+/kv1Mf3Oc7bMv7T91P/lE/5HfzvqDxP+PZuA4R+RCvoL/u/P8M8S2pi6tUKS6zaLj6X3YPvpe9f5//tmOG+qrv49n/iml13r+m/fHOPETyjr13N/nse1mqas+flq/PkT//k7JHw3wf8yOP5xRCL9m8WN/FMo4NiXXxSZ3y32vzf/B9AAeGYPxMVfj0nAtf7Wpxk44/8A \ No newline at end of file +7L3XtqRYki36NfXYNdDiEQ2OdDS83IEGRzvCga8/rB2RVZkZWd3VfSrr9h03Isd2scARy8ymTZu23PMvKNcf0juZan3Mi+4vCJQff0H5vyAIDFPE/QRGzm8jBIl8G6jeTf59p78POM1VfB+Evo9uTV4sv9lxHcdubabfDmbjMBTZ+pux5P0eP7/drRy73551SqrihwEnS7ofR4MmX+tvoxRC/n1cLpqq/uXMMEF/29Inv+z8/U6WOsnHz6+GUOEvKPcex/Xbq/7gig5M3i/z8u1z4j/Y+rcLexfD+s98QK8lvoecLQijD0Q71/C2nP9A0W+H2ZNu+37H3692PX+Zgve4DXkBjgL9BWU/dbMWzpRkYOvnNvo9Vq99d7+D75ddkhYdm2Rt9fUxbuzG971pGId7f3ZZ32P7t4m8p4Atm677Zae/IGhZFkSWgfFxWMWkbzrgN9zYN9l9UU4yLPeT7nzf4bu7wOj9PumaarjfZPd0FPfR2DxZ6q+rhv924l+dKCfpFAL38+M0fp/ZvXivxfGroe/TKhVjX6zv897ll60Y/Vcc//ap726OoN+t/vm705DQ97H6Vw6D/TKYfHfU6m+H/7st7xffzfnHpp2CeCD2sv1/3jHa9ButhAnxHzj9B6YluvX73IFg+GbEe3TegBeyP87z3zb9xiV+GQTH+Y/lywjMvQOMTMevP0FU4Jlv7slv0m29jXEfunjfE7v8ciX3nX27mG/7/ie+B//Xvvejd1XvJG9uw/7Gw3KaIH7nQNT9Hti7uQOf+e5I6zj94s/WuDRrM/7GvX7ZXfvdDn2T5+Dqf3TIenw3133S5Jfr/bqC7/f6df33nTVDdb/7D+p3ofE9fv4Vzkogv/FUHIX/iv/gqwT1B75K/VmuSv7gqXxyT1PSFsALizUBb++XjKX836HTH/gDWSREAf2IRHlSUGX2g1eh/ww2/RZt/pW2w/6KEPTf/1G/MSVK/Qg6f0ukvzYk8WcZ8r+CnO+IcFtT+2Zc56n9Ytf/LiD8E+b+F5r0P/OcP8vcGIT/Ff+vLfxHaeVPs/Av7vQrE3+DdDBfX4b4v4LwP5hmmswhkvwDa+IFlWM/WPPHbP/dIL/Ge+yfMfkfJ4Tfgva/IqbRv8Lob60M/2hlgiT/SqL/TkPDPxj6j2LZWd/FPYl31vqzg/l/jt2/2kJkVJGWf8wO/1t48C+wPIFBf/0nAPzfG97Iv500Yjdp/GKLv+ON3tCUzRdnZLa1vqf5jsUvjvXnU0fsjwqTLEeoP4CRfwYl/gERTXDoz0wfv2d6GP4HsPJHPO9P8y7kx2rzl+TxO+PdxfIEXjb9V33Ofj0zy/StxAezlvzypmwOYOV/SMjTcV3H/h9a6vsZ+BxQTJT59hYRJ4BoXOOzpv2BVKkamfuf4Xi14FX3K+9zP7Adx+hg3I65cQAvEp/VfcGXM+GM+//2f7dri8xT+s07+V/6KP3mLN9eiU2rBzS4eiW0HbfTGerBPgWD+XwUlolMpspY7jPq/FO3OAaTWGbk2GersR+tn1jcEVDwWS5klSAEs7E494MmVLKDhiR1v37kUCc8fRsbTDgj8HneakW4Tc4you2FMhR39LHfcS6elj6thEOYDd9AXfumfWN0uHfeedggyfXnwVJoykP06xTwITe0ukTDFSkg9YOXIibf5JMtDTTfks7R1HyyVIk29XKaPdd8BCRpzmFAAb+1yleU3ZkghgeETIpmw5mJ8fF7iti1dZEKfyCL2x0ZLWixuCADix+57CIRa6X2YzC5fEfjPBxJuMQXeXugkUxu1pm9uzalN/+OAlZXXkPuZM2Tt++KWJxlrkgnCOH2i12sx6KfTwLnFSUo4sfQfiQQTY8kepxwD27CSunCvWObVZ9eam/KZaj5gHltLxmFigqbdrsmi5podD89PyoDnon3fGJIijylMy5C116HnkigfXg5eam/91PH95Sk049fXRxzIeaM7yKYLx1hpd5/o4QcojkCPCL4SA1q8Lp0ERY33ZDOfphOeEg686ApCJFvS7HkJDy1h+hZfPu0dohzTJZfWgd9LC+klTL3GT8ebq1OBZ9c0tOQLvAZc6G5TSG5x9p2sDH7u7EGCgbmh8E69JEsDGfSqnQJkCNbUeIknwWnSaZRmQdFlDd0sG0fPb1Suj/Be0FfDjMlu1koqsG9LeXULMTAjGSCh7xvEGHdp5MEgiGz+wfxLjNtWl6NBsQtC/feupVQKUns55LfK2bfxxRkfjxxFWzqQu4l88zHXQJMXSJycz5Ou3cpUHnYoK1bAUEvduOpEQanDE8C9445pSO5kpw5aQyTsjr9nJBNa00Iuui81uT1Tb6zgKoqNBr75+zsFrhavdfNd4wNRDwV72FxXnGO4EnE3NOuCu974sSWf7xYeqbKhPQNW1DRI9P67HN/eFi626FZYfDr4enPoiua4dcMDEtlOp6c6y/YJNtl6bS435nghfjOKex9gB5JaDyoHak76gxMscK9ZTxnTAovMcICb6s1GCOlnVrb99C0r14dj03icZdb5/wagn2DNoOd7c2QHyo+CG+qJinglZ6JyVXm1pAnwGeKXwesVxOr4P1jAmExYXU1S6kbaMK676R8GOgSEsAFJLoS9DMLE0rcsj56E1zCTuxAxezCPLXlbMQWSmurhvXH2nwUsa6AhaPP436U4X45+X2bDtiY4NOxM7fNzvajnHqw5Q6zvu4TyLKjctskacrqPngQlMFLfZ2NohaWQ7GSdDWxGk999ipZTk1KZfGg3twSBhdxaTGl1ouiq94Ozjj7XNy7sv1gTE/PWKtDKnetd9iL8KHE83FKRTw3m54HC8XbwGMsXNZ8l80/MfNmj4nLTei4AQpQGD62CC+fnkZNI+hxRG8fVuJVSJHUPQ5pYA90UhA+GwT0MLDUph01kqLz9MfgohIOkVFplc/4mEOV1DDO5t5XvIsx382TsGGapb72qOrd9wxwxcRwEV4ylMSeI314WZ85DhGuUc+7ymvMdHkzzqcrFbWtD8EijalTFBxcd4fuLYNA7yRVK+mTszAjW5hHh9mVc6l25VJVH6WKCpmPiRyxRVc3TuV1VY+DZVx0WGtT6pG6cCucUGJbp+v0iuSNAYqkW6O7wkPUe5/kpKogg0l/yDH0SRHo1T+xOxldzqTBLVzBQf1W/D44ptemZ6/IRWrtdTQ0nrXP4fQkBwCV9gnarf1AqRI/pqE260lCdCO5p5kM6lTQe4SOPD/ZagcgrixcLmd3dotsSIEfNtTqH/PRPs/PWe+VckInj3j3UcfEaqEFh5/V5t/ZiUsLT7gQoYSA35KOGmveJVhrAjfdTQUSvfWOLTzWjjrIMj1LNd3XUuoNjOeo9mSKLtRfnqsHvbTad544WuIgvOOiP+/Y0RuVj4XXQ69ffg0y4kIoqLkcZ27rtZXFuCPVcSpe4WMKnsY0PE0FZt62w0vYcwkKfwGp4Or8NvBCddEm9UGmiTlssliHRvbFmWGS9ndWspHXzMkcVixiGjf+46jJPpzZJYplYjXbvgtgz/9seinmDtuDnNbDl5A+cFkwmVB5pbMUYdEnmAH+TezMyKf/EdmJn+ZQH7MrmVKQle4ThkmeSafvNqHlmI+NPXfKexOMblvuNDiui8xob6/TE2+rU5OGDcA7Wcxie3Zug6IBNNsDXdA51fVZzbxwbQot8lzmSwku4nyTLqO7C1xuQ6g9OnUwdqP/TA3R1Yijpv3Kw8q0wR0qVPG54KMSt9TVCNqnFw5RX+o+HxEDHyMluisJ9t32PRzEYkxKJY9qsf163OhmDpjQSft2Oq6EpBIES9t47+ilh59OernK62UVN24H6sF3UbU80zAUlWl6qDNNDjorFvEYLeTO+IFvvmIDe4Rvn+sS40hXul7pLItIbHrWaD+R2+PkH4Y+BcscAFYwicG7ydm3uBXIex3o/Ai7BIkEJp7a9cXYGKDkgUUK5OF6UBGzmr81wLlfylN+6FiOnFhvCQ0N8poapN1rQoLbYw9hztbINVXYtAGQPu6A05KHSN+fiZGsa5YkUWNsebE+AO74yc+BE8NOYqWNKr2Nl7LBmHO++Jdc1Lk+bs7WcPGhDZG5KOY8HXLmcnFjT9aerWItn1VvXePLHoZUT05YRCNLXM6Dnia91oakeVRaNOb8/EnQXmqcLV4PRAnApdnUi799SbSopsPUKyFqiXOkB0P49AhIUhCbZztSRBdwn54MykMXvGAKuKpf74wkZiBPiJmf+6rNjrhDXO09AO3k+7Urg3HsYzv3Tft+QuU9XqNhxqvJ5ntZ8rFkS7vHsOOKhrWfR50ln3RvcrZbwWM0sYuaQ1z2UVejvnNJdeeRuoebZEmO7PWVnJwglh/Bc3RN+RMZz8dt+vo+WjAwDjLPJc+8WANPTfYoVux2e5YH/Mt/3eir1hRJWXrXykqFgehrq+lRoTMTuhmjqLo2lLVwX5M2I0L4Hi8d0B2XSxKqR9rVsUMPsq+d4clFWx6F0SU9fRnZ2dEOkV7DaQuzfh+X1vHmCIrXhxDiY0Nw6TXCgXOzLC68CnsGzAzUQrx5W3gXV3gUGjMJVJPNTPGhvmT4yR2kdMNSvLXkwLNCDD+j/BXxwZZSHQSKxZtum6IA2TikJqKBso/So+TLUwxuN7FXe/vQeAf96qqwGxppzaH4Iy413/+QM/FRN+AAzOLRVxaybSrFEPp6Zn1CAALMJlKSHALmJtLbvobafxXirHqPnh1pgk4pjMtoJCmr8TUbo7jSrmvWFTO9hBOpAuuaYXB/vbJHwEMAFwMUMz4ZwKj2GS5pGS7aQ1DXkoxBKN5/xid9JXLljob0oXKOtD9Gxjoc/trtz2fwK8zN9T6gRBYvLPyLvMnsOVgoeYOLCJnAxOrVdr213JXKjXE+B5hxSENWJFmPLB8lW8r7QRsPeIxfLszhtHmnTSmB6M6mnCvhubx7Bw3lvE/XjJRqA0wmgiG0jaVYEzFQijGO55u2inORooBq+o8U0t/IX3nzvkvpb9XyMm5gJ/Y9rsn3of+god8rYn7xzpMh+RPkUIwEvVTsH3Q4cIj6Kw79fSv9g6CB/YGeAaPwXzH6T9I0sB97Hr9Srv7ThtX/SJf6n8igv9es/qE+/j/Qw/8EoepvRvx/TajCf+xy/Kpppd+uXxU9uOE/37i3bdNv/erfCZJUVmS/Ny71z9jxz9IbUYr8XbsK/gPFEUHof6chf+xiCK7202z/+eqVP7bbv7P7hP/Yh2CGpDtvAFp+Wu/XTSQU+Sv0T1gP/lts/nvsR/xU+n8q/T+V/p9K/0+l/6fS/1Pp/6n0/1T6fyr9P5X+n0r/T6X/p9L/U+n/qfSDZc5/hdC/a/kk+b9d6sd//J7KT1Hjp6jxU9T4KWr8FDV+iho/RY2fosZPUeOnqPFT1PgpavwUNX6KGj9Fjf//iRokRPxW1PhfpGn84Q8A/dH3ff8/JmaAP0ZgOEYBooZKde3nfm7C9lvZjmxHzoWVG5c3hxfjM4kNod8NCU2g0QyefX9AgoKfp1ApavaePu9FmbFx3NST2PEbeaH9kIyusz1K8PPad3DRNp8B/nhyeDBNqgPRowCnbY1P7dS1+H6fRKch1Ak5RBnVg9rP3Lohy8CnfT+piyxunxBRX2kajnkK7P1nHPS4VN9e1wyjcEwF/tivgfuPab4NKIz9/ROMUn3fS7hv9WvQuV99DT44hvlfeKxW+f3BwV7P78fi/pODP2/rfg0qT4b7X3isv03Ns/r9wf94an48OPPDNP+vOtY/8Ie0J5T4nkBOJi3vwXSb1gumGdIa8TZI+AGIyvMuRBiCwWQq6Sx7xPnmjrP7CNJTzjQDEJy0LOmC8MuiGUKGH1FnIQ+J/yhRxIMCRTXxGE32wQjPtHfGj9dVk/lIj9IxyF1OPV0C6Y8sHwT84rqOZ0atXFBlbBb3CTZsN/8Ttz4HlVBGG4Ak4WwG0tmVvUGKNwo2QvvyvR8UYW9vRn9/THL2ap7MbxxkO/hcRpUiyN3dFHT5ZGZAXiMeCfgeQpaWjohYfZ9ZCVtGUM9DyxLfZYWIH/LeXXgQNWp+n1ccUI8OLsBHMUQ8MInhG53UBuvFYvfhPJjC1DRm4s2y4Kr2mE//goPEILAjdTmCynU2CHlypZHxfR9sZ7tBF68ihr4b0eTQx4dV0UrCLvf1BjAEb58OuT6i8mAEau35ciVfdJtrEUMy5WbxVRq8Sa1W1dB9Mov+EN7+GlqD5rGMEa8oD9jLxb1umHoK+5k9AYXgdykgp2/UiHOps9DlJxTcbO9+X6z++CVV1FXdxoMDZxSeV1Ym4IkwjDohnzl+wgx7MLoM9JQoi+jV9kUP3E8O9xaHGLK1a9zTkSmOeSu8or1sL2iho/XiRfsURjCo+Dt4P+4a+9s1EA5+P3qEVzxXRWoSwKjhxFxonpD0q3VXZ+b81lGdddcsV4GAaAgulcv32OgvcaaxATn3ew5jJmcyU2+uOvzowFwFeyV7WKJeNXRvC91drUTYtCKafuFjjAfcehFeDbA4mJA2jZoQVIwKdFfM4eHcOM8aPptcJD5a5fMZwYjQiep3a0noirL6SS2hZT1Y0hqu4eqOtG92n4Aa3HZUYEEN0irBijKskla8iXFtnOhtahlhq9tSY04T5BIx+VBZiVqye72Qugla+bUFMC+4SPtLpNaHJ43oXr4fM3afkGbHk1JBaFystdKpZ7jgzw+76ez7tnvYxkuO4jJiwTT7pb8fObjf8QnIfMNC287DIUoir3wbYqmqQjJfaDylGzyOPyqgIgi0tmXFlE3+OYrMGzikKAGNXFAeTSljjxnyjZzMtbZL3U5K0OWf/qE3FVupBZmMllOVSxaUNDuZy6Q8TybDQIl7dub7OD5MxIshQrxaqTMgKO3nt7sJDyCk5g0wnoK+gUTffHg82jTrzXfsN2TjKxHR6JWmDYlfEkYEPR2gK8gHdozkUW8YPjwCJeIXIgv7e0OTMJGA6R2MP8UntzhGrt11XGUPOxfXAE3Ic08PT5s+sDvz6A50D3Dq5JqIDQkB0vQCuBTpyS9IWI5kCeDHNZEnrRdTjkt69B2j4aO8LaTNY7gNTPaMXsHSGMYSYU71GeOT76oXgziMXNx0Cz2Tp2jhmH0jM9TFAHdWyxjb1Z4oN/HH/BB3AQBYPE7vd0ef66UK12r5nR0e6OD1bv3RVLqkzw9z1/OiBm9Htif6XDxlpfnKdx8rAR5iq/MRyb2XvoyIxkwCBdgmX+tdjHvh0LdUuR9cm5iv4hG8Msp6KY4ua4RAoRVxx3oZEzjhwgNqPbhnwLPtU3pU7NCWy5Qepv+MnU4v7RGOG3b3444/DqSw8iA1p2mVzfoZQ9XyAvGFIwxTmNcek8UNBcRdvX/PWjIDJ5p9gX20C1SuQW4O8nkoqRGcqylX5RMWyUSgiC0i6iYy9tLagDoR0GnIvKpgvwrCRaZn9P5c2jpzeUjtUEQv1hj650t+UUNOW+5nwweewcJaU0Ki4VUtA+nxUWSvnX7ZUUlQFfwyUtK+PCI2UPJA8pJ4IHiItLUh4m9HJ7cuv3H08Uv+LJvWlPWdjBcTRQGPT4kwvZi6ZuaPreFR7264lsp06hRlwNF5K8bYOBuhD5MmzPM8KPpmhgqTU3lJ8AXSZcOlpX8WYS2NLdDSFxBxZTbIBH3tMwjmt2zLV/vY+RNXGrhIllfSnbvTz9cwvGgn7tMiuAvDzzeMZ5kpUw30zPvYMK7wffusSG4Nd+C6EG4zSWF6JvU0cqn1HV6ZuPOdAurEgZZQoG7ApdW535B6VGBUuMMP59mVDHak2KsSKgC/v/xmMvGE/LafPpsXmxwkIyj58W4sA5S2oGsD0AP0AfdmKCkSi9BJLt3AAaBwqnQnkRC5ZtX7NIXabJTeImPgvp+uJBW81cVm4w5DiTZwwoMgJ0tmdIVx2tx0lp2mCkdYEkN+CTSQvBfZQwS89g5w3tKabD95NZJeyGieuvuu8IE14P0gveodzZqokOqc9lZZ9p0wEUJmUPiNR1VbCO6Alj5bUXyeSTQdEaVv6GZRDtLQ/Pt5gx2wiLFr6DRR6VcO3UOAKlDCRZcfuR+e2fEk9jrlIiRWg9KhJzI9b2MB0TBwK90x7k542s16Yqj72GbDdwki8xgd92QLmfig+5goCebPvN5hX7zJ/ajoYk7OfvzMz8K66xv6GTwy17CKNI+6AY/WsOaC9vR3OZreaQvmvDiUJa547cKBzHAVEfWRPbLcCDTfKFpS2dn4CCxy0zpBa9RNfkG9i5J8hc7WR63CqXAoU3ZKCIug51Y9y3kIbX695ywsAZ0Zp8y3tPiFUjL0GuIU5mWvGJXWxchy0fYDavcH5ntI6fPICfyAKTYf8SZr03KPn9DyJh7sI0/7WNqoVcZ6ujCsDwipZd7OAC4lC7rs5NT1oEQu+gDMj8A9NxPcYqPEa+on/c1Id44TzWl9cDs2kQfjy5R+OyjrYDr73B817CYOMBj8ygHr8JlteaLj2gisC3XUq0Y/Dr8WOPRkxqgOoFEMuK7yX6eDToSb+QgAnSLDG9OphTGx4bG1o+hxuWfDL/iOToYmvXjKmbo4KOweOxlBBwLJ+Sbf3Nxw1ViUA7hE6OV5N0S7lPpkDGWgbWtk1Q0htd4rv6czwDIvRsfc15Y/nofPHSw1P8AmwsJqpQsDw4/95JKIw1nEsXp5W6QlHf8EnRYlxFms7cautjjbQUgry2+oR1YGqhy5GB+ywcCnnPcrbEZ87n0miRLIDDBs4CvecMZQX8qhQyfEtIO+mlHKdsM5lBjCSZJNSkMDdW63I/3FXeGTcQt4KImOJdErjoV+RELbH5+v+hQV6kbrko/jUx/cNzZp7LbJ5M3Y2FN/REotpdQKecxruHDIBoAo41VIhZuUwOkW7kdfth66WJ+7hLHuzK7hw5zh6ojK0wZi29wQU58I6QVlmCG5WJ+DjiQStrpUvktE9l2BjwDn2aLIGvij64cX2QEMa3CS+Jpr9mZsOHA13XYcoWsVENO7ydOIM3CtPO+lNElb3u2U6ohzfhJlW6cVq4U1lJ5dOA8rK7otpW13bJE9ox9Vw2e++g1+yXki/VzpjmevUuInU3aT9BS3dj83pSJy1RmH9I4kQLvNnV+ewCsP2R2JxWzCfamwKcwNPpSBAhg70qvSBl/eTjbbRlWfqPF5VuRCpvWKe92XgiyB+iRgow0HFROVGQ3+OJNKRN++LzQ95bw8+OZBgQUhxdPEPtW88PMzhBKiwxz1gyLvDrXgXpxA9nkN4Y6d5XBAuaCF8J5EW82QaDn1sUnG9L5srm6RZWM9LHQdG5dYSmfoTwqGYOhi+zvp1GRCgea2HbbU0UXHx5PImpk4W6yjDd4Yjsclm6mChjQpn/hMb1fILeTzwIpV4CW0LQoisdnFbCXf+matPLoPV93J2zCKwH81s0vkULr3kj7WyX4OHQ52ECbRhV/kzqnt/Ibgu9S9+g6BrrY6ueMhji/GaD7xUxecYWQmiFZEvho8BhtAX1uQPYYsktQ/10Vxj49UOUhi9YA/q2Z8kCI/+LtdVY8dCuyZs2k5IxIfJ3T22lc+DoC/lTv6XBgrNcn22vTK/DBpX/mAWLLLkvYTEX/LlkVf4iCHUpLTaZRwGec2lAGtbSugA20yFDeJXl4f3faGDKcTHWRnX/wiykdMO4nj2R9/Gc/eOcCds+nOTMyGogOo0EArRppJ3IGjYBePNbd1l4hOPb8xvlhTBWCRyIVIHQxn/2Yljpo0Sm4A62AU1IDRNI9J/EWpDLMAIKXLlMBHAyFGMb4JqhqKnH+F3aNDP2+NMorZZsvAuqKeoHHSebnLBSP0Rg+Ztx1SJjDvimbZtyw8W/WaOlMRXwP3eW0aNeqVFjUYpRl36ss0M15BVinXyTSb2m7tz6voF28y+4DxPpLHQE8Rg3HOebuJtMmOnV/bQB/1ly4hGC+YJr7Uh4d5hSiI55rxAwTUocWArgTgUeE7Vz44KxFFuFYhFn1A9jZdGr3IZBCXI90kTjrBRLedJlTJnWi8rrueJocwVokTC42SQABmcd19pF81bOs/ZlBydKsmhdDbDGPDjzpo97ALmBqh0RkEOlsayXRnXKmwVTYl7lqZccwu6tR3qZP1nuQS7gJvFy2LTFNak9HwDb41KUb8MckZ8gYQ934H0Yztav/oXOJADYGxYWmK9/SZ3aRWrOc7GB4txKrEncY/FPN66KLTPlVdnhVDhT2/CMpPg07pqbnGqjwyCZO1XIv8Kgq0GbbS857R1K0oDNZYD/xPAVi6eJJiL1TYTdp1zmjUSw/IdEcV/0WX2CeJx+1ROulXz2oGhW34cq/dUoMaDJWdE4L1VfamFRUUbzLP8o+zV1GB8aguRbEd8MylUthz2U2pNJEXS5LLqQf0rO2Ml4lPDINE7U4QPaM0emrpQAyw5vzAujr3Up+bjA3Aroduo7VOIMM/fAd6OHS4Yv0X0MNo8SUhNDyFnSGiCTgvRyYCMUCpeliHdbhyjYpUoDCHK17ky8RVe0MAaWjQavDTVx/rTGZtH8XFgnal0GuCVjpEtU9Tgh8nVZgAd4xkozLzqVSS5VzgdOuXGGexbwxPIdrYJF5Ppl3ryLVHCWyT5gYLOxFRmNTa0IGSyj7kQ43hsXSDyE6ZnRzwAUIKdvr2rFaWn/TkI3T1KRjxSKkgdUMG48H8dWaUWEpUK4guiuMbP9uBfoxbMvqYs0kpPO4vk5SWFW/eB4Z9iMi6gpc1PfQopF6xjcnBK3RUYh3fBrnEWLniLs0uu+KBbDLIjzyrFKbDX7lAb/AntxqlFFiRvU4dQUvWCFqwtCnLmZy/86t6seYneV7n6mvewjdBVaObr1lN81Whyiw/EnIxPHyc+zDZeS0io6FOKog3YAe99xrnmzoUsotcG+hlXZ9W/fgWSY5wCZuNLck3FUlqgxcD63F1Wt+vs1IA1XAbdku485yd2Q/e42LJVF6CFW1i2aF8cVPMtgBpcpQ0c9UjlbE2dW49+avzdp9IrCcbXnIZhq/HHajNTIrWZmmUXZ6UlqDlJayna/NsWmVPkRWIJLGa0pYdpwo9Px41B8XDqUfQJtuvqShJwvRDBC0Ybf2SXyqpDeM33xqtXMaPk3wa7aM/F+2VhV0lM5iF0+XsTYL1KZ7xxezHg/tQigs9rNFhIcZ/1GNk+WVJ+vRna2oGNQgM/1pSpJwLKD6oVM0ncbLAvRxMNYAfsWA/+LxQjPWSUSpWCbAkKb9EUIHEbHuJ75bJ9CGTWWnjghee2eVF2O7KGEwwIN1JM2znmfyV8wYrHXgOSjl76MoQa1F5jeCuq+OvpFHrzKwIQeJkL0Zz33BdQtld011WpfTmTFkMI8txeHYktW4PAIqeYHrW69PGpvOJYmzazO1bYryh5b1CMn+QyGzY0JHzUuWmyqM8pk+OZcM0b5DgPcC9iYI1IvFHHz5Om667bNvI65qiiaaJPHwb2rrJ+TTQ2zLYZ/HCNHtFyKi8XnBSqCRgTeJk8taLF+6Q3TRM8VKh9p4RI/F071spE9kA5wejIMFaHmcjIWwLviAdrE14TQiJl5zNI6h2kLaBceNDBPXJkU4v+MH0OWdZa4Vmm9bOavWEHExF2SRQMxtGuYaQsidRplWx6HsoUFizkn0pNE2VvBfrpkeov8rVaTx4FCiVmRCDhS2B6ZWoVgLh1b2qGtnfpAPtro48okv1XuL4tOSsYab4uLJIreQ3O/rDdcZ5Ac9isBIwd+J7rQK/0H1MVDk8W5ZWFqPKrd60ZW07h8cQE26Awrov5KkNs+c8BVqu0+0qMnHMDbir9swrRWL+SKorr8AQUsVyTPF2jvY4oFG1eIglTo/FyRfhYUuxBan+uHLhEVTRwhc4WLTGd2lJyaTpYfjR56Pkdgd1u8YbPQmiWkM9Vj4Qk2oDHprwnmo8M2yY7TrcyOQ5+64SjfkQMswWTXlXNx5F4fSBPxUMkl0O7lEKiBLVMX4q9o6nOWP785t7oQhY6IMAGsYCY+1MxcjvCUWW+lH6v6iKYJ+bQdyPNxRKzjv3IThacRwkNJd7zTgXApol5RaMuu9X8+xR7+LFNattWxhHTp3OL22ZCBox8CFSjJH3/uVF5qXEqxPSrI+HpRo/vD3OX2OBkTK5LaibQ28b6t9X7otEYQwuBcfw4xORedu+HDUS4YrBHyzuE7BxhLsCxW89sz474jTxF3FA38RMwVXWYlGBZ3fZ1/CjAEl9rTFcGJH9Y6yV93unrRXmnqDoA6qAxnkL08Vm9qkWLRUEnYW4WVVfMO70ChnbzDg5UBRwR1WVvHe95IPLy9YgkFnlsxAf9booreRKKsRzlPyR5nCLNUbysS/NRvx2nWGEDIhmZfcHxitPqPrUDMsIvHsmIUoAqfUujh54UnDOR7Et6fUmH9vh2cxew1WA14MV+x6MB8FOcnVqiOiNsu5+s8w5C77a51MZkW4w3GkXpEsepO35wqQzokOAjzxNxl5qYVmEEtUjnufiTlMOuHuSA54OF7lJxaAfNWyvovJn+p2QJtp0nlLOIU1288rd/Lcw3+09QRckCTfsHbK80e9BDztG6/fr/cSJFvUNGySY4ZT9RxGmHUqizrJDnZA87VYQZ3LvZTH/3oYRyw2a5e1xLahK9V3QN5u6ZudmagDAwyoDfI9VLDbOMucjEYMYv4/mKFDpOoLrGcfkk4CdXoJNlCuyk+Nyg1Xf12pjj8OsezPTtKliddoQZtgLXh0gqytBC+xTz6et1bBiJNHXUadAe/0cbr08ucrvEL6n3ypSXOKBZnuZwtXxwT4PQ2/QQs10MWlfOFj1mOnR4nsjati7d7RydtKkxGOA6AJE4JExtZ9LIWl89l2R/Hw8iHWjdwE7W3WAlfdK73gh6A7saVBCfaZcX/03UJPvKpJ8wguHMzqX2AtOGKXs64LIjK9FLcO0wYlTvSqfpoSG5ZpoKtZAHkAul1otnJrxRaRlUIZ+t4dfgX8d/Ak6YV4BysloZh5SlscvEs8H4RV+u0bk8QKUsEDKLdxPcI2s18pJUOiTRit2ShJY4sft5TFXfMP1nkuIUyYP+L2n3HRyCzy/KthlX36W0sJef3h7kadUoODRYyFFcFhbzhc2cJkoPEx5ZhkM+EJ4nkf9IQFjPOq92b6tYGIbPXP2ocwHx3mkaoXkYLXV2TS6arwrSA7EveWYrFLl5wG0GmBe4cOYb5F61WrVCj7cOfVC5HrD3aDN7/wz3D76+CzHTx0ENKSPfVMWuNu4Oag6GOjj45TM4f6z/Xg9e6M5BMUWIPT7m5OL0mahaCPhVPcw7MhpAi1iaJuhcMiWwAPXDK7Pgt4bzUpRdEMzPzYRYAeu3R5nNtdL4saFfc5dIS3dtxjAJSGu2qUEEgbEE1pBW+csvbzKYQ6JVNzldcFrugQ1ya+uJMBF9hbV+mVSoMJF+qxkluf6JUbzOQwcBy8iz+GeKqFlB6sjhR6GcTN5u+ZCBEmJtAoX/lOu26NBVzV7t/kjAUtJ3/E1bF0uKtXkOVl487cRTiZfhaNMJLwbNtjq8IBWOYXmx5o/AG2NB8HQn53v+HcFJKXmsRu+g9Lglq/OJvtqFqpQapI5RXJLRp1kMFJPWNFXtsLlmXxJNgwoUFCkhiblTOEBfJVADPiKib2L0PIcsNp9MWaGs5dxbWhgDPCpOi6glTPWoOPrDw5iiemBSvl+qYfrletMULzRnq6MDOjXYv4pvWvxumFxFJomE4V3/eQiFc2iIsRA+w1U+ayUGJjU9LY9gjkZRctvrOlCO9ko0XOAaMHCqZeloehpdTt3+X4kjbdbEtjO3rUmgT1d5k7akgXYrkhY+6EWfYURUdENIU1A0Fg9PxLdcoKSKSK2iEug5k03CVSk2wIBlw83UKE1BP8bDdHTjEcg8E+veejDoVDRcD0ueI+uh3NXjiKX3HwvPwvnToYvlCZdbDsoFg9jLbVwuNtJyTnP3emAYXamh7TVlGvAm9TNDIhwWIONsNIhzOvy/LhCtHFuQltyJXTM5j1NKVdyQUsgC69WTEdys+Hq6HS7BCh1FehmECTgZfyuRPADwtjMRAi6uEDvRF9B7ahOb7T3z1ZUBGgt82NVrAfnFqgAPJ+YnK03U3844yk+Q2M5yESNegNyCedlCEN3XduKzVENoWAZpsh0Os9xSs3cPLg4rIqvQNrLrNN72pTJGGO1QPsRw0g4GyftoGNzJ87JBZYnc5ZS/Av8DBsLPelc2IEDbeaQIrY2dtRCi+wxGwksU1MCf5zqOKkdMBV6GkmX7zzQGH0p00WaERyfooCnpCyQj+uhCf6Te71y7PkurJo6vDs2Wc82K1aR6NmCtpVec/tN6RvcsMyCbUgjGdgmeCcMIik1vWMkxooiFlYqs1QLU75qCdHz8jL9WBVAadKgy9xQOLQvgE/MqKb1XEorabTKsJRffDh9HarlH037vjhlhE275d0d1+r3wELAXOzgLJIZx0jjfYBQVl8J0sEZ/PTsHFPt2B5216eFyhr5smNiAWXUj/r6QKH53L135IYbyGnofdWocsLMXTX6x76I9jemOLyMC14INj9cvVTrPFjeKDZ5U5Q2oAFGDoQjQiqOp3saWgz/yQgguIsh7S0onZ4d8rI26jPVu+4mhI1Zl0sxdccnN+n+bCwHKa0y9EjQIvgqxV37VfYofhFx+Sn3Vas7zxo7n+jwnJNHcoxVxDshIp261GMN7eTGcRMOB+dk3lCaq0NjekrUUcQepvyCBCe4M74UiZJ7VvtWXAHc9IeRQTmPlU2JH4Vaz86uoD5uF2YKUanZYU7HqPw1ykfEN/PNTF58Ij1hKifH8JAnuYd6/klIGxS0MV4QmHDF3bDy0+SP2kcwSzvoZwOK1XwRGIvIGi3Njju7ftN4vfdH3D8oescdyXIVeuphtTMd7yDBI+lc0lrl6WNJ4fQ+2IrBCPN0M9jZs0SzFU7R6poUzI3soBVbhPxGrIDwgwSLrJeJRg/zLV/YNFrpR940PVDH1gCIaSwjNev62NoJqvUkvWD2M3Uldaqhm4OsObCiSsx12F2ASxbd5icVXygjALnJTfz1sIuxwEvfRldYB1/1Y4PUvWFxkb1QwyIDKch1szyhxAlUMOEL4ZM8EUNbtGvSJGyFDXcDLg5bjyFce6m+nVGJmqCcS6ialz7YXGRClmwQPWFio5bzxAUSZyTjZALhYY5vPSTyMFIqGh/KZveVpZpQGDc00T5ebatMZLXWjvopyyOHPGo5LT8cUOIQLEjZd5UQywt64mzM56syN9dEUU3fpkBT2LGeQdpXhDMQ9Pa7LN+2/PLkLcp3dWEqP5nfDIBV7NkTytCNAwI/F0HlcT9xq88ahDymz24ob9jceEYwjeZNfJ7tXZVdIO18ZJU1OGFGGw7AUo0+QdcgCPU1HhHFZVcy6dmZlK6XqpVwClIrXRk+FdsiPJ/GfO69e8x4r8/FfOKoifhWsK+E6bllj1qHROAKzJ75O99MPCW+1EAtAvHM7zBFRvAln/QbnsPnY1zc9I79emRvaPvQZ0OJvMzYzUOD73vCcDz8QMpLG7oNZRL9lNFBOuyjuFGE4dLTfdJdG9lbUT6VB6THQFzYNn2waKQm9521Z2rPI4/h4NXUJ3Uspgx5swe0OtnVMpKUZO950OjueYV4hk8R/I0CHtI8lPoDTlSPmyKWfQ6J2RXGNHlaVUa5q/eTFpSE7YNZq9t2WpUTYNyY7ZGJGOBLLSzVCC+6qzWrK0mzD9MyDY2rN2YJE5l9gkknfIu0vcmG63shHnt3PqeRb6cHSwFuvtw0uI2kmDGDFiMP57YvdDObLOzigXwXCg4eOBgP+i+FB/v2u2rVhEzgu871WSWEB/kAZSYpowXoYjwteVD8Ic1AV+RTKU+OCw+9VusYWopI0kkyR76+xoMYXZpAQwHKIVRRQQbrnRk0Up0Bf/FPLvdgxqE2yro+zKhXmfANRj46U8+cudiT8A5ICbbjgI7Cmx+WRvCgzh252pdCUqi4DOuS4bouGM2MjwnpuHuGeQUebbN9w/2d9UU5MC7+hhhEYgSIpcx5nAV+UKU+qd0FrRKUDWVva/kx4845FClxQ0mY8ry7Po+5EG0VRMrkku7gMwMpezRU1oE8XBo5zb4JL4KBVQvyBRKuXjocORETz3+7jdH+zHeBAAe7KX9OibygrmGF7a7Lvn3lmL8+YyjwjvFpgyR91m+bVU53hR068mhNpmCUKDhGA9WfLx35hqYFWtR+LwC6HJe5Wg9j5vZA4Q26HuRGEeKTazzk8iBJqPZiEsb4mTkWf3szbn/TfG24+a14sFFwWsTWyphigLqDJZ84ta4HNRan+Ui2VQb3ug6IG7TpICZDWQVv9hQcB5jStogAzuNF0tFaNHr9290mQ7Q5eSZvyTFhPdwuaMkvdNUlRi8LsMDU3uTyvmwBUJIEBqHQzIuym08C9k63oCj8zFYDOtlsBkgEnCQ7tKGT5k4hG/DKaTACbE1i0B84OMgAfWASwx9mD0RRmmRnxnzy6+xKj8xlzagSk6QTpPu0IkSWc4QVFgC6N3gIjolJWoM9ETxO6Q1oS/WTgVCY8QTwo7jsntRODBEHwzKdSiahES6w7ilXMWu8d01OERQgTkHvPewiYVScO7q5sauezXaEjDjJAXp7dktIPtfsMgjGK2JynqItHQAl8IAncHdQ80wCsOG8Pp8yiBvxiqJaI9EuobfOZtnJnhmNr4nGsCdWYGx1FLxk0CvmkUtgJZTYUBdosk7BkzjV1pLzMunGfXOoVz85bt6zmmhMn4rbSLd8zimVnUyVpSFaBbkqH9ntVeHuNCjt9pCASZzHq3msOerQZcqsDlThQj7ACRHUupH+2vAWD/nxgSEEm9XDp96t9wShHOociN2jtDm4EEpt0hXvu8+u+dvvn6Ti0GBJB5iIWbM8EqIufmtUqpZtEZIfEs73LyxxXzdx6aLZZ20YuxGPdReEGW2jf8mioIYo+iQDWIXjBRuCu/TiFGXtnzDPvhqYn7EtoqTnoFwch3GhoGYFm/o0b0uTkjPqKUqiu1fWTnpcK8e5Dk2+fkREZnJPzDSO682YuH3iHugEyhNHjdUoek9+b8kcDagyfvukjdSsdgSGvlQRPvKVAMq54bZ+ZTLohwRhgOY5tcPeRppGCWUi/8XVezL1fU1hT2DwVFLDUutrCWF62dbAN3lqD3RN47EkwTOzDi6dt+CjDSh/RD93upEW1ztLRnb4cAoBLLDSkdYPg/iDjgCUd7ZqJN6EENo+zNk0kgYpxfqtaV+dzq5fLK6Ki35UDNu97vgX3iFpatcFdz1jKbWXu45Qt+7zZsciHNquUODxGzjndYimZQ31N9stbG0NwpKLT92p5KvrV6Ul/UJj1Gde+1e8UuWoG4/xQKD3wCTPp61ytlBXz3fnoWrR3THUt09o4ipGb7mYgEVR4TwGtP9e8xapw1VvT6ZilqWTuZp4Trrf3JcVZYJuSXfliJO7x1ZpRl68F75qilm0iyY8EIddkCWkfXjd8+YSacJxrHgQa2Vqg7wMBZuUzMW0a0LugOHFilJctPZoSFCwOyv0CXMEW/m8Nh8X6GqIV4opNWzuRs06Vp/Gn0rdbxdRELSWn81bI+eTYhC+syz2+emIt33NKG5z/tiSaAFHkRxKN7kqHF30WR8ZSiKisUKUO8TQNw+3MQlS1InINh98ea/TKQO2/BcMkciHOFeuyfDXGgVSbc0mph1r6z7MjZgWhI/vGpxBWJjBH57oJ8kRNUYCBdRiN3J0sz3fels0leVGTsfvb6vvr4+l4Wy+cwS34xDyhoBatosNRBD0w7nn9cTJKaApFsInDZLPEXILZV21lRMMi3u3NwgBxc2Q0z7eg6fkBp+oGB4eVBw5mchorSlueso8xTK6eMY1/2IZkCjoT/AwyeQNeov24V4U8tVGz5KsxYm4ItzQVl99Gu0sIR1WQhdIKEvP07cbw+uy932vnUPoGM82cvgosnIf+0XV39hLAjI/1FFEp3p3feMrXxQDEH0zlWGaaPymnU9sZAzu1UmPoDA6WJcuwKxn4CrPk0qpJlFpsSz0pqdnQ3KRp6d+6nSMnoX0lYPSeR6VPKBV6dXX4qz2evEsDt/y9kyGFh9JM3HqOizXiNgQ1c4BSjTQpAVKFDD263cWDGszqjK9RojHGJugauYMa5VJlPYFg5xzux8+v9JcyTnyYzzHR3D59kUKzu3pAaLtc7SaToGzCkjHAD0PTLuxiiPKk0IhI9ReRnBX0p9ooyB/egi4+8X8Hn0DKYiypF2bxz3AkStQdjivcAwrMiW3MAjx1kegcQQJvC9PzvDZUFbqtLRq5psr3A5wUZwgUMb6wlLFYrQnuaIYHfufyNM8kKxRJvA8vv6wOFooVEs8YLEHesVqAJHGyvSsdAFBPDdAEJYzjGjoczSKy+V5fU5F6ZdXg74bqDCckj49dOpjHDuGRBDbD0MEfXvamt1O6WhY83kOc79JQ5bS7IfICk4ZgOBFwemL2p/YNRF3Kd6qnbGHo5nL/tkJwqvyPpP95mqSoOlt+FrBSp4oTm92hKvlh6Q7cyf3JQzmMDO3euFjKw7xDZMrfpX4PSP4NVRH78gmGsPfCTJMmj1XSDs8cieCxsiUXXdwfF9d8QRRu0wMVVI2PpHEOlTjYEq3eUDvA/OfIxzUjyozGa9nnKfMhp5e5G3PXke51k3MPtPSuPU/HtRhlLczlKgXPfpVuWKOZZjzyT9nAaIM9Po4TPNcGc6QtRwng3H325irlF7p0trwFXbono2dTfVbbcn+m7qi2pJTH20ewKInin7xaIoSSONNJD7Y3bMybfKLvsv83tHfDDqopRWf6vhmDjeeK5JdGR/bxo+Qb9nLHUj61RBk0gXzkeaig90cQkVDd2F0qCiOMn6Or2ZIbBQSp5h8H6vKCUzVUblWix9MbIqJ6/lyK1nktctS19yF6fmG86PVax750pFLWPBKegLLf0AX8KKKMMxZyBbsJKRy45jVVwzLFAsazuGUlUxuPuyP7WpaFuOBeEO19nbd41N0VGd3wt4h6NtaCvSBk+Y2+A9K8aCdeAlHLXyCWoadDsjv4TYd0Ox70lLmVBGtNZTWqGkxWz29PizFFG/q8Si0V3wEXw0Slj8W6gTCu2SPdzzy0UtFO+DrDbTNpCuL58s+11blN0jUnCWaodQsidDtrC277cdtdGU+ss176nsVf9LDrMMYM/EvIiwwqC5girvKti8GpqO5TfJ/WLpqNemNLfg0NxdDKGbmycQ4Yho9/VXv78CB/Xl3pdaBqjrQZuqEDmR95nsxvr1FZQSIq8hK+ntON0zKimLNMi/wxbpxDBMpEj7O0SuU5j6BZo3dJEZDYRvznBSt8YmjFgqvxdWYN4tvkcFXc6ZRqB4xiiqYqvMiYLrcVSKOa9zkWFjCfIpUc77rgXJHYy/l4D6y8ShOk39GVY+j4W5m3Melr5JO61pidAdctKyZ39z24eirDKXz+vSBnDV0kx+2Mkmd/rXtAcoHfdvNUnvdCL5Qj4sIX3WrkA9brGp61Tb49Plt/Z3krle5L9shPnZSswJ/Ldh7OqlnPzLEKZZ36hcQV0CKpzN0nO8vV0fB4C0a+zpwapkrD4v+A03+2nSQlsC53W6BaJJUjrlOaktokcBOKkEe/AfJyuhP1VhWAV/DZ5ikR4/RF6OklFmRoO4cdXZMVOHYH9Qnio9bPUgES4Z63bAXs9wLwsi8TVGlJTyHQCvECcplCERXwaTA3eu2fADPBZq8YOZi6ok/D5s4499H6HWn4bz2PBXifr+Zgg7zpPOXObkUYPXncUKW1wWNnXTomj3X0UQEPzGSIkuBd2gcC/2yUqLArB/LLaCzyb4CRwc/S1IfySLf6BEWUKjpKEJczYbR1NNCQBJtFDk3Uu1vNiCwZexaOrvJ/TX064RzP/LfN4vjzf59MamA68BFd1C3kzt8VGVW9i/D1XG94VRTDknS/NIboj/hTVMMY0qfb0lsNsjCcLpch063txGhEyJxIjt/mUrNizL1eOZvxF24Hidwxw/nmEgoZ2EAa2/qTQWGaQ+gBZFnf2Kgf2pkK3LG9+4ho6K6QNTbQCb8ggBSRo+IGwwbw0rKUNOKY+lfJeFD9pKny29kPDdHOyc8lvlSylXp5tfV99SLHEJz1nlAe3wmd/bKwio+S7+E0BWNLnUgul8OnUOxXeAEHK/fjyRe7vDMv89KyuelKgNAtvFq/cjIQJB1zO3izMXSu2myKbxm0siKMX+yLSb0y88OIyU/vx6Jh1R3vDvO8s2L3rQAhEtBgiLV0fej+thn1XPeT+sIDA0Wdqp4iqHp8xGo3Ss7UD3nvI4gXowG5PNh4mgJB072rMf2X2nyt5lnVuS7R2haX+a9xWj5+qVLlSxNtUo+LwS/sMbjk75ChA78lJINRgVTW16DOS87mY2u0ln2bq1RF9vmeqOosFcEXBJR1YXdnH0vDgyksSmUxbVhq9H6RW+vp0zfcpiLGzbhGzOgDhd8oGY1P/QNUvSPEegXnbkHA7fn8liuEKZ5EiM03r7RXPhErhgXT4gZNWvzLzaQLd4FcE0dK3uf8spD1xX/MMSExsHX8F4AOY08dIVEAHSnn0FG3WQNtnZsPoLbTOESpuxTklfxC+FmtHHAN/SxvH2h5lEfcGT/PnhLlP+KBaxToeQsV/T5teLFZgSq4tHyfZs0T7e/dmD6eAJ5gg/rB8qSYnYVzI1juL6g0fDH+efta498i8OBiTf5UBFdgxzH9gxKQYk0rBE0qIyed/030rr9FUCYgh8sMxCtvDj3bhuy4yV+zBT2UvOZFQpDn/TaOkAV7R8stTWb/l6IZzWCMJ6q76RqNKESy6nWiFBCQEp9Ve3FxXOfKrwzteKZTOEQ21FbRz9HWOQi/6/XYBMLLO1ptVgQC32jr/747byj+y7FEr+fp/ziVb2FhArOktAYzOUz884nNjXkHDPO5J3q/XKfl9CMO1f0OBPN70tVir4Vptz1HnN+j+wmnGQE6LauZFmv4CoqnhEFPPkeJGBo9+djHJntryDbW2+I6DbDRyu5iWjU85mI8vO4r2JdMU/UsWsQXalr0gpG5u7aHwmgigUdT3xgktVhLHjhAsUzo4mZ5nn8RO94yh36SP7TE2XxIlKKOND1TzvVIbaIotQ5QAwZmJxfGy5mf67kwoXS6HKRgudO+uNmQhqNjaPZdR0aLiiGFtGx/JH94RA3KEvrAShzsErjF8bMqNEO+Ds7dGXIpkvgw+zs/8L38QSbhm3Izb99RhgyqeBfO8Wquf9dGAMGbMMbqRDqT1plZuc5d011BChGLgmEX+fz/dkT8W2+0wCFdQXy4HRCQ1iISSH8+l5PpWlyoMkIOMf6FoLFwF0Kb+VtCYnKEnqdKomSpnD+g1rwzg784+b2ngGgF44HWZUszBPlSjO/DhhzwcJhKROfcjAmQ/nV/w71h8evZ94JU4Dt12xFSj1NS+uiMXEsl24KgfFX6eCbEbW5ZCoPCjMDobTbahapatCeK+AaUWBTODhMcDRY4MYJw+pU2x1B+aORJ8t9bNCqw3yB4fkQmxWVkxXvDLIoqP732OrFynuawceTyzIgrl8+GqY1rccXY52uPT+V2jEE0hbmaz2sXGp+8p2lLqA+r2sh12qJSwH7zOBumTn6OqI145gBvIxxkMLUyqZzNOE4PR1kaMWVTlW2EW0ItcIfljU42YZau/NVfuW0Uwx4IsNkvi5qs90ZzxiSEsyoCjJpGZd5FLvE7D+izL4ETln9LayCdjyWoct61PpQBzRzMOOCgPUyptNjKXM7GcieY6vX9ZVDW04fG0vOXB+MXWO5RFA3PxgzC5VV/7gucACf+IifbyL8VTTxKGcn3JLptSqufIR6Bk/TQCtvB2rGVkXu5oy2BJYu8mDfhC9AuezzL+eIlaJFCtmAMIj/HKtz2eWVt2vD77zIPi+bLnxhvL2JG9IlV8r0xUjg+MPZl22TA8WHES0IMAHK0lEmQFiuAc7cZZAOagjE1Q5uG82dL0IeBnwr0prN6JEU+bPvkzVt25emgPZfVj2RCQVkDEobJ1AG3OWdFoR5mQipjaF4JYSZ5rbqKH3lMeX71+1qaY2F+PzgaShKcKYaPtbu2qtitJqNctftqf/14GTylPVOJNEyTyeVpirc7apqjoQ9y5+xynW88KPKxnWZyWonSaYWcvi0EITmgectvQbpgYrghd8mBPWmsIWhgvD7PRdxpF6oU90AP1WajEYWyouq48HL7B72YjecQBxb4zlxdoYL+zc4Q3aF1+efgBf731H16Se9RfwKXoRN+iO1JQwzPL6uVK3CaBKt+mgxUiOg3X9IXQi1uq4jcD6HQY4kG5ogC5ZpHjb4s7znwSK3NIRQ3w6r+Nsn5T0ZKS1RX0/3zW4k7dLM/pA+8sbVXl15lwMy0VFxly3I1TRuNcXgvW9aiGQa1No/n6+VDsjOFCBjJoL2uJGSs7YulX7tKYeSCA5jSCdfOH5kj3cFoXqMbUy4eM4JcFZ+npmV+GjEWy+CYa0QBWlNgKeJSlC4mo3wwSnjZ5yN07iNsz5FmQzVCQVQmeoTaB8EfQaiVWUb8VnYDHV4TaHMwlGn/qU8s6czD6Gx9foiLn2d4KCLruUFuck1hNiZdos6jLrDRyjZGaROi8E4BwZL3rPGNeW5fWN9MA/aUxY4wASjkQsLzbSSRPcf+9AE9+W8C4FFWGrWT7zP/X1PjzgpABkMDf7GXQGm9sYKSSvLBk0z55Fj2OZLFvtFPNVW78LcougbO7m5is+/pQZDy2Pje0YtOWyrSTnkGkQy3UuYNGntX/HEFKw0S/fdyVB8XtL9NgJYLVep/xS52J7s7eVjB9DEdxIV6vJa+rHl8hYNpicrM5ccUMkY4RFUstwlKC74GC7D+l19wlmsxURnPEFl/dllTxYFV5XX/lbFQ0G8bnR29w0VCHm8eKBW9/GvdVDV1lV3xDdmMIxUWJzbYzmyr6/LMMwVh0KrzAALnaAzRpfvA8vL9fqCtBZ89JmrIUb7rbsKarBoYWgBVIDkj3wmPBvrcjdxQ5Aoq2ieGHQ+Yine2qYaRh5ZC1AFBO8hP3Vmn4CTWwwXhL/oRIbHKYJGgi9HnIQ7iYNVOD5qS5koFP/G33N7RIZu+J0PbU8eTJBqaypIenrhWXVSUBmoa4Nrfs/3UcRgG/+BCRytyZLrt+8A45vGL5Bh8Xt2X+aYxy+agrqQnQKtxWZc1pQbjWtASbgAN4vGHfP567Cu6cKXbl5bVj6QZz6/Wrk956gisruOqKDWWp3BBj47uu/eBZMttIU5C+N9tYuRaOYLGiKP6D2Nir96nc2U6mrObiIM0AbZ8Ym1G3WtYDzDiG5x9RDo2ZM2ChKWAUL/+kZPeSdK88ucNeYFkR2xU3nNETsfKae4gwAT68/s2ELmm27Yo1jrvXzCqnQUxD5ZimgdO5kA4ASpW/FH0XiudZ/PX6F+0KguiRg9tcTJx+3vupcbspho/ZL6lbWHcYi3ntN/NGbWc61kEd0Sf8wO1DMy4wTSZAOaaMUcmAs/V90X/UFwMWo905vfz/h81ouF/4DhnPVu4JbtrxbzpgByMPgtN0SOH8FDsjv33xQ/QQImM3wWVQexYedP3JDSQjYi/m4AaxKmn3IZDw58WhfNev6h/rj5vGaFlruQar+EOfMV8YgUyYriWMlrd/lcRSLi4S3oZ/4V9xjxB0ccxuAIAl9K9Ru2w8y79JMpkIsH75szWnSixVkJKRItgbJIpiYd7Y6BC+XE3dWMJ8U6as53vliOH8KfVBPswD0Li3G2fk8jNOTGwt3w/3RhY81x4VhtB7G6QDMzHF4a65dfquQI0+/NMIOhVvRtDYPbm5wV2ct40W8kIg11L5hqZRI7m1mgmozdp8ZxIoWYHsr9q5Cg9X3w9poQfRlP2ih78uNA4wsM0vaUuEUcfp3g/7WhH8MCYjj7cfW8a2Z0aAhhY2rf5xlvWhHsMBi3YzhpItERn2ofg8RNc176RUNXoG7LoRUvVijwI0GaNNoWPiJloB0em61adssrH1h6CrA/hDjLg9WfqoA7+7D0T6jN5+/zUFdl/U0Dxdpu/ohLdLqVhGCRVpNWyNmCbtAwKuz0ShVuU5B8kKTDmGIphjIaCI/yZ1tw3jjXweWZ7xEqjnbgCB0Alsp+5JLrglW8zwvdHVJyJLE4PBQ2vg3T3NrLBc9kulY2eV7mVLYX/KbKayHts5V0TqeV/RKefTgos8ZBS652F/rOcepca0jOH6URluYHu+QtBvY8YMXLis/chL4PNpZVje2X/Rhj0c3YDaDg2nIGnH4ASrL9ZiIZ+QtM9ZtDitclB5EKhX13D78oR4gXrG9PO1Z+6T5FVSyDo0EpQ5nRhojaa1ZfY+xHoyDGGsejqLtz93myddtFrSEtX2+SELUmN5MwYzuXXp0pb5iPbajoUoZeL/iVUeoyfH/pzHt/RhcBY2BbQvXYvJkd6tFTikr4gjTq9fau71MbA/uSTiy7EyjZ36/8A5tXfHcoaKvpjLgqvrcSlluvGj59aMxnCqyzq1ODs2ru2pOyTS5A0dKV/2zuF5oKgqLsWPSAIIOJ3G0CGZZyY5SilaDeKGukunqnkuNrW+iH3nOJIsjDjxGCeKKylhZDXduJ4l84DoXCnM3U7n/xHx6heDqvCrXtrC/qjDQyZTbovb8y8F4eEeErbfuQqN4l7U6rpyPwYsNGnsGLUMguFCz8qS2iT5tak27FEe5cqxFHmOJAGPk8+6Vl5zWY5LReO4mvUr5zhmp9N1mjlA6OhBtRzqYqGjv4CKNbulMwbLePH72/52zfHJcQXUPmNPa4TOcKhnWXkGmq/EakwjOw5ZRzT/eA+RZ3fhVY8bfFAGIeX1sGidwtUIZiB0Y3jVo/cxZaGJlN+C79Hr4muKDTsfLUI5FBw0WsCkh1YHrOL2/EaMyZxDCXHGdtFXMECaD3idYfI+M6Jl8rJ9EltacqEwBjqCzTxO/9tI7K1i7ZKZLvc7SgOtmdIgoIZsZtnz30i/hg9NoRsxNCgfQRny+OSe7iAA0I2YSXlNz8FmvLV4ALrY7hneauRXYIK54q/eiRD6JCPkNcNo5GfsbPa0323r+o5nQfzMNPxFH9ANHQTuLEuz+gw3+//OF8q42EgAdANu5GiKbU3vjpVmBH4e+3E3Jfe0fNGT/lNsiohCxUOwvqWWK0BxiMxeRRjYTeRtt4p/hVcYaWG4eWAoV6210xrEoEb3Ur+0pj7mvXguluQarm40iWPr6LrCoE2R3nBNNhwtN8a7jhFbadUa7pzt1SnAniIbfu87bG/4qB3X2hhqIhpd/wOPHiViKQAdXDy/kkDdlOBhh0lXdFpnCJSsvS8bqb7HJ6CLsr+343xU7JERIIqafLz7WGgVS6whSiuLho2lRPSFXmyFwHC9ry7RaCcqWHbPZVGXg0DvmLf605jV7/f7I3xTN2W3AM48cy7sUT2yYQeTTLqH/aOazD8bmeHbPRfdBdLH6N2CmgVr0PrTz3zLh3bSK+Qz6s9FxT+QWSN7C8pXD43AThEDr08BRKsiA0mzzo8J9byWE5VA9xNYoCeNsb6yVeCq04uLO6ssKRuPgAWy8Gg4cXxbrGBEtkBi/4S+voIJjfIPbLQd5KK+RQddJBf0fo9xtaHEGjBAHfH6f2ox/6kVFz6Cebf/jku5eg/ZC9CTa1m6SE448mKUdwnIqVKUyymAnYsOER5p0WOW+rOGkTXWSzEqSRu4lk837TYRTCWTgH8j3RcQNZXxyu3TwfhtTzo1U3R4JPV9ol80GzHMMiKPzYKlg2V/6vVkJupEA9LwmGy4gcUIqb5OoNAvGmg6iCWC6Ye8PjmMZmv34hb498UJDeQMjEQgYGFYHSDg6snpTavRicYzm+q7zGOfCmLOIMfDS2G/klmsqGtr9JpRvFAFEQQmHfQqWLHbU8C7CG4yVYNhOI8AfqE3VS1mXR1iUV53kZrSlG14zaEVQk8Xx4DbKiPgEek/WZx6Hz+iLhQhhhytXvPMp12XnKbdkcLxy8mp0MhjDPLHEP4nG9gZcbJZj+hTnANrpzJm9LQo9tJCcNvNZpNtU3/ltFhRj2DnQP6Oi3bM6jNtqnsnPsb9+5IboefZqUkIdP8wXAE81/QxxjncsNYP+r+buy1djixYPJ1r4FaY5ApnG+V7VUryKC6kIa17xLKxGH/ohD2tmQGSVo/v2VAkjeYLpusG05eExMaFiZ53OHQT5pAfsc6A2Ir9Up7dMxbIpiOBB74zTN6uPHXs+TeRBS2CxR52eIjlb5rUub15alk5r7Y7bTJ8T2h/hqGlD4zxe4ouevpi1f3QaigIbhA9shseoki77/IGYXw5rWgXLikb151i49MLOfmQS1EiEsEqQqz9LWHWitkp83BGb57CqMHiwGxZ8BsoPo3FyzTZZX0RdDu/WFhHPLi2FBls30b0doXhqGxMiM7DHpGQ+26NA/PFs17Rju8DPuyyDUjYuPoPjwzECR/Anjs4YRRuttjyovwckePNBNLaVEGywzfLOPPtcT1yvhDbQzot7n9CSHQpyKdY3kFboRqCZ7bc2rskJB/QB0QkeOXPf6pAktjGPh4mFYI2ajC3XQmxFVG/0bUgRv5IiIK5IoZLMCJRmOWhnmxagqweF0xmm8+Us/DiO1Oth8JXqSsvDFtzWLCs1wh4fbu6RJ+WskuXkwMLpZf5ve3qxyCwj5BhwSXnxRVJ6wEu/P/aOy/Zm/z14bQpMWSHSObPfbM6Y9f3+DAAjoBoyhKLyN9UdN3H7Qb1ow+Ml9qfJfFSsvpUK3J3J1z2RWsfwX/qS9oA/ZOSTbCOXTRvmT+eMp7Mn99aOh4ez+QkqpY6u9tphUvEQZRFArBpn4eLOd8oBKCyCj8Ebu4qUNtTQwk8CNSNVx21+wIIBFL4SARwdmRlptSbz8xT3HHEJ9oJ8Fl8zfhEOjX8y4xq2S3Qv2UWOMVa0vHPGsHqkSvBCtmP+9hkkcqzluTTsv6Z9IU9gkjCCRCsh609cXPK6fl6dra9MP2vqk0zQGQ0I21hDk1wuaPzmasiDg8LfTMfhNEcAHQG2NY+S8ZH/MS7ZdouQSa9wn9/1+mixmRdSagy+GTNCgmeqyJDJc/CDdxEujU0NPoNgsu+EHq9yu6IawgGJ+tmKzVunkDRfbkKk/pgW7S8V6Xj1Lc8KxRQ6jyIIvemcC8YH9FSF40iCzq1OyPXK4dhaPE+Mq/XUEzuEoahRz8fsNNGWz6iMKznBIRyJL95oYWtyHo8+HrGj90seq1MoiZLwaTpwF+qbcBCTTXse/WNiCBGeZT/x7qsAhp1rC1cXiir8y9z4evrfLz+fXwYoz15B1kGnobvKhowzOzq/lVuM6H5LU4Ruu/wzMNQ05/sy2i4peWCEr+t/8xh+fsHoz9z2ZLvVuF8/qC2oVRC4VO/Yjf3hgVQ5cW6NFGOPfweR0VAs47969gNR97WoufFPJFHLecZfoNjPRQe13COwPBlx+efPKQ8uo99kK0W9fBFWBpGbZJnRoyvAVpUufumPDU7k7xhNy1gF0e6WnUiRoQ0e8eWqxB7Yc4dXBwqgxQXSQDUMVi2JBplE//RXCvp07UPRqrr/nMA7BEiNrucboRQyJF88BRadi56L09jRkAvBxFWX0seDo72+nIdgkaJMrKrBnnxJ8TUyXJhr+mVi19KWMBUE5aGI2EaLiorwo67HADUziDmazxDWehgLIkOQo0t7ncD42g4SjFkxcvQPFm0GNlqq8JdLYWTCnpgdnDymvuw0RNEr51ja3YWpq+bmZVqVUUTrqvCfcaOk13llqpRR4rb7cwThorVAsb+CvPUioxs4nb/YKXCCn9xDHBsNtMAhorKOiky2OnVAgCJ7XrqHWuWK+KHsXzfqnVGm9+nF7vu0mXpDwwdtSQGh9NGmftCtNSMikjdtuCM1stiyW+G/HZ06GV7ozxflCxvvuZuShSHa8588Li3WuOkiwpQA3f7apT7ZSPbX/2ihXfHx5wnHsO4orCg7JjjCq7llBvHWjWoP6KKNzFRSgozyJGq0BpXZHnGiesajO4GJYUBKJcUDQl2/c1QZ3PYhdg8hDwAUWiaexshUhGO0LQl9dfxDY9SNKvwXkZfuxCGAdzAeXKvN7ifavYOeD3fJ4GbtduMnib5YS2YFcOrXQqUcjq0T+0LvZkfXuY+v4C6BBNkHQ8hSjNVxDux0tFVaYn3enbtgT09qtFTV+MJVxRPXHvyl9ZZQXM5IsFmTXSj3kl9rZtDYSxmmTQec09YUYDc2cD+sFRxdwrREs4Aog0dRp/Z4wJnxhnErqE90IizQJlHl/m4Az7A73VqTFcBHK4CKful+ix7Md3n0kay/FkJ43IkdZ5cXbnu+nInd/tXzQMG/LfLFE6sNSFjYuYPaWxShE68kxprzF9mLdHtCPVy8GvsZnZeHpRrdkoSAM728S0T1/4+2XO9WU6JV+2KWlBaSo/citzD4HMhiUE1KZAsyEsxir/a0odnzyu4YLXVFUT/D0t3Ft9JJWybR71i5Ov7CrzeLLtWJ/bI/RrwEEAzUVEogc2/n72zQbMp3frrpU5Q6dI16ekeE99dzUaJ9eer6Jnol+oIlOm+Vl+3VV0Ovx4slKQu+kZhTeOSrQ/iYadHrG7pbCyeWqh71TCz00YEPhCwfVqPzred+sUSdCoZla3oUAIbV4AymjIYxATReaM1YYWrObqLTBa4oUdqvP9n7euN9GeWoPFPwPLerPz3q+2ONsNvfyEgDPiTcJTV+zo7Sh1RZFSKvXL+u0P5+jUd0hzm3ogNrONT4e4OzwpDzG+roXTWAaEEZLSqEhI4WBOSc0PVfTm0kpqe8Mz6Ufh0K9lCO9LVh1bcxAVCqkYkNBT9Mo5FrVrf0sgd+zVzp2nFdv+OKNkZ99bAyj1eup/uNY7BlLPQRuM2CxUn8J4JcWCbDwCVx6IWpz5uZlxQ6kaBN0Lnd3o0s+kdbJwM3KVJD0ntpfG98I0per8zS4N0+F7J3Teck988f8ovaJxTf5BXIxHFdtPpvpIhSrsOcrZbrJi57BcA+mPXd2Iu8fuKnNw/tDSaffqE32yR/8Pj54lzaj+994AYwKVGoN80tkFzYyx2p7I/m4TTC1XqR96uxFBdspWUqrBdIbQ/n1qu2MCW7Q+H8+eMI9gkedFnZt9ois5My3FCjFdeVehG43/zy6Wt+nj0FrgRjOJkNOcLGbWt8wEZKVAq3R7psf0GX0Efgbus0af/gEBo2x7APYof6YcRjFsxRISi+m0vD5VUR3P0/MWnQHCangVocxd42ZE9VxHK7QKqi77HF02OkdHcEQ7qiSSiofhtnnkv6uxJuX40iMO84nFzVtHL6YjlXbcEKRjxcNWa1KAPC5faP5TOJ7xnica69S0w5I0xRnexbcSjs7yp6TFP4ryxtQBxE2gftqAcYMy/bg9U/j9U5RPy+BXl77mpWMskDswKdIfDE0yYe5PhCwEX4Ce7UHFk4l+fNogsswU1tV/nGlH+6iTqD6aefsgeXQ74+DOkYBFMzrl70UK7OlRycRIgMQwx03wDqxp9zXoofUhO7m2RGHT1QFXwNXMHGR5uk5+CsuLKfH30Q1Ia2WvgEkxAZOwLWk90IGE2OYTHM7vewl1LUXvY/pwLtdNoi/DeaGUHVTCRoi+YNrTu+y+zQg3FxVx/41d6mx6q5bZM5hifjQ9IF9LX6Vs6O/UNdI35ysVOZ8dLlRSbKdncWIX0hWrGXl0w4z7Ck+Ptib38Duy9+jau7rQSMtnnjN86Vr6P4wWmt9YucQE9esicH3TQnHD1r15wVdL+Bn+e1vbCw/yZTGE4HIbXug9B/9YpxRnGxfYf7Y76lO8vccBczqGnYGLhGLy7FKTcOEL2NMdwutKg1S298xLzxOWvxaR9PA+XNpOPNRn5mgXIL5kfxwbody2wSHmacCkQVjLyD3jO6sM8Rgl/BJDFS6KtQEy643f7MLNixpll6BKWGO+dKuAd4X7at0ToKQKvHi5UFgWA0iElTGK31pPp7HHzY6bQDcgnfLmGvWfeufG8u+UtdBaO+9nXdiEJ7PklI2Vv6YxsvPT2MeEg3iEp4fEZQWdrTCWCa7O8+ly+bbmwxQ40i5f3dATYGbzqwPKu9eWZDod9pH2BMs1BkGJzhGdM7OuPoMliFvylk9dCIy5506ZS+ALHdTwlO8ae8WfjcOHg5QkehTOqKy5ND+xhHkQSCOCUMAW7pr+cn0FHdVB3wHLaipTuf1N/0MG49en6cQ7qjirLigavtNs+Vip1m4mkcOI1Y791+nGhfiy09ZDB0aXo8r0/RLZTRWhhpclYGYQ1uTpNEOwE/q2CHCYTeQRoAJnFAJt9mPkC2LA/SUto8mYgYCiVkzT1VW3tzvZxzuK14JFqvzvzaKZOyPRYxH+n3AH4yUN6muEeoeLFLu5AsAj++cyVEIB6OfLvHyvdBKH1ZI+ipWa5rX8/wKSeGVlBWTrcQ9uP3TdKefFRT2eyzl30UMGTg7Mdss0l3gdH0zVr3+zUxlJopeY4u1tNpNvZAbkWBkO5xEcsyjhSxeFR4wq9A9YxOc1d4SYsDqXHTDYWFjB2FdSUt6IGWxHSbKeYKOFsyh8gyIrNYk4BPGeteAxNv9gr0v+o+fVOeHNnDDQy5DCWtC4q+NnxserWwQ2MMYFba0GtVPHOVVq5Mg4VF5ahZIl6q7Zn+SP+cCUCquQGjnu5F/PrzGSwBxRjXjmOfObemQkr9UlORvd43Ftk96THGiFw8M+gnUlhFy1biZOILLE4bRGjIknvhdZVrXjlnzDRG4PjdwHKcY5KYxggDlpnSTPp7S3d/CbNIHpcvtb3nqbyBGIJr8OgC2Lryj9iGHmrRJjSGLT31AIrgTh6uiaMFF2qJQoeIT0ki2F7Qrlp1rpimRbydFBf5aP4f+kwSktxyp33rE3EBZlcXUZxLA8gR6yklthsfEC8m1b6VH4nOaLAciQShJJ4VtKcw3nXNjarSteajw7yBBmEd0toFf63mZTQAniEV2MGWnOLZ7vTjyW9UsBygEpMz5AS/1RRuE3F3BkI2/TUNujRZpkAj1VQXMxP4oCJt1N++LIfnOdcMwl4Mo4z4Wa3YXDJ8Q7irTLCXs6QmJeeyQ7MZxyYPwdXO1T9gDIaM6x4Pg2FvUql9v/7ZohPDi3325StDrjPbyueaNuNXJbNkDtu2wuEv8dN0jviOOxQoZgxPhFx32Vz7RBKkJ1w9wSKKwxlBfcytlvb/JePCZjv01Ip5xQL4DLZhxEQk1TN8mN7fe1BhH4SAVfRELz+y/hFSVZ0I+u9S6JeJf7cCybjbBXcAkEZfrPivfVLe+qAKgvClh0y5eBNgMZFyO4vw7xS4ANp3hCEqNMgajmR4Uq2O4KwxDNYyiZILeLAJnMd8pLEosIjy4Z7NKaplH+ZlcW7VClpo0Y2VxBMQAhR7ryyCc+g3jgM5+0gDXQs2qx8p8gwJmi9wNuEsWTwJ7dxDqV6bxNwQmE2JiRqYmp35xhIuTvs+mljF7cbn/aXAfav9GDo3HClio5skWsev7+tymIwc3614CW/KKL43Vvm4cY2OITX6J30nzz3QZfsIPcxBe84JnGaW76y40yFpPFwYjS/QkJxsYugOEg1SVpNb0FHqArxuILGkYTPYQqMQmzCQVS1goF52xjye7OAFrCKf7edhaeLLadnOs9FW8l/aC1QzOLKfBwXrs5xaHOTQAUX8dYv+wzUfBJii+hGjZDsv52wP/FbYR8S64+YDehFKdr+xLaxgkDeWL36IAMUntquqbmH2LEYiYoCOePdnv1AXqQkcXw+BaoC/KbcKkw4BtksK9r963cvIAd9i2gx6gtA5wu/XYqnEXk+jGzJ5VTGxWGl2gvtXbhc2vTccD/REzGjdzQyZYeYiYRaEKnDBnDGJd6DwYzGRQHR1h3k+vrcZ6EHH74UjhDeiLuO0/6SYK2WZ1PgtMhk/Pp4X+3XnSc8Y46jZyltEiRy5jLLrqOjBnX8PAOaZY7VL3BV3OOlw4G8SoyzEtFGzaO5NPVGOlL5DetGT56ypxwcm9tuZ9uf200RkQ86rRuQUOQN8fi6A86nLQ1UTYTSwJmxjRDv67sT/Fr6gejjiIFW+D94DtXvr+ftS/TadBaYGZ/eYT7gSx/2TEYpbM78fSQXI94Bm2sXqYO+XX1OWMaCWDVTNVaW+fkMQ6n3bc4tJbyenI7EAKQIcNy/XNyKEDb/AO3c0DCupJyXfBbsH/YJjIPHRgoe9DoQ+vCvKXEZz3gSyJ2nhf/l6ZGOEHhPHFZAShGiFUGAY4/mMEhmf55G7qK9+ryHT7FvtZfXp/4qpDfYPTyCTgajc9iQOvXDex4u5O/QQuxoJJr3So2TzWHIljWV5/AWX04o5+SYRWFxdI4ydwPbZ1t4Pf96G8ASksGWYCpMxFvy71chjNzsSDPj1Dng8kgAeSkS3eUUKMnn5bo3BaF7Ski5H1Y1SZYu0WFr1R6IIYZvj8jUHOozt7BMNbgp+2iYJFGevIhc2tlMe5jGx8WZn6QyFr36g8m4QiwyYHUjGSmvIqgWiqLRprRlKsnbMueyJE9wG5gFwrzyJKIChj2x9T5KuiWL5vKIGxwRKN1yWehzEfheQWhgUXGa4L8PdFwH85G9Zrvl00QFtfAvnI/3Anur8+Kif5g22YTDwFHcIzYrf9faRei+XRWAT8SRN9ovIL44xkRyaFLk+8AG9Y9HFqtFSzHKmqrvqOpNKzUMshde5kY1kGZXGpE6gDuYe7iCxPhANpayG2bedOTljoFgLEz8WyQt5lak1VwcPs/6H+LyBjnnC0JEMMhZ5AjMwXR53CnxeQhRQr3VSD335Zo/5vT65ZxdfVddQH9E4nytiViPbi8UN5GE+fNrcR64ClvCQExxgfQP6dmo7x48/R75+NSKvsQwmDxmAxvOLolldgmgFM5wEE13UEy+z969zpC5oj3ALzKTAa8xpHosKoQ+Bhc8RsJvwlMN7IF+Mudhe9XoxVFpMsNIpYDlbsOGto7fua3NCK88NRO0feFg4l8BQj0QcjM0HOBM/YY9PitcgQL11fWrTCZt/d2/8aZQT54Prb/tyBk3gmNPzdNPS+TW3Mt2iN/hzw6BpqM9fLlJpzjqRtXqhIPT6P/oLKr21qE//bTdXh0PM3ztl0s7Kssu6DdY8JmwRdrMVHNLtz+7fnQnTVMdcgiy6tCXvogtZ9CWgwfWrezM4Zsc5MNucLaPz7Hu1ysXRDN8OBwqVRe9Ty5LokeiWapPPGBnk9W9/V00RfafL9Nr8v0sVCG+tBQZu9eJdJTVfgb2SzDLM9aVRLRO4NHCEPZARNKsDBE6sp/bsoj6di5koNnWvWbdk152/n1sUyeT/LjLWvGRU3tggm+ZkK5cndE4FOuruiiyqP9fwwwXqoatfRE1xowEbjTLSr3LEjW/vhFJ73tyEXwnBn5pdOGfNoDqGwWHhHMMRx4pHJd3IE4p4eawnPS46wrK/sb7xQ/5KfKE/DT1SarvhevJ55Jsd80/e7rQDlReb+yQzpc4PG+7MgjG5eI6iLKKjSPjB4ZLW5njaF9kBs3NJhi0HjpnNuYQdJGOj+++40dh1fH74iGQFLceDogX/M5VHYCcqEW3FdRd+W1kPKHgJ6H230RJnmjosCA19W31EUVwfkk25DFOJ7rvbKr5FGsIVCCnBWmDn5PWlqZZUfwiTMIjOG6RWLD83OPa16Yy6SwXij4/ItEkxatOi8YyQb5+pZ//C/5+6RQCNTsFvf9zx6GYv82cnnIMP0rMohcIubA8NsKNImbkGSkfFyRxom5Z0aj+yzW5QVV75kH/8sUbXzMbxAy7+KV2SmnRpntUV0fM0v6sQRaMYE3DdUk+z2L/95g21br37hGrYc4UUs+lqM/e1cJNS2S7P8a3f4Yu78LenSEjGb0om+Jd1QiZhgCIZ19ddG3DSpoT0D7SL6CcsNr6g4yWeM5NDVXvxQoHAg4IvHkFMwH/J+DLeinpjxufuv+JFaDGO86O1N2ejpFlIs0xgpuyT5txe+rbAKfdEm6ukhoe9Ds/9+TmcXg0LecKWKdgDQN+P/11olgpqdKHCGQzYO0DnZctPosm8coBireA7JDvojzvjNp++HprFhrIN/gf6A0R7S89gwIEucZRzYCPcIb9bLxX8HjKI5ECaluCrTY9z/Lmem0UQHr/GNyOmu5w3vw6Ij7q2QtflR9Xs9x0fMC01s4v6IB7cBSnMGOej1JXMGnvVL6NLuzN9TZaTwYzGOwlQmpOlWthDHGjDS2Kg4JRjoXM/dTiXbYA3JLZ7yAyATxvtTUpf0AMS8UvCL/ZA+L8OJOqU9EDBGZP5sPy77lim5yqM/rXBT1B/+//dOyEck4bn72925bUie2ggdL1TChHgV/W6uRRL5L3qj48B8tcnqxHwR0OLvrosdAf0yY3XY9pcuInuecdBu6XnB+4WCC2WLfMqTrPpqvgetz20OS/GGWN65kuMgXlBac8RLu4rzIsvOWdpR+LoYw74IpyZeSvv+qvmkGJKA1onCUZNY9KwDw33zIf9EgvBftIDuVb5yKZNZG6aeCtRoKlhmi5nDcUJ7/jd3MK+8n5QuTYsmsQINwFAvpoWq9RFe/vABiLJYIxcci+a1BQf6ibZR4+a16vpZImzULk4KOe/OdI8vX0jTXk6tBCje4JBD9DX24Ve5tx9WAuNSREKgodVaUvhj2ZphHCBo+q2yjoZn0LuG/ZgJdXEN8L8DBZoVd8iZlK4yO2XrcqwHQj5hOW33lo4AwHNAayD1SqRnYoHI7DN8UmQZ/dGrvPWIj8AV97L67PQUrvhLlz8jmv2sLtWzslq81wVp05qJAq34SS9EcI8Uq+k6neOkR1qVTbnO/XfUf5cxSfnjk+egri9OHJLDTwwhBZeNMuLEK1xbRvUNH+Ndf2Jv++C0//5eAJiO8btmeOwnyg84DmuxLlqdfks/KO0Holq9dMotdvr7t03pOp5C7JS/yRRQmvu2Je8bJTs0AYZ1eounnuvpv3uoGpsRgfv5Mxlf5WOsoAvCXNb1WFF+7wMr0hLJqcL3S27hSYAmN/YritgO5xcq+jT+Q64RpnkFOAil+p49ChR0CQajvEHnpURiGg1K0zAmBi5KFWm7u8IvUazvd37/VSHMCaCcPJrOzE8usbEzuOrGKQ3V7DYsoCb6Yg4mF0rum4B7RkQ6kix6KJuTS67gtSCoqLYIT8rYr9Br2zx9Do4YN57g04jm9tPlW/EtbSn7YsOnDy9qY4Qi+t9d7M+ynyH4+2rplfKNp8p8f+XpzRHdunMJZVPHk4kwJc4zaMiDaW8npV/CKVj8aeBU9m0vJ7qrw3+xfPoCx9t/8r3wUT2+1cfvMOFiUP7QBqDt+RuPLRT1iA9ZPWWd4k4/eHJQpBo+bHf5DBnH8Qve2JA/p6D3CICU799sf8yC9Tkq+8Zot+p5VBeFSVoFjOBDCoIr8T5TCd4CrJcTu8Y3j+xgbtk29invHic3gVc8wfNXiv3f372NojbZIJZ311+2r5BR6m2P1Gca6S4SCdNtNDoDtS2QiDAIgXEiHFc/K6CFiE1812f1jTRYR61UgPbSXeudg5J3W9yyo72hlgHBz2oBVhOHpRZ8Pm3pIj/t6PrI7h6vTzpMZGwWuYUki95/gXP0ZhCh8X02lZfujQbbz98V8tqA3C8FyFDdNAvGL6jCJSKjnCD8A58S7GF1Gw5EuZlElHK+/7Bf1T+lj+LJ6Fdc7K5JseRJHtZJJZI87uZMq7E0WvMvAOLQmH/5FR66QCh/hn3V1Q1mifJUQjFnIreVRlTvtoAS04hNAX/AkijgMsOcN+AqbuaMONrDZkreyU+ZQFEaLfgVGbEI2irZ7gdMwgRNF7F4/yFIXibuM41O86uToClNw6t2ByB+2r6EXqw4+LnUC34yv7UhPghPoaXZ6N+SCLty3dnh0PoGGo863blhRv4akmi+Mbk0Uqi9SDROiGH73hWnf+xbAn+exmSufb7PgR8nRXXzPP6LDG4kvrEhl7bIbEnJJxtbZfEp3/9l+3kALa7YPqTGH4aQr06udUwXLlyUohTPz9A9OZ7+VTsbfKTPWsWAyG60j0QU0MbNAMSLwzX7yGpSfaNbFLfCLPusnrqhXqKae5XQ8+s6jmsGYlbMzWz5ulfJLESif/fvfqGOe3MNfTvtz4VhE29IBtx7KFYdBFEOVObjbZTxn9yt9k6Y2rj52H9Lp5Cx/h3o2LcN+5nY78dw/sWkS0vXlAh/CzKgTB4WLkZLWZR9LrWllb9rlond+lsG/EsBXgOX1bMo4iIIns0yyMw2Qbe9BsCKGgQB82NboTOk5JbGBBoJ2n6gKpgl6qxkDQhgN4QIX2MNSqlITeevoZCF27yV7jcUH+l3aibI2lXVXbQ5KtnwpL7YSp97TybebZxhNz4jOv86sx22vy01sGprlId/9SdIJdbAaVQQwWXXutrNxDAFIMPu4bZXGnX8wvaos19J4GSz5a9Hr3GWDrerPHFFjNSSpStgNzkqbTDsWXPh/DVKMnO+v96zt4/jef+wjPy6LfvRkl3qecTZOKJdnn3CuzWoUu+AQpv5CkFWRn9LL+SUjCVVSj2tXCrNXuyQPAbrky2tfpbW7Bnak/pzNMHCgfFgA9loJl2WjnnaSvS3iubxu/mCu1VQ3pTLBwglpNWbqB36EYa58Uea9WobaWiuS0jA6I82e0UGzaxOGETF0/s+Nljzcvt9dOi8KPAUdxGlVQP7tRsamMfxHdvm535luZB6U9LxJAX3r2yqfGtzSEkfFNJB1E8+E91ukZNXJY3FLIzczVx/2SJgFetZNbRLtcXt3CBkuyhCZZVizbYOSfqCCVwmvVMTlcnhCxyDyMWAJnoBt1SKxwNfpExbJZ113RTE46HPXwse7gUn4vSCL1SOsCr3583mZj8EINGVSb1/yfHuz+cLnOIKYJsFRWBn2PL+8yXP1BLhWrKbTZVsBoykLTelz9Tz+kaZ8vWH8H84EjqkjTBL312gjIJZG/fGMPB9M3uSwDwnS5b4z/oNXKB+7Pgmw9M7e2UTeqpG0i2O1tWyMqSdUcdGfqiHavI0m3FyUuuVoeQ3gyADK230rlLK5YevAMxrN+rXlxOKSxtD1M+63oKPf3eLNKko2lBt34bfZEsZj7dRvDm0T804LPHGsiyJK867cZWVNGN5MYhOJnu4Fjq9Jo0bF9abzkFDJpVmL718bUd1/QVEdj3zG+HjgAXXAzBgm2EhJhc4gbOZ1t+H8Sf+m13SDhgjKSv826KuwqwKgP/fJax/6ue/0RdVGGBydp/4s+1ms+Hj8AXC5hPTtchmKkgJYY7XdK8is6pvyQWhOmL8NUu7CqKLjhTj2FFWRBGtRcvc3kchybV9HgsytVgNkj4rN5zClEyQIsvAwVxs8SyS5DbhMIBSA1XKvihXcxd/J0C88u4yQBRXdtdmL4eJhLaqKiAkBM7f2GF+uiutPze4vllUgOakNKm6VTki7WzqWj0Nm08AI5/kbwkzCPX3Np982Q+x313FwpyHcqPfh0N3rIrpKBSoN572X+tYTt7Nw29QWDksp2ViFEysS/cM33DtGZj4a/xotaIsNSo/Pb/fqhlCXGiqjePloPAJYLiTsmkt0XXr6/wMwxEZ7LG7TQRXW2Zw6XvkeYma2/wKkrWb6gvj2hDjgbx29rQT6gc9j1+Ml1NYIOWDVONzwfJEvNn666zCkXO1pa1WC0/VzSYB9p4V1k54fd6ge3l5Pu4pGN9jo/Zf656Pn3yVu2k/blErbvElzCRtr4X/xphtN9OiCDBebS/yfJLPhWidYsQDzgT0Of/+T9x7LbuO5GijTzOX00Ev8pLeS/QUeXOC3ntSNE9/mFq7qqu7akxEz/yjHVpbdMk0SOADEgksMF3uJuxftzY5Auuryd8i5myoAfNJX2nfNSdBE2LUx8t6pjx2lTvTDyVPAOfsLPAoy17sqee/8dFr4cGOjPwxDHWIak+snlX69mAvyzZtZoMWN3FYm/l+Yc20gQvfBnGsFW1ooHebRvObl576yCoyF9qZVqVkRCosBj2rBSC/q9LqzxTuFiAmvdawyKTym7xXg4M8nJ10mg+aBTLGuZAA3ha+0A1Iu4wCqKWkpKELpTgO0M8N0Ydzu0IBqzsqHvXqIJyZ7mgXNH6XAR+50M0Yt3DvpLpNk/teop6caJEeYJfwlowjxrdciO20dM3Q2tcl1QAjHLWRU0P0l7v7CPTJuZZcfLJPicf8QIvsnJGSieZPrIS37LQkXjv8NzCqvZ/6zXkGn4NQyk2XNJ91xTUZtPWEFqYoDcDk6czUE311/jdhSNMNAoHnSctW341PQbHkI1nBBev436C4zUmSpRM7HTfE0WYQ4DVIS7TlZn5XJIOX0oT8+1JFZDXf26hw/EUUHOdhz9CUOL9/Ybl7i8xsviCi/XjVCUVsuEvZZL6Bkrxsvtfq4Y3XkBCUzXxTqudgajM+b93Q8A1lZhj5swfySVAkQO5eD/ftm92P/Rz70RuIl/wObQT+fCbejL7zWJqFb9oLfkxgXpFwDEgbsLbDhVnqbpyTw5+rIpP5FMwRMAruYeTfpeYGK4h6NmP/Jz8wUFrjzwtCUFd6t8A7z0CaxWsisVzWy03n0ngcLFAiZrlf/b0Szc9ne2UbCrQ2bHvDQpc+y+2LzXZVEwhKffPqJ8v514geyKJpySBIPKHOwC/9lkIybdCwPytyqsZaVV7U57lCV+yGvmXwVUHfL3VGlXW/zr8WYqTxLWeGh6R/vglzhHCOxRx2KeuTle8unPFJn2RHebdWsLMxWJbN1T3NfgxSQrn3+uSWgjyvZDo/SaqNBn04cVFnWKHK1bQulcoRVT8hyW+TbtT1C/mroZub85nzAfSQrMgeoRu7TWiY6X1EAHv4B31eJAn0rNjDrXBOJS4OPsjjFhB5PN9z1OpXjsaDF/PNv3l91UZND7/hUV0BIMtLjF6bhmQPYK9dHN1iy5w4H1tMlGi+PxnXH4knRkBfc9MtkKEd+7qwknFajwvbIPgnfPgUEGQlObw+tOHSmHhFqyYhB59/EDzwcRSLv6mngaXK6h3AGvL+Qzyu1UgBvaHz+5Um+MfQyvlJccjQkWmbZRFACcP+GvpeF+jpCWKZM1Jw8+qPvWr2557/5GdNtywZqc+DQQ/SEA2Eg5V4eYjt7L1vTkrkMAWW7D+ivjGo8jAiMsleCHF5eYnc9TULHsvQmcpXYUFudvCxqANzXHKFqIvD4Ic4KigZG+vwFgwTkbln9wR+RAW8oF7kvK7PhL4fcH6XgM/4N8oTokXENs83RaJVmuFcLFFNSEa9y1hAIJ36IzkOsE57Q6wriWoJPb/hjBK6Y3WSTpjwQ2TFOhkAN8gyHdv+q6q2bdMuOljDB+Aj2fhYKCOlYcKIjY+ZigFFyehLqpcPmIc9es+91MpWje2V/IY7FunhKBp6fbwSn8/SD5/X/HkYBzSsyjC0MvVkyxXVE3M8FRzQU/dSuGwzxYLdC/9FM6QU6hacfBdhb9XOOd8zrw8kd82zG4L44UwFQpYL1OfWqYC2ymED5sHgF8B99FeD/T/+/YMAfk78n/4GHKhwfjv4Vcn/099/6jKdzucP+bzH8YeL4UWfBjPtnM34vZNvBaexN7Nj2X9Db1SP/BsCjdGc9eu/ofcjCPxz6pPNa3b84RTK/xvKdoeYDV22zrfeA/129fdnzp8TGIr+HO9Vupa/bsKQn3NlVhXlr1c9fj0XLT/Hxe9lf/MlfN8IKPpgs7b9rQLf3whUpT/P6KXIdZC9+e8bUlP21c+G/e/or5d9onbLfu77N4Ro79cyaXWTPLSsZ/tzgZi2AZzPh37996W67rO3tgDBCNgd99vF+1fx6/9vIcsY9f/bpey/OgqU0w9zF4FWf5+G/ltvUbM5zubh5txE1I33iT5exj+8HQiNbwX+J5v276CO/8oL/tVWaxxt/L9u8f9hc/+5qQrH3DMGAsPAbss6dP9F7f7j09958ttZ5B9agKzAFx1l9rJaM3uMEnB6nyNQgXLt7pnKwb8aaH/rfyshv46FqKtawCbYoatu4Q7ZUX9TKKTbvz3w6y3ofRy1VdHfv5ObN2UzuKFqW3Zoh/lbDTT/fu7zgFdVSdTSvx7oqjQFpfxPcDfk8fgbQFB/5G8g99c/8TcMe/wNefyZxSHI/xaLw/6Cxf3TOBXzAFxJ/rt9gEDJjdeyZI3i30qA/tO+oXDkHzoGR//cMQRF/u2B/rljUBz613tm9MOe+OTN/zeHaNVtlPyOiH+n/twzXLRGbXTeRPTPXXT3UJ9moDjov6boNFrK773gYFnnocn8Xy1F/kye2ffz3yb8XzMF+yvC/2cCX4fxn+bL32v0hxpAEEH8B1L+P6ep/3p2/PMII9TfP/8vBxv7s6TnsnaNtKjJlv9ktOH/erT/cYDBuBRzlFZ3F/6hi6lHCj0efx78FM/IFPuLsf3rofxtxNssX/87FPOPQ90P/V9zu391lH9dRTHqbyQF4w8KRiEMxR7EP0x7+C+mPYpSN1FgOP7rCfLPRAFD/1tEQcB/IorfJBvoWSBcfwb97wL1z538B1n738AG2F9JZ8B27GGbE0CKv0vWnyr8B6J1KaMR/PyRsOmwxb9P73+VVf0DYwCfv2ZifyQ9L5vTqI/+iYzJvyDYf6R+EgL//kNq/18jVBj+G4xS8AP9jVAf/0SoyJ8IFSHQvyEkhRM48n3uL4X4/xql/qYp/YFS9a1dKxYIrKH4f8HB8jylCOIv0RWSJP8Rr/kXRNZ/GwPC/3t0Av0NRn4b8S+l/AOd/BWMeZB/u/ne3+nkz1RCkn+D/gcU2r8Wc9B/jfZ+Yx1VFxVgiL7/08t4A7pfPCP67SCvDkBDvw+SFsVZawxLtVYDGKx4WIHu8B+N4n+Cg/4Z+JO/VeQ+SG+ivtnlzyEi1GNW/BsC+oMCDvkYTZt2EypWQTPA3cmmGfnHiIGZbGC9yzEsaNpxIVotfqwfjAk8o1xwwIA/dsHHfzx2C34A9pCfY2Ac4Y/fjST3xx/S8X7p7/eHv4wr9PfN9M8l/nuSsaD26bL0qdXFpjvg3Atclb4XOfrUHbb4w/Ghc/exSXPfcn65af9W+o+NmL/Jv4RSiSa0k9qSEy9DkapCG69jQMqisGsXv+ksSRschSWiUEeIB8mi0oZIuxl2/tkeZC+LbWPY6fvpQrvzZrjQL0dbGs/QexJOa9VZt9Yv3+yNC1MNqSAyEd5j34MCm8Fi/9iSa8Tu53/e7YzgeA3fVqmdwMWErY8zFAPCbSxRlso1FvHr1StNWEN9JFlQwg0fDU3R9MRR/cQ/SZd8dKfBXza56xV56hUMnl8TtN1SUcA0H7/kn5KLoTAkUNLIh+9nnXTtnortJ65+r1ebiW13121IJWt/VeQn7vUt8K1P0LlbgFCrhnhY5Acf3cb2X/Wt6MIQv2cJX1DaBKGW9G2NMfJ1sblk8tczf3H1H571/tNn/3T1j8+6nXfFyAF/W9N6W/QGZl/bx+uI2z8hoON6tO82f2LJgyKf2jwfH1MOU2WWJn9dd2IkhFJEOEMHZ37oglFiX+jD+913H+AGS33759U+P2FvocFbaY27T2Xu+DnfgNqEfvhWLvAO43scFF7nnQkC+hnQMv97m77zqw3bpH/+RZus7igTpDSjd/mHEsMleD8HV2wxWfj9yQKMxu+1+8M9v7+5/oeWcndffZL2uQf+szXu/goQYYl85QzeVhv+vcf+8T5APSz1pU1L9K4AVcZE+n00ioHU0C9t2RYffOl4/2TciMVv5kvd9/OkgZagrDl8m0Xo42AWncoFFshA5lwhpvmbSRQ88/O9a/zz5RhT5+hd55jiZlK7KzFFIjF7cH8Lhd2b+6ZBZvZE5cxBZfcd3CyxJmZw5v76Hhu/Sv8p9VsS+O73U8VP0f/KF5TO0gwv03zCiKbO8CbN8ibP8SZwEbkZYPo1Fxf6l8v+8cMUrkTvpkybMtgXSpo3O7vvMN0/3XlzUY6hXZE2E7AcQg03o9MBjzXpA1yV1v3+a9Av6WPSjBRsiuKGSrjiR6KycbvGC/vC8NrGA3KqhvODxL069CkyI9mPkZomcfg9PdL3MG/yHL/MIAweSJNQb7BIN+b3HYJReuK7uT6t96nQBSTMUAmH8CQO2uJSTBo7Qp5OvXXl+VAQa/KV8abAz3RKU+z3oXJls9EQ2w1nEu89g+3RbzJ2gGcSA7XTEjIS8HJadli/8lJB5UmMxgITppga23miQvJGm2TxEt67N2dCDzaS4nNE3ExqekEcUYPgZpd7vohpNoIW8Q7rWrq1EizVR8aH/XjnTp2/Zh7WW9nGpIn4iIjURJYyBjB0Pq3JU4/godkeWXNhIzs+xZmHt6hpuScacEKzc4nwqYsscAZr98sdb2j2dq4JdeZIPz3gHS9EchagZvap7Q+UY4eAxzliIzi7bs58WcCDnZksP4/VIK3Jx3N/tZT0TTIZ1vmovi2w7tr1HqJEiwanVP+sZHQ1xfHg8zSKgsB+0PvWNVM8GyPY9hy9p45u4+5jQ2n5YR6414wrp57TDM3CHFYW8JKs6wVi0OJCaiM1gWrE2OMzzZ5ra7u1GxlpGv04HyafCb3S1Jxt9qYdD5lReL3m+pl5SLXY56XO0iJqc6QJy6Nal+tN+rlgxnXewlnpoeueabkpbNGDfYrFyA2M52X4mClobbiGSk0AlH2DrF8o4fgmcFWft/bh9eT58BuiWybFvHwsfNb49Bkau+q4UrdF/czIEecIK53nHSG0V+VMb0jsVA3ihneUMeeEROr+gXpHsagrvva3ca18BvzNHd4hFCyEgjb3KG36LsjA/QpERODXinkCX6idSZm1vxAQfpqhGOCkDRvWNIFV3176hhF9wDXw2QgIsPeB8GG02/Z1zaI2zn2s5eDqUlemROpemZvoGS5zY9Nxa9uAP0KfJoJlxNMPVnqwHawyzIqc1nvxPYonlMrZv9V6lu5beMuf5KY3sXsW8i6bet3LlpQGGOMnW7yQ+JvvYMTtZ7e2aoes46Y/X/tQvd4RXiXfSCbC8LLn+pWKr0MzhjX+SHHp1nq5/xNHoVnjgRo7wlUvXM98LPg5y7OCPZl4GUzeL3iW//AmDWC2L0/mWcY0b076i4uZwc2b5BuJ8TJfBo7AuIHM0JZw8rXOuocJOOSNG4ubq9EMY2pkDhzoP9Q7OoygdB0xXNjpblDYiO1SgHlNUzEFr+N5T+HLuildQSv+lWtKhLaTdo/NCWY/40x5D/Q5BvVwGIR0Yvzn5G2bHTv9A/eR5xOVrFz1Zk3SZkMiJRd6sTMx6A1CKztYMJ8xL3/n8BpwXTTTT9zaBHbOmAU1H2fv1jzKwn5oiOEcfhQF5JdmcBC7GhgVHAhGojq+EM7BNQ6Gh+9yNljXdTRt5sQTq1TfG3fR3WEt8ZIxCADL88NRSduFcVQF61xmpKnkfi1Z45xERu8P17k5j1GNDnYqcyeuGSVTHM+B8njpFs87B7cL5QzM5A7E9lYwDbUXGHrLjqP3s3jUymQcZ+M+cSxDm3KZH2EtsjLwxhMwi7bWZJ7h1z2l/WKeXoniLSnKPd08QI7lyOCSdxBGoV4lApICa+nm1opP1ESnd+O1bkd2q39pwuOhhQwfApfQ0/g0OSPkauSoEvYGXeNFr5CFyVzqELr5PN+wgsSto63dPWuyUWGp+W6uWgXc0y/osbLo0ppL4AvQXkr5YMzyYwoK7CzfWdmoaN4ob4IQM3dPQI5xBmO5Hr3FIPCUNJdNdEpcuZb0kBCObzifDM6Hk1ND+yoImvvcCD2fPjeif+f4i3TBhmf5ISBvDgMOVT28BYbHb71VSUK9UEkZhR/fyC0J9nlUacufod45s/WW9gYTcGIIwDuE9mpinVGIAjVKD6paIOs4IHVtq+nFtVxYL0mFssIoaf5ePF/jOgLfrU4KY3U3NblBV/UZTtPseImJO8Gw8khIHUPdGU0gP85qtWwvtbTGfD7Xx0faT+WFRDHiZ4QbktW1poouIilRq25UoJ4UWNabKKVoLM9aFGyfPIboNFaMj7aJIT3LbFHFPFonXVi06NdOm1BIApI3VxJmeprSdwc38MA9ERVlFBDeTqjKi+UmXSZSevZfi6wH21qGjp/W3pNrOCdFNka4ZifT5/d6A4WvB9W7++57OIgJ7E+znzlotxtZrEqQ6kCgdtoGN5ZnwmtSYi/aPb7o5O0nW8ozmKlB/eQwFzzuMVPEtI7HnvtG+OpKCfq8I64nCX31l+pRQFjY9cxnDebICS7Chk5eyd7pM3sGH7ihXQn+ZB9sq/Lqa9gQlstYPyHV2Yg4jN99GaFTUVBD5i+cm+31OmlAvyhl/lQFgoVkN0r0iTOSnhZxl2lQhFSj8hosL6SM2esCb8Jzpnl2uhfcs8VTRwh4JXHq58h9Qv3Y8I1m6sOq4U2LtexVw6j0pBK1YjcCW72TNXLonnBvcxuLgi6Q5ekOh1BAGv98QNknTIbKmlge8rVOBj5btN7lbxkfPtXXFUjRuaHO8BCCaerDG1BCxSVVfLOumQ3JRK4nXCyO4eM3ShC1qs71iNy+nodn1D4Bw6q3FX2IhyGpwmtns2f6HBknU1dE6/NaNO8pju3vZQM9oqaT074PaQ6/e26FZzRb39CKhFfXoOwDgl6txpYYxr4esen6EfDXXz7glcXjIBwLxAYSlO8+98M9midr33iCex6Rm3j8N1lXCE8DWnyj1AQD/FLFqaiAZViAAFHxNAdZAIjeWPr+7sKtDfwOpe9zPGPxtuzwIn8qjOuoJXPQrmAFhQwrZRKKh3DRlzkINJQJTjFqDPl3ybTzjLFLZNfoK9baEoQ1Mp6DtODY6Sts86R699Moyah7rAitLWLjbFCOiN65SM+mDwkCDoj7iz9wadONtcI+ZG66iIhgNg76K4jXHEMhOUvFGXNCpyC+Ab5lMi3tkBkuuTpJJ1GKiXSYEcuAGzGCvb77AIAtVDA1rwJBHQVZrvUHL8N0b1AlHSBaA4LAul4gLCbiz8UNKvoGD2yqxR0/znlBeSqIR1yaKJO7+9yQoDHncCDjOvEquSzJS7FIC3lsmONjD5LRGsAkl6V9E9KgLKeIONG700PMvg7RrLfnhUIWNKKxnVwAuALUjQlukLd9mylgm0NjeoJdjLowv6gbotgwo+jX3qhRSYhu0bYH32TfBlb5wPr5qNvf6HyF5L3B3t+oJwSsGZ8WYovEW38jPsBdRTTqK5i0Z7gKMODKQBqbZ6t6Ugr7FfAnxzs1xF5QYN9PAeNEyUH26LCOkeZv41GKlxpe+CkIRYTzbwOvNHto3FmXF/1ZToK6FxdgAMEHEqFUtsnrAwmHgFZovbOG8EL056govq56pX7pit9G2sjNr2dmNsvjwwaTEMeerOMgeYp8CN1kFxSUfTPfyT4Rip/GK2DFR1qbzzfSHusRA5W8BsMgf7Yc3fy12ngbdZulMtao1OxaxXkejkz3c4J9D+Kbmpk+Ol8RqcJAY8SqGrK7029O+BYnHG+BDEmjrEWHrDehqKaOOJSBf154D3OJx749g3/zQyh0ZqfIUMDhxSOi9wBnXbJktYm9Ab1tMyEbK/Z+jsdOlrX8bBMr+DYKpIzkcxkQnK9i7q0osX4zOkHRNs20mVPYF+p+AzSy7ZZtiJoDkpWXXOLR8TiY00SKxb6k/oadO4kBbJ7LuiOf9cMMmh0xIjs5XvMS2XguhHIPv0RLjBncF6QYbeDnxRvxLanFZpFOJruFttAYx3MsiU6IcxfroE1+4TXpqmmSV7VjznuzS5zcMOhe7Lz0DaXE232Zl3zBdg8nS1qMv+V/4X+jOtwAtjtWdYn0i11Wy02HgIxL61DBNrHRsNsMMrQRq157KTz4miMyubYuRoxsrLBPH7MaDTVkQXMQL6rP9uNegte12/WG+Jc6AR622Uz8PI2MR1RcTFCrE4nkoZbPtiJqKExwTRqgW8VZeLvZBFiImu31eSwXxMcBkRCng8oeW7+sbsWwwYV3pOulE4/yU03pxkFX5OqktRofbxFvzH7v5XkT9p3SUos2S4VAhddwkeOL1MzXQzn9L74y95VF+oUhb+Wt3WydIWfWWHIjCxX4Y96vMyyfwoZH8yxTbMq8+gRSPdrU3QOq0dOcnnlrX9frSI1byFdeYNWL/yCYY2lDldGFaXnUWqOsp1c5J7PxWa6/hTzbzUubnm0Xz+7jnmS+5CTdJNj9JuvDnFkKKz9Nv+abAmdv+Ye1Edu1md3zsppSmqJuYYnYlziKuJ3fnNX7pCX9frzYDgW4Ycg4/0UA1+2zHF/ysXpKJ1py48s1wW4vDJ8qnL2VS2EMNEvhdgeixo6N67Zcr6c9ZRtSWZzzGdOWYdQbukqX/vRUjRtw6/QOBBJhjWrGWOuenejY0Ji/4aYPHo1MFqiUHM93GfP2o5pV/WUokVZxcx0uPUxmMHsZdFGJV5NAFumtJjHTBcyye0wuixVVg/5RMiMTOh6usvLFf3Mf0tm0ULDIhLhEMnYwoSJ642M2Jc8P37tFmk4WoT5KOb8b1IwP/fCGuo7jyNbUvfLsUNDYXfqKCO5hb5hL0JSWbVmve+EYhVGtF9d7fNg6Fy2lPJG3fiOEeC6bw1w79hidX+nTy7rxehrxMbtZTpTZ0aPMkXTB4wg7NRZehmTfeGUe3Sg52VtHU8+ac8bhhTPHDveVR4whMMJLQhwpyvZqW+pQbDhUu1N4qiyZMYfQMpQrtDuCiibYj2DVGlLnKPtO+yzKNy5DYgp9lOcbWC/6ZemCZHpK2vKNbvg0s0lTmsi/mRfgK6fGg3/PUYqtW+uLteUG3lUlVEKZ+hqWQ2GnVVhjr2T50lYAfIVWNe3VbBkFNFf5WT648cZy4w2eu7GFSBeJwP4sYwAlmQZ4gzddWzYDR65vnde6sYZN13onMiTHWkMiAZ53ma1yLCnvmzwQbkhBA2sfQzskQ4E5ODlvg4YQxdVOfPs8reVC09RqgOZhdgQ6gt3yAu7pN+j3yXP97nv8KFdPAmWSe2EQEhP4nBD+5hoLG0etJz7d5lWj6ojsZfi4ETSCuHO9r0u9ovLVS6kr3i0EjwuY3dnPTL+cHF7eSFkXTMzHrR37QfcGKyakf3WIo/Y3VAZo8QM6in5m8s3ptI5shS5Uzcc0YGhH9amLic8xv1TdqYVtVuYzqdHc2FtNwj08rb56lDRT0ZzUx7U+5uy5ZoTggv5G3DZ7OS4x9wqkTrqHcil0eWjqpeE3XSOfjEbVI8v49qPlMJPXhLxvuEnuRZbxd0FgRzTzXIF6x3x10G06rogV/TzZjAX5Rn+gcg1X1y7Wya2+hXM79p7vt2OXhF5YkutNJV1snTsqBgTH+S2wSwu4FsaS8z7tJYkRXGXLvBUfYtVU8PIYupW+NU8zywuRfr/bAPJpxHfn5Kmlt0Y+TD4SnRr6xMz+TQzd2bQJLiDthx0bzdpHzPi8nL0f0aezZDhiFc6gd0v/5Msgdk5Pf/hW7WVeMhG+tyxBOEDkRaXwaq382dVozE3uxb/NwMPbx9w5tzqCq77zTKYqgrDUQL87agA+Q6ZobIrmWl5TN4fhuFJyvHlOBTVQWN4Fgm0CxEz1To5WNhqT+xRuNq+7DT/373M+l4+hdsfrVbBP6Ym8WyVGlo9q0NeLHfdr0qZM/aZofgwDob5fWunNK+o089kA2KYM1n4Lr3ZKy+RLAfZHsW9FpyxA/4ZZCQxu56Sz0zu61eZtzB9TurSe9pXIsTvzOk02V/YKOy/UNPnJghSCTLX3oT1OV0JqGbH3jJoVYaRHYBMK8zAxX0U+TfLg+zWFfNbjFFX01VEM5+c3Vvqjhm/8bcFdaUEJLJjX2n/jAO0r/CBBTkZGRZ0t3ixnbafTFlbP4UBoFiGnznpXP01zsjrqn+L1aeTiJxc9PmXhomchQWkmCrFFx1+WDKDDXFeTRGjQcrXte0x7a0pbLZ6Ksr5RGNhPzagHUajNwTTzsrbeGlCXvBGTWr36MD1FfU8F59L7ONWLgBiSxzJ1n7qE1LjWDfWMTCwsJqZAU1irV0Mkj4aY2hnDezzWol5FGnV8tkT1TeCGsrArXp7gSEa74XT2UuCHqt5T106K9Rswk8Ga+L3cYrUFo7cdLNhAt3VjqJMq9bSXD/7SrR57fyj8E+O+M/bbPRmdMQiD5+ES+K9diytQ/WjAQthRf+UsI77R4GUuxfEJA49/O29fO/2ay9kybMddsKGZRfOXM1UtA6uxC40G63nNWipsn24Ya/bkrNHYkbRPbAKImgkeWnoFZ7eGeelLt/RKUtj2NU5D/Zcj2sPLczmJp9WITO4JK5cF5BCRn23ougrFkgUtGgYQ8djRlYBIPoHCqhV2b1VK/SVCSh3FeD2Sry8PAMCRkzwv+mZnX/E5+qRByhvawVUE1ZYiAwv58yY/jVPZuT7sJ+IgDm0aM9a/SYPWmA65XuoRbLl2P3eAoPJT5aTme5t8b7zFUm264xM6b43g6mgvfcFjJCXtQ7U676Qk9xOTHn+qcsIOOLIlWN2NhK8CxevmM2cryeo4+ZaFbNPzIVmk2uxElz1epKqWI4aGyDRM4iuTRcqlakVs670sPQ1iI0TsJXY6E/sMS4KkDEm/FFkrXZ+HVtGO0s4n5ujWwStdxzZ2EstylEuO1W5IorzrxrVv9oS74lt9n2pSXW/d0wOFaZMbLRFKDdycP6UMecjsJk1yEBMgCqJtDXfgZ91LZhasJlFt7a4nxLzVS/BjMPmAUu58NUmG7PKeauBDUoTDMB4/62m3lg/W00yZo/eEZ3foN7nM/MhkWuBLvdTNwJVrhblbUyeBWPKMC9+y+CxGsbrVu0K/TMRFVLtV/GaJNUZh3T0hFdqErDoMXg4W2DX5R8u1aOzao0NRQWtT4UUZwJI78FMaic+sBYtyV/x57Qen6fTqSZk4uhnVmsSbCo6uIZE4vPhgOSbZ89VJnaZZjjl+sy/YXXMW1q6tb7t1denYPvU8T0wzN2rk+Ebpgi7YWG2eenxDp+ck3X6kqh4iyXoEYYPbSi/ApWelOIciUog2AVIlg44LP/lk4ehXPGkvduL9OGN/0hPBMfxIbaP0WlCyVYh1f36CbMBhd3a7Wc8QvkQKyUgNCdse0nadYNK9HcqXhzVpttPV5iZ2H/7D/oZFovf46VsvNJVwsAdbNGRoIfsDYfN3PbgJcDkRk7V4KrYFdvvub5AZ4yN277fzTY73eeMkU5henLEvsA6F97EOrH7cLV48toOagO7TD1ujl//ZllxPNBC78qaZpCXgezyYl9CHRunpU7POeWZXTaYon+LIoJQ281zfJofSKYCCx31pYjiLyL7e2irpogko1lvIwSOoE+WkJfLGghu9q0M1TX7ytu3nPKFLWHb+G6I4gwG6HHqPmfUicw89byYTo/E3vDOtZGrF6fLk92KmXZVs2i2ELAGdVwPapmDvulM/uyGBR8KWEHjaCkpanTySVrgW0i5ZXSTA8TcT+aI58OLQQZH8CbRv+NoaqfG+GQr+dXIk2cA/u1Oz7TWPWS4pD0NdWEU+nqaHA7KlaWiOrITaYZD+5kgehZ7076LL7HbS94C1FaRWVkl9XuyB1ZWelyKA0usLhBtCiwONUONp+ADyfXfB1qRy1dQIS2AND0RJ4oQxgU7ieYu+ks4xTAxsb6EHFFWUDAPbdTVSK5GgbeoSX9tUA8J5VhsEpSq5QJeDeqdIoOKhbe3HF4DX4ys3nzuV0eMlDtuxCJXnKuSr0pOrnsLluOTHzdgybwwwsPxDf7ePxl/6S/ePB39ldHEQYCCdnWo9EAmaFr1x5LKuf37ItUYw4kYtL2JYAli3Y8ZCMSH1YfbYX2aT9eNTPzC1EKkq5zi8Utnq5DSY+Bg3BHg70TYIRS3TSYNAgFG9mM7Ln4JHLhHybMpg4u5WRHEjv8S5AJHHhfH1UguiqaeIyLYJ8R+n7b/fEXwKIsblCAjFos46eatVY7KMwbE9GHTJoAal2RpuHgPx0MbjySgZGuwogtbL0VNSyz+VlvBccUuOK44eRZmu4ghxcfx4ZjuqZm08us9AzSK/uFQFm7au7DRVLQS7WRFs8WCm3mD/EyS6yamb7SdPU0usxlbjx5VnzgdFD83Qbrhifpo2Hd6RncqO+sQIo0DIGzS0xdtU8L6/702R55m4y3K2yIAiC9BoEnGxMnQ0Fm/wBbsLXqECxaMi25qUgkZgvF/NUSuxXbS0IEszY00r8RLVF2nSgwTvTFbJ16iSDCbxiQ5bfAOh6AYmb6sY+ofHKvzldW8YdjhPuT4Op8RO5q7wjBLY67la35B13/GKiPo5ep63gtUFwsmJxWwjYhgD8T0M5bJLBwVIxD7mSQkjQOalvb2Nee9gLgYvdNi4slCObvsgpSN5gFGF7oOpXkQ5yGaBgT1prw9gHAMqY0EbATfZs/XYLSRPmmlMN5oWyPRZ5ts3gYv3YM0jzUxEKrH7057b2EwKkd2EuQhNlkcv2J7sFzFDPeJqVmRhTZ2BldTuSgzNPeTo/tthu9ybTbfufFMXYII+Fw05fMdTYDPpKP755CZUHBrzBR1SUasg4GvmiJ/kEA4q0+sPrHb9ROT4uwkZYkCOZcTghC5bxeWP5cUBdh3pyT6q2asZU+VtsU9DqolcqsuoEkVVcyR1hX3kE9kq/NGL2rOweLsEulxX7j15JHIBsUDp4Zs20WYK3wn9fiJEnC09MxJXc51Lc7HjFEbTlEnaY3hsaJgU1NupQcJQxs8eX14NWFGJRY6SuZTzFLWt8eRaH7CEUe4bN//9MN6PzDh8PMwXKl+xin9ZIdq2qi+9VOIziKp7UiM0gCX8C3t5ZDeQ5yTDqDhWJflxM5kmBgoo+8/K9zL7rU7LlQ0+CZGtBgNVg5s2W/iQ9Lu8lYrzowf8dHfDBqun4oDldpO9ITHS+jtsPgSLzIccPrjLURN8GlxXmauRJTrksRyPghwMWFgyzFA5yoqFb2A4aDOh7LqlwbGLb/mrFDm7PPgoRu2ZaxAVIt38ur0I8Zk+XhRvTvXVM81ATtc21y5hQ09lpo10exlHhjv5xOzPHBBjawsPW9+2mzPbRVRePrs1AvS+kVwMrOc68G248b4mXtFp9ugSld+0DknyfujMHLMSFtc1XiIYtM9QDffx/HwI/mo5irsyPLkAua0a47Nin9jR0O+p1tQIEGP1Ft0LD10SmhO0eb09IyHlOnMiH4ttwwZSUDDWfb/hhMDRfZSeqh6q+tIJsdAuDWwZBs8G90gFxZBPix2+9n7q+a6E8/A5ZMdnMJJc4hEXH98uxXStbip8hlXoJKJ28oQf8SPhtmPtr1ktBaATMuk3mmycfw7Atn2IOA9XeL7UyPC3tgs7Oj1fMpZwaNo+gnYOmzZ76dNgC/F2hMVAS+OukMeptZq5A/tTYRgWI2hJ/NYyZm9T6+Fa+uFDBbSMok5lUde3be8zT/e5Hdp6UNIFlAh6AQEcwnzebyjbPT96D2N+huHbGdu4+LCir4bMCTqZyHUPUhQykC2t4LQrdSxC5XnlBJAYeakhOvhdx/g5J/bXEsS5rWbDSayIoR85TkDz3Ie0akUfO3flPE/z+OEh2gqbKiTru0HVTpYyhiXH3dwNTahzrt/PFsksLYPREQbWuv64kBrZK67TKy/V1H3JiVoM8QqSdeS971bRxOtwEmves4RFJMlZ3sjm8hc+gzO1GNcNsqny7sIlurSEGpXsVUWdB7IFB2vFDtWty5VCY5LatG+GVyqyTLXS2+NXe8oZJgLAzPIShzW05TEFmBRyGucwwzVijeSVk93YS93d2j0e0rRFsEnSY4JAbM7zLt58NQNKzLm1i63rJuEjfAFTR02lC2ePHchZLLyVz3pTeJ64ohXGY3VJLa0E0PQ2zFGG7pOPxM9uOAHMSwYHlHlru1atcR4MixNu+8paFiVujYuMA/zjzykjz8MeDKRbGFYqipTPYHzAhWsn3tedncWq+e0EFRqd01ReJJBKND2IfYkF5y0CI7t8qt5IdHPhCnA0KD6tG/mg8ITdL+3jxhLbNhN+O4QRfF3bVM9+Bw8UNS1qZI1w0cZLVs2JCPA+jwRau/nQbMrhu7M7y6fpzckQ6YZWRFI9orUl/fibBAwgGRskORRa8sAxtFpf2wJCZuTpTT8z4MfKLcuAvhm6BkkbtRrUgYCtX3J+pW+S1W2cciGwAN2RF8w+vNQ/zCWVaaMETO6lyMo0a15bvdpjSb2aEA7B2T3xfV7OJkOmixOWQadvMYjld9vX+Cq+yBMk2RCIxbsOyVwFW3xp+6u3+3kt7NX23ESpPEuGNEb7YOQLwi57CzPFZjjyAabEcPgvLk2WIcDYF477wKS009Ncq4T6mdBR9YsBWATrCqmF7tVoMfQ+LBfxnl1yn8jCaQb5KZh4BBjFO472JFK9dbL0I7kpCqX3cHwYmNMJIkqNKlpB0NNR/8aESaPnNHf7usuV2LNQ41OrbcoG/8WqCzYj1TZBlBoMcVqugwM3QVAATY+YNIAJE9dJSVxU1GPy00AYU8detibzoh0jb8Av6tAMHDa2AS6kswaGtTP7iJ6FEi8bz58AhanN61xJgEEfk6y5t5hIrTPO6gmjHi7U5zF69oX3wgoM8cO5HdrzoVYhPgmd3CPNgESbKFnrEBZPau5vxsm1TxGq8LqDHYlxljriRHNvwgdpjZ4d1/vRIR7c5aD1F6dhwSirHrGjt+ofowE1MfMcoWl+IBCe2bbjLYHcwo7TsDgLXE3ehW4fbptyXa9quor0s1gUYPTVc37ylrgesGfRYALjMfVZQlh+vrZM6nbmkQ4mMJZXt2KlYvWsN6wEkF7iPa23B3O2FTrttN/K2HcBUD3XeVsvks4+TDngQ9Spugl5hsIj4/vzAb4TYrqE9Q2w7H50vvHI4iQSqFDUP1Hd2rb66usafSZjaGq30CZr/CbjhKjxzAWqiC4egudub+vweAUGmgjSxHau+2vdUJJnLNiYHH6IPAQ+z7F36BC6wany1DWTt+yf9C31opGMBtzFj4y7R5Nen/T1cGwp3JoYIDNbeZR1ywytRQx4AfZ5AiqNeMYzv5VNFNejWAg6tYErUROhmJ17eBsM/BYjhyDte36V07MdRllAnxncLh+Q0FsYa/WylmWy3NP6jBSGuVGRixRpw/zmr2uIaJ+xbVz+Vpw/pTp1JwJbwsfq7qkgfRYH6OP6fLKip0xVNNvRaDmJIgnQOWcKRBrcFXQD0Ypt0QujwMifz+e4W4q43znxDZz3YfynnKxDcM1hf2vUh4L1YkqOYfAuy6sdMIJ/aXogH4EBwpUzBlBVH/6awKnQqdbjHt16EovOy55znVhkIAlKJ7x7McKBSvAGVYxGV9TZGxe13au0pifwsxM0H2r9ZL2icpOjRlmFkIMH1LmnCL7HfDAt5zwNYd8Lb0icHGE5BP7FuxH6anPn8wIe7Be/xdBdSCNSPH5pYmHkoU9/MIw6cvTMH5nCOKFk9pGII0Q33FyUAlCjb04vHfUjXZ+OXmwZxszcxUmb9jOoYMUoPh5g7eQDQvUiZJgDzdrBvSXakkc2dvihY+PwEqhHAuLlJiEuvfHo6poZHbH8YanjjblmNb1yJgmi+VZ95mQTqQ5+rrJRxxRGWrdW9MXcsKepzY6OuiJfIhqNy4EFqBmGGmlP4O2TnuHtC9g52h+DEpnhsNz3U01lr1Zqoyz7ehrNUktMfU3ydZOTNFyDnFxM6qDEzjHmDSEgSfhEQX/guEF9/qGttEiGfSefbjSpbdO54tIvtqDQwCEor5aJeo2R3UMFdb3yiQvOtxIhWuy6gmG8YdcfG3imHk6mSfPv/S709fkSeqe4wY4C6m2llZ6RUAZ8TgK8r9/nNxbh5l1wb9TEE1FCwVWhg7nr/nrOwKsNgEdp8T76rLRrisdB2hpAFO2BTyI9/iHZXJtwsRnVbm+8t8ojN6vPYgbxOPcgtle3dt/YrUE7kds4ulMYUNV1Oh81fT6QW/LF39UO42vdpAf+u7ooszTwsf3uGvjxcOJtntaBP5PFHEUolovNMfpT2kmX+31l8X7SIb+KLHAOFy2EfwC5wpPfBUWBpgrJB2Yoy6pIduXPjN8HIq4KVpRV4IQSx0VT8LceiIiB7ksGFnsWtJyp1VZGaY1C5NrKwbihM70vLt9rFD8SWRQFTGGak99hjd+8reYhFzUgDxADTdq46aZjAuIe0aJHax523CIBNm70jydcSBJAk1hWV2puhmASh55vDiNL32Ruay6LdGS7DQh9JlTzgobsmLB0U+8a/sFUgpCLyjEaNO8m6jI+PmhDH+axStbQEGgyDuYOjSflZ7HuYalDIvKfDm49SnuHDk8AhK/Nyn5qywmWvTICqCde/bwYw01dMbMX4S0vnHWc12EGYqh3DGo/SCjsKlrxsBMS5ws76HZZoAFzsMLZ0ZcXOWd/fcQ3pA9C4AWcfIFFoqlCAAYJOZ2t6NUfXj3/2B/iFEv0wldayivEIkFejb9ZIWgxzSy8brsl1Z6EJcnYrfJhubWzVbQw8+LAlnwFtUdfDM/QWQF3TH2qN1BS8L768tQzKT5AhgZ+xgJlTzTKj0mi7W6KZXoLXn4oRlOP6REyNxreuala2uTmQmYjaK1Ac8o+eCwqpyah4dDpN5dHJ286f7HbOYq93OJyW9euQExnTVep0jWfyevEAfg/NR80XW7Mf7WMZfdqOBT3XxNXkr2d3sXMIz0DfJ9iyJUV+JkVTy3d5xlj8ASDZMcOD7U+jbCxk74xdZPnmujKh0x7fIPKTZJy8Vm7nfp4C/lGfJP2psp1laeNMSRiVUzuO/kuNiDXHt08QmHrlcfcKVN6OSvswN4kmlsr3yeH7Ik5IaqyeUFT3zjCdLvXe3dgtVzQhWFDwYKyQ5iuot2wr28QaDVPet/yK+7uyW86P4LZAYfQyZ44NdHqxYQSdrQIwyU3Vj72hpaODzZ13O0aNmeqM38sBBQ8iRhH1bXSjOOaSnkjGyhclelVJQcNwzZbri2ojj9ZQ2+f2EMlKlkO0kh662BUldNwTVUx6TjxJmzRXW+kF+Qlt/Iyv8uUqLrGPlnI6GGR01idl5BjvABElZhlOouz2bMsiKhLhYRPAhnyTXNNxJKe2ZjZyxXSc6OhqZqU1ghiMqh4PaHbpK45CWNd85rMUqNbU9CocdeDs2TKNiRnmMQKpvkmChygUOX3NktQwGglfC+TSv5UgCtlxuL6n+vDICGcLf4LnmqwpZ9xNHzXQ0nNdM13sv0bAo8mMcKkJorCDOY9eITeCbPcjk4wjif9cbHx5tIfQ/bLpdz57Tnaw67JssuT886sh3asbtCd5uF2V7haHZdIPB9kfJZsEL49ps1vV7cKNmPggPNz2xcfriGjtuHLQSgcS6ZVVoYYWXs1hMTqYibzvKIrUnlGA+J5QTFCEAAQtczJRf2s5dX9tB9j5IwGQmogbpjl+a4WCyNezHUK4TN8nnI3eufxkP0ptEW/Q3TgysEQPfMCzCsxBHStElFuGPHwHLrgFvYG23pHrai7wS+zBXoozYvC8ykDSIfr3NRSjkAahtTwbxB+nukOzr7AJpZRGX6ka5uNd9uo0PjRgkbWJFrhpVTJUL01eRmkbnzR8PPw2QWhCTOIYfKo7bix7HeLMxs33PMPen3qnNGswVpzEBsV7YVWy+eJXRPaLLiyiGVZRsNXUzt05ukl/hwzrFE0HhVb6SM+cujtLBZioqVP7zKWpsFBuV39WdUKa0r5VdM8v5PFvPMn7rrXRLfDcbDuu9U/Q17LrwBDrPdhW5VgUIWBlGdhUWxYG4uGHeMghx+HjgxVjhDoOURzqAqZpORj5QxLGZEYzdr+kNTkp1YIFh0O9WOor0PpjvP9je1I9CLT8B4ugDDjjIefZsWXvhlFZ2T0jHHai/ysL203SjGgPyD6teDQTtWlBRC3HoXLJds6Iod3PfcUOcm1KCQlo8Yuy8Cac16iGAqL/NiDOpHHuUgmQ6Z6Gl+n+7Luk6v5+rB+/aQaPywwVbZdJg7Zjp89ht35lgbXj5dSwihnAal5IyMfexWoevX+nOxNs+omisrBJsRMPPjb2LSJgk780zv1VhqAGYMlw+vLtEQTMBF3ALgg6WpRiR3h1dYjWvT1EF/0WLRpQFA3KgQ20H3Xzl0jw0q+WbMO0zeCYaWFD7GTgKsw+k5ehr2smOS6Z/ljVBCg/pQpGVcaUandnVFk+ExCtqE0F3mZ0epkF4eVFT0VBxc9PtKlX9xBkeduVxDMQF5EF37x4EfFol/a+3o1p3PJZuSxirZ7lYFkcKOSHN+L4gc7nh3n3DOlUxq3OZ/ys7D3czOzZMoPlX1Hq1uw8MAVp6eDxOcMt3IWOfMA3Nm6Nk7rhGx0/NjxW5l0w7dheu4a1LDM8ThO4whr+rJVDO+56dfK2xeLULIZlzlhaByRxk3GbSfJ7h+sjioPyPYllqC5gpKHRTMnuLjkl19vwxyYmwK2605xz63TN8E4X2IB3pQqQTfsUQfqlKWtUnwts3IOhXMez9+FM5J2CN+sjOuZAySUcydwsWBcg9ADEE80vW+vEnVUziafpS6AobxVUKHIXzK8m9iuT7PUT/QymONIhviteEV7/Bjy5toz33/mj+87apmIG3NQUpq+e//wrSahWXLnkBs0Klyxy6JaHYBV38RrA9WY5wjTtmzXPq5b3WleEuuP9ZnAitC2kilzb9DDP2nlwDr6zt1Ylv0VDOBGtRV9C3PLfIKdZ0xgDjQbXeDW/XosQlbXMyGSzMPN22fARKR6jbtS1nrRFco0DlGJokrPfuMID8I6T48EDRF2FvFeWV50ipw4YKwtJHkI61MP+RsHYNzpHGpQkPqGoTXzPXy35rKSsOGj0rn7rJq/bSng0e8Gt/L3CAIAnY+8+UXnDF24PNg18Pv+A4Yvv+j8ffORRLJ4VTxIlt6xXABPU3fjgCehCWxAvEEkTrk66y5xV5wGQwUh5McLQd6igZxUFfGcV/D29zBtfdmwSi7DI4XxM/R9Kt48X/CmT+ZzdCUb7R8d9hZb1DVujRosizZcN/udM6PHfH6IW3/WgMW1r/uj9gaHosLcBTCAA6sAZvSCMdnXYDjJewPpwIKG+jnY5aSq0bu07cZWj90ZS0rCq31ywNRuhlejFXVq8CNQ7NoaOejHLkIj8froW6wuw3PzkT2RnJfSvQyE8+g9X7F5H1Z3jA+Eg3QJH3HB8ymv3vGP7hGI9OGROYVcyStc4S0kKDTxN+95Kv0rcuGvF5IgtN802ahW5IO5t+/i3H20ryg85c7S//8Z+44tyY1ky6+ZPbRYQmsZ0LuADOiAFl8/8CiS0z3vTE/zkMWszAwId3Oze00aUbc1ctA7sxZOSMOwCFJ5Gy+BJgLAgRl9dIpj85magZckecMKMc06keozZ8InjcB0cn3AQvlMEt+qGHSy6NWWbO0o3G0DxNFuHB7mp6vm7Fy9HmQ2mcbPKWrIZ73LMe+GVR01RFFnMZm5RurI1bspfEaW3gR/lnZEiOjYt7FDhLllU0wyXt9IKswAgU3gEpt7mqxTwSrskG1QsYwp5XhUNPD5aJQ8HmGPy2rTryJ5yAVbPbpR8bX4/RxgnX7J1OfdhcObRiGM0dRZE/fPtH0QI5zXWHuMUaso52Gw8Ic8TQbVWzZEMPLmJ+DyQpFiWT/Zl2ceeynhC9byYycOTBOdvkkZmHuQeyfAeAfNcrqYTh8w0l3Ns0HLs/xocSc3SBkedPryE41i+tKKV0qE5AmnMHF+tGmDl+gpvHkOgd80kkoiCEmGSU28pPuLXQs/b6UhPFZ3uAVCfQDsqpEofeOfz3pjw28wkPf8n2FhbCyojSi+1FXB0VQaKo9AzK6Gb6w0mdic3xOOBqiLGK+t7gUtrzAqogtoRdi5Kt2HfFlzl4oWb5xHn8kPiliMdtN8NX4pAsR2/Ru07RWDGrwuOKlx+MvF/UaA1vxqsHjyxO5fZgErv2ZbxvpY5E1XdKjcng4r1M8+4votWQKl0HkqMmb/99syNlhfeMeqN34pd0zNl1kEKPFCujzY1ywyclxf6G/ursG+y/PKFDnQOQXeIkkOEutUVLl6VavUEun6TGrCbdtfWGXB++/HKKW9U9gRiccUBzJQug8ONiPxXZD3p5dCPJ05HnWANXkrR/ariTuYLWQ5SxY/RNxg/vNoRUrh8erNpvvm+CJOPtXDwXekkUjEHrSPNMvN/thouYfrN5/vFMNTS9jIzpgF6z4zwc1oxDydrKnZSc3EMgqiOHmyGGwAY0DpGt4bjr1wxh7eE/za2DOcbKKb85VfkvuiaNtSOxEuU3QQ4JgCxg3UCr26TYmg08zXGjux/r2c7m4TtWahNK2+AxkhHkWQjBHpTEWLfqAXeWv0Y9HdT93IWF3c22G/ZQPWjGAvkeXDZyZUByph12H2aYKyVjKXvqYB7TDI5thEpvOv87lXdTL5taA0g22ALXMJPCKwLj24EEeKH8DfAyKtc7T8zcrOb/g9vWt3APSz/yYemx/2WJbAMSY+R7Yy1edF0kmx8DchMQSYiiPmr7ldUpNWv5eRvEuU3lRmIUAi8/DziukG1NC53g06UUp+upcfKgEYgQnsq5XueZOx7cdAkwC3lBH1SX/EGoFPm/WHJCq6He0SaaCRwZpP9Rtf9tiCQvkyhaKk5kXIVfSDhzIga4z4JjNtBPLh/WP3xr/t3sM9/tg953/YPbdyuc+Sg8HkzL95pX757iAvtHkBV151WYJ8dpUIx6XxOujHBEBimG9okQaEPczk8JFLCbF+j4ROTP7eynH53jCteqqLVyMIGVQPmKm9ll5Oa/jle0hGgX3c5vZlHizdS3RZ3MYmCdPf3Z2A8NbnIdbRCBOwgdE6Y+sNZUuBkA0QW7lVCg5GEbtwH8DmHTjrYeR8WmF02YP8DhYvbXxwv8nzWKO4JBMWCZx2k3ESKCc0k+W4pg00bCoSu6zH4CXMa/KtdSscqkNevvNntd8G02fcKi6Gu6FW1mHpUKjIr6Kh4pDvDZQiuqlWxXzLarkdPEN01su+KlLD4GfETMZdbKFDsmOzNsHRqyccKEhKoqZ25THU6zqoYWSx0FoZKRPnaLhp3HGFZFMsYOE3ATYm7ou7fiFD54Fu2kNNvqn/er1htj5OXqR2NJsujTrry3ZPOcd6d7NwNVsQXtCXXkvjj8HcjOVQDpjJJSeG4Chfbb6fc4hHzx5Jms7rUDqPUScbyx95AmTA4HDeW7iKrve6829V1jdeC1PxzTXYKvg+ETT7Fj26bx5CHnJJ3Xa7+U6KacL7F1Ruj3VqFWK7hVsVYLEVX7uB3zodCAVCPjr0sb5CO0YGnrGyadxjx2t95jQmRum9QCjzGyxyCctgzINNdA04uQBAjwT8WTewuCZJGH7PYtzqym+SSKt4+5UuK6H8873PhO0uIIJRqqB7+yNfl9bpWioc7qHUbvVZP6TWQnj3c7RyEM/NXcD0jXwKI0LMo3+3TU2FiYGKsIr4FxZAq6VBoK9IFHnBqmWi23r1KWVr7rpXxBpqrz5sr0rN7vOWOOZkBBikAtQP3PDZzZ+8QX6ZH7wWJV5kYpB3Ob5HZMcrQ4uxR41FPdOvtXwZIh+TJcw6v+ldPqOZpf8WeReoqG2XS98d2WbBWEyiPscH1efEFaZTjpu4gVy1EUNILcRIf8MQEgR+sBQ9u6Uv18k0ZdDOdimaDKizXLbzi1bYw/ZVEd7yLMo4JLqhJHjBoa19qEeBzMhgkBHaWZv6QdcLbvLASCNpHzJs7E4z9W4o9kErucdMmK+RMHeyWTKnG7E2mzmS8rjWGiaUanzuK4NR88V5uUdY+DB7FDRyRC50YAuyViXrbahsTywBZ+cOhnnl8F23VSlYwSGCWnUx/1b7ynIfCUY7qVt1d7RHPfb8LNBEJJfyc1WwymcnHwnkWy8Ns17I00r/aGTyzCLvZpBp8ftRLlR8NUKeOvQkwmsO1BIoKmFRY0xf/bfZBWpDysY4T2l71+FBn99YIf7YX8SFMKe8ueHRUik9g7EN4sE5DUC8ZAAEb9XnaQdum9CetNKBJb3u4V6bBzFTorcpwqmZ6Qa+xWxp3mJzbaTZDtfhSR4eT+GAnHIa4QnaLm4ycYfhI4fu8b9sVTHbh+mHTbShGLIeTv9ia4zxtp5TXcFtw+RATB415oFTfPDAHFAOo7GgZxygTo9uPwyR+Zs1uQKjgA4goXgKiuQqpnwYDxM6/pUFPZzpDwuCmBenAw/VUud11uP4QAgV6wTj/fFn4W2cNcw73PdiooHGRzL4WTM+rpiHMArHliBmaMXxR5i58XoTobJmbD1gx2eNavF9OCqf13WNGjtBC+a4vdEjYubZd/vX4o5UNfdXbLR64Zjq4QlwA/MKq2jE590mOv8GNuYhbKk1OuwaUDEsEwZhnTIVijMFbD3G53b7oF5KoX9h4ge3gzFtIuBg7n2iYO+s9vNnki7LrwaFeYzqLiP/Uo9+oJWUK0JYnc5rE7O1JI1UVCXm6Lm6zsAQReSLojRCXqS+kWP3edQaxFH7pxUcS2grTOBF/j11fR2jC1OBGT5sFvW8uTLrI1baOsI0Blm/FLqI2Y5vWr3FeVM+OikK/QfjMzleL6Q5ZM59dxtTV01dMAp4V8KQVzesJ5dhzION/bVKWc1M9J4qaHpFyJODgMLUyld4j5jcRY2E0L/uK3/CgrzYJq361VPRXTi+5ugXzGii8GEdLCJH3rm7jMfeOT/21xtB1szhJt/cfnngr+pMufWlew9qUuX7+wsdysrHfJ90Hmv6ORIS0A7sKY0Cg/Ega+i9EX0eOFCDh6AGfz5h0XFOrWWFgE8PTF0MbhqccTohRlBkF0eglzuZ0xFIaxtiCuuSL0qYL6nnyS2OAyaEK+KjLU596DH7oXmG/mDJ5nOa21NlfS5f2q2mLIi1a3cB+OlSIz7SACr0bGzraV2YkJYwcsW6FnP7WoXz8A6PiWAhietlUlEks5egA2Cbz5bKyKpLCyv72kQtetsK0x617uyFChtzznOGencOG+nrt0KaX7GGfCRmpGSrEYgCoP6K//RMeY++GlIff0wpQ+CAvAlg0H3r5ZvgiaOk/+px9PP5cMwiCycpcc0JCpHDF85en8Mkubr1Ac2apuJ13A9FCPOFOJ1kH1QtVkal5ozYYdjRxh4svzZhTNYiVDLcoh75xo4yDlMdzormPZAZwKatTbLVGo8+DVvsVHyAvxSSyTjmwVemESmevzy6khqbQ2LSThMm/wiyVH3ojOd8VEbF/Zbde2V4zUShJRoHq6DI4kPS+O0/xsHqUTOgTzG3vyYrcJMo6qnscnBRMz5oNMDYhj+oge0GVP+LrGqV/2UwCcYQfNG3PE2htDkm19mpirEt4SU7R8PwXN/KerVDmgiBwLR0m+bs+5WAKsmFfUdIH3F0WYbHcjZvkU0XzZMwzTUiNtSmWPjM7eVTGT9c2cBA2OFX1pRJ7rSRadUIJKbsXOxEAYM/JvcrfPCeyyEkP2rDpXr2aDBoEGWVSd9jelKFc7esM34XHkl+mQ7GXPtHWCoAt3D2wPsPqu1qn+ISgXvvXpelzwuUaLEUXFVZJ77xdsc3l1kqyvq61pizVJV4EDvEfmMQtRil/stJjsL5Xe7LD5DMpdO7puiaMLvT+IDLvbZlolYDHj/TFIt1GR6KnuHucMZmGB0uI8Y8q3IfIVUU/ahCJeB8WeObqT/BHX6TC5mYn9gxyJiNDZnmLKSqSrKgbgidsmPZPbRKv1e7iJWkejcZ30rOt2o+u+MpeePsJy2JESh7+axJuHRD7WEMQzr9PnVUrzXM5cdviPicrrXYmJwBNEVXl8FXkJl/5JyBGg8MmWUh3UmZ9KN87nqBYlPWj/pl0Uona5ssHEBJIo/u/VBt1Ru+IHJez2OXeAZaLJ/agxEYlg2PR3nwMHfHh6LXIXEKWI+4jl0FQjp17vd0ls6FGx9hiKiSakdYMtjHCXqkfPlr9AsSMWy0sb4mS6yqDMX2xWOmOreOikWJZt5BqdjrAVdSO7mQmmDldRTydsovPl8cF48J5YsHYCmCDKvn7sF9sbSLdYEpdV9r7Uv3K1NsMMRRxucoCM8mo+ZRYL9ON63BR+yuCvzBjrwLLJOqikXFYY7v0JdOF0ONKkrV92aU8Q+IpGMDzWvV2wieRvGEFYTGEgTPb75OL/EUTlgABnE/0yZaYnK9fE8nR4wvjDCyl0hTREzlqkzzw018bYH/dgVcuAqte+OCyzxKdpKobH20LY9wAsIXi5ui+YVSn1lOPaRnu1dd8KGq2HrLUz0IZUmOukvDZ3DlrVAOtPUa+oTamid8CGTqi+gAh7VfhRswAt2agSTIGkNGAk1IhL+nMMPYqjzs1mRYdN01/CyHSwHZNI5xBZW6lIasiQyex+0IVoiFXWel9r3vCD9cCtmS4KoNFCQdoZgYRZJHMmuFF19QTTchtMwmhmuIVOzc7a8kO21mJjLnMa2kizH5cbnqS3OK099QoQXQGDLtYZ7ywHEnnjCYoJzCW9RVx5MWRlq44oVWHFXJtk3fCU/C4A1TUQBsSrszx785qP4lP6WKrbGTi20q0dv1WD4awJtE3jhn3Ow0342u14wrHLEIFsNJOUNVxnNsQ795CF1njKoJbAUElbakzaFwEPbIGDWjKDXAsIKF6QGki6LJGHyu73Yukl9hDTmTXOZPVY+u80G6iula5gggKyHIy47pkB61mvf5rmGruVWRi+M1JuCqXySJ8YMzrrbKTIA3ZQ0gjsmmR81V2Kl45zXWWTqybutQIEllc9Wp9kfIdzFcUVrs6EehEFi14h6r95GXthR94uGlIGUjpAxn4QNG6big0kA+W5QOCl+rbPYVgMmhqD4tOQ6fjK+1ek5BJLP7Bvl5YyNXrfLCdccXqurbhcrecEoccc7H7JFJRGr8fX7Hco9gNnvUZ+ZmRaaefSmJkkKiCBUvjzkWr6bkfKnioBsiwqPbvGFRWFGxJMkLoDb2aCVk2I8fa2/0dQ7usO50i1+8FerqmroMRfNAeUnymTodDeAxPmNXdzasXQkmfaKZ7gpu+T7tQ4Zkxhz835RFUV8lzZqMncaTcv+NBEzdSYIgUVt1x2KMG5nCCgVt2dnE4R7ywQqqgqxQlZmADWIA5kKizc8r5vQMzyeVrT6aEcU/2KMQOVUoZFkxHTPIMjpzSygh5NJ8H0p/sEsgGpnMKT35cNTnSvhj7o5zc2/9Hcj0X1Gf5H+hbPpeCgL7XwhXB6zlHpAmVSPgCubL/wg+aDIYgT94hQPAnuFrxf4OPzYRmS8XUph5wTLixzC64yV29/OFLhwMw50Gy6iPhQWOKDb/BuIHCgXYsHpzT1//9HfG0ugvdsKfmDV81kyCu1wSqkKCl3QwiIKH6r/6HIN2osSffs1UrfzVvzmXlsdWfEyfY/kUVTuF9zeTww7lT2fdv/sqg963/1wz693efqn/9FLO0RzVh+zWe/pKLuq0vBbXb+bSb+XSo+fzNXwXIQ7FUbU+n2/+vva/XP//7tv868Ybh+qeRw6t1Mo/v//3f/+83/MOng/Ryn/V/5pe0+fdQTfif7veP92oTcjvg397t//RJxr9fDKOOvWG2TPYxTPJ339dmtHgipHglYRxo4Deyf/2vP+nP/azX//ls/6n53S/SR//x+s869gn9Z/n/Nf1/rPmj5wIsGm1bvesOZKErpT19Kr82YsjjtQHf/3e7ZGNX39qOOvN7v++zp9r/Vkzq++2DHU/6fN7L/83nPDZGb3vvgk/Xq6gQIbv3Gbz3JaPT8P3EcdzINOPD8uPb/N2TqOpMNc79ucqYBWJd4jfuSQ+qx6oLv+f7vzsDho8IAqH/vXO7r/f+f7v7/ysY5OHcJcO7v+4s83RoPOz6XlAZtVv3getO6h76v0/1/i/eDrL+/8/3V+STrj/6en+uet/sxsW/9/f9fUfduN3V/7LZ33wySX6CiR6T/l/eo0/IJWGUtQcU5SpHMiojIY5zRczeqHYPLv8+5n2172s1rySUHy+p/opQi/2v58l6ifXzdd+5BaswOenLxrsfPYFeodJb7X09Q4D0CH+1w/9Xz7/dx/vf9c1f+u5vzqAg+709q/n97/0t3/2NgJogga+oMTuVFcQ/cKc19LpM1kMfzj0KICjoERQJTI84cHqD1KTr0/abv3Wf1tHlC9t46SxFrpe7UUUjzuWtkBctmJBMCMQz6PGRlJv6SIRDV+EGUfIPguTak7joZ7tgeAGRcOkt0fffpu6biO/r4L8efdHtKVOIy+jdQL0BNTf4OheakVzHBCKylxzZynUjzieW2dAF1kuJyhmpQSBRDrPe2XWc2YTOU2c3hueUPJ4bVg0wxQIFRLBz2lnROicEjXw4wUOcNshmwJ+nKV2//ms22bKlu9tdVZFYQzZuYtRZj7QXa7zJLsDpEvO8oecwEgMMW8w36D3cwQxvDW1gfsaP96aM53kCl5BkiSEwIx8r7z5ZTBlyoJmZizP2PCL2c0cgci5z3HtvJGhG+n1eyExOpzafXv0vTejb+fJUtegcJc1GbT6swppJtLx0LCK42UlDYMyn1H/Nu/FrEH2mcmXgSF8WduvQC38hR45URMGj/rpryxQQAlhLK/dMyuyRSMerNz7zVQP1b/hXakBn6QT8oY/VWUi4G8hi9YpUTZN3RNSQZrTPQ1LUv8aqKN3DA9kuY4LwmCtXP724a3Y45DJGP2NSLI1qjIPLHAhjG7v5GoOD4FM4a5DELvdjgjnMPveltcMVgvL3ku1LyAYmRDgQ51dIXdX63cskYvoOhxIjTJpAT7NxXZxtAR3zAzLFCvik7RvWTxPESozHnuOYYz/fH9g0fd1bQTiRisqgHwSi9N0rs6ClkimzFKPblDo5YJmroP/OtUHMect/rZJsqOw3RGb+6XXqmlv229PIcli8O2DpSTfWqW95tNx0D50qJTyRlH0pY3I/DyTYZpI0nF8dINN8kE9FzS/TtJHlmQkwLgP8degHxmk6dsnkjoejkgYkfxbxUyzE8hbQKDO/FQwJAoXctfjFHBqxdqvYK6fR0u925wGwPUwzcTMB4seIxxVdgeBAb1HtKDP1zBPniT50IRyy8Y+qgtpGzG/kkspkZ4Vj+PUc0/uPtjjxApyx+T23adu8VMJfrvGssn/aD5+Ql/grZFBpOPrGcOfd8TH5PBsxiPODBIMlx6Rym/fNwaKO58lJ9sAMQsJjxXoIrScMsBTjvzC6UX/HotTYEmKIOecNoxwY2QyKGt8WR6k/EmVAB4GRw3uPzucjAyp9E1/ZM8bQBizAYYwv/n9Zn4uVjl5T0Q5TUgbYh9teXa4+bhTzMzYQztslL5V0TZCkCeMvl/Nm/eQ07bseTuJb7zXf85C8Tb0V6DTR9mlcpyzht2jnGgOxAECTlnzvdGVcv4+OcUFNCYLKAxA8aKAVc2OpjLhw/piEI7nY8aL0mCcRPvw4+a4Ev95qm5icOKD51ozW86LGm9GljBqaF9XN8qOlgoJAnIIJ5KogsFboI+GyRmzhT/pCw93t+zG7TyomJnOaVinOYKxRssLOHXcNRXFQFPULatq/yQ0+TBTf+1NG3ZSeufmEAu5BfvWQGeTG3YAqYnRlHltg61ibwIlv67IP7Ri5i5kS1hriDncKDwtpDTdWJaGQdGdSSOox9ZfxFhhSPY8TW+5Lr0im/jRU2wTbFXC8wqIzx7VO8FokCAUbV/GQ0mYR0PHOh875NBtShbv12M2584W5zTBpnlOOxSIX+El3/5riAJ+M/DevDfO4GSYgiMyxyD7jm6Q9zHtSzzOtaWSpWHiPKDhgeBVx1ooytBsKQOOommiKS8Elc9ol7TV9IwI8svdPx7cXCD6SGSgaEns6YFEc/3DepkHL4tWM0Qjm66TgM1ttsrpvj4WfosLVqaCDl5UxTvvpPw0B43ucfkrjM+/TjoF8w4ym7IQw/QN0ohiAX/d25Kwdpw4hkXOahKcvOVM2cfA0YlHsBDoA4UOa3JUawCS/5nv+oeo6QYBMgtXl6YXH1/rELIyvqQUvHI+2lb+jACkU0gXJUl86dD+HlfV/JgNQ6IyE/PmPG0gQXXAgF3qXm5eqKdzkOLW4I5Ps4LnAXTgfBxvLw+S5qlUS3Po1Vq4CR8HhoTN7LOHXVC1wI4aah9NwCzupigKZ29l5zuYg8P7Sbg6l081KcwRY+pxEH36r4DInUDk9X1sBI3c6woiAaJ0huIBvCQDCVymIh/hd6vtJ83weztWKlr2ZBcO0/bVKpAUGixe6TCTNoFjkWfho7f8ZH7MTEc4lswr3/ZAeZ77TF9XhsdSjCK/RfGGoZHy2kTpQUURV3yPPYpXhrNZ5yIr6UvZ3ahIwzBOX7LhNIAQrIbMFgy0t0kXkWMhXk5HrwSRDifUNe6O+xoahttHGn1BapdO46HIoqIAtguKLiaAHdYtzB1FjvwBlQH0HMfAHGkjFesjFoAUNGiFHmewoMCH/aKF42NHa3uNX+RmAwblOoL8SMJEKNtnxT1ki/NedAZIEmNGeIDo/az+Z4wgYqc9suWv6rTGNL/khPAe4PPtDE7HsVqHNZMZJnPwW3snim0mW/IGOEoC4bFD25Ve4Le4SoO4Mt90pyZObt8f7I81jxqO034p8vuY4jT36pMiRZeCM3j5V8CKfNlprUWF77RKJNa2LEdqyhGneJRMqg7QAGpavCxbGYlsJZL3DtnnMLXbes9jhvb2Itpk27N0wufEZnJR/tbP1ZgdHUYtpj6MPcLq4eh0zJoBrxiYmk+LGigSwWo2CYpNC+hOK1pSPtYiTDDfogz7KGk7ZcNAyj6vkLtsnjuwRCrO5JXnuJGgTUPzX+Y5Y9o6iWa5eU7iFZFswVqG76kqFes/9nBlqyECeUFa5jIaOpgDKrq1ubAbA5f6F5RfLDxGyuStxecWNVBmPfQEwyJcnfe9xz7RJqNUqfS/Nn+1LYFyoBuxCYVChT26Xv4EYrZV7AuJBoCQva+evijnTU1+9L2ZCJF4+dXghYA2AMiV5Dls5GD/bJTlTSL1+aRIBAntFGaeek6sdiCh8G2KUWWCtzq98ko7LHDWHKVb1mOIVOVZu+BesYe9pBwiJu+MkQe43ewlWJUPVbEOamPPaZqwfFW2MXq9pd7PSnLFu3H3WPjZ6k62e7HdXVRuyGtA0Sz9a++85/A3ZSJvOA4daczzr3FkZGah1YckCNXnmGJgQS29HOCajrUI2ugVldXJQwwx4A2vYzRjpbmznvTbwDqAzm+CNYcG+pD8fDAKh4BWlyz3yfBfF3h/Y6IWOJTD9xtC5T/Wm/lFzFutu2jFTr9GJzOwvFlcZhBRvrN0VrAlSzC+4LUZsngPvISY2In4NhSt481YCvPq7v1c/Q/S+8dLwa2V7j7iZ7o1U62J7Tf9CqAsomvmQHd84Gxn5zj5doK0jr90J5DAwJe+85s+hOS+Xqy2+Qk3AP5d8atyDvatuC9F+r4sQUbTEmiRyEeGRzDATFm1xD+MJGWPVMgRK1winBgbG0+GJo0M/HqB6huR9BtIsKi05zkzIx86Z6uOiIWjGvcuaT0yO8gLPefFT1emuqaO5vEoj4+uNJ82ZJnX6T1LFmfY++FVUFCZJeORN45M0DlpfI7rjOgtdh5QjHRkl/qG9hFwpIeJZN3bPRMpiqBVxDyPeDh6U9KHmgifs7mbPN+wzfX5oH1HDFCqJdR3vREiD4UYlmk8pP4FSisWRFOD9oiMh2KdwMr0vAuvDv7sHbDrlQki95PvQWgJltRGrBm/CVi8YWfRLBVvb/2D1Jd4kXXwGIM5jrruveJ5ViXNRE4hTFjXH7nIrm+YLWmu31cIh/uaK7g0vQb/jVjmh66Q2fDdAvqobywtI+pLsa9etaCFkobXHwMcV7X7NUnGtou6KSDChGlhmqopEo/z1neAjhssHQ7mD0cjHOYX1oy55Usvdhqw0XA7gZqFd6wcqG3R6/v5dCJHXU4NcEbOjP5SdSX0oAsTqaR3z1phYoUhANSe3Xxn+qKjIuEFWgsA4kWXB4pB4yVpHzhQUTieRhLNhscQAYTIW4DvyBCKt8Pw+a6OQm8S4qSU/T0i8wugzknkxL2anl69kTZQtdiQ5Gc51xrGepFEQf6OQ9oXp1dobvMGXrINU0RptENUngTui6TfLC0fvkQTu8Mw0b4GwuDxf5D6Y8VtLo/Naenk8ejCdxGf1nrSy/fcPecswrDWLDhHd7Q2yjy7600zs+TjL5by9vKsy8cF4s4mYZ7jCbTk4ucLeGv580tTXSfjS8EOTdHowOJOMAjRsHTrWtvhHNo8dRbEZUK/M/TsRIyStyQNKYtCEbOIkqqaxsTDJjrWwbU39SToUNyy0dHXsKrH/hoFL+bGTpd+1OQDRf3tpbrc3g5KPxCDs/SxfEoPgaGXI6zoXy97yxcizUlRxMyatLylNjoK0iK8Mzu+yQpkG+3d7Q5D44Km1Qhmy/tmQ5xVLBKpH5pHCwMQmAfQwXWiyMZcokKgPqBrB/oE+0KX1Fkb8XLmvdIUEWEKn/Otd34wuH8bKp31Lyw5ufCIsqWk9GbcYp4NmteoOF+F2DfJJ3wR/7Sqq7uOBw1T06H1p/3JqvVmQMDZL3kbMoTmiGNNhaMHljTPydZudS23JNTzr8LsZvogXRSRlfIOHqGxtGgpM3Rz8qPuowTn+FZQeUs1hpn8dD6NbmBMEcvqcKIa0tJBAR8hEEMvQZcoPiZrI0+wPXF5+U0/pmu6fh4OppFgrlwnpjjLpW6rcK1G7zEKUYRGSWRrvlKMS8i8MeH9LVo2VZnKNyuPvrGV/nwht5aYnDHJTAHWe9O/95qdztfTgZ5m+QZSZpJ71eJn+FIfVwTBxGZ8Hi4odphMCw1Obo3NHWU3G7iElE3dzj0HMn0LDM113mbNNnm9HkrsF+8vp8DtlR6oPqlMZHm0H3/Ph3o9YA1PPuH3OzGfR5tXGrmWvIW9Ia8FgSEXzSQ87Orvhgnlwxblii7kV51/sMBesze5ay0Ikr2m7QDYN3q9krcnJPurSLpUChx6Eol3OdZjcFtLaAD728Zfwog709Vity124ap8WtRAI5265QfoZtwRmF+0DksCH5T7nQj22Umqx4nn+z1SdRCrB6+Kwiy91A0gP5ZaYPnmI95Didgu1T+MS72Q1ShAfDDaaWIMv0GWLh+2HnaG0O3eMM8iw6w/yUZibuQXuaO/oq4N2A6VSeG3I8O19ewZE71hVx5LVWQIhBIcfXG6EFQ8zG81GF7O8OtrsHiyKJQFlWvkaHmfjSaoKo0QiopzWhYIUE09/HoQhEvW+Lw4qpLGlB7w1R2fX3bE82+ry21305z1xVjEtq9MJNxsBkdXHKWfr8U8HRDVREYGd+pDuX029GXxYlnOeKdQ5ZEaFDQez2hl3sJ5/yjeKervYiG7vtKSeHlr8Zuj14IO68iSHYetZCI5+wq8Q/N5FBfwHRJl8xaHcWDvF/DwTL/iT+oURJ8zRo+dJ+sbMniWCR67M8GPXHP1yrzWb6MWWy96qwle6NoGrKcMYesu9pvcwuWju7SH4aw4sOds0IlOYpQvrzU46Y55zF9wGVNxvJv7b68k7lf2/NbWlz6/wahQ8dQP6HClneAETIoZy2fy0g7ziZo7KjTD3NODW0tdUgLI2wlcH9WT2xaRMFG/tUrPbsY4BUwQmKaNBfXT1Zh4vRngudEs0bn04IEu3Hu8ziCHZZ+VXwWUyRsdu4t0oN6Y+d/YtHE8gX1MotPX8k7oJjKUQPxKVQKLIoqACtwKmrsJjpVsZsUlDnK7NHwjJ6Wate0Hel9WjgIfZDYdXEy8UhBAFaD66q6SwA7D/waUXn4KV5mqqLK+wuY0yjqdc+BlLd1Osuwwodiadmudr7fSITiOdHLPtkFZ3ISHZuMwc92mW0eUbtfoAA+mOGfAo3N+MYusvAULqygpR1AOKtaigLEZj1IRo881dIfjStxilDLkwi+/SZy9LmcvPjkH8QoQiOPhQe8YT+kGwTbNjeCLJfiNFhlTUbICcSUTiUxxPF4KlhHJ/udBkH3JnzDGOQSuyryKxbFy/OWvSo1eEGzkI9RrOn8dE2NsJrqtEiXO2QxNVjGa5FjH1i8oCfITmv2hffUPylLkRNX6Ld/DfuzetuNeiCXBXnzoweJD0moZhSVJFz7BtNRCUWXTw2UfNnCkuKrBNBXKj6+K7v1g1YY1x95WN9Jy6J+t8SHX/YOFTBtq26KhuGmi/rxb24isBH9SKm84EnZuSh86C+w+BA24gd6VlNx/7BVf6UAE7Xmr8EaqTfJQnaDs6F7+fo13ky7D0aZpeca9xTT8+D7i7kwfpMoVSmiFpgwtaTU38PfS9E/AXRXSMguIISpYKaC4sgIz3LkDAbyH9clixnMDnsH8Ky42iIox0TGYsI+gqZmLlQ0reYq7nTEXGQW6lih/jWqriHg9mgJ4ZQKG+AzhQ599JW3e5GR9QEkoq3NM46Wj4drbl5xFWrTBd1vPvxfmwS13XFFyvspYStpYdsjOXVEn9kYn+65YmHkvOSn4PPKKaBSPIDo6MC0kjIXxHYpUEXwd+E4UK4VPs+ysX0ew2ql7Z9MDKku/JCZvsn9az2KKFMQEVCAumM+YwUEav4BOX3UTWzgSE63LbGecZHnvIJCRBzmwV54krSPYWs7Y0Q2emwcpp6VOD5Clwgq86N6VfgUDpAQDN54P1oQayUwu+caDqTfTtmaCYZx3kUSwvG1MM0IYIs5AIpjIb1+OPN38ao+aAIVOWBqeHgNd3UiA9yR0btcEUpjdTPYLV/UQuTnYr9XxL7//jCKsbhxJJZKvlvbleZfAHrE3bQ/5YpgmfENoJVjSiUgNd+1WIpIvv33XdPVJJdNZsSmbm3WsykFCtGnDizWiydNZCa+B3B4zCkzyAQz/nKnMQBXZXdB7HzrG2jK/7+eCKKfvtj0HpKl53Xt7aqV8cbLTQmlP3WzPKohMQBHMl3sPsMHyMxqeKaNH4JogMvW+jgbswduJ0Dyt9MlP3PgaiAmr8CS3YR/4nhu/rphXVRlZyYQUaCjD9jUeO/bkW8ZR2VBiAwYiZGY2wweN2lEkmY+dGB5ctb6b1zHHFCrQsC6AdJYg0I1NJqaC5mLxnq5CqAiFIlI6XP0Ch376XLWKGILfdCy42BxxAb2jwBe33/bnrmVjZdy6sl2s/cA3HhrHo/qGzEtkK7MzmLb9DGE3Y6an3UKPHJrVLT8CeH9ZLg2V+CcpPplypf3iwh/8eiW864Y4ZUM3vpE63nQfwmK4JqXLkRSTJmJ+ccSk/Zx3AfNoubElUmjfpn7ejDaglztPrxbKinXnoSJkOPsglSRCIbuP3ipar4ettN++YR+S3h+eWWA7h9A0UcpTO40WKm5wgXz0jKFL/HpokbwvYxvqw8ALgl9SxblbMegmuTQTxlGNEF3Sxz9foh56zUpzxNhIMiwzAYGSwCGPK3fGSnFRfpnQh8Wo/KCSR0I0DryBNMcx54WRdw95RlmDRvTYFlIg3+jXIhkYkg4It1V+33R5FyHVORMkDQTTNPu2AY7sZrSG30uIk/xw/7qJA58qcPjGMzrswMCDlEAQDEvBT96WFaJNhlYda491RniXDNlB+dx/yCKCoMf13lLgA/0aTMoOMQETMiRtpfPHucE692/yyfcnYvavrdxMgPBWiRdWkJAD7XlHuCXDvoESuroke5u3o618uP4vrhtJ4FlO9hCAe0wldtcec5Ku0c8POLLJOAxmyh25+t7BPX4jcNdVN2wKH2r7DpPG/iVpKw1QCQAZeLQEXP8L4IveEDJ7nTbwhpm0MNp0+4geupBMrmGfsCmmc/ta5iuz+QeUvuBU6VANY4ZHURaeZb2XTx/8ijDWIFwDPJX4g4eqDNwJvDZvgzvjiQXC1pnp2WsB4tov9zdO9q3WPGbc6OIhb+Yt111HPxvG37/yg8XWWQiDvtcNtoFCo98uy8wrVGO2RwntB1dFrOLgNg6F+SiqArYb+AK10D/7Woo0+VIilWm3NWVcwNw+ijrJKS3C6FTs6HnjdmOWFlMdKLzZ81e+haYqvS0nCw1cnPC8/jNj5REqcQmW7NojoP9R4MQw/zyAId/29hupu8J8zvIUNZV1mTelFl/kGe3VQRKDdcG4Dp0lDrSDZ9+m42Q56OPOmhWDsednRuTfFGQuVdpo55vr82sxA7wXRL3N9rfSehwqXZfcEpSjH817WMD+WgMQkjKvHKWPhqz3lrmwL+Atp2vi3PbFYtyDA88MqY6ENweH9gUJAwG0UaGtA/r+cZ1FLPSyHqZSRyLqgDFMvzGu44W4JHYY37ngeJhituyfUh0Fy+necSMSeQAkaGNhgquYXBcPXbywzKIJKKzgTCvn8txy4sRv/ZHudmhtXvlinnuDPiUkCNJ85hnpK9auqTjF5dkEspEQNjZKSbo62G58Khub7Q8wAc0G/GDxTol08Z7iuggoArs5WLZ63Iv4Dw1eGlAi4cHyYPKxuCnuljwYKeNF13Hf5hUJslcTFmtCBzU6LKdW7dI/Ym8MtoO+GPTSepBrKgcgQb8nJOj17Zx1us8CUbvOSUuVqD9lIx/bir91DPVwgqYI783OFadw6Gj5tMOxaG4cij1/Ek370is8Qt/KccyMr1bHGwk9KvyKag9sV8DcdRDWaQnAwLyozx1dZGRqwHziHb7YaWwyoM3M/UTDBGgHBaZ6QJEffq/MEbBGvKxMtgUFvedN6hidsRJ9gFwIe4hflipkKAXocYGFdep9ix5r+74si5pZyB8CoI8gnu2BkOl3yYAtwVALtlSQdEzvUILaif8cO7jcy8Jgtei60nGts/Coi/HzdS8U2Zv5Kpol5Saq6AWsqGgq2ZIjAHF26NrHwAa6tS3y+jKynrP6uilM9huybi6CHUtBBx7KH4HLQWYs02K/V910nkmfV2Iuk61R/FfVeWyZ/D3VHAv9IgxlPQSm7CIoenYuchY/zUR/p3U+1vRmJcv3Qn/modDuu1QP/VaBcQhKqOaZm9jb7UZUCJDM1+bIZpUXTXZhAJLGHooFA5E8IB0hKH4zZFJ5k6QPyOXMnzFubYU28a2l7Finbso+MdHw62EFuGIlYgBalJQ91L+yBDVnddQOaN2O2rVsfp1u9iMm9bU/TfdMjKY25g0gX8P7XMD5KyHnmPVvVuMOuAoHA2BFcn3xoijQbjrtewO3WpOayzLhkh5Km13babsUp5vjQBc9tLj8wn50BZ8ztiVNTWW5rmjlxzGP4kBfif1b72haauhkxYLJ6du1C9ziL6QypivJluIFsZCaG5zFEq1r+cBtkK6KZMwPlyPh8qzEzfRDwceB6+LbsQ3F5ySrG7T3MPPjG0Uxvfky0Ii/GWe7u/cEML3ejEAVU4g6PAhm+esK7Q/d92GEL2BBqvpN0dS0bQHLrElHzZbYybwkc/rpgQZ+z68cyQ5QX9mgRz9gzTsPXNLLx3MwiP0sW9BhglAlEqMxWJAjrn3JlvBO5ayTwS6F0rH2EDJ/lawrehtWQcsVmFQ4ynKJ7vZvaHkQS37ct/3dY0S3+WAbEwtyLwAshqZIZJ+0pmdfmMuu0Fc9AKHB02/z7ek41hhVq8NHB87hPO90OGnk+UP8JVMZfEdNlm3vX7cp9djZfml6POHdg0Vi0yo7rK/MGRhwLV5iErG8KwwsvRySA+Uy2buB71IEWeUkb1JzZqmai67dp56c3vTt6h0p3CryhxWVDlDnVxQtbOsA2lzN3jyTeB3KaqPjeRHhv5kTXyyc4O+teVVFiraZhFHg7XHgkOev/P9Yvj4i8bQHrqaKEsWUw4esKHEuq4fMipSG3pULh/ebF2oGRfiXmR907klp73YgIrFKv8FL8JssK2xqmVRbf0kNQmDGFpK+VBpEeKD8AqDm2HgknmaZiZg9E+T2Ey8hX49O2HElJrcDr1W0oVJE4seP1RLl/gJoig2uiFhdCah6BF6g7NnxD8RYb17BN/1ANGYRVEjyX9bXEdR7rThGV4XvXisRxFVxD5SCXZD59NiuAyqAv2eq9XeQuvZadiF+Xwzeq4bGthJ0Rx/U7zB7M5ZSaRAiv1H2YO1SHsvba19TCQNd9muEZObs9uB5yP4SIg3V5YktdLN/cOw1anLuuiOhBvP4M4Et/MfEnxbcYur0fkX5F4aRJWtz5ddosxDujBPxG2WUF0ivkw+AT1NN1YGoq55evgT42DRJdU0s+FaFx25/Lgl4/8pyuXVjwEZfPpsZxEfjiVFpCLhAaWIgs16Xx+sMneNNmDdvwSejULJqTWw0PwzaLUZ2+CpoezGrkT5Lg+4Q2DXxCwItEbSLpMam46bEDHWWVUigFuItyQXcJ+w+k7IbMG893fBOcVxGBpguNVHTARsvn/7EdIc4TjAz8w5LMcap9EoKXHZk5nMp5TLMbUnhaM5vjHUfS9IiG3MiaQ5eEEjYL5jIozdlAXRwHe753QaTtneECufIH1/TF9izYJyK4RS9kQkFhil9CGx7t4Qbl02Zow8vAjNf24fS1Af+Aa3d76Jf7Pm1InZ7kJ3XnOUrH79CitNXoLN7yiuLwkQkAw0g38zCqXRGJbD9XkLhJFB5rZ8eUpEZrn/407X1gZXXm9ik3vCRzqVNy0JnFxE8CBZHqqtqa54Fuj/Q/sICR8C5RfXlFqkGc66UHqGFgaajP6zl0vQW3CSN+7xHxhQDbwST1rEoVlUNoG4mOoIjRwVdcb3jMb8WGY7vs2UUvedIetHEG6OpJZaYHQFje4GszPSdjVIAnQLY3XIwMnndr33H5E+QVD/HoKJ9c/XrNywNW6Gw/BiNS66+QnBHYg/nHTHcGTwEEuD8+EvwMPUh84t5z1V6dOULYNlTcoVdyyJxBG/zK1j6MZ6x7aowsfMPbfULlmYEXepo23xAWsSOvr9xtpah1jyyyRn2nN7RVUR5vRLeKJS/WmwLFmFwnj6oSLxp8Y4eEX++P4XtVGUU3EOHHVC2zt1A35vCyupfe6nmDcSLDES6QXPxgNymlGe9Jgp0El3B5a7G/9a9zpy+UZoF3rSm9VpXhivUyDo2GsCq3BpT6aBstGR1xV0avQRL+glnZnQxgnljXEX7QLEMyAtmuwR9YR2CL69Dl0SXSTCyfV0XvC+75wixgKSRDFbWB7mpCs1QRJ7OUkEIhl/wVOzERNN7NH3nF0TtOJmf+BsfdH2sDB3me8P0GQd0DBFjMooA/BqZ9zvX7oHQbMz4Ii6U5SpvCng12l+w00M5GeQSPQQQwbSKKxgV2YtE07ubyKDs16sIRkkecTc8wV6LAgGaaxsnfTYRh15N67hz4clQhJWoAnUDKu+EWYq7Xs3kXpwvqH3vxbTaZZ0/BlUnVsra2a/c0zWPI6HF7KNPh+2KeemBgPeeS9tLszyfBcJZirpYs/lXbvvt0mFFsyqcJD161Eq1uxBqBPdq3gdGvElEE3+utb1PgEEezXhKb4DvRI4tg/rUuunS3pfDYXrEBcrgiQYaSUGVbQ8/YWzXedPmj0rynHU1czynbFIHQZiVp7Zup3HQF5sNO4EkizPH5hW3fEWYQMQCjfgKAXeKVfSQqraWwkahM2nfsK6gcavaHxVaY3PBX/GB1F50Hr86wwjuAWLUCd75AUaE5LBe7oYtbBMVwGX761bRe2HlPiJDh6N2I319/a2sWaiaOhFPYOuAlQYNIXslMtspbXK/bZR1QwGJcMBe1kNieKJaSOmE/CRVAcPgYV2BxhMc0XH/UVdjUOD2zQBhWX8thG8i71Gx5A9fntj1OIQrF9XVQCHWCh02unN5StIXafEfwLRGOt30nKDHoQbanc7yCcrT8GhV1NP07AGElGpxkfX+stQsjolpdvTlDfyGZlHqQjmMaPoLtNPBYJUpugQsSpefs/OFjZLZUwp6DDYlTf7CZ6/hHeDDwbJQ2s+EOBcDTQRvba5zWNUibJODH1LRHrZGXvPzD2oiTPdJDjZRNwgsyZL7dZHeOnI5TCeDee8iEVWTslgPXtTJhdPA6AIRGA2qVBVq5rChhx0e0wE0h0LHN/c/lH9/pxtjKAyyfbh5iyjXtO2ueAni/RvOsAXMN44ItzeyOrObD1BSS1W3ztmS4ikIizzhINa1N5gzIZz4FsL1UMRWmJdMcLxKgxx2/LqTWDUqYz624mMIWkPJBKP523NIX5VIa76Nf0XZ69UEwQ1NwWuva8BFgQ2L10w+dcM0y30chvc7XBBqoicxCt+FRJp2Vm3tgybUi/3UtbmNqOGd+W55c2HP1gg8bQn6xZ3yf/N0FduuI0Hya2YvhqUYLSZrJ7aY8etHdV/P9KbPI19ZVZkZEUlA4XBBEraalKsGslid29E3SN2CT35/fXsnPu75mP9NKKz0D1dEfeny31pwT2o9RXFp+EAFK8MGJxog2G+OMtjuU3m9uZZhhEtTGVmBIOQzE0uQjzne5gNTOLNxG1QpLPbSgfwJEW7CkcAtfyDyfwUDr1Pzb/ZaGiFc5UubPX5/a5alSN4E4Liq7Ht1q/Z06fF184gwqTNuj5DoKsaJX7enuR40BsR9h6lF/hagiuVkPLIWh6D4F/5i5tNRiLE1j2FrMekxPkdOXB0YNTWD731x7PDooR0qgYrkNYA8QHYPYJJ+w5X/0FcUpAzgWpZuKyB0A/UCqxKRDSHnAu63mTjwP4BnnL5xDrGmbremgUsQ1egzRFalV5RiA0eMCbxmnKpYrZhAl2xtGn5F0cO8rKeHn+wqLWPeCbwH/fagAT5TZ9rt14yMecriZxg3gaLu2YtxRumnU5tP0Wk+9qb1VpQceJJouEJuU2FBZf2fckeAau3mWA6dyWfyV6tF4MSfGyiM6XNbdJYyMsecQx7NhL/YEXRmWvTg1I8CVSfqH7bQixNcGj2fiNC+nVoodnaw1HYFuLeuu3SJOu89Krn+ZHZAcntAEJFRfjLP1poiEGnfFSOZCybVVVfKshmpx3kD9xlCywWm/xFdrMD68P2c4Vrk8FOZTJfAL3PwJ1zC8ZNSbSTcw2tppWxulCdqKuhz2WkD2KCwbvTNzL2nX+BZDU3HPQInfkkngJO8bYDuLHuVtbC6WDLN7OhOvAbPlpbIzUzIP7ee4bIB5yyunHmVsFZGqo49eS0uU1wVf0lopY05i5XfvJsc143VgP6pHPrI0RmGwX8byQshqKSLx5DdPpzYH5YJAUoj+jwpsSF7TY2jrgWzkvshwQMLhx4BNQ+6HQbsB9ssHAUvbWVcmb3VCGMV8Nh6ZHb3pjcNYS/9yV0SVSKpT96diudSD02h/vw4A3CIJhorvH2q5mPlw0VZMN1T/eUr2ATQEUWKVZCNnL7S8EvbKmBxUvnse1V5KzPc2gsaCqdgny/EvBYFbrPEzfDfM+zlbTa+s2XYS8bKvwJeL6jCcgcx2grYF4JJvE1fNssqCb3HL8T+1b/J9q6/RVAUq/no2gOYeBGratYaj5/fACIv0ruYtOEbm1L1JaL17HyxSWG5PmYrIKUoKmFYO9TxoL/h7sm+HMD75GcA857ju+HpD6+drB+SG8KomqtgFSsG0Hv9V4YuUdbPvrFxMnRXIn4OciJM9r1HaVTnXnmfWEC51JG66tsKGXB7CervSlKZU9rHDB/J7O/JYkI4IucFZl+RIzzw18QfvX2taWETGpjY5YvuGcRiBkZwDTefEAJuQX9TOAFCKDpH/IJ6qiv8DlpcGtXvxTHfKKpIjXcUmJtPaevHkkSTY7CwAKK5zCGsp5Ssvf8IAFc/QThfTueYL7WVFaCe56H6+ieZjQm+gC4Z4IAc8jnxGTsH9nKXmG9R91NkEsFAJ2MWZ+iDWZ+OQZTxiJ+ZHtq+v/+WPdnSFPC9Y1VoXyxGSLvg5cJ1j7GkVHjbk+aiaXxaHiZwRAMFiTbrIh+2JFvV753fRCbuUJfnWurxV2TmGsaVOBuj2VvrGgsfcJOyl4hQzUBjcFAj5jmWTNLOBoX6be9GiyKAbJHtrB8+GGuVi6xzDr5C6gtagpH5A9ILiI8KLhzF6Ue0Z/MQFh1FUAgQtAJEHDQ2HCqAS1H/IFNFmNqWrE0zH6E4XACz/1T+8DKN4ERqlaHtGkL7El4eN2aFZ3joISAyb913X0fKGzsr9mBY3jMgX1oZ1Vek86YbJh9o/IMcTLXPkO+nt4Tuh0E31siBQTakVnclfSwToVapZblQ8lJcr0PKtnEqKQzL/ociKUl9rSFPeTwAX6qioLX6M0RIQbYevjvBY88WQsXVFl0GSsYXqvH1+VSfAwKREHY3t0J4hgk/Pt5kcvWUW/Z3XJVk9namll6wtBaOXGd/5y/FAzoiirZQXWEXeMowUlV3lg9wKI1UsRrejaPixh2dftPub4zREbDwr1Lp6vxGVJOPyCY4hfETKrjqbtEINFQu9vqncJPQ5qeIKy4MuARtA+SpMHtYKNsZarxZhCIOx466t6PvdrvcGTu1ipoGx4hhKnhDF2LhHL0iPt+JoSeT04Hvli55EQT92HgSg77RcAfngHEtj3BWzSJMSljmAMLZh68pzB0A8VbaSIZMnciNlb7xLdy2OpibRmLS+CjbYouCQfskdFxj33btO4dm+jNHmWjRKxU+1f+GAhTGIs52f32YiBDqMfu6duEVw84WQXMKrr8cEjmRfvyM4qNggRrWW3w72p5S5CXaA6cxIvOG5zL5EfKfVFijBSQR1jZfX5XxqoRmslpVi0QLASMFuoudR5S8Nufom3ovTDZFFVyWegGqhe6HtqJ0UxoowAZpK1BmptAFeckbCjQ/cb+82zB1JjHzikv0HVhQlVmUXVeUr5FTAKET9csxcZnxogpkT7XzuyZh3VO+09obIWcAmE/9A8EMcvb7l0b4gToPrn1KF/9YGK5+ypIiKJBgMomIlOQHy6cHfa8ZlB5BmQdAfXxNPzcef0nrI8dwxIJWBgxxQNvI6lCyPuNrzXG8SMvkARBq2Nz76G0WFMftCdzPuNFrQApjY2mFiTU54D2MfgGA7vM6BGpdv04+hsOmNAyzKIh8DmZt/jTg5M07DmE+7BMCaVG5XSZUbW/imdYN9l1JyOj1kFIRFdi7NifJ/BGl+itrp9KKYf4jfX/YDSQA0+X31xbGyZaIud783YBL1ee/adJfIHMFKjxEg1qalPfRQ1ONVU6XvB/1Zf+JWGKthNuCuMWW6rJ0V7JNU1RaZO36cDZ3lxUN2pzScPoJ/D2Kmr/Cl86aNr9/PZAVBmtiRPLX5z5RTMxEStmTXoT50uPw1wSA0PXw3hMG+hWySnGoFMWUzVnnBIQgv/PgZ6mS4Nh1tcUDYZ0VQzUDx+4iXp0E3PhbziYQetGhQIxhjdmgp1/uv78JHwheQOQGuZD1BfvmRuhBLaQhvACRt8X8PGTVvpfEiZb6lJkJYI1FyLcmngJb/SWxqnOtenXP5CWMIdQeb9am+WdXEVQAvR1cdi8x8QMbZssULhujCi3p1M34gwR3vxxmA1L/TdeWcEKgRDzTU4tfXegOneAzuJtgbTxi2y0PYlJ+vhrVfHCM7oz8GxTrG6paGy2ztm2xnPRqRoQiei9e5w6CJzqXrnkvV5lzf/Dws3mPUumzSaBpLD0xbthST509Sbyo9fzqqX5k2j3Ff73YwQ02ViomN336m7k50ilHfq+GJ5ZSqCEhnEN8HPBV7NG4w/ooE3piEH7Qhiua15Sw1w6u+lS8t9CcJ+zgLx0jonx8SRQtZD/+tp05Bu+BWs2TjnOHwzAjJ4i0fEnz97fBxvXnoFgXtyR+Uhp8ofcQ3P7iMwvOsBW/ovxDn+Bd/sa+dqRYaeoN3poNpAUbqvsVKwFjIb588TS21YfRXWKatbzmWiJa/cI6DGPFCTo3kGc6JjiIdDY8dcJdfzWCPonJSTO9lrvHoDaEKntlcYwEej8+lHxGSMjvQhtgk64pwQJFfKWamjKs+BrqaVQ8ijIO9ckjeUxJCmepSWy26gFoFpsj4609Wy7N/brbyPwzR3FD0shGQcbh84gjGZW3+m09XxztL2fx2GKoY0I0Uxn1kd98Zoj+7XhwiJnr0KK9/rGR/j81VGZGRdSrr3+jt9SWTsf1tpL5Cje8Ud2/3VgiprOx8nBvC1kOeAAVZGlocMh2dN/vzzCAmvMRurIqzSav8utHjJV1s+P+0c15AfmVxP5Mz2N8IL3ig2ow/MxGZaozGQsyk9d9BT7Q8dSIpD8kWynyzuHIR/7wnwFK5egpbtdNd5564uuPOWfKo+6VWxlA0hRTcUZp7fBKVYG9fctdu10kn5fsixCSdSIXecgCs0r3VlrEtHpaJG1feOuOmKBqU7MmKYMOv99GQj+mikMTTiBeVNf6EbMtfyOT6AopolhGgKEDAPOs9MANSOOwAy7T1ujNf3l6hvj9zTu/J+7YxW9uIRDrQF9H/sqth6Lu3/D+RXbegymhMl98pnjRw0t6Dx7OpD+VPwfubYt5wdt+vlDRiB4XYkondSEOjuJHdy6I3ucg0v7GMYtid0V/6LMqVGyax3ibV4ukQT3dl/El+Pf5qVoMqWFC5tyInBCKND1noPA6h3Bjj3N/6t58QrVAHJ+ejcqvlVi2bv5m9ZeGfv+83+/RNPPBP+JOTmjMiWDpt6jlMj2U/IK53zkyRdJCzuRv5jXwX0B2e7if7i7JX0uT+OMZR8A8ksspsaJMRHpukyMSXsKxBHt2CDIz/Usl60ibeGHXsY5lH/tIuDkoUPQF+GjbSqHVAZGA9kC+KfAa2feX54UlNrd19d/ciHwjngf7jfkhgS4T7UOwf70gXEE7Vy+p06cer0+Kt23VBsDbsiK2PGf3jb6/gQ0M7jA31xz05lYvYYyQgvDyTsJtiNUxg7bjCVF0XWa+G6BnWwewyQtFp8/zXgDtGSCyTGEPeEJSagpjhee9hCorP3LHKaekmtzENJnDKr+EpuLuojy/JlqXPBOIiY3d8GhFNZx7R0uE+2rYwuPn4slX0/jVXVy0eNnVWiVg3B/L+R8wo/Ex1K/oO7AV9RZBJvo3jatb++Mk0BEt5+qq31rzR4Am9AJzo/bn5tncHCe+L6UadlWfkrJDE5gkYsRcn+2QgJCX5NJij/JRLAxDTYozNlmiQr7qJm0S83XkctWhhGgOCC+a/fa6GW+/Pz5YWvSJlE4x4n+Ov0QodXvTrzDkEU+0AaCjegigXe+WTMK6kd9iMnZ09/0v46qgJBhMFNCu2YBS0hBqN5LccBsFrsblYsEgfAM1d2DL78svp6QLkQDHsr9BmkWbHtsvPrBDtNyRPaAG8k49j8SZjigFbFtaFdLivqWDUoOx5FOi6vc2I4dHvBy6P0zRYI5L9E++LKOfJLn7PCcfOPy13OgS28nbXJ6JRvSyaMJ125eeab43lvzmutKo6X/r/xh6KXgbPn/zGvCQoPZrl+vDUJHS2bX2whR56gF/+8V8N6HTkU4CCz5ScJgmEoKvoeEwdCRnF4hUD8jr4Ah9QjrzcTJ0i/mts//1CwQmCTbK/U03KG9DomS3TXg3gUUClkfZngNGayN6zHn1ekhMnlWOqdSpm/C6jYvvjbtsmYKB5X/kf9Q17lPN5D7gWAtbgcqQH9XRH9h7duDh+Cz350oQFwhffYamGnFx1dtY4KGF2taOtxsnJX0cvjLvoSlrpCdMUDy5yuYQH+MBFr+K4FZiF4EVxN843TrRh93hHWMF7K52+UD+/Yov28xbJ1zzTOdenO7hyWNVbTLdiGMI5AKBDqQ/GZmggaeElh6tj3CpK7XXvclro8WnB+CEVpst7BVqbsHkexyo09KG/CBGMV9XJFE0BE6haZLBF+qkg4AY/1c1QNhegrRhnJuQZBt0nBbkeYwnah392lM1mHIrvkFU0d/LM13TchoU5tdGt54dTFmHZxPNR2g5dFqZEUG/7F8XJmDeO9wgARwwoEpRh40DU1q4WDT11+SnpNrrKu+P+ZhaJixVsBk9brwQqdGl2jvGMY1YlVzs3PxcTaYk58DYWNBM8ZVrKG8k1wHdn/4ztD0dJbvvoNjK21kp2i8pRNm94uW0Ex7fxn7tekT7HVNcCK86Ynh0/brKoHQYOvsrI3sZ6Wxp5KEciVu81ogqSa58Du40wr+135lUadHnwwApeE4Aag3YRTMDWn4SeDQ/NU6QXE8i94BcGLmxSbOSuJa6sQndwa7tOIFc6Jcvket2ukuHxYy0FdL2AAtyrgn5KfsNfBX/t/zs6xxl6f8e6GBOxjZfSnEBjjFMLz+Lbi+D1r52cYW6TCi2H09f/DAfyA0gje5vc+KadkgRBgBURPMz+RTT6F9C5ZZe8hYY3jiL9cY1PJIPmScjK5cQbkYt1r0+3o2zFVgXGDEoqtg6Wr2ewk+XCkHj21DVx1WZeG3sz/HxbFOWZlWZp5i2G5TokOPPK2LwBFG9kHTXVNhByw39eexi8xFUt6as8t165YlloMzD5fh4Zl8K/AN8FcGhh9LxlTgG/rC51LHPZkoOa+iifub3eLv48Jd5kfWww2haUCl3YZVUCM8SH2iT2KR9XRse7MHis05bUi+IM4Rd0y/gZ1gIHbDtoR4j9QRdzfpDVa4Ng5tD70qdZglV7+DpK61+4ud6li+/ZGd51aYs052m2jw1KuT6D19+hVyMdVzVfl1/DF8/6Io58SF57y/tWHPybyKR/XIu5+vFnWBhiSxYEZa0HcO2E7tbJ1MaQLKH8zbpRgHJ2MYhm57/fVhhN+LDdkJniwwb9kAMz7UvXFYN7+/XzzjCaIcl7RubuJYXpQnoULz/GP56PQ5ixNsjEt/vkUWFS4opqAssFgCq2PtLixsotljz82/WTjHTPmYQExOTBX0rq1S3Xfvbr52oZEoNyd/xBFwSfR1vtdCWzOzm1vEFtn8WqF+jB9bqLmdTVqapxiygkpO1+CWelYSY7DtJ4zPq3oNSlOmgfXP08sr8Rnwy2iy619rs9uTlswgVWuvB3T+Z9vKmuO8bGjFkPgVFxxZJ8vihhNekPUwuVOa/ubx9/1xw7BoWPLC19kIs49quL4pH1AQL50vFh3mn/eSHrvfXITc101vDoaGg75PjimYo+Gn7XwImpDkG9nVcUXpKKb5pB/xV68BTtRVZzK4ex/fmKt6Inpz1G163WSBguOKkaRraRCbDMyIzozmjeo+zMcwTNY58N0/Qe/tik5+TSsRJ4IWLziIqy8tmqDNN3hBXcilxQJHoW1+ua77qvAtEGzofmZlH/dB3oYd7mEMiV2LMC/fFycatkOnTjcLYJbmXPuWBoJk3wUMvyFzsP1+fVCzd9L45JxCz5io8ZnlsO1F1p7TkYQCboL5ZhrybZj56QJjAoLHX9P4NHHsBDb9xJeOriKggho0JTt1VKUkvZUrG4HnrjEGhPauqn4cg2why4FOzX0oE3JU7sSirMTz8Am0pbl18Y1E4is8e+hk+gIlyYB+Zk9BccwRq2XpCKcc/ErwYXiAepGey4QoXRfiqOhj4JCI3dyiIqACvK8DNqEEtC/ImLHXlAzK0Zfg5XlxssMOARxynOraCaca4/JlK/HzWFlCiD+EKL2v+DbladA2X9K6/8ehf4dD2Qv8QqCy4e4C6Cymvvy/ZPf7S5O3BjkLEfk+C64D0M+6FyJr6oXIczf8pFIzhbYEqaTZxSpemVVfIlriQFthlgIkatugjiwxN6xjTfwP1a1I4JhfWFyedhZRMu0dxxS9ls3bM5TkehPVKo4n4m6qPNAoKLMPPPejQy2kBElzghvMhMeN8jrY+hx41qCxdzZBZZMiB7cvsl9OKQuh+tFTqmv5lS1lnC8pNrREnofJXGpWYrrI/bFf2BhcQ/IVZjMyynlVHO5X2/pXBHGgbIGd45sv+JP0kEaUKkJaY71OHa/xPRRsps2/9yKGU+nOFDKGUr43oZZ4Z3ovEIbRHieKTl2xFU/6GAx/Dy++11Ew6b/Ofur6sq//Q6FPookuud2B8od4pRGcD8pp7fcqzPRu/u0uXKX7H+CMEcYRz6Mvaa/eJnRFGCvwRkfoCuQ7c/rzu6NRg+CsxZbGC85yLpIcUL38Je0OA6yCi2Eb9HKGChVUFtyK8PY+bxo+qosLUycdMFcOCil1ORIVeEceWcewP91DTo1Kk1lov4tGMFh2k4CRwY+2eJQ0HTTGb4qtHpfAEN1g7QkI/fV3kXaGKGVbLyTzxTr+GNpQ+K85EW7qdAFM8wHAqS7s7JlKarrJECWa9xbjX2In+ZBT1Vy7p9LTBooNBwhqU98Q53o33Cf9G6pEqE74vZrgH5X4GW4CEmsYcIy0UfJ8AynRfzqvkBef2i5/Uru8Ks7kU7iPCroLhWUvuhxDVtrOr+YUhxciAzBwUbX9ddxmI94BgHu1t8lTUMXhu3UJlZXugDrv1MW6a+oaFwbTxuU2ng0OC3qCe/6Uh1JIWwTvwBAkOW7RdVeTMDmD7HgrVbm62728H4BftumOlGHyzs/fUtFrl0OfKkBfzRX+rkBpfkkZVRrfDNCfm0IrKutupP8JJ2tgJN5MkakZTQbcwi/rj7rP4XCFKiTkFP8ofnwIKLpco5GPodGqI/Qk6kwzw8Nonf0ZveVwyYNUgP6Vo8KeoqCas0i/AXEZYytNn0Mmq9RuzraOT25uRi52jeuQv32MfQBG19BvojXpExBLfGXjmkpnVqARxOCRgwISSEqKzc75Nkb0St0+2PFJhyU1O+VTlpiwIAd9O1r92mg5qFU8uxCnR4mnH7M6xF2drGQUEuWwfpVUZdMVmTw9f3FDafhpdPgb6k3TNcFvR5SQ1Ntpe0fvK7mxifDn6Mi0oY1xlfYx2kASJC2pQ/K37/+6Z1OK4L7fQAJoDPXEbMOJACtshzwV6//TUl+Pkv19n/cQ2BkpJ7Hmt/mhM2mKghDu1r68aHFymPbuKhAS12AiDTinXEskOv+Ce2kvLWGtZFc9oDhJQ+CcO/f07Uy2aNL7UVz51dXkFmw1LS3K/nwHzer1F34j1F9pOFm9+wOdwQ7dCw+ZXajgPZe7lQv0QHsai5N6ppETQLQk3OtX9XJeoGNymmK/bv3bPPliKHJUQJHNgplvx12g28vBrkne/KTaCMOXPwS8sLtaJYnaITfYDZWx9Bm9p+DLJKJ4KImgpJhMfo3xBk9Rs6UWdg3eS0QC4kI06yJHMEjJywzUaH2BKKaT4MsZQ/DKxT0/8lbj9ErSSnP3UUXtO0ziACxlG/M9q1opu8P1Up9XctLYY6MT4+u2fxVipUhuRvuNSGYtQFYTf4d68v6vtgyPTudnlz9L35ASKH7RDWqQm8ZwKiHqkUkgtk1ITvG98ir7w272d8yQR1/Ba/11JK0seyVyJksBSNSTvK+V3MQstFEqBD6DiKHfgT8XTwigjKlHGQtuWBI2iEO0KhNzykfrMO+AiIFOAbVIQZWVx1UCDQqo0i5Gm9r88axuncCyMLYoNhLf0YWT134iSshggnQtexyhIyDWhlzNWONDJEkyh8erHWTzPcd0fRzXiRzfCL2z7S++cDeYVJdRHqFBaUU5Ds8gMN9kuc8u3Tx5CaIWumYrp7/X7jEfuMlTJ04eFwJgv8olU4dXKw3WC9QxfufgDnB0xZzycmns7wvheVGcJIR8nzJp0jEcqZtsl0ZQqb1KZ/2NCvIn4PAg+djqj0ESHJ5WEOUmT+h4iJLEUU3FuI8ochS5dAsQS8Ny1NwQD1O0YJokXKcm9fgcQafCLn2gjh/RTx1/srAt38CiO6BTagb2yUbV5n0r7yOQHR6KwP1yZ/IZWCm6Z9mhhiPWqMgUabukRlydOLM0aNKfs4pKe21dVcOHbk8cIwh6vA1P62ks3kIBg/1r3gHYF15T4vXK6BYHN9Z5bzxSRfkAASHosfb1THo36dY6Jslz4uWat+9cVITnf9+9GD8DLSOMIgXnX3++2xRS6vOT//d2OsIfA1uJmOKQBLr4uASMJSW0QQ+tTlqfcsfs19F7AYW+k4m/w7t0/z5M3Ia/jrjDNRT1JCXeoZ9p9d9uuU5Xq/jZFCvM8MxlKECg01MTPMTJgsIpUubZfa3vjma4Mc2LDMk3z2UqvR0/8SAf3aPcKqqX60eT7lyioWcKdb7wg+ht7ZnSDlZSyW0UbqAnpSCPK/6AobAp1cNoAxf3VaEUA//U8vejxRT6i/tffQ5TCLOmCy1UcA4QUyglLu5xx/wpBPD/2RlymhL7qOqyfei4mDdHNDq6xs2RvPBc7kQm077dVF2/72RrWtZrfFrwHuJuOKJE6PU8LUkIvE2IjfwQJ+h8itYkD7hXXfwnEeD2uusZ/P45AWMsbOQqc9RKLkvfYIk6doHOM0EA147xCTBH0QHsCBc0jiX/rqXNJAcMoBuYn2sEZ2uUNxDlY547jUme+krp9u+nX2ZOkhpWia9kQtbWTxQVJdIdLv6wYXapz/CG6iKpSx3poGREhsHi2YvyPbtHn30oVTEJdJH4CMhkHlyPwuDdQdb9rcXqMkGD7wovPoggpBDy+PCZdtQ04XSgFhWKZtYNE3fmFFpF6uZlgu0MGdnuxUNu37jX2CLMu9XAHuFmHgv6k5BB2uuobiDIofjd3j2hFIRQceL73Bz0boqo9+p4avc9D1DR91KgYDMg+CKpulUX8tH2T6L/X1ASxjN3R1xztW91kvdT6bL3PSR2UNgNjur83Di6MlSXYoyCyirysFL5UitymGtNEhw39hS9QIz8jCs16SrK++BTgX6yhQn094JQ5gr/S+AoqRbEAMCI05y3vUqWdJF5yn7RBwe+ClziTH6SyE+6L/BV9PwaGHxZ+9BYMoqdub1TgNykMFAafQle/biYxf6CCRmZGWxf2u3aGlOohRbwWu6ZKFpbCcS8niNOuMvGp5i5BzvI+9tmPsXQ5Q0zZlunet3N0/Xf9lL/2DnrqIqdoGucPB4m38aJ87M9S0D1CByndlXVp/zV422rs/+0Z429GXV4IP9ydkT26z64BsKsuCzR09GNjhRwm8xcVpEKdz4dxAAZA2v3+VwhwcuhWf6oPYbBjWadWKdUbKnz29ZTpIg1SpRul1F70/m6h58Zl3gw/cKGKAUIM4Q2pE/pG2AQMPBZJqGKD+aNeuygG9efMOwZ1R876GbHNNV4Cnjt3B3GOyDEvp0fSd3vBMPX+7h1Oq1nRStIkR9rKWJH3u7V2SLFUwuGWFiOMHHTawfYXmh9NnOOkQszPHKkqudsV6qzo756v4I239muH9sp2B/hpojZejTXTe1w0ROpJWd2Z61phE1Jmv9wlTIOq4hh1Dj9+olF28n9wTnM8VX2Kn9LMk0IDlr44xPbziEcYob7QfvpcscNKfllM3pc5+sg2cfvtRlWIvzyU1SVCUsz4frFKzN+40qV1sKyTRyxj5h8CNrv4bvBu9A0Dj4xiSx2JDl1PHTLsHIOOukKJHpEWNG9P33KZH4KTdaDwe0KbsDnGtR2XLD2zH31Iz8BgK/yL+5qmhXXiux01L/45Fw3dq46TvaExsNK/Inv5WMrWB+4py7XiUneBEQ0FxhahNNBME7f1Yz95q6vOYBzNfc19M0UL7AhHEKOS+bvKZfEioFMR1xfz52WWbTJD8NpT7+ivnKVndy+Eudm/WiD/7lG7n1JrROQAIZdtFL4EOIJKfrXxox0niuEWEtk1S9Nd0ktMDbvvT/1J49Xv/SW2jG2r6+4UoDSqPKRkkfuTBSpoMhz3/isI1dC3mXoicNQbJeljq7e0B4QZV7L7NxS9Jg1W+ohYHu+joJ3splPqgyZEqk3dKFCfiPlx5LCEAwye35B4feL3zM//Nptn7Bqp8TA3mfhhs9glZWbmdj4XKZ1z7NoQaQknH3YR7ZLcVjR4XnsWpTBwRYgPZgGgiu6utUm/QXvyX6td9Rs1sU2VNFgNsjHdy3DuTyp2YHYfiNm1r6oDTG+Zey1PCQVzl1RvnJBuorlN5jkTPq65l3zqg6cm8VlTNWaiZsbcn8NgFl8/tURV20U8La7kOjDd4HZQeVtL7LxpdFJnpI6LnZKzS6IGsuqwryznB1En3X3nP1oAicgT+psJsspgxiCL5Lc7n1qJ36IzSOqlG4MZlq0VwG0W+MZCkqdAX5f+GZGMDmBnQOH3kbNnldebNJZQ4tAvQUMq9ATGlxyco7a+/U/tKGectoz64sLqdJG5JHeIjbVapP1zBhA2XwkqDfad5hncj5fS0d7FiTnjY6fyt26R7OxKEvBHB/DAPa+d9Awuyv4qGzraDaw+jG+S2lUF7wV19haQPqQfcFXiLWB4YoZn/frFgt6TIAJ6gZrW3wbV4UjRdJzQTMAKeDQIjCOF759IOdcWzsvoyIX/C5sW1xR29Gr7sDgSfoqT3FRR+WicSwiiuorh/lH3GrTPaC0PjhbZ3kBaIDR5QAsznl/Gu3lYxT3XOTkKHkDEU0zZdGjbfZlUBzFxmi3oUkooiEeEQOWU83OhTbCisjs+kdO8/7qGNC6xMvaOxXYzLWezr67H2pCRz3EH4b4d5ziarTIgYhZqRoaKFUOrt6/f6qUK0iSP5Xh0znuenweCbhZUyqXRS6JKoFTBZw3xacV7vpjJMWct3TjFJE4HLgRdvqR9+7sJA0gHEIhk+bX0wo3OmmTq38B+5UZFqXFPn59Aim5hBjA2h2qiM9TtI3x+a/tZisbEcN2Lsfzct5ow2vcdtYKAYFAxOXMeIs/QGavzC19fB5n1tqRlpEFr1T0EsK89gOFu7AcnSaPng65m3v4fsPSX2sD70BnOnUPaIh1OnlVYb9pngXODryApTDAR/eOF1VKzBY/ag5v7SufH8rKQtNIEAAXlbVvtP0uK8WJrwJKbEkBFFdzAlhARG8vhkS8JwdfI5JaT6GMEZoEidfTL/9MCOL58dMKD1gdPsdqdN6jS5g4pP/q2s04auZCVLSvEBYVDMBLIHLqh4jAvV7kYxnoowR/nhQY8shw5SdDm9NaR+fnBzXJ+aOv90W14AHflSMOvC0e9/MKo+nnD24ccuK39xVoPxsWzu35fup5yK03kBkFqlIb/oaue+VQNQZ3bZ74F3/UpxxDhOZes9P4tjV/yxLR/75dvCOSP5BkYqjGGP/nzshQYP7ZGxuutdK7fDEV8qSTVpF93X8Its79vsdQHM0mNbZdBsRwbbN8GsTmTmQibILF+zCxugh3n/OthxI6VkzOY+hRkN2QfLMvW/b4Iww3/NWGvuZCBInSs71UMn/uvWTUXtcO+9/7iY16O+oeI8sS0YPtmnAZpB4dMJi362apsR1tgQfJIIMnfXODOZZIW8gpeYSPHly0jggBrMxoQCm6bhwEUEEaiMPAWW9xsgYLdQuMGMq0GELqFlTU5O+oBvFwaf2i3xtTPWmxpxU39tq2jOhtJaHYNXxQuWfEvqvpohLd0tkyYlXdq2sbS6r/W5TaZP9/NgeNKm1QcSAFWJvxtylbhZI30aIkFejnw+ThFvQftTcBTs4bI3MyfYi1sjJXQTMUrXXwAHYrhu9LF9eozmphiUdcblV4yhI/+ylEgOrP+/E0OPyxvqqzJ3wPksYKpN8N8UL6dK57CsljIDNJhyQqkmR2u77o2CBQUfDxskmnjtQzVOqK1fr56HyxJwN/FADtJdEJ1n0FgrS9bqNTMvhYsE9e5L9I4NB0rg8JpTvdIPo9HHiUJpSVoKzDT0CerO3DuQcS63MilwsNMJLTFj0qMAbv61Kdis/yQkPMLEOdFyfBQbIGb331Xfent0yYsSMytz/DXSV7ZMcm1UC8KItAmdhlR/YPgCTu78QIbwty1XlLRRacNQEYIrqjewsy6hdjWVQLKylVbE8chegtadC5bBe4Hfim4v1VuMKeU+ZqEjgyJOebjOhtMbtmySgsjFzmmzGC2RbqqkyvRYBRX1s4iFlh/SgGbY4dceJsbDs1RoCFi/o3VkY342Dl+LlnvDXqYQbi6+sT8+6rD5tdzcpUU0xj/KaurrJeo4jxZV4y82y8Il+G+tS8uV6ZkjF8jlRiBIxrZuROeAEJIJDqwQdut3SGCMD+DurBC53p/c050Eg1x55desmPQ2ClpfCAwf1PyQSJfPFumtQqV6L7gg0D+W14cmHlgMO+D1cExp3XjpnU0aSOg+0F3M/jNsXFE4g2WsdGGLcV+cqza166rObLYu2R5TWAKldAv1Q2HlFlS0d2x22qcqeLhf9NjWhxtgH2o2cnshtzbYOQOq8y6uRgz/HyAgUz8yOXFtK2kKngvtURHDBS8eV9h9hkZRdZpZvKmh/tAJn4HsPzPnGClzJSP8ZpUhFyghxF8rpyvkqeeMr7JK4guFZu0SyqEFzQdQ1RIf1tXGVpqZKL2z63a1qrvot9xBb/DHMzuwiPoDck3+77YHPiSh8JlfLYhjazA+SI9w4t5/rih99LNz1FWTrVL6LF4z9+qpLqG683GSNDAWaUgtcEasnK6XztpqUNk6tn8a4PmB/SLMUAI4FuHk8UgXMbvz8xRQ2FA/nAKDbkw5fcFFEjGznyiX7qr4WNVWbQx4lpHTJz65SKquBvgT2fFyzfKJXEqiwOItFa9UJavGNX435yQZAH9VpAqwq6XD/ZO53fmni6hDYjBn4qZGh1VhPhW2MRmtot6x3gtijsXy+TZeUmLjcuJNIISsF9fEivr4gDM9PEE+YZ2p8yulscYV3qExg73yngnS+cquzbMcKKsVcXF8rwxgOU/ANkPn/fa9KrOwqEwCKCkyQELTEaKO30bOR0xvPNJd5aM8VgJcyP5p+MXlFUEX3JkxL+kFU169wOGrrD4TpkHll5/25+M3fYtI8kZ/vXJwBEwoGP/5e4zhmzxlWgNq/2og+QcH19V/BDQVpo/Er362CTNmGO6Fc8poKQsVzzsUr59S2pPMaQM13Ot/U8ghAF9kbs57GfHda+D0t84+aFIfCwDbEumfTnpCizbFGUinfPgwT2VSi+bw1pXlhH6lB/AFEpW8asP+fqw7MBQRDCHafNVWf4IdwRk1VxNlgl6X6uTU0KrFuM9bwP4h3ShtLO3opMtYtmnQVCru5LU+saHFx4KDjk97Cr8t7ca+X55TUtDISuKVjyZdXkmdw0SkCgONdHA7CCTylQhkOvSgzKaxagJhftLzlvBKKb+PEkOg1nuCOvSU2Ovq2TQcQ5evKI+sLVB5ziBbNaOzmf+NY3kx/LvC/DDohvJI3UclpXed7eua7r8LbLUtmqqhJ0TcozbGcs3oa96WWMPTq/h5KIain6L5IfXA7TNWciFggCxLg+Y00brwqcNEc07mFT2Hn+s5+L0oqm0O2LgqvSvLdcid4jRWnwDHuED88GkxfPBbq0SBb6XzUr0/XxVtCJAUeNt57nCU+QHOnyf9xqunpkv+eOBZuXnOYZHgbct0/Y7qxq3dMw3L2XhFVBrnC3n0o6s9VmAnoTHDN6UAd74wmkrKu6Xn4Ki9mbB1ZehvchXdeHcsEZwgqI4j64OxfdBPiTlpR52NC7P5jLukfOAC3b52fWQoF/iwDe+TbQO9ZEHnFw6JTT2Hv2JFxy0l3QipwZOTh8UrD0c2/n0W6JlJgRO9VGAN7qLwuI9ym6FylTV/LXnnJp8Z14vmE1kmuyXolTFm6paJaC+vUodDJvbpu3WnKdzeRJkRZMP7anhkynAvKiz76dtugUbCy2A3sa/4RV5FSC3qppIprC80hqkOyQgIrWBzaenXBxaARez9LkXKR5WsVXVxJ2nwUm5ggpRrmEXfuHP0vhgNyk3q7dug6P5kPD6vx+KW1+TJNaJpCKUpUuCJmcmJbibqD1U4rx2rNqVBCHMmJnk7iL2TjdPiwMJyap5+3ApjPFunZNkc6mhPJiuudBFKZHJEyZxD5Gl3OaGP7c8TVMe+zs76WWpm1etqlh9l8tOsfVMy9cIVAj45f7Idvr316+KQrMF7qXzPN+uB51st7S3aKGNKyxSUbBTOgdrtkxmjrdFo2/ZXYzqfYodzm+LvXtCeop6o12QKer7HLR/HIyEaS9cYYmVs0FEprQPG8DHCxxI1f/AOj976N/seKH47nX4Y6waBWpQlyOACPzN80xSZ5tt/lgsjreuHZBrsQH152JfHno2ubHouSKWDPwL7YGbUo97/nkpq6hGPnv4vjIG7qtUWW6B9y0V745i2ck/v78lQFns875IJLN0fszXiQUVD1Fa2/Nj8vuMfuFKupwp5whT2c8aa09nwD32LtUhpMrvVu9Ge0evb5zL3oiGeKDPU9NP282vVRMUAbUkRjq2buq/ril/Xtw4sFFXFY9SvQ9hDPEdfPx+RZ/lL9VoRR9ou+BfxYyYphsg1rHFhIpMKA1VHBSDWJPbB/A9Bdp8zAovfTUZg73sXPLHdv9GlYWrM8ZYIIj+TeriSz9M/SjQBXzzCF0UZVAhMZhvRPOa6PZ/lQqqdFjRRO5Ryju5y/oXGzt+lAdw5ctIH7mEXnmOZgXbunXy4h4eWWYuYoxXJwcdZUUoNJ34jR7T37SW3q/YMzp7bnBDaoMczYO9T3stIn5Vbe4iG6zDiKkrM/whX39Tj9O0H3ePXLrz4ULM48Larys7oTdwQ50r+JufO6CxPizcdz6+1k8W1liwfSrG/gab42iiFn8pZTpsvqMSvMgnMFrn8C2BRxTbmssCuaC1w78ucyOwnuSjEeG/zWbQdbp/B+ccgKaFpNUy2GesF29Jyh+6F4KtfkGgzGTP4wLQjTnY3MPVf2JiipvxxgQsHIwMOWsXFeFTj0+W4twsEeDLeITjpvWPlAc799u2n1V+zDpqcCNe8yKDFv9l6u8HXTIzcAlZmxxNJKnX5DSJdTgx4+dRtcv0U4KdmY6qMjRQOXrfS0HcGjXP9geweVjKArfJuSA7kP7lsKro6E4ZC8YfvgMxRGCrF7yawXzNUw9ye381vqzPJt/YDGkfAI97Cg+Xf90a2uw6S1GInFUDnLFLUaff79dmKAdSZxAb5oBBYueWWLXJD5BkMgjkZs7vRt09LLCxy+xIXezMjz+ROurYcXZHD3dIbSQ14Mop/m+Z/SGhqv/XvPw0UHi/ZiLkDmMHdcC6Z+b4wBNX3BpFp5nrj2ZlRVTlHV2BL8DCapaDzwqvKHToqlWpJnsdKuVxL/IhBzpgVH/g+mmQJoOdy0zpcXRLbJhjBgb9dYKQyzaPXoKHSDt53cjnr7ceG8Wo3RXezBFEGyY2mVA8NFT0Vz5Td4kvD0g38T72JxhfnLxFgwRXNyV7qp/YDE1VYSb7hbdDednNc7IbmlVagSiCnJwVznJbhQpRmk5Qyg9TkHrT3PP4voaDj2/JE3iGhf2SW6nmns7+L8Vo0w9R+Qy/0HYN7NvJoxsdbTI+jqIszaKsu3xhx08mOyZ+gpAoOs73t9JLKgdpX7cV4QJ4V7NfQdls3jZJI4RnKdwlXKSUe/sblCQ8CEq4lRteGg7FXtmAlL9zBsi6mF99l4DoUJOa67r5l9d2CJkNwTpW76NnpoqbU1qcoKTOM2ooSr0wJyj60ipdF5VcDcAxtjD/Af7Na2JM19ogwbrLS1NXKPkMyvzrOpzsco8Lkpj09QVqJl8SHr5exTz+YVThPD8beX5UAIInVBpuylSYh26lqHvNAyiobNbBKxeJZK5je+zhVvFhPV/Mfy+ks3uq15ZL3f82u1pY4dBhZZ8vq6zNSJ9M1TPW3oPmccK9wT0ZwVTc++SArFlJisESXaSMuhYa3Mf8G0fkGj/TOX8b0ijDj0nPHHGoDZsJovFX/wkgmOGSn4y33bjddtAS1+z+ls2z9GFr9zn47Ube6sjT0dbJdTJpfzfXdH7Ale0HY0F+tHuz+sEINS2FkceOXKbyH4frsdeX+BehW1b7TFpK2olZ2szneOZgSIA7VhtEIF0F1jQpwjojU7+uGyRA5fzeDWXUAZroJ5OH6Y+dHA4RSlhYJ/i9EmsJpFn1KB25DvOLKX7pzn40ZonRWeREgfG/UH59NEXGc+07HR3iW1XljvE5jlsu9i9B8ZOV7M4Tj3yxysMF8saD2CWX9alx7kfrB9LW3/IUCDLjWg2qGYTyJyLMo4CbQwo3x1D0fk+9ugCEFLOu8sB5lSBZ8tOUoZ5nhw0OK3K8s79Ocf+mnN+qt2eqmCSfXQj6f4LfDHle6izW1atIds8aj0rHiwxyFyfPvfPhUx/oSeTrYSzREy5jidLoCIqM1YldGlS3GDyX0JaBgSjZmmfr7xzzdVKs34gUj2+YDviNCgj3bMI9npMyoLCL2SGwooc92jGhRN31nBfBaHhMd40CK37AOaiAoP1doScdEy9H5r/QZ8FZK2Y/RcKhp6F0K1c07nQtt775Ak0xntzUyiOQgrcye35teeYxeyuzYjF0Xq8dxsT17hjVarcuUVK+94CqLrsZ2R0oY3aSlgqUPxy9wdH5U2Qz3KtvhkJhKIVDnpbY3tYte5sHogSK6CwYqqk2q60ajamgH2yg712rRbdQhV2deMnS4HIXf2/Qbif/koNPIUt6ccN6d+O0Mo7j/qVZqTlfKhDg+l30ObrPTp3TmWUALyPy862z4xk2TDEuaDH86dCPi1Dtj4Ey4ECazN1T7XMzNZoPr+t0SC4dijSfSo41Sq7r2AVzgKz8cW1hiToGgVaYZDOU/0V8YBPZhLBJiv9UnZpp1nyUtqe5sULJI65eF2hPE5G67sN+sJK7oSTEKTHzy5HQMc9apruytCUn7Fungt2xXnzB/746+SEuDf6dPqOXOhBb+OffGG7hwf6my0Uk9vs2xbTklXMXuBYkHs/FitGOASXEXy0oHFfR/yBXlfQISwxE3s/25RhuIdtghnbp1ypekuCL2cZFVcYyyVqHhFkIVfZ1a7WWowaU5vuBfLNMloocCg9WET4Crvwvc9eV5agSZFcz/3jziTfCe/jDCuGNsKsfUtVv1jB9+nRLVQglmZER90aGKVGOFfLC3TvnUpi+fZBL/aqjLM8JhXiNAr5De79+y0l363wgg3tTWAhpr6aAzl9r27/TZa9KIKljc/ilCLDHqPYozecamwCEBzmzHGj7JZS6FJt1+wz9r6EwVDNJr6B4r28L9SDPM+c59e92JfBTsM0r5EGFb/G20Zc4658jfhDN+ioUrd31b8w9E78qtPy2fhXplKjdE3HIndYXu8Tb4T33GiZpRMwLCKZjmVwb2o5zclgUFRj75UlyQKcg+sOaoOB1vaPDcfyOJ1Vj7hOI04OoKJdK+QS3k1E04o7vmCZXCEkBoutDDng997GN6QzuXJfRNv5UjDw8Y2pqkOrzEi/Z8PFKbNszU98b8OIqW/IDT3grR1x4mXj2mrWHyfozYxd5KyZX7FG+CcZVb35bJ+8FVFuXvCT2aDIzlUE3LENMOC86XHBGBcK0cCQeXz7rW132BmrkFwUP3D1Jyg5XPDEpJ5C/ejK2N8/jC6UJjpK2sMwj6s9ALqOJWJ+Bmb/UgIM5fyE7XbvvX9EcmtufDzHhESge3sEM1ktpf+ed743QuoQS3b3x0ElTEKIlGoNWlBZJtG15nscptLQlnWqiHZQnOWWFLfCDAyYdB5HIyLlRLC/O3isL18pk4bgx6off9nZEAKduKsie3WxRx8VoMjdz/Hlz7eEN6yLL/nVH+jBKJfEhbls8AgFPf2X5Z3awGR6GksIJHvTc8Bmu3WhVYL2IxA4mbmhr8cYdacr3dpoeXhJEl0hdkJKK2IdJ13CdZSqfcdPR7CCuakOM/gLymVi/5udTlHUlatqCITT2r6JDBV8OcGwWfZTqW1FJy4qXWMQlSN+LQq3U+66ycq+F64u7Lq+C3nygxpsRvhqdg4FXVgHyXopXANIFwVa3tPn2+84YsDdB/jqAnJ/XIM7+mz6O+oQlqZrzAoK8dt2/BezOTknec84NvXO1XD2WDBqYlMggeiIV6/eX3ZqM3iI41+i/W5FQGe1DMh/osK+CnqWhNmU1dd3Lo4cj1tJ3Yn6yOVAe6sOFa+J2Ul91w8qLPrcxuRjVnX3lESjRy7r0ShFwMWcMrOPpondrbknW9I3Q5OyiYolfqLCPS7Ds52fS1FdPWUjc1HdHiIylpjqZvV/w9jpVPfGmT8zTwsAtVzqXeNMoeHk81J3DhVfjEi03L6haisJ+EK08a9zsvFZb9L9ppj06cNKEVbFFE/u2Y0Ox5rXzOMn3bZcq4/tzdnaFKRlw9N+z8jaDBTO4637E7obGXrlg7hxIO0PFfAekSJQLrpv3yrhc1JdTlmmrOlKz10Q4DvxQd9Nrsv31pV9Ou1j2wWwN8CO6H4sRI16Aaw22tpBsMY/UsgioQcadWxxU30Zzdw8YWy7tmWCmYWszSLYicXFLrWzU5lirF9Y5m3L0Lh+Hyj0Dp3j+1bsAMMzdHq7ID8d3Vr9x+li3AgaRjZjzhnFVZ0C03m5Y9XRdF/uQ8bdaw4FOwO2x0KLSFDoce1dRUsi64SB2dlhA8rkoyAFr8Fh/RL9I/y/kAr3AbhXx8TCZMDWGHZOvi6RNAHli2kdUmtoPyf25gHwtWSn3J6VFnelM/oFX5ldCPvQZo5r7Dw+jBPR+aPK0zsFo9fvHSiIVUjO7okTifYJRb+VXTA/uLpTVh8v1nRZfiy40VxW874arhsXozc68yzYJ+OgA1BTzBXYoWc0r2btSVhHpVP39ih8YCNzCxq1hSyzCRWZw3Mv3i917t9lOVCwwMkdCb3Uqffh1W/gb8pqP3RFa2r9VqRBFz9prmxhQAWb4BDVZZz2GA4s9iSXYIiCiI8v1jqVDo1R5ZbsnKXiN7tsiXiI7YQKETKz0IlWpfXAED3yjwjlqwycUGoi9dIv8KPwdYiJt34mYlX1ukf0LKOGea8X7ON52WoriGbn6VzMi07Q+L2d/ZE9RvpSSsWq2FxtKUeRmts++naCWCACVXeE0yWOWA8n1c4RZL03k2MfWEgFwESyNQSwt0I8JY/vDDkSS68D+5arQJqIroi9MedczIULnzpcr8G5h42qRhS/fyV57xCavmSrWPh8EIfMxNTYa/4yHDSvEo1JN/VO2FF9b4XeEI27auCXTI+vgs3ISiDkuP7q6OuV5jUeiYuSNepX87v3gmVv5ZxrHVhbR5Z64yBuEMH2dhScAVlGWyW0bUVFt90f66uuvNWwmhPOxZEB0zLxWT9GzXUlIoXjd7+D9MWN6+siVQHYYvJ4yJmZqbKy6OWrauaHFGXSucQmU6FNapHUCOq+QYwotFO14vqf7uEeUkSmaGsPv5sCtpN/smKzl89P1BvYyGLlHIGoU+Ujk3j4y1SePTZcLRivplV365bbGIzR7hAWC+3+9GljWYkAqOZLjnP2OVs/yDH04lE2jtxeqx6+q8zr115ypSyds9i116xM6MPCo8lro5Dmt7SiW1pFyO1Wv4kCU8Vd23mHJL+O0DYcKcNC7jafAG7jMWfeZS51aoU17piKmqjOoS9WgM9VDeX9rHznC4gZP2hC7sbFd+ACwDnmQ+4BT3yItKXXagBiIMOG/cJtaWeU0HUCpXRm13jIB0F+1b9gaaUOvdotFJKH5djhDq/axuaxgFwekCpmI7B+tH2oZACWBmFXdgwXbUSfivijIBn0/K9BJ2SLdy+dhXur0ILHdDXRvHrgVVbF6RWjKg6a5aceEgRP19r9OeA5xD3rxifz23MpvfLYTSdT3w/kDQdwqIVEdT1VEd4acd9FjKTeSfZOf+t0axE1tPPwrc4e6GCWwI/5hoIJpsL6P1jV+UTedf0kE0n+5GV8hg+UuV+6MDZ1wCMJfqRaAbbkQu0/0U/FqXxueacsmpPrTa6IwX3dAqjIrSg/z8tn4l5Xq4sIPMKyKC/EWXxLM9WutYchckJVcAi/5aLRnTObNIg3qyETpDH0ZnqSYYvi1GKQWyjrtqas82+uwxyioL9vJAcdV6zkuaIZMPv5y+K/P9EBjMu43SQr5TJ17nawSfqidgrVkJ+S3iBbGdmYcRqQHxfP82Q0lwF8vrJzTGcWqMnn2E+IXx7MlvMsPJebz1pCCDBjSpmWH+dhwbNudH7EY2TWOkT4zJGww3pJ01cmVD4mPek1QnPcbzhcKqUrSmvZliCuAo+KQA/PkWX63KzNFZ1524YdvElVepSi8p5Q8dI9KH3aR7ue0DxPqhQiWux3ILpRgIgjT0LZ6xce6jMnpteBQ90rpy4sW4IymSMLnHWiMZrGQ0JQeV8Vs56DWukMGCvLSw4g0FZ/Iv1f5wMJQUt1HsePqqZpfCM5Kxgh+mSu9kgJTuJSOluSZXdipO83o5zKry2mBZf7+dCC7iPxJWS1Qtd1BQr4e5I26ZpeKiHOrr3GdHl6Cx9CVYQLX16Ny7AU7rrq2MdaRl0P74VrHHx0qvlxhvHHvCMDXH1M53C8E7JthtTLSy1yLDK2N9sYexbbeLcWqpFXA/SC/X1a0l+JctOA47JX3Q98n9mfmgbbfmaZQ6MKR6LPf/LmvpUB7C0d8fqddKV4qQ0F0CeR6lKoe+UroalB8pFQUkrmKkgLSWpNm5sABBUrtbgUOPIOAKzQfjaLd+CLvjLhICHge8Z9FclI1kZZ5B163Xe2XBZdjOcG6WjOd7i2JAlfLt9g2DBtsI/ErOyU8NDm8HKAwZSwGSIzugDnf85mXRsu2fvloaivMqToP0hK+w/wcRPdh2esPs8Dq6+fAMITrvulXMjZ1XG8riBDPng3WOV33vkvxoy7lyc6eo3CRMWxlDVcCxcqwScackVFli31vqIWuVWqc8VMIlIRPylsJnDK0GIuXOuEdvhEIVqEN8gMhJvTNeQUtg1F63yOOLfgf9wo3GYd0FlG9BPvK2+nb0qTJ4rLKKWeWD31PHjRYINpIjPh3OVTv2b8f5h6xygnZJduiDYO/9/leUN7n6LbsfReYPu8d++uXL4VGWYwQ6prb7Bey/9pkv6TLtfFp2D6obV+lt3KV7PJ82Z7H5Gv8ysmXdP5CvVBah5VfRAgcicc2O0VfBn+y/F1rZlqQU54TemG5dJDxQaNZ/GuSkuOz/EMbtDu8lJcyE1pAcSRBX/HKzIw/siaMnO0vyxEdfEeSFnsdFUY7HcwfVNt4A0eGwEI6WFOk5bWHvb8fS4Xk0kCzcNXYL4uYGvVtxY8Gs6Dp3ex+bqVt4v0c2q+M4ONfCv559a8gqT/AidKG1kBx8LPVZUByywRa3W+UceWnGeuV2n79XgWUpiUzNgtnzk9+yW4VTYhc4EG3NFaivXDmagFWSjHA0QgZmEwHtkUc1/HzznFAyKshReCv34aft3KYkwwVqpeTPYtf0sSKLP89D0ZstArTmc4P7K9LEwILswto/iNCe0TWyKYRlUIrqtgM5zRGsL/Cmv18e8YXv0Y0cWOWm4+OnHG+14IZYanafk0RmN4ayhsY3BQrZtloa+LmpOYuQCzToCn43EZzQN/tXkh0QL+4eCYzYXB65oECZvKg5s4jD/RTfIi+SJS+88WMigZgofSLE8fhA9/Tc8W9M0jo5e0zzyxtAnR1WxCL++JIrorlvXImsBo0Sr2trtdkSRXOs17MFSUxg2dUdX0jo4AiAevrjWH2XzOBkYOn8hbVnS827sSv5QAkrL9Bn+rXmynVTaBS4gVRNYsrZ31js4lm0L2d320zZSliVIzWO8GNlLVHuO7tF25o6FPWZ0Wu64ZBjwImyvG4LJJx3Me4IOZLUs/jAH2w2UYcq42MPm36arOufV+H8RhQ4yE+/fM132wBHZU3/vkaEYMoiYJjFdZGRRqIDvKlrS3ezvfXh1IidbITPNe6yj5+yFZasnubYzu/8f+Nw8AM0ajJXQJ66yPNXnQEiLfi5q+fPIb5aaoUoQgBDxBfKW/jDpXAjOT9nCkZR22GxOIsWxqHfXO7XJltih467x0mcaJbRb/ZnQVjESXpUCjjp4RBvUg2rT+uG81/59FsC8cQY06HhP46gXNNZfPlxoqugSOCtL9dprqAvyFxzH+NnX+jg+y0WekDFR776wmmizCPnvNOEKbSdLpTUiZmwAeYVeQdvTGn2KR+abG3RDVVQ28ICzeWtZJDSoNOvs+s3uzGd9KtJOYvR440Hv6Zj22IrAUtb0ibaaiKtG/sGR3/6x6t087SV5re5lMowuNRfkreUaY1qoOfOxBn/rvSMhgWuD4p5DMDZe+aTdF6rFIGrjxGF9MWh7aDMWxfPVeQPq2Ve9HP+SP+6msSxTcmy71d/gDg2/WBwoLCHPrJDM83BbyuEc//923O/OqP9lY3XekN+jyOf9fxXtEDzXD+wk4sta1zoqYxBvxx/cB0XjgXK8r/oPz/oGyWriXxTAX3CVjTOaCX9B7BhYbr14L/fl5d4C2fckz8/M+WrZu+wE+kyBFD2fEyJIGKB6AlNssmEv1J3IcMhuKQBGoXhw6e511nPde/vrjqCKJfGg8g8OE7EMsSIADJ6/PFZ46PoFR3LPpkLpZh+snir9iRfiY8c0nLnkTZa/n1HfdFMaEcCJntZj3Icv84JRMqN+zoZ4y/knzY66XlY58OkC6E3JjimMzRNGGr8RuQhN5NLMVuPwmOn+S9oJUHOeG7Q80a/e7D8LPt9zGxNvP/6A97zDpTg1c2B5wbdJ7kAQikUuNfCSzn1xqzWitqT9EbRcMUjun+VxKkezgDsEbanlRoTW9676i/BWbeESQ1sqvOsi01+C40sBU2D9ZDb5VS1bf9m4GEq49AQBlTusr5AHb4Fod74161w5Qu25cmwyjPXxZ71ifmnLN526mJ4wKjf0zWUUpHyrGR+d2Mr+sYB0IZOf43GnGn0SA4GftGPD4Ny/4G1Urk2GrsGWkBYbbJRalRFpcuxX/MO2IEQQAXae+HLCjQuUdNrUx3qfpfvoIwBZMEtzz+rjGea2oFUVsUO1jdteYkmOGyPUrhDYSaeXEoJgr3kVoSFL03zlUHjlP+Pssy1P+9ZuzYei0EkZ6z5x6yHT9PAm7AVax9pRHoCyrKbkxIhaeCOILgGQ6xer/JMxhTvgogUmxTp3zjQQrtlQWuPUvJ/b6GrYOlFEPj+byBRnSvN836Oj74+exWjo+w0oSqBRTHYC3WbPZfA0t283/VMn6jw3kDc8FbuZMJPW8cN4BznHgjNXs8Y+B+fWJ8a9s6qq8Yt5m6mVayLs7jZyaAEFiHwONdri1Qqb2gbNl+ngEZxp5p4sAwER1k3ZEfPV2rk1QIVVl/M9BajxkIM46l9uWcjclu33QK9LJu/obGJzzAMANdGOLRFhAunwW8FbslfavmERdwb1KJLaLyYxyqBQ9NLAvynwl//r6aX2oJUboELTep6UhJnwwC9vvtJT4WSRmKTJ2cpYK5/p+sCjz5fhOXXbheLwjK71pFvpj/XjOmwCP2WlfJQYLrjzH/btRXhINKp290w7fX3JClQ12QxjuXSj7Ajzmez7Isymh9Wuwf+9CAb2l8OaVMEOPb0YwLJOqFMG1HKIiGxrmP5iwxKYr275MAJgk1PByUKp3P7MkADBXNw3v7jg6NLQzhdUlMjpN+6+n8jkjubR5apKdo6BHscur+nJivSPgMrcWdRGU4VfT9hVE9q6lQ5G3/7R+O5E5w5sb2u4YPXy2jCtU27Qf7ala6xYUGzPHisdg+U+nioTWXdCI2/HaEzoIwJ5XH5UabKt0Djl0MhTra0d4orFICu0981cL79doaCcXN8dt/UuFm1d/IH4ZCuebHdssi6P4cEfwkv9BE1R4KO2tXpvMHCNOxXNwWn6EqJxRFmG8QlgyKq/9R1mcvlYGWnje0+J/97I5fvHYPR04ApyjoV/Fx5OaVyKe3TrL90YC0MN5PJ2Z5WkEQUU1A7/3a5bWv5RFd8dvAVQibCI6n5G+x2l8dlS0Qg7zSilolgrMAz4DFubWlXxquInLnaz8YfotBKSx9OszJUMuMTB2aD0LB9lzhf+RWSMaTeOCErYPVBttTZYGoF7+82MkXuZ3kpvvgkUVUIfgk4dw47D/pV8U6gKMou160fKocxm+MKHoOGgmV4Vvoe1UD9lkagWuK8nVN8PQivXb4pRGs/PtwbW23CZb74k3mw3TUBEQImK99Cd/bXy0OaC/tQ2V8opBQSbze0eBVr1uJ4Gpp7J/iBpsdVJPcgZOFqEvpRcEhkNl17ejij3mBU79yBslkRCx+7AyNURTxaGvviiJoWMix+jdVzdjfAqJ0Tu8UWdS+D++kNpOgUTA7/PJW6fjfLX8NKjmOwekt2VFim+GhMenWWoOHzPT3CtOmIf3s15LP8D9nxre6UcV/TKADR517Yg8aoghPjM6egpehDw3k4a3kXu3IT8qLL/zXtBeR3XcI7w+nuUmEvufcCmh7uJPIv7/RhwbjDn7Hn782ClUUFr92UWyWx0SRuB5lEl/aI+rVX2up3rFXO4GEvjaPap7JTg/wnwzuuD7GI9F4v/v62Yu2LUPoOoOY1xnusNu7wzIgpNtyIndbvsavxTGY5wjpup0dqfNLvXI+DOgHNq3iN7sQchu/Dsi+PG3CqkLi4D/kVSDGkEYpWC6UcHVXpDSw381+fKGnYGX0YDfnQG8syx/pZd3E+5UJ10dFwQHD1Fr0HfWY09MAMIhtuV5y3pTD54sieUpXs55BOeRFGaSXtR1Z01wZDx+kG3zIT2KXc1eFURfkYVnlmpoTaWe7dfSVBbDw9cpmEEHYCGtK574QaaBgF7vSCvfgA9BQjeTabbMqnv/q6wnz2CcwT8oRV2WgJet3FNyMqknTfk0SOfURizwJYkr8CIBbB3Qo5phXoS3w9Bgsy062/inupFWYwQ1SuiTC3osWXosW9vC2gFxWO5hcqqhvw8G/oK4jW+MwjL4j2lQfPf+85zv8ex/LuNzoSF++G9JytJOEScbNggRWTbEOGyOu5UsWZD5mI7VcR6fsvn175fBA8I4VGKFtFCI5CWfyBM/rsSqfBx/ZrQDu6cswLoGAEFmgGZVAku+HRaNyAokZ4tQiNIw1GCIvk5NoJX/2HB5rOknXUdCP6BEacsJTNucKVoffCGG4bXehxtCaIVYi9Mi+zwMmNvYEttJ4rx8YQqIHXPBbSHb3lgRVAZeG418CP6qQrA5+vjxWxGhe/RdGp8YHMpGQ9b7Q5wrhpoT2I+X00Fpm7i/3SPu1RwRcAoaTCJeL9DNWTRIr8gmlPvhlhKSbfryUEdNiFtTd9FrPG7Vtf1ZZdpl6wA1/Vpzekcx6YXv7/aFqVLdURGPs0rqdKYYhuJDz7ynvBVy9IvSAn4Ftud4X+3nOxKdcQA7AaE1TVCWCOlvoBlPeliT9kI8d1EGskkGlCMGs/Cj66L18ZiTmTJbx7Mb5wAz2mS5zo3DtgavRsHP59LpwKZBwAa7Wmrq/v7x14md6xv4mzjnJ/wjoO1L7pcm5j4pwpWH7p6Q7FYbMnCs3gygKH5YzJAFJsdhzIi0vDHo3573evm0QZdOn7OfZon/7n4asviAPyhfZnTD5Ou78hVDZWPtEt+2c2uI8YEx73UcrpKRaBJaBWvdtdPhMMbXUMFY4UNFI1tuKKneWGdeIPvMH8L+5SV1cgEYilU6ApZFAKB9LfFMzJN7gqZhayV0+UFNfxfhVk29l0raVrxAbt7fB1E3FFGzPrjmvLbgdwTD69RoTq8IiGxAmVt35oNLuyBvAJpzkolnwoaKKiSHJLfwlsFN8rqceYpbSW2yBchCamcomzpdbJfWk2OZrJJjoAu46EkmyB6cC62tgz6TQ52i8ftpMuJ61cBXHchL/RTRo/GZi72iT1EOFTjrVT8w/EgTLhSMPtFcMAy7/62oCtP2MvNuCrMZiVVjQZUFUSNYfMq9N864ESSZAy3fYg+mXpon9U0BXZ0ScmI1zttO+BYyW8GAEaBbmvGnhJT67FP19HWUHPjy0ft+tO8W/H42Zo7ukL0fFekln8Hxi2Q6Kmg3H8ZY2nnIOj3F+5cmajqfIbP4Go1GBtgbF6c/ovyX5a/b5M4ZLfxfHXR1mHzRgvyn4YWNvOfqUDY2KqXEh+uRyBGl1X4CQMvwak79HHrFJEcpguCU8KIL09yM/OJSPKpsJPwix8Gz2q4N/1dWHEiLmSSYtgO+KWQLur5IwHfILosvFiEryrFkL93nuQKbpO/4Tf5l/fQn72aVWGPXF6jDqDFyGoXgCPukljV/j1J9dNo72TVSiI4ELxvPyVXmfWgNNbL/wIIyo56LowSOMxwVgpQvsf3fC2hjk+RCVVeyNftmKWDGJ9cvL95Tb0aL1iWX7wYpi7Pll/8zD/f2iDBG8C2JhLxN2VKmjASFeLKPkAbaLflKayIR8m2usLSLqQDfgT/1DzyxX7IJoUEAza3F6t++Xx5D7N8737PC3rYIx+EC5wqKM2+r1UO5n4SDm5m6zL89yApEERnrSxcO5d/z4GSsS299o0Bj1w+nASRg4eNR1O0l0n6pLrHgo53aEGoy0pC5WBnbCnkG+dah48QXlRVSk5ujVeQGO7nicMlzoXaZUbMeI4ItVQSXt0Kgtb6nF4GoGVRZKww09RdmGNu/svWo8HoIVvo83OvmyZQcf5nmq+KG7wwvVqmezjZYsvsM74jxPHZsYnXvkNdML+jFar8SRkwrzhZwmXwOOXvAEZvhjl+qDT0U+JYsRvPUQF+GuSLzvYjSV+2WlzzbUsAYnsy6tOmQjsOFLb+SOqQfklQboqGSYfF8J62SWyHQ33GSVBLQuQPgXB5K+i/slrI925jJuon9lvdh3MJW/4jUrxQRVdu9zer7DlmZQ1IIP/yUqTqQqA3V7S0uMnN3KXYWQ+2TkwsDnl7SPNWaLW/n9fU1+Hv20Pftbp03Tr7bTWwo8AofdIQFcjf4RudelrNWnt9wP8saq+9E7iUZGZrHDw1oYSAiCI9HNC1GvgzGKyLelxJYGr0LRqDRojLEsz8dvCH9dB6vkbHDRiAQe+u9cRiEsetgr6szmYXoHx7EW2Xq+zNqS3zIlFQmoS70MyRvvrBI7oMUOgwFg0tiWpRNErZl1jS/cDRR2wWPS1AuYxHtUK+ieFqpVl6ucqHUrLGudbbrRfC/4d0c7ALLZnW9Mqr9phaFgc2hAClELF5XlbN304NQ8d3cew/dVJnYDocQ6cZ9Ny20iL3MhcMBKIflCp5TcTkJut/k24La/JOGBvNg2JEOXwY/aqV9Sg2VHXPq7Q45CUmlu8sVFLdPocPAIWtQJ59ZDjWD1MiHCgAShHl+cSGc/SiQQ50Gkv6PshSOW1lL6ffrEPkRHYmcd74FqZ97haxKVb1x54I3FVzZbxa3v4+3kV9YJvFR0aRWKuopHZqtM3wUfSyu/G84FEbx7o/FreZfNXT1d9d2nlHBzOWW5KdlTzNXd8mtK5pJCUJ07jFiuTopsDUBSZEI3rQxAj5Izki86U9UXSwY0NLBuqd6KBxUDArmW4NSxQruTP6W/ZHvhfFjmQ4nDHjG2BFLGRgwWYSa+185g3fGmtpWpbcWrt3RYtYh84zqBPRK7rSSR8TNIAyGOo/52iVjwUIQTzf6rzaX++i8BDM+W7a/WFWPxPL8ibsnp5k2qerAUy89rkhDZK7GRFHpVsTFVsM19pEe9Y1oKiRy7aHjzao2MrYR4Iyx+PFuTcRVDiYIuuQllpgC5iYw2G+qmFba6j1qyONpvILt03+YTJwHzPbNFp+DjoFCvS5ISQkQfTdNEX5Q2LUTHDFloyNf7nhmSWWrlXivHZ6yPgZAya0xNLNJOK1Bgd12Gi8SxOFxScrFhMCyH/EKF5AW4WErQz/xPO1ykhs39mnWT5FCh7q+sROBp7iPwyLeB7F+27rZTjwQhn6j9mFfwa/m7RNEVPQj+Aay/CnMo9JrG90KXkIYT/MO9cX6pfIYwmvDCggvUXxDnao3s2o2q09B91VI7Y8YUqfzYaNUfa/pR2EmSzr3+eg97BdxWluhbwnZcsyby/tZYtf1sHTEBD7TVkARtEJZJGHt3uFaG2kybV2Xeyl8IA/gw68Vd8i7X8s7WSPfvo12no2VjBoJzEup9GoOU80Jo76tDtKCvH3MnlEVz+tc6HSo11wdz6iNaCeC8IF28Vl6RNP6xqSH2lneDF3NVLY6KVgGC/Bn32RqXqcRSmNqXKzTHQGTJq12+SVTBtXC0QCHzoN2z+IAX3aw87pmmYdHIEBteD4xU5MFayJQIKEq2+ls4J4bRiPXDmOiy5ODT+yrU4JwkMJbQXpqJBPQLpeXXYGoQXV6VYeBF2Qp/Dgbg08gGGBe7Sgqw7nM3AKxUdejuGjDfS9+kFBL5I4rtwK8o8ouR2dxpMon3bh/chxmrV47dW+fljLakIkp5BWt9JHvXQRZhZXgFz+Mz9bnI6h7VUwFs3Rbu329oJ9HZp135bIcjB4fPfDHIF0UB788PqAGtBQzGHBLpcKcPojpSvuIgel9AKHK0GDrNr9INnAjJ26yK1yo56jvGmnhhtHylhdt1dbXUtZDitvDS0qbfY7k1AywJhjIiiROlp+U7FAkoei9aSFeE6ZBLNfLMUln5d05v83czkPuRgnUiq/CUG5v/1DPUnTMIUfMV00eShuaSLqKJ3RtsnMMels33jQtlBn2lb/DAy+cV6jMWXWHoqNG0spOq6zeu0o6+iEUGuhj4Hc+bE+MrFPUBAvXLsgYAE2C9eG+3oGRJKzyH8gufRmDQzSEL7CYer1sT7Rcja/NYTn8w9fCwzyV4dDy3rJ4uK2mBWi+sxjT9KzQntmrpL4TT5aQdFFmO5K/r6oJyZaMmVUKZgFBGQPMXElq66PuxNc88AgQpr3QmFP5A+pzLhAKOXNARpMHxa9zhxvur+TZe3fbvxt3xcHrL1BCaP4sTIFLBX+uf2Fkf09zG4625cAH3UX3cWtsHVA1N9XC9VfmrNVQIAGabRREvXTt4DR/BRMK+zUM4q3avgr3fCJxbEJpun4F5VrhdpGwqnDmM1kpym8Qo9+2LOLtbkx1cUDiuUbVkEdeqgtVOHMghHEFNz2fVv6pW8tPQxJ6BqY/sEJjOCH0r2r4uE/wEuS+hx2jVCjnyHXKKn2bmtWu72FT4Ck5Vl0UfNgYOFxS+bmRl0mBfJZWE7W54bFO3l2osNNff6gBUHF0tT2fgXXJt8BcfbOaZZ6mzGZwPrM07WTaVHdSimwYqqKuaD7LEoLMtkK7C6cWINC6kkKEZjJGCG8Ac7jsysyEBNpSYk5vJE3SKzE97TuSibMC+DS/99SUBTyVwzsHPX2GUjmAElRp5V37AYZqfZbM199kxXBSThzl9zPwVonkY7Pn92VCPy7+VTXw/6Rs5l92uPlCI5KIDZ0Cf7OE+zwBX/Cp7kMszpQ5Ba0SIy5j8pSflUa0NhJhvi+McypOZtHW9CpmuA9FmSoC/zQMXVtr2QVNOmEUHlIQLh7C95OffssiskNahC5EVu/Y7DaDvZXK7h1NfNVcVNbZejH/Z1ptBNrFBDU27Ym8f1dQtFjRZtn6SbBT75YFL+yYdfCpD78GZSVIr5YmAd+rbVYAPVjvYRzrwZDOBpWqH8JBZGXiPjZpVIODC8xbhlDrFlL/q0e3Mmexil8WxJWHwQ4GqzK7pVwl5Xcx+UlpRfp2je8V8WEuckj6sQxtxESFUg3Bc0GKe5c6UYN8++icfPjodb4FNBFOEPL1PmNfUFux5bsps+wxK57UKB4t4R8lyl8AWSj/2GOL6nW0I8TwnrZMkavsWTdESGshUhwoQYT6ariPERG/dJUc/ocelkQzidr/sSLtr2hDXn3sUCObQv6F6vAL3VoBCXsljjRi++TmF2JxmSnsa8RNRjOWdx/LR4HbawFEQdvI8a1qG0ZJahdelVbWhsGkUTLpNdRQo8YCgBwubUkLDmfFgqIkv2sQP6IcP73CCLgXs1EsWzOcXLw5icWFkmbv2S8/fM8pGX+OcpeO6HJRPZL4fj7dfpG3qxm2v6nT9zoKavoWaYZotlaw1Tv2un3t3wKF37iN/D0etwWgNhjphaBACMABWvOm7+V2UmGu7NsMzXN8y3tZK0+e8X73VSvFLvP0TWCqjG5hnKTXt49rAXc5W4gb++6pbEfKVb3DVel3knKwGL+w8Ay1Fj9BfTdHeotcKDKd8CkKW6dIkFv5s69Vn1sTAxVna8D21IRwQ+iUV4/jByLrIukTLy3P3eayEFBR7cFPv95sxOZ9VpcUc3RwXmXe+5ly0ZbD+yoeBxPH1BewIpZeZHRGIFUHiF8R0gertxw3tP8lrbcYWBEa8XHmSbnN08kf1LmCL8N6rjBElsC8gkB6CZt0hzZDYlYeoL7fIKtqjsURTZoTWZWDTxBl8/SxO4iuK8tGzNCEb3XSkzjLA3O653ISOEnyndh9zfXMmzFs32BTZ8IyI3mQNhumNBZunjTBcrFK4d8OqnCAOWM6j+/uGqIdVPjdyZEDV8C2N2jnAIdI597W6OyGWtpJRFIZ/G7GT4/wzH2ZO1QxdYxhONoUuhskUFe1kHaTVgwR+sL938Z8zS68t78Cz0HIfDkPLcxQ8QLJwiVrRse6CnHRivxJbbHsapMs8gbOb705ToTYkESH0Aq03krG4PnOod2jgagTmcOotAjJei2TuRUk3vyLM9D5/i1LqMtARUjxXjKtuK3RVNBa2DI+0ZxyBA63rRa8Z2NlovjOxnWZ8WA6HUQ+tkH9w+Iw+wnGy1Cx3fQyvaNaDs7h6OzYrJE/NrWCCwoS9KxXSBEHG/bw2YPEjs+bJYcXJCjqSbTNaWiX0sxq4BJhljQQl89gF4Mjlb2JYj9q9BQXC/e6OyvtKAbPcfpEXtvwiTmaV0OIK1dFeWcNM1vN4lOL8zfs3H734WH01H4ifJbrFKJwfKEm871UYp1sa6baYE2ko+fsPGQO8Y9NFs1Qx9ZXIjhXFfOyWZWTwzJdxdZqXZOLf9iUIkA3CfLY+RoqqWZQwkh9Lf8bk5GpmXQYEjA9bi5SfdaZ/TUgMGpxHsW3Xa8N8OQQcqJhtK4dlkfM3Gr7nF13gKQRlEeHsCz1P+320bXGhBcR8q9uwPoVbvB79D4YZFAuUQBiS+fQJfRWmYHEVPELmt50VRjXuk+N59earuR4dhuDyR4CMHhIodWlQ9PRMZ/lM+eZCL8yQ7uKxWIUGYlVJD6hGfIDBkc4jnOubuqG2ZA6ceB1jYM3uMHFw7QYPkeLbwv6ES8Ha2KlwA4iLYsQhvptQEnLv5byN1NAQw952P/GzbJJK7txvw1lDC6J6dbc0aLn3kB67PXBGPmvTvKld95O26tXKJhw8FuaF8xzbu8rDADcGg6/s3Shw5/bGZnLyh3m99o3WkxTihRAcIYNzNWtDLeW4deKxWI+eCBMCVNITEUKsps+Oo7jcfeuwIA9mMEv6kE7f/Vj9FohM8Ws07DIiSrpiR0ynIVuNC2CoFZ/xguqtfcxUovZ3Y1JR2pMf9F/nWgCOq+F+jD30UHqDmx/Tya4XCj3Urz8gR121C0wv4LDQe/8wtkM+4DLnNKukg2oN3Hez7XLq4t5hDHuh3TKMfooD/2TXA2AIZIt7aEbUD6pZr4RnnCvdxojwhtFL79Rcw5JGR3YMNVySDtkEOKtPNR1alNIbM5Q4SbCZpDO6rCmJvmQVOEizvYMFz0aN0RXuHsyY18PHwDdrbtuyy6pgs8Ew+T27S4tRBk3izdo4yDxWVZfGz8Xlxu0Wia6f9vTe8UK7Vsz4Qm0ARdh7HaVPXVljXIfvbZ1sIJvX91kZFaz5vkcv6jCZH5cjAcMj5IiHgGZiLj/4dQrwCOi7dDwAFD8v0YBCW97dqsEdRaS1i8GSbHBiKBs8nJDiURJ3AnziJC7e9sJ9MWM2P7aunFyXVhyzcpytvnw76Zbu8qao6xqcAyBUXXaqOGyDVxUuy2kYBmk/pYHJB4wcND+xGVr3xTQjq6MqQ0E1SmtbnDaqe5+ve3L5e6BPFdBX/ekwLnSklfHJzI1o0KWPTgCErZK9S1cJ+mz1hmqEK/XRrjVjz0RnfKnSpt7f99KmLbMMy3uu1T1v/PiFFZ2Us2TtvhRN/15QXPZw2qf87TnF9WVRxCcfuGLSNH1bZjeZZmLAabhFtLH+yWRZkdc7WvpjR+bFXY7lRb3Arh42G3sn9bNzYwdVVjtY83NYv4Jh5cYuuI+4RruD8dyvPtrEShOG1IhqTF0n9xoM992AaY/RaRhJ/I7oEHdJlwATbzceSeCPwP+qsYBFhQkp0oYcJSwS7mmWLpFaRNyl/Fpw1KargyS7pGZ5kkExZpU3DOo1iB9/FVmWbzXF87sE6UbjYZqi9wJlVTrq3DayNvt6DjrsleQouk3gUKRAlbmr5+RBUbLOUs0FZYdgBomgENACLHV+0swtQnV4IL6fWWnAfURK/XR295EiWuAfpnggbuUW7ZJs8FbQO9eljySpVVQE0nv4taF7mKyFxnR2rqa+hBXKXvxlEwDRVQFl0GwWZZacTSfCT1doMNa1iOpBJ3K604SUtLB8RTM8diZhN125efLEhfqZVARiKzPySuxHLeE+j9nsEq2KM0g39L5FmIPPwzidSSWyFdKUR14Slvvw/QY1RgPs1acEUnaeBAwCVlD3XDnGdV/exNftoc6OW1n8wl8BP7kRn6BFBfxTpMm5OI3vcWlt7/Vd+1MF1cYHohfioShAV10N8vMiAeb/cZmXudGKpMy3w342h0d8h+F7Ug569MPbWFsSj1B9zuBCWvPElTS7M7mwY9sG96ALGDj75jop7+2Ru5UgplvHeCEDiS/frEoiVNIwXn2/1yFP+81qEBwxgaOzf3ePQdAIbJuBKgBq7D2WMRkvWpfYNn+GXJMZaiAFJOBLbFaDmhqm33BLdEkiAb8fI5SGNwHfjiSzte5LGP5OlOkr9epLqkskc2uV0+YqWGnDIel21imCHEZ2SeTk2c6WGFT0Hn6u2R5oHIdEoR75LzFAI7yaXZd6g41OjLKV3f3MksX1/oRH9OhMQ5GLGBSUdcaNUIiWTMT4d/z6LqUAnGE6McN5GhsaMB6Gz+6bj7QwokaFGmdZuk8nWhXP+B50QsHEgvRmA5hyEhW3mICcGKW5mE8OSQThweDJUfCPHnXtyfj7hHUz7rYe3+H3zXqve8OR3k8zlJcQszxmNxd70WkYBzI4ey2CJKwSgs96jRJ7ju+/pIGMbg890KcnK2ph3aviR/0yvsTSUK3SH/5+mA9NQPdf8Acu+2t1zce7/VTVF9lbgyhJQqZlvHV0K/K8lbsMU1zrExJR4IpqmzaZxv1RWMaLGz43U6pwAoTBzN2CAO5chij0z2wbKoCUOgOd/YGGE9BJrPFd0bl+5dcnrB9c71w/trwLSBInMuJi97Texmea+fNbTDckHOtdqNay0iKM2US0I9PHe3SwXWj6lrVVW0roI8ldy5SVIX1PZx4PvAxhPOKAo0gy8zYqphG6A6grE6S5NqLARqQHwGCgW1NPMIzXFEIoo9OYo6Jbqm5tmgFvBv9V7Ze7KD4qFZs8x/gD7mKr67/a4D17HUmWRsUew7aWJDnBZl/NLbE18PnQ6mpsKP5XG0gb9sCL33i2zdS8Wo2eyJyZNe3G4XJyctPzxfhkPNyWjojr17BqThyJ/UYpWuYyiPIYzutZ5hnCokqDHCZ0PfN8EetjSwntdSEEms3hOGtwfxlIO87B70i+tW6S3oxpxSSkl28U9mmZeTZqZSoH4kj7O6dLmcWzopv5X99yFL9R/Vv5BZXnMFp9t/72axNHhqKwPTgdU3wiFtJ8QBdKbWU25eqpYURq3zwFIkex0khMWKa+fWFW0R1gv5QdhBPtr5atQWu3qZhOpcB+vV+l1F9XvM12LZ5BcOLrHWVhmoym4m+XRbeXfVE80D3AyKdnayDmJ/G28y0hNygUL3IxlpEblAVeptxprb28kKmKAn0tPOFNA+gowsYGdz4CnGP7Czve/ncK/HR3N1JGaJN59b/ayJON9ZLzveCy50AfIH59hw8X8XfcP+zhZUY55ZjFmp2HlMGOCCdRSdGi+5JOxwT87UYkLuucfERRT/m1OUKW7MzAyr+Y/sMCggPhyqKxY3kY0EdqHkSO9J+UqUWLHSyVtjs1NGgq+bAtXVrPx92OLDHDD+KFlym0Beq+zYdVb9Kc+pVbB4p0KyNOpnzEe7XNR/UQ3sIwxYN/aVLfNckzGR91oVWhgGd1/JW4HcXvkuTn0gFW3mIDi/6UGlq59I2lZoru5rpp/s9V3peruDmBxlOf+71+WLQEP8UGNVTx717oMiqdKvNSvQwrdaXAEOedZ4Kh5A84WeVHNmaxo67bhl6/ejSFPMZV/eiHS0E8z4jYBk/A2ax45/O1PQZrl8C8gZ8kDR2gF7TiqRCnhHNiGPYArZ+3jeEMLuVlAvqFCMQTrK/Q/gjBJxn8Migsv/SlQCRi8+6w/dKqk1Ze0JsqK2v5zGTjbHMWuaM+q4EJIIfgc8QLd2+o++zCjTPxoY1biI+hhFHeo+QPgIA08td3avYfNhqcW7A4+2aAzA4xupIXlZdanFWg3q7qUxrwLncVSX4+87OH+nSX+8mG9eTbHgyR4rY1a5/hZhFnoKLjXXzUMertJMscl4UPZNDoXPUZg57g0bU1/Vr6TmA9vqmciTcCpsGzDwlUr2NmbEyHF/lpGaq2o3HeKKSHmaN9R04GRv2RHtJ7a2KT6Ep0pTrWKV9wNFbHhzOyetY/+NUxgI4go23pvtYt4VrZh+Pmn1ZNthRNaS/tUjpLU0ByIGqj9f1SscRSTi7xPslHxXhq+CpZOLJM6qLvrp3Zo/3QPjBsa3eONVk3jdCog78lBCTtLPT6FTyZf/EJnv8gUrjSyvaEZdF4AHG4BIQb5p9tjNNXa9IKoHzlAAOX5/K6Ftxmw1WGS0poOAcaEGYbLBcXYWuF7bslx93Jwyup3cjvdaIs77kJWV13QB1u5d6NBGM5eml8gIVkS7Y5gmPRVc9X+O1fovm1FSso1u06lo1u93FfQ08pCb63Ffk9Y82rqWzEOC0cLLFhGtO04JSWEDQBR4kMwJaRAqGYQBDLCIpqiNOV0DSIqXvmmIYdGhQPZfdwGEtuZnjzVn0aSYkVEpjxQz/zl7qe3n/U227OhQVH+B+Ciljx62XSIuuYIHYFZHjw+0HBW99MamLWIScY/R3A1whMbCQVeuXlpSsZuPleUSAIHlaMLkXIKAaFELy/tk+1D1K1hMc2Y6DzXU1tF0lPRbzqnT3P2ktoS8eDgHpGKJLWGufBz4CUl6joXS4lysmQlCpOzRAf6KDrA5M7hTDPyfs7tEPGby2m+kaSPoYco5irihLU+kuHfDaAS8lrp1gEP73PCSTAGNNtL3deICvUPoquzVToMgBvNV87jZJzpLd6Rnh6Nu+y2fOe5NZgO97ou/y6cMFdQuy6X9f6ICFst0NSq8KvdNQ5Mu2E97R+qXKjCNyqyvTa9+C8pgfesBcOa+HEE/764uvhGB8hGItgRiohGYUrPfFI794Mc/AfoOGNeZrxBUZ7hIHyhxieD0cpAua11UihVq+0/1UH70Vz4fYNA/cvj09EqyC0scfjo4nIP+8g4RsIAPyUZb6hktS963DOiMgXa3dOVvMdg4HeawvHBJDbBP7Wk4ag5UtyMxZuPjGjWAyR0Z6flxDxO1cwjzbdiPukSY5YdCWB2FPkjI/BbwZRZWnz7yAAtlAvFJJSwzHtEStRuZMuJGFW/MX+H294D0gLGlz3VxbScr64OoQETqMikGr6X+wCagB+F553CpLcqEG8dXfSqsPnat4Mekk8GEYR2D76XrgmI8OZBkFRdn/P7gb4d4tAxPqGhwu09A+jv/UTImYhVDnuL1HKOM8gPes0Ge47/l/23mtJYhzZEvya+3jLqIPxSK215ssYg1rLCIqvXyKyqm73dO+smlnbNctIi8ykBkDA/bjDj0PjCUb/phOpa8hfHkr1bC15wn7YOYxcyttk3ojj4VjVtGCE73we4/UMDJzkM/PPKP311kph4K+KnHxjCL1rsGnnoX3cucsG68b8w5cpQ1lFWdumn54bXDgy/CTBm3/sBw7whwAaGzhlcM58ZowQVj8th9WyOgvx+4Ebprn0IU1zBfP26cmbfZPfFzc0bhnQfhk3HCVc9cZzPFClL1bzT7FVnDrPcaKAMP4jC4P1U+acU6VGW8pCvtGTLMqE+/FRIFmeU+i4PpDkp1aIpG+Ur+z44W9g5VhpYJi1wxp+5PVzPRqHewlrwckdFlYPBRgf3iHmM8nuEiC3UKhgk0AIRP30SlFlOl/hQ31hGioeooGL7YMnOU6WwO1lcUJosk1x05EjsDI8vy4D3ESzarowjoh17oG5hma0yAem/dRXJKXW/eT2Q4qs+dq6GnOvhwkeiAtrTvtgMu7PF3/QT8btw+P9HCL1BVvoDNTOhLswKXLtoUEC8xcryRFdZTa/vs7ktsrmY6JmPwQpPUEvS7OaOcgf7tdddEgAcXGtgDPOl/njPmNgmkDuCPwBkPNDEywrKEqsxY3mryvLG76S3HKSLEa/eQYiC0dhrv7ybGiaKZfCf71bkW2uDIG56BSuj41+votHuFqIST8MKw2y2mXw13tUoSr2nTH91uoRNrkTDcOSDgNoNDDmWk/6trPe3oO4S8dP8efILa74B+8FavWlwjZ57Cyh/dP/yMpnkM+bA37LTzY/poAMrHL2scD9NNhpk2j5hTEC+6WLyOyTE1VJF6SL99T858as4B2rf67uFevF9ZLFym+zGunt3dgXUr/7LaWqn+tnzTdaJ4LiWG5ZYT1WAZZFYM9r6tP9KS3wk9OLsZNqEH8eqReJzVOPP3T004vMVpZsmbrei+iA9cxpCFk/7I2TAIIyZ+TRw4u4d0SORKcmt+n+0yiy5GhEXUysPXQJZYvVusja8tZV8SlTgA/4DZ2xxgYquWs+oN64DeMTLLfuKH/yGxFOGaYJW2cLMk5ps63F4cPn6yAj/0WDt3Uyg0uwyAiiE1JMmHsLQ7GvXuVZiNK+XLqs/6IJCpnSxknbR5tz8JAgf/b3UiDYUxieGScdqrbyQc4r+LC0t20o/5SC0SXPZarzCg3yYa9GohXTpSJzwvzQv9iIZWQf9setPlnnxgo+8fHHlCn/7BtVzin9BC/MqkWYsEc/z6X4Jz0py8hlvcKCl51I4Z+Uuu+64TVXSkaCR7q3SYhal72h00AaURJ7aYxM40rzXeGIWu92x0bOFFk+ZX44n984eqjxwZIDvD3oJJ/7H/cD/NN8XW5/1Vxlmdp0NJhV9y5xRAwbZdFCDY36GYhYlNicrTNIL7TCKopdRxjvss2ycCR+BAAjmTgraQrK3ja3Hmb5Sx7oNLFSkyrTn8bRW0O+xzlRdVOFiyzIBcb7qorrqHsUhFVmK2bp395AMbuWMNpsYP1Wafv/iJn7/86Hfn3HDk8yLMWWKiSQVIlSyc8x2fZwbmnlsiy/lG7A6v4PEOcFTcmSDxvYgyBaJbA95LyDMNqhp3MNi+n8J4r9nPjJly0/fk6EwS6U+w+U6Q8hH/t8W25RB/159P788SBR7EEiBEQST/zPO5w/hwnyDwKF/v7A8M/Rvc626s/bY9DPviqvy+rPwqH4H0A9gd3J+rOr/PvR35nyb4EARjqYvOv+Kt/3fwSqs59rpiAeiE/R/rclRuv+/ZTChPhP7M8yfJLunf+c97Nj3c7uzx1rlUzg37pPyvsv/f1LrVOeggJC957kr42iPvL7cTRosjpNOjV55Z05rvVWj8N9/DVu29j/wwlUV5fgwDZO994sWStwPQvfG+u2jG0e/Nk4YE8xDptTX6AsMPlXQe6NLLnNL5T62bw1wFD+MvZ/Gfu/jP1fxv4vY/+Xsf/L2P9l7P8y9v/jl7H/y9j/Zez/xy9j/5ex/8vY/2Xs/zL2fxn7v4z9X8b+L2P/l7H/y9j/Zez/MvZ/Gfu/jP1fxv4vY/+Xsf/L2P9l7P8y9n8Z+7+M/V/G/i9j/5ex/8vY/2Xs/zL2fxn7v4z9X8b+L2P/l7H/y9j/Zez/MvZ/Gfu/jP1fxv4vY/+Xsf/L2P9l7P8y9n8Z+7+M/V/G/i9j///E5/87jH0Uf/6BPW+BDT/QJ4xC/0zYh0nk31D0iT8w8l9Z+n8x9//nU/Sxf6HoC+C//56mP75vjTLkzDgM/8XMB5R5ZuzG5XsOev/w4NF0uSRZnf/XsWEcALW/qLvuH07HKRSi8b+5+P/dyX+z9aH/+4z/5M+t9C7LbVKjdLX13b+h+yN/bf9ZX/jfZR2YxhrchfvcN1v/yiLwZ/KC/rirPFV/JPuK/VHe7fffQNaA/5bev7qx/PPufNLXHXj3zNjXt6EFOQmIv4c0539lH8Rw+A8CQ0C+CAJ0xH/ug/i/ZonAoD8I5N/0QOx/UQ/862H/V5JE/B/2hw4coJO0LZfxPWT/0OuK7+d/v8v896/9X/NR/JUYotq2+wrqBwmNUz7sVb22fyRTklb5H+NSAqkNzgUTTHU6Dn/9/c82KdrkP+/Tt/w/t+XuBX+ArBL/K1OHQMQfBAQjT+JPWUT8syR64n/8m55AIH88/40s+ofd//M7w7+Ko//fZwxpphykDAGWHYjDArR8p41lu7xtAoujHIq+8QYw/DCLieywmuJbq7oeRClfBHyfZnG39vK++gv8ckru9Y/bXsmNFKX9uQ3AC3f8/P3ZGYzZ9MPp/zk/Lv9Wld+Tvoe4H7htQ53uMdSpNuVbc8E+AxwVvwdZ6tRcpvyH7UNj720QdwU+P8QU/q+7/4BU7hZ3FZSJFKGez3d64tVP4hO8eYEufJux6sW9NYakTPaJpQLfJIgPSYIMZjPfplN83g9ykISuNZ0s1D1od0OajYNqcsTpjH2dcDu7yfutMQJrMC9MMcWSyAV4fwU+FDk09gqOd3pN2H39z7PdCWxvcWhX6peIxDTHGQsR4bW2IInV9hLwyxjkNm6gIRFtKGXHj4pmaHbiqHbin7RPP5rb4oZD7lpNnloNg+u3FO3emcBjaoBf0s+dy7E0RXCniYtDvUn7bs+E7vOq/y5Xlwtdf5dtzMTbCKjJz2vQ3lFgf6Lee0cgAA3xsSSIPpqD7X+Wt6ZKU/juJQJe7lLkuWahPb2Av4PWL4n885p/c/SfrvX/h9f+y9F/vNbr/euFHPC3Np3/TkIQaeMEeJOw+ycG/biZnLvOn5foQ0nwfPsBPmUspkgMRf55/L8S4rj4v02IYzLPb/sYnf6JBxuNQrkz7zaV2ONnfwtKEwdxKF/gGeZ3Oyr93j9TBLQz6Mvc33X6jq/utvkG/d/Uye6PKkUqKwmrf7hjvEahPnpCh0n831eW4G38Xbp/OOfvJzf/VFP2bqtP2ul7FOidebdXdBv1SSCfUWh38X+12D+fB3oP8/z2TVvwrwiVp1T8+22UI6mi377l2Fz07cf7J2cn7BXS3959X0+aaAXutcShVcYBDkbRKV/AM8sAP9qL4oBZwdE/37vEP1+WtjSW2jWWLm8htXsiXaYivUf3t5SZvb1PGiV6TxXWGhVm38HJImNhJmvtxnfb/PPuP3f93gl89/uq8ufW/0++4O4MRXMSxaW0YGk0Z1EMZ3EsZwF/8C0AQZKKuybaV8r+k4FQeiK1WxJlSTKwE6xbnN1nWN6/nAmsNpryBMpKgWv5Od6CTgMy1qIOcFTcgL1jUob4sShajN6y7MVyvOFHqjCvbnutjIHhjYNH5FyP5wd5Dco4ZMiC5NPP/UkcDufHbZotb2l5GVYURw+kTZ8hIFNNxX0Gb1a+ELbXp/M/NboyiGkphEv4Igu9X5WQtk6C6G7z7qvzISP2HMjT3QM/8ynOr2CI5StfzJZ436g69cMFsC5C8uUCJzMNdfMa0yJwzqw7rF1FJaPSLCRTifHz6zl1y/yMSQSGydLgwx1k3xkA6wFfEuIWUrMBsUQDJg0v7zSIeTGjDvEP+1r7reZtJUCmh/MIC7cpjIWDtU5yMHEmPgIitoktAzb7qduzrxzRQ3V8smHjVnKDJ2sd/qpk1Z6qwNPgFCIRPC+yxGms2y9vui2E0L1m1F0S7fTBHAGfSHmEWvmncT5QgR08/ioQB8GZ7e0ul30Br9JsB8VLibKGfOi70T1FYIVfcVNMSmgD51g/+IicrCqcPQe9ltDNEqaDK7IkiSLnQe3vvp1fizkBJ1ISzj3VvfqPA2XVh37gfjttrHLOC7TwS1zbYIKtaVaIRssLaczMAiiQdiY9y/Wtc7zGS8wsS368POlnRq8ssxaHufuOjywovF1Lo+c+Uq/OeSmLuArqkqj8+qi39QrJoOCtV1N0cF756LbnamHx7+TB6EI5sSPt+zk+5TLamJ6pPGcAyoBj175Qwg0s4Ila3t3DH8jzEbREv86ydQVYrDf4/Blbp+7ZSnME7czJCWcJO1uWHSFUo3bnEBJ6RYXYMUxy+pyRRNk/0ODK9vN6XXtoXhuXAyegy7mEjMVQ1BX+U51BV+fgYQMqIgoa2TqBv3enM3obLgQChXt+p0Jh055n4PgZRBCl0zxgEIBLRwRguBEBjPbvfdvypHsVAdaxcH0pG10hzSAvbaLH69I61KtzQKYpGvq0CSwhvnYw4oPpYYWmN+S0wzXwnxwh1+7XWcHplRfyofRJ7/4m9Hop7ZKlNYNki1mE0UH6fq0kHnI9jHjD4jV245LNqx1OYx9rI0zwOgURziM/Gs7SGJlgHKo5bq+P+Kq8Rqv+xSPCmA/U3BG2NnAtD7DoZy/H8M5s4VU0+3/Cs+JHNqkAs31lMsfQlnVL0j+lmBXdskm6kRgncVXk8rQXSTRl8yfXaIx3WEBC3rixvKUaRdOWSn6dnJ9nmBxmVHmuEK/MfFcoboVuLcG4pp6vJ7xN5z2EL/vu6TJac0ahygnazer9bk4w+ml3LgYUTFmhPg7zYPgF+uy/387LHR54gOg6KtqF4i+qqC6mSIoeZDALMWotQsk7YCwvmF+EBbxFbJ8slI7bb55ZcnpFrcc5eA2HMnAQm0K8xB9Z3oFiw3Hjx6nj3lZM0rwuhHVxlYVBnBSPqCDtgKuqCyucWK0E/rQL3g6rqZ9OUQREXhBPctattKvIWO/RE/VM78eSDc6KZBJ+2N4rOOzZaiC6mz1x1azo8tDHp8+Jt3reWbhbn+5Iz95IvEMZU1FnhaFQcl1tWISjkWfzOFtPx7Ecbat1ecSNwEhgWpHHbMre0mWBjXtIB+UyG6nsrxnK6l4RIcd65HDFuQgtP40KIVKDUbO318gB0RC91k/X9j7y2+zLUg6PbWT8ELiInuanLWi+UBJXEbEQNI2fGDEDk4XYI1T70UNYRl6dq249yAs2ycxzuaur1BGrByU11TZV2UsFooO6S64etFV9LF6G3fU7KlsFLVo5JAgh9/b0AlIKY9gBvdUgIAhb61twK1y+1uwQEZZr2YCMzodbPMfOKAmK/dwIvZg/N6IPC9wgPTCFIj14JGSxb2gq/I5Mn3sPdi3yzfpMqyT+BGZhi3DAoXJX/bzqnbU6f+1uMAGnJg8ihym/IbYFhZ6gRNnxrFfIPg5I2bp6NtiOjZs1rVGGn0Q12EvdmLYJcF97MX4pu6VKLbopejzPi+unFu5G48Yh8fMYm95sI+lx1pvt+Jmttpaub4+PuJ+ygSQvJMgJLybra8tkTUAyolG8pER9MbLtkKjEZKrORuCdgDzG5DQ3jEveM036ttWhsnV0brYyaDlsvTqjkAg0byGn9KxbIrAHwUh6n4iC0vICJvrq6mLYWZOIjFoCY5W06L1VsRtkja+zLetmyJvmr8XNtSXcbqAAlPc77HMgLg5iBtMbjg5iFnkvsRmFIJWRQJ2si24sT8fXLL/8ZPe5spfeVAZUgR4tz1H5FDAbPe53JgtZ85oGFvgI6L4SoU+YsANJaFuw1o8SwuJ+oD9btCRudBEOdHJyHmZ6rkcfuKU8Ef7kH+xdFzUwkm+Me5nbJ372DiKMkwf2xG79hFqyMHB2cbbrpED/RZ/WT1EgmE93s0J1nBa1rHz1uQolSD3Jxmj78dNc/D7yZ7ygW73X/OgeLb4yQYCtzCqfowgI5ePAN5ppDruB3+pLzY0GRkX9mSo18yawzT8Zs4DuARda76ksqRJZdW88+BJSOf0B5Z84HWt7ZjgoUPtvPAal9UUo4eOnzkAki6yxY5PjMQRTzw9nQunzVT1LMN35slqSTjyfvxgcw0EmOhp/bop7PRJvaJZRTzodCKzmvaEP4TBFhTd2JtczfaLdXNkQdSgawbqHOLaH6xu0iJLNbhce4hJnwEXA68liA92mEX7TgHsfEGR0KlNhGGM8XpYXJCAsfP2AR5aPg3BtH8B8mQQ1Obyj1RnnxhOsfiRe6nNgWI0xPI/oN/9NE42woQhzWWtg0gECnYqjWMgGQPTG0vd3529r4G8ofe/jaJtzJJcTuFOmPVep6IPyeDsqJViu0lg4+Iu6rJGnoJx3y0mlyf/STDtHm7tI9q22YZ0jQlgr4YUpVR52BjLT6s/B+7RyOmk+I0Bbhzg4E1UTovUeMjDZQ4TA/OJucAcuvjVzq7EPWVgeIiCYg4P2il5bgaGQlGfCgrmxWxJfOr5EZpUT0+Ml1SfppnI5ky49YTkICEYwYwHg5MsYtFS/9oGYlKRGe3ASTA3ms6IiRG1BYI3nR/xqIcFS3qBiaPHIeXa4G7wKjpd1GfGJSxUkcvf0NxK11hKP5KtJ/VqqKvKSbdJGHm/MDbAHSastEJLr2oWEOMrrKSBuEvZajDnXIVjNW79QyIYm9OWkFwCuAHVjvBcV3dDlMggUaC2fd8pJ4xfjeUMUB6Zl7dpbJakIwSu77uDa/FvBuhiZoJg0B/SLrBT9EEwnJgPBY+2k24gjEKEWIiA7Hl0mk7aBQXvGGw/oyDzQxtbZKb6YwUH9naTvlRgzoMi5rwLOiYqFnMllXDMrQvNRCZcSX/jJ82WCc6GJ16oztt6iSaumVzOv7OUFBED0gQQokxzy+kD8waM12uyMyRuIpk+yHGiKX2mXJgddok7sYui51a6PDxPN/OvlSxoOpuKkg+9np3xCuQOcr1JAxMKn9UtYDpDO4Yo36UzNhIFCXqNpkuc3NuWWr/Wbc1CvXWtzSyrVaRSc4+DE8j5fZqIQPhd6SE4jIRUYWIxY3UBOfwbtCd/qhOXsbl30SVKTQ9LaWFAyVxirKDgvfIDZ1GdC3+RCboz53uplCYpYvHwk1B7hjEdWjDozN6B3HDpmXrKzn9Oxk1Uj6V1qR99KvVcX4QoJdLhAwbzbUGKCdnKjsmvb+W3N8VAq+w3QyK5f32PSHpAkG1KFJ8fjoE8LKVfnEocbdu4kBrB5IWmudDYPK2p3xEyc9DCWNXHwgo+lATYEW3jReMCLL7SF9YszX7emFtpVPOn8Vtp8ax76VBE9/yo8rIfekoE3pKdkaVE3rrXs7S6yUkuje7lzYgxyXnHOUBUVVzL9w83TDuNu/V8GBRAuN4Dtj01ZE+1i1s32sjEiX5V9KCB6bTKdLodMdcJqY6/4B9ewRC419kULiYOVzhlgdquipsSrLuInzdl9vIv3++59hRBnKCBBCP926Jd+mjmHKLiQonYvEOlDqfSuJhooTnFVHKHbxFk5p33zMJ+0b+PzWC+Ie0VESpwuKvlMY9j9hmGjB+9IP4gnnhSnklGti27I1YtbPT1CAW+tYR+k5c3v+1PNbMqqZALljfEiJ4NULeMhn8EXX1n7xiDDSpO38da9HY0mF8ZcCzOPZfhj3Y8z7eCJjY9WrzJszv3mBFo9eSu7D0wj3Zr1onOuyzgy81bytR/ZzRo8CPpYu1ihNX5eH43aytvp1+5Jv7m80EK+yHfrUme961+L97gHWSC6aT/zzvCWtHHJbZmRdCtouLbEmVv/YV3C9F3uDJykZE9VVt5xhTiXMAm4U9yS1f9kFRU+DKZHAW4YczYwCMBQOqvJkI7Nl3vBltpAagjmbWD4XOPMbVzyU6TaMru70HPqmVfTVdulO3P+RmqbdT9T1tG0ckNX8dJ0X1HZEbdP/0AgAVaf7fRSe70XXAeaihBuh+jRSmSJiumhh9WLcx71omiGKSdqzS5NvA4wmcPMZVJlLVxtCtmkv1nEQpUww+wvcl3tpB61j5ybOd9zcJ1XBtcAE4XK5/UJC3SMiyTtRDMqoDc+ZjLy/HCDV2bZbBPKo5KKu0Lt9NAOf2ya1ytxVGWvfSfmVWYXvyqCfThvzCOop5q/80Hz4ymJk0Yrr3B6OBqbrJU0k7d9w8d4IVnj0rjOlJxf7TNImmno5utYvLwgqvwYUPpI++hxxL3y4g1TdG68skxekp7MbaMpZ8O602jg9LHDQ+0TUwyc8CL/SmT5bXTd85AdOFb6k9cVhszpg+/op8d3O4IKFgjXsxsVaQqUCbMhT4o3myOvJ/qozhB4L4Z17aN01kV1tQHg0q18VuU2CW7hBeTKqXLgR5/El31bfS91vYF3XfM1X2WBihVQ3Ks11jobWRnqBoAv3ymWs1kdLYPqyj/TBzfeWG0QZXJjC4EqU575mcagvmEUN97gLM+RrMiVmtvmtW+s4VCN1gs0yTL2mIpA5l1WJx9rxgUWB5QbUlLA20dTLkk/wRic3dCkIET21BN/f3R7vdAss1tgeVg9gU4osFNwX7tBf0CeG+gR6Ee+BhIYk6yBQciLwJeUCN6euTKvpPMF3WuNBlUmZK/ix42gEcRbmn1bmw2VrkHMPOGuIbicx5ze0XPtcgt4DZGqKekX9+qcVxD1IZgxIYOrR1xluKEyQIvfbLOUnku3pFN7suP7WLEe84ih/XPIPEzQp+JSNLfh34u8nGmDFubeqSLu41+G5s6KyzNZ0ua4tseS61tO8B5ob8TrcsP1iGWQIWXWfJTNoMtHMz+LdRW8iHQy6wFZpzBI1sNKjRkJb7hJ7mWec/eNQPQRrW/AvKO/Nuh7Pq6EEYIifZsrAmLGmmeh4srWvzTy3dzKuZsGPwi6qU9jP67I7e4l/cs+d1SICJYNOuCX5nE1folueDpr+kJwhamKTngIdVvD62PsN+q2PK28KAUqDLsICigk8JZUV7PbIh/nAElOFdUxawiJsT/bLsV5pPswU6va+4SZH8PdhwnV3TXHEbt0R61fB52ropd7+tojsBs/99OZCPx1jeIRIq9nBm/2xp19g77Y2bu40Ip8vHssvXubI7gSuHo61wmEZSb6hn5cbTwyJ1NbttdqzP0Sx9P2lF5v362hFoqr+4YgMJpYnoNboLWDvsh9jt8Op3kttwzhuZzrx1T6wzBKRhd1JOzkF7J+FJO6DGbar1mdcyUB9sFjHAklNNTKXzbUbZezBbBNHu39Vl7dnFXptwc4H9m5DZ2qBO0b5xVwuJ2zxsxhcpvN76l4zNnagRjBWyO/vIXTKLK9ciPu/VhVJZ0pQSqkeh9iZ5qvlFRzYh9oJS/jREtAvCr9sLBAQT5t+uCGLYMCxmdlRQiUSYgX3f668hr4xt823Fc2lMK8dW0DDxyz+wY/yA8wlhTUfb/etrt18+nwm++y2jdV//NsduXTtiejoQEI/2ulEoDSzxuf83gF6dKeqoVCTNlzly0B6LA09SwSKrReXRdO2WDPWae+5rJqbhT2ANQv5SBKpT3odlm3zt+i5yW9iVmpjSHOTkHbM969tOGVaWVEjOljnftPU0HKq9FM5UwsLC5nukQzWG02UyCPlpi7BcMH/KUmg4K0yqR3RA2iCWOUgT3h8nlXNLs3TuWGDD8U5R66Tlpu4xcRYO0rXG+12oG39z4YED367qdYI5Wn7qwf3NDsAQs/T/zzwgN3Gt73YHSnKI70wwOUSpDLjN++jGAgQphJMwqGFkI0Mqy1PD5x5HOhGwbqGTRswVRxN+28Ay0MWhjuXHc0rLw8aDIZ32+3SmaG7I0x1kAuKoUdaadjM0DUdPRQsys6+y0uqkC8tVeawU6gsioaGK7gjIbvsSJHKQmZ3gNWqkrIJZIgf6PbxpdrHnVoHEHEY0c3AiK5FIrrjt/9Ta40Q4DkJnnhzUQaXxkAgCMr+n4CpnPODV+STxZlnKkebE08u0qgYb7Q7+6nsgqzNIejIy7iUpa5YENImpRK98hlKEf0LtT7uuN+tDPXbmaF7znwp1stNZY36dB5WwRXT/mZAU+JmHYPxe798yl6nxfpc6cipcyII+8Ua/qJCBRgeN1y5uxESZnmwLaR96w/RJtU2p3o84dBKko1YWiMzOMsGLkkPL1nIwtds1eVr0JMggiDyMxn6pxxRZBPU9QuWVIrL+CgTXCSrA+IJblt8FrTsDczC1U1SRXLqDckkcOm9ZxbPOGeECrhqaT1FWq+Fsl0l95oiZCbdUqGTyVBPrJ4aZsexAw6BdF1pjdyi+anCwNmk55d420nRIfKxQcvMPiAUe5+LUma7Ivh2cKHKPOHaT5+5tNuKx/Mp1kSS+0px+zQX3qZ/tHJFM9VWqVZkSc1Mn3XpkkjoeJoD7518VlOQn2bd6V2WYiHKE4nB+36UmmZ8faUlCkLsps4MlwschryHz3Xgrmrjx5FebXLeONpAk/uyM1ZIuh5ByblrtfH2A9W1ajNF3Nh8vJnZxHhMzr6lkRe8cVF6zFLfqDMyjwv0ovl3s4Fe1vBwOr1Hrp+2zzq5ZxaUaSWVZgNciDAqw1dsLk53PMBvAFqQVLdR6ybMRHtRxS3uCMPPFz5doazKCLGaBshdTpqOA/CD70YToyf+Af/5b7243wFs5byrhkkSpdk14qSnUxsu/6J8hGHvcXrFy1HuAopRTMzRez9EN/XCQZd6D4DadzS9n166tK+vEfwcEBgCUntLz2wDTQTcRA6K5gStJLDgTBF2IxeCkJOhHQrddmxQYqAPQS8w4/Qh6EL/LjFJ8RJurT8V84YYB4KH14a8Pqxt3rxmR5qI2rIPkyDXsHnvRZaqhpA14dK2hHw/T5ogx9is/K1ud2WInfqNpflT3nkUEZZRaG9Z/epPQEKnva1fcF5Qg7Nu6vTPpmBYf2OWXgCZXq6WYWEWHSjd2Ws5zlIQ8fRlxld46oPQujJmt8s+Oj9zmyDLHz0vIXMC31lXxtSzpWa1aQ5GIRcvWrJcjoIWSOqqEe0ywCj0230fkzhiXBEBJ7f5VPc3CIRN7jhsz7dPCTC8ZBOAsEaOWHsoUT6RCqYgm0apMGHdiw542RJsoXpDzBx8rexTHkhyg9TWRlZOnTLx0G3pShoSez0ucPTC7eO9FFq6RCWfe50s7ZHjCMjjbyJin4xB9bUWlEJAEpvxjd3anmgCWrq5jdPLXhO0ZDy1Twn+JsuM3q7EctPKXQS+q36KqrAMCFy/JUaUVSWcwyQMFVSrZCoa5sK37rsm5R8UVoEfdZSia7HM8yQSMFjx96PLwBvJqOw9P2ZUyDBy/tY+dr3ZNKotfRq5ng9LulxC7bcnyIMTP9QoFmK17f/ZfvHh786ujyIb0Lz/dn5X6qM4E8Tm/eD/iG3BsGIG7UYxLhGsOa8aBvF+CyAmWM3rDYfJl07MKUUnnXBsnitMPXJqjDxMW8IELrJe+TLRqLSFoGAoDLo3i903ifXBNHbKprZuxbJq5UMYSkZ0E6TYSgl0TZzQuTvGQkepxOEYQKfvICxBQJi65VFI2+zakrXKTreDxpdc6hFKaaB28dIPNTp0Gk5R6MdRdBmPYan2HG63BG+J7zT43olj7LKNmGC2Nfroec7quTda/L0SMmToATsvfndV72qKCXvtBuCrT5MN284+ESpZrHK2wlS3VJTu3WU1+MqcveDoodqqjdcsT5tl41h4mSSq+gYYZYIeYOGrgwtGR+G+9wM0c/UW9ezQ0YUWYFFkwqrnaOTufpjwDt9ZMQy9JpkyVHFDFQC44J6STqR6ZO108DUlD1vhCEoBmlRowjvdF5L16SQNCZyqQbbXAuh6BsM3k42tQ+H1bjh9yEMu6wvXx+XlV9u7m3wghKYoW92ClwT3/eVEI0++b6/gdkFwi2I1eoSYpwiIRzHat3F4/kl+BzLLMcJ6OaV8w7NZe9h9vXlKzKv2kZZqhuijEqkEUZlaojmZhWkKF94GvbFvTmAcwyYjCVlRuzsLPZjt5Eibecpe1MUT2Z6VbwFAHj9B2MdWW4hYoXdn+58T+0sE/ndMVe+zYvEgJ3ZMYgFGhBPtRMba5sczKT2V2qq3iEl9+8e26XBavtt59qmBANUX1XkCFxfhq20f3K6zs6oMLaWAR1i2SggR3HuCp/04I9nrjUfWOmHmSjwsI1pYkSOdcLglKo62eOO1WCBuE60dJ+U3GinTA5tRjfFhijEpkpqQVBUV1Q2OEA+iaPAH61sfBt7vS+eqraNDWefRC6gFp5aHFIW2s5xmFKhjhCvfB3oibja61zbi5nmOJnnXFQf4+ONxmn5DN2GBAELQf74ymogiiosceXce7q6oL5bX2q0EUtp+T7xHYQPM3zk5hHgcbE+iw2rOcOO0a5TAtFQiM8oKN75nKARTOFfmOGT/UieswSjwlRX5MfLJYoYn8DY1+vAz51QmdcrHwMSIjsVBqYGO78d/kNSYXUbFedHi7j5boY3rJyyC6bbLeaGxEgX7LD14G2yGAv4YC9XSfF59Dx5qSeG6JHHejxKcjRhfs0xU2Gf9ovvwe2htwXl160Njl0Ipa9R5O7SGKDYc889k6gR8ZbX3UUIevYwnpw1N9dAtyM5X++l8QgH0uWFMrO3YR457hYzvesF6Iydwz8c7f2+JbNTJtUVMO+Wh8Ibyb2A91wDsQ033leFKzmtAV2TCmCEKE3Dh0YvL0bEXk2DVwgG7QvUwMNr0R98sNmu7G00R65AbyvmpNeMjh0tFc6N+rMiQB0K3oXHHgktKdoaoW+mpNTkbhJgL8d0gBbkzW3fbzjBs9SQZKeixYq29vyL79YWtk2TY6L7TUXlWMyrExv7MA9cX8FFrI/58RnNtBA5xMOn0HvSfadZMpdjNToLqJPq8OP1SNn3sQ3XolQ8sAnpTAJQ6FV8DiC2A4g4D4/XDSUxg3fXxz2VnYYEEsVl3SPqlrjtckObR4d/vY+4HCkRJOw9TrVTrR34n0rTtGleTV+hmtN7l9kPz9aOACqhdRK0Z570Q9cNAa17+vtQt+MpgvyKPLWCRafiYtlvKNvrH22AsSDH8Pf5cnDhYSdfC5nlNTKVmuEClCTIEb8Mbk/sGeRZFLUbQULiZ6bg4ncZX/qSOl9PEOt1qgOnL1mIg8R1I4pjP6TdyNrUexvr+6rPjQ/BkZlMJpnAi+putuUprlj2lm5o+jyXJtQ7JLfVHEYnGHjrhuNCGmSv2V6r/UxV9rUgGiHGa0jSkHDf7bJ9beNJbMXAEDaRpmd1I5srWLkczpVy2t6Q86zuJlyTS02fk5wbddL7YG2daKuZsb5tuYpvLVKd97fpV7IkPTsx9LnNmQuaTgAws/3UZUx1fcwRJsasyrr0eE1YK/rV7LTO2vS3dY/HFGUTTJoOGM8Tb1e/b28Z7YgSS2HvQud5afyIDeDqaJ7ZyjpT3wIYGsqf7e7hReoJdvya6kvsKDmC5tC0Jgm6dz7SIL/hBHAvmSww5u33tamt+6AZnPA6I+8YlLgtLvIV4R+QUE1axj0aSa807UwQngGNcREbb71wH3d3BquX0I1qNDnnubpIoJUoahSGCovOWwUmTqUr/kT0S+nxcDLKAaWZxShzhDOs3ePGEu/3QgTdGCfwdb3nZgl6eHw+51VJ7Akuu9ea10sqALzPIZHavQNosaQ47J3eDijq7eaIeEMrIq0fydaRwQuIRZBiwXB6YG915IFjaL0Z7xXw+Yvs7j8LkMfyrcuAvRl7JkmZjRI1EY9t3+5sZCHJaA7+9CAwAd2TF8w8/Cw4rDWTKLMCQs6QJXleVL+rje5YM78h+IN3d18Iz8t9S5Dl4YRtUlkoRC8p7IYG3wSDPEG6HZ5Y/esQrY13BEPdjcEZlq10Nsf3Urn2bQlSafWDkQaEXc47zmWHZskHGBLjERhslq5jhDEGjgfApbRT89IohPKZ0UkJyhF4BJsaafjeaNUXFB62h/h6n9478nheLGCLvCaAUfzj6E4i0zo3zz6il6FQdr+ODw2zGkEkmVknG+CBTtp3NcEs0eel37ddqoWBgdrguTmWZHJfrLpiC1K/Z+ipROMrq7bRhdsoKoGlR8wqwISp52YkLsjKMQdZxE+Z66zvNveTHSNvwC9o0AICNt4jXIpnAxxrZ/4RfBslDAcvdIDClNY4N8Dr4h+zpHq3msjs85U3M/Z8eNBQvNBzKH0DKzEkiJdu7M6HUsf4zPfSgLQjkrwF0d7GuNSfy3ALTrbTBajGmx52Rdpdm4QVrL2NH6Q9+c6r2Y8e8eG+ALW/WBWLJknxiR29Tf8XGj1nelkSNCsOBMJzx3H9NZI62HVbBmdAqElYas7hdRnbD4qqKciwCGUJ3r5yLjpnC9sB+zYFBjD+en7WGJZ0452L/U4/stECzvL6NqwUrFm0lhEB0kt93Q59mHXs2O3m/TbGvhOAyrkt7+0iqfxDVyM+Jr2iWZBvyhwyhZ9vzn4hW+PmBljOMLkgvSf9ShP+GQvaJ2k6x1GMoWlQPZ1iS72VNtngdzdOiQbPPWCKaMLB+947tA+fk2FgiSDtyym0YGvap+ibKzalRxAjD54rCiyMXUIzWUWa+3b21/2TheIgmOlkwv3rkbP326Q2nboeriPG7/YFkJkjP6qmo8fOJka8xKGfyM2Eo33rW9hU9vwnA0GnOrIVaiFPemcf/hv+ZuZ3CdK5x1c16904STyq53C3fiLg+5sa5bLXdba90/5MTwzzkrIQnqQDc+9g22JE/Uxd63G34fyplLk/EdjmP3Z/DwXxs7rAHteWkxF8ea6TxUkm201lkYfOJZch0mSvqB+JTujKgZ94Wvp8PsddU5Cq7B4TBVDMHzrQpXQbo2uJh9uiPmRsEDJyiqOwqq5uxAjOULVIOiITA3U2gan6CLYUzvhesR/3221moez9XF+a1CYjkZd7PhyEBAcmQQiKmEyeoDE3Lup6o7JnHcTZ8WoAdUG6XUn1lpJW3viYhUfUvYcIvr+4aF7PZR7jYeBDSJhdfj14zuC8BDW6wv0Y36UfufcLum/SCk8Ov1ShNIs4oD4Y9jwK9CweuUy7sWgNiYAjRD/eUvQJoMbQnn42aUe26a5WvnOMXtiLFd/qz0sFM0av4wHmTj6A6I6QcQEsaxf31+SdPvKpxw8Nm0aDfz5SJxePNMbFEE+uvl3QCSsetjLdmGtRsqug0yhZbtNnSd/Cs4f1TTKb1xMj7dsq+mJu2FeVdkcnTZYuAU2m9cAi1IpjlXRm8PRZy/HOAH6On9QONJnjsDQMc/PMjU7skvy75iW9iB0xDw3JNW1BUnDDAyd35qLEztLWDSEgkf8k0XDguPn8/FNdKYGMh146vWRWurb3hHVYHV6mQEBQAfKxGlPiDFD5vIxiZqMzlBNEfXkeb5oh7AVTCy/Ph5ur4vJ3u/NDcxr84JY32JFBue2s1nISykHMSYQPTXh2IHbk7V/wYDaEjsgx7ynQQd9lN/QFRLUB8Ciu/kdb5G7L8FeUdSZQRXsEkqjjH5Ip1BkX2knp99YPFQ65RX3+ohGf9Q7ibfRbDyyMKepm8j1N3hxHz/o63Y+S6Q/k1nyv72yH+fVuUiP3nV2UABX7T9bAT4QT53CUBuKZbPooY6FaHZbWdHEnv3mJfmYW7ytd8mvIguBwwUa4B9ArHPmdUOSpZykGwA1l2zXJbNyZc/tIvOqSESQFBKG8XmVbcrcdiAiRFogm9gLZhM/M7mqzsic+8Rz5oL3YncOLLfYGxY9UEgQek+n25HZY5d7+u+EgDzWh73qiFOnglpdNKViUjBJ8SvWx41YJsHmjfzxlY5IAlsS6eWJ7CwSLOLTi7dKSCLTatBWSQCWO1xrARVQvKxozU8pQbbOr+AdTCEIqa9ds0aKfn5f5CUAdhrh4KWQDjZEq4WDsUHhafVb7fi1NTCSB7uL2o3J26PB50PHVRd5PdT3BtFdOAPPEb/SLNr3ME3Jn5UNpZe3jvA4rEmKtp1HnQUJxX1Oyj52QsFzYQXXrCo2Yi5Xujhp+4p7D9RFCSBv5yI9Y6QKTRHONAAwSsxpTU1swGgP32B/C/BKplavVjJOJVYT8Bg8ZPuow1Sr9/n1rqj2NK5J2OvnDsFvvKGhpFeWBrQXIx02jBs3RVF7CPd2cyg2UZHyovzL1TMsP0KFRkDPA2BPM6mORaLdbQpXdipcby8nSXtQEWW8K3tm5Xrv0lkJWy6sdT7HyPvoMKmUWoeLQGbSXT6UhVRjM+5yEQepwqWsajyfms6HqTO7bz+z3Alh4im4/aLbemP/qaNsZlHgs798WLqd7N4flwiEDDWKfXpAnybCel7qa7cuC0XiKQZLrxIfSnGbcOunQWprFsW1yFWOuPoDUoWZRvri8e5/adCv5VghJ561ITV1krTmmQl3OXph+JxuQa09uGSEzzcZh3pzLg5SXTuS8RYrd6iAgx1zH3BhVmKKkniLwWVDd3uz9gTVSSZWmA0UryoxxtglOyxigzIlSpENgBzV7tyT+ze5D70BCaORAnKpgD0L65He0jOO1MDfu5Y8d9TqYzPXe1/h25yYPppL/SW1sHnXfiQuOq8rTn5hIZutcq2spammmfRfqimq4zphap2MPhaglKcoSMdTAW5VP07MU2aJeqT9jq+b5E7UihtRJ6xJWGVH3rXMykDnAAqsyGicix3QBiCrS63yWZ7vneZQ8LwXiPylkSnefaxOG9K3Wyg2Pz843Bc31LHdm9CKjmtNSqkubhhUxxrOu2apUqrN49TntWnRWdNXF5AKTWEm3D+AFGKFY4fYuT1EgaEV8r9Ja+tRAKuXm6gWf60MjMZyvgQHPDSAv066K71osKmDBFTffCxACTpEYYT3n5xMz6XD0Ca3nF6mb3GiaTurjYdMtpT+mFFRrtXNvfXLGXZUkjyOXnd4O9di8qD+tw+uveLN7NhU5Lsq5PH1D+Psxv4Nu8+robY4sCH7uhvLDtmTStVw18qVrS5TCSBAtqUZLiIwm5BLHyZosVmcyIr4flRMEAQDRSKxUNnojbd6n+5gTa7YQ0gB1Q696WK82Rhj0dfKxHuun1E/+eTykYI4dIegRDYRy0MRAG0B4pSaPbnUqSC0tHL5LlezK3GBb658b6r1hw+qAHUpxAq/rEoB0uMbO3dPlSdMUWy5MgJnWH6xzARLLJI8/2rXLp7tuz9j8sYImxiI63pDrdKxDVVpHsZ8MCtaPgFkRirCiF0wejfNqbSfscPrNjvf4g4xPU9CqPdpb8V0VaOA7tVhmZkspq2Sr8iVJEhobbeNSua9VuD7lWCurHCp04kd4FFDorjZioVVA7RKWZdHx9Prmsyk11laS0VAct5PlsnMn7nnXTHXjcTBe2GmfsWgkI8IQOzwcu+bNZ2ki1VnaTyZuzFXFjmmU4o9LJaYiJQikj8kSK3wuysVUu+NaJSRGMU4wpg35aWSCQcdD+ZiKccj9cQKGG30Rg0C3nI/zLpj59vHTqrkqsJLkTMyBNk9nlfTmUnezEiLqEwNDx6Xcus9KoG79Jy5VTOcKLN4PrC6womc/kYxMWqeqInspOPFJP7EkePlQL3A4m0hkTNe6+Q26r5ohvdpvDOs3TqoN4hJTJMejXzHTc4tPMzvXUeD4YcgVjLI20Jo3Mgowo0SVawiWdG/bTbNQVIre/It+jcF7artURmdO90+tE0fgxmDI+PoKLcECQsQbAS5I+0aQXy5vdM2ElkMzvi5qKrssIp43KgQ+0H1Xz10l41q6RbMGUzeCYcSVi7GTgOs4+Q5emrnsF8n2evXjVOCh4ZSeEi63gtx4Oy1L8JnGTPtUPcSwks3NLxaramouDzZ5fMRLu9jjSZ67U0MwDfkJVQblg5tkmzLU8DLa070kK/EZWd392kRyuFVIlhsE4YMdes+690jp5dZrT13SS2c/31aezsWhMGGyeSUDj2x5+loBDGx2Y21y4QC4czR1mrcZeVOvx47fxqQXh6ble1vUwBLL4TiFI4wVSHY5hks7bLW/rzYh5wsusfzYugKFW7TXzaIzPBgNlR+QE4gMQbHlUxpX1Zrh8pKMoHmPS2S9ZUDXnV8Du80jmFLlKizC20ohqJY5mkiZ86yTy69nViqgeCley3fijKRcIrBq89ILgIQK9gQhFrRnEloE8udn9+l1qkzy2RaL2EcwVHQyypeFIcG7he3avIjDTK2jNU1kjN+GV7K/HmPRXnseBHrx+D6jkYhXa41yRlF36x+B3aYUQ+4scoNGmS13SVDqA4jqu/M6wDTmWMJybMdzjus2d1pDZIKpOVNY5rtOtCQ2BC385xIZCcird2NZ5s9kADeqralbmduWDphndGSNFJNc3wWorsfK502zEAJJP7yi0yM6IZVr2uWq0cq+lOdpTCoUlQcG2I/zyG/L/EjRGGEWAR/k1aAy5MSBYO0g0UeY4Pn4WTZ92qkCatHvKtKUaoXjl5rLiPwbn+Te2xfF+otSwKFfglv1dwYBgM4nzvqic5oqPQ6wBv7mH9Bc9UXn4S1HUtHmFOEgGWrHCh5c/bwrByIJLeAD4kwidavN3XaRvV5ZNNYQQn782HTDfCRnRUF814jCYI+zLpBMu2JzPJHpIEfDU/aX5YLf2mzpkyc66PDosVDoUM+8LWowLdqy/RL07oIey/khbvtZBR7XoRmOxh/d5zMuPAADvrl0rcSAMSlQYTgtBhP5rpCifA5mPZ/15F/q+8ZWj92dqqeI1/vsgqHdjkarlk1mcmA9dbprkIN67AI0EcZHe7+UddTfAbKnomvIvWEirE/txYYt+7h50+tAWEgT8Qnn/eDpNzv+0XwCET8csmSQJ/qlx4d8ikIzd8seXR6MxIO/UUg8363fpQDVshitvQvLcw/QoX7iGXtWgRZ270b0e2tRghlpKBpBSvfNCiCJAHBghpVKMnS2kMt3rcUEloh5UYmXujA6fDwR+BmfFWgoj4qjS+b9TuTd2hCNDwp37wFinnYU7HrVlUt6bG4PIpt07esU1cSj/ogRawdlHTZEXqfRI7W1lyWWSZN7lCgkBHsUZghWW+zbyCKCzDBJKh7PKRRy3UdgHbjElv75qF+ckZsB3aB8EZHSfoto4PNRSHHcgx4X5abf+Mcu5nR5y0bJU6LkHsDq0xHJKumCIXmiEEYp8qLwn2p+V4gWLFuk3MqolaRj12i4ehw6haotHSDY42K/C/WiSL5uVTqx1K0vBXzFWnbs+IFqwsPTSQ2z98en42C8gxbxtepW71PCVS6L9hQX8ZbiVqY9RHhQn6cXKyTVF0a0kTwkzjiJ8cstTRu8QP83xr5qS3Yr2/Jr+l0Mj2LGEL+FMMQhhq9v7Ti2u6rv6OrysH3yZGYINqw154K5T+HNcwj8ppFU+h3+HCY18ZLuL3Yt/LyVhvB43eEWCPUBsKtGovSNfz7rjQ03CFF7z58MC2NjQW1E8aWuCo6m0lB5BGJ2NXxjpcnE5vyecDRAXcR4bXUvaHmFURFdQCvCzlXpPuTLmrtUtHjjPPpMflDEYrSb5qvxSxEgtuvfK4ieBjV4XbBT4/BXi/uNAK359WDx5Indv8oCVn7Ntoz1scibruhQuT0dVqiffcT1W7IESqHzVGTM/u+3ZWywvvCOVW/8Uu6Ymi+zCFDihXR5sK9ZZOS4vtDf3F2DfZfnlSnynxYu3iJJDgrrVFS5elWr1BLp+kxqwm3bX1hlwfvvxyilvVPYEYnHFQe/Q7YfHGxG4rsg708vhXg6czzqAG/yVo7s1xN3MFvIcpYsfoi4wfzn0YqUwuPVm033zfFFnHyqh4PvSCORiD1oH2mWm/3x0XIP128+3ymGp5awkZ0xC9Z9ZoKb0Yh5OllTs5OaiWUUZHHyZDHYAAZ6YqzhveHYC2fs4T0BDSIXDCeb6OZ85ZfkvijattROhMsUHQQ4poBzA71Cr25TIug087XGTqx/L6e720StWShNq+9ARojHECRjRDpT0aIf6EXeGv14dPdTNzJWF/d22G/ZgDUj2Etk+fCZCdWBSth1mH2aoKyVzKWvaUA7DLI5NpHp/Ot87lWdTH4tKM1gG+DLXAKPCKxLDy7EkeIH8PeASOscLX/S/vkNv6d37Q6AfvbfxGPzwx7LEgTGxGfLVqb6vEg6KRb+JiSG2EHGN3/N7ZKatPq9jORdovSmMgsBCpmHX1RMN6CGzvVu0IlS8tO9/FDgIAFwutzVSve8ydj2Y6BJgFvKiPqkP2KNwKfN+kMSFd2Odok00MhgzafKAT97fEGhfJlCUVLzIuQq+sFDGZA1RnyTmTaC9eH94/fGv/0eC8T9gN9z/offcyuX+yy5UB0l829RqV+9O6gLbX5HwFaXJchnV4lwXBqvg35cACSG+YYWaUDYw0wOH7mUEOv3SOjE5O+tHJfvDdOqp7p4NYKUQfWAmdpr6eW0hl+9h2QU2Mdtbl/mfxKQosviNjZJmP7u7gSktz4PsY5GmIANjNYZW28oWwqEbIDYyq1SsDGK2IX7ADbvwFkPI+fTCqPLHtR3sHhp44P7TZ7HGsUlmbBI4LSbjJNAOaGZLMc1baBhU5HYZT0GL2Fek2+tW+FQHfLynT+j/TaYPuNWcTHcDbWyDkuHQkV+HQ0Vh3xvYBTRTbUq5ltWy+3gGaKzXvZVkRoGPyNmMu5iCx2SHZu1CY5ePeFAQVISNbUrj6Ne10ENI4uF1spImThHw03jjiskm2IBA78JsDFxX9z1Cxk6D3TTHmryTf3X6w2z9XHyIrWj2XRp1FlftnvKOda7m4Wr2YLwgr70Whp/DOZmLIdyziRa5cQQHOWrzfezD/HomSNJ03kdSucx6mRj+bOeABkwOJz3Fq6i673u/FuV9Y3XwlR8cw22Cr5PBM2+RY/tm4eQh1xSt91uvpNimvD+BZXb451ahdhu4VYFWGzF127gt04HQoGQjw19vK/QjpGBZ6xsGvfY8VqfOY2JUXovEMr8BoNcwkC2lLWJrgE7FwDokYA/6wYG1yQJw+9ZjFtd+U0SaRVvv9ZlJZR/sfeZsN0FZDDKByb+1teldbqWCod7KLVbfdYPqbUQ0KdkHZGDeG7uAqZv5FMYEWIe/bttaipMDFSEVcS/sABaLQ0CuiJR5AWrlolu69WnlK25614Ra6i9+rC9KjW7z1vimJMRYFAKUD9ww2c3f/IG+WV+8FqUeJGJQd3l+B6RHa8MLcYeMxb1TL/W8mWIfEyWMOuAanPRZzSz9N8i7wITte1y6bsj2ywYi0nU5/ig+py4wnTKcRM3kKs2YgiphRjpbxhCgsAPlqJnt/TlOpmmDNrZLkWTAXOWy3Z+0Qp72L4qwlueRRmHRDeUBC84tLUP9RiQGRkMMkI7a1M/6HrBTR4YaSTtQ4aN3Wmm3g3FPpCQe9yE+RoJcyfBsVbdiLXZzJGUx7XWMKFU43NfGcj1FuflHmHhw+xR0MgRudCBLchalay3obI9sQScnftnhKwcvuu2KgUrOETQqy7m32pfWe4jwWgndavujvaox56fBZqI5FJ+rgpW+ezkI4F866Vh1gt5Wukfi0yeWeTdDDItfj/KhYqvRshTh55EeM2BXgJFJSxqjOmr/za7QG1I2RjnKW3vOjzo8xsrxB//i7gQ5pQ3NzxWKqVncGyWeHAOODJGJH/qxqs+TzsI24T2pJUOLOl1D/faPIiZEr1NEU7NTDfwLWZL8xabayPNdrgOT/LweAoH5JTTCE/QdnGTiTsMHzl0j/9Vq4rZPkw/bKINxZD1cPoXW2OMt/Xs6gpuGyYHy+QxYx7YxQf/O/3SYTQWaMYB6vTY9sMQmb9ZkyswClAACcVTUCRXMeXDeJjQ8a8sCMjQ/lgQxLw4HUSoljqvsx7HB0KoWCcY748/C2/jrGHe4b4XEw00PpLBz5vxccU8hFE4tgQxQyuOP8LMjdebCJU1Y+sBOz5rVIvvw1H5vK5r1NgJWjDH7Y0eETPPvtu/Fnekqrm/YqPVC8dUD0+AG5hXWEUjPu820fmf1PlD2FJrdNg1oGJYJgzCOmUqFGcK+HqMz+32Qb2UQv/SxA9u5wDYARzMvU8UzJ3Vfv46ZY5fDQrzGNVdRv6lHv1AKylXhLA6ndcmZmtJGqmoSszRc3WdwbsEI18UpRHyIvWNHLvPY9Ygjto/reBYQlthAi/y76nr6xhdmIoEpXJZ1PPmyqzPstLWEaYxyPqV0EXMdnzT6i3Om/LRSVHoPxifyfF6Ic0hc+6725i6auqCUcC7Eoa8umE9uQxjHmzsr1XKamai91RB0ytCnhwEDKZWvsJ7xOQuaiSE/qmv/EkL8mKbtOpXT0V34fiao18wo4nCh3WwiBx55+4yHnvn/NhfbwRZM4ebfHP71YG/qjPl1pfuPahJle/vL3UoKx/zfdJ5rOnnSEjAOrCnNAoMxoOqofdG9HngQA0egh78+YRFxzm1lhUCPj0wdTG4aXDG6YQYQZFdHIFe7mRORyCtbYgprEu+KGG+pJ4ntzgOmBCuiI+2OPWhx+yH5sERoMnmc5rbU2V9Ll/araYsiLVrdwH46VIjPtIAKvRsbOtpXZiQljByxboWc/tahfPwDo+JYCGJ62VSUSSzl6ADYJvPlsrIqksLK/vaRC162wrTHrXu7IUKG3POs4d6dw4b6eu3QppfsYZ8JGakZKsRiAKg/or/9Ex5j74aUh9/TClD4MB6E5SbebVevgmeOEr6rx9HP58PxyyycJIS15ygEDl84ez1OUySq1ug5M5OU/E67ocihPlCnE6yD6oWK6NSc0bsMOxoYw+WX5swJmsRKhluUY98Y0cZh6kOZ0XzHsgMYNPWJtlqjUefhi12Kj4gXgrJZBzz4CvTiBTPXx5bSY3NITFppwmTfwRZqj50xnM+KqPifsvuvTK8ZqLQEo2DVdBk8SFp/PYf52D1qBnQp5jbX5MVuEkU9VR2ObioGR8IDTC24Q9qYLsB1f8yq1rlfxlMgjEEX/QtT1MobY7JdXaqYmxLeMnO0TA817eyXu2QJkIgMS3dpjn7fiWgSnJh3xHSRxxdluHxnM1bZNNF8yRMc42IDbUpFj5ze/lUxg9XNjAQdviVNWWSO21kWjUCiSk7FztRwOCPy/0KH7zncgjJj9pwqZ49GgwaRFll0veYnlTh3C3rjN+FR5JfpYMx1/4RlgrALZw98P6Darvap7hE4N6712Xp8wIlWiwFV1XWiW+83fHNZZaKsr6uNeYsVSUexA6x3xhkLUap/3KSo3B+l/vyAyRz6fSuKbomzO40PuByr22ZqNVAxM80xWJdhoeiZ7g7nLEZRofLiDHPqtxHSBVFP6pQCThf1vhm6k9wB/bXvxzzEzsGGbOxIdOchVRVSRbUDaFTdiy7h1bp92oXsZJU7ybjW8n5Vs1ndzwlb5z9pCUxAm0vnzUJl26oPYxhSKffp47qtYa5/PgNEZ/TtRYbkzOApujqMvgKMvOPnDNQ47HOW2Ih3UmZ9KN87nqBYlPWj/pl0Uona5ssHMBIIo/t/VBt1Ru+IHJez2OXeAZaLJ/agxEYlg2Px3jwMHfHh6LXIXEKWI+4jl0FQjp17vd0ls6FGx9hiKiSakdYMtjHCXqkfPlr9AsSMWy0sb4mS6yqDMX2xWOmOreOikWJZt5BqdjrAVdSO7mQmmDldRTydsovPl8cF48J5YsHYCiCDKvn7sF9sbSLdYEpdV9r7Uv3K1NsMMRRxmcrCM8ko+ZRYD+lm9bgI3ZXBf5gR94FnklVxaLiMMd36Euni6FGFaXqezPK+AdE0rGB5rXqbQRPo3jCCkJjCYLnN1+nl3gKJywAg7ifaxMtMblevqeTI8YXRhjZS6QpIqZyVab54Sa+tsB/uwIuXIXWvXHBZR4jO0lUtj7Wlkc4AeGLxU3R/EKpzyynHtKz3asu+FBVbL3lqR6ksiRH3aXhM7jyVigH2noNfUJtzRM+BCr1RXSAw9qvwg04gW7NQBFkjSEjgSYkwt9TmGFsVR52azIsuu4afpbDpYBqGse4gkpdSkPWRAbP43YEI8TCrrNS+953hB8uhWxJcNUGCpKOUEyMIskjmbXCiy+oppsQWmYTwzVEKnbu9leSnTYzE5nzmFbSxZj8uFz1pTnF6W+o0AJoDJn2ME954LgTTxhMUE7hLeqq40kLIy1c8UIrjqpk26bvhCdh8IapKAA2pd2Z498cVP+Kn1LF1tjJxTaV6O16LB8L4E0ib5wzbnaa70bXa8YVjlgEi+GknKEq49m2od88hK4zRtUEvgKCSlvS5lA4CHtkjJpRFHCGKCtYmB5AuiiajMHn+m7nIvkV1pAzyWX+VPXoOh+kq5iuZY4AshKCvOyYDulRq3mf7xq2mlsVuTheYwKu+mWSGD8442qrzAREU9YA4phsesxchZ2Kd15jnaUj67YOBYpUNledan+EfBfDFaXFjn4UCoFVK+7xeh95aUvRJx5eCko2QspwFj5glI4LKg3Us0XpoPC1ymZfAbgciurTkuPwyfhaq+cURDK7b1CfNzZy1SovXHd8oaq+XajsDafEEed8zB6ZRKTG3+d3LPcIZrPHfGZuVmTq2ZeSKCkkilDx8rhj8WpKzpcqDrohIjy6zRsWhRUVS5K8AGpjj1ZChv34sfZGX+fgDutOt/jFW6GurqnLUDQPjJckn6nT0QAe4zN2dWfD2pVg0iea6a7glu/TPmRIZszBj8AUifoqadZk7DSelHsHhjh1JwmCRG3VHYsxbmQKK5QEaCNxuId8sIKqICtUZb9z1TEAcyHR5ucVc3qG55PKVh/LiOIf7DGInCoUsqyYjhlkGZ25JZQQcmm+D6U/2CUQjUzmlJ58OOpzJfxxd8e5ubf+DmT6r6xP8r9QNn0vBdDT5uqAtdwD0qRqBFzBfPkfwQcigxH4H69wANgzfK3Y3+HHJiLz5UIKMy9YRvwYRne8xO5+vtCFg2G402AZ9fGwIBDF5t9A/EChABtWb+7p6x99ZyyN/mIn/IlZw2fNJLjLJaEqJHhJB4MoeKj+S+cYyIkSf/SaqVr5S785l5bHV3xMn2P5FFU7hfc3k8MO5Y+y7t+6ykD79p9rZr3b2y/1Hy3lHM1Rfchuvaev5KJOy2tx/WYu/VYuPXo+X8N3EeJQHFXr8/nm72v/y/X/b93mnxpvHKp7Hjm0Uiv//P7f//3zfs87eD5EK/+V/jW9ps+7AzXif7veP2rUJuT3wb+92//QiUY/n4yjTr1h9gx28Uzy959KMxpcMRK8kjBuFKCd/G/P+3/0sZ/5+i+f9T89p/tN+vg/XucZxz6p/zznv473nzF/1okAm1brds+YI0noSllPr8qfuTjiSH3w1+/dnrXx06eGs97s/u/r/LnWnzGz+m7LUPeTPr/38kEiCsyM3nffhB8vV1Agw3dus3luy8en4fuI4zmQ6ceH5ce3eTun0VSY6x37cxUwisQ7xO9cEp9RD1SX/093fmYHDR4QhUP/emf33+98//d3fsaxyUO4Swf3f9zZ5mig/Gx6Hliz6jfvg9Yd1D31/p9j/F88neX9/5/ur5VOuP/p6f65638zGxb/39/19R9m43dX/stnffDJJfoKJHpP+X+0xh+QSkMpao4pylQOZFRGw5zmixm9UGyeWf79TPvrXlZrXkkoPt9T/RShF/vf9xL1W9fN137WLRiBz89eNNj5zAv0DpPeaunrHQZAIf6nh/4vn/9bx/vfbc3fdu4vBXCgTm//NL//Rd/+mdsIoAkaxIISu1NdQfQLc15Lp89kMfzh0KMAgYISQZXI8IQHqz9ITb4+abv1W/9tHVG+tI2TxlroerUXUTzuWNoCedkKHAgrBuJ51NhI6i1dJKLhizDjCNlnYVLNaTzUsz2Q3KBomPT26NtvU9dt5PdVkL/o/oi21GnkZbROgJ6A/hsc3UutaI4DQlGZa+4shfoRx3PrDOgiy+UExayUIJBI53mvzHrObCKnidN7wxNKHq8Ni2aYAqlCIvgF7YwInVOiBnG8wAFhO2RTwI+z1O4/n3XbTNnyva3OqiiMITt3McrMB7rLdZ5kd4B0yVn+kBMKmpnzBvMNej9HkMNb0z+HqB1vzZlOcgWvIEkSQmBGvlfe/DKYMmWBmBnLMzb8YnYzRyBy7nNcO29k6EZ6/V5IjA6ndt8efe/N6Nt5stQ1aNxlTQat/oxCmol0PDSs4nhZScOgzWfUv817MWtQfWbyZWAIX9b2K9ALf6FHTtSEwaN++msLFFBCGMtr98yKbNGIByP3fjPVQ/VveFdqwCfphLzhT1WZCPhbyKJ1SpRNU/eEVJDmdE/DktQ/AXX0juGBLNdxQRislcvfPLwVexwyGaO/EUm2RlXmgQUuhNHtnVzN4SGQKdx1CHK32xHhHGbf2/KawWhh2Xup9gUkIxMCfKizK+Tuav2OJXIRXYcDpVEmLcCnudgujpbgjplhmWJFfJL2LYvnKUJlxmPPNozxX+wPDPq+ro1A3GhFBZBPYnGaztVZ0BLJlFnq0Q0KvVwg5jr4r1N9EHPe4m+bJDsK2x2xuV96rZr2tv3mFJIsBt8+WEryrVXaaz4dB+1Dh0opbxRFX9qIzM8zGaaJJB3HRzeYJB/0c0Hz6yR9ZElGAhz3If4E+pFBmr59Iqnj4YiEEcm/Ucw0O4G8BSTqzE8FQ6JwIXc9TgGnVqz9Cub6ebTUu81pAFwP00zMfLDoMcJRZXcQwl7GES3o8zXMkydJPjSh3LKxj+pC2kbMr+RSSqRnxOM49dyTuw/2OLGC3DG5ffepW/xMgt+usWzyP5qPn9AXRGtkkOn4esbw5x3xMTk8m/GIM4MEw6VHpPLb942B5s5nyMk2QMxCwmMFuggtpwzwlCO/cHrRv8fiFFiSIsg5pw0j3BiZDMoaX5YHKX9SJYCHwVGD+88MJyNDKn3TH9nzBhDGbIAhzG9+v5lfiFVO3hNRThPShthHW54Zbj7uFDMz9tAOG6VvVbSNENQJo+9X8+Y95LQte95O4hvv9Z+9ULwN/RXo9FF2qRznrGH3KCeaA3GAhFPWfG90pZy/d05xAYvJAgoDULwoYFWzo6lM+LC+GITj+ZjxojQYJ9E+/Lg5rsR/nqqbGJz44LnWzJbzosabkSWMGtrX1Y2yo6VCgoAawokkqmDwFuijYXLGbOFv9YWHu1t243YeVMxM5zSs0xzBWKPlBYI67pqKYqAp6pZVtX8SmnyYqb/2pg07Kb1zc4iF3IJ9a2CzyQ07wKqJ0ZR5bYOtYm8CJb+uyD+0YuYuZEtYa4g53Cg8LaQ03ViWhkHRnUkjqMfWX8ZYYUj2PE1vuS69Ipv4sVNsE2xVwvMKyM8e1TvBaFAgFG1fxkNJmEdDxzofP+TQbUoW79fjNufOFuc0waZ5TjsULL/CS7791xAF/GbgvXlvnMHJMAVHZI5B9h3doO5j2pd4nGtLJUvDxHlAwwPBq461UJSh2VIGbEXTRFNeCCqf0S5pq+kZEeSXu388uLlA9pHIQNOS2NMDieb6h/UyD14WrWaIRjZdJwGT22yV0319LPwWF6xMBR28qIp33kn5aQ4a3ePy1xiff510CuYdVDZlIYbpG6QRxQL+urclYe04cQyLnNUk2HnLmbKPg6MTj2AhoAOFDmtyVGsAiv+Z7/qHqOkGASoLV5emFx9f6xCyMr6kFLxyPtpW/pwApFNIFyVJfOnQ/h5X1fyYDUOiMhPz5jxtoEB1wIBf6l5uXqinc5Di1uCOT7OC5wF04Hwcby8PkuapVEtz6NVauAkfB4aEzeyzh11QtcCOGmofTcAs7qYoCmdvZec7mIPD+0m4OpdPNSnMEWPqcRB9+q+AyJ1A5PV9bASN3OsKMgGidIbiAaIkAwlCpiIf4Xer7SfN8Hs7Vipa9mQXDtP21SpQFBosXukwkzaBbZFn4WO3/GR+3ExHOJbMK9/2QHme+0xfV4bHUowiv0XxhqGR8tpE6UFFEVd8jz2KV4azWeciK+lL2d2oSMMwTl+y4cBhrKLVkNmCAXmbdBE5FuLldPRKkOlwQl3j7rivoWG4faTRF6R26TQeiiwqCuC7oOhiAthh3cLcUeTIH1AZQM92DMyRNlKxPmIBrIIGrdDjDBYUxLBftHB87Ghtr/GL3GzAoFxHkB9JmAhl+6y4h2xx3ovOAElizAgPEL2f0f+MEUTstEe2/FWd1pjml5wQ3gN8vp3B6ThW67BmMsNkDn5r70SxzWRL3gBHSSA9dmi70gv8FldpEFfmm+7UxMnt+4P98eZRw3Har0R+H1Oc5l59UqToUnAGL/8aWJEvO621qPCdVonE2pblSE054hSPkUnVARpAT4uXZSsjka1E8t4h+xymdlvveczQ3l5Em2x7lk747NhMLsrf+Lkas6PDqMXUh7FHWD0cnY5ZM+AVA1PzaVEDRSJYzf4d21pAd1rRkvKxFmGC+RZl2MdI2ykbBlL2eYXcZfPcgSVScSavPMeNBG0amv8yzx7T1kk0y81zEq+IZAvWMnxPValY//GHK1sNEagL0jKX0dDBHFDRrc2F3Ri41L+g/WLhMVImby0+t6iBMuuhJxgW4eq87z32iTYZpUql/8n81bYE2oFuxCYUChX26Hr5E8jZVrEvJBoAQva+evqinDc1+dH3ZiJE4uVXgxcC2gAgV5LnsJGD/fNRljeJ1OeTIhEktFOYeeo5sdqBhMK3KUaVCd7q9Mor7bDAXnOUblmPIVKVZ+yCe8Ue9pJyiJi8M0Ye4Hazl2BVPlTFOqiNPbtpwvJV2cbo9ZZ6PyvJFe/G3WPhZ6o72e7FdndRuSGvAUWz9K+5857N35SJvOE4dKQxz7/GkZGZhVYfkiBUn2OKgQe19HKAazrWImijV1RWJw8xxIA3vI7RjJXmznrSbwPrADq/CdYcGuhD8vPBKBwCpC5Z7pPhPxV4f2OiFgSUw/cbQuU/3pv5ZcxbrbtoxU6/RiczsLxZXGYQUb6zdFawJUswvuC1GbJ4D7yEmNiJ+DYUrePNWArz6u79XP0P0vvHS8Gtle4+4me6NVOtie13+hVAWUTXzIHu+CDYzs5x8u0EaR1/5U6ggIEvfed3+hCS+3qx2uYn3AD4d8WvyjnYt+K+FOn7sgQZTUugRSIfGR7BADNl1RL/MJKUPatCjljhEuHE2Nh4MjRpZODXC3TfiKTfQIJFpT3PmRn50DlbdUQsHNW4d0nrWbODvNBzXvxsZapr6mgej/H46ErzaUOWeZ3eM2Rxhr0fXgUFlVkyHnnjyASdk8bnuM6I3mLnAcVIR3apb2gfAUd6mEjWvd0zkaIIWkXM84iHozclfaiJ8Dmbu8nzDdtcnw/ad8QAo1pCfdcbIfJQiGGZxkPqX6C1YkE0NWiPyHgo1gm8TM+78Orgz9wBv16ZIHM/+R6ElmBIbcSa8ZuAxRt2Fs1S8fbWP0h9iRdZB48zmOOo694rnmdV0kzkFMKEdf1ZF9n1DbMlzfX7CuFwX3MFl6bX4L8Ry/zQFTIbvltAH/WNpWVEfSn21asWtFDS8PrjgOOqdr8mydh2UTcFRJgwLUxTNUXicd76DtBxg6XDwfzhaITD/NKaMbd86cVOAzYabidQs/COlQO1LXp9P59O5KjLqQHOyJnRX6quhB50YSKV9O5ZK0ysMASA2rOb70xfdFQkvIC0ACBedHmgGDRekvaBAxWF42kk0Wx4HBFAiLwF+I4MoXg7DJ/v6ij0JiFOStnfIzK/AOqcRE7cq+np1RtpA1WLDUl+hnOtYawXSRTU7zikfXF6heY2b+Al2zBFlEY7ROVJ4L5I+s3S8uFLNLE7DBPtayAMHv8HqT9e3Oby2JyWTh6PLnwX8WmtJ718z91zziIMa82Cc3RHa6PMs7veNDNLPv5iKW8vz7p8XCDubBLm2Z7ASi5+voC3lj+/MtV1Mr4U7NAUjQ4s7gSDEA1Lt661Hc6hzVNnQVwm9NtDz0zEKHlL0pCyKBQxiyipqmlMPGyiYx1ce1NPgg7FLRsdfQ2reuyvUfBibux06cdMPlDU316qy+3toPQDMThLH8un9BAYejnCiv5p2Vu+EGlOiiJm1qTlLbXRUZAW4Z3Z8U1WsLbR3t3uMDQuaFqNYLa8bzbEWcUikfqhebQwAIF5AB1cJ4pszCUqBOoDunZgT7AvdEmdtREvZ94rTRERpvA533rnB4P7t6HSWf/CkpMLjyhbSkpvxi3m2aB5jYrzVYh9k3zCF/FPq7q663jQMDUdWn/a31q13gxIOPslb0OG0BxxrKlw9MCS5tnZ2q2u5ZaEev5VmN1MH6SLIrJS3sGzaCwtWsoM3Zz8qPsowTm+FVTeUo1hJj+dT6MbOKaIZXU4UQ1p6aCAjxCIoZegSxQfk7WRJ9ieuLz8ph/XNV2/CAfTSDBXrhNTnOVSt1W4VqP3OIUoQqMksjVfKcYlZN6Y8P4WLZuqTOWblUff2Ep/vpBbS0zOmGSmAO+96d97zU7n6+nATrN8Aykzyb1q8TN8qY8rgmRiMz4PFxQ7TKaFBie3xuaOspsNXELKpm7nnoM1fQsMzXXeZs02eb0eSuwX7y+nwO2VHqg+qUxkebQff8+Hej1gDU8+4fc7MZ/HmlcauZa8hb0hrwWJIRfNJDzs6u+GCeXDFuWKLuRXnX+wwF6zN7lrLUiSvabtANg3er2Styck+6tIulQKHHoSiXc51mNwW0toAP/bxl/CiDvT1WK3LXbhqnxa1ICQTt3yA3Qz7gjcL1qHJYEPyv1OBPvsJNXjxPP9Hqk6iNWDV0Vhll7qBpAfSy2wfPMR76FEbJfqH8alXshqFCA/GO00MYbfIEuXD1sPO0Podm+YZ5Fh1p9iIzE38ovc0V9T1wZ8h8qk8NuR4dp65oyJ3rArj6UqMgRCCY6+OF0IOh7mtxoML2f46RosniwKZUHlGjla3mejCapKI4Si4pyWBQJ0Uw8/DYJwyRqfF0dV0pjSA7G64/Orjnj+bXW57W6as74Yi9j2lYmEm81g64qj9Iu1mKcDsprIyOBOfSi3z4a+LF4syxnvFKo8UoOCxuMZrcxbOO8fwztF/V0sZNdXWhIvby1+c/Ra0GEdWbLjsJVMJGdfgXdoPo/hArFDomze4jAO7P0CEZ7p1/xJnYLoc8bosfNkfUMGzzLBY3cm+JFrrl6Z1/pt1GLrRW81wQtd24D1lCFs3cV+k1u4fHSX9jCcFQf2nA060UmM8uW1BifdMY/7Cy5jKo53c/8dlcT9yp7f2vrS5zc4KlQ89QM6XGknOAGTYsbymby0w3yi5o4KzTD39ODWUpeUAPJ2AtdH9eS2RSRM1G+t0rObMU4BEwSmaWNB/Ww1Jl5vBkRuNEt0Lj14oAv3Hq8zyGHZZ+VXAWXyRsfuIh2oN2b+NzZtHE9gH5Po9LW8E7qJDCUQv1KVwKKIIqADt4LmboJjJZtZcYmD3C4N38hJqWZt+4Hel5WjIAaZTQcXE68UJFAFqL66qySww/C/AaWXn8JVpiqqrK+wOY2yTucceFlLt5MsO0wotqbdWufrrXQIjiOd3LNtUBY34aHZOMxct+nWEaXbNTogginOGYjonF/MIitvwcIqSsoRtIOKtShgbMajVMTocw3d4bgStxilDLnwy+8kzl6XsxefnIN4BQjE8fCgd4yndINgm+ZG8MUS/I4WGVNRsgJxJROJTHE8XgqWEcn+F0GQfcmfMMY5BK7KvIrFsXL81a9KjV4QbOQj1Gs6f4qJMTYT3VaJEudshiarGE1yrGPrF5QE+QnN/tC++gdlKXKiav2W72E/dm/bcS/EkmAvPvRg8SFptYzCkqQLn2BaaqGosunhsg8bBFJc1WCaCuXHV0X3frBqw5pjb6sbaTn0z9b4kOv+wUKmDbVt0VDcNFF/3q1tRFaCPymVNxwJOzelD50Fdh+CBsJA70pK7j/+iq90sATteavwRqpN8lCdoOzoXv5+jXeTLsPRpml5xr3FNPz4PuLuTB+kyhVKaIWmDC1pNTfw99L0T8BdFdIyC8ghKlgpoLiyAjfcuQMBoof1yWLGcwOewfwrLjaIijHRMZiwj6CpmYuVDSt5irudMRcZBbaWKH9CtVVEvB5LAaIyAUN8hvChz76SNm9ysj6gJZTVOabx0tFw7e1LziIt2uC7reffC/PgljuuKDlfZSwlbSw7ZOeuqBN7o5N9VyzMvJecFHweeUU0ikcQHR2YFhLGwvgORaoIvg58J4qVwqdZdtavI1jt1L2z6QGVpV8SkzfZP6tnMUUKcgIqWC6Yz5jBQRq/hE5fdRNbOBITrctsZ5xkee8gkJEHObBXniStI9haztjRDZ6bByWnpU4PkKXCCrzo3pV+BQOUBIMwng/GhBrJTC75xoOpN9O2ZoJhnHeRRLC8bUwzQhgizkAimMhvX4483fxqj5oAhU5YGp4eA1vdSID3JHRu1wRSmN1M9gtX9RC5OdhP6vhX339GEVY3jqQSyVdL+/K8S+CP2Ju2h3wxTBO+IbQSLOlEpIa7disRyZffvmu6+qSS6azYlM3NOlblICHatOHFGtHk6ayE10BujxkFJvkAhn/OVGagiuwu6L0PHWNtmd/3c0GU03fbng3S1LzuvT21Ur442WmhtKdutmcVRCagCebLvQfYYPkZDc+U0SNwTZCZel9HA+bg7URonlb65CdufA3EhFV4ktuwD2LPjV9XzKuqjKxkQgoIyrB9jceOPfmWcVQ2lNiAgQiZmc3wQaN2FEnm4yeGB1et7+Z1zDGFCjSsC6CcJQh0Y5OJqaC5WLynqxAqQqGIlA5Xv8Chnz1XrSKG4DcdCy42R1xA7yiIxe23/blr2VgZt65sF2s/8I2HxvGYviHzEtnK7AymbT9D2M2Y6Wm30COHZnXLjwDeX5ZLQyX+SYpPplxpv7jwB79eCe+6IU7Z0I1vpI433YewGK5J6XIkxaSJmF8eMWk/513APFpubIkU2repnzejDejlztOrhbJi3XmoCBnOPkgliVDI7qO3itbrYSvtt2/Yh6T3h2cW2M4hNE2U8tROo4WKG1wgHz1j6BK/Hlok78vYhvow8ILgl1Rx7lYM1CSXZsI4qhGiS/r450vUQ69ZaY4YG0mGZSYgUBIE5HHlzlgpLsovE/qwGJUfVPJIiMZBNJDmOOa8MPLuIc8oayBEj20hBeqNfhLJwJF0YHFb5fdNl3cRUp0zQdJAME2zbxvgyG5Ga/i9hDjJD/dPTRzEVEHAN57RYQcOHpQEgmRYCn7ytqwQbTK06lh7rDPCu2TIDsrn/kMWEQQ9rveWghjo12BSdogJmJAhaSudP8EN1rl/J598f0vM/snKzQRIb5V4YQUJOdCed4RbMuwbaKGrS7K3eTvayofr//K6kQSe5WQPAYTHVGJ37TEn6Rr9/IAjm4zDYKbckavvHdzjdwTuuuqGTeFDbd9h0ti/Im2lASYBIAOPlkDofwF80RtCZq/TBt4wkxZGm26fpYcuJJNr2Cdsiuncvpb5ymz+AaUvOFU6VMOY4TGUhWdZ7+XTB78mjDUI1wBPJf7goSoDdwKvzdvgznhigbR1Znr2WoC89sv9HSf7VmseM2508ZA385brrqOfCePvX/vBYusshEHf6wbTQKHRb5Zl5hWqMdujhPaDqyJWcXAbh8J8FFUB2w18gV7on38tRZp8KZHKtNuaMi5gbh9FneSUFmF0Knb0vHG7MUuLqQ4U3uz5K99CU5XelpOFBi5OeF7/mbHyCJW4BEN27RGw/ygIYph/HsCQb3v7Ham7wnzO8hQ1lXWZN6UWX+QZ7dVBEoN1wbgOnSUOrINn36bjZDnQcWfNisHY8zMj8u8UZC5V2mjnm+vzk5gB0Qui3mb7W2k9DpWuS24JytGP5T0s4H+tASySMq8cpY+GrPeWubAvEC2na+Lc9sVi3IMDzwypjoQ3B4f2BQmDBWijQlsH9P3jOotY6GU9TKWORNQBY5h+Y1zHC3FJ7DC+c8HxMMVs2T+lOgqW073jRiTyAKygjYUJrmJyXTx08cIyiyagsIIzrZzLc8uJE7/1Z3W3Q2vzyhfz3BvolJAgSfOZZ6SvWLum4hSXZxOsjYSwsVFK0tXBduNT2dhsf4ALaDYQB4t3SqSL9xTXRUAR2M3BstXjXsR/aPDSgBIJD5YHJx+Lm+JuyYORMl50HfdtXpEgezVhsSZ0UKPDcmrVLv2z7I3BdtAXg15aD2pN5QAU6PeEBL2+nbNO91kgatc5aakS9ads5GNb8beOoR5O0BThvdm54hQOHS2fdjgWzY1DsedPomlfeoVH6Fs5jpnx1ep4I6FHhV9R7YHtCjh3HaR1WgIwMC/qc0cXGZkaMJ94hy92GpsMWDNzP9EwAdZBgakeUOSH3ytzBLwRLyuTbUFB73mTOkZnrEQfsC6EPcQvSxUylAL0uMDCOvW+RY+1fV+WRc0s5A8B0EcQz/ZAyPS7ZMCUYKgFWyooOqZ3KEHtxH+2HVzuZWGwWnRd6bjWWXjUxfj5uheK7M18Fc2SchNV9AJWVDSVbMkRgDw7dO1jYAPb2hZ5fRlZz1l93RQm+w1ZNxfBjKVAgYfyRxBykBnLtNjvVTedZ9LnlZjLZGsU/1V1Hlsmf081x0K/CENZD4EpuwiKnpmLnMVPM9HfaZ2PNb1ZyfK90J95KLT7LtVDv1XgHIISqnnmJvZ2uxEVAiTztTmyWeVFk10YgKSxh2LBQCQPSEcIit8MmVTeJOkDcjnzZ4xbW6FNfGspO9apm7JPTDT8NKwAV6xEDECLkrKH+teWoOasjtoBrdtRu5bNT+lmP2JSX/vTdM/EaGpj3gDyNbzPBYK/EnKOWf9mNe6Aq3AwAFYk1xcvigLtptO+N3CrNam5LBMu6aG02bWdtktxujkObNFDi8sv7EdX8DljW9LUVJbrilZ+HPMoDvSV2L/xjqalhk5WLJicvl27wC3+QipjupJsKV4QC6m5wVks0bqWD8IG6apIxvxwORIuz0rcTD8UfByELr4d21B8TrK6QXsPMz++URTTmy8Di/g742x3954ArtebEahiClGHB8Esf6rQ/tB9H0b4Ah6kqt8UTU3bFrDMmnTUbImdzEsyp58eEPB7fuVIdoD6ygY9+gFr3nngkl4+noNB7GfZAoUJQpVIjMZgQY649iVbwjuVs04GsxRKx9pDyPxVsq7obVgFkiswqXCU5RLd7d/Q8iCW/Lhv+7vHiG7zwTYmFuReAFgMTZHIPmlNz7wwl12hr3oAiwZPv823p+NYY1StDh8bOIfzvNPhpJHnD/GXTGXwHTVZtr1/3abUY2f7lenxhHcPFolNq+ywvjJn4IBr8RKTiOVdYWDp5ZAcKJfJ3g18lyLIKid5k5ozS9VcdO0+9eT0pm9X70jhVpE/rKh0gDm/omhhWwfQ5mr25pnE61BWGx3Piwj/nTnxxcIJ/t6aV1WkaJtJGAXeHgcOef7a/4/l6yMST3vgaqooUUw5fMiKEueyesisSGnoXblweL95oWZQhH+Z+UHnnpT2bgcyEqv0O3gJfpNlhU0tk2rrr6hBCMzYQtKXSoMMD5RfANQcG4/E0ywzEbNngtx+4iXk69EJO67E5HbgtYo2VIpI/PjxWqLcXwBNscEVEasrAVOPwAuUPTP+gRjrzSv4ph+IxiyCCkn+y/o6gnqvFcfoqvDdayWCuCrugVGwCzKfHt91QAWI90y1/g5S117LLsTvi8F71dDYVoLu6IP6HWZvxlIqDULkN8oerF3KY3l77WsqYWDLfkJIZs5uD56H7C8h0lBdnthCN/sHx16jJueuOxJqMI8/F9jCf1z8acEtpk7vV5R/YRhZsjZXfkKbhXBnnIjfKKO8QHmdfAB8mmqqDpa66unlS4CPTZNU18SCb1V47PbnkoD3ryyXWzcGfPTls5lBfDSeGJWGgAuUJgYy63V5vM7QOd6EefMWfDIKJavWxEbzw6DdYmSHr4K2F7Ma6TM06A6BWRO/INESQbtIamw6bkrMUGdZhQRqId6SXCB8wu4zKbsB89bTDe8Ux2VkgOlSEzUdMPHy6U9Md4jjBDMz77AUY5xKr6QgZEdmPpdSLsPclhSO5vzGWPfxJC2yMSeS5uAFwQr7JRN59KYsgA6uwz2/22DS9o5Q4Rz542v6An8WjFMxnKI3MqHAMKUPgWnvlnDjsilz9OFFYOZr+1Ca+sA/YLX7XfSLPb9WxG4PsvOas3zl41dIcfoKdHZPeWVRmIhkoAHUm1k4lc6oBKbfSyicBCav9dNDKjLD9Q9/urY+sPJ6E5vUGz7SubRpWejsIoIHweJIdVVtzbNA9wfaX1gQCDi3qL7cItVgzpXSI7QwIDr6w1ouTW/BTdK4z3tkTDHwRjBpHYtiVdUA6maiIzhyVNAV1zse85PIcHyfLaPoPUfSiybeGE0tscTsCDi2F6yVmb6zUQqgUwCzWw5GJq/7te+Y/AmS6hcYVLRvrn79hqVhKxSWH6NxydVXCO5I7OG8I4Y7g4dAApwffwkepj5kfjHvuUqPrnwBLHtKrrBrWSSO4G1+DUs/xjO2XRUmdv6hrX7B0oygSx1tmw8oi9jR9zfO1jLUmmdtcoY9p3d0FVFer4Q3CuWvF9uCRRjspw8qEm9avKNniT/fn8J2qjIK7qHDDihb525g701hZfWvvVTzBvJFBiLdQFw8ILcp5VmviQKdRFdwuavxv3WvM6dvlGaBN61pvdaV4Qo1so6NBrAqt8ZUOigbLVldcZdGL8GQfsKZGV2MYN4YV9E+MCwD8oLZLkFfWIfgy+vQJdFlEoxsX9cF78vuOUIsIGkkg5H1QW2qQjMUkaezVBCC4Rc8FTsx0fQeTd/5BVE7TuYn/sYHXR8rQ4f53jB9xgGKIWJMRhGAXyPzfufaPRCajRlfxIWyXOVNAa9G+wtmeigng1yihwAimFZxBaMie5FoencTGZT9tIpglOQRd8MT7LUoEKC5tnHSZxNx6NW0jjsXngxFWIkqUDeg8k6Ypbjr1UzuxfmC2vdeTKtd1vnjUHVipayd/co9XfM4ElrMPvp02K6Ylx4IeO+5tL00y/NZIJylqIs1m3/ttt8uHVY0q8JJ0qPHrFS7C6FGcK/mfWDEm0Q08Rda2/sEOOTRjKf0BvhO5NgyqE+tmy7tfTkcpkdcoAyeaKCRFFTZ9vATxnadN23+qCTPWVczx3PKJnUQhFl5aut2Ggd9sdmwE0iyOHNsXnHLV4QJlligEV8h4E6xih5S1dZS2Ch0Ju0b1hU0blX7Y0JrbC74Kz6Q2ovO49dnGME9QIw6wTs/wIiQHNbL3bCFbaICuGx/3Sp6L6zcR2TocNRupK+vv5U1C1VTJ+IJbB2w0qAhZK9EZjulTe63jbJuKCARDtjLekgMT1QLKZ2Qn6QqYBg8rCvQeIItOu4/6moMCty+GbBY1p+E8E3kPSqW/OHLE7seh3DloroaKMRaocNGdy5PSfoiLf4DmNZIp5ueE/Q41MC601k+QXkaHq2KepqePYCQUi0ust5flprFMTHNjr68gd/QLEpdKIcRTX8BOR0MVpmiS8CgdPk5O1/YKJk9paDHYVPS5C989hreAT4cLAul/UyIczHQRPDW5jqHVS3CNjn4IRXtYWvkNT//oCbCdJ/kYBN1g8CQLLlfF+mtI5fDdDI4710kompSFuvBizq5cBo4ukAEToMqVYWaOWzoYYfHdADNodDxzf0P5d/f6cYYCoNsH27eIso1bbsrXoJ4/w5n2ALmG0eE2xtZndnNBxippapb52xJ8RSERZ5wkOvaG8yZEE58C+F6KGIrzEsmOF6lQQ47ft1JrBqVMR9f8TEEraFkgtH87dmkr0qkNd/Gv6Ls9WqC4Iam4LXXNeCiwIfFayafumGa5T4Ow/sdLgg10ZMYhe9CIk07q7b2QRPqxX7q2txG1PDOfLe8ubBnawSRtgT94k4JIhwvkIStvspZg7BYnTtRHKSvgn9/fn17Bz5u+Zj/FAor3eCKqC9ffFwLr4NaDlGcGz5QwZFhgxsNEOw3exms16E81lzLMOJFUxlZASfkM1+WIG9rvKwbpnBm5VaoUljsoQP5HSLcF0eCV/kBnj8WTLxOrZ/2WhohXOVLqzPGnyXLUiRvAjBdVRaf3aLdXbrHrzwiLOpI2j0kuopxk8fsaS8PGgPiusLUJj8ziIrlZDKyNoegeAzHmHV3FGKuzW06WkJ6jM+RX64OzJqawHufHDvceuiESqAieQ0gDwi7BzBJP+7Kv+kzClIGcC1bdxTgukH0AqveIhtC7gnMb/PlwB8Az7h94+5iTV2vmgYmQVQjY4jsSq8oxQGGGBN4zTxUsVowgS7Z2jL9iqKHaV4ODz/YRZrHvBN4D/psQQNsps6066cZGev43zxdRYLkSJB8zd7FcBRjiil1E6eY8fWrqJ7duXVNd5YyFO5uZk6y+BnGTaCoe/ZinFH66dTmU3Saj71pvRUlB54kGq6Q21RYUFn/p9wRoFq7OZZDZ/KZ/NVqETjx5wYKY/rcFp2ljMwx55BHM+EvdgSdmRY9OPWjQNWJ+oct9OIEl0bPJyK0b6cWip0dLLVdAe6t6y5dos57X5VcfzI7ILk9IIjIKD+ZZ2tNEYi074qRzAWT6qorZdmM1OO8gfsMoeUC0/+ILlZgffh+znAtcvipTKZL4Jc5+BMu4fhJqTYS7uG1tFI2N8oTNRX0uey0AWxQWDf6Zube0y/wrIam4x6BE7+kE8CbvG2A7ix7lbWwulgyzezoTrwGz5aWyM1MyD+3nuGyAecsrpx5lbBWRqqOPXktLlNcFX9JaKWNOYuV37ybHNeN1YD+qRz6yNEZhsF/G8kLIaiki8eQ3T6c2B+WCQFKI/o8KbEhe02No64Fs5L7IcEDC4ceATUPuh0G7AfbLBwFL21lXJm91QhjFfDYemR296Y3DWEv/cldElUiqU/enYrnUg9Nof78OANwiCYaK7x9quZj5cNFWTDdU/3lK9gE0BFFilWQjZy+0vBL2ypgcVL57HtVeSsz3NoLGgqnYJ8vxLwWBW6zxM3w3zPs5W02vrNl2EvGyr8CXi+ownIHMdoK2BeCSbxNXzbLKgm9xy/E/tW/yfauv0VQFKv56NoDmHgRq2rWGo+f3wAiL9K7mLThG5tS9SWi9ex8sUlhuT5mKyClKCphWDvU8aC/4e7JvhzAefIzgHnP8d3w9IfXTtYPyQ1hVM1VsIoVA+i9/itDlyjrZ9/YOBm6KxE/BzkRJvveozSqc6+8TyygXOpIXfVthQy4vQT1dyWpzCntY4aPZPb3ZDEhHJHzArOvyBEe+Gvij96+1rSwCQ1M7PJF9wxiMQMjuIabTwgBt6C/KZwAIRSdI35BPdUVfgctLo3q9+KYbxRVpMY7CszNp7T1Y0miyTFYWADRXOYQ1lNK1t5/BICrnyCcL6dzzJfaygpQz/NQff2TzMYEX0CXDHBADvmc+IydA3u5S8y3qPspMolgoJMxizP0waxPxyDKeMTPTA9t399/y55saQr43rEqtC8WI6RdcLhw3WMsKRXe9qS5aBqflocJHNFAQaLNusiHLclW9XvnN5GJO9TluZZ6/BWZuYZxJc7GaPbWusbCB9yk7CUiVDPQGBzUiHmOJZO0s0Ghftu70aIIIFtkO+uHD8Za5SLrnIOvkPqClmBk/oD0AuKjggtHcfoR7dk8hEVHERQCBK0AEQeNDYcK4FLUP8hUEaa2JWvTzEcoDhfA7D+VP7xMIziRWmVou4bQvoSXx41Z4RkeegiIzFv33deR8sbOij0YlvcMyJdWRvUV6bzphskHGv8gB1PtM+T76S2h+2HQjTVyYJANqdVdSR/LRKhValkulLwU1+uQsm2cSgrDsv+hSEpSX2vIUx4PwJeqKGit/gwRUpCth+9O8NizhVBxtUWXgZLxhWp8fT7V54BAJITdza0QnmHCj483mVw95Zb9va5KMns7U0svWFoLR66zv/OX4gEdEUVbqK6wCzxlGKmqO8sHeCmNVLEa3o2j4sYdnX7T7m+M0RGw8K9S6er8RlSTj8gmOIXxEyq46m7RCDRULvb6p3CT0OaniCsuDLgEbQPkqTB7WCjbGWq8WYQiDseOurej73a73Bk7tYqaBseIYSp4Qxdi4Ry9Ij7fiaEnk9OB75YueREE/dh4EoO+0XAH54BxLY9wVs0iTEpY5gDC2YevKcwdAPFW2kiGTJ3IjZW+8S3ctjqYm0Zi0vgo22KLgkH7JHRcY9927TuHZvozR5lo0SsVPtX/hgIUxiLOdn99mIgQ6jH7unbhFcPOFkFzCq6/HBI5kX78jOKjYIEa1lt8O9qeUuQl2gOnMSLzhucy+RHyn1RYowUkEdY2X1+V8aqEZrJaVYtECwEjBbqLnUeUvDbn6Jt6L0w2RRVclnoBqoXuh7aidFMaKMAGaStQZqbQBXnJGwo0P3G/vNswdSYx84pL9B1YUJVZlF1XlK+RUwChE/XLMXGZ8aIKZE+187smYd1TvtPaGyFnAJhP/QPBDHL2+5dG+IE6D659Shf/WBiufsqSIiiQYDKJiJTkB8unB32vGZQeQZkHQH18TT83Hn9J6yPHcMSCVgYMcUDbyOpQsj7ja81xvEjL5AEQatjc++htFhTH7Qncz7jRa0AKY2NphYk1OeA9jH4BgO7zOgRqXb9OPobDpjQMsyiIfA5mbf404M2bdxzCfNgnBNKicrtMqNrexDOtG+y7kpDR6yGlIiqwd21OkvkjSvVX1k6lFcP8R/r+sBtIAKbL768tjJMtEXO9+bsBl6rPf9Okv0DmClR4iAa1NCnvo4emGqucLnk/6sv+E7HEWgm3BXGLLdVl6a5km6aotMja9eFs7i4rGrQ5peH0E/h7FDV/hS+dNW1+/3ogKwzWxIjkr899opiYiZSyJ70I86XH4a8JAKHr4b0nDPQrZJXiUCmKKZuzzgkIQX7nwc9SJcGx62qLB8I6K4ZqBo7dRbw6Cbjxt5xNIPSiQ4EYwxqzQU+/3H9/CB8IXkDkBrmQ9QX75kboQS2kIbwAkbfF/Dxk1b6XxImW+pSZCWCNRci3Jp4CW/0lsapzrXp1z+QljCHUHm/WpvlnVxFUAL0dXHYvMfEDG2bLFC4bowot6dTN+IMEd78cZgNS/03XlnBCoEQ801OLX13oDp3gM7ibYG08YtstD2JSfr4a1XxwjO6M/BsU6xuqWhsts7ZtsZz0akaEInovXucOgic6l655L1eZc3/w8LN5j1Lps0mgaSw9MW7YUk+dPUm8qPX86ql+ZNo9xX+92MENNlYqJjd9+pu5OdIpR36vhieWUqghIZxDfBzwVezRuMP6KBN6YhB+0IYrmteUsNcOrvpUvLfQnCfs4C8dI6J8fEkULWQ//radOQbnQK3mSce5w2GYkRNEWr6k+fvbYOP6c1Csi1sSPykNvtB7CG5/8ZkFZ9iKX1H+oU9wlr+xrx0pVpp6g7dmA2nBhup+xUrAWIgvXzyNbfVhdJeYZi2vuZaIVr+wDsNYcYLODeSZjgkOIp0NT51w11+NoE9ictJMr+XuMagNocpeWRwjgd6PDyWfERLyu9AG2KRrSrBAEV+ppqYMK76GehoVj6KMQ33ySB5TksJZahKbrXoAmsXmyHhrz5ZLc7/uNjL/zFHckDSyUZBx+DziSEblrX5bzxdH+8tZPLYY6pgQzVRGfeQ3nxmifzseHGLmOrRor39spP9PDZWZURH16uvf6C21pdNxva1kvsINb1T3bzeWiOlsrDzc20KWAx5ABVkaGhyyHd33+zMMoOZ8hK6sSrPJq/z6EWNl3ey4f3RzXkB+JbE/0/MYH0iv+KAaDD+zUZnqTMaCzOR1X4EPdDw1IukPyVaKvHM48pE//GeAUjl6itt1052nnvj6Y86Z8qh75VYGkDTFVJxRWju8UlVgb99y124Xyecl+yKEZJ3IRR6ywKzSvZUWMa2eFknbF966Iyao2tSsScqgw++3kdCPqeLQhBOIF9W1fsRsy9/IJLpCiiiWEWDoAMA8Kz1wA9I47IDLtDV681+eniF+f/PO74k7dvGbWwjEOtDXkb9y66Go+ze8f5Gd98WUUJkvPlO86OElvQcPZ9Kfyp8D97bFvOBtP1+oaESPCzGlk7oQB0fxozsXRO9zEGl/45hFsbuiP/RZFSo2zWO8zatF0qCe7sv4Evz7/FQthtQwIXNuRE4IRZqeM1B4nUO4sce5P3VvPqFaII5Pz0bl10osWzd/s/pLQ79/3u/3aJr54B9xJyc05kSw9FvUcpkeSn7B3O8cmSJpIWfyN/Ma+C8guz3cT3eX5K+lSfzxjCNgHsnllFhRJiI9t8kRCS/hWII9OwSZmf6lknWkTbyw61jHso99JNwcFCj6Any0baXQ6oBIQHsg3xR4jez7y/PCEpvbuvpvbkS+Ec+D/cb8kECXifYh2L9eEK6gnauX1OlTj9cnxdu2agPgbVkRW56z+0bf38AGBneYm2sOenOrlzBGSEF4eSfhNsTqmEHb8YQoui4z3w3Qs60D2OSFotPneS+A9gwQWaawBzwhKTWFscLzXkKVlR+545RTUk1uYprMYZVfQlNxd1GeXxOtS54JxMTGbni0ohrOvaMlwn01bOHxc/Hkq2n86i4uWrzsaq0SMO6P5fwPmNH4GOpX9B3YinqLIBP9m8bVrf1xEuiIlnN11W+t+SNAE3qBuVH7c/Nsbo4T35dSDbuqT0nZoQlMEjFirs92SEDIS3JpsUf5KBaGoSbFGZssUSFfdZM2ifk6crnqUEI0B4QXzX573Yy33x8fLC36REqnGPE/x18ilLq96VcY8ogn2gDQUT0E0K53SyZh3chvMRk7uvv+l3FVUBIMJgpo12xAKWkItRtJbriNAlfjcrFgEL6Bmjuw5ffll1PShUiAY9nfIM2iTY/tFx/YIVruyB5QA3mnnkfiTEeUArYtrQppcd/SQanBWPIpUfV7m5HDI14O3R+maDDHJfonX5bRT5LcfZ6TDxz+Wm50ie3kbS7PRCN6WTThuu1LzzTfG0t+c11p1PS/9X8MvRS8DZ+/eQ14SFD7tcv1YahI6exae2GKPPWAv/1ivpvQ6UgngQUfKXiZJhKCr6HhMHQkZxeIVA/I6+AIfUI683EydIv5rbP/9QsEJgk2yv1NNyhvQ6Jkt014N4FFApZH2Z4DRmsjesx59XpITJ5VjqnUqZvwuo2L7427bJmCgeV/5H/UNe5TzeQ+4FgLW4HKkB/V0R/Ye3bg4fgs9+dKEBcIX32GphpxcdXbWOChhdrWjrcbJyV9HL4y76Epa6QnTFA8ucrmEB/jARa/iuBWYheBFcTfON060Yfd4R1jBeyudvlA/v2KL9vMWydc80znXpzu4cljVW0y3YhjCOQCgQ6kPxmZoIGnhJYerY9wqSu1173Ja6PFpwfghFabLewVam7B5HscqNPShvwgRjFfVyRRNATeQtMkgy/USQcBMf6vaoCwvQRpwzg3Ick26DgtyPMYT9Q6+rWnajDlVnyDqKK/l2e6puU0KMyvjW49O5iyDs8mmo/Qcui0MiOCftm/LkzAvHe4QQI4YECVog4bB6a0cLFo6q/JT0m111XeH/MxtUxYqmAzetx4IVKjS7V3jGMasSq52Ln5uZpMSc6BsbGgmeIr11DeSK4Duj/9Z2h7Okp230GxlbezUrRfUoiye8XLaSc8vo392vWI9jumuBBedcTw6Pp1lUHpMHT2V0b2MtLZ0shDORK3eK0RVZJc+RzcaYR/a78zqdKiz4cBUvCcANQasItmBrT8JPBofmqcILmeRO4BuTByY5NmJXEtdWMTuoNd23ECudAvXyLX7XSXDosZaSuk7QEW5FwT8lP2G/gq/m/52dc5ytL/PdDBnIxtvpTiAhxjmF5+Ft1eBq197eIKdZlQbD+evvhhPpAbQBrd3+bENe2QIgwAqIjmZ/IpptG/hMotveQtMLxxFuuNa3gkHzJPRlYuIdyMWqx7fbwbZyuwLjBiUFSxdbR6PYWfLhWCxrehqo+rMvHa2J/j49mmLM2qMk8xbTco0SHHn1fE4AmieiHprqmwg5Yb+vPYxeYjqG5NWeW79coTy0CZh8vx8cy+FPgH+CqCQw+l4ytxDPxhc6ljn82UHNbQRf3M7/F28eEv8yLrYYfRtKBS7sIqqRCeJT7QJrFJ+7o2PNiDxWedtqReEGcIu6ZfwM+wEDpg20M9RuoJupr1h6pcGwY3h96VOs0Sqt7B01da/cTP9SxffsnO8qpNWaY7TbV5alTI9R++/Aq5GOu4qv26/hi+ftAVc+JD8t5f2rHm5N9EIvvlXM7XizvBwhJZsCIsaTuGbSd2t06mNIBkD+dt0o0CkrGNQzY9//uwwm7Eh+2EzhYZNuyBGJ5rX7isGt7fr59xhNEOS9o3NnEtL0oT0KF4/zH89XocxIi3RyS+3yOLCpcUU1AXWCwAVLH3lxY3UGyx5uffrJ1ipn3MICYmJgv6Vlapbrv2t187UcmUGpK/4wm4JPo63mqhLZnZza3jC2z/LFC/Rg+s1V3OpqxMU41ZQCUna/FLPCsJMdl3ksZn1L0vSlGmg/bN0csr8xvxyWiz6F5rs9uTl88iVGitB3f/ZNrLm+K+b2jEkPkUFB1bJMnjhxJek/YwuVCZ/+by9v1zwbFrWPDA1toLsYxru74oHlETLJwvFR/mnfaTH7reX4fc1ExvDYeGgr5PjiuaoeCn7X8JmJDmGNjXcUXpKaX4ph3wV60DT9VWZDG7ehzfm6t4I3py1m943WaBgOGKk6ZpaBOZDM+IzIzmjOo9zsYwT9Q48t08Qe/ti01+TioRJ4EDF51FVJaXzVBnmrwhruRS4oAi0be+XNd81XkXiDZ0PjIzj/qh70IP9zCHRK7EmBfui5ONWyHTpxuFsUtyL33KA0Ezb4KHXpC52H++PqlYuul9c04gZs1VeMzy2Hai6k5pycMANkF9swx5N8189IAwgUFjr+n9Gzj2Ahp+40rGVxFRQQwbE5y6q1KSXsqUjMHz1hmDQntWVT8PQbYR5MCnZr+UCLgrd2JRVmN4+AXaUty6+MaicBSfPfQzfAAT5cA+MiehueYI1LL1hFKOfyQ4GF4gHqRnsuEKF0X4qjoY+CQiN3coiKgAryvAzahBLQvyJix15QMytGX4OV5cbLDDgEccpzq2gmnGuPyZSvx81hZQog/hCi9r/g25WnQNl/Suv/HoX+HQ9kL/EKgsuHuAugspr78v2T3+0uTtwY5CxH5PguuA9DPuhcia+qFyHM3/KRSM4W2BKmk2cUqXplVXyJa4kBbYZYCJGrboI4sMTesY038D9WtSOCYX1hcnnYWUTLtHccUvZbN2zOU5HoT1SqOJ+JuqjzQKCizDzz3o0MtpARJc4IbzITHjfI62PoceNagsXc2QWWTIge3L7JfTikLofrRU6pr+ZUtZZwvKTa0RJ6HyVxqVmK6yP2xX9gYXEPyFWYzMsp5VRzuV9v6VwRxoGyBneObL/iT9JBGlCpCWmO9Th2v8T0UbKbNv/cihlPpzhQyhlK+N6GWeGd6LxCG0R4nik5dsRVP+hgMfw8vvtdRMOm/zn7q+rKv/0OhT6KJLrndgfKHeKURnA/Kae33Ksz0bv7tLlyl+x/gjBHGEc+jL2mv3iZ0RRgr8EZH6ArkO3P687ujUYPgrMWWxgvc5F0kPKV7+EvaGANdBRLGN+jlCBQurCm5FeHseN40fVUWFqZOPmSqGBRW7nIgKvSKOLePYH+6hpkelSK21XsSjGS06SMFJ4MbaPUsaDppiNsVXj0rhCW6wdoSEfvq6yLtCFTOslpN54p1+DW0ofVacibZ0OwGmeIDhVJZ2d0ykNF1liRLMeotxr7ET/cko6q9c0ulpg0UHg4Q1KO+Jc7wb7xP+jdQjVSZ8D2a4B+V+BluAhJrGHCMtFHyfAMp0X86r5AXn9ouf1K7vCrO5FO4jwq6C4VlL7ocQ1bazq/mFIcXIgMwcFG1/XXcZiPeAYB7tbfJU1DF4bt1CZWV7oA679TFumvqGhcG08blNp4NDgt6gnv+lIdSSFsE78AQJDlu0XVXkzA5g+x4K1W5utu9vB+AX7bpjpRh8s7P3rWm1yqHPlSEv5ov+ViE1viSNqoxuh2lOzKEVlXW3U3+Ek7SxE24mSdSMpoJuYRb1x91n8blClBJzCn6UPz4FFFwuUcjH0OnUEPsTdCYZ4OG1T/6M3vK4ZMCqQX5K0eBPUVFNWKVfgLmMsJSnz6CTVes3ZltHJ7c3Ixc7R/XIX77HPoAiauk30Bv1iIglvjPwzCUzq1EJ4nBIwIAJJSVEZ+d8myJ7JW6fbHmkwpKbnPKpyk1ZEAK+nax/7TQd1CqeXIhTosXTjtmdYy/O1jIKCHLZPkqrMuiKzZ4evrihtP00unwM9CfpmuG2ostJamy0vaL3ld3ZxPhy9GVaUMa4yvoY7SAJEhfUoPhb9//dM6nFcV9uoQE0B3riNmDEgRS2Q54L9P7fU1+Ok/9+nfUT2xgoJbHntfqjMWmLgRLu1L6+anBwmfbsKhIS1GIjDDqlXEskO/yCe2ovLWOtZVU8ozlIQOGfOPT370y1aNL4Ul/51NXlFWw2LC3J/X4GzOv1Fn0j1l9oO1m8+QGfww3dCg2bX6nhPJS5lwv1Q3gYi5J7p5ISQbck3OhU93NdomJwm2K+bv/aPftgKXJUQpDMgZluxV+j2cjDr0ne/abYCMKUPwe/sLhYJ4rZITbZD5Sx9Rmc0vBlklE8FUTQUkwmPkb5giap2dKLOgfvJKMBcCEbdZAjmSVk5IZrND7AlFJI8WWMofhlYp+e+Ctx+yVoJTn7qaP2nKZxABcyjPif1awV3eD7qU6ruWltMdCJ8fXbP4uxUqU2In3HpTIWoSoIv8O9eX9X2wevTOdmlz9L35MTKH7QDmmRmsRzKiDqkUohtUxKTfC+8Sn6wm/3ds6TRFzDa/13Ja0seSRzJUoCS9WQvEfK72IWWiiUAh9AxVHuwJ+Kp4VRRlSijIW2LQkaRSHaFQi55SP1mXfARUCmANukIMrK4qqBBoVUaRYjTe1/edY2TuFYGFsUGwhv6cPI6r8RJWUxQDoXvI5RkJBrQi9nrHCgkyWYQuPVj7N4nuO6P45qxI9uhF/Y9pfeORvMK0qoj1ChtKKchmaRGW6yXeaWb588hNAKXTMV09/r9xmP3GWokqcPC4ExX+QTqcKrlYfrBOsZvnLxBzg7Ys54ODX3doTxvajOEkI+Tpg16RiPVMy2S6IpVd6kMv/HhHgT8XkQfOx0RqGJDk8qCXOSJvU9REhiKabi3EaUOQpdugSIJeC5a28IBqjbMUwSL1KSe/0OINLgDz/RRg7pp46/2FkX7uBRHNEptAN7ZaNq8z6V9pHJD45EYX+4MvkNrRTcMu3RwhDrVWUKNNzSIy5PnFiaNWhO2cUlPbevquDCtyePEYQ9Xgem9LWXbiABwf617gHtCq4p8XvldAsCm+s9t54pIv2AAJD0WPp6pzwa9escE2W58HPNWvevK0Jyvu/fjR6Al5HGEQLzrr/fbYspdHnJ//vTjrCHwNbiZjikAS6+LgEjCUltEEPrU5an3LH7NfRewGFvpOJv8O7dP8+TNyGv464wzUU9SQl3qGfafXfbrlOV6v42RQrzPDMZShAoNNTEzzEyYLCKVLm2X2t745muDHNiwzJN89lKr0dP/EgH92j3Cqql+tHk+5coqFnCnW+8IPobe2Z0g5WUsltFG6gJ6Ugjyv+gKGwKdXDaAMX91WhFAP/1PL3o8UU+ov7X30OUwizpgstVHAOEFMoJS7uccf8KQTw/9kZcpoS+6jqsn3ouJg3RzQ6usbNkbzwXO5EJtO+3VRdv+9ka1rWa3xa8B7ibjiiROj1PC1JCLxNiI38ECfofIrWJA+4V138JxHg9rrrGf7+OQFjLGzkKvOslFiXvsUWcOkHnGKGBasZ5hZgi6IH2BAqaRxL/1lPnkgKGUQzMT7SDM7TLG4hzsM4dx6XOfCV1+3bTr7MnSQ0rRdeyIWprJ4sLkugOl35ZMbpU5/hDdBFVpY710DIiQmDxbMX4H92iz7+VKpiEukj8BGQyDi5H4HFvoOp+1+L0GCHB9oUXn0URUgh4fHlMumobcLpQCgrFMmsHibrzCy0i9XIzwXaHDOz2YqG2b91r7BFmXerhDnCzDgX9Sckh7HTVNxBlUPxu7h7RikIoOPB87w96NkRVe/Q9NXqfh6hp+qhRMRiQfRBU3SqL+Gn7JtF/r6kJYhm7o6852re6yXqp9dl6n5M6KG0GxnR/bxxcGCtLsEdBZBV5WSl8qRS5TTWmiQ4b+gtfoEZ+RhSa9ZRkffEpwL9YQ4X6esApcwR/pfEVVIpiAWBEaM5b3qVKO0m85D5pg4KfgkOcyQ9S2Qn3Rf6Kvh8Dww8LP3oLBtFTtzcq8JsUBgqDT6GrXzeTmD9QQSMzo60L+107Q0r1kCJei11TJQtL4biXE8RpV5n4VHOXIGd5H/vsx1i6nCGmbMt07+kcXf9dP+WvvYOeusgpmsb5w0HibbwoH/uzFHSP0EFKd2Vd2n8N3rYa+397xvibUZcXwg93Z2SP7rNrAOyqywINHf3YWCGHyfxFBalQ5/NhHIABkHa//xUCnBy61Z/qQxjsWNapVUr1hgqffT1lukiDVOlGKbUXvb9b6LlxmTfDD1yoYoAQQ3hD6oS+ETYBA49FEqrYYP6o1y6KQf05845B3ZGzfkZsc42XgOfO3UGcI3LMy+mR9N1eMEy9v3uH02pWtJI0yZG2Mlbk/W6tHVIslXC4pcUIIweddrD9heZHE+c4qRDzM0eqSu52hTor+rvnK3jjrf3aob2y3QF+m6iNV2PN9B4XDZF6UlZ35rpW2ISU2S93CdOgqjhGncOPn2iUnfwfnNMcT1Wf4qc086TQgKUvDrH9POIRRqgvtJ8+V+ywkl8Wk/dljj6yTdx+u1EV4i8PZXWJkBQzvl+sEvM3rnRpHSzr5BHLmPmHgM0uvhu8G33DwCOj2FJHokPXU4cMO8ego65QokekBc3b07dc5ofgZB0o/J7QJmyOcW3HJUvP7Ecf0jMw2Ar/4r6maWGd+G5HzYt/zkVD96rjZG9oDKz0r8hePpay9YF7ynKtuNRdYERDgbFFKA0008Rt/dhP3uqqMxhHc19z30zRAjvCEcSoZP6uclm8COhUxPXF/HmZZZvMELz21Dv6K2fp2d0LYW72rxbIv3vU7qfUGhE5QMhlG4UvAY6gkl9t/GjHiWK4hUR2zdJ0l/QSU8Pu+1t/0nj1e3+JLWPb6ro7BSiNKg8pWeT+ZIEKmgzHvf8KQjX0baaeCLzqjZL0sdVb2gPCjCvZ/RuKXpMGK31ELI/3UdBOdtMp9UETItWmbhSoT8T8OHJYwgEGz29IvD7xe+bnf5vNM3aN1HiYm0z8sFnskjIzczufi5TOOXZtiLSEkw+7iHZJbisaPK89i1IYuCLEB7MAUEV319qk36A9+a/VrvqNmtimShqsBtmY7mU49ycVOzC7D8Ts2lfVAaa3zL2Wp4SCuUuqN05IN9HcJvOcCR/X3Es+9cFTk/isqRozUTNj7s9hMIuvn1qiqu0inhZXch2YbnA7qLytJXbeNDqpM1LHxU7J2SVRA1l12FeW84Ook+6+8x8tgETkCf3NBFllMGOQRfLbnU+txG/RGST10o3BDMvWCuA2C3xjIclToK9L/4xIRgewM6Dw+8jZs8rrTRpLKHHol6AhFXoC40sOzlFb3/6ndpQzTltGfXFhdbrIXJI7xMZaLdL+OQMIm68ElQb7TvMM7sdL6Wjv4sSc8bFT+Vu3SHZ2JQn4owN44J7XTnoGF2V/lQ0d7QZWH8Y3Se2qgveCOnsLSB/SD7gq8RYwPDHDs379YkHvSRABvUBN62+D6nCkaDpOaCZgBTwaBMaRwvdPpJxrC+dldOTC/4VNi2sKO3q1fVgcCT/FSW6qqHw0ziUEUV3FcP+oew3aZ7SWB68W2d5AWiA0eUALM55fxrt5WMU91zk5Ch5AxFNM2XRo232ZVAcxcZot6FJKKIhHhEDllPNzoU2worI7PpHTvP+6hjQusTL2jsV2My1ns6+ux9qQkc9xB+G+Hec4mq0yIGIWakaGihVDq7ev3+qlCtIkj+V4dM57np8Hgm4WVMql0UuiSqBUwWcN8WnFe76YyTFnLd04xSROBy4EXb6kffu7CQNIBxCIZPm19MKNzppk6t/AfuVGRalxT5+fQIpuYQYwNodqojPU7SN8fmv7WYrGxHDdi7H83LeaMNr3jFpBQDComJw5D5Fn6IzV+YWvr4PMelvSMtKgteoeAtjXHsBwN/aDk6TR80FXM2//D1j6S23gfegM584hbZEOJ88qrDfts8C5wVeQFCaYiP7xwmqp2YJH7cHNfaXzY3lZSFppAoCC8rat9p8lxXixNWDJTQmgogpuYEuIiI3l8MiXhOBrZHLLSfQxArNAkTr65f9pARxfPjrhQeuDp1jtzhtUaXOHlB9921knjVzIypYV4oLCIRgJZA7dUHGYl6tcDGM9lOCP80IDHlmOnCRoc3rryPz84GY5P7T1/uo2PIC7cqTh14WjXn5hVP284e1DDtzW/mKtB+Pi2V2/L11PuZUmcoMgNUrD/9BVz3yqhqDO7TPfgu/6lGOI8JxLVnr/lsYveWLav/fLNwTyR/IMDNUYw5/8eVkKjB9bI+P1VjrXb4YivlSSatKvuy/hltnfUyz1wUxSY9tlUCzHBtu3QWzOZCbCJkisHzOLm2DHOf96GLFj5eQMpj4F2Q3ZB8uydb8vwnDDf03Yay5koAgd63sVw+f+a1bNRe2w771/+JiXo/4hojwxLdi+GadB2sEhk0mLfrYq29EWWJA8EkjyNxe4c5mkhbyCV9jI8WXLiCDA2owGhILb5mEABYSRKAy8xRY3W6Bgt9C4gUyrAYRuYWVNzo56AIdL4w/t1pj6WYstrbip37Z1VGcjCc2u4YvCJSv+RVUfjfCWzpYJs/JOTdtYWv3Xutwm8+e7OXBcaZOKAynAyoS/TdkqnKyRHi2xQC8HPh+nqPegvQl4atYQmZv5U6yFjbESmql4pYsPoEMxfFe6uF59RhNTLOp6o9JLhvDRXzkKRGfWn7/J4YflTZU1+XuAPFYw9WaYD8q3c8VTWBYLmUE6LFmBNLPD9V3XBoGCgo+HTTJtvJahWke01s9X74MlCfi7GGAniU6o7jMIrPVlC5Wa2deCZeI690Uah6ZjZVA4zekeyefxyKMkobQEbQVmGvpkdQfOPYhYlxu5VHiYiYS2+FGJMWBXn/pUbJYfEnJ+AeK8KBkeii1w87vvqi+9fdqEBYm59Rn+OskrOya5FupFQQTaxC4jqn8QPGFnN15gQ5i71ksquui0AcgIwRXVW5hZtxDbukpAWblqa+I4RG9Bi85lq8D9wC8F97fKDeaUMl+T0JEhMcd8XGeDyS1bVmlh5CLHlBnMtkhXdXIlGoziytpZxALrTylgc+yQC29zw6E5CjREzL+xOrIRHzvHzyXrvUEPMwhXV5+Yf486bH49J1dJMY3xn7K6ynqJKs6TdcXIu/2CcBnuW/vicmVKxvg1UokROKKRnTvhCSCERKIDG7Td2h0iCPMzqAsrdK73N+dEJ9EQd37pJTsGjZ2SxgcC8zclHyTyxbNlWqtQie4LPgjkv+XFgZkHBvM+WB285rRu3LSOJm0EdD/obga/OTaOSLzBMjbasKXYT45V+9p1NUcWe5csrwlMoRL6pbrhkDJLKro7dluNM1U8/G96TIujDbAPNTuZ3ZB7G4zcYZVZNxdjhp8PMJCJH7m8mLaVVAXvpZboiIGCN+8rzD4jo8g6zUze9HAfyMTvAJb/mROslJnyMV6TipAL9DCCz5XzVfLUU8Y3eQXRpWKTdkmF8IKmY4gK6W/rKkNLjUzU/rlV21r1XfQ7ruB3mIPZXXgEvSH5Zt+DzYEveShcxmcb0sgKvF+kZ3gxzx839F66+TnKyql2CT0W7/lblVTXcL3ZGAkaOKsUpDZYQ1ZO92snLXWITD2bf23Q/IB+MQYIAXzrcLIYhMv4/Zk5aigMyB9OoSEXpvweQIFk7Mwn+qW7Gj5WlUUbI651xMSpXy6iirsB/nRWvHyjXBKnsjiASGvVC2X5ilGN/80JSRbQbwWpIux6+WDvdH5n7ukS2oAY/KmYqdFRRYhvhU1sZruod4zXorhzsUyenZe02LicSCMoAfv1JbGyLg7ATB9PkG9od8rsanmMcaVHaOxwr4x3snSusmvDDCfKWlVcLM8bA1j+A5D98HmvTa/qLBwKgwBKmhywwGSkuNO3kdMRwzufdGfJGI+VMDeSfzp+QVlF8CVHRvxLWtGkdz9g6AqL75R5YOn1t/3J2G3fMpKc4V+fDBwBAzr2X+4+Y8gWX4nWsNqPOkjO8fFVxQ8BbaX5I9Grj03SjDmmW/GcAkrKcsXDLuXbt6T2FEPKcD3X2v8EQhjQF7mbw352XPc6KP2Nkx+KxMcywLZk2peTrsCyTVEm0jkPHtxTqfSyOax1ZRmhT/kBTKFkFb/6kK8Pyw4MRQRzmDZfleWPcEdAVs3VZJmg91idnBJatRjveRvAP6QLpZ29FZ1sEcs+DYJa3ZWk1jc+vPBQcMjpYVfhv73VyPfLa1oaCllRtOLJrMszuWuQgERxqIkGZgeZVKYKgVyXHpTRLEZNKNxfct4KRjH150lyGMxyR1iXnhp7XSWDjnPw4hX1ga0NOscJZLN2dD7zr2kkP5Z/D8APi24kj9RxWFZ6z25d13T5W2SpbdVUCTsn5Bi3M5ZvQl/1ssYevL2Gk4tqKPotkh9eD9A2ZyEXCgLEujxgThutC582RDTvYFLZe/yxnovTi6bS7oiBq9K/tlyL3CFGa/ENeIQPzAeTFs8Hu7VKFPheNivR9/NV0YoARY23necKT5Ef6PB93mu4ema+5I8HmpWf5xgeBd62TNvvrGrc0jHfvJSFV0CtcbacSzuy1mcBehIeM3hTBnjjC6etqLhffgqK2psFV1+G9iJf1YVzwxrBGxTFeXR1KL4P8iEpL/Wwo3F5Npdxj5wHXLDLz66HBP0SB77xbaJ1qI884OTSKaGx9+hPvOCgvaQTOTXw5vRBwdrDsZ1PvyVaZkLgrT4K8EZ3UVi8R9mtUJmqmr/2nFOT78zrBbOJTJP9UpSqeFNVqwTUt1epg2Fz27TdmvN0Lk+CrGjyoT01fDIFmBd19v20TbdgY6EF0Nv4N7wirwLkVlUTyRSWV1qDdIcERKQ2sPn0lItDK+Bilj73IsXDKraqmrjzNDgpV1AhyjXswi/8WRof7CblZvXWbXA0HxJe//dDcetrksQ6kVSEsnRJ0OTMpAR3E7WHSpzXjlW7kiCEGTOT3F3E3unmaXEgIVk1bx8uhTHerXOSbC41lAfTNRe6KCUyecIk7iGylNvc8OeWp2nKY39nJ70sdfOqVRWr73LZKbaeafkagQoBv9wf2U7//vpVUWi2wL10nufb9aCT7Zb2Fi20cYVFKgp2SudgzZbJzPG2aPQtu4tRvU+xw/ltsXdPSE9Rb7QLMkV9n4P2j4ORMO2FKyyxcjaIyJT2YQP4eIEDqfofWOdnD/2bHS8U370Of4xVo0AN6nIEEIG/eZ5J6myzzR+LxfHWtQNyLTag/lzsy0PPJjcWPVfEkoF/oT1wU+pxzz8vZRXVyGcP31fGwH2VKsst8L6l4t1RLDv55/e3BCiLfd4XiWSWzo/5OrGg4iFKa3t+TH6f0S9cSZcz5RxhKvtZY+3pDLjH3qU6hFT53erdaO/o9Y1z2RvREA/0eWr6abv5tWqCIqCWxEjH1k391zXlz4sbBzbqquJRqvchjCG+g4/fr+iz/KUaregDbRf8q5gR03QDxDq2mFCRCaWhioNiEGty+wC+p0Cbj1nhpa8mY7CXnUv+2O7fqLJwdcYYCwTRv0ldfOmHqR8FuoBvHqGLogwqJAbzjWheE93+r1JBlQ4rmsg9Snknd1n/YmPHj/IArnwZ6SOX0CvP0axgW7dOXtzDI8vMRYzx6uSgo6wIhaYTv9Fj+pvW0vsVe0Znzw1uSG2Qo3mw92mvRcSvqs1dZIN1GDF1ZYY/5Otv6nGa9uPukUt3PlyIeVxY+3VlJ/QGbqhzBX/zcwc01oeF+87H1/rJwhoLtk/F2N9gcxxN1OIvpUyHzXdUghf5BEbrHL4l8IhiW3NZIBe0dvjXZW4E1pN8NCL8t9kMuk737+CcA9C0kLRaBvuM9eItSflD90Kw1S8IlJnseVwAujEHm3u4+k9MTHEz3piAhYORIWftoiJ86vHJUpybJQJ8GY9w3LT+kfJg537b9rPKj1lHDW7Ea15k0OK/TP39oEtmBi4ha5OjiST1mpwmsQ4nZvw8qnaZfkqwM9NRVYYGKkfveymIW6Pm2f4ANg9LWeA2ORdkB9K/HFYVHd0pY8H4w3cghghs9YJXM5iveepBbu+vxpf12eQbmyHtA+BxT+Hh8q9bQ5tdZykKkbNqgDN2Ker0+/3aDOVA6gxiwxwwSOzcEqs2+QGSTAaB3Mz53ai7hwU2dpkdqYud+fEnUkcdO87u6OEOqY2kBlw5xf8tsz8kVPX/mpefBgrv10yE3GHsoA5Y98wcH3jiiluj6DRz/dGsrIiqvKMr8AVYWM1y8FnhFYUOXbUq1WSvQ6U87kU+5EAHjOoPXD8N0mSwc5kpPY5uiQ1zzMCgv04Qctnm0UvwEGknrxv5/PXWY6MYtbvCmzmCaMPEJhOKh4aK/spn6i7x5QHpJt7H/gTji5O3aJDg6qZkT/UTm6GpKsxkv/B2KC+7eU52Q7NKKxBFkJOzwlluq1AhStMJSvlhClJvmnse32M4+PiWPIFnWNgvuZVq7uns/1KMNv0Qlc/wC23XwL6dPLrR0Sbj4yjK0izKussXdvxksmPiJwiJouN8fyu9pHKQ9nVbES6AdzX7FZTN5m2TNEJ4lsJdwkVKube/QUnCg6CEW7nhpeFQ7JUNSPk7Z4Csi/nVdwmIDjWpua6bf3lth5DZEKxj9T56Zqq4OaXFCUrqPKOGotQLc4KiL63SdVHJ1QC8xhbmP8C/eU2M6VobJFh3eWnqCiWfQZl/XYeTXe5xQRKTvr5AzeRLwsPXq5jHP4wqnOdnI8+PCkDwhErDTZkK89CtFHWveQAFlc06eOUikcx1bI893Co+rOeL+e+FdHZP9dpyqfvfZlcLKxw6rOzzZZW1GemTqXrG2nvQPE64N7gnI5iKe58ckDUrSTFYoouUUddCg/uYf+OIXONnOudvQxpl+DHpmSMOtWEzQTT+6j8BBDNc8pPxthu32w5a4prd37J5lj5s7T4Hv93IWx15Oto6uU4m7e/mms4PuLL9YCzIj3ZvVj8YoaalMPLYkctU/uNwPfb6Ev8idMtqn0lLSTsxS5v5HM8cDAlwx2qDCKSrwJomRVhnZOrXdYMEqJzfu6GMOkAT/WTyMP2xk8MhQgkL6wS/V2ItgTSrHqUj12F+McUv3dmPxiwxOoucKDD+F8qvj6bIeK59p6NDfKuq3DE+x3HLxf4lKH6ykt154pEvVnm4QN54ELvksj41zv1o/UDa+lueAkFmXKtBNYNQ/kSEeRRwc0jh5hiK3u+pVxeAkGLWVR44rxIkS36aMtTz7LDBYUWOd/bXKe7flPNb9fZMFZPkswtB/0/wmyHPS53FunoVye5Z41HpeJFB7uLkuXc+fOoDPYl8PYwlesJlLFEaHUGRsTqxS4PqFoPnEtoyMBAlW/Ns/Z1jvk6K9RuR4vEN0wG/UQHhnk24x3NSBhR2MTsEVvSwRzsmlKi7nvMiGA2P6a5RYMUPOAcVELS/K/SkY+LlyPwX+iw4a8Xsp0g49DSUbuWKxp2u5dY3X6ApxpObWnkEUvBWZs+vLc88Zm9lViyGzuu1w5i43h2jWu3WJUrK9x5Q1WU3I7sDZcxO0lKB8oejNzg6f4pshnv1zVAoDKVwyNMS29u6ZW/zQJRAEZ0FQzXVZrVVozEV9IMN9L1rtegWqrCrEy9ZGlzu4u8N2u3kX3LwKWRJL25Y726cVsZx3L80KzXnSwUCXL+LPkf32alzOrMM4GVEfr51djzDhinGBS2GPx36cRGq/TFQBhxIk7l7qn1upkbz4XWdDsmlQ5HmU8mxRsl1HbtgDpCVP64tLFHHINAKk2yG8r+ID2wimxA2SfGfqlMzzZqP0vY0N1YoecTV6wLtaSJS133YD1ZyN5SEOCVmfjkSOuZZy3RXlrbkhH3rVLA71osv+N9XJz/EpcG/02f0UgdiC//8G8MtPNjfdLmIxH7fppiWvHLuAteCxOO5WDHaMaCE+KsFheMq+h/kqpIeYYmByPvZvhzDLWQbzNAu/VrFSxJ8Mdu4qMpYJlnrkDALocq+bq3WctSA0nw/kG+WyVKRQ+HBKsJHwJUC5Vghy92jc26F6dsXufy0X5RmGaEQ2ijgB3T061ZMH/eXDWTw7AoLIe3d5ND1t9r2X3bZK2NI6tgM1hQB9hjVHqX5Wr8mAOFBxiwn2m6E8ivEZt3rof9bKAz9mLhXULz/7Av1Is8r4zn138cVQKdgGy3kwYRv8bFRTZw/9fl9Ec2q5YreHp/ty70Hvyq0XFl/E+mUqD1iccic1he72DvgI/MaJm5EzAsIpmOZTB/ajnMyWBQVGPvrk+SAT0E+L2uCAu2uotNx/I4nVWPuY4j7BFFeLKVSB4+TUjTijtWXJlcISQCi60MOqJ7H2H7pFO5cl9F3/lKMLLy+1NQgZa2Jt2z4eCm27ZWq1Q5UXGWP/8AT3soRF94mnmqz/jJZf2bsPGvF+P56lG+C5/rtfvuLqwVMW5e8+OvRZGoqw8ewDDHm/pe568pyVAmyq5l/vPnEG+G9+APhvRF29UOq+s0apk+fbqkKiSQzMuLeyDBedLjgjAqEaeHIe3z5rG91aQnUyC8KHrh74oQdrvfEJJxA/urJ2N48jy+UJjhK2sL8E1F/BnIZTcSqB2b+UgMO5vyF7HTllr+iOTS3Px9iwiNQPLyDGayXkv7+dL43QusSSnRX4qGTJCBESzQGLcstkmjb/DyPU2hpSzrVWDsoT3LyAlvgBwdMOg4ikZFzo1henL1XGq6FycLvxqgeftvbEQGcuokge3azRR33RuO5md91ybWHN6yLLPvXHenDKOVETdy2eAQCnvzK8s/sYDM8DMWZEzzoueFTXLvRIsN6EXk7mLihrcUbd6Qp39tpeniJEV0idUGKC2IfJl3DdZYpfMZNRrODuKINMfoLyGds/ZqfT1Ha5ahpC4bQ2L+KDgV8OcCxmfVRom9ZIS0rnmMRFyN9LwqVUu27ysq9Fq4v7rq8Air5QH1vRvhqdA4GXlkFyHsuXgFIFwRb3dLm2+87Y8BKgvx1ADnr1yDOfkkfR3XCklTMnwyCvHbdvxnszk5O3vOHG3rnarlqzBk0MCmRQfRYytbvL7s1Hr1FcK7RL1uRUBmtJpkaOuwro2dpqExZTVz38ujheGtJGZt1OgfKQ324cI3dTuqLblh50ec25iNGVWdfnwiU6GVdeqUIOJtTBtbxZNG79WNJ1vSN0Pjsomx5v1BhH5dg2c960tRXT1nIu6nujhAZS010Mi1f8PY6VT32pvrN08LALVcy53jTKHh+PNSdw4VX4xItNy+omovCfhCtPGvc7LxWW/S/Sao9OnDShFWxRRP7tmNDsea18zjJ922XKGNZn51dYEoKHP33rJRmsGAGd92P2N3Q2CsXzJ0Daaeo+NkBKRLljOvmvTAuF/XlhGXaoorU9DURjgM/1N30mnR/femX0y6WfTBbA/yIbm0xYsQLcKXB1haSLeaRWhoBNci4c4uD6tvox90DxpZzeyaYadjaFJKtSFzcXMsbtTnW4oV1zqYcvcu/Q+WegVP889W7ADDM3R6uyA/HMq1KnD7WLYNBZCPmlDCu6gyI1tsNq5qu62IfMl6qFRzoBNweCy0qTabDb+/KcgpZNxzEzg4LSD4XBTlgDR7rj+gX6f+FXKAX2K0gag+TCVNj2DH+ukjSBJAnJn1EJYn9kNyfC8jX4pVyf1KaVanOfGp4ZX4l5EOfMYq5r3kYJaDyocnTOgej1e+1FUcqpKZ2QYlEeYJRb/lXTA7uzpTVh/O1TLKvRWeaqwred8NVw2L0ZmfKvI0DPjoANcV8gR1yVvNy9i6UVUQ6VS9f7wcGArewcWvY8hbhLDU47uX72e6VbboTBQuMzBHTW5VINb9uC39DXlPbHaElfalKmSh61l7ZxIAKMMPHqMk66zEc2NuTWILNAiI60o/esXRo5CqvbPckBa/RLS3iJbITJkDIxEovUpXaB0fwwDcqnKM21KHQQOylW2St8HeIibR9x2Ka9x+L7F9ACfdcK97HUdpJLopn5OpfzYhM06pfzv7InqJ8KSVl1XTPNpSiyM1sn307QS0RACq7wkn8ebMcSK6fI8x6aSLHPraWCICLYGkMYmmBfowZ2x92IJJcB/YvV4Q2EV0RfWFKWc2ECJ07n6/Au4WNq0VmvnzHe+URm7ymqlj5fBCETG1qbDT+GQ8bVohHpZp6nbcUX1nhd4Qjbtq4JdUj6+DTfBKI+Z3Xuro6+XmNR6xi5I16hVz2fvDMrfwzjWMri+hyT1zkDUKYvM7MEwCryPP4to0oK7a7lr76+msNmwrhfCwpEB3zU6mn6NmuJCTQe93voKzNNz3VciGQHQavp4yJqfo2Vt0cNe3c0OwMOte4BEr0KS3SOgGdV8gxhRaKdvyzJ/u4R5SRKpr6hsvmwK243+w3Wcln3fUG9jIYuUcgahT5SORKH5mqk8emywWjlfTCzv18W98jNHuEBYL7f70aWNZiQCo58sE5u4xWz/IMfTiUTaO3F6q/X0XndeqvOVOXTNjsW+rWx3Rg4FHhtdDJc1rbUSytI/l2ql7BgSjjr+yUYc4v47QNhwpwUNm+p8AbuNRZ95lLnEqhTXumIqaoUqhL1KAz1UMpv5WPHGF2gydtiN3Y2C58AFiHPMh9wKlvluSUOm1ADESY8F+4Ta2scpoOoNSujFqlTAD0V+wbtkba0KvdYhFxaJYOZ2jFPjaXFezigBQhE5H9o/VDLQWgJBDTonuwYDvqxLvPMrJBy2cFOildpHupH+alTg8S291A9+aBW1EVq1aEpjxompt2jBk4Vm//64Tn8O5BLz6R356v8huf7UQS9f1wriGIWyUkqt5TEdGdIX+66LGUG8mWZF2VrUHc1MbDvzJ3qItRAjviNQNlTIP1fbSu7xd1058viUD6LzfjK6Sw3H2UO2VDJxyC8FeqBWBbLsTuE60LXu0rwzNt2YRUf3pNFObrDkhVZkXpYV4++/5lpbq48AMMq+JCvMXnBHP9WmsYMhekORfDy2c02vNNfppFGtSRiZIZ+jI8STHZ8GsxSC2UddpTV3i212GPUVBftvMBHFet5ndGM2Rc+8vhv+rpgcbku98kKeRTde51soj5oXIy1pKdkN8iWhjbmXEYkR4Uz/NnN5QAf72wfE5mFCvy+NlPiJ8dz5bwLj+UmLrUkIwMGNKmZYepbfht250fsRjZNY6RPDMkbDDeknTRyYUPiY96jVGc9xvOFzKpiJOK9mWIy4Cj4pAD8+RZfrcLM0FnXnbhh28SxadIUHhPKHnoHpU+7CLdz0kfxtQLESx3O5BdyMFEEKahbdWKj1X+JqfXgkPdK6EvL1qAM5oiCZ93oDGaxUxCE3pcFbOdg0rrDhkoyEsPI9JUfOLzvfIHFoaS6j6KHVdP1fxCcJozRvDLXOmVBJjCJXe0+JPamZ2404zWl1lcTgss8/enA9lF5E/KaoGq7Q4S8vXg06hreqmIOLf6+q6Sw4vxN3SlmMD11agce8aOq65tjHV88qGtudbxR4d6X64w3rh3BOD2x5QP9wsB+2ZYrZT0UtciQ2ujvbFHsa13c7HIaRVwP8jvlxXtpfdHtOB32CvlQ98n9mfmgbbfmSZT6MyR6LPf/LmvpEArheN9fqddyV4qQ0F0DuR6lIoe+UroalB8pBQUkrqKkgDSWpFm6sABBUrtbhkOPIOAKzS1RtHu+yLvlLhICHge8Z9FchI1lpZ5B163Xe2XBZffcox1lWY6XSmJAlfJt9g2DBtsI/ErOyU8NDm8HKAwZewNkBjdAXO+f2ZeGi3b+uWjqa0wJ+o8SEtYhp9zEN2HZa8/zAKrr58DwxCu+6Zf8dhU72pbQYR4+mywzum68s7FWl3yk509R+EiY9jyCi4EipVhk3xzRkrlLfa9oRa6VqlxxjoTKAmflFIJnDy0GIuXOqEMSwSCVWiD/EB4E/rmvIKWwSi97xHHFvzavcJNxiGdRVQvxr7ydvq2NGmyuKxywpn5Q9/jBw1miDYSI/5dDtV79m/N3CNWOCG7pFu0YfD3PssF5X2ObvPed4Hp88q3v375XGiUxQihrrnNfiH7r032S7JcG5+E7YPa9lUqlStnl+dm++dNvsavHH9J5y/UC6V1WPlFhMCReGyzk/V58CfL37VipgU55TmmF5ZLBhkfNJrFvyYpOT7LP7RBu8NLeSkzoQUURxL09V6ZmfFH1oSRs/1lOaKD70jSYq+jwming/mDahslcGQILKSDNUVaXnvYe/lYKuQjDTQLF439soipUUvr/WgwC5rKZvc/VtLG3s+h/UoJ/v1LwT+v/hXEVQ2cKG1oDRQHP1tdBiQ3j6HV/UYpl9fNWK3U9uv3KqA0LZlvM3Pmz8kv6a2iMfEReNAtjZVoL5y5SoCVXAxwNEIGJtWBbRHHdazLDw4IeTEkCPz127AulcOcZChTvQ/Zs/glTazI8t/zYMRGKzCd6fzA/ro0IbAwu4DmPyK0R2SFbBpRKLSiis1wTmME+yus2c/dUz77NaJ5N2a++ejIGWe5ZswIS8X2a4rA9NaQ38DgJlg2y0ZbETcnNXcGYpkGTcHnNpoD+m73TKID+sW9ZzIVBqdnHihgxg9q7jzyQOusJvosVvrOF1MqGoCF0i9OHIcavqfnintnkND7tM88s7QJ0NVtQSzuiyO5Kpb3+jCB1aBR4m1VtcZLonCe9WKuKH4zeEoV1zcyMigSsL7aGGb/NRMYOXjKb1Hd+WzjTvxaDkDC+hv0qX6VTK5uApUQL4iqWFw5qxubTTSF7u38bpspSxGjYrTeCW6krD3CdaWfuaGhT2mfZh9dNwx6FDBRfo/LIhnHfYwLYr4k9TwO0AebbcSx2MiobpNXm3ZteR3GY0CNh/j0z22+6QI6Km/8cxsRgyiJgt8qrI2KNBAd5Etbm5XO99eHUiJ1shM817ry/v2QrSRn9/aD7fzG/zcOAzNEoyJ3CeitWpq96AgQb8XNXz95DPOTRMlCEQIeIL5QSuMOlcCM5P2cKRlHbYbE3mm6NA5bcrtcmG2CHjrvHSZxoltBl+zOgrGIknQolPFTwqBeJJtUtetG8995NNvCb4gxp0NCf53Auaaw+XxjRdfAEUHaS5cpLuBviB3zX2Pn3+ggO2lW+kCFx/56gukizKPnvBOEqTSd7uSUiRnwAWYVKaMSc7JN6pcWKyWqKRp6Q1i4sayVHBIadPJ9ZvVmN76TbiU2fzlypPHwz8/Yhsia0fKGtKmGqkhbYs/o+F/3aJ12lr7Q9PYzhSI8Hnmd844yrVEV/NyBOPPflZbBsMD1SSH1DJS9azZZ67FKHrjyGF1Mmx3aDsawffWPgvRJpdyLfs61+KuvSWTfN5nv7fIHAEvXBwoLCj/QT2Z4vsngdY14/r+7OfOrP9pb3XSlN+jzOP5dx3tZDzTD+Qs7sdS2+hAVjTHgj+sHpvPCubei/A/K/w/KpsmaE89UcHXAms4BvaRyBBcarl8Jfvm8usBbPuGY9/M/m7du8gI/kSJHDGXHS5EYyh6AFtssG0t0HbsPGQzFIQ7U7h06+OfTddZz/euLq44g+rnxAAIfvgMxzwECkLz+s/jMUQtKcb9Fn/yIeZjU6fsrdqSfCs9c0rInUfaaf33HfVFMKAdCartpD7LcaydnQuWGHf1846/4M+zV0vJvnw6QLoTcN8UxqaNpwlbhNyAJvRtbit3WMY6f5L2ghQc5YdmhZoV+92H42fb7mFib+X/0hz1mnanAK5sDzg36E38CEEilvn8lsJxfa8xiLag9QW8UDRP4Tfe/kiDdwxmANdL2uEAretN7R/0tMFNGkNTIrjrLttTgu9DAVtg8WA+9VUpVS/s3AzFXHYGAMqZ05fMB7PAtDvfGvSqHyV22z02GUZ6/LPasz5tzzqa0ExPHBUavTdZRckf6YCPz+zK+qt44EMrI8b/RiDuNBsHx2DfiUTcs+xtUK5Fjq7FnpAWE2cYXpUbpO3cpvjbviBEEAVyklQ9ZUKBzj5pKme5c9b98AWEKJglufvxdYzzXVAqitih2sLprzXEww3l75EIJhJp5cSgmCveRWBIUlRvnqgPHKX+fZRnq/14z9tt6LQSRnLPnHrL9fp4EfAFXsPaVRKAvqCi7b0LKPBXEEQTPcIjV+02ewZjylQGRYpsq4RsPUmgvz3DtWUrudxu2CpZcDI3n8wYa0b3eNOvrqPHz2a0cH2G5CRULKI7BWqzZ7L8Gluzm/6pl/EaH8wbmgrdyJxP6p3HcAP7gRIlU7PGMgfv1ifGtbeuovmDcZupmWkm79+f9zAQQAusQeLz7aAuUay8oXbafZ0CGsWeaODBMRAdZd2StJ2txkgqhKutvBlrrMQNhyrHUvpyzMdltSSdAL+vmb2h8zAMMM9CZIR5tBuHymcFbtlvSt2gecQHfTSpviyj8Nw5VgofGlgX5z4Q/f1/NL7WEyF2ClpvEdKS4jwcB+/32Eh+LpAxZqk7OUsBc/09WBZ4sS+KyM9frBUH5XavIF/Pfa8YUeMReqyI+SHD9MX6+G/UV4aDQ6Rvd8O01N2TuUBek8c6lkg/wY47nsyyLMlqfZHttHxrwLY0vJ5cJYiwdzbhAol4I03aEgmhonKs1Z3mTomj/PglgklDBw0Gp0vnMngzAUNY8vLfv6NDYwhBel9jkOOm3ns7viOTe5qFFeoqGHsHOp+7PifmKhHpoLe4kCsMpou8vjOpZTYUib/tv/3Akd4IzN7bfNXz4aimVqbZpP9hXs5LtnWnAHC8ei+0zlSweWnFxJ2LDb0foLAhzUnlcbrSp0D3g2MVQqKMdrURhlRLYfeKLFt6v19ZIKG6O375OhJtVfyN/GArlmrXt5lnQ/Tki+El+obGqPRR21q5U5w8QpmO5uC0+Q1VOKIow3yAsGRRX/6Osz17KAy05b2jx6/3sjl+8dg9HTgAnKOhXUTty84rl01sn2a41IC2M99OJ6ScpIIgoJqD3fu3y2tfyiK74beAihE0ExxPyt1jtr47KFojBp9CySiWCMwPPgL0/1pZ8abiIyJ2v/GD4LQalsPTpMCdDLTMydehnEDK25zK/llshHk/igRO2DlYbbE+VBaKe/fJiJ1/kdpKb7oNHFlGF4JOEP8Zh/0m/KlYBHEXp9aLlU+UwfmNE0XPQSCgM30LLVQ3YZ2kErsny1zXB04v02uGXRrDy5eHa2m4TLPfFm9SH6agJiBAwX/sSvre/WhzQXlpNpXyskFBOvMpo8IrXrURwsTT2T3GDzQ6qSe7AyUJUufSi4BDI7Lp2dPbHvMCpXz6DZDLiLdZ2ir5RFPFoa++yLGhYyLH6kipm7G8BUfpD7xSZVb4P76Q2k6BRMDv88lbp97+v/DWo5DgGp7d4R4lthofGpFtrDR4y098rTJuG9LNfy2eG/zkzvsWNKv5jAh046twTe9AQRXhidPYUvAx9aCAPbyX3Ykd+Up594b+mvYjsliG8P5zmJhH6nj9WQNvDHUf+/Y1qGow7+B1//tooFFGY/dpFsennTWSx61Em8aU9olr9tZKqHXu1E0joaz9RxTPp6QH+k8Id17/xSDTKsq+evWjbMoSuM4h5neEOu707zANCui0ncrfla/xaHIN5jpCu29mROr/U68OHAf3AplX8phdCbuPXAdmXp01YRUgcfE1eGWIMSZSA5UIJV3dFSgP73ezHF3oKVkoPdnMO9May/JFc1k2Ur1S4ahUFBwxTa9F31GNOTwPAILb5esmfJh/qL4p8ErqY9RT6QF6UQnpe2ZE1zYXx8EG6wYfPSezyx1Vh1AV5WFa+JuZE2uluHX1hASx8vdIZRBA2wprQH1+INFCwi11phXvwAWioRnLttlkFz3/19YR5rA7Mk3LEVRloyfodBTejatK0X5HEh6rF7BMHb0qsBcCtAzoUP5hXoC3w9Bgsy062Xmd33CrM4AYJnRNh70ULr0ULe3hbQC6rHUwulVW34eBfUNeRrXAYRsuINtVHzz/v+Q7/3scyLjc60pfvhrQc7SRhku9mQQKroliHfSOu5UsWZD5mI7FcR6fsvi29fHggeMcKjNA2ChGfhDN5guf1WPGZBx/ZrQDu6cswLoGAEFmgGZVA4m/NolE+gcQMcWoRGsYaDJGXyYm1nD97Dn9rOklXUdCP6BEacsxTNucKVoffCGG4bXehxtCaIZYj9MiW5wETG3sCW2mUaw1DSPSAC34Lye7e4qDI4Nxw/EvgRxWS1cH/LI8VMZpX/4XRqfGBTMRktS/0uUK4KaH9SDk9tOap+8s90n7tEQGXgOE4wuUsqceiid+KfEKJD34ZIcmmHy9lxLQ3C+pueq3njdq2P6ssu0w14IY/K07vSGa1sL1d1lSF6paKaIydW7czvWEIzuTP95T3DC5eEXrAz8C2j95n+3nORJ0vIAdgtKYpKmJBnS10gylvi+N++Iwd1EGskkK5CMGs/Cj6qFzqGXlzJst4duPUMIPV02VuFK49cDUadu4zvS5cCiRcgIu1ou7vL2+d+Jmesb+Jc44/fwS0jNR+aT5crSJcbtj+KelOgSEz58rNIIpCzXKGJCAJ9vacSPtkBr2b815t3zaI0qnO+3m26N/+pyGrz8iD8kV2J0y+enf+QqjsW6uj23ZObXEeMKa97qMVElLNAstArfs2OnymmEpqGCscqGgkq21FlTtNjWtEn/kD+N/cpO6dgUYihU6ApZFAKB9LfBMzJErwVEylfFw+UBNfxfhVk29l0raVLxAbt7fB1E3FFGzPrjivzbgdwTD69Rpjq8AiGxAmVt35oNDuyBvAJpzkrFnwoaCyiSHJLfwlsFP8R088xMylUmyBchCamUonzpdbJfGkt81XSDDRGdx1JBKnD04F1tfAnkmhz9F4/bSZcD1r4SqO5cT+i2jQd8m8vaONEw8VOulU6zf/SBAsZ4480F42DLj8r6sJ0PYzUrYZWYzZqrCgy4KokKw/pF6bfLocJJkALd9hD6ZfmubtnwK6OiPivNn3h+20bwajOTwYAZqGH9608ByfXYr+vo68Ax8eWr/v1p3iy0djftBd0pejYL24M3g+tmwHRc2G43hLG0/5A4/vz/WJ12Q8RWbzNxiNMrQ1KE5/Rv/NyV+zz58xXPo7O+7iMPugAftNwQ8bK+WozhsaFRPjQvTJ5QjS6r4AIaX4NcZ/jzxikyLkwXBLeJAFye9HfnAotSqbMT8Ib+HZ7FcH/6qrDzlEzJNMWgDfZbME3F85YTrkF0SXixEVf9JmzdznuQOZpu/3n/jL/OtL2M8utcKoz1aHUWfgMgzFE/BJL278Cqf+7LJxtCVRiI4ELhjPy1flfWoNNLb9zIMwopqzrAePMB4XgJUusP/dCWtj8PkMUV68vdHPWxHLJrF6eZ894XY0a31i2X6wIht7ftnrebi/X5QhgjIjFvYyYUeVOhoQ4sUych5gu+gnpbFMyLe5vrVFRB3oBvypf+iZ5YpdEA0KaGYtTmVbvjyG3L/vz54e/rYVMAYfKJdZlHFbvR7K/SwcxNzcbfrlWU4g4sBITjp7OPeOHz9jRWJ7iQaNUT2cDpyEgYNHXbfjWPepKseyh3JuR6jBSEvqYmFgJ+wZZKlD2YvPKC+iIvWDXp0X4OiOvxOGC73LlLLtGBF8sQoopx0ateUtsRhcTaHCQmm4oaco3dCmTMtV4/EQrPB9lOjky5Yd1MzzVO+H7g4vVCuezTZasliGd8R5njo2b3TukddML2httF6OIycVfhZymnwNOHrBE5jhj12qDz4V+YTMRvDWQ1yEuyLxvrPRVO6XlTzbUMManEy7pOiQjcCGL72RO6YekJcboKOSYfJ9IayTmSPT3XCTlRPQugDhXxxI+i7ul7Bq7fzIuIn+lfViy2DKf8VrVooJivTe5+Qsw5ZmUNSCD/8lKk6kKgN1e0tLjJzdyl2BkPtkfISB/1zSPlaYLW7593ebz3n00/bsb502Tb/YTm/J8AgcdocEcDX6R+Rel7IWdW+5NVJixf3onVgjIzPb4WHNDCQEwZHo5oWo18EYRXy2JceWBi9C0Sg0aHxj6eczfkP46zpYIaeDi0Yk8NB/5zwKYdHDXlFnNg/TOziOtcjW82XWlvyWyalIQF3qZUjeeKeF2AEtdhgMAJPGtiydIGrNrGt85m6gsAv+Jk09g0m8R7WM7mmhWHW5+BCVboV5pbNNN5rlgn93tAMgm935xqT6m1YYCjaHBqQQtXBWWM7WTQ9O/XzcncfwfZWJ3UAosYrdZ9Nym8jLXAgcsFJIvtApIbeTkNttvg247S9JeCAvtg3x0KXwo3aql9Rg6fHO/d0hRyEuNDf+4qKWanQ4eAQt6oRz66FGsHoeE2FAglCPL04ksx/FEojzIJLfUfbCEUtrKf0+1W8foiOxs45yoNqZd/iKROUbVx54Y/GFzRbv1vfxdvIL6wReKjq3MkVdxSO1VabvgtrS8u+Gc0EE795o/FrepXNXTVd19wkl3NyHstyE7Cnm6m75NcVzTiGozh3GWy5OimwNQFJkQjetFECPnDPiLzpTxReLBzQ0sG4pSsWDsgGBXEtwqrdCu5M/Jb9ke+F8WOZDicMeMbYYUsZGDBZhJr7XzmDdUVLbylS24lVbMqxaRJa4TmCPxG4rSaT8DNJAiOOovl0sZjwU4USz/2pzqb/+SwDDs3n7q3XFWDzPr4ibc7p5k6oeLNny85rERPqKbSSBXsXbmArY5mrpUe+YlkAixy4a3rxaI2UL4b0RFj+ercm4iqFEQRffhDJTgNxERpsOVdMKW9VHLZkd7TeQXbpvPxMnAfM9s1mn4OOgUK9LkmJCRB9N00RflDYtRMcMWWjIV3nPDMkslXKvheMzVm0gpMwaU/MWaacVKLC7LsNF3m9xuKT4YsNgWA75hQrxC3CxhKCf+Z92OEsMm/s16ybJoUDdX1mJwNPcR+CRbwPZv2zdbaceCULqqK3NK/i1/F2i6IoeBP8A1l+FORR6TWO50Dmk4QT/cG+cXwqfIYwmvLDgAvUXxLlYI7tyo+I0dF+11M6YMUXKaxst+mNNaoWdJOncq6/3sFfAbWWJviVsxzVrIu9vhRXbz9YRE/BAWw1J0AZhmYSxd4drpajNtJ8i/7TyF8IAPkx7cZe8y7W8szWS/fto1+lo2TcDwR8S6n0ag5TzQmjvq0O0oK+1uRPKojn9a50OlZqrgzn1ES0EcF6QLF4rr0jy/rGp4e0tZYNnc1EsjooWAYL8GffZGpcpxxKY2pcrNMdAZMmrXb5xVMCVcLRAIfOg3bP4gBfdLDzumaZh0cgQG14PjFTkwVrIhAgoSrb6WzgnhtGItWZMdFk+4NP7KlTgnCQwltBemokE9Aul5ddgahCdX4Vh4FneCn8OBuDTSAcYF7tCCrCuvhsAVooqdHcNmO+lbxIKifwRxXbgVxT5xUht7jSZ2CvbB/dhxurlY1fqvJzSlpRFCa9grY+kZRWkEZaHV/A8PlOdi6zuUTVlwNZt4f79hnYcnX3S5c92OD7g8JnPBvmiKOD9+QE1oLWAwZhDIhnu5EFUR8IXHETvCwhFjhZDp/lVuoETIS7NInutkqOWb6x5L4z2WWnhdl1dzXUtpLgtvLSk6fe33JoBFgdDHpHEidLT8h2yGBS9Fy2ky8Jk+EgV8sxSXvj3h97m72Yg9yMF60QW4Sk3Nl9XM9SdMwhR8xXTR+KG5uIuoondG2ycwx6WzfeNC6UGfSUleOClfoX6jEVXGDpqNK3spOr6jau0oy9iloIuBn7H8+bE+ApF1UCgflnWAGACrPfe2y3IWdIKzyH/wqcRGHRzyAK7icfr1kT7xcjaPObTH0w9PKy+BI9+zy2rJ8tKWqDWC6sxTf8KzYktWvoL4XQ+aQdF5iP567q6oFzeqHERUyYglBHQ/JmE5i5aPrbmmUeAIOWVToXMH0ifc5lQwJELOoIkOH6NO9z3/mq+jVe1fdm4Ox5OpUwNofmzOAEiZfy1/omdVZvmNh6l5sIZ3EfVcWttH1AVNFXDVaryV2uoEADMNo0iXrp28Bo+gomEfZuHcFbtXhl7lwj8sSA02eqBeVa4XaR0ypw5jNZCcpvYyPftizi7W5EdnFE4rlGVZBHXqoLVjh3IIRxBTc5n1b+qlvPT0Lw9A1Mf2SEwnRH6VrR9XSb4CXJfQo/RqhVyZBlyip+k5rVru9gU+ApOVZdFHzYGDhcUvm5kZZJgXyWVhO1ueGxTt+fqW2iuv9UBqDi6Wp5Owbv42uAvPtjMM89SZzM4H1ibd7JsIjuoRTcNlFFXMR9kjkFnmyFdgdOLEWlcSCFDMxgjBTeAOdx3ZKZDDGwoMcc384nRKTLr9pzIRdmAfRte+utLAp5K4JyDn7/CKB3BCCo18q78gMPkc+bN1txnx3DRmzzMqTY/rxD9hMH+uesN9bjPt7CJb52UyLnsdlFDIfIRHTgF+mQP93kGuOJX2YNcnil1CFojQlzG5C89KY9qbSDELC2OcyhPZpLW9Qpkug5EmykB/jYPXFhp2wdNOWEWHVASzhzC9uKff8si00xahy5EVuza7ySAvpfJ7R5OfdWPqqhv68X4l22VDLKJDWpo2vX29lFN3GxB42XrJ8lGsV8euLRv0sEnMlQOzkySWi5PBLxT364AfLDYwT7SgSebCSxVO4SHzMrAe2xUrAIBF563CKfUKab8VY9uZ854F7v0/bYkDH4oUJHaFf3KIa97s3VCK8qvc3SvmA9reSekD+vQRlxECFUgHBe0mGe5MyHY0kf/5MNHp6MU2FgwRcjT+5h5TW3GnuemzLbPoPSnUuFgEe8oXu4c2ELpxx5DXL/TDSGe56R1kkRt36IpWkIDmepQASLMR9N1hBjrrbt80Dr0uCSSQdzulx1pd00a4vpzjwLBHPoSqsYrcG8FKOSVPNaI4ZufU4j90ExuTyN+IoqxlJ+3fDS4nTRwFISdPM+almK0pBbhdWlFZShsEgWTblMdBUo8IOjBwqYU03BqPBhq4rM29gP64cM7HKNLBjvVkgbz+cWzg1hcGFnmrv3S8/eM0tHXOGfpuO4Dyicy39rj7Rdpm7px26s6Xb+zoKZvoWaYZkslK41Tv2t97w449P74yN/DUWswWoOhThgahAAMgBVv+m4usxxzbddmeIbrW8bbWmmqz/vVW630fom3fwJLZXQD8yylptWuDdzlbCFu4L+vumUhX/gGV6zXRc7xavDCzjPQkvUI/dUUrRS9VmA4pc4IWaZzk1j4s61Wn1ljAxdnacP3xIZwQOiXRHy/H4ysi6xLtLw8d/VjJaQg24ObKsuSMTmfVaXFHN0PLjLlZ/1w0ZbC+uszDCSOry9gRyg9T+2IQKwIEr8gpgtUbz9uaP9JXmsztiAw4uXKk3Sbo/N5VO8CtgjvvfI3ogT2BQTSQ9C0O6QZErv8EPXlFllFezSWaMqM0LoMbJo4g6/14sS+oii1niYx2eimI3WWAeZ2/8hN6CjBd2r38aNvzoR56wabIhueEdGbrMEwvbFg87QRhosVClc2rMoJ4oB9eHQvb4h6WOXzRY4MqBq+JVE7BzhEOue+FncnvKUtZxSF4Uvj7Xxw/pkP80NVDF1hGE42mS6G8RRl7WQdpNWDBH6wv3fxnzNLryzvwNPQch8OQ8tzFDxAMnOJStGx7oKcZGK/EpttexIkyzyBs5vvTlOhNsQRIfQCrTeSsbg+c6h3aOBqBOZw6i0CMl6LZO5ZTje/Isz0Pn+zXOpS0BFSPFeMK24rdFX0LWwpHmnPOAIHWteLXlOws9HPzrztJOXDfDiMamiFT43DZ1QLx8lSs9z1b3hF0x6cxVXbsVkheWpuARMUJuxdrpAmCDLu57UBix+ZFU8OK04W0BFvm9HSKqGfxcDFwCxrJCiZxy4ARy5/E8N61O4tKBDusjsK7ysFzHL72Sez5RdxMquEZleojvbKGma8nsejFOfvpy/56MW/1VdTQ/ws0S1G4fxASeJ9r8I43dJIt9kcS0PO33/IGOAdm86apXhTX4nsWFH8jN2yjAye+jKuTvMST3xpX4IA2SDMZ+vfSFY0ixJG8mPpzzc5uZpZ5QEB48PWInm9zvSvCYlBg/Motu16bZgvh4ADFbNt5bAscv5Gw/f8ogs8haAsIpx+oedpv4+2zS40g5hvcRtWnbnZ69H/YJhBtkAxhCGpT5/QV2EyFlfBI6R+21lhVOE+OZ5Xb76a69FhCC7XAmT0kECpS4Oip2c6Sz19Nhd6YYZ0Z4/FyjQQq0p6QDXiAwyOdB7hXEvqhtqcOXDidYyBNbvDxMGVGzxEim8zuw6XjLWxU+EGEBfFiMP7bkJJ+HgvpzQSQ0MMe9v92E/TScq5c78NZw0tiOrV3dKg5d5Deuz2wBn5tE0+TeW6ddKqVyubcPBYmBfOc2zvKg8D3BgMvtKyUeDO7Y3N5OSaeb32jdbjBOKFEBwhg3M1a0Mt5bh14rFYj54IYwJU0hMRQiymesdRXO6+VZiRBzOYOX1Ip+/WVr8FIpP9Gg27jIiSrtgR02nIVuMCGGq9z/eC6q19zFSs9ndjUlHSkzX6r3MtAMfFcD/GHnoovcHNj+lk1wuFHurXH5CjrtoFphdwWKjca8Z2yAdcfjjNyumgWAO3bLZdTlzcO4xhz7RbhtE6O/A6vR4AQyDbu4dmRK1RzXrFPONcyTZGhDeMXnIn5hrmNDqyY6jhknTIJsBZfaLp0KLk3piixEmCzSSd0WVNcfQli8BBmq0MFjwdNUZXuHsw37wePga+WT+2LbusCjYbDJPfs7u0N8qg8XuzNg4yj1XVpbG+uI9xu1ms66c9lTueadeKGV+oDaAIK9dRqqvCGt9VWG7rZAPZvL7Pyqhgzfc9elGHyfy4HAkYHiFHPAQ0E3P5wa9TgEdA36XjAaD4eYkGFNo+3a0a3JFFWrsYLMkGJ4aywcMJKR4lcSfAJ07i3tueuS9mTOfH1uWT69KKYxaOs1WXb8fd0l3eFHVdg3MAhKrLTmWHbfCqwqUfGoZB2k9uYPIBIwfNT2yKVn02zcjqqMqQUY3S2hanjeref9Y9vvw90KcC6Kv+dBgXOpLCqFNzIxp06aMTAGErZ+/cVYI+Xb2hGOFCfbRrxdgz0RlfKrep8lsubdIyy7CUc6Xun8Z/v7Cskz4sWbkvRdO/F/TOezjpE/72nOz6sijikw9cMWmavi2zm0wzNuAk3CLaWP9kMi/Iq4yW/tiReXGXY3lRL7Crh83Gyrh6du7bQZXVDtbPOaxfwbA+xi64j7hGu4Px3K8+2sRKE4ZUiGpMXSf3Ggz33YBpj9FpGEn8jujw7uIuBibebjySwB+B/1VjAYsKE1KkDR+UsEi4p1k6RyoRcZf8a8FRm6wOEu+Smn7iFHpjVn7DoF6DWPuryLJ8qyme38VINxoP0xS9Fyir0lHntpGV2Vdz0GGv+IOi2wQORTJUmbtqjh8UJess1VxQeghmEAsKAS3AUn9OmrlFqAoPxPdTKwm4WqTUurO7WopogX+Y4oG4hZu1S7zBW0bvXJc8kqQWURZI5fBrQ/cwWQt90+m5mvoSFih78ZdNAERXBJRBs2mUWnI6nQg/XaHBWNciqgcdy8lOE1LcwvIVzfDYmYTddPnmyRMX6mdcEIitzMgrth+1hPs8ZrNLtCrOIN1QeYswB5+HcTqTSqQrpCmPvMQsV/P9BjVGA+xVnQMpO08CBgErqHuuHOO6L2/iq/ZQZ8ctLH7hr4Cf3IiP0awA/inS5Fycxvd3bm3lWlb+VECVUUP0QjwUBeiqq0F+XiTA/GuXeZkbrUjKfDtsvTk84jsM35Ny0KM1b2NtTjxCVZ/BhbTmiStJeqdyZr9tG3wHncHA2TdXcX5vj9ytBDHdOsYLKUh8+aZFHKGShvFqWa7DJ+k3q0FwxASOzr7sHoOgEdg2A1UA1Fg55m/yvWhdbNv8GXJNaqiBFJCAL7FpBWpqmH7DLdEliQRcPkYoCW8Cvh1JZivdlzC8jJXpK/XqS6pyJHUrldPmIlhpwyHpdtYpghxGdonl+NnOlhgU9B7W12wPNI5DolCN/JcYoBFeza5LvMFGJ0bZ8u5+Zsnien/CI3p0piH7iBgU5FXKjVCI5kzE+Pf79V1yATjDdGKGP8nb0IDxMHx233ykhRE1ytR3mib7dKJF9ozvQScUTCxIbzaAKcdRdosxyIlRmoupP5BEEB4MnhwF/+hR156Mv09YN+Nu6/Edft+s97o3HOn9JEV5CTHzY3Y/Yi86DeNABmevWRCHRUzwaa9RYs/x/Zc0kNHtoQf69GRBLax7FfyoX8aXWBqqVfrD3w/zoQno/gv+wGV/La75KNu6KL7I3hpEThIyLeOto1uR563cZZjiWp2QiAJXVNu08TTuj8IyXtxQ30yuwjEQBvPjZgRw5zJEptezbagAUuoMdPYHGk5AJ7HGd0Xn6vW56rB6cL1z/djyLiDxO5YRF7un9TbqaebPbzbdkHCsd6Zay0qLMGYT0Y5MtffoYDvT9C1tizaX0EeSu5bJC0P6ns48HngewnjEAUeRZH7aKJtG6A6gLo+R5tqIDBuRHgCDgW5NPcYwXlMIIY9OY46ybim6tWkGvBn8V7Ff7qL4qJRt8vzGH3D3trr+qw3es9eReGlU7DFsa06SE2z2xdwSWwOfD60uxobif7WBtGEPvHeJp9tMzavV6LHMmWnTbhwuxyc3PTfGJ+PhtnREXL+GVXPsSOw3StD8I4Moj+G8nmWeISwqNMhhQtczzxexPraU0F4XQqDpHI6zBveXgbTjHPyO5FvrJunNmFZMQnr5RmGflplnoxamciCOtJcfOpdZPM26mf/1LUfxG9W/hZ9Rnw+MFt+tv/3KxJEhy2wPTsYEn4iFNB/QhVJbnk4f9dQwIrFvngKRo1huxCYsU98+M4voDrBfyg7CifZXS9egtdtETKZcYL/er1LqryveZrsWzyA48fWOPDNNRlPx0mXR7WVfFA90DzDyydkaiFnH3naWEnKDQvEi98ZScoPSwEuVO6m0lxcyRZahr4UnvGkAHUXYt8GdjwB/sP2FHaX/nQI/2d2NlBHaZF79rzbyZGO95HwvOO850AeIX8vw4SL+jvuHPbzM6EM5Zram5yGlsCPCcZRTtOi+pNMxAX+7EYlLO+czoqin/NocIUt6pmDlX0xfs4DgQLiyaOyYHwZUS82DyJG+TphKtNjBUmm7U0ODpuKabencej7udmSOGX7wXniZQlug7tvPsOpN8qF+5daBIt3yiJMpH/FebVOrHsJbGKZ48C9N6rvGn1TGR11oVSjgWR1/xW5H8bsk+R/pACtvsYFF17mGFi59Y4mZoLu5bpr/c5X3+SpuTqDxVH2Xa82iOfgpNqihin/3TJdR6VSZl+qlWK4rGYY45ScVDOXzgJNVfmRjFjvqum3o9atHk8nju6ge/XApiOcZEdvgMTibFe/PfG2PwdolMG/gJ3FDB+gFrXgivBPCOTEMe4DWz9vGcAaX8DIB/UIE3hOsr9D+CEEdD34eZJaf+1IgEm/z7rD90oqTVl5QSeWFtdQz2TjbnEbuqM9qYALIIfgc8cLdG+rqXbhx5n1o4xbiYyhhlPco+QMgII389Z2a/YeNBucWLM6+GSCzQ4yu+EV9cu2dFqDerupTGvAudwVJ1vX87KE+2eV+smE9/rYHQyS4bc1aPdws4gxUdJRZrY5Rb8dp6rgsfCCDRn9UnzHoCR5dW9Ovpe8E1uObwpl4I2AaPK1JoHodM2XfdHiRdctQlR2N80YhPcwcbRk5KRh1LT2k99bEJtaV6Ep0rFO+4Giseh/OyOpp/+BXxwA6goy2pftat4RreR+Om39aFdlSNKW9tEvpLE0ByYGojVb3S8ViSzm52KvjWsV4avgqaTiyTOKiZdfO7NHWtA8M29qdY0VWTSM06uBvMQFJOwu9fgVP5l98guc/iBQutLw9YVk0HkAcLgHhhp96G9/JqzVpBVC+fICBy3N5XQtus+EqwzklNJwDDQizDZaLi7C1wvbdkuPufMIrrtzI73Uiz++5CVldd0AdbuXejRhjOXppfICFZEu2OYJj0VX/rHDpX6L5tRUryNbtOpaNbvdxX0NPyQm+txW5nLHm1RQ2YpwWDpbYMI1pWnBKiwmagKNYBmDLSIBQTCCIZQRFNcTpimkaxNQ9c0zDDg2Kh7J7OIw5NzO8eas+jSTECgnMWNPP/CWup/e1etvNubDgCL8mqIgVv14qLbKOCWKXQYYHlw8K3vpmUmOzCjnB6O8AvkZgYiMp0wvvk7uSgZvligJB8LBsdClCRjEohOD9tdXFPkjFEh7bjIHOdxW1XSQ9Ze9V7+x51l5CmzseBNQzQpG01jgPfgakPEdF73IpUY6HOFdxaob4QAddH5iPkwnzHJffoR1Sfmsx1Tfi5DHkGMVcRRSj1l865LMBXEpeO8Ui+Kk8J5AAY0y3vdyfDFmh9lF0bapClwF4q/naaZScI73VU8LT03mXzZ73JLcC2/FGy/zrwhl3CW/X/bpWjYSw3Q5xpQq/0lHnyLQT3tP6pcqNInCrKtNr34Pzmh54w144rIUTT/jri6+GY3yEYMyCGSmEeBSu5MQjvSsZ5uBroOGNeZrxBUZ7hIE+DzE8H46SBcxrq5BMLV5J/6sO3ovmwu0bBr4/P+qIVkFoY4+/jyYi/7yDhG8gAPBTlllCOal71+GcEfFZrN05Wc13DAYq1xZ+E0BuY/hbTRqC5i/JTVm4qd+MYjFESnv+J4eI37mCebTJRtwnTXLEoisxxJ4iZ9QGvxlEkSbNv4MA2EK9UIhzDce0R6xE5Y67kIRZ8Rf7f5TwHpAWNLjuryyk5XxxdQgJnEZFINX0v9gF1AD8LjzvBCS5UYN46+6kFYfPVbwZ9JJ4MIwisH30vXBNRoYzCYIs7/6e3Q3w7xaBiPUNDxdo6R9Gf+snRMxCqHLcX6KUcZ5BclZJPNz3WxcJzviVE6lrKFjIV0W3tjphf9k5nFqq38l6EAfp2tW0YETg7uR406GJU2Jm/YvSXx+rFIXB+lKTXwyhfw8O65L67s1dNtgP5h9+mTKMXZS1YwWf6wsXrgrTFFh58jhxgD8kMNnAKYMLFp1xUlT9zRxWq9osxRuJm5a19BHLCgW3Bezkz4ElHosXmY8OaH8ZNwIj3fVXFERgSlNeDy65fbl1nuNEAWHirkqD/TfmXNCURl/KQn3QkyqrhLcHKNAs9BS5XgA0+aUXMhWYZZqdf/kbWDlWOthm7bBGu7ruN9m4QiqthaB2WFSRL0A+/FPOZ4o/FJDcwqCSQwEl8O6n9IO+piuNSC3FdFQ+ZROXW1KkBEFVwNer8oSwVPvBLVd9g87w4roMcPOeNcuDcUSucx+cNTSjTZGY/ve8MqW03p47pPK25/vb1Zh3kxa4If6/3L3X0uPGki76NHO5OgDCX8IbEt7zRgFvCO+Jpz8o/t0aSa21lmL2aPae8yskkSAJVGVlZX6ZlUacM8YDh3HfF/5gKNZpg2OluvARwyYyArUzYA5MSvzrUCGR/ZGVZEvOfTQ+vs7ossrGY6BHLwAlPQGXJWnFHuRX7tc1dEgEcXEvEWPtT+aPQz2BaQI5PfAHQPZXmmBRQmFkTk44flxZbveR5KYdpU/kU2cgNDEE5qtPng3DsMWUe/H6krj6TG8wH77Fc7OQ7dM8wlEDVP7KsFIh8zV13nztKuSBfk5MP7Migjqzw66bkq4DRAN77uXKHzprr2sTN0m/5d93bn4+v/Ce/yg/qbB19rSnwPriP7L02Nu28sBvuaUjMfikbxajh/rOVqNvi0SKD4wRuU+6iMJRvPSQNVE+BfeRfd2YE91j9t6zcz61/IwVqfReaXVrrV3fJ1K7+JZ+PLbzq+cbo+F+fkyXrDCJWYQVCdjz6oNyvkYL/OTMpO/kw39uROKGUk1pz40Jv7jIeCmypdDnOkk26GfOQLd54y6cBBCUMd6IFp6kvcGzW/hWlVeyfxFFkW0Vr/KBs7omoi2pnCdFnVbtIVEKDfIBP6EzZl9DBX+OB9Tql2H8Bu3W7fv3/MYbf++GAZ1HE9Lf8mKZky0EVHyQoRczYLXebOfg3K0H0QkJKo6tiSLoR68KHESrn1y6tP2gCfo2JLWdvIhXxsNddPvO74WIc2+xo1JePh7qLPiZcMe66XXZhsrXKFhNdh22fJ+BThLWrEdqPpyP2xixX+lfXMixigd7/VK9OfvCCh6+eX3CFt95o8z4ezvAEzurISru4ddzaYFihvvU82l758BiR3LwPaXu0ze84gtZj7BQcxf59qiKVtcYII1omTtVVmGwe/3pcETPF93RnjckTkjYr5zPTxw9VHug5YBgdRopZN7mbMA/LVTF8mPmD46tDFuFucfeRLaEor0imYiu0l8bEQ0ji7c09taKL3GWpKbB9bV4pWnQ418CgJUNjJPVO8JdNrcWpFmsdEwSmYlBF8kXcbSXrlz7HC+bocQkDtQCE7zHA9MQ58hxs0hn1NQ+3ECzuxqx6qij7VKq+7/KzP2f+WPiz94RSJajueIBiSRdIHT09ZliuRg/vZSiKD4p3SCr+z9AnBc0RFPWLeDK7aaWItdC9uoH4Q5R9tlNhv0PBP364pZNS3Z8fREGlxD+PxC2PcSsb7NlukQd9OPT2+0bhkAUjEAXexPkDf+6w/vr4xt++3q/V+lSfv8JCn1dK7OqKL8PB8G+AYUELkfz16Xi14d9zsY/QwCo6GCzpvkxos/rG1SlX78Z/GeHb/nrl+mJVO1KyUGE/4NAvs8qatbs63tfF+bl3Xy/MJfRAF5WbVRc/2cABaokah5RnDVGP1dL1XfX53G/LH17faEBHzBR8iqmfu1Stm/66XMrJP/8/eYedFMV4LdLP1xXo3nIEjDrvDqya9jM55H0j6vQjyvX63JZhosY9Jcmyrri2xpn07ekB2Go+/CPpO8WsKAXAhqaPko/TYyhT8UoCGATIUqyuO9f/4C/DV3xd7IBSlDfUJiiYBKnMBz7QeD3H1b8N1xw/QDHf+YDFP5G3v4mPkCx/z4+aKs0Bb/5F3wAXVwAQX/CB7/+9r/MCk21Zf+4gPAS/aPqvw1Rt5RZ312Dy75V/V/jja9fN9Er+0fTF/0/lvY7j3ye+J8T+poL/vkDQ/4+h+S6dzb9mP9vCPPrB38br13c9Y0gYAy7hA0E+O33vEbi34B58Uehg8HfsNufyB3oGwT/XXLnZ357XPS2l34Ci/pH1vuQG7ABB1/k28trNe3h2sHXhX2KgOgol7b5/vG8TP0r879PEL2uFFOUVhetf8OBRBbhGeCivGqa31wHVfPyBFy/mMSuTvAIGP3nEuv7uybLl+8/EqK2agDF2b6tkmsedgQSbSDV/nVsP57X9V32d7IDglLfSArGiB866A8aCMG/oT/zA4JQ34gbimE/FNfPrAFD0N/FGPi/F0T9etk33UXFrvtPaZBGc/nhEOivqKh/vZq/btTfctV36dceFzcN5bdon5FvZZrPvyTNOn99/Q+8JGAkhvwZ/31f998zAxCLn7+fWPj2Vzjrj/z69zEVAX3DEehLmcEEQhK/Z6ofW/s3HIVi3zCc+sJAH7GE/ol2+9sgDvXvOeonbfObhR/6qls+g8KY/8C4P2OTHxrxt2tw+42Gqtrio5lAVkU7JxE4CuGiJfoFyLxv81b8ln9/FmHw37mcOAJ/gyn8evG1mtQfVvNnkIr8iUDA/i55QEL/T61edJnTGZhSNAy/zNm0VcknMZS+3tpfb3/h+jaquvmfr+tf3PG/FyYkBP75OzmBuPAlRf5qrMC/xw4E9mfYAcf+hBf+LpBKwv9P8cKPnfzbtVf7uGqyX65L/+uW/0KN3yiURH8V07+X6zD2J5LgT+T43ycJ/gIy+F9gqqYI9RqjBd6HDu6xb0nTr2k+Xez2rcuWL8Hyp4bJx2vUNOtR9Y/LKPlF6qfqvH4WNb8wF0F+caYLDHyxxd9u0WLENwImf9Xmf0CVJPQNhf7zD/7Z4ED+xMtxWbf438U55L/nnN8jvbTf/wxW/iQu/qn5+hNanKZ+n2/frn1eDVlKg7fg7oBmF5wCLwFxUfCUrl+SjxTA/nuQ4B8kSZ4n6Y38c6voV6b/n7VTbgTxDb2M6B+Q8g8YBP4TFwn0jSR/fBvYKf+TgugvAMr/BYLoS8R826tX1WZpFX3rQVF0AbwfwHtwwNO3bf+pErGU66dO5uff+KP0oqTMfpGqLfsFOEk+6u4mwDcIGo5//Nmnf7dUoqhvOImh5O94B/+ZdTBgjPwPsgt1+4ld5C7vpzb68MC1dy9StaBF2/99KPMxStjrP9ei/TcDmF/Z+F+Jnb8m276TCP7n2+bvFlgYTnyDyP+UPujvmA5Dvl147T8V4M8aEIX/RAEi336cCPyf8OCfzvAvQKf/G0ZUenEaKKr7saDAxV+471d+uWTkUkyZbT4+2DqbvrPjv17w/yp//u38Qn2DCQqhfhhXf3CZXNyEE/8KMf3ZsRDgMuRv4hfifx2/qO//n7AK8ODfLqnyq7PsD1iI+NkK+1N/zDf8v8Gh9udz+gtnRYDkw880gv8JNW5Q8uXMjeIfd4D+tbLH0G8/zi6/kwahftb1+A+Xxe+c17e/TdX/TBkrm/t1SjKgwKLu4u32c/IEGU20ABDwE+X+D446bv9Mu6bU54Dqrx1p/Hsd/F/fDdg/Wf+f1/l/ZBH/dJB/wcn0/yS6B8LxAvdfb28CQNv/olK984ndKFlaBXERrw3WOvCFt8eoHh/8x/eiFwJ5xquLaI4LqbW8PWxovtsd8RVhMaNaHbYKa1d9JOp1iAC7TUhAhxPGCOAlXkhjXh/saZ6ftlGNYumOC4vmdS8JpIIOuF6CO2xxi2bisaj269NcKYAG5Z18msA4ELVBnyhS501uCm3URIlmUk8Ycx/pJg3aX+jTOI74GnfikQR3OA/jcNUCz/MpkOpim2NGOIAn8hQzmv21jhAITdXETI+o7fF+rQaTlWsVzTqFttmqDNkNOTYNzwIQ55bqTweE0aVQs67DZghZ4N24Xt9pb1XiUi88nmFVPnyAEBNbcxve9Cy00+GkCqJKL0vfd+2Hj3nRM5A4JkL86bneUmcCYawwHcyDQPdLrMd+bhKgzj5V8S/pQ7E0pe9dwDBWmiQqLwhvM+6pADRFZt6dcv1cMswAiTOZqDN9sz6Rgq8OveN2twy0usVOh+HaRqoEnRiQGRUDIIgrhUbVZdnLz+phzScHhPhPMZQig5v2bKa3n0aAGgjHwuYjQw9FxCU9qz7V9A1w2Xud2/bEae3FpAROEBBskOIpRQ480wfVpNJjlkFLIyZLQ/pma0hdOv0d5IjA1kIHfZbn19CIFUEfGNbW0cXzQcLe6XWSSdrV9TvM4M2EoikI8K5MU+biM4ir59tZpM1HQzNX6HKwz3nayGzKgEFLoMmn1u7spAouIqCWMBOgD0c7EjuLzgIUYmRuDWdJyAKy69DiMZzsO2unPaMCJ9/1lbi9joUP18R1StNg9kY+KCinGjX5VF/rQGS2lt7v4afTc0xIX/WYmImQqb41ovVrywxCADnLnGHriabUGl3ilOAcQPnBP5l3fZPrLY8UHce6tUt8pb0bMRTWroCMByJQW5blwWIhKFtYZL0jOLdRmNNIoDICYGK72gpJMM/NS4ZziWtUCPoQF3Fk1faENEAZUpsAqTwbXs47tOWPdtIz69zPNBiQu0RVXSjaGa5QvJ1RYWQPClkft0H4NDbzZhGka4g6qs1sdov0ILsZUSKTDCGECfPiw2e+Je78TA19fmpkSVptD/K+ioCb2Bzhb9knOj6f6XRyXRMBuSS94ZMNsmcZ3kiVW6ppZa0pTrAlXE12k5WDWFTPxe5ECxrIbrUZuH0V10I/2f29bEVPOOCmN2mjd5gPQD2zCkm2LfY3opcyd12iKOiUGDXnKBYUHVkK3tLSNGmM8U3Z60otdIoFwur3sAqX3RIEu1HiFR4WWvqJ9MWanNvcStjKTDJ26v0on6PLXTuWk0deSJiU7/mIjvLtEM5Vp/KKAoMob6d4f9oBrPnU48l0gRm6SMknrbeNundTbQXzkhsLNoyvoqREP8NgbbHN0qXJpe08kagOhKM+oySUDRCgK2Enn4KyGjXg1n3VFHPF2BZSnpG+vGfkicvdpo8wyiLRYUpOygvXpC97GLAhqRDHcw19uVRzLPBw1FmBdCSsm69b8Gq4h/044MjrNoaF346UpUamPmwQEwMk9kjn8NbgL9COiFsrNBUTS06B11kfvQW/496dZKBHZ9qzoqePjuBYEnusccRkYUMJcK+yS6Wa9KeUO432Cg03s/mppahq78PAmOtu1EwiFwgas2VlOSQt0zQGQaDCaDipWwRqENV3b4Z9Tnx5lZp5MrvJ77QnvP58w5IjQoCYvJ9rzIGuN8W9cbOSFkLQXVJNcjAi4hqTG94Ypa1m3EZJ+pDj/mFEnXwnzxvqYiQDKD5m0eBRHOl4RezK54AZiXICZiFSjs3h3o22rWCG1H8C2QgPClc99UzF5BvLjV7ZZaFMF6JtfcrBF9SudXd9i4XtkaIKfAOGiz+IZhePoVqMrwmUxU32ip9CGmgSx3svJ7/ONFFXEI+ZBu7TluXFm1YORxCEaWCOEdHRFmgxTCLLq8Z3FXIDnnCKwCisVX3oXn2jQZj8wS6TT8y5nwJDiYNxvqb2x/SQyRQ1VJp6k3ARiUIe60nHX/LQCLfH44a2uU+BOElOeBeli9QBS+IPXZ8x1zNkl1pNRmVmKuMFK3tRh/5SHkFwyN0+J/GIZMxk4ejzJiyNyUjbu73kl5GYjaZieUIc/qCr7QjXbXYgPOsT7uh5XQpxUhJ2k+IYOgYFaBTwzc3Dlg3kQip3gwAm3XrXmmRHc7K3XgN+8n3eUgOeArFOocR7g/mxe3n9sSSSY8xWtA3EkpFktpznRsjTGLy5KmdtiuAo7QmjmNKet9h4gOhzawtSTNcOLZvEIXmt6ZirOIzp/EVhZev6g1HktMsh8jHChkDqnVtB8bR8egSIJMVFxYMMYSn2oomSfEZmlN5GQJD/tMlJrFLd87zVUYh50rvBggJ/tE9tTnKJEShl0kFeFUQ68C7iCRJ5qWHC2i6ovS5gLunxx2MeWCOyUoMM6eI8nbHWFSo4QWGH2PV7qiHZfa/frXUDyCWoI1XM5K0tS7HngMv2Md56nMgH/ICz10tqW+2gb7i9Y9W1Rrfgjo6fIq57AD2T59sGMplhOyxuZlFJeXTAhIFELBYKrvlIda8jtxu/1uqNQ0821CHV2Y7KO8YjEiYMrZbDV3gn1GTuxqbV7fZmUNdp8IMT8PgtqWZK3g1hZCA9t03SR7nbAQp0IUVsOPEC8TuF0106kuLdx0X4ITkdWQij6/E6luROy5x0+slV4EkJPh+4sE1dKOBv/6bWRV6SmANC9LRiVPE8WyELmfoqi3X7gnMdtRYtc5BLLqnUqPmyd5f2BDed4t2Q/ckxQ3Aeyqx2Tfs091nhGrL0RyMPgwJsZCuTW+gu26yQpGMmOLRffyrMRun7viixDD3MqTZEPcJhZtngPvqCwatIa1gZCnQwPWcyOLs47tYUqz+93ZcUttLsMskeaW9VHlYB8dqbSlkjqzs6VW58Wqo6ondLHzgudCtZveImZ+aHiVNcXOCtouTnXA6BDIskD+TfAxSdWWvC+ZT4Aup24B+M6Dn3t1BM1+Jxsg2kwZ0HYetkmvI7141udp7P2fNmRheITt3IMGQK5Nq2bq2DPEIPk5AM1VrDUF5UjJiG1LzsI7ceJ2YqKqOez/2ZigXDDdNKQJet160U27BndOwst3nhodBnLjCKwhbl3uhcAOS6XUUtWbqvkruepDJ8JSXq+zJNjK7YeDnIbLIjM5Z8089ssdIlHEqzfi+fLOxms+bE1KFCQ9686x0arVUYHmtI1PVUuctFpu6O8PC2HqydhIxD8PIUWeYO0S6aPJXmIFeLUy9di0JZy6dRij54JsTzTdhaitaXBX5KMygCIdyiLawoGqkLGxF8SlPj5y0J1cdEbGh5o9XX3Je4RINcOVuHuER3XdWNTv5+IQTbIILGiCsyZQJJ9snTJI2udcILwbk63XdjROaSCR9ZQObn8tpDuXAAUlxl/ZZzhSSyRcoZvfM8RSAL82eCUtVOaRIqSTKIXWK48lMfMltI1YM+OR/iTnXnmyrepdBTPFltl5I69t10A2BiXPYCKVLm5nUZsDEuEAgQk98fE5fnD3LpkAEbT05Gbs47RyywOXfl7buw8+RfxOuyaRbPci5cCHKwYh0Uoyky83Fz2Y4fEtMUg4bETd1opkNx10O58SzCKwpJ7I5XlzWKocxsewCseqJXAzgJiNxfCIgXD92bXpstKOEgOg52NBGJcYWSVNxsgZwWRT9kH+URviReDl89FjZ6AI3dk0j/hMpkpDgDszeqT0SqjsVZUzYt5wmzE+7a2XGfyjkewNrJxfsEizVNdCGV1CMHL9korgpu6AXnz9JzSsdSjqc3kvJNtE1/rcFuyuZLTgomkDB2aj2xXgGG4/58qL3aJ3E3ai5bc23uFgCuAQHARPuaizrRs/ILJt7wU3neB0+dKWQ+RJScxFSz8qhs9RsAWbKxccpUL601DqMMoBeQEg3VuRTQciwIXueherzVzI0NBE+qgoHst9iEjOCg7AX1nctOLCRdsFHjyEd1iVd/ez6MNmigA+vkmWrH7ZMTGetBhQ5t8144uE/QYlfMmWBbawYpwUInHaqUu4PbPCpDcQsgL2wSP9LzuQraHmrsolkjfW9i8fUWHiFQdWRtOy7a4Ut8cQEAqM9UEpYn7vrWmD80vZkIdxfeQ+gIdyrncPO5Zl0jp/Osbrd3SNGAAu5tF6WVm8jWE8xcfNDIIZfcdtP5OaVDQjmfXtO20BbAggrnQwYkMF3a4okjI9zf4DFS29CDbwM/vAV3Jl+ro57NZaM0rnsztsetUxOSGuas89hOAvlka+nfyvuGH06l1sGIzEbsP4yklyui9oz6UxRGRi0xCH17ldXLImu8+Di0AJRNzAYueZQMPJFoo5AjEpmBsrzQKLEuqGwZaVLyuHqrrfv92gDoGreSrgHzUmyaUcHuHhyS7QPI6Lhb4pLrSbyX0zt/qa6dIdslwurqqd23LZxQF3ris3mskzp7SR74pg8j6NPp0Vf9gg38IYk2Wm0bPzXBqol7NAjNp77LDdncCBaMh/92Lx20zrxYIKQhl3xVKm8PN0a46l5a2yVVmG0dREl1Hr0mCkgb2xMlfRCvTXiNQ1/vpKYP1FtVewPZFhmMm40Z/K52d0afrL4+HW8kggj2b4Sy7YP5NrEdCobwUfrAGC7Ojg7mbC/7aGNIHWEa9yUobGBWboDScapi5IO2rkGrQjDO7rRtNVJ6u7H6MVOSkQWjcjRYuJbHC/HqE9+7gOqh+zEERhfjRkIj1oAdd0h8VSxJ7cuiXRardDIHkYzv8KiWZuWJyzhpbyIr72/FtGb76V+KvHyqSzhyRONXwaWuaDLYE6ilbdVOewyLzUyscWgo26fAyryWsTW69vBMmgIU6qMclp7/ysxd4nJY3CE352WrY3ZG1/gLNVZH6he03JVBdOP4ICMkXa6QlsbrBZ9E6eHflbY7H0WIYmo9XOb+e3AT4S4mMWA95/lmqBEVyxc/2jSaU4sPWyyVmsw5SLawVgzGd0+IxIKJaZIVKt7za+pVvp56JL8TTx8qWRZIKSrfHJvUUOF+9+3TmKjufY19909Jw0Kx0S1H0FplmzG0OF53iijP90DawsWnkOi33HyvXPXu36WEg3Tg6jkejteNChFofEc73gr60nTvTOcsLWSSh60z/fGepRtOGXLxqazkL7QXzf4Lq11aJzxt9LZSNeGXpTHvfVRP6OSfFhrrMEjtE68l16pLCeJbjypkx1Wvg1SSZt4iZkks+uUbBF46l4UUBWLPBkVXoq03Ni+ugnTbGZPK4W5K8QxTrs4gB1ZsgDiYmB4y5WHxRn4erBSMjCX7wJ/VF370njxdNpFZ4p3E5XDKdCuzciakZaTZtGjXiGqTasumVZm6kqsQtgKHwtAXdzv17smbaB1XU4i+zvotrvfKXD0W5bTZDiIS8u3jvDlqOj/W41Xls47OZgkHvK28o1VQn3oZZMLLvjVnXh7v8qHEYZPuWaaYSK8hzLB5bqvsnzz9i2ZDv16I8JItTgsX7fFp0KbEekTnT0seKjsbzEN8CZh+NA4b51TeLrc+m+gsC2YRwGcIqF+hl/B05eQoL/fcsPLT5Wpezqlgms0N6OUZBHkEgy8jstRFkkm7Tz1FY8U6GtVa6RBdWFpmB9YkJwthtQtwkOaTcsLhPdLQHXRh6B+B5FJRkpnEqDYhKavL/Yh6lMIts+6KCa2hRKHx+F7ST3FjRkyKD+C5NIVLnvY01LJ2YbOvR5FggZzRkc62ispprtVZ2GAqbvu8LPbiQF+8Y2YKSLjUeC6rtAKSDviB8ZI89Vx5TBYb3yXejN+vsrqRSXbwXitdbPkCjMn5VjiGvHDu5J5cFx8S3fu2g06vuTBIO6hmWV+unfTcXmUZsOL5Xm1OVMzachrgzoCYY7aaIpB4mYAl5u1AL9xYlQi9sD1itzeVedoj9HRe/ILdXy7RZLJ9C3ZS8u7O4Y6TeJQovpqJptD8myEmAlLXY+UMRYxbeKDsp3gXmCyc7lxrirh2WSxP9SRnDkswoxPs3CzedVrO/E2z5UTICDtCHkomFeSoBY8LAmXWdBgurG5vdC5MZ+JXWOxWd0XV80ABOi5FWgzPN+OVm+GR8zUjod6oto+sRzsuvqkIOvE2IcKat74xZq1j7hofpoiyKV4xNtXzndTtGVxWZZf7dBxOmjeKVUPD9Hi8T81+pzbL21YLI29t1WxCYuCsHrbnfYJ4Jxc6mdXG0RvVibaDAuprNdbTnr00WWzFjvlBh6dx8pmMr7xFE/NN05BOaNsjNLRLAyW3wazF5/Em7ZKADY12oVy17x5mhRzyJjbQjFEw1HN7PYvVvuRRNObVofB95R8+9bwpuurwpc+QOHcPVocSAgOkj5vSiWrj5oBK7hcfoUjcDicuITz4kOVrjLuMssJZHnbzkNq+NiaPOvgmKi8jFbKaM5qG19Mjm0cbuCipTq9orwBc0Wn87TpvPpiwp9mi5jt9+V5vSwf16D2JaxSGttqML+oH6UbUMT7NJ49w3Gsr4dZs7OUhumnKEV4CD4P+OepGNUSkUGUVU+e4loJ5tOjN2uVeU9DiFOdLeT2aLX8y+O7dZZ2R0syRge8vUJSN9E28MUNiAbhvnQ1E31KSF+XcGCWUU02iyb0qiN6RfAaQgDwIhTsHvgpM4ELPk9m9L0wCwe3JAUeMQCjPw7tB/uY30tkbR1knhALlTdT2gkPxlBnoU0IkFgOJS0kbCI6dXAnD5HrbgeIP2wOynl16rdpJmOPi3VGy33P2CLDg4MwjizTfA3V3stOiooVN6GpCQL0rv85L5a65/QO6nsvuat5YaM3O6JGD9ct5xNM5k2xdFtuRFpoS2ZY4JU4K3BYvHEGL0BAskGABEA5wXU+oh7048/aVsvxTr+f//hPNH/kWxLfbb2I9oN+H2f9JNuhFvG8U8vMJKA5/Q/4bImb/fFo/nYHaDq1x9ONCPP/2NPTnI8qfTjH/i/lZ/7WjzgD8h5O/mnJzlWwMn6NOOtBsC5LpaUYviQsuNPsFh87rxYO/uII9VIZWxgQ0m2WYdPCEEvJ5WNVbbYttrPxq4I2hcfA985w7UL0rl0SEm1Tki0yE57hT8YyDqtC3trB1cfA+9j0otMlKlgr8+s6eijMlN+VlWTJcjCiNzLmrxqK7XKP3a9CFISmvZz3YFh/+es+ktVrDVvpUsna9IrcUSZFHl5yPlno/3+ShOy/scdLvxym/H8H1+wo+Mx+DwqBYrt/XP+79m/vzz0Crk7a5xtNsccW8n2KIh76ypYFJyZX86/d//Pvr/K45OC5EyW0JpRKNP97Umrx/pU8dg90hCvvj5FeVpZb4mjtb/f5eslQusYhdMEuD3Nb73dyuZ6zf6beGoCA7UpYJSx6Pmt4S2MIS0d2u35wx4r3Dm3fh/bCWi/7392cZNPaPNTkHsF5/caz/apzW8GzDf3mfi47ts/oa52/p/UXzi094WNNfVnPR/Pb0LTFpqUX+Wos9DIAh/JnbxRtKk9woOGm15o/3+brXF830tlkTxCrj63s2aNvGgJV5tM3w5Pq3xcuQ6pqnVl+P5cJDvcxl0zEhzQ133Q1P7TQPtS5Qy9m36y6AinjkY2cqChfVPcXi/tWTr9VBvOV5cdhvn2z9/snnX3/yRcc69eEm7qyfnmyw1LWGjOY4gGeVIW29l9UpW+z8Uxr/hdHpzr8f3XdOx61/Nbpfn/pXVkPn/vpT7X+xGp+ncgOXtF6ZitTbE6kt5jDmix8ZMxIpKEa0PkbowoTUQq3pQ7Pp3vGF+lrlz2f378/SX9r76QvXNcWNb9Rs/H4vkR++rgfj4ltAgfIjL2r0uNYFivxnq7+od+R76zXW6z21/ub35AP5E1nzQ86x1EfmuC9LvH5zffdLPn7kJWgOBw45KdCc/Gk0isULbqZNS262iST44Gz+tWfAbs1viBxcgI+himZJpHcZv9Z2bYeXKUjv+8qKfcU3rdIKCBY2DKUDnFN8Wmp7wrFXaE88XlT2FFRXgGmTT8qZju9m7SCO8WkRSVIw4WzB0K5j06zEYGfAS3shCuRFHmqaB8sI+nGBI2sM2fJ7Vu87hICGM2cSQ22PYal+eFSWpNITQfUYx2/Bg+OcPGlZrQ7MOozPFXuSUv9e0WCCSXAGi3ugnq2gBsgU4xXoJOqZ4CjhtsqfHhmx0Zblsq6apLvOWiVF4IeQkVooqaUd1aQPjmA2cAZETFJJjCABR0hr1FWp7egz4rzE2lexqD26m+NBLGAKoijecFRNt8KZbJXOYyYH3+FoA7bpTUtvEDG1KXa/LN+u6alleN9CpDvu5+lQ51b3rpE+56qaARTVaKT4okKcCFTY1YxsOklOwcfHihzqaNYq4EPSuNxT+YEx3AIUvnkje4pXuMohbjyD/nQ8gvN9/t4crSBeSMABykURXfT1fsKbXIFqftSTOOGyKLQbeOczSBXjeV1XLS5mhDaeYzc/KxMcUCBnCHdEvvTzjUZfUv5Zh0g2+i6RUGoICOKlFnnq6eBGKPU6n+96d26Qxp+VDyJc1j3AWNQ419meALXQJJqLbQaHOk8c/KgxitvZVI8zFIlZsEwWwGCN4uFDmw0LQ3LwxETVNaHAy+crkoTjEKA84dBrG4Yg8kQHrMRty1Lz+IkUpAe5BBrG8VQcGSUSdJ7EDlUjkG0Rm7R3rn0ouwelLywyCKIh0c0U6tN+VIpmrOtnTSFRp7G1RGOCe+m5saTjvlMutCukHCEIYt/723SNSdW027NhueAEi+Sm4BRzsg/Cvc3PHs/AOTfo/c3eOnEc2qeo9Lsp4GogfaiY3I0n5MzggFIrCxgS+PftrPrRY5WCMWxvqq6hxc6pjR0wuNC7hmoadew9HBRGA92Yt7oHM3K9hjniIAha5fI16dugysS1R91CysWneFE8DGPHOthzZ/YDzYgNlV5RG1vZRyS4ryWUNC4AZjB2QAM4O5OA22Nw1O5rjlj/3B2DdvAjgXjVovpb4b6i89Nv7iI58fJuWiZioQy98XtKqmCUPTezj6yN+uzgGYLEiSmlVNVfaYnw8gqbZya+ZL/swV1nKt75tcLPnibktm735JoBhNIrcMFOEbedNAEc7NIzGvF8HG8vHy3v87XCdWmNIT2hrFcYCHUql5XrA+sIiew64pzbYejGtB74EG7V117IIvVhew9qz5tYClNGNVqEFbQO3z9tPuvhRBbS/LFzsjeQmMzHsAM+Jh4t6g2JJdyFH7OKm46LqjZ5hzECaf3SSjE5/BpVM9IYXmLpvZ500yb7k5ZElOxe9rvpJfMe888b6PA2Enjhdc4MlXdUSujV/3Cfv1ubbtRW40DZRDdmzZj17vUVkr+Bu8VaYkG4DFllTYrKPfC7tGuxu7SaAZsxtbGTj/qX4TdUQGYTK7oDrgmRmLbXzlDQCEeIwRI4U/Mm9n1bn4zehSymZs7dJ+8PdZ5rGkE2Og6gFgW3KN4yTTDHoTnz+/0oCBDJKDG1txZPjpNBHfW9iJ4oCJgQgnWgHYSAOcQ39ePSQyb1iokssi+1OTWGMMVPdJymuEEA+2XOc2gHVeCxk4a3OlpZlZVgEg6IFIWMM/h02x23OeynSleIXNUwDhzZebxT7Esmy129xjTYipqGxBzvFS59f4trRU03XrKtrXTg+tOfDU9QcBbdUh2BpI+ScRIHnud7ReO1pFnmEyxuvRZmM7ioP2RvWB4zyrPJgjOjZ17WO4VsYZ6Br6WDGY/etIETnsRH0ccK3fFsBm+3V47rG4bv3SwlFQF23nzEzKXgqKeDM9Cn9023PPdi8Uw1pulh+YqJe6g4CPhcLIqaXWypfEhPuJyUscIs72v+UQLQg7w1wfMZvh+X4dkvilZqNU0gEh1y2jR+tUNDgV5qbCvNlMPcCWGtMdOlGN5xADowS9PZ8p2gODK+xylkv3RMg/cdvfn15DK7kZEVz/R3xNhrj56tVZZl1ljzxjVRE4O3A7cebDpWBD8FtPYIvaBsB/4mNTyeVue+4tTtXBYfyBDx8IUdnPR0n0atAhdg5+u+HRTNba++UJC8JRq/G9fhXqSfwDonN+nxPoJtkSb+Jbfc53SpmQY3dYmTh9eOcBxbjoMlwX0uBIH7QrCapm75exXECxUFbDbsWxAuNGsw5psoxIE0ml4Wu64fB6JmQdFJQa+JZEY/LbBngWUgTop7Jwehfab/uLNn2FZQ153urX7Mt8qi4rDLkiDLgO6CgjftwSZjZdqG3Pb0ApUedG1HT+spNRaqPeQBF9RIgeyHN386ENoUv5dGsLze/XA7GY9G2AYnSpEfcXktF8y5rWHaCmYHiUJI8xcQPS/ql30A4RvlEC/uXRx6H6dv6Yk7F/AZGpV9YGj1gO8a3Y1a576MDc/WiXgRJ8BRIghW2O+b3PLcGhaxFxZaRDXK00yNs0S/tHlQs+xdAR3ttj7GKNZun1mMzBmrchJwF91vAzMulSBzzb0Q8OWV5z05pjczu4RMrHRQB1yKTpIstEi8RIJzdsllUaVZW8ehu9fpBJTGvI7c9K8dm0hZ/qGfdac3pOvvIVnSRg8ru/mgQkbzOFlFlXScFU8WceZufMpTZtAZF5Qol/rMjzD3QmjmEtJGzPiemJS2z74Njt3Rp5gdTztNMfWJ1DXFDfS1x+7LKGj56phPJwskHb4n2BYrYrb8qg8XpugCULbvnlj0Hem0DhGsSpuZlYbzx0BDYzVzKCER5z081qCGEv0yT1A0wJRp21q0DFYJIXO5BfDyVhmfmuHnzcBlEuG34G27nw7nRejyzzsAQsa2OI9ZPk5ydIPhpIObyEl2jWU8UgMglxNHtxKd8dFRujMKZFnGtwDiX6OfOMoxMvf95vNDnfUK7UXKaKfFfdfBXjPlZl72LlDki3beuaCX9RKzn8BJWurg12rM3iKXZMGYiIFeu2lE00Ve+8COxNZNcmLBmn5zGPha6kYyWuG1WYhUE+8OQZL4+9o51+av86e0Yhi0xyHH2X1PS/RMKZeRwBflPoZAg+qPvIMrKrwH0EotiKSMzk0VPE51GvquLhR7VOPjVNEGoPMTZ7SuhkqCm3ZaZm/XkHuGLRMM2IOau9LBC1RI9aMIQqQv7Q1imen76968KdmIB7WRaFhadTZR8SDdGCrJmJzBaZd3Xsltdi54CdGhGXAvX9D3iNZl2m7O7Vjc8ta6uy1j+kI1pVCO511TKnwFgO2DsvCmnryH6eogpGgKn0PDi0uvA2c8CP/nctfkPo3/UveRLYZW+isA/5YwKKyJDgU7kITrSiKk1i8cyZ7SnmABKFXAJMUcfjCSmFxcIQUM/xbgp7oy4ajexZ6GbRt4tQXCrSFeJ+OWY7WEuMw5QzEF1O+VsLUI/eLZTpqpKc0+sjJ+3JVe2y/hUT7kunz5DG0fzkWyMEGjy66CvELLaYc4sdsIHeOdS7EHLTizkXokLe7JW4mgrQc20mWJJE1kHU8xCKBFQB0Hv2z0Oqd25cmXR33Wabqiq+Vy3isKaCBUc6htWtW/XSZEN4/9LrY2KNo93+6K99oD9TKxDqBlWs6CFxO71g7o9UIDB5Kj60BIDkhq3PQJO3FYOGFzvusK9jof5a16C2+i8i5lMIVB00QLlibFsx6J0Ydx/f3FF8l78JM5Th/n24f9bUllTBztzo1uulZSxW1SXSuDSiVC4zwgB5KxW0WHZlLs7C8FHBaVNWgEbRhZVWcQrsEUP47FGAj7cT42gI5rNO52+stGw00arP0RsvNAzUbsMUF3mp6S+Gco74ihU0t0/fopBU1KdnBCTPTDVh6y70BvVCCfrXVUMh3KNA6g9mSlG91mDRnwNgiqkD6n1TuCQv1bvJewpyBwOPYEknSXIgIIkdOBvSNBCPbqunJYTJlaxZsZk8awB9oAoM6Bp/i5aM6jiG4vT7mHqihd5FwqGG0FAgHRPCZhvNlHgaQGp2I5U9NZEAcbRKZPz7IJKmIoaXdFCt9Mmg62xeM7h/tC6pcWN9g01Ma5kfq98aMsPPTloObh2BzzyHy/uutwimxIpeYpaLN615Jn6c66HDlp0qT9DLFH/aQbkCsgFLObzmDWUgny05llVAcSNimSQjoGM72OD7q5WZbK8Cff4Mgjw98a9NlD10qECHGKYhczCBTQsyAqiqaOHKwhfeW9t7oa+QcUvphgbytYeYTuEng2faKHRV1i8oKi7morFru9Ornt8M6c21A6xMuAoebdLyhgTpS6ywd3M0ZuWlLH+Sm+gj0jdNw5kn14LoC3kdZaT99X39C4qN6kO0PShUnB3AKlpDgkUz8RQjIGV09ZUqcc4T3lAl0bkCfoAL3FRl9x25y24i4LNzpzWVeP0p3G3FNVqKS10efB+nuQzDn5qPs15BivtnvZHGR8W0UXdwWsfCnWwzIdqBvrBqnK14dX9YgG8XRuzhmQytd7GN4VOLhgSX3t7PupLPn69B/pINObFl9IF7lJcn56F9Po92DOE2Q1071qgyfGci9e4XRF7SaibFwKWcPRKxjmAT8VVZwbyOOCG0RTs9c8ZReV7j2HMy3+dtKTulTX+P54OOhahNl8GensyOfqVfhL0TuXUggCJHgGxt2Vs3726QjloyF7MbFCF65WONSJLlQ5QFYl0imtEYkMtPf6GM4lOczBeQA5zXA1JE8Ea1dC2Q1kaQngwLHur8F52QYTcXaHn+edSU1502o4h+RVWY8tBTx98jTFNs6qTwbxti+T2M2igZXh1zvekceo0IHuUG44HJfpdYE17Fn6wzDS5SXNizux5JyORpDzAiFVFpKImN9Uw4ry+WUtSgWVSXaVlqhnLElEbPcXiG2zx3UH2Dew7Wfk8M/Nzp5NLHomNQp4lPdV75367KtA/77CAVfDRrPuofXKNv5duJRwBxHQ1YvroJO2eqB+kcrPcayTz+jJG0cjKg4rHFHUk5UXKjunCPwk2soKkB9DzrB0cgHnIHho5MqXxaW8b4uagUi0YKPw3h+8JJ5Lpuo2Gn8YraodWYLq/pfUTFVwUImAOXAr0B0KHcORKcGVfq0ZHUSwJfW5ItD4jeTNx2w2Pog3mCLF62yzA+lj4+xIAp9nZHonet0pVwonizi4kWSYUhKPgwSJjgZT8+ekdjmhV8Q7nTufeO3yTn9ijJnXQ3o1J8XqA8rcDOOdCLiVTGDrCr348bVohwmiZm89jZnVLp8u47uS8GYYVo1iqHCIO+TVDkff8/QFp+0leMegPbOZaNri/gzn6B5GLLVklF8FumSaTCHhz6MtwBzq8hJcwHeI53UkdH3HnDbw8IxgaA158ILLqr3DTKM++DSWJLzDbLT3Ma7ZaqHtZaiVbG0FZ9HAhN5rh7akyq/NmxmeJ/92kU3cfH+STdgxV+hARiFIZ3vxDqqhL/XnvdUx26P6/OGVxNzCmKL7Yj+m6A2i64/HDu2WuOEsj4ohrbt0mht+OpJTQ/qanzoP77zHFiEC5G16los8nqch3PynMlQKNVkJbWYwjqP3e5+RH1mNCu+IBp6buy6Y74d3QRc26t+Hl8KSy0h2BiXSSoXWLO6I0yfuEGoGhj1hFxWp2J6jJ1UHquwJg1g8YUFAbiAPoYCmZoRDOZkYYQ691MhVV00JsWIM44Lebz1FgA8yGXc2xO14+sSDVu/mnePorrqDRz7yMrPksQgKfeBXs5aX8Zg8J3lRr1GSTNoXXprx0g87kpsbht0aqWVeXp6duIMkfTexzfrQ9yBe370JPJjClACPzjGgOlE4M+oXwTPv6U/RcYFHmYRDyIB+TBV0+v2Cn0IQ08TMzTxwjLQPKbG559EJb+8GsRzcPRrakZuONzRtxbls/gTjin0siLonLMRTJGIMC+eMoQWi/XgQJFd0R5Q2d54tEqdgMDTvQfE5QawfGc4E7o20xwO02hFCdMKbtRBE1lzVu6SgFMEypvF4Q08vPaDJ7V52e6EsWXoq93ZNN7/tm8gwrfdNF2En3B/e7ELioquZLopvbIQp8QUFhUF1b2M3gCPFUlS6LhCutwuqdb3l3i0pGulNT0m+e7zUkli2EvXpl39f5zuCaRriTpu+9rcF5w5S4VRTRI9Vbn1zhq3LQANuoKgQn+eXvuKKB2BBY1oLrBYrjdgV08sbqpWGQY3qeO72VxznR9jqdM310R42R3whVTaTfd3XJGiOi6mGh/f9UXrsu7i96BmcIcpoziOYvHz6w1sdDryH1cGg6vUAjkbdd5itEBmigqnSfhtAYz1lC+MX0hg2G63NEkiZFPB8AD7JIsDtS1IAr4xH42XnX+azK8d1RIx6GYLxP1i6duJetYx1ICaBEgxw9eW450xfuOUMC1JKFwmNCQNNdsk8C/JAI2Q0zoKB6WhOCd7lbnZAIVgAUcGO3n1cnWnXJAnlhi0d1whCIXNxkhyVvXuLEVtnMl6gMndzfHRG4yP1dDqLwZmAAtgFdWnN2wn1c6DTFs3IZKZIB8s8GQkr6k7kedLtQg7MO30+XyZv3FPaCE4wbg6EsuUPqoN0BZbh+eG844FXQdwgcOO5gCZkTyRSztUOTEb066U9UZR13gTuzZGB3lUfhvDDE3E6cF+2KY0ntxj9nYd8089V5xECWV2LwO55UqlR4bdMayaindmihYjVRG3wIFBnnzmCAK1qU1Tw53CP2/w4c6CPmJMyunRWNQ0+IaTgdfG4iTX73vSnQNjuK6qoooxFzVzQMZnqpS/yTrzdxxXLloAiDnPBnRqyWlTNUNEFMLw8YomGCqJ5Q9HWNbS+Jm7bThmej8O6XhukrriHEzlKIQ8Y0dx9cYutZEsKiHiCfg8DG3WwynAT4h8x/fjES4OTqei912ANIjNA0rh4jO7TCt8dPqIF9kwN2AW+59qtCtouCjXJaZ8sQUpSW2GhaYyuru6FAT0NYIHwiZZM8E4hRhCI2qUnugtXLVFt71NIIjwFP3iQHuB5D3WV8DGj2FA4x3fGF7hM4jHlL26GQR95ruhZCMERFfIWOgWsR20I8MVtp1GelaQutFUVhoW+SvjEfHW/RF+XOE9JT4wEpgw3uTGrOlHjpiN7Ck3Kmu4evNm6RUE5Vj6zMpHfcTtbcIm97SdnWT5GGtCJrcQDq5sS12m2jqm8J4Rn/enMQ1LPV3mcGcwh+crkt+w+1NU1M0qFbGsa7ReUZMvGQZlPs8ZOyM8AgYw2iBSkWnZDfg1tzVxGers7WoZu7I2i8FwaX2OvI8IKZ7fykdBUjr0vs0ja5v7lP7qO43k3J7Nj00NoHsS5HlGWrPngLZbuYQsP36kXisX7WpRgifZwhAAOeUw+E0YMs3ygfRcWgrxERIeAKFARkqFYlj7eKHG2kKPm1RGuObr6JAdkx/dGPkIDmFvPh4jKz8wnG3OExA6n63pbV2AjWwl1x87ZxwiuO0EaiAd8qsDhG05ItwEFDwJjwWFYDD6JdN1H6gQpGsboqwR33hJkePn1/C4JcJzql3ONgQ90UOmY6UIcxiVIXHPzy7nBmCcCwPXwYTEDiOhzwsHxVo5luvckOspxdn99dtsK4mirnGgNzgjW/LL1P+e6gQjGcjA7D9xjCr5ZRp8SVAV6b4HxP/uu02J2T5VoA89IQarIsjxUg8S6yjj9Z30tKYhFq4FIAMjAocRPtg6wF53Op7cqruEV1Si+N6jXxXrITNDpHS39OhuPddA1OzG4C5TacCw3yB2lu0tQZo6uR3PZehzIDVg8f/GwWOR2DioS8CQwbc4AT8ae+qdcg+YYSwbOtW3LBs6hSKk4VD2R2blFdCRVTUNdC8adLFDks/FgIBQa3idYBhIJPqss0bavhEyL4PcPXBXQgoVfoc9Pe1ZksFHDbzx+TB/9mgsUYcuBQr/WJaYtYLmVsjJKMSXAyJhtyHFiRq3lOl3sCLwa0yCdfF3kzpoS2R3cHHectpzQfPflMAcke28BkP/Ip3vY1wBU6TTWGoxxgbmU4UhyzKs8rfN7+CaOYCt2Au/0N4w9oCPHgHRwjFMzzSSlPjHGBY0yRzndJODjjNhYfgUbV7/LSv0yDCi8WidjKO4tBuWWRaxPhKUuybvrQP/qHWCSPC1MuQ26pHXmKTPewFtOVfixbrNOWzsLxgwppojVO4u0GQEDBjQQ/lV51PmxdWYhe+RVN+aPW0DuMIo+TpRtOD7M8Q3GNtbbL0sxmbcyV3peN5sorAU89QAHrQyMswWdPoT9IbzRRKdwyC/g5J5P+bGm+IGdj4u7X93L4OQBdawTxOQT4JCmnKZbWzBGRYYxJk2fzoVP3EB78RkvJrqpZWGgk1ECFVCvwA8WbqRAZdEYVplH4ujJwpLeYk7AlRSYNDCJ+AvLg2aXwipb6/PCSAknWKYVae+Al5wK1xkN2sneZFileM3txfZqZ5iITSPve9vcXUvyQNhmi4uQPTTmMp5HdlOaxoxzBa/KvJb2dcGiB4o4GE6RuBMxU8HKLNLrLmWyDJKqu2xM5fN+H6gF7qGhME0t4YrFdHr8EWRuQb52dJN3laPBsc7rkzHrBG1qPgRaIjvUxSPfZsa+BsV8BG07EP8JpIMMky0wkS/7Xp4CoI04SR4NHfJaxxmVPjhCOfj/eLqKdceRZvlKYliK0WKydmKLGZ/+qk7Pf3sx8x33aVuuqsyMiIT6gXMhHCF+m6qQoRSgxwUW1qk3FT3W9n1ZFjWzkn8IgD6D72INhEwnJQO2BENN2FRBox59QDFqxf5rdnB5lMWH1aL7TsetzsKzLsbf5NwocjTLXTRrys1U0QtYUdFUvMdnAPLs0H2MAbh1nG2LvL4/Wc+Zfd0UBjuFrJOLYMdSUCZL+X83MsmMaZjsdNdN5xn0dcfGOlsaxU+qzmPr7B+pZpvohDCU+RKYsoug6N25yF79NBP9g9b5r6Y3G1kmK/1bhkJ7nlI99UcFwSEooZpnHuJo9wdRIUAy3d2WjSovmuzGACT9eigWDET8gnSEoPj9I5NKQpI+IJcLf31xcy+0mW9N5cA6dVeOmYkGUGMB6LBUiRiAFiVlDTXwNaOaszpqBbRuRe1WNsQfBDi/pL71l+Fc8aepP8sOkO/H+91A/JWQa8z6hNW4E67C4QOwIrm5vCgKtJPOx9HArdakxrrOuKSH0m7VVtquxeXkOPBFLy0uJ9iP7uB3fS1JU1NZrita+eOYZ3Gibmz9rXc0rzV0sWLB5PTjWAVu8jdSfeY7ztbChVhIzT+cyRKtY/pANkg3RfosL5cj4fKqxN3wQ8HHgXQxdWxD8TnJ6h/ae5n5OUXRl959GXhE4IWHwzl6AoReb0GgiilEHR4EowSvQP7QTS8jdP+6y+qEoql53wOW2eKOWkyxk3lJ5vTLky0MpMTP+ACor2zQsx+wJskDh/Ty8Ro+xHGVLShAJlSJxGgMFuSIa13ZFJJUzjoZ7FIonVsPIcukZF3RW7AKhiXApMJRpkN0j/9A64tY8vN5rOn4IrrFB/sYm5Dzd1UsuKBN9klzfveFua0KdesBHBo8nZqpp79fjVG1Onx94BIuy0GHs0Zef4i/ZKoP31GzaVnH5DSl/rX3vzI9nvCewSSxeZNt1leWTAFg7RbjiOUdYWDp9ZRsKJfJ3gl8hyLIKid5g1oyU9UcdOt+9Wz3hm9VSaRwm8ifZlTawJ3fUbSyrQ1oc7V4y0LidSirjY7nRYRjAOFPWDjD06N5VUWKlhGHUeAd38Amr78bzM918hGJpz3wbqooUUw5/MiKEpeyesmsSGnoUzlw+CS8UDMowrtGftK5J6W904GMxCZ14ADACVlW2Nwyqbb9FTUIgfE1kdRVaZDhgfIbgJpz55HvvMhMxByZILe/7xry9WiHHVdicjvwWkV/VIqI/e8btUS5vwGaYoM7IjZHAq4egVcoe3f8BzFmwiv4rp+IxqyCCkm+a062oD5bxTG6KkxHrUQQV33/Gp2sgsznN3adUAH0nrnWkyB1rK3sQvy5GbxXPxrbStAT/VC/w6z9s5ZKgxD5g7Ina5XyWD5e684lDHwZqFISjJzdXzwPWRMh0lBdXthKN8cPx9xRk3PHGQk1WMa/ENjC/0L8ZcItps6JG+UTDCNr1uZ/1+ROhfBknIg/KKO4oLxOPgE+TTVV/7ua2NNLV4DPXZNUx8CCqSo89r/JQID3byyXmw8o8xdvn80+xE/jiVFpCLhAaWIgs16Xx/sK7TMhjIc34YtRKFk1ZzZaXgbtFCM7TAra3sz2Sd+lQQ8I7Jo4/bVyQYdIamw67sqXoa6yCgnURLw1vv9uKj4WUnYCJtHTHe8U22FkgOlSAzVssPHy5c9Md4rjDDMLb7MU87mUXkmBZEdmPpdSDsM8phSOxpJgrPNGkhbZmQtJc/AF/26SBRGDRx/KBOgAXLg67YNBWwdChUvkj+48gXgWjHMxXKI3MqHAMKUPutrFbg13LpszWx9cAjPc/Udp6gv/gNfuD9EvjvzeEKs9yc5rrtLNx0lIcfoOdPZIeWVVmIhkoAHUm5k4lS6oBLbfiykcDBdhWz89pSL7OP7pz/feB2Ze72KTesNPutY2LQudXUXwINg3Uh1V2/Is0P2B9lcWCAHXHtW3U6QazDlSeoYmBvqs/7CWQ9N78JA07vMe+aUYeCeYtP6KYlXVAOpmoi3YclTQFdfbHpMDRGX7PltGUbJEkksTCUZT61diDgTL/p2VhX6yUQqgSwC7Ww6fTN6O+zgw+RfE1Z8wqGhTrk5+w9KwGQrrH6NxyM1XCO6MreF6Ioa7gpdAApz/nQgepn5kfjPJUqVnV4IxIOIlOcKhZZE4gm8Dkgn2H+MZ264KYyv/0Wa/YmlG0KWOts0PlEUcaDJ9s60MteY9m9zHWtInuosorzfCG4US+EvahMW/Ns8fKhIJLT7Re8Tf1+ewnauMgnvotALK0rkH+HtD2Fh9stZq2UG+6INID2hgCch9TnnWa6JAJ9ENvN3d+FPd68zlf0qjwJvWMN1tY7hCjcxzpwGsys0xlU7KQktWV5y10UuwpL9wYUYHI5gE4yraB45lQFyY7WLUxToEX91Tl0SHiTGyde8bPtbDs4WvgKSRDFbWB7WpCs1QRJ4uUkEIH7/gqa/9JZreo+knvyHqwMn8whN80PWx+ugw338Mn7F5ANy+ZBQB+DUySZJrz0BoFvaZEAfKcpU3BLwarQns9FDOH3KNXgKIYFrFFYyKHEWs6d1DZFAGsIwHoySPODseY+6qQIDmWp+LvpqIQ++mtZ2l8GQowkpUgboBlQ/CKMVDrxbyKC4XapOjmDerrPM3oOrERpkHO8k9XfM4EprMMfp02G6Yl54I+N5LaXlplueLQNhrURdb9oLoP4iRDhuaVeEs6dHrVqrDgdBP8GzGc2JEQiKa+CetHX0MAvJofOf0AfhO5NgyqC+tm28tuW0O0yMuUAZP/KCRFFTZ/vITxnLshDb+qCTPmXezfJeUjesgCLPy0rb9+pz0zWbDQSDxai9f4/62fEUY4IgFGjEJAXeJVfSSqraWwkahM+nYsa6gcbM6XhdaY0vB398Tqb3oOsHnmBHcA8SoE7z9BxgRksN6uRv2sI3/JmFYk1NFycrKfUSGNkcdn9Sd/L2sWaiaOxGPYfOElQYNIWsjMssuLfJ4LJR1QgGJcMBetlNieKJaSemC/DhVAcPgYV2BxguY6Hj8UdfPoMBtwoDDsol/VTdE3qNiyZ++PLPbeQp3LqrbB4VYM7TZ6MnlOU5d0uR/gGmNdLrrOUGPQw28O53lM5Sn4dmqqKfp2QsIKdXkIjOZWGoRx9gwOvr2Bn5Hsyh1oBxGNN21wcrDKlN08d8slvxa7An+lMyRUtAbsClp9lc+c4ckwIeTZaG0XwhxKQaaCBJtqXNY1SJsl4M/pKK9bI28l/cPaiBM94tPNlZ3CCzJmvt1kT46cttMJ4vAGRBRNSur+eJFnVw5rQKyIggaVKkq1MJhQw/bPKYDaA6Ftm8c/yj/kaQ781EYZP9xyx5RjmFZXeEK4hOCUpw9YKZvRDj9J6szq/kBJ7VWdWtfLSlegrDKMw5yXUeD2TPCiYkQbqcitsKyZoLtVRpks+PkzGLVqIzxxorfR9AaSiYYzd9fI3UrkdZ8C59E2evVGME/moLXXgdGCP0VPHy3TL70j2GUxzgMSRKuCDXTsxiFSSGRhpVVe/uiCfVmf3Vt7CP68a78ML2lsBZz/Bu3gU64XQKFwwVJ2GpSrhrIYnVuR98gdQs++UXgb0583PMxx8GvVvqHK6K+dPlvLbgntZ6iuDR8oBb7jAxONECw3xxlsN2n8npzLcMIl6YysgJByGcmliAfc7zNB6ZwZuM2qFJY7KUD+RMi3IQjgVv+QOT/CgZep6YI2FEaIVzlS5s9fn9rlqVI3gRgu6rse3Wr9nTp8XXziDCpM26PkOgqxolft6e5HjQGxH2HqUX+QBsjm5PxyFocguJf+IuZT0chxtY8hq3FpMf4HDlxdWDU1Ay+98Wxw6OHdqgEKpLXAPIA2T2ASfoNV/5DX1GQgvkTrKXbCgjdQL3AqkRkQ8i5gPttJg78D+AZp2+cQ6yp261p4BJENfoMkVXpFaXYwBFjAq8ZpypWKybQJVubhl9R9DAv6+nhJ7tKy5h3Au9Bv/3fGBydabdfMzLmKYufYdwEirpnL8YZpZ9ObT5Fp/nYm9ZbUXLgSaLhCrlNhQWV9X/KHQGqtZtjOXQmn8lfrRaBE39uoDCmz23RWcrIHHMOeTQT/mJH0Jlp0YNTP3ADCqv+YQu9OMGh0fOJCO3bqYViZwdLbVeAe+u6S5eo896tkutPZgcktwcEERnlJ/NsrSkCkfZdMZK5YFJddaUsm5F6nDdwnyG0XGD6H9HFCqwP388ZrkUOP5XJdAn8Mgd/wiUcPynVRsI9vJZWyuZGeaKmgj6XnTaADQrrRt/M3Hv6BZ7V0HTcI3Dil3QC2MnbBujOsldZC6uLJdPMju7Ea/BsaYnczIT8c+sZLhtwzuLKmVcJa2Wk6tiT1+IyxVXxl4RW2pizWPnNu8lx3VgN6J/KoY8cnWEYuGtEHAohqKSLx5DdPpzYH5YJAUoj+jwpsSF7TY2jrgWzkvshwQMLhx4BNQ+6HQbsB9ssHAUvbWVcmb3VCGMV8Nh6ZHb3pjcNYS/9yV0SVSKpT96diudSD02h/vw4A3CIJhorvH2q5mPlw0VZMN1T/eUr2ATQEUWKVZCNnL7S8EvbwOgyViqffa8qb2WGW3tBQ+EU7POFmNeiwGmWuBn+e4a9vM3Gd7YMe8lY+VfA6wVVWO4gRlsB+0Iwibfpy2ZZJaH3+IXYv/o32d7lAqGEYjUfXXsAEy9iVc1a4/HzG0DkRXoXkzZ8Y1OqvkS0np0vNiks18dsBaQURSUMa4c6HvQ33D3ZlwNYT34GMO85vhue/vDayfohuSGMqrkKVrHib8btXxm6RFk/+8bGydBdifg5yIkw2fcepVGde+V9YgHlUkfqqm8rZMDtJai/K0llTmkfM3wks78niwnhiJwXmH1FjvDAr4k/evta08ImNDCxyxfdM4jFDIxPHW4+IQTcgsAPACqbReeIX1BPdYXfQYtLo/q9OOYbRRWp8Y4Cc/Mpbf1YkmhyDBYWQDSXOYT1lJK19x8B4OonCOfL6RzzpbayAtTzPFRf/ySzMcEX0CUDHJBDPic+Y+fAXu4S8y3qfopM4kvJGWMWZ+iDWZ+OQZTxiJ+ZHtq+v4HezNjSFPC9Y1VoXyxGSLtgceG6x1hSKrztSXPRND4tDxM4ooGCRJt1kQ9bkq3q985vIhN3qMtzLfX4KzJzDeNKnI3R7K11jYUPOEnZS0SoZqAxOKgR8xxLJmlng0L9tnejRRFAtsh21g8fjLXKRdY5B18h9QUtwcj8AekFxEcFF47i9CPas3kIi44iKAQIWgEiDhobDhXApah/kKkiTG1L1qaZj1AcLoDZfyp/eJlGcCK1ytB2DaF9CS+PG7PCMzz0EBCZt+67ryPljZ0VezAs7xmQL62M6ivSedMNkw80/kEOptpnyPfTW0L3w6Aba+TAtEdSq7uSPpaJUKvUslwoeSmu1yFl2ziVFIZl/0ORlKS+1pCnPB6AL1VR0Fr9GSKkIFsP353gsWcLoeJqiy4DJeML1fj6fKrP8Td8GnY3t0J4hgk/Pt5kcvWUW/a3XZVk9namll6wtBaOXGd/5y/FAzoiirZQXWEXeMowUlV3lg+wKY1UsRrejaPixh2dftNOAhMTjoCFf5VKV+c3opp8RDbBKYyfUMFVd4tGoKFysdc/hZuENj9FXHFhwCVoGyBPhdnDQtnOUOPNIhRxOHbUvR19t9vlztipVdQ0OEYMU8EbuhAL5+gV8flODD2ZnA58t3TJiyDox8aTGPSNhjs4B4xreYSzahZhUsIyBxDOPnxNYX8zxUWljWTI1IncWOkb38Jtq4O5aSQmjY+yLbYoGLRPQsc19m3XvnNopj9zlIkWvVLhU5X/MefCWMTZ7q8PExFCPWZf1y68YtjZImhOwfWXQyIn0o+fUXwULFDDeotvR9tTirxEe+A0RmTe8FwmP0L+kwprtIAkwtrm66syXpXQTFarapFoIWCkQHex84iS1+YcfVPvhcmmqILLUi9AtdD90FaUbkoDBdggbQXKzBS6IC95Q4HmJ+6XdxumziRmXnGJvgMLqjKLsuuK8jVyCiB0on45Ji4zXlSB7Kl2ftckrHvKd1p7I+QMAPOpfyCYQc5+/9IIP1DnwbVP6eIfC8PVT1lSBAUSTCYRkZL8YPn0oO8xg9IjKPMAqI+v6efG4y9pfeQYjljQyoCb6NA2sjqUrM/4WnMcL9IyeQCEGjb3PnqbBcVxewL3M270GpDC2FhaYWJNDngPo18AoPu8DoFa16+Tj+GwKQ3DLAoin4NZmz8N2HnzjkOYD/uEQFpUbpcJVdubeKZ1g31XEjJ6PaRURAX2rs1JMn9Eqf7K2qm0Ypj/SN8fdgMJwHT5/bWFcbIlYq43fzfgUvUZdMB0XyBzBSo8RINampT30UNTjVVOl7wf9WX/iVhirYTbgrjFluqydFeyTVNUWmTt+nA2d5fV36i+NJx+An+Pouav8KWzps3vXw9khUOwH+Svz32imJiJlLInvQjzpcfhrwkAoevhvScM9CtkleJQKYopm7POCQhBfufBz1IlwbHraosHwjorhmoGjt1FvDoJuLEAPiQQetGhQIxhjdmgp1/uvy/CB4IXELlBLmR9oehyRuhBLaQhvACRt8X8PGTVvofEiZb6lJkJYI1FyLcmngJb/SWxqnOtenXP5CWMIdQeb9am+WdXEVQAvR0cdi8x8QMbZssULhujCi3p1M34gwR3vxxmA1L/TdeWcEKgRDzTU4tfXegOneAzuJtgbTxi2y0PYlJ+vhrVfHCM7oz8GxTrG6paGy2ztm2xnPRqRoQiei9e5w6CJzqXrnkvV5lzf/Dws3mPUumzSaBpLD0xbthST509Sbyo9fzqqX5k2j3Ff73YwU05jaqY3PTpb+bmSKcc+b0anlhKoYaEcA7xccBXsUfjDuujTOiJQfhBG65oXlPCXjs46lPxnkJznrCDv3SMiPLxJVG0kP3423bmGKwDtZonHecOh2FGThBp+ZLm72+DjevPQbEubkn8pDT4Qu8hOP3FZxacYSt+RfmHPsFa/sa+dqRYaeoN3poNpAUbqvsVKwFjIb588TS21YfRXWKatbzmWiJa/cI6DGPFCTo3kGc6JjiIdDY8dcJdfzWCPonJSTO9lrvHoDaEKntlcYwEej8+lHxGSMjvQhtgk64pwQJFfKWamjKs+BrqaVQ8ijIO9ckjeUxJCmepSWy26gFoFpsj4609Wy7N/brbyPwzR3FD0shGQcbh84gjGZW3+m09XxztL2fx2GKoY0I0Uxn1kd98Zoj+7XhwiJnr0KK9/rGR/j81VGZGRdSrr3+jt9SWTsf1tpL5Cje8Ud2/3VgiprOx8nBvC1kOeAAVZGlocMh2dN/vzzCAmvMRurIqzSav8utHjJV1syMYpj2Di+/YxP5Mz2N8IL3ig2ow/MxGZaozGQsyk9d9BT7Q8dSIpD8kWynyzuHIR/7wnwFK5egpbtdNd5564uuPOWfKo+6VWxlA0hRTcUZp7fBKVYG9fctdu10kn5fsixCSdSIXecgCs0r3VlrEtHpaJG1feOuOmKBqU7MmKYMOv99GQj+mikMTTiBeVNf6EbMtfyOT6AopolhGgKEDAPOs9MANSOOwAy7T1ujNf3l6hvg9gLreE3fs4je3EIh1oK8jf+XWQ1HXB7F/kZ13Y0qozBefKV708JLeg4cz6U/lz4F722Je8LafL1Q0oseFmNJJXYiDo/jRnQui9zmItL9xzKLYXdEf+qwKFZvmMd7m1SL/Rk9+GV+Cf5+fqsWQGiZkzo3ICaFI03MGCq9zCDf2OPen7s0nVAvE8enZqPxaiWXr5m9Wf2no98/7/R5NMx/8I+7khMac+AHKjpbL9FDyC+Z+58gUSQs5E8L8FyP/Sqsf7qe7S/LX0iT+eMYRMI/kckqsKBORntvkiISXcCzBnh2CzEz/Usk60iZe2HWsY9nHPhJuDgoUfQE+2rZSaHVAJKA9kG8KvEb2/eV5YYnNbV39Nzci34jnwX5jfkigy0T7EOxfLwhX0M7VS+r0qcfrk+JtW7UB8LasiC3P2X2j729gA4M7zM01B7251UsYI6QgvLyTcBtidcyg7XhCFF2Xme8G6NnWAWzyQtHp87wHQHsGiCxT+O8iBVJqCmOF572EKis/cscpp6Sa3MQ0mcMqv4Sm4u6iPL8mWpc8E4iJjd3waEU1nHtHS4T7atjC4+fiyVfT+NVdXLR42dVaJWDcH8t5/9Naj6F+Rd+Brai3CDLRv2lc3dofJ4GOaDlXV/3Wmj8CNKEXmBu1PzfP5uY48X0p1bCr+pSUHZrAJBEj5vpshwSEvCSXFnuUj2JhGGpSnLHJEhXyVTdpk5ivI5erDiVEc0B40ey31814+/3xwdKiT6R0ihH/c/wlQqnbm36FIY94og0AHdVDAO16t2QS1o38FpOxo7vvn4yrgpJgMFFAu2YDSklDqN1IcsNtFLgal4sFg/AN1NyBLb8vv5ySLkQCHPu7PKQq2vTYfvGBHaLljuwBNZB36nkkznREKYah7KtCWty3dFBqMJZ8SlT93mbk8IiXQ/eHKRrMcYn+yZdl9JMkd5/n5AOHv5YbXWI7eZvLM9GIXhZNuG770jPN98aS31xXGjVdARIRQy8Fb8Pnb14DHhLUfu1yfRgqUjq71l6YIk894G+/mO8mdDrSSWDBRwo200RC8DU0HIaO5OwCkeoBeR0coU9IZz5Ohm4xv3X2v36BwCRZt1P/phuUtyFRstsmvJvAIgHLo2zPAaO1ET3mvHo9JCbPKsdU6tRNeN3GxffGXbZMn/k/8j/qGvepZnIfcKyFrUBlyI/q6A/sPTvwcHyW+3MliAuErz5DU424uOptLPDQQm1rx9uNk5I+Dl+Z99CUNdITJiieXGVziI/xYABcBKcSuwisINg/xJ/ow+7wjrECdle7fCD/fsWXbeatE655pnMvTvfw5LGqNpluxDEEcoFAB9KfjEzQwFNCS4/WR7jUldrr3uS10eLTA3BCq80W9go1t2DyPQ7UaWlDfhCjmK8rkigaArvQNMngC3XSQUCM/6saIGwvQdowzk1Isg06TgvyPMYTtY5+7al6B4rGG0QV/T080zUtp0Fhfm1069nBlHV4NtF8hJZDp5UZEfTL/nVhAua9ww0SwAEDqhR12DgwpYWLRVN/TX5Kqr2u8v6Yj6llwlIFm9HjxguRGl2qvWMc04hVycXOzc/VZEpyDoyNBc0UX7mG8kZyHdD96T9D29NRsvsOiq28nZWi/ZJClN0rXk474fFt7NeuR7TfMcWF8KojhkfXr6sMSoehs78yspeRzpZGHsqRuMVrjaiS5Mrn4E4jBBHvm0mVFn0+DJCC5wSg1oBdNDOg5SeBR/NT4wTJ9SRyD8iFkRubNCuJa6kbm9Ad7NqOE8iFfvkSuW6nu3RYzEhbIW0PsCDnmpCfst/AV/EggExf5yhL//dAB3MytvlSigtwjGF6+Vl0exm09rWLK9RlQrH9ePrih/lAbgBpdAfQOte0Q4owAKAimp/Jp5hG/xIqt/SSt8DwxlmsN67hkXzIPBlZuYRwM2qx7vXxbpytwLoSoKyp2DpavZ7CT5cKQePbUNXHVZl4bezP8fFsU5ZmVZmnmLYblOiQ488rYvAEUb2QdNdU2EHLDf157GLzEVS3pqzy3XrliWWgzMPl+HhmXwr8A3wVwaGH0vGVOAb+sLnUsc9mSg5r6KJ+5vd4u/jwl3mR9bDDaFpQKXdhlVQIzxIfaJPYpH1dGx7sweKzTltSL4gzhF3TL+BnWAgdsO2hHiP1BF3N+kNVrg2Dm0PvSp1mCVXv4OkrrX7i53qWL79kZ3nVpizTnabaPDUq5PoPX36FXIx1XNV+XX8MXz/oijnxIXnvL+1Yc/JvIpH9ci7n68WdYGGJLFgRlrQdw7YTu1snUxpAsofzNulGAcnYxiGbnv99WGE34sN2QmeLDBv2/i7q0r5wWTW8v18/4wijHZa0b2ziWl6UJqBD8f5j+Ov1OIgRb49IfL9HFhUuKaagLrBYAKhi7y8tbqDYYs3Pv1k7xUz7mEFMTEwW9K2sUt127W+/dqKSKTUkf8cTcEn0dbzVQlsys5tbxxfY/lmgfo0eWKu7nE1ZmaYas4BKTtbil3hWEmKy7ySNz6h7N0pRpoP2zdHLK/Mb8clos+hea7Pbk5fPIlRorQd3/2Tay5vivm9oxJD5FBQdWyTJ44cSXpP2MLlQmb8Ad/f9c8Gxa1jwwNbaC7GMa7u+KB5REyycLxUf5p32kx+63l+H3NRMbw2HhoK+T44rmqHgp+1/CZiQ5hjY13FF6Sml+KYd8FetA0/VVmQxu3oc35ureCN6ctZveN1mgYDhipOmaWgTmQzPiMyM5ozqPc7GME/UOPLdPEHv7YtNfk4qESeBBRedRVSWl81QZ5q8Ia7kUuKAItG3vlzXfNV5F4g2dD4yM4/6oe9CD/cwh0SuxJgX7ouTjVsh06cbhbFLci99ygNBM2+Ch16Qudh/vj6pWLrpfXNOIGbNVXjM8th2oupOacnDADZBfbMMeTfNfPSAMIFBY6/p/Rs49gIafuNKxuAWiSCGjQlO3VUpSS9lSsbgeeuMQaE9q6qfhyDbCHLgU7NfSgTclTuxKKsxPPwCbSluXXxjUTiKzx76GT6AiXJgH5mT0FxzBGrZekIpxz8SLAwvEA/SM9lwhYsifFUdDHwSkZs7FERUgNcV4GbUoJYFeROWuvIBGdoy/BwvLjbYYcAjjlMdW8E0Y1z+TCV+PmsLKNGHcIWXNf+GXC26hkt619949K9waHuhfwhUFtw9QN2FlNffl+wef2ny9mBHIWK/J8F1QPoZ90JkTf1QOY7m/xQKxvC2QJU0mzilS9OqK2RLXEgL7DLARA1b9JFFhqZ1jGnxT/kihWNyYX1x0llIybR7FFf8UjZrx1ye40FYrzSaiL+p+kijoMAy/NyDDr2cFiDBBW44HxIzzudo63PoUYPK0tUMmUWGnAVuWfpyWlEI3Y+WSl3Tv2wp62xBuak14iRU/kqjEtNV9oftyt7gAoK/MIuRWdaz6min0t6/MpgDbQPkDM982Z+knySiVAHSEvN96nCN/6loI2X2rR85lFJ/rpAhlPK1Eb3MM8N7kTiE9ihRfPKSrWjK33DgY3j5PZaaSedt/lPXl3X1Hxp9Cl10yfUOjC/UO4XobEBec69PebZn43d36TLF7xh/hCCOcA59WXvtPrEzwkiBPyJSXyDXgduf1x2dGgx/JaYsVrCfc5H0kOLlL2FvCHAcRBTbqJ8jVLCwquBUhLfncdP4UVVUmDr5mKliWFCxy4mo0Cvi2DKO/eEeanpUitRa60U8mtGigxScBE6s3bOk4aApZlN89agUnuAGa0dI6Kevi7wrVDHDajmZJ97p19CG0mfFmWhLtxNgigcYTmVpd8dEStNVlijBrLcY9xo70Z+Mov7KJZ2eNlh0MEhYg/KeOMe78T7h30g9UmXCd2GGe1DuZ7AFSKhpzDHSQsH3CaBM9+W8Sl5wbr/4Se36rjCbS+E+IuwqGJ615H4IUW07u5pfGFKMDMjMQdH213WXgXgPCObR3iZPRR2D59YtVFa2B+qwWx/jpqlvWBhMG5/bdDo4JOgN6vlfGkItaRG8A0+Q4LBF21VFzuwAtu+hUO3mZvv+dgB+0a47VorBNzt7d02rVQ59rgx5MV8EhKkXYEjSqMrodpjmxBxaUVl3O/VHOEkbO+FmkkTNaCroFmZRf9x9Fp8rRCkxp+BH+eNTQMHlEoV8DJ1ODbE/QWeSAR5e++TP6C2PSwasGuSnFA3+FBXVhFX6BZjLCEt5+gw6WbV+Y7Z1dHJ7M3Kxc1SP/OV77AMoopZ+A71Rj4hY4jsDz1wysxqVIA6HBAyYUFJCdHbOtymyV+L2yZZHKiy5ySmfqtyUBSHg28n6107TQa3iyYU4JVo87ZjdOfbibC2jgCCX7aO0KoOu2Ozp4YsbSttPo8vHQH+SrhluK7qcpMZG2yt6X9mdTYwvR1+mBWWMq6yP0Q6SIHFBDYq/df/fPZNaHPflFhpAc6AnbgNGHEhhO+S5QO/fnvpynPz366yf2MZAKYk9r9UfjUlb7O8SBPv6qsHBZdqzq0hIUIuNMOiUci2R7PAL7qm9tIy1llXxjOYgAYV/4tDfvzPVoknjS33lU1eXVzX66aUlud/PgHm93qJvxPoLbSeLNz/gc7ihW6Fh8ys1nIcy93KhfggPY1Fy71RSIuiWhBud6n6uS1QMblPM1+1fu2cfLEWOSgiSOTDTrfhrNBt5+DXJu98UG0GY8ufgFxYX60QxO8Qm+4Eytj6DVRq+TDKKp4IIWorJxMcoX9AkNVt6UefgnWQ0AC5kow5yJLOEjNxwjcbn7z41SPFljKH4ZWKfnvgrcfslaCU5+6mj9pymcQAXMoz4n9WsFd3g+6lOq7lpbTHQifH12z+LsVKlNiJ9x6UyFqEqCL/DvXl/R9sHW6Zzs8ufpe/JCRQ/aIe0SE3iORUQ9UilkFompSZ43/gUfeG3ezvnSSKu4bX+u5JWljySuRIlgaVqSN4l5XcxCy0USoEPoOIod+BPxdPCKCMqUcZC25YEjaIQ7QqE3PKR+szg8m0WZAqwTQqirCyuGmhQSJVmMdLU/pdnbeMUjoWxRbGB8JY+jKz+G1FSFgOkc8HrGAUJuSb0csYKBzpZgik0Xv04i+c5rvvjqEb86Eb4hW1/6Z2zwbyihPoIFUorymloFpnhJttlbvn2yUMIrdA1UzH9PX6f8chdhip5+rAQGPNFPpEqvFp5uE6wnuErF3+AsyPmjIdTc29HGN+L6iwh5OOEWZOO8UjFbLskmlLlTSrzf0yINxGfB8HHTmcUmujwpJIwJ2lS30OEJJZiKs5tRJmj0KVLgFgCnrv2hmCAuh3DJPEiJbnX7wAiDX74iTZySD91/MXOunAHj+KITqGd7NhGVG3ep9I+MvnBkSjsD1cmv6GVglOmPVoYYr2qTIGGW3rE5YkTS7MGzSm7uKTn9lUVXPj25DGCsMfrwJS+9tINJCDYv9Y9oF3BNSV+r5xuQWBzvefWM0WkHxAAkh5LX++UR6N+nWOiLBd+rlnr/nVFSM73/d3oAXgZaRwhMO/6+922mEKXl/y/r3aEPQS2FjfDIQ1w8XUJGElIaoMYWp+yPOWO3a+hBFwj0UjF3+Ddu3+eJ29CXsddYZqLepIS7lDPtPvutl2nKtWB20JZYZ5nJkMJAoWGmvg5RgYMVpEq1/ZrbW8805VhTmxYpmk+W+n16Ikf6eAe7V5BtVQ/mnz/EgU1S7jzjRdEf2PPjO5MakLZraIN1IR0pBHlf1AUNoU6OG2A4v5qtCKA/3qeXvT4Ih9R/+vvIUphlnTB5SqOAUIK5YSlXc64f4Ugnh97Iy5TQl91HdZPPReThuhmB9fYWbI3noudyATa99uqi7f9bA3rWs1vC94D3E1HlEidnqcFKaGXCbGRP4IE/Q+R2sQB54rrvwRivB5XXeO/jyMQ1vJGjgJ7vcSi5D22iFMn6BwjNFDNOK8QUwQ90J5AQfNI4t966lxSwDCKgfmJdnCGdnkDcQ7WueO41JmvpG7fbvp19iSpYaXoWjZEbe1kcUES3eHSLytGl+ocf4guoqrUsR5aRkQILJ6tGP+jW/QJqtlZTEJdJH4CMhkHlyPwuDdQdb9rcXqMkGD7wovPoggpBDy+PCZdtQ04XSgFhWKZtYNE3fmFFpF6uZlgu0PmgYQD1Pate409wqxLPdwBbtahoD8pOYSdrvoGogyK383dI1pRCAUHnu/9Qc+GqGqPvqdG7/MQNU0fNSoGA7IPgqpbZRE/bd8k+u81NUEsY3f0NUf7VjdZL7U+W+9zUgelzcCY7u+NgwNjZQn2KIisIi8rhS+VIrepxjTRYUN/4QvUyM+IQrOekqwvPgX4F2uoUF8POGWOvzvt6goqRbEAMCI05y3vUqWdJF5yn7RBwatgEWfyg1R2wn2Rv6Lvx8Dww8KP3oJB9NTtjQr8JoWBwuBT6OrXzSTmD1TQyMxo68J+186QUj2kiNdi11TJwlI47uUEcdpVJj7V3CXIWd7HPvsxli5niCnbMt27OkfXf9dP+WvvoKcucoqmcf5wkHgbL8rH/iwF3SN0kNJdWZf2X4O3rcY+qFJl+ZtRlxfCD3dnZI/us2sA7KrLAg0d/dhYIYfJ/EUFqVDn82EcgAGQdr//FQKcHLrVn+pDGOxY1qlVSvWGCp99PWW6SINU6UYptRe9v1vouXGZN8MPXKhigBBDeEPqhL4RNgEDj0USqthg/qjXLopB/TnzjkHdkbN+RmxzjZeA587dQZwjcszL6ZH03V4wTL2/e4fTala0kjTJkbYyVuT9bq0dUiyVcLilxQgjB512sP2F5kcT5zipEPMzR6pK7naFOiv6u+creOOt/dqhvbLdAT5N1MarsWZ6j4uGSD0pqztzXStsQsrsl7uEaVBVHKPO4cdPNMpO/g/OaY6nqk/xU5p5UmjA0heH2H4euOAW6gvtp88VO6zkl8XkfZmjj2wTt99uVIX4y0NZXSIkxYzvF6vE/I0rXVoHyzp5xDJm/iFgs4vvBu9G3zDwyCi21JHo0PXUIcPOMeioK5ToEWlB8/b0LZf5IThZBwq/J7QJm2Nc23HJ0jP70Yf0DAy2wr+4r2laWCe+21Hz4p9z0dC96jjZGxoDK/0rspePpWx94J6yXCsudRcY0VBgbBFKA800cVs/9pO3uuoMxtHc19w3U7TAjnAEMSqZv6tcFi8COhVxfTF/XmbZJjMErz31jv7KWXp290KYm/0L3EnN7lG7n1JrROQAIZdtFL4EOIJKfrXxox0niuEWEtk1S9Nd0ktMDbvvp/6k8er3/hJbxrbVdXcKUBpVHlKyyP3JAhU0GY57/xWEaujbTD0R2OqNkvSx1VvaA8KMK9n9G4pek1aAAWJ5vI+CdrKbTqkPmhCpNnWjQH0i5seRwxIOMHh+Q+L1id8zP/fAVICMXSM1HuYmEz9sFrukzMzczucipXOOXRsiLeHkwy6iXZLbigbPa8+iFAauCPHBLABU0d21Nuk3aE/+a7WrfqMmtqmSBqtBNqZ7Gc79ScUOzO4DMbv2VXWA6S1zr+UpoWDukuqNE9JNNLfJPGfCxzX3kk998NQkPmuqxkzUzJj7cxjM4uunlqhqu4inxZVcB6Yb3A4qb2uJnTeNTuqM1HGxU3J2SdRAVh32leX8IOqku+/8RwsgEXlCfzNBVhnMGGSR/HbnUyvxW3QGSb10YzDDsrUCuM0C31hI8hTo69I/I5LRAewMKPw+cvas8nqTxhJKHPolaEiFnsD4koNz1Na3/6kd5YzTllFfXFidLjKX5A6xsVaLtH/OAMLmK0Glwb7TPIP78VI62rs4MWd87FQooMiTnV1JAv7oAB6457WTnsFF2V9lQ0e7gdWH8U1Su6rgvaDO3gLSh/QDjkq8BQxPzPCsX79Y0HsSREAvUNP626A6HCmajhOaCVgBjwaBcaTw/RMp59rCeRkdufB/YdPimsKOXm0fFkfCT3GSmyoqH41zCUFUVzHcP+peg/YZreXB1iLbG0gLhCYPaGHG88t4Nw+ruOc6J0fBA4h4iimbDm27L5PqICZOswVdSglcCMQSApVTzs+FNsGKyu74RE7z/usa0rjEytg7FtvNtJzNvroea0NGPscdhPt2nONotsqAiFmoGRkqVgyt3r5+q5cqSJM8luPROe95fh4IullQKZdGL4kqgVIFnzXEpxXv+WImx5y1dOMUkzgduBB0+ZL27e8mDCAdQCCS5dfSCzc6a5KpfwP7lRsVpcY9fX4CKbqFGcDYHKqJzlC3j/D5re1nKRoTw3UvxvJz32rCaN81agUBwaBicuY8RJ6hM1bnF76+DjLrbUnLSIPWqnsIYF97AMPd2A9OkkbPB13NvP0fsPSX2sD70BnOnUPaIh1OnlVYb9pngXODryApTDAR/eOF1VKzBY/ag5v7SufH8rKQtNIEAAXlbVvtP0uK8WJrwCU3JYCKKjiBLSEiNpbDI18Sgq+RyS0n0ccIzAJF6uiX/6cFcHz56IQHrQ+eYrU7b1ClzR1SfvRtZ500ciErW1aICwqHYCSQOXRDxWFernIxjPVQgj/OCw14ZDlykqDN6a0j8/ODm+X80Nb70W14AHflSMOvC0e9/MKo+nnD24ccuK39xVoPxsWzu35fup5yK03kBkFqlIb/oaue+VQNQZ3bZ74F3/UpxxDhOZes9P4tjV/yxLR/75dvCOSP5BkYqjGGP/nzshQYP7ZGxuutdK7fDEV8qSTVpF93X8Its7+rWOqDmaTGtsugWI4Ntm+D2JzJTIRNkFg/ZhY3wY5z/vUwYsfKyRlMfQqyG7IPlmXrfl+E4Yb/mrDXXMhAETrW9yqGz/3XrJqL2mHfe3/4mJej/iGiPDEt2L4Zp0HawSGTSYt+tirb0RZYkDwSSPI3F7hzmaSFvIJX2MjxZcuIIMDajAaEgtvmYQAFhJEoDLzFFjdboGC30LiBTKsBhG5hZU3Ojnr4u+0ef2i3xtTPWmxpxU39tq2jOhtJaHYNXxQuWfEvqvpohLd0tkyYlXdq2sbS6r/W5TaZP9/NgeNKm1QcSAFWJjDABlU4WSM9WmKBXg58Pk5R70F7E/DUrCEyN/OnWAsbYyU0U/FKFx9Ah2L4rnRxvfqMJqZY1PVGpZcM4aO/chSIzqw/f5PDD8ubKmvy9wB5rGDqzTAflG/niqewLBYyg3RYsgJpZofru64NAgUFbw+bZNp4LUO1jmitn6/eB0sS8HcxwE4SnVDdZxADMgGFSs3sa8EycZ37Io1D07EyKJzmdI/k83jkUZJQWoK2AjMNfbK6A+ceRKzLjVwqPMxEQlv8qMQYsKtPfSo2yw8JOb8AcV6UDA/FFrj53XfVl94+bcKCxNz6DH+d5JUdk1wL9aIgAm1ilxHVPwiesLMbL7AhzF3rJRVddNp/N9aBI6q3MLNuIbZ1lYCyctXWxHGI3oIWnctWgfuBXwrub5UbzCllviahI0Nijvm4zgaTW7as0sLIRY4pM5htka7q5Eo0GMWVtbOIBdafUsDm2CEX3uaGQ3MUaIiYf2N1ZCM+do6fS9Z7gx5mEK6uPjH/LnXY/HpOrpJiGuM/ZXWV9RJVnCfripF3+wXhMty39sXlypSM8WukEiNwRCM7d8L7ux8xEh3YoO3W7hBBmJ9BXVihc72/OSc6iYa480sv2TFo7JQ0PhCYvyn5IJEvni3TWoVKdF/wRiD/LS8OzDwwmPfBguu+2bRu3LSOJm0EdD/obga/OTaOSLzBMjbasKXYT45V+9p1NUcWe5csrwlMoRL6pbrhkDJLKro7dluNM1U8/G96TIujDbAPNTuZ3ZB7G4zcYZVZNxdjhp8PMJCJH7m8mLaVVAXvpZboiIGCN+8rzD4jo8g6zUze9HAfyMTvAJb/mROslJnyMV6TipAL9DCC95XzVfLUU8Y3eQXRpWKTdkmF8IKmY4gKCcC4D0NLjUzU/rlV21r1XfQ7ruB3mIPZXXgEvSH5Zt+FzYEveShcxmcb0sgK7C/SM7yY548bei/d/Bxl5VS7hB6L9/xdlVTXcL3ZGAkaOKsUpDZYQ1ZO92snLXWITD2bf23Q/IB+MQYIAXzrcLIYhMv4/Zk5aigMyB9OoSEXpvwuQIFk7Mwn+qW7Gj5WlUUbI651xMSpXy6iirsB/nRWvHyjXBKnsjiASGvVC2X5ilGN/80JSRbQbwWpIux6+WDvdH5n7ukS2oAY/KmYqdFRRYhvhU1sZruod4zXorhzsUyenZe02LicSCMoAfv1JbGyLg7ATB9PkG9od8rsanmMcaVHaOxwr4x3snSusmvDDCfKWlVcLM8bA1j+A5D98HmPTa/qLBwKgwBKmhxwgclIcadvI6cjhnc+6c6SMR4rYW4k/3T8grKK4EuOjPiXtKJJ737A0BUW3ynzwNLr7/YnY7d9y0hyhn99MnAEDOjYf7n7jCFbfCVaw2o/6iA5x8dXFT8EtJXmj0SvPjZJM+aYbsVzCigpyxUPu5Rv35LaUwwpw/Vca/8TCGFAX+RuDvvZcd3roPQ3Tn4oEh/LANuSaV9OuqrBm8hEOufBg3sqlV42h7WuLCP0KT+AKZSs4lcf8vVh2YGhiGAO0+arsvwR7gjIqrmaLBP0LquTU0KrFuM9bwP4h3ShtLO3opMtYtmnQVCru5LU+saHFx4KDjk97Cr8t7ca+X55TUtDISuKVjyZdXkmdw0SkCgONdHA7CCTylQhkOvSgzKaxagJhftLzlvBKKb+PEkOg1nuCOvSU2Ovq2TQcQ5evKI+sLVB5ziBbNaOzmf+NY3kx/LvAvhh0Y3kkToOy0rv2q3rmi7AO6faVk2VsHNCjnE7Y/km9FUva+zB7jWcXFRD0W+R/PB6gLY5C7lQECDW5QFz2mhd+LQhonkHk8re44/1XJxeNJV2Rwxclf615VrkDjFai2/AI3xgPpi0eD7YrVWiwPeyWYm+n6+KVgQoarztPFd4ivxAh+/zXsPVM/MlfzzQrPw8x/Ao8Lbl/9h7r+ZXkSxf9NPM46nAm0eMQIAQ3r7cwHsnPJ/+ktq1+1Z1VZs5033O3IhRRe2/QJCZZK78LcMy41LuRYXrL8zRDunDS8DXOPnsn2ZgdfUD7El4yOB17uK189gNScadXM0oaq0/uHxraLfkK1tw+tYHsIKCMA3WCwrPjbxIyo5tbKstnk2fuE1OPf4wcnV9eQR9Kw587RhEY1Lqs8fJTyt577VDS+GA3eYQd2RXwMq9eglrNtMw1W6JlEQDxU2FSwJodGaZztuU0TwKTZbTez+n1OiY03zAbPSkye6T5bJwUkUjuVTQydTGsKmhGVbF2S8ujdwkq9O+2RV81B4wL7zYu7XlpcPvD/0AsY3f5BVp4SKnLGtIIrG81LxJq48AR2pcg4/3Z7YpGZxNonp+xLCfhUaWI2saezPmMspDuZr98B9+z98qdpLPerbnpTcVB3rc+FeiuB5oJDGPJOWjLJ0TNDkxMcGdRGWjImc3Q9HMJGBh74mJztZnz3ixldAVkaSYFpWLYYy3qpQk60P2nr1maR86y0Uyurwo7CAyfzbp25kanqYpmy33Vry11MUuZlkogs9hxNi8x/m9CWQI4HK3JStdfuNVUWjSAV2a1xW0HYhkO8W1QTNlmGGB8t2VenGwYjzJxLQXf3B0ow3RVxdjm1kuoX2OSEdRN7dzE0m+x0E728aImHKLKywxcwbgyJSisi683YIDKTsq/OInG/3mjn9kwVp5JaNXKLAGtSkCFIFvPs8oNpfJ4LePzvH6sQLlWqiB/7nQ5dsrGa1QsC0Bi3r+Fu0BTMnbOZV2zEryO51sfJ2ZN+7IVJ4vrh3kkn364dNM1fJbBCgJHd4RiGgSd1W7QcwteIhSmo4folIdnMwSX89E2geYSkp9qOwXA+jYPmSTEAunne0T7cxXdeJccnM0xAZxnsprN6z0mJWH9EB1kRG3pR27wNKe6i039qzfFtklFfcg3n14uqrTzej1+b5q1H0VWg64LJgBU15vwOvYbEQFxhP7InSzXqjIRQX6ngQtDqZ7x2vWmDd7GKnoDM0a+IWOyxPG6ICJfjN18bnjxY7vvh74YhMvQXgCD4leuzmaXfunUxYy8NJhBQ05BzFtn23S3bKx6fipCxfOE+l8i3gVtqno7jIv7fNjbTaZJxbyHo726baU7qPQuOMnuo3fbC2dU7C7v3dcb3nUApmKDdtqc3wE/Cia1EIW+AUj2kuaYJW88aYaxnHdzg45XqbKeZjNeZVTFUZEL4BCzcP95s/t0fDVf7hg2gK9fD7m8GE4VIh9E5vjaCRn31fKtFcHg+Teko/7bszN0R88Ihn6lGfIAc0tHljMicCvKB3ePl4uBoPO41lunLkBNc0j9YbB1KH62J8oL9E1exhyABhl8rRtzgXRmL3BXVz1NSbGuBYujMvC7sCQk3JQPj52+KhL5skSLv4ZNm9YlO4SU3flymUp9VzVKr/G3+GcZgn0cW5N/W7oeDI9F5GVxtFEFNt1SpNYixMTvm9F8xlLyV2ZcSuKtwI8R8/zkxGnQk2ToQJtHhYT16pTzk02pLt1WFkwX2YePt5f+Q7wkAdb3MKr5k7HNHbg3d7Xx5d12CgINY92gOBxjt5m8TesofX6YikKeSZFDyfsJ6viIAgMhjIheQK8YXIZJDRPkZXrdAMvmd4EcjJ7sFBnBz/Y0GJWpMpWpuR3pPJbdpiswcZNUhlIBUA5xQMwRTYRlZ1v8PJVQ955b5NHajKGW7mstSemA5C44Gbf37X0dSl6kvlF2tIFeAAWlpMUtOUdvmfSRSNTdXIDKmVzt+RD9rTLyE7PdWMvjm92yhOpw9ElMmCO6Rm0bB+P9Gnw6PGwEXEljxNRv7H12CD4zSrxWoogSj+y0Yji3ltGy/wa20O49YB4Ec5tvdzhlpMXvxfh4qSetuxEBkNThZc8ncxeoTRvpyla34qe664ggHdyujc9m8KTiFwz3fx5MRn5qutzGu5p2PjwFO0Hz7Cwk3MzVZ/j3n1fMRr0RRQOw39oowL720z9Ex0MMty2LM+1LK/a9MMOavI0NXwHLFEwzaCc6U/8dOOuagrCAuJdxQYPaTF4QyPfHjyJ3iriAiWdyzdR0uNCUMIqLO9QcCi08xq88jd3F5k/WvBaRWB0qEjFsqw04JUVQqb3Q99mW30lmoxrY5ztwKXOfleQH9teSlD0oRSvlyClsguWsYF5FeCbXYfYS2ncCGsPO46tR84nUOIcx2Ymh7UdkMjENxbIyfMQce9GFW37IaM+9l1dyF2VgRA8omJ/UprEXHQj+u29PYAFlU1aeOZ8gUxf2BrauJ6prO0IaXmLdEZHdcrnkNdvZVcdy0zaK4z91iorzX+Nmmy/586GpmHE7d7amYcmWefOAbNmIUpvlmh9aXgp3ptTtW86IutdauZeLkgt9SUT7yliUgs2EUTtzM7lQjDDReUTb9phOQ23IY7JKj+Lrb/6pVknt1zfafNCrpbWd659kkawWJpZAihbN0aHHH+1J1nFCDnOHwOPbemTSksOf4V2l+MBQjesoo5KTBqRlhuMul2T20cAjuUaeZCWBCuK6GPtO5EDy3IjYOUMzpp6Vy4avXYm9eKSHU0OeeTwYx7hmyTmHJhm5S03n5WXHkxWxiurKswnRCeBEx6ME0DpoSrSE0+VYNxaxNGLwhrCfRiWVOhuBcWJZrLdd9x3hCL1PpA9bMQqWqxDDVM36CV4bR3k+4MgE65RoIpBKGckvNR3ucmjcG3wBLu8qtkCQkg2vWQegFcOXpaUitRX02Sy7qb7pr13xy6sQcw5jXzamoyJz731QPyPW06QbcfmRz86GUnOSeFRcbslg9TCyX1tHXh/9fQo8FU/5OgO56FIKbQP+e/ZDC0aeLe8eS6i9TcGuGSj7Y2zckxgxli3EDEenjDt8gvlEtZee2s4RblLYQezQqBED7s1Q0QJL8s2bwlGwUO6rSVYclzORB8I2p0FutMhcevIfACpH5zVQ1bNIg7d31I7c1ltjcfnfC3Og6YY+1lX0vUgH/bMrOmxpInNrM2TFbK+tTtle49cZw1+Jbfzx4/ymw6o4jDqgV2BZcyI4lyC0oujF9jfS+mpeWsRJCjkeaLXp3GOrU3VsKe2IZIrCeYHQxXZYJVZoTEZxIP19LkqlWBl8mOVR17UFThfhfJm2s3oHE9XzZ7iKzvhV3vitDQMwxrQrFjvtyrg4q8z61J0ncwqpRP9DVBG4KfzxQ67VzPZ8EGz/muHviyEakoGSgCA1Im1xop6MhWa9jd0miQX91mcjjnHvnOubdkPZgKzsmoZj4/fMgg0wySboHzp865BJCPCRjFeyi9qolntkpqO5oYCJbewuCHQGEcitqyLVbGcO6HIwykhcfKBeGG2/hnPQlc+KWGcL8pdTf2WL/gyeJEqcShwuTvMK38BYwt//UjD/biwb3Y5n8TKoM7GT1qYZ4YrbmTzXCi9m8GlHmGguJlpSa+vyFVEHcISPZF2k3GYbyt7GiCHdu5UMp6T4MGM90EV78/4VFrESzyoMI5TqZQUfUNxum5IkCRPMUshb2Olh/rApQzl2EeSWltrnhLTNbfkUiqlHycJIRHK8MA3aOvmJRtVq0x60r1WiYWQ5qxT6PiWtv3xdtnOQ0hs2QRWpAdsM7IxiNMxBxoQwt2E+exosxBSmQn1vFZ99y0oDJVM2Eko3qnrh7olzyPhOflHcxmwU7C14vEgw7dwGagiTGq1B7dEMyup9Go2dQm4e+JniX4W+jcjneQ3Wyj0idk4QhvaG7wlds2EtYDZLsG0LJO8+qblzAQWBAnGvnGSHMAURL21JshVzsLfTdNpeVJ+T10Icarrp9knlyr3MmOKRqyhCGhyhpAISHSdxwGr5zY0AR3DrWUxr5U/pHfiHQE11kheKcL5fDt4LjTNEcvFCqy40hp+hSe8efqcd2p4rEyvW5N1JsZIk0YIz8CmHA2Mq1ydpgyLD8i2LtphYNNkrEm9+tbfQsjZ/m6Bd1TATQtHgkFxWEdv4wLAyNcLHph7wojtz2BkIu5BfvPJGPY0DQpKExwlrl6W+NQPBvkZNESvemZaqB4Hc64gG11axTdpDs1t902Mt7uSjbcwg3Vi1F1J69gDNH88kW4L3DOjCLhoCe/+lWY6STRNdhz78WhoXTzk8LVTtmhmOfaBbzlgVHHgiYwcK8XywmQrsTfnGgsH9bu89dvO8Alg1I0eT9uoV7/lAjSc6imoCq7Z7X7+PJ/OeflqP4gZURGXIezuA4++afkntjcYHobC1HRv6bnmY/x1oXmKdQISmJiwoo3Ovy//JS2XWXfwJ0RUkVQfYpgTWz+qL1xlmdxhrGjQWojLGw+jF6B8hvq3+Pnox22Gasbj/aiNb0aHHD5NYNhMOz9S1zQXPzOeYT4XIl0nPEqp3DaZfXYvb1a487RzqOBdOVjfnlKrHAysshKg90w4XRAuCLa6/poup2vfPVYQ5LcCyFEpvTA5Bb3v5QGLYj4lKQTZzbwtKWxNZkZeU8L1nXk2XDlkDOpqlMAgaiim8/KNbg0H+/Mwz8EpGoGQmVdFMhW0G2dKT2Jfak85sqzTpvs9eEVFqFXx5Eq36sN5c2i1Ype3/cwLDrcyieCXrXEmPkjRy1r0TBFwOsUMrOLRR23nRBf1cfHR8Gj99BMo6GMbPu5nO6rxJSsdpSNBXV4tITC6HKlkXCjwqhyyGtpjFfD0o+c+ZzRleF1LeLbfqjuHP5TaIhpu+qByJjy2nWie04ubTGU2BGeJ4teNgePrMUuGoGFLM9QUq50bj5N817SRNBTV0Ro5JsXA0H9NUqG5H+zNnddNdhc0dNIJc0dPGjEqJBtQioRnyrXTlr9PC3WeEcs0eenLsTISpgnfqrtm1/GmLLRiNh/d2Jm1BnZEq9IZwecfcPmC9dUjG8wmX7EPYJCxpgYH2bfRxNpcxnhmxkQwY782MfTUfeFjZa+slut9zhWsNVdp7yw+8KRrAkbxZFFbF2iYm9GfvuMNRVwWOL3PawoDz0bMLGBcVhngrbe99XI8z5O9lfFCLmFXJeBm/9CCVKcqHNhnmlHIvOLAd7b/gOBz4fF02TePdbv/9fRfIAvgArvmRGVjT0J7MewQLhYS1S5kC1HnU1Fk3Eru1wTkvMKZsr5UmpaxyiQVPDPfFPKew7zzqat4GCWg4laTx3lyB73bKj30ZUiOjZwSiOIAo16zRYh27kql2YGzuYjSRafTlyU/7GXF5bfOqPXGFFkTury/A9UUcx5sn7EvO2OvXJoFpJXVQgluMRCYhd/XC/sEApzGb45THCfd7KKJNyJnAZPZQ3otI7Hi5/XDX5BdV0ZLvKKukMVUEGx9Kw2iRx8ww4eoxprz3u9YYIsswaYu4e9xorYs7b0zmZfWaxRdZbAKnVAEdsQeEDKyokLKYnPLETywjT6O4dVX3qOG2FPVyUriLw8TaOMKhTjrEp3sFADCHdcI174XRpQJwuFb6vJ6+5qmV4q53bQnSQslxawcb+mKUhS5as29b0eoIVygys5wFCYBy4Hg+snHdOUlcOzNawkXmAg+9Zv4NAAfQ8Zw+g2QJNeC/cvlnkH4p0+fmFSUEyFAx8ZnM7BuYcOsk6nzvMKttIn1OceyUDq863pMpb1Yf/jBPAxYIm5I1dQqayi+1L1lgH1uXLlPrPr6zsfZ+CCmIKtUeTaz4xz2UMbIC7XzZ9E57j23zy9rHJqngH6ukfPt/uFFypHaD6BVZFl4GW8/zderEhd1/paGjR/etH9iQDpaUsqHYBuW+IigYN4ut6i0gB6rZ/4gWwyejycmxHLwnlVteL2OFU0Pt7Xe54MSHOrlv9oHOs2QqT0ayN/wZIu2YfOpdyy95AAu6h3Xw241ArJ8HlXbvTHlzTw7BKIGgfcFrnCQsTx4bDwtMFpRzY3MydY5GKDJJnTg3P+t1cCyOgNCyZEE54zCn23dfqv9Lq0velVQNVDy1m7lb3GmNhqxydHltQtp9437ud1AB8+9mpZiaRXJ1kO2cw54GS9Ps/Ay/jOMa7/LQA4qmmB07Z6LzXmbuMgsJVozJspn8jKG2kh2W03epWIpHWT30gs8aU1s75VtvVsAa5Fbcu9xakmjjJLHFZCBABOOghvUzEqHZgKV2nqievEkgPSXbys2+6++k9uPToSeVpjc+5VvQ33q7ib0SO4xPtndqO+9YiCUuEKct7cs2AwqEXRpStZoca9AK8Yf8fpUt+Ylj7cktlmuak89N6MyVs4ITdnQONXNEDJwKF/OYnpHH3SgFp/Ar3dTTu2wrUCijuNNFQRxs4j4ZTDmPt2+n0nr35xyJdmCrMqieRMXtfLwN80damHUgx3wioFSpsa6zp/nQKEuOllIBFK/sRnLI4afbSJdMeuZXu9631QtQLblPOw60Crn5a5825rx1CDZGZWRwhzVBKHKrCDempfDBt+oVAt/fAWGWbIgXuczgjm/pTXeT86NMy6EP8nwbo6ATOqP2MsD40cTtDA8STFp/y0xSH0o/TDGNrcNu8VupiArhpkAHVcupyClGTKsnM/uKNV4i8Zk0K2i6PGxPHUqmYd8X5opqz9Nj199+jE0E2MyAt1Ltu1MlicC/fXEsimaUCzPwns/IU6631vCPh1PZKrihaSky5AG/TSZyoADw2gdn8XItjbf0T1DjxXGG5LO22fuQMINryGK807NOY9UzMOopJ0nxKXAULE/Xe3gWX4zci1CJ/5pwbe+SeRJHqHwFlHPvr0hvd8EupuizgspBXno1roj2yMDE0Fo79dazvhQZgE5Kh8capWIPm3/A4zRFEk4vAkN/iSkIhrRwyxpzeSWr3Z/AoA8Vc8nNckhkuXMbrHQE2XrBnZcPmRtgeA4Y97uN3KlkyLACj+Z+QqT2EiNyBontDq1/DQbwJmXLwayH4E/KL0BUNvuJOSoblLLc3zKiDA16hyU0W6HeACdMfbgunKQ9i1lh1l9rYy+J1nfVFxjOoNJBaf1GC7c3l3Q/T5m/aUgYN/0sx6TdmzppKevtD10KLZ2VibkGS0D3Q9yus+MdmKQCDoceJ1U3Or7yH7ZPED7jalTiU5NkT661Zm6UnRfxWMPjmXcpFSRGQqiM0DXg5h3yCKi85vifSmnkNiSpAgorSWpxSbsUiDV7priwDIIdIW6elG0FZzkFRMnCQHLI/7lSGYkh+Jn2oDVbZO7zwd/Bs8Qa8uXZraFKDy48nkJTc2w7joQ37RTj1tN9k4TAOYTC4AkRreAnW/JxIuDbujfeDS5eUyRPPXixyu85OgF69ay56/MAsvK14DxfpzXRSvhUJdBuc7AQzy+N1hrtm1xZUIlf7KDnWxT4vx3v2YlnD8o9glrZMC9YyprsOWCGuicxdocqvRBifgoFZJrZp7O6LzYPgqvQCBYhlbIcR8Boa6m4jYMRqldh5jGw6ms01ufOKSyiGyH2PJcD8cQx9dT+MzPiNOyW30Pb2kwRV4DMeDLZ5fte/9WzDVguemxn3j1VwxerqP4oLzD0U3WORZgfXYROPPCZ49a+rw9qK0vrfuQ3WKQ3Sf6nCsfec0ttW2zWEhnxn7uzrYkIJVheYYLaf5w9UJpFZa+HiGwL+zrZKZd5v6g5WUumfGDHM8ppD8sF/VPvH/RLL5opGg6LH+rDa/LOyVFmoiXS3EkQZ/BzEyMM7AajBzNN8oR7R1TFD/GPEjM6zAxp5eNdwEMGQ8WUsGaIg3/urX34uZUSCL2NAvntaHoxFjLhR7cCKZDY1FvTqJHTWh/DdpKTPDBNwT/ODvFDcsKGFEaT+8pDr63+hMouVkIzdbix1xW1UM5U+u33usDpWlRC7TUnJKD/8SXjIZE8uBBtTRWpG1v4soHLGWCi6M+0jOxCniLMMxDVSQ4UMjzPkLgxWm8qpB2bXxCqWwnZMfipziyAssvx84I9SvHVKZ1XGOxaOLBwuwHFP8RoM0nS2R9EblES7JQ98c4+LAzwy/j7j3m028hmqDWstVBB+59FHPKDLCYr9+iCEyn99kFGG6EpdPz3ZTExYn1lQJfpv4l4VPjTy59NVsq0i6tcMFExo/e7JhbFNDCW2pubXJHq7QiujSUutYRYsrvAYdST04Y+gq+xvuKa2MQz06ae55ZWgPS1aVDLO4IAzlLuq0kjKvXqB/Za1nO4SeSOFtXmNMPAwaPqfxc/HcK+Q+sK1eG2b7FBAYOHrNLkDc+XbkDPz87UMK6C9SpVgomk9cHFREKRJUsLh3lhU0aGkPXeizrqj1Fn5ExWm0fli/NHcK1hZNa3lsd4y5OE1V9v+nhgQnPYPh8xPd+7cMH0RRRPvYd1MFma2HIV9Kvmkhp4rYpzv19M9D3rfh0dzdL/AEVlVf+7kbAIEqk4ECGX4Mk9kQLOeLapIW5fOtQiqRKtg/b0s+sC25lK8rYrUmwjV/5n+N4Y2/hXZKbCHCrEifb313EnnHtW08ew5woklJPgIAFiM+l4n15kqv5z+2YqCeOGgyJBXH8qU224LZnrjURuqu8vWvEga45XbAbC8YiiOIuUe8vCIN8kWxUVpblTz/eR7MNHECMNu4i+q0EztW5wWcrK1hvHHmIW2Ex+QnsDaGp/VrY+Ts6yIjqmd7Rx81/7YdmIcyNc/YB3FTqVjUzSsPe8A5mFSn8AjPTVew+DVaIVJ3X9IqwcK3rM9lHNKjke8/qxa58K15SqH1j5Mj3rX8mQ+Mhc0o/V6SJX6iMNAV2j47/Vo9WafPT5S+1SUZPgIc9qzLelMbZL92vORBnfl6pvxkWmD4ppJoA2FtanTY2K2Wu9Rz8k2nS/bWBMayLmkhIF5XS9VGPqRK++TWJdAnIbGs+PwTAwnIAYEFeAn1phufrFJ5nn+d/9mZOSrc3l7yqUvemj33/9TreTjuADMfX7USXmzIhShpjwMdyXM1UcC6QpP9A+f9A2XzoF6u6svsIRn49vnXpqj3vM9zQVTenhqyov9ENUq2fNyxn+73jP5Ab6aAx+mT9AtpDELUU+Q6yVs8Pdoi2rv6jW/8LxX9cuGWfJTt+XPi9F338B8p1h5gNXbaArPbQr7/iCPILqKEJ7jp/nAKFgcDhXqVL+eMUTf04VWZVUS4/7/wFRn+cjuYfp4q/NA9w+UenYLMfXNa2P8fw/Y5AVfp3ngOnf32OqF2zH9f9B0K0d89sWt2ACM2/Tsx9dlqH5dfp+l/zd4JvuoZgBFT5+fnj/a349e+3kXmM+n93K/uvcwXa6YdPF4Gn/t4N/VO9WNlnq0CdP4iv5mS4l/SeVyLqxvu3Pp7H3wwESFvfsfwrn/J/geH+Vzr4r07A8779/pHZoqqN4qqtlv/jE/B/8en/+lFfQ5TeF7NRG/VJ1d9CHARW6NdJ4oY+WT83OiTnPxj13z793Vk/zyK/ezLkRhNwfi+rJbPGKAGn908EBlYuXfuFqP8ixqH3cdRWt4yI8rdKtGQfcEHVttzQDp/vMND8+7nPA4Crkqhlfr2hq9IUtML+ESjh/ywkIhT8C0X8DhLRP8FEDCN/Qcg/4iKC/JtAkYD+b4Lib+kZg6A/NmOXnypffiVL8wGqPny/yjzL/fpV+379Hzr8T9Ah+Ts6RMj/DnSI/oEO+Wgu4yH6pL8utHprFMs9WT8xar4PwDSBn+4xVKAONPRVNyFriZb79++pv1rsz7D2aQaGAv3jNU/vIXyvBQc3v8haNkqa4tvGz5Xrhx6szbx8hib7zXJSSIwSxF9+8X6dXOSPC5/iGZVi9/niE6XVvba/+Y0mU4gk/2ly+5U+sT8jt/Keruu+KGp/ffh/ATnBv5fy/kzMw+hffl71W0oifyLPv56UsD+BtL+igt+s8VxG6bD/OiV/WXDoz2bwb27Mu40RtNwd9xKO5S/R5zPsM/LLvfTVmKUMOAStg2mCfgFEkYIJxEAv/bAkX8LA/5xW/rPr/gdUSVKE+nPi+gvi/J54f6XofwF5UAT8C00TBI0TOAKTKP07aiGpP1ILCf1CUT+vhlEI+xMd4d9GOPj/EM7/LwgHx4hf8P9epEP9tyGdeu1Gqf8d5cD4T8JBwYJ+r3xmEegTx/82Z/vjQv/X6Kutxuevj/mndPMnhCbcHxT9I3HS38+/htbQ3/MwGP8jKv0Eqt/SEv3voiXyf2DoXwJDP+kN/teQyY1IMPwbRIJ/D0ko/N8Mkkjif8jovyEZkX/F2H5PRiT930skQv8JIgJMY/yTGcfBf7+uzG/O//j8mcZEfD//qWlGoGTo+yxZovjncKC/r7D8lV2a+BONhYT+BO4x7N82x39mgvmvaK3/SUUVJVAaTf8ZRTVJMvy7Uf5Tu/FPbR+/06z/uN5/nxj/8Wb7P7Kaoxf2xJY3/88nRKtupSU/Iv5sNR/HkvXpl0IR6NEXVZ/9PbsE/I9X+Pcr9TdMB3me0t/t9Ec7E5IkfxPu/mrl/hrsl2H8SWL6MFdLNfwpb3j91QV/4RF/tIL97pXUP0Fb/3pq+fkr9Dto+BPtlPgzQRD+t9ES8meS4A+TZvwf/4xplvh7pllw8X1R/oPI/r/r/jjnv2nif7dTp6/yKgOmPGZdvpao6EsZPx/n8yc22x8D/MPp+K/P/eHEv+vJsL83nf+bj/CvhYG/JfX8dldTf3NX/ymIRDj0NzW2f99W/AUh6N98/urNCflHUZug6F9I9I+7819hZvzzzUn+Y7b9UyCuuqgA8/X9y8zjLbD8FK5/HuTVAZb9bwJoPCzL0P3Npfu1Bz6Nluim1h+HiDCCd3pc5bKauUOKWAzAjeBtOeXDKe5vPviHlzgmAH8rSR/7r6OB/7ZMSGI+M5YQBjjR7pbQXveX12NnGO5QWUaeEvE+wbLp6Aol5D1gVeveW2zhZSjSVWjhWOwzPz78gWl9uSQi3Kbio8hEeI57lch4qAo8cws6hwDHsedCgUVV0rMg7mv2VJxpqS3fDsfyMSq3Eu+sbw7bpRpT7kEX+lNuwnq0zEfwlzaTzgQJ6Ib0ae5aRW0pmqKvPrleHX2GJ3VodoO/LuZ8XdL58u/7K/jKPBwK/GK5769/tv2b9h+h/66Trr3H025xxZ6hGBCBJ2+pb9BSJf3l+p///+X57mewHYiWuhJKnwzxOuk1Of8yP3UMtoQo7K/rsaocvcT3s3PV79uSnuUSi/il9W/I6dzfPdvdx/rr/K0BQi8vtCwTjjpeNbMlsIknorPd91wx6p4B4lqhF9RSMfy+fY7FYu9Yk2sE6/VPjvXvjdMcwy74u+3c89iF1Y9x/na+f8z5TScP+K01ZnvPORJ6pph09CL9WIs98OUrdL/PdtOG3CYIDSfdu/3rdn609WPOtK5dE9Qs4/s6ywFhemBlXl07hvxwmg8JUh3jetd3t3xwqI6DGLYBvZ1g15zgel/GodYFZtr7drcCZpGIPPxKReGedVc2+b/X8706qLuEN4X9tmfz9z1f/3zP9zzWqQe3cW/+oWedo+81ZN+2DWhWHtPObcxe3mL7b87xPzE6zf7Ho/uV0gnz743uL73+M6uh8f98r9bfWY1vr/zIJ51bpiJ9uiK9xTzO/qBH1ohEGorR9xCjTGFAaqHWzPG2mMH2hPpe5e9vyq99ac37DD3hPic7MULP+u/3EvWl63rUb7oFM1B+8aLGjntdoMgLO62hz8hz13us9zG9/uZ+6oX+Cdb8xDmO/mKO05jifc997Q98/OLlvbbftGg0cyN1qLey+RCc7P1ZcqNLnoIH3KabPfvmGUBQyVftB0sX7ZI8zzJu1m7txsYQnqeycuJQPdpO7gQUD1qW1kDOn4IF7uiucOwVNpCvhs5CQXUEmDEeSTkzsWLUNmrr30qgFA2T9uaP3Tq17UqOVkaCgIhsQBvqUNPcX745xIC3PI5uuZLVP7zeuPpKYqgbcDzVDpfOkvQZopgWEwTiv74ecx33rn2jDuJrxUPqOZwr5n9gCjihE279DaL00U9MVCAVj2sAD1ZklcDPSax3X3/M91Nz7LVKCt8LID01Meqd9nSbvniS3W7ZjSU/z5Kc0G+5mxpzVHo7hoy8bljTQZUIfI8UYzrIZf26NwL/RjXdCvtjqUweszm4hmd02GK2d4pA5KdLceW4kL4d6GU8kQDtD+W6bPra6sHR03Cuqvlbr4356VUZJwId9DUrGXaS0zBIIj28xjqa3xWIFn3zuas+RlZ3CpAx5UT3lKgIlUedeH5/3ZGJx5Cfm/0uyAb94WsYRUwx1PsFb1IFXDPpkLzgsijeCDjyWLSKibyuq44QM/I9XVM/h5UBohDQK4B7Ml+GGWGw5pl/1yGS9KFPnhg9+iTZqEWeul+PT4xurvCsd1BV6HFVHnCAXncf5zD9Wmfr8/VVTaK52IBLPRsS4KZWL5CrrV5XIJKzYBqcCeJW6Ad8vGfdxNH86yupam+hIMqwiZ7CcQhQnvDYvQ2/Ad8goTrLb8tSP4gLLSgXcn74uxZHRoskkyexTdcoZJnk9tx7xzrk3YXSBo90kmwpbDOE+rJelfzW1/W7ppCoMfhaYjHJN1quL+m077QD7TIlRSiKWsqAfJKvby4SthzvX2CRnBR4Y3+sg3SQORwIEPMAqjiwHNKL09iFojzshkCo/vOn/2cI2TMPHrcsYEh4nMhVDZPLyQWrW+6nuocW29d76kEmSUx5Y2/gIzrAfqG3EIg12f0Zvb/DPHmQJKPy+ZoMnV9l4jpgTvHMxVC8ZzwIYts8uGtn9wPLyA17NlEXf0Mhmt1pluD55oGTr4Af0Ajypj1BANhoq/2PZ8SHcLd1xiaOBHqoJj0ghdNEFwaiNe4pJxsXeWciHkjQSSgpBTyWj4GfuVfWRUN2PFiSIshPSquqtzJP0s0rfJ7Z+MZ+yYX73pDd68cKhwNDSl3d7cn9BBDGrN8aqxG/XQwJUvM8w2gi8mlCGg8rlRl4NJfmFDAfjHMLHaUvWdBVD6TtQiOrjngbOXRN/6wHMQZb9WMvZJH6stwXvedt/AxSVtU7lBPePbGDbJdJPV7oQhk/d052AsQEKYEFEkR6PLCi3tD4STjwa1YJw3Yw1aIUGCfRzivNFJeCH6NqJwYnSjxV6o9mWNRwMU8Ro/rGAgkYDSV+hAhw1p5IonB7e4ZKBXsmzOp9qc/bzU3Ta7O1oezDtEbNGvXuDhWanyAC31xiQXAVSV6TonIOQnnu79hZurcOGzG9cR8P87gZGyuA2eSK7YBqAjRmrLXXZSwiUHI0Bd54ux/uRNaQ1fqAw9XMVjxKeanzXDMoujGxD3UYaKI4JYZkj+Ntz+f5Ksg6+Pq/u2sR8rwEwif2IgoxGjgs++vI2CgJ86hnaMfNhwy6ickssm62+Wl14ROH2PT5xC0KyC+zw7EbVeGBXwy81dHKqdwTpmCfTDFIv/wLhCBP2xwMn0qTyVx94zwIE3QfdrEvmST19RozYCu+32jMP9zCYZRTXCv6gzyelrmVNlyfIP8WkYCEXEJH9ySavkrWTmx4npWKIern2zRCsLj1Whjt6GDemJ2wNGW0a1EFb0RhXtY7jW5BnoHL0tGIJ/ez3Voem3gY9lohhchmcLg1OaFtOLH38zOpSLDz5iNmbwZHhzbBQuE3fGUJ92JxDTVmmPFHlJfwUgng6r2YND07+FJ5kJbwOSXhhVEqa/5lAtCLQlo/DIPzdaubwyK/y3fNkOiTCfj3Z1pB+qke+yaPs8w0kw9jJ4W1xg2HZh82KN/EGqVhb/lO0jwVK3EKWY2GA997DPHqj8PuekZVD3ZQUH2vXWY2V0mSOH3NW8fADBzeDsJ8celUkY+Pz7xfgeuX3fhAnu2DSKtrXwkauZbFAxgiHp6wg7wkPfnNjcv7+NUo20Ez/NYMhYzmHdl6/bSOSpGC6M3Zzg1mUiawLdLEA1EN4edmMy1haE9eGpsd5XmunEbzCQ+54PtOg+I1QyP5uQriLRX5XDbumx8sDKezxkkW4kjpLYi46IdpJGtOARKCVpPJjN17nY1ngWMh/hkPdg6CJw3vpXBX0FVQ318OUr9mpDLpOOizxM+ybySDfzIubLBm9t5QZE9vodKF7u3ovgdajYVqDx6ACmq0QPfDndHXN4vBYy91f2nOYUQu1mVQriXIUnxMhLSWC24ja5B2gtFDohAwj1sQve7ZLwcfIjbaJhv+LA5tiNPzGRL2LfiMrcq9cKx6wcqb6ad37zT6RmTrh2zIC8hR4jfbtbJJ3YNfgyJ2g+Id0a0cGql+ldgPbu7XHKfIIJR+G2Kc5qwuzGJ0zjiVf4I6uAoystNSCRLfKoVALE2eD9SUIkZ2g0ws91APsnraSbIwItmIJG/vT4fD5HbtbJvpm8v26TfbHLnh3Ts2eWb5d/5MhdnQflACqmT0AZZ340UH7NvlJRWT02mWXUkkWEUnQTKrDLrighalUpsfE8w3KMPeIK3HrOeKSWl53KmDepehmB2hlaa4GqJ1TfMjc+8xZZmEd77aRmhn/lODlQTfYlnMlr/ww4Uteh/E4CqJySho/+5RwazeM7sycP4aGWiqZh4jn+SlBMfq11Ci3eoJhvm4/Nm2Div99YlSudQB8RKpdBGEklyITkgU+tj803ImEJ1VBM4jVIAgpG+L/Zql46Imxx8vxkdE/mnVePZAayDI5eTRr2Svf3mUZk8CVZYx4kOPZvISWz4mVtkR7zHW2SAzbiRPVloouwb2miG187L3vizdc+deC3ZrLzGHCGGUMM8eblZ9dheppArWQHXs3k0Tli7SOvhWJHZOkpML3g6bzcL3UrdPvROazUSfNXn2KJrEv66dfW/+Og+fK45DexzwvDUMzJOZaflWEh5FuU/fTPHaK+/hig4UH1rpBX3Kk42ogsurdsso6kJzRzW9LhVrgXR+Eey7r6GS5D87I3HIPeSB5coEB/rg21kZvwHxfF4UQT+iYrII5MxllEZpT1rS41Ftnwz8XDUuUQk/3Vg6ydicJRjnYTcJMtu3eAkxgeHzjSdoe8RoEmO113YsTol0zm5JuLbQbSmU06W85YpYgcD2lbKItv64L8PRQNXzTxCO7UNcBg2kh/K/YVaOwYPJR1LnlS36u/RAxgvWFEaZM7Cx4EaKdJynCKl1Q6BZ+NwT3Aeh4WxSzMFXRhKTmyqePvs4BThUVzaYVEUcGNiyvrXLSaeGHhoVdzz3TshbndNlQ8C8QQ46k9Rumu2fM/1Jsy9Wxi9FHt77DR7lS6rLxmMZ67DvKQsSLLr1Ksgt3jljkxeOTNAxKXyKvxjBnvXUpRhxT045grYB6Ei3JpK0kXmEou9Di4DZNnHr6HVO73L4KI/6qtN0xVbT4d0m8hkAqjnUtZ3qIbcK0c/TsIudBdJgz4giu83uq7eKdQAu0/EmvBj4vXaArxdvkHhkcmwIzcGU6oj2wS8CFi7YmBVNxpvrVSLVKZxk5d7M4BP4bRsteJoUYT2RkwcT2vmDLpJz9JI5Tl/X6cHetqQSLk5W70SI9i7pAvmojplBpRxhce5TI8VanaxBMyX21g8GHBSVOb5JRtezqs4g4g3Tj2kqJl/Yj+sFqrsINRb3O/NDRyMMUIVGOAJuHulZj13W7y/DlRPvCqQd1TV6ie67w6ffplQPJ+SHeVnyS/Js6MQEKuzMo5KYQGIIIGp/zHRjuqyl/IcFgrOB4kXnO4pBwykqJezKKBxMA4kmffaN+IN4Deg7TwjFm74vx8WQ6FVEjJjSx91/j0DUOYiUuJa3/SoipHFlJVDF5z2dSwVjnUCiIF7WIPWTexVoqvMqnrM1k/mxv0FUGrqmRdIRSz93R6SJzWAYf1vcR2/zPyT1m4vrXBq8p7l9DnvrRVlwaMtBz+Ox2caReV6laHCKbmil5mlyVavyTsLSmTUpstOkTYcZ4o46ZFqQnk4oZiedv+UZSwxohsukjhRs0BSN9ixuuP3D7+d2WSrd+3g6Tx0Zcb6h7x66VyJAyUsU+5hFIZ+ZBVGW3+rEw290qNxzq6vp8YKChvX3roLlV+AsvmsxF3aY9A2TtyjqrJZsclvTS11P9MbcBc9DvBUYet694psFuNSch68YMYq8kzrOL7Hx94zUCPtI9jFcAG2jnblenqee0LSo7kezx6QPkoJFfLmkeTRTgQJzC3RwFUpP9ZOjD1e+ha4N4Ak2QqfYaithGZ+tUCQBYTKHc7Qo3RncuVSZTjoLCw/O2/1kzqlXPawBz7q1NUjGKBHbKjqEI+BlI5sv07ChfqpbtCqbL61qEQMCjp2c1yH1Ue9BoMiwf4sl9b2zlUte8jX0XukoMds7viVdFHlK+eXeRKMp/pwn6Gqke9X5Ic7xzUPmNVntP2TZOjS6BpNbsOwLDmVVnFvI5X0EYujZbUPJwZ7KwBNsR5x2etE365rOr4WDqUWYy5eJyY58rprCW4rBvpmC76N+6OuKI2XD7DER9ojGrGFjmSmcd2HTF7bQ5QiZlcikzJtMJMC919d4LclhjPYL4DTL15D0ITmrEsp+pErzm0iuHu7BudkGk3GmwOGlsKkhbe8aziFplddjSwFNXw+G5lp71T46eVq3Suxk0chJcHPGO/qaZMbXbNoJxuNWvW5hDQ9LbxwnprzRvFDIJec1LILsBuQkMNFExL22Glfskd/a4rOgs6dVpSXm6ksSkZvSgBQ31rTuQPb1LSuM7Ee4WVnYxqJr0JNARPlQDe6lzZ4K+G8TjIQatG9TCcwm2x5n4dCCAuLsq4bvoYsxB8B+0crLCbyXrih86EcryjYnHFE0UJUbyDsvC4+PaMkrkPxYaoafF+/zNkoEei7/0LjkE1nUDKS/9TeaGLzRTeK5ZKt+Y4iX3qnvI0swzfuBmqmanuSGgmfgV8A7ZCaGI+MJV9q9ZowfweZzyGWBIRDqYbxmo/VAyO4nkt3eMnqQXWSa7afwyDMqVchBs8uVJqgi9hGKClL6+SBA4uyeAY/mzUnt8MIgiwqT28BWt5fKr4HGzevZtBfNaSPGIrp+JgJhJqDWOCsM4tfW8j4MkJ8JGRjcqHbpcljPeQony3JqFEOFTSqQW9s8o+RpA6fdDbyT313ZTLZdoYTBHClBxNFLRnuVrz0Ngy2eRHh0BXiGuryBC9gOibyOhH7o2csCFp4JDK2ljofgcOpgs59JGz0GT5KHzW6M+1WuuWphrGWs5WztBHv5ZhQ41x7rKPWxtic7htfjdNBN3DzvIxmwbazQgU6Cn87W4h50y9zszz3VKduj+vpplcSdQv9EymK9PtE3vcvx2qHdFDeCe2BiwGgOk+a6l07Up6W8t5faL/dSYpMUgeRtuKaDvsJLFxAvlMdKpj9mwhgZTBCYogwZ9cVqTDgjBlhuFE0wzpd7iy5cNJyHm8JPh31aGZQ8VzowZ3FH7SFxxuCt43gIO5hIx9YchXTtq5IrjGIRwoKAIiBXUwF92gkOpOTDCnPgpnquOmpKihWr67fofWopCmyQybRzAWGBt+3CA6rO9swJbFed0aVeeZmZ0lT4hTY+VqOWlun4uHbS0M30fBqMJzRvvdEOK5JaBMeR9tmxjZtnF2GjydB/uHZ9absfr+dgAAum8EmARecYMY0s7BnzCj/MBwbIDZXwwNiERymfeX0q6PKGhbgEP2bImZ8fwDDSvZ6JxYdHL5wuAnE83L9axpba/qG/3yvBZ7ML6q+IQyyImissZCiSMY4Hc8YyAtl9LQhPR3QmjDH2B1ckdsHiWD4QwJ4v1q+MYH0HoazpiEFV3wD7EO1aCCJnrKrylDGa5FhDf51Q6KYH9HH6xupuKUt6hrLSrenmdUMb6YZ5IpoI28H+cmcHEhdNzTRRPPEJpsUG8gud7k9914EhxZRVpi5QfrAKunPcRemXFIu0dqCfnnM0akkuW4l5TOMp66yg+PuNOp9NWwdkIfiDknnVELFjlTrPmGHzVtCAGSgqxPD6wa/44gVIUP+sBV6L1ZvcZcPNW7p7jqMa1fHc700c50fQaUzND9EetEd8S6pcJnma935Cc1x8ang8lVfpcmeBNMwM3iFKWP5AcWkBbLg1ewJYD6uDxdS7A57BnDPIVogKMMFQGa/zoan+ZAvrFc8paDfmPT9RgLVEPgKbZOET1o0UwCrjMkTZe7f67EhxHZGTVn5r2b84prbjQTX1dSQ/Ai3o4GxjO9fM3HLLFRTUM12eWEzqWLI/jaugDixCJ/0qWJiJ5pR8ODxi+TSK+xDt75jiEerMOAZFygi+9HwrCIXEx0lyVNbuLnpsXsl0C5W5kxOTPelf1NOYLAbvBGRALpjDvN2dVL8vdLqindjMEBl/mT96womaHbnuE7klB/ZMw7AxHrqSMrp/gXHzIAVE/qJ7SJNhCZ5f9hmPD/WbPxKIFmBOqIFMnjlf2zAVMU3zDjGMs0+ScOdIxxTVgyHicEWC8Z3GMp7TxS/6oDwgz/By1X4FAKtr8ZukjE71ikCyd/shu5krOohcDcwCHX2T/x2+j1W1IcpEOCpxlx9XDvgRe9F6n87q+w1fEFo8NPFAxJo7Ny0USMtpooouylh8Gws2JZ96GYq8FxFlWvFs8WnyMBbCriGzw9QMEx0ghpdH/GSggmxPKNr6ltHWxOm6T0bk0/jN3FFX/MuObLmQRpxsFU/cYjPZkgIiQ+0e0MhFPayy/Af1jph5+aDNb42gc6/BGkSGj6Zx8Zqc0AzOnpiwAg9THXaA7bl2qoKxikJNcsajym/FiQoPDH1yNHUvdCjUgQbySN7JB95pVPd98X3zif6Wq5aotvZPQKEPGn49QJZ3132p65OYMpoLhGs6s0dBSBQR097iZDj0xXNZywIIjujgYWIfn3PpDQW2uO3Sy6t6qgtjVoVuYk0JX7in7jf09YkdPrVET2BadxKEXdUPPW0auqfQR17T3YU3SzNpKMfLMCsT6Yy72YRL/LRC3jQ9nNKhC1/JF163JaExXB3T+UAKYe0z3/eIYVMeVwbzaL6yOZIpY13dT0arkGV+JquBkmzZeCjzGE7fSSn0UUjv/EhGq2XXpWbsavZW0rvdfmfYxiE0TeTPqZkGDRVWOEPKV8LQOX7eatFzm4fGe/U9/3g4OZUdmxZA8yjO9YRxVP3wT7F0Dkt4eXa90Bwx1OITfjIugf4oMCxdCSsGWT4yngMLfl6iok1CNPAyZmmOYw5Qu6uDbDWvjmDNsdWjeIAdwJwMGAnwCRW0fIzo/Mo8qjUmSOwJpq63dQU6spnQCn7NHk7y/QUyKbnApgoMvsEH7TfA4EFNHvAy7JuqNtI0D60TtGhZfagSwj6fkO7md/994hMEPSzXCgrdsqPKxGwfEDDxhMQ1N34YN1jjQoFwPX5JTAcQfX0I8HorxzPNDcmetu3dW8N+W79Fo3Oy03ndX/Nb1/++1/VFMJaD3R/APCYTm6kPKQlqrH4FRzYc+v4dc3sqRxvo40fq6uWl6hTeV/rlhfW9pPc5qQaQACQDmxaB6X8G+qLde8xWxTW8Ym/6Meh0c5MeOpNMCnIa19l0rKP2thKdv4VSC46lFlUwpr+BMrM1LZrLzv2WoV9cb3HxWOR3HiqSb+l4YNrRQc94qIHX1snb1pcMvNe2TAsYhyK54jH1QmcbiZjoWbUtfS8Yf3GAkc/6i4UwaDxBfmmWQv3vKj8Zy5MDtkMJ5SuuCljBwU3gPT57VmSwXsMnEb8+X/6aCzRpSb7MNOsSM+a31J0kT8+YFmB0yjb0uHC9fucaU+wovOqf8Xk96iK315TMlG8Smv+Xp+vYchyJkV+zd3pzpLeid+KNXvTefv0yq2e3DzOv1NUSlZkAIgJIwPP634KVZ6h8S7Bk9xEB/48CEcP49wAf+bH2BjzjBvM5y1PUXNZl3pTa9yav6KhOkhjMG8Z16Cpx4B086zFsO8tp8BWMisHY67cgMtA4Ey5V2ujgm/tXf/4RA5qo98WaKq3HodJxyD1GOfr1vKcJ4q85gENS5pWt9NGQ9d66FNYN1HK6Jq79WE3GOTnwzJBqS3hzcmhfkGCcnWihQlsH9PPHdVax0Mt6mEsdiagTxjD9wbiOF74lccD4wQXnyxSz9fiV6iiYdpd8G5HIA3CCdhYmuIrJdfHUxRvLTJqAwgrOtHIprz0nLvzR39PdDq3FKxPmOQ8YIEOCJM1vWZC+Yq2a+qa4vBjgbMSEhY1SnG42dnx+lYUt1g+EgGYHOtj3oES6SOZvXQQUgT0cLJs97kX8jwZfGlAi4cXyLECzu+Ls8YuRMl50bCcx7kiQvZowWQM6qdFmObVq1/499p/BslGXQW+t7zTfkf9mAPaEBLlTZ2/zcxWI2nV2WqpE/Ssb+dw3PNEx1MMJmiK8hF0qTuHQ0fRpm2PR/HMq1vKLNW2iN3iEpsq2jYyvNtsbCT0q/IpqT+xQzg/PgLROSwAG5kV9busiI1MD5hNJ6LLz2GR/XUmPCw1j4B0UmOoBRX75vbJEIBrxsjJbJhT0njerY3R9lehvsqJwhPhtqkKGUoAeF1hYp95U9Fjb92VZ1MxK/iEA+gy+izUQMp2Uf20uMdSETRV0/6IPKEat2H/NDi6PsviwWnTf6bjVWXjWxfibnBtFjma5i2ZNuZkqegErKpqK9/gMQJ4duo8xsIBvbYu8vj9Zz5l93RQGO4Wsk4tgx1IHWLc/AslBZkzDZKe7bjrPoK87NtbZ0ih+UnUeW2f/SDXbRCeEocyXwJRdBEXvzkX26qeZ6B+0zn81vdnIMlnp3zIU2vOU6qk/KggOQQnVPPMQR7s/iAoBkunutmxUedFkN5goKX49FAsGIn5BOkJQ/P6RSSUhSR+Qy4W/vri5F9rMt6ZyYJ26K8fMRAOosQB0WKpE0ESOLSlrqIGvGdWc1VEroHUrareyIf4gwPkl9a2/DOeKP039WXaAfD/e7wbir4RcY9YnrMadcBUOH4AVyc3lRVGgnXQ+jgZutSY11nXGJT2Udqu20nYtLifHgS96aXE5wX4Eelh/LUlTU1muK1r545hncaJubP2tdzSvNXSxYsHk9ONYBW7yN1J95jvO1sKFWEjNP5zJEq1j+kA2SDdF+iwvlyPh8qrE3fBDwQfDDMWpYxuKz0lW/9Dey8zPKYq+9O7/zaUCXng4nKMnQOj1FgSqmELU4UEwSvAK5A/d9DJCF0SQqk4ompr3PWCZLe6oxRQ7mZdkTr882cJASvyM/yZDlg169gPWJHngkF4+XsOHOK6yBXXHhCqRGI3BghxxrSubQpLKWffXuC+Uzq2HkGVSsq7oLVgF4/ZgUuEo0yG6x3+g9UUs+fk81nR8Ed3ig32MTci5AbAYmiKWfdKc331hbqtC3XoAhwZPp2bq6e9XY1StDl8fuITLctDhrJHXH+IvmerDd9RsWtYxOU2pf+39r0yPJ7xnMEls3mSb9ZUlUwBYu8U4YnlHGFh6PSUbymWydwLfoQiyykneoJbMVDUH3bpfPdu94VtVEincJvKnGZU2cOd3FK3sv+HF1eItC4nXoaw2Op4XEY4BhD9h4QxPj+ZVFSlaRhxGgXd8A5u8QM5TOdfJRySe9sC7qaJEMeXwIytKXMrqJbMipaFP5cDhk/BCzaAI7xr5SeeelPZOBzISm9SBAwAnZFlhc8uk2vZX1CAExtdEUlelQYYHykF7evHceeQ7LzITMUcmyO3vu4Z8Pdphx5WY3A68VtEflSJi//tGLVHub+tvrP0dEZsjAVePwCuUvTv+gxgz4RV8109EY1ZBhSTfNSdbUJ+t4hhdFaajViKIq749cApWQebzG7tOqAB6z1zrSZA61lZ2If7cDN6rH41tJeiJfqjfYdb+WUulQYj8QdmTtUp5LB+vdecS/pv7C76gkbP7i+chayJEGqrLC1vp5vjhmDtqcu44I6EGy/gXAlv4X4i/TLjF1Dlxo3yCYWTN2lwBr0+F8GSciD8oo7igvE4+AT5NNVUHR1319NIV4HPXJNUxsGCqCo/d/70l4P0by+XmAy7TiLfPZh/ip/HEqDQEXKA0MZBZr8vjfYX2mRDGw5vwxSiUrJozGy0vg3aKkR0mBW1vZvuk79KgBwR2TZxAoiWCDpHU2HTclS9DXWUVEqiJeGt8A/mEPRZSdgIm0dMd7xTbYUD7cDE1UMMGGy9f/sx0pzjOMLPwNksxn0vplRRIdmTmcynlMMxjSuFoLAnGOm8kaZGduZA0B18QnLC/ZCKPPpQJ0MF9Ote0DwZtHQgVLpE/ujO4XCIG41wMl+iNTCgwTOlDYNu7Ndy5bM5sfXAJzHD3H6WpL/wDXrs/RL848ntDrPYkO6+5SjcfJyHF6TvQ2SPllVVhIpKBBlBvZuJUuqAS2H4vpnASuLzWT0+pyD6Of/rzvfeBmde72KTe8JOutU3LQmdXETwI9o1UR9W2PAt0f6D9lQVCwLVH9e0UqQZzjpSeoYmB9qJ/WMuh6T14SBr3eY/8Ugy8E0xaf0WxqmoAdTPRFmw5KuiK622PyQGisn2fLaMoWSLJpYkEo6n1KzEHgmX/zspCP9koBdAlgN0th08mb8d9HJj8C+LqTxhUtClXJ79hadgMhfWP0Tjk5isEd8bWcD0Rw13BSyABzv9OBA9TPzK/mWSp0rMrXYBlL8kRDi2LxL9mqSCZ8Dd9vR7brgpjK//RZr9iaUbQpY62zQ+URRxoMn2zrQy15j2b3Mda0ie6iyivN8IbhRL4S9qExb+ZhD9UJBJafKL3iL+vz2E7VxkF99BpBZSlcw/w94awsfpkrdWy/w1GQ6QH3FsJyH1OedZrokAn0Q283d34U93rzOV/SqPAm9Yw3W1juEKNzHOnAazKzTGVTspCS1ZXnLXRS7Ckv3BhRgcjmATjKvpvdPKAuDDbxaiLdQi+uqcuiQ4TY2Tr3jd8rIdnC18BSSMZrKwPalMVmqGIPF2kghA+fsFTX/tLNL1H009+Q9SBk/mFJ/ig62P10WG+/xg+Y/MAuH3JKALwa2SSJNeegdAs7DMhDpTlKm8IeDVaf01fh3L+kGv0EkAE0yquYFTkKGJN7x4ig/7m9XgwSvKIs+Mx5q4KBGiu9bnoq4k49G5a21kKT4YirEQVqBtQ+SCMUjz0aiGP4nKhNjmKebPKOn8Dqk5slHmwk9zTNY8jockco0+H7YZ56YmA772Ulpdmeb4IhL0WdbFlL4j+gxjpsKFZFc6SHr1upTocCP0Ez2Y8J0YkJKKJf9La0ccgII/Gd04fgO9Eji2D+tK6+daS2+YwPeICZfDEDxpJQZXtLz9hLMdOaOOPSvKceTfLd0nZuA6CMCsvbduvz0nfbDYcBBKv9vI17m/LV4QBjligEZMQcJdYRS+pamspbBQ6k44d6woaN6vjdaE1thT8/T2R2ouuE3yOGcE9QIw6wdt/gBEhOayXu2EP21gFcNmanCpKVlbuIzK0Oer4pO7k72XNQtXciXgMmyesNGgIWRuRWXZpkcdjoawTCkiEA/aynRLDE9VKShfkx6kKGAYP6wo0XsBEx+OPun4GBW4TBhyWTfyruiHyHhVL/vTlmd3OU7hzUd0+KMSaoc1GTy7PceqSJv8DTGuk013PCXocauDd6SyfoTwNz1ZFPU3PXkBIqSYXmcnEUos4xobR0bc38DuaRakD5TCi6S6Y04TBKlN0MViULr8We4I/JXOkFPQGbEqa/ZXP3CEJ8OFkWSjtF0JcioEmgkRb6hxWtQjb5eAPqWgvWyPv5f2DGgjT/eKTjdUdAkuy5n5dpI+O3DbTySJwBkRUzcpqvnhRJ1dOq4CsCIIGVaoKtXDY0MM2j+kAmkOh7RvHP8p/JOnOfBQG2X/cskeUY1hWV7iC+ISgFGcPmOkbEU7/yerMan7ASa1V3dpXS4qXIKzyjINc19Fg9oxwYiKE26mIrbCsmWB7lQbZ7Dg5s1g1KmO8seL3EbSGkglG8/fXSN1KpDXfwidR9no1RvCPpuC11zXgTUEM+26ZfOkfwyiPcRiSJFwRaqZnMQqTQiINK6v29kUT6s3+6trYR/TjXflhekthLeYIlLYYnXC7BAqHC5Kw1aRc9d+o6tyOvkHqFnzyi8DfnPi452OOg1+t9A9XRH3p8t9acE9qPUVxafhALfYZGZxogGC/Ocpgu0/l9eZahhEuTWVkBYKQz0wsQT7meJsPTOHMxm1QpbDYSwfyJ0S4CUcCt/yByP8VDLxOTRGwozRCuMqXNnv8/tYsS5G8CcB2Vdn36lbt6dLj6+YRYVJn3B4h0VWME79uT3M9aAyI+w5Ti/yB24tsTsYja3EIin/hL2Y+HYUYW/MYthaTHuNz5MTVgVFTM/jeF8cOjx7aoRKoSF4DyANk9wAm6Tdc+Q99RUHKAK5l6bbyN3oZyBNVIrIh5FzA/TYTB/4H8IzTN84h1tTt1jRwCaIafYbIqvSKUmzgiDGB14xTFasVE+iSrU3Dryh6mJf19PCTXaVlzDuB96DfHjTAZ+pMu/2akTFPWfwM4yZQ1D17Mc4o/XRq8yk6zcfetN6KkgNPEg1XyG0qLOhvujJwxgSo1m6O5dCZfCZ/tVoETvy5gcKYPrdFZykjc8w55NFM+IsdQWemRQ9O/ShQdaL+YQu9OMGh0fOJCO3bqYViZ0EH5hXg3rru0iXqvHer5PqT2QHJ7QFBREb5yTxba4pApH1XjGQumFRXXSnLZqQe5w3cZwgtF5j+R3SxAuvD93OGa5HDT2UyXQK/zMGfcAnHT0q1kXAPr6WVsrlRnqipoM9lpw1gg8K60Tcz955+gWc1NB33CJz4JZ0AdvK2Abqz7FXWwupiyTSzozvxGjxbWiI3MyH/3HqGywacs7hy5lXCWhmpOvbktbhMcVX8JaGVNuYsVn7zbnJcN1YD+qdy6CNHZxj2N5F7KISgki4eQ3b7cGJ/WCYEKI3o86TEhuw1NY66FsxK7ocEDywcegTUPOh2GLAfbLNwFLy0lXFl9lYjjAUTzkU9Mrt705uGsJf+5C6JKpHUJ+9OxXOph6ZQf36cAThEE40V3j5V87Hy4aIsmO6p/vIVbALoiCLFKshGTl9p+KVtFbA4qXz2vaq8lRlu7QUNhVOwzxdiNNBLVNwlbob/nmEvb7PxnS3DXjJW/hXwekEVljuI0VbAvhBM4m36sllWSeg9fiH2r/5Ntne5QCihWM1H1x7AxItYVbPWePz8BhB5kd7FpA3f2JSqLxGtZ+eLTQrL9TFbmf6G4YVh7VDHg/6Guyf7cgDryc8A5j3Hd8PTH147WT8kN4RRNVfBKlYM6v/8DVJmM4myfvaNjZOhuxLxc5ATYbLvPUqjOvfK+8QCyqWO1FXfVsiA20tQf1eSypzSPmb4SGZ/TxYTwhE5LzD7ihzhgV8Tf/T2taaFTWhgYpcvumcQi38T/4abTwgBtyDwA4DKZtE54hfUU13hd9Di0qh+L475RlFFaryjwNx8Sls/liSaHIOFBRDNZQ5hPaVk7f1HALj6CcL5cjrHfKmtrPwNYQ7V1z/JbEzwBXTJf6N1IJ8Tn7FzYC93ifkWdT9FJvGl5IwxizP0waxPxyDKeMTPTA9t399Ab2ZsaQr43rEqtC8WI6RdsLhw3WMsKRXe9qS5aBqflocJHNFAQaLNusiHLclW9XvnN5GJO9TluZZ6/BWZuYZxJc7GaPbWusbCB5yk7CUiVDPQGBzUiHmOJZO0s0Ghftu70aIIIFtkO+uHD8Za5SLrnIOvkPqClmBk/oD0AuKjggtHcfoR7dk8hEVHERQCBK0AEQeNDYcK4FLUP8hUEaa2JWvTzEcoDhfA7D+VP7xMIziRWmVou4bQvoSXx41Z4RkeegiIzFv33deR8sbOij0YlvcMyJdWRvUV6bzphskHGv8gB1PtM+T76S2h+2HQjTVyYCgLqdVdSR/LRKhValkulLwU1+uQsm2cSgrDsv+hSEpSX2vIUx4PwJeqKGit/gwRUpCth+9O8Fgws1xcbdFloGR8oRpfn0/1OSAQCWF3cyuEZ5jw4+NNJldPuWV/21VJZm9naukFS2vhyHX2d/5SPKAjomgL1RV2gacMI1V1Z/kAm9JIFavh3TgqbtzR6TftJDB84AhY+FepdHV+I6rJR2QTnML4CRVcdbdoBBoqF3v9U7hJaPNTxBUXBlyCtgHyVJg9LJTtDDXeLEIRh2NH3dvRd7td7oydWkVNg2PEMBW8oQuxcI5eEZ/vxNCTyenAd0uXvAiCfmw8iUHfaLiDc8C4lkc4q2YRJiUscwDh7MPXFOYOgHgrbSRDpk7kxkrf+BZuWx3MTSMxaXyUbbFFwaB9EjqusW+79p1DM/2Zo0y06JUKn6r8jzkXxiLOdn99mIgQ6jH7unbhFcPOFkFzCq6/HBI5kX78jOKjYIEa1lt8O9qeUuQl2gOnMSLzhucy+RHyn1RYowUkEdY2X1+V8aqEZrJaVYtECwEjBbqLnUeUvDbn6Jt6L0w2RRVclnoBqoXuh7aidFMaKMAGaStQZqbQBXnJGwo0P3G/vNswdSYx84pL9B1YUJVZlF1XlK+RUwChE/XLMXGZ8aIKZE+187smYd1TvtPaGyFnAJhP/QPBDHL2+5dG+IE6D659Shf/WBiufsqSIiiQYDKJiJTkB8unB32PGZQeQZkHQH18TT83Hn9J6yPHcMSCVmYCFtlGVoeS9Rlfa47jRVomD4BQw+beR2+zoDhuT+B+xo1eA1IYG0srTKzJAe9h9AsAdJ/XIVDr+nXyMRw2pWGYRUHkczBr86cBO2/ecQjzYZ8QSIvK7TKhansTz7RusO9KQkavh5SKqMDetTlJ5o8o1V9ZO5VWDPMf6fvDbiABmC6/v2thnGyJmOvN3w24VP1v3mj3BTJXoMJDNKilSXkfPTTVWOV0yftRfxMuwC/XSrgtiFtsqS5LdyXbNEWlRdauD2dzd1nR4JpTGk4/gb9HUfNX+NJZ0+b3rweywiHYD/LX5z5RTMxEStmTXoT50uPw1wSA0PXw3hMG+hWySnGoFMWUzVnnBIQgv/PgZ6mS4Nh1tcUDYZ0VQzUDx+4iXp0E3FgAHxIIvehQIMawxmzQ0y/33xfhA8ELiNwgF7K+UHQ5I/SgFtIQXoDI22J+HrJq30PiREt9yswEsMYi5FsTT4Gt/pJY1blWvbpn8hLGEGqPN2vT/LOrCCqA3g4Ou5eY+IENs2UKl41RhZZ06mb8QYK7Xw6zAan/pmtLOCFQIp7pqcWvLnSHTvAZ3E2wNh6x7ZYHMSk/X41qPjhGd0b+DYr1DVWtjZZZ27ZYTno1I0IRvRevcwfBE51L17yXq8y5P3j42bxHqfTZJNA0lp4YN2ypp86eJF7Uen71VD8y7Z7iv7vYwU05jaqY3PTpb+bmSKcc+b0anlhKoYaEcA7xccBXsUfjDuujTOiJQfhBG65oXlPCXjs46lPxnkJznrCDv3SMiPLxJVG0kP3423ZmMKOYpVbzpOPc4TDMyAkiLV/S/P1tsHH9OSjWxS2Jn5QGX+g9BKe/+MyCM2zFr/gb5QVA3vUb+9qRYqWpN3hrNpAWbKjuV6wEjIX48sXT2FYfRneJadbymmuJaPUL6zCMFSfo3ECe6ZjgINLZ8NQJd/3VCPokJifN9FruHoPaEKrslcUxErj78aHkM0JCfhfaAJt0TQkWKOIr1dSUYcXXUE+j4lGUcahPHsljSlI4S01is1UPQLPYHBlv7dlyae7X3UbmnzmKG5JGNgoyDp9HHMmovNVv6/niaH85i8cWQx0TopnKqI/85jND9G/Hg0PMXIcW7fWPjfT/qaEyMyqiXn39G72ltnQ6rreVzFe44Y3q/u3GEjGdjZWHe1vIcsADqCBLQ4NDtqP7fn+GAdScj9CVVWk2eZVfP2KsrJsd949uzn9jCRP7Mz2P8YH0ig+qwfAzG5WpzmQsyExe9xX4QMdTI5L+kGylyDuHIx/5w38GKJWjp7hdN9156omvP+acKY+6V25lAElTTMUZpbXDK1UF9vYtd+12kXxesi9CSNaJXOQhC8wq3VtpEdPqaZG0feGtO2KCqk3NmqQMOvx+Gwn9mCoOTTiBeFFd60fMtvyNTKIrpIhiGQGGDgDMs9IDNyCNww64TFujN//l6Rni9wDqek/csYvf3EIg1oG+jvyVWw9FXR/E/kV23o0poTJffKZ40cNLeg8ezqQ/lT8H7m2LecHbfr5Q0YgeF2JKJ3UhDo7iR3cuiN7nINL+xjGLYndFf+izKlRsmsd4m1eLpEE93ZfxJfj3+alaDKlhQubciIAZSU3PGSi8ziHc2OPcn7o3n1AtEMenZ6PyayWWrZu/Wf2lod8/7/d7NM188I+4kxMac+IHKDtaLtNDyS+Y+50jUyQt5EwI81+M/Cutfrif7i7J35Um8cczjoB5JJdTYkWZiPTcJkckvIRjCfbsEGRm+pdK1pE28cKuYx3LPvaRcHNQoOgL8NG2lUKr+5uM7oF8U+A1su8vzwtLbG7r6r++EflGPA/2G/ND4v+mNhF/s8Y3rqCdq5fU6VOP1yfF27ZqA+BtWRFbnrP7Rt/fwAYGd5ibaw56c6uXMEZIQXh5J+E2xOqYQdvxhCi6LjPfDdCzrQPY5IWi0+d5D4D2DBBZprAHPCEpNYWxwvNeQpWVH7njlFNSTW5imsxhlV9CU3F3UZ5fE61LngnExMZueLSiGs69oyXCfTVs4fFz8eSrafzqLi5avOxqrRIw7o/lvP9prcdQv6LvwFbUWwSZ6N80rm7tj5NAR7Scq6t+a83/m7GtF5gbtT83z+bmOPF9KdWwq/qUlB2awCQRI+b6bIcEhLwklxZ7lI9iYRhqUpyxyRIV8lU3aZOYryOXqw4lRHNAeNHst9fNePv98cHSok+kdIoR/3P8JUKp25t+hSGPeKINAB3VQwDterdkEtaN/BaTsaO775+Mq4KSYDBRQLtmA0pJQ6jdSHLDbRS4GpeLBYPwDdTcgS2/L7+cki5EAhzLACaoijY9tl98YIdouSN7QA3knXoeiTMdUYphKPuqkBb3LR2UGowlnxJVv7cZOTzi5dD9YYoGc1yif/JlGf0kyd3nOfnA4a/lRpfYTt7m8kw0opdFE67bvvRM872x5DfXlUZNV4BExNBLwdvw+ZvXgIcEtV+7XB+GipTOrrUXpshTD/jbL+a7CZ2OdBJY8JGCzTSREHwNDYehIzm7QKR6QF4HR+gT0pmPk6FbzG+d/e++QGCSrNupf90NytuQKNltE95NYJGA5VG254DR2ogec169HhKTZ5VjKnXqJrxu4+J74y5bps/8H/kfdY37VDO5DzjWwlagMuRHdfQH9p4deDg+y/25EsQFwlefoalGXFz1NhYwEr5t7Xi7cVLSx+Er8x6askZ6wgTFk6tsDvExHgyAi+BUYheBFQT7h/gTfdgd3jFWwO5qlw/k36/4ss28dcI1z3TuxekenjxW1SbTjTiGQC4Q6ED6k5EJGnhKaOnR+giXulJ73Zu8Nlp8egBOaLXZwl6h5hZMvseBOi1tyA9iFPN1RRJFQ2AXmiYZfKFOOgiI8X9VA4TtJUgbxrkJSbZBx2lBnsd4otbRrz1V70DReIOoor+HZ7qm5TQozK+Nbj07mLIOzyaaj9By6LQyI4J+2b9bmIB573CDBHDAgCpFHTYOTGnhYtHUX5Ofkmqvq7w/5mNqmbBUwWb0uPFCpEaXau8YxzRiVXKxc/NzNZmSnANjY0EzxVeuobyRXAd0f/rP0PZ0lOy+g2Irb2elaL+kEGX3ipfTTnh8G/u16xHtd0xxIbzqiOHR9esqg9Jh6OyvjOxlpLOlkYdyJG7xWiOqJLnyObjTCEHE+2ZSpUWfDwOk4DkBqDVgF80MaPlJ4NH81DhBcj2J3ANyYeTGJs1K4lrqxiZ0B7u24wRyoV++RK7b6S4dFjPSVkjbAyzIuSbkp+w38FU8CCDT1znK0v890MGcjG2+lOICHGOYXn4W3V4GrX3t4gp1mVBsP56++GE+kBtAGt0BtM417ZAiDACoiOZn8imm0b+Eyi295C0wvHEW641reCQfMk9GVi4h3IxarHt9vBtnK7CuBChrKraOVq+n8NOlQtD4NlT1cVUmXhv7c3w825SlWVXmKabtBiU65Pjzihg8QVQvJN01FXbQckN/HrvYfATVrSmrfLdeeWIZKPNwOT6e2ZcCg7HMIoJDD6XjK3EM/GFzqWOfzZQc1tBF/czv8Xbx4S/zIuthh9G0oFLuwiqpEJ4lPtAmsUn7ujY82IPFZ522pF4QZwi7pl/Az7AQOmDbQz1G6gm6mvWHqlwbBjeH3pU6zRKq3sHTV1r9xM/1LF9+yc7yqk1ZpjtNtXlqVMj1H778CrkY67iq/br+GL5+0BVz4kPy3l/asebkX0ci++VczteLO8HCElmwIixpO4ZtJ3a3TqY0gGQP523SjQKSsY1DNj3/+7DCbsSH7YTOFhk27IEYnmtfuKwa3t+vn3GE0Q5L2jc2cS0vShPQoXj/Mfz1ehzEiLdHJL7fI4sKlxRTUBdYLABUsfeXFjdQbLHm51+vnWKmfcwgJiYmC/pWVqluu/a3XztRyZQakr/jCbgk+jreaqEtmdnNreMLbP8sUL9GD6zVXc6mrExTjVlAJSdr8Us8Kwkx2XeSxmfUvRulKNNB++bo5ZX5jfhktFl0r7XZ7cnLZxEqtNaDu38y7eVNcd83NGLIfAqKji2S5PFDCa9Je5hcqMx/s7r7/rng2DUseGBr7YVYxrVdXxSPqAkWzpeKD/NO+8kPXe+vQ25qpreGQ0NB3yfHFc1Q8NP2vwRMSHMM7Ou4ovSUUnzTv/G1rQNP1VZkMbt6HN+bq3gjenLWb3jdZoGA4YqTpmloE5kMz4jMjOaM6j3OxjBP1Djy3TxB7+2LTX5OKhEngQUXnUVUlpfNUGeavCGu5FLigCLRt75c13zVeReINnQ+MjOP+qHvQg/3MIdErsSYF+6Lk41bIdOnG4WxS3IvfcoDQTNvgodekLnYf74+qVi66X1zTiBmzVV4zPLYdqLqTmnJwwA2QX2zDHk3zXz0gDCBQWOv6f0bOPYCGn7jSsZXEVFBDBsTnLqrUpJeypSMwfPWGYNCe1ZVPw9BthHkwKdmv5QIuCt3YlFWY3j4BdpS3Lr4xqJwFJ899DN8ABPlwD4yJ6G55gjUsvWEUo5/JFgYXiAepGey4QoXRfiqOmj4JCI3dyiICKYgswLcjBrUsiBvwlJXPiBDW4af48XFBjsMeMRxqmMrmGaMy5+pxM9nbQEl+hCu8LLm35CrRddwSe/6G4/+FQ5tL/QPgcqCuweou5Dy+vuS3eMvTd4e7ChE7PckuA5IP+NeiKypHyrH0fyfQsEY3haokmYTp3RpWnWFbIkLaYFdBuioYYs+ssjQtI4xLf4pX6RwTC6sL046CymZdo/iil/KZu2Yy3M8COuVRhPxN1UfaRQUWIafe9Chl9MCJLjADedDYsb5HG19Dj1qUFm6miGzyJCzUhCZOK0ohO5HS6Wu6V+2lHW2oNzUGnESKn+lUYnpKvvDdmVvcAHBX5jFyCzrWXW0U2nvXxnMgbYBcoZnvuxP0k8SUaoAaYn5PnW4xv9UtJEy+9aPHEqpP1fIEEr52ohe5pnhvUgcQnuUKD55yVY05W848DG8/B5LzaTzNv+p68u6+g+NPoUuuuR6B8YX6p1CdDYgr7nXpzzbs/G7u3SZ4neMP0IQRziHvqy9dp/YGWGkwB8RqS+Q68Dtz+uOTg2GvxJTFivYz7lIekjx8pewN2CaDCui2Eb9HKGChVUFpyK8PY+bxo+qosLUycdMFcOCil1ORIVeEceWcewP91DTo1Kk1lov4tGMFh2k4CRwYu2eJQ0HTTGb4qtHpfAEN1g7QkI/fV3kXaGKGVbLyTzxTr+GNpQ+K85EW7qdAFM8wHAqS7s7JlKarrJECXq9xbjX2In+ZBT1Vy7p9LTBooNBwhqU98Q53o33Cf9a6pEqE74LM9yDcj+DLUBCTWOOkRYKvk8AZbov51XygnP7xU9q13eF2VwK9xFhV8HwrCX3Q4hq29nV/MKQYmRAZg6Ktr9bdxmI94BgHu1t8lTUMXhu3UJlZXugDrv1MW6a+oaFwbTxuU2ng0OC3qCe/6Uh1JIWwTvwBAkOW7RdVeTMDmD7HgrVbm62728H4BftumOlGHyzs3fXtFrl0OfKkBfzRUCYegGGJI2qjG6HaU7MoRWVdbdTf4STtLETbiZJ1Iymgm5hFvXH3WfxuUKUEnMKfpQ/PgUUXC5RyMfQ6dQQ+xN0Jhng4bVP/oze8rhkwKpBfkrR4E9RUU1YpV+AuYywlKfPoJNV6zdmW0cntzcjFztH9chfvsc+gCJq6TfQG/WIiCW+M/DMJTOr0d9s9ZCAARNKSojOzvk2RfZK3D7Z8kiFJTc55VOVm7IgBHw7Wf/aaTqoVTy5EKdEi6cdszvHXpytZRQQ5LJ9lFZl0BWbPT18cUNp+2l0+RjoT9I1w21Fl5PU2Gh7Re8ru7OJ8eXoy7SgjHGV9THaQRIkLqhB8bfu/2/PpBbHfbmFBtAc6InbgBEHUtgOeS7Q+7envhwn//066ye2MVBKYs9r9Udj0hYDJdypfX3V4OAy7dlVJCSoxUYYdEq5lkh2+AX31F5axlrLqnhGc5CAwj9x6O/fmWrRpPGlvvKpq8urGv300pLc72fAvF5v0Tdi/YW2k8WbH/A53NCt0LD5lRrOQ5l7uVA/hIexKLl3KikRdEvCjU51P9clKga3Kebr9q/dsw+WIkclBMkcmOlW/F00G3n4Ncm73xQbQZjy5+AXFhfrRDE7xCb7gTK2PoNVGr5MMoqngghaisnExyhf0CQ1W3pR5+CdZDQALmSjDnIks4SM3HCNxgeYUgopvowxFL9M7NMTfyVuvwStJGc/ddSe0zQO4EKGEf+zmrWiG3w/1Wk1N60tBjoxvn77ZzFWqtRGpO+4VMYiVAXhd7g37+9o+2DLdG52+bP0PTmB4gftkBapSTynAqIeqRRSy6TUBO8bn6Iv/HZv5zxJxDW81n9X0sqSRzJXoiSwVA3Ju6T8LmahhUIp8AFUHOUO/Kl4WhhlRCXKWGjbkqBRFKJdgZBbPlKfeQdcBGQKsE0KoqwsrhpoUEiVZjHS1P6XZ23jFI6FsUWxgfCWPoys/mtRUhYDpHPB6xgFCbkm9HLGCgc6WYIpNF79OIvnOa7746hG/OhG+IVtf+mds8G8ooT6CBVKK8ppaBaZ4SbbZW759slDCK3QNVMx/T1+n/HIXYYqefqwEBjzRT6RKrxaebhOsJ7hKxd/gLMj5oyHU3NvRxjfi+osIeTjhFmTjvFIxWy7JJpS5U0q839MiDcRnwfBx05nFJro8KSSMCdpUt9DhCSWYirObUSZo9ClS4BYAp679oZggLodwyTxIiW51+8AIg1++Ik2ckg/dfzFzrpwB4/iiE6hnezYRlRt3qfSPjL5wZEo7A9XJr+hlYJTpj1aGGK9qkyBhlt6xOWJE0uzBs0pu7ik5/ZVFVz49uQxgrDH68CUvvbSDSQg2L+re0C7gmtK/F453YLA5nrPrWeKSD8gACQ9lr7eKY9G/TrHRFku/Fyz1v27FSE53/d3owfgZaRxhMC86+9322IKXV7y/77aEfYQ2FrcDIc0wMXXJWAkIakNYmh9yvKUO3a/ht4DOOyNVPw13r3753nyJuR13BWmuagnKeEO9Uy7727bdapSXQ40ImGeZyZDCQKFhpr4OUYGDFaRKtf2a21vPNOVYU5sWKZpPlvp9eiJH+ngHu1eQbVUP5p8/xIFNUu4840XRH9jz4zuTGpC2a2iDdSEdKQR5X9QFDaFOjhtgOL+arQigP96nl70+CIfUf+730OUwizpgstVHAOEFMoJS7uccf8KQTw/9kZcpoS+6jqsn3ouJg3RzQ6usbNkbzwXO5EJtO+3VRdv+9ka1rWa3xa8B7ibjiiROj1PC1JCLxNiI38ECfofIrWJA84V138JxHg9rrrGfx9HIKzljRwF9nqJRcl7bBGnTnBzjNBANeO8QkwR9EB7AgXNI4l/66lzSQHDKAbmJ9rBGdrlDcQ5WOeO41JnvpK6fbvp19mTpIaVomvZELW1k8UFSXSHS7+sGF2qc/whuoiqUsd6aBkRIbB4tmL8j27RJ6hmZzEJdZH4CchkHFyOwOPeQNX9rsXpMUKC7QsvPosipBDw+PKYdNU24HShFBSKZdYOEnXnF1pE6uVmgu0OmQcSDlDbt+419gizLvVwB7hZh4L+pOQQdrrqG4gyKH43d49oRSEUHHi+9wc9G6KqPfqeGr3PQ9Q0fdSoGAzIPgiqbpVF/LR9k+i/19QEsYzd0dcc7VvdZL3U+my9z0kdlDYDY7q/Nw4OjJUl2KMgsoq8rBS+VIrcphrTRIcN/YUvUCM/IwrNekqyvvgU4F+soUJ9PeCUOYK/0vgKKkWxADAiNOct71KlnSRecp+0QcGrYBFn8oNUdsJ9kb+i78fA8MPCj96CQfTU7Y0K/CaFgcLgU+jq180k5g9U0MjMaOvCftfOkFI9pIjXYtdUycJSOO7lBHHaVSY+1dwlyFnexz77MZYuZ4gp2zLduzpH13/XT/lr76CnLnKKpnH+cJB4Gy/Kx/4sBd0jdJDSXVmX9t8Fb1uNfVClyvI3oy4vhB/uzsge3WfXANhVlwUaOvqxsUIOk/mLClKhzufDOAADIO1+/ysEODl0qz/VhzDYsaxTq5TqDRU++3rKdJEGqdKNUmoven+30HPjMm+GH7hQxQAhhvCG1Al9I2wCGh6LJFSxwfxRr10Ug/pz5h2DuiNn/YzY5hovAc+du4M4R+SYl9Mj6bu9YJh6f/cOp9WsaCVpkiNtZazI+91aO6RYKuFwS4sRRg467WD7C82PJs5xUiHmZ45UldztCnVW9HfPV/DGW/u1Q3tluwN8mqiNV2PN9B4XDZF6UlZ35rpW2ISU2S93CdOgqjhGncOPn2iUnfwfnNMcT1Wf4qc086TQgKUvDrH9POIRRqgvtJ8+V+ywkl8Wk/dljj6yTdx+u1EV4i8PZXWJkBQzvl+sEvM3rnRpHSzr5BHLmPmHgM0uvhu8G33DwCOj2FJHokPXU4cMO8ego65QokekBc3b07dc5ofgZB0o/J7QJmyOcW3HJUvP7Ecf0jMw2Ar/4r6maWGd+G5HzYt/zkVD96rjZG9oDKz0r8hePpay9YF7ynKtuNRdYERDgbFFKA0008Rt/dhP3uqqMxhHc19z30zRAjvCEcSoZP6uclm8COhUxPXF/HmZZZvMELz21Dv6K2fp2d0LYW72rxbIv3vU7qfUGhE5QMhlG4UvAY6gkl9t/GjHiWK4hUR2zdJ0l/QSU8Pu+6k/abz6vb/ElrFtdd2dApRGlYeULHJ/skAFTYbj3n8FoRr6NlNPBLZ6oyR9bPWW9oAw40p2/4ai16QVYIBYHu+joJ3splPqgyZEqk3dKFCfiPlx5LCEAwye35B4feL3zM890BUgY9dIjYe5ycQPm8UuKTMzt/O5SOmcY9eGSEs4+bCLaJfktqLB89qzKIWBK0J8MAsAVXR3rU36Da4n/121q36jJrapkgarQTamexnO/UnFDvTuAzG79lV1gOktc6/lKaFg7pLqjRPSTTS3yTxnwsc195JPffDUJD5rqsZM1MyY+3MYzOLrp5aoaruIp8WVXAe6G9wOKm9riZ03jU7qjNRxsVNydknUQFYd9pXl/CDqpLvv/EcLIBF5Qn89QVYZ9Bhkkfx251Mr8Vt0Bkm9dGMww7K1ArjNAt9YSPIU6OvSPyOS0QHsDCj8PnL2rPJ6k8YSShz6JWhIhZ7A+JKDc9TWt/+pHeWM05ZRX1xYnS4yl+QOsbFWi7R/zgDC5itBpcG+0zyD+/FSOtq7ODFnfOxUKKDIk51dSQL+6AAeuOe1k57BRdlfZUNHu4HVh/FNUruq4L2gzt4C0of0A45KvAUMT8zwrF+/WNB7EkRAL1DT+tugOhwpmo4TmglYAY8GgXGk8P0TKefawnkZHbnwf2HT4prCjl5tHxZHwk9xkpsqKh+NcwlBVFcx3D/qXoPrM1rLg61FtjeQFghNHtDCjOeX8W4eVnHPdU6OggcQ8RRTNh3adl8m1UFMnGYLupQSmAPEEgKVU87PhTbBisru+ERO8/7rGtK4xMrYOxbbzbSczb66HmtDRj7HHYT7dpzjaLbKgIhZqBkZKlYMrd6+fquXKkiTPJbj0TnveX4eCLpZUCmXRi+JKoFSBZ81xKcV7/liJsectXTjFJM4HbgQdPmS9u3vJgwgHUAgkuXX0gs3OmuSqX8D+5UbFaXGPX1+Aim6hRnA2Byqic5Qt4/w+a3tZykaE8N1L8byc99qwmjfNWoFAcGgYnLmPESeoTNW5xe+vg4y621Jy0iD1qp7CGBfewDD3dgPTpJGzwddzbz9P2DpL7WB96EznDuHtEU6nDyrsN60zwLnBl9BUphgIvrHC6ulZgsetQc395XOj+VlIWmlCQAKytu22n+WFOPF1oAhNyWAiio4gS0hIjaWwyNfEoKvkcktJ9HHCMwCRerol/+nBXB8+eiEB60PnmK1O29Qpc0dUn70bWedNHIhK1tWiAsKh2AkkDl0Q8VhXq5yMYz1UII/zgsNeGQ5cpKgzemtI/Pzg5vl/NDW+9FteAB35UjDrwtHvfzCqPp5w9uHHLit/cVaD9rFs7t+X7qecitN5AZBapSG/6GrnvlUDUGd22e+Bd/1KccQ4TmXrPT+LY1f8sS0f++Xbwjkj+QZGKoxhj/587IUGD+2RsbrrXSu3wxFfKkk1aRfd1/CLbO/q1jqg5mkxrbLoFiODbZvg9icyUyETZBYP2YWN8GOc/7dYcSOlZMzmPoUZDdkHyzL1v2+CMMN/13CXnMhA0XoWN+rGD73X7NqLmqHfe/94WNejvqHiPLEtGD7ZpwGaQeHTCYt+tmqbEdbYEHySCDJX1/gzmWSFvIKXmEjx5ctI4IAazMaEApum4cBFBBGojDwFlvcbIGC3ULjBjKtBhC6hZU1OTvqASwujT+0W2PqZy22tOKmftvWUZ2NJDS7hi8Kl6z4F1V9NMJbOlsmzMo7NW1jafXf1eU2mT/fzYHjSptUHEgBViYwwAZVOFkjPVpigV4OfD5OUe/B9SbgqVlDZG7mT7EWNsZKaKbilS4+gA7F8F3p4nr1GU1MsajrjUovGcJHf+UoEJ1Zf/4mhx+WN1XW5O8B8ljB1JthPijfzhVPYVksZAbpsGQF0swO13ddGwQKCt4eNsm08VqGah3RWj9fvQ+WJODvYoCdJDqhus8gBmQCCpWa2deCZeI690Uah6ZjZVA4zekeyefxyKMkobQEbQVmGvpkdQfOPYhYlxu5VHiYiYS2+FGJMWBXn/pUbJYfEnJ+AeK8KBkeii1w87vvqi+9fdqEBYm59Rn+bpJXdkxyLdSLggi0iV1GVP8geMLObrzAhjB3rZdUdNFpA5ARgiOqtzCzbiG2dZWAsnLV1sRxiN6CFp3LVoH7gV8K7m+VG8wpZb4moSNDYo75uM4Gk1u2rNLCyEWOKTOYbZGu6uRKNBjFlbWziAXWn1LA5tghF97mhkNzFGiImH9tdWQjPnaOn0vWe4MeZhCurj4x/y512Px6Tq6SYhrjP2V1lfUSVZwn64qRd/sF4TLct/bF5cqUjPFrpBIjcEQjO3fCE0AIiUQHNmi7tTtEEOZnUBdW6Fzvr8+JTqIh7vzSS3YMGjsljQ8E5q9LPkjki2fLtFahEt0XvBHIf8uLAzMPDPp9sDrY5rRu3LSOJm0EdD/obga/OTaOSLzBMjbasKXYT45V+9p1NUcWe5csrwl0oRL6pbrhkDJLKro7dluNM1U8/K97TIujDbAPNTuZ3ZB7G7TcYZVZNxdjhp8PMJCJH7m8mLaVVAXvpZboiIGCN+8rzD4jo8g6zUze9HAfyMTvAJb/mROslJnyMV6TipAL3GEE7yvnq+Spp4xv8gqiS8Um7ZIK4QVNxxAVEoBxH4aWGpmo/XOrtrXqu+h3XMHvMAezu/AIekPyzb4LmwNf8lC4jM82pJEV2F+kZ3gxzx839F66+TnKyql2CT0W7/kblVTXcL3ZGAkucFYpSG2whqyc7tdOWuoQmXo2/65B8wP6xRggBPCtw8liEC7j92fmqKEwIH84hYZcmPK7AAWSsTOf6JfuavhYVRZtjLjWEROnfrmIKu4G+NNZ8fKNckmcyuIAIq1VL5TlK0Y1/tcnJFnAfStIFWHXywd7p/M7c0+X0AbE4E/FTI2OKkJ8K2xiM9tFvWO8FsWdi2Xy7LykxcblRBpBCdivL4mVdXEAZvp4gnxDu1NmV8tjjCs9QmOHe2W8k6VzlV0bZjhR1qriYnneGMDyH4Dsh897bP63u/dadtxI00WfpiPOuVAFvLmEtyQ8AfBGAQ8Q3gN8+o1kldSSqtTdMz2amb2XVGuRCZfI/PJ3+ZtW1VnYFzoBuDTZoIBJT3G7ZyG7LfpnOuj2lDAuK2FOIJc6fkBJQfA5Rwb8pbSiUevcQNIVFl8pY8Pi41P96b5annmPUoa/aDIgBAyI2L909xFDlucRaS9WK6mN5GwPn1V8E9BaGm8SPXvYII2YbTgFzynApSxVXOxQwrYmtXfWxQzXcrX11UAIA/VFbka/HW3HOTZKv/jkjSLxPn9gSzSs004XFbiJTMRj+njjrkrFh8VhtSPLCL3Lb6Ap5KziFTfyomHJhqGIYHTD4qmyfBPOAJhVUzWaBugaVjulhFrN+nNcOnAhnSn16M7oYIlYcnshqNkcUWyGz831NwWH7BZ2FD5szZd8XnpNTUM+K4rmczCqfI/OCmxAojj0CjpmBTupTOEDc128UffXdK8IhftszpuPXoy9cZBsBjOdHtald4VdpJJB+/FxySvqGzYXaO8HsJu1ouOehsY9Kln+GgDPz5qe3GLbZlnpGrt5nuMJUOdYW4qhEFZOSDFuZUzPgEL1MPsWzN6Lk7Oiy9olkN+8/kDrlIUc6PFAzMMFy2mhdeFW+4jmbkwsu2+vr8Zsd4Mhtxqi44r4E5ZrkivEaDW+AIpwg/nHoD3HjV1qJXh4bjIqQXgLVbQggFPjaaWpwlPkDdo8j3dfXDUyIVnywGblpSmGBw93mYal3IsKN3XMMw5l4hXga5xM+1T3rHmbgD0JfzL4K3/gL0/YLUXFvfyWUdT6mnD10tAuyVd14PRu9mAGRXHsHR16nhv5Jik3drHt5fBsKuMuOXa4YOW3VfcJ+lIc+JdnEbVN3eQOJ6dG8e9ri5biAT/qQ9qRXQMzp3cKVm+2Zd/aJdISAwKz+lYANTqzzORdyqqFwlDV9FrPKTV49jgfMBvJNNlOWa6KJ1XUyoMKW5XaGDa1DMupOFfn0uiRZK+0q3cNHwwB5kWdve626CZ8n2gBxDZ+klekxQM5VdVAEoXllfpOOl0EOFL9sPh4l7NNy+BslG7nJD27WaxVNXLGobNjLqN8lHuxEz/xe36/YScpv2Z3Xjpb8yDhon8lipuhQRLzQFIBytI5QZMjExPcSVQuKnFu3Rf1TAIWdh+Z6GwC9owXV3s+JCQpxuXGxTDGO1VKkq9D9eXOcIyJznKJjN5+9GwhMpfr9O6NNU/TlMuWeyNdWuriFrMqFuF0WDE273F+LQIVAnS53ZKVLj/xqig0mgCX9vsdNi2IZDultUYzrZ9hkQoeK6VzsGbJZGK7S9B7ptU8Ub2Nsc0ul6d7DkhLURe3eySKevWD9raNkTDtEldYYuYswJEp7cY+4O0SHEjVu8E6P7roJ3e8kIVr5ZeMWaHAGtSkoJ6z+MnnGcX2Mlr8Npkcbx4rUK7FF/A/F9t805PBeYquI2JRx1+iPSBT6naOpRuzinpPRxdfZ+aOeyqV58vDDXPFPYOnbKe38lMEKHl6vCcS0SjtN+MiYo+Chyitbvk+Km+9lzmSLifK3sNUUpp95eoMwLF7qDYhFV4zuyfa2np14lxycTTEBXGemr5bTnrMmqAIqCkx0rY0Qxs6hny75MaODZoieyvF1Yl79zwfN6+d0ff02Wo0gxu0HHBZMD2m6XfA69hsQEXGl7ri+cg6sSKXG9D3FGjxMNM/9Nlg7uxhpZLX12sYFCaujhhjAib6ydTF554fe8FDF/DFJXRRlIGHRGdcHM19BadXFirw0mFFAzl7KW3kJmkv2dj2gvQBF56MtIFD6IVra+ZjmZdGnpzNJfPEQe790ciPhjIDFBp2/ES34ZOtpfUKdg/2luscn1ogW3Nh91Yfk4gfRZ06yALrMGLoygjfyIveVP0wrNvZIodu3zgfczm/8qrCiugFINQ+Hp/8uR361LuJC8ctNEtZmJ+C5VFP7JPYHEcjNftsKdP+K+yVxyX5PO61vXmmwCOKZY55hhzQ3OChw5wIrEdpfw/wcrEYdB7OcuPsDahpPmnWDHbrq8mdorxE10yw1BAwykR2Xe4BojE7i3tz1ceYGOPGc2EeLPzoGXLUDirAhxYfTMU+WeKBT/3m94vWvqX0sXLlspRmfjOq4IXfn3OaJdDkXZr6daNDZjouIiuDo4kodl8pTWINToz4vhX1NJTKY2WGrSjuGvAcPc8pI06NGkfrBrR5WEoezivlHsmGtJcOq4q2budP4f6R7wAPEdjiEl6Nx3iMQwv29j4+vqzHRuHT8GkPCB7n4G8Of5E19LXqLEUhclJ0cMJOWRWHYWgxlA2pI+AN44NBnvYpseor3cAm051ATmYPF+psYYF9OsyKVNnKlPyOVEHD9qPTu7hNaj2pAVJO8YCYIpuEqt4nePn9gvzzWiZCajPWo3qwzp7YHqDEBTcHwW6k+lszkywo0oYuwAuwsJqk4F7+Efg2XdQq9Uougkq53CX5kB39YFSv49qhk4Y7O+aJ0uLoElkwx3QMWjaCkMoWjx6Ci0greZzI7RNbj/ViUK8Kb6QIonUDGw0o7t9VtMzfQ3OIlx4QL+K5re9Hf8nJS9BJcHFSsqt6kcXQVOEnspe5K5TmzThG610zc/MhimBPzvRHuS58hcgN+5HLbyYj9dfrHPtrGDb+eUquwDMs7OXcTL3OYW8/W4wW/SYKj+En2qrA+rbT4ER7i3xuW5bnRpZXTTqx/S2RbQPfAUsUbTssZ3qK5UfcVnVBOEC8q9hQUBaLtwzy7sOj5K8SLlLKuXwSJQlvBCWcwvEPDYeebv4CW/72/kDmyQj1VQJGh4rUHMdJQ15bIWS8C+Y2uzc9MVTcGOJsBy517r2Cgtj1U4KiD63QdVFJ1QeYxhrmb4C+ua8npmv1I8Kaw41jR8j5BEq849js5HC2A5KY+KIFaiIfEu5fVMXYvsqowr7fFnK/qUAIHlCpOylDYd50LQXNtTyABZVNGnjmApFMdWx9uriZ3VjXE9PyEumslmq16VDXT2VXE8ts2i+s/dIqKyPQB0N173PrQmM/4G7n7IxgKM65c8CsWUjKnSWaQOl1zb9zN+OTjsi5l4a9lwvyUrqSifcUsakFGwni5c3e+wHBDBeVMl43/XJaj5o4RqecFtfUu6Vex0e53tNaR94Nbe5cI5NWuDiGXQJStm6MCXnB6o7qDSPUOBd6HttSmUpLDtefbpvjIULXrHYbtJi0IiO3mNv2Hh9dBMix+kIE0lFgTZMCrLknaug4jwhYOcPzRd2rBxrpO5P6cckONocIOSzMA3xBYs6BaVbdcluu/PRgsjJe2ZvGTE90FDlRYLwQSo+bpsh4qoXD1iCeWRRO/9z7fknF9lJQvGgmm33HA08sUn+C3H4jVslhPaof294swbZ1mO8CQSZcrUEVg1DeQPhp8OBGn8KN3hfd8l3NDhBCslFXeUC8crBZUmpKV42jzT42M7DdvT12cQ1jzqvV0zVUTJL3xgfxP49yhFw3tifzaFUkOUeNR6XtkgxSByf3tfHgXe/oQeSrrs/RHc6fEqXRARTcZ/vp0MC75c5zEW3eMcAla2OvvZVjQjvG2oWI8ecJ0w9+oR6Es7/89TlG+YPCDmaFQIkedqv7iBJ1x7UvCUbDn3TzUmDFe3A2KiBoexboTj+JS0fmQ+g24az5ZG9ZxKH7XWlmLns5wzGd+uIJNMW48qtS3gIpuDOzpseSJi6z1jIrZl3jttp2H7jW6YNKbeYpiPILB1RxWK+eXYFlzIriXIHSN0cvcLCXimz4axEmKOT7kt+lcY6tdVWzp7EhykMR7QlDNdVitVmjMRXEg3X0uWqV6GSqsKoDL5kanK9ieTHtevAO+XHLZEnPTlhvTpxW+r5fQ5qVXvulCjxw/czaFF1Hu0rpxLwDKiPy46mz/e6/mKyf0Kz72KHfDkLVJQMlgIC8EmeNtdvJVGjaXaTTJrm4y+J0yDn2nnNNw06YDczKN8cSpqBhEGiGSTZB+TLgHxaRDAgbxXip6tRIs8ZbqVua6wuU3J7FRQKtYSBix3mzNyznTijycUpMvLwndMw1p+EsTG1KCevUqcdqm5d8wZehTt6IQ4PL3WP0XAfGFv79NQ238MY+2eUCEivDVzZMaWGfGa49Ipfnnsq97h+U8Ay1R2Y7iv4RuYqoRViiI9J2tA777mSyBXJo516l4jkJXsy6H1RxnwZZaxA/8aHCOk6t0lL0DsXpuiFhkshSlkL+xirCTcCVDOVYIUmdrbFPhWnrS3IptTKIk4RQCK0X8A3a2nnJhptTJh35eK8KCyH1+Uqh41Pa9uvusps/IalhE1hTBNhlVKuXxmMODSCEPxJm2tF6IZQyE1/zWnXtp6AwVDLPVkHx9rZO1CV5HgnPqV9vlwE7BfvSfB5k+BbfFqqJ463aw0uimbVU0evttoTcNfCzQsuF+clIpwT19hS7xK49sXm6G7wl7ot5vkTMfRBMwzKJ3tUNZyewKCow9omT5ABNQW6X1gQ9tLMIdtv2Gp5U72P7hLjbI0izKVeqx9uOKRpx+iKkyRlCIiDRtT4HrJ5bX4d0DDeOw+grfyj3xD9CangheaWJp3z38Fys6yNWixVYcZX1+RGe8FoOOP808Fgb9UuT9UbGSpNafJ6hS3kG6Fe5enX5LCaQbV1yn6FLk7GhdLe7eRefnBvsDtijAm5aOBL2msd6ZhMXgIx8vOCBuecZsd0ZDkzECeQnn4zljmOvoTTBUdLqZ0lAfWWQU28gZtUx40J1OBhzDdno0ik+SXNobrsuYvz9obh4AzNYK0XtO2k8t4fmyZfopsB9O4qAi5Z47/Q0M0mirrPj2A+hpk3pUJ/6TrmSneXYBF9ywHDDgScycqwUy4ujq8X+nBssHL7u5aXftlZAAKNuJMiu9VqDhgvR5/gaw6rg6t3t5kmWvfMd3LpeyoiKeFvi/hDw6JOWf2Q7i+Fh6Jnaj0t6fvExrr/RPMVaEQltTFzR2uTv70BXlrf9auHpidwk8iZIz5zYuuGm4zeWyT3GiXqjgbi89jF6Acrn0/wUPx+CuMlQwxLuwsv6ZHTI4dMGhs20DaLbmubSNOMZFnBPpG1FoVTKbVNZudX9WePO082hgn+o4Xr3tdeNg4FVVgF4z8TzAcIFwVI39fHttc29wwqC/FQAOSqtE0evoPe9PGBJysckhSC3nrclhZ3Rzsj3mHBda581V/YZgz4MSmSQ21NK5+UT3frs3Umwz94rapFQGb0imQrarTOlR6krDVmNHOd06W4P9ah4GlU8PpRL9eH8+ek0Ups33cyLHrcyiRiUjXUmAUjRyzr0TBFwOsYMfMOj6dbMiSmZwxKgz6MJ0inUUGHrp8e0HdWgq1pLmUj4Kt8NITKmGt3IuNDgVTvU29MdqpCnhY6bzmjM8NdLwbP9Ut05XNBeDlFz44SqmShsO1HLo86NtjZbordEsX7RwEEXZsUSDWyp+xfFGufG4yTf1k2k9EV1NFaOKTEw9L9HpTAeE3bnzvcFuzfUt8oJc0dHWjEqJhtQikQ55Zpxy++ng3pyxDJ1XgZqrA2EbcOX6m64r3jTFlqz68m0dmZ9ATuiU5mMGPACXOqwufpkjbmkHgeADDLOWOMg+zaaONuDseTMGglm6NY6hmQzECcn07OX+trnXMMae1X21uFDX3mPwCieLLfmATTMzerOwPP7Ii4LnN7nNYWBZyNmFzCu3hjgrbfdzXI4z5O9lPFCLeHHjYDrfaJF5ZXe4NA904xC5hUHvrPdBILPRUF+sHcea/fg4+m/QA6gC+yaE5WLyYShM2z/XBwkej0gV4zagIoi61JyPyYgT3/OlPNBaVrGNyap4Jn5pJD3Peaej23FwygBFZeaPMzjozfbrTKfgQqpsZVTIlEcoNdrtojRzr1TZfbgbC6idDHpVHdUwV1WXL2bzO21MUVWPx98sAPVFPMEtstY3c3Yd67MItKot0ILLzEQmIXvbx2bQhFO4zvHaZ6Xbm5RxxuRs4DJ7E96LSOp4ud14t+Q+6qshtCjtlClVBRdcystokMFmOGfqMHa897tWOhKLMGmDyLY4+TWsLR/z1ReWd+D9NB6pzAJTWQHTICQgZU0UpXqS47ggW1UOHq9q3zhBbHnzSQrhX/7mEhb76cYZ21ikq0GiHDL1eJ73wsrykTxCJzbot8DwzArzd4u7CnKQikxq8ZbuqIURa5Gfa3bAaqJB1BlZzh6JiHLgeD6McBMTRc59uK1xAOYCKbXnZhqQB+fjOV1G4Ak14D1y+W+RQRnQJ+YUpQjIULHxmczsG5h/WySqSe/n1vpEqs8x6pYevzj4TOVobNB/5V5WLBCXCTVuFVZTfGl6S89HHDDyk3xLTB3Ps4GgRjDrLqps50dZ78/VYx8o24uF633uMZW/rDGvpZFdHoPXOB2gh9pR+oKQKvIsufbugdpvr4rabnNn9KwseCP+xQD6BhJqR6iazmSEEHhvL0fRWWE9FDJuUA2GDwfMibGanifb0av68eKpsejce6nQIkepQd6I6DjDNmGUEPBhidbtPVbQN1jRVdDuHjtuPlsVyskS/momvaOaXdGbhGI6kU+ELnCQ4by4LHhdEBvpVtuZV62zmEPjS5hAuf+T60GljUZEEqOJDhnFcHsmu791u3KqtOrht5CLW/cRv0UZ2qiARs9U13bJ/2440Hu1tDBc3rdUCx9Q7L1UN2cA17Gi2wXfsZP/bB2uwrkoKIOh4fbcbE9byMX2aVCG9ZIBUxexlATqY/GUHelWEoP2f30Dd70RWz3lW38SwBrkEty73BqSaOMUocVwECECU/DLWpmlcOwgUrtyKhZyASQ/vJtxeZA71q1mUzi6RuFzd31fOtfp/nYxA7JfSYg24vq+3oMhJKHGOfNJQvW/Y0I2zQlX2hxzUAjxZP0nqpL81KHSxLbnMfNHTtuRlWsnBGacqFhfNX9k4Gf6ttbbP/owhbU4hP59bqV9/LYRiRRz/PHCoK4WUKCMhzygG7uctIEF6dcSbYgq7Ko78SbWnn4k+YOdTBKYHu8YqCUeWFtG8xzqFFvOllIBLp9YjMWIYblJlHeMevbfvfwP6lagGzL+dj7QKucV9vy7hqWbECqN2gDhXk3G4Qqs6J0aV4eG36iUh1c+AgMs+JAvMlnBHN+SmvcZe4RZ9wTnpL+Xh8hmbwmqVN7JohGaGF4kmLS7lNikJoo87CGJnctt8EupqBqlp0AHVctxzClGfJZedPuadVwicZk2K6S5POxOrY3Mn/yXWmnrCnbPr8GtNDXI2MzIt0pruuNji8B/fXEsjEaUSzPntd6Qrx0v5aEe3q+xFSFjqTkgyEtWraZyoJDy2q8gMXI5mXfo2uEhBXGa5LOGzn3IPEir08U570X5wmplD+jkvZkiEuBoWKXH8bBs/xm5UaEjrzswJe+SeRJHqHwFlFy11wkvdtEuh2j1n9SGiKYzrojm5CBgSCMu76WM96XWUgO2oRDjRbRpxtMwBhNkYTH21AfjGIqoRHdz4pRj49Sb3YZEMjz5gekoXhEspzZJRb6kupchB1XD9VYIDjOmPvjE7nSKhFghVNm688ktlIrcoYRrU4jP+0acOblQwPZSeQPyqwBqW12EvJuj+SlzvGpIuJY3+awjHb3iYfQGWMC15a9sm8p2883fWXMPcm6uuJq2+ttKjwdoX/j7v4Aj9+HrHtrCFg33WzGpBs7JumbK+32LYqtrZOJeUarQPeDvHaa0VYKE9GEQ79Vikt9H9gPmwfUfmNeqUKntkQf7eqNbSk99ELYw2MZNiXVVIaC6AzgupfyFlkkdL5TfKDkFBI7ihIBpbUkjdiGHxRItbumOLAMAl3hVekU7YQn+Y6Jk4SA5RH/cCQ7Up/SNG7A6rap7TThcig/sabUDbspJFHgSvkt1i+Gfaw98Uk7JVxqsn/agGDKWAgkMboB7HxLRl7qTcv8xKOptTBG6thJk1/4ydGJzqVlzx+ZBVa1jwHjLpzvN609+1cZlusMPMTja4E1dtMU70ys1Ck72NG1FS64d2tWwrlAsTJskCF3j6msxpY3VEPnLL3svkoFSsIHpVAeduabjMlLjVD4BQLBKrRC3kMIidtqa4+awahb2yK2JXiVc/qrjEM3FlHdJ7bI6+FZ0qDL4jTLEWdkl/r+vKTBFNF7oseXaVfda/1WzLvHcttnp3gNVgxe3kcxobzH0XXWeg5gfW4RevPCZ8JLme4+1LzeRjuR7WKR7RRN58pHfn1JbdssFcqZsdP1sC0JSa1f5OdC2l9dvVD6BisfjxA4EPd1tNM2e3zF8jKXzDAhhzw+6Ynlok7GO51m8cUgJdtj+Utt0N/+qWjKSOgPiiMJ+gxnZmS8njVg5Kg/UY5o59mSNFlzrzD6YWNep1r3AhgyBBa6gTlFal6/tPfi4lRIInU0C+cvSzOJ4aUWZnhRMBMaitfmJWZUP92PQVuLCT78hOAfZ6s9nmUFjCi1b3YUB19LXQZKbvaEZmcJYi6rXn05U+un3quA0rRkhEZqj8nBT/FbRZ9EIvCgWhor0a4/cqUAK5n4wNEA6Zj4BniL2M99VSQ4UMjzLkLgxav9qlB2Y5ChVHUTsmXxUxpYkeWXY2fEl55jN6bxHtbi0ITAwuwEiv+I0BaQJbLqRK7Qiiq+umPoA9ibYd26nh7z6acQTfgystVDe+5+FHPK9LCUr5+iCExrdtkbMNwIS0f5XpfEm5Ne7xT4MnW6go91MD7od72lEv2gNS4cyVjo7Ja5RAHjeUnNjUvuaJVWRJs+lbbxxJgKOsChbicn9l0Fv4frjPfGIL6b1Nc4s7QBpKu3CbG4J/bkrJiuljAP84UGkbuW5fycIoVzTY05g2fI4DGVn0twT6FAwNpyZZjtU0yg5+Ahe4vqxqcrd+DntAMlrH2DOtVawWTqKlARoUFUyeLKUb6x0UBj6L0ey7oashQwKkbfGsEJlLlFuKbwUse/34a4jdPkdrvf6V7ARDnsp0m67++9nxBDk9Rj30EdbPYl9vlKBlUdaXXc1MW53y8Ger8Un/Z6zBJPoKLyyl+PETGIkig4VGG9V6SOaCBPWuu0sJdPHUqJvJGN4DrmmbXhpWxFGbvVCbbxK/9LP+7YXbyX5CYBulVJoxvsD8SdceNTTx7DvChSUl+EgAWIz5Xi/vaVhxHI2zFSMo5aDImFcTy9bLbgNjk36gjdb7y7G8SBrjldsBsL+iJK0q5Q9w8RBvki2aisHCcYv+5HszUcQowx7BL6qQTOvXKLz1ZWdO44Ikhb4TD5CewNT9v4Vtj50zvIil4zvaPCxX9dwXAQ5qJz7gHcVF7Nzc4oA7vDOxhVpAgKzE5XqZ1qrJCoV/6iV4SFX6Y5k11Eg0q+16i+2ZVvpLfyND4xcuT90j+TvvaROaXlFaljHVWRusCu3vGf6tE32p7aXL/VyeCLcL9nVcbbyjAH5eNjDsSZX8407wwLTJ8UUo2A2DvGK61dVskejtwHJ1Onu76BPqzLLVGQNiqV93Q7xkr85Nck0iUks62evgqAheMBggX5CfTBDM+/UnieA57/5Wn2qLV7/VbXm9Le6WPfv53Hu2kLKMPxcTsx1bpMiJLGGPDjeA/D1nAuVJS/ofzfUHZepr7O/CpdyqsBvlryvluc6p2Br9T1vYnirGH7Kc0mrm/6CVyHoNDn5zpctVGRfT18Hen67rrwep2LAkJDNGXd8jkfuZUS30LO6gfhDtHOu5tM5yeU+Hrilk1Ldnw9EQZNqPA3lGsPKevbbAHZ7qFvR1EY/UJS9N9/vt3h/HYxdh3+2rJ/e6XrG4V9bSqzqii/9QfDvoAsC6A5mr82Fb8+DZDvr30ANOHgsqb5pUufzwhUpV+vGfxnR2x5/fP0RKt2pZUgIn5C4W+vFTVr9vW8vyFEcz2ZveaFKJbPCH1tAKN9nZpHydcziXHtQTvXt9UlJEFO1F2MBbo5fz90fZ2Xs/n9+eA+P82fabvWDwRjoJrQ36/4+lChKypQgBFyhut58y99AFzk043fdw0IJ39vQ373WGTq1y7N0m+Y2ctqyT43vRr2KRqutnJpm2+HiylKqwsKv8EPTaYQSYKOV03zm/YUz6gU+yEwv7b8cu43pP0RrWU/Ve+rLfrl2QBbVRI1TFNdfB7ll374KwEKI18Qgv5zgELfoZOgoO/hSUB/FTbx77D5iKaqXwHKmDT9qe9+xQUopfQvQA2/oPZBzR/Q9u9h5vfzj/wYRXme0gTxBxjgfzrr4CwxaqsGzMb3K+zbbb51GP4DOP9q0kb+Q+Ag2I+QQ3yPHBj5L4DOD18J+1Oqdt3/YnhQUkbTnC2/A8u65D9R/0HaBf0poH4loj+4S/J1nsAdpiL+/65RuF4T+uXP///14Z9H5d8gwHyGrtkyAJXfHP92b3C466cWkBJwrMmWJZt+mi/IVl3x/fFrnpefqgvhABjg4C/v8fXIMl1Ay6/zf7nyG5ygSzhMf3/XXy+Mo6QuPsvmpz+8HoJRX98MwehvH/DfveR/YDz/Kh70Z8+1s7lfJ1ArFlLmvomWqr80MOij50GOpYPfS9Sl0ZRel0QtWLpdPA+/503/nGX9N6Hl+xf+fwJG/1vQwqzLh6n/ASZLP12CJ2BanP5vSTJgVP85O/otg4H+FVYSfeM8yTWPQDT+C0UOGvpHMjEGod9xDhT9gcyB/lWMA/6ec9yv2dzA7AlHlqxfZ/YP8/ILC9eB/mH2c/U5C+Xjfln69vez82N2/90U/FiC/L0ACkGsIAJxYy6jAXSlPS7BYyi/RO91yr4kfZr9fF0Crhz6CtxY2K77z9968lVb+nW9/UBa/ecCyDeYIX8paNBL3iB/L2KQ9PdAoX6gOsF/EU5+kVx+gxNx7RIw7/N38OjXpbmUGa7vuiwBPftlVf5mLq//RPD870THH089QpIwTPwZTtJoLj/CK/QbwP0pOP9FRP4JhUH+IIuCR0bz8PVF8+oA/fgh/L4D7T5jX5qojdPo5/zbWP77KP3LUIn/CJX4Fxz6+w/8HUaxH2H0v0C3/zFG/1y3v3hv9y9xOuRPON1X0YevogZM9N9519cb/wn7+t9JJuex+XnKhv56HpBG/q8hixhBfiH/CEDsnwHwB8wU/6uYKbj+DwD8g4T0B4D8kchRDCX8SyTsO2X8d4o3+PnXIfWhVZ+hwNnrf+gLjOGXvPnr37/hPBA/wZGv33/fRuE/Phv+tP7xDr+c/e3vH+4N/6Htl558d/bv743z/yLJTao56X+eoxwIC+0ABjZTkg/ZjYuvH/hrnqthBtCfsvlb24+uj4YorppqOb9UQB+4Pvwc/W6y/+9ZWShJ/yMpFSG/t9vi/63rCv1uXXFRUmbfrafv+PBv1snvYc7/aCV87OU/GPhv7XzVgo3QpgIW/HZOIrDp/OnIz3aWVvPP5tSna7J8mbd/Tle/W7a/Wu2/t679wNT1PycHIDB8keHfgOWfS6o/AguG/FVg+V6jkav5UknBO1yS2oULBDKGpWqvGZ6+g9AvJOMXLPxTUvyPp/lPaPAfcfp5GvNLK/QbzJXLMswf4QR4zu37/uVSrpPmQ8D+9rW02ZIlwJ/jKzqTmIZ+GtdsOn8asgkYI6Iuyb4MH1b/HyE51F9KcfB/bFGlvteLMegL8t+q8XxvjdcvpeZj2PgnmPl+Nr+b8P+kkvI/S1P+jC6m0RJdEP36FREB1hCuerCGvUOaVPRgd/PueKXgFdenG/jFKxwTXn+5xdeIDJzAynfOeVhXe6HkTFlXn03RZnfE5n190IX9Ovu4sYw6JtLVIMiHaguil6nvpfORPXg88AOl0Jp5JayguJP+sDxNqzg5CqEyZ4WJg8RxRe4O0gDv/MEf6Rfrtm1ztypvF20xdPZ8EqMy04WNOYZo4W7La9EgMZyJxY7YRzntj5shKpV1PYVjM9QksLjpHIFE/DZftjSPMpwGXoKf7XI685Or7T3JqJGDnCvoExMsqyjKkmU5jpMEQVAUJbSscNXCaBlZWcCk95YCvyQoKGJkOu3w5VNRyqA4qDCADsdkvVzTzAtxLnjKk8yMS4XC/oSzHCdP+83NqOamctKbwZvyu0iztE0tXXlM2it0H8qdXs5aKMOJOw8e1gshUQWoyDOhViXaZKBSRzlbuvMJ5+5GwCiWHMh7lAD6sy9YpvcMmnDbvlG+wt01h+GfZb44nFMJbrSTOI1pKAM8F/PbYu5bwvLH/eYLt+GNMubHj4OxzxC45qQexi48DRW10lehgHlxwVDEts1Ewsh7TLHDbq6sd+qRvDHMozNpMWA8SpIL+ECBV1MJ7/eNnZkXTbmMsTFD/Ij5THhi7BY6ECM85bKdjcdWwliQPqBPzlhvlx8wfNobq4eBJiRuZG+7Q9SZdZuV6IkHamMlsQfDVebiu4ZJZ5O0pGX7A1rfJl9IZ3bBfKh/aaQ8GjffvDMUxXa7mUnOcdNAjJkg5UL+YvBORUHUz+EM7tM4PIoZSQU4uYky2x/52w89dDKXSHad0dGw17ugdq6jeOCcwNHAiXQnUdF87nkpY4LMoqq+5NUWAs9N6RNjq0HT07l6XjHTLEgBgXHkCVRSBbFESowLzVDR3QHOi/ThiorjeecrhP0mh8q4yA2mwPown2uMiREdZfu2SA1NklBebFT5TFVFo5hpTBMPtaaEAW9mxAWVpFZVAxfG0zBwjd7ZSaFBl8SpgkvgHZJXn/xtpXYKOX6ravERxrFg7Rm9gOlngefDvgUqhn8C6bmKekkin/mSuCqvwajDDQ/2haIMTCkEsy9BggVG33PKAqHbLMpA4YxYs1u2o6Yc48x3d+L+fo4dfsHD2kEii0B9hfxbitAzNaZM1EyKzdoe0cMLGV17Q+FZWosjf5FHsjBKHuDKpuBv3eF8wzL79Z7d0XWNQiR57si+dFa7Go9dCBg9AcMbTuaLRtQ4BqwXEJIKKpUjlhn0luU0FhCgxJHyybRoXbMqKYSsz7bHvcKax0SKfndYjZnYew6eL8cQcuZOsoQqbXd0OxEFWxrWhamX4cmoNysYxh4kv2n9MzpyIUJvZFaQNL8VamzAQWsFO6hTdlLOJy+KPWvNQ/Vwhy9yqJheNVrr8Cji0POaOc+OSLl6pYUQHETH4HMKnHehPWE3f3ZI/d7uGe6vBgmcbxi4jZtXCEgZjxb2vYiydkmOoNcoYXvJp9xkVQ3NwDdZbWNFPdDOBP5FCZeruV2a+9Vp36XTmaY1qtxE/KkmcJXv2GNbgHcafIRD3M7vl3Usrjh1lEZKJe08x1chBaGIsP6rlhf63mUZNiHEcX4cw6TNEycxI+BAftzFNgR4QuC7n7lngL8sGGYPXgPdNx4EwBn3Zjf7vovdp55C/ZbNR+DiNp4HAXBUK3X/7uE3nNQT+PmUdRnZFlTmiNbXK4K55RUM/DBEt3Z88n4q79DGUD2hkdCQI2hsB/Jr1jnw7nkNlznBI7WPrw/2GZW60+q+GQWN6ZYQxaKf0ZErw6w6c2l7/DHXF1WolBzNhj11xw95P2cdF4V2S5aSxjyRGl6iWANEG9n7YFVR1NcE4zInVbNPlEi5CFkl40pVX3MtEPrsHp4VSt3HoVnN+YgBowbcFc39flHk4SxX1vxUOcnZ+FMiT6cKEHHCsuzsmAxKKrmGFeZpz49138PUSgs+yKR3/U4f1DQyd/nEYr3DQAoVw+futf7qLAQTX5WJw7h/kWzwQAnhGWleX8e4rOUL+9SQR1VEpuHqdfOEN02RLzGFNnZTxjaOK3lCEbcN2gFH69ZMErrNTGJiwvjplKOC3hP5nO9Rj5fsK1Df4ZERxjgZVjzMpYgzhgA7onh1IkarQ3/KMoPRdZSA/GfdOicOYn+SpLzRG3y8uTeiuFC6nex9IWQAxTAULj7msE8iVfNuPXw1VuIw0xQJIRRqogLOihusvMtVBuKpiim883rdNwKY0Z1IjWspkncPI83iILRIOQaMWUKUoHo7sVhxVXtVq4WkfK35zPs2H8qmEsCARiq9cKgP+DZDy2uFKqHSquI2vw1Y8bYH27TyGySs/gQnZUQ8KChgnxWaRMTm27r9Bknm6pRUEh3vbhY5ovI0TJhxo2jbqQkn73dxbk9OWhn5hEEooiOFjkiY2YHmBa9fDytYSKZcBFbjp5kFEWp6SKHijdAGxrpfQnbd3VVqOUTI6Txv/ZTC8WznTWbQRMwMCOwAntpcVW8d4sRu8mQ9hMmLwDfQ4pOGln+9Tj80Z9zbzp4jy4uxYDu2YHgsv+kgrYzjXtZqMrliKFNPBqScCjhtUHpSdtKUQPoswHbST/S827oEI8c2MlcEADaQZdNdwTLI7WYvsIcSmGGMbJoyPGnOfd58yKuHr+6Kj+B4HLQimzb3jp07uSSZVR9vSL56hoYbUSR0WXslqc8KTqIdEp+lQ7Bqx+fNV+r0gHPoeXLOuOp9SMxLtN4VVEJJ6f1G5XC5o7cAu+UJAwwDekgtj3CMjJm/FZQ5WQegLdcBRJ2pqdLct3yBkCiL6Cbf32uS2eCqZrmoY0mHysNKgOu5G/pMnJDkC00SBdCGsGHxjSsnp9Ep5lxjSMLfNWxNg4wxLzM3Vj3DSFEMEU8zMCCF48UlS4jrcJplmR23oeNLmLibq2v3b+JaefyoU9ZIRJtg2reuZe7HRgSkbzKrjKVFiyMZZThHCk+MsTqrOWeMgF6iDcDtHHnRFEiWYaI9cPccWBnEHvQCswYcfHsq614m2IBpASWigh2FWMDOrfnw6ZX2TKylGQdr4pv8crbz4qOYqcEhdOuTfHtOtKQ95c1SWJ8U4wXiR6KdCT8EccXDbeFFeXm5H1WYGo19h08t1XaD3UgRo+as67fYL7PnjqfZ+5oSCBRSYAt7B5frQLuaILS3Gy579nZJkwtwFBVpVQinm7W9avHsHkhYze/SJujWxOLBi2lmZ57m/dXMBzSQBU8rgYIXcaXfCLcrHoeJJxxIzvNcgmuVbp5AHKVlxHsPZy8e7aBICItQQ7gTqt1d7mMhC8mwRq4jOsgcKdZ7LOYhb7Kyb3DyWO1g2kpOD0qZGN6UlJFzbRyXJMUme1aLWbQt99nKYyRIYlC9lVXheMTY58bkL9bJBz8MN/p9CqabVmePEKOzDGlZgxK3rODaeVbocQ5jsu/VeQsNz6rPgUttQYbOJaLrTnNWaHNag5SCsXnd4XCKtQzyDLREFJ1/00andpgLe0POSCen44h/aQHD9NVCwXrJU38+u1mC5GYXNpA753ETo+iooy2MP4Gys1wRL0T8EF/jeG9JRGqu26o3OvffA0XS8ps3TDphc3UKbSBdBkzwzhfWjMjtsMnrfKZtH29f/CRsJbkgQpZn9VoqLK5q8s0RcD+x51khMhGoi8KIF/5tjdOc5pYuTSy6zSWXBSKxywLSmBAJM7fo7gvIY0iLnSQobANJV9jS13C5WLj0lB88ZprhLk5Hc0p6DEpKclvyMTGsF9hMe4c5A9OSgHSJC8RzSX6qkr4M9Lj0EHWEunZ2KnknrleaXUQAgZBRY1eRPOdGzpbZBZib8GHUzNQS0Y6VhQyLVPuQWEzNmG1P7pUOK7eHSNnN/EpQiOMXObxJEI4LmH27zWyC9QngF++Z8dXeUvHUqmP1VI+Kl1t5L9zjeA80zugekAr8/RCpW84RCCkYq82qoxNAjiWuHLmvD6FMcfd1wjBDuzcHMKtRSy8xdx5CzELRZ3vK1HqjEBwW0AieHQ6KP0kXPcuQzlaoyBCeE96oTCZ95+/nWu5230MAgulRCaN6IMtLEEYFdjiWy1SWKU2mKKxXaVmKvdf3mA0UVuzx1rijl9xMV1bbvT65L8N8kF7Jm9zg7Lxei4/8/qZAVl8LnED98G79TVB6S+Z8i0qsgPq4fzNnhb2KcDdZRO7uKMYbO3W+LjL+qbBspvVzFVU5dEwiIyj0vqZWH1Na1dqJOpmAuiK8cL3RfbYZhQV+9NLWP4YQHhcbIpb3pfBwjLslKhTP2LqN6bRfwhPgwMErtEjc6Rbk3hnKaMyqrE61aDAq1rgrdDggIhOMd/Fa36MQDSpMtWYZC8+Qe0CxAQTSu6IY6Msw2EuiBrHJpjtEk804jjkrpv+V5fGiySBW9yaP/WpfPqJyjKd7rw4yQuW393XE+hzp3E/BpfxWlVai2kxhCvWLzHgb8NYXrXRA6ulqgY8OEn513a/fwPI93iTtOhcUMSISsjxxcZrCVZVHEqCh7qV3CdvHRe6q1+ZWGruoJhr28wOk1FNuiSjXJr+a7F01yEJYX8O7sz86lp3g7v1mS2oHaiWzGGsKbQaYOZV/G8vnACh7WjpARdGhj7TBqHt+HKgtvqJKau3tXShZL1ZLAvJaDs2M6OXSKiyv324ru+yXLilvWv7e9V3MuAVhwgF6CIjSy1jLyV+pWbJDOKygnk0LHEM5FctkSDdIvV+Y5i5RRUtQrpkw6W7mvA5fwBPWpmDQi2XGaWWxN7pibKBmeeUDxi7JGuqEG3pYgl9zVG8ooaA05xynI3uou50MwvX9gHIbawi5UDjP+qSIHGvuyMywzW6p6hdbfvVDZg/M16Lo7grCrPEspwns3vzax88S0G1YNOaIuuiUKSJvGTdvcMIRwwArJxdy9tpF2sDhh+NE4fuxp/e7oXJWX01E5Asq9llQ3LWgDPG6m8xZuAhAVPRzLTbBAigvQTDGNXwPZoKbwJ/rdJd9m9qGBVMdKWYsWxuYx65eChMXielubEyGj8gg+XP4wLAw8GsgQnZTcpcETYD3iG9bzjLYCO+RorN1TeBa1z54viojKxVmzNIuGYDmS0q+GwgTqU8gAmc3XfDsT8plaHJiuNeJ1nwq5F2qztq1BYRx5xeI/wSmENj37KU5V1l3oD28URbua4kpi6Z19O9PfspbhYzX4kvGg0eGisu+vn2xW1XJfF0ffXQPR1K0q7p5XeJItLA3/GDe472kDhkwAx+dlQVTsHYojEbN3PbZx5NTnqkDAkjWl9SUnQMQZvpZuYxh0f2eUoF5PzRFYInxmH2iza2SWhIWBLV8LebD7lF7KSmfypNCRmP3Jp3H1zCzovhQ+GuVHe4ogdDM/IWUJHoJUve5hogseVxqlV2qQ2h11fVId1cUQcG8a6oTlR/X1zqSMkiW5/nYq1XjDE7cmEKMpfUl4W30/oF6wqtaNbCADod6vk2R5dWLrES9O59F3Pl7U/bcsddH7V0k+LdPccSHikJdIphsRobeJmOBODAz2649pErxTVKkNFp5pzK80XJPzajz8BKOJFtVNC3ZrYTWXGcvTaEodlDhQIysxBGAJsVynoO1EhoViU0BdcmqQMvKPrFNqjvhJcmNT9UPdxtOVJTJnhAjKA8LW1we9ADDjUhcE39/IEwXB0F50wSRvL1VZ69my9OefV4XcdEnKjLIv/Tg+/HTeYNxjHG/ia/SwF+CWaz8YGF8ZVf4j8cb9khiDI/gxVkBV+5dMtZjLd2C+uBSW4BmxT2vBWCbHrCBRSeku3xftpwdXASppSOORu8ZwoD6oazPgBBbkAvTgm9pFC+7KRzycK0xItQCfU64M8ibDAn0bQrT+ZiqeF+8/XtLNLBOH+wLaIuQABZMtXUxMusfY7zQAGPIarUc95fuMNL4Fwr+4xb0Fxz9fr/ot/tD9BfsBz73v2n+r98iIn/k79Eva/e/Ymva7ftm/n97M5qA8S/4bzej/+if9j1MyP9Gx4V/EG2WVtu/7JD2p/7e/yG3tv/kXfZvA/UbX3gE+tsPfex/+JTb2iwXAYOWrIs6YJWD/tR97r/ipX76dx/w777vP46gKKMJeN399YPwPzgC14FfoxmbKs+SM7ke84979OfNn4XyXxYf8Iv37r8caob+iBr/wW/vTz3u2ipNm+yvJIAwgnwhfssYf08BfxCZhmF/9+P9LRH8xXnx3yGCj9Di9F4lTvrBn3AW6uFS/vQjGvgXOd2kWR6tH+D8BX430QBcvb7004fT9pciBDb6P1tEaT9Vv/79ubwG9Wr/5nPzl0099ksg6q8B1vQXFP6NN+z3MhIM/cgTC/5Con/R3P/isfPfMfn/guvrf3b616Hpo/TLXtVVm6VV9A0E4PsAvgO/q75tP7EQ4lKunxj/zz/wi/kA52e52rKfAW4+Ihkiwp8osJ9+dPSvhg5O0V9oGsfIHxIN+EIEhP0GSNT3Hn2X2PUDYfu/QpCqjLCOF14tMUS3f1IbtU29/ydoyHJx/v5L9TH9znO2zL+0/dT/5RP+R3876g8T/j2bgOEfkQr6C/7vz/DPEtqYurVCkus2i4+l92D76XvX+f/7Zjhvqq7+PZ/4ppdd6/pv3xzjxE8o69dzf57HtZqmrPn5avz5E//5VyPhE5FI/2ZxI/8UCjj25RdF5neL/e/N/wE0AJ7ZA3Hx12MScK2/9WkGzvg/ \ No newline at end of file diff --git a/docs/imgs/kyuubi_ecosystem.drawio.png b/docs/imgs/kyuubi_ecosystem.drawio.png index 19de7adb5..72d221d10 100644 Binary files a/docs/imgs/kyuubi_ecosystem.drawio.png and b/docs/imgs/kyuubi_ecosystem.drawio.png differ diff --git a/docs/index.rst b/docs/index.rst index fbd299e7b..e86041ffc 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -179,6 +179,7 @@ What's Next :glob: quick_start/index + configuration/settings deployment/index Security monitor/index @@ -216,7 +217,13 @@ What's Next :caption: Contributing :maxdepth: 2 - develop_tools/index + contributing/code/index + contributing/doc/index + +.. toctree:: + :caption: Community + :maxdepth: 2 + community/index .. toctree:: diff --git a/docs/make.bat b/docs/make.bat index 1f441aefc..b8c48a2db 100644 --- a/docs/make.bat +++ b/docs/make.bat @@ -38,7 +38,7 @@ if errorlevel 9009 ( echo.may add the Sphinx directory to PATH. echo. echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ + echo.https://www.sphinx-doc.org/ exit /b 1 ) diff --git a/docs/monitor/logging.md b/docs/monitor/logging.md index 8d373f5a9..9dce6e22a 100644 --- a/docs/monitor/logging.md +++ b/docs/monitor/logging.md @@ -114,7 +114,7 @@ For example, we can disable the console appender and enable the file appender li - + @@ -265,5 +265,5 @@ You will both get the final results and the corresponding operation logs telling - [Monitoring Kyuubi - Server Metrics](metrics.md) - [Trouble Shooting](trouble_shooting.md) - Spark Online Documentation - - [Monitoring and Instrumentation](http://spark.apache.org/docs/latest/monitoring.html) + - [Monitoring and Instrumentation](https://spark.apache.org/docs/latest/monitoring.html) diff --git a/docs/monitor/metrics.md b/docs/monitor/metrics.md index 1d1fa326a..561014c37 100644 --- a/docs/monitor/metrics.md +++ b/docs/monitor/metrics.md @@ -44,10 +44,12 @@ These metrics include: |--------------------------------------------------|----------------------------------------|-----------|-------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `kyuubi.exec.pool.threads.alive` | | gauge | 1.2.0 |
      threads keepAlive in the backend executive thread pool
      | | `kyuubi.exec.pool.threads.active` | | gauge | 1.2.0 |
      threads active in the backend executive thread pool
      | +| `kyuubi.exec.pool.work_queue.size` | | gauge | 1.7.0 |
      work queue size in the backend executive thread pool
      | | `kyuubi.connection.total` | | counter | 1.2.0 |
      cumulative connection count
      | | `kyuubi.connection.total` | `${sessionType}` | counter | 1.7.0 |
      cumulative connection count with session type `${sessionType}`
      | | `kyuubi.connection.opened` | | gauge | 1.2.0 |
      current active connection count
      | | `kyuubi.connection.opened` | `${user}` | counter | 1.2.0 |
      current active connections count requested by a `${user}`
      | +| `kyuubi.connection.opened` | `${user}`
      `${sessionType}` | counter | 1.7.0 |
      current active connections count requested by a `${user}` with session type `${sessionType}`
      | | `kyuubi.connection.opened` | `${sessionType}` | counter | 1.7.0 |
      current active connections count with session type `${sessionType}`
      | | `kyuubi.connection.failed` | | counter | 1.2.0 |
      cumulative failed connection count
      | | `kyuubi.connection.failed` | `${user}` | counter | 1.2.0 |
      cumulative failed connections for a `${user}`
      | diff --git a/docs/overview/architecture.md b/docs/overview/architecture.md index ec4dc0d8d..4df5e24a4 100644 --- a/docs/overview/architecture.md +++ b/docs/overview/architecture.md @@ -107,7 +107,7 @@ and these applications can be placed in different shared domains for other conne Kyuubi does not occupy any resources from the Cluster Manager(e.g. Yarn) during startup and will give all resources back if there is not any active session interacting with a `SparkContext`. -Spark also provides [Dynamic Resource Allocation](http://spark.apache.org/docs/latest/job-scheduling.html#dynamic-resource-allocation) to dynamically adjust the resources your application occupies based on the workload. It means +Spark also provides [Dynamic Resource Allocation](https://spark.apache.org/docs/latest/job-scheduling.html#dynamic-resource-allocation) to dynamically adjust the resources your application occupies based on the workload. It means that your application may give resources back to the cluster if they are no longer used and request them again later when there is demand. This feature is handy if multiple applications share resources in your Spark cluster. @@ -172,5 +172,5 @@ We also create a [Submarine: Spark Security](https://mvnrepository.com/artifact/ ## Conclusions -Kyuubi is a unified multi-tenant JDBC interface for large-scale data processing and analytics, built on top of [Apache Spark™](http://spark.apache.org/). +Kyuubi is a unified multi-tenant JDBC interface for large-scale data processing and analytics, built on top of [Apache Spark™](https://spark.apache.org/). It extends the Spark Thrift Server's scenarios in enterprise applications, the most important of which is multi-tenancy support. diff --git a/docs/overview/kyuubi_vs_hive.md b/docs/overview/kyuubi_vs_hive.md index f69215240..80038c178 100644 --- a/docs/overview/kyuubi_vs_hive.md +++ b/docs/overview/kyuubi_vs_hive.md @@ -32,16 +32,14 @@ have multiple reducer stages. ## Differences Between Kyuubi and HiveServer2 -- - -| Kyuubi | HiveServer2 | -|--------------------------------|--------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| ** Language ** | Spark SQL | Hive QL | -| ** Optimizer ** | Spark SQL Catalyst | Hive Optimizer | -| ** Engine ** | up to Spark 3.x | MapReduce/[up to Spark 2.3](https://cwiki.apache.org/confluence/display/Hive/Hive+on+Spark%3A+Getting+Started#HiveonSpark:GettingStarted-VersionCompatibility)/Tez | -| ** Performance ** | High | Low | -| ** Compatibility with Spark ** | Good | Bad(need to rebuild on a specific version) | -| ** Data Types ** | [Spark Data Types](http://spark.apache.org/docs/latest/sql-ref-datatypes.html) | [Hive Data Types](https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Types) | +| | Kyuubi | HiveServer2 | +|------------------------------|---------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Language** | Spark SQL | Hive QL | +| **Optimizer** | Spark SQL Catalyst | Hive Optimizer | +| **Engine** | up to Spark 3.x | MapReduce/[up to Spark 2.3](https://cwiki.apache.org/confluence/display/Hive/Hive+on+Spark%3A+Getting+Started#HiveonSpark:GettingStarted-VersionCompatibility)/Tez | +| **Performance** | High | Low | +| **Compatibility with Spark** | Good | Bad(need to rebuild on a specific version) | +| **Data Types** | [Spark Data Types](https://spark.apache.org/docs/latest/sql-ref-datatypes.html) | [Hive Data Types](https://cwiki.apache.org/confluence/display/Hive/LanguageManual+Types) | ## Performance diff --git a/docs/overview/kyuubi_vs_thriftserver.md b/docs/overview/kyuubi_vs_thriftserver.md index 00a03c3b2..66f900c74 100644 --- a/docs/overview/kyuubi_vs_thriftserver.md +++ b/docs/overview/kyuubi_vs_thriftserver.md @@ -19,7 +19,7 @@ ## Introductions -The Apache Spark [Thrift JDBC/ODBC Server](http://spark.apache.org/docs/latest/sql-distributed-sql-engine.html) is a Thrift service implemented by the Apache Spark community based on HiveServer2. +The Apache Spark [Thrift JDBC/ODBC Server](https://spark.apache.org/docs/latest/sql-distributed-sql-engine.html) is a Thrift service implemented by the Apache Spark community based on HiveServer2. Designed to be seamlessly compatible with HiveServer2, it provides Spark SQL capabilities to end-users in a pure SQL way through a JDBC interface. This "out-of-the-box" model minimizes the barriers and costs for users to use Spark. diff --git a/docs/quick_start/quick_start.rst b/docs/quick_start/quick_start.rst index ca73fba35..9d0a7d30c 100644 --- a/docs/quick_start/quick_start.rst +++ b/docs/quick_start/quick_start.rst @@ -34,17 +34,17 @@ For quick start deployment, we need to prepare the following stuffs: use Spark for demonstration. These essential components are JVM-based applications. So, the JRE needs to be -pre-installed and the `JAVA_HOME` is correctly set to each component. +pre-installed and the ``JAVA_HOME`` is correctly set to each component. ================ ============ =============== =========================================== Component Role Version Remarks ================ ============ =============== =========================================== - **Java** JRE 8/11 Officially released against JDK8 + **Java** JRE 8/11/17 Officially released against JDK8 **Kyuubi** Gateway \ |release| \ - Kyuubi Server Engine lib - Kyuubi Engine Beeline - Kyuubi Hive Beeline - **Spark** Engine >=3.0.0 A Spark distribution - **Flink** Engine >=1.14.0 A Flink distribution + **Spark** Engine >=3.1 A Spark distribution + **Flink** Engine 1.16/1.17/1.18 A Flink distribution **Trino** Engine >=363 A Trino cluster **Doris** Engine N/A A Doris cluster **Hive** Engine - 3.1.x - A Hive distribution @@ -143,7 +143,7 @@ To install Spark, you need to unpack the tarball. For example, .. code-block:: - $ tar zxf spark-3.3.1-bin-hadoop3.tgz + $ tar zxf spark-3.3.2-bin-hadoop3.tgz Configuration ~~~~~~~~~~~~~ diff --git a/docs/quick_start/quick_start_with_helm.md b/docs/quick_start/quick_start_with_helm.md index 99c787841..0733a4de7 100644 --- a/docs/quick_start/quick_start_with_helm.md +++ b/docs/quick_start/quick_start_with_helm.md @@ -15,105 +15,106 @@ - limitations under the License. --> -# Getting Started With Kyuubi on kubernetes +# Getting Started with Helm -## Running kyuubi with helm +## Running Kyuubi with Helm -[Helm](https://helm.sh/) is the package manager for Kubernetes，it can be used to find, share, and use software built for Kubernetes. +[Helm](https://helm.sh/) is the package manager for Kubernetes, it can be used to find, share, and use software built for Kubernetes. -### Get helm and Install +### Install Helm -Please go to [Install Helm](https://helm.sh/docs/intro/install/) page to get and install an appropriate release version for yourself. +Please go to [Installing Helm](https://helm.sh/docs/intro/install/) page to get and install an appropriate release version for yourself. ### Get Kyuubi Started -#### [Optional] Create namespace on kubernetes +#### Install the chart -```bash -create ns kyuubi +```shell +helm install kyuubi ${KYUUBI_HOME}/charts/kyuubi -n kyuubi --create-namespace ``` -#### Get kyuubi started +It will print release info with notes, including the ways to get Kyuubi accessed within Kubernetes cluster and exposed externally depending on the configuration provided. -```bash -helm install kyuubi-helm ${KYUUBI_HOME}/charts/kyuubi -n ${namespace_name} -``` - -It will print variables and the way to get kyuubi expose ip and port. - -```bash -NAME: kyuubi-helm -LAST DEPLOYED: Wed Oct 20 15:22:47 2021 +```shell +NAME: kyuubi +LAST DEPLOYED: Sat Feb 11 20:59:00 2023 NAMESPACE: kyuubi STATUS: deployed REVISION: 1 TEST SUITE: None NOTES: -Get kyuubi expose URL by running these commands: - export NODE_PORT=$(kubectl get --namespace kyuubi -o jsonpath="{.spec.ports[0].nodePort}" services kyuubi-svc) - export NODE_IP=$(kubectl get nodes --namespace kyuubi -o jsonpath="{.items[0].status.addresses[0].address}") - echo $NODE_IP:$NODE_PORT +The chart has been installed! + +In order to check the release status, use: + helm status kyuubi -n kyuubi + or for more detailed info + helm get all kyuubi -n kyuubi + +************************ +******* Services ******* +************************ +THRIFT_BINARY: +- To access kyuubi-thrift-binary service within the cluster, use the following URL: + kyuubi-thrift-binary.kyuubi.svc.cluster.local +- To access kyuubi-thrift-binary service from outside the cluster for debugging, run the following command: + kubectl port-forward svc/kyuubi-thrift-binary 10009:10009 -n kyuubi + and use 127.0.0.1:10009 ``` -#### Using hive beeline +#### Uninstall the chart -[Using Hive Beeline](./quick_start.html#using-hive-beeline) to opening a connection. +```shell +helm uninstall kyuubi -n kyuubi +``` -#### Remove kyuubi +#### Configure chart release -```bash -helm uninstall kyuubi-helm -n ${namespace_name} -``` +Specify configuration properties using `--set` flag. +For example, to install the chart with `replicaCount` set to `1`, use the following command: -#### Edit server config +```shell +helm install kyuubi ${KYUUBI_HOME}/charts/kyuubi -n kyuubi --create-namespace --set replicaCount=1 +``` -Modify `values.yaml` under `${KYUUBI_HOME}/docker/helm`: +Also, custom values file can be used to override default property values. For example, create `myvalues.yaml` to specify `replicaCount` and `resources`: ```yaml -# Kyuubi server numbers -replicaCount: 2 - -image: - repository: apache/kyuubi - pullPolicy: Always - # Overrides the image tag whose default is the chart appVersion. - tag: "master-snapshot" - -server: - bind: - host: 0.0.0.0 - port: 10009 - conf: - mountPath: /opt/kyuubi/conf - -service: - type: NodePort - # The default port limit of kubernetes is 30000-32767 - # to change: - # vim kube-apiserver.yaml (usually under path: /etc/kubernetes/manifests/) - # add or change line 'service-node-port-range=1-32767' under kube-apiserver - port: 30009 +replicaCount: 1 + +resources: + requests: + cpu: 2 + memory: 4Gi + limits: + cpu: 4 + memory: 10Gi +``` + +and use it to override default chart values with `-f` flag: + +```shell +helm install kyuubi ${KYUUBI_HOME}/charts/kyuubi -n kyuubi --create-namespace -f myvalues.yaml ``` -#### Get server log +#### Access logs -List all server pods: +List all pods in the release namespace: -```bash -kubectl get po -n ${namespace_name} +```shell +kubectl get pod -n kyuubi ``` -The server pods will print: +Find Kyuubi pods: -```text -NAME READY STATUS RESTARTS AGE -kyuubi-server-585d8944c5-m7j5s 1/1 Running 0 30m -kyuubi-server-32sdsa1245-2d2sj 1/1 Running 0 30m +```shell +NAME READY STATUS RESTARTS AGE +kyuubi-5b6d496c98-kbhws 1/1 Running 0 38m +kyuubi-5b6d496c98-lqldk 1/1 Running 0 38m ``` -then, use pod name to get logs: +Then, use pod name to get logs: -```bash -kubectl -n ${namespace_name} logs kyuubi-server-585d8944c5-m7j5s +```shell +kubectl logs kyuubi-5b6d496c98-kbhws -n kyuubi ``` diff --git a/docs/quick_start/quick_start_with_jdbc.md b/docs/quick_start/quick_start_with_jdbc.md index c22cc1b65..abd4fbec4 100644 --- a/docs/quick_start/quick_start_with_jdbc.md +++ b/docs/quick_start/quick_start_with_jdbc.md @@ -15,82 +15,82 @@ - limitations under the License. --> -# Getting Started With Hive JDBC +# Getting Started with Hive JDBC -## How to install JDBC driver +## How to get the Kyuubi JDBC driver -Kyuubi JDBC driver is fully compatible with the 2.3.* version of hive JDBC driver, so we reuse hive JDBC driver to connect to Kyuubi server. +Kyuubi Thrift API is fully compatible with HiveServer2, so technically, it allows to use any Hive JDBC driver to connect +Kyuubi Server. But it's recommended to use [Kyuubi Hive JDBC driver](../client/jdbc/kyuubi_jdbc), which is forked from +Hive 3.1.x JDBC driver, aims to support some missing functionalities of the original Hive JDBC driver. -Add repository to your maven configuration file which may reside in `$MAVEN_HOME/conf/settings.xml`. +The driver is available from Maven Central: ```xml - - - central maven repo - central maven repo https - https://repo.maven.apache.org/maven2 - - -``` - -You can add below dependency to your `pom.xml` file in your application. - -```xml - - - org.apache.hive - hive-jdbc - 2.3.7 - - org.apache.hadoop - hadoop-common - - 2.7.4 + org.apache.kyuubi + kyuubi-hive-jdbc-shaded + 1.7.0 ``` -## Use JDBC driver with kerberos +## Connect to non-kerberized Kyuubi Server -The below java code is using a keytab file to login and connect to Kyuubi server by JDBC. +The following java code connects directly to the Kyuubi Server by JDBC without using kerberos authentication. ```java package org.apache.kyuubi.examples; -import java.io.IOException; -import java.security.PrivilegedExceptionAction; import java.sql.*; -import org.apache.hadoop.security.UserGroupInformation; - -public class JDBCTest { - - private static String driverName = "org.apache.hive.jdbc.HiveDriver"; - private static String kyuubiJdbcUrl = "jdbc:hive2://localhost:10009/default;"; - - public static void main(String[] args) throws ClassNotFoundException, SQLException { - String principal = args[0]; // kerberos principal - String keytab = args[1]; // keytab file location - Configuration configuration = new Configuration(); - configuration.set(HADOOP_SECURITY_AUTHENTICATION, "kerberos"); - UserGroupInformation.setConfiguration(configuration); - UserGroupInformation ugi = UserGroupInformation.loginUserFromKeytabAndReturnUGI(principal, keytab); - - Class.forName(driverName); - Connection conn = ugi.doAs(new PrivilegedExceptionAction(){ - public Connection run() throws SQLException { - return DriverManager.getConnection(kyuubiJdbcUrl); - } - }); - Statement st = conn.createStatement(); - ResultSet res = st.executeQuery("show databases"); - while (res.next()) { - System.out.println(res.getString(1)); +public class KyuubiJDBC { + + private static String driverName = "org.apache.kyuubi.jdbc.KyuubiHiveDriver"; + private static String kyuubiJdbcUrl = "jdbc:kyuubi://localhost:10009/default;"; + + public static void main(String[] args) throws SQLException { + try (Connection conn = DriverManager.getConnection(kyuubiJdbcUrl)) { + try (Statement stmt = conn.createStatement()) { + try (ResultSet rs = stmt.executeQuery("show databases")) { + while (rs.next()) { + System.out.println(rs.getString(1)); + } + } + } + } + } +} +``` + +## Connect to Kerberized Kyuubi Server + +The following Java code uses a keytab file to login and connect to Kyuubi Server by JDBC. + +```java +package org.apache.kyuubi.examples; + +import java.sql.*; + +public class KyuubiJDBCDemo { + + private static String driverName = "org.apache.kyuubi.jdbc.KyuubiHiveDriver"; + private static String kyuubiJdbcUrlTemplate = "jdbc:kyuubi://localhost:10009/default;" + + "kyuubiClientPrincipal=%s;kyuubiClientKeytab=%s;kyuubiServerPrincipal=%s"; + + public static void main(String[] args) throws SQLException { + String clientPrincipal = args[0]; // Kerberos principal + String clientKeytab = args[1]; // Keytab file location + String serverPrincipal = args[2]; // Kerberos principal used by Kyuubi Server + String kyuubiJdbcUrl = String.format(kyuubiJdbcUrlTemplate, clientPrincipal, clientKeytab, serverPrincipal); + try (Connection conn = DriverManager.getConnection(kyuubiJdbcUrl)) { + try (Statement stmt = conn.createStatement()) { + try (ResultSet rs = stmt.executeQuery("show databases")) { + while (rs.next()) { + System.out.println(rs.getString(1)); + } } - res.close(); - st.close(); - conn.close(); + } } + } } ``` diff --git a/docs/quick_start/quick_start_with_jupyter.md b/docs/quick_start/quick_start_with_jupyter.md index 44b3faa57..608da9284 100644 --- a/docs/quick_start/quick_start_with_jupyter.md +++ b/docs/quick_start/quick_start_with_jupyter.md @@ -15,5 +15,5 @@ - limitations under the License. --> -# Getting Started With Hive Jupyter Lap +# Getting Started with Jupyter Lap diff --git a/docs/requirements.txt b/docs/requirements.txt index 8a5ee7e12..8e1f5c471 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -17,11 +17,12 @@ # under the License. # -# we shall bypass markdown-3.4.1, see details in KYUUBI-3126 -markdown==3.3.7 +markdown==3.4.1 recommonmark==0.7.1 sphinx==4.5.0 sphinx-book-theme==0.3.3 -sphinx-markdown-tables==0.0.15 -sphinx-notfound-page==0.8 +sphinx-markdown-tables==0.0.17 +sphinx-notfound-page==0.8.3 sphinx-togglebutton===0.3.2 +sphinxemoji===0.2.0 +sphinx-copybutton===0.5.2 diff --git a/docs/security/authentication.rst b/docs/security/authentication.rst index f16a452c8..00bf368ff 100644 --- a/docs/security/authentication.rst +++ b/docs/security/authentication.rst @@ -43,4 +43,4 @@ The related configurations can be found at `Authentication Configurations`_ jdbc ../extensions/server/authentication -.. _Authentication Configurations: ../deployment/settings.html#authentication +.. _Authentication Configurations: ../configuration/settings.html#authentication diff --git a/docs/security/authorization/spark/build.md b/docs/security/authorization/spark/build.md index 2756cc356..17e8e00f4 100644 --- a/docs/security/authorization/spark/build.md +++ b/docs/security/authorization/spark/build.md @@ -19,7 +19,7 @@ ## Build with Apache Maven -Kyuubi Spark AuthZ Plugin is built using [Apache Maven](http://maven.apache.org). +Kyuubi Spark AuthZ Plugin is built using [Apache Maven](https://maven.apache.org). To build it, `cd` to the root direct of kyuubi project and run: ```shell @@ -31,6 +31,19 @@ After a while, if everything goes well, you will get the plugin finally in two p - The main plugin jar, which is under `./extensions/spark/kyuubi-spark-authz/target/kyuubi-spark-authz_${scala.binary.version}-${project.version}.jar` - The least transitive dependencies needed, which are under `./extensions/spark/kyuubi-spark-authz/target/scala-${scala.binary.version}/jars` +## Build shaded jar with Apache Maven + +Apache Kyuubi also provides the shaded jar for the Spark AuthZ plugin, You can run the AuthZ plugin using just a shaded jar without the additional dependency of jars, +To build it, `cd` to the root direct of kyuubi project and run: + +```shell +build/mvn clean package -pl :kyuubi-spark-authz-shaded_2.12 -DskipTests -am +``` + +After a while, if everything goes well, you will get the plugin finally: + +- The shaded AuthZ plugin jar, which is under `./extensions/spark/kyuubi-spark-authz-shaded/target/kyuubi-spark-authz-shaded_${scala.binary.version}-${project.version}.jar` + ### Build against Different Apache Spark Versions The maven option `spark.version` is used for specifying Spark version to compile with and generate corresponding transitive dependencies. @@ -51,7 +64,7 @@ The available `spark.version`s are shown in the following table. | 3.3.x | √ | - | | 3.2.x | √ | - | | 3.1.x | √ | - | -| 3.0.x | √ | - | +| 3.0.x | x | EOL since v1.9.0 | | 2.4.x and earlier | × | [PR 2367](https://github.com/apache/kyuubi/pull/2367) is used to track how we work with older releases with scala 2.11 | Currently, Spark released with Scala 2.12 are supported. @@ -68,17 +81,18 @@ build/mvn clean package -pl :kyuubi-spark-authz_2.12 -DskipTests -Dranger.versio The available `ranger.version`s are shown in the following table. -| Ranger Version | Supported | Remark | -|:--------------:|:---------:|:------:| -| 2.3.x | √ | - | -| 2.2.x | √ | - | -| 2.1.x | √ | - | -| 2.0.x | √ | - | -| 1.2.x | √ | - | -| 1.1.x | √ | - | -| 1.0.x | √ | - | -| 0.7.x | √ | - | -| 0.6.x | √ | - | +| Ranger Version | Supported | Remark | +|:--------------:|:---------:|:-----------------------------------------------------------------------------------------:| +| 2.4.x | √ | - | +| 2.3.x | √ | - | +| 2.2.x | √ | - | +| 2.1.x | √ | - | +| 2.0.x | √ | - | +| 1.2.x | √ | - | +| 1.1.x | √ | - | +| 1.0.x | √ | - | +| 0.7.x | √ | - | +| 0.6.x | X | [KYUUBI-4672](https://github.com/apache/kyuubi/issues/4672) reported unresolved failures. | Currently, all ranger releases are supported. diff --git a/docs/security/authorization/spark/install.md b/docs/security/authorization/spark/install.md index f820f53c4..ff4131c6f 100644 --- a/docs/security/authorization/spark/install.md +++ b/docs/security/authorization/spark/install.md @@ -31,7 +31,7 @@ ## Install -With the `kyuubi-spark-authz_*.jar` and its transitive dependencies available for spark runtime classpath, such as +Use either the shaded jar `kyuubi-spark-authz-shaded_*.jar` or the `kyuubi-spark-authz_*.jar` with its transitive dependencies available for spark runtime classpath, such as - Copied to `$SPARK_HOME/jars`, or - Specified to `spark.jars` configuration diff --git a/docs/security/authorization/spark/overview.rst b/docs/security/authorization/spark/overview.rst index fcbaa880b..364d6485f 100644 --- a/docs/security/authorization/spark/overview.rst +++ b/docs/security/authorization/spark/overview.rst @@ -106,4 +106,4 @@ You can specify config `spark.kyuubi.conf.restricted.list` values to disable cha 2. A set statement with key equal to `spark.sql.optimizer.excludedRules` and value containing `org.apache.kyuubi.plugin.spark.authz.ranger.*` also does not allow modification. .. _Apache Ranger: https://ranger.apache.org/ -.. _Spark Configurations: ../../../deployment/settings.html#spark-configurations +.. _Spark Configurations: ../../../configuration/settings.html#spark-configurations diff --git a/docs/security/kerberos.rst b/docs/security/kerberos.rst index c4bca8e82..2505fa30d 100644 --- a/docs/security/kerberos.rst +++ b/docs/security/kerberos.rst @@ -115,4 +115,5 @@ Refresh all the kyuubi server instances Restart all the kyuubi server instances or `Refresh Configurations`_ to activate the settings. .. _Hadoop Impersonation: https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/Superusers.html -.. _Refresh Configurations: ..tools/kyuubi-admin.html#refresh-config +.. _configurations: ../client/advanced/kerberos.html +.. _Refresh Configurations: ../tools/kyuubi-admin.html#refresh-config diff --git a/docs/security/kinit.md b/docs/security/kinit.md index e9dfbc491..0d613e000 100644 --- a/docs/security/kinit.md +++ b/docs/security/kinit.md @@ -104,5 +104,5 @@ hadoop.proxyuser..hosts * ## Further Readings - [Hadoop in Secure Mode](https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-common/SecureMode.html) -- [Use Kerberos for authentication in Spark](http://spark.apache.org/docs/latest/security.html#kerberos) +- [Use Kerberos for authentication in Spark](https://spark.apache.org/docs/latest/security.html#kerberos) diff --git a/docs/security/ldap.md b/docs/security/ldap.md new file mode 100644 index 000000000..7994afb51 --- /dev/null +++ b/docs/security/ldap.md @@ -0,0 +1,60 @@ + + +# Configure Kyuubi to use LDAP Authentication + +Kyuubi can be configured to enable frontend LDAP authentication for clients, such as the BeeLine, or the JDBC and ODBC drivers. +At present, only simple LDAP authentication mechanism involving username and password is supported. The client sends +a username and password to the Kyuubi server, and the Kyuubi server validates these credentials using an external LDAP service. + +## Enable LDAP Authentication + +To enable LDAP authentication for Kyuubi, LDAP-related configurations is required to be configured in +`$KYUUBI_HOME/conf/kyuubi-defaults.conf` on each node where Kyuubi server is installed. + +For example, + +```properties example +kyuubi.authentication=LDAP +kyuubi.authentication.ldap.baseDN=dc=org +kyuubi.authentication.ldap.domain=apache.org +kyuubi.authentication.ldap.binddn=uid=kyuubi,OU=Users,DC=apache,DC=org +kyuubi.authentication.ldap.bindpw=kyuubi123123 +kyuubi.authentication.ldap.url=ldap://hostname.com:389/ +``` + +## User and Group Filter in LDAP + +Kyuubi also supports complex LDAP cases as [Apache Hive](https://cwiki.apache.org/confluence/display/Hive/User+and+Group+Filter+Support+with+LDAP+Atn+Provider+in+HiveServer2#UserandGroupFilterSupportwithLDAPAtnProviderinHiveServer2-UserandGroupFilterSupportwithLDAP) does. + +For example, + +```properties example +# Group Membership +kyuubi.authentication.ldap.groupClassKey=groupOfNames +kyuubi.authentication.ldap.groupDNPattern=CN=%s,OU=Groups,DC=apache,DC=org +kyuubi.authentication.ldap.groupFilter=group1,group2 +kyuubi.authentication.ldap.groupMembershipKey=memberUid +# User Search List +kyuubi.authentication.ldap.userDNPattern=CN=%s,CN=Users,DC=apache,DC=org +kyuubi.authentication.ldap.userFilter=hive-admin,hive,hive-test,hive-user +# Custom Query +kyuubi.authentication.ldap.customLDAPQuery=(&(objectClass=group)(objectClass=top)(instanceType=4)(cn=Domain*)), (&(objectClass=person)(|(sAMAccountName=admin)(|(memberOf=CN=Domain Admins,CN=Users,DC=domain,DC=com)(memberOf=CN=Administrators,CN=Builtin,DC=domain,DC=com)))) +``` + +Please refer to [Settings for LDAP authentication in Kyuubi](../configuration/settings.html?highlight=LDAP#authentication) +for all configurations. diff --git a/docs/security/ldap.rst b/docs/security/ldap.rst deleted file mode 100644 index 35cfcd6de..000000000 --- a/docs/security/ldap.rst +++ /dev/null @@ -1,21 +0,0 @@ -.. Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - -.. http://www.apache.org/licenses/LICENSE-2.0 - -.. Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - -Configure Kyuubi to use LDAP Authentication -=============================================== - -.. warning:: - the page is still in-progress. diff --git a/docs/tools/kyuubi-admin.rst b/docs/tools/kyuubi-admin.rst index cf60f67b1..bd37f7e68 100644 --- a/docs/tools/kyuubi-admin.rst +++ b/docs/tools/kyuubi-admin.rst @@ -69,6 +69,12 @@ Usage: ``bin/kyuubi-admin refresh config [options] []`` - Description * - hadoopConf - The hadoop conf used for proxy user verification. + * - userDefaultsConf + - The user defaults configs with key in format in the form of `___{username}___.{config key}` from default property file. + * - unlimitedUsers + - The users without maximum connections limitation. + * - denyUsers + - The user in the deny list will be denied to connect to kyuubi server. .. _list_engine: @@ -93,6 +99,17 @@ Usage: ``bin/kyuubi-admin list engine [options]`` - The subdomain for the share level of an engine. If not specified, it will read the configuration item kyuubi.engine.share.level.subdomain from kyuubi-defaults.conf. * - --hs2ProxyUser - The proxy user to impersonate. When specified, it will list engines for the hs2ProxyUser. + * - -a --all + - All the engine. + +.. _list_server: + +List Servers +------------------------------------- + +Prints a table of the key information about the servers. + +Usage: ``bin/kyuubi-admin list server`` .. _delete_engine: diff --git a/docs/tools/kyuubi-ctl.md b/docs/tools/kyuubi-ctl.md deleted file mode 100644 index aae67584e..000000000 --- a/docs/tools/kyuubi-ctl.md +++ /dev/null @@ -1,183 +0,0 @@ - - -# Managing kyuubi servers and engines Tool - -## Usage - -```shell -bin/kyuubi-ctl --help -``` - -Output - -```shell -kyuubi 1.6.0-SNAPSHOT -Usage: kyuubi-ctl [create|get|delete|list] [options] - - -zk, --zk-quorum - The connection string for the zookeeper ensemble, using zk quorum manually. - -n, --namespace The namespace, using kyuubi-defaults/conf if absent. - -s, --host Hostname or IP address of a service. - -p, --port Listening port of a service. - -v, --version Using the compiled KYUUBI_VERSION default, change it if the active service is running in another. - -b, --verbose Print additional debug output. - -Command: create [server] - -Command: create server - Expose Kyuubi server instance to another domain. - -Command: get [server|engine] [options] - Get the service/engine node info, host and port needed. -Command: get server - Get Kyuubi server info of domain -Command: get engine - Get Kyuubi engine info belong to a user. - -u, --user The user name this engine belong to. - -et, --engine-type - The engine type this engine belong to. - -es, --engine-subdomain - The engine subdomain this engine belong to. - -esl, --engine-share-level - The engine share level this engine belong to. - -Command: delete [server|engine] [options] - Delete the specified service/engine node, host and port needed. -Command: delete server - Delete the specified service node for a domain -Command: delete engine - Delete the specified engine node for user. - -u, --user The user name this engine belong to. - -et, --engine-type - The engine type this engine belong to. - -es, --engine-subdomain - The engine subdomain this engine belong to. - -esl, --engine-share-level - The engine share level this engine belong to. - -Command: list [server|engine] [options] - List all the service/engine nodes for a particular domain. -Command: list server - List all the service nodes for a particular domain -Command: list engine - List all the engine nodes for a user - -u, --user The user name this engine belong to. - -et, --engine-type - The engine type this engine belong to. - -es, --engine-subdomain - The engine subdomain this engine belong to. - -esl, --engine-share-level - The engine share level this engine belong to. - - -h, --help Show help message and exit. -``` - -## Manage kyuubi servers - -You can specify the zookeeper address(`--zk-quorum`) and namespace(`--namespace`), version(`--version`) parameters to query a specific kyuubi server cluster. - -### List server - -List all the service nodes for a particular domain. - -```shell -bin/kyuubi-ctl list server -``` - -### Create server - -Expose Kyuubi server instance to another domain. - -First read `kyuubi.ha.zookeeper.namespace` in `conf/kyuubi-defaults.conf`, if there are server instances under this namespace, register them in the new namespace specified by the `--namespace` parameter. - -```shell -bin/kyuubi-ctl create server --namespace XXX -``` - -### Get server - -Get Kyuubi server info of domain. - -```shell -bin/kyuubi-ctl get server --host XXX --port YYY -``` - -### Delete server - -Delete the specified service node for a domain. - -After the server node is deleted, the kyuubi server stops opening new sessions and waits for all currently open sessions to be closed before the process exits. - -```shell -bin/kyuubi-ctl delete server --host XXX --port YYY -``` - -## Manage kyuubi engines - -You can also specify the engine type(`--engine-type`), engine share level subdomain(`--engine-subdomain`) and engine share level(`--engine-share-level`). - -If not specified, the configuration item `kyuubi.engine.type` of `kyuubi-defaults.conf` read, the default value is `SPARK_SQL`, `kyuubi.engine.share.level.subdomain`, the default value is `default`, `kyuubi.engine.share.level`, the default value is `USER`. - -If the engine pool mode is enabled through `kyuubi.engine.pool.size`, the subdomain consists of `kyuubi.engine.pool.name` and a number below size, e.g. `engine-pool-0` . - -`--engine-share-level` supports the following enum values. -* CONNECTION - -The engine Ref Id (UUID) must be specified via `--engine-subdomain`. -* USER: - -Default Value. -* GROUP: - -The `--user` parameter is the group name corresponding to the user. -* SERVER: - -The `--user` parameter is the user who started the kyuubi server. - -### List engine - -List all the engine nodes for a user. - -```shell -bin/kyuubi-ctl list engine --user AAA -``` - -The management share level is SERVER, the user who starts the kyuubi server is A, the engine is TRINO, and the subdomain is adhoc. - -```shell -bin/kyuubi-ctl list engine --user A --engine-type TRINO --engine-subdomain adhoc --engine-share-level SERVER -``` - -### Get engine - -Get Kyuubi engine info belong to a user. - -```shell -bin/kyuubi-ctl get engine --user AAA --host XXX --port YYY -``` - -### Delete engine - -Delete the specified engine node for user. - -After the engine node is deleted, the kyuubi engine stops opening new sessions and waits for all currently open sessions to be closed before the process exits. - -```shell -bin/kyuubi-ctl delete engine --user AAA --host XXX --port YYY -``` - diff --git a/docs/tools/kyuubi-ctl.rst b/docs/tools/kyuubi-ctl.rst new file mode 100644 index 000000000..4a9308fed --- /dev/null +++ b/docs/tools/kyuubi-ctl.rst @@ -0,0 +1,213 @@ +.. Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +Administrator CLI +================= + +.. _usage: + +Usage +----- +.. code-block:: bash + + bin/kyuubi-ctl --help + +Output + +.. parsed-literal:: + + kyuubi |release| + Usage: kyuubi-ctl [create|get|delete|list] [options] + + -zk, --zk-quorum + The connection string for the zookeeper ensemble, using zk quorum manually. + -n, --namespace The namespace, using kyuubi-defaults/conf if absent. + -s, --host Hostname or IP address of a service. + -p, --port Listening port of a service. + -v, --version Using the compiled KYUUBI_VERSION default, change it if the active service is running in another. + -b, --verbose Print additional debug output. + + Command: create [server] + + Command: create server + Expose Kyuubi server instance to another domain. + + Command: get [server|engine] [options] + Get the service/engine node info, host and port needed. + Command: get server + Get Kyuubi server info of domain + Command: get engine + Get Kyuubi engine info belong to a user. + -u, --user The user name this engine belong to. + -et, --engine-type + The engine type this engine belong to. + -es, --engine-subdomain + The engine subdomain this engine belong to. + -esl, --engine-share-level + The engine share level this engine belong to. + + Command: delete [server|engine] [options] + Delete the specified service/engine node, host and port needed. + Command: delete server + Delete the specified service node for a domain + Command: delete engine + Delete the specified engine node for user. + -u, --user The user name this engine belong to. + -et, --engine-type + The engine type this engine belong to. + -es, --engine-subdomain + The engine subdomain this engine belong to. + -esl, --engine-share-level + The engine share level this engine belong to. + + Command: list [server|engine] [options] + List all the service/engine nodes for a particular domain. + Command: list server + List all the service nodes for a particular domain + Command: list engine + List all the engine nodes for a user + -u, --user The user name this engine belong to. + -et, --engine-type + The engine type this engine belong to. + -es, --engine-subdomain + The engine subdomain this engine belong to. + -esl, --engine-share-level + The engine share level this engine belong to. + + -h, --help Show help message and exit. + +.. _manage_kyuubi_servers: + +Manage kyuubi servers +--------------------- + +You can specify the zookeeper address(``--zk-quorum``) and namespace(``--namespace``), version(``--version``) parameters to query a specific kyuubi server cluster. + +.. _list_servers: + +List server +*********** + +List all the service nodes for a particular domain. + +.. code-block:: bash + + bin/kyuubi-ctl list server + +.. _create_servers: + +Create server +*********** +Expose Kyuubi server instance to another domain. + +First read ``kyuubi.ha.zookeeper.namespace`` in ``conf/kyuubi-defaults.conf``, if there are server instances under this namespace, register them in the new namespace specified by the ``--namespace`` parameter. + +.. code-block:: bash + + bin/kyuubi-ctl create server --namespace XXX + +.. _get_servers: + +Get server +*********** + +Get Kyuubi server info of domain. + +.. code-block:: bash + + bin/kyuubi-ctl get server --host XXX --port YYY + +.. _delete_servers: + +Delete server +*********** + +Delete the specified service node for a domain. + +After the server node is deleted, the kyuubi server stops opening new sessions and waits for all currently open sessions to be closed before the process exits. + +.. code-block:: bash + + bin/kyuubi-ctl delete server --host XXX --port YYY + +.. _manage_kyuubi_engines: + +Manage kyuubi engines +--------------------- + +You can also specify the engine type(``--engine-type``), engine share level subdomain(``--engine-subdomain``) and engine share level(``--engine-share-level``). + +If not specified, the configuration item ``kyuubi.engine.type`` of ``kyuubi-defaults.conf`` read, the default value is ``SPARK_SQL``, ``kyuubi.engine.share.level.subdomain``, the default value is ``default``, ``kyuubi.engine.share.level``, the default value is ``USER``. + +If the engine pool mode is enabled through ``kyuubi.engine.pool.size``, the subdomain consists of ``kyuubi.engine.pool.name`` and a number below size, e.g. ``engine-pool-0`` . + +``--engine-share-level`` supports the following enum values. + +- CONNECTION + +The engine Ref Id (UUID) must be specified via ``--engine-subdomain``. + +- USER: + +Default Value. + +- GROUP: + +The ``--user`` parameter is the group name corresponding to the user. + +- SERVER: + +The ``--user`` parameter is the user who started the kyuubi server. + +.. _list_engines: + +List engine +*********** + +List all the engine nodes for a user. + +.. code-block:: bash + + bin/kyuubi-ctl list engine --user AAA + +The management share level is SERVER, the user who starts the kyuubi server is A, the engine is TRINO, and the subdomain is adhoc. + +.. code-block:: bash + + bin/kyuubi-ctl list engine --user A --engine-type TRINO --engine-subdomain adhoc --engine-share-level SERVER + +.. _get_engines: + +Get engine +*********** + +Get Kyuubi engine info belong to a user. + +.. code-block:: bash + + bin/kyuubi-ctl get engine --user AAA --host XXX --port YYY + +.. _delete_engines: + +Delete engine +************* + +Delete the specified engine node for user. + +After the engine node is deleted, the kyuubi engine stops opening new sessions and waits for all currently open sessions to be closed before the process exits. + +.. code-block:: bash + + bin/kyuubi-ctl delete engine --user AAA --host XXX --port YYY \ No newline at end of file diff --git a/extensions/README.md b/extensions/README.md index 92eac9097..5725f0f9b 100644 --- a/extensions/README.md +++ b/extensions/README.md @@ -1,25 +1,24 @@ - +- Licensed to the Apache Software Foundation (ASF) under one or more +- contributor license agreements. See the NOTICE file distributed with +- this work for additional information regarding copyright ownership. +- The ASF licenses this file to You under the Apache License, Version 2.0 +- (the "License"); you may not use this file except in compliance with +- the License. You may obtain a copy of the License at +- +- http://www.apache.org/licenses/LICENSE-2.0 +- +- Unless required by applicable law or agreed to in writing, software +- distributed under the License is distributed on an "AS IS" BASIS, +- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +- See the License for the specific language governing permissions and +- limitations under the License. +--> + # For developers This folder contains plugins/extension for kyuubi server and different engine types. - - ext - kyuubi-server - spark @@ -27,4 +26,5 @@ This folder contains plugins/extension for kyuubi server and different engine ty - trino - hive - others - - ... \ No newline at end of file + - ... + diff --git a/extensions/server/kyuubi-server-plugin/pom.xml b/extensions/server/kyuubi-server-plugin/pom.xml index b7dfe0ae8..12c1699fc 100644 --- a/extensions/server/kyuubi-server-plugin/pom.xml +++ b/extensions/server/kyuubi-server-plugin/pom.xml @@ -21,7 +21,7 @@ org.apache.kyuubi kyuubi-parent - 1.7.0-SNAPSHOT + 1.9.0-SNAPSHOT ../../../pom.xml diff --git a/extensions/spark/kyuubi-extension-spark-3-1/pom.xml b/extensions/spark/kyuubi-extension-spark-3-1/pom.xml index 5bd4b2fd5..a7fcbabe5 100644 --- a/extensions/spark/kyuubi-extension-spark-3-1/pom.xml +++ b/extensions/spark/kyuubi-extension-spark-3-1/pom.xml @@ -21,11 +21,11 @@ org.apache.kyuubi kyuubi-parent - 1.7.0-SNAPSHOT + 1.9.0-SNAPSHOT ../../../pom.xml - kyuubi-extension-spark-3-1_2.12 + kyuubi-extension-spark-3-1_${scala.binary.version} jar Kyuubi Dev Spark Extensions (for Spark 3.1) https://kyuubi.apache.org/ @@ -125,10 +125,21 @@ jakarta.xml.bind-api test + + + org.apache.logging.log4j + log4j-1.2-api + test + + + + org.apache.logging.log4j + log4j-slf4j-impl + test + - org.apache.maven.plugins @@ -137,7 +148,7 @@ false - org.apache.kyuubi:kyuubi-extension-spark-common_${scala.binary.version} + org.apache.kyuubi:*       diff --git a/extensions/spark/kyuubi-extension-spark-3-1/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLExtension.scala b/extensions/spark/kyuubi-extension-spark-3-1/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLExtension.scala index cd312de95..f952b56f3 100644 --- a/extensions/spark/kyuubi-extension-spark-3-1/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLExtension.scala +++ b/extensions/spark/kyuubi-extension-spark-3-1/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLExtension.scala @@ -20,7 +20,7 @@ package org.apache.kyuubi.sql import org.apache.spark.sql.SparkSessionExtensions import org.apache.kyuubi.sql.sqlclassification.KyuubiSqlClassification -import org.apache.kyuubi.sql.watchdog.{ForcedMaxOutputRowsRule, MaxPartitionStrategy} +import org.apache.kyuubi.sql.watchdog.{ForcedMaxOutputRowsRule, MaxScanStrategy} // scalastyle:off line.size.limit /** @@ -40,6 +40,6 @@ class KyuubiSparkSQLExtension extends (SparkSessionExtensions => Unit) { // watchdog extension extensions.injectOptimizerRule(ForcedMaxOutputRowsRule) - extensions.injectPlannerStrategy(MaxPartitionStrategy) + extensions.injectPlannerStrategy(MaxScanStrategy) } } diff --git a/extensions/spark/kyuubi-extension-spark-3-1/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLParser.scala b/extensions/spark/kyuubi-extension-spark-3-1/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLParser.scala index 2f12a82e2..87c10bc34 100644 --- a/extensions/spark/kyuubi-extension-spark-3-1/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLParser.scala +++ b/extensions/spark/kyuubi-extension-spark-3-1/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLParser.scala @@ -21,19 +21,21 @@ import org.antlr.v4.runtime._ import org.antlr.v4.runtime.atn.PredictionMode import org.antlr.v4.runtime.misc.{Interval, ParseCancellationException} import org.apache.spark.sql.AnalysisException -import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier} +import org.apache.spark.sql.catalyst.{FunctionIdentifier, SQLConfHelper, TableIdentifier} import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.catalyst.parser.{ParseErrorListener, ParseException, ParserInterface, PostProcessor} import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.trees.Origin import org.apache.spark.sql.types.{DataType, StructType} -abstract class KyuubiSparkSQLParserBase extends ParserInterface { +abstract class KyuubiSparkSQLParserBase extends ParserInterface with SQLConfHelper { def delegate: ParserInterface - def astBuilder: KyuubiSparkSQLAstBuilderBase + def astBuilder: KyuubiSparkSQLAstBuilder override def parsePlan(sqlText: String): LogicalPlan = parse(sqlText) { parser => astBuilder.visit(parser.singleStatement()) match { + case optimize: UnparsedPredicateOptimize => + astBuilder.buildOptimizeStatement(optimize, delegate.parseExpression) case plan: LogicalPlan => plan case _ => delegate.parsePlan(sqlText) } @@ -105,7 +107,7 @@ abstract class KyuubiSparkSQLParserBase extends ParserInterface { class SparkKyuubiSparkSQLParser( override val delegate: ParserInterface) extends KyuubiSparkSQLParserBase { - def astBuilder: KyuubiSparkSQLAstBuilderBase = new KyuubiSparkSQLAstBuilder + def astBuilder: KyuubiSparkSQLAstBuilder = new KyuubiSparkSQLAstBuilder } /* Copied from Apache Spark's to avoid dependency on Spark Internals */ diff --git a/extensions/spark/kyuubi-extension-spark-3-1/src/main/scala/org/apache/kyuubi/sql/sqlclassification/KyuubiGetSqlClassification.scala b/extensions/spark/kyuubi-extension-spark-3-1/src/main/scala/org/apache/kyuubi/sql/sqlclassification/KyuubiGetSqlClassification.scala index e8aadc850..b94cdf346 100644 --- a/extensions/spark/kyuubi-extension-spark-3-1/src/main/scala/org/apache/kyuubi/sql/sqlclassification/KyuubiGetSqlClassification.scala +++ b/extensions/spark/kyuubi-extension-spark-3-1/src/main/scala/org/apache/kyuubi/sql/sqlclassification/KyuubiGetSqlClassification.scala @@ -55,7 +55,7 @@ object KyuubiGetSqlClassification extends Logging { * You need to make sure that the configuration item: SQL_CLASSIFICATION_ENABLED * is true * @param simpleName: the analyzied_logical_plan's getSimpleName - * @return: This sql's classification + * @return This sql's classification */ def getSqlClassification(simpleName: String): String = { jsonNode.map { json => diff --git a/extensions/spark/kyuubi-extension-spark-3-1/src/test/scala/org/apache/spark/sql/ZorderSuite.scala b/extensions/spark/kyuubi-extension-spark-3-1/src/test/scala/org/apache/spark/sql/ZorderSuite.scala index fd04e27db..29a166abf 100644 --- a/extensions/spark/kyuubi-extension-spark-3-1/src/test/scala/org/apache/spark/sql/ZorderSuite.scala +++ b/extensions/spark/kyuubi-extension-spark-3-1/src/test/scala/org/apache/spark/sql/ZorderSuite.scala @@ -17,6 +17,20 @@ package org.apache.spark.sql -class ZorderWithCodegenEnabledSuite extends ZorderWithCodegenEnabledSuiteBase {} +import org.apache.spark.sql.catalyst.parser.ParserInterface -class ZorderWithCodegenDisabledSuite extends ZorderWithCodegenDisabledSuiteBase {} +import org.apache.kyuubi.sql.SparkKyuubiSparkSQLParser + +trait ParserSuite { self: ZorderSuiteBase => + override def createParser: ParserInterface = { + new SparkKyuubiSparkSQLParser(spark.sessionState.sqlParser) + } +} + +class ZorderWithCodegenEnabledSuite + extends ZorderWithCodegenEnabledSuiteBase + with ParserSuite {} + +class ZorderWithCodegenDisabledSuite + extends ZorderWithCodegenDisabledSuiteBase + with ParserSuite {} diff --git a/extensions/spark/kyuubi-extension-spark-3-2/pom.xml b/extensions/spark/kyuubi-extension-spark-3-2/pom.xml index daab162b7..b1ddcecf8 100644 --- a/extensions/spark/kyuubi-extension-spark-3-2/pom.xml +++ b/extensions/spark/kyuubi-extension-spark-3-2/pom.xml @@ -21,11 +21,11 @@ org.apache.kyuubi kyuubi-parent - 1.7.0-SNAPSHOT + 1.9.0-SNAPSHOT ../../../pom.xml - kyuubi-extension-spark-3-2_2.12 + kyuubi-extension-spark-3-2_${scala.binary.version} jar Kyuubi Dev Spark Extensions (for Spark 3.2) https://kyuubi.apache.org/ @@ -125,10 +125,21 @@ jakarta.xml.bind-api test + + + org.apache.logging.log4j + log4j-1.2-api + test + + + + org.apache.logging.log4j + log4j-slf4j-impl + test + - org.apache.maven.plugins @@ -137,7 +148,7 @@ false - org.apache.kyuubi:kyuubi-extension-spark-common_${scala.binary.version} + org.apache.kyuubi:* diff --git a/extensions/spark/kyuubi-extension-spark-3-2/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLExtension.scala b/extensions/spark/kyuubi-extension-spark-3-2/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLExtension.scala index ef9da41be..97e777042 100644 --- a/extensions/spark/kyuubi-extension-spark-3-2/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLExtension.scala +++ b/extensions/spark/kyuubi-extension-spark-3-2/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLExtension.scala @@ -19,7 +19,7 @@ package org.apache.kyuubi.sql import org.apache.spark.sql.SparkSessionExtensions -import org.apache.kyuubi.sql.watchdog.{ForcedMaxOutputRowsRule, MaxPartitionStrategy} +import org.apache.kyuubi.sql.watchdog.{ForcedMaxOutputRowsRule, MaxScanStrategy} // scalastyle:off line.size.limit /** @@ -38,6 +38,6 @@ class KyuubiSparkSQLExtension extends (SparkSessionExtensions => Unit) { // watchdog extension extensions.injectOptimizerRule(ForcedMaxOutputRowsRule) - extensions.injectPlannerStrategy(MaxPartitionStrategy) + extensions.injectPlannerStrategy(MaxScanStrategy) } } diff --git a/extensions/spark/kyuubi-extension-spark-3-2/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLParser.scala b/extensions/spark/kyuubi-extension-spark-3-2/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLParser.scala index 2f12a82e2..87c10bc34 100644 --- a/extensions/spark/kyuubi-extension-spark-3-2/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLParser.scala +++ b/extensions/spark/kyuubi-extension-spark-3-2/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLParser.scala @@ -21,19 +21,21 @@ import org.antlr.v4.runtime._ import org.antlr.v4.runtime.atn.PredictionMode import org.antlr.v4.runtime.misc.{Interval, ParseCancellationException} import org.apache.spark.sql.AnalysisException -import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier} +import org.apache.spark.sql.catalyst.{FunctionIdentifier, SQLConfHelper, TableIdentifier} import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.catalyst.parser.{ParseErrorListener, ParseException, ParserInterface, PostProcessor} import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.trees.Origin import org.apache.spark.sql.types.{DataType, StructType} -abstract class KyuubiSparkSQLParserBase extends ParserInterface { +abstract class KyuubiSparkSQLParserBase extends ParserInterface with SQLConfHelper { def delegate: ParserInterface - def astBuilder: KyuubiSparkSQLAstBuilderBase + def astBuilder: KyuubiSparkSQLAstBuilder override def parsePlan(sqlText: String): LogicalPlan = parse(sqlText) { parser => astBuilder.visit(parser.singleStatement()) match { + case optimize: UnparsedPredicateOptimize => + astBuilder.buildOptimizeStatement(optimize, delegate.parseExpression) case plan: LogicalPlan => plan case _ => delegate.parsePlan(sqlText) } @@ -105,7 +107,7 @@ abstract class KyuubiSparkSQLParserBase extends ParserInterface { class SparkKyuubiSparkSQLParser( override val delegate: ParserInterface) extends KyuubiSparkSQLParserBase { - def astBuilder: KyuubiSparkSQLAstBuilderBase = new KyuubiSparkSQLAstBuilder + def astBuilder: KyuubiSparkSQLAstBuilder = new KyuubiSparkSQLAstBuilder } /* Copied from Apache Spark's to avoid dependency on Spark Internals */ diff --git a/extensions/spark/kyuubi-extension-spark-3-2/src/test/scala/org/apache/spark/sql/ZorderSuite.scala b/extensions/spark/kyuubi-extension-spark-3-2/src/test/scala/org/apache/spark/sql/ZorderSuite.scala index fd04e27db..29a166abf 100644 --- a/extensions/spark/kyuubi-extension-spark-3-2/src/test/scala/org/apache/spark/sql/ZorderSuite.scala +++ b/extensions/spark/kyuubi-extension-spark-3-2/src/test/scala/org/apache/spark/sql/ZorderSuite.scala @@ -17,6 +17,20 @@ package org.apache.spark.sql -class ZorderWithCodegenEnabledSuite extends ZorderWithCodegenEnabledSuiteBase {} +import org.apache.spark.sql.catalyst.parser.ParserInterface -class ZorderWithCodegenDisabledSuite extends ZorderWithCodegenDisabledSuiteBase {} +import org.apache.kyuubi.sql.SparkKyuubiSparkSQLParser + +trait ParserSuite { self: ZorderSuiteBase => + override def createParser: ParserInterface = { + new SparkKyuubiSparkSQLParser(spark.sessionState.sqlParser) + } +} + +class ZorderWithCodegenEnabledSuite + extends ZorderWithCodegenEnabledSuiteBase + with ParserSuite {} + +class ZorderWithCodegenDisabledSuite + extends ZorderWithCodegenDisabledSuiteBase + with ParserSuite {} diff --git a/extensions/spark/kyuubi-extension-spark-3-3/pom.xml b/extensions/spark/kyuubi-extension-spark-3-3/pom.xml index cc8291213..9b1a30af0 100644 --- a/extensions/spark/kyuubi-extension-spark-3-3/pom.xml +++ b/extensions/spark/kyuubi-extension-spark-3-3/pom.xml @@ -21,11 +21,11 @@ org.apache.kyuubi kyuubi-parent - 1.7.0-SNAPSHOT + 1.9.0-SNAPSHOT ../../../pom.xml - kyuubi-extension-spark-3-3_2.12 + kyuubi-extension-spark-3-3_${scala.binary.version} jar Kyuubi Dev Spark Extensions (for Spark 3.3) https://kyuubi.apache.org/ @@ -37,6 +37,14 @@ ${project.version} + + org.apache.kyuubi + kyuubi-download + ${project.version} + pom + test + + org.apache.kyuubi kyuubi-extension-spark-common_${scala.binary.version} @@ -45,6 +53,14 @@ test + + org.apache.kyuubi + kyuubi-util-scala_${scala.binary.version} + ${project.version} + test-jar + test + + org.scala-lang scala-library @@ -130,6 +146,38 @@ + + org.codehaus.mojo + build-helper-maven-plugin + + + regex-property + + regex-property + + + spark.home + ${project.basedir}/../../../externals/kyuubi-download/target/${spark.archive.name} + (.+)\.tgz + $1 + + + + + + org.scalatest + scalatest-maven-plugin + + + + ${spark.home} + ${scala.binary.version} + + + org.apache.maven.plugins maven-shade-plugin @@ -137,7 +185,7 @@ false - org.apache.kyuubi:kyuubi-extension-spark-common_${scala.binary.version} + org.apache.kyuubi:* diff --git a/extensions/spark/kyuubi-extension-spark-3-3/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLExtension.scala b/extensions/spark/kyuubi-extension-spark-3-3/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLExtension.scala index ef9da41be..792315d89 100644 --- a/extensions/spark/kyuubi-extension-spark-3-3/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLExtension.scala +++ b/extensions/spark/kyuubi-extension-spark-3-3/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLExtension.scala @@ -17,9 +17,9 @@ package org.apache.kyuubi.sql -import org.apache.spark.sql.SparkSessionExtensions +import org.apache.spark.sql.{FinalStageResourceManager, InjectCustomResourceProfile, SparkSessionExtensions} -import org.apache.kyuubi.sql.watchdog.{ForcedMaxOutputRowsRule, MaxPartitionStrategy} +import org.apache.kyuubi.sql.watchdog.{ForcedMaxOutputRowsRule, MaxScanStrategy} // scalastyle:off line.size.limit /** @@ -38,6 +38,9 @@ class KyuubiSparkSQLExtension extends (SparkSessionExtensions => Unit) { // watchdog extension extensions.injectOptimizerRule(ForcedMaxOutputRowsRule) - extensions.injectPlannerStrategy(MaxPartitionStrategy) + extensions.injectPlannerStrategy(MaxScanStrategy) + + extensions.injectQueryStagePrepRule(FinalStageResourceManager(_)) + extensions.injectQueryStagePrepRule(InjectCustomResourceProfile) } } diff --git a/extensions/spark/kyuubi-extension-spark-3-3/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLParser.scala b/extensions/spark/kyuubi-extension-spark-3-3/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLParser.scala index af1711ebb..c4418c33c 100644 --- a/extensions/spark/kyuubi-extension-spark-3-3/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLParser.scala +++ b/extensions/spark/kyuubi-extension-spark-3-3/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLParser.scala @@ -21,19 +21,21 @@ import org.antlr.v4.runtime._ import org.antlr.v4.runtime.atn.PredictionMode import org.antlr.v4.runtime.misc.{Interval, ParseCancellationException} import org.apache.spark.sql.AnalysisException -import org.apache.spark.sql.catalyst.{FunctionIdentifier, TableIdentifier} +import org.apache.spark.sql.catalyst.{FunctionIdentifier, SQLConfHelper, TableIdentifier} import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.catalyst.parser.{ParseErrorListener, ParseException, ParserInterface, PostProcessor} import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.trees.Origin import org.apache.spark.sql.types.{DataType, StructType} -abstract class KyuubiSparkSQLParserBase extends ParserInterface { +abstract class KyuubiSparkSQLParserBase extends ParserInterface with SQLConfHelper { def delegate: ParserInterface - def astBuilder: KyuubiSparkSQLAstBuilderBase + def astBuilder: KyuubiSparkSQLAstBuilder override def parsePlan(sqlText: String): LogicalPlan = parse(sqlText) { parser => astBuilder.visit(parser.singleStatement()) match { + case optimize: UnparsedPredicateOptimize => + astBuilder.buildOptimizeStatement(optimize, delegate.parseExpression) case plan: LogicalPlan => plan case _ => delegate.parsePlan(sqlText) } @@ -113,7 +115,7 @@ abstract class KyuubiSparkSQLParserBase extends ParserInterface { class SparkKyuubiSparkSQLParser( override val delegate: ParserInterface) extends KyuubiSparkSQLParserBase { - def astBuilder: KyuubiSparkSQLAstBuilderBase = new KyuubiSparkSQLAstBuilder + def astBuilder: KyuubiSparkSQLAstBuilder = new KyuubiSparkSQLAstBuilder } /* Copied from Apache Spark's to avoid dependency on Spark Internals */ diff --git a/extensions/spark/kyuubi-extension-spark-3-3/src/main/scala/org/apache/spark/sql/FinalStageResourceManager.scala b/extensions/spark/kyuubi-extension-spark-3-3/src/main/scala/org/apache/spark/sql/FinalStageResourceManager.scala new file mode 100644 index 000000000..32fb9f5ce --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-3/src/main/scala/org/apache/spark/sql/FinalStageResourceManager.scala @@ -0,0 +1,283 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +import scala.annotation.tailrec +import scala.collection.mutable +import scala.collection.mutable.ArrayBuffer + +import org.apache.spark.{ExecutorAllocationClient, MapOutputTrackerMaster, SparkContext, SparkEnv} +import org.apache.spark.internal.Logging +import org.apache.spark.resource.ResourceProfile +import org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend +import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.execution.{FilterExec, ProjectExec, SortExec, SparkPlan} +import org.apache.spark.sql.execution.adaptive._ +import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec +import org.apache.spark.sql.execution.exchange.{ENSURE_REQUIREMENTS, ShuffleExchangeExec} + +import org.apache.kyuubi.sql.{KyuubiSQLConf, MarkNumOutputColumnsRule} + +/** + * This rule assumes the final write stage has less cores requirement than previous, otherwise + * this rule would take no effect. + * + * It provide a feature: + * 1. Kill redundant executors before running final write stage + */ +case class FinalStageResourceManager(session: SparkSession) + extends Rule[SparkPlan] with FinalRebalanceStageHelper { + override def apply(plan: SparkPlan): SparkPlan = { + if (!conf.getConf(KyuubiSQLConf.FINAL_WRITE_STAGE_EAGERLY_KILL_EXECUTORS_ENABLED)) { + return plan + } + + if (!MarkNumOutputColumnsRule.isWrite(session, plan)) { + return plan + } + + val sc = session.sparkContext + val dra = sc.getConf.getBoolean("spark.dynamicAllocation.enabled", false) + val coresPerExecutor = sc.getConf.getInt("spark.executor.cores", 1) + val minExecutors = sc.getConf.getInt("spark.dynamicAllocation.minExecutors", 0) + val maxExecutors = sc.getConf.getInt("spark.dynamicAllocation.maxExecutors", Int.MaxValue) + val factor = conf.getConf(KyuubiSQLConf.FINAL_WRITE_STAGE_PARTITION_FACTOR) + val hasImprovementRoom = maxExecutors - 1 > minExecutors * factor + // Fast fail if: + // 1. DRA off + // 2. only work with yarn and k8s + // 3. maxExecutors is not bigger than minExecutors * factor + if (!dra || !sc.schedulerBackend.isInstanceOf[CoarseGrainedSchedulerBackend] || + !hasImprovementRoom) { + return plan + } + + val stageOpt = findFinalRebalanceStage(plan) + if (stageOpt.isEmpty) { + return plan + } + + // It's not safe to kill executors if this plan contains table cache. + // If the executor loses then the rdd would re-compute those partition. + if (hasTableCache(plan) && + conf.getConf(KyuubiSQLConf.FINAL_WRITE_STAGE_SKIP_KILLING_EXECUTORS_FOR_TABLE_CACHE)) { + return plan + } + + // TODO: move this to query stage optimizer when updating Spark to 3.5.x + // Since we are in `prepareQueryStage`, the AQE shuffle read has not been applied. + // So we need to apply it by self. + val shuffleRead = queryStageOptimizerRules.foldLeft(stageOpt.get.asInstanceOf[SparkPlan]) { + case (latest, rule) => rule.apply(latest) + } + val (targetCores, stage) = shuffleRead match { + case AQEShuffleReadExec(stage: ShuffleQueryStageExec, partitionSpecs) => + (partitionSpecs.length, stage) + case stage: ShuffleQueryStageExec => + // we can still kill executors if no AQE shuffle read, e.g., `.repartition(2)` + (stage.shuffle.numPartitions, stage) + case _ => + // it should never happen in current Spark, but to be safe do nothing if happens + logWarning("BUG, Please report to Apache Kyuubi community") + return plan + } + // The condition whether inject custom resource profile: + // - target executors < active executors + // - active executors - target executors > min executors + val numActiveExecutors = sc.getExecutorIds().length + val targetExecutors = (math.ceil(targetCores.toFloat / coresPerExecutor) * factor).toInt + .max(1) + val hasBenefits = targetExecutors < numActiveExecutors && + (numActiveExecutors - targetExecutors) > minExecutors + logInfo(s"The snapshot of current executors view, " + + s"active executors: $numActiveExecutors, min executor: $minExecutors, " + + s"target executors: $targetExecutors, has benefits: $hasBenefits") + if (hasBenefits) { + val shuffleId = stage.plan.asInstanceOf[ShuffleExchangeExec].shuffleDependency.shuffleId + val numReduce = stage.plan.asInstanceOf[ShuffleExchangeExec].numPartitions + // Now, there is only a final rebalance stage waiting to execute and all tasks of previous + // stage are finished. Kill redundant existed executors eagerly so the tasks of final + // stage can be centralized scheduled. + killExecutors(sc, targetExecutors, shuffleId, numReduce) + } + + plan + } + + /** + * The priority of kill executors follow: + * 1. kill executor who is younger than other (The older the JIT works better) + * 2. kill executor who produces less shuffle data first + */ + private def findExecutorToKill( + sc: SparkContext, + targetExecutors: Int, + shuffleId: Int, + numReduce: Int): Seq[String] = { + val tracker = SparkEnv.get.mapOutputTracker.asInstanceOf[MapOutputTrackerMaster] + val shuffleStatusOpt = tracker.shuffleStatuses.get(shuffleId) + if (shuffleStatusOpt.isEmpty) { + return Seq.empty + } + val shuffleStatus = shuffleStatusOpt.get + val executorToBlockSize = new mutable.HashMap[String, Long] + shuffleStatus.withMapStatuses { mapStatus => + mapStatus.foreach { status => + var i = 0 + var sum = 0L + while (i < numReduce) { + sum += status.getSizeForBlock(i) + i += 1 + } + executorToBlockSize.getOrElseUpdate(status.location.executorId, sum) + } + } + + val backend = sc.schedulerBackend.asInstanceOf[CoarseGrainedSchedulerBackend] + val executorsWithRegistrationTs = backend.getExecutorsWithRegistrationTs() + val existedExecutors = executorsWithRegistrationTs.keys.toSet + val expectedNumExecutorToKill = existedExecutors.size - targetExecutors + if (expectedNumExecutorToKill < 1) { + return Seq.empty + } + + val executorIdsToKill = new ArrayBuffer[String]() + // We first kill executor who does not hold shuffle block. It would happen because + // the last stage is running fast and finished in a short time. The existed executors are + // from previous stages that have not been killed by DRA, so we can not find it by tracking + // shuffle status. + // We should evict executors by their alive time first and retain all of executors which + // have better locality for shuffle block. + executorsWithRegistrationTs.toSeq.sortBy(_._2).foreach { case (id, _) => + if (executorIdsToKill.length < expectedNumExecutorToKill && + !executorToBlockSize.contains(id)) { + executorIdsToKill.append(id) + } + } + + // Evict the rest executors according to the shuffle block size + executorToBlockSize.toSeq.sortBy(_._2).foreach { case (id, _) => + if (executorIdsToKill.length < expectedNumExecutorToKill && existedExecutors.contains(id)) { + executorIdsToKill.append(id) + } + } + + executorIdsToKill.toSeq + } + + private def killExecutors( + sc: SparkContext, + targetExecutors: Int, + shuffleId: Int, + numReduce: Int): Unit = { + val executorAllocationClient = sc.schedulerBackend.asInstanceOf[ExecutorAllocationClient] + + val executorsToKill = + if (conf.getConf(KyuubiSQLConf.FINAL_WRITE_STAGE_EAGERLY_KILL_EXECUTORS_KILL_ALL)) { + executorAllocationClient.getExecutorIds() + } else { + findExecutorToKill(sc, targetExecutors, shuffleId, numReduce) + } + logInfo(s"Request to kill executors, total count ${executorsToKill.size}, " + + s"[${executorsToKill.mkString(", ")}].") + if (executorsToKill.isEmpty) { + return + } + + // Note, `SparkContext#killExecutors` does not allow with DRA enabled, + // see `https://github.com/apache/spark/pull/20604`. + // It may cause the status in `ExecutorAllocationManager` inconsistent with + // `CoarseGrainedSchedulerBackend` for a while. But it should be synchronous finally. + // + // We should adjust target num executors, otherwise `YarnAllocator` might re-request original + // target executors if DRA has not updated target executors yet. + // Note, DRA would re-adjust executors if there are more tasks to be executed, so we are safe. + // + // * We kill executor + // * YarnAllocator re-request target executors + // * DRA can not release executors since they are new added + // ----------------------------------------------------------------> timeline + executorAllocationClient.killExecutors( + executorIds = executorsToKill, + adjustTargetNumExecutors = true, + countFailures = false, + force = false) + + FinalStageResourceManager.getAdjustedTargetExecutors(sc) + .filter(_ < targetExecutors).foreach { adjustedExecutors => + val delta = targetExecutors - adjustedExecutors + logInfo(s"Target executors after kill ($adjustedExecutors) is lower than required " + + s"($targetExecutors). Requesting $delta additional executor(s).") + executorAllocationClient.requestExecutors(delta) + } + } + + @transient private val queryStageOptimizerRules: Seq[Rule[SparkPlan]] = Seq( + OptimizeSkewInRebalancePartitions, + CoalesceShufflePartitions(session), + OptimizeShuffleWithLocalRead) +} + +object FinalStageResourceManager extends Logging { + + private[sql] def getAdjustedTargetExecutors(sc: SparkContext): Option[Int] = { + sc.schedulerBackend match { + case schedulerBackend: CoarseGrainedSchedulerBackend => + try { + val field = classOf[CoarseGrainedSchedulerBackend] + .getDeclaredField("requestedTotalExecutorsPerResourceProfile") + field.setAccessible(true) + schedulerBackend.synchronized { + val requestedTotalExecutorsPerResourceProfile = + field.get(schedulerBackend).asInstanceOf[mutable.HashMap[ResourceProfile, Int]] + val defaultRp = sc.resourceProfileManager.defaultResourceProfile + requestedTotalExecutorsPerResourceProfile.get(defaultRp) + } + } catch { + case e: Exception => + logWarning("Failed to get requestedTotalExecutors of Default ResourceProfile", e) + None + } + case _ => None + } + } +} + +trait FinalRebalanceStageHelper extends AdaptiveSparkPlanHelper { + @tailrec + final protected def findFinalRebalanceStage(plan: SparkPlan): Option[ShuffleQueryStageExec] = { + plan match { + case p: ProjectExec => findFinalRebalanceStage(p.child) + case f: FilterExec => findFinalRebalanceStage(f.child) + case s: SortExec if !s.global => findFinalRebalanceStage(s.child) + case stage: ShuffleQueryStageExec + if stage.isMaterialized && stage.mapStats.isDefined && + stage.plan.isInstanceOf[ShuffleExchangeExec] && + stage.plan.asInstanceOf[ShuffleExchangeExec].shuffleOrigin != ENSURE_REQUIREMENTS => + Some(stage) + case _ => None + } + } + + final protected def hasTableCache(plan: SparkPlan): Boolean = { + find(plan) { + case _: InMemoryTableScanExec => true + case _ => false + }.isDefined + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-3/src/main/scala/org/apache/spark/sql/InjectCustomResourceProfile.scala b/extensions/spark/kyuubi-extension-spark-3-3/src/main/scala/org/apache/spark/sql/InjectCustomResourceProfile.scala new file mode 100644 index 000000000..30c042b2a --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-3/src/main/scala/org/apache/spark/sql/InjectCustomResourceProfile.scala @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.execution.{CustomResourceProfileExec, SparkPlan} +import org.apache.spark.sql.execution.adaptive._ + +import org.apache.kyuubi.sql.{KyuubiSQLConf, MarkNumOutputColumnsRule} + +/** + * Inject custom resource profile for final write stage, so we can specify custom + * executor resource configs. + */ +case class InjectCustomResourceProfile(session: SparkSession) + extends Rule[SparkPlan] with FinalRebalanceStageHelper { + override def apply(plan: SparkPlan): SparkPlan = { + if (!conf.getConf(KyuubiSQLConf.FINAL_WRITE_STAGE_RESOURCE_ISOLATION_ENABLED)) { + return plan + } + + if (!MarkNumOutputColumnsRule.isWrite(session, plan)) { + return plan + } + + val stage = findFinalRebalanceStage(plan) + if (stage.isEmpty) { + return plan + } + + // TODO: Ideally, We can call `CoarseGrainedSchedulerBackend.requestTotalExecutors` eagerly + // to reduce the task submit pending time, but it may lose task locality. + // + // By default, it would request executors when catch stage submit event. + injectCustomResourceProfile(plan, stage.get.id) + } + + private def injectCustomResourceProfile(plan: SparkPlan, id: Int): SparkPlan = { + plan match { + case stage: ShuffleQueryStageExec if stage.id == id => + CustomResourceProfileExec(stage) + case _ => plan.mapChildren(child => injectCustomResourceProfile(child, id)) + } + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-3/src/main/scala/org/apache/spark/sql/execution/CustomResourceProfileExec.scala b/extensions/spark/kyuubi-extension-spark-3-3/src/main/scala/org/apache/spark/sql/execution/CustomResourceProfileExec.scala new file mode 100644 index 000000000..3698140fb --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-3/src/main/scala/org/apache/spark/sql/execution/CustomResourceProfileExec.scala @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution + +import org.apache.spark.network.util.{ByteUnit, JavaUtils} +import org.apache.spark.rdd.RDD +import org.apache.spark.resource.{ExecutorResourceRequests, ResourceProfileBuilder} +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions.{Attribute, SortOrder} +import org.apache.spark.sql.catalyst.plans.physical.Partitioning +import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics} +import org.apache.spark.sql.vectorized.ColumnarBatch +import org.apache.spark.util.Utils + +import org.apache.kyuubi.sql.KyuubiSQLConf._ + +/** + * This node wraps the final executed plan and inject custom resource profile to the RDD. + * It assumes that, the produced RDD would create the `ResultStage` in `DAGScheduler`, + * so it makes resource isolation between previous and final stage. + * + * Note that, Spark does not support config `minExecutors` for each resource profile. + * Which means, it would retain `minExecutors` for each resource profile. + * So, suggest set `spark.dynamicAllocation.minExecutors` to 0 if enable this feature. + */ +case class CustomResourceProfileExec(child: SparkPlan) extends UnaryExecNode { + override def output: Seq[Attribute] = child.output + override def outputPartitioning: Partitioning = child.outputPartitioning + override def outputOrdering: Seq[SortOrder] = child.outputOrdering + override def supportsColumnar: Boolean = child.supportsColumnar + override def supportsRowBased: Boolean = child.supportsRowBased + override protected def doCanonicalize(): SparkPlan = child.canonicalized + + private val executorCores = conf.getConf(FINAL_WRITE_STAGE_EXECUTOR_CORES).getOrElse( + sparkContext.getConf.getInt("spark.executor.cores", 1)) + private val executorMemory = conf.getConf(FINAL_WRITE_STAGE_EXECUTOR_MEMORY).getOrElse( + sparkContext.getConf.get("spark.executor.memory", "2G")) + private val executorMemoryOverhead = + conf.getConf(FINAL_WRITE_STAGE_EXECUTOR_MEMORY_OVERHEAD) + .getOrElse(sparkContext.getConf.get("spark.executor.memoryOverhead", "1G")) + private val executorOffHeapMemory = conf.getConf(FINAL_WRITE_STAGE_EXECUTOR_OFF_HEAP_MEMORY) + + override lazy val metrics: Map[String, SQLMetric] = { + val base = Map( + "executorCores" -> SQLMetrics.createMetric(sparkContext, "executor cores"), + "executorMemory" -> SQLMetrics.createMetric(sparkContext, "executor memory (MiB)"), + "executorMemoryOverhead" -> SQLMetrics.createMetric( + sparkContext, + "executor memory overhead (MiB)")) + val addition = executorOffHeapMemory.map(_ => + "executorOffHeapMemory" -> + SQLMetrics.createMetric(sparkContext, "executor off heap memory (MiB)")).toMap + base ++ addition + } + + private def wrapResourceProfile[T](rdd: RDD[T]): RDD[T] = { + if (Utils.isTesting) { + // do nothing for local testing + return rdd + } + + metrics("executorCores") += executorCores + metrics("executorMemory") += JavaUtils.byteStringAs(executorMemory, ByteUnit.MiB) + metrics("executorMemoryOverhead") += JavaUtils.byteStringAs( + executorMemoryOverhead, + ByteUnit.MiB) + executorOffHeapMemory.foreach(m => + metrics("executorOffHeapMemory") += JavaUtils.byteStringAs(m, ByteUnit.MiB)) + + val executionId = sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY) + SQLMetrics.postDriverMetricUpdates(sparkContext, executionId, metrics.values.toSeq) + + val resourceProfileBuilder = new ResourceProfileBuilder() + val executorResourceRequests = new ExecutorResourceRequests() + executorResourceRequests.cores(executorCores) + executorResourceRequests.memory(executorMemory) + executorResourceRequests.memoryOverhead(executorMemoryOverhead) + executorOffHeapMemory.foreach(executorResourceRequests.offHeapMemory) + resourceProfileBuilder.require(executorResourceRequests) + rdd.withResources(resourceProfileBuilder.build()) + rdd + } + + override protected def doExecute(): RDD[InternalRow] = { + val rdd = child.execute() + wrapResourceProfile(rdd) + } + + override protected def doExecuteColumnar(): RDD[ColumnarBatch] = { + val rdd = child.executeColumnar() + wrapResourceProfile(rdd) + } + + override protected def withNewChildInternal(newChild: SparkPlan): SparkPlan = { + this.copy(child = newChild) + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-3/src/test/resources/log4j2-test.xml b/extensions/spark/kyuubi-extension-spark-3-3/src/test/resources/log4j2-test.xml index bfc40dd6d..3110216c1 100644 --- a/extensions/spark/kyuubi-extension-spark-3-3/src/test/resources/log4j2-test.xml +++ b/extensions/spark/kyuubi-extension-spark-3-3/src/test/resources/log4j2-test.xml @@ -21,14 +21,14 @@ - + - + diff --git a/extensions/spark/kyuubi-extension-spark-3-3/src/test/scala/org/apache/spark/sql/FinalStageResourceManagerSuite.scala b/extensions/spark/kyuubi-extension-spark-3-3/src/test/scala/org/apache/spark/sql/FinalStageResourceManagerSuite.scala new file mode 100644 index 000000000..4b9991ef6 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-3/src/test/scala/org/apache/spark/sql/FinalStageResourceManagerSuite.scala @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +import org.apache.spark.SparkConf +import org.scalatest.time.{Minutes, Span} + +import org.apache.kyuubi.sql.KyuubiSQLConf +import org.apache.kyuubi.tags.SparkLocalClusterTest + +@SparkLocalClusterTest +class FinalStageResourceManagerSuite extends KyuubiSparkSQLExtensionTest { + + override def sparkConf(): SparkConf = { + // It is difficult to run spark in local-cluster mode when spark.testing is set. + sys.props.remove("spark.testing") + + super.sparkConf().set("spark.master", "local-cluster[3, 1, 1024]") + .set("spark.dynamicAllocation.enabled", "true") + .set("spark.dynamicAllocation.initialExecutors", "3") + .set("spark.dynamicAllocation.minExecutors", "1") + .set("spark.dynamicAllocation.shuffleTracking.enabled", "true") + .set(KyuubiSQLConf.FINAL_STAGE_CONFIG_ISOLATION.key, "true") + .set(KyuubiSQLConf.FINAL_WRITE_STAGE_EAGERLY_KILL_EXECUTORS_ENABLED.key, "true") + } + + test("[KYUUBI #5136][Bug] Final Stage hangs forever") { + // Prerequisite to reproduce the bug: + // 1. Dynamic allocation is enabled. + // 2. Dynamic allocation min executors is 1. + // 3. target executors < active executors. + // 4. No active executor is left after FinalStageResourceManager killed executors. + // This is possible because FinalStageResourceManager retained executors may already be + // requested to be killed but not died yet. + // 5. Final Stage required executors is 1. + withSQLConf( + (KyuubiSQLConf.FINAL_WRITE_STAGE_EAGERLY_KILL_EXECUTORS_KILL_ALL.key, "true")) { + withTable("final_stage") { + eventually(timeout(Span(10, Minutes))) { + sql( + "CREATE TABLE final_stage AS SELECT id, count(*) as num FROM (SELECT 0 id) GROUP BY id") + } + assert(FinalStageResourceManager.getAdjustedTargetExecutors(spark.sparkContext).get == 1) + } + } + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-3/src/test/scala/org/apache/spark/sql/InjectResourceProfileSuite.scala b/extensions/spark/kyuubi-extension-spark-3-3/src/test/scala/org/apache/spark/sql/InjectResourceProfileSuite.scala new file mode 100644 index 000000000..b0767b187 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-3/src/test/scala/org/apache/spark/sql/InjectResourceProfileSuite.scala @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +import org.apache.spark.scheduler.{SparkListener, SparkListenerEvent} +import org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate + +import org.apache.kyuubi.sql.KyuubiSQLConf + +class InjectResourceProfileSuite extends KyuubiSparkSQLExtensionTest { + private def checkCustomResourceProfile(sqlString: String, exists: Boolean): Unit = { + @volatile var lastEvent: SparkListenerSQLAdaptiveExecutionUpdate = null + val listener = new SparkListener { + override def onOtherEvent(event: SparkListenerEvent): Unit = { + event match { + case e: SparkListenerSQLAdaptiveExecutionUpdate => lastEvent = e + case _ => + } + } + } + + spark.sparkContext.addSparkListener(listener) + try { + sql(sqlString).collect() + spark.sparkContext.listenerBus.waitUntilEmpty() + assert(lastEvent != null) + var current = lastEvent.sparkPlanInfo + var shouldStop = false + while (!shouldStop) { + if (current.nodeName != "CustomResourceProfile") { + if (current.children.isEmpty) { + assert(!exists) + shouldStop = true + } else { + current = current.children.head + } + } else { + assert(exists) + shouldStop = true + } + } + } finally { + spark.sparkContext.removeSparkListener(listener) + } + } + + test("Inject resource profile") { + withTable("t") { + withSQLConf( + "spark.sql.adaptive.forceApply" -> "true", + KyuubiSQLConf.FINAL_STAGE_CONFIG_ISOLATION.key -> "true", + KyuubiSQLConf.FINAL_WRITE_STAGE_RESOURCE_ISOLATION_ENABLED.key -> "true") { + + sql("CREATE TABLE t (c1 int, c2 string) USING PARQUET") + + checkCustomResourceProfile("INSERT INTO TABLE t VALUES(1, 'a')", false) + checkCustomResourceProfile("SELECT 1", false) + checkCustomResourceProfile( + "INSERT INTO TABLE t SELECT /*+ rebalance */ * FROM VALUES(1, 'a')", + true) + } + } + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-3/src/test/scala/org/apache/spark/sql/ZorderSuite.scala b/extensions/spark/kyuubi-extension-spark-3-3/src/test/scala/org/apache/spark/sql/ZorderSuite.scala index 90fc17e24..a08366f1d 100644 --- a/extensions/spark/kyuubi-extension-spark-3-3/src/test/scala/org/apache/spark/sql/ZorderSuite.scala +++ b/extensions/spark/kyuubi-extension-spark-3-3/src/test/scala/org/apache/spark/sql/ZorderSuite.scala @@ -17,13 +17,14 @@ package org.apache.spark.sql +import org.apache.spark.sql.catalyst.parser.ParserInterface import org.apache.spark.sql.catalyst.plans.logical.{RebalancePartitions, Sort} import org.apache.spark.sql.internal.SQLConf -import org.apache.kyuubi.sql.KyuubiSQLConf +import org.apache.kyuubi.sql.{KyuubiSQLConf, SparkKyuubiSparkSQLParser} import org.apache.kyuubi.sql.zorder.Zorder -trait ZorderWithCodegenEnabledSuiteBase33 extends ZorderWithCodegenEnabledSuiteBase { +trait ZorderSuiteSpark33 extends ZorderSuiteBase { test("Add rebalance before zorder") { Seq("true" -> false, "false" -> true).foreach { case (useOriginalOrdering, zorder) => @@ -106,6 +107,18 @@ trait ZorderWithCodegenEnabledSuiteBase33 extends ZorderWithCodegenEnabledSuiteB } } -class ZorderWithCodegenEnabledSuite extends ZorderWithCodegenEnabledSuiteBase33 {} +trait ParserSuite { self: ZorderSuiteBase => + override def createParser: ParserInterface = { + new SparkKyuubiSparkSQLParser(spark.sessionState.sqlParser) + } +} + +class ZorderWithCodegenEnabledSuite + extends ZorderWithCodegenEnabledSuiteBase + with ZorderSuiteSpark33 + with ParserSuite {} -class ZorderWithCodegenDisabledSuite extends ZorderWithCodegenEnabledSuiteBase33 {} +class ZorderWithCodegenDisabledSuite + extends ZorderWithCodegenDisabledSuiteBase + with ZorderSuiteSpark33 + with ParserSuite {} diff --git a/extensions/spark/kyuubi-spark-connector-kudu/pom.xml b/extensions/spark/kyuubi-extension-spark-3-4/pom.xml similarity index 64% rename from extensions/spark/kyuubi-spark-connector-kudu/pom.xml rename to extensions/spark/kyuubi-extension-spark-3-4/pom.xml index f3d667fce..ee5b5f155 100644 --- a/extensions/spark/kyuubi-spark-connector-kudu/pom.xml +++ b/extensions/spark/kyuubi-extension-spark-3-4/pom.xml @@ -21,13 +21,13 @@ org.apache.kyuubi kyuubi-parent - 1.7.0-SNAPSHOT + 1.9.0-SNAPSHOT ../../../pom.xml - kyuubi-spark-connector-kudu_2.12 + kyuubi-extension-spark-3-4_${scala.binary.version} jar - Kyuubi Spark Kudu Connector + Kyuubi Dev Spark Extensions (for Spark 3.4) https://kyuubi.apache.org/ @@ -38,20 +38,14 @@ - org.apache.logging.log4j - log4j-api - provided - - - - org.apache.logging.log4j - log4j-core + org.apache.spark + spark-sql_${scala.binary.version} provided org.apache.spark - spark-sql_${scala.binary.version} + spark-hive_${scala.binary.version} provided @@ -62,48 +56,45 @@ - org.apache.kudu - kudu-client - - - - org.apache.spark - spark-catalyst_${scala.binary.version} - test-jar + org.apache.kyuubi + kyuubi-download + ${project.version} + pom test - org.scalatestplus - scalacheck-1-17_${scala.binary.version} + org.apache.kyuubi + kyuubi-util-scala_${scala.binary.version} + ${project.version} + test-jar test - com.dimafeng - testcontainers-scala-scalatest_${scala.binary.version} + org.apache.spark + spark-core_${scala.binary.version} + test-jar test org.apache.spark - spark-sql_${scala.binary.version} - ${spark.version} + spark-catalyst_${scala.binary.version} test-jar test - org.apache.kyuubi - kyuubi-common_${scala.binary.version} - ${project.version} + org.scalatestplus + scalacheck-1-17_${scala.binary.version} test - org.apache.kyuubi - kyuubi-common_${scala.binary.version} - ${project.version} + org.apache.spark + spark-sql_${scala.binary.version} + ${spark.version} test-jar test @@ -136,16 +127,55 @@ jakarta.xml.bind-api test + + + org.apache.logging.log4j + log4j-slf4j-impl + test + - org.apache.maven.plugins - maven-dependency-plugin + org.codehaus.mojo + build-helper-maven-plugin + + + regex-property + + regex-property + + + spark.home + ${project.basedir}/../../../externals/kyuubi-download/target/${spark.archive.name} + (.+)\.tgz + $1 + + + + + + org.scalatest + scalatest-maven-plugin + + + + ${spark.home} + ${scala.binary.version} + + + + + org.antlr + antlr4-maven-plugin - true + true + ${project.basedir}/src/main/antlr4 @@ -156,43 +186,9 @@ false - org.apache.kudu:kudu-client - com.stumbleupon:async + org.apache.kyuubi:* - - - org.apache.kudu:kudu-client - - META-INF/maven/** - META-INF/native/** - META-INF/native-image/** - MANIFEST.MF - LICENSE - LICENSE.txt - NOTICE - NOTICE.txt - *.properties - **/*.proto - - - - - - org.apache.kudu - ${kyuubi.shade.packageName}.org.apache.kudu - - org.apache.kudu.** - - - - com.stumbleupon:async - ${kyuubi.shade.packageName}.com.stumbleupon.async - - com.stumbleupon.async.** - - - @@ -203,20 +199,6 @@ - - - org.apache.maven.plugins - maven-jar-plugin - - - prepare-test-jar - - test-jar - - test-compile - - - target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes diff --git a/extensions/spark/kyuubi-extension-spark-3-4/src/main/antlr4/org/apache/kyuubi/sql/KyuubiSparkSQL.g4 b/extensions/spark/kyuubi-extension-spark-3-4/src/main/antlr4/org/apache/kyuubi/sql/KyuubiSparkSQL.g4 new file mode 100644 index 000000000..e52b7f5cf --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/main/antlr4/org/apache/kyuubi/sql/KyuubiSparkSQL.g4 @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +grammar KyuubiSparkSQL; + +@members { + /** + * Verify whether current token is a valid decimal token (which contains dot). + * Returns true if the character that follows the token is not a digit or letter or underscore. + * + * For example: + * For char stream "2.3", "2." is not a valid decimal token, because it is followed by digit '3'. + * For char stream "2.3_", "2.3" is not a valid decimal token, because it is followed by '_'. + * For char stream "2.3W", "2.3" is not a valid decimal token, because it is followed by 'W'. + * For char stream "12.0D 34.E2+0.12 " 12.0D is a valid decimal token because it is followed + * by a space. 34.E2 is a valid decimal token because it is followed by symbol '+' + * which is not a digit or letter or underscore. + */ + public boolean isValidDecimal() { + int nextChar = _input.LA(1); + if (nextChar >= 'A' && nextChar <= 'Z' || nextChar >= '0' && nextChar <= '9' || + nextChar == '_') { + return false; + } else { + return true; + } + } + } + +tokens { + DELIMITER +} + +singleStatement + : statement EOF + ; + +statement + : OPTIMIZE multipartIdentifier whereClause? zorderClause #optimizeZorder + | .*? #passThrough + ; + +whereClause + : WHERE partitionPredicate = predicateToken + ; + +zorderClause + : ZORDER BY order+=multipartIdentifier (',' order+=multipartIdentifier)* + ; + +// We don't have an expression rule in our grammar here, so we just grab the tokens and defer +// parsing them to later. +predicateToken + : .+? + ; + +multipartIdentifier + : parts+=identifier ('.' parts+=identifier)* + ; + +identifier + : strictIdentifier + ; + +strictIdentifier + : IDENTIFIER #unquotedIdentifier + | quotedIdentifier #quotedIdentifierAlternative + | nonReserved #unquotedIdentifier + ; + +quotedIdentifier + : BACKQUOTED_IDENTIFIER + ; + +nonReserved + : AND + | BY + | FALSE + | DATE + | INTERVAL + | OPTIMIZE + | OR + | TABLE + | TIMESTAMP + | TRUE + | WHERE + | ZORDER + ; + +AND: 'AND'; +BY: 'BY'; +FALSE: 'FALSE'; +DATE: 'DATE'; +INTERVAL: 'INTERVAL'; +OPTIMIZE: 'OPTIMIZE'; +OR: 'OR'; +TABLE: 'TABLE'; +TIMESTAMP: 'TIMESTAMP'; +TRUE: 'TRUE'; +WHERE: 'WHERE'; +ZORDER: 'ZORDER'; + +MINUS: '-'; + +BIGINT_LITERAL + : DIGIT+ 'L' + ; + +SMALLINT_LITERAL + : DIGIT+ 'S' + ; + +TINYINT_LITERAL + : DIGIT+ 'Y' + ; + +INTEGER_VALUE + : DIGIT+ + ; + +DECIMAL_VALUE + : DIGIT+ EXPONENT + | DECIMAL_DIGITS EXPONENT? {isValidDecimal()}? + ; + +DOUBLE_LITERAL + : DIGIT+ EXPONENT? 'D' + | DECIMAL_DIGITS EXPONENT? 'D' {isValidDecimal()}? + ; + +BIGDECIMAL_LITERAL + : DIGIT+ EXPONENT? 'BD' + | DECIMAL_DIGITS EXPONENT? 'BD' {isValidDecimal()}? + ; + +BACKQUOTED_IDENTIFIER + : '`' ( ~'`' | '``' )* '`' + ; + +IDENTIFIER + : (LETTER | DIGIT | '_')+ + ; + +fragment DECIMAL_DIGITS + : DIGIT+ '.' DIGIT* + | '.' DIGIT+ + ; + +fragment EXPONENT + : 'E' [+-]? DIGIT+ + ; + +fragment DIGIT + : [0-9] + ; + +fragment LETTER + : [A-Z] + ; + +SIMPLE_COMMENT + : '--' ~[\r\n]* '\r'? '\n'? -> channel(HIDDEN) + ; + +BRACKETED_COMMENT + : '/*' .*? '*/' -> channel(HIDDEN) + ; + +WS : [ \r\n\t]+ -> channel(HIDDEN) + ; + +// Catch-all for anything we can't recognize. +// We use this to be able to ignore and recover all the text +// when splitting statements with DelimiterLexer +UNRECOGNIZED + : . + ; diff --git a/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/DropIgnoreNonexistent.scala b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/DropIgnoreNonexistent.scala new file mode 100644 index 000000000..e33632b8b --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/DropIgnoreNonexistent.scala @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.sql + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.analysis.{UnresolvedFunctionName, UnresolvedRelation} +import org.apache.spark.sql.catalyst.plans.logical.{DropFunction, DropNamespace, LogicalPlan, NoopCommand, UncacheTable} +import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.execution.command.{AlterTableDropPartitionCommand, DropTableCommand} + +import org.apache.kyuubi.sql.KyuubiSQLConf._ + +case class DropIgnoreNonexistent(session: SparkSession) extends Rule[LogicalPlan] { + + override def apply(plan: LogicalPlan): LogicalPlan = { + if (conf.getConf(DROP_IGNORE_NONEXISTENT)) { + plan match { + case i @ AlterTableDropPartitionCommand(_, _, false, _, _) => + i.copy(ifExists = true) + case i @ DropTableCommand(_, false, _, _) => + i.copy(ifExists = true) + case i @ DropNamespace(_, false, _) => + i.copy(ifExists = true) + case UncacheTable(u: UnresolvedRelation, false, _) => + NoopCommand("UNCACHE TABLE", u.multipartIdentifier) + case DropFunction(u: UnresolvedFunctionName, false) => + NoopCommand("DROP FUNCTION", u.multipartIdentifier) + case _ => plan + } + } else { + plan + } + } + +} diff --git a/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/InferRebalanceAndSortOrders.scala b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/InferRebalanceAndSortOrders.scala new file mode 100644 index 000000000..fcbf5c0a1 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/InferRebalanceAndSortOrders.scala @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql + +import scala.annotation.tailrec + +import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeSet, Expression, NamedExpression, UnaryExpression} +import org.apache.spark.sql.catalyst.planning.ExtractEquiJoinKeys +import org.apache.spark.sql.catalyst.plans.{FullOuter, Inner, LeftAnti, LeftOuter, LeftSemi, RightOuter} +import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, Filter, LogicalPlan, Project, Sort, SubqueryAlias, View} + +/** + * Infer the columns for Rebalance and Sort to improve the compression ratio. + * + * For example + * {{{ + * INSERT INTO TABLE t PARTITION(p='a') + * SELECT * FROM t1 JOIN t2 on t1.c1 = t2.c1 + * }}} + * the inferred columns are: t1.c1 + */ +object InferRebalanceAndSortOrders { + + type PartitioningAndOrdering = (Seq[Expression], Seq[Expression]) + + private def getAliasMap(named: Seq[NamedExpression]): Map[Expression, Attribute] = { + @tailrec + def throughUnary(e: Expression): Expression = e match { + case u: UnaryExpression if u.deterministic => + throughUnary(u.child) + case _ => e + } + + named.flatMap { + case a @ Alias(child, _) => + Some((throughUnary(child).canonicalized, a.toAttribute)) + case _ => None + }.toMap + } + + def infer(plan: LogicalPlan): Option[PartitioningAndOrdering] = { + def candidateKeys( + input: LogicalPlan, + output: AttributeSet = AttributeSet.empty): Option[PartitioningAndOrdering] = { + input match { + case ExtractEquiJoinKeys(joinType, leftKeys, rightKeys, _, _, _, _, _) => + joinType match { + case LeftSemi | LeftAnti | LeftOuter => Some((leftKeys, leftKeys)) + case RightOuter => Some((rightKeys, rightKeys)) + case Inner | FullOuter => + if (output.isEmpty) { + Some((leftKeys ++ rightKeys, leftKeys ++ rightKeys)) + } else { + assert(leftKeys.length == rightKeys.length) + val keys = leftKeys.zip(rightKeys).flatMap { case (left, right) => + if (left.references.subsetOf(output)) { + Some(left) + } else if (right.references.subsetOf(output)) { + Some(right) + } else { + None + } + } + Some((keys, keys)) + } + case _ => None + } + case agg: Aggregate => + val aliasMap = getAliasMap(agg.aggregateExpressions) + Some(( + agg.groupingExpressions.map(p => aliasMap.getOrElse(p.canonicalized, p)), + agg.groupingExpressions.map(o => aliasMap.getOrElse(o.canonicalized, o)))) + case s: Sort => Some((s.order.map(_.child), s.order.map(_.child))) + case p: Project => + val aliasMap = getAliasMap(p.projectList) + candidateKeys(p.child, p.references).map { case (partitioning, ordering) => + ( + partitioning.map(p => aliasMap.getOrElse(p.canonicalized, p)), + ordering.map(o => aliasMap.getOrElse(o.canonicalized, o))) + } + case f: Filter => candidateKeys(f.child, output) + case s: SubqueryAlias => candidateKeys(s.child, output) + case v: View => candidateKeys(v.child, output) + + case _ => None + } + } + + candidateKeys(plan).map { case (partitioning, ordering) => + ( + partitioning.filter(_.references.subsetOf(plan.outputSet)), + ordering.filter(_.references.subsetOf(plan.outputSet))) + } + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/InsertShuffleNodeBeforeJoin.scala b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/InsertShuffleNodeBeforeJoin.scala new file mode 100644 index 000000000..1a02e8c1e --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/InsertShuffleNodeBeforeJoin.scala @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql + +import org.apache.spark.sql.catalyst.plans.physical.Distribution +import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.execution.{SortExec, SparkPlan} +import org.apache.spark.sql.execution.adaptive.QueryStageExec +import org.apache.spark.sql.execution.aggregate.BaseAggregateExec +import org.apache.spark.sql.execution.exchange.{Exchange, ShuffleExchangeExec} +import org.apache.spark.sql.execution.joins.{ShuffledHashJoinExec, SortMergeJoinExec} +import org.apache.spark.sql.internal.SQLConf + +import org.apache.kyuubi.sql.KyuubiSQLConf._ + +/** + * Insert shuffle node before join if it doesn't exist to make `OptimizeSkewedJoin` works. + */ +object InsertShuffleNodeBeforeJoin extends Rule[SparkPlan] { + + override def apply(plan: SparkPlan): SparkPlan = { + // this rule has no meaning without AQE + if (!conf.getConf(FORCE_SHUFFLE_BEFORE_JOIN) || + !conf.getConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED)) { + return plan + } + + val newPlan = insertShuffleBeforeJoin(plan) + if (plan.fastEquals(newPlan)) { + plan + } else { + // make sure the output partitioning and ordering will not be broken. + KyuubiEnsureRequirements.apply(newPlan) + } + } + + // Since spark 3.3, insertShuffleBeforeJoin shouldn't be applied if join is skewed. + private def insertShuffleBeforeJoin(plan: SparkPlan): SparkPlan = plan transformUp { + case smj @ SortMergeJoinExec(_, _, _, _, l, r, isSkewJoin) if !isSkewJoin => + smj.withNewChildren(checkAndInsertShuffle(smj.requiredChildDistribution.head, l) :: + checkAndInsertShuffle(smj.requiredChildDistribution(1), r) :: Nil) + + case shj: ShuffledHashJoinExec if !shj.isSkewJoin => + if (!shj.left.isInstanceOf[Exchange] && !shj.right.isInstanceOf[Exchange]) { + shj.withNewChildren(withShuffleExec(shj.requiredChildDistribution.head, shj.left) :: + withShuffleExec(shj.requiredChildDistribution(1), shj.right) :: Nil) + } else if (!shj.left.isInstanceOf[Exchange]) { + shj.withNewChildren( + withShuffleExec(shj.requiredChildDistribution.head, shj.left) :: shj.right :: Nil) + } else if (!shj.right.isInstanceOf[Exchange]) { + shj.withNewChildren( + shj.left :: withShuffleExec(shj.requiredChildDistribution(1), shj.right) :: Nil) + } else { + shj + } + } + + private def checkAndInsertShuffle( + distribution: Distribution, + child: SparkPlan): SparkPlan = child match { + case SortExec(_, _, _: Exchange, _) => + child + case SortExec(_, _, _: QueryStageExec, _) => + child + case sort @ SortExec(_, _, agg: BaseAggregateExec, _) => + sort.withNewChildren(withShuffleExec(distribution, agg) :: Nil) + case _ => + withShuffleExec(distribution, child) + } + + private def withShuffleExec(distribution: Distribution, child: SparkPlan): SparkPlan = { + val numPartitions = distribution.requiredNumPartitions + .getOrElse(conf.numShufflePartitions) + ShuffleExchangeExec(distribution.createPartitioning(numPartitions), child) + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/KyuubiEnsureRequirements.scala b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/KyuubiEnsureRequirements.scala new file mode 100644 index 000000000..a17e0a465 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/KyuubiEnsureRequirements.scala @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql + +import org.apache.spark.sql.catalyst.expressions.SortOrder +import org.apache.spark.sql.catalyst.plans.physical.{BroadcastDistribution, Distribution, UnspecifiedDistribution} +import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.execution.{SortExec, SparkPlan} +import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ShuffleExchangeExec} + +/** + * Copy from Apache Spark `EnsureRequirements` + * 1. remove reorder join predicates + * 2. remove shuffle pruning + */ +object KyuubiEnsureRequirements extends Rule[SparkPlan] { + private def ensureDistributionAndOrdering(operator: SparkPlan): SparkPlan = { + val requiredChildDistributions: Seq[Distribution] = operator.requiredChildDistribution + val requiredChildOrderings: Seq[Seq[SortOrder]] = operator.requiredChildOrdering + var children: Seq[SparkPlan] = operator.children + assert(requiredChildDistributions.length == children.length) + assert(requiredChildOrderings.length == children.length) + + // Ensure that the operator's children satisfy their output distribution requirements. + children = children.zip(requiredChildDistributions).map { + case (child, distribution) if child.outputPartitioning.satisfies(distribution) => + child + case (child, BroadcastDistribution(mode)) => + BroadcastExchangeExec(mode, child) + case (child, distribution) => + val numPartitions = distribution.requiredNumPartitions + .getOrElse(conf.numShufflePartitions) + ShuffleExchangeExec(distribution.createPartitioning(numPartitions), child) + } + + // Get the indexes of children which have specified distribution requirements and need to have + // same number of partitions. + val childrenIndexes = requiredChildDistributions.zipWithIndex.filter { + case (UnspecifiedDistribution, _) => false + case (_: BroadcastDistribution, _) => false + case _ => true + }.map(_._2) + + val childrenNumPartitions = + childrenIndexes.map(children(_).outputPartitioning.numPartitions).toSet + + if (childrenNumPartitions.size > 1) { + // Get the number of partitions which is explicitly required by the distributions. + val requiredNumPartitions = { + val numPartitionsSet = childrenIndexes.flatMap { + index => requiredChildDistributions(index).requiredNumPartitions + }.toSet + assert( + numPartitionsSet.size <= 1, + s"$operator have incompatible requirements of the number of partitions for its children") + numPartitionsSet.headOption + } + + // If there are non-shuffle children that satisfy the required distribution, we have + // some tradeoffs when picking the expected number of shuffle partitions: + // 1. We should avoid shuffling these children. + // 2. We should have a reasonable parallelism. + val nonShuffleChildrenNumPartitions = + childrenIndexes.map(children).filterNot(_.isInstanceOf[ShuffleExchangeExec]) + .map(_.outputPartitioning.numPartitions) + val expectedChildrenNumPartitions = + if (nonShuffleChildrenNumPartitions.nonEmpty) { + if (nonShuffleChildrenNumPartitions.length == childrenIndexes.length) { + // Here we pick the max number of partitions among these non-shuffle children. + nonShuffleChildrenNumPartitions.max + } else { + // Here we pick the max number of partitions among these non-shuffle children as the + // expected number of shuffle partitions. However, if it's smaller than + // `conf.numShufflePartitions`, we pick `conf.numShufflePartitions` as the + // expected number of shuffle partitions. + math.max(nonShuffleChildrenNumPartitions.max, conf.defaultNumShufflePartitions) + } + } else { + childrenNumPartitions.max + } + + val targetNumPartitions = requiredNumPartitions.getOrElse(expectedChildrenNumPartitions) + + children = children.zip(requiredChildDistributions).zipWithIndex.map { + case ((child, distribution), index) if childrenIndexes.contains(index) => + if (child.outputPartitioning.numPartitions == targetNumPartitions) { + child + } else { + val defaultPartitioning = distribution.createPartitioning(targetNumPartitions) + child match { + // If child is an exchange, we replace it with a new one having defaultPartitioning. + case ShuffleExchangeExec(_, c, _) => ShuffleExchangeExec(defaultPartitioning, c) + case _ => ShuffleExchangeExec(defaultPartitioning, child) + } + } + + case ((child, _), _) => child + } + } + + // Now that we've performed any necessary shuffles, add sorts to guarantee output orderings: + children = children.zip(requiredChildOrderings).map { case (child, requiredOrdering) => + // If child.outputOrdering already satisfies the requiredOrdering, we do not need to sort. + if (SortOrder.orderingSatisfies(child.outputOrdering, requiredOrdering)) { + child + } else { + SortExec(requiredOrdering, global = false, child = child) + } + } + + operator.withNewChildren(children) + } + + def apply(plan: SparkPlan): SparkPlan = plan.transformUp { + case operator: SparkPlan => + ensureDistributionAndOrdering(operator) + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/KyuubiQueryStagePreparation.scala b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/KyuubiQueryStagePreparation.scala new file mode 100644 index 000000000..a7fcbecd4 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/KyuubiQueryStagePreparation.scala @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.execution.SparkPlan +import org.apache.spark.sql.execution.adaptive.QueryStageExec +import org.apache.spark.sql.execution.command.{ResetCommand, SetCommand} +import org.apache.spark.sql.execution.exchange.{BroadcastExchangeLike, ReusedExchangeExec, ShuffleExchangeLike} +import org.apache.spark.sql.internal.SQLConf + +import org.apache.kyuubi.sql.KyuubiSQLConf._ + +/** + * This rule split stage into two parts: + * 1. previous stage + * 2. final stage + * For final stage, we can inject extra config. It's useful if we use repartition to optimize + * small files that needs bigger shuffle partition size than previous. + * + * Let's say we have a query with 3 stages, then the logical machine like: + * + * Set/Reset Command -> cleanup previousStage config if user set the spark config. + * Query -> AQE -> stage1 -> preparation (use previousStage to overwrite spark config) + * -> AQE -> stage2 -> preparation (use spark config) + * -> AQE -> stage3 -> preparation (use finalStage config to overwrite spark config, + * store spark config to previousStage.) + * + * An example of the new finalStage config: + * `spark.sql.adaptive.advisoryPartitionSizeInBytes` -> + * `spark.sql.finalStage.adaptive.advisoryPartitionSizeInBytes` + */ +case class FinalStageConfigIsolation(session: SparkSession) extends Rule[SparkPlan] { + import FinalStageConfigIsolation._ + + override def apply(plan: SparkPlan): SparkPlan = { + // this rule has no meaning without AQE + if (!conf.getConf(FINAL_STAGE_CONFIG_ISOLATION) || + !conf.getConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED)) { + return plan + } + + if (isFinalStage(plan)) { + // We can not get the whole plan at query preparation phase to detect if current plan is + // for writing, so we depend on a tag which is been injected at post resolution phase. + // Note: we should still do clean up previous config for non-final stage to avoid such case: + // the first statement is write, but the second statement is query. + if (conf.getConf(FINAL_STAGE_CONFIG_ISOLATION_WRITE_ONLY) && + !WriteUtils.isWrite(session, plan)) { + return plan + } + + // set config for final stage + session.conf.getAll.filter(_._1.startsWith(FINAL_STAGE_CONFIG_PREFIX)).foreach { + case (k, v) => + val sparkConfigKey = s"spark.sql.${k.substring(FINAL_STAGE_CONFIG_PREFIX.length)}" + val previousStageConfigKey = + s"$PREVIOUS_STAGE_CONFIG_PREFIX${k.substring(FINAL_STAGE_CONFIG_PREFIX.length)}" + // store the previous config only if we have not stored, to avoid some query only + // have one stage that will overwrite real config. + if (!session.sessionState.conf.contains(previousStageConfigKey)) { + val originalValue = + if (session.conf.getOption(sparkConfigKey).isDefined) { + session.sessionState.conf.getConfString(sparkConfigKey) + } else { + // the default value of config is None, so we need to use a internal tag + INTERNAL_UNSET_CONFIG_TAG + } + logInfo(s"Store config: $sparkConfigKey to previousStage, " + + s"original value: $originalValue ") + session.sessionState.conf.setConfString(previousStageConfigKey, originalValue) + } + logInfo(s"For final stage: set $sparkConfigKey = $v.") + session.conf.set(sparkConfigKey, v) + } + } else { + // reset config for previous stage + session.conf.getAll.filter(_._1.startsWith(PREVIOUS_STAGE_CONFIG_PREFIX)).foreach { + case (k, v) => + val sparkConfigKey = s"spark.sql.${k.substring(PREVIOUS_STAGE_CONFIG_PREFIX.length)}" + logInfo(s"For previous stage: set $sparkConfigKey = $v.") + if (v == INTERNAL_UNSET_CONFIG_TAG) { + session.conf.unset(sparkConfigKey) + } else { + session.conf.set(sparkConfigKey, v) + } + // unset config so that we do not need to reset configs for every previous stage + session.conf.unset(k) + } + } + + plan + } + + /** + * Currently formula depend on AQE in Spark 3.1.1, not sure it can work in future. + */ + private def isFinalStage(plan: SparkPlan): Boolean = { + var shuffleNum = 0 + var broadcastNum = 0 + var reusedNum = 0 + var queryStageNum = 0 + + def collectNumber(p: SparkPlan): SparkPlan = { + p transform { + case shuffle: ShuffleExchangeLike => + shuffleNum += 1 + shuffle + + case broadcast: BroadcastExchangeLike => + broadcastNum += 1 + broadcast + + case reusedExchangeExec: ReusedExchangeExec => + reusedNum += 1 + reusedExchangeExec + + // query stage is leaf node so we need to transform it manually + // compatible with Spark 3.5: + // SPARK-42101: table cache is a independent query stage, so do not need include it. + case queryStage: QueryStageExec if queryStage.nodeName != "TableCacheQueryStage" => + queryStageNum += 1 + collectNumber(queryStage.plan) + queryStage + } + } + collectNumber(plan) + + if (shuffleNum == 0) { + // we don not care about broadcast stage here since it won't change partition number. + true + } else if (shuffleNum + broadcastNum + reusedNum == queryStageNum) { + true + } else { + false + } + } +} +object FinalStageConfigIsolation { + final val SQL_PREFIX = "spark.sql." + final val FINAL_STAGE_CONFIG_PREFIX = "spark.sql.finalStage." + final val PREVIOUS_STAGE_CONFIG_PREFIX = "spark.sql.previousStage." + final val INTERNAL_UNSET_CONFIG_TAG = "__INTERNAL_UNSET_CONFIG_TAG__" + + def getPreviousStageConfigKey(configKey: String): Option[String] = { + if (configKey.startsWith(SQL_PREFIX)) { + Some(s"$PREVIOUS_STAGE_CONFIG_PREFIX${configKey.substring(SQL_PREFIX.length)}") + } else { + None + } + } +} + +case class FinalStageConfigIsolationCleanRule(session: SparkSession) extends Rule[LogicalPlan] { + import FinalStageConfigIsolation._ + + override def apply(plan: LogicalPlan): LogicalPlan = plan match { + case set @ SetCommand(Some((k, Some(_)))) if k.startsWith(SQL_PREFIX) => + checkAndUnsetPreviousStageConfig(k) + set + + case reset @ ResetCommand(Some(k)) if k.startsWith(SQL_PREFIX) => + checkAndUnsetPreviousStageConfig(k) + reset + + case other => other + } + + private def checkAndUnsetPreviousStageConfig(configKey: String): Unit = { + getPreviousStageConfigKey(configKey).foreach { previousStageConfigKey => + if (session.sessionState.conf.contains(previousStageConfigKey)) { + logInfo(s"For previous stage: unset $previousStageConfigKey") + session.conf.unset(previousStageConfigKey) + } + } + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/KyuubiSQLConf.scala b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/KyuubiSQLConf.scala new file mode 100644 index 000000000..6f45dae12 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/KyuubiSQLConf.scala @@ -0,0 +1,276 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql + +import org.apache.spark.network.util.ByteUnit +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.internal.SQLConf._ + +object KyuubiSQLConf { + + val INSERT_REPARTITION_BEFORE_WRITE = + buildConf("spark.sql.optimizer.insertRepartitionBeforeWrite.enabled") + .doc("Add repartition node at the top of query plan. An approach of merging small files.") + .version("1.2.0") + .booleanConf + .createWithDefault(true) + + val INSERT_REPARTITION_NUM = + buildConf("spark.sql.optimizer.insertRepartitionNum") + .doc(s"The partition number if ${INSERT_REPARTITION_BEFORE_WRITE.key} is enabled. " + + s"If AQE is disabled, the default value is ${SQLConf.SHUFFLE_PARTITIONS.key}. " + + "If AQE is enabled, the default value is none that means depend on AQE. " + + "This config is used for Spark 3.1 only.") + .version("1.2.0") + .intConf + .createOptional + + val DYNAMIC_PARTITION_INSERTION_REPARTITION_NUM = + buildConf("spark.sql.optimizer.dynamicPartitionInsertionRepartitionNum") + .doc(s"The partition number of each dynamic partition if " + + s"${INSERT_REPARTITION_BEFORE_WRITE.key} is enabled. " + + "We will repartition by dynamic partition columns to reduce the small file but that " + + "can cause data skew. This config is to extend the partition of dynamic " + + "partition column to avoid skew but may generate some small files.") + .version("1.2.0") + .intConf + .createWithDefault(100) + + val FORCE_SHUFFLE_BEFORE_JOIN = + buildConf("spark.sql.optimizer.forceShuffleBeforeJoin.enabled") + .doc("Ensure shuffle node exists before shuffled join (shj and smj) to make AQE " + + "`OptimizeSkewedJoin` works (complex scenario join, multi table join).") + .version("1.2.0") + .booleanConf + .createWithDefault(false) + + val FINAL_STAGE_CONFIG_ISOLATION = + buildConf("spark.sql.optimizer.finalStageConfigIsolation.enabled") + .doc("If true, the final stage support use different config with previous stage. " + + "The prefix of final stage config key should be `spark.sql.finalStage.`." + + "For example, the raw spark config: `spark.sql.adaptive.advisoryPartitionSizeInBytes`, " + + "then the final stage config should be: " + + "`spark.sql.finalStage.adaptive.advisoryPartitionSizeInBytes`.") + .version("1.2.0") + .booleanConf + .createWithDefault(false) + + val SQL_CLASSIFICATION = "spark.sql.analyzer.classification" + val SQL_CLASSIFICATION_ENABLED = + buildConf("spark.sql.analyzer.classification.enabled") + .doc("When true, allows Kyuubi engine to judge this SQL's classification " + + s"and set `$SQL_CLASSIFICATION` back into sessionConf. " + + "Through this configuration item, Spark can optimizing configuration dynamic") + .version("1.4.0") + .booleanConf + .createWithDefault(false) + + val INSERT_ZORDER_BEFORE_WRITING = + buildConf("spark.sql.optimizer.insertZorderBeforeWriting.enabled") + .doc("When true, we will follow target table properties to insert zorder or not. " + + "The key properties are: 1) kyuubi.zorder.enabled; if this property is true, we will " + + "insert zorder before writing data. 2) kyuubi.zorder.cols; string split by comma, we " + + "will zorder by these cols.") + .version("1.4.0") + .booleanConf + .createWithDefault(true) + + val ZORDER_GLOBAL_SORT_ENABLED = + buildConf("spark.sql.optimizer.zorderGlobalSort.enabled") + .doc("When true, we do a global sort using zorder. Note that, it can cause data skew " + + "issue if the zorder columns have less cardinality. When false, we only do local sort " + + "using zorder.") + .version("1.4.0") + .booleanConf + .createWithDefault(true) + + val REBALANCE_BEFORE_ZORDER = + buildConf("spark.sql.optimizer.rebalanceBeforeZorder.enabled") + .doc("When true, we do a rebalance before zorder in case data skew. " + + "Note that, if the insertion is dynamic partition we will use the partition " + + "columns to rebalance. Note that, this config only affects with Spark 3.3.x") + .version("1.6.0") + .booleanConf + .createWithDefault(false) + + val REBALANCE_ZORDER_COLUMNS_ENABLED = + buildConf("spark.sql.optimizer.rebalanceZorderColumns.enabled") + .doc(s"When true and ${REBALANCE_BEFORE_ZORDER.key} is true, we do rebalance before " + + s"Z-Order. If it's dynamic partition insert, the rebalance expression will include " + + s"both partition columns and Z-Order columns. Note that, this config only " + + s"affects with Spark 3.3.x") + .version("1.6.0") + .booleanConf + .createWithDefault(false) + + val TWO_PHASE_REBALANCE_BEFORE_ZORDER = + buildConf("spark.sql.optimizer.twoPhaseRebalanceBeforeZorder.enabled") + .doc(s"When true and ${REBALANCE_BEFORE_ZORDER.key} is true, we do two phase rebalance " + + s"before Z-Order for the dynamic partition write. The first phase rebalance using " + + s"dynamic partition column; The second phase rebalance using dynamic partition column + " + + s"Z-Order columns. Note that, this config only affects with Spark 3.3.x") + .version("1.6.0") + .booleanConf + .createWithDefault(false) + + val ZORDER_USING_ORIGINAL_ORDERING_ENABLED = + buildConf("spark.sql.optimizer.zorderUsingOriginalOrdering.enabled") + .doc(s"When true and ${REBALANCE_BEFORE_ZORDER.key} is true, we do sort by " + + s"the original ordering i.e. lexicographical order. Note that, this config only " + + s"affects with Spark 3.3.x") + .version("1.6.0") + .booleanConf + .createWithDefault(false) + + val WATCHDOG_MAX_PARTITIONS = + buildConf("spark.sql.watchdog.maxPartitions") + .doc("Set the max partition number when spark scans a data source. " + + "Enable maxPartitions Strategy by specifying this configuration. " + + "Add maxPartitions Strategy to avoid scan excessive partitions " + + "on partitioned table, it's optional that works with defined") + .version("1.4.0") + .intConf + .createOptional + + val WATCHDOG_MAX_FILE_SIZE = + buildConf("spark.sql.watchdog.maxFileSize") + .doc("Set the maximum size in bytes of files when spark scans a data source. " + + "Enable maxFileSize Strategy by specifying this configuration. " + + "Add maxFileSize Strategy to avoid scan excessive size of files," + + " it's optional that works with defined") + .version("1.8.0") + .bytesConf(ByteUnit.BYTE) + .createOptional + + val WATCHDOG_FORCED_MAXOUTPUTROWS = + buildConf("spark.sql.watchdog.forcedMaxOutputRows") + .doc("Add ForcedMaxOutputRows rule to avoid huge output rows of non-limit query " + + "unexpectedly, it's optional that works with defined") + .version("1.4.0") + .intConf + .createOptional + + val DROP_IGNORE_NONEXISTENT = + buildConf("spark.sql.optimizer.dropIgnoreNonExistent") + .doc("Do not report an error if DROP DATABASE/TABLE/VIEW/FUNCTION/PARTITION specifies " + + "a non-existent database/table/view/function/partition") + .version("1.5.0") + .booleanConf + .createWithDefault(false) + + val INFER_REBALANCE_AND_SORT_ORDERS = + buildConf("spark.sql.optimizer.inferRebalanceAndSortOrders.enabled") + .doc("When ture, infer columns for rebalance and sort orders from original query, " + + "e.g. the join keys from join. It can avoid compression ratio regression.") + .version("1.7.0") + .booleanConf + .createWithDefault(false) + + val INFER_REBALANCE_AND_SORT_ORDERS_MAX_COLUMNS = + buildConf("spark.sql.optimizer.inferRebalanceAndSortOrdersMaxColumns") + .doc("The max columns of inferred columns.") + .version("1.7.0") + .intConf + .checkValue(_ > 0, "must be positive number") + .createWithDefault(3) + + val INSERT_REPARTITION_BEFORE_WRITE_IF_NO_SHUFFLE = + buildConf("spark.sql.optimizer.insertRepartitionBeforeWriteIfNoShuffle.enabled") + .doc("When true, add repartition even if the original plan does not have shuffle.") + .version("1.7.0") + .booleanConf + .createWithDefault(false) + + val FINAL_STAGE_CONFIG_ISOLATION_WRITE_ONLY = + buildConf("spark.sql.optimizer.finalStageConfigIsolationWriteOnly.enabled") + .doc("When true, only enable final stage isolation for writing.") + .version("1.7.0") + .booleanConf + .createWithDefault(true) + + val FINAL_WRITE_STAGE_EAGERLY_KILL_EXECUTORS_ENABLED = + buildConf("spark.sql.finalWriteStage.eagerlyKillExecutors.enabled") + .doc("When true, eagerly kill redundant executors before running final write stage.") + .version("1.8.0") + .booleanConf + .createWithDefault(false) + + val FINAL_WRITE_STAGE_EAGERLY_KILL_EXECUTORS_KILL_ALL = + buildConf("spark.sql.finalWriteStage.eagerlyKillExecutors.killAll") + .doc("When true, eagerly kill all executors before running final write stage. " + + "Mainly for test.") + .version("1.8.0") + .booleanConf + .createWithDefault(false) + + val FINAL_WRITE_STAGE_SKIP_KILLING_EXECUTORS_FOR_TABLE_CACHE = + buildConf("spark.sql.finalWriteStage.skipKillingExecutorsForTableCache") + .doc("When true, skip killing executors if the plan has table caches.") + .version("1.8.0") + .booleanConf + .createWithDefault(true) + + val FINAL_WRITE_STAGE_PARTITION_FACTOR = + buildConf("spark.sql.finalWriteStage.retainExecutorsFactor") + .doc("If the target executors * factor < active executors, and " + + "target executors * factor > min executors, then kill redundant executors.") + .version("1.8.0") + .doubleConf + .checkValue(_ >= 1, "must be bigger than or equal to 1") + .createWithDefault(1.2) + + val FINAL_WRITE_STAGE_RESOURCE_ISOLATION_ENABLED = + buildConf("spark.sql.finalWriteStage.resourceIsolation.enabled") + .doc( + "When true, make final write stage resource isolation using custom RDD resource profile.") + .version("1.8.0") + .booleanConf + .createWithDefault(false) + + val FINAL_WRITE_STAGE_EXECUTOR_CORES = + buildConf("spark.sql.finalWriteStage.executorCores") + .doc("Specify the executor core request for final write stage. " + + "It would be passed to the RDD resource profile.") + .version("1.8.0") + .intConf + .createOptional + + val FINAL_WRITE_STAGE_EXECUTOR_MEMORY = + buildConf("spark.sql.finalWriteStage.executorMemory") + .doc("Specify the executor on heap memory request for final write stage. " + + "It would be passed to the RDD resource profile.") + .version("1.8.0") + .stringConf + .createOptional + + val FINAL_WRITE_STAGE_EXECUTOR_MEMORY_OVERHEAD = + buildConf("spark.sql.finalWriteStage.executorMemoryOverhead") + .doc("Specify the executor memory overhead request for final write stage. " + + "It would be passed to the RDD resource profile.") + .version("1.8.0") + .stringConf + .createOptional + + val FINAL_WRITE_STAGE_EXECUTOR_OFF_HEAP_MEMORY = + buildConf("spark.sql.finalWriteStage.executorOffHeapMemory") + .doc("Specify the executor off heap memory request for final write stage. " + + "It would be passed to the RDD resource profile.") + .version("1.8.0") + .stringConf + .createOptional +} diff --git a/kyuubi-server/web-ui/src/pinia/modules/layout.ts b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/KyuubiSQLExtensionException.scala similarity index 74% rename from kyuubi-server/web-ui/src/pinia/modules/layout.ts rename to extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/KyuubiSQLExtensionException.scala index b8743a86e..88c5a988f 100644 --- a/kyuubi-server/web-ui/src/pinia/modules/layout.ts +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/KyuubiSQLExtensionException.scala @@ -15,18 +15,14 @@ * limitations under the License. */ -import { defineStore } from 'pinia' -import { ref } from 'vue' +package org.apache.kyuubi.sql -export const useStore = defineStore('aside', () => { - const isCollapse = ref(false) +import java.sql.SQLException - function changeCollapse() { - isCollapse.value = !isCollapse.value - } +class KyuubiSQLExtensionException(reason: String, cause: Throwable) + extends SQLException(reason, cause) { - return { - isCollapse, - changeCollapse + def this(reason: String) = { + this(reason, null) } -}) +} diff --git a/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLAstBuilder.scala b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLAstBuilder.scala new file mode 100644 index 000000000..cc00bf88e --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLAstBuilder.scala @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql + +import scala.collection.JavaConverters.asScalaBufferConverter +import scala.collection.mutable.ListBuffer + +import org.antlr.v4.runtime.ParserRuleContext +import org.antlr.v4.runtime.misc.Interval +import org.antlr.v4.runtime.tree.ParseTree +import org.apache.spark.sql.catalyst.SQLConfHelper +import org.apache.spark.sql.catalyst.analysis.{UnresolvedAttribute, UnresolvedRelation, UnresolvedStar} +import org.apache.spark.sql.catalyst.expressions._ +import org.apache.spark.sql.catalyst.parser.ParserUtils.withOrigin +import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, Project, Sort} + +import org.apache.kyuubi.sql.KyuubiSparkSQLParser._ +import org.apache.kyuubi.sql.zorder.{OptimizeZorderStatement, Zorder} + +class KyuubiSparkSQLAstBuilder extends KyuubiSparkSQLBaseVisitor[AnyRef] with SQLConfHelper { + + def buildOptimizeStatement( + unparsedPredicateOptimize: UnparsedPredicateOptimize, + parseExpression: String => Expression): LogicalPlan = { + + val UnparsedPredicateOptimize(tableIdent, tablePredicate, orderExpr) = + unparsedPredicateOptimize + + val predicate = tablePredicate.map(parseExpression) + verifyPartitionPredicates(predicate) + val table = UnresolvedRelation(tableIdent) + val tableWithFilter = predicate match { + case Some(expr) => Filter(expr, table) + case None => table + } + val query = + Sort( + SortOrder(orderExpr, Ascending, NullsLast, Seq.empty) :: Nil, + conf.getConf(KyuubiSQLConf.ZORDER_GLOBAL_SORT_ENABLED), + Project(Seq(UnresolvedStar(None)), tableWithFilter)) + OptimizeZorderStatement(tableIdent, query) + } + + private def verifyPartitionPredicates(predicates: Option[Expression]): Unit = { + predicates.foreach { + case p if !isLikelySelective(p) => + throw new KyuubiSQLExtensionException(s"unsupported partition predicates: ${p.sql}") + case _ => + } + } + + /** + * Forked from Apache Spark's org.apache.spark.sql.catalyst.expressions.PredicateHelper + * The `PredicateHelper.isLikelySelective()` is available since Spark-3.3, forked for Spark + * that is lower than 3.3. + * + * Returns whether an expression is likely to be selective + */ + private def isLikelySelective(e: Expression): Boolean = e match { + case Not(expr) => isLikelySelective(expr) + case And(l, r) => isLikelySelective(l) || isLikelySelective(r) + case Or(l, r) => isLikelySelective(l) && isLikelySelective(r) + case _: StringRegexExpression => true + case _: BinaryComparison => true + case _: In | _: InSet => true + case _: StringPredicate => true + case BinaryPredicate(_) => true + case _: MultiLikeBase => true + case _ => false + } + + private object BinaryPredicate { + def unapply(expr: Expression): Option[Expression] = expr match { + case _: Contains => Option(expr) + case _: StartsWith => Option(expr) + case _: EndsWith => Option(expr) + case _ => None + } + } + + /** + * Create an expression from the given context. This method just passes the context on to the + * visitor and only takes care of typing (We assume that the visitor returns an Expression here). + */ + protected def expression(ctx: ParserRuleContext): Expression = typedVisit(ctx) + + protected def multiPart(ctx: ParserRuleContext): Seq[String] = typedVisit(ctx) + + override def visitSingleStatement(ctx: SingleStatementContext): LogicalPlan = { + visit(ctx.statement()).asInstanceOf[LogicalPlan] + } + + override def visitOptimizeZorder( + ctx: OptimizeZorderContext): UnparsedPredicateOptimize = withOrigin(ctx) { + val tableIdent = multiPart(ctx.multipartIdentifier()) + + val predicate = Option(ctx.whereClause()) + .map(_.partitionPredicate) + .map(extractRawText(_)) + + val zorderCols = ctx.zorderClause().order.asScala + .map(visitMultipartIdentifier) + .map(UnresolvedAttribute(_)) + .toSeq + + val orderExpr = + if (zorderCols.length == 1) { + zorderCols.head + } else { + Zorder(zorderCols) + } + UnparsedPredicateOptimize(tableIdent, predicate, orderExpr) + } + + override def visitPassThrough(ctx: PassThroughContext): LogicalPlan = null + + override def visitMultipartIdentifier(ctx: MultipartIdentifierContext): Seq[String] = + withOrigin(ctx) { + ctx.parts.asScala.map(_.getText).toSeq + } + + override def visitZorderClause(ctx: ZorderClauseContext): Seq[UnresolvedAttribute] = + withOrigin(ctx) { + val res = ListBuffer[UnresolvedAttribute]() + ctx.multipartIdentifier().forEach { identifier => + res += UnresolvedAttribute(identifier.parts.asScala.map(_.getText).toSeq) + } + res.toSeq + } + + private def typedVisit[T](ctx: ParseTree): T = { + ctx.accept(this).asInstanceOf[T] + } + + private def extractRawText(exprContext: ParserRuleContext): String = { + // Extract the raw expression which will be parsed later + exprContext.getStart.getInputStream.getText(new Interval( + exprContext.getStart.getStartIndex, + exprContext.getStop.getStopIndex)) + } +} + +/** + * a logical plan contains an unparsed expression that will be parsed by spark. + */ +trait UnparsedExpressionLogicalPlan extends LogicalPlan { + override def output: Seq[Attribute] = throw new UnsupportedOperationException() + + override def children: Seq[LogicalPlan] = throw new UnsupportedOperationException() + + protected def withNewChildrenInternal( + newChildren: IndexedSeq[LogicalPlan]): LogicalPlan = + throw new UnsupportedOperationException() +} + +case class UnparsedPredicateOptimize( + tableIdent: Seq[String], + tablePredicate: Option[String], + orderExpr: Expression) extends UnparsedExpressionLogicalPlan {} diff --git a/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLCommonExtension.scala b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLCommonExtension.scala new file mode 100644 index 000000000..f39ad3cc3 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLCommonExtension.scala @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql + +import org.apache.spark.sql.SparkSessionExtensions + +import org.apache.kyuubi.sql.zorder.{InsertZorderBeforeWritingDatasource33, InsertZorderBeforeWritingHive33, ResolveZorder} + +class KyuubiSparkSQLCommonExtension extends (SparkSessionExtensions => Unit) { + override def apply(extensions: SparkSessionExtensions): Unit = { + KyuubiSparkSQLCommonExtension.injectCommonExtensions(extensions) + } +} + +object KyuubiSparkSQLCommonExtension { + def injectCommonExtensions(extensions: SparkSessionExtensions): Unit = { + // inject zorder parser and related rules + extensions.injectParser { case (_, parser) => new SparkKyuubiSparkSQLParser(parser) } + extensions.injectResolutionRule(ResolveZorder) + + // Note that: + // InsertZorderBeforeWritingDatasource and InsertZorderBeforeWritingHive + // should be applied before + // RepartitionBeforeWriting and RebalanceBeforeWriting + // because we can only apply one of them (i.e. Global Sort or Repartition/Rebalance) + extensions.injectPostHocResolutionRule(InsertZorderBeforeWritingDatasource33) + extensions.injectPostHocResolutionRule(InsertZorderBeforeWritingHive33) + extensions.injectPostHocResolutionRule(FinalStageConfigIsolationCleanRule) + + extensions.injectQueryStagePrepRule(_ => InsertShuffleNodeBeforeJoin) + + extensions.injectQueryStagePrepRule(FinalStageConfigIsolation(_)) + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLExtension.scala b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLExtension.scala new file mode 100644 index 000000000..792315d89 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLExtension.scala @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql + +import org.apache.spark.sql.{FinalStageResourceManager, InjectCustomResourceProfile, SparkSessionExtensions} + +import org.apache.kyuubi.sql.watchdog.{ForcedMaxOutputRowsRule, MaxScanStrategy} + +// scalastyle:off line.size.limit +/** + * Depend on Spark SQL Extension framework, we can use this extension follow steps + * 1. move this jar into $SPARK_HOME/jars + * 2. add config into `spark-defaults.conf`: `spark.sql.extensions=org.apache.kyuubi.sql.KyuubiSparkSQLExtension` + */ +// scalastyle:on line.size.limit +class KyuubiSparkSQLExtension extends (SparkSessionExtensions => Unit) { + override def apply(extensions: SparkSessionExtensions): Unit = { + KyuubiSparkSQLCommonExtension.injectCommonExtensions(extensions) + + extensions.injectPostHocResolutionRule(RebalanceBeforeWritingDatasource) + extensions.injectPostHocResolutionRule(RebalanceBeforeWritingHive) + extensions.injectPostHocResolutionRule(DropIgnoreNonexistent) + + // watchdog extension + extensions.injectOptimizerRule(ForcedMaxOutputRowsRule) + extensions.injectPlannerStrategy(MaxScanStrategy) + + extensions.injectQueryStagePrepRule(FinalStageResourceManager(_)) + extensions.injectQueryStagePrepRule(InjectCustomResourceProfile) + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLParser.scala b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLParser.scala new file mode 100644 index 000000000..c4418c33c --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLParser.scala @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql + +import org.antlr.v4.runtime._ +import org.antlr.v4.runtime.atn.PredictionMode +import org.antlr.v4.runtime.misc.{Interval, ParseCancellationException} +import org.apache.spark.sql.AnalysisException +import org.apache.spark.sql.catalyst.{FunctionIdentifier, SQLConfHelper, TableIdentifier} +import org.apache.spark.sql.catalyst.expressions.Expression +import org.apache.spark.sql.catalyst.parser.{ParseErrorListener, ParseException, ParserInterface, PostProcessor} +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.catalyst.trees.Origin +import org.apache.spark.sql.types.{DataType, StructType} + +abstract class KyuubiSparkSQLParserBase extends ParserInterface with SQLConfHelper { + def delegate: ParserInterface + def astBuilder: KyuubiSparkSQLAstBuilder + + override def parsePlan(sqlText: String): LogicalPlan = parse(sqlText) { parser => + astBuilder.visit(parser.singleStatement()) match { + case optimize: UnparsedPredicateOptimize => + astBuilder.buildOptimizeStatement(optimize, delegate.parseExpression) + case plan: LogicalPlan => plan + case _ => delegate.parsePlan(sqlText) + } + } + + protected def parse[T](command: String)(toResult: KyuubiSparkSQLParser => T): T = { + val lexer = new KyuubiSparkSQLLexer( + new UpperCaseCharStream(CharStreams.fromString(command))) + lexer.removeErrorListeners() + lexer.addErrorListener(ParseErrorListener) + + val tokenStream = new CommonTokenStream(lexer) + val parser = new KyuubiSparkSQLParser(tokenStream) + parser.addParseListener(PostProcessor) + parser.removeErrorListeners() + parser.addErrorListener(ParseErrorListener) + + try { + try { + // first, try parsing with potentially faster SLL mode + parser.getInterpreter.setPredictionMode(PredictionMode.SLL) + toResult(parser) + } catch { + case _: ParseCancellationException => + // if we fail, parse with LL mode + tokenStream.seek(0) // rewind input stream + parser.reset() + + // Try Again. + parser.getInterpreter.setPredictionMode(PredictionMode.LL) + toResult(parser) + } + } catch { + case e: ParseException if e.command.isDefined => + throw e + case e: ParseException => + throw e.withCommand(command) + case e: AnalysisException => + val position = Origin(e.line, e.startPosition) + throw new ParseException(Option(command), e.message, position, position) + } + } + + override def parseExpression(sqlText: String): Expression = { + delegate.parseExpression(sqlText) + } + + override def parseTableIdentifier(sqlText: String): TableIdentifier = { + delegate.parseTableIdentifier(sqlText) + } + + override def parseFunctionIdentifier(sqlText: String): FunctionIdentifier = { + delegate.parseFunctionIdentifier(sqlText) + } + + override def parseMultipartIdentifier(sqlText: String): Seq[String] = { + delegate.parseMultipartIdentifier(sqlText) + } + + override def parseTableSchema(sqlText: String): StructType = { + delegate.parseTableSchema(sqlText) + } + + override def parseDataType(sqlText: String): DataType = { + delegate.parseDataType(sqlText) + } + + /** + * This functions was introduced since spark-3.3, for more details, please see + * https://github.com/apache/spark/pull/34543 + */ + override def parseQuery(sqlText: String): LogicalPlan = { + delegate.parseQuery(sqlText) + } +} + +class SparkKyuubiSparkSQLParser( + override val delegate: ParserInterface) + extends KyuubiSparkSQLParserBase { + def astBuilder: KyuubiSparkSQLAstBuilder = new KyuubiSparkSQLAstBuilder +} + +/* Copied from Apache Spark's to avoid dependency on Spark Internals */ +class UpperCaseCharStream(wrapped: CodePointCharStream) extends CharStream { + override def consume(): Unit = wrapped.consume() + override def getSourceName(): String = wrapped.getSourceName + override def index(): Int = wrapped.index + override def mark(): Int = wrapped.mark + override def release(marker: Int): Unit = wrapped.release(marker) + override def seek(where: Int): Unit = wrapped.seek(where) + override def size(): Int = wrapped.size + + override def getText(interval: Interval): String = wrapped.getText(interval) + + // scalastyle:off + override def LA(i: Int): Int = { + val la = wrapped.LA(i) + if (la == 0 || la == IntStream.EOF) la + else Character.toUpperCase(la) + } + // scalastyle:on +} diff --git a/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/RebalanceBeforeWriting.scala b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/RebalanceBeforeWriting.scala new file mode 100644 index 000000000..3cbacdd2f --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/RebalanceBeforeWriting.scala @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.expressions.{Ascending, Attribute, SortOrder} +import org.apache.spark.sql.catalyst.plans.logical._ + +trait RepartitionBuilderWithRebalance extends RepartitionBuilder { + override def buildRepartition( + dynamicPartitionColumns: Seq[Attribute], + query: LogicalPlan): LogicalPlan = { + if (!conf.getConf(KyuubiSQLConf.INFER_REBALANCE_AND_SORT_ORDERS) || + dynamicPartitionColumns.nonEmpty) { + RebalancePartitions(dynamicPartitionColumns, query) + } else { + val maxColumns = conf.getConf(KyuubiSQLConf.INFER_REBALANCE_AND_SORT_ORDERS_MAX_COLUMNS) + val inferred = InferRebalanceAndSortOrders.infer(query) + if (inferred.isDefined) { + val (partitioning, ordering) = inferred.get + val rebalance = RebalancePartitions(partitioning.take(maxColumns), query) + if (ordering.nonEmpty) { + val sortOrders = ordering.take(maxColumns).map(o => SortOrder(o, Ascending)) + Sort(sortOrders, false, rebalance) + } else { + rebalance + } + } else { + RebalancePartitions(dynamicPartitionColumns, query) + } + } + } + + override def canInsertRepartitionByExpression(plan: LogicalPlan): Boolean = { + super.canInsertRepartitionByExpression(plan) && { + plan match { + case _: RebalancePartitions => false + case _ => true + } + } + } +} + +/** + * For datasource table, there two commands can write data to table + * 1. InsertIntoHadoopFsRelationCommand + * 2. CreateDataSourceTableAsSelectCommand + * This rule add a RebalancePartitions node between write and query + */ +case class RebalanceBeforeWritingDatasource(session: SparkSession) + extends RepartitionBeforeWritingDatasourceBase + with RepartitionBuilderWithRebalance {} + +/** + * For Hive table, there two commands can write data to table + * 1. InsertIntoHiveTable + * 2. CreateHiveTableAsSelectCommand + * This rule add a RebalancePartitions node between write and query + */ +case class RebalanceBeforeWritingHive(session: SparkSession) + extends RepartitionBeforeWritingHiveBase + with RepartitionBuilderWithRebalance {} diff --git a/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/RepartitionBeforeWritingBase.scala b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/RepartitionBeforeWritingBase.scala new file mode 100644 index 000000000..3ebb9740f --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/RepartitionBeforeWritingBase.scala @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql + +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.catalyst.plans.logical._ +import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand +import org.apache.spark.sql.hive.execution.InsertIntoHiveTable +import org.apache.spark.sql.internal.StaticSQLConf + +trait RepartitionBuilder extends Rule[LogicalPlan] with RepartitionBeforeWriteHelper { + def buildRepartition( + dynamicPartitionColumns: Seq[Attribute], + query: LogicalPlan): LogicalPlan +} + +/** + * For datasource table, there two commands can write data to table + * 1. InsertIntoHadoopFsRelationCommand + * 2. CreateDataSourceTableAsSelectCommand + * This rule add a repartition node between write and query + */ +abstract class RepartitionBeforeWritingDatasourceBase extends RepartitionBuilder { + + override def apply(plan: LogicalPlan): LogicalPlan = { + if (conf.getConf(KyuubiSQLConf.INSERT_REPARTITION_BEFORE_WRITE)) { + addRepartition(plan) + } else { + plan + } + } + + private def addRepartition(plan: LogicalPlan): LogicalPlan = plan match { + case i @ InsertIntoHadoopFsRelationCommand(_, sp, _, pc, bucket, _, _, query, _, _, _, _) + if query.resolved && bucket.isEmpty && canInsertRepartitionByExpression(query) => + val dynamicPartitionColumns = pc.filterNot(attr => sp.contains(attr.name)) + i.copy(query = buildRepartition(dynamicPartitionColumns, query)) + + case u @ Union(children, _, _) => + u.copy(children = children.map(addRepartition)) + + case _ => plan + } +} + +/** + * For Hive table, there two commands can write data to table + * 1. InsertIntoHiveTable + * 2. CreateHiveTableAsSelectCommand + * This rule add a repartition node between write and query + */ +abstract class RepartitionBeforeWritingHiveBase extends RepartitionBuilder { + override def apply(plan: LogicalPlan): LogicalPlan = { + if (conf.getConf(StaticSQLConf.CATALOG_IMPLEMENTATION) == "hive" && + conf.getConf(KyuubiSQLConf.INSERT_REPARTITION_BEFORE_WRITE)) { + addRepartition(plan) + } else { + plan + } + } + + def addRepartition(plan: LogicalPlan): LogicalPlan = plan match { + case i @ InsertIntoHiveTable(table, partition, query, _, _, _, _, _, _, _, _) + if query.resolved && table.bucketSpec.isEmpty && canInsertRepartitionByExpression(query) => + val dynamicPartitionColumns = partition.filter(_._2.isEmpty).keys + .flatMap(name => query.output.find(_.name == name)).toSeq + i.copy(query = buildRepartition(dynamicPartitionColumns, query)) + + case u @ Union(children, _, _) => + u.copy(children = children.map(addRepartition)) + + case _ => plan + } +} + +trait RepartitionBeforeWriteHelper extends Rule[LogicalPlan] { + private def hasBenefit(plan: LogicalPlan): Boolean = { + def probablyHasShuffle: Boolean = plan.find { + case _: Join => true + case _: Aggregate => true + case _: Distinct => true + case _: Deduplicate => true + case _: Window => true + case s: Sort if s.global => true + case _: RepartitionOperation => true + case _: GlobalLimit => true + case _ => false + }.isDefined + + conf.getConf(KyuubiSQLConf.INSERT_REPARTITION_BEFORE_WRITE_IF_NO_SHUFFLE) || probablyHasShuffle + } + + def canInsertRepartitionByExpression(plan: LogicalPlan): Boolean = { + def canInsert(p: LogicalPlan): Boolean = p match { + case Project(_, child) => canInsert(child) + case SubqueryAlias(_, child) => canInsert(child) + case Limit(_, _) => false + case _: Sort => false + case _: RepartitionByExpression => false + case _: Repartition => false + case _ => true + } + + // 1. make sure AQE is enabled, otherwise it is no meaning to add a shuffle + // 2. make sure it does not break the semantics of original plan + // 3. try to avoid adding a shuffle if it has potential performance regression + conf.adaptiveExecutionEnabled && canInsert(plan) && hasBenefit(plan) + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/WriteUtils.scala b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/WriteUtils.scala new file mode 100644 index 000000000..89dd83194 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/WriteUtils.scala @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.execution.{SparkPlan, UnionExec} +import org.apache.spark.sql.execution.command.DataWritingCommandExec +import org.apache.spark.sql.execution.datasources.v2.V2TableWriteExec + +object WriteUtils { + def isWrite(session: SparkSession, plan: SparkPlan): Boolean = { + plan match { + case _: DataWritingCommandExec => true + case _: V2TableWriteExec => true + case u: UnionExec if u.children.nonEmpty => u.children.forall(isWrite(session, _)) + case _ => false + } + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/watchdog/ForcedMaxOutputRowsBase.scala b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/watchdog/ForcedMaxOutputRowsBase.scala new file mode 100644 index 000000000..4f897d1b6 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/watchdog/ForcedMaxOutputRowsBase.scala @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql.watchdog + +import org.apache.spark.sql.catalyst.analysis.MultiInstanceRelation +import org.apache.spark.sql.catalyst.dsl.expressions._ +import org.apache.spark.sql.catalyst.expressions.Alias +import org.apache.spark.sql.catalyst.plans.logical._ +import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.execution.command.DataWritingCommand + +import org.apache.kyuubi.sql.KyuubiSQLConf + +/* + * Add ForcedMaxOutputRows rule for output rows limitation + * to avoid huge output rows of non_limit query unexpectedly + * mainly applied to cases as below: + * + * case 1: + * {{{ + * SELECT [c1, c2, ...] + * }}} + * + * case 2: + * {{{ + * WITH CTE AS ( + * ...) + * SELECT [c1, c2, ...] FROM CTE ... + * }}} + * + * The Logical Rule add a GlobalLimit node before root project + * */ +trait ForcedMaxOutputRowsBase extends Rule[LogicalPlan] { + + protected def isChildAggregate(a: Aggregate): Boolean + + protected def canInsertLimitInner(p: LogicalPlan): Boolean = p match { + case Aggregate(_, Alias(_, "havingCondition") :: Nil, _) => false + case agg: Aggregate => !isChildAggregate(agg) + case _: RepartitionByExpression => true + case _: Distinct => true + case _: Filter => true + case _: Project => true + case Limit(_, _) => true + case _: Sort => true + case Union(children, _, _) => + if (children.exists(_.isInstanceOf[DataWritingCommand])) { + false + } else { + true + } + case _: MultiInstanceRelation => true + case _: Join => true + case _ => false + } + + protected def canInsertLimit(p: LogicalPlan, maxOutputRowsOpt: Option[Int]): Boolean = { + maxOutputRowsOpt match { + case Some(forcedMaxOutputRows) => canInsertLimitInner(p) && + !p.maxRows.exists(_ <= forcedMaxOutputRows) + case None => false + } + } + + override def apply(plan: LogicalPlan): LogicalPlan = { + val maxOutputRowsOpt = conf.getConf(KyuubiSQLConf.WATCHDOG_FORCED_MAXOUTPUTROWS) + plan match { + case p if p.resolved && canInsertLimit(p, maxOutputRowsOpt) => + Limit( + maxOutputRowsOpt.get, + plan) + case _ => plan + } + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/watchdog/ForcedMaxOutputRowsRule.scala b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/watchdog/ForcedMaxOutputRowsRule.scala new file mode 100644 index 000000000..a3d990b10 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/watchdog/ForcedMaxOutputRowsRule.scala @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql.watchdog + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, CommandResult, LogicalPlan, Union, WithCTE} +import org.apache.spark.sql.execution.command.DataWritingCommand + +case class ForcedMaxOutputRowsRule(sparkSession: SparkSession) extends ForcedMaxOutputRowsBase { + + override protected def isChildAggregate(a: Aggregate): Boolean = false + + override protected def canInsertLimitInner(p: LogicalPlan): Boolean = p match { + case WithCTE(plan, _) => this.canInsertLimitInner(plan) + case plan: LogicalPlan => plan match { + case Union(children, _, _) => !children.exists { + case _: DataWritingCommand => true + case p: CommandResult if p.commandLogicalPlan.isInstanceOf[DataWritingCommand] => true + case _ => false + } + case _ => super.canInsertLimitInner(plan) + } + } + + override protected def canInsertLimit(p: LogicalPlan, maxOutputRowsOpt: Option[Int]): Boolean = { + p match { + case WithCTE(plan, _) => this.canInsertLimit(plan, maxOutputRowsOpt) + case _ => super.canInsertLimit(p, maxOutputRowsOpt) + } + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/watchdog/KyuubiWatchDogException.scala b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/watchdog/KyuubiWatchDogException.scala new file mode 100644 index 000000000..e44309192 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/watchdog/KyuubiWatchDogException.scala @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql.watchdog + +import org.apache.kyuubi.sql.KyuubiSQLExtensionException + +final class MaxPartitionExceedException( + private val reason: String = "", + private val cause: Throwable = None.orNull) + extends KyuubiSQLExtensionException(reason, cause) + +final class MaxFileSizeExceedException( + private val reason: String = "", + private val cause: Throwable = None.orNull) + extends KyuubiSQLExtensionException(reason, cause) diff --git a/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/watchdog/MaxScanStrategy.scala b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/watchdog/MaxScanStrategy.scala new file mode 100644 index 000000000..1ed55ebc2 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/watchdog/MaxScanStrategy.scala @@ -0,0 +1,305 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql.watchdog + +import org.apache.hadoop.fs.Path +import org.apache.spark.sql.{PruneFileSourcePartitionHelper, SparkSession, Strategy} +import org.apache.spark.sql.catalyst.SQLConfHelper +import org.apache.spark.sql.catalyst.catalog.{CatalogTable, HiveTableRelation} +import org.apache.spark.sql.catalyst.planning.ScanOperation +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.execution.SparkPlan +import org.apache.spark.sql.execution.datasources.{CatalogFileIndex, HadoopFsRelation, InMemoryFileIndex, LogicalRelation} +import org.apache.spark.sql.types.StructType + +import org.apache.kyuubi.sql.KyuubiSQLConf + +/** + * Add MaxScanStrategy to avoid scan excessive partitions or files + * 1. Check if scan exceed maxPartition of partitioned table + * 2. Check if scan exceed maxFileSize (calculated by hive table and partition statistics) + * This Strategy Add Planner Strategy after LogicalOptimizer + * @param session + */ +case class MaxScanStrategy(session: SparkSession) + extends Strategy + with SQLConfHelper + with PruneFileSourcePartitionHelper { + override def apply(plan: LogicalPlan): Seq[SparkPlan] = { + val maxScanPartitionsOpt = conf.getConf(KyuubiSQLConf.WATCHDOG_MAX_PARTITIONS) + val maxFileSizeOpt = conf.getConf(KyuubiSQLConf.WATCHDOG_MAX_FILE_SIZE) + if (maxScanPartitionsOpt.isDefined || maxFileSizeOpt.isDefined) { + checkScan(plan, maxScanPartitionsOpt, maxFileSizeOpt) + } + Nil + } + + private def checkScan( + plan: LogicalPlan, + maxScanPartitionsOpt: Option[Int], + maxFileSizeOpt: Option[Long]): Unit = { + plan match { + case ScanOperation(_, _, _, relation: HiveTableRelation) => + if (relation.isPartitioned) { + relation.prunedPartitions match { + case Some(prunedPartitions) => + if (maxScanPartitionsOpt.exists(_ < prunedPartitions.size)) { + throw new MaxPartitionExceedException( + s""" + |SQL job scan hive partition: ${prunedPartitions.size} + |exceed restrict of hive scan maxPartition ${maxScanPartitionsOpt.get} + |You should optimize your SQL logical according partition structure + |or shorten query scope such as p_date, detail as below: + |Table: ${relation.tableMeta.qualifiedName} + |Owner: ${relation.tableMeta.owner} + |Partition Structure: ${relation.partitionCols.map(_.name).mkString(", ")} + |""".stripMargin) + } + lazy val scanFileSize = prunedPartitions.flatMap(_.stats).map(_.sizeInBytes).sum + if (maxFileSizeOpt.exists(_ < scanFileSize)) { + throw partTableMaxFileExceedError( + scanFileSize, + maxFileSizeOpt.get, + Some(relation.tableMeta), + prunedPartitions.flatMap(_.storage.locationUri).map(_.toString), + relation.partitionCols.map(_.name)) + } + case _ => + lazy val scanPartitions: Int = session + .sessionState.catalog.externalCatalog.listPartitionNames( + relation.tableMeta.database, + relation.tableMeta.identifier.table).size + if (maxScanPartitionsOpt.exists(_ < scanPartitions)) { + throw new MaxPartitionExceedException( + s""" + |Your SQL job scan a whole huge table without any partition filter, + |You should optimize your SQL logical according partition structure + |or shorten query scope such as p_date, detail as below: + |Table: ${relation.tableMeta.qualifiedName} + |Owner: ${relation.tableMeta.owner} + |Partition Structure: ${relation.partitionCols.map(_.name).mkString(", ")} + |""".stripMargin) + } + + lazy val scanFileSize: BigInt = + relation.tableMeta.stats.map(_.sizeInBytes).getOrElse { + session + .sessionState.catalog.externalCatalog.listPartitions( + relation.tableMeta.database, + relation.tableMeta.identifier.table).flatMap(_.stats).map(_.sizeInBytes).sum + } + if (maxFileSizeOpt.exists(_ < scanFileSize)) { + throw new MaxFileSizeExceedException( + s""" + |Your SQL job scan a whole huge table without any partition filter, + |You should optimize your SQL logical according partition structure + |or shorten query scope such as p_date, detail as below: + |Table: ${relation.tableMeta.qualifiedName} + |Owner: ${relation.tableMeta.owner} + |Partition Structure: ${relation.partitionCols.map(_.name).mkString(", ")} + |""".stripMargin) + } + } + } else { + lazy val scanFileSize = relation.tableMeta.stats.map(_.sizeInBytes).sum + if (maxFileSizeOpt.exists(_ < scanFileSize)) { + throw nonPartTableMaxFileExceedError( + scanFileSize, + maxFileSizeOpt.get, + Some(relation.tableMeta)) + } + } + case ScanOperation( + _, + _, + filters, + relation @ LogicalRelation( + fsRelation @ HadoopFsRelation( + fileIndex: InMemoryFileIndex, + partitionSchema, + _, + _, + _, + _), + _, + _, + _)) => + if (fsRelation.partitionSchema.nonEmpty) { + val (partitionKeyFilters, dataFilter) = + getPartitionKeyFiltersAndDataFilters( + SparkSession.active, + relation, + partitionSchema, + filters, + relation.output) + val prunedPartitions = fileIndex.listFiles( + partitionKeyFilters.toSeq, + dataFilter) + if (maxScanPartitionsOpt.exists(_ < prunedPartitions.size)) { + throw maxPartitionExceedError( + prunedPartitions.size, + maxScanPartitionsOpt.get, + relation.catalogTable, + fileIndex.rootPaths, + fsRelation.partitionSchema) + } + lazy val scanFileSize = prunedPartitions.flatMap(_.files).map(_.getLen).sum + if (maxFileSizeOpt.exists(_ < scanFileSize)) { + throw partTableMaxFileExceedError( + scanFileSize, + maxFileSizeOpt.get, + relation.catalogTable, + fileIndex.rootPaths.map(_.toString), + fsRelation.partitionSchema.map(_.name)) + } + } else { + lazy val scanFileSize = fileIndex.sizeInBytes + if (maxFileSizeOpt.exists(_ < scanFileSize)) { + throw nonPartTableMaxFileExceedError( + scanFileSize, + maxFileSizeOpt.get, + relation.catalogTable) + } + } + case ScanOperation( + _, + _, + filters, + logicalRelation @ LogicalRelation( + fsRelation @ HadoopFsRelation( + catalogFileIndex: CatalogFileIndex, + partitionSchema, + _, + _, + _, + _), + _, + _, + _)) => + if (fsRelation.partitionSchema.nonEmpty) { + val (partitionKeyFilters, _) = + getPartitionKeyFiltersAndDataFilters( + SparkSession.active, + logicalRelation, + partitionSchema, + filters, + logicalRelation.output) + + val fileIndex = catalogFileIndex.filterPartitions( + partitionKeyFilters.toSeq) + + lazy val prunedPartitionSize = fileIndex.partitionSpec().partitions.size + if (maxScanPartitionsOpt.exists(_ < prunedPartitionSize)) { + throw maxPartitionExceedError( + prunedPartitionSize, + maxScanPartitionsOpt.get, + logicalRelation.catalogTable, + catalogFileIndex.rootPaths, + fsRelation.partitionSchema) + } + + lazy val scanFileSize = fileIndex + .listFiles(Nil, Nil).flatMap(_.files).map(_.getLen).sum + if (maxFileSizeOpt.exists(_ < scanFileSize)) { + throw partTableMaxFileExceedError( + scanFileSize, + maxFileSizeOpt.get, + logicalRelation.catalogTable, + catalogFileIndex.rootPaths.map(_.toString), + fsRelation.partitionSchema.map(_.name)) + } + } else { + lazy val scanFileSize = catalogFileIndex.sizeInBytes + if (maxFileSizeOpt.exists(_ < scanFileSize)) { + throw nonPartTableMaxFileExceedError( + scanFileSize, + maxFileSizeOpt.get, + logicalRelation.catalogTable) + } + } + case _ => + } + } + + def maxPartitionExceedError( + prunedPartitionSize: Int, + maxPartitionSize: Int, + tableMeta: Option[CatalogTable], + rootPaths: Seq[Path], + partitionSchema: StructType): Throwable = { + val truncatedPaths = + if (rootPaths.length > 5) { + rootPaths.slice(0, 5).mkString(",") + """... """ + (rootPaths.length - 5) + " more paths" + } else { + rootPaths.mkString(",") + } + + new MaxPartitionExceedException( + s""" + |SQL job scan data source partition: $prunedPartitionSize + |exceed restrict of data source scan maxPartition $maxPartitionSize + |You should optimize your SQL logical according partition structure + |or shorten query scope such as p_date, detail as below: + |Table: ${tableMeta.map(_.qualifiedName).getOrElse("")} + |Owner: ${tableMeta.map(_.owner).getOrElse("")} + |RootPaths: $truncatedPaths + |Partition Structure: ${partitionSchema.map(_.name).mkString(", ")} + |""".stripMargin) + } + + private def partTableMaxFileExceedError( + scanFileSize: Number, + maxFileSize: Long, + tableMeta: Option[CatalogTable], + rootPaths: Seq[String], + partitions: Seq[String]): Throwable = { + val truncatedPaths = + if (rootPaths.length > 5) { + rootPaths.slice(0, 5).mkString(",") + """... """ + (rootPaths.length - 5) + " more paths" + } else { + rootPaths.mkString(",") + } + + new MaxFileSizeExceedException( + s""" + |SQL job scan file size in bytes: $scanFileSize + |exceed restrict of table scan maxFileSize $maxFileSize + |You should optimize your SQL logical according partition structure + |or shorten query scope such as p_date, detail as below: + |Table: ${tableMeta.map(_.qualifiedName).getOrElse("")} + |Owner: ${tableMeta.map(_.owner).getOrElse("")} + |RootPaths: $truncatedPaths + |Partition Structure: ${partitions.mkString(", ")} + |""".stripMargin) + } + + private def nonPartTableMaxFileExceedError( + scanFileSize: Number, + maxFileSize: Long, + tableMeta: Option[CatalogTable]): Throwable = { + new MaxFileSizeExceedException( + s""" + |SQL job scan file size in bytes: $scanFileSize + |exceed restrict of table scan maxFileSize $maxFileSize + |detail as below: + |Table: ${tableMeta.map(_.qualifiedName).getOrElse("")} + |Owner: ${tableMeta.map(_.owner).getOrElse("")} + |Location: ${tableMeta.map(_.location).getOrElse("")} + |""".stripMargin) + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/zorder/InsertZorderBeforeWriting.scala b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/zorder/InsertZorderBeforeWriting.scala new file mode 100644 index 000000000..b3f98ec6d --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/zorder/InsertZorderBeforeWriting.scala @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql.zorder + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.catalog.CatalogTable +import org.apache.spark.sql.catalyst.expressions.{Ascending, Attribute, Expression, NullsLast, SortOrder} +import org.apache.spark.sql.catalyst.plans.logical._ +import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand +import org.apache.spark.sql.hive.execution.InsertIntoHiveTable + +import org.apache.kyuubi.sql.{KyuubiSQLConf, KyuubiSQLExtensionException} + +trait InsertZorderHelper33 extends Rule[LogicalPlan] with ZorderBuilder { + private val KYUUBI_ZORDER_ENABLED = "kyuubi.zorder.enabled" + private val KYUUBI_ZORDER_COLS = "kyuubi.zorder.cols" + + def isZorderEnabled(props: Map[String, String]): Boolean = { + props.contains(KYUUBI_ZORDER_ENABLED) && + "true".equalsIgnoreCase(props(KYUUBI_ZORDER_ENABLED)) && + props.contains(KYUUBI_ZORDER_COLS) + } + + def getZorderColumns(props: Map[String, String]): Seq[String] = { + val cols = props.get(KYUUBI_ZORDER_COLS) + assert(cols.isDefined) + cols.get.split(",").map(_.trim) + } + + def canInsertZorder(query: LogicalPlan): Boolean = query match { + case Project(_, child) => canInsertZorder(child) + // TODO: actually, we can force zorder even if existed some shuffle + case _: Sort => false + case _: RepartitionByExpression => false + case _: Repartition => false + case _ => true + } + + def insertZorder( + catalogTable: CatalogTable, + plan: LogicalPlan, + dynamicPartitionColumns: Seq[Attribute]): LogicalPlan = { + if (!canInsertZorder(plan)) { + return plan + } + val cols = getZorderColumns(catalogTable.properties) + val resolver = session.sessionState.conf.resolver + val output = plan.output + val bound = cols.flatMap(col => output.find(attr => resolver(attr.name, col))) + if (bound.size < cols.size) { + logWarning(s"target table does not contain all zorder cols: ${cols.mkString(",")}, " + + s"please check your table properties ${KYUUBI_ZORDER_COLS}.") + plan + } else { + if (conf.getConf(KyuubiSQLConf.ZORDER_GLOBAL_SORT_ENABLED) && + conf.getConf(KyuubiSQLConf.REBALANCE_BEFORE_ZORDER)) { + throw new KyuubiSQLExtensionException(s"${KyuubiSQLConf.ZORDER_GLOBAL_SORT_ENABLED.key} " + + s"and ${KyuubiSQLConf.REBALANCE_BEFORE_ZORDER.key} can not be enabled together.") + } + if (conf.getConf(KyuubiSQLConf.ZORDER_GLOBAL_SORT_ENABLED) && + dynamicPartitionColumns.nonEmpty) { + logWarning(s"Dynamic partition insertion with global sort may produce small files.") + } + + val zorderExpr = + if (bound.length == 1) { + bound + } else if (conf.getConf(KyuubiSQLConf.ZORDER_USING_ORIGINAL_ORDERING_ENABLED)) { + bound.asInstanceOf[Seq[Expression]] + } else { + buildZorder(bound) :: Nil + } + val (global, orderExprs, child) = + if (conf.getConf(KyuubiSQLConf.ZORDER_GLOBAL_SORT_ENABLED)) { + (true, zorderExpr, plan) + } else if (conf.getConf(KyuubiSQLConf.REBALANCE_BEFORE_ZORDER)) { + val rebalanceExpr = + if (dynamicPartitionColumns.isEmpty) { + // static partition insert + bound + } else if (conf.getConf(KyuubiSQLConf.REBALANCE_ZORDER_COLUMNS_ENABLED)) { + // improve data compression ratio + dynamicPartitionColumns.asInstanceOf[Seq[Expression]] ++ bound + } else { + dynamicPartitionColumns.asInstanceOf[Seq[Expression]] + } + // for dynamic partition insert, Spark always sort the partition columns, + // so here we sort partition columns + zorder. + val rebalance = + if (dynamicPartitionColumns.nonEmpty && + conf.getConf(KyuubiSQLConf.TWO_PHASE_REBALANCE_BEFORE_ZORDER)) { + // improve compression ratio + RebalancePartitions( + rebalanceExpr, + RebalancePartitions(dynamicPartitionColumns, plan)) + } else { + RebalancePartitions(rebalanceExpr, plan) + } + (false, dynamicPartitionColumns.asInstanceOf[Seq[Expression]] ++ zorderExpr, rebalance) + } else { + (false, zorderExpr, plan) + } + val order = orderExprs.map { expr => + SortOrder(expr, Ascending, NullsLast, Seq.empty) + } + Sort(order, global, child) + } + } + + override def buildZorder(children: Seq[Expression]): ZorderBase = Zorder(children) + + def session: SparkSession + def applyInternal(plan: LogicalPlan): LogicalPlan + + final override def apply(plan: LogicalPlan): LogicalPlan = { + if (conf.getConf(KyuubiSQLConf.INSERT_ZORDER_BEFORE_WRITING)) { + applyInternal(plan) + } else { + plan + } + } +} + +case class InsertZorderBeforeWritingDatasource33(session: SparkSession) + extends InsertZorderHelper33 { + override def applyInternal(plan: LogicalPlan): LogicalPlan = plan match { + case insert: InsertIntoHadoopFsRelationCommand + if insert.query.resolved && + insert.bucketSpec.isEmpty && insert.catalogTable.isDefined && + isZorderEnabled(insert.catalogTable.get.properties) => + val dynamicPartition = + insert.partitionColumns.filterNot(attr => insert.staticPartitions.contains(attr.name)) + val newQuery = insertZorder(insert.catalogTable.get, insert.query, dynamicPartition) + if (newQuery.eq(insert.query)) { + insert + } else { + insert.copy(query = newQuery) + } + + case _ => plan + } +} + +case class InsertZorderBeforeWritingHive33(session: SparkSession) + extends InsertZorderHelper33 { + override def applyInternal(plan: LogicalPlan): LogicalPlan = plan match { + case insert: InsertIntoHiveTable + if insert.query.resolved && + insert.table.bucketSpec.isEmpty && isZorderEnabled(insert.table.properties) => + val dynamicPartition = insert.partition.filter(_._2.isEmpty).keys + .flatMap(name => insert.query.output.find(_.name == name)).toSeq + val newQuery = insertZorder(insert.table, insert.query, dynamicPartition) + if (newQuery.eq(insert.query)) { + insert + } else { + insert.copy(query = newQuery) + } + + case _ => plan + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/zorder/InsertZorderBeforeWritingBase.scala b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/zorder/InsertZorderBeforeWritingBase.scala new file mode 100644 index 000000000..2c59d148e --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/zorder/InsertZorderBeforeWritingBase.scala @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql.zorder + +import java.util.Locale + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.catalog.CatalogTable +import org.apache.spark.sql.catalyst.expressions.{Ascending, Expression, NullsLast, SortOrder} +import org.apache.spark.sql.catalyst.plans.logical._ +import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand +import org.apache.spark.sql.hive.execution.InsertIntoHiveTable + +import org.apache.kyuubi.sql.KyuubiSQLConf + +/** + * TODO: shall we forbid zorder if it's dynamic partition inserts ? + * Insert zorder before writing datasource if the target table properties has zorder properties + */ +abstract class InsertZorderBeforeWritingDatasourceBase + extends InsertZorderHelper { + override def applyInternal(plan: LogicalPlan): LogicalPlan = plan match { + case insert: InsertIntoHadoopFsRelationCommand + if insert.query.resolved && insert.bucketSpec.isEmpty && insert.catalogTable.isDefined && + isZorderEnabled(insert.catalogTable.get.properties) => + val newQuery = insertZorder(insert.catalogTable.get, insert.query) + if (newQuery.eq(insert.query)) { + insert + } else { + insert.copy(query = newQuery) + } + case _ => plan + } +} + +/** + * TODO: shall we forbid zorder if it's dynamic partition inserts ? + * Insert zorder before writing hive if the target table properties has zorder properties + */ +abstract class InsertZorderBeforeWritingHiveBase + extends InsertZorderHelper { + override def applyInternal(plan: LogicalPlan): LogicalPlan = plan match { + case insert: InsertIntoHiveTable + if insert.query.resolved && insert.table.bucketSpec.isEmpty && + isZorderEnabled(insert.table.properties) => + val newQuery = insertZorder(insert.table, insert.query) + if (newQuery.eq(insert.query)) { + insert + } else { + insert.copy(query = newQuery) + } + case _ => plan + } +} + +trait ZorderBuilder { + def buildZorder(children: Seq[Expression]): ZorderBase +} + +trait InsertZorderHelper extends Rule[LogicalPlan] with ZorderBuilder { + private val KYUUBI_ZORDER_ENABLED = "kyuubi.zorder.enabled" + private val KYUUBI_ZORDER_COLS = "kyuubi.zorder.cols" + + def isZorderEnabled(props: Map[String, String]): Boolean = { + props.contains(KYUUBI_ZORDER_ENABLED) && + "true".equalsIgnoreCase(props(KYUUBI_ZORDER_ENABLED)) && + props.contains(KYUUBI_ZORDER_COLS) + } + + def getZorderColumns(props: Map[String, String]): Seq[String] = { + val cols = props.get(KYUUBI_ZORDER_COLS) + assert(cols.isDefined) + cols.get.split(",").map(_.trim.toLowerCase(Locale.ROOT)) + } + + def canInsertZorder(query: LogicalPlan): Boolean = query match { + case Project(_, child) => canInsertZorder(child) + // TODO: actually, we can force zorder even if existed some shuffle + case _: Sort => false + case _: RepartitionByExpression => false + case _: Repartition => false + case _ => true + } + + def insertZorder(catalogTable: CatalogTable, plan: LogicalPlan): LogicalPlan = { + if (!canInsertZorder(plan)) { + return plan + } + val cols = getZorderColumns(catalogTable.properties) + val attrs = plan.output.map(attr => (attr.name, attr)).toMap + if (cols.exists(!attrs.contains(_))) { + logWarning(s"target table does not contain all zorder cols: ${cols.mkString(",")}, " + + s"please check your table properties ${KYUUBI_ZORDER_COLS}.") + plan + } else { + val bound = cols.map(attrs(_)) + val orderExpr = + if (bound.length == 1) { + bound.head + } else { + buildZorder(bound) + } + // TODO: We can do rebalance partitions before local sort of zorder after SPARK 3.3 + // see https://github.com/apache/spark/pull/34542 + Sort( + SortOrder(orderExpr, Ascending, NullsLast, Seq.empty) :: Nil, + conf.getConf(KyuubiSQLConf.ZORDER_GLOBAL_SORT_ENABLED), + plan) + } + } + + def applyInternal(plan: LogicalPlan): LogicalPlan + + final override def apply(plan: LogicalPlan): LogicalPlan = { + if (conf.getConf(KyuubiSQLConf.INSERT_ZORDER_BEFORE_WRITING)) { + applyInternal(plan) + } else { + plan + } + } +} + +/** + * TODO: shall we forbid zorder if it's dynamic partition inserts ? + * Insert zorder before writing datasource if the target table properties has zorder properties + */ +case class InsertZorderBeforeWritingDatasource(session: SparkSession) + extends InsertZorderBeforeWritingDatasourceBase { + override def buildZorder(children: Seq[Expression]): ZorderBase = Zorder(children) +} + +/** + * TODO: shall we forbid zorder if it's dynamic partition inserts ? + * Insert zorder before writing hive if the target table properties has zorder properties + */ +case class InsertZorderBeforeWritingHive(session: SparkSession) + extends InsertZorderBeforeWritingHiveBase { + override def buildZorder(children: Seq[Expression]): ZorderBase = Zorder(children) +} diff --git a/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/zorder/OptimizeZorderCommandBase.scala b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/zorder/OptimizeZorderCommandBase.scala new file mode 100644 index 000000000..21d1cf2a2 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/zorder/OptimizeZorderCommandBase.scala @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql.zorder + +import org.apache.spark.sql.{Row, SparkSession} +import org.apache.spark.sql.catalyst.catalog.CatalogTable +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.execution.SparkPlan +import org.apache.spark.sql.execution.command.DataWritingCommand +import org.apache.spark.sql.hive.execution.InsertIntoHiveTable + +import org.apache.kyuubi.sql.KyuubiSQLExtensionException + +/** + * A runnable command for zorder, we delegate to real command to execute + */ +abstract class OptimizeZorderCommandBase extends DataWritingCommand { + def catalogTable: CatalogTable + + override def outputColumnNames: Seq[String] = query.output.map(_.name) + + private def isHiveTable: Boolean = { + catalogTable.provider.isEmpty || + (catalogTable.provider.isDefined && "hive".equalsIgnoreCase(catalogTable.provider.get)) + } + + private def getWritingCommand(session: SparkSession): DataWritingCommand = { + // TODO: Support convert hive relation to datasource relation, can see + // [[org.apache.spark.sql.hive.RelationConversions]] + InsertIntoHiveTable( + catalogTable, + catalogTable.partitionColumnNames.map(p => (p, None)).toMap, + query, + overwrite = true, + ifPartitionNotExists = false, + outputColumnNames) + } + + override def run(session: SparkSession, child: SparkPlan): Seq[Row] = { + // TODO: Support datasource relation + // TODO: Support read and insert overwrite the same table for some table format + if (!isHiveTable) { + throw new KyuubiSQLExtensionException("only support hive table") + } + + val command = getWritingCommand(session) + command.run(session, child) + DataWritingCommand.propogateMetrics(session.sparkContext, command, metrics) + Seq.empty + } +} + +/** + * A runnable command for zorder, we delegate to real command to execute + */ +case class OptimizeZorderCommand( + catalogTable: CatalogTable, + query: LogicalPlan) + extends OptimizeZorderCommandBase { + protected def withNewChildInternal(newChild: LogicalPlan): LogicalPlan = { + copy(query = newChild) + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/zorder/OptimizeZorderStatementBase.scala b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/zorder/OptimizeZorderStatementBase.scala new file mode 100644 index 000000000..895f9e24b --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/zorder/OptimizeZorderStatementBase.scala @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql.zorder + +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, UnaryNode} + +/** + * A zorder statement that contains we parsed from SQL. + * We should convert this plan to certain command at Analyzer. + */ +case class OptimizeZorderStatement( + tableIdentifier: Seq[String], + query: LogicalPlan) extends UnaryNode { + override def child: LogicalPlan = query + override def output: Seq[Attribute] = child.output + protected def withNewChildInternal(newChild: LogicalPlan): LogicalPlan = + copy(query = newChild) +} diff --git a/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/zorder/ResolveZorderBase.scala b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/zorder/ResolveZorderBase.scala new file mode 100644 index 000000000..9f735caa7 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/zorder/ResolveZorderBase.scala @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql.zorder + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.catalog.{CatalogTable, HiveTableRelation} +import org.apache.spark.sql.catalyst.expressions.AttributeSet +import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, SubqueryAlias} +import org.apache.spark.sql.catalyst.rules.Rule + +import org.apache.kyuubi.sql.KyuubiSQLExtensionException + +/** + * Resolve `OptimizeZorderStatement` to `OptimizeZorderCommand` + */ +abstract class ResolveZorderBase extends Rule[LogicalPlan] { + def session: SparkSession + def buildOptimizeZorderCommand( + catalogTable: CatalogTable, + query: LogicalPlan): OptimizeZorderCommandBase + + protected def checkQueryAllowed(query: LogicalPlan): Unit = query foreach { + case Filter(condition, SubqueryAlias(_, tableRelation: HiveTableRelation)) => + if (tableRelation.partitionCols.isEmpty) { + throw new KyuubiSQLExtensionException("Filters are only supported for partitioned table") + } + + val partitionKeyIds = AttributeSet(tableRelation.partitionCols) + if (condition.references.isEmpty || !condition.references.subsetOf(partitionKeyIds)) { + throw new KyuubiSQLExtensionException("Only partition column filters are allowed") + } + + case _ => + } + + protected def getTableIdentifier(tableIdent: Seq[String]): TableIdentifier = tableIdent match { + case Seq(tbl) => TableIdentifier.apply(tbl) + case Seq(db, tbl) => TableIdentifier.apply(tbl, Some(db)) + case _ => throw new KyuubiSQLExtensionException( + "only support session catalog table, please use db.table instead") + } + + override def apply(plan: LogicalPlan): LogicalPlan = plan match { + case statement: OptimizeZorderStatement if statement.query.resolved => + checkQueryAllowed(statement.query) + val tableIdentifier = getTableIdentifier(statement.tableIdentifier) + val catalogTable = session.sessionState.catalog.getTableMetadata(tableIdentifier) + buildOptimizeZorderCommand(catalogTable, statement.query) + + case _ => plan + } +} + +/** + * Resolve `OptimizeZorderStatement` to `OptimizeZorderCommand` + */ +case class ResolveZorder(session: SparkSession) extends ResolveZorderBase { + override def buildOptimizeZorderCommand( + catalogTable: CatalogTable, + query: LogicalPlan): OptimizeZorderCommandBase = { + OptimizeZorderCommand(catalogTable, query) + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/zorder/ZorderBase.scala b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/zorder/ZorderBase.scala new file mode 100644 index 000000000..e4d98ccbe --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/zorder/ZorderBase.scala @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql.zorder + +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.analysis.TypeCheckResult +import org.apache.spark.sql.catalyst.expressions.Expression +import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode, FalseLiteral} +import org.apache.spark.sql.catalyst.expressions.codegen.Block._ +import org.apache.spark.sql.types.{BinaryType, DataType} + +import org.apache.kyuubi.sql.KyuubiSQLExtensionException + +abstract class ZorderBase extends Expression { + override def foldable: Boolean = children.forall(_.foldable) + override def nullable: Boolean = false + override def dataType: DataType = BinaryType + override def prettyName: String = "zorder" + + override def checkInputDataTypes(): TypeCheckResult = { + try { + defaultNullValues + TypeCheckResult.TypeCheckSuccess + } catch { + case e: KyuubiSQLExtensionException => + TypeCheckResult.TypeCheckFailure(e.getMessage) + } + } + + @transient + private[this] lazy val defaultNullValues: Array[Any] = + children.map(_.dataType) + .map(ZorderBytesUtils.defaultValue) + .toArray + + override def eval(input: InternalRow): Any = { + val childrenValues = children.zipWithIndex.map { + case (child: Expression, index) => + val v = child.eval(input) + if (v == null) { + defaultNullValues(index) + } else { + v + } + } + ZorderBytesUtils.interleaveBits(childrenValues.toArray) + } + + override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { + val evals = children.map(_.genCode(ctx)) + val defaultValues = ctx.addReferenceObj("defaultValues", defaultNullValues) + val values = ctx.freshName("values") + val util = ZorderBytesUtils.getClass.getName.stripSuffix("$") + val inputs = evals.zipWithIndex.map { + case (eval, index) => + s""" + |${eval.code} + |if (${eval.isNull}) { + | $values[$index] = $defaultValues[$index]; + |} else { + | $values[$index] = ${eval.value}; + |} + |""".stripMargin + } + ev.copy( + code = + code""" + |byte[] ${ev.value} = null; + |Object[] $values = new Object[${evals.length}]; + |${inputs.mkString("\n")} + |${ev.value} = $util.interleaveBits($values); + |""".stripMargin, + isNull = FalseLiteral) + } +} + +case class Zorder(children: Seq[Expression]) extends ZorderBase { + protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Expression = + copy(children = newChildren) +} diff --git a/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/zorder/ZorderBytesUtils.scala b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/zorder/ZorderBytesUtils.scala new file mode 100644 index 000000000..d249f1dc3 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/kyuubi/sql/zorder/ZorderBytesUtils.scala @@ -0,0 +1,517 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql.zorder + +import java.lang.{Double => jDouble, Float => jFloat} + +import org.apache.spark.sql.types._ +import org.apache.spark.unsafe.types.UTF8String + +import org.apache.kyuubi.sql.KyuubiSQLExtensionException + +object ZorderBytesUtils { + final private val BIT_8_MASK = 1 << 7 + final private val BIT_16_MASK = 1 << 15 + final private val BIT_32_MASK = 1 << 31 + final private val BIT_64_MASK = 1L << 63 + + def interleaveBits(inputs: Array[Any]): Array[Byte] = { + inputs.length match { + // it's a more fast approach, use O(8 * 8) + // can see http://graphics.stanford.edu/~seander/bithacks.html#InterleaveTableObvious + case 1 => longToByte(toLong(inputs(0))) + case 2 => interleave2Longs(toLong(inputs(0)), toLong(inputs(1))) + case 3 => interleave3Longs(toLong(inputs(0)), toLong(inputs(1)), toLong(inputs(2))) + case 4 => + interleave4Longs(toLong(inputs(0)), toLong(inputs(1)), toLong(inputs(2)), toLong(inputs(3))) + case 5 => interleave5Longs( + toLong(inputs(0)), + toLong(inputs(1)), + toLong(inputs(2)), + toLong(inputs(3)), + toLong(inputs(4))) + case 6 => interleave6Longs( + toLong(inputs(0)), + toLong(inputs(1)), + toLong(inputs(2)), + toLong(inputs(3)), + toLong(inputs(4)), + toLong(inputs(5))) + case 7 => interleave7Longs( + toLong(inputs(0)), + toLong(inputs(1)), + toLong(inputs(2)), + toLong(inputs(3)), + toLong(inputs(4)), + toLong(inputs(5)), + toLong(inputs(6))) + case 8 => interleave8Longs( + toLong(inputs(0)), + toLong(inputs(1)), + toLong(inputs(2)), + toLong(inputs(3)), + toLong(inputs(4)), + toLong(inputs(5)), + toLong(inputs(6)), + toLong(inputs(7))) + + case _ => + // it's the default approach, use O(64 * n), n is the length of inputs + interleaveBitsDefault(inputs.map(toByteArray)) + } + } + + private def interleave2Longs(l1: Long, l2: Long): Array[Byte] = { + // output 8 * 16 bits + val result = new Array[Byte](16) + var i = 0 + while (i < 8) { + val tmp1 = ((l1 >> (i * 8)) & 0xFF).toShort + val tmp2 = ((l2 >> (i * 8)) & 0xFF).toShort + + var z = 0 + var j = 0 + while (j < 8) { + val x_masked = tmp1 & (1 << j) + val y_masked = tmp2 & (1 << j) + z |= (x_masked << j) + z |= (y_masked << (j + 1)) + j = j + 1 + } + result((7 - i) * 2 + 1) = (z & 0xFF).toByte + result((7 - i) * 2) = ((z >> 8) & 0xFF).toByte + i = i + 1 + } + result + } + + private def interleave3Longs(l1: Long, l2: Long, l3: Long): Array[Byte] = { + // output 8 * 24 bits + val result = new Array[Byte](24) + var i = 0 + while (i < 8) { + val tmp1 = ((l1 >> (i * 8)) & 0xFF).toInt + val tmp2 = ((l2 >> (i * 8)) & 0xFF).toInt + val tmp3 = ((l3 >> (i * 8)) & 0xFF).toInt + + var z = 0 + var j = 0 + while (j < 8) { + val r1_mask = tmp1 & (1 << j) + val r2_mask = tmp2 & (1 << j) + val r3_mask = tmp3 & (1 << j) + z |= (r1_mask << (2 * j)) | (r2_mask << (2 * j + 1)) | (r3_mask << (2 * j + 2)) + j = j + 1 + } + result((7 - i) * 3 + 2) = (z & 0xFF).toByte + result((7 - i) * 3 + 1) = ((z >> 8) & 0xFF).toByte + result((7 - i) * 3) = ((z >> 16) & 0xFF).toByte + i = i + 1 + } + result + } + + private def interleave4Longs(l1: Long, l2: Long, l3: Long, l4: Long): Array[Byte] = { + // output 8 * 32 bits + val result = new Array[Byte](32) + var i = 0 + while (i < 8) { + val tmp1 = ((l1 >> (i * 8)) & 0xFF).toInt + val tmp2 = ((l2 >> (i * 8)) & 0xFF).toInt + val tmp3 = ((l3 >> (i * 8)) & 0xFF).toInt + val tmp4 = ((l4 >> (i * 8)) & 0xFF).toInt + + var z = 0 + var j = 0 + while (j < 8) { + val r1_mask = tmp1 & (1 << j) + val r2_mask = tmp2 & (1 << j) + val r3_mask = tmp3 & (1 << j) + val r4_mask = tmp4 & (1 << j) + z |= (r1_mask << (3 * j)) | (r2_mask << (3 * j + 1)) | (r3_mask << (3 * j + 2)) | + (r4_mask << (3 * j + 3)) + j = j + 1 + } + result((7 - i) * 4 + 3) = (z & 0xFF).toByte + result((7 - i) * 4 + 2) = ((z >> 8) & 0xFF).toByte + result((7 - i) * 4 + 1) = ((z >> 16) & 0xFF).toByte + result((7 - i) * 4) = ((z >> 24) & 0xFF).toByte + i = i + 1 + } + result + } + + private def interleave5Longs( + l1: Long, + l2: Long, + l3: Long, + l4: Long, + l5: Long): Array[Byte] = { + // output 8 * 40 bits + val result = new Array[Byte](40) + var i = 0 + while (i < 8) { + val tmp1 = ((l1 >> (i * 8)) & 0xFF).toLong + val tmp2 = ((l2 >> (i * 8)) & 0xFF).toLong + val tmp3 = ((l3 >> (i * 8)) & 0xFF).toLong + val tmp4 = ((l4 >> (i * 8)) & 0xFF).toLong + val tmp5 = ((l5 >> (i * 8)) & 0xFF).toLong + + var z = 0L + var j = 0 + while (j < 8) { + val r1_mask = tmp1 & (1 << j) + val r2_mask = tmp2 & (1 << j) + val r3_mask = tmp3 & (1 << j) + val r4_mask = tmp4 & (1 << j) + val r5_mask = tmp5 & (1 << j) + z |= (r1_mask << (4 * j)) | (r2_mask << (4 * j + 1)) | (r3_mask << (4 * j + 2)) | + (r4_mask << (4 * j + 3)) | (r5_mask << (4 * j + 4)) + j = j + 1 + } + result((7 - i) * 5 + 4) = (z & 0xFF).toByte + result((7 - i) * 5 + 3) = ((z >> 8) & 0xFF).toByte + result((7 - i) * 5 + 2) = ((z >> 16) & 0xFF).toByte + result((7 - i) * 5 + 1) = ((z >> 24) & 0xFF).toByte + result((7 - i) * 5) = ((z >> 32) & 0xFF).toByte + i = i + 1 + } + result + } + + private def interleave6Longs( + l1: Long, + l2: Long, + l3: Long, + l4: Long, + l5: Long, + l6: Long): Array[Byte] = { + // output 8 * 48 bits + val result = new Array[Byte](48) + var i = 0 + while (i < 8) { + val tmp1 = ((l1 >> (i * 8)) & 0xFF).toLong + val tmp2 = ((l2 >> (i * 8)) & 0xFF).toLong + val tmp3 = ((l3 >> (i * 8)) & 0xFF).toLong + val tmp4 = ((l4 >> (i * 8)) & 0xFF).toLong + val tmp5 = ((l5 >> (i * 8)) & 0xFF).toLong + val tmp6 = ((l6 >> (i * 8)) & 0xFF).toLong + + var z = 0L + var j = 0 + while (j < 8) { + val r1_mask = tmp1 & (1 << j) + val r2_mask = tmp2 & (1 << j) + val r3_mask = tmp3 & (1 << j) + val r4_mask = tmp4 & (1 << j) + val r5_mask = tmp5 & (1 << j) + val r6_mask = tmp6 & (1 << j) + z |= (r1_mask << (5 * j)) | (r2_mask << (5 * j + 1)) | (r3_mask << (5 * j + 2)) | + (r4_mask << (5 * j + 3)) | (r5_mask << (5 * j + 4)) | (r6_mask << (5 * j + 5)) + j = j + 1 + } + result((7 - i) * 6 + 5) = (z & 0xFF).toByte + result((7 - i) * 6 + 4) = ((z >> 8) & 0xFF).toByte + result((7 - i) * 6 + 3) = ((z >> 16) & 0xFF).toByte + result((7 - i) * 6 + 2) = ((z >> 24) & 0xFF).toByte + result((7 - i) * 6 + 1) = ((z >> 32) & 0xFF).toByte + result((7 - i) * 6) = ((z >> 40) & 0xFF).toByte + i = i + 1 + } + result + } + + private def interleave7Longs( + l1: Long, + l2: Long, + l3: Long, + l4: Long, + l5: Long, + l6: Long, + l7: Long): Array[Byte] = { + // output 8 * 56 bits + val result = new Array[Byte](56) + var i = 0 + while (i < 8) { + val tmp1 = ((l1 >> (i * 8)) & 0xFF).toLong + val tmp2 = ((l2 >> (i * 8)) & 0xFF).toLong + val tmp3 = ((l3 >> (i * 8)) & 0xFF).toLong + val tmp4 = ((l4 >> (i * 8)) & 0xFF).toLong + val tmp5 = ((l5 >> (i * 8)) & 0xFF).toLong + val tmp6 = ((l6 >> (i * 8)) & 0xFF).toLong + val tmp7 = ((l7 >> (i * 8)) & 0xFF).toLong + + var z = 0L + var j = 0 + while (j < 8) { + val r1_mask = tmp1 & (1 << j) + val r2_mask = tmp2 & (1 << j) + val r3_mask = tmp3 & (1 << j) + val r4_mask = tmp4 & (1 << j) + val r5_mask = tmp5 & (1 << j) + val r6_mask = tmp6 & (1 << j) + val r7_mask = tmp7 & (1 << j) + z |= (r1_mask << (6 * j)) | (r2_mask << (6 * j + 1)) | (r3_mask << (6 * j + 2)) | + (r4_mask << (6 * j + 3)) | (r5_mask << (6 * j + 4)) | (r6_mask << (6 * j + 5)) | + (r7_mask << (6 * j + 6)) + j = j + 1 + } + result((7 - i) * 7 + 6) = (z & 0xFF).toByte + result((7 - i) * 7 + 5) = ((z >> 8) & 0xFF).toByte + result((7 - i) * 7 + 4) = ((z >> 16) & 0xFF).toByte + result((7 - i) * 7 + 3) = ((z >> 24) & 0xFF).toByte + result((7 - i) * 7 + 2) = ((z >> 32) & 0xFF).toByte + result((7 - i) * 7 + 1) = ((z >> 40) & 0xFF).toByte + result((7 - i) * 7) = ((z >> 48) & 0xFF).toByte + i = i + 1 + } + result + } + + private def interleave8Longs( + l1: Long, + l2: Long, + l3: Long, + l4: Long, + l5: Long, + l6: Long, + l7: Long, + l8: Long): Array[Byte] = { + // output 8 * 64 bits + val result = new Array[Byte](64) + var i = 0 + while (i < 8) { + val tmp1 = ((l1 >> (i * 8)) & 0xFF).toLong + val tmp2 = ((l2 >> (i * 8)) & 0xFF).toLong + val tmp3 = ((l3 >> (i * 8)) & 0xFF).toLong + val tmp4 = ((l4 >> (i * 8)) & 0xFF).toLong + val tmp5 = ((l5 >> (i * 8)) & 0xFF).toLong + val tmp6 = ((l6 >> (i * 8)) & 0xFF).toLong + val tmp7 = ((l7 >> (i * 8)) & 0xFF).toLong + val tmp8 = ((l8 >> (i * 8)) & 0xFF).toLong + + var z = 0L + var j = 0 + while (j < 8) { + val r1_mask = tmp1 & (1 << j) + val r2_mask = tmp2 & (1 << j) + val r3_mask = tmp3 & (1 << j) + val r4_mask = tmp4 & (1 << j) + val r5_mask = tmp5 & (1 << j) + val r6_mask = tmp6 & (1 << j) + val r7_mask = tmp7 & (1 << j) + val r8_mask = tmp8 & (1 << j) + z |= (r1_mask << (7 * j)) | (r2_mask << (7 * j + 1)) | (r3_mask << (7 * j + 2)) | + (r4_mask << (7 * j + 3)) | (r5_mask << (7 * j + 4)) | (r6_mask << (7 * j + 5)) | + (r7_mask << (7 * j + 6)) | (r8_mask << (7 * j + 7)) + j = j + 1 + } + result((7 - i) * 8 + 7) = (z & 0xFF).toByte + result((7 - i) * 8 + 6) = ((z >> 8) & 0xFF).toByte + result((7 - i) * 8 + 5) = ((z >> 16) & 0xFF).toByte + result((7 - i) * 8 + 4) = ((z >> 24) & 0xFF).toByte + result((7 - i) * 8 + 3) = ((z >> 32) & 0xFF).toByte + result((7 - i) * 8 + 2) = ((z >> 40) & 0xFF).toByte + result((7 - i) * 8 + 1) = ((z >> 48) & 0xFF).toByte + result((7 - i) * 8) = ((z >> 56) & 0xFF).toByte + i = i + 1 + } + result + } + + def interleaveBitsDefault(arrays: Array[Array[Byte]]): Array[Byte] = { + var totalLength = 0 + var maxLength = 0 + arrays.foreach { array => + totalLength += array.length + maxLength = maxLength.max(array.length * 8) + } + val result = new Array[Byte](totalLength) + var resultBit = 0 + + var bit = 0 + while (bit < maxLength) { + val bytePos = bit / 8 + val bitPos = bit % 8 + + for (arr <- arrays) { + val len = arr.length + if (bytePos < len) { + val resultBytePos = totalLength - 1 - resultBit / 8 + val resultBitPos = resultBit % 8 + result(resultBytePos) = + updatePos(result(resultBytePos), resultBitPos, arr(len - 1 - bytePos), bitPos) + resultBit += 1 + } + } + bit += 1 + } + result + } + + def updatePos(a: Byte, apos: Int, b: Byte, bpos: Int): Byte = { + var temp = (b & (1 << bpos)).toByte + if (apos > bpos) { + temp = (temp << (apos - bpos)).toByte + } else if (apos < bpos) { + temp = (temp >> (bpos - apos)).toByte + } + val atemp = (a & (1 << apos)).toByte + if (atemp == temp) { + return a + } + (a ^ (1 << apos)).toByte + } + + def toLong(a: Any): Long = { + a match { + case b: Boolean => (if (b) 1 else 0).toLong ^ BIT_64_MASK + case b: Byte => b.toLong ^ BIT_64_MASK + case s: Short => s.toLong ^ BIT_64_MASK + case i: Int => i.toLong ^ BIT_64_MASK + case l: Long => l ^ BIT_64_MASK + case f: Float => java.lang.Float.floatToRawIntBits(f).toLong ^ BIT_64_MASK + case d: Double => java.lang.Double.doubleToRawLongBits(d) ^ BIT_64_MASK + case str: UTF8String => str.getPrefix + case dec: Decimal => dec.toLong ^ BIT_64_MASK + case other: Any => + throw new KyuubiSQLExtensionException("Unsupported z-order type: " + other.getClass) + } + } + + def toByteArray(a: Any): Array[Byte] = { + a match { + case bo: Boolean => + booleanToByte(bo) + case b: Byte => + byteToByte(b) + case s: Short => + shortToByte(s) + case i: Int => + intToByte(i) + case l: Long => + longToByte(l) + case f: Float => + floatToByte(f) + case d: Double => + doubleToByte(d) + case str: UTF8String => + // truncate or padding str to 8 byte + paddingTo8Byte(str.getBytes) + case dec: Decimal => + longToByte(dec.toLong) + case other: Any => + throw new KyuubiSQLExtensionException("Unsupported z-order type: " + other.getClass) + } + } + + def booleanToByte(a: Boolean): Array[Byte] = { + if (a) { + byteToByte(1.toByte) + } else { + byteToByte(0.toByte) + } + } + + def byteToByte(a: Byte): Array[Byte] = { + val tmp = (a ^ BIT_8_MASK).toByte + Array(tmp) + } + + def shortToByte(a: Short): Array[Byte] = { + val tmp = a ^ BIT_16_MASK + Array(((tmp >> 8) & 0xFF).toByte, (tmp & 0xFF).toByte) + } + + def intToByte(a: Int): Array[Byte] = { + val result = new Array[Byte](4) + var i = 0 + val tmp = a ^ BIT_32_MASK + while (i <= 3) { + val offset = i * 8 + result(3 - i) = ((tmp >> offset) & 0xFF).toByte + i += 1 + } + result + } + + def longToByte(a: Long): Array[Byte] = { + val result = new Array[Byte](8) + var i = 0 + val tmp = a ^ BIT_64_MASK + while (i <= 7) { + val offset = i * 8 + result(7 - i) = ((tmp >> offset) & 0xFF).toByte + i += 1 + } + result + } + + def floatToByte(a: Float): Array[Byte] = { + val fi = jFloat.floatToRawIntBits(a) + intToByte(fi) + } + + def doubleToByte(a: Double): Array[Byte] = { + val dl = jDouble.doubleToRawLongBits(a) + longToByte(dl) + } + + def paddingTo8Byte(a: Array[Byte]): Array[Byte] = { + val len = a.length + if (len == 8) { + a + } else if (len > 8) { + val result = new Array[Byte](8) + System.arraycopy(a, 0, result, 0, 8) + result + } else { + val result = new Array[Byte](8) + System.arraycopy(a, 0, result, 8 - len, len) + result + } + } + + def defaultByteArrayValue(dataType: DataType): Array[Byte] = toByteArray { + defaultValue(dataType) + } + + def defaultValue(dataType: DataType): Any = { + dataType match { + case BooleanType => + true + case ByteType => + Byte.MaxValue + case ShortType => + Short.MaxValue + case IntegerType | DateType => + Int.MaxValue + case LongType | TimestampType | _: DecimalType => + Long.MaxValue + case FloatType => + Float.MaxValue + case DoubleType => + Double.MaxValue + case StringType => + // we pad string to 8 bytes so it's equal to long + UTF8String.fromBytes(longToByte(Long.MaxValue)) + case other: Any => + throw new KyuubiSQLExtensionException(s"Unsupported z-order type: ${other.catalogString}") + } + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/spark/sql/FinalStageResourceManager.scala b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/spark/sql/FinalStageResourceManager.scala new file mode 100644 index 000000000..81873476c --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/spark/sql/FinalStageResourceManager.scala @@ -0,0 +1,289 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +import scala.annotation.tailrec +import scala.collection.mutable +import scala.collection.mutable.ArrayBuffer + +import org.apache.spark.{ExecutorAllocationClient, MapOutputTrackerMaster, SparkContext, SparkEnv} +import org.apache.spark.internal.Logging +import org.apache.spark.resource.ResourceProfile +import org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend +import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.execution.{FilterExec, ProjectExec, SortExec, SparkPlan} +import org.apache.spark.sql.execution.adaptive._ +import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec +import org.apache.spark.sql.execution.command.DataWritingCommandExec +import org.apache.spark.sql.execution.datasources.WriteFilesExec +import org.apache.spark.sql.execution.datasources.v2.V2TableWriteExec +import org.apache.spark.sql.execution.exchange.{ENSURE_REQUIREMENTS, ShuffleExchangeExec} + +import org.apache.kyuubi.sql.{KyuubiSQLConf, WriteUtils} + +/** + * This rule assumes the final write stage has less cores requirement than previous, otherwise + * this rule would take no effect. + * + * It provide a feature: + * 1. Kill redundant executors before running final write stage + */ +case class FinalStageResourceManager(session: SparkSession) + extends Rule[SparkPlan] with FinalRebalanceStageHelper { + override def apply(plan: SparkPlan): SparkPlan = { + if (!conf.getConf(KyuubiSQLConf.FINAL_WRITE_STAGE_EAGERLY_KILL_EXECUTORS_ENABLED)) { + return plan + } + + if (!WriteUtils.isWrite(session, plan)) { + return plan + } + + val sc = session.sparkContext + val dra = sc.getConf.getBoolean("spark.dynamicAllocation.enabled", false) + val coresPerExecutor = sc.getConf.getInt("spark.executor.cores", 1) + val minExecutors = sc.getConf.getInt("spark.dynamicAllocation.minExecutors", 0) + val maxExecutors = sc.getConf.getInt("spark.dynamicAllocation.maxExecutors", Int.MaxValue) + val factor = conf.getConf(KyuubiSQLConf.FINAL_WRITE_STAGE_PARTITION_FACTOR) + val hasImprovementRoom = maxExecutors - 1 > minExecutors * factor + // Fast fail if: + // 1. DRA off + // 2. only work with yarn and k8s + // 3. maxExecutors is not bigger than minExecutors * factor + if (!dra || !sc.schedulerBackend.isInstanceOf[CoarseGrainedSchedulerBackend] || + !hasImprovementRoom) { + return plan + } + + val stageOpt = findFinalRebalanceStage(plan) + if (stageOpt.isEmpty) { + return plan + } + + // It's not safe to kill executors if this plan contains table cache. + // If the executor loses then the rdd would re-compute those partition. + if (hasTableCache(plan) && + conf.getConf(KyuubiSQLConf.FINAL_WRITE_STAGE_SKIP_KILLING_EXECUTORS_FOR_TABLE_CACHE)) { + return plan + } + + // TODO: move this to query stage optimizer when updating Spark to 3.5.x + // Since we are in `prepareQueryStage`, the AQE shuffle read has not been applied. + // So we need to apply it by self. + val shuffleRead = queryStageOptimizerRules.foldLeft(stageOpt.get.asInstanceOf[SparkPlan]) { + case (latest, rule) => rule.apply(latest) + } + val (targetCores, stage) = shuffleRead match { + case AQEShuffleReadExec(stage: ShuffleQueryStageExec, partitionSpecs) => + (partitionSpecs.length, stage) + case stage: ShuffleQueryStageExec => + // we can still kill executors if no AQE shuffle read, e.g., `.repartition(2)` + (stage.shuffle.numPartitions, stage) + case _ => + // it should never happen in current Spark, but to be safe do nothing if happens + logWarning("BUG, Please report to Apache Kyuubi community") + return plan + } + // The condition whether inject custom resource profile: + // - target executors < active executors + // - active executors - target executors > min executors + val numActiveExecutors = sc.getExecutorIds().length + val targetExecutors = (math.ceil(targetCores.toFloat / coresPerExecutor) * factor).toInt + .max(1) + val hasBenefits = targetExecutors < numActiveExecutors && + (numActiveExecutors - targetExecutors) > minExecutors + logInfo(s"The snapshot of current executors view, " + + s"active executors: $numActiveExecutors, min executor: $minExecutors, " + + s"target executors: $targetExecutors, has benefits: $hasBenefits") + if (hasBenefits) { + val shuffleId = stage.plan.asInstanceOf[ShuffleExchangeExec].shuffleDependency.shuffleId + val numReduce = stage.plan.asInstanceOf[ShuffleExchangeExec].numPartitions + // Now, there is only a final rebalance stage waiting to execute and all tasks of previous + // stage are finished. Kill redundant existed executors eagerly so the tasks of final + // stage can be centralized scheduled. + killExecutors(sc, targetExecutors, shuffleId, numReduce) + } + + plan + } + + /** + * The priority of kill executors follow: + * 1. kill executor who is younger than other (The older the JIT works better) + * 2. kill executor who produces less shuffle data first + */ + private def findExecutorToKill( + sc: SparkContext, + targetExecutors: Int, + shuffleId: Int, + numReduce: Int): Seq[String] = { + val tracker = SparkEnv.get.mapOutputTracker.asInstanceOf[MapOutputTrackerMaster] + val shuffleStatusOpt = tracker.shuffleStatuses.get(shuffleId) + if (shuffleStatusOpt.isEmpty) { + return Seq.empty + } + val shuffleStatus = shuffleStatusOpt.get + val executorToBlockSize = new mutable.HashMap[String, Long] + shuffleStatus.withMapStatuses { mapStatus => + mapStatus.foreach { status => + var i = 0 + var sum = 0L + while (i < numReduce) { + sum += status.getSizeForBlock(i) + i += 1 + } + executorToBlockSize.getOrElseUpdate(status.location.executorId, sum) + } + } + + val backend = sc.schedulerBackend.asInstanceOf[CoarseGrainedSchedulerBackend] + val executorsWithRegistrationTs = backend.getExecutorsWithRegistrationTs() + val existedExecutors = executorsWithRegistrationTs.keys.toSet + val expectedNumExecutorToKill = existedExecutors.size - targetExecutors + if (expectedNumExecutorToKill < 1) { + return Seq.empty + } + + val executorIdsToKill = new ArrayBuffer[String]() + // We first kill executor who does not hold shuffle block. It would happen because + // the last stage is running fast and finished in a short time. The existed executors are + // from previous stages that have not been killed by DRA, so we can not find it by tracking + // shuffle status. + // We should evict executors by their alive time first and retain all of executors which + // have better locality for shuffle block. + executorsWithRegistrationTs.toSeq.sortBy(_._2).foreach { case (id, _) => + if (executorIdsToKill.length < expectedNumExecutorToKill && + !executorToBlockSize.contains(id)) { + executorIdsToKill.append(id) + } + } + + // Evict the rest executors according to the shuffle block size + executorToBlockSize.toSeq.sortBy(_._2).foreach { case (id, _) => + if (executorIdsToKill.length < expectedNumExecutorToKill && existedExecutors.contains(id)) { + executorIdsToKill.append(id) + } + } + + executorIdsToKill.toSeq + } + + private def killExecutors( + sc: SparkContext, + targetExecutors: Int, + shuffleId: Int, + numReduce: Int): Unit = { + val executorAllocationClient = sc.schedulerBackend.asInstanceOf[ExecutorAllocationClient] + + val executorsToKill = + if (conf.getConf(KyuubiSQLConf.FINAL_WRITE_STAGE_EAGERLY_KILL_EXECUTORS_KILL_ALL)) { + executorAllocationClient.getExecutorIds() + } else { + findExecutorToKill(sc, targetExecutors, shuffleId, numReduce) + } + logInfo(s"Request to kill executors, total count ${executorsToKill.size}, " + + s"[${executorsToKill.mkString(", ")}].") + if (executorsToKill.isEmpty) { + return + } + + // Note, `SparkContext#killExecutors` does not allow with DRA enabled, + // see `https://github.com/apache/spark/pull/20604`. + // It may cause the status in `ExecutorAllocationManager` inconsistent with + // `CoarseGrainedSchedulerBackend` for a while. But it should be synchronous finally. + // + // We should adjust target num executors, otherwise `YarnAllocator` might re-request original + // target executors if DRA has not updated target executors yet. + // Note, DRA would re-adjust executors if there are more tasks to be executed, so we are safe. + // + // * We kill executor + // * YarnAllocator re-request target executors + // * DRA can not release executors since they are new added + // ----------------------------------------------------------------> timeline + executorAllocationClient.killExecutors( + executorIds = executorsToKill, + adjustTargetNumExecutors = true, + countFailures = false, + force = false) + + FinalStageResourceManager.getAdjustedTargetExecutors(sc) + .filter(_ < targetExecutors).foreach { adjustedExecutors => + val delta = targetExecutors - adjustedExecutors + logInfo(s"Target executors after kill ($adjustedExecutors) is lower than required " + + s"($targetExecutors). Requesting $delta additional executor(s).") + executorAllocationClient.requestExecutors(delta) + } + } + + @transient private val queryStageOptimizerRules: Seq[Rule[SparkPlan]] = Seq( + OptimizeSkewInRebalancePartitions, + CoalesceShufflePartitions(session), + OptimizeShuffleWithLocalRead) +} + +object FinalStageResourceManager extends Logging { + + private[sql] def getAdjustedTargetExecutors(sc: SparkContext): Option[Int] = { + sc.schedulerBackend match { + case schedulerBackend: CoarseGrainedSchedulerBackend => + try { + val field = classOf[CoarseGrainedSchedulerBackend] + .getDeclaredField("requestedTotalExecutorsPerResourceProfile") + field.setAccessible(true) + schedulerBackend.synchronized { + val requestedTotalExecutorsPerResourceProfile = + field.get(schedulerBackend).asInstanceOf[mutable.HashMap[ResourceProfile, Int]] + val defaultRp = sc.resourceProfileManager.defaultResourceProfile + requestedTotalExecutorsPerResourceProfile.get(defaultRp) + } + } catch { + case e: Exception => + logWarning("Failed to get requestedTotalExecutors of Default ResourceProfile", e) + None + } + case _ => None + } + } +} + +trait FinalRebalanceStageHelper extends AdaptiveSparkPlanHelper { + @tailrec + final protected def findFinalRebalanceStage(plan: SparkPlan): Option[ShuffleQueryStageExec] = { + plan match { + case write: DataWritingCommandExec => findFinalRebalanceStage(write.child) + case write: V2TableWriteExec => findFinalRebalanceStage(write.child) + case write: WriteFilesExec => findFinalRebalanceStage(write.child) + case p: ProjectExec => findFinalRebalanceStage(p.child) + case f: FilterExec => findFinalRebalanceStage(f.child) + case s: SortExec if !s.global => findFinalRebalanceStage(s.child) + case stage: ShuffleQueryStageExec + if stage.isMaterialized && stage.mapStats.isDefined && + stage.plan.isInstanceOf[ShuffleExchangeExec] && + stage.plan.asInstanceOf[ShuffleExchangeExec].shuffleOrigin != ENSURE_REQUIREMENTS => + Some(stage) + case _ => None + } + } + + final protected def hasTableCache(plan: SparkPlan): Boolean = { + find(plan) { + case _: InMemoryTableScanExec => true + case _ => false + }.isDefined + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/spark/sql/InjectCustomResourceProfile.scala b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/spark/sql/InjectCustomResourceProfile.scala new file mode 100644 index 000000000..64421d6bf --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/spark/sql/InjectCustomResourceProfile.scala @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.execution.{CustomResourceProfileExec, SparkPlan} +import org.apache.spark.sql.execution.adaptive._ + +import org.apache.kyuubi.sql.{KyuubiSQLConf, WriteUtils} + +/** + * Inject custom resource profile for final write stage, so we can specify custom + * executor resource configs. + */ +case class InjectCustomResourceProfile(session: SparkSession) + extends Rule[SparkPlan] with FinalRebalanceStageHelper { + override def apply(plan: SparkPlan): SparkPlan = { + if (!conf.getConf(KyuubiSQLConf.FINAL_WRITE_STAGE_RESOURCE_ISOLATION_ENABLED)) { + return plan + } + + if (!WriteUtils.isWrite(session, plan)) { + return plan + } + + val stage = findFinalRebalanceStage(plan) + if (stage.isEmpty) { + return plan + } + + // TODO: Ideally, We can call `CoarseGrainedSchedulerBackend.requestTotalExecutors` eagerly + // to reduce the task submit pending time, but it may lose task locality. + // + // By default, it would request executors when catch stage submit event. + injectCustomResourceProfile(plan, stage.get.id) + } + + private def injectCustomResourceProfile(plan: SparkPlan, id: Int): SparkPlan = { + plan match { + case stage: ShuffleQueryStageExec if stage.id == id => + CustomResourceProfileExec(stage) + case _ => plan.mapChildren(child => injectCustomResourceProfile(child, id)) + } + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/spark/sql/PruneFileSourcePartitionHelper.scala b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/spark/sql/PruneFileSourcePartitionHelper.scala new file mode 100644 index 000000000..ce496eb47 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/spark/sql/PruneFileSourcePartitionHelper.scala @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +import org.apache.spark.sql.catalyst.expressions.{AttributeReference, AttributeSet, Expression, ExpressionSet, PredicateHelper, SubqueryExpression} +import org.apache.spark.sql.catalyst.plans.logical.LeafNode +import org.apache.spark.sql.execution.datasources.DataSourceStrategy +import org.apache.spark.sql.types.StructType + +trait PruneFileSourcePartitionHelper extends PredicateHelper { + + def getPartitionKeyFiltersAndDataFilters( + sparkSession: SparkSession, + relation: LeafNode, + partitionSchema: StructType, + filters: Seq[Expression], + output: Seq[AttributeReference]): (ExpressionSet, Seq[Expression]) = { + val normalizedFilters = DataSourceStrategy.normalizeExprs( + filters.filter(f => f.deterministic && !SubqueryExpression.hasSubquery(f)), + output) + val partitionColumns = + relation.resolve(partitionSchema, sparkSession.sessionState.analyzer.resolver) + val partitionSet = AttributeSet(partitionColumns) + val (partitionFilters, dataFilters) = normalizedFilters.partition(f => + f.references.subsetOf(partitionSet)) + val extraPartitionFilter = + dataFilters.flatMap(extractPredicatesWithinOutputSet(_, partitionSet)) + + (ExpressionSet(partitionFilters ++ extraPartitionFilter), dataFilters) + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/spark/sql/execution/CustomResourceProfileExec.scala b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/spark/sql/execution/CustomResourceProfileExec.scala new file mode 100644 index 000000000..3698140fb --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/main/scala/org/apache/spark/sql/execution/CustomResourceProfileExec.scala @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution + +import org.apache.spark.network.util.{ByteUnit, JavaUtils} +import org.apache.spark.rdd.RDD +import org.apache.spark.resource.{ExecutorResourceRequests, ResourceProfileBuilder} +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions.{Attribute, SortOrder} +import org.apache.spark.sql.catalyst.plans.physical.Partitioning +import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics} +import org.apache.spark.sql.vectorized.ColumnarBatch +import org.apache.spark.util.Utils + +import org.apache.kyuubi.sql.KyuubiSQLConf._ + +/** + * This node wraps the final executed plan and inject custom resource profile to the RDD. + * It assumes that, the produced RDD would create the `ResultStage` in `DAGScheduler`, + * so it makes resource isolation between previous and final stage. + * + * Note that, Spark does not support config `minExecutors` for each resource profile. + * Which means, it would retain `minExecutors` for each resource profile. + * So, suggest set `spark.dynamicAllocation.minExecutors` to 0 if enable this feature. + */ +case class CustomResourceProfileExec(child: SparkPlan) extends UnaryExecNode { + override def output: Seq[Attribute] = child.output + override def outputPartitioning: Partitioning = child.outputPartitioning + override def outputOrdering: Seq[SortOrder] = child.outputOrdering + override def supportsColumnar: Boolean = child.supportsColumnar + override def supportsRowBased: Boolean = child.supportsRowBased + override protected def doCanonicalize(): SparkPlan = child.canonicalized + + private val executorCores = conf.getConf(FINAL_WRITE_STAGE_EXECUTOR_CORES).getOrElse( + sparkContext.getConf.getInt("spark.executor.cores", 1)) + private val executorMemory = conf.getConf(FINAL_WRITE_STAGE_EXECUTOR_MEMORY).getOrElse( + sparkContext.getConf.get("spark.executor.memory", "2G")) + private val executorMemoryOverhead = + conf.getConf(FINAL_WRITE_STAGE_EXECUTOR_MEMORY_OVERHEAD) + .getOrElse(sparkContext.getConf.get("spark.executor.memoryOverhead", "1G")) + private val executorOffHeapMemory = conf.getConf(FINAL_WRITE_STAGE_EXECUTOR_OFF_HEAP_MEMORY) + + override lazy val metrics: Map[String, SQLMetric] = { + val base = Map( + "executorCores" -> SQLMetrics.createMetric(sparkContext, "executor cores"), + "executorMemory" -> SQLMetrics.createMetric(sparkContext, "executor memory (MiB)"), + "executorMemoryOverhead" -> SQLMetrics.createMetric( + sparkContext, + "executor memory overhead (MiB)")) + val addition = executorOffHeapMemory.map(_ => + "executorOffHeapMemory" -> + SQLMetrics.createMetric(sparkContext, "executor off heap memory (MiB)")).toMap + base ++ addition + } + + private def wrapResourceProfile[T](rdd: RDD[T]): RDD[T] = { + if (Utils.isTesting) { + // do nothing for local testing + return rdd + } + + metrics("executorCores") += executorCores + metrics("executorMemory") += JavaUtils.byteStringAs(executorMemory, ByteUnit.MiB) + metrics("executorMemoryOverhead") += JavaUtils.byteStringAs( + executorMemoryOverhead, + ByteUnit.MiB) + executorOffHeapMemory.foreach(m => + metrics("executorOffHeapMemory") += JavaUtils.byteStringAs(m, ByteUnit.MiB)) + + val executionId = sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY) + SQLMetrics.postDriverMetricUpdates(sparkContext, executionId, metrics.values.toSeq) + + val resourceProfileBuilder = new ResourceProfileBuilder() + val executorResourceRequests = new ExecutorResourceRequests() + executorResourceRequests.cores(executorCores) + executorResourceRequests.memory(executorMemory) + executorResourceRequests.memoryOverhead(executorMemoryOverhead) + executorOffHeapMemory.foreach(executorResourceRequests.offHeapMemory) + resourceProfileBuilder.require(executorResourceRequests) + rdd.withResources(resourceProfileBuilder.build()) + rdd + } + + override protected def doExecute(): RDD[InternalRow] = { + val rdd = child.execute() + wrapResourceProfile(rdd) + } + + override protected def doExecuteColumnar(): RDD[ColumnarBatch] = { + val rdd = child.executeColumnar() + wrapResourceProfile(rdd) + } + + override protected def withNewChildInternal(newChild: SparkPlan): SparkPlan = { + this.copy(child = newChild) + } +} diff --git a/extensions/spark/kyuubi-spark-connector-kudu/src/test/resources/log4j2-test.xml b/extensions/spark/kyuubi-extension-spark-3-4/src/test/resources/log4j2-test.xml similarity index 98% rename from extensions/spark/kyuubi-spark-connector-kudu/src/test/resources/log4j2-test.xml rename to extensions/spark/kyuubi-extension-spark-3-4/src/test/resources/log4j2-test.xml index bfc40dd6d..3110216c1 100644 --- a/extensions/spark/kyuubi-spark-connector-kudu/src/test/resources/log4j2-test.xml +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/test/resources/log4j2-test.xml @@ -21,14 +21,14 @@ - + - + diff --git a/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/DropIgnoreNonexistentSuite.scala b/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/DropIgnoreNonexistentSuite.scala new file mode 100644 index 000000000..bbc61fb44 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/DropIgnoreNonexistentSuite.scala @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +import org.apache.spark.sql.catalyst.plans.logical.{DropNamespace, NoopCommand} +import org.apache.spark.sql.execution.command._ + +import org.apache.kyuubi.sql.KyuubiSQLConf + +class DropIgnoreNonexistentSuite extends KyuubiSparkSQLExtensionTest { + + test("drop ignore nonexistent") { + withSQLConf(KyuubiSQLConf.DROP_IGNORE_NONEXISTENT.key -> "true") { + // drop nonexistent database + val df1 = sql("DROP DATABASE nonexistent_database") + assert(df1.queryExecution.analyzed.asInstanceOf[DropNamespace].ifExists == true) + + // drop nonexistent function + val df4 = sql("DROP FUNCTION nonexistent_function") + assert(df4.queryExecution.analyzed.isInstanceOf[NoopCommand]) + + // drop nonexistent PARTITION + withTable("test") { + sql("CREATE TABLE IF NOT EXISTS test(i int) PARTITIONED BY (p int)") + val df5 = sql("ALTER TABLE test DROP PARTITION (p = 1)") + assert(df5.queryExecution.analyzed + .asInstanceOf[AlterTableDropPartitionCommand].ifExists == true) + } + } + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/FinalStageConfigIsolationSuite.scala b/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/FinalStageConfigIsolationSuite.scala new file mode 100644 index 000000000..96c8ae6e8 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/FinalStageConfigIsolationSuite.scala @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +import org.apache.spark.sql.execution.adaptive.{AQEShuffleReadExec, QueryStageExec} +import org.apache.spark.sql.internal.SQLConf + +import org.apache.kyuubi.sql.{FinalStageConfigIsolation, KyuubiSQLConf} + +class FinalStageConfigIsolationSuite extends KyuubiSparkSQLExtensionTest { + override protected def beforeAll(): Unit = { + super.beforeAll() + setupData() + } + + test("final stage config set reset check") { + withSQLConf( + KyuubiSQLConf.FINAL_STAGE_CONFIG_ISOLATION.key -> "true", + KyuubiSQLConf.FINAL_STAGE_CONFIG_ISOLATION_WRITE_ONLY.key -> "false", + "spark.sql.finalStage.adaptive.coalescePartitions.minPartitionNum" -> "1", + "spark.sql.finalStage.adaptive.advisoryPartitionSizeInBytes" -> "100") { + // use loop to double check final stage config doesn't affect the sql query each other + (1 to 3).foreach { _ => + sql("SELECT COUNT(*) FROM VALUES(1) as t(c)").collect() + assert(spark.sessionState.conf.getConfString( + "spark.sql.previousStage.adaptive.coalescePartitions.minPartitionNum") === + FinalStageConfigIsolation.INTERNAL_UNSET_CONFIG_TAG) + assert(spark.sessionState.conf.getConfString( + "spark.sql.adaptive.coalescePartitions.minPartitionNum") === + "1") + assert(spark.sessionState.conf.getConfString( + "spark.sql.finalStage.adaptive.coalescePartitions.minPartitionNum") === + "1") + + // 64MB + assert(spark.sessionState.conf.getConfString( + "spark.sql.previousStage.adaptive.advisoryPartitionSizeInBytes") === + "67108864b") + assert(spark.sessionState.conf.getConfString( + "spark.sql.adaptive.advisoryPartitionSizeInBytes") === + "100") + assert(spark.sessionState.conf.getConfString( + "spark.sql.finalStage.adaptive.advisoryPartitionSizeInBytes") === + "100") + } + + sql("SET spark.sql.adaptive.advisoryPartitionSizeInBytes=1") + assert(spark.sessionState.conf.getConfString( + "spark.sql.adaptive.advisoryPartitionSizeInBytes") === + "1") + assert(!spark.sessionState.conf.contains( + "spark.sql.previousStage.adaptive.advisoryPartitionSizeInBytes")) + + sql("SET a=1") + assert(spark.sessionState.conf.getConfString("a") === "1") + + sql("RESET spark.sql.adaptive.coalescePartitions.minPartitionNum") + assert(!spark.sessionState.conf.contains( + "spark.sql.adaptive.coalescePartitions.minPartitionNum")) + assert(!spark.sessionState.conf.contains( + "spark.sql.previousStage.adaptive.coalescePartitions.minPartitionNum")) + + sql("RESET a") + assert(!spark.sessionState.conf.contains("a")) + } + } + + test("final stage config isolation") { + def checkPartitionNum( + sqlString: String, + previousPartitionNum: Int, + finalPartitionNum: Int): Unit = { + val df = sql(sqlString) + df.collect() + val shuffleReaders = collect(df.queryExecution.executedPlan) { + case customShuffleReader: AQEShuffleReadExec => customShuffleReader + } + assert(shuffleReaders.nonEmpty) + // reorder stage by stage id to ensure we get the right stage + val sortedShuffleReaders = shuffleReaders.sortWith { + case (s1, s2) => + s1.child.asInstanceOf[QueryStageExec].id < s2.child.asInstanceOf[QueryStageExec].id + } + if (sortedShuffleReaders.length > 1) { + assert(sortedShuffleReaders.head.partitionSpecs.length === previousPartitionNum) + } + assert(sortedShuffleReaders.last.partitionSpecs.length === finalPartitionNum) + assert(df.rdd.partitions.length === finalPartitionNum) + } + + withSQLConf( + SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", + SQLConf.COALESCE_PARTITIONS_MIN_PARTITION_NUM.key -> "1", + SQLConf.SHUFFLE_PARTITIONS.key -> "3", + KyuubiSQLConf.FINAL_STAGE_CONFIG_ISOLATION.key -> "true", + KyuubiSQLConf.FINAL_STAGE_CONFIG_ISOLATION_WRITE_ONLY.key -> "false", + "spark.sql.adaptive.advisoryPartitionSizeInBytes" -> "1", + "spark.sql.adaptive.coalescePartitions.minPartitionSize" -> "1", + "spark.sql.finalStage.adaptive.advisoryPartitionSizeInBytes" -> "10000000") { + + // use loop to double check final stage config doesn't affect the sql query each other + (1 to 3).foreach { _ => + checkPartitionNum( + "SELECT c1, count(*) FROM t1 GROUP BY c1", + 1, + 1) + + checkPartitionNum( + "SELECT c2, count(*) FROM (SELECT c1, count(*) as c2 FROM t1 GROUP BY c1) GROUP BY c2", + 3, + 1) + + checkPartitionNum( + "SELECT t1.c1, count(*) FROM t1 JOIN t2 ON t1.c2 = t2.c2 GROUP BY t1.c1", + 3, + 1) + + checkPartitionNum( + """ + | SELECT /*+ REPARTITION */ + | t1.c1, count(*) FROM t1 + | JOIN t2 ON t1.c2 = t2.c2 + | JOIN t3 ON t1.c1 = t3.c1 + | GROUP BY t1.c1 + |""".stripMargin, + 3, + 1) + + // one shuffle reader + checkPartitionNum( + """ + | SELECT /*+ BROADCAST(t1) */ + | t1.c1, t2.c2 FROM t1 + | JOIN t2 ON t1.c2 = t2.c2 + | DISTRIBUTE BY c1 + |""".stripMargin, + 1, + 1) + + // test ReusedExchange + checkPartitionNum( + """ + |SELECT /*+ REPARTITION */ t0.c2 FROM ( + |SELECT t1.c1, (count(*) + c1) as c2 FROM t1 GROUP BY t1.c1 + |) t0 JOIN ( + |SELECT t1.c1, (count(*) + c1) as c2 FROM t1 GROUP BY t1.c1 + |) t1 ON t0.c2 = t1.c2 + |""".stripMargin, + 3, + 1) + + // one shuffle reader + checkPartitionNum( + """ + |SELECT t0.c1 FROM ( + |SELECT t1.c1 FROM t1 GROUP BY t1.c1 + |) t0 JOIN ( + |SELECT t1.c1 FROM t1 GROUP BY t1.c1 + |) t1 ON t0.c1 = t1.c1 + |""".stripMargin, + 1, + 1) + } + } + } + + test("final stage config isolation write only") { + withSQLConf( + KyuubiSQLConf.FINAL_STAGE_CONFIG_ISOLATION.key -> "true", + KyuubiSQLConf.FINAL_STAGE_CONFIG_ISOLATION_WRITE_ONLY.key -> "true", + "spark.sql.finalStage.adaptive.advisoryPartitionSizeInBytes" -> "7") { + sql("set spark.sql.adaptive.advisoryPartitionSizeInBytes=5") + sql("SELECT * FROM t1").count() + assert(spark.conf.getOption("spark.sql.adaptive.advisoryPartitionSizeInBytes") + .contains("5")) + + withTable("tmp") { + sql("CREATE TABLE t1 USING PARQUET SELECT /*+ repartition */ 1 AS c1, 'a' AS c2") + assert(spark.conf.getOption("spark.sql.adaptive.advisoryPartitionSizeInBytes") + .contains("7")) + } + + sql("SELECT * FROM t1").count() + assert(spark.conf.getOption("spark.sql.adaptive.advisoryPartitionSizeInBytes") + .contains("5")) + } + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/FinalStageResourceManagerSuite.scala b/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/FinalStageResourceManagerSuite.scala new file mode 100644 index 000000000..4b9991ef6 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/FinalStageResourceManagerSuite.scala @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +import org.apache.spark.SparkConf +import org.scalatest.time.{Minutes, Span} + +import org.apache.kyuubi.sql.KyuubiSQLConf +import org.apache.kyuubi.tags.SparkLocalClusterTest + +@SparkLocalClusterTest +class FinalStageResourceManagerSuite extends KyuubiSparkSQLExtensionTest { + + override def sparkConf(): SparkConf = { + // It is difficult to run spark in local-cluster mode when spark.testing is set. + sys.props.remove("spark.testing") + + super.sparkConf().set("spark.master", "local-cluster[3, 1, 1024]") + .set("spark.dynamicAllocation.enabled", "true") + .set("spark.dynamicAllocation.initialExecutors", "3") + .set("spark.dynamicAllocation.minExecutors", "1") + .set("spark.dynamicAllocation.shuffleTracking.enabled", "true") + .set(KyuubiSQLConf.FINAL_STAGE_CONFIG_ISOLATION.key, "true") + .set(KyuubiSQLConf.FINAL_WRITE_STAGE_EAGERLY_KILL_EXECUTORS_ENABLED.key, "true") + } + + test("[KYUUBI #5136][Bug] Final Stage hangs forever") { + // Prerequisite to reproduce the bug: + // 1. Dynamic allocation is enabled. + // 2. Dynamic allocation min executors is 1. + // 3. target executors < active executors. + // 4. No active executor is left after FinalStageResourceManager killed executors. + // This is possible because FinalStageResourceManager retained executors may already be + // requested to be killed but not died yet. + // 5. Final Stage required executors is 1. + withSQLConf( + (KyuubiSQLConf.FINAL_WRITE_STAGE_EAGERLY_KILL_EXECUTORS_KILL_ALL.key, "true")) { + withTable("final_stage") { + eventually(timeout(Span(10, Minutes))) { + sql( + "CREATE TABLE final_stage AS SELECT id, count(*) as num FROM (SELECT 0 id) GROUP BY id") + } + assert(FinalStageResourceManager.getAdjustedTargetExecutors(spark.sparkContext).get == 1) + } + } + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/InjectResourceProfileSuite.scala b/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/InjectResourceProfileSuite.scala new file mode 100644 index 000000000..b0767b187 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/InjectResourceProfileSuite.scala @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +import org.apache.spark.scheduler.{SparkListener, SparkListenerEvent} +import org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate + +import org.apache.kyuubi.sql.KyuubiSQLConf + +class InjectResourceProfileSuite extends KyuubiSparkSQLExtensionTest { + private def checkCustomResourceProfile(sqlString: String, exists: Boolean): Unit = { + @volatile var lastEvent: SparkListenerSQLAdaptiveExecutionUpdate = null + val listener = new SparkListener { + override def onOtherEvent(event: SparkListenerEvent): Unit = { + event match { + case e: SparkListenerSQLAdaptiveExecutionUpdate => lastEvent = e + case _ => + } + } + } + + spark.sparkContext.addSparkListener(listener) + try { + sql(sqlString).collect() + spark.sparkContext.listenerBus.waitUntilEmpty() + assert(lastEvent != null) + var current = lastEvent.sparkPlanInfo + var shouldStop = false + while (!shouldStop) { + if (current.nodeName != "CustomResourceProfile") { + if (current.children.isEmpty) { + assert(!exists) + shouldStop = true + } else { + current = current.children.head + } + } else { + assert(exists) + shouldStop = true + } + } + } finally { + spark.sparkContext.removeSparkListener(listener) + } + } + + test("Inject resource profile") { + withTable("t") { + withSQLConf( + "spark.sql.adaptive.forceApply" -> "true", + KyuubiSQLConf.FINAL_STAGE_CONFIG_ISOLATION.key -> "true", + KyuubiSQLConf.FINAL_WRITE_STAGE_RESOURCE_ISOLATION_ENABLED.key -> "true") { + + sql("CREATE TABLE t (c1 int, c2 string) USING PARQUET") + + checkCustomResourceProfile("INSERT INTO TABLE t VALUES(1, 'a')", false) + checkCustomResourceProfile("SELECT 1", false) + checkCustomResourceProfile( + "INSERT INTO TABLE t SELECT /*+ rebalance */ * FROM VALUES(1, 'a')", + true) + } + } + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/InsertShuffleNodeBeforeJoinSuite.scala b/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/InsertShuffleNodeBeforeJoinSuite.scala new file mode 100644 index 000000000..f0d384657 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/InsertShuffleNodeBeforeJoinSuite.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class InsertShuffleNodeBeforeJoinSuite extends InsertShuffleNodeBeforeJoinSuiteBase diff --git a/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/InsertShuffleNodeBeforeJoinSuiteBase.scala b/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/InsertShuffleNodeBeforeJoinSuiteBase.scala new file mode 100644 index 000000000..c657dee49 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/InsertShuffleNodeBeforeJoinSuiteBase.scala @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +import org.apache.spark.SparkConf +import org.apache.spark.sql.execution.exchange.{ENSURE_REQUIREMENTS, ShuffleExchangeLike} +import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf} + +import org.apache.kyuubi.sql.KyuubiSQLConf + +trait InsertShuffleNodeBeforeJoinSuiteBase extends KyuubiSparkSQLExtensionTest { + override protected def beforeAll(): Unit = { + super.beforeAll() + setupData() + } + + override def sparkConf(): SparkConf = { + super.sparkConf() + .set( + StaticSQLConf.SPARK_SESSION_EXTENSIONS.key, + "org.apache.kyuubi.sql.KyuubiSparkSQLCommonExtension") + } + + test("force shuffle before join") { + def checkShuffleNodeNum(sqlString: String, num: Int): Unit = { + var expectedResult: Seq[Row] = Seq.empty + withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") { + expectedResult = sql(sqlString).collect() + } + val df = sql(sqlString) + checkAnswer(df, expectedResult) + assert( + collect(df.queryExecution.executedPlan) { + case shuffle: ShuffleExchangeLike if shuffle.shuffleOrigin == ENSURE_REQUIREMENTS => + shuffle + }.size == num) + } + + withSQLConf( + SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", + KyuubiSQLConf.FORCE_SHUFFLE_BEFORE_JOIN.key -> "true") { + Seq("SHUFFLE_HASH", "MERGE").foreach { joinHint => + // positive case + checkShuffleNodeNum( + s""" + |SELECT /*+ $joinHint(t2, t3) */ t1.c1, t1.c2, t2.c1, t3.c1 from t1 + | JOIN t2 ON t1.c1 = t2.c1 + | JOIN t3 ON t1.c1 = t3.c1 + | """.stripMargin, + 4) + + // negative case + checkShuffleNodeNum( + s""" + |SELECT /*+ $joinHint(t2, t3) */ t1.c1, t1.c2, t2.c1, t3.c1 from t1 + | JOIN t2 ON t1.c1 = t2.c1 + | JOIN t3 ON t1.c2 = t3.c2 + | """.stripMargin, + 4) + } + + checkShuffleNodeNum( + """ + |SELECT t1.c1, t2.c1, t3.c2 from t1 + | JOIN t2 ON t1.c1 = t2.c1 + | JOIN ( + | SELECT c2, count(*) FROM t1 GROUP BY c2 + | ) t3 ON t1.c1 = t3.c2 + | """.stripMargin, + 5) + + checkShuffleNodeNum( + """ + |SELECT t1.c1, t2.c1, t3.c1 from t1 + | JOIN t2 ON t1.c1 = t2.c1 + | JOIN ( + | SELECT c1, count(*) FROM t1 GROUP BY c1 + | ) t3 ON t1.c1 = t3.c1 + | """.stripMargin, + 5) + } + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/KyuubiSparkSQLExtensionTest.scala b/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/KyuubiSparkSQLExtensionTest.scala new file mode 100644 index 000000000..dd9ffbf16 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/KyuubiSparkSQLExtensionTest.scala @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +import org.apache.hadoop.hive.conf.HiveConf.ConfVars +import org.apache.spark.SparkConf +import org.apache.spark.sql.execution.QueryExecution +import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper +import org.apache.spark.sql.execution.command.{DataWritingCommand, DataWritingCommandExec} +import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf} +import org.apache.spark.sql.test.SQLTestData.TestData +import org.apache.spark.sql.test.SQLTestUtils +import org.apache.spark.sql.util.QueryExecutionListener +import org.apache.spark.util.Utils + +import org.apache.kyuubi.sql.KyuubiSQLConf + +trait KyuubiSparkSQLExtensionTest extends QueryTest + with SQLTestUtils + with AdaptiveSparkPlanHelper { + sys.props.put("spark.testing", "1") + + private var _spark: Option[SparkSession] = None + protected def spark: SparkSession = _spark.getOrElse { + throw new RuntimeException("test spark session don't initial before using it.") + } + + override protected def beforeAll(): Unit = { + if (_spark.isEmpty) { + _spark = Option(SparkSession.builder() + .master("local[1]") + .config(sparkConf) + .enableHiveSupport() + .getOrCreate()) + } + super.beforeAll() + } + + override protected def afterAll(): Unit = { + super.afterAll() + cleanupData() + _spark.foreach(_.stop) + } + + protected def setupData(): Unit = { + val self = spark + import self.implicits._ + spark.sparkContext.parallelize( + (1 to 100).map(i => TestData(i, i.toString)), + 10) + .toDF("c1", "c2").createOrReplaceTempView("t1") + spark.sparkContext.parallelize( + (1 to 10).map(i => TestData(i, i.toString)), + 5) + .toDF("c1", "c2").createOrReplaceTempView("t2") + spark.sparkContext.parallelize( + (1 to 50).map(i => TestData(i, i.toString)), + 2) + .toDF("c1", "c2").createOrReplaceTempView("t3") + } + + private def cleanupData(): Unit = { + spark.sql("DROP VIEW IF EXISTS t1") + spark.sql("DROP VIEW IF EXISTS t2") + spark.sql("DROP VIEW IF EXISTS t3") + } + + def sparkConf(): SparkConf = { + val basePath = Utils.createTempDir() + "/" + getClass.getCanonicalName + val metastorePath = basePath + "/metastore_db" + val warehousePath = basePath + "/warehouse" + new SparkConf() + .set( + StaticSQLConf.SPARK_SESSION_EXTENSIONS.key, + "org.apache.kyuubi.sql.KyuubiSparkSQLExtension") + .set(KyuubiSQLConf.SQL_CLASSIFICATION_ENABLED.key, "true") + .set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "true") + .set("spark.hadoop.hive.exec.dynamic.partition.mode", "nonstrict") + .set("spark.hadoop.hive.metastore.client.capability.check", "false") + .set( + ConfVars.METASTORECONNECTURLKEY.varname, + s"jdbc:derby:;databaseName=$metastorePath;create=true") + .set(StaticSQLConf.WAREHOUSE_PATH, warehousePath) + .set("spark.ui.enabled", "false") + } + + def withListener(sqlString: String)(callback: DataWritingCommand => Unit): Unit = { + withListener(sql(sqlString))(callback) + } + + def withListener(df: => DataFrame)(callback: DataWritingCommand => Unit): Unit = { + val listener = new QueryExecutionListener { + override def onFailure(f: String, qe: QueryExecution, e: Exception): Unit = {} + + override def onSuccess(funcName: String, qe: QueryExecution, duration: Long): Unit = { + qe.executedPlan match { + case write: DataWritingCommandExec => callback(write.cmd) + case _ => + } + } + } + spark.listenerManager.register(listener) + try { + df.collect() + sparkContext.listenerBus.waitUntilEmpty() + } finally { + spark.listenerManager.unregister(listener) + } + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/RebalanceBeforeWritingSuite.scala b/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/RebalanceBeforeWritingSuite.scala new file mode 100644 index 000000000..1d9630f49 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/RebalanceBeforeWritingSuite.scala @@ -0,0 +1,271 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, RebalancePartitions, Sort} +import org.apache.spark.sql.execution.command.DataWritingCommand +import org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand +import org.apache.spark.sql.hive.HiveUtils +import org.apache.spark.sql.hive.execution.InsertIntoHiveTable + +import org.apache.kyuubi.sql.KyuubiSQLConf + +class RebalanceBeforeWritingSuite extends KyuubiSparkSQLExtensionTest { + + test("check rebalance exists") { + def check(df: => DataFrame, expectedRebalanceNum: Int = 1): Unit = { + withSQLConf(KyuubiSQLConf.INSERT_REPARTITION_BEFORE_WRITE_IF_NO_SHUFFLE.key -> "true") { + withListener(df) { write => + assert(write.collect { + case r: RebalancePartitions => r + }.size == expectedRebalanceNum) + } + } + withSQLConf(KyuubiSQLConf.INSERT_REPARTITION_BEFORE_WRITE_IF_NO_SHUFFLE.key -> "false") { + withListener(df) { write => + assert(write.collect { + case r: RebalancePartitions => r + }.isEmpty) + } + } + } + + // It's better to set config explicitly in case of we change the default value. + withSQLConf(KyuubiSQLConf.INSERT_REPARTITION_BEFORE_WRITE.key -> "true") { + Seq("USING PARQUET", "").foreach { storage => + withTable("tmp1") { + sql(s"CREATE TABLE tmp1 (c1 int) $storage PARTITIONED BY (c2 string)") + check(sql("INSERT INTO TABLE tmp1 PARTITION(c2='a') " + + "SELECT * FROM VALUES(1),(2) AS t(c1)")) + } + + withTable("tmp1", "tmp2") { + sql(s"CREATE TABLE tmp1 (c1 int) $storage PARTITIONED BY (c2 string)") + sql(s"CREATE TABLE tmp2 (c1 int) $storage PARTITIONED BY (c2 string)") + check( + sql( + """FROM VALUES(1),(2) + |INSERT INTO TABLE tmp1 PARTITION(c2='a') SELECT * + |INSERT INTO TABLE tmp2 PARTITION(c2='a') SELECT * + |""".stripMargin), + 2) + } + + withTable("tmp1") { + sql(s"CREATE TABLE tmp1 (c1 int) $storage") + check(sql("INSERT INTO TABLE tmp1 SELECT * FROM VALUES(1),(2),(3) AS t(c1)")) + } + + withTable("tmp1", "tmp2") { + sql(s"CREATE TABLE tmp1 (c1 int) $storage") + sql(s"CREATE TABLE tmp2 (c1 int) $storage") + check( + sql( + """FROM VALUES(1),(2),(3) + |INSERT INTO TABLE tmp1 SELECT * + |INSERT INTO TABLE tmp2 SELECT * + |""".stripMargin), + 2) + } + + withTable("tmp1") { + sql(s"CREATE TABLE tmp1 $storage AS SELECT * FROM VALUES(1),(2),(3) AS t(c1)") + } + + withTable("tmp1") { + sql(s"CREATE TABLE tmp1 $storage PARTITIONED BY(c2) AS " + + s"SELECT * FROM VALUES(1, 'a'),(2, 'b') AS t(c1, c2)") + } + } + } + } + + test("check rebalance does not exists") { + def check(df: DataFrame): Unit = { + withListener(df) { write => + assert(write.collect { + case r: RebalancePartitions => r + }.isEmpty) + } + } + + withSQLConf( + KyuubiSQLConf.INSERT_REPARTITION_BEFORE_WRITE.key -> "true", + KyuubiSQLConf.INSERT_REPARTITION_BEFORE_WRITE_IF_NO_SHUFFLE.key -> "true") { + // test no write command + check(sql("SELECT * FROM VALUES(1, 'a'),(2, 'b') AS t(c1, c2)")) + check(sql("SELECT count(*) FROM VALUES(1, 'a'),(2, 'b') AS t(c1, c2)")) + + // test not supported plan + withTable("tmp1") { + sql(s"CREATE TABLE tmp1 (c1 int) PARTITIONED BY (c2 string)") + check(sql("INSERT INTO TABLE tmp1 PARTITION(c2) " + + "SELECT /*+ repartition(10) */ * FROM VALUES(1, 'a'),(2, 'b') AS t(c1, c2)")) + check(sql("INSERT INTO TABLE tmp1 PARTITION(c2) " + + "SELECT * FROM VALUES(1, 'a'),(2, 'b') AS t(c1, c2) ORDER BY c1")) + check(sql("INSERT INTO TABLE tmp1 PARTITION(c2) " + + "SELECT * FROM VALUES(1, 'a'),(2, 'b') AS t(c1, c2) LIMIT 10")) + } + } + + withSQLConf(KyuubiSQLConf.INSERT_REPARTITION_BEFORE_WRITE.key -> "false") { + Seq("USING PARQUET", "").foreach { storage => + withTable("tmp1") { + sql(s"CREATE TABLE tmp1 (c1 int) $storage PARTITIONED BY (c2 string)") + check(sql("INSERT INTO TABLE tmp1 PARTITION(c2) " + + "SELECT * FROM VALUES(1, 'a'),(2, 'b') AS t(c1, c2)")) + } + + withTable("tmp1") { + sql(s"CREATE TABLE tmp1 (c1 int) $storage") + check(sql("INSERT INTO TABLE tmp1 SELECT * FROM VALUES(1),(2),(3) AS t(c1)")) + } + } + } + } + + test("test dynamic partition write") { + def checkRepartitionExpression(sqlString: String): Unit = { + withListener(sqlString) { write => + assert(write.isInstanceOf[InsertIntoHiveTable]) + assert(write.collect { + case r: RebalancePartitions if r.partitionExpressions.size == 1 => + assert(r.partitionExpressions.head.asInstanceOf[Attribute].name === "c2") + r + }.size == 1) + } + } + + withSQLConf( + KyuubiSQLConf.INSERT_REPARTITION_BEFORE_WRITE.key -> "true", + KyuubiSQLConf.DYNAMIC_PARTITION_INSERTION_REPARTITION_NUM.key -> "2", + KyuubiSQLConf.INSERT_REPARTITION_BEFORE_WRITE_IF_NO_SHUFFLE.key -> "true") { + Seq("USING PARQUET", "").foreach { storage => + withTable("tmp1") { + sql(s"CREATE TABLE tmp1 (c1 int) $storage PARTITIONED BY (c2 string)") + checkRepartitionExpression("INSERT INTO TABLE tmp1 SELECT 1 as c1, 'a' as c2 ") + } + + withTable("tmp1") { + checkRepartitionExpression( + "CREATE TABLE tmp1 PARTITIONED BY(C2) SELECT 1 as c1, 'a' as c2") + } + } + } + } + + test("OptimizedCreateHiveTableAsSelectCommand") { + withSQLConf( + HiveUtils.CONVERT_METASTORE_PARQUET.key -> "true", + HiveUtils.CONVERT_METASTORE_CTAS.key -> "true", + KyuubiSQLConf.INSERT_REPARTITION_BEFORE_WRITE_IF_NO_SHUFFLE.key -> "true") { + withTable("t") { + withListener("CREATE TABLE t STORED AS parquet AS SELECT 1 as a") { write => + assert(write.isInstanceOf[InsertIntoHadoopFsRelationCommand]) + assert(write.collect { + case _: RebalancePartitions => true + }.size == 1) + } + } + } + } + + test("Infer rebalance and sorder orders") { + def checkShuffleAndSort(dataWritingCommand: LogicalPlan, sSize: Int, rSize: Int): Unit = { + assert(dataWritingCommand.isInstanceOf[DataWritingCommand]) + val plan = dataWritingCommand.asInstanceOf[DataWritingCommand].query + assert(plan.collect { + case s: Sort => s + }.size == sSize) + assert(plan.collect { + case r: RebalancePartitions if r.partitionExpressions.size == rSize => r + }.nonEmpty || rSize == 0) + } + + withView("v") { + withTable("t", "input1", "input2") { + withSQLConf(KyuubiSQLConf.INFER_REBALANCE_AND_SORT_ORDERS.key -> "true") { + sql(s"CREATE TABLE t (c1 int, c2 long) USING PARQUET PARTITIONED BY (p string)") + sql(s"CREATE TABLE input1 USING PARQUET AS SELECT * FROM VALUES(1,2),(1,3)") + sql(s"CREATE TABLE input2 USING PARQUET AS SELECT * FROM VALUES(1,3),(1,3)") + sql(s"CREATE VIEW v as SELECT col1, count(*) as col2 FROM input1 GROUP BY col1") + + val df0 = sql( + s""" + |INSERT INTO TABLE t PARTITION(p='a') + |SELECT /*+ broadcast(input2) */ input1.col1, input2.col1 + |FROM input1 + |JOIN input2 + |ON input1.col1 = input2.col1 + |""".stripMargin) + checkShuffleAndSort(df0.queryExecution.analyzed, 1, 1) + + val df1 = sql( + s""" + |INSERT INTO TABLE t PARTITION(p='a') + |SELECT /*+ broadcast(input2) */ input1.col1, input1.col2 + |FROM input1 + |LEFT JOIN input2 + |ON input1.col1 = input2.col1 and input1.col2 = input2.col2 + |""".stripMargin) + checkShuffleAndSort(df1.queryExecution.analyzed, 1, 2) + + val df2 = sql( + s""" + |INSERT INTO TABLE t PARTITION(p='a') + |SELECT col1 as c1, count(*) as c2 + |FROM input1 + |GROUP BY col1 + |HAVING count(*) > 0 + |""".stripMargin) + checkShuffleAndSort(df2.queryExecution.analyzed, 1, 1) + + // dynamic partition + val df3 = sql( + s""" + |INSERT INTO TABLE t PARTITION(p) + |SELECT /*+ broadcast(input2) */ input1.col1, input1.col2, input1.col2 + |FROM input1 + |JOIN input2 + |ON input1.col1 = input2.col1 + |""".stripMargin) + checkShuffleAndSort(df3.queryExecution.analyzed, 0, 1) + + // non-deterministic + val df4 = sql( + s""" + |INSERT INTO TABLE t PARTITION(p='a') + |SELECT col1 + rand(), count(*) as c2 + |FROM input1 + |GROUP BY col1 + |""".stripMargin) + checkShuffleAndSort(df4.queryExecution.analyzed, 0, 0) + + // view + val df5 = sql( + s""" + |INSERT INTO TABLE t PARTITION(p='a') + |SELECT * FROM v + |""".stripMargin) + checkShuffleAndSort(df5.queryExecution.analyzed, 1, 1) + } + } + } + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/WatchDogSuite.scala b/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/WatchDogSuite.scala new file mode 100644 index 000000000..957089340 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/WatchDogSuite.scala @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +class WatchDogSuite extends WatchDogSuiteBase {} diff --git a/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/WatchDogSuiteBase.scala b/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/WatchDogSuiteBase.scala new file mode 100644 index 000000000..a202e813c --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/WatchDogSuiteBase.scala @@ -0,0 +1,601 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +import java.io.File + +import scala.collection.JavaConverters._ + +import org.apache.commons.io.FileUtils +import org.apache.spark.sql.catalyst.plans.logical.{GlobalLimit, LogicalPlan} + +import org.apache.kyuubi.sql.KyuubiSQLConf +import org.apache.kyuubi.sql.watchdog.{MaxFileSizeExceedException, MaxPartitionExceedException} + +trait WatchDogSuiteBase extends KyuubiSparkSQLExtensionTest { + override protected def beforeAll(): Unit = { + super.beforeAll() + setupData() + } + + case class LimitAndExpected(limit: Int, expected: Int) + + val limitAndExpecteds = List(LimitAndExpected(1, 1), LimitAndExpected(11, 10)) + + private def checkMaxPartition: Unit = { + withSQLConf(KyuubiSQLConf.WATCHDOG_MAX_PARTITIONS.key -> "100") { + checkAnswer(sql("SELECT count(distinct(p)) FROM test"), Row(10) :: Nil) + } + withSQLConf(KyuubiSQLConf.WATCHDOG_MAX_PARTITIONS.key -> "5") { + sql("SELECT * FROM test where p=1").queryExecution.sparkPlan + + sql(s"SELECT * FROM test WHERE p in (${Range(0, 5).toList.mkString(",")})") + .queryExecution.sparkPlan + + intercept[MaxPartitionExceedException]( + sql("SELECT * FROM test where p != 1").queryExecution.sparkPlan) + + intercept[MaxPartitionExceedException]( + sql("SELECT * FROM test").queryExecution.sparkPlan) + + intercept[MaxPartitionExceedException](sql( + s"SELECT * FROM test WHERE p in (${Range(0, 6).toList.mkString(",")})") + .queryExecution.sparkPlan) + } + } + + test("watchdog with scan maxPartitions -- hive") { + Seq("textfile", "parquet").foreach { format => + withTable("test", "temp") { + sql( + s""" + |CREATE TABLE test(i int) + |PARTITIONED BY (p int) + |STORED AS $format""".stripMargin) + spark.range(0, 10, 1).selectExpr("id as col") + .createOrReplaceTempView("temp") + + for (part <- Range(0, 10)) { + sql( + s""" + |INSERT OVERWRITE TABLE test PARTITION (p='$part') + |select col from temp""".stripMargin) + } + checkMaxPartition + } + } + } + + test("watchdog with scan maxPartitions -- data source") { + withTempDir { dir => + withTempView("test") { + spark.range(10).selectExpr("id", "id as p") + .write + .partitionBy("p") + .mode("overwrite") + .save(dir.getCanonicalPath) + spark.read.load(dir.getCanonicalPath).createOrReplaceTempView("test") + checkMaxPartition + } + } + } + + test("test watchdog: simple SELECT STATEMENT") { + + withSQLConf(KyuubiSQLConf.WATCHDOG_FORCED_MAXOUTPUTROWS.key -> "10") { + + List("", "ORDER BY c1", "ORDER BY c2").foreach { sort => + List("", " DISTINCT").foreach { distinct => + assert(sql( + s""" + |SELECT $distinct * + |FROM t1 + |$sort + |""".stripMargin).queryExecution.optimizedPlan.isInstanceOf[GlobalLimit]) + } + } + + limitAndExpecteds.foreach { case LimitAndExpected(limit, expected) => + List("", "ORDER BY c1", "ORDER BY c2").foreach { sort => + List("", "DISTINCT").foreach { distinct => + assert(sql( + s""" + |SELECT $distinct * + |FROM t1 + |$sort + |LIMIT $limit + |""".stripMargin).queryExecution.optimizedPlan.maxRows.contains(expected)) + } + } + } + } + } + + test("test watchdog: SELECT ... WITH AGGREGATE STATEMENT ") { + + withSQLConf(KyuubiSQLConf.WATCHDOG_FORCED_MAXOUTPUTROWS.key -> "10") { + + assert(!sql("SELECT count(*) FROM t1") + .queryExecution.optimizedPlan.isInstanceOf[GlobalLimit]) + + val sorts = List("", "ORDER BY cnt", "ORDER BY c1", "ORDER BY cnt, c1", "ORDER BY c1, cnt") + val havingConditions = List("", "HAVING cnt > 1") + + havingConditions.foreach { having => + sorts.foreach { sort => + assert(sql( + s""" + |SELECT c1, COUNT(*) as cnt + |FROM t1 + |GROUP BY c1 + |$having + |$sort + |""".stripMargin).queryExecution.optimizedPlan.isInstanceOf[GlobalLimit]) + } + } + + limitAndExpecteds.foreach { case LimitAndExpected(limit, expected) => + havingConditions.foreach { having => + sorts.foreach { sort => + assert(sql( + s""" + |SELECT c1, COUNT(*) as cnt + |FROM t1 + |GROUP BY c1 + |$having + |$sort + |LIMIT $limit + |""".stripMargin).queryExecution.optimizedPlan.maxRows.contains(expected)) + } + } + } + } + } + + test("test watchdog: SELECT with CTE forceMaxOutputRows") { + // simple CTE + val q1 = + """ + |WITH t2 AS ( + | SELECT * FROM t1 + |) + |""".stripMargin + + // nested CTE + val q2 = + """ + |WITH + | t AS (SELECT * FROM t1), + | t2 AS ( + | WITH t3 AS (SELECT * FROM t1) + | SELECT * FROM t3 + | ) + |""".stripMargin + withSQLConf(KyuubiSQLConf.WATCHDOG_FORCED_MAXOUTPUTROWS.key -> "10") { + + val sorts = List("", "ORDER BY c1", "ORDER BY c2") + + sorts.foreach { sort => + Seq(q1, q2).foreach { withQuery => + assert(sql( + s""" + |$withQuery + |SELECT * FROM t2 + |$sort + |""".stripMargin).queryExecution.optimizedPlan.isInstanceOf[GlobalLimit]) + } + } + + limitAndExpecteds.foreach { case LimitAndExpected(limit, expected) => + sorts.foreach { sort => + Seq(q1, q2).foreach { withQuery => + assert(sql( + s""" + |$withQuery + |SELECT * FROM t2 + |$sort + |LIMIT $limit + |""".stripMargin).queryExecution.optimizedPlan.maxRows.contains(expected)) + } + } + } + } + } + + test("test watchdog: SELECT AGGREGATE WITH CTE forceMaxOutputRows") { + + withSQLConf(KyuubiSQLConf.WATCHDOG_FORCED_MAXOUTPUTROWS.key -> "10") { + + assert(!sql( + """ + |WITH custom_cte AS ( + |SELECT * FROM t1 + |) + | + |SELECT COUNT(*) + |FROM custom_cte + |""".stripMargin).queryExecution + .analyzed.isInstanceOf[GlobalLimit]) + + val sorts = List("", "ORDER BY cnt", "ORDER BY c1", "ORDER BY cnt, c1", "ORDER BY c1, cnt") + val havingConditions = List("", "HAVING cnt > 1") + + havingConditions.foreach { having => + sorts.foreach { sort => + assert(sql( + s""" + |WITH custom_cte AS ( + |SELECT * FROM t1 + |) + | + |SELECT c1, COUNT(*) as cnt + |FROM custom_cte + |GROUP BY c1 + |$having + |$sort + |""".stripMargin).queryExecution.optimizedPlan.isInstanceOf[GlobalLimit]) + } + } + + limitAndExpecteds.foreach { case LimitAndExpected(limit, expected) => + havingConditions.foreach { having => + sorts.foreach { sort => + assert(sql( + s""" + |WITH custom_cte AS ( + |SELECT * FROM t1 + |) + | + |SELECT c1, COUNT(*) as cnt + |FROM custom_cte + |GROUP BY c1 + |$having + |$sort + |LIMIT $limit + |""".stripMargin).queryExecution.optimizedPlan.maxRows.contains(expected)) + } + } + } + } + } + + test("test watchdog: UNION Statement for forceMaxOutputRows") { + + withSQLConf(KyuubiSQLConf.WATCHDOG_FORCED_MAXOUTPUTROWS.key -> "10") { + + List("", "ALL").foreach { x => + assert(sql( + s""" + |SELECT c1, c2 FROM t1 + |UNION $x + |SELECT c1, c2 FROM t2 + |UNION $x + |SELECT c1, c2 FROM t3 + |""".stripMargin) + .queryExecution.optimizedPlan.isInstanceOf[GlobalLimit]) + } + + val sorts = List("", "ORDER BY cnt", "ORDER BY c1", "ORDER BY cnt, c1", "ORDER BY c1, cnt") + val havingConditions = List("", "HAVING cnt > 1") + + List("", "ALL").foreach { x => + havingConditions.foreach { having => + sorts.foreach { sort => + assert(sql( + s""" + |SELECT c1, count(c2) as cnt + |FROM t1 + |GROUP BY c1 + |$having + |UNION $x + |SELECT c1, COUNT(c2) as cnt + |FROM t2 + |GROUP BY c1 + |$having + |UNION $x + |SELECT c1, COUNT(c2) as cnt + |FROM t3 + |GROUP BY c1 + |$having + |$sort + |""".stripMargin) + .queryExecution.optimizedPlan.isInstanceOf[GlobalLimit]) + } + } + } + + limitAndExpecteds.foreach { case LimitAndExpected(limit, expected) => + assert(sql( + s""" + |SELECT c1, c2 FROM t1 + |UNION + |SELECT c1, c2 FROM t2 + |UNION + |SELECT c1, c2 FROM t3 + |LIMIT $limit + |""".stripMargin) + .queryExecution.optimizedPlan.maxRows.contains(expected)) + } + } + } + + test("test watchdog: Select View Statement for forceMaxOutputRows") { + withSQLConf(KyuubiSQLConf.WATCHDOG_FORCED_MAXOUTPUTROWS.key -> "3") { + withTable("tmp_table", "tmp_union") { + withView("tmp_view", "tmp_view2") { + sql(s"create table tmp_table (a int, b int)") + sql(s"insert into tmp_table values (1,10),(2,20),(3,30),(4,40),(5,50)") + sql(s"create table tmp_union (a int, b int)") + sql(s"insert into tmp_union values (6,60),(7,70),(8,80),(9,90),(10,100)") + sql(s"create view tmp_view2 as select * from tmp_union") + assert(!sql( + s""" + |CREATE VIEW tmp_view + |as + |SELECT * FROM + |tmp_table + |""".stripMargin) + .queryExecution.optimizedPlan.isInstanceOf[GlobalLimit]) + + assert(sql( + s""" + |SELECT * FROM + |tmp_view + |""".stripMargin) + .queryExecution.optimizedPlan.maxRows.contains(3)) + + assert(sql( + s""" + |SELECT * FROM + |tmp_view + |limit 11 + |""".stripMargin) + .queryExecution.optimizedPlan.maxRows.contains(3)) + + assert(sql( + s""" + |SELECT * FROM + |(select * from tmp_view + |UNION + |select * from tmp_view2) + |ORDER BY a + |DESC + |""".stripMargin) + .collect().head.get(0) === 10) + } + } + } + } + + test("test watchdog: Insert Statement for forceMaxOutputRows") { + + withSQLConf(KyuubiSQLConf.WATCHDOG_FORCED_MAXOUTPUTROWS.key -> "10") { + withTable("tmp_table", "tmp_insert") { + spark.sql(s"create table tmp_table (a int, b int)") + spark.sql(s"insert into tmp_table values (1,10),(2,20),(3,30),(4,40),(5,50)") + val multiInsertTableName1: String = "tmp_tbl1" + val multiInsertTableName2: String = "tmp_tbl2" + sql(s"drop table if exists $multiInsertTableName1") + sql(s"drop table if exists $multiInsertTableName2") + sql(s"create table $multiInsertTableName1 like tmp_table") + sql(s"create table $multiInsertTableName2 like tmp_table") + assert(!sql( + s""" + |FROM tmp_table + |insert into $multiInsertTableName1 select * limit 2 + |insert into $multiInsertTableName2 select * + |""".stripMargin) + .queryExecution.optimizedPlan.isInstanceOf[GlobalLimit]) + } + } + } + + test("test watchdog: Distribute by for forceMaxOutputRows") { + + withSQLConf(KyuubiSQLConf.WATCHDOG_FORCED_MAXOUTPUTROWS.key -> "10") { + withTable("tmp_table") { + spark.sql(s"create table tmp_table (a int, b int)") + spark.sql(s"insert into tmp_table values (1,10),(2,20),(3,30),(4,40),(5,50)") + assert(sql( + s""" + |SELECT * + |FROM tmp_table + |DISTRIBUTE BY a + |""".stripMargin) + .queryExecution.optimizedPlan.isInstanceOf[GlobalLimit]) + } + } + } + + test("test watchdog: Subquery for forceMaxOutputRows") { + withSQLConf(KyuubiSQLConf.WATCHDOG_FORCED_MAXOUTPUTROWS.key -> "1") { + withTable("tmp_table1") { + sql("CREATE TABLE spark_catalog.`default`.tmp_table1(KEY INT, VALUE STRING) USING PARQUET") + sql("INSERT INTO TABLE spark_catalog.`default`.tmp_table1 " + + "VALUES (1, 'aa'),(2,'bb'),(3, 'cc'),(4,'aa'),(5,'cc'),(6, 'aa')") + assert( + sql("select * from tmp_table1").queryExecution.optimizedPlan.isInstanceOf[GlobalLimit]) + val testSqlText = + """ + |select count(*) + |from tmp_table1 + |where tmp_table1.key in ( + |select distinct tmp_table1.key + |from tmp_table1 + |where tmp_table1.value = "aa" + |) + |""".stripMargin + val plan = sql(testSqlText).queryExecution.optimizedPlan + assert(!findGlobalLimit(plan)) + checkAnswer(sql(testSqlText), Row(3) :: Nil) + } + + def findGlobalLimit(plan: LogicalPlan): Boolean = plan match { + case _: GlobalLimit => true + case p if p.children.isEmpty => false + case p => p.children.exists(findGlobalLimit) + } + + } + } + + test("test watchdog: Join for forceMaxOutputRows") { + withSQLConf(KyuubiSQLConf.WATCHDOG_FORCED_MAXOUTPUTROWS.key -> "1") { + withTable("tmp_table1", "tmp_table2") { + sql("CREATE TABLE spark_catalog.`default`.tmp_table1(KEY INT, VALUE STRING) USING PARQUET") + sql("INSERT INTO TABLE spark_catalog.`default`.tmp_table1 " + + "VALUES (1, 'aa'),(2,'bb'),(3, 'cc'),(4,'aa'),(5,'cc'),(6, 'aa')") + sql("CREATE TABLE spark_catalog.`default`.tmp_table2(KEY INT, VALUE STRING) USING PARQUET") + sql("INSERT INTO TABLE spark_catalog.`default`.tmp_table2 " + + "VALUES (1, 'aa'),(2,'bb'),(3, 'cc'),(4,'aa'),(5,'cc'),(6, 'aa')") + val testSqlText = + """ + |select a.*,b.* + |from tmp_table1 a + |join + |tmp_table2 b + |on a.KEY = b.KEY + |""".stripMargin + val plan = sql(testSqlText).queryExecution.optimizedPlan + assert(findGlobalLimit(plan)) + } + + def findGlobalLimit(plan: LogicalPlan): Boolean = plan match { + case _: GlobalLimit => true + case p if p.children.isEmpty => false + case p => p.children.exists(findGlobalLimit) + } + } + } + + private def checkMaxFileSize(tableSize: Long, nonPartTableSize: Long): Unit = { + withSQLConf(KyuubiSQLConf.WATCHDOG_MAX_FILE_SIZE.key -> tableSize.toString) { + checkAnswer(sql("SELECT count(distinct(p)) FROM test"), Row(10) :: Nil) + } + + withSQLConf(KyuubiSQLConf.WATCHDOG_MAX_FILE_SIZE.key -> (tableSize / 2).toString) { + sql("SELECT * FROM test where p=1").queryExecution.sparkPlan + + sql(s"SELECT * FROM test WHERE p in (${Range(0, 3).toList.mkString(",")})") + .queryExecution.sparkPlan + + intercept[MaxFileSizeExceedException]( + sql("SELECT * FROM test where p != 1").queryExecution.sparkPlan) + + intercept[MaxFileSizeExceedException]( + sql("SELECT * FROM test").queryExecution.sparkPlan) + + intercept[MaxFileSizeExceedException](sql( + s"SELECT * FROM test WHERE p in (${Range(0, 6).toList.mkString(",")})") + .queryExecution.sparkPlan) + } + + withSQLConf(KyuubiSQLConf.WATCHDOG_MAX_FILE_SIZE.key -> nonPartTableSize.toString) { + checkAnswer(sql("SELECT count(*) FROM test_non_part"), Row(10000) :: Nil) + } + + withSQLConf(KyuubiSQLConf.WATCHDOG_MAX_FILE_SIZE.key -> (nonPartTableSize - 1).toString) { + intercept[MaxFileSizeExceedException]( + sql("SELECT * FROM test_non_part").queryExecution.sparkPlan) + } + } + + test("watchdog with scan maxFileSize -- hive") { + Seq(false).foreach { convertMetastoreParquet => + withTable("test", "test_non_part", "temp") { + spark.range(10000).selectExpr("id as col") + .createOrReplaceTempView("temp") + + // partitioned table + sql( + s""" + |CREATE TABLE test(i int) + |PARTITIONED BY (p int) + |STORED AS parquet""".stripMargin) + for (part <- Range(0, 10)) { + sql( + s""" + |INSERT OVERWRITE TABLE test PARTITION (p='$part') + |select col from temp""".stripMargin) + } + + val tablePath = new File(spark.sessionState.catalog.externalCatalog + .getTable("default", "test").location) + val tableSize = FileUtils.listFiles(tablePath, Array("parquet"), true).asScala + .map(_.length()).sum + assert(tableSize > 0) + + // non-partitioned table + sql( + s""" + |CREATE TABLE test_non_part(i int) + |STORED AS parquet""".stripMargin) + sql( + s""" + |INSERT OVERWRITE TABLE test_non_part + |select col from temp""".stripMargin) + sql("ANALYZE TABLE test_non_part COMPUTE STATISTICS") + + val nonPartTablePath = new File(spark.sessionState.catalog.externalCatalog + .getTable("default", "test_non_part").location) + val nonPartTableSize = FileUtils.listFiles(nonPartTablePath, Array("parquet"), true).asScala + .map(_.length()).sum + assert(nonPartTableSize > 0) + + // check + withSQLConf("spark.sql.hive.convertMetastoreParquet" -> convertMetastoreParquet.toString) { + checkMaxFileSize(tableSize, nonPartTableSize) + } + } + } + } + + test("watchdog with scan maxFileSize -- data source") { + withTempDir { dir => + withTempView("test", "test_non_part") { + // partitioned table + val tablePath = new File(dir, "test") + spark.range(10).selectExpr("id", "id as p") + .write + .partitionBy("p") + .mode("overwrite") + .parquet(tablePath.getCanonicalPath) + spark.read.load(tablePath.getCanonicalPath).createOrReplaceTempView("test") + + val tableSize = FileUtils.listFiles(tablePath, Array("parquet"), true).asScala + .map(_.length()).sum + assert(tableSize > 0) + + // non-partitioned table + val nonPartTablePath = new File(dir, "test_non_part") + spark.range(10000).selectExpr("id", "id as p") + .write + .mode("overwrite") + .parquet(nonPartTablePath.getCanonicalPath) + spark.read.load(nonPartTablePath.getCanonicalPath).createOrReplaceTempView("test_non_part") + + val nonPartTableSize = FileUtils.listFiles(nonPartTablePath, Array("parquet"), true).asScala + .map(_.length()).sum + assert(tableSize > 0) + + // check + checkMaxFileSize(tableSize, nonPartTableSize) + } + } + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/ZorderCoreBenchmark.scala b/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/ZorderCoreBenchmark.scala new file mode 100644 index 000000000..9b1614fce --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/ZorderCoreBenchmark.scala @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +import org.apache.spark.SparkConf +import org.apache.spark.benchmark.Benchmark +import org.apache.spark.sql.benchmark.KyuubiBenchmarkBase +import org.apache.spark.sql.internal.StaticSQLConf + +import org.apache.kyuubi.sql.zorder.ZorderBytesUtils + +/** + * Benchmark to measure performance with zorder core. + * + * {{{ + * RUN_BENCHMARK=1 ./build/mvn clean test \ + * -pl extensions/spark/kyuubi-extension-spark-3-1 -am \ + * -Pspark-3.1,kyuubi-extension-spark-3-1 \ + * -Dtest=none -DwildcardSuites=org.apache.spark.sql.ZorderCoreBenchmark + * }}} + */ +class ZorderCoreBenchmark extends KyuubiSparkSQLExtensionTest with KyuubiBenchmarkBase { + private val runBenchmark = sys.env.contains("RUN_BENCHMARK") + private val numRows = 1 * 1000 * 1000 + + private def randomInt(numColumns: Int): Seq[Array[Any]] = { + (1 to numRows).map { l => + val arr = new Array[Any](numColumns) + (0 until numColumns).foreach(col => arr(col) = l) + arr + } + } + + private def randomLong(numColumns: Int): Seq[Array[Any]] = { + (1 to numRows).map { l => + val arr = new Array[Any](numColumns) + (0 until numColumns).foreach(col => arr(col) = l.toLong) + arr + } + } + + private def interleaveMultiByteArrayBenchmark(): Unit = { + val benchmark = + new Benchmark(s"$numRows rows zorder core benchmark", numRows, output = output) + benchmark.addCase("2 int columns benchmark", 3) { _ => + randomInt(2).foreach(ZorderBytesUtils.interleaveBits) + } + + benchmark.addCase("3 int columns benchmark", 3) { _ => + randomInt(3).foreach(ZorderBytesUtils.interleaveBits) + } + + benchmark.addCase("4 int columns benchmark", 3) { _ => + randomInt(4).foreach(ZorderBytesUtils.interleaveBits) + } + + benchmark.addCase("2 long columns benchmark", 3) { _ => + randomLong(2).foreach(ZorderBytesUtils.interleaveBits) + } + + benchmark.addCase("3 long columns benchmark", 3) { _ => + randomLong(3).foreach(ZorderBytesUtils.interleaveBits) + } + + benchmark.addCase("4 long columns benchmark", 3) { _ => + randomLong(4).foreach(ZorderBytesUtils.interleaveBits) + } + + benchmark.run() + } + + private def paddingTo8ByteBenchmark() { + val iterations = 10 * 1000 * 1000 + + val b2 = Array('a'.toByte, 'b'.toByte) + val benchmark = + new Benchmark(s"$iterations iterations paddingTo8Byte benchmark", iterations, output = output) + benchmark.addCase("2 length benchmark", 3) { _ => + (1 to iterations).foreach(_ => ZorderBytesUtils.paddingTo8Byte(b2)) + } + + val b16 = Array.tabulate(16) { i => i.toByte } + benchmark.addCase("16 length benchmark", 3) { _ => + (1 to iterations).foreach(_ => ZorderBytesUtils.paddingTo8Byte(b16)) + } + + benchmark.run() + } + + test("zorder core benchmark") { + assume(runBenchmark) + + withHeader { + interleaveMultiByteArrayBenchmark() + paddingTo8ByteBenchmark() + } + } + + override def sparkConf(): SparkConf = { + super.sparkConf().remove(StaticSQLConf.SPARK_SESSION_EXTENSIONS.key) + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/ZorderSuite.scala b/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/ZorderSuite.scala new file mode 100644 index 000000000..c2fa16197 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/ZorderSuite.scala @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +import org.apache.spark.sql.catalyst.parser.ParserInterface +import org.apache.spark.sql.catalyst.plans.logical.{RebalancePartitions, Sort} +import org.apache.spark.sql.internal.SQLConf + +import org.apache.kyuubi.sql.{KyuubiSQLConf, SparkKyuubiSparkSQLParser} +import org.apache.kyuubi.sql.zorder.Zorder + +trait ZorderSuiteSpark extends ZorderSuiteBase { + + test("Add rebalance before zorder") { + Seq("true" -> false, "false" -> true).foreach { case (useOriginalOrdering, zorder) => + withSQLConf( + KyuubiSQLConf.ZORDER_GLOBAL_SORT_ENABLED.key -> "false", + KyuubiSQLConf.REBALANCE_BEFORE_ZORDER.key -> "true", + KyuubiSQLConf.REBALANCE_ZORDER_COLUMNS_ENABLED.key -> "true", + KyuubiSQLConf.ZORDER_USING_ORIGINAL_ORDERING_ENABLED.key -> useOriginalOrdering) { + withTable("t") { + sql( + """ + |CREATE TABLE t (c1 int, c2 string) PARTITIONED BY (d string) + | TBLPROPERTIES ( + |'kyuubi.zorder.enabled'= 'true', + |'kyuubi.zorder.cols'= 'c1,C2') + |""".stripMargin) + val p = sql("INSERT INTO TABLE t PARTITION(d='a') SELECT * FROM VALUES(1,'a')") + .queryExecution.analyzed + assert(p.collect { + case sort: Sort + if !sort.global && + ((sort.order.exists(_.child.isInstanceOf[Zorder]) && zorder) || + (!sort.order.exists(_.child.isInstanceOf[Zorder]) && !zorder)) => sort + }.size == 1) + assert(p.collect { + case rebalance: RebalancePartitions + if rebalance.references.map(_.name).exists(_.equals("c1")) => rebalance + }.size == 1) + + val p2 = sql("INSERT INTO TABLE t PARTITION(d) SELECT * FROM VALUES(1,'a','b')") + .queryExecution.analyzed + assert(p2.collect { + case sort: Sort + if (!sort.global && Seq("c1", "c2", "d").forall(x => + sort.references.map(_.name).exists(_.equals(x)))) && + ((sort.order.exists(_.child.isInstanceOf[Zorder]) && zorder) || + (!sort.order.exists(_.child.isInstanceOf[Zorder]) && !zorder)) => sort + }.size == 1) + assert(p2.collect { + case rebalance: RebalancePartitions + if Seq("c1", "c2", "d").forall(x => + rebalance.references.map(_.name).exists(_.equals(x))) => rebalance + }.size == 1) + } + } + } + } + + test("Two phase rebalance before Z-Order") { + withSQLConf( + SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> + "org.apache.spark.sql.catalyst.optimizer.CollapseRepartition", + KyuubiSQLConf.ZORDER_GLOBAL_SORT_ENABLED.key -> "false", + KyuubiSQLConf.REBALANCE_BEFORE_ZORDER.key -> "true", + KyuubiSQLConf.TWO_PHASE_REBALANCE_BEFORE_ZORDER.key -> "true", + KyuubiSQLConf.REBALANCE_ZORDER_COLUMNS_ENABLED.key -> "true") { + withTable("t") { + sql( + """ + |CREATE TABLE t (c1 int) PARTITIONED BY (d string) + | TBLPROPERTIES ( + |'kyuubi.zorder.enabled'= 'true', + |'kyuubi.zorder.cols'= 'c1') + |""".stripMargin) + val p = sql("INSERT INTO TABLE t PARTITION(d) SELECT * FROM VALUES(1,'a')") + val rebalance = p.queryExecution.optimizedPlan.innerChildren + .flatMap(_.collect { case r: RebalancePartitions => r }) + assert(rebalance.size == 2) + assert(rebalance.head.partitionExpressions.flatMap(_.references.map(_.name)) + .contains("d")) + assert(rebalance.head.partitionExpressions.flatMap(_.references.map(_.name)) + .contains("c1")) + + assert(rebalance(1).partitionExpressions.flatMap(_.references.map(_.name)) + .contains("d")) + assert(!rebalance(1).partitionExpressions.flatMap(_.references.map(_.name)) + .contains("c1")) + } + } + } +} + +trait ParserSuite { self: ZorderSuiteBase => + override def createParser: ParserInterface = { + new SparkKyuubiSparkSQLParser(spark.sessionState.sqlParser) + } +} + +class ZorderWithCodegenEnabledSuite + extends ZorderWithCodegenEnabledSuiteBase + with ZorderSuiteSpark + with ParserSuite {} +class ZorderWithCodegenDisabledSuite + extends ZorderWithCodegenDisabledSuiteBase + with ZorderSuiteSpark + with ParserSuite {} diff --git a/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/ZorderSuiteBase.scala b/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/ZorderSuiteBase.scala new file mode 100644 index 000000000..2d3eec957 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/ZorderSuiteBase.scala @@ -0,0 +1,768 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +import org.apache.spark.SparkConf +import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier} +import org.apache.spark.sql.catalyst.analysis.{UnresolvedAttribute, UnresolvedFunction, UnresolvedRelation, UnresolvedStar} +import org.apache.spark.sql.catalyst.expressions.{Alias, Ascending, AttributeReference, EqualTo, Expression, ExpressionEvalHelper, Literal, NullsLast, SortOrder} +import org.apache.spark.sql.catalyst.parser.{ParseException, ParserInterface} +import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, OneRowRelation, Project, Sort} +import org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand +import org.apache.spark.sql.functions._ +import org.apache.spark.sql.hive.execution.InsertIntoHiveTable +import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf} +import org.apache.spark.sql.types._ + +import org.apache.kyuubi.sql.{KyuubiSQLConf, KyuubiSQLExtensionException} +import org.apache.kyuubi.sql.zorder.{OptimizeZorderCommandBase, OptimizeZorderStatement, Zorder, ZorderBytesUtils} + +trait ZorderSuiteBase extends KyuubiSparkSQLExtensionTest with ExpressionEvalHelper { + override def sparkConf(): SparkConf = { + super.sparkConf() + .set( + StaticSQLConf.SPARK_SESSION_EXTENSIONS.key, + "org.apache.kyuubi.sql.KyuubiSparkSQLCommonExtension") + } + + test("optimize unpartitioned table") { + withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") { + withTable("up") { + sql(s"DROP TABLE IF EXISTS up") + + val target = Seq( + Seq(0, 0), + Seq(1, 0), + Seq(0, 1), + Seq(1, 1), + Seq(2, 0), + Seq(3, 0), + Seq(2, 1), + Seq(3, 1), + Seq(0, 2), + Seq(1, 2), + Seq(0, 3), + Seq(1, 3), + Seq(2, 2), + Seq(3, 2), + Seq(2, 3), + Seq(3, 3)) + sql(s"CREATE TABLE up (c1 INT, c2 INT, c3 INT)") + sql(s"INSERT INTO TABLE up VALUES" + + "(0,0,2),(0,1,2),(0,2,1),(0,3,3)," + + "(1,0,4),(1,1,2),(1,2,1),(1,3,3)," + + "(2,0,2),(2,1,1),(2,2,5),(2,3,5)," + + "(3,0,3),(3,1,4),(3,2,9),(3,3,0)") + + val e = intercept[KyuubiSQLExtensionException] { + sql("OPTIMIZE up WHERE c1 > 1 ZORDER BY c1, c2") + } + assert(e.getMessage == "Filters are only supported for partitioned table") + + sql("OPTIMIZE up ZORDER BY c1, c2") + val res = sql("SELECT c1, c2 FROM up").collect() + + assert(res.length == 16) + + for (i <- target.indices) { + val t = target(i) + val r = res(i) + assert(t(0) == r.getInt(0)) + assert(t(1) == r.getInt(1)) + } + } + } + } + + test("optimize partitioned table") { + withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") { + withTable("p") { + sql("DROP TABLE IF EXISTS p") + + val target = Seq( + Seq(0, 0), + Seq(1, 0), + Seq(0, 1), + Seq(1, 1), + Seq(2, 0), + Seq(3, 0), + Seq(2, 1), + Seq(3, 1), + Seq(0, 2), + Seq(1, 2), + Seq(0, 3), + Seq(1, 3), + Seq(2, 2), + Seq(3, 2), + Seq(2, 3), + Seq(3, 3)) + + sql(s"CREATE TABLE p (c1 INT, c2 INT, c3 INT) PARTITIONED BY (id INT)") + sql(s"ALTER TABLE p ADD PARTITION (id = 1)") + sql(s"ALTER TABLE p ADD PARTITION (id = 2)") + sql(s"INSERT INTO TABLE p PARTITION (id = 1) VALUES" + + "(0,0,2),(0,1,2),(0,2,1),(0,3,3)," + + "(1,0,4),(1,1,2),(1,2,1),(1,3,3)," + + "(2,0,2),(2,1,1),(2,2,5),(2,3,5)," + + "(3,0,3),(3,1,4),(3,2,9),(3,3,0)") + sql(s"INSERT INTO TABLE p PARTITION (id = 2) VALUES" + + "(0,0,2),(0,1,2),(0,2,1),(0,3,3)," + + "(1,0,4),(1,1,2),(1,2,1),(1,3,3)," + + "(2,0,2),(2,1,1),(2,2,5),(2,3,5)," + + "(3,0,3),(3,1,4),(3,2,9),(3,3,0)") + + sql(s"OPTIMIZE p ZORDER BY c1, c2") + + val res1 = sql(s"SELECT c1, c2 FROM p WHERE id = 1").collect() + val res2 = sql(s"SELECT c1, c2 FROM p WHERE id = 2").collect() + + assert(res1.length == 16) + assert(res2.length == 16) + + for (i <- target.indices) { + val t = target(i) + val r1 = res1(i) + assert(t(0) == r1.getInt(0)) + assert(t(1) == r1.getInt(1)) + + val r2 = res2(i) + assert(t(0) == r2.getInt(0)) + assert(t(1) == r2.getInt(1)) + } + } + } + } + + test("optimize partitioned table with filters") { + withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") { + withTable("p") { + sql("DROP TABLE IF EXISTS p") + + val target1 = Seq( + Seq(0, 0), + Seq(1, 0), + Seq(0, 1), + Seq(1, 1), + Seq(2, 0), + Seq(3, 0), + Seq(2, 1), + Seq(3, 1), + Seq(0, 2), + Seq(1, 2), + Seq(0, 3), + Seq(1, 3), + Seq(2, 2), + Seq(3, 2), + Seq(2, 3), + Seq(3, 3)) + val target2 = Seq( + Seq(0, 0), + Seq(0, 1), + Seq(0, 2), + Seq(0, 3), + Seq(1, 0), + Seq(1, 1), + Seq(1, 2), + Seq(1, 3), + Seq(2, 0), + Seq(2, 1), + Seq(2, 2), + Seq(2, 3), + Seq(3, 0), + Seq(3, 1), + Seq(3, 2), + Seq(3, 3)) + sql(s"CREATE TABLE p (c1 INT, c2 INT, c3 INT) PARTITIONED BY (id INT)") + sql(s"ALTER TABLE p ADD PARTITION (id = 1)") + sql(s"ALTER TABLE p ADD PARTITION (id = 2)") + sql(s"INSERT INTO TABLE p PARTITION (id = 1) VALUES" + + "(0,0,2),(0,1,2),(0,2,1),(0,3,3)," + + "(1,0,4),(1,1,2),(1,2,1),(1,3,3)," + + "(2,0,2),(2,1,1),(2,2,5),(2,3,5)," + + "(3,0,3),(3,1,4),(3,2,9),(3,3,0)") + sql(s"INSERT INTO TABLE p PARTITION (id = 2) VALUES" + + "(0,0,2),(0,1,2),(0,2,1),(0,3,3)," + + "(1,0,4),(1,1,2),(1,2,1),(1,3,3)," + + "(2,0,2),(2,1,1),(2,2,5),(2,3,5)," + + "(3,0,3),(3,1,4),(3,2,9),(3,3,0)") + + val e = intercept[KyuubiSQLExtensionException]( + sql(s"OPTIMIZE p WHERE id = 1 AND c1 > 1 ZORDER BY c1, c2")) + assert(e.getMessage == "Only partition column filters are allowed") + + sql(s"OPTIMIZE p WHERE id = 1 ZORDER BY c1, c2") + + val res1 = sql(s"SELECT c1, c2 FROM p WHERE id = 1").collect() + val res2 = sql(s"SELECT c1, c2 FROM p WHERE id = 2").collect() + + assert(res1.length == 16) + assert(res2.length == 16) + + for (i <- target1.indices) { + val t1 = target1(i) + val r1 = res1(i) + assert(t1(0) == r1.getInt(0)) + assert(t1(1) == r1.getInt(1)) + + val t2 = target2(i) + val r2 = res2(i) + assert(t2(0) == r2.getInt(0)) + assert(t2(1) == r2.getInt(1)) + } + } + } + } + + test("optimize zorder with datasource table") { + // TODO remove this if we support datasource table + withTable("t") { + sql("CREATE TABLE t (c1 int, c2 int) USING PARQUET") + val msg = intercept[KyuubiSQLExtensionException] { + sql("OPTIMIZE t ZORDER BY c1, c2") + }.getMessage + assert(msg.contains("only support hive table")) + } + } + + private def checkZorderTable( + enabled: Boolean, + cols: String, + planHasRepartition: Boolean, + resHasSort: Boolean): Unit = { + def checkSort(plan: LogicalPlan): Unit = { + assert(plan.isInstanceOf[Sort] === resHasSort) + plan match { + case sort: Sort => + val colArr = cols.split(",") + val refs = + if (colArr.length == 1) { + sort.order.head + .child.asInstanceOf[AttributeReference] :: Nil + } else { + sort.order.head + .child.asInstanceOf[Zorder].children.map(_.references.head) + } + assert(refs.size === colArr.size) + refs.zip(colArr).foreach { case (ref, col) => + assert(ref.name === col.trim) + } + case _ => + } + } + + val repartition = + if (planHasRepartition) { + "/*+ repartition */" + } else { + "" + } + withSQLConf("spark.sql.shuffle.partitions" -> "1") { + // hive + withSQLConf("spark.sql.hive.convertMetastoreParquet" -> "false") { + withTable("zorder_t1", "zorder_t2_true", "zorder_t2_false") { + sql( + s""" + |CREATE TABLE zorder_t1 (c1 int, c2 string, c3 long, c4 double) STORED AS PARQUET + |TBLPROPERTIES ( + | 'kyuubi.zorder.enabled' = '$enabled', + | 'kyuubi.zorder.cols' = '$cols') + |""".stripMargin) + val df1 = sql(s""" + |INSERT INTO TABLE zorder_t1 + |SELECT $repartition * FROM VALUES(1,'a',2,4D),(2,'b',3,6D) + |""".stripMargin) + assert(df1.queryExecution.analyzed.isInstanceOf[InsertIntoHiveTable]) + checkSort(df1.queryExecution.analyzed.children.head) + + Seq("true", "false").foreach { optimized => + withSQLConf( + "spark.sql.hive.convertMetastoreCtas" -> optimized, + "spark.sql.hive.convertMetastoreParquet" -> optimized) { + + withListener( + s""" + |CREATE TABLE zorder_t2_$optimized STORED AS PARQUET + |TBLPROPERTIES ( + | 'kyuubi.zorder.enabled' = '$enabled', + | 'kyuubi.zorder.cols' = '$cols') + | + |SELECT $repartition * FROM + |VALUES(1,'a',2,4D),(2,'b',3,6D) AS t(c1 ,c2 , c3, c4) + |""".stripMargin) { write => + if (optimized.toBoolean) { + assert(write.isInstanceOf[InsertIntoHadoopFsRelationCommand]) + } else { + assert(write.isInstanceOf[InsertIntoHiveTable]) + } + checkSort(write.query) + } + } + } + } + } + + // datasource + withTable("zorder_t3", "zorder_t4") { + sql( + s""" + |CREATE TABLE zorder_t3 (c1 int, c2 string, c3 long, c4 double) USING PARQUET + |TBLPROPERTIES ( + | 'kyuubi.zorder.enabled' = '$enabled', + | 'kyuubi.zorder.cols' = '$cols') + |""".stripMargin) + val df1 = sql(s""" + |INSERT INTO TABLE zorder_t3 + |SELECT $repartition * FROM VALUES(1,'a',2,4D),(2,'b',3,6D) + |""".stripMargin) + assert(df1.queryExecution.analyzed.isInstanceOf[InsertIntoHadoopFsRelationCommand]) + checkSort(df1.queryExecution.analyzed.children.head) + + withListener( + s""" + |CREATE TABLE zorder_t4 USING PARQUET + |TBLPROPERTIES ( + | 'kyuubi.zorder.enabled' = '$enabled', + | 'kyuubi.zorder.cols' = '$cols') + | + |SELECT $repartition * FROM + |VALUES(1,'a',2,4D),(2,'b',3,6D) AS t(c1 ,c2 , c3, c4) + |""".stripMargin) { write => + assert(write.isInstanceOf[InsertIntoHadoopFsRelationCommand]) + checkSort(write.query) + } + } + } + } + + test("Support insert zorder by table properties") { + withSQLConf(KyuubiSQLConf.INSERT_ZORDER_BEFORE_WRITING.key -> "false") { + checkZorderTable(true, "c1", false, false) + checkZorderTable(false, "c1", false, false) + } + withSQLConf(KyuubiSQLConf.INSERT_ZORDER_BEFORE_WRITING.key -> "true") { + checkZorderTable(true, "", false, false) + checkZorderTable(true, "c5", false, false) + checkZorderTable(true, "c1,c5", false, false) + checkZorderTable(false, "c3", false, false) + checkZorderTable(true, "c3", true, false) + checkZorderTable(true, "c3", false, true) + checkZorderTable(true, "c2,c4", false, true) + checkZorderTable(true, "c4, c2, c1, c3", false, true) + } + } + + test("zorder: check unsupported data type") { + def checkZorderPlan(zorder: Expression): Unit = { + val msg = intercept[AnalysisException] { + val plan = Project(Seq(Alias(zorder, "c")()), OneRowRelation()) + spark.sessionState.analyzer.checkAnalysis(plan) + }.getMessage + // before Spark 3.2.0 the null type catalog string is null, after Spark 3.2.0 it's void + // see https://github.com/apache/spark/pull/33437 + assert(msg.contains("Unsupported z-order type:") && + (msg.contains("null") || msg.contains("void"))) + } + + checkZorderPlan(Zorder(Seq(Literal(null, NullType)))) + checkZorderPlan(Zorder(Seq(Literal(1, IntegerType), Literal(null, NullType)))) + } + + test("zorder: check supported data type") { + val children = Seq( + Literal.create(false, BooleanType), + Literal.create(null, BooleanType), + Literal.create(1.toByte, ByteType), + Literal.create(null, ByteType), + Literal.create(1.toShort, ShortType), + Literal.create(null, ShortType), + Literal.create(1, IntegerType), + Literal.create(null, IntegerType), + Literal.create(1L, LongType), + Literal.create(null, LongType), + Literal.create(1f, FloatType), + Literal.create(null, FloatType), + Literal.create(1d, DoubleType), + Literal.create(null, DoubleType), + Literal.create("1", StringType), + Literal.create(null, StringType), + Literal.create(1L, TimestampType), + Literal.create(null, TimestampType), + Literal.create(1, DateType), + Literal.create(null, DateType), + Literal.create(BigDecimal(1, 1), DecimalType(1, 1)), + Literal.create(null, DecimalType(1, 1))) + val zorder = Zorder(children) + val plan = Project(Seq(Alias(zorder, "c")()), OneRowRelation()) + spark.sessionState.analyzer.checkAnalysis(plan) + assert(zorder.foldable) + +// // scalastyle:off +// val resultGen = org.apache.commons.codec.binary.Hex.encodeHex( +// zorder.eval(InternalRow.fromSeq(children)).asInstanceOf[Array[Byte]], false) +// resultGen.grouped(2).zipWithIndex.foreach { case (char, i) => +// print("0x" + char(0) + char(1) + ", ") +// if ((i + 1) % 10 == 0) { +// println() +// } +// } +// // scalastyle:on + + val expected = Array( + 0xFB, 0xEA, 0xAA, 0xBA, 0xAE, 0xAB, 0xAA, 0xEA, 0xBA, 0xAE, 0xAB, 0xAA, 0xEA, 0xBA, 0xA6, + 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, + 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xBA, 0xBB, 0xAA, 0xAA, 0xAA, + 0xBA, 0xAA, 0xBA, 0xAA, 0xBA, 0xAA, 0xBA, 0xAA, 0xBA, 0xAA, 0xBA, 0xAA, 0x9A, 0xAA, 0xAA, + 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xEA, + 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, + 0xAA, 0xAA, 0xBE, 0xAA, 0xAA, 0x8A, 0xBA, 0xAA, 0x2A, 0xEA, 0xA8, 0xAA, 0xAA, 0xA2, 0xAA, + 0xAA, 0x8A, 0xAA, 0xAA, 0x2F, 0xEB, 0xFE) + .map(_.toByte) + checkEvaluation(zorder, expected, InternalRow.fromSeq(children)) + } + + private def checkSort(input: DataFrame, expected: Seq[Row], dataType: Array[DataType]): Unit = { + withTempDir { dir => + input.repartition(3).write.mode("overwrite").format("parquet").save(dir.getCanonicalPath) + val df = spark.read.format("parquet") + .load(dir.getCanonicalPath) + .repartition(1) + assert(df.schema.fields.map(_.dataType).sameElements(dataType)) + val exprs = Seq("c1", "c2").map(col).map(_.expr) + val sortOrder = SortOrder(Zorder(exprs), Ascending, NullsLast, Seq.empty) + val zorderSort = Sort(Seq(sortOrder), true, df.logicalPlan) + val result = Dataset.ofRows(spark, zorderSort) + checkAnswer(result, expected) + } + } + + test("sort with zorder -- boolean column") { + val schema = StructType(StructField("c1", BooleanType) :: StructField("c2", BooleanType) :: Nil) + val nonNullDF = spark.createDataFrame( + spark.sparkContext.parallelize( + Seq(Row(false, false), Row(false, true), Row(true, false), Row(true, true))), + schema) + val expected = + Row(false, false) :: Row(true, false) :: Row(false, true) :: Row(true, true) :: Nil + checkSort(nonNullDF, expected, Array(BooleanType, BooleanType)) + val df = spark.createDataFrame( + spark.sparkContext.parallelize( + Seq(Row(false, false), Row(false, null), Row(null, false), Row(null, null))), + schema) + val expected2 = + Row(false, false) :: Row(null, false) :: Row(false, null) :: Row(null, null) :: Nil + checkSort(df, expected2, Array(BooleanType, BooleanType)) + } + + test("sort with zorder -- int column") { + // TODO: add more datatype unit test + val session = spark + import session.implicits._ + // generate 4 * 4 matrix + val len = 3 + val input = spark.range(len + 1).selectExpr("cast(id as int) as c1") + .select($"c1", explode(sequence(lit(0), lit(len))) as "c2") + val expected = + Row(0, 0) :: Row(1, 0) :: Row(0, 1) :: Row(1, 1) :: + Row(2, 0) :: Row(3, 0) :: Row(2, 1) :: Row(3, 1) :: + Row(0, 2) :: Row(1, 2) :: Row(0, 3) :: Row(1, 3) :: + Row(2, 2) :: Row(3, 2) :: Row(2, 3) :: Row(3, 3) :: Nil + checkSort(input, expected, Array(IntegerType, IntegerType)) + + // contains null value case. + val nullDF = spark.range(1).selectExpr("cast(null as int) as c1") + val input2 = spark.range(len).selectExpr("cast(id as int) as c1") + .union(nullDF) + .select( + $"c1", + explode(concat(sequence(lit(0), lit(len - 1)), array(lit(null)))) as "c2") + val expected2 = Row(0, 0) :: Row(1, 0) :: Row(0, 1) :: Row(1, 1) :: + Row(2, 0) :: Row(2, 1) :: Row(0, 2) :: Row(1, 2) :: + Row(2, 2) :: Row(null, 0) :: Row(null, 1) :: Row(null, 2) :: + Row(0, null) :: Row(1, null) :: Row(2, null) :: Row(null, null) :: Nil + checkSort(input2, expected2, Array(IntegerType, IntegerType)) + } + + test("sort with zorder -- string column") { + val schema = StructType(StructField("c1", StringType) :: StructField("c2", StringType) :: Nil) + val rdd = spark.sparkContext.parallelize(Seq( + Row("a", "a"), + Row("a", "b"), + Row("a", "c"), + Row("a", "d"), + Row("b", "a"), + Row("b", "b"), + Row("b", "c"), + Row("b", "d"), + Row("c", "a"), + Row("c", "b"), + Row("c", "c"), + Row("c", "d"), + Row("d", "a"), + Row("d", "b"), + Row("d", "c"), + Row("d", "d"))) + val input = spark.createDataFrame(rdd, schema) + val expected = Row("a", "a") :: Row("b", "a") :: Row("c", "a") :: Row("a", "b") :: + Row("a", "c") :: Row("b", "b") :: Row("c", "b") :: Row("b", "c") :: + Row("c", "c") :: Row("d", "a") :: Row("d", "b") :: Row("d", "c") :: + Row("a", "d") :: Row("b", "d") :: Row("c", "d") :: Row("d", "d") :: Nil + checkSort(input, expected, Array(StringType, StringType)) + + val rdd2 = spark.sparkContext.parallelize(Seq( + Row(null, "a"), + Row("a", "b"), + Row("a", "c"), + Row("a", null), + Row("b", "a"), + Row(null, "b"), + Row("b", null), + Row("b", "d"), + Row("c", "a"), + Row("c", null), + Row(null, "c"), + Row("c", "d"), + Row("d", null), + Row("d", "b"), + Row("d", "c"), + Row(null, "d"), + Row(null, null))) + val input2 = spark.createDataFrame(rdd2, schema) + val expected2 = Row("b", "a") :: Row("c", "a") :: Row("a", "b") :: Row("a", "c") :: + Row("d", "b") :: Row("d", "c") :: Row("b", "d") :: Row("c", "d") :: + Row(null, "a") :: Row(null, "b") :: Row(null, "c") :: Row(null, "d") :: + Row("a", null) :: Row("b", null) :: Row("c", null) :: Row("d", null) :: + Row(null, null) :: Nil + checkSort(input2, expected2, Array(StringType, StringType)) + } + + test("test special value of short int long type") { + val df1 = spark.createDataFrame(Seq( + (-1, -1L), + (Int.MinValue, Int.MinValue.toLong), + (1, 1L), + (Int.MaxValue - 1, Int.MaxValue.toLong), + (Int.MaxValue - 1, Int.MaxValue.toLong - 1), + (Int.MaxValue, Int.MaxValue.toLong + 1), + (Int.MaxValue, Int.MaxValue.toLong))).toDF("c1", "c2") + val expected1 = + Row(Int.MinValue, Int.MinValue.toLong) :: + Row(-1, -1L) :: + Row(1, 1L) :: + Row(Int.MaxValue - 1, Int.MaxValue.toLong - 1) :: + Row(Int.MaxValue - 1, Int.MaxValue.toLong) :: + Row(Int.MaxValue, Int.MaxValue.toLong) :: + Row(Int.MaxValue, Int.MaxValue.toLong + 1) :: Nil + checkSort(df1, expected1, Array(IntegerType, LongType)) + + val df2 = spark.createDataFrame(Seq( + (-1, -1.toShort), + (Short.MinValue.toInt, Short.MinValue), + (1, 1.toShort), + (Short.MaxValue.toInt, (Short.MaxValue - 1).toShort), + (Short.MaxValue.toInt + 1, (Short.MaxValue - 1).toShort), + (Short.MaxValue.toInt, Short.MaxValue), + (Short.MaxValue.toInt + 1, Short.MaxValue))).toDF("c1", "c2") + val expected2 = + Row(Short.MinValue.toInt, Short.MinValue) :: + Row(-1, -1.toShort) :: + Row(1, 1.toShort) :: + Row(Short.MaxValue.toInt, Short.MaxValue - 1) :: + Row(Short.MaxValue.toInt, Short.MaxValue) :: + Row(Short.MaxValue.toInt + 1, Short.MaxValue - 1) :: + Row(Short.MaxValue.toInt + 1, Short.MaxValue) :: Nil + checkSort(df2, expected2, Array(IntegerType, ShortType)) + + val df3 = spark.createDataFrame(Seq( + (-1L, -1.toShort), + (Short.MinValue.toLong, Short.MinValue), + (1L, 1.toShort), + (Short.MaxValue.toLong, (Short.MaxValue - 1).toShort), + (Short.MaxValue.toLong + 1, (Short.MaxValue - 1).toShort), + (Short.MaxValue.toLong, Short.MaxValue), + (Short.MaxValue.toLong + 1, Short.MaxValue))).toDF("c1", "c2") + val expected3 = + Row(Short.MinValue.toLong, Short.MinValue) :: + Row(-1L, -1.toShort) :: + Row(1L, 1.toShort) :: + Row(Short.MaxValue.toLong, Short.MaxValue - 1) :: + Row(Short.MaxValue.toLong, Short.MaxValue) :: + Row(Short.MaxValue.toLong + 1, Short.MaxValue - 1) :: + Row(Short.MaxValue.toLong + 1, Short.MaxValue) :: Nil + checkSort(df3, expected3, Array(LongType, ShortType)) + } + + test("skip zorder if only requires one column") { + withTable("t") { + withSQLConf("spark.sql.hive.convertMetastoreParquet" -> "false") { + sql("CREATE TABLE t (c1 int, c2 string) stored as parquet") + val order1 = sql("OPTIMIZE t ZORDER BY c1").queryExecution.analyzed + .asInstanceOf[OptimizeZorderCommandBase].query.asInstanceOf[Sort].order.head.child + assert(!order1.isInstanceOf[Zorder]) + assert(order1.isInstanceOf[AttributeReference]) + } + } + } + + test("Add config to control if zorder using global sort") { + withTable("t") { + withSQLConf(KyuubiSQLConf.ZORDER_GLOBAL_SORT_ENABLED.key -> "false") { + sql( + """ + |CREATE TABLE t (c1 int, c2 string) TBLPROPERTIES ( + |'kyuubi.zorder.enabled'= 'true', + |'kyuubi.zorder.cols'= 'c1,c2') + |""".stripMargin) + val p1 = sql("OPTIMIZE t ZORDER BY c1, c2").queryExecution.analyzed + assert(p1.collect { + case shuffle: Sort if !shuffle.global => shuffle + }.size == 1) + + val p2 = sql("INSERT INTO TABLE t SELECT * FROM VALUES(1,'a')").queryExecution.analyzed + assert(p2.collect { + case shuffle: Sort if !shuffle.global => shuffle + }.size == 1) + } + } + } + + test("fast approach test") { + Seq[Seq[Any]]( + Seq(1L, 2L), + Seq(1L, 2L, 3L), + Seq(1L, 2L, 3L, 4L), + Seq(1L, 2L, 3L, 4L, 5L), + Seq(1L, 2L, 3L, 4L, 5L, 6L), + Seq(1L, 2L, 3L, 4L, 5L, 6L, 7L), + Seq(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L)) + .foreach { inputs => + assert(java.util.Arrays.equals( + ZorderBytesUtils.interleaveBits(inputs.toArray), + ZorderBytesUtils.interleaveBitsDefault(inputs.map(ZorderBytesUtils.toByteArray).toArray))) + } + } + + test("OPTIMIZE command is parsed as expected") { + val parser = createParser + val globalSort = spark.conf.get(KyuubiSQLConf.ZORDER_GLOBAL_SORT_ENABLED) + + assert(parser.parsePlan("OPTIMIZE p zorder by c1") === + OptimizeZorderStatement( + Seq("p"), + Sort( + SortOrder(UnresolvedAttribute("c1"), Ascending, NullsLast, Seq.empty) :: Nil, + globalSort, + Project(Seq(UnresolvedStar(None)), UnresolvedRelation(TableIdentifier("p")))))) + + assert(parser.parsePlan("OPTIMIZE p zorder by c1, c2") === + OptimizeZorderStatement( + Seq("p"), + Sort( + SortOrder( + Zorder(Seq(UnresolvedAttribute("c1"), UnresolvedAttribute("c2"))), + Ascending, + NullsLast, + Seq.empty) :: Nil, + globalSort, + Project(Seq(UnresolvedStar(None)), UnresolvedRelation(TableIdentifier("p")))))) + + assert(parser.parsePlan("OPTIMIZE p where id = 1 zorder by c1") === + OptimizeZorderStatement( + Seq("p"), + Sort( + SortOrder(UnresolvedAttribute("c1"), Ascending, NullsLast, Seq.empty) :: Nil, + globalSort, + Project( + Seq(UnresolvedStar(None)), + Filter( + EqualTo(UnresolvedAttribute("id"), Literal(1)), + UnresolvedRelation(TableIdentifier("p"))))))) + + assert(parser.parsePlan("OPTIMIZE p where id = 1 zorder by c1, c2") === + OptimizeZorderStatement( + Seq("p"), + Sort( + SortOrder( + Zorder(Seq(UnresolvedAttribute("c1"), UnresolvedAttribute("c2"))), + Ascending, + NullsLast, + Seq.empty) :: Nil, + globalSort, + Project( + Seq(UnresolvedStar(None)), + Filter( + EqualTo(UnresolvedAttribute("id"), Literal(1)), + UnresolvedRelation(TableIdentifier("p"))))))) + + assert(parser.parsePlan("OPTIMIZE p where id = current_date() zorder by c1") === + OptimizeZorderStatement( + Seq("p"), + Sort( + SortOrder(UnresolvedAttribute("c1"), Ascending, NullsLast, Seq.empty) :: Nil, + globalSort, + Project( + Seq(UnresolvedStar(None)), + Filter( + EqualTo( + UnresolvedAttribute("id"), + UnresolvedFunction("current_date", Seq.empty, false)), + UnresolvedRelation(TableIdentifier("p"))))))) + + // TODO: add following case support + intercept[ParseException] { + parser.parsePlan("OPTIMIZE p zorder by (c1)") + } + + intercept[ParseException] { + parser.parsePlan("OPTIMIZE p zorder by (c1, c2)") + } + } + + test("OPTIMIZE partition predicates constraint") { + withTable("p") { + sql("CREATE TABLE p (c1 INT, c2 INT) PARTITIONED BY (event_date DATE)") + val e1 = intercept[KyuubiSQLExtensionException] { + sql("OPTIMIZE p WHERE event_date = current_date as c ZORDER BY c1, c2") + } + assert(e1.getMessage.contains("unsupported partition predicates")) + + val e2 = intercept[KyuubiSQLExtensionException] { + sql("OPTIMIZE p WHERE c1 = 1 ZORDER BY c1, c2") + } + assert(e2.getMessage == "Only partition column filters are allowed") + } + } + + def createParser: ParserInterface +} + +trait ZorderWithCodegenEnabledSuiteBase extends ZorderSuiteBase { + override def sparkConf(): SparkConf = { + val conf = super.sparkConf + conf.set(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key, "true") + conf + } +} + +trait ZorderWithCodegenDisabledSuiteBase extends ZorderSuiteBase { + override def sparkConf(): SparkConf = { + val conf = super.sparkConf + conf.set(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key, "false") + conf.set(SQLConf.CODEGEN_FACTORY_MODE.key, "NO_CODEGEN") + conf + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/benchmark/KyuubiBenchmarkBase.scala b/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/benchmark/KyuubiBenchmarkBase.scala new file mode 100644 index 000000000..b891a7224 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-4/src/test/scala/org/apache/spark/sql/benchmark/KyuubiBenchmarkBase.scala @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.benchmark + +import java.io.{File, FileOutputStream, OutputStream} + +import scala.collection.JavaConverters._ + +import com.google.common.reflect.ClassPath +import org.scalatest.Assertions._ + +trait KyuubiBenchmarkBase { + var output: Option[OutputStream] = None + + private val prefix = { + val benchmarkClasses = ClassPath.from(Thread.currentThread.getContextClassLoader) + .getTopLevelClassesRecursive("org.apache.spark.sql").asScala.toArray + assert(benchmarkClasses.nonEmpty) + val benchmark = benchmarkClasses.find(_.load().getName.endsWith("Benchmark")) + val targetDirOrProjDir = + new File(benchmark.get.load().getProtectionDomain.getCodeSource.getLocation.toURI) + .getParentFile.getParentFile + if (targetDirOrProjDir.getName == "target") { + targetDirOrProjDir.getParentFile.getCanonicalPath + "/" + } else { + targetDirOrProjDir.getCanonicalPath + "/" + } + } + + def withHeader(func: => Unit): Unit = { + val version = System.getProperty("java.version").split("\\D+")(0).toInt + val jdkString = if (version > 8) s"-jdk$version" else "" + val resultFileName = + s"${this.getClass.getSimpleName.replace("$", "")}$jdkString-results.txt" + val dir = new File(s"${prefix}benchmarks/") + if (!dir.exists()) { + // scalastyle:off println + println(s"Creating ${dir.getAbsolutePath} for benchmark results.") + // scalastyle:on println + dir.mkdirs() + } + val file = new File(dir, resultFileName) + if (!file.exists()) { + file.createNewFile() + } + output = Some(new FileOutputStream(file)) + + func + + output.foreach { o => + if (o != null) { + o.close() + } + } + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/pom.xml b/extensions/spark/kyuubi-extension-spark-3-5/pom.xml new file mode 100644 index 000000000..e78a88a80 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/pom.xml @@ -0,0 +1,206 @@ + + + + 4.0.0 + + org.apache.kyuubi + kyuubi-parent + 1.9.0-SNAPSHOT + ../../../pom.xml + + + kyuubi-extension-spark-3-5_${scala.binary.version} + jar + Kyuubi Dev Spark Extensions (for Spark 3.5) + https://kyuubi.apache.org/ + + + + org.scala-lang + scala-library + provided + + + + org.apache.spark + spark-sql_${scala.binary.version} + provided + + + + org.apache.spark + spark-hive_${scala.binary.version} + provided + + + + org.apache.hadoop + hadoop-client-api + provided + + + + org.apache.kyuubi + kyuubi-download + ${project.version} + pom + test + + + + org.apache.kyuubi + kyuubi-util-scala_${scala.binary.version} + ${project.version} + test-jar + test + + + + org.apache.spark + spark-core_${scala.binary.version} + test-jar + test + + + + org.apache.spark + spark-catalyst_${scala.binary.version} + test-jar + test + + + + org.scalatestplus + scalacheck-1-17_${scala.binary.version} + test + + + + org.apache.spark + spark-sql_${scala.binary.version} + ${spark.version} + test-jar + test + + + + org.apache.hadoop + hadoop-client-runtime + test + + + + + commons-collections + commons-collections + test + + + + commons-io + commons-io + test + + + + jakarta.xml.bind + jakarta.xml.bind-api + test + + + + org.apache.logging.log4j + log4j-slf4j-impl + test + + + + + + + + org.codehaus.mojo + build-helper-maven-plugin + + + regex-property + + regex-property + + + spark.home + ${project.basedir}/../../../externals/kyuubi-download/target/${spark.archive.name} + (.+)\.tgz + $1 + + + + + + org.scalatest + scalatest-maven-plugin + + + + ${spark.home} + ${scala.binary.version} + + + + + org.antlr + antlr4-maven-plugin + + true + ${project.basedir}/src/main/antlr4 + + + + + org.apache.maven.plugins + maven-shade-plugin + + false + + + org.apache.kyuubi:* + + + + + + + shade + + package + + + + + target/scala-${scala.binary.version}/classes + target/scala-${scala.binary.version}/test-classes + + diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/main/antlr4/org/apache/kyuubi/sql/KyuubiSparkSQL.g4 b/extensions/spark/kyuubi-extension-spark-3-5/src/main/antlr4/org/apache/kyuubi/sql/KyuubiSparkSQL.g4 new file mode 100644 index 000000000..e52b7f5cf --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/main/antlr4/org/apache/kyuubi/sql/KyuubiSparkSQL.g4 @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +grammar KyuubiSparkSQL; + +@members { + /** + * Verify whether current token is a valid decimal token (which contains dot). + * Returns true if the character that follows the token is not a digit or letter or underscore. + * + * For example: + * For char stream "2.3", "2." is not a valid decimal token, because it is followed by digit '3'. + * For char stream "2.3_", "2.3" is not a valid decimal token, because it is followed by '_'. + * For char stream "2.3W", "2.3" is not a valid decimal token, because it is followed by 'W'. + * For char stream "12.0D 34.E2+0.12 " 12.0D is a valid decimal token because it is followed + * by a space. 34.E2 is a valid decimal token because it is followed by symbol '+' + * which is not a digit or letter or underscore. + */ + public boolean isValidDecimal() { + int nextChar = _input.LA(1); + if (nextChar >= 'A' && nextChar <= 'Z' || nextChar >= '0' && nextChar <= '9' || + nextChar == '_') { + return false; + } else { + return true; + } + } + } + +tokens { + DELIMITER +} + +singleStatement + : statement EOF + ; + +statement + : OPTIMIZE multipartIdentifier whereClause? zorderClause #optimizeZorder + | .*? #passThrough + ; + +whereClause + : WHERE partitionPredicate = predicateToken + ; + +zorderClause + : ZORDER BY order+=multipartIdentifier (',' order+=multipartIdentifier)* + ; + +// We don't have an expression rule in our grammar here, so we just grab the tokens and defer +// parsing them to later. +predicateToken + : .+? + ; + +multipartIdentifier + : parts+=identifier ('.' parts+=identifier)* + ; + +identifier + : strictIdentifier + ; + +strictIdentifier + : IDENTIFIER #unquotedIdentifier + | quotedIdentifier #quotedIdentifierAlternative + | nonReserved #unquotedIdentifier + ; + +quotedIdentifier + : BACKQUOTED_IDENTIFIER + ; + +nonReserved + : AND + | BY + | FALSE + | DATE + | INTERVAL + | OPTIMIZE + | OR + | TABLE + | TIMESTAMP + | TRUE + | WHERE + | ZORDER + ; + +AND: 'AND'; +BY: 'BY'; +FALSE: 'FALSE'; +DATE: 'DATE'; +INTERVAL: 'INTERVAL'; +OPTIMIZE: 'OPTIMIZE'; +OR: 'OR'; +TABLE: 'TABLE'; +TIMESTAMP: 'TIMESTAMP'; +TRUE: 'TRUE'; +WHERE: 'WHERE'; +ZORDER: 'ZORDER'; + +MINUS: '-'; + +BIGINT_LITERAL + : DIGIT+ 'L' + ; + +SMALLINT_LITERAL + : DIGIT+ 'S' + ; + +TINYINT_LITERAL + : DIGIT+ 'Y' + ; + +INTEGER_VALUE + : DIGIT+ + ; + +DECIMAL_VALUE + : DIGIT+ EXPONENT + | DECIMAL_DIGITS EXPONENT? {isValidDecimal()}? + ; + +DOUBLE_LITERAL + : DIGIT+ EXPONENT? 'D' + | DECIMAL_DIGITS EXPONENT? 'D' {isValidDecimal()}? + ; + +BIGDECIMAL_LITERAL + : DIGIT+ EXPONENT? 'BD' + | DECIMAL_DIGITS EXPONENT? 'BD' {isValidDecimal()}? + ; + +BACKQUOTED_IDENTIFIER + : '`' ( ~'`' | '``' )* '`' + ; + +IDENTIFIER + : (LETTER | DIGIT | '_')+ + ; + +fragment DECIMAL_DIGITS + : DIGIT+ '.' DIGIT* + | '.' DIGIT+ + ; + +fragment EXPONENT + : 'E' [+-]? DIGIT+ + ; + +fragment DIGIT + : [0-9] + ; + +fragment LETTER + : [A-Z] + ; + +SIMPLE_COMMENT + : '--' ~[\r\n]* '\r'? '\n'? -> channel(HIDDEN) + ; + +BRACKETED_COMMENT + : '/*' .*? '*/' -> channel(HIDDEN) + ; + +WS : [ \r\n\t]+ -> channel(HIDDEN) + ; + +// Catch-all for anything we can't recognize. +// We use this to be able to ignore and recover all the text +// when splitting statements with DelimiterLexer +UNRECOGNIZED + : . + ; diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/DropIgnoreNonexistent.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/DropIgnoreNonexistent.scala new file mode 100644 index 000000000..e33632b8b --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/DropIgnoreNonexistent.scala @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.sql + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.analysis.{UnresolvedFunctionName, UnresolvedRelation} +import org.apache.spark.sql.catalyst.plans.logical.{DropFunction, DropNamespace, LogicalPlan, NoopCommand, UncacheTable} +import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.execution.command.{AlterTableDropPartitionCommand, DropTableCommand} + +import org.apache.kyuubi.sql.KyuubiSQLConf._ + +case class DropIgnoreNonexistent(session: SparkSession) extends Rule[LogicalPlan] { + + override def apply(plan: LogicalPlan): LogicalPlan = { + if (conf.getConf(DROP_IGNORE_NONEXISTENT)) { + plan match { + case i @ AlterTableDropPartitionCommand(_, _, false, _, _) => + i.copy(ifExists = true) + case i @ DropTableCommand(_, false, _, _) => + i.copy(ifExists = true) + case i @ DropNamespace(_, false, _) => + i.copy(ifExists = true) + case UncacheTable(u: UnresolvedRelation, false, _) => + NoopCommand("UNCACHE TABLE", u.multipartIdentifier) + case DropFunction(u: UnresolvedFunctionName, false) => + NoopCommand("DROP FUNCTION", u.multipartIdentifier) + case _ => plan + } + } else { + plan + } + } + +} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/DynamicShufflePartitions.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/DynamicShufflePartitions.scala new file mode 100644 index 000000000..03d93d076 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/DynamicShufflePartitions.scala @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.sql + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.plans.physical.{HashPartitioning, RangePartitioning, RoundRobinPartitioning} +import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.execution.{FileSourceScanExec, SparkPlan} +import org.apache.spark.sql.execution.adaptive.ShuffleQueryStageExec +import org.apache.spark.sql.execution.exchange.{REPARTITION_BY_NUM, ShuffleExchangeExec, ValidateRequirements} +import org.apache.spark.sql.hive.HiveSparkPlanHelper.HiveTableScanExec +import org.apache.spark.sql.internal.SQLConf._ + +import org.apache.kyuubi.sql.KyuubiSQLConf.{DYNAMIC_SHUFFLE_PARTITIONS, DYNAMIC_SHUFFLE_PARTITIONS_MAX_NUM} + +/** + * Dynamically adjust the number of shuffle partitions according to the input data size + */ +case class DynamicShufflePartitions(spark: SparkSession) extends Rule[SparkPlan] { + + override def apply(plan: SparkPlan): SparkPlan = { + if (!conf.getConf(DYNAMIC_SHUFFLE_PARTITIONS) || !conf.getConf(ADAPTIVE_EXECUTION_ENABLED)) { + plan + } else { + val maxDynamicShufflePartitions = conf.getConf(DYNAMIC_SHUFFLE_PARTITIONS_MAX_NUM) + + def collectScanSizes(plan: SparkPlan): Seq[Long] = plan match { + case FileSourceScanExec(relation, _, _, _, _, _, _, _, _) => + Seq(relation.location.sizeInBytes) + case t: HiveTableScanExec => + t.relation.prunedPartitions match { + case Some(partitions) => Seq(partitions.flatMap(_.stats).map(_.sizeInBytes.toLong).sum) + case None => Seq(t.relation.computeStats().sizeInBytes.toLong) + .filter(_ != conf.defaultSizeInBytes) + } + case stage: ShuffleQueryStageExec if stage.isMaterialized && stage.mapStats.isDefined => + Seq(stage.mapStats.get.bytesByPartitionId.sum) + case p => + p.children.flatMap(collectScanSizes) + } + + val scanSizes = collectScanSizes(plan) + if (scanSizes.isEmpty) { + return plan + } + + val targetSize = conf.getConf(ADVISORY_PARTITION_SIZE_IN_BYTES) + val targetShufflePartitions = Math.min( + Math.max(scanSizes.sum / targetSize + 1, conf.numShufflePartitions).toInt, + maxDynamicShufflePartitions) + + val newPlan = plan transformUp { + case exchange @ ShuffleExchangeExec(outputPartitioning, _, shuffleOrigin, _) + if shuffleOrigin != REPARTITION_BY_NUM => + val newOutPartitioning = outputPartitioning match { + case RoundRobinPartitioning(numPartitions) + if targetShufflePartitions != numPartitions => + Some(RoundRobinPartitioning(targetShufflePartitions)) + case HashPartitioning(expressions, numPartitions) + if targetShufflePartitions != numPartitions => + Some(HashPartitioning(expressions, targetShufflePartitions)) + case RangePartitioning(ordering, numPartitions) + if targetShufflePartitions != numPartitions => + Some(RangePartitioning(ordering, targetShufflePartitions)) + case _ => None + } + if (newOutPartitioning.isDefined) { + exchange.copy(outputPartitioning = newOutPartitioning.get) + } else { + exchange + } + } + + if (ValidateRequirements.validate(newPlan)) { + newPlan + } else { + logInfo("DynamicShufflePartitions rule generated an invalid plan. " + + "Falling back to the original plan.") + plan + } + } + } + +} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/InferRebalanceAndSortOrders.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/InferRebalanceAndSortOrders.scala new file mode 100644 index 000000000..fcbf5c0a1 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/InferRebalanceAndSortOrders.scala @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql + +import scala.annotation.tailrec + +import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeSet, Expression, NamedExpression, UnaryExpression} +import org.apache.spark.sql.catalyst.planning.ExtractEquiJoinKeys +import org.apache.spark.sql.catalyst.plans.{FullOuter, Inner, LeftAnti, LeftOuter, LeftSemi, RightOuter} +import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, Filter, LogicalPlan, Project, Sort, SubqueryAlias, View} + +/** + * Infer the columns for Rebalance and Sort to improve the compression ratio. + * + * For example + * {{{ + * INSERT INTO TABLE t PARTITION(p='a') + * SELECT * FROM t1 JOIN t2 on t1.c1 = t2.c1 + * }}} + * the inferred columns are: t1.c1 + */ +object InferRebalanceAndSortOrders { + + type PartitioningAndOrdering = (Seq[Expression], Seq[Expression]) + + private def getAliasMap(named: Seq[NamedExpression]): Map[Expression, Attribute] = { + @tailrec + def throughUnary(e: Expression): Expression = e match { + case u: UnaryExpression if u.deterministic => + throughUnary(u.child) + case _ => e + } + + named.flatMap { + case a @ Alias(child, _) => + Some((throughUnary(child).canonicalized, a.toAttribute)) + case _ => None + }.toMap + } + + def infer(plan: LogicalPlan): Option[PartitioningAndOrdering] = { + def candidateKeys( + input: LogicalPlan, + output: AttributeSet = AttributeSet.empty): Option[PartitioningAndOrdering] = { + input match { + case ExtractEquiJoinKeys(joinType, leftKeys, rightKeys, _, _, _, _, _) => + joinType match { + case LeftSemi | LeftAnti | LeftOuter => Some((leftKeys, leftKeys)) + case RightOuter => Some((rightKeys, rightKeys)) + case Inner | FullOuter => + if (output.isEmpty) { + Some((leftKeys ++ rightKeys, leftKeys ++ rightKeys)) + } else { + assert(leftKeys.length == rightKeys.length) + val keys = leftKeys.zip(rightKeys).flatMap { case (left, right) => + if (left.references.subsetOf(output)) { + Some(left) + } else if (right.references.subsetOf(output)) { + Some(right) + } else { + None + } + } + Some((keys, keys)) + } + case _ => None + } + case agg: Aggregate => + val aliasMap = getAliasMap(agg.aggregateExpressions) + Some(( + agg.groupingExpressions.map(p => aliasMap.getOrElse(p.canonicalized, p)), + agg.groupingExpressions.map(o => aliasMap.getOrElse(o.canonicalized, o)))) + case s: Sort => Some((s.order.map(_.child), s.order.map(_.child))) + case p: Project => + val aliasMap = getAliasMap(p.projectList) + candidateKeys(p.child, p.references).map { case (partitioning, ordering) => + ( + partitioning.map(p => aliasMap.getOrElse(p.canonicalized, p)), + ordering.map(o => aliasMap.getOrElse(o.canonicalized, o))) + } + case f: Filter => candidateKeys(f.child, output) + case s: SubqueryAlias => candidateKeys(s.child, output) + case v: View => candidateKeys(v.child, output) + + case _ => None + } + } + + candidateKeys(plan).map { case (partitioning, ordering) => + ( + partitioning.filter(_.references.subsetOf(plan.outputSet)), + ordering.filter(_.references.subsetOf(plan.outputSet))) + } + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/InsertShuffleNodeBeforeJoin.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/InsertShuffleNodeBeforeJoin.scala new file mode 100644 index 000000000..1a02e8c1e --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/InsertShuffleNodeBeforeJoin.scala @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql + +import org.apache.spark.sql.catalyst.plans.physical.Distribution +import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.execution.{SortExec, SparkPlan} +import org.apache.spark.sql.execution.adaptive.QueryStageExec +import org.apache.spark.sql.execution.aggregate.BaseAggregateExec +import org.apache.spark.sql.execution.exchange.{Exchange, ShuffleExchangeExec} +import org.apache.spark.sql.execution.joins.{ShuffledHashJoinExec, SortMergeJoinExec} +import org.apache.spark.sql.internal.SQLConf + +import org.apache.kyuubi.sql.KyuubiSQLConf._ + +/** + * Insert shuffle node before join if it doesn't exist to make `OptimizeSkewedJoin` works. + */ +object InsertShuffleNodeBeforeJoin extends Rule[SparkPlan] { + + override def apply(plan: SparkPlan): SparkPlan = { + // this rule has no meaning without AQE + if (!conf.getConf(FORCE_SHUFFLE_BEFORE_JOIN) || + !conf.getConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED)) { + return plan + } + + val newPlan = insertShuffleBeforeJoin(plan) + if (plan.fastEquals(newPlan)) { + plan + } else { + // make sure the output partitioning and ordering will not be broken. + KyuubiEnsureRequirements.apply(newPlan) + } + } + + // Since spark 3.3, insertShuffleBeforeJoin shouldn't be applied if join is skewed. + private def insertShuffleBeforeJoin(plan: SparkPlan): SparkPlan = plan transformUp { + case smj @ SortMergeJoinExec(_, _, _, _, l, r, isSkewJoin) if !isSkewJoin => + smj.withNewChildren(checkAndInsertShuffle(smj.requiredChildDistribution.head, l) :: + checkAndInsertShuffle(smj.requiredChildDistribution(1), r) :: Nil) + + case shj: ShuffledHashJoinExec if !shj.isSkewJoin => + if (!shj.left.isInstanceOf[Exchange] && !shj.right.isInstanceOf[Exchange]) { + shj.withNewChildren(withShuffleExec(shj.requiredChildDistribution.head, shj.left) :: + withShuffleExec(shj.requiredChildDistribution(1), shj.right) :: Nil) + } else if (!shj.left.isInstanceOf[Exchange]) { + shj.withNewChildren( + withShuffleExec(shj.requiredChildDistribution.head, shj.left) :: shj.right :: Nil) + } else if (!shj.right.isInstanceOf[Exchange]) { + shj.withNewChildren( + shj.left :: withShuffleExec(shj.requiredChildDistribution(1), shj.right) :: Nil) + } else { + shj + } + } + + private def checkAndInsertShuffle( + distribution: Distribution, + child: SparkPlan): SparkPlan = child match { + case SortExec(_, _, _: Exchange, _) => + child + case SortExec(_, _, _: QueryStageExec, _) => + child + case sort @ SortExec(_, _, agg: BaseAggregateExec, _) => + sort.withNewChildren(withShuffleExec(distribution, agg) :: Nil) + case _ => + withShuffleExec(distribution, child) + } + + private def withShuffleExec(distribution: Distribution, child: SparkPlan): SparkPlan = { + val numPartitions = distribution.requiredNumPartitions + .getOrElse(conf.numShufflePartitions) + ShuffleExchangeExec(distribution.createPartitioning(numPartitions), child) + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/KyuubiEnsureRequirements.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/KyuubiEnsureRequirements.scala new file mode 100644 index 000000000..586cad838 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/KyuubiEnsureRequirements.scala @@ -0,0 +1,464 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql + +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions.SortOrder +import org.apache.spark.sql.catalyst.plans._ +import org.apache.spark.sql.catalyst.plans.JoinType +import org.apache.spark.sql.catalyst.plans.physical._ +import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.catalyst.util.InternalRowComparableWrapper +import org.apache.spark.sql.execution.{SortExec, SparkPlan} +import org.apache.spark.sql.execution.datasources.v2.BatchScanExec +import org.apache.spark.sql.execution.exchange._ +import org.apache.spark.sql.execution.joins.{ShuffledHashJoinExec, SortMergeJoinExec} +import org.apache.spark.sql.internal.SQLConf + +/** + * Copy from Apache Spark `EnsureRequirements` + * 1. remove reorder join predicates + * 2. remove shuffle pruning + */ +object KyuubiEnsureRequirements extends Rule[SparkPlan] { + + private def ensureDistributionAndOrdering( + parent: Option[SparkPlan], + originalChildren: Seq[SparkPlan], + requiredChildDistributions: Seq[Distribution], + requiredChildOrderings: Seq[Seq[SortOrder]], + shuffleOrigin: ShuffleOrigin): Seq[SparkPlan] = { + assert(requiredChildDistributions.length == originalChildren.length) + assert(requiredChildOrderings.length == originalChildren.length) + // Ensure that the operator's children satisfy their output distribution requirements. + var children = originalChildren.zip(requiredChildDistributions).map { + case (child, distribution) if child.outputPartitioning.satisfies(distribution) => + child + case (child, BroadcastDistribution(mode)) => + BroadcastExchangeExec(mode, child) + case (child, distribution) => + val numPartitions = distribution.requiredNumPartitions + .getOrElse(conf.numShufflePartitions) + ShuffleExchangeExec(distribution.createPartitioning(numPartitions), child, shuffleOrigin) + } + + // Get the indexes of children which have specified distribution requirements and need to be + // co-partitioned. + val childrenIndexes = requiredChildDistributions.zipWithIndex.filter { + case (_: ClusteredDistribution, _) => true + case _ => false + }.map(_._2) + + // Special case: if all sides of the join are single partition and it's physical size less than + // or equal spark.sql.maxSinglePartitionBytes. + val preferSinglePartition = childrenIndexes.forall { i => + children(i).outputPartitioning == SinglePartition && + children(i).logicalLink + .forall(_.stats.sizeInBytes <= conf.getConf(SQLConf.MAX_SINGLE_PARTITION_BYTES)) + } + + // If there are more than one children, we'll need to check partitioning & distribution of them + // and see if extra shuffles are necessary. + if (childrenIndexes.length > 1 && !preferSinglePartition) { + val specs = childrenIndexes.map(i => { + val requiredDist = requiredChildDistributions(i) + assert( + requiredDist.isInstanceOf[ClusteredDistribution], + s"Expected ClusteredDistribution but found ${requiredDist.getClass.getSimpleName}") + i -> children(i).outputPartitioning.createShuffleSpec( + requiredDist.asInstanceOf[ClusteredDistribution]) + }).toMap + + // Find out the shuffle spec that gives better parallelism. Currently this is done by + // picking the spec with the largest number of partitions. + // + // NOTE: this is not optimal for the case when there are more than 2 children. Consider: + // (10, 10, 11) + // where the number represent the number of partitions for each child, it's better to pick 10 + // here since we only need to shuffle one side - we'd need to shuffle two sides if we pick 11. + // + // However this should be sufficient for now since in Spark nodes with multiple children + // always have exactly 2 children. + + // Whether we should consider `spark.sql.shuffle.partitions` and ensure enough parallelism + // during shuffle. To achieve a good trade-off between parallelism and shuffle cost, we only + // consider the minimum parallelism iff ALL children need to be re-shuffled. + // + // A child needs to be re-shuffled iff either one of below is true: + // 1. It can't create partitioning by itself, i.e., `canCreatePartitioning` returns false + // (as for the case of `RangePartitioning`), therefore it needs to be re-shuffled + // according to other shuffle spec. + // 2. It already has `ShuffleExchangeLike`, so we can re-use existing shuffle without + // introducing extra shuffle. + // + // On the other hand, in scenarios such as: + // HashPartitioning(5) <-> HashPartitioning(6) + // while `spark.sql.shuffle.partitions` is 10, we'll only re-shuffle the left side and make it + // HashPartitioning(6). + val shouldConsiderMinParallelism = specs.forall(p => + !p._2.canCreatePartitioning || children(p._1).isInstanceOf[ShuffleExchangeLike]) + // Choose all the specs that can be used to shuffle other children + val candidateSpecs = specs + .filter(_._2.canCreatePartitioning) + .filter(p => + !shouldConsiderMinParallelism || + children(p._1).outputPartitioning.numPartitions >= conf.defaultNumShufflePartitions) + val bestSpecOpt = if (candidateSpecs.isEmpty) { + None + } else { + // When choosing specs, we should consider those children with no `ShuffleExchangeLike` node + // first. For instance, if we have: + // A: (No_Exchange, 100) <---> B: (Exchange, 120) + // it's better to pick A and change B to (Exchange, 100) instead of picking B and insert a + // new shuffle for A. + val candidateSpecsWithoutShuffle = candidateSpecs.filter { case (k, _) => + !children(k).isInstanceOf[ShuffleExchangeLike] + } + val finalCandidateSpecs = if (candidateSpecsWithoutShuffle.nonEmpty) { + candidateSpecsWithoutShuffle + } else { + candidateSpecs + } + // Pick the spec with the best parallelism + Some(finalCandidateSpecs.values.maxBy(_.numPartitions)) + } + + // Check if the following conditions are satisfied: + // 1. There are exactly two children (e.g., join). Note that Spark doesn't support + // multi-way join at the moment, so this check should be sufficient. + // 2. All children are of `KeyGroupedPartitioning`, and they are compatible with each other + // If both are true, skip shuffle. + val isKeyGroupCompatible = parent.isDefined && + children.length == 2 && childrenIndexes.length == 2 && { + val left = children.head + val right = children(1) + val newChildren = checkKeyGroupCompatible( + parent.get, + left, + right, + requiredChildDistributions) + if (newChildren.isDefined) { + children = newChildren.get + } + newChildren.isDefined + } + + children = children.zip(requiredChildDistributions).zipWithIndex.map { + case ((child, _), idx) if isKeyGroupCompatible || !childrenIndexes.contains(idx) => + child + case ((child, dist), idx) => + if (bestSpecOpt.isDefined && bestSpecOpt.get.isCompatibleWith(specs(idx))) { + child + } else { + val newPartitioning = bestSpecOpt.map { bestSpec => + // Use the best spec to create a new partitioning to re-shuffle this child + val clustering = dist.asInstanceOf[ClusteredDistribution].clustering + bestSpec.createPartitioning(clustering) + }.getOrElse { + // No best spec available, so we create default partitioning from the required + // distribution + val numPartitions = dist.requiredNumPartitions + .getOrElse(conf.numShufflePartitions) + dist.createPartitioning(numPartitions) + } + + child match { + case ShuffleExchangeExec(_, c, so, ps) => + ShuffleExchangeExec(newPartitioning, c, so, ps) + case _ => ShuffleExchangeExec(newPartitioning, child) + } + } + } + } + + // Now that we've performed any necessary shuffles, add sorts to guarantee output orderings: + children = children.zip(requiredChildOrderings).map { case (child, requiredOrdering) => + // If child.outputOrdering already satisfies the requiredOrdering, we do not need to sort. + if (SortOrder.orderingSatisfies(child.outputOrdering, requiredOrdering)) { + child + } else { + SortExec(requiredOrdering, global = false, child = child) + } + } + + children + } + + /** + * Checks whether two children, `left` and `right`, of a join operator have compatible + * `KeyGroupedPartitioning`, and can benefit from storage-partitioned join. + * + * Returns the updated new children if the check is successful, otherwise `None`. + */ + private def checkKeyGroupCompatible( + parent: SparkPlan, + left: SparkPlan, + right: SparkPlan, + requiredChildDistribution: Seq[Distribution]): Option[Seq[SparkPlan]] = { + parent match { + case smj: SortMergeJoinExec => + checkKeyGroupCompatible(left, right, smj.joinType, requiredChildDistribution) + case sj: ShuffledHashJoinExec => + checkKeyGroupCompatible(left, right, sj.joinType, requiredChildDistribution) + case _ => + None + } + } + + private def checkKeyGroupCompatible( + left: SparkPlan, + right: SparkPlan, + joinType: JoinType, + requiredChildDistribution: Seq[Distribution]): Option[Seq[SparkPlan]] = { + assert(requiredChildDistribution.length == 2) + + var newLeft = left + var newRight = right + + val specs = Seq(left, right).zip(requiredChildDistribution).map { case (p, d) => + if (!d.isInstanceOf[ClusteredDistribution]) return None + val cd = d.asInstanceOf[ClusteredDistribution] + val specOpt = createKeyGroupedShuffleSpec(p.outputPartitioning, cd) + if (specOpt.isEmpty) return None + specOpt.get + } + + val leftSpec = specs.head + val rightSpec = specs(1) + + var isCompatible = false + if (!conf.v2BucketingPushPartValuesEnabled) { + isCompatible = leftSpec.isCompatibleWith(rightSpec) + } else { + logInfo("Pushing common partition values for storage-partitioned join") + isCompatible = leftSpec.areKeysCompatible(rightSpec) + + // Partition expressions are compatible. Regardless of whether partition values + // match from both sides of children, we can calculate a superset of partition values and + // push-down to respective data sources so they can adjust their output partitioning by + // filling missing partition keys with empty partitions. As result, we can still avoid + // shuffle. + // + // For instance, if two sides of a join have partition expressions + // `day(a)` and `day(b)` respectively + // (the join query could be `SELECT ... FROM t1 JOIN t2 on t1.a = t2.b`), but + // with different partition values: + // `day(a)`: [0, 1] + // `day(b)`: [1, 2, 3] + // Following the case 2 above, we don't have to shuffle both sides, but instead can + // just push the common set of partition values: `[0, 1, 2, 3]` down to the two data + // sources. + if (isCompatible) { + val leftPartValues = leftSpec.partitioning.partitionValues + val rightPartValues = rightSpec.partitioning.partitionValues + + logInfo( + s""" + |Left side # of partitions: ${leftPartValues.size} + |Right side # of partitions: ${rightPartValues.size} + |""".stripMargin) + + // As partition keys are compatible, we can pick either left or right as partition + // expressions + val partitionExprs = leftSpec.partitioning.expressions + + var mergedPartValues = InternalRowComparableWrapper + .mergePartitions(leftSpec.partitioning, rightSpec.partitioning, partitionExprs) + .map(v => (v, 1)) + + logInfo(s"After merging, there are ${mergedPartValues.size} partitions") + + var replicateLeftSide = false + var replicateRightSide = false + var applyPartialClustering = false + + // This means we allow partitions that are not clustered on their values, + // that is, multiple partitions with the same partition value. In the + // following, we calculate how many partitions that each distinct partition + // value has, and pushdown the information to scans, so they can adjust their + // final input partitions respectively. + if (conf.v2BucketingPartiallyClusteredDistributionEnabled) { + logInfo("Calculating partially clustered distribution for " + + "storage-partitioned join") + + // Similar to `OptimizeSkewedJoin`, we need to check join type and decide + // whether partially clustered distribution can be applied. For instance, the + // optimization cannot be applied to a left outer join, where the left hand + // side is chosen as the side to replicate partitions according to stats. + // Otherwise, query result could be incorrect. + val canReplicateLeft = canReplicateLeftSide(joinType) + val canReplicateRight = canReplicateRightSide(joinType) + + if (!canReplicateLeft && !canReplicateRight) { + logInfo("Skipping partially clustered distribution as it cannot be applied for " + + s"join type '$joinType'") + } else { + val leftLink = left.logicalLink + val rightLink = right.logicalLink + + replicateLeftSide = + if (leftLink.isDefined && rightLink.isDefined && + leftLink.get.stats.sizeInBytes > 1 && + rightLink.get.stats.sizeInBytes > 1) { + logInfo( + s""" + |Using plan statistics to determine which side of join to fully + |cluster partition values: + |Left side size (in bytes): ${leftLink.get.stats.sizeInBytes} + |Right side size (in bytes): ${rightLink.get.stats.sizeInBytes} + |""".stripMargin) + leftLink.get.stats.sizeInBytes < rightLink.get.stats.sizeInBytes + } else { + // As a simple heuristic, we pick the side with fewer number of partitions + // to apply the grouping & replication of partitions + logInfo("Using number of partitions to determine which side of join " + + "to fully cluster partition values") + leftPartValues.size < rightPartValues.size + } + + replicateRightSide = !replicateLeftSide + + // Similar to skewed join, we need to check the join type to see whether replication + // of partitions can be applied. For instance, replication should not be allowed for + // the left-hand side of a right outer join. + if (replicateLeftSide && !canReplicateLeft) { + logInfo("Left-hand side is picked but cannot be applied to join type " + + s"'$joinType'. Skipping partially clustered distribution.") + replicateLeftSide = false + } else if (replicateRightSide && !canReplicateRight) { + logInfo("Right-hand side is picked but cannot be applied to join type " + + s"'$joinType'. Skipping partially clustered distribution.") + replicateRightSide = false + } else { + val partValues = if (replicateLeftSide) rightPartValues else leftPartValues + val numExpectedPartitions = partValues + .map(InternalRowComparableWrapper(_, partitionExprs)) + .groupBy(identity) + .mapValues(_.size) + + mergedPartValues = mergedPartValues.map { case (partVal, numParts) => + ( + partVal, + numExpectedPartitions.getOrElse( + InternalRowComparableWrapper(partVal, partitionExprs), + numParts)) + } + + logInfo("After applying partially clustered distribution, there are " + + s"${mergedPartValues.map(_._2).sum} partitions.") + applyPartialClustering = true + } + } + } + + // Now we need to push-down the common partition key to the scan in each child + newLeft = populatePartitionValues( + left, + mergedPartValues, + applyPartialClustering, + replicateLeftSide) + newRight = populatePartitionValues( + right, + mergedPartValues, + applyPartialClustering, + replicateRightSide) + } + } + + if (isCompatible) Some(Seq(newLeft, newRight)) else None + } + + // Similar to `OptimizeSkewedJoin.canSplitRightSide` + private def canReplicateLeftSide(joinType: JoinType): Boolean = { + joinType == Inner || joinType == Cross || joinType == RightOuter + } + + // Similar to `OptimizeSkewedJoin.canSplitLeftSide` + private def canReplicateRightSide(joinType: JoinType): Boolean = { + joinType == Inner || joinType == Cross || joinType == LeftSemi || + joinType == LeftAnti || joinType == LeftOuter + } + + // Populate the common partition values down to the scan nodes + private def populatePartitionValues( + plan: SparkPlan, + values: Seq[(InternalRow, Int)], + applyPartialClustering: Boolean, + replicatePartitions: Boolean): SparkPlan = plan match { + case scan: BatchScanExec => + scan.copy(spjParams = scan.spjParams.copy( + commonPartitionValues = Some(values), + applyPartialClustering = applyPartialClustering, + replicatePartitions = replicatePartitions)) + case node => + node.mapChildren(child => + populatePartitionValues( + child, + values, + applyPartialClustering, + replicatePartitions)) + } + + /** + * Tries to create a [[KeyGroupedShuffleSpec]] from the input partitioning and distribution, if + * the partitioning is a [[KeyGroupedPartitioning]] (either directly or indirectly), and + * satisfies the given distribution. + */ + private def createKeyGroupedShuffleSpec( + partitioning: Partitioning, + distribution: ClusteredDistribution): Option[KeyGroupedShuffleSpec] = { + def check(partitioning: KeyGroupedPartitioning): Option[KeyGroupedShuffleSpec] = { + val attributes = partitioning.expressions.flatMap(_.collectLeaves()) + val clustering = distribution.clustering + + val satisfies = if (SQLConf.get.getConf(SQLConf.REQUIRE_ALL_CLUSTER_KEYS_FOR_CO_PARTITION)) { + attributes.length == clustering.length && attributes.zip(clustering).forall { + case (l, r) => l.semanticEquals(r) + } + } else { + partitioning.satisfies(distribution) + } + + if (satisfies) { + Some(partitioning.createShuffleSpec(distribution).asInstanceOf[KeyGroupedShuffleSpec]) + } else { + None + } + } + + partitioning match { + case p: KeyGroupedPartitioning => check(p) + case PartitioningCollection(partitionings) => + val specs = partitionings.map(p => createKeyGroupedShuffleSpec(p, distribution)) + assert(specs.forall(_.isEmpty) || specs.forall(_.isDefined)) + specs.head + case _ => None + } + } + + def apply(plan: SparkPlan): SparkPlan = plan.transformUp { + case operator: SparkPlan => + val newChildren = ensureDistributionAndOrdering( + Some(operator), + operator.children, + operator.requiredChildDistribution, + operator.requiredChildOrdering, + ENSURE_REQUIREMENTS) + operator.withNewChildren(newChildren) + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/KyuubiQueryStagePreparation.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/KyuubiQueryStagePreparation.scala new file mode 100644 index 000000000..a7fcbecd4 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/KyuubiQueryStagePreparation.scala @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.execution.SparkPlan +import org.apache.spark.sql.execution.adaptive.QueryStageExec +import org.apache.spark.sql.execution.command.{ResetCommand, SetCommand} +import org.apache.spark.sql.execution.exchange.{BroadcastExchangeLike, ReusedExchangeExec, ShuffleExchangeLike} +import org.apache.spark.sql.internal.SQLConf + +import org.apache.kyuubi.sql.KyuubiSQLConf._ + +/** + * This rule split stage into two parts: + * 1. previous stage + * 2. final stage + * For final stage, we can inject extra config. It's useful if we use repartition to optimize + * small files that needs bigger shuffle partition size than previous. + * + * Let's say we have a query with 3 stages, then the logical machine like: + * + * Set/Reset Command -> cleanup previousStage config if user set the spark config. + * Query -> AQE -> stage1 -> preparation (use previousStage to overwrite spark config) + * -> AQE -> stage2 -> preparation (use spark config) + * -> AQE -> stage3 -> preparation (use finalStage config to overwrite spark config, + * store spark config to previousStage.) + * + * An example of the new finalStage config: + * `spark.sql.adaptive.advisoryPartitionSizeInBytes` -> + * `spark.sql.finalStage.adaptive.advisoryPartitionSizeInBytes` + */ +case class FinalStageConfigIsolation(session: SparkSession) extends Rule[SparkPlan] { + import FinalStageConfigIsolation._ + + override def apply(plan: SparkPlan): SparkPlan = { + // this rule has no meaning without AQE + if (!conf.getConf(FINAL_STAGE_CONFIG_ISOLATION) || + !conf.getConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED)) { + return plan + } + + if (isFinalStage(plan)) { + // We can not get the whole plan at query preparation phase to detect if current plan is + // for writing, so we depend on a tag which is been injected at post resolution phase. + // Note: we should still do clean up previous config for non-final stage to avoid such case: + // the first statement is write, but the second statement is query. + if (conf.getConf(FINAL_STAGE_CONFIG_ISOLATION_WRITE_ONLY) && + !WriteUtils.isWrite(session, plan)) { + return plan + } + + // set config for final stage + session.conf.getAll.filter(_._1.startsWith(FINAL_STAGE_CONFIG_PREFIX)).foreach { + case (k, v) => + val sparkConfigKey = s"spark.sql.${k.substring(FINAL_STAGE_CONFIG_PREFIX.length)}" + val previousStageConfigKey = + s"$PREVIOUS_STAGE_CONFIG_PREFIX${k.substring(FINAL_STAGE_CONFIG_PREFIX.length)}" + // store the previous config only if we have not stored, to avoid some query only + // have one stage that will overwrite real config. + if (!session.sessionState.conf.contains(previousStageConfigKey)) { + val originalValue = + if (session.conf.getOption(sparkConfigKey).isDefined) { + session.sessionState.conf.getConfString(sparkConfigKey) + } else { + // the default value of config is None, so we need to use a internal tag + INTERNAL_UNSET_CONFIG_TAG + } + logInfo(s"Store config: $sparkConfigKey to previousStage, " + + s"original value: $originalValue ") + session.sessionState.conf.setConfString(previousStageConfigKey, originalValue) + } + logInfo(s"For final stage: set $sparkConfigKey = $v.") + session.conf.set(sparkConfigKey, v) + } + } else { + // reset config for previous stage + session.conf.getAll.filter(_._1.startsWith(PREVIOUS_STAGE_CONFIG_PREFIX)).foreach { + case (k, v) => + val sparkConfigKey = s"spark.sql.${k.substring(PREVIOUS_STAGE_CONFIG_PREFIX.length)}" + logInfo(s"For previous stage: set $sparkConfigKey = $v.") + if (v == INTERNAL_UNSET_CONFIG_TAG) { + session.conf.unset(sparkConfigKey) + } else { + session.conf.set(sparkConfigKey, v) + } + // unset config so that we do not need to reset configs for every previous stage + session.conf.unset(k) + } + } + + plan + } + + /** + * Currently formula depend on AQE in Spark 3.1.1, not sure it can work in future. + */ + private def isFinalStage(plan: SparkPlan): Boolean = { + var shuffleNum = 0 + var broadcastNum = 0 + var reusedNum = 0 + var queryStageNum = 0 + + def collectNumber(p: SparkPlan): SparkPlan = { + p transform { + case shuffle: ShuffleExchangeLike => + shuffleNum += 1 + shuffle + + case broadcast: BroadcastExchangeLike => + broadcastNum += 1 + broadcast + + case reusedExchangeExec: ReusedExchangeExec => + reusedNum += 1 + reusedExchangeExec + + // query stage is leaf node so we need to transform it manually + // compatible with Spark 3.5: + // SPARK-42101: table cache is a independent query stage, so do not need include it. + case queryStage: QueryStageExec if queryStage.nodeName != "TableCacheQueryStage" => + queryStageNum += 1 + collectNumber(queryStage.plan) + queryStage + } + } + collectNumber(plan) + + if (shuffleNum == 0) { + // we don not care about broadcast stage here since it won't change partition number. + true + } else if (shuffleNum + broadcastNum + reusedNum == queryStageNum) { + true + } else { + false + } + } +} +object FinalStageConfigIsolation { + final val SQL_PREFIX = "spark.sql." + final val FINAL_STAGE_CONFIG_PREFIX = "spark.sql.finalStage." + final val PREVIOUS_STAGE_CONFIG_PREFIX = "spark.sql.previousStage." + final val INTERNAL_UNSET_CONFIG_TAG = "__INTERNAL_UNSET_CONFIG_TAG__" + + def getPreviousStageConfigKey(configKey: String): Option[String] = { + if (configKey.startsWith(SQL_PREFIX)) { + Some(s"$PREVIOUS_STAGE_CONFIG_PREFIX${configKey.substring(SQL_PREFIX.length)}") + } else { + None + } + } +} + +case class FinalStageConfigIsolationCleanRule(session: SparkSession) extends Rule[LogicalPlan] { + import FinalStageConfigIsolation._ + + override def apply(plan: LogicalPlan): LogicalPlan = plan match { + case set @ SetCommand(Some((k, Some(_)))) if k.startsWith(SQL_PREFIX) => + checkAndUnsetPreviousStageConfig(k) + set + + case reset @ ResetCommand(Some(k)) if k.startsWith(SQL_PREFIX) => + checkAndUnsetPreviousStageConfig(k) + reset + + case other => other + } + + private def checkAndUnsetPreviousStageConfig(configKey: String): Unit = { + getPreviousStageConfigKey(configKey).foreach { previousStageConfigKey => + if (session.sessionState.conf.contains(previousStageConfigKey)) { + logInfo(s"For previous stage: unset $previousStageConfigKey") + session.conf.unset(previousStageConfigKey) + } + } + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/KyuubiSQLConf.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/KyuubiSQLConf.scala new file mode 100644 index 000000000..597cb250b --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/KyuubiSQLConf.scala @@ -0,0 +1,292 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql + +import org.apache.spark.network.util.ByteUnit +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.internal.SQLConf._ + +object KyuubiSQLConf { + + val INSERT_REPARTITION_BEFORE_WRITE = + buildConf("spark.sql.optimizer.insertRepartitionBeforeWrite.enabled") + .doc("Add repartition node at the top of query plan. An approach of merging small files.") + .version("1.2.0") + .booleanConf + .createWithDefault(true) + + val INSERT_REPARTITION_NUM = + buildConf("spark.sql.optimizer.insertRepartitionNum") + .doc(s"The partition number if ${INSERT_REPARTITION_BEFORE_WRITE.key} is enabled. " + + s"If AQE is disabled, the default value is ${SQLConf.SHUFFLE_PARTITIONS.key}. " + + "If AQE is enabled, the default value is none that means depend on AQE. " + + "This config is used for Spark 3.1 only.") + .version("1.2.0") + .intConf + .createOptional + + val DYNAMIC_PARTITION_INSERTION_REPARTITION_NUM = + buildConf("spark.sql.optimizer.dynamicPartitionInsertionRepartitionNum") + .doc(s"The partition number of each dynamic partition if " + + s"${INSERT_REPARTITION_BEFORE_WRITE.key} is enabled. " + + "We will repartition by dynamic partition columns to reduce the small file but that " + + "can cause data skew. This config is to extend the partition of dynamic " + + "partition column to avoid skew but may generate some small files.") + .version("1.2.0") + .intConf + .createWithDefault(100) + + val FORCE_SHUFFLE_BEFORE_JOIN = + buildConf("spark.sql.optimizer.forceShuffleBeforeJoin.enabled") + .doc("Ensure shuffle node exists before shuffled join (shj and smj) to make AQE " + + "`OptimizeSkewedJoin` works (complex scenario join, multi table join).") + .version("1.2.0") + .booleanConf + .createWithDefault(false) + + val FINAL_STAGE_CONFIG_ISOLATION = + buildConf("spark.sql.optimizer.finalStageConfigIsolation.enabled") + .doc("If true, the final stage support use different config with previous stage. " + + "The prefix of final stage config key should be `spark.sql.finalStage.`." + + "For example, the raw spark config: `spark.sql.adaptive.advisoryPartitionSizeInBytes`, " + + "then the final stage config should be: " + + "`spark.sql.finalStage.adaptive.advisoryPartitionSizeInBytes`.") + .version("1.2.0") + .booleanConf + .createWithDefault(false) + + val SQL_CLASSIFICATION = "spark.sql.analyzer.classification" + val SQL_CLASSIFICATION_ENABLED = + buildConf("spark.sql.analyzer.classification.enabled") + .doc("When true, allows Kyuubi engine to judge this SQL's classification " + + s"and set `$SQL_CLASSIFICATION` back into sessionConf. " + + "Through this configuration item, Spark can optimizing configuration dynamic") + .version("1.4.0") + .booleanConf + .createWithDefault(false) + + val INSERT_ZORDER_BEFORE_WRITING = + buildConf("spark.sql.optimizer.insertZorderBeforeWriting.enabled") + .doc("When true, we will follow target table properties to insert zorder or not. " + + "The key properties are: 1) kyuubi.zorder.enabled; if this property is true, we will " + + "insert zorder before writing data. 2) kyuubi.zorder.cols; string split by comma, we " + + "will zorder by these cols.") + .version("1.4.0") + .booleanConf + .createWithDefault(true) + + val ZORDER_GLOBAL_SORT_ENABLED = + buildConf("spark.sql.optimizer.zorderGlobalSort.enabled") + .doc("When true, we do a global sort using zorder. Note that, it can cause data skew " + + "issue if the zorder columns have less cardinality. When false, we only do local sort " + + "using zorder.") + .version("1.4.0") + .booleanConf + .createWithDefault(true) + + val REBALANCE_BEFORE_ZORDER = + buildConf("spark.sql.optimizer.rebalanceBeforeZorder.enabled") + .doc("When true, we do a rebalance before zorder in case data skew. " + + "Note that, if the insertion is dynamic partition we will use the partition " + + "columns to rebalance. Note that, this config only affects with Spark 3.3.x") + .version("1.6.0") + .booleanConf + .createWithDefault(false) + + val REBALANCE_ZORDER_COLUMNS_ENABLED = + buildConf("spark.sql.optimizer.rebalanceZorderColumns.enabled") + .doc(s"When true and ${REBALANCE_BEFORE_ZORDER.key} is true, we do rebalance before " + + s"Z-Order. If it's dynamic partition insert, the rebalance expression will include " + + s"both partition columns and Z-Order columns. Note that, this config only " + + s"affects with Spark 3.3.x") + .version("1.6.0") + .booleanConf + .createWithDefault(false) + + val TWO_PHASE_REBALANCE_BEFORE_ZORDER = + buildConf("spark.sql.optimizer.twoPhaseRebalanceBeforeZorder.enabled") + .doc(s"When true and ${REBALANCE_BEFORE_ZORDER.key} is true, we do two phase rebalance " + + s"before Z-Order for the dynamic partition write. The first phase rebalance using " + + s"dynamic partition column; The second phase rebalance using dynamic partition column + " + + s"Z-Order columns. Note that, this config only affects with Spark 3.3.x") + .version("1.6.0") + .booleanConf + .createWithDefault(false) + + val ZORDER_USING_ORIGINAL_ORDERING_ENABLED = + buildConf("spark.sql.optimizer.zorderUsingOriginalOrdering.enabled") + .doc(s"When true and ${REBALANCE_BEFORE_ZORDER.key} is true, we do sort by " + + s"the original ordering i.e. lexicographical order. Note that, this config only " + + s"affects with Spark 3.3.x") + .version("1.6.0") + .booleanConf + .createWithDefault(false) + + val WATCHDOG_MAX_PARTITIONS = + buildConf("spark.sql.watchdog.maxPartitions") + .doc("Set the max partition number when spark scans a data source. " + + "Enable maxPartitions Strategy by specifying this configuration. " + + "Add maxPartitions Strategy to avoid scan excessive partitions " + + "on partitioned table, it's optional that works with defined") + .version("1.4.0") + .intConf + .createOptional + + val WATCHDOG_MAX_FILE_SIZE = + buildConf("spark.sql.watchdog.maxFileSize") + .doc("Set the maximum size in bytes of files when spark scans a data source. " + + "Enable maxFileSize Strategy by specifying this configuration. " + + "Add maxFileSize Strategy to avoid scan excessive size of files," + + " it's optional that works with defined") + .version("1.8.0") + .bytesConf(ByteUnit.BYTE) + .createOptional + + val WATCHDOG_FORCED_MAXOUTPUTROWS = + buildConf("spark.sql.watchdog.forcedMaxOutputRows") + .doc("Add ForcedMaxOutputRows rule to avoid huge output rows of non-limit query " + + "unexpectedly, it's optional that works with defined") + .version("1.4.0") + .intConf + .createOptional + + val DROP_IGNORE_NONEXISTENT = + buildConf("spark.sql.optimizer.dropIgnoreNonExistent") + .doc("Do not report an error if DROP DATABASE/TABLE/VIEW/FUNCTION/PARTITION specifies " + + "a non-existent database/table/view/function/partition") + .version("1.5.0") + .booleanConf + .createWithDefault(false) + + val INFER_REBALANCE_AND_SORT_ORDERS = + buildConf("spark.sql.optimizer.inferRebalanceAndSortOrders.enabled") + .doc("When ture, infer columns for rebalance and sort orders from original query, " + + "e.g. the join keys from join. It can avoid compression ratio regression.") + .version("1.7.0") + .booleanConf + .createWithDefault(false) + + val INFER_REBALANCE_AND_SORT_ORDERS_MAX_COLUMNS = + buildConf("spark.sql.optimizer.inferRebalanceAndSortOrdersMaxColumns") + .doc("The max columns of inferred columns.") + .version("1.7.0") + .intConf + .checkValue(_ > 0, "must be positive number") + .createWithDefault(3) + + val INSERT_REPARTITION_BEFORE_WRITE_IF_NO_SHUFFLE = + buildConf("spark.sql.optimizer.insertRepartitionBeforeWriteIfNoShuffle.enabled") + .doc("When true, add repartition even if the original plan does not have shuffle.") + .version("1.7.0") + .booleanConf + .createWithDefault(false) + + val FINAL_STAGE_CONFIG_ISOLATION_WRITE_ONLY = + buildConf("spark.sql.optimizer.finalStageConfigIsolationWriteOnly.enabled") + .doc("When true, only enable final stage isolation for writing.") + .version("1.7.0") + .booleanConf + .createWithDefault(true) + + val FINAL_WRITE_STAGE_EAGERLY_KILL_EXECUTORS_ENABLED = + buildConf("spark.sql.finalWriteStage.eagerlyKillExecutors.enabled") + .doc("When true, eagerly kill redundant executors before running final write stage.") + .version("1.8.0") + .booleanConf + .createWithDefault(false) + + val FINAL_WRITE_STAGE_EAGERLY_KILL_EXECUTORS_KILL_ALL = + buildConf("spark.sql.finalWriteStage.eagerlyKillExecutors.killAll") + .doc("When true, eagerly kill all executors before running final write stage. " + + "Mainly for test.") + .version("1.8.0") + .booleanConf + .createWithDefault(false) + + val FINAL_WRITE_STAGE_SKIP_KILLING_EXECUTORS_FOR_TABLE_CACHE = + buildConf("spark.sql.finalWriteStage.skipKillingExecutorsForTableCache") + .doc("When true, skip killing executors if the plan has table caches.") + .version("1.8.0") + .booleanConf + .createWithDefault(true) + + val FINAL_WRITE_STAGE_PARTITION_FACTOR = + buildConf("spark.sql.finalWriteStage.retainExecutorsFactor") + .doc("If the target executors * factor < active executors, and " + + "target executors * factor > min executors, then kill redundant executors.") + .version("1.8.0") + .doubleConf + .checkValue(_ >= 1, "must be bigger than or equal to 1") + .createWithDefault(1.2) + + val FINAL_WRITE_STAGE_RESOURCE_ISOLATION_ENABLED = + buildConf("spark.sql.finalWriteStage.resourceIsolation.enabled") + .doc( + "When true, make final write stage resource isolation using custom RDD resource profile.") + .version("1.8.0") + .booleanConf + .createWithDefault(false) + + val FINAL_WRITE_STAGE_EXECUTOR_CORES = + buildConf("spark.sql.finalWriteStage.executorCores") + .doc("Specify the executor core request for final write stage. " + + "It would be passed to the RDD resource profile.") + .version("1.8.0") + .intConf + .createOptional + + val FINAL_WRITE_STAGE_EXECUTOR_MEMORY = + buildConf("spark.sql.finalWriteStage.executorMemory") + .doc("Specify the executor on heap memory request for final write stage. " + + "It would be passed to the RDD resource profile.") + .version("1.8.0") + .stringConf + .createOptional + + val FINAL_WRITE_STAGE_EXECUTOR_MEMORY_OVERHEAD = + buildConf("spark.sql.finalWriteStage.executorMemoryOverhead") + .doc("Specify the executor memory overhead request for final write stage. " + + "It would be passed to the RDD resource profile.") + .version("1.8.0") + .stringConf + .createOptional + + val FINAL_WRITE_STAGE_EXECUTOR_OFF_HEAP_MEMORY = + buildConf("spark.sql.finalWriteStage.executorOffHeapMemory") + .doc("Specify the executor off heap memory request for final write stage. " + + "It would be passed to the RDD resource profile.") + .version("1.8.0") + .stringConf + .createOptional + + val DYNAMIC_SHUFFLE_PARTITIONS = + buildConf("spark.sql.optimizer.dynamicShufflePartitions") + .doc("If true, adjust the number of shuffle partitions dynamically based on the job" + + " input size. The new number of partitions is the maximum input size" + + " divided by `spark.sql.adaptive.advisoryPartitionSizeInBytes`.") + .version("1.9.0") + .booleanConf + .createWithDefault(false) + + val DYNAMIC_SHUFFLE_PARTITIONS_MAX_NUM = + buildConf("spark.sql.optimizer.dynamicShufflePartitions.maxNum") + .doc("The maximum partition number of DynamicShufflePartitions.") + .version("1.9.0") + .intConf + .createWithDefault(2000) +} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/KyuubiSQLExtensionException.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/KyuubiSQLExtensionException.scala new file mode 100644 index 000000000..88c5a988f --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/KyuubiSQLExtensionException.scala @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql + +import java.sql.SQLException + +class KyuubiSQLExtensionException(reason: String, cause: Throwable) + extends SQLException(reason, cause) { + + def this(reason: String) = { + this(reason, null) + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLAstBuilder.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLAstBuilder.scala new file mode 100644 index 000000000..cc00bf88e --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLAstBuilder.scala @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql + +import scala.collection.JavaConverters.asScalaBufferConverter +import scala.collection.mutable.ListBuffer + +import org.antlr.v4.runtime.ParserRuleContext +import org.antlr.v4.runtime.misc.Interval +import org.antlr.v4.runtime.tree.ParseTree +import org.apache.spark.sql.catalyst.SQLConfHelper +import org.apache.spark.sql.catalyst.analysis.{UnresolvedAttribute, UnresolvedRelation, UnresolvedStar} +import org.apache.spark.sql.catalyst.expressions._ +import org.apache.spark.sql.catalyst.parser.ParserUtils.withOrigin +import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, Project, Sort} + +import org.apache.kyuubi.sql.KyuubiSparkSQLParser._ +import org.apache.kyuubi.sql.zorder.{OptimizeZorderStatement, Zorder} + +class KyuubiSparkSQLAstBuilder extends KyuubiSparkSQLBaseVisitor[AnyRef] with SQLConfHelper { + + def buildOptimizeStatement( + unparsedPredicateOptimize: UnparsedPredicateOptimize, + parseExpression: String => Expression): LogicalPlan = { + + val UnparsedPredicateOptimize(tableIdent, tablePredicate, orderExpr) = + unparsedPredicateOptimize + + val predicate = tablePredicate.map(parseExpression) + verifyPartitionPredicates(predicate) + val table = UnresolvedRelation(tableIdent) + val tableWithFilter = predicate match { + case Some(expr) => Filter(expr, table) + case None => table + } + val query = + Sort( + SortOrder(orderExpr, Ascending, NullsLast, Seq.empty) :: Nil, + conf.getConf(KyuubiSQLConf.ZORDER_GLOBAL_SORT_ENABLED), + Project(Seq(UnresolvedStar(None)), tableWithFilter)) + OptimizeZorderStatement(tableIdent, query) + } + + private def verifyPartitionPredicates(predicates: Option[Expression]): Unit = { + predicates.foreach { + case p if !isLikelySelective(p) => + throw new KyuubiSQLExtensionException(s"unsupported partition predicates: ${p.sql}") + case _ => + } + } + + /** + * Forked from Apache Spark's org.apache.spark.sql.catalyst.expressions.PredicateHelper + * The `PredicateHelper.isLikelySelective()` is available since Spark-3.3, forked for Spark + * that is lower than 3.3. + * + * Returns whether an expression is likely to be selective + */ + private def isLikelySelective(e: Expression): Boolean = e match { + case Not(expr) => isLikelySelective(expr) + case And(l, r) => isLikelySelective(l) || isLikelySelective(r) + case Or(l, r) => isLikelySelective(l) && isLikelySelective(r) + case _: StringRegexExpression => true + case _: BinaryComparison => true + case _: In | _: InSet => true + case _: StringPredicate => true + case BinaryPredicate(_) => true + case _: MultiLikeBase => true + case _ => false + } + + private object BinaryPredicate { + def unapply(expr: Expression): Option[Expression] = expr match { + case _: Contains => Option(expr) + case _: StartsWith => Option(expr) + case _: EndsWith => Option(expr) + case _ => None + } + } + + /** + * Create an expression from the given context. This method just passes the context on to the + * visitor and only takes care of typing (We assume that the visitor returns an Expression here). + */ + protected def expression(ctx: ParserRuleContext): Expression = typedVisit(ctx) + + protected def multiPart(ctx: ParserRuleContext): Seq[String] = typedVisit(ctx) + + override def visitSingleStatement(ctx: SingleStatementContext): LogicalPlan = { + visit(ctx.statement()).asInstanceOf[LogicalPlan] + } + + override def visitOptimizeZorder( + ctx: OptimizeZorderContext): UnparsedPredicateOptimize = withOrigin(ctx) { + val tableIdent = multiPart(ctx.multipartIdentifier()) + + val predicate = Option(ctx.whereClause()) + .map(_.partitionPredicate) + .map(extractRawText(_)) + + val zorderCols = ctx.zorderClause().order.asScala + .map(visitMultipartIdentifier) + .map(UnresolvedAttribute(_)) + .toSeq + + val orderExpr = + if (zorderCols.length == 1) { + zorderCols.head + } else { + Zorder(zorderCols) + } + UnparsedPredicateOptimize(tableIdent, predicate, orderExpr) + } + + override def visitPassThrough(ctx: PassThroughContext): LogicalPlan = null + + override def visitMultipartIdentifier(ctx: MultipartIdentifierContext): Seq[String] = + withOrigin(ctx) { + ctx.parts.asScala.map(_.getText).toSeq + } + + override def visitZorderClause(ctx: ZorderClauseContext): Seq[UnresolvedAttribute] = + withOrigin(ctx) { + val res = ListBuffer[UnresolvedAttribute]() + ctx.multipartIdentifier().forEach { identifier => + res += UnresolvedAttribute(identifier.parts.asScala.map(_.getText).toSeq) + } + res.toSeq + } + + private def typedVisit[T](ctx: ParseTree): T = { + ctx.accept(this).asInstanceOf[T] + } + + private def extractRawText(exprContext: ParserRuleContext): String = { + // Extract the raw expression which will be parsed later + exprContext.getStart.getInputStream.getText(new Interval( + exprContext.getStart.getStartIndex, + exprContext.getStop.getStopIndex)) + } +} + +/** + * a logical plan contains an unparsed expression that will be parsed by spark. + */ +trait UnparsedExpressionLogicalPlan extends LogicalPlan { + override def output: Seq[Attribute] = throw new UnsupportedOperationException() + + override def children: Seq[LogicalPlan] = throw new UnsupportedOperationException() + + protected def withNewChildrenInternal( + newChildren: IndexedSeq[LogicalPlan]): LogicalPlan = + throw new UnsupportedOperationException() +} + +case class UnparsedPredicateOptimize( + tableIdent: Seq[String], + tablePredicate: Option[String], + orderExpr: Expression) extends UnparsedExpressionLogicalPlan {} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLCommonExtension.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLCommonExtension.scala new file mode 100644 index 000000000..ad95ac429 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLCommonExtension.scala @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql + +import org.apache.spark.sql.SparkSessionExtensions + +import org.apache.kyuubi.sql.zorder.{InsertZorderBeforeWritingDatasource33, InsertZorderBeforeWritingHive33, ResolveZorder} + +class KyuubiSparkSQLCommonExtension extends (SparkSessionExtensions => Unit) { + override def apply(extensions: SparkSessionExtensions): Unit = { + KyuubiSparkSQLCommonExtension.injectCommonExtensions(extensions) + } +} + +object KyuubiSparkSQLCommonExtension { + def injectCommonExtensions(extensions: SparkSessionExtensions): Unit = { + // inject zorder parser and related rules + extensions.injectParser { case (_, parser) => new SparkKyuubiSparkSQLParser(parser) } + extensions.injectResolutionRule(ResolveZorder) + + // Note that: + // InsertZorderBeforeWritingDatasource and InsertZorderBeforeWritingHive + // should be applied before + // RepartitionBeforeWriting and RebalanceBeforeWriting + // because we can only apply one of them (i.e. Global Sort or Repartition/Rebalance) + extensions.injectPostHocResolutionRule(InsertZorderBeforeWritingDatasource33) + extensions.injectPostHocResolutionRule(InsertZorderBeforeWritingHive33) + extensions.injectPostHocResolutionRule(FinalStageConfigIsolationCleanRule) + + extensions.injectQueryStagePrepRule(_ => InsertShuffleNodeBeforeJoin) + extensions.injectQueryStagePrepRule(DynamicShufflePartitions) + + extensions.injectQueryStagePrepRule(FinalStageConfigIsolation(_)) + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLExtension.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLExtension.scala new file mode 100644 index 000000000..792315d89 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLExtension.scala @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql + +import org.apache.spark.sql.{FinalStageResourceManager, InjectCustomResourceProfile, SparkSessionExtensions} + +import org.apache.kyuubi.sql.watchdog.{ForcedMaxOutputRowsRule, MaxScanStrategy} + +// scalastyle:off line.size.limit +/** + * Depend on Spark SQL Extension framework, we can use this extension follow steps + * 1. move this jar into $SPARK_HOME/jars + * 2. add config into `spark-defaults.conf`: `spark.sql.extensions=org.apache.kyuubi.sql.KyuubiSparkSQLExtension` + */ +// scalastyle:on line.size.limit +class KyuubiSparkSQLExtension extends (SparkSessionExtensions => Unit) { + override def apply(extensions: SparkSessionExtensions): Unit = { + KyuubiSparkSQLCommonExtension.injectCommonExtensions(extensions) + + extensions.injectPostHocResolutionRule(RebalanceBeforeWritingDatasource) + extensions.injectPostHocResolutionRule(RebalanceBeforeWritingHive) + extensions.injectPostHocResolutionRule(DropIgnoreNonexistent) + + // watchdog extension + extensions.injectOptimizerRule(ForcedMaxOutputRowsRule) + extensions.injectPlannerStrategy(MaxScanStrategy) + + extensions.injectQueryStagePrepRule(FinalStageResourceManager(_)) + extensions.injectQueryStagePrepRule(InjectCustomResourceProfile) + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLParser.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLParser.scala new file mode 100644 index 000000000..c4418c33c --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLParser.scala @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql + +import org.antlr.v4.runtime._ +import org.antlr.v4.runtime.atn.PredictionMode +import org.antlr.v4.runtime.misc.{Interval, ParseCancellationException} +import org.apache.spark.sql.AnalysisException +import org.apache.spark.sql.catalyst.{FunctionIdentifier, SQLConfHelper, TableIdentifier} +import org.apache.spark.sql.catalyst.expressions.Expression +import org.apache.spark.sql.catalyst.parser.{ParseErrorListener, ParseException, ParserInterface, PostProcessor} +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.catalyst.trees.Origin +import org.apache.spark.sql.types.{DataType, StructType} + +abstract class KyuubiSparkSQLParserBase extends ParserInterface with SQLConfHelper { + def delegate: ParserInterface + def astBuilder: KyuubiSparkSQLAstBuilder + + override def parsePlan(sqlText: String): LogicalPlan = parse(sqlText) { parser => + astBuilder.visit(parser.singleStatement()) match { + case optimize: UnparsedPredicateOptimize => + astBuilder.buildOptimizeStatement(optimize, delegate.parseExpression) + case plan: LogicalPlan => plan + case _ => delegate.parsePlan(sqlText) + } + } + + protected def parse[T](command: String)(toResult: KyuubiSparkSQLParser => T): T = { + val lexer = new KyuubiSparkSQLLexer( + new UpperCaseCharStream(CharStreams.fromString(command))) + lexer.removeErrorListeners() + lexer.addErrorListener(ParseErrorListener) + + val tokenStream = new CommonTokenStream(lexer) + val parser = new KyuubiSparkSQLParser(tokenStream) + parser.addParseListener(PostProcessor) + parser.removeErrorListeners() + parser.addErrorListener(ParseErrorListener) + + try { + try { + // first, try parsing with potentially faster SLL mode + parser.getInterpreter.setPredictionMode(PredictionMode.SLL) + toResult(parser) + } catch { + case _: ParseCancellationException => + // if we fail, parse with LL mode + tokenStream.seek(0) // rewind input stream + parser.reset() + + // Try Again. + parser.getInterpreter.setPredictionMode(PredictionMode.LL) + toResult(parser) + } + } catch { + case e: ParseException if e.command.isDefined => + throw e + case e: ParseException => + throw e.withCommand(command) + case e: AnalysisException => + val position = Origin(e.line, e.startPosition) + throw new ParseException(Option(command), e.message, position, position) + } + } + + override def parseExpression(sqlText: String): Expression = { + delegate.parseExpression(sqlText) + } + + override def parseTableIdentifier(sqlText: String): TableIdentifier = { + delegate.parseTableIdentifier(sqlText) + } + + override def parseFunctionIdentifier(sqlText: String): FunctionIdentifier = { + delegate.parseFunctionIdentifier(sqlText) + } + + override def parseMultipartIdentifier(sqlText: String): Seq[String] = { + delegate.parseMultipartIdentifier(sqlText) + } + + override def parseTableSchema(sqlText: String): StructType = { + delegate.parseTableSchema(sqlText) + } + + override def parseDataType(sqlText: String): DataType = { + delegate.parseDataType(sqlText) + } + + /** + * This functions was introduced since spark-3.3, for more details, please see + * https://github.com/apache/spark/pull/34543 + */ + override def parseQuery(sqlText: String): LogicalPlan = { + delegate.parseQuery(sqlText) + } +} + +class SparkKyuubiSparkSQLParser( + override val delegate: ParserInterface) + extends KyuubiSparkSQLParserBase { + def astBuilder: KyuubiSparkSQLAstBuilder = new KyuubiSparkSQLAstBuilder +} + +/* Copied from Apache Spark's to avoid dependency on Spark Internals */ +class UpperCaseCharStream(wrapped: CodePointCharStream) extends CharStream { + override def consume(): Unit = wrapped.consume() + override def getSourceName(): String = wrapped.getSourceName + override def index(): Int = wrapped.index + override def mark(): Int = wrapped.mark + override def release(marker: Int): Unit = wrapped.release(marker) + override def seek(where: Int): Unit = wrapped.seek(where) + override def size(): Int = wrapped.size + + override def getText(interval: Interval): String = wrapped.getText(interval) + + // scalastyle:off + override def LA(i: Int): Int = { + val la = wrapped.LA(i) + if (la == 0 || la == IntStream.EOF) la + else Character.toUpperCase(la) + } + // scalastyle:on +} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/RebalanceBeforeWriting.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/RebalanceBeforeWriting.scala new file mode 100644 index 000000000..3cbacdd2f --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/RebalanceBeforeWriting.scala @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.expressions.{Ascending, Attribute, SortOrder} +import org.apache.spark.sql.catalyst.plans.logical._ + +trait RepartitionBuilderWithRebalance extends RepartitionBuilder { + override def buildRepartition( + dynamicPartitionColumns: Seq[Attribute], + query: LogicalPlan): LogicalPlan = { + if (!conf.getConf(KyuubiSQLConf.INFER_REBALANCE_AND_SORT_ORDERS) || + dynamicPartitionColumns.nonEmpty) { + RebalancePartitions(dynamicPartitionColumns, query) + } else { + val maxColumns = conf.getConf(KyuubiSQLConf.INFER_REBALANCE_AND_SORT_ORDERS_MAX_COLUMNS) + val inferred = InferRebalanceAndSortOrders.infer(query) + if (inferred.isDefined) { + val (partitioning, ordering) = inferred.get + val rebalance = RebalancePartitions(partitioning.take(maxColumns), query) + if (ordering.nonEmpty) { + val sortOrders = ordering.take(maxColumns).map(o => SortOrder(o, Ascending)) + Sort(sortOrders, false, rebalance) + } else { + rebalance + } + } else { + RebalancePartitions(dynamicPartitionColumns, query) + } + } + } + + override def canInsertRepartitionByExpression(plan: LogicalPlan): Boolean = { + super.canInsertRepartitionByExpression(plan) && { + plan match { + case _: RebalancePartitions => false + case _ => true + } + } + } +} + +/** + * For datasource table, there two commands can write data to table + * 1. InsertIntoHadoopFsRelationCommand + * 2. CreateDataSourceTableAsSelectCommand + * This rule add a RebalancePartitions node between write and query + */ +case class RebalanceBeforeWritingDatasource(session: SparkSession) + extends RepartitionBeforeWritingDatasourceBase + with RepartitionBuilderWithRebalance {} + +/** + * For Hive table, there two commands can write data to table + * 1. InsertIntoHiveTable + * 2. CreateHiveTableAsSelectCommand + * This rule add a RebalancePartitions node between write and query + */ +case class RebalanceBeforeWritingHive(session: SparkSession) + extends RepartitionBeforeWritingHiveBase + with RepartitionBuilderWithRebalance {} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/RepartitionBeforeWritingBase.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/RepartitionBeforeWritingBase.scala new file mode 100644 index 000000000..3ebb9740f --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/RepartitionBeforeWritingBase.scala @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql + +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.catalyst.plans.logical._ +import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand +import org.apache.spark.sql.hive.execution.InsertIntoHiveTable +import org.apache.spark.sql.internal.StaticSQLConf + +trait RepartitionBuilder extends Rule[LogicalPlan] with RepartitionBeforeWriteHelper { + def buildRepartition( + dynamicPartitionColumns: Seq[Attribute], + query: LogicalPlan): LogicalPlan +} + +/** + * For datasource table, there two commands can write data to table + * 1. InsertIntoHadoopFsRelationCommand + * 2. CreateDataSourceTableAsSelectCommand + * This rule add a repartition node between write and query + */ +abstract class RepartitionBeforeWritingDatasourceBase extends RepartitionBuilder { + + override def apply(plan: LogicalPlan): LogicalPlan = { + if (conf.getConf(KyuubiSQLConf.INSERT_REPARTITION_BEFORE_WRITE)) { + addRepartition(plan) + } else { + plan + } + } + + private def addRepartition(plan: LogicalPlan): LogicalPlan = plan match { + case i @ InsertIntoHadoopFsRelationCommand(_, sp, _, pc, bucket, _, _, query, _, _, _, _) + if query.resolved && bucket.isEmpty && canInsertRepartitionByExpression(query) => + val dynamicPartitionColumns = pc.filterNot(attr => sp.contains(attr.name)) + i.copy(query = buildRepartition(dynamicPartitionColumns, query)) + + case u @ Union(children, _, _) => + u.copy(children = children.map(addRepartition)) + + case _ => plan + } +} + +/** + * For Hive table, there two commands can write data to table + * 1. InsertIntoHiveTable + * 2. CreateHiveTableAsSelectCommand + * This rule add a repartition node between write and query + */ +abstract class RepartitionBeforeWritingHiveBase extends RepartitionBuilder { + override def apply(plan: LogicalPlan): LogicalPlan = { + if (conf.getConf(StaticSQLConf.CATALOG_IMPLEMENTATION) == "hive" && + conf.getConf(KyuubiSQLConf.INSERT_REPARTITION_BEFORE_WRITE)) { + addRepartition(plan) + } else { + plan + } + } + + def addRepartition(plan: LogicalPlan): LogicalPlan = plan match { + case i @ InsertIntoHiveTable(table, partition, query, _, _, _, _, _, _, _, _) + if query.resolved && table.bucketSpec.isEmpty && canInsertRepartitionByExpression(query) => + val dynamicPartitionColumns = partition.filter(_._2.isEmpty).keys + .flatMap(name => query.output.find(_.name == name)).toSeq + i.copy(query = buildRepartition(dynamicPartitionColumns, query)) + + case u @ Union(children, _, _) => + u.copy(children = children.map(addRepartition)) + + case _ => plan + } +} + +trait RepartitionBeforeWriteHelper extends Rule[LogicalPlan] { + private def hasBenefit(plan: LogicalPlan): Boolean = { + def probablyHasShuffle: Boolean = plan.find { + case _: Join => true + case _: Aggregate => true + case _: Distinct => true + case _: Deduplicate => true + case _: Window => true + case s: Sort if s.global => true + case _: RepartitionOperation => true + case _: GlobalLimit => true + case _ => false + }.isDefined + + conf.getConf(KyuubiSQLConf.INSERT_REPARTITION_BEFORE_WRITE_IF_NO_SHUFFLE) || probablyHasShuffle + } + + def canInsertRepartitionByExpression(plan: LogicalPlan): Boolean = { + def canInsert(p: LogicalPlan): Boolean = p match { + case Project(_, child) => canInsert(child) + case SubqueryAlias(_, child) => canInsert(child) + case Limit(_, _) => false + case _: Sort => false + case _: RepartitionByExpression => false + case _: Repartition => false + case _ => true + } + + // 1. make sure AQE is enabled, otherwise it is no meaning to add a shuffle + // 2. make sure it does not break the semantics of original plan + // 3. try to avoid adding a shuffle if it has potential performance regression + conf.adaptiveExecutionEnabled && canInsert(plan) && hasBenefit(plan) + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/WriteUtils.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/WriteUtils.scala new file mode 100644 index 000000000..89dd83194 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/WriteUtils.scala @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.execution.{SparkPlan, UnionExec} +import org.apache.spark.sql.execution.command.DataWritingCommandExec +import org.apache.spark.sql.execution.datasources.v2.V2TableWriteExec + +object WriteUtils { + def isWrite(session: SparkSession, plan: SparkPlan): Boolean = { + plan match { + case _: DataWritingCommandExec => true + case _: V2TableWriteExec => true + case u: UnionExec if u.children.nonEmpty => u.children.forall(isWrite(session, _)) + case _ => false + } + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/watchdog/ForcedMaxOutputRowsBase.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/watchdog/ForcedMaxOutputRowsBase.scala new file mode 100644 index 000000000..4f897d1b6 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/watchdog/ForcedMaxOutputRowsBase.scala @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql.watchdog + +import org.apache.spark.sql.catalyst.analysis.MultiInstanceRelation +import org.apache.spark.sql.catalyst.dsl.expressions._ +import org.apache.spark.sql.catalyst.expressions.Alias +import org.apache.spark.sql.catalyst.plans.logical._ +import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.execution.command.DataWritingCommand + +import org.apache.kyuubi.sql.KyuubiSQLConf + +/* + * Add ForcedMaxOutputRows rule for output rows limitation + * to avoid huge output rows of non_limit query unexpectedly + * mainly applied to cases as below: + * + * case 1: + * {{{ + * SELECT [c1, c2, ...] + * }}} + * + * case 2: + * {{{ + * WITH CTE AS ( + * ...) + * SELECT [c1, c2, ...] FROM CTE ... + * }}} + * + * The Logical Rule add a GlobalLimit node before root project + * */ +trait ForcedMaxOutputRowsBase extends Rule[LogicalPlan] { + + protected def isChildAggregate(a: Aggregate): Boolean + + protected def canInsertLimitInner(p: LogicalPlan): Boolean = p match { + case Aggregate(_, Alias(_, "havingCondition") :: Nil, _) => false + case agg: Aggregate => !isChildAggregate(agg) + case _: RepartitionByExpression => true + case _: Distinct => true + case _: Filter => true + case _: Project => true + case Limit(_, _) => true + case _: Sort => true + case Union(children, _, _) => + if (children.exists(_.isInstanceOf[DataWritingCommand])) { + false + } else { + true + } + case _: MultiInstanceRelation => true + case _: Join => true + case _ => false + } + + protected def canInsertLimit(p: LogicalPlan, maxOutputRowsOpt: Option[Int]): Boolean = { + maxOutputRowsOpt match { + case Some(forcedMaxOutputRows) => canInsertLimitInner(p) && + !p.maxRows.exists(_ <= forcedMaxOutputRows) + case None => false + } + } + + override def apply(plan: LogicalPlan): LogicalPlan = { + val maxOutputRowsOpt = conf.getConf(KyuubiSQLConf.WATCHDOG_FORCED_MAXOUTPUTROWS) + plan match { + case p if p.resolved && canInsertLimit(p, maxOutputRowsOpt) => + Limit( + maxOutputRowsOpt.get, + plan) + case _ => plan + } + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/watchdog/ForcedMaxOutputRowsRule.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/watchdog/ForcedMaxOutputRowsRule.scala new file mode 100644 index 000000000..a3d990b10 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/watchdog/ForcedMaxOutputRowsRule.scala @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql.watchdog + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.plans.logical.{Aggregate, CommandResult, LogicalPlan, Union, WithCTE} +import org.apache.spark.sql.execution.command.DataWritingCommand + +case class ForcedMaxOutputRowsRule(sparkSession: SparkSession) extends ForcedMaxOutputRowsBase { + + override protected def isChildAggregate(a: Aggregate): Boolean = false + + override protected def canInsertLimitInner(p: LogicalPlan): Boolean = p match { + case WithCTE(plan, _) => this.canInsertLimitInner(plan) + case plan: LogicalPlan => plan match { + case Union(children, _, _) => !children.exists { + case _: DataWritingCommand => true + case p: CommandResult if p.commandLogicalPlan.isInstanceOf[DataWritingCommand] => true + case _ => false + } + case _ => super.canInsertLimitInner(plan) + } + } + + override protected def canInsertLimit(p: LogicalPlan, maxOutputRowsOpt: Option[Int]): Boolean = { + p match { + case WithCTE(plan, _) => this.canInsertLimit(plan, maxOutputRowsOpt) + case _ => super.canInsertLimit(p, maxOutputRowsOpt) + } + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/watchdog/KyuubiWatchDogException.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/watchdog/KyuubiWatchDogException.scala new file mode 100644 index 000000000..e44309192 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/watchdog/KyuubiWatchDogException.scala @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql.watchdog + +import org.apache.kyuubi.sql.KyuubiSQLExtensionException + +final class MaxPartitionExceedException( + private val reason: String = "", + private val cause: Throwable = None.orNull) + extends KyuubiSQLExtensionException(reason, cause) + +final class MaxFileSizeExceedException( + private val reason: String = "", + private val cause: Throwable = None.orNull) + extends KyuubiSQLExtensionException(reason, cause) diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/watchdog/MaxScanStrategy.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/watchdog/MaxScanStrategy.scala new file mode 100644 index 000000000..1ed55ebc2 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/watchdog/MaxScanStrategy.scala @@ -0,0 +1,305 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql.watchdog + +import org.apache.hadoop.fs.Path +import org.apache.spark.sql.{PruneFileSourcePartitionHelper, SparkSession, Strategy} +import org.apache.spark.sql.catalyst.SQLConfHelper +import org.apache.spark.sql.catalyst.catalog.{CatalogTable, HiveTableRelation} +import org.apache.spark.sql.catalyst.planning.ScanOperation +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.execution.SparkPlan +import org.apache.spark.sql.execution.datasources.{CatalogFileIndex, HadoopFsRelation, InMemoryFileIndex, LogicalRelation} +import org.apache.spark.sql.types.StructType + +import org.apache.kyuubi.sql.KyuubiSQLConf + +/** + * Add MaxScanStrategy to avoid scan excessive partitions or files + * 1. Check if scan exceed maxPartition of partitioned table + * 2. Check if scan exceed maxFileSize (calculated by hive table and partition statistics) + * This Strategy Add Planner Strategy after LogicalOptimizer + * @param session + */ +case class MaxScanStrategy(session: SparkSession) + extends Strategy + with SQLConfHelper + with PruneFileSourcePartitionHelper { + override def apply(plan: LogicalPlan): Seq[SparkPlan] = { + val maxScanPartitionsOpt = conf.getConf(KyuubiSQLConf.WATCHDOG_MAX_PARTITIONS) + val maxFileSizeOpt = conf.getConf(KyuubiSQLConf.WATCHDOG_MAX_FILE_SIZE) + if (maxScanPartitionsOpt.isDefined || maxFileSizeOpt.isDefined) { + checkScan(plan, maxScanPartitionsOpt, maxFileSizeOpt) + } + Nil + } + + private def checkScan( + plan: LogicalPlan, + maxScanPartitionsOpt: Option[Int], + maxFileSizeOpt: Option[Long]): Unit = { + plan match { + case ScanOperation(_, _, _, relation: HiveTableRelation) => + if (relation.isPartitioned) { + relation.prunedPartitions match { + case Some(prunedPartitions) => + if (maxScanPartitionsOpt.exists(_ < prunedPartitions.size)) { + throw new MaxPartitionExceedException( + s""" + |SQL job scan hive partition: ${prunedPartitions.size} + |exceed restrict of hive scan maxPartition ${maxScanPartitionsOpt.get} + |You should optimize your SQL logical according partition structure + |or shorten query scope such as p_date, detail as below: + |Table: ${relation.tableMeta.qualifiedName} + |Owner: ${relation.tableMeta.owner} + |Partition Structure: ${relation.partitionCols.map(_.name).mkString(", ")} + |""".stripMargin) + } + lazy val scanFileSize = prunedPartitions.flatMap(_.stats).map(_.sizeInBytes).sum + if (maxFileSizeOpt.exists(_ < scanFileSize)) { + throw partTableMaxFileExceedError( + scanFileSize, + maxFileSizeOpt.get, + Some(relation.tableMeta), + prunedPartitions.flatMap(_.storage.locationUri).map(_.toString), + relation.partitionCols.map(_.name)) + } + case _ => + lazy val scanPartitions: Int = session + .sessionState.catalog.externalCatalog.listPartitionNames( + relation.tableMeta.database, + relation.tableMeta.identifier.table).size + if (maxScanPartitionsOpt.exists(_ < scanPartitions)) { + throw new MaxPartitionExceedException( + s""" + |Your SQL job scan a whole huge table without any partition filter, + |You should optimize your SQL logical according partition structure + |or shorten query scope such as p_date, detail as below: + |Table: ${relation.tableMeta.qualifiedName} + |Owner: ${relation.tableMeta.owner} + |Partition Structure: ${relation.partitionCols.map(_.name).mkString(", ")} + |""".stripMargin) + } + + lazy val scanFileSize: BigInt = + relation.tableMeta.stats.map(_.sizeInBytes).getOrElse { + session + .sessionState.catalog.externalCatalog.listPartitions( + relation.tableMeta.database, + relation.tableMeta.identifier.table).flatMap(_.stats).map(_.sizeInBytes).sum + } + if (maxFileSizeOpt.exists(_ < scanFileSize)) { + throw new MaxFileSizeExceedException( + s""" + |Your SQL job scan a whole huge table without any partition filter, + |You should optimize your SQL logical according partition structure + |or shorten query scope such as p_date, detail as below: + |Table: ${relation.tableMeta.qualifiedName} + |Owner: ${relation.tableMeta.owner} + |Partition Structure: ${relation.partitionCols.map(_.name).mkString(", ")} + |""".stripMargin) + } + } + } else { + lazy val scanFileSize = relation.tableMeta.stats.map(_.sizeInBytes).sum + if (maxFileSizeOpt.exists(_ < scanFileSize)) { + throw nonPartTableMaxFileExceedError( + scanFileSize, + maxFileSizeOpt.get, + Some(relation.tableMeta)) + } + } + case ScanOperation( + _, + _, + filters, + relation @ LogicalRelation( + fsRelation @ HadoopFsRelation( + fileIndex: InMemoryFileIndex, + partitionSchema, + _, + _, + _, + _), + _, + _, + _)) => + if (fsRelation.partitionSchema.nonEmpty) { + val (partitionKeyFilters, dataFilter) = + getPartitionKeyFiltersAndDataFilters( + SparkSession.active, + relation, + partitionSchema, + filters, + relation.output) + val prunedPartitions = fileIndex.listFiles( + partitionKeyFilters.toSeq, + dataFilter) + if (maxScanPartitionsOpt.exists(_ < prunedPartitions.size)) { + throw maxPartitionExceedError( + prunedPartitions.size, + maxScanPartitionsOpt.get, + relation.catalogTable, + fileIndex.rootPaths, + fsRelation.partitionSchema) + } + lazy val scanFileSize = prunedPartitions.flatMap(_.files).map(_.getLen).sum + if (maxFileSizeOpt.exists(_ < scanFileSize)) { + throw partTableMaxFileExceedError( + scanFileSize, + maxFileSizeOpt.get, + relation.catalogTable, + fileIndex.rootPaths.map(_.toString), + fsRelation.partitionSchema.map(_.name)) + } + } else { + lazy val scanFileSize = fileIndex.sizeInBytes + if (maxFileSizeOpt.exists(_ < scanFileSize)) { + throw nonPartTableMaxFileExceedError( + scanFileSize, + maxFileSizeOpt.get, + relation.catalogTable) + } + } + case ScanOperation( + _, + _, + filters, + logicalRelation @ LogicalRelation( + fsRelation @ HadoopFsRelation( + catalogFileIndex: CatalogFileIndex, + partitionSchema, + _, + _, + _, + _), + _, + _, + _)) => + if (fsRelation.partitionSchema.nonEmpty) { + val (partitionKeyFilters, _) = + getPartitionKeyFiltersAndDataFilters( + SparkSession.active, + logicalRelation, + partitionSchema, + filters, + logicalRelation.output) + + val fileIndex = catalogFileIndex.filterPartitions( + partitionKeyFilters.toSeq) + + lazy val prunedPartitionSize = fileIndex.partitionSpec().partitions.size + if (maxScanPartitionsOpt.exists(_ < prunedPartitionSize)) { + throw maxPartitionExceedError( + prunedPartitionSize, + maxScanPartitionsOpt.get, + logicalRelation.catalogTable, + catalogFileIndex.rootPaths, + fsRelation.partitionSchema) + } + + lazy val scanFileSize = fileIndex + .listFiles(Nil, Nil).flatMap(_.files).map(_.getLen).sum + if (maxFileSizeOpt.exists(_ < scanFileSize)) { + throw partTableMaxFileExceedError( + scanFileSize, + maxFileSizeOpt.get, + logicalRelation.catalogTable, + catalogFileIndex.rootPaths.map(_.toString), + fsRelation.partitionSchema.map(_.name)) + } + } else { + lazy val scanFileSize = catalogFileIndex.sizeInBytes + if (maxFileSizeOpt.exists(_ < scanFileSize)) { + throw nonPartTableMaxFileExceedError( + scanFileSize, + maxFileSizeOpt.get, + logicalRelation.catalogTable) + } + } + case _ => + } + } + + def maxPartitionExceedError( + prunedPartitionSize: Int, + maxPartitionSize: Int, + tableMeta: Option[CatalogTable], + rootPaths: Seq[Path], + partitionSchema: StructType): Throwable = { + val truncatedPaths = + if (rootPaths.length > 5) { + rootPaths.slice(0, 5).mkString(",") + """... """ + (rootPaths.length - 5) + " more paths" + } else { + rootPaths.mkString(",") + } + + new MaxPartitionExceedException( + s""" + |SQL job scan data source partition: $prunedPartitionSize + |exceed restrict of data source scan maxPartition $maxPartitionSize + |You should optimize your SQL logical according partition structure + |or shorten query scope such as p_date, detail as below: + |Table: ${tableMeta.map(_.qualifiedName).getOrElse("")} + |Owner: ${tableMeta.map(_.owner).getOrElse("")} + |RootPaths: $truncatedPaths + |Partition Structure: ${partitionSchema.map(_.name).mkString(", ")} + |""".stripMargin) + } + + private def partTableMaxFileExceedError( + scanFileSize: Number, + maxFileSize: Long, + tableMeta: Option[CatalogTable], + rootPaths: Seq[String], + partitions: Seq[String]): Throwable = { + val truncatedPaths = + if (rootPaths.length > 5) { + rootPaths.slice(0, 5).mkString(",") + """... """ + (rootPaths.length - 5) + " more paths" + } else { + rootPaths.mkString(",") + } + + new MaxFileSizeExceedException( + s""" + |SQL job scan file size in bytes: $scanFileSize + |exceed restrict of table scan maxFileSize $maxFileSize + |You should optimize your SQL logical according partition structure + |or shorten query scope such as p_date, detail as below: + |Table: ${tableMeta.map(_.qualifiedName).getOrElse("")} + |Owner: ${tableMeta.map(_.owner).getOrElse("")} + |RootPaths: $truncatedPaths + |Partition Structure: ${partitions.mkString(", ")} + |""".stripMargin) + } + + private def nonPartTableMaxFileExceedError( + scanFileSize: Number, + maxFileSize: Long, + tableMeta: Option[CatalogTable]): Throwable = { + new MaxFileSizeExceedException( + s""" + |SQL job scan file size in bytes: $scanFileSize + |exceed restrict of table scan maxFileSize $maxFileSize + |detail as below: + |Table: ${tableMeta.map(_.qualifiedName).getOrElse("")} + |Owner: ${tableMeta.map(_.owner).getOrElse("")} + |Location: ${tableMeta.map(_.location).getOrElse("")} + |""".stripMargin) + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/zorder/InsertZorderBeforeWriting.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/zorder/InsertZorderBeforeWriting.scala new file mode 100644 index 000000000..b3f98ec6d --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/zorder/InsertZorderBeforeWriting.scala @@ -0,0 +1,177 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql.zorder + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.catalog.CatalogTable +import org.apache.spark.sql.catalyst.expressions.{Ascending, Attribute, Expression, NullsLast, SortOrder} +import org.apache.spark.sql.catalyst.plans.logical._ +import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand +import org.apache.spark.sql.hive.execution.InsertIntoHiveTable + +import org.apache.kyuubi.sql.{KyuubiSQLConf, KyuubiSQLExtensionException} + +trait InsertZorderHelper33 extends Rule[LogicalPlan] with ZorderBuilder { + private val KYUUBI_ZORDER_ENABLED = "kyuubi.zorder.enabled" + private val KYUUBI_ZORDER_COLS = "kyuubi.zorder.cols" + + def isZorderEnabled(props: Map[String, String]): Boolean = { + props.contains(KYUUBI_ZORDER_ENABLED) && + "true".equalsIgnoreCase(props(KYUUBI_ZORDER_ENABLED)) && + props.contains(KYUUBI_ZORDER_COLS) + } + + def getZorderColumns(props: Map[String, String]): Seq[String] = { + val cols = props.get(KYUUBI_ZORDER_COLS) + assert(cols.isDefined) + cols.get.split(",").map(_.trim) + } + + def canInsertZorder(query: LogicalPlan): Boolean = query match { + case Project(_, child) => canInsertZorder(child) + // TODO: actually, we can force zorder even if existed some shuffle + case _: Sort => false + case _: RepartitionByExpression => false + case _: Repartition => false + case _ => true + } + + def insertZorder( + catalogTable: CatalogTable, + plan: LogicalPlan, + dynamicPartitionColumns: Seq[Attribute]): LogicalPlan = { + if (!canInsertZorder(plan)) { + return plan + } + val cols = getZorderColumns(catalogTable.properties) + val resolver = session.sessionState.conf.resolver + val output = plan.output + val bound = cols.flatMap(col => output.find(attr => resolver(attr.name, col))) + if (bound.size < cols.size) { + logWarning(s"target table does not contain all zorder cols: ${cols.mkString(",")}, " + + s"please check your table properties ${KYUUBI_ZORDER_COLS}.") + plan + } else { + if (conf.getConf(KyuubiSQLConf.ZORDER_GLOBAL_SORT_ENABLED) && + conf.getConf(KyuubiSQLConf.REBALANCE_BEFORE_ZORDER)) { + throw new KyuubiSQLExtensionException(s"${KyuubiSQLConf.ZORDER_GLOBAL_SORT_ENABLED.key} " + + s"and ${KyuubiSQLConf.REBALANCE_BEFORE_ZORDER.key} can not be enabled together.") + } + if (conf.getConf(KyuubiSQLConf.ZORDER_GLOBAL_SORT_ENABLED) && + dynamicPartitionColumns.nonEmpty) { + logWarning(s"Dynamic partition insertion with global sort may produce small files.") + } + + val zorderExpr = + if (bound.length == 1) { + bound + } else if (conf.getConf(KyuubiSQLConf.ZORDER_USING_ORIGINAL_ORDERING_ENABLED)) { + bound.asInstanceOf[Seq[Expression]] + } else { + buildZorder(bound) :: Nil + } + val (global, orderExprs, child) = + if (conf.getConf(KyuubiSQLConf.ZORDER_GLOBAL_SORT_ENABLED)) { + (true, zorderExpr, plan) + } else if (conf.getConf(KyuubiSQLConf.REBALANCE_BEFORE_ZORDER)) { + val rebalanceExpr = + if (dynamicPartitionColumns.isEmpty) { + // static partition insert + bound + } else if (conf.getConf(KyuubiSQLConf.REBALANCE_ZORDER_COLUMNS_ENABLED)) { + // improve data compression ratio + dynamicPartitionColumns.asInstanceOf[Seq[Expression]] ++ bound + } else { + dynamicPartitionColumns.asInstanceOf[Seq[Expression]] + } + // for dynamic partition insert, Spark always sort the partition columns, + // so here we sort partition columns + zorder. + val rebalance = + if (dynamicPartitionColumns.nonEmpty && + conf.getConf(KyuubiSQLConf.TWO_PHASE_REBALANCE_BEFORE_ZORDER)) { + // improve compression ratio + RebalancePartitions( + rebalanceExpr, + RebalancePartitions(dynamicPartitionColumns, plan)) + } else { + RebalancePartitions(rebalanceExpr, plan) + } + (false, dynamicPartitionColumns.asInstanceOf[Seq[Expression]] ++ zorderExpr, rebalance) + } else { + (false, zorderExpr, plan) + } + val order = orderExprs.map { expr => + SortOrder(expr, Ascending, NullsLast, Seq.empty) + } + Sort(order, global, child) + } + } + + override def buildZorder(children: Seq[Expression]): ZorderBase = Zorder(children) + + def session: SparkSession + def applyInternal(plan: LogicalPlan): LogicalPlan + + final override def apply(plan: LogicalPlan): LogicalPlan = { + if (conf.getConf(KyuubiSQLConf.INSERT_ZORDER_BEFORE_WRITING)) { + applyInternal(plan) + } else { + plan + } + } +} + +case class InsertZorderBeforeWritingDatasource33(session: SparkSession) + extends InsertZorderHelper33 { + override def applyInternal(plan: LogicalPlan): LogicalPlan = plan match { + case insert: InsertIntoHadoopFsRelationCommand + if insert.query.resolved && + insert.bucketSpec.isEmpty && insert.catalogTable.isDefined && + isZorderEnabled(insert.catalogTable.get.properties) => + val dynamicPartition = + insert.partitionColumns.filterNot(attr => insert.staticPartitions.contains(attr.name)) + val newQuery = insertZorder(insert.catalogTable.get, insert.query, dynamicPartition) + if (newQuery.eq(insert.query)) { + insert + } else { + insert.copy(query = newQuery) + } + + case _ => plan + } +} + +case class InsertZorderBeforeWritingHive33(session: SparkSession) + extends InsertZorderHelper33 { + override def applyInternal(plan: LogicalPlan): LogicalPlan = plan match { + case insert: InsertIntoHiveTable + if insert.query.resolved && + insert.table.bucketSpec.isEmpty && isZorderEnabled(insert.table.properties) => + val dynamicPartition = insert.partition.filter(_._2.isEmpty).keys + .flatMap(name => insert.query.output.find(_.name == name)).toSeq + val newQuery = insertZorder(insert.table, insert.query, dynamicPartition) + if (newQuery.eq(insert.query)) { + insert + } else { + insert.copy(query = newQuery) + } + + case _ => plan + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/zorder/InsertZorderBeforeWritingBase.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/zorder/InsertZorderBeforeWritingBase.scala new file mode 100644 index 000000000..2c59d148e --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/zorder/InsertZorderBeforeWritingBase.scala @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql.zorder + +import java.util.Locale + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.catalog.CatalogTable +import org.apache.spark.sql.catalyst.expressions.{Ascending, Expression, NullsLast, SortOrder} +import org.apache.spark.sql.catalyst.plans.logical._ +import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand +import org.apache.spark.sql.hive.execution.InsertIntoHiveTable + +import org.apache.kyuubi.sql.KyuubiSQLConf + +/** + * TODO: shall we forbid zorder if it's dynamic partition inserts ? + * Insert zorder before writing datasource if the target table properties has zorder properties + */ +abstract class InsertZorderBeforeWritingDatasourceBase + extends InsertZorderHelper { + override def applyInternal(plan: LogicalPlan): LogicalPlan = plan match { + case insert: InsertIntoHadoopFsRelationCommand + if insert.query.resolved && insert.bucketSpec.isEmpty && insert.catalogTable.isDefined && + isZorderEnabled(insert.catalogTable.get.properties) => + val newQuery = insertZorder(insert.catalogTable.get, insert.query) + if (newQuery.eq(insert.query)) { + insert + } else { + insert.copy(query = newQuery) + } + case _ => plan + } +} + +/** + * TODO: shall we forbid zorder if it's dynamic partition inserts ? + * Insert zorder before writing hive if the target table properties has zorder properties + */ +abstract class InsertZorderBeforeWritingHiveBase + extends InsertZorderHelper { + override def applyInternal(plan: LogicalPlan): LogicalPlan = plan match { + case insert: InsertIntoHiveTable + if insert.query.resolved && insert.table.bucketSpec.isEmpty && + isZorderEnabled(insert.table.properties) => + val newQuery = insertZorder(insert.table, insert.query) + if (newQuery.eq(insert.query)) { + insert + } else { + insert.copy(query = newQuery) + } + case _ => plan + } +} + +trait ZorderBuilder { + def buildZorder(children: Seq[Expression]): ZorderBase +} + +trait InsertZorderHelper extends Rule[LogicalPlan] with ZorderBuilder { + private val KYUUBI_ZORDER_ENABLED = "kyuubi.zorder.enabled" + private val KYUUBI_ZORDER_COLS = "kyuubi.zorder.cols" + + def isZorderEnabled(props: Map[String, String]): Boolean = { + props.contains(KYUUBI_ZORDER_ENABLED) && + "true".equalsIgnoreCase(props(KYUUBI_ZORDER_ENABLED)) && + props.contains(KYUUBI_ZORDER_COLS) + } + + def getZorderColumns(props: Map[String, String]): Seq[String] = { + val cols = props.get(KYUUBI_ZORDER_COLS) + assert(cols.isDefined) + cols.get.split(",").map(_.trim.toLowerCase(Locale.ROOT)) + } + + def canInsertZorder(query: LogicalPlan): Boolean = query match { + case Project(_, child) => canInsertZorder(child) + // TODO: actually, we can force zorder even if existed some shuffle + case _: Sort => false + case _: RepartitionByExpression => false + case _: Repartition => false + case _ => true + } + + def insertZorder(catalogTable: CatalogTable, plan: LogicalPlan): LogicalPlan = { + if (!canInsertZorder(plan)) { + return plan + } + val cols = getZorderColumns(catalogTable.properties) + val attrs = plan.output.map(attr => (attr.name, attr)).toMap + if (cols.exists(!attrs.contains(_))) { + logWarning(s"target table does not contain all zorder cols: ${cols.mkString(",")}, " + + s"please check your table properties ${KYUUBI_ZORDER_COLS}.") + plan + } else { + val bound = cols.map(attrs(_)) + val orderExpr = + if (bound.length == 1) { + bound.head + } else { + buildZorder(bound) + } + // TODO: We can do rebalance partitions before local sort of zorder after SPARK 3.3 + // see https://github.com/apache/spark/pull/34542 + Sort( + SortOrder(orderExpr, Ascending, NullsLast, Seq.empty) :: Nil, + conf.getConf(KyuubiSQLConf.ZORDER_GLOBAL_SORT_ENABLED), + plan) + } + } + + def applyInternal(plan: LogicalPlan): LogicalPlan + + final override def apply(plan: LogicalPlan): LogicalPlan = { + if (conf.getConf(KyuubiSQLConf.INSERT_ZORDER_BEFORE_WRITING)) { + applyInternal(plan) + } else { + plan + } + } +} + +/** + * TODO: shall we forbid zorder if it's dynamic partition inserts ? + * Insert zorder before writing datasource if the target table properties has zorder properties + */ +case class InsertZorderBeforeWritingDatasource(session: SparkSession) + extends InsertZorderBeforeWritingDatasourceBase { + override def buildZorder(children: Seq[Expression]): ZorderBase = Zorder(children) +} + +/** + * TODO: shall we forbid zorder if it's dynamic partition inserts ? + * Insert zorder before writing hive if the target table properties has zorder properties + */ +case class InsertZorderBeforeWritingHive(session: SparkSession) + extends InsertZorderBeforeWritingHiveBase { + override def buildZorder(children: Seq[Expression]): ZorderBase = Zorder(children) +} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/zorder/OptimizeZorderCommandBase.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/zorder/OptimizeZorderCommandBase.scala new file mode 100644 index 000000000..21d1cf2a2 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/zorder/OptimizeZorderCommandBase.scala @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql.zorder + +import org.apache.spark.sql.{Row, SparkSession} +import org.apache.spark.sql.catalyst.catalog.CatalogTable +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.execution.SparkPlan +import org.apache.spark.sql.execution.command.DataWritingCommand +import org.apache.spark.sql.hive.execution.InsertIntoHiveTable + +import org.apache.kyuubi.sql.KyuubiSQLExtensionException + +/** + * A runnable command for zorder, we delegate to real command to execute + */ +abstract class OptimizeZorderCommandBase extends DataWritingCommand { + def catalogTable: CatalogTable + + override def outputColumnNames: Seq[String] = query.output.map(_.name) + + private def isHiveTable: Boolean = { + catalogTable.provider.isEmpty || + (catalogTable.provider.isDefined && "hive".equalsIgnoreCase(catalogTable.provider.get)) + } + + private def getWritingCommand(session: SparkSession): DataWritingCommand = { + // TODO: Support convert hive relation to datasource relation, can see + // [[org.apache.spark.sql.hive.RelationConversions]] + InsertIntoHiveTable( + catalogTable, + catalogTable.partitionColumnNames.map(p => (p, None)).toMap, + query, + overwrite = true, + ifPartitionNotExists = false, + outputColumnNames) + } + + override def run(session: SparkSession, child: SparkPlan): Seq[Row] = { + // TODO: Support datasource relation + // TODO: Support read and insert overwrite the same table for some table format + if (!isHiveTable) { + throw new KyuubiSQLExtensionException("only support hive table") + } + + val command = getWritingCommand(session) + command.run(session, child) + DataWritingCommand.propogateMetrics(session.sparkContext, command, metrics) + Seq.empty + } +} + +/** + * A runnable command for zorder, we delegate to real command to execute + */ +case class OptimizeZorderCommand( + catalogTable: CatalogTable, + query: LogicalPlan) + extends OptimizeZorderCommandBase { + protected def withNewChildInternal(newChild: LogicalPlan): LogicalPlan = { + copy(query = newChild) + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/zorder/OptimizeZorderStatementBase.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/zorder/OptimizeZorderStatementBase.scala new file mode 100644 index 000000000..895f9e24b --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/zorder/OptimizeZorderStatementBase.scala @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql.zorder + +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, UnaryNode} + +/** + * A zorder statement that contains we parsed from SQL. + * We should convert this plan to certain command at Analyzer. + */ +case class OptimizeZorderStatement( + tableIdentifier: Seq[String], + query: LogicalPlan) extends UnaryNode { + override def child: LogicalPlan = query + override def output: Seq[Attribute] = child.output + protected def withNewChildInternal(newChild: LogicalPlan): LogicalPlan = + copy(query = newChild) +} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/zorder/ResolveZorderBase.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/zorder/ResolveZorderBase.scala new file mode 100644 index 000000000..9f735caa7 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/zorder/ResolveZorderBase.scala @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql.zorder + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.catalog.{CatalogTable, HiveTableRelation} +import org.apache.spark.sql.catalyst.expressions.AttributeSet +import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, SubqueryAlias} +import org.apache.spark.sql.catalyst.rules.Rule + +import org.apache.kyuubi.sql.KyuubiSQLExtensionException + +/** + * Resolve `OptimizeZorderStatement` to `OptimizeZorderCommand` + */ +abstract class ResolveZorderBase extends Rule[LogicalPlan] { + def session: SparkSession + def buildOptimizeZorderCommand( + catalogTable: CatalogTable, + query: LogicalPlan): OptimizeZorderCommandBase + + protected def checkQueryAllowed(query: LogicalPlan): Unit = query foreach { + case Filter(condition, SubqueryAlias(_, tableRelation: HiveTableRelation)) => + if (tableRelation.partitionCols.isEmpty) { + throw new KyuubiSQLExtensionException("Filters are only supported for partitioned table") + } + + val partitionKeyIds = AttributeSet(tableRelation.partitionCols) + if (condition.references.isEmpty || !condition.references.subsetOf(partitionKeyIds)) { + throw new KyuubiSQLExtensionException("Only partition column filters are allowed") + } + + case _ => + } + + protected def getTableIdentifier(tableIdent: Seq[String]): TableIdentifier = tableIdent match { + case Seq(tbl) => TableIdentifier.apply(tbl) + case Seq(db, tbl) => TableIdentifier.apply(tbl, Some(db)) + case _ => throw new KyuubiSQLExtensionException( + "only support session catalog table, please use db.table instead") + } + + override def apply(plan: LogicalPlan): LogicalPlan = plan match { + case statement: OptimizeZorderStatement if statement.query.resolved => + checkQueryAllowed(statement.query) + val tableIdentifier = getTableIdentifier(statement.tableIdentifier) + val catalogTable = session.sessionState.catalog.getTableMetadata(tableIdentifier) + buildOptimizeZorderCommand(catalogTable, statement.query) + + case _ => plan + } +} + +/** + * Resolve `OptimizeZorderStatement` to `OptimizeZorderCommand` + */ +case class ResolveZorder(session: SparkSession) extends ResolveZorderBase { + override def buildOptimizeZorderCommand( + catalogTable: CatalogTable, + query: LogicalPlan): OptimizeZorderCommandBase = { + OptimizeZorderCommand(catalogTable, query) + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/zorder/ZorderBase.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/zorder/ZorderBase.scala new file mode 100644 index 000000000..e4d98ccbe --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/zorder/ZorderBase.scala @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql.zorder + +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.analysis.TypeCheckResult +import org.apache.spark.sql.catalyst.expressions.Expression +import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode, FalseLiteral} +import org.apache.spark.sql.catalyst.expressions.codegen.Block._ +import org.apache.spark.sql.types.{BinaryType, DataType} + +import org.apache.kyuubi.sql.KyuubiSQLExtensionException + +abstract class ZorderBase extends Expression { + override def foldable: Boolean = children.forall(_.foldable) + override def nullable: Boolean = false + override def dataType: DataType = BinaryType + override def prettyName: String = "zorder" + + override def checkInputDataTypes(): TypeCheckResult = { + try { + defaultNullValues + TypeCheckResult.TypeCheckSuccess + } catch { + case e: KyuubiSQLExtensionException => + TypeCheckResult.TypeCheckFailure(e.getMessage) + } + } + + @transient + private[this] lazy val defaultNullValues: Array[Any] = + children.map(_.dataType) + .map(ZorderBytesUtils.defaultValue) + .toArray + + override def eval(input: InternalRow): Any = { + val childrenValues = children.zipWithIndex.map { + case (child: Expression, index) => + val v = child.eval(input) + if (v == null) { + defaultNullValues(index) + } else { + v + } + } + ZorderBytesUtils.interleaveBits(childrenValues.toArray) + } + + override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { + val evals = children.map(_.genCode(ctx)) + val defaultValues = ctx.addReferenceObj("defaultValues", defaultNullValues) + val values = ctx.freshName("values") + val util = ZorderBytesUtils.getClass.getName.stripSuffix("$") + val inputs = evals.zipWithIndex.map { + case (eval, index) => + s""" + |${eval.code} + |if (${eval.isNull}) { + | $values[$index] = $defaultValues[$index]; + |} else { + | $values[$index] = ${eval.value}; + |} + |""".stripMargin + } + ev.copy( + code = + code""" + |byte[] ${ev.value} = null; + |Object[] $values = new Object[${evals.length}]; + |${inputs.mkString("\n")} + |${ev.value} = $util.interleaveBits($values); + |""".stripMargin, + isNull = FalseLiteral) + } +} + +case class Zorder(children: Seq[Expression]) extends ZorderBase { + protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]): Expression = + copy(children = newChildren) +} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/zorder/ZorderBytesUtils.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/zorder/ZorderBytesUtils.scala new file mode 100644 index 000000000..d249f1dc3 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/kyuubi/sql/zorder/ZorderBytesUtils.scala @@ -0,0 +1,517 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql.zorder + +import java.lang.{Double => jDouble, Float => jFloat} + +import org.apache.spark.sql.types._ +import org.apache.spark.unsafe.types.UTF8String + +import org.apache.kyuubi.sql.KyuubiSQLExtensionException + +object ZorderBytesUtils { + final private val BIT_8_MASK = 1 << 7 + final private val BIT_16_MASK = 1 << 15 + final private val BIT_32_MASK = 1 << 31 + final private val BIT_64_MASK = 1L << 63 + + def interleaveBits(inputs: Array[Any]): Array[Byte] = { + inputs.length match { + // it's a more fast approach, use O(8 * 8) + // can see http://graphics.stanford.edu/~seander/bithacks.html#InterleaveTableObvious + case 1 => longToByte(toLong(inputs(0))) + case 2 => interleave2Longs(toLong(inputs(0)), toLong(inputs(1))) + case 3 => interleave3Longs(toLong(inputs(0)), toLong(inputs(1)), toLong(inputs(2))) + case 4 => + interleave4Longs(toLong(inputs(0)), toLong(inputs(1)), toLong(inputs(2)), toLong(inputs(3))) + case 5 => interleave5Longs( + toLong(inputs(0)), + toLong(inputs(1)), + toLong(inputs(2)), + toLong(inputs(3)), + toLong(inputs(4))) + case 6 => interleave6Longs( + toLong(inputs(0)), + toLong(inputs(1)), + toLong(inputs(2)), + toLong(inputs(3)), + toLong(inputs(4)), + toLong(inputs(5))) + case 7 => interleave7Longs( + toLong(inputs(0)), + toLong(inputs(1)), + toLong(inputs(2)), + toLong(inputs(3)), + toLong(inputs(4)), + toLong(inputs(5)), + toLong(inputs(6))) + case 8 => interleave8Longs( + toLong(inputs(0)), + toLong(inputs(1)), + toLong(inputs(2)), + toLong(inputs(3)), + toLong(inputs(4)), + toLong(inputs(5)), + toLong(inputs(6)), + toLong(inputs(7))) + + case _ => + // it's the default approach, use O(64 * n), n is the length of inputs + interleaveBitsDefault(inputs.map(toByteArray)) + } + } + + private def interleave2Longs(l1: Long, l2: Long): Array[Byte] = { + // output 8 * 16 bits + val result = new Array[Byte](16) + var i = 0 + while (i < 8) { + val tmp1 = ((l1 >> (i * 8)) & 0xFF).toShort + val tmp2 = ((l2 >> (i * 8)) & 0xFF).toShort + + var z = 0 + var j = 0 + while (j < 8) { + val x_masked = tmp1 & (1 << j) + val y_masked = tmp2 & (1 << j) + z |= (x_masked << j) + z |= (y_masked << (j + 1)) + j = j + 1 + } + result((7 - i) * 2 + 1) = (z & 0xFF).toByte + result((7 - i) * 2) = ((z >> 8) & 0xFF).toByte + i = i + 1 + } + result + } + + private def interleave3Longs(l1: Long, l2: Long, l3: Long): Array[Byte] = { + // output 8 * 24 bits + val result = new Array[Byte](24) + var i = 0 + while (i < 8) { + val tmp1 = ((l1 >> (i * 8)) & 0xFF).toInt + val tmp2 = ((l2 >> (i * 8)) & 0xFF).toInt + val tmp3 = ((l3 >> (i * 8)) & 0xFF).toInt + + var z = 0 + var j = 0 + while (j < 8) { + val r1_mask = tmp1 & (1 << j) + val r2_mask = tmp2 & (1 << j) + val r3_mask = tmp3 & (1 << j) + z |= (r1_mask << (2 * j)) | (r2_mask << (2 * j + 1)) | (r3_mask << (2 * j + 2)) + j = j + 1 + } + result((7 - i) * 3 + 2) = (z & 0xFF).toByte + result((7 - i) * 3 + 1) = ((z >> 8) & 0xFF).toByte + result((7 - i) * 3) = ((z >> 16) & 0xFF).toByte + i = i + 1 + } + result + } + + private def interleave4Longs(l1: Long, l2: Long, l3: Long, l4: Long): Array[Byte] = { + // output 8 * 32 bits + val result = new Array[Byte](32) + var i = 0 + while (i < 8) { + val tmp1 = ((l1 >> (i * 8)) & 0xFF).toInt + val tmp2 = ((l2 >> (i * 8)) & 0xFF).toInt + val tmp3 = ((l3 >> (i * 8)) & 0xFF).toInt + val tmp4 = ((l4 >> (i * 8)) & 0xFF).toInt + + var z = 0 + var j = 0 + while (j < 8) { + val r1_mask = tmp1 & (1 << j) + val r2_mask = tmp2 & (1 << j) + val r3_mask = tmp3 & (1 << j) + val r4_mask = tmp4 & (1 << j) + z |= (r1_mask << (3 * j)) | (r2_mask << (3 * j + 1)) | (r3_mask << (3 * j + 2)) | + (r4_mask << (3 * j + 3)) + j = j + 1 + } + result((7 - i) * 4 + 3) = (z & 0xFF).toByte + result((7 - i) * 4 + 2) = ((z >> 8) & 0xFF).toByte + result((7 - i) * 4 + 1) = ((z >> 16) & 0xFF).toByte + result((7 - i) * 4) = ((z >> 24) & 0xFF).toByte + i = i + 1 + } + result + } + + private def interleave5Longs( + l1: Long, + l2: Long, + l3: Long, + l4: Long, + l5: Long): Array[Byte] = { + // output 8 * 40 bits + val result = new Array[Byte](40) + var i = 0 + while (i < 8) { + val tmp1 = ((l1 >> (i * 8)) & 0xFF).toLong + val tmp2 = ((l2 >> (i * 8)) & 0xFF).toLong + val tmp3 = ((l3 >> (i * 8)) & 0xFF).toLong + val tmp4 = ((l4 >> (i * 8)) & 0xFF).toLong + val tmp5 = ((l5 >> (i * 8)) & 0xFF).toLong + + var z = 0L + var j = 0 + while (j < 8) { + val r1_mask = tmp1 & (1 << j) + val r2_mask = tmp2 & (1 << j) + val r3_mask = tmp3 & (1 << j) + val r4_mask = tmp4 & (1 << j) + val r5_mask = tmp5 & (1 << j) + z |= (r1_mask << (4 * j)) | (r2_mask << (4 * j + 1)) | (r3_mask << (4 * j + 2)) | + (r4_mask << (4 * j + 3)) | (r5_mask << (4 * j + 4)) + j = j + 1 + } + result((7 - i) * 5 + 4) = (z & 0xFF).toByte + result((7 - i) * 5 + 3) = ((z >> 8) & 0xFF).toByte + result((7 - i) * 5 + 2) = ((z >> 16) & 0xFF).toByte + result((7 - i) * 5 + 1) = ((z >> 24) & 0xFF).toByte + result((7 - i) * 5) = ((z >> 32) & 0xFF).toByte + i = i + 1 + } + result + } + + private def interleave6Longs( + l1: Long, + l2: Long, + l3: Long, + l4: Long, + l5: Long, + l6: Long): Array[Byte] = { + // output 8 * 48 bits + val result = new Array[Byte](48) + var i = 0 + while (i < 8) { + val tmp1 = ((l1 >> (i * 8)) & 0xFF).toLong + val tmp2 = ((l2 >> (i * 8)) & 0xFF).toLong + val tmp3 = ((l3 >> (i * 8)) & 0xFF).toLong + val tmp4 = ((l4 >> (i * 8)) & 0xFF).toLong + val tmp5 = ((l5 >> (i * 8)) & 0xFF).toLong + val tmp6 = ((l6 >> (i * 8)) & 0xFF).toLong + + var z = 0L + var j = 0 + while (j < 8) { + val r1_mask = tmp1 & (1 << j) + val r2_mask = tmp2 & (1 << j) + val r3_mask = tmp3 & (1 << j) + val r4_mask = tmp4 & (1 << j) + val r5_mask = tmp5 & (1 << j) + val r6_mask = tmp6 & (1 << j) + z |= (r1_mask << (5 * j)) | (r2_mask << (5 * j + 1)) | (r3_mask << (5 * j + 2)) | + (r4_mask << (5 * j + 3)) | (r5_mask << (5 * j + 4)) | (r6_mask << (5 * j + 5)) + j = j + 1 + } + result((7 - i) * 6 + 5) = (z & 0xFF).toByte + result((7 - i) * 6 + 4) = ((z >> 8) & 0xFF).toByte + result((7 - i) * 6 + 3) = ((z >> 16) & 0xFF).toByte + result((7 - i) * 6 + 2) = ((z >> 24) & 0xFF).toByte + result((7 - i) * 6 + 1) = ((z >> 32) & 0xFF).toByte + result((7 - i) * 6) = ((z >> 40) & 0xFF).toByte + i = i + 1 + } + result + } + + private def interleave7Longs( + l1: Long, + l2: Long, + l3: Long, + l4: Long, + l5: Long, + l6: Long, + l7: Long): Array[Byte] = { + // output 8 * 56 bits + val result = new Array[Byte](56) + var i = 0 + while (i < 8) { + val tmp1 = ((l1 >> (i * 8)) & 0xFF).toLong + val tmp2 = ((l2 >> (i * 8)) & 0xFF).toLong + val tmp3 = ((l3 >> (i * 8)) & 0xFF).toLong + val tmp4 = ((l4 >> (i * 8)) & 0xFF).toLong + val tmp5 = ((l5 >> (i * 8)) & 0xFF).toLong + val tmp6 = ((l6 >> (i * 8)) & 0xFF).toLong + val tmp7 = ((l7 >> (i * 8)) & 0xFF).toLong + + var z = 0L + var j = 0 + while (j < 8) { + val r1_mask = tmp1 & (1 << j) + val r2_mask = tmp2 & (1 << j) + val r3_mask = tmp3 & (1 << j) + val r4_mask = tmp4 & (1 << j) + val r5_mask = tmp5 & (1 << j) + val r6_mask = tmp6 & (1 << j) + val r7_mask = tmp7 & (1 << j) + z |= (r1_mask << (6 * j)) | (r2_mask << (6 * j + 1)) | (r3_mask << (6 * j + 2)) | + (r4_mask << (6 * j + 3)) | (r5_mask << (6 * j + 4)) | (r6_mask << (6 * j + 5)) | + (r7_mask << (6 * j + 6)) + j = j + 1 + } + result((7 - i) * 7 + 6) = (z & 0xFF).toByte + result((7 - i) * 7 + 5) = ((z >> 8) & 0xFF).toByte + result((7 - i) * 7 + 4) = ((z >> 16) & 0xFF).toByte + result((7 - i) * 7 + 3) = ((z >> 24) & 0xFF).toByte + result((7 - i) * 7 + 2) = ((z >> 32) & 0xFF).toByte + result((7 - i) * 7 + 1) = ((z >> 40) & 0xFF).toByte + result((7 - i) * 7) = ((z >> 48) & 0xFF).toByte + i = i + 1 + } + result + } + + private def interleave8Longs( + l1: Long, + l2: Long, + l3: Long, + l4: Long, + l5: Long, + l6: Long, + l7: Long, + l8: Long): Array[Byte] = { + // output 8 * 64 bits + val result = new Array[Byte](64) + var i = 0 + while (i < 8) { + val tmp1 = ((l1 >> (i * 8)) & 0xFF).toLong + val tmp2 = ((l2 >> (i * 8)) & 0xFF).toLong + val tmp3 = ((l3 >> (i * 8)) & 0xFF).toLong + val tmp4 = ((l4 >> (i * 8)) & 0xFF).toLong + val tmp5 = ((l5 >> (i * 8)) & 0xFF).toLong + val tmp6 = ((l6 >> (i * 8)) & 0xFF).toLong + val tmp7 = ((l7 >> (i * 8)) & 0xFF).toLong + val tmp8 = ((l8 >> (i * 8)) & 0xFF).toLong + + var z = 0L + var j = 0 + while (j < 8) { + val r1_mask = tmp1 & (1 << j) + val r2_mask = tmp2 & (1 << j) + val r3_mask = tmp3 & (1 << j) + val r4_mask = tmp4 & (1 << j) + val r5_mask = tmp5 & (1 << j) + val r6_mask = tmp6 & (1 << j) + val r7_mask = tmp7 & (1 << j) + val r8_mask = tmp8 & (1 << j) + z |= (r1_mask << (7 * j)) | (r2_mask << (7 * j + 1)) | (r3_mask << (7 * j + 2)) | + (r4_mask << (7 * j + 3)) | (r5_mask << (7 * j + 4)) | (r6_mask << (7 * j + 5)) | + (r7_mask << (7 * j + 6)) | (r8_mask << (7 * j + 7)) + j = j + 1 + } + result((7 - i) * 8 + 7) = (z & 0xFF).toByte + result((7 - i) * 8 + 6) = ((z >> 8) & 0xFF).toByte + result((7 - i) * 8 + 5) = ((z >> 16) & 0xFF).toByte + result((7 - i) * 8 + 4) = ((z >> 24) & 0xFF).toByte + result((7 - i) * 8 + 3) = ((z >> 32) & 0xFF).toByte + result((7 - i) * 8 + 2) = ((z >> 40) & 0xFF).toByte + result((7 - i) * 8 + 1) = ((z >> 48) & 0xFF).toByte + result((7 - i) * 8) = ((z >> 56) & 0xFF).toByte + i = i + 1 + } + result + } + + def interleaveBitsDefault(arrays: Array[Array[Byte]]): Array[Byte] = { + var totalLength = 0 + var maxLength = 0 + arrays.foreach { array => + totalLength += array.length + maxLength = maxLength.max(array.length * 8) + } + val result = new Array[Byte](totalLength) + var resultBit = 0 + + var bit = 0 + while (bit < maxLength) { + val bytePos = bit / 8 + val bitPos = bit % 8 + + for (arr <- arrays) { + val len = arr.length + if (bytePos < len) { + val resultBytePos = totalLength - 1 - resultBit / 8 + val resultBitPos = resultBit % 8 + result(resultBytePos) = + updatePos(result(resultBytePos), resultBitPos, arr(len - 1 - bytePos), bitPos) + resultBit += 1 + } + } + bit += 1 + } + result + } + + def updatePos(a: Byte, apos: Int, b: Byte, bpos: Int): Byte = { + var temp = (b & (1 << bpos)).toByte + if (apos > bpos) { + temp = (temp << (apos - bpos)).toByte + } else if (apos < bpos) { + temp = (temp >> (bpos - apos)).toByte + } + val atemp = (a & (1 << apos)).toByte + if (atemp == temp) { + return a + } + (a ^ (1 << apos)).toByte + } + + def toLong(a: Any): Long = { + a match { + case b: Boolean => (if (b) 1 else 0).toLong ^ BIT_64_MASK + case b: Byte => b.toLong ^ BIT_64_MASK + case s: Short => s.toLong ^ BIT_64_MASK + case i: Int => i.toLong ^ BIT_64_MASK + case l: Long => l ^ BIT_64_MASK + case f: Float => java.lang.Float.floatToRawIntBits(f).toLong ^ BIT_64_MASK + case d: Double => java.lang.Double.doubleToRawLongBits(d) ^ BIT_64_MASK + case str: UTF8String => str.getPrefix + case dec: Decimal => dec.toLong ^ BIT_64_MASK + case other: Any => + throw new KyuubiSQLExtensionException("Unsupported z-order type: " + other.getClass) + } + } + + def toByteArray(a: Any): Array[Byte] = { + a match { + case bo: Boolean => + booleanToByte(bo) + case b: Byte => + byteToByte(b) + case s: Short => + shortToByte(s) + case i: Int => + intToByte(i) + case l: Long => + longToByte(l) + case f: Float => + floatToByte(f) + case d: Double => + doubleToByte(d) + case str: UTF8String => + // truncate or padding str to 8 byte + paddingTo8Byte(str.getBytes) + case dec: Decimal => + longToByte(dec.toLong) + case other: Any => + throw new KyuubiSQLExtensionException("Unsupported z-order type: " + other.getClass) + } + } + + def booleanToByte(a: Boolean): Array[Byte] = { + if (a) { + byteToByte(1.toByte) + } else { + byteToByte(0.toByte) + } + } + + def byteToByte(a: Byte): Array[Byte] = { + val tmp = (a ^ BIT_8_MASK).toByte + Array(tmp) + } + + def shortToByte(a: Short): Array[Byte] = { + val tmp = a ^ BIT_16_MASK + Array(((tmp >> 8) & 0xFF).toByte, (tmp & 0xFF).toByte) + } + + def intToByte(a: Int): Array[Byte] = { + val result = new Array[Byte](4) + var i = 0 + val tmp = a ^ BIT_32_MASK + while (i <= 3) { + val offset = i * 8 + result(3 - i) = ((tmp >> offset) & 0xFF).toByte + i += 1 + } + result + } + + def longToByte(a: Long): Array[Byte] = { + val result = new Array[Byte](8) + var i = 0 + val tmp = a ^ BIT_64_MASK + while (i <= 7) { + val offset = i * 8 + result(7 - i) = ((tmp >> offset) & 0xFF).toByte + i += 1 + } + result + } + + def floatToByte(a: Float): Array[Byte] = { + val fi = jFloat.floatToRawIntBits(a) + intToByte(fi) + } + + def doubleToByte(a: Double): Array[Byte] = { + val dl = jDouble.doubleToRawLongBits(a) + longToByte(dl) + } + + def paddingTo8Byte(a: Array[Byte]): Array[Byte] = { + val len = a.length + if (len == 8) { + a + } else if (len > 8) { + val result = new Array[Byte](8) + System.arraycopy(a, 0, result, 0, 8) + result + } else { + val result = new Array[Byte](8) + System.arraycopy(a, 0, result, 8 - len, len) + result + } + } + + def defaultByteArrayValue(dataType: DataType): Array[Byte] = toByteArray { + defaultValue(dataType) + } + + def defaultValue(dataType: DataType): Any = { + dataType match { + case BooleanType => + true + case ByteType => + Byte.MaxValue + case ShortType => + Short.MaxValue + case IntegerType | DateType => + Int.MaxValue + case LongType | TimestampType | _: DecimalType => + Long.MaxValue + case FloatType => + Float.MaxValue + case DoubleType => + Double.MaxValue + case StringType => + // we pad string to 8 bytes so it's equal to long + UTF8String.fromBytes(longToByte(Long.MaxValue)) + case other: Any => + throw new KyuubiSQLExtensionException(s"Unsupported z-order type: ${other.catalogString}") + } + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/spark/sql/FinalStageResourceManager.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/spark/sql/FinalStageResourceManager.scala new file mode 100644 index 000000000..81873476c --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/spark/sql/FinalStageResourceManager.scala @@ -0,0 +1,289 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +import scala.annotation.tailrec +import scala.collection.mutable +import scala.collection.mutable.ArrayBuffer + +import org.apache.spark.{ExecutorAllocationClient, MapOutputTrackerMaster, SparkContext, SparkEnv} +import org.apache.spark.internal.Logging +import org.apache.spark.resource.ResourceProfile +import org.apache.spark.scheduler.cluster.CoarseGrainedSchedulerBackend +import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.execution.{FilterExec, ProjectExec, SortExec, SparkPlan} +import org.apache.spark.sql.execution.adaptive._ +import org.apache.spark.sql.execution.columnar.InMemoryTableScanExec +import org.apache.spark.sql.execution.command.DataWritingCommandExec +import org.apache.spark.sql.execution.datasources.WriteFilesExec +import org.apache.spark.sql.execution.datasources.v2.V2TableWriteExec +import org.apache.spark.sql.execution.exchange.{ENSURE_REQUIREMENTS, ShuffleExchangeExec} + +import org.apache.kyuubi.sql.{KyuubiSQLConf, WriteUtils} + +/** + * This rule assumes the final write stage has less cores requirement than previous, otherwise + * this rule would take no effect. + * + * It provide a feature: + * 1. Kill redundant executors before running final write stage + */ +case class FinalStageResourceManager(session: SparkSession) + extends Rule[SparkPlan] with FinalRebalanceStageHelper { + override def apply(plan: SparkPlan): SparkPlan = { + if (!conf.getConf(KyuubiSQLConf.FINAL_WRITE_STAGE_EAGERLY_KILL_EXECUTORS_ENABLED)) { + return plan + } + + if (!WriteUtils.isWrite(session, plan)) { + return plan + } + + val sc = session.sparkContext + val dra = sc.getConf.getBoolean("spark.dynamicAllocation.enabled", false) + val coresPerExecutor = sc.getConf.getInt("spark.executor.cores", 1) + val minExecutors = sc.getConf.getInt("spark.dynamicAllocation.minExecutors", 0) + val maxExecutors = sc.getConf.getInt("spark.dynamicAllocation.maxExecutors", Int.MaxValue) + val factor = conf.getConf(KyuubiSQLConf.FINAL_WRITE_STAGE_PARTITION_FACTOR) + val hasImprovementRoom = maxExecutors - 1 > minExecutors * factor + // Fast fail if: + // 1. DRA off + // 2. only work with yarn and k8s + // 3. maxExecutors is not bigger than minExecutors * factor + if (!dra || !sc.schedulerBackend.isInstanceOf[CoarseGrainedSchedulerBackend] || + !hasImprovementRoom) { + return plan + } + + val stageOpt = findFinalRebalanceStage(plan) + if (stageOpt.isEmpty) { + return plan + } + + // It's not safe to kill executors if this plan contains table cache. + // If the executor loses then the rdd would re-compute those partition. + if (hasTableCache(plan) && + conf.getConf(KyuubiSQLConf.FINAL_WRITE_STAGE_SKIP_KILLING_EXECUTORS_FOR_TABLE_CACHE)) { + return plan + } + + // TODO: move this to query stage optimizer when updating Spark to 3.5.x + // Since we are in `prepareQueryStage`, the AQE shuffle read has not been applied. + // So we need to apply it by self. + val shuffleRead = queryStageOptimizerRules.foldLeft(stageOpt.get.asInstanceOf[SparkPlan]) { + case (latest, rule) => rule.apply(latest) + } + val (targetCores, stage) = shuffleRead match { + case AQEShuffleReadExec(stage: ShuffleQueryStageExec, partitionSpecs) => + (partitionSpecs.length, stage) + case stage: ShuffleQueryStageExec => + // we can still kill executors if no AQE shuffle read, e.g., `.repartition(2)` + (stage.shuffle.numPartitions, stage) + case _ => + // it should never happen in current Spark, but to be safe do nothing if happens + logWarning("BUG, Please report to Apache Kyuubi community") + return plan + } + // The condition whether inject custom resource profile: + // - target executors < active executors + // - active executors - target executors > min executors + val numActiveExecutors = sc.getExecutorIds().length + val targetExecutors = (math.ceil(targetCores.toFloat / coresPerExecutor) * factor).toInt + .max(1) + val hasBenefits = targetExecutors < numActiveExecutors && + (numActiveExecutors - targetExecutors) > minExecutors + logInfo(s"The snapshot of current executors view, " + + s"active executors: $numActiveExecutors, min executor: $minExecutors, " + + s"target executors: $targetExecutors, has benefits: $hasBenefits") + if (hasBenefits) { + val shuffleId = stage.plan.asInstanceOf[ShuffleExchangeExec].shuffleDependency.shuffleId + val numReduce = stage.plan.asInstanceOf[ShuffleExchangeExec].numPartitions + // Now, there is only a final rebalance stage waiting to execute and all tasks of previous + // stage are finished. Kill redundant existed executors eagerly so the tasks of final + // stage can be centralized scheduled. + killExecutors(sc, targetExecutors, shuffleId, numReduce) + } + + plan + } + + /** + * The priority of kill executors follow: + * 1. kill executor who is younger than other (The older the JIT works better) + * 2. kill executor who produces less shuffle data first + */ + private def findExecutorToKill( + sc: SparkContext, + targetExecutors: Int, + shuffleId: Int, + numReduce: Int): Seq[String] = { + val tracker = SparkEnv.get.mapOutputTracker.asInstanceOf[MapOutputTrackerMaster] + val shuffleStatusOpt = tracker.shuffleStatuses.get(shuffleId) + if (shuffleStatusOpt.isEmpty) { + return Seq.empty + } + val shuffleStatus = shuffleStatusOpt.get + val executorToBlockSize = new mutable.HashMap[String, Long] + shuffleStatus.withMapStatuses { mapStatus => + mapStatus.foreach { status => + var i = 0 + var sum = 0L + while (i < numReduce) { + sum += status.getSizeForBlock(i) + i += 1 + } + executorToBlockSize.getOrElseUpdate(status.location.executorId, sum) + } + } + + val backend = sc.schedulerBackend.asInstanceOf[CoarseGrainedSchedulerBackend] + val executorsWithRegistrationTs = backend.getExecutorsWithRegistrationTs() + val existedExecutors = executorsWithRegistrationTs.keys.toSet + val expectedNumExecutorToKill = existedExecutors.size - targetExecutors + if (expectedNumExecutorToKill < 1) { + return Seq.empty + } + + val executorIdsToKill = new ArrayBuffer[String]() + // We first kill executor who does not hold shuffle block. It would happen because + // the last stage is running fast and finished in a short time. The existed executors are + // from previous stages that have not been killed by DRA, so we can not find it by tracking + // shuffle status. + // We should evict executors by their alive time first and retain all of executors which + // have better locality for shuffle block. + executorsWithRegistrationTs.toSeq.sortBy(_._2).foreach { case (id, _) => + if (executorIdsToKill.length < expectedNumExecutorToKill && + !executorToBlockSize.contains(id)) { + executorIdsToKill.append(id) + } + } + + // Evict the rest executors according to the shuffle block size + executorToBlockSize.toSeq.sortBy(_._2).foreach { case (id, _) => + if (executorIdsToKill.length < expectedNumExecutorToKill && existedExecutors.contains(id)) { + executorIdsToKill.append(id) + } + } + + executorIdsToKill.toSeq + } + + private def killExecutors( + sc: SparkContext, + targetExecutors: Int, + shuffleId: Int, + numReduce: Int): Unit = { + val executorAllocationClient = sc.schedulerBackend.asInstanceOf[ExecutorAllocationClient] + + val executorsToKill = + if (conf.getConf(KyuubiSQLConf.FINAL_WRITE_STAGE_EAGERLY_KILL_EXECUTORS_KILL_ALL)) { + executorAllocationClient.getExecutorIds() + } else { + findExecutorToKill(sc, targetExecutors, shuffleId, numReduce) + } + logInfo(s"Request to kill executors, total count ${executorsToKill.size}, " + + s"[${executorsToKill.mkString(", ")}].") + if (executorsToKill.isEmpty) { + return + } + + // Note, `SparkContext#killExecutors` does not allow with DRA enabled, + // see `https://github.com/apache/spark/pull/20604`. + // It may cause the status in `ExecutorAllocationManager` inconsistent with + // `CoarseGrainedSchedulerBackend` for a while. But it should be synchronous finally. + // + // We should adjust target num executors, otherwise `YarnAllocator` might re-request original + // target executors if DRA has not updated target executors yet. + // Note, DRA would re-adjust executors if there are more tasks to be executed, so we are safe. + // + // * We kill executor + // * YarnAllocator re-request target executors + // * DRA can not release executors since they are new added + // ----------------------------------------------------------------> timeline + executorAllocationClient.killExecutors( + executorIds = executorsToKill, + adjustTargetNumExecutors = true, + countFailures = false, + force = false) + + FinalStageResourceManager.getAdjustedTargetExecutors(sc) + .filter(_ < targetExecutors).foreach { adjustedExecutors => + val delta = targetExecutors - adjustedExecutors + logInfo(s"Target executors after kill ($adjustedExecutors) is lower than required " + + s"($targetExecutors). Requesting $delta additional executor(s).") + executorAllocationClient.requestExecutors(delta) + } + } + + @transient private val queryStageOptimizerRules: Seq[Rule[SparkPlan]] = Seq( + OptimizeSkewInRebalancePartitions, + CoalesceShufflePartitions(session), + OptimizeShuffleWithLocalRead) +} + +object FinalStageResourceManager extends Logging { + + private[sql] def getAdjustedTargetExecutors(sc: SparkContext): Option[Int] = { + sc.schedulerBackend match { + case schedulerBackend: CoarseGrainedSchedulerBackend => + try { + val field = classOf[CoarseGrainedSchedulerBackend] + .getDeclaredField("requestedTotalExecutorsPerResourceProfile") + field.setAccessible(true) + schedulerBackend.synchronized { + val requestedTotalExecutorsPerResourceProfile = + field.get(schedulerBackend).asInstanceOf[mutable.HashMap[ResourceProfile, Int]] + val defaultRp = sc.resourceProfileManager.defaultResourceProfile + requestedTotalExecutorsPerResourceProfile.get(defaultRp) + } + } catch { + case e: Exception => + logWarning("Failed to get requestedTotalExecutors of Default ResourceProfile", e) + None + } + case _ => None + } + } +} + +trait FinalRebalanceStageHelper extends AdaptiveSparkPlanHelper { + @tailrec + final protected def findFinalRebalanceStage(plan: SparkPlan): Option[ShuffleQueryStageExec] = { + plan match { + case write: DataWritingCommandExec => findFinalRebalanceStage(write.child) + case write: V2TableWriteExec => findFinalRebalanceStage(write.child) + case write: WriteFilesExec => findFinalRebalanceStage(write.child) + case p: ProjectExec => findFinalRebalanceStage(p.child) + case f: FilterExec => findFinalRebalanceStage(f.child) + case s: SortExec if !s.global => findFinalRebalanceStage(s.child) + case stage: ShuffleQueryStageExec + if stage.isMaterialized && stage.mapStats.isDefined && + stage.plan.isInstanceOf[ShuffleExchangeExec] && + stage.plan.asInstanceOf[ShuffleExchangeExec].shuffleOrigin != ENSURE_REQUIREMENTS => + Some(stage) + case _ => None + } + } + + final protected def hasTableCache(plan: SparkPlan): Boolean = { + find(plan) { + case _: InMemoryTableScanExec => true + case _ => false + }.isDefined + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/spark/sql/InjectCustomResourceProfile.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/spark/sql/InjectCustomResourceProfile.scala new file mode 100644 index 000000000..64421d6bf --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/spark/sql/InjectCustomResourceProfile.scala @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.execution.{CustomResourceProfileExec, SparkPlan} +import org.apache.spark.sql.execution.adaptive._ + +import org.apache.kyuubi.sql.{KyuubiSQLConf, WriteUtils} + +/** + * Inject custom resource profile for final write stage, so we can specify custom + * executor resource configs. + */ +case class InjectCustomResourceProfile(session: SparkSession) + extends Rule[SparkPlan] with FinalRebalanceStageHelper { + override def apply(plan: SparkPlan): SparkPlan = { + if (!conf.getConf(KyuubiSQLConf.FINAL_WRITE_STAGE_RESOURCE_ISOLATION_ENABLED)) { + return plan + } + + if (!WriteUtils.isWrite(session, plan)) { + return plan + } + + val stage = findFinalRebalanceStage(plan) + if (stage.isEmpty) { + return plan + } + + // TODO: Ideally, We can call `CoarseGrainedSchedulerBackend.requestTotalExecutors` eagerly + // to reduce the task submit pending time, but it may lose task locality. + // + // By default, it would request executors when catch stage submit event. + injectCustomResourceProfile(plan, stage.get.id) + } + + private def injectCustomResourceProfile(plan: SparkPlan, id: Int): SparkPlan = { + plan match { + case stage: ShuffleQueryStageExec if stage.id == id => + CustomResourceProfileExec(stage) + case _ => plan.mapChildren(child => injectCustomResourceProfile(child, id)) + } + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/spark/sql/PruneFileSourcePartitionHelper.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/spark/sql/PruneFileSourcePartitionHelper.scala new file mode 100644 index 000000000..ce496eb47 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/spark/sql/PruneFileSourcePartitionHelper.scala @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +import org.apache.spark.sql.catalyst.expressions.{AttributeReference, AttributeSet, Expression, ExpressionSet, PredicateHelper, SubqueryExpression} +import org.apache.spark.sql.catalyst.plans.logical.LeafNode +import org.apache.spark.sql.execution.datasources.DataSourceStrategy +import org.apache.spark.sql.types.StructType + +trait PruneFileSourcePartitionHelper extends PredicateHelper { + + def getPartitionKeyFiltersAndDataFilters( + sparkSession: SparkSession, + relation: LeafNode, + partitionSchema: StructType, + filters: Seq[Expression], + output: Seq[AttributeReference]): (ExpressionSet, Seq[Expression]) = { + val normalizedFilters = DataSourceStrategy.normalizeExprs( + filters.filter(f => f.deterministic && !SubqueryExpression.hasSubquery(f)), + output) + val partitionColumns = + relation.resolve(partitionSchema, sparkSession.sessionState.analyzer.resolver) + val partitionSet = AttributeSet(partitionColumns) + val (partitionFilters, dataFilters) = normalizedFilters.partition(f => + f.references.subsetOf(partitionSet)) + val extraPartitionFilter = + dataFilters.flatMap(extractPredicatesWithinOutputSet(_, partitionSet)) + + (ExpressionSet(partitionFilters ++ extraPartitionFilter), dataFilters) + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/spark/sql/execution/CustomResourceProfileExec.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/spark/sql/execution/CustomResourceProfileExec.scala new file mode 100644 index 000000000..3698140fb --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/spark/sql/execution/CustomResourceProfileExec.scala @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution + +import org.apache.spark.network.util.{ByteUnit, JavaUtils} +import org.apache.spark.rdd.RDD +import org.apache.spark.resource.{ExecutorResourceRequests, ResourceProfileBuilder} +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions.{Attribute, SortOrder} +import org.apache.spark.sql.catalyst.plans.physical.Partitioning +import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics} +import org.apache.spark.sql.vectorized.ColumnarBatch +import org.apache.spark.util.Utils + +import org.apache.kyuubi.sql.KyuubiSQLConf._ + +/** + * This node wraps the final executed plan and inject custom resource profile to the RDD. + * It assumes that, the produced RDD would create the `ResultStage` in `DAGScheduler`, + * so it makes resource isolation between previous and final stage. + * + * Note that, Spark does not support config `minExecutors` for each resource profile. + * Which means, it would retain `minExecutors` for each resource profile. + * So, suggest set `spark.dynamicAllocation.minExecutors` to 0 if enable this feature. + */ +case class CustomResourceProfileExec(child: SparkPlan) extends UnaryExecNode { + override def output: Seq[Attribute] = child.output + override def outputPartitioning: Partitioning = child.outputPartitioning + override def outputOrdering: Seq[SortOrder] = child.outputOrdering + override def supportsColumnar: Boolean = child.supportsColumnar + override def supportsRowBased: Boolean = child.supportsRowBased + override protected def doCanonicalize(): SparkPlan = child.canonicalized + + private val executorCores = conf.getConf(FINAL_WRITE_STAGE_EXECUTOR_CORES).getOrElse( + sparkContext.getConf.getInt("spark.executor.cores", 1)) + private val executorMemory = conf.getConf(FINAL_WRITE_STAGE_EXECUTOR_MEMORY).getOrElse( + sparkContext.getConf.get("spark.executor.memory", "2G")) + private val executorMemoryOverhead = + conf.getConf(FINAL_WRITE_STAGE_EXECUTOR_MEMORY_OVERHEAD) + .getOrElse(sparkContext.getConf.get("spark.executor.memoryOverhead", "1G")) + private val executorOffHeapMemory = conf.getConf(FINAL_WRITE_STAGE_EXECUTOR_OFF_HEAP_MEMORY) + + override lazy val metrics: Map[String, SQLMetric] = { + val base = Map( + "executorCores" -> SQLMetrics.createMetric(sparkContext, "executor cores"), + "executorMemory" -> SQLMetrics.createMetric(sparkContext, "executor memory (MiB)"), + "executorMemoryOverhead" -> SQLMetrics.createMetric( + sparkContext, + "executor memory overhead (MiB)")) + val addition = executorOffHeapMemory.map(_ => + "executorOffHeapMemory" -> + SQLMetrics.createMetric(sparkContext, "executor off heap memory (MiB)")).toMap + base ++ addition + } + + private def wrapResourceProfile[T](rdd: RDD[T]): RDD[T] = { + if (Utils.isTesting) { + // do nothing for local testing + return rdd + } + + metrics("executorCores") += executorCores + metrics("executorMemory") += JavaUtils.byteStringAs(executorMemory, ByteUnit.MiB) + metrics("executorMemoryOverhead") += JavaUtils.byteStringAs( + executorMemoryOverhead, + ByteUnit.MiB) + executorOffHeapMemory.foreach(m => + metrics("executorOffHeapMemory") += JavaUtils.byteStringAs(m, ByteUnit.MiB)) + + val executionId = sparkContext.getLocalProperty(SQLExecution.EXECUTION_ID_KEY) + SQLMetrics.postDriverMetricUpdates(sparkContext, executionId, metrics.values.toSeq) + + val resourceProfileBuilder = new ResourceProfileBuilder() + val executorResourceRequests = new ExecutorResourceRequests() + executorResourceRequests.cores(executorCores) + executorResourceRequests.memory(executorMemory) + executorResourceRequests.memoryOverhead(executorMemoryOverhead) + executorOffHeapMemory.foreach(executorResourceRequests.offHeapMemory) + resourceProfileBuilder.require(executorResourceRequests) + rdd.withResources(resourceProfileBuilder.build()) + rdd + } + + override protected def doExecute(): RDD[InternalRow] = { + val rdd = child.execute() + wrapResourceProfile(rdd) + } + + override protected def doExecuteColumnar(): RDD[ColumnarBatch] = { + val rdd = child.executeColumnar() + wrapResourceProfile(rdd) + } + + override protected def withNewChildInternal(newChild: SparkPlan): SparkPlan = { + this.copy(child = newChild) + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/spark/sql/hive/HiveSparkPlanHelper.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/spark/sql/hive/HiveSparkPlanHelper.scala new file mode 100644 index 000000000..aa9a04596 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/main/scala/org/apache/spark/sql/hive/HiveSparkPlanHelper.scala @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.hive + +object HiveSparkPlanHelper { + type HiveTableScanExec = org.apache.spark.sql.hive.execution.HiveTableScanExec +} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/test/resources/log4j2-test.xml b/extensions/spark/kyuubi-extension-spark-3-5/src/test/resources/log4j2-test.xml new file mode 100644 index 000000000..3110216c1 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/test/resources/log4j2-test.xml @@ -0,0 +1,43 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/DropIgnoreNonexistentSuite.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/DropIgnoreNonexistentSuite.scala new file mode 100644 index 000000000..bbc61fb44 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/DropIgnoreNonexistentSuite.scala @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +import org.apache.spark.sql.catalyst.plans.logical.{DropNamespace, NoopCommand} +import org.apache.spark.sql.execution.command._ + +import org.apache.kyuubi.sql.KyuubiSQLConf + +class DropIgnoreNonexistentSuite extends KyuubiSparkSQLExtensionTest { + + test("drop ignore nonexistent") { + withSQLConf(KyuubiSQLConf.DROP_IGNORE_NONEXISTENT.key -> "true") { + // drop nonexistent database + val df1 = sql("DROP DATABASE nonexistent_database") + assert(df1.queryExecution.analyzed.asInstanceOf[DropNamespace].ifExists == true) + + // drop nonexistent function + val df4 = sql("DROP FUNCTION nonexistent_function") + assert(df4.queryExecution.analyzed.isInstanceOf[NoopCommand]) + + // drop nonexistent PARTITION + withTable("test") { + sql("CREATE TABLE IF NOT EXISTS test(i int) PARTITIONED BY (p int)") + val df5 = sql("ALTER TABLE test DROP PARTITION (p = 1)") + assert(df5.queryExecution.analyzed + .asInstanceOf[AlterTableDropPartitionCommand].ifExists == true) + } + } + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/DynamicShufflePartitionsSuite.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/DynamicShufflePartitionsSuite.scala new file mode 100644 index 000000000..6668675a5 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/DynamicShufflePartitionsSuite.scala @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +import org.apache.spark.sql.execution.{CommandResultExec, SparkPlan} +import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanExec, ShuffleQueryStageExec} +import org.apache.spark.sql.execution.exchange.{ENSURE_REQUIREMENTS, ShuffleExchangeExec} +import org.apache.spark.sql.hive.HiveUtils.CONVERT_METASTORE_PARQUET +import org.apache.spark.sql.internal.SQLConf._ + +import org.apache.kyuubi.sql.KyuubiSQLConf.{DYNAMIC_SHUFFLE_PARTITIONS, DYNAMIC_SHUFFLE_PARTITIONS_MAX_NUM} + +class DynamicShufflePartitionsSuite extends KyuubiSparkSQLExtensionTest { + + override protected def beforeAll(): Unit = { + super.beforeAll() + setupData() + } + + test("test dynamic shuffle partitions") { + def collectExchanges(plan: SparkPlan): Seq[ShuffleExchangeExec] = { + plan match { + case p: CommandResultExec => collectExchanges(p.commandPhysicalPlan) + case p: AdaptiveSparkPlanExec => collectExchanges(p.finalPhysicalPlan) + case p: ShuffleQueryStageExec => collectExchanges(p.plan) + case p: ShuffleExchangeExec => p +: collectExchanges(p.child) + case p => p.children.flatMap(collectExchanges) + } + } + + // datasource scan + withTable("table1", "table2", "table3") { + sql("create table table1 stored as parquet as select c1, c2 from t1") + sql("create table table2 stored as parquet as select c1, c2 from t2") + sql("create table table3 (c1 int, c2 string) stored as parquet") + sql("ANALYZE TABLE table1 COMPUTE STATISTICS") + sql("ANALYZE TABLE table2 COMPUTE STATISTICS") + + val initialPartitionNum: Int = 2 + Seq(false, true).foreach { dynamicShufflePartitions => + val maxDynamicShufflePartitions = if (dynamicShufflePartitions) { + Seq(8, 2000) + } else { + Seq(2000) + } + maxDynamicShufflePartitions.foreach { maxDynamicShufflePartitionNum => + withSQLConf( + DYNAMIC_SHUFFLE_PARTITIONS.key -> dynamicShufflePartitions.toString, + DYNAMIC_SHUFFLE_PARTITIONS_MAX_NUM.key -> maxDynamicShufflePartitionNum.toString, + AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", + COALESCE_PARTITIONS_INITIAL_PARTITION_NUM.key -> initialPartitionNum.toString, + ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "500") { + val df = sql("insert overwrite table3 " + + " select a.c1 as c1, b.c2 as c2 from table1 a join table2 b on a.c1 = b.c1") + + val exchanges = collectExchanges(df.queryExecution.executedPlan) + val (joinExchanges, rebalanceExchanges) = exchanges + .partition(_.shuffleOrigin == ENSURE_REQUIREMENTS) + // table scan size: 7369 3287 + assert(joinExchanges.size == 2) + if (dynamicShufflePartitions) { + joinExchanges.foreach(e => + assert(e.outputPartitioning.numPartitions + == Math.min(22, maxDynamicShufflePartitionNum))) + } else { + joinExchanges.foreach(e => + assert(e.outputPartitioning.numPartitions == initialPartitionNum)) + } + + assert(rebalanceExchanges.size == 1) + if (dynamicShufflePartitions) { + if (maxDynamicShufflePartitionNum == 8) { + // shuffle query size: 1424 451 + assert(rebalanceExchanges.head.outputPartitioning.numPartitions == + Math.min(4, maxDynamicShufflePartitionNum)) + } else { + // shuffle query size: 2057 664 + assert(rebalanceExchanges.head.outputPartitioning.numPartitions == + Math.min(6, maxDynamicShufflePartitionNum)) + } + } else { + assert( + rebalanceExchanges.head.outputPartitioning.numPartitions == initialPartitionNum) + } + } + + // hive table scan + withSQLConf( + DYNAMIC_SHUFFLE_PARTITIONS.key -> dynamicShufflePartitions.toString, + DYNAMIC_SHUFFLE_PARTITIONS_MAX_NUM.key -> maxDynamicShufflePartitionNum.toString, + AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", + COALESCE_PARTITIONS_INITIAL_PARTITION_NUM.key -> initialPartitionNum.toString, + ADVISORY_PARTITION_SIZE_IN_BYTES.key -> "500", + CONVERT_METASTORE_PARQUET.key -> "false") { + val df = sql("insert overwrite table3 " + + " select a.c1 as c1, b.c2 as c2 from table1 a join table2 b on a.c1 = b.c1") + + val exchanges = collectExchanges(df.queryExecution.executedPlan) + val (joinExchanges, rebalanceExchanges) = exchanges + .partition(_.shuffleOrigin == ENSURE_REQUIREMENTS) + // table scan size: 7369 3287 + assert(joinExchanges.size == 2) + if (dynamicShufflePartitions) { + joinExchanges.foreach(e => + assert(e.outputPartitioning.numPartitions == + Math.min(22, maxDynamicShufflePartitionNum))) + } else { + joinExchanges.foreach(e => + assert(e.outputPartitioning.numPartitions == initialPartitionNum)) + } + // shuffle query size: 5154 720 + assert(rebalanceExchanges.size == 1) + if (dynamicShufflePartitions) { + assert(rebalanceExchanges.head.outputPartitioning.numPartitions + == Math.min(12, maxDynamicShufflePartitionNum)) + } else { + assert(rebalanceExchanges.head.outputPartitioning.numPartitions == + initialPartitionNum) + } + } + } + } + } + } + +} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/FinalStageConfigIsolationSuite.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/FinalStageConfigIsolationSuite.scala new file mode 100644 index 000000000..96c8ae6e8 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/FinalStageConfigIsolationSuite.scala @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +import org.apache.spark.sql.execution.adaptive.{AQEShuffleReadExec, QueryStageExec} +import org.apache.spark.sql.internal.SQLConf + +import org.apache.kyuubi.sql.{FinalStageConfigIsolation, KyuubiSQLConf} + +class FinalStageConfigIsolationSuite extends KyuubiSparkSQLExtensionTest { + override protected def beforeAll(): Unit = { + super.beforeAll() + setupData() + } + + test("final stage config set reset check") { + withSQLConf( + KyuubiSQLConf.FINAL_STAGE_CONFIG_ISOLATION.key -> "true", + KyuubiSQLConf.FINAL_STAGE_CONFIG_ISOLATION_WRITE_ONLY.key -> "false", + "spark.sql.finalStage.adaptive.coalescePartitions.minPartitionNum" -> "1", + "spark.sql.finalStage.adaptive.advisoryPartitionSizeInBytes" -> "100") { + // use loop to double check final stage config doesn't affect the sql query each other + (1 to 3).foreach { _ => + sql("SELECT COUNT(*) FROM VALUES(1) as t(c)").collect() + assert(spark.sessionState.conf.getConfString( + "spark.sql.previousStage.adaptive.coalescePartitions.minPartitionNum") === + FinalStageConfigIsolation.INTERNAL_UNSET_CONFIG_TAG) + assert(spark.sessionState.conf.getConfString( + "spark.sql.adaptive.coalescePartitions.minPartitionNum") === + "1") + assert(spark.sessionState.conf.getConfString( + "spark.sql.finalStage.adaptive.coalescePartitions.minPartitionNum") === + "1") + + // 64MB + assert(spark.sessionState.conf.getConfString( + "spark.sql.previousStage.adaptive.advisoryPartitionSizeInBytes") === + "67108864b") + assert(spark.sessionState.conf.getConfString( + "spark.sql.adaptive.advisoryPartitionSizeInBytes") === + "100") + assert(spark.sessionState.conf.getConfString( + "spark.sql.finalStage.adaptive.advisoryPartitionSizeInBytes") === + "100") + } + + sql("SET spark.sql.adaptive.advisoryPartitionSizeInBytes=1") + assert(spark.sessionState.conf.getConfString( + "spark.sql.adaptive.advisoryPartitionSizeInBytes") === + "1") + assert(!spark.sessionState.conf.contains( + "spark.sql.previousStage.adaptive.advisoryPartitionSizeInBytes")) + + sql("SET a=1") + assert(spark.sessionState.conf.getConfString("a") === "1") + + sql("RESET spark.sql.adaptive.coalescePartitions.minPartitionNum") + assert(!spark.sessionState.conf.contains( + "spark.sql.adaptive.coalescePartitions.minPartitionNum")) + assert(!spark.sessionState.conf.contains( + "spark.sql.previousStage.adaptive.coalescePartitions.minPartitionNum")) + + sql("RESET a") + assert(!spark.sessionState.conf.contains("a")) + } + } + + test("final stage config isolation") { + def checkPartitionNum( + sqlString: String, + previousPartitionNum: Int, + finalPartitionNum: Int): Unit = { + val df = sql(sqlString) + df.collect() + val shuffleReaders = collect(df.queryExecution.executedPlan) { + case customShuffleReader: AQEShuffleReadExec => customShuffleReader + } + assert(shuffleReaders.nonEmpty) + // reorder stage by stage id to ensure we get the right stage + val sortedShuffleReaders = shuffleReaders.sortWith { + case (s1, s2) => + s1.child.asInstanceOf[QueryStageExec].id < s2.child.asInstanceOf[QueryStageExec].id + } + if (sortedShuffleReaders.length > 1) { + assert(sortedShuffleReaders.head.partitionSpecs.length === previousPartitionNum) + } + assert(sortedShuffleReaders.last.partitionSpecs.length === finalPartitionNum) + assert(df.rdd.partitions.length === finalPartitionNum) + } + + withSQLConf( + SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", + SQLConf.COALESCE_PARTITIONS_MIN_PARTITION_NUM.key -> "1", + SQLConf.SHUFFLE_PARTITIONS.key -> "3", + KyuubiSQLConf.FINAL_STAGE_CONFIG_ISOLATION.key -> "true", + KyuubiSQLConf.FINAL_STAGE_CONFIG_ISOLATION_WRITE_ONLY.key -> "false", + "spark.sql.adaptive.advisoryPartitionSizeInBytes" -> "1", + "spark.sql.adaptive.coalescePartitions.minPartitionSize" -> "1", + "spark.sql.finalStage.adaptive.advisoryPartitionSizeInBytes" -> "10000000") { + + // use loop to double check final stage config doesn't affect the sql query each other + (1 to 3).foreach { _ => + checkPartitionNum( + "SELECT c1, count(*) FROM t1 GROUP BY c1", + 1, + 1) + + checkPartitionNum( + "SELECT c2, count(*) FROM (SELECT c1, count(*) as c2 FROM t1 GROUP BY c1) GROUP BY c2", + 3, + 1) + + checkPartitionNum( + "SELECT t1.c1, count(*) FROM t1 JOIN t2 ON t1.c2 = t2.c2 GROUP BY t1.c1", + 3, + 1) + + checkPartitionNum( + """ + | SELECT /*+ REPARTITION */ + | t1.c1, count(*) FROM t1 + | JOIN t2 ON t1.c2 = t2.c2 + | JOIN t3 ON t1.c1 = t3.c1 + | GROUP BY t1.c1 + |""".stripMargin, + 3, + 1) + + // one shuffle reader + checkPartitionNum( + """ + | SELECT /*+ BROADCAST(t1) */ + | t1.c1, t2.c2 FROM t1 + | JOIN t2 ON t1.c2 = t2.c2 + | DISTRIBUTE BY c1 + |""".stripMargin, + 1, + 1) + + // test ReusedExchange + checkPartitionNum( + """ + |SELECT /*+ REPARTITION */ t0.c2 FROM ( + |SELECT t1.c1, (count(*) + c1) as c2 FROM t1 GROUP BY t1.c1 + |) t0 JOIN ( + |SELECT t1.c1, (count(*) + c1) as c2 FROM t1 GROUP BY t1.c1 + |) t1 ON t0.c2 = t1.c2 + |""".stripMargin, + 3, + 1) + + // one shuffle reader + checkPartitionNum( + """ + |SELECT t0.c1 FROM ( + |SELECT t1.c1 FROM t1 GROUP BY t1.c1 + |) t0 JOIN ( + |SELECT t1.c1 FROM t1 GROUP BY t1.c1 + |) t1 ON t0.c1 = t1.c1 + |""".stripMargin, + 1, + 1) + } + } + } + + test("final stage config isolation write only") { + withSQLConf( + KyuubiSQLConf.FINAL_STAGE_CONFIG_ISOLATION.key -> "true", + KyuubiSQLConf.FINAL_STAGE_CONFIG_ISOLATION_WRITE_ONLY.key -> "true", + "spark.sql.finalStage.adaptive.advisoryPartitionSizeInBytes" -> "7") { + sql("set spark.sql.adaptive.advisoryPartitionSizeInBytes=5") + sql("SELECT * FROM t1").count() + assert(spark.conf.getOption("spark.sql.adaptive.advisoryPartitionSizeInBytes") + .contains("5")) + + withTable("tmp") { + sql("CREATE TABLE t1 USING PARQUET SELECT /*+ repartition */ 1 AS c1, 'a' AS c2") + assert(spark.conf.getOption("spark.sql.adaptive.advisoryPartitionSizeInBytes") + .contains("7")) + } + + sql("SELECT * FROM t1").count() + assert(spark.conf.getOption("spark.sql.adaptive.advisoryPartitionSizeInBytes") + .contains("5")) + } + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/FinalStageResourceManagerSuite.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/FinalStageResourceManagerSuite.scala new file mode 100644 index 000000000..4b9991ef6 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/FinalStageResourceManagerSuite.scala @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +import org.apache.spark.SparkConf +import org.scalatest.time.{Minutes, Span} + +import org.apache.kyuubi.sql.KyuubiSQLConf +import org.apache.kyuubi.tags.SparkLocalClusterTest + +@SparkLocalClusterTest +class FinalStageResourceManagerSuite extends KyuubiSparkSQLExtensionTest { + + override def sparkConf(): SparkConf = { + // It is difficult to run spark in local-cluster mode when spark.testing is set. + sys.props.remove("spark.testing") + + super.sparkConf().set("spark.master", "local-cluster[3, 1, 1024]") + .set("spark.dynamicAllocation.enabled", "true") + .set("spark.dynamicAllocation.initialExecutors", "3") + .set("spark.dynamicAllocation.minExecutors", "1") + .set("spark.dynamicAllocation.shuffleTracking.enabled", "true") + .set(KyuubiSQLConf.FINAL_STAGE_CONFIG_ISOLATION.key, "true") + .set(KyuubiSQLConf.FINAL_WRITE_STAGE_EAGERLY_KILL_EXECUTORS_ENABLED.key, "true") + } + + test("[KYUUBI #5136][Bug] Final Stage hangs forever") { + // Prerequisite to reproduce the bug: + // 1. Dynamic allocation is enabled. + // 2. Dynamic allocation min executors is 1. + // 3. target executors < active executors. + // 4. No active executor is left after FinalStageResourceManager killed executors. + // This is possible because FinalStageResourceManager retained executors may already be + // requested to be killed but not died yet. + // 5. Final Stage required executors is 1. + withSQLConf( + (KyuubiSQLConf.FINAL_WRITE_STAGE_EAGERLY_KILL_EXECUTORS_KILL_ALL.key, "true")) { + withTable("final_stage") { + eventually(timeout(Span(10, Minutes))) { + sql( + "CREATE TABLE final_stage AS SELECT id, count(*) as num FROM (SELECT 0 id) GROUP BY id") + } + assert(FinalStageResourceManager.getAdjustedTargetExecutors(spark.sparkContext).get == 1) + } + } + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/InjectResourceProfileSuite.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/InjectResourceProfileSuite.scala new file mode 100644 index 000000000..b0767b187 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/InjectResourceProfileSuite.scala @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +import org.apache.spark.scheduler.{SparkListener, SparkListenerEvent} +import org.apache.spark.sql.execution.ui.SparkListenerSQLAdaptiveExecutionUpdate + +import org.apache.kyuubi.sql.KyuubiSQLConf + +class InjectResourceProfileSuite extends KyuubiSparkSQLExtensionTest { + private def checkCustomResourceProfile(sqlString: String, exists: Boolean): Unit = { + @volatile var lastEvent: SparkListenerSQLAdaptiveExecutionUpdate = null + val listener = new SparkListener { + override def onOtherEvent(event: SparkListenerEvent): Unit = { + event match { + case e: SparkListenerSQLAdaptiveExecutionUpdate => lastEvent = e + case _ => + } + } + } + + spark.sparkContext.addSparkListener(listener) + try { + sql(sqlString).collect() + spark.sparkContext.listenerBus.waitUntilEmpty() + assert(lastEvent != null) + var current = lastEvent.sparkPlanInfo + var shouldStop = false + while (!shouldStop) { + if (current.nodeName != "CustomResourceProfile") { + if (current.children.isEmpty) { + assert(!exists) + shouldStop = true + } else { + current = current.children.head + } + } else { + assert(exists) + shouldStop = true + } + } + } finally { + spark.sparkContext.removeSparkListener(listener) + } + } + + test("Inject resource profile") { + withTable("t") { + withSQLConf( + "spark.sql.adaptive.forceApply" -> "true", + KyuubiSQLConf.FINAL_STAGE_CONFIG_ISOLATION.key -> "true", + KyuubiSQLConf.FINAL_WRITE_STAGE_RESOURCE_ISOLATION_ENABLED.key -> "true") { + + sql("CREATE TABLE t (c1 int, c2 string) USING PARQUET") + + checkCustomResourceProfile("INSERT INTO TABLE t VALUES(1, 'a')", false) + checkCustomResourceProfile("SELECT 1", false) + checkCustomResourceProfile( + "INSERT INTO TABLE t SELECT /*+ rebalance */ * FROM VALUES(1, 'a')", + true) + } + } + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/InsertShuffleNodeBeforeJoinSuite.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/InsertShuffleNodeBeforeJoinSuite.scala new file mode 100644 index 000000000..f0d384657 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/InsertShuffleNodeBeforeJoinSuite.scala @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +class InsertShuffleNodeBeforeJoinSuite extends InsertShuffleNodeBeforeJoinSuiteBase diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/InsertShuffleNodeBeforeJoinSuiteBase.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/InsertShuffleNodeBeforeJoinSuiteBase.scala new file mode 100644 index 000000000..c657dee49 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/InsertShuffleNodeBeforeJoinSuiteBase.scala @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +import org.apache.spark.SparkConf +import org.apache.spark.sql.execution.exchange.{ENSURE_REQUIREMENTS, ShuffleExchangeLike} +import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf} + +import org.apache.kyuubi.sql.KyuubiSQLConf + +trait InsertShuffleNodeBeforeJoinSuiteBase extends KyuubiSparkSQLExtensionTest { + override protected def beforeAll(): Unit = { + super.beforeAll() + setupData() + } + + override def sparkConf(): SparkConf = { + super.sparkConf() + .set( + StaticSQLConf.SPARK_SESSION_EXTENSIONS.key, + "org.apache.kyuubi.sql.KyuubiSparkSQLCommonExtension") + } + + test("force shuffle before join") { + def checkShuffleNodeNum(sqlString: String, num: Int): Unit = { + var expectedResult: Seq[Row] = Seq.empty + withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") { + expectedResult = sql(sqlString).collect() + } + val df = sql(sqlString) + checkAnswer(df, expectedResult) + assert( + collect(df.queryExecution.executedPlan) { + case shuffle: ShuffleExchangeLike if shuffle.shuffleOrigin == ENSURE_REQUIREMENTS => + shuffle + }.size == num) + } + + withSQLConf( + SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "-1", + KyuubiSQLConf.FORCE_SHUFFLE_BEFORE_JOIN.key -> "true") { + Seq("SHUFFLE_HASH", "MERGE").foreach { joinHint => + // positive case + checkShuffleNodeNum( + s""" + |SELECT /*+ $joinHint(t2, t3) */ t1.c1, t1.c2, t2.c1, t3.c1 from t1 + | JOIN t2 ON t1.c1 = t2.c1 + | JOIN t3 ON t1.c1 = t3.c1 + | """.stripMargin, + 4) + + // negative case + checkShuffleNodeNum( + s""" + |SELECT /*+ $joinHint(t2, t3) */ t1.c1, t1.c2, t2.c1, t3.c1 from t1 + | JOIN t2 ON t1.c1 = t2.c1 + | JOIN t3 ON t1.c2 = t3.c2 + | """.stripMargin, + 4) + } + + checkShuffleNodeNum( + """ + |SELECT t1.c1, t2.c1, t3.c2 from t1 + | JOIN t2 ON t1.c1 = t2.c1 + | JOIN ( + | SELECT c2, count(*) FROM t1 GROUP BY c2 + | ) t3 ON t1.c1 = t3.c2 + | """.stripMargin, + 5) + + checkShuffleNodeNum( + """ + |SELECT t1.c1, t2.c1, t3.c1 from t1 + | JOIN t2 ON t1.c1 = t2.c1 + | JOIN ( + | SELECT c1, count(*) FROM t1 GROUP BY c1 + | ) t3 ON t1.c1 = t3.c1 + | """.stripMargin, + 5) + } + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/KyuubiSparkSQLExtensionTest.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/KyuubiSparkSQLExtensionTest.scala new file mode 100644 index 000000000..dd9ffbf16 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/KyuubiSparkSQLExtensionTest.scala @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +import org.apache.hadoop.hive.conf.HiveConf.ConfVars +import org.apache.spark.SparkConf +import org.apache.spark.sql.execution.QueryExecution +import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper +import org.apache.spark.sql.execution.command.{DataWritingCommand, DataWritingCommandExec} +import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf} +import org.apache.spark.sql.test.SQLTestData.TestData +import org.apache.spark.sql.test.SQLTestUtils +import org.apache.spark.sql.util.QueryExecutionListener +import org.apache.spark.util.Utils + +import org.apache.kyuubi.sql.KyuubiSQLConf + +trait KyuubiSparkSQLExtensionTest extends QueryTest + with SQLTestUtils + with AdaptiveSparkPlanHelper { + sys.props.put("spark.testing", "1") + + private var _spark: Option[SparkSession] = None + protected def spark: SparkSession = _spark.getOrElse { + throw new RuntimeException("test spark session don't initial before using it.") + } + + override protected def beforeAll(): Unit = { + if (_spark.isEmpty) { + _spark = Option(SparkSession.builder() + .master("local[1]") + .config(sparkConf) + .enableHiveSupport() + .getOrCreate()) + } + super.beforeAll() + } + + override protected def afterAll(): Unit = { + super.afterAll() + cleanupData() + _spark.foreach(_.stop) + } + + protected def setupData(): Unit = { + val self = spark + import self.implicits._ + spark.sparkContext.parallelize( + (1 to 100).map(i => TestData(i, i.toString)), + 10) + .toDF("c1", "c2").createOrReplaceTempView("t1") + spark.sparkContext.parallelize( + (1 to 10).map(i => TestData(i, i.toString)), + 5) + .toDF("c1", "c2").createOrReplaceTempView("t2") + spark.sparkContext.parallelize( + (1 to 50).map(i => TestData(i, i.toString)), + 2) + .toDF("c1", "c2").createOrReplaceTempView("t3") + } + + private def cleanupData(): Unit = { + spark.sql("DROP VIEW IF EXISTS t1") + spark.sql("DROP VIEW IF EXISTS t2") + spark.sql("DROP VIEW IF EXISTS t3") + } + + def sparkConf(): SparkConf = { + val basePath = Utils.createTempDir() + "/" + getClass.getCanonicalName + val metastorePath = basePath + "/metastore_db" + val warehousePath = basePath + "/warehouse" + new SparkConf() + .set( + StaticSQLConf.SPARK_SESSION_EXTENSIONS.key, + "org.apache.kyuubi.sql.KyuubiSparkSQLExtension") + .set(KyuubiSQLConf.SQL_CLASSIFICATION_ENABLED.key, "true") + .set(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key, "true") + .set("spark.hadoop.hive.exec.dynamic.partition.mode", "nonstrict") + .set("spark.hadoop.hive.metastore.client.capability.check", "false") + .set( + ConfVars.METASTORECONNECTURLKEY.varname, + s"jdbc:derby:;databaseName=$metastorePath;create=true") + .set(StaticSQLConf.WAREHOUSE_PATH, warehousePath) + .set("spark.ui.enabled", "false") + } + + def withListener(sqlString: String)(callback: DataWritingCommand => Unit): Unit = { + withListener(sql(sqlString))(callback) + } + + def withListener(df: => DataFrame)(callback: DataWritingCommand => Unit): Unit = { + val listener = new QueryExecutionListener { + override def onFailure(f: String, qe: QueryExecution, e: Exception): Unit = {} + + override def onSuccess(funcName: String, qe: QueryExecution, duration: Long): Unit = { + qe.executedPlan match { + case write: DataWritingCommandExec => callback(write.cmd) + case _ => + } + } + } + spark.listenerManager.register(listener) + try { + df.collect() + sparkContext.listenerBus.waitUntilEmpty() + } finally { + spark.listenerManager.unregister(listener) + } + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/RebalanceBeforeWritingSuite.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/RebalanceBeforeWritingSuite.scala new file mode 100644 index 000000000..1d9630f49 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/RebalanceBeforeWritingSuite.scala @@ -0,0 +1,271 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, RebalancePartitions, Sort} +import org.apache.spark.sql.execution.command.DataWritingCommand +import org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand +import org.apache.spark.sql.hive.HiveUtils +import org.apache.spark.sql.hive.execution.InsertIntoHiveTable + +import org.apache.kyuubi.sql.KyuubiSQLConf + +class RebalanceBeforeWritingSuite extends KyuubiSparkSQLExtensionTest { + + test("check rebalance exists") { + def check(df: => DataFrame, expectedRebalanceNum: Int = 1): Unit = { + withSQLConf(KyuubiSQLConf.INSERT_REPARTITION_BEFORE_WRITE_IF_NO_SHUFFLE.key -> "true") { + withListener(df) { write => + assert(write.collect { + case r: RebalancePartitions => r + }.size == expectedRebalanceNum) + } + } + withSQLConf(KyuubiSQLConf.INSERT_REPARTITION_BEFORE_WRITE_IF_NO_SHUFFLE.key -> "false") { + withListener(df) { write => + assert(write.collect { + case r: RebalancePartitions => r + }.isEmpty) + } + } + } + + // It's better to set config explicitly in case of we change the default value. + withSQLConf(KyuubiSQLConf.INSERT_REPARTITION_BEFORE_WRITE.key -> "true") { + Seq("USING PARQUET", "").foreach { storage => + withTable("tmp1") { + sql(s"CREATE TABLE tmp1 (c1 int) $storage PARTITIONED BY (c2 string)") + check(sql("INSERT INTO TABLE tmp1 PARTITION(c2='a') " + + "SELECT * FROM VALUES(1),(2) AS t(c1)")) + } + + withTable("tmp1", "tmp2") { + sql(s"CREATE TABLE tmp1 (c1 int) $storage PARTITIONED BY (c2 string)") + sql(s"CREATE TABLE tmp2 (c1 int) $storage PARTITIONED BY (c2 string)") + check( + sql( + """FROM VALUES(1),(2) + |INSERT INTO TABLE tmp1 PARTITION(c2='a') SELECT * + |INSERT INTO TABLE tmp2 PARTITION(c2='a') SELECT * + |""".stripMargin), + 2) + } + + withTable("tmp1") { + sql(s"CREATE TABLE tmp1 (c1 int) $storage") + check(sql("INSERT INTO TABLE tmp1 SELECT * FROM VALUES(1),(2),(3) AS t(c1)")) + } + + withTable("tmp1", "tmp2") { + sql(s"CREATE TABLE tmp1 (c1 int) $storage") + sql(s"CREATE TABLE tmp2 (c1 int) $storage") + check( + sql( + """FROM VALUES(1),(2),(3) + |INSERT INTO TABLE tmp1 SELECT * + |INSERT INTO TABLE tmp2 SELECT * + |""".stripMargin), + 2) + } + + withTable("tmp1") { + sql(s"CREATE TABLE tmp1 $storage AS SELECT * FROM VALUES(1),(2),(3) AS t(c1)") + } + + withTable("tmp1") { + sql(s"CREATE TABLE tmp1 $storage PARTITIONED BY(c2) AS " + + s"SELECT * FROM VALUES(1, 'a'),(2, 'b') AS t(c1, c2)") + } + } + } + } + + test("check rebalance does not exists") { + def check(df: DataFrame): Unit = { + withListener(df) { write => + assert(write.collect { + case r: RebalancePartitions => r + }.isEmpty) + } + } + + withSQLConf( + KyuubiSQLConf.INSERT_REPARTITION_BEFORE_WRITE.key -> "true", + KyuubiSQLConf.INSERT_REPARTITION_BEFORE_WRITE_IF_NO_SHUFFLE.key -> "true") { + // test no write command + check(sql("SELECT * FROM VALUES(1, 'a'),(2, 'b') AS t(c1, c2)")) + check(sql("SELECT count(*) FROM VALUES(1, 'a'),(2, 'b') AS t(c1, c2)")) + + // test not supported plan + withTable("tmp1") { + sql(s"CREATE TABLE tmp1 (c1 int) PARTITIONED BY (c2 string)") + check(sql("INSERT INTO TABLE tmp1 PARTITION(c2) " + + "SELECT /*+ repartition(10) */ * FROM VALUES(1, 'a'),(2, 'b') AS t(c1, c2)")) + check(sql("INSERT INTO TABLE tmp1 PARTITION(c2) " + + "SELECT * FROM VALUES(1, 'a'),(2, 'b') AS t(c1, c2) ORDER BY c1")) + check(sql("INSERT INTO TABLE tmp1 PARTITION(c2) " + + "SELECT * FROM VALUES(1, 'a'),(2, 'b') AS t(c1, c2) LIMIT 10")) + } + } + + withSQLConf(KyuubiSQLConf.INSERT_REPARTITION_BEFORE_WRITE.key -> "false") { + Seq("USING PARQUET", "").foreach { storage => + withTable("tmp1") { + sql(s"CREATE TABLE tmp1 (c1 int) $storage PARTITIONED BY (c2 string)") + check(sql("INSERT INTO TABLE tmp1 PARTITION(c2) " + + "SELECT * FROM VALUES(1, 'a'),(2, 'b') AS t(c1, c2)")) + } + + withTable("tmp1") { + sql(s"CREATE TABLE tmp1 (c1 int) $storage") + check(sql("INSERT INTO TABLE tmp1 SELECT * FROM VALUES(1),(2),(3) AS t(c1)")) + } + } + } + } + + test("test dynamic partition write") { + def checkRepartitionExpression(sqlString: String): Unit = { + withListener(sqlString) { write => + assert(write.isInstanceOf[InsertIntoHiveTable]) + assert(write.collect { + case r: RebalancePartitions if r.partitionExpressions.size == 1 => + assert(r.partitionExpressions.head.asInstanceOf[Attribute].name === "c2") + r + }.size == 1) + } + } + + withSQLConf( + KyuubiSQLConf.INSERT_REPARTITION_BEFORE_WRITE.key -> "true", + KyuubiSQLConf.DYNAMIC_PARTITION_INSERTION_REPARTITION_NUM.key -> "2", + KyuubiSQLConf.INSERT_REPARTITION_BEFORE_WRITE_IF_NO_SHUFFLE.key -> "true") { + Seq("USING PARQUET", "").foreach { storage => + withTable("tmp1") { + sql(s"CREATE TABLE tmp1 (c1 int) $storage PARTITIONED BY (c2 string)") + checkRepartitionExpression("INSERT INTO TABLE tmp1 SELECT 1 as c1, 'a' as c2 ") + } + + withTable("tmp1") { + checkRepartitionExpression( + "CREATE TABLE tmp1 PARTITIONED BY(C2) SELECT 1 as c1, 'a' as c2") + } + } + } + } + + test("OptimizedCreateHiveTableAsSelectCommand") { + withSQLConf( + HiveUtils.CONVERT_METASTORE_PARQUET.key -> "true", + HiveUtils.CONVERT_METASTORE_CTAS.key -> "true", + KyuubiSQLConf.INSERT_REPARTITION_BEFORE_WRITE_IF_NO_SHUFFLE.key -> "true") { + withTable("t") { + withListener("CREATE TABLE t STORED AS parquet AS SELECT 1 as a") { write => + assert(write.isInstanceOf[InsertIntoHadoopFsRelationCommand]) + assert(write.collect { + case _: RebalancePartitions => true + }.size == 1) + } + } + } + } + + test("Infer rebalance and sorder orders") { + def checkShuffleAndSort(dataWritingCommand: LogicalPlan, sSize: Int, rSize: Int): Unit = { + assert(dataWritingCommand.isInstanceOf[DataWritingCommand]) + val plan = dataWritingCommand.asInstanceOf[DataWritingCommand].query + assert(plan.collect { + case s: Sort => s + }.size == sSize) + assert(plan.collect { + case r: RebalancePartitions if r.partitionExpressions.size == rSize => r + }.nonEmpty || rSize == 0) + } + + withView("v") { + withTable("t", "input1", "input2") { + withSQLConf(KyuubiSQLConf.INFER_REBALANCE_AND_SORT_ORDERS.key -> "true") { + sql(s"CREATE TABLE t (c1 int, c2 long) USING PARQUET PARTITIONED BY (p string)") + sql(s"CREATE TABLE input1 USING PARQUET AS SELECT * FROM VALUES(1,2),(1,3)") + sql(s"CREATE TABLE input2 USING PARQUET AS SELECT * FROM VALUES(1,3),(1,3)") + sql(s"CREATE VIEW v as SELECT col1, count(*) as col2 FROM input1 GROUP BY col1") + + val df0 = sql( + s""" + |INSERT INTO TABLE t PARTITION(p='a') + |SELECT /*+ broadcast(input2) */ input1.col1, input2.col1 + |FROM input1 + |JOIN input2 + |ON input1.col1 = input2.col1 + |""".stripMargin) + checkShuffleAndSort(df0.queryExecution.analyzed, 1, 1) + + val df1 = sql( + s""" + |INSERT INTO TABLE t PARTITION(p='a') + |SELECT /*+ broadcast(input2) */ input1.col1, input1.col2 + |FROM input1 + |LEFT JOIN input2 + |ON input1.col1 = input2.col1 and input1.col2 = input2.col2 + |""".stripMargin) + checkShuffleAndSort(df1.queryExecution.analyzed, 1, 2) + + val df2 = sql( + s""" + |INSERT INTO TABLE t PARTITION(p='a') + |SELECT col1 as c1, count(*) as c2 + |FROM input1 + |GROUP BY col1 + |HAVING count(*) > 0 + |""".stripMargin) + checkShuffleAndSort(df2.queryExecution.analyzed, 1, 1) + + // dynamic partition + val df3 = sql( + s""" + |INSERT INTO TABLE t PARTITION(p) + |SELECT /*+ broadcast(input2) */ input1.col1, input1.col2, input1.col2 + |FROM input1 + |JOIN input2 + |ON input1.col1 = input2.col1 + |""".stripMargin) + checkShuffleAndSort(df3.queryExecution.analyzed, 0, 1) + + // non-deterministic + val df4 = sql( + s""" + |INSERT INTO TABLE t PARTITION(p='a') + |SELECT col1 + rand(), count(*) as c2 + |FROM input1 + |GROUP BY col1 + |""".stripMargin) + checkShuffleAndSort(df4.queryExecution.analyzed, 0, 0) + + // view + val df5 = sql( + s""" + |INSERT INTO TABLE t PARTITION(p='a') + |SELECT * FROM v + |""".stripMargin) + checkShuffleAndSort(df5.queryExecution.analyzed, 1, 1) + } + } + } + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/WatchDogSuite.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/WatchDogSuite.scala new file mode 100644 index 000000000..957089340 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/WatchDogSuite.scala @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +class WatchDogSuite extends WatchDogSuiteBase {} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/WatchDogSuiteBase.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/WatchDogSuiteBase.scala new file mode 100644 index 000000000..a202e813c --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/WatchDogSuiteBase.scala @@ -0,0 +1,601 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +import java.io.File + +import scala.collection.JavaConverters._ + +import org.apache.commons.io.FileUtils +import org.apache.spark.sql.catalyst.plans.logical.{GlobalLimit, LogicalPlan} + +import org.apache.kyuubi.sql.KyuubiSQLConf +import org.apache.kyuubi.sql.watchdog.{MaxFileSizeExceedException, MaxPartitionExceedException} + +trait WatchDogSuiteBase extends KyuubiSparkSQLExtensionTest { + override protected def beforeAll(): Unit = { + super.beforeAll() + setupData() + } + + case class LimitAndExpected(limit: Int, expected: Int) + + val limitAndExpecteds = List(LimitAndExpected(1, 1), LimitAndExpected(11, 10)) + + private def checkMaxPartition: Unit = { + withSQLConf(KyuubiSQLConf.WATCHDOG_MAX_PARTITIONS.key -> "100") { + checkAnswer(sql("SELECT count(distinct(p)) FROM test"), Row(10) :: Nil) + } + withSQLConf(KyuubiSQLConf.WATCHDOG_MAX_PARTITIONS.key -> "5") { + sql("SELECT * FROM test where p=1").queryExecution.sparkPlan + + sql(s"SELECT * FROM test WHERE p in (${Range(0, 5).toList.mkString(",")})") + .queryExecution.sparkPlan + + intercept[MaxPartitionExceedException]( + sql("SELECT * FROM test where p != 1").queryExecution.sparkPlan) + + intercept[MaxPartitionExceedException]( + sql("SELECT * FROM test").queryExecution.sparkPlan) + + intercept[MaxPartitionExceedException](sql( + s"SELECT * FROM test WHERE p in (${Range(0, 6).toList.mkString(",")})") + .queryExecution.sparkPlan) + } + } + + test("watchdog with scan maxPartitions -- hive") { + Seq("textfile", "parquet").foreach { format => + withTable("test", "temp") { + sql( + s""" + |CREATE TABLE test(i int) + |PARTITIONED BY (p int) + |STORED AS $format""".stripMargin) + spark.range(0, 10, 1).selectExpr("id as col") + .createOrReplaceTempView("temp") + + for (part <- Range(0, 10)) { + sql( + s""" + |INSERT OVERWRITE TABLE test PARTITION (p='$part') + |select col from temp""".stripMargin) + } + checkMaxPartition + } + } + } + + test("watchdog with scan maxPartitions -- data source") { + withTempDir { dir => + withTempView("test") { + spark.range(10).selectExpr("id", "id as p") + .write + .partitionBy("p") + .mode("overwrite") + .save(dir.getCanonicalPath) + spark.read.load(dir.getCanonicalPath).createOrReplaceTempView("test") + checkMaxPartition + } + } + } + + test("test watchdog: simple SELECT STATEMENT") { + + withSQLConf(KyuubiSQLConf.WATCHDOG_FORCED_MAXOUTPUTROWS.key -> "10") { + + List("", "ORDER BY c1", "ORDER BY c2").foreach { sort => + List("", " DISTINCT").foreach { distinct => + assert(sql( + s""" + |SELECT $distinct * + |FROM t1 + |$sort + |""".stripMargin).queryExecution.optimizedPlan.isInstanceOf[GlobalLimit]) + } + } + + limitAndExpecteds.foreach { case LimitAndExpected(limit, expected) => + List("", "ORDER BY c1", "ORDER BY c2").foreach { sort => + List("", "DISTINCT").foreach { distinct => + assert(sql( + s""" + |SELECT $distinct * + |FROM t1 + |$sort + |LIMIT $limit + |""".stripMargin).queryExecution.optimizedPlan.maxRows.contains(expected)) + } + } + } + } + } + + test("test watchdog: SELECT ... WITH AGGREGATE STATEMENT ") { + + withSQLConf(KyuubiSQLConf.WATCHDOG_FORCED_MAXOUTPUTROWS.key -> "10") { + + assert(!sql("SELECT count(*) FROM t1") + .queryExecution.optimizedPlan.isInstanceOf[GlobalLimit]) + + val sorts = List("", "ORDER BY cnt", "ORDER BY c1", "ORDER BY cnt, c1", "ORDER BY c1, cnt") + val havingConditions = List("", "HAVING cnt > 1") + + havingConditions.foreach { having => + sorts.foreach { sort => + assert(sql( + s""" + |SELECT c1, COUNT(*) as cnt + |FROM t1 + |GROUP BY c1 + |$having + |$sort + |""".stripMargin).queryExecution.optimizedPlan.isInstanceOf[GlobalLimit]) + } + } + + limitAndExpecteds.foreach { case LimitAndExpected(limit, expected) => + havingConditions.foreach { having => + sorts.foreach { sort => + assert(sql( + s""" + |SELECT c1, COUNT(*) as cnt + |FROM t1 + |GROUP BY c1 + |$having + |$sort + |LIMIT $limit + |""".stripMargin).queryExecution.optimizedPlan.maxRows.contains(expected)) + } + } + } + } + } + + test("test watchdog: SELECT with CTE forceMaxOutputRows") { + // simple CTE + val q1 = + """ + |WITH t2 AS ( + | SELECT * FROM t1 + |) + |""".stripMargin + + // nested CTE + val q2 = + """ + |WITH + | t AS (SELECT * FROM t1), + | t2 AS ( + | WITH t3 AS (SELECT * FROM t1) + | SELECT * FROM t3 + | ) + |""".stripMargin + withSQLConf(KyuubiSQLConf.WATCHDOG_FORCED_MAXOUTPUTROWS.key -> "10") { + + val sorts = List("", "ORDER BY c1", "ORDER BY c2") + + sorts.foreach { sort => + Seq(q1, q2).foreach { withQuery => + assert(sql( + s""" + |$withQuery + |SELECT * FROM t2 + |$sort + |""".stripMargin).queryExecution.optimizedPlan.isInstanceOf[GlobalLimit]) + } + } + + limitAndExpecteds.foreach { case LimitAndExpected(limit, expected) => + sorts.foreach { sort => + Seq(q1, q2).foreach { withQuery => + assert(sql( + s""" + |$withQuery + |SELECT * FROM t2 + |$sort + |LIMIT $limit + |""".stripMargin).queryExecution.optimizedPlan.maxRows.contains(expected)) + } + } + } + } + } + + test("test watchdog: SELECT AGGREGATE WITH CTE forceMaxOutputRows") { + + withSQLConf(KyuubiSQLConf.WATCHDOG_FORCED_MAXOUTPUTROWS.key -> "10") { + + assert(!sql( + """ + |WITH custom_cte AS ( + |SELECT * FROM t1 + |) + | + |SELECT COUNT(*) + |FROM custom_cte + |""".stripMargin).queryExecution + .analyzed.isInstanceOf[GlobalLimit]) + + val sorts = List("", "ORDER BY cnt", "ORDER BY c1", "ORDER BY cnt, c1", "ORDER BY c1, cnt") + val havingConditions = List("", "HAVING cnt > 1") + + havingConditions.foreach { having => + sorts.foreach { sort => + assert(sql( + s""" + |WITH custom_cte AS ( + |SELECT * FROM t1 + |) + | + |SELECT c1, COUNT(*) as cnt + |FROM custom_cte + |GROUP BY c1 + |$having + |$sort + |""".stripMargin).queryExecution.optimizedPlan.isInstanceOf[GlobalLimit]) + } + } + + limitAndExpecteds.foreach { case LimitAndExpected(limit, expected) => + havingConditions.foreach { having => + sorts.foreach { sort => + assert(sql( + s""" + |WITH custom_cte AS ( + |SELECT * FROM t1 + |) + | + |SELECT c1, COUNT(*) as cnt + |FROM custom_cte + |GROUP BY c1 + |$having + |$sort + |LIMIT $limit + |""".stripMargin).queryExecution.optimizedPlan.maxRows.contains(expected)) + } + } + } + } + } + + test("test watchdog: UNION Statement for forceMaxOutputRows") { + + withSQLConf(KyuubiSQLConf.WATCHDOG_FORCED_MAXOUTPUTROWS.key -> "10") { + + List("", "ALL").foreach { x => + assert(sql( + s""" + |SELECT c1, c2 FROM t1 + |UNION $x + |SELECT c1, c2 FROM t2 + |UNION $x + |SELECT c1, c2 FROM t3 + |""".stripMargin) + .queryExecution.optimizedPlan.isInstanceOf[GlobalLimit]) + } + + val sorts = List("", "ORDER BY cnt", "ORDER BY c1", "ORDER BY cnt, c1", "ORDER BY c1, cnt") + val havingConditions = List("", "HAVING cnt > 1") + + List("", "ALL").foreach { x => + havingConditions.foreach { having => + sorts.foreach { sort => + assert(sql( + s""" + |SELECT c1, count(c2) as cnt + |FROM t1 + |GROUP BY c1 + |$having + |UNION $x + |SELECT c1, COUNT(c2) as cnt + |FROM t2 + |GROUP BY c1 + |$having + |UNION $x + |SELECT c1, COUNT(c2) as cnt + |FROM t3 + |GROUP BY c1 + |$having + |$sort + |""".stripMargin) + .queryExecution.optimizedPlan.isInstanceOf[GlobalLimit]) + } + } + } + + limitAndExpecteds.foreach { case LimitAndExpected(limit, expected) => + assert(sql( + s""" + |SELECT c1, c2 FROM t1 + |UNION + |SELECT c1, c2 FROM t2 + |UNION + |SELECT c1, c2 FROM t3 + |LIMIT $limit + |""".stripMargin) + .queryExecution.optimizedPlan.maxRows.contains(expected)) + } + } + } + + test("test watchdog: Select View Statement for forceMaxOutputRows") { + withSQLConf(KyuubiSQLConf.WATCHDOG_FORCED_MAXOUTPUTROWS.key -> "3") { + withTable("tmp_table", "tmp_union") { + withView("tmp_view", "tmp_view2") { + sql(s"create table tmp_table (a int, b int)") + sql(s"insert into tmp_table values (1,10),(2,20),(3,30),(4,40),(5,50)") + sql(s"create table tmp_union (a int, b int)") + sql(s"insert into tmp_union values (6,60),(7,70),(8,80),(9,90),(10,100)") + sql(s"create view tmp_view2 as select * from tmp_union") + assert(!sql( + s""" + |CREATE VIEW tmp_view + |as + |SELECT * FROM + |tmp_table + |""".stripMargin) + .queryExecution.optimizedPlan.isInstanceOf[GlobalLimit]) + + assert(sql( + s""" + |SELECT * FROM + |tmp_view + |""".stripMargin) + .queryExecution.optimizedPlan.maxRows.contains(3)) + + assert(sql( + s""" + |SELECT * FROM + |tmp_view + |limit 11 + |""".stripMargin) + .queryExecution.optimizedPlan.maxRows.contains(3)) + + assert(sql( + s""" + |SELECT * FROM + |(select * from tmp_view + |UNION + |select * from tmp_view2) + |ORDER BY a + |DESC + |""".stripMargin) + .collect().head.get(0) === 10) + } + } + } + } + + test("test watchdog: Insert Statement for forceMaxOutputRows") { + + withSQLConf(KyuubiSQLConf.WATCHDOG_FORCED_MAXOUTPUTROWS.key -> "10") { + withTable("tmp_table", "tmp_insert") { + spark.sql(s"create table tmp_table (a int, b int)") + spark.sql(s"insert into tmp_table values (1,10),(2,20),(3,30),(4,40),(5,50)") + val multiInsertTableName1: String = "tmp_tbl1" + val multiInsertTableName2: String = "tmp_tbl2" + sql(s"drop table if exists $multiInsertTableName1") + sql(s"drop table if exists $multiInsertTableName2") + sql(s"create table $multiInsertTableName1 like tmp_table") + sql(s"create table $multiInsertTableName2 like tmp_table") + assert(!sql( + s""" + |FROM tmp_table + |insert into $multiInsertTableName1 select * limit 2 + |insert into $multiInsertTableName2 select * + |""".stripMargin) + .queryExecution.optimizedPlan.isInstanceOf[GlobalLimit]) + } + } + } + + test("test watchdog: Distribute by for forceMaxOutputRows") { + + withSQLConf(KyuubiSQLConf.WATCHDOG_FORCED_MAXOUTPUTROWS.key -> "10") { + withTable("tmp_table") { + spark.sql(s"create table tmp_table (a int, b int)") + spark.sql(s"insert into tmp_table values (1,10),(2,20),(3,30),(4,40),(5,50)") + assert(sql( + s""" + |SELECT * + |FROM tmp_table + |DISTRIBUTE BY a + |""".stripMargin) + .queryExecution.optimizedPlan.isInstanceOf[GlobalLimit]) + } + } + } + + test("test watchdog: Subquery for forceMaxOutputRows") { + withSQLConf(KyuubiSQLConf.WATCHDOG_FORCED_MAXOUTPUTROWS.key -> "1") { + withTable("tmp_table1") { + sql("CREATE TABLE spark_catalog.`default`.tmp_table1(KEY INT, VALUE STRING) USING PARQUET") + sql("INSERT INTO TABLE spark_catalog.`default`.tmp_table1 " + + "VALUES (1, 'aa'),(2,'bb'),(3, 'cc'),(4,'aa'),(5,'cc'),(6, 'aa')") + assert( + sql("select * from tmp_table1").queryExecution.optimizedPlan.isInstanceOf[GlobalLimit]) + val testSqlText = + """ + |select count(*) + |from tmp_table1 + |where tmp_table1.key in ( + |select distinct tmp_table1.key + |from tmp_table1 + |where tmp_table1.value = "aa" + |) + |""".stripMargin + val plan = sql(testSqlText).queryExecution.optimizedPlan + assert(!findGlobalLimit(plan)) + checkAnswer(sql(testSqlText), Row(3) :: Nil) + } + + def findGlobalLimit(plan: LogicalPlan): Boolean = plan match { + case _: GlobalLimit => true + case p if p.children.isEmpty => false + case p => p.children.exists(findGlobalLimit) + } + + } + } + + test("test watchdog: Join for forceMaxOutputRows") { + withSQLConf(KyuubiSQLConf.WATCHDOG_FORCED_MAXOUTPUTROWS.key -> "1") { + withTable("tmp_table1", "tmp_table2") { + sql("CREATE TABLE spark_catalog.`default`.tmp_table1(KEY INT, VALUE STRING) USING PARQUET") + sql("INSERT INTO TABLE spark_catalog.`default`.tmp_table1 " + + "VALUES (1, 'aa'),(2,'bb'),(3, 'cc'),(4,'aa'),(5,'cc'),(6, 'aa')") + sql("CREATE TABLE spark_catalog.`default`.tmp_table2(KEY INT, VALUE STRING) USING PARQUET") + sql("INSERT INTO TABLE spark_catalog.`default`.tmp_table2 " + + "VALUES (1, 'aa'),(2,'bb'),(3, 'cc'),(4,'aa'),(5,'cc'),(6, 'aa')") + val testSqlText = + """ + |select a.*,b.* + |from tmp_table1 a + |join + |tmp_table2 b + |on a.KEY = b.KEY + |""".stripMargin + val plan = sql(testSqlText).queryExecution.optimizedPlan + assert(findGlobalLimit(plan)) + } + + def findGlobalLimit(plan: LogicalPlan): Boolean = plan match { + case _: GlobalLimit => true + case p if p.children.isEmpty => false + case p => p.children.exists(findGlobalLimit) + } + } + } + + private def checkMaxFileSize(tableSize: Long, nonPartTableSize: Long): Unit = { + withSQLConf(KyuubiSQLConf.WATCHDOG_MAX_FILE_SIZE.key -> tableSize.toString) { + checkAnswer(sql("SELECT count(distinct(p)) FROM test"), Row(10) :: Nil) + } + + withSQLConf(KyuubiSQLConf.WATCHDOG_MAX_FILE_SIZE.key -> (tableSize / 2).toString) { + sql("SELECT * FROM test where p=1").queryExecution.sparkPlan + + sql(s"SELECT * FROM test WHERE p in (${Range(0, 3).toList.mkString(",")})") + .queryExecution.sparkPlan + + intercept[MaxFileSizeExceedException]( + sql("SELECT * FROM test where p != 1").queryExecution.sparkPlan) + + intercept[MaxFileSizeExceedException]( + sql("SELECT * FROM test").queryExecution.sparkPlan) + + intercept[MaxFileSizeExceedException](sql( + s"SELECT * FROM test WHERE p in (${Range(0, 6).toList.mkString(",")})") + .queryExecution.sparkPlan) + } + + withSQLConf(KyuubiSQLConf.WATCHDOG_MAX_FILE_SIZE.key -> nonPartTableSize.toString) { + checkAnswer(sql("SELECT count(*) FROM test_non_part"), Row(10000) :: Nil) + } + + withSQLConf(KyuubiSQLConf.WATCHDOG_MAX_FILE_SIZE.key -> (nonPartTableSize - 1).toString) { + intercept[MaxFileSizeExceedException]( + sql("SELECT * FROM test_non_part").queryExecution.sparkPlan) + } + } + + test("watchdog with scan maxFileSize -- hive") { + Seq(false).foreach { convertMetastoreParquet => + withTable("test", "test_non_part", "temp") { + spark.range(10000).selectExpr("id as col") + .createOrReplaceTempView("temp") + + // partitioned table + sql( + s""" + |CREATE TABLE test(i int) + |PARTITIONED BY (p int) + |STORED AS parquet""".stripMargin) + for (part <- Range(0, 10)) { + sql( + s""" + |INSERT OVERWRITE TABLE test PARTITION (p='$part') + |select col from temp""".stripMargin) + } + + val tablePath = new File(spark.sessionState.catalog.externalCatalog + .getTable("default", "test").location) + val tableSize = FileUtils.listFiles(tablePath, Array("parquet"), true).asScala + .map(_.length()).sum + assert(tableSize > 0) + + // non-partitioned table + sql( + s""" + |CREATE TABLE test_non_part(i int) + |STORED AS parquet""".stripMargin) + sql( + s""" + |INSERT OVERWRITE TABLE test_non_part + |select col from temp""".stripMargin) + sql("ANALYZE TABLE test_non_part COMPUTE STATISTICS") + + val nonPartTablePath = new File(spark.sessionState.catalog.externalCatalog + .getTable("default", "test_non_part").location) + val nonPartTableSize = FileUtils.listFiles(nonPartTablePath, Array("parquet"), true).asScala + .map(_.length()).sum + assert(nonPartTableSize > 0) + + // check + withSQLConf("spark.sql.hive.convertMetastoreParquet" -> convertMetastoreParquet.toString) { + checkMaxFileSize(tableSize, nonPartTableSize) + } + } + } + } + + test("watchdog with scan maxFileSize -- data source") { + withTempDir { dir => + withTempView("test", "test_non_part") { + // partitioned table + val tablePath = new File(dir, "test") + spark.range(10).selectExpr("id", "id as p") + .write + .partitionBy("p") + .mode("overwrite") + .parquet(tablePath.getCanonicalPath) + spark.read.load(tablePath.getCanonicalPath).createOrReplaceTempView("test") + + val tableSize = FileUtils.listFiles(tablePath, Array("parquet"), true).asScala + .map(_.length()).sum + assert(tableSize > 0) + + // non-partitioned table + val nonPartTablePath = new File(dir, "test_non_part") + spark.range(10000).selectExpr("id", "id as p") + .write + .mode("overwrite") + .parquet(nonPartTablePath.getCanonicalPath) + spark.read.load(nonPartTablePath.getCanonicalPath).createOrReplaceTempView("test_non_part") + + val nonPartTableSize = FileUtils.listFiles(nonPartTablePath, Array("parquet"), true).asScala + .map(_.length()).sum + assert(tableSize > 0) + + // check + checkMaxFileSize(tableSize, nonPartTableSize) + } + } + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/ZorderCoreBenchmark.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/ZorderCoreBenchmark.scala new file mode 100644 index 000000000..9b1614fce --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/ZorderCoreBenchmark.scala @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +import org.apache.spark.SparkConf +import org.apache.spark.benchmark.Benchmark +import org.apache.spark.sql.benchmark.KyuubiBenchmarkBase +import org.apache.spark.sql.internal.StaticSQLConf + +import org.apache.kyuubi.sql.zorder.ZorderBytesUtils + +/** + * Benchmark to measure performance with zorder core. + * + * {{{ + * RUN_BENCHMARK=1 ./build/mvn clean test \ + * -pl extensions/spark/kyuubi-extension-spark-3-1 -am \ + * -Pspark-3.1,kyuubi-extension-spark-3-1 \ + * -Dtest=none -DwildcardSuites=org.apache.spark.sql.ZorderCoreBenchmark + * }}} + */ +class ZorderCoreBenchmark extends KyuubiSparkSQLExtensionTest with KyuubiBenchmarkBase { + private val runBenchmark = sys.env.contains("RUN_BENCHMARK") + private val numRows = 1 * 1000 * 1000 + + private def randomInt(numColumns: Int): Seq[Array[Any]] = { + (1 to numRows).map { l => + val arr = new Array[Any](numColumns) + (0 until numColumns).foreach(col => arr(col) = l) + arr + } + } + + private def randomLong(numColumns: Int): Seq[Array[Any]] = { + (1 to numRows).map { l => + val arr = new Array[Any](numColumns) + (0 until numColumns).foreach(col => arr(col) = l.toLong) + arr + } + } + + private def interleaveMultiByteArrayBenchmark(): Unit = { + val benchmark = + new Benchmark(s"$numRows rows zorder core benchmark", numRows, output = output) + benchmark.addCase("2 int columns benchmark", 3) { _ => + randomInt(2).foreach(ZorderBytesUtils.interleaveBits) + } + + benchmark.addCase("3 int columns benchmark", 3) { _ => + randomInt(3).foreach(ZorderBytesUtils.interleaveBits) + } + + benchmark.addCase("4 int columns benchmark", 3) { _ => + randomInt(4).foreach(ZorderBytesUtils.interleaveBits) + } + + benchmark.addCase("2 long columns benchmark", 3) { _ => + randomLong(2).foreach(ZorderBytesUtils.interleaveBits) + } + + benchmark.addCase("3 long columns benchmark", 3) { _ => + randomLong(3).foreach(ZorderBytesUtils.interleaveBits) + } + + benchmark.addCase("4 long columns benchmark", 3) { _ => + randomLong(4).foreach(ZorderBytesUtils.interleaveBits) + } + + benchmark.run() + } + + private def paddingTo8ByteBenchmark() { + val iterations = 10 * 1000 * 1000 + + val b2 = Array('a'.toByte, 'b'.toByte) + val benchmark = + new Benchmark(s"$iterations iterations paddingTo8Byte benchmark", iterations, output = output) + benchmark.addCase("2 length benchmark", 3) { _ => + (1 to iterations).foreach(_ => ZorderBytesUtils.paddingTo8Byte(b2)) + } + + val b16 = Array.tabulate(16) { i => i.toByte } + benchmark.addCase("16 length benchmark", 3) { _ => + (1 to iterations).foreach(_ => ZorderBytesUtils.paddingTo8Byte(b16)) + } + + benchmark.run() + } + + test("zorder core benchmark") { + assume(runBenchmark) + + withHeader { + interleaveMultiByteArrayBenchmark() + paddingTo8ByteBenchmark() + } + } + + override def sparkConf(): SparkConf = { + super.sparkConf().remove(StaticSQLConf.SPARK_SESSION_EXTENSIONS.key) + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/ZorderSuite.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/ZorderSuite.scala new file mode 100644 index 000000000..c2fa16197 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/ZorderSuite.scala @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +import org.apache.spark.sql.catalyst.parser.ParserInterface +import org.apache.spark.sql.catalyst.plans.logical.{RebalancePartitions, Sort} +import org.apache.spark.sql.internal.SQLConf + +import org.apache.kyuubi.sql.{KyuubiSQLConf, SparkKyuubiSparkSQLParser} +import org.apache.kyuubi.sql.zorder.Zorder + +trait ZorderSuiteSpark extends ZorderSuiteBase { + + test("Add rebalance before zorder") { + Seq("true" -> false, "false" -> true).foreach { case (useOriginalOrdering, zorder) => + withSQLConf( + KyuubiSQLConf.ZORDER_GLOBAL_SORT_ENABLED.key -> "false", + KyuubiSQLConf.REBALANCE_BEFORE_ZORDER.key -> "true", + KyuubiSQLConf.REBALANCE_ZORDER_COLUMNS_ENABLED.key -> "true", + KyuubiSQLConf.ZORDER_USING_ORIGINAL_ORDERING_ENABLED.key -> useOriginalOrdering) { + withTable("t") { + sql( + """ + |CREATE TABLE t (c1 int, c2 string) PARTITIONED BY (d string) + | TBLPROPERTIES ( + |'kyuubi.zorder.enabled'= 'true', + |'kyuubi.zorder.cols'= 'c1,C2') + |""".stripMargin) + val p = sql("INSERT INTO TABLE t PARTITION(d='a') SELECT * FROM VALUES(1,'a')") + .queryExecution.analyzed + assert(p.collect { + case sort: Sort + if !sort.global && + ((sort.order.exists(_.child.isInstanceOf[Zorder]) && zorder) || + (!sort.order.exists(_.child.isInstanceOf[Zorder]) && !zorder)) => sort + }.size == 1) + assert(p.collect { + case rebalance: RebalancePartitions + if rebalance.references.map(_.name).exists(_.equals("c1")) => rebalance + }.size == 1) + + val p2 = sql("INSERT INTO TABLE t PARTITION(d) SELECT * FROM VALUES(1,'a','b')") + .queryExecution.analyzed + assert(p2.collect { + case sort: Sort + if (!sort.global && Seq("c1", "c2", "d").forall(x => + sort.references.map(_.name).exists(_.equals(x)))) && + ((sort.order.exists(_.child.isInstanceOf[Zorder]) && zorder) || + (!sort.order.exists(_.child.isInstanceOf[Zorder]) && !zorder)) => sort + }.size == 1) + assert(p2.collect { + case rebalance: RebalancePartitions + if Seq("c1", "c2", "d").forall(x => + rebalance.references.map(_.name).exists(_.equals(x))) => rebalance + }.size == 1) + } + } + } + } + + test("Two phase rebalance before Z-Order") { + withSQLConf( + SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> + "org.apache.spark.sql.catalyst.optimizer.CollapseRepartition", + KyuubiSQLConf.ZORDER_GLOBAL_SORT_ENABLED.key -> "false", + KyuubiSQLConf.REBALANCE_BEFORE_ZORDER.key -> "true", + KyuubiSQLConf.TWO_PHASE_REBALANCE_BEFORE_ZORDER.key -> "true", + KyuubiSQLConf.REBALANCE_ZORDER_COLUMNS_ENABLED.key -> "true") { + withTable("t") { + sql( + """ + |CREATE TABLE t (c1 int) PARTITIONED BY (d string) + | TBLPROPERTIES ( + |'kyuubi.zorder.enabled'= 'true', + |'kyuubi.zorder.cols'= 'c1') + |""".stripMargin) + val p = sql("INSERT INTO TABLE t PARTITION(d) SELECT * FROM VALUES(1,'a')") + val rebalance = p.queryExecution.optimizedPlan.innerChildren + .flatMap(_.collect { case r: RebalancePartitions => r }) + assert(rebalance.size == 2) + assert(rebalance.head.partitionExpressions.flatMap(_.references.map(_.name)) + .contains("d")) + assert(rebalance.head.partitionExpressions.flatMap(_.references.map(_.name)) + .contains("c1")) + + assert(rebalance(1).partitionExpressions.flatMap(_.references.map(_.name)) + .contains("d")) + assert(!rebalance(1).partitionExpressions.flatMap(_.references.map(_.name)) + .contains("c1")) + } + } + } +} + +trait ParserSuite { self: ZorderSuiteBase => + override def createParser: ParserInterface = { + new SparkKyuubiSparkSQLParser(spark.sessionState.sqlParser) + } +} + +class ZorderWithCodegenEnabledSuite + extends ZorderWithCodegenEnabledSuiteBase + with ZorderSuiteSpark + with ParserSuite {} +class ZorderWithCodegenDisabledSuite + extends ZorderWithCodegenDisabledSuiteBase + with ZorderSuiteSpark + with ParserSuite {} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/ZorderSuiteBase.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/ZorderSuiteBase.scala new file mode 100644 index 000000000..2d3eec957 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/ZorderSuiteBase.scala @@ -0,0 +1,768 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql + +import org.apache.spark.SparkConf +import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier} +import org.apache.spark.sql.catalyst.analysis.{UnresolvedAttribute, UnresolvedFunction, UnresolvedRelation, UnresolvedStar} +import org.apache.spark.sql.catalyst.expressions.{Alias, Ascending, AttributeReference, EqualTo, Expression, ExpressionEvalHelper, Literal, NullsLast, SortOrder} +import org.apache.spark.sql.catalyst.parser.{ParseException, ParserInterface} +import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, OneRowRelation, Project, Sort} +import org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand +import org.apache.spark.sql.functions._ +import org.apache.spark.sql.hive.execution.InsertIntoHiveTable +import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf} +import org.apache.spark.sql.types._ + +import org.apache.kyuubi.sql.{KyuubiSQLConf, KyuubiSQLExtensionException} +import org.apache.kyuubi.sql.zorder.{OptimizeZorderCommandBase, OptimizeZorderStatement, Zorder, ZorderBytesUtils} + +trait ZorderSuiteBase extends KyuubiSparkSQLExtensionTest with ExpressionEvalHelper { + override def sparkConf(): SparkConf = { + super.sparkConf() + .set( + StaticSQLConf.SPARK_SESSION_EXTENSIONS.key, + "org.apache.kyuubi.sql.KyuubiSparkSQLCommonExtension") + } + + test("optimize unpartitioned table") { + withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") { + withTable("up") { + sql(s"DROP TABLE IF EXISTS up") + + val target = Seq( + Seq(0, 0), + Seq(1, 0), + Seq(0, 1), + Seq(1, 1), + Seq(2, 0), + Seq(3, 0), + Seq(2, 1), + Seq(3, 1), + Seq(0, 2), + Seq(1, 2), + Seq(0, 3), + Seq(1, 3), + Seq(2, 2), + Seq(3, 2), + Seq(2, 3), + Seq(3, 3)) + sql(s"CREATE TABLE up (c1 INT, c2 INT, c3 INT)") + sql(s"INSERT INTO TABLE up VALUES" + + "(0,0,2),(0,1,2),(0,2,1),(0,3,3)," + + "(1,0,4),(1,1,2),(1,2,1),(1,3,3)," + + "(2,0,2),(2,1,1),(2,2,5),(2,3,5)," + + "(3,0,3),(3,1,4),(3,2,9),(3,3,0)") + + val e = intercept[KyuubiSQLExtensionException] { + sql("OPTIMIZE up WHERE c1 > 1 ZORDER BY c1, c2") + } + assert(e.getMessage == "Filters are only supported for partitioned table") + + sql("OPTIMIZE up ZORDER BY c1, c2") + val res = sql("SELECT c1, c2 FROM up").collect() + + assert(res.length == 16) + + for (i <- target.indices) { + val t = target(i) + val r = res(i) + assert(t(0) == r.getInt(0)) + assert(t(1) == r.getInt(1)) + } + } + } + } + + test("optimize partitioned table") { + withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") { + withTable("p") { + sql("DROP TABLE IF EXISTS p") + + val target = Seq( + Seq(0, 0), + Seq(1, 0), + Seq(0, 1), + Seq(1, 1), + Seq(2, 0), + Seq(3, 0), + Seq(2, 1), + Seq(3, 1), + Seq(0, 2), + Seq(1, 2), + Seq(0, 3), + Seq(1, 3), + Seq(2, 2), + Seq(3, 2), + Seq(2, 3), + Seq(3, 3)) + + sql(s"CREATE TABLE p (c1 INT, c2 INT, c3 INT) PARTITIONED BY (id INT)") + sql(s"ALTER TABLE p ADD PARTITION (id = 1)") + sql(s"ALTER TABLE p ADD PARTITION (id = 2)") + sql(s"INSERT INTO TABLE p PARTITION (id = 1) VALUES" + + "(0,0,2),(0,1,2),(0,2,1),(0,3,3)," + + "(1,0,4),(1,1,2),(1,2,1),(1,3,3)," + + "(2,0,2),(2,1,1),(2,2,5),(2,3,5)," + + "(3,0,3),(3,1,4),(3,2,9),(3,3,0)") + sql(s"INSERT INTO TABLE p PARTITION (id = 2) VALUES" + + "(0,0,2),(0,1,2),(0,2,1),(0,3,3)," + + "(1,0,4),(1,1,2),(1,2,1),(1,3,3)," + + "(2,0,2),(2,1,1),(2,2,5),(2,3,5)," + + "(3,0,3),(3,1,4),(3,2,9),(3,3,0)") + + sql(s"OPTIMIZE p ZORDER BY c1, c2") + + val res1 = sql(s"SELECT c1, c2 FROM p WHERE id = 1").collect() + val res2 = sql(s"SELECT c1, c2 FROM p WHERE id = 2").collect() + + assert(res1.length == 16) + assert(res2.length == 16) + + for (i <- target.indices) { + val t = target(i) + val r1 = res1(i) + assert(t(0) == r1.getInt(0)) + assert(t(1) == r1.getInt(1)) + + val r2 = res2(i) + assert(t(0) == r2.getInt(0)) + assert(t(1) == r2.getInt(1)) + } + } + } + } + + test("optimize partitioned table with filters") { + withSQLConf(SQLConf.SHUFFLE_PARTITIONS.key -> "1") { + withTable("p") { + sql("DROP TABLE IF EXISTS p") + + val target1 = Seq( + Seq(0, 0), + Seq(1, 0), + Seq(0, 1), + Seq(1, 1), + Seq(2, 0), + Seq(3, 0), + Seq(2, 1), + Seq(3, 1), + Seq(0, 2), + Seq(1, 2), + Seq(0, 3), + Seq(1, 3), + Seq(2, 2), + Seq(3, 2), + Seq(2, 3), + Seq(3, 3)) + val target2 = Seq( + Seq(0, 0), + Seq(0, 1), + Seq(0, 2), + Seq(0, 3), + Seq(1, 0), + Seq(1, 1), + Seq(1, 2), + Seq(1, 3), + Seq(2, 0), + Seq(2, 1), + Seq(2, 2), + Seq(2, 3), + Seq(3, 0), + Seq(3, 1), + Seq(3, 2), + Seq(3, 3)) + sql(s"CREATE TABLE p (c1 INT, c2 INT, c3 INT) PARTITIONED BY (id INT)") + sql(s"ALTER TABLE p ADD PARTITION (id = 1)") + sql(s"ALTER TABLE p ADD PARTITION (id = 2)") + sql(s"INSERT INTO TABLE p PARTITION (id = 1) VALUES" + + "(0,0,2),(0,1,2),(0,2,1),(0,3,3)," + + "(1,0,4),(1,1,2),(1,2,1),(1,3,3)," + + "(2,0,2),(2,1,1),(2,2,5),(2,3,5)," + + "(3,0,3),(3,1,4),(3,2,9),(3,3,0)") + sql(s"INSERT INTO TABLE p PARTITION (id = 2) VALUES" + + "(0,0,2),(0,1,2),(0,2,1),(0,3,3)," + + "(1,0,4),(1,1,2),(1,2,1),(1,3,3)," + + "(2,0,2),(2,1,1),(2,2,5),(2,3,5)," + + "(3,0,3),(3,1,4),(3,2,9),(3,3,0)") + + val e = intercept[KyuubiSQLExtensionException]( + sql(s"OPTIMIZE p WHERE id = 1 AND c1 > 1 ZORDER BY c1, c2")) + assert(e.getMessage == "Only partition column filters are allowed") + + sql(s"OPTIMIZE p WHERE id = 1 ZORDER BY c1, c2") + + val res1 = sql(s"SELECT c1, c2 FROM p WHERE id = 1").collect() + val res2 = sql(s"SELECT c1, c2 FROM p WHERE id = 2").collect() + + assert(res1.length == 16) + assert(res2.length == 16) + + for (i <- target1.indices) { + val t1 = target1(i) + val r1 = res1(i) + assert(t1(0) == r1.getInt(0)) + assert(t1(1) == r1.getInt(1)) + + val t2 = target2(i) + val r2 = res2(i) + assert(t2(0) == r2.getInt(0)) + assert(t2(1) == r2.getInt(1)) + } + } + } + } + + test("optimize zorder with datasource table") { + // TODO remove this if we support datasource table + withTable("t") { + sql("CREATE TABLE t (c1 int, c2 int) USING PARQUET") + val msg = intercept[KyuubiSQLExtensionException] { + sql("OPTIMIZE t ZORDER BY c1, c2") + }.getMessage + assert(msg.contains("only support hive table")) + } + } + + private def checkZorderTable( + enabled: Boolean, + cols: String, + planHasRepartition: Boolean, + resHasSort: Boolean): Unit = { + def checkSort(plan: LogicalPlan): Unit = { + assert(plan.isInstanceOf[Sort] === resHasSort) + plan match { + case sort: Sort => + val colArr = cols.split(",") + val refs = + if (colArr.length == 1) { + sort.order.head + .child.asInstanceOf[AttributeReference] :: Nil + } else { + sort.order.head + .child.asInstanceOf[Zorder].children.map(_.references.head) + } + assert(refs.size === colArr.size) + refs.zip(colArr).foreach { case (ref, col) => + assert(ref.name === col.trim) + } + case _ => + } + } + + val repartition = + if (planHasRepartition) { + "/*+ repartition */" + } else { + "" + } + withSQLConf("spark.sql.shuffle.partitions" -> "1") { + // hive + withSQLConf("spark.sql.hive.convertMetastoreParquet" -> "false") { + withTable("zorder_t1", "zorder_t2_true", "zorder_t2_false") { + sql( + s""" + |CREATE TABLE zorder_t1 (c1 int, c2 string, c3 long, c4 double) STORED AS PARQUET + |TBLPROPERTIES ( + | 'kyuubi.zorder.enabled' = '$enabled', + | 'kyuubi.zorder.cols' = '$cols') + |""".stripMargin) + val df1 = sql(s""" + |INSERT INTO TABLE zorder_t1 + |SELECT $repartition * FROM VALUES(1,'a',2,4D),(2,'b',3,6D) + |""".stripMargin) + assert(df1.queryExecution.analyzed.isInstanceOf[InsertIntoHiveTable]) + checkSort(df1.queryExecution.analyzed.children.head) + + Seq("true", "false").foreach { optimized => + withSQLConf( + "spark.sql.hive.convertMetastoreCtas" -> optimized, + "spark.sql.hive.convertMetastoreParquet" -> optimized) { + + withListener( + s""" + |CREATE TABLE zorder_t2_$optimized STORED AS PARQUET + |TBLPROPERTIES ( + | 'kyuubi.zorder.enabled' = '$enabled', + | 'kyuubi.zorder.cols' = '$cols') + | + |SELECT $repartition * FROM + |VALUES(1,'a',2,4D),(2,'b',3,6D) AS t(c1 ,c2 , c3, c4) + |""".stripMargin) { write => + if (optimized.toBoolean) { + assert(write.isInstanceOf[InsertIntoHadoopFsRelationCommand]) + } else { + assert(write.isInstanceOf[InsertIntoHiveTable]) + } + checkSort(write.query) + } + } + } + } + } + + // datasource + withTable("zorder_t3", "zorder_t4") { + sql( + s""" + |CREATE TABLE zorder_t3 (c1 int, c2 string, c3 long, c4 double) USING PARQUET + |TBLPROPERTIES ( + | 'kyuubi.zorder.enabled' = '$enabled', + | 'kyuubi.zorder.cols' = '$cols') + |""".stripMargin) + val df1 = sql(s""" + |INSERT INTO TABLE zorder_t3 + |SELECT $repartition * FROM VALUES(1,'a',2,4D),(2,'b',3,6D) + |""".stripMargin) + assert(df1.queryExecution.analyzed.isInstanceOf[InsertIntoHadoopFsRelationCommand]) + checkSort(df1.queryExecution.analyzed.children.head) + + withListener( + s""" + |CREATE TABLE zorder_t4 USING PARQUET + |TBLPROPERTIES ( + | 'kyuubi.zorder.enabled' = '$enabled', + | 'kyuubi.zorder.cols' = '$cols') + | + |SELECT $repartition * FROM + |VALUES(1,'a',2,4D),(2,'b',3,6D) AS t(c1 ,c2 , c3, c4) + |""".stripMargin) { write => + assert(write.isInstanceOf[InsertIntoHadoopFsRelationCommand]) + checkSort(write.query) + } + } + } + } + + test("Support insert zorder by table properties") { + withSQLConf(KyuubiSQLConf.INSERT_ZORDER_BEFORE_WRITING.key -> "false") { + checkZorderTable(true, "c1", false, false) + checkZorderTable(false, "c1", false, false) + } + withSQLConf(KyuubiSQLConf.INSERT_ZORDER_BEFORE_WRITING.key -> "true") { + checkZorderTable(true, "", false, false) + checkZorderTable(true, "c5", false, false) + checkZorderTable(true, "c1,c5", false, false) + checkZorderTable(false, "c3", false, false) + checkZorderTable(true, "c3", true, false) + checkZorderTable(true, "c3", false, true) + checkZorderTable(true, "c2,c4", false, true) + checkZorderTable(true, "c4, c2, c1, c3", false, true) + } + } + + test("zorder: check unsupported data type") { + def checkZorderPlan(zorder: Expression): Unit = { + val msg = intercept[AnalysisException] { + val plan = Project(Seq(Alias(zorder, "c")()), OneRowRelation()) + spark.sessionState.analyzer.checkAnalysis(plan) + }.getMessage + // before Spark 3.2.0 the null type catalog string is null, after Spark 3.2.0 it's void + // see https://github.com/apache/spark/pull/33437 + assert(msg.contains("Unsupported z-order type:") && + (msg.contains("null") || msg.contains("void"))) + } + + checkZorderPlan(Zorder(Seq(Literal(null, NullType)))) + checkZorderPlan(Zorder(Seq(Literal(1, IntegerType), Literal(null, NullType)))) + } + + test("zorder: check supported data type") { + val children = Seq( + Literal.create(false, BooleanType), + Literal.create(null, BooleanType), + Literal.create(1.toByte, ByteType), + Literal.create(null, ByteType), + Literal.create(1.toShort, ShortType), + Literal.create(null, ShortType), + Literal.create(1, IntegerType), + Literal.create(null, IntegerType), + Literal.create(1L, LongType), + Literal.create(null, LongType), + Literal.create(1f, FloatType), + Literal.create(null, FloatType), + Literal.create(1d, DoubleType), + Literal.create(null, DoubleType), + Literal.create("1", StringType), + Literal.create(null, StringType), + Literal.create(1L, TimestampType), + Literal.create(null, TimestampType), + Literal.create(1, DateType), + Literal.create(null, DateType), + Literal.create(BigDecimal(1, 1), DecimalType(1, 1)), + Literal.create(null, DecimalType(1, 1))) + val zorder = Zorder(children) + val plan = Project(Seq(Alias(zorder, "c")()), OneRowRelation()) + spark.sessionState.analyzer.checkAnalysis(plan) + assert(zorder.foldable) + +// // scalastyle:off +// val resultGen = org.apache.commons.codec.binary.Hex.encodeHex( +// zorder.eval(InternalRow.fromSeq(children)).asInstanceOf[Array[Byte]], false) +// resultGen.grouped(2).zipWithIndex.foreach { case (char, i) => +// print("0x" + char(0) + char(1) + ", ") +// if ((i + 1) % 10 == 0) { +// println() +// } +// } +// // scalastyle:on + + val expected = Array( + 0xFB, 0xEA, 0xAA, 0xBA, 0xAE, 0xAB, 0xAA, 0xEA, 0xBA, 0xAE, 0xAB, 0xAA, 0xEA, 0xBA, 0xA6, + 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, + 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xBA, 0xBB, 0xAA, 0xAA, 0xAA, + 0xBA, 0xAA, 0xBA, 0xAA, 0xBA, 0xAA, 0xBA, 0xAA, 0xBA, 0xAA, 0xBA, 0xAA, 0x9A, 0xAA, 0xAA, + 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xEA, + 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, + 0xAA, 0xAA, 0xBE, 0xAA, 0xAA, 0x8A, 0xBA, 0xAA, 0x2A, 0xEA, 0xA8, 0xAA, 0xAA, 0xA2, 0xAA, + 0xAA, 0x8A, 0xAA, 0xAA, 0x2F, 0xEB, 0xFE) + .map(_.toByte) + checkEvaluation(zorder, expected, InternalRow.fromSeq(children)) + } + + private def checkSort(input: DataFrame, expected: Seq[Row], dataType: Array[DataType]): Unit = { + withTempDir { dir => + input.repartition(3).write.mode("overwrite").format("parquet").save(dir.getCanonicalPath) + val df = spark.read.format("parquet") + .load(dir.getCanonicalPath) + .repartition(1) + assert(df.schema.fields.map(_.dataType).sameElements(dataType)) + val exprs = Seq("c1", "c2").map(col).map(_.expr) + val sortOrder = SortOrder(Zorder(exprs), Ascending, NullsLast, Seq.empty) + val zorderSort = Sort(Seq(sortOrder), true, df.logicalPlan) + val result = Dataset.ofRows(spark, zorderSort) + checkAnswer(result, expected) + } + } + + test("sort with zorder -- boolean column") { + val schema = StructType(StructField("c1", BooleanType) :: StructField("c2", BooleanType) :: Nil) + val nonNullDF = spark.createDataFrame( + spark.sparkContext.parallelize( + Seq(Row(false, false), Row(false, true), Row(true, false), Row(true, true))), + schema) + val expected = + Row(false, false) :: Row(true, false) :: Row(false, true) :: Row(true, true) :: Nil + checkSort(nonNullDF, expected, Array(BooleanType, BooleanType)) + val df = spark.createDataFrame( + spark.sparkContext.parallelize( + Seq(Row(false, false), Row(false, null), Row(null, false), Row(null, null))), + schema) + val expected2 = + Row(false, false) :: Row(null, false) :: Row(false, null) :: Row(null, null) :: Nil + checkSort(df, expected2, Array(BooleanType, BooleanType)) + } + + test("sort with zorder -- int column") { + // TODO: add more datatype unit test + val session = spark + import session.implicits._ + // generate 4 * 4 matrix + val len = 3 + val input = spark.range(len + 1).selectExpr("cast(id as int) as c1") + .select($"c1", explode(sequence(lit(0), lit(len))) as "c2") + val expected = + Row(0, 0) :: Row(1, 0) :: Row(0, 1) :: Row(1, 1) :: + Row(2, 0) :: Row(3, 0) :: Row(2, 1) :: Row(3, 1) :: + Row(0, 2) :: Row(1, 2) :: Row(0, 3) :: Row(1, 3) :: + Row(2, 2) :: Row(3, 2) :: Row(2, 3) :: Row(3, 3) :: Nil + checkSort(input, expected, Array(IntegerType, IntegerType)) + + // contains null value case. + val nullDF = spark.range(1).selectExpr("cast(null as int) as c1") + val input2 = spark.range(len).selectExpr("cast(id as int) as c1") + .union(nullDF) + .select( + $"c1", + explode(concat(sequence(lit(0), lit(len - 1)), array(lit(null)))) as "c2") + val expected2 = Row(0, 0) :: Row(1, 0) :: Row(0, 1) :: Row(1, 1) :: + Row(2, 0) :: Row(2, 1) :: Row(0, 2) :: Row(1, 2) :: + Row(2, 2) :: Row(null, 0) :: Row(null, 1) :: Row(null, 2) :: + Row(0, null) :: Row(1, null) :: Row(2, null) :: Row(null, null) :: Nil + checkSort(input2, expected2, Array(IntegerType, IntegerType)) + } + + test("sort with zorder -- string column") { + val schema = StructType(StructField("c1", StringType) :: StructField("c2", StringType) :: Nil) + val rdd = spark.sparkContext.parallelize(Seq( + Row("a", "a"), + Row("a", "b"), + Row("a", "c"), + Row("a", "d"), + Row("b", "a"), + Row("b", "b"), + Row("b", "c"), + Row("b", "d"), + Row("c", "a"), + Row("c", "b"), + Row("c", "c"), + Row("c", "d"), + Row("d", "a"), + Row("d", "b"), + Row("d", "c"), + Row("d", "d"))) + val input = spark.createDataFrame(rdd, schema) + val expected = Row("a", "a") :: Row("b", "a") :: Row("c", "a") :: Row("a", "b") :: + Row("a", "c") :: Row("b", "b") :: Row("c", "b") :: Row("b", "c") :: + Row("c", "c") :: Row("d", "a") :: Row("d", "b") :: Row("d", "c") :: + Row("a", "d") :: Row("b", "d") :: Row("c", "d") :: Row("d", "d") :: Nil + checkSort(input, expected, Array(StringType, StringType)) + + val rdd2 = spark.sparkContext.parallelize(Seq( + Row(null, "a"), + Row("a", "b"), + Row("a", "c"), + Row("a", null), + Row("b", "a"), + Row(null, "b"), + Row("b", null), + Row("b", "d"), + Row("c", "a"), + Row("c", null), + Row(null, "c"), + Row("c", "d"), + Row("d", null), + Row("d", "b"), + Row("d", "c"), + Row(null, "d"), + Row(null, null))) + val input2 = spark.createDataFrame(rdd2, schema) + val expected2 = Row("b", "a") :: Row("c", "a") :: Row("a", "b") :: Row("a", "c") :: + Row("d", "b") :: Row("d", "c") :: Row("b", "d") :: Row("c", "d") :: + Row(null, "a") :: Row(null, "b") :: Row(null, "c") :: Row(null, "d") :: + Row("a", null) :: Row("b", null) :: Row("c", null) :: Row("d", null) :: + Row(null, null) :: Nil + checkSort(input2, expected2, Array(StringType, StringType)) + } + + test("test special value of short int long type") { + val df1 = spark.createDataFrame(Seq( + (-1, -1L), + (Int.MinValue, Int.MinValue.toLong), + (1, 1L), + (Int.MaxValue - 1, Int.MaxValue.toLong), + (Int.MaxValue - 1, Int.MaxValue.toLong - 1), + (Int.MaxValue, Int.MaxValue.toLong + 1), + (Int.MaxValue, Int.MaxValue.toLong))).toDF("c1", "c2") + val expected1 = + Row(Int.MinValue, Int.MinValue.toLong) :: + Row(-1, -1L) :: + Row(1, 1L) :: + Row(Int.MaxValue - 1, Int.MaxValue.toLong - 1) :: + Row(Int.MaxValue - 1, Int.MaxValue.toLong) :: + Row(Int.MaxValue, Int.MaxValue.toLong) :: + Row(Int.MaxValue, Int.MaxValue.toLong + 1) :: Nil + checkSort(df1, expected1, Array(IntegerType, LongType)) + + val df2 = spark.createDataFrame(Seq( + (-1, -1.toShort), + (Short.MinValue.toInt, Short.MinValue), + (1, 1.toShort), + (Short.MaxValue.toInt, (Short.MaxValue - 1).toShort), + (Short.MaxValue.toInt + 1, (Short.MaxValue - 1).toShort), + (Short.MaxValue.toInt, Short.MaxValue), + (Short.MaxValue.toInt + 1, Short.MaxValue))).toDF("c1", "c2") + val expected2 = + Row(Short.MinValue.toInt, Short.MinValue) :: + Row(-1, -1.toShort) :: + Row(1, 1.toShort) :: + Row(Short.MaxValue.toInt, Short.MaxValue - 1) :: + Row(Short.MaxValue.toInt, Short.MaxValue) :: + Row(Short.MaxValue.toInt + 1, Short.MaxValue - 1) :: + Row(Short.MaxValue.toInt + 1, Short.MaxValue) :: Nil + checkSort(df2, expected2, Array(IntegerType, ShortType)) + + val df3 = spark.createDataFrame(Seq( + (-1L, -1.toShort), + (Short.MinValue.toLong, Short.MinValue), + (1L, 1.toShort), + (Short.MaxValue.toLong, (Short.MaxValue - 1).toShort), + (Short.MaxValue.toLong + 1, (Short.MaxValue - 1).toShort), + (Short.MaxValue.toLong, Short.MaxValue), + (Short.MaxValue.toLong + 1, Short.MaxValue))).toDF("c1", "c2") + val expected3 = + Row(Short.MinValue.toLong, Short.MinValue) :: + Row(-1L, -1.toShort) :: + Row(1L, 1.toShort) :: + Row(Short.MaxValue.toLong, Short.MaxValue - 1) :: + Row(Short.MaxValue.toLong, Short.MaxValue) :: + Row(Short.MaxValue.toLong + 1, Short.MaxValue - 1) :: + Row(Short.MaxValue.toLong + 1, Short.MaxValue) :: Nil + checkSort(df3, expected3, Array(LongType, ShortType)) + } + + test("skip zorder if only requires one column") { + withTable("t") { + withSQLConf("spark.sql.hive.convertMetastoreParquet" -> "false") { + sql("CREATE TABLE t (c1 int, c2 string) stored as parquet") + val order1 = sql("OPTIMIZE t ZORDER BY c1").queryExecution.analyzed + .asInstanceOf[OptimizeZorderCommandBase].query.asInstanceOf[Sort].order.head.child + assert(!order1.isInstanceOf[Zorder]) + assert(order1.isInstanceOf[AttributeReference]) + } + } + } + + test("Add config to control if zorder using global sort") { + withTable("t") { + withSQLConf(KyuubiSQLConf.ZORDER_GLOBAL_SORT_ENABLED.key -> "false") { + sql( + """ + |CREATE TABLE t (c1 int, c2 string) TBLPROPERTIES ( + |'kyuubi.zorder.enabled'= 'true', + |'kyuubi.zorder.cols'= 'c1,c2') + |""".stripMargin) + val p1 = sql("OPTIMIZE t ZORDER BY c1, c2").queryExecution.analyzed + assert(p1.collect { + case shuffle: Sort if !shuffle.global => shuffle + }.size == 1) + + val p2 = sql("INSERT INTO TABLE t SELECT * FROM VALUES(1,'a')").queryExecution.analyzed + assert(p2.collect { + case shuffle: Sort if !shuffle.global => shuffle + }.size == 1) + } + } + } + + test("fast approach test") { + Seq[Seq[Any]]( + Seq(1L, 2L), + Seq(1L, 2L, 3L), + Seq(1L, 2L, 3L, 4L), + Seq(1L, 2L, 3L, 4L, 5L), + Seq(1L, 2L, 3L, 4L, 5L, 6L), + Seq(1L, 2L, 3L, 4L, 5L, 6L, 7L), + Seq(1L, 2L, 3L, 4L, 5L, 6L, 7L, 8L)) + .foreach { inputs => + assert(java.util.Arrays.equals( + ZorderBytesUtils.interleaveBits(inputs.toArray), + ZorderBytesUtils.interleaveBitsDefault(inputs.map(ZorderBytesUtils.toByteArray).toArray))) + } + } + + test("OPTIMIZE command is parsed as expected") { + val parser = createParser + val globalSort = spark.conf.get(KyuubiSQLConf.ZORDER_GLOBAL_SORT_ENABLED) + + assert(parser.parsePlan("OPTIMIZE p zorder by c1") === + OptimizeZorderStatement( + Seq("p"), + Sort( + SortOrder(UnresolvedAttribute("c1"), Ascending, NullsLast, Seq.empty) :: Nil, + globalSort, + Project(Seq(UnresolvedStar(None)), UnresolvedRelation(TableIdentifier("p")))))) + + assert(parser.parsePlan("OPTIMIZE p zorder by c1, c2") === + OptimizeZorderStatement( + Seq("p"), + Sort( + SortOrder( + Zorder(Seq(UnresolvedAttribute("c1"), UnresolvedAttribute("c2"))), + Ascending, + NullsLast, + Seq.empty) :: Nil, + globalSort, + Project(Seq(UnresolvedStar(None)), UnresolvedRelation(TableIdentifier("p")))))) + + assert(parser.parsePlan("OPTIMIZE p where id = 1 zorder by c1") === + OptimizeZorderStatement( + Seq("p"), + Sort( + SortOrder(UnresolvedAttribute("c1"), Ascending, NullsLast, Seq.empty) :: Nil, + globalSort, + Project( + Seq(UnresolvedStar(None)), + Filter( + EqualTo(UnresolvedAttribute("id"), Literal(1)), + UnresolvedRelation(TableIdentifier("p"))))))) + + assert(parser.parsePlan("OPTIMIZE p where id = 1 zorder by c1, c2") === + OptimizeZorderStatement( + Seq("p"), + Sort( + SortOrder( + Zorder(Seq(UnresolvedAttribute("c1"), UnresolvedAttribute("c2"))), + Ascending, + NullsLast, + Seq.empty) :: Nil, + globalSort, + Project( + Seq(UnresolvedStar(None)), + Filter( + EqualTo(UnresolvedAttribute("id"), Literal(1)), + UnresolvedRelation(TableIdentifier("p"))))))) + + assert(parser.parsePlan("OPTIMIZE p where id = current_date() zorder by c1") === + OptimizeZorderStatement( + Seq("p"), + Sort( + SortOrder(UnresolvedAttribute("c1"), Ascending, NullsLast, Seq.empty) :: Nil, + globalSort, + Project( + Seq(UnresolvedStar(None)), + Filter( + EqualTo( + UnresolvedAttribute("id"), + UnresolvedFunction("current_date", Seq.empty, false)), + UnresolvedRelation(TableIdentifier("p"))))))) + + // TODO: add following case support + intercept[ParseException] { + parser.parsePlan("OPTIMIZE p zorder by (c1)") + } + + intercept[ParseException] { + parser.parsePlan("OPTIMIZE p zorder by (c1, c2)") + } + } + + test("OPTIMIZE partition predicates constraint") { + withTable("p") { + sql("CREATE TABLE p (c1 INT, c2 INT) PARTITIONED BY (event_date DATE)") + val e1 = intercept[KyuubiSQLExtensionException] { + sql("OPTIMIZE p WHERE event_date = current_date as c ZORDER BY c1, c2") + } + assert(e1.getMessage.contains("unsupported partition predicates")) + + val e2 = intercept[KyuubiSQLExtensionException] { + sql("OPTIMIZE p WHERE c1 = 1 ZORDER BY c1, c2") + } + assert(e2.getMessage == "Only partition column filters are allowed") + } + } + + def createParser: ParserInterface +} + +trait ZorderWithCodegenEnabledSuiteBase extends ZorderSuiteBase { + override def sparkConf(): SparkConf = { + val conf = super.sparkConf + conf.set(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key, "true") + conf + } +} + +trait ZorderWithCodegenDisabledSuiteBase extends ZorderSuiteBase { + override def sparkConf(): SparkConf = { + val conf = super.sparkConf + conf.set(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key, "false") + conf.set(SQLConf.CODEGEN_FACTORY_MODE.key, "NO_CODEGEN") + conf + } +} diff --git a/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/benchmark/KyuubiBenchmarkBase.scala b/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/benchmark/KyuubiBenchmarkBase.scala new file mode 100644 index 000000000..b891a7224 --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-3-5/src/test/scala/org/apache/spark/sql/benchmark/KyuubiBenchmarkBase.scala @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.benchmark + +import java.io.{File, FileOutputStream, OutputStream} + +import scala.collection.JavaConverters._ + +import com.google.common.reflect.ClassPath +import org.scalatest.Assertions._ + +trait KyuubiBenchmarkBase { + var output: Option[OutputStream] = None + + private val prefix = { + val benchmarkClasses = ClassPath.from(Thread.currentThread.getContextClassLoader) + .getTopLevelClassesRecursive("org.apache.spark.sql").asScala.toArray + assert(benchmarkClasses.nonEmpty) + val benchmark = benchmarkClasses.find(_.load().getName.endsWith("Benchmark")) + val targetDirOrProjDir = + new File(benchmark.get.load().getProtectionDomain.getCodeSource.getLocation.toURI) + .getParentFile.getParentFile + if (targetDirOrProjDir.getName == "target") { + targetDirOrProjDir.getParentFile.getCanonicalPath + "/" + } else { + targetDirOrProjDir.getCanonicalPath + "/" + } + } + + def withHeader(func: => Unit): Unit = { + val version = System.getProperty("java.version").split("\\D+")(0).toInt + val jdkString = if (version > 8) s"-jdk$version" else "" + val resultFileName = + s"${this.getClass.getSimpleName.replace("$", "")}$jdkString-results.txt" + val dir = new File(s"${prefix}benchmarks/") + if (!dir.exists()) { + // scalastyle:off println + println(s"Creating ${dir.getAbsolutePath} for benchmark results.") + // scalastyle:on println + dir.mkdirs() + } + val file = new File(dir, resultFileName) + if (!file.exists()) { + file.createNewFile() + } + output = Some(new FileOutputStream(file)) + + func + + output.foreach { o => + if (o != null) { + o.close() + } + } + } +} diff --git a/extensions/spark/kyuubi-extension-spark-common/pom.xml b/extensions/spark/kyuubi-extension-spark-common/pom.xml index 2c587fd78..259931a2e 100644 --- a/extensions/spark/kyuubi-extension-spark-common/pom.xml +++ b/extensions/spark/kyuubi-extension-spark-common/pom.xml @@ -21,11 +21,11 @@ org.apache.kyuubi kyuubi-parent - 1.7.0-SNAPSHOT + 1.9.0-SNAPSHOT ../../../pom.xml - kyuubi-extension-spark-common_2.12 + kyuubi-extension-spark-common_${scala.binary.version} jar Kyuubi Dev Spark Extensions Common (for Spark 3) https://kyuubi.apache.org/ @@ -110,10 +110,21 @@ jakarta.xml.bind-api test + + + org.apache.logging.log4j + log4j-1.2-api + test + + + + org.apache.logging.log4j + log4j-slf4j-impl + test + - org.antlr diff --git a/extensions/spark/kyuubi-extension-spark-common/src/main/antlr4/org/apache/kyuubi/sql/KyuubiSparkSQL.g4 b/extensions/spark/kyuubi-extension-spark-common/src/main/antlr4/org/apache/kyuubi/sql/KyuubiSparkSQL.g4 index 63e2bf848..e52b7f5cf 100644 --- a/extensions/spark/kyuubi-extension-spark-common/src/main/antlr4/org/apache/kyuubi/sql/KyuubiSparkSQL.g4 +++ b/extensions/spark/kyuubi-extension-spark-common/src/main/antlr4/org/apache/kyuubi/sql/KyuubiSparkSQL.g4 @@ -55,53 +55,23 @@ statement ; whereClause - : WHERE booleanExpression + : WHERE partitionPredicate = predicateToken ; zorderClause : ZORDER BY order+=multipartIdentifier (',' order+=multipartIdentifier)* ; -booleanExpression - : query #logicalQuery - | left=booleanExpression operator=AND right=booleanExpression #logicalBinary - | left=booleanExpression operator=OR right=booleanExpression #logicalBinary - ; - -query - : '('? multipartIdentifier comparisonOperator constant ')'? - ; - -comparisonOperator - : EQ | NEQ | NEQJ | LT | LTE | GT | GTE | NSEQ - ; - -constant - : NULL #nullLiteral - | identifier STRING #typeConstructor - | number #numericLiteral - | booleanValue #booleanLiteral - | STRING+ #stringLiteral +// We don't have an expression rule in our grammar here, so we just grab the tokens and defer +// parsing them to later. +predicateToken + : .+? ; multipartIdentifier : parts+=identifier ('.' parts+=identifier)* ; -booleanValue - : TRUE | FALSE - ; - -number - : MINUS? DECIMAL_VALUE #decimalLiteral - | MINUS? INTEGER_VALUE #integerLiteral - | MINUS? BIGINT_LITERAL #bigIntLiteral - | MINUS? SMALLINT_LITERAL #smallIntLiteral - | MINUS? TINYINT_LITERAL #tinyIntLiteral - | MINUS? DOUBLE_LITERAL #doubleLiteral - | MINUS? BIGDECIMAL_LITERAL #bigDecimalLiteral - ; - identifier : strictIdentifier ; @@ -136,7 +106,6 @@ BY: 'BY'; FALSE: 'FALSE'; DATE: 'DATE'; INTERVAL: 'INTERVAL'; -NULL: 'NULL'; OPTIMIZE: 'OPTIMIZE'; OR: 'OR'; TABLE: 'TABLE'; @@ -145,22 +114,8 @@ TRUE: 'TRUE'; WHERE: 'WHERE'; ZORDER: 'ZORDER'; -EQ : '=' | '=='; -NSEQ: '<=>'; -NEQ : '<>'; -NEQJ: '!='; -LT : '<'; -LTE : '<=' | '!>'; -GT : '>'; -GTE : '>=' | '!<'; - MINUS: '-'; -STRING - : '\'' ( ~('\''|'\\') | ('\\' .) )* '\'' - | '"' ( ~('"'|'\\') | ('\\' .) )* '"' - ; - BIGINT_LITERAL : DIGIT+ 'L' ; diff --git a/extensions/spark/kyuubi-extension-spark-common/src/main/scala/org/apache/kyuubi/sql/KyuubiQueryStagePreparation.scala b/extensions/spark/kyuubi-extension-spark-common/src/main/scala/org/apache/kyuubi/sql/KyuubiQueryStagePreparation.scala index 360a2645e..fee65b350 100644 --- a/extensions/spark/kyuubi-extension-spark-common/src/main/scala/org/apache/kyuubi/sql/KyuubiQueryStagePreparation.scala +++ b/extensions/spark/kyuubi-extension-spark-common/src/main/scala/org/apache/kyuubi/sql/KyuubiQueryStagePreparation.scala @@ -133,7 +133,9 @@ case class FinalStageConfigIsolation(session: SparkSession) extends Rule[SparkPl reusedExchangeExec // query stage is leaf node so we need to transform it manually - case queryStage: QueryStageExec => + // compatible with Spark 3.5: + // SPARK-42101: table cache is a independent query stage, so do not need include it. + case queryStage: QueryStageExec if queryStage.nodeName != "TableCacheQueryStage" => queryStageNum += 1 collectNumber(queryStage.plan) queryStage diff --git a/extensions/spark/kyuubi-extension-spark-common/src/main/scala/org/apache/kyuubi/sql/KyuubiSQLConf.scala b/extensions/spark/kyuubi-extension-spark-common/src/main/scala/org/apache/kyuubi/sql/KyuubiSQLConf.scala index 0fe9f649e..6f45dae12 100644 --- a/extensions/spark/kyuubi-extension-spark-common/src/main/scala/org/apache/kyuubi/sql/KyuubiSQLConf.scala +++ b/extensions/spark/kyuubi-extension-spark-common/src/main/scala/org/apache/kyuubi/sql/KyuubiSQLConf.scala @@ -17,6 +17,7 @@ package org.apache.kyuubi.sql +import org.apache.spark.network.util.ByteUnit import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.internal.SQLConf._ @@ -33,7 +34,8 @@ object KyuubiSQLConf { buildConf("spark.sql.optimizer.insertRepartitionNum") .doc(s"The partition number if ${INSERT_REPARTITION_BEFORE_WRITE.key} is enabled. " + s"If AQE is disabled, the default value is ${SQLConf.SHUFFLE_PARTITIONS.key}. " + - "If AQE is enabled, the default value is none that means depend on AQE.") + "If AQE is enabled, the default value is none that means depend on AQE. " + + "This config is used for Spark 3.1 only.") .version("1.2.0") .intConf .createOptional @@ -138,13 +140,23 @@ object KyuubiSQLConf { val WATCHDOG_MAX_PARTITIONS = buildConf("spark.sql.watchdog.maxPartitions") .doc("Set the max partition number when spark scans a data source. " + - "Enable MaxPartitionStrategy by specifying this configuration. " + + "Enable maxPartitions Strategy by specifying this configuration. " + "Add maxPartitions Strategy to avoid scan excessive partitions " + "on partitioned table, it's optional that works with defined") .version("1.4.0") .intConf .createOptional + val WATCHDOG_MAX_FILE_SIZE = + buildConf("spark.sql.watchdog.maxFileSize") + .doc("Set the maximum size in bytes of files when spark scans a data source. " + + "Enable maxFileSize Strategy by specifying this configuration. " + + "Add maxFileSize Strategy to avoid scan excessive size of files," + + " it's optional that works with defined") + .version("1.8.0") + .bytesConf(ByteUnit.BYTE) + .createOptional + val WATCHDOG_FORCED_MAXOUTPUTROWS = buildConf("spark.sql.watchdog.forcedMaxOutputRows") .doc("Add ForcedMaxOutputRows rule to avoid huge output rows of non-limit query " + @@ -190,4 +202,75 @@ object KyuubiSQLConf { .version("1.7.0") .booleanConf .createWithDefault(true) + + val FINAL_WRITE_STAGE_EAGERLY_KILL_EXECUTORS_ENABLED = + buildConf("spark.sql.finalWriteStage.eagerlyKillExecutors.enabled") + .doc("When true, eagerly kill redundant executors before running final write stage.") + .version("1.8.0") + .booleanConf + .createWithDefault(false) + + val FINAL_WRITE_STAGE_EAGERLY_KILL_EXECUTORS_KILL_ALL = + buildConf("spark.sql.finalWriteStage.eagerlyKillExecutors.killAll") + .doc("When true, eagerly kill all executors before running final write stage. " + + "Mainly for test.") + .version("1.8.0") + .booleanConf + .createWithDefault(false) + + val FINAL_WRITE_STAGE_SKIP_KILLING_EXECUTORS_FOR_TABLE_CACHE = + buildConf("spark.sql.finalWriteStage.skipKillingExecutorsForTableCache") + .doc("When true, skip killing executors if the plan has table caches.") + .version("1.8.0") + .booleanConf + .createWithDefault(true) + + val FINAL_WRITE_STAGE_PARTITION_FACTOR = + buildConf("spark.sql.finalWriteStage.retainExecutorsFactor") + .doc("If the target executors * factor < active executors, and " + + "target executors * factor > min executors, then kill redundant executors.") + .version("1.8.0") + .doubleConf + .checkValue(_ >= 1, "must be bigger than or equal to 1") + .createWithDefault(1.2) + + val FINAL_WRITE_STAGE_RESOURCE_ISOLATION_ENABLED = + buildConf("spark.sql.finalWriteStage.resourceIsolation.enabled") + .doc( + "When true, make final write stage resource isolation using custom RDD resource profile.") + .version("1.8.0") + .booleanConf + .createWithDefault(false) + + val FINAL_WRITE_STAGE_EXECUTOR_CORES = + buildConf("spark.sql.finalWriteStage.executorCores") + .doc("Specify the executor core request for final write stage. " + + "It would be passed to the RDD resource profile.") + .version("1.8.0") + .intConf + .createOptional + + val FINAL_WRITE_STAGE_EXECUTOR_MEMORY = + buildConf("spark.sql.finalWriteStage.executorMemory") + .doc("Specify the executor on heap memory request for final write stage. " + + "It would be passed to the RDD resource profile.") + .version("1.8.0") + .stringConf + .createOptional + + val FINAL_WRITE_STAGE_EXECUTOR_MEMORY_OVERHEAD = + buildConf("spark.sql.finalWriteStage.executorMemoryOverhead") + .doc("Specify the executor memory overhead request for final write stage. " + + "It would be passed to the RDD resource profile.") + .version("1.8.0") + .stringConf + .createOptional + + val FINAL_WRITE_STAGE_EXECUTOR_OFF_HEAP_MEMORY = + buildConf("spark.sql.finalWriteStage.executorOffHeapMemory") + .doc("Specify the executor off heap memory request for final write stage. " + + "It would be passed to the RDD resource profile.") + .version("1.8.0") + .stringConf + .createOptional } diff --git a/extensions/spark/kyuubi-extension-spark-common/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLAstBuilder.scala b/extensions/spark/kyuubi-extension-spark-common/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLAstBuilder.scala index 9f1958b09..cc00bf88e 100644 --- a/extensions/spark/kyuubi-extension-spark-common/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLAstBuilder.scala +++ b/extensions/spark/kyuubi-extension-spark-common/src/main/scala/org/apache/kyuubi/sql/KyuubiSparkSQLAstBuilder.scala @@ -17,37 +17,81 @@ package org.apache.kyuubi.sql -import java.time.LocalDate -import java.util.Locale - import scala.collection.JavaConverters.asScalaBufferConverter -import scala.collection.mutable.{ArrayBuffer, ListBuffer} -import scala.util.control.NonFatal +import scala.collection.mutable.ListBuffer import org.antlr.v4.runtime.ParserRuleContext -import org.antlr.v4.runtime.tree.{ParseTree, TerminalNode} -import org.apache.commons.codec.binary.Hex -import org.apache.spark.sql.AnalysisException +import org.antlr.v4.runtime.misc.Interval +import org.antlr.v4.runtime.tree.ParseTree +import org.apache.spark.sql.catalyst.SQLConfHelper import org.apache.spark.sql.catalyst.analysis.{UnresolvedAttribute, UnresolvedRelation, UnresolvedStar} import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.catalyst.parser.ParseException -import org.apache.spark.sql.catalyst.parser.ParserUtils.{string, stringWithoutUnescape, withOrigin} +import org.apache.spark.sql.catalyst.parser.ParserUtils.withOrigin import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, Project, Sort} -import org.apache.spark.sql.catalyst.util.DateTimeUtils.{getZoneId, localDateToDays, stringToTimestamp} -import org.apache.spark.sql.catalyst.util.IntervalUtils -import org.apache.spark.sql.hive.HiveAnalysis.conf -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types._ -import org.apache.spark.unsafe.types.UTF8String import org.apache.kyuubi.sql.KyuubiSparkSQLParser._ -import org.apache.kyuubi.sql.zorder.{OptimizeZorderStatement, OptimizeZorderStatementBase, Zorder, ZorderBase} +import org.apache.kyuubi.sql.zorder.{OptimizeZorderStatement, Zorder} + +class KyuubiSparkSQLAstBuilder extends KyuubiSparkSQLBaseVisitor[AnyRef] with SQLConfHelper { + + def buildOptimizeStatement( + unparsedPredicateOptimize: UnparsedPredicateOptimize, + parseExpression: String => Expression): LogicalPlan = { -abstract class KyuubiSparkSQLAstBuilderBase extends KyuubiSparkSQLBaseVisitor[AnyRef] { - def buildZorder(child: Seq[Expression]): ZorderBase - def buildOptimizeZorderStatement( - tableIdentifier: Seq[String], - query: LogicalPlan): OptimizeZorderStatementBase + val UnparsedPredicateOptimize(tableIdent, tablePredicate, orderExpr) = + unparsedPredicateOptimize + + val predicate = tablePredicate.map(parseExpression) + verifyPartitionPredicates(predicate) + val table = UnresolvedRelation(tableIdent) + val tableWithFilter = predicate match { + case Some(expr) => Filter(expr, table) + case None => table + } + val query = + Sort( + SortOrder(orderExpr, Ascending, NullsLast, Seq.empty) :: Nil, + conf.getConf(KyuubiSQLConf.ZORDER_GLOBAL_SORT_ENABLED), + Project(Seq(UnresolvedStar(None)), tableWithFilter)) + OptimizeZorderStatement(tableIdent, query) + } + + private def verifyPartitionPredicates(predicates: Option[Expression]): Unit = { + predicates.foreach { + case p if !isLikelySelective(p) => + throw new KyuubiSQLExtensionException(s"unsupported partition predicates: ${p.sql}") + case _ => + } + } + + /** + * Forked from Apache Spark's org.apache.spark.sql.catalyst.expressions.PredicateHelper + * The `PredicateHelper.isLikelySelective()` is available since Spark-3.3, forked for Spark + * that is lower than 3.3. + * + * Returns whether an expression is likely to be selective + */ + private def isLikelySelective(e: Expression): Boolean = e match { + case Not(expr) => isLikelySelective(expr) + case And(l, r) => isLikelySelective(l) || isLikelySelective(r) + case Or(l, r) => isLikelySelective(l) && isLikelySelective(r) + case _: StringRegexExpression => true + case _: BinaryComparison => true + case _: In | _: InSet => true + case _: StringPredicate => true + case BinaryPredicate(_) => true + case _: MultiLikeBase => true + case _ => false + } + + private object BinaryPredicate { + def unapply(expr: Expression): Option[Expression] = expr match { + case _: Contains => Option(expr) + case _: StartsWith => Option(expr) + case _: EndsWith => Option(expr) + case _ => None + } + } /** * Create an expression from the given context. This method just passes the context on to the @@ -62,21 +106,12 @@ abstract class KyuubiSparkSQLAstBuilderBase extends KyuubiSparkSQLBaseVisitor[An } override def visitOptimizeZorder( - ctx: OptimizeZorderContext): LogicalPlan = withOrigin(ctx) { + ctx: OptimizeZorderContext): UnparsedPredicateOptimize = withOrigin(ctx) { val tableIdent = multiPart(ctx.multipartIdentifier()) - val table = UnresolvedRelation(tableIdent) - - val whereClause = - if (ctx.whereClause() == null) { - None - } else { - Option(expression(ctx.whereClause().booleanExpression())) - } - val tableWithFilter = whereClause match { - case Some(expr) => Filter(expr, table) - case None => table - } + val predicate = Option(ctx.whereClause()) + .map(_.partitionPredicate) + .map(extractRawText(_)) val zorderCols = ctx.zorderClause().order.asScala .map(visitMultipartIdentifier) @@ -87,364 +122,53 @@ abstract class KyuubiSparkSQLAstBuilderBase extends KyuubiSparkSQLBaseVisitor[An if (zorderCols.length == 1) { zorderCols.head } else { - buildZorder(zorderCols) + Zorder(zorderCols) } - val query = - Sort( - SortOrder(orderExpr, Ascending, NullsLast, Seq.empty) :: Nil, - conf.getConf(KyuubiSQLConf.ZORDER_GLOBAL_SORT_ENABLED), - Project(Seq(UnresolvedStar(None)), tableWithFilter)) - - buildOptimizeZorderStatement(tableIdent, query) + UnparsedPredicateOptimize(tableIdent, predicate, orderExpr) } override def visitPassThrough(ctx: PassThroughContext): LogicalPlan = null - override def visitQuery(ctx: QueryContext): Expression = withOrigin(ctx) { - val left = new UnresolvedAttribute(multiPart(ctx.multipartIdentifier())) - val right = expression(ctx.constant()) - val operator = ctx.comparisonOperator().getChild(0).asInstanceOf[TerminalNode] - operator.getSymbol.getType match { - case KyuubiSparkSQLParser.EQ => - EqualTo(left, right) - case KyuubiSparkSQLParser.NSEQ => - EqualNullSafe(left, right) - case KyuubiSparkSQLParser.NEQ | KyuubiSparkSQLParser.NEQJ => - Not(EqualTo(left, right)) - case KyuubiSparkSQLParser.LT => - LessThan(left, right) - case KyuubiSparkSQLParser.LTE => - LessThanOrEqual(left, right) - case KyuubiSparkSQLParser.GT => - GreaterThan(left, right) - case KyuubiSparkSQLParser.GTE => - GreaterThanOrEqual(left, right) - } - } - - override def visitLogicalBinary(ctx: LogicalBinaryContext): Expression = withOrigin(ctx) { - val expressionType = ctx.operator.getType - val expressionCombiner = expressionType match { - case KyuubiSparkSQLParser.AND => And.apply _ - case KyuubiSparkSQLParser.OR => Or.apply _ - } - - // Collect all similar left hand contexts. - val contexts = ArrayBuffer(ctx.right) - var current = ctx.left - def collectContexts: Boolean = current match { - case lbc: LogicalBinaryContext if lbc.operator.getType == expressionType => - contexts += lbc.right - current = lbc.left - true - case _ => - contexts += current - false - } - while (collectContexts) { - // No body - all updates take place in the collectContexts. - } - - // Reverse the contexts to have them in the same sequence as in the SQL statement & turn them - // into expressions. - val expressions = contexts.reverseMap(expression) - - // Create a balanced tree. - def reduceToExpressionTree(low: Int, high: Int): Expression = high - low match { - case 0 => - expressions(low) - case 1 => - expressionCombiner(expressions(low), expressions(high)) - case x => - val mid = low + x / 2 - expressionCombiner( - reduceToExpressionTree(low, mid), - reduceToExpressionTree(mid + 1, high)) - } - reduceToExpressionTree(0, expressions.size - 1) - } - override def visitMultipartIdentifier(ctx: MultipartIdentifierContext): Seq[String] = withOrigin(ctx) { - ctx.parts.asScala.map(_.getText) + ctx.parts.asScala.map(_.getText).toSeq } override def visitZorderClause(ctx: ZorderClauseContext): Seq[UnresolvedAttribute] = withOrigin(ctx) { val res = ListBuffer[UnresolvedAttribute]() ctx.multipartIdentifier().forEach { identifier => - res += UnresolvedAttribute(identifier.parts.asScala.map(_.getText)) + res += UnresolvedAttribute(identifier.parts.asScala.map(_.getText).toSeq) } - res - } - - /** - * Create a NULL literal expression. - */ - override def visitNullLiteral(ctx: NullLiteralContext): Literal = withOrigin(ctx) { - Literal(null) - } - - /** - * Create a Boolean literal expression. - */ - override def visitBooleanLiteral(ctx: BooleanLiteralContext): Literal = withOrigin(ctx) { - if (ctx.getText.toBoolean) { - Literal.TrueLiteral - } else { - Literal.FalseLiteral + res.toSeq } - } - - /** - * Create a typed Literal expression. A typed literal has the following SQL syntax: - * {{{ - * [TYPE] '[VALUE]' - * }}} - * Currently Date, Timestamp, Interval and Binary typed literals are supported. - */ - override def visitTypeConstructor(ctx: TypeConstructorContext): Literal = withOrigin(ctx) { - val value = string(ctx.STRING) - val valueType = ctx.identifier.getText.toUpperCase(Locale.ROOT) - - def toLiteral[T](f: UTF8String => Option[T], t: DataType): Literal = { - f(UTF8String.fromString(value)).map(Literal(_, t)).getOrElse { - throw new ParseException(s"Cannot parse the $valueType value: $value", ctx) - } - } - try { - valueType match { - case "DATE" => - toLiteral(stringToDate, DateType) - case "TIMESTAMP" => - val zoneId = getZoneId(SQLConf.get.sessionLocalTimeZone) - toLiteral(stringToTimestamp(_, zoneId), TimestampType) - case "INTERVAL" => - val interval = - try { - IntervalUtils.stringToInterval(UTF8String.fromString(value)) - } catch { - case e: IllegalArgumentException => - val ex = new ParseException("Cannot parse the INTERVAL value: " + value, ctx) - ex.setStackTrace(e.getStackTrace) - throw ex - } - Literal(interval, CalendarIntervalType) - case "X" => - val padding = if (value.length % 2 != 0) "0" else "" - - Literal(Hex.decodeHex(padding + value)) - case other => - throw new ParseException(s"Literals of type '$other' are currently not supported.", ctx) - } - } catch { - case e: IllegalArgumentException => - val message = Option(e.getMessage).getOrElse(s"Exception parsing $valueType") - throw new ParseException(message, ctx) - } - } - - /** - * Create a String literal expression. - */ - override def visitStringLiteral(ctx: StringLiteralContext): Literal = withOrigin(ctx) { - Literal(createString(ctx)) - } - - /** - * Create a decimal literal for a regular decimal number. - */ - override def visitDecimalLiteral(ctx: DecimalLiteralContext): Literal = withOrigin(ctx) { - Literal(BigDecimal(ctx.getText).underlying()) - } - - /** Create a numeric literal expression. */ - private def numericLiteral( - ctx: NumberContext, - rawStrippedQualifier: String, - minValue: BigDecimal, - maxValue: BigDecimal, - typeName: String)(converter: String => Any): Literal = withOrigin(ctx) { - try { - val rawBigDecimal = BigDecimal(rawStrippedQualifier) - if (rawBigDecimal < minValue || rawBigDecimal > maxValue) { - throw new ParseException( - s"Numeric literal ${rawStrippedQualifier} does not " + - s"fit in range [${minValue}, ${maxValue}] for type ${typeName}", - ctx) - } - Literal(converter(rawStrippedQualifier)) - } catch { - case e: NumberFormatException => - throw new ParseException(e.getMessage, ctx) - } - } - - /** - * Create a Byte Literal expression. - */ - override def visitTinyIntLiteral(ctx: TinyIntLiteralContext): Literal = { - val rawStrippedQualifier = ctx.getText.substring(0, ctx.getText.length - 1) - numericLiteral( - ctx, - rawStrippedQualifier, - Byte.MinValue, - Byte.MaxValue, - ByteType.simpleString)(_.toByte) - } - - /** - * Create an integral literal expression. The code selects the most narrow integral type - * possible, either a BigDecimal, a Long or an Integer is returned. - */ - override def visitIntegerLiteral(ctx: IntegerLiteralContext): Literal = withOrigin(ctx) { - BigDecimal(ctx.getText) match { - case v if v.isValidInt => - Literal(v.intValue) - case v if v.isValidLong => - Literal(v.longValue) - case v => Literal(v.underlying()) - } - } - - /** - * Create a Short Literal expression. - */ - override def visitSmallIntLiteral(ctx: SmallIntLiteralContext): Literal = { - val rawStrippedQualifier = ctx.getText.substring(0, ctx.getText.length - 1) - numericLiteral( - ctx, - rawStrippedQualifier, - Short.MinValue, - Short.MaxValue, - ShortType.simpleString)(_.toShort) - } - - /** - * Create a Long Literal expression. - */ - override def visitBigIntLiteral(ctx: BigIntLiteralContext): Literal = { - val rawStrippedQualifier = ctx.getText.substring(0, ctx.getText.length - 1) - numericLiteral( - ctx, - rawStrippedQualifier, - Long.MinValue, - Long.MaxValue, - LongType.simpleString)(_.toLong) - } - - /** - * Create a Double Literal expression. - */ - override def visitDoubleLiteral(ctx: DoubleLiteralContext): Literal = { - val rawStrippedQualifier = ctx.getText.substring(0, ctx.getText.length - 1) - numericLiteral( - ctx, - rawStrippedQualifier, - Double.MinValue, - Double.MaxValue, - DoubleType.simpleString)(_.toDouble) - } - - /** - * Create a BigDecimal Literal expression. - */ - override def visitBigDecimalLiteral(ctx: BigDecimalLiteralContext): Literal = { - val raw = ctx.getText.substring(0, ctx.getText.length - 2) - try { - Literal(BigDecimal(raw).underlying()) - } catch { - case e: AnalysisException => - throw new ParseException(e.message, ctx) - } - } - - /** - * Create a String from a string literal context. This supports multiple consecutive string - * literals, these are concatenated, for example this expression "'hello' 'world'" will be - * converted into "helloworld". - * - * Special characters can be escaped by using Hive/C-style escaping. - */ - private def createString(ctx: StringLiteralContext): String = { - if (conf.escapedStringLiterals) { - ctx.STRING().asScala.map(stringWithoutUnescape).mkString - } else { - ctx.STRING().asScala.map(string).mkString - } - } private def typedVisit[T](ctx: ParseTree): T = { ctx.accept(this).asInstanceOf[T] } - private def stringToDate(s: UTF8String): Option[Int] = { - def isValidDigits(segment: Int, digits: Int): Boolean = { - // An integer is able to represent a date within [+-]5 million years. - var maxDigitsYear = 7 - (segment == 0 && digits >= 4 && digits <= maxDigitsYear) || - (segment != 0 && digits > 0 && digits <= 2) - } - if (s == null || s.trimAll().numBytes() == 0) { - return None - } - val segments: Array[Int] = Array[Int](1, 1, 1) - var sign = 1 - var i = 0 - var currentSegmentValue = 0 - var currentSegmentDigits = 0 - val bytes = s.trimAll().getBytes - var j = 0 - if (bytes(j) == '-' || bytes(j) == '+') { - sign = if (bytes(j) == '-') -1 else 1 - j += 1 - } - while (j < bytes.length && (i < 3 && !(bytes(j) == ' ' || bytes(j) == 'T'))) { - val b = bytes(j) - if (i < 2 && b == '-') { - if (!isValidDigits(i, currentSegmentDigits)) { - return None - } - segments(i) = currentSegmentValue - currentSegmentValue = 0 - currentSegmentDigits = 0 - i += 1 - } else { - val parsedValue = b - '0'.toByte - if (parsedValue < 0 || parsedValue > 9) { - return None - } else { - currentSegmentValue = currentSegmentValue * 10 + parsedValue - currentSegmentDigits += 1 - } - } - j += 1 - } - if (!isValidDigits(i, currentSegmentDigits)) { - return None - } - if (i < 2 && j < bytes.length) { - // For the `yyyy` and `yyyy-[m]m` formats, entire input must be consumed. - return None - } - segments(i) = currentSegmentValue - try { - val localDate = LocalDate.of(sign * segments(0), segments(1), segments(2)) - Some(localDateToDays(localDate)) - } catch { - case NonFatal(_) => None - } + private def extractRawText(exprContext: ParserRuleContext): String = { + // Extract the raw expression which will be parsed later + exprContext.getStart.getInputStream.getText(new Interval( + exprContext.getStart.getStartIndex, + exprContext.getStop.getStopIndex)) } } -class KyuubiSparkSQLAstBuilder extends KyuubiSparkSQLAstBuilderBase { - override def buildZorder(child: Seq[Expression]): ZorderBase = { - Zorder(child) - } +/** + * a logical plan contains an unparsed expression that will be parsed by spark. + */ +trait UnparsedExpressionLogicalPlan extends LogicalPlan { + override def output: Seq[Attribute] = throw new UnsupportedOperationException() - override def buildOptimizeZorderStatement( - tableIdentifier: Seq[String], - query: LogicalPlan): OptimizeZorderStatementBase = { - OptimizeZorderStatement(tableIdentifier, query) - } + override def children: Seq[LogicalPlan] = throw new UnsupportedOperationException() + + protected def withNewChildrenInternal( + newChildren: IndexedSeq[LogicalPlan]): LogicalPlan = + throw new UnsupportedOperationException() } + +case class UnparsedPredicateOptimize( + tableIdent: Seq[String], + tablePredicate: Option[String], + orderExpr: Expression) extends UnparsedExpressionLogicalPlan {} diff --git a/extensions/spark/kyuubi-extension-spark-common/src/main/scala/org/apache/kyuubi/sql/watchdog/KyuubiWatchDogException.scala b/extensions/spark/kyuubi-extension-spark-common/src/main/scala/org/apache/kyuubi/sql/watchdog/KyuubiWatchDogException.scala index b3c58afdf..e44309192 100644 --- a/extensions/spark/kyuubi-extension-spark-common/src/main/scala/org/apache/kyuubi/sql/watchdog/KyuubiWatchDogException.scala +++ b/extensions/spark/kyuubi-extension-spark-common/src/main/scala/org/apache/kyuubi/sql/watchdog/KyuubiWatchDogException.scala @@ -23,3 +23,8 @@ final class MaxPartitionExceedException( private val reason: String = "", private val cause: Throwable = None.orNull) extends KyuubiSQLExtensionException(reason, cause) + +final class MaxFileSizeExceedException( + private val reason: String = "", + private val cause: Throwable = None.orNull) + extends KyuubiSQLExtensionException(reason, cause) diff --git a/extensions/spark/kyuubi-extension-spark-common/src/main/scala/org/apache/kyuubi/sql/watchdog/MaxPartitionStrategy.scala b/extensions/spark/kyuubi-extension-spark-common/src/main/scala/org/apache/kyuubi/sql/watchdog/MaxPartitionStrategy.scala deleted file mode 100644 index 61ab07adf..000000000 --- a/extensions/spark/kyuubi-extension-spark-common/src/main/scala/org/apache/kyuubi/sql/watchdog/MaxPartitionStrategy.scala +++ /dev/null @@ -1,185 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kyuubi.sql.watchdog - -import org.apache.hadoop.fs.Path -import org.apache.spark.sql.{PruneFileSourcePartitionHelper, SparkSession, Strategy} -import org.apache.spark.sql.catalyst.SQLConfHelper -import org.apache.spark.sql.catalyst.catalog.{CatalogTable, HiveTableRelation} -import org.apache.spark.sql.catalyst.planning.ScanOperation -import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan -import org.apache.spark.sql.execution.SparkPlan -import org.apache.spark.sql.execution.datasources.{CatalogFileIndex, HadoopFsRelation, InMemoryFileIndex, LogicalRelation} -import org.apache.spark.sql.types.StructType - -import org.apache.kyuubi.sql.KyuubiSQLConf - -/** - * Add maxPartitions Strategy to avoid scan excessive partitions on partitioned table - * 1 Check if scan exceed maxPartition - * 2 Check if Using partitionFilter on partitioned table - * This Strategy Add Planner Strategy after LogicalOptimizer - */ -case class MaxPartitionStrategy(session: SparkSession) - extends Strategy - with SQLConfHelper - with PruneFileSourcePartitionHelper { - override def apply(plan: LogicalPlan): Seq[SparkPlan] = { - val maxScanPartitionsOpt = conf.getConf(KyuubiSQLConf.WATCHDOG_MAX_PARTITIONS) - - if (maxScanPartitionsOpt.isDefined) { - checkRelationMaxPartitions(plan, maxScanPartitionsOpt.get) - } - Nil - } - - private def checkRelationMaxPartitions( - plan: LogicalPlan, - maxScanPartitions: Int): Unit = { - plan match { - case ScanOperation(_, _, relation: HiveTableRelation) if relation.isPartitioned => - relation.prunedPartitions match { - case Some(prunedPartitions) => - if (prunedPartitions.size > maxScanPartitions) { - throw new MaxPartitionExceedException( - s""" - |SQL job scan hive partition: ${prunedPartitions.size} - |exceed restrict of hive scan maxPartition $maxScanPartitions - |You should optimize your SQL logical according partition structure - |or shorten query scope such as p_date, detail as below: - |Table: ${relation.tableMeta.qualifiedName} - |Owner: ${relation.tableMeta.owner} - |Partition Structure: ${relation.partitionCols.map(_.name).mkString(", ")} - |""".stripMargin) - } - case _ => - val totalPartitions = session - .sessionState.catalog.externalCatalog.listPartitionNames( - relation.tableMeta.database, - relation.tableMeta.identifier.table) - if (totalPartitions.size > maxScanPartitions) { - throw new MaxPartitionExceedException( - s""" - |Your SQL job scan a whole huge table without any partition filter, - |You should optimize your SQL logical according partition structure - |or shorten query scope such as p_date, detail as below: - |Table: ${relation.tableMeta.qualifiedName} - |Owner: ${relation.tableMeta.owner} - |Partition Structure: ${relation.partitionCols.map(_.name).mkString(", ")} - |""".stripMargin) - } - } - case ScanOperation( - _, - filters, - relation @ LogicalRelation( - fsRelation @ HadoopFsRelation( - fileIndex: InMemoryFileIndex, - partitionSchema, - _, - _, - _, - _), - _, - _, - _)) if fsRelation.partitionSchema.nonEmpty => - val (partitionKeyFilters, dataFilter) = - getPartitionKeyFiltersAndDataFilters( - fsRelation.sparkSession, - relation, - partitionSchema, - filters, - relation.output) - val prunedPartitionSize = fileIndex.listFiles( - partitionKeyFilters.toSeq, - dataFilter) - .size - if (prunedPartitionSize > maxScanPartitions) { - throw maxPartitionExceedError( - prunedPartitionSize, - maxScanPartitions, - relation.catalogTable, - fileIndex.rootPaths, - fsRelation.partitionSchema) - } - case ScanOperation( - _, - filters, - logicalRelation @ LogicalRelation( - fsRelation @ HadoopFsRelation( - catalogFileIndex: CatalogFileIndex, - partitionSchema, - _, - _, - _, - _), - _, - _, - _)) if fsRelation.partitionSchema.nonEmpty => - val (partitionKeyFilters, _) = - getPartitionKeyFiltersAndDataFilters( - fsRelation.sparkSession, - logicalRelation, - partitionSchema, - filters, - logicalRelation.output) - - val prunedPartitionSize = - catalogFileIndex.filterPartitions( - partitionKeyFilters.toSeq) - .partitionSpec() - .partitions - .size - if (prunedPartitionSize > maxScanPartitions) { - throw maxPartitionExceedError( - prunedPartitionSize, - maxScanPartitions, - logicalRelation.catalogTable, - catalogFileIndex.rootPaths, - fsRelation.partitionSchema) - } - case _ => - } - } - - def maxPartitionExceedError( - prunedPartitionSize: Int, - maxPartitionSize: Int, - tableMeta: Option[CatalogTable], - rootPaths: Seq[Path], - partitionSchema: StructType): Throwable = { - val truncatedPaths = - if (rootPaths.length > 5) { - rootPaths.slice(0, 5).mkString(",") + """... """ + (rootPaths.length - 5) + " more paths" - } else { - rootPaths.mkString(",") - } - - new MaxPartitionExceedException( - s""" - |SQL job scan data source partition: $prunedPartitionSize - |exceed restrict of data source scan maxPartition $maxPartitionSize - |You should optimize your SQL logical according partition structure - |or shorten query scope such as p_date, detail as below: - |Table: ${tableMeta.map(_.qualifiedName).getOrElse("")} - |Owner: ${tableMeta.map(_.owner).getOrElse("")} - |RootPaths: $truncatedPaths - |Partition Structure: ${partitionSchema.map(_.name).mkString(", ")} - |""".stripMargin) - } -} diff --git a/extensions/spark/kyuubi-extension-spark-common/src/main/scala/org/apache/kyuubi/sql/watchdog/MaxScanStrategy.scala b/extensions/spark/kyuubi-extension-spark-common/src/main/scala/org/apache/kyuubi/sql/watchdog/MaxScanStrategy.scala new file mode 100644 index 000000000..0ee693fcb --- /dev/null +++ b/extensions/spark/kyuubi-extension-spark-common/src/main/scala/org/apache/kyuubi/sql/watchdog/MaxScanStrategy.scala @@ -0,0 +1,303 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.sql.watchdog + +import org.apache.hadoop.fs.Path +import org.apache.spark.sql.{PruneFileSourcePartitionHelper, SparkSession, Strategy} +import org.apache.spark.sql.catalyst.SQLConfHelper +import org.apache.spark.sql.catalyst.catalog.{CatalogTable, HiveTableRelation} +import org.apache.spark.sql.catalyst.planning.ScanOperation +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.execution.SparkPlan +import org.apache.spark.sql.execution.datasources.{CatalogFileIndex, HadoopFsRelation, InMemoryFileIndex, LogicalRelation} +import org.apache.spark.sql.types.StructType + +import org.apache.kyuubi.sql.KyuubiSQLConf + +/** + * Add MaxScanStrategy to avoid scan excessive partitions or files + * 1. Check if scan exceed maxPartition of partitioned table + * 2. Check if scan exceed maxFileSize (calculated by hive table and partition statistics) + * This Strategy Add Planner Strategy after LogicalOptimizer + * @param session + */ +case class MaxScanStrategy(session: SparkSession) + extends Strategy + with SQLConfHelper + with PruneFileSourcePartitionHelper { + override def apply(plan: LogicalPlan): Seq[SparkPlan] = { + val maxScanPartitionsOpt = conf.getConf(KyuubiSQLConf.WATCHDOG_MAX_PARTITIONS) + val maxFileSizeOpt = conf.getConf(KyuubiSQLConf.WATCHDOG_MAX_FILE_SIZE) + if (maxScanPartitionsOpt.isDefined || maxFileSizeOpt.isDefined) { + checkScan(plan, maxScanPartitionsOpt, maxFileSizeOpt) + } + Nil + } + + private def checkScan( + plan: LogicalPlan, + maxScanPartitionsOpt: Option[Int], + maxFileSizeOpt: Option[Long]): Unit = { + plan match { + case ScanOperation(_, _, relation: HiveTableRelation) => + if (relation.isPartitioned) { + relation.prunedPartitions match { + case Some(prunedPartitions) => + if (maxScanPartitionsOpt.exists(_ < prunedPartitions.size)) { + throw new MaxPartitionExceedException( + s""" + |SQL job scan hive partition: ${prunedPartitions.size} + |exceed restrict of hive scan maxPartition ${maxScanPartitionsOpt.get} + |You should optimize your SQL logical according partition structure + |or shorten query scope such as p_date, detail as below: + |Table: ${relation.tableMeta.qualifiedName} + |Owner: ${relation.tableMeta.owner} + |Partition Structure: ${relation.partitionCols.map(_.name).mkString(", ")} + |""".stripMargin) + } + lazy val scanFileSize = prunedPartitions.flatMap(_.stats).map(_.sizeInBytes).sum + if (maxFileSizeOpt.exists(_ < scanFileSize)) { + throw partTableMaxFileExceedError( + scanFileSize, + maxFileSizeOpt.get, + Some(relation.tableMeta), + prunedPartitions.flatMap(_.storage.locationUri).map(_.toString), + relation.partitionCols.map(_.name)) + } + case _ => + lazy val scanPartitions: Int = session + .sessionState.catalog.externalCatalog.listPartitionNames( + relation.tableMeta.database, + relation.tableMeta.identifier.table).size + if (maxScanPartitionsOpt.exists(_ < scanPartitions)) { + throw new MaxPartitionExceedException( + s""" + |Your SQL job scan a whole huge table without any partition filter, + |You should optimize your SQL logical according partition structure + |or shorten query scope such as p_date, detail as below: + |Table: ${relation.tableMeta.qualifiedName} + |Owner: ${relation.tableMeta.owner} + |Partition Structure: ${relation.partitionCols.map(_.name).mkString(", ")} + |""".stripMargin) + } + + lazy val scanFileSize: BigInt = + relation.tableMeta.stats.map(_.sizeInBytes).getOrElse { + session + .sessionState.catalog.externalCatalog.listPartitions( + relation.tableMeta.database, + relation.tableMeta.identifier.table).flatMap(_.stats).map(_.sizeInBytes).sum + } + if (maxFileSizeOpt.exists(_ < scanFileSize)) { + throw new MaxFileSizeExceedException( + s""" + |Your SQL job scan a whole huge table without any partition filter, + |You should optimize your SQL logical according partition structure + |or shorten query scope such as p_date, detail as below: + |Table: ${relation.tableMeta.qualifiedName} + |Owner: ${relation.tableMeta.owner} + |Partition Structure: ${relation.partitionCols.map(_.name).mkString(", ")} + |""".stripMargin) + } + } + } else { + lazy val scanFileSize = relation.tableMeta.stats.map(_.sizeInBytes).sum + if (maxFileSizeOpt.exists(_ < scanFileSize)) { + throw nonPartTableMaxFileExceedError( + scanFileSize, + maxFileSizeOpt.get, + Some(relation.tableMeta)) + } + } + case ScanOperation( + _, + filters, + relation @ LogicalRelation( + fsRelation @ HadoopFsRelation( + fileIndex: InMemoryFileIndex, + partitionSchema, + _, + _, + _, + _), + _, + _, + _)) => + if (fsRelation.partitionSchema.nonEmpty) { + val (partitionKeyFilters, dataFilter) = + getPartitionKeyFiltersAndDataFilters( + fsRelation.sparkSession, + relation, + partitionSchema, + filters, + relation.output) + val prunedPartitions = fileIndex.listFiles( + partitionKeyFilters.toSeq, + dataFilter) + if (maxScanPartitionsOpt.exists(_ < prunedPartitions.size)) { + throw maxPartitionExceedError( + prunedPartitions.size, + maxScanPartitionsOpt.get, + relation.catalogTable, + fileIndex.rootPaths, + fsRelation.partitionSchema) + } + lazy val scanFileSize = prunedPartitions.flatMap(_.files).map(_.getLen).sum + if (maxFileSizeOpt.exists(_ < scanFileSize)) { + throw partTableMaxFileExceedError( + scanFileSize, + maxFileSizeOpt.get, + relation.catalogTable, + fileIndex.rootPaths.map(_.toString), + fsRelation.partitionSchema.map(_.name)) + } + } else { + lazy val scanFileSize = fileIndex.sizeInBytes + if (maxFileSizeOpt.exists(_ < scanFileSize)) { + throw nonPartTableMaxFileExceedError( + scanFileSize, + maxFileSizeOpt.get, + relation.catalogTable) + } + } + case ScanOperation( + _, + filters, + logicalRelation @ LogicalRelation( + fsRelation @ HadoopFsRelation( + catalogFileIndex: CatalogFileIndex, + partitionSchema, + _, + _, + _, + _), + _, + _, + _)) => + if (fsRelation.partitionSchema.nonEmpty) { + val (partitionKeyFilters, _) = + getPartitionKeyFiltersAndDataFilters( + fsRelation.sparkSession, + logicalRelation, + partitionSchema, + filters, + logicalRelation.output) + + val fileIndex = catalogFileIndex.filterPartitions( + partitionKeyFilters.toSeq) + + lazy val prunedPartitionSize = fileIndex.partitionSpec().partitions.size + if (maxScanPartitionsOpt.exists(_ < prunedPartitionSize)) { + throw maxPartitionExceedError( + prunedPartitionSize, + maxScanPartitionsOpt.get, + logicalRelation.catalogTable, + catalogFileIndex.rootPaths, + fsRelation.partitionSchema) + } + + lazy val scanFileSize = fileIndex + .listFiles(Nil, Nil).flatMap(_.files).map(_.getLen).sum + if (maxFileSizeOpt.exists(_ < scanFileSize)) { + throw partTableMaxFileExceedError( + scanFileSize, + maxFileSizeOpt.get, + logicalRelation.catalogTable, + catalogFileIndex.rootPaths.map(_.toString), + fsRelation.partitionSchema.map(_.name)) + } + } else { + lazy val scanFileSize = catalogFileIndex.sizeInBytes + if (maxFileSizeOpt.exists(_ < scanFileSize)) { + throw nonPartTableMaxFileExceedError( + scanFileSize, + maxFileSizeOpt.get, + logicalRelation.catalogTable) + } + } + case _ => + } + } + + def maxPartitionExceedError( + prunedPartitionSize: Int, + maxPartitionSize: Int, + tableMeta: Option[CatalogTable], + rootPaths: Seq[Path], + partitionSchema: StructType): Throwable = { + val truncatedPaths = + if (rootPaths.length > 5) { + rootPaths.slice(0, 5).mkString(",") + """... """ + (rootPaths.length - 5) + " more paths" + } else { + rootPaths.mkString(",") + } + + new MaxPartitionExceedException( + s""" + |SQL job scan data source partition: $prunedPartitionSize + |exceed restrict of data source scan maxPartition $maxPartitionSize + |You should optimize your SQL logical according partition structure + |or shorten query scope such as p_date, detail as below: + |Table: ${tableMeta.map(_.qualifiedName).getOrElse("")} + |Owner: ${tableMeta.map(_.owner).getOrElse("")} + |RootPaths: $truncatedPaths + |Partition Structure: ${partitionSchema.map(_.name).mkString(", ")} + |""".stripMargin) + } + + private def partTableMaxFileExceedError( + scanFileSize: Number, + maxFileSize: Long, + tableMeta: Option[CatalogTable], + rootPaths: Seq[String], + partitions: Seq[String]): Throwable = { + val truncatedPaths = + if (rootPaths.length > 5) { + rootPaths.slice(0, 5).mkString(",") + """... """ + (rootPaths.length - 5) + " more paths" + } else { + rootPaths.mkString(",") + } + + new MaxFileSizeExceedException( + s""" + |SQL job scan file size in bytes: $scanFileSize + |exceed restrict of table scan maxFileSize $maxFileSize + |You should optimize your SQL logical according partition structure + |or shorten query scope such as p_date, detail as below: + |Table: ${tableMeta.map(_.qualifiedName).getOrElse("")} + |Owner: ${tableMeta.map(_.owner).getOrElse("")} + |RootPaths: $truncatedPaths + |Partition Structure: ${partitions.mkString(", ")} + |""".stripMargin) + } + + private def nonPartTableMaxFileExceedError( + scanFileSize: Number, + maxFileSize: Long, + tableMeta: Option[CatalogTable]): Throwable = { + new MaxFileSizeExceedException( + s""" + |SQL job scan file size in bytes: $scanFileSize + |exceed restrict of table scan maxFileSize $maxFileSize + |detail as below: + |Table: ${tableMeta.map(_.qualifiedName).getOrElse("")} + |Owner: ${tableMeta.map(_.owner).getOrElse("")} + |Location: ${tableMeta.map(_.location).getOrElse("")} + |""".stripMargin) + } +} diff --git a/extensions/spark/kyuubi-extension-spark-common/src/main/scala/org/apache/kyuubi/sql/zorder/OptimizeZorderStatementBase.scala b/extensions/spark/kyuubi-extension-spark-common/src/main/scala/org/apache/kyuubi/sql/zorder/OptimizeZorderStatementBase.scala index a9bb5a5d7..895f9e24b 100644 --- a/extensions/spark/kyuubi-extension-spark-common/src/main/scala/org/apache/kyuubi/sql/zorder/OptimizeZorderStatementBase.scala +++ b/extensions/spark/kyuubi-extension-spark-common/src/main/scala/org/apache/kyuubi/sql/zorder/OptimizeZorderStatementBase.scala @@ -20,24 +20,15 @@ package org.apache.kyuubi.sql.zorder import org.apache.spark.sql.catalyst.expressions.Attribute import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, UnaryNode} -/** - * A zorder statement that contains we parsed from SQL. - * We should convert this plan to certain command at Analyzer. - */ -abstract class OptimizeZorderStatementBase extends UnaryNode { - def tableIdentifier: Seq[String] - def query: LogicalPlan - override def child: LogicalPlan = query - override def output: Seq[Attribute] = child.output -} - /** * A zorder statement that contains we parsed from SQL. * We should convert this plan to certain command at Analyzer. */ case class OptimizeZorderStatement( tableIdentifier: Seq[String], - query: LogicalPlan) extends OptimizeZorderStatementBase { + query: LogicalPlan) extends UnaryNode { + override def child: LogicalPlan = query + override def output: Seq[Attribute] = child.output protected def withNewChildInternal(newChild: LogicalPlan): LogicalPlan = copy(query = newChild) } diff --git a/extensions/spark/kyuubi-extension-spark-common/src/main/scala/org/apache/kyuubi/sql/zorder/ResolveZorderBase.scala b/extensions/spark/kyuubi-extension-spark-common/src/main/scala/org/apache/kyuubi/sql/zorder/ResolveZorderBase.scala index cdead0b06..9f735caa7 100644 --- a/extensions/spark/kyuubi-extension-spark-common/src/main/scala/org/apache/kyuubi/sql/zorder/ResolveZorderBase.scala +++ b/extensions/spark/kyuubi-extension-spark-common/src/main/scala/org/apache/kyuubi/sql/zorder/ResolveZorderBase.scala @@ -57,7 +57,7 @@ abstract class ResolveZorderBase extends Rule[LogicalPlan] { } override def apply(plan: LogicalPlan): LogicalPlan = plan match { - case statement: OptimizeZorderStatementBase if statement.query.resolved => + case statement: OptimizeZorderStatement if statement.query.resolved => checkQueryAllowed(statement.query) val tableIdentifier = getTableIdentifier(statement.tableIdentifier) val catalogTable = session.sessionState.catalog.getTableMetadata(tableIdentifier) diff --git a/extensions/spark/kyuubi-extension-spark-common/src/test/resources/log4j2-test.xml b/extensions/spark/kyuubi-extension-spark-common/src/test/resources/log4j2-test.xml index bfc40dd6d..3110216c1 100644 --- a/extensions/spark/kyuubi-extension-spark-common/src/test/resources/log4j2-test.xml +++ b/extensions/spark/kyuubi-extension-spark-common/src/test/resources/log4j2-test.xml @@ -21,14 +21,14 @@ - + - + diff --git a/extensions/spark/kyuubi-extension-spark-common/src/test/scala/org/apache/spark/sql/KyuubiSparkSQLExtensionTest.scala b/extensions/spark/kyuubi-extension-spark-common/src/test/scala/org/apache/spark/sql/KyuubiSparkSQLExtensionTest.scala index fd81948c6..e58ac726c 100644 --- a/extensions/spark/kyuubi-extension-spark-common/src/test/scala/org/apache/spark/sql/KyuubiSparkSQLExtensionTest.scala +++ b/extensions/spark/kyuubi-extension-spark-common/src/test/scala/org/apache/spark/sql/KyuubiSparkSQLExtensionTest.scala @@ -29,6 +29,8 @@ import org.apache.kyuubi.sql.KyuubiSQLConf trait KyuubiSparkSQLExtensionTest extends QueryTest with SQLTestUtils with AdaptiveSparkPlanHelper { + sys.props.put("spark.testing", "1") + private var _spark: Option[SparkSession] = None protected def spark: SparkSession = _spark.getOrElse { throw new RuntimeException("test spark session don't initial before using it.") diff --git a/extensions/spark/kyuubi-extension-spark-common/src/test/scala/org/apache/spark/sql/WatchDogSuiteBase.scala b/extensions/spark/kyuubi-extension-spark-common/src/test/scala/org/apache/spark/sql/WatchDogSuiteBase.scala index e6ecd28c9..a202e813c 100644 --- a/extensions/spark/kyuubi-extension-spark-common/src/test/scala/org/apache/spark/sql/WatchDogSuiteBase.scala +++ b/extensions/spark/kyuubi-extension-spark-common/src/test/scala/org/apache/spark/sql/WatchDogSuiteBase.scala @@ -17,10 +17,15 @@ package org.apache.spark.sql +import java.io.File + +import scala.collection.JavaConverters._ + +import org.apache.commons.io.FileUtils import org.apache.spark.sql.catalyst.plans.logical.{GlobalLimit, LogicalPlan} import org.apache.kyuubi.sql.KyuubiSQLConf -import org.apache.kyuubi.sql.watchdog.MaxPartitionExceedException +import org.apache.kyuubi.sql.watchdog.{MaxFileSizeExceedException, MaxPartitionExceedException} trait WatchDogSuiteBase extends KyuubiSparkSQLExtensionTest { override protected def beforeAll(): Unit = { @@ -371,7 +376,7 @@ trait WatchDogSuiteBase extends KyuubiSparkSQLExtensionTest { |ORDER BY a |DESC |""".stripMargin) - .collect().head.get(0).equals(10)) + .collect().head.get(0) === 10) } } } @@ -477,4 +482,120 @@ trait WatchDogSuiteBase extends KyuubiSparkSQLExtensionTest { } } } + + private def checkMaxFileSize(tableSize: Long, nonPartTableSize: Long): Unit = { + withSQLConf(KyuubiSQLConf.WATCHDOG_MAX_FILE_SIZE.key -> tableSize.toString) { + checkAnswer(sql("SELECT count(distinct(p)) FROM test"), Row(10) :: Nil) + } + + withSQLConf(KyuubiSQLConf.WATCHDOG_MAX_FILE_SIZE.key -> (tableSize / 2).toString) { + sql("SELECT * FROM test where p=1").queryExecution.sparkPlan + + sql(s"SELECT * FROM test WHERE p in (${Range(0, 3).toList.mkString(",")})") + .queryExecution.sparkPlan + + intercept[MaxFileSizeExceedException]( + sql("SELECT * FROM test where p != 1").queryExecution.sparkPlan) + + intercept[MaxFileSizeExceedException]( + sql("SELECT * FROM test").queryExecution.sparkPlan) + + intercept[MaxFileSizeExceedException](sql( + s"SELECT * FROM test WHERE p in (${Range(0, 6).toList.mkString(",")})") + .queryExecution.sparkPlan) + } + + withSQLConf(KyuubiSQLConf.WATCHDOG_MAX_FILE_SIZE.key -> nonPartTableSize.toString) { + checkAnswer(sql("SELECT count(*) FROM test_non_part"), Row(10000) :: Nil) + } + + withSQLConf(KyuubiSQLConf.WATCHDOG_MAX_FILE_SIZE.key -> (nonPartTableSize - 1).toString) { + intercept[MaxFileSizeExceedException]( + sql("SELECT * FROM test_non_part").queryExecution.sparkPlan) + } + } + + test("watchdog with scan maxFileSize -- hive") { + Seq(false).foreach { convertMetastoreParquet => + withTable("test", "test_non_part", "temp") { + spark.range(10000).selectExpr("id as col") + .createOrReplaceTempView("temp") + + // partitioned table + sql( + s""" + |CREATE TABLE test(i int) + |PARTITIONED BY (p int) + |STORED AS parquet""".stripMargin) + for (part <- Range(0, 10)) { + sql( + s""" + |INSERT OVERWRITE TABLE test PARTITION (p='$part') + |select col from temp""".stripMargin) + } + + val tablePath = new File(spark.sessionState.catalog.externalCatalog + .getTable("default", "test").location) + val tableSize = FileUtils.listFiles(tablePath, Array("parquet"), true).asScala + .map(_.length()).sum + assert(tableSize > 0) + + // non-partitioned table + sql( + s""" + |CREATE TABLE test_non_part(i int) + |STORED AS parquet""".stripMargin) + sql( + s""" + |INSERT OVERWRITE TABLE test_non_part + |select col from temp""".stripMargin) + sql("ANALYZE TABLE test_non_part COMPUTE STATISTICS") + + val nonPartTablePath = new File(spark.sessionState.catalog.externalCatalog + .getTable("default", "test_non_part").location) + val nonPartTableSize = FileUtils.listFiles(nonPartTablePath, Array("parquet"), true).asScala + .map(_.length()).sum + assert(nonPartTableSize > 0) + + // check + withSQLConf("spark.sql.hive.convertMetastoreParquet" -> convertMetastoreParquet.toString) { + checkMaxFileSize(tableSize, nonPartTableSize) + } + } + } + } + + test("watchdog with scan maxFileSize -- data source") { + withTempDir { dir => + withTempView("test", "test_non_part") { + // partitioned table + val tablePath = new File(dir, "test") + spark.range(10).selectExpr("id", "id as p") + .write + .partitionBy("p") + .mode("overwrite") + .parquet(tablePath.getCanonicalPath) + spark.read.load(tablePath.getCanonicalPath).createOrReplaceTempView("test") + + val tableSize = FileUtils.listFiles(tablePath, Array("parquet"), true).asScala + .map(_.length()).sum + assert(tableSize > 0) + + // non-partitioned table + val nonPartTablePath = new File(dir, "test_non_part") + spark.range(10000).selectExpr("id", "id as p") + .write + .mode("overwrite") + .parquet(nonPartTablePath.getCanonicalPath) + spark.read.load(nonPartTablePath.getCanonicalPath).createOrReplaceTempView("test_non_part") + + val nonPartTableSize = FileUtils.listFiles(nonPartTablePath, Array("parquet"), true).asScala + .map(_.length()).sum + assert(tableSize > 0) + + // check + checkMaxFileSize(tableSize, nonPartTableSize) + } + } + } } diff --git a/extensions/spark/kyuubi-extension-spark-common/src/test/scala/org/apache/spark/sql/ZorderSuiteBase.scala b/extensions/spark/kyuubi-extension-spark-common/src/test/scala/org/apache/spark/sql/ZorderSuiteBase.scala index b24533e69..e0f86f85d 100644 --- a/extensions/spark/kyuubi-extension-spark-common/src/test/scala/org/apache/spark/sql/ZorderSuiteBase.scala +++ b/extensions/spark/kyuubi-extension-spark-common/src/test/scala/org/apache/spark/sql/ZorderSuiteBase.scala @@ -18,9 +18,11 @@ package org.apache.spark.sql import org.apache.spark.SparkConf -import org.apache.spark.sql.catalyst.InternalRow -import org.apache.spark.sql.catalyst.expressions.{Alias, Ascending, AttributeReference, Expression, ExpressionEvalHelper, Literal, NullsLast, SortOrder} -import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, OneRowRelation, Project, Sort} +import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier} +import org.apache.spark.sql.catalyst.analysis.{UnresolvedAttribute, UnresolvedFunction, UnresolvedRelation, UnresolvedStar} +import org.apache.spark.sql.catalyst.expressions.{Alias, Ascending, AttributeReference, EqualTo, Expression, ExpressionEvalHelper, Literal, NullsLast, SortOrder} +import org.apache.spark.sql.catalyst.parser.{ParseException, ParserInterface} +import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, OneRowRelation, Project, Sort} import org.apache.spark.sql.execution.command.CreateDataSourceTableAsSelectCommand import org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand import org.apache.spark.sql.functions._ @@ -29,7 +31,7 @@ import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf} import org.apache.spark.sql.types._ import org.apache.kyuubi.sql.{KyuubiSQLConf, KyuubiSQLExtensionException} -import org.apache.kyuubi.sql.zorder.{OptimizeZorderCommandBase, Zorder, ZorderBytesUtils} +import org.apache.kyuubi.sql.zorder.{OptimizeZorderCommandBase, OptimizeZorderStatement, Zorder, ZorderBytesUtils} trait ZorderSuiteBase extends KyuubiSparkSQLExtensionTest with ExpressionEvalHelper { override def sparkConf(): SparkConf = { @@ -245,20 +247,22 @@ trait ZorderSuiteBase extends KyuubiSparkSQLExtensionTest with ExpressionEvalHel resHasSort: Boolean): Unit = { def checkSort(plan: LogicalPlan): Unit = { assert(plan.isInstanceOf[Sort] === resHasSort) - if (plan.isInstanceOf[Sort]) { - val colArr = cols.split(",") - val refs = - if (colArr.length == 1) { - plan.asInstanceOf[Sort].order.head - .child.asInstanceOf[AttributeReference] :: Nil - } else { - plan.asInstanceOf[Sort].order.head - .child.asInstanceOf[Zorder].children.map(_.references.head) + plan match { + case sort: Sort => + val colArr = cols.split(",") + val refs = + if (colArr.length == 1) { + sort.order.head + .child.asInstanceOf[AttributeReference] :: Nil + } else { + sort.order.head + .child.asInstanceOf[Zorder].children.map(_.references.head) + } + assert(refs.size === colArr.size) + refs.zip(colArr).foreach { case (ref, col) => + assert(ref.name === col.trim) } - assert(refs.size === colArr.size) - refs.zip(colArr).foreach { case (ref, col) => - assert(ref.name === col.trim) - } + case _ => } } @@ -652,6 +656,99 @@ trait ZorderSuiteBase extends KyuubiSparkSQLExtensionTest with ExpressionEvalHel ZorderBytesUtils.interleaveBitsDefault(inputs.map(ZorderBytesUtils.toByteArray).toArray))) } } + + test("OPTIMIZE command is parsed as expected") { + val parser = createParser + val globalSort = spark.conf.get(KyuubiSQLConf.ZORDER_GLOBAL_SORT_ENABLED) + + assert(parser.parsePlan("OPTIMIZE p zorder by c1") === + OptimizeZorderStatement( + Seq("p"), + Sort( + SortOrder(UnresolvedAttribute("c1"), Ascending, NullsLast, Seq.empty) :: Nil, + globalSort, + Project(Seq(UnresolvedStar(None)), UnresolvedRelation(TableIdentifier("p")))))) + + assert(parser.parsePlan("OPTIMIZE p zorder by c1, c2") === + OptimizeZorderStatement( + Seq("p"), + Sort( + SortOrder( + Zorder(Seq(UnresolvedAttribute("c1"), UnresolvedAttribute("c2"))), + Ascending, + NullsLast, + Seq.empty) :: Nil, + globalSort, + Project(Seq(UnresolvedStar(None)), UnresolvedRelation(TableIdentifier("p")))))) + + assert(parser.parsePlan("OPTIMIZE p where id = 1 zorder by c1") === + OptimizeZorderStatement( + Seq("p"), + Sort( + SortOrder(UnresolvedAttribute("c1"), Ascending, NullsLast, Seq.empty) :: Nil, + globalSort, + Project( + Seq(UnresolvedStar(None)), + Filter( + EqualTo(UnresolvedAttribute("id"), Literal(1)), + UnresolvedRelation(TableIdentifier("p"))))))) + + assert(parser.parsePlan("OPTIMIZE p where id = 1 zorder by c1, c2") === + OptimizeZorderStatement( + Seq("p"), + Sort( + SortOrder( + Zorder(Seq(UnresolvedAttribute("c1"), UnresolvedAttribute("c2"))), + Ascending, + NullsLast, + Seq.empty) :: Nil, + globalSort, + Project( + Seq(UnresolvedStar(None)), + Filter( + EqualTo(UnresolvedAttribute("id"), Literal(1)), + UnresolvedRelation(TableIdentifier("p"))))))) + + assert(parser.parsePlan("OPTIMIZE p where id = current_date() zorder by c1") === + OptimizeZorderStatement( + Seq("p"), + Sort( + SortOrder(UnresolvedAttribute("c1"), Ascending, NullsLast, Seq.empty) :: Nil, + globalSort, + Project( + Seq(UnresolvedStar(None)), + Filter( + EqualTo( + UnresolvedAttribute("id"), + UnresolvedFunction("current_date", Seq.empty, false)), + UnresolvedRelation(TableIdentifier("p"))))))) + + // TODO: add following case support + intercept[ParseException] { + parser.parsePlan("OPTIMIZE p zorder by (c1)") + } + + intercept[ParseException] { + parser.parsePlan("OPTIMIZE p zorder by (c1, c2)") + } + } + + test("OPTIMIZE partition predicates constraint") { + withTable("p") { + sql("CREATE TABLE p (c1 INT, c2 INT) PARTITIONED BY (event_date DATE)") + val e1 = intercept[KyuubiSQLExtensionException] { + sql("OPTIMIZE p WHERE event_date = current_date as c ZORDER BY c1, c2") + } + assert(e1.getMessage.contains("unsupported partition predicates")) + + val e2 = intercept[KyuubiSQLExtensionException] { + sql("OPTIMIZE p WHERE c1 = 1 ZORDER BY c1, c2") + } + assert(e2.getMessage == "Only partition column filters are allowed") + } + } + + def createParser: ParserInterface } trait ZorderWithCodegenEnabledSuiteBase extends ZorderSuiteBase { diff --git a/extensions/spark/kyuubi-extension-spark-common/src/test/scala/org/apache/spark/sql/benchmark/KyuubiBenchmarkBase.scala b/extensions/spark/kyuubi-extension-spark-common/src/test/scala/org/apache/spark/sql/benchmark/KyuubiBenchmarkBase.scala index c8c1b021d..b891a7224 100644 --- a/extensions/spark/kyuubi-extension-spark-common/src/test/scala/org/apache/spark/sql/benchmark/KyuubiBenchmarkBase.scala +++ b/extensions/spark/kyuubi-extension-spark-common/src/test/scala/org/apache/spark/sql/benchmark/KyuubiBenchmarkBase.scala @@ -22,6 +22,7 @@ import java.io.{File, FileOutputStream, OutputStream} import scala.collection.JavaConverters._ import com.google.common.reflect.ClassPath +import org.scalatest.Assertions._ trait KyuubiBenchmarkBase { var output: Option[OutputStream] = None diff --git a/extensions/spark/kyuubi-extension-spark-jdbc-dialect/pom.xml b/extensions/spark/kyuubi-extension-spark-jdbc-dialect/pom.xml index 5588805e9..ea571644e 100644 --- a/extensions/spark/kyuubi-extension-spark-jdbc-dialect/pom.xml +++ b/extensions/spark/kyuubi-extension-spark-jdbc-dialect/pom.xml @@ -21,12 +21,12 @@ org.apache.kyuubi kyuubi-parent - 1.7.0-SNAPSHOT + 1.9.0-SNAPSHOT ../../../pom.xml - kyuubi-extension-spark-jdbc-dialect_2.12 + kyuubi-extension-spark-jdbc-dialect_${scala.binary.version} jar Kyuubi Spark JDBC Dialect plugin https://kyuubi.apache.org/ diff --git a/extensions/spark/kyuubi-extension-spark-jdbc-dialect/src/test/scala/resources/log4j2-test.xml b/extensions/spark/kyuubi-extension-spark-jdbc-dialect/src/test/scala/resources/log4j2-test.xml index bfc40dd6d..3110216c1 100644 --- a/extensions/spark/kyuubi-extension-spark-jdbc-dialect/src/test/scala/resources/log4j2-test.xml +++ b/extensions/spark/kyuubi-extension-spark-jdbc-dialect/src/test/scala/resources/log4j2-test.xml @@ -21,14 +21,14 @@ - + - + diff --git a/extensions/spark/kyuubi-spark-authz-shaded/pom.xml b/extensions/spark/kyuubi-spark-authz-shaded/pom.xml new file mode 100644 index 000000000..10edeb1fb --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz-shaded/pom.xml @@ -0,0 +1,337 @@ + + + + 4.0.0 + + org.apache.kyuubi + kyuubi-parent + 1.9.0-SNAPSHOT + ../../../pom.xml + + + kyuubi-spark-authz-shaded_${scala.binary.version} + jar + Kyuubi Dev Spark Authorization Extension Shaded + https://kyuubi.apache.org/ + + + + 1.0.0 + 1.19.4 + 5.7.0 + + + + + org.apache.kyuubi + kyuubi-spark-authz_${scala.binary.version} + ${project.version} + + + org.apache.kyuubi + kyuubi-util-scala_${scala.binary.version} + ${project.version} + + + org.apache.ranger + ranger-plugins-common + ${ranger.version} + + + com.sun.jersey + jersey-bundle + + + org.apache.ranger + ranger-plugin-classloader + + + org.apache.ranger + ranger-plugins-audit + + + log4j + log4j + + + ch.qos.logback + logback-classic + + + commons-lang + commons-lang + + + commons-logging + commons-logging + + + org.apache.httpcomponents + * + + + org.apache.hadoop + hadoop-common + + + javax.ws.rs + jsr311-api + + + com.kstruct + gethostname4j + + + net.java.dev.jna + jna + + + net.java.dev.jna + jna-platform + + + + + + com.sun.jersey + jersey-client + ${jersey.client.version} + + + javax.ws.rs + jsr311-api + + + + + + com.kstruct + gethostname4j + ${gethostname4j.version} + + + + net.java.dev.jna + jna + ${jna.version} + + + + net.java.dev.jna + jna-platform + ${jna.version} + + + + org.apache.ranger + ranger-plugins-audit + ${ranger.version} + + + org.apache.ranger + ranger-plugins-cred + + + org.apache.kafka + * + + + org.apache.solr + solr-solrj + + + org.elasticsearch + * + + + org.elasticsearch.client + * + + + org.elasticsearch.plugin + * + + + org.apache.lucene + * + + + log4j + log4j + + + commons-lang + commons-lang + + + commons-logging + commons-logging + + + com.carrotsearch + hppc + + + org.apache.httpcomponents + * + + + org.apache.hive + hive-storage-api + + + org.apache.orc + orc-core + + + org.apache.hadoop + hadoop-common + + + com.google.guava + guava + + + joda-time + joda-time + + + org.apache.logging.log4j + * + + + com.amazonaws + aws-java-sdk-bundle + + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + false + + + org.apache.kyuubi:kyuubi-util-scala_${scala.binary.version} + org.apache.kyuubi:kyuubi-spark-authz_${scala.binary.version} + org.apache.kyuubi:kyuubi-util + org.apache.ranger:ranger-plugins-common + org.apache.ranger:ranger-plugins-audit + org.codehaus.jackson:jackson-jaxrs + org.codehaus.jackson:jackson-core-asl + org.codehaus.jackson:jackson-mapper-asl + com.sun.jersey:jersey-client + com.sun.jersey:jersey-core + com.kstruct:gethostname4j + net.java.dev.jna:jna + net.java.dev.jna:jna-platform + + + + + *:* + + **/*.proto + META-INF/*.SF + META-INF/LGPL2.1 + META-INF/AL2.0 + META-INF/ASL2.0 + META-INF/*.DSA + META-INF/*.RSA + META-INF/DEPENDENCIES + META-INF/LICENSE.txt + META-INF/NOTICE.txt + META-INF/maven/** + LICENSE.txt + NOTICE.txt + mozilla/** + **/module-info.class + + + + + + org.codehaus.jackson + ${kyuubi.shade.packageName}.org.codehaus.jackson + + + com.sun.jersey + ${kyuubi.shade.packageName}.com.sun.jersey + + + com.sun.ws.rs.ext + ${kyuubi.shade.packageName}.com.sun.ws.rs.ext + + + com.kstruct.gethostname4j + ${kyuubi.shade.packageName}.com.kstruct.gethostname4j + + + org.apache.hadoop.security + ${kyuubi.shade.packageName}.org.apache.hadoop.security + + org.apache.hadoop.security.KrbPasswordSaverLoginModule + org.apache.hadoop.security.SecureClientLogin + org.apache.hadoop.security.SecureClientLoginConfiguration + + + + + + + + + + + shade + + package + + + + + + + + + + scala-2.13 + + + + org.apache.maven.plugins + maven-shade-plugin + + + + net.java.dev.jna:jna + net.java.dev.jna:jna-platform + + + + + + + + + diff --git a/extensions/spark/kyuubi-spark-authz-shaded/src/main/resources/META-INF/LICENSE b/extensions/spark/kyuubi-spark-authz-shaded/src/main/resources/META-INF/LICENSE new file mode 100644 index 000000000..1e6d25e88 --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz-shaded/src/main/resources/META-INF/LICENSE @@ -0,0 +1,225 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +------------------------------------------------------------------------------------ + +This project bundles some components that are licensed under the + +Apache License Version 2.0 +-------------------------- +org.apache.ranger:ranger-plugins-common +org.apache.ranger:ranger-plugins-audit +org.codehaus.jackson:jackson-jaxrs +org.codehaus.jackson:jackson-core-asl +org.codehaus.jackson:jackson-mapper-asl +net.java.dev.jna:jna +net.java.dev.jna:jna-platform + +Common Development and Distribution License (CDDL) 1.1 +------------------------------------------------------ +com.sun.jersey:jersey-client +com.sun.jersey:jersey-core + +MIT license +----------- +com.kstruct:gethostname4j diff --git a/extensions/spark/kyuubi-spark-authz-shaded/src/main/resources/META-INF/NOTICE b/extensions/spark/kyuubi-spark-authz-shaded/src/main/resources/META-INF/NOTICE new file mode 100644 index 000000000..9afa0f86d --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz-shaded/src/main/resources/META-INF/NOTICE @@ -0,0 +1,12 @@ +Apache Kyuubi +Copyright 2021-2023 The Apache Software Foundation. + +This product includes software developed at +The Apache Software Foundation (https://www.apache.org/). + +-------------------------------------------------------------------------------- + +This binary artifact contains + +Apache Ranger +Copyright 2014-2022 The Apache Software Foundation diff --git a/extensions/spark/kyuubi-spark-authz/README.md b/extensions/spark/kyuubi-spark-authz/README.md index c257e30e1..9657b5b7a 100644 --- a/extensions/spark/kyuubi-spark-authz/README.md +++ b/extensions/spark/kyuubi-spark-authz/README.md @@ -1,19 +1,19 @@ +- Licensed to the Apache Software Foundation (ASF) under one or more +- contributor license agreements. See the NOTICE file distributed with +- this work for additional information regarding copyright ownership. +- The ASF licenses this file to You under the Apache License, Version 2.0 +- (the "License"); you may not use this file except in compliance with +- the License. You may obtain a copy of the License at +- +- http://www.apache.org/licenses/LICENSE-2.0 +- +- Unless required by applicable law or agreed to in writing, software +- distributed under the License is distributed on an "AS IS" BASIS, +- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +- See the License for the specific language governing permissions and +- limitations under the License. +--> # Kyuubi Spark AuthZ Extension @@ -26,26 +26,27 @@ ## Build ```shell -build/mvn clean package -pl :kyuubi-spark-authz_2.12 -Dspark.version=3.2.1 -Dranger.version=2.3.0 +build/mvn clean package -DskipTests -pl :kyuubi-spark-authz_2.12 -am -Dspark.version=3.2.1 -Dranger.version=2.4.0 ``` - ### Supported Apache Spark Versions `-Dspark.version=` - [x] master -- [x] 3.3.x (default) +- [x] 3.4.x (default) +- [x] 3.3.x - [x] 3.2.x - [x] 3.1.x -- [x] 3.0.x +- [ ] 3.0.x - [ ] 2.4.x and earlier ### Supported Apache Ranger Versions `-Dranger.version=` -- [x] 2.3.x (default) +- [x] 2.4.x (default) +- [x] 2.3.x - [x] 2.2.x - [x] 2.1.x - [x] 2.0.x @@ -53,4 +54,5 @@ build/mvn clean package -pl :kyuubi-spark-authz_2.12 -Dspark.version=3.2.1 -Dran - [x] 1.1.x - [x] 1.0.x - [x] 0.7.x -- [x] 0.6.x +- [ ] 0.6.x + diff --git a/extensions/spark/kyuubi-spark-authz/pom.xml b/extensions/spark/kyuubi-spark-authz/pom.xml index d537e5a1c..c2d9f7595 100644 --- a/extensions/spark/kyuubi-spark-authz/pom.xml +++ b/extensions/spark/kyuubi-spark-authz/pom.xml @@ -21,27 +21,38 @@ org.apache.kyuubi kyuubi-parent - 1.7.0-SNAPSHOT + 1.9.0-SNAPSHOT ../../../pom.xml - kyuubi-spark-authz_2.12 + kyuubi-spark-authz_${scala.binary.version} jar Kyuubi Dev Spark Authorization Extension https://kyuubi.apache.org/ + 1.0.0 + 1.19.4 5.7.0 + + org.apache.kyuubi + kyuubi-util-scala_${scala.binary.version} + ${project.version} + org.apache.ranger ranger-plugins-common ${ranger.version} + + com.sun.jersey + jersey-bundle + org.apache.ranger ranger-plugin-classloader @@ -101,6 +112,18 @@ + + com.sun.jersey + jersey-client + ${jersey.client.version} + + + javax.ws.rs + jsr311-api + + + + com.kstruct gethostname4j @@ -300,10 +323,21 @@ scala-collection-compat_${scala.binary.version} test + + + org.apache.paimon + paimon-spark-${paimon.spark.binary.version} + test + + + + io.delta + ${delta.artifact}_${scala.binary.version} + test + - ${project.basedir}/src/test/resources @@ -313,4 +347,48 @@ target/scala-${scala.binary.version}/test-classes + + + + spark-authz-hudi-test + + + org.apache.hudi + hudi-spark${hudi.spark.binary.version}-bundle_${scala.binary.version} + ${hudi.version} + test + + + + + + gen-policy + + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-test-source + + add-test-source + + generate-sources + + + src/test/gen/scala + + + + + + + + + diff --git a/extensions/spark/kyuubi-spark-authz/src/main/resources/META-INF/services/org.apache.kyuubi.plugin.spark.authz.serde.FunctionExtractor b/extensions/spark/kyuubi-spark-authz/src/main/resources/META-INF/services/org.apache.kyuubi.plugin.spark.authz.serde.FunctionExtractor index 4686bb033..2facb004a 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/resources/META-INF/services/org.apache.kyuubi.plugin.spark.authz.serde.FunctionExtractor +++ b/extensions/spark/kyuubi-spark-authz/src/main/resources/META-INF/services/org.apache.kyuubi.plugin.spark.authz.serde.FunctionExtractor @@ -17,4 +17,5 @@ org.apache.kyuubi.plugin.spark.authz.serde.ExpressionInfoFunctionExtractor org.apache.kyuubi.plugin.spark.authz.serde.FunctionIdentifierFunctionExtractor +org.apache.kyuubi.plugin.spark.authz.serde.QualifiedNameStringFunctionExtractor org.apache.kyuubi.plugin.spark.authz.serde.StringFunctionExtractor diff --git a/extensions/spark/kyuubi-spark-authz/src/main/resources/META-INF/services/org.apache.kyuubi.plugin.spark.authz.serde.FunctionTypeExtractor b/extensions/spark/kyuubi-spark-authz/src/main/resources/META-INF/services/org.apache.kyuubi.plugin.spark.authz.serde.FunctionTypeExtractor index 475f47afc..3bb0ee6c2 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/resources/META-INF/services/org.apache.kyuubi.plugin.spark.authz.serde.FunctionTypeExtractor +++ b/extensions/spark/kyuubi-spark-authz/src/main/resources/META-INF/services/org.apache.kyuubi.plugin.spark.authz.serde.FunctionTypeExtractor @@ -17,4 +17,5 @@ org.apache.kyuubi.plugin.spark.authz.serde.ExpressionInfoFunctionTypeExtractor org.apache.kyuubi.plugin.spark.authz.serde.FunctionIdentifierFunctionTypeExtractor +org.apache.kyuubi.plugin.spark.authz.serde.FunctionNameFunctionTypeExtractor org.apache.kyuubi.plugin.spark.authz.serde.TempMarkerFunctionTypeExtractor diff --git a/extensions/spark/kyuubi-spark-authz/src/main/resources/META-INF/services/org.apache.kyuubi.plugin.spark.authz.serde.QueryExtractor b/extensions/spark/kyuubi-spark-authz/src/main/resources/META-INF/services/org.apache.kyuubi.plugin.spark.authz.serde.QueryExtractor index c659114f9..2406a40e1 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/resources/META-INF/services/org.apache.kyuubi.plugin.spark.authz.serde.QueryExtractor +++ b/extensions/spark/kyuubi-spark-authz/src/main/resources/META-INF/services/org.apache.kyuubi.plugin.spark.authz.serde.QueryExtractor @@ -15,5 +15,6 @@ # limitations under the License. # +org.apache.kyuubi.plugin.spark.authz.serde.HudiMergeIntoSourceTableExtractor org.apache.kyuubi.plugin.spark.authz.serde.LogicalPlanOptionQueryExtractor org.apache.kyuubi.plugin.spark.authz.serde.LogicalPlanQueryExtractor diff --git a/extensions/spark/kyuubi-spark-authz/src/main/resources/META-INF/services/org.apache.kyuubi.plugin.spark.authz.serde.TableExtractor b/extensions/spark/kyuubi-spark-authz/src/main/resources/META-INF/services/org.apache.kyuubi.plugin.spark.authz.serde.TableExtractor index f4d7eb503..dc35a8f51 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/resources/META-INF/services/org.apache.kyuubi.plugin.spark.authz.serde.TableExtractor +++ b/extensions/spark/kyuubi-spark-authz/src/main/resources/META-INF/services/org.apache.kyuubi.plugin.spark.authz.serde.TableExtractor @@ -18,8 +18,16 @@ org.apache.kyuubi.plugin.spark.authz.serde.CatalogTableOptionTableExtractor org.apache.kyuubi.plugin.spark.authz.serde.CatalogTableTableExtractor org.apache.kyuubi.plugin.spark.authz.serde.DataSourceV2RelationTableExtractor +org.apache.kyuubi.plugin.spark.authz.serde.ExpressionSeqTableExtractor +org.apache.kyuubi.plugin.spark.authz.serde.HudiDataSourceV2RelationTableExtractor +org.apache.kyuubi.plugin.spark.authz.serde.HudiMergeIntoTargetTableExtractor +org.apache.kyuubi.plugin.spark.authz.serde.HudiCallProcedureInputTableExtractor +org.apache.kyuubi.plugin.spark.authz.serde.HudiCallProcedureOutputTableExtractor org.apache.kyuubi.plugin.spark.authz.serde.IdentifierTableExtractor org.apache.kyuubi.plugin.spark.authz.serde.LogicalRelationTableExtractor org.apache.kyuubi.plugin.spark.authz.serde.ResolvedDbObjectNameTableExtractor +org.apache.kyuubi.plugin.spark.authz.serde.ResolvedIdentifierTableExtractor org.apache.kyuubi.plugin.spark.authz.serde.ResolvedTableTableExtractor +org.apache.kyuubi.plugin.spark.authz.serde.StringTableExtractor org.apache.kyuubi.plugin.spark.authz.serde.TableIdentifierTableExtractor +org.apache.kyuubi.plugin.spark.authz.serde.TableTableExtractor diff --git a/extensions/spark/kyuubi-spark-authz/src/main/resources/META-INF/services/org.apache.kyuubi.plugin.spark.authz.serde.URIExtractor b/extensions/spark/kyuubi-spark-authz/src/main/resources/META-INF/services/org.apache.kyuubi.plugin.spark.authz.serde.URIExtractor new file mode 100644 index 000000000..0b77fa26e --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/main/resources/META-INF/services/org.apache.kyuubi.plugin.spark.authz.serde.URIExtractor @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.apache.kyuubi.plugin.spark.authz.serde.StringURIExtractor diff --git a/extensions/spark/kyuubi-spark-authz/src/main/resources/database_command_spec.json b/extensions/spark/kyuubi-spark-authz/src/main/resources/database_command_spec.json index 4eb4b3ef8..c640ed89b 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/resources/database_command_spec.json +++ b/extensions/spark/kyuubi-spark-authz/src/main/resources/database_command_spec.json @@ -22,6 +22,11 @@ "fieldExtractor" : "CatalogPluginCatalogExtractor" }, "isInput" : false + }, { + "fieldName" : "name", + "fieldExtractor" : "ResolvedNamespaceDatabaseExtractor", + "catalogDesc" : null, + "isInput" : false } ], "opType" : "CREATEDATABASE" }, { @@ -45,6 +50,11 @@ }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.SetCatalogAndNamespace", "databaseDescs" : [ { + "fieldName" : "child", + "fieldExtractor" : "ResolvedNamespaceDatabaseExtractor", + "catalogDesc" : null, + "isInput" : true + }, { "fieldName" : "child", "fieldExtractor" : "ResolvedDBObjectNameDatabaseExtractor", "catalogDesc" : null, diff --git a/extensions/spark/kyuubi-spark-authz/src/main/resources/function_command_spec.json b/extensions/spark/kyuubi-spark-authz/src/main/resources/function_command_spec.json index c93985614..0b71245d2 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/resources/function_command_spec.json +++ b/extensions/spark/kyuubi-spark-authz/src/main/resources/function_command_spec.json @@ -1,6 +1,16 @@ [ { "classname" : "org.apache.spark.sql.execution.command.CreateFunctionCommand", "functionDescs" : [ { + "fieldName" : "identifier", + "fieldExtractor" : "FunctionIdentifierFunctionExtractor", + "databaseDesc" : null, + "functionTypeDesc" : { + "fieldName" : "isTemp", + "fieldExtractor" : "TempMarkerFunctionTypeExtractor", + "skipTypes" : [ "TEMP" ] + }, + "isInput" : false + }, { "fieldName" : "functionName", "fieldExtractor" : "StringFunctionExtractor", "databaseDesc" : { @@ -44,6 +54,16 @@ }, { "classname" : "org.apache.spark.sql.execution.command.DropFunctionCommand", "functionDescs" : [ { + "fieldName" : "identifier", + "fieldExtractor" : "FunctionIdentifierFunctionExtractor", + "databaseDesc" : null, + "functionTypeDesc" : { + "fieldName" : "isTemp", + "fieldExtractor" : "TempMarkerFunctionTypeExtractor", + "skipTypes" : [ "TEMP" ] + }, + "isInput" : false + }, { "fieldName" : "functionName", "fieldExtractor" : "StringFunctionExtractor", "databaseDesc" : { diff --git a/extensions/spark/kyuubi-spark-authz/src/main/resources/scan_command_spec.json b/extensions/spark/kyuubi-spark-authz/src/main/resources/scan_command_spec.json index 9a6aef4ed..40a0d81c2 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/resources/scan_command_spec.json +++ b/extensions/spark/kyuubi-spark-authz/src/main/resources/scan_command_spec.json @@ -1,29 +1,89 @@ [ { - "classname" : "org.apache.kyuubi.plugin.spark.authz.util.PermanentViewMarker", + "classname" : "org.apache.kyuubi.plugin.spark.authz.rule.permanentview.PermanentViewMarker", "scanDescs" : [ { "fieldName" : "catalogTable", "fieldExtractor" : "CatalogTableTableExtractor", "catalogDesc" : null - } ] + } ], + "functionDescs" : [ ] }, { "classname" : "org.apache.spark.sql.catalyst.catalog.HiveTableRelation", "scanDescs" : [ { "fieldName" : "tableMeta", "fieldExtractor" : "CatalogTableTableExtractor", "catalogDesc" : null - } ] + } ], + "functionDescs" : [ ] }, { "classname" : "org.apache.spark.sql.execution.datasources.LogicalRelation", "scanDescs" : [ { "fieldName" : "catalogTable", "fieldExtractor" : "CatalogTableOptionTableExtractor", "catalogDesc" : null - } ] + } ], + "functionDescs" : [ ] }, { "classname" : "org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation", "scanDescs" : [ { "fieldName" : null, "fieldExtractor" : "DataSourceV2RelationTableExtractor", "catalogDesc" : null + } ], + "functionDescs" : [ ] +}, { + "classname" : "org.apache.spark.sql.hive.HiveGenericUDF", + "scanDescs" : [ ], + "functionDescs" : [ { + "fieldName" : "name", + "fieldExtractor" : "QualifiedNameStringFunctionExtractor", + "databaseDesc" : null, + "functionTypeDesc" : { + "fieldName" : "name", + "fieldExtractor" : "FunctionNameFunctionTypeExtractor", + "skipTypes" : [ "TEMP", "SYSTEM" ] + }, + "isInput" : true + } ] +}, { + "classname" : "org.apache.spark.sql.hive.HiveGenericUDTF", + "scanDescs" : [ ], + "functionDescs" : [ { + "fieldName" : "name", + "fieldExtractor" : "QualifiedNameStringFunctionExtractor", + "databaseDesc" : null, + "functionTypeDesc" : { + "fieldName" : "name", + "fieldExtractor" : "FunctionNameFunctionTypeExtractor", + "skipTypes" : [ "TEMP", "SYSTEM" ] + }, + "isInput" : true + } ] +}, { + "classname" : "org.apache.spark.sql.hive.HiveSimpleUDF", + "scanDescs" : [ ], + "functionDescs" : [ { + "fieldName" : "name", + "fieldExtractor" : "QualifiedNameStringFunctionExtractor", + "databaseDesc" : null, + "functionTypeDesc" : { + "fieldName" : "name", + "fieldExtractor" : "FunctionNameFunctionTypeExtractor", + "skipTypes" : [ "TEMP", "SYSTEM" ] + }, + "isInput" : true + } ] +}, { + "classname" : "org.apache.spark.sql.hive.HiveUDAFFunction", + "scanDescs" : [ ], + "functionDescs" : [ { + "fieldName" : "name", + "fieldExtractor" : "QualifiedNameStringFunctionExtractor", + "databaseDesc" : null, + "functionTypeDesc" : { + "fieldName" : "name", + "fieldExtractor" : "FunctionNameFunctionTypeExtractor", + "skipTypes" : [ "TEMP", "SYSTEM" ] + }, + "isInput" : true } ] } ] \ No newline at end of file diff --git a/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json b/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json index 3b9b8f24e..67e027c6e 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json +++ b/extensions/spark/kyuubi-spark-authz/src/main/resources/table_command_spec.json @@ -11,7 +11,8 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "ALTERTABLE_ADDCOLS", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.AddPartitions", "tableDescs" : [ { @@ -25,7 +26,8 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "ALTERTABLE_ADDPARTS", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.AlterColumn", "tableDescs" : [ { @@ -39,7 +41,8 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "ALTERTABLE_ADDCOLS", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.AlterTable", "tableDescs" : [ { @@ -53,7 +56,8 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "ALTERTABLE_PROPERTIES", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.AppendData", "tableDescs" : [ { @@ -74,7 +78,8 @@ "queryDescs" : [ { "fieldName" : "query", "fieldExtractor" : "LogicalPlanQueryExtractor" - } ] + } ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.CacheTable", "tableDescs" : [ ], @@ -82,7 +87,8 @@ "queryDescs" : [ { "fieldName" : "table", "fieldExtractor" : "LogicalPlanQueryExtractor" - } ] + } ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.CacheTableAsSelect", "tableDescs" : [ ], @@ -90,7 +96,8 @@ "queryDescs" : [ { "fieldName" : "plan", "fieldExtractor" : "LogicalPlanQueryExtractor" - } ] + } ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.CommentOnTable", "tableDescs" : [ { @@ -104,10 +111,20 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "ALTERTABLE_PROPERTIES", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.CreateTable", "tableDescs" : [ { + "fieldName" : "child", + "fieldExtractor" : "ResolvedIdentifierTableExtractor", + "columnDesc" : null, + "actionTypeDesc" : null, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : false, + "setCurrentDatabaseIfMissing" : false + }, { "fieldName" : "tableName", "fieldExtractor" : "IdentifierTableExtractor", "columnDesc" : null, @@ -130,10 +147,20 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "CREATETABLE", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.CreateTableAsSelect", "tableDescs" : [ { + "fieldName" : "name", + "fieldExtractor" : "ResolvedIdentifierTableExtractor", + "columnDesc" : null, + "actionTypeDesc" : null, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : false, + "setCurrentDatabaseIfMissing" : false + }, { "fieldName" : "tableName", "fieldExtractor" : "IdentifierTableExtractor", "columnDesc" : null, @@ -146,7 +173,7 @@ "isInput" : false, "setCurrentDatabaseIfMissing" : false }, { - "fieldName" : "left", + "fieldName" : "name", "fieldExtractor" : "ResolvedDbObjectNameTableExtractor", "columnDesc" : null, "actionTypeDesc" : null, @@ -159,7 +186,8 @@ "queryDescs" : [ { "fieldName" : "query", "fieldExtractor" : "LogicalPlanQueryExtractor" - } ] + } ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.CreateV2Table", "tableDescs" : [ { @@ -176,9 +204,10 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "CREATETABLE", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { - "classname" : "org.apache.spark.sql.catalyst.plans.logical.DeleteFromIcebergTable", + "classname" : "org.apache.spark.sql.catalyst.plans.logical.DeleteFromTable", "tableDescs" : [ { "fieldName" : "table", "fieldExtractor" : "DataSourceV2RelationTableExtractor", @@ -194,31 +223,23 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "QUERY", - "queryDescs" : [ { - "fieldName" : "query", - "fieldExtractor" : "LogicalPlanQueryExtractor" - } ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { - "classname" : "org.apache.spark.sql.catalyst.plans.logical.DeleteFromTable", + "classname" : "org.apache.spark.sql.catalyst.plans.logical.DescribeRelation", "tableDescs" : [ { - "fieldName" : "table", - "fieldExtractor" : "DataSourceV2RelationTableExtractor", + "fieldName" : "relation", + "fieldExtractor" : "ResolvedTableTableExtractor", "columnDesc" : null, - "actionTypeDesc" : { - "fieldName" : null, - "fieldExtractor" : null, - "actionType" : "UPDATE" - }, + "actionTypeDesc" : null, "tableTypeDesc" : null, "catalogDesc" : null, - "isInput" : false, - "setCurrentDatabaseIfMissing" : false + "isInput" : true, + "setCurrentDatabaseIfMissing" : true } ], - "opType" : "QUERY", - "queryDescs" : [ { - "fieldName" : "query", - "fieldExtractor" : "LogicalPlanQueryExtractor" - } ] + "opType" : "DESCTABLE", + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.DropColumns", "tableDescs" : [ { @@ -232,7 +253,8 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "ALTERTABLE_ADDCOLS", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.DropPartitions", "tableDescs" : [ { @@ -246,42 +268,32 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "ALTERTABLE_DROPPARTS", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.DropTable", "tableDescs" : [ { "fieldName" : "child", - "fieldExtractor" : "ResolvedTableTableExtractor", + "fieldExtractor" : "ResolvedIdentifierTableExtractor", "columnDesc" : null, "actionTypeDesc" : null, "tableTypeDesc" : null, "catalogDesc" : null, "isInput" : false, "setCurrentDatabaseIfMissing" : false - } ], - "opType" : "DROPTABLE", - "queryDescs" : [ ] -}, { - "classname" : "org.apache.spark.sql.catalyst.plans.logical.MergeIntoIcebergTable", - "tableDescs" : [ { - "fieldName" : "targetTable", - "fieldExtractor" : "DataSourceV2RelationTableExtractor", + }, { + "fieldName" : "child", + "fieldExtractor" : "ResolvedTableTableExtractor", "columnDesc" : null, - "actionTypeDesc" : { - "fieldName" : null, - "fieldExtractor" : null, - "actionType" : "UPDATE" - }, + "actionTypeDesc" : null, "tableTypeDesc" : null, "catalogDesc" : null, "isInput" : false, "setCurrentDatabaseIfMissing" : false } ], - "opType" : "QUERY", - "queryDescs" : [ { - "fieldName" : "sourceTable", - "fieldExtractor" : "LogicalPlanQueryExtractor" - } ] + "opType" : "DROPTABLE", + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.MergeIntoTable", "tableDescs" : [ { @@ -302,7 +314,8 @@ "queryDescs" : [ { "fieldName" : "sourceTable", "fieldExtractor" : "LogicalPlanQueryExtractor" - } ] + } ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.OverwriteByExpression", "tableDescs" : [ { @@ -323,7 +336,8 @@ "queryDescs" : [ { "fieldName" : "query", "fieldExtractor" : "LogicalPlanQueryExtractor" - } ] + } ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.OverwritePartitionsDynamic", "tableDescs" : [ { @@ -344,7 +358,8 @@ "queryDescs" : [ { "fieldName" : "query", "fieldExtractor" : "LogicalPlanQueryExtractor" - } ] + } ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.RefreshTable", "tableDescs" : [ { @@ -358,7 +373,8 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "QUERY", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.RenameColumn", "tableDescs" : [ { @@ -372,7 +388,8 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "ALTERTABLE_RENAMECOL", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.RenamePartitions", "tableDescs" : [ { @@ -386,7 +403,8 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "ALTERTABLE_RENAMEPART", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.RepairTable", "tableDescs" : [ { @@ -400,7 +418,8 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "MSCK", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.ReplaceColumns", "tableDescs" : [ { @@ -414,10 +433,42 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "ALTERTABLE_REPLACECOLS", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] +}, { + "classname" : "org.apache.spark.sql.catalyst.plans.logical.ReplaceData", + "tableDescs" : [ { + "fieldName" : "originalTable", + "fieldExtractor" : "DataSourceV2RelationTableExtractor", + "columnDesc" : null, + "actionTypeDesc" : { + "fieldName" : null, + "fieldExtractor" : null, + "actionType" : "UPDATE" + }, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : false, + "setCurrentDatabaseIfMissing" : false + } ], + "opType" : "QUERY", + "queryDescs" : [ { + "fieldName" : "query", + "fieldExtractor" : "LogicalPlanQueryExtractor" + } ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.ReplaceTable", "tableDescs" : [ { + "fieldName" : "child", + "fieldExtractor" : "ResolvedIdentifierTableExtractor", + "columnDesc" : null, + "actionTypeDesc" : null, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : false, + "setCurrentDatabaseIfMissing" : false + }, { "fieldName" : "tableName", "fieldExtractor" : "IdentifierTableExtractor", "columnDesc" : null, @@ -440,10 +491,20 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "CREATETABLE", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.ReplaceTableAsSelect", "tableDescs" : [ { + "fieldName" : "name", + "fieldExtractor" : "ResolvedIdentifierTableExtractor", + "columnDesc" : null, + "actionTypeDesc" : null, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : false, + "setCurrentDatabaseIfMissing" : false + }, { "fieldName" : "tableName", "fieldExtractor" : "IdentifierTableExtractor", "columnDesc" : null, @@ -456,7 +517,7 @@ "isInput" : false, "setCurrentDatabaseIfMissing" : false }, { - "fieldName" : "left", + "fieldName" : "name", "fieldExtractor" : "ResolvedDbObjectNameTableExtractor", "columnDesc" : null, "actionTypeDesc" : null, @@ -469,7 +530,23 @@ "queryDescs" : [ { "fieldName" : "query", "fieldExtractor" : "LogicalPlanQueryExtractor" - } ] + } ], + "uriDescs" : [ ] +}, { + "classname" : "org.apache.spark.sql.catalyst.plans.logical.SetTableProperties", + "tableDescs" : [ { + "fieldName" : "table", + "fieldExtractor" : "ResolvedTableTableExtractor", + "columnDesc" : null, + "actionTypeDesc" : null, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : false, + "setCurrentDatabaseIfMissing" : false + } ], + "opType" : "ALTERTABLE_PROPERTIES", + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.ShowCreateTable", "tableDescs" : [ { @@ -483,7 +560,8 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "SHOW_CREATETABLE", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.ShowTableProperties", "tableDescs" : [ { @@ -497,7 +575,8 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "SHOW_TBLPROPERTIES", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.TruncatePartition", "tableDescs" : [ { @@ -511,7 +590,8 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "ALTERTABLE_DROPPARTS", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.TruncateTable", "tableDescs" : [ { @@ -525,49 +605,8 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "TRUNCATETABLE", - "queryDescs" : [ ] -}, { - "classname" : "org.apache.spark.sql.catalyst.plans.logical.UnresolvedMergeIntoIcebergTable", - "tableDescs" : [ { - "fieldName" : "targetTable", - "fieldExtractor" : "DataSourceV2RelationTableExtractor", - "columnDesc" : null, - "actionTypeDesc" : { - "fieldName" : null, - "fieldExtractor" : null, - "actionType" : "UPDATE" - }, - "tableTypeDesc" : null, - "catalogDesc" : null, - "isInput" : false, - "setCurrentDatabaseIfMissing" : false - } ], - "opType" : "QUERY", - "queryDescs" : [ { - "fieldName" : "sourceTable", - "fieldExtractor" : "LogicalPlanQueryExtractor" - } ] -}, { - "classname" : "org.apache.spark.sql.catalyst.plans.logical.UpdateIcebergTable", - "tableDescs" : [ { - "fieldName" : "table", - "fieldExtractor" : "DataSourceV2RelationTableExtractor", - "columnDesc" : null, - "actionTypeDesc" : { - "fieldName" : null, - "fieldExtractor" : null, - "actionType" : "UPDATE" - }, - "tableTypeDesc" : null, - "catalogDesc" : null, - "isInput" : false, - "setCurrentDatabaseIfMissing" : false - } ], - "opType" : "QUERY", - "queryDescs" : [ { - "fieldName" : "query", - "fieldExtractor" : "LogicalPlanQueryExtractor" - } ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.catalyst.plans.logical.UpdateTable", "tableDescs" : [ { @@ -585,10 +624,8 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "QUERY", - "queryDescs" : [ { - "fieldName" : "query", - "fieldExtractor" : "LogicalPlanQueryExtractor" - } ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.execution.command.AlterTableAddColumnsCommand", "tableDescs" : [ { @@ -605,7 +642,8 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "ALTERTABLE_ADDCOLS", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.execution.command.AlterTableAddPartitionCommand", "tableDescs" : [ { @@ -622,7 +660,8 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "ALTERTABLE_ADDPARTS", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.execution.command.AlterTableChangeColumnCommand", "tableDescs" : [ { @@ -639,7 +678,8 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "ALTERTABLE_REPLACECOLS", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.execution.command.AlterTableDropPartitionCommand", "tableDescs" : [ { @@ -656,7 +696,8 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "ALTERTABLE_DROPPARTS", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.execution.command.AlterTableRecoverPartitionsCommand", "tableDescs" : [ { @@ -670,30 +711,14 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "MSCK", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.execution.command.AlterTableRenameCommand", "tableDescs" : [ { "fieldName" : "oldName", "fieldExtractor" : "TableIdentifierTableExtractor", "columnDesc" : null, - "actionTypeDesc" : { - "fieldName" : null, - "fieldExtractor" : null, - "actionType" : "DELETE" - }, - "tableTypeDesc" : { - "fieldName" : "oldName", - "fieldExtractor" : "TableIdentifierTableTypeExtractor", - "skipTypes" : [ "TEMP_VIEW" ] - }, - "catalogDesc" : null, - "isInput" : false, - "setCurrentDatabaseIfMissing" : false - }, { - "fieldName" : "newName", - "fieldExtractor" : "TableIdentifierTableExtractor", - "columnDesc" : null, "actionTypeDesc" : null, "tableTypeDesc" : { "fieldName" : "oldName", @@ -705,7 +730,8 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "ALTERTABLE_RENAME", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.execution.command.AlterTableRenamePartitionCommand", "tableDescs" : [ { @@ -722,7 +748,8 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "ALTERTABLE_RENAMEPART", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.execution.command.AlterTableSerDePropertiesCommand", "tableDescs" : [ { @@ -739,7 +766,8 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "ALTERTABLE_SERDEPROPERTIES", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.execution.command.AlterTableSetLocationCommand", "tableDescs" : [ { @@ -756,7 +784,8 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "ALTERTABLE_LOCATION", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.execution.command.AlterTableSetPropertiesCommand", "tableDescs" : [ { @@ -770,7 +799,8 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "ALTERTABLE_PROPERTIES", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.execution.command.AlterTableUnsetPropertiesCommand", "tableDescs" : [ { @@ -784,7 +814,8 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "ALTERTABLE_PROPERTIES", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.execution.command.AlterViewAsCommand", "tableDescs" : [ { @@ -805,10 +836,20 @@ "queryDescs" : [ { "fieldName" : "query", "fieldExtractor" : "LogicalPlanQueryExtractor" - } ] + } ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.execution.command.AnalyzeColumnCommand", "tableDescs" : [ { + "fieldName" : "tableIdent", + "fieldExtractor" : "TableIdentifierTableExtractor", + "columnDesc" : null, + "actionTypeDesc" : null, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : false, + "setCurrentDatabaseIfMissing" : false + }, { "fieldName" : "tableIdent", "fieldExtractor" : "TableIdentifierTableExtractor", "columnDesc" : { @@ -833,11 +874,21 @@ "isInput" : true, "setCurrentDatabaseIfMissing" : false } ], - "opType" : "ANALYZE_TABLE", - "queryDescs" : [ ] + "opType" : "ALTERTABLE_PROPERTIES", + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.execution.command.AnalyzePartitionCommand", "tableDescs" : [ { + "fieldName" : "tableIdent", + "fieldExtractor" : "TableIdentifierTableExtractor", + "columnDesc" : null, + "actionTypeDesc" : null, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : false, + "setCurrentDatabaseIfMissing" : false + }, { "fieldName" : "tableIdent", "fieldExtractor" : "TableIdentifierTableExtractor", "columnDesc" : { @@ -850,8 +901,9 @@ "isInput" : true, "setCurrentDatabaseIfMissing" : false } ], - "opType" : "ANALYZE_TABLE", - "queryDescs" : [ ] + "opType" : "ALTERTABLE_PROPERTIES", + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.execution.command.AnalyzeTableCommand", "tableDescs" : [ { @@ -861,14 +913,9 @@ "actionTypeDesc" : null, "tableTypeDesc" : null, "catalogDesc" : null, - "isInput" : true, + "isInput" : false, "setCurrentDatabaseIfMissing" : false - } ], - "opType" : "ANALYZE_TABLE", - "queryDescs" : [ ] -}, { - "classname" : "org.apache.spark.sql.execution.command.AnalyzeTablesCommand", - "tableDescs" : [ { + }, { "fieldName" : "tableIdent", "fieldExtractor" : "TableIdentifierTableExtractor", "columnDesc" : null, @@ -878,8 +925,9 @@ "isInput" : true, "setCurrentDatabaseIfMissing" : false } ], - "opType" : "ANALYZE_TABLE", - "queryDescs" : [ ] + "opType" : "ALTERTABLE_PROPERTIES", + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.execution.command.CacheTableCommand", "tableDescs" : [ ], @@ -887,7 +935,8 @@ "queryDescs" : [ { "fieldName" : "plan", "fieldExtractor" : "LogicalPlanOptionQueryExtractor" - } ] + } ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.execution.command.CreateDataSourceTableAsSelectCommand", "tableDescs" : [ { @@ -898,13 +947,14 @@ "tableTypeDesc" : null, "catalogDesc" : null, "isInput" : false, - "setCurrentDatabaseIfMissing" : false + "setCurrentDatabaseIfMissing" : true } ], "opType" : "CREATETABLE_AS_SELECT", "queryDescs" : [ { "fieldName" : "query", "fieldExtractor" : "LogicalPlanQueryExtractor" - } ] + } ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.execution.command.CreateDataSourceTableCommand", "tableDescs" : [ { @@ -915,10 +965,11 @@ "tableTypeDesc" : null, "catalogDesc" : null, "isInput" : false, - "setCurrentDatabaseIfMissing" : false + "setCurrentDatabaseIfMissing" : true } ], "opType" : "CREATETABLE", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.execution.command.CreateTableCommand", "tableDescs" : [ { @@ -929,10 +980,11 @@ "tableTypeDesc" : null, "catalogDesc" : null, "isInput" : false, - "setCurrentDatabaseIfMissing" : false + "setCurrentDatabaseIfMissing" : true } ], "opType" : "CREATETABLE", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.execution.command.CreateTableLikeCommand", "tableDescs" : [ { @@ -955,7 +1007,8 @@ "setCurrentDatabaseIfMissing" : true } ], "opType" : "CREATETABLE", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.execution.command.CreateViewCommand", "tableDescs" : [ { @@ -979,7 +1032,8 @@ }, { "fieldName" : "child", "fieldExtractor" : "LogicalPlanQueryExtractor" - } ] + } ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.execution.command.DescribeColumnCommand", "tableDescs" : [ { @@ -996,7 +1050,8 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "DESCTABLE", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.execution.command.DescribeTableCommand", "tableDescs" : [ { @@ -1013,7 +1068,8 @@ "setCurrentDatabaseIfMissing" : true } ], "opType" : "DESCTABLE", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.execution.command.DropTableCommand", "tableDescs" : [ { @@ -1031,7 +1087,8 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "DROPTABLE", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.execution.command.InsertIntoDataSourceDirCommand", "tableDescs" : [ ], @@ -1039,7 +1096,8 @@ "queryDescs" : [ { "fieldName" : "query", "fieldExtractor" : "LogicalPlanQueryExtractor" - } ] + } ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.execution.command.LoadDataCommand", "tableDescs" : [ { @@ -1060,7 +1118,8 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "LOAD", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.execution.command.RefreshTableCommand", "tableDescs" : [ { @@ -1074,7 +1133,8 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "QUERY", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.execution.command.RepairTableCommand", "tableDescs" : [ { @@ -1088,7 +1148,8 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "MSCK", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.execution.command.ShowColumnsCommand", "tableDescs" : [ { @@ -1102,7 +1163,8 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "SHOWCOLUMNS", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.execution.command.ShowCreateTableAsSerdeCommand", "tableDescs" : [ { @@ -1116,7 +1178,8 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "SHOW_CREATETABLE", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.execution.command.ShowCreateTableCommand", "tableDescs" : [ { @@ -1130,7 +1193,8 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "SHOW_CREATETABLE", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.execution.command.ShowPartitionsCommand", "tableDescs" : [ { @@ -1147,7 +1211,8 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "SHOWPARTITIONS", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.execution.command.ShowTablePropertiesCommand", "tableDescs" : [ { @@ -1161,7 +1226,8 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "SHOW_TBLPROPERTIES", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.execution.command.TruncateTableCommand", "tableDescs" : [ { @@ -1178,12 +1244,32 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "TRUNCATETABLE", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] +}, { + "classname" : "org.apache.spark.sql.execution.datasources.CreateTable", + "tableDescs" : [ { + "fieldName" : "tableDesc", + "fieldExtractor" : "CatalogTableTableExtractor", + "columnDesc" : null, + "actionTypeDesc" : null, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : false, + "setCurrentDatabaseIfMissing" : false + } ], + "opType" : "CREATETABLE", + "queryDescs" : [ { + "fieldName" : "query", + "fieldExtractor" : "LogicalPlanOptionQueryExtractor" + } ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.execution.datasources.CreateTempViewUsing", "tableDescs" : [ ], "opType" : "CREATEVIEW", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.execution.datasources.InsertIntoDataSourceCommand", "tableDescs" : [ { @@ -1204,7 +1290,8 @@ "queryDescs" : [ { "fieldName" : "query", "fieldExtractor" : "LogicalPlanQueryExtractor" - } ] + } ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand", "tableDescs" : [ { @@ -1228,15 +1315,8 @@ "queryDescs" : [ { "fieldName" : "query", "fieldExtractor" : "LogicalPlanQueryExtractor" - } ] -}, { - "classname" : "org.apache.spark.sql.execution.datasources.InsertIntoHiveDirCommand", - "tableDescs" : [ ], - "opType" : "QUERY", - "queryDescs" : [ { - "fieldName" : "query", - "fieldExtractor" : "LogicalPlanQueryExtractor" - } ] + } ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.execution.datasources.RefreshTable", "tableDescs" : [ { @@ -1250,7 +1330,8 @@ "setCurrentDatabaseIfMissing" : false } ], "opType" : "QUERY", - "queryDescs" : [ ] + "queryDescs" : [ ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand", "tableDescs" : [ ], @@ -1258,7 +1339,8 @@ "queryDescs" : [ { "fieldName" : "query", "fieldExtractor" : "LogicalPlanQueryExtractor" - } ] + } ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.hive.execution.CreateHiveTableAsSelectCommand", "tableDescs" : [ { @@ -1278,7 +1360,17 @@ "queryDescs" : [ { "fieldName" : "query", "fieldExtractor" : "LogicalPlanQueryExtractor" - } ] + } ], + "uriDescs" : [ ] +}, { + "classname" : "org.apache.spark.sql.hive.execution.InsertIntoHiveDirCommand", + "tableDescs" : [ ], + "opType" : "QUERY", + "queryDescs" : [ { + "fieldName" : "query", + "fieldExtractor" : "LogicalPlanQueryExtractor" + } ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.hive.execution.InsertIntoHiveTable", "tableDescs" : [ { @@ -1302,7 +1394,8 @@ "queryDescs" : [ { "fieldName" : "query", "fieldExtractor" : "LogicalPlanQueryExtractor" - } ] + } ], + "uriDescs" : [ ] }, { "classname" : "org.apache.spark.sql.hive.execution.OptimizedCreateHiveTableAsSelectCommand", "tableDescs" : [ { @@ -1322,5 +1415,572 @@ "queryDescs" : [ { "fieldName" : "query", "fieldExtractor" : "LogicalPlanQueryExtractor" + } ], + "uriDescs" : [ ] +}, { + "classname" : "org.apache.spark.sql.catalyst.plans.logical.Call", + "tableDescs" : [ { + "fieldName" : "args", + "fieldExtractor" : "ExpressionSeqTableExtractor", + "columnDesc" : null, + "actionTypeDesc" : null, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : false, + "setCurrentDatabaseIfMissing" : false + } ], + "opType" : "ALTERTABLE_PROPERTIES", + "queryDescs" : [ ], + "uriDescs" : [ ] +}, { + "classname" : "org.apache.spark.sql.catalyst.plans.logical.DeleteFromIcebergTable", + "tableDescs" : [ { + "fieldName" : "table", + "fieldExtractor" : "DataSourceV2RelationTableExtractor", + "columnDesc" : null, + "actionTypeDesc" : { + "fieldName" : null, + "fieldExtractor" : null, + "actionType" : "UPDATE" + }, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : false, + "setCurrentDatabaseIfMissing" : false + } ], + "opType" : "QUERY", + "queryDescs" : [ ], + "uriDescs" : [ ] +}, { + "classname" : "org.apache.spark.sql.catalyst.plans.logical.MergeIntoIcebergTable", + "tableDescs" : [ { + "fieldName" : "targetTable", + "fieldExtractor" : "DataSourceV2RelationTableExtractor", + "columnDesc" : null, + "actionTypeDesc" : { + "fieldName" : null, + "fieldExtractor" : null, + "actionType" : "UPDATE" + }, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : false, + "setCurrentDatabaseIfMissing" : false + } ], + "opType" : "QUERY", + "queryDescs" : [ { + "fieldName" : "sourceTable", + "fieldExtractor" : "LogicalPlanQueryExtractor" + } ], + "uriDescs" : [ ] +}, { + "classname" : "org.apache.spark.sql.catalyst.plans.logical.UnresolvedMergeIntoIcebergTable", + "tableDescs" : [ { + "fieldName" : "targetTable", + "fieldExtractor" : "DataSourceV2RelationTableExtractor", + "columnDesc" : null, + "actionTypeDesc" : { + "fieldName" : null, + "fieldExtractor" : null, + "actionType" : "UPDATE" + }, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : false, + "setCurrentDatabaseIfMissing" : false + } ], + "opType" : "QUERY", + "queryDescs" : [ { + "fieldName" : "sourceTable", + "fieldExtractor" : "LogicalPlanQueryExtractor" + } ], + "uriDescs" : [ ] +}, { + "classname" : "org.apache.spark.sql.catalyst.plans.logical.UpdateIcebergTable", + "tableDescs" : [ { + "fieldName" : "table", + "fieldExtractor" : "DataSourceV2RelationTableExtractor", + "columnDesc" : null, + "actionTypeDesc" : { + "fieldName" : null, + "fieldExtractor" : null, + "actionType" : "UPDATE" + }, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : false, + "setCurrentDatabaseIfMissing" : false + } ], + "opType" : "QUERY", + "queryDescs" : [ ], + "uriDescs" : [ ] +}, { + "classname" : "org.apache.spark.sql.hudi.command.AlterHoodieTableAddColumnsCommand", + "tableDescs" : [ { + "fieldName" : "tableId", + "fieldExtractor" : "TableIdentifierTableExtractor", + "columnDesc" : { + "fieldName" : "colsToAdd", + "fieldExtractor" : "StructFieldSeqColumnExtractor" + }, + "actionTypeDesc" : null, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : false, + "setCurrentDatabaseIfMissing" : false + } ], + "opType" : "ALTERTABLE_ADDCOLS", + "queryDescs" : [ ], + "uriDescs" : [ ] +}, { + "classname" : "org.apache.spark.sql.hudi.command.AlterHoodieTableChangeColumnCommand", + "tableDescs" : [ { + "fieldName" : "tableIdentifier", + "fieldExtractor" : "TableIdentifierTableExtractor", + "columnDesc" : { + "fieldName" : "columnName", + "fieldExtractor" : "StringColumnExtractor" + }, + "actionTypeDesc" : null, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : false, + "setCurrentDatabaseIfMissing" : false + } ], + "opType" : "ALTERTABLE_REPLACECOLS", + "queryDescs" : [ ], + "uriDescs" : [ ] +}, { + "classname" : "org.apache.spark.sql.hudi.command.AlterHoodieTableDropPartitionCommand", + "tableDescs" : [ { + "fieldName" : "tableIdentifier", + "fieldExtractor" : "TableIdentifierTableExtractor", + "columnDesc" : { + "fieldName" : "partitionSpecs", + "fieldExtractor" : "PartitionSeqColumnExtractor" + }, + "actionTypeDesc" : null, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : false, + "setCurrentDatabaseIfMissing" : false + } ], + "opType" : "ALTERTABLE_DROPPARTS", + "queryDescs" : [ ], + "uriDescs" : [ ] +}, { + "classname" : "org.apache.spark.sql.hudi.command.AlterHoodieTableRenameCommand", + "tableDescs" : [ { + "fieldName" : "oldName", + "fieldExtractor" : "TableIdentifierTableExtractor", + "columnDesc" : null, + "actionTypeDesc" : null, + "tableTypeDesc" : { + "fieldName" : "oldName", + "fieldExtractor" : "TableIdentifierTableTypeExtractor", + "skipTypes" : [ "TEMP_VIEW" ] + }, + "catalogDesc" : null, + "isInput" : false, + "setCurrentDatabaseIfMissing" : false + } ], + "opType" : "ALTERTABLE_RENAME", + "queryDescs" : [ ], + "uriDescs" : [ ] +}, { + "classname" : "org.apache.spark.sql.hudi.command.AlterTableCommand", + "tableDescs" : [ { + "fieldName" : "table", + "fieldExtractor" : "CatalogTableTableExtractor", + "columnDesc" : null, + "actionTypeDesc" : null, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : false, + "setCurrentDatabaseIfMissing" : false + } ], + "opType" : "ALTERTABLE_PROPERTIES", + "queryDescs" : [ ], + "uriDescs" : [ ] +}, { + "classname" : "org.apache.spark.sql.hudi.command.CallProcedureHoodieCommand", + "tableDescs" : [ { + "fieldName" : "clone", + "fieldExtractor" : "HudiCallProcedureInputTableExtractor", + "columnDesc" : null, + "actionTypeDesc" : { + "fieldName" : null, + "fieldExtractor" : null, + "actionType" : "OTHER" + }, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : true, + "setCurrentDatabaseIfMissing" : true + }, { + "fieldName" : "clone", + "fieldExtractor" : "HudiCallProcedureOutputTableExtractor", + "columnDesc" : null, + "actionTypeDesc" : { + "fieldName" : null, + "fieldExtractor" : null, + "actionType" : "UPDATE" + }, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : false, + "setCurrentDatabaseIfMissing" : true + } ], + "opType" : "QUERY", + "queryDescs" : [ ], + "uriDescs" : [ ] +}, { + "classname" : "org.apache.spark.sql.hudi.command.CompactionHoodiePathCommand", + "tableDescs" : [ ], + "opType" : "CREATETABLE", + "queryDescs" : [ ], + "uriDescs" : [ { + "fieldName" : "path", + "fieldExtractor" : "StringURIExtractor", + "isInput" : false + } ] +}, { + "classname" : "org.apache.spark.sql.hudi.command.CompactionHoodieTableCommand", + "tableDescs" : [ { + "fieldName" : "table", + "fieldExtractor" : "CatalogTableTableExtractor", + "columnDesc" : null, + "actionTypeDesc" : null, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : false, + "setCurrentDatabaseIfMissing" : false + } ], + "opType" : "CREATETABLE", + "queryDescs" : [ ], + "uriDescs" : [ ] +}, { + "classname" : "org.apache.spark.sql.hudi.command.CompactionShowHoodiePathCommand", + "tableDescs" : [ ], + "opType" : "SHOW_TBLPROPERTIES", + "queryDescs" : [ ], + "uriDescs" : [ { + "fieldName" : "path", + "fieldExtractor" : "StringURIExtractor", + "isInput" : true } ] +}, { + "classname" : "org.apache.spark.sql.hudi.command.CompactionShowHoodieTableCommand", + "tableDescs" : [ { + "fieldName" : "table", + "fieldExtractor" : "CatalogTableTableExtractor", + "columnDesc" : null, + "actionTypeDesc" : null, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : true, + "setCurrentDatabaseIfMissing" : false + } ], + "opType" : "SHOW_TBLPROPERTIES", + "queryDescs" : [ ], + "uriDescs" : [ ] +}, { + "classname" : "org.apache.spark.sql.hudi.command.CreateHoodieTableAsSelectCommand", + "tableDescs" : [ { + "fieldName" : "table", + "fieldExtractor" : "CatalogTableTableExtractor", + "columnDesc" : null, + "actionTypeDesc" : null, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : false, + "setCurrentDatabaseIfMissing" : false + } ], + "opType" : "CREATETABLE_AS_SELECT", + "queryDescs" : [ { + "fieldName" : "query", + "fieldExtractor" : "LogicalPlanQueryExtractor" + } ], + "uriDescs" : [ ] +}, { + "classname" : "org.apache.spark.sql.hudi.command.CreateHoodieTableCommand", + "tableDescs" : [ { + "fieldName" : "table", + "fieldExtractor" : "CatalogTableTableExtractor", + "columnDesc" : null, + "actionTypeDesc" : null, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : false, + "setCurrentDatabaseIfMissing" : false + } ], + "opType" : "CREATETABLE", + "queryDescs" : [ ], + "uriDescs" : [ ] +}, { + "classname" : "org.apache.spark.sql.hudi.command.CreateHoodieTableLikeCommand", + "tableDescs" : [ { + "fieldName" : "targetTable", + "fieldExtractor" : "TableIdentifierTableExtractor", + "columnDesc" : null, + "actionTypeDesc" : null, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : false, + "setCurrentDatabaseIfMissing" : true + }, { + "fieldName" : "sourceTable", + "fieldExtractor" : "TableIdentifierTableExtractor", + "columnDesc" : null, + "actionTypeDesc" : null, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : true, + "setCurrentDatabaseIfMissing" : true + } ], + "opType" : "CREATETABLE", + "queryDescs" : [ ], + "uriDescs" : [ ] +}, { + "classname" : "org.apache.spark.sql.hudi.command.CreateIndexCommand", + "tableDescs" : [ { + "fieldName" : "table", + "fieldExtractor" : "CatalogTableTableExtractor", + "columnDesc" : null, + "actionTypeDesc" : null, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : false, + "setCurrentDatabaseIfMissing" : false + } ], + "opType" : "CREATEINDEX", + "queryDescs" : [ ], + "uriDescs" : [ ] +}, { + "classname" : "org.apache.spark.sql.hudi.command.DeleteHoodieTableCommand", + "tableDescs" : [ { + "fieldName" : "dft", + "fieldExtractor" : "HudiDataSourceV2RelationTableExtractor", + "columnDesc" : null, + "actionTypeDesc" : { + "fieldName" : null, + "fieldExtractor" : null, + "actionType" : "UPDATE" + }, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : false, + "setCurrentDatabaseIfMissing" : false + } ], + "opType" : "QUERY", + "queryDescs" : [ ], + "uriDescs" : [ ] +}, { + "classname" : "org.apache.spark.sql.hudi.command.DropHoodieTableCommand", + "tableDescs" : [ { + "fieldName" : "tableIdentifier", + "fieldExtractor" : "TableIdentifierTableExtractor", + "columnDesc" : null, + "actionTypeDesc" : null, + "tableTypeDesc" : { + "fieldName" : "tableIdentifier", + "fieldExtractor" : "TableIdentifierTableTypeExtractor", + "skipTypes" : [ "TEMP_VIEW" ] + }, + "catalogDesc" : null, + "isInput" : false, + "setCurrentDatabaseIfMissing" : false + } ], + "opType" : "DROPTABLE", + "queryDescs" : [ ], + "uriDescs" : [ ] +}, { + "classname" : "org.apache.spark.sql.hudi.command.DropIndexCommand", + "tableDescs" : [ { + "fieldName" : "table", + "fieldExtractor" : "CatalogTableTableExtractor", + "columnDesc" : null, + "actionTypeDesc" : null, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : false, + "setCurrentDatabaseIfMissing" : false + } ], + "opType" : "DROPINDEX", + "queryDescs" : [ ], + "uriDescs" : [ ] +}, { + "classname" : "org.apache.spark.sql.hudi.command.InsertIntoHoodieTableCommand", + "tableDescs" : [ { + "fieldName" : "logicalRelation", + "fieldExtractor" : "LogicalRelationTableExtractor", + "columnDesc" : null, + "actionTypeDesc" : { + "fieldName" : "overwrite", + "fieldExtractor" : "OverwriteOrInsertActionTypeExtractor", + "actionType" : null + }, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : false, + "setCurrentDatabaseIfMissing" : false + } ], + "opType" : "QUERY", + "queryDescs" : [ { + "fieldName" : "query", + "fieldExtractor" : "LogicalPlanQueryExtractor" + } ], + "uriDescs" : [ ] +}, { + "classname" : "org.apache.spark.sql.hudi.command.MergeIntoHoodieTableCommand", + "tableDescs" : [ { + "fieldName" : "mergeInto", + "fieldExtractor" : "HudiMergeIntoTargetTableExtractor", + "columnDesc" : null, + "actionTypeDesc" : { + "fieldName" : null, + "fieldExtractor" : null, + "actionType" : "UPDATE" + }, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : false, + "setCurrentDatabaseIfMissing" : false + } ], + "opType" : "QUERY", + "queryDescs" : [ { + "fieldName" : "mergeInto", + "fieldExtractor" : "HudiMergeIntoSourceTableExtractor" + } ], + "uriDescs" : [ ] +}, { + "classname" : "org.apache.spark.sql.hudi.command.RefreshIndexCommand", + "tableDescs" : [ { + "fieldName" : "table", + "fieldExtractor" : "CatalogTableTableExtractor", + "columnDesc" : null, + "actionTypeDesc" : null, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : false, + "setCurrentDatabaseIfMissing" : false + } ], + "opType" : "ALTERINDEX_REBUILD", + "queryDescs" : [ ], + "uriDescs" : [ ] +}, { + "classname" : "org.apache.spark.sql.hudi.command.RepairHoodieTableCommand", + "tableDescs" : [ { + "fieldName" : "tableName", + "fieldExtractor" : "TableIdentifierTableExtractor", + "columnDesc" : null, + "actionTypeDesc" : null, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : false, + "setCurrentDatabaseIfMissing" : false + } ], + "opType" : "MSCK", + "queryDescs" : [ ], + "uriDescs" : [ ] +}, { + "classname" : "org.apache.spark.sql.hudi.command.ShowHoodieTablePartitionsCommand", + "tableDescs" : [ { + "fieldName" : "tableIdentifier", + "fieldExtractor" : "TableIdentifierTableExtractor", + "columnDesc" : { + "fieldName" : "specOpt", + "fieldExtractor" : "PartitionOptionColumnExtractor" + }, + "actionTypeDesc" : null, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : true, + "setCurrentDatabaseIfMissing" : false + } ], + "opType" : "SHOWPARTITIONS", + "queryDescs" : [ ], + "uriDescs" : [ ] +}, { + "classname" : "org.apache.spark.sql.hudi.command.ShowIndexesCommand", + "tableDescs" : [ { + "fieldName" : "table", + "fieldExtractor" : "CatalogTableTableExtractor", + "columnDesc" : null, + "actionTypeDesc" : null, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : true, + "setCurrentDatabaseIfMissing" : false + } ], + "opType" : "SHOWINDEXES", + "queryDescs" : [ ], + "uriDescs" : [ ] +}, { + "classname" : "org.apache.spark.sql.hudi.command.Spark31AlterTableCommand", + "tableDescs" : [ { + "fieldName" : "table", + "fieldExtractor" : "CatalogTableTableExtractor", + "columnDesc" : null, + "actionTypeDesc" : null, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : false, + "setCurrentDatabaseIfMissing" : false + } ], + "opType" : "ALTERTABLE_PROPERTIES", + "queryDescs" : [ ], + "uriDescs" : [ ] +}, { + "classname" : "org.apache.spark.sql.hudi.command.TruncateHoodieTableCommand", + "tableDescs" : [ { + "fieldName" : "tableIdentifier", + "fieldExtractor" : "TableIdentifierTableExtractor", + "columnDesc" : { + "fieldName" : "partitionSpec", + "fieldExtractor" : "PartitionOptionColumnExtractor" + }, + "actionTypeDesc" : null, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : false, + "setCurrentDatabaseIfMissing" : false + } ], + "opType" : "TRUNCATETABLE", + "queryDescs" : [ ], + "uriDescs" : [ ] +}, { + "classname" : "org.apache.spark.sql.hudi.command.UpdateHoodieTableCommand", + "tableDescs" : [ { + "fieldName" : "ut", + "fieldExtractor" : "HudiDataSourceV2RelationTableExtractor", + "columnDesc" : null, + "actionTypeDesc" : { + "fieldName" : null, + "fieldExtractor" : null, + "actionType" : "UPDATE" + }, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : false, + "setCurrentDatabaseIfMissing" : false + } ], + "opType" : "QUERY", + "queryDescs" : [ ], + "uriDescs" : [ ] +}, { + "classname" : "org.apache.spark.sql.delta.commands.CreateDeltaTableCommand", + "tableDescs" : [ { + "fieldName" : "table", + "fieldExtractor" : "CatalogTableTableExtractor", + "columnDesc" : null, + "actionTypeDesc" : null, + "tableTypeDesc" : null, + "catalogDesc" : null, + "isInput" : false, + "setCurrentDatabaseIfMissing" : false + } ], + "opType" : "CREATETABLE", + "queryDescs" : [ ], + "uriDescs" : [ ] } ] \ No newline at end of file diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ObjectType.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ObjectType.scala index 39f03147e..fe53440c1 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ObjectType.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ObjectType.scala @@ -23,15 +23,17 @@ object ObjectType extends Enumeration { type ObjectType = Value - val DATABASE, TABLE, VIEW, COLUMN, FUNCTION = Value + val DATABASE, TABLE, VIEW, COLUMN, FUNCTION, INDEX, URI = Value def apply(obj: PrivilegeObject, opType: OperationType): ObjectType = { obj.privilegeObjectType match { case PrivilegeObjectType.DATABASE => DATABASE + case PrivilegeObjectType.TABLE_OR_VIEW if opType.toString.contains("INDEX") => INDEX case PrivilegeObjectType.TABLE_OR_VIEW if obj.columns.nonEmpty => COLUMN case PrivilegeObjectType.TABLE_OR_VIEW if opType.toString.contains("VIEW") => VIEW case PrivilegeObjectType.TABLE_OR_VIEW => TABLE case PrivilegeObjectType.FUNCTION => FUNCTION + case PrivilegeObjectType.DFS_URL | PrivilegeObjectType.LOCAL_URI => URI } } } diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/OperationType.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/OperationType.scala index 046ab3e2a..3f2062b20 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/OperationType.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/OperationType.scala @@ -20,7 +20,8 @@ package org.apache.kyuubi.plugin.spark.authz object OperationType extends Enumeration { type OperationType = Value - + // According to https://scalameta.org/scalafmt/docs/known-issues.html + // format: off val ALTERDATABASE, ALTERDATABASE_LOCATION, ALTERTABLE_ADDCOLS, ALTERTABLE_ADDPARTS, ALTERTABLE_RENAMECOL, ALTERTABLE_REPLACECOLS, ALTERTABLE_DROPPARTS, ALTERTABLE_RENAMEPART, ALTERTABLE_RENAME, ALTERTABLE_PROPERTIES, ALTERTABLE_SERDEPROPERTIES, ALTERTABLE_LOCATION, @@ -28,5 +29,7 @@ object OperationType extends Enumeration { CREATETABLE_AS_SELECT, CREATEFUNCTION, CREATEVIEW, DESCDATABASE, DESCFUNCTION, DESCTABLE, DROPDATABASE, DROPFUNCTION, DROPTABLE, DROPVIEW, EXPLAIN, LOAD, MSCK, QUERY, RELOADFUNCTION, SHOWCONF, SHOW_CREATETABLE, SHOWCOLUMNS, SHOWDATABASES, SHOWFUNCTIONS, SHOWPARTITIONS, - SHOWTABLES, SHOW_TBLPROPERTIES, SWITCHDATABASE, TRUNCATETABLE = Value + SHOWTABLES, SHOW_TBLPROPERTIES, SWITCHDATABASE, TRUNCATETABLE, + CREATEINDEX, DROPINDEX, ALTERINDEX_REBUILD, SHOWINDEXES = Value + // format: on } diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/PrivilegeObject.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/PrivilegeObject.scala index 195aa7989..0fe145b4a 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/PrivilegeObject.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/PrivilegeObject.scala @@ -17,11 +17,12 @@ package org.apache.kyuubi.plugin.spark.authz +import java.net.URI import javax.annotation.Nonnull import org.apache.kyuubi.plugin.spark.authz.PrivilegeObjectActionType.PrivilegeObjectActionType import org.apache.kyuubi.plugin.spark.authz.PrivilegeObjectType._ -import org.apache.kyuubi.plugin.spark.authz.serde.{Database, Function, Table} +import org.apache.kyuubi.plugin.spark.authz.serde.{Database, Function, Table, Uri} /** * Build a Spark logical plan to different `PrivilegeObject`s @@ -86,4 +87,19 @@ object PrivilegeObject { None ) // TODO: Support catalog for function } + + def apply(uri: Uri): PrivilegeObject = { + val privilegeObjectType = Option(new URI(uri.path).getScheme) match { + case Some("file") => LOCAL_URI + case _ => DFS_URL + } + new PrivilegeObject( + privilegeObjectType, + PrivilegeObjectActionType.OTHER, + uri.path, + null, + Nil, + None, + None) + } } diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/PrivilegeObjectType.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/PrivilegeObjectType.scala index f514fcb82..4020392f2 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/PrivilegeObjectType.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/PrivilegeObjectType.scala @@ -20,5 +20,5 @@ package org.apache.kyuubi.plugin.spark.authz object PrivilegeObjectType extends Enumeration { type PrivilegeObjectType = Value - val DATABASE, TABLE_OR_VIEW, FUNCTION = Value + val DATABASE, TABLE_OR_VIEW, FUNCTION, LOCAL_URI, DFS_URL = Value } diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/PrivilegesBuilder.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/PrivilegesBuilder.scala index cee79b87d..833499280 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/PrivilegesBuilder.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/PrivilegesBuilder.scala @@ -26,8 +26,11 @@ import org.slf4j.LoggerFactory import org.apache.kyuubi.plugin.spark.authz.OperationType.OperationType import org.apache.kyuubi.plugin.spark.authz.PrivilegeObjectActionType._ +import org.apache.kyuubi.plugin.spark.authz.rule.Authorization.KYUUBI_AUTHZ_TAG +import org.apache.kyuubi.plugin.spark.authz.rule.permanentview.PermanentViewMarker import org.apache.kyuubi.plugin.spark.authz.serde._ import org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils._ +import org.apache.kyuubi.util.reflect.ReflectUtils._ object PrivilegesBuilder { @@ -63,7 +66,13 @@ object PrivilegesBuilder { def mergeProjection(table: Table, plan: LogicalPlan): Unit = { if (projectionList.isEmpty) { - privilegeObjects += PrivilegeObject(table, plan.output.map(_.name)) + plan match { + case pvm: PermanentViewMarker + if pvm.isSubqueryExpressionPlaceHolder || pvm.output.isEmpty => + privilegeObjects += PrivilegeObject(table, pvm.outputColNames) + case _ => + privilegeObjects += PrivilegeObject(table, plan.output.map(_.name)) + } } else { val cols = (projectionList ++ conditionList).flatMap(collectLeaves) .filter(plan.outputSet.contains).map(_.name).distinct @@ -72,6 +81,8 @@ object PrivilegesBuilder { } plan match { + case p if p.getTagValue(KYUUBI_AUTHZ_TAG).nonEmpty => + case p: Project => buildQuery(p.child, privilegeObjects, p.projectList, conditionList, spark) case j: Join => @@ -95,6 +106,12 @@ object PrivilegesBuilder { val cols = conditionList ++ sortCols buildQuery(s.child, privilegeObjects, projectionList, cols, spark) + case a: Aggregate => + val aggCols = + (a.aggregateExpressions ++ a.groupingExpressions).flatMap(e => collectLeaves(e)) + val cols = conditionList ++ aggCols + buildQuery(a.child, privilegeObjects, projectionList, cols, spark) + case scan if isKnownScan(scan) && scan.resolved => getScanSpec(scan).tables(scan, spark).foreach(mergeProjection(_, scan)) @@ -144,7 +161,7 @@ object PrivilegesBuilder { } } catch { case e: Exception => - LOG.warn(tableDesc.error(plan, e)) + LOG.debug(tableDesc.error(plan, e)) Nil } } @@ -162,7 +179,7 @@ object PrivilegesBuilder { } } catch { case e: Exception => - LOG.warn(databaseDesc.error(plan, e)) + LOG.debug(databaseDesc.error(plan, e)) } } desc.operationType @@ -176,6 +193,23 @@ object PrivilegesBuilder { outputObjs ++= getTablePriv(td) } } + spec.uriDescs.foreach { ud => + try { + val uri = ud.extract(plan) + uri match { + case Some(uri) => + if (ud.isInput) { + inputObjs += PrivilegeObject(uri) + } else { + outputObjs += PrivilegeObject(uri) + } + case None => + } + } catch { + case e: Exception => + LOG.debug(ud.error(plan, e)) + } + } spec.queries(plan).foreach(buildQuery(_, inputObjs, spark = spark)) spec.operationType @@ -193,7 +227,7 @@ object PrivilegesBuilder { } } catch { case e: Exception => - LOG.warn(fd.error(plan, e)) + LOG.debug(fd.error(plan, e)) } } spec.operationType @@ -202,7 +236,39 @@ object PrivilegesBuilder { } } - type PrivilegesAndOpType = (Seq[PrivilegeObject], Seq[PrivilegeObject], OperationType) + type PrivilegesAndOpType = (Iterable[PrivilegeObject], Iterable[PrivilegeObject], OperationType) + + /** + * Build input privilege objects from a Spark's LogicalPlan for hive permanent udf + * + * @param plan A Spark LogicalPlan + */ + def buildFunctions( + plan: LogicalPlan, + spark: SparkSession): PrivilegesAndOpType = { + val inputObjs = new ArrayBuffer[PrivilegeObject] + plan match { + case command: Command if isKnownTableCommand(command) => + val spec = getTableCommandSpec(command) + val functionPrivAndOpType = spec.queries(plan) + .map(plan => buildFunctions(plan, spark)) + functionPrivAndOpType.map(_._1) + .reduce(_ ++ _) + .foreach(functionPriv => inputObjs += functionPriv) + + case plan => plan transformAllExpressions { + case hiveFunction: Expression if isKnownFunction(hiveFunction) => + val functionSpec: ScanSpec = getFunctionSpec(hiveFunction) + if (functionSpec.functionDescs + .exists(!_.functionTypeDesc.get.skip(hiveFunction, spark))) { + functionSpec.functions(hiveFunction).foreach(func => + inputObjs += PrivilegeObject(func)) + } + hiveFunction + } + } + (inputObjs, Seq.empty, OperationType.QUERY) + } /** * Build input and output privilege objects from a Spark's LogicalPlan diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/AccessRequest.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/AccessRequest.scala index 4997dda3b..8fc8028e6 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/AccessRequest.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/AccessRequest.scala @@ -27,7 +27,7 @@ import org.apache.ranger.plugin.policyengine.{RangerAccessRequestImpl, RangerPol import org.apache.kyuubi.plugin.spark.authz.OperationType.OperationType import org.apache.kyuubi.plugin.spark.authz.ranger.AccessType._ -import org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils.{invoke, invokeAs} +import org.apache.kyuubi.util.reflect.ReflectUtils._ case class AccessRequest private (accessType: AccessType) extends RangerAccessRequestImpl @@ -50,7 +50,7 @@ object AccessRequest { "getRolesFromUserAndGroups", (classOf[String], userName), (classOf[JSet[String]], userGroups)) - invoke(req, "setUserRoles", (classOf[JSet[String]], roles)) + invokeAs[Unit](req, "setUserRoles", (classOf[JSet[String]], roles)) } catch { case _: Exception => } @@ -61,7 +61,7 @@ object AccessRequest { } try { val clusterName = invokeAs[String](SparkRangerAdminPlugin, "getClusterName") - invoke(req, "setClusterName", (classOf[String], clusterName)) + invokeAs[Unit](req, "setClusterName", (classOf[String], clusterName)) } catch { case _: Exception => } @@ -74,8 +74,8 @@ object AccessRequest { private def getUserGroupsFromUserStore(user: UserGroupInformation): Option[JSet[String]] = { try { - val storeEnricher = invoke(SparkRangerAdminPlugin, "getUserStoreEnricher") - val userStore = invoke(storeEnricher, "getRangerUserStore") + val storeEnricher = invokeAs[AnyRef](SparkRangerAdminPlugin, "getUserStoreEnricher") + val userStore = invokeAs[AnyRef](storeEnricher, "getRangerUserStore") val userGroupMapping = invokeAs[JHashMap[String, JSet[String]]](userStore, "getUserGroupMapping") Some(userGroupMapping.get(user.getShortUserName)) diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/AccessResource.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/AccessResource.scala index 47a0292c7..858dc1c37 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/AccessResource.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/AccessResource.scala @@ -17,6 +17,9 @@ package org.apache.kyuubi.plugin.spark.authz.ranger +import java.io.File +import java.util + import scala.language.implicitConversions import org.apache.ranger.plugin.policyengine.RangerAccessResourceImpl @@ -35,6 +38,7 @@ class AccessResource private (val objectType: ObjectType, val catalog: Option[St val columnStr = getColumn if (columnStr == null) Nil else columnStr.split(",").filter(_.nonEmpty) } + def getUrl: String = getValue("url") } object AccessResource { @@ -57,9 +61,19 @@ object AccessResource { resource.setValue("database", firstLevelResource) resource.setValue("table", secondLevelResource) resource.setValue("column", thirdLevelResource) - case TABLE | VIEW => // fixme spark have added index support + case TABLE | VIEW | INDEX => resource.setValue("database", firstLevelResource) resource.setValue("table", secondLevelResource) + case URI => + val objectList = new util.ArrayList[String] + Option(firstLevelResource) + .filter(_.nonEmpty) + .foreach { path => + val s = path.stripSuffix(File.separator) + objectList.add(s) + objectList.add(s + File.separator) + } + resource.setValue("url", objectList) } resource.setServiceDef(SparkRangerAdminPlugin.getServiceDef) owner.foreach(resource.setOwnerUser) diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/AccessType.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/AccessType.scala index 52e3c0176..7f1ddb68e 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/AccessType.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/AccessType.scala @@ -25,7 +25,7 @@ object AccessType extends Enumeration { type AccessType = Value - val NONE, CREATE, ALTER, DROP, SELECT, UPDATE, USE, READ, WRITE, ALL, ADMIN = Value + val NONE, CREATE, ALTER, DROP, SELECT, UPDATE, USE, READ, WRITE, ALL, ADMIN, INDEX = Value def apply(obj: PrivilegeObject, opType: OperationType, isInput: Boolean): AccessType = { obj.actionType match { @@ -35,32 +35,42 @@ object AccessType extends Enumeration { case CREATETABLE | CREATEVIEW | CREATETABLE_AS_SELECT if obj.privilegeObjectType == TABLE_OR_VIEW => if (isInput) SELECT else CREATE - // new table new `CREATE` privilege here and the old table gets `DELETE` via actionType - case ALTERTABLE_RENAME => CREATE + case CREATETABLE + if obj.privilegeObjectType == DFS_URL || obj.privilegeObjectType == LOCAL_URI => + if (isInput) SELECT else CREATE case ALTERDATABASE | ALTERDATABASE_LOCATION | ALTERTABLE_ADDCOLS | ALTERTABLE_ADDPARTS | ALTERTABLE_DROPPARTS | ALTERTABLE_LOCATION | + ALTERTABLE_RENAME | ALTERTABLE_PROPERTIES | ALTERTABLE_RENAMECOL | ALTERTABLE_RENAMEPART | ALTERTABLE_REPLACECOLS | ALTERTABLE_SERDEPROPERTIES | ALTERVIEW_RENAME | - MSCK => ALTER + MSCK | + ALTERINDEX_REBUILD => ALTER case ALTERVIEW_AS => if (isInput) SELECT else ALTER - case DROPDATABASE | DROPTABLE | DROPFUNCTION | DROPVIEW => DROP + case DROPDATABASE | DROPTABLE | DROPFUNCTION | DROPVIEW | DROPINDEX => DROP case LOAD => if (isInput) SELECT else UPDATE case QUERY | SHOW_CREATETABLE | SHOW_TBLPROPERTIES | SHOWPARTITIONS | + SHOWINDEXES | ANALYZE_TABLE => SELECT case SHOWCOLUMNS | DESCTABLE => SELECT - case SHOWDATABASES | SWITCHDATABASE | DESCDATABASE | SHOWTABLES | SHOWFUNCTIONS => USE + case SHOWDATABASES | + SWITCHDATABASE | + DESCDATABASE | + SHOWTABLES | + SHOWFUNCTIONS | + DESCFUNCTION => USE case TRUNCATETABLE => UPDATE + case CREATEINDEX => INDEX case _ => NONE } case PrivilegeObjectActionType.DELETE => DROP diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/RangerConfigProvider.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerConfigProvider.scala similarity index 78% rename from extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/RangerConfigProvider.scala rename to extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerConfigProvider.scala index 83fe048e6..05d8cc64f 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/RangerConfigProvider.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerConfigProvider.scala @@ -15,11 +15,12 @@ * limitations under the License. */ -package org.apache.kyuubi.plugin.spark.authz.util +package org.apache.kyuubi.plugin.spark.authz.ranger import org.apache.hadoop.conf.Configuration -import org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils._ +import org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils.isRanger21orGreater +import org.apache.kyuubi.util.reflect.ReflectUtils.invokeAs trait RangerConfigProvider { @@ -33,15 +34,13 @@ trait RangerConfigProvider { * org.apache.ranger.authorization.hadoop.config.RangerConfiguration * for Ranger 2.0 and below */ - def getRangerConf: Configuration = { + val getRangerConf: Configuration = { if (isRanger21orGreater) { // for Ranger 2.1+ - invokeAs[Configuration](this, "getConfig") + invokeAs(this, "getConfig") } else { // for Ranger 2.0 and below - invokeStaticAs[Configuration]( - Class.forName("org.apache.ranger.authorization.hadoop.config.RangerConfiguration"), - "getInstance") + invokeAs("org.apache.ranger.authorization.hadoop.config.RangerConfiguration", "getInstance") } } } diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerSparkExtension.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerSparkExtension.scala index f4dcb3f9f..01645ff97 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerSparkExtension.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerSparkExtension.scala @@ -19,7 +19,11 @@ package org.apache.kyuubi.plugin.spark.authz.ranger import org.apache.spark.sql.SparkSessionExtensions -import org.apache.kyuubi.plugin.spark.authz.util.{RuleEliminateMarker, RuleEliminateViewMarker} +import org.apache.kyuubi.plugin.spark.authz.rule.{RuleEliminateMarker, RuleEliminatePermanentViewMarker} +import org.apache.kyuubi.plugin.spark.authz.rule.config.AuthzConfigurationChecker +import org.apache.kyuubi.plugin.spark.authz.rule.datamasking.{RuleApplyDataMaskingStage0, RuleApplyDataMaskingStage1} +import org.apache.kyuubi.plugin.spark.authz.rule.permanentview.RuleApplyPermanentViewMarker +import org.apache.kyuubi.plugin.spark.authz.rule.rowfilter.{FilterDataSourceV2Strategy, RuleApplyRowFilter, RuleReplaceShowObjectCommands} /** * ACL Management for Apache Spark SQL with Apache Ranger, enabling: @@ -36,16 +40,18 @@ import org.apache.kyuubi.plugin.spark.authz.util.{RuleEliminateMarker, RuleElimi * @since 1.6.0 */ class RangerSparkExtension extends (SparkSessionExtensions => Unit) { - SparkRangerAdminPlugin.init() + SparkRangerAdminPlugin.initialize() override def apply(v1: SparkSessionExtensions): Unit = { v1.injectCheckRule(AuthzConfigurationChecker) v1.injectResolutionRule(_ => new RuleReplaceShowObjectCommands()) v1.injectResolutionRule(_ => new RuleApplyPermanentViewMarker()) - v1.injectResolutionRule(new RuleApplyRowFilterAndDataMasking(_)) + v1.injectResolutionRule(RuleApplyRowFilter) + v1.injectResolutionRule(RuleApplyDataMaskingStage0) + v1.injectResolutionRule(RuleApplyDataMaskingStage1) v1.injectOptimizerRule(_ => new RuleEliminateMarker()) v1.injectOptimizerRule(new RuleAuthorization(_)) - v1.injectOptimizerRule(_ => new RuleEliminateViewMarker()) + v1.injectOptimizerRule(_ => new RuleEliminatePermanentViewMarker()) v1.injectPlannerStrategy(new FilterDataSourceV2Strategy(_)) } } diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RuleApplyRowFilterAndDataMasking.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RuleApplyRowFilterAndDataMasking.scala deleted file mode 100644 index b6961c924..000000000 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RuleApplyRowFilterAndDataMasking.scala +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kyuubi.plugin.spark.authz.ranger - -import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.expressions.Alias -import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, Project} -import org.apache.spark.sql.catalyst.rules.Rule - -import org.apache.kyuubi.plugin.spark.authz.ObjectType -import org.apache.kyuubi.plugin.spark.authz.serde._ -import org.apache.kyuubi.plugin.spark.authz.util.{PermanentViewMarker, RowFilterAndDataMaskingMarker} -import org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils._ - -class RuleApplyRowFilterAndDataMasking(spark: SparkSession) extends Rule[LogicalPlan] { - private def mapChildren(plan: LogicalPlan)(f: LogicalPlan => LogicalPlan): LogicalPlan = { - val newChildren = plan match { - case cmd if isKnownTableCommand(cmd) => - val tableCommandSpec = getTableCommandSpec(cmd) - val queries = tableCommandSpec.queries(cmd) - cmd.children.map { - case c if queries.contains(c) => f(c) - case other => other - } - case _ => - plan.children.map(f) - } - plan.withNewChildren(newChildren) - } - - override def apply(plan: LogicalPlan): LogicalPlan = { - mapChildren(plan) { - case p: RowFilterAndDataMaskingMarker => p - case scan if isKnownScan(scan) && scan.resolved => - val tables = getScanSpec(scan).tables(scan, spark) - tables.headOption.map(applyFilterAndMasking(scan, _)).getOrElse(scan) - case other => apply(other) - } - } - - private def applyFilterAndMasking( - plan: LogicalPlan, - table: Table): LogicalPlan = { - val ugi = getAuthzUgi(spark.sparkContext) - val opType = operationType(plan) - val parse = spark.sessionState.sqlParser.parseExpression _ - val are = AccessResource(ObjectType.TABLE, table.database.orNull, table.table, null) - val art = AccessRequest(are, ugi, opType, AccessType.SELECT) - val filterExprStr = SparkRangerAdminPlugin.getFilterExpr(art) - val newOutput = plan.output.map { attr => - val are = - AccessResource(ObjectType.COLUMN, table.database.orNull, table.table, attr.name) - val art = AccessRequest(are, ugi, opType, AccessType.SELECT) - val maskExprStr = SparkRangerAdminPlugin.getMaskingExpr(art) - if (maskExprStr.isEmpty) { - attr - } else { - val maskExpr = parse(maskExprStr.get) - plan match { - case _: PermanentViewMarker => - Alias(maskExpr, attr.name)(exprId = attr.exprId) - case _ => - Alias(maskExpr, attr.name)() - } - } - } - - if (filterExprStr.isEmpty) { - Project(newOutput, RowFilterAndDataMaskingMarker(plan)) - } else { - val filterExpr = parse(filterExprStr.get) - Project(newOutput, Filter(filterExpr, RowFilterAndDataMaskingMarker(plan))) - } - } -} diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RuleAuthorization.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RuleAuthorization.scala index 1c73acc49..afb4f7c54 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RuleAuthorization.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RuleAuthorization.scala @@ -22,29 +22,15 @@ import scala.collection.mutable.ArrayBuffer import org.apache.ranger.plugin.policyengine.RangerAccessRequest import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan -import org.apache.spark.sql.catalyst.rules.Rule -import org.apache.spark.sql.catalyst.trees.TreeNodeTag import org.apache.kyuubi.plugin.spark.authz._ import org.apache.kyuubi.plugin.spark.authz.ObjectType._ -import org.apache.kyuubi.plugin.spark.authz.ranger.RuleAuthorization.KYUUBI_AUTHZ_TAG import org.apache.kyuubi.plugin.spark.authz.ranger.SparkRangerAdminPlugin._ -import org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils._; -class RuleAuthorization(spark: SparkSession) extends Rule[LogicalPlan] { - override def apply(plan: LogicalPlan): LogicalPlan = plan match { - case p if !plan.getTagValue(KYUUBI_AUTHZ_TAG).contains(true) => - RuleAuthorization.checkPrivileges(spark, p) - p.setTagValue(KYUUBI_AUTHZ_TAG, true) - p - case p => p // do nothing if checked privileges already. - } -} - -object RuleAuthorization { +import org.apache.kyuubi.plugin.spark.authz.rule.Authorization +import org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils._ - val KYUUBI_AUTHZ_TAG = TreeNodeTag[Boolean]("__KYUUBI_AUTHZ_TAG") - - def checkPrivileges(spark: SparkSession, plan: LogicalPlan): Unit = { +class RuleAuthorization(spark: SparkSession) extends Authorization(spark) { + override def checkPrivileges(spark: SparkSession, plan: LogicalPlan): Unit = { val auditHandler = new SparkRangerAuditHandler val ugi = getAuthzUgi(spark.sparkContext) val (inputs, outputs, opType) = PrivilegesBuilder.build(plan, spark) @@ -54,7 +40,7 @@ object RuleAuthorization { requests += AccessRequest(resource, ugi, opType, AccessType.USE) } - def addAccessRequest(objects: Seq[PrivilegeObject], isInput: Boolean): Unit = { + def addAccessRequest(objects: Iterable[PrivilegeObject], isInput: Boolean): Unit = { objects.foreach { obj => val resource = AccessResource(obj, opType) val accessType = ranger.AccessType(obj, opType, isInput) @@ -85,7 +71,7 @@ object RuleAuthorization { } case _ => Seq(request) } - } + }.toSeq if (authorizeInSingleCall) { verify(requestArrays.flatten, auditHandler) diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/SparkRangerAdminPlugin.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/SparkRangerAdminPlugin.scala index 7ece55fe5..66f34db91 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/SparkRangerAdminPlugin.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/SparkRangerAdminPlugin.scala @@ -18,18 +18,18 @@ package org.apache.kyuubi.plugin.spark.authz.ranger import scala.collection.JavaConverters._ -import scala.collection.mutable.ArrayBuffer -import scala.collection.mutable.LinkedHashMap +import scala.collection.mutable.{ArrayBuffer, LinkedHashMap} +import org.apache.hadoop.util.ShutdownHookManager import org.apache.ranger.plugin.policyengine.RangerAccessRequest import org.apache.ranger.plugin.service.RangerBasePlugin +import org.slf4j.LoggerFactory import org.apache.kyuubi.plugin.spark.authz.AccessControlException -import org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils._ -import org.apache.kyuubi.plugin.spark.authz.util.RangerConfigProvider object SparkRangerAdminPlugin extends RangerBasePlugin("spark", "sparkSql") with RangerConfigProvider { + final private val LOG = LoggerFactory.getLogger(getClass) /** * For a Spark SQL query, it may contain 0 or more privilege objects to verify, e.g. a typical @@ -60,6 +60,29 @@ object SparkRangerAdminPlugin extends RangerBasePlugin("spark", "sparkSql") s"ranger.plugin.$getServiceType.use.usergroups.from.userstore.enabled", false) + /** + * plugin initialization + * with cleanup shutdown hook registered + */ + def initialize(): Unit = { + this.init() + registerCleanupShutdownHook(this) + } + + /** + * register shutdown hook for plugin cleanup + */ + private def registerCleanupShutdownHook(plugin: RangerBasePlugin): Unit = { + ShutdownHookManager.get().addShutdownHook( + () => { + if (plugin != null) { + LOG.info(s"clean up ranger plugin, appId: ${plugin.getAppId}") + plugin.cleanup() + } + }, + Integer.MAX_VALUE) + } + def getFilterExpr(req: AccessRequest): Option[String] = { val result = evalRowFilterPolicies(req, null) Option(result) @@ -84,11 +107,8 @@ object SparkRangerAdminPlugin extends RangerBasePlugin("spark", "sparkSql") } else if (result.getMaskTypeDef != null) { result.getMaskTypeDef.getName match { case "MASK" => regexp_replace(col) - case "MASK_SHOW_FIRST_4" if isSparkVersionAtLeast("3.1") => - regexp_replace(col, hasLen = true) case "MASK_SHOW_FIRST_4" => - val right = regexp_replace(s"substr($col, 5)") - s"concat(substr($col, 0, 4), $right)" + regexp_replace(col, hasLen = true) case "MASK_SHOW_LAST_4" => val left = regexp_replace(s"left($col, length($col) - 4)") s"concat($left, right($col, 4))" @@ -111,7 +131,8 @@ object SparkRangerAdminPlugin extends RangerBasePlugin("spark", "sparkSql") val upper = s"regexp_replace($expr, '[A-Z]', 'X'$pos)" val lower = s"regexp_replace($upper, '[a-z]', 'x'$pos)" val digits = s"regexp_replace($lower, '[0-9]', 'n'$pos)" - digits + val other = s"regexp_replace($digits, '[^A-Za-z0-9]', 'U'$pos)" + other } /** diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/Authorization.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/Authorization.scala new file mode 100644 index 000000000..6c7d0afa6 --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/Authorization.scala @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.plugin.spark.authz.rule + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Subquery} +import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.spark.sql.catalyst.trees.TreeNodeTag + +import org.apache.kyuubi.plugin.spark.authz.rule.Authorization._ +import org.apache.kyuubi.plugin.spark.authz.rule.permanentview.PermanentViewMarker + +abstract class Authorization(spark: SparkSession) extends Rule[LogicalPlan] { + override def apply(plan: LogicalPlan): LogicalPlan = { + plan match { + case plan if isAuthChecked(plan) => plan // do nothing if checked privileges already. + case p => + checkPrivileges(spark, p) + markAuthChecked(p) + } + } + + def checkPrivileges(spark: SparkSession, plan: LogicalPlan): Unit +} + +object Authorization { + + val KYUUBI_AUTHZ_TAG = TreeNodeTag[Unit]("__KYUUBI_AUTHZ_TAG") + + private def markAllNodesAuthChecked(plan: LogicalPlan): LogicalPlan = { + plan.transformDown { case p => + p.setTagValue(KYUUBI_AUTHZ_TAG, ()) + p + } + } + + protected def markAuthChecked(plan: LogicalPlan): LogicalPlan = { + plan.setTagValue(KYUUBI_AUTHZ_TAG, ()) + plan transformDown { + case pvm: PermanentViewMarker => + markAllNodesAuthChecked(pvm) + case subquery: Subquery => + markAllNodesAuthChecked(subquery) + } + } + + protected def isAuthChecked(plan: LogicalPlan): Boolean = { + plan match { + case subquery: Subquery => isAuthChecked(subquery.child) + case p => p.getTagValue(KYUUBI_AUTHZ_TAG).nonEmpty + } + } +} diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/RuleEliminateMarker.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/RuleEliminateMarker.scala new file mode 100644 index 000000000..3da11ad05 --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/RuleEliminateMarker.scala @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.plugin.spark.authz.rule + +import org.apache.spark.sql.catalyst.expressions.SubqueryExpression +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.catalyst.rules.Rule + +import org.apache.kyuubi.plugin.spark.authz.rule.datamasking.{DataMaskingStage0Marker, DataMaskingStage1Marker} +import org.apache.kyuubi.plugin.spark.authz.rule.rowfilter.RowFilterMarker + +class RuleEliminateMarker extends Rule[LogicalPlan] { + override def apply(plan: LogicalPlan): LogicalPlan = { + plan.transformUp { case p => + p.transformExpressionsUp { + case p: SubqueryExpression => + p.withNewPlan(apply(p.plan)) + } match { + case marker0: DataMaskingStage0Marker => marker0.child + case marker1: DataMaskingStage1Marker => marker1.child + case rf: RowFilterMarker => rf.child + case other => other + } + } + } +} diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/RuleEliminateViewMarker.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/RuleEliminatePermanentViewMarker.scala similarity index 66% rename from extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/RuleEliminateViewMarker.scala rename to extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/RuleEliminatePermanentViewMarker.scala index 9bda84a03..864ada55f 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/RuleEliminateViewMarker.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/RuleEliminatePermanentViewMarker.scala @@ -15,16 +15,23 @@ * limitations under the License. */ -package org.apache.kyuubi.plugin.spark.authz.util +package org.apache.kyuubi.plugin.spark.authz.rule +import org.apache.spark.sql.catalyst.expressions.SubqueryExpression import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.rules.Rule +import org.apache.kyuubi.plugin.spark.authz.rule.permanentview.PermanentViewMarker + /** - * Transforming up [[org.apache.kyuubi.plugin.spark.authz.util.PermanentViewMarker]] + * Transforming up [[PermanentViewMarker]] */ -class RuleEliminateViewMarker extends Rule[LogicalPlan] { +class RuleEliminatePermanentViewMarker extends Rule[LogicalPlan] { override def apply(plan: LogicalPlan): LogicalPlan = { - plan.transformUp { case pvm: PermanentViewMarker => pvm.child } + plan.transformUp { + case pvm: PermanentViewMarker => pvm.child.transformAllExpressions { + case s: SubqueryExpression => s.withNewPlan(apply(s.plan)) + } + } } } diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/RuleHelper.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/RuleHelper.scala new file mode 100644 index 000000000..c163cafe9 --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/RuleHelper.scala @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.plugin.spark.authz.rule + +import org.apache.hadoop.security.UserGroupInformation +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.expressions.Expression +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.catalyst.rules.Rule + +import org.apache.kyuubi.plugin.spark.authz.serde.{getTableCommandSpec, isKnownTableCommand} +import org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils + +trait RuleHelper extends Rule[LogicalPlan] { + + def spark: SparkSession + + final protected val parse: String => Expression = spark.sessionState.sqlParser.parseExpression _ + + protected def mapChildren(plan: LogicalPlan)(f: LogicalPlan => LogicalPlan): LogicalPlan = { + val newChildren = plan match { + case cmd if isKnownTableCommand(cmd) => + val tableCommandSpec = getTableCommandSpec(cmd) + val queries = tableCommandSpec.queries(cmd) + cmd.children.map { + case c if queries.contains(c) => f(c) + case other => other + } + case _ => + plan.children.map(f) + } + plan.withNewChildren(newChildren) + } + + def ugi: UserGroupInformation = AuthZUtils.getAuthzUgi(spark.sparkContext) + +} diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/AuthzConfigurationChecker.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/config/AuthzConfigurationChecker.scala similarity index 97% rename from extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/AuthzConfigurationChecker.scala rename to extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/config/AuthzConfigurationChecker.scala index 56ab27d22..3ab2c3fd6 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/AuthzConfigurationChecker.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/config/AuthzConfigurationChecker.scala @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.kyuubi.plugin.spark.authz.ranger +package org.apache.kyuubi.plugin.spark.authz.rule.config import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/datamasking/DataMaskingStage0Marker.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/datamasking/DataMaskingStage0Marker.scala new file mode 100644 index 000000000..c1d3a7532 --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/datamasking/DataMaskingStage0Marker.scala @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.plugin.spark.authz.rule.datamasking + +import org.apache.spark.sql.catalyst.expressions.{Attribute, ExprId} +import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, UnaryNode} + +import org.apache.kyuubi.plugin.spark.authz.util.WithInternalChild +case class DataMaskingStage0Marker(child: LogicalPlan, scan: LogicalPlan) + extends UnaryNode with WithInternalChild { + + def exprToMaskers(): Map[ExprId, Attribute] = { + scan.output.map(_.exprId).zip(child.output).flatMap { case (id, expr) => + if (id == expr.exprId) None else Some(id -> expr) + }.toMap + } + + override def output: Seq[Attribute] = child.output + + override def withNewChildInternal(newChild: LogicalPlan): LogicalPlan = copy(child = newChild) + +} diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/ObjectFilterPlaceHolder.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/datamasking/DataMaskingStage1Marker.scala similarity index 69% rename from extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/ObjectFilterPlaceHolder.scala rename to extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/datamasking/DataMaskingStage1Marker.scala index a5d1c0d3b..1c30879e4 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/ObjectFilterPlaceHolder.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/datamasking/DataMaskingStage1Marker.scala @@ -15,12 +15,17 @@ * limitations under the License. */ -package org.apache.kyuubi.plugin.spark.authz.util +package org.apache.kyuubi.plugin.spark.authz.rule.datamasking import org.apache.spark.sql.catalyst.expressions.Attribute -import org.apache.spark.sql.catalyst.plans.logical.{LeafNode, LogicalPlan, Statistics} +import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, UnaryNode} + +import org.apache.kyuubi.plugin.spark.authz.util.WithInternalChild + +case class DataMaskingStage1Marker(child: LogicalPlan) extends UnaryNode with WithInternalChild { -case class ObjectFilterPlaceHolder(child: LogicalPlan) extends LeafNode { override def output: Seq[Attribute] = child.output - override def computeStats(): Statistics = child.stats + + override def withNewChildInternal(newChild: LogicalPlan): LogicalPlan = copy(child = newChild) + } diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/datamasking/RuleApplyDataMaskingStage0.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/datamasking/RuleApplyDataMaskingStage0.scala new file mode 100644 index 000000000..27cde1621 --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/datamasking/RuleApplyDataMaskingStage0.scala @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.plugin.spark.authz.rule.datamasking + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.expressions.Alias +import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project} + +import org.apache.kyuubi.plugin.spark.authz.ObjectType +import org.apache.kyuubi.plugin.spark.authz.OperationType.QUERY +import org.apache.kyuubi.plugin.spark.authz.ranger._ +import org.apache.kyuubi.plugin.spark.authz.rule.RuleHelper +import org.apache.kyuubi.plugin.spark.authz.serde._ + +/** + * The full data masking rule contains two separate stages. + * + * Step1: RuleApplyDataMaskingStage0 + * - lookup the full plan for supported scans + * - once found, get masker configuration from external column by column + * - use spark sql parser to generate an unresolved expression for each masker + * - add a projection with new output on the right top of the original scan if the output has + * changed + * - Add DataMaskingStage0Marker to track the original expression and its masker expression. + * + * Step2: Spark native rules will resolve our newly added maskers + * + * Step3: [[RuleApplyDataMaskingStage1]] + */ +case class RuleApplyDataMaskingStage0(spark: SparkSession) extends RuleHelper { + + override def apply(plan: LogicalPlan): LogicalPlan = { + val newPlan = mapChildren(plan) { + case p: DataMaskingStage0Marker => p + case p: DataMaskingStage1Marker => p + case scan if isKnownScan(scan) && scan.resolved => + val tables = getScanSpec(scan).tables(scan, spark) + tables.headOption.map(applyMasking(scan, _)).getOrElse(scan) + case other => apply(other) + } + newPlan + } + + private def applyMasking( + plan: LogicalPlan, + table: Table): LogicalPlan = { + val newOutput = plan.output.map { attr => + val are = + AccessResource(ObjectType.COLUMN, table.database.orNull, table.table, attr.name) + val art = AccessRequest(are, ugi, QUERY, AccessType.SELECT) + val maskExprStr = SparkRangerAdminPlugin.getMaskingExpr(art) + maskExprStr.map(parse).map(Alias(_, attr.name)()).getOrElse(attr) + } + if (newOutput == plan.output) { + plan + } else { + DataMaskingStage0Marker(Project(newOutput, plan), plan) + } + } +} diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/datamasking/RuleApplyDataMaskingStage1.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/datamasking/RuleApplyDataMaskingStage1.scala new file mode 100644 index 000000000..b0069c9a5 --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/datamasking/RuleApplyDataMaskingStage1.scala @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.plugin.spark.authz.rule.datamasking + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.expressions.NamedExpression +import org.apache.spark.sql.catalyst.plans.logical.{Command, LogicalPlan} + +import org.apache.kyuubi.plugin.spark.authz.rule.RuleHelper +import org.apache.kyuubi.plugin.spark.authz.serde._ + +/** + * See [[RuleApplyDataMaskingStage0]] also. + * + * This is the second step for data masking. It will fulfill the missing attributes that + * have a related masker expression buffered by DataMaskingStage0Marker. + */ +case class RuleApplyDataMaskingStage1(spark: SparkSession) extends RuleHelper { + + override def apply(plan: LogicalPlan): LogicalPlan = { + + plan match { + case marker0: DataMaskingStage0Marker => marker0 + case marker1: DataMaskingStage1Marker => marker1 + case cmd if isKnownTableCommand(cmd) => + val tableCommandSpec = getTableCommandSpec(cmd) + val queries = tableCommandSpec.queries(cmd) + cmd.mapChildren { + case marker0: DataMaskingStage0Marker => marker0 + case marker1: DataMaskingStage1Marker => marker1 + case query if queries.contains(query) && query.resolved => + applyDataMasking(query) + case o => o + } + case cmd: Command if cmd.childrenResolved => + cmd.mapChildren(applyDataMasking) + case cmd: Command => cmd + case other if other.resolved => applyDataMasking(other) + case other => other + } + } + + private def applyDataMasking(plan: LogicalPlan): LogicalPlan = { + assert(plan.resolved, "the current masking approach relies on a resolved plan") + def replaceOriginExprWithMasker(plan: LogicalPlan): LogicalPlan = plan match { + case m: DataMaskingStage0Marker => m + case m: DataMaskingStage1Marker => m + case p => + val maskerExprs = p.collect { + case marker: DataMaskingStage0Marker if marker.resolved => marker.exprToMaskers() + }.flatten.toMap + if (maskerExprs.isEmpty) { + p + } else { + val t = p.transformExpressionsUp { + case e: NamedExpression => maskerExprs.getOrElse(e.exprId, e) + } + t.withNewChildren(t.children.map(replaceOriginExprWithMasker)) + } + } + val newPlan = replaceOriginExprWithMasker(plan) + + if (newPlan == plan) { + plan + } else { + DataMaskingStage1Marker(newPlan) + } + } +} diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/PermanentViewMarker.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/permanentview/PermanentViewMarker.scala similarity index 78% rename from extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/PermanentViewMarker.scala rename to extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/permanentview/PermanentViewMarker.scala index 69b55e0fc..18b58e4d8 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/PermanentViewMarker.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/permanentview/PermanentViewMarker.scala @@ -15,13 +15,19 @@ * limitations under the License. */ -package org.apache.kyuubi.plugin.spark.authz.util +package org.apache.kyuubi.plugin.spark.authz.rule.permanentview import org.apache.spark.sql.catalyst.catalog.CatalogTable import org.apache.spark.sql.catalyst.expressions.Attribute import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, UnaryNode} -case class PermanentViewMarker(child: LogicalPlan, catalogTable: CatalogTable) extends UnaryNode +import org.apache.kyuubi.plugin.spark.authz.util.WithInternalChild + +case class PermanentViewMarker( + child: LogicalPlan, + catalogTable: CatalogTable, + outputColNames: Seq[String], + isSubqueryExpressionPlaceHolder: Boolean = false) extends UnaryNode with WithInternalChild { override def output: Seq[Attribute] = child.output diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RuleApplyPermanentViewMarker.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/permanentview/RuleApplyPermanentViewMarker.scala similarity index 65% rename from extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RuleApplyPermanentViewMarker.scala rename to extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/permanentview/RuleApplyPermanentViewMarker.scala index 424df7e0b..1a0024abb 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RuleApplyPermanentViewMarker.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/permanentview/RuleApplyPermanentViewMarker.scala @@ -15,20 +15,20 @@ * limitations under the License. */ -package org.apache.kyuubi.plugin.spark.authz.ranger +package org.apache.kyuubi.plugin.spark.authz.rule.permanentview +import org.apache.spark.sql.catalyst.expressions.SubqueryExpression import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, View} import org.apache.spark.sql.catalyst.rules.Rule import org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils._ -import org.apache.kyuubi.plugin.spark.authz.util.PermanentViewMarker /** - * Adding [[org.apache.kyuubi.plugin.spark.authz.util.PermanentViewMarker]] for permanent views + * Adding [[PermanentViewMarker]] for permanent views * for marking catalogTable of views used by privilege checking * in [[org.apache.kyuubi.plugin.spark.authz.ranger.RuleAuthorization]]. - * [[org.apache.kyuubi.plugin.spark.authz.util.PermanentViewMarker]] must be transformed up later - * in [[org.apache.kyuubi.plugin.spark.authz.util.RuleEliminateViewMarker]] optimizer. + * [[PermanentViewMarker]] must be transformed up later + * in [[org.apache.kyuubi.plugin.spark.authz.rule.RuleEliminatePermanentViewMarker]] optimizer. */ class RuleApplyPermanentViewMarker extends Rule[LogicalPlan] { @@ -36,7 +36,16 @@ class RuleApplyPermanentViewMarker extends Rule[LogicalPlan] { plan mapChildren { case p: PermanentViewMarker => p case permanentView: View if hasResolvedPermanentView(permanentView) => - PermanentViewMarker(permanentView, permanentView.desc) + val resolved = permanentView.transformAllExpressions { + case subquery: SubqueryExpression => + subquery.withNewPlan(plan = + PermanentViewMarker( + subquery.plan, + permanentView.desc, + permanentView.output.map(_.name), + true)) + } + PermanentViewMarker(resolved, resolved.desc, resolved.output.map(_.name)) case other => apply(other) } } diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/FilterDataSourceV2Strategy.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/rowfilter/FilterDataSourceV2Strategy.scala similarity index 63% rename from extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/FilterDataSourceV2Strategy.scala rename to extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/rowfilter/FilterDataSourceV2Strategy.scala index 1109464ac..17c766555 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/FilterDataSourceV2Strategy.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/rowfilter/FilterDataSourceV2Strategy.scala @@ -14,21 +14,29 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.kyuubi.plugin.spark.authz.ranger +package org.apache.kyuubi.plugin.spark.authz.rule.rowfilter import org.apache.spark.sql.{SparkSession, Strategy} -import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project} import org.apache.spark.sql.execution.SparkPlan -import org.apache.kyuubi.plugin.spark.authz.util.ObjectFilterPlaceHolder - class FilterDataSourceV2Strategy(spark: SparkSession) extends Strategy { override def apply(plan: LogicalPlan): Seq[SparkPlan] = plan match { + // For Spark 3.1 and below, `ColumnPruning` rule will set `ObjectFilterPlaceHolder#child` to + // `Project` + case ObjectFilterPlaceHolder(Project(_, child)) if child.nodeName == "ShowNamespaces" => + spark.sessionState.planner.plan(child) + .map(FilteredShowNamespaceExec(_, spark.sparkContext)).toSeq + + // For Spark 3.2 and above case ObjectFilterPlaceHolder(child) if child.nodeName == "ShowNamespaces" => - spark.sessionState.planner.plan(child).map(FilteredShowNamespaceExec).toSeq + spark.sessionState.planner.plan(child) + .map(FilteredShowNamespaceExec(_, spark.sparkContext)).toSeq case ObjectFilterPlaceHolder(child) if child.nodeName == "ShowTables" => - spark.sessionState.planner.plan(child).map(FilteredShowTablesExec).toSeq + spark.sessionState.planner.plan(child) + .map(FilteredShowTablesExec(_, spark.sparkContext)).toSeq + case _ => Nil } } diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/FilteredShowObjectsExec.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/rowfilter/FilteredShowObjectsExec.scala similarity index 59% rename from extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/FilteredShowObjectsExec.scala rename to extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/rowfilter/FilteredShowObjectsExec.scala index 7cc777d9b..0bb421356 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/FilteredShowObjectsExec.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/rowfilter/FilteredShowObjectsExec.scala @@ -14,36 +14,43 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.kyuubi.plugin.spark.authz.ranger +package org.apache.kyuubi.plugin.spark.authz.rule.rowfilter import org.apache.hadoop.security.UserGroupInformation +import org.apache.spark.SparkContext import org.apache.spark.rdd.RDD import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.expressions.Attribute import org.apache.spark.sql.execution.{LeafExecNode, SparkPlan} import org.apache.kyuubi.plugin.spark.authz.{ObjectType, OperationType} +import org.apache.kyuubi.plugin.spark.authz.ranger.{AccessRequest, AccessResource, AccessType, SparkRangerAdminPlugin} import org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils trait FilteredShowObjectsExec extends LeafExecNode { - def delegated: SparkPlan + def result: Array[InternalRow] - final override def output: Seq[Attribute] = delegated.output - - final private lazy val result = { - delegated.executeCollect().filter(isAllowed(_, AuthZUtils.getAuthzUgi(sparkContext))) - } + override def output: Seq[Attribute] final override def doExecute(): RDD[InternalRow] = { sparkContext.parallelize(result, 1) } +} - protected def isAllowed(r: InternalRow, ugi: UserGroupInformation): Boolean +trait FilteredShowObjectsCheck { + def isAllowed(r: InternalRow, ugi: UserGroupInformation): Boolean } -case class FilteredShowNamespaceExec(delegated: SparkPlan) extends FilteredShowObjectsExec { +case class FilteredShowNamespaceExec(result: Array[InternalRow], output: Seq[Attribute]) + extends FilteredShowObjectsExec {} +object FilteredShowNamespaceExec extends FilteredShowObjectsCheck { + def apply(delegated: SparkPlan, sc: SparkContext): FilteredShowNamespaceExec = { + val result = delegated.executeCollect() + .filter(isAllowed(_, AuthZUtils.getAuthzUgi(sc))) + new FilteredShowNamespaceExec(result, delegated.output) + } - override protected def isAllowed(r: InternalRow, ugi: UserGroupInformation): Boolean = { + override def isAllowed(r: InternalRow, ugi: UserGroupInformation): Boolean = { val database = r.getString(0) val resource = AccessResource(ObjectType.DATABASE, database, null, null) val request = AccessRequest(resource, ugi, OperationType.SHOWDATABASES, AccessType.USE) @@ -52,8 +59,16 @@ case class FilteredShowNamespaceExec(delegated: SparkPlan) extends FilteredShowO } } -case class FilteredShowTablesExec(delegated: SparkPlan) extends FilteredShowObjectsExec { - override protected def isAllowed(r: InternalRow, ugi: UserGroupInformation): Boolean = { +case class FilteredShowTablesExec(result: Array[InternalRow], output: Seq[Attribute]) + extends FilteredShowObjectsExec {} +object FilteredShowTablesExec extends FilteredShowObjectsCheck { + def apply(delegated: SparkPlan, sc: SparkContext): FilteredShowNamespaceExec = { + val result = delegated.executeCollect() + .filter(isAllowed(_, AuthZUtils.getAuthzUgi(sc))) + new FilteredShowNamespaceExec(result, delegated.output) + } + + override def isAllowed(r: InternalRow, ugi: UserGroupInformation): Boolean = { val database = r.getString(0) val table = r.getString(1) val isTemp = r.getBoolean(2) diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/rowfilter/ObjectFilterPlaceHolder.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/rowfilter/ObjectFilterPlaceHolder.scala new file mode 100644 index 000000000..6a7f1beab --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/rowfilter/ObjectFilterPlaceHolder.scala @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.plugin.spark.authz.rule.rowfilter + +import org.apache.spark.sql.catalyst.expressions.Attribute +import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, UnaryNode} + +import org.apache.kyuubi.plugin.spark.authz.util.WithInternalChild + +case class ObjectFilterPlaceHolder(child: LogicalPlan) extends UnaryNode + with WithInternalChild { + + override def output: Seq[Attribute] = child.output + + override def withNewChildInternal(newChild: LogicalPlan): LogicalPlan = { + // `FilterDataSourceV2Strategy` requires child.nodename not changed + if (child.nodeName == newChild.nodeName) { + copy(newChild) + } else { + this + } + } +} diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/RowFilterAndDataMaskingMarker.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/rowfilter/RowFilterMarker.scala similarity index 80% rename from extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/RowFilterAndDataMaskingMarker.scala rename to extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/rowfilter/RowFilterMarker.scala index 357e9bfc2..f4295a094 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/RowFilterAndDataMaskingMarker.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/rowfilter/RowFilterMarker.scala @@ -15,17 +15,17 @@ * limitations under the License. */ -package org.apache.kyuubi.plugin.spark.authz.util +package org.apache.kyuubi.plugin.spark.authz.rule.rowfilter import org.apache.spark.sql.catalyst.expressions.Attribute import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, UnaryNode} -case class RowFilterAndDataMaskingMarker(child: LogicalPlan) extends UnaryNode - with WithInternalChild { +import org.apache.kyuubi.plugin.spark.authz.util.WithInternalChild + +case class RowFilterMarker(child: LogicalPlan) extends UnaryNode with WithInternalChild { override def output: Seq[Attribute] = child.output - override def withNewChildInternal(newChild: LogicalPlan): LogicalPlan = - copy(child = newChild) + override def withNewChildInternal(newChild: LogicalPlan): LogicalPlan = copy(child = newChild) } diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/rowfilter/RuleApplyRowFilter.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/rowfilter/RuleApplyRowFilter.scala new file mode 100644 index 000000000..defee4005 --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/rowfilter/RuleApplyRowFilter.scala @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.plugin.spark.authz.rule.rowfilter + +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan} + +import org.apache.kyuubi.plugin.spark.authz.ObjectType +import org.apache.kyuubi.plugin.spark.authz.OperationType.QUERY +import org.apache.kyuubi.plugin.spark.authz.ranger._ +import org.apache.kyuubi.plugin.spark.authz.rule.RuleHelper +import org.apache.kyuubi.plugin.spark.authz.serde._ + +case class RuleApplyRowFilter(spark: SparkSession) extends RuleHelper { + + override def apply(plan: LogicalPlan): LogicalPlan = { + val newPlan = mapChildren(plan) { + case p: RowFilterMarker => p + case scan if isKnownScan(scan) && scan.resolved => + val tables = getScanSpec(scan).tables(scan, spark) + tables.headOption.map(applyFilter(scan, _)).getOrElse(scan) + case other => apply(other) + } + newPlan + } + + private def applyFilter( + plan: LogicalPlan, + table: Table): LogicalPlan = { + val are = AccessResource(ObjectType.TABLE, table.database.orNull, table.table, null) + val art = AccessRequest(are, ugi, QUERY, AccessType.SELECT) + val filterExpr = SparkRangerAdminPlugin.getFilterExpr(art).map(parse) + val filtered = filterExpr.foldLeft(plan)((p, expr) => Filter(expr, RowFilterMarker(p))) + filtered + } +} diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RuleReplaceShowObjectCommands.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/rowfilter/RuleReplaceShowObjectCommands.scala similarity index 83% rename from extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RuleReplaceShowObjectCommands.scala rename to extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/rowfilter/RuleReplaceShowObjectCommands.scala index 08d2b4fd0..1728234a8 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RuleReplaceShowObjectCommands.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/rule/rowfilter/RuleReplaceShowObjectCommands.scala @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.kyuubi.plugin.spark.authz.ranger +package org.apache.kyuubi.plugin.spark.authz.rule.rowfilter import org.apache.hadoop.security.UserGroupInformation import org.apache.spark.sql.{Row, SparkSession} @@ -25,16 +25,15 @@ import org.apache.spark.sql.catalyst.rules.Rule import org.apache.spark.sql.execution.command.{RunnableCommand, ShowColumnsCommand} import org.apache.kyuubi.plugin.spark.authz.{ObjectType, OperationType} -import org.apache.kyuubi.plugin.spark.authz.util.{AuthZUtils, ObjectFilterPlaceHolder, WithInternalChildren} +import org.apache.kyuubi.plugin.spark.authz.ranger.{AccessRequest, AccessResource, AccessType, SparkRangerAdminPlugin} +import org.apache.kyuubi.plugin.spark.authz.util.{AuthZUtils, WithInternalChildren} +import org.apache.kyuubi.util.reflect.ReflectUtils._ class RuleReplaceShowObjectCommands extends Rule[LogicalPlan] { override def apply(plan: LogicalPlan): LogicalPlan = plan match { case r: RunnableCommand if r.nodeName == "ShowTablesCommand" => FilteredShowTablesCommand(r) case n: LogicalPlan if n.nodeName == "ShowTables" => ObjectFilterPlaceHolder(n) - // show databases in spark2.4.x - case r: RunnableCommand if r.nodeName == "ShowDatabasesCommand" => - FilteredShowDatabasesCommand(r) case n: LogicalPlan if n.nodeName == "ShowNamespaces" => ObjectFilterPlaceHolder(n) case r: RunnableCommand if r.nodeName == "ShowFunctionsCommand" => @@ -48,7 +47,7 @@ class RuleReplaceShowObjectCommands extends Rule[LogicalPlan] { case class FilteredShowTablesCommand(delegated: RunnableCommand) extends FilteredShowObjectCommand(delegated) { - var isExtended: Boolean = AuthZUtils.getFieldVal(delegated, "isExtended").asInstanceOf[Boolean] + private val isExtended = getField[Boolean](delegated, "isExtended") override protected def isAllowed(r: Row, ugi: UserGroupInformation): Boolean = { val database = r.getString(0) @@ -63,18 +62,6 @@ case class FilteredShowTablesCommand(delegated: RunnableCommand) } } -case class FilteredShowDatabasesCommand(delegated: RunnableCommand) - extends FilteredShowObjectCommand(delegated) { - - override protected def isAllowed(r: Row, ugi: UserGroupInformation): Boolean = { - val database = r.getString(0) - val resource = AccessResource(ObjectType.DATABASE, database, null, null) - val request = AccessRequest(resource, ugi, OperationType.SHOWDATABASES, AccessType.USE) - val result = SparkRangerAdminPlugin.isAccessAllowed(request) - result != null && result.getIsAllowed - } -} - abstract class FilteredShowObjectCommand(delegated: RunnableCommand) extends RunnableCommand with WithInternalChildren { diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/CommandSpec.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/CommandSpec.scala index d72d78932..7b306551c 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/CommandSpec.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/CommandSpec.scala @@ -19,6 +19,7 @@ package org.apache.kyuubi.plugin.spark.authz.serde import com.fasterxml.jackson.annotation.JsonIgnore import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.expressions.Expression import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.slf4j.LoggerFactory @@ -42,6 +43,10 @@ trait CommandSpec extends { final def operationType: OperationType = OperationType.withName(opType) } +trait CommandSpecs[T <: CommandSpec] { + def specs: Seq[T] +} + /** * A specification describe a database command * @@ -78,14 +83,15 @@ case class TableCommandSpec( classname: String, tableDescs: Seq[TableDesc], opType: String = OperationType.QUERY.toString, - queryDescs: Seq[QueryDesc] = Nil) extends CommandSpec { + queryDescs: Seq[QueryDesc] = Nil, + uriDescs: Seq[UriDesc] = Nil) extends CommandSpec { def queries: LogicalPlan => Seq[LogicalPlan] = plan => { queryDescs.flatMap { qd => try { qd.extract(plan) } catch { case e: Exception => - LOG.warn(qd.error(plan, e)) + LOG.debug(qd.error(plan, e)) None } } @@ -94,7 +100,8 @@ case class TableCommandSpec( case class ScanSpec( classname: String, - scanDescs: Seq[ScanDesc]) extends CommandSpec { + scanDescs: Seq[ScanDesc], + functionDescs: Seq[FunctionDesc] = Seq.empty) extends CommandSpec { override def opType: String = OperationType.QUERY.toString def tables: (LogicalPlan, SparkSession) => Seq[Table] = (plan, spark) => { scanDescs.flatMap { td => @@ -102,7 +109,19 @@ case class ScanSpec( td.extract(plan, spark) } catch { case e: Exception => - LOG.warn(td.error(plan, e)) + LOG.debug(td.error(plan, e)) + None + } + } + } + + def functions: (Expression) => Seq[Function] = (expr) => { + functionDescs.flatMap { fd => + try { + Some(fd.extract(expr)) + } catch { + case e: Exception => + LOG.debug(fd.error(expr, e)) None } } diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/Descriptor.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/Descriptor.scala index d8c866b88..4869fc1da 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/Descriptor.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/Descriptor.scala @@ -23,18 +23,9 @@ import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.kyuubi.plugin.spark.authz.PrivilegeObjectActionType import org.apache.kyuubi.plugin.spark.authz.PrivilegeObjectActionType.PrivilegeObjectActionType -import org.apache.kyuubi.plugin.spark.authz.serde.ActionTypeExtractor.actionTypeExtractors -import org.apache.kyuubi.plugin.spark.authz.serde.CatalogExtractor.catalogExtractors -import org.apache.kyuubi.plugin.spark.authz.serde.ColumnExtractor.columnExtractors -import org.apache.kyuubi.plugin.spark.authz.serde.DatabaseExtractor.dbExtractors -import org.apache.kyuubi.plugin.spark.authz.serde.FunctionExtractor.functionExtractors import org.apache.kyuubi.plugin.spark.authz.serde.FunctionType.FunctionType -import org.apache.kyuubi.plugin.spark.authz.serde.FunctionTypeExtractor.functionTypeExtractors -import org.apache.kyuubi.plugin.spark.authz.serde.QueryExtractor.queryExtractors -import org.apache.kyuubi.plugin.spark.authz.serde.TableExtractor.tableExtractors import org.apache.kyuubi.plugin.spark.authz.serde.TableType.TableType -import org.apache.kyuubi.plugin.spark.authz.serde.TableTypeExtractor.tableTypeExtractors -import org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils._ +import org.apache.kyuubi.util.reflect.ReflectUtils._ /** * A database object(such as database, table, function) descriptor describes its name and getter @@ -81,8 +72,8 @@ case class ColumnDesc( fieldName: String, fieldExtractor: String) extends Descriptor { override def extract(v: AnyRef): Seq[String] = { - val columnsVal = invoke(v, fieldName) - val columnExtractor = columnExtractors(fieldExtractor) + val columnsVal = invokeAs[AnyRef](v, fieldName) + val columnExtractor = lookupExtractor[ColumnExtractor](fieldExtractor) columnExtractor(columnsVal) } } @@ -100,8 +91,8 @@ case class DatabaseDesc( catalogDesc: Option[CatalogDesc] = None, isInput: Boolean = false) extends Descriptor { override def extract(v: AnyRef): Database = { - val databaseVal = invoke(v, fieldName) - val databaseExtractor = dbExtractors(fieldExtractor) + val databaseVal = invokeAs[AnyRef](v, fieldName) + val databaseExtractor = lookupExtractor[DatabaseExtractor](fieldExtractor) val db = databaseExtractor(databaseVal) if (db.catalog.isEmpty && catalogDesc.nonEmpty) { val maybeCatalog = catalogDesc.get.extract(v) @@ -128,8 +119,8 @@ case class FunctionTypeDesc( } def extract(v: AnyRef, spark: SparkSession): FunctionType = { - val functionTypeVal = invoke(v, fieldName) - val functionTypeExtractor = functionTypeExtractors(fieldExtractor) + val functionTypeVal = invokeAs[AnyRef](v, fieldName) + val functionTypeExtractor = lookupExtractor[FunctionTypeExtractor](fieldExtractor) functionTypeExtractor(functionTypeVal, spark) } @@ -154,8 +145,8 @@ case class FunctionDesc( functionTypeDesc: Option[FunctionTypeDesc] = None, isInput: Boolean = false) extends Descriptor { override def extract(v: AnyRef): Function = { - val functionVal = invoke(v, fieldName) - val functionExtractor = functionExtractors(fieldExtractor) + val functionVal = invokeAs[AnyRef](v, fieldName) + val functionExtractor = lookupExtractor[FunctionExtractor](fieldExtractor) var function = functionExtractor(functionVal) if (function.database.isEmpty) { val maybeDatabase = databaseDesc.map(_.extract(v)) @@ -179,8 +170,8 @@ case class QueryDesc( fieldName: String, fieldExtractor: String = "LogicalPlanQueryExtractor") extends Descriptor { override def extract(v: AnyRef): Option[LogicalPlan] = { - val queryVal = invoke(v, fieldName) - val queryExtractor = queryExtractors(fieldExtractor) + val queryVal = invokeAs[AnyRef](v, fieldName) + val queryExtractor = lookupExtractor[QueryExtractor](fieldExtractor) queryExtractor(queryVal) } } @@ -201,8 +192,8 @@ case class TableTypeDesc( } def extract(v: AnyRef, spark: SparkSession): TableType = { - val tableTypeVal = invoke(v, fieldName) - val tableTypeExtractor = tableTypeExtractors(fieldExtractor) + val tableTypeVal = invokeAs[AnyRef](v, fieldName) + val tableTypeExtractor = lookupExtractor[TableTypeExtractor](fieldExtractor) tableTypeExtractor(tableTypeVal, spark) } @@ -239,8 +230,8 @@ case class TableDesc( } def extract(v: AnyRef, spark: SparkSession): Option[Table] = { - val tableVal = invoke(v, fieldName) - val tableExtractor = tableExtractors(fieldExtractor) + val tableVal = invokeAs[AnyRef](v, fieldName) + val tableExtractor = lookupExtractor[TableExtractor](fieldExtractor) val maybeTable = tableExtractor(spark, tableVal) maybeTable.map { t => if (t.catalog.isEmpty && catalogDesc.nonEmpty) { @@ -266,9 +257,9 @@ case class ActionTypeDesc( actionType: Option[String] = None) extends Descriptor { override def extract(v: AnyRef): PrivilegeObjectActionType = { actionType.map(PrivilegeObjectActionType.withName).getOrElse { - val actionTypeVal = invoke(v, fieldName) - val extractor = actionTypeExtractors(fieldExtractor) - extractor(actionTypeVal) + val actionTypeVal = invokeAs[AnyRef](v, fieldName) + val actionTypeExtractor = lookupExtractor[ActionTypeExtractor](fieldExtractor) + actionTypeExtractor(actionTypeVal) } } } @@ -283,9 +274,9 @@ case class CatalogDesc( fieldName: String = "catalog", fieldExtractor: String = "CatalogPluginCatalogExtractor") extends Descriptor { override def extract(v: AnyRef): Option[String] = { - val catalogVal = invoke(v, fieldName) - val extractor = catalogExtractors(fieldExtractor) - extractor(catalogVal) + val catalogVal = invokeAs[AnyRef](v, fieldName) + val catalogExtractor = lookupExtractor[CatalogExtractor](fieldExtractor) + catalogExtractor(catalogVal) } } @@ -301,9 +292,9 @@ case class ScanDesc( val tableVal = if (fieldName == null) { v } else { - invoke(v, fieldName) + invokeAs[AnyRef](v, fieldName) } - val tableExtractor = tableExtractors(fieldExtractor) + val tableExtractor = lookupExtractor[TableExtractor](fieldExtractor) val maybeTable = tableExtractor(spark, tableVal) maybeTable.map { t => if (t.catalog.isEmpty && catalogDesc.nonEmpty) { @@ -315,3 +306,21 @@ case class ScanDesc( } } } + +/** + * Function Descriptor + * + * @param fieldName the field name or method name of this function field + * @param fieldExtractor the key of a [[FunctionExtractor]] instance + * @param isInput read or write + */ +case class UriDesc( + fieldName: String, + fieldExtractor: String, + isInput: Boolean = false) extends Descriptor { + override def extract(v: AnyRef): Option[Uri] = { + val uriVal = invokeAs[AnyRef](v, fieldName) + val uriExtractor = lookupExtractor[URIExtractor](fieldExtractor) + uriExtractor(uriVal) + } +} diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/Function.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/Function.scala index b7a0010b4..ba19972ed 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/Function.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/Function.scala @@ -21,8 +21,8 @@ package org.apache.kyuubi.plugin.spark.authz.serde * :: Developer API :: * * Represents a function identity - * + * @param catalog * @param database * @param functionName */ -case class Function(database: Option[String], functionName: String) +case class Function(catalog: Option[String], database: Option[String], functionName: String) diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/Uri.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/Uri.scala new file mode 100644 index 000000000..aa9af8732 --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/Uri.scala @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.plugin.spark.authz.serde + +/** + * :: Developer API :: + * + * Represents a URI identity + * @param path + */ +case class Uri(path: String) diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/catalogExtractors.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/catalogExtractors.scala index 0b7d71223..e48becb32 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/catalogExtractors.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/catalogExtractors.scala @@ -17,7 +17,7 @@ package org.apache.kyuubi.plugin.spark.authz.serde -import org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils._ +import org.apache.kyuubi.util.reflect.ReflectUtils._ trait CatalogExtractor extends (AnyRef => Option[String]) with Extractor @@ -43,7 +43,7 @@ class CatalogPluginOptionCatalogExtractor extends CatalogExtractor { override def apply(v1: AnyRef): Option[String] = { v1 match { case Some(catalogPlugin: AnyRef) => - new CatalogPluginCatalogExtractor().apply(catalogPlugin) + lookupExtractor[CatalogPluginCatalogExtractor].apply(catalogPlugin) case _ => None } } diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/databaseExtractors.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/databaseExtractors.scala index 4e9270e78..713d3e3fb 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/databaseExtractors.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/databaseExtractors.scala @@ -18,6 +18,7 @@ package org.apache.kyuubi.plugin.spark.authz.serde import org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils._ +import org.apache.kyuubi.util.reflect.ReflectUtils._ trait DatabaseExtractor extends (AnyRef => Database) with Extractor @@ -68,9 +69,9 @@ class StringSeqOptionDatabaseExtractor extends DatabaseExtractor { */ class ResolvedNamespaceDatabaseExtractor extends DatabaseExtractor { override def apply(v1: AnyRef): Database = { - val catalogVal = invoke(v1, "catalog") - val catalog = new CatalogPluginCatalogExtractor().apply(catalogVal) - val namespace = getFieldVal[Seq[String]](v1, "namespace") + val catalogVal = invokeAs[AnyRef](v1, "catalog") + val catalog = lookupExtractor[CatalogPluginCatalogExtractor].apply(catalogVal) + val namespace = getField[Seq[String]](v1, "namespace") Database(catalog, quote(namespace)) } } @@ -80,9 +81,9 @@ class ResolvedNamespaceDatabaseExtractor extends DatabaseExtractor { */ class ResolvedDBObjectNameDatabaseExtractor extends DatabaseExtractor { override def apply(v1: AnyRef): Database = { - val catalogVal = invoke(v1, "catalog") - val catalog = new CatalogPluginCatalogExtractor().apply(catalogVal) - val namespace = getFieldVal[Seq[String]](v1, "nameParts") + val catalogVal = invokeAs[AnyRef](v1, "catalog") + val catalog = lookupExtractor[CatalogPluginCatalogExtractor].apply(catalogVal) + val namespace = getField[Seq[String]](v1, "nameParts") Database(catalog, quote(namespace)) } } diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/functionExtractors.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/functionExtractors.scala index 894a6cb8f..bcd5f2665 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/functionExtractors.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/functionExtractors.scala @@ -20,12 +20,26 @@ package org.apache.kyuubi.plugin.spark.authz.serde import org.apache.spark.sql.catalyst.FunctionIdentifier import org.apache.spark.sql.catalyst.expressions.ExpressionInfo +import org.apache.kyuubi.plugin.spark.authz.serde.FunctionExtractor.buildFunctionFromQualifiedName + trait FunctionExtractor extends (AnyRef => Function) with Extractor object FunctionExtractor { val functionExtractors: Map[String, FunctionExtractor] = { loadExtractorsToMap[FunctionExtractor] } + + private[authz] def buildFunctionFromQualifiedName(qualifiedName: String): Function = { + val parts: Array[String] = qualifiedName.split("\\.") + val (catalog, database, functionName) = if (parts.length == 3) { + (Some(parts.head), Some(parts.tail.head), parts.last) + } else if (parts.length == 2) { + (None, Some(parts.head), parts.last) + } else { + (None, None, qualifiedName) + } + Function(catalog, database, functionName) + } } /** @@ -33,7 +47,17 @@ object FunctionExtractor { */ class StringFunctionExtractor extends FunctionExtractor { override def apply(v1: AnyRef): Function = { - Function(None, v1.asInstanceOf[String]) + Function(None, None, v1.asInstanceOf[String]) + } +} + +/** + * * String + */ +class QualifiedNameStringFunctionExtractor extends FunctionExtractor { + override def apply(v1: AnyRef): Function = { + val qualifiedName: String = v1.asInstanceOf[String] + buildFunctionFromQualifiedName(qualifiedName) } } @@ -43,7 +67,7 @@ class StringFunctionExtractor extends FunctionExtractor { class FunctionIdentifierFunctionExtractor extends FunctionExtractor { override def apply(v1: AnyRef): Function = { val identifier = v1.asInstanceOf[FunctionIdentifier] - Function(identifier.database, identifier.funcName) + Function(None, identifier.database, identifier.funcName) } } @@ -53,6 +77,6 @@ class FunctionIdentifierFunctionExtractor extends FunctionExtractor { class ExpressionInfoFunctionExtractor extends FunctionExtractor { override def apply(v1: AnyRef): Function = { val info = v1.asInstanceOf[ExpressionInfo] - Function(Option(info.getDb), info.getName) + Function(None, Option(info.getDb), info.getName) } } diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/functionTypeExtractors.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/functionTypeExtractors.scala index 4c5e9dc84..c134b5018 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/functionTypeExtractors.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/functionTypeExtractors.scala @@ -19,8 +19,11 @@ package org.apache.kyuubi.plugin.spark.authz.serde import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.FunctionIdentifier +import org.apache.spark.sql.catalyst.catalog.SessionCatalog +import org.apache.kyuubi.plugin.spark.authz.serde.FunctionExtractor.buildFunctionFromQualifiedName import org.apache.kyuubi.plugin.spark.authz.serde.FunctionType.{FunctionType, PERMANENT, SYSTEM, TEMP} +import org.apache.kyuubi.plugin.spark.authz.serde.FunctionTypeExtractor.getFunctionType object FunctionType extends Enumeration { type FunctionType = Value @@ -33,6 +36,19 @@ object FunctionTypeExtractor { val functionTypeExtractors: Map[String, FunctionTypeExtractor] = { loadExtractorsToMap[FunctionTypeExtractor] } + + def getFunctionType(fi: FunctionIdentifier, catalog: SessionCatalog): FunctionType = { + fi match { + case temp if catalog.isTemporaryFunction(temp) => + TEMP + case permanent if catalog.isPersistentFunction(permanent) => + PERMANENT + case system if catalog.isRegisteredFunction(system) => + SYSTEM + case _ => + TEMP + } + } } /** @@ -53,9 +69,9 @@ class TempMarkerFunctionTypeExtractor extends FunctionTypeExtractor { */ class ExpressionInfoFunctionTypeExtractor extends FunctionTypeExtractor { override def apply(v1: AnyRef, spark: SparkSession): FunctionType = { - val function = new ExpressionInfoFunctionExtractor().apply(v1) + val function = lookupExtractor[ExpressionInfoFunctionExtractor].apply(v1) val fi = FunctionIdentifier(function.functionName, function.database) - new FunctionIdentifierFunctionTypeExtractor().apply(fi, spark) + lookupExtractor[FunctionIdentifierFunctionTypeExtractor].apply(fi, spark) } } @@ -66,14 +82,18 @@ class FunctionIdentifierFunctionTypeExtractor extends FunctionTypeExtractor { override def apply(v1: AnyRef, spark: SparkSession): FunctionType = { val catalog = spark.sessionState.catalog val fi = v1.asInstanceOf[FunctionIdentifier] - if (catalog.isTemporaryFunction(fi)) { - TEMP - } else if (catalog.isPersistentFunction(fi)) { - PERMANENT - } else if (catalog.isRegisteredFunction(fi)) { - SYSTEM - } else { - TEMP - } + getFunctionType(fi, catalog) + } +} + +/** + * String + */ +class FunctionNameFunctionTypeExtractor extends FunctionTypeExtractor { + override def apply(v1: AnyRef, spark: SparkSession): FunctionType = { + val catalog: SessionCatalog = spark.sessionState.catalog + val qualifiedName: String = v1.asInstanceOf[String] + val function = buildFunctionFromQualifiedName(qualifiedName) + getFunctionType(FunctionIdentifier(function.functionName, function.database), catalog) } } diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/package.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/package.scala index a52a558a0..1c5ffb629 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/package.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/package.scala @@ -17,9 +17,6 @@ package org.apache.kyuubi.plugin.spark.authz -import java.util.ServiceLoader - -import scala.collection.JavaConverters._ import scala.reflect.ClassTag import com.fasterxml.jackson.core.`type`.TypeReference @@ -28,16 +25,24 @@ import com.fasterxml.jackson.module.scala.DefaultScalaModule import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.kyuubi.plugin.spark.authz.OperationType.{OperationType, QUERY} +import org.apache.kyuubi.plugin.spark.authz.serde.ActionTypeExtractor.actionTypeExtractors +import org.apache.kyuubi.plugin.spark.authz.serde.CatalogExtractor.catalogExtractors +import org.apache.kyuubi.plugin.spark.authz.serde.ColumnExtractor.columnExtractors +import org.apache.kyuubi.plugin.spark.authz.serde.DatabaseExtractor.dbExtractors +import org.apache.kyuubi.plugin.spark.authz.serde.FunctionExtractor.functionExtractors +import org.apache.kyuubi.plugin.spark.authz.serde.FunctionTypeExtractor.functionTypeExtractors +import org.apache.kyuubi.plugin.spark.authz.serde.QueryExtractor.queryExtractors +import org.apache.kyuubi.plugin.spark.authz.serde.TableExtractor.tableExtractors +import org.apache.kyuubi.plugin.spark.authz.serde.TableTypeExtractor.tableTypeExtractors +import org.apache.kyuubi.plugin.spark.authz.serde.URIExtractor.uriExtractors +import org.apache.kyuubi.util.reflect.ReflectUtils._ package object serde { final val mapper = JsonMapper.builder().addModule(DefaultScalaModule).build() - def loadExtractorsToMap[T <: Extractor](implicit ct: ClassTag[T]): Map[String, T] = { - ServiceLoader.load(ct.runtimeClass).iterator().asScala - .map { case e: Extractor => (e.key, e.asInstanceOf[T]) } - .toMap - } + def loadExtractorsToMap[T <: Extractor](implicit ct: ClassTag[T]): Map[String, T] = + loadFromServiceLoader[T]()(ct).map { e: T => (e.key, e) }.toMap final lazy val DB_COMMAND_SPECS: Map[String, DatabaseCommandSpec] = { val is = getClass.getClassLoader.getResourceAsStream("database_command_spec.json") @@ -68,7 +73,8 @@ package object serde { final private lazy val SCAN_SPECS: Map[String, ScanSpec] = { val is = getClass.getClassLoader.getResourceAsStream("scan_command_spec.json") mapper.readValue(is, new TypeReference[Array[ScanSpec]] {}) - .map(e => (e.classname, e)).toMap + .map(e => (e.classname, e)) + .filter(t => t._2.scanDescs.nonEmpty).toMap } def isKnownScan(r: AnyRef): Boolean = { @@ -79,6 +85,21 @@ package object serde { SCAN_SPECS(r.getClass.getName) } + final private lazy val FUNCTION_SPECS: Map[String, ScanSpec] = { + val is = getClass.getClassLoader.getResourceAsStream("scan_command_spec.json") + mapper.readValue(is, new TypeReference[Array[ScanSpec]] {}) + .map(e => (e.classname, e)) + .filter(t => t._2.functionDescs.nonEmpty).toMap + } + + def isKnownFunction(r: AnyRef): Boolean = { + FUNCTION_SPECS.contains(r.getClass.getName) + } + + def getFunctionSpec(r: AnyRef): ScanSpec = { + FUNCTION_SPECS(r.getClass.getName) + } + def operationType(plan: LogicalPlan): OperationType = { val classname = plan.getClass.getName TABLE_COMMAND_SPECS.get(classname) @@ -87,4 +108,34 @@ package object serde { .map(s => s.operationType) .getOrElse(QUERY) } + + /** + * get extractor instance by extractor class name + * @param extractorKey explicitly load extractor by its simple class name. + * null by default means get extractor by extractor class. + * @param ct class tag of extractor class type + * @tparam T extractor class type + * @return + */ + def lookupExtractor[T <: Extractor](extractorKey: String)( + implicit ct: ClassTag[T]): T = { + val extractorClass = ct.runtimeClass + val extractors: Map[String, Extractor] = extractorClass match { + case c if classOf[CatalogExtractor].isAssignableFrom(c) => catalogExtractors + case c if classOf[DatabaseExtractor].isAssignableFrom(c) => dbExtractors + case c if classOf[TableExtractor].isAssignableFrom(c) => tableExtractors + case c if classOf[TableTypeExtractor].isAssignableFrom(c) => tableTypeExtractors + case c if classOf[ColumnExtractor].isAssignableFrom(c) => columnExtractors + case c if classOf[QueryExtractor].isAssignableFrom(c) => queryExtractors + case c if classOf[FunctionExtractor].isAssignableFrom(c) => functionExtractors + case c if classOf[FunctionTypeExtractor].isAssignableFrom(c) => functionTypeExtractors + case c if classOf[ActionTypeExtractor].isAssignableFrom(c) => actionTypeExtractors + case c if classOf[URIExtractor].isAssignableFrom(c) => uriExtractors + case _ => throw new IllegalArgumentException(s"Unknown extractor type: $ct") + } + extractors(extractorKey).asInstanceOf[T] + } + + def lookupExtractor[T <: Extractor](implicit ct: ClassTag[T]): T = + lookupExtractor[T](ct.runtimeClass.getSimpleName)(ct) } diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/pathExtractors.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/pathExtractors.scala new file mode 100644 index 000000000..81fa8411b --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/pathExtractors.scala @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.plugin.spark.authz.serde + +trait URIExtractor extends (AnyRef => Option[Uri]) with Extractor + +object URIExtractor { + val uriExtractors: Map[String, URIExtractor] = { + loadExtractorsToMap[URIExtractor] + } +} + +/** + * String + */ +class StringURIExtractor extends URIExtractor { + override def apply(v1: AnyRef): Option[Uri] = { + Some(Uri(v1.asInstanceOf[String])) + } +} diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/queryExtractors.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/queryExtractors.scala index f6fc19ac2..4ac87e100 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/queryExtractors.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/queryExtractors.scala @@ -19,6 +19,8 @@ package org.apache.kyuubi.plugin.spark.authz.serde import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.kyuubi.util.reflect.ReflectUtils.invokeAs + trait QueryExtractor extends (AnyRef => Option[LogicalPlan]) with Extractor object QueryExtractor { @@ -44,3 +46,9 @@ class LogicalPlanOptionQueryExtractor extends QueryExtractor { v1.asInstanceOf[Option[LogicalPlan]] } } + +class HudiMergeIntoSourceTableExtractor extends QueryExtractor { + override def apply(v1: AnyRef): Option[LogicalPlan] = { + new LogicalPlanQueryExtractor().apply(invokeAs[LogicalPlan](v1, "sourceTable")) + } +} diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/tableExtractors.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/tableExtractors.scala index c848381d4..a54b58c33 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/tableExtractors.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/serde/tableExtractors.scala @@ -18,15 +18,20 @@ package org.apache.kyuubi.plugin.spark.authz.serde import java.util.{Map => JMap} +import java.util.LinkedHashMap import scala.collection.JavaConverters._ import org.apache.spark.sql.SparkSession -import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.catalyst.{InternalRow, TableIdentifier} import org.apache.spark.sql.catalyst.catalog.CatalogTable -import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.apache.spark.sql.catalyst.expressions.Expression +import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, SubqueryAlias} +import org.apache.spark.sql.types.DataType +import org.apache.spark.unsafe.types.UTF8String import org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils._ +import org.apache.kyuubi.util.reflect.ReflectUtils._ /** * A trait for extracting database and table as string tuple @@ -46,10 +51,25 @@ object TableExtractor { */ def getOwner(v: AnyRef): Option[String] = { // org.apache.spark.sql.connector.catalog.Table - val table = invoke(v, "table") + val table = invokeAs[AnyRef](v, "table") val properties = invokeAs[JMap[String, String]](table, "properties").asScala properties.get("owner") } + + def getOwner(spark: SparkSession, catalogName: String, tableIdent: AnyRef): Option[String] = { + try { + val catalogManager = invokeAs[AnyRef](spark.sessionState, "catalogManager") + val catalog = invokeAs[AnyRef](catalogManager, "catalog", (classOf[String], catalogName)) + val table = invokeAs[AnyRef]( + catalog, + "loadTable", + (Class.forName("org.apache.spark.sql.connector.catalog.Identifier"), tableIdent)) + getOwner(table) + } catch { + // Exception may occur due to invalid reflection or table not found + case _: Exception => None + } + } } /** @@ -74,10 +94,14 @@ class TableIdentifierTableExtractor extends TableExtractor { */ class CatalogTableTableExtractor extends TableExtractor { override def apply(spark: SparkSession, v1: AnyRef): Option[Table] = { - val catalogTable = v1.asInstanceOf[CatalogTable] - val identifier = catalogTable.identifier - val owner = Option(catalogTable.owner).filter(_.nonEmpty) - Some(Table(None, identifier.database, identifier.table, owner)) + if (null == v1) { + None + } else { + val catalogTable = v1.asInstanceOf[CatalogTable] + val identifier = catalogTable.identifier + val owner = Option(catalogTable.owner).filter(_.nonEmpty) + Some(Table(None, identifier.database, identifier.table, owner)) + } } } @@ -87,7 +111,7 @@ class CatalogTableTableExtractor extends TableExtractor { class CatalogTableOptionTableExtractor extends TableExtractor { override def apply(spark: SparkSession, v1: AnyRef): Option[Table] = { val catalogTable = v1.asInstanceOf[Option[CatalogTable]] - catalogTable.flatMap(new CatalogTableTableExtractor().apply(spark, _)) + catalogTable.flatMap(lookupExtractor[CatalogTableTableExtractor].apply(spark, _)) } } @@ -96,10 +120,10 @@ class CatalogTableOptionTableExtractor extends TableExtractor { */ class ResolvedTableTableExtractor extends TableExtractor { override def apply(spark: SparkSession, v1: AnyRef): Option[Table] = { - val catalogVal = invoke(v1, "catalog") - val catalog = new CatalogPluginCatalogExtractor().apply(catalogVal) - val identifier = invoke(v1, "identifier") - val maybeTable = new IdentifierTableExtractor().apply(spark, identifier) + val catalogVal = invokeAs[AnyRef](v1, "catalog") + val catalog = lookupExtractor[CatalogPluginCatalogExtractor].apply(catalogVal) + val identifier = invokeAs[AnyRef](v1, "identifier") + val maybeTable = lookupExtractor[IdentifierTableExtractor].apply(spark, identifier) val maybeOwner = TableExtractor.getOwner(v1) maybeTable.map(_.copy(catalog = catalog, owner = maybeOwner)) } @@ -116,6 +140,34 @@ class IdentifierTableExtractor extends TableExtractor { } } +/** + * java.lang.String + * with concat parts by "." + */ +class StringTableExtractor extends TableExtractor { + override def apply(spark: SparkSession, v1: AnyRef): Option[Table] = { + val tableNameArr = v1.asInstanceOf[String].split("\\.") + val maybeTable = tableNameArr.length match { + case 1 => Table(None, None, tableNameArr(0), None) + case 2 => Table(None, Some(tableNameArr(0)), tableNameArr(1), None) + case 3 => Table(Some(tableNameArr(0)), Some(tableNameArr(1)), tableNameArr(2), None) + } + Option(maybeTable) + } +} + +/** + * Seq[org.apache.spark.sql.catalyst.expressions.Expression] + */ +class ExpressionSeqTableExtractor extends TableExtractor { + override def apply(spark: SparkSession, v1: AnyRef): Option[Table] = { + val expressions = v1.asInstanceOf[Seq[Expression]] + // Iceberg will rearrange the parameters according to the parameter order + // defined in the procedure, where the table parameters are currently always the first. + lookupExtractor[StringTableExtractor].apply(spark, expressions.head.toString()) + } +} + /** * org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation */ @@ -128,13 +180,12 @@ class DataSourceV2RelationTableExtractor extends TableExtractor { case Some(v2Relation) => val maybeCatalogPlugin = invokeAs[Option[AnyRef]](v2Relation, "catalog") val maybeCatalog = maybeCatalogPlugin.flatMap(catalogPlugin => - new CatalogPluginCatalogExtractor().apply(catalogPlugin)) - val maybeIdentifier = invokeAs[Option[AnyRef]](v2Relation, "identifier") - maybeIdentifier.flatMap { id => - val maybeTable = new IdentifierTableExtractor().apply(spark, id) - val maybeOwner = TableExtractor.getOwner(v2Relation) - maybeTable.map(_.copy(catalog = maybeCatalog, owner = maybeOwner)) - } + lookupExtractor[CatalogPluginCatalogExtractor].apply(catalogPlugin)) + lookupExtractor[TableTableExtractor].apply(spark, invokeAs[AnyRef](v2Relation, "table")) + .map { table => + val maybeOwner = TableExtractor.getOwner(v2Relation) + table.copy(catalog = maybeCatalog, owner = maybeOwner) + } } } } @@ -146,7 +197,7 @@ class LogicalRelationTableExtractor extends TableExtractor { override def apply(spark: SparkSession, v1: AnyRef): Option[Table] = { val maybeCatalogTable = invokeAs[Option[AnyRef]](v1, "catalogTable") maybeCatalogTable.flatMap { ct => - new CatalogTableTableExtractor().apply(spark, ct) + lookupExtractor[CatalogTableTableExtractor].apply(spark, ct) } } } @@ -156,11 +207,287 @@ class LogicalRelationTableExtractor extends TableExtractor { */ class ResolvedDbObjectNameTableExtractor extends TableExtractor { override def apply(spark: SparkSession, v1: AnyRef): Option[Table] = { - val catalogVal = invoke(v1, "catalog") - val catalog = new CatalogPluginCatalogExtractor().apply(catalogVal) + val catalogVal = invokeAs[AnyRef](v1, "catalog") + val catalog = lookupExtractor[CatalogPluginCatalogExtractor].apply(catalogVal) val nameParts = invokeAs[Seq[String]](v1, "nameParts") val namespace = nameParts.init.toArray val table = nameParts.last Some(Table(catalog, Some(quote(namespace)), table, None)) } } + +/** + * org.apache.spark.sql.catalyst.analysis.ResolvedIdentifier + */ +class ResolvedIdentifierTableExtractor extends TableExtractor { + override def apply(spark: SparkSession, v1: AnyRef): Option[Table] = { + v1.getClass.getName match { + case "org.apache.spark.sql.catalyst.analysis.ResolvedIdentifier" => + val catalogVal = invokeAs[AnyRef](v1, "catalog") + val catalog = lookupExtractor[CatalogPluginCatalogExtractor].apply(catalogVal) + val identifier = invokeAs[AnyRef](v1, "identifier") + val maybeTable = lookupExtractor[IdentifierTableExtractor].apply(spark, identifier) + val owner = catalog.flatMap(name => TableExtractor.getOwner(spark, name, identifier)) + maybeTable.map(_.copy(catalog = catalog, owner = owner)) + case _ => None + } + } +} + +/** + * org.apache.spark.sql.connector.catalog.Table + */ +class TableTableExtractor extends TableExtractor { + override def apply(spark: SparkSession, v1: AnyRef): Option[Table] = { + val tableName = invokeAs[String](v1, "name") + lookupExtractor[StringTableExtractor].apply(spark, tableName) + } +} + +class HudiDataSourceV2RelationTableExtractor extends TableExtractor { + override def apply(spark: SparkSession, v1: AnyRef): Option[Table] = { + invokeAs[LogicalPlan](v1, "table") match { + // Match multipartIdentifier with tableAlias + case SubqueryAlias(_, SubqueryAlias(identifier, _)) => + lookupExtractor[StringTableExtractor].apply(spark, identifier.toString()) + // Match multipartIdentifier without tableAlias + case SubqueryAlias(identifier, _) => + lookupExtractor[StringTableExtractor].apply(spark, identifier.toString()) + } + } +} + +class HudiMergeIntoTargetTableExtractor extends TableExtractor { + override def apply(spark: SparkSession, v1: AnyRef): Option[Table] = { + invokeAs[LogicalPlan](v1, "targetTable") match { + // Match multipartIdentifier with tableAlias + case SubqueryAlias(_, SubqueryAlias(identifier, relation)) => + lookupExtractor[StringTableExtractor].apply(spark, identifier.toString()) + // Match multipartIdentifier without tableAlias + case SubqueryAlias(identifier, _) => + lookupExtractor[StringTableExtractor].apply(spark, identifier.toString()) + } + } +} + +abstract class HudiCallProcedureTableExtractor extends TableExtractor { + + protected def extractTableIdentifier( + procedure: AnyRef, + args: AnyRef, + tableParameterKey: String): Option[String] = { + val tableIdentifierParameter = + invokeAs[Array[AnyRef]](procedure, "parameters") + .find(invokeAs[String](_, "name").equals(tableParameterKey)) + .getOrElse(throw new IllegalArgumentException(s"Could not find param $tableParameterKey")) + val tableIdentifierParameterIndex = invokeAs[LinkedHashMap[String, Int]](args, "map") + .getOrDefault(tableParameterKey, INVALID_INDEX) + tableIdentifierParameterIndex match { + case INVALID_INDEX => + None + case argsIndex => + val dataType = invokeAs[DataType](tableIdentifierParameter, "dataType") + val row = invokeAs[InternalRow](args, "internalRow") + val tableName = InternalRow.getAccessor(dataType, true)(row, argsIndex) + Option(tableName.asInstanceOf[UTF8String].toString) + } + } + + case class ProcedureArgsInputOutputPair( + input: Option[String] = None, + output: Option[String] = None) + + protected val PROCEDURE_CLASS_PATH = "org.apache.spark.sql.hudi.command.procedures" + + protected val INVALID_INDEX = -1 + + // These pairs are used to get the procedure input/output args which user passed in call command. + protected val procedureArgsInputOutputPairs: Map[String, ProcedureArgsInputOutputPair] = Map( + ( + s"$PROCEDURE_CLASS_PATH.ArchiveCommitsProcedure", + ProcedureArgsInputOutputPair(output = Some("table"))), + ( + s"$PROCEDURE_CLASS_PATH.CommitsCompareProcedure", + ProcedureArgsInputOutputPair(input = Some("table"))), + ( + s"$PROCEDURE_CLASS_PATH.CopyToTableProcedure", + ProcedureArgsInputOutputPair( + input = Some("table"), + output = Some("new_table"))), + ( + s"$PROCEDURE_CLASS_PATH.CopyToTempViewProcedure", + ProcedureArgsInputOutputPair(input = Some("table"))), + ( + s"$PROCEDURE_CLASS_PATH.CreateMetadataTableProcedure", + ProcedureArgsInputOutputPair(output = Some("table"))), + ( + s"$PROCEDURE_CLASS_PATH.CreateSavepointProcedure", + ProcedureArgsInputOutputPair(output = Some("table"))), + ( + s"$PROCEDURE_CLASS_PATH.DeleteMarkerProcedure", + ProcedureArgsInputOutputPair(output = Some("table"))), + ( + s"$PROCEDURE_CLASS_PATH.DeleteMetadataTableProcedure", + ProcedureArgsInputOutputPair(output = Some("table"))), + ( + s"$PROCEDURE_CLASS_PATH.DeleteSavepointProcedure", + ProcedureArgsInputOutputPair(output = Some("table"))), + ( + s"$PROCEDURE_CLASS_PATH.ExportInstantsProcedure", + ProcedureArgsInputOutputPair(input = Some("table"))), + ( + s"$PROCEDURE_CLASS_PATH.HdfsParquetImportProcedure", + ProcedureArgsInputOutputPair(output = Some("table"))), + ( + s"$PROCEDURE_CLASS_PATH.HelpProcedure", + ProcedureArgsInputOutputPair()), + ( + s"$PROCEDURE_CLASS_PATH.HiveSyncProcedure", + ProcedureArgsInputOutputPair(input = Some("table"))), + ( + s"$PROCEDURE_CLASS_PATH.InitMetadataTableProcedure", + ProcedureArgsInputOutputPair(output = Some("table"))), + ( + s"$PROCEDURE_CLASS_PATH.RepairAddpartitionmetaProcedure", + ProcedureArgsInputOutputPair(output = Some("table"))), + ( + s"$PROCEDURE_CLASS_PATH.RepairCorruptedCleanFilesProcedure", + ProcedureArgsInputOutputPair(output = Some("table"))), + ( + s"$PROCEDURE_CLASS_PATH.RepairDeduplicateProcedure", + ProcedureArgsInputOutputPair(output = Some("table"))), + ( + s"$PROCEDURE_CLASS_PATH.RepairMigratePartitionMetaProcedure", + ProcedureArgsInputOutputPair(output = Some("table"))), + ( + s"$PROCEDURE_CLASS_PATH.RepairOverwriteHoodiePropsProcedure", + ProcedureArgsInputOutputPair(output = Some("table"))), + ( + s"$PROCEDURE_CLASS_PATH.RollbackToInstantTimeProcedure", + ProcedureArgsInputOutputPair(output = Some("table"))), + ( + s"$PROCEDURE_CLASS_PATH.RollbackToSavepointProcedure", + ProcedureArgsInputOutputPair(output = Some("table"))), + ( + s"$PROCEDURE_CLASS_PATH.RunBootstrapProcedure", + ProcedureArgsInputOutputPair( + input = Some("table"), + output = Some("table"))), + ( + s"$PROCEDURE_CLASS_PATH.RunCleanProcedure", + ProcedureArgsInputOutputPair( + input = Some("table"), + output = Some("table"))), + ( + s"$PROCEDURE_CLASS_PATH.RunClusteringProcedure", + ProcedureArgsInputOutputPair( + input = Some("table"), + output = Some("table"))), + ( + s"$PROCEDURE_CLASS_PATH.RunCompactionProcedure", + ProcedureArgsInputOutputPair( + input = Some("table"), + output = Some("table"))), + ( + s"$PROCEDURE_CLASS_PATH.ShowArchivedCommitsProcedure", + ProcedureArgsInputOutputPair(input = Some("table"))), + ( + s"$PROCEDURE_CLASS_PATH.ShowBootstrapMappingProcedure", + ProcedureArgsInputOutputPair(input = Some("table"))), + ( + s"$PROCEDURE_CLASS_PATH.ShowClusteringProcedure", + ProcedureArgsInputOutputPair(input = Some("table"))), + ( + s"$PROCEDURE_CLASS_PATH.ShowCommitExtraMetadataProcedure", + ProcedureArgsInputOutputPair(input = Some("table"))), + ( + s"$PROCEDURE_CLASS_PATH.ShowCommitFilesProcedure", + ProcedureArgsInputOutputPair(input = Some("table"))), + ( + s"$PROCEDURE_CLASS_PATH.ShowCommitPartitionsProcedure", + ProcedureArgsInputOutputPair(input = Some("table"))), + ( + s"$PROCEDURE_CLASS_PATH.ShowCommitWriteStatsProcedure", + ProcedureArgsInputOutputPair(input = Some("table"))), + ( + s"$PROCEDURE_CLASS_PATH.ShowCompactionProcedure", + ProcedureArgsInputOutputPair(input = Some("table"))), + ( + s"$PROCEDURE_CLASS_PATH.ShowFileSystemViewProcedure", + ProcedureArgsInputOutputPair(input = Some("table"))), + ( + s"$PROCEDURE_CLASS_PATH.ShowFsPathDetailProcedure", + ProcedureArgsInputOutputPair()), + ( + s"$PROCEDURE_CLASS_PATH.ShowHoodieLogFileMetadataProcedure", + ProcedureArgsInputOutputPair(input = Some("table"))), + ( + s"$PROCEDURE_CLASS_PATH.ShowHoodieLogFileRecordsProcedure", + ProcedureArgsInputOutputPair(input = Some("table"))), + ( + s"$PROCEDURE_CLASS_PATH.ShowInvalidParquetProcedure", + ProcedureArgsInputOutputPair()), + ( + s"$PROCEDURE_CLASS_PATH.ShowMetadataTableFilesProcedure", + ProcedureArgsInputOutputPair(input = Some("table"))), + ( + s"$PROCEDURE_CLASS_PATH.ShowMetadataTablePartitionsProcedure", + ProcedureArgsInputOutputPair(input = Some("table"))), + ( + s"$PROCEDURE_CLASS_PATH.ShowMetadataTableStatsProcedure", + ProcedureArgsInputOutputPair(input = Some("table"))), + ( + s"$PROCEDURE_CLASS_PATH.ShowRollbacksProcedure", + ProcedureArgsInputOutputPair(input = Some("table"))), + ( + s"$PROCEDURE_CLASS_PATH.ShowSavepointsProcedure", + ProcedureArgsInputOutputPair(input = Some("table"))), + ( + s"$PROCEDURE_CLASS_PATH.ShowTablePropertiesProcedure", + ProcedureArgsInputOutputPair(input = Some("table"))), + ( + s"$PROCEDURE_CLASS_PATH.StatsFileSizeProcedure", + ProcedureArgsInputOutputPair(input = Some("table"))), + ( + s"$PROCEDURE_CLASS_PATH.StatsWriteAmplificationProcedure", + ProcedureArgsInputOutputPair(input = Some("table"))), + ( + s"$PROCEDURE_CLASS_PATH.UpgradeOrDowngradeProcedure", + ProcedureArgsInputOutputPair(output = Some("table"))), + ( + s"$PROCEDURE_CLASS_PATH.ValidateHoodieSyncProcedure", + ProcedureArgsInputOutputPair( + input = Some("src_table"), + output = Some("dst_table"))), + ( + s"$PROCEDURE_CLASS_PATH.ValidateMetadataTableFilesProcedure", + ProcedureArgsInputOutputPair(input = Some("table")))) +} + +class HudiCallProcedureOutputTableExtractor + extends HudiCallProcedureTableExtractor { + override def apply(spark: SparkSession, v1: AnyRef): Option[Table] = { + val procedure = invokeAs[AnyRef](v1, "procedure") + val args = invokeAs[AnyRef](v1, "args") + procedureArgsInputOutputPairs.get(procedure.getClass.getName) + .filter(_.output.isDefined) + .map { argsPairs => + val tableIdentifier = extractTableIdentifier(procedure, args, argsPairs.output.get) + lookupExtractor[StringTableExtractor].apply(spark, tableIdentifier.get).orNull + } + } +} + +class HudiCallProcedureInputTableExtractor + extends HudiCallProcedureTableExtractor { + override def apply(spark: SparkSession, v1: AnyRef): Option[Table] = { + val procedure = invokeAs[AnyRef](v1, "procedure") + val args = invokeAs[AnyRef](v1, "args") + procedureArgsInputOutputPairs.get(procedure.getClass.getName) + .filter(_.input.isDefined) + .map { argsPairs => + val tableIdentifier = extractTableIdentifier(procedure, args, argsPairs.input.get) + lookupExtractor[StringTableExtractor].apply(spark, tableIdentifier.get).orNull + } + } +} diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/AuthZUtils.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/AuthZUtils.scala index 5773e1c93..2477c9e45 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/AuthZUtils.scala +++ b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/AuthZUtils.scala @@ -23,8 +23,6 @@ import java.security.interfaces.ECPublicKey import java.security.spec.X509EncodedKeySpec import java.util.Base64 -import scala.util.{Failure, Success, Try} - import org.apache.commons.lang3.StringUtils import org.apache.hadoop.security.UserGroupInformation import org.apache.ranger.plugin.service.RangerBasePlugin @@ -33,67 +31,12 @@ import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, View} import org.apache.kyuubi.plugin.spark.authz.AccessControlException import org.apache.kyuubi.plugin.spark.authz.util.ReservedKeys._ +import org.apache.kyuubi.util.SemanticVersion +import org.apache.kyuubi.util.reflect.DynConstructors +import org.apache.kyuubi.util.reflect.ReflectUtils._ private[authz] object AuthZUtils { - /** - * fixme error handling need improve here - */ - def getFieldVal[T](o: Any, name: String): T = { - Try { - val field = o.getClass.getDeclaredField(name) - field.setAccessible(true) - field.get(o) - } match { - case Success(value) => value.asInstanceOf[T] - case Failure(e) => - val candidates = o.getClass.getDeclaredFields.map(_.getName).mkString("[", ",", "]") - throw new RuntimeException(s"$name not in ${o.getClass} $candidates", e) - } - } - - def getFieldValOpt[T](o: Any, name: String): Option[T] = Try(getFieldVal[T](o, name)).toOption - - def invoke( - obj: AnyRef, - methodName: String, - args: (Class[_], AnyRef)*): AnyRef = { - try { - val (types, values) = args.unzip - val method = obj.getClass.getMethod(methodName, types: _*) - method.setAccessible(true) - method.invoke(obj, values: _*) - } catch { - case e: NoSuchMethodException => - val candidates = obj.getClass.getMethods.map(_.getName).mkString("[", ",", "]") - throw new RuntimeException(s"$methodName not in ${obj.getClass} $candidates", e) - } - } - - def invokeAs[T]( - obj: AnyRef, - methodName: String, - args: (Class[_], AnyRef)*): T = { - invoke(obj, methodName, args: _*).asInstanceOf[T] - } - - def invokeStatic( - obj: Class[_], - methodName: String, - args: (Class[_], AnyRef)*): AnyRef = { - val (types, values) = args.unzip - val method = obj.getMethod(methodName, types: _*) - method.setAccessible(true) - method.invoke(obj, values: _*) - } - - def invokeStaticAs[T]( - obj: Class[_], - methodName: String, - args: (Class[_], AnyRef)*): T = { - invokeStatic(obj, methodName, args: _*).asInstanceOf[T] - } - /** * Get the active session user * @param spark spark context instance @@ -118,8 +61,8 @@ private[authz] object AuthZUtils { def hasResolvedPermanentView(plan: LogicalPlan): Boolean = { plan match { - case view: View if view.resolved && isSparkVersionAtLeast("3.1.0") => - !getFieldVal[Boolean](view, "isTempView") + case view: View if view.resolved => + !getField[Boolean](view, "isTempView") case _ => false } @@ -127,7 +70,12 @@ private[authz] object AuthZUtils { lazy val isRanger21orGreater: Boolean = { try { - classOf[RangerBasePlugin].getConstructor(classOf[String], classOf[String], classOf[String]) + DynConstructors.builder().impl( + classOf[RangerBasePlugin], + classOf[String], + classOf[String], + classOf[String]) + .buildChecked[RangerBasePlugin]() true } catch { case _: NoSuchMethodException => @@ -135,30 +83,15 @@ private[authz] object AuthZUtils { } } - def isSparkVersionAtMost(targetVersionString: String): Boolean = { - SemanticVersion(SPARK_VERSION).isVersionAtMost(targetVersionString) - } + lazy val SPARK_RUNTIME_VERSION: SemanticVersion = SemanticVersion(SPARK_VERSION) + lazy val isSparkV32OrGreater: Boolean = SPARK_RUNTIME_VERSION >= "3.2" + lazy val isSparkV33OrGreater: Boolean = SPARK_RUNTIME_VERSION >= "3.3" + lazy val isSparkV34OrGreater: Boolean = SPARK_RUNTIME_VERSION >= "3.4" + lazy val isSparkV35OrGreater: Boolean = SPARK_RUNTIME_VERSION >= "3.5" - def isSparkVersionAtLeast(targetVersionString: String): Boolean = { - SemanticVersion(SPARK_VERSION).isVersionAtLeast(targetVersionString) - } - - def isSparkVersionEqualTo(targetVersionString: String): Boolean = { - SemanticVersion(SPARK_VERSION).isVersionEqualTo(targetVersionString) - } - - /** - * check if spark version satisfied - * first param is option of supported most spark version, - * and secont param is option of supported least spark version - * - * @return - */ - def passSparkVersionCheck: (Option[String], Option[String]) => Boolean = - (mostSparkVersion, leastSparkVersion) => { - mostSparkVersion.forall(isSparkVersionAtMost) && - leastSparkVersion.forall(isSparkVersionAtLeast) - } + lazy val SCALA_RUNTIME_VERSION: SemanticVersion = + SemanticVersion(scala.util.Properties.versionNumberString) + lazy val isScalaV213: Boolean = SCALA_RUNTIME_VERSION >= "2.13" def quoteIfNeeded(part: String): String = { if (part.matches("[a-zA-Z0-9_]+") && !part.matches("\\d+")) { diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/SemanticVersion.scala b/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/SemanticVersion.scala deleted file mode 100644 index 4d7e89725..000000000 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/SemanticVersion.scala +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kyuubi.plugin.spark.authz.util - -/** - * Encapsulate a component Spark version for the convenience of version checks. - * Copy from org.apache.kyuubi.engine.ComponentVersion - */ -case class SemanticVersion(majorVersion: Int, minorVersion: Int) { - - def isVersionAtMost(targetVersionString: String): Boolean = { - this.compareVersion( - targetVersionString, - (targetMajor: Int, targetMinor: Int, runtimeMajor: Int, runtimeMinor: Int) => - (runtimeMajor < targetMajor) || { - runtimeMajor == targetMajor && runtimeMinor <= targetMinor - }) - } - - def isVersionAtLeast(targetVersionString: String): Boolean = { - this.compareVersion( - targetVersionString, - (targetMajor: Int, targetMinor: Int, runtimeMajor: Int, runtimeMinor: Int) => - (runtimeMajor > targetMajor) || { - runtimeMajor == targetMajor && runtimeMinor >= targetMinor - }) - } - - def isVersionEqualTo(targetVersionString: String): Boolean = { - this.compareVersion( - targetVersionString, - (targetMajor: Int, targetMinor: Int, runtimeMajor: Int, runtimeMinor: Int) => - runtimeMajor == targetMajor && runtimeMinor == targetMinor) - } - - def compareVersion( - targetVersionString: String, - callback: (Int, Int, Int, Int) => Boolean): Boolean = { - val targetVersion = SemanticVersion(targetVersionString) - val targetMajor = targetVersion.majorVersion - val targetMinor = targetVersion.minorVersion - callback(targetMajor, targetMinor, this.majorVersion, this.minorVersion) - } - - override def toString: String = s"$majorVersion.$minorVersion" -} - -object SemanticVersion { - - def apply(versionString: String): SemanticVersion = { - """^(\d+)\.(\d+)(\..*)?$""".r.findFirstMatchIn(versionString) match { - case Some(m) => - SemanticVersion(m.group(1).toInt, m.group(2).toInt) - case None => - throw new IllegalArgumentException(s"Tried to parse '$versionString' as a project" + - s" version string, but it could not find the major and minor version numbers.") - } - } -} diff --git a/extensions/spark/kyuubi-spark-authz/src/test/gen/scala/org/apache/kyuubi/plugin/spark/authz/gen/PolicyJsonFileGenerator.scala b/extensions/spark/kyuubi-spark-authz/src/test/gen/scala/org/apache/kyuubi/plugin/spark/authz/gen/PolicyJsonFileGenerator.scala new file mode 100644 index 000000000..7996c8ecc --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/test/gen/scala/org/apache/kyuubi/plugin/spark/authz/gen/PolicyJsonFileGenerator.scala @@ -0,0 +1,361 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.plugin.spark.authz.gen + +import java.nio.charset.StandardCharsets +import java.nio.file.{Files, Paths, StandardOpenOption} +import java.util.UUID + +import com.fasterxml.jackson.annotation.JsonInclude.Include +import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper} +import com.fasterxml.jackson.databind.json.JsonMapper +import com.fasterxml.jackson.databind.node.ObjectNode +import com.fasterxml.jackson.module.scala.DefaultScalaModule +import org.apache.ranger.plugin.model.RangerPolicy +import org.scalatest.funsuite.AnyFunSuite + +// scalastyle:off +import org.apache.kyuubi.plugin.spark.authz.RangerTestNamespace._ +import org.apache.kyuubi.plugin.spark.authz.RangerTestUsers._ +import org.apache.kyuubi.plugin.spark.authz.gen.KRangerPolicyItemAccess.allowTypes +import org.apache.kyuubi.plugin.spark.authz.gen.KRangerPolicyResource._ +import org.apache.kyuubi.plugin.spark.authz.gen.RangerAccessType._ +import org.apache.kyuubi.plugin.spark.authz.gen.RangerClassConversions._ +import org.apache.kyuubi.util.AssertionUtils._ + +/** + * Generates the policy file to test/main/resources dir. + * + * To run the test suite: + * {{{ + * KYUUBI_UPDATE=0 dev/gen/gen_ranger_policy_json.sh + * }}} + * + * To regenerate the ranger policy file: + * {{{ + * dev/gen/gen_ranger_policy_json.sh + * }}} + */ +class PolicyJsonFileGenerator extends AnyFunSuite { + // scalastyle:on + final private val mapper: ObjectMapper = JsonMapper.builder() + .addModule(DefaultScalaModule) + .serializationInclusion(Include.NON_NULL) + .build() + + test("check ranger policy file") { + val pluginHome = getClass.getProtectionDomain.getCodeSource.getLocation.getPath + .split("target").head + val policyFileName = "sparkSql_hive_jenkins.json" + val policyFilePath = + Paths.get(pluginHome, "src", "test", "resources", policyFileName) + val generatedStr = mapper.writerWithDefaultPrettyPrinter() + .writeValueAsString(servicePolicies) + + if (sys.env.get("KYUUBI_UPDATE").contains("1")) { + // scalastyle:off println + println(s"Writing ranger policies to $policyFileName.") + // scalastyle:on println + Files.write( + policyFilePath, + generatedStr.getBytes(StandardCharsets.UTF_8), + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING) + } else { + assertFileContent( + policyFilePath, + Seq(generatedStr), + "dev/gen/gen_ranger_policy_json.sh", + splitFirstExpectedLine = true) + } + } + + private def servicePolicies: JsonNode = { + val inputStream = Thread.currentThread().getContextClassLoader + .getResourceAsStream("policies_base.json") + val rootObjNode = mapper.readTree(inputStream).asInstanceOf[ObjectNode] + val policies = genPolicies + // scalastyle:off println + println(s"Generated ${policies.size} policies.") + // scalastyle:on println + rootObjNode.set("policies", mapper.readTree(mapper.writeValueAsString(policies))) + } + + private def genPolicies: Iterable[RangerPolicy] = { + List[RangerPolicy]( + // access for all + policyAccessForAllUrl, + policyAccessForAllDbTableColumns, + policyAccessForAllDbUdf, + // access + policyAccessForDbAllColumns, + policyAccessForDefaultDbSrcTable, + policyAccessForDefaultBobUse, + policyAccessForDefaultBobSelect, + policyAccessForPermViewAccessOnly, + policyAccessForTable2AccessOnly, + // row filter + policyFilterForSrcTableKeyLessThan20, + policyFilterForPermViewKeyLessThan20, + // data masking + policyMaskForPermView, + policyMaskForPermViewUser, + policyMaskNullifyForValue2, + policyMaskShowFirst4ForValue3, + policyMaskDateShowYearForValue4, + policyMaskShowFirst4ForValue5) + // fill the id and guid with auto-increased index + .zipWithIndex + .map { + case (p, index) => + p.setId(index) + p.setGuid(UUID.nameUUIDFromBytes(index.toString.getBytes()).toString) + p + } + } + + // resources + private val allDatabaseRes = databaseRes("*") + private val allTableRes = tableRes("*") + private val allColumnRes = columnRes("*") + private val srcTableRes = tableRes("src") + + // policy type + private val POLICY_TYPE_ACCESS: Int = 0 + private val POLICY_TYPE_DATAMASK: Int = 1 + private val POLICY_TYPE_ROWFILTER: Int = 2 + + // policies + private val policyAccessForAllUrl = KRangerPolicy( + name = "all - url", + description = "Policy for all - url", + resources = Map("url" -> KRangerPolicyResource( + values = List("*"), + isRecursive = true)), + policyItems = List(KRangerPolicyItem( + users = List(admin), + accesses = allowTypes(select, update, create, drop, alter, index, lock, all, read, write), + delegateAdmin = true))) + + private val policyAccessForAllDbTableColumns = KRangerPolicy( + name = "all - database, table, column", + description = "Policy for all - database, table, column", + resources = Map(allDatabaseRes, allTableRes, allColumnRes), + policyItems = List(KRangerPolicyItem( + users = List(admin), + accesses = allowTypes(select, update, create, drop, alter, index, lock, all, read, write), + delegateAdmin = true))) + + private val policyAccessForAllDbUdf = KRangerPolicy( + name = "all - database, udf", + description = "Policy for all - database, udf", + resources = Map(allDatabaseRes, "udf" -> KRangerPolicyResource(values = List("*"))), + policyItems = List(KRangerPolicyItem( + users = List(admin), + accesses = allowTypes(select, update, create, drop, alter, index, lock, all, read, write), + delegateAdmin = true))) + + private val policyAccessForDbAllColumns = KRangerPolicy( + name = "all - database, udf", + description = "Policy for all - database, udf", + resources = Map( + databaseRes(defaultDb, sparkCatalog, icebergNamespace, namespace1), + allTableRes, + allColumnRes), + policyItems = List( + KRangerPolicyItem( + users = List(bob, permViewUser, ownerPlaceHolder), + accesses = allowTypes(select, update, create, drop, alter, index, lock, all, read, write), + delegateAdmin = true), + KRangerPolicyItem( + users = List(defaultTableOwner, createOnlyUser), + accesses = allowTypes(create), + delegateAdmin = true))) + + private val policyAccessForDefaultDbSrcTable = KRangerPolicy( + name = "default_kent", + resources = Map( + databaseRes(defaultDb, sparkCatalog), + srcTableRes, + columnRes("key")), + policyItems = List( + KRangerPolicyItem( + users = List(kent), + accesses = allowTypes(select, update, create, drop, alter, index, lock, all, read, write), + delegateAdmin = true), + KRangerPolicyItem( + users = List(defaultTableOwner, createOnlyUser), + accesses = allowTypes(create), + delegateAdmin = true))) + + private val policyFilterForSrcTableKeyLessThan20 = KRangerPolicy( + name = "src_key_less_than_20", + policyType = POLICY_TYPE_ROWFILTER, + resources = Map( + databaseRes(defaultDb), + srcTableRes), + rowFilterPolicyItems = List( + KRangerRowFilterPolicyItem( + rowFilterInfo = KRangerPolicyItemRowFilterInfo(filterExpr = "key<20"), + accesses = allowTypes(select), + users = List(bob, permViewUser)))) + + private val policyFilterForPermViewKeyLessThan20 = KRangerPolicy( + name = "perm_view_key_less_than_20", + policyType = POLICY_TYPE_ROWFILTER, + resources = Map( + databaseRes(defaultDb), + tableRes("perm_view")), + rowFilterPolicyItems = List( + KRangerRowFilterPolicyItem( + rowFilterInfo = KRangerPolicyItemRowFilterInfo(filterExpr = "key<20"), + accesses = allowTypes(select), + users = List(permViewUser)))) + + private val policyAccessForDefaultBobUse = KRangerPolicy( + name = "default_bob_use", + resources = Map( + databaseRes("default_bob", sparkCatalog), + tableRes("table_use*"), + allColumnRes), + policyItems = List( + KRangerPolicyItem( + users = List(bob), + accesses = allowTypes(update), + delegateAdmin = true))) + + private val policyAccessForDefaultBobSelect = KRangerPolicy( + name = "default_bob_select", + resources = Map( + databaseRes("default_bob", sparkCatalog), + tableRes("table_select*"), + allColumnRes), + policyItems = List( + KRangerPolicyItem( + users = List(bob), + accesses = allowTypes(select, use), + delegateAdmin = true))) + + private val policyMaskForPermView = KRangerPolicy( + name = "src_value_hash_perm_view", + policyType = POLICY_TYPE_DATAMASK, + resources = Map( + databaseRes(defaultDb, sparkCatalog), + srcTableRes, + columnRes("value1")), + dataMaskPolicyItems = List( + KRangerDataMaskPolicyItem( + dataMaskInfo = KRangerPolicyItemDataMaskInfo(dataMaskType = "MASK_HASH"), + users = List(bob), + accesses = allowTypes(select), + delegateAdmin = true))) + + private val policyMaskForPermViewUser = KRangerPolicy( + name = "src_value_hash", + policyType = POLICY_TYPE_DATAMASK, + resources = Map( + databaseRes(defaultDb, sparkCatalog), + tableRes("perm_view"), + columnRes("value1")), + dataMaskPolicyItems = List( + KRangerDataMaskPolicyItem( + dataMaskInfo = KRangerPolicyItemDataMaskInfo(dataMaskType = "MASK_HASH"), + users = List(permViewUser), + accesses = allowTypes(select), + delegateAdmin = true))) + + private val policyMaskNullifyForValue2 = KRangerPolicy( + name = "src_value2_nullify", + policyType = POLICY_TYPE_DATAMASK, + resources = Map( + databaseRes(defaultDb, sparkCatalog, icebergNamespace, namespace1), + srcTableRes, + columnRes("value2")), + dataMaskPolicyItems = List( + KRangerDataMaskPolicyItem( + dataMaskInfo = KRangerPolicyItemDataMaskInfo(dataMaskType = "MASK"), + users = List(bob), + accesses = allowTypes(select), + delegateAdmin = true))) + + private val policyMaskShowFirst4ForValue3 = KRangerPolicy( + name = "src_value3_sf4", + policyType = POLICY_TYPE_DATAMASK, + resources = Map( + databaseRes(defaultDb, sparkCatalog), + srcTableRes, + columnRes("value3")), + dataMaskPolicyItems = List( + KRangerDataMaskPolicyItem( + dataMaskInfo = KRangerPolicyItemDataMaskInfo(dataMaskType = "MASK_SHOW_FIRST_4"), + users = List(bob), + accesses = allowTypes(select), + delegateAdmin = true))) + + private val policyMaskDateShowYearForValue4 = KRangerPolicy( + name = "src_value4_sf4", + policyType = POLICY_TYPE_DATAMASK, + resources = Map( + databaseRes(defaultDb, sparkCatalog), + srcTableRes, + columnRes("value4")), + dataMaskPolicyItems = List( + KRangerDataMaskPolicyItem( + dataMaskInfo = KRangerPolicyItemDataMaskInfo(dataMaskType = "MASK_DATE_SHOW_YEAR"), + users = List(bob), + accesses = allowTypes(select), + delegateAdmin = true))) + + private val policyMaskShowFirst4ForValue5 = KRangerPolicy( + name = "src_value5_sf4", + policyType = POLICY_TYPE_DATAMASK, + resources = Map( + databaseRes(defaultDb, sparkCatalog), + srcTableRes, + columnRes("value5")), + dataMaskPolicyItems = List( + KRangerDataMaskPolicyItem( + dataMaskInfo = KRangerPolicyItemDataMaskInfo(dataMaskType = "MASK_SHOW_LAST_4"), + users = List(bob), + accesses = allowTypes(select), + delegateAdmin = true))) + + private val policyAccessForPermViewAccessOnly = KRangerPolicy( + name = "someone_access_perm_view", + resources = Map( + databaseRes(defaultDb), + tableRes("perm_view"), + allColumnRes), + policyItems = List( + KRangerPolicyItem( + users = List(permViewOnlyUser), + accesses = allowTypes(select), + delegateAdmin = true))) + + private val policyAccessForTable2AccessOnly = KRangerPolicy( + name = "someone_access_table2", + resources = Map( + databaseRes(defaultDb), + tableRes("table2"), + allColumnRes), + policyItems = List( + KRangerPolicyItem( + users = List(table2OnlyUser), + accesses = allowTypes(select), + delegateAdmin = true))) +} diff --git a/extensions/spark/kyuubi-spark-authz/src/test/gen/scala/org/apache/kyuubi/plugin/spark/authz/gen/RangerGenWrapper.scala b/extensions/spark/kyuubi-spark-authz/src/test/gen/scala/org/apache/kyuubi/plugin/spark/authz/gen/RangerGenWrapper.scala new file mode 100644 index 000000000..71bce3759 --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/test/gen/scala/org/apache/kyuubi/plugin/spark/authz/gen/RangerGenWrapper.scala @@ -0,0 +1,211 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.plugin.spark.authz.gen + +import scala.collection.convert.ImplicitConversions._ +import scala.language.implicitConversions + +import org.apache.ranger.plugin.model.RangerPolicy +import org.apache.ranger.plugin.model.RangerPolicy._ + +import org.apache.kyuubi.plugin.spark.authz.gen.RangerClassConversions._ + +trait RangerObjectGenerator[T] { + def get: T +} + +object RangerClassConversions { + implicit def getRangerObject[T](g: RangerObjectGenerator[T]): T = g.get +} + +case class KRangerPolicy( + service: String = "hive_jenkins", + name: String, + policyType: Int = 0, + description: String = "", + isAuditEnabled: Boolean = true, + resources: Map[String, RangerPolicyResource] = Map.empty, + conditions: List[RangerPolicyItemCondition] = List.empty, + policyItems: List[RangerPolicyItem] = List.empty, + denyPolicyItems: List[RangerPolicyItem] = List.empty, + allowExceptions: List[RangerPolicyItem] = List.empty, + denyExceptions: List[RangerPolicyItem] = List.empty, + dataMaskPolicyItems: List[RangerDataMaskPolicyItem] = List.empty, + rowFilterPolicyItems: List[RangerRowFilterPolicyItem] = List.empty, + id: Int = 0, + guid: String = "", + isEnabled: Boolean = true, + version: Int = 1) extends RangerObjectGenerator[RangerPolicy] { + override def get: RangerPolicy = { + val p = new RangerPolicy() + p.setService(service) + p.setName(name) + p.setPolicyType(policyType) + p.setDescription(description) + p.setIsAuditEnabled(isAuditEnabled) + p.setResources(resources) + p.setConditions(conditions) + p.setPolicyItems(policyItems) + p.setAllowExceptions(allowExceptions) + p.setDenyExceptions(denyExceptions) + p.setDataMaskPolicyItems(dataMaskPolicyItems) + p.setRowFilterPolicyItems(rowFilterPolicyItems) + p.setId(id) + p.setGuid(guid) + p.setIsAuditEnabled(isEnabled) + p.setVersion(version) + p + } +} + +case class KRangerPolicyResource( + values: List[String] = List.empty, + isExcludes: Boolean = false, + isRecursive: Boolean = false) extends RangerObjectGenerator[RangerPolicyResource] { + override def get: RangerPolicyResource = { + val r = new RangerPolicyResource() + r.setValues(values) + r.setIsExcludes(isExcludes) + r.setIsRecursive(isRecursive) + r + } +} + +object KRangerPolicyResource { + def databaseRes(values: String*): (String, RangerPolicyResource) = + "database" -> KRangerPolicyResource(values.toList) + + def tableRes(values: String*): (String, RangerPolicyResource) = + "table" -> KRangerPolicyResource(values.toList) + + def columnRes(values: String*): (String, RangerPolicyResource) = + "column" -> KRangerPolicyResource(values.toList) +} + +case class KRangerPolicyItemCondition( + `type`: String, + values: List[String]) extends RangerObjectGenerator[RangerPolicyItemCondition] { + override def get: RangerPolicyItemCondition = { + val c = new RangerPolicyItemCondition() + c.setType(`type`) + c.setValues(values) + c + } +} + +case class KRangerPolicyItem( + accesses: List[RangerPolicyItemAccess] = List.empty, + users: List[String] = List.empty, + groups: List[String] = List.empty, + conditions: List[RangerPolicyItemCondition] = List.empty, + delegateAdmin: Boolean = false) extends RangerObjectGenerator[RangerPolicyItem] { + override def get: RangerPolicyItem = { + val i = new RangerPolicyItem() + i.setAccesses(accesses) + i.setUsers(users) + i.setGroups(groups) + i.setConditions(conditions) + i.setDelegateAdmin(delegateAdmin) + i + } +} + +case class KRangerPolicyItemAccess( + `type`: String, + isAllowed: Boolean) extends RangerObjectGenerator[RangerPolicyItemAccess] { + override def get: RangerPolicyItemAccess = { + val a = new RangerPolicyItemAccess + a.setType(`type`) + a.setIsAllowed(isAllowed) + a + } +} + +object KRangerPolicyItemAccess { + def allowTypes(types: String*): List[RangerPolicyItemAccess] = + types.map(t => KRangerPolicyItemAccess(t, isAllowed = true).get).toList +} + +case class KRangerDataMaskPolicyItem( + dataMaskInfo: RangerPolicyItemDataMaskInfo, + accesses: List[RangerPolicyItemAccess] = List.empty, + users: List[String] = List.empty, + groups: List[String] = List.empty, + conditions: List[RangerPolicyItemCondition] = List.empty, + delegateAdmin: Boolean = false) extends RangerObjectGenerator[RangerDataMaskPolicyItem] { + override def get: RangerDataMaskPolicyItem = { + val i = new RangerDataMaskPolicyItem + i.setDataMaskInfo(dataMaskInfo) + i.setAccesses(accesses) + i.setUsers(users) + i.setGroups(groups) + i.setConditions(conditions) + i.setDelegateAdmin(delegateAdmin) + i + } +} + +case class KRangerPolicyItemDataMaskInfo( + dataMaskType: String) extends RangerObjectGenerator[RangerPolicyItemDataMaskInfo] { + override def get: RangerPolicyItemDataMaskInfo = { + val i = new RangerPolicyItemDataMaskInfo + i.setDataMaskType(dataMaskType) + i + } +} + +case class KRangerRowFilterPolicyItem( + rowFilterInfo: RangerPolicyItemRowFilterInfo, + accesses: List[RangerPolicyItemAccess] = List.empty, + users: List[String] = List.empty, + groups: List[String] = List.empty, + conditions: List[RangerPolicyItemCondition] = List.empty, + delegateAdmin: Boolean = false) extends RangerObjectGenerator[RangerRowFilterPolicyItem] { + override def get: RangerRowFilterPolicyItem = { + val i = new RangerRowFilterPolicyItem + i.setRowFilterInfo(rowFilterInfo) + i.setAccesses(accesses) + i.setUsers(users) + i.setGroups(groups) + i.setConditions(conditions) + i.setDelegateAdmin(delegateAdmin) + i + } +} + +case class KRangerPolicyItemRowFilterInfo( + filterExpr: String) extends RangerObjectGenerator[RangerPolicyItemRowFilterInfo] { + override def get: RangerPolicyItemRowFilterInfo = { + val i = new RangerPolicyItemRowFilterInfo + i.setFilterExpr(filterExpr) + i + } +} + +object RangerAccessType { + val select = "select" + val update = "update" + val create = "create" + val drop = "drop" + val alter = "alter" + val index = "index" + val lock = "lock" + val all = "all" + val read = "read" + val write = "write" + val use = "use" +} diff --git a/extensions/spark/kyuubi-spark-authz/src/test/resources/log4j2-test.xml b/extensions/spark/kyuubi-spark-authz/src/test/resources/log4j2-test.xml index 5e01ed4ab..709407281 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/resources/log4j2-test.xml +++ b/extensions/spark/kyuubi-spark-authz/src/test/resources/log4j2-test.xml @@ -21,21 +21,21 @@ - + - + - + diff --git a/extensions/spark/kyuubi-spark-authz/src/test/resources/policies_base.json b/extensions/spark/kyuubi-spark-authz/src/test/resources/policies_base.json new file mode 100644 index 000000000..aea5d2a9c --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/test/resources/policies_base.json @@ -0,0 +1,1678 @@ +{ + "serviceName": "hive_jenkins", + "serviceId": 1, + "policyVersion": 85, + "policyUpdateTime": "20190429-21:36:09.000-+0800", + "policies": [ + { + "service": "hive_jenkins", + "name": "all - url", + "policyType": 0, + "policyPriority": 0, + "description": "Policy for all - url", + "isAuditEnabled": true, + "resources": { + "url": { + "values": [ + "*" + ], + "isExcludes": false, + "isRecursive": true + } + }, + "policyItems": [ + { + "accesses": [ + { + "type": "select", + "isAllowed": true + }, + { + "type": "update", + "isAllowed": true + }, + { + "type": "create", + "isAllowed": true + }, + { + "type": "drop", + "isAllowed": true + }, + { + "type": "alter", + "isAllowed": true + }, + { + "type": "index", + "isAllowed": true + }, + { + "type": "lock", + "isAllowed": true + }, + { + "type": "all", + "isAllowed": true + }, + { + "type": "read", + "isAllowed": true + }, + { + "type": "write", + "isAllowed": true + } + ], + "users": [ + "admin" + ], + "groups": [], + "conditions": [], + "delegateAdmin": true + } + ], + "denyPolicyItems": [], + "allowExceptions": [], + "denyExceptions": [], + "dataMaskPolicyItems": [], + "rowFilterPolicyItems": [], + "options": {}, + "validitySchedules": [], + "policyLabels": [], + "id": 1, + "guid": "cf7e6725-492f-434f-bffe-6bb4e3147246", + "isEnabled": true, + "version": 1 + }, + { + "service": "hive_jenkins", + "name": "all - database, table, column", + "policyType": 0, + "policyPriority": 0, + "description": "Policy for all - database, table, column", + "isAuditEnabled": true, + "resources": { + "database": { + "values": [ + "*" + ], + "isExcludes": false, + "isRecursive": false + }, + "column": { + "values": [ + "*" + ], + "isExcludes": false, + "isRecursive": false + }, + "table": { + "values": [ + "*" + ], + "isExcludes": false, + "isRecursive": false + } + }, + "policyItems": [ + { + "accesses": [ + { + "type": "select", + "isAllowed": true + }, + { + "type": "update", + "isAllowed": true + }, + { + "type": "create", + "isAllowed": true + }, + { + "type": "drop", + "isAllowed": true + }, + { + "type": "alter", + "isAllowed": true + }, + { + "type": "index", + "isAllowed": true + }, + { + "type": "lock", + "isAllowed": true + }, + { + "type": "all", + "isAllowed": true + }, + { + "type": "read", + "isAllowed": true + }, + { + "type": "write", + "isAllowed": true + } + ], + "users": [ + "admin" + ], + "groups": [], + "conditions": [], + "delegateAdmin": true + } + ], + "denyPolicyItems": [], + "allowExceptions": [], + "denyExceptions": [], + "dataMaskPolicyItems": [], + "rowFilterPolicyItems": [], + "options": {}, + "validitySchedules": [], + "policyLabels": [], + "id": 2, + "guid": "3b96138a-af4d-48bc-9544-58c5bfa1979b", + "isEnabled": true, + "version": 1 + }, + { + "service": "hive_jenkins", + "name": "all - database, udf", + "policyType": 0, + "policyPriority": 0, + "description": "Policy for all - database, udf", + "isAuditEnabled": true, + "resources": { + "database": { + "values": [ + "*" + ], + "isExcludes": false, + "isRecursive": false + }, + "udf": { + "values": [ + "*" + ], + "isExcludes": false, + "isRecursive": false + } + }, + "policyItems": [ + { + "accesses": [ + { + "type": "select", + "isAllowed": true + }, + { + "type": "update", + "isAllowed": true + }, + { + "type": "create", + "isAllowed": true + }, + { + "type": "drop", + "isAllowed": true + }, + { + "type": "alter", + "isAllowed": true + }, + { + "type": "index", + "isAllowed": true + }, + { + "type": "lock", + "isAllowed": true + }, + { + "type": "all", + "isAllowed": true + }, + { + "type": "read", + "isAllowed": true + }, + { + "type": "write", + "isAllowed": true + } + ], + "users": [ + "admin" + ], + "groups": [], + "conditions": [], + "delegateAdmin": true + } + ], + "denyPolicyItems": [], + "allowExceptions": [], + "denyExceptions": [], + "dataMaskPolicyItems": [], + "rowFilterPolicyItems": [], + "options": {}, + "validitySchedules": [], + "policyLabels": [], + "id": 3, + "guid": "db08fbb0-61da-4f33-8144-ccd89816151d", + "isEnabled": true, + "version": 1 + }, + { + "service": "hive_jenkins", + "name": "default", + "policyType": 0, + "policyPriority": 0, + "description": "", + "isAuditEnabled": true, + "resources": { + "database": { + "values": [ + "default", + "spark_catalog", + "iceberg_ns", + "ns1" + ], + "isExcludes": false, + "isRecursive": false + }, + "column": { + "values": [ + "*" + ], + "isExcludes": false, + "isRecursive": false + }, + "table": { + "values": [ + "*" + ], + "isExcludes": false, + "isRecursive": false + } + }, + "policyItems": [ + { + "accesses": [ + { + "type": "select", + "isAllowed": true + }, + { + "type": "update", + "isAllowed": true + }, + { + "type": "create", + "isAllowed": true + }, + { + "type": "drop", + "isAllowed": true + }, + { + "type": "alter", + "isAllowed": true + }, + { + "type": "index", + "isAllowed": true + }, + { + "type": "lock", + "isAllowed": true + }, + { + "type": "all", + "isAllowed": true + }, + { + "type": "read", + "isAllowed": true + }, + { + "type": "write", + "isAllowed": true + } + ], + "users": [ + "bob", + "perm_view_user", + "{OWNER}" + ], + "groups": [], + "conditions": [], + "delegateAdmin": false + }, + { + "accesses": [ + { + "type": "select", + "isAllowed": false + }, + { + "type": "update", + "isAllowed": false + }, + { + "type": "create", + "isAllowed": true + }, + { + "type": "drop", + "isAllowed": false + }, + { + "type": "alter", + "isAllowed": false + }, + { + "type": "index", + "isAllowed": false + }, + { + "type": "lock", + "isAllowed": false + }, + { + "type": "all", + "isAllowed": false + }, + { + "type": "read", + "isAllowed": false + }, + { + "type": "write", + "isAllowed": false + } + ], + "users": [ + "default_table_owner", + "create_only_user" + ], + "groups": [], + "conditions": [], + "delegateAdmin": false + } + ], + "denyPolicyItems": [], + "allowExceptions": [], + "denyExceptions": [], + "dataMaskPolicyItems": [], + "rowFilterPolicyItems": [], + "options": {}, + "validitySchedules": [], + "policyLabels": [ + "" + ], + "id": 5, + "guid": "2db6099d-e4f1-41df-9d24-f2f47bed618e", + "isEnabled": true, + "version": 5 + }, + { + "service": "hive_jenkins", + "name": "default_kent", + "policyType": 0, + "policyPriority": 0, + "description": "", + "isAuditEnabled": true, + "resources": { + "database": { + "values": [ + "default", + "spark_catalog" + ], + "isExcludes": false, + "isRecursive": false + }, + "column": { + "values": [ + "key" + ], + "isExcludes": false, + "isRecursive": false + }, + "table": { + "values": [ + "src" + ], + "isExcludes": false, + "isRecursive": false + } + }, + "policyItems": [ + { + "accesses": [ + { + "type": "select", + "isAllowed": true + }, + { + "type": "update", + "isAllowed": true + }, + { + "type": "create", + "isAllowed": true + }, + { + "type": "drop", + "isAllowed": true + }, + { + "type": "alter", + "isAllowed": true + }, + { + "type": "index", + "isAllowed": true + }, + { + "type": "lock", + "isAllowed": true + }, + { + "type": "all", + "isAllowed": true + }, + { + "type": "read", + "isAllowed": true + }, + { + "type": "write", + "isAllowed": true + } + ], + "users": [ + "kent" + ], + "groups": [], + "conditions": [], + "delegateAdmin": false + } + ], + "denyPolicyItems": [], + "allowExceptions": [], + "denyExceptions": [], + "dataMaskPolicyItems": [], + "rowFilterPolicyItems": [], + "options": {}, + "validitySchedules": [], + "policyLabels": [ + "" + ], + "id": 5, + "guid": "fd24db19-f7cc-4e13-a8ba-bbd5a07a2d8d", + "isEnabled": true, + "version": 5 + }, + { + "service": "hive_jenkins", + "name": "src_key _less_than_20", + "policyType": 2, + "policyPriority": 0, + "description": "", + "isAuditEnabled": true, + "resources": { + "database": { + "values": [ + "default" + ], + "isExcludes": false, + "isRecursive": false + }, + "table": { + "values": [ + "src" + ], + "isExcludes": false, + "isRecursive": false + } + }, + "policyItems": [], + "denyPolicyItems": [], + "allowExceptions": [], + "denyExceptions": [], + "dataMaskPolicyItems": [], + "rowFilterPolicyItems": [ + { + "rowFilterInfo": { + "filterExpr": "key\u003c20" + }, + "accesses": [ + { + "type": "select", + "isAllowed": true + } + ], + "users": [ + "bob" + ], + "groups": [], + "conditions": [], + "delegateAdmin": false + } + ], + "serviceType": "hive", + "options": {}, + "validitySchedules": [], + "policyLabels": [ + "" + ], + "id": 4, + "guid": "f588a9ed-f7b1-48f7-9d0d-c12cf2b9b7ed", + "isEnabled": true, + "version": 26 + }, + { + "service": "hive_jenkins", + "name": "src_key_less_than_20_perm_view", + "policyType": 2, + "policyPriority": 0, + "description": "", + "isAuditEnabled": true, + "resources": { + "database": { + "values": [ + "default" + ], + "isExcludes": false, + "isRecursive": false + }, + "table": { + "values": [ + "perm_view" + ], + "isExcludes": false, + "isRecursive": false + } + }, + "policyItems": [], + "denyPolicyItems": [], + "allowExceptions": [], + "denyExceptions": [], + "dataMaskPolicyItems": [], + "rowFilterPolicyItems": [ + { + "rowFilterInfo": { + "filterExpr": "key\u003c20" + }, + "accesses": [ + { + "type": "select", + "isAllowed": true + } + ], + "users": [ + "perm_view_user" + ], + "groups": [], + "conditions": [], + "delegateAdmin": false + } + ], + "serviceType": "hive", + "options": {}, + "validitySchedules": [], + "policyLabels": [ + "" + ], + "id": 22, + "guid": "c240a7ea-9d26-4db2-b925-d5dbe49bd447 \n", + "isEnabled": true, + "version": 26 + }, + { + "service": "hive_jenkins", + "name": "default_bob_use", + "policyType": 0, + "policyPriority": 0, + "description": "", + "isAuditEnabled": true, + "resources": { + "database": { + "values": [ + "default_bob", + "spark_catalog" + ], + "isExcludes": false, + "isRecursive": false + }, + "column": { + "values": [ + "*" + ], + "isExcludes": false, + "isRecursive": false + }, + "table": { + "values": [ + "table_use*" + ], + "isExcludes": false, + "isRecursive": false + } + }, + "policyItems": [ + { + "accesses": [ + { + "type": "update", + "isAllowed": true + } + ], + "users": [ + "bob" + ], + "groups": [], + "conditions": [], + "delegateAdmin": false + } + ], + "denyPolicyItems": [], + "allowExceptions": [], + "denyExceptions": [], + "dataMaskPolicyItems": [], + "rowFilterPolicyItems": [], + "options": {}, + "validitySchedules": [], + "policyLabels": [ + "" + ], + "id": 5, + "guid": "2eb6099d-e4f1-41df-9d24-f2f47bed618e", + "isEnabled": true, + "version": 5 + }, + { + "service": "hive_jenkins", + "name": "default_bob_select", + "policyType": 0, + "policyPriority": 0, + "description": "", + "isAuditEnabled": true, + "resources": { + "database": { + "values": [ + "default_bob", + "spark_catalog" + ], + "isExcludes": false, + "isRecursive": false + }, + "column": { + "values": [ + "*" + ], + "isExcludes": false, + "isRecursive": false + }, + "table": { + "values": [ + "table_select*" + ], + "isExcludes": false, + "isRecursive": false + } + }, + "policyItems": [ + { + "accesses": [ + { + "type": "select", + "isAllowed": true + }, + { + "type": "use", + "isAllowed": true + } + ], + "users": [ + "bob" + ], + "groups": [], + "conditions": [], + "delegateAdmin": false + } + ], + "denyPolicyItems": [], + "allowExceptions": [], + "denyExceptions": [], + "dataMaskPolicyItems": [], + "rowFilterPolicyItems": [], + "options": {}, + "validitySchedules": [], + "policyLabels": [ + "" + ], + "id": 5, + "guid": "2fb6099d-e4f1-41df-9d24-f2f47bed618e", + "isEnabled": true, + "version": 5 + }, + { + "service": "hive_jenkins", + "name": "src_value_hash_perm_view", + "policyType": 1, + "policyPriority": 0, + "description": "", + "isAuditEnabled": true, + "resources": { + "database": { + "values": [ + "default", + "spark_catalog" + ], + "isExcludes": false, + "isRecursive": false + }, + "column": { + "values": [ + "value1" + ], + "isExcludes": false, + "isRecursive": false + }, + "table": { + "values": [ + "src" + ], + "isExcludes": false, + "isRecursive": false + } + }, + "policyItems": [], + "denyPolicyItems": [], + "allowExceptions": [], + "denyExceptions": [], + "dataMaskPolicyItems": [ + { + "dataMaskInfo": { + "dataMaskType": "MASK_HASH" + }, + "accesses": [ + { + "type": "select", + "isAllowed": true + } + ], + "users": [ + "bob" + ], + "groups": [], + "conditions": [], + "delegateAdmin": false + } + ], + "rowFilterPolicyItems": [], + "options": {}, + "validitySchedules": [], + "policyLabels": [ + "" + ], + "id": 5, + "guid": "ed1868a1-bf79-4721-a3d5-6815cc7d4986", + "isEnabled": true, + "version": 1 + }, + { + "service": "hive_jenkins", + "name": "src_value_hash", + "policyType": 1, + "policyPriority": 0, + "description": "", + "isAuditEnabled": true, + "resources": { + "database": { + "values": [ + "default", + "spark_catalog" + ], + "isExcludes": false, + "isRecursive": false + }, + "column": { + "values": [ + "value1" + ], + "isExcludes": false, + "isRecursive": false + }, + "table": { + "values": [ + "perm_view" + ], + "isExcludes": false, + "isRecursive": false + } + }, + "policyItems": [], + "denyPolicyItems": [], + "allowExceptions": [], + "denyExceptions": [], + "dataMaskPolicyItems": [ + { + "dataMaskInfo": { + "dataMaskType": "MASK_HASH" + }, + "accesses": [ + { + "type": "select", + "isAllowed": true + } + ], + "users": [ + "perm_view_user" + ], + "groups": [], + "conditions": [], + "delegateAdmin": false + } + ], + "rowFilterPolicyItems": [], + "options": {}, + "validitySchedules": [], + "policyLabels": [ + "" + ], + "id": 20, + "guid": "bfeddeab-50d0-4902-985f-42559efa39c3", + "isEnabled": true, + "version": 1 + }, + { + "service": "hive_jenkins", + "name": "src_value2_nullify", + "policyType": 1, + "policyPriority": 0, + "description": "", + "isAuditEnabled": true, + "resources": { + "database": { + "values": [ + "default", + "spark_catalog", + "iceberg_ns", + "ns1" + ], + "isExcludes": false, + "isRecursive": false + }, + "column": { + "values": [ + "value2" + ], + "isExcludes": false, + "isRecursive": false + }, + "table": { + "values": [ + "src" + ], + "isExcludes": false, + "isRecursive": false + } + }, + "policyItems": [], + "denyPolicyItems": [], + "allowExceptions": [], + "denyExceptions": [], + "dataMaskPolicyItems": [ + { + "dataMaskInfo": { + "dataMaskType": "MASK" + }, + "accesses": [ + { + "type": "select", + "isAllowed": true + } + ], + "users": [ + "bob" + ], + "groups": [], + "conditions": [], + "delegateAdmin": false + } + ], + "rowFilterPolicyItems": [], + "options": {}, + "validitySchedules": [], + "policyLabels": [ + "" + ], + "id": 6, + "guid": "98a04cd7-8d14-4466-adc9-126d87a3af69", + "isEnabled": true, + "version": 1 + }, + { + "service": "hive_jenkins", + "name": "src_value3_sf4", + "policyType": 1, + "policyPriority": 0, + "description": "", + "isAuditEnabled": true, + "resources": { + "database": { + "values": [ + "default", + "spark_catalog" + ], + "isExcludes": false, + "isRecursive": false + }, + "column": { + "values": [ + "value3" + ], + "isExcludes": false, + "isRecursive": false + }, + "table": { + "values": [ + "src" + ], + "isExcludes": false, + "isRecursive": false + } + }, + "policyItems": [], + "denyPolicyItems": [], + "allowExceptions": [], + "denyExceptions": [], + "dataMaskPolicyItems": [ + { + "dataMaskInfo": { + "dataMaskType": "MASK_SHOW_FIRST_4" + }, + "accesses": [ + { + "type": "select", + "isAllowed": true + } + ], + "users": [ + "bob" + ], + "groups": [], + "conditions": [], + "delegateAdmin": false + } + ], + "rowFilterPolicyItems": [], + "options": {}, + "validitySchedules": [], + "policyLabels": [ + "" + ], + "id": 7, + "guid": "9d50a525-b24c-4cf5-a885-d10d426368d1", + "isEnabled": true, + "version": 1 + }, + { + "service": "hive_jenkins", + "name": "src_value4_sf4", + "policyType": 1, + "policyPriority": 0, + "description": "", + "isAuditEnabled": true, + "resources": { + "database": { + "values": [ + "default", + "spark_catalog" + ], + "isExcludes": false, + "isRecursive": false + }, + "column": { + "values": [ + "value4" + ], + "isExcludes": false, + "isRecursive": false + }, + "table": { + "values": [ + "src" + ], + "isExcludes": false, + "isRecursive": false + } + }, + "policyItems": [], + "denyPolicyItems": [], + "allowExceptions": [], + "denyExceptions": [], + "dataMaskPolicyItems": [ + { + "dataMaskInfo": { + "dataMaskType": "MASK_DATE_SHOW_YEAR" + }, + "accesses": [ + { + "type": "select", + "isAllowed": true + } + ], + "users": [ + "bob" + ], + "groups": [], + "conditions": [], + "delegateAdmin": false + } + ], + "rowFilterPolicyItems": [], + "options": {}, + "validitySchedules": [], + "policyLabels": [ + "" + ], + "id": 8, + "guid": "9d50a526-b24c-4cf5-a885-d10d426368d1", + "isEnabled": true, + "version": 1 + }, + { + "service": "hive_jenkins", + "name": "src_value5_show_last_4", + "policyType": 1, + "policyPriority": 0, + "description": "", + "isAuditEnabled": true, + "resources": { + "database": { + "values": [ + "default", + "spark_catalog" + ], + "isExcludes": false, + "isRecursive": false + }, + "column": { + "values": [ + "value5" + ], + "isExcludes": false, + "isRecursive": false + }, + "table": { + "values": [ + "src" + ], + "isExcludes": false, + "isRecursive": false + } + }, + "policyItems": [], + "denyPolicyItems": [], + "allowExceptions": [], + "denyExceptions": [], + "dataMaskPolicyItems": [ + { + "dataMaskInfo": { + "dataMaskType": "MASK_SHOW_LAST_4" + }, + "accesses": [ + { + "type": "select", + "isAllowed": true + } + ], + "users": [ + "bob" + ], + "groups": [], + "conditions": [], + "delegateAdmin": false + } + ], + "rowFilterPolicyItems": [], + "options": {}, + "validitySchedules": [], + "policyLabels": [ + "" + ], + "id": 32, + "guid": "b3f1f1e0-2bd6-4b20-8a32-a531006ae151", + "isEnabled": true, + "version": 1 + }, + { + "service": "hive_jenkins", + "name": "someone_access_perm_view", + "policyType": 0, + "policyPriority": 0, + "description": "", + "isAuditEnabled": true, + "resources": { + "database": { + "values": [ + "default" + ], + "isExcludes": false, + "isRecursive": false + }, + "column": { + "values": [ + "*" + ], + "isExcludes": false, + "isRecursive": false + }, + "table": { + "values": [ + "perm_view" + ], + "isExcludes": false, + "isRecursive": false + } + }, + "policyItems": [ + { + "accesses": [ + { + "type": "select", + "isAllowed": true + } + ], + "users": [ + "user_perm_view_only" + ], + "groups": [], + "conditions": [], + "delegateAdmin": false + } + ], + "denyPolicyItems": [], + "allowExceptions": [], + "denyExceptions": [], + "dataMaskPolicyItems": [], + "rowFilterPolicyItems": [], + "options": {}, + "validitySchedules": [], + "policyLabels": [ + "" + ], + "id": 123, + "guid": "2fb6099d-e421-41df-9d24-f2f47bed618e", + "isEnabled": true, + "version": 5 + } + ], + "serviceDef": { + "name": "hive", + "implClass": "org.apache.ranger.services.hive.RangerServiceHive", + "label": "Hive Server2", + "description": "Hive Server2", + "options": { + "enableDenyAndExceptionsInPolicies": "true" + }, + "configs": [ + { + "itemId": 1, + "name": "username", + "type": "string", + "mandatory": true, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "Username" + }, + { + "itemId": 2, + "name": "password", + "type": "password", + "mandatory": true, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "Password" + }, + { + "itemId": 3, + "name": "jdbc.driverClassName", + "type": "string", + "mandatory": true, + "defaultValue": "org.apache.hive.jdbc.HiveDriver", + "validationRegEx": "", + "validationMessage": "", + "uiHint": "" + }, + { + "itemId": 4, + "name": "jdbc.url", + "type": "string", + "mandatory": true, + "defaultValue": "", + "validationRegEx": "", + "validationMessage": "", + "uiHint": "{\"TextFieldWithIcon\":true, \"info\": \"1.For Remote Mode, eg.\u003cbr\u003ejdbc:hive2://\u0026lt;host\u0026gt;:\u0026lt;port\u0026gt;\u003cbr\u003e2.For Embedded Mode (no host or port), eg.\u003cbr\u003ejdbc:hive2:///;initFile\u003d\u0026lt;file\u0026gt;\u003cbr\u003e3.For HTTP Mode, eg.\u003cbr\u003ejdbc:hive2://\u0026lt;host\u0026gt;:\u0026lt;port\u0026gt;/;\u003cbr\u003etransportMode\u003dhttp;httpPath\u003d\u0026lt;httpPath\u0026gt;\u003cbr\u003e4.For SSL Mode, eg.\u003cbr\u003ejdbc:hive2://\u0026lt;host\u0026gt;:\u0026lt;port\u0026gt;/;ssl\u003dtrue;\u003cbr\u003esslTrustStore\u003dtStore;trustStorePassword\u003dpw\u003cbr\u003e5.For ZooKeeper Mode, eg.\u003cbr\u003ejdbc:hive2://\u0026lt;host\u0026gt;/;serviceDiscoveryMode\u003d\u003cbr\u003ezooKeeper;zooKeeperNamespace\u003dhiveserver2\u003cbr\u003e6.For Kerberos Mode, eg.\u003cbr\u003ejdbc:hive2://\u0026lt;host\u0026gt;:\u0026lt;port\u0026gt;/;\u003cbr\u003eprincipal\u003dhive/domain@EXAMPLE.COM\u003cbr\u003e\"}" + }, + { + "itemId": 5, + "name": "commonNameForCertificate", + "type": "string", + "mandatory": false, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "Common Name for Certificate" + } + ], + "resources": [ + { + "itemId": 1, + "name": "database", + "type": "string", + "level": 10, + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "true" + }, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "Hive Database", + "description": "Hive Database", + "accessTypeRestrictions": [], + "isValidLeaf": false + }, + { + "itemId": 5, + "name": "url", + "type": "string", + "level": 10, + "mandatory": true, + "lookupSupported": false, + "recursiveSupported": true, + "excludesSupported": false, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "false" + }, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "URL", + "description": "URL", + "accessTypeRestrictions": [], + "isValidLeaf": true + }, + { + "itemId": 2, + "name": "table", + "type": "string", + "level": 20, + "parent": "database", + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "true" + }, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "Hive Table", + "description": "Hive Table", + "accessTypeRestrictions": [], + "isValidLeaf": false + }, + { + "itemId": 3, + "name": "udf", + "type": "string", + "level": 20, + "parent": "database", + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "true" + }, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "Hive UDF", + "description": "Hive UDF", + "accessTypeRestrictions": [], + "isValidLeaf": true + }, + { + "itemId": 4, + "name": "column", + "type": "string", + "level": 30, + "parent": "table", + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "true" + }, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "", + "label": "Hive Column", + "description": "Hive Column", + "accessTypeRestrictions": [], + "isValidLeaf": true + } + ], + "accessTypes": [ + { + "itemId": 1, + "name": "select", + "label": "select", + "impliedGrants": [] + }, + { + "itemId": 2, + "name": "update", + "label": "update", + "impliedGrants": [] + }, + { + "itemId": 3, + "name": "create", + "label": "Create", + "impliedGrants": [] + }, + { + "itemId": 4, + "name": "drop", + "label": "Drop", + "impliedGrants": [] + }, + { + "itemId": 5, + "name": "alter", + "label": "Alter", + "impliedGrants": [] + }, + { + "itemId": 6, + "name": "index", + "label": "Index", + "impliedGrants": [] + }, + { + "itemId": 7, + "name": "lock", + "label": "Lock", + "impliedGrants": [] + }, + { + "itemId": 8, + "name": "all", + "label": "All", + "impliedGrants": [ + "select", + "update", + "create", + "drop", + "alter", + "index", + "lock", + "read", + "write" + ] + }, + { + "itemId": 9, + "name": "read", + "label": "Read", + "impliedGrants": [] + }, + { + "itemId": 10, + "name": "write", + "label": "Write", + "impliedGrants": [] + } + ], + "policyConditions": [], + "contextEnrichers": [], + "enums": [], + "dataMaskDef": { + "maskTypes": [ + { + "itemId": 1, + "name": "MASK", + "label": "Redact", + "description": "Replace lowercase with \u0027x\u0027, uppercase with \u0027X\u0027, digits with \u00270\u0027", + "transformer": "mask({col})", + "dataMaskOptions": {} + }, + { + "itemId": 2, + "name": "MASK_SHOW_LAST_4", + "label": "Partial mask: show last 4", + "description": "Show last 4 characters; replace rest with \u0027x\u0027", + "transformer": "mask_show_last_n({col}, 4, \u0027x\u0027, \u0027x\u0027, \u0027x\u0027, -1, \u00271\u0027)", + "dataMaskOptions": {} + }, + { + "itemId": 3, + "name": "MASK_SHOW_FIRST_4", + "label": "Partial mask: show first 4", + "description": "Show first 4 characters; replace rest with \u0027x\u0027", + "transformer": "mask_show_first_n({col}, 4, \u0027x\u0027, \u0027x\u0027, \u0027x\u0027, -1, \u00271\u0027)", + "dataMaskOptions": {} + }, + { + "itemId": 4, + "name": "MASK_HASH", + "label": "Hash", + "description": "Hash the value", + "transformer": "mask_hash({col})", + "dataMaskOptions": {} + }, + { + "itemId": 5, + "name": "MASK_NULL", + "label": "Nullify", + "description": "Replace with NULL", + "dataMaskOptions": {} + }, + { + "itemId": 6, + "name": "MASK_NONE", + "label": "Unmasked (retain original value)", + "description": "No masking", + "dataMaskOptions": {} + }, + { + "itemId": 12, + "name": "MASK_DATE_SHOW_YEAR", + "label": "Date: show only year", + "description": "Date: show only year", + "transformer": "mask({col}, \u0027x\u0027, \u0027x\u0027, \u0027x\u0027, -1, \u00271\u0027, 1, 0, -1)", + "dataMaskOptions": {} + }, + { + "itemId": 13, + "name": "CUSTOM", + "label": "Custom", + "description": "Custom", + "dataMaskOptions": {} + } + ], + "accessTypes": [ + { + "itemId": 1, + "name": "select", + "label": "select", + "impliedGrants": [] + } + ], + "resources": [ + { + "itemId": 1, + "name": "database", + "type": "string", + "level": 10, + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": false, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "false", + "ignoreCase": "true" + }, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "{ \"singleValue\":true }", + "label": "Hive Database", + "description": "Hive Database", + "accessTypeRestrictions": [], + "isValidLeaf": false + }, + { + "itemId": 2, + "name": "table", + "type": "string", + "level": 20, + "parent": "database", + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": false, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "false", + "ignoreCase": "true" + }, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "{ \"singleValue\":true }", + "label": "Hive Table", + "description": "Hive Table", + "accessTypeRestrictions": [], + "isValidLeaf": false + }, + { + "itemId": 4, + "name": "column", + "type": "string", + "level": 30, + "parent": "table", + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": false, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "false", + "ignoreCase": "true" + }, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "{ \"singleValue\":true }", + "label": "Hive Column", + "description": "Hive Column", + "accessTypeRestrictions": [], + "isValidLeaf": true + } + ] + }, + "rowFilterDef": { + "accessTypes": [ + { + "itemId": 1, + "name": "select", + "label": "select", + "impliedGrants": [] + } + ], + "resources": [ + { + "itemId": 1, + "name": "database", + "type": "string", + "level": 10, + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": false, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "false", + "ignoreCase": "true" + }, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "{ \"singleValue\":true }", + "label": "Hive Database", + "description": "Hive Database", + "accessTypeRestrictions": [], + "isValidLeaf": false + }, + { + "itemId": 2, + "name": "table", + "type": "string", + "level": 20, + "parent": "database", + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": false, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "false", + "ignoreCase": "true" + }, + "validationRegEx": "", + "validationMessage": "", + "uiHint": "{ \"singleValue\":true }", + "label": "Hive Table", + "description": "Hive Table", + "accessTypeRestrictions": [], + "isValidLeaf": true + } + ] + }, + "id": 3, + "guid": "3e1afb5a-184a-4e82-9d9c-87a5cacc243c", + "isEnabled": true, + "createTime": "20190401-20:14:36.000-+0800", + "updateTime": "20190401-20:14:36.000-+0800", + "version": 1 + }, + "auditMode": "audit-default" +} diff --git a/extensions/spark/kyuubi-spark-authz/src/test/resources/sparkSql_hive_jenkins.json b/extensions/spark/kyuubi-spark-authz/src/test/resources/sparkSql_hive_jenkins.json index b5b069c46..76d8c788a 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/resources/sparkSql_hive_jenkins.json +++ b/extensions/spark/kyuubi-spark-authz/src/test/resources/sparkSql_hive_jenkins.json @@ -1,1611 +1,1402 @@ { - "serviceName": "hive_jenkins", - "serviceId": 1, - "policyVersion": 85, - "policyUpdateTime": "20190429-21:36:09.000-+0800", - "policies": [ - { - "service": "hive_jenkins", - "name": "all - url", - "policyType": 0, - "policyPriority": 0, - "description": "Policy for all - url", - "isAuditEnabled": true, - "resources": { - "url": { - "values": [ - "*" - ], - "isExcludes": false, - "isRecursive": true - } - }, - "policyItems": [ - { - "accesses": [ - { - "type": "select", - "isAllowed": true - }, - { - "type": "update", - "isAllowed": true - }, - { - "type": "create", - "isAllowed": true - }, - { - "type": "drop", - "isAllowed": true - }, - { - "type": "alter", - "isAllowed": true - }, - { - "type": "index", - "isAllowed": true - }, - { - "type": "lock", - "isAllowed": true - }, - { - "type": "all", - "isAllowed": true - }, - { - "type": "read", - "isAllowed": true - }, - { - "type": "write", - "isAllowed": true - } - ], - "users": [ - "admin" - ], - "groups": [], - "conditions": [], - "delegateAdmin": true - } - ], - "denyPolicyItems": [], - "allowExceptions": [], - "denyExceptions": [], - "dataMaskPolicyItems": [], - "rowFilterPolicyItems": [], - "options": {}, - "validitySchedules": [], - "policyLabels": [], - "id": 1, - "guid": "cf7e6725-492f-434f-bffe-6bb4e3147246", - "isEnabled": true, - "version": 1 + "serviceName" : "hive_jenkins", + "serviceId" : 1, + "policyVersion" : 85, + "policyUpdateTime" : "20190429-21:36:09.000-+0800", + "policies" : [ { + "id" : 0, + "guid" : "cfcd2084-95d5-35ef-a6e7-dff9f98764da", + "isEnabled" : true, + "version" : 1, + "service" : "hive_jenkins", + "name" : "all - url", + "policyType" : 0, + "policyPriority" : 0, + "description" : "Policy for all - url", + "isAuditEnabled" : true, + "resources" : { + "url" : { + "values" : [ "*" ], + "isExcludes" : false, + "isRecursive" : true + } }, - { - "service": "hive_jenkins", - "name": "all - database, table, column", - "policyType": 0, - "policyPriority": 0, - "description": "Policy for all - database, table, column", - "isAuditEnabled": true, - "resources": { - "database": { - "values": [ - "*" - ], - "isExcludes": false, - "isRecursive": false - }, - "column": { - "values": [ - "*" - ], - "isExcludes": false, - "isRecursive": false - }, - "table": { - "values": [ - "*" - ], - "isExcludes": false, - "isRecursive": false - } + "conditions" : [ ], + "policyItems" : [ { + "accesses" : [ { + "type" : "select", + "isAllowed" : true + }, { + "type" : "update", + "isAllowed" : true + }, { + "type" : "create", + "isAllowed" : true + }, { + "type" : "drop", + "isAllowed" : true + }, { + "type" : "alter", + "isAllowed" : true + }, { + "type" : "index", + "isAllowed" : true + }, { + "type" : "lock", + "isAllowed" : true + }, { + "type" : "all", + "isAllowed" : true + }, { + "type" : "read", + "isAllowed" : true + }, { + "type" : "write", + "isAllowed" : true + } ], + "users" : [ "admin" ], + "groups" : [ ], + "roles" : [ ], + "conditions" : [ ], + "delegateAdmin" : true + } ], + "denyPolicyItems" : [ ], + "allowExceptions" : [ ], + "denyExceptions" : [ ], + "dataMaskPolicyItems" : [ ], + "rowFilterPolicyItems" : [ ], + "options" : { }, + "validitySchedules" : [ ], + "policyLabels" : [ ], + "isDenyAllElse" : false + }, { + "id" : 1, + "guid" : "c4ca4238-a0b9-3382-8dcc-509a6f75849b", + "isEnabled" : true, + "version" : 1, + "service" : "hive_jenkins", + "name" : "all - database, table, column", + "policyType" : 0, + "policyPriority" : 0, + "description" : "Policy for all - database, table, column", + "isAuditEnabled" : true, + "resources" : { + "database" : { + "values" : [ "*" ], + "isExcludes" : false, + "isRecursive" : false }, - "policyItems": [ - { - "accesses": [ - { - "type": "select", - "isAllowed": true - }, - { - "type": "update", - "isAllowed": true - }, - { - "type": "create", - "isAllowed": true - }, - { - "type": "drop", - "isAllowed": true - }, - { - "type": "alter", - "isAllowed": true - }, - { - "type": "index", - "isAllowed": true - }, - { - "type": "lock", - "isAllowed": true - }, - { - "type": "all", - "isAllowed": true - }, - { - "type": "read", - "isAllowed": true - }, - { - "type": "write", - "isAllowed": true - } - ], - "users": [ - "admin" - ], - "groups": [], - "conditions": [], - "delegateAdmin": true - } - ], - "denyPolicyItems": [], - "allowExceptions": [], - "denyExceptions": [], - "dataMaskPolicyItems": [], - "rowFilterPolicyItems": [], - "options": {}, - "validitySchedules": [], - "policyLabels": [], - "id": 2, - "guid": "3b96138a-af4d-48bc-9544-58c5bfa1979b", - "isEnabled": true, - "version": 1 - }, - { - "service": "hive_jenkins", - "name": "all - database, udf", - "policyType": 0, - "policyPriority": 0, - "description": "Policy for all - database, udf", - "isAuditEnabled": true, - "resources": { - "database": { - "values": [ - "*" - ], - "isExcludes": false, - "isRecursive": false - }, - "udf": { - "values": [ - "*" - ], - "isExcludes": false, - "isRecursive": false - } + "column" : { + "values" : [ "*" ], + "isExcludes" : false, + "isRecursive" : false }, - "policyItems": [ - { - "accesses": [ - { - "type": "select", - "isAllowed": true - }, - { - "type": "update", - "isAllowed": true - }, - { - "type": "create", - "isAllowed": true - }, - { - "type": "drop", - "isAllowed": true - }, - { - "type": "alter", - "isAllowed": true - }, - { - "type": "index", - "isAllowed": true - }, - { - "type": "lock", - "isAllowed": true - }, - { - "type": "all", - "isAllowed": true - }, - { - "type": "read", - "isAllowed": true - }, - { - "type": "write", - "isAllowed": true - } - ], - "users": [ - "admin" - ], - "groups": [], - "conditions": [], - "delegateAdmin": true - } - ], - "denyPolicyItems": [], - "allowExceptions": [], - "denyExceptions": [], - "dataMaskPolicyItems": [], - "rowFilterPolicyItems": [], - "options": {}, - "validitySchedules": [], - "policyLabels": [], - "id": 3, - "guid": "db08fbb0-61da-4f33-8144-ccd89816151d", - "isEnabled": true, - "version": 1 + "table" : { + "values" : [ "*" ], + "isExcludes" : false, + "isRecursive" : false + } }, - { - "service": "hive_jenkins", - "name": "default", - "policyType": 0, - "policyPriority": 0, - "description": "", - "isAuditEnabled": true, - "resources": { - "database": { - "values": [ - "default", - "spark_catalog", - "iceberg_ns" - ], - "isExcludes": false, - "isRecursive": false - }, - "column": { - "values": [ - "*" - ], - "isExcludes": false, - "isRecursive": false - }, - "table": { - "values": [ - "*" - ], - "isExcludes": false, - "isRecursive": false - } + "conditions" : [ ], + "policyItems" : [ { + "accesses" : [ { + "type" : "select", + "isAllowed" : true + }, { + "type" : "update", + "isAllowed" : true + }, { + "type" : "create", + "isAllowed" : true + }, { + "type" : "drop", + "isAllowed" : true + }, { + "type" : "alter", + "isAllowed" : true + }, { + "type" : "index", + "isAllowed" : true + }, { + "type" : "lock", + "isAllowed" : true + }, { + "type" : "all", + "isAllowed" : true + }, { + "type" : "read", + "isAllowed" : true + }, { + "type" : "write", + "isAllowed" : true + } ], + "users" : [ "admin" ], + "groups" : [ ], + "roles" : [ ], + "conditions" : [ ], + "delegateAdmin" : true + } ], + "denyPolicyItems" : [ ], + "allowExceptions" : [ ], + "denyExceptions" : [ ], + "dataMaskPolicyItems" : [ ], + "rowFilterPolicyItems" : [ ], + "options" : { }, + "validitySchedules" : [ ], + "policyLabels" : [ ], + "isDenyAllElse" : false + }, { + "id" : 2, + "guid" : "c81e728d-9d4c-3f63-af06-7f89cc14862c", + "isEnabled" : true, + "version" : 1, + "service" : "hive_jenkins", + "name" : "all - database, udf", + "policyType" : 0, + "policyPriority" : 0, + "description" : "Policy for all - database, udf", + "isAuditEnabled" : true, + "resources" : { + "database" : { + "values" : [ "*" ], + "isExcludes" : false, + "isRecursive" : false }, - "policyItems": [ - { - "accesses": [ - { - "type": "select", - "isAllowed": true - }, - { - "type": "update", - "isAllowed": true - }, - { - "type": "create", - "isAllowed": true - }, - { - "type": "drop", - "isAllowed": true - }, - { - "type": "alter", - "isAllowed": true - }, - { - "type": "index", - "isAllowed": true - }, - { - "type": "lock", - "isAllowed": true - }, - { - "type": "all", - "isAllowed": true - }, - { - "type": "read", - "isAllowed": true - }, - { - "type": "write", - "isAllowed": true - } - ], - "users": [ - "bob", - "perm_view_user", - "{OWNER}" - ], - "groups": [], - "conditions": [], - "delegateAdmin": false - }, { - "accesses": [ - { - "type": "select", - "isAllowed": false - }, - { - "type": "update", - "isAllowed": false - }, - { - "type": "create", - "isAllowed": true - }, - { - "type": "drop", - "isAllowed": false - }, - { - "type": "alter", - "isAllowed": false - }, - { - "type": "index", - "isAllowed": false - }, - { - "type": "lock", - "isAllowed": false - }, - { - "type": "all", - "isAllowed": false - }, - { - "type": "read", - "isAllowed": false - }, - { - "type": "write", - "isAllowed": false - } - ], - "users": [ - "default_table_owner", - "create_only_user" - ], - "groups": [], - "conditions": [], - "delegateAdmin": false - } - ], - "denyPolicyItems": [], - "allowExceptions": [], - "denyExceptions": [], - "dataMaskPolicyItems": [], - "rowFilterPolicyItems": [], - "options": {}, - "validitySchedules": [], - "policyLabels": [ - "" - ], - "id": 5, - "guid": "2db6099d-e4f1-41df-9d24-f2f47bed618e", - "isEnabled": true, - "version": 5 + "udf" : { + "values" : [ "*" ], + "isExcludes" : false, + "isRecursive" : false + } }, - { - "service": "hive_jenkins", - "name": "default_kent", - "policyType": 0, - "policyPriority": 0, - "description": "", - "isAuditEnabled": true, - "resources": { - "database": { - "values": [ - "default", - "spark_catalog" - ], - "isExcludes": false, - "isRecursive": false - }, - "column": { - "values": [ - "key" - ], - "isExcludes": false, - "isRecursive": false - }, - "table": { - "values": [ - "src" - ], - "isExcludes": false, - "isRecursive": false - } + "conditions" : [ ], + "policyItems" : [ { + "accesses" : [ { + "type" : "select", + "isAllowed" : true + }, { + "type" : "update", + "isAllowed" : true + }, { + "type" : "create", + "isAllowed" : true + }, { + "type" : "drop", + "isAllowed" : true + }, { + "type" : "alter", + "isAllowed" : true + }, { + "type" : "index", + "isAllowed" : true + }, { + "type" : "lock", + "isAllowed" : true + }, { + "type" : "all", + "isAllowed" : true + }, { + "type" : "read", + "isAllowed" : true + }, { + "type" : "write", + "isAllowed" : true + } ], + "users" : [ "admin" ], + "groups" : [ ], + "roles" : [ ], + "conditions" : [ ], + "delegateAdmin" : true + } ], + "denyPolicyItems" : [ ], + "allowExceptions" : [ ], + "denyExceptions" : [ ], + "dataMaskPolicyItems" : [ ], + "rowFilterPolicyItems" : [ ], + "options" : { }, + "validitySchedules" : [ ], + "policyLabels" : [ ], + "isDenyAllElse" : false + }, { + "id" : 3, + "guid" : "eccbc87e-4b5c-32fe-a830-8fd9f2a7baf3", + "isEnabled" : true, + "version" : 1, + "service" : "hive_jenkins", + "name" : "all - database, udf", + "policyType" : 0, + "policyPriority" : 0, + "description" : "Policy for all - database, udf", + "isAuditEnabled" : true, + "resources" : { + "database" : { + "values" : [ "default", "spark_catalog", "iceberg_ns", "ns1" ], + "isExcludes" : false, + "isRecursive" : false + }, + "column" : { + "values" : [ "*" ], + "isExcludes" : false, + "isRecursive" : false }, - "policyItems": [ - { - "accesses": [ - { - "type": "select", - "isAllowed": true - }, - { - "type": "update", - "isAllowed": true - }, - { - "type": "create", - "isAllowed": true - }, - { - "type": "drop", - "isAllowed": true - }, - { - "type": "alter", - "isAllowed": true - }, - { - "type": "index", - "isAllowed": true - }, - { - "type": "lock", - "isAllowed": true - }, - { - "type": "all", - "isAllowed": true - }, - { - "type": "read", - "isAllowed": true - }, - { - "type": "write", - "isAllowed": true - } - ], - "users": [ - "kent" - ], - "groups": [], - "conditions": [], - "delegateAdmin": false - } - ], - "denyPolicyItems": [], - "allowExceptions": [], - "denyExceptions": [], - "dataMaskPolicyItems": [], - "rowFilterPolicyItems": [], - "options": {}, - "validitySchedules": [], - "policyLabels": [ - "" - ], - "id": 5, - "guid": "fd24db19-f7cc-4e13-a8ba-bbd5a07a2d8d", - "isEnabled": true, - "version": 5 + "table" : { + "values" : [ "*" ], + "isExcludes" : false, + "isRecursive" : false + } }, - { - "service": "hive_jenkins", - "name": "src_key _less_than_20", - "policyType": 2, - "policyPriority": 0, - "description": "", - "isAuditEnabled": true, - "resources": { - "database": { - "values": [ - "default" - ], - "isExcludes": false, - "isRecursive": false - }, - "table": { - "values": [ - "src" - ], - "isExcludes": false, - "isRecursive": false - } + "conditions" : [ ], + "policyItems" : [ { + "accesses" : [ { + "type" : "select", + "isAllowed" : true + }, { + "type" : "update", + "isAllowed" : true + }, { + "type" : "create", + "isAllowed" : true + }, { + "type" : "drop", + "isAllowed" : true + }, { + "type" : "alter", + "isAllowed" : true + }, { + "type" : "index", + "isAllowed" : true + }, { + "type" : "lock", + "isAllowed" : true + }, { + "type" : "all", + "isAllowed" : true + }, { + "type" : "read", + "isAllowed" : true + }, { + "type" : "write", + "isAllowed" : true + } ], + "users" : [ "bob", "perm_view_user", "{OWNER}" ], + "groups" : [ ], + "roles" : [ ], + "conditions" : [ ], + "delegateAdmin" : true + }, { + "accesses" : [ { + "type" : "create", + "isAllowed" : true + } ], + "users" : [ "default_table_owner", "create_only_user" ], + "groups" : [ ], + "roles" : [ ], + "conditions" : [ ], + "delegateAdmin" : true + } ], + "denyPolicyItems" : [ ], + "allowExceptions" : [ ], + "denyExceptions" : [ ], + "dataMaskPolicyItems" : [ ], + "rowFilterPolicyItems" : [ ], + "options" : { }, + "validitySchedules" : [ ], + "policyLabels" : [ ], + "isDenyAllElse" : false + }, { + "id" : 4, + "guid" : "a87ff679-a2f3-371d-9181-a67b7542122c", + "isEnabled" : true, + "version" : 1, + "service" : "hive_jenkins", + "name" : "default_kent", + "policyType" : 0, + "policyPriority" : 0, + "description" : "", + "isAuditEnabled" : true, + "resources" : { + "database" : { + "values" : [ "default", "spark_catalog" ], + "isExcludes" : false, + "isRecursive" : false }, - "policyItems": [], - "denyPolicyItems": [], - "allowExceptions": [], - "denyExceptions": [], - "dataMaskPolicyItems": [], - "rowFilterPolicyItems": [ - { - "rowFilterInfo": { - "filterExpr": "key\u003c20" - }, - "accesses": [ - { - "type": "select", - "isAllowed": true - } - ], - "users": [ - "bob" - ], - "groups": [], - "conditions": [], - "delegateAdmin": false - } - ], - "serviceType": "hive", - "options": {}, - "validitySchedules": [], - "policyLabels": [ - "" - ], - "id": 4, - "guid": "f588a9ed-f7b1-48f7-9d0d-c12cf2b9b7ed", - "isEnabled": true, - "version": 26 - },{ - "service": "hive_jenkins", - "name": "src_key_less_than_20_perm_view", - "policyType": 2, - "policyPriority": 0, - "description": "", - "isAuditEnabled": true, - "resources": { - "database": { - "values": [ - "default" - ], - "isExcludes": false, - "isRecursive": false - }, - "table": { - "values": [ - "perm_view" - ], - "isExcludes": false, - "isRecursive": false - } + "column" : { + "values" : [ "key" ], + "isExcludes" : false, + "isRecursive" : false }, - "policyItems": [], - "denyPolicyItems": [], - "allowExceptions": [], - "denyExceptions": [], - "dataMaskPolicyItems": [], - "rowFilterPolicyItems": [ - { - "rowFilterInfo": { - "filterExpr": "key\u003c20" - }, - "accesses": [ - { - "type": "select", - "isAllowed": true - } - ], - "users": [ - "perm_view_user" - ], - "groups": [], - "conditions": [], - "delegateAdmin": false - } - ], - "serviceType": "hive", - "options": {}, - "validitySchedules": [], - "policyLabels": [ - "" - ], - "id": 22, - "guid": "c240a7ea-9d26-4db2-b925-d5dbe49bd447 \n", - "isEnabled": true, - "version": 26 + "table" : { + "values" : [ "src" ], + "isExcludes" : false, + "isRecursive" : false + } }, - { - "service": "hive_jenkins", - "name": "default_bob_use", - "policyType": 0, - "policyPriority": 0, - "description": "", - "isAuditEnabled": true, - "resources": { - "database": { - "values": [ - "default_bob", - "spark_catalog" - ], - "isExcludes": false, - "isRecursive": false - }, - "column": { - "values": [ - "*" - ], - "isExcludes": false, - "isRecursive": false - }, - "table": { - "values": [ - "table_use*" - ], - "isExcludes": false, - "isRecursive": false - } + "conditions" : [ ], + "policyItems" : [ { + "accesses" : [ { + "type" : "select", + "isAllowed" : true + }, { + "type" : "update", + "isAllowed" : true + }, { + "type" : "create", + "isAllowed" : true + }, { + "type" : "drop", + "isAllowed" : true + }, { + "type" : "alter", + "isAllowed" : true + }, { + "type" : "index", + "isAllowed" : true + }, { + "type" : "lock", + "isAllowed" : true + }, { + "type" : "all", + "isAllowed" : true + }, { + "type" : "read", + "isAllowed" : true + }, { + "type" : "write", + "isAllowed" : true + } ], + "users" : [ "kent" ], + "groups" : [ ], + "roles" : [ ], + "conditions" : [ ], + "delegateAdmin" : true + }, { + "accesses" : [ { + "type" : "create", + "isAllowed" : true + } ], + "users" : [ "default_table_owner", "create_only_user" ], + "groups" : [ ], + "roles" : [ ], + "conditions" : [ ], + "delegateAdmin" : true + } ], + "denyPolicyItems" : [ ], + "allowExceptions" : [ ], + "denyExceptions" : [ ], + "dataMaskPolicyItems" : [ ], + "rowFilterPolicyItems" : [ ], + "options" : { }, + "validitySchedules" : [ ], + "policyLabels" : [ ], + "isDenyAllElse" : false + }, { + "id" : 5, + "guid" : "e4da3b7f-bbce-3345-9777-2b0674a318d5", + "isEnabled" : true, + "version" : 1, + "service" : "hive_jenkins", + "name" : "default_bob_use", + "policyType" : 0, + "policyPriority" : 0, + "description" : "", + "isAuditEnabled" : true, + "resources" : { + "database" : { + "values" : [ "default_bob", "spark_catalog" ], + "isExcludes" : false, + "isRecursive" : false }, - "policyItems": [ - { - "accesses": [ - { - "type": "update", - "isAllowed": true - } - ], - "users": [ - "bob" - ], - "groups": [], - "conditions": [], - "delegateAdmin": false - } - ], - "denyPolicyItems": [], - "allowExceptions": [], - "denyExceptions": [], - "dataMaskPolicyItems": [], - "rowFilterPolicyItems": [], - "options": {}, - "validitySchedules": [], - "policyLabels": [ - "" - ], - "id": 5, - "guid": "2eb6099d-e4f1-41df-9d24-f2f47bed618e", - "isEnabled": true, - "version": 5 - }, - { - "service": "hive_jenkins", - "name": "default_bob_select", - "policyType": 0, - "policyPriority": 0, - "description": "", - "isAuditEnabled": true, - "resources": { - "database": { - "values": [ - "default_bob", - "spark_catalog" - ], - "isExcludes": false, - "isRecursive": false - }, - "column": { - "values": [ - "*" - ], - "isExcludes": false, - "isRecursive": false - }, - "table": { - "values": [ - "table_select*" - ], - "isExcludes": false, - "isRecursive": false - } + "column" : { + "values" : [ "*" ], + "isExcludes" : false, + "isRecursive" : false }, - "policyItems": [ - { - "accesses": [ - { - "type": "select", - "isAllowed": true - }, - { - "type": "use", - "isAllowed": true - } - ], - "users": [ - "bob" - ], - "groups": [], - "conditions": [], - "delegateAdmin": false - } - ], - "denyPolicyItems": [], - "allowExceptions": [], - "denyExceptions": [], - "dataMaskPolicyItems": [], - "rowFilterPolicyItems": [], - "options": {}, - "validitySchedules": [], - "policyLabels": [ - "" - ], - "id": 5, - "guid": "2fb6099d-e4f1-41df-9d24-f2f47bed618e", - "isEnabled": true, - "version": 5 + "table" : { + "values" : [ "table_use*" ], + "isExcludes" : false, + "isRecursive" : false + } }, - { - "service": "hive_jenkins", - "name": "src_value_hash_perm_view", - "policyType": 1, - "policyPriority": 0, - "description": "", - "isAuditEnabled": true, - "resources": { - "database": { - "values": [ - "default", - "spark_catalog" - ], - "isExcludes": false, - "isRecursive": false - }, - "column": { - "values": [ - "value1" - ], - "isExcludes": false, - "isRecursive": false - }, - "table": { - "values": [ - "src" - ], - "isExcludes": false, - "isRecursive": false - } + "conditions" : [ ], + "policyItems" : [ { + "accesses" : [ { + "type" : "update", + "isAllowed" : true + } ], + "users" : [ "bob" ], + "groups" : [ ], + "roles" : [ ], + "conditions" : [ ], + "delegateAdmin" : true + } ], + "denyPolicyItems" : [ ], + "allowExceptions" : [ ], + "denyExceptions" : [ ], + "dataMaskPolicyItems" : [ ], + "rowFilterPolicyItems" : [ ], + "options" : { }, + "validitySchedules" : [ ], + "policyLabels" : [ ], + "isDenyAllElse" : false + }, { + "id" : 6, + "guid" : "1679091c-5a88-3faf-afb5-e6087eb1b2dc", + "isEnabled" : true, + "version" : 1, + "service" : "hive_jenkins", + "name" : "default_bob_select", + "policyType" : 0, + "policyPriority" : 0, + "description" : "", + "isAuditEnabled" : true, + "resources" : { + "database" : { + "values" : [ "default_bob", "spark_catalog" ], + "isExcludes" : false, + "isRecursive" : false }, - "policyItems": [], - "denyPolicyItems": [], - "allowExceptions": [], - "denyExceptions": [], - "dataMaskPolicyItems": [ - { - "dataMaskInfo": { - "dataMaskType": "MASK_HASH" - }, - "accesses": [ - { - "type": "select", - "isAllowed": true - } - ], - "users": [ - "bob" - ], - "groups": [], - "conditions": [], - "delegateAdmin": false - } - ], - "rowFilterPolicyItems": [], - "options": {}, - "validitySchedules": [], - "policyLabels": [ - "" - ], - "id": 5, - "guid": "ed1868a1-bf79-4721-a3d5-6815cc7d4986", - "isEnabled": true, - "version": 1 - },{ - "service": "hive_jenkins", - "name": "src_value_hash", - "policyType": 1, - "policyPriority": 0, - "description": "", - "isAuditEnabled": true, - "resources": { - "database": { - "values": [ - "default", - "spark_catalog" - ], - "isExcludes": false, - "isRecursive": false - }, - "column": { - "values": [ - "value1" - ], - "isExcludes": false, - "isRecursive": false - }, - "table": { - "values": [ - "perm_view" - ], - "isExcludes": false, - "isRecursive": false - } + "column" : { + "values" : [ "*" ], + "isExcludes" : false, + "isRecursive" : false }, - "policyItems": [], - "denyPolicyItems": [], - "allowExceptions": [], - "denyExceptions": [], - "dataMaskPolicyItems": [ - { - "dataMaskInfo": { - "dataMaskType": "MASK_HASH" - }, - "accesses": [ - { - "type": "select", - "isAllowed": true - } - ], - "users": [ - "perm_view_user" - ], - "groups": [], - "conditions": [], - "delegateAdmin": false - } - ], - "rowFilterPolicyItems": [], - "options": {}, - "validitySchedules": [], - "policyLabels": [ - "" - ], - "id": 20, - "guid": "bfeddeab-50d0-4902-985f-42559efa39c3", - "isEnabled": true, - "version": 1 + "table" : { + "values" : [ "table_select*" ], + "isExcludes" : false, + "isRecursive" : false + } }, - { - "service": "hive_jenkins", - "name": "src_value2_nullify", - "policyType": 1, - "policyPriority": 0, - "description": "", - "isAuditEnabled": true, - "resources": { - "database": { - "values": [ - "default", - "spark_catalog" - ], - "isExcludes": false, - "isRecursive": false - }, - "column": { - "values": [ - "value2" - ], - "isExcludes": false, - "isRecursive": false - }, - "table": { - "values": [ - "src" - ], - "isExcludes": false, - "isRecursive": false - } + "conditions" : [ ], + "policyItems" : [ { + "accesses" : [ { + "type" : "select", + "isAllowed" : true + }, { + "type" : "use", + "isAllowed" : true + } ], + "users" : [ "bob" ], + "groups" : [ ], + "roles" : [ ], + "conditions" : [ ], + "delegateAdmin" : true + } ], + "denyPolicyItems" : [ ], + "allowExceptions" : [ ], + "denyExceptions" : [ ], + "dataMaskPolicyItems" : [ ], + "rowFilterPolicyItems" : [ ], + "options" : { }, + "validitySchedules" : [ ], + "policyLabels" : [ ], + "isDenyAllElse" : false + }, { + "id" : 7, + "guid" : "8f14e45f-ceea-367a-9a36-dedd4bea2543", + "isEnabled" : true, + "version" : 1, + "service" : "hive_jenkins", + "name" : "someone_access_perm_view", + "policyType" : 0, + "policyPriority" : 0, + "description" : "", + "isAuditEnabled" : true, + "resources" : { + "database" : { + "values" : [ "default" ], + "isExcludes" : false, + "isRecursive" : false + }, + "column" : { + "values" : [ "*" ], + "isExcludes" : false, + "isRecursive" : false }, - "policyItems": [], - "denyPolicyItems": [], - "allowExceptions": [], - "denyExceptions": [], - "dataMaskPolicyItems": [ - { - "dataMaskInfo": { - "dataMaskType": "MASK" - }, - "accesses": [ - { - "type": "select", - "isAllowed": true - } - ], - "users": [ - "bob" - ], - "groups": [], - "conditions": [], - "delegateAdmin": false - } - ], - "rowFilterPolicyItems": [], - "options": {}, - "validitySchedules": [], - "policyLabels": [ - "" - ], - "id": 6, - "guid": "98a04cd7-8d14-4466-adc9-126d87a3af69", - "isEnabled": true, - "version": 1 + "table" : { + "values" : [ "perm_view" ], + "isExcludes" : false, + "isRecursive" : false + } }, - { - "service": "hive_jenkins", - "name": "src_value3_sf4", - "policyType": 1, - "policyPriority": 0, - "description": "", - "isAuditEnabled": true, - "resources": { - "database": { - "values": [ - "default", - "spark_catalog" - ], - "isExcludes": false, - "isRecursive": false - }, - "column": { - "values": [ - "value3" - ], - "isExcludes": false, - "isRecursive": false - }, - "table": { - "values": [ - "src" - ], - "isExcludes": false, - "isRecursive": false - } + "conditions" : [ ], + "policyItems" : [ { + "accesses" : [ { + "type" : "select", + "isAllowed" : true + } ], + "users" : [ "user_perm_view_only" ], + "groups" : [ ], + "roles" : [ ], + "conditions" : [ ], + "delegateAdmin" : true + } ], + "denyPolicyItems" : [ ], + "allowExceptions" : [ ], + "denyExceptions" : [ ], + "dataMaskPolicyItems" : [ ], + "rowFilterPolicyItems" : [ ], + "options" : { }, + "validitySchedules" : [ ], + "policyLabels" : [ ], + "isDenyAllElse" : false + }, { + "id" : 8, + "guid" : "c9f0f895-fb98-3b91-99f5-1fd0297e236d", + "isEnabled" : true, + "version" : 1, + "service" : "hive_jenkins", + "name" : "someone_access_table2", + "policyType" : 0, + "policyPriority" : 0, + "description" : "", + "isAuditEnabled" : true, + "resources" : { + "database" : { + "values" : [ "default" ], + "isExcludes" : false, + "isRecursive" : false + }, + "column" : { + "values" : [ "*" ], + "isExcludes" : false, + "isRecursive" : false }, - "policyItems": [], - "denyPolicyItems": [], - "allowExceptions": [], - "denyExceptions": [], - "dataMaskPolicyItems": [ - { - "dataMaskInfo": { - "dataMaskType": "MASK_SHOW_FIRST_4" - }, - "accesses": [ - { - "type": "select", - "isAllowed": true - } - ], - "users": [ - "bob" - ], - "groups": [], - "conditions": [], - "delegateAdmin": false - } - ], - "rowFilterPolicyItems": [], - "options": {}, - "validitySchedules": [], - "policyLabels": [ - "" - ], - "id": 7, - "guid": "9d50a525-b24c-4cf5-a885-d10d426368d1", - "isEnabled": true, - "version": 1 + "table" : { + "values" : [ "table2" ], + "isExcludes" : false, + "isRecursive" : false + } }, - { - "service": "hive_jenkins", - "name": "src_value4_sf4", - "policyType": 1, - "policyPriority": 0, - "description": "", - "isAuditEnabled": true, - "resources": { - "database": { - "values": [ - "default", - "spark_catalog" - ], - "isExcludes": false, - "isRecursive": false - }, - "column": { - "values": [ - "value4" - ], - "isExcludes": false, - "isRecursive": false - }, - "table": { - "values": [ - "src" - ], - "isExcludes": false, - "isRecursive": false - } + "conditions" : [ ], + "policyItems" : [ { + "accesses" : [ { + "type" : "select", + "isAllowed" : true + } ], + "users" : [ "user_table2_only" ], + "groups" : [ ], + "roles" : [ ], + "conditions" : [ ], + "delegateAdmin" : true + } ], + "denyPolicyItems" : [ ], + "allowExceptions" : [ ], + "denyExceptions" : [ ], + "dataMaskPolicyItems" : [ ], + "rowFilterPolicyItems" : [ ], + "options" : { }, + "validitySchedules" : [ ], + "policyLabels" : [ ], + "isDenyAllElse" : false + }, { + "id" : 9, + "guid" : "45c48cce-2e2d-3fbd-aa1a-fc51c7c6ad26", + "isEnabled" : true, + "version" : 1, + "service" : "hive_jenkins", + "name" : "src_key_less_than_20", + "policyType" : 2, + "policyPriority" : 0, + "description" : "", + "isAuditEnabled" : true, + "resources" : { + "database" : { + "values" : [ "default" ], + "isExcludes" : false, + "isRecursive" : false }, - "policyItems": [], - "denyPolicyItems": [], - "allowExceptions": [], - "denyExceptions": [], - "dataMaskPolicyItems": [ - { - "dataMaskInfo": { - "dataMaskType": "MASK_DATE_SHOW_YEAR" - }, - "accesses": [ - { - "type": "select", - "isAllowed": true - } - ], - "users": [ - "bob" - ], - "groups": [], - "conditions": [], - "delegateAdmin": false - } - ], - "rowFilterPolicyItems": [], - "options": {}, - "validitySchedules": [], - "policyLabels": [ - "" - ], - "id": 8, - "guid": "9d50a526-b24c-4cf5-a885-d10d426368d1", - "isEnabled": true, - "version": 1 + "table" : { + "values" : [ "src" ], + "isExcludes" : false, + "isRecursive" : false + } }, - { - "service": "hive_jenkins", - "name": "src_value5_show_last_4", - "policyType": 1, - "policyPriority": 0, - "description": "", - "isAuditEnabled": true, - "resources": { - "database": { - "values": [ - "default", - "spark_catalog" - ], - "isExcludes": false, - "isRecursive": false - }, - "column": { - "values": [ - "value5" - ], - "isExcludes": false, - "isRecursive": false - }, - "table": { - "values": [ - "src" - ], - "isExcludes": false, - "isRecursive": false - } + "conditions" : [ ], + "policyItems" : [ ], + "denyPolicyItems" : [ ], + "allowExceptions" : [ ], + "denyExceptions" : [ ], + "dataMaskPolicyItems" : [ ], + "rowFilterPolicyItems" : [ { + "accesses" : [ { + "type" : "select", + "isAllowed" : true + } ], + "users" : [ "bob", "perm_view_user" ], + "groups" : [ ], + "roles" : [ ], + "conditions" : [ ], + "delegateAdmin" : false, + "rowFilterInfo" : { + "filterExpr" : "key<20" + } + } ], + "options" : { }, + "validitySchedules" : [ ], + "policyLabels" : [ ], + "isDenyAllElse" : false + }, { + "id" : 10, + "guid" : "d3d94468-02a4-3259-b55d-38e6d163e820", + "isEnabled" : true, + "version" : 1, + "service" : "hive_jenkins", + "name" : "perm_view_key_less_than_20", + "policyType" : 2, + "policyPriority" : 0, + "description" : "", + "isAuditEnabled" : true, + "resources" : { + "database" : { + "values" : [ "default" ], + "isExcludes" : false, + "isRecursive" : false }, - "policyItems": [], - "denyPolicyItems": [], - "allowExceptions": [], - "denyExceptions": [], - "dataMaskPolicyItems": [ - { - "dataMaskInfo": { - "dataMaskType": "MASK_SHOW_LAST_4" - }, - "accesses": [ - { - "type": "select", - "isAllowed": true - } - ], - "users": [ - "bob" - ], - "groups": [], - "conditions": [], - "delegateAdmin": false - } - ], - "rowFilterPolicyItems": [], - "options": {}, - "validitySchedules": [], - "policyLabels": [ - "" - ], - "id": 32, - "guid": "b3f1f1e0-2bd6-4b20-8a32-a531006ae151", - "isEnabled": true, - "version": 1 - } - ], - "serviceDef": { - "name": "hive", - "implClass": "org.apache.ranger.services.hive.RangerServiceHive", - "label": "Hive Server2", - "description": "Hive Server2", - "options": { - "enableDenyAndExceptionsInPolicies": "true" + "table" : { + "values" : [ "perm_view" ], + "isExcludes" : false, + "isRecursive" : false + } }, - "configs": [ - { - "itemId": 1, - "name": "username", - "type": "string", - "mandatory": true, - "validationRegEx": "", - "validationMessage": "", - "uiHint": "", - "label": "Username" + "conditions" : [ ], + "policyItems" : [ ], + "denyPolicyItems" : [ ], + "allowExceptions" : [ ], + "denyExceptions" : [ ], + "dataMaskPolicyItems" : [ ], + "rowFilterPolicyItems" : [ { + "accesses" : [ { + "type" : "select", + "isAllowed" : true + } ], + "users" : [ "perm_view_user" ], + "groups" : [ ], + "roles" : [ ], + "conditions" : [ ], + "delegateAdmin" : false, + "rowFilterInfo" : { + "filterExpr" : "key<20" + } + } ], + "options" : { }, + "validitySchedules" : [ ], + "policyLabels" : [ ], + "isDenyAllElse" : false + }, { + "id" : 11, + "guid" : "6512bd43-d9ca-36e0-ac99-0b0a82652dca", + "isEnabled" : true, + "version" : 1, + "service" : "hive_jenkins", + "name" : "src_value_hash_perm_view", + "policyType" : 1, + "policyPriority" : 0, + "description" : "", + "isAuditEnabled" : true, + "resources" : { + "database" : { + "values" : [ "default", "spark_catalog" ], + "isExcludes" : false, + "isRecursive" : false }, - { - "itemId": 2, - "name": "password", - "type": "password", - "mandatory": true, - "validationRegEx": "", - "validationMessage": "", - "uiHint": "", - "label": "Password" + "column" : { + "values" : [ "value1" ], + "isExcludes" : false, + "isRecursive" : false }, - { - "itemId": 3, - "name": "jdbc.driverClassName", - "type": "string", - "mandatory": true, - "defaultValue": "org.apache.hive.jdbc.HiveDriver", - "validationRegEx": "", - "validationMessage": "", - "uiHint": "" + "table" : { + "values" : [ "src" ], + "isExcludes" : false, + "isRecursive" : false + } + }, + "conditions" : [ ], + "policyItems" : [ ], + "denyPolicyItems" : [ ], + "allowExceptions" : [ ], + "denyExceptions" : [ ], + "dataMaskPolicyItems" : [ { + "accesses" : [ { + "type" : "select", + "isAllowed" : true + } ], + "users" : [ "bob" ], + "groups" : [ ], + "roles" : [ ], + "conditions" : [ ], + "delegateAdmin" : true, + "dataMaskInfo" : { + "dataMaskType" : "MASK_HASH" + } + } ], + "rowFilterPolicyItems" : [ ], + "options" : { }, + "validitySchedules" : [ ], + "policyLabels" : [ ], + "isDenyAllElse" : false + }, { + "id" : 12, + "guid" : "c20ad4d7-6fe9-3759-aa27-a0c99bff6710", + "isEnabled" : true, + "version" : 1, + "service" : "hive_jenkins", + "name" : "src_value_hash", + "policyType" : 1, + "policyPriority" : 0, + "description" : "", + "isAuditEnabled" : true, + "resources" : { + "database" : { + "values" : [ "default", "spark_catalog" ], + "isExcludes" : false, + "isRecursive" : false }, - { - "itemId": 4, - "name": "jdbc.url", - "type": "string", - "mandatory": true, - "defaultValue": "", - "validationRegEx": "", - "validationMessage": "", - "uiHint": "{\"TextFieldWithIcon\":true, \"info\": \"1.For Remote Mode, eg.\u003cbr\u003ejdbc:hive2://\u0026lt;host\u0026gt;:\u0026lt;port\u0026gt;\u003cbr\u003e2.For Embedded Mode (no host or port), eg.\u003cbr\u003ejdbc:hive2:///;initFile\u003d\u0026lt;file\u0026gt;\u003cbr\u003e3.For HTTP Mode, eg.\u003cbr\u003ejdbc:hive2://\u0026lt;host\u0026gt;:\u0026lt;port\u0026gt;/;\u003cbr\u003etransportMode\u003dhttp;httpPath\u003d\u0026lt;httpPath\u0026gt;\u003cbr\u003e4.For SSL Mode, eg.\u003cbr\u003ejdbc:hive2://\u0026lt;host\u0026gt;:\u0026lt;port\u0026gt;/;ssl\u003dtrue;\u003cbr\u003esslTrustStore\u003dtStore;trustStorePassword\u003dpw\u003cbr\u003e5.For ZooKeeper Mode, eg.\u003cbr\u003ejdbc:hive2://\u0026lt;host\u0026gt;/;serviceDiscoveryMode\u003d\u003cbr\u003ezooKeeper;zooKeeperNamespace\u003dhiveserver2\u003cbr\u003e6.For Kerberos Mode, eg.\u003cbr\u003ejdbc:hive2://\u0026lt;host\u0026gt;:\u0026lt;port\u0026gt;/;\u003cbr\u003eprincipal\u003dhive/domain@EXAMPLE.COM\u003cbr\u003e\"}" + "column" : { + "values" : [ "value1" ], + "isExcludes" : false, + "isRecursive" : false }, - { - "itemId": 5, - "name": "commonNameForCertificate", - "type": "string", - "mandatory": false, - "validationRegEx": "", - "validationMessage": "", - "uiHint": "", - "label": "Common Name for Certificate" + "table" : { + "values" : [ "perm_view" ], + "isExcludes" : false, + "isRecursive" : false } - ], - "resources": [ - { - "itemId": 1, - "name": "database", - "type": "string", - "level": 10, - "mandatory": true, - "lookupSupported": true, - "recursiveSupported": false, - "excludesSupported": true, - "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", - "matcherOptions": { - "wildCard": "true", - "ignoreCase": "true" - }, - "validationRegEx": "", - "validationMessage": "", - "uiHint": "", - "label": "Hive Database", - "description": "Hive Database", - "accessTypeRestrictions": [], - "isValidLeaf": false + }, + "conditions" : [ ], + "policyItems" : [ ], + "denyPolicyItems" : [ ], + "allowExceptions" : [ ], + "denyExceptions" : [ ], + "dataMaskPolicyItems" : [ { + "accesses" : [ { + "type" : "select", + "isAllowed" : true + } ], + "users" : [ "perm_view_user" ], + "groups" : [ ], + "roles" : [ ], + "conditions" : [ ], + "delegateAdmin" : true, + "dataMaskInfo" : { + "dataMaskType" : "MASK_HASH" + } + } ], + "rowFilterPolicyItems" : [ ], + "options" : { }, + "validitySchedules" : [ ], + "policyLabels" : [ ], + "isDenyAllElse" : false + }, { + "id" : 13, + "guid" : "c51ce410-c124-310e-8db5-e4b97fc2af39", + "isEnabled" : true, + "version" : 1, + "service" : "hive_jenkins", + "name" : "src_value2_nullify", + "policyType" : 1, + "policyPriority" : 0, + "description" : "", + "isAuditEnabled" : true, + "resources" : { + "database" : { + "values" : [ "default", "spark_catalog", "iceberg_ns", "ns1" ], + "isExcludes" : false, + "isRecursive" : false }, - { - "itemId": 5, - "name": "url", - "type": "string", - "level": 10, - "mandatory": true, - "lookupSupported": false, - "recursiveSupported": true, - "excludesSupported": false, - "matcher": "org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", - "matcherOptions": { - "wildCard": "true", - "ignoreCase": "false" - }, - "validationRegEx": "", - "validationMessage": "", - "uiHint": "", - "label": "URL", - "description": "URL", - "accessTypeRestrictions": [], - "isValidLeaf": true + "column" : { + "values" : [ "value2" ], + "isExcludes" : false, + "isRecursive" : false }, - { - "itemId": 2, - "name": "table", - "type": "string", - "level": 20, - "parent": "database", - "mandatory": true, - "lookupSupported": true, - "recursiveSupported": false, - "excludesSupported": true, - "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", - "matcherOptions": { - "wildCard": "true", - "ignoreCase": "true" - }, - "validationRegEx": "", - "validationMessage": "", - "uiHint": "", - "label": "Hive Table", - "description": "Hive Table", - "accessTypeRestrictions": [], - "isValidLeaf": false + "table" : { + "values" : [ "src" ], + "isExcludes" : false, + "isRecursive" : false + } + }, + "conditions" : [ ], + "policyItems" : [ ], + "denyPolicyItems" : [ ], + "allowExceptions" : [ ], + "denyExceptions" : [ ], + "dataMaskPolicyItems" : [ { + "accesses" : [ { + "type" : "select", + "isAllowed" : true + } ], + "users" : [ "bob" ], + "groups" : [ ], + "roles" : [ ], + "conditions" : [ ], + "delegateAdmin" : true, + "dataMaskInfo" : { + "dataMaskType" : "MASK" + } + } ], + "rowFilterPolicyItems" : [ ], + "options" : { }, + "validitySchedules" : [ ], + "policyLabels" : [ ], + "isDenyAllElse" : false + }, { + "id" : 14, + "guid" : "aab32389-22bc-325a-af60-6eb525ffdc56", + "isEnabled" : true, + "version" : 1, + "service" : "hive_jenkins", + "name" : "src_value3_sf4", + "policyType" : 1, + "policyPriority" : 0, + "description" : "", + "isAuditEnabled" : true, + "resources" : { + "database" : { + "values" : [ "default", "spark_catalog" ], + "isExcludes" : false, + "isRecursive" : false }, - { - "itemId": 3, - "name": "udf", - "type": "string", - "level": 20, - "parent": "database", - "mandatory": true, - "lookupSupported": true, - "recursiveSupported": false, - "excludesSupported": true, - "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", - "matcherOptions": { - "wildCard": "true", - "ignoreCase": "true" - }, - "validationRegEx": "", - "validationMessage": "", - "uiHint": "", - "label": "Hive UDF", - "description": "Hive UDF", - "accessTypeRestrictions": [], - "isValidLeaf": true + "column" : { + "values" : [ "value3" ], + "isExcludes" : false, + "isRecursive" : false }, - { - "itemId": 4, - "name": "column", - "type": "string", - "level": 30, - "parent": "table", - "mandatory": true, - "lookupSupported": true, - "recursiveSupported": false, - "excludesSupported": true, - "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", - "matcherOptions": { - "wildCard": "true", - "ignoreCase": "true" - }, - "validationRegEx": "", - "validationMessage": "", - "uiHint": "", - "label": "Hive Column", - "description": "Hive Column", - "accessTypeRestrictions": [], - "isValidLeaf": true + "table" : { + "values" : [ "src" ], + "isExcludes" : false, + "isRecursive" : false + } + }, + "conditions" : [ ], + "policyItems" : [ ], + "denyPolicyItems" : [ ], + "allowExceptions" : [ ], + "denyExceptions" : [ ], + "dataMaskPolicyItems" : [ { + "accesses" : [ { + "type" : "select", + "isAllowed" : true + } ], + "users" : [ "bob" ], + "groups" : [ ], + "roles" : [ ], + "conditions" : [ ], + "delegateAdmin" : true, + "dataMaskInfo" : { + "dataMaskType" : "MASK_SHOW_FIRST_4" } - ], - "accessTypes": [ - { - "itemId": 1, - "name": "select", - "label": "select", - "impliedGrants": [] + } ], + "rowFilterPolicyItems" : [ ], + "options" : { }, + "validitySchedules" : [ ], + "policyLabels" : [ ], + "isDenyAllElse" : false + }, { + "id" : 15, + "guid" : "9bf31c7f-f062-336a-96d3-c8bd1f8f2ff3", + "isEnabled" : true, + "version" : 1, + "service" : "hive_jenkins", + "name" : "src_value4_sf4", + "policyType" : 1, + "policyPriority" : 0, + "description" : "", + "isAuditEnabled" : true, + "resources" : { + "database" : { + "values" : [ "default", "spark_catalog" ], + "isExcludes" : false, + "isRecursive" : false }, - { - "itemId": 2, - "name": "update", - "label": "update", - "impliedGrants": [] + "column" : { + "values" : [ "value4" ], + "isExcludes" : false, + "isRecursive" : false }, - { - "itemId": 3, - "name": "create", - "label": "Create", - "impliedGrants": [] + "table" : { + "values" : [ "src" ], + "isExcludes" : false, + "isRecursive" : false + } + }, + "conditions" : [ ], + "policyItems" : [ ], + "denyPolicyItems" : [ ], + "allowExceptions" : [ ], + "denyExceptions" : [ ], + "dataMaskPolicyItems" : [ { + "accesses" : [ { + "type" : "select", + "isAllowed" : true + } ], + "users" : [ "bob" ], + "groups" : [ ], + "roles" : [ ], + "conditions" : [ ], + "delegateAdmin" : true, + "dataMaskInfo" : { + "dataMaskType" : "MASK_DATE_SHOW_YEAR" + } + } ], + "rowFilterPolicyItems" : [ ], + "options" : { }, + "validitySchedules" : [ ], + "policyLabels" : [ ], + "isDenyAllElse" : false + }, { + "id" : 16, + "guid" : "c74d97b0-1eae-357e-84aa-9d5bade97baf", + "isEnabled" : true, + "version" : 1, + "service" : "hive_jenkins", + "name" : "src_value5_sf4", + "policyType" : 1, + "policyPriority" : 0, + "description" : "", + "isAuditEnabled" : true, + "resources" : { + "database" : { + "values" : [ "default", "spark_catalog" ], + "isExcludes" : false, + "isRecursive" : false }, - { - "itemId": 4, - "name": "drop", - "label": "Drop", - "impliedGrants": [] + "column" : { + "values" : [ "value5" ], + "isExcludes" : false, + "isRecursive" : false }, - { - "itemId": 5, - "name": "alter", - "label": "Alter", - "impliedGrants": [] + "table" : { + "values" : [ "src" ], + "isExcludes" : false, + "isRecursive" : false + } + }, + "conditions" : [ ], + "policyItems" : [ ], + "denyPolicyItems" : [ ], + "allowExceptions" : [ ], + "denyExceptions" : [ ], + "dataMaskPolicyItems" : [ { + "accesses" : [ { + "type" : "select", + "isAllowed" : true + } ], + "users" : [ "bob" ], + "groups" : [ ], + "roles" : [ ], + "conditions" : [ ], + "delegateAdmin" : true, + "dataMaskInfo" : { + "dataMaskType" : "MASK_SHOW_LAST_4" + } + } ], + "rowFilterPolicyItems" : [ ], + "options" : { }, + "validitySchedules" : [ ], + "policyLabels" : [ ], + "isDenyAllElse" : false + } ], + "serviceDef" : { + "name" : "hive", + "implClass" : "org.apache.ranger.services.hive.RangerServiceHive", + "label" : "Hive Server2", + "description" : "Hive Server2", + "options" : { + "enableDenyAndExceptionsInPolicies" : "true" + }, + "configs" : [ { + "itemId" : 1, + "name" : "username", + "type" : "string", + "mandatory" : true, + "validationRegEx" : "", + "validationMessage" : "", + "uiHint" : "", + "label" : "Username" + }, { + "itemId" : 2, + "name" : "password", + "type" : "password", + "mandatory" : true, + "validationRegEx" : "", + "validationMessage" : "", + "uiHint" : "", + "label" : "Password" + }, { + "itemId" : 3, + "name" : "jdbc.driverClassName", + "type" : "string", + "mandatory" : true, + "defaultValue" : "org.apache.hive.jdbc.HiveDriver", + "validationRegEx" : "", + "validationMessage" : "", + "uiHint" : "" + }, { + "itemId" : 4, + "name" : "jdbc.url", + "type" : "string", + "mandatory" : true, + "defaultValue" : "", + "validationRegEx" : "", + "validationMessage" : "", + "uiHint" : "{\"TextFieldWithIcon\":true, \"info\": \"1.For Remote Mode, eg.
      jdbc:hive2://<host>:<port>
      2.For Embedded Mode (no host or port), eg.
      jdbc:hive2:///;initFile=<file>
      3.For HTTP Mode, eg.
      jdbc:hive2://<host>:<port>/;
      transportMode=http;httpPath=<httpPath>
      4.For SSL Mode, eg.
      jdbc:hive2://<host>:<port>/;ssl=true;
      sslTrustStore=tStore;trustStorePassword=pw
      5.For ZooKeeper Mode, eg.
      jdbc:hive2://<host>/;serviceDiscoveryMode=
      zooKeeper;zooKeeperNamespace=hiveserver2
      6.For Kerberos Mode, eg.
      jdbc:hive2://<host>:<port>/;
      principal=hive/domain@EXAMPLE.COM
      \"}" + }, { + "itemId" : 5, + "name" : "commonNameForCertificate", + "type" : "string", + "mandatory" : false, + "validationRegEx" : "", + "validationMessage" : "", + "uiHint" : "", + "label" : "Common Name for Certificate" + } ], + "resources" : [ { + "itemId" : 1, + "name" : "database", + "type" : "string", + "level" : 10, + "mandatory" : true, + "lookupSupported" : true, + "recursiveSupported" : false, + "excludesSupported" : true, + "matcher" : "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions" : { + "wildCard" : "true", + "ignoreCase" : "true" }, - { - "itemId": 6, - "name": "index", - "label": "Index", - "impliedGrants": [] + "validationRegEx" : "", + "validationMessage" : "", + "uiHint" : "", + "label" : "Hive Database", + "description" : "Hive Database", + "accessTypeRestrictions" : [ ], + "isValidLeaf" : false + }, { + "itemId" : 5, + "name" : "url", + "type" : "string", + "level" : 10, + "mandatory" : true, + "lookupSupported" : false, + "recursiveSupported" : true, + "excludesSupported" : false, + "matcher" : "org.apache.ranger.plugin.resourcematcher.RangerPathResourceMatcher", + "matcherOptions" : { + "wildCard" : "true", + "ignoreCase" : "false" }, - { - "itemId": 7, - "name": "lock", - "label": "Lock", - "impliedGrants": [] + "validationRegEx" : "", + "validationMessage" : "", + "uiHint" : "", + "label" : "URL", + "description" : "URL", + "accessTypeRestrictions" : [ ], + "isValidLeaf" : true + }, { + "itemId" : 2, + "name" : "table", + "type" : "string", + "level" : 20, + "parent" : "database", + "mandatory" : true, + "lookupSupported" : true, + "recursiveSupported" : false, + "excludesSupported" : true, + "matcher" : "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions" : { + "wildCard" : "true", + "ignoreCase" : "true" }, - { - "itemId": 8, - "name": "all", - "label": "All", - "impliedGrants": [ - "select", - "update", - "create", - "drop", - "alter", - "index", - "lock", - "read", - "write" - ] + "validationRegEx" : "", + "validationMessage" : "", + "uiHint" : "", + "label" : "Hive Table", + "description" : "Hive Table", + "accessTypeRestrictions" : [ ], + "isValidLeaf" : false + }, { + "itemId" : 3, + "name" : "udf", + "type" : "string", + "level" : 20, + "parent" : "database", + "mandatory" : true, + "lookupSupported" : true, + "recursiveSupported" : false, + "excludesSupported" : true, + "matcher" : "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions" : { + "wildCard" : "true", + "ignoreCase" : "true" }, - { - "itemId": 9, - "name": "read", - "label": "Read", - "impliedGrants": [] + "validationRegEx" : "", + "validationMessage" : "", + "uiHint" : "", + "label" : "Hive UDF", + "description" : "Hive UDF", + "accessTypeRestrictions" : [ ], + "isValidLeaf" : true + }, { + "itemId" : 4, + "name" : "column", + "type" : "string", + "level" : 30, + "parent" : "table", + "mandatory" : true, + "lookupSupported" : true, + "recursiveSupported" : false, + "excludesSupported" : true, + "matcher" : "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions" : { + "wildCard" : "true", + "ignoreCase" : "true" }, - { - "itemId": 10, - "name": "write", - "label": "Write", - "impliedGrants": [] - } - ], - "policyConditions": [], - "contextEnrichers": [], - "enums": [], - "dataMaskDef": { - "maskTypes": [ - { - "itemId": 1, - "name": "MASK", - "label": "Redact", - "description": "Replace lowercase with \u0027x\u0027, uppercase with \u0027X\u0027, digits with \u00270\u0027", - "transformer": "mask({col})", - "dataMaskOptions": {} - }, - { - "itemId": 2, - "name": "MASK_SHOW_LAST_4", - "label": "Partial mask: show last 4", - "description": "Show last 4 characters; replace rest with \u0027x\u0027", - "transformer": "mask_show_last_n({col}, 4, \u0027x\u0027, \u0027x\u0027, \u0027x\u0027, -1, \u00271\u0027)", - "dataMaskOptions": {} - }, - { - "itemId": 3, - "name": "MASK_SHOW_FIRST_4", - "label": "Partial mask: show first 4", - "description": "Show first 4 characters; replace rest with \u0027x\u0027", - "transformer": "mask_show_first_n({col}, 4, \u0027x\u0027, \u0027x\u0027, \u0027x\u0027, -1, \u00271\u0027)", - "dataMaskOptions": {} - }, - { - "itemId": 4, - "name": "MASK_HASH", - "label": "Hash", - "description": "Hash the value", - "transformer": "mask_hash({col})", - "dataMaskOptions": {} + "validationRegEx" : "", + "validationMessage" : "", + "uiHint" : "", + "label" : "Hive Column", + "description" : "Hive Column", + "accessTypeRestrictions" : [ ], + "isValidLeaf" : true + } ], + "accessTypes" : [ { + "itemId" : 1, + "name" : "select", + "label" : "select", + "impliedGrants" : [ ] + }, { + "itemId" : 2, + "name" : "update", + "label" : "update", + "impliedGrants" : [ ] + }, { + "itemId" : 3, + "name" : "create", + "label" : "Create", + "impliedGrants" : [ ] + }, { + "itemId" : 4, + "name" : "drop", + "label" : "Drop", + "impliedGrants" : [ ] + }, { + "itemId" : 5, + "name" : "alter", + "label" : "Alter", + "impliedGrants" : [ ] + }, { + "itemId" : 6, + "name" : "index", + "label" : "Index", + "impliedGrants" : [ ] + }, { + "itemId" : 7, + "name" : "lock", + "label" : "Lock", + "impliedGrants" : [ ] + }, { + "itemId" : 8, + "name" : "all", + "label" : "All", + "impliedGrants" : [ "select", "update", "create", "drop", "alter", "index", "lock", "read", "write" ] + }, { + "itemId" : 9, + "name" : "read", + "label" : "Read", + "impliedGrants" : [ ] + }, { + "itemId" : 10, + "name" : "write", + "label" : "Write", + "impliedGrants" : [ ] + } ], + "policyConditions" : [ ], + "contextEnrichers" : [ ], + "enums" : [ ], + "dataMaskDef" : { + "maskTypes" : [ { + "itemId" : 1, + "name" : "MASK", + "label" : "Redact", + "description" : "Replace lowercase with 'x', uppercase with 'X', digits with '0'", + "transformer" : "mask({col})", + "dataMaskOptions" : { } + }, { + "itemId" : 2, + "name" : "MASK_SHOW_LAST_4", + "label" : "Partial mask: show last 4", + "description" : "Show last 4 characters; replace rest with 'x'", + "transformer" : "mask_show_last_n({col}, 4, 'x', 'x', 'x', -1, '1')", + "dataMaskOptions" : { } + }, { + "itemId" : 3, + "name" : "MASK_SHOW_FIRST_4", + "label" : "Partial mask: show first 4", + "description" : "Show first 4 characters; replace rest with 'x'", + "transformer" : "mask_show_first_n({col}, 4, 'x', 'x', 'x', -1, '1')", + "dataMaskOptions" : { } + }, { + "itemId" : 4, + "name" : "MASK_HASH", + "label" : "Hash", + "description" : "Hash the value", + "transformer" : "mask_hash({col})", + "dataMaskOptions" : { } + }, { + "itemId" : 5, + "name" : "MASK_NULL", + "label" : "Nullify", + "description" : "Replace with NULL", + "dataMaskOptions" : { } + }, { + "itemId" : 6, + "name" : "MASK_NONE", + "label" : "Unmasked (retain original value)", + "description" : "No masking", + "dataMaskOptions" : { } + }, { + "itemId" : 12, + "name" : "MASK_DATE_SHOW_YEAR", + "label" : "Date: show only year", + "description" : "Date: show only year", + "transformer" : "mask({col}, 'x', 'x', 'x', -1, '1', 1, 0, -1)", + "dataMaskOptions" : { } + }, { + "itemId" : 13, + "name" : "CUSTOM", + "label" : "Custom", + "description" : "Custom", + "dataMaskOptions" : { } + } ], + "accessTypes" : [ { + "itemId" : 1, + "name" : "select", + "label" : "select", + "impliedGrants" : [ ] + } ], + "resources" : [ { + "itemId" : 1, + "name" : "database", + "type" : "string", + "level" : 10, + "mandatory" : true, + "lookupSupported" : true, + "recursiveSupported" : false, + "excludesSupported" : false, + "matcher" : "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions" : { + "wildCard" : "false", + "ignoreCase" : "true" }, - { - "itemId": 5, - "name": "MASK_NULL", - "label": "Nullify", - "description": "Replace with NULL", - "dataMaskOptions": {} + "validationRegEx" : "", + "validationMessage" : "", + "uiHint" : "{ \"singleValue\":true }", + "label" : "Hive Database", + "description" : "Hive Database", + "accessTypeRestrictions" : [ ], + "isValidLeaf" : false + }, { + "itemId" : 2, + "name" : "table", + "type" : "string", + "level" : 20, + "parent" : "database", + "mandatory" : true, + "lookupSupported" : true, + "recursiveSupported" : false, + "excludesSupported" : false, + "matcher" : "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions" : { + "wildCard" : "false", + "ignoreCase" : "true" }, - { - "itemId": 6, - "name": "MASK_NONE", - "label": "Unmasked (retain original value)", - "description": "No masking", - "dataMaskOptions": {} + "validationRegEx" : "", + "validationMessage" : "", + "uiHint" : "{ \"singleValue\":true }", + "label" : "Hive Table", + "description" : "Hive Table", + "accessTypeRestrictions" : [ ], + "isValidLeaf" : false + }, { + "itemId" : 4, + "name" : "column", + "type" : "string", + "level" : 30, + "parent" : "table", + "mandatory" : true, + "lookupSupported" : true, + "recursiveSupported" : false, + "excludesSupported" : false, + "matcher" : "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions" : { + "wildCard" : "false", + "ignoreCase" : "true" }, - { - "itemId": 12, - "name": "MASK_DATE_SHOW_YEAR", - "label": "Date: show only year", - "description": "Date: show only year", - "transformer": "mask({col}, \u0027x\u0027, \u0027x\u0027, \u0027x\u0027, -1, \u00271\u0027, 1, 0, -1)", - "dataMaskOptions": {} - }, - { - "itemId": 13, - "name": "CUSTOM", - "label": "Custom", - "description": "Custom", - "dataMaskOptions": {} - } - ], - "accessTypes": [ - { - "itemId": 1, - "name": "select", - "label": "select", - "impliedGrants": [] - } - ], - "resources": [ - { - "itemId": 1, - "name": "database", - "type": "string", - "level": 10, - "mandatory": true, - "lookupSupported": true, - "recursiveSupported": false, - "excludesSupported": false, - "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", - "matcherOptions": { - "wildCard": "false", - "ignoreCase": "true" - }, - "validationRegEx": "", - "validationMessage": "", - "uiHint": "{ \"singleValue\":true }", - "label": "Hive Database", - "description": "Hive Database", - "accessTypeRestrictions": [], - "isValidLeaf": false - }, - { - "itemId": 2, - "name": "table", - "type": "string", - "level": 20, - "parent": "database", - "mandatory": true, - "lookupSupported": true, - "recursiveSupported": false, - "excludesSupported": false, - "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", - "matcherOptions": { - "wildCard": "false", - "ignoreCase": "true" - }, - "validationRegEx": "", - "validationMessage": "", - "uiHint": "{ \"singleValue\":true }", - "label": "Hive Table", - "description": "Hive Table", - "accessTypeRestrictions": [], - "isValidLeaf": false - }, - { - "itemId": 4, - "name": "column", - "type": "string", - "level": 30, - "parent": "table", - "mandatory": true, - "lookupSupported": true, - "recursiveSupported": false, - "excludesSupported": false, - "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", - "matcherOptions": { - "wildCard": "false", - "ignoreCase": "true" - }, - "validationRegEx": "", - "validationMessage": "", - "uiHint": "{ \"singleValue\":true }", - "label": "Hive Column", - "description": "Hive Column", - "accessTypeRestrictions": [], - "isValidLeaf": true - } - ] + "validationRegEx" : "", + "validationMessage" : "", + "uiHint" : "{ \"singleValue\":true }", + "label" : "Hive Column", + "description" : "Hive Column", + "accessTypeRestrictions" : [ ], + "isValidLeaf" : true + } ] }, - "rowFilterDef": { - "accessTypes": [ - { - "itemId": 1, - "name": "select", - "label": "select", - "impliedGrants": [] - } - ], - "resources": [ - { - "itemId": 1, - "name": "database", - "type": "string", - "level": 10, - "mandatory": true, - "lookupSupported": true, - "recursiveSupported": false, - "excludesSupported": false, - "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", - "matcherOptions": { - "wildCard": "false", - "ignoreCase": "true" - }, - "validationRegEx": "", - "validationMessage": "", - "uiHint": "{ \"singleValue\":true }", - "label": "Hive Database", - "description": "Hive Database", - "accessTypeRestrictions": [], - "isValidLeaf": false + "rowFilterDef" : { + "accessTypes" : [ { + "itemId" : 1, + "name" : "select", + "label" : "select", + "impliedGrants" : [ ] + } ], + "resources" : [ { + "itemId" : 1, + "name" : "database", + "type" : "string", + "level" : 10, + "mandatory" : true, + "lookupSupported" : true, + "recursiveSupported" : false, + "excludesSupported" : false, + "matcher" : "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions" : { + "wildCard" : "false", + "ignoreCase" : "true" + }, + "validationRegEx" : "", + "validationMessage" : "", + "uiHint" : "{ \"singleValue\":true }", + "label" : "Hive Database", + "description" : "Hive Database", + "accessTypeRestrictions" : [ ], + "isValidLeaf" : false + }, { + "itemId" : 2, + "name" : "table", + "type" : "string", + "level" : 20, + "parent" : "database", + "mandatory" : true, + "lookupSupported" : true, + "recursiveSupported" : false, + "excludesSupported" : false, + "matcher" : "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions" : { + "wildCard" : "false", + "ignoreCase" : "true" }, - { - "itemId": 2, - "name": "table", - "type": "string", - "level": 20, - "parent": "database", - "mandatory": true, - "lookupSupported": true, - "recursiveSupported": false, - "excludesSupported": false, - "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", - "matcherOptions": { - "wildCard": "false", - "ignoreCase": "true" - }, - "validationRegEx": "", - "validationMessage": "", - "uiHint": "{ \"singleValue\":true }", - "label": "Hive Table", - "description": "Hive Table", - "accessTypeRestrictions": [], - "isValidLeaf": true - } - ] + "validationRegEx" : "", + "validationMessage" : "", + "uiHint" : "{ \"singleValue\":true }", + "label" : "Hive Table", + "description" : "Hive Table", + "accessTypeRestrictions" : [ ], + "isValidLeaf" : true + } ] }, - "id": 3, - "guid": "3e1afb5a-184a-4e82-9d9c-87a5cacc243c", - "isEnabled": true, - "createTime": "20190401-20:14:36.000-+0800", - "updateTime": "20190401-20:14:36.000-+0800", - "version": 1 + "id" : 3, + "guid" : "3e1afb5a-184a-4e82-9d9c-87a5cacc243c", + "isEnabled" : true, + "createTime" : "20190401-20:14:36.000-+0800", + "updateTime" : "20190401-20:14:36.000-+0800", + "version" : 1 }, - "auditMode": "audit-default" -} + "auditMode" : "audit-default" +} \ No newline at end of file diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/FunctionPrivilegesBuilderSuite.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/FunctionPrivilegesBuilderSuite.scala new file mode 100644 index 000000000..ad4b57faa --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/FunctionPrivilegesBuilderSuite.scala @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.plugin.spark.authz + +import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan +import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach} +// scalastyle:off +import org.scalatest.funsuite.AnyFunSuite + +import org.apache.kyuubi.plugin.spark.authz.OperationType.QUERY +import org.apache.kyuubi.plugin.spark.authz.ranger.AccessType + +abstract class FunctionPrivilegesBuilderSuite extends AnyFunSuite + with SparkSessionProvider with BeforeAndAfterAll with BeforeAndAfterEach { + // scalastyle:on + + protected def withTable(t: String)(f: String => Unit): Unit = { + try { + f(t) + } finally { + sql(s"DROP TABLE IF EXISTS $t") + } + } + + protected def withDatabase(t: String)(f: String => Unit): Unit = { + try { + f(t) + } finally { + sql(s"DROP DATABASE IF EXISTS $t") + } + } + + protected def checkColumns(plan: LogicalPlan, cols: Seq[String]): Unit = { + val (in, out, _) = PrivilegesBuilder.build(plan, spark) + assert(out.isEmpty, "Queries shall not check output privileges") + val po = in.head + assert(po.actionType === PrivilegeObjectActionType.OTHER) + assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) + assert(po.columns === cols) + } + + protected def checkColumns(query: String, cols: Seq[String]): Unit = { + checkColumns(sql(query).queryExecution.optimizedPlan, cols) + } + + protected val reusedDb: String = getClass.getSimpleName + protected val reusedDb2: String = getClass.getSimpleName + "2" + protected val reusedTable: String = reusedDb + "." + getClass.getSimpleName + protected val reusedTableShort: String = reusedTable.split("\\.").last + protected val reusedPartTable: String = reusedTable + "_part" + protected val reusedPartTableShort: String = reusedPartTable.split("\\.").last + protected val functionCount = 3 + protected val functionNamePrefix = "kyuubi_fun_" + protected val tempFunNamePrefix = "kyuubi_temp_fun_" + + override def beforeAll(): Unit = { + sql(s"CREATE DATABASE IF NOT EXISTS $reusedDb") + sql(s"CREATE DATABASE IF NOT EXISTS $reusedDb2") + sql(s"CREATE TABLE IF NOT EXISTS $reusedTable" + + s" (key int, value string) USING parquet") + sql(s"CREATE TABLE IF NOT EXISTS $reusedPartTable" + + s" (key int, value string, pid string) USING parquet" + + s" PARTITIONED BY(pid)") + // scalastyle:off + (0 until functionCount).foreach { index => + { + sql(s"CREATE FUNCTION ${reusedDb}.${functionNamePrefix}${index} AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFMaskHash'") + sql(s"CREATE FUNCTION ${reusedDb2}.${functionNamePrefix}${index} AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFMaskHash'") + sql(s"CREATE TEMPORARY FUNCTION ${tempFunNamePrefix}${index} AS 'org.apache.hadoop.hive.ql.udf.generic.GenericUDFMaskHash'") + } + } + sql(s"USE ${reusedDb2}") + // scalastyle:on + super.beforeAll() + } + + override def afterAll(): Unit = { + Seq(reusedTable, reusedPartTable).foreach { t => + sql(s"DROP TABLE IF EXISTS $t") + } + + Seq(reusedDb, reusedDb2).foreach { db => + (0 until functionCount).foreach { index => + sql(s"DROP FUNCTION ${db}.${functionNamePrefix}${index}") + } + sql(s"DROP DATABASE IF EXISTS ${db}") + } + + spark.stop() + super.afterAll() + } +} + +class HiveFunctionPrivilegesBuilderSuite extends FunctionPrivilegesBuilderSuite { + + override protected val catalogImpl: String = "hive" + + test("Function Call Query") { + val plan = sql(s"SELECT kyuubi_fun_1('data'), " + + s"kyuubi_fun_2(value), " + + s"${reusedDb}.kyuubi_fun_0(value), " + + s"kyuubi_temp_fun_1('data2')," + + s"kyuubi_temp_fun_2(key) " + + s"FROM $reusedTable").queryExecution.analyzed + val (inputs, _, _) = PrivilegesBuilder.buildFunctions(plan, spark) + assert(inputs.size === 3) + inputs.foreach { po => + assert(po.actionType === PrivilegeObjectActionType.OTHER) + assert(po.privilegeObjectType === PrivilegeObjectType.FUNCTION) + assert(po.dbname startsWith reusedDb.toLowerCase) + assert(po.objectName startsWith functionNamePrefix.toLowerCase) + val accessType = ranger.AccessType(po, QUERY, isInput = true) + assert(accessType === AccessType.SELECT) + } + } + + test("Function Call Query with Quoted Name") { + val plan = sql(s"SELECT `kyuubi_fun_1`('data'), " + + s"`kyuubi_fun_2`(value), " + + s"`${reusedDb}`.`kyuubi_fun_0`(value), " + + s"`kyuubi_temp_fun_1`('data2')," + + s"`kyuubi_temp_fun_2`(key) " + + s"FROM $reusedTable").queryExecution.analyzed + val (inputs, _, _) = PrivilegesBuilder.buildFunctions(plan, spark) + assert(inputs.size === 3) + inputs.foreach { po => + assert(po.actionType === PrivilegeObjectActionType.OTHER) + assert(po.privilegeObjectType === PrivilegeObjectType.FUNCTION) + assert(po.dbname startsWith reusedDb.toLowerCase) + assert(po.objectName startsWith functionNamePrefix.toLowerCase) + val accessType = ranger.AccessType(po, QUERY, isInput = true) + assert(accessType === AccessType.SELECT) + } + } + + test("Simple Function Call Query") { + val plan = sql(s"SELECT kyuubi_fun_1('data'), " + + s"kyuubi_fun_0('value'), " + + s"${reusedDb}.kyuubi_fun_0('value'), " + + s"${reusedDb}.kyuubi_fun_2('value'), " + + s"kyuubi_temp_fun_1('data2')," + + s"kyuubi_temp_fun_2('key') ").queryExecution.analyzed + val (inputs, _, _) = PrivilegesBuilder.buildFunctions(plan, spark) + assert(inputs.size === 4) + inputs.foreach { po => + assert(po.actionType === PrivilegeObjectActionType.OTHER) + assert(po.privilegeObjectType === PrivilegeObjectType.FUNCTION) + assert(po.dbname startsWith reusedDb.toLowerCase) + assert(po.objectName startsWith functionNamePrefix.toLowerCase) + val accessType = ranger.AccessType(po, QUERY, isInput = true) + assert(accessType === AccessType.SELECT) + } + } + + test("Function Call In CAST Command") { + val table = "castTable" + withTable(table) { table => + val plan = sql(s"CREATE TABLE ${table} " + + s"SELECT kyuubi_fun_1('data') col1, " + + s"${reusedDb2}.kyuubi_fun_2(value) col2, " + + s"kyuubi_fun_0(value) col3, " + + s"kyuubi_fun_2('value') col4, " + + s"${reusedDb}.kyuubi_fun_2('value') col5, " + + s"${reusedDb}.kyuubi_fun_1('value') col6, " + + s"kyuubi_temp_fun_1('data2') col7, " + + s"kyuubi_temp_fun_2(key) col8 " + + s"FROM ${reusedTable} WHERE ${reusedDb2}.kyuubi_fun_1(key)='123'").queryExecution.analyzed + val (inputs, _, _) = PrivilegesBuilder.buildFunctions(plan, spark) + assert(inputs.size === 7) + inputs.foreach { po => + assert(po.actionType === PrivilegeObjectActionType.OTHER) + assert(po.privilegeObjectType === PrivilegeObjectType.FUNCTION) + assert(po.dbname startsWith reusedDb.toLowerCase) + assert(po.objectName startsWith functionNamePrefix.toLowerCase) + val accessType = ranger.AccessType(po, QUERY, isInput = true) + assert(accessType === AccessType.SELECT) + } + } + } + +} diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/IcebergCatalogPrivilegesBuilderSuite.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/IcebergCatalogPrivilegesBuilderSuite.scala index d89d0696f..b8d51bc2c 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/IcebergCatalogPrivilegesBuilderSuite.scala +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/IcebergCatalogPrivilegesBuilderSuite.scala @@ -22,13 +22,15 @@ import org.scalatest.Outcome import org.apache.kyuubi.Utils import org.apache.kyuubi.plugin.spark.authz.OperationType._ import org.apache.kyuubi.plugin.spark.authz.ranger.AccessType +import org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils._ +import org.apache.kyuubi.tags.IcebergTest +import org.apache.kyuubi.util.AssertionUtils._ +@IcebergTest class IcebergCatalogPrivilegesBuilderSuite extends V2CommandsPrivilegesSuite { override protected val catalogImpl: String = "hive" override protected val sqlExtensions: String = - if (isSparkV32OrGreater) { - "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions" - } else "" + "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions" override protected def format = "iceberg" override protected val supportsUpdateTable = false @@ -38,20 +40,17 @@ class IcebergCatalogPrivilegesBuilderSuite extends V2CommandsPrivilegesSuite { override protected val supportsPartitionManagement = false override def beforeAll(): Unit = { - if (isSparkV32OrGreater) { - spark.conf.set( - s"spark.sql.catalog.$catalogV2", - "org.apache.iceberg.spark.SparkCatalog") - spark.conf.set(s"spark.sql.catalog.$catalogV2.type", "hadoop") - spark.conf.set( - s"spark.sql.catalog.$catalogV2.warehouse", - Utils.createTempDir("iceberg-hadoop").toString) - } + spark.conf.set( + s"spark.sql.catalog.$catalogV2", + "org.apache.iceberg.spark.SparkCatalog") + spark.conf.set(s"spark.sql.catalog.$catalogV2.type", "hadoop") + spark.conf.set( + s"spark.sql.catalog.$catalogV2.warehouse", + Utils.createTempDir("iceberg-hadoop").toString) super.beforeAll() } override def withFixture(test: NoArgTest): Outcome = { - assume(isSparkV32OrGreater) test() } @@ -59,13 +58,21 @@ class IcebergCatalogPrivilegesBuilderSuite extends V2CommandsPrivilegesSuite { val plan = sql(s"DELETE FROM $catalogTable WHERE key = 1 ").queryExecution.analyzed val (inputs, outputs, operationType) = PrivilegesBuilder.build(plan, spark) assert(operationType === QUERY) - assert(inputs.isEmpty) + if (isSparkV34OrGreater) { + assert(inputs.size === 1) + val po = inputs.head + assertEqualsIgnoreCase(namespace)(po.dbname) + assertEqualsIgnoreCase(catalogTableShort)(po.objectName) + assertContains(po.columns, "key", "value") + } else { + assert(inputs.size === 0) + } assert(outputs.size === 1) val po = outputs.head assert(po.actionType === PrivilegeObjectActionType.UPDATE) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po.dbname === namespace) - assert(po.objectName === catalogTableShort) + assertEqualsIgnoreCase(namespace)(po.dbname) + assertEqualsIgnoreCase(catalogTableShort)(po.objectName) assert(po.columns.isEmpty) checkV2TableOwner(po) val accessType = AccessType(po, operationType, isInput = false) @@ -76,13 +83,21 @@ class IcebergCatalogPrivilegesBuilderSuite extends V2CommandsPrivilegesSuite { val plan = sql(s"UPDATE $catalogTable SET value = 'b' WHERE key = 1 ").queryExecution.analyzed val (inputs, outputs, operationType) = PrivilegesBuilder.build(plan, spark) assert(operationType === QUERY) - assert(inputs.isEmpty) + if (isSparkV35OrGreater) { + assert(inputs.size === 1) + val po = inputs.head + assertEqualsIgnoreCase(namespace)(po.dbname) + assertEqualsIgnoreCase(catalogTableShort)(po.objectName) + assertContains(po.columns, "key", "value") + } else { + assert(inputs.size === 0) + } assert(outputs.size === 1) val po = outputs.head assert(po.actionType === PrivilegeObjectActionType.UPDATE) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po.dbname === namespace) - assert(po.objectName === catalogTableShort) + assertEqualsIgnoreCase(namespace)(po.dbname) + assertEqualsIgnoreCase(catalogTableShort)(po.objectName) assert(po.columns.isEmpty) checkV2TableOwner(po) val accessType = AccessType(po, operationType, isInput = false) @@ -100,12 +115,24 @@ class IcebergCatalogPrivilegesBuilderSuite extends V2CommandsPrivilegesSuite { s"WHEN NOT MATCHED THEN INSERT *").queryExecution.analyzed val (inputs, outputs, operationType) = PrivilegesBuilder.build(plan, spark) assert(operationType === QUERY) - assert(inputs.size === 1) - val po0 = inputs.head + if (isSparkV35OrGreater) { + assert(inputs.size === 2) + val po = inputs.head + assert(po.actionType === PrivilegeObjectActionType.OTHER) + assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) + assertEqualsIgnoreCase(namespace)(po.dbname) + assertEqualsIgnoreCase(table)(po.objectName) + assertContains(po.columns, "key", "value") + // The properties of RowLevelOperationTable are empty, so owner is none + assert(po.owner.isEmpty) + } else { + assert(inputs.size === 1) + } + val po0 = inputs.last assert(po0.actionType === PrivilegeObjectActionType.OTHER) assert(po0.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po0.dbname === namespace) - assert(po0.objectName === catalogTableShort) + assertEqualsIgnoreCase(namespace)(po0.dbname) + assertEqualsIgnoreCase(catalogTableShort)(po0.objectName) assert(po0.columns === Seq("key", "value")) checkV2TableOwner(po0) @@ -113,12 +140,34 @@ class IcebergCatalogPrivilegesBuilderSuite extends V2CommandsPrivilegesSuite { val po = outputs.head assert(po.actionType === PrivilegeObjectActionType.UPDATE) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po.dbname === namespace) - assert(po.objectName === table) + assertEqualsIgnoreCase(namespace)(po.dbname) + assertEqualsIgnoreCase(table)(po.objectName) assert(po.columns.isEmpty) checkV2TableOwner(po) val accessType = AccessType(po, operationType, isInput = false) assert(accessType === AccessType.UPDATE) } } + + test("RewriteDataFilesProcedure") { + val table = "RewriteDataFilesProcedure" + withV2Table(table) { tableId => + sql(s"CREATE TABLE IF NOT EXISTS $tableId (key int, value String) USING iceberg") + sql(s"INSERT INTO $tableId VALUES (1, 'a'), (2, 'b'), (3, 'c')") + + val plan = sql(s"CALL $catalogV2.system.rewrite_data_files (table => '$tableId')") + .queryExecution.analyzed + val (inputs, outputs, operationType) = PrivilegesBuilder.build(plan, spark) + assert(operationType === ALTERTABLE_PROPERTIES) + assert(inputs.size === 0) + assert(outputs.size === 1) + val po = outputs.head + assert(po.actionType === PrivilegeObjectActionType.OTHER) + assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) + assertEqualsIgnoreCase(namespace)(po.dbname) + assertEqualsIgnoreCase(table)(po.objectName) + val accessType = AccessType(po, operationType, isInput = false) + assert(accessType === AccessType.ALTER) + } + } } diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/PrivilegesBuilderSuite.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/PrivilegesBuilderSuite.scala index 15f58deb3..696777441 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/PrivilegesBuilderSuite.scala +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/PrivilegesBuilderSuite.scala @@ -30,9 +30,11 @@ import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach} import org.scalatest.funsuite.AnyFunSuite import org.apache.kyuubi.plugin.spark.authz.OperationType._ +import org.apache.kyuubi.plugin.spark.authz.RangerTestNamespace._ +import org.apache.kyuubi.plugin.spark.authz.RangerTestUsers._ import org.apache.kyuubi.plugin.spark.authz.ranger.AccessType -import org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils -import org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils.isSparkVersionAtMost +import org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils._ +import org.apache.kyuubi.util.AssertionUtils._ abstract class PrivilegesBuilderSuite extends AnyFunSuite with SparkSessionProvider with BeforeAndAfterAll with BeforeAndAfterEach { @@ -110,7 +112,7 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite } test("AlterDatabasePropertiesCommand") { - assume(isSparkVersionAtMost("3.2")) + assume(SPARK_RUNTIME_VERSION <= "3.2") val plan = sql("ALTER DATABASE default SET DBPROPERTIES (abc = '123')").queryExecution.analyzed val (in, out, operationType) = PrivilegesBuilder.build(plan, spark) assertResult(plan.getClass.getName)( @@ -122,8 +124,8 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.DATABASE) assert(po.catalog.isEmpty) - assert(po.dbname === "default") - assert(po.objectName === "default") + assertEqualsIgnoreCase(defaultDb)(po.dbname) + assertEqualsIgnoreCase(defaultDb)(po.objectName) assert(po.columns.isEmpty) } @@ -143,25 +145,22 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite val (in, out, operationType) = PrivilegesBuilder.build(plan, spark) assert(operationType === ALTERTABLE_RENAME) assert(in.isEmpty) - assert(out.size === 2) + assert(out.size === 1) out.foreach { po => assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) assert(po.catalog.isEmpty) - assert(po.dbname equalsIgnoreCase reusedDb) - assert(Set(oldTableShort, "efg").contains(po.objectName)) + assertEqualsIgnoreCase(reusedDb)(po.dbname) + assertExistsIgnoreCase(po.objectName)(Set(oldTableShort, "efg")) assert(po.columns.isEmpty) val accessType = ranger.AccessType(po, operationType, isInput = false) - assert(Set(AccessType.CREATE, AccessType.DROP).contains(accessType)) - if (accessType == AccessType.DROP) { - checkTableOwner(po) - } + assert(accessType == AccessType.ALTER) } } } } test("CreateDatabaseCommand") { - assume(isSparkVersionAtMost("3.2")) + assume(SPARK_RUNTIME_VERSION <= "3.2") withDatabase("CreateDatabaseCommand") { db => val plan = sql(s"CREATE DATABASE $db").queryExecution.analyzed val (in, out, operationType) = PrivilegesBuilder.build(plan, spark) @@ -174,8 +173,8 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.DATABASE) assert(po.catalog.isEmpty) - assert(po.dbname === "CreateDatabaseCommand") - assert(po.objectName === "CreateDatabaseCommand") + assertEqualsIgnoreCase(db)(po.dbname) + assertEqualsIgnoreCase(db)(po.objectName) assert(po.columns.isEmpty) val accessType = ranger.AccessType(po, operationType, isInput = false) assert(accessType === AccessType.CREATE) @@ -183,7 +182,7 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite } test("DropDatabaseCommand") { - assume(isSparkVersionAtMost("3.2")) + assume(SPARK_RUNTIME_VERSION <= "3.2") withDatabase("DropDatabaseCommand") { db => sql(s"CREATE DATABASE $db") val plan = sql(s"DROP DATABASE DropDatabaseCommand").queryExecution.analyzed @@ -197,8 +196,8 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.DATABASE) assert(po.catalog.isEmpty) - assert(po.dbname === "DropDatabaseCommand") - assert(po.objectName === "DropDatabaseCommand") + assertEqualsIgnoreCase(db)(po.dbname) + assertEqualsIgnoreCase(db)(po.objectName) assert(po.columns.isEmpty) val accessType = ranger.AccessType(po, operationType, isInput = false) assert(accessType === AccessType.DROP) @@ -215,8 +214,8 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) assert(po.catalog.isEmpty) - assert(po.dbname equalsIgnoreCase reusedDb) - assert(po.objectName === reusedPartTableShort) + assertEqualsIgnoreCase(reusedDb)(po.dbname) + assertEqualsIgnoreCase(reusedPartTableShort)(po.objectName) assert(po.columns.head === "pid") checkTableOwner(po) val accessType = ranger.AccessType(po, operationType, isInput = false) @@ -233,8 +232,8 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) assert(po.catalog.isEmpty) - assert(po.dbname equalsIgnoreCase reusedDb) - assert(po.objectName === reusedPartTableShort) + assertEqualsIgnoreCase(reusedDb)(po.dbname) + assertEqualsIgnoreCase(reusedPartTableShort)(po.objectName) assert(po.columns.head === "pid") checkTableOwner(po) val accessType = ranger.AccessType(po, operationType, isInput = false) @@ -266,8 +265,8 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) assert(po.catalog.isEmpty) - assert(po.dbname equalsIgnoreCase reusedDb) - assert(po.objectName equalsIgnoreCase tableName.split("\\.").last) + assertEqualsIgnoreCase(reusedDb)(po.dbname) + assertEqualsIgnoreCase(tableName.split("\\.").last)(po.objectName) assert(po.columns.isEmpty) checkTableOwner(po) val accessType = ranger.AccessType(po, operationType, isInput = false) @@ -289,8 +288,8 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) assert(po.catalog.isEmpty) - assert(po.dbname equalsIgnoreCase reusedDb) - assert(po.objectName === reusedPartTableShort) + assertEqualsIgnoreCase(reusedDb)(po.dbname) + assertEqualsIgnoreCase(reusedPartTableShort)(po.objectName) assert(po.columns.head === "pid") checkTableOwner(po) val accessType = ranger.AccessType(po, operationType, isInput = false) @@ -312,8 +311,8 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) assert(po.catalog.isEmpty) - assert(po.dbname === reusedDb) - assert(po.objectName === reusedPartTableShort) + assertEqualsIgnoreCase(reusedDb)(po.dbname) + assertEqualsIgnoreCase(reusedPartTableShort)(po.objectName) assert(po.columns.head === "pid") checkTableOwner(po) val accessType = ranger.AccessType(po, operationType, isInput = false) @@ -334,8 +333,8 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) assert(po.catalog.isEmpty) - assert(po.dbname === reusedDb) - assert(po.objectName === reusedTable.split("\\.").last) + assertEqualsIgnoreCase(reusedDb)(po.dbname) + assertEqualsIgnoreCase(reusedTableShort)(po.objectName) assert(po.columns.isEmpty) checkTableOwner(po) val accessType = ranger.AccessType(po, operationType, isInput = false) @@ -353,8 +352,8 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite val po0 = in.head assert(po0.actionType === PrivilegeObjectActionType.OTHER) assert(po0.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po0.dbname equalsIgnoreCase reusedDb) - assert(po0.objectName equalsIgnoreCase reusedPartTableShort) + assertEqualsIgnoreCase(reusedDb)(po0.dbname) + assertEqualsIgnoreCase(reusedPartTableShort)(po0.objectName) if (isSparkV32OrGreater) { // Query in AlterViewAsCommand can not be resolved before SPARK-34698 assert(po0.columns === Seq("key", "value", "pid")) @@ -368,8 +367,8 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) assert(po.catalog.isEmpty) - assert(po.dbname === (if (isSparkV2) null else "default")) - assert(po.objectName === "AlterViewAsCommand") + assertEqualsIgnoreCase(defaultDb)(po.dbname) + assertEqualsIgnoreCase("AlterViewAsCommand")(po.objectName) checkTableOwner(po) assert(po.columns.isEmpty) val accessType = ranger.AccessType(po, operationType, isInput = false) @@ -380,41 +379,62 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite val plan = sql(s"ANALYZE TABLE $reusedPartTable PARTITION (pid=1)" + s" COMPUTE STATISTICS FOR COLUMNS key").queryExecution.analyzed val (in, out, operationType) = PrivilegesBuilder.build(plan, spark) - assert(operationType === ANALYZE_TABLE) + assert(operationType === ALTERTABLE_PROPERTIES) assert(in.size === 1) val po0 = in.head assert(po0.actionType === PrivilegeObjectActionType.OTHER) assert(po0.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po0.dbname equalsIgnoreCase reusedDb) - assert(po0.objectName equalsIgnoreCase reusedPartTableShort) + assertEqualsIgnoreCase(reusedDb)(po0.dbname) + assertEqualsIgnoreCase(reusedPartTableShort)(po0.objectName) // ignore this check as it behaves differently across spark versions assert(po0.columns === Seq("key")) checkTableOwner(po0) val accessType0 = ranger.AccessType(po0, operationType, isInput = true) - assert(accessType0 === AccessType.SELECT) + assert(accessType0 === AccessType.ALTER) + + assert(out.size === 1) + val po1 = out.head + assert(po1.actionType === PrivilegeObjectActionType.OTHER) + assert(po1.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) + assertEqualsIgnoreCase(reusedDb)(po1.dbname) + assertEqualsIgnoreCase(reusedPartTableShort)(po1.objectName) + // ignore this check as it behaves differently across spark versions + assert(po1.columns.isEmpty) + checkTableOwner(po1) + val accessType1 = ranger.AccessType(po1, operationType, isInput = true) + assert(accessType1 === AccessType.ALTER) - assert(out.size === 0) } test("AnalyzePartitionCommand") { val plan = sql(s"ANALYZE TABLE $reusedPartTable" + s" PARTITION (pid = 1) COMPUTE STATISTICS").queryExecution.analyzed val (in, out, operationType) = PrivilegesBuilder.build(plan, spark) - assert(operationType === ANALYZE_TABLE) + assert(operationType === ALTERTABLE_PROPERTIES) assert(in.size === 1) val po0 = in.head assert(po0.actionType === PrivilegeObjectActionType.OTHER) assert(po0.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po0.dbname equalsIgnoreCase reusedDb) - assert(po0.objectName equalsIgnoreCase reusedPartTableShort) + assertEqualsIgnoreCase(reusedDb)(po0.dbname) + assertEqualsIgnoreCase(reusedPartTableShort)(po0.objectName) // ignore this check as it behaves differently across spark versions assert(po0.columns === Seq("pid")) checkTableOwner(po0) val accessType0 = ranger.AccessType(po0, operationType, isInput = true) - assert(accessType0 === AccessType.SELECT) + assert(accessType0 === AccessType.ALTER) - assert(out.size === 0) + assert(out.size === 1) + val po1 = out.head + assert(po1.actionType === PrivilegeObjectActionType.OTHER) + assert(po1.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) + assertEqualsIgnoreCase(reusedDb)(po1.dbname) + assertEqualsIgnoreCase(reusedPartTableShort)(po1.objectName) + // ignore this check as it behaves differently across spark versions + assert(po1.columns.isEmpty) + checkTableOwner(po1) + val accessType1 = ranger.AccessType(po1, operationType, isInput = true) + assert(accessType1 === AccessType.ALTER) } test("AnalyzeTableCommand") { @@ -422,20 +442,30 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite .queryExecution.analyzed val (in, out, operationType) = PrivilegesBuilder.build(plan, spark) - assert(operationType === ANALYZE_TABLE) + assert(operationType === ALTERTABLE_PROPERTIES) assert(in.size === 1) val po0 = in.head assert(po0.actionType === PrivilegeObjectActionType.OTHER) assert(po0.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po0.dbname equalsIgnoreCase reusedDb) - assert(po0.objectName equalsIgnoreCase reusedPartTableShort) + assertEqualsIgnoreCase(reusedDb)(po0.dbname) + assertEqualsIgnoreCase(reusedPartTableShort)(po0.objectName) // ignore this check as it behaves differently across spark versions assert(po0.columns.isEmpty) checkTableOwner(po0) val accessType0 = ranger.AccessType(po0, operationType, isInput = true) - assert(accessType0 === AccessType.SELECT) + assert(accessType0 === AccessType.ALTER) - assert(out.size === 0) + assert(out.size === 1) + val po1 = out.head + assert(po1.actionType === PrivilegeObjectActionType.OTHER) + assert(po1.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) + assertEqualsIgnoreCase(reusedDb)(po1.dbname) + assertEqualsIgnoreCase(reusedPartTableShort)(po1.objectName) + // ignore this check as it behaves differently across spark versions + assert(po1.columns.isEmpty) + checkTableOwner(po1) + val accessType1 = ranger.AccessType(po1, operationType, isInput = true) + assert(accessType1 === AccessType.ALTER) } test("AnalyzeTablesCommand") { @@ -448,8 +478,8 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite val po0 = in.head assert(po0.actionType === PrivilegeObjectActionType.OTHER) assert(po0.privilegeObjectType === PrivilegeObjectType.DATABASE) - assert(po0.dbname equalsIgnoreCase reusedDb) - assert(po0.objectName equalsIgnoreCase reusedDb) + assertEqualsIgnoreCase(reusedDb)(po0.dbname) + assertEqualsIgnoreCase(reusedDb)(po0.objectName) // ignore this check as it behaves differently across spark versions assert(po0.columns.isEmpty) val accessType0 = ranger.AccessType(po0, operationType, isInput = true) @@ -466,8 +496,8 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite val po0 = in.head assert(po0.actionType === PrivilegeObjectActionType.OTHER) assert(po0.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po0.dbname equalsIgnoreCase reusedDb) - assert(po0.objectName equalsIgnoreCase reusedDb) + assertEqualsIgnoreCase(reusedDb)(po0.dbname) + assertEqualsIgnoreCase(reusedDb)(po0.objectName) assert(po0.columns.isEmpty) checkTableOwner(po0) val accessType0 = ranger.AccessType(po0, operationType, isInput = true) @@ -485,8 +515,8 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite val po0 = in.head assert(po0.actionType === PrivilegeObjectActionType.OTHER) assert(po0.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po0.dbname equalsIgnoreCase reusedDb) - assert(po0.objectName equalsIgnoreCase reusedTable.split("\\.").last) + assertEqualsIgnoreCase(reusedDb)(po0.dbname) + assertEqualsIgnoreCase(reusedTableShort)(po0.objectName) if (isSparkV32OrGreater) { assert(po0.columns.head === "key") checkTableOwner(po0) @@ -508,8 +538,8 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite val po0 = in.head assert(po0.actionType === PrivilegeObjectActionType.OTHER) assert(po0.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po0.dbname equalsIgnoreCase reusedDb) - assert(po0.objectName equalsIgnoreCase reusedTable.split("\\.").last) + assertEqualsIgnoreCase(reusedDb)(po0.dbname) + assertEqualsIgnoreCase(reusedTableShort)(po0.objectName) if (isSparkV32OrGreater) { assert(po0.columns === Seq("key", "value")) checkTableOwner(po0) @@ -524,8 +554,8 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) assert(po.catalog.isEmpty) - assert(po.dbname === (if (isSparkV2) null else "default")) - assert(po.objectName === "CreateViewCommand") + assertEqualsIgnoreCase(defaultDb)(po.dbname) + assertEqualsIgnoreCase("CreateViewCommand")(po.objectName) assert(po.columns.isEmpty) val accessType = ranger.AccessType(po, operationType, isInput = false) assert(accessType === AccessType.CREATE) @@ -544,8 +574,8 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) assert(po.catalog.isEmpty) - assert(po.dbname === (if (isSparkV2) null else "default")) - assert(po.objectName === tableName) + assertEqualsIgnoreCase(defaultDb)(po.dbname) + assertEqualsIgnoreCase(tableName)(po.objectName) assert(po.columns.isEmpty) val accessType = ranger.AccessType(po, operationType, isInput = false) assert(accessType === AccessType.CREATE) @@ -591,9 +621,9 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.FUNCTION) assert(po.catalog.isEmpty) - val db = if (isSparkV33OrGreater) "default" else null - assert(po.dbname === db) - assert(po.objectName === "CreateFunctionCommand") + val db = if (isSparkV33OrGreater) defaultDb else null + assertEqualsIgnoreCase(db)(po.dbname) + assertEqualsIgnoreCase("CreateFunctionCommand")(po.objectName) assert(po.columns.isEmpty) val accessType = ranger.AccessType(po, operationType, isInput = false) assert(accessType === AccessType.CREATE) @@ -623,16 +653,15 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.FUNCTION) assert(po.catalog.isEmpty) - val db = if (isSparkV33OrGreater) "default" else null - assert(po.dbname === db) - assert(po.objectName === "DropFunctionCommand") + val db = if (isSparkV33OrGreater) defaultDb else null + assertEqualsIgnoreCase(db)(po.dbname) + assertEqualsIgnoreCase("DropFunctionCommand")(po.objectName) assert(po.columns.isEmpty) val accessType = ranger.AccessType(po, operationType, isInput = false) assert(accessType === AccessType.DROP) } test("RefreshFunctionCommand") { - assume(AuthZUtils.isSparkVersionAtLeast("3.1")) sql(s"CREATE FUNCTION RefreshFunctionCommand AS '${getClass.getCanonicalName}'") val plan = sql("REFRESH FUNCTION RefreshFunctionCommand") .queryExecution.analyzed @@ -644,9 +673,9 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.FUNCTION) assert(po.catalog.isEmpty) - val db = if (isSparkV33OrGreater) "default" else null - assert(po.dbname === db) - assert(po.objectName === "RefreshFunctionCommand") + val db = if (isSparkV33OrGreater) defaultDb else null + assertEqualsIgnoreCase(db)(po.dbname) + assertEqualsIgnoreCase("RefreshFunctionCommand")(po.objectName) assert(po.columns.isEmpty) val accessType = ranger.AccessType(po, operationType, isInput = false) assert(accessType === AccessType.NONE) @@ -661,8 +690,8 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite val po0 = in.head assert(po0.actionType === PrivilegeObjectActionType.OTHER) assert(po0.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po0.dbname equalsIgnoreCase reusedDb) - assert(po0.objectName equalsIgnoreCase reusedTable.split("\\.").last) + assertEqualsIgnoreCase(reusedDb)(po0.dbname) + assertEqualsIgnoreCase(reusedTableShort)(po0.objectName) assert(po0.columns.isEmpty) checkTableOwner(po0) val accessType0 = ranger.AccessType(po0, operationType, isInput = true) @@ -673,8 +702,8 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) assert(po.catalog.isEmpty) - assert(po.dbname equalsIgnoreCase reusedDb) - assert(po.objectName === "CreateTableLikeCommand") + assertEqualsIgnoreCase(reusedDb)(po.dbname) + assertEqualsIgnoreCase("CreateTableLikeCommand")(po.objectName) assert(po.columns.isEmpty) val accessType = ranger.AccessType(po, operationType, isInput = false) assert(accessType === AccessType.CREATE) @@ -692,8 +721,8 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite val po0 = in.head assert(po0.actionType === PrivilegeObjectActionType.OTHER) assert(po0.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po0.dbname equalsIgnoreCase reusedDb) - assert(po0.objectName equalsIgnoreCase reusedTable.split("\\.").last) + assertEqualsIgnoreCase(reusedDb)(po0.dbname) + assertEqualsIgnoreCase(reusedTableShort)(po0.objectName) assert(po0.columns.isEmpty) checkTableOwner(po0) val accessType0 = ranger.AccessType(po0, operationType, isInput = true) @@ -704,8 +733,8 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) assert(po.catalog.isEmpty) - assert(po.dbname equalsIgnoreCase reusedDb) - assert(po.objectName === "CreateTableLikeCommandWithoutDatabase") + assertEqualsIgnoreCase(reusedDb)(po.dbname) + assertEqualsIgnoreCase("CreateTableLikeCommandWithoutDatabase")(po.objectName) assert(po.columns.isEmpty) val accessType = ranger.AccessType(po, operationType, isInput = false) assert(accessType === AccessType.CREATE) @@ -730,8 +759,8 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) assert(po.catalog.isEmpty) - assert(po.dbname equalsIgnoreCase reusedDb) - assert(po.objectName equalsIgnoreCase reusedTable.split("\\.").last) + assertEqualsIgnoreCase(reusedDb)(po.dbname) + assertEqualsIgnoreCase(reusedTableShort)(po.objectName) assert(po.columns === Seq("key")) checkTableOwner(po) val accessType = ranger.AccessType(po, operationType, isInput = false) @@ -749,8 +778,8 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) assert(po.catalog.isEmpty) - assert(po.dbname equalsIgnoreCase reusedDb) - assert(po.objectName equalsIgnoreCase reusedTable.split("\\.").last) + assertEqualsIgnoreCase(reusedDb)(po.dbname) + assertEqualsIgnoreCase(reusedTableShort)(po.objectName) assert(po.columns.isEmpty) checkTableOwner(po) val accessType = ranger.AccessType(po, operationType, isInput = false) @@ -760,7 +789,7 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite } test("DescribeDatabaseCommand") { - assume(isSparkVersionAtMost("3.2")) + assume(SPARK_RUNTIME_VERSION <= "3.2") val plan = sql(s"DESC DATABASE $reusedDb").queryExecution.analyzed val (in, out, operationType) = PrivilegesBuilder.build(plan, spark) assert(operationType === DESCDATABASE) @@ -769,8 +798,8 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.DATABASE) assert(po.catalog.isEmpty) - assert(po.dbname equalsIgnoreCase reusedDb) - assert(po.objectName equalsIgnoreCase reusedDb) + assertEqualsIgnoreCase(reusedDb)(po.dbname) + assertEqualsIgnoreCase(reusedDb)(po.objectName) assert(po.columns.isEmpty) val accessType = ranger.AccessType(po, operationType, isInput = false) assert(accessType === AccessType.USE) @@ -788,8 +817,8 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite val po0 = in.head assert(po0.actionType === PrivilegeObjectActionType.OTHER) assert(po0.privilegeObjectType === PrivilegeObjectType.DATABASE) - assert(po0.dbname equalsIgnoreCase reusedDb) - assert(po0.objectName equalsIgnoreCase reusedDb) + assertEqualsIgnoreCase(reusedDb)(po0.dbname) + assertEqualsIgnoreCase(reusedDb)(po0.objectName) assert(po0.columns.isEmpty) val accessType0 = ranger.AccessType(po0, operationType, isInput = false) assert(accessType0 === AccessType.USE) @@ -811,8 +840,8 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) assert(po.catalog.isEmpty) - assert(po.dbname equalsIgnoreCase reusedDb) - assert(po.objectName === reusedPartTableShort) + assertEqualsIgnoreCase(reusedDb)(po.dbname) + assertEqualsIgnoreCase(reusedPartTableShort)(po.objectName) assert(po.columns.head === "pid") checkTableOwner(po) val accessType = ranger.AccessType(po, operationType, isInput = false) @@ -827,8 +856,8 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite val po0 = in.head assert(po0.actionType === PrivilegeObjectActionType.OTHER) assert(po0.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po0.dbname equalsIgnoreCase reusedDb) - assert(po0.objectName equalsIgnoreCase reusedTable.split("\\.").last) + assertEqualsIgnoreCase(reusedDb)(po0.dbname) + assertEqualsIgnoreCase(reusedTableShort)(po0.objectName) assert(po0.columns.isEmpty) checkTableOwner(po0) val accessType0 = ranger.AccessType(po0, operationType, isInput = true) @@ -845,8 +874,8 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite val po0 = in.head assert(po0.actionType === PrivilegeObjectActionType.OTHER) assert(po0.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po0.dbname equalsIgnoreCase reusedDb) - assert(po0.objectName equalsIgnoreCase reusedTable.split("\\.").last) + assertEqualsIgnoreCase(reusedDb)(po0.dbname) + assertEqualsIgnoreCase(reusedTableShort)(po0.objectName) assert(po0.columns.isEmpty) checkTableOwner(po0) val accessType0 = ranger.AccessType(po0, operationType, isInput = true) @@ -863,8 +892,8 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite val po0 = in.head assert(po0.actionType === PrivilegeObjectActionType.OTHER) assert(po0.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po0.dbname equalsIgnoreCase reusedDb) - assert(po0.objectName equalsIgnoreCase reusedTable.split("\\.").last) + assertEqualsIgnoreCase(reusedDb)(po0.dbname) + assertEqualsIgnoreCase(reusedTableShort)(po0.objectName) assert(po0.columns.isEmpty) checkTableOwner(po0) val accessType0 = ranger.AccessType(po0, operationType, isInput = true) @@ -882,8 +911,8 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite val po0 = in.head assert(po0.actionType === PrivilegeObjectActionType.OTHER) assert(po0.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po0.dbname equalsIgnoreCase reusedDb) - assert(po0.objectName equalsIgnoreCase reusedPartTableShort) + assertEqualsIgnoreCase(reusedDb)(po0.dbname) + assertEqualsIgnoreCase(reusedPartTableShort)(po0.objectName) assert(po0.columns === Seq("pid")) checkTableOwner(po0) val accessType0 = ranger.AccessType(po0, operationType, isInput = true) @@ -918,8 +947,8 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) assert(po.catalog.isEmpty) - assert(po.dbname equalsIgnoreCase reusedDb) - assert(po.objectName equalsIgnoreCase tableName.split("\\.").last) + assertEqualsIgnoreCase(reusedDb)(po.dbname) + assertEqualsIgnoreCase(tableName.split("\\.").last)(po.objectName) assert(po.columns.isEmpty) checkTableOwner(po) val accessType = ranger.AccessType(po, operationType, isInput = false) @@ -934,8 +963,8 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) assert(po.catalog.isEmpty) - assert(po.dbname equalsIgnoreCase reusedDb) - assert(po.objectName equalsIgnoreCase reusedTableShort) + assertEqualsIgnoreCase(reusedDb)(po.dbname) + assertEqualsIgnoreCase(reusedTableShort)(po.objectName) assert(po.columns.take(2) === Seq("key", "value")) checkTableOwner(po) } @@ -959,7 +988,6 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite } test("Query: CTE") { - assume(!isSparkV2) checkColumns( s""" |with t(c) as (select coalesce(max(key), pid, 1) from $reusedPartTable group by pid) @@ -1010,8 +1038,8 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) assert(po.catalog.isEmpty) - assert(po.dbname equalsIgnoreCase reusedDb) - assert(po.objectName startsWith reusedTableShort.toLowerCase) + assertEqualsIgnoreCase(reusedDb)(po.dbname) + assertStartsWithIgnoreCase(reusedTableShort)(po.objectName) assert( po.columns === Seq("value", "pid", "key"), s"$reusedPartTable both 'key', 'value' and 'pid' should be authenticated") @@ -1037,8 +1065,8 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) assert(po.catalog.isEmpty) - assert(po.dbname equalsIgnoreCase reusedDb) - assert(po.objectName startsWith reusedTableShort.toLowerCase) + assertEqualsIgnoreCase(reusedDb)(po.dbname) + assertStartsWithIgnoreCase(reusedTableShort)(po.objectName) assert( po.columns === Seq("value", "key", "pid"), s"$reusedPartTable both 'key', 'value' and 'pid' should be authenticated") @@ -1067,8 +1095,8 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) assert(po.catalog.isEmpty) - assert(po.dbname equalsIgnoreCase reusedDb) - assert(po.objectName startsWith reusedTableShort.toLowerCase) + assertEqualsIgnoreCase(reusedDb)(po.dbname) + assertStartsWithIgnoreCase(reusedTableShort)(po.objectName) assert( po.columns === Seq("key", "value"), s"$reusedPartTable 'key' is the join key and 'pid' is omitted") @@ -1096,8 +1124,8 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) assert(po.catalog.isEmpty) - assert(po.dbname equalsIgnoreCase reusedDb) - assert(po.objectName startsWith reusedTableShort.toLowerCase) + assertEqualsIgnoreCase(reusedDb)(po.dbname) + assertStartsWithIgnoreCase(reusedTableShort)(po.objectName) assert( po.columns === Seq("key", "value"), s"$reusedPartTable both 'key' and 'value' should be authenticated") @@ -1126,8 +1154,8 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) assert(po.catalog.isEmpty) - assert(po.dbname equalsIgnoreCase reusedDb) - assert(po.objectName startsWith reusedTableShort.toLowerCase) + assertEqualsIgnoreCase(reusedDb)(po.dbname) + assertStartsWithIgnoreCase(reusedTableShort)(po.objectName) assert( po.columns === Seq("key", "value"), s"$reusedPartTable both 'key' and 'value' should be authenticated") @@ -1152,8 +1180,8 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) assert(po.catalog.isEmpty) - assert(po.dbname equalsIgnoreCase reusedDb) - assert(po.objectName startsWith reusedTableShort.toLowerCase) + assertEqualsIgnoreCase(reusedDb)(po.dbname) + assertStartsWithIgnoreCase(reusedTableShort)(po.objectName) assert( po.columns === Seq("key", "value"), s"$reusedPartTable both 'key' and 'value' should be authenticated") @@ -1178,8 +1206,8 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) assert(po.catalog.isEmpty) - assert(po.dbname equalsIgnoreCase reusedDb) - assert(po.objectName startsWith reusedTableShort.toLowerCase) + assertEqualsIgnoreCase(reusedDb)(po.dbname) + assertStartsWithIgnoreCase(reusedTableShort)(po.objectName) assert( po.columns === Seq("key", "value", "pid"), s"$reusedPartTable both 'key', 'value' and 'pid' should be authenticated") @@ -1222,8 +1250,8 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) assert(po.catalog.isEmpty) - assert(po.dbname equalsIgnoreCase reusedDb) - assert(po.objectName === getClass.getSimpleName) + assertEqualsIgnoreCase(reusedDb)(po.dbname) + assertEqualsIgnoreCase(reusedTableShort)(po.objectName) assert(po.columns.head === "a") checkTableOwner(po) val accessType = ranger.AccessType(po, operationType, isInput = false) @@ -1231,7 +1259,6 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite } test("AlterTableChangeColumnCommand") { - assume(!isSparkV2) val plan = sql(s"ALTER TABLE $reusedTable" + s" ALTER COLUMN value COMMENT 'alter column'").queryExecution.analyzed val (in, out, operationType) = PrivilegesBuilder.build(plan, spark) @@ -1242,8 +1269,8 @@ abstract class PrivilegesBuilderSuite extends AnyFunSuite assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) assert(po.catalog.isEmpty) - assert(po.dbname equalsIgnoreCase reusedDb) - assert(po.objectName === getClass.getSimpleName) + assertEqualsIgnoreCase(reusedDb)(po.dbname) + assertEqualsIgnoreCase(reusedTableShort)(po.objectName) assert(po.columns.head === "value") checkTableOwner(po) val accessType = ranger.AccessType(po, operationType, isInput = false) @@ -1256,7 +1283,7 @@ class InMemoryPrivilegeBuilderSuite extends PrivilegesBuilderSuite { // some hive version does not support set database location test("AlterDatabaseSetLocationCommand") { - assume(isSparkVersionAtMost("3.2")) + assume(SPARK_RUNTIME_VERSION <= "3.2") val newLoc = spark.conf.get("spark.sql.warehouse.dir") + "/new_db_location" val plan = sql(s"ALTER DATABASE default SET LOCATION '$newLoc'") .queryExecution.analyzed @@ -1270,8 +1297,8 @@ class InMemoryPrivilegeBuilderSuite extends PrivilegesBuilderSuite { assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.DATABASE) assert(po.catalog.isEmpty) - assert(po.dbname === "default") - assert(po.objectName === "default") + assertEqualsIgnoreCase(defaultDb)(po.dbname) + assertEqualsIgnoreCase(defaultDb)(po.objectName) assert(po.columns.isEmpty) val accessType = ranger.AccessType(po, operationType, isInput = false) assert(accessType === AccessType.ALTER) @@ -1287,8 +1314,8 @@ class InMemoryPrivilegeBuilderSuite extends PrivilegesBuilderSuite { val po0 = in.head assert(po0.actionType === PrivilegeObjectActionType.OTHER) assert(po0.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po0.dbname equalsIgnoreCase reusedDb) - assert(po0.objectName equalsIgnoreCase reusedTable.split("\\.").last) + assertEqualsIgnoreCase(reusedDb)(po0.dbname) + assertEqualsIgnoreCase(reusedTableShort)(po0.objectName) assert(po0.columns === Seq("key", "value")) checkTableOwner(po0) val accessType0 = ranger.AccessType(po0, operationType, isInput = true) @@ -1299,8 +1326,8 @@ class InMemoryPrivilegeBuilderSuite extends PrivilegesBuilderSuite { assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) assert(po.catalog.isEmpty) - assert(po.dbname === (if (isSparkV2) null else "default")) - assert(po.objectName === "CreateDataSourceTableAsSelectCommand") + assertEqualsIgnoreCase(defaultDb)(po.dbname) + assertEqualsIgnoreCase("CreateDataSourceTableAsSelectCommand")(po.objectName) if (catalogImpl == "hive") { assert(po.columns === Seq("key", "value")) } else { @@ -1313,10 +1340,9 @@ class InMemoryPrivilegeBuilderSuite extends PrivilegesBuilderSuite { class HiveCatalogPrivilegeBuilderSuite extends PrivilegesBuilderSuite { - override protected val catalogImpl: String = if (isSparkV2) "in-memory" else "hive" + override protected val catalogImpl: String = "hive" test("AlterTableSerDePropertiesCommand") { - assume(!isSparkV2) withTable("AlterTableSerDePropertiesCommand") { t => sql(s"CREATE TABLE $t (key int, pid int) USING hive PARTITIONED BY (pid)") sql(s"ALTER TABLE $t ADD IF NOT EXISTS PARTITION (pid=1)") @@ -1331,8 +1357,8 @@ class HiveCatalogPrivilegeBuilderSuite extends PrivilegesBuilderSuite { assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) assert(po.catalog.isEmpty) - assert(po.dbname === "default") - assert(po.objectName === t) + assertEqualsIgnoreCase(defaultDb)(po.dbname) + assertEqualsIgnoreCase(t)(po.objectName) assert(po.columns.head === "pid") checkTableOwner(po) val accessType = ranger.AccessType(po, operationType, isInput = false) @@ -1341,7 +1367,6 @@ class HiveCatalogPrivilegeBuilderSuite extends PrivilegesBuilderSuite { } test("CreateTableCommand") { - assume(!isSparkV2) withTable("CreateTableCommand") { _ => val plan = sql(s"CREATE TABLE CreateTableCommand(a int, b string) USING hive") .queryExecution.analyzed @@ -1353,8 +1378,8 @@ class HiveCatalogPrivilegeBuilderSuite extends PrivilegesBuilderSuite { assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) assert(po.catalog.isEmpty) - assert(po.dbname === "default") - assert(po.objectName === "CreateTableCommand") + assertEqualsIgnoreCase(defaultDb)(po.dbname) + assertEqualsIgnoreCase("CreateTableCommand")(po.objectName) assert(po.columns.isEmpty) val accessType = ranger.AccessType(po, operationType, isInput = false) assert(accessType === AccessType.CREATE) @@ -1362,7 +1387,6 @@ class HiveCatalogPrivilegeBuilderSuite extends PrivilegesBuilderSuite { } test("CreateHiveTableAsSelectCommand") { - assume(!isSparkV2) val plan = sql(s"CREATE TABLE CreateHiveTableAsSelectCommand USING hive" + s" AS SELECT key, value FROM $reusedTable") .queryExecution.analyzed @@ -1373,8 +1397,8 @@ class HiveCatalogPrivilegeBuilderSuite extends PrivilegesBuilderSuite { val po0 = in.head assert(po0.actionType === PrivilegeObjectActionType.OTHER) assert(po0.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po0.dbname equalsIgnoreCase reusedDb) - assert(po0.objectName equalsIgnoreCase reusedTable.split("\\.").last) + assertEqualsIgnoreCase(reusedDb)(po0.dbname) + assertEqualsIgnoreCase(reusedTableShort)(po0.objectName) assert(po0.columns === Seq("key", "value")) checkTableOwner(po0) val accessType0 = ranger.AccessType(po0, operationType, isInput = true) @@ -1385,15 +1409,14 @@ class HiveCatalogPrivilegeBuilderSuite extends PrivilegesBuilderSuite { assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) assert(po.catalog.isEmpty) - assert(po.dbname === "default") - assert(po.objectName === "CreateHiveTableAsSelectCommand") + assertEqualsIgnoreCase(defaultDb)(po.dbname) + assertEqualsIgnoreCase("CreateHiveTableAsSelectCommand")(po.objectName) assert(po.columns === Seq("key", "value")) val accessType = ranger.AccessType(po, operationType, isInput = false) assert(accessType === AccessType.CREATE) } test("LoadDataCommand") { - assume(!isSparkV2) val dataPath = getClass.getClassLoader.getResource("data.txt").getPath val tableName = reusedDb + "." + "LoadDataToTable" withTable(tableName) { _ => @@ -1413,7 +1436,7 @@ class HiveCatalogPrivilegeBuilderSuite extends PrivilegesBuilderSuite { val po0 = out.head assert(po0.actionType === PrivilegeObjectActionType.INSERT_OVERWRITE) assert(po0.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po0.dbname equalsIgnoreCase reusedDb) + assertEqualsIgnoreCase(reusedDb)(po0.dbname) assert(po0.objectName equalsIgnoreCase tableName.split("\\.").last) assert(po0.columns.isEmpty) checkTableOwner(po0) @@ -1423,12 +1446,11 @@ class HiveCatalogPrivilegeBuilderSuite extends PrivilegesBuilderSuite { } test("InsertIntoDatasourceDirCommand") { - assume(!isSparkV2) val tableDirectory = getClass.getResource("/").getPath + "table_directory" val directory = File(tableDirectory).createDirectory() val plan = sql( s""" - |INSERT OVERWRITE DIRECTORY '$directory.path' + |INSERT OVERWRITE DIRECTORY '${directory.path}' |USING parquet |SELECT * FROM $reusedPartTable""".stripMargin) .queryExecution.analyzed @@ -1438,7 +1460,7 @@ class HiveCatalogPrivilegeBuilderSuite extends PrivilegesBuilderSuite { val po0 = in.head assert(po0.actionType === PrivilegeObjectActionType.OTHER) assert(po0.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po0.dbname equalsIgnoreCase reusedDb) + assertEqualsIgnoreCase(reusedDb)(po0.dbname) assert(po0.objectName equalsIgnoreCase reusedPartTable.split("\\.").last) assert(po0.columns === Seq("key", "value", "pid")) checkTableOwner(po0) @@ -1449,7 +1471,6 @@ class HiveCatalogPrivilegeBuilderSuite extends PrivilegesBuilderSuite { } test("InsertIntoDataSourceCommand") { - assume(!isSparkV2) val tableName = "InsertIntoDataSourceTable" withTable(tableName) { _ => // sql(s"CREATE TABLE $tableName (a int, b string) USING parquet") @@ -1483,8 +1504,8 @@ class HiveCatalogPrivilegeBuilderSuite extends PrivilegesBuilderSuite { assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) assert(po.catalog.isEmpty) - assert(po.dbname equalsIgnoreCase reusedDb) - assert(po.objectName equalsIgnoreCase reusedTable.split("\\.").last) + assertEqualsIgnoreCase(reusedDb)(po.dbname) + assertEqualsIgnoreCase(reusedTableShort)(po.objectName) assert(po.columns === Seq("key", "value")) checkTableOwner(po) val accessType = ranger.AccessType(po, operationType, isInput = true) @@ -1496,8 +1517,8 @@ class HiveCatalogPrivilegeBuilderSuite extends PrivilegesBuilderSuite { assert(po.actionType === PrivilegeObjectActionType.INSERT) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) assert(po.catalog.isEmpty) - assert(po.dbname equalsIgnoreCase "default") - assert(po.objectName equalsIgnoreCase tableName) + assertEqualsIgnoreCase(defaultDb)(po.dbname) + assertEqualsIgnoreCase(tableName)(po.objectName) assert(po.columns.isEmpty) checkTableOwner(po) val accessType = ranger.AccessType(po, operationType, isInput = false) @@ -1508,7 +1529,6 @@ class HiveCatalogPrivilegeBuilderSuite extends PrivilegesBuilderSuite { } test("InsertIntoHadoopFsRelationCommand") { - assume(!isSparkV2) val tableName = "InsertIntoHadoopFsRelationTable" withTable(tableName) { _ => sql(s"CREATE TABLE $tableName (a int, b string) USING parquet") @@ -1526,8 +1546,8 @@ class HiveCatalogPrivilegeBuilderSuite extends PrivilegesBuilderSuite { assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) assert(po.catalog.isEmpty) - assert(po.dbname equalsIgnoreCase reusedDb) - assert(po.objectName equalsIgnoreCase reusedTable.split("\\.").last) + assertEqualsIgnoreCase(reusedDb)(po.dbname) + assertEqualsIgnoreCase(reusedTableShort)(po.objectName) assert(po.columns === Seq("key", "value")) checkTableOwner(po) val accessType = ranger.AccessType(po, operationType, isInput = false) @@ -1539,8 +1559,8 @@ class HiveCatalogPrivilegeBuilderSuite extends PrivilegesBuilderSuite { assert(po.actionType === PrivilegeObjectActionType.INSERT) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) assert(po.catalog.isEmpty) - assert(po.dbname equalsIgnoreCase "default") - assert(po.objectName equalsIgnoreCase tableName) + assertEqualsIgnoreCase(defaultDb)(po.dbname) + assertEqualsIgnoreCase(tableName)(po.objectName) assert(po.columns === Seq("a", "b")) checkTableOwner(po) val accessType = ranger.AccessType(po, operationType, isInput = false) @@ -1549,13 +1569,12 @@ class HiveCatalogPrivilegeBuilderSuite extends PrivilegesBuilderSuite { } } - test("InsertIntoHiveDirCommand") { - assume(!isSparkV2) + test("InsertIntoDataSourceDirCommand") { val tableDirectory = getClass.getResource("/").getPath + "table_directory" val directory = File(tableDirectory).createDirectory() val plan = sql( s""" - |INSERT OVERWRITE DIRECTORY '$directory.path' + |INSERT OVERWRITE DIRECTORY '${directory.path}' |USING parquet |SELECT * FROM $reusedPartTable""".stripMargin) .queryExecution.analyzed @@ -1565,7 +1584,32 @@ class HiveCatalogPrivilegeBuilderSuite extends PrivilegesBuilderSuite { val po0 = in.head assert(po0.actionType === PrivilegeObjectActionType.OTHER) assert(po0.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po0.dbname equalsIgnoreCase reusedDb) + assertEqualsIgnoreCase(reusedDb)(po0.dbname) + assert(po0.objectName equalsIgnoreCase reusedPartTable.split("\\.").last) + assert(po0.columns === Seq("key", "value", "pid")) + checkTableOwner(po0) + val accessType0 = ranger.AccessType(po0, operationType, isInput = true) + assert(accessType0 === AccessType.SELECT) + + assert(out.isEmpty) + } + + test("InsertIntoHiveDirCommand") { + val tableDirectory = getClass.getResource("/").getPath + "table_directory" + val directory = File(tableDirectory).createDirectory() + val plan = sql( + s""" + |INSERT OVERWRITE DIRECTORY '${directory.path}' + |ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' + |SELECT * FROM $reusedPartTable""".stripMargin) + .queryExecution.analyzed + val (in, out, operationType) = PrivilegesBuilder.build(plan, spark) + assert(operationType === QUERY) + assert(in.size === 1) + val po0 = in.head + assert(po0.actionType === PrivilegeObjectActionType.OTHER) + assert(po0.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) + assertEqualsIgnoreCase(reusedDb)(po0.dbname) assert(po0.objectName equalsIgnoreCase reusedPartTable.split("\\.").last) assert(po0.columns === Seq("key", "value", "pid")) checkTableOwner(po0) @@ -1576,7 +1620,6 @@ class HiveCatalogPrivilegeBuilderSuite extends PrivilegesBuilderSuite { } test("InsertIntoHiveTableCommand") { - assume(!isSparkV2) val tableName = "InsertIntoHiveTable" withTable(tableName) { _ => sql(s"CREATE TABLE $tableName (a int, b string) USING hive") @@ -1595,8 +1638,8 @@ class HiveCatalogPrivilegeBuilderSuite extends PrivilegesBuilderSuite { assert(po.actionType === PrivilegeObjectActionType.INSERT) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) assert(po.catalog.isEmpty) - assert(po.dbname equalsIgnoreCase "default") - assert(po.objectName equalsIgnoreCase tableName) + assertEqualsIgnoreCase(defaultDb)(po.dbname) + assertEqualsIgnoreCase(tableName)(po.objectName) assert(po.columns === Seq("a", "b")) checkTableOwner(po) val accessType = ranger.AccessType(po, operationType, isInput = false) @@ -1606,7 +1649,6 @@ class HiveCatalogPrivilegeBuilderSuite extends PrivilegesBuilderSuite { } test("ShowCreateTableAsSerdeCommand") { - assume(!isSparkV2) withTable("ShowCreateTableAsSerdeCommand") { t => sql(s"CREATE TABLE $t (key int, pid int) USING hive PARTITIONED BY (pid)") val plan = sql(s"SHOW CREATE TABLE $t AS SERDE").queryExecution.analyzed @@ -1616,8 +1658,8 @@ class HiveCatalogPrivilegeBuilderSuite extends PrivilegesBuilderSuite { val po0 = in.head assert(po0.actionType === PrivilegeObjectActionType.OTHER) assert(po0.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po0.dbname === "default") - assert(po0.objectName === t) + assertEqualsIgnoreCase(defaultDb)(po0.dbname) + assertEqualsIgnoreCase(t)(po0.objectName) assert(po0.columns.isEmpty) checkTableOwner(po0) val accessType0 = ranger.AccessType(po0, operationType, isInput = true) @@ -1628,7 +1670,6 @@ class HiveCatalogPrivilegeBuilderSuite extends PrivilegesBuilderSuite { } test("OptimizedCreateHiveTableAsSelectCommand") { - assume(!isSparkV2) val plan = sql( s"CREATE TABLE OptimizedCreateHiveTableAsSelectCommand STORED AS parquet AS SELECT 1 as a") .queryExecution.analyzed @@ -1642,12 +1683,54 @@ class HiveCatalogPrivilegeBuilderSuite extends PrivilegesBuilderSuite { assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) assert(po.catalog.isEmpty) - assert(po.dbname === "default") - assert(po.objectName === "OptimizedCreateHiveTableAsSelectCommand") + assertEqualsIgnoreCase(defaultDb)(po.dbname) + assertEqualsIgnoreCase("OptimizedCreateHiveTableAsSelectCommand")(po.objectName) assert(po.columns === Seq("a")) val accessType = ranger.AccessType(po, operationType, isInput = false) assert(accessType === AccessType.CREATE) } + + test("KYUUBI #4532: Displays the columns involved in extracting the aggregation operator") { + // case1: There is no project operator involving all columns. + val plan1 = sql(s"SELECT COUNT(key), MAX(value) FROM $reusedPartTable GROUP BY pid") + .queryExecution.optimizedPlan + val (in1, out1, _) = PrivilegesBuilder.build(plan1, spark) + assert(in1.size === 1) + assert(out1.isEmpty) + val pi1 = in1.head + assert(pi1.columns.size === 3) + assert(pi1.columns === Seq("key", "value", "pid")) + + // case2: Some columns are involved, and the group column is not selected. + val plan2 = sql(s"SELECT COUNT(key) FROM $reusedPartTable GROUP BY pid") + .queryExecution.optimizedPlan + val (in2, out2, _) = PrivilegesBuilder.build(plan2, spark) + assert(in2.size === 1) + assert(out2.isEmpty) + val pi2 = in2.head + assert(pi2.columns.size === 2) + assert(pi2.columns === Seq("key", "pid")) + + // case3: Some columns are involved, and the group column is selected. + val plan3 = sql(s"SELECT COUNT(key), pid FROM $reusedPartTable GROUP BY pid") + .queryExecution.optimizedPlan + val (in3, out3, _) = PrivilegesBuilder.build(plan3, spark) + assert(in3.size === 1) + assert(out3.isEmpty) + val pi3 = in3.head + assert(pi3.columns.size === 2) + assert(pi3.columns === Seq("key", "pid")) + + // case4: HAVING & GROUP clause + val plan4 = sql(s"SELECT COUNT(key) FROM $reusedPartTable GROUP BY pid HAVING MAX(key) > 1000") + .queryExecution.optimizedPlan + val (in4, out4, _) = PrivilegesBuilder.build(plan4, spark) + assert(in4.size === 1) + assert(out4.isEmpty) + val pi4 = in4.head + assert(pi4.columns.size === 2) + assert(pi4.columns === Seq("key", "pid")) + } } case class SimpleInsert(userSpecifiedSchema: StructType)(@transient val sparkSession: SparkSession) diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/RangerTestResources.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/RangerTestResources.scala new file mode 100644 index 000000000..4f870d504 --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/RangerTestResources.scala @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.plugin.spark.authz + +object RangerTestUsers { + // authorized users used in policy generation + val admin = "admin" + val alice = "alice" + val bob = "bob" + val kent = "kent" + val permViewUser = "perm_view_user" + val ownerPlaceHolder = "{OWNER}" + val createOnlyUser = "create_only_user" + val defaultTableOwner = "default_table_owner" + val permViewOnlyUser = "user_perm_view_only" + val table2OnlyUser = "user_table2_only" + + // non-authorized users + val invisibleUser = "i_am_invisible" + val denyUser = "denyuser" + val denyUser2 = "denyuser2" + val someone = "someone" +} + +object RangerTestNamespace { + val defaultDb = "default" + val sparkCatalog = "spark_catalog" + val icebergNamespace = "iceberg_ns" + val hudiNamespace = "hudi_ns" + val deltaNamespace = "delta_ns" + val namespace1 = "ns1" + val namespace2 = "ns2" +} diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/SparkSessionProvider.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/SparkSessionProvider.scala index 0ab88917b..7aa4d99e4 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/SparkSessionProvider.scala +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/SparkSessionProvider.scala @@ -22,29 +22,28 @@ import java.security.PrivilegedExceptionAction import org.apache.hadoop.security.UserGroupInformation import org.apache.spark.SparkConf -import org.apache.spark.sql.{DataFrame, SparkSession, SparkSessionExtensions} +import org.apache.spark.sql.{DataFrame, Row, SparkSession, SparkSessionExtensions} +import org.scalatest.Assertions._ import org.apache.kyuubi.Utils +import org.apache.kyuubi.plugin.spark.authz.RangerTestUsers._ +import org.apache.kyuubi.plugin.spark.authz.V2JdbcTableCatalogPrivilegesBuilderSuite._ +import org.apache.kyuubi.plugin.spark.authz.ranger.DeltaCatalogRangerSparkExtensionSuite._ import org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils._ trait SparkSessionProvider { protected val catalogImpl: String protected def format: String = if (catalogImpl == "hive") "hive" else "parquet" - protected val isSparkV2: Boolean = isSparkVersionAtMost("2.4") - protected val isSparkV31OrGreater: Boolean = isSparkVersionAtLeast("3.1") - protected val isSparkV32OrGreater: Boolean = isSparkVersionAtLeast("3.2") - protected val isSparkV33OrGreater: Boolean = isSparkVersionAtLeast("3.3") - protected val extension: SparkSessionExtensions => Unit = _ => Unit + protected val extension: SparkSessionExtensions => Unit = _ => () protected val sqlExtensions: String = "" - protected val defaultTableOwner = "default_table_owner" protected val extraSparkConf: SparkConf = new SparkConf() protected lazy val spark: SparkSession = { val metastore = { val path = Utils.createTempDir(prefix = "hms") - Files.delete(path) + Files.deleteIfExists(path) path } val ret = SparkSession.builder() @@ -71,4 +70,47 @@ trait SparkSessionProvider { protected val sql: String => DataFrame = spark.sql + protected def doAs[T](user: String, f: => T): T = { + UserGroupInformation.createRemoteUser(user).doAs[T]( + new PrivilegedExceptionAction[T] { + override def run(): T = f + }) + } + protected def withCleanTmpResources[T](res: Seq[(String, String)])(f: => T): T = { + try { + f + } finally { + res.foreach { + case (t, "table") => doAs( + admin, { + val purgeOption = + if (isSparkV32OrGreater && isCatalogSupportPurge( + spark.sessionState.catalogManager.currentCatalog.name())) { + "PURGE" + } else "" + sql(s"DROP TABLE IF EXISTS $t $purgeOption") + }) + case (db, "database") => doAs(admin, sql(s"DROP DATABASE IF EXISTS $db")) + case (fn, "function") => doAs(admin, sql(s"DROP FUNCTION IF EXISTS $fn")) + case (view, "view") => doAs(admin, sql(s"DROP VIEW IF EXISTS $view")) + case (cacheTable, "cache") => if (isSparkV32OrGreater) { + doAs(admin, sql(s"UNCACHE TABLE IF EXISTS $cacheTable")) + } + case (_, e) => + throw new RuntimeException(s"the resource whose resource type is $e cannot be cleared") + } + } + } + + protected def checkAnswer(user: String, query: String, result: Seq[Row]): Unit = { + doAs(user, assert(sql(query).collect() === result)) + } + + private def isCatalogSupportPurge(catalogName: String): Boolean = { + val unsupportedCatalogs = Set(v2JdbcTableCatalogClassName, deltaCatalogClassName) + spark.conf.getOption(s"spark.sql.catalog.$catalogName") match { + case Some(catalog) if !unsupportedCatalogs.contains(catalog) => true + case _ => false + } + } } diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/V2CommandsPrivilegesSuite.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/V2CommandsPrivilegesSuite.scala index 9d3e6d42d..149c9ba8f 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/V2CommandsPrivilegesSuite.scala +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/V2CommandsPrivilegesSuite.scala @@ -23,8 +23,11 @@ import org.apache.hadoop.security.UserGroupInformation import org.apache.spark.sql.execution.QueryExecution import org.apache.kyuubi.plugin.spark.authz.OperationType._ +import org.apache.kyuubi.plugin.spark.authz.RangerTestNamespace._ import org.apache.kyuubi.plugin.spark.authz.ranger.AccessType import org.apache.kyuubi.plugin.spark.authz.serde.{Database, DB_COMMAND_SPECS} +import org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils._ +import org.apache.kyuubi.util.AssertionUtils._ abstract class V2CommandsPrivilegesSuite extends PrivilegesBuilderSuite { @@ -99,9 +102,9 @@ abstract class V2CommandsPrivilegesSuite extends PrivilegesBuilderSuite { val po = outputs.head assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po.catalog === Some(catalogV2)) - assert(po.dbname === namespace) - assert(po.objectName === table) + assertEqualsIgnoreCase(Some(catalogV2))(po.catalog) + assertEqualsIgnoreCase(namespace)(po.dbname) + assertEqualsIgnoreCase(table)(po.objectName) assert(po.columns.isEmpty) assert(po.owner.isEmpty) val accessType = AccessType(po, operationType, isInput = false) @@ -121,9 +124,9 @@ abstract class V2CommandsPrivilegesSuite extends PrivilegesBuilderSuite { val po0 = inputs.head assert(po0.actionType === PrivilegeObjectActionType.OTHER) assert(po0.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po0.catalog === None) - assert(po0.dbname equalsIgnoreCase reusedDb) - assert(po0.objectName equalsIgnoreCase reusedTableShort) + assert(po0.catalog.isEmpty) + assertEqualsIgnoreCase(reusedDb)(po0.dbname) + assertEqualsIgnoreCase(reusedTableShort)(po0.objectName) assert(po0.columns.take(2) === Seq("key", "value")) checkTableOwner(po0) @@ -131,9 +134,9 @@ abstract class V2CommandsPrivilegesSuite extends PrivilegesBuilderSuite { val po = outputs.head assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po.catalog === Some(catalogV2)) - assert(po.dbname === namespace) - assert(po.objectName === table) + assertEqualsIgnoreCase(Some(catalogV2))(po.catalog) + assertEqualsIgnoreCase(namespace)(po.dbname) + assertEqualsIgnoreCase(table)(po.objectName) assert(po.columns.isEmpty) assert(po.owner.isEmpty) val accessType = AccessType(po, operationType, isInput = false) @@ -154,11 +157,15 @@ abstract class V2CommandsPrivilegesSuite extends PrivilegesBuilderSuite { val po = outputs.head assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po.catalog === Some(catalogV2)) - assert(po.dbname === namespace) - assert(po.objectName === table) + assertEqualsIgnoreCase(Some(catalogV2))(po.catalog) + assertEqualsIgnoreCase(namespace)(po.dbname) + assertEqualsIgnoreCase(table)(po.objectName) assert(po.columns.isEmpty) - assert(po.owner.isEmpty) + if (isSparkV34OrGreater) { + checkV2TableOwner(po) + } else { + assert(po.owner.isEmpty) + } val accessType = AccessType(po, operationType, isInput = false) assert(accessType === AccessType.CREATE) } @@ -176,9 +183,9 @@ abstract class V2CommandsPrivilegesSuite extends PrivilegesBuilderSuite { val po0 = inputs.head assert(po0.actionType === PrivilegeObjectActionType.OTHER) assert(po0.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po0.catalog === None) - assert(po0.dbname equalsIgnoreCase reusedDb) - assert(po0.objectName equalsIgnoreCase reusedTableShort) + assert(po0.catalog.isEmpty) + assertEqualsIgnoreCase(reusedDb)(po0.dbname) + assertEqualsIgnoreCase(reusedTableShort)(po0.objectName) assert(po0.columns.take(2) === Seq("key", "value")) checkTableOwner(po0) @@ -186,11 +193,15 @@ abstract class V2CommandsPrivilegesSuite extends PrivilegesBuilderSuite { val po = outputs.head assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po.catalog === Some(catalogV2)) - assert(po.dbname === namespace) - assert(po.objectName === table) + assertEqualsIgnoreCase(Some(catalogV2))(po.catalog) + assertEqualsIgnoreCase(namespace)(po.dbname) + assertEqualsIgnoreCase(table)(po.objectName) assert(po.columns.isEmpty) - assert(po.owner.isEmpty) + if (isSparkV34OrGreater) { + checkV2TableOwner(po) + } else { + assert(po.owner.isEmpty) + } val accessType = AccessType(po, operationType, isInput = false) assert(accessType === AccessType.CREATE) } @@ -207,9 +218,9 @@ abstract class V2CommandsPrivilegesSuite extends PrivilegesBuilderSuite { val po = outputs.head assert(po.actionType === PrivilegeObjectActionType.INSERT) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po.catalog === Some(catalogV2)) - assert(po.dbname === namespace) - assert(po.objectName === catalogTableShort) + assertEqualsIgnoreCase(Some(catalogV2))(po.catalog) + assertEqualsIgnoreCase(namespace)(po.dbname) + assertEqualsIgnoreCase(catalogTableShort)(po.objectName) assert(po.columns.isEmpty) checkV2TableOwner(po) val accessType = AccessType(po, operationType, isInput = false) @@ -229,9 +240,9 @@ abstract class V2CommandsPrivilegesSuite extends PrivilegesBuilderSuite { val po = outputs.head assert(po.actionType === PrivilegeObjectActionType.UPDATE) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po.catalog === Some(catalogV2)) - assert(po.dbname === namespace) - assert(po.objectName === catalogTableShort) + assertEqualsIgnoreCase(Some(catalogV2))(po.catalog) + assertEqualsIgnoreCase(namespace)(po.dbname) + assertEqualsIgnoreCase(catalogTableShort)(po.objectName) assert(po.columns.isEmpty) checkV2TableOwner(po) val accessType = AccessType(po, operationType, isInput = false) @@ -249,9 +260,9 @@ abstract class V2CommandsPrivilegesSuite extends PrivilegesBuilderSuite { val po = outputs.head assert(po.actionType === PrivilegeObjectActionType.UPDATE) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po.catalog === Some(catalogV2)) - assert(po.dbname === namespace) - assert(po.objectName === catalogTableShort) + assertEqualsIgnoreCase(Some(catalogV2))(po.catalog) + assertEqualsIgnoreCase(namespace)(po.dbname) + assertEqualsIgnoreCase(catalogTableShort)(po.objectName) assert(po.columns.isEmpty) checkV2TableOwner(po) val accessType = AccessType(po, operationType, isInput = false) @@ -267,9 +278,9 @@ abstract class V2CommandsPrivilegesSuite extends PrivilegesBuilderSuite { val po = outputs.head assert(po.actionType === PrivilegeObjectActionType.INSERT_OVERWRITE) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po.catalog === Some(catalogV2)) - assert(po.dbname === namespace) - assert(po.objectName === catalogTableShort) + assertEqualsIgnoreCase(Some(catalogV2))(po.catalog) + assertEqualsIgnoreCase(namespace)(po.dbname) + assertEqualsIgnoreCase(catalogTableShort)(po.objectName) assert(po.columns.isEmpty) checkV2TableOwner(po) val accessType = AccessType(po, operationType, isInput = false) @@ -290,9 +301,9 @@ abstract class V2CommandsPrivilegesSuite extends PrivilegesBuilderSuite { val po = outputs.head assert(po.actionType === PrivilegeObjectActionType.INSERT_OVERWRITE) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po.catalog === Some(catalogV2)) - assert(po.dbname === namespace) - assert(po.objectName === catalogPartTableShort) + assertEqualsIgnoreCase(Some(catalogV2))(po.catalog) + assertEqualsIgnoreCase(namespace)(po.dbname) + assertEqualsIgnoreCase(catalogPartTableShort)(po.objectName) assert(po.columns.isEmpty) checkV2TableOwner(po) val accessType = AccessType(po, operationType, isInput = false) @@ -315,9 +326,9 @@ abstract class V2CommandsPrivilegesSuite extends PrivilegesBuilderSuite { val po = outputs.head assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po.catalog === Some(catalogV2)) - assert(po.dbname === namespace) - assert(po.objectName === catalogPartTableShort) + assertEqualsIgnoreCase(Some(catalogV2))(po.catalog) + assertEqualsIgnoreCase(namespace)(po.dbname) + assertEqualsIgnoreCase(catalogPartTableShort)(po.objectName) assert(po.columns.isEmpty) checkV2TableOwner(po) val accessType = AccessType(po, operationType, isInput = false) @@ -337,9 +348,9 @@ abstract class V2CommandsPrivilegesSuite extends PrivilegesBuilderSuite { val po = outputs.head assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po.catalog === Some(catalogV2)) - assert(po.dbname === namespace) - assert(po.objectName === catalogPartTableShort) + assertEqualsIgnoreCase(Some(catalogV2))(po.catalog) + assertEqualsIgnoreCase(namespace)(po.dbname) + assertEqualsIgnoreCase(catalogPartTableShort)(po.objectName) assert(po.columns.isEmpty) checkV2TableOwner(po) val accessType = AccessType(po, operationType, isInput = false) @@ -359,9 +370,9 @@ abstract class V2CommandsPrivilegesSuite extends PrivilegesBuilderSuite { val po = outputs.head assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po.catalog === Some(catalogV2)) - assert(po.dbname === namespace) - assert(po.objectName === catalogPartTableShort) + assertEqualsIgnoreCase(Some(catalogV2))(po.catalog) + assertEqualsIgnoreCase(namespace)(po.dbname) + assertEqualsIgnoreCase(catalogPartTableShort)(po.objectName) assert(po.columns.isEmpty) checkV2TableOwner(po) val accessType = AccessType(po, operationType, isInput = false) @@ -382,9 +393,9 @@ abstract class V2CommandsPrivilegesSuite extends PrivilegesBuilderSuite { val po = outputs.head assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po.catalog === Some(catalogV2)) - assert(po.dbname === namespace) - assert(po.objectName === catalogPartTableShort) + assertEqualsIgnoreCase(Some(catalogV2))(po.catalog) + assertEqualsIgnoreCase(namespace)(po.dbname) + assertEqualsIgnoreCase(catalogPartTableShort)(po.objectName) assert(po.columns.isEmpty) checkV2TableOwner(po) val accessType = AccessType(po, operationType, isInput = false) @@ -403,9 +414,9 @@ abstract class V2CommandsPrivilegesSuite extends PrivilegesBuilderSuite { val po = outputs.head assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po.catalog === Some(catalogV2)) - assert(po.dbname === namespace) - assert(po.objectName === catalogTableShort) + assertEqualsIgnoreCase(Some(catalogV2))(po.catalog) + assertEqualsIgnoreCase(namespace)(po.dbname) + assertEqualsIgnoreCase(catalogTableShort)(po.objectName) assert(po.columns.isEmpty) checkV2TableOwner(po) val accessType = AccessType(po, operationType, isInput = false) @@ -425,9 +436,9 @@ abstract class V2CommandsPrivilegesSuite extends PrivilegesBuilderSuite { val po = outputs.head assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po.catalog === Some(catalogV2)) - assert(po.dbname === namespace) - assert(po.objectName === table) + assertEqualsIgnoreCase(Some(catalogV2))(po.catalog) + assertEqualsIgnoreCase(namespace)(po.dbname) + assertEqualsIgnoreCase(table)(po.objectName) assert(po.columns.isEmpty) checkV2TableOwner(po) val accessType = AccessType(po, operationType, isInput = false) @@ -452,9 +463,9 @@ abstract class V2CommandsPrivilegesSuite extends PrivilegesBuilderSuite { val po0 = inputs.head assert(po0.actionType === PrivilegeObjectActionType.OTHER) assert(po0.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po0.catalog === Some(catalogV2)) - assert(po0.dbname === namespace) - assert(po0.objectName === catalogTableShort) + assertEqualsIgnoreCase(Some(catalogV2))(po0.catalog) + assertEqualsIgnoreCase(namespace)(po0.dbname) + assertEqualsIgnoreCase(catalogTableShort)(po0.objectName) assert(po0.columns === Seq("key", "value")) checkV2TableOwner(po0) @@ -462,9 +473,9 @@ abstract class V2CommandsPrivilegesSuite extends PrivilegesBuilderSuite { val po = outputs.head assert(po.actionType === PrivilegeObjectActionType.UPDATE) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po.catalog === Some(catalogV2)) - assert(po.dbname === namespace) - assert(po.objectName === table) + assertEqualsIgnoreCase(Some(catalogV2))(po.catalog) + assertEqualsIgnoreCase(namespace)(po.dbname) + assertEqualsIgnoreCase(table)(po.objectName) assert(po.columns.isEmpty) checkV2TableOwner(po) val accessType = AccessType(po, operationType, isInput = false) @@ -485,9 +496,9 @@ abstract class V2CommandsPrivilegesSuite extends PrivilegesBuilderSuite { val po = outputs.head assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po.catalog === Some(catalogV2)) - assert(po.dbname === namespace) - assert(po.objectName === catalogPartTableShort) + assertEqualsIgnoreCase(Some(catalogV2))(po.catalog) + assertEqualsIgnoreCase(namespace)(po.dbname) + assertEqualsIgnoreCase(catalogPartTableShort)(po.objectName) assert(po.columns.isEmpty) checkV2TableOwner(po) val accessType = AccessType(po, operationType, isInput = false) @@ -506,15 +517,33 @@ abstract class V2CommandsPrivilegesSuite extends PrivilegesBuilderSuite { val po = outputs.head assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po.catalog === Some(catalogV2)) - assert(po.dbname === namespace) - assert(po.objectName === catalogTableShort) + assertEqualsIgnoreCase(Some(catalogV2))(po.catalog) + assertEqualsIgnoreCase(namespace)(po.dbname) + assertEqualsIgnoreCase(catalogTableShort)(po.objectName) assert(po.columns.isEmpty) checkV2TableOwner(po) val accessType = AccessType(po, operationType, isInput = false) assert(accessType === AccessType.UPDATE) } + test("DescribeTable") { + val plan = executePlan(s"DESCRIBE TABLE $catalogTable").analyzed + val (inputs, outputs, operationType) = PrivilegesBuilder.build(plan, spark) + assert(operationType === DESCTABLE) + assert(inputs.size === 1) + val po = inputs.head + assert(po.actionType === PrivilegeObjectActionType.OTHER) + assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) + assertEqualsIgnoreCase(Some(catalogV2))(po.catalog) + assertEqualsIgnoreCase(namespace)(po.dbname) + assertEqualsIgnoreCase(catalogTableShort)(po.objectName) + assert(po.columns.isEmpty) + checkV2TableOwner(po) + val accessType = AccessType(po, operationType, isInput = true) + assert(accessType === AccessType.SELECT) + assert(outputs.size === 0) + } + // with V2AlterTableCommand test("AddColumns") { @@ -532,9 +561,9 @@ abstract class V2CommandsPrivilegesSuite extends PrivilegesBuilderSuite { val po = outputs.head assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po.catalog === Some(catalogV2)) - assert(po.dbname === namespace) - assert(po.objectName === table) + assertEqualsIgnoreCase(Some(catalogV2))(po.catalog) + assertEqualsIgnoreCase(namespace)(po.dbname) + assertEqualsIgnoreCase(table)(po.objectName) assert(po.columns.isEmpty) checkV2TableOwner(po) val accessType = AccessType(po, operationType, isInput = false) @@ -557,9 +586,9 @@ abstract class V2CommandsPrivilegesSuite extends PrivilegesBuilderSuite { val po = outputs.head assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po.catalog === Some(catalogV2)) - assert(po.dbname === namespace) - assert(po.objectName === table) + assertEqualsIgnoreCase(Some(catalogV2))(po.catalog) + assertEqualsIgnoreCase(namespace)(po.dbname) + assertEqualsIgnoreCase(table)(po.objectName) assert(po.columns.isEmpty) checkV2TableOwner(po) val accessType = AccessType(po, operationType, isInput = false) @@ -582,9 +611,9 @@ abstract class V2CommandsPrivilegesSuite extends PrivilegesBuilderSuite { val po = outputs.head assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po.catalog === Some(catalogV2)) - assert(po.dbname === namespace) - assert(po.objectName === table) + assertEqualsIgnoreCase(Some(catalogV2))(po.catalog) + assertEqualsIgnoreCase(namespace)(po.dbname) + assertEqualsIgnoreCase(table)(po.objectName) assert(po.columns.isEmpty) checkV2TableOwner(po) val accessType = AccessType(po, operationType, isInput = false) @@ -607,9 +636,9 @@ abstract class V2CommandsPrivilegesSuite extends PrivilegesBuilderSuite { val po = outputs.head assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po.catalog === Some(catalogV2)) - assert(po.dbname === namespace) - assert(po.objectName === table) + assertEqualsIgnoreCase(Some(catalogV2))(po.catalog) + assertEqualsIgnoreCase(namespace)(po.dbname) + assertEqualsIgnoreCase(table)(po.objectName) assert(po.columns.isEmpty) checkV2TableOwner(po) val accessType = AccessType(po, operationType, isInput = false) @@ -632,9 +661,9 @@ abstract class V2CommandsPrivilegesSuite extends PrivilegesBuilderSuite { val po = outputs.head assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.TABLE_OR_VIEW) - assert(po.catalog === Some(catalogV2)) - assert(po.dbname === namespace) - assert(po.objectName === table) + assertEqualsIgnoreCase(Some(catalogV2))(po.catalog) + assertEqualsIgnoreCase(namespace)(po.dbname) + assertEqualsIgnoreCase(table)(po.objectName) assert(po.columns.isEmpty) checkV2TableOwner(po) val accessType = AccessType(po, operationType, isInput = false) @@ -649,7 +678,7 @@ abstract class V2CommandsPrivilegesSuite extends PrivilegesBuilderSuite { val spec = DB_COMMAND_SPECS(plan1.getClass.getName) var db: Database = null spec.databaseDescs.find { d => - Try(db = d.extract(plan1)).isSuccess + Try { db = d.extract(plan1) }.isSuccess } withClue(sql1) { assert(db.catalog === None) @@ -670,8 +699,8 @@ abstract class V2CommandsPrivilegesSuite extends PrivilegesBuilderSuite { assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.DATABASE) assert(po.catalog.get === sparkSessionCatalogName) - assert(po.dbname === "default") - assert(po.objectName === "default") + assertEqualsIgnoreCase(defaultDb)(po.dbname) + assertEqualsIgnoreCase(defaultDb)(po.objectName) assert(po.columns.isEmpty) } @@ -689,8 +718,8 @@ abstract class V2CommandsPrivilegesSuite extends PrivilegesBuilderSuite { assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.DATABASE) assert(po.catalog.get === sparkSessionCatalogName) - assert(po.dbname === "CreateNamespace") - assert(po.objectName === "CreateNamespace") + assertEqualsIgnoreCase("CreateNamespace")(po.dbname) + assertEqualsIgnoreCase("CreateNamespace")(po.objectName) assert(po.columns.isEmpty) val accessType = ranger.AccessType(po, operationType, isInput = false) assert(accessType === AccessType.CREATE) @@ -714,8 +743,8 @@ abstract class V2CommandsPrivilegesSuite extends PrivilegesBuilderSuite { assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.DATABASE) assert(po.catalog.get === sparkSessionCatalogName) - assert(po.dbname === "default") - assert(po.objectName === "default") + assertEqualsIgnoreCase(defaultDb)(po.dbname) + assertEqualsIgnoreCase(defaultDb)(po.objectName) assert(po.columns.isEmpty) val accessType = ranger.AccessType(po, operationType, isInput = false) assert(accessType === AccessType.ALTER) @@ -733,8 +762,8 @@ abstract class V2CommandsPrivilegesSuite extends PrivilegesBuilderSuite { assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.DATABASE) assert(po.catalog.get === sparkSessionCatalogName) - assert(po.dbname equalsIgnoreCase reusedDb) - assert(po.objectName equalsIgnoreCase reusedDb) + assertEqualsIgnoreCase(reusedDb)(po.dbname) + assertEqualsIgnoreCase(reusedDb)(po.objectName) assert(po.columns.isEmpty) val accessType = ranger.AccessType(po, operationType, isInput = false) assert(accessType === AccessType.USE) @@ -757,8 +786,8 @@ abstract class V2CommandsPrivilegesSuite extends PrivilegesBuilderSuite { assert(po.actionType === PrivilegeObjectActionType.OTHER) assert(po.privilegeObjectType === PrivilegeObjectType.DATABASE) assert(po.catalog.get === sparkSessionCatalogName) - assert(po.dbname === "DropNameSpace") - assert(po.objectName === "DropNameSpace") + assertEqualsIgnoreCase(db)(po.dbname) + assertEqualsIgnoreCase(db)(po.objectName) assert(po.columns.isEmpty) val accessType = ranger.AccessType(po, operationType, isInput = false) assert(accessType === AccessType.DROP) diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/V2JdbcTableCatalogPrivilegesBuilderSuite.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/V2JdbcTableCatalogPrivilegesBuilderSuite.scala index f85689406..d1a6f4ae8 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/V2JdbcTableCatalogPrivilegesBuilderSuite.scala +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/V2JdbcTableCatalogPrivilegesBuilderSuite.scala @@ -22,7 +22,9 @@ import scala.util.Try import org.scalatest.Outcome +import org.apache.kyuubi.plugin.spark.authz.V2JdbcTableCatalogPrivilegesBuilderSuite._ import org.apache.kyuubi.plugin.spark.authz.serde._ +import org.apache.kyuubi.util.AssertionUtils._ class V2JdbcTableCatalogPrivilegesBuilderSuite extends V2CommandsPrivilegesSuite { override protected val catalogImpl: String = "in-memory" @@ -37,15 +39,11 @@ class V2JdbcTableCatalogPrivilegesBuilderSuite extends V2CommandsPrivilegesSuite val jdbcUrl: String = s"$dbUrl;create=true" override def beforeAll(): Unit = { - if (isSparkV31OrGreater) { - spark.conf.set( - s"spark.sql.catalog.$catalogV2", - "org.apache.spark.sql.execution.datasources.v2.jdbc.JDBCTableCatalog") - spark.conf.set(s"spark.sql.catalog.$catalogV2.url", jdbcUrl) - spark.conf.set( - s"spark.sql.catalog.$catalogV2.driver", - "org.apache.derby.jdbc.AutoloadedDriver") - } + spark.conf.set(s"spark.sql.catalog.$catalogV2", v2JdbcTableCatalogClassName) + spark.conf.set(s"spark.sql.catalog.$catalogV2.url", jdbcUrl) + spark.conf.set( + s"spark.sql.catalog.$catalogV2.driver", + "org.apache.derby.jdbc.AutoloadedDriver") super.beforeAll() } @@ -59,7 +57,6 @@ class V2JdbcTableCatalogPrivilegesBuilderSuite extends V2CommandsPrivilegesSuite } override def withFixture(test: NoArgTest): Outcome = { - assume(isSparkV31OrGreater) test() } @@ -77,12 +74,12 @@ class V2JdbcTableCatalogPrivilegesBuilderSuite extends V2CommandsPrivilegesSuite val spec = TABLE_COMMAND_SPECS(plan.getClass.getName) var table: Table = null spec.tableDescs.find { d => - Try(table = d.extract(plan, spark).get).isSuccess + Try { table = d.extract(plan, spark).get }.isSuccess } withClue(str) { - assert(table.catalog === Some(catalogV2)) - assert(table.database === Some(ns1)) - assert(table.table === tbl) + assertEqualsIgnoreCase(Some(catalogV2))(table.catalog) + assertEqualsIgnoreCase(Some(ns1))(table.database) + assertEqualsIgnoreCase(tbl)(table.table) assert(table.owner.isEmpty) } } @@ -102,12 +99,12 @@ class V2JdbcTableCatalogPrivilegesBuilderSuite extends V2CommandsPrivilegesSuite val spec = TABLE_COMMAND_SPECS(plan.getClass.getName) var table: Table = null spec.tableDescs.find { d => - Try(table = d.extract(plan, spark).get).isSuccess + Try { table = d.extract(plan, spark).get }.isSuccess } withClue(sql1) { - assert(table.catalog === Some(catalogV2)) - assert(table.database === Some(ns1)) - assert(table.table === tbl) + assertEqualsIgnoreCase(Some(catalogV2))(table.catalog) + assertEqualsIgnoreCase(Some(ns1))(table.database) + assertEqualsIgnoreCase(tbl)(table.table) assert(table.owner.isEmpty) } } @@ -125,11 +122,11 @@ class V2JdbcTableCatalogPrivilegesBuilderSuite extends V2CommandsPrivilegesSuite val plan = executePlan(sql1).analyzed val spec = TABLE_COMMAND_SPECS(plan.getClass.getName) var table: Table = null - spec.tableDescs.find { d => Try(table = d.extract(plan, spark).get).isSuccess } + spec.tableDescs.find { d => Try { table = d.extract(plan, spark).get }.isSuccess } withClue(sql1) { - assert(table.catalog === Some(catalogV2)) - assert(table.database === Some(ns1)) - assert(table.table === tbl) + assertEqualsIgnoreCase(Some(catalogV2))(table.catalog) + assertEqualsIgnoreCase(Some(ns1))(table.database) + assertEqualsIgnoreCase(tbl)(table.table) assert(table.owner.isEmpty) } } @@ -144,11 +141,11 @@ class V2JdbcTableCatalogPrivilegesBuilderSuite extends V2CommandsPrivilegesSuite val spec = DB_COMMAND_SPECS(plan.getClass.getName) var db: Database = null spec.databaseDescs.find { d => - Try(db = d.extract(plan)).isSuccess + Try { db = d.extract(plan) }.isSuccess } withClue(sql) { - assert(db.catalog === Some(catalogV2)) - assert(db.database === ns1) + assertEqualsIgnoreCase(Some(catalogV2))(db.catalog) + assertEqualsIgnoreCase(ns1)(db.database) } } @@ -163,12 +160,17 @@ class V2JdbcTableCatalogPrivilegesBuilderSuite extends V2CommandsPrivilegesSuite val spec = DB_COMMAND_SPECS(plan.getClass.getName) var db: Database = null spec.databaseDescs.find { d => - Try(db = d.extract(plan)).isSuccess + Try { db = d.extract(plan) }.isSuccess } withClue(sql1) { - assert(db.catalog === Some(catalogV2)) - assert(db.database === ns1) + assertEqualsIgnoreCase(Some(catalogV2))(db.catalog) + assertEqualsIgnoreCase(ns1)(db.database) } } } } + +object V2JdbcTableCatalogPrivilegesBuilderSuite { + val v2JdbcTableCatalogClassName: String = + "org.apache.spark.sql.execution.datasources.v2.jdbc.JDBCTableCatalog" +} diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/DatabaseCommands.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/DatabaseCommands.scala index e947579e9..4436d9566 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/DatabaseCommands.scala +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/DatabaseCommands.scala @@ -20,7 +20,7 @@ package org.apache.kyuubi.plugin.spark.authz.gen import org.apache.kyuubi.plugin.spark.authz.OperationType._ import org.apache.kyuubi.plugin.spark.authz.serde._ -object DatabaseCommands { +object DatabaseCommands extends CommandSpecs[DatabaseCommandSpec] { val AlterDatabaseProperties = { DatabaseCommandSpec( @@ -58,9 +58,10 @@ object DatabaseCommands { "namespace", classOf[StringSeqDatabaseExtractor], catalogDesc = Some(CatalogDesc())) + val databaseDesc3 = DatabaseDesc("name", classOf[ResolvedNamespaceDatabaseExtractor]) DatabaseCommandSpec( "org.apache.spark.sql.catalyst.plans.logical.CreateNamespace", - Seq(databaseDesc1, databaseDesc2), + Seq(databaseDesc1, databaseDesc2, databaseDesc3), CREATEDATABASE) } @@ -97,12 +98,12 @@ object DatabaseCommands { val SetCatalogAndNamespace = { val cmd = "org.apache.spark.sql.catalyst.plans.logical.SetCatalogAndNamespace" - val databaseDesc1 = + val resolvedDbObjectDatabaseDesc = DatabaseDesc( "child", classOf[ResolvedDBObjectNameDatabaseExtractor], isInput = true) - val databaseDesc2 = + val stringSeqOptionDatabaseDesc = DatabaseDesc( "namespace", classOf[StringSeqOptionDatabaseExtractor], @@ -110,7 +111,15 @@ object DatabaseCommands { fieldName = "catalogName", fieldExtractor = classOf[StringOptionCatalogExtractor])), isInput = true) - DatabaseCommandSpec(cmd, Seq(databaseDesc1, databaseDesc2), SWITCHDATABASE) + val resolvedNamespaceDatabaseDesc = + DatabaseDesc( + "child", + classOf[ResolvedNamespaceDatabaseExtractor], + isInput = true) + DatabaseCommandSpec( + cmd, + Seq(resolvedNamespaceDatabaseDesc, resolvedDbObjectDatabaseDesc, stringSeqOptionDatabaseDesc), + SWITCHDATABASE) } val SetNamespace = { @@ -132,7 +141,7 @@ object DatabaseCommands { DatabaseCommandSpec(cmd, Seq(databaseDesc), DESCDATABASE) } - val data: Array[DatabaseCommandSpec] = Array( + override def specs: Seq[DatabaseCommandSpec] = Seq( AlterDatabaseProperties, AlterDatabaseProperties.copy( classname = "org.apache.spark.sql.execution.command.AlterDatabaseSetLocationCommand", diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/DeltaCommands.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/DeltaCommands.scala new file mode 100644 index 000000000..6435a64f5 --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/DeltaCommands.scala @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.plugin.spark.authz.gen + +import org.apache.kyuubi.plugin.spark.authz.OperationType._ +import org.apache.kyuubi.plugin.spark.authz.serde._ + +object DeltaCommands extends CommandSpecs[TableCommandSpec] { + + val CreateDeltaTableCommand = { + val cmd = "org.apache.spark.sql.delta.commands.CreateDeltaTableCommand" + val tableDesc = TableDesc("table", classOf[CatalogTableTableExtractor]) + TableCommandSpec(cmd, Seq(tableDesc), CREATETABLE) + } + + override def specs: Seq[TableCommandSpec] = Seq( + CreateDeltaTableCommand) +} diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/FunctionCommands.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/FunctionCommands.scala index 46c7f0efa..d5c849dd6 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/FunctionCommands.scala +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/FunctionCommands.scala @@ -21,7 +21,7 @@ import org.apache.kyuubi.plugin.spark.authz.OperationType._ import org.apache.kyuubi.plugin.spark.authz.serde._ import org.apache.kyuubi.plugin.spark.authz.serde.FunctionType.{SYSTEM, TEMP} -object FunctionCommands { +object FunctionCommands extends CommandSpecs[FunctionCommandSpec] { val CreateFunction = { val cmd = "org.apache.spark.sql.execution.command.CreateFunctionCommand" @@ -35,8 +35,12 @@ object FunctionCommands { "functionName", classOf[StringFunctionExtractor], Some(databaseDesc), - Some(functionTypeDesc)) - FunctionCommandSpec(cmd, Seq(functionDesc), CREATEFUNCTION) + functionTypeDesc = Some(functionTypeDesc)) + val functionIdentifierDesc = FunctionDesc( + "identifier", + classOf[FunctionIdentifierFunctionExtractor], + functionTypeDesc = Some(functionTypeDesc)) + FunctionCommandSpec(cmd, Seq(functionIdentifierDesc, functionDesc), CREATEFUNCTION) } val DescribeFunction = { @@ -79,9 +83,9 @@ object FunctionCommands { FunctionCommandSpec(cmd, Seq(functionDesc), RELOADFUNCTION) } - val data: Array[FunctionCommandSpec] = Array( + override def specs: Seq[FunctionCommandSpec] = Seq( CreateFunction, DropFunction, DescribeFunction, - RefreshFunction).sortBy(_.classname) + RefreshFunction) } diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/HudiCommands.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/HudiCommands.scala new file mode 100644 index 000000000..381f8081a --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/HudiCommands.scala @@ -0,0 +1,288 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.plugin.spark.authz.gen + +import org.apache.kyuubi.plugin.spark.authz.OperationType._ +import org.apache.kyuubi.plugin.spark.authz.PrivilegeObjectActionType._ +import org.apache.kyuubi.plugin.spark.authz.serde._ +import org.apache.kyuubi.plugin.spark.authz.serde.TableType._ + +object HudiCommands extends CommandSpecs[TableCommandSpec] { + val AlterHoodieTableAddColumnsCommand = { + val cmd = "org.apache.spark.sql.hudi.command.AlterHoodieTableAddColumnsCommand" + val columnDesc = ColumnDesc("colsToAdd", classOf[StructFieldSeqColumnExtractor]) + val tableDesc = TableDesc("tableId", classOf[TableIdentifierTableExtractor], Some(columnDesc)) + TableCommandSpec(cmd, Seq(tableDesc), ALTERTABLE_ADDCOLS) + } + + val AlterHoodieTableChangeColumnCommand = { + val cmd = "org.apache.spark.sql.hudi.command.AlterHoodieTableChangeColumnCommand" + val columnDesc = ColumnDesc("columnName", classOf[StringColumnExtractor]) + val tableDesc = + TableDesc("tableIdentifier", classOf[TableIdentifierTableExtractor], Some(columnDesc)) + TableCommandSpec(cmd, Seq(tableDesc), ALTERTABLE_REPLACECOLS) + } + + val AlterHoodieTableDropPartitionCommand = { + val cmd = "org.apache.spark.sql.hudi.command.AlterHoodieTableDropPartitionCommand" + val columnDesc = ColumnDesc("partitionSpecs", classOf[PartitionSeqColumnExtractor]) + val tableDesc = + TableDesc("tableIdentifier", classOf[TableIdentifierTableExtractor], Some(columnDesc)) + TableCommandSpec(cmd, Seq(tableDesc), ALTERTABLE_DROPPARTS) + } + + val AlterHoodieTableRenameCommand = { + val cmd = "org.apache.spark.sql.hudi.command.AlterHoodieTableRenameCommand" + val oldTableTableTypeDesc = + TableTypeDesc( + "oldName", + classOf[TableIdentifierTableTypeExtractor], + Seq(TEMP_VIEW)) + val oldTableD = TableDesc( + "oldName", + classOf[TableIdentifierTableExtractor], + tableTypeDesc = Some(oldTableTableTypeDesc)) + + TableCommandSpec(cmd, Seq(oldTableD), ALTERTABLE_RENAME) + } + + val AlterTableCommand = { + val cmd = "org.apache.spark.sql.hudi.command.AlterTableCommand" + val tableDesc = TableDesc("table", classOf[CatalogTableTableExtractor], None) + TableCommandSpec(cmd, Seq(tableDesc), ALTERTABLE_PROPERTIES) + } + + val Spark31AlterTableCommand = { + val cmd = "org.apache.spark.sql.hudi.command.Spark31AlterTableCommand" + val tableDesc = TableDesc("table", classOf[CatalogTableTableExtractor], None) + TableCommandSpec(cmd, Seq(tableDesc), ALTERTABLE_PROPERTIES) + } + + val CreateHoodieTableCommand = { + val cmd = "org.apache.spark.sql.hudi.command.CreateHoodieTableCommand" + val tableDesc = TableDesc("table", classOf[CatalogTableTableExtractor]) + TableCommandSpec(cmd, Seq(tableDesc), CREATETABLE) + } + + val CreateHoodieTableAsSelectCommand = { + val cmd = "org.apache.spark.sql.hudi.command.CreateHoodieTableAsSelectCommand" + CreateHoodieTableCommand.copy( + classname = cmd, + opType = CREATETABLE_AS_SELECT, + queryDescs = Seq(QueryDesc("query"))) + } + + val CreateHoodieTableLikeCommand = { + val cmd = "org.apache.spark.sql.hudi.command.CreateHoodieTableLikeCommand" + val tableDesc1 = TableDesc( + "targetTable", + classOf[TableIdentifierTableExtractor], + setCurrentDatabaseIfMissing = true) + val tableDesc2 = TableDesc( + "sourceTable", + classOf[TableIdentifierTableExtractor], + isInput = true, + setCurrentDatabaseIfMissing = true) + TableCommandSpec(cmd, Seq(tableDesc1, tableDesc2), CREATETABLE) + } + + val DropHoodieTableCommand = { + val cmd = "org.apache.spark.sql.hudi.command.DropHoodieTableCommand" + val tableTypeDesc = + TableTypeDesc( + "tableIdentifier", + classOf[TableIdentifierTableTypeExtractor], + Seq(TEMP_VIEW)) + TableCommandSpec( + cmd, + Seq(TableDesc( + "tableIdentifier", + classOf[TableIdentifierTableExtractor], + tableTypeDesc = Some(tableTypeDesc))), + DROPTABLE) + } + + val RepairHoodieTableCommand = { + val cmd = "org.apache.spark.sql.hudi.command.RepairHoodieTableCommand" + TableCommandSpec(cmd, Seq(TableDesc("tableName", classOf[TableIdentifierTableExtractor])), MSCK) + } + + val TruncateHoodieTableCommand = { + val cmd = "org.apache.spark.sql.hudi.command.TruncateHoodieTableCommand" + val columnDesc = ColumnDesc("partitionSpec", classOf[PartitionOptionColumnExtractor]) + val tableDesc = + TableDesc( + "tableIdentifier", + classOf[TableIdentifierTableExtractor], + columnDesc = Some(columnDesc)) + TableCommandSpec(cmd, Seq(tableDesc), TRUNCATETABLE) + } + + val CompactionHoodieTableCommand = { + val cmd = "org.apache.spark.sql.hudi.command.CompactionHoodieTableCommand" + val tableDesc = TableDesc("table", classOf[CatalogTableTableExtractor]) + TableCommandSpec(cmd, Seq(tableDesc), CREATETABLE) + } + + val CompactionShowHoodieTableCommand = { + val cmd = "org.apache.spark.sql.hudi.command.CompactionShowHoodieTableCommand" + val tableDesc = TableDesc("table", classOf[CatalogTableTableExtractor], isInput = true) + TableCommandSpec(cmd, Seq(tableDesc), SHOW_TBLPROPERTIES) + } + + val CompactionHoodiePathCommand = { + val cmd = "org.apache.spark.sql.hudi.command.CompactionHoodiePathCommand" + val uriDesc = UriDesc("path", classOf[StringURIExtractor]) + TableCommandSpec( + cmd, + Seq.empty, + CREATETABLE, + uriDescs = Seq(uriDesc)) + } + + val CompactionShowHoodiePathCommand = { + val cmd = "org.apache.spark.sql.hudi.command.CompactionShowHoodiePathCommand" + val uriDesc = UriDesc("path", classOf[StringURIExtractor], isInput = true) + TableCommandSpec(cmd, Seq.empty, SHOW_TBLPROPERTIES, uriDescs = Seq(uriDesc)) + } + + val CreateIndexCommand = { + val cmd = "org.apache.spark.sql.hudi.command.CreateIndexCommand" + val tableDesc = TableDesc("table", classOf[CatalogTableTableExtractor]) + TableCommandSpec(cmd, Seq(tableDesc), CREATEINDEX) + } + + val DropIndexCommand = { + val cmd = "org.apache.spark.sql.hudi.command.DropIndexCommand" + val tableDesc = TableDesc("table", classOf[CatalogTableTableExtractor]) + TableCommandSpec(cmd, Seq(tableDesc), DROPINDEX) + } + + val ShowIndexCommand = { + val cmd = "org.apache.spark.sql.hudi.command.ShowIndexesCommand" + val tableDesc = TableDesc("table", classOf[CatalogTableTableExtractor], isInput = true) + TableCommandSpec(cmd, Seq(tableDesc), SHOWINDEXES) + } + + val RefreshIndexCommand = { + val cmd = "org.apache.spark.sql.hudi.command.RefreshIndexCommand" + val tableDesc = TableDesc("table", classOf[CatalogTableTableExtractor]) + TableCommandSpec(cmd, Seq(tableDesc), ALTERINDEX_REBUILD) + } + + val InsertIntoHoodieTableCommand = { + val cmd = "org.apache.spark.sql.hudi.command.InsertIntoHoodieTableCommand" + val tableDesc = TableDesc( + "logicalRelation", + classOf[LogicalRelationTableExtractor], + actionTypeDesc = + Some(ActionTypeDesc("overwrite", classOf[OverwriteOrInsertActionTypeExtractor]))) + TableCommandSpec(cmd, Seq(tableDesc), queryDescs = Seq(QueryDesc("query"))) + } + + val ShowHoodieTablePartitionsCommand = { + val cmd = "org.apache.spark.sql.hudi.command.ShowHoodieTablePartitionsCommand" + val columnDesc = ColumnDesc("specOpt", classOf[PartitionOptionColumnExtractor]) + val tableDesc = TableDesc( + "tableIdentifier", + classOf[TableIdentifierTableExtractor], + isInput = true, + columnDesc = Some(columnDesc)) + TableCommandSpec(cmd, Seq(tableDesc), SHOWPARTITIONS) + } + + val DeleteHoodieTableCommand = { + val cmd = "org.apache.spark.sql.hudi.command.DeleteHoodieTableCommand" + val actionTypeDesc = ActionTypeDesc(actionType = Some(UPDATE)) + val tableDesc = + TableDesc( + "dft", + classOf[HudiDataSourceV2RelationTableExtractor], + actionTypeDesc = Some(actionTypeDesc)) + TableCommandSpec(cmd, Seq(tableDesc)) + } + + val UpdateHoodieTableCommand = { + val cmd = "org.apache.spark.sql.hudi.command.UpdateHoodieTableCommand" + val actionTypeDesc = ActionTypeDesc(actionType = Some(UPDATE)) + val tableDesc = + TableDesc( + "ut", + classOf[HudiDataSourceV2RelationTableExtractor], + actionTypeDesc = Some(actionTypeDesc)) + TableCommandSpec(cmd, Seq(tableDesc)) + } + + val MergeIntoHoodieTableCommand = { + val cmd = "org.apache.spark.sql.hudi.command.MergeIntoHoodieTableCommand" + val actionTypeDesc = ActionTypeDesc(actionType = Some(UPDATE)) + val tableDesc = + TableDesc( + "mergeInto", + classOf[HudiMergeIntoTargetTableExtractor], + actionTypeDesc = Some(actionTypeDesc)) + val queryDescs = QueryDesc("mergeInto", classOf[HudiMergeIntoSourceTableExtractor]) + TableCommandSpec(cmd, Seq(tableDesc), queryDescs = Seq(queryDescs)) + } + + val CallProcedureHoodieCommand = { + val cmd = "org.apache.spark.sql.hudi.command.CallProcedureHoodieCommand" + TableCommandSpec( + cmd, + Seq( + TableDesc( + "clone", + classOf[HudiCallProcedureInputTableExtractor], + actionTypeDesc = Some(ActionTypeDesc(actionType = Some(OTHER))), + isInput = true, + setCurrentDatabaseIfMissing = true), + TableDesc( + "clone", + classOf[HudiCallProcedureOutputTableExtractor], + actionTypeDesc = Some(ActionTypeDesc(actionType = Some(UPDATE))), + setCurrentDatabaseIfMissing = true))) + } + + override def specs: Seq[TableCommandSpec] = Seq( + AlterHoodieTableAddColumnsCommand, + AlterHoodieTableChangeColumnCommand, + AlterHoodieTableDropPartitionCommand, + AlterHoodieTableRenameCommand, + AlterTableCommand, + CallProcedureHoodieCommand, + CreateHoodieTableAsSelectCommand, + CreateHoodieTableCommand, + CreateHoodieTableLikeCommand, + CreateIndexCommand, + CompactionHoodiePathCommand, + CompactionHoodieTableCommand, + CompactionShowHoodiePathCommand, + CompactionShowHoodieTableCommand, + DeleteHoodieTableCommand, + DropHoodieTableCommand, + DropIndexCommand, + InsertIntoHoodieTableCommand, + MergeIntoHoodieTableCommand, + RefreshIndexCommand, + RepairHoodieTableCommand, + ShowIndexCommand, + TruncateHoodieTableCommand, + ShowHoodieTablePartitionsCommand, + Spark31AlterTableCommand, + UpdateHoodieTableCommand) +} diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/IcebergCommands.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/IcebergCommands.scala index 208e73c51..59f8eb7a6 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/IcebergCommands.scala +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/IcebergCommands.scala @@ -17,10 +17,11 @@ package org.apache.kyuubi.plugin.spark.authz.gen +import org.apache.kyuubi.plugin.spark.authz.OperationType import org.apache.kyuubi.plugin.spark.authz.PrivilegeObjectActionType._ import org.apache.kyuubi.plugin.spark.authz.serde._ -object IcebergCommands { +object IcebergCommands extends CommandSpecs[TableCommandSpec] { val DeleteFromIcebergTable = { val cmd = "org.apache.spark.sql.catalyst.plans.logical.DeleteFromIcebergTable" @@ -30,7 +31,7 @@ object IcebergCommands { "table", classOf[DataSourceV2RelationTableExtractor], actionTypeDesc = Some(actionTypeDesc)) - TableCommandSpec(cmd, Seq(tableDesc), queryDescs = Seq(QueryDesc("query"))) + TableCommandSpec(cmd, Seq(tableDesc)) } val UpdateIcebergTable = { @@ -49,7 +50,14 @@ object IcebergCommands { TableCommandSpec(cmd, Seq(tableDesc), queryDescs = Seq(queryDesc)) } - val data: Array[TableCommandSpec] = Array( + val CallProcedure = { + val cmd = "org.apache.spark.sql.catalyst.plans.logical.Call" + val td = TableDesc("args", classOf[ExpressionSeqTableExtractor]) + TableCommandSpec(cmd, Seq(td), opType = OperationType.ALTERTABLE_PROPERTIES) + } + + override def specs: Seq[TableCommandSpec] = Seq( + CallProcedure, DeleteFromIcebergTable, UpdateIcebergTable, MergeIntoIcebergTable, diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/JsonSpecFileGenerator.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/JsonSpecFileGenerator.scala index 7c7ed138b..5fb4ace10 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/JsonSpecFileGenerator.scala +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/JsonSpecFileGenerator.scala @@ -18,37 +18,67 @@ package org.apache.kyuubi.plugin.spark.authz.gen import java.nio.charset.StandardCharsets -import java.nio.file.{Files, Paths} +import java.nio.file.{Files, Paths, StandardOpenOption} + +//scalastyle:off +import org.scalatest.funsuite.AnyFunSuite import org.apache.kyuubi.plugin.spark.authz.serde.{mapper, CommandSpec} +import org.apache.kyuubi.plugin.spark.authz.serde.CommandSpecs +import org.apache.kyuubi.util.AssertionUtils._ /** * Generates the default command specs to src/main/resources dir. * - * Usage: - * mvn scala:run -DmainClass=this class -pl :kyuubi-spark-authz_2.12 + * To run the test suite: + * {{{ + * KYUUBI_UPDATE=0 dev/gen/gen_ranger_spec_json.sh + * }}} + * + * To regenerate the ranger policy file: + * {{{ + * dev/gen/gen_ranger_spec_json.sh + * }}} */ -object JsonSpecFileGenerator { - - def main(args: Array[String]): Unit = { - writeCommandSpecJson("database", DatabaseCommands.data) - writeCommandSpecJson("table", TableCommands.data ++ IcebergCommands.data) - writeCommandSpecJson("function", FunctionCommands.data) - writeCommandSpecJson("scan", Scans.data) +class JsonSpecFileGenerator extends AnyFunSuite { + // scalastyle:on + test("check spec json files") { + writeCommandSpecJson("database", Seq(DatabaseCommands)) + writeCommandSpecJson("table", Seq(TableCommands, IcebergCommands, HudiCommands, DeltaCommands)) + writeCommandSpecJson("function", Seq(FunctionCommands)) + writeCommandSpecJson("scan", Seq(Scans)) } - def writeCommandSpecJson[T <: CommandSpec](commandType: String, specArr: Array[T]): Unit = { + def writeCommandSpecJson[T <: CommandSpec]( + commandType: String, + specsArr: Seq[CommandSpecs[T]]): Unit = { val pluginHome = getClass.getProtectionDomain.getCodeSource.getLocation.getPath .split("target").head val filename = s"${commandType}_command_spec.json" - val writer = { - val p = Paths.get(pluginHome, "src", "main", "resources", filename) - Files.newBufferedWriter(p, StandardCharsets.UTF_8) + val filePath = Paths.get(pluginHome, "src", "main", "resources", filename) + + val allSpecs = specsArr.flatMap(_.specs.sortBy(_.classname)) + val duplicatedClassnames = allSpecs.groupBy(_.classname).values + .filter(_.size > 1).flatMap(specs => specs.map(_.classname)).toSet + withClue(s"Unexpected duplicated classnames: $duplicatedClassnames")( + assertResult(0)(duplicatedClassnames.size)) + val generatedStr = mapper.writerWithDefaultPrettyPrinter().writeValueAsString(allSpecs) + + if (sys.env.get("KYUUBI_UPDATE").contains("1")) { + // scalastyle:off println + println(s"writing ${allSpecs.length} specs to $filename") + // scalastyle:on println + Files.write( + filePath, + generatedStr.getBytes(StandardCharsets.UTF_8), + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING) + } else { + assertFileContent( + filePath, + Seq(generatedStr), + "dev/gen/gen_ranger_spec_json.sh", + splitFirstExpectedLine = true) } - // scalastyle:off println - println(s"writing ${specArr.length} specs to $filename") - // scalastyle:on println - mapper.writerWithDefaultPrettyPrinter().writeValue(writer, specArr.sortBy(_.classname)) - writer.close() } } diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/Scans.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/Scans.scala index 7bd8260bb..7771a2dd2 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/Scans.scala +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/Scans.scala @@ -18,8 +18,9 @@ package org.apache.kyuubi.plugin.spark.authz.gen import org.apache.kyuubi.plugin.spark.authz.serde._ +import org.apache.kyuubi.plugin.spark.authz.serde.FunctionType._ -object Scans { +object Scans extends CommandSpecs[ScanSpec] { val HiveTableRelation = { val r = "org.apache.spark.sql.catalyst.catalog.HiveTableRelation" @@ -49,7 +50,7 @@ object Scans { } val PermanentViewMarker = { - val r = "org.apache.kyuubi.plugin.spark.authz.util.PermanentViewMarker" + val r = "org.apache.kyuubi.plugin.spark.authz.rule.permanentview.PermanentViewMarker" val tableDesc = ScanDesc( "catalogTable", @@ -57,9 +58,34 @@ object Scans { ScanSpec(r, Seq(tableDesc)) } - val data: Array[ScanSpec] = Array( + val HiveSimpleUDF = { + ScanSpec( + "org.apache.spark.sql.hive.HiveSimpleUDF", + Seq.empty, + Seq(FunctionDesc( + "name", + classOf[QualifiedNameStringFunctionExtractor], + functionTypeDesc = Some(FunctionTypeDesc( + "name", + classOf[FunctionNameFunctionTypeExtractor], + Seq(TEMP, SYSTEM))), + isInput = true))) + } + + val HiveGenericUDF = HiveSimpleUDF.copy(classname = "org.apache.spark.sql.hive.HiveGenericUDF") + + val HiveUDAFFunction = HiveSimpleUDF.copy(classname = + "org.apache.spark.sql.hive.HiveUDAFFunction") + + val HiveGenericUDTF = HiveSimpleUDF.copy(classname = "org.apache.spark.sql.hive.HiveGenericUDTF") + + override def specs: Seq[ScanSpec] = Seq( HiveTableRelation, LogicalRelation, DataSourceV2Relation, - PermanentViewMarker) + PermanentViewMarker, + HiveSimpleUDF, + HiveGenericUDF, + HiveUDAFFunction, + HiveGenericUDTF) } diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/TableCommands.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/TableCommands.scala index ef981515a..e397ba487 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/TableCommands.scala +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/gen/TableCommands.scala @@ -22,7 +22,7 @@ import org.apache.kyuubi.plugin.spark.authz.PrivilegeObjectActionType._ import org.apache.kyuubi.plugin.spark.authz.serde._ import org.apache.kyuubi.plugin.spark.authz.serde.TableType._ -object TableCommands { +object TableCommands extends CommandSpecs[TableCommandSpec] { // table extractors val tite = classOf[TableIdentifierTableExtractor] val tableNameDesc = TableDesc("tableName", tite) @@ -30,6 +30,8 @@ object TableCommands { val resolvedTableDesc = TableDesc("child", classOf[ResolvedTableTableExtractor]) val resolvedDbObjectNameDesc = TableDesc("child", classOf[ResolvedDbObjectNameTableExtractor]) + val resolvedIdentifierTableDesc = + TableDesc("child", classOf[ResolvedIdentifierTableExtractor]) val overwriteActionTypeDesc = ActionTypeDesc("overwrite", classOf[OverwriteOrInsertActionTypeExtractor]) val queryQueryDesc = QueryDesc("query") @@ -102,7 +104,6 @@ object TableCommands { val AlterTableRename = { val cmd = "org.apache.spark.sql.execution.command.AlterTableRenameCommand" - val actionTypeDesc = ActionTypeDesc(actionType = Some(DELETE)) val oldTableTableTypeDesc = TableTypeDesc( @@ -112,12 +113,9 @@ object TableCommands { val oldTableD = TableDesc( "oldName", tite, - tableTypeDesc = Some(oldTableTableTypeDesc), - actionTypeDesc = Some(actionTypeDesc)) + tableTypeDesc = Some(oldTableTableTypeDesc)) - val newTableD = - TableDesc("newName", tite, tableTypeDesc = Some(oldTableTableTypeDesc)) - TableCommandSpec(cmd, Seq(oldTableD, newTableD), ALTERTABLE_RENAME) + TableCommandSpec(cmd, Seq(oldTableD), ALTERTABLE_RENAME) } // this is for spark 3.1 or below @@ -183,7 +181,8 @@ object TableCommands { val cd2 = cd1.copy(fieldExtractor = classOf[StringSeqOptionColumnExtractor]) val td1 = tableIdentDesc.copy(columnDesc = Some(cd1), isInput = true) val td2 = td1.copy(columnDesc = Some(cd2)) - TableCommandSpec(cmd, Seq(td1, td2), ANALYZE_TABLE) + // AnalyzeColumn will update table properties, here we use ALTERTABLE_PROPERTIES + TableCommandSpec(cmd, Seq(tableIdentDesc, td1, td2), ALTERTABLE_PROPERTIES) } val AnalyzePartition = { @@ -191,16 +190,18 @@ object TableCommands { val columnDesc = ColumnDesc("partitionSpec", classOf[PartitionColumnExtractor]) TableCommandSpec( cmd, - Seq(tableIdentDesc.copy(columnDesc = Some(columnDesc), isInput = true)), - ANALYZE_TABLE) + // AnalyzePartition will update table properties, here we use ALTERTABLE_PROPERTIES + Seq(tableIdentDesc, tableIdentDesc.copy(columnDesc = Some(columnDesc), isInput = true)), + ALTERTABLE_PROPERTIES) } val AnalyzeTable = { val cmd = "org.apache.spark.sql.execution.command.AnalyzeTableCommand" TableCommandSpec( cmd, - Seq(tableIdentDesc.copy(isInput = true)), - ANALYZE_TABLE) + // AnalyzeTable will update table properties, here we use ALTERTABLE_PROPERTIES + Seq(tableIdentDesc, tableIdentDesc.copy(isInput = true)), + ALTERTABLE_PROPERTIES) } val CreateTableV2 = { @@ -209,7 +210,10 @@ object TableCommands { "tableName", classOf[IdentifierTableExtractor], catalogDesc = Some(CatalogDesc())) - TableCommandSpec(cmd, Seq(tableDesc, resolvedDbObjectNameDesc), CREATETABLE) + TableCommandSpec( + cmd, + Seq(resolvedIdentifierTableDesc, tableDesc, resolvedDbObjectNameDesc), + CREATETABLE) } val CreateV2Table = { @@ -229,7 +233,10 @@ object TableCommands { catalogDesc = Some(CatalogDesc())) TableCommandSpec( cmd, - Seq(tableDesc, resolvedDbObjectNameDesc.copy(fieldName = "left")), + Seq( + resolvedIdentifierTableDesc.copy(fieldName = "name"), + tableDesc, + resolvedDbObjectNameDesc.copy(fieldName = "name")), CREATETABLE_AS_SELECT, Seq(queryQueryDesc)) } @@ -250,6 +257,17 @@ object TableCommands { TableCommandSpec(cmd, Seq(tableDesc), queryDescs = Seq(queryQueryDesc)) } + val ReplaceData = { + val cmd = "org.apache.spark.sql.catalyst.plans.logical.ReplaceData" + val actionTypeDesc = ActionTypeDesc(actionType = Some(UPDATE)) + val tableDesc = + TableDesc( + "originalTable", + classOf[DataSourceV2RelationTableExtractor], + actionTypeDesc = Some(actionTypeDesc)) + TableCommandSpec(cmd, Seq(tableDesc), queryDescs = Seq(queryQueryDesc)) + } + val UpdateTable = { val cmd = "org.apache.spark.sql.catalyst.plans.logical.UpdateTable" val actionTypeDesc = ActionTypeDesc(actionType = Some(UPDATE)) @@ -258,7 +276,7 @@ object TableCommands { "table", classOf[DataSourceV2RelationTableExtractor], actionTypeDesc = Some(actionTypeDesc)) - TableCommandSpec(cmd, Seq(tableDesc), queryDescs = Seq(queryQueryDesc)) + TableCommandSpec(cmd, Seq(tableDesc)) } val DeleteFromTable = { @@ -350,9 +368,17 @@ object TableCommands { TableCommandSpec(cmd, Nil, CREATEVIEW) } + val CreateTable = { + val cmd = "org.apache.spark.sql.execution.datasources.CreateTable" + val tableDesc = TableDesc("tableDesc", classOf[CatalogTableTableExtractor]) + val queryDesc = QueryDesc("query", "LogicalPlanOptionQueryExtractor") + TableCommandSpec(cmd, Seq(tableDesc), CREATETABLE, queryDescs = Seq(queryDesc)) + } + val CreateDataSourceTable = { val cmd = "org.apache.spark.sql.execution.command.CreateDataSourceTableCommand" - val tableDesc = TableDesc("table", classOf[CatalogTableTableExtractor]) + val tableDesc = + TableDesc("table", classOf[CatalogTableTableExtractor], setCurrentDatabaseIfMissing = true) TableCommandSpec(cmd, Seq(tableDesc), CREATETABLE) } @@ -410,6 +436,16 @@ object TableCommands { TableCommandSpec(cmd, Seq(tableDesc), DESCTABLE) } + val DescribeRelationTable = { + val cmd = "org.apache.spark.sql.catalyst.plans.logical.DescribeRelation" + val tableDesc = TableDesc( + "relation", + classOf[ResolvedTableTableExtractor], + isInput = true, + setCurrentDatabaseIfMissing = true) + TableCommandSpec(cmd, Seq(tableDesc), DESCTABLE) + } + val DropTable = { val cmd = "org.apache.spark.sql.execution.command.DropTableCommand" val tableTypeDesc = @@ -425,8 +461,7 @@ object TableCommands { val DropTableV2 = { val cmd = "org.apache.spark.sql.catalyst.plans.logical.DropTable" - val tableDesc1 = resolvedTableDesc - TableCommandSpec(cmd, Seq(tableDesc1), DROPTABLE) + TableCommandSpec(cmd, Seq(resolvedIdentifierTableDesc, resolvedTableDesc), DROPTABLE) } val MergeIntoTable = { @@ -559,7 +594,13 @@ object TableCommands { TableCommandSpec(cmd, Seq(tableIdentDesc.copy(isInput = true))) } - val data: Array[TableCommandSpec] = Array( + val SetTableProperties = { + val cmd = "org.apache.spark.sql.catalyst.plans.logical.SetTableProperties" + val tableDesc = TableDesc("table", classOf[ResolvedTableTableExtractor]) + TableCommandSpec(cmd, Seq(tableDesc), ALTERTABLE_PROPERTIES) + } + + override def specs: Seq[TableCommandSpec] = Seq( AddPartitions, DropPartitions, RenamePartitions, @@ -587,8 +628,6 @@ object TableCommands { AnalyzeColumn, AnalyzePartition, AnalyzeTable, - AnalyzeTable.copy(classname = - "org.apache.spark.sql.execution.command.AnalyzeTablesCommand"), AppendDataV2, CacheTable, CacheTableAsSelect, @@ -601,6 +640,7 @@ object TableCommands { CreateHiveTableAsSelect, CreateHiveTableAsSelect.copy(classname = "org.apache.spark.sql.hive.execution.OptimizedCreateHiveTableAsSelectCommand"), + CreateTable, CreateTableLike, CreateTableV2, CreateTableV2.copy(classname = @@ -614,6 +654,7 @@ object TableCommands { DeleteFromTable, DescribeColumn, DescribeTable, + DescribeRelationTable, DropTable, DropTableV2, InsertIntoDataSource, @@ -622,7 +663,7 @@ object TableCommands { "org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand"), InsertIntoHadoopFsRelationCommand, InsertIntoDataSourceDir.copy(classname = - "org.apache.spark.sql.execution.datasources.InsertIntoHiveDirCommand"), + "org.apache.spark.sql.hive.execution.InsertIntoHiveDirCommand"), InsertIntoHiveTable, LoadData, MergeIntoTable, @@ -632,6 +673,8 @@ object TableCommands { RefreshTable, RefreshTableV2, RefreshTable3d0, + ReplaceData, + SetTableProperties, ShowColumns, ShowCreateTable, ShowCreateTable.copy(classname = diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/DeltaCatalogRangerSparkExtensionSuite.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/DeltaCatalogRangerSparkExtensionSuite.scala new file mode 100644 index 000000000..e833f03cd --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/DeltaCatalogRangerSparkExtensionSuite.scala @@ -0,0 +1,195 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.plugin.spark.authz.ranger + +import org.scalatest.Outcome + +import org.apache.kyuubi.Utils +import org.apache.kyuubi.plugin.spark.authz.AccessControlException +import org.apache.kyuubi.plugin.spark.authz.RangerTestNamespace._ +import org.apache.kyuubi.plugin.spark.authz.RangerTestUsers._ +import org.apache.kyuubi.plugin.spark.authz.ranger.DeltaCatalogRangerSparkExtensionSuite._ +import org.apache.kyuubi.tags.DeltaTest +import org.apache.kyuubi.util.AssertionUtils._ + +/** + * Tests for RangerSparkExtensionSuite on Delta Lake + */ +@DeltaTest +class DeltaCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSuite { + override protected val catalogImpl: String = "hive" + override protected val sqlExtensions: String = "io.delta.sql.DeltaSparkSessionExtension" + + val namespace1 = deltaNamespace + val table1 = "table1_delta" + val table2 = "table2_delta" + + override def withFixture(test: NoArgTest): Outcome = { + test() + } + + override def beforeAll(): Unit = { + spark.conf.set(s"spark.sql.catalog.$sparkCatalog", deltaCatalogClassName) + spark.conf.set( + s"spark.sql.catalog.$sparkCatalog.warehouse", + Utils.createTempDir("delta-hadoop").toString) + super.beforeAll() + } + + override def afterAll(): Unit = { + super.afterAll() + spark.sessionState.catalog.reset() + spark.sessionState.conf.clear() + } + + test("create table") { + withCleanTmpResources(Seq( + (s"$namespace1.$table1", "table"), + (s"$namespace1.$table2", "table"), + (s"$namespace1", "database"))) { + doAs(admin, sql(s"CREATE DATABASE IF NOT EXISTS $namespace1")) + val createNonPartitionTableSql = + s""" + |CREATE TABLE IF NOT EXISTS $namespace1.$table1 ( + | id INT, + | firstName STRING, + | middleName STRING, + | lastName STRING, + | gender STRING, + | birthDate TIMESTAMP, + | ssn STRING, + | salary INT + |) USING DELTA + |""".stripMargin + interceptContains[AccessControlException] { + doAs(someone, sql(createNonPartitionTableSql)) + }(s"does not have [create] privilege on [$namespace1/$table1]") + doAs(admin, createNonPartitionTableSql) + + val createPartitionTableSql = + s""" + |CREATE TABLE IF NOT EXISTS $namespace1.$table2 ( + | id INT, + | firstName STRING, + | middleName STRING, + | lastName STRING, + | gender STRING, + | birthDate TIMESTAMP, + | ssn STRING, + | salary INT + |) + |USING DELTA + |PARTITIONED BY (gender) + |""".stripMargin + interceptContains[AccessControlException] { + doAs(someone, sql(createPartitionTableSql)) + }(s"does not have [create] privilege on [$namespace1/$table2]") + doAs(admin, createPartitionTableSql) + } + } + + test("create or replace table") { + withCleanTmpResources(Seq((s"$namespace1.$table1", "table"), (s"$namespace1", "database"))) { + val createOrReplaceTableSql = + s""" + |CREATE OR REPLACE TABLE $namespace1.$table1 ( + | id INT, + | firstName STRING, + | middleName STRING, + | lastName STRING, + | gender STRING, + | birthDate TIMESTAMP, + | ssn STRING, + | salary INT + |) USING DELTA + |""".stripMargin + interceptContains[AccessControlException] { + doAs(someone, sql(createOrReplaceTableSql)) + }(s"does not have [create] privilege on [$namespace1/$table1]") + doAs(admin, createOrReplaceTableSql) + } + } + + test("alter table") { + withCleanTmpResources(Seq((s"$namespace1.$table1", "table"), (s"$namespace1", "database"))) { + doAs(admin, sql(s"CREATE DATABASE IF NOT EXISTS $namespace1")) + doAs( + admin, + sql( + s""" + |CREATE TABLE IF NOT EXISTS $namespace1.$table1 ( + | id INT, + | firstName STRING, + | middleName STRING, + | lastName STRING, + | gender STRING, + | birthDate TIMESTAMP, + | ssn STRING, + | salary INT + |) + |USING DELTA + |PARTITIONED BY (gender) + |""".stripMargin)) + + // add columns + interceptContains[AccessControlException]( + doAs(someone, sql(s"ALTER TABLE $namespace1.$table1 ADD COLUMNS (age int)")))( + s"does not have [alter] privilege on [$namespace1/$table1]") + + // change column + interceptContains[AccessControlException]( + doAs( + someone, + sql(s"ALTER TABLE $namespace1.$table1" + + s" CHANGE COLUMN gender gender STRING AFTER birthDate")))( + s"does not have [alter] privilege on [$namespace1/$table1]") + + // replace columns + interceptContains[AccessControlException]( + doAs( + someone, + sql(s"ALTER TABLE $namespace1.$table1" + + s" REPLACE COLUMNS (id INT, firstName STRING)")))( + s"does not have [alter] privilege on [$namespace1/$table1]") + + // rename column + interceptContains[AccessControlException]( + doAs( + someone, + sql(s"ALTER TABLE $namespace1.$table1" + + s" RENAME COLUMN birthDate TO dateOfBirth")))( + s"does not have [alter] privilege on [$namespace1/$table1]") + + // drop column + interceptContains[AccessControlException]( + doAs(someone, sql(s"ALTER TABLE $namespace1.$table1 DROP COLUMN birthDate")))( + s"does not have [alter] privilege on [$namespace1/$table1]") + + // set properties + interceptContains[AccessControlException]( + doAs( + someone, + sql(s"ALTER TABLE $namespace1.$table1" + + s" SET TBLPROPERTIES ('delta.appendOnly' = 'true')")))( + s"does not have [alter] privilege on [$namespace1/$table1]") + } + } +} + +object DeltaCatalogRangerSparkExtensionSuite { + val deltaCatalogClassName: String = "org.apache.spark.sql.delta.catalog.DeltaCatalog" +} diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/HudiCatalogRangerSparkExtensionSuite.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/HudiCatalogRangerSparkExtensionSuite.scala new file mode 100644 index 000000000..8cff1698d --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/HudiCatalogRangerSparkExtensionSuite.scala @@ -0,0 +1,619 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.plugin.spark.authz.ranger + +import org.apache.spark.SparkConf +import org.scalatest.Outcome + +import org.apache.kyuubi.Utils +import org.apache.kyuubi.plugin.spark.authz.AccessControlException +import org.apache.kyuubi.plugin.spark.authz.RangerTestNamespace._ +import org.apache.kyuubi.plugin.spark.authz.RangerTestUsers._ +import org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils._ +import org.apache.kyuubi.tags.HudiTest +import org.apache.kyuubi.util.AssertionUtils.interceptContains + +/** + * Tests for RangerSparkExtensionSuite on Hudi SQL. + * Run this test should enbale `hudi` profile. + */ +@HudiTest +class HudiCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSuite { + override protected val catalogImpl: String = "in-memory" + // TODO: Apache Hudi not support Spark 3.5 and Scala 2.13 yet, + // should change after Apache Hudi support Spark 3.5 and Scala 2.13. + private def isSupportedVersion = !isSparkV35OrGreater && !isScalaV213 + + override protected val sqlExtensions: String = + if (isSupportedVersion) { + "org.apache.spark.sql.hudi.HoodieSparkSessionExtension" + } else { + "" + } + + override protected val extraSparkConf: SparkConf = + new SparkConf() + .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer") + + val namespace1 = hudiNamespace + val table1 = "table1_hoodie" + val table2 = "table2_hoodie" + val outputTable1 = "outputTable_hoodie" + val index1 = "table_hoodie_index1" + + override def withFixture(test: NoArgTest): Outcome = { + assume(isSupportedVersion) + test() + } + + override def beforeAll(): Unit = { + if (isSupportedVersion) { + if (isSparkV32OrGreater) { + spark.conf.set( + s"spark.sql.catalog.$sparkCatalog", + "org.apache.spark.sql.hudi.catalog.HoodieCatalog") + spark.conf.set(s"spark.sql.catalog.$sparkCatalog.type", "hadoop") + spark.conf.set( + s"spark.sql.catalog.$sparkCatalog.warehouse", + Utils.createTempDir("hudi-hadoop").toString) + } + super.beforeAll() + } + } + + override def afterAll(): Unit = { + if (isSupportedVersion) { + super.afterAll() + spark.sessionState.catalog.reset() + spark.sessionState.conf.clear() + } + } + + test("AlterTableCommand") { + withCleanTmpResources(Seq((s"$namespace1.$table1", "table"), (namespace1, "database"))) { + doAs(admin, sql(s"CREATE DATABASE IF NOT EXISTS $namespace1")) + doAs( + admin, + sql( + s""" + |CREATE TABLE IF NOT EXISTS $namespace1.$table1(id int, name string, city string) + |USING hudi + |OPTIONS ( + | type = 'cow', + | primaryKey = 'id', + | 'hoodie.datasource.hive_sync.enable' = 'false' + |) + |PARTITIONED BY(city) + |""".stripMargin)) + + // AlterHoodieTableAddColumnsCommand + interceptContains[AccessControlException]( + doAs(someone, sql(s"ALTER TABLE $namespace1.$table1 ADD COLUMNS(age int)")))( + s"does not have [alter] privilege on [$namespace1/$table1/age]") + + // AlterHoodieTableChangeColumnCommand + interceptContains[AccessControlException]( + doAs(someone, sql(s"ALTER TABLE $namespace1.$table1 CHANGE COLUMN id id bigint")))( + s"does not have [alter] privilege" + + s" on [$namespace1/$table1/id]") + + // AlterHoodieTableDropPartitionCommand + interceptContains[AccessControlException]( + doAs(someone, sql(s"ALTER TABLE $namespace1.$table1 DROP PARTITION (city='test')")))( + s"does not have [alter] privilege" + + s" on [$namespace1/$table1/city]") + + // AlterHoodieTableRenameCommand + interceptContains[AccessControlException]( + doAs(someone, sql(s"ALTER TABLE $namespace1.$table1 RENAME TO $namespace1.$table2")))( + s"does not have [alter] privilege" + + s" on [$namespace1/$table1]") + + // AlterTableCommand && Spark31AlterTableCommand + try { + sql("set hoodie.schema.on.read.enable=true") + interceptContains[AccessControlException]( + doAs(someone, sql(s"ALTER TABLE $namespace1.$table1 ADD COLUMNS(age int)")))( + s"does not have [alter] privilege on [$namespace1/$table1]") + } finally { + sql("set hoodie.schema.on.read.enable=false") + } + } + } + + test("CreateHoodieTableCommand") { + withCleanTmpResources(Seq((namespace1, "database"))) { + doAs(admin, sql(s"CREATE DATABASE IF NOT EXISTS $namespace1")) + interceptContains[AccessControlException]( + doAs( + someone, + sql( + s""" + |CREATE TABLE IF NOT EXISTS $namespace1.$table1(id int, name string, city string) + |USING HUDI + |OPTIONS ( + | type = 'cow', + | primaryKey = 'id', + | 'hoodie.datasource.hive_sync.enable' = 'false' + |) + |PARTITIONED BY(city) + |""".stripMargin)))(s"does not have [create] privilege on [$namespace1/$table1]") + } + } + + test("CreateHoodieTableAsSelectCommand") { + withCleanTmpResources(Seq((s"$namespace1.$table1", "table"), (namespace1, "database"))) { + doAs(admin, sql(s"CREATE DATABASE IF NOT EXISTS $namespace1")) + doAs( + admin, + sql( + s""" + |CREATE TABLE IF NOT EXISTS $namespace1.$table1(id int, name string, city string) + |USING HUDI + |OPTIONS ( + | type = 'cow', + | primaryKey = 'id', + | 'hoodie.datasource.hive_sync.enable' = 'false' + |) + |PARTITIONED BY(city) + |""".stripMargin)) + interceptContains[AccessControlException]( + doAs( + someone, + sql( + s""" + |CREATE TABLE IF NOT EXISTS $namespace1.$table2 + |USING HUDI + |AS + |SELECT id FROM $namespace1.$table1 + |""".stripMargin)))(s"does not have [select] privilege on [$namespace1/$table1/id]") + } + } + + test("CreateHoodieTableLikeCommand") { + withCleanTmpResources(Seq( + (s"$namespace1.$table1", "table"), + (s"$namespace1.$table2", "table"), + (namespace1, "database"))) { + doAs(admin, sql(s"CREATE DATABASE IF NOT EXISTS $namespace1")) + doAs( + admin, + sql( + s""" + |CREATE TABLE IF NOT EXISTS $namespace1.$table1(id int, name string, city string) + |USING HUDI + |OPTIONS ( + | type = 'cow', + | primaryKey = 'id', + | 'hoodie.datasource.hive_sync.enable' = 'false' + |) + |PARTITIONED BY(city) + |""".stripMargin)) + + val createTableSql = + s""" + |CREATE TABLE IF NOT EXISTS $namespace1.$table2 + |LIKE $namespace1.$table1 + |USING HUDI + |""".stripMargin + interceptContains[AccessControlException] { + doAs( + someone, + sql( + createTableSql)) + }(s"does not have [select] privilege on [$namespace1/$table1]") + doAs(admin, sql(createTableSql)) + } + } + + test("DropHoodieTableCommand") { + withCleanTmpResources(Seq((namespace1, "database"))) { + doAs(admin, sql(s"CREATE DATABASE IF NOT EXISTS $namespace1")) + doAs( + admin, + sql( + s""" + |CREATE TABLE IF NOT EXISTS $namespace1.$table1(id int, name string, city string) + |USING HUDI + |OPTIONS ( + | type = 'cow', + | primaryKey = 'id', + | 'hoodie.datasource.hive_sync.enable' = 'false' + |) + |PARTITIONED BY(city) + |""".stripMargin)) + + val dropTableSql = s"DROP TABLE IF EXISTS $namespace1.$table1" + interceptContains[AccessControlException] { + doAs(someone, sql(dropTableSql)) + }(s"does not have [drop] privilege on [$namespace1/$table1]") + doAs(admin, sql(dropTableSql)) + } + } + + test("RepairHoodieTableCommand") { + withCleanTmpResources(Seq((s"$namespace1.$table1", "table"), (namespace1, "database"))) { + doAs(admin, sql(s"CREATE DATABASE IF NOT EXISTS $namespace1")) + doAs( + admin, + sql( + s""" + |CREATE TABLE IF NOT EXISTS $namespace1.$table1(id int, name string, city string) + |USING HUDI + |OPTIONS ( + | type = 'cow', + | primaryKey = 'id', + | 'hoodie.datasource.hive_sync.enable' = 'false' + |) + |PARTITIONED BY(city) + |""".stripMargin)) + + val repairTableSql = s"MSCK REPAIR TABLE $namespace1.$table1" + interceptContains[AccessControlException] { + doAs(someone, sql(repairTableSql)) + }(s"does not have [alter] privilege on [$namespace1/$table1]") + doAs(admin, sql(repairTableSql)) + } + } + + test("TruncateHoodieTableCommand") { + withCleanTmpResources(Seq((s"$namespace1.$table1", "table"), (namespace1, "database"))) { + doAs(admin, sql(s"CREATE DATABASE IF NOT EXISTS $namespace1")) + doAs( + admin, + sql( + s""" + |CREATE TABLE IF NOT EXISTS $namespace1.$table1(id int, name string, city string) + |USING HUDI + |OPTIONS ( + | type = 'cow', + | primaryKey = 'id', + | 'hoodie.datasource.hive_sync.enable' = 'false' + |) + |PARTITIONED BY(city) + |""".stripMargin)) + + val truncateTableSql = s"TRUNCATE TABLE $namespace1.$table1" + interceptContains[AccessControlException] { + doAs(someone, sql(truncateTableSql)) + }(s"does not have [update] privilege on [$namespace1/$table1]") + doAs(admin, sql(truncateTableSql)) + } + } + + test("CompactionHoodieTableCommand / CompactionShowHoodieTableCommand") { + withCleanTmpResources(Seq((s"$namespace1.$table1", "table"), (namespace1, "database"))) { + doAs(admin, sql(s"CREATE DATABASE IF NOT EXISTS $namespace1")) + doAs( + admin, + sql( + s""" + |CREATE TABLE IF NOT EXISTS $namespace1.$table1(id int, name string, city string) + |USING HUDI + |OPTIONS ( + | type = 'mor', + | primaryKey = 'id', + | 'hoodie.datasource.hive_sync.enable' = 'false' + |) + |PARTITIONED BY(city) + |""".stripMargin)) + + val compactionTable = s"RUN COMPACTION ON $namespace1.$table1" + interceptContains[AccessControlException] { + doAs(someone, sql(compactionTable)) + }(s"does not have [create] privilege on [$namespace1/$table1]") + doAs(admin, sql(compactionTable)) + + val showCompactionTable = s"SHOW COMPACTION ON $namespace1.$table1" + interceptContains[AccessControlException] { + doAs(someone, sql(showCompactionTable)) + }(s"does not have [select] privilege on [$namespace1/$table1]") + doAs(admin, sql(showCompactionTable)) + } + } + + test("CompactionHoodiePathCommand / CompactionShowHoodiePathCommand") { + withSingleCallEnabled { + withCleanTmpResources(Seq.empty) { + val path1 = "hdfs://demo/test/hudi/path" + val compactOnPath = s"RUN COMPACTION ON '$path1'" + interceptContains[AccessControlException]( + doAs(someone, sql(compactOnPath)))( + s"does not have [create] privilege on [[$path1, $path1/]]") + + val showCompactOnPath = s"SHOW COMPACTION ON '$path1'" + interceptContains[AccessControlException]( + doAs(someone, sql(showCompactOnPath)))( + s"does not have [select] privilege on [[$path1, $path1/]]") + + val path2 = "file:///demo/test/hudi/path" + val compactOnPath2 = s"RUN COMPACTION ON '$path2'" + interceptContains[AccessControlException]( + doAs(someone, sql(compactOnPath2)))( + s"does not have [create] privilege on [[$path2, $path2/]]") + + val showCompactOnPath2 = s"SHOW COMPACTION ON '$path2'" + interceptContains[AccessControlException]( + doAs(someone, sql(showCompactOnPath2)))( + s"does not have [select] privilege on [[$path2, $path2/]]") + + val path3 = "hdfs://demo/test/hudi/path" + val compactOnPath3 = s"RUN COMPACTION ON '$path3'" + interceptContains[AccessControlException]( + doAs(someone, sql(compactOnPath3)))( + s"does not have [create] privilege on [[$path3, $path3/]]") + + val showCompactOnPath3 = s"SHOW COMPACTION ON '$path3/'" + interceptContains[AccessControlException]( + doAs(someone, sql(showCompactOnPath3)))( + s"does not have [select] privilege on [[$path3, $path3/]]") + } + } + } + + test("InsertIntoHoodieTableCommand") { + withSingleCallEnabled { + withCleanTmpResources(Seq( + (s"$namespace1.$table1", "table"), + (s"$namespace1.$table2", "table"), + (namespace1, "database"))) { + doAs(admin, sql(s"CREATE DATABASE IF NOT EXISTS $namespace1")) + doAs( + admin, + sql( + s""" + |CREATE TABLE IF NOT EXISTS $namespace1.$table1(id int, name string, city string) + |USING HUDI + |OPTIONS ( + | type = 'cow', + | primaryKey = 'id', + | 'hoodie.datasource.hive_sync.enable' = 'false' + |) + |PARTITIONED BY(city) + |""".stripMargin)) + + doAs( + admin, + sql( + s""" + |CREATE TABLE IF NOT EXISTS $namespace1.$table2(id int, name string, city string) + |USING $format + |""".stripMargin)) + + val insertIntoHoodieTableSql = + s""" + |INSERT INTO $namespace1.$table1 + |PARTITION(city = 'hangzhou') + |SELECT id, name + |FROM $namespace1.$table2 + |WHERE city = 'hangzhou' + |""".stripMargin + interceptContains[AccessControlException] { + doAs(someone, sql(insertIntoHoodieTableSql)) + }(s"does not have [select] privilege on " + + s"[$namespace1/$table2/id,$namespace1/$table2/name,hudi_ns/$table2/city], " + + s"[update] privilege on [$namespace1/$table1]") + } + } + } + + test("ShowHoodieTablePartitionsCommand") { + withSingleCallEnabled { + withCleanTmpResources(Seq( + (s"$namespace1.$table1", "table"), + (s"$namespace1.$table2", "table"), + (namespace1, "database"))) { + doAs(admin, sql(s"CREATE DATABASE IF NOT EXISTS $namespace1")) + doAs( + admin, + sql( + s""" + |CREATE TABLE IF NOT EXISTS $namespace1.$table1(id int, name string, city string) + |USING HUDI + |OPTIONS ( + | type = 'cow', + | primaryKey = 'id', + | 'hoodie.datasource.hive_sync.enable' = 'false' + |) + |PARTITIONED BY(city) + |""".stripMargin)) + + val showPartitionsSql = s"SHOW PARTITIONS $namespace1.$table1" + interceptContains[AccessControlException] { + doAs(someone, sql(showPartitionsSql)) + }(s"does not have [select] privilege on [$namespace1/$table1]") + doAs(admin, sql(showPartitionsSql)) + + val showPartitionSpecSql = + s"SHOW PARTITIONS $namespace1.$table1 PARTITION (city = 'hangzhou')" + interceptContains[AccessControlException] { + doAs(someone, sql(showPartitionSpecSql)) + }(s"does not have [select] privilege on [$namespace1/$table1/city]") + doAs(admin, sql(showPartitionSpecSql)) + } + } + } + + test("DeleteHoodieTableCommand/UpdateHoodieTableCommand/MergeIntoHoodieTableCommand") { + withSingleCallEnabled { + withCleanTmpResources(Seq( + (s"$namespace1.$table1", "table"), + (s"$namespace1.$table2", "table"), + (namespace1, "database"))) { + doAs(admin, sql(s"CREATE DATABASE IF NOT EXISTS $namespace1")) + doAs( + admin, + sql( + s""" + |CREATE TABLE IF NOT EXISTS $namespace1.$table1(id int, name string, city string) + |USING HUDI + |OPTIONS ( + | type = 'cow', + | primaryKey = 'id', + | 'hoodie.datasource.hive_sync.enable' = 'false' + |) + |PARTITIONED BY(city) + |""".stripMargin)) + + doAs( + admin, + sql( + s""" + |CREATE TABLE IF NOT EXISTS $namespace1.$table2(id int, name string, city string) + |USING HUDI + |OPTIONS ( + | type = 'cow', + | primaryKey = 'id', + | 'hoodie.datasource.hive_sync.enable' = 'false' + |) + |PARTITIONED BY(city) + |""".stripMargin)) + + val deleteFrom = s"DELETE FROM $namespace1.$table1 WHERE id = 10" + interceptContains[AccessControlException] { + doAs(someone, sql(deleteFrom)) + }(s"does not have [update] privilege on [$namespace1/$table1]") + doAs(admin, sql(deleteFrom)) + + val updateSql = s"UPDATE $namespace1.$table1 SET name = 'test' WHERE id > 10" + interceptContains[AccessControlException] { + doAs(someone, sql(updateSql)) + }(s"does not have [update] privilege on [$namespace1/$table1]") + doAs(admin, sql(updateSql)) + + val mergeIntoSQL = + s""" + |MERGE INTO $namespace1.$table1 target + |USING $namespace1.$table2 source + |ON target.id = source.id + |WHEN MATCHED + |AND target.name == 'test' + | THEN UPDATE SET id = source.id, name = source.name, city = source.city + |""".stripMargin + interceptContains[AccessControlException] { + doAs(someone, sql(mergeIntoSQL)) + }(s"does not have [select] privilege on " + + s"[$namespace1/$table2/id,$namespace1/$table2/name,$namespace1/$table2/city]") + doAs(admin, sql(mergeIntoSQL)) + } + } + } + + test("CallProcedureHoodieCommand") { + withSingleCallEnabled { + withCleanTmpResources(Seq( + (s"$namespace1.$table1", "table"), + (s"$namespace1.$table2", "table"), + (namespace1, "database"))) { + doAs(admin, sql(s"CREATE DATABASE IF NOT EXISTS $namespace1")) + doAs( + admin, + sql( + s""" + |CREATE TABLE IF NOT EXISTS $namespace1.$table1(id int, name string, city string) + |USING HUDI + |OPTIONS ( + | type = 'cow', + | primaryKey = 'id', + | 'hoodie.datasource.hive_sync.enable' = 'false' + |) + |PARTITIONED BY(city) + |""".stripMargin)) + doAs( + admin, + sql( + s""" + |CREATE TABLE IF NOT EXISTS $namespace1.$table2(id int, name string, city string) + |USING HUDI + |OPTIONS ( + | type = 'cow', + | primaryKey = 'id', + | 'hoodie.datasource.hive_sync.enable' = 'false' + |) + |PARTITIONED BY(city) + |""".stripMargin)) + + val copy_to_table = + s"CALL copy_to_table(table => '$namespace1.$table1', new_table => '$namespace1.$table2')" + interceptContains[AccessControlException] { + doAs(someone, sql(copy_to_table)) + }(s"does not have [select] privilege on [$namespace1/$table1]") + doAs(admin, sql(copy_to_table)) + + val show_table_properties = s"CALL show_table_properties(table => '$namespace1.$table1')" + interceptContains[AccessControlException] { + doAs(someone, sql(show_table_properties)) + }(s"does not have [select] privilege on [$namespace1/$table1]") + doAs(admin, sql(show_table_properties)) + } + } + } + + test("IndexBasedCommand") { + assume( + !isSparkV33OrGreater, + "Hudi index creation not supported on Spark 3.3 or greater currently") + withCleanTmpResources(Seq((s"$namespace1.$table1", "table"), (namespace1, "database"))) { + doAs(admin, sql(s"CREATE DATABASE IF NOT EXISTS $namespace1")) + doAs( + admin, + sql( + s""" + |CREATE TABLE IF NOT EXISTS $namespace1.$table1(id int, name string, city string) + |USING HUDI + |OPTIONS ( + | type = 'cow', + | primaryKey = 'id', + | 'hoodie.datasource.hive_sync.enable' = 'false' + |) + |PARTITIONED BY(city) + |""".stripMargin)) + + // CreateIndexCommand + val createIndex = s"CREATE INDEX $index1 ON $namespace1.$table1 USING LUCENE (id)" + interceptContains[AccessControlException]( + doAs( + someone, + sql(createIndex)))(s"does not have [index] privilege on [$namespace1/$table1]") + doAs(admin, sql(createIndex)) + + // RefreshIndexCommand + val refreshIndex = s"REFRESH INDEX $index1 ON $namespace1.$table1" + interceptContains[AccessControlException]( + doAs( + someone, + sql(refreshIndex)))(s"does not have [alter] privilege on [$namespace1/$table1]") + doAs(admin, sql(refreshIndex)) + + // ShowIndexesCommand + val showIndex = s"SHOW INDEXES FROM TABLE $namespace1.$table1" + interceptContains[AccessControlException]( + doAs( + someone, + sql(showIndex)))(s"does not have [select] privilege on [$namespace1/$table1]") + doAs(admin, sql(showIndex)) + + // DropIndexCommand + val dropIndex = s"DROP INDEX $index1 ON $namespace1.$table1" + interceptContains[AccessControlException]( + doAs( + someone, + sql(dropIndex)))(s"does not have [drop] privilege on [$namespace1/$table1]") + doAs(admin, sql(dropIndex)) + } + } +} diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/IcebergCatalogRangerSparkExtensionSuite.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/IcebergCatalogRangerSparkExtensionSuite.scala index a2634bb26..28e13aff3 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/IcebergCatalogRangerSparkExtensionSuite.scala +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/IcebergCatalogRangerSparkExtensionSuite.scala @@ -16,55 +16,74 @@ */ package org.apache.kyuubi.plugin.spark.authz.ranger -// scalastyle:off +import java.sql.Timestamp +import java.text.SimpleDateFormat + import scala.util.Try +import org.apache.spark.sql.Row +import org.scalatest.Outcome + +// scalastyle:off import org.apache.kyuubi.Utils import org.apache.kyuubi.plugin.spark.authz.AccessControlException +import org.apache.kyuubi.plugin.spark.authz.RangerTestNamespace._ +import org.apache.kyuubi.plugin.spark.authz.RangerTestUsers._ +import org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils._ +import org.apache.kyuubi.tags.IcebergTest +import org.apache.kyuubi.util.AssertionUtils._ /** * Tests for RangerSparkExtensionSuite * on Iceberg catalog with DataSource V2 API. */ +@IcebergTest class IcebergCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSuite { override protected val catalogImpl: String = "hive" override protected val sqlExtensions: String = - if (isSparkV32OrGreater) - "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions" - else "" + "org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions" val catalogV2 = "local" - val namespace1 = "iceberg_ns" + val namespace1 = icebergNamespace val table1 = "table1" val outputTable1 = "outputTable1" + val bobNamespace = "default_bob" + val bobSelectTable = "table_select_bob_1" + + override def withFixture(test: NoArgTest): Outcome = { + test() + } override def beforeAll(): Unit = { - if (isSparkV32OrGreater) { - spark.conf.set( - s"spark.sql.catalog.$catalogV2", - "org.apache.iceberg.spark.SparkCatalog") - spark.conf.set(s"spark.sql.catalog.$catalogV2.type", "hadoop") - spark.conf.set( - s"spark.sql.catalog.$catalogV2.warehouse", - Utils.createTempDir("iceberg-hadoop").toString) - - super.beforeAll() - - doAs("admin", sql(s"CREATE DATABASE IF NOT EXISTS $catalogV2.$namespace1")) - doAs( - "admin", - sql(s"CREATE TABLE IF NOT EXISTS $catalogV2.$namespace1.$table1" + - " (id int, name string, city string) USING iceberg")) + spark.conf.set( + s"spark.sql.catalog.$catalogV2", + "org.apache.iceberg.spark.SparkCatalog") + spark.conf.set(s"spark.sql.catalog.$catalogV2.type", "hadoop") + spark.conf.set( + s"spark.sql.catalog.$catalogV2.warehouse", + Utils.createTempDir("iceberg-hadoop").toString) - doAs( - "admin", - sql(s"INSERT INTO $catalogV2.$namespace1.$table1" + - " (id , name , city ) VALUES (1, 'liangbowen','Guangzhou')")) - doAs( - "admin", - sql(s"CREATE TABLE IF NOT EXISTS $catalogV2.$namespace1.$outputTable1" + - " (id int, name string, city string) USING iceberg")) - } + super.beforeAll() + + doAs(admin, sql(s"CREATE DATABASE IF NOT EXISTS $catalogV2.$namespace1")) + doAs( + admin, + sql(s"CREATE TABLE IF NOT EXISTS $catalogV2.$namespace1.$table1" + + " (id int, name string, city string) USING iceberg")) + + doAs( + admin, + sql(s"INSERT INTO $catalogV2.$namespace1.$table1" + + " (id , name , city ) VALUES (1, 'liangbowen','Guangzhou')")) + doAs( + admin, + sql(s"CREATE TABLE IF NOT EXISTS $catalogV2.$namespace1.$outputTable1" + + " (id int, name string, city string) USING iceberg")) + + doAs( + admin, + sql(s"CREATE TABLE IF NOT EXISTS $catalogV2.$bobNamespace.$bobSelectTable" + + " (id int, name string, city string) USING iceberg")) } override def afterAll(): Unit = { @@ -74,11 +93,9 @@ class IcebergCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSuite } test("[KYUUBI #3515] MERGE INTO") { - assume(isSparkV32OrGreater) - val mergeIntoSql = s""" - |MERGE INTO $catalogV2.$namespace1.$outputTable1 AS target + |MERGE INTO $catalogV2.$bobNamespace.$bobSelectTable AS target |USING $catalogV2.$namespace1.$table1 AS source |ON target.id = source.id |WHEN MATCHED AND (target.name='delete') THEN DELETE @@ -88,65 +105,64 @@ class IcebergCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSuite // MergeIntoTable: Using a MERGE INTO Statement val e1 = intercept[AccessControlException]( doAs( - "someone", + someone, sql(mergeIntoSql))) assert(e1.getMessage.contains(s"does not have [select] privilege" + s" on [$namespace1/$table1/id]")) - try { - SparkRangerAdminPlugin.getRangerConf.setBoolean( - s"ranger.plugin.${SparkRangerAdminPlugin.getServiceType}.authorize.in.single.call", - true) - val e2 = intercept[AccessControlException]( - doAs( - "someone", - sql(mergeIntoSql))) - assert(e2.getMessage.contains(s"does not have" + - s" [select] privilege" + - s" on [$namespace1/$table1/id,$namespace1/table1/name,$namespace1/$table1/city]," + - s" [update] privilege on [$namespace1/$outputTable1]")) - } finally { - SparkRangerAdminPlugin.getRangerConf.setBoolean( - s"ranger.plugin.${SparkRangerAdminPlugin.getServiceType}.authorize.in.single.call", - false) + withSingleCallEnabled { + interceptContains[AccessControlException](doAs(someone, sql(mergeIntoSql)))( + if (isSparkV35OrGreater) { + s"does not have [select] privilege on [$namespace1/table1/id" + + s",$namespace1/$table1/name,$namespace1/$table1/city]" + } else { + "does not have " + + s"[select] privilege on [$namespace1/$table1/id,$namespace1/$table1/name,$namespace1/$table1/city]," + + s" [update] privilege on [$bobNamespace/$bobSelectTable]" + }) + + interceptContains[AccessControlException] { + doAs(bob, sql(mergeIntoSql)) + }(s"does not have [update] privilege on [$bobNamespace/$bobSelectTable]") } - doAs("admin", sql(mergeIntoSql)) + doAs(admin, sql(mergeIntoSql)) } test("[KYUUBI #3515] UPDATE TABLE") { - assume(isSparkV32OrGreater) - // UpdateTable - val e1 = intercept[AccessControlException]( - doAs( - "someone", - sql(s"UPDATE $catalogV2.$namespace1.$table1 SET city='Guangzhou' " + - " WHERE id=1"))) - assert(e1.getMessage.contains(s"does not have [update] privilege" + - s" on [$namespace1/$table1]")) + interceptContains[AccessControlException] { + doAs(someone, sql(s"UPDATE $catalogV2.$namespace1.$table1 SET city='Guangzhou' WHERE id=1")) + }(if (isSparkV35OrGreater) { + s"does not have [select] privilege on [$namespace1/$table1/id]" + } else { + s"does not have [update] privilege on [$namespace1/$table1]" + }) doAs( - "admin", + admin, sql(s"UPDATE $catalogV2.$namespace1.$table1 SET city='Guangzhou' " + " WHERE id=1")) } test("[KYUUBI #3515] DELETE FROM TABLE") { - assume(isSparkV32OrGreater) - // DeleteFromTable - val e6 = intercept[AccessControlException]( - doAs("someone", sql(s"DELETE FROM $catalogV2.$namespace1.$table1 WHERE id=2"))) - assert(e6.getMessage.contains(s"does not have [update] privilege" + - s" on [$namespace1/$table1]")) + interceptContains[AccessControlException] { + doAs(someone, sql(s"DELETE FROM $catalogV2.$namespace1.$table1 WHERE id=2")) + }(if (isSparkV34OrGreater) { + s"does not have [select] privilege on [$namespace1/$table1/id]" + } else { + s"does not have [update] privilege on [$namespace1/$table1]" + }) - doAs("admin", sql(s"DELETE FROM $catalogV2.$namespace1.$table1 WHERE id=2")) + interceptContains[AccessControlException] { + doAs(bob, sql(s"DELETE FROM $catalogV2.$bobNamespace.$bobSelectTable WHERE id=2")) + }(s"does not have [update] privilege on [$bobNamespace/$bobSelectTable]") + + doAs(admin, sql(s"DELETE FROM $catalogV2.$namespace1.$table1 WHERE id=2")) } test("[KYUUBI #3666] Support {OWNER} variable for queries run on CatalogV2") { - assume(isSparkV32OrGreater) - val table = "owner_variable" val select = s"SELECT key FROM $catalogV2.$namespace1.$table" @@ -164,7 +180,7 @@ class IcebergCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSuite }.isSuccess)) doAs( - "create_only_user", { + createOnlyUser, { val e = intercept[AccessControlException](sql(select).collect()) assert(e.getMessage === errorMessage("select", s"$namespace1/$table/key")) }) @@ -179,17 +195,17 @@ class IcebergCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSuite (s"$catalogV2.default.src", "table"), (s"$catalogV2.default.outputTable2", "table"))) { doAs( - "admin", + admin, sql(s"CREATE TABLE IF NOT EXISTS $catalogV2.default.src" + " (id int, name string, key string) USING iceberg")) doAs( - "admin", + admin, sql(s"INSERT INTO $catalogV2.default.src" + " (id , name , key ) VALUES " + "(1, 'liangbowen1','10')" + ", (2, 'liangbowen2','20')")) doAs( - "admin", + admin, sql(s"CREATE TABLE IF NOT EXISTS $catalogV2.$namespace1.$outputTable2" + " (id int, name string, key string) USING iceberg")) @@ -201,20 +217,20 @@ class IcebergCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSuite |WHEN NOT MATCHED THEN INSERT (id, name, key) VALUES (source.id, source.name, source.key) """.stripMargin - doAs("admin", sql(mergeIntoSql)) + doAs(admin, sql(mergeIntoSql)) doAs( - "admin", { + admin, { val countOutputTable = sql(s"select count(1) from $catalogV2.$namespace1.$outputTable2").collect() val rowCount = countOutputTable(0).get(0) assert(rowCount === 2) }) - doAs("admin", sql(s"truncate table $catalogV2.$namespace1.$outputTable2")) + doAs(admin, sql(s"truncate table $catalogV2.$namespace1.$outputTable2")) // source table with row filter `key`<20 - doAs("bob", sql(mergeIntoSql)) + doAs(bob, sql(mergeIntoSql)) doAs( - "admin", { + admin, { val countOutputTable = sql(s"select count(1) from $catalogV2.$namespace1.$outputTable2").collect() val rowCount = countOutputTable(0).get(0) @@ -222,4 +238,130 @@ class IcebergCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSuite }) } } + + test("[KYUUBI #4255] DESCRIBE TABLE") { + val e1 = intercept[AccessControlException]( + doAs(someone, sql(s"DESCRIBE TABLE $catalogV2.$namespace1.$table1").explain())) + assert(e1.getMessage.contains(s"does not have [select] privilege" + + s" on [$namespace1/$table1]")) + } + + test("CALL RewriteDataFilesProcedure") { + val tableName = "table_select_call_command_table" + val table = s"$catalogV2.$namespace1.$tableName" + val initDataFilesCount = 2 + val rewriteDataFiles1 = s"CALL $catalogV2.system.rewrite_data_files " + + s"(table => '$table', options => map('min-input-files','$initDataFilesCount'))" + val rewriteDataFiles2 = s"CALL $catalogV2.system.rewrite_data_files " + + s"(table => '$table', options => map('min-input-files','${initDataFilesCount + 1}'))" + + withCleanTmpResources(Seq((table, "table"))) { + doAs( + admin, { + sql(s"CREATE TABLE IF NOT EXISTS $table (id int, name string) USING iceberg") + // insert 2 data files + (0 until initDataFilesCount) + .foreach(i => sql(s"INSERT INTO $table VALUES ($i, 'user_$i')")) + }) + + interceptContains[AccessControlException](doAs(someone, sql(rewriteDataFiles1)))( + s"does not have [alter] privilege on [$namespace1/$tableName]") + interceptContains[AccessControlException](doAs(someone, sql(rewriteDataFiles2)))( + s"does not have [alter] privilege on [$namespace1/$tableName]") + + /** + * Case 1: Number of input data files equals or greater than minimum expected. + * Two logical plans triggered + * when ( input-files(2) >= min-input-files(2) ): + * + * == Physical Plan 1 == + * Call (1) + * + * == Physical Plan 2 == + * AppendData (3) + * +- * ColumnarToRow (2) + * +- BatchScan local.iceberg_ns.call_command_table (1) + */ + doAs( + admin, { + val result1 = sql(rewriteDataFiles1).collect() + // rewritten results into 2 data files + assert(result1(0)(0) === initDataFilesCount) + }) + + /** + * Case 2: Number of input data files less than minimum expected. + * Only one logical plan triggered + * when ( input-files(2) < min-input-files(3) ) + * + * == Physical Plan == + * Call (1) + */ + doAs( + admin, { + val result2 = sql(rewriteDataFiles2).collect() + assert(result2(0)(0) === 0) + }) + } + } + + private def prepareExampleIcebergTable(table: String, initSnapshots: Int): Unit = { + doAs(admin, sql(s"CREATE TABLE IF NOT EXISTS $table (id int, name string) USING iceberg")) + (0 until initSnapshots).foreach(i => + doAs(admin, sql(s"INSERT INTO $table VALUES ($i, 'user_$i')"))) + } + + private def getFirstSnapshot(table: String): Row = { + val existedSnapshots = + sql(s"SELECT * FROM $table.snapshots ORDER BY committed_at ASC LIMIT 1").collect() + existedSnapshots(0) + } + + test("CALL rollback_to_snapshot") { + val tableName = "table_rollback_to_snapshot" + val table = s"$catalogV2.$namespace1.$tableName" + withCleanTmpResources(Seq((table, "table"))) { + prepareExampleIcebergTable(table, 2) + val targetSnapshotId = getFirstSnapshot(table).getAs[Long]("snapshot_id") + val callRollbackToSnapshot = + s"CALL $catalogV2.system.rollback_to_snapshot (table => '$table', snapshot_id => $targetSnapshotId)" + + interceptContains[AccessControlException](doAs(someone, sql(callRollbackToSnapshot)))( + s"does not have [alter] privilege on [$namespace1/$tableName]") + doAs(admin, sql(callRollbackToSnapshot)) + } + } + + test("CALL rollback_to_timestamp") { + val tableName = "table_rollback_to_timestamp" + val table = s"$catalogV2.$namespace1.$tableName" + withCleanTmpResources(Seq((table, "table"))) { + prepareExampleIcebergTable(table, 2) + val callRollbackToTimestamp = { + val targetSnapshotCommittedAt = getFirstSnapshot(table).getAs[Timestamp]("committed_at") + val targetTimestamp = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS") + .format(targetSnapshotCommittedAt.getTime + 1) + s"CALL $catalogV2.system.rollback_to_timestamp (table => '$table', timestamp => TIMESTAMP '$targetTimestamp')" + } + + interceptContains[AccessControlException](doAs(someone, sql(callRollbackToTimestamp)))( + s"does not have [alter] privilege on [$namespace1/$tableName]") + doAs(admin, sql(callRollbackToTimestamp)) + } + } + + test("CALL set_current_snapshot") { + val tableName = "table_set_current_snapshot" + val table = s"$catalogV2.$namespace1.$tableName" + withCleanTmpResources(Seq((table, "table"))) { + prepareExampleIcebergTable(table, 2) + val targetSnapshotId = getFirstSnapshot(table).getAs[Long]("snapshot_id") + val callSetCurrentSnapshot = + s"CALL $catalogV2.system.set_current_snapshot (table => '$table', snapshot_id => $targetSnapshotId)" + + interceptContains[AccessControlException](doAs(someone, sql(callSetCurrentSnapshot)))( + s"does not have [alter] privilege on [$namespace1/$tableName]") + doAs(admin, sql(callSetCurrentSnapshot)) + } + } } diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/PaimonCatalogRangerSparkExtensionSuite.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/PaimonCatalogRangerSparkExtensionSuite.scala new file mode 100644 index 000000000..62cd9d627 --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/PaimonCatalogRangerSparkExtensionSuite.scala @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.plugin.spark.authz.ranger + +import org.scalatest.Outcome + +import org.apache.kyuubi.Utils +import org.apache.kyuubi.plugin.spark.authz.AccessControlException +import org.apache.kyuubi.plugin.spark.authz.RangerTestUsers._ +import org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils._ +import org.apache.kyuubi.tags.PaimonTest +import org.apache.kyuubi.util.AssertionUtils._ + +/** + * Tests for RangerSparkExtensionSuite on Paimon + */ +@PaimonTest +class PaimonCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSuite { + override protected val catalogImpl: String = "hive" + private def isSupportedVersion = !isSparkV35OrGreater + + val catalogV2 = "paimon_catalog" + val namespace1 = "paimon_ns" + val table1 = "table1" + + override def withFixture(test: NoArgTest): Outcome = { + assume(isSupportedVersion) + test() + } + + override def beforeAll(): Unit = { + if (isSupportedVersion) { + spark.conf.set(s"spark.sql.catalog.$catalogV2", "org.apache.paimon.spark.SparkCatalog") + spark.conf.set( + s"spark.sql.catalog.$catalogV2.warehouse", + Utils.createTempDir(catalogV2).toString) + super.beforeAll() + } + + doAs(admin, sql(s"CREATE DATABASE IF NOT EXISTS $catalogV2.$namespace1")) + } + + override def afterAll(): Unit = { + if (isSupportedVersion) { + doAs(admin, sql(s"DROP DATABASE IF EXISTS $catalogV2.$namespace1")) + + super.afterAll() + spark.sessionState.catalog.reset() + spark.sessionState.conf.clear() + } + } + + test("CreateTable") { + withCleanTmpResources(Seq((s"$catalogV2.$namespace1.$table1", "table"))) { + val createTable = + s""" + |CREATE TABLE IF NOT EXISTS $catalogV2.$namespace1.$table1 + |(id int, name string, city string) + |USING paimon + |OPTIONS ( + | primaryKey = 'id' + |) + |""".stripMargin + + interceptContains[AccessControlException] { + doAs(someone, sql(createTable)) + }(s"does not have [create] privilege on [$namespace1/$table1]") + doAs(admin, createTable) + } + } +} diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerLocalClient.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerLocalClient.scala index d25ea716a..d7473a580 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerLocalClient.scala +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerLocalClient.scala @@ -19,6 +19,7 @@ package org.apache.kyuubi.plugin.spark.authz.ranger import java.text.SimpleDateFormat +import com.fasterxml.jackson.databind.DeserializationFeature import com.fasterxml.jackson.databind.json.JsonMapper import org.apache.ranger.admin.client.RangerAdminRESTClient import org.apache.ranger.plugin.util.ServicePolicies @@ -27,6 +28,7 @@ class RangerLocalClient extends RangerAdminRESTClient with RangerClientHelper { private val mapper = new JsonMapper() .setDateFormat(new SimpleDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z")) + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) private val policies: ServicePolicies = { val loader = Thread.currentThread().getContextClassLoader diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerSparkExtensionSuite.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerSparkExtensionSuite.scala index 8f95a3f9f..d3f190687 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerSparkExtensionSuite.scala +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/RangerSparkExtensionSuite.scala @@ -17,14 +17,10 @@ package org.apache.kyuubi.plugin.spark.authz.ranger -import java.security.PrivilegedExceptionAction -import java.sql.Timestamp - import scala.util.Try -import org.apache.commons.codec.digest.DigestUtils import org.apache.hadoop.security.UserGroupInformation -import org.apache.spark.sql.{Row, SparkSessionExtensions} +import org.apache.spark.sql.SparkSessionExtensions import org.apache.spark.sql.catalyst.analysis.NoSuchTableException import org.apache.spark.sql.catalyst.catalog.HiveTableRelation import org.apache.spark.sql.catalyst.plans.logical.Statistics @@ -35,21 +31,17 @@ import org.scalatest.BeforeAndAfterAll import org.scalatest.funsuite.AnyFunSuite import org.apache.kyuubi.plugin.spark.authz.{AccessControlException, SparkSessionProvider} -import org.apache.kyuubi.plugin.spark.authz.ranger.RuleAuthorization.KYUUBI_AUTHZ_TAG -import org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils.getFieldVal - +import org.apache.kyuubi.plugin.spark.authz.RangerTestNamespace._ +import org.apache.kyuubi.plugin.spark.authz.RangerTestUsers._ +import org.apache.kyuubi.plugin.spark.authz.rule.Authorization.KYUUBI_AUTHZ_TAG +import org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils._ +import org.apache.kyuubi.util.AssertionUtils._ +import org.apache.kyuubi.util.reflect.ReflectUtils._ abstract class RangerSparkExtensionSuite extends AnyFunSuite with SparkSessionProvider with BeforeAndAfterAll { // scalastyle:on override protected val extension: SparkSessionExtensions => Unit = new RangerSparkExtension - protected def doAs[T](user: String, f: => T): T = { - UserGroupInformation.createRemoteUser(user).doAs[T]( - new PrivilegedExceptionAction[T] { - override def run(): T = f - }) - } - override def afterAll(): Unit = { spark.stop() super.afterAll() @@ -62,24 +54,6 @@ abstract class RangerSparkExtensionSuite extends AnyFunSuite s"Permission denied: user [$user] does not have [$privilege] privilege on [$resource]" } - protected def withCleanTmpResources[T](res: Seq[(String, String)])(f: => T): T = { - try { - f - } finally { - res.foreach { - case (t, "table") => doAs("admin", sql(s"DROP TABLE IF EXISTS $t")) - case (db, "database") => doAs("admin", sql(s"DROP DATABASE IF EXISTS $db")) - case (fn, "function") => doAs("admin", sql(s"DROP FUNCTION IF EXISTS $fn")) - case (view, "view") => doAs("admin", sql(s"DROP VIEW IF EXISTS $view")) - case (cacheTable, "cache") => if (isSparkV32OrGreater) { - doAs("admin", sql(s"UNCACHE TABLE IF EXISTS $cacheTable")) - } - case (_, e) => - throw new RuntimeException(s"the resource whose resource type is $e cannot be cleared") - } - } - } - /** * Drops temporary view `viewNames` after calling `f`. */ @@ -116,20 +90,35 @@ abstract class RangerSparkExtensionSuite extends AnyFunSuite } } + /** + * Enables authorizing in single call mode, + * and disables authorizing in single call mode after calling `f` + */ + protected def withSingleCallEnabled(f: => Unit): Unit = { + val singleCallConfig = + s"ranger.plugin.${SparkRangerAdminPlugin.getServiceType}.authorize.in.single.call" + try { + SparkRangerAdminPlugin.getRangerConf.setBoolean(singleCallConfig, true) + f + } finally { + SparkRangerAdminPlugin.getRangerConf.setBoolean(singleCallConfig, false) + } + } + test("[KYUUBI #3226] RuleAuthorization: Should check privileges once only.") { - val logicalPlan = doAs("admin", sql("SHOW TABLES").queryExecution.logical) + val logicalPlan = doAs(admin, sql("SHOW TABLES").queryExecution.logical) val rule = new RuleAuthorization(spark) (1 until 10).foreach { i => if (i == 1) { assert(logicalPlan.getTagValue(KYUUBI_AUTHZ_TAG).isEmpty) } else { - assert(logicalPlan.getTagValue(KYUUBI_AUTHZ_TAG).getOrElse(false)) + assert(logicalPlan.getTagValue(KYUUBI_AUTHZ_TAG).nonEmpty) } rule.apply(logicalPlan) } - assert(logicalPlan.getTagValue(KYUUBI_AUTHZ_TAG).getOrElse(false)) + assert(logicalPlan.getTagValue(KYUUBI_AUTHZ_TAG).nonEmpty) } test("[KYUUBI #3226]: Another session should also check even if the plan is cached.") { @@ -145,13 +134,13 @@ abstract class RangerSparkExtensionSuite extends AnyFunSuite withCleanTmpResources(Seq((testTable, "table"))) { // create tmp table doAs( - "admin", { + admin, { sql(create) // session1: first query, should auth once.[LogicalRelation] val df = sql(select) val plan1 = df.queryExecution.optimizedPlan - assert(plan1.getTagValue(KYUUBI_AUTHZ_TAG).getOrElse(false)) + assert(plan1.getTagValue(KYUUBI_AUTHZ_TAG).nonEmpty) // cache df.cache() @@ -159,7 +148,7 @@ abstract class RangerSparkExtensionSuite extends AnyFunSuite // session1: second query, should auth once.[InMemoryRelation] // (don't need to check in again, but it's okay to check in once) val plan2 = sql(select).queryExecution.optimizedPlan - assert(plan1 != plan2 && plan2.getTagValue(KYUUBI_AUTHZ_TAG).getOrElse(false)) + assert(plan1 != plan2 && plan2.getTagValue(KYUUBI_AUTHZ_TAG).nonEmpty) // session2: should auth once. val otherSessionDf = spark.newSession().sql(select) @@ -170,7 +159,7 @@ abstract class RangerSparkExtensionSuite extends AnyFunSuite // make sure it use cache. assert(plan3.isInstanceOf[InMemoryRelation]) // auth once only. - assert(plan3.getTagValue(KYUUBI_AUTHZ_TAG).getOrElse(false)) + assert(plan3.getTagValue(KYUUBI_AUTHZ_TAG).nonEmpty) }) } } @@ -184,18 +173,18 @@ abstract class RangerSparkExtensionSuite extends AnyFunSuite val e = intercept[AccessControlException](sql(create)) assert(e.getMessage === errorMessage("create", "mydb")) withCleanTmpResources(Seq((testDb, "database"))) { - doAs("admin", assert(Try { sql(create) }.isSuccess)) - doAs("admin", assert(Try { sql(alter) }.isSuccess)) + doAs(admin, assert(Try { sql(create) }.isSuccess)) + doAs(admin, assert(Try { sql(alter) }.isSuccess)) val e1 = intercept[AccessControlException](sql(alter)) assert(e1.getMessage === errorMessage("alter", "mydb")) val e2 = intercept[AccessControlException](sql(drop)) assert(e2.getMessage === errorMessage("drop", "mydb")) - doAs("kent", Try(sql("SHOW DATABASES")).isSuccess) + doAs(kent, Try(sql("SHOW DATABASES")).isSuccess) } } test("auth: tables") { - val db = "default" + val db = defaultDb val table = "src" val col = "key" @@ -207,14 +196,14 @@ abstract class RangerSparkExtensionSuite extends AnyFunSuite assert(e.getMessage === errorMessage("create")) withCleanTmpResources(Seq((s"$db.$table", "table"))) { - doAs("bob", assert(Try { sql(create0) }.isSuccess)) - doAs("bob", assert(Try { sql(alter0) }.isSuccess)) + doAs(bob, assert(Try { sql(create0) }.isSuccess)) + doAs(bob, assert(Try { sql(alter0) }.isSuccess)) val e1 = intercept[AccessControlException](sql(drop0)) assert(e1.getMessage === errorMessage("drop")) - doAs("bob", assert(Try { sql(alter0) }.isSuccess)) - doAs("bob", assert(Try { sql(select).collect() }.isSuccess)) - doAs("kent", assert(Try { sql(s"SELECT key FROM $db.$table").collect() }.isSuccess)) + doAs(bob, assert(Try { sql(alter0) }.isSuccess)) + doAs(bob, assert(Try { sql(select).collect() }.isSuccess)) + doAs(kent, assert(Try { sql(s"SELECT key FROM $db.$table").collect() }.isSuccess)) Seq( select, @@ -225,10 +214,10 @@ abstract class RangerSparkExtensionSuite extends AnyFunSuite s"SELECT key FROM $db.$table WHERE value in (SELECT value as key FROM $db.$table)") .foreach { q => doAs( - "kent", { + kent, { withClue(q) { val e = intercept[AccessControlException](sql(q).collect()) - assert(e.getMessage === errorMessage("select", "default/src/value", "kent")) + assert(e.getMessage === errorMessage("select", "default/src/value", kent)) } }) } @@ -236,221 +225,15 @@ abstract class RangerSparkExtensionSuite extends AnyFunSuite } test("auth: functions") { - val db = "default" + val db = defaultDb val func = "func" val create0 = s"CREATE FUNCTION IF NOT EXISTS $db.$func AS 'abc.mnl.xyz'" doAs( - "kent", { + kent, { val e = intercept[AccessControlException](sql(create0)) assert(e.getMessage === errorMessage("create", "default/func")) }) - doAs("admin", assert(Try(sql(create0)).isSuccess)) - } - - test("row level filter") { - val db = "default" - val table = "src" - val col = "key" - val create = s"CREATE TABLE IF NOT EXISTS $db.$table ($col int, value int) USING $format" - - withCleanTmpResources(Seq((s"$db.${table}2", "table"), (s"$db.$table", "table"))) { - doAs("admin", assert(Try { sql(create) }.isSuccess)) - doAs("admin", sql(s"INSERT INTO $db.$table SELECT 1, 1")) - doAs("admin", sql(s"INSERT INTO $db.$table SELECT 20, 2")) - doAs("admin", sql(s"INSERT INTO $db.$table SELECT 30, 3")) - - doAs( - "kent", - assert(sql(s"SELECT key FROM $db.$table order by key").collect() === - Seq(Row(1), Row(20), Row(30)))) - - Seq( - s"SELECT value FROM $db.$table", - s"SELECT value as key FROM $db.$table", - s"SELECT max(value) FROM $db.$table", - s"SELECT coalesce(max(value), 1) FROM $db.$table", - s"SELECT value FROM $db.$table WHERE value in (SELECT value as key FROM $db.$table)") - .foreach { q => - doAs( - "bob", { - withClue(q) { - assert(sql(q).collect() === Seq(Row(1))) - } - }) - } - doAs( - "bob", { - sql(s"CREATE TABLE $db.src2 using $format AS SELECT value FROM $db.$table") - assert(sql(s"SELECT value FROM $db.${table}2").collect() === Seq(Row(1))) - }) - } - } - - test("[KYUUBI #3581]: row level filter on permanent view") { - assume(isSparkV31OrGreater) - - val db = "default" - val table = "src" - val permView = "perm_view" - val col = "key" - val create = s"CREATE TABLE IF NOT EXISTS $db.$table ($col int, value int) USING $format" - val createView = - s"CREATE OR REPLACE VIEW $db.$permView" + - s" AS SELECT * FROM $db.$table" - - withCleanTmpResources(Seq( - (s"$db.$table", "table"), - (s"$db.$permView", "view"))) { - doAs("admin", assert(Try { sql(create) }.isSuccess)) - doAs("admin", assert(Try { sql(createView) }.isSuccess)) - doAs("admin", sql(s"INSERT INTO $db.$table SELECT 1, 1")) - doAs("admin", sql(s"INSERT INTO $db.$table SELECT 20, 2")) - doAs("admin", sql(s"INSERT INTO $db.$table SELECT 30, 3")) - - Seq( - s"SELECT value FROM $db.$permView", - s"SELECT value as key FROM $db.$permView", - s"SELECT max(value) FROM $db.$permView", - s"SELECT coalesce(max(value), 1) FROM $db.$permView", - s"SELECT value FROM $db.$permView WHERE value in (SELECT value as key FROM $db.$permView)") - .foreach { q => - doAs( - "perm_view_user", { - withClue(q) { - assert(sql(q).collect() === Seq(Row(1))) - } - }) - } - } - } - - test("data masking") { - val db = "default" - val table = "src" - val col = "key" - val create = - s"CREATE TABLE IF NOT EXISTS $db.$table" + - s" ($col int, value1 int, value2 string, value3 string, value4 timestamp, value5 string)" + - s" USING $format" - - withCleanTmpResources(Seq( - (s"$db.${table}2", "table"), - (s"$db.$table", "table"))) { - doAs("admin", assert(Try { sql(create) }.isSuccess)) - doAs( - "admin", - sql( - s"INSERT INTO $db.$table SELECT 1, 1, 'hello', 'world', " + - s"timestamp'2018-11-17 12:34:56', 'World'")) - doAs( - "admin", - sql( - s"INSERT INTO $db.$table SELECT 20, 2, 'kyuubi', 'y', " + - s"timestamp'2018-11-17 12:34:56', 'world'")) - doAs( - "admin", - sql( - s"INSERT INTO $db.$table SELECT 30, 3, 'spark', 'a'," + - s" timestamp'2018-11-17 12:34:56', 'world'")) - - doAs( - "kent", - assert(sql(s"SELECT key FROM $db.$table order by key").collect() === - Seq(Row(1), Row(20), Row(30)))) - - Seq( - s"SELECT value1, value2, value3, value4, value5 FROM $db.$table", - s"SELECT value1 as key, value2, value3, value4, value5 FROM $db.$table", - s"SELECT max(value1), max(value2), max(value3), max(value4), max(value5) FROM $db.$table", - s"SELECT coalesce(max(value1), 1), coalesce(max(value2), 1), coalesce(max(value3), 1), " + - s"coalesce(max(value4), timestamp '2018-01-01 22:33:44'), coalesce(max(value5), 1) " + - s"FROM $db.$table", - s"SELECT value1, value2, value3, value4, value5 FROM $db.$table WHERE value2 in" + - s" (SELECT value2 as key FROM $db.$table)") - .foreach { q => - doAs( - "bob", { - withClue(q) { - assert(sql(q).collect() === - Seq( - Row( - DigestUtils.md5Hex("1"), - "xxxxx", - "worlx", - Timestamp.valueOf("2018-01-01 00:00:00"), - "Xorld"))) - } - }) - } - doAs( - "bob", { - sql(s"CREATE TABLE $db.src2 using $format AS SELECT value1 FROM $db.$table") - assert(sql(s"SELECT value1 FROM $db.${table}2").collect() === - Seq(Row(DigestUtils.md5Hex("1")))) - }) - } - } - - test("[KYUUBI #3581]: data masking on permanent view") { - assume(isSparkV31OrGreater) - - val db = "default" - val table = "src" - val permView = "perm_view" - val col = "key" - val create = - s"CREATE TABLE IF NOT EXISTS $db.$table" + - s" ($col int, value1 int, value2 string)" + - s" USING $format" - - val createView = - s"CREATE OR REPLACE VIEW $db.$permView" + - s" AS SELECT * FROM $db.$table" - - withCleanTmpResources(Seq( - (s"$db.$table", "table"), - (s"$db.$permView", "view"))) { - doAs("admin", assert(Try { sql(create) }.isSuccess)) - doAs("admin", assert(Try { sql(createView) }.isSuccess)) - doAs( - "admin", - sql( - s"INSERT INTO $db.$table SELECT 1, 1, 'hello'")) - - Seq( - s"SELECT value1, value2 FROM $db.$permView") - .foreach { q => - doAs( - "perm_view_user", { - withClue(q) { - assert(sql(q).collect() === - Seq( - Row( - DigestUtils.md5Hex("1"), - "hello"))) - } - }) - } - } - } - - test("KYUUBI #2390: RuleEliminateMarker stays in analyze phase for data masking") { - val db = "default" - val table = "src" - val create = - s"CREATE TABLE IF NOT EXISTS $db.$table (key int, value1 int) USING $format" - - withCleanTmpResources(Seq((s"$db.$table", "table"))) { - doAs("admin", sql(create)) - doAs("admin", sql(s"INSERT INTO $db.$table SELECT 1, 1")) - // scalastyle: off - doAs( - "bob", { - assert(sql(s"select * from $db.$table").collect() === - Seq(Row(1, DigestUtils.md5Hex("1")))) - assert(Try(sql(s"select * from $db.$table").show(1)).isSuccess) - }) - } + doAs(admin, assert(Try(sql(create0)).isSuccess)) } test("show tables") { @@ -461,13 +244,14 @@ abstract class RangerSparkExtensionSuite extends AnyFunSuite (s"$db.$table", "table"), (s"$db.${table}for_show", "table"), (s"$db", "database"))) { - doAs("admin", sql(s"CREATE DATABASE IF NOT EXISTS $db")) - doAs("admin", sql(s"CREATE TABLE IF NOT EXISTS $db.$table (key int) USING $format")) - doAs("admin", sql(s"CREATE TABLE IF NOT EXISTS $db.${table}for_show (key int) USING $format")) - - doAs("admin", assert(sql(s"show tables from $db").collect().length === 2)) - doAs("bob", assert(sql(s"show tables from $db").collect().length === 0)) - doAs("i_am_invisible", assert(sql(s"show tables from $db").collect().length === 0)) + doAs(admin, sql(s"CREATE DATABASE IF NOT EXISTS $db")) + doAs(admin, sql(s"CREATE TABLE IF NOT EXISTS $db.$table (key int) USING $format")) + doAs(admin, sql(s"CREATE TABLE IF NOT EXISTS $db.${table}for_show (key int) USING $format")) + + doAs(admin, assert(sql(s"show tables from $db").collect().length === 2)) + doAs(bob, assert(sql(s"show tables from $db").collect().length === 0)) + doAs(invisibleUser, assert(sql(s"show tables from $db").collect().length === 0)) + doAs(invisibleUser, assert(sql(s"show tables from $db").limit(1).isEmpty)) } } @@ -475,18 +259,19 @@ abstract class RangerSparkExtensionSuite extends AnyFunSuite val db = "default2" withCleanTmpResources(Seq((db, "database"))) { - doAs("admin", sql(s"CREATE DATABASE IF NOT EXISTS $db")) - doAs("admin", assert(sql(s"SHOW DATABASES").collect().length == 2)) - doAs("admin", assert(sql(s"SHOW DATABASES").collectAsList().get(0).getString(0) == "default")) - doAs("admin", assert(sql(s"SHOW DATABASES").collectAsList().get(1).getString(0) == s"$db")) - - doAs("bob", assert(sql(s"SHOW DATABASES").collect().length == 1)) - doAs("bob", assert(sql(s"SHOW DATABASES").collectAsList().get(0).getString(0) == "default")) + doAs(admin, sql(s"CREATE DATABASE IF NOT EXISTS $db")) + doAs(admin, assert(sql(s"SHOW DATABASES").collect().length == 2)) + doAs(admin, assert(sql(s"SHOW DATABASES").collectAsList().get(0).getString(0) == defaultDb)) + doAs(admin, assert(sql(s"SHOW DATABASES").collectAsList().get(1).getString(0) == s"$db")) + + doAs(bob, assert(sql(s"SHOW DATABASES").collect().length == 1)) + doAs(bob, assert(sql(s"SHOW DATABASES").collectAsList().get(0).getString(0) == defaultDb)) + doAs(invisibleUser, assert(sql(s"SHOW DATABASES").limit(1).isEmpty)) } } test("show functions") { - val default = "default" + val default = defaultDb val db3 = "default3" val function1 = "function1" @@ -494,41 +279,41 @@ abstract class RangerSparkExtensionSuite extends AnyFunSuite (s"$default.$function1", "function"), (s"$db3.$function1", "function"), (db3, "database"))) { - doAs("admin", sql(s"CREATE FUNCTION $function1 AS 'Function1'")) - doAs("admin", assert(sql(s"show user functions $default.$function1").collect().length == 1)) - doAs("bob", assert(sql(s"show user functions $default.$function1").collect().length == 0)) + doAs(admin, sql(s"CREATE FUNCTION $function1 AS 'Function1'")) + doAs(admin, assert(sql(s"show user functions $default.$function1").collect().length == 1)) + doAs(bob, assert(sql(s"show user functions $default.$function1").collect().length == 0)) - doAs("admin", sql(s"CREATE DATABASE IF NOT EXISTS $db3")) - doAs("admin", sql(s"CREATE FUNCTION $db3.$function1 AS 'Function1'")) + doAs(admin, sql(s"CREATE DATABASE IF NOT EXISTS $db3")) + doAs(admin, sql(s"CREATE FUNCTION $db3.$function1 AS 'Function1'")) - doAs("admin", assert(sql(s"show user functions $db3.$function1").collect().length == 1)) - doAs("bob", assert(sql(s"show user functions $db3.$function1").collect().length == 0)) + doAs(admin, assert(sql(s"show user functions $db3.$function1").collect().length == 1)) + doAs(bob, assert(sql(s"show user functions $db3.$function1").collect().length == 0)) - doAs("admin", assert(sql(s"show system functions").collect().length > 0)) - doAs("bob", assert(sql(s"show system functions").collect().length > 0)) + doAs(admin, assert(sql(s"show system functions").collect().length > 0)) + doAs(bob, assert(sql(s"show system functions").collect().length > 0)) - val adminSystemFunctionCount = doAs("admin", sql(s"show system functions").collect().length) - val bobSystemFunctionCount = doAs("bob", sql(s"show system functions").collect().length) + val adminSystemFunctionCount = doAs(admin, sql(s"show system functions").collect().length) + val bobSystemFunctionCount = doAs(bob, sql(s"show system functions").collect().length) assert(adminSystemFunctionCount == bobSystemFunctionCount) } } test("show columns") { - val db = "default" + val db = defaultDb val table = "src" val col = "key" val create = s"CREATE TABLE IF NOT EXISTS $db.$table ($col int, value int) USING $format" withCleanTmpResources(Seq((s"$db.$table", "table"))) { - doAs("admin", sql(create)) + doAs(admin, sql(create)) - doAs("admin", assert(sql(s"SHOW COLUMNS IN $table").count() == 2)) - doAs("admin", assert(sql(s"SHOW COLUMNS IN $db.$table").count() == 2)) - doAs("admin", assert(sql(s"SHOW COLUMNS IN $table IN $db").count() == 2)) + doAs(admin, assert(sql(s"SHOW COLUMNS IN $table").count() == 2)) + doAs(admin, assert(sql(s"SHOW COLUMNS IN $db.$table").count() == 2)) + doAs(admin, assert(sql(s"SHOW COLUMNS IN $table IN $db").count() == 2)) - doAs("kent", assert(sql(s"SHOW COLUMNS IN $table").count() == 1)) - doAs("kent", assert(sql(s"SHOW COLUMNS IN $db.$table").count() == 1)) - doAs("kent", assert(sql(s"SHOW COLUMNS IN $table IN $db").count() == 1)) + doAs(kent, assert(sql(s"SHOW COLUMNS IN $table").count() == 1)) + doAs(kent, assert(sql(s"SHOW COLUMNS IN $db.$table").count() == 1)) + doAs(kent, assert(sql(s"SHOW COLUMNS IN $table IN $db").count() == 1)) } } @@ -543,24 +328,24 @@ abstract class RangerSparkExtensionSuite extends AnyFunSuite (s"$db.${table}_select2", "table"), (s"$db.${table}_select3", "table"), (s"$db", "database"))) { - doAs("admin", sql(s"CREATE DATABASE IF NOT EXISTS $db")) - doAs("admin", sql(s"CREATE TABLE IF NOT EXISTS $db.${table}_use1 (key int) USING $format")) - doAs("admin", sql(s"CREATE TABLE IF NOT EXISTS $db.${table}_use2 (key int) USING $format")) - doAs("admin", sql(s"CREATE TABLE IF NOT EXISTS $db.${table}_select1 (key int) USING $format")) - doAs("admin", sql(s"CREATE TABLE IF NOT EXISTS $db.${table}_select2 (key int) USING $format")) - doAs("admin", sql(s"CREATE TABLE IF NOT EXISTS $db.${table}_select3 (key int) USING $format")) + doAs(admin, sql(s"CREATE DATABASE IF NOT EXISTS $db")) + doAs(admin, sql(s"CREATE TABLE IF NOT EXISTS $db.${table}_use1 (key int) USING $format")) + doAs(admin, sql(s"CREATE TABLE IF NOT EXISTS $db.${table}_use2 (key int) USING $format")) + doAs(admin, sql(s"CREATE TABLE IF NOT EXISTS $db.${table}_select1 (key int) USING $format")) + doAs(admin, sql(s"CREATE TABLE IF NOT EXISTS $db.${table}_select2 (key int) USING $format")) + doAs(admin, sql(s"CREATE TABLE IF NOT EXISTS $db.${table}_select3 (key int) USING $format")) doAs( - "admin", + admin, assert(sql(s"show table extended from $db like '$table*'").collect().length === 5)) doAs( - "bob", + bob, assert(sql(s"show tables from $db").collect().length === 5)) doAs( - "bob", + bob, assert(sql(s"show table extended from $db like '$table*'").collect().length === 3)) doAs( - "i_am_invisible", + invisibleUser, assert(sql(s"show table extended from $db like '$table*'").collect().length === 0)) } } @@ -572,48 +357,48 @@ abstract class RangerSparkExtensionSuite extends AnyFunSuite val globalTempView2 = "global_temp_view2" // create or replace view - doAs("denyuser", sql(s"CREATE TEMPORARY VIEW $tempView AS select * from values(1)")) + doAs(denyUser, sql(s"CREATE TEMPORARY VIEW $tempView AS select * from values(1)")) doAs( - "denyuser", + denyUser, sql(s"CREATE GLOBAL TEMPORARY VIEW $globalTempView AS SELECT * FROM values(1)")) // rename view - doAs("denyuser2", sql(s"ALTER VIEW $tempView RENAME TO $tempView2")) + doAs(denyUser2, sql(s"ALTER VIEW $tempView RENAME TO $tempView2")) doAs( - "denyuser2", + denyUser2, sql(s"ALTER VIEW global_temp.$globalTempView RENAME TO global_temp.$globalTempView2")) - doAs("admin", sql(s"DROP VIEW IF EXISTS $tempView2")) - doAs("admin", sql(s"DROP VIEW IF EXISTS global_temp.$globalTempView2")) - doAs("admin", assert(sql("show tables from global_temp").collect().length == 0)) + doAs(admin, sql(s"DROP VIEW IF EXISTS $tempView2")) + doAs(admin, sql(s"DROP VIEW IF EXISTS global_temp.$globalTempView2")) + doAs(admin, assert(sql("show tables from global_temp").collect().length == 0)) } test("[KYUUBI #3426] Drop temp view should be skipped permission check") { val tempView = "temp_view" val globalTempView = "global_temp_view" - doAs("denyuser", sql(s"CREATE TEMPORARY VIEW $tempView AS select * from values(1)")) + doAs(denyUser, sql(s"CREATE TEMPORARY VIEW $tempView AS select * from values(1)")) doAs( - "denyuser", + denyUser, sql(s"CREATE OR REPLACE TEMPORARY VIEW $tempView" + s" AS select * from values(1)")) doAs( - "denyuser", + denyUser, sql(s"CREATE GLOBAL TEMPORARY VIEW $globalTempView AS SELECT * FROM values(1)")) doAs( - "denyuser", + denyUser, sql(s"CREATE OR REPLACE GLOBAL TEMPORARY VIEW $globalTempView" + s" AS select * from values(1)")) // global_temp will contain the temporary view, even if it is not global - doAs("admin", assert(sql("show tables from global_temp").collect().length == 2)) + doAs(admin, assert(sql("show tables from global_temp").collect().length == 2)) - doAs("denyuser2", sql(s"DROP VIEW IF EXISTS $tempView")) - doAs("denyuser2", sql(s"DROP VIEW IF EXISTS global_temp.$globalTempView")) + doAs(denyUser2, sql(s"DROP VIEW IF EXISTS $tempView")) + doAs(denyUser2, sql(s"DROP VIEW IF EXISTS global_temp.$globalTempView")) - doAs("admin", assert(sql("show tables from global_temp").collect().length == 0)) + doAs(admin, assert(sql("show tables from global_temp").collect().length == 0)) } test("[KYUUBI #3428] AlterViewAsCommand should be skipped permission check") { @@ -621,26 +406,26 @@ abstract class RangerSparkExtensionSuite extends AnyFunSuite val globalTempView = "global_temp_view" // create or replace view - doAs("denyuser", sql(s"CREATE TEMPORARY VIEW $tempView AS select * from values(1)")) + doAs(denyUser, sql(s"CREATE TEMPORARY VIEW $tempView AS select * from values(1)")) doAs( - "denyuser", + denyUser, sql(s"CREATE OR REPLACE TEMPORARY VIEW $tempView" + s" AS select * from values(1)")) doAs( - "denyuser", + denyUser, sql(s"CREATE GLOBAL TEMPORARY VIEW $globalTempView AS SELECT * FROM values(1)")) doAs( - "denyuser", + denyUser, sql(s"CREATE OR REPLACE GLOBAL TEMPORARY VIEW $globalTempView" + s" AS select * from values(1)")) // rename view - doAs("denyuser2", sql(s"ALTER VIEW $tempView AS SELECT * FROM values(1)")) - doAs("denyuser2", sql(s"ALTER VIEW global_temp.$globalTempView AS SELECT * FROM values(1)")) + doAs(denyUser2, sql(s"ALTER VIEW $tempView AS SELECT * FROM values(1)")) + doAs(denyUser2, sql(s"ALTER VIEW global_temp.$globalTempView AS SELECT * FROM values(1)")) - doAs("admin", sql(s"DROP VIEW IF EXISTS $tempView")) - doAs("admin", sql(s"DROP VIEW IF EXISTS global_temp.$globalTempView")) - doAs("admin", assert(sql("show tables from global_temp").collect().length == 0)) + doAs(admin, sql(s"DROP VIEW IF EXISTS $tempView")) + doAs(admin, sql(s"DROP VIEW IF EXISTS global_temp.$globalTempView")) + doAs(admin, assert(sql("show tables from global_temp").collect().length == 0)) } test("[KYUUBI #3343] pass temporary view creation") { @@ -649,28 +434,39 @@ abstract class RangerSparkExtensionSuite extends AnyFunSuite withTempView(tempView) { doAs( - "denyuser", + denyUser, assert(Try(sql(s"CREATE TEMPORARY VIEW $tempView AS select * from values(1)")).isSuccess)) doAs( - "denyuser", + denyUser, Try(sql(s"CREATE OR REPLACE TEMPORARY VIEW $tempView" + s" AS select * from values(1)")).isSuccess) } withGlobalTempView(globalTempView) { doAs( - "denyuser", + denyUser, Try( sql( s"CREATE GLOBAL TEMPORARY VIEW $globalTempView AS SELECT * FROM values(1)")).isSuccess) doAs( - "denyuser", + denyUser, Try(sql(s"CREATE OR REPLACE GLOBAL TEMPORARY VIEW $globalTempView" + s" AS select * from values(1)")).isSuccess) } - doAs("admin", assert(sql("show tables from global_temp").collect().length == 0)) + doAs(admin, assert(sql("show tables from global_temp").collect().length == 0)) + } + + test("[KYUUBI #5172] Check USE permissions for DESCRIBE FUNCTION") { + val fun = s"$defaultDb.function1" + + withCleanTmpResources(Seq((s"$fun", "function"))) { + doAs(admin, sql(s"CREATE FUNCTION $fun AS 'Function1'")) + doAs(admin, sql(s"DESC FUNCTION $fun").collect().length == 1) + val e = intercept[AccessControlException](doAs(denyUser, sql(s"DESC FUNCTION $fun"))) + assert(e.getMessage === errorMessage("_any", "default/function1", denyUser)) + } } } @@ -680,16 +476,15 @@ class InMemoryCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSuite class HiveCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSuite { override protected val catalogImpl: String = "hive" - test("table stats must be specified") { val table = "hive_src" withCleanTmpResources(Seq((table, "table"))) { - doAs("admin", sql(s"CREATE TABLE IF NOT EXISTS $table (id int)")) + doAs(admin, sql(s"CREATE TABLE IF NOT EXISTS $table (id int)")) doAs( - "admin", { + admin, { val hiveTableRelation = sql(s"SELECT * FROM $table") .queryExecution.optimizedPlan.collectLeaves().head.asInstanceOf[HiveTableRelation] - assert(getFieldVal[Option[Statistics]](hiveTableRelation, "tableStats").nonEmpty) + assert(getField[Option[Statistics]](hiveTableRelation, "tableStats").nonEmpty) }) } } @@ -697,9 +492,9 @@ class HiveCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSuite { test("HiveTableRelation should be able to be converted to LogicalRelation") { val table = "hive_src" withCleanTmpResources(Seq((table, "table"))) { - doAs("admin", sql(s"CREATE TABLE IF NOT EXISTS $table (id int) STORED AS PARQUET")) + doAs(admin, sql(s"CREATE TABLE IF NOT EXISTS $table (id int) STORED AS PARQUET")) doAs( - "admin", { + admin, { val relation = sql(s"SELECT * FROM $table") .queryExecution.optimizedPlan.collectLeaves().head assert(relation.isInstanceOf[LogicalRelation]) @@ -717,7 +512,7 @@ class HiveCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSuite { (s"$db.$table1", "table"), (s"$db", "database"))) { doAs( - "admin", { + admin, { sql(s"CREATE DATABASE IF NOT EXISTS $db") sql(s"CREATE TABLE IF NOT EXISTS $db.$table1(id int) STORED AS PARQUET") sql(s"INSERT INTO $db.$table1 SELECT 1") @@ -738,16 +533,16 @@ class HiveCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSuite { (adminPermView, "view"), (permView, "view"), (table, "table"))) { - doAs("admin", sql(s"CREATE TABLE IF NOT EXISTS $table (id int)")) + doAs(admin, sql(s"CREATE TABLE IF NOT EXISTS $table (id int)")) - doAs("admin", sql(s"CREATE VIEW ${adminPermView} AS SELECT * FROM $table")) + doAs(admin, sql(s"CREATE VIEW ${adminPermView} AS SELECT * FROM $table")) val e1 = intercept[AccessControlException]( - doAs("someone", sql(s"CREATE VIEW $permView AS SELECT 1 as a"))) + doAs(someone, sql(s"CREATE VIEW $permView AS SELECT 1 as a"))) assert(e1.getMessage.contains(s"does not have [create] privilege on [default/$permView]")) val e2 = intercept[AccessControlException]( - doAs("someone", sql(s"CREATE VIEW $permView AS SELECT * FROM $table"))) + doAs(someone, sql(s"CREATE VIEW $permView AS SELECT * FROM $table"))) if (isSparkV32OrGreater) { assert(e2.getMessage.contains(s"does not have [select] privilege on [default/$table/id]")) } else { @@ -757,32 +552,52 @@ class HiveCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSuite { } test("[KYUUBI #3326] check persisted view and skip shadowed table") { + val db1 = defaultDb val table = "hive_src" val permView = "perm_view" - val db1 = "default" - val db2 = "db2" withCleanTmpResources(Seq( (s"$db1.$table", "table"), - (s"$db2.$permView", "view"), - (db2, "database"))) { - doAs("admin", sql(s"CREATE TABLE IF NOT EXISTS $db1.$table (id int)")) - - doAs("admin", sql(s"CREATE DATABASE IF NOT EXISTS $db2")) - doAs("admin", sql(s"CREATE VIEW $db2.$permView AS SELECT * FROM $table")) + (s"$db1.$permView", "view"))) { + doAs(admin, sql(s"CREATE TABLE IF NOT EXISTS $db1.$table (id int, name string)")) + doAs(admin, sql(s"CREATE VIEW $db1.$permView AS SELECT * FROM $db1.$table")) + // KYUUBI #3326: with no privileges to the permanent view or the source table val e1 = intercept[AccessControlException]( - doAs("someone", sql(s"select * from $db2.$permView")).show(0)) - if (isSparkV31OrGreater) { - assert(e1.getMessage.contains(s"does not have [select] privilege on [$db2/$permView/id]")) - } else { - assert(e1.getMessage.contains(s"does not have [select] privilege on [$db1/$table/id]")) - } + doAs( + someone, { + sql(s"select * from $db1.$permView").collect() + })) + assert(e1.getMessage.contains(s"does not have [select] privilege on [$db1/$permView/id]")) + } + } + + test("KYUUBI #4504: query permanent view with privilege to permanent view only") { + val db1 = defaultDb + val table = "hive_src" + val permView = "perm_view" + val userPermViewOnly = permViewOnlyUser + + withCleanTmpResources(Seq( + (s"$db1.$table", "table"), + (s"$db1.$permView", "view"))) { + doAs(admin, sql(s"CREATE TABLE IF NOT EXISTS $db1.$table (id int, name string)")) + doAs(admin, sql(s"CREATE VIEW $db1.$permView AS SELECT * FROM $db1.$table")) + + // query all columns of the permanent view + // with access privileges to the permanent view but no privilege to the source table + val sql1 = s"SELECT * FROM $db1.$permView" + doAs(userPermViewOnly, { sql(sql1).collect() }) + + // query the second column of permanent view with multiple columns + // with access privileges to the permanent view but no privilege to the source table + val sql2 = s"SELECT name FROM $db1.$permView" + doAs(userPermViewOnly, { sql(sql2).collect() }) } } test("[KYUUBI #3371] support throws all disallowed privileges in exception") { - val db1 = "default" + val db1 = defaultDb val srcTable1 = "hive_src1" val srcTable2 = "hive_src2" val sinkTable1 = "hive_sink1" @@ -792,17 +607,17 @@ class HiveCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSuite { (s"$db1.$srcTable2", "table"), (s"$db1.$sinkTable1", "table"))) { doAs( - "admin", + admin, sql(s"CREATE TABLE IF NOT EXISTS $db1.$srcTable1" + s" (id int, name string, city string)")) doAs( - "admin", + admin, sql(s"CREATE TABLE IF NOT EXISTS $db1.$srcTable2" + s" (id int, age int)")) doAs( - "admin", + admin, sql(s"CREATE TABLE IF NOT EXISTS $db1.$sinkTable1" + s" (id int, age int, name string, city string)")) @@ -811,25 +626,17 @@ class HiveCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSuite { s" FROM $db1.$srcTable1 as tb1" + s" JOIN $db1.$srcTable2 as tb2" + s" on tb1.id = tb2.id" - val e1 = intercept[AccessControlException](doAs("someone", sql(insertSql1))) + val e1 = intercept[AccessControlException](doAs(someone, sql(insertSql1))) assert(e1.getMessage.contains(s"does not have [select] privilege on [$db1/$srcTable1/id]")) - try { - SparkRangerAdminPlugin.getRangerConf.setBoolean( - s"ranger.plugin.${SparkRangerAdminPlugin.getServiceType}.authorize.in.single.call", - true) - val e2 = intercept[AccessControlException](doAs("someone", sql(insertSql1))) + withSingleCallEnabled { + val e2 = intercept[AccessControlException](doAs(someone, sql(insertSql1))) assert(e2.getMessage.contains(s"does not have" + s" [select] privilege on" + s" [$db1/$srcTable1/id,$db1/$srcTable1/name,$db1/$srcTable1/city," + s"$db1/$srcTable2/age,$db1/$srcTable2/id]," + s" [update] privilege on [$db1/$sinkTable1/id,$db1/$sinkTable1/age," + s"$db1/$sinkTable1/name,$db1/$sinkTable1/city]")) - } finally { - // revert to default value - SparkRangerAdminPlugin.getRangerConf.setBoolean( - s"ranger.plugin.${SparkRangerAdminPlugin.getServiceType}.authorize.in.single.call", - false) } } } @@ -837,7 +644,7 @@ class HiveCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSuite { test("[KYUUBI #3411] skip checking cache table") { if (isSparkV32OrGreater) { // cache table sql supported since 3.2.0 - val db1 = "default" + val db1 = defaultDb val srcTable1 = "hive_src1" val cacheTable1 = "cacheTable1" val cacheTable2 = "cacheTable2" @@ -852,23 +659,23 @@ class HiveCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSuite { (s"$db1.$cacheTable4", "cache"))) { doAs( - "admin", + admin, sql(s"CREATE TABLE IF NOT EXISTS $db1.$srcTable1" + s" (id int, name string, city string)")) val e1 = intercept[AccessControlException]( - doAs("someone", sql(s"CACHE TABLE $cacheTable2 select * from $db1.$srcTable1"))) + doAs(someone, sql(s"CACHE TABLE $cacheTable2 select * from $db1.$srcTable1"))) assert( e1.getMessage.contains(s"does not have [select] privilege on [$db1/$srcTable1/id]")) - doAs("admin", sql(s"CACHE TABLE $cacheTable3 SELECT 1 AS a, 2 AS b ")) - doAs("someone", sql(s"CACHE TABLE $cacheTable4 select 1 as a, 2 as b ")) + doAs(admin, sql(s"CACHE TABLE $cacheTable3 SELECT 1 AS a, 2 AS b ")) + doAs(someone, sql(s"CACHE TABLE $cacheTable4 select 1 as a, 2 as b ")) } } } test("[KYUUBI #3608] Support {OWNER} variable for queries") { - val db = "default" + val db = defaultDb val table = "owner_variable" val select = s"SELECT key FROM $db.$table" @@ -887,7 +694,7 @@ class HiveCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSuite { }.isSuccess)) doAs( - "create_only_user", { + createOnlyUser, { val e = intercept[AccessControlException](sql(select).collect()) assert(e.getMessage === errorMessage("select", s"$db/$table/key")) }) @@ -901,10 +708,328 @@ class HiveCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSuite { Seq( (s"$db.$table", "table"), (s"$db", "database"))) { - doAs("admin", sql(s"CREATE DATABASE IF NOT EXISTS $db")) - doAs("admin", sql(s"CREATE TABLE IF NOT EXISTS $db.$table (key int) USING $format")) + doAs(admin, sql(s"CREATE DATABASE IF NOT EXISTS $db")) + doAs(admin, sql(s"CREATE TABLE IF NOT EXISTS $db.$table (key int) USING $format")) sql("SHOW DATABASES").queryExecution.optimizedPlan.stats sql(s"SHOW TABLES IN $db").queryExecution.optimizedPlan.stats } } + + test("[KYUUBI #4658] insert overwrite hive directory") { + val db1 = defaultDb + val table = "src" + + withCleanTmpResources(Seq((s"$db1.$table", "table"))) { + doAs(admin, sql(s"CREATE TABLE IF NOT EXISTS $db1.$table (id int, name string)")) + val e = intercept[AccessControlException]( + doAs( + someone, + sql( + s"""INSERT OVERWRITE DIRECTORY '/tmp/test_dir' ROW FORMAT DELIMITED FIELDS + | TERMINATED BY ',' + | SELECT * FROM $db1.$table;""".stripMargin))) + assert(e.getMessage.contains(s"does not have [select] privilege on [$db1/$table/id]")) + } + } + + test("[KYUUBI #4658] insert overwrite datasource directory") { + val db1 = defaultDb + val table = "src" + + withCleanTmpResources(Seq((s"$db1.$table", "table"))) { + doAs(admin, sql(s"CREATE TABLE IF NOT EXISTS $db1.$table (id int, name string)")) + val e = intercept[AccessControlException]( + doAs( + someone, + sql( + s"""INSERT OVERWRITE DIRECTORY '/tmp/test_dir' + | USING parquet + | SELECT * FROM $db1.$table;""".stripMargin))) + assert(e.getMessage.contains(s"does not have [select] privilege on [$db1/$table/id]")) + } + } + + test("[KYUUBI #5417] should not check scalar-subquery in permanent view") { + val db1 = defaultDb + val table1 = "table1" + val table2 = "table2" + val view1 = "view1" + withCleanTmpResources( + Seq((s"$db1.$table1", "table"), (s"$db1.$table2", "table"), (s"$db1.$view1", "view"))) { + doAs(admin, sql(s"CREATE TABLE IF NOT EXISTS $db1.$table1 (id int, scope int)")) + doAs(admin, sql(s"CREATE TABLE IF NOT EXISTS $db1.$table2 (id int, scope int)")) + + val e1 = intercept[AccessControlException] { + doAs( + someone, + sql( + s""" + |WITH temp AS ( + | SELECT max(scope) max_scope + | FROM $db1.$table1) + |SELECT id as new_id FROM $db1.$table2 + |WHERE scope = (SELECT max_scope FROM temp) + |""".stripMargin).show()) + } + // Will first check subquery privilege. + assert(e1.getMessage.contains(s"does not have [select] privilege on [$db1/$table1/scope]")) + + doAs( + admin, + sql( + s""" + |CREATE VIEW $db1.$view1 + |AS + |WITH temp AS ( + | SELECT max(scope) max_scope + | FROM $db1.$table1) + |SELECT id as new_id FROM $db1.$table2 + |WHERE scope = (SELECT max_scope FROM temp) + |""".stripMargin)) + // Will just check permanent view privilege. + val e2 = intercept[AccessControlException]( + doAs(someone, sql(s"SELECT * FROM $db1.$view1".stripMargin).show())) + assert(e2.getMessage.contains(s"does not have [select] privilege on [$db1/$view1/new_id]")) + } + } + + test("[KYUUBI #5417] should not check in-subquery in permanent view") { + val db1 = defaultDb + val table1 = "table1" + val table2 = "table2" + val view1 = "view1" + withCleanTmpResources( + Seq((s"$db1.$table1", "table"), (s"$db1.$table2", "table"), (s"$db1.$view1", "view"))) { + doAs(admin, sql(s"CREATE TABLE IF NOT EXISTS $db1.$table1 (id int, scope int)")) + doAs(admin, sql(s"CREATE TABLE IF NOT EXISTS $db1.$table2 (id int, scope int)")) + + val e1 = intercept[AccessControlException] { + doAs( + someone, + sql( + s""" + |WITH temp AS ( + | SELECT max(scope) max_scope + | FROM $db1.$table1) + |SELECT id as new_id FROM $db1.$table2 + |WHERE scope in (SELECT max_scope FROM temp) + |""".stripMargin).show()) + } + // Will first check subquery privilege. + assert(e1.getMessage.contains(s"does not have [select] privilege on [$db1/$table1/scope]")) + + doAs( + admin, + sql( + s""" + |CREATE VIEW $db1.$view1 + |AS + |WITH temp AS ( + | SELECT max(scope) max_scope + | FROM $db1.$table1) + |SELECT id as new_id FROM $db1.$table2 + |WHERE scope in (SELECT max_scope FROM temp) + |""".stripMargin)) + // Will just check permanent view privilege. + val e2 = intercept[AccessControlException]( + doAs(someone, sql(s"SELECT * FROM $db1.$view1".stripMargin).show())) + assert(e2.getMessage.contains(s"does not have [select] privilege on [$db1/$view1/new_id]")) + } + } + + test("[KYUUBI #5475] Check permanent view's subquery should check view's correct privilege") { + val db1 = defaultDb + val table1 = "table1" + val table2 = "table2" + val view1 = "view1" + withSingleCallEnabled { + withCleanTmpResources( + Seq((s"$db1.$table1", "table"), (s"$db1.$table2", "table"), (s"$db1.$view1", "view"))) { + doAs(admin, sql(s"CREATE TABLE IF NOT EXISTS $db1.$table1(id int, scope int)")) + doAs( + admin, + sql( + s""" + | CREATE TABLE IF NOT EXISTS $db1.$table2( + | id int, + | name string, + | age int, + | scope int) + | """.stripMargin)) + doAs( + admin, + sql( + s""" + |CREATE VIEW $db1.$view1 + |AS + |WITH temp AS ( + | SELECT max(scope) max_scope + | FROM $db1.$table1) + |SELECT id, name, max(scope) as max_scope, sum(age) sum_age + |FROM $db1.$table2 + |WHERE scope in (SELECT max_scope FROM temp) + |GROUP BY id, name + |""".stripMargin)) + // Will just check permanent view privilege. + val e2 = intercept[AccessControlException]( + doAs( + someone, + sql(s"SELECT id as new_id, name, max_scope FROM $db1.$view1".stripMargin).show())) + assert(e2.getMessage.contains( + s"does not have [select] privilege on " + + s"[$db1/$view1/id,$db1/$view1/name,$db1/$view1/max_scope,$db1/$view1/sum_age]")) + } + } + } + + test("[KYUUBI #5492] saveAsTable create DataSource table miss db info") { + val table1 = "table1" + withSingleCallEnabled { + withCleanTmpResources(Seq.empty) { + val df = doAs( + admin, + sql(s"SELECT * FROM VALUES(1, 100),(2, 200),(3, 300) AS t(id, scope)")).persist() + interceptContains[AccessControlException]( + doAs(someone, df.write.mode("overwrite").saveAsTable(table1)))( + s"does not have [create] privilege on [$defaultDb/$table1]") + } + } + } + + test("[KYUUBI #5472] Permanent View should pass column when child plan no output ") { + val db1 = defaultDb + val table1 = "table1" + val view1 = "view1" + val view2 = "view2" + withSingleCallEnabled { + withCleanTmpResources( + Seq((s"$db1.$table1", "table"), (s"$db1.$view1", "view"), (s"$db1.$view2", "view"))) { + doAs(admin, sql(s"CREATE TABLE IF NOT EXISTS $db1.$table1 (id int, scope int)")) + doAs(admin, sql(s"CREATE VIEW $db1.$view1 AS SELECT * FROM $db1.$table1")) + doAs( + admin, + sql( + s""" + |CREATE VIEW $db1.$view2 + |AS + |SELECT count(*) as cnt, sum(id) as sum_id FROM $db1.$table1 + """.stripMargin)) + interceptContains[AccessControlException]( + doAs(someone, sql(s"SELECT count(*) FROM $db1.$table1").show()))( + s"does not have [select] privilege on [$db1/$table1/id,$db1/$table1/scope]") + + interceptContains[AccessControlException]( + doAs(someone, sql(s"SELECT count(*) FROM $db1.$view1").show()))( + s"does not have [select] privilege on [$db1/$view1/id,$db1/$view1/scope]") + + interceptContains[AccessControlException]( + doAs(someone, sql(s"SELECT count(*) FROM $db1.$view2").show()))( + s"does not have [select] privilege on [$db1/$view2/cnt,$db1/$view2/sum_id]") + + interceptContains[AccessControlException]( + doAs(someone, sql(s"SELECT count(id) FROM $db1.$table1 WHERE id > 10").show()))( + s"does not have [select] privilege on [$db1/$table1/id]") + + interceptContains[AccessControlException]( + doAs(someone, sql(s"SELECT count(id) FROM $db1.$view1 WHERE id > 10").show()))( + s"does not have [select] privilege on [$db1/$view1/id]") + + interceptContains[AccessControlException]( + doAs(someone, sql(s"SELECT count(sum_id) FROM $db1.$view2 WHERE sum_id > 10").show()))( + s"does not have [select] privilege on [$db1/$view2/sum_id]") + + interceptContains[AccessControlException]( + doAs(someone, sql(s"SELECT count(scope) FROM $db1.$table1 WHERE id > 10").show()))( + s"does not have [select] privilege on [$db1/$table1/scope,$db1/$table1/id]") + + interceptContains[AccessControlException]( + doAs(someone, sql(s"SELECT count(scope) FROM $db1.$view1 WHERE id > 10").show()))( + s"does not have [select] privilege on [$db1/$view1/scope,$db1/$view1/id]") + + interceptContains[AccessControlException]( + doAs(someone, sql(s"SELECT count(cnt) FROM $db1.$view2 WHERE sum_id > 10").show()))( + s"does not have [select] privilege on [$db1/$view2/cnt,$db1/$view2/sum_id]") + } + } + } + + test("[KYUUBI #5503][AUTHZ] Check plan auth checked should not set tag to all child nodes") { + assume(isSparkV32OrGreater, "Spark 3.1 not support lateral subquery.") + val db1 = defaultDb + val table1 = "table1" + val table2 = "table2" + val perm_view = "perm_view" + withSingleCallEnabled { + withCleanTmpResources( + Seq( + (s"$db1.$table1", "table"), + (s"$db1.$table2", "table"), + (s"$db1.$perm_view", "view"))) { + doAs(admin, sql(s"CREATE TABLE IF NOT EXISTS $db1.$table1 (id int, scope int)")) + doAs(admin, sql(s"CREATE TABLE IF NOT EXISTS $db1.$table2 (id int, scope int)")) + doAs(admin, sql(s"CREATE VIEW $db1.$perm_view AS SELECT * FROM $db1.$table2")) + interceptContains[AccessControlException]( + doAs( + someone, + sql( + s""" + |SELECT t1.id + |FROM $db1.$table1 t1, + |LATERAL ( + | SELECT * + | FROM $db1.$perm_view t2 + | WHERE t1.id = t2.id + |) + |""".stripMargin).show()))( + s"does not have [select] privilege on " + + s"[$db1/$perm_view/id,$db1/$perm_view/scope]") + interceptContains[AccessControlException]( + doAs( + permViewOnlyUser, + sql( + s""" + |SELECT t1.id + |FROM $db1.$table1 t1, + |LATERAL ( + | SELECT * + | FROM $db1.$perm_view t2 + | WHERE t1.id = t2.id + |) + |""".stripMargin).show()))( + s"does not have [select] privilege on " + + s"[$db1/$table1/id]") + + interceptContains[AccessControlException]( + doAs( + someone, + sql( + s""" + |SELECT t1.id + |FROM $db1.$table1 t1, + |LATERAL ( + | SELECT * + | FROM $db1.$table2 t2 + | WHERE t1.id = t2.id + |) + |""".stripMargin).show()))( + s"does not have [select] privilege on " + + s"[$db1/$table2/id,$db1/$table2/scope]") + interceptContains[AccessControlException]( + doAs( + table2OnlyUser, + sql( + s""" + |SELECT t1.id + |FROM $db1.$table1 t1, + |LATERAL ( + | SELECT * + | FROM $db1.$table2 t2 + | WHERE t1.id = t2.id + |) + |""".stripMargin).show()))( + s"does not have [select] privilege on " + + s"[$db1/$table1/id]") + } + } + } } diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/SparkRangerAdminPluginSuite.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/SparkRangerAdminPluginSuite.scala index 8711a7287..301ae87c5 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/SparkRangerAdminPluginSuite.scala +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/SparkRangerAdminPluginSuite.scala @@ -22,6 +22,8 @@ import org.apache.hadoop.security.UserGroupInformation import org.scalatest.funsuite.AnyFunSuite import org.apache.kyuubi.plugin.spark.authz.{ObjectType, OperationType} +import org.apache.kyuubi.plugin.spark.authz.RangerTestNamespace._ +import org.apache.kyuubi.plugin.spark.authz.RangerTestUsers._ import org.apache.kyuubi.plugin.spark.authz.ranger.SparkRangerAdminPlugin._ class SparkRangerAdminPluginSuite extends AnyFunSuite { @@ -29,13 +31,13 @@ class SparkRangerAdminPluginSuite extends AnyFunSuite { test("get filter expression") { val bob = UserGroupInformation.createRemoteUser("bob") - val are = AccessResource(ObjectType.TABLE, "default", "src", null) + val are = AccessResource(ObjectType.TABLE, defaultDb, "src", null) def buildAccessRequest(ugi: UserGroupInformation): AccessRequest = { AccessRequest(are, ugi, OperationType.QUERY, AccessType.SELECT) } val maybeString = getFilterExpr(buildAccessRequest(bob)) assert(maybeString.get === "key<20") - Seq("admin", "alice").foreach { user => + Seq(admin, alice).foreach { user => val ugi = UserGroupInformation.createRemoteUser(user) val maybeString = getFilterExpr(buildAccessRequest(ugi)) assert(maybeString.isEmpty) @@ -45,18 +47,21 @@ class SparkRangerAdminPluginSuite extends AnyFunSuite { test("get data masker") { val bob = UserGroupInformation.createRemoteUser("bob") def buildAccessRequest(ugi: UserGroupInformation, column: String): AccessRequest = { - val are = AccessResource(ObjectType.COLUMN, "default", "src", column) + val are = AccessResource(ObjectType.COLUMN, defaultDb, "src", column) AccessRequest(are, ugi, OperationType.QUERY, AccessType.SELECT) } assert(getMaskingExpr(buildAccessRequest(bob, "value1")).get === "md5(cast(value1 as string))") assert(getMaskingExpr(buildAccessRequest(bob, "value2")).get === - "regexp_replace(regexp_replace(regexp_replace(value2, '[A-Z]', 'X'), '[a-z]', 'x')," + - " '[0-9]', 'n')") + "regexp_replace(regexp_replace(regexp_replace(regexp_replace(value2, '[A-Z]', 'X')," + + " '[a-z]', 'x'), '[0-9]', 'n'), '[^A-Za-z0-9]', 'U')") assert(getMaskingExpr(buildAccessRequest(bob, "value3")).get contains "regexp_replace") assert(getMaskingExpr(buildAccessRequest(bob, "value4")).get === "date_trunc('YEAR', value4)") - assert(getMaskingExpr(buildAccessRequest(bob, "value5")).get contains "regexp_replace") + assert(getMaskingExpr(buildAccessRequest(bob, "value5")).get === + "concat(regexp_replace(regexp_replace(regexp_replace(regexp_replace(" + + "left(value5, length(value5) - 4), '[A-Z]', 'X'), '[a-z]', 'x')," + + " '[0-9]', 'n'), '[^A-Za-z0-9]', 'U'), right(value5, 4))") - Seq("admin", "alice").foreach { user => + Seq(admin, alice).foreach { user => val ugi = UserGroupInformation.createRemoteUser(user) val maybeString = getMaskingExpr(buildAccessRequest(ugi, "value1")) assert(maybeString.isEmpty) diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/V2JdbcTableCatalogRangerSparkExtensionSuite.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/V2JdbcTableCatalogRangerSparkExtensionSuite.scala index 6bdab9d9d..046052d55 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/V2JdbcTableCatalogRangerSparkExtensionSuite.scala +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/V2JdbcTableCatalogRangerSparkExtensionSuite.scala @@ -22,6 +22,10 @@ import scala.util.Try // scalastyle:off import org.apache.kyuubi.plugin.spark.authz.AccessControlException +import org.apache.kyuubi.plugin.spark.authz.RangerTestNamespace._ +import org.apache.kyuubi.plugin.spark.authz.RangerTestUsers._ +import org.apache.kyuubi.plugin.spark.authz.V2JdbcTableCatalogPrivilegesBuilderSuite._ +import org.apache.kyuubi.plugin.spark.authz.util.AuthZUtils._ /** * Tests for RangerSparkExtensionSuite @@ -32,8 +36,6 @@ class V2JdbcTableCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSu val catalogV2 = "testcat" val jdbcCatalogV2 = "jdbc2" - val namespace1 = "ns1" - val namespace2 = "ns2" val table1 = "table1" val table2 = "table2" val outputTable1 = "outputTable1" @@ -43,27 +45,23 @@ class V2JdbcTableCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSu val jdbcUrl: String = s"$dbUrl;create=true" override def beforeAll(): Unit = { - if (isSparkV31OrGreater) { - spark.conf.set( - s"spark.sql.catalog.$catalogV2", - "org.apache.spark.sql.execution.datasources.v2.jdbc.JDBCTableCatalog") - spark.conf.set(s"spark.sql.catalog.$catalogV2.url", jdbcUrl) - spark.conf.set( - s"spark.sql.catalog.$catalogV2.driver", - "org.apache.derby.jdbc.AutoloadedDriver") - - super.beforeAll() - - doAs("admin", sql(s"CREATE DATABASE IF NOT EXISTS $catalogV2.$namespace1")) - doAs( - "admin", - sql(s"CREATE TABLE IF NOT EXISTS $catalogV2.$namespace1.$table1" + - " (id int, name string, city string)")) - doAs( - "admin", - sql(s"CREATE TABLE IF NOT EXISTS $catalogV2.$namespace1.$outputTable1" + - " (id int, name string, city string)")) - } + spark.conf.set(s"spark.sql.catalog.$catalogV2", v2JdbcTableCatalogClassName) + spark.conf.set(s"spark.sql.catalog.$catalogV2.url", jdbcUrl) + spark.conf.set( + s"spark.sql.catalog.$catalogV2.driver", + "org.apache.derby.jdbc.AutoloadedDriver") + + super.beforeAll() + + doAs(admin, sql(s"CREATE DATABASE IF NOT EXISTS $catalogV2.$namespace1")) + doAs( + admin, + sql(s"CREATE TABLE IF NOT EXISTS $catalogV2.$namespace1.$table1" + + " (id int, name string, city string)")) + doAs( + admin, + sql(s"CREATE TABLE IF NOT EXISTS $catalogV2.$namespace1.$outputTable1" + + " (id int, name string, city string)")) } override def afterAll(): Unit = { @@ -78,48 +76,47 @@ class V2JdbcTableCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSu } test("[KYUUBI #3424] CREATE DATABASE") { - assume(isSparkV31OrGreater) - // create database val e1 = intercept[AccessControlException]( - doAs("someone", sql(s"CREATE DATABASE IF NOT EXISTS $catalogV2.$namespace2").explain())) + doAs(someone, sql(s"CREATE DATABASE IF NOT EXISTS $catalogV2.$namespace2").explain())) assert(e1.getMessage.contains(s"does not have [create] privilege" + s" on [$namespace2]")) } test("[KYUUBI #3424] DROP DATABASE") { - assume(isSparkV31OrGreater) - // create database val e1 = intercept[AccessControlException]( - doAs("someone", sql(s"DROP DATABASE IF EXISTS $catalogV2.$namespace2").explain())) + doAs(someone, sql(s"DROP DATABASE IF EXISTS $catalogV2.$namespace2").explain())) assert(e1.getMessage.contains(s"does not have [drop] privilege" + s" on [$namespace2]")) } test("[KYUUBI #3424] SELECT TABLE") { - assume(isSparkV31OrGreater) - // select val e1 = intercept[AccessControlException]( - doAs("someone", sql(s"select city, id from $catalogV2.$namespace1.$table1").explain())) + doAs(someone, sql(s"select city, id from $catalogV2.$namespace1.$table1").explain())) assert(e1.getMessage.contains(s"does not have [select] privilege" + - s" on [$namespace1/$table1/id]")) + s" on [$namespace1/$table1/city]")) } - test("[KYUUBI #3424] CREATE TABLE") { - assume(isSparkV31OrGreater) + test("[KYUUBI #4255] DESCRIBE TABLE") { + val e1 = intercept[AccessControlException]( + doAs(someone, sql(s"DESCRIBE TABLE $catalogV2.$namespace1.$table1").explain())) + assert(e1.getMessage.contains(s"does not have [select] privilege" + + s" on [$namespace1/$table1]")) + } + test("[KYUUBI #3424] CREATE TABLE") { // CreateTable val e2 = intercept[AccessControlException]( - doAs("someone", sql(s"CREATE TABLE IF NOT EXISTS $catalogV2.$namespace1.$table2"))) + doAs(someone, sql(s"CREATE TABLE IF NOT EXISTS $catalogV2.$namespace1.$table2"))) assert(e2.getMessage.contains(s"does not have [create] privilege" + s" on [$namespace1/$table2]")) // CreateTableAsSelect val e21 = intercept[AccessControlException]( doAs( - "someone", + someone, sql(s"CREATE TABLE IF NOT EXISTS $catalogV2.$namespace1.$table2" + s" AS select * from $catalogV2.$namespace1.$table1"))) assert(e21.getMessage.contains(s"does not have [select] privilege" + @@ -127,22 +124,18 @@ class V2JdbcTableCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSu } test("[KYUUBI #3424] DROP TABLE") { - assume(isSparkV31OrGreater) - // DropTable val e3 = intercept[AccessControlException]( - doAs("someone", sql(s"DROP TABLE $catalogV2.$namespace1.$table1"))) + doAs(someone, sql(s"DROP TABLE $catalogV2.$namespace1.$table1"))) assert(e3.getMessage.contains(s"does not have [drop] privilege" + s" on [$namespace1/$table1]")) } test("[KYUUBI #3424] INSERT TABLE") { - assume(isSparkV31OrGreater) - // AppendData: Insert Using a VALUES Clause val e4 = intercept[AccessControlException]( doAs( - "someone", + someone, sql(s"INSERT INTO $catalogV2.$namespace1.$outputTable1 (id, name, city)" + s" VALUES (1, 'bowenliang123', 'Guangzhou')"))) assert(e4.getMessage.contains(s"does not have [update] privilege" + @@ -151,7 +144,7 @@ class V2JdbcTableCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSu // AppendData: Insert Using a TABLE Statement val e42 = intercept[AccessControlException]( doAs( - "someone", + someone, sql(s"INSERT INTO $catalogV2.$namespace1.$outputTable1 (id, name, city)" + s" TABLE $catalogV2.$namespace1.$table1"))) assert(e42.getMessage.contains(s"does not have [select] privilege" + @@ -160,7 +153,7 @@ class V2JdbcTableCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSu // AppendData: Insert Using a SELECT Statement val e43 = intercept[AccessControlException]( doAs( - "someone", + someone, sql(s"INSERT INTO $catalogV2.$namespace1.$outputTable1 (id, name, city)" + s" SELECT * from $catalogV2.$namespace1.$table1"))) assert(e43.getMessage.contains(s"does not have [select] privilege" + @@ -169,7 +162,7 @@ class V2JdbcTableCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSu // OverwriteByExpression: Insert Overwrite val e44 = intercept[AccessControlException]( doAs( - "someone", + someone, sql(s"INSERT OVERWRITE $catalogV2.$namespace1.$outputTable1 (id, name, city)" + s" VALUES (1, 'bowenliang123', 'Guangzhou')"))) assert(e44.getMessage.contains(s"does not have [update] privilege" + @@ -177,8 +170,6 @@ class V2JdbcTableCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSu } test("[KYUUBI #3424] MERGE INTO") { - assume(isSparkV31OrGreater) - val mergeIntoSql = s""" |MERGE INTO $catalogV2.$namespace1.$outputTable1 AS target @@ -191,37 +182,28 @@ class V2JdbcTableCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSu // MergeIntoTable: Using a MERGE INTO Statement val e1 = intercept[AccessControlException]( doAs( - "someone", + someone, sql(mergeIntoSql))) assert(e1.getMessage.contains(s"does not have [select] privilege" + s" on [$namespace1/$table1/id]")) - try { - SparkRangerAdminPlugin.getRangerConf.setBoolean( - s"ranger.plugin.${SparkRangerAdminPlugin.getServiceType}.authorize.in.single.call", - true) + withSingleCallEnabled { val e2 = intercept[AccessControlException]( doAs( - "someone", + someone, sql(mergeIntoSql))) assert(e2.getMessage.contains(s"does not have" + s" [select] privilege" + s" on [$namespace1/$table1/id,$namespace1/table1/name,$namespace1/$table1/city]," + s" [update] privilege on [$namespace1/$outputTable1]")) - } finally { - SparkRangerAdminPlugin.getRangerConf.setBoolean( - s"ranger.plugin.${SparkRangerAdminPlugin.getServiceType}.authorize.in.single.call", - false) } } test("[KYUUBI #3424] UPDATE TABLE") { - assume(isSparkV31OrGreater) - // UpdateTable val e5 = intercept[AccessControlException]( doAs( - "someone", + someone, sql(s"UPDATE $catalogV2.$namespace1.$table1 SET city='Hangzhou' " + " WHERE id=1"))) assert(e5.getMessage.contains(s"does not have [update] privilege" + @@ -229,22 +211,18 @@ class V2JdbcTableCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSu } test("[KYUUBI #3424] DELETE FROM TABLE") { - assume(isSparkV31OrGreater) - // DeleteFromTable val e6 = intercept[AccessControlException]( - doAs("someone", sql(s"DELETE FROM $catalogV2.$namespace1.$table1 WHERE id=1"))) + doAs(someone, sql(s"DELETE FROM $catalogV2.$namespace1.$table1 WHERE id=1"))) assert(e6.getMessage.contains(s"does not have [update] privilege" + s" on [$namespace1/$table1]")) } test("[KYUUBI #3424] CACHE TABLE") { - assume(isSparkV31OrGreater) - // CacheTable val e7 = intercept[AccessControlException]( doAs( - "someone", + someone, sql(s"CACHE TABLE $cacheTable1" + s" AS select * from $catalogV2.$namespace1.$table1"))) if (isSparkV32OrGreater) { @@ -261,7 +239,7 @@ class V2JdbcTableCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSu val e1 = intercept[AccessControlException]( doAs( - "someone", + someone, sql(s"TRUNCATE TABLE $catalogV2.$namespace1.$table1"))) assert(e1.getMessage.contains(s"does not have [update] privilege" + s" on [$namespace1/$table1]")) @@ -272,19 +250,17 @@ class V2JdbcTableCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSu val e1 = intercept[AccessControlException]( doAs( - "someone", + someone, sql(s"MSCK REPAIR TABLE $catalogV2.$namespace1.$table1"))) assert(e1.getMessage.contains(s"does not have [alter] privilege" + s" on [$namespace1/$table1]")) } test("[KYUUBI #3424] ALTER TABLE") { - assume(isSparkV31OrGreater) - // AddColumns val e61 = intercept[AccessControlException]( doAs( - "someone", + someone, sql(s"ALTER TABLE $catalogV2.$namespace1.$table1 ADD COLUMNS (age int) ").explain())) assert(e61.getMessage.contains(s"does not have [alter] privilege" + s" on [$namespace1/$table1]")) @@ -292,7 +268,7 @@ class V2JdbcTableCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSu // DropColumns val e62 = intercept[AccessControlException]( doAs( - "someone", + someone, sql(s"ALTER TABLE $catalogV2.$namespace1.$table1 DROP COLUMNS city ").explain())) assert(e62.getMessage.contains(s"does not have [alter] privilege" + s" on [$namespace1/$table1]")) @@ -300,7 +276,7 @@ class V2JdbcTableCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSu // RenameColumn val e63 = intercept[AccessControlException]( doAs( - "someone", + someone, sql(s"ALTER TABLE $catalogV2.$namespace1.$table1 RENAME COLUMN city TO city2 ").explain())) assert(e63.getMessage.contains(s"does not have [alter] privilege" + s" on [$namespace1/$table1]")) @@ -308,7 +284,7 @@ class V2JdbcTableCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSu // AlterColumn val e64 = intercept[AccessControlException]( doAs( - "someone", + someone, sql(s"ALTER TABLE $catalogV2.$namespace1.$table1 " + s"ALTER COLUMN city COMMENT 'city' "))) assert(e64.getMessage.contains(s"does not have [alter] privilege" + @@ -316,12 +292,10 @@ class V2JdbcTableCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSu } test("[KYUUBI #3424] COMMENT ON") { - assume(isSparkV31OrGreater) - // CommentOnNamespace val e1 = intercept[AccessControlException]( doAs( - "someone", + someone, sql(s"COMMENT ON DATABASE $catalogV2.$namespace1 IS 'xYz' ").explain())) assert(e1.getMessage.contains(s"does not have [alter] privilege" + s" on [$namespace1]")) @@ -329,7 +303,7 @@ class V2JdbcTableCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSu // CommentOnNamespace val e2 = intercept[AccessControlException]( doAs( - "someone", + someone, sql(s"COMMENT ON NAMESPACE $catalogV2.$namespace1 IS 'xYz' ").explain())) assert(e2.getMessage.contains(s"does not have [alter] privilege" + s" on [$namespace1]")) @@ -337,7 +311,7 @@ class V2JdbcTableCatalogRangerSparkExtensionSuite extends RangerSparkExtensionSu // CommentOnTable val e3 = intercept[AccessControlException]( doAs( - "someone", + someone, sql(s"COMMENT ON TABLE $catalogV2.$namespace1.$table1 IS 'xYz' ").explain())) assert(e3.getMessage.contains(s"does not have [alter] privilege" + s" on [$namespace1/$table1]")) diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/datamasking/DataMaskingForHiveHiveParquetSuite.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/datamasking/DataMaskingForHiveHiveParquetSuite.scala new file mode 100644 index 000000000..ccc694f9b --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/datamasking/DataMaskingForHiveHiveParquetSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.plugin.spark.authz.ranger.datamasking + +class DataMaskingForHiveHiveParquetSuite extends DataMaskingTestBase { + override protected val catalogImpl: String = "hive" + override protected def format: String = "USING hive OPTIONS(fileFormat='parquet')" +} diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/datamasking/DataMaskingForHiveParquetSuite.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/datamasking/DataMaskingForHiveParquetSuite.scala new file mode 100644 index 000000000..ba254abbd --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/datamasking/DataMaskingForHiveParquetSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.plugin.spark.authz.ranger.datamasking + +class DataMaskingForHiveParquetSuite extends DataMaskingTestBase { + override protected val catalogImpl: String = "hive" + override protected def format: String = "USING parquet" +} diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/datamasking/DataMaskingForIcebergSuite.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/datamasking/DataMaskingForIcebergSuite.scala new file mode 100644 index 000000000..405e53fc2 --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/datamasking/DataMaskingForIcebergSuite.scala @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.plugin.spark.authz.ranger.datamasking + +import org.apache.spark.SparkConf +import org.scalatest.Outcome + +import org.apache.kyuubi.Utils + +class DataMaskingForIcebergSuite extends DataMaskingTestBase { + override protected val extraSparkConf: SparkConf = { + new SparkConf() + .set("spark.sql.defaultCatalog", "testcat") + .set( + "spark.sql.catalog.testcat", + "org.apache.iceberg.spark.SparkCatalog") + .set(s"spark.sql.catalog.testcat.type", "hadoop") + .set( + "spark.sql.catalog.testcat.warehouse", + Utils.createTempDir("iceberg-hadoop").toString) + } + + override protected val catalogImpl: String = "in-memory" + + override protected def format: String = "USING iceberg" + + override def beforeAll(): Unit = { + super.beforeAll() + } + + override def afterAll(): Unit = { + super.afterAll() + } + + override def withFixture(test: NoArgTest): Outcome = { + test() + } +} diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/datamasking/DataMaskingForInMemoryParquetSuite.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/datamasking/DataMaskingForInMemoryParquetSuite.scala new file mode 100644 index 000000000..1bfb71e79 --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/datamasking/DataMaskingForInMemoryParquetSuite.scala @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.plugin.spark.authz.ranger.datamasking + +class DataMaskingForInMemoryParquetSuite extends DataMaskingTestBase { + + override protected val catalogImpl: String = "in-memory" + override protected def format: String = "USING parquet" +} diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/datamasking/DataMaskingForJDBCV2Suite.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/datamasking/DataMaskingForJDBCV2Suite.scala new file mode 100644 index 000000000..411d98cf9 --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/datamasking/DataMaskingForJDBCV2Suite.scala @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.plugin.spark.authz.ranger.datamasking +import java.sql.DriverManager + +import scala.util.Try + +import org.apache.spark.SparkConf +import org.scalatest.Outcome + +import org.apache.kyuubi.plugin.spark.authz.V2JdbcTableCatalogPrivilegesBuilderSuite._ + +class DataMaskingForJDBCV2Suite extends DataMaskingTestBase { + override protected val extraSparkConf: SparkConf = { + new SparkConf() + .set("spark.sql.defaultCatalog", "testcat") + .set("spark.sql.catalog.testcat", v2JdbcTableCatalogClassName) + .set(s"spark.sql.catalog.testcat.url", "jdbc:derby:memory:testcat;create=true") + .set( + s"spark.sql.catalog.testcat.driver", + "org.apache.derby.jdbc.AutoloadedDriver") + } + + override protected val catalogImpl: String = "in-memory" + + override protected def format: String = "" + + override def beforeAll(): Unit = { + super.beforeAll() + } + + override def afterAll(): Unit = { + super.afterAll() + // cleanup db + Try { + DriverManager.getConnection(s"jdbc:derby:memory:testcat;shutdown=true") + } + } + + override def withFixture(test: NoArgTest): Outcome = { + test() + } +} diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/datamasking/DataMaskingTestBase.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/datamasking/DataMaskingTestBase.scala new file mode 100644 index 000000000..d8877b7f9 --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/datamasking/DataMaskingTestBase.scala @@ -0,0 +1,327 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.plugin.spark.authz.ranger.datamasking + +import java.sql.Timestamp + +import scala.util.Try + +// scalastyle:off +import org.apache.commons.codec.digest.DigestUtils.md5Hex +import org.apache.spark.sql.{Row, SparkSessionExtensions} +import org.scalatest.BeforeAndAfterAll +import org.scalatest.funsuite.AnyFunSuite + +import org.apache.kyuubi.plugin.spark.authz.RangerTestUsers._ +import org.apache.kyuubi.plugin.spark.authz.SparkSessionProvider +import org.apache.kyuubi.plugin.spark.authz.ranger.RangerSparkExtension + +/** + * Base trait for data masking tests, derivative classes shall name themselves following: + * DataMaskingFor CatalogImpl? FileFormat? Additions? Suite + */ +trait DataMaskingTestBase extends AnyFunSuite with SparkSessionProvider with BeforeAndAfterAll { +// scalastyle:on + override protected val extension: SparkSessionExtensions => Unit = new RangerSparkExtension + + private def setup(): Unit = { + sql(s"CREATE TABLE IF NOT EXISTS default.src" + + "(key int," + + " value1 int," + + " value2 string," + + " value3 string," + + " value4 timestamp," + + " value5 string)" + + s" $format") + + // NOTICE: `bob` has a row filter `key < 20` + sql("INSERT INTO default.src " + + "SELECT 1, 1, 'hello', 'world', timestamp'2018-11-17 12:34:56', 'World'") + sql("INSERT INTO default.src " + + "SELECT 20, 2, 'kyuubi', 'y', timestamp'2018-11-17 12:34:56', 'world'") + sql("INSERT INTO default.src " + + "SELECT 30, 3, 'spark', 'a', timestamp'2018-11-17 12:34:56', 'world'") + + // scalastyle:off + val value1 = "hello WORD 123 ~!@# AßþΔЙקم๗ቐあア叶葉엽" + val value2 = "AßþΔЙקم๗ቐあア叶葉엽 hello WORD 123 ~!@#" + // AßþΔЙקم๗ቐあア叶葉엽 reference https://zh.wikipedia.org/zh-cn/Unicode#XML.E5.92.8CUnicode + // scalastyle:on + sql(s"INSERT INTO default.src " + + s"SELECT 10, 4, '$value1', '$value1', timestamp'2018-11-17 12:34:56', '$value1'") + sql("INSERT INTO default.src " + + s"SELECT 11, 5, '$value2', '$value2', timestamp'2018-11-17 12:34:56', '$value2'") + + sql(s"CREATE TABLE default.unmasked $format AS SELECT * FROM default.src") + } + + private def cleanup(): Unit = { + sql("DROP TABLE IF EXISTS default.src") + sql("DROP TABLE IF EXISTS default.unmasked") + } + + override def beforeAll(): Unit = { + doAs(admin, setup()) + super.beforeAll() + } + override def afterAll(): Unit = { + doAs(admin, cleanup()) + spark.stop + super.afterAll() + } + + test("simple query with a user doesn't have mask rules") { + checkAnswer( + kent, + "SELECT key FROM default.src order by key", + Seq(Row(1), Row(10), Row(11), Row(20), Row(30))) + } + + test("simple query with a user has mask rules") { + val result = + Seq(Row(md5Hex("1"), "xxxxx", "worlx", Timestamp.valueOf("2018-01-01 00:00:00"), "Xorld")) + checkAnswer( + bob, + "SELECT value1, value2, value3, value4, value5 FROM default.src " + + "where key = 1", + result) + checkAnswer( + bob, + "SELECT value1 as key, value2, value3, value4, value5 FROM default.src where key = 1", + result) + } + + test("star") { + val result = + Seq(Row(1, md5Hex("1"), "xxxxx", "worlx", Timestamp.valueOf("2018-01-01 00:00:00"), "Xorld")) + checkAnswer(bob, "SELECT * FROM default.src where key = 1", result) + } + + test("simple udf") { + val result = + Seq(Row(md5Hex("1"), "xxxxx", "worlx", Timestamp.valueOf("2018-01-01 00:00:00"), "Xorld")) + checkAnswer( + bob, + "SELECT max(value1), max(value2), max(value3), max(value4), max(value5) FROM default.src" + + " where key = 1", + result) + } + + test("complex udf") { + val result = + Seq(Row(md5Hex("1"), "xxxxx", "worlx", Timestamp.valueOf("2018-01-01 00:00:00"), "Xorld")) + checkAnswer( + bob, + "SELECT coalesce(max(value1), 1), coalesce(max(value2), 1), coalesce(max(value3), 1), " + + "coalesce(max(value4), timestamp '2018-01-01 22:33:44'), coalesce(max(value5), 1) " + + "FROM default.src where key = 1", + result) + } + + test("in subquery") { + val result = + Seq(Row(md5Hex("1"), "xxxxx", "worlx", Timestamp.valueOf("2018-01-01 00:00:00"), "Xorld")) + checkAnswer( + bob, + "SELECT value1, value2, value3, value4, value5 FROM default.src WHERE value2 in " + + "(SELECT value2 as key FROM default.src where key = 1)", + result) + } + + test("create a unmasked table as select from a masked one") { + withCleanTmpResources(Seq(("default.src2", "table"))) { + doAs( + bob, + sql(s"CREATE TABLE default.src2 $format AS SELECT value1 FROM default.src " + + s"where key = 1")) + checkAnswer(bob, "SELECT value1 FROM default.src2", Seq(Row(md5Hex("1")))) + } + } + + test("insert into a unmasked table from a masked one") { + withCleanTmpResources(Seq(("default.src2", "table"), ("default.src3", "table"))) { + doAs(bob, sql(s"CREATE TABLE default.src2 (value1 string) $format")) + doAs( + bob, + sql(s"INSERT INTO default.src2 SELECT value1 from default.src " + + s"where key = 1")) + doAs( + bob, + sql(s"INSERT INTO default.src2 SELECT value1 as v from default.src " + + s"where key = 1")) + checkAnswer(bob, "SELECT value1 FROM default.src2", Seq(Row(md5Hex("1")), Row(md5Hex("1")))) + doAs(bob, sql(s"CREATE TABLE default.src3 (k int, value string) $format")) + doAs( + bob, + sql(s"INSERT INTO default.src3 SELECT key, value1 from default.src " + + s"where key = 1")) + doAs( + bob, + sql(s"INSERT INTO default.src3 SELECT key, value1 as v from default.src " + + s"where key = 1")) + checkAnswer(bob, "SELECT value FROM default.src3", Seq(Row(md5Hex("1")), Row(md5Hex("1")))) + } + } + + test("join on an unmasked table") { + val s = "SELECT a.value1, b.value1 FROM default.src a" + + " join default.unmasked b on a.value1=b.value1" + checkAnswer(bob, s, Nil) + checkAnswer(bob, s, Nil) // just for testing query multiple times, don't delete it + } + + test("self join on a masked table") { + val s = "SELECT a.value1, b.value1 FROM default.src a" + + " join default.src b on a.value1=b.value1 where a.key = 1 and b.key = 1 " + checkAnswer(bob, s, Seq(Row(md5Hex("1"), md5Hex("1")))) + // just for testing query multiple times, don't delete it + checkAnswer(bob, s, Seq(Row(md5Hex("1"), md5Hex("1")))) + } + + test("self join on a masked table and filter the masked column with original value") { + val s = "SELECT a.value1, b.value1 FROM default.src a" + + " join default.src b on a.value1=b.value1" + + " where a.value1='1' and b.value1='1'" + checkAnswer(bob, s, Nil) + checkAnswer(bob, s, Nil) // just for testing query multiple times, don't delete it + } + + test("self join on a masked table and filter the masked column with masked value") { + // scalastyle:off + val s = "SELECT a.value1, b.value1 FROM default.src a" + + " join default.src b on a.value1=b.value1" + + s" where a.value1='${md5Hex("1")}' and b.value1='${md5Hex("1")}'" + // TODO: The v1 an v2 relations generate different implicit type cast rules for filters + // so the bellow test failed in derivative classes that us v2 data source, e.g., DataMaskingForIcebergSuite + // For the issue itself, we might need check the spark logic first + // DataMaskingStage1Marker Project [value1#178, value1#183] + // +- Project [value1#178, value1#183] + // +- Filter ((cast(value1#178 as int) = cast(c4ca4238a0b923820dcc509a6f75849b as int)) AND (cast(value1#183 as int) = cast(c4ca4238a0b923820dcc509a6f75849b as int))) + // +- Join Inner, (value1#178 = value1#183) + // :- SubqueryAlias a + // : +- SubqueryAlias testcat.default.src + // : +- Filter (key#166 < 20) + // : +- RowFilterMarker + // : +- DataMaskingStage0Marker RelationV2[key#166, value1#167, value2#168, value3#169, value4#170, value5#171] default.src + // : +- Project [key#166, md5(cast(cast(value1#167 as string) as binary)) AS value1#178, regexp_replace(regexp_replace(regexp_replace(value2#168, [A-Z], X, 1), [a-z], x, 1), [0-9], n, 1) AS value2#179, regexp_replace(regexp_replace(regexp_replace(value3#169, [A-Z], X, 5), [a-z], x, 5), [0-9], n, 5) AS value3#180, date_trunc(YEAR, value4#170, Some(Asia/Shanghai)) AS value4#181, concat(regexp_replace(regexp_replace(regexp_replace(left(value5#171, (length(value5#171) - 4)), [A-Z], X, 1), [a-z], x, 1), [0-9], n, 1), right(value5#171, 4)) AS value5#182] + // : +- RelationV2[key#166, value1#167, value2#168, value3#169, value4#170, value5#171] default.src + // +- SubqueryAlias b + // +- SubqueryAlias testcat.default.src + // +- Filter (key#172 < 20) + // +- RowFilterMarker + // +- DataMaskingStage0Marker RelationV2[key#172, value1#173, value2#174, value3#175, value4#176, value5#177] default.src + // +- Project [key#172, md5(cast(cast(value1#173 as string) as binary)) AS value1#183, regexp_replace(regexp_replace(regexp_replace(value2#174, [A-Z], X, 1), [a-z], x, 1), [0-9], n, 1) AS value2#184, regexp_replace(regexp_replace(regexp_replace(value3#175, [A-Z], X, 5), [a-z], x, 5), [0-9], n, 5) AS value3#185, date_trunc(YEAR, value4#176, Some(Asia/Shanghai)) AS value4#186, concat(regexp_replace(regexp_replace(regexp_replace(left(value5#177, (length(value5#177) - 4)), [A-Z], X, 1), [a-z], x, 1), [0-9], n, 1), right(value5#177, 4)) AS value5#187] + // +- RelationV2[key#172, value1#173, value2#174, value3#175, value4#176, value5#177] default.src + // + // + // Project [value1#143, value1#148] + // +- Filter ((value1#143 = c4ca4238a0b923820dcc509a6f75849b) AND (value1#148 = c4ca4238a0b923820dcc509a6f75849b)) + // +- Join Inner, (value1#143 = value1#148) + // :- SubqueryAlias a + // : +- SubqueryAlias spark_catalog.default.src + // : +- Filter (key#60 < 20) + // : +- RowFilterMarker + // : +- DataMaskingStage0Marker Relation default.src[key#60,value1#61,value2#62,value3#63,value4#64,value5#65] parquet + // : +- Project [key#60, md5(cast(cast(value1#61 as string) as binary)) AS value1#143, regexp_replace(regexp_replace(regexp_replace(value2#62, [A-Z], X, 1), [a-z], x, 1), [0-9], n, 1) AS value2#144, regexp_replace(regexp_replace(regexp_replace(value3#63, [A-Z], X, 5), [a-z], x, 5), [0-9], n, 5) AS value3#145, date_trunc(YEAR, value4#64, Some(Asia/Shanghai)) AS value4#146, concat(regexp_replace(regexp_replace(regexp_replace(left(value5#65, (length(value5#65) - 4)), [A-Z], X, 1), [a-z], x, 1), [0-9], n, 1), right(value5#65, 4)) AS value5#147] + // : +- Relation default.src[key#60,value1#61,value2#62,value3#63,value4#64,value5#65] parquet + // +- SubqueryAlias b + // +- SubqueryAlias spark_catalog.default.src + // +- Filter (key#153 < 20) + // +- RowFilterMarker + // +- DataMaskingStage0Marker Relation default.src[key#60,value1#61,value2#62,value3#63,value4#64,value5#65] parquet + // +- Project [key#153, md5(cast(cast(value1#154 as string) as binary)) AS value1#148, regexp_replace(regexp_replace(regexp_replace(value2#155, [A-Z], X, 1), [a-z], x, 1), [0-9], n, 1) AS value2#149, regexp_replace(regexp_replace(regexp_replace(value3#156, [A-Z], X, 5), [a-z], x, 5), [0-9], n, 5) AS value3#150, date_trunc(YEAR, value4#157, Some(Asia/Shanghai)) AS value4#151, concat(regexp_replace(regexp_replace(regexp_replace(left(value5#158, (length(value5#158) - 4)), [A-Z], X, 1), [a-z], x, 1), [0-9], n, 1), right(value5#158, 4)) AS value5#152] + // +- Relation default.src[key#153,value1#154,value2#155,value3#156,value4#157,value5#158] parquet + // checkAnswer(bob, s, Seq(Row(md5Hex("1"), md5Hex("1")))) + // + // + // scalastyle:on + + // So here we use value2 to avoid type casting + val s2 = "SELECT a.value1, b.value1 FROM default.src a" + + " join default.src b on a.value1=b.value1" + + s" where a.value2='xxxxx' and b.value2='xxxxx'" + checkAnswer(bob, s2, Seq(Row(md5Hex("1"), md5Hex("1")))) + // just for testing query multiple times, don't delete it + checkAnswer(bob, s2, Seq(Row(md5Hex("1"), md5Hex("1")))) + } + + test("union an unmasked table") { + val s = """ + SELECT value1 from ( + SELECT a.value1 FROM default.src a where a.key = 1 + union + (SELECT b.value1 FROM default.unmasked b) + ) c order by value1 + """ + checkAnswer(bob, s, Seq(Row("1"), Row("2"), Row("3"), Row("4"), Row("5"), Row(md5Hex("1")))) + } + + test("union a masked table") { + val s = "SELECT a.value1 FROM default.src a where a.key = 1 union" + + " (SELECT b.value1 FROM default.src b where b.key = 1)" + checkAnswer(bob, s, Seq(Row(md5Hex("1")))) + } + + test("KYUUBI #3581: permanent view should lookup rule on itself not the raw table") { + val supported = doAs( + permViewUser, + Try(sql("CREATE OR REPLACE VIEW default.perm_view AS SELECT * FROM default.src")).isSuccess) + assume(supported, s"view support for '$format' has not been implemented yet") + + withCleanTmpResources(Seq(("default.perm_view", "view"))) { + checkAnswer( + permViewUser, + "SELECT value1, value2 FROM default.src where key = 1", + Seq(Row(1, "hello"))) + checkAnswer( + permViewUser, + "SELECT value1, value2 FROM default.perm_view where key = 1", + Seq(Row(md5Hex("1"), "hello"))) + } + } + + // This test only includes a small subset of UCS-2 characters. + // But in theory, it should work for all characters + test("test MASK,MASK_SHOW_FIRST_4,MASK_SHOW_LAST_4 rule with non-English character set") { + val s1 = s"SELECT * FROM default.src where key = 10" + val s2 = s"SELECT * FROM default.src where key = 11" + // scalastyle:off + checkAnswer( + bob, + s1, + Seq(Row( + 10, + md5Hex("4"), + "xxxxxUXXXXUnnnUUUUUUXUUUUUUUUUUUUU", + "hellxUXXXXUnnnUUUUUUXUUUUUUUUUUUUU", + Timestamp.valueOf("2018-01-01 00:00:00"), + "xxxxxUXXXXUnnnUUUUUUXUUUUUUUUUア叶葉엽"))) + checkAnswer( + bob, + s2, + Seq(Row( + 11, + md5Hex("5"), + "XUUUUUUUUUUUUUUxxxxxUXXXXUnnnUUUUU", + "AßþΔUUUUUUUUUUUxxxxxUXXXXUnnnUUUUU", + Timestamp.valueOf("2018-01-01 00:00:00"), + "XUUUUUUUUUUUUUUxxxxxUXXXXUnnnU~!@#"))) + // scalastyle:on + } + +} diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/rowfiltering/RowFilteringForHiveHiveParquetSuite.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/rowfiltering/RowFilteringForHiveHiveParquetSuite.scala new file mode 100644 index 000000000..142a2f825 --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/rowfiltering/RowFilteringForHiveHiveParquetSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.plugin.spark.authz.ranger.rowfiltering + +class RowFilteringForHiveHiveParquetSuite extends RowFilteringTestBase { + override protected val catalogImpl: String = "hive" + override protected def format: String = "USING hive OPTIONS(fileFormat='parquet')" +} diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/rowfiltering/RowFilteringForHiveParquetSuite.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/rowfiltering/RowFilteringForHiveParquetSuite.scala new file mode 100644 index 000000000..9727643cf --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/rowfiltering/RowFilteringForHiveParquetSuite.scala @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.plugin.spark.authz.ranger.rowfiltering + +class RowFilteringForHiveParquetSuite extends RowFilteringTestBase { + override protected val catalogImpl: String = "hive" + override protected def format: String = "USING parquet" +} diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/rowfiltering/RowFilteringForIcebergSuite.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/rowfiltering/RowFilteringForIcebergSuite.scala new file mode 100644 index 000000000..57a9e29b6 --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/rowfiltering/RowFilteringForIcebergSuite.scala @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.plugin.spark.authz.ranger.rowfiltering + +import org.apache.spark.SparkConf +import org.scalatest.Outcome + +import org.apache.kyuubi.Utils + +class RowFilteringForIcebergSuite extends RowFilteringTestBase { + override protected val extraSparkConf: SparkConf = { + new SparkConf() + .set("spark.sql.defaultCatalog", "testcat") + .set( + "spark.sql.catalog.testcat", + "org.apache.iceberg.spark.SparkCatalog") + .set(s"spark.sql.catalog.testcat.type", "hadoop") + .set( + "spark.sql.catalog.testcat.warehouse", + Utils.createTempDir("iceberg-hadoop").toString) + } + + override protected val catalogImpl: String = "in-memory" + + override protected def format: String = "USING iceberg" + + override def beforeAll(): Unit = { + super.beforeAll() + } + + override def afterAll(): Unit = { + super.afterAll() + } + + override def withFixture(test: NoArgTest): Outcome = { + test() + } +} diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/rowfiltering/RowFilteringForInMemoryParquetSuite.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/rowfiltering/RowFilteringForInMemoryParquetSuite.scala new file mode 100644 index 000000000..9baaa2a31 --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/rowfiltering/RowFilteringForInMemoryParquetSuite.scala @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.plugin.spark.authz.ranger.rowfiltering + +class RowFilteringForInMemoryParquetSuite extends RowFilteringTestBase { + + override protected val catalogImpl: String = "in-memory" + override protected def format: String = "USING parquet" +} diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/rowfiltering/RowFilteringForJDBCV2Suite.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/rowfiltering/RowFilteringForJDBCV2Suite.scala new file mode 100644 index 000000000..bfe1cd9e4 --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/rowfiltering/RowFilteringForJDBCV2Suite.scala @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.plugin.spark.authz.ranger.rowfiltering + +import java.sql.DriverManager + +import scala.util.Try + +import org.apache.spark.SparkConf +import org.scalatest.Outcome + +import org.apache.kyuubi.plugin.spark.authz.V2JdbcTableCatalogPrivilegesBuilderSuite._ + +class RowFilteringForJDBCV2Suite extends RowFilteringTestBase { + override protected val extraSparkConf: SparkConf = { + new SparkConf() + .set("spark.sql.defaultCatalog", "testcat") + .set("spark.sql.catalog.testcat", v2JdbcTableCatalogClassName) + .set(s"spark.sql.catalog.testcat.url", "jdbc:derby:memory:testcat;create=true") + .set( + s"spark.sql.catalog.testcat.driver", + "org.apache.derby.jdbc.AutoloadedDriver") + } + + override protected val catalogImpl: String = "in-memory" + + override protected def format: String = "" + + override def beforeAll(): Unit = { + super.beforeAll() + } + + override def afterAll(): Unit = { + super.afterAll() + // cleanup db + Try { + DriverManager.getConnection(s"jdbc:derby:memory:testcat;shutdown=true") + } + } + + override def withFixture(test: NoArgTest): Outcome = { + test() + } +} diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/rowfiltering/RowFilteringTestBase.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/rowfiltering/RowFilteringTestBase.scala new file mode 100644 index 000000000..3d0890d19 --- /dev/null +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/rowfiltering/RowFilteringTestBase.scala @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.plugin.spark.authz.ranger.rowfiltering + +// scalastyle:off +import scala.util.Try + +import org.apache.spark.sql.{Row, SparkSessionExtensions} +import org.scalatest.BeforeAndAfterAll +import org.scalatest.funsuite.AnyFunSuite + +import org.apache.kyuubi.plugin.spark.authz.RangerTestUsers._ +import org.apache.kyuubi.plugin.spark.authz.SparkSessionProvider +import org.apache.kyuubi.plugin.spark.authz.ranger.RangerSparkExtension + +/** + * Base trait for row filtering tests, derivative classes shall name themselves following: + * RowFilteringFor CatalogImpl? FileFormat? Additions? Suite + */ +trait RowFilteringTestBase extends AnyFunSuite with SparkSessionProvider with BeforeAndAfterAll { +// scalastyle:on + override protected val extension: SparkSessionExtensions => Unit = new RangerSparkExtension + + private def setup(): Unit = { + sql(s"CREATE TABLE IF NOT EXISTS default.src(key int, value int) $format") + sql("INSERT INTO default.src SELECT 1, 1") + sql("INSERT INTO default.src SELECT 20, 2") + sql("INSERT INTO default.src SELECT 30, 3") + } + + private def cleanup(): Unit = { + sql("DROP TABLE IF EXISTS default.src") + } + + override def beforeAll(): Unit = { + doAs(admin, setup()) + super.beforeAll() + } + override def afterAll(): Unit = { + doAs(admin, cleanup()) + spark.stop + super.afterAll() + } + + test("user without row filtering rule") { + checkAnswer( + kent, + "SELECT key FROM default.src order order by key", + Seq(Row(1), Row(20), Row(30))) + } + + test("simple query projecting filtering column") { + checkAnswer(bob, "SELECT key FROM default.src", Seq(Row(1))) + } + + test("simple query projecting non filtering column") { + checkAnswer(bob, "SELECT value FROM default.src", Seq(Row(1))) + } + + test("simple query projecting non filtering column with udf max") { + checkAnswer(bob, "SELECT max(value) FROM default.src", Seq(Row(1))) + } + + test("simple query projecting non filtering column with udf coalesce") { + checkAnswer(bob, "SELECT coalesce(max(value), 1) FROM default.src", Seq(Row(1))) + } + + test("in subquery") { + checkAnswer( + bob, + "SELECT value FROM default.src WHERE value in (SELECT value as key FROM default.src)", + Seq(Row(1))) + } + + test("ctas") { + withCleanTmpResources(Seq(("default.src2", "table"))) { + doAs(bob, sql(s"CREATE TABLE default.src2 $format AS SELECT value FROM default.src")) + val query = "select value from default.src2" + checkAnswer(admin, query, Seq(Row(1))) + checkAnswer(bob, query, Seq(Row(1))) + } + } + + test("[KYUUBI #3581]: row level filter on permanent view") { + val supported = doAs( + permViewUser, + Try(sql("CREATE OR REPLACE VIEW default.perm_view AS SELECT * FROM default.src")).isSuccess) + assume(supported, s"view support for '$format' has not been implemented yet") + + withCleanTmpResources(Seq((s"default.perm_view", "view"))) { + checkAnswer( + admin, + "SELECT key FROM default.perm_view order order by key", + Seq(Row(1), Row(20), Row(30))) + checkAnswer(bob, "SELECT key FROM default.perm_view", Seq(Row(1))) + checkAnswer(bob, "SELECT value FROM default.perm_view", Seq(Row(1))) + checkAnswer(bob, "SELECT max(value) FROM default.perm_view", Seq(Row(1))) + checkAnswer(bob, "SELECT coalesce(max(value), 1) FROM default.perm_view", Seq(Row(1))) + checkAnswer( + bob, + "SELECT value FROM default.perm_view WHERE value in " + + "(SELECT value as key FROM default.perm_view)", + Seq(Row(1))) + } + } +} diff --git a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/AuthzConfigurationCheckerSuite.scala b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/rule/AuthzConfigurationCheckerSuite.scala similarity index 92% rename from extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/AuthzConfigurationCheckerSuite.scala rename to extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/rule/AuthzConfigurationCheckerSuite.scala index cd5757e54..10fa0af9e 100644 --- a/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/ranger/AuthzConfigurationCheckerSuite.scala +++ b/extensions/spark/kyuubi-spark-authz/src/test/scala/org/apache/kyuubi/plugin/spark/authz/rule/AuthzConfigurationCheckerSuite.scala @@ -15,13 +15,15 @@ * limitations under the License. */ -package org.apache.kyuubi.plugin.spark.authz.ranger +package org.apache.kyuubi.plugin.spark.authz.rule import org.scalatest.BeforeAndAfterAll // scalastyle:off import org.scalatest.funsuite.AnyFunSuite import org.apache.kyuubi.plugin.spark.authz.{AccessControlException, SparkSessionProvider} +import org.apache.kyuubi.plugin.spark.authz.ranger.RuleAuthorization +import org.apache.kyuubi.plugin.spark.authz.rule.config.AuthzConfigurationChecker class AuthzConfigurationCheckerSuite extends AnyFunSuite with SparkSessionProvider with BeforeAndAfterAll { diff --git a/extensions/spark/kyuubi-spark-connector-common/pom.xml b/extensions/spark/kyuubi-spark-connector-common/pom.xml index e9fa8fcb4..1fc0f5768 100644 --- a/extensions/spark/kyuubi-spark-connector-common/pom.xml +++ b/extensions/spark/kyuubi-spark-connector-common/pom.xml @@ -21,16 +21,22 @@ org.apache.kyuubi kyuubi-parent - 1.7.0-SNAPSHOT + 1.9.0-SNAPSHOT ../../../pom.xml - kyuubi-spark-connector-common_2.12 + kyuubi-spark-connector-common_${scala.binary.version} jar Kyuubi Spark Connector Common https://kyuubi.apache.org/ + + org.apache.kyuubi + kyuubi-util-scala_${scala.binary.version} + ${project.version} + + org.scala-lang scala-library @@ -87,10 +93,21 @@ scalacheck-1-17_${scala.binary.version} test + + + org.apache.logging.log4j + log4j-1.2-api + test + + + + org.apache.logging.log4j + log4j-slf4j-impl + test + - org.apache.maven.plugins diff --git a/extensions/spark/kyuubi-spark-connector-common/src/main/scala/org/apache/kyuubi/spark/connector/common/SemanticVersion.scala b/extensions/spark/kyuubi-spark-connector-common/src/main/scala/org/apache/kyuubi/spark/connector/common/SemanticVersion.scala deleted file mode 100644 index 200937ca6..000000000 --- a/extensions/spark/kyuubi-spark-connector-common/src/main/scala/org/apache/kyuubi/spark/connector/common/SemanticVersion.scala +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kyuubi.spark.connector.common - -/** - * Encapsulate a component Spark version for the convenience of version checks. - * Copy from org.apache.kyuubi.engine.ComponentVersion - */ -case class SemanticVersion(majorVersion: Int, minorVersion: Int) { - - def isVersionAtMost(targetVersionString: String): Boolean = { - this.compareVersion( - targetVersionString, - (targetMajor: Int, targetMinor: Int, runtimeMajor: Int, runtimeMinor: Int) => - (runtimeMajor < targetMajor) || { - runtimeMajor == targetMajor && runtimeMinor <= targetMinor - }) - } - - def isVersionAtLeast(targetVersionString: String): Boolean = { - this.compareVersion( - targetVersionString, - (targetMajor: Int, targetMinor: Int, runtimeMajor: Int, runtimeMinor: Int) => - (runtimeMajor > targetMajor) || { - runtimeMajor == targetMajor && runtimeMinor >= targetMinor - }) - } - - def isVersionEqualTo(targetVersionString: String): Boolean = { - this.compareVersion( - targetVersionString, - (targetMajor: Int, targetMinor: Int, runtimeMajor: Int, runtimeMinor: Int) => - runtimeMajor == targetMajor && runtimeMinor == targetMinor) - } - - def compareVersion( - targetVersionString: String, - callback: (Int, Int, Int, Int) => Boolean): Boolean = { - val targetVersion = SemanticVersion(targetVersionString) - val targetMajor = targetVersion.majorVersion - val targetMinor = targetVersion.minorVersion - callback(targetMajor, targetMinor, this.majorVersion, this.minorVersion) - } - - override def toString: String = s"$majorVersion.$minorVersion" -} - -object SemanticVersion { - - def apply(versionString: String): SemanticVersion = { - """^(\d+)\.(\d+)(\..*)?$""".r.findFirstMatchIn(versionString) match { - case Some(m) => - SemanticVersion(m.group(1).toInt, m.group(2).toInt) - case None => - throw new IllegalArgumentException(s"Tried to parse '$versionString' as a project" + - s" version string, but it could not find the major and minor version numbers.") - } - } -} diff --git a/extensions/spark/kyuubi-spark-connector-common/src/main/scala/org/apache/kyuubi/spark/connector/common/SparkUtils.scala b/extensions/spark/kyuubi-spark-connector-common/src/main/scala/org/apache/kyuubi/spark/connector/common/SparkUtils.scala index c1a659fbf..fcb99ebe6 100644 --- a/extensions/spark/kyuubi-spark-connector-common/src/main/scala/org/apache/kyuubi/spark/connector/common/SparkUtils.scala +++ b/extensions/spark/kyuubi-spark-connector-common/src/main/scala/org/apache/kyuubi/spark/connector/common/SparkUtils.scala @@ -19,17 +19,8 @@ package org.apache.kyuubi.spark.connector.common import org.apache.spark.SPARK_VERSION -object SparkUtils { - - def isSparkVersionAtMost(targetVersionString: String): Boolean = { - SemanticVersion(SPARK_VERSION).isVersionAtMost(targetVersionString) - } +import org.apache.kyuubi.util.SemanticVersion - def isSparkVersionAtLeast(targetVersionString: String): Boolean = { - SemanticVersion(SPARK_VERSION).isVersionAtLeast(targetVersionString) - } - - def isSparkVersionEqualTo(targetVersionString: String): Boolean = { - SemanticVersion(SPARK_VERSION).isVersionEqualTo(targetVersionString) - } +object SparkUtils { + lazy val SPARK_RUNTIME_VERSION: SemanticVersion = SemanticVersion(SPARK_VERSION) } diff --git a/extensions/spark/kyuubi-spark-connector-common/src/test/resources/log4j2-test.xml b/extensions/spark/kyuubi-spark-connector-common/src/test/resources/log4j2-test.xml index bfc40dd6d..3110216c1 100644 --- a/extensions/spark/kyuubi-spark-connector-common/src/test/resources/log4j2-test.xml +++ b/extensions/spark/kyuubi-spark-connector-common/src/test/resources/log4j2-test.xml @@ -21,14 +21,14 @@ - + - + diff --git a/extensions/spark/kyuubi-spark-connector-hive/pom.xml b/extensions/spark/kyuubi-spark-connector-hive/pom.xml index a97dfa053..4f46138e9 100644 --- a/extensions/spark/kyuubi-spark-connector-hive/pom.xml +++ b/extensions/spark/kyuubi-spark-connector-hive/pom.xml @@ -21,18 +21,17 @@ org.apache.kyuubi kyuubi-parent - 1.7.0-SNAPSHOT + 1.9.0-SNAPSHOT ../../../pom.xml - kyuubi-spark-connector-hive_2.12 + kyuubi-spark-connector-hive_${scala.binary.version} jar Kyuubi Spark Hive Connector A Kyuubi hive connector based on Spark V2 DataSource https://kyuubi.apache.org/ - org.apache.kyuubi kyuubi-spark-connector-common_${scala.binary.version} @@ -40,34 +39,34 @@ - org.apache.kyuubi - kyuubi-spark-connector-common_${scala.binary.version} - ${project.version} - test-jar - test + com.google.guava + guava - org.scala-lang - scala-library + org.apache.spark + spark-hive_${scala.binary.version} provided - org.slf4j - slf4j-api + org.apache.hadoop + hadoop-client-api provided - org.apache.spark - spark-sql_${scala.binary.version} - provided + org.apache.kyuubi + kyuubi-spark-connector-common_${scala.binary.version} + ${project.version} + test-jar + test - com.google.guava - guava + org.scalatestplus + scalacheck-1-17_${scala.binary.version} + test @@ -84,17 +83,6 @@ test - - org.apache.spark - spark-hive_${scala.binary.version} - - - - org.scalatestplus - scalacheck-1-17_${scala.binary.version} - test - - org.apache.spark spark-sql_${scala.binary.version} @@ -117,15 +105,10 @@ test - - org.apache.hadoop - hadoop-client-api - - org.apache.hadoop hadoop-client-runtime - runtime + test +- Licensed to the Apache Software Foundation (ASF) under one or more +- contributor license agreements. See the NOTICE file distributed with +- this work for additional information regarding copyright ownership. +- The ASF licenses this file to You under the Apache License, Version 2.0 +- (the "License"); you may not use this file except in compliance with +- the License. You may obtain a copy of the License at +- +- http://www.apache.org/licenses/LICENSE-2.0 +- +- Unless required by applicable law or agreed to in writing, software +- distributed under the License is distributed on an "AS IS" BASIS, +- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +- See the License for the specific language governing permissions and +- limitations under the License. +--> # Kyuubi Spark Listener Extension ## Functions - [x] All `listener` extensions can be implemented in this module, like `QueryExecutionListener` and `ExtraListener` -- [x] Add `SparkOperationLineageQueryExecutionListener` to extends spark `QueryExecutionListener` +- [x] Add `SparkOperationLineageQueryExecutionListener` to extends spark `QueryExecutionListener` - [x] SQL lineage parsing will be triggered after SQL execution and will be written to the json logger file ## Build ```shell -build/mvn clean package -pl :kyuubi-spark-lineage_2.12 -Dspark.version=3.2.1 +build/mvn clean package -DskipTests -pl :kyuubi-spark-lineage_2.12 -am -Dspark.version=3.2.1 ``` ### Supported Apache Spark Versions @@ -34,6 +34,8 @@ build/mvn clean package -pl :kyuubi-spark-lineage_2.12 -Dspark.version=3.2.1 `-Dspark.version=` - [x] master +- [ ] 3.4.x - [x] 3.3.x (default) - [x] 3.2.x - [x] 3.1.x + diff --git a/extensions/spark/kyuubi-spark-lineage/pom.xml b/extensions/spark/kyuubi-spark-lineage/pom.xml index 74c05299d..270bf4d04 100644 --- a/extensions/spark/kyuubi-spark-lineage/pom.xml +++ b/extensions/spark/kyuubi-spark-lineage/pom.xml @@ -21,16 +21,21 @@ org.apache.kyuubi kyuubi-parent - 1.7.0-SNAPSHOT + 1.9.0-SNAPSHOT ../../../pom.xml - kyuubi-spark-lineage_2.12 + kyuubi-spark-lineage_${scala.binary.version} jar Kyuubi Dev Spark Lineage Extension https://kyuubi.apache.org/ + + org.apache.kyuubi + kyuubi-util-scala_${scala.binary.version} + ${project.version} + org.apache.spark spark-sql_${scala.binary.version} @@ -38,16 +43,101 @@ - commons-collections - commons-collections - test + org.apache.kyuubi + kyuubi-common_${scala.binary.version} + ${project.version} + provided org.apache.kyuubi - kyuubi-common_${scala.binary.version} + kyuubi-events_${scala.binary.version} ${project.version} - test + provided + + + + commons-collections + commons-collections + provided + + + + com.google.guava + guava + provided + + + + com.fasterxml.jackson.core + jackson-annotations + provided + + + + com.fasterxml.jackson.core + jackson-core + provided + + + + com.fasterxml.jackson.core + jackson-databind + provided + + + + org.apache.httpcomponents + httpclient + provided + + + + commons-lang + commons-lang + provided + + + + org.apache.commons + commons-lang3 + provided + + + + org.apache.atlas + atlas-client-v2 + ${atlas.version} + + + org.slf4j + slf4j-log4j12 + + + org.slf4j + slf4j-api + + + org.slf4j + jul-to-slf4j + + + commons-logging + commons-logging + + + org.apache.hadoop + hadoop-common + + + org.springframework + spring-context + + + org.apache.commons + commons-text + + @@ -77,11 +167,9 @@ spark-hive_${scala.binary.version} test - - ${project.basedir}/src/test/resources diff --git a/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/events/OperationLineageEvent.scala b/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/Lineage.scala similarity index 82% rename from extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/events/OperationLineageEvent.scala rename to extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/Lineage.scala index c69b45709..730deeb01 100644 --- a/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/events/OperationLineageEvent.scala +++ b/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/Lineage.scala @@ -15,9 +15,7 @@ * limitations under the License. */ -package org.apache.kyuubi.plugin.lineage.events - -import org.apache.spark.scheduler.SparkListenerEvent +package org.apache.kyuubi.plugin.lineage case class ColumnLineage(column: String, originalColumns: Set[String]) @@ -34,8 +32,9 @@ class Lineage( override def equals(other: Any): Boolean = other match { case otherLineage: Lineage => - otherLineage.inputTables == inputTables && otherLineage.outputTables == outputTables && - otherLineage.columnLineage == columnLineage + otherLineage.inputTables.toSet == inputTables.toSet && + otherLineage.outputTables.toSet == outputTables.toSet && + otherLineage.columnLineage.toSet == columnLineage.toSet case _ => false } @@ -60,9 +59,3 @@ object Lineage { new Lineage(inputTables, outputTables, newColumnLineage) } } - -case class OperationLineageEvent( - executionId: Long, - eventTime: Long, - lineage: Option[Lineage], - exception: Option[Throwable]) extends SparkListenerEvent diff --git a/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/LineageDispatcher.scala b/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/LineageDispatcher.scala new file mode 100644 index 000000000..b993f1428 --- /dev/null +++ b/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/LineageDispatcher.scala @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.plugin.lineage + +import org.apache.spark.sql.execution.QueryExecution + +import org.apache.kyuubi.plugin.lineage.dispatcher.{KyuubiEventDispatcher, SparkEventDispatcher} +import org.apache.kyuubi.plugin.lineage.dispatcher.atlas.AtlasLineageDispatcher + +trait LineageDispatcher { + + def send(qe: QueryExecution, lineage: Option[Lineage]): Unit + + def onFailure(qe: QueryExecution, exception: Exception): Unit = {} + +} + +object LineageDispatcher { + + def apply(dispatcherType: String): LineageDispatcher = { + LineageDispatcherType.withName(dispatcherType) match { + case LineageDispatcherType.SPARK_EVENT => new SparkEventDispatcher() + case LineageDispatcherType.KYUUBI_EVENT => new KyuubiEventDispatcher() + case LineageDispatcherType.ATLAS => new AtlasLineageDispatcher() + case _ => throw new UnsupportedOperationException( + s"Unsupported lineage dispatcher: $dispatcherType.") + } + } + +} diff --git a/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/LineageDispatcherType.scala b/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/LineageDispatcherType.scala new file mode 100644 index 000000000..8e07f6d77 --- /dev/null +++ b/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/LineageDispatcherType.scala @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.plugin.lineage + +object LineageDispatcherType extends Enumeration { + type LineageDispatcherType = Value + + val SPARK_EVENT, KYUUBI_EVENT, ATLAS = Value +} diff --git a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/RuleEliminateMarker.scala b/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/LineageParserProvider.scala similarity index 74% rename from extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/RuleEliminateMarker.scala rename to extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/LineageParserProvider.scala index d2da72570..665efef10 100644 --- a/extensions/spark/kyuubi-spark-authz/src/main/scala/org/apache/kyuubi/plugin/spark/authz/util/RuleEliminateMarker.scala +++ b/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/LineageParserProvider.scala @@ -15,13 +15,14 @@ * limitations under the License. */ -package org.apache.kyuubi.plugin.spark.authz.util +package org.apache.kyuubi.plugin.lineage +import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan -import org.apache.spark.sql.catalyst.rules.Rule -class RuleEliminateMarker extends Rule[LogicalPlan] { - override def apply(plan: LogicalPlan): LogicalPlan = { - plan.transformUp { case rf: RowFilterAndDataMaskingMarker => rf.child } +import org.apache.kyuubi.plugin.lineage.helper.SparkSQLLineageParseHelper +object LineageParserProvider { + def parse(spark: SparkSession, plan: LogicalPlan): Lineage = { + SparkSQLLineageParseHelper(spark).parse(plan) } } diff --git a/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/SparkOperationLineageQueryExecutionListener.scala b/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/SparkOperationLineageQueryExecutionListener.scala index c27d2eb8b..b83117cde 100644 --- a/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/SparkOperationLineageQueryExecutionListener.scala +++ b/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/SparkOperationLineageQueryExecutionListener.scala @@ -17,24 +17,25 @@ package org.apache.kyuubi.plugin.lineage -import org.apache.spark.kyuubi.lineage.SparkContextHelper +import org.apache.spark.kyuubi.lineage.{LineageConf, SparkContextHelper} import org.apache.spark.sql.execution.QueryExecution import org.apache.spark.sql.util.QueryExecutionListener -import org.apache.kyuubi.plugin.lineage.events.OperationLineageEvent import org.apache.kyuubi.plugin.lineage.helper.SparkSQLLineageParseHelper class SparkOperationLineageQueryExecutionListener extends QueryExecutionListener { + private lazy val dispatchers: Seq[LineageDispatcher] = { + SparkContextHelper.getConf(LineageConf.DISPATCHERS).map(LineageDispatcher(_)) + } + override def onSuccess(funcName: String, qe: QueryExecution, durationNs: Long): Unit = { val lineage = - SparkSQLLineageParseHelper(qe.sparkSession).transformToLineage(qe.id, qe.optimizedPlan) - val event = OperationLineageEvent(qe.id, System.currentTimeMillis(), lineage, None) - SparkContextHelper.postEventToSparkListenerBus(event) + SparkSQLLineageParseHelper(qe.sparkSession).transformToLineage(qe.id, qe.analyzed) + dispatchers.foreach(_.send(qe, lineage)) } override def onFailure(funcName: String, qe: QueryExecution, exception: Exception): Unit = { - val event = OperationLineageEvent(qe.id, System.currentTimeMillis(), None, Some(exception)) - SparkContextHelper.postEventToSparkListenerBus(event) + dispatchers.foreach(_.onFailure(qe, exception)) } } diff --git a/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/dispatcher/KyuubiEventDispatcher.scala b/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/dispatcher/KyuubiEventDispatcher.scala new file mode 100644 index 000000000..6a9e65948 --- /dev/null +++ b/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/dispatcher/KyuubiEventDispatcher.scala @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.plugin.lineage.dispatcher + +import org.apache.spark.sql.execution.QueryExecution + +import org.apache.kyuubi.Utils +import org.apache.kyuubi.events.{EventBus, KyuubiEvent} +import org.apache.kyuubi.plugin.lineage.{Lineage, LineageDispatcher} + +class KyuubiEventDispatcher extends LineageDispatcher { + + override def send(qe: QueryExecution, lineage: Option[Lineage]): Unit = { + val event = OperationLineageKyuubiEvent(qe.id, System.currentTimeMillis(), lineage, None) + EventBus.post(event) + } + + override def onFailure(qe: QueryExecution, exception: Exception): Unit = { + val event = + OperationLineageKyuubiEvent(qe.id, System.currentTimeMillis(), None, Some(exception)) + EventBus.post(event) + } + +} + +case class OperationLineageKyuubiEvent( + executionId: Long, + eventTime: Long, + lineage: Option[Lineage], + exception: Option[Throwable]) extends KyuubiEvent { + override def partitions: Seq[(String, String)] = + ("day", Utils.getDateFromTimestamp(eventTime)) :: Nil +} diff --git a/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/dispatcher/SparkEventDispatcher.scala b/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/dispatcher/SparkEventDispatcher.scala new file mode 100644 index 000000000..36fbbb7d4 --- /dev/null +++ b/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/dispatcher/SparkEventDispatcher.scala @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.plugin.lineage.dispatcher + +import org.apache.spark.kyuubi.lineage.SparkContextHelper +import org.apache.spark.scheduler.SparkListenerEvent +import org.apache.spark.sql.execution.QueryExecution + +import org.apache.kyuubi.plugin.lineage.{Lineage, LineageDispatcher} + +class SparkEventDispatcher extends LineageDispatcher { + + override def send(qe: QueryExecution, lineage: Option[Lineage]): Unit = { + val event = OperationLineageSparkEvent(qe.id, System.currentTimeMillis(), lineage, None) + SparkContextHelper.postEventToSparkListenerBus(event) + } + + override def onFailure(qe: QueryExecution, exception: Exception): Unit = { + val event = OperationLineageSparkEvent(qe.id, System.currentTimeMillis(), None, Some(exception)) + SparkContextHelper.postEventToSparkListenerBus(event) + } +} + +case class OperationLineageSparkEvent( + executionId: Long, + eventTime: Long, + lineage: Option[Lineage], + exception: Option[Throwable]) extends SparkListenerEvent diff --git a/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/dispatcher/atlas/AtlasClient.scala b/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/dispatcher/atlas/AtlasClient.scala new file mode 100644 index 000000000..15b127182 --- /dev/null +++ b/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/dispatcher/atlas/AtlasClient.scala @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.plugin.lineage.dispatcher.atlas + +import java.util.Locale + +import com.google.common.annotations.VisibleForTesting +import org.apache.atlas.AtlasClientV2 +import org.apache.atlas.model.instance.AtlasEntity +import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo +import org.apache.commons.lang3.StringUtils +import org.apache.hadoop.util.ShutdownHookManager + +import org.apache.kyuubi.plugin.lineage.dispatcher.atlas.AtlasClientConf._ + +trait AtlasClient extends AutoCloseable { + def send(entities: Seq[AtlasEntity]): Unit +} + +class AtlasRestClient(conf: AtlasClientConf) extends AtlasClient { + + private val atlasClient: AtlasClientV2 = { + val serverUrl = conf.get(ATLAS_REST_ENDPOINT).split(",") + val username = conf.get(CLIENT_USERNAME) + val password = conf.get(CLIENT_PASSWORD) + if (StringUtils.isNoneBlank(username, password)) { + new AtlasClientV2(serverUrl, Array(username, password)) + } else { + new AtlasClientV2(serverUrl: _*) + } + } + + override def send(entities: Seq[AtlasEntity]): Unit = { + val entitiesWithExtInfo = new AtlasEntitiesWithExtInfo() + entities.foreach(entitiesWithExtInfo.addEntity) + atlasClient.createEntities(entitiesWithExtInfo) + } + + override def close(): Unit = { + if (atlasClient != null) { + atlasClient.close() + } + } +} + +object AtlasClient { + + @volatile private var client: AtlasClient = _ + + def getClient(): AtlasClient = { + if (client == null) { + AtlasClient.synchronized { + if (client == null) { + val clientConf = AtlasClientConf.getConf() + client = clientConf.get(CLIENT_TYPE).toLowerCase(Locale.ROOT) match { + case "rest" => new AtlasRestClient(clientConf) + case unknown => throw new RuntimeException(s"Unsupported client type: $unknown.") + } + registerCleanupShutdownHook(client) + } + } + } + client + } + + private def registerCleanupShutdownHook(client: AtlasClient): Unit = { + ShutdownHookManager.get.addShutdownHook( + () => { + if (client != null) { + client.close() + } + }, + Integer.MAX_VALUE) + } + + @VisibleForTesting + private[dispatcher] def setClient(newClient: AtlasClient): Unit = { + client = newClient + } + +} diff --git a/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/dispatcher/atlas/AtlasClientConf.scala b/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/dispatcher/atlas/AtlasClientConf.scala new file mode 100644 index 000000000..03b1a83e0 --- /dev/null +++ b/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/dispatcher/atlas/AtlasClientConf.scala @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.plugin.lineage.dispatcher.atlas + +import org.apache.atlas.ApplicationProperties +import org.apache.commons.configuration.Configuration +import org.apache.spark.kyuubi.lineage.SparkContextHelper + +class AtlasClientConf(configuration: Configuration) { + + def get(entry: ConfigEntry): String = { + configuration.getProperty(entry.key) match { + case s: String => s + case l: List[_] => l.mkString(",") + case o if o != null => o.toString + case _ => entry.defaultValue + } + } + +} + +object AtlasClientConf { + + private lazy val clientConf: AtlasClientConf = { + val conf = ApplicationProperties.get() + SparkContextHelper.globalSparkContext.getConf.getAllWithPrefix("spark.atlas.") + .foreach { case (k, v) => conf.setProperty(s"atlas.$k", v) } + new AtlasClientConf(conf) + } + + def getConf(): AtlasClientConf = clientConf + + val ATLAS_REST_ENDPOINT = ConfigEntry("atlas.rest.address", "http://localhost:21000") + + val CLIENT_TYPE = ConfigEntry("atlas.client.type", "rest") + val CLIENT_USERNAME = ConfigEntry("atlas.client.username", null) + val CLIENT_PASSWORD = ConfigEntry("atlas.client.password", null) + + val CLUSTER_NAME = ConfigEntry("atlas.cluster.name", "primary") + + val COLUMN_LINEAGE_ENABLED = ConfigEntry("atlas.hook.spark.column.lineage.enabled", "true") +} diff --git a/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/dispatcher/atlas/AtlasEntityHelper.scala b/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/dispatcher/atlas/AtlasEntityHelper.scala new file mode 100644 index 000000000..cfa19b7aa --- /dev/null +++ b/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/dispatcher/atlas/AtlasEntityHelper.scala @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.plugin.lineage.dispatcher.atlas + +import scala.collection.JavaConverters._ + +import org.apache.atlas.model.instance.{AtlasEntity, AtlasObjectId, AtlasRelatedObjectId} +import org.apache.spark.kyuubi.lineage.{LineageConf, SparkContextHelper} +import org.apache.spark.sql.execution.QueryExecution + +import org.apache.kyuubi.plugin.lineage.Lineage +import org.apache.kyuubi.plugin.lineage.helper.SparkListenerHelper + +/** + * The helpers for Atlas spark entities from Lineage. + * The Atlas spark models refer to : + * https://github.com/apache/atlas/blob/master/addons/models/1000-Hadoop/1100-spark_model.json + */ +object AtlasEntityHelper { + + /** + * Generate `spark_process` Atlas Entity from Lineage. + * @param qe + * @param lineage + * @return + */ + def processEntity(qe: QueryExecution, lineage: Lineage): AtlasEntity = { + val entity = new AtlasEntity(PROCESS_TYPE) + + val appId = SparkContextHelper.globalSparkContext.applicationId + val appName = SparkContextHelper.globalSparkContext.appName match { + case "Spark shell" => s"Spark Job $appId" + case default => s"$default $appId" + } + + entity.setAttribute("qualifiedName", appId) + entity.setAttribute("name", appName) + entity.setAttribute("currUser", SparkListenerHelper.currentUser) + SparkListenerHelper.sessionUser.foreach(entity.setAttribute("remoteUser", _)) + entity.setAttribute("executionId", qe.id) + entity.setAttribute("details", qe.toString()) + entity.setAttribute("sparkPlanDescription", qe.sparkPlan.toString()) + + // TODO add entity type instead of parsing from string + val inputs = lineage.inputTables.flatMap(tableObjectId).map { objId => + relatedObjectId(objId, RELATIONSHIP_DATASET_PROCESS_INPUTS) + } + val outputs = lineage.outputTables.flatMap(tableObjectId).map { objId => + relatedObjectId(objId, RELATIONSHIP_PROCESS_DATASET_OUTPUTS) + } + + entity.setRelationshipAttribute("inputs", inputs.asJava) + entity.setRelationshipAttribute("outputs", outputs.asJava) + + entity + } + + /** + * Generate `spark_column_lineage` Atlas Entity from Lineage. + * @param processEntity + * @param lineage + * @return + */ + def columnLineageEntities(processEntity: AtlasEntity, lineage: Lineage): Seq[AtlasEntity] = { + lineage.columnLineage.flatMap(columnLineage => { + val inputs = columnLineage.originalColumns.flatMap(columnObjectId).map { objId => + relatedObjectId(objId, RELATIONSHIP_DATASET_PROCESS_INPUTS) + } + val outputs = Option(columnLineage.column).flatMap(columnObjectId).map { objId => + relatedObjectId(objId, RELATIONSHIP_PROCESS_DATASET_OUTPUTS) + }.toSeq + + if (inputs.nonEmpty && outputs.nonEmpty) { + val entity = new AtlasEntity(COLUMN_LINEAGE_TYPE) + val outputColumnName = buildColumnQualifiedName(columnLineage.column).get + val qualifiedName = + s"${processEntity.getAttribute("qualifiedName")}:${outputColumnName}" + entity.setAttribute("qualifiedName", qualifiedName) + entity.setAttribute("name", qualifiedName) + entity.setRelationshipAttribute("inputs", inputs.asJava) + entity.setRelationshipAttribute("outputs", outputs.asJava) + entity.setRelationshipAttribute( + "process", + relatedObjectId(objectId(processEntity), RELATIONSHIP_SPARK_PROCESS_COLUMN_LINEAGE)) + Some(entity) + } else { + None + } + }) + } + + def tableObjectId(tableName: String): Option[AtlasObjectId] = { + buildTableQualifiedName(tableName) + .map(new AtlasObjectId(HIVE_TABLE_TYPE, "qualifiedName", _)) + } + + def buildTableQualifiedName(tableName: String): Option[String] = { + val defaultCatalog = LineageConf.DEFAULT_CATALOG + tableName.split('.') match { + case Array(`defaultCatalog`, db, table) => + Some(s"${db.toLowerCase}.${table.toLowerCase}@$cluster") + case _ => + None + } + } + + def columnObjectId(columnName: String): Option[AtlasObjectId] = { + buildColumnQualifiedName(columnName) + .map(new AtlasObjectId(HIVE_COLUMN_TYPE, "qualifiedName", _)) + } + + def buildColumnQualifiedName(columnName: String): Option[String] = { + val defaultCatalog = LineageConf.DEFAULT_CATALOG + columnName.split('.') match { + case Array(`defaultCatalog`, db, table, column) => + Some(s"${db.toLowerCase}.${table.toLowerCase}.${column.toLowerCase}@$cluster") + case _ => + None + } + } + + def objectId(entity: AtlasEntity): AtlasObjectId = { + val objId = new AtlasObjectId(entity.getGuid, entity.getTypeName) + objId.setUniqueAttributes(Map("qualifiedName" -> entity.getAttribute("qualifiedName")).asJava) + objId + } + + def relatedObjectId(objectId: AtlasObjectId, relationshipType: String): AtlasRelatedObjectId = { + new AtlasRelatedObjectId(objectId, relationshipType) + } + + lazy val cluster = AtlasClientConf.getConf().get(AtlasClientConf.CLUSTER_NAME) + lazy val columnLineageEnabled = + AtlasClientConf.getConf().get(AtlasClientConf.COLUMN_LINEAGE_ENABLED).toBoolean + + val HIVE_TABLE_TYPE = "hive_table" + val HIVE_COLUMN_TYPE = "hive_column" + val PROCESS_TYPE = "spark_process" + val COLUMN_LINEAGE_TYPE = "spark_column_lineage" + val RELATIONSHIP_DATASET_PROCESS_INPUTS = "dataset_process_inputs" + val RELATIONSHIP_PROCESS_DATASET_OUTPUTS = "process_dataset_outputs" + val RELATIONSHIP_SPARK_PROCESS_COLUMN_LINEAGE = "spark_process_column_lineages" + +} diff --git a/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/dispatcher/atlas/AtlasLineageDispatcher.scala b/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/dispatcher/atlas/AtlasLineageDispatcher.scala new file mode 100644 index 000000000..c66b51107 --- /dev/null +++ b/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/dispatcher/atlas/AtlasLineageDispatcher.scala @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.plugin.lineage.dispatcher.atlas + +import org.apache.spark.internal.Logging +import org.apache.spark.sql.execution.QueryExecution + +import org.apache.kyuubi.plugin.lineage.{Lineage, LineageDispatcher} +import org.apache.kyuubi.plugin.lineage.dispatcher.atlas.AtlasEntityHelper.columnLineageEnabled + +class AtlasLineageDispatcher extends LineageDispatcher with Logging { + + override def send(qe: QueryExecution, lineageOpt: Option[Lineage]): Unit = { + try { + lineageOpt.filter(l => l.inputTables.nonEmpty || l.outputTables.nonEmpty).foreach(lineage => { + val processEntity = AtlasEntityHelper.processEntity(qe, lineage) + val columnLineageEntities = if (lineage.columnLineage.nonEmpty && columnLineageEnabled) { + AtlasEntityHelper.columnLineageEntities(processEntity, lineage) + } else { + Seq.empty + } + AtlasClient.getClient().send(processEntity +: columnLineageEntities) + }) + } catch { + case t: Throwable => + logWarning("Send lineage to atlas failed.", t) + } + } + + override def onFailure(qe: QueryExecution, exception: Exception): Unit = { + // ignore + } + +} diff --git a/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/dispatcher/atlas/ConfigEntry.scala b/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/dispatcher/atlas/ConfigEntry.scala new file mode 100644 index 000000000..3f9d9831d --- /dev/null +++ b/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/dispatcher/atlas/ConfigEntry.scala @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.plugin.lineage.dispatcher.atlas + +case class ConfigEntry(key: String, defaultValue: String) diff --git a/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/helper/SemanticVersion.scala b/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/helper/SemanticVersion.scala deleted file mode 100644 index a4a8b2e0e..000000000 --- a/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/helper/SemanticVersion.scala +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kyuubi.plugin.lineage.helper - -/** - * Encapsulate a component (Kyuubi/Spark/Hive/Flink etc.) version - * for the convenience of version checks. - */ -case class SemanticVersion(majorVersion: Int, minorVersion: Int) { - - def isVersionAtMost(targetVersionString: String): Boolean = { - this.compareVersion( - targetVersionString, - (targetMajor: Int, targetMinor: Int, runtimeMajor: Int, runtimeMinor: Int) => - (runtimeMajor < targetMajor) || { - runtimeMajor == targetMajor && runtimeMinor <= targetMinor - }) - } - - def isVersionAtLeast(targetVersionString: String): Boolean = { - this.compareVersion( - targetVersionString, - (targetMajor: Int, targetMinor: Int, runtimeMajor: Int, runtimeMinor: Int) => - (runtimeMajor > targetMajor) || { - runtimeMajor == targetMajor && runtimeMinor >= targetMinor - }) - } - - def isVersionEqualTo(targetVersionString: String): Boolean = { - this.compareVersion( - targetVersionString, - (targetMajor: Int, targetMinor: Int, runtimeMajor: Int, runtimeMinor: Int) => - runtimeMajor == targetMajor && runtimeMinor == targetMinor) - } - - def compareVersion( - targetVersionString: String, - callback: (Int, Int, Int, Int) => Boolean): Boolean = { - val targetVersion = SemanticVersion(targetVersionString) - val targetMajor = targetVersion.majorVersion - val targetMinor = targetVersion.minorVersion - callback(targetMajor, targetMinor, this.majorVersion, this.minorVersion) - } - - override def toString: String = s"$majorVersion.$minorVersion" -} - -object SemanticVersion { - - def apply(versionString: String): SemanticVersion = { - """^(\d+)\.(\d+)(\..*)?$""".r.findFirstMatchIn(versionString) match { - case Some(m) => - SemanticVersion(m.group(1).toInt, m.group(2).toInt) - case None => - throw new IllegalArgumentException(s"Tried to parse '$versionString' as a project" + - s" version string, but it could not find the major and minor version numbers.") - } - } -} diff --git a/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/helper/SparkListenerHelper.scala b/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/helper/SparkListenerHelper.scala index f2808a4e9..6093e8660 100644 --- a/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/helper/SparkListenerHelper.scala +++ b/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/helper/SparkListenerHelper.scala @@ -17,25 +17,20 @@ package org.apache.kyuubi.plugin.lineage.helper +import org.apache.hadoop.security.UserGroupInformation import org.apache.spark.SPARK_VERSION +import org.apache.spark.kyuubi.lineage.SparkContextHelper + +import org.apache.kyuubi.util.SemanticVersion object SparkListenerHelper { - lazy val sparkMajorMinorVersion: (Int, Int) = { - val runtimeSparkVer = org.apache.spark.SPARK_VERSION - val runtimeVersion = SemanticVersion(runtimeSparkVer) - (runtimeVersion.majorVersion, runtimeVersion.minorVersion) - } + lazy val SPARK_RUNTIME_VERSION: SemanticVersion = SemanticVersion(SPARK_VERSION) - def isSparkVersionAtMost(targetVersionString: String): Boolean = { - SemanticVersion(SPARK_VERSION).isVersionAtMost(targetVersionString) - } + def currentUser: String = UserGroupInformation.getCurrentUser.getShortUserName - def isSparkVersionAtLeast(targetVersionString: String): Boolean = { - SemanticVersion(SPARK_VERSION).isVersionAtLeast(targetVersionString) - } + def sessionUser: Option[String] = + Option(SparkContextHelper.globalSparkContext.getLocalProperty(KYUUBI_SESSION_USER)) - def isSparkVersionEqualTo(targetVersionString: String): Boolean = { - SemanticVersion(SPARK_VERSION).isVersionEqualTo(targetVersionString) - } + final val KYUUBI_SESSION_USER = "kyuubi.session.user" } diff --git a/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/helper/SparkSQLLineageParseHelper.scala b/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/helper/SparkSQLLineageParseHelper.scala index f70e09126..273111464 100644 --- a/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/helper/SparkSQLLineageParseHelper.scala +++ b/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/kyuubi/plugin/lineage/helper/SparkSQLLineageParseHelper.scala @@ -18,15 +18,15 @@ package org.apache.kyuubi.plugin.lineage.helper import scala.collection.immutable.ListMap -import scala.util.{Failure, Success, Try} +import scala.util.Try import org.apache.spark.internal.Logging +import org.apache.spark.kyuubi.lineage.{LineageConf, SparkContextHelper} import org.apache.spark.sql.SparkSession import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.analysis.{NamedRelation, PersistedView, ViewType} import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, HiveTableRelation} -import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeSet, Expression, NamedExpression} -import org.apache.spark.sql.catalyst.expressions.ScalarSubquery +import org.apache.spark.sql.catalyst.expressions.{Alias, Attribute, AttributeSet, Expression, NamedExpression, ScalarSubquery} import org.apache.spark.sql.catalyst.expressions.aggregate.Count import org.apache.spark.sql.catalyst.plans.{LeftAnti, LeftSemi} import org.apache.spark.sql.catalyst.plans.logical._ @@ -36,8 +36,9 @@ import org.apache.spark.sql.execution.columnar.InMemoryRelation import org.apache.spark.sql.execution.datasources.LogicalRelation import org.apache.spark.sql.execution.datasources.v2.{DataSourceV2Relation, DataSourceV2ScanRelation} -import org.apache.kyuubi.plugin.lineage.events.Lineage -import org.apache.kyuubi.plugin.lineage.helper.SparkListenerHelper.isSparkVersionAtMost +import org.apache.kyuubi.plugin.lineage.Lineage +import org.apache.kyuubi.plugin.lineage.helper.SparkListenerHelper.SPARK_RUNTIME_VERSION +import org.apache.kyuubi.util.reflect.ReflectUtils._ trait LineageParser { def sparkSession: SparkSession @@ -52,7 +53,7 @@ trait LineageParser { val columnsLineage = extractColumnsLineage(plan, ListMap[Attribute, AttributeSet]()).toList.collect { case (k, attrs) => - k.name -> attrs.map(_.qualifiedName).toSet + k.name -> attrs.map(attr => (attr.qualifier :+ attr.name).mkString(".")).toSet } val (inputTables, outputTables) = columnsLineage.foldLeft((List[String](), List[String]())) { case ((inputs, outputs), (out, in)) => @@ -128,7 +129,7 @@ trait LineageParser { exp.toAttribute, if (!containsCountAll(exp.child)) references else references + exp.toAttribute.withName(AGGREGATE_COUNT_COLUMN_IDENTIFIER)) - case a: Attribute => a -> a.references + case a: Attribute => a -> AttributeSet(a) } ListMap(exps: _*) } @@ -149,6 +150,9 @@ trait LineageParser { attr.withQualifier(attr.qualifier.init) case attr if attr.name.equalsIgnoreCase(AGGREGATE_COUNT_COLUMN_IDENTIFIER) => attr.withQualifier(qualifier) + case attr if isNameWithQualifier(attr, qualifier) => + val newName = attr.name.split('.').last.stripPrefix("`").stripSuffix("`") + attr.withName(newName).withQualifier(qualifier) }) } } else { @@ -160,6 +164,12 @@ trait LineageParser { } } + private def isNameWithQualifier(attr: Attribute, qualifier: Seq[String]): Boolean = { + val nameTokens = attr.name.split('.') + val namespace = nameTokens.init.mkString(".") + nameTokens.length > 1 && namespace.endsWith(qualifier.mkString(".")) + } + private def mergeRelationColumnLineage( parentColumnsLineage: AttributeMap[AttributeSet], relationOutput: Seq[Attribute], @@ -180,40 +190,41 @@ trait LineageParser { plan match { // For command case p if p.nodeName == "CommandResult" => - val commandPlan = getPlanField[LogicalPlan]("commandLogicalPlan", plan) + val commandPlan = getField[LogicalPlan](plan, "commandLogicalPlan") extractColumnsLineage(commandPlan, parentColumnsLineage) case p if p.nodeName == "AlterViewAsCommand" => val query = - if (isSparkVersionAtMost("3.1")) { + if (SPARK_RUNTIME_VERSION <= "3.1") { sparkSession.sessionState.analyzer.execute(getQuery(plan)) } else { getQuery(plan) } - val view = getPlanField[TableIdentifier]("name", plan).unquotedString + val view = getV1TableName(getField[TableIdentifier](plan, "name").unquotedString) extractColumnsLineage(query, parentColumnsLineage).map { case (k, v) => k.withName(s"$view.${k.name}") -> v } case p if p.nodeName == "CreateViewCommand" - && getPlanField[ViewType]("viewType", plan) == PersistedView => - val view = getPlanField[TableIdentifier]("name", plan).unquotedString + && getField[ViewType](plan, "viewType") == PersistedView => + val view = getV1TableName(getField[TableIdentifier](plan, "name").unquotedString) val outputCols = - getPlanField[Seq[(String, Option[String])]]("userSpecifiedColumns", plan).map(_._1) + getField[Seq[(String, Option[String])]](plan, "userSpecifiedColumns").map(_._1) val query = - if (isSparkVersionAtMost("3.1")) { - sparkSession.sessionState.analyzer.execute(getPlanField[LogicalPlan]("child", plan)) + if (SPARK_RUNTIME_VERSION <= "3.1") { + sparkSession.sessionState.analyzer.execute(getField[LogicalPlan](plan, "child")) } else { - getPlanField[LogicalPlan]("plan", plan) + getField[LogicalPlan](plan, "plan") } - extractColumnsLineage(query, parentColumnsLineage).zipWithIndex.map { + val lineages = extractColumnsLineage(query, parentColumnsLineage).zipWithIndex.map { case ((k, v), i) if outputCols.nonEmpty => k.withName(s"$view.${outputCols(i)}") -> v case ((k, v), _) => k.withName(s"$view.${k.name}") -> v - } + }.toSeq + ListMap[Attribute, AttributeSet](lineages: _*) case p if p.nodeName == "CreateDataSourceTableAsSelectCommand" => - val table = getPlanField[CatalogTable]("table", plan).qualifiedName + val table = getV1TableName(getField[CatalogTable](plan, "table").qualifiedName) extractColumnsLineage(getQuery(plan), parentColumnsLineage).map { case (k, v) => k.withName(s"$table.${k.name}") -> v } @@ -221,7 +232,7 @@ trait LineageParser { case p if p.nodeName == "CreateHiveTableAsSelectCommand" || p.nodeName == "OptimizedCreateHiveTableAsSelectCommand" => - val table = getPlanField[CatalogTable]("tableDesc", plan).qualifiedName + val table = getV1TableName(getField[CatalogTable](plan, "tableDesc").qualifiedName) extractColumnsLineage(getQuery(plan), parentColumnsLineage).map { case (k, v) => k.withName(s"$table.${k.name}") -> v } @@ -230,17 +241,17 @@ trait LineageParser { if p.nodeName == "CreateTableAsSelect" || p.nodeName == "ReplaceTableAsSelect" => val (table, namespace, catalog) = - if (isSparkVersionAtMost("3.2")) { + if (SPARK_RUNTIME_VERSION <= "3.2") { ( - getPlanField[Identifier]("tableName", plan).name, - getPlanField[Identifier]("tableName", plan).namespace.mkString("."), - getPlanField[TableCatalog]("catalog", plan).name()) + getField[Identifier](plan, "tableName").name, + getField[Identifier](plan, "tableName").namespace.mkString("."), + getField[TableCatalog](plan, "catalog").name()) } else { ( - getPlanMethod[Identifier]("tableName", plan).name(), - getPlanMethod[Identifier]("tableName", plan).namespace().mkString("."), - getCurrentPlanField[CatalogPlugin]( - getPlanMethod[LogicalPlan]("left", plan), + invokeAs[Identifier](plan, "tableName").name(), + invokeAs[Identifier](plan, "tableName").namespace().mkString("."), + getField[CatalogPlugin]( + invokeAs[LogicalPlan](plan, "name"), "catalog").name()) } extractColumnsLineage(getQuery(plan), parentColumnsLineage).map { case (k, v) => @@ -248,8 +259,9 @@ trait LineageParser { } case p if p.nodeName == "InsertIntoDataSourceCommand" => - val logicalRelation = getPlanField[LogicalRelation]("logicalRelation", plan) - val table = logicalRelation.catalogTable.map(_.qualifiedName).getOrElse("") + val logicalRelation = getField[LogicalRelation](plan, "logicalRelation") + val table = logicalRelation + .catalogTable.map(t => getV1TableName(t.qualifiedName)).getOrElse("") extractColumnsLineage(getQuery(plan), parentColumnsLineage).map { case (k, v) if table.nonEmpty => k.withName(s"$table.${k.name}") -> v @@ -257,8 +269,9 @@ trait LineageParser { case p if p.nodeName == "InsertIntoHadoopFsRelationCommand" => val table = - getPlanField[Option[CatalogTable]]("catalogTable", plan).map(_.qualifiedName).getOrElse( - "") + getField[Option[CatalogTable]](plan, "catalogTable") + .map(t => getV1TableName(t.qualifiedName)) + .getOrElse("") extractColumnsLineage(getQuery(plan), parentColumnsLineage).map { case (k, v) if table.nonEmpty => k.withName(s"$table.${k.name}") -> v @@ -268,15 +281,15 @@ trait LineageParser { if p.nodeName == "InsertIntoDataSourceDirCommand" || p.nodeName == "InsertIntoHiveDirCommand" => val dir = - getPlanField[CatalogStorageFormat]("storage", plan).locationUri.map(_.toString).getOrElse( - "") + getField[CatalogStorageFormat](plan, "storage").locationUri.map(_.toString) + .getOrElse("") extractColumnsLineage(getQuery(plan), parentColumnsLineage).map { case (k, v) if dir.nonEmpty => k.withName(s"`$dir`.${k.name}") -> v } case p if p.nodeName == "InsertIntoHiveTable" => - val table = getPlanField[CatalogTable]("table", plan).qualifiedName + val table = getV1TableName(getField[CatalogTable](plan, "table").qualifiedName) extractColumnsLineage(getQuery(plan), parentColumnsLineage).map { case (k, v) => k.withName(s"$table.${k.name}") -> v } @@ -288,14 +301,14 @@ trait LineageParser { if p.nodeName == "AppendData" || p.nodeName == "OverwriteByExpression" || p.nodeName == "OverwritePartitionsDynamic" => - val table = getPlanField[NamedRelation]("table", plan).name + val table = getV2TableName(getField[NamedRelation](plan, "table")) extractColumnsLineage(getQuery(plan), parentColumnsLineage).map { case (k, v) => k.withName(s"$table.${k.name}") -> v } case p if p.nodeName == "MergeIntoTable" => - val matchedActions = getPlanField[Seq[MergeAction]]("matchedActions", plan) - val notMatchedActions = getPlanField[Seq[MergeAction]]("notMatchedActions", plan) + val matchedActions = getField[Seq[MergeAction]](plan, "matchedActions") + val notMatchedActions = getField[Seq[MergeAction]](plan, "notMatchedActions") val allAssignments = (matchedActions ++ notMatchedActions).collect { case UpdateAction(_, assignments) => assignments case InsertAction(_, assignments) => assignments @@ -303,19 +316,24 @@ trait LineageParser { val nextColumnsLlineage = ListMap(allAssignments.map { assignment => ( assignment.key.asInstanceOf[Attribute], - AttributeSet(assignment.value.asInstanceOf[Attribute])) + assignment.value.references) }: _*) - val targetTable = getPlanField[LogicalPlan]("targetTable", plan) - val sourceTable = getPlanField[LogicalPlan]("sourceTable", plan) + val targetTable = getField[LogicalPlan](plan, "targetTable") + val sourceTable = getField[LogicalPlan](plan, "sourceTable") val targetColumnsLineage = extractColumnsLineage( targetTable, nextColumnsLlineage.map { case (k, _) => (k, AttributeSet(k)) }) val sourceColumnsLineage = extractColumnsLineage(sourceTable, nextColumnsLlineage) val targetColumnsWithTargetTable = targetColumnsLineage.values.flatten.map { column => - column.withName(s"${column.qualifiedName}") + val unquotedQualifiedName = (column.qualifier :+ column.name).mkString(".") + column.withName(unquotedQualifiedName) } ListMap(targetColumnsWithTargetTable.zip(sourceColumnsLineage.values).toSeq: _*) + case p if p.nodeName == "WithCTE" => + val optimized = sparkSession.sessionState.optimizer.execute(p) + extractColumnsLineage(optimized, parentColumnsLineage) + // For query case p: Project => val nextColumnsLineage = @@ -327,6 +345,45 @@ trait LineageParser { joinColumnsLineage(parentColumnsLineage, getSelectColumnLineage(p.aggregateExpressions)) p.children.map(extractColumnsLineage(_, nextColumnsLineage)).reduce(mergeColumnsLineage) + case p: Expand => + val references = + p.projections.transpose.map(_.flatMap(x => x.references)).map(AttributeSet(_)) + + val childColumnsLineage = ListMap(p.output.zip(references): _*) + val nextColumnsLineage = + joinColumnsLineage(parentColumnsLineage, childColumnsLineage) + p.children.map(extractColumnsLineage(_, nextColumnsLineage)).reduce(mergeColumnsLineage) + + case p: Generate => + val generateColumnsLineageWithId = + ListMap(p.generatorOutput.map(attrRef => (attrRef.toAttribute.exprId, p.references)): _*) + + val nextColumnsLineage = parentColumnsLineage.map { + case (key, attrRefs) => + key -> AttributeSet(attrRefs.flatMap(attr => + generateColumnsLineageWithId.getOrElse( + attr.exprId, + AttributeSet(attr)))) + } + p.children.map(extractColumnsLineage(_, nextColumnsLineage)).reduce(mergeColumnsLineage) + + case p: Window => + val windowColumnsLineage = + ListMap(p.windowExpressions.map(exp => (exp.toAttribute, exp.references)): _*) + + val nextColumnsLineage = if (parentColumnsLineage.isEmpty) { + ListMap(p.child.output.map(attr => (attr, attr.references)): _*) ++ windowColumnsLineage + } else { + parentColumnsLineage.map { + case (k, _) if windowColumnsLineage.contains(k) => + k -> windowColumnsLineage(k) + case (k, attrs) => + k -> AttributeSet(attrs.flatten(attr => + windowColumnsLineage.getOrElse(attr, AttributeSet(attr)))) + } + } + p.children.map(extractColumnsLineage(_, nextColumnsLineage)).reduce(mergeColumnsLineage) + case p: Join => p.joinType match { case LeftSemi | LeftAnti => @@ -337,38 +394,69 @@ trait LineageParser { } case p: Union => - // merge all children in to one derivedColumns - val childrenUnion = - p.children.map(extractColumnsLineage(_, ListMap[Attribute, AttributeSet]())).map( - _.values).reduce { - (left, right) => - left.zip(right).map(attr => attr._1 ++ attr._2) + val childrenColumnsLineage = + // support for the multi-insert statement + if (p.output.isEmpty) { + p.children + .map(extractColumnsLineage(_, ListMap[Attribute, AttributeSet]())) + .reduce(mergeColumnsLineage) + } else { + // merge all children in to one derivedColumns + val childrenUnion = + p.children.map(extractColumnsLineage(_, ListMap[Attribute, AttributeSet]())).map( + _.values).reduce { + (left, right) => + left.zip(right).map(attr => attr._1 ++ attr._2) + } + ListMap(p.output.zip(childrenUnion): _*) } - val childrenColumnsLineage = ListMap(p.output.zip(childrenUnion): _*) joinColumnsLineage(parentColumnsLineage, childrenColumnsLineage) case p: LogicalRelation if p.catalogTable.nonEmpty => - val tableName = p.catalogTable.get.qualifiedName + val tableName = getV1TableName(p.catalogTable.get.qualifiedName) joinRelationColumnLineage(parentColumnsLineage, p.output, Seq(tableName)) case p: HiveTableRelation => - val tableName = p.tableMeta.qualifiedName + val tableName = getV1TableName(p.tableMeta.qualifiedName) joinRelationColumnLineage(parentColumnsLineage, p.output, Seq(tableName)) case p: DataSourceV2ScanRelation => - val tableName = p.name + val tableName = getV2TableName(p) joinRelationColumnLineage(parentColumnsLineage, p.output, Seq(tableName)) // For creating the view from v2 table, the logical plan of table will // be the `DataSourceV2Relation` not the `DataSourceV2ScanRelation`. // because the view from the table is not going to read it. case p: DataSourceV2Relation => - val tableName = p.name + val tableName = getV2TableName(p) joinRelationColumnLineage(parentColumnsLineage, p.output, Seq(tableName)) case p: LocalRelation => joinRelationColumnLineage(parentColumnsLineage, p.output, Seq(LOCAL_TABLE_IDENTIFIER)) + case _: OneRowRelation => + parentColumnsLineage.map { + case (k, attrs) => + k -> AttributeSet(attrs.map { + case attr + if attr.qualifier.nonEmpty && attr.qualifier.last.equalsIgnoreCase( + SUBQUERY_COLUMN_IDENTIFIER) => + attr.withQualifier(attr.qualifier.init) + case attr => attr + }) + } + + case p: View => + if (!p.isTempView && SparkContextHelper.getConf( + LineageConf.SKIP_PARSING_PERMANENT_VIEW_ENABLED)) { + val viewName = getV1TableName(p.desc.qualifiedName) + joinRelationColumnLineage(parentColumnsLineage, p.output, Seq(viewName)) + } else { + val viewColumnsLineage = + extractColumnsLineage(p.child, ListMap[Attribute, AttributeSet]()) + mergeRelationColumnLineage(parentColumnsLineage, p.output, viewColumnsLineage) + } + case p: InMemoryRelation => // get logical plan from cachedPlan val cachedTableLogical = findSparkPlanLogicalLink(Seq(p.cacheBuilder.cachedPlan)) @@ -391,47 +479,32 @@ trait LineageParser { } } - private def getPlanField[T](field: String, plan: LogicalPlan): T = { - getFieldVal[T](plan, field) - } - - private def getCurrentPlanField[T](curPlan: LogicalPlan, field: String): T = { - getFieldVal[T](curPlan, field) - } - - private def getPlanMethod[T](name: String, plan: LogicalPlan): T = { - getMethod[T](plan, name) - } - - private def getQuery(plan: LogicalPlan): LogicalPlan = { - getPlanField[LogicalPlan]("query", plan) - } + private def getQuery(plan: LogicalPlan): LogicalPlan = getField[LogicalPlan](plan, "query") - private def getFieldVal[T](o: Any, name: String): T = { - Try { - val field = o.getClass.getDeclaredField(name) - field.setAccessible(true) - field.get(o) - } match { - case Success(value) => value.asInstanceOf[T] - case Failure(e) => - val candidates = o.getClass.getDeclaredFields.map(_.getName).mkString("[", ",", "]") - throw new RuntimeException(s"$name not in $candidates", e) + private def getV2TableName(plan: NamedRelation): String = { + plan match { + case relation: DataSourceV2ScanRelation => + val catalog = relation.relation.catalog.map(_.name()).getOrElse(LineageConf.DEFAULT_CATALOG) + val database = relation.relation.identifier.get.namespace().mkString(".") + val table = relation.relation.identifier.get.name() + s"$catalog.$database.$table" + case relation: DataSourceV2Relation => + val catalog = relation.catalog.map(_.name()).getOrElse(LineageConf.DEFAULT_CATALOG) + val database = relation.identifier.get.namespace().mkString(".") + val table = relation.identifier.get.name() + s"$catalog.$database.$table" + case _ => + plan.name } } - private def getMethod[T](o: Any, name: String): T = { - Try { - val method = o.getClass.getDeclaredMethod(name) - method.invoke(o) - } match { - case Success(value) => value.asInstanceOf[T] - case Failure(e) => - val candidates = o.getClass.getDeclaredMethods.map(_.getName).mkString("[", ",", "]") - throw new RuntimeException(s"$name not in $candidates", e) + private def getV1TableName(qualifiedName: String): String = { + qualifiedName.split("\\.") match { + case Array(database, table) => + Seq(LineageConf.DEFAULT_CATALOG, database, table).filter(_.nonEmpty).mkString(".") + case _ => qualifiedName } } - } case class SparkSQLLineageParseHelper(sparkSession: SparkSession) extends LineageParser diff --git a/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/spark/kyuubi/lineage/LineageConf.scala b/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/spark/kyuubi/lineage/LineageConf.scala new file mode 100644 index 000000000..e264b1f35 --- /dev/null +++ b/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/spark/kyuubi/lineage/LineageConf.scala @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.kyuubi.lineage + +import org.apache.spark.internal.config.ConfigBuilder +import org.apache.spark.sql.internal.SQLConf + +import org.apache.kyuubi.plugin.lineage.LineageDispatcherType + +object LineageConf { + + val SKIP_PARSING_PERMANENT_VIEW_ENABLED = + ConfigBuilder("spark.kyuubi.plugin.lineage.skip.parsing.permanent.view.enabled") + .doc("Whether to skip the lineage parsing of permanent views") + .version("1.8.0") + .booleanConf + .createWithDefault(false) + + val DISPATCHERS = ConfigBuilder("spark.kyuubi.plugin.lineage.dispatchers") + .doc("The lineage dispatchers are implementations of " + + "`org.apache.kyuubi.plugin.lineage.LineageDispatcher` for dispatching lineage events.
        " + + "
      • SPARK_EVENT: send lineage event to spark event bus
        • " + + "
      • KYUUBI_EVENT: send lineage event to kyuubi event bus
        • " + + "
      • ATLAS: send lineage to apache atlas
        • " + + "
      ") + .version("1.8.0") + .stringConf + .toSequence + .checkValue( + _.toSet.subsetOf(LineageDispatcherType.values.map(_.toString)), + "Unsupported lineage dispatchers") + .createWithDefault(Seq(LineageDispatcherType.SPARK_EVENT.toString)) + + val DEFAULT_CATALOG: String = SQLConf.get.getConf(SQLConf.DEFAULT_CATALOG) + +} diff --git a/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/spark/kyuubi/lineage/SparkContextHelper.scala b/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/spark/kyuubi/lineage/SparkContextHelper.scala index e6272364f..6e0f0e5c8 100644 --- a/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/spark/kyuubi/lineage/SparkContextHelper.scala +++ b/extensions/spark/kyuubi-spark-lineage/src/main/scala/org/apache/spark/kyuubi/lineage/SparkContextHelper.scala @@ -18,6 +18,7 @@ package org.apache.spark.kyuubi.lineage import org.apache.spark.SparkContext +import org.apache.spark.internal.config.ConfigEntry import org.apache.spark.scheduler.SparkListenerEvent import org.apache.spark.sql.SparkSession @@ -31,4 +32,11 @@ object SparkContextHelper { sc.listenerBus.post(event) } + def getConf[T](entry: ConfigEntry[T]): T = { + globalSparkContext.getConf.get(entry) + } + + def setConf[T](entry: ConfigEntry[T], value: T): Unit = { + globalSparkContext.conf.set(entry, value) + } } diff --git a/extensions/spark/kyuubi-spark-lineage/src/test/resources/atlas-application.properties b/extensions/spark/kyuubi-spark-lineage/src/test/resources/atlas-application.properties new file mode 100644 index 000000000..e6dc52f98 --- /dev/null +++ b/extensions/spark/kyuubi-spark-lineage/src/test/resources/atlas-application.properties @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +atlas.cluster.name=test diff --git a/extensions/spark/kyuubi-spark-lineage/src/test/resources/log4j2-test.xml b/extensions/spark/kyuubi-spark-lineage/src/test/resources/log4j2-test.xml index bfc40dd6d..3110216c1 100644 --- a/extensions/spark/kyuubi-spark-lineage/src/test/resources/log4j2-test.xml +++ b/extensions/spark/kyuubi-spark-lineage/src/test/resources/log4j2-test.xml @@ -21,14 +21,14 @@ - + - + diff --git a/extensions/spark/kyuubi-spark-lineage/src/test/scala/org/apache/kyuubi/plugin/lineage/dispatcher/atlas/AtlasLineageDispatcherSuite.scala b/extensions/spark/kyuubi-spark-lineage/src/test/scala/org/apache/kyuubi/plugin/lineage/dispatcher/atlas/AtlasLineageDispatcherSuite.scala new file mode 100644 index 000000000..8e8d18f21 --- /dev/null +++ b/extensions/spark/kyuubi-spark-lineage/src/test/scala/org/apache/kyuubi/plugin/lineage/dispatcher/atlas/AtlasLineageDispatcherSuite.scala @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.plugin.lineage.dispatcher.atlas + +import java.util + +import scala.collection.JavaConverters._ + +import org.apache.atlas.model.instance.{AtlasEntity, AtlasObjectId} +import org.apache.commons.lang3.StringUtils +import org.apache.spark.SparkConf +import org.apache.spark.kyuubi.lineage.LineageConf.{DEFAULT_CATALOG, DISPATCHERS, SKIP_PARSING_PERMANENT_VIEW_ENABLED} +import org.apache.spark.kyuubi.lineage.SparkContextHelper +import org.apache.spark.sql.SparkListenerExtensionTest +import org.scalatest.concurrent.PatienceConfiguration.Timeout +import org.scalatest.time.SpanSugar._ + +import org.apache.kyuubi.KyuubiFunSuite +import org.apache.kyuubi.plugin.lineage.Lineage +import org.apache.kyuubi.plugin.lineage.dispatcher.atlas.AtlasEntityHelper.{buildColumnQualifiedName, buildTableQualifiedName, COLUMN_LINEAGE_TYPE, PROCESS_TYPE} +import org.apache.kyuubi.plugin.lineage.helper.SparkListenerHelper.SPARK_RUNTIME_VERSION + +class AtlasLineageDispatcherSuite extends KyuubiFunSuite with SparkListenerExtensionTest { + val catalogName = + if (SPARK_RUNTIME_VERSION <= "3.1") "org.apache.spark.sql.connector.InMemoryTableCatalog" + else "org.apache.spark.sql.connector.catalog.InMemoryTableCatalog" + + override protected val catalogImpl: String = "hive" + + override def sparkConf(): SparkConf = { + super.sparkConf() + .set("spark.sql.catalog.v2_catalog", catalogName) + .set( + "spark.sql.queryExecutionListeners", + "org.apache.kyuubi.plugin.lineage.SparkOperationLineageQueryExecutionListener") + .set(DISPATCHERS.key, "ATLAS") + .set(SKIP_PARSING_PERMANENT_VIEW_ENABLED.key, "true") + } + + override def afterAll(): Unit = { + spark.stop() + super.afterAll() + } + + test("altas lineage capture: insert into select sql") { + val mockAtlasClient = new MockAtlasClient() + AtlasClient.setClient(mockAtlasClient) + + withTable("test_table0") { _ => + spark.sql("create table test_table0(a string, b int, c int)") + spark.sql("create table test_table1(a string, d int)") + spark.sql("insert into test_table1 select a, b + c as d from test_table0").collect() + val expected = Lineage( + List(s"$DEFAULT_CATALOG.default.test_table0"), + List(s"$DEFAULT_CATALOG.default.test_table1"), + List( + ( + s"$DEFAULT_CATALOG.default.test_table1.a", + Set(s"$DEFAULT_CATALOG.default.test_table0.a")), + ( + s"$DEFAULT_CATALOG.default.test_table1.d", + Set( + s"$DEFAULT_CATALOG.default.test_table0.b", + s"$DEFAULT_CATALOG.default.test_table0.c")))) + eventually(Timeout(5.seconds)) { + assert(mockAtlasClient.getEntities != null && mockAtlasClient.getEntities.nonEmpty) + } + checkAtlasProcessEntity(mockAtlasClient.getEntities.head, expected) + checkAtlasColumnLineageEntities( + mockAtlasClient.getEntities.head, + mockAtlasClient.getEntities.tail, + expected) + } + + } + + def checkAtlasProcessEntity(entity: AtlasEntity, expected: Lineage): Unit = { + assert(entity.getTypeName == PROCESS_TYPE) + + val appId = SparkContextHelper.globalSparkContext.applicationId + assert(entity.getAttribute("qualifiedName") == appId) + assert(entity.getAttribute("name") + == s"${SparkContextHelper.globalSparkContext.appName} $appId") + assert(StringUtils.isNotBlank(entity.getAttribute("currUser").asInstanceOf[String])) + assert(entity.getAttribute("executionId") != null) + assert(StringUtils.isNotBlank(entity.getAttribute("details").asInstanceOf[String])) + assert(StringUtils.isNotBlank(entity.getAttribute("sparkPlanDescription").asInstanceOf[String])) + + val inputs = entity.getRelationshipAttribute("inputs") + .asInstanceOf[util.Collection[AtlasObjectId]].asScala.map(getQualifiedName) + val outputs = entity.getRelationshipAttribute("outputs") + .asInstanceOf[util.Collection[AtlasObjectId]].asScala.map(getQualifiedName) + assertResult(expected.inputTables + .flatMap(buildTableQualifiedName(_).toSeq))(inputs) + assertResult(expected.outputTables + .flatMap(buildTableQualifiedName(_).toSeq))(outputs) + } + + def checkAtlasColumnLineageEntities( + processEntity: AtlasEntity, + entities: Seq[AtlasEntity], + expected: Lineage): Unit = { + assert(entities.size == expected.columnLineage.size) + + entities.zip(expected.columnLineage).foreach { + case (entity, expectedLineage) => + assert(entity.getTypeName == COLUMN_LINEAGE_TYPE) + val expectedQualifiedName = + s"${processEntity.getAttribute("qualifiedName")}:" + + s"${buildColumnQualifiedName(expectedLineage.column).get}" + assert(entity.getAttribute("qualifiedName") == expectedQualifiedName) + assert(entity.getAttribute("name") == expectedQualifiedName) + + val inputs = entity.getRelationshipAttribute("inputs") + .asInstanceOf[util.Collection[AtlasObjectId]].asScala.map(getQualifiedName) + assertResult(expectedLineage.originalColumns + .flatMap(buildColumnQualifiedName(_).toSet))(inputs.toSet) + + val outputs = entity.getRelationshipAttribute("outputs") + .asInstanceOf[util.Collection[AtlasObjectId]].asScala.map(getQualifiedName) + assert(outputs.size == 1) + assert(buildColumnQualifiedName(expectedLineage.column).toSeq.head == outputs.head) + + assert(getQualifiedName(entity.getRelationshipAttribute("process").asInstanceOf[ + AtlasObjectId]) == processEntity.getAttribute("qualifiedName")) + } + } + + // Pre-set cluster name for testing in `test/resources/atlas-application.properties` + private val cluster = "test" + + def getQualifiedName(objId: AtlasObjectId): String = { + objId.getUniqueAttributes.get("qualifiedName").asInstanceOf[String] + } + + class MockAtlasClient() extends AtlasClient { + private var _entities: Seq[AtlasEntity] = _ + + override def send(entities: Seq[AtlasEntity]): Unit = { + _entities = entities + } + + def getEntities: Seq[AtlasEntity] = _entities + + override def close(): Unit = {} + } +} diff --git a/extensions/spark/kyuubi-spark-lineage/src/test/scala/org/apache/kyuubi/plugin/lineage/events/OperationLineageEventSuite.scala b/extensions/spark/kyuubi-spark-lineage/src/test/scala/org/apache/kyuubi/plugin/lineage/events/OperationLineageEventSuite.scala index 6eeebbd3c..378eb3bb4 100644 --- a/extensions/spark/kyuubi-spark-lineage/src/test/scala/org/apache/kyuubi/plugin/lineage/events/OperationLineageEventSuite.scala +++ b/extensions/spark/kyuubi-spark-lineage/src/test/scala/org/apache/kyuubi/plugin/lineage/events/OperationLineageEventSuite.scala @@ -20,16 +20,20 @@ package org.apache.kyuubi.plugin.lineage.events import java.util.concurrent.{CountDownLatch, TimeUnit} import org.apache.spark.SparkConf +import org.apache.spark.kyuubi.lineage.LineageConf._ import org.apache.spark.scheduler.{SparkListener, SparkListenerEvent} import org.apache.spark.sql.SparkListenerExtensionTest import org.apache.kyuubi.KyuubiFunSuite -import org.apache.kyuubi.plugin.lineage.helper.SparkListenerHelper.isSparkVersionAtMost +import org.apache.kyuubi.events.EventBus +import org.apache.kyuubi.plugin.lineage.Lineage +import org.apache.kyuubi.plugin.lineage.dispatcher.{OperationLineageKyuubiEvent, OperationLineageSparkEvent} +import org.apache.kyuubi.plugin.lineage.helper.SparkListenerHelper.SPARK_RUNTIME_VERSION class OperationLineageEventSuite extends KyuubiFunSuite with SparkListenerExtensionTest { val catalogName = - if (isSparkVersionAtMost("3.1")) "org.apache.spark.sql.connector.InMemoryTableCatalog" + if (SPARK_RUNTIME_VERSION <= "3.1") "org.apache.spark.sql.connector.InMemoryTableCatalog" else "org.apache.spark.sql.connector.catalog.InMemoryTableCatalog" override protected val catalogImpl: String = "hive" @@ -40,18 +44,21 @@ class OperationLineageEventSuite extends KyuubiFunSuite with SparkListenerExtens .set( "spark.sql.queryExecutionListeners", "org.apache.kyuubi.plugin.lineage.SparkOperationLineageQueryExecutionListener") + .set(DISPATCHERS.key, "SPARK_EVENT,KYUUBI_EVENT") + .set(SKIP_PARSING_PERMANENT_VIEW_ENABLED.key, "true") } test("operation lineage event capture: for execute sql") { - val countDownLatch = new CountDownLatch(1) - var actual: Lineage = null + val countDownLatch = new CountDownLatch(2) + // get lineage from spark event + var actualSparkEventLineage: Lineage = null spark.sparkContext.addSparkListener(new SparkListener { override def onOtherEvent(event: SparkListenerEvent): Unit = { event match { - case lineageEvent: OperationLineageEvent => + case lineageEvent: OperationLineageSparkEvent => lineageEvent.lineage.foreach { case lineage if lineage.inputTables.nonEmpty => - actual = lineage + actualSparkEventLineage = lineage countDownLatch.countDown() } case _ => @@ -59,17 +66,28 @@ class OperationLineageEventSuite extends KyuubiFunSuite with SparkListenerExtens } }) + // get lineage from kyuubi event + var actualKyuubiEventLineage: Lineage = null + EventBus.register[OperationLineageKyuubiEvent] { lineageEvent: OperationLineageKyuubiEvent => + lineageEvent.lineage.foreach { + case lineage if lineage.inputTables.nonEmpty => + actualKyuubiEventLineage = lineage + countDownLatch.countDown() + } + } + withTable("test_table0") { _ => spark.sql("create table test_table0(a string, b string)") spark.sql("select a as col0, b as col1 from test_table0").collect() val expected = Lineage( - List("default.test_table0"), + List(s"$DEFAULT_CATALOG.default.test_table0"), List(), List( - ("col0", Set("default.test_table0.a")), - ("col1", Set("default.test_table0.b")))) + ("col0", Set(s"$DEFAULT_CATALOG.default.test_table0.a")), + ("col1", Set(s"$DEFAULT_CATALOG.default.test_table0.b")))) countDownLatch.await(20, TimeUnit.SECONDS) - assert(actual == expected) + assert(actualSparkEventLineage == expected) + assert(actualKyuubiEventLineage == expected) } } @@ -77,16 +95,17 @@ class OperationLineageEventSuite extends KyuubiFunSuite with SparkListenerExtens val countDownLatch = new CountDownLatch(1) var executionId: Long = -1 val expected = Lineage( - List("default.table1", "default.table0"), + List(s"$DEFAULT_CATALOG.default.table1", s"$DEFAULT_CATALOG.default.table0"), List(), List( - ("aa", Set("default.table1.a")), - ("bb", Set("default.table0.b")))) + ("aa", Set(s"$DEFAULT_CATALOG.default.table1.a")), + ("bb", Set(s"$DEFAULT_CATALOG.default.table0.b")))) spark.sparkContext.addSparkListener(new SparkListener { override def onOtherEvent(event: SparkListenerEvent): Unit = { event match { - case lineageEvent: OperationLineageEvent if executionId == lineageEvent.executionId => + case lineageEvent: OperationLineageSparkEvent + if executionId == lineageEvent.executionId => lineageEvent.lineage.foreach { lineage => assert(lineage == expected) countDownLatch.countDown() @@ -116,4 +135,40 @@ class OperationLineageEventSuite extends KyuubiFunSuite with SparkListenerExtens } } + test("test for skip parsing permanent view") { + val countDownLatch = new CountDownLatch(1) + var actual: Lineage = null + spark.sparkContext.addSparkListener(new SparkListener { + override def onOtherEvent(event: SparkListenerEvent): Unit = { + event match { + case lineageEvent: OperationLineageSparkEvent => + lineageEvent.lineage.foreach { + case lineage if lineage.inputTables.nonEmpty && lineage.outputTables.isEmpty => + actual = lineage + countDownLatch.countDown() + } + case _ => + } + } + }) + + withTable("t1") { _ => + spark.sql("CREATE TABLE t1 (a string, b string, c string) USING hive") + spark.sql("CREATE VIEW t2 as select * from t1") + spark.sql( + s"select a as k, b" + + s" from t2" + + s" where a in ('HELLO') and c = 'HELLO'").collect() + + val expected = Lineage( + List(s"$DEFAULT_CATALOG.default.t2"), + List(), + List( + ("k", Set(s"$DEFAULT_CATALOG.default.t2.a")), + ("b", Set(s"$DEFAULT_CATALOG.default.t2.b")))) + countDownLatch.await(20, TimeUnit.SECONDS) + assert(actual == expected) + } + } + } diff --git a/extensions/spark/kyuubi-spark-lineage/src/test/scala/org/apache/kyuubi/plugin/lineage/helper/SparkSQLLineageParserHelperSuite.scala b/extensions/spark/kyuubi-spark-lineage/src/test/scala/org/apache/kyuubi/plugin/lineage/helper/SparkSQLLineageParserHelperSuite.scala index 6652be9ea..3c19163db 100644 --- a/extensions/spark/kyuubi-spark-lineage/src/test/scala/org/apache/kyuubi/plugin/lineage/helper/SparkSQLLineageParserHelperSuite.scala +++ b/extensions/spark/kyuubi-spark-lineage/src/test/scala/org/apache/kyuubi/plugin/lineage/helper/SparkSQLLineageParserHelperSuite.scala @@ -17,10 +17,10 @@ package org.apache.kyuubi.plugin.lineage.helper -import scala.collection.immutable.List import scala.reflect.io.File import org.apache.spark.SparkConf +import org.apache.spark.kyuubi.lineage.{LineageConf, SparkContextHelper} import org.apache.spark.sql.{DataFrame, SparkListenerExtensionTest, SparkSession, SQLContext} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType} @@ -28,16 +28,17 @@ import org.apache.spark.sql.sources.{BaseRelation, InsertableRelation, SchemaRel import org.apache.spark.sql.types.{IntegerType, StringType, StructType} import org.apache.kyuubi.KyuubiFunSuite -import org.apache.kyuubi.plugin.lineage.events.Lineage -import org.apache.kyuubi.plugin.lineage.helper.SparkListenerHelper.isSparkVersionAtMost +import org.apache.kyuubi.plugin.lineage.Lineage +import org.apache.kyuubi.plugin.lineage.helper.SparkListenerHelper.SPARK_RUNTIME_VERSION class SparkSQLLineageParserHelperSuite extends KyuubiFunSuite with SparkListenerExtensionTest { val catalogName = - if (isSparkVersionAtMost("3.1")) "org.apache.spark.sql.connector.InMemoryTableCatalog" + if (SPARK_RUNTIME_VERSION <= "3.1") "org.apache.spark.sql.connector.InMemoryTableCatalog" else "org.apache.spark.sql.connector.catalog.InMemoryTableCatalog" + val DEFAULT_CATALOG = LineageConf.DEFAULT_CATALOG override protected val catalogImpl: String = "hive" override def sparkConf(): SparkConf = { @@ -74,22 +75,28 @@ class SparkSQLLineageParserHelperSuite extends KyuubiFunSuite withView("alterviewascommand", "alterviewascommand1") { _ => spark.sql("create view alterviewascommand as select key from test_db0.test_table0") val ret0 = - exectractLineage("alter view alterviewascommand as select key from test_db0.test_table0") + extractLineage("alter view alterviewascommand as select key from test_db0.test_table0") assert(ret0 == Lineage( - List("test_db0.test_table0"), - List("default.alterviewascommand"), - List(("default.alterviewascommand.key", Set("test_db0.test_table0.key"))))) + List(s"$DEFAULT_CATALOG.test_db0.test_table0"), + List(s"$DEFAULT_CATALOG.default.alterviewascommand"), + List(( + s"$DEFAULT_CATALOG.default.alterviewascommand.key", + Set(s"$DEFAULT_CATALOG.test_db0.test_table0.key"))))) spark.sql("create view alterviewascommand1 as select * from test_db0.test_table0") val ret1 = - exectractLineage("alter view alterviewascommand1 as select * from test_db0.test_table0") + extractLineage("alter view alterviewascommand1 as select * from test_db0.test_table0") assert(ret1 == Lineage( - List("test_db0.test_table0"), - List("default.alterviewascommand1"), + List(s"$DEFAULT_CATALOG.test_db0.test_table0"), + List(s"$DEFAULT_CATALOG.default.alterviewascommand1"), List( - ("default.alterviewascommand1.key", Set("test_db0.test_table0.key")), - ("default.alterviewascommand1.value", Set("test_db0.test_table0.value"))))) + ( + s"$DEFAULT_CATALOG.default.alterviewascommand1.key", + Set(s"$DEFAULT_CATALOG.test_db0.test_table0.key")), + ( + s"$DEFAULT_CATALOG.default.alterviewascommand1.value", + Set(s"$DEFAULT_CATALOG.test_db0.test_table0.value"))))) } } @@ -101,16 +108,16 @@ class SparkSQLLineageParserHelperSuite extends KyuubiFunSuite ddls.split("\n").filter(_.nonEmpty).foreach(spark.sql(_).collect()) withView("test_view") { _ => - val result = exectractLineage( + val result = extractLineage( "create view test_view(a, b, c) as" + " select col1 as a, col2 as b, col3 as c from v2_catalog.db.tbb") assert(result == Lineage( List("v2_catalog.db.tbb"), - List("default.test_view"), + List(s"$DEFAULT_CATALOG.default.test_view"), List( - ("default.test_view.a", Set("v2_catalog.db.tbb.col1")), - ("default.test_view.b", Set("v2_catalog.db.tbb.col2")), - ("default.test_view.c", Set("v2_catalog.db.tbb.col3"))))) + (s"$DEFAULT_CATALOG.default.test_view.a", Set("v2_catalog.db.tbb.col1")), + (s"$DEFAULT_CATALOG.default.test_view.b", Set("v2_catalog.db.tbb.col2")), + (s"$DEFAULT_CATALOG.default.test_view.c", Set("v2_catalog.db.tbb.col3"))))) } } @@ -122,36 +129,36 @@ class SparkSQLLineageParserHelperSuite extends KyuubiFunSuite ddls.split("\n").filter(_.nonEmpty).foreach(spark.sql(_).collect()) withTable("v2_catalog.db.tb0") { _ => val ret0 = - exectractLineage( + extractLineage( s"insert into table v2_catalog.db.tb0 " + s"select key as col1, value as col2 from test_db0.test_table0") assert(ret0 == Lineage( - List("test_db0.test_table0"), + List(s"$DEFAULT_CATALOG.test_db0.test_table0"), List("v2_catalog.db.tb0"), List( - ("v2_catalog.db.tb0.col1", Set("test_db0.test_table0.key")), - ("v2_catalog.db.tb0.col2", Set("test_db0.test_table0.value"))))) + ("v2_catalog.db.tb0.col1", Set(s"$DEFAULT_CATALOG.test_db0.test_table0.key")), + ("v2_catalog.db.tb0.col2", Set(s"$DEFAULT_CATALOG.test_db0.test_table0.value"))))) val ret1 = - exectractLineage( + extractLineage( s"insert overwrite table v2_catalog.db.tb0 partition(col2) " + s"select key as col1, value as col2 from test_db0.test_table0") assert(ret1 == Lineage( - List("test_db0.test_table0"), + List(s"$DEFAULT_CATALOG.test_db0.test_table0"), List("v2_catalog.db.tb0"), List( - ("v2_catalog.db.tb0.col1", Set("test_db0.test_table0.key")), - ("v2_catalog.db.tb0.col2", Set("test_db0.test_table0.value"))))) + ("v2_catalog.db.tb0.col1", Set(s"$DEFAULT_CATALOG.test_db0.test_table0.key")), + ("v2_catalog.db.tb0.col2", Set(s"$DEFAULT_CATALOG.test_db0.test_table0.value"))))) val ret2 = - exectractLineage( + extractLineage( s"insert overwrite table v2_catalog.db.tb0 partition(col2 = 'bb') " + s"select key as col1 from test_db0.test_table0") assert(ret2 == Lineage( - List("test_db0.test_table0"), + List(s"$DEFAULT_CATALOG.test_db0.test_table0"), List("v2_catalog.db.tb0"), List( - ("v2_catalog.db.tb0.col1", Set("test_db0.test_table0.key")), + ("v2_catalog.db.tb0.col1", Set(s"$DEFAULT_CATALOG.test_db0.test_table0.key")), ("v2_catalog.db.tb0.col2", Set())))) } } @@ -165,13 +172,13 @@ class SparkSQLLineageParserHelperSuite extends KyuubiFunSuite |""".stripMargin ddls.split("\n").filter(_.nonEmpty).foreach(spark.sql(_).collect()) withTable("v2_catalog.db.target_t", "v2_catalog.db.source_t") { _ => - val ret0 = exectractLineageWithoutExecuting("MERGE INTO v2_catalog.db.target_t AS target " + + val ret0 = extractLineageWithoutExecuting("MERGE INTO v2_catalog.db.target_t AS target " + "USING v2_catalog.db.source_t AS source " + "ON target.id = source.id " + "WHEN MATCHED THEN " + " UPDATE SET target.name = source.name, target.price = source.price " + "WHEN NOT MATCHED THEN " + - " INSERT (id, name, price) VALUES (source.id, source.name, source.price)") + " INSERT (id, name, price) VALUES (cast(source.id as int), source.name, source.price)") assert(ret0 == Lineage( List("v2_catalog.db.source_t"), List("v2_catalog.db.target_t"), @@ -180,7 +187,7 @@ class SparkSQLLineageParserHelperSuite extends KyuubiFunSuite ("v2_catalog.db.target_t.name", Set("v2_catalog.db.source_t.name")), ("v2_catalog.db.target_t.price", Set("v2_catalog.db.source_t.price"))))) - val ret1 = exectractLineageWithoutExecuting("MERGE INTO v2_catalog.db.target_t AS target " + + val ret1 = extractLineageWithoutExecuting("MERGE INTO v2_catalog.db.target_t AS target " + "USING v2_catalog.db.source_t AS source " + "ON target.id = source.id " + "WHEN MATCHED THEN " + @@ -195,7 +202,7 @@ class SparkSQLLineageParserHelperSuite extends KyuubiFunSuite ("v2_catalog.db.target_t.name", Set("v2_catalog.db.source_t.name")), ("v2_catalog.db.target_t.price", Set("v2_catalog.db.source_t.price"))))) - val ret2 = exectractLineageWithoutExecuting("MERGE INTO v2_catalog.db.target_t AS target " + + val ret2 = extractLineageWithoutExecuting("MERGE INTO v2_catalog.db.target_t AS target " + "USING (select a.id, a.name, b.price " + "from v2_catalog.db.source_t a join v2_catalog.db.pivot_t b) AS source " + "ON target.id = source.id " + @@ -217,32 +224,44 @@ class SparkSQLLineageParserHelperSuite extends KyuubiFunSuite test("columns lineage extract - CreateViewCommand") { withView("createviewcommand", "createviewcommand1", "createviewcommand2") { _ => - val ret0 = exectractLineage( + val ret0 = extractLineage( "create view createviewcommand(a, b) as select key, value from test_db0.test_table0") assert(ret0 == Lineage( - List("test_db0.test_table0"), - List("default.createviewcommand"), + List(s"$DEFAULT_CATALOG.test_db0.test_table0"), + List(s"$DEFAULT_CATALOG.default.createviewcommand"), List( - ("default.createviewcommand.a", Set("test_db0.test_table0.key")), - ("default.createviewcommand.b", Set("test_db0.test_table0.value"))))) + ( + s"$DEFAULT_CATALOG.default.createviewcommand.a", + Set(s"$DEFAULT_CATALOG.test_db0.test_table0.key")), + ( + s"$DEFAULT_CATALOG.default.createviewcommand.b", + Set(s"$DEFAULT_CATALOG.test_db0.test_table0.value"))))) - val ret1 = exectractLineage( + val ret1 = extractLineage( "create view createviewcommand1 as select key, value from test_db0.test_table0") assert(ret1 == Lineage( - List("test_db0.test_table0"), - List("default.createviewcommand1"), + List(s"$DEFAULT_CATALOG.test_db0.test_table0"), + List(s"$DEFAULT_CATALOG.default.createviewcommand1"), List( - ("default.createviewcommand1.key", Set("test_db0.test_table0.key")), - ("default.createviewcommand1.value", Set("test_db0.test_table0.value"))))) + ( + s"$DEFAULT_CATALOG.default.createviewcommand1.key", + Set(s"$DEFAULT_CATALOG.test_db0.test_table0.key")), + ( + s"$DEFAULT_CATALOG.default.createviewcommand1.value", + Set(s"$DEFAULT_CATALOG.test_db0.test_table0.value"))))) - val ret2 = exectractLineage( + val ret2 = extractLineage( "create view createviewcommand2 as select * from test_db0.test_table0") assert(ret2 == Lineage( - List("test_db0.test_table0"), - List("default.createviewcommand2"), + List(s"$DEFAULT_CATALOG.test_db0.test_table0"), + List(s"$DEFAULT_CATALOG.default.createviewcommand2"), List( - ("default.createviewcommand2.key", Set("test_db0.test_table0.key")), - ("default.createviewcommand2.value", Set("test_db0.test_table0.value"))))) + ( + s"$DEFAULT_CATALOG.default.createviewcommand2.key", + Set(s"$DEFAULT_CATALOG.test_db0.test_table0.key")), + ( + s"$DEFAULT_CATALOG.default.createviewcommand2.value", + Set(s"$DEFAULT_CATALOG.test_db0.test_table0.value"))))) } } @@ -250,67 +269,81 @@ class SparkSQLLineageParserHelperSuite extends KyuubiFunSuite withTable("createdatasourcetableasselectcommand", "createdatasourcetableasselectcommand1") { _ => val ret0 = - exectractLineage("create table createdatasourcetableasselectcommand using parquet" + + extractLineage("create table createdatasourcetableasselectcommand using parquet" + " AS SELECT key, value FROM test_db0.test_table0") assert(ret0 == Lineage( - List("test_db0.test_table0"), - List("default.createdatasourcetableasselectcommand"), + List(s"$DEFAULT_CATALOG.test_db0.test_table0"), + List(s"$DEFAULT_CATALOG.default.createdatasourcetableasselectcommand"), List( - ("default.createdatasourcetableasselectcommand.key", Set("test_db0.test_table0.key")), ( - "default.createdatasourcetableasselectcommand.value", - Set("test_db0.test_table0.value"))))) + s"$DEFAULT_CATALOG.default.createdatasourcetableasselectcommand.key", + Set(s"$DEFAULT_CATALOG.test_db0.test_table0.key")), + ( + s"$DEFAULT_CATALOG.default.createdatasourcetableasselectcommand.value", + Set(s"$DEFAULT_CATALOG.test_db0.test_table0.value"))))) val ret1 = - exectractLineage("create table createdatasourcetableasselectcommand1 using parquet" + + extractLineage("create table createdatasourcetableasselectcommand1 using parquet" + " AS SELECT * FROM test_db0.test_table0") assert(ret1 == Lineage( - List("test_db0.test_table0"), - List("default.createdatasourcetableasselectcommand1"), + List(s"$DEFAULT_CATALOG.test_db0.test_table0"), + List(s"$DEFAULT_CATALOG.default.createdatasourcetableasselectcommand1"), List( - ("default.createdatasourcetableasselectcommand1.key", Set("test_db0.test_table0.key")), ( - "default.createdatasourcetableasselectcommand1.value", - Set("test_db0.test_table0.value"))))) + s"$DEFAULT_CATALOG.default.createdatasourcetableasselectcommand1.key", + Set(s"$DEFAULT_CATALOG.test_db0.test_table0.key")), + ( + s"$DEFAULT_CATALOG.default.createdatasourcetableasselectcommand1.value", + Set(s"$DEFAULT_CATALOG.test_db0.test_table0.value"))))) } } test("columns lineage extract - CreateHiveTableAsSelectCommand") { withTable("createhivetableasselectcommand", "createhivetableasselectcommand1") { _ => - val ret0 = exectractLineage("create table createhivetableasselectcommand using hive" + + val ret0 = extractLineage("create table createhivetableasselectcommand using hive" + " as select key, value from test_db0.test_table0") assert(ret0 == Lineage( - List("test_db0.test_table0"), - List("default.createhivetableasselectcommand"), + List(s"$DEFAULT_CATALOG.test_db0.test_table0"), + List(s"$DEFAULT_CATALOG.default.createhivetableasselectcommand"), List( - ("default.createhivetableasselectcommand.key", Set("test_db0.test_table0.key")), - ("default.createhivetableasselectcommand.value", Set("test_db0.test_table0.value"))))) + ( + s"$DEFAULT_CATALOG.default.createhivetableasselectcommand.key", + Set(s"$DEFAULT_CATALOG.test_db0.test_table0.key")), + ( + s"$DEFAULT_CATALOG.default.createhivetableasselectcommand.value", + Set(s"$DEFAULT_CATALOG.test_db0.test_table0.value"))))) - val ret1 = exectractLineage("create table createhivetableasselectcommand1 using hive" + + val ret1 = extractLineage("create table createhivetableasselectcommand1 using hive" + " as select * from test_db0.test_table0") assert(ret1 == Lineage( - List("test_db0.test_table0"), - List("default.createhivetableasselectcommand1"), + List(s"$DEFAULT_CATALOG.test_db0.test_table0"), + List(s"$DEFAULT_CATALOG.default.createhivetableasselectcommand1"), List( - ("default.createhivetableasselectcommand1.key", Set("test_db0.test_table0.key")), - ("default.createhivetableasselectcommand1.value", Set("test_db0.test_table0.value"))))) + ( + s"$DEFAULT_CATALOG.default.createhivetableasselectcommand1.key", + Set(s"$DEFAULT_CATALOG.test_db0.test_table0.key")), + ( + s"$DEFAULT_CATALOG.default.createhivetableasselectcommand1.value", + Set(s"$DEFAULT_CATALOG.test_db0.test_table0.value"))))) } } test("columns lineage extract - OptimizedCreateHiveTableAsSelectCommand") { withTable("optimizedcreatehivetableasselectcommand") { _ => val ret = - exectractLineage( + extractLineage( "create table optimizedcreatehivetableasselectcommand stored as parquet " + "as select * from test_db0.test_table0") assert(ret == Lineage( - List("test_db0.test_table0"), - List("default.optimizedcreatehivetableasselectcommand"), + List(s"$DEFAULT_CATALOG.test_db0.test_table0"), + List(s"$DEFAULT_CATALOG.default.optimizedcreatehivetableasselectcommand"), List( - ("default.optimizedcreatehivetableasselectcommand.key", Set("test_db0.test_table0.key")), ( - "default.optimizedcreatehivetableasselectcommand.value", - Set("test_db0.test_table0.value"))))) + s"$DEFAULT_CATALOG.default.optimizedcreatehivetableasselectcommand.key", + Set(s"$DEFAULT_CATALOG.test_db0.test_table0.key")), + ( + s"$DEFAULT_CATALOG.default.optimizedcreatehivetableasselectcommand.value", + Set(s"$DEFAULT_CATALOG.test_db0.test_table0.value"))))) } } @@ -318,27 +351,31 @@ class SparkSQLLineageParserHelperSuite extends KyuubiFunSuite withTable( "v2_catalog.db.createhivetableasselectcommand", "v2_catalog.db.createhivetableasselectcommand1") { _ => - val ret0 = exectractLineage("create table v2_catalog.db.createhivetableasselectcommand" + + val ret0 = extractLineage("create table v2_catalog.db.createhivetableasselectcommand" + " as select key, value from test_db0.test_table0") assert(ret0 == Lineage( - List("test_db0.test_table0"), + List(s"$DEFAULT_CATALOG.test_db0.test_table0"), List("v2_catalog.db.createhivetableasselectcommand"), List( - ("v2_catalog.db.createhivetableasselectcommand.key", Set("test_db0.test_table0.key")), + ( + "v2_catalog.db.createhivetableasselectcommand.key", + Set(s"$DEFAULT_CATALOG.test_db0.test_table0.key")), ( "v2_catalog.db.createhivetableasselectcommand.value", - Set("test_db0.test_table0.value"))))) + Set(s"$DEFAULT_CATALOG.test_db0.test_table0.value"))))) - val ret1 = exectractLineage("create table v2_catalog.db.createhivetableasselectcommand1" + + val ret1 = extractLineage("create table v2_catalog.db.createhivetableasselectcommand1" + " as select * from test_db0.test_table0") assert(ret1 == Lineage( - List("test_db0.test_table0"), + List(s"$DEFAULT_CATALOG.test_db0.test_table0"), List("v2_catalog.db.createhivetableasselectcommand1"), List( - ("v2_catalog.db.createhivetableasselectcommand1.key", Set("test_db0.test_table0.key")), + ( + "v2_catalog.db.createhivetableasselectcommand1.key", + Set(s"$DEFAULT_CATALOG.test_db0.test_table0.key")), ( "v2_catalog.db.createhivetableasselectcommand1.value", - Set("test_db0.test_table0.value"))))) + Set(s"$DEFAULT_CATALOG.test_db0.test_table0.value"))))) } } @@ -363,36 +400,48 @@ class SparkSQLLineageParserHelperSuite extends KyuubiFunSuite spark.sessionState.catalog.createTable(newTable, ignoreIfExists = false) val ret0 = - exectractLineage( + extractLineage( s"insert into table $tableName select key, value from test_db0.test_table0") assert(ret0 == Lineage( - List("test_db0.test_table0"), - List("default.insertintodatasourcecommand"), + List(s"$DEFAULT_CATALOG.test_db0.test_table0"), + List(s"$DEFAULT_CATALOG.default.insertintodatasourcecommand"), List( - ("default.insertintodatasourcecommand.a", Set("test_db0.test_table0.key")), - ("default.insertintodatasourcecommand.b", Set("test_db0.test_table0.value"))))) + ( + s"$DEFAULT_CATALOG.default.insertintodatasourcecommand.a", + Set(s"$DEFAULT_CATALOG.test_db0.test_table0.key")), + ( + s"$DEFAULT_CATALOG.default.insertintodatasourcecommand.b", + Set(s"$DEFAULT_CATALOG.test_db0.test_table0.value"))))) val ret1 = - exectractLineage( + extractLineage( s"insert into table $tableName select * from test_db0.test_table0") assert(ret1 == Lineage( - List("test_db0.test_table0"), - List("default.insertintodatasourcecommand"), + List(s"$DEFAULT_CATALOG.test_db0.test_table0"), + List(s"$DEFAULT_CATALOG.default.insertintodatasourcecommand"), List( - ("default.insertintodatasourcecommand.a", Set("test_db0.test_table0.key")), - ("default.insertintodatasourcecommand.b", Set("test_db0.test_table0.value"))))) + ( + s"$DEFAULT_CATALOG.default.insertintodatasourcecommand.a", + Set(s"$DEFAULT_CATALOG.test_db0.test_table0.key")), + ( + s"$DEFAULT_CATALOG.default.insertintodatasourcecommand.b", + Set(s"$DEFAULT_CATALOG.test_db0.test_table0.value"))))) val ret2 = - exectractLineage( + extractLineage( s"insert into table $tableName " + s"select (select key from test_db0.test_table1 limit 1) + 1 as aa, " + s"value as bb from test_db0.test_table0") assert(ret2 == Lineage( - List("test_db0.test_table1", "test_db0.test_table0"), - List("default.insertintodatasourcecommand"), + List(s"$DEFAULT_CATALOG.test_db0.test_table1", s"$DEFAULT_CATALOG.test_db0.test_table0"), + List(s"$DEFAULT_CATALOG.default.insertintodatasourcecommand"), List( - ("default.insertintodatasourcecommand.a", Set("test_db0.test_table1.key")), - ("default.insertintodatasourcecommand.b", Set("test_db0.test_table0.value"))))) + ( + s"$DEFAULT_CATALOG.default.insertintodatasourcecommand.a", + Set(s"$DEFAULT_CATALOG.test_db0.test_table1.key")), + ( + s"$DEFAULT_CATALOG.default.insertintodatasourcecommand.b", + Set(s"$DEFAULT_CATALOG.test_db0.test_table0.value"))))) } } @@ -402,15 +451,19 @@ class SparkSQLLineageParserHelperSuite extends KyuubiFunSuite withTable(tableName) { _ => spark.sql(s"CREATE TABLE $tableName (a int, b string) USING parquet") val ret0 = - exectractLineage( + extractLineage( s"insert into table $tableName select key, value from test_db0.test_table0") assert(ret0 == Lineage( - List("test_db0.test_table0"), - List("default.insertintohadoopfsrelationcommand"), + List(s"$DEFAULT_CATALOG.test_db0.test_table0"), + List(s"$DEFAULT_CATALOG.default.insertintohadoopfsrelationcommand"), List( - ("default.insertintohadoopfsrelationcommand.a", Set("test_db0.test_table0.key")), - ("default.insertintohadoopfsrelationcommand.b", Set("test_db0.test_table0.value"))))) + ( + s"$DEFAULT_CATALOG.default.insertintohadoopfsrelationcommand.a", + Set(s"$DEFAULT_CATALOG.test_db0.test_table0.key")), + ( + s"$DEFAULT_CATALOG.default.insertintohadoopfsrelationcommand.b", + Set(s"$DEFAULT_CATALOG.test_db0.test_table0.value"))))) } } @@ -418,33 +471,33 @@ class SparkSQLLineageParserHelperSuite extends KyuubiFunSuite test("columns lineage extract - InsertIntoDatasourceDirCommand") { val tableDirectory = getClass.getResource("/").getPath + "table_directory" val directory = File(tableDirectory).createDirectory() - val ret0 = exectractLineage(s""" - |INSERT OVERWRITE DIRECTORY '$directory.path' - |USING parquet - |SELECT * FROM test_db0.test_table_part0""".stripMargin) + val ret0 = extractLineage(s""" + |INSERT OVERWRITE DIRECTORY '$directory.path' + |USING parquet + |SELECT * FROM test_db0.test_table_part0""".stripMargin) assert(ret0 == Lineage( - List("test_db0.test_table_part0"), + List(s"$DEFAULT_CATALOG.test_db0.test_table_part0"), List(s"""`$directory.path`"""), List( - (s"""`$directory.path`.key""", Set("test_db0.test_table_part0.key")), - (s"""`$directory.path`.value""", Set("test_db0.test_table_part0.value")), - (s"""`$directory.path`.pid""", Set("test_db0.test_table_part0.pid"))))) + (s"""`$directory.path`.key""", Set(s"$DEFAULT_CATALOG.test_db0.test_table_part0.key")), + (s"""`$directory.path`.value""", Set(s"$DEFAULT_CATALOG.test_db0.test_table_part0.value")), + (s"""`$directory.path`.pid""", Set(s"$DEFAULT_CATALOG.test_db0.test_table_part0.pid"))))) } test("columns lineage extract - InsertIntoHiveDirCommand") { val tableDirectory = getClass.getResource("/").getPath + "table_directory" val directory = File(tableDirectory).createDirectory() - val ret0 = exectractLineage(s""" - |INSERT OVERWRITE DIRECTORY '$directory.path' - |USING parquet - |SELECT * FROM test_db0.test_table_part0""".stripMargin) + val ret0 = extractLineage(s""" + |INSERT OVERWRITE DIRECTORY '$directory.path' + |USING parquet + |SELECT * FROM test_db0.test_table_part0""".stripMargin) assert(ret0 == Lineage( - List("test_db0.test_table_part0"), + List(s"$DEFAULT_CATALOG.test_db0.test_table_part0"), List(s"""`$directory.path`"""), List( - (s"""`$directory.path`.key""", Set("test_db0.test_table_part0.key")), - (s"""`$directory.path`.value""", Set("test_db0.test_table_part0.value")), - (s"""`$directory.path`.pid""", Set("test_db0.test_table_part0.pid"))))) + (s"""`$directory.path`.key""", Set(s"$DEFAULT_CATALOG.test_db0.test_table_part0.key")), + (s"""`$directory.path`.value""", Set(s"$DEFAULT_CATALOG.test_db0.test_table_part0.value")), + (s"""`$directory.path`.pid""", Set(s"$DEFAULT_CATALOG.test_db0.test_table_part0.pid"))))) } test("columns lineage extract - InsertIntoHiveTable") { @@ -452,41 +505,45 @@ class SparkSQLLineageParserHelperSuite extends KyuubiFunSuite withTable(tableName) { _ => spark.sql(s"CREATE TABLE $tableName (a int, b string) USING hive") val ret0 = - exectractLineage( + extractLineage( s"insert into table $tableName select * from test_db0.test_table0") assert(ret0 == Lineage( - List("test_db0.test_table0"), - List(s"default.$tableName"), + List(s"$DEFAULT_CATALOG.test_db0.test_table0"), + List(s"$DEFAULT_CATALOG.default.$tableName"), List( - (s"default.$tableName.a", Set("test_db0.test_table0.key")), - (s"default.$tableName.b", Set("test_db0.test_table0.value"))))) + ( + s"$DEFAULT_CATALOG.default.$tableName.a", + Set(s"$DEFAULT_CATALOG.test_db0.test_table0.key")), + ( + s"$DEFAULT_CATALOG.default.$tableName.b", + Set(s"$DEFAULT_CATALOG.test_db0.test_table0.value"))))) } } test("columns lineage extract - logical relation sql") { - val ret0 = exectractLineage("select key, value from test_db0.test_table0") + val ret0 = extractLineage("select key, value from test_db0.test_table0") assert(ret0 == Lineage( - List("test_db0.test_table0"), + List(s"$DEFAULT_CATALOG.test_db0.test_table0"), List(), List( - ("key", Set("test_db0.test_table0.key")), - ("value", Set("test_db0.test_table0.value"))))) + ("key", Set(s"$DEFAULT_CATALOG.test_db0.test_table0.key")), + ("value", Set(s"$DEFAULT_CATALOG.test_db0.test_table0.value"))))) - val ret1 = exectractLineage("select * from test_db0.test_table_part0") + val ret1 = extractLineage("select * from test_db0.test_table_part0") assert(ret1 == Lineage( - List("test_db0.test_table_part0"), + List(s"$DEFAULT_CATALOG.test_db0.test_table_part0"), List(), List( - ("key", Set("test_db0.test_table_part0.key")), - ("value", Set("test_db0.test_table_part0.value")), - ("pid", Set("test_db0.test_table_part0.pid"))))) + ("key", Set(s"$DEFAULT_CATALOG.test_db0.test_table_part0.key")), + ("value", Set(s"$DEFAULT_CATALOG.test_db0.test_table_part0.value")), + ("pid", Set(s"$DEFAULT_CATALOG.test_db0.test_table_part0.pid"))))) } test("columns lineage extract - not generate lineage sql") { - val ret0 = exectractLineage("create table test_table1(a string, b string, c string)") + val ret0 = extractLineage("create table test_table1(a string, b string, c string)") assert(ret0 == Lineage(List[String](), List[String](), List[(String, Set[String])]())) } @@ -499,14 +556,14 @@ class SparkSQLLineageParserHelperSuite extends KyuubiFunSuite ddls.split("\n").filter(_.nonEmpty).foreach(spark.sql(_).collect()) withTable("v2_catalog.db.tb") { _ => val sql0 = "select col1 from v2_catalog.db.tb" - val ret0 = exectractLineage(sql0) + val ret0 = extractLineage(sql0) assert(ret0 == Lineage( List("v2_catalog.db.tb"), List(), List("col1" -> Set("v2_catalog.db.tb.col1")))) val sql1 = "select col1, hash(hash(col1)) as col2 from v2_catalog.db.tb" - val ret1 = exectractLineage(sql1) + val ret1 = extractLineage(sql1) assert(ret1 == Lineage( List("v2_catalog.db.tb"), List(), @@ -514,7 +571,7 @@ class SparkSQLLineageParserHelperSuite extends KyuubiFunSuite val sql2 = "select col1, case col1 when '1' then 's1' else col1 end col2 from v2_catalog.db.tb" - val ret2 = exectractLineage(sql2) + val ret2 = extractLineage(sql2) assert(ret2 == Lineage( List("v2_catalog.db.tb"), List(), @@ -523,7 +580,7 @@ class SparkSQLLineageParserHelperSuite extends KyuubiFunSuite val sql3 = "select col1 as col2, 'col2' as col2, 'col2', first(col3) as col2 " + "from v2_catalog.db.tb group by col1" - val ret3 = exectractLineage(sql3) + val ret3 = extractLineage(sql3) assert(ret3 == Lineage( List("v2_catalog.db.tb"), List[String](), @@ -536,7 +593,7 @@ class SparkSQLLineageParserHelperSuite extends KyuubiFunSuite val sql4 = "select col1 as col2, sum(hash(col1) + hash(hash(col1))) " + "from v2_catalog.db.tb group by col1" - val ret4 = exectractLineage(sql4) + val ret4 = extractLineage(sql4) assert(ret4 == Lineage( List("v2_catalog.db.tb"), List(), @@ -552,7 +609,7 @@ class SparkSQLLineageParserHelperSuite extends KyuubiFunSuite | on t1.col2 = t2.col3 | group by 1 |""".stripMargin - val ret5 = exectractLineage(sql5) + val ret5 = extractLineage(sql5) assert(ret5 == Lineage( List("v2_catalog.db.tb"), List(), @@ -578,26 +635,26 @@ class SparkSQLLineageParserHelperSuite extends KyuubiFunSuite |select tmp0_0 as a0, tmp1_0 as a1 from tmp0 join tmp1 where tmp1_0 = tmp0_0 |""".stripMargin val sql0ExpectResult = Lineage( - List("default.tmp0", "default.tmp1"), + List(s"$DEFAULT_CATALOG.default.tmp0", s"$DEFAULT_CATALOG.default.tmp1"), List(), List( - "a0" -> Set("default.tmp0.tmp0_0"), - "a1" -> Set("default.tmp1.tmp1_0"))) + "a0" -> Set(s"$DEFAULT_CATALOG.default.tmp0.tmp0_0"), + "a1" -> Set(s"$DEFAULT_CATALOG.default.tmp1.tmp1_0"))) val sql1 = """ |select count(tmp1_0) as cnt, tmp1_1 from tmp1 group by tmp1_1 |""".stripMargin val sql1ExpectResult = Lineage( - List("default.tmp1"), + List(s"$DEFAULT_CATALOG.default.tmp1"), List(), List( - "cnt" -> Set("default.tmp1.tmp1_0"), - "tmp1_1" -> Set("default.tmp1.tmp1_1"))) + "cnt" -> Set(s"$DEFAULT_CATALOG.default.tmp1.tmp1_0"), + "tmp1_1" -> Set(s"$DEFAULT_CATALOG.default.tmp1.tmp1_1"))) - val ret0 = exectractLineage(sql0) + val ret0 = extractLineage(sql0) assert(ret0 == sql0ExpectResult) - val ret1 = exectractLineage(sql1) + val ret1 = extractLineage(sql1) assert(ret1 == sql1ExpectResult) } } @@ -657,17 +714,17 @@ class SparkSQLLineageParserHelperSuite extends KyuubiFunSuite |FROM goods_cat_new |LIMIT 10""".stripMargin - val ret0 = exectractLineage(sql0) + val ret0 = extractLineage(sql0) assert(ret0 == Lineage( List( - "test_db.goods_detail0", + s"$DEFAULT_CATALOG.test_db.goods_detail0", "v2_catalog.test_db_v2.goods_detail1", "v2_catalog.test_db_v2.mall_icon_schedule", "v2_catalog.test_db_v2.mall_icon"), List(), List( - ("goods_id", Set("test_db.goods_detail0.goods_id")), - ("cate_grory", Set("test_db.goods_detail0.cat_id")), + ("goods_id", Set(s"$DEFAULT_CATALOG.test_db.goods_detail0.goods_id")), + ("cate_grory", Set(s"$DEFAULT_CATALOG.test_db.goods_detail0.cat_id")), ("cat_id", Set("v2_catalog.test_db_v2.goods_detail1.cat_id")), ("product_id", Set("v2_catalog.test_db_v2.goods_detail1.product_id")), ("start_time", Set("v2_catalog.test_db_v2.mall_icon_schedule.start_time")), @@ -691,7 +748,7 @@ class SparkSQLLineageParserHelperSuite extends KyuubiFunSuite |on t1.col1 = t2.col1 |""".stripMargin - val ret0 = exectractLineage(sql0) + val ret0 = extractLineage(sql0) assert( ret0 == Lineage( List("v2_catalog.db.tb1", "v2_catalog.db.tb2"), @@ -726,14 +783,26 @@ class SparkSQLLineageParserHelperSuite extends KyuubiFunSuite |select a, b, c as c from test_db.test_table1 |) a |""".stripMargin - val ret0 = exectractLineage(sql0) + val ret0 = extractLineage(sql0) assert(ret0 == Lineage( - List("test_db.test_table0", "test_db.test_table1"), + List(s"$DEFAULT_CATALOG.test_db.test_table0", s"$DEFAULT_CATALOG.test_db.test_table1"), List(), List( - ("a", Set("test_db.test_table0.a", "test_db.test_table1.a")), - ("b", Set("test_db.test_table0.b", "test_db.test_table1.b")), - ("c", Set("test_db.test_table0.b", "test_db.test_table1.c"))))) + ( + "a", + Set( + s"$DEFAULT_CATALOG.test_db.test_table0.a", + s"$DEFAULT_CATALOG.test_db.test_table1.a")), + ( + "b", + Set( + s"$DEFAULT_CATALOG.test_db.test_table0.b", + s"$DEFAULT_CATALOG.test_db.test_table1.b")), + ( + "c", + Set( + s"$DEFAULT_CATALOG.test_db.test_table0.b", + s"$DEFAULT_CATALOG.test_db.test_table1.c"))))) } } @@ -767,20 +836,22 @@ class SparkSQLLineageParserHelperSuite extends KyuubiFunSuite |GROUP BY |stat_date, channel_id, sub_channel_id, user_type, country_name |""".stripMargin - val ret0 = exectractLineage(sql0) + val ret0 = extractLineage(sql0) assert(ret0 == Lineage( - List("test_db.test_order_item"), + List(s"$DEFAULT_CATALOG.test_db.test_order_item"), List(), List( - ("stat_date", Set("test_db.test_order_item.stat_date")), - ("channel_id", Set("test_db.test_order_item.channel_id")), - ("sub_channel_id", Set("test_db.test_order_item.sub_channel_id")), - ("user_type", Set("test_db.test_order_item.user_type")), - ("country_name", Set("test_db.test_order_item.country_name")), - ("get_count0", Set("test_db.test_order_item.order_id")), + ("stat_date", Set(s"$DEFAULT_CATALOG.test_db.test_order_item.stat_date")), + ("channel_id", Set(s"$DEFAULT_CATALOG.test_db.test_order_item.channel_id")), + ("sub_channel_id", Set(s"$DEFAULT_CATALOG.test_db.test_order_item.sub_channel_id")), + ("user_type", Set(s"$DEFAULT_CATALOG.test_db.test_order_item.user_type")), + ("country_name", Set(s"$DEFAULT_CATALOG.test_db.test_order_item.country_name")), + ("get_count0", Set(s"$DEFAULT_CATALOG.test_db.test_order_item.order_id")), ( "get_amount0", - Set("test_db.test_order_item.goods_count", "test_db.test_order_item.shop_price")), + Set( + s"$DEFAULT_CATALOG.test_db.test_order_item.goods_count", + s"$DEFAULT_CATALOG.test_db.test_order_item.shop_price")), ("add_time", Set[String]())))) val sql1 = """ @@ -805,104 +876,120 @@ class SparkSQLLineageParserHelperSuite extends KyuubiFunSuite |) a |GROUP BY a.channel_id, a.sub_channel_id, a.country_name |""".stripMargin - val ret1 = exectractLineage(sql1) + val ret1 = extractLineage(sql1) assert(ret1 == Lineage( - List("test_db.test_order_item", "test_db.test_p0_order_item"), + List( + s"$DEFAULT_CATALOG.test_db.test_order_item", + s"$DEFAULT_CATALOG.test_db.test_p0_order_item"), List(), List( ( "channel_id", - Set("test_db.test_order_item.channel_id", "test_db.test_p0_order_item.channel_id")), + Set( + s"$DEFAULT_CATALOG.test_db.test_order_item.channel_id", + s"$DEFAULT_CATALOG.test_db.test_p0_order_item.channel_id")), ( "sub_channel_id", Set( - "test_db.test_order_item.sub_channel_id", - "test_db.test_p0_order_item.sub_channel_id")), + s"$DEFAULT_CATALOG.test_db.test_order_item.sub_channel_id", + s"$DEFAULT_CATALOG.test_db.test_p0_order_item.sub_channel_id")), ( "country_name", - Set("test_db.test_order_item.country_name", "test_db.test_p0_order_item.country_name")), - ("get_count0", Set("test_db.test_order_item.order_id")), + Set( + s"$DEFAULT_CATALOG.test_db.test_order_item.country_name", + s"$DEFAULT_CATALOG.test_db.test_p0_order_item.country_name")), + ("get_count0", Set(s"$DEFAULT_CATALOG.test_db.test_order_item.order_id")), ( "get_amount0", - Set("test_db.test_order_item.goods_count", "test_db.test_order_item.shop_price")), + Set( + s"$DEFAULT_CATALOG.test_db.test_order_item.goods_count", + s"$DEFAULT_CATALOG.test_db.test_order_item.shop_price")), ("add_time", Set[String]())))) } } test("columns lineage extract - agg sql") { val sql0 = """select key as a, count(*) as b, 1 as c from test_db0.test_table0 group by key""" - val ret0 = exectractLineage(sql0) + val ret0 = extractLineage(sql0) assert(ret0 == Lineage( - List("test_db0.test_table0"), + List(s"$DEFAULT_CATALOG.test_db0.test_table0"), List(), List( - ("a", Set("test_db0.test_table0.key")), - ("b", Set("test_db0.test_table0.__count__")), + ("a", Set(s"$DEFAULT_CATALOG.test_db0.test_table0.key")), + ("b", Set(s"$DEFAULT_CATALOG.test_db0.test_table0.__count__")), ("c", Set())))) val sql1 = """select count(*) as a, 1 as b from test_db0.test_table0""" - val ret1 = exectractLineage(sql1) + val ret1 = extractLineage(sql1) assert(ret1 == Lineage( - List("test_db0.test_table0"), + List(s"$DEFAULT_CATALOG.test_db0.test_table0"), List(), List( - ("a", Set("test_db0.test_table0.__count__")), + ("a", Set(s"$DEFAULT_CATALOG.test_db0.test_table0.__count__")), ("b", Set())))) val sql2 = """select every(key == 1) as a, 1 as b from test_db0.test_table0""" - val ret2 = exectractLineage(sql2) + val ret2 = extractLineage(sql2) assert(ret2 == Lineage( - List("test_db0.test_table0"), + List(s"$DEFAULT_CATALOG.test_db0.test_table0"), List(), List( - ("a", Set("test_db0.test_table0.key")), + ("a", Set(s"$DEFAULT_CATALOG.test_db0.test_table0.key")), ("b", Set())))) val sql3 = """select count(*) as a, 1 as b from test_db0.test_table0""" - val ret3 = exectractLineage(sql3) + val ret3 = extractLineage(sql3) assert(ret3 == Lineage( - List("test_db0.test_table0"), + List(s"$DEFAULT_CATALOG.test_db0.test_table0"), List(), List( - ("a", Set("test_db0.test_table0.__count__")), + ("a", Set(s"$DEFAULT_CATALOG.test_db0.test_table0.__count__")), ("b", Set())))) val sql4 = """select first(key) as a, 1 as b from test_db0.test_table0""" - val ret4 = exectractLineage(sql4) + val ret4 = extractLineage(sql4) assert(ret4 == Lineage( - List("test_db0.test_table0"), + List(s"$DEFAULT_CATALOG.test_db0.test_table0"), List(), List( - ("a", Set("test_db0.test_table0.key")), + ("a", Set(s"$DEFAULT_CATALOG.test_db0.test_table0.key")), ("b", Set())))) val sql5 = """select avg(key) as a, 1 as b from test_db0.test_table0""" - val ret5 = exectractLineage(sql5) + val ret5 = extractLineage(sql5) assert(ret5 == Lineage( - List("test_db0.test_table0"), + List(s"$DEFAULT_CATALOG.test_db0.test_table0"), List(), List( - ("a", Set("test_db0.test_table0.key")), + ("a", Set(s"$DEFAULT_CATALOG.test_db0.test_table0.key")), ("b", Set())))) val sql6 = """select count(value) + sum(key) as a, | 1 as b from test_db0.test_table0""".stripMargin - val ret6 = exectractLineage(sql6) + val ret6 = extractLineage(sql6) assert(ret6 == Lineage( - List("test_db0.test_table0"), + List(s"$DEFAULT_CATALOG.test_db0.test_table0"), List(), List( - ("a", Set("test_db0.test_table0.value", "test_db0.test_table0.key")), + ( + "a", + Set( + s"$DEFAULT_CATALOG.test_db0.test_table0.value", + s"$DEFAULT_CATALOG.test_db0.test_table0.key")), ("b", Set())))) val sql7 = """select count(*) + sum(key) as a, 1 as b from test_db0.test_table0""" - val ret7 = exectractLineage(sql7) + val ret7 = extractLineage(sql7) assert(ret7 == Lineage( - List("test_db0.test_table0"), + List(s"$DEFAULT_CATALOG.test_db0.test_table0"), List(), List( - ("a", Set("test_db0.test_table0.__count__", "test_db0.test_table0.key")), + ( + "a", + Set( + s"$DEFAULT_CATALOG.test_db0.test_table0.__count__", + s"$DEFAULT_CATALOG.test_db0.test_table0.key")), ("b", Set())))) } @@ -920,26 +1007,26 @@ class SparkSQLLineageParserHelperSuite extends KyuubiFunSuite """ |select b.a as aa, t0_cached.b0 as bb from t0_cached join table1 b on b.a = t0_cached.a0 |""".stripMargin - val ret0 = exectractLineage(sql0) + val ret0 = extractLineage(sql0) assert(ret0 == Lineage( - List("default.table1", "default.table0"), + List(s"$DEFAULT_CATALOG.default.table1", s"$DEFAULT_CATALOG.default.table0"), List(), List( - ("aa", Set("default.table1.a")), - ("bb", Set("default.table0.b"))))) + ("aa", Set(s"$DEFAULT_CATALOG.default.table1.a")), + ("bb", Set(s"$DEFAULT_CATALOG.default.table0.b"))))) val df0 = spark.sql("select a as a0, b as b0 from table0 where a = 2") df0.cache() val df1 = spark.sql("select a, b from table1") val df = df0.join(df1).select(df0("a0").alias("aa"), df1("b").alias("bb")) - val optimized = df.queryExecution.optimizedPlan - val ret1 = SparkSQLLineageParseHelper(spark).transformToLineage(0, optimized).get + val analyzed = df.queryExecution.analyzed + val ret1 = SparkSQLLineageParseHelper(spark).transformToLineage(0, analyzed).get assert(ret1 == Lineage( - List("default.table0", "default.table1"), + List(s"$DEFAULT_CATALOG.default.table0", s"$DEFAULT_CATALOG.default.table1"), List(), List( - ("aa", Set("default.table0.a")), - ("bb", Set("default.table1.b"))))) + ("aa", Set(s"$DEFAULT_CATALOG.default.table0.a")), + ("bb", Set(s"$DEFAULT_CATALOG.default.table1.b"))))) } } @@ -955,158 +1042,414 @@ class SparkSQLLineageParserHelperSuite extends KyuubiFunSuite """ |select a as aa, bb, cc from (select b as bb, c as cc from table1) t0, table0 |""".stripMargin - val ret0 = exectractLineage(sql0) + val ret0 = extractLineage(sql0) assert(ret0 == Lineage( - List("default.table0", "default.table1"), + List(s"$DEFAULT_CATALOG.default.table0", s"$DEFAULT_CATALOG.default.table1"), List(), List( - ("aa", Set("default.table0.a")), - ("bb", Set("default.table1.b")), - ("cc", Set("default.table1.c"))))) + ("aa", Set(s"$DEFAULT_CATALOG.default.table0.a")), + ("bb", Set(s"$DEFAULT_CATALOG.default.table1.b")), + ("cc", Set(s"$DEFAULT_CATALOG.default.table1.c"))))) val sql1 = """ |select (select a from table1) as aa, b as bb from table1 |""".stripMargin - val ret1 = exectractLineage(sql1) + val ret1 = extractLineage(sql1) assert(ret1 == Lineage( - List("default.table1"), + List(s"$DEFAULT_CATALOG.default.table1"), List(), List( - ("aa", Set("default.table1.a")), - ("bb", Set("default.table1.b"))))) + ("aa", Set(s"$DEFAULT_CATALOG.default.table1.a")), + ("bb", Set(s"$DEFAULT_CATALOG.default.table1.b"))))) val sql2 = """ |select (select count(*) from table0) as aa, b as bb from table1 |""".stripMargin - val ret2 = exectractLineage(sql2) + val ret2 = extractLineage(sql2) assert(ret2 == Lineage( - List("default.table0", "default.table1"), + List(s"$DEFAULT_CATALOG.default.table0", s"$DEFAULT_CATALOG.default.table1"), List(), List( - ("aa", Set("default.table0.__count__")), - ("bb", Set("default.table1.b"))))) + ("aa", Set(s"$DEFAULT_CATALOG.default.table0.__count__")), + ("bb", Set(s"$DEFAULT_CATALOG.default.table1.b"))))) // ListQuery val sql3 = """ |select * from table0 where table0.a in (select a from table1) |""".stripMargin - val ret3 = exectractLineage(sql3) + val ret3 = extractLineage(sql3) assert(ret3 == Lineage( - List("default.table0"), + List(s"$DEFAULT_CATALOG.default.table0"), List(), List( - ("a", Set("default.table0.a")), - ("b", Set("default.table0.b")), - ("c", Set("default.table0.c"))))) + ("a", Set(s"$DEFAULT_CATALOG.default.table0.a")), + ("b", Set(s"$DEFAULT_CATALOG.default.table0.b")), + ("c", Set(s"$DEFAULT_CATALOG.default.table0.c"))))) // Exists val sql4 = """ |select * from table0 where exists (select * from table1 where table0.c = table1.c) |""".stripMargin - val ret4 = exectractLineage(sql4) + val ret4 = extractLineage(sql4) assert(ret4 == Lineage( - List("default.table0"), + List(s"$DEFAULT_CATALOG.default.table0"), List(), List( - ("a", Set("default.table0.a")), - ("b", Set("default.table0.b")), - ("c", Set("default.table0.c"))))) + ("a", Set(s"$DEFAULT_CATALOG.default.table0.a")), + ("b", Set(s"$DEFAULT_CATALOG.default.table0.b")), + ("c", Set(s"$DEFAULT_CATALOG.default.table0.c"))))) val sql5 = """ |select * from table0 where exists (select * from table1 where c = "odone") |""".stripMargin - val ret5 = exectractLineage(sql5) + val ret5 = extractLineage(sql5) assert(ret5 == Lineage( - List("default.table0"), + List(s"$DEFAULT_CATALOG.default.table0"), List(), List( - ("a", Set("default.table0.a")), - ("b", Set("default.table0.b")), - ("c", Set("default.table0.c"))))) + ("a", Set(s"$DEFAULT_CATALOG.default.table0.a")), + ("b", Set(s"$DEFAULT_CATALOG.default.table0.b")), + ("c", Set(s"$DEFAULT_CATALOG.default.table0.c"))))) val sql6 = """ |select * from table0 where not exists (select * from table1 where c = "odone") |""".stripMargin - val ret6 = exectractLineage(sql6) + val ret6 = extractLineage(sql6) assert(ret6 == Lineage( - List("default.table0"), + List(s"$DEFAULT_CATALOG.default.table0"), List(), List( - ("a", Set("default.table0.a")), - ("b", Set("default.table0.b")), - ("c", Set("default.table0.c"))))) + ("a", Set(s"$DEFAULT_CATALOG.default.table0.a")), + ("b", Set(s"$DEFAULT_CATALOG.default.table0.b")), + ("c", Set(s"$DEFAULT_CATALOG.default.table0.c"))))) val sql7 = """ |select * from table0 where table0.a not in (select a from table1) |""".stripMargin - val ret7 = exectractLineage(sql7) + val ret7 = extractLineage(sql7) assert(ret7 == Lineage( - List("default.table0"), + List(s"$DEFAULT_CATALOG.default.table0"), List(), List( - ("a", Set("default.table0.a")), - ("b", Set("default.table0.b")), - ("c", Set("default.table0.c"))))) + ("a", Set(s"$DEFAULT_CATALOG.default.table0.a")), + ("b", Set(s"$DEFAULT_CATALOG.default.table0.b")), + ("c", Set(s"$DEFAULT_CATALOG.default.table0.c"))))) val sql8 = """ |select (select a from table1) + 1, b as bb from table1 |""".stripMargin - val ret8 = exectractLineage(sql8) + val ret8 = extractLineage(sql8) assert(ret8 == Lineage( - List("default.table1"), + List(s"$DEFAULT_CATALOG.default.table1"), List(), List( - ("(scalarsubquery() + 1)", Set("default.table1.a")), - ("bb", Set("default.table1.b"))))) + ("(scalarsubquery() + 1)", Set(s"$DEFAULT_CATALOG.default.table1.a")), + ("bb", Set(s"$DEFAULT_CATALOG.default.table1.b"))))) val sql9 = """ |select (select a from table1 limit 1) + 1 as aa, b as bb from table1 |""".stripMargin - val ret9 = exectractLineage(sql9) + val ret9 = extractLineage(sql9) assert(ret9 == Lineage( - List("default.table1"), + List(s"$DEFAULT_CATALOG.default.table1"), List(), List( - ("aa", Set("default.table1.a")), - ("bb", Set("default.table1.b"))))) + ("aa", Set(s"$DEFAULT_CATALOG.default.table1.a")), + ("bb", Set(s"$DEFAULT_CATALOG.default.table1.b"))))) val sql10 = """ |select (select a from table1 limit 1) + (select a from table0 limit 1) + 1 as aa, | b as bb from table1 |""".stripMargin - val ret10 = exectractLineage(sql10) + val ret10 = extractLineage(sql10) assert(ret10 == Lineage( - List("default.table1", "default.table0"), + List(s"$DEFAULT_CATALOG.default.table1", s"$DEFAULT_CATALOG.default.table0"), + List(), + List( + ("aa", Set(s"$DEFAULT_CATALOG.default.table1.a", s"$DEFAULT_CATALOG.default.table0.a")), + ("bb", Set(s"$DEFAULT_CATALOG.default.table1.b"))))) + + val sql11 = + """ + |select tmp.a, b from (select * from table1) tmp; + |""".stripMargin + + val ret11 = extractLineage(sql11) + assert(ret11 == Lineage( + List(s"$DEFAULT_CATALOG.default.table1"), List(), List( - ("aa", Set("default.table1.a", "default.table0.a")), - ("bb", Set("default.table1.b"))))) + ("a", Set(s"$DEFAULT_CATALOG.default.table1.a")), + ("b", Set(s"$DEFAULT_CATALOG.default.table1.b"))))) + } + } + + test("test group by") { + withTable("t1", "t2", "v2_catalog.db.t1", "v2_catalog.db.t2") { _ => + spark.sql("CREATE TABLE t1 (a string, b string, c string) USING hive") + spark.sql("CREATE TABLE t2 (a string, b string, c string) USING hive") + spark.sql("CREATE TABLE v2_catalog.db.t1 (a string, b string, c string)") + spark.sql("CREATE TABLE v2_catalog.db.t2 (a string, b string, c string)") + val ret0 = + extractLineage( + s"insert into table t1 select a," + + s"concat_ws('/', collect_set(b))," + + s"count(distinct(b)) * count(distinct(c))" + + s"from t2 group by a") + assert(ret0 == Lineage( + List(s"$DEFAULT_CATALOG.default.t2"), + List(s"$DEFAULT_CATALOG.default.t1"), + List( + (s"$DEFAULT_CATALOG.default.t1.a", Set(s"$DEFAULT_CATALOG.default.t2.a")), + (s"$DEFAULT_CATALOG.default.t1.b", Set(s"$DEFAULT_CATALOG.default.t2.b")), + ( + s"$DEFAULT_CATALOG.default.t1.c", + Set(s"$DEFAULT_CATALOG.default.t2.b", s"$DEFAULT_CATALOG.default.t2.c"))))) + + val ret1 = + extractLineage( + s"insert into table v2_catalog.db.t1 select a," + + s"concat_ws('/', collect_set(b))," + + s"count(distinct(b)) * count(distinct(c))" + + s"from v2_catalog.db.t2 group by a") + assert(ret1 == Lineage( + List("v2_catalog.db.t2"), + List("v2_catalog.db.t1"), + List( + ("v2_catalog.db.t1.a", Set("v2_catalog.db.t2.a")), + ("v2_catalog.db.t1.b", Set("v2_catalog.db.t2.b")), + ("v2_catalog.db.t1.c", Set("v2_catalog.db.t2.b", "v2_catalog.db.t2.c"))))) + + val ret2 = + extractLineage( + s"insert into table v2_catalog.db.t1 select a," + + s"count(distinct(b+c))," + + s"count(distinct(b)) * count(distinct(c))" + + s"from v2_catalog.db.t2 group by a") + assert(ret2 == Lineage( + List("v2_catalog.db.t2"), + List("v2_catalog.db.t1"), + List( + ("v2_catalog.db.t1.a", Set("v2_catalog.db.t2.a")), + ("v2_catalog.db.t1.b", Set("v2_catalog.db.t2.b", "v2_catalog.db.t2.c")), + ("v2_catalog.db.t1.c", Set("v2_catalog.db.t2.b", "v2_catalog.db.t2.c"))))) + } + } + + test("test grouping sets") { + withTable("t1", "t2") { _ => + spark.sql("CREATE TABLE t1 (a string, b string, c string) USING hive") + spark.sql("CREATE TABLE t2 (a string, b string, c string, d string) USING hive") + val ret0 = + extractLineage( + s"insert into table t1 select a,b,GROUPING__ID " + + s"from t2 group by a,b,c,d grouping sets ((a,b,c), (a,b,d))") + assert(ret0 == Lineage( + List(s"$DEFAULT_CATALOG.default.t2"), + List(s"$DEFAULT_CATALOG.default.t1"), + List( + (s"$DEFAULT_CATALOG.default.t1.a", Set(s"$DEFAULT_CATALOG.default.t2.a")), + (s"$DEFAULT_CATALOG.default.t1.b", Set(s"$DEFAULT_CATALOG.default.t2.b")), + (s"$DEFAULT_CATALOG.default.t1.c", Set())))) + } + } + + test("test cache table with window function") { + withTable("t1", "t2") { _ => + spark.sql("CREATE TABLE t1 (a string, b string) USING hive") + spark.sql("CREATE TABLE t2 (a string, b string) USING hive") + + spark.sql( + s"cache table c1 select * from (" + + s"select a, b, row_number() over (partition by a order by b asc ) rank from t2)" + + s" where rank=1") + val ret0 = extractLineage("insert overwrite table t1 select a, b from c1") + assert(ret0 == Lineage( + List(s"$DEFAULT_CATALOG.default.t2"), + List(s"$DEFAULT_CATALOG.default.t1"), + List( + (s"$DEFAULT_CATALOG.default.t1.a", Set(s"$DEFAULT_CATALOG.default.t2.a")), + (s"$DEFAULT_CATALOG.default.t1.b", Set(s"$DEFAULT_CATALOG.default.t2.b"))))) + + val ret1 = extractLineage("insert overwrite table t1 select a, rank from c1") + assert(ret1 == Lineage( + List(s"$DEFAULT_CATALOG.default.t2"), + List(s"$DEFAULT_CATALOG.default.t1"), + List( + (s"$DEFAULT_CATALOG.default.t1.a", Set(s"$DEFAULT_CATALOG.default.t2.a")), + ( + s"$DEFAULT_CATALOG.default.t1.b", + Set(s"$DEFAULT_CATALOG.default.t2.a", s"$DEFAULT_CATALOG.default.t2.b"))))) + + spark.sql( + s"cache table c2 select * from (" + + s"select b, a, row_number() over (partition by a order by b asc ) rank from t2)" + + s" where rank=1") + val ret2 = extractLineage("insert overwrite table t1 select a, b from c2") + assert(ret2 == Lineage( + List(s"$DEFAULT_CATALOG.default.t2"), + List(s"$DEFAULT_CATALOG.default.t1"), + List( + (s"$DEFAULT_CATALOG.default.t1.a", Set(s"$DEFAULT_CATALOG.default.t2.a")), + (s"$DEFAULT_CATALOG.default.t1.b", Set(s"$DEFAULT_CATALOG.default.t2.b"))))) + + spark.sql( + s"cache table c3 select * from (" + + s"select a as aa, b as bb, row_number() over (partition by a order by b asc ) rank" + + s" from t2) where rank=1") + val ret3 = extractLineage("insert overwrite table t1 select aa, bb from c3") + assert(ret3 == Lineage( + List(s"$DEFAULT_CATALOG.default.t2"), + List(s"$DEFAULT_CATALOG.default.t1"), + List( + (s"$DEFAULT_CATALOG.default.t1.a", Set(s"$DEFAULT_CATALOG.default.t2.a")), + (s"$DEFAULT_CATALOG.default.t1.b", Set(s"$DEFAULT_CATALOG.default.t2.b"))))) + } + } + + test("test count()") { + withTable("t1", "t2") { _ => + spark.sql("CREATE TABLE t1 (a string, b string, c string) USING hive") + spark.sql("CREATE TABLE t2 (a string, b string, c string) USING hive") + val ret0 = extractLineage("insert into t1 select 1,2,(select count(distinct" + + " ifnull(get_json_object(a, '$.b.imei'), get_json_object(a, '$.b.android_id'))) from t2)") + + assert(ret0 == Lineage( + List(s"$DEFAULT_CATALOG.default.t2"), + List(s"$DEFAULT_CATALOG.default.t1"), + List( + (s"$DEFAULT_CATALOG.default.t1.a", Set()), + (s"$DEFAULT_CATALOG.default.t1.b", Set()), + (s"$DEFAULT_CATALOG.default.t1.c", Set(s"$DEFAULT_CATALOG.default.t2.a"))))) + } + } + + test("test create view from view") { + withTable("t1") { _ => + spark.sql("CREATE TABLE t1 (a string, b string, c string) USING hive") + withView("t2") { _ => + spark.sql("CREATE VIEW t2 as select * from t1") + val ret0 = + extractLineage( + s"create or replace view view_tst comment 'view'" + + s" as select a as k,b" + + s" from t2" + + s" where a in ('HELLO') and c = 'HELLO'") + assert(ret0 == Lineage( + List(s"$DEFAULT_CATALOG.default.t1"), + List(s"$DEFAULT_CATALOG.default.view_tst"), + List( + (s"$DEFAULT_CATALOG.default.view_tst.k", Set(s"$DEFAULT_CATALOG.default.t1.a")), + (s"$DEFAULT_CATALOG.default.view_tst.b", Set(s"$DEFAULT_CATALOG.default.t1.b"))))) + } + } + } + + test("test for skip parsing permanent view") { + withTable("t1") { _ => + SparkContextHelper.setConf(LineageConf.SKIP_PARSING_PERMANENT_VIEW_ENABLED, true) + spark.sql("CREATE TABLE t1 (a string, b string, c string) USING hive") + withView("t2") { _ => + spark.sql("CREATE VIEW t2 as select * from t1") + val ret0 = + extractLineage( + s"select a as k, b" + + s" from t2" + + s" where a in ('HELLO') and c = 'HELLO'") + assert(ret0 == Lineage( + List(s"$DEFAULT_CATALOG.default.t2"), + List(), + List( + ("k", Set(s"$DEFAULT_CATALOG.default.t2.a")), + ("b", Set(s"$DEFAULT_CATALOG.default.t2.b"))))) + } + } + } + + test("test the statement with FROM xxx INSERT xxx") { + withTable("t1", "t2", "t3") { _ => + spark.sql("CREATE TABLE t1 (a string, b string) USING hive") + spark.sql("CREATE TABLE t2 (a string, b string) USING hive") + spark.sql("CREATE TABLE t3 (a string, b string) USING hive") + val ret0 = extractLineage("from (select a,b from t1)" + + " insert overwrite table t2 select a,b where a=1" + + " insert overwrite table t3 select a,b where b=1") + assert(ret0 == Lineage( + List(s"$DEFAULT_CATALOG.default.t1"), + List(s"$DEFAULT_CATALOG.default.t2", s"$DEFAULT_CATALOG.default.t3"), + List( + (s"$DEFAULT_CATALOG.default.t2.a", Set(s"$DEFAULT_CATALOG.default.t1.a")), + (s"$DEFAULT_CATALOG.default.t2.b", Set(s"$DEFAULT_CATALOG.default.t1.b")), + (s"$DEFAULT_CATALOG.default.t3.a", Set(s"$DEFAULT_CATALOG.default.t1.a")), + (s"$DEFAULT_CATALOG.default.t3.b", Set(s"$DEFAULT_CATALOG.default.t1.b"))))) + } + } + + test("test lateral view explode") { + withTable("t1", "t2") { _ => + spark.sql("CREATE TABLE t1 (a string, b string, c string, d string) USING hive") + spark.sql("CREATE TABLE t2 (a string, b string, c string, d string) USING hive") + + val ret0 = extractLineage("insert into t1 select 1, t2.b, cc.action, t2.d " + + "from t2 lateral view explode(split(c,'\\},\\{')) cc as action") + assert(ret0 == Lineage( + List(s"$DEFAULT_CATALOG.default.t2"), + List(s"$DEFAULT_CATALOG.default.t1"), + List( + (s"$DEFAULT_CATALOG.default.t1.a", Set()), + (s"$DEFAULT_CATALOG.default.t1.b", Set(s"$DEFAULT_CATALOG.default.t2.b")), + (s"$DEFAULT_CATALOG.default.t1.c", Set(s"$DEFAULT_CATALOG.default.t2.c")), + (s"$DEFAULT_CATALOG.default.t1.d", Set(s"$DEFAULT_CATALOG.default.t2.d"))))) + + val ret1 = extractLineage("insert into t1 select 1, t2.b, cc.action0, dd.action1 " + + "from t2 " + + "lateral view explode(split(c,'\\},\\{')) cc as action0 " + + "lateral view explode(split(d,'\\},\\{')) dd as action1") + assert(ret1 == Lineage( + List(s"$DEFAULT_CATALOG.default.t2"), + List(s"$DEFAULT_CATALOG.default.t1"), + List( + (s"$DEFAULT_CATALOG.default.t1.a", Set()), + (s"$DEFAULT_CATALOG.default.t1.b", Set(s"$DEFAULT_CATALOG.default.t2.b")), + (s"$DEFAULT_CATALOG.default.t1.c", Set(s"$DEFAULT_CATALOG.default.t2.c")), + (s"$DEFAULT_CATALOG.default.t1.d", Set(s"$DEFAULT_CATALOG.default.t2.d"))))) + + val ret2 = extractLineage("insert into t1 select 1, t2.b, dd.pos, dd.action1 " + + "from t2 " + + "lateral view posexplode(split(d,'\\},\\{')) dd as pos, action1") + assert(ret2 == Lineage( + List(s"$DEFAULT_CATALOG.default.t2"), + List(s"$DEFAULT_CATALOG.default.t1"), + List( + (s"$DEFAULT_CATALOG.default.t1.a", Set()), + (s"$DEFAULT_CATALOG.default.t1.b", Set(s"$DEFAULT_CATALOG.default.t2.b")), + (s"$DEFAULT_CATALOG.default.t1.c", Set(s"$DEFAULT_CATALOG.default.t2.d")), + (s"$DEFAULT_CATALOG.default.t1.d", Set(s"$DEFAULT_CATALOG.default.t2.d"))))) } } - private def exectractLineageWithoutExecuting(sql: String): Lineage = { + private def extractLineageWithoutExecuting(sql: String): Lineage = { val parsed = spark.sessionState.sqlParser.parsePlan(sql) val analyzed = spark.sessionState.analyzer.execute(parsed) spark.sessionState.analyzer.checkAnalysis(analyzed) - val optimized = spark.sessionState.optimizer.execute(analyzed) - SparkSQLLineageParseHelper(spark).transformToLineage(0, optimized).get + SparkSQLLineageParseHelper(spark).transformToLineage(0, analyzed).get } - private def exectractLineage(sql: String): Lineage = { + private def extractLineage(sql: String): Lineage = { val parsed = spark.sessionState.sqlParser.parsePlan(sql) val qe = spark.sessionState.executePlan(parsed) - val optimized = qe.optimizedPlan - SparkSQLLineageParseHelper(spark).transformToLineage(0, optimized).get + val analyzed = qe.analyzed + SparkSQLLineageParseHelper(spark).transformToLineage(0, analyzed).get } } diff --git a/extensions/spark/kyuubi-spark-lineage/src/test/scala/org/apache/spark/sql/SparkListenerExtenstionTest.scala b/extensions/spark/kyuubi-spark-lineage/src/test/scala/org/apache/spark/sql/SparkListenerExtensionTest.scala similarity index 100% rename from extensions/spark/kyuubi-spark-lineage/src/test/scala/org/apache/spark/sql/SparkListenerExtenstionTest.scala rename to extensions/spark/kyuubi-spark-lineage/src/test/scala/org/apache/spark/sql/SparkListenerExtensionTest.scala diff --git a/externals/kyuubi-chat-engine/pom.xml b/externals/kyuubi-chat-engine/pom.xml new file mode 100644 index 000000000..3639ceed3 --- /dev/null +++ b/externals/kyuubi-chat-engine/pom.xml @@ -0,0 +1,90 @@ + + + + 4.0.0 + + org.apache.kyuubi + kyuubi-parent + 1.9.0-SNAPSHOT + ../../pom.xml + + + kyuubi-chat-engine_${scala.binary.version} + jar + Kyuubi Project Engine Chat + https://kyuubi.apache.org/ + + + + + org.apache.kyuubi + kyuubi-common_${scala.binary.version} + ${project.version} + + + + org.apache.kyuubi + kyuubi-ha_${scala.binary.version} + ${project.version} + + + + com.theokanning.openai-gpt3-java + service + ${openai.java.version} + + + + org.apache.kyuubi + kyuubi-common_${scala.binary.version} + ${project.version} + test-jar + test + + + + org.apache.kyuubi + ${hive.jdbc.artifact} + ${project.version} + test + + + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + prepare-test-jar + + test-jar + + test-compile + + + + + target/scala-${scala.binary.version}/classes + target/scala-${scala.binary.version}/test-classes + + + diff --git a/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/ChatBackendService.scala b/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/ChatBackendService.scala new file mode 100644 index 000000000..fdc710e2c --- /dev/null +++ b/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/ChatBackendService.scala @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.engine.chat + +import org.apache.kyuubi.engine.chat.session.ChatSessionManager +import org.apache.kyuubi.service.AbstractBackendService +import org.apache.kyuubi.session.SessionManager + +class ChatBackendService + extends AbstractBackendService("ChatBackendService") { + + override val sessionManager: SessionManager = new ChatSessionManager() + +} diff --git a/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/ChatEngine.scala b/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/ChatEngine.scala new file mode 100644 index 000000000..c1fdea953 --- /dev/null +++ b/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/ChatEngine.scala @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.engine.chat + +import ChatEngine.currentEngine + +import org.apache.kyuubi.{Logging, Utils} +import org.apache.kyuubi.Utils.{addShutdownHook, JDBC_ENGINE_SHUTDOWN_PRIORITY} +import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.ha.HighAvailabilityConf.HA_ZK_CONN_RETRY_POLICY +import org.apache.kyuubi.ha.client.RetryPolicies +import org.apache.kyuubi.service.Serverable +import org.apache.kyuubi.util.SignalRegister + +class ChatEngine extends Serverable("ChatEngine") { + + override val backendService = new ChatBackendService() + override val frontendServices = Seq(new ChatTBinaryFrontendService(this)) + + override def start(): Unit = { + super.start() + // Start engine self-terminating checker after all services are ready and it can be reached by + // all servers in engine spaces. + backendService.sessionManager.startTerminatingChecker(() => { + currentEngine.foreach(_.stop()) + }) + } + + override protected def stopServer(): Unit = {} +} + +object ChatEngine extends Logging { + + val kyuubiConf: KyuubiConf = KyuubiConf() + + var currentEngine: Option[ChatEngine] = None + + def startEngine(): Unit = { + currentEngine = Some(new ChatEngine()) + currentEngine.foreach { engine => + engine.initialize(kyuubiConf) + engine.start() + addShutdownHook( + () => { + engine.stop() + }, + JDBC_ENGINE_SHUTDOWN_PRIORITY + 1) + } + } + + def main(args: Array[String]): Unit = { + SignalRegister.registerLogger(logger) + + try { + Utils.fromCommandLineArgs(args, kyuubiConf) + kyuubiConf.setIfMissing(KyuubiConf.FRONTEND_THRIFT_BINARY_BIND_PORT, 0) + kyuubiConf.setIfMissing(HA_ZK_CONN_RETRY_POLICY, RetryPolicies.N_TIME.toString) + + startEngine() + } catch { + case t: Throwable if currentEngine.isDefined => + currentEngine.foreach { engine => + engine.stop() + } + error("Failed to create Chat Engine", t) + throw t + case t: Throwable => + error("Failed to create Chat Engine.", t) + throw t + } + } +} diff --git a/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/ChatTBinaryFrontendService.scala b/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/ChatTBinaryFrontendService.scala new file mode 100644 index 000000000..80702c97c --- /dev/null +++ b/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/ChatTBinaryFrontendService.scala @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.engine.chat + +import org.apache.kyuubi.ha.client.{EngineServiceDiscovery, ServiceDiscovery} +import org.apache.kyuubi.service.{Serverable, Service, TBinaryFrontendService} + +class ChatTBinaryFrontendService(override val serverable: Serverable) + extends TBinaryFrontendService("ChatTBinaryFrontend") { + + /** + * An optional `ServiceDiscovery` for [[FrontendService]] to expose itself + */ + override lazy val discoveryService: Option[Service] = + if (ServiceDiscovery.supportServiceDiscovery(conf)) { + Some(new EngineServiceDiscovery(this)) + } else { + None + } +} diff --git a/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/operation/ChatOperation.scala b/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/operation/ChatOperation.scala new file mode 100644 index 000000000..b0b1806f8 --- /dev/null +++ b/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/operation/ChatOperation.scala @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.engine.chat.operation + +import org.apache.hive.service.rpc.thrift._ + +import org.apache.kyuubi.{KyuubiSQLException, Utils} +import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.engine.chat.schema.{RowSet, SchemaHelper} +import org.apache.kyuubi.operation.{AbstractOperation, FetchIterator, OperationState} +import org.apache.kyuubi.operation.FetchOrientation.{FETCH_FIRST, FETCH_NEXT, FETCH_PRIOR, FetchOrientation} +import org.apache.kyuubi.session.Session + +abstract class ChatOperation(session: Session) extends AbstractOperation(session) { + + protected var iter: FetchIterator[Array[String]] = _ + + protected lazy val conf: KyuubiConf = session.sessionManager.getConf + + override def getNextRowSetInternal( + order: FetchOrientation, + rowSetSize: Int): TFetchResultsResp = { + validateDefaultFetchOrientation(order) + assertState(OperationState.FINISHED) + setHasResultSet(true) + order match { + case FETCH_NEXT => + iter.fetchNext() + case FETCH_PRIOR => + iter.fetchPrior(rowSetSize) + case FETCH_FIRST => + iter.fetchAbsolute(0) + } + + val taken = iter.take(rowSetSize) + val resultRowSet = RowSet.toTRowSet(taken.toSeq, 1, getProtocolVersion) + resultRowSet.setStartRowOffset(iter.getPosition) + val resp = new TFetchResultsResp(OK_STATUS) + resp.setResults(resultRowSet) + resp.setHasMoreRows(false) + resp + } + + override def cancel(): Unit = { + cleanup(OperationState.CANCELED) + } + + override def close(): Unit = { + cleanup(OperationState.CLOSED) + } + + protected def onError(cancel: Boolean = false): PartialFunction[Throwable, Unit] = { + // We should use Throwable instead of Exception since `java.lang.NoClassDefFoundError` + // could be thrown. + case e: Throwable => + withLockRequired { + val errMsg = Utils.stringifyException(e) + if (state == OperationState.TIMEOUT) { + val ke = KyuubiSQLException(s"Timeout operating $opType: $errMsg") + setOperationException(ke) + throw ke + } else if (isTerminalState(state)) { + setOperationException(KyuubiSQLException(errMsg)) + warn(s"Ignore exception in terminal state with $statementId: $errMsg") + } else { + error(s"Error operating $opType: $errMsg", e) + val ke = KyuubiSQLException(s"Error operating $opType: $errMsg", e) + setOperationException(ke) + setState(OperationState.ERROR) + throw ke + } + } + } + + override protected def beforeRun(): Unit = { + setState(OperationState.PENDING) + setHasResultSet(true) + } + + override protected def afterRun(): Unit = {} + + override def getResultSetMetadata: TGetResultSetMetadataResp = { + val tTableSchema = SchemaHelper.stringTTableSchema("reply") + val resp = new TGetResultSetMetadataResp + resp.setSchema(tTableSchema) + resp.setStatus(OK_STATUS) + resp + } + + override def shouldRunAsync: Boolean = false +} diff --git a/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/operation/ChatOperationManager.scala b/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/operation/ChatOperationManager.scala new file mode 100644 index 000000000..1e8916517 --- /dev/null +++ b/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/operation/ChatOperationManager.scala @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.engine.chat.operation + +import java.util + +import org.apache.kyuubi.KyuubiSQLException +import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.engine.chat.provider.ChatProvider +import org.apache.kyuubi.operation.{Operation, OperationManager} +import org.apache.kyuubi.session.Session + +class ChatOperationManager( + conf: KyuubiConf, + chatProvider: ChatProvider) extends OperationManager("ChatOperationManager") { + + override def newExecuteStatementOperation( + session: Session, + statement: String, + confOverlay: Map[String, String], + runAsync: Boolean, + queryTimeout: Long): Operation = { + val executeStatement = + new ExecuteStatement( + session, + statement, + runAsync, + queryTimeout, + chatProvider) + addOperation(executeStatement) + } + + override def newGetTypeInfoOperation(session: Session): Operation = { + throw KyuubiSQLException.featureNotSupported() + } + + override def newGetCatalogsOperation(session: Session): Operation = { + throw KyuubiSQLException.featureNotSupported() + } + + override def newGetSchemasOperation( + session: Session, + catalog: String, + schema: String): Operation = { + throw KyuubiSQLException.featureNotSupported() + } + + override def newGetTablesOperation( + session: Session, + catalogName: String, + schemaName: String, + tableName: String, + tableTypes: util.List[String]): Operation = { + throw KyuubiSQLException.featureNotSupported() + } + + override def newGetTableTypesOperation(session: Session): Operation = { + throw KyuubiSQLException.featureNotSupported() + } + + override def newGetColumnsOperation( + session: Session, + catalogName: String, + schemaName: String, + tableName: String, + columnName: String): Operation = { + throw KyuubiSQLException.featureNotSupported() + } + + override def newGetFunctionsOperation( + session: Session, + catalogName: String, + schemaName: String, + functionName: String): Operation = { + throw KyuubiSQLException.featureNotSupported() + } + + override def newGetPrimaryKeysOperation( + session: Session, + catalogName: String, + schemaName: String, + tableName: String): Operation = { + throw KyuubiSQLException.featureNotSupported() + } + + override def newGetCrossReferenceOperation( + session: Session, + primaryCatalog: String, + primarySchema: String, + primaryTable: String, + foreignCatalog: String, + foreignSchema: String, + foreignTable: String): Operation = { + throw KyuubiSQLException.featureNotSupported() + } + + override def getQueryId(operation: Operation): String = { + throw KyuubiSQLException.featureNotSupported() + } + + override def newSetCurrentCatalogOperation(session: Session, catalog: String): Operation = { + throw KyuubiSQLException.featureNotSupported() + } + + override def newGetCurrentCatalogOperation(session: Session): Operation = { + throw KyuubiSQLException.featureNotSupported() + } + + override def newSetCurrentDatabaseOperation(session: Session, database: String): Operation = { + throw KyuubiSQLException.featureNotSupported() + } + + override def newGetCurrentDatabaseOperation(session: Session): Operation = { + throw KyuubiSQLException.featureNotSupported() + } +} diff --git a/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/operation/ExecuteStatement.scala b/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/operation/ExecuteStatement.scala new file mode 100644 index 000000000..754a51932 --- /dev/null +++ b/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/operation/ExecuteStatement.scala @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.engine.chat.operation + +import org.apache.kyuubi.Logging +import org.apache.kyuubi.engine.chat.provider.ChatProvider +import org.apache.kyuubi.operation.{ArrayFetchIterator, OperationState} +import org.apache.kyuubi.operation.log.OperationLog +import org.apache.kyuubi.session.Session + +class ExecuteStatement( + session: Session, + override val statement: String, + override val shouldRunAsync: Boolean, + queryTimeout: Long, + chatProvider: ChatProvider) + extends ChatOperation(session) with Logging { + + private val operationLog: OperationLog = OperationLog.createOperationLog(session, getHandle) + override def getOperationLog: Option[OperationLog] = Option(operationLog) + + override protected def runInternal(): Unit = { + addTimeoutMonitor(queryTimeout) + if (shouldRunAsync) { + val asyncOperation = new Runnable { + override def run(): Unit = { + executeStatement() + } + } + val chatSessionManager = session.sessionManager + val backgroundHandle = chatSessionManager.submitBackgroundOperation(asyncOperation) + setBackgroundHandle(backgroundHandle) + } else { + executeStatement() + } + } + + private def executeStatement(): Unit = { + setState(OperationState.RUNNING) + + try { + val reply = chatProvider.ask(session.handle.identifier.toString, statement) + iter = new ArrayFetchIterator(Array(Array(reply))) + + setState(OperationState.FINISHED) + } catch { + onError(true) + } finally { + shutdownTimeoutMonitor() + } + } +} diff --git a/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/provider/ChatGPTProvider.scala b/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/provider/ChatGPTProvider.scala new file mode 100644 index 000000000..aae8b488a --- /dev/null +++ b/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/provider/ChatGPTProvider.scala @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.engine.chat.provider + +import java.net.{InetSocketAddress, Proxy, URL} +import java.time.Duration +import java.util +import java.util.concurrent.TimeUnit + +import scala.collection.JavaConverters._ + +import com.google.common.cache.{CacheBuilder, CacheLoader, LoadingCache} +import com.theokanning.openai.OpenAiApi +import com.theokanning.openai.completion.chat.{ChatCompletionRequest, ChatMessage, ChatMessageRole} +import com.theokanning.openai.service.OpenAiService +import com.theokanning.openai.service.OpenAiService.{defaultClient, defaultObjectMapper, defaultRetrofit} + +import org.apache.kyuubi.config.KyuubiConf + +class ChatGPTProvider(conf: KyuubiConf) extends ChatProvider { + + private val gptApiKey = conf.get(KyuubiConf.ENGINE_CHAT_GPT_API_KEY).getOrElse { + throw new IllegalArgumentException( + s"'${KyuubiConf.ENGINE_CHAT_GPT_API_KEY.key}' must be configured, " + + s"which could be got at https://platform.openai.com/account/api-keys") + } + + private val openAiService: OpenAiService = { + val builder = defaultClient( + gptApiKey, + Duration.ofMillis(conf.get(KyuubiConf.ENGINE_CHAT_GPT_HTTP_SOCKET_TIMEOUT))) + .newBuilder + .connectTimeout(Duration.ofMillis(conf.get(KyuubiConf.ENGINE_CHAT_GPT_HTTP_CONNECT_TIMEOUT))) + + conf.get(KyuubiConf.ENGINE_CHAT_GPT_HTTP_PROXY) match { + case Some(httpProxyUrl) => + val url = new URL(httpProxyUrl) + val proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(url.getHost, url.getPort)) + builder.proxy(proxy) + case _ => + } + + val retrofit = defaultRetrofit(builder.build(), defaultObjectMapper) + val api = retrofit.create(classOf[OpenAiApi]) + new OpenAiService(api) + } + + private var sessionUser: Option[String] = None + + private val chatHistory: LoadingCache[String, util.ArrayDeque[ChatMessage]] = + CacheBuilder.newBuilder() + .expireAfterWrite(10, TimeUnit.MINUTES) + .build(new CacheLoader[String, util.ArrayDeque[ChatMessage]] { + override def load(sessionId: String): util.ArrayDeque[ChatMessage] = + new util.ArrayDeque[ChatMessage] + }) + + override def open(sessionId: String, user: Option[String]): Unit = { + sessionUser = user + chatHistory.getIfPresent(sessionId) + } + + override def ask(sessionId: String, q: String): String = { + val messages = chatHistory.get(sessionId) + try { + messages.addLast(new ChatMessage(ChatMessageRole.USER.value(), q)) + val completionRequest = ChatCompletionRequest.builder() + .model(conf.get(KyuubiConf.ENGINE_CHAT_GPT_MODEL)) + .messages(messages.asScala.toList.asJava) + .user(sessionUser.orNull) + .n(1) + .build() + val responseText = openAiService.createChatCompletion(completionRequest) + .getChoices.get(0).getMessage.getContent + responseText + } catch { + case e: Throwable => + messages.removeLast() + s"Chat failed. Error: ${e.getMessage}" + } + } + + override def close(sessionId: String): Unit = { + chatHistory.invalidate(sessionId) + } +} diff --git a/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/provider/ChatProvider.scala b/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/provider/ChatProvider.scala new file mode 100644 index 000000000..06d719380 --- /dev/null +++ b/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/provider/ChatProvider.scala @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.engine.chat.provider + +import scala.util.control.NonFatal + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.scala.{ClassTagExtensions, DefaultScalaModule} + +import org.apache.kyuubi.{KyuubiException, Logging} +import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.util.reflect.DynConstructors + +trait ChatProvider { + + def open(sessionId: String, user: Option[String] = None): Unit + + def ask(sessionId: String, q: String): String + + def close(sessionId: String): Unit +} + +object ChatProvider extends Logging { + + val mapper: ObjectMapper with ClassTagExtensions = + new ObjectMapper().registerModule(DefaultScalaModule) :: ClassTagExtensions + + def load(conf: KyuubiConf): ChatProvider = { + val groupProviderClass = conf.get(KyuubiConf.ENGINE_CHAT_PROVIDER) + try { + DynConstructors.builder(classOf[ChatProvider]) + .impl(groupProviderClass, classOf[KyuubiConf]) + .impl(groupProviderClass) + .buildChecked + .newInstanceChecked(conf) + } catch { + case _: ClassCastException => + throw new KyuubiException( + s"Class $groupProviderClass is not a child of '${classOf[ChatProvider].getName}'.") + case NonFatal(e) => + throw new IllegalArgumentException(s"Error while instantiating '$groupProviderClass': ", e) + } + } +} diff --git a/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/provider/EchoProvider.scala b/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/provider/EchoProvider.scala new file mode 100644 index 000000000..1116ea785 --- /dev/null +++ b/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/provider/EchoProvider.scala @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.engine.chat.provider + +class EchoProvider extends ChatProvider { + + override def open(sessionId: String, user: Option[String]): Unit = {} + + override def ask(sessionId: String, q: String): String = + "This is ChatKyuubi, nice to meet you!" + + override def close(sessionId: String): Unit = {} +} diff --git a/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/provider/Message.scala b/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/provider/Message.scala new file mode 100644 index 000000000..e2162be9f --- /dev/null +++ b/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/provider/Message.scala @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.engine.chat.provider + +case class Message(role: String, content: String) diff --git a/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/schema/RowSet.scala b/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/schema/RowSet.scala new file mode 100644 index 000000000..3bb4ba7df --- /dev/null +++ b/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/schema/RowSet.scala @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.engine.chat.schema + +import java.util + +import org.apache.hive.service.rpc.thrift._ + +import org.apache.kyuubi.util.RowSetUtils._ + +object RowSet { + + def emptyTRowSet(): TRowSet = { + new TRowSet(0, new java.util.ArrayList[TRow](0)) + } + + def toTRowSet( + rows: Seq[Array[String]], + columnSize: Int, + protocolVersion: TProtocolVersion): TRowSet = { + if (protocolVersion.getValue < TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V6.getValue) { + toRowBasedSet(rows, columnSize) + } else { + toColumnBasedSet(rows, columnSize) + } + } + + def toRowBasedSet(rows: Seq[Array[String]], columnSize: Int): TRowSet = { + val rowSize = rows.length + val tRows = new java.util.ArrayList[TRow](rowSize) + var i = 0 + while (i < rowSize) { + val row = rows(i) + val tRow = new TRow() + var j = 0 + val columnSize = row.length + while (j < columnSize) { + val columnValue = stringTColumnValue(j, row) + tRow.addToColVals(columnValue) + j += 1 + } + i += 1 + tRows.add(tRow) + } + new TRowSet(0, tRows) + } + + def toColumnBasedSet(rows: Seq[Array[String]], columnSize: Int): TRowSet = { + val rowSize = rows.length + val tRowSet = new TRowSet(0, new util.ArrayList[TRow](rowSize)) + var i = 0 + while (i < columnSize) { + val tColumn = toTColumn(rows, i) + tRowSet.addToColumns(tColumn) + i += 1 + } + tRowSet + } + + private def toTColumn(rows: Seq[Array[String]], ordinal: Int): TColumn = { + val nulls = new java.util.BitSet() + val values = getOrSetAsNull[String](rows, ordinal, nulls, "") + TColumn.stringVal(new TStringColumn(values, nulls)) + } + + private def getOrSetAsNull[String]( + rows: Seq[Array[String]], + ordinal: Int, + nulls: util.BitSet, + defaultVal: String): util.List[String] = { + val size = rows.length + val ret = new util.ArrayList[String](size) + var idx = 0 + while (idx < size) { + val row = rows(idx) + val isNull = row(ordinal) == null + if (isNull) { + nulls.set(idx, true) + ret.add(idx, defaultVal) + } else { + ret.add(idx, row(ordinal)) + } + idx += 1 + } + ret + } + + private def stringTColumnValue(ordinal: Int, row: Array[String]): TColumnValue = { + val tStringValue = new TStringValue + if (row(ordinal) != null) tStringValue.setValue(row(ordinal)) + TColumnValue.stringVal(tStringValue) + } +} diff --git a/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/schema/SchemaHelper.scala b/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/schema/SchemaHelper.scala new file mode 100644 index 000000000..8ccfdda2f --- /dev/null +++ b/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/schema/SchemaHelper.scala @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.engine.chat.schema + +import java.util.Collections + +import org.apache.hive.service.rpc.thrift._ + +object SchemaHelper { + + def stringTTypeQualifiers: TTypeQualifiers = { + val ret = new TTypeQualifiers() + val qualifiers = Collections.emptyMap[String, TTypeQualifierValue]() + ret.setQualifiers(qualifiers) + ret + } + + def stringTTypeDesc: TTypeDesc = { + val typeEntry = new TPrimitiveTypeEntry(TTypeId.STRING_TYPE) + typeEntry.setTypeQualifiers(stringTTypeQualifiers) + val tTypeDesc = new TTypeDesc() + tTypeDesc.addToTypes(TTypeEntry.primitiveEntry(typeEntry)) + tTypeDesc + } + + def stringTColumnDesc(fieldName: String, pos: Int): TColumnDesc = { + val tColumnDesc = new TColumnDesc() + tColumnDesc.setColumnName(fieldName) + tColumnDesc.setTypeDesc(stringTTypeDesc) + tColumnDesc.setPosition(pos) + tColumnDesc + } + + def stringTTableSchema(fieldsName: String*): TTableSchema = { + val tTableSchema = new TTableSchema() + fieldsName.zipWithIndex.foreach { case (f, i) => + tTableSchema.addToColumns(stringTColumnDesc(f, i)) + } + tTableSchema + } +} diff --git a/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/session/ChatSessionImpl.scala b/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/session/ChatSessionImpl.scala new file mode 100644 index 000000000..6ec6d0626 --- /dev/null +++ b/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/session/ChatSessionImpl.scala @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.engine.chat.session + +import org.apache.hive.service.rpc.thrift.{TGetInfoType, TGetInfoValue, TProtocolVersion} + +import org.apache.kyuubi.{KYUUBI_VERSION, KyuubiSQLException} +import org.apache.kyuubi.config.KyuubiReservedKeys.KYUUBI_SESSION_HANDLE_KEY +import org.apache.kyuubi.session.{AbstractSession, SessionHandle, SessionManager} + +class ChatSessionImpl( + protocol: TProtocolVersion, + user: String, + password: String, + ipAddress: String, + conf: Map[String, String], + sessionManager: SessionManager) + extends AbstractSession(protocol, user, password, ipAddress, conf, sessionManager) { + + override val handle: SessionHandle = + conf.get(KYUUBI_SESSION_HANDLE_KEY).map(SessionHandle.fromUUID).getOrElse(SessionHandle()) + + private val chatProvider = sessionManager.asInstanceOf[ChatSessionManager].chatProvider + + override def open(): Unit = { + info(s"Starting to open chat session.") + chatProvider.open(handle.identifier.toString, Some(user)) + super.open() + info(s"The chat session is started.") + } + + override def getInfo(infoType: TGetInfoType): TGetInfoValue = withAcquireRelease() { + infoType match { + case TGetInfoType.CLI_SERVER_NAME | TGetInfoType.CLI_DBMS_NAME => + TGetInfoValue.stringValue("Kyuubi Chat Engine") + case TGetInfoType.CLI_DBMS_VER => + TGetInfoValue.stringValue(KYUUBI_VERSION) + case TGetInfoType.CLI_ODBC_KEYWORDS => TGetInfoValue.stringValue("Unimplemented") + case TGetInfoType.CLI_MAX_COLUMN_NAME_LEN => + TGetInfoValue.lenValue(128) + case TGetInfoType.CLI_MAX_SCHEMA_NAME_LEN => + TGetInfoValue.lenValue(128) + case TGetInfoType.CLI_MAX_TABLE_NAME_LEN => + TGetInfoValue.lenValue(128) + case _ => throw KyuubiSQLException(s"Unrecognized GetInfoType value: $infoType") + } + } + + override def close(): Unit = { + chatProvider.close(handle.identifier.toString) + super.close() + } + +} diff --git a/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/session/ChatSessionManager.scala b/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/session/ChatSessionManager.scala new file mode 100644 index 000000000..33a9dd450 --- /dev/null +++ b/externals/kyuubi-chat-engine/src/main/scala/org/apache/kyuubi/engine/chat/session/ChatSessionManager.scala @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.engine.chat.session + +import org.apache.hive.service.rpc.thrift.TProtocolVersion + +import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.config.KyuubiConf.ENGINE_SHARE_LEVEL +import org.apache.kyuubi.config.KyuubiReservedKeys.KYUUBI_SESSION_HANDLE_KEY +import org.apache.kyuubi.engine.ShareLevel +import org.apache.kyuubi.engine.chat.ChatEngine +import org.apache.kyuubi.engine.chat.operation.ChatOperationManager +import org.apache.kyuubi.engine.chat.provider.ChatProvider +import org.apache.kyuubi.operation.OperationManager +import org.apache.kyuubi.session.{Session, SessionHandle, SessionManager} + +class ChatSessionManager(name: String) + extends SessionManager(name) { + + def this() = this(classOf[ChatSessionManager].getSimpleName) + + override protected def isServer: Boolean = false + + lazy val chatProvider: ChatProvider = ChatProvider.load(conf) + + override lazy val operationManager: OperationManager = + new ChatOperationManager(conf, chatProvider) + + override def initialize(conf: KyuubiConf): Unit = { + this.conf = conf + super.initialize(conf) + } + + override protected def createSession( + protocol: TProtocolVersion, + user: String, + password: String, + ipAddress: String, + conf: Map[String, String]): Session = { + conf.get(KYUUBI_SESSION_HANDLE_KEY).map(SessionHandle.fromUUID) + .flatMap(getSessionOption).getOrElse { + new ChatSessionImpl(protocol, user, password, ipAddress, conf, this) + } + } + + override def closeSession(sessionHandle: SessionHandle): Unit = { + super.closeSession(sessionHandle) + if (conf.get(ENGINE_SHARE_LEVEL) == ShareLevel.CONNECTION.toString) { + info("Session stopped due to shared level is Connection.") + stopSession() + } + } + + private def stopSession(): Unit = { + ChatEngine.currentEngine.foreach(_.stop()) + } +} diff --git a/externals/kyuubi-chat-engine/src/test/resources/log4j2-test.xml b/externals/kyuubi-chat-engine/src/test/resources/log4j2-test.xml new file mode 100644 index 000000000..356d64590 --- /dev/null +++ b/externals/kyuubi-chat-engine/src/test/resources/log4j2-test.xml @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/externals/kyuubi-chat-engine/src/test/scala/org/apache/kyuubi/engine/chat/WithChatEngine.scala b/externals/kyuubi-chat-engine/src/test/scala/org/apache/kyuubi/engine/chat/WithChatEngine.scala new file mode 100644 index 000000000..287fdde2f --- /dev/null +++ b/externals/kyuubi-chat-engine/src/test/scala/org/apache/kyuubi/engine/chat/WithChatEngine.scala @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.kyuubi.engine.chat + +import org.apache.kyuubi.KyuubiFunSuite +import org.apache.kyuubi.config.KyuubiConf + +trait WithChatEngine extends KyuubiFunSuite { + + protected var engine: ChatEngine = _ + protected var connectionUrl: String = _ + + protected val kyuubiConf: KyuubiConf = ChatEngine.kyuubiConf + + def withKyuubiConf: Map[String, String] + + override def beforeAll(): Unit = { + super.beforeAll() + startChatEngine() + } + + override def afterAll(): Unit = { + stopChatEngine() + super.afterAll() + } + + def stopChatEngine(): Unit = { + if (engine != null) { + engine.stop() + engine = null + } + } + + def startChatEngine(): Unit = { + withKyuubiConf.foreach { case (k, v) => + System.setProperty(k, v) + kyuubiConf.set(k, v) + } + ChatEngine.startEngine() + engine = ChatEngine.currentEngine.get + connectionUrl = engine.frontendServices.head.connectionUrl + } + + protected def jdbcConnectionUrl: String = s"jdbc:hive2://$connectionUrl/;" + +} diff --git a/externals/kyuubi-chat-engine/src/test/scala/org/apache/kyuubi/engine/chat/operation/ChatOperationSuite.scala b/externals/kyuubi-chat-engine/src/test/scala/org/apache/kyuubi/engine/chat/operation/ChatOperationSuite.scala new file mode 100644 index 000000000..b14407a26 --- /dev/null +++ b/externals/kyuubi-chat-engine/src/test/scala/org/apache/kyuubi/engine/chat/operation/ChatOperationSuite.scala @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.engine.chat.operation + +import org.apache.kyuubi.config.KyuubiConf._ +import org.apache.kyuubi.engine.chat.WithChatEngine +import org.apache.kyuubi.operation.HiveJDBCTestHelper + +class ChatOperationSuite extends HiveJDBCTestHelper with WithChatEngine { + + override def withKyuubiConf: Map[String, String] = Map( + ENGINE_CHAT_PROVIDER.key -> "echo") + + override protected def jdbcUrl: String = jdbcConnectionUrl + + test("test echo chat provider") { + withJdbcStatement() { stmt => + val result = stmt.executeQuery("Hello, Kyuubi") + assert(result.next()) + val expected = "This is ChatKyuubi, nice to meet you!" + assert(result.getString("reply") === expected) + assert(!result.next()) + } + } +} diff --git a/externals/kyuubi-download/pom.xml b/externals/kyuubi-download/pom.xml index b0479f7ed..b21e3e5a2 100644 --- a/externals/kyuubi-download/pom.xml +++ b/externals/kyuubi-download/pom.xml @@ -21,7 +21,7 @@ org.apache.kyuubi kyuubi-parent - 1.7.0-SNAPSHOT + 1.9.0-SNAPSHOT ../../pom.xml @@ -36,6 +36,7 @@ com.googlecode.maven-download-plugin download-maven-plugin + ${maven.plugin.download.cache.path} ${project.build.directory} 60000 3 diff --git a/externals/kyuubi-flink-sql-engine/pom.xml b/externals/kyuubi-flink-sql-engine/pom.xml index c93993607..eec5c1cd9 100644 --- a/externals/kyuubi-flink-sql-engine/pom.xml +++ b/externals/kyuubi-flink-sql-engine/pom.xml @@ -21,11 +21,11 @@ org.apache.kyuubi kyuubi-parent - 1.7.0-SNAPSHOT + 1.9.0-SNAPSHOT ../../pom.xml - kyuubi-flink-sql-engine_2.12 + kyuubi-flink-sql-engine_${scala.binary.version} jar Kyuubi Project Engine Flink SQL https://kyuubi.apache.org/ @@ -59,55 +59,49 @@ org.apache.flink - flink-streaming-java${flink.module.scala.suffix} + flink-streaming-java provided org.apache.flink - flink-clients${flink.module.scala.suffix} + flink-clients provided org.apache.flink - flink-sql-client${flink.module.scala.suffix} + flink-sql-client provided org.apache.flink - flink-table-common - provided - - - - org.apache.flink - flink-table-api-java + flink-sql-gateway provided org.apache.flink - flink-table-api-java-bridge${flink.module.scala.suffix} + flink-table-common provided org.apache.flink - flink-table-planner_${scala.binary.version} + flink-table-api-java provided org.apache.flink - flink-table-runtime${flink.module.scala.suffix} + flink-table-api-java-bridge provided org.apache.flink - flink-sql-parser + flink-table-runtime provided @@ -126,9 +120,47 @@ ${project.version} test + + + org.apache.kyuubi + kyuubi-zookeeper_${scala.binary.version} + ${project.version} + test + + org.apache.flink - flink-test-utils${flink.module.scala.suffix} + flink-test-utils + test + + + + org.apache.hadoop + hadoop-client-minicluster + test + + + + org.bouncycastle + bcprov-jdk15on + test + + + + org.bouncycastle + bcpkix-jdk15on + test + + + + jakarta.activation + jakarta.activation-api + test + + + + jakarta.xml.bind + jakarta.xml.bind-api test       @@ -142,20 +174,15 @@ false - org.apache.kyuubi:kyuubi-common_${scala.binary.version} - org.apache.kyuubi:kyuubi-ha_${scala.binary.version} com.fasterxml.jackson.core:* com.fasterxml.jackson.module:* com.google.guava:failureaccess com.google.guava:guava commons-codec:commons-codec org.apache.commons:commons-lang3 - org.apache.curator:curator-client - org.apache.curator:curator-framework - org.apache.curator:curator-recipes org.apache.hive:hive-service-rpc org.apache.thrift:* - org.apache.zookeeper:* + org.apache.kyuubi:* @@ -184,13 +211,6 @@ com.fasterxml.jackson.** - - org.apache.curator - ${kyuubi.shade.packageName}.org.apache.curator - - org.apache.curator.** - - com.google.common ${kyuubi.shade.packageName}.com.google.common @@ -234,20 +254,6 @@ org.apache.thrift.** - - org.apache.jute - ${kyuubi.shade.packageName}.org.apache.jute - - org.apache.jute.** - - - - org.apache.zookeeper - ${kyuubi.shade.packageName}.org.apache.zookeeper - - org.apache.zookeeper.** - - diff --git a/externals/kyuubi-flink-sql-engine/src/main/java/org/apache/flink/client/deployment/application/executors/EmbeddedExecutorFactory.java b/externals/kyuubi-flink-sql-engine/src/main/java/org/apache/flink/client/deployment/application/executors/EmbeddedExecutorFactory.java new file mode 100644 index 000000000..558db74a3 --- /dev/null +++ b/externals/kyuubi-flink-sql-engine/src/main/java/org/apache/flink/client/deployment/application/executors/EmbeddedExecutorFactory.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.client.deployment.application.executors; + +import static org.apache.flink.util.Preconditions.checkNotNull; +import static org.apache.flink.util.Preconditions.checkState; + +import java.util.Collection; +import java.util.concurrent.ConcurrentLinkedQueue; +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.JobID; +import org.apache.flink.api.common.time.Time; +import org.apache.flink.client.cli.ClientOptions; +import org.apache.flink.client.deployment.application.EmbeddedJobClient; +import org.apache.flink.configuration.Configuration; +import org.apache.flink.configuration.DeploymentOptions; +import org.apache.flink.core.execution.PipelineExecutor; +import org.apache.flink.core.execution.PipelineExecutorFactory; +import org.apache.flink.runtime.dispatcher.DispatcherGateway; +import org.apache.flink.util.concurrent.ScheduledExecutor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Copied from Apache Flink to exposed the DispatcherGateway for Kyuubi statements. */ +@Internal +public class EmbeddedExecutorFactory implements PipelineExecutorFactory { + + private static Collection bootstrapJobIds; + + private static Collection submittedJobIds; + + private static DispatcherGateway dispatcherGateway; + + private static ScheduledExecutor retryExecutor; + + private static final Object bootstrapLock = new Object(); + + private static final long BOOTSTRAP_WAIT_INTERVAL = 10_000L; + + private static final int BOOTSTRAP_WAIT_RETRIES = 3; + + private static final Logger LOGGER = LoggerFactory.getLogger(EmbeddedExecutorFactory.class); + + public EmbeddedExecutorFactory() { + LOGGER.debug( + "{} loaded in thread {} with classloader {}.", + this.getClass().getCanonicalName(), + Thread.currentThread().getName(), + this.getClass().getClassLoader().toString()); + } + + /** + * Creates an {@link EmbeddedExecutorFactory}. + * + * @param submittedJobIds a list that is going to be filled with the job ids of the new jobs that + * will be submitted. This is essentially used to return the submitted job ids to the caller. + * @param dispatcherGateway the dispatcher of the cluster which is going to be used to submit + * jobs. + */ + public EmbeddedExecutorFactory( + final Collection submittedJobIds, + final DispatcherGateway dispatcherGateway, + final ScheduledExecutor retryExecutor) { + // there should be only one instance of EmbeddedExecutorFactory + LOGGER.debug( + "{} initiated in thread {} with classloader {}.", + this.getClass().getCanonicalName(), + Thread.currentThread().getName(), + this.getClass().getClassLoader().toString()); + checkState(EmbeddedExecutorFactory.submittedJobIds == null); + checkState(EmbeddedExecutorFactory.dispatcherGateway == null); + checkState(EmbeddedExecutorFactory.retryExecutor == null); + synchronized (bootstrapLock) { + // submittedJobIds would be always 1, because we create a new list to avoid concurrent access + // issues + LOGGER.debug("Bootstrapping EmbeddedExecutorFactory."); + EmbeddedExecutorFactory.submittedJobIds = + new ConcurrentLinkedQueue<>(checkNotNull(submittedJobIds)); + EmbeddedExecutorFactory.bootstrapJobIds = submittedJobIds; + EmbeddedExecutorFactory.dispatcherGateway = checkNotNull(dispatcherGateway); + EmbeddedExecutorFactory.retryExecutor = checkNotNull(retryExecutor); + bootstrapLock.notifyAll(); + } + } + + @Override + public String getName() { + return EmbeddedExecutor.NAME; + } + + @Override + public boolean isCompatibleWith(final Configuration configuration) { + // override Flink's implementation to allow usage in Kyuubi + LOGGER.debug("Matching execution target: {}", configuration.get(DeploymentOptions.TARGET)); + return configuration.get(DeploymentOptions.TARGET).equalsIgnoreCase("yarn-application") + && configuration.toMap().getOrDefault("yarn.tags", "").toLowerCase().contains("kyuubi"); + } + + @Override + public PipelineExecutor getExecutor(final Configuration configuration) { + checkNotNull(configuration); + Collection executorJobIDs; + synchronized (bootstrapLock) { + // wait in a loop to avoid spurious wakeups + int retry = 0; + while (bootstrapJobIds == null && retry < BOOTSTRAP_WAIT_RETRIES) { + try { + LOGGER.debug("Waiting for bootstrap to complete. Wait retries: {}.", retry); + bootstrapLock.wait(BOOTSTRAP_WAIT_INTERVAL); + retry++; + } catch (InterruptedException e) { + throw new RuntimeException("Interrupted while waiting for bootstrap.", e); + } + } + if (bootstrapJobIds == null) { + throw new RuntimeException( + "Bootstrap of Flink SQL engine timed out after " + + BOOTSTRAP_WAIT_INTERVAL * BOOTSTRAP_WAIT_RETRIES + + " ms. Please check the engine log for more details."); + } + } + if (bootstrapJobIds.size() > 0) { + LOGGER.info("Submitting new Kyuubi job. Job submitted: {}.", submittedJobIds.size()); + executorJobIDs = submittedJobIds; + } else { + LOGGER.info("Bootstrapping Flink SQL engine with the initial SQL."); + executorJobIDs = bootstrapJobIds; + } + return new EmbeddedExecutor( + executorJobIDs, + dispatcherGateway, + (jobId, userCodeClassloader) -> { + final Time timeout = + Time.milliseconds(configuration.get(ClientOptions.CLIENT_TIMEOUT).toMillis()); + return new EmbeddedJobClient( + jobId, dispatcherGateway, retryExecutor, timeout, userCodeClassloader); + }); + } +} diff --git a/externals/kyuubi-flink-sql-engine/src/main/java/org/apache/kyuubi/engine/flink/result/Constants.java b/externals/kyuubi-flink-sql-engine/src/main/java/org/apache/kyuubi/engine/flink/result/Constants.java deleted file mode 100644 index b683eb76a..000000000 --- a/externals/kyuubi-flink-sql-engine/src/main/java/org/apache/kyuubi/engine/flink/result/Constants.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kyuubi.engine.flink.result; - -/** Constant column names. */ -public class Constants { - - public static final String TABLE_TYPE = "TABLE"; - public static final String VIEW_TYPE = "VIEW"; - - public static final String[] SUPPORTED_TABLE_TYPES = new String[] {TABLE_TYPE, VIEW_TYPE}; -} diff --git a/externals/kyuubi-flink-sql-engine/src/main/java/org/apache/kyuubi/engine/flink/result/ResultSet.java b/externals/kyuubi-flink-sql-engine/src/main/java/org/apache/kyuubi/engine/flink/result/ResultSet.java deleted file mode 100644 index 66f03a159..000000000 --- a/externals/kyuubi-flink-sql-engine/src/main/java/org/apache/kyuubi/engine/flink/result/ResultSet.java +++ /dev/null @@ -1,178 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kyuubi.engine.flink.result; - -import com.google.common.collect.Iterators; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Iterator; -import java.util.List; -import java.util.Objects; -import javax.annotation.Nullable; -import org.apache.flink.table.api.ResultKind; -import org.apache.flink.table.api.TableResult; -import org.apache.flink.table.catalog.Column; -import org.apache.flink.table.catalog.ResolvedSchema; -import org.apache.flink.types.Row; -import org.apache.flink.util.Preconditions; -import org.apache.kyuubi.operation.ArrayFetchIterator; -import org.apache.kyuubi.operation.FetchIterator; - -/** - * A set of one statement execution result containing result kind, columns, rows of data and change - * flags for streaming mode. - */ -public class ResultSet { - - private final ResultKind resultKind; - private final List columns; - private final FetchIterator data; - - // null in batch mode - // - // list of boolean in streaming mode, - // true if the corresponding row is an append row, false if its a retract row - private final List changeFlags; - - private ResultSet( - ResultKind resultKind, - List columns, - FetchIterator data, - @Nullable List changeFlags) { - this.resultKind = Preconditions.checkNotNull(resultKind, "resultKind must not be null"); - this.columns = Preconditions.checkNotNull(columns, "columns must not be null"); - this.data = Preconditions.checkNotNull(data, "data must not be null"); - this.changeFlags = changeFlags; - if (changeFlags != null) { - Preconditions.checkArgument( - Iterators.size((Iterator) data) == changeFlags.size(), - "the size of data and the size of changeFlags should be equal"); - } - } - - public List getColumns() { - return columns; - } - - public FetchIterator getData() { - return data; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - ResultSet resultSet = (ResultSet) o; - return resultKind.equals(resultSet.resultKind) - && columns.equals(resultSet.columns) - && data.equals(resultSet.data) - && Objects.equals(changeFlags, resultSet.changeFlags); - } - - @Override - public int hashCode() { - return Objects.hash(resultKind, columns, data, changeFlags); - } - - @Override - public String toString() { - return "ResultSet{" - + "resultKind=" - + resultKind - + ", columns=" - + columns - + ", data=" - + data - + ", changeFlags=" - + changeFlags - + '}'; - } - - public static ResultSet fromTableResult(TableResult tableResult) { - ResolvedSchema schema = tableResult.getResolvedSchema(); - // collect all rows from table result as list - // this is ok as TableResult contains limited rows - List rows = new ArrayList<>(); - tableResult.collect().forEachRemaining(rows::add); - return builder() - .resultKind(tableResult.getResultKind()) - .columns(schema.getColumns()) - .data(rows.toArray(new Row[0])) - .build(); - } - - public static Builder builder() { - return new Builder(); - } - - /** Builder for {@link ResultSet}. */ - public static class Builder { - private ResultKind resultKind = null; - private List columns = null; - private FetchIterator data = null; - private List changeFlags = null; - - private Builder() {} - - /** Set {@link ResultKind}. */ - public Builder resultKind(ResultKind resultKind) { - this.resultKind = resultKind; - return this; - } - - /** Set columns. */ - public Builder columns(Column... columns) { - this.columns = Arrays.asList(columns); - return this; - } - - /** Set columns. */ - public Builder columns(List columns) { - this.columns = columns; - return this; - } - - /** Set data. */ - public Builder data(FetchIterator data) { - this.data = data; - return this; - } - - /** Set data. */ - public Builder data(Row[] data) { - this.data = new ArrayFetchIterator<>(data); - return this; - } - - /** Set change flags. */ - public Builder changeFlags(List changeFlags) { - this.changeFlags = changeFlags; - return this; - } - - /** Returns a {@link ResultSet} instance. */ - public ResultSet build() { - return new ResultSet(resultKind, columns, data, changeFlags); - } - } -} diff --git a/externals/kyuubi-flink-sql-engine/src/main/resources/META-INF/services/org.apache.flink.core.execution.PipelineExecutorFactory b/externals/kyuubi-flink-sql-engine/src/main/resources/META-INF/services/org.apache.flink.core.execution.PipelineExecutorFactory new file mode 100644 index 000000000..c394c07a7 --- /dev/null +++ b/externals/kyuubi-flink-sql-engine/src/main/resources/META-INF/services/org.apache.flink.core.execution.PipelineExecutorFactory @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.flink.client.deployment.application.executors.EmbeddedExecutorFactory \ No newline at end of file diff --git a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/FlinkEngineUtils.scala b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/FlinkEngineUtils.scala index e271944a7..06165272d 100644 --- a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/FlinkEngineUtils.scala +++ b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/FlinkEngineUtils.scala @@ -18,33 +18,43 @@ package org.apache.kyuubi.engine.flink import java.io.File +import java.lang.{Boolean => JBoolean} import java.net.URL +import java.util.{ArrayList => JArrayList, Collections => JCollections, List => JList} import scala.collection.JavaConverters._ +import scala.collection.convert.ImplicitConversions._ -import org.apache.commons.cli.{CommandLine, DefaultParser, Option, Options, ParseException} +import org.apache.commons.cli.{CommandLine, DefaultParser, Options} +import org.apache.flink.api.common.JobID +import org.apache.flink.client.cli.{CustomCommandLine, DefaultCLI, GenericCLI} +import org.apache.flink.configuration.Configuration import org.apache.flink.core.fs.Path import org.apache.flink.runtime.util.EnvironmentInformation import org.apache.flink.table.client.SqlClientException -import org.apache.flink.table.client.cli.CliOptions +import org.apache.flink.table.client.cli.CliOptionsParser import org.apache.flink.table.client.cli.CliOptionsParser._ -import org.apache.flink.table.client.gateway.context.SessionContext -import org.apache.flink.table.client.gateway.local.LocalExecutor +import org.apache.flink.table.gateway.service.context.{DefaultContext, SessionContext} +import org.apache.flink.table.gateway.service.result.ResultFetcher +import org.apache.flink.table.gateway.service.session.Session +import org.apache.flink.util.JarUtils -import org.apache.kyuubi.Logging -import org.apache.kyuubi.engine.SemanticVersion +import org.apache.kyuubi.{KyuubiException, Logging} +import org.apache.kyuubi.util.SemanticVersion +import org.apache.kyuubi.util.reflect._ +import org.apache.kyuubi.util.reflect.ReflectUtils._ object FlinkEngineUtils extends Logging { - val MODE_EMBEDDED = "embedded" - val EMBEDDED_MODE_CLIENT_OPTIONS: Options = getEmbeddedModeClientOptions(new Options); + val EMBEDDED_MODE_CLIENT_OPTIONS: Options = getEmbeddedModeClientOptions(new Options) - val SUPPORTED_FLINK_VERSIONS: Array[SemanticVersion] = - Array("1.14", "1.15", "1.16").map(SemanticVersion.apply) + private def SUPPORTED_FLINK_VERSIONS = Set("1.16", "1.17", "1.18").map(SemanticVersion.apply) + + val FLINK_RUNTIME_VERSION: SemanticVersion = SemanticVersion(EnvironmentInformation.getVersion) def checkFlinkVersion(): Unit = { val flinkVersion = EnvironmentInformation.getVersion - if (SUPPORTED_FLINK_VERSIONS.contains(SemanticVersion(flinkVersion))) { + if (SUPPORTED_FLINK_VERSIONS.contains(FLINK_RUNTIME_VERSION)) { info(s"The current Flink version is $flinkVersion") } else { throw new UnsupportedOperationException( @@ -53,56 +63,90 @@ object FlinkEngineUtils extends Logging { } } - def isFlinkVersionAtMost(targetVersionString: String): Boolean = - SemanticVersion(EnvironmentInformation.getVersion).isVersionAtMost(targetVersionString) - - def isFlinkVersionAtLeast(targetVersionString: String): Boolean = - SemanticVersion(EnvironmentInformation.getVersion).isVersionAtLeast(targetVersionString) - - def isFlinkVersionEqualTo(targetVersionString: String): Boolean = - SemanticVersion(EnvironmentInformation.getVersion).isVersionEqualTo(targetVersionString) - - def parseCliOptions(args: Array[String]): CliOptions = { - val (mode, modeArgs) = - if (args.isEmpty || args(0).startsWith("-")) (MODE_EMBEDDED, args) - else (args(0), args.drop(1)) - val options = parseEmbeddedModeClient(modeArgs) - if (mode == MODE_EMBEDDED) { - if (options.isPrintHelp) { - printHelpEmbeddedModeClient() + /** + * Copied and modified from [[org.apache.flink.table.client.cli.CliOptionsParser]] + * to avoid loading flink-python classes which we doesn't support yet. + */ + private def discoverDependencies( + jars: JList[URL], + libraries: JList[URL]): JList[URL] = { + val dependencies: JList[URL] = new JArrayList[URL] + try { // find jar files + for (url <- jars) { + JarUtils.checkJarFile(url) + dependencies.add(url) } - options - } else { - throw new SqlClientException("Other mode is not supported yet.") + // find jar files in library directories + libraries.foreach { libUrl => + val dir: File = new File(libUrl.toURI) + if (!dir.isDirectory) throw new SqlClientException(s"Directory expected: $dir") + if (!dir.canRead) throw new SqlClientException(s"Directory cannot be read: $dir") + val files: Array[File] = dir.listFiles + if (files == null) throw new SqlClientException(s"Directory cannot be read: $dir") + files.filter { f => f.isFile && f.getAbsolutePath.toLowerCase.endsWith(".jar") } + .foreach { f => + val url: URL = f.toURI.toURL + JarUtils.checkJarFile(url) + dependencies.add(url) + } + } + } catch { + case e: Exception => + throw new SqlClientException("Could not load all required JAR files.", e) } + dependencies } - def getSessionContext(localExecutor: LocalExecutor, sessionId: String): SessionContext = { - val method = classOf[LocalExecutor].getDeclaredMethod("getSessionContext", classOf[String]) - method.setAccessible(true) - method.invoke(localExecutor, sessionId).asInstanceOf[SessionContext] + def getDefaultContext( + args: Array[String], + flinkConf: Configuration, + flinkConfDir: String): DefaultContext = { + val parser = new DefaultParser + val line = parser.parse(EMBEDDED_MODE_CLIENT_OPTIONS, args, true) + val jars: JList[URL] = Option(checkUrls(line, CliOptionsParser.OPTION_JAR)) + .getOrElse(JCollections.emptyList()) + val libDirs: JList[URL] = Option(checkUrls(line, CliOptionsParser.OPTION_LIBRARY)) + .getOrElse(JCollections.emptyList()) + val dependencies: JList[URL] = discoverDependencies(jars, libDirs) + if (FLINK_RUNTIME_VERSION === "1.16") { + val commandLines: JList[CustomCommandLine] = + Seq(new GenericCLI(flinkConf, flinkConfDir), new DefaultCLI).asJava + DynConstructors.builder() + .impl( + classOf[DefaultContext], + classOf[Configuration], + classOf[JList[CustomCommandLine]]) + .build() + .newInstance(flinkConf, commandLines) + .asInstanceOf[DefaultContext] + } else if (FLINK_RUNTIME_VERSION >= "1.17") { + invokeAs[DefaultContext]( + classOf[DefaultContext], + "load", + (classOf[Configuration], flinkConf), + (classOf[JList[URL]], dependencies), + (classOf[Boolean], JBoolean.TRUE), + (classOf[Boolean], JBoolean.FALSE)) + } else { + throw new KyuubiException( + s"Flink version ${EnvironmentInformation.getVersion} are not supported currently.") + } } - def parseEmbeddedModeClient(args: Array[String]): CliOptions = + def getSessionContext(session: Session): SessionContext = getField(session, "sessionContext") + + def getResultJobId(resultFetch: ResultFetcher): Option[JobID] = { + if (FLINK_RUNTIME_VERSION <= "1.16") { + return None + } try { - val parser = new DefaultParser - val line = parser.parse(EMBEDDED_MODE_CLIENT_OPTIONS, args, true) - val jarUrls = checkUrls(line, OPTION_JAR) - val libraryUrls = checkUrls(line, OPTION_LIBRARY) - new CliOptions( - line.hasOption(OPTION_HELP.getOpt), - checkSessionId(line), - checkUrl(line, OPTION_INIT_FILE), - checkUrl(line, OPTION_FILE), - if (jarUrls != null && jarUrls.nonEmpty) jarUrls.asJava else null, - if (libraryUrls != null && libraryUrls.nonEmpty) libraryUrls.asJava else null, - line.getOptionValue(OPTION_UPDATE.getOpt), - line.getOptionValue(OPTION_HISTORY.getOpt), - null) + Option(getField[JobID](resultFetch, "jobID")) } catch { - case e: ParseException => - throw new SqlClientException(e.getMessage) + case _: NullPointerException => None + case e: Throwable => + throw new IllegalStateException("Unexpected error occurred while fetching query ID", e) } + } def checkSessionId(line: CommandLine): String = { val sessionId = line.getOptionValue(OPTION_SESSION.getOpt) @@ -111,13 +155,13 @@ object FlinkEngineUtils extends Logging { } else sessionId } - def checkUrl(line: CommandLine, option: Option): URL = { - val urls: List[URL] = checkUrls(line, option) + def checkUrl(line: CommandLine, option: org.apache.commons.cli.Option): URL = { + val urls: JList[URL] = checkUrls(line, option) if (urls != null && urls.nonEmpty) urls.head else null } - def checkUrls(line: CommandLine, option: Option): List[URL] = { + def checkUrls(line: CommandLine, option: org.apache.commons.cli.Option): JList[URL] = { if (line.hasOption(option.getOpt)) { line.getOptionValues(option.getOpt).distinct.map((url: String) => { checkFilePath(url) diff --git a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/FlinkSQLBackendService.scala b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/FlinkSQLBackendService.scala index d049e3c80..9802f1955 100644 --- a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/FlinkSQLBackendService.scala +++ b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/FlinkSQLBackendService.scala @@ -17,7 +17,7 @@ package org.apache.kyuubi.engine.flink -import org.apache.flink.table.client.gateway.context.DefaultContext +import org.apache.flink.table.gateway.service.context.DefaultContext import org.apache.kyuubi.engine.flink.session.FlinkSQLSessionManager import org.apache.kyuubi.service.AbstractBackendService diff --git a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/FlinkSQLEngine.scala b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/FlinkSQLEngine.scala index 06fdc65ae..8838799bc 100644 --- a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/FlinkSQLEngine.scala +++ b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/FlinkSQLEngine.scala @@ -18,23 +18,21 @@ package org.apache.kyuubi.engine.flink import java.io.File -import java.net.URL import java.nio.file.Paths -import java.time.Instant +import java.time.Duration import java.util.concurrent.CountDownLatch import scala.collection.JavaConverters._ -import scala.collection.mutable.ListBuffer -import org.apache.flink.client.cli.{DefaultCLI, GenericCLI} -import org.apache.flink.configuration.{Configuration, DeploymentOptions, GlobalConfiguration} -import org.apache.flink.table.client.SqlClientException -import org.apache.flink.table.client.gateway.context.DefaultContext -import org.apache.flink.util.JarUtils +import org.apache.flink.configuration.{Configuration, DeploymentOptions, GlobalConfiguration, PipelineOptions} +import org.apache.flink.table.api.TableEnvironment +import org.apache.flink.table.gateway.api.config.SqlGatewayServiceConfigOptions +import org.apache.flink.table.gateway.service.context.DefaultContext -import org.apache.kyuubi.{KyuubiSQLException, Logging, Utils} +import org.apache.kyuubi.{Logging, Utils} import org.apache.kyuubi.Utils.{addShutdownHook, currentUser, FLINK_ENGINE_SHUTDOWN_PRIORITY} import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.config.KyuubiReservedKeys.{KYUUBI_ENGINE_NAME, KYUUBI_SESSION_USER_KEY} import org.apache.kyuubi.engine.flink.FlinkSQLEngine.{countDownLatch, currentEngine} import org.apache.kyuubi.service.Serverable import org.apache.kyuubi.util.SignalRegister @@ -71,9 +69,12 @@ object FlinkSQLEngine extends Logging { def main(args: Array[String]): Unit = { SignalRegister.registerLogger(logger) + info(s"Flink SQL engine classpath: ${System.getProperty("java.class.path")}") + FlinkEngineUtils.checkFlinkVersion() try { + kyuubiConf.loadFileDefaults() Utils.fromCommandLineArgs(args, kyuubiConf) val flinkConfDir = sys.env.getOrElse( "FLINK_CONF_DIR", { @@ -93,51 +94,33 @@ object FlinkSQLEngine extends Logging { flinkConf.addAll(Configuration.fromMap(flinkConfFromArgs.asJava)) val executionTarget = flinkConf.getString(DeploymentOptions.TARGET) - // set cluster name for per-job and application mode - executionTarget match { - case "yarn-per-job" | "yarn-application" => - if (!flinkConf.containsKey("yarn.application.name")) { - val appName = s"kyuubi_${user}_flink_${Instant.now}" - flinkConf.setString("yarn.application.name", appName) - } - case "kubernetes-application" => - if (!flinkConf.containsKey("kubernetes.cluster-id")) { - val appName = s"kyuubi-${user}-flink-${Instant.now}" - flinkConf.setString("kubernetes.cluster-id", appName) - } - case other => - debug(s"Skip generating app name for execution target $other") - } - - val cliOptions = FlinkEngineUtils.parseCliOptions(args) - val jars = if (cliOptions.getJars != null) cliOptions.getJars.asScala else List.empty - val libDirs = - if (cliOptions.getLibraryDirs != null) cliOptions.getLibraryDirs.asScala else List.empty - val dependencies = discoverDependencies(jars, libDirs) - val engineContext = new DefaultContext( - dependencies.asJava, - flinkConf, - Seq(new GenericCLI(flinkConf, flinkConfDir), new DefaultCLI).asJava) + setDeploymentConf(executionTarget, flinkConf) kyuubiConf.setIfMissing(KyuubiConf.FRONTEND_THRIFT_BINARY_BIND_PORT, 0) + val engineContext = FlinkEngineUtils.getDefaultContext(args, flinkConf, flinkConfDir) startEngine(engineContext) - info("started engine...") + info("Flink engine started") + + if ("yarn-application".equalsIgnoreCase(executionTarget)) { + bootstrapFlinkApplicationExecutor() + } // blocking main thread countDownLatch.await() } catch { case t: Throwable if currentEngine.isDefined => + error("Fatal error occurs, thus stopping the engines", t) currentEngine.foreach { engine => - error(t) engine.stop() } case t: Throwable => - error("Create FlinkSQL Engine Failed", t) + error("Failed to create FlinkSQL Engine", t) } } def startEngine(engineContext: DefaultContext): Unit = { + debug(s"Starting Flink SQL engine with default configuration: ${engineContext.getFlinkConfig}") currentEngine = Some(new FlinkSQLEngine(engineContext)) currentEngine.foreach { engine => engine.initialize(kyuubiConf) @@ -146,36 +129,39 @@ object FlinkSQLEngine extends Logging { } } - private def discoverDependencies( - jars: Seq[URL], - libraries: Seq[URL]): List[URL] = { - try { - var dependencies: ListBuffer[URL] = ListBuffer() - // find jar files - jars.foreach { url => - JarUtils.checkJarFile(url) - dependencies = dependencies += url - } - // find jar files in library directories - libraries.foreach { libUrl => - val dir: File = new File(libUrl.toURI) - if (!dir.isDirectory) throw new SqlClientException("Directory expected: " + dir) - else if (!dir.canRead) throw new SqlClientException("Directory cannot be read: " + dir) - val files: Array[File] = dir.listFiles - if (files == null) throw new SqlClientException("Directory cannot be read: " + dir) - files.foreach { f => - // only consider jars - if (f.isFile && f.getAbsolutePath.toLowerCase.endsWith(".jar")) { - val url: URL = f.toURI.toURL - JarUtils.checkJarFile(url) - dependencies = dependencies += url - } + private def bootstrapFlinkApplicationExecutor() = { + // trigger an execution to initiate EmbeddedExecutor with the default flink conf + val flinkConf = new Configuration() + flinkConf.set(PipelineOptions.NAME, "kyuubi-bootstrap-sql") + debug(s"Running bootstrap Flink SQL in application mode with flink conf: $flinkConf.") + val tableEnv = TableEnvironment.create(flinkConf) + val res = tableEnv.executeSql("select 'kyuubi'") + res.await() + info("Bootstrap Flink SQL finished.") + } + + private def setDeploymentConf(executionTarget: String, flinkConf: Configuration): Unit = { + // forward kyuubi engine variables to flink configuration + kyuubiConf.getOption("flink.app.name") + .foreach(flinkConf.setString(KYUUBI_ENGINE_NAME, _)) + + kyuubiConf.getOption(KYUUBI_SESSION_USER_KEY) + .foreach(flinkConf.setString(KYUUBI_SESSION_USER_KEY, _)) + + // force disable Flink's session timeout + flinkConf.set( + SqlGatewayServiceConfigOptions.SQL_GATEWAY_SESSION_IDLE_TIMEOUT, + Duration.ofMillis(0)) + + executionTarget match { + case "yarn-per-job" | "yarn-application" => + if (flinkConf.containsKey("high-availability.cluster-id")) { + flinkConf.setString( + "yarn.application.id", + flinkConf.toMap.get("high-availability.cluster-id")) } - } - dependencies.toList - } catch { - case e: Exception => - throw KyuubiSQLException(s"Could not load all required JAR files.", e) + case other => + debug(s"Skip setting deployment conf for execution target $other") } } } diff --git a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/ExecuteStatement.scala b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/ExecuteStatement.scala index 93d013556..0e0c476e2 100644 --- a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/ExecuteStatement.scala +++ b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/ExecuteStatement.scala @@ -17,39 +17,25 @@ package org.apache.kyuubi.engine.flink.operation -import java.time.LocalDate -import java.util - -import scala.collection.JavaConverters._ -import scala.collection.mutable.ArrayBuffer +import scala.concurrent.duration.Duration import org.apache.flink.api.common.JobID -import org.apache.flink.table.api.{ResultKind, TableResult} -import org.apache.flink.table.client.gateway.TypedResult -import org.apache.flink.table.data.{GenericArrayData, GenericMapData, RowData} -import org.apache.flink.table.data.binary.{BinaryArrayData, BinaryMapData} -import org.apache.flink.table.operations.{Operation, QueryOperation} -import org.apache.flink.table.operations.command._ -import org.apache.flink.table.types.DataType -import org.apache.flink.table.types.logical._ -import org.apache.flink.types.Row +import org.apache.flink.table.gateway.api.operation.OperationHandle import org.apache.kyuubi.Logging -import org.apache.kyuubi.engine.flink.FlinkEngineUtils._ -import org.apache.kyuubi.engine.flink.result.ResultSet -import org.apache.kyuubi.engine.flink.schema.RowSet.toHiveString +import org.apache.kyuubi.engine.flink.FlinkEngineUtils +import org.apache.kyuubi.engine.flink.result.ResultSetUtil import org.apache.kyuubi.operation.OperationState import org.apache.kyuubi.operation.log.OperationLog -import org.apache.kyuubi.reflection.DynMethods import org.apache.kyuubi.session.Session -import org.apache.kyuubi.util.RowSetUtils class ExecuteStatement( session: Session, override val statement: String, override val shouldRunAsync: Boolean, queryTimeout: Long, - resultMaxRows: Int) + resultMaxRows: Int, + resultFetchTimeout: Duration) extends FlinkOperation(session) with Logging { private val operationLog: OperationLog = @@ -65,10 +51,6 @@ class ExecuteStatement( setHasResultSet(true) } - override protected def afterRun(): Unit = { - OperationLog.removeCurrentOperationLog() - } - override protected def runInternal(): Unit = { addTimeoutMonitor(queryTimeout) executeStatement() @@ -77,21 +59,11 @@ class ExecuteStatement( private def executeStatement(): Unit = { try { setState(OperationState.RUNNING) - val operation = executor.parseStatement(sessionId, statement) - operation match { - case queryOperation: QueryOperation => runQueryOperation(queryOperation) - case setOperation: SetOperation => - resultSet = OperationUtils.runSetOperation(setOperation, executor, sessionId) - case resetOperation: ResetOperation => - resultSet = OperationUtils.runResetOperation(resetOperation, executor, sessionId) - case addJarOperation: AddJarOperation if isFlinkVersionAtMost("1.15") => - resultSet = OperationUtils.runAddJarOperation(addJarOperation, executor, sessionId) - case removeJarOperation: RemoveJarOperation => - resultSet = OperationUtils.runRemoveJarOperation(removeJarOperation, executor, sessionId) - case showJarsOperation: ShowJarsOperation if isFlinkVersionAtMost("1.15") => - resultSet = OperationUtils.runShowJarOperation(showJarsOperation, executor, sessionId) - case operation: Operation => runOperation(operation) - } + val resultFetcher = executor.executeStatement( + new OperationHandle(getHandle.identifier), + statement) + jobId = FlinkEngineUtils.getResultJobId(resultFetcher) + resultSet = ResultSetUtil.fromResultFetcher(resultFetcher, resultMaxRows, resultFetchTimeout) setState(OperationState.FINISHED) } catch { onError(cancel = true) @@ -99,168 +71,4 @@ class ExecuteStatement( shutdownTimeoutMonitor() } } - - private def runQueryOperation(operation: QueryOperation): Unit = { - var resultId: String = null - try { - val resultDescriptor = executor.executeQuery(sessionId, operation) - val dataTypes = resultDescriptor.getResultSchema.getColumnDataTypes.asScala.toList - - resultId = resultDescriptor.getResultId - - val rows = new ArrayBuffer[Row]() - var loop = true - - while (loop) { - Thread.sleep(50) // slow the processing down - - val pageSize = Math.min(500, resultMaxRows) - val result = executor.snapshotResult(sessionId, resultId, pageSize) - result.getType match { - case TypedResult.ResultType.PAYLOAD => - (1 to result.getPayload).foreach { page => - if (rows.size < resultMaxRows) { - // FLINK-24461 retrieveResultPage method changes the return type from Row to RowData - val retrieveResultPage = DynMethods.builder("retrieveResultPage") - .impl(executor.getClass, classOf[String], classOf[Int]) - .build(executor) - val _page = Integer.valueOf(page) - if (isFlinkVersionEqualTo("1.14")) { - val result = retrieveResultPage.invoke[util.List[Row]](resultId, _page) - rows ++= result.asScala - } else if (isFlinkVersionAtLeast("1.15")) { - val result = retrieveResultPage.invoke[util.List[RowData]](resultId, _page) - rows ++= result.asScala.map(r => convertToRow(r, dataTypes)) - } - } else { - loop = false - } - } - case TypedResult.ResultType.EOS => loop = false - case TypedResult.ResultType.EMPTY => - } - } - - resultSet = ResultSet.builder - .resultKind(ResultKind.SUCCESS_WITH_CONTENT) - .columns(resultDescriptor.getResultSchema.getColumns) - .data(rows.slice(0, resultMaxRows).toArray[Row]) - .build - } finally { - if (resultId != null) { - cleanupQueryResult(resultId) - } - } - } - - private def runOperation(operation: Operation): Unit = { - // FLINK-24461 executeOperation method changes the return type - // from TableResult to TableResultInternal - val executeOperation = DynMethods.builder("executeOperation") - .impl(executor.getClass, classOf[String], classOf[Operation]) - .build(executor) - val result = executeOperation.invoke[TableResult](sessionId, operation) - jobId = result.getJobClient.asScala.map(_.getJobID) - result.await() - resultSet = ResultSet.fromTableResult(result) - } - - private def cleanupQueryResult(resultId: String): Unit = { - try { - executor.cancelQuery(sessionId, resultId) - } catch { - case t: Throwable => - warn(s"Failed to clean result set $resultId in session $sessionId", t) - } - } - - private[this] def convertToRow(r: RowData, dataTypes: List[DataType]): Row = { - val row = Row.withPositions(r.getRowKind, r.getArity) - for (i <- 0 until r.getArity) { - val dataType = dataTypes(i) - dataType.getLogicalType match { - case arrayType: ArrayType => - val arrayData = r.getArray(i) - if (arrayData == null) { - row.setField(i, null) - } - arrayData match { - case d: GenericArrayData => - row.setField(i, d.toObjectArray) - case d: BinaryArrayData => - row.setField(i, d.toObjectArray(arrayType.getElementType)) - case _ => - } - case _: BinaryType => - row.setField(i, r.getBinary(i)) - case _: BigIntType => - row.setField(i, r.getLong(i)) - case _: BooleanType => - row.setField(i, r.getBoolean(i)) - case _: VarCharType | _: CharType => - row.setField(i, r.getString(i)) - case t: DecimalType => - row.setField(i, r.getDecimal(i, t.getPrecision, t.getScale).toBigDecimal) - case _: DateType => - val date = RowSetUtils.formatLocalDate(LocalDate.ofEpochDay(r.getInt(i))) - row.setField(i, date) - case t: TimestampType => - val ts = RowSetUtils - .formatLocalDateTime(r.getTimestamp(i, t.getPrecision) - .toLocalDateTime) - row.setField(i, ts) - case _: TinyIntType => - row.setField(i, r.getByte(i)) - case _: SmallIntType => - row.setField(i, r.getShort(i)) - case _: IntType => - row.setField(i, r.getInt(i)) - case _: FloatType => - row.setField(i, r.getFloat(i)) - case mapType: MapType => - val mapData = r.getMap(i) - if (mapData != null && mapData.size > 0) { - val keyType = mapType.getKeyType - val valueType = mapType.getValueType - mapData match { - case d: BinaryMapData => - val kvArray = toArray(keyType, valueType, d) - val map: util.Map[Any, Any] = new util.HashMap[Any, Any] - for (i <- kvArray._1.indices) { - val value: Any = kvArray._2(i) - map.put(kvArray._1(i), value) - } - row.setField(i, map) - case d: GenericMapData => // TODO - } - } else { - row.setField(i, null) - } - case _: DoubleType => - row.setField(i, r.getDouble(i)) - case t: RowType => - val fieldDataTypes = DynMethods.builder("getFieldDataTypes") - .impl(classOf[DataType], classOf[DataType]) - .buildStatic - .invoke[util.List[DataType]](dataType) - .asScala.toList - val internalRowData = r.getRow(i, t.getFieldCount) - val internalRow = convertToRow(internalRowData, fieldDataTypes) - row.setField(i, internalRow) - case t => - val hiveString = toHiveString((row.getField(i), t)) - row.setField(i, hiveString) - } - } - row - } - - private[this] def toArray( - keyType: LogicalType, - valueType: LogicalType, - arrayData: BinaryMapData): (Array[_], Array[_]) = { - - arrayData.keyArray().toObjectArray(keyType) -> arrayData.valueArray().toObjectArray(valueType) - } - } diff --git a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/FlinkOperation.scala b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/FlinkOperation.scala index 2859d659e..1424b721c 100644 --- a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/FlinkOperation.scala +++ b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/FlinkOperation.scala @@ -18,12 +18,17 @@ package org.apache.kyuubi.engine.flink.operation import java.io.IOException +import java.time.ZoneId +import java.util.concurrent.TimeoutException import scala.collection.JavaConverters.collectionAsScalaIterableConverter +import scala.collection.mutable.ListBuffer -import org.apache.flink.table.client.gateway.Executor -import org.apache.flink.table.client.gateway.context.SessionContext -import org.apache.hive.service.rpc.thrift.{TGetResultSetMetadataResp, TRowSet, TTableSchema} +import org.apache.flink.configuration.Configuration +import org.apache.flink.table.gateway.service.context.SessionContext +import org.apache.flink.table.gateway.service.operation.OperationExecutor +import org.apache.flink.types.Row +import org.apache.hive.service.rpc.thrift.{TFetchResultsResp, TGetResultSetMetadataResp, TTableSchema} import org.apache.kyuubi.{KyuubiSQLException, Utils} import org.apache.kyuubi.engine.flink.result.ResultSet @@ -36,12 +41,16 @@ import org.apache.kyuubi.session.Session abstract class FlinkOperation(session: Session) extends AbstractOperation(session) { + protected val flinkSession: org.apache.flink.table.gateway.service.session.Session = + session.asInstanceOf[FlinkSessionImpl].fSession + + protected val executor: OperationExecutor = flinkSession.createExecutor( + Configuration.fromMap(flinkSession.getSessionConfig)) + protected val sessionContext: SessionContext = { session.asInstanceOf[FlinkSessionImpl].sessionContext } - protected val executor: Executor = session.asInstanceOf[FlinkSessionImpl].executor - protected val sessionId: String = session.handle.identifier.toString protected var resultSet: ResultSet = _ @@ -52,7 +61,7 @@ abstract class FlinkOperation(session: Session) extends AbstractOperation(sessio } override protected def afterRun(): Unit = { - state.synchronized { + withLockRequired { if (!isTerminalState(state)) { setState(OperationState.FINISHED) } @@ -66,6 +75,10 @@ abstract class FlinkOperation(session: Session) extends AbstractOperation(sessio override def close(): Unit = { cleanup(OperationState.CLOSED) + // the result set may be null if the operation ends exceptionally + if (resultSet != null) { + resultSet.close + } try { getOperationLog.foreach(_.close()) } catch { @@ -85,22 +98,50 @@ abstract class FlinkOperation(session: Session) extends AbstractOperation(sessio resp } - override def getNextRowSet(order: FetchOrientation, rowSetSize: Int): TRowSet = { + override def getNextRowSetInternal( + order: FetchOrientation, + rowSetSize: Int): TFetchResultsResp = { validateDefaultFetchOrientation(order) assertState(OperationState.FINISHED) setHasResultSet(true) order match { - case FETCH_NEXT => resultSet.getData.fetchNext() case FETCH_PRIOR => resultSet.getData.fetchPrior(rowSetSize); case FETCH_FIRST => resultSet.getData.fetchAbsolute(0); + case FETCH_NEXT => // ignored because new data are fetched lazily + } + val batch = new ListBuffer[Row] + try { + // there could be null values at the end of the batch + // because Flink could return an EOS + var rows = 0 + while (resultSet.getData.hasNext && rows < rowSetSize) { + Option(resultSet.getData.next()).foreach { r => batch += r; rows += 1 } + } + } catch { + case e: TimeoutException => + // ignore and return the current batch if there's some data + // otherwise, rethrow the timeout exception + if (batch.nonEmpty) { + debug(s"Timeout fetching more data for $opType operation. " + + s"Returning the current fetched data.") + } else { + throw e + } + } + val timeZone = Option(flinkSession.getSessionConfig.get("table.local-time-zone")) + val zoneId = timeZone match { + case Some(tz) => ZoneId.of(tz) + case None => ZoneId.systemDefault() } - val token = resultSet.getData.take(rowSetSize) val resultRowSet = RowSet.resultSetToTRowSet( - token.toList, + batch.toList, resultSet, + zoneId, getProtocolVersion) - resultRowSet.setStartRowOffset(resultSet.getData.getPosition) - resultRowSet + val resp = new TFetchResultsResp(OK_STATUS) + resp.setResults(resultRowSet) + resp.setHasMoreRows(resultSet.getData.hasNext) + resp } override def shouldRunAsync: Boolean = false @@ -109,7 +150,7 @@ abstract class FlinkOperation(session: Session) extends AbstractOperation(sessio // We should use Throwable instead of Exception since `java.lang.NoClassDefFoundError` // could be thrown. case e: Throwable => - state.synchronized { + withLockRequired { val errMsg = Utils.stringifyException(e) if (state == OperationState.TIMEOUT) { val ke = KyuubiSQLException(s"Timeout operating $opType: $errMsg") diff --git a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/FlinkSQLOperationManager.scala b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/FlinkSQLOperationManager.scala index d7b5e297d..d5c0629ee 100644 --- a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/FlinkSQLOperationManager.scala +++ b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/FlinkSQLOperationManager.scala @@ -20,9 +20,12 @@ package org.apache.kyuubi.engine.flink.operation import java.util import scala.collection.JavaConverters._ +import scala.concurrent.duration.{Duration, DurationLong} +import scala.language.postfixOps import org.apache.kyuubi.KyuubiSQLException import org.apache.kyuubi.config.KyuubiConf._ +import org.apache.kyuubi.engine.flink.FlinkEngineUtils import org.apache.kyuubi.engine.flink.result.Constants import org.apache.kyuubi.engine.flink.session.FlinkSessionImpl import org.apache.kyuubi.operation.{NoneMode, Operation, OperationManager, PlanOnlyMode} @@ -44,7 +47,8 @@ class FlinkSQLOperationManager extends OperationManager("FlinkSQLOperationManage runAsync: Boolean, queryTimeout: Long): Operation = { val flinkSession = session.asInstanceOf[FlinkSessionImpl] - if (flinkSession.sessionContext.getConfigMap.getOrDefault( + val sessionConfig = flinkSession.fSession.getSessionConfig + if (sessionConfig.getOrDefault( ENGINE_OPERATION_CONVERT_CATALOG_DATABASE_ENABLED.key, operationConvertCatalogDatabaseDefault.toString).toBoolean) { val catalogDatabaseOperation = processCatalogDatabase(session, statement, confOverlay) @@ -53,23 +57,42 @@ class FlinkSQLOperationManager extends OperationManager("FlinkSQLOperationManage } } - val mode = PlanOnlyMode.fromString(flinkSession.sessionContext.getConfigMap.getOrDefault( - OPERATION_PLAN_ONLY_MODE.key, - operationModeDefault)) + val mode = PlanOnlyMode.fromString( + sessionConfig.getOrDefault( + OPERATION_PLAN_ONLY_MODE.key, + operationModeDefault)) - flinkSession.sessionContext.set(OPERATION_PLAN_ONLY_MODE.key, mode.name) + val sessionContext = FlinkEngineUtils.getSessionContext(flinkSession.fSession) + sessionContext.set(OPERATION_PLAN_ONLY_MODE.key, mode.name) val resultMaxRows = flinkSession.normalizedConf.getOrElse( ENGINE_FLINK_MAX_ROWS.key, resultMaxRowsDefault.toString).toInt + + val resultFetchTimeout = + flinkSession.normalizedConf.get(ENGINE_FLINK_FETCH_TIMEOUT.key).map(_.toLong milliseconds) + .getOrElse(Duration.Inf) + val op = mode match { case NoneMode => // FLINK-24427 seals calcite classes which required to access in async mode, considering // there is no much benefit in async mode, here we just ignore `runAsync` and always run // statement in sync mode as a workaround - new ExecuteStatement(session, statement, false, queryTimeout, resultMaxRows) + new ExecuteStatement( + session, + statement, + false, + queryTimeout, + resultMaxRows, + resultFetchTimeout) case mode => - new PlanOnlyStatement(session, statement, mode) + new PlanOnlyStatement( + session, + statement, + mode, + queryTimeout, + resultMaxRows, + resultFetchTimeout) } addOperation(op) } diff --git a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/GetCatalogs.scala b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/GetCatalogs.scala index 11dd760e4..245371681 100644 --- a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/GetCatalogs.scala +++ b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/GetCatalogs.scala @@ -17,6 +17,8 @@ package org.apache.kyuubi.engine.flink.operation +import scala.collection.convert.ImplicitConversions._ + import org.apache.kyuubi.engine.flink.result.ResultSetUtil import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant.TABLE_CAT import org.apache.kyuubi.session.Session @@ -25,8 +27,8 @@ class GetCatalogs(session: Session) extends FlinkOperation(session) { override protected def runInternal(): Unit = { try { - val tableEnv = sessionContext.getExecutionContext.getTableEnvironment - val catalogs = tableEnv.listCatalogs.toList + val catalogManager = sessionContext.getSessionState.catalogManager + val catalogs = catalogManager.listCatalogs.toList resultSet = ResultSetUtil.stringListToResultSet(catalogs, TABLE_CAT) } catch onError() } diff --git a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/GetColumns.scala b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/GetColumns.scala index 6ce2a6ac7..b1a7c0c3e 100644 --- a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/GetColumns.scala +++ b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/GetColumns.scala @@ -21,7 +21,7 @@ import scala.collection.JavaConverters._ import org.apache.commons.lang3.StringUtils import org.apache.flink.table.api.{DataTypes, ResultKind} -import org.apache.flink.table.catalog.Column +import org.apache.flink.table.catalog.{Column, ObjectIdentifier} import org.apache.flink.table.types.logical._ import org.apache.flink.types.Row @@ -40,17 +40,17 @@ class GetColumns( override protected def runInternal(): Unit = { try { - val tableEnv = sessionContext.getExecutionContext.getTableEnvironment val catalogName = - if (StringUtils.isEmpty(catalogNameOrEmpty)) tableEnv.getCurrentCatalog + if (StringUtils.isEmpty(catalogNameOrEmpty)) executor.getCurrentCatalog else catalogNameOrEmpty val schemaNameRegex = toJavaRegex(schemaNamePattern) val tableNameRegex = toJavaRegex(tableNamePattern) val columnNameRegex = toJavaRegex(columnNamePattern).r - val columns = tableEnv.getCatalog(catalogName).asScala.toArray.flatMap { flinkCatalog => + val catalogManager = sessionContext.getSessionState.catalogManager + val columns = catalogManager.getCatalog(catalogName).asScala.toArray.flatMap { flinkCatalog => SchemaHelper.getSchemasWithPattern(flinkCatalog, schemaNameRegex) .flatMap { schemaName => SchemaHelper.getFlinkTablesWithPattern( @@ -60,7 +60,8 @@ class GetColumns( tableNameRegex) .filter { _._2.isDefined } .flatMap { case (tableName, _) => - val flinkTable = tableEnv.from(s"`$catalogName`.`$schemaName`.`$tableName`") + val flinkTable = catalogManager.getTable( + ObjectIdentifier.of(catalogName, schemaName, tableName)).get() val resolvedSchema = flinkTable.getResolvedSchema resolvedSchema.getColumns.asScala.toArray.zipWithIndex .filter { case (column, _) => diff --git a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/GetCurrentCatalog.scala b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/GetCurrentCatalog.scala index 988072e8d..5f82de4a6 100644 --- a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/GetCurrentCatalog.scala +++ b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/GetCurrentCatalog.scala @@ -18,15 +18,20 @@ package org.apache.kyuubi.engine.flink.operation import org.apache.kyuubi.engine.flink.result.ResultSetUtil +import org.apache.kyuubi.operation.log.OperationLog import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant.TABLE_CAT import org.apache.kyuubi.session.Session class GetCurrentCatalog(session: Session) extends FlinkOperation(session) { + private val operationLog: OperationLog = + OperationLog.createOperationLog(session, getHandle) + + override def getOperationLog: Option[OperationLog] = Option(operationLog) + override protected def runInternal(): Unit = { try { - val tableEnv = sessionContext.getExecutionContext.getTableEnvironment - val catalog = tableEnv.getCurrentCatalog + val catalog = executor.getCurrentCatalog resultSet = ResultSetUtil.stringListToResultSet(List(catalog), TABLE_CAT) } catch onError() } diff --git a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/GetCurrentDatabase.scala b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/GetCurrentDatabase.scala index 8315a18d3..107609c06 100644 --- a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/GetCurrentDatabase.scala +++ b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/GetCurrentDatabase.scala @@ -18,15 +18,20 @@ package org.apache.kyuubi.engine.flink.operation import org.apache.kyuubi.engine.flink.result.ResultSetUtil +import org.apache.kyuubi.operation.log.OperationLog import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant.TABLE_SCHEM import org.apache.kyuubi.session.Session class GetCurrentDatabase(session: Session) extends FlinkOperation(session) { + private val operationLog: OperationLog = + OperationLog.createOperationLog(session, getHandle) + + override def getOperationLog: Option[OperationLog] = Option(operationLog) + override protected def runInternal(): Unit = { try { - val tableEnv = sessionContext.getExecutionContext.getTableEnvironment - val database = tableEnv.getCurrentDatabase + val database = sessionContext.getSessionState.catalogManager.getCurrentDatabase resultSet = ResultSetUtil.stringListToResultSet(List(database), TABLE_SCHEM) } catch onError() } diff --git a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/GetFunctions.scala b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/GetFunctions.scala index ab870ab79..85f34a29a 100644 --- a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/GetFunctions.scala +++ b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/GetFunctions.scala @@ -20,9 +20,10 @@ package org.apache.kyuubi.engine.flink.operation import java.sql.DatabaseMetaData import scala.collection.JavaConverters._ +import scala.collection.convert.ImplicitConversions._ import org.apache.commons.lang3.StringUtils -import org.apache.flink.table.api.{DataTypes, ResultKind, TableEnvironment} +import org.apache.flink.table.api.{DataTypes, ResultKind} import org.apache.flink.table.catalog.Column import org.apache.flink.types.Row @@ -42,17 +43,20 @@ class GetFunctions( try { val schemaPattern = toJavaRegex(schemaName) val functionPattern = toJavaRegex(functionName) - val tableEnv: TableEnvironment = sessionContext.getExecutionContext.getTableEnvironment + val functionCatalog = sessionContext.getSessionState.functionCatalog + val catalogManager = sessionContext.getSessionState.catalogManager + val systemFunctions = filterPattern( - tableEnv.listFunctions().diff(tableEnv.listUserDefinedFunctions()), + functionCatalog.getFunctions + .diff(functionCatalog.getUserDefinedFunctions), functionPattern) .map { f => Row.of(null, null, f, null, Integer.valueOf(DatabaseMetaData.functionResultUnknown), null) - } - val catalogFunctions = tableEnv.listCatalogs() + }.toArray + val catalogFunctions = catalogManager.listCatalogs() .filter { c => StringUtils.isEmpty(catalogName) || c == catalogName } .flatMap { c => - val catalog = tableEnv.getCatalog(c).get() + val catalog = catalogManager.getCatalog(c).get() filterPattern(catalog.listDatabases().asScala, schemaPattern) .flatMap { d => filterPattern(catalog.listFunctions(d).asScala, functionPattern) @@ -66,7 +70,7 @@ class GetFunctions( null) } } - } + }.toArray resultSet = ResultSet.builder.resultKind(ResultKind.SUCCESS_WITH_CONTENT) .columns( Column.physical(FUNCTION_CAT, DataTypes.STRING()), diff --git a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/GetPrimaryKeys.scala b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/GetPrimaryKeys.scala index b534feb1f..5b9060cf1 100644 --- a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/GetPrimaryKeys.scala +++ b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/GetPrimaryKeys.scala @@ -21,8 +21,9 @@ import scala.collection.JavaConverters._ import org.apache.commons.lang3.StringUtils import org.apache.flink.table.api.{DataTypes, ResultKind} -import org.apache.flink.table.catalog.Column +import org.apache.flink.table.catalog.{Column, ObjectIdentifier} import org.apache.flink.types.Row +import org.apache.flink.util.FlinkException import org.apache.kyuubi.engine.flink.result.ResultSet import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant._ @@ -37,22 +38,25 @@ class GetPrimaryKeys( override protected def runInternal(): Unit = { try { - val tableEnv = sessionContext.getExecutionContext.getTableEnvironment + val catalogManager = sessionContext.getSessionState.catalogManager val catalogName = - if (StringUtils.isEmpty(catalogNameOrEmpty)) tableEnv.getCurrentCatalog + if (StringUtils.isEmpty(catalogNameOrEmpty)) catalogManager.getCurrentCatalog else catalogNameOrEmpty val schemaName = if (StringUtils.isEmpty(schemaNameOrEmpty)) { - if (catalogName != tableEnv.getCurrentCatalog) { - tableEnv.getCatalog(catalogName).get().getDefaultDatabase + if (catalogName != executor.getCurrentCatalog) { + catalogManager.getCatalog(catalogName).get().getDefaultDatabase } else { - tableEnv.getCurrentDatabase + catalogManager.getCurrentDatabase } } else schemaNameOrEmpty - val flinkTable = tableEnv.from(s"`$catalogName`.`$schemaName`.`$tableName`") + val flinkTable = catalogManager + .getTable(ObjectIdentifier.of(catalogName, schemaName, tableName)) + .orElseThrow(() => + new FlinkException(s"Table `$catalogName`.`$schemaName`.`$tableName`` not found.")) val resolvedSchema = flinkTable.getResolvedSchema val primaryKeySchema = resolvedSchema.getPrimaryKey @@ -102,5 +106,4 @@ class GetPrimaryKeys( ) // format: on } - } diff --git a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/GetSchemas.scala b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/GetSchemas.scala index 6715b2320..f56ddd8b1 100644 --- a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/GetSchemas.scala +++ b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/GetSchemas.scala @@ -18,9 +18,10 @@ package org.apache.kyuubi.engine.flink.operation import scala.collection.JavaConverters._ +import scala.collection.convert.ImplicitConversions._ import org.apache.commons.lang3.StringUtils -import org.apache.flink.table.api.{DataTypes, ResultKind, TableEnvironment} +import org.apache.flink.table.api.{DataTypes, ResultKind} import org.apache.flink.table.catalog.Column import org.apache.flink.types.Row @@ -35,14 +36,14 @@ class GetSchemas(session: Session, catalogName: String, schema: String) override protected def runInternal(): Unit = { try { val schemaPattern = toJavaRegex(schema) - val tableEnv: TableEnvironment = sessionContext.getExecutionContext.getTableEnvironment - val schemas = tableEnv.listCatalogs() + val catalogManager = sessionContext.getSessionState.catalogManager + val schemas = catalogManager.listCatalogs() .filter { c => StringUtils.isEmpty(catalogName) || c == catalogName } .flatMap { c => - val catalog = tableEnv.getCatalog(c).get() + val catalog = catalogManager.getCatalog(c).get() filterPattern(catalog.listDatabases().asScala, schemaPattern) .map { d => Row.of(d, c) } - } + }.toArray resultSet = ResultSet.builder.resultKind(ResultKind.SUCCESS_WITH_CONTENT) .columns( Column.physical(TABLE_SCHEM, DataTypes.STRING()), diff --git a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/GetTables.scala b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/GetTables.scala index a4e55715a..325a50167 100644 --- a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/GetTables.scala +++ b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/GetTables.scala @@ -37,16 +37,16 @@ class GetTables( override protected def runInternal(): Unit = { try { - val tableEnv = sessionContext.getExecutionContext.getTableEnvironment + val catalogManager = sessionContext.getSessionState.catalogManager val catalogName = - if (StringUtils.isEmpty(catalogNameOrEmpty)) tableEnv.getCurrentCatalog + if (StringUtils.isEmpty(catalogNameOrEmpty)) catalogManager.getCurrentCatalog else catalogNameOrEmpty val schemaNameRegex = toJavaRegex(schemaNamePattern) val tableNameRegex = toJavaRegex(tableNamePattern) - val tables = tableEnv.getCatalog(catalogName).asScala.toArray.flatMap { flinkCatalog => + val tables = catalogManager.getCatalog(catalogName).asScala.toArray.flatMap { flinkCatalog => SchemaHelper.getSchemasWithPattern(flinkCatalog, schemaNameRegex) .flatMap { schemaName => SchemaHelper.getFlinkTablesWithPattern( diff --git a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/OperationUtils.scala b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/OperationUtils.scala deleted file mode 100644 index 7d624948c..000000000 --- a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/OperationUtils.scala +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kyuubi.engine.flink.operation - -import java.util - -import scala.collection.JavaConverters._ -import scala.collection.mutable.ArrayBuffer - -import org.apache.flink.table.api.{DataTypes, ResultKind} -import org.apache.flink.table.catalog.Column -import org.apache.flink.table.client.gateway.Executor -import org.apache.flink.table.operations.command._ -import org.apache.flink.types.Row - -import org.apache.kyuubi.engine.flink.result.{ResultSet, ResultSetUtil} -import org.apache.kyuubi.engine.flink.result.ResultSetUtil.successResultSet -import org.apache.kyuubi.reflection.DynMethods - -object OperationUtils { - - /** - * Runs a SetOperation with executor. Returns when SetOperation is executed successfully. - * - * @param setOperation Set operation. - * @param executor A gateway for communicating with Flink and other external systems. - * @param sessionId Id of the session. - * @return A ResultSet of SetOperation execution. - */ - def runSetOperation( - setOperation: SetOperation, - executor: Executor, - sessionId: String): ResultSet = { - if (setOperation.getKey.isPresent) { - val key: String = setOperation.getKey.get.trim - - if (setOperation.getValue.isPresent) { - val newValue: String = setOperation.getValue.get.trim - executor.setSessionProperty(sessionId, key, newValue) - } - - val value = executor.getSessionConfigMap(sessionId).getOrDefault(key, "") - ResultSet.builder - .resultKind(ResultKind.SUCCESS_WITH_CONTENT) - .columns( - Column.physical("key", DataTypes.STRING()), - Column.physical("value", DataTypes.STRING())) - .data(Array(Row.of(key, value))) - .build - } else { - // show all properties if set without key - val properties: util.Map[String, String] = executor.getSessionConfigMap(sessionId) - - val entries = ArrayBuffer.empty[Row] - properties.forEach((key, value) => entries.append(Row.of(key, value))) - - if (entries.nonEmpty) { - val prettyEntries = entries.sortBy(_.getField(0).asInstanceOf[String]) - ResultSet.builder - .resultKind(ResultKind.SUCCESS_WITH_CONTENT) - .columns( - Column.physical("key", DataTypes.STRING()), - Column.physical("value", DataTypes.STRING())) - .data(prettyEntries.toArray) - .build - } else { - ResultSet.builder - .resultKind(ResultKind.SUCCESS_WITH_CONTENT) - .columns( - Column.physical("key", DataTypes.STRING()), - Column.physical("value", DataTypes.STRING())) - .data(Array[Row]()) - .build - } - } - } - - /** - * Runs a ResetOperation with executor. Returns when ResetOperation is executed successfully. - * - * @param resetOperation Reset operation. - * @param executor A gateway for communicating with Flink and other external systems. - * @param sessionId Id of the session. - * @return A ResultSet of ResetOperation execution. - */ - def runResetOperation( - resetOperation: ResetOperation, - executor: Executor, - sessionId: String): ResultSet = { - if (resetOperation.getKey.isPresent) { - // reset the given property - executor.resetSessionProperty(sessionId, resetOperation.getKey.get()) - } else { - // reset all properties - executor.resetSessionProperties(sessionId) - } - successResultSet - } - - /** - * Runs a AddJarOperation with the executor. Currently only jars on local filesystem - * are supported. - * - * @param addJarOperation Add-jar operation. - * @param executor A gateway for communicating with Flink and other external systems. - * @param sessionId Id of the session. - * @return A ResultSet of AddJarOperation execution. - */ - def runAddJarOperation( - addJarOperation: AddJarOperation, - executor: Executor, - sessionId: String): ResultSet = { - // Removed by FLINK-27790 - val addJar = DynMethods.builder("addJar") - .impl(executor.getClass, classOf[String], classOf[String]) - .build(executor) - addJar.invoke[Void](sessionId, addJarOperation.getPath) - successResultSet - } - - /** - * Runs a RemoveJarOperation with the executor. Only jars added by AddJarOperation could - * be removed. - * - * @param removeJarOperation Remove-jar operation. - * @param executor A gateway for communicating with Flink and other external systems. - * @param sessionId Id of the session. - * @return A ResultSet of RemoveJarOperation execution. - */ - def runRemoveJarOperation( - removeJarOperation: RemoveJarOperation, - executor: Executor, - sessionId: String): ResultSet = { - executor.removeJar(sessionId, removeJarOperation.getPath) - successResultSet - } - - /** - * Runs a ShowJarsOperation with the executor. Returns the jars of the current session. - * - * @param showJarsOperation Show-jar operation. - * @param executor A gateway for communicating with Flink and other external systems. - * @param sessionId Id of the session. - * @return A ResultSet of ShowJarsOperation execution. - */ - def runShowJarOperation( - showJarsOperation: ShowJarsOperation, - executor: Executor, - sessionId: String): ResultSet = { - // Removed by FLINK-27790 - val listJars = DynMethods.builder("listJars") - .impl(executor.getClass, classOf[String]) - .build(executor) - val jars = listJars.invoke[util.List[String]](sessionId) - ResultSetUtil.stringListToResultSet(jars.asScala.toList, "jar") - } -} diff --git a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/PlanOnlyStatement.scala b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/PlanOnlyStatement.scala index afe04a307..1284bfd73 100644 --- a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/PlanOnlyStatement.scala +++ b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/PlanOnlyStatement.scala @@ -17,10 +17,13 @@ package org.apache.kyuubi.engine.flink.operation +import scala.concurrent.duration.Duration + +import com.google.common.base.Preconditions import org.apache.flink.table.api.TableEnvironment +import org.apache.flink.table.gateway.api.operation.OperationHandle import org.apache.flink.table.operations.command._ -import org.apache.kyuubi.engine.flink.FlinkEngineUtils.isFlinkVersionAtMost import org.apache.kyuubi.engine.flink.result.ResultSetUtil import org.apache.kyuubi.operation.{ExecutionMode, ParseMode, PhysicalMode, PlanOnlyMode, UnknownMode} import org.apache.kyuubi.operation.PlanOnlyMode.{notSupportedModeError, unknownModeError} @@ -33,7 +36,10 @@ import org.apache.kyuubi.session.Session class PlanOnlyStatement( session: Session, override val statement: String, - mode: PlanOnlyMode) extends FlinkOperation(session) { + mode: PlanOnlyMode, + queryTimeout: Long, + resultMaxRows: Int, + resultFetchTimeout: Duration) extends FlinkOperation(session) { private val operationLog: OperationLog = OperationLog.createOperationLog(session, getHandle) private val lineSeparator: String = System.lineSeparator() @@ -45,19 +51,22 @@ class PlanOnlyStatement( } override protected def runInternal(): Unit = { + addTimeoutMonitor(queryTimeout) try { - val operation = executor.parseStatement(sessionId, statement) + val operations = executor.getTableEnvironment.getParser.parse(statement) + Preconditions.checkArgument( + operations.size() == 1, + "Plan-only mode supports single statement only", + null) + val operation = operations.get(0) operation match { - case setOperation: SetOperation => - resultSet = OperationUtils.runSetOperation(setOperation, executor, sessionId) - case resetOperation: ResetOperation => - resultSet = OperationUtils.runResetOperation(resetOperation, executor, sessionId) - case addJarOperation: AddJarOperation if isFlinkVersionAtMost("1.15") => - resultSet = OperationUtils.runAddJarOperation(addJarOperation, executor, sessionId) - case removeJarOperation: RemoveJarOperation => - resultSet = OperationUtils.runRemoveJarOperation(removeJarOperation, executor, sessionId) - case showJarsOperation: ShowJarsOperation if isFlinkVersionAtMost("1.15") => - resultSet = OperationUtils.runShowJarOperation(showJarsOperation, executor, sessionId) + case _: SetOperation | _: ResetOperation | _: AddJarOperation | _: RemoveJarOperation | + _: ShowJarsOperation => + val resultFetcher = executor.executeStatement( + new OperationHandle(getHandle.identifier), + statement) + resultSet = + ResultSetUtil.fromResultFetcher(resultFetcher, resultMaxRows, resultFetchTimeout); case _ => explainOperation(statement) } } catch { @@ -66,7 +75,7 @@ class PlanOnlyStatement( } private def explainOperation(statement: String): Unit = { - val tableEnv: TableEnvironment = sessionContext.getExecutionContext.getTableEnvironment + val tableEnv: TableEnvironment = executor.getTableEnvironment val explainPlans = tableEnv.explainSql(statement).split(s"$lineSeparator$lineSeparator") val operationPlan = mode match { diff --git a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/SetCurrentCatalog.scala b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/SetCurrentCatalog.scala index 489cc6384..f279ccda6 100644 --- a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/SetCurrentCatalog.scala +++ b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/SetCurrentCatalog.scala @@ -17,15 +17,21 @@ package org.apache.kyuubi.engine.flink.operation +import org.apache.kyuubi.operation.log.OperationLog import org.apache.kyuubi.session.Session class SetCurrentCatalog(session: Session, catalog: String) extends FlinkOperation(session) { + private val operationLog: OperationLog = + OperationLog.createOperationLog(session, getHandle) + + override def getOperationLog: Option[OperationLog] = Option(operationLog) + override protected def runInternal(): Unit = { try { - val tableEnv = sessionContext.getExecutionContext.getTableEnvironment - tableEnv.useCatalog(catalog) + val catalogManager = sessionContext.getSessionState.catalogManager + catalogManager.setCurrentCatalog(catalog) setHasResultSet(false) } catch onError() } diff --git a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/SetCurrentDatabase.scala b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/SetCurrentDatabase.scala index 0d3598405..70535e834 100644 --- a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/SetCurrentDatabase.scala +++ b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/operation/SetCurrentDatabase.scala @@ -17,15 +17,21 @@ package org.apache.kyuubi.engine.flink.operation +import org.apache.kyuubi.operation.log.OperationLog import org.apache.kyuubi.session.Session class SetCurrentDatabase(session: Session, database: String) extends FlinkOperation(session) { + private val operationLog: OperationLog = + OperationLog.createOperationLog(session, getHandle) + + override def getOperationLog: Option[OperationLog] = Option(operationLog) + override protected def runInternal(): Unit = { try { - val tableEnv = sessionContext.getExecutionContext.getTableEnvironment - tableEnv.useDatabase(database) + val catalogManager = sessionContext.getSessionState.catalogManager + catalogManager.setCurrentDatabase(database) setHasResultSet(false) } catch onError() } diff --git a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/result/Constants.scala b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/result/Constants.scala new file mode 100644 index 000000000..ca582b2e3 --- /dev/null +++ b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/result/Constants.scala @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.engine.flink.result + +object Constants { + val TABLE_TYPE: String = "TABLE" + val VIEW_TYPE: String = "VIEW" + val SUPPORTED_TABLE_TYPES: Array[String] = Array[String](TABLE_TYPE, VIEW_TYPE) +} diff --git a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/result/QueryResultFetchIterator.scala b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/result/QueryResultFetchIterator.scala new file mode 100644 index 000000000..60ae08d9d --- /dev/null +++ b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/result/QueryResultFetchIterator.scala @@ -0,0 +1,176 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.engine.flink.result + +import java.util +import java.util.concurrent.Executors + +import scala.collection.convert.ImplicitConversions._ +import scala.concurrent.{Await, ExecutionContext, ExecutionContextExecutor, Future} +import scala.concurrent.duration.Duration + +import com.google.common.util.concurrent.ThreadFactoryBuilder +import org.apache.flink.table.api.DataTypes +import org.apache.flink.table.catalog.ResolvedSchema +import org.apache.flink.table.data.RowData +import org.apache.flink.table.data.conversion.DataStructureConverters +import org.apache.flink.table.gateway.service.result.ResultFetcher +import org.apache.flink.table.types.DataType +import org.apache.flink.types.Row + +import org.apache.kyuubi.Logging +import org.apache.kyuubi.engine.flink.shim.FlinkResultSet +import org.apache.kyuubi.operation.FetchIterator + +class QueryResultFetchIterator( + resultFetcher: ResultFetcher, + maxRows: Int = 1000000, + resultFetchTimeout: Duration = Duration.Inf) extends FetchIterator[Row] with Logging { + + val schema: ResolvedSchema = resultFetcher.getResultSchema + + val dataTypes: util.List[DataType] = schema.getColumnDataTypes + + var token: Long = 0 + + var pos: Long = 0 + + var fetchStart: Long = 0 + + var bufferedRows: Array[Row] = new Array[Row](0) + + var hasNext: Boolean = true + + val FETCH_INTERVAL_MS: Long = 1000 + + private val executor = Executors.newSingleThreadScheduledExecutor( + new ThreadFactoryBuilder().setNameFormat("flink-query-iterator-%d").setDaemon(true).build) + + implicit private val executionContext: ExecutionContextExecutor = + ExecutionContext.fromExecutor(executor) + + /** + * Begin a fetch block, forward from the current position. + * + * Throws TimeoutException if no data is fetched within the timeout. + */ + override def fetchNext(): Unit = { + if (!hasNext) { + return + } + val future = Future(() -> { + var fetched = false + // if no timeout is set, this would block until some rows are fetched + debug(s"Fetching from result store with timeout $resultFetchTimeout ms") + while (!fetched && !Thread.interrupted()) { + val rs = resultFetcher.fetchResults(token, maxRows - bufferedRows.length) + val flinkRs = new FlinkResultSet(rs) + // TODO: replace string-based match when Flink 1.16 support is dropped + flinkRs.getResultType.name() match { + case "EOS" => + debug("EOS received, no more data to fetch.") + fetched = true + hasNext = false + case "NOT_READY" => + // if flink jobs are not ready, continue to retry + debug("Result not ready, retrying...") + case "PAYLOAD" => + val fetchedData = flinkRs.getData + // if no data fetched, continue to retry + if (!fetchedData.isEmpty) { + debug(s"Fetched ${fetchedData.length} rows from result store.") + fetched = true + bufferedRows ++= fetchedData.map(rd => convertToRow(rd, dataTypes.toList)) + fetchStart = pos + } else { + debug("No data fetched, retrying...") + } + case _ => + throw new RuntimeException(s"Unexpected result type: ${flinkRs.getResultType}") + } + if (hasNext) { + val nextToken = flinkRs.getNextToken + if (nextToken == null) { + hasNext = false + } else { + token = nextToken + } + } + Thread.sleep(FETCH_INTERVAL_MS) + } + }) + Await.result(future, resultFetchTimeout) + } + + /** + * Begin a fetch block, moving the iterator to the given position. + * Resets the fetch start offset. + * + * @param pos index to move a position of iterator. + */ + override def fetchAbsolute(pos: Long): Unit = { + val effectivePos = Math.max(pos, 0) + if (effectivePos < bufferedRows.length) { + this.fetchStart = effectivePos + return + } + throw new IllegalArgumentException(s"Cannot skip to an unreachable position $effectivePos.") + } + + override def getFetchStart: Long = fetchStart + + override def getPosition: Long = pos + + /** + * @return returns row if any and null if no more rows can be fetched. + */ + override def next(): Row = { + if (pos < bufferedRows.length) { + debug(s"Fetching from buffered rows at pos $pos.") + val row = bufferedRows(pos.toInt) + pos += 1 + if (pos >= maxRows) { + hasNext = false + } + row + } else { + // block until some rows are fetched or TimeoutException is thrown + fetchNext() + if (hasNext) { + val row = bufferedRows(pos.toInt) + pos += 1 + if (pos >= maxRows) { + hasNext = false + } + row + } else { + null + } + } + } + + def close(): Unit = { + resultFetcher.close() + executor.shutdown() + } + + private[this] def convertToRow(r: RowData, dataTypes: List[DataType]): Row = { + val converter = DataStructureConverters.getConverter(DataTypes.ROW(dataTypes: _*)) + converter.toExternal(r).asInstanceOf[Row] + } +} diff --git a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/result/ResultSet.scala b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/result/ResultSet.scala new file mode 100644 index 000000000..b8d407297 --- /dev/null +++ b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/result/ResultSet.scala @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.engine.flink.result + +import java.util + +import scala.collection.JavaConverters._ + +import com.google.common.collect.Iterators +import org.apache.flink.api.common.JobID +import org.apache.flink.table.api.{DataTypes, ResultKind} +import org.apache.flink.table.catalog.Column +import org.apache.flink.types.Row + +import org.apache.kyuubi.operation.{ArrayFetchIterator, FetchIterator} + +case class ResultSet( + resultKind: ResultKind, + columns: util.List[Column], + data: FetchIterator[Row], + // null in batch mode + // list of boolean in streaming mode, + // true if the corresponding row is an append row, false if its a retract row + changeFlags: Option[util.List[Boolean]]) { + + require(resultKind != null, "resultKind must not be null") + require(columns != null, "columns must not be null") + require(data != null, "data must not be null") + changeFlags.foreach { flags => + require( + Iterators.size(data.asInstanceOf[util.Iterator[_]]) == flags.size, + "the size of data and the size of changeFlags should be equal") + } + + def getColumns: util.List[Column] = columns + + def getData: FetchIterator[Row] = data + + def close: Unit = { + data match { + case queryIte: QueryResultFetchIterator => queryIte.close() + case _ => + } + } +} + +/** + * A set of one statement execution result containing result kind, columns, rows of data and change + * flags for streaming mode. + */ +object ResultSet { + + def fromJobId(jobID: JobID): ResultSet = { + val data: Array[Row] = if (jobID != null) { + Array(Row.of(jobID.toString)) + } else { + // should not happen + Array(Row.of("(Empty Job ID)")) + } + builder + .resultKind(ResultKind.SUCCESS_WITH_CONTENT) + .columns(Column.physical("result", DataTypes.STRING())) + .data(data) + .build + } + + def builder: Builder = new ResultSet.Builder + + class Builder { + private var resultKind: ResultKind = _ + private var columns: util.List[Column] = _ + private var data: FetchIterator[Row] = _ + private var changeFlags: Option[util.List[Boolean]] = None + + def resultKind(resultKind: ResultKind): ResultSet.Builder = { + this.resultKind = resultKind + this + } + + def columns(columns: Column*): ResultSet.Builder = { + this.columns = columns.asJava + this + } + + def columns(columns: util.List[Column]): ResultSet.Builder = { + this.columns = columns + this + } + + def data(data: FetchIterator[Row]): ResultSet.Builder = { + this.data = data + this + } + + def data(data: Array[Row]): ResultSet.Builder = { + this.data = new ArrayFetchIterator[Row](data) + this + } + + def changeFlags(changeFlags: util.List[Boolean]): ResultSet.Builder = { + this.changeFlags = Some(changeFlags) + this + } + + def build: ResultSet = new ResultSet(resultKind, columns, data, changeFlags) + } +} diff --git a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/result/ResultSetUtil.scala b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/result/ResultSetUtil.scala index ded271cf1..8b722f1e5 100644 --- a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/result/ResultSetUtil.scala +++ b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/result/ResultSetUtil.scala @@ -15,11 +15,14 @@ * limitations under the License. */ -package org.apache.kyuubi.engine.flink.result; +package org.apache.kyuubi.engine.flink.result + +import scala.concurrent.duration.Duration import org.apache.flink.table.api.DataTypes import org.apache.flink.table.api.ResultKind import org.apache.flink.table.catalog.Column +import org.apache.flink.table.gateway.service.result.ResultFetcher import org.apache.flink.types.Row /** Utility object for building ResultSet. */ @@ -54,4 +57,20 @@ object ResultSetUtil { .columns(Column.physical("result", DataTypes.STRING)) .data(Array[Row](Row.of("OK"))) .build + + def fromResultFetcher( + resultFetcher: ResultFetcher, + maxRows: Int, + resultFetchTimeout: Duration): ResultSet = { + if (maxRows <= 0) { + throw new IllegalArgumentException("maxRows should be positive") + } + val schema = resultFetcher.getResultSchema + val ite = new QueryResultFetchIterator(resultFetcher, maxRows, resultFetchTimeout) + ResultSet.builder + .resultKind(ResultKind.SUCCESS_WITH_CONTENT) + .columns(schema.getColumns) + .data(ite) + .build + } } diff --git a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/schema/RowSet.scala b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/schema/RowSet.scala index 2b3ae50b7..c446396d5 100644 --- a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/schema/RowSet.scala +++ b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/schema/RowSet.scala @@ -21,7 +21,9 @@ import java.{lang, util} import java.nio.ByteBuffer import java.nio.charset.StandardCharsets import java.sql.{Date, Timestamp} -import java.time.{LocalDate, LocalDateTime} +import java.time.{Instant, LocalDate, LocalDateTime, ZonedDateTime, ZoneId} +import java.time.format.{DateTimeFormatter, DateTimeFormatterBuilder, TextStyle} +import java.time.temporal.ChronoField import java.util.Collections import scala.collection.JavaConverters._ @@ -42,15 +44,16 @@ object RowSet { def resultSetToTRowSet( rows: Seq[Row], resultSet: ResultSet, + zoneId: ZoneId, protocolVersion: TProtocolVersion): TRowSet = { if (protocolVersion.getValue < TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V6.getValue) { - toRowBaseSet(rows, resultSet) + toRowBaseSet(rows, resultSet, zoneId) } else { - toColumnBasedSet(rows, resultSet) + toColumnBasedSet(rows, resultSet, zoneId) } } - def toRowBaseSet(rows: Seq[Row], resultSet: ResultSet): TRowSet = { + def toRowBaseSet(rows: Seq[Row], resultSet: ResultSet, zoneId: ZoneId): TRowSet = { val rowSize = rows.size val tRows = new util.ArrayList[TRow](rowSize) var i = 0 @@ -60,7 +63,7 @@ object RowSet { val columnSize = row.getArity var j = 0 while (j < columnSize) { - val columnValue = toTColumnValue(j, row, resultSet) + val columnValue = toTColumnValue(j, row, resultSet, zoneId) tRow.addToColVals(columnValue) j += 1 } @@ -71,14 +74,14 @@ object RowSet { new TRowSet(0, tRows) } - def toColumnBasedSet(rows: Seq[Row], resultSet: ResultSet): TRowSet = { + def toColumnBasedSet(rows: Seq[Row], resultSet: ResultSet, zoneId: ZoneId): TRowSet = { val size = rows.length val tRowSet = new TRowSet(0, new util.ArrayList[TRow](size)) val columnSize = resultSet.getColumns.size() var i = 0 while (i < columnSize) { val field = resultSet.getColumns.get(i) - val tColumn = toTColumn(rows, i, field.getDataType.getLogicalType) + val tColumn = toTColumn(rows, i, field.getDataType.getLogicalType, zoneId) tRowSet.addToColumns(tColumn) i += 1 } @@ -88,7 +91,8 @@ object RowSet { private def toTColumnValue( ordinal: Int, row: Row, - resultSet: ResultSet): TColumnValue = { + resultSet: ResultSet, + zoneId: ZoneId): TColumnValue = { val column = resultSet.getColumns.get(ordinal) val logicalType = column.getDataType.getLogicalType @@ -153,6 +157,12 @@ object RowSet { s"for type ${t.getClass}.") } TColumnValue.stringVal(tStringValue) + case _: LocalZonedTimestampType => + val tStringValue = new TStringValue + val fieldValue = row.getField(ordinal) + tStringValue.setValue(TIMESTAMP_LZT_FORMATTER.format( + ZonedDateTime.ofInstant(fieldValue.asInstanceOf[Instant], zoneId))) + TColumnValue.stringVal(tStringValue) case t => val tStringValue = new TStringValue if (row.getField(ordinal) != null) { @@ -166,7 +176,11 @@ object RowSet { ByteBuffer.wrap(bitSet.toByteArray) } - private def toTColumn(rows: Seq[Row], ordinal: Int, logicalType: LogicalType): TColumn = { + private def toTColumn( + rows: Seq[Row], + ordinal: Int, + logicalType: LogicalType, + zoneId: ZoneId): TColumn = { val nulls = new java.util.BitSet() // for each column, determine the conversion class by sampling the first non-value value // if there's no row, set the entire column empty @@ -211,6 +225,12 @@ object RowSet { s"for type ${t.getClass}.") } TColumn.stringVal(new TStringColumn(values, nulls)) + case _: LocalZonedTimestampType => + val values = getOrSetAsNull[Instant](rows, ordinal, nulls, Instant.EPOCH) + .toArray().map(v => + TIMESTAMP_LZT_FORMATTER.format( + ZonedDateTime.ofInstant(v.asInstanceOf[Instant], zoneId))) + TColumn.stringVal(new TStringColumn(values.toList.asJava, nulls)) case _ => var i = 0 val rowSize = rows.length @@ -303,11 +323,14 @@ object RowSet { case _: DecimalType => TTypeId.DECIMAL_TYPE case _: DateType => TTypeId.DATE_TYPE case _: TimestampType => TTypeId.TIMESTAMP_TYPE + case _: LocalZonedTimestampType => TTypeId.TIMESTAMPLOCALTZ_TYPE case _: ArrayType => TTypeId.ARRAY_TYPE case _: MapType => TTypeId.MAP_TYPE case _: RowType => TTypeId.STRUCT_TYPE case _: BinaryType => TTypeId.BINARY_TYPE - case t @ (_: ZonedTimestampType | _: LocalZonedTimestampType | _: MultisetType | + case _: VarBinaryType => TTypeId.BINARY_TYPE + case _: TimeType => TTypeId.STRING_TYPE + case t @ (_: ZonedTimestampType | _: MultisetType | _: YearMonthIntervalType | _: DayTimeIntervalType) => throw new IllegalArgumentException( "Flink data type `%s` is not supported currently".format(t.asSummaryString()), @@ -368,11 +391,33 @@ object RowSet { // Only match string in nested type values "\"" + s + "\"" - case (bin: Array[Byte], _: BinaryType) => + case (bin: Array[Byte], _ @(_: BinaryType | _: VarBinaryType)) => new String(bin, StandardCharsets.UTF_8) case (other, _) => other.toString } } + + /** should stay in sync with org.apache.kyuubi.jdbc.hive.common.TimestampTZUtil */ + var TIMESTAMP_LZT_FORMATTER: DateTimeFormatter = { + val builder = new DateTimeFormatterBuilder + // Date part + builder.append(DateTimeFormatter.ofPattern("yyyy-MM-dd")) + // Time part + builder + .optionalStart + .appendLiteral(" ") + .append(DateTimeFormatter.ofPattern("HH:mm:ss")) + .optionalStart + .appendFraction(ChronoField.NANO_OF_SECOND, 1, 9, true) + .optionalEnd + .optionalEnd + + // Zone part + builder.optionalStart.appendLiteral(" ").optionalEnd + builder.optionalStart.appendZoneText(TextStyle.NARROW).optionalEnd + + builder.toFormatter + } } diff --git a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/session/FlinkSQLSessionManager.scala b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/session/FlinkSQLSessionManager.scala index 8a3fc7446..b7cd46217 100644 --- a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/session/FlinkSQLSessionManager.scala +++ b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/session/FlinkSQLSessionManager.scala @@ -17,11 +17,17 @@ package org.apache.kyuubi.engine.flink.session -import org.apache.flink.table.client.gateway.context.DefaultContext -import org.apache.flink.table.client.gateway.local.LocalExecutor +import scala.collection.JavaConverters._ +import scala.collection.JavaConverters.mapAsJavaMap + +import org.apache.flink.table.gateway.api.session.SessionEnvironment +import org.apache.flink.table.gateway.rest.util.SqlGatewayRestAPIVersion +import org.apache.flink.table.gateway.service.context.DefaultContext import org.apache.hive.service.rpc.thrift.TProtocolVersion +import org.apache.kyuubi.config.KyuubiReservedKeys.KYUUBI_SESSION_HANDLE_KEY import org.apache.kyuubi.engine.flink.operation.FlinkSQLOperationManager +import org.apache.kyuubi.engine.flink.shim.FlinkSessionManager import org.apache.kyuubi.session.{Session, SessionHandle, SessionManager} class FlinkSQLSessionManager(engineContext: DefaultContext) @@ -30,11 +36,11 @@ class FlinkSQLSessionManager(engineContext: DefaultContext) override protected def isServer: Boolean = false val operationManager = new FlinkSQLOperationManager() - val executor = new LocalExecutor(engineContext) + val sessionManager = new FlinkSessionManager(engineContext) override def start(): Unit = { super.start() - executor.start() + sessionManager.start() } override protected def createSession( @@ -43,18 +49,42 @@ class FlinkSQLSessionManager(engineContext: DefaultContext) password: String, ipAddress: String, conf: Map[String, String]): Session = { - new FlinkSessionImpl( - protocol, - user, - password, - ipAddress, - conf, - this, - executor) + conf.get(KYUUBI_SESSION_HANDLE_KEY).map(SessionHandle.fromUUID).flatMap( + getSessionOption).getOrElse { + val flinkInternalSession = sessionManager.openSession( + SessionEnvironment.newBuilder + .setSessionEndpointVersion(SqlGatewayRestAPIVersion.V1) + .addSessionConfig(mapAsJavaMap(conf)) + .build) + val sessionConfig = flinkInternalSession.getSessionConfig + sessionConfig.putAll(conf.asJava) + val session = new FlinkSessionImpl( + protocol, + user, + password, + ipAddress, + conf, + this, + flinkInternalSession) + session + } + } + + override def getSessionOption(sessionHandle: SessionHandle): Option[Session] = { + val session = super.getSessionOption(sessionHandle) + session.foreach(s => s.asInstanceOf[FlinkSessionImpl].fSession.touch()) + session } override def closeSession(sessionHandle: SessionHandle): Unit = { + val fSession = super.getSessionOption(sessionHandle) + fSession.foreach(s => + sessionManager.closeSession(s.asInstanceOf[FlinkSessionImpl].fSession.getSessionHandle)) super.closeSession(sessionHandle) - executor.closeSession(sessionHandle.toString) + } + + override def stop(): Unit = synchronized { + sessionManager.stop() + super.stop() } } diff --git a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/session/FlinkSessionImpl.scala b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/session/FlinkSessionImpl.scala index 03d9ce42e..b8d1f8569 100644 --- a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/session/FlinkSessionImpl.scala +++ b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/session/FlinkSessionImpl.scala @@ -19,15 +19,19 @@ package org.apache.kyuubi.engine.flink.session import scala.util.control.NonFatal +import org.apache.flink.configuration.Configuration import org.apache.flink.runtime.util.EnvironmentInformation import org.apache.flink.table.client.gateway.SqlExecutionException -import org.apache.flink.table.client.gateway.context.SessionContext -import org.apache.flink.table.client.gateway.local.LocalExecutor +import org.apache.flink.table.gateway.api.operation.OperationHandle +import org.apache.flink.table.gateway.service.context.SessionContext +import org.apache.flink.table.gateway.service.session.{Session => FSession} import org.apache.hive.service.rpc.thrift.{TGetInfoType, TGetInfoValue, TProtocolVersion} import org.apache.kyuubi.KyuubiSQLException +import org.apache.kyuubi.config.KyuubiReservedKeys.KYUUBI_SESSION_HANDLE_KEY import org.apache.kyuubi.engine.flink.FlinkEngineUtils -import org.apache.kyuubi.session.{AbstractSession, SessionManager} +import org.apache.kyuubi.engine.flink.udf.KDFRegistry +import org.apache.kyuubi.session.{AbstractSession, SessionHandle, SessionManager, USE_CATALOG, USE_DATABASE} class FlinkSessionImpl( protocol: TProtocolVersion, @@ -36,13 +40,19 @@ class FlinkSessionImpl( ipAddress: String, conf: Map[String, String], sessionManager: SessionManager, - val executor: LocalExecutor) + val fSession: FSession) extends AbstractSession(protocol, user, password, ipAddress, conf, sessionManager) { - lazy val sessionContext: SessionContext = { - FlinkEngineUtils.getSessionContext(executor, handle.identifier.toString) + override val handle: SessionHandle = + conf.get(KYUUBI_SESSION_HANDLE_KEY).map(SessionHandle.fromUUID) + .getOrElse(SessionHandle.fromUUID(fSession.getSessionHandle.getIdentifier.toString)) + + val sessionContext: SessionContext = { + FlinkEngineUtils.getSessionContext(fSession) } + KDFRegistry.registerAll(sessionContext) + private def setModifiableConfig(key: String, value: String): Unit = { try { sessionContext.set(key, value) @@ -52,26 +62,33 @@ class FlinkSessionImpl( } override def open(): Unit = { - executor.openSession(handle.identifier.toString) - normalizedConf.foreach { - case ("use:catalog", catalog) => - val tableEnv = sessionContext.getExecutionContext.getTableEnvironment - try { - tableEnv.useCatalog(catalog) - } catch { - case NonFatal(e) => + val executor = fSession.createExecutor(Configuration.fromMap(fSession.getSessionConfig)) + + val (useCatalogAndDatabaseConf, otherConf) = normalizedConf.partition { case (k, _) => + Array(USE_CATALOG, USE_DATABASE).contains(k) + } + + useCatalogAndDatabaseConf.get(USE_CATALOG).foreach { catalog => + try { + executor.executeStatement(OperationHandle.create, s"USE CATALOG $catalog") + } catch { + case NonFatal(e) => + throw e + } + } + + useCatalogAndDatabaseConf.get("use:database").foreach { database => + try { + executor.executeStatement(OperationHandle.create, s"USE $database") + } catch { + case NonFatal(e) => + if (database != "default") { throw e - } - case ("use:database", database) => - val tableEnv = sessionContext.getExecutionContext.getTableEnvironment - try { - tableEnv.useDatabase(database) - } catch { - case NonFatal(e) => - if (database != "default") { - throw e - } - } + } + } + } + + otherConf.foreach { case (key, value) => setModifiableConfig(key, value) } super.open() diff --git a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/shim/FlinkResultSet.scala b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/shim/FlinkResultSet.scala new file mode 100644 index 000000000..7fb05c844 --- /dev/null +++ b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/shim/FlinkResultSet.scala @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.engine.flink.shim + +import java.lang.{Long => JLong} +import java.util + +import org.apache.flink.table.data.RowData +import org.apache.flink.table.gateway.api.results.ResultSet.ResultType + +import org.apache.kyuubi.util.reflect.ReflectUtils._ + +class FlinkResultSet(resultSet: AnyRef) { + + def getData: util.List[RowData] = invokeAs(resultSet, "getData") + + def getNextToken: JLong = invokeAs(resultSet, "getNextToken") + + def getResultType: ResultType = invokeAs(resultSet, "getResultType") +} diff --git a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/shim/FlinkSessionManager.scala b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/shim/FlinkSessionManager.scala new file mode 100644 index 000000000..89414ac4c --- /dev/null +++ b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/shim/FlinkSessionManager.scala @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.engine.flink.shim + +import org.apache.flink.table.gateway.api.session.{SessionEnvironment, SessionHandle} +import org.apache.flink.table.gateway.service.context.DefaultContext +import org.apache.flink.table.gateway.service.session.Session + +import org.apache.kyuubi.engine.flink.FlinkEngineUtils.FLINK_RUNTIME_VERSION +import org.apache.kyuubi.util.reflect._ +import org.apache.kyuubi.util.reflect.ReflectUtils._ + +class FlinkSessionManager(engineContext: DefaultContext) { + + val sessionManager: AnyRef = { + if (FLINK_RUNTIME_VERSION === "1.16") { + DynConstructors.builder().impl( + "org.apache.flink.table.gateway.service.session.SessionManager", + classOf[DefaultContext]) + .build() + .newInstance(engineContext) + } else { + DynConstructors.builder().impl( + "org.apache.flink.table.gateway.service.session.SessionManagerImpl", + classOf[DefaultContext]) + .build() + .newInstance(engineContext) + } + } + + def start(): Unit = invokeAs(sessionManager, "start") + + def stop(): Unit = invokeAs(sessionManager, "stop") + + def getSession(sessionHandle: SessionHandle): Session = + invokeAs(sessionManager, "getSession", (classOf[SessionHandle], sessionHandle)) + + def openSession(environment: SessionEnvironment): Session = + invokeAs(sessionManager, "openSession", (classOf[SessionEnvironment], environment)) + + def closeSession(sessionHandle: SessionHandle): Unit = + invokeAs(sessionManager, "closeSession", (classOf[SessionHandle], sessionHandle)) +} diff --git a/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/udf/KDFRegistry.scala b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/udf/KDFRegistry.scala new file mode 100644 index 000000000..9ccbe7940 --- /dev/null +++ b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/udf/KDFRegistry.scala @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.engine.flink.udf + +import java.util + +import scala.collection.mutable.ArrayBuffer + +import org.apache.flink.configuration.Configuration +import org.apache.flink.table.functions.{ScalarFunction, UserDefinedFunction} +import org.apache.flink.table.gateway.service.context.SessionContext + +import org.apache.kyuubi.{KYUUBI_VERSION, Utils} +import org.apache.kyuubi.config.KyuubiReservedKeys.{KYUUBI_ENGINE_NAME, KYUUBI_SESSION_USER_KEY} +import org.apache.kyuubi.engine.flink.FlinkEngineUtils.FLINK_RUNTIME_VERSION +import org.apache.kyuubi.util.reflect.DynMethods + +object KDFRegistry { + + def createKyuubiDefinedFunctions(sessionContext: SessionContext): Array[KyuubiDefinedFunction] = { + + val kyuubiDefinedFunctions = new ArrayBuffer[KyuubiDefinedFunction] + + val flinkConfigMap: util.Map[String, String] = { + if (FLINK_RUNTIME_VERSION === "1.16") { + DynMethods + .builder("getConfigMap") + .impl(classOf[SessionContext]) + .build() + .invoke(sessionContext) + .asInstanceOf[util.Map[String, String]] + } else { + DynMethods + .builder("getSessionConf") + .impl(classOf[SessionContext]) + .build() + .invoke(sessionContext) + .asInstanceOf[Configuration] + .toMap + } + } + + val kyuubi_version: KyuubiDefinedFunction = create( + "kyuubi_version", + new KyuubiVersionFunction(flinkConfigMap), + "Return the version of Kyuubi Server", + "string", + "1.8.0") + kyuubiDefinedFunctions += kyuubi_version + + val engineName: KyuubiDefinedFunction = create( + "kyuubi_engine_name", + new EngineNameFunction(flinkConfigMap), + "Return the application name for the associated query engine", + "string", + "1.8.0") + kyuubiDefinedFunctions += engineName + + val engineId: KyuubiDefinedFunction = create( + "kyuubi_engine_id", + new EngineIdFunction(flinkConfigMap), + "Return the application id for the associated query engine", + "string", + "1.8.0") + kyuubiDefinedFunctions += engineId + + val systemUser: KyuubiDefinedFunction = create( + "kyuubi_system_user", + new SystemUserFunction(flinkConfigMap), + "Return the system user name for the associated query engine", + "string", + "1.8.0") + kyuubiDefinedFunctions += systemUser + + val sessionUser: KyuubiDefinedFunction = create( + "kyuubi_session_user", + new SessionUserFunction(flinkConfigMap), + "Return the session username for the associated query engine", + "string", + "1.8.0") + kyuubiDefinedFunctions += sessionUser + + kyuubiDefinedFunctions.toArray + } + + def create( + name: String, + udf: UserDefinedFunction, + description: String, + returnType: String, + since: String): KyuubiDefinedFunction = { + val kdf = KyuubiDefinedFunction(name, udf, description, returnType, since) + kdf + } + + def registerAll(sessionContext: SessionContext): Unit = { + val functions = createKyuubiDefinedFunctions(sessionContext) + for (func <- functions) { + sessionContext.getSessionState.functionCatalog + .registerTemporarySystemFunction(func.name, func.udf, true) + } + } +} + +class KyuubiVersionFunction(confMap: util.Map[String, String]) extends ScalarFunction { + def eval(): String = KYUUBI_VERSION +} + +class EngineNameFunction(confMap: util.Map[String, String]) extends ScalarFunction { + def eval(): String = { + confMap match { + case m if m.containsKey("yarn.application.name") => m.get("yarn.application.name") + case m if m.containsKey("kubernetes.cluster-id") => m.get("kubernetes.cluster-id") + case m => m.getOrDefault(KYUUBI_ENGINE_NAME, "unknown-engine-name") + } + } +} + +class EngineIdFunction(confMap: util.Map[String, String]) extends ScalarFunction { + def eval(): String = { + confMap match { + case m if m.containsKey("yarn.application.id") => m.get("yarn.application.id") + case m if m.containsKey("kubernetes.cluster-id") => m.get("kubernetes.cluster-id") + case m => m.getOrDefault("high-availability.cluster-id", "unknown-engine-id") + } + } +} + +class SystemUserFunction(confMap: util.Map[String, String]) extends ScalarFunction { + def eval(): String = Utils.currentUser +} + +class SessionUserFunction(confMap: util.Map[String, String]) extends ScalarFunction { + def eval(): String = confMap.getOrDefault(KYUUBI_SESSION_USER_KEY, "unknown-user") +} diff --git a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkHudiOperationSuite.scala b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/udf/KyuubiDefinedFunction.scala similarity index 65% rename from externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkHudiOperationSuite.scala rename to externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/udf/KyuubiDefinedFunction.scala index c5e8be37a..5cfce86d6 100644 --- a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkHudiOperationSuite.scala +++ b/externals/kyuubi-flink-sql-engine/src/main/scala/org/apache/kyuubi/engine/flink/udf/KyuubiDefinedFunction.scala @@ -15,16 +15,20 @@ * limitations under the License. */ -package org.apache.kyuubi.engine.spark.operation +package org.apache.kyuubi.engine.flink.udf -import org.apache.kyuubi.engine.spark.WithSparkSQLEngine -import org.apache.kyuubi.operation.HudiMetadataTests -import org.apache.kyuubi.tags.HudiTest +import org.apache.flink.table.functions.UserDefinedFunction -@HudiTest -class SparkHudiOperationSuite extends WithSparkSQLEngine with HudiMetadataTests { - - override protected def jdbcUrl: String = getJdbcUrl - - override def withKyuubiConf: Map[String, String] = extraConfigs -} +/** + * A wrapper for Flink's [[UserDefinedFunction]] + * + * @param name function name + * @param udf user-defined function + * @param description function description + */ +case class KyuubiDefinedFunction( + name: String, + udf: UserDefinedFunction, + description: String, + returnType: String, + since: String) diff --git a/externals/kyuubi-flink-sql-engine/src/test/resources/log4j2-test.xml b/externals/kyuubi-flink-sql-engine/src/test/resources/log4j2-test.xml index bfc40dd6d..3110216c1 100644 --- a/externals/kyuubi-flink-sql-engine/src/test/resources/log4j2-test.xml +++ b/externals/kyuubi-flink-sql-engine/src/test/resources/log4j2-test.xml @@ -21,14 +21,14 @@ - + - + diff --git a/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/WithDiscoveryFlinkSQLEngine.scala b/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/WithDiscoveryFlinkSQLEngine.scala new file mode 100644 index 000000000..c352429ea --- /dev/null +++ b/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/WithDiscoveryFlinkSQLEngine.scala @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.engine.flink + +import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.ha.client.{DiscoveryClient, DiscoveryClientProvider} + +trait WithDiscoveryFlinkSQLEngine { + + protected def namespace: String + + protected def conf: KyuubiConf + + def withDiscoveryClient(f: DiscoveryClient => Unit): Unit = { + DiscoveryClientProvider.withDiscoveryClient(conf)(f) + } + + def getFlinkEngineServiceUrl: String = { + var hostPort: Option[(String, Int)] = None + var retries = 0 + while (hostPort.isEmpty && retries < 10) { + withDiscoveryClient(client => hostPort = client.getServerHost(namespace)) + retries += 1 + Thread.sleep(1000L) + } + if (hostPort.isEmpty) { + throw new RuntimeException("Time out retrieving Flink engine service url.") + } + // delay the access to thrift service because the thrift service + // may not be ready although it's registered + Thread.sleep(3000L) + s"jdbc:hive2://${hostPort.get._1}:${hostPort.get._2}" + } +} diff --git a/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/WithFlinkSQLEngine.scala b/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/WithFlinkSQLEngine.scala deleted file mode 100644 index fbfb8df29..000000000 --- a/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/WithFlinkSQLEngine.scala +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kyuubi.engine.flink - -import scala.collection.JavaConverters._ - -import org.apache.flink.client.cli.{CustomCommandLine, DefaultCLI} -import org.apache.flink.configuration.{Configuration, RestOptions} -import org.apache.flink.runtime.minicluster.{MiniCluster, MiniClusterConfiguration} -import org.apache.flink.table.client.gateway.context.DefaultContext - -import org.apache.kyuubi.{KyuubiFunSuite, Utils} -import org.apache.kyuubi.config.KyuubiConf -import org.apache.kyuubi.engine.flink.util.TestUserClassLoaderJar - -trait WithFlinkSQLEngine extends KyuubiFunSuite { - - protected val flinkConfig = new Configuration() - protected var miniCluster: MiniCluster = _ - protected var engine: FlinkSQLEngine = _ - // conf will be loaded until start flink engine - def withKyuubiConf: Map[String, String] - val kyuubiConf: KyuubiConf = FlinkSQLEngine.kyuubiConf - - protected var connectionUrl: String = _ - - protected val GENERATED_UDF_CLASS: String = "LowerUDF" - - protected val GENERATED_UDF_CODE: String = - s""" - public class $GENERATED_UDF_CLASS extends org.apache.flink.table.functions.ScalarFunction { - public String eval(String str) { - return str.toLowerCase(); - } - } - """ - - override def beforeAll(): Unit = { - startMiniCluster() - startFlinkEngine() - super.beforeAll() - } - - override def afterAll(): Unit = { - super.afterAll() - stopFlinkEngine() - miniCluster.close() - } - - def startFlinkEngine(): Unit = { - withKyuubiConf.foreach { case (k, v) => - System.setProperty(k, v) - kyuubiConf.set(k, v) - } - val udfJar = TestUserClassLoaderJar.createJarFile( - Utils.createTempDir("test-jar").toFile, - "test-classloader-udf.jar", - GENERATED_UDF_CLASS, - GENERATED_UDF_CODE) - val engineContext = new DefaultContext( - List(udfJar.toURI.toURL).asJava, - flinkConfig, - List[CustomCommandLine](new DefaultCLI).asJava) - FlinkSQLEngine.startEngine(engineContext) - engine = FlinkSQLEngine.currentEngine.get - connectionUrl = engine.frontendServices.head.connectionUrl - } - - def stopFlinkEngine(): Unit = { - if (engine != null) { - engine.stop() - engine = null - } - } - - private def startMiniCluster(): Unit = { - val cfg = new MiniClusterConfiguration.Builder() - .setConfiguration(flinkConfig) - .setNumSlotsPerTaskManager(1) - .build - miniCluster = new MiniCluster(cfg) - miniCluster.start() - flinkConfig.setString(RestOptions.ADDRESS, miniCluster.getRestAddress.get().getHost) - flinkConfig.setInteger(RestOptions.PORT, miniCluster.getRestAddress.get().getPort) - } - - protected def getJdbcUrl: String = s"jdbc:hive2://$connectionUrl/;" - -} diff --git a/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/WithFlinkSQLEngineLocal.scala b/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/WithFlinkSQLEngineLocal.scala new file mode 100644 index 000000000..92c1bcd83 --- /dev/null +++ b/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/WithFlinkSQLEngineLocal.scala @@ -0,0 +1,228 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.engine.flink + +import java.io.{File, FilenameFilter} +import java.lang.ProcessBuilder.Redirect +import java.net.URI +import java.nio.file.{Files, Paths} + +import scala.collection.JavaConverters._ +import scala.collection.mutable.ArrayBuffer + +import org.apache.flink.configuration.{Configuration, RestOptions} +import org.apache.flink.runtime.minicluster.{MiniCluster, MiniClusterConfiguration} + +import org.apache.kyuubi.{KYUUBI_VERSION, KyuubiException, KyuubiFunSuite, SCALA_COMPILE_VERSION, Utils} +import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.config.KyuubiConf._ +import org.apache.kyuubi.ha.HighAvailabilityConf.HA_ADDRESSES +import org.apache.kyuubi.zookeeper.EmbeddedZookeeper +import org.apache.kyuubi.zookeeper.ZookeeperConf.{ZK_CLIENT_PORT, ZK_CLIENT_PORT_ADDRESS} + +trait WithFlinkSQLEngineLocal extends KyuubiFunSuite with WithFlinkTestResources { + + protected val flinkConfig = new Configuration() + + protected var miniCluster: MiniCluster = _ + + protected var engineProcess: Process = _ + + private var zkServer: EmbeddedZookeeper = _ + + protected val conf: KyuubiConf = FlinkSQLEngine.kyuubiConf + + protected def engineRefId: String + + def withKyuubiConf: Map[String, String] + + protected var connectionUrl: String = _ + + override def beforeAll(): Unit = { + withKyuubiConf.foreach { case (k, v) => + if (k.startsWith("flink.")) { + flinkConfig.setString(k.stripPrefix("flink."), v) + } + } + withKyuubiConf.foreach { case (k, v) => + System.setProperty(k, v) + conf.set(k, v) + } + + zkServer = new EmbeddedZookeeper() + conf.set(ZK_CLIENT_PORT, 0).set(ZK_CLIENT_PORT_ADDRESS, "localhost") + zkServer.initialize(conf) + zkServer.start() + conf.set(HA_ADDRESSES, zkServer.getConnectString) + + val envs = scala.collection.mutable.Map[String, String]() + val kyuubiExternals = Utils.getCodeSourceLocation(getClass) + .split("externals").head + val flinkHome = { + val candidates = Paths.get(kyuubiExternals, "externals", "kyuubi-download", "target") + .toFile.listFiles(f => f.getName.contains("flink")) + if (candidates == null) None else candidates.map(_.toPath).headOption + } + if (flinkHome.isDefined) { + envs("FLINK_HOME") = flinkHome.get.toString + envs("FLINK_CONF_DIR") = Paths.get(flinkHome.get.toString, "conf").toString + } + envs("JAVA_HOME") = System.getProperty("java.home") + envs("JAVA_EXEC") = Paths.get(envs("JAVA_HOME"), "bin", "java").toString + + startMiniCluster() + startFlinkEngine(envs.toMap) + super.beforeAll() + } + + override def afterAll(): Unit = { + super.afterAll() + if (engineProcess != null) { + engineProcess.destroy() + engineProcess = null + } + if (miniCluster != null) { + miniCluster.close() + miniCluster = null + } + if (zkServer != null) { + zkServer.stop() + zkServer = null + } + } + + def startFlinkEngine(envs: Map[String, String]): Unit = { + val flinkHome = envs("FLINK_HOME") + val processBuilder: ProcessBuilder = new ProcessBuilder + processBuilder.environment().putAll(envs.asJava) + + conf.set(ENGINE_FLINK_EXTRA_CLASSPATH, udfJar.getAbsolutePath) + val command = new ArrayBuffer[String]() + + command += envs("JAVA_EXEC") + + val memory = conf.get(ENGINE_FLINK_MEMORY) + command += s"-Xmx$memory" + val javaOptions = conf.get(ENGINE_FLINK_JAVA_OPTIONS) + if (javaOptions.isDefined) { + command += javaOptions.get + } + + command += "-cp" + val classpathEntries = new java.util.LinkedHashSet[String] + // flink engine runtime jar + mainResource(envs).foreach(classpathEntries.add) + // flink sql jars + Paths.get(flinkHome) + .resolve("opt") + .toFile + .listFiles(new FilenameFilter { + override def accept(dir: File, name: String): Boolean = { + name.toLowerCase.startsWith("flink-sql-client") || + name.toLowerCase.startsWith("flink-sql-gateway") + } + }).foreach(jar => classpathEntries.add(jar.getAbsolutePath)) + + // jars from flink lib + classpathEntries.add(s"$flinkHome${File.separator}lib${File.separator}*") + + // classpath contains flink configurations, default to flink.home/conf + classpathEntries.add(envs.getOrElse("FLINK_CONF_DIR", "")) + // classpath contains hadoop configurations + val cp = System.getProperty("java.class.path") + // exclude kyuubi flink engine jar that has SPI for EmbeddedExecutorFactory + // which can't be initialized on the client side + val hadoopJars = cp.split(":").filter(s => !s.contains("flink")) + hadoopJars.foreach(classpathEntries.add) + val extraCp = conf.get(ENGINE_FLINK_EXTRA_CLASSPATH) + extraCp.foreach(classpathEntries.add) + if (hadoopJars.isEmpty && extraCp.isEmpty) { + mainResource(envs).foreach { path => + val devHadoopJars = Paths.get(path).getParent + .resolve(s"scala-$SCALA_COMPILE_VERSION") + .resolve("jars") + if (!Files.exists(devHadoopJars)) { + throw new KyuubiException(s"The path $devHadoopJars does not exists. " + + s"Please set FLINK_HADOOP_CLASSPATH or ${ENGINE_FLINK_EXTRA_CLASSPATH.key}" + + s" for configuring location of hadoop client jars, etc.") + } + classpathEntries.add(s"$devHadoopJars${File.separator}*") + } + } + command += classpathEntries.asScala.mkString(File.pathSeparator) + command += "org.apache.kyuubi.engine.flink.FlinkSQLEngine" + + conf.getAll.foreach { case (k, v) => + command += "--conf" + command += s"$k=$v" + } + + processBuilder.command(command.toList.asJava) + processBuilder.redirectOutput(Redirect.INHERIT) + processBuilder.redirectError(Redirect.INHERIT) + + info(s"staring flink local engine...") + engineProcess = processBuilder.start() + } + + private def startMiniCluster(): Unit = { + val cfg = new MiniClusterConfiguration.Builder() + .setConfiguration(flinkConfig) + .setNumSlotsPerTaskManager(1) + .setNumTaskManagers(2) + .build + miniCluster = new MiniCluster(cfg) + miniCluster.start() + flinkConfig.setString(RestOptions.ADDRESS, miniCluster.getRestAddress.get().getHost) + flinkConfig.setInteger(RestOptions.PORT, miniCluster.getRestAddress.get().getPort) + } + + protected def getJdbcUrl: String = s"jdbc:hive2://$connectionUrl/;" + + def mainResource(env: Map[String, String]): Option[String] = { + val module = "kyuubi-flink-sql-engine" + val shortName = "flink" + // 1. get the main resource jar for user specified config first + val jarName = s"${module}_$SCALA_COMPILE_VERSION-$KYUUBI_VERSION.jar" + conf.getOption(s"kyuubi.session.engine.$shortName.main.resource").filter { + userSpecified => + // skip check exist if not local file. + val uri = new URI(userSpecified) + val schema = if (uri.getScheme != null) uri.getScheme else "file" + schema match { + case "file" => Files.exists(Paths.get(userSpecified)) + case _ => true + } + }.orElse { + // 2. get the main resource jar from system build default + env.get(KYUUBI_HOME).toSeq + .flatMap { p => + Seq( + Paths.get(p, "externals", "engines", shortName, jarName), + Paths.get(p, "externals", module, "target", jarName)) + } + .find(Files.exists(_)).map(_.toAbsolutePath.toFile.getCanonicalPath) + }.orElse { + // 3. get the main resource from dev environment + val cwd = Utils.getCodeSourceLocation(getClass).split("externals") + assert(cwd.length > 1) + Option(Paths.get(cwd.head, "externals", module, "target", jarName)) + .map(_.toAbsolutePath.toFile.getCanonicalPath) + } + } +} diff --git a/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/WithFlinkSQLEngineOnYarn.scala b/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/WithFlinkSQLEngineOnYarn.scala new file mode 100644 index 000000000..49fb947a3 --- /dev/null +++ b/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/WithFlinkSQLEngineOnYarn.scala @@ -0,0 +1,266 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.engine.flink + +import java.io.{File, FilenameFilter, FileWriter} +import java.lang.ProcessBuilder.Redirect +import java.net.URI +import java.nio.file.{Files, Paths} + +import scala.collection.JavaConverters._ +import scala.collection.mutable.{ArrayBuffer, ListBuffer} + +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.hdfs.MiniDFSCluster +import org.apache.hadoop.yarn.conf.YarnConfiguration +import org.apache.hadoop.yarn.server.MiniYARNCluster + +import org.apache.kyuubi.{KYUUBI_VERSION, KyuubiFunSuite, SCALA_COMPILE_VERSION, Utils} +import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.config.KyuubiConf.{ENGINE_FLINK_APPLICATION_JARS, KYUUBI_HOME} +import org.apache.kyuubi.ha.HighAvailabilityConf.HA_ADDRESSES +import org.apache.kyuubi.zookeeper.EmbeddedZookeeper +import org.apache.kyuubi.zookeeper.ZookeeperConf.{ZK_CLIENT_PORT, ZK_CLIENT_PORT_ADDRESS} + +trait WithFlinkSQLEngineOnYarn extends KyuubiFunSuite with WithFlinkTestResources { + + protected def engineRefId: String + + protected val conf: KyuubiConf = new KyuubiConf(false) + + private var hdfsCluster: MiniDFSCluster = _ + + private var yarnCluster: MiniYARNCluster = _ + + private var zkServer: EmbeddedZookeeper = _ + + def withKyuubiConf: Map[String, String] = testExtraConf + + private val yarnConf: YarnConfiguration = { + val yarnConfig = new YarnConfiguration() + + // configurations copied from org.apache.flink.yarn.YarnTestBase + yarnConfig.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 32) + yarnConfig.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 4096) + + yarnConfig.setBoolean(YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME, true) + yarnConfig.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2) + yarnConfig.setInt(YarnConfiguration.RM_MAX_COMPLETED_APPLICATIONS, 2) + yarnConfig.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, 4) + yarnConfig.setInt(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC, 3600) + yarnConfig.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, false) + // memory is overwritten in the MiniYARNCluster. + // so we have to change the number of cores for testing. + yarnConfig.setInt(YarnConfiguration.NM_VCORES, 666) + yarnConfig.setFloat(YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE, 99.0f) + yarnConfig.setInt(YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS, 1000) + yarnConfig.setInt(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, 5000) + + // capacity-scheduler.xml is missing in hadoop-client-minicluster so this is a workaround + yarnConfig.set("yarn.scheduler.capacity.root.queues", "default,four_cores_queue") + + yarnConfig.setInt("yarn.scheduler.capacity.root.default.capacity", 100) + yarnConfig.setFloat("yarn.scheduler.capacity.root.default.user-limit-factor", 1) + yarnConfig.setInt("yarn.scheduler.capacity.root.default.maximum-capacity", 100) + yarnConfig.set("yarn.scheduler.capacity.root.default.state", "RUNNING") + yarnConfig.set("yarn.scheduler.capacity.root.default.acl_submit_applications", "*") + yarnConfig.set("yarn.scheduler.capacity.root.default.acl_administer_queue", "*") + + yarnConfig.setInt("yarn.scheduler.capacity.root.four_cores_queue.maximum-capacity", 100) + yarnConfig.setInt("yarn.scheduler.capacity.root.four_cores_queue.maximum-applications", 10) + yarnConfig.setInt("yarn.scheduler.capacity.root.four_cores_queue.maximum-allocation-vcores", 4) + yarnConfig.setFloat("yarn.scheduler.capacity.root.four_cores_queue.user-limit-factor", 1) + yarnConfig.set("yarn.scheduler.capacity.root.four_cores_queue.acl_submit_applications", "*") + yarnConfig.set("yarn.scheduler.capacity.root.four_cores_queue.acl_administer_queue", "*") + + yarnConfig.setInt("yarn.scheduler.capacity.node-locality-delay", -1) + // Set bind host to localhost to avoid java.net.BindException + yarnConfig.set(YarnConfiguration.RM_BIND_HOST, "localhost") + yarnConfig.set(YarnConfiguration.NM_BIND_HOST, "localhost") + + yarnConfig + } + + override def beforeAll(): Unit = { + zkServer = new EmbeddedZookeeper() + conf.set(ZK_CLIENT_PORT, 0).set(ZK_CLIENT_PORT_ADDRESS, "localhost") + zkServer.initialize(conf) + zkServer.start() + conf.set(HA_ADDRESSES, zkServer.getConnectString) + + hdfsCluster = new MiniDFSCluster.Builder(new Configuration) + .numDataNodes(1) + .checkDataNodeAddrConfig(true) + .checkDataNodeHostConfig(true) + .build() + + val hdfsServiceUrl = s"hdfs://localhost:${hdfsCluster.getNameNodePort}" + yarnConf.set("fs.defaultFS", hdfsServiceUrl) + yarnConf.addResource(hdfsCluster.getConfiguration(0)) + + val cp = System.getProperty("java.class.path") + // exclude kyuubi flink engine jar that has SPI for EmbeddedExecutorFactory + // which can't be initialized on the client side + val hadoopJars = cp.split(":").filter(s => !s.contains("flink") && !s.contains("log4j")) + val hadoopClasspath = hadoopJars.mkString(":") + yarnConf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH, hadoopClasspath) + + yarnCluster = new MiniYARNCluster("flink-engine-cluster", 1, 1, 1) + yarnCluster.init(yarnConf) + yarnCluster.start() + + val hadoopConfDir = Utils.createTempDir().toFile + val writer = new FileWriter(new File(hadoopConfDir, "core-site.xml")) + yarnCluster.getConfig.writeXml(writer) + writer.close() + + val envs = scala.collection.mutable.Map[String, String]() + val kyuubiExternals = Utils.getCodeSourceLocation(getClass) + .split("externals").head + val flinkHome = { + val candidates = Paths.get(kyuubiExternals, "externals", "kyuubi-download", "target") + .toFile.listFiles(f => f.getName.contains("flink")) + if (candidates == null) None else candidates.map(_.toPath).headOption + } + if (flinkHome.isDefined) { + envs("FLINK_HOME") = flinkHome.get.toString + envs("FLINK_CONF_DIR") = Paths.get(flinkHome.get.toString, "conf").toString + } + envs("HADOOP_CLASSPATH") = hadoopClasspath + envs("HADOOP_CONF_DIR") = hadoopConfDir.getAbsolutePath + + startFlinkEngine(envs.toMap) + + super.beforeAll() + } + + private def startFlinkEngine(envs: Map[String, String]): Unit = { + val processBuilder: ProcessBuilder = new ProcessBuilder + processBuilder.environment().putAll(envs.asJava) + + conf.set(ENGINE_FLINK_APPLICATION_JARS, udfJar.getAbsolutePath) + val flinkExtraJars = extraFlinkJars(envs("FLINK_HOME")) + val command = new ArrayBuffer[String]() + + command += s"${envs("FLINK_HOME")}${File.separator}bin/flink" + command += "run-application" + command += "-t" + command += "yarn-application" + command += s"-Dyarn.ship-files=${flinkExtraJars.mkString(";")}" + command += s"-Dyarn.application.name=kyuubi_user_flink_paul" + command += s"-Dyarn.tags=KYUUBI,$engineRefId" + command += "-Djobmanager.memory.process.size=1g" + command += "-Dtaskmanager.memory.process.size=1g" + command += "-Dcontainerized.master.env.FLINK_CONF_DIR=." + command += "-Dcontainerized.taskmanager.env.FLINK_CONF_DIR=." + command += s"-Dcontainerized.master.env.HADOOP_CONF_DIR=${envs("HADOOP_CONF_DIR")}" + command += s"-Dcontainerized.taskmanager.env.HADOOP_CONF_DIR=${envs("HADOOP_CONF_DIR")}" + command += "-Dexecution.target=yarn-application" + command += "-c" + command += "org.apache.kyuubi.engine.flink.FlinkSQLEngine" + command += s"${mainResource(envs).get}" + + for ((k, v) <- withKyuubiConf) { + conf.set(k, v) + } + + for ((k, v) <- conf.getAll) { + command += "--conf" + command += s"$k=$v" + } + + processBuilder.command(command.toList.asJava) + processBuilder.redirectOutput(Redirect.INHERIT) + processBuilder.redirectError(Redirect.INHERIT) + + info(s"staring flink yarn-application cluster for engine $engineRefId..") + val process = processBuilder.start() + process.waitFor() + info(s"flink yarn-application cluster for engine $engineRefId has started") + } + + def extraFlinkJars(flinkHome: String): Array[String] = { + // locate flink sql jars + val flinkExtraJars = new ListBuffer[String] + val flinkSQLJars = Paths.get(flinkHome) + .resolve("opt") + .toFile + .listFiles(new FilenameFilter { + override def accept(dir: File, name: String): Boolean = { + name.toLowerCase.startsWith("flink-sql-client") || + name.toLowerCase.startsWith("flink-sql-gateway") + } + }).map(f => f.getAbsolutePath).sorted + flinkExtraJars ++= flinkSQLJars + + val userJars = conf.get(ENGINE_FLINK_APPLICATION_JARS) + userJars.foreach(jars => flinkExtraJars ++= jars.split(",")) + flinkExtraJars.toArray + } + + /** + * Copied form org.apache.kyuubi.engine.ProcBuilder + * The engine jar or other runnable jar containing the main method + */ + def mainResource(env: Map[String, String]): Option[String] = { + // 1. get the main resource jar for user specified config first + val module = "kyuubi-flink-sql-engine" + val shortName = "flink" + val jarName = s"${module}_$SCALA_COMPILE_VERSION-$KYUUBI_VERSION.jar" + conf.getOption(s"kyuubi.session.engine.$shortName.main.resource").filter { userSpecified => + // skip check exist if not local file. + val uri = new URI(userSpecified) + val schema = if (uri.getScheme != null) uri.getScheme else "file" + schema match { + case "file" => Files.exists(Paths.get(userSpecified)) + case _ => true + } + }.orElse { + // 2. get the main resource jar from system build default + env.get(KYUUBI_HOME).toSeq + .flatMap { p => + Seq( + Paths.get(p, "externals", "engines", shortName, jarName), + Paths.get(p, "externals", module, "target", jarName)) + } + .find(Files.exists(_)).map(_.toAbsolutePath.toFile.getCanonicalPath) + }.orElse { + // 3. get the main resource from dev environment + val cwd = Utils.getCodeSourceLocation(getClass).split("externals") + assert(cwd.length > 1) + Option(Paths.get(cwd.head, "externals", module, "target", jarName)) + .map(_.toAbsolutePath.toFile.getCanonicalPath) + } + } + + override def afterAll(): Unit = { + super.afterAll() + if (yarnCluster != null) { + yarnCluster.stop() + yarnCluster = null + } + if (hdfsCluster != null) { + hdfsCluster.shutdown() + hdfsCluster = null + } + if (zkServer != null) { + zkServer.stop() + zkServer = null + } + } +} diff --git a/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/WithFlinkTestResources.scala b/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/WithFlinkTestResources.scala new file mode 100644 index 000000000..3b1d65cb2 --- /dev/null +++ b/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/WithFlinkTestResources.scala @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.engine.flink + +import java.io.File + +import org.apache.kyuubi.Utils +import org.apache.kyuubi.engine.flink.util.TestUserClassLoaderJar + +trait WithFlinkTestResources { + + protected val GENERATED_UDF_CLASS: String = "LowerUDF" + + protected val GENERATED_UDF_CODE: String = + s""" + public class $GENERATED_UDF_CLASS extends org.apache.flink.table.functions.ScalarFunction { + public String eval(String str) { + return str.toLowerCase(); + } + } + """ + + protected val udfJar: File = TestUserClassLoaderJar.createJarFile( + Utils.createTempDir("test-jar").toFile, + "test-classloader-udf.jar", + GENERATED_UDF_CLASS, + GENERATED_UDF_CODE) + + protected val savepointDir: File = Utils.createTempDir("savepoints").toFile + + protected val testExtraConf: Map[String, String] = Map( + "flink.pipeline.name" -> "test-job", + "flink.state.savepoints.dir" -> savepointDir.toURI.toString) +} diff --git a/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/operation/FlinkOperationLocalSuite.scala b/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/operation/FlinkOperationLocalSuite.scala new file mode 100644 index 000000000..279cbea22 --- /dev/null +++ b/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/operation/FlinkOperationLocalSuite.scala @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.engine.flink.operation + +import java.util.UUID + +import org.apache.kyuubi.{KYUUBI_VERSION, Utils} +import org.apache.kyuubi.config.KyuubiConf._ +import org.apache.kyuubi.config.KyuubiReservedKeys.KYUUBI_SESSION_USER_KEY +import org.apache.kyuubi.engine.ShareLevel +import org.apache.kyuubi.engine.flink.{WithDiscoveryFlinkSQLEngine, WithFlinkSQLEngineLocal} +import org.apache.kyuubi.ha.HighAvailabilityConf.{HA_ENGINE_REF_ID, HA_NAMESPACE} +import org.apache.kyuubi.operation.NoneMode + +class FlinkOperationLocalSuite extends FlinkOperationSuite + with WithDiscoveryFlinkSQLEngine with WithFlinkSQLEngineLocal { + + protected def jdbcUrl: String = getFlinkEngineServiceUrl + + override def withKyuubiConf: Map[String, String] = { + Map( + "flink.execution.target" -> "remote", + "flink.high-availability.cluster-id" -> "flink-mini-cluster", + "flink.app.name" -> "kyuubi_connection_flink_paul", + HA_NAMESPACE.key -> namespace, + HA_ENGINE_REF_ID.key -> engineRefId, + ENGINE_TYPE.key -> "FLINK_SQL", + ENGINE_SHARE_LEVEL.key -> shareLevel, + OPERATION_PLAN_ONLY_MODE.key -> NoneMode.name, + KYUUBI_SESSION_USER_KEY -> "paullin") ++ testExtraConf + } + + override protected def engineRefId: String = UUID.randomUUID().toString + + def namespace: String = "/kyuubi/flink-local-engine-test" + + def shareLevel: String = ShareLevel.USER.toString + + def engineType: String = "flink" + + test("execute statement - kyuubi defined functions") { + withJdbcStatement() { statement => + var resultSet = statement.executeQuery("select kyuubi_version() as kyuubi_version") + assert(resultSet.next()) + assert(resultSet.getString(1) === KYUUBI_VERSION) + + resultSet = statement.executeQuery("select kyuubi_engine_name() as engine_name") + assert(resultSet.next()) + assert(resultSet.getString(1).equals(s"kyuubi_connection_flink_paul")) + + resultSet = statement.executeQuery("select kyuubi_engine_id() as engine_id") + assert(resultSet.next()) + assert(resultSet.getString(1) === "flink-mini-cluster") + + resultSet = statement.executeQuery("select kyuubi_system_user() as `system_user`") + assert(resultSet.next()) + assert(resultSet.getString(1) === Utils.currentUser) + + resultSet = statement.executeQuery("select kyuubi_session_user() as `session_user`") + assert(resultSet.next()) + assert(resultSet.getString(1) === "paullin") + } + } +} diff --git a/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/operation/FlinkOperationOnYarnSuite.scala b/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/operation/FlinkOperationOnYarnSuite.scala new file mode 100644 index 000000000..401c3b0bd --- /dev/null +++ b/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/operation/FlinkOperationOnYarnSuite.scala @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.engine.flink.operation + +import java.util.UUID + +import org.apache.kyuubi.{KYUUBI_VERSION, Utils} +import org.apache.kyuubi.config.KyuubiConf.{ENGINE_SHARE_LEVEL, ENGINE_TYPE} +import org.apache.kyuubi.config.KyuubiReservedKeys.KYUUBI_SESSION_USER_KEY +import org.apache.kyuubi.engine.ShareLevel +import org.apache.kyuubi.engine.flink.{WithDiscoveryFlinkSQLEngine, WithFlinkSQLEngineOnYarn} +import org.apache.kyuubi.ha.HighAvailabilityConf.{HA_ENGINE_REF_ID, HA_NAMESPACE} + +class FlinkOperationOnYarnSuite extends FlinkOperationSuite + with WithDiscoveryFlinkSQLEngine with WithFlinkSQLEngineOnYarn { + + protected def jdbcUrl: String = getFlinkEngineServiceUrl + + override def withKyuubiConf: Map[String, String] = { + Map( + HA_NAMESPACE.key -> namespace, + HA_ENGINE_REF_ID.key -> engineRefId, + ENGINE_TYPE.key -> "FLINK_SQL", + ENGINE_SHARE_LEVEL.key -> shareLevel, + KYUUBI_SESSION_USER_KEY -> "paullin") ++ testExtraConf + } + + override protected def engineRefId: String = UUID.randomUUID().toString + + def namespace: String = "/kyuubi/flink-yarn-application-test" + + def shareLevel: String = ShareLevel.USER.toString + + def engineType: String = "flink" + + test("execute statement - kyuubi defined functions") { + withJdbcStatement() { statement => + var resultSet = statement.executeQuery("select kyuubi_version() as kyuubi_version") + assert(resultSet.next()) + assert(resultSet.getString(1) === KYUUBI_VERSION) + + resultSet = statement.executeQuery("select kyuubi_engine_name() as engine_name") + assert(resultSet.next()) + assert(resultSet.getString(1).equals(s"kyuubi_user_flink_paul")) + + resultSet = statement.executeQuery("select kyuubi_engine_id() as engine_id") + assert(resultSet.next()) + assert(resultSet.getString(1).startsWith("application_")) + + resultSet = statement.executeQuery("select kyuubi_system_user() as `system_user`") + assert(resultSet.next()) + assert(resultSet.getString(1) === Utils.currentUser) + + resultSet = statement.executeQuery("select kyuubi_session_user() as `session_user`") + assert(resultSet.next()) + assert(resultSet.getString(1) === "paullin") + } + } +} diff --git a/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/operation/FlinkOperationSuite.scala b/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/operation/FlinkOperationSuite.scala index c75124c39..9469cf286 100644 --- a/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/operation/FlinkOperationSuite.scala +++ b/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/operation/FlinkOperationSuite.scala @@ -17,43 +17,29 @@ package org.apache.kyuubi.engine.flink.operation +import java.nio.file.Paths import java.sql.DatabaseMetaData import java.util.UUID import scala.collection.JavaConverters._ import org.apache.flink.api.common.JobID +import org.apache.flink.configuration.PipelineOptions import org.apache.flink.table.types.logical.LogicalTypeRoot import org.apache.hive.service.rpc.thrift._ -import org.scalatest.concurrent.PatienceConfiguration.Timeout -import org.scalatest.time.SpanSugar._ import org.apache.kyuubi.Utils import org.apache.kyuubi.config.KyuubiConf._ -import org.apache.kyuubi.engine.flink.FlinkEngineUtils._ -import org.apache.kyuubi.engine.flink.WithFlinkSQLEngine +import org.apache.kyuubi.engine.flink.FlinkEngineUtils.FLINK_RUNTIME_VERSION +import org.apache.kyuubi.engine.flink.WithFlinkTestResources import org.apache.kyuubi.engine.flink.result.Constants import org.apache.kyuubi.engine.flink.util.TestUserClassLoaderJar -import org.apache.kyuubi.jdbc.hive.KyuubiStatement -import org.apache.kyuubi.operation.{HiveJDBCTestHelper, NoneMode} +import org.apache.kyuubi.jdbc.hive.{KyuubiSQLException, KyuubiStatement} +import org.apache.kyuubi.jdbc.hive.common.TimestampTZ +import org.apache.kyuubi.operation.HiveJDBCTestHelper import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant._ -import org.apache.kyuubi.service.ServiceState._ -class FlinkOperationSuite extends WithFlinkSQLEngine with HiveJDBCTestHelper { - override def withKyuubiConf: Map[String, String] = - Map(OPERATION_PLAN_ONLY_MODE.key -> NoneMode.name) - - override protected def jdbcUrl: String = - s"jdbc:hive2://${engine.frontendServices.head.connectionUrl}/;" - - ignore("release session if shared level is CONNECTION") { - logger.info(s"jdbc url is $jdbcUrl") - assert(engine.getServiceState == STARTED) - withJdbcStatement() { _ => } - eventually(Timeout(20.seconds)) { - assert(engine.getServiceState == STOPPED) - } - } +abstract class FlinkOperationSuite extends HiveJDBCTestHelper with WithFlinkTestResources { test("get catalogs") { withJdbcStatement() { statement => @@ -649,6 +635,62 @@ class FlinkOperationSuite extends WithFlinkSQLEngine with HiveJDBCTestHelper { } } + test("execute statement - show/stop jobs") { + if (FLINK_RUNTIME_VERSION >= "1.17") { + // use a bigger value to ensure all tasks of the streaming query run until + // we explicitly stop the job. + withSessionConf()(Map(ENGINE_FLINK_MAX_ROWS.key -> "10000"))(Map.empty) { + withMultipleConnectionJdbcStatement()({ statement => + statement.executeQuery( + "create table tbl_a (a int) with (" + + "'connector' = 'datagen', " + + "'rows-per-second'='10')") + statement.executeQuery("create table tbl_b (a int) with ('connector' = 'blackhole')") + val insertResult1 = statement.executeQuery("insert into tbl_b select * from tbl_a") + assert(insertResult1.next()) + val jobId1 = insertResult1.getString(1) + + Thread.sleep(5000) + + val showResult = statement.executeQuery("show jobs") + val metadata = showResult.getMetaData + assert(metadata.getColumnName(1) === "job id") + assert(metadata.getColumnType(1) === java.sql.Types.VARCHAR) + assert(metadata.getColumnName(2) === "job name") + assert(metadata.getColumnType(2) === java.sql.Types.VARCHAR) + assert(metadata.getColumnName(3) === "status") + assert(metadata.getColumnType(3) === java.sql.Types.VARCHAR) + assert(metadata.getColumnName(4) === "start time") + assert(metadata.getColumnType(4) === java.sql.Types.OTHER) + + var isFound = false + while (showResult.next()) { + if (showResult.getString(1) === jobId1) { + isFound = true + assert(showResult.getString(2) === "test-job") + assert(showResult.getString(3) === "RUNNING") + assert(showResult.getObject(4).isInstanceOf[TimestampTZ]) + } + } + assert(isFound) + + val stopResult1 = statement.executeQuery(s"stop job '$jobId1'") + assert(stopResult1.next()) + assert(stopResult1.getString(1) === "OK") + + val insertResult2 = statement.executeQuery("insert into tbl_b select * from tbl_a") + assert(insertResult2.next()) + val jobId2 = insertResult2.getString(1) + + val stopResult2 = statement.executeQuery(s"stop job '$jobId2' with savepoint") + assert(stopResult2.getMetaData.getColumnName(1).equals("savepoint path")) + assert(stopResult2.next()) + assert(Paths.get(stopResult2.getString(1)).getFileName.toString.startsWith("savepoint-")) + }) + } + } + } + test("execute statement - select column name with dots") { withJdbcStatement() { statement => val resultSet = statement.executeQuery("select 'tmp.hello'") @@ -756,30 +798,54 @@ class FlinkOperationSuite extends WithFlinkSQLEngine with HiveJDBCTestHelper { } } - test("execute statement - select array") { + test("execute statement - select timestamp with local time zone") { + withJdbcStatement() { statement => + statement.executeQuery("CREATE VIEW T1 AS SELECT TO_TIMESTAMP_LTZ(4001, 3)") + statement.executeQuery("SET 'table.local-time-zone' = 'UTC'") + val resultSetUTC = statement.executeQuery("SELECT * FROM T1") + val metaData = resultSetUTC.getMetaData + assert(metaData.getColumnType(1) === java.sql.Types.OTHER) + assert(resultSetUTC.next()) + assert(resultSetUTC.getString(1) === "1970-01-01 00:00:04.001 UTC") + + statement.executeQuery("SET 'table.local-time-zone' = 'America/Los_Angeles'") + val resultSetPST = statement.executeQuery("SELECT * FROM T1") + assert(resultSetPST.next()) + assert(resultSetPST.getString(1) === "1969-12-31 16:00:04.001 America/Los_Angeles") + } + } + + test("execute statement - select time") { withJdbcStatement() { statement => val resultSet = - statement.executeQuery("select array ['v1', 'v2', 'v3']") + statement.executeQuery( + "select time '00:00:03', time '00:00:05.123456789'") + val metaData = resultSet.getMetaData + assert(metaData.getColumnType(1) === java.sql.Types.VARCHAR) + assert(metaData.getColumnType(2) === java.sql.Types.VARCHAR) + assert(resultSet.next()) + assert(resultSet.getString(1) == "00:00:03") + assert(resultSet.getString(2) == "00:00:05.123") + } + } + + test("execute statement - select array") { + withJdbcStatement() { statement => + val resultSet = statement.executeQuery("select array ['v1', 'v2', 'v3']") val metaData = resultSet.getMetaData assert(metaData.getColumnType(1) === java.sql.Types.ARRAY) assert(resultSet.next()) - if (isFlinkVersionEqualTo("1.14")) { - val expected = """["v1","v2","v3"]""" - assert(resultSet.getObject(1).toString == expected) - } - if (isFlinkVersionAtLeast("1.15")) { - val expected = "[v1,v2,v3]" - assert(resultSet.getObject(1).toString == expected) - } + val expected = "[\"v1\",\"v2\",\"v3\"]" + assert(resultSet.getObject(1).toString == expected) } } test("execute statement - select map") { withJdbcStatement() { statement => - val resultSet = - statement.executeQuery("select map ['k1', 'v1', 'k2', 'v2']") + val resultSet = statement.executeQuery("select map ['k1', 'v1', 'k2', 'v2']") assert(resultSet.next()) - assert(resultSet.getString(1) == "{k1=v1, k2=v2}") + assert(List("{k1=v1, k2=v2}", "{k2=v2, k1=v1}") + .contains(resultSet.getString(1))) val metaData = resultSet.getMetaData assert(metaData.getColumnType(1) === java.sql.Types.JAVA_OBJECT) } @@ -787,17 +853,10 @@ class FlinkOperationSuite extends WithFlinkSQLEngine with HiveJDBCTestHelper { test("execute statement - select row") { withJdbcStatement() { statement => - val resultSet = - statement.executeQuery("select (1, '2', true)") + val resultSet = statement.executeQuery("select (1, '2', true)") assert(resultSet.next()) - if (isFlinkVersionEqualTo("1.14")) { - val expected = """{INT NOT NULL:1,CHAR(1) NOT NULL:"2",BOOLEAN NOT NULL:true}""" - assert(resultSet.getString(1) == expected) - } - if (isFlinkVersionAtLeast("1.15")) { - val expected = """{INT NOT NULL:1,CHAR(1) NOT NULL:2,BOOLEAN NOT NULL:true}""" - assert(resultSet.getString(1) == expected) - } + val expected = """{INT NOT NULL:1,CHAR(1) NOT NULL:"2",BOOLEAN NOT NULL:true}""" + assert(resultSet.getString(1) == expected) val metaData = resultSet.getMetaData assert(metaData.getColumnType(1) === java.sql.Types.STRUCT) } @@ -807,25 +866,30 @@ class FlinkOperationSuite extends WithFlinkSQLEngine with HiveJDBCTestHelper { withJdbcStatement() { statement => val resultSet = statement.executeQuery("select encode('kyuubi', 'UTF-8')") assert(resultSet.next()) - if (isFlinkVersionEqualTo("1.14")) { - assert(resultSet.getString(1) == "kyuubi") - } - if (isFlinkVersionAtLeast("1.15")) { - // TODO: validate table results after FLINK-28882 is resolved - assert(resultSet.getString(1) == "k") - } + // TODO: validate table results after FLINK-28882 is resolved + assert(resultSet.getString(1) == "k") + val metaData = resultSet.getMetaData + assert(metaData.getColumnType(1) === java.sql.Types.BINARY) + } + } + + test("execute statement - select varbinary") { + withJdbcStatement() { statement => + val resultSet = statement.executeQuery("select cast('kyuubi' as varbinary)") + assert(resultSet.next()) + assert(resultSet.getString(1) == "kyuubi") val metaData = resultSet.getMetaData assert(metaData.getColumnType(1) === java.sql.Types.BINARY) } } test("execute statement - select float") { - withJdbcStatement()({ statement => + withJdbcStatement() { statement => val resultSet = statement.executeQuery("SELECT cast(0.1 as float)") assert(resultSet.next()) assert(resultSet.getString(1) == "0.1") assert(resultSet.getFloat(1) == 0.1f) - }) + } } test("execute statement - select count") { @@ -876,20 +940,15 @@ class FlinkOperationSuite extends WithFlinkSQLEngine with HiveJDBCTestHelper { } test("execute statement - create/drop catalog") { - withJdbcStatement()({ statement => - val createResult = { + withJdbcStatement() { statement => + val createResult = statement.executeQuery("create catalog cat_a with ('type'='generic_in_memory')") - } - if (isFlinkVersionAtLeast("1.15")) { - assert(createResult.next()) - assert(createResult.getString(1) === "OK") - } + assert(createResult.next()) + assert(createResult.getString(1) === "OK") val dropResult = statement.executeQuery("drop catalog cat_a") - if (isFlinkVersionAtLeast("1.15")) { - assert(dropResult.next()) - assert(dropResult.getString(1) === "OK") - } - }) + assert(dropResult.next()) + assert(dropResult.getString(1) === "OK") + } } test("execute statement - set/get catalog") { @@ -903,36 +962,31 @@ class FlinkOperationSuite extends WithFlinkSQLEngine with HiveJDBCTestHelper { statement.getConnection.setCatalog("cat_a") val changedCatalog = statement.getConnection.getCatalog assert(changedCatalog == "cat_a") + statement.getConnection.setCatalog("default_catalog") assert(statement.execute("drop catalog cat_a")) } } } test("execute statement - create/alter/drop database") { - withJdbcStatement()({ statement => + withJdbcStatement() { statement => val createResult = statement.executeQuery("create database db_a") - if (isFlinkVersionAtLeast("1.15")) { - assert(createResult.next()) - assert(createResult.getString(1) === "OK") - } + assert(createResult.next()) + assert(createResult.getString(1) === "OK") val alterResult = statement.executeQuery("alter database db_a set ('k1' = 'v1')") - if (isFlinkVersionAtLeast("1.15")) { - assert(alterResult.next()) - assert(alterResult.getString(1) === "OK") - } + assert(alterResult.next()) + assert(alterResult.getString(1) === "OK") val dropResult = statement.executeQuery("drop database db_a") - if (isFlinkVersionAtLeast("1.15")) { - assert(dropResult.next()) - assert(dropResult.getString(1) === "OK") - } - }) + assert(dropResult.next()) + assert(dropResult.getString(1) === "OK") + } } test("execute statement - set/get database") { withSessionConf()( Map(ENGINE_OPERATION_CONVERT_CATALOG_DATABASE_ENABLED.key -> "true"))( Map.empty) { - withJdbcStatement()({ statement => + withJdbcStatement() { statement => statement.executeQuery("create database db_a") val schema = statement.getConnection.getSchema assert(schema == "default_database") @@ -940,102 +994,113 @@ class FlinkOperationSuite extends WithFlinkSQLEngine with HiveJDBCTestHelper { val changedSchema = statement.getConnection.getSchema assert(changedSchema == "db_a") assert(statement.execute("drop database db_a")) - }) + } } } test("execute statement - create/alter/drop table") { - withJdbcStatement()({ statement => - val createResult = { + withJdbcStatement() { statement => + val createResult = statement.executeQuery("create table tbl_a (a string) with ('connector' = 'blackhole')") - } - if (isFlinkVersionAtLeast("1.15")) { - assert(createResult.next()) - assert(createResult.getString(1) === "OK") - } + assert(createResult.next()) + assert(createResult.getString(1) === "OK") val alterResult = statement.executeQuery("alter table tbl_a rename to tbl_b") - if (isFlinkVersionAtLeast("1.15")) { - assert(alterResult.next()) - assert(alterResult.getString(1) === "OK") - } + assert(alterResult.next()) + assert(alterResult.getString(1) === "OK") val dropResult = statement.executeQuery("drop table tbl_b") - if (isFlinkVersionAtLeast("1.15")) { - assert(dropResult.next()) - assert(dropResult.getString(1) === "OK") - } - }) + assert(dropResult.next()) + assert(dropResult.getString(1) === "OK") + } } test("execute statement - create/alter/drop view") { - withMultipleConnectionJdbcStatement()({ statement => + withMultipleConnectionJdbcStatement() { statement => val createResult = statement.executeQuery("create view view_a as select 1") - if (isFlinkVersionAtLeast("1.15")) { - assert(createResult.next()) - assert(createResult.getString(1) === "OK") - } + assert(createResult.next()) + assert(createResult.getString(1) === "OK") val alterResult = statement.executeQuery("alter view view_a rename to view_b") - if (isFlinkVersionAtLeast("1.15")) { - assert(alterResult.next()) - assert(alterResult.getString(1) === "OK") - } + assert(alterResult.next()) + assert(alterResult.getString(1) === "OK") val dropResult = statement.executeQuery("drop view view_b") - if (isFlinkVersionAtLeast("1.15")) { - assert(dropResult.next()) - assert(dropResult.getString(1) === "OK") - } - }) + assert(dropResult.next()) + assert(dropResult.getString(1) === "OK") + } } - test("execute statement - insert into") { - withMultipleConnectionJdbcStatement()({ statement => + test("execute statement - batch insert into") { + withMultipleConnectionJdbcStatement() { statement => statement.executeQuery("create table tbl_a (a int) with ('connector' = 'blackhole')") val resultSet = statement.executeQuery("insert into tbl_a select 1") val metadata = resultSet.getMetaData - assert(metadata.getColumnName(1) == "default_catalog.default_database.tbl_a") - assert(metadata.getColumnType(1) == java.sql.Types.BIGINT) + assert(metadata.getColumnName(1) === "job id") + assert(metadata.getColumnType(1) === java.sql.Types.VARCHAR) + assert(resultSet.next()) + assert(resultSet.getString(1).length == 32) + } + } + + test("execute statement - streaming insert into") { + withMultipleConnectionJdbcStatement()({ statement => + // Flink currently doesn't support stop job statement, thus use a finite stream + statement.executeQuery( + "create table tbl_a (a int) with (" + + "'connector' = 'datagen', " + + "'rows-per-second'='10', " + + "'number-of-rows'='100')") + statement.executeQuery("create table tbl_b (a int) with ('connector' = 'blackhole')") + val resultSet = statement.executeQuery("insert into tbl_b select * from tbl_a") + val metadata = resultSet.getMetaData + assert(metadata.getColumnName(1) === "job id") + assert(metadata.getColumnType(1) === java.sql.Types.VARCHAR) assert(resultSet.next()) - assert(resultSet.getLong(1) == -1L) + val jobId = resultSet.getString(1) + assert(jobId.length == 32) + + if (FLINK_RUNTIME_VERSION >= "1.17") { + val stopResult = statement.executeQuery(s"stop job '$jobId'") + assert(stopResult.next()) + assert(stopResult.getString(1) === "OK") + } }) } test("execute statement - set properties") { - withMultipleConnectionJdbcStatement()({ statement => + withMultipleConnectionJdbcStatement() { statement => val resultSet = statement.executeQuery("set table.dynamic-table-options.enabled = true") val metadata = resultSet.getMetaData - assert(metadata.getColumnName(1) == "key") - assert(metadata.getColumnName(2) == "value") + assert(metadata.getColumnName(1) == "result") assert(resultSet.next()) - assert(resultSet.getString(1) == "table.dynamic-table-options.enabled") - assert(resultSet.getString(2) == "true") - }) + assert(resultSet.getString(1) == "OK") + } } test("execute statement - show properties") { - withMultipleConnectionJdbcStatement()({ statement => + withMultipleConnectionJdbcStatement() { statement => val resultSet = statement.executeQuery("set") val metadata = resultSet.getMetaData assert(metadata.getColumnName(1) == "key") assert(metadata.getColumnName(2) == "value") assert(resultSet.next()) - }) + } } test("execute statement - reset property") { - withMultipleConnectionJdbcStatement()({ statement => - statement.executeQuery("set pipeline.jars = my.jar") - statement.executeQuery("reset pipeline.jars") + val originalName = "test-job" // defined in WithFlinkTestResource + withMultipleConnectionJdbcStatement() { statement => + statement.executeQuery(s"set ${PipelineOptions.NAME.key()} = wrong-name") + statement.executeQuery(s"reset ${PipelineOptions.NAME.key()}") val resultSet = statement.executeQuery("set") // Flink does not support set key without value currently, // thus read all rows to find the desired one var success = false while (resultSet.next()) { - if (resultSet.getString(1) == "pipeline.jars" && - !resultSet.getString(2).contains("my.jar")) { + if (resultSet.getString(1) == PipelineOptions.NAME.key() && + resultSet.getString(2).equals(originalName)) { success = true } } assert(success) - }) + } } test("execute statement - select udf") { @@ -1073,7 +1138,8 @@ class FlinkOperationSuite extends WithFlinkSQLEngine with HiveJDBCTestHelper { test("ensure result max rows") { withSessionConf()(Map(ENGINE_FLINK_MAX_ROWS.key -> "200"))(Map.empty) { withJdbcStatement() { statement => - statement.execute("create table tbl_src (a bigint) with ('connector' = 'datagen')") + statement.execute("create table tbl_src (a bigint) with (" + + "'connector' = 'datagen', 'number-of-rows' = '1000')") val resultSet = statement.executeQuery(s"select a from tbl_src") var rows = 0 while (resultSet.next()) { @@ -1084,7 +1150,31 @@ class FlinkOperationSuite extends WithFlinkSQLEngine with HiveJDBCTestHelper { } } - test("execute statement - add/remove/show jar") { + test("execute statement - add/show jar") { + val jarName = s"newly-added-${UUID.randomUUID()}.jar" + val newJar = TestUserClassLoaderJar.createJarFile( + Utils.createTempDir("add-jar-test").toFile, + jarName, + GENERATED_UDF_CLASS, + GENERATED_UDF_CODE).toPath + + withMultipleConnectionJdbcStatement()({ statement => + statement.execute(s"add jar '$newJar'") + + val showJarsResultAdded = statement.executeQuery("show jars") + var exists = false + while (showJarsResultAdded.next()) { + if (showJarsResultAdded.getString(1).contains(jarName)) { + exists = true + } + } + assert(exists) + }) + } + + // ignored because Flink gateway doesn't support remove-jar statements + // see org.apache.flink.table.gateway.service.operation.OperationExecutor#callRemoveJar(..) + ignore("execute statement - remove jar") { val jarName = s"newly-added-${UUID.randomUUID()}.jar" val newJar = TestUserClassLoaderJar.createJarFile( Utils.createTempDir("add-jar-test").toFile, @@ -1154,9 +1244,25 @@ class FlinkOperationSuite extends WithFlinkSQLEngine with HiveJDBCTestHelper { assert(stmt.asInstanceOf[KyuubiStatement].getQueryId === null) stmt.executeQuery("insert into tbl_a values (1)") val queryId = stmt.asInstanceOf[KyuubiStatement].getQueryId - assert(queryId !== null) - // parse the string to check if it's valid Flink job id - assert(JobID.fromHexString(queryId) !== null) + // Flink 1.16 doesn't support query id via ResultFetcher + if (FLINK_RUNTIME_VERSION >= "1.17") { + assert(queryId !== null) + // parse the string to check if it's valid Flink job id + assert(JobID.fromHexString(queryId) !== null) + } } } + + test("test result fetch timeout") { + val exception = intercept[KyuubiSQLException]( + withSessionConf()(Map(ENGINE_FLINK_FETCH_TIMEOUT.key -> "60000"))() { + withJdbcStatement("tbl_a") { stmt => + stmt.executeQuery("create table tbl_a (a int) " + + "with ('connector' = 'datagen', 'rows-per-second'='0')") + val resultSet = stmt.executeQuery("select * from tbl_a") + while (resultSet.next()) {} + } + }) + assert(exception.getMessage === "Futures timed out after [60000 milliseconds]") + } } diff --git a/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/operation/PlanOnlyOperationSuite.scala b/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/operation/PlanOnlyOperationSuite.scala index 1194f3582..17c49464f 100644 --- a/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/operation/PlanOnlyOperationSuite.scala +++ b/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/operation/PlanOnlyOperationSuite.scala @@ -18,21 +18,33 @@ package org.apache.kyuubi.engine.flink.operation import java.sql.Statement +import java.util.UUID import org.apache.kyuubi.config.KyuubiConf -import org.apache.kyuubi.engine.flink.WithFlinkSQLEngine +import org.apache.kyuubi.engine.flink.{WithDiscoveryFlinkSQLEngine, WithFlinkSQLEngineLocal} +import org.apache.kyuubi.ha.HighAvailabilityConf.{HA_ENGINE_REF_ID, HA_NAMESPACE} import org.apache.kyuubi.operation.{AnalyzeMode, ExecutionMode, HiveJDBCTestHelper, ParseMode, PhysicalMode} -class PlanOnlyOperationSuite extends WithFlinkSQLEngine with HiveJDBCTestHelper { +class PlanOnlyOperationSuite extends WithFlinkSQLEngineLocal + with HiveJDBCTestHelper with WithDiscoveryFlinkSQLEngine { + + override protected def engineRefId: String = UUID.randomUUID().toString + + override protected def namespace: String = "/kyuubi/flink-plan-only-test" + + def engineType: String = "flink" override def withKyuubiConf: Map[String, String] = Map( + "flink.execution.target" -> "remote", + HA_NAMESPACE.key -> namespace, + HA_ENGINE_REF_ID.key -> engineRefId, + KyuubiConf.ENGINE_TYPE.key -> "FLINK_SQL", KyuubiConf.ENGINE_SHARE_LEVEL.key -> "user", KyuubiConf.OPERATION_PLAN_ONLY_MODE.key -> ParseMode.name, - KyuubiConf.ENGINE_SHARE_LEVEL_SUBDOMAIN.key -> "plan-only") + KyuubiConf.ENGINE_SHARE_LEVEL_SUBDOMAIN.key -> "plan-only") ++ testExtraConf - override protected def jdbcUrl: String = - s"jdbc:hive2://${engine.frontendServices.head.connectionUrl}/;" + override protected def jdbcUrl: String = getFlinkEngineServiceUrl test("Plan only operation with system defaults") { withJdbcStatement() { statement => diff --git a/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/result/ResultSetSuite.scala b/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/result/ResultSetSuite.scala index 9190456b3..9ee5c658b 100644 --- a/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/result/ResultSetSuite.scala +++ b/externals/kyuubi-flink-sql-engine/src/test/scala/org/apache/kyuubi/engine/flink/result/ResultSetSuite.scala @@ -17,6 +17,8 @@ package org.apache.kyuubi.engine.flink.result +import java.time.ZoneId + import org.apache.flink.table.api.{DataTypes, ResultKind} import org.apache.flink.table.catalog.Column import org.apache.flink.table.data.StringData @@ -44,9 +46,10 @@ class ResultSetSuite extends KyuubiFunSuite { .data(rowsNew) .build - assert(RowSet.toRowBaseSet(rowsNew, resultSetNew) - === RowSet.toRowBaseSet(rowsOld, resultSetOld)) - assert(RowSet.toColumnBasedSet(rowsNew, resultSetNew) - === RowSet.toColumnBasedSet(rowsOld, resultSetOld)) + val timeZone = ZoneId.of("America/Los_Angeles") + assert(RowSet.toRowBaseSet(rowsNew, resultSetNew, timeZone) + === RowSet.toRowBaseSet(rowsOld, resultSetOld, timeZone)) + assert(RowSet.toColumnBasedSet(rowsNew, resultSetNew, timeZone) + === RowSet.toColumnBasedSet(rowsOld, resultSetOld, timeZone)) } } diff --git a/externals/kyuubi-hive-sql-engine/pom.xml b/externals/kyuubi-hive-sql-engine/pom.xml index 1dbc31947..caed7e27c 100644 --- a/externals/kyuubi-hive-sql-engine/pom.xml +++ b/externals/kyuubi-hive-sql-engine/pom.xml @@ -21,11 +21,11 @@ org.apache.kyuubi kyuubi-parent - 1.7.0-SNAPSHOT + 1.9.0-SNAPSHOT ../../pom.xml - kyuubi-hive-sql-engine_2.12 + kyuubi-hive-sql-engine_${scala.binary.version} jar Kyuubi Project Engine Hive SQL https://kyuubi.apache.org/ @@ -163,6 +163,12 @@ HikariCP test + + + com.vladsch.flexmark + flexmark-all + test + @@ -179,12 +185,7 @@ com.fasterxml.jackson.core:jackson-core com.fasterxml.jackson.core:jackson-databind com.fasterxml.jackson.module:jackson-module-scala_${scala.binary.version} - org.apache.kyuubi:kyuubi-common_${scala.binary.version} - org.apache.kyuubi:kyuubi-events_${scala.binary.version} - org.apache.kyuubi:kyuubi-ha_${scala.binary.version} - org.apache.curator:curator-client - org.apache.curator:curator-framework - org.apache.curator:curator-recipes + org.apache.kyuubi:* @@ -205,15 +206,6 @@ - - - org.apache.curator - ${kyuubi.shade.packageName}.org.apache.curator - - org.apache.curator.** - - - diff --git a/externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/HiveSQLEngine.scala b/externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/HiveSQLEngine.scala index 839da710e..3cc426c43 100644 --- a/externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/HiveSQLEngine.scala +++ b/externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/HiveSQLEngine.scala @@ -18,6 +18,7 @@ package org.apache.kyuubi.engine.hive import java.security.PrivilegedExceptionAction +import java.time.Instant import scala.util.control.NonFatal @@ -65,6 +66,7 @@ object HiveSQLEngine extends Logging { var currentEngine: Option[HiveSQLEngine] = None val hiveConf = new HiveConf() val kyuubiConf = new KyuubiConf() + val user = UserGroupInformation.getCurrentUser.getShortUserName def startEngine(): Unit = { try { @@ -97,6 +99,8 @@ object HiveSQLEngine extends Logging { } val engine = new HiveSQLEngine() + val appName = s"kyuubi_${user}_hive_${Instant.now}" + hiveConf.setIfUnset("hive.engine.name", appName) info(s"Starting ${engine.getName}") engine.initialize(kyuubiConf) EventBus.post(HiveEngineEvent(engine)) diff --git a/externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/operation/HiveOperation.scala b/externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/operation/HiveOperation.scala index 81affdff3..9759fa00b 100644 --- a/externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/operation/HiveOperation.scala +++ b/externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/operation/HiveOperation.scala @@ -21,9 +21,10 @@ import java.util.concurrent.Future import org.apache.hive.service.cli.operation.{Operation, OperationManager} import org.apache.hive.service.cli.session.{HiveSession, SessionManager => HiveSessionManager} -import org.apache.hive.service.rpc.thrift.{TGetResultSetMetadataResp, TRowSet} +import org.apache.hive.service.rpc.thrift.{TFetchResultsResp, TGetResultSetMetadataResp} import org.apache.kyuubi.KyuubiSQLException +import org.apache.kyuubi.config.KyuubiReservedKeys.KYUUBI_SESSION_USER_KEY import org.apache.kyuubi.engine.hive.session.HiveSessionImpl import org.apache.kyuubi.operation.{AbstractOperation, FetchOrientation, OperationState, OperationStatus} import org.apache.kyuubi.operation.FetchOrientation.FetchOrientation @@ -43,12 +44,14 @@ abstract class HiveOperation(session: Session) extends AbstractOperation(session override def beforeRun(): Unit = { setState(OperationState.RUNNING) + hive.getHiveConf.set(KYUUBI_SESSION_USER_KEY, session.user) } override def afterRun(): Unit = { - state.synchronized { + withLockRequired { if (!isTerminalState(state)) { setState(OperationState.FINISHED) + hive.getHiveConf.unset(KYUUBI_SESSION_USER_KEY) } } } @@ -92,22 +95,31 @@ abstract class HiveOperation(session: Session) extends AbstractOperation(session resp } - override def getNextRowSet(order: FetchOrientation, rowSetSize: Int): TRowSet = { + override def getNextRowSetInternal( + order: FetchOrientation, + rowSetSize: Int): TFetchResultsResp = { val tOrder = FetchOrientation.toTFetchOrientation(order) val hiveOrder = org.apache.hive.service.cli.FetchOrientation.getFetchOrientation(tOrder) val rowSet = internalHiveOperation.getNextRowSet(hiveOrder, rowSetSize) - rowSet.toTRowSet + val resp = new TFetchResultsResp(OK_STATUS) + resp.setResults(rowSet.toTRowSet) + resp.setHasMoreRows(false) + resp } - def getOperationLogRowSet(order: FetchOrientation, rowSetSize: Int): TRowSet = { + def getOperationLogRowSet(order: FetchOrientation, rowSetSize: Int): TFetchResultsResp = { val tOrder = FetchOrientation.toTFetchOrientation(order) val hiveOrder = org.apache.hive.service.cli.FetchOrientation.getFetchOrientation(tOrder) val handle = internalHiveOperation.getHandle - delegatedOperationManager.getOperationLogRowSet( + val rowSet = delegatedOperationManager.getOperationLogRowSet( handle, hiveOrder, rowSetSize, hive.getHiveConf).toTRowSet + val resp = new TFetchResultsResp(OK_STATUS) + resp.setResults(rowSet) + resp.setHasMoreRows(false) + resp } override def isTimedOut: Boolean = internalHiveOperation.isTimedOut(System.currentTimeMillis) diff --git a/externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/operation/HiveOperationManager.scala b/externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/operation/HiveOperationManager.scala index 0762a2938..4e41e742e 100644 --- a/externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/operation/HiveOperationManager.scala +++ b/externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/operation/HiveOperationManager.scala @@ -20,7 +20,7 @@ package org.apache.kyuubi.engine.hive.operation import java.util.List import org.apache.hadoop.hive.conf.HiveConf.ConfVars -import org.apache.hive.service.rpc.thrift.TRowSet +import org.apache.hive.service.rpc.thrift.TFetchResultsResp import org.apache.kyuubi.config.KyuubiConf._ import org.apache.kyuubi.engine.hive.session.HiveSessionImpl @@ -154,7 +154,7 @@ class HiveOperationManager() extends OperationManager("HiveOperationManager") { override def getOperationLogRowSet( opHandle: OperationHandle, order: FetchOrientation, - maxRows: Int): TRowSet = { + maxRows: Int): TFetchResultsResp = { val operation = getOperation(opHandle).asInstanceOf[HiveOperation] operation.getOperationLogRowSet(order, maxRows) } diff --git a/externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/session/HiveSessionImpl.scala b/externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/session/HiveSessionImpl.scala index 3b85f94df..5069b1379 100644 --- a/externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/session/HiveSessionImpl.scala +++ b/externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/session/HiveSessionImpl.scala @@ -27,6 +27,7 @@ import org.apache.hive.service.rpc.thrift.{TGetInfoType, TGetInfoValue, TProtoco import org.apache.kyuubi.KyuubiSQLException import org.apache.kyuubi.engine.hive.events.HiveSessionEvent +import org.apache.kyuubi.engine.hive.udf.KDFRegistry import org.apache.kyuubi.events.EventBus import org.apache.kyuubi.operation.{Operation, OperationHandle} import org.apache.kyuubi.session.{AbstractSession, SessionHandle, SessionManager} @@ -48,6 +49,7 @@ class HiveSessionImpl( val confClone = new HashMap[String, String]() confClone.putAll(conf.asJava) // pass conf.asScala not support `put` method hive.open(confClone) + KDFRegistry.registerAll() EventBus.post(sessionEvent) } diff --git a/externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/session/HiveSessionManager.scala b/externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/session/HiveSessionManager.scala index dc807429c..d09912770 100644 --- a/externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/session/HiveSessionManager.scala +++ b/externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/session/HiveSessionManager.scala @@ -28,6 +28,7 @@ import org.apache.hive.service.cli.session.{HiveSessionImplwithUGI => ImportedHi import org.apache.hive.service.rpc.thrift.TProtocolVersion import org.apache.kyuubi.config.KyuubiConf.ENGINE_SHARE_LEVEL +import org.apache.kyuubi.config.KyuubiReservedKeys.KYUUBI_SESSION_HANDLE_KEY import org.apache.kyuubi.engine.ShareLevel import org.apache.kyuubi.engine.hive.HiveSQLEngine import org.apache.kyuubi.engine.hive.operation.HiveOperationManager @@ -72,33 +73,38 @@ class HiveSessionManager(engine: HiveSQLEngine) extends SessionManager("HiveSess password: String, ipAddress: String, conf: Map[String, String]): Session = { - val sessionHandle = SessionHandle() - val hive = { - val sessionWithUGI = new ImportedHiveSessionImpl( - new ImportedSessionHandle(sessionHandle.toTSessionHandle, protocol), + conf.get(KYUUBI_SESSION_HANDLE_KEY).map(SessionHandle.fromUUID).flatMap( + getSessionOption).getOrElse { + val sessionHandle = + conf.get(KYUUBI_SESSION_HANDLE_KEY).map(SessionHandle.fromUUID).getOrElse(SessionHandle()) + val hive = { + val sessionWithUGI = new ImportedHiveSessionImpl( + new ImportedSessionHandle(sessionHandle.toTSessionHandle, protocol), + protocol, + user, + password, + HiveSQLEngine.hiveConf, + ipAddress, + null, + Seq(ipAddress).asJava) + val proxy = HiveSessionProxy.getProxy(sessionWithUGI, sessionWithUGI.getSessionUgi) + sessionWithUGI.setProxySession(proxy) + proxy + } + hive.setSessionManager(internalSessionManager) + hive.setOperationManager(internalSessionManager.getOperationManager) + operationLogRoot.foreach(dir => hive.setOperationLogSessionDir(new File(dir))) + new HiveSessionImpl( protocol, user, password, - HiveSQLEngine.hiveConf, ipAddress, - null, - Seq(ipAddress).asJava) - val proxy = HiveSessionProxy.getProxy(sessionWithUGI, sessionWithUGI.getSessionUgi) - sessionWithUGI.setProxySession(proxy) - proxy + conf, + this, + sessionHandle, + hive) } - hive.setSessionManager(internalSessionManager) - hive.setOperationManager(internalSessionManager.getOperationManager) - operationLogRoot.foreach(dir => hive.setOperationLogSessionDir(new File(dir))) - new HiveSessionImpl( - protocol, - user, - password, - ipAddress, - conf, - this, - sessionHandle, - hive) + } override def closeSession(sessionHandle: SessionHandle): Unit = { diff --git a/externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/udf/KDFRegistry.scala b/externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/udf/KDFRegistry.scala new file mode 100644 index 000000000..5ff468b77 --- /dev/null +++ b/externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/udf/KDFRegistry.scala @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.engine.hive.udf + +import scala.collection.mutable.ArrayBuffer + +import org.apache.hadoop.hive.ql.exec.{FunctionRegistry, UDFArgumentLengthException} +import org.apache.hadoop.hive.ql.session.SessionState +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector +import org.apache.hadoop.hive.serde2.objectinspector.primitive.{PrimitiveObjectInspectorFactory, StringObjectInspector} + +import org.apache.kyuubi.{KYUUBI_VERSION, Utils} +import org.apache.kyuubi.config.KyuubiReservedKeys.{KYUUBI_ENGINE_ID, KYUUBI_SESSION_USER_KEY} + +object KDFRegistry { + + @transient + val registeredFunctions = new ArrayBuffer[KyuubiDefinedFunction]() + + val kyuubi_version: KyuubiDefinedFunction = create( + "kyuubi_version", + new KyuubiVersionFunction, + "Return the version of Kyuubi Server", + "string", + "1.8.0") + + val engine_name: KyuubiDefinedFunction = create( + "engine_name", + new EngineNameFunction, + "Return the name of engine", + "string", + "1.8.0") + + val engine_id: KyuubiDefinedFunction = create( + "engine_id", + new EngineIdFunction, + "Return the id of engine", + "string", + "1.8.0") + + val system_user: KyuubiDefinedFunction = create( + "system_user", + new SystemUserFunction, + "Return the system user", + "string", + "1.8.0") + + val session_user: KyuubiDefinedFunction = create( + "session_user", + new SessionUserFunction, + "Return the session user", + "string", + "1.8.0") + + def create( + name: String, + udf: GenericUDF, + description: String, + returnType: String, + since: String): KyuubiDefinedFunction = { + val kdf = KyuubiDefinedFunction(name, udf, description, returnType, since) + registeredFunctions += kdf + kdf + } + + def registerAll(): Unit = { + for (func <- registeredFunctions) { + FunctionRegistry.registerTemporaryUDF(func.name, func.udf.getClass) + } + } +} + +class KyuubiVersionFunction() extends GenericUDF { + private val returnOI: StringObjectInspector = + PrimitiveObjectInspectorFactory.javaStringObjectInspector + override def initialize(arguments: Array[ObjectInspector]): ObjectInspector = { + if (arguments.length != 0) { + throw new UDFArgumentLengthException("The function kyuubi_version() takes no arguments, got " + + arguments.length) + } + returnOI + } + + override def evaluate(arguments: Array[GenericUDF.DeferredObject]): AnyRef = KYUUBI_VERSION + + override def getDisplayString(children: Array[String]): String = "kyuubi_version()" +} + +class EngineNameFunction() extends GenericUDF { + private val returnOI: StringObjectInspector = + PrimitiveObjectInspectorFactory.javaStringObjectInspector + override def initialize(arguments: Array[ObjectInspector]): ObjectInspector = { + if (arguments.length != 0) { + throw new UDFArgumentLengthException("The function engine_name() takes no arguments, got " + + arguments.length) + } + returnOI + } + override def evaluate(arguments: Array[GenericUDF.DeferredObject]): AnyRef = + SessionState.get.getConf.get("hive.engine.name", "") + override def getDisplayString(children: Array[String]): String = "engine_name()" +} + +class EngineIdFunction() extends GenericUDF { + private val returnOI: StringObjectInspector = + PrimitiveObjectInspectorFactory.javaStringObjectInspector + override def initialize(arguments: Array[ObjectInspector]): ObjectInspector = { + if (arguments.length != 0) { + throw new UDFArgumentLengthException("The function engine_id() takes no arguments, got " + + arguments.length) + } + returnOI + } + + override def evaluate(arguments: Array[GenericUDF.DeferredObject]): AnyRef = + SessionState.get.getConf.get(KYUUBI_ENGINE_ID, "") + + override def getDisplayString(children: Array[String]): String = "engine_id()" +} + +class SystemUserFunction() extends GenericUDF { + private val returnOI: StringObjectInspector = + PrimitiveObjectInspectorFactory.javaStringObjectInspector + override def initialize(arguments: Array[ObjectInspector]): ObjectInspector = { + if (arguments.length != 0) { + throw new UDFArgumentLengthException("The function system_user() takes no arguments, got " + + arguments.length) + } + returnOI + } + + override def evaluate(arguments: Array[GenericUDF.DeferredObject]): AnyRef = Utils.currentUser + + override def getDisplayString(children: Array[String]): String = "system_user()" +} + +class SessionUserFunction() extends GenericUDF { + private val returnOI: StringObjectInspector = + PrimitiveObjectInspectorFactory.javaStringObjectInspector + override def initialize(arguments: Array[ObjectInspector]): ObjectInspector = { + if (arguments.length != 0) { + throw new UDFArgumentLengthException("The function session_user() takes no arguments, got " + + arguments.length) + } + returnOI + } + + override def evaluate(arguments: Array[GenericUDF.DeferredObject]): AnyRef = { + SessionState.get.getConf.get(KYUUBI_SESSION_USER_KEY, "") + } + + override def getDisplayString(children: Array[String]): String = "session_user()" +} diff --git a/externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/udf/KyuubiDefinedFunction.scala b/externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/udf/KyuubiDefinedFunction.scala new file mode 100644 index 000000000..ee91a804e --- /dev/null +++ b/externals/kyuubi-hive-sql-engine/src/main/scala/org/apache/kyuubi/engine/hive/udf/KyuubiDefinedFunction.scala @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.engine.hive.udf + +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF + +/** + * A wrapper for Hive's [[UserDefinedFunction]] + * + * @param name function name + * @param udf user-defined function + * @param description function description + */ +case class KyuubiDefinedFunction( + name: String, + udf: GenericUDF, + description: String, + returnType: String, + since: String) diff --git a/externals/kyuubi-hive-sql-engine/src/test/resources/log4j2-test.xml b/externals/kyuubi-hive-sql-engine/src/test/resources/log4j2-test.xml index bfc40dd6d..3110216c1 100644 --- a/externals/kyuubi-hive-sql-engine/src/test/resources/log4j2-test.xml +++ b/externals/kyuubi-hive-sql-engine/src/test/resources/log4j2-test.xml @@ -21,14 +21,14 @@ - + - + diff --git a/externals/kyuubi-hive-sql-engine/src/test/scala/org/apache/kyuubi/engine/hive/operation/HiveOperationSuite.scala b/externals/kyuubi-hive-sql-engine/src/test/scala/org/apache/kyuubi/engine/hive/operation/HiveOperationSuite.scala index f949ec37a..eb10e0b41 100644 --- a/externals/kyuubi-hive-sql-engine/src/test/scala/org/apache/kyuubi/engine/hive/operation/HiveOperationSuite.scala +++ b/externals/kyuubi-hive-sql-engine/src/test/scala/org/apache/kyuubi/engine/hive/operation/HiveOperationSuite.scala @@ -19,7 +19,7 @@ package org.apache.kyuubi.engine.hive.operation import org.apache.commons.lang3.{JavaVersion, SystemUtils} -import org.apache.kyuubi.{HiveEngineTests, Utils} +import org.apache.kyuubi.{HiveEngineTests, KYUUBI_VERSION, Utils} import org.apache.kyuubi.engine.hive.HiveSQLEngine import org.apache.kyuubi.jdbc.hive.KyuubiStatement @@ -49,4 +49,20 @@ class HiveOperationSuite extends HiveEngineTests { assert(kyuubiStatement.getQueryId != null) } } + + test("kyuubi defined function - kyuubi_version") { + withJdbcStatement("hive_engine_test") { statement => + val rs = statement.executeQuery("SELECT kyuubi_version()") + assert(rs.next()) + assert(rs.getString(1) == KYUUBI_VERSION) + } + } + + test("kyuubi defined function - engine_name") { + withJdbcStatement("hive_engine_test") { statement => + val rs = statement.executeQuery("SELECT engine_name()") + assert(rs.next()) + assert(rs.getString(1).nonEmpty) + } + } } diff --git a/externals/kyuubi-hive-sql-engine/src/test/scala/org/apache/kyuubi/engine/hive/udf/KyuubiDefinedFunctionSuite.scala b/externals/kyuubi-hive-sql-engine/src/test/scala/org/apache/kyuubi/engine/hive/udf/KyuubiDefinedFunctionSuite.scala new file mode 100644 index 000000000..08cb143e0 --- /dev/null +++ b/externals/kyuubi-hive-sql-engine/src/test/scala/org/apache/kyuubi/engine/hive/udf/KyuubiDefinedFunctionSuite.scala @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.engine.hive.udf + +import java.nio.file.Paths + +import org.apache.kyuubi.{KyuubiFunSuite, MarkdownBuilder, Utils} +import org.apache.kyuubi.util.GoldenFileUtils._ + +/** + * End-to-end test cases for configuration doc file + * The golden result file is "docs/extensions/engines/hive/functions.md". + * + * To run the entire test suite: + * {{{ + * KYUUBI_UPDATE=0 dev/gen/gen_hive_kdf_docs.sh + * }}} + * + * To re-generate golden files for entire suite, run: + * {{{ + * dev/gen/gen_hive_kdf_docs.sh + * }}} + */ +class KyuubiDefinedFunctionSuite extends KyuubiFunSuite { + + private val kyuubiHome: String = Utils.getCodeSourceLocation(getClass) + .split("kyuubi-hive-sql-engine")(0) + private val markdown = + Paths.get(kyuubiHome, "..", "docs", "extensions", "engines", "hive", "functions.md") + .toAbsolutePath + + test("verify or update kyuubi hive sql functions") { + val builder = MarkdownBuilder(licenced = true, getClass.getName) + + builder += "# Auxiliary SQL Functions" += + """Kyuubi provides several auxiliary SQL functions as supplement to Hive's + | [Built-in Functions](https://cwiki.apache.org/confluence/display/hive/languagemanual+udf# + |LanguageManualUDF-Built-inFunctions)""" ++= + """ + | Name | Description | Return Type | Since + | --- | --- | --- | --- + |""" + KDFRegistry.registeredFunctions.foreach { func => + builder += s"${func.name} | ${func.description} | ${func.returnType} | ${func.since}" + } + + verifyOrRegenerateGoldenFile(markdown, builder.toMarkdown, "dev/gen/gen_hive_kdf_docs.sh") + } +} diff --git a/externals/kyuubi-jdbc-engine/pom.xml b/externals/kyuubi-jdbc-engine/pom.xml index 8853cec64..3c21fed57 100644 --- a/externals/kyuubi-jdbc-engine/pom.xml +++ b/externals/kyuubi-jdbc-engine/pom.xml @@ -21,11 +21,11 @@ org.apache.kyuubi kyuubi-parent - 1.7.0-SNAPSHOT + 1.9.0-SNAPSHOT ../../pom.xml - kyuubi-jdbc-engine_2.12 + kyuubi-jdbc-engine_${scala.binary.version} jar Kyuubi Project Engine JDBC https://kyuubi.apache.org/ diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcSQLEngine.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcSQLEngine.scala index 618098f31..6e0647f6c 100644 --- a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcSQLEngine.scala +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/JdbcSQLEngine.scala @@ -19,7 +19,9 @@ package org.apache.kyuubi.engine.jdbc import org.apache.kyuubi.{Logging, Utils} import org.apache.kyuubi.Utils.{addShutdownHook, JDBC_ENGINE_SHUTDOWN_PRIORITY} import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.config.KyuubiConf.ENGINE_JDBC_INITIALIZE_SQL import org.apache.kyuubi.engine.jdbc.JdbcSQLEngine.currentEngine +import org.apache.kyuubi.engine.jdbc.util.KyuubiJdbcUtils import org.apache.kyuubi.ha.HighAvailabilityConf.HA_ZK_CONN_RETRY_POLICY import org.apache.kyuubi.ha.client.RetryPolicies import org.apache.kyuubi.service.Serverable @@ -71,6 +73,8 @@ object JdbcSQLEngine extends Logging { kyuubiConf.setIfMissing(HA_ZK_CONN_RETRY_POLICY, RetryPolicies.N_TIME.toString) startEngine() + + KyuubiJdbcUtils.initializeJdbcSession(kyuubiConf, kyuubiConf.get(ENGINE_JDBC_INITIALIZE_SQL)) } catch { case t: Throwable if currentEngine.isDefined => currentEngine.foreach { engine => diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/connection/ConnectionProvider.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/connection/ConnectionProvider.scala index 798c92fbe..cb6e4b6c5 100644 --- a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/connection/ConnectionProvider.scala +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/connection/ConnectionProvider.scala @@ -16,26 +16,25 @@ */ package org.apache.kyuubi.engine.jdbc.connection -import java.sql.{Connection, DriverManager} -import java.util.ServiceLoader - -import scala.collection.mutable.ArrayBuffer +import java.sql.{Connection, Driver, DriverManager} import org.apache.kyuubi.Logging import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.config.KyuubiConf.{ENGINE_JDBC_CONNECTION_PROVIDER, ENGINE_JDBC_CONNECTION_URL, ENGINE_JDBC_DRIVER_CLASS} +import org.apache.kyuubi.util.reflect.DynClasses +import org.apache.kyuubi.util.reflect.ReflectUtils._ abstract class AbstractConnectionProvider extends Logging { protected val providers = loadProviders() def getProviderClass(kyuubiConf: KyuubiConf): String = { - val specifiedDriverClass = kyuubiConf.get(ENGINE_JDBC_DRIVER_CLASS) - specifiedDriverClass.foreach(Class.forName) - - specifiedDriverClass.getOrElse { + val driverClass: Class[_ <: Driver] = Option( + DynClasses.builder().impl(kyuubiConf.get(ENGINE_JDBC_DRIVER_CLASS).get) + .orNull().build[Driver]()).getOrElse { val url = kyuubiConf.get(ENGINE_JDBC_CONNECTION_URL).get - DriverManager.getDriver(url).getClass.getCanonicalName + DriverManager.getDriver(url).getClass } + driverClass.getCanonicalName } def create(kyuubiConf: KyuubiConf): Connection = { @@ -69,27 +68,12 @@ abstract class AbstractConnectionProvider extends Logging { selectedProvider.getConnection(kyuubiConf) } - def loadProviders(): Seq[JdbcConnectionProvider] = { - val loader = ServiceLoader.load( - classOf[JdbcConnectionProvider], - Thread.currentThread().getContextClassLoader) - val providers = ArrayBuffer[JdbcConnectionProvider]() - - val iterator = loader.iterator() - while (iterator.hasNext) { - try { - val provider = iterator.next() + def loadProviders(): Seq[JdbcConnectionProvider] = + loadFromServiceLoader[JdbcConnectionProvider]() + .map { provider => info(s"Loaded provider: $provider") - providers += provider - } catch { - case t: Throwable => - warn(s"Loaded of the provider failed with the exception", t) - } - } - - // TODO support disable provider - providers - } + provider + }.toSeq } object ConnectionProvider extends AbstractConnectionProvider diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/DorisDialect.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/DorisDialect.scala index c2ae29953..f7c1ace64 100644 --- a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/DorisDialect.scala +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/DorisDialect.scala @@ -15,7 +15,7 @@ * limitations under the License. */ package org.apache.kyuubi.engine.jdbc.dialect -import java.sql.{Connection, ResultSet, Statement} +import java.sql.{Connection, Statement} import java.util import scala.collection.JavaConverters._ @@ -23,34 +23,19 @@ import scala.collection.mutable.ArrayBuffer import org.apache.commons.lang3.StringUtils -import org.apache.kyuubi.KyuubiSQLException import org.apache.kyuubi.engine.jdbc.doris.{DorisRowSetHelper, DorisSchemaHelper} import org.apache.kyuubi.engine.jdbc.schema.{RowSetHelper, SchemaHelper} -import org.apache.kyuubi.operation.Operation import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant._ import org.apache.kyuubi.session.Session class DorisDialect extends JdbcDialect { override def createStatement(connection: Connection, fetchSize: Int): Statement = { - val statement = - connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY) + val statement = super.createStatement(connection, fetchSize) statement.setFetchSize(Integer.MIN_VALUE) statement } - override def getTypeInfoOperation(session: Session): Operation = { - throw KyuubiSQLException.featureNotSupported() - } - - override def getCatalogsOperation(session: Session): Operation = { - throw KyuubiSQLException.featureNotSupported() - } - - override def getSchemasOperation(session: Session): Operation = { - throw KyuubiSQLException.featureNotSupported() - } - override def getTablesQuery( catalog: String, schema: String, @@ -96,10 +81,6 @@ class DorisDialect extends JdbcDialect { query.toString() } - override def getTableTypesOperation(session: Session): Operation = { - throw KyuubiSQLException.featureNotSupported() - } - override def getColumnsQuery( session: Session, catalogName: String, @@ -139,18 +120,6 @@ class DorisDialect extends JdbcDialect { query.toString() } - override def getFunctionsOperation(session: Session): Operation = { - throw KyuubiSQLException.featureNotSupported() - } - - override def getPrimaryKeysOperation(session: Session): Operation = { - throw KyuubiSQLException.featureNotSupported() - } - - override def getCrossReferenceOperation(session: Session): Operation = { - throw KyuubiSQLException.featureNotSupported() - } - override def getRowSetHelper(): RowSetHelper = { new DorisRowSetHelper } diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/JdbcDialect.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/JdbcDialect.scala index b7ac7f43b..62e20a1d2 100644 --- a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/JdbcDialect.scala +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/JdbcDialect.scala @@ -16,29 +16,38 @@ */ package org.apache.kyuubi.engine.jdbc.dialect -import java.sql.{Connection, Statement} +import java.sql.{Connection, ResultSet, Statement} import java.util -import java.util.ServiceLoader -import scala.collection.JavaConverters._ - -import org.apache.kyuubi.{KyuubiException, Logging} +import org.apache.kyuubi.{KyuubiException, KyuubiSQLException, Logging} import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.config.KyuubiConf.{ENGINE_JDBC_CONNECTION_URL, ENGINE_JDBC_SHORT_NAME} import org.apache.kyuubi.engine.jdbc.schema.{RowSetHelper, SchemaHelper} import org.apache.kyuubi.engine.jdbc.util.SupportServiceLoader import org.apache.kyuubi.operation.Operation import org.apache.kyuubi.session.Session +import org.apache.kyuubi.util.reflect.ReflectUtils._ abstract class JdbcDialect extends SupportServiceLoader with Logging { - def createStatement(connection: Connection, fetchSize: Int = 1000): Statement + def createStatement(connection: Connection, fetchSize: Int = 1000): Statement = { + val statement = + connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY) + statement.setFetchSize(fetchSize) + statement + } - def getTypeInfoOperation(session: Session): Operation + def getTypeInfoOperation(session: Session): Operation = { + throw KyuubiSQLException.featureNotSupported() + } - def getCatalogsOperation(session: Session): Operation + def getCatalogsOperation(session: Session): Operation = { + throw KyuubiSQLException.featureNotSupported() + } - def getSchemasOperation(session: Session): Operation + def getSchemasOperation(session: Session): Operation = { + throw KyuubiSQLException.featureNotSupported() + } def getTablesQuery( catalog: String, @@ -46,7 +55,9 @@ abstract class JdbcDialect extends SupportServiceLoader with Logging { tableName: String, tableTypes: util.List[String]): String - def getTableTypesOperation(session: Session): Operation + def getTableTypesOperation(session: Session): Operation = { + throw KyuubiSQLException.featureNotSupported() + } def getColumnsQuery( session: Session, @@ -55,16 +66,21 @@ abstract class JdbcDialect extends SupportServiceLoader with Logging { tableName: String, columnName: String): String - def getFunctionsOperation(session: Session): Operation + def getFunctionsOperation(session: Session): Operation = { + throw KyuubiSQLException.featureNotSupported() + } - def getPrimaryKeysOperation(session: Session): Operation + def getPrimaryKeysOperation(session: Session): Operation = { + throw KyuubiSQLException.featureNotSupported() + } - def getCrossReferenceOperation(session: Session): Operation + def getCrossReferenceOperation(session: Session): Operation = { + throw KyuubiSQLException.featureNotSupported() + } def getRowSetHelper(): RowSetHelper def getSchemaHelper(): SchemaHelper - } object JdbcDialects extends Logging { @@ -75,9 +91,8 @@ object JdbcDialects extends Logging { assert(url.length > 5 && url.substring(5).contains(":")) url.substring(5, url.indexOf(":", 5)) } - val serviceLoader = - ServiceLoader.load(classOf[JdbcDialect], Thread.currentThread().getContextClassLoader) - serviceLoader.asScala.filter(_.name().equalsIgnoreCase(shortName)).toList match { + loadFromServiceLoader[JdbcDialect]() + .filter(_.name().equalsIgnoreCase(shortName)).toList match { case Nil => throw new KyuubiException(s"Don't find jdbc dialect implement for jdbc engine: $shortName.") case head :: Nil => diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/PhoenixDialect.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/PhoenixDialect.scala index 0cce14b42..4c8e8f265 100644 --- a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/PhoenixDialect.scala +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/dialect/PhoenixDialect.scala @@ -15,8 +15,6 @@ * limitations under the License. */ package org.apache.kyuubi.engine.jdbc.dialect - -import java.sql.{Connection, ResultSet, Statement} import java.util import scala.collection.JavaConverters._ @@ -24,34 +22,13 @@ import scala.collection.mutable.ArrayBuffer import org.apache.commons.lang3.StringUtils -import org.apache.kyuubi.KyuubiSQLException import org.apache.kyuubi.engine.jdbc.phoenix.{PhoenixRowSetHelper, PhoenixSchemaHelper} import org.apache.kyuubi.engine.jdbc.schema.{RowSetHelper, SchemaHelper} -import org.apache.kyuubi.operation.Operation import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant._ import org.apache.kyuubi.session.Session class PhoenixDialect extends JdbcDialect { - override def createStatement(connection: Connection, fetchSize: Int): Statement = { - val statement = - connection.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY) - statement.setFetchSize(fetchSize) - statement - } - - override def getTypeInfoOperation(session: Session): Operation = { - throw KyuubiSQLException.featureNotSupported() - } - - override def getCatalogsOperation(session: Session): Operation = { - throw KyuubiSQLException.featureNotSupported() - } - - override def getSchemasOperation(session: Session): Operation = { - throw KyuubiSQLException.featureNotSupported() - } - override def getTablesQuery( catalog: String, schema: String, @@ -91,10 +68,6 @@ class PhoenixDialect extends JdbcDialect { query.toString() } - override def getTableTypesOperation(session: Session): Operation = { - throw KyuubiSQLException.featureNotSupported() - } - override def getColumnsQuery( session: Session, catalogName: String, @@ -127,18 +100,6 @@ class PhoenixDialect extends JdbcDialect { query.toString() } - override def getFunctionsOperation(session: Session): Operation = { - throw KyuubiSQLException.featureNotSupported() - } - - override def getPrimaryKeysOperation(session: Session): Operation = { - throw KyuubiSQLException.featureNotSupported() - } - - override def getCrossReferenceOperation(session: Session): Operation = { - throw KyuubiSQLException.featureNotSupported() - } - override def getRowSetHelper(): RowSetHelper = { new PhoenixRowSetHelper } diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/doris/DorisRowSetHelper.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/doris/DorisRowSetHelper.scala index 1ce43c7a4..a92942cec 100644 --- a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/doris/DorisRowSetHelper.scala +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/doris/DorisRowSetHelper.scala @@ -16,125 +16,21 @@ */ package org.apache.kyuubi.engine.jdbc.doris -import java.sql.{Date, Types} -import java.time.LocalDateTime - -import scala.collection.JavaConverters._ - import org.apache.hive.service.rpc.thrift._ -import org.apache.kyuubi.engine.jdbc.schema.{Column, RowSetHelper} -import org.apache.kyuubi.util.RowSetUtils.{bitSetToBuffer, formatDate, formatLocalDateTime} +import org.apache.kyuubi.engine.jdbc.schema.RowSetHelper class DorisRowSetHelper extends RowSetHelper { - protected def toTColumn( - rows: Seq[Seq[Any]], - ordinal: Int, - sqlType: Int): TColumn = { - val nulls = new java.util.BitSet() - sqlType match { - case Types.BIT => - val values = getOrSetAsNull[java.lang.Boolean](rows, ordinal, nulls, true) - TColumn.boolVal(new TBoolColumn(values, nulls)) - - case Types.TINYINT | Types.SMALLINT | Types.INTEGER => - val values = getOrSetAsNull[java.lang.Integer](rows, ordinal, nulls, 0) - TColumn.i32Val(new TI32Column(values, nulls)) - - case Types.BIGINT => - val values = getOrSetAsNull[java.lang.Long](rows, ordinal, nulls, 0L) - TColumn.i64Val(new TI64Column(values, nulls)) - - case Types.REAL => - val values = getOrSetAsNull[java.lang.Float](rows, ordinal, nulls, 0.toFloat) - .asScala.map(n => java.lang.Double.valueOf(n.toString)).asJava - TColumn.doubleVal(new TDoubleColumn(values, nulls)) - - case Types.DOUBLE => - val values = getOrSetAsNull[java.lang.Double](rows, ordinal, nulls, 0.toDouble) - TColumn.doubleVal(new TDoubleColumn(values, nulls)) - - case Types.CHAR | Types.VARCHAR => - val values = getOrSetAsNull[String](rows, ordinal, nulls, "") - TColumn.stringVal(new TStringColumn(values, nulls)) - - case _ => - val rowSize = rows.length - val values = new java.util.ArrayList[String](rowSize) - var i = 0 - while (i < rowSize) { - val row = rows(i) - nulls.set(i, row(ordinal) == null) - val value = - if (row(ordinal) == null) { - "" - } else { - toHiveString(row(ordinal), sqlType) - } - values.add(value) - i += 1 - } - TColumn.stringVal(new TStringColumn(values, nulls)) - } - } - - protected def toTColumnValue(ordinal: Int, row: List[Any], types: List[Column]): TColumnValue = { - types(ordinal).sqlType match { - case Types.BIT => - val boolValue = new TBoolValue - if (row(ordinal) != null) boolValue.setValue(row(ordinal).asInstanceOf[Boolean]) - TColumnValue.boolVal(boolValue) - - case Types.TINYINT | Types.SMALLINT | Types.INTEGER => - val tI32Value = new TI32Value - if (row(ordinal) != null) tI32Value.setValue(row(ordinal).asInstanceOf[Int]) - TColumnValue.i32Val(tI32Value) - - case Types.BIGINT => - val tI64Value = new TI64Value - if (row(ordinal) != null) tI64Value.setValue(row(ordinal).asInstanceOf[Long]) - TColumnValue.i64Val(tI64Value) - - case Types.REAL => - val tDoubleValue = new TDoubleValue - if (row(ordinal) != null) { - val doubleValue = java.lang.Double.valueOf(row(ordinal).asInstanceOf[Float].toString) - tDoubleValue.setValue(doubleValue) - } - TColumnValue.doubleVal(tDoubleValue) - - case Types.DOUBLE => - val tDoubleValue = new TDoubleValue - if (row(ordinal) != null) tDoubleValue.setValue(row(ordinal).asInstanceOf[Double]) - TColumnValue.doubleVal(tDoubleValue) + override def toTinyIntTColumn(rows: Seq[Seq[Any]], ordinal: Int): TColumn = + toIntegerTColumn(rows, ordinal) - case Types.CHAR | Types.VARCHAR => - val tStringValue = new TStringValue - if (row(ordinal) != null) tStringValue.setValue(row(ordinal).asInstanceOf[String]) - TColumnValue.stringVal(tStringValue) + override def toSmallIntTColumn(rows: Seq[Seq[Any]], ordinal: Int): TColumn = + toIntegerTColumn(rows, ordinal) - case _ => - val tStrValue = new TStringValue - if (row(ordinal) != null) { - tStrValue.setValue( - toHiveString(row(ordinal), types(ordinal).sqlType)) - } - TColumnValue.stringVal(tStrValue) - } - } + override def toTinyIntTColumnValue(row: List[Any], ordinal: Int): TColumnValue = + toIntegerTColumnValue(row, ordinal) - protected def toHiveString(data: Any, sqlType: Int): String = { - (data, sqlType) match { - case (date: Date, Types.DATE) => - formatDate(date) - case (dateTime: LocalDateTime, Types.TIMESTAMP) => - formatLocalDateTime(dateTime) - case (decimal: java.math.BigDecimal, Types.DECIMAL) => - decimal.toPlainString - // TODO support bitmap and hll - case (other, _) => - other.toString - } - } + override def toSmallIntTColumnValue(row: List[Any], ordinal: Int): TColumnValue = + toIntegerTColumnValue(row, ordinal) } diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/doris/DorisSchemaHelper.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/doris/DorisSchemaHelper.scala index ca8bb6ec3..b323d3731 100644 --- a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/doris/DorisSchemaHelper.scala +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/doris/DorisSchemaHelper.scala @@ -16,44 +16,13 @@ */ package org.apache.kyuubi.engine.jdbc.doris -import java.sql.Types - import org.apache.hive.service.rpc.thrift._ import org.apache.kyuubi.engine.jdbc.schema.SchemaHelper class DorisSchemaHelper extends SchemaHelper { - override def toTTypeId(sqlType: Int): TTypeId = sqlType match { - case Types.BIT => - TTypeId.BOOLEAN_TYPE - - case Types.TINYINT | Types.SMALLINT | Types.INTEGER => - TTypeId.INT_TYPE - - case Types.BIGINT => - TTypeId.BIGINT_TYPE - - case Types.REAL => - TTypeId.FLOAT_TYPE - - case Types.DOUBLE => - TTypeId.DOUBLE_TYPE - - case Types.CHAR | Types.VARCHAR => - TTypeId.STRING_TYPE - - case Types.DATE => - TTypeId.DATE_TYPE - - case Types.TIMESTAMP => - TTypeId.TIMESTAMP_TYPE - - case Types.DECIMAL => - TTypeId.DECIMAL_TYPE + override def tinyIntToTTypeId: TTypeId = TTypeId.INT_TYPE - // TODO add more type support - case _ => - TTypeId.STRING_TYPE - } + override def smallIntToTTypeId: TTypeId = TTypeId.INT_TYPE } diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/operation/JdbcOperation.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/operation/JdbcOperation.scala index 6cac42f49..2ca173757 100644 --- a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/operation/JdbcOperation.scala +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/operation/JdbcOperation.scala @@ -16,7 +16,7 @@ */ package org.apache.kyuubi.engine.jdbc.operation -import org.apache.hive.service.rpc.thrift.{TGetResultSetMetadataResp, TRowSet} +import org.apache.hive.service.rpc.thrift.{TFetchResultsResp, TGetResultSetMetadataResp, TRowSet} import org.apache.kyuubi.{KyuubiSQLException, Utils} import org.apache.kyuubi.config.KyuubiConf @@ -36,7 +36,9 @@ abstract class JdbcOperation(session: Session) extends AbstractOperation(session protected lazy val dialect: JdbcDialect = JdbcDialects.get(conf) - override def getNextRowSet(order: FetchOrientation, rowSetSize: Int): TRowSet = { + override def getNextRowSetInternal( + order: FetchOrientation, + rowSetSize: Int): TFetchResultsResp = { validateDefaultFetchOrientation(order) assertState(OperationState.FINISHED) setHasResultSet(true) @@ -51,7 +53,10 @@ abstract class JdbcOperation(session: Session) extends AbstractOperation(session val taken = iter.take(rowSetSize) val resultRowSet = toTRowSet(taken) resultRowSet.setStartRowOffset(iter.getPosition) - resultRowSet + val resp = new TFetchResultsResp(OK_STATUS) + resp.setResults(resultRowSet) + resp.setHasMoreRows(false) + resp } override def cancel(): Unit = { @@ -66,7 +71,7 @@ abstract class JdbcOperation(session: Session) extends AbstractOperation(session // We should use Throwable instead of Exception since `java.lang.NoClassDefFoundError` // could be thrown. case e: Throwable => - state.synchronized { + withLockRequired { val errMsg = Utils.stringifyException(e) if (state == OperationState.TIMEOUT) { val ke = KyuubiSQLException(s"Timeout operating $opType: $errMsg") diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/phoenix/PhoenixRowSetHelper.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/phoenix/PhoenixRowSetHelper.scala index a1f6d4ac2..67d9d09e5 100644 --- a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/phoenix/PhoenixRowSetHelper.scala +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/phoenix/PhoenixRowSetHelper.scala @@ -16,144 +16,6 @@ */ package org.apache.kyuubi.engine.jdbc.phoenix -import java.sql.{Date, Types} -import java.time.LocalDateTime +import org.apache.kyuubi.engine.jdbc.schema.RowSetHelper -import scala.collection.JavaConverters._ - -import org.apache.hive.service.rpc.thrift._ - -import org.apache.kyuubi.engine.jdbc.schema.{Column, RowSetHelper} -import org.apache.kyuubi.util.RowSetUtils.{bitSetToBuffer, formatDate, formatLocalDateTime} - -class PhoenixRowSetHelper extends RowSetHelper { - - protected def toTColumn( - rows: Seq[Seq[Any]], - ordinal: Int, - sqlType: Int): TColumn = { - val nulls = new java.util.BitSet() - sqlType match { - - case Types.BIT => - val values = getOrSetAsNull[java.lang.Boolean](rows, ordinal, nulls, true) - TColumn.boolVal(new TBoolColumn(values, nulls)) - - case Types.TINYINT => - val values = getOrSetAsNull[java.lang.Byte](rows, ordinal, nulls, 0.toByte) - TColumn.byteVal(new TByteColumn(values, nulls)) - - case Types.SMALLINT => - val values = getOrSetAsNull[java.lang.Short](rows, ordinal, nulls, 0.toShort) - TColumn.i16Val(new TI16Column(values, nulls)) - - case Types.INTEGER => - val values = getOrSetAsNull[java.lang.Integer](rows, ordinal, nulls, 0) - TColumn.i32Val(new TI32Column(values, nulls)) - - case Types.BIGINT => - val values = getOrSetAsNull[java.lang.Long](rows, ordinal, nulls, 0L) - TColumn.i64Val(new TI64Column(values, nulls)) - - case Types.REAL => - val values = getOrSetAsNull[java.lang.Float](rows, ordinal, nulls, 0.toFloat) - .asScala.map(n => java.lang.Double.valueOf(n.toString)).asJava - TColumn.doubleVal(new TDoubleColumn(values, nulls)) - - case Types.DOUBLE => - val values = getOrSetAsNull[java.lang.Double](rows, ordinal, nulls, 0.toDouble) - TColumn.doubleVal(new TDoubleColumn(values, nulls)) - - case Types.CHAR | Types.VARCHAR => - val values = getOrSetAsNull[String](rows, ordinal, nulls, "") - TColumn.stringVal(new TStringColumn(values, nulls)) - - case _ => - val rowSize = rows.length - val values = new java.util.ArrayList[String](rowSize) - var i = 0 - while (i < rowSize) { - val row = rows(i) - nulls.set(i, row(ordinal) == null) - val value = - if (row(ordinal) == null) { - "" - } else { - toHiveString(row(ordinal), sqlType) - } - values.add(value) - i += 1 - } - TColumn.stringVal(new TStringColumn(values, nulls)) - } - } - - protected def toTColumnValue(ordinal: Int, row: List[Any], types: List[Column]): TColumnValue = { - types(ordinal).sqlType match { - case Types.BIT => - val boolValue = new TBoolValue - if (row(ordinal) != null) boolValue.setValue(row(ordinal).asInstanceOf[Boolean]) - TColumnValue.boolVal(boolValue) - - case Types.TINYINT => - val byteValue = new TByteValue() - if (row(ordinal) != null) byteValue.setValue(row(ordinal).asInstanceOf[Byte]) - TColumnValue.byteVal(byteValue) - - case Types.SMALLINT => - val tI16Value = new TI16Value() - if (row(ordinal) != null) tI16Value.setValue(row(ordinal).asInstanceOf[Short]) - TColumnValue.i16Val(tI16Value) - - case Types.INTEGER => - val tI32Value = new TI32Value - if (row(ordinal) != null) tI32Value.setValue(row(ordinal).asInstanceOf[Int]) - TColumnValue.i32Val(tI32Value) - - case Types.BIGINT => - val tI64Value = new TI64Value - if (row(ordinal) != null) tI64Value.setValue(row(ordinal).asInstanceOf[Long]) - TColumnValue.i64Val(tI64Value) - - case Types.REAL => - val tDoubleValue = new TDoubleValue - if (row(ordinal) != null) { - val doubleValue = java.lang.Double.valueOf(row(ordinal).asInstanceOf[Float].toString) - tDoubleValue.setValue(doubleValue) - } - TColumnValue.doubleVal(tDoubleValue) - - case Types.DOUBLE => - val tDoubleValue = new TDoubleValue - if (row(ordinal) != null) tDoubleValue.setValue(row(ordinal).asInstanceOf[Double]) - TColumnValue.doubleVal(tDoubleValue) - - case Types.CHAR | Types.VARCHAR => - val tStringValue = new TStringValue - if (row(ordinal) != null) tStringValue.setValue(row(ordinal).asInstanceOf[String]) - TColumnValue.stringVal(tStringValue) - - case _ => - val tStrValue = new TStringValue - if (row(ordinal) != null) { - tStrValue.setValue( - toHiveString(row(ordinal), types(ordinal).sqlType)) - } - TColumnValue.stringVal(tStrValue) - } - } - - protected def toHiveString(data: Any, sqlType: Int): String = { - (data, sqlType) match { - case (date: Date, Types.DATE) => - formatDate(date) - case (dateTime: LocalDateTime, Types.TIMESTAMP) => - formatLocalDateTime(dateTime) - case (decimal: java.math.BigDecimal, Types.DECIMAL) => - decimal.toPlainString - // TODO support bitmap and hll - case (other, _) => - other.toString - } - } -} +class PhoenixRowSetHelper extends RowSetHelper {} diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/phoenix/PhoenixSchemaHelper.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/phoenix/PhoenixSchemaHelper.scala index f5e04f7ca..938956cdc 100644 --- a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/phoenix/PhoenixSchemaHelper.scala +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/phoenix/PhoenixSchemaHelper.scala @@ -16,50 +16,6 @@ */ package org.apache.kyuubi.engine.jdbc.phoenix -import java.sql.Types - -import org.apache.hive.service.rpc.thrift._ - import org.apache.kyuubi.engine.jdbc.schema.SchemaHelper -class PhoenixSchemaHelper extends SchemaHelper { - - override def toTTypeId(sqlType: Int): TTypeId = sqlType match { - case Types.BIT => - TTypeId.BOOLEAN_TYPE - - case Types.TINYINT => - TTypeId.TINYINT_TYPE - - case Types.SMALLINT => - TTypeId.SMALLINT_TYPE - - case Types.INTEGER => - TTypeId.INT_TYPE - - case Types.BIGINT => - TTypeId.BIGINT_TYPE - - case Types.REAL => - TTypeId.FLOAT_TYPE - - case Types.DOUBLE => - TTypeId.DOUBLE_TYPE - - case Types.CHAR | Types.VARCHAR => - TTypeId.STRING_TYPE - - case Types.DATE => - TTypeId.DATE_TYPE - - case Types.TIMESTAMP => - TTypeId.TIMESTAMP_TYPE - - case Types.DECIMAL => - TTypeId.DECIMAL_TYPE - - // TODO add more type support - case _ => - TTypeId.STRING_TYPE - } -} +class PhoenixSchemaHelper extends SchemaHelper {} diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/schema/RowSetHelper.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/schema/RowSetHelper.scala index d489ed8a2..74b4cec10 100644 --- a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/schema/RowSetHelper.scala +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/schema/RowSetHelper.scala @@ -16,10 +16,16 @@ */ package org.apache.kyuubi.engine.jdbc.schema -import java.util +import java.{lang, util} +import java.sql.{Date, Types} +import java.time.LocalDateTime + +import scala.collection.JavaConverters._ import org.apache.hive.service.rpc.thrift._ +import org.apache.kyuubi.util.RowSetUtils.{bitSetToBuffer, formatDate, formatLocalDateTime} + abstract class RowSetHelper { def toTRowSet( @@ -70,9 +76,73 @@ abstract class RowSetHelper { protected def toTColumn( rows: Seq[Seq[Any]], ordinal: Int, - sqlType: Int): TColumn + sqlType: Int): TColumn = { + sqlType match { + case Types.BIT => + toBitTColumn(rows, ordinal) + + case Types.TINYINT => + toTinyIntTColumn(rows, ordinal) + + case Types.SMALLINT => + toSmallIntTColumn(rows, ordinal) + + case Types.INTEGER => + toIntegerTColumn(rows, ordinal) + + case Types.BIGINT => + toBigIntTColumn(rows, ordinal) + + case Types.REAL => + toRealTColumn(rows, ordinal) + + case Types.DOUBLE => + toDoubleTColumn(rows, ordinal) + + case Types.CHAR => + toCharTColumn(rows, ordinal) + + case Types.VARCHAR => + toVarcharTColumn(rows, ordinal) + + case _ => + toDefaultTColumn(rows, ordinal, sqlType) + } + } + + protected def toTColumnValue(ordinal: Int, row: List[Any], types: List[Column]): TColumnValue = { + types(ordinal).sqlType match { + case Types.BIT => + toBitTColumnValue(row, ordinal) + + case Types.TINYINT => + toTinyIntTColumnValue(row, ordinal) + + case Types.SMALLINT => + toSmallIntTColumnValue(row, ordinal) + + case Types.INTEGER => + toIntegerTColumnValue(row, ordinal) + + case Types.BIGINT => + toBigIntTColumnValue(row, ordinal) + + case Types.REAL => + toRealTColumnValue(row, ordinal) + + case Types.DOUBLE => + toDoubleTColumnValue(row, ordinal) - protected def toTColumnValue(ordinal: Int, row: List[Any], types: List[Column]): TColumnValue + case Types.CHAR => + toCharTColumnValue(row, ordinal) + + case Types.VARCHAR => + toVarcharTColumnValue(row, ordinal) + + case _ => + toDefaultTColumnValue(row, ordinal, types) + } + } protected def getOrSetAsNull[T]( rows: Seq[Seq[Any]], @@ -95,4 +165,159 @@ abstract class RowSetHelper { } ret } + + protected def toDefaultTColumn(rows: Seq[Seq[Any]], ordinal: Int, sqlType: Int): TColumn = { + val nulls = new java.util.BitSet() + val rowSize = rows.length + val values = new util.ArrayList[String](rowSize) + var i = 0 + while (i < rowSize) { + val row = rows(i) + nulls.set(i, row(ordinal) == null) + val value = + if (row(ordinal) == null) { + "" + } else { + toHiveString(row(ordinal), sqlType) + } + values.add(value) + i += 1 + } + TColumn.stringVal(new TStringColumn(values, nulls)) + } + + protected def toBitTColumn(rows: Seq[Seq[Any]], ordinal: Int): TColumn = { + val nulls = new java.util.BitSet() + val values = getOrSetAsNull[java.lang.Boolean](rows, ordinal, nulls, true) + TColumn.boolVal(new TBoolColumn(values, nulls)) + } + + protected def toTinyIntTColumn(rows: Seq[Seq[Any]], ordinal: Int): TColumn = { + val nulls = new java.util.BitSet() + val values = getOrSetAsNull[java.lang.Byte](rows, ordinal, nulls, 0.toByte) + TColumn.byteVal(new TByteColumn(values, nulls)) + } + + protected def toSmallIntTColumn(rows: Seq[Seq[Any]], ordinal: Int): TColumn = { + val nulls = new java.util.BitSet() + val values = getOrSetAsNull[java.lang.Short](rows, ordinal, nulls, 0.toShort) + TColumn.i16Val(new TI16Column(values, nulls)) + } + + protected def toIntegerTColumn(rows: Seq[Seq[Any]], ordinal: Int): TColumn = { + val nulls = new java.util.BitSet() + val values = getOrSetAsNull[java.lang.Integer](rows, ordinal, nulls, 0) + TColumn.i32Val(new TI32Column(values, nulls)) + } + + protected def toBigIntTColumn(rows: Seq[Seq[Any]], ordinal: Int): TColumn = { + val nulls = new java.util.BitSet() + val values = getOrSetAsNull[lang.Long](rows, ordinal, nulls, 0L) + TColumn.i64Val(new TI64Column(values, nulls)) + } + + protected def toRealTColumn(rows: Seq[Seq[Any]], ordinal: Int): TColumn = { + val nulls = new java.util.BitSet() + val values = getOrSetAsNull[lang.Float](rows, ordinal, nulls, 0.toFloat) + .asScala.map(n => java.lang.Double.valueOf(n.toString)).asJava + TColumn.doubleVal(new TDoubleColumn(values, nulls)) + } + + protected def toDoubleTColumn(rows: Seq[Seq[Any]], ordinal: Int): TColumn = { + val nulls = new java.util.BitSet() + val values = getOrSetAsNull[lang.Double](rows, ordinal, nulls, 0.toDouble) + TColumn.doubleVal(new TDoubleColumn(values, nulls)) + } + + protected def toCharTColumn(rows: Seq[Seq[Any]], ordinal: Int): TColumn = { + toVarcharTColumn(rows, ordinal) + } + + protected def toVarcharTColumn(rows: Seq[Seq[Any]], ordinal: Int): TColumn = { + val nulls = new java.util.BitSet() + val values = getOrSetAsNull[String](rows, ordinal, nulls, "") + TColumn.stringVal(new TStringColumn(values, nulls)) + } + + // ========================================================== + + protected def toBitTColumnValue(row: List[Any], ordinal: Int): TColumnValue = { + val boolValue = new TBoolValue + if (row(ordinal) != null) boolValue.setValue(row(ordinal).asInstanceOf[Boolean]) + TColumnValue.boolVal(boolValue) + } + + protected def toTinyIntTColumnValue(row: List[Any], ordinal: Int): TColumnValue = { + val byteValue = new TByteValue + if (row(ordinal) != null) byteValue.setValue(row(ordinal).asInstanceOf[Byte]) + TColumnValue.byteVal(byteValue) + } + + protected def toSmallIntTColumnValue(row: List[Any], ordinal: Int): TColumnValue = { + val tI16Value = new TI16Value + if (row(ordinal) != null) tI16Value.setValue(row(ordinal).asInstanceOf[Short]) + TColumnValue.i16Val(tI16Value) + } + + protected def toIntegerTColumnValue(row: List[Any], ordinal: Int): TColumnValue = { + val tI32Value = new TI32Value + if (row(ordinal) != null) tI32Value.setValue(row(ordinal).asInstanceOf[Int]) + TColumnValue.i32Val(tI32Value) + } + + protected def toBigIntTColumnValue(row: List[Any], ordinal: Int): TColumnValue = { + val tI64Value = new TI64Value + if (row(ordinal) != null) tI64Value.setValue(row(ordinal).asInstanceOf[Long]) + TColumnValue.i64Val(tI64Value) + } + + protected def toRealTColumnValue(row: List[Any], ordinal: Int): TColumnValue = { + val tDoubleValue = new TDoubleValue + if (row(ordinal) != null) { + val doubleValue = java.lang.Double.valueOf(row(ordinal).asInstanceOf[Float].toString) + tDoubleValue.setValue(doubleValue) + } + TColumnValue.doubleVal(tDoubleValue) + } + + protected def toDoubleTColumnValue(row: List[Any], ordinal: Int): TColumnValue = { + val tDoubleValue = new TDoubleValue + if (row(ordinal) != null) tDoubleValue.setValue(row(ordinal).asInstanceOf[Double]) + TColumnValue.doubleVal(tDoubleValue) + } + + protected def toCharTColumnValue(row: List[Any], ordinal: Int): TColumnValue = { + toVarcharTColumnValue(row, ordinal) + } + + protected def toVarcharTColumnValue(row: List[Any], ordinal: Int): TColumnValue = { + val tStringValue = new TStringValue + if (row(ordinal) != null) tStringValue.setValue(row(ordinal).asInstanceOf[String]) + TColumnValue.stringVal(tStringValue) + } + + protected def toDefaultTColumnValue( + row: List[Any], + ordinal: Int, + types: List[Column]): TColumnValue = { + val tStrValue = new TStringValue + if (row(ordinal) != null) { + tStrValue.setValue( + toHiveString(row(ordinal), types(ordinal).sqlType)) + } + TColumnValue.stringVal(tStrValue) + } + + protected def toHiveString(data: Any, sqlType: Int): String = { + (data, sqlType) match { + case (date: Date, Types.DATE) => + formatDate(date) + case (dateTime: LocalDateTime, Types.TIMESTAMP) => + formatLocalDateTime(dateTime) + case (decimal: java.math.BigDecimal, Types.DECIMAL) => + decimal.toPlainString + case (other, _) => + other.toString + } + } } diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/schema/SchemaHelper.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/schema/SchemaHelper.scala index 3be3c7d42..455eb2a92 100644 --- a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/schema/SchemaHelper.scala +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/schema/SchemaHelper.scala @@ -16,6 +16,7 @@ */ package org.apache.kyuubi.engine.jdbc.schema +import java.sql.Types import java.util.Collections import scala.collection.JavaConverters._ @@ -62,6 +63,97 @@ abstract class SchemaHelper { ret } - protected def toTTypeId(sqlType: Int): TTypeId + protected def toTTypeId(sqlType: Int): TTypeId = sqlType match { + case Types.BIT => + bitToTTypeId + case Types.TINYINT => + tinyIntToTTypeId + + case Types.SMALLINT => + smallIntToTTypeId + + case Types.INTEGER => + integerToTTypeId + + case Types.BIGINT => + bigintToTTypeId + + case Types.REAL => + realToTTypeId + + case Types.DOUBLE => + doubleToTTypeId + + case Types.CHAR => + charToTTypeId + + case Types.VARCHAR => + varcharToTTypeId + + case Types.DATE => + dateToTTypeId + + case Types.TIMESTAMP => + timestampToTTypeId + + case Types.DECIMAL => + decimalToTTypeId + + // TODO add more type support + case _ => + defaultToTTypeId + } + + protected def bitToTTypeId: TTypeId = { + TTypeId.BOOLEAN_TYPE + } + + protected def tinyIntToTTypeId: TTypeId = { + TTypeId.TINYINT_TYPE + } + + protected def smallIntToTTypeId: TTypeId = { + TTypeId.SMALLINT_TYPE + } + + protected def integerToTTypeId: TTypeId = { + TTypeId.INT_TYPE + } + + protected def bigintToTTypeId: TTypeId = { + TTypeId.BIGINT_TYPE + } + + protected def realToTTypeId: TTypeId = { + TTypeId.FLOAT_TYPE + } + + protected def doubleToTTypeId: TTypeId = { + TTypeId.DOUBLE_TYPE + } + + protected def charToTTypeId: TTypeId = { + TTypeId.STRING_TYPE + } + + protected def varcharToTTypeId: TTypeId = { + TTypeId.STRING_TYPE + } + + protected def dateToTTypeId: TTypeId = { + TTypeId.DATE_TYPE + } + + protected def timestampToTTypeId: TTypeId = { + TTypeId.TIMESTAMP_TYPE + } + + protected def decimalToTTypeId: TTypeId = { + TTypeId.DECIMAL_TYPE + } + + protected def defaultToTTypeId: TTypeId = { + TTypeId.STRING_TYPE + } } diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/session/JdbcSessionImpl.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/session/JdbcSessionImpl.scala index 63fb2dd07..8b36e5a56 100644 --- a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/session/JdbcSessionImpl.scala +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/session/JdbcSessionImpl.scala @@ -23,8 +23,12 @@ import scala.util.{Failure, Success, Try} import org.apache.hive.service.rpc.thrift.{TGetInfoType, TGetInfoValue, TProtocolVersion} import org.apache.kyuubi.KyuubiSQLException +import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.config.KyuubiConf._ +import org.apache.kyuubi.config.KyuubiReservedKeys.KYUUBI_SESSION_HANDLE_KEY import org.apache.kyuubi.engine.jdbc.connection.ConnectionProvider -import org.apache.kyuubi.session.{AbstractSession, SessionManager} +import org.apache.kyuubi.engine.jdbc.util.KyuubiJdbcUtils +import org.apache.kyuubi.session.{AbstractSession, SessionHandle, SessionManager} class JdbcSessionImpl( protocol: TProtocolVersion, @@ -35,11 +39,23 @@ class JdbcSessionImpl( sessionManager: SessionManager) extends AbstractSession(protocol, user, password, ipAddress, conf, sessionManager) { + override val handle: SessionHandle = + conf.get(KYUUBI_SESSION_HANDLE_KEY).map(SessionHandle.fromUUID).getOrElse(SessionHandle()) + private[jdbc] var sessionConnection: Connection = _ private var databaseMetaData: DatabaseMetaData = _ - private val kyuubiConf = sessionManager.getConf + private val kyuubiConf: KyuubiConf = normalizeConf + + private def normalizeConf: KyuubiConf = { + val kyuubiConf = sessionManager.getConf.clone + if (kyuubiConf.get(ENGINE_JDBC_CONNECTION_PROPAGATECREDENTIAL)) { + kyuubiConf.set(ENGINE_JDBC_CONNECTION_USER, user) + kyuubiConf.set(ENGINE_JDBC_CONNECTION_PASSWORD, password) + } + kyuubiConf + } override def open(): Unit = { info(s"Starting to open jdbc session.") @@ -47,6 +63,10 @@ class JdbcSessionImpl( sessionConnection = ConnectionProvider.create(kyuubiConf) databaseMetaData = sessionConnection.getMetaData } + KyuubiJdbcUtils.initializeJdbcSession( + kyuubiConf, + sessionConnection, + kyuubiConf.get(ENGINE_JDBC_SESSION_INITIALIZE_SQL)) super.open() info(s"The jdbc session is started.") } diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/session/JdbcSessionManager.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/session/JdbcSessionManager.scala index db8f60c3c..09958e050 100644 --- a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/session/JdbcSessionManager.scala +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/session/JdbcSessionManager.scala @@ -20,6 +20,7 @@ import org.apache.hive.service.rpc.thrift.TProtocolVersion import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.config.KyuubiConf.ENGINE_SHARE_LEVEL +import org.apache.kyuubi.config.KyuubiReservedKeys.KYUUBI_SESSION_HANDLE_KEY import org.apache.kyuubi.engine.ShareLevel import org.apache.kyuubi.engine.jdbc.JdbcSQLEngine import org.apache.kyuubi.engine.jdbc.operation.JdbcOperationManager @@ -46,7 +47,10 @@ class JdbcSessionManager(name: String) password: String, ipAddress: String, conf: Map[String, String]): Session = { - new JdbcSessionImpl(protocol, user, password, ipAddress, conf, this) + conf.get(KYUUBI_SESSION_HANDLE_KEY).map(SessionHandle.fromUUID).flatMap( + getSessionOption).getOrElse { + new JdbcSessionImpl(protocol, user, password, ipAddress, conf, this) + } } override def closeSession(sessionHandle: SessionHandle): Unit = { diff --git a/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/util/KyuubiJdbcUtils.scala b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/util/KyuubiJdbcUtils.scala new file mode 100644 index 000000000..7107045ff --- /dev/null +++ b/externals/kyuubi-jdbc-engine/src/main/scala/org/apache/kyuubi/engine/jdbc/util/KyuubiJdbcUtils.scala @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.engine.jdbc.util + +import java.sql.Connection + +import org.apache.kyuubi.{KyuubiSQLException, Logging} +import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.engine.jdbc.connection.ConnectionProvider +import org.apache.kyuubi.engine.jdbc.dialect.{JdbcDialect, JdbcDialects} +import org.apache.kyuubi.util.JdbcUtils + +object KyuubiJdbcUtils extends Logging { + + def initializeJdbcSession(kyuubiConf: KyuubiConf, initializationSQLs: Seq[String]): Unit = { + JdbcUtils.withCloseable(ConnectionProvider.create(kyuubiConf)) { connection => + initializeJdbcSession(kyuubiConf, connection, initializationSQLs) + } + } + + def initializeJdbcSession( + kyuubiConf: KyuubiConf, + connection: Connection, + initializationSQLs: Seq[String]): Unit = { + if (initializationSQLs == null || initializationSQLs.isEmpty) { + return + } + try { + val dialect: JdbcDialect = JdbcDialects.get(kyuubiConf) + JdbcUtils.withCloseable(dialect.createStatement(connection)) { statement => + initializationSQLs.foreach { sql => + debug(s"Execute initialization sql: $sql") + statement.execute(sql) + } + } + } catch { + case e: Exception => + error("Failed to execute initialization sql.", e) + throw KyuubiSQLException(e) + } + } +} diff --git a/externals/kyuubi-jdbc-engine/src/test/resources/log4j2-test.xml b/externals/kyuubi-jdbc-engine/src/test/resources/log4j2-test.xml index bfc40dd6d..3110216c1 100644 --- a/externals/kyuubi-jdbc-engine/src/test/resources/log4j2-test.xml +++ b/externals/kyuubi-jdbc-engine/src/test/resources/log4j2-test.xml @@ -21,14 +21,14 @@ - + - + diff --git a/externals/kyuubi-spark-sql-engine/pom.xml b/externals/kyuubi-spark-sql-engine/pom.xml index 3d3259c80..555e41a44 100644 --- a/externals/kyuubi-spark-sql-engine/pom.xml +++ b/externals/kyuubi-spark-sql-engine/pom.xml @@ -21,11 +21,11 @@ org.apache.kyuubi kyuubi-parent - 1.7.0-SNAPSHOT + 1.9.0-SNAPSHOT ../../pom.xml - kyuubi-spark-sql-engine_2.12 + kyuubi-spark-sql-engine_${scala.binary.version} jar Kyuubi Project Engine Spark SQL https://kyuubi.apache.org/ @@ -65,6 +65,13 @@ provided + + org.apache.spark + spark-sql_${scala.binary.version} + test-jar + test + + org.apache.spark spark-repl_${scala.binary.version} @@ -140,69 +147,77 @@ - org.apache.parquet - parquet-avro - test - - - - org.apache.spark - spark-avro_${scala.binary.version} - test - - - - org.apache.hudi - hudi-common + io.delta + ${delta.artifact}_${scala.binary.version} test - org.apache.hudi - hudi-spark-common_${scala.binary.version} + org.apache.kyuubi + kyuubi-zookeeper_${scala.binary.version} + ${project.version} test - org.apache.hudi - hudi-spark_${scala.binary.version} + com.dimafeng + testcontainers-scala-scalatest_${scala.binary.version} test - org.apache.hudi - hudi-spark3.1.x_${scala.binary.version} + io.etcd + jetcd-launcher test - io.delta - delta-core_${scala.binary.version} + com.vladsch.flexmark + flexmark-all test org.apache.kyuubi - kyuubi-zookeeper_${scala.binary.version} + kyuubi-spark-lineage_${scala.binary.version} ${project.version} test - - - io.etcd - jetcd-launcher - test - - - - com.vladsch.flexmark - flexmark-all - test - + + + org.codehaus.mojo + build-helper-maven-plugin + + + add-scala-sources + + add-source + + generate-sources + + + src/main/scala-${scala.binary.version} + + + + + add-scala-test-sources + + add-test-source + + generate-test-sources + + + src/test/scala-${scala.binary.version} + + + + + org.apache.maven.plugins maven-shade-plugin @@ -223,15 +238,9 @@ io.perfmark:perfmark-api io.vertx:* net.jodah:failsafe - org.apache.curator:curator-client - org.apache.curator:curator-framework - org.apache.curator:curator-recipes org.apache.hive:hive-service-rpc - org.apache.kyuubi:kyuubi-common_${scala.binary.version} - org.apache.kyuubi:kyuubi-events_${scala.binary.version} - org.apache.kyuubi:kyuubi-ha_${scala.binary.version} + org.apache.kyuubi:* org.apache.thrift:* - org.apache.zookeeper:zookeeper org.checkerframework:checker-qual org.codehaus.mojo:animal-sniffer-annotations @@ -256,27 +265,6 @@ - - org.apache.curator - ${kyuubi.shade.packageName}.org.apache.curator - - org.apache.curator.** - - - - org.apache.zookeeper - ${kyuubi.shade.packageName}.org.apache.zookeeper - - org.apache.zookeeper.** - - - - org.apache.jute - ${kyuubi.shade.packageName}.org.apache.jute - - org.apache.jute.** - - org.apache.hive.service.rpc.thrift ${kyuubi.shade.packageName}.org.apache.hive.service.rpc.thrift diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/repl/KyuubiSparkILoop.scala b/externals/kyuubi-spark-sql-engine/src/main/scala-2.12/org/apache/kyuubi/engine/spark/repl/KyuubiSparkILoop.scala similarity index 90% rename from externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/repl/KyuubiSparkILoop.scala rename to externals/kyuubi-spark-sql-engine/src/main/scala-2.12/org/apache/kyuubi/engine/spark/repl/KyuubiSparkILoop.scala index 27090fae4..fbbda89ed 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/repl/KyuubiSparkILoop.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala-2.12/org/apache/kyuubi/engine/spark/repl/KyuubiSparkILoop.scala @@ -17,22 +17,23 @@ package org.apache.kyuubi.engine.spark.repl -import java.io.{ByteArrayOutputStream, File} +import java.io.{ByteArrayOutputStream, File, PrintWriter} import java.util.concurrent.locks.ReentrantLock import scala.tools.nsc.Settings -import scala.tools.nsc.interpreter.IR -import scala.tools.nsc.interpreter.JPrintWriter +import scala.tools.nsc.interpreter.Results import org.apache.spark.SparkContext import org.apache.spark.repl.SparkILoop import org.apache.spark.sql.{DataFrame, SparkSession} import org.apache.spark.util.MutableURLClassLoader +import org.apache.kyuubi.Utils + private[spark] case class KyuubiSparkILoop private ( spark: SparkSession, output: ByteArrayOutputStream) - extends SparkILoop(None, new JPrintWriter(output)) { + extends SparkILoop(None, new PrintWriter(output)) { import KyuubiSparkILoop._ val result = new DataFrameHolder(spark) @@ -100,7 +101,7 @@ private[spark] case class KyuubiSparkILoop private ( def clearResult(statementId: String): Unit = result.unset(statementId) - def interpretWithRedirectOutError(statement: String): IR.Result = withLockRequired { + def interpretWithRedirectOutError(statement: String): Results.Result = withLockRequired { Console.withOut(output) { Console.withErr(output) { this.interpret(statement) @@ -124,10 +125,5 @@ private[spark] object KyuubiSparkILoop { } private val lock = new ReentrantLock() - private def withLockRequired[T](block: => T): T = { - try { - lock.lock() - block - } finally lock.unlock() - } + private def withLockRequired[T](block: => T): T = Utils.withLockRequired(lock)(block) } diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala-2.13/org/apache/kyuubi/engine/spark/repl/KyuubiSparkILoop.scala b/externals/kyuubi-spark-sql-engine/src/main/scala-2.13/org/apache/kyuubi/engine/spark/repl/KyuubiSparkILoop.scala new file mode 100644 index 000000000..a63d71a78 --- /dev/null +++ b/externals/kyuubi-spark-sql-engine/src/main/scala-2.13/org/apache/kyuubi/engine/spark/repl/KyuubiSparkILoop.scala @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.engine.spark.repl + +import java.io.{ByteArrayOutputStream, File, PrintWriter} +import java.util.concurrent.locks.ReentrantLock + +import scala.tools.nsc.Settings +import scala.tools.nsc.interpreter.{IMain, Results} + +import org.apache.spark.SparkContext +import org.apache.spark.repl.SparkILoop +import org.apache.spark.sql.{DataFrame, SparkSession} +import org.apache.spark.util.MutableURLClassLoader + +import org.apache.kyuubi.Utils + +private[spark] case class KyuubiSparkILoop private ( + spark: SparkSession, + output: ByteArrayOutputStream) + extends SparkILoop(null, new PrintWriter(output)) { + import KyuubiSparkILoop._ + + val result = new DataFrameHolder(spark) + + private def initialize(): Unit = withLockRequired { + val settings = new Settings + val interpArguments = List( + "-Yrepl-class-based", + "-Yrepl-outdir", + s"${spark.sparkContext.getConf.get("spark.repl.class.outputDir")}") + settings.processArguments(interpArguments, processAll = true) + settings.usejavacp.value = true + val currentClassLoader = Thread.currentThread().getContextClassLoader + settings.embeddedDefaults(currentClassLoader) + this.createInterpreter(settings) + val iMain = this.intp.asInstanceOf[IMain] + iMain.initializeCompiler() + try { + this.compilerClasspath + iMain.ensureClassLoader() + var classLoader: ClassLoader = Thread.currentThread().getContextClassLoader + while (classLoader != null) { + classLoader match { + case loader: MutableURLClassLoader => + val allJars = loader.getURLs.filter { u => + val file = new File(u.getPath) + u.getProtocol == "file" && file.isFile && + file.getName.contains("scala-lang_scala-reflect") + } + this.addUrlsToClassPath(allJars: _*) + classLoader = null + case _ => + classLoader = classLoader.getParent + } + } + + this.addUrlsToClassPath( + classOf[DataFrameHolder].getProtectionDomain.getCodeSource.getLocation) + } finally { + Thread.currentThread().setContextClassLoader(currentClassLoader) + } + + this.beQuietDuring { + // SparkSession/SparkContext and their implicits + this.bind("spark", classOf[SparkSession].getCanonicalName, spark, List("""@transient""")) + this.bind( + "sc", + classOf[SparkContext].getCanonicalName, + spark.sparkContext, + List("""@transient""")) + + this.interpret("import org.apache.spark.SparkContext._") + this.interpret("import spark.implicits._") + this.interpret("import spark.sql") + this.interpret("import org.apache.spark.sql.functions._") + + // for feeding results to client, e.g. beeline + this.bind( + "result", + classOf[DataFrameHolder].getCanonicalName, + result) + } + } + + def getResult(statementId: String): DataFrame = result.get(statementId) + + def clearResult(statementId: String): Unit = result.unset(statementId) + + def interpretWithRedirectOutError(statement: String): Results.Result = withLockRequired { + Console.withOut(output) { + Console.withErr(output) { + this.interpret(statement) + } + } + } + + def getOutput: String = { + val res = output.toString.trim + output.reset() + res + } +} + +private[spark] object KyuubiSparkILoop { + def apply(spark: SparkSession): KyuubiSparkILoop = { + val os = new ByteArrayOutputStream() + val iLoop = new KyuubiSparkILoop(spark, os) + iLoop.initialize() + iLoop + } + + private val lock = new ReentrantLock() + private def withLockRequired[T](block: => T): T = Utils.withLockRequired(lock)(block) +} diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/KyuubiSparkUtil.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/KyuubiSparkUtil.scala index 2c3e7195c..b9fb93259 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/KyuubiSparkUtil.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/KyuubiSparkUtil.scala @@ -21,12 +21,12 @@ import java.time.{Instant, LocalDateTime, ZoneId} import scala.annotation.meta.getter -import org.apache.spark.SparkContext +import org.apache.spark.{SPARK_VERSION, SparkContext} import org.apache.spark.sql.SparkSession import org.apache.spark.util.kvstore.KVIndex import org.apache.kyuubi.Logging -import org.apache.kyuubi.engine.SemanticVersion +import org.apache.kyuubi.util.SemanticVersion object KyuubiSparkUtil extends Logging { @@ -95,9 +95,7 @@ object KyuubiSparkUtil extends Logging { } } - lazy val sparkMajorMinorVersion: (Int, Int) = { - val runtimeSparkVer = org.apache.spark.SPARK_VERSION - val runtimeVersion = SemanticVersion(runtimeSparkVer) - (runtimeVersion.majorVersion, runtimeVersion.minorVersion) - } + // Given that we are on the Spark SQL engine side, the [[org.apache.spark.SPARK_VERSION]] can be + // represented as the runtime version of the Spark SQL engine. + lazy val SPARK_ENGINE_RUNTIME_VERSION: SemanticVersion = SemanticVersion(SPARK_VERSION) } diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkSQLEngine.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkSQLEngine.scala index 42e7c44a1..ba84e1b1b 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkSQLEngine.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkSQLEngine.scala @@ -17,7 +17,6 @@ package org.apache.kyuubi.engine.spark -import java.net.InetAddress import java.time.Instant import java.util.{Locale, UUID} import java.util.concurrent.{CountDownLatch, ScheduledExecutorService, ThreadPoolExecutor, TimeUnit} @@ -36,7 +35,8 @@ import org.apache.kyuubi.{KyuubiException, Logging, Utils} import org.apache.kyuubi.Utils._ import org.apache.kyuubi.config.{KyuubiConf, KyuubiReservedKeys} import org.apache.kyuubi.config.KyuubiConf._ -import org.apache.kyuubi.config.KyuubiReservedKeys.KYUUBI_ENGINE_SUBMIT_TIME_KEY +import org.apache.kyuubi.config.KyuubiReservedKeys.{KYUUBI_ENGINE_SUBMIT_TIME_KEY, KYUUBI_ENGINE_URL} +import org.apache.kyuubi.engine.ShareLevel import org.apache.kyuubi.engine.spark.SparkSQLEngine.{countDownLatch, currentEngine} import org.apache.kyuubi.engine.spark.events.{EngineEvent, EngineEventsStore, SparkEventHandlerRegister} import org.apache.kyuubi.engine.spark.session.SparkSessionImpl @@ -80,6 +80,12 @@ case class SparkSQLEngine(spark: SparkSession) extends Serverable("SparkSQLEngin assert(currentEngine.isDefined) currentEngine.get.stop() }) + + val maxInitTimeout = conf.get(ENGINE_SPARK_MAX_INITIAL_WAIT) + if (conf.get(ENGINE_SHARE_LEVEL) == ShareLevel.CONNECTION.toString && + maxInitTimeout > 0) { + startFastFailChecker(maxInitTimeout) + } } override def stop(): Unit = if (shutdown.compareAndSet(false, true)) { @@ -114,6 +120,27 @@ case class SparkSQLEngine(spark: SparkSession) extends Serverable("SparkSQLEngin stopEngineExec.get.execute(stopTask) } + private[kyuubi] def startFastFailChecker(maxTimeout: Long): Unit = { + val startedTime = System.currentTimeMillis() + Utils.tryLogNonFatalError { + ThreadUtils.runInNewThread("spark-engine-failfast-checker") { + if (!shutdown.get) { + while (backendService.sessionManager.getOpenSessionCount <= 0 && + System.currentTimeMillis() - startedTime < maxTimeout) { + info(s"Waiting for the initial connection") + Thread.sleep(Duration(10, TimeUnit.SECONDS).toMillis) + } + if (backendService.sessionManager.getOpenSessionCount <= 0) { + error(s"Spark engine has been terminated because no incoming connection" + + s" for more than $maxTimeout ms, de-registering from engine discovery space.") + assert(currentEngine.isDefined) + currentEngine.get.stop() + } + } + } + } + } + override protected def stopServer(): Unit = { countDownLatch.countDown() } @@ -165,6 +192,10 @@ object SparkSQLEngine extends Logging { private val sparkSessionCreated = new AtomicBoolean(false) + // Kubernetes pod name max length - '-exec-' - Int.MAX_VALUE.length + // 253 - 10 - 6 + val EXECUTOR_POD_NAME_PREFIX_MAX_LENGTH = 237 + SignalRegister.registerLogger(logger) setupConf() @@ -189,7 +220,6 @@ object SparkSQLEngine extends Logging { _kyuubiConf = KyuubiConf() val rootDir = _sparkConf.getOption("spark.repl.classdir").getOrElse(getLocalDir(_sparkConf)) val outputDir = Utils.createTempDir(prefix = "repl", root = rootDir) - _sparkConf.setIfMissing("spark.sql.execution.topKSortFallbackThreshold", "10000") _sparkConf.setIfMissing("spark.sql.legacy.castComplexTypesToString.enabled", "true") _sparkConf.setIfMissing("spark.master", "local") _sparkConf.set( @@ -223,7 +253,7 @@ object SparkSQLEngine extends Logging { if (!isOnK8sClusterMode) { // set driver host to ip instead of kyuubi pod name - _sparkConf.set("spark.driver.host", InetAddress.getLocalHost.getHostAddress) + _sparkConf.setIfMissing("spark.driver.host", Utils.findLocalInetAddress.getHostAddress) } } @@ -259,6 +289,7 @@ object SparkSQLEngine extends Logging { KyuubiSparkUtil.initializeSparkSession( session, kyuubiConf.get(ENGINE_INITIALIZE_SQL) ++ kyuubiConf.get(ENGINE_SESSION_INITIALIZE_SQL)) + session.sparkContext.setLocalProperty(KYUUBI_ENGINE_URL, KyuubiSparkUtil.engineUrl) session } @@ -345,7 +376,7 @@ object SparkSQLEngine extends Logging { case i: InterruptedException if !sparkSessionCreated.get => error( s"The Engine main thread was interrupted, possibly due to `createSpark` timeout." + - s" The `kyuubi.session.engine.initialize.timeout` is ($initTimeout ms) " + + s" The `${ENGINE_INIT_TIMEOUT.key}` is ($initTimeout ms) " + s" and submitted at $submitTime.", i) case t: Throwable => error(s"Failed to instantiate SparkSession: ${t.getMessage}", t) @@ -359,7 +390,7 @@ object SparkSQLEngine extends Logging { private def startInitTimeoutChecker(startTime: Long, timeout: Long): Unit = { val mainThread = Thread.currentThread() - new Thread( + val checker = new Thread( () => { while (System.currentTimeMillis() - startTime < timeout && !sparkSessionCreated.get()) { Thread.sleep(500) @@ -368,7 +399,9 @@ object SparkSQLEngine extends Logging { mainThread.interrupt() } }, - "CreateSparkTimeoutChecker").start() + "CreateSparkTimeoutChecker") + checker.setDaemon(true) + checker.start() } private def isOnK8sClusterMode: Boolean = { @@ -390,8 +423,4 @@ object SparkSQLEngine extends Logging { s"kyuubi-${UUID.randomUUID()}" } } - - // Kubernetes pod name max length - '-exec-' - Int.MAX_VALUE.length - // 253 - 10 - 6 - val EXECUTOR_POD_NAME_PREFIX_MAX_LENGTH = 237 } diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkTBinaryFrontendService.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkTBinaryFrontendService.scala index d4eaf3454..c2563b32b 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkTBinaryFrontendService.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/SparkTBinaryFrontendService.scala @@ -19,6 +19,7 @@ package org.apache.kyuubi.engine.spark import scala.collection.JavaConverters._ +import org.apache.hadoop.conf.Configuration import org.apache.hadoop.io.Text import org.apache.hadoop.security.{Credentials, UserGroupInformation} import org.apache.hadoop.security.token.{Token, TokenIdentifier} @@ -27,11 +28,13 @@ import org.apache.spark.SparkContext import org.apache.spark.kyuubi.SparkContextHelper import org.apache.kyuubi.{KyuubiSQLException, Logging} +import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.config.KyuubiReservedKeys._ import org.apache.kyuubi.ha.client.{EngineServiceDiscovery, ServiceDiscovery} import org.apache.kyuubi.service.{Serverable, Service, TBinaryFrontendService} import org.apache.kyuubi.service.TFrontendService._ import org.apache.kyuubi.util.KyuubiHadoopUtils +import org.apache.kyuubi.util.reflect.DynConstructors class SparkTBinaryFrontendService( override val serverable: Serverable) @@ -94,13 +97,23 @@ class SparkTBinaryFrontendService( } override def attributes: Map[String, String] = { - Map(KYUUBI_ENGINE_ID -> KyuubiSparkUtil.engineId) + val extraAttributes = conf.get(KyuubiConf.ENGINE_SPARK_REGISTER_ATTRIBUTES).map { attr => + attr -> KyuubiSparkUtil.globalSparkContext.getConf.get(attr, "") + }.toMap + val attributes = extraAttributes ++ Map(KYUUBI_ENGINE_ID -> KyuubiSparkUtil.engineId) + // TODO Support Spark Web UI Enabled SSL + sc.uiWebUrl match { + case Some(url) => attributes ++ Map(KYUUBI_ENGINE_URL -> url.split("//").last) + case None => attributes + } } } object SparkTBinaryFrontendService extends Logging { val HIVE_DELEGATION_TOKEN = new Text("HIVE_DELEGATION_TOKEN") + val HIVE_CONF_CLASSNAME = "org.apache.hadoop.hive.conf.HiveConf" + @volatile private var _hiveConf: Configuration = _ private[spark] def renewDelegationToken(sc: SparkContext, delegationToken: String): Unit = { val newCreds = KyuubiHadoopUtils.decodeCredentials(delegationToken) @@ -124,7 +137,7 @@ object SparkTBinaryFrontendService extends Logging { newTokens: Map[Text, Token[_ <: TokenIdentifier]], oldCreds: Credentials, updateCreds: Credentials): Unit = { - val metastoreUris = sc.hadoopConfiguration.getTrimmed("hive.metastore.uris", "") + val metastoreUris = hiveConf(sc.hadoopConfiguration).getTrimmed("hive.metastore.uris", "") // `HiveMetaStoreClient` selects the first token whose service is "" and kind is // "HIVE_DELEGATION_TOKEN" to authenticate. @@ -195,4 +208,25 @@ object SparkTBinaryFrontendService extends Logging { 1 } } + + private[kyuubi] def hiveConf(hadoopConf: Configuration): Configuration = { + if (_hiveConf == null) { + synchronized { + if (_hiveConf == null) { + _hiveConf = + try { + DynConstructors.builder() + .impl(HIVE_CONF_CLASSNAME, classOf[Configuration], classOf[Class[_]]) + .build[Configuration]() + .newInstance(hadoopConf, Class.forName(HIVE_CONF_CLASSNAME)) + } catch { + case e: Throwable => + warn("Fail to create Hive Configuration", e) + hadoopConf + } + } + } + } + _hiveConf + } } diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/ExecutePython.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/ExecutePython.scala index e48ff6e5b..badd83530 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/ExecutePython.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/ExecutePython.scala @@ -40,7 +40,7 @@ import org.apache.kyuubi.{KyuubiSQLException, Logging, Utils} import org.apache.kyuubi.config.KyuubiConf.{ENGINE_SPARK_PYTHON_ENV_ARCHIVE, ENGINE_SPARK_PYTHON_ENV_ARCHIVE_EXEC_PATH, ENGINE_SPARK_PYTHON_HOME_ARCHIVE} import org.apache.kyuubi.config.KyuubiReservedKeys.{KYUUBI_SESSION_USER_KEY, KYUUBI_STATEMENT_ID_KEY} import org.apache.kyuubi.engine.spark.KyuubiSparkUtil._ -import org.apache.kyuubi.operation.{ArrayFetchIterator, OperationState} +import org.apache.kyuubi.operation.{ArrayFetchIterator, OperationHandle, OperationState} import org.apache.kyuubi.operation.log.OperationLog import org.apache.kyuubi.session.Session @@ -49,7 +49,8 @@ class ExecutePython( override val statement: String, override val shouldRunAsync: Boolean, queryTimeout: Long, - worker: SessionPythonWorker) extends SparkOperation(session) { + worker: SessionPythonWorker, + override protected val handle: OperationHandle) extends SparkOperation(session) { private val operationLog: OperationLog = OperationLog.createOperationLog(session, getHandle) override def getOperationLog: Option[OperationLog] = Option(operationLog) @@ -77,30 +78,31 @@ class ExecutePython( OperationLog.removeCurrentOperationLog() } - private def executePython(): Unit = withLocalProperties { + private def executePython(): Unit = try { - setState(OperationState.RUNNING) - info(diagnostics) - addOperationListener() - val response = worker.runCode(statement) - val status = response.map(_.content.status).getOrElse("UNKNOWN_STATUS") - if (PythonResponse.OK_STATUS.equalsIgnoreCase(status)) { - val output = response.map(_.content.getOutput()).getOrElse("") - val ename = response.map(_.content.getEname()).getOrElse("") - val evalue = response.map(_.content.getEvalue()).getOrElse("") - val traceback = response.map(_.content.getTraceback()).getOrElse(Array.empty) - iter = - new ArrayFetchIterator[Row](Array(Row(output, status, ename, evalue, Row(traceback: _*)))) - setState(OperationState.FINISHED) - } else { - throw KyuubiSQLException(s"Interpret error:\n$statement\n $response") + withLocalProperties { + setState(OperationState.RUNNING) + info(diagnostics) + addOperationListener() + val response = worker.runCode(statement) + val status = response.map(_.content.status).getOrElse("UNKNOWN_STATUS") + if (PythonResponse.OK_STATUS.equalsIgnoreCase(status)) { + val output = response.map(_.content.getOutput()).getOrElse("") + val ename = response.map(_.content.getEname()).getOrElse("") + val evalue = response.map(_.content.getEvalue()).getOrElse("") + val traceback = response.map(_.content.getTraceback()).getOrElse(Seq.empty) + iter = + new ArrayFetchIterator[Row](Array(Row(output, status, ename, evalue, traceback))) + setState(OperationState.FINISHED) + } else { + throw KyuubiSQLException(s"Interpret error:\n$statement\n $response") + } } } catch { onError(cancel = true) } finally { shutdownTimeoutMonitor() } - } override protected def runInternal(): Unit = { addTimeoutMonitor(queryTimeout) @@ -180,12 +182,7 @@ case class SessionPythonWorker( new BufferedReader(new InputStreamReader(workerProcess.getInputStream), 1) private val lock = new ReentrantLock() - private def withLockRequired[T](block: => T): T = { - try { - lock.lock() - block - } finally lock.unlock() - } + private def withLockRequired[T](block: => T): T = Utils.withLockRequired(lock)(block) /** * Run the python code and return the response. This method maybe invoked internally, @@ -210,7 +207,7 @@ case class SessionPythonWorker( stdin.flush() val pythonResponse = Option(stdout.readLine()).map(ExecutePython.fromJson[PythonResponse](_)) // throw exception if internal python code fail - if (internal && pythonResponse.map(_.content.status) != Some(PythonResponse.OK_STATUS)) { + if (internal && !pythonResponse.map(_.content.status).contains(PythonResponse.OK_STATUS)) { throw KyuubiSQLException(s"Internal python code $code failure: $pythonResponse") } pythonResponse @@ -328,7 +325,7 @@ object ExecutePython extends Logging { } // for test - def defaultSparkHome(): String = { + def defaultSparkHome: String = { val homeDirFilter: FilenameFilter = (dir: File, name: String) => dir.isDirectory && name.contains("spark-") && !name.contains("-engine") // get from kyuubi-server/../externals/kyuubi-download/target @@ -418,7 +415,7 @@ case class PythonResponseContent( data: Map[String, String], ename: String, evalue: String, - traceback: Array[String], + traceback: Seq[String], status: String) { def getOutput(): String = { Option(data) @@ -431,7 +428,7 @@ case class PythonResponseContent( def getEvalue(): String = { Option(evalue).getOrElse("") } - def getTraceback(): Array[String] = { - Option(traceback).getOrElse(Array.empty) + def getTraceback(): Seq[String] = { + Option(traceback).getOrElse(Seq.empty) } } diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/ExecuteScala.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/ExecuteScala.scala index ff686cca0..691c4fb32 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/ExecuteScala.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/ExecuteScala.scala @@ -31,7 +31,7 @@ import org.apache.spark.sql.types.StructType import org.apache.kyuubi.KyuubiSQLException import org.apache.kyuubi.engine.spark.KyuubiSparkUtil._ import org.apache.kyuubi.engine.spark.repl.KyuubiSparkILoop -import org.apache.kyuubi.operation.{ArrayFetchIterator, OperationState} +import org.apache.kyuubi.operation.{ArrayFetchIterator, OperationHandle, OperationState} import org.apache.kyuubi.operation.log.OperationLog import org.apache.kyuubi.session.Session @@ -51,7 +51,8 @@ class ExecuteScala( repl: KyuubiSparkILoop, override val statement: String, override val shouldRunAsync: Boolean, - queryTimeout: Long) + queryTimeout: Long, + override protected val handle: OperationHandle) extends SparkOperation(session) { private val operationLog: OperationLog = OperationLog.createOperationLog(session, getHandle) @@ -76,59 +77,60 @@ class ExecuteScala( OperationLog.removeCurrentOperationLog() } - private def executeScala(): Unit = withLocalProperties { + private def executeScala(): Unit = try { - setState(OperationState.RUNNING) - info(diagnostics) - Thread.currentThread().setContextClassLoader(spark.sharedState.jarClassLoader) - addOperationListener() - val legacyOutput = repl.getOutput - if (legacyOutput.nonEmpty) { - warn(s"Clearing legacy output from last interpreting:\n $legacyOutput") - } - val replUrls = repl.classLoader.getParent.asInstanceOf[URLClassLoader].getURLs - spark.sharedState.jarClassLoader.getURLs.filterNot(replUrls.contains).foreach { jar => - try { - if ("file".equals(jar.toURI.getScheme)) { - repl.addUrlsToClassPath(jar) - } else { - spark.sparkContext.addFile(jar.toString) - val localJarFile = new File(SparkFiles.get(new Path(jar.toURI.getPath).getName)) - val localJarUrl = localJarFile.toURI.toURL - if (!replUrls.contains(localJarUrl)) { - repl.addUrlsToClassPath(localJarUrl) + withLocalProperties { + setState(OperationState.RUNNING) + info(diagnostics) + Thread.currentThread().setContextClassLoader(spark.sharedState.jarClassLoader) + addOperationListener() + val legacyOutput = repl.getOutput + if (legacyOutput.nonEmpty) { + warn(s"Clearing legacy output from last interpreting:\n $legacyOutput") + } + val replUrls = repl.classLoader.getParent.asInstanceOf[URLClassLoader].getURLs + spark.sharedState.jarClassLoader.getURLs.filterNot(replUrls.contains).foreach { jar => + try { + if ("file".equals(jar.toURI.getScheme)) { + repl.addUrlsToClassPath(jar) + } else { + spark.sparkContext.addFile(jar.toString) + val localJarFile = new File(SparkFiles.get(new Path(jar.toURI.getPath).getName)) + val localJarUrl = localJarFile.toURI.toURL + if (!replUrls.contains(localJarUrl)) { + repl.addUrlsToClassPath(localJarUrl) + } } + } catch { + case e: Throwable => error(s"Error adding $jar to repl class path", e) } - } catch { - case e: Throwable => error(s"Error adding $jar to repl class path", e) } - } - repl.interpretWithRedirectOutError(statement) match { - case Success => - iter = { - result = repl.getResult(statementId) - if (result != null) { - new ArrayFetchIterator[Row](result.collect()) - } else { - val output = repl.getOutput - debug("scala repl output:\n" + output) - new ArrayFetchIterator[Row](Array(Row(output))) + repl.interpretWithRedirectOutError(statement) match { + case Success => + iter = { + result = repl.getResult(statementId) + if (result != null) { + new ArrayFetchIterator[Row](result.collect()) + } else { + val output = repl.getOutput + debug("scala repl output:\n" + output) + new ArrayFetchIterator[Row](Array(Row(output))) + } } - } - case Error => - throw KyuubiSQLException(s"Interpret error:\n$statement\n ${repl.getOutput}") - case Incomplete => - throw KyuubiSQLException(s"Incomplete code:\n$statement") + case Error => + throw KyuubiSQLException(s"Interpret error:\n$statement\n ${repl.getOutput}") + case Incomplete => + throw KyuubiSQLException(s"Incomplete code:\n$statement") + } + setState(OperationState.FINISHED) } - setState(OperationState.FINISHED) } catch { onError(cancel = true) } finally { repl.clearResult(statementId) shutdownTimeoutMonitor() } - } override protected def runInternal(): Unit = { addTimeoutMonitor(queryTimeout) diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/ExecuteStatement.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/ExecuteStatement.scala index 2cdc2b500..17d8a7412 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/ExecuteStatement.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/ExecuteStatement.scala @@ -21,14 +21,15 @@ import java.util.concurrent.RejectedExecutionException import scala.collection.JavaConverters._ -import org.apache.spark.sql.{DataFrame, Row} -import org.apache.spark.sql.kyuubi.SparkDatasetHelper +import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.kyuubi.SparkDatasetHelper._ import org.apache.spark.sql.types._ import org.apache.kyuubi.{KyuubiSQLException, Logging} import org.apache.kyuubi.config.KyuubiConf.OPERATION_RESULT_MAX_ROWS import org.apache.kyuubi.engine.spark.KyuubiSparkUtil._ -import org.apache.kyuubi.operation.{ArrayFetchIterator, IterableFetchIterator, OperationState} +import org.apache.kyuubi.engine.spark.session.SparkSessionImpl +import org.apache.kyuubi.operation.{ArrayFetchIterator, FetchIterator, IterableFetchIterator, OperationHandle, OperationState} import org.apache.kyuubi.operation.log.OperationLog import org.apache.kyuubi.session.Session @@ -37,7 +38,8 @@ class ExecuteStatement( override val statement: String, override val shouldRunAsync: Boolean, queryTimeout: Long, - incrementalCollect: Boolean) + incrementalCollect: Boolean, + override protected val handle: OperationHandle) extends SparkOperation(session) with Logging { private val operationLog: OperationLog = OperationLog.createOperationLog(session, getHandle) @@ -62,59 +64,35 @@ class ExecuteStatement( OperationLog.removeCurrentOperationLog() } - private def executeStatement(): Unit = withLocalProperties { + protected def incrementalCollectResult(resultDF: DataFrame): Iterator[Any] = { + resultDF.toLocalIterator().asScala + } + + protected def fullCollectResult(resultDF: DataFrame): Array[_] = { + resultDF.collect() + } + + protected def takeResult(resultDF: DataFrame, maxRows: Int): Array[_] = { + resultDF.take(maxRows) + } + + protected def executeStatement(): Unit = try { - setState(OperationState.RUNNING) - info(diagnostics) - Thread.currentThread().setContextClassLoader(spark.sharedState.jarClassLoader) - addOperationListener() - result = spark.sql(statement) - - iter = - if (incrementalCollect) { - info("Execute in incremental collect mode") - if (arrowEnabled) { - new IterableFetchIterator[Array[Byte]](new Iterable[Array[Byte]] { - override def iterator: Iterator[Array[Byte]] = SparkDatasetHelper.toArrowBatchRdd( - convertComplexType(result)).toLocalIterator - }) - } else { - new IterableFetchIterator[Row](new Iterable[Row] { - override def iterator: Iterator[Row] = result.toLocalIterator().asScala - }) - } - } else { - val resultMaxRows = spark.conf.getOption(OPERATION_RESULT_MAX_ROWS.key).map(_.toInt) - .getOrElse(session.sessionManager.getConf.get(OPERATION_RESULT_MAX_ROWS)) - if (resultMaxRows <= 0) { - info("Execute in full collect mode") - if (arrowEnabled) { - new ArrayFetchIterator( - SparkDatasetHelper.toArrowBatchRdd( - convertComplexType(result)).collect()) - } else { - new ArrayFetchIterator(result.collect()) - } - } else { - info(s"Execute with max result rows[$resultMaxRows]") - if (arrowEnabled) { - // this will introduce shuffle and hurt performance - new ArrayFetchIterator( - SparkDatasetHelper.toArrowBatchRdd( - convertComplexType(result.limit(resultMaxRows))).collect()) - } else { - new ArrayFetchIterator(result.take(resultMaxRows)) - } - } - } - setCompiledStateIfNeeded() - setState(OperationState.FINISHED) + withLocalProperties { + setState(OperationState.RUNNING) + info(diagnostics) + Thread.currentThread().setContextClassLoader(spark.sharedState.jarClassLoader) + addOperationListener() + result = spark.sql(statement) + iter = collectAsIterator(result) + setCompiledStateIfNeeded() + setState(OperationState.FINISHED) + } } catch { onError(cancel = true) } finally { shutdownTimeoutMonitor() } - } override protected def runInternal(): Unit = { addTimeoutMonitor(queryTimeout) @@ -164,17 +142,82 @@ class ExecuteStatement( } } - // TODO:(fchen) make this configurable - val kyuubiBeelineConvertToString = true - - def convertComplexType(df: DataFrame): DataFrame = { - if (kyuubiBeelineConvertToString) { - SparkDatasetHelper.convertTopLevelComplexTypeToHiveString(df) + override def getResultSetMetadataHints(): Seq[String] = + Seq( + s"__kyuubi_operation_result_format__=$resultFormat", + s"__kyuubi_operation_result_arrow_timestampAsString__=$timestampAsString") + + private def collectAsIterator(resultDF: DataFrame): FetchIterator[_] = { + val resultMaxRows = spark.conf.getOption(OPERATION_RESULT_MAX_ROWS.key).map(_.toInt) + .getOrElse(session.sessionManager.getConf.get(OPERATION_RESULT_MAX_ROWS)) + if (incrementalCollect) { + if (resultMaxRows > 0) { + warn(s"Ignore ${OPERATION_RESULT_MAX_ROWS.key} on incremental collect mode.") + } + info("Execute in incremental collect mode") + new IterableFetchIterator[Any](new Iterable[Any] { + override def iterator: Iterator[Any] = incrementalCollectResult(resultDF) + }) } else { - df + val internalArray = if (resultMaxRows <= 0) { + info("Execute in full collect mode") + fullCollectResult(resultDF) + } else { + info(s"Execute with max result rows[$resultMaxRows]") + takeResult(resultDF, resultMaxRows) + } + new ArrayFetchIterator(internalArray) } } +} - override def getResultSetMetadataHints(): Seq[String] = - Seq(s"__kyuubi_operation_result_format__=$resultFormat") +class ArrowBasedExecuteStatement( + session: Session, + override val statement: String, + override val shouldRunAsync: Boolean, + queryTimeout: Long, + incrementalCollect: Boolean, + override protected val handle: OperationHandle) + extends ExecuteStatement( + session, + statement, + shouldRunAsync, + queryTimeout, + incrementalCollect, + handle) { + + checkUseLargeVarType() + + override protected def incrementalCollectResult(resultDF: DataFrame): Iterator[Any] = { + toArrowBatchLocalIterator(convertComplexType(resultDF)) + } + + override protected def fullCollectResult(resultDF: DataFrame): Array[_] = { + executeCollect(convertComplexType(resultDF)) + } + + override protected def takeResult(resultDF: DataFrame, maxRows: Int): Array[_] = { + executeCollect(convertComplexType(resultDF.limit(maxRows))) + } + + override protected def isArrowBasedOperation: Boolean = true + + override val resultFormat = "arrow" + + private def convertComplexType(df: DataFrame): DataFrame = { + convertTopLevelComplexTypeToHiveString(df, timestampAsString) + } + + def checkUseLargeVarType(): Unit = { + // TODO: largeVarType support, see SPARK-39979. + val useLargeVarType = session.asInstanceOf[SparkSessionImpl].spark + .conf + .get("spark.sql.execution.arrow.useLargeVarType", "false") + .toBoolean + if (useLargeVarType) { + throw new KyuubiSQLException( + "`spark.sql.execution.arrow.useLargeVarType = true` not support now.", + null) + } + } } diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetCatalogs.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetCatalogs.scala index 6d818e53e..c8e587300 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetCatalogs.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetCatalogs.scala @@ -19,7 +19,7 @@ package org.apache.kyuubi.engine.spark.operation import org.apache.spark.sql.types.StructType -import org.apache.kyuubi.engine.spark.shim.SparkCatalogShim +import org.apache.kyuubi.engine.spark.util.SparkCatalogUtils import org.apache.kyuubi.operation.IterableFetchIterator import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant.TABLE_CAT import org.apache.kyuubi.session.Session @@ -33,7 +33,7 @@ class GetCatalogs(session: Session) extends SparkOperation(session) { override protected def runInternal(): Unit = { try { - iter = new IterableFetchIterator(SparkCatalogShim().getCatalogs(spark).toList) + iter = new IterableFetchIterator(SparkCatalogUtils.getCatalogs(spark)) } catch onError() } } diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetColumns.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetColumns.scala index e78516981..3a0ab7d5b 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetColumns.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetColumns.scala @@ -19,7 +19,7 @@ package org.apache.kyuubi.engine.spark.operation import org.apache.spark.sql.types._ -import org.apache.kyuubi.engine.spark.shim.SparkCatalogShim +import org.apache.kyuubi.engine.spark.util.SparkCatalogUtils import org.apache.kyuubi.operation.IterableFetchIterator import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant._ import org.apache.kyuubi.session.Session @@ -115,8 +115,8 @@ class GetColumns( val schemaPattern = toJavaRegex(schemaName) val tablePattern = toJavaRegex(tableName) val columnPattern = toJavaRegex(columnName) - iter = new IterableFetchIterator(SparkCatalogShim() - .getColumns(spark, catalogName, schemaPattern, tablePattern, columnPattern).toList) + iter = new IterableFetchIterator(SparkCatalogUtils + .getColumns(spark, catalogName, schemaPattern, tablePattern, columnPattern)) } catch { onError() } diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetCurrentCatalog.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetCurrentCatalog.scala index 66d707ec0..1d85d3d5a 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetCurrentCatalog.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetCurrentCatalog.scala @@ -17,15 +17,20 @@ package org.apache.kyuubi.engine.spark.operation +import org.apache.spark.sql.Row import org.apache.spark.sql.types.StructType -import org.apache.kyuubi.engine.spark.shim.SparkCatalogShim import org.apache.kyuubi.operation.IterableFetchIterator +import org.apache.kyuubi.operation.log.OperationLog import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant.TABLE_CAT import org.apache.kyuubi.session.Session class GetCurrentCatalog(session: Session) extends SparkOperation(session) { + private val operationLog: OperationLog = OperationLog.createOperationLog(session, getHandle) + + override def getOperationLog: Option[OperationLog] = Option(operationLog) + override protected def resultSchema: StructType = { new StructType() .add(TABLE_CAT, "string", nullable = true, "Catalog name.") @@ -33,7 +38,8 @@ class GetCurrentCatalog(session: Session) extends SparkOperation(session) { override protected def runInternal(): Unit = { try { - iter = new IterableFetchIterator(Seq(SparkCatalogShim().getCurrentCatalog(spark))) + val currentCatalogName = spark.sessionState.catalogManager.currentCatalog.name() + iter = new IterableFetchIterator(Seq(Row(currentCatalogName))) } catch onError() } } diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetCurrentDatabase.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetCurrentDatabase.scala index bcf3ad2a5..2478fb6a4 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetCurrentDatabase.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetCurrentDatabase.scala @@ -17,15 +17,21 @@ package org.apache.kyuubi.engine.spark.operation +import org.apache.spark.sql.Row import org.apache.spark.sql.types.StructType -import org.apache.kyuubi.engine.spark.shim.SparkCatalogShim +import org.apache.kyuubi.engine.spark.util.SparkCatalogUtils.quoteIfNeeded import org.apache.kyuubi.operation.IterableFetchIterator +import org.apache.kyuubi.operation.log.OperationLog import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant.TABLE_SCHEM import org.apache.kyuubi.session.Session class GetCurrentDatabase(session: Session) extends SparkOperation(session) { + private val operationLog: OperationLog = OperationLog.createOperationLog(session, getHandle) + + override def getOperationLog: Option[OperationLog] = Option(operationLog) + override protected def resultSchema: StructType = { new StructType() .add(TABLE_SCHEM, "string", nullable = true, "Schema name.") @@ -33,7 +39,9 @@ class GetCurrentDatabase(session: Session) extends SparkOperation(session) { override protected def runInternal(): Unit = { try { - iter = new IterableFetchIterator(Seq(SparkCatalogShim().getCurrentDatabase(spark))) + val currentDatabaseName = + spark.sessionState.catalogManager.currentNamespace.map(quoteIfNeeded).mkString(".") + iter = new IterableFetchIterator(Seq(Row(currentDatabaseName))) } catch onError() } } diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetSchemas.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetSchemas.scala index 3937f528d..46dc7634a 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetSchemas.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetSchemas.scala @@ -19,7 +19,7 @@ package org.apache.kyuubi.engine.spark.operation import org.apache.spark.sql.types.StructType -import org.apache.kyuubi.engine.spark.shim.SparkCatalogShim +import org.apache.kyuubi.engine.spark.util.SparkCatalogUtils import org.apache.kyuubi.operation.IterableFetchIterator import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant._ import org.apache.kyuubi.session.Session @@ -40,7 +40,7 @@ class GetSchemas(session: Session, catalogName: String, schema: String) override protected def runInternal(): Unit = { try { val schemaPattern = toJavaRegex(schema) - val rows = SparkCatalogShim().getSchemas(spark, catalogName, schemaPattern) + val rows = SparkCatalogUtils.getSchemas(spark, catalogName, schemaPattern) iter = new IterableFetchIterator(rows) } catch onError() } diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetTableTypes.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetTableTypes.scala index 1d2cae381..1029175b2 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetTableTypes.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetTableTypes.scala @@ -20,7 +20,7 @@ package org.apache.kyuubi.engine.spark.operation import org.apache.spark.sql.Row import org.apache.spark.sql.types.StructType -import org.apache.kyuubi.engine.spark.shim.SparkCatalogShim +import org.apache.kyuubi.engine.spark.util.SparkCatalogUtils import org.apache.kyuubi.operation.IterableFetchIterator import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant._ import org.apache.kyuubi.session.Session @@ -33,6 +33,6 @@ class GetTableTypes(session: Session) } override protected def runInternal(): Unit = { - iter = new IterableFetchIterator(SparkCatalogShim.sparkTableTypes.map(Row(_)).toList) + iter = new IterableFetchIterator(SparkCatalogUtils.sparkTableTypes.map(Row(_)).toList) } } diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetTables.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetTables.scala index 4093c61c1..980e4fdb1 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetTables.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/GetTables.scala @@ -19,7 +19,8 @@ package org.apache.kyuubi.engine.spark.operation import org.apache.spark.sql.types.StructType -import org.apache.kyuubi.engine.spark.shim.SparkCatalogShim +import org.apache.kyuubi.config.KyuubiConf.OPERATION_GET_TABLES_IGNORE_TABLE_PROPERTIES +import org.apache.kyuubi.engine.spark.util.SparkCatalogUtils import org.apache.kyuubi.operation.IterableFetchIterator import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant._ import org.apache.kyuubi.session.Session @@ -32,6 +33,12 @@ class GetTables( tableTypes: Set[String]) extends SparkOperation(session) { + protected val ignoreTableProperties = + spark.conf.getOption(OPERATION_GET_TABLES_IGNORE_TABLE_PROPERTIES.key) match { + case Some(s) => s.toBoolean + case _ => session.sessionManager.getConf.get(OPERATION_GET_TABLES_IGNORE_TABLE_PROPERTIES) + } + override def statement: String = { super.statement + s" [catalog: $catalog," + @@ -66,14 +73,19 @@ class GetTables( try { val schemaPattern = toJavaRegex(schema) val tablePattern = toJavaRegex(tableName) - val sparkShim = SparkCatalogShim() val catalogTablesAndViews = - sparkShim.getCatalogTablesOrViews(spark, catalog, schemaPattern, tablePattern, tableTypes) + SparkCatalogUtils.getCatalogTablesOrViews( + spark, + catalog, + schemaPattern, + tablePattern, + tableTypes, + ignoreTableProperties) val allTableAndViews = if (tableTypes.exists("VIEW".equalsIgnoreCase)) { catalogTablesAndViews ++ - sparkShim.getTempViews(spark, catalog, schemaPattern, tablePattern) + SparkCatalogUtils.getTempViews(spark, catalog, schemaPattern, tablePattern) } else { catalogTablesAndViews } diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/PlanOnlyStatement.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/PlanOnlyStatement.scala index b7e5451ec..4f8808313 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/PlanOnlyStatement.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/PlanOnlyStatement.scala @@ -17,14 +17,17 @@ package org.apache.kyuubi.engine.spark.operation -import org.apache.spark.sql.Row +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.module.scala.DefaultScalaModule +import org.apache.spark.kyuubi.SparkUtilsHelper +import org.apache.spark.sql.{Row, SparkSession} import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.StructType import org.apache.kyuubi.KyuubiSQLException -import org.apache.kyuubi.config.KyuubiConf.{OPERATION_PLAN_ONLY_EXCLUDES, OPERATION_PLAN_ONLY_OUT_STYLE} -import org.apache.kyuubi.operation.{AnalyzeMode, ArrayFetchIterator, ExecutionMode, IterableFetchIterator, JsonStyle, OptimizeMode, OptimizeWithStatsMode, ParseMode, PhysicalMode, PlainStyle, PlanOnlyMode, PlanOnlyStyle, UnknownMode, UnknownStyle} +import org.apache.kyuubi.config.KyuubiConf.{LINEAGE_PARSER_PLUGIN_PROVIDER, OPERATION_PLAN_ONLY_EXCLUDES, OPERATION_PLAN_ONLY_OUT_STYLE} +import org.apache.kyuubi.operation.{AnalyzeMode, ArrayFetchIterator, ExecutionMode, IterableFetchIterator, JsonStyle, LineageMode, OperationHandle, OptimizeMode, OptimizeWithStatsMode, ParseMode, PhysicalMode, PlainStyle, PlanOnlyMode, PlanOnlyStyle, UnknownMode, UnknownStyle} import org.apache.kyuubi.operation.PlanOnlyMode.{notSupportedModeError, unknownModeError} import org.apache.kyuubi.operation.PlanOnlyStyle.{notSupportedStyleError, unknownStyleError} import org.apache.kyuubi.operation.log.OperationLog @@ -36,12 +39,13 @@ import org.apache.kyuubi.session.Session class PlanOnlyStatement( session: Session, override val statement: String, - mode: PlanOnlyMode) + mode: PlanOnlyMode, + override protected val handle: OperationHandle) extends SparkOperation(session) { private val operationLog: OperationLog = OperationLog.createOperationLog(session, getHandle) - private val planExcludes: Seq[String] = { - spark.conf.getOption(OPERATION_PLAN_ONLY_EXCLUDES.key).map(_.split(",").map(_.trim).toSeq) + private val planExcludes: Set[String] = { + spark.conf.getOption(OPERATION_PLAN_ONLY_EXCLUDES.key).map(_.split(",").map(_.trim).toSet) .getOrElse(session.sessionManager.getConf.get(OPERATION_PLAN_ONLY_EXCLUDES)) } @@ -65,28 +69,29 @@ class PlanOnlyStatement( super.beforeRun() } - override protected def runInternal(): Unit = withLocalProperties { + override protected def runInternal(): Unit = try { - SQLConf.withExistingConf(spark.sessionState.conf) { - val parsed = spark.sessionState.sqlParser.parsePlan(statement) - - parsed match { - case cmd if planExcludes.contains(cmd.getClass.getSimpleName) => - result = spark.sql(statement) - iter = new ArrayFetchIterator(result.collect()) - - case plan => style match { - case PlainStyle => explainWithPlainStyle(plan) - case JsonStyle => explainWithJsonStyle(plan) - case UnknownStyle => unknownStyleError(style) - case other => throw notSupportedStyleError(other, "Spark SQL") - } + withLocalProperties { + SQLConf.withExistingConf(spark.sessionState.conf) { + val parsed = spark.sessionState.sqlParser.parsePlan(statement) + + parsed match { + case cmd if planExcludes.contains(cmd.getClass.getSimpleName) => + result = spark.sql(statement) + iter = new ArrayFetchIterator(result.collect()) + + case plan => style match { + case PlainStyle => explainWithPlainStyle(plan) + case JsonStyle => explainWithJsonStyle(plan) + case UnknownStyle => unknownStyleError(style) + case other => throw notSupportedStyleError(other, "Spark SQL") + } + } } } } catch { onError() } - } private def explainWithPlainStyle(plan: LogicalPlan): Unit = { mode match { @@ -117,6 +122,9 @@ class PlanOnlyStatement( case ExecutionMode => val executed = spark.sql(statement).queryExecution.executedPlan iter = new IterableFetchIterator(Seq(Row(executed.toString()))) + case LineageMode => + val result = parseLineage(spark, plan) + iter = new IterableFetchIterator(Seq(Row(result))) case UnknownMode => throw unknownModeError(mode) case _ => throw notSupportedModeError(mode, "Spark SQL") } @@ -141,10 +149,39 @@ class PlanOnlyStatement( case ExecutionMode => val executed = spark.sql(statement).queryExecution.executedPlan iter = new IterableFetchIterator(Seq(Row(executed.toJSON))) + case LineageMode => + val result = parseLineage(spark, plan) + iter = new IterableFetchIterator(Seq(Row(result))) case UnknownMode => throw unknownModeError(mode) case _ => throw KyuubiSQLException(s"The operation mode $mode" + " doesn't support in Spark SQL engine.") } } + + private def parseLineage(spark: SparkSession, plan: LogicalPlan): String = { + val analyzed = spark.sessionState.analyzer.execute(plan) + spark.sessionState.analyzer.checkAnalysis(analyzed) + val optimized = spark.sessionState.optimizer.execute(analyzed) + val parserProviderClass = session.sessionManager.getConf.get(LINEAGE_PARSER_PLUGIN_PROVIDER) + + try { + if (!SparkUtilsHelper.classesArePresent( + parserProviderClass)) { + throw new Exception(s"'$parserProviderClass' not found," + + " need to install kyuubi-spark-lineage plugin before using the 'lineage' mode") + } + + val lineage = Class.forName(parserProviderClass) + .getMethod("parse", classOf[SparkSession], classOf[LogicalPlan]) + .invoke(null, spark, optimized) + + val mapper = new ObjectMapper().registerModule(DefaultScalaModule) + mapper.writeValueAsString(lineage) + } catch { + case e: Throwable => + throw KyuubiSQLException(s"Extract columns lineage failed: ${e.getMessage}", e) + } + } + } diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SetCurrentCatalog.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SetCurrentCatalog.scala index 4e8c0aa69..88105b086 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SetCurrentCatalog.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SetCurrentCatalog.scala @@ -19,18 +19,23 @@ package org.apache.kyuubi.engine.spark.operation import org.apache.spark.sql.types.StructType -import org.apache.kyuubi.engine.spark.shim.SparkCatalogShim +import org.apache.kyuubi.engine.spark.util.SparkCatalogUtils +import org.apache.kyuubi.operation.log.OperationLog import org.apache.kyuubi.session.Session class SetCurrentCatalog(session: Session, catalog: String) extends SparkOperation(session) { + private val operationLog: OperationLog = OperationLog.createOperationLog(session, getHandle) + + override def getOperationLog: Option[OperationLog] = Option(operationLog) + override protected def resultSchema: StructType = { new StructType() } override protected def runInternal(): Unit = { try { - SparkCatalogShim().setCurrentCatalog(spark, catalog) + SparkCatalogUtils.setCurrentCatalog(spark, catalog) setHasResultSet(false) } catch onError() } diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SetCurrentDatabase.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SetCurrentDatabase.scala index 0a21bc839..d227f5fd2 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SetCurrentDatabase.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SetCurrentDatabase.scala @@ -19,19 +19,23 @@ package org.apache.kyuubi.engine.spark.operation import org.apache.spark.sql.types.StructType -import org.apache.kyuubi.engine.spark.shim.SparkCatalogShim +import org.apache.kyuubi.operation.log.OperationLog import org.apache.kyuubi.session.Session class SetCurrentDatabase(session: Session, database: String) extends SparkOperation(session) { + private val operationLog: OperationLog = OperationLog.createOperationLog(session, getHandle) + + override def getOperationLog: Option[OperationLog] = Option(operationLog) + override protected def resultSchema: StructType = { new StructType() } override protected def runInternal(): Unit = { try { - SparkCatalogShim().setCurrentDatabase(spark, database) + spark.sessionState.catalogManager.setCurrentNamespace(Array(database)) setHasResultSet(false) } catch onError() } diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SparkOperation.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SparkOperation.scala index 842ff944f..1de360f07 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SparkOperation.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SparkOperation.scala @@ -20,10 +20,11 @@ package org.apache.kyuubi.engine.spark.operation import java.io.IOException import java.time.ZoneId -import org.apache.hive.service.rpc.thrift.{TGetResultSetMetadataResp, TProgressUpdateResp, TRowSet} +import org.apache.hive.service.rpc.thrift.{TFetchResultsResp, TGetResultSetMetadataResp, TProgressUpdateResp, TRowSet} import org.apache.spark.kyuubi.{SparkProgressMonitor, SQLOperationListener} import org.apache.spark.kyuubi.SparkUtilsHelper.redact import org.apache.spark.sql.{DataFrame, Row, SparkSession} +import org.apache.spark.sql.execution.SQLExecution import org.apache.spark.sql.types.StructType import org.apache.kyuubi.{KyuubiSQLException, Utils} @@ -100,13 +101,13 @@ abstract class SparkOperation(session: Session) super.getStatus } - override def cleanup(targetState: OperationState): Unit = state.synchronized { + override def cleanup(targetState: OperationState): Unit = withLockRequired { operationListener.foreach(_.cleanup()) if (!isTerminalState(state)) { setState(targetState) Option(getBackgroundHandle).foreach(_.cancel(true)) - if (!spark.sparkContext.isStopped) spark.sparkContext.cancelJobGroup(statementId) } + if (!spark.sparkContext.isStopped) spark.sparkContext.cancelJobGroup(statementId) } protected val forceCancel = @@ -135,27 +136,35 @@ abstract class SparkOperation(session: Session) spark.sparkContext.setLocalProperty protected def withLocalProperties[T](f: => T): T = { - try { - spark.sparkContext.setJobGroup(statementId, redactedStatement, forceCancel) - spark.sparkContext.setLocalProperty(KYUUBI_SESSION_USER_KEY, session.user) - spark.sparkContext.setLocalProperty(KYUUBI_STATEMENT_ID_KEY, statementId) - schedulerPool match { - case Some(pool) => - spark.sparkContext.setLocalProperty(SPARK_SCHEDULER_POOL_KEY, pool) - case None => - } - if (isSessionUserSignEnabled) { - setSessionUserSign() - } + SQLExecution.withSQLConfPropagated(spark) { + val originalSession = SparkSession.getActiveSession + try { + SparkSession.setActiveSession(spark) + spark.sparkContext.setJobGroup(statementId, redactedStatement, forceCancel) + spark.sparkContext.setLocalProperty(KYUUBI_SESSION_USER_KEY, session.user) + spark.sparkContext.setLocalProperty(KYUUBI_STATEMENT_ID_KEY, statementId) + schedulerPool match { + case Some(pool) => + spark.sparkContext.setLocalProperty(SPARK_SCHEDULER_POOL_KEY, pool) + case None => + } + if (isSessionUserSignEnabled) { + setSessionUserSign() + } - f - } finally { - spark.sparkContext.setLocalProperty(SPARK_SCHEDULER_POOL_KEY, null) - spark.sparkContext.setLocalProperty(KYUUBI_SESSION_USER_KEY, null) - spark.sparkContext.setLocalProperty(KYUUBI_STATEMENT_ID_KEY, null) - spark.sparkContext.clearJobGroup() - if (isSessionUserSignEnabled) { - clearSessionUserSign() + f + } finally { + spark.sparkContext.setLocalProperty(SPARK_SCHEDULER_POOL_KEY, null) + spark.sparkContext.setLocalProperty(KYUUBI_SESSION_USER_KEY, null) + spark.sparkContext.setLocalProperty(KYUUBI_STATEMENT_ID_KEY, null) + spark.sparkContext.clearJobGroup() + if (isSessionUserSignEnabled) { + clearSessionUserSign() + } + originalSession match { + case Some(session) => SparkSession.setActiveSession(session) + case None => SparkSession.clearActiveSession() + } } } } @@ -165,15 +174,16 @@ abstract class SparkOperation(session: Session) // could be thrown. case e: Throwable => if (cancel && !spark.sparkContext.isStopped) spark.sparkContext.cancelJobGroup(statementId) - state.synchronized { + withLockRequired { val errMsg = Utils.stringifyException(e) if (state == OperationState.TIMEOUT) { val ke = KyuubiSQLException(s"Timeout operating $opType: $errMsg") setOperationException(ke) throw ke } else if (isTerminalState(state)) { - setOperationException(KyuubiSQLException(errMsg)) - warn(s"Ignore exception in terminal state with $statementId: $errMsg") + val ke = KyuubiSQLException(errMsg) + setOperationException(ke) + throw ke } else { error(s"Error operating $opType: $errMsg", e) val ke = KyuubiSQLException(s"Error operating $opType: $errMsg", e) @@ -191,7 +201,7 @@ abstract class SparkOperation(session: Session) } override protected def afterRun(): Unit = { - state.synchronized { + withLockRequired { if (!isTerminalState(state)) { setState(OperationState.FINISHED) } @@ -223,10 +233,12 @@ abstract class SparkOperation(session: Session) resp } - override def getNextRowSet(order: FetchOrientation, rowSetSize: Int): TRowSet = - withLocalProperties { - var resultRowSet: TRowSet = null - try { + override def getNextRowSetInternal( + order: FetchOrientation, + rowSetSize: Int): TFetchResultsResp = { + var resultRowSet: TRowSet = null + try { + withLocalProperties { validateDefaultFetchOrientation(order) assertState(OperationState.FINISHED) setHasResultSet(true) @@ -236,7 +248,7 @@ abstract class SparkOperation(session: Session) case FETCH_FIRST => iter.fetchAbsolute(0); } resultRowSet = - if (arrowEnabled) { + if (isArrowBasedOperation) { if (iter.hasNext) { val taken = iter.next().asInstanceOf[Array[Byte]] RowSet.toTRowSet(taken, getProtocolVersion) @@ -246,28 +258,28 @@ abstract class SparkOperation(session: Session) } else { val taken = iter.take(rowSetSize) RowSet.toTRowSet( - taken.toList.asInstanceOf[List[Row]], + taken.toSeq.asInstanceOf[Seq[Row]], resultSchema, - getProtocolVersion, - timeZone) + getProtocolVersion) } resultRowSet.setStartRowOffset(iter.getPosition) - } catch onError(cancel = true) + } + } catch onError(cancel = true) - resultRowSet - } + val resp = new TFetchResultsResp(OK_STATUS) + resp.setResults(resultRowSet) + resp.setHasMoreRows(false) + resp + } override def shouldRunAsync: Boolean = false - protected def arrowEnabled(): Boolean = { - resultFormat().equalsIgnoreCase("arrow") && - // TODO: (fchen) make all operation support arrow - getClass.getCanonicalName == classOf[ExecuteStatement].getCanonicalName - } + protected def isArrowBasedOperation: Boolean = false + + protected def resultFormat: String = "thrift" - protected def resultFormat(): String = { - // TODO: respect the config of the operation ExecuteStatement, if it was set. - spark.conf.get("kyuubi.operation.result.format", "thrift") + protected def timestampAsString: Boolean = { + spark.conf.get("kyuubi.operation.result.arrow.timestampAsString", "false").toBoolean } protected def setSessionUserSign(): Unit = { diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SparkSQLOperationManager.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SparkSQLOperationManager.scala index 5c5ed0f98..ab0828746 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SparkSQLOperationManager.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/SparkSQLOperationManager.scala @@ -23,10 +23,11 @@ import scala.collection.JavaConverters._ import org.apache.kyuubi.KyuubiSQLException import org.apache.kyuubi.config.KyuubiConf._ +import org.apache.kyuubi.config.KyuubiReservedKeys.KYUUBI_OPERATION_HANDLE_KEY import org.apache.kyuubi.engine.spark.repl.KyuubiSparkILoop import org.apache.kyuubi.engine.spark.session.SparkSessionImpl -import org.apache.kyuubi.engine.spark.shim.SparkCatalogShim -import org.apache.kyuubi.operation.{NoneMode, Operation, OperationManager, PlanOnlyMode} +import org.apache.kyuubi.engine.spark.util.SparkCatalogUtils +import org.apache.kyuubi.operation.{NoneMode, Operation, OperationHandle, OperationManager, PlanOnlyMode} import org.apache.kyuubi.session.{Session, SessionHandle} class SparkSQLOperationManager private (name: String) extends OperationManager(name) { @@ -70,6 +71,8 @@ class SparkSQLOperationManager private (name: String) extends OperationManager(n val lang = OperationLanguages(confOverlay.getOrElse( OPERATION_LANGUAGE.key, spark.conf.get(OPERATION_LANGUAGE.key, operationLanguageDefault))) + val opHandle = confOverlay.get(KYUUBI_OPERATION_HANDLE_KEY).map( + OperationHandle.apply).getOrElse(OperationHandle()) val operation = lang match { case OperationLanguages.SQL => @@ -82,20 +85,39 @@ class SparkSQLOperationManager private (name: String) extends OperationManager(n case NoneMode => val incrementalCollect = spark.conf.getOption(OPERATION_INCREMENTAL_COLLECT.key) .map(_.toBoolean).getOrElse(operationIncrementalCollectDefault) - new ExecuteStatement(session, statement, runAsync, queryTimeout, incrementalCollect) + // TODO: respect the config of the operation ExecuteStatement, if it was set. + val resultFormat = spark.conf.get("kyuubi.operation.result.format", "thrift") + resultFormat.toLowerCase match { + case "arrow" => + new ArrowBasedExecuteStatement( + session, + statement, + runAsync, + queryTimeout, + incrementalCollect, + opHandle) + case _ => + new ExecuteStatement( + session, + statement, + runAsync, + queryTimeout, + incrementalCollect, + opHandle) + } case mode => - new PlanOnlyStatement(session, statement, mode) + new PlanOnlyStatement(session, statement, mode, opHandle) } case OperationLanguages.SCALA => val repl = sessionToRepl.getOrElseUpdate(session.handle, KyuubiSparkILoop(spark)) - new ExecuteScala(session, repl, statement, runAsync, queryTimeout) + new ExecuteScala(session, repl, statement, runAsync, queryTimeout, opHandle) case OperationLanguages.PYTHON => try { ExecutePython.init() val worker = sessionToPythonProcess.getOrElseUpdate( session.handle, ExecutePython.createSessionPythonWorker(spark, session)) - new ExecutePython(session, statement, runAsync, queryTimeout, worker) + new ExecutePython(session, statement, runAsync, queryTimeout, worker, opHandle) } catch { case e: Throwable => spark.conf.set(OPERATION_LANGUAGE.key, OperationLanguages.SQL.toString) @@ -157,7 +179,7 @@ class SparkSQLOperationManager private (name: String) extends OperationManager(n tableTypes: java.util.List[String]): Operation = { val tTypes = if (tableTypes == null || tableTypes.isEmpty) { - SparkCatalogShim.sparkTableTypes + SparkCatalogUtils.sparkTableTypes } else { tableTypes.asScala.toSet } @@ -209,6 +231,6 @@ class SparkSQLOperationManager private (name: String) extends OperationManager(n } override def getQueryId(operation: Operation): String = { - throw KyuubiSQLException.featureNotSupported() + operation.getHandle.identifier.toString } } diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/schema/RowSet.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/schema/RowSet.scala index 8cc88156b..4f935ce49 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/schema/RowSet.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/schema/RowSet.scala @@ -18,22 +18,24 @@ package org.apache.kyuubi.engine.spark.schema import java.nio.ByteBuffer -import java.nio.charset.StandardCharsets -import java.sql.Timestamp -import java.time._ -import java.util.Date import scala.collection.JavaConverters._ import org.apache.hive.service.rpc.thrift._ import org.apache.spark.sql.Row +import org.apache.spark.sql.execution.HiveResult import org.apache.spark.sql.types._ -import org.apache.kyuubi.engine.spark.schema.SchemaHelper.TIMESTAMP_NTZ import org.apache.kyuubi.util.RowSetUtils._ object RowSet { + def toHiveString(valueAndType: (Any, DataType), nested: Boolean = false): String = { + // compatible w/ Spark 3.1 and above + val timeFormatters = HiveResult.getTimeFormatters + HiveResult.toHiveString(valueAndType, nested, timeFormatters) + } + def toTRowSet( bytes: Array[Byte], protocolVersion: TProtocolVersion): TRowSet = { @@ -58,26 +60,25 @@ object RowSet { def toTRowSet( rows: Seq[Row], schema: StructType, - protocolVersion: TProtocolVersion, - timeZone: ZoneId): TRowSet = { + protocolVersion: TProtocolVersion): TRowSet = { if (protocolVersion.getValue < TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V6.getValue) { - toRowBasedSet(rows, schema, timeZone) + toRowBasedSet(rows, schema) } else { - toColumnBasedSet(rows, schema, timeZone) + toColumnBasedSet(rows, schema) } } - def toRowBasedSet(rows: Seq[Row], schema: StructType, timeZone: ZoneId): TRowSet = { - var i = 0 + def toRowBasedSet(rows: Seq[Row], schema: StructType): TRowSet = { val rowSize = rows.length val tRows = new java.util.ArrayList[TRow](rowSize) + var i = 0 while (i < rowSize) { val row = rows(i) val tRow = new TRow() var j = 0 val columnSize = row.length while (j < columnSize) { - val columnValue = toTColumnValue(j, row, schema, timeZone) + val columnValue = toTColumnValue(j, row, schema) tRow.addToColVals(columnValue) j += 1 } @@ -87,21 +88,21 @@ object RowSet { new TRowSet(0, tRows) } - def toColumnBasedSet(rows: Seq[Row], schema: StructType, timeZone: ZoneId): TRowSet = { + def toColumnBasedSet(rows: Seq[Row], schema: StructType): TRowSet = { val rowSize = rows.length val tRowSet = new TRowSet(0, new java.util.ArrayList[TRow](rowSize)) var i = 0 val columnSize = schema.length while (i < columnSize) { val field = schema(i) - val tColumn = toTColumn(rows, i, field.dataType, timeZone) + val tColumn = toTColumn(rows, i, field.dataType) tRowSet.addToColumns(tColumn) i += 1 } tRowSet } - private def toTColumn(rows: Seq[Row], ordinal: Int, typ: DataType, timeZone: ZoneId): TColumn = { + private def toTColumn(rows: Seq[Row], ordinal: Int, typ: DataType): TColumn = { val nulls = new java.util.BitSet() typ match { case BooleanType => @@ -151,13 +152,7 @@ object RowSet { while (i < rowSize) { val row = rows(i) nulls.set(i, row.isNullAt(ordinal)) - val value = - if (row.isNullAt(ordinal)) { - "" - } else { - toHiveString((row.get(ordinal), typ), timeZone) - } - values.add(value) + values.add(toHiveString(row.get(ordinal) -> typ)) i += 1 } TColumn.stringVal(new TStringColumn(values, nulls)) @@ -189,8 +184,7 @@ object RowSet { private def toTColumnValue( ordinal: Int, row: Row, - types: StructType, - timeZone: ZoneId): TColumnValue = { + types: StructType): TColumnValue = { types(ordinal).dataType match { case BooleanType => val boolValue = new TBoolValue @@ -238,69 +232,12 @@ object RowSet { case _ => val tStrValue = new TStringValue if (!row.isNullAt(ordinal)) { - tStrValue.setValue( - toHiveString((row.get(ordinal), types(ordinal).dataType), timeZone)) + tStrValue.setValue(toHiveString(row.get(ordinal) -> types(ordinal).dataType)) } TColumnValue.stringVal(tStrValue) } } - /** - * A simpler impl of Spark's toHiveString - */ - def toHiveString(dataWithType: (Any, DataType), timeZone: ZoneId): String = { - dataWithType match { - case (null, _) => - // Only match nulls in nested type values - "null" - - case (d: Date, DateType) => - formatDate(d) - - case (ld: LocalDate, DateType) => - formatLocalDate(ld) - - case (t: Timestamp, TimestampType) => - formatTimestamp(t, Option(timeZone)) - - case (t: LocalDateTime, ntz) if ntz.getClass.getSimpleName.equals(TIMESTAMP_NTZ) => - formatLocalDateTime(t) - - case (i: Instant, TimestampType) => - formatInstant(i, Option(timeZone)) - - case (bin: Array[Byte], BinaryType) => - new String(bin, StandardCharsets.UTF_8) - - case (decimal: java.math.BigDecimal, DecimalType()) => - decimal.toPlainString - - case (s: String, StringType) => - // Only match string in nested type values - "\"" + s + "\"" - - case (d: Duration, _) => toDayTimeIntervalString(d) - - case (p: Period, _) => toYearMonthIntervalString(p) - - case (seq: scala.collection.Seq[_], ArrayType(typ, _)) => - seq.map(v => (v, typ)).map(e => toHiveString(e, timeZone)).mkString("[", ",", "]") - - case (m: Map[_, _], MapType(kType, vType, _)) => - m.map { case (key, value) => - toHiveString((key, kType), timeZone) + ":" + toHiveString((value, vType), timeZone) - }.toSeq.sorted.mkString("{", ",", "}") - - case (struct: Row, StructType(fields)) => - struct.toSeq.zip(fields).map { case (v, t) => - s""""${t.name}":${toHiveString((v, t.dataType), timeZone)}""" - }.mkString("{", ",", "}") - - case (other, _) => - other.toString - } - } - private def toTColumn(data: Array[Byte]): TColumn = { val values = new java.util.ArrayList[ByteBuffer](1) values.add(ByteBuffer.wrap(data)) diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/session/SparkSQLSessionManager.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/session/SparkSQLSessionManager.scala index 76c6a6505..79f38ce35 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/session/SparkSQLSessionManager.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/session/SparkSQLSessionManager.scala @@ -20,10 +20,12 @@ package org.apache.kyuubi.engine.spark.session import java.util.concurrent.{ScheduledExecutorService, TimeUnit} import org.apache.hive.service.rpc.thrift.TProtocolVersion +import org.apache.spark.api.python.KyuubiPythonGatewayServer import org.apache.spark.sql.SparkSession import org.apache.kyuubi.KyuubiSQLException import org.apache.kyuubi.config.KyuubiConf._ +import org.apache.kyuubi.config.KyuubiReservedKeys.KYUUBI_SESSION_HANDLE_KEY import org.apache.kyuubi.engine.ShareLevel import org.apache.kyuubi.engine.ShareLevel._ import org.apache.kyuubi.engine.spark.{KyuubiSparkUtil, SparkSQLEngine} @@ -93,6 +95,7 @@ class SparkSQLSessionManager private (name: String, spark: SparkSession) override def stop(): Unit = { super.stop() + KyuubiPythonGatewayServer.shutdown() userIsolatedSparkSessionThread.foreach(_.shutdown()) } @@ -135,21 +138,24 @@ class SparkSQLSessionManager private (name: String, spark: SparkSession) password: String, ipAddress: String, conf: Map[String, String]): Session = { - val sparkSession = - try { - getOrNewSparkSession(user) - } catch { - case e: Exception => throw KyuubiSQLException(e) - } + conf.get(KYUUBI_SESSION_HANDLE_KEY).map(SessionHandle.fromUUID).flatMap( + getSessionOption).getOrElse { + val sparkSession = + try { + getOrNewSparkSession(user) + } catch { + case e: Exception => throw KyuubiSQLException(e) + } - new SparkSessionImpl( - protocol, - user, - password, - ipAddress, - conf, - this, - sparkSession) + new SparkSessionImpl( + protocol, + user, + password, + ipAddress, + conf, + this, + sparkSession) + } } override def closeSession(sessionHandle: SessionHandle): Unit = { @@ -164,7 +170,12 @@ class SparkSQLSessionManager private (name: String, spark: SparkSession) } } } - super.closeSession(sessionHandle) + try { + super.closeSession(sessionHandle) + } catch { + case e: KyuubiSQLException => + warn(s"Error closing session ${sessionHandle}", e) + } if (shareLevel == ShareLevel.CONNECTION) { info("Session stopped due to shared level is Connection.") stopSession() diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/session/SparkSessionImpl.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/session/SparkSessionImpl.scala index 5bf1ec084..8d9012cbd 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/session/SparkSessionImpl.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/session/SparkSessionImpl.scala @@ -17,17 +17,19 @@ package org.apache.kyuubi.engine.spark.session +import org.apache.commons.lang3.StringUtils import org.apache.hive.service.rpc.thrift.{TGetInfoType, TGetInfoValue, TProtocolVersion} import org.apache.spark.sql.{AnalysisException, SparkSession} import org.apache.kyuubi.KyuubiSQLException +import org.apache.kyuubi.config.KyuubiReservedKeys.KYUUBI_SESSION_HANDLE_KEY import org.apache.kyuubi.engine.spark.events.SessionEvent import org.apache.kyuubi.engine.spark.operation.SparkSQLOperationManager -import org.apache.kyuubi.engine.spark.shim.SparkCatalogShim import org.apache.kyuubi.engine.spark.udf.KDFRegistry +import org.apache.kyuubi.engine.spark.util.SparkCatalogUtils import org.apache.kyuubi.events.EventBus import org.apache.kyuubi.operation.{Operation, OperationHandle} -import org.apache.kyuubi.session.{AbstractSession, SessionManager} +import org.apache.kyuubi.session._ class SparkSessionImpl( protocol: TProtocolVersion, @@ -39,6 +41,9 @@ class SparkSessionImpl( val spark: SparkSession) extends AbstractSession(protocol, user, password, ipAddress, conf, sessionManager) { + override val handle: SessionHandle = + conf.get(KYUUBI_SESSION_HANDLE_KEY).map(SessionHandle.fromUUID).getOrElse(SessionHandle()) + private def setModifiableConfig(key: String, value: String): Unit = { try { spark.conf.set(key, value) @@ -50,22 +55,35 @@ class SparkSessionImpl( private val sessionEvent = SessionEvent(this) override def open(): Unit = { - normalizedConf.foreach { - case ("use:catalog", catalog) => - try { - SparkCatalogShim().setCurrentCatalog(spark, catalog) - } catch { - case e if e.getMessage.contains("Cannot find catalog plugin class for catalog") => - warn(e.getMessage()) - } - case ("use:database", database) => - try { - SparkCatalogShim().setCurrentDatabase(spark, database) - } catch { - case e - if database == "default" && e.getMessage != null && - e.getMessage.contains("not found") => - } + + val (useCatalogAndDatabaseConf, otherConf) = normalizedConf.partition { case (k, _) => + Array(USE_CATALOG, USE_DATABASE).contains(k) + } + + useCatalogAndDatabaseConf.get(USE_CATALOG).foreach { catalog => + try { + SparkCatalogUtils.setCurrentCatalog(spark, catalog) + } catch { + case e if e.getMessage.contains("Cannot find catalog plugin class for catalog") => + warn(e.getMessage()) + } + } + + useCatalogAndDatabaseConf.get("use:database").foreach { database => + try { + spark.sessionState.catalogManager.setCurrentNamespace(Array(database)) + } catch { + case e + if database == "default" && + StringUtils.containsAny( + e.getMessage, + "not found", + "SCHEMA_NOT_FOUND", + "is not authorized to perform: glue:GetDatabase") => + } + } + + otherConf.foreach { case (key, value) => setModifiableConfig(key, value) } KDFRegistry.registerAll(spark) diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/shim/CatalogShim_v2_4.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/shim/CatalogShim_v2_4.scala deleted file mode 100644 index 1aa322d31..000000000 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/shim/CatalogShim_v2_4.scala +++ /dev/null @@ -1,189 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kyuubi.engine.spark.shim - -import java.util.regex.Pattern - -import org.apache.spark.sql.{Row, SparkSession} -import org.apache.spark.sql.catalyst.TableIdentifier - -class CatalogShim_v2_4 extends SparkCatalogShim { - - override def getCatalogs(spark: SparkSession): Seq[Row] = { - Seq(Row(SparkCatalogShim.SESSION_CATALOG)) - } - - override protected def catalogExists(spark: SparkSession, catalog: String): Boolean = false - - override def setCurrentCatalog(spark: SparkSession, catalog: String): Unit = {} - - override def getCurrentCatalog(spark: SparkSession): Row = { - Row(SparkCatalogShim.SESSION_CATALOG) - } - - override def getSchemas( - spark: SparkSession, - catalogName: String, - schemaPattern: String): Seq[Row] = { - (spark.sessionState.catalog.listDatabases(schemaPattern) ++ - getGlobalTempViewManager(spark, schemaPattern)).map(Row(_, "")) - } - - def setCurrentDatabase(spark: SparkSession, databaseName: String): Unit = { - spark.sessionState.catalog.setCurrentDatabase(databaseName) - } - - def getCurrentDatabase(spark: SparkSession): Row = { - Row(spark.sessionState.catalog.getCurrentDatabase) - } - - override protected def getGlobalTempViewManager( - spark: SparkSession, - schemaPattern: String): Seq[String] = { - val database = spark.sharedState.globalTempViewManager.database - Option(database).filter(_.matches(schemaPattern)).toSeq - } - - override def getCatalogTablesOrViews( - spark: SparkSession, - catalogName: String, - schemaPattern: String, - tablePattern: String, - tableTypes: Set[String]): Seq[Row] = { - val catalog = spark.sessionState.catalog - val databases = catalog.listDatabases(schemaPattern) - - databases.flatMap { db => - val identifiers = catalog.listTables(db, tablePattern, includeLocalTempViews = false) - catalog.getTablesByName(identifiers) - .filter(t => matched(tableTypes, t.tableType.name)).map { t => - val typ = if (t.tableType.name == "VIEW") "VIEW" else "TABLE" - Row( - catalogName, - t.database, - t.identifier.table, - typ, - t.comment.getOrElse(""), - null, - null, - null, - null, - null) - } - } - } - - override def getTempViews( - spark: SparkSession, - catalogName: String, - schemaPattern: String, - tablePattern: String): Seq[Row] = { - val views = getViews(spark, schemaPattern, tablePattern) - views.map { ident => - Row(catalogName, ident.database.orNull, ident.table, "VIEW", "", null, null, null, null, null) - } - } - - override protected def getViews( - spark: SparkSession, - schemaPattern: String, - tablePattern: String): Seq[TableIdentifier] = { - val db = getGlobalTempViewManager(spark, schemaPattern) - if (db.nonEmpty) { - spark.sessionState.catalog.listTables(db.head, tablePattern) - } else { - spark.sessionState.catalog.listLocalTempViews(tablePattern) - } - } - - override def getColumns( - spark: SparkSession, - catalogName: String, - schemaPattern: String, - tablePattern: String, - columnPattern: String): Seq[Row] = { - - val cp = columnPattern.r.pattern - val byCatalog = getColumnsByCatalog(spark, catalogName, schemaPattern, tablePattern, cp) - val byGlobalTmpDB = getColumnsByGlobalTempViewManager(spark, schemaPattern, tablePattern, cp) - val byLocalTmp = getColumnsByLocalTempViews(spark, tablePattern, cp) - - byCatalog ++ byGlobalTmpDB ++ byLocalTmp - } - - protected def getColumnsByCatalog( - spark: SparkSession, - catalogName: String, - schemaPattern: String, - tablePattern: String, - columnPattern: Pattern): Seq[Row] = { - val catalog = spark.sessionState.catalog - - val databases = catalog.listDatabases(schemaPattern) - - databases.flatMap { db => - val identifiers = catalog.listTables(db, tablePattern, includeLocalTempViews = true) - catalog.getTablesByName(identifiers).flatMap { t => - val tableSchema = - if (t.provider.getOrElse("").equalsIgnoreCase("delta")) { - spark.table(f"${db}.${t.identifier.table}").schema - } else { - t.schema - } - tableSchema.zipWithIndex.filter(f => columnPattern.matcher(f._1.name).matches()) - .map { case (f, i) => toColumnResult(catalogName, t.database, t.identifier.table, f, i) } - } - } - } - - protected def getColumnsByGlobalTempViewManager( - spark: SparkSession, - schemaPattern: String, - tablePattern: String, - columnPattern: Pattern): Seq[Row] = { - val catalog = spark.sessionState.catalog - - getGlobalTempViewManager(spark, schemaPattern).flatMap { globalTmpDb => - catalog.globalTempViewManager.listViewNames(tablePattern).flatMap { v => - catalog.globalTempViewManager.get(v).map { plan => - plan.schema.zipWithIndex.filter(f => columnPattern.matcher(f._1.name).matches()) - .map { case (f, i) => - toColumnResult(SparkCatalogShim.SESSION_CATALOG, globalTmpDb, v, f, i) - } - } - }.flatten - } - } - - protected def getColumnsByLocalTempViews( - spark: SparkSession, - tablePattern: String, - columnPattern: Pattern): Seq[Row] = { - val catalog = spark.sessionState.catalog - - catalog.listLocalTempViews(tablePattern) - .map(v => (v, catalog.getTempView(v.table).get)) - .flatMap { case (v, plan) => - plan.schema.zipWithIndex - .filter(f => columnPattern.matcher(f._1.name).matches()) - .map { case (f, i) => - toColumnResult(SparkCatalogShim.SESSION_CATALOG, null, v.table, f, i) - } - } - } -} diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/shim/CatalogShim_v3_0.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/shim/CatalogShim_v3_0.scala deleted file mode 100644 index d60f94ac7..000000000 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/shim/CatalogShim_v3_0.scala +++ /dev/null @@ -1,215 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kyuubi.engine.spark.shim - -import java.util.regex.Pattern - -import org.apache.spark.sql.{Row, SparkSession} -import org.apache.spark.sql.connector.catalog.{CatalogExtension, CatalogPlugin, SupportsNamespaces, TableCatalog} - -import org.apache.kyuubi.engine.spark.shim.SparkCatalogShim.SESSION_CATALOG - -class CatalogShim_v3_0 extends CatalogShim_v2_4 { - - override def getCatalogs(spark: SparkSession): Seq[Row] = { - - // A [[CatalogManager]] is session unique - val catalogMgr = spark.sessionState.catalogManager - // get the custom v2 session catalog or default spark_catalog - val sessionCatalog = invoke(catalogMgr, "v2SessionCatalog") - val defaultCatalog = catalogMgr.currentCatalog - - val defaults = Seq(sessionCatalog, defaultCatalog).distinct - .map(invoke(_, "name").asInstanceOf[String]) - val catalogs = getField(catalogMgr, "catalogs") - .asInstanceOf[scala.collection.Map[String, _]] - (catalogs.keys ++: defaults).distinct.map(Row(_)) - } - - private def getCatalog(spark: SparkSession, catalogName: String): CatalogPlugin = { - val catalogManager = spark.sessionState.catalogManager - if (catalogName == null || catalogName.isEmpty) { - catalogManager.currentCatalog - } else { - catalogManager.catalog(catalogName) - } - } - - override def catalogExists(spark: SparkSession, catalog: String): Boolean = { - spark.sessionState.catalogManager.isCatalogRegistered(catalog) - } - - override def setCurrentCatalog(spark: SparkSession, catalog: String): Unit = { - // SPARK-36841(3.3.0) Ensure setCurrentCatalog method catalog must exist - if (spark.sessionState.catalogManager.isCatalogRegistered(catalog)) { - spark.sessionState.catalogManager.setCurrentCatalog(catalog) - } else { - throw new IllegalArgumentException(s"Cannot find catalog plugin class for catalog '$catalog'") - } - } - - override def getCurrentCatalog(spark: SparkSession): Row = { - Row(spark.sessionState.catalogManager.currentCatalog.name()) - } - - private def listAllNamespaces( - catalog: SupportsNamespaces, - namespaces: Array[Array[String]]): Array[Array[String]] = { - val children = namespaces.flatMap { ns => - catalog.listNamespaces(ns) - } - if (children.isEmpty) { - namespaces - } else { - namespaces ++: listAllNamespaces(catalog, children) - } - } - - private def listAllNamespaces(catalog: CatalogPlugin): Array[Array[String]] = { - catalog match { - case catalog: CatalogExtension => - // DSv2 does not support pass schemaPattern transparently - catalog.defaultNamespace() +: catalog.listNamespaces(Array()) - case catalog: SupportsNamespaces => - val rootSchema = catalog.listNamespaces() - val allSchemas = listAllNamespaces(catalog, rootSchema) - allSchemas - } - } - - /** - * Forked from Apache Spark's org.apache.spark.sql.connector.catalog.CatalogV2Implicits - */ - private def quoteIfNeeded(part: String): String = { - if (part.contains(".") || part.contains("`")) { - s"`${part.replace("`", "``")}`" - } else { - part - } - } - - private def listNamespacesWithPattern( - catalog: CatalogPlugin, - schemaPattern: String): Array[Array[String]] = { - val p = schemaPattern.r.pattern - listAllNamespaces(catalog).filter { ns => - val quoted = ns.map(quoteIfNeeded).mkString(".") - p.matcher(quoted).matches() - }.distinct - } - - private def getSchemasWithPattern(catalog: CatalogPlugin, schemaPattern: String): Seq[String] = { - val p = schemaPattern.r.pattern - listAllNamespaces(catalog).flatMap { ns => - val quoted = ns.map(quoteIfNeeded).mkString(".") - if (p.matcher(quoted).matches()) { - Some(quoted) - } else { - None - } - }.distinct - } - - override def getSchemas( - spark: SparkSession, - catalogName: String, - schemaPattern: String): Seq[Row] = { - val catalog = getCatalog(spark, catalogName) - var schemas = getSchemasWithPattern(catalog, schemaPattern) - if (catalogName == SparkCatalogShim.SESSION_CATALOG) { - val viewMgr = getGlobalTempViewManager(spark, schemaPattern) - schemas = schemas ++ viewMgr - } - schemas.map(Row(_, catalog.name)) - } - - override def setCurrentDatabase(spark: SparkSession, databaseName: String): Unit = { - spark.sessionState.catalogManager.setCurrentNamespace(Array(databaseName)) - } - - override def getCurrentDatabase(spark: SparkSession): Row = { - Row(spark.sessionState.catalogManager.currentNamespace.map(quoteIfNeeded).mkString(".")) - } - - override def getCatalogTablesOrViews( - spark: SparkSession, - catalogName: String, - schemaPattern: String, - tablePattern: String, - tableTypes: Set[String]): Seq[Row] = { - val catalog = getCatalog(spark, catalogName) - val namespaces = listNamespacesWithPattern(catalog, schemaPattern) - catalog match { - case builtin if builtin.name() == SESSION_CATALOG => - super.getCatalogTablesOrViews( - spark, - SESSION_CATALOG, - schemaPattern, - tablePattern, - tableTypes) - case tc: TableCatalog => - val tp = tablePattern.r.pattern - val identifiers = namespaces.flatMap { ns => - tc.listTables(ns).filter(i => tp.matcher(quoteIfNeeded(i.name())).matches()) - } - identifiers.map { ident => - val table = tc.loadTable(ident) - // TODO: restore view type for session catalog - val comment = table.properties().getOrDefault(TableCatalog.PROP_COMMENT, "") - val schema = ident.namespace().map(quoteIfNeeded).mkString(".") - val tableName = quoteIfNeeded(ident.name()) - Row(catalog.name(), schema, tableName, "TABLE", comment, null, null, null, null, null) - } - case _ => Seq.empty[Row] - } - } - - override protected def getColumnsByCatalog( - spark: SparkSession, - catalogName: String, - schemaPattern: String, - tablePattern: String, - columnPattern: Pattern): Seq[Row] = { - val catalog = getCatalog(spark, catalogName) - - catalog match { - case builtin if builtin.name() == SESSION_CATALOG => - super.getColumnsByCatalog( - spark, - SESSION_CATALOG, - schemaPattern, - tablePattern, - columnPattern) - - case tc: TableCatalog => - val namespaces = listNamespacesWithPattern(catalog, schemaPattern) - val tp = tablePattern.r.pattern - val identifiers = namespaces.flatMap { ns => - tc.listTables(ns).filter(i => tp.matcher(quoteIfNeeded(i.name())).matches()) - } - identifiers.flatMap { ident => - val table = tc.loadTable(ident) - val namespace = ident.namespace().map(quoteIfNeeded).mkString(".") - val tableName = quoteIfNeeded(ident.name()) - - table.schema.zipWithIndex.filter(f => columnPattern.matcher(f._1.name).matches()) - .map { case (f, i) => toColumnResult(tc.name(), namespace, tableName, f, i) } - } - } - } -} diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/shim/SparkCatalogShim.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/shim/SparkCatalogShim.scala deleted file mode 100644 index bc5792823..000000000 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/shim/SparkCatalogShim.scala +++ /dev/null @@ -1,182 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kyuubi.engine.spark.shim - -import org.apache.spark.sql.{Row, SparkSession} -import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.types.StructField - -import org.apache.kyuubi.Logging -import org.apache.kyuubi.engine.spark.KyuubiSparkUtil.sparkMajorMinorVersion -import org.apache.kyuubi.engine.spark.schema.SchemaHelper - -/** - * A shim that defines the interface interact with Spark's catalogs - */ -trait SparkCatalogShim extends Logging { - - // /////////////////////////////////////////////////////////////////////////////////////////////// - // Catalog // - // /////////////////////////////////////////////////////////////////////////////////////////////// - - /** - * Get all register catalogs in Spark's `CatalogManager` - */ - def getCatalogs(spark: SparkSession): Seq[Row] - - protected def catalogExists(spark: SparkSession, catalog: String): Boolean - - def setCurrentCatalog(spark: SparkSession, catalog: String): Unit - - def getCurrentCatalog(spark: SparkSession): Row - - // /////////////////////////////////////////////////////////////////////////////////////////////// - // Schema // - // /////////////////////////////////////////////////////////////////////////////////////////////// - - /** - * a list of [[Row]]s, with 2 fields `schemaName: String, catalogName: String` - */ - def getSchemas(spark: SparkSession, catalogName: String, schemaPattern: String): Seq[Row] - - def setCurrentDatabase(spark: SparkSession, databaseName: String): Unit - - def getCurrentDatabase(spark: SparkSession): Row - - protected def getGlobalTempViewManager(spark: SparkSession, schemaPattern: String): Seq[String] - - // /////////////////////////////////////////////////////////////////////////////////////////////// - // Table & View // - // /////////////////////////////////////////////////////////////////////////////////////////////// - - def getCatalogTablesOrViews( - spark: SparkSession, - catalogName: String, - schemaPattern: String, - tablePattern: String, - tableTypes: Set[String]): Seq[Row] - - def getTempViews( - spark: SparkSession, - catalogName: String, - schemaPattern: String, - tablePattern: String): Seq[Row] - - protected def getViews( - spark: SparkSession, - schemaPattern: String, - tablePattern: String): Seq[TableIdentifier] - - // /////////////////////////////////////////////////////////////////////////////////////////////// - // Columns // - // /////////////////////////////////////////////////////////////////////////////////////////////// - - def getColumns( - spark: SparkSession, - catalogName: String, - schemaPattern: String, - tablePattern: String, - columnPattern: String): Seq[Row] - - protected def toColumnResult( - catalog: String, - db: String, - table: String, - col: StructField, - pos: Int): Row = { - // format: off - Row( - catalog, // TABLE_CAT - db, // TABLE_SCHEM - table, // TABLE_NAME - col.name, // COLUMN_NAME - SchemaHelper.toJavaSQLType(col.dataType), // DATA_TYPE - col.dataType.sql, // TYPE_NAME - SchemaHelper.getColumnSize(col.dataType).orNull, // COLUMN_SIZE - null, // BUFFER_LENGTH - SchemaHelper.getDecimalDigits(col.dataType).orNull, // DECIMAL_DIGITS - SchemaHelper.getNumPrecRadix(col.dataType).orNull, // NUM_PREC_RADIX - if (col.nullable) 1 else 0, // NULLABLE - col.getComment().getOrElse(""), // REMARKS - null, // COLUMN_DEF - null, // SQL_DATA_TYPE - null, // SQL_DATETIME_SUB - null, // CHAR_OCTET_LENGTH - pos, // ORDINAL_POSITION - "YES", // IS_NULLABLE - null, // SCOPE_CATALOG - null, // SCOPE_SCHEMA - null, // SCOPE_TABLE - null, // SOURCE_DATA_TYPE - "NO" // IS_AUTO_INCREMENT - ) - // format: on - } - - // /////////////////////////////////////////////////////////////////////////////////////////////// - // Miscellaneous // - // /////////////////////////////////////////////////////////////////////////////////////////////// - - protected def invoke( - obj: Any, - methodName: String, - args: (Class[_], AnyRef)*): Any = { - val (types, values) = args.unzip - val method = obj.getClass.getMethod(methodName, types: _*) - method.setAccessible(true) - method.invoke(obj, values.toSeq: _*) - } - - protected def invoke( - clazz: Class[_], - obj: AnyRef, - methodName: String, - args: (Class[_], AnyRef)*): AnyRef = { - val (types, values) = args.unzip - val method = clazz.getMethod(methodName, types: _*) - method.setAccessible(true) - method.invoke(obj, values.toSeq: _*) - } - - protected def getField(o: Any, fieldName: String): Any = { - val field = o.getClass.getDeclaredField(fieldName) - field.setAccessible(true) - field.get(o) - } - - protected def matched(tableTypes: Set[String], tableType: String): Boolean = { - val typ = if (tableType.equalsIgnoreCase("VIEW")) "VIEW" else "TABLE" - tableTypes.exists(typ.equalsIgnoreCase) - } - -} - -object SparkCatalogShim { - def apply(): SparkCatalogShim = { - sparkMajorMinorVersion match { - case (3, _) => new CatalogShim_v3_0 - case (2, _) => new CatalogShim_v2_4 - case _ => - throw new IllegalArgumentException(s"Not Support spark version $sparkMajorMinorVersion") - } - } - - val SESSION_CATALOG: String = "spark_catalog" - - val sparkTableTypes = Set("VIEW", "TABLE") -} diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/udf/KDFRegistry.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/udf/KDFRegistry.scala index f4612a3d0..a2d50d151 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/udf/KDFRegistry.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/udf/KDFRegistry.scala @@ -25,7 +25,7 @@ import org.apache.spark.sql.expressions.UserDefinedFunction import org.apache.spark.sql.functions.udf import org.apache.kyuubi.{KYUUBI_VERSION, Utils} -import org.apache.kyuubi.config.KyuubiReservedKeys.KYUUBI_SESSION_USER_KEY +import org.apache.kyuubi.config.KyuubiReservedKeys.{KYUUBI_ENGINE_URL, KYUUBI_SESSION_USER_KEY} object KDFRegistry { @@ -73,6 +73,16 @@ object KDFRegistry { "string", "1.4.0") + val engine_url: KyuubiDefinedFunction = create( + "engine_url", + udf { () => + Option(TaskContext.get()).map(_.getLocalProperty(KYUUBI_ENGINE_URL)) + .getOrElse(throw new RuntimeException("Unable to get engine url")) + }, + "Return the engine url for the associated query engine", + "string", + "1.8.0") + def create( name: String, udf: UserDefinedFunction, diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/udf/KyuubiDefinedFunction.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/udf/KyuubiDefinedFunction.scala index 30228bf72..6bc2e3ddb 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/udf/KyuubiDefinedFunction.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/udf/KyuubiDefinedFunction.scala @@ -20,7 +20,7 @@ package org.apache.kyuubi.engine.spark.udf import org.apache.spark.sql.expressions.UserDefinedFunction /** - * A wrapper for Spark' [[UserDefinedFunction]] + * A wrapper for Spark's [[UserDefinedFunction]] * * @param name function name * @param udf user-defined function diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/util/SparkCatalogUtils.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/util/SparkCatalogUtils.scala new file mode 100644 index 000000000..18a14494e --- /dev/null +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/util/SparkCatalogUtils.scala @@ -0,0 +1,373 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.engine.spark.util + +import java.util.regex.Pattern + +import org.apache.commons.lang3.StringUtils +import org.apache.spark.sql.{Row, SparkSession} +import org.apache.spark.sql.catalyst.TableIdentifier +import org.apache.spark.sql.connector.catalog.{CatalogExtension, CatalogPlugin, SupportsNamespaces, TableCatalog} +import org.apache.spark.sql.types.StructField + +import org.apache.kyuubi.Logging +import org.apache.kyuubi.engine.spark.schema.SchemaHelper +import org.apache.kyuubi.util.reflect.ReflectUtils._ + +/** + * A shim that defines the interface interact with Spark's catalogs + */ +object SparkCatalogUtils extends Logging { + + private val VIEW = "VIEW" + private val TABLE = "TABLE" + + val SESSION_CATALOG: String = "spark_catalog" + val sparkTableTypes: Set[String] = Set(VIEW, TABLE) + + // /////////////////////////////////////////////////////////////////////////////////////////////// + // Catalog // + // /////////////////////////////////////////////////////////////////////////////////////////////// + + /** + * Get all register catalogs in Spark's `CatalogManager` + */ + def getCatalogs(spark: SparkSession): Seq[Row] = { + + // A [[CatalogManager]] is session unique + val catalogMgr = spark.sessionState.catalogManager + // get the custom v2 session catalog or default spark_catalog + val sessionCatalog = invokeAs[AnyRef](catalogMgr, "v2SessionCatalog") + val defaultCatalog = catalogMgr.currentCatalog + + val defaults = Seq(sessionCatalog, defaultCatalog).distinct.map(invokeAs[String](_, "name")) + val catalogs = getField[scala.collection.Map[String, _]](catalogMgr, "catalogs") + (catalogs.keys ++: defaults).distinct.map(Row(_)) + } + + def getCatalog(spark: SparkSession, catalogName: String): CatalogPlugin = { + val catalogManager = spark.sessionState.catalogManager + if (StringUtils.isBlank(catalogName)) { + catalogManager.currentCatalog + } else { + catalogManager.catalog(catalogName) + } + } + + def setCurrentCatalog(spark: SparkSession, catalog: String): Unit = { + // SPARK-36841(3.3.0) Ensure setCurrentCatalog method catalog must exist + if (spark.sessionState.catalogManager.isCatalogRegistered(catalog)) { + spark.sessionState.catalogManager.setCurrentCatalog(catalog) + } else { + throw new IllegalArgumentException(s"Cannot find catalog plugin class for catalog '$catalog'") + } + } + + // /////////////////////////////////////////////////////////////////////////////////////////////// + // Schema // + // /////////////////////////////////////////////////////////////////////////////////////////////// + + /** + * a list of [[Row]]s, with 2 fields `schemaName: String, catalogName: String` + */ + def getSchemas( + spark: SparkSession, + catalogName: String, + schemaPattern: String): Seq[Row] = { + if (catalogName == SparkCatalogUtils.SESSION_CATALOG) { + (spark.sessionState.catalog.listDatabases(schemaPattern) ++ + getGlobalTempViewManager(spark, schemaPattern)) + .map(Row(_, SparkCatalogUtils.SESSION_CATALOG)) + } else { + val catalog = getCatalog(spark, catalogName) + getSchemasWithPattern(catalog, schemaPattern).map(Row(_, catalog.name)) + } + } + + private def getGlobalTempViewManager( + spark: SparkSession, + schemaPattern: String): Seq[String] = { + val database = spark.sharedState.globalTempViewManager.database + Option(database).filter(_.matches(schemaPattern)).toSeq + } + + private def listAllNamespaces( + catalog: SupportsNamespaces, + namespaces: Array[Array[String]]): Array[Array[String]] = { + val children = namespaces.flatMap { ns => + catalog.listNamespaces(ns) + } + if (children.isEmpty) { + namespaces + } else { + namespaces ++: listAllNamespaces(catalog, children) + } + } + + private def listAllNamespaces(catalog: CatalogPlugin): Array[Array[String]] = { + catalog match { + case catalog: CatalogExtension => + // DSv2 does not support pass schemaPattern transparently + catalog.defaultNamespace() +: catalog.listNamespaces(Array()) + case catalog: SupportsNamespaces => + val rootSchema = catalog.listNamespaces() + val allSchemas = listAllNamespaces(catalog, rootSchema) + allSchemas + } + } + + private def listNamespacesWithPattern( + catalog: CatalogPlugin, + schemaPattern: String): Array[Array[String]] = { + listAllNamespaces(catalog).filter { ns => + val quoted = ns.map(quoteIfNeeded).mkString(".") + schemaPattern.r.pattern.matcher(quoted).matches() + }.map(_.toList).toList.distinct.map(_.toArray).toArray + } + + private def getSchemasWithPattern(catalog: CatalogPlugin, schemaPattern: String): Seq[String] = { + val p = schemaPattern.r.pattern + listAllNamespaces(catalog).flatMap { ns => + val quoted = ns.map(quoteIfNeeded).mkString(".") + if (p.matcher(quoted).matches()) Some(quoted) else None + }.distinct + } + + // /////////////////////////////////////////////////////////////////////////////////////////////// + // Table & View // + // /////////////////////////////////////////////////////////////////////////////////////////////// + + def getCatalogTablesOrViews( + spark: SparkSession, + catalogName: String, + schemaPattern: String, + tablePattern: String, + tableTypes: Set[String], + ignoreTableProperties: Boolean = false): Seq[Row] = { + val catalog = getCatalog(spark, catalogName) + val namespaces = listNamespacesWithPattern(catalog, schemaPattern) + catalog match { + case builtin if builtin.name() == SESSION_CATALOG => + val catalog = spark.sessionState.catalog + val databases = catalog.listDatabases(schemaPattern) + + def isMatchedTableType(tableTypes: Set[String], tableType: String): Boolean = { + val typ = if (tableType.equalsIgnoreCase(VIEW)) VIEW else TABLE + tableTypes.exists(typ.equalsIgnoreCase) + } + + databases.flatMap { db => + val identifiers = catalog.listTables(db, tablePattern, includeLocalTempViews = false) + catalog.getTablesByName(identifiers) + .filter(t => isMatchedTableType(tableTypes, t.tableType.name)).map { t => + val typ = if (t.tableType.name == VIEW) VIEW else TABLE + Row( + catalogName, + t.database, + t.identifier.table, + typ, + t.comment.getOrElse(""), + null, + null, + null, + null, + null) + } + } + case tc: TableCatalog => + val tp = tablePattern.r.pattern + val identifiers = namespaces.flatMap { ns => + tc.listTables(ns).filter(i => tp.matcher(quoteIfNeeded(i.name())).matches()) + } + identifiers.map { ident => + // TODO: restore view type for session catalog + val comment = if (ignoreTableProperties) "" + else { // load table is a time consuming operation + tc.loadTable(ident).properties().getOrDefault(TableCatalog.PROP_COMMENT, "") + } + val schema = ident.namespace().map(quoteIfNeeded).mkString(".") + val tableName = quoteIfNeeded(ident.name()) + Row(catalog.name(), schema, tableName, TABLE, comment, null, null, null, null, null) + } + case _ => Seq.empty[Row] + } + } + + private def getColumnsByCatalog( + spark: SparkSession, + catalogName: String, + schemaPattern: String, + tablePattern: String, + columnPattern: Pattern): Seq[Row] = { + val catalog = getCatalog(spark, catalogName) + + catalog match { + case tc: TableCatalog => + val namespaces = listNamespacesWithPattern(catalog, schemaPattern) + val tp = tablePattern.r.pattern + val identifiers = namespaces.flatMap { ns => + tc.listTables(ns).filter(i => tp.matcher(quoteIfNeeded(i.name())).matches()) + } + identifiers.flatMap { ident => + val table = tc.loadTable(ident) + val namespace = ident.namespace().map(quoteIfNeeded).mkString(".") + val tableName = quoteIfNeeded(ident.name()) + + table.schema.zipWithIndex.filter(f => columnPattern.matcher(f._1.name).matches()) + .map { case (f, i) => toColumnResult(tc.name(), namespace, tableName, f, i) } + } + + case builtin if builtin.name() == SESSION_CATALOG => + val catalog = spark.sessionState.catalog + val databases = catalog.listDatabases(schemaPattern) + databases.flatMap { db => + val identifiers = catalog.listTables(db, tablePattern, includeLocalTempViews = true) + catalog.getTablesByName(identifiers).flatMap { t => + t.schema.zipWithIndex.filter(f => columnPattern.matcher(f._1.name).matches()) + .map { case (f, i) => + toColumnResult(catalogName, t.database, t.identifier.table, f, i) + } + } + } + } + } + + def getTempViews( + spark: SparkSession, + catalogName: String, + schemaPattern: String, + tablePattern: String): Seq[Row] = { + val views = getViews(spark, schemaPattern, tablePattern) + views.map { ident => + Row(catalogName, ident.database.orNull, ident.table, VIEW, "", null, null, null, null, null) + } + } + + private def getViews( + spark: SparkSession, + schemaPattern: String, + tablePattern: String): Seq[TableIdentifier] = { + val db = getGlobalTempViewManager(spark, schemaPattern) + if (db.nonEmpty) { + spark.sessionState.catalog.listTables(db.head, tablePattern) + } else { + spark.sessionState.catalog.listLocalTempViews(tablePattern) + } + } + + // /////////////////////////////////////////////////////////////////////////////////////////////// + // Columns // + // /////////////////////////////////////////////////////////////////////////////////////////////// + + def getColumns( + spark: SparkSession, + catalogName: String, + schemaPattern: String, + tablePattern: String, + columnPattern: String): Seq[Row] = { + + val cp = columnPattern.r.pattern + val byCatalog = getColumnsByCatalog(spark, catalogName, schemaPattern, tablePattern, cp) + val byGlobalTmpDB = getColumnsByGlobalTempViewManager(spark, schemaPattern, tablePattern, cp) + val byLocalTmp = getColumnsByLocalTempViews(spark, tablePattern, cp) + + byCatalog ++ byGlobalTmpDB ++ byLocalTmp + } + + private def getColumnsByGlobalTempViewManager( + spark: SparkSession, + schemaPattern: String, + tablePattern: String, + columnPattern: Pattern): Seq[Row] = { + val catalog = spark.sessionState.catalog + + getGlobalTempViewManager(spark, schemaPattern).flatMap { globalTmpDb => + catalog.globalTempViewManager.listViewNames(tablePattern).flatMap { v => + catalog.globalTempViewManager.get(v).map { plan => + plan.schema.zipWithIndex.filter(f => columnPattern.matcher(f._1.name).matches()) + .map { case (f, i) => + toColumnResult(SparkCatalogUtils.SESSION_CATALOG, globalTmpDb, v, f, i) + } + } + }.flatten + } + } + + private def getColumnsByLocalTempViews( + spark: SparkSession, + tablePattern: String, + columnPattern: Pattern): Seq[Row] = { + val catalog = spark.sessionState.catalog + + catalog.listLocalTempViews(tablePattern) + .map(v => (v, catalog.getTempView(v.table).get)) + .flatMap { case (v, plan) => + plan.schema.zipWithIndex + .filter(f => columnPattern.matcher(f._1.name).matches()) + .map { case (f, i) => + toColumnResult(SparkCatalogUtils.SESSION_CATALOG, null, v.table, f, i) + } + } + } + + private def toColumnResult( + catalog: String, + db: String, + table: String, + col: StructField, + pos: Int): Row = { + // format: off + Row( + catalog, // TABLE_CAT + db, // TABLE_SCHEM + table, // TABLE_NAME + col.name, // COLUMN_NAME + SchemaHelper.toJavaSQLType(col.dataType), // DATA_TYPE + col.dataType.sql, // TYPE_NAME + SchemaHelper.getColumnSize(col.dataType).orNull, // COLUMN_SIZE + null, // BUFFER_LENGTH + SchemaHelper.getDecimalDigits(col.dataType).orNull, // DECIMAL_DIGITS + SchemaHelper.getNumPrecRadix(col.dataType).orNull, // NUM_PREC_RADIX + if (col.nullable) 1 else 0, // NULLABLE + col.getComment().getOrElse(""), // REMARKS + null, // COLUMN_DEF + null, // SQL_DATA_TYPE + null, // SQL_DATETIME_SUB + null, // CHAR_OCTET_LENGTH + pos, // ORDINAL_POSITION + "YES", // IS_NULLABLE + null, // SCOPE_CATALOG + null, // SCOPE_SCHEMA + null, // SCOPE_TABLE + null, // SOURCE_DATA_TYPE + "NO" // IS_AUTO_INCREMENT + ) + // format: on + } + + /** + * Forked from Apache Spark's [[org.apache.spark.sql.catalyst.util.quoteIfNeeded]] + */ + def quoteIfNeeded(part: String): String = { + if (part.matches("[a-zA-Z0-9_]+") && !part.matches("\\d+")) { + part + } else { + s"`${part.replace("`", "``")}`" + } + } +} diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/api/python/KyuubiPythonGatewayServer.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/api/python/KyuubiPythonGatewayServer.scala index 7e15ffe05..8cf8d685c 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/api/python/KyuubiPythonGatewayServer.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/api/python/KyuubiPythonGatewayServer.scala @@ -30,10 +30,12 @@ object KyuubiPythonGatewayServer extends Logging { val CONNECTION_FILE_PATH = Utils.createTempDir() + "/connection.info" - def start(): Unit = { + private var gatewayServer: Py4JServer = _ + + def start(): Unit = synchronized { val sparkConf = new SparkConf() - val gatewayServer: Py4JServer = new Py4JServer(sparkConf) + gatewayServer = new Py4JServer(sparkConf) gatewayServer.start() val boundPort: Int = gatewayServer.getListeningPort @@ -65,4 +67,11 @@ object KyuubiPythonGatewayServer extends Logging { System.exit(1) } } + + def shutdown(): Unit = synchronized { + if (gatewayServer != null) { + logInfo("shutting down the python gateway server.") + gatewayServer.shutdown() + } + } } diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SQLOperationListener.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SQLOperationListener.scala index 1a57fcf29..686cb1f35 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SQLOperationListener.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SQLOperationListener.scala @@ -20,6 +20,8 @@ package org.apache.spark.kyuubi import java.util.Properties import java.util.concurrent.ConcurrentHashMap +import scala.collection.JavaConverters._ + import org.apache.spark.scheduler._ import org.apache.spark.sql.SparkSession import org.apache.spark.sql.execution.ui.SparkListenerSQLExecutionEnd @@ -44,8 +46,8 @@ class SQLOperationListener( spark: SparkSession) extends StatsReportListener with Logging { private val operationId: String = operation.getHandle.identifier.toString - private lazy val activeJobs = new java.util.HashSet[Int]() - private lazy val activeStages = new ConcurrentHashMap[StageAttempt, StageInfo]() + private lazy val activeJobs = new ConcurrentHashMap[Int, SparkJobInfo]() + private lazy val activeStages = new ConcurrentHashMap[SparkStageAttempt, SparkStageInfo]() private var executionId: Option[Long] = None private val conf: KyuubiConf = operation.getSession.sessionManager.getConf @@ -53,6 +55,7 @@ class SQLOperationListener( if (conf.get(ENGINE_SPARK_SHOW_PROGRESS)) { Some(new SparkConsoleProgressBar( operation, + activeJobs, activeStages, conf.get(ENGINE_SPARK_SHOW_PROGRESS_UPDATE_INTERVAL), conf.get(ENGINE_SPARK_SHOW_PROGRESS_TIME_FORMAT))) @@ -79,9 +82,10 @@ class SQLOperationListener( } } - override def onJobStart(jobStart: SparkListenerJobStart): Unit = activeJobs.synchronized { + override def onJobStart(jobStart: SparkListenerJobStart): Unit = { if (sameGroupId(jobStart.properties)) { val jobId = jobStart.jobId + val stageIds = jobStart.stageInfos.map(_.stageId).toSet val stageSize = jobStart.stageInfos.size if (executionId.isEmpty) { executionId = Option(jobStart.properties.getProperty(SPARK_SQL_EXECUTION_ID_KEY)) @@ -93,17 +97,19 @@ class SQLOperationListener( case _ => } } + activeJobs.put( + jobId, + new SparkJobInfo(stageSize, stageIds)) withOperationLog { - activeJobs.add(jobId) info(s"Query [$operationId]: Job $jobId started with $stageSize stages," + s" ${activeJobs.size()} active jobs running") } } } - override def onJobEnd(jobEnd: SparkListenerJobEnd): Unit = activeJobs.synchronized { + override def onJobEnd(jobEnd: SparkListenerJobEnd): Unit = { val jobId = jobEnd.jobId - if (activeJobs.remove(jobId)) { + if (activeJobs.remove(jobId) != null) { val hint = jobEnd.jobResult match { case JobSucceeded => "succeeded" case _ => "failed" // TODO: Handle JobFailed(exception: Exception) @@ -120,10 +126,10 @@ class SQLOperationListener( val stageInfo = stageSubmitted.stageInfo val stageId = stageInfo.stageId val attemptNumber = stageInfo.attemptNumber() - val stageAttempt = StageAttempt(stageId, attemptNumber) + val stageAttempt = SparkStageAttempt(stageId, attemptNumber) activeStages.put( stageAttempt, - new StageInfo(stageId, stageInfo.numTasks)) + new SparkStageInfo(stageId, stageInfo.numTasks)) withOperationLog { info(s"Query [$operationId]: Stage $stageId.$attemptNumber started " + s"with ${stageInfo.numTasks} tasks, ${activeStages.size()} active stages running") @@ -134,28 +140,37 @@ class SQLOperationListener( override def onStageCompleted(stageCompleted: SparkListenerStageCompleted): Unit = { val stageInfo = stageCompleted.stageInfo - val stageAttempt = StageAttempt(stageInfo.stageId, stageInfo.attemptNumber()) + val stageId = stageInfo.stageId + val stageAttempt = SparkStageAttempt(stageInfo.stageId, stageInfo.attemptNumber()) activeStages.synchronized { if (activeStages.remove(stageAttempt) != null) { + stageInfo.getStatusString match { + case "succeeded" => + activeJobs.asScala.foreach { case (_, jobInfo) => + if (jobInfo.stageIds.contains(stageId)) { + jobInfo.numCompleteStages.getAndIncrement() + } + } + } withOperationLog(super.onStageCompleted(stageCompleted)) } } } override def onTaskStart(taskStart: SparkListenerTaskStart): Unit = activeStages.synchronized { - val stageAttempt = StageAttempt(taskStart.stageId, taskStart.stageAttemptId) + val stageAttempt = SparkStageAttempt(taskStart.stageId, taskStart.stageAttemptId) if (activeStages.containsKey(stageAttempt)) { - activeStages.get(stageAttempt).numActiveTasks += 1 + activeStages.get(stageAttempt).numActiveTasks.getAndIncrement() super.onTaskStart(taskStart) } } override def onTaskEnd(taskEnd: SparkListenerTaskEnd): Unit = activeStages.synchronized { - val stageAttempt = StageAttempt(taskEnd.stageId, taskEnd.stageAttemptId) + val stageAttempt = SparkStageAttempt(taskEnd.stageId, taskEnd.stageAttemptId) if (activeStages.containsKey(stageAttempt)) { - activeStages.get(stageAttempt).numActiveTasks -= 1 + activeStages.get(stageAttempt).numActiveTasks.getAndDecrement() if (taskEnd.reason == org.apache.spark.Success) { - activeStages.get(stageAttempt).numCompleteTasks += 1 + activeStages.get(stageAttempt).numCompleteTasks.getAndIncrement() } super.onTaskEnd(taskEnd) } diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SparkConsoleProgressBar.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SparkConsoleProgressBar.scala index fc2ebd5f8..feb0d16a1 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SparkConsoleProgressBar.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SparkConsoleProgressBar.scala @@ -29,7 +29,8 @@ import org.apache.kyuubi.operation.Operation class SparkConsoleProgressBar( operation: Operation, - liveStages: ConcurrentHashMap[StageAttempt, StageInfo], + liveJobs: ConcurrentHashMap[Int, SparkJobInfo], + liveStages: ConcurrentHashMap[SparkStageAttempt, SparkStageInfo], updatePeriodMSec: Long, timeFormat: String) extends Logging { @@ -72,21 +73,36 @@ class SparkConsoleProgressBar( } } + /** + * Use stageId to find stage's jobId + * @param stageId + * @return jobId (Optional) + */ + private def findJobId(stageId: Int): Option[Int] = { + liveJobs.asScala.collectFirst { + case (jobId, jobInfo) if jobInfo.stageIds.contains(stageId) => jobId + } + } + /** * Show progress bar in console. The progress bar is displayed in the next line * after your last output, keeps overwriting itself to hold in one line. The logging will follow * the progress bar, then progress bar will be showed in next line without overwrite logs. */ - private def show(now: Long, stages: Seq[StageInfo]): Unit = { + private def show(now: Long, stages: Seq[SparkStageInfo]): Unit = { val width = TerminalWidth / stages.size val bar = stages.map { s => val total = s.numTasks - val header = s"[Stage ${s.stageId}:" + val jobHeader = findJobId(s.stageId).map(jobId => + s"[Job $jobId (${liveJobs.get(jobId).numCompleteStages} " + + s"/ ${liveJobs.get(jobId).numStages}) Stages] ").getOrElse( + "[There is no job about this stage] ") + val header = jobHeader + s"[Stage ${s.stageId}:" val tailer = s"(${s.numCompleteTasks} + ${s.numActiveTasks}) / $total]" - val w = width - header.length - tailer.length + val w = width + jobHeader.length - header.length - tailer.length val bar = if (w > 0) { - val percent = w * s.numCompleteTasks / total + val percent = w * s.numCompleteTasks.get / total (0 until w).map { i => if (i < percent) "=" else if (i == percent) ">" else " " }.mkString("") diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SparkProgressMonitor.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SparkProgressMonitor.scala index a46cbecc2..1d9ef53ea 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SparkProgressMonitor.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SparkProgressMonitor.scala @@ -136,12 +136,8 @@ class SparkProgressMonitor(spark: SparkSession, jobGroup: String) { trimmedVName = s.substring(0, COLUMN_1_WIDTH - 2) trimmedVName += ".." } else trimmedVName += " " - val result = new StringBuilder(trimmedVName) val toFill = (spaceRemaining * percent).toInt - for (i <- 0 until toFill) { - result.append(".") - } - result.toString + s"$trimmedVName${"." * toFill}" } private def getCompletedStages: Int = { diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SparkSQLEngineListener.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SparkSQLEngineListener.scala index 8e32b5329..48f157a43 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SparkSQLEngineListener.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SparkSQLEngineListener.scala @@ -40,9 +40,9 @@ import org.apache.kyuubi.service.{Serverable, ServiceState} class SparkSQLEngineListener(server: Serverable) extends SparkListener with Logging { // the conf of server is null before initialized, use lazy val here - private lazy val deregisterExceptions: Seq[String] = + private lazy val deregisterExceptions: Set[String] = server.getConf.get(ENGINE_DEREGISTER_EXCEPTION_CLASSES) - private lazy val deregisterMessages: Seq[String] = + private lazy val deregisterMessages: Set[String] = server.getConf.get(ENGINE_DEREGISTER_EXCEPTION_MESSAGES) private lazy val deregisterExceptionTTL: Long = server.getConf.get(ENGINE_DEREGISTER_EXCEPTION_TTL) @@ -74,7 +74,7 @@ class SparkSQLEngineListener(server: Serverable) extends SparkListener with Logg case JobFailed(e) if e != null => val cause = findCause(e) var deregisterInfo: Option[String] = None - if (deregisterExceptions.exists(_.equals(cause.getClass.getCanonicalName))) { + if (deregisterExceptions.contains(cause.getClass.getCanonicalName)) { deregisterInfo = Some("Job failed exception class is in the set of " + s"${ENGINE_DEREGISTER_EXCEPTION_CLASSES.key}, deregistering the engine.") } else if (deregisterMessages.exists(stringifyException(cause).contains)) { diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SparkUtilsHelper.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SparkUtilsHelper.scala index e2f51e648..106be3fc7 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SparkUtilsHelper.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/SparkUtilsHelper.scala @@ -43,4 +43,13 @@ object SparkUtilsHelper extends Logging { def getLocalDir(conf: SparkConf): String = { Utils.getLocalDir(conf) } + + def classesArePresent(className: String): Boolean = { + try { + Utils.classForName(className) + true + } catch { + case _: ClassNotFoundException | _: NoClassDefFoundError => false + } + } } diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/StageStatus.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/StageStatus.scala index 144570862..29644f9f4 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/StageStatus.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/kyuubi/StageStatus.scala @@ -17,11 +17,17 @@ package org.apache.spark.kyuubi -case class StageAttempt(stageId: Int, stageAttemptId: Int) { +import java.util.concurrent.atomic.AtomicInteger + +case class SparkStageAttempt(stageId: Int, stageAttemptId: Int) { override def toString: String = s"Stage $stageId (Attempt $stageAttemptId)" } -class StageInfo(val stageId: Int, val numTasks: Int) { - var numActiveTasks = 0 - var numCompleteTasks = 0 +class SparkStageInfo(val stageId: Int, val numTasks: Int) { + val numActiveTasks = new AtomicInteger(0) + val numCompleteTasks = new AtomicInteger(0) +} + +class SparkJobInfo(val numStages: Int, val stageIds: Set[Int]) { + val numCompleteStages = new AtomicInteger(0) } diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/sql/execution/arrow/KyuubiArrowConverters.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/sql/execution/arrow/KyuubiArrowConverters.scala new file mode 100644 index 000000000..5c4d7086f --- /dev/null +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/sql/execution/arrow/KyuubiArrowConverters.scala @@ -0,0 +1,352 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.arrow + +import java.io.{ByteArrayInputStream, ByteArrayOutputStream} +import java.lang.{Boolean => JBoolean} +import java.nio.channels.Channels + +import scala.collection.JavaConverters._ +import scala.collection.mutable.ArrayBuffer + +import org.apache.arrow.vector._ +import org.apache.arrow.vector.ipc.{ArrowStreamWriter, ReadChannel, WriteChannel} +import org.apache.arrow.vector.ipc.message.{IpcOption, MessageSerializer} +import org.apache.arrow.vector.types.pojo.{Schema => ArrowSchema} +import org.apache.spark.TaskContext +import org.apache.spark.internal.Logging +import org.apache.spark.sql.SparkSession +import org.apache.spark.sql.catalyst.{InternalRow, SQLConfHelper} +import org.apache.spark.sql.catalyst.expressions.UnsafeRow +import org.apache.spark.sql.execution.CollectLimitExec +import org.apache.spark.sql.types._ +import org.apache.spark.sql.util.ArrowUtils +import org.apache.spark.util.Utils + +import org.apache.kyuubi.util.reflect.DynMethods + +object KyuubiArrowConverters extends SQLConfHelper with Logging { + + type Batch = (Array[Byte], Long) + + /** + * this method is to slice the input Arrow record batch byte array `bytes`, starting from `start` + * and taking `length` number of elements. + */ + def slice( + schema: StructType, + timeZoneId: String, + bytes: Array[Byte], + start: Int, + length: Int): Array[Byte] = { + val in = new ByteArrayInputStream(bytes) + val out = new ByteArrayOutputStream(bytes.length) + + var vectorSchemaRoot: VectorSchemaRoot = null + var slicedVectorSchemaRoot: VectorSchemaRoot = null + + val sliceAllocator = ArrowUtils.rootAllocator.newChildAllocator( + "slice", + 0, + Long.MaxValue) + val arrowSchema = toArrowSchema(schema, timeZoneId, true, false) + vectorSchemaRoot = VectorSchemaRoot.create(arrowSchema, sliceAllocator) + try { + val recordBatch = MessageSerializer.deserializeRecordBatch( + new ReadChannel(Channels.newChannel(in)), + sliceAllocator) + val vectorLoader = new VectorLoader(vectorSchemaRoot) + vectorLoader.load(recordBatch) + recordBatch.close() + slicedVectorSchemaRoot = vectorSchemaRoot.slice(start, length) + + val unloader = new VectorUnloader(slicedVectorSchemaRoot) + val writeChannel = new WriteChannel(Channels.newChannel(out)) + val batch = unloader.getRecordBatch() + MessageSerializer.serialize(writeChannel, batch) + batch.close() + out.toByteArray() + } finally { + in.close() + out.close() + if (vectorSchemaRoot != null) { + vectorSchemaRoot.getFieldVectors.asScala.foreach(_.close()) + vectorSchemaRoot.close() + } + if (slicedVectorSchemaRoot != null) { + slicedVectorSchemaRoot.getFieldVectors.asScala.foreach(_.close()) + slicedVectorSchemaRoot.close() + } + sliceAllocator.close() + } + } + + /** + * Forked from `org.apache.spark.sql.execution.SparkPlan#executeTake()`, the algorithm can be + * summarized in the following steps: + * 1. If the limit specified in the CollectLimitExec object is 0, the function returns an empty + * array of batches. + * 2. Otherwise, execute the child query plan of the CollectLimitExec object to obtain an RDD of + * data to collect. + * 3. Use an iterative approach to collect data in batches until the specified limit is reached. + * In each iteration, it selects a subset of the partitions of the RDD to scan and tries to + * collect data from them. + * 4. For each partition subset, we use the runJob method of the Spark context to execute a + * closure that scans the partition data and converts it to Arrow batches. + * 5. Check if the collected data reaches the specified limit. If not, it selects another subset + * of partitions to scan and repeats the process until the limit is reached or all partitions + * have been scanned. + * 6. Return an array of all the collected Arrow batches. + * + * Note that: + * 1. The returned Arrow batches row count >= limit, if the input df has more than the `limit` + * row count + * 2. We don't implement the `takeFromEnd` logical + * + * @return + */ + def takeAsArrowBatches( + collectLimitExec: CollectLimitExec, + maxRecordsPerBatch: Long, + maxEstimatedBatchSize: Long, + timeZoneId: String): Array[Batch] = { + val n = collectLimitExec.limit + val schema = collectLimitExec.schema + if (n == 0) { + new Array[Batch](0) + } else { + val limitScaleUpFactor = Math.max(conf.limitScaleUpFactor, 2) + // TODO: refactor and reuse the code from RDD's take() + val childRDD = collectLimitExec.child.execute() + val buf = new ArrayBuffer[Batch] + var bufferedRowSize = 0L + val totalParts = childRDD.partitions.length + var partsScanned = 0 + while (bufferedRowSize < n && partsScanned < totalParts) { + // The number of partitions to try in this iteration. It is ok for this number to be + // greater than totalParts because we actually cap it at totalParts in runJob. + var numPartsToTry = limitInitialNumPartitions + if (partsScanned > 0) { + // If we didn't find any rows after the previous iteration, multiply by + // limitScaleUpFactor and retry. Otherwise, interpolate the number of partitions we need + // to try, but overestimate it by 50%. We also cap the estimation in the end. + if (buf.isEmpty) { + numPartsToTry = partsScanned * limitScaleUpFactor + } else { + val left = n - bufferedRowSize + // As left > 0, numPartsToTry is always >= 1 + numPartsToTry = Math.ceil(1.5 * left * partsScanned / bufferedRowSize).toInt + numPartsToTry = Math.min(numPartsToTry, partsScanned * limitScaleUpFactor) + } + } + + val partsToScan = + partsScanned.until(math.min(partsScanned + numPartsToTry, totalParts)) + + // TODO: SparkPlan.session introduced in SPARK-35798, replace with SparkPlan.session once we + // drop Spark-3.1.x support. + val sc = SparkSession.active.sparkContext + val res = sc.runJob( + childRDD, + (it: Iterator[InternalRow]) => { + val batches = toBatchIterator( + it, + schema, + maxRecordsPerBatch, + maxEstimatedBatchSize, + n, + timeZoneId) + batches.map(b => b -> batches.rowCountInLastBatch).toArray + }, + partsToScan) + + var i = 0 + while (bufferedRowSize < n && i < res.length) { + var j = 0 + val batches = res(i) + while (j < batches.length && n > bufferedRowSize) { + val batch = batches(j) + val (_, batchSize) = batch + buf += batch + bufferedRowSize += batchSize + j += 1 + } + i += 1 + } + partsScanned += partsToScan.size + } + + buf.toArray + } + } + + /** + * Spark introduced the config `spark.sql.limit.initialNumPartitions` since 3.4.0. see SPARK-40211 + */ + private def limitInitialNumPartitions: Int = { + conf.getConfString("spark.sql.limit.initialNumPartitions", "1") + .toInt + } + + /** + * Different from [[org.apache.spark.sql.execution.arrow.ArrowConverters.toBatchIterator]], + * each output arrow batch contains this batch row count. + */ + def toBatchIterator( + rowIter: Iterator[InternalRow], + schema: StructType, + maxRecordsPerBatch: Long, + maxEstimatedBatchSize: Long, + limit: Long, + timeZoneId: String): ArrowBatchIterator = { + new ArrowBatchIterator( + rowIter, + schema, + maxRecordsPerBatch, + maxEstimatedBatchSize, + limit, + timeZoneId, + TaskContext.get) + } + + /** + * This class ArrowBatchIterator is derived from + * [[org.apache.spark.sql.execution.arrow.ArrowConverters.ArrowBatchWithSchemaIterator]], + * with two key differences: + * 1. there is no requirement to write the schema at the batch header + * 2. iteration halts when `rowCount` equals `limit` + * Note that `limit < 0` means no limit, and return all rows the in the iterator. + */ + private[sql] class ArrowBatchIterator( + rowIter: Iterator[InternalRow], + schema: StructType, + maxRecordsPerBatch: Long, + maxEstimatedBatchSize: Long, + limit: Long, + timeZoneId: String, + context: TaskContext) + extends Iterator[Array[Byte]] { + + protected val arrowSchema = toArrowSchema(schema, timeZoneId, true, false) + private val allocator = + ArrowUtils.rootAllocator.newChildAllocator( + s"to${this.getClass.getSimpleName}", + 0, + Long.MaxValue) + + private val root = VectorSchemaRoot.create(arrowSchema, allocator) + protected val unloader = new VectorUnloader(root) + protected val arrowWriter = ArrowWriter.create(root) + + Option(context).foreach { + _.addTaskCompletionListener[Unit] { _ => + root.close() + allocator.close() + } + } + + override def hasNext: Boolean = (rowIter.hasNext && (rowCount < limit || limit < 0)) || { + root.close() + allocator.close() + false + } + + var rowCountInLastBatch: Long = 0 + var rowCount: Long = 0 + + override def next(): Array[Byte] = { + val out = new ByteArrayOutputStream() + val writeChannel = new WriteChannel(Channels.newChannel(out)) + + rowCountInLastBatch = 0 + var estimatedBatchSize = 0L + Utils.tryWithSafeFinally { + + // Always write the first row. + while (rowIter.hasNext && ( + // For maxBatchSize and maxRecordsPerBatch, respect whatever smaller. + // If the size in bytes is positive (set properly), always write the first row. + rowCountInLastBatch == 0 && maxEstimatedBatchSize > 0 || + // If the size in bytes of rows are 0 or negative, unlimit it. + estimatedBatchSize <= 0 || + estimatedBatchSize < maxEstimatedBatchSize || + // If the size of rows are 0 or negative, unlimit it. + maxRecordsPerBatch <= 0 || + rowCountInLastBatch < maxRecordsPerBatch || + rowCount < limit || + limit < 0)) { + val row = rowIter.next() + arrowWriter.write(row) + estimatedBatchSize += (row match { + case ur: UnsafeRow => ur.getSizeInBytes + // Trying to estimate the size of the current row + case _: InternalRow => schema.defaultSize + }) + rowCountInLastBatch += 1 + rowCount += 1 + } + arrowWriter.finish() + val batch = unloader.getRecordBatch() + MessageSerializer.serialize(writeChannel, batch) + + // Always write the Ipc options at the end. + ArrowStreamWriter.writeEndOfStream(writeChannel, ARROW_IPC_OPTION_DEFAULT) + + batch.close() + } { + arrowWriter.reset() + } + + out.toByteArray + } + } + + // the signature of function [[ArrowUtils.toArrowSchema]] is changed in SPARK-41971 (since Spark + // 3.5) + private lazy val toArrowSchemaMethod = DynMethods.builder("toArrowSchema") + .impl( // for Spark 3.4 or previous + "org.apache.spark.sql.util.ArrowUtils", + classOf[StructType], + classOf[String]) + .impl( // for Spark 3.5 or later + "org.apache.spark.sql.util.ArrowUtils", + classOf[StructType], + classOf[String], + classOf[Boolean], + classOf[Boolean]) + .build() + + /** + * this function uses reflective calls to the [[ArrowUtils.toArrowSchema]]. + */ + private def toArrowSchema( + schema: StructType, + timeZone: String, + errorOnDuplicatedFieldNames: JBoolean, + largeVarTypes: JBoolean): ArrowSchema = { + toArrowSchemaMethod.invoke[ArrowSchema]( + ArrowUtils, + schema, + timeZone, + errorOnDuplicatedFieldNames, + largeVarTypes) + } + + // IpcOption.DEFAULT was introduced in ARROW-11081(ARROW-4.0.0), add this for adapt Spark-3.1/3.2 + final private val ARROW_IPC_OPTION_DEFAULT = new IpcOption() +} diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/sql/kyuubi/SparkDatasetHelper.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/sql/kyuubi/SparkDatasetHelper.scala index 23f7df213..c0f9d61c2 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/sql/kyuubi/SparkDatasetHelper.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/sql/kyuubi/SparkDatasetHelper.scala @@ -17,35 +17,105 @@ package org.apache.spark.sql.kyuubi -import java.time.ZoneId +import scala.collection.mutable.ArrayBuffer +import org.apache.spark.SparkContext +import org.apache.spark.internal.Logging +import org.apache.spark.network.util.{ByteUnit, JavaUtils} import org.apache.spark.rdd.RDD -import org.apache.spark.sql.{DataFrame, Dataset, Row} +import org.apache.spark.sql.{DataFrame, Dataset, Row, SparkSession} +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.execution.{CollectLimitExec, LocalTableScanExec, SparkPlan, SQLExecution} +import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec +import org.apache.spark.sql.execution.arrow.KyuubiArrowConverters +import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics} import org.apache.spark.sql.functions._ -import org.apache.spark.sql.types.{ArrayType, DataType, MapType, StructField, StructType} +import org.apache.spark.sql.types._ +import org.apache.kyuubi.engine.spark.KyuubiSparkUtil import org.apache.kyuubi.engine.spark.schema.RowSet +import org.apache.kyuubi.engine.spark.util.SparkCatalogUtils.quoteIfNeeded +import org.apache.kyuubi.util.reflect.DynMethods +import org.apache.kyuubi.util.reflect.ReflectUtils._ + +object SparkDatasetHelper extends Logging { + + def executeCollect(df: DataFrame): Array[Array[Byte]] = withNewExecutionId(df) { + executeArrowBatchCollect(df.queryExecution.executedPlan) + } + + def executeArrowBatchCollect: SparkPlan => Array[Array[Byte]] = { + case adaptiveSparkPlan: AdaptiveSparkPlanExec => + executeArrowBatchCollect(finalPhysicalPlan(adaptiveSparkPlan)) + // TODO: avoid extra shuffle if `offset` > 0 + case collectLimit: CollectLimitExec if offset(collectLimit) > 0 => + logWarning("unsupported offset > 0, an extra shuffle will be introduced.") + toArrowBatchRdd(collectLimit).collect() + case collectLimit: CollectLimitExec if collectLimit.limit >= 0 => + doCollectLimit(collectLimit) + case collectLimit: CollectLimitExec if collectLimit.limit < 0 => + executeArrowBatchCollect(collectLimit.child) + // TODO: replace with pattern match once we drop Spark 3.1 support. + case command: SparkPlan if isCommandResultExec(command) => + doCommandResultExec(command) + case localTableScan: LocalTableScanExec => + doLocalTableScan(localTableScan) + case plan: SparkPlan => + toArrowBatchRdd(plan).collect() + } -object SparkDatasetHelper { def toArrowBatchRdd[T](ds: Dataset[T]): RDD[Array[Byte]] = { ds.toArrowBatchRdd } - def convertTopLevelComplexTypeToHiveString(df: DataFrame): DataFrame = { - val timeZone = ZoneId.of(df.sparkSession.sessionState.conf.sessionLocalTimeZone) + /** + * Forked from [[Dataset.toArrowBatchRdd(plan: SparkPlan)]]. + * Convert to an RDD of serialized ArrowRecordBatches. + */ + def toArrowBatchRdd(plan: SparkPlan): RDD[Array[Byte]] = { + val schemaCaptured = plan.schema + // TODO: SparkPlan.session introduced in SPARK-35798, replace with SparkPlan.session once we + // drop Spark 3.1 support. + val maxRecordsPerBatch = SparkSession.active.sessionState.conf.arrowMaxRecordsPerBatch + val timeZoneId = SparkSession.active.sessionState.conf.sessionLocalTimeZone + // note that, we can't pass the lazy variable `maxBatchSize` directly, this is because input + // arguments are serialized and sent to the executor side for execution. + val maxBatchSizePerBatch = maxBatchSize + plan.execute().mapPartitionsInternal { iter => + KyuubiArrowConverters.toBatchIterator( + iter, + schemaCaptured, + maxRecordsPerBatch, + maxBatchSizePerBatch, + -1, + timeZoneId) + } + } + + def toArrowBatchLocalIterator(df: DataFrame): Iterator[Array[Byte]] = { + withNewExecutionId(df) { + toArrowBatchRdd(df).toLocalIterator + } + } + + def convertTopLevelComplexTypeToHiveString( + df: DataFrame, + timestampAsString: Boolean): DataFrame = { val quotedCol = (name: String) => col(quoteIfNeeded(name)) - // an udf to call `RowSet.toHiveString` on complex types(struct/array/map). + // an udf to call `RowSet.toHiveString` on complex types(struct/array/map) and timestamp type. val toHiveStringUDF = udf[String, Row, String]((row, schemaDDL) => { val dt = DataType.fromDDL(schemaDDL) dt match { case StructType(Array(StructField(_, st: StructType, _, _))) => - RowSet.toHiveString((row, st), timeZone) + RowSet.toHiveString((row, st), nested = true) case StructType(Array(StructField(_, at: ArrayType, _, _))) => - RowSet.toHiveString((row.toSeq.head, at), timeZone) + RowSet.toHiveString((row.toSeq.head, at), nested = true) case StructType(Array(StructField(_, mt: MapType, _, _))) => - RowSet.toHiveString((row.toSeq.head, mt), timeZone) + RowSet.toHiveString((row.toSeq.head, mt), nested = true) + case StructType(Array(StructField(_, tt: TimestampType, _, _))) => + RowSet.toHiveString((row.toSeq.head, tt), nested = true) case _ => throw new UnsupportedOperationException } @@ -54,22 +124,158 @@ object SparkDatasetHelper { val cols = df.schema.map { case sf @ StructField(name, _: StructType, _, _) => toHiveStringUDF(quotedCol(name), lit(sf.toDDL)).as(name) - case sf @ StructField(name, (_: MapType | _: ArrayType), _, _) => + case sf @ StructField(name, _: MapType | _: ArrayType, _, _) => + toHiveStringUDF(struct(quotedCol(name)), lit(sf.toDDL)).as(name) + case sf @ StructField(name, _: TimestampType, _, _) if timestampAsString => toHiveStringUDF(struct(quotedCol(name)), lit(sf.toDDL)).as(name) case StructField(name, _, _, _) => quotedCol(name) } df.select(cols: _*) } + private lazy val maxBatchSize: Long = { + // respect spark connect config + KyuubiSparkUtil.globalSparkContext + .getConf + .getOption("spark.connect.grpc.arrow.maxBatchSize") + .orElse(Option("4m")) + .map(JavaUtils.byteStringAs(_, ByteUnit.MiB)) + .get + } + + private def doCollectLimit(collectLimit: CollectLimitExec): Array[Array[Byte]] = { + // TODO: SparkPlan.session introduced in SPARK-35798, replace with SparkPlan.session once we + // drop Spark-3.1.x support. + val timeZoneId = SparkSession.active.sessionState.conf.sessionLocalTimeZone + val maxRecordsPerBatch = SparkSession.active.sessionState.conf.arrowMaxRecordsPerBatch + + val batches = KyuubiArrowConverters.takeAsArrowBatches( + collectLimit, + maxRecordsPerBatch, + maxBatchSize, + timeZoneId) + + // note that the number of rows in the returned arrow batches may be >= `limit`, perform + // the slicing operation of result + val result = ArrayBuffer[Array[Byte]]() + var i = 0 + var rest = collectLimit.limit + while (i < batches.length && rest > 0) { + val (batch, size) = batches(i) + if (size <= rest) { + result += batch + // returned ArrowRecordBatch has less than `limit` row count, safety to do conversion + rest -= size.toInt + } else { // size > rest + result += KyuubiArrowConverters.slice(collectLimit.schema, timeZoneId, batch, 0, rest) + rest = 0 + } + i += 1 + } + result.toArray + } + + private lazy val commandResultExecRowsMethod = DynMethods.builder("rows") + .impl("org.apache.spark.sql.execution.CommandResultExec") + .build() + + private def doCommandResultExec(command: SparkPlan): Array[Array[Byte]] = { + val spark = SparkSession.active + // TODO: replace with `command.rows` once we drop Spark 3.1 support. + val rows = commandResultExecRowsMethod.invoke[Seq[InternalRow]](command) + command.longMetric("numOutputRows").add(rows.size) + sendDriverMetrics(spark.sparkContext, command.metrics) + KyuubiArrowConverters.toBatchIterator( + rows.iterator, + command.schema, + spark.sessionState.conf.arrowMaxRecordsPerBatch, + maxBatchSize, + -1, + spark.sessionState.conf.sessionLocalTimeZone).toArray + } + + private def doLocalTableScan(localTableScan: LocalTableScanExec): Array[Array[Byte]] = { + val spark = SparkSession.active + localTableScan.longMetric("numOutputRows").add(localTableScan.rows.size) + sendDriverMetrics(spark.sparkContext, localTableScan.metrics) + KyuubiArrowConverters.toBatchIterator( + localTableScan.rows.iterator, + localTableScan.schema, + spark.sessionState.conf.arrowMaxRecordsPerBatch, + maxBatchSize, + -1, + spark.sessionState.conf.sessionLocalTimeZone).toArray + } + /** - * Fork from Apache Spark-3.3.1 org.apache.spark.sql.catalyst.util.quoteIfNeeded to adapt to - * Spark-3.1.x + * This method provides a reflection-based implementation of + * [[AdaptiveSparkPlanExec.finalPhysicalPlan]] that enables us to adapt to the Spark runtime + * without patching SPARK-41914. + * + * TODO: Once we drop support for Spark 3.1.x, we can directly call + * [[AdaptiveSparkPlanExec.finalPhysicalPlan]]. */ - def quoteIfNeeded(part: String): String = { - if (part.matches("[a-zA-Z0-9_]+") && !part.matches("\\d+")) { - part - } else { - s"`${part.replace("`", "``")}`" + def finalPhysicalPlan(adaptiveSparkPlanExec: AdaptiveSparkPlanExec): SparkPlan = { + withFinalPlanUpdate(adaptiveSparkPlanExec, identity) + } + + /** + * A reflection-based implementation of [[AdaptiveSparkPlanExec.withFinalPlanUpdate]]. + */ + private def withFinalPlanUpdate[T]( + adaptiveSparkPlanExec: AdaptiveSparkPlanExec, + fun: SparkPlan => T): T = { + val plan = invokeAs[SparkPlan](adaptiveSparkPlanExec, "getFinalPhysicalPlan") + val result = fun(plan) + invokeAs[Unit](adaptiveSparkPlanExec, "finalPlanUpdate") + result + } + + /** + * offset support was add since Spark-3.4(set SPARK-28330), to ensure backward compatibility with + * earlier versions of Spark, this function uses reflective calls to the "offset". + */ + private def offset(collectLimitExec: CollectLimitExec): Int = { + Option( + DynMethods.builder("offset") + .impl(collectLimitExec.getClass) + .orNoop() + .build() + .invoke[Int](collectLimitExec)) + .getOrElse(0) + } + + private def isCommandResultExec(sparkPlan: SparkPlan): Boolean = { + // scalastyle:off line.size.limit + // the CommandResultExec was introduced in SPARK-35378 (Spark 3.2), after SPARK-35378 the + // physical plan of runnable command is CommandResultExec. + // for instance: + // ``` + // scala> spark.sql("show tables").queryExecution.executedPlan + // res0: org.apache.spark.sql.execution.SparkPlan = + // CommandResult , [namespace#0, tableName#1, isTemporary#2] + // +- ShowTables [namespace#0, tableName#1, isTemporary#2], V2SessionCatalog(spark_catalog), [default] + // + // scala > spark.sql("show tables").queryExecution.executedPlan.getClass + // res1: Class[_ <: org.apache.spark.sql.execution.SparkPlan] = class org.apache.spark.sql.execution.CommandResultExec + // ``` + // scalastyle:on line.size.limit + sparkPlan.getClass.getName == "org.apache.spark.sql.execution.CommandResultExec" + } + + /** + * refer to org.apache.spark.sql.Dataset#withAction(), assign a new execution id for arrow-based + * operation, so that we can track the arrow-based queries on the UI tab. + */ + private def withNewExecutionId[T](df: DataFrame)(body: => T): T = { + SQLExecution.withNewExecutionId(df.queryExecution, Some("collectAsArrow")) { + df.queryExecution.executedPlan.resetMetrics() + body } } + + private def sendDriverMetrics(sc: SparkContext, metrics: Map[String, SQLMetric]): Unit = { + val executionId = sc.getLocalProperty(SQLExecution.EXECUTION_ID_KEY) + SQLMetrics.postDriverMetricUpdates(sc, executionId, metrics.values.toSeq) + } } diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EnginePage.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EnginePage.scala index 0aba0c7c5..7188ac62f 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EnginePage.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EnginePage.scala @@ -29,7 +29,7 @@ import org.apache.commons.text.StringEscapeUtils import org.apache.spark.ui.TableSourceUtil._ import org.apache.spark.ui.UIUtils._ -import org.apache.kyuubi.{KYUUBI_VERSION, Utils} +import org.apache.kyuubi._ import org.apache.kyuubi.engine.spark.events.{SessionEvent, SparkOperationEvent} case class EnginePage(parent: EngineTab) extends WebUIPage("") { @@ -58,6 +58,15 @@ case class EnginePage(parent: EngineTab) extends WebUIPage("") { Kyuubi Version: {KYUUBI_VERSION} +
    • + Compilation Revision: + {REVISION.substring(0, 7)} ({REVISION_TIME}), branch {BRANCH} +
      • +
    • + Compilation with: + Spark {SPARK_COMPILE_VERSION}, Scala {SCALA_COMPILE_VERSION}, + Hadoop {HADOOP_COMPILE_VERSION}, Hive {HIVE_COMPILE_VERSION} +
    • Started at: {new Date(parent.startTime)} @@ -84,6 +93,10 @@ case class EnginePage(parent: EngineTab) extends WebUIPage("") { Background execution pool threads active: {engine.backendService.sessionManager.getActiveCount}
      • +
    • + Background execution pool work queue size: + {engine.backendService.sessionManager.getWorkQueueSize} +
      • }.getOrElse(Seq.empty) }
    @@ -288,7 +301,7 @@ case class EnginePage(parent: EngineTab) extends WebUIPage("") {
- + } @@ -382,7 +395,7 @@ private class StatementStatsPagedTable( {if (event.completeTime > 0) formatDate(event.completeTime)}
{session.name} {formatDate(session.startTime)} {if (session.endTime > 0) formatDate(session.endTime)} {formatDurationVerbose(session.duration)} {formatDuration(session.duration)} {session.totalOperations}
- {formatDurationVerbose(event.duration)} + {formatDuration(event.duration)} diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EngineSessionPage.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EngineSessionPage.scala index 1f34ae64f..cdfc6d313 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EngineSessionPage.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EngineSessionPage.scala @@ -42,7 +42,7 @@ case class EngineSessionPage(parent: EngineTab) require(parameterId != null && parameterId.nonEmpty, "Missing id parameter") val content = store.synchronized { // make sure all parts in this page are consistent - val sessionStat = store.getSession(parameterId).getOrElse(null) + val sessionStat = store.getSession(parameterId).orNull require(sessionStat != null, "Invalid sessionID[" + parameterId + "]") val redactionPattern = parent.sparkUI match { @@ -51,7 +51,7 @@ case class EngineSessionPage(parent: EngineTab) } val sessionPropertiesTable = - if (sessionStat.conf != null && !sessionStat.conf.isEmpty) { + if (sessionStat.conf != null && sessionStat.conf.nonEmpty) { val table = UIUtils.listingTable( propertyHeader, propertyRow, @@ -78,8 +78,18 @@ case class EngineSessionPage(parent: EngineTab)

User {sessionStat.username}, IP {sessionStat.ip}, - Server {sessionStat.serverIp}, + Server {sessionStat.serverIp} +

++ +

Session created at {formatDate(sessionStat.startTime)}, + { + if (sessionStat.endTime > 0) { + s""" + | ended at ${formatDate(sessionStat.endTime)}, + | after ${formatDuration(sessionStat.duration)}. + |""".stripMargin + } + } Total run {sessionStat.totalOperations} SQL

++ sessionPropertiesTable ++ diff --git a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EngineTab.scala b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EngineTab.scala index b7cebbd97..52edcf220 100644 --- a/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EngineTab.scala +++ b/externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/spark/ui/EngineTab.scala @@ -26,7 +26,7 @@ import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.engine.spark.SparkSQLEngine import org.apache.kyuubi.engine.spark.events.EngineEventsStore import org.apache.kyuubi.service.ServiceState -import org.apache.kyuubi.util.ClassUtils +import org.apache.kyuubi.util.reflect.{DynClasses, DynMethods} /** * Note that [[SparkUITab]] is private for Spark @@ -62,31 +62,35 @@ case class EngineTab( sparkUI.foreach { ui => try { - // Spark shade the jetty package so here we use reflection - val sparkServletContextHandlerClz = loadSparkServletContextHandler - val attachHandlerMethod = Class.forName("org.apache.spark.ui.SparkUI") - .getMethod("attachHandler", sparkServletContextHandlerClz) - val createRedirectHandlerMethod = Class.forName("org.apache.spark.ui.JettyUtils") - .getMethod( - "createRedirectHandler", + // [KYUUBI #3627]: the official spark release uses the shaded and relocated jetty classes, + // but if we use sbt to build for testing, e.g. docker image, it still uses the vanilla + // jetty classes. + val sparkServletContextHandlerClz = DynClasses.builder() + .impl("org.sparkproject.jetty.servlet.ServletContextHandler") + .impl("org.eclipse.jetty.servlet.ServletContextHandler") + .buildChecked() + val attachHandlerMethod = DynMethods.builder("attachHandler") + .impl("org.apache.spark.ui.SparkUI", sparkServletContextHandlerClz) + .buildChecked(ui) + val createRedirectHandlerMethod = DynMethods.builder("createRedirectHandler") + .impl( + "org.apache.spark.ui.JettyUtils", classOf[String], classOf[String], - classOf[(HttpServletRequest) => Unit], + classOf[HttpServletRequest => Unit], classOf[String], classOf[Set[String]]) + .buildStaticChecked() attachHandlerMethod .invoke( - ui, createRedirectHandlerMethod - .invoke(null, "/kyuubi/stop", "/kyuubi", handleKillRequest _, "", Set("GET", "POST"))) + .invoke("/kyuubi/stop", "/kyuubi", handleKillRequest _, "", Set("GET", "POST"))) attachHandlerMethod .invoke( - ui, createRedirectHandlerMethod .invoke( - null, "/kyuubi/gracefulstop", "/kyuubi", handleGracefulKillRequest _, @@ -105,18 +109,6 @@ case class EngineTab( cause) } - private def loadSparkServletContextHandler: Class[_] = { - // [KYUUBI #3627]: the official spark release uses the shaded and relocated jetty classes, - // but if use sbt to build for testing, e.g. docker image, it still uses vanilla jetty classes. - val shaded = "org.sparkproject.jetty.servlet.ServletContextHandler" - val vanilla = "org.eclipse.jetty.servlet.ServletContextHandler" - if (ClassUtils.classIsLoadable(shaded)) { - Class.forName(shaded) - } else { - Class.forName(vanilla) - } - } - def handleKillRequest(request: HttpServletRequest): Unit = { if (killEnabled && engine.isDefined && engine.get.getServiceState != ServiceState.STOPPED) { engine.get.stop() diff --git a/externals/kyuubi-spark-sql-engine/src/test/resources/log4j2-test.xml b/externals/kyuubi-spark-sql-engine/src/test/resources/log4j2-test.xml index bfc40dd6d..3110216c1 100644 --- a/externals/kyuubi-spark-sql-engine/src/test/resources/log4j2-test.xml +++ b/externals/kyuubi-spark-sql-engine/src/test/resources/log4j2-test.xml @@ -21,14 +21,14 @@ - + - + diff --git a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/EtcdShareLevelSparkEngineSuite.scala b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/EtcdShareLevelSparkEngineSuite.scala index 46dc3b54c..727b232e3 100644 --- a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/EtcdShareLevelSparkEngineSuite.scala +++ b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/EtcdShareLevelSparkEngineSuite.scala @@ -17,9 +17,7 @@ package org.apache.kyuubi.engine.spark -import org.apache.kyuubi.config.KyuubiConf.ENGINE_CHECK_INTERVAL -import org.apache.kyuubi.config.KyuubiConf.ENGINE_SHARE_LEVEL -import org.apache.kyuubi.config.KyuubiConf.ENGINE_SPARK_MAX_LIFETIME +import org.apache.kyuubi.config.KyuubiConf.{ENGINE_CHECK_INTERVAL, ENGINE_SHARE_LEVEL, ENGINE_SPARK_MAX_INITIAL_WAIT, ENGINE_SPARK_MAX_LIFETIME} import org.apache.kyuubi.engine.ShareLevel import org.apache.kyuubi.engine.ShareLevel.ShareLevel @@ -30,6 +28,7 @@ trait EtcdShareLevelSparkEngineSuite etcdConf ++ Map( ENGINE_SHARE_LEVEL.key -> shareLevel.toString, ENGINE_SPARK_MAX_LIFETIME.key -> "PT20s", + ENGINE_SPARK_MAX_INITIAL_WAIT.key -> "0", ENGINE_CHECK_INTERVAL.key -> "PT5s") } } diff --git a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/IndividualSparkSuite.scala b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/IndividualSparkSuite.scala index c6789d14d..8fca1d0ca 100644 --- a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/IndividualSparkSuite.scala +++ b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/IndividualSparkSuite.scala @@ -114,7 +114,7 @@ class SparkEngineSuites extends KyuubiFunSuite { } assert(SparkSQLEngine.currentEngine.isEmpty) val errorMsg = s"The Engine main thread was interrupted, possibly due to `createSpark`" + - s" timeout. The `kyuubi.session.engine.initialize.timeout` is ($timeout ms) " + + s" timeout. The `${ENGINE_INIT_TIMEOUT.key}` is ($timeout ms) " + s" and submitted at $submitTime." assert(logAppender.loggingEvents.exists( _.getMessage.getFormattedMessage.equals(errorMsg))) diff --git a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/SchedulerPoolSuite.scala b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/SchedulerPoolSuite.scala index af8c90cf2..a07f7d783 100644 --- a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/SchedulerPoolSuite.scala +++ b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/SchedulerPoolSuite.scala @@ -19,6 +19,9 @@ package org.apache.kyuubi.engine.spark import java.util.concurrent.Executors +import scala.concurrent.duration.SECONDS + +import org.apache.spark.KyuubiSparkContextHelper import org.apache.spark.scheduler.{SparkListener, SparkListenerJobEnd, SparkListenerJobStart} import org.scalatest.concurrent.PatienceConfiguration.Timeout import org.scalatest.time.SpanSugar.convertIntToGrainOfTime @@ -76,33 +79,36 @@ class SchedulerPoolSuite extends WithSparkSQLEngine with HiveJDBCTestHelper { eventually(Timeout(3.seconds)) { assert(job0Started) } - Seq(1, 0).foreach { priority => - threads.execute(() => { - priority match { - case 0 => - withJdbcStatement() { statement => - statement.execute("SET kyuubi.operation.scheduler.pool=p0") - statement.execute("SELECT java_method('java.lang.Thread', 'sleep', 1500l)" + - "FROM range(1, 3, 1, 2)") - } - - case 1 => - withJdbcStatement() { statement => - statement.execute("SET kyuubi.operation.scheduler.pool=p1") - statement.execute("SELECT java_method('java.lang.Thread', 'sleep', 1500l)" + - " FROM range(1, 3, 1, 2)") - } - } - }) + threads.execute(() => { + // job name job1 + withJdbcStatement() { statement => + statement.execute("SET kyuubi.operation.scheduler.pool=p1") + statement.execute("SELECT java_method('java.lang.Thread', 'sleep', 1500l)" + + " FROM range(1, 3, 1, 2)") + } + }) + // make sure job1 started before job2 + eventually(Timeout(2.seconds)) { + assert(job1StartTime > 0) } + + threads.execute(() => { + // job name job2 + withJdbcStatement() { statement => + statement.execute("SET kyuubi.operation.scheduler.pool=p0") + statement.execute("SELECT java_method('java.lang.Thread', 'sleep', 1500l)" + + "FROM range(1, 3, 1, 2)") + } + }) threads.shutdown() - eventually(Timeout(20.seconds)) { - // We can not ensure that job1 is started before job2 so here using abs. - assert(Math.abs(job1StartTime - job2StartTime) < 1000) - // Job1 minShare is 2(total resource) so that job2 should be allocated tasks after - // job1 finished. - assert(job2FinishTime - job1FinishTime >= 1000) - } + threads.awaitTermination(20, SECONDS) + // make sure the SparkListener has received the finished events for job1 and job2. + KyuubiSparkContextHelper.waitListenerBus(spark) + // job1 should be started before job2 + assert(job1StartTime < job2StartTime) + // job2 minShare is 2(total resource) so that job1 should be allocated tasks after + // job2 finished. + assert(job2FinishTime < job1FinishTime) } finally { spark.sparkContext.removeSparkListener(listener) } diff --git a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/SparkEngineRegisterSuite.scala b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/SparkEngineRegisterSuite.scala new file mode 100644 index 000000000..8c636af76 --- /dev/null +++ b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/SparkEngineRegisterSuite.scala @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.engine.spark + +import java.util.UUID + +import org.apache.kyuubi.config.KyuubiReservedKeys.{KYUUBI_ENGINE_ID, KYUUBI_ENGINE_URL} + +trait SparkEngineRegisterSuite extends WithDiscoverySparkSQLEngine { + + override def withKyuubiConf: Map[String, String] = + super.withKyuubiConf ++ Map("spark.ui.enabled" -> "true") + + override val namespace: String = s"/kyuubi/deregister_test/${UUID.randomUUID.toString}" + + test("Spark Engine Register Zookeeper with spark ui info") { + withDiscoveryClient(client => { + val info = client.getChildren(namespace).head.split(";") + assert(info.exists(_.startsWith(KYUUBI_ENGINE_ID))) + assert(info.exists(_.startsWith(KYUUBI_ENGINE_URL))) + }) + } +} + +class ZookeeperSparkEngineRegisterSuite extends SparkEngineRegisterSuite + with WithEmbeddedZookeeper { + + override def withKyuubiConf: Map[String, String] = + super.withKyuubiConf ++ zookeeperConf +} + +class EtcdSparkEngineRegisterSuite extends SparkEngineRegisterSuite + with WithEtcdCluster { + override def withKyuubiConf: Map[String, String] = super.withKyuubiConf ++ etcdConf +} diff --git a/extensions/spark/kyuubi-spark-connector-kudu/src/test/scala/org/apache/kyuubi/spark/connector/kudu/KuduClientSuite.scala b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/SparkTBinaryFrontendServiceSuite.scala similarity index 70% rename from extensions/spark/kyuubi-spark-connector-kudu/src/test/scala/org/apache/kyuubi/spark/connector/kudu/KuduClientSuite.scala rename to externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/SparkTBinaryFrontendServiceSuite.scala index eebb4719c..5f81e51f8 100644 --- a/extensions/spark/kyuubi-spark-connector-kudu/src/test/scala/org/apache/kyuubi/spark/connector/kudu/KuduClientSuite.scala +++ b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/SparkTBinaryFrontendServiceSuite.scala @@ -15,18 +15,15 @@ * limitations under the License. */ -package org.apache.kyuubi.spark.connector.kudu +package org.apache.kyuubi.engine.spark -import org.apache.kudu.client.KuduClient +import org.apache.hadoop.conf.Configuration import org.apache.kyuubi.KyuubiFunSuite -class KuduClientSuite extends KyuubiFunSuite with KuduMixin { - - test("kudu client") { - val builder = new KuduClient.KuduClientBuilder(kuduMasterUrl) - val kuduClient = builder.build() - - assert(kuduClient.findLeaderMasterServer().getPort === kuduMasterPort) +class SparkTBinaryFrontendServiceSuite extends KyuubiFunSuite { + test("new hive conf") { + val hiveConf = SparkTBinaryFrontendService.hiveConf(new Configuration()) + assert(hiveConf.getClass().getName == SparkTBinaryFrontendService.HIVE_CONF_CLASSNAME) } } diff --git a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/WithSparkSQLEngine.scala b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/WithSparkSQLEngine.scala index 629a8374b..3b98c2efb 100644 --- a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/WithSparkSQLEngine.scala +++ b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/WithSparkSQLEngine.scala @@ -21,7 +21,7 @@ import org.apache.spark.sql.SparkSession import org.apache.kyuubi.{KyuubiFunSuite, Utils} import org.apache.kyuubi.config.KyuubiConf -import org.apache.kyuubi.engine.spark.KyuubiSparkUtil.sparkMajorMinorVersion +import org.apache.kyuubi.engine.spark.KyuubiSparkUtil.SPARK_ENGINE_RUNTIME_VERSION trait WithSparkSQLEngine extends KyuubiFunSuite { protected var spark: SparkSession = _ @@ -34,14 +34,8 @@ trait WithSparkSQLEngine extends KyuubiFunSuite { // Affected by such configuration' default value // engine.initialize.sql='SHOW DATABASES' - protected var initJobId: Int = { - sparkMajorMinorVersion match { - case (3, minor) if minor >= 2 => 1 // SPARK-35378 - case (3, _) => 0 - case _ => - throw new IllegalArgumentException(s"Not Support spark version $sparkMajorMinorVersion") - } - } + // SPARK-35378 + protected lazy val initJobId: Int = if (SPARK_ENGINE_RUNTIME_VERSION >= "3.2") 1 else 0 override def beforeAll(): Unit = { startSparkEngine() diff --git a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/ZookeeperShareLevelSparkEngineSuite.scala b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/ZookeeperShareLevelSparkEngineSuite.scala index 4ef96e61a..f24abb36c 100644 --- a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/ZookeeperShareLevelSparkEngineSuite.scala +++ b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/ZookeeperShareLevelSparkEngineSuite.scala @@ -19,6 +19,7 @@ package org.apache.kyuubi.engine.spark import org.apache.kyuubi.config.KyuubiConf.ENGINE_CHECK_INTERVAL import org.apache.kyuubi.config.KyuubiConf.ENGINE_SHARE_LEVEL +import org.apache.kyuubi.config.KyuubiConf.ENGINE_SPARK_MAX_INITIAL_WAIT import org.apache.kyuubi.config.KyuubiConf.ENGINE_SPARK_MAX_LIFETIME import org.apache.kyuubi.engine.ShareLevel import org.apache.kyuubi.engine.ShareLevel.ShareLevel @@ -30,6 +31,7 @@ trait ZookeeperShareLevelSparkEngineSuite zookeeperConf ++ Map( ENGINE_SHARE_LEVEL.key -> shareLevel.toString, ENGINE_SPARK_MAX_LIFETIME.key -> "PT20s", + ENGINE_SPARK_MAX_INITIAL_WAIT.key -> "0", ENGINE_CHECK_INTERVAL.key -> "PT5s") } } diff --git a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkArrowbasedOperationSuite.scala b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkArrowbasedOperationSuite.scala index e46456914..d3d4a56d7 100644 --- a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkArrowbasedOperationSuite.scala +++ b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkArrowbasedOperationSuite.scala @@ -17,13 +17,36 @@ package org.apache.kyuubi.engine.spark.operation +import java.lang.{Boolean => JBoolean} import java.sql.Statement +import java.util.{Locale, Set => JSet} +import org.apache.spark.{KyuubiSparkContextHelper, TaskContext} +import org.apache.spark.scheduler.{SparkListener, SparkListenerJobStart} +import org.apache.spark.sql.{QueryTest, Row, SparkSession} +import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.plans.logical.Project +import org.apache.spark.sql.execution.{CollectLimitExec, LocalTableScanExec, QueryExecution, SparkPlan} +import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanExec +import org.apache.spark.sql.execution.exchange.Exchange +import org.apache.spark.sql.execution.joins.{BroadcastHashJoinExec, SortMergeJoinExec} +import org.apache.spark.sql.execution.metric.SparkMetricsTestUtils +import org.apache.spark.sql.functions.col +import org.apache.spark.sql.internal.SQLConf +import org.apache.spark.sql.kyuubi.SparkDatasetHelper +import org.apache.spark.sql.types.StructType +import org.apache.spark.sql.util.QueryExecutionListener + +import org.apache.kyuubi.KyuubiException import org.apache.kyuubi.config.KyuubiConf -import org.apache.kyuubi.engine.spark.WithSparkSQLEngine +import org.apache.kyuubi.engine.spark.{SparkSQLEngine, WithSparkSQLEngine} +import org.apache.kyuubi.engine.spark.session.SparkSessionImpl import org.apache.kyuubi.operation.SparkDataTypeTests +import org.apache.kyuubi.util.reflect.{DynFields, DynMethods} +import org.apache.kyuubi.util.reflect.ReflectUtils._ -class SparkArrowbasedOperationSuite extends WithSparkSQLEngine with SparkDataTypeTests { +class SparkArrowbasedOperationSuite extends WithSparkSQLEngine with SparkDataTypeTests + with SparkMetricsTestUtils { override protected def jdbcUrl: String = getJdbcUrl @@ -35,6 +58,23 @@ class SparkArrowbasedOperationSuite extends WithSparkSQLEngine with SparkDataTyp override def resultFormat: String = "arrow" + override def beforeEach(): Unit = { + super.beforeEach() + withJdbcStatement() { statement => + checkResultSetFormat(statement, "arrow") + } + spark.catalog.listTables() + .collect() + .foreach { table => + if (table.isTemporary) { + spark.catalog.dropTempView(table.name) + } else { + spark.sql(s"DROP TABLE IF EXISTS ${table.name}") + } + () + } + } + test("detect resultSet format") { withJdbcStatement() { statement => checkResultSetFormat(statement, "arrow") @@ -43,7 +83,314 @@ class SparkArrowbasedOperationSuite extends WithSparkSQLEngine with SparkDataTyp } } - def checkResultSetFormat(statement: Statement, expectFormat: String): Unit = { + test("Spark session timezone format") { + withJdbcStatement() { statement => + def check(expect: String): Unit = { + val query = + """ + |SELECT + | from_utc_timestamp( + | from_unixtime( + | 1670404535000 / 1000, 'yyyy-MM-dd HH:mm:ss' + | ), + | 'GMT+08:00' + | ) + |""".stripMargin + val resultSet = statement.executeQuery(query) + assert(resultSet.next()) + assert(resultSet.getString(1) == expect) + } + + def setTimeZone(timeZone: String): Unit = { + val rs = statement.executeQuery(s"set spark.sql.session.timeZone=$timeZone") + assert(rs.next()) + } + + Seq("true", "false").foreach { timestampAsString => + statement.executeQuery( + s"set ${KyuubiConf.ARROW_BASED_ROWSET_TIMESTAMP_AS_STRING.key}=$timestampAsString") + checkArrowBasedRowSetTimestampAsString(statement, timestampAsString) + setTimeZone("UTC") + check("2022-12-07 17:15:35.0") + setTimeZone("GMT+8") + check("2022-12-08 01:15:35.0") + } + } + } + + test("assign a new execution id for arrow-based result") { + val listener = new SQLMetricsListener + withJdbcStatement() { statement => + withSparkListener(listener) { + val result = statement.executeQuery("select 1 as c1") + assert(result.next()) + assert(result.getInt("c1") == 1) + } + } + + assert(listener.queryExecution.analyzed.isInstanceOf[Project]) + } + + test("arrow-based query metrics") { + val listener = new SQLMetricsListener + withJdbcStatement() { statement => + withSparkListener(listener) { + val result = statement.executeQuery("select 1 as c1") + assert(result.next()) + assert(result.getInt("c1") == 1) + } + } + + val metrics = listener.queryExecution.executedPlan.collectLeaves().head.metrics + assert(metrics.contains("numOutputRows")) + assert(metrics("numOutputRows").value === 1) + } + + test("SparkDatasetHelper.executeArrowBatchCollect should return expect row count") { + val returnSize = Seq( + 0, // spark optimizer guaranty the `limit != 0`, it's just for the sanity check + 7, // less than one partition + 10, // equal to one partition + 13, // between one and two partitions, run two jobs + 20, // equal to two partitions + 29, // between two and three partitions + 1000, // all partitions + 1001) // more than total row count + + def runAndCheck(sparkPlan: SparkPlan, expectSize: Int): Unit = { + val arrowBinary = SparkDatasetHelper.executeArrowBatchCollect(sparkPlan) + val rows = fromBatchIterator( + arrowBinary.iterator, + sparkPlan.schema, + "", + true, + KyuubiSparkContextHelper.dummyTaskContext()) + assert(rows.size == expectSize) + } + + val excludedRules = Seq( + "org.apache.spark.sql.catalyst.optimizer.EliminateLimits", + "org.apache.spark.sql.catalyst.optimizer.OptimizeLimitZero", + "org.apache.spark.sql.execution.adaptive.AQEPropagateEmptyRelation").mkString(",") + withSQLConf( + SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> excludedRules, + SQLConf.ADAPTIVE_OPTIMIZER_EXCLUDED_RULES.key -> excludedRules) { + // aqe + // outermost AdaptiveSparkPlanExec + spark.range(1000) + .repartitionByRange(100, col("id")) + .createOrReplaceTempView("t_1") + spark.sql("select * from t_1") + .foreachPartition { p: Iterator[Row] => + assert(p.length == 10) + () + } + returnSize.foreach { size => + val df = spark.sql(s"select * from t_1 limit $size") + val headPlan = df.queryExecution.executedPlan.collectLeaves().head + if (SPARK_ENGINE_RUNTIME_VERSION >= "3.2") { + assert(headPlan.isInstanceOf[AdaptiveSparkPlanExec]) + val finalPhysicalPlan = + SparkDatasetHelper.finalPhysicalPlan(headPlan.asInstanceOf[AdaptiveSparkPlanExec]) + assert(finalPhysicalPlan.isInstanceOf[CollectLimitExec]) + } + if (size > 1000) { + runAndCheck(df.queryExecution.executedPlan, 1000) + } else { + runAndCheck(df.queryExecution.executedPlan, size) + } + } + + // outermost CollectLimitExec + spark.range(0, 1000, 1, numPartitions = 100) + .createOrReplaceTempView("t_2") + spark.sql("select * from t_2") + .foreachPartition { p: Iterator[Row] => + assert(p.length == 10) + () + } + returnSize.foreach { size => + val df = spark.sql(s"select * from t_2 limit $size") + val plan = df.queryExecution.executedPlan + assert(plan.isInstanceOf[CollectLimitExec]) + if (size > 1000) { + runAndCheck(df.queryExecution.executedPlan, 1000) + } else { + runAndCheck(df.queryExecution.executedPlan, size) + } + } + } + } + + test("aqe should work properly") { + + val s = spark + import s.implicits._ + + spark.sparkContext.parallelize( + (1 to 100).map(i => TestData(i, i.toString))).toDF() + .createOrReplaceTempView("testData") + spark.sparkContext.parallelize( + TestData2(1, 1) :: + TestData2(1, 2) :: + TestData2(2, 1) :: + TestData2(2, 2) :: + TestData2(3, 1) :: + TestData2(3, 2) :: Nil, + 2).toDF() + .createOrReplaceTempView("testData2") + + withSQLConf( + SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "true", + SQLConf.SHUFFLE_PARTITIONS.key -> "5", + SQLConf.AUTO_BROADCASTJOIN_THRESHOLD.key -> "80") { + val (plan, adaptivePlan) = runAdaptiveAndVerifyResult( + """ + |SELECT * FROM( + | SELECT * FROM testData join testData2 ON key = a where value = '1' + |) LIMIT 1 + |""".stripMargin) + val smj = plan.collect { case smj: SortMergeJoinExec => smj } + val bhj = adaptivePlan.collect { case bhj: BroadcastHashJoinExec => bhj } + assert(smj.size == 1) + assert(bhj.size == 1) + } + } + + test("result offset support") { + assume(SPARK_ENGINE_RUNTIME_VERSION >= "3.4") + var numStages = 0 + val listener = new SparkListener { + override def onJobStart(jobStart: SparkListenerJobStart): Unit = { + numStages = jobStart.stageInfos.length + } + } + withJdbcStatement() { statement => + withSparkListener(listener) { + withPartitionedTable("t_3") { + statement.executeQuery("select * from t_3 limit 10 offset 10") + } + } + } + // the extra shuffle be introduced if the `offset` > 0 + assert(numStages == 2) + } + + test("arrow serialization should not introduce extra shuffle for outermost limit") { + var numStages = 0 + val listener = new SparkListener { + override def onJobStart(jobStart: SparkListenerJobStart): Unit = { + numStages = jobStart.stageInfos.length + } + } + withJdbcStatement() { statement => + withSparkListener(listener) { + withPartitionedTable("t_3") { + statement.executeQuery("select * from t_3 limit 1000") + } + } + } + // Should be only one stage since there is no shuffle. + assert(numStages == 1) + } + + test("CommandResultExec should not trigger job") { + val listener = new JobCountListener + val l2 = new SQLMetricsListener + val nodeName = spark.sql("SHOW TABLES").queryExecution.executedPlan.getClass.getName + if (SPARK_ENGINE_RUNTIME_VERSION < "3.2") { + assert(nodeName == "org.apache.spark.sql.execution.command.ExecutedCommandExec") + } else { + assert(nodeName == "org.apache.spark.sql.execution.CommandResultExec") + } + withJdbcStatement("table_1") { statement => + statement.executeQuery("CREATE TABLE table_1 (id bigint) USING parquet") + withSparkListener(listener) { + withSparkListener(l2) { + val resultSet = statement.executeQuery("SHOW TABLES") + assert(resultSet.next()) + assert(resultSet.getString("tableName") == "table_1") + } + } + } + + if (SPARK_ENGINE_RUNTIME_VERSION < "3.2") { + // Note that before Spark 3.2, a LocalTableScan SparkPlan will be submitted, and the issue of + // preventing LocalTableScan from triggering a job submission was addressed in [KYUUBI #4710]. + assert(l2.queryExecution.executedPlan.getClass.getName == + "org.apache.spark.sql.execution.LocalTableScanExec") + } else { + assert(l2.queryExecution.executedPlan.getClass.getName == + "org.apache.spark.sql.execution.CommandResultExec") + } + assert(listener.numJobs == 0) + } + + test("LocalTableScanExec should not trigger job") { + val listener = new JobCountListener + withJdbcStatement("view_1") { statement => + withSparkListener(listener) { + withAllSessions { s => + import s.implicits._ + Seq((1, "a")).toDF("c1", "c2").createOrReplaceTempView("view_1") + val plan = s.sql("select * from view_1").queryExecution.executedPlan + assert(plan.isInstanceOf[LocalTableScanExec]) + } + val resultSet = statement.executeQuery("select * from view_1") + assert(resultSet.next()) + assert(!resultSet.next()) + } + } + assert(listener.numJobs == 0) + } + + test("LocalTableScanExec metrics") { + val listener = new SQLMetricsListener + withJdbcStatement("view_1") { statement => + withSparkListener(listener) { + withAllSessions { s => + import s.implicits._ + Seq((1, "a")).toDF("c1", "c2").createOrReplaceTempView("view_1") + } + val result = statement.executeQuery("select * from view_1") + assert(result.next()) + assert(!result.next()) + } + } + + val metrics = listener.queryExecution.executedPlan.collectLeaves().head.metrics + assert(metrics.contains("numOutputRows")) + assert(metrics("numOutputRows").value === 1) + } + + test("post LocalTableScanExec driver-side metrics") { + val expectedMetrics = Map( + 0L -> (("LocalTableScan", Map("number of output rows" -> "2")))) + withTables("view_1") { + val s = spark + import s.implicits._ + Seq((1, "a"), (2, "b")).toDF("c1", "c2").createOrReplaceTempView("view_1") + val df = spark.sql("SELECT * FROM view_1") + val metrics = getSparkPlanMetrics(df) + assert(metrics == expectedMetrics) + } + } + + test("post CommandResultExec driver-side metrics") { + spark.sql("show tables").show(truncate = false) + assume(SPARK_ENGINE_RUNTIME_VERSION >= "3.2") + val expectedMetrics = Map( + 0L -> (("CommandResult", Map("number of output rows" -> "2")))) + withTables("table_1", "table_2") { + spark.sql("CREATE TABLE table_1 (id bigint) USING parquet") + spark.sql("CREATE TABLE table_2 (id bigint) USING parquet") + val df = spark.sql("SHOW TABLES") + val metrics = getSparkPlanMetrics(df) + assert(metrics == expectedMetrics) + } + } + + private def checkResultSetFormat(statement: Statement, expectFormat: String): Unit = { val query = s""" |SELECT '$${hivevar:${KyuubiConf.OPERATION_RESULT_FORMAT.key}}' AS col @@ -52,4 +399,197 @@ class SparkArrowbasedOperationSuite extends WithSparkSQLEngine with SparkDataTyp assert(resultSet.next()) assert(resultSet.getString("col") === expectFormat) } + + private def checkArrowBasedRowSetTimestampAsString( + statement: Statement, + expect: String): Unit = { + val query = + s""" + |SELECT '$${hivevar:${KyuubiConf.ARROW_BASED_ROWSET_TIMESTAMP_AS_STRING.key}}' AS col + |""".stripMargin + val resultSet = statement.executeQuery(query) + assert(resultSet.next()) + assert(resultSet.getString("col") === expect) + } + + // since all the new sessions have their owner listener bus, we should register the listener + // in the current session. + private def withSparkListener[T](listener: QueryExecutionListener)(body: => T): T = { + withAllSessions(s => s.listenerManager.register(listener)) + try { + val result = body + KyuubiSparkContextHelper.waitListenerBus(spark) + result + } finally { + withAllSessions(s => s.listenerManager.unregister(listener)) + } + } + + // since all the new sessions have their owner listener bus, we should register the listener + // in the current session. + private def withSparkListener[T](listener: SparkListener)(body: => T): T = { + withAllSessions(s => s.sparkContext.addSparkListener(listener)) + try { + val result = body + KyuubiSparkContextHelper.waitListenerBus(spark) + result + } finally { + withAllSessions(s => s.sparkContext.removeSparkListener(listener)) + } + } + + private def withPartitionedTable[T](viewName: String)(body: => T): T = { + withAllSessions { spark => + spark.range(0, 1000, 1, numPartitions = 100) + .createOrReplaceTempView(viewName) + } + try { + body + } finally { + withAllSessions { spark => + spark.sql(s"DROP VIEW IF EXISTS $viewName") + } + } + } + + private def withAllSessions(op: SparkSession => Unit): Unit = { + SparkSQLEngine.currentEngine.get + .backendService + .sessionManager + .allSessions() + .map(_.asInstanceOf[SparkSessionImpl].spark) + .foreach(op(_)) + } + + private def runAdaptiveAndVerifyResult(query: String): (SparkPlan, SparkPlan) = { + val dfAdaptive = spark.sql(query) + val planBefore = dfAdaptive.queryExecution.executedPlan + val result = dfAdaptive.collect() + withSQLConf(SQLConf.ADAPTIVE_EXECUTION_ENABLED.key -> "false") { + val df = spark.sql(query) + QueryTest.checkAnswer(df, df.collect().toSeq) + } + val planAfter = dfAdaptive.queryExecution.executedPlan + val adaptivePlan = planAfter.asInstanceOf[AdaptiveSparkPlanExec].executedPlan + val exchanges = adaptivePlan.collect { + case e: Exchange => e + } + assert(exchanges.isEmpty, "The final plan should not contain any Exchange node.") + (dfAdaptive.queryExecution.sparkPlan, adaptivePlan) + } + + /** + * Sets all SQL configurations specified in `pairs`, calls `f`, and then restores all SQL + * configurations. + */ + protected def withSQLConf(pairs: (String, String)*)(f: => Unit): Unit = { + val conf = SQLConf.get + val (keys, values) = pairs.unzip + val currentValues = keys.map { key => + if (conf.contains(key)) { + Some(conf.getConfString(key)) + } else { + None + } + } + (keys, values).zipped.foreach { (k, v) => + if (isStaticConfigKey(k)) { + throw new KyuubiException(s"Cannot modify the value of a static config: $k") + } + conf.setConfString(k, v) + } + try f + finally { + keys.zip(currentValues).foreach { + case (key, Some(value)) => conf.setConfString(key, value) + case (key, None) => conf.unsetConf(key) + } + } + } + + private def withTables[T](tableNames: String*)(f: => T): T = { + try { + f + } finally { + tableNames.foreach { name => + if (name.toUpperCase(Locale.ROOT).startsWith("VIEW")) { + spark.sql(s"DROP VIEW IF EXISTS $name") + } else { + spark.sql(s"DROP TABLE IF EXISTS $name") + } + } + } + } + + /** + * This method provides a reflection-based implementation of [[SQLConf.isStaticConfigKey]] to + * adapt Spark-3.1.x + * + * TODO: Once we drop support for Spark 3.1.x, we can directly call + * [[SQLConf.isStaticConfigKey()]]. + */ + private def isStaticConfigKey(key: String): Boolean = + getField[JSet[String]]((SQLConf.getClass, SQLConf), "staticConfKeys").contains(key) + + // the signature of function [[ArrowConverters.fromBatchIterator]] is changed in SPARK-43528 + // (since Spark 3.5) + private lazy val fromBatchIteratorMethod = DynMethods.builder("fromBatchIterator") + .hiddenImpl( // for Spark 3.4 or previous + "org.apache.spark.sql.execution.arrow.ArrowConverters$", + classOf[Iterator[Array[Byte]]], + classOf[StructType], + classOf[String], + classOf[TaskContext]) + .hiddenImpl( // for Spark 3.5 or later + "org.apache.spark.sql.execution.arrow.ArrowConverters$", + classOf[Iterator[Array[Byte]]], + classOf[StructType], + classOf[String], + classOf[Boolean], + classOf[TaskContext]) + .build() + + def fromBatchIterator( + arrowBatchIter: Iterator[Array[Byte]], + schema: StructType, + timeZoneId: String, + errorOnDuplicatedFieldNames: JBoolean, + context: TaskContext): Iterator[InternalRow] = { + val className = "org.apache.spark.sql.execution.arrow.ArrowConverters$" + val instance = DynFields.builder().impl(className, "MODULE$").build[Object]().get(null) + if (SPARK_ENGINE_RUNTIME_VERSION >= "3.5") { + fromBatchIteratorMethod.invoke[Iterator[InternalRow]]( + instance, + arrowBatchIter, + schema, + timeZoneId, + errorOnDuplicatedFieldNames, + context) + } else { + fromBatchIteratorMethod.invoke[Iterator[InternalRow]]( + instance, + arrowBatchIter, + schema, + timeZoneId, + context) + } + } + + class JobCountListener extends SparkListener { + var numJobs = 0 + override def onJobStart(jobStart: SparkListenerJobStart): Unit = { + numJobs += 1 + } + } + + class SQLMetricsListener extends QueryExecutionListener { + var queryExecution: QueryExecution = _ + override def onSuccess(funcName: String, qe: QueryExecution, durationNs: Long): Unit = { + queryExecution = qe + } + override def onFailure(funcName: String, qe: QueryExecution, exception: Exception): Unit = {} + } } + +case class TestData(key: Int, value: String) +case class TestData2(a: Int, b: Int) diff --git a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkCatalogDatabaseOperationSuite.scala b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkCatalogDatabaseOperationSuite.scala index 46208bff1..5ee01bda1 100644 --- a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkCatalogDatabaseOperationSuite.scala +++ b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkCatalogDatabaseOperationSuite.scala @@ -22,7 +22,7 @@ import org.apache.spark.sql.util.CaseInsensitiveStringMap import org.apache.kyuubi.config.KyuubiConf.ENGINE_OPERATION_CONVERT_CATALOG_DATABASE_ENABLED import org.apache.kyuubi.engine.spark.WithSparkSQLEngine -import org.apache.kyuubi.engine.spark.shim.SparkCatalogShim +import org.apache.kyuubi.engine.spark.util.SparkCatalogUtils import org.apache.kyuubi.operation.HiveJDBCTestHelper class SparkCatalogDatabaseOperationSuite extends WithSparkSQLEngine with HiveJDBCTestHelper { @@ -37,7 +37,7 @@ class SparkCatalogDatabaseOperationSuite extends WithSparkSQLEngine with HiveJDB test("set/get current catalog") { withJdbcStatement() { statement => val catalog = statement.getConnection.getCatalog - assert(catalog == SparkCatalogShim.SESSION_CATALOG) + assert(catalog == SparkCatalogUtils.SESSION_CATALOG) statement.getConnection.setCatalog("dummy") val changedCatalog = statement.getConnection.getCatalog assert(changedCatalog == "dummy") @@ -61,7 +61,7 @@ class DummyCatalog extends CatalogPlugin { _name = name } - private var _name: String = null + private var _name: String = _ override def name(): String = _name diff --git a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkOperationSuite.scala b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkOperationSuite.scala index 30bbf8b77..adab0231d 100644 --- a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkOperationSuite.scala +++ b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/operation/SparkOperationSuite.scala @@ -32,14 +32,14 @@ import org.apache.spark.sql.catalyst.analysis.FunctionRegistry import org.apache.spark.sql.types._ import org.apache.kyuubi.config.KyuubiConf -import org.apache.kyuubi.engine.SemanticVersion import org.apache.kyuubi.engine.spark.WithSparkSQLEngine import org.apache.kyuubi.engine.spark.schema.SchemaHelper.TIMESTAMP_NTZ -import org.apache.kyuubi.engine.spark.shim.SparkCatalogShim +import org.apache.kyuubi.engine.spark.util.SparkCatalogUtils +import org.apache.kyuubi.jdbc.hive.KyuubiStatement import org.apache.kyuubi.operation.{HiveMetadataTests, SparkQueryTests} import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant._ import org.apache.kyuubi.util.KyuubiHadoopUtils -import org.apache.kyuubi.util.SparkVersionUtil.isSparkVersionAtLeast +import org.apache.kyuubi.util.SemanticVersion class SparkOperationSuite extends WithSparkSQLEngine with HiveMetadataTests with SparkQueryTests { @@ -50,7 +50,7 @@ class SparkOperationSuite extends WithSparkSQLEngine with HiveMetadataTests with withJdbcStatement() { statement => val meta = statement.getConnection.getMetaData val types = meta.getTableTypes - val expected = SparkCatalogShim.sparkTableTypes.toIterator + val expected = SparkCatalogUtils.sparkTableTypes.toIterator while (types.next()) { assert(types.getString(TABLE_TYPE) === expected.next()) } @@ -93,12 +93,12 @@ class SparkOperationSuite extends WithSparkSQLEngine with HiveMetadataTests with .add("c17", "struct", nullable = true, "17") // since spark3.3.0 - if (SPARK_ENGINE_VERSION >= "3.3") { + if (SPARK_ENGINE_RUNTIME_VERSION >= "3.3") { schema = schema.add("c18", "interval day", nullable = true, "18") .add("c19", "interval year", nullable = true, "19") } // since spark3.4.0 - if (SPARK_ENGINE_VERSION >= "3.4") { + if (SPARK_ENGINE_RUNTIME_VERSION >= "3.4") { schema = schema.add("c20", "timestamp_ntz", nullable = true, "20") } @@ -144,7 +144,7 @@ class SparkOperationSuite extends WithSparkSQLEngine with HiveMetadataTests with var pos = 0 while (rowSet.next()) { - assert(rowSet.getString(TABLE_CAT) === SparkCatalogShim.SESSION_CATALOG) + assert(rowSet.getString(TABLE_CAT) === SparkCatalogUtils.SESSION_CATALOG) assert(rowSet.getString(TABLE_SCHEM) === defaultSchema) assert(rowSet.getString(TABLE_NAME) === tableName) assert(rowSet.getString(COLUMN_NAME) === schema(pos).name) @@ -202,7 +202,7 @@ class SparkOperationSuite extends WithSparkSQLEngine with HiveMetadataTests with val data = statement.getConnection.getMetaData val rowSet = data.getColumns("", "global_temp", viewName, null) while (rowSet.next()) { - assert(rowSet.getString(TABLE_CAT) === SparkCatalogShim.SESSION_CATALOG) + assert(rowSet.getString(TABLE_CAT) === SparkCatalogUtils.SESSION_CATALOG) assert(rowSet.getString(TABLE_SCHEM) === "global_temp") assert(rowSet.getString(TABLE_NAME) === viewName) assert(rowSet.getString(COLUMN_NAME) === "i") @@ -229,7 +229,7 @@ class SparkOperationSuite extends WithSparkSQLEngine with HiveMetadataTests with val data = statement.getConnection.getMetaData val rowSet = data.getColumns("", "global_temp", viewName, "n") while (rowSet.next()) { - assert(rowSet.getString(TABLE_CAT) === SparkCatalogShim.SESSION_CATALOG) + assert(rowSet.getString(TABLE_CAT) === SparkCatalogUtils.SESSION_CATALOG) assert(rowSet.getString(TABLE_SCHEM) === "global_temp") assert(rowSet.getString(TABLE_NAME) === viewName) assert(rowSet.getString(COLUMN_NAME) === "n") @@ -307,28 +307,28 @@ class SparkOperationSuite extends WithSparkSQLEngine with HiveMetadataTests with val tFetchResultsReq1 = new TFetchResultsReq(opHandle, TFetchOrientation.FETCH_NEXT, 1) val tFetchResultsResp1 = client.FetchResults(tFetchResultsReq1) assert(tFetchResultsResp1.getStatus.getStatusCode === TStatusCode.SUCCESS_STATUS) - val idSeq1 = tFetchResultsResp1.getResults.getColumns.get(0).getI64Val.getValues.asScala.toSeq + val idSeq1 = tFetchResultsResp1.getResults.getColumns.get(0).getI64Val.getValues.asScala assertResult(Seq(0L))(idSeq1) // fetch next from first row val tFetchResultsReq2 = new TFetchResultsReq(opHandle, TFetchOrientation.FETCH_NEXT, 1) val tFetchResultsResp2 = client.FetchResults(tFetchResultsReq2) assert(tFetchResultsResp2.getStatus.getStatusCode === TStatusCode.SUCCESS_STATUS) - val idSeq2 = tFetchResultsResp2.getResults.getColumns.get(0).getI64Val.getValues.asScala.toSeq + val idSeq2 = tFetchResultsResp2.getResults.getColumns.get(0).getI64Val.getValues.asScala assertResult(Seq(1L))(idSeq2) // fetch prior from second row, expected got first row val tFetchResultsReq3 = new TFetchResultsReq(opHandle, TFetchOrientation.FETCH_PRIOR, 1) val tFetchResultsResp3 = client.FetchResults(tFetchResultsReq3) assert(tFetchResultsResp3.getStatus.getStatusCode === TStatusCode.SUCCESS_STATUS) - val idSeq3 = tFetchResultsResp3.getResults.getColumns.get(0).getI64Val.getValues.asScala.toSeq + val idSeq3 = tFetchResultsResp3.getResults.getColumns.get(0).getI64Val.getValues.asScala assertResult(Seq(0L))(idSeq3) // fetch first val tFetchResultsReq4 = new TFetchResultsReq(opHandle, TFetchOrientation.FETCH_FIRST, 3) val tFetchResultsResp4 = client.FetchResults(tFetchResultsReq4) assert(tFetchResultsResp4.getStatus.getStatusCode === TStatusCode.SUCCESS_STATUS) - val idSeq4 = tFetchResultsResp4.getResults.getColumns.get(0).getI64Val.getValues.asScala.toSeq + val idSeq4 = tFetchResultsResp4.getResults.getColumns.get(0).getI64Val.getValues.asScala assertResult(Seq(0L, 1L))(idSeq4) } } @@ -350,7 +350,7 @@ class SparkOperationSuite extends WithSparkSQLEngine with HiveMetadataTests with val tFetchResultsResp1 = client.FetchResults(tFetchResultsReq1) assert(tFetchResultsResp1.getStatus.getStatusCode === TStatusCode.SUCCESS_STATUS) val idSeq1 = tFetchResultsResp1.getResults.getColumns.get(0) - .getI64Val.getValues.asScala.toSeq + .getI64Val.getValues.asScala assertResult(Seq(0L))(idSeq1) // fetch next from first row @@ -358,7 +358,7 @@ class SparkOperationSuite extends WithSparkSQLEngine with HiveMetadataTests with val tFetchResultsResp2 = client.FetchResults(tFetchResultsReq2) assert(tFetchResultsResp2.getStatus.getStatusCode === TStatusCode.SUCCESS_STATUS) val idSeq2 = tFetchResultsResp2.getResults.getColumns.get(0) - .getI64Val.getValues.asScala.toSeq + .getI64Val.getValues.asScala assertResult(Seq(1L))(idSeq2) // fetch prior from second row, expected got first row @@ -366,7 +366,7 @@ class SparkOperationSuite extends WithSparkSQLEngine with HiveMetadataTests with val tFetchResultsResp3 = client.FetchResults(tFetchResultsReq3) assert(tFetchResultsResp3.getStatus.getStatusCode === TStatusCode.SUCCESS_STATUS) val idSeq3 = tFetchResultsResp3.getResults.getColumns.get(0) - .getI64Val.getValues.asScala.toSeq + .getI64Val.getValues.asScala assertResult(Seq(0L))(idSeq3) // fetch first @@ -374,7 +374,7 @@ class SparkOperationSuite extends WithSparkSQLEngine with HiveMetadataTests with val tFetchResultsResp4 = client.FetchResults(tFetchResultsReq4) assert(tFetchResultsResp4.getStatus.getStatusCode === TStatusCode.SUCCESS_STATUS) val idSeq4 = tFetchResultsResp4.getResults.getColumns.get(0) - .getI64Val.getValues.asScala.toSeq + .getI64Val.getValues.asScala assertResult(Seq(0L, 1L))(idSeq4) } } @@ -511,7 +511,7 @@ class SparkOperationSuite extends WithSparkSQLEngine with HiveMetadataTests with val status = tOpenSessionResp.getStatus val errorMessage = status.getErrorMessage assert(status.getStatusCode === TStatusCode.ERROR_STATUS) - if (isSparkVersionAtLeast("3.4")) { + if (SPARK_ENGINE_RUNTIME_VERSION >= "3.4") { assert(errorMessage.contains("[SCHEMA_NOT_FOUND]")) assert(errorMessage.contains(s"The schema `$dbName` cannot be found.")) } else { @@ -729,6 +729,14 @@ class SparkOperationSuite extends WithSparkSQLEngine with HiveMetadataTests with } } + test("KYUUBI #5030: Support get query id in Spark engine") { + withJdbcStatement() { stmt => + stmt.executeQuery("SELECT 1") + val queryId = stmt.asInstanceOf[KyuubiStatement].getQueryId + assert(queryId != null && queryId.nonEmpty) + } + } + private def whenMetaStoreURIsSetTo(uris: String)(func: String => Unit): Unit = { val conf = spark.sparkContext.hadoopConfiguration val origin = conf.get("hive.metastore.uris", "") diff --git a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/schema/RowSetSuite.scala b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/schema/RowSetSuite.scala index 803eea3e6..5d2ba4a0d 100644 --- a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/schema/RowSetSuite.scala +++ b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/schema/RowSetSuite.scala @@ -20,7 +20,7 @@ package org.apache.kyuubi.engine.spark.schema import java.nio.ByteBuffer import java.nio.charset.StandardCharsets import java.sql.{Date, Timestamp} -import java.time.{Instant, LocalDate, ZoneId} +import java.time.{Instant, LocalDate} import scala.collection.JavaConverters._ @@ -30,7 +30,6 @@ import org.apache.spark.sql.types._ import org.apache.spark.unsafe.types.CalendarInterval import org.apache.kyuubi.KyuubiFunSuite -import org.apache.kyuubi.engine.spark.schema.RowSet.toHiveString class RowSetSuite extends KyuubiFunSuite { @@ -97,10 +96,9 @@ class RowSetSuite extends KyuubiFunSuite { .add("q", "timestamp") private val rows: Seq[Row] = (0 to 10).map(genRow) ++ Seq(Row.fromSeq(Seq.fill(17)(null))) - private val zoneId: ZoneId = ZoneId.systemDefault() test("column based set") { - val tRowSet = RowSet.toColumnBasedSet(rows, schema, zoneId) + val tRowSet = RowSet.toColumnBasedSet(rows, schema) assert(tRowSet.getColumns.size() === schema.size) assert(tRowSet.getRowsSize === 0) @@ -159,22 +157,22 @@ class RowSetSuite extends KyuubiFunSuite { val decCol = cols.next().getStringVal decCol.getValues.asScala.zipWithIndex.foreach { - case (b, 11) => assert(b.isEmpty) + case (b, 11) => assert(b === "NULL") case (b, i) => assert(b === s"$i.$i") } val dateCol = cols.next().getStringVal dateCol.getValues.asScala.zipWithIndex.foreach { - case (b, 11) => assert(b.isEmpty) + case (b, 11) => assert(b === "NULL") case (b, i) => - assert(b === toHiveString((Date.valueOf(s"2018-11-${i + 1}"), DateType), zoneId)) + assert(b === RowSet.toHiveString(Date.valueOf(s"2018-11-${i + 1}") -> DateType)) } val tsCol = cols.next().getStringVal tsCol.getValues.asScala.zipWithIndex.foreach { - case (b, 11) => assert(b.isEmpty) + case (b, 11) => assert(b === "NULL") case (b, i) => assert(b === - toHiveString((Timestamp.valueOf(s"2018-11-17 13:33:33.$i"), TimestampType), zoneId)) + RowSet.toHiveString(Timestamp.valueOf(s"2018-11-17 13:33:33.$i") -> TimestampType)) } val binCol = cols.next().getBinaryVal @@ -185,29 +183,27 @@ class RowSetSuite extends KyuubiFunSuite { val arrCol = cols.next().getStringVal arrCol.getValues.asScala.zipWithIndex.foreach { - case (b, 11) => assert(b === "") - case (b, i) => assert(b === toHiveString( - (Array.fill(i)(java.lang.Double.valueOf(s"$i.$i")).toSeq, ArrayType(DoubleType)), - zoneId)) + case (b, 11) => assert(b === "NULL") + case (b, i) => assert(b === RowSet.toHiveString( + Array.fill(i)(java.lang.Double.valueOf(s"$i.$i")).toSeq -> ArrayType(DoubleType))) } val mapCol = cols.next().getStringVal mapCol.getValues.asScala.zipWithIndex.foreach { - case (b, 11) => assert(b === "") - case (b, i) => assert(b === toHiveString( - (Map(i -> java.lang.Double.valueOf(s"$i.$i")), MapType(IntegerType, DoubleType)), - zoneId)) + case (b, 11) => assert(b === "NULL") + case (b, i) => assert(b === RowSet.toHiveString( + Map(i -> java.lang.Double.valueOf(s"$i.$i")) -> MapType(IntegerType, DoubleType))) } val intervalCol = cols.next().getStringVal intervalCol.getValues.asScala.zipWithIndex.foreach { - case (b, 11) => assert(b === "") + case (b, 11) => assert(b === "NULL") case (b, i) => assert(b === new CalendarInterval(i, i, i).toString) } } test("row based set") { - val tRowSet = RowSet.toRowBasedSet(rows, schema, zoneId) + val tRowSet = RowSet.toRowBasedSet(rows, schema) assert(tRowSet.getColumnCount === 0) assert(tRowSet.getRowsSize === rows.size) val iter = tRowSet.getRowsIterator @@ -237,7 +233,7 @@ class RowSetSuite extends KyuubiFunSuite { assert(r6.get(9).getStringVal.getValue === "2018-11-06") val r7 = iter.next().getColVals - assert(r7.get(10).getStringVal.getValue === "2018-11-17 13:33:33.600") + assert(r7.get(10).getStringVal.getValue === "2018-11-17 13:33:33.6") assert(r7.get(11).getStringVal.getValue === new String( Array.fill[Byte](6)(6.toByte), StandardCharsets.UTF_8)) @@ -245,7 +241,7 @@ class RowSetSuite extends KyuubiFunSuite { val r8 = iter.next().getColVals assert(r8.get(12).getStringVal.getValue === Array.fill(7)(7.7d).mkString("[", ",", "]")) assert(r8.get(13).getStringVal.getValue === - toHiveString((Map(7 -> 7.7d), MapType(IntegerType, DoubleType)), zoneId)) + RowSet.toHiveString(Map(7 -> 7.7d) -> MapType(IntegerType, DoubleType))) val r9 = iter.next().getColVals assert(r9.get(14).getStringVal.getValue === new CalendarInterval(8, 8, 8).toString) @@ -253,7 +249,7 @@ class RowSetSuite extends KyuubiFunSuite { test("to row set") { TProtocolVersion.values().foreach { proto => - val set = RowSet.toTRowSet(rows, schema, proto, zoneId) + val set = RowSet.toTRowSet(rows, schema, proto) if (proto.getValue < TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V6.getValue) { assert(!set.isSetColumns, proto.toString) assert(set.isSetRows, proto.toString) diff --git a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/session/SessionSuite.scala b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/session/SessionSuite.scala index 5e0b6c28e..b89c560b3 100644 --- a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/session/SessionSuite.scala +++ b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/session/SessionSuite.scala @@ -27,7 +27,9 @@ import org.apache.kyuubi.service.ServiceState._ class SessionSuite extends WithSparkSQLEngine with HiveJDBCTestHelper { override def withKyuubiConf: Map[String, String] = { - Map(ENGINE_SHARE_LEVEL.key -> "CONNECTION") + Map( + ENGINE_SHARE_LEVEL.key -> "CONNECTION", + ENGINE_SPARK_MAX_INITIAL_WAIT.key -> "0") } override protected def beforeEach(): Unit = { diff --git a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/udf/KyuubiDefinedFunctionSuite.scala b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/udf/KyuubiDefinedFunctionSuite.scala index dc0513ed3..7a3f8c940 100644 --- a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/udf/KyuubiDefinedFunctionSuite.scala +++ b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/engine/spark/udf/KyuubiDefinedFunctionSuite.scala @@ -19,26 +19,23 @@ package org.apache.kyuubi.engine.spark.udf import java.nio.file.Paths -import scala.collection.mutable.ArrayBuffer +import org.apache.kyuubi.{KyuubiFunSuite, MarkdownBuilder, Utils} +import org.apache.kyuubi.util.GoldenFileUtils._ -import org.apache.kyuubi.{KyuubiFunSuite, TestUtils, Utils} - -// scalastyle:off line.size.limit /** * End-to-end test cases for configuration doc file - * The golden result file is "docs/sql/functions.md". + * The golden result file is "docs/extensions/engines/spark/functions.md". * * To run the entire test suite: * {{{ - * build/mvn clean test -pl externals/kyuubi-spark-sql-engine -am -Pflink-provided,spark-provided,hive-provided -DwildcardSuites=org.apache.kyuubi.engine.spark.udf.KyuubiDefinedFunctionSuite + * KYUUBI_UPDATE=0 dev/gen/gen_spark_kdf_docs.sh * }}} * * To re-generate golden files for entire suite, run: * {{{ - * KYUUBI_UPDATE=1 build/mvn clean test -pl externals/kyuubi-spark-sql-engine -am -Pflink-provided,spark-provided,hive-provided -DwildcardSuites=org.apache.kyuubi.engine.spark.udf.KyuubiDefinedFunctionSuite + * dev/gen/gen_spark_kdf_docs.sh * }}} */ -// scalastyle:on line.size.limit class KyuubiDefinedFunctionSuite extends KyuubiFunSuite { private val kyuubiHome: String = Utils.getCodeSourceLocation(getClass) @@ -48,45 +45,20 @@ class KyuubiDefinedFunctionSuite extends KyuubiFunSuite { .toAbsolutePath test("verify or update kyuubi spark sql functions") { - val newOutput = new ArrayBuffer[String]() - newOutput += "" - newOutput += "" - newOutput += "" - newOutput += "" - newOutput += "" - newOutput += "# Auxiliary SQL Functions" - newOutput += "" - newOutput += "Kyuubi provides several auxiliary SQL functions as supplement to Spark's " + - "[Built-in Functions](https://spark.apache.org/docs/latest/api/sql/index.html#" + - "built-in-functions)" - newOutput += "" - newOutput += "Name | Description | Return Type | Since" - newOutput += "--- | --- | --- | ---" - KDFRegistry + val builder = MarkdownBuilder(licenced = true, getClass.getName) + + builder += "# Auxiliary SQL Functions" += + """Kyuubi provides several auxiliary SQL functions as supplement to Spark's + | [Built-in Functions](https://spark.apache.org/docs/latest/api/sql/index.html# + |built-in-functions)""" ++= + """ + | Name | Description | Return Type | Since + | --- | --- | --- | --- + |""" KDFRegistry.registeredFunctions.foreach { func => - newOutput += s"${func.name} | ${func.description} | ${func.returnType} | ${func.since}" + builder += s"${func.name} | ${func.description} | ${func.returnType} | ${func.since}" } - newOutput += "" - TestUtils.verifyOutput( - markdown, - newOutput, - getClass.getCanonicalName, - "externals/kyuubi-spark-sql-engine") + + verifyOrRegenerateGoldenFile(markdown, builder.toMarkdown, "dev/gen/gen_spark_kdf_docs.sh") } } diff --git a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/jdbc/KyuubiHiveDriverSuite.scala b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/jdbc/KyuubiHiveDriverSuite.scala index 4d3c75498..ae68440df 100644 --- a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/jdbc/KyuubiHiveDriverSuite.scala +++ b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/kyuubi/jdbc/KyuubiHiveDriverSuite.scala @@ -22,7 +22,7 @@ import java.util.Properties import org.apache.kyuubi.IcebergSuiteMixin import org.apache.kyuubi.engine.spark.WithSparkSQLEngine -import org.apache.kyuubi.engine.spark.shim.SparkCatalogShim +import org.apache.kyuubi.engine.spark.util.SparkCatalogUtils import org.apache.kyuubi.jdbc.hive.{KyuubiConnection, KyuubiStatement} import org.apache.kyuubi.tags.IcebergTest @@ -47,15 +47,15 @@ class KyuubiHiveDriverSuite extends WithSparkSQLEngine with IcebergSuiteMixin { val metaData = connection.getMetaData assert(metaData.getClass.getName === "org.apache.kyuubi.jdbc.hive.KyuubiDatabaseMetaData") val statement = connection.createStatement() - val table1 = s"${SparkCatalogShim.SESSION_CATALOG}.default.kyuubi_hive_jdbc" + val table1 = s"${SparkCatalogUtils.SESSION_CATALOG}.default.kyuubi_hive_jdbc" val table2 = s"$catalog.default.hdp_cat_tbl" try { statement.execute(s"CREATE TABLE $table1(key int) USING parquet") statement.execute(s"CREATE TABLE $table2(key int) USING $format") - val resultSet1 = metaData.getTables(SparkCatalogShim.SESSION_CATALOG, "default", "%", null) + val resultSet1 = metaData.getTables(SparkCatalogUtils.SESSION_CATALOG, "default", "%", null) assert(resultSet1.next()) - assert(resultSet1.getString(1) === SparkCatalogShim.SESSION_CATALOG) + assert(resultSet1.getString(1) === SparkCatalogUtils.SESSION_CATALOG) assert(resultSet1.getString(2) === "default") assert(resultSet1.getString(3) === "kyuubi_hive_jdbc") diff --git a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/spark/KyuubiSparkContextHelper.scala b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/spark/KyuubiSparkContextHelper.scala new file mode 100644 index 000000000..1b662eadf --- /dev/null +++ b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/spark/KyuubiSparkContextHelper.scala @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark + +import org.apache.spark.sql.SparkSession + +/** + * A place to invoke non-public APIs of [[SparkContext]], for test only. + */ +object KyuubiSparkContextHelper { + + def waitListenerBus(spark: SparkSession): Unit = { + spark.sparkContext.listenerBus.waitUntilEmpty() + } + + def dummyTaskContext(): TaskContextImpl = TaskContext.empty() +} diff --git a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/spark/kyuubi/SQLOperationListenerSuite.scala b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/spark/kyuubi/SQLOperationListenerSuite.scala index 04277fca4..f732f7c38 100644 --- a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/spark/kyuubi/SQLOperationListenerSuite.scala +++ b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/spark/kyuubi/SQLOperationListenerSuite.scala @@ -22,13 +22,16 @@ import scala.collection.JavaConverters.asScalaBufferConverter import org.apache.hive.service.rpc.thrift.{TExecuteStatementReq, TFetchOrientation, TFetchResultsReq, TOperationHandle} import org.scalatest.time.SpanSugar._ +import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.config.KyuubiConf.OPERATION_SPARK_LISTENER_ENABLED import org.apache.kyuubi.engine.spark.WithSparkSQLEngine import org.apache.kyuubi.operation.HiveJDBCTestHelper class SQLOperationListenerSuite extends WithSparkSQLEngine with HiveJDBCTestHelper { - override def withKyuubiConf: Map[String, String] = Map.empty + override def withKyuubiConf: Map[String, String] = Map( + KyuubiConf.ENGINE_SPARK_SHOW_PROGRESS.key -> "true", + KyuubiConf.ENGINE_SPARK_SHOW_PROGRESS_UPDATE_INTERVAL.key -> "200") override protected def jdbcUrl: String = getJdbcUrl @@ -54,6 +57,24 @@ class SQLOperationListenerSuite extends WithSparkSQLEngine with HiveJDBCTestHelp } } + test("operation listener with progress job info") { + val sql = "SELECT java_method('java.lang.Thread', 'sleep', 10000l) FROM range(1, 3, 1, 2);" + withSessionHandle { (client, handle) => + val req = new TExecuteStatementReq() + req.setSessionHandle(handle) + req.setStatement(sql) + val tExecuteStatementResp = client.ExecuteStatement(req) + val opHandle = tExecuteStatementResp.getOperationHandle + val fetchResultsReq = new TFetchResultsReq(opHandle, TFetchOrientation.FETCH_NEXT, 1000) + fetchResultsReq.setFetchType(1.toShort) + eventually(timeout(90.seconds), interval(500.milliseconds)) { + val resultsResp = client.FetchResults(fetchResultsReq) + val logs = resultsResp.getResults.getColumns.get(0).getStringVal.getValues.asScala + assert(logs.exists(_.matches(".*\\[Job .* Stages\\] \\[Stage .*\\]"))) + } + } + } + test("SQLOperationListener configurable") { val sql = "select /*+ REPARTITION(3, a) */ a from values(1) t(a);" withSessionHandle { (client, handle) => diff --git a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/spark/kyuubi/SparkSQLEngineDeregisterSuite.scala b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/spark/kyuubi/SparkSQLEngineDeregisterSuite.scala index 8dc93759b..4dddcd4ee 100644 --- a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/spark/kyuubi/SparkSQLEngineDeregisterSuite.scala +++ b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/spark/kyuubi/SparkSQLEngineDeregisterSuite.scala @@ -24,9 +24,8 @@ import org.apache.spark.sql.internal.SQLConf.ANSI_ENABLED import org.scalatest.time.SpanSugar.convertIntToGrainOfTime import org.apache.kyuubi.config.KyuubiConf._ -import org.apache.kyuubi.engine.spark.KyuubiSparkUtil.sparkMajorMinorVersion -import org.apache.kyuubi.engine.spark.WithDiscoverySparkSQLEngine -import org.apache.kyuubi.engine.spark.WithEmbeddedZookeeper +import org.apache.kyuubi.engine.spark.{WithDiscoverySparkSQLEngine, WithEmbeddedZookeeper} +import org.apache.kyuubi.engine.spark.KyuubiSparkUtil.SPARK_ENGINE_RUNTIME_VERSION import org.apache.kyuubi.service.ServiceState abstract class SparkSQLEngineDeregisterSuite @@ -61,10 +60,11 @@ abstract class SparkSQLEngineDeregisterSuite class SparkSQLEngineDeregisterExceptionSuite extends SparkSQLEngineDeregisterSuite { override def withKyuubiConf: Map[String, String] = { super.withKyuubiConf ++ Map(ENGINE_DEREGISTER_EXCEPTION_CLASSES.key -> { - sparkMajorMinorVersion match { + if (SPARK_ENGINE_RUNTIME_VERSION >= "3.3") { // see https://issues.apache.org/jira/browse/SPARK-35958 - case (3, minor) if minor > 2 => "org.apache.spark.SparkArithmeticException" - case _ => classOf[ArithmeticException].getCanonicalName + "org.apache.spark.SparkArithmeticException" + } else { + classOf[ArithmeticException].getCanonicalName } }) @@ -94,10 +94,11 @@ class SparkSQLEngineDeregisterExceptionTTLSuite zookeeperConf ++ Map( ANSI_ENABLED.key -> "true", ENGINE_DEREGISTER_EXCEPTION_CLASSES.key -> { - sparkMajorMinorVersion match { + if (SPARK_ENGINE_RUNTIME_VERSION >= "3.3") { // see https://issues.apache.org/jira/browse/SPARK-35958 - case (3, minor) if minor > 2 => "org.apache.spark.SparkArithmeticException" - case _ => classOf[ArithmeticException].getCanonicalName + "org.apache.spark.SparkArithmeticException" + } else { + classOf[ArithmeticException].getCanonicalName } }, ENGINE_DEREGISTER_JOB_MAX_FAILURES.key -> maxJobFailures.toString, diff --git a/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/spark/sql/execution/metric/SparkMetricsTestUtils.scala b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/spark/sql/execution/metric/SparkMetricsTestUtils.scala new file mode 100644 index 000000000..7ab06f0ef --- /dev/null +++ b/externals/kyuubi-spark-sql-engine/src/test/scala/org/apache/spark/sql/execution/metric/SparkMetricsTestUtils.scala @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.metric + +import org.apache.spark.sql.DataFrame +import org.apache.spark.sql.execution.SparkPlanInfo +import org.apache.spark.sql.execution.ui.SparkPlanGraph +import org.apache.spark.sql.kyuubi.SparkDatasetHelper + +import org.apache.kyuubi.engine.spark.WithSparkSQLEngine + +trait SparkMetricsTestUtils { + this: WithSparkSQLEngine => + + private lazy val statusStore = spark.sharedState.statusStore + private def currentExecutionIds(): Set[Long] = { + spark.sparkContext.listenerBus.waitUntilEmpty(10000) + statusStore.executionsList.map(_.executionId).toSet + } + + protected def getSparkPlanMetrics(df: DataFrame): Map[Long, (String, Map[String, Any])] = { + val previousExecutionIds = currentExecutionIds() + SparkDatasetHelper.executeCollect(df) + spark.sparkContext.listenerBus.waitUntilEmpty(10000) + val executionIds = currentExecutionIds().diff(previousExecutionIds) + assert(executionIds.size === 1) + val executionId = executionIds.head + val metricValues = statusStore.executionMetrics(executionId) + SparkPlanGraph(SparkPlanInfo.fromSparkPlan(df.queryExecution.executedPlan)).allNodes + .map { node => + val nodeMetrics = node.metrics.map { metric => + val metricValue = metricValues(metric.accumulatorId) + (metric.name, metricValue) + }.toMap + (node.id, node.name -> nodeMetrics) + }.toMap + } +} diff --git a/externals/kyuubi-trino-engine/pom.xml b/externals/kyuubi-trino-engine/pom.xml index 7e2f67370..7d91e4a86 100644 --- a/externals/kyuubi-trino-engine/pom.xml +++ b/externals/kyuubi-trino-engine/pom.xml @@ -21,11 +21,11 @@ org.apache.kyuubi kyuubi-parent - 1.7.0-SNAPSHOT + 1.9.0-SNAPSHOT ../../pom.xml - kyuubi-trino-engine_2.12 + kyuubi-trino-engine_${scala.binary.version} jar Kyuubi Project Engine Trino https://kyuubi.apache.org/ diff --git a/externals/kyuubi-trino-engine/src/main/scala/org/apache/kyuubi/engine/trino/operation/ExecuteStatement.scala b/externals/kyuubi-trino-engine/src/main/scala/org/apache/kyuubi/engine/trino/operation/ExecuteStatement.scala index eb1b27300..3e7cce80c 100644 --- a/externals/kyuubi-trino-engine/src/main/scala/org/apache/kyuubi/engine/trino/operation/ExecuteStatement.scala +++ b/externals/kyuubi-trino-engine/src/main/scala/org/apache/kyuubi/engine/trino/operation/ExecuteStatement.scala @@ -19,7 +19,7 @@ package org.apache.kyuubi.engine.trino.operation import java.util.concurrent.RejectedExecutionException -import org.apache.hive.service.rpc.thrift.TRowSet +import org.apache.hive.service.rpc.thrift.TFetchResultsResp import org.apache.kyuubi.{KyuubiSQLException, Logging} import org.apache.kyuubi.engine.trino.TrinoStatement @@ -82,7 +82,9 @@ class ExecuteStatement( } } - override def getNextRowSet(order: FetchOrientation, rowSetSize: Int): TRowSet = { + override def getNextRowSetInternal( + order: FetchOrientation, + rowSetSize: Int): TFetchResultsResp = { validateDefaultFetchOrientation(order) assertState(OperationState.FINISHED) setHasResultSet(true) @@ -97,7 +99,10 @@ class ExecuteStatement( val taken = iter.take(rowSetSize) val resultRowSet = RowSet.toTRowSet(taken.toList, schema, getProtocolVersion) resultRowSet.setStartRowOffset(iter.getPosition) - resultRowSet + val fetchResultsResp = new TFetchResultsResp(OK_STATUS) + fetchResultsResp.setResults(resultRowSet) + fetchResultsResp.setHasMoreRows(false) + fetchResultsResp } private def executeStatement(trinoStatement: TrinoStatement): Unit = { diff --git a/externals/kyuubi-trino-engine/src/main/scala/org/apache/kyuubi/engine/trino/operation/GetCurrentCatalog.scala b/externals/kyuubi-trino-engine/src/main/scala/org/apache/kyuubi/engine/trino/operation/GetCurrentCatalog.scala index 3d8c7fd6c..504a53a41 100644 --- a/externals/kyuubi-trino-engine/src/main/scala/org/apache/kyuubi/engine/trino/operation/GetCurrentCatalog.scala +++ b/externals/kyuubi-trino-engine/src/main/scala/org/apache/kyuubi/engine/trino/operation/GetCurrentCatalog.scala @@ -23,11 +23,16 @@ import io.trino.client.ClientStandardTypes.VARCHAR import io.trino.client.ClientTypeSignature.VARCHAR_UNBOUNDED_LENGTH import org.apache.kyuubi.operation.IterableFetchIterator +import org.apache.kyuubi.operation.log.OperationLog import org.apache.kyuubi.session.Session class GetCurrentCatalog(session: Session) extends TrinoOperation(session) { + private val operationLog: OperationLog = OperationLog.createOperationLog(session, getHandle) + + override def getOperationLog: Option[OperationLog] = Option(operationLog) + override protected def runInternal(): Unit = { try { val session = trinoContext.clientSession.get diff --git a/externals/kyuubi-trino-engine/src/main/scala/org/apache/kyuubi/engine/trino/operation/GetCurrentDatabase.scala b/externals/kyuubi-trino-engine/src/main/scala/org/apache/kyuubi/engine/trino/operation/GetCurrentDatabase.scala index 3bf2987b4..3ab598ef0 100644 --- a/externals/kyuubi-trino-engine/src/main/scala/org/apache/kyuubi/engine/trino/operation/GetCurrentDatabase.scala +++ b/externals/kyuubi-trino-engine/src/main/scala/org/apache/kyuubi/engine/trino/operation/GetCurrentDatabase.scala @@ -23,11 +23,16 @@ import io.trino.client.ClientStandardTypes.VARCHAR import io.trino.client.ClientTypeSignature.VARCHAR_UNBOUNDED_LENGTH import org.apache.kyuubi.operation.IterableFetchIterator +import org.apache.kyuubi.operation.log.OperationLog import org.apache.kyuubi.session.Session class GetCurrentDatabase(session: Session) extends TrinoOperation(session) { + private val operationLog: OperationLog = OperationLog.createOperationLog(session, getHandle) + + override def getOperationLog: Option[OperationLog] = Option(operationLog) + override protected def runInternal(): Unit = { try { val session = trinoContext.clientSession.get diff --git a/externals/kyuubi-trino-engine/src/main/scala/org/apache/kyuubi/engine/trino/operation/SetCurrentCatalog.scala b/externals/kyuubi-trino-engine/src/main/scala/org/apache/kyuubi/engine/trino/operation/SetCurrentCatalog.scala index 09ba4262f..16836b0a9 100644 --- a/externals/kyuubi-trino-engine/src/main/scala/org/apache/kyuubi/engine/trino/operation/SetCurrentCatalog.scala +++ b/externals/kyuubi-trino-engine/src/main/scala/org/apache/kyuubi/engine/trino/operation/SetCurrentCatalog.scala @@ -19,11 +19,16 @@ package org.apache.kyuubi.engine.trino.operation import io.trino.client.ClientSession +import org.apache.kyuubi.operation.log.OperationLog import org.apache.kyuubi.session.Session class SetCurrentCatalog(session: Session, catalog: String) extends TrinoOperation(session) { + private val operationLog: OperationLog = OperationLog.createOperationLog(session, getHandle) + + override def getOperationLog: Option[OperationLog] = Option(operationLog) + override protected def runInternal(): Unit = { try { val session = trinoContext.clientSession.get diff --git a/externals/kyuubi-trino-engine/src/main/scala/org/apache/kyuubi/engine/trino/operation/SetCurrentDatabase.scala b/externals/kyuubi-trino-engine/src/main/scala/org/apache/kyuubi/engine/trino/operation/SetCurrentDatabase.scala index f25cc9e0c..aa4697f5f 100644 --- a/externals/kyuubi-trino-engine/src/main/scala/org/apache/kyuubi/engine/trino/operation/SetCurrentDatabase.scala +++ b/externals/kyuubi-trino-engine/src/main/scala/org/apache/kyuubi/engine/trino/operation/SetCurrentDatabase.scala @@ -19,11 +19,16 @@ package org.apache.kyuubi.engine.trino.operation import io.trino.client.ClientSession +import org.apache.kyuubi.operation.log.OperationLog import org.apache.kyuubi.session.Session class SetCurrentDatabase(session: Session, database: String) extends TrinoOperation(session) { + private val operationLog: OperationLog = OperationLog.createOperationLog(session, getHandle) + + override def getOperationLog: Option[OperationLog] = Option(operationLog) + override protected def runInternal(): Unit = { try { val session = trinoContext.clientSession.get diff --git a/externals/kyuubi-trino-engine/src/main/scala/org/apache/kyuubi/engine/trino/operation/TrinoOperation.scala b/externals/kyuubi-trino-engine/src/main/scala/org/apache/kyuubi/engine/trino/operation/TrinoOperation.scala index 6e40f65f2..11eaa1bc1 100644 --- a/externals/kyuubi-trino-engine/src/main/scala/org/apache/kyuubi/engine/trino/operation/TrinoOperation.scala +++ b/externals/kyuubi-trino-engine/src/main/scala/org/apache/kyuubi/engine/trino/operation/TrinoOperation.scala @@ -21,7 +21,7 @@ import java.io.IOException import io.trino.client.Column import io.trino.client.StatementClient -import org.apache.hive.service.rpc.thrift.{TGetResultSetMetadataResp, TRowSet} +import org.apache.hive.service.rpc.thrift.{TFetchResultsResp, TGetResultSetMetadataResp} import org.apache.kyuubi.KyuubiSQLException import org.apache.kyuubi.Utils @@ -54,7 +54,9 @@ abstract class TrinoOperation(session: Session) extends AbstractOperation(sessio resp } - override def getNextRowSet(order: FetchOrientation, rowSetSize: Int): TRowSet = { + override def getNextRowSetInternal( + order: FetchOrientation, + rowSetSize: Int): TFetchResultsResp = { validateDefaultFetchOrientation(order) assertState(OperationState.FINISHED) setHasResultSet(true) @@ -66,7 +68,10 @@ abstract class TrinoOperation(session: Session) extends AbstractOperation(sessio val taken = iter.take(rowSetSize) val resultRowSet = RowSet.toTRowSet(taken.toList, schema, getProtocolVersion) resultRowSet.setStartRowOffset(iter.getPosition) - resultRowSet + val resp = new TFetchResultsResp(OK_STATUS) + resp.setResults(resultRowSet) + resp.setHasMoreRows(false) + resp } override protected def beforeRun(): Unit = { @@ -75,7 +80,7 @@ abstract class TrinoOperation(session: Session) extends AbstractOperation(sessio } override protected def afterRun(): Unit = { - state.synchronized { + withLockRequired { if (!isTerminalState(state)) { setState(OperationState.FINISHED) } @@ -108,7 +113,7 @@ abstract class TrinoOperation(session: Session) extends AbstractOperation(sessio // could be thrown. case e: Throwable => if (cancel && trino.isRunning) trino.cancelLeafStage() - state.synchronized { + withLockRequired { val errMsg = Utils.stringifyException(e) if (state == OperationState.TIMEOUT) { val ke = KyuubiSQLException(s"Timeout operating $opType: $errMsg") diff --git a/externals/kyuubi-trino-engine/src/main/scala/org/apache/kyuubi/engine/trino/session/TrinoSessionImpl.scala b/externals/kyuubi-trino-engine/src/main/scala/org/apache/kyuubi/engine/trino/session/TrinoSessionImpl.scala index a19d74d58..0b3ac01a9 100644 --- a/externals/kyuubi-trino-engine/src/main/scala/org/apache/kyuubi/engine/trino/session/TrinoSessionImpl.scala +++ b/externals/kyuubi-trino-engine/src/main/scala/org/apache/kyuubi/engine/trino/session/TrinoSessionImpl.scala @@ -22,19 +22,23 @@ import java.time.ZoneId import java.util.{Collections, Locale, Optional} import java.util.concurrent.TimeUnit +import scala.collection.JavaConverters._ + import io.airlift.units.Duration import io.trino.client.ClientSession +import io.trino.client.OkHttpUtil import okhttp3.OkHttpClient import org.apache.hive.service.rpc.thrift.{TGetInfoType, TGetInfoValue, TProtocolVersion} import org.apache.kyuubi.KyuubiSQLException import org.apache.kyuubi.Utils.currentUser import org.apache.kyuubi.config.{KyuubiConf, KyuubiReservedKeys} +import org.apache.kyuubi.config.KyuubiReservedKeys.KYUUBI_SESSION_HANDLE_KEY import org.apache.kyuubi.engine.trino.{TrinoConf, TrinoContext, TrinoStatement} import org.apache.kyuubi.engine.trino.event.TrinoSessionEvent import org.apache.kyuubi.events.EventBus import org.apache.kyuubi.operation.{Operation, OperationHandle} -import org.apache.kyuubi.session.{AbstractSession, SessionManager} +import org.apache.kyuubi.session.{AbstractSession, SessionHandle, SessionManager, USE_CATALOG, USE_DATABASE} class TrinoSessionImpl( protocol: TProtocolVersion, @@ -45,47 +49,53 @@ class TrinoSessionImpl( sessionManager: SessionManager) extends AbstractSession(protocol, user, password, ipAddress, conf, sessionManager) { + val sessionConf: KyuubiConf = sessionManager.getConf + + override val handle: SessionHandle = + conf.get(KYUUBI_SESSION_HANDLE_KEY).map(SessionHandle.fromUUID).getOrElse(SessionHandle()) + + private val username: String = sessionConf + .getOption(KyuubiReservedKeys.KYUUBI_SESSION_USER_KEY).getOrElse(currentUser) + var trinoContext: TrinoContext = _ private var clientSession: ClientSession = _ - private var catalogName: String = null - private var databaseName: String = null - + private var catalogName: String = _ + private var databaseName: String = _ private val sessionEvent = TrinoSessionEvent(this) override def open(): Unit = { - normalizedConf.foreach { - case ("use:catalog", catalog) => catalogName = catalog - case ("use:database", database) => databaseName = database - case _ => // do nothing + + val (useCatalogAndDatabaseConf, _) = normalizedConf.partition { case (k, _) => + Array(USE_CATALOG, USE_DATABASE).contains(k) } - val httpClient = new OkHttpClient.Builder().build() + useCatalogAndDatabaseConf.foreach { + case (USE_CATALOG, catalog) => catalogName = catalog + case (USE_DATABASE, database) => databaseName = database + } + if (catalogName == null) { + catalogName = sessionConf.get(KyuubiConf.ENGINE_TRINO_CONNECTION_CATALOG) + .getOrElse(throw KyuubiSQLException("Trino default catalog can not be null!")) + } clientSession = createClientSession() - trinoContext = TrinoContext(httpClient, clientSession) + trinoContext = TrinoContext(createHttpClient(), clientSession) super.open() EventBus.post(sessionEvent) } private def createClientSession(): ClientSession = { - val sessionConf = sessionManager.getConf val connectionUrl = sessionConf.get(KyuubiConf.ENGINE_TRINO_CONNECTION_URL).getOrElse( throw KyuubiSQLException("Trino server url can not be null!")) - if (catalogName == null) { - catalogName = sessionConf.get( - KyuubiConf.ENGINE_TRINO_CONNECTION_CATALOG).getOrElse( - throw KyuubiSQLException("Trino default catalog can not be null!")) - } - - val user = sessionConf - .getOption(KyuubiReservedKeys.KYUUBI_SESSION_USER_KEY).getOrElse(currentUser) val clientRequestTimeout = sessionConf.get(TrinoConf.CLIENT_REQUEST_TIMEOUT) + val properties = getTrinoSessionConf(sessionConf).asJava + new ClientSession( URI.create(connectionUrl), - user, + username, Optional.empty(), "kyuubi", Optional.empty(), @@ -98,7 +108,7 @@ class TrinoSessionImpl( Locale.getDefault, Collections.emptyMap(), Collections.emptyMap(), - Collections.emptyMap(), + properties, Collections.emptyMap(), Collections.emptyMap(), null, @@ -106,6 +116,37 @@ class TrinoSessionImpl( true) } + private def createHttpClient(): OkHttpClient = { + val keystorePath = sessionConf.get(KyuubiConf.ENGINE_TRINO_CONNECTION_KEYSTORE_PATH) + val keystorePassword = sessionConf.get(KyuubiConf.ENGINE_TRINO_CONNECTION_KEYSTORE_PASSWORD) + val keystoreType = sessionConf.get(KyuubiConf.ENGINE_TRINO_CONNECTION_KEYSTORE_TYPE) + val truststorePath = sessionConf.get(KyuubiConf.ENGINE_TRINO_CONNECTION_TRUSTSTORE_PATH) + val truststorePassword = sessionConf.get(KyuubiConf.ENGINE_TRINO_CONNECTION_TRUSTSTORE_PASSWORD) + val truststoreType = sessionConf.get(KyuubiConf.ENGINE_TRINO_CONNECTION_TRUSTSTORE_TYPE) + + val serverScheme = clientSession.getServer.getScheme + + val builder = new OkHttpClient.Builder() + + OkHttpUtil.setupSsl( + builder, + Optional.ofNullable(keystorePath.orNull), + Optional.ofNullable(keystorePassword.orNull), + Optional.ofNullable(keystoreType.orNull), + Optional.ofNullable(truststorePath.orNull), + Optional.ofNullable(truststorePassword.orNull), + Optional.ofNullable(truststoreType.orNull)) + + sessionConf.get(KyuubiConf.ENGINE_TRINO_CONNECTION_PASSWORD).foreach { password => + require( + serverScheme.equalsIgnoreCase("https"), + "Trino engine using username/password requires HTTPS to be enabled") + builder.addInterceptor(OkHttpUtil.basicAuth(username, password)) + } + + builder.build() + } + override protected def runOperation(operation: Operation): OperationHandle = { sessionEvent.totalOperations += 1 super.runOperation(operation) @@ -133,6 +174,12 @@ class TrinoSessionImpl( resultSet.next().head.toString } + private def getTrinoSessionConf(sessionConf: KyuubiConf): Map[String, String] = { + val trinoSessionConf = sessionConf.getAll.filterKeys(_.startsWith("trino.")) + .map { case (k, v) => (k.stripPrefix("trino."), v) } + trinoSessionConf.toMap + } + override def close(): Unit = { sessionEvent.endTime = System.currentTimeMillis() EventBus.post(sessionEvent) diff --git a/externals/kyuubi-trino-engine/src/main/scala/org/apache/kyuubi/engine/trino/session/TrinoSessionManager.scala b/externals/kyuubi-trino-engine/src/main/scala/org/apache/kyuubi/engine/trino/session/TrinoSessionManager.scala index 6d56d5c05..e18b8f758 100644 --- a/externals/kyuubi-trino-engine/src/main/scala/org/apache/kyuubi/engine/trino/session/TrinoSessionManager.scala +++ b/externals/kyuubi-trino-engine/src/main/scala/org/apache/kyuubi/engine/trino/session/TrinoSessionManager.scala @@ -20,6 +20,7 @@ package org.apache.kyuubi.engine.trino.session import org.apache.hive.service.rpc.thrift.TProtocolVersion import org.apache.kyuubi.config.KyuubiConf.ENGINE_SHARE_LEVEL +import org.apache.kyuubi.config.KyuubiReservedKeys.KYUUBI_SESSION_HANDLE_KEY import org.apache.kyuubi.engine.ShareLevel import org.apache.kyuubi.engine.trino.TrinoSqlEngine import org.apache.kyuubi.engine.trino.operation.TrinoOperationManager @@ -36,7 +37,10 @@ class TrinoSessionManager password: String, ipAddress: String, conf: Map[String, String]): Session = { - new TrinoSessionImpl(protocol, user, password, ipAddress, conf, this) + conf.get(KYUUBI_SESSION_HANDLE_KEY).map(SessionHandle.fromUUID).flatMap( + getSessionOption).getOrElse { + new TrinoSessionImpl(protocol, user, password, ipAddress, conf, this) + } } override def closeSession(sessionHandle: SessionHandle): Unit = { diff --git a/externals/kyuubi-trino-engine/src/test/resources/log4j2-test.xml b/externals/kyuubi-trino-engine/src/test/resources/log4j2-test.xml index bfc40dd6d..3110216c1 100644 --- a/externals/kyuubi-trino-engine/src/test/resources/log4j2-test.xml +++ b/externals/kyuubi-trino-engine/src/test/resources/log4j2-test.xml @@ -21,14 +21,14 @@ - + - + diff --git a/externals/kyuubi-trino-engine/src/test/scala/org/apache/kyuubi/engine/trino/TrinoStatementSuite.scala b/externals/kyuubi-trino-engine/src/test/scala/org/apache/kyuubi/engine/trino/TrinoStatementSuite.scala index fc9f1af5f..dec753ad4 100644 --- a/externals/kyuubi-trino-engine/src/test/scala/org/apache/kyuubi/engine/trino/TrinoStatementSuite.scala +++ b/externals/kyuubi-trino-engine/src/test/scala/org/apache/kyuubi/engine/trino/TrinoStatementSuite.scala @@ -30,15 +30,15 @@ class TrinoStatementSuite extends WithTrinoContainerServer { assert(schema.size === 1) assert(schema(0).getName === "_col0") - assert(resultSet.toIterator.hasNext) - assert(resultSet.toIterator.next() === List(1)) + assert(resultSet.hasNext) + assert(resultSet.next() === List(1)) val trinoStatement2 = TrinoStatement(trinoContext, kyuubiConf, "show schemas") val schema2 = trinoStatement2.getColumns val resultSet2 = trinoStatement2.execute() assert(schema2.size === 1) - assert(resultSet2.toIterator.hasNext) + assert(resultSet2.hasNext) } } diff --git a/externals/kyuubi-trino-engine/src/test/scala/org/apache/kyuubi/engine/trino/operation/TrinoOperationSuite.scala b/externals/kyuubi-trino-engine/src/test/scala/org/apache/kyuubi/engine/trino/operation/TrinoOperationSuite.scala index a6f125af5..90939a3e4 100644 --- a/externals/kyuubi-trino-engine/src/test/scala/org/apache/kyuubi/engine/trino/operation/TrinoOperationSuite.scala +++ b/externals/kyuubi-trino-engine/src/test/scala/org/apache/kyuubi/engine/trino/operation/TrinoOperationSuite.scala @@ -590,14 +590,14 @@ class TrinoOperationSuite extends WithTrinoEngine with TrinoQueryTests { val tFetchResultsReq1 = new TFetchResultsReq(opHandle, TFetchOrientation.FETCH_NEXT, 1) val tFetchResultsResp1 = client.FetchResults(tFetchResultsReq1) assert(tFetchResultsResp1.getStatus.getStatusCode === TStatusCode.SUCCESS_STATUS) - val idSeq1 = tFetchResultsResp1.getResults.getColumns.get(0).getI32Val.getValues.asScala.toSeq + val idSeq1 = tFetchResultsResp1.getResults.getColumns.get(0).getI32Val.getValues.asScala assertResult(Seq(0L))(idSeq1) // fetch next from first row val tFetchResultsReq2 = new TFetchResultsReq(opHandle, TFetchOrientation.FETCH_NEXT, 1) val tFetchResultsResp2 = client.FetchResults(tFetchResultsReq2) assert(tFetchResultsResp2.getStatus.getStatusCode === TStatusCode.SUCCESS_STATUS) - val idSeq2 = tFetchResultsResp2.getResults.getColumns.get(0).getI32Val.getValues.asScala.toSeq + val idSeq2 = tFetchResultsResp2.getResults.getColumns.get(0).getI32Val.getValues.asScala assertResult(Seq(1L))(idSeq2) val tFetchResultsReq3 = new TFetchResultsReq(opHandle, TFetchOrientation.FETCH_PRIOR, 1) @@ -607,7 +607,7 @@ class TrinoOperationSuite extends WithTrinoEngine with TrinoQueryTests { } else { assert(tFetchResultsResp3.getStatus.getStatusCode === TStatusCode.SUCCESS_STATUS) val idSeq3 = - tFetchResultsResp3.getResults.getColumns.get(0).getI32Val.getValues.asScala.toSeq + tFetchResultsResp3.getResults.getColumns.get(0).getI32Val.getValues.asScala assertResult(Seq(0L))(idSeq3) } @@ -618,7 +618,7 @@ class TrinoOperationSuite extends WithTrinoEngine with TrinoQueryTests { } else { assert(tFetchResultsResp4.getStatus.getStatusCode === TStatusCode.SUCCESS_STATUS) val idSeq4 = - tFetchResultsResp4.getResults.getColumns.get(0).getI32Val.getValues.asScala.toSeq + tFetchResultsResp4.getResults.getColumns.get(0).getI32Val.getValues.asScala assertResult(Seq(0L, 1L))(idSeq4) } } @@ -771,8 +771,8 @@ class TrinoOperationSuite extends WithTrinoEngine with TrinoQueryTests { assert(schema.size === 1) assert(schema(0).getName === "_col0") - assert(resultSet.toIterator.hasNext) - version = resultSet.toIterator.next().head.toString + assert(resultSet.hasNext) + version = resultSet.next().head.toString } version } diff --git a/integration-tests/kyuubi-flink-it/pom.xml b/integration-tests/kyuubi-flink-it/pom.xml index 7f9a84a85..15699be1d 100644 --- a/integration-tests/kyuubi-flink-it/pom.xml +++ b/integration-tests/kyuubi-flink-it/pom.xml @@ -21,11 +21,11 @@ org.apache.kyuubi integration-tests - 1.7.0-SNAPSHOT + 1.9.0-SNAPSHOT ../pom.xml - kyuubi-flink-it_2.12 + kyuubi-flink-it_${scala.binary.version} Kyuubi Test Flink SQL IT https://kyuubi.apache.org/ @@ -75,10 +75,45 @@ org.apache.flink - flink-table-runtime${flink.module.scala.suffix} + flink-table-runtime + test + + + + + org.apache.hadoop + hadoop-client-minicluster + test + + + + org.bouncycastle + bcprov-jdk15on + test + + + + org.bouncycastle + bcpkix-jdk15on + test + + + + jakarta.activation + jakarta.activation-api + test + + + + jakarta.xml.bind + jakarta.xml.bind-api test + + target/scala-${scala.binary.version}/classes + target/scala-${scala.binary.version}/test-classes + diff --git a/integration-tests/kyuubi-flink-it/src/test/resources/log4j2-test.xml b/integration-tests/kyuubi-flink-it/src/test/resources/log4j2-test.xml index bfc40dd6d..3110216c1 100644 --- a/integration-tests/kyuubi-flink-it/src/test/resources/log4j2-test.xml +++ b/integration-tests/kyuubi-flink-it/src/test/resources/log4j2-test.xml @@ -21,14 +21,14 @@ - + - + diff --git a/integration-tests/kyuubi-flink-it/src/test/scala/org/apache/kyuubi/it/flink/WithKyuubiServerAndYarnMiniCluster.scala b/integration-tests/kyuubi-flink-it/src/test/scala/org/apache/kyuubi/it/flink/WithKyuubiServerAndYarnMiniCluster.scala new file mode 100644 index 000000000..de9a8ae2d --- /dev/null +++ b/integration-tests/kyuubi-flink-it/src/test/scala/org/apache/kyuubi/it/flink/WithKyuubiServerAndYarnMiniCluster.scala @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.it.flink + +import java.io.{File, FileWriter} +import java.nio.file.Paths + +import org.apache.hadoop.yarn.conf.YarnConfiguration + +import org.apache.kyuubi.{KyuubiFunSuite, Utils, WithKyuubiServer} +import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.config.KyuubiConf.KYUUBI_ENGINE_ENV_PREFIX +import org.apache.kyuubi.server.{MiniDFSService, MiniYarnService} + +trait WithKyuubiServerAndYarnMiniCluster extends KyuubiFunSuite with WithKyuubiServer { + + val kyuubiHome: String = Utils.getCodeSourceLocation(getClass).split("integration-tests").head + + override protected val conf: KyuubiConf = new KyuubiConf(false) + + protected var miniHdfsService: MiniDFSService = _ + + protected var miniYarnService: MiniYarnService = _ + + private val yarnConf: YarnConfiguration = { + val yarnConfig = new YarnConfiguration() + + // configurations copied from org.apache.flink.yarn.YarnTestBase + yarnConfig.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 32) + yarnConfig.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, 4096) + + yarnConfig.setBoolean(YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME, true) + yarnConfig.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2) + yarnConfig.setInt(YarnConfiguration.RM_MAX_COMPLETED_APPLICATIONS, 2) + yarnConfig.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, 4) + yarnConfig.setInt(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC, 3600) + yarnConfig.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, false) + // memory is overwritten in the MiniYARNCluster. + // so we have to change the number of cores for testing. + yarnConfig.setInt(YarnConfiguration.NM_VCORES, 666) + yarnConfig.setFloat(YarnConfiguration.NM_MAX_PER_DISK_UTILIZATION_PERCENTAGE, 99.0f) + yarnConfig.setInt(YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS, 1000) + yarnConfig.setInt(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, 5000) + + // capacity-scheduler.xml is missing in hadoop-client-minicluster so this is a workaround + yarnConfig.set("yarn.scheduler.capacity.root.queues", "default,four_cores_queue") + + yarnConfig.setInt("yarn.scheduler.capacity.root.default.capacity", 100) + yarnConfig.setFloat("yarn.scheduler.capacity.root.default.user-limit-factor", 1) + yarnConfig.setInt("yarn.scheduler.capacity.root.default.maximum-capacity", 100) + yarnConfig.set("yarn.scheduler.capacity.root.default.state", "RUNNING") + yarnConfig.set("yarn.scheduler.capacity.root.default.acl_submit_applications", "*") + yarnConfig.set("yarn.scheduler.capacity.root.default.acl_administer_queue", "*") + + yarnConfig.setInt("yarn.scheduler.capacity.root.four_cores_queue.maximum-capacity", 100) + yarnConfig.setInt("yarn.scheduler.capacity.root.four_cores_queue.maximum-applications", 10) + yarnConfig.setInt("yarn.scheduler.capacity.root.four_cores_queue.maximum-allocation-vcores", 4) + yarnConfig.setFloat("yarn.scheduler.capacity.root.four_cores_queue.user-limit-factor", 1) + yarnConfig.set("yarn.scheduler.capacity.root.four_cores_queue.acl_submit_applications", "*") + yarnConfig.set("yarn.scheduler.capacity.root.four_cores_queue.acl_administer_queue", "*") + + yarnConfig.setInt("yarn.scheduler.capacity.node-locality-delay", -1) + // Set bind host to localhost to avoid java.net.BindException + yarnConfig.set(YarnConfiguration.RM_BIND_HOST, "localhost") + yarnConfig.set(YarnConfiguration.NM_BIND_HOST, "localhost") + + yarnConfig + } + + override def beforeAll(): Unit = { + miniHdfsService = new MiniDFSService() + miniHdfsService.initialize(conf) + miniHdfsService.start() + + val hdfsServiceUrl = s"hdfs://localhost:${miniHdfsService.getDFSPort}" + yarnConf.set("fs.defaultFS", hdfsServiceUrl) + yarnConf.addResource(miniHdfsService.getHadoopConf) + + val cp = System.getProperty("java.class.path") + // exclude kyuubi flink engine jar that has SPI for EmbeddedExecutorFactory + // which can't be initialized on the client side + val hadoopJars = cp.split(":").filter(s => !s.contains("flink")) + val hadoopClasspath = hadoopJars.mkString(":") + yarnConf.set("yarn.application.classpath", hadoopClasspath) + + miniYarnService = new MiniYarnService() + miniYarnService.setYarnConf(yarnConf) + miniYarnService.initialize(conf) + miniYarnService.start() + + val hadoopConfDir = Utils.createTempDir().toFile + val writer = new FileWriter(new File(hadoopConfDir, "core-site.xml")) + yarnConf.writeXml(writer) + writer.close() + + val flinkHome = { + val candidates = Paths.get(kyuubiHome, "externals", "kyuubi-download", "target") + .toFile.listFiles(f => f.getName.contains("flink")) + if (candidates == null) None else candidates.map(_.toPath).headOption + } + if (flinkHome.isEmpty) { + throw new IllegalStateException(s"Flink home not found in $kyuubiHome/externals") + } + + conf.set(s"$KYUUBI_ENGINE_ENV_PREFIX.KYUUBI_HOME", kyuubiHome) + conf.set(s"$KYUUBI_ENGINE_ENV_PREFIX.FLINK_HOME", flinkHome.get.toString) + conf.set( + s"$KYUUBI_ENGINE_ENV_PREFIX.FLINK_CONF_DIR", + s"${flinkHome.get.toString}${File.separator}conf") + conf.set(s"$KYUUBI_ENGINE_ENV_PREFIX.HADOOP_CLASSPATH", hadoopClasspath) + conf.set(s"$KYUUBI_ENGINE_ENV_PREFIX.HADOOP_CONF_DIR", hadoopConfDir.getAbsolutePath) + conf.set(s"flink.containerized.master.env.HADOOP_CLASSPATH", hadoopClasspath) + conf.set(s"flink.containerized.master.env.HADOOP_CONF_DIR", hadoopConfDir.getAbsolutePath) + conf.set(s"flink.containerized.taskmanager.env.HADOOP_CONF_DIR", hadoopConfDir.getAbsolutePath) + + super.beforeAll() + } + + override def afterAll(): Unit = { + super.afterAll() + if (miniYarnService != null) { + miniYarnService.stop() + miniYarnService = null + } + if (miniHdfsService != null) { + miniHdfsService.stop() + miniHdfsService = null + } + } +} diff --git a/integration-tests/kyuubi-flink-it/src/test/scala/org/apache/kyuubi/it/flink/operation/FlinkOperationSuite.scala b/integration-tests/kyuubi-flink-it/src/test/scala/org/apache/kyuubi/it/flink/operation/FlinkOperationSuite.scala index 893e0020a..55476bfd0 100644 --- a/integration-tests/kyuubi-flink-it/src/test/scala/org/apache/kyuubi/it/flink/operation/FlinkOperationSuite.scala +++ b/integration-tests/kyuubi-flink-it/src/test/scala/org/apache/kyuubi/it/flink/operation/FlinkOperationSuite.scala @@ -31,7 +31,7 @@ class FlinkOperationSuite extends WithKyuubiServerAndFlinkMiniCluster override val conf: KyuubiConf = KyuubiConf() .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME", kyuubiHome) .set(ENGINE_TYPE, "FLINK_SQL") - .set("flink.parallelism.default", "6") + .set("flink.parallelism.default", "2") override protected def jdbcUrl: String = getJdbcUrl @@ -72,7 +72,7 @@ class FlinkOperationSuite extends WithKyuubiServerAndFlinkMiniCluster var success = false while (resultSet.next() && !success) { if (resultSet.getString(1) == "parallelism.default" && - resultSet.getString(2) == "6") { + resultSet.getString(2) == "2") { success = true } } diff --git a/integration-tests/kyuubi-flink-it/src/test/scala/org/apache/kyuubi/it/flink/operation/FlinkOperationSuiteOnYarn.scala b/integration-tests/kyuubi-flink-it/src/test/scala/org/apache/kyuubi/it/flink/operation/FlinkOperationSuiteOnYarn.scala new file mode 100644 index 000000000..ee6b9bb98 --- /dev/null +++ b/integration-tests/kyuubi-flink-it/src/test/scala/org/apache/kyuubi/it/flink/operation/FlinkOperationSuiteOnYarn.scala @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.it.flink.operation + +import org.apache.hive.service.rpc.thrift.{TGetInfoReq, TGetInfoType} + +import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.config.KyuubiConf._ +import org.apache.kyuubi.it.flink.WithKyuubiServerAndYarnMiniCluster +import org.apache.kyuubi.operation.HiveJDBCTestHelper +import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant.TABLE_CAT + +class FlinkOperationSuiteOnYarn extends WithKyuubiServerAndYarnMiniCluster + with HiveJDBCTestHelper { + + override protected def jdbcUrl: String = { + // delay the access to thrift service because the thrift service + // may not be ready although it's registered + Thread.sleep(3000L) + getJdbcUrl + } + + override def beforeAll(): Unit = { + conf + .set(s"$KYUUBI_ENGINE_ENV_PREFIX.$KYUUBI_HOME", kyuubiHome) + .set(ENGINE_TYPE, "FLINK_SQL") + .set("flink.execution.target", "yarn-application") + .set("flink.parallelism.default", "2") + super.beforeAll() + } + + test("get catalogs for flink sql") { + withJdbcStatement() { statement => + val meta = statement.getConnection.getMetaData + val catalogs = meta.getCatalogs + val expected = Set("default_catalog").toIterator + while (catalogs.next()) { + assert(catalogs.getString(TABLE_CAT) === expected.next()) + } + assert(!expected.hasNext) + assert(!catalogs.next()) + } + } + + test("execute statement - create/alter/drop table") { + withJdbcStatement() { statement => + statement.executeQuery("create table tbl_a (a string) with ('connector' = 'blackhole')") + assert(statement.execute("alter table tbl_a rename to tbl_b")) + assert(statement.execute("drop table tbl_b")) + } + } + + test("execute statement - select column name with dots") { + withJdbcStatement() { statement => + val resultSet = statement.executeQuery("select 'tmp.hello'") + assert(resultSet.next()) + assert(resultSet.getString(1) === "tmp.hello") + } + } + + test("set kyuubi conf into flink conf") { + withJdbcStatement() { statement => + val resultSet = statement.executeQuery("SET") + // Flink does not support set key without value currently, + // thus read all rows to find the desired one + var success = false + while (resultSet.next() && !success) { + if (resultSet.getString(1) == "parallelism.default" && + resultSet.getString(2) == "2") { + success = true + } + } + assert(success) + } + } + + test("server info provider - server") { + withSessionConf(Map(KyuubiConf.SERVER_INFO_PROVIDER.key -> "SERVER"))()() { + withSessionHandle { (client, handle) => + val req = new TGetInfoReq() + req.setSessionHandle(handle) + req.setInfoType(TGetInfoType.CLI_DBMS_NAME) + assert(client.GetInfo(req).getInfoValue.getStringValue === "Apache Kyuubi") + } + } + } + + test("server info provider - engine") { + withSessionConf(Map(KyuubiConf.SERVER_INFO_PROVIDER.key -> "ENGINE"))()() { + withSessionHandle { (client, handle) => + val req = new TGetInfoReq() + req.setSessionHandle(handle) + req.setInfoType(TGetInfoType.CLI_DBMS_NAME) + assert(client.GetInfo(req).getInfoValue.getStringValue === "Apache Flink") + } + } + } +} diff --git a/integration-tests/kyuubi-hive-it/pom.xml b/integration-tests/kyuubi-hive-it/pom.xml index 8b9813a2b..c4e9f320c 100644 --- a/integration-tests/kyuubi-hive-it/pom.xml +++ b/integration-tests/kyuubi-hive-it/pom.xml @@ -21,11 +21,11 @@ org.apache.kyuubi integration-tests - 1.7.0-SNAPSHOT + 1.9.0-SNAPSHOT ../pom.xml - kyuubi-hive-it_2.12 + kyuubi-hive-it_${scala.binary.version} Kyuubi Test Hive IT https://kyuubi.apache.org/ @@ -69,4 +69,9 @@ test + + + target/scala-${scala.binary.version}/classes + target/scala-${scala.binary.version}/test-classes + diff --git a/integration-tests/kyuubi-hive-it/src/test/resources/log4j2-test.xml b/integration-tests/kyuubi-hive-it/src/test/resources/log4j2-test.xml index bfc40dd6d..3110216c1 100644 --- a/integration-tests/kyuubi-hive-it/src/test/resources/log4j2-test.xml +++ b/integration-tests/kyuubi-hive-it/src/test/resources/log4j2-test.xml @@ -21,14 +21,14 @@ - + - + diff --git a/integration-tests/kyuubi-hive-it/src/test/scala/org/apache/kyuubi/it/hive/operation/KyuubiOperationHiveEnginePerUserSuite.scala b/integration-tests/kyuubi-hive-it/src/test/scala/org/apache/kyuubi/it/hive/operation/KyuubiOperationHiveEnginePerUserSuite.scala index a4e6bb150..07e2bc0f2 100644 --- a/integration-tests/kyuubi-hive-it/src/test/scala/org/apache/kyuubi/it/hive/operation/KyuubiOperationHiveEnginePerUserSuite.scala +++ b/integration-tests/kyuubi-hive-it/src/test/scala/org/apache/kyuubi/it/hive/operation/KyuubiOperationHiveEnginePerUserSuite.scala @@ -61,4 +61,21 @@ class KyuubiOperationHiveEnginePerUserSuite extends WithKyuubiServer with HiveEn } } } + + test("kyuubi defined function - system_user, session_user") { + withJdbcStatement("hive_engine_test") { statement => + val rs = statement.executeQuery("SELECT system_user(), session_user()") + assert(rs.next()) + assert(rs.getString(1) === Utils.currentUser) + assert(rs.getString(2) === Utils.currentUser) + } + } + + test("kyuubi defined function - engine_id") { + withJdbcStatement("hive_engine_test") { statement => + val rs = statement.executeQuery("SELECT engine_id()") + assert(rs.next()) + assert(rs.getString(1).nonEmpty) + } + } } diff --git a/integration-tests/kyuubi-jdbc-it/pom.xml b/integration-tests/kyuubi-jdbc-it/pom.xml index 0aef12fb3..95ffd2038 100644 --- a/integration-tests/kyuubi-jdbc-it/pom.xml +++ b/integration-tests/kyuubi-jdbc-it/pom.xml @@ -21,11 +21,11 @@ org.apache.kyuubi integration-tests - 1.7.0-SNAPSHOT + 1.9.0-SNAPSHOT ../pom.xml - kyuubi-jdbc-it_2.12 + kyuubi-jdbc-it_${scala.binary.version} Kyuubi Test Jdbc IT https://kyuubi.apache.org/ @@ -114,5 +114,7 @@ + target/scala-${scala.binary.version}/classes + target/scala-${scala.binary.version}/test-classes diff --git a/integration-tests/kyuubi-jdbc-it/src/test/resources/log4j2-test.xml b/integration-tests/kyuubi-jdbc-it/src/test/resources/log4j2-test.xml index bfc40dd6d..3110216c1 100644 --- a/integration-tests/kyuubi-jdbc-it/src/test/resources/log4j2-test.xml +++ b/integration-tests/kyuubi-jdbc-it/src/test/resources/log4j2-test.xml @@ -21,14 +21,14 @@ - + - + diff --git a/integration-tests/kyuubi-kubernetes-it/pom.xml b/integration-tests/kyuubi-kubernetes-it/pom.xml index cb04e73c1..a4334e497 100644 --- a/integration-tests/kyuubi-kubernetes-it/pom.xml +++ b/integration-tests/kyuubi-kubernetes-it/pom.xml @@ -15,17 +15,15 @@ ~ See the License for the specific language governing permissions and ~ limitations under the License. --> - - + 4.0.0 org.apache.kyuubi integration-tests - 1.7.0-SNAPSHOT + 1.9.0-SNAPSHOT ../pom.xml - 4.0.0 kubernetes-integration-tests_2.12 Kyuubi Test Kubernetes IT @@ -62,12 +60,6 @@ test - - io.fabric8 - kubernetes-client - test - - org.apache.hadoop hadoop-client-minicluster diff --git a/integration-tests/kyuubi-kubernetes-it/src/test/resources/log4j2-test.xml b/integration-tests/kyuubi-kubernetes-it/src/test/resources/log4j2-test.xml index bfc40dd6d..3110216c1 100644 --- a/integration-tests/kyuubi-kubernetes-it/src/test/resources/log4j2-test.xml +++ b/integration-tests/kyuubi-kubernetes-it/src/test/resources/log4j2-test.xml @@ -21,14 +21,14 @@ - + - + diff --git a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/MiniKube.scala b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/MiniKube.scala index cd373873a..f4cd557bb 100644 --- a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/MiniKube.scala +++ b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/MiniKube.scala @@ -17,7 +17,11 @@ package org.apache.kyuubi.kubernetes.test -import io.fabric8.kubernetes.client.{Config, DefaultKubernetesClient} +import io.fabric8.kubernetes.client.{Config, KubernetesClient, KubernetesClientBuilder} +import io.fabric8.kubernetes.client.okhttp.OkHttpClientFactory +import okhttp3.{Dispatcher, OkHttpClient} + +import org.apache.kyuubi.util.ThreadUtils /** * This code copied from Aapache Spark @@ -44,7 +48,7 @@ object MiniKube { executeMinikube(true, "ip").head } - def getKubernetesClient: DefaultKubernetesClient = { + def getKubernetesClient: KubernetesClient = { // only the three-part version number is matched (the optional suffix like "-beta.0" is dropped) val versionArrayOpt = "\\d+\\.\\d+\\.\\d+".r .findFirstIn(minikubeVersionString.split(VERSION_PREFIX)(1)) @@ -65,7 +69,18 @@ object MiniKube { "For minikube version a three-part version number is expected (the optional " + "non-numeric suffix is intentionally dropped)") } + // https://github.com/fabric8io/kubernetes-client/issues/3547 + val dispatcher = new Dispatcher( + ThreadUtils.newDaemonCachedThreadPool("kubernetes-dispatcher")) + val factoryWithCustomDispatcher = new OkHttpClientFactory() { + override protected def additionalConfig(builder: OkHttpClient.Builder): Unit = { + builder.dispatcher(dispatcher) + } + } - new DefaultKubernetesClient(Config.autoConfigure("minikube")) + new KubernetesClientBuilder() + .withConfig(Config.autoConfigure("minikube")) + .withHttpClientFactory(factoryWithCustomDispatcher) + .build() } } diff --git a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/WithKyuubiServerOnKubernetes.scala b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/WithKyuubiServerOnKubernetes.scala index ed9cbce09..595fdd431 100644 --- a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/WithKyuubiServerOnKubernetes.scala +++ b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/WithKyuubiServerOnKubernetes.scala @@ -18,14 +18,14 @@ package org.apache.kyuubi.kubernetes.test import io.fabric8.kubernetes.api.model.Pod -import io.fabric8.kubernetes.client.DefaultKubernetesClient +import io.fabric8.kubernetes.client.KubernetesClient import org.apache.kyuubi.KyuubiFunSuite trait WithKyuubiServerOnKubernetes extends KyuubiFunSuite { protected def connectionConf: Map[String, String] = Map.empty - lazy val miniKubernetesClient: DefaultKubernetesClient = MiniKube.getKubernetesClient + lazy val miniKubernetesClient: KubernetesClient = MiniKube.getKubernetesClient lazy val kyuubiPod: Pod = miniKubernetesClient.pods().withName("kyuubi-test").get() lazy val kyuubiServerIp: String = kyuubiPod.getStatus.getPodIP lazy val miniKubeIp: String = MiniKube.getIp diff --git a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala index c8894679d..95e15e6eb 100644 --- a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala +++ b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/deployment/KyuubiOnKubernetesTestsSuite.scala @@ -54,7 +54,9 @@ class KyuubiOnKubernetesWithSparkTestsBase extends WithKyuubiServerOnKubernetes super.connectionConf ++ Map( "spark.master" -> s"k8s://$miniKubeApiMaster", - "spark.kubernetes.container.image" -> "apache/spark:3.3.1", + // We should update spark docker image in ./github/workflows/master.yml at the same time + "spark.kubernetes.container.image" -> "apache/spark:3.4.1", + "spark.kubernetes.container.image.pullPolicy" -> "IfNotPresent", "spark.executor.memory" -> "512M", "spark.driver.memory" -> "1024M", "spark.kubernetes.driver.request.cores" -> "250m", diff --git a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala index 798618e4c..09532efe3 100644 --- a/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala +++ b/integration-tests/kyuubi-kubernetes-it/src/test/scala/org/apache/kyuubi/kubernetes/test/spark/SparkOnKubernetesTestsSuite.scala @@ -17,21 +17,23 @@ package org.apache.kyuubi.kubernetes.test.spark -import scala.collection.JavaConverters._ +import java.util.UUID + import scala.concurrent.duration._ import org.apache.hadoop.conf.Configuration import org.apache.hadoop.net.NetUtils -import org.apache.kyuubi.{BatchTestHelper, KyuubiException, Logging, Utils, WithKyuubiServer, WithSimpleDFSService} +import org.apache.kyuubi._ +import org.apache.kyuubi.client.util.BatchUtils._ import org.apache.kyuubi.config.KyuubiConf -import org.apache.kyuubi.config.KyuubiConf.FRONTEND_THRIFT_BINARY_BIND_HOST -import org.apache.kyuubi.engine.{ApplicationInfo, ApplicationOperation, KubernetesApplicationOperation} +import org.apache.kyuubi.config.KyuubiConf._ +import org.apache.kyuubi.engine.{ApplicationInfo, ApplicationManagerInfo, ApplicationOperation, KubernetesApplicationOperation} import org.apache.kyuubi.engine.ApplicationState.{FAILED, NOT_FOUND, RUNNING} import org.apache.kyuubi.engine.spark.SparkProcessBuilder import org.apache.kyuubi.kubernetes.test.MiniKube import org.apache.kyuubi.operation.SparkQueryTests -import org.apache.kyuubi.session.{KyuubiBatchSessionImpl, KyuubiSessionManager} +import org.apache.kyuubi.session.KyuubiSessionManager import org.apache.kyuubi.util.Validator.KUBERNETES_EXECUTOR_POD_NAME_PREFIX import org.apache.kyuubi.zookeeper.ZookeeperConf.ZK_CLIENT_PORT_ADDRESS @@ -41,19 +43,23 @@ abstract class SparkOnKubernetesSuiteBase MiniKube.getKubernetesClient.getMasterUrl.toString } + protected val appMgrInfo = + ApplicationManagerInfo(Some(s"k8s://$apiServerAddress"), Some("minikube"), None) + protected def sparkOnK8sConf: KyuubiConf = { // TODO Support more Spark version // Spark official docker image: https://hub.docker.com/r/apache/spark/tags KyuubiConf().set("spark.master", s"k8s://$apiServerAddress") - .set("spark.kubernetes.container.image", "apache/spark:v3.2.1") + .set("spark.kubernetes.container.image", "apache/spark:3.4.1") .set("spark.kubernetes.container.image.pullPolicy", "IfNotPresent") .set("spark.executor.instances", "1") .set("spark.executor.memory", "512M") .set("spark.driver.memory", "512M") .set("spark.kubernetes.driver.request.cores", "250m") .set("spark.kubernetes.executor.request.cores", "250m") - .set("kyuubi.kubernetes.context", "minikube") - .set("kyuubi.frontend.protocols", "THRIFT_BINARY,REST") + .set(KUBERNETES_CONTEXT.key, "minikube") + .set(FRONTEND_PROTOCOLS.key, "THRIFT_BINARY,REST") + .set(ENGINE_INIT_TIMEOUT.key, "PT10M") } } @@ -122,6 +128,7 @@ class SparkClusterModeOnKubernetesSuite override protected def jdbcUrl: String = getJdbcUrl } +// [KYUUBI #4467] KubernetesApplicationOperator doesn't support client mode class KyuubiOperationKubernetesClusterClientModeSuite extends SparkClientModeOnKubernetesSuiteBase { private lazy val k8sOperation: KubernetesApplicationOperation = { @@ -133,31 +140,39 @@ class KyuubiOperationKubernetesClusterClientModeSuite private def sessionManager: KyuubiSessionManager = server.backendService.sessionManager.asInstanceOf[KyuubiSessionManager] - test("Spark Client Mode On Kubernetes Kyuubi KubernetesApplicationOperation Suite") { - val batchRequest = newSparkBatchRequest(conf.getAll) + ignore("Spark Client Mode On Kubernetes Kyuubi KubernetesApplicationOperation Suite") { + val batchRequest = newSparkBatchRequest(conf.getAll ++ Map( + KYUUBI_BATCH_ID_KEY -> UUID.randomUUID().toString)) val sessionHandle = sessionManager.openBatchSession( "kyuubi", "passwd", "localhost", - batchRequest.getConf.asScala.toMap, batchRequest) eventually(timeout(3.minutes), interval(50.milliseconds)) { - val state = k8sOperation.getApplicationInfoByTag(sessionHandle.identifier.toString) + val state = k8sOperation.getApplicationInfoByTag( + appMgrInfo, + sessionHandle.identifier.toString) assert(state.id != null) assert(state.name != null) assert(state.state == RUNNING) } - val killResponse = k8sOperation.killApplicationByTag(sessionHandle.identifier.toString) + val killResponse = k8sOperation.killApplicationByTag( + appMgrInfo, + sessionHandle.identifier.toString) assert(killResponse._1) assert(killResponse._2 startsWith "Succeeded to terminate:") - val appInfo = k8sOperation.getApplicationInfoByTag(sessionHandle.identifier.toString) + val appInfo = k8sOperation.getApplicationInfoByTag( + appMgrInfo, + sessionHandle.identifier.toString) assert(appInfo == ApplicationInfo(null, null, NOT_FOUND)) - val failKillResponse = k8sOperation.killApplicationByTag(sessionHandle.identifier.toString) + val failKillResponse = k8sOperation.killApplicationByTag( + appMgrInfo, + sessionHandle.identifier.toString) assert(!failKillResponse._1) assert(failKillResponse._2 === ApplicationOperation.NOT_FOUND) } @@ -193,37 +208,44 @@ class KyuubiOperationKubernetesClusterClusterModeSuite "spark.kubernetes.driver.pod.name", driverPodNamePrefix + "-" + System.currentTimeMillis()) - val batchRequest = newSparkBatchRequest(conf.getAll) + val batchRequest = newSparkBatchRequest(conf.getAll ++ Map( + KYUUBI_BATCH_ID_KEY -> UUID.randomUUID().toString)) val sessionHandle = sessionManager.openBatchSession( "runner", "passwd", "localhost", - batchRequest.getConf.asScala.toMap, batchRequest) - val session = sessionManager.getSession(sessionHandle).asInstanceOf[KyuubiBatchSessionImpl] - val batchJobSubmissionOp = session.batchJobSubmissionOp - - eventually(timeout(3.minutes), interval(50.milliseconds)) { - val appInfo = batchJobSubmissionOp.getOrFetchCurrentApplicationInfo - assert(appInfo.nonEmpty) - assert(appInfo.exists(_.state == RUNNING)) - assert(appInfo.exists(_.name.startsWith(driverPodNamePrefix))) + // wait for driver pod start + eventually(timeout(3.minutes), interval(5.second)) { + // trigger k8sOperation init here + val appInfo = k8sOperation.getApplicationInfoByTag( + appMgrInfo, + sessionHandle.identifier.toString) + assert(appInfo.state == RUNNING) + assert(appInfo.name.startsWith(driverPodNamePrefix)) } - val killResponse = k8sOperation.killApplicationByTag(sessionHandle.identifier.toString) + val killResponse = k8sOperation.killApplicationByTag( + appMgrInfo, + sessionHandle.identifier.toString) assert(killResponse._1) - assert(killResponse._2 startsWith "Operation of deleted appId:") + assert(killResponse._2 endsWith "is completed") + assert(killResponse._2 contains sessionHandle.identifier.toString) eventually(timeout(3.minutes), interval(50.milliseconds)) { - val appInfo = k8sOperation.getApplicationInfoByTag(sessionHandle.identifier.toString) + val appInfo = k8sOperation.getApplicationInfoByTag( + appMgrInfo, + sessionHandle.identifier.toString) // We may kill engine start but not ready // An EOF Error occurred when the driver was starting assert(appInfo.state == FAILED || appInfo.state == NOT_FOUND) } - val failKillResponse = k8sOperation.killApplicationByTag(sessionHandle.identifier.toString) + val failKillResponse = k8sOperation.killApplicationByTag( + appMgrInfo, + sessionHandle.identifier.toString) assert(!failKillResponse._1) } } diff --git a/integration-tests/kyuubi-trino-it/pom.xml b/integration-tests/kyuubi-trino-it/pom.xml index e62e58d1d..c93d43c00 100644 --- a/integration-tests/kyuubi-trino-it/pom.xml +++ b/integration-tests/kyuubi-trino-it/pom.xml @@ -21,11 +21,11 @@ org.apache.kyuubi integration-tests - 1.7.0-SNAPSHOT + 1.9.0-SNAPSHOT ../pom.xml - kyuubi-trino-it_2.12 + kyuubi-trino-it_${scala.binary.version} Kyuubi Test Trino IT https://kyuubi.apache.org/ @@ -88,4 +88,9 @@ + + + target/scala-${scala.binary.version}/classes + target/scala-${scala.binary.version}/test-classes + diff --git a/integration-tests/kyuubi-trino-it/src/test/resources/log4j2-test.xml b/integration-tests/kyuubi-trino-it/src/test/resources/log4j2-test.xml index bfc40dd6d..3110216c1 100644 --- a/integration-tests/kyuubi-trino-it/src/test/resources/log4j2-test.xml +++ b/integration-tests/kyuubi-trino-it/src/test/resources/log4j2-test.xml @@ -21,14 +21,14 @@ - + - + diff --git a/integration-tests/kyuubi-trino-it/src/test/scala/org/apache/kyuubi/it/trino/server/TrinoFrontendSuite.scala b/integration-tests/kyuubi-trino-it/src/test/scala/org/apache/kyuubi/it/trino/server/TrinoFrontendSuite.scala new file mode 100644 index 000000000..7575bf8a9 --- /dev/null +++ b/integration-tests/kyuubi-trino-it/src/test/scala/org/apache/kyuubi/it/trino/server/TrinoFrontendSuite.scala @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.it.trino.server + +import scala.util.control.NonFatal + +import org.apache.kyuubi.WithKyuubiServer +import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.operation.SparkMetadataTests + +/** + * This test is for Trino jdbc driver with Kyuubi Server and Spark engine: + * + * ------------------------------------------------------------- + * | JDBC | + * | Trino-driver ----> Kyuubi Server --> Spark Engine | + * | | + * ------------------------------------------------------------- + */ +class TrinoFrontendSuite extends WithKyuubiServer with SparkMetadataTests { + + test("execute statement - select 11 where 1=1") { + withJdbcStatement() { statement => + val resultSet = statement.executeQuery("SELECT 11 where 1<1") + while (resultSet.next()) { + assert(resultSet.getInt(1) === 11) + } + } + } + + test("execute preparedStatement - select 11 where 1 = 1") { + withJdbcPrepareStatement("select 11 where 1 = ? ") { statement => + statement.setInt(1, 1) + val rs = statement.executeQuery() + while (rs.next()) { + assert(rs.getInt(1) == 11) + } + } + } + + override protected val conf: KyuubiConf = { + KyuubiConf().set(KyuubiConf.FRONTEND_PROTOCOLS, Seq("TRINO")) + } + + override protected def jdbcUrl: String = { + s"jdbc:trino://${server.frontendServices.head.connectionUrl}/;" + } + + // trino jdbc driver requires enable SSL if specify password + override protected val password: String = "" + + override def beforeAll(): Unit = { + super.beforeAll() + // eagerly start spark engine before running test, it's a workaround for trino jdbc driver + // since it does not support changing http connect timeout + try { + withJdbcStatement() { statement => + statement.execute("SELECT 1") + } + } catch { + case NonFatal(_) => + } + } +} diff --git a/integration-tests/kyuubi-zookeeper-it/pom.xml b/integration-tests/kyuubi-zookeeper-it/pom.xml index eaeff5898..869fd40b2 100644 --- a/integration-tests/kyuubi-zookeeper-it/pom.xml +++ b/integration-tests/kyuubi-zookeeper-it/pom.xml @@ -21,11 +21,11 @@ org.apache.kyuubi integration-tests - 1.7.0-SNAPSHOT + 1.9.0-SNAPSHOT ../pom.xml - kyuubi-zookeeper-it_2.12 + kyuubi-zookeeper-it_${scala.binary.version} Kyuubi Test Zookeeper IT https://kyuubi.apache.org/ diff --git a/integration-tests/kyuubi-zookeeper-it/src/test/resources/log4j2-test.xml b/integration-tests/kyuubi-zookeeper-it/src/test/resources/log4j2-test.xml index bfc40dd6d..3110216c1 100644 --- a/integration-tests/kyuubi-zookeeper-it/src/test/resources/log4j2-test.xml +++ b/integration-tests/kyuubi-zookeeper-it/src/test/resources/log4j2-test.xml @@ -21,14 +21,14 @@ - + - + diff --git a/integration-tests/pom.xml b/integration-tests/pom.xml index 4e3431afb..35d0b4f9e 100644 --- a/integration-tests/pom.xml +++ b/integration-tests/pom.xml @@ -21,7 +21,7 @@ org.apache.kyuubi kyuubi-parent - 1.7.0-SNAPSHOT + 1.9.0-SNAPSHOT integration-tests diff --git a/kyuubi-assembly/pom.xml b/kyuubi-assembly/pom.xml index 725126f84..4fa0d9a0f 100644 --- a/kyuubi-assembly/pom.xml +++ b/kyuubi-assembly/pom.xml @@ -22,11 +22,11 @@ org.apache.kyuubi kyuubi-parent - 1.7.0-SNAPSHOT + 1.9.0-SNAPSHOT ../pom.xml - kyuubi-assembly_2.12 + kyuubi-assembly_${scala.binary.version} pom Kyuubi Project Assembly https://kyuubi.apache.org/ @@ -69,28 +69,18 @@ - org.apache.hadoop - hadoop-client-api + org.apache.kyuubi + ${kyuubi-shaded-zookeeper.artifacts} org.apache.hadoop - hadoop-client-runtime - - - - org.apache.curator - curator-framework - - - - org.apache.curator - curator-client + hadoop-client-api - org.apache.curator - curator-recipes + org.apache.hadoop + hadoop-client-runtime diff --git a/kyuubi-common/pom.xml b/kyuubi-common/pom.xml index 26cdc271d..0d5c491b5 100644 --- a/kyuubi-common/pom.xml +++ b/kyuubi-common/pom.xml @@ -21,20 +21,20 @@ org.apache.kyuubi kyuubi-parent - 1.7.0-SNAPSHOT + 1.9.0-SNAPSHOT ../pom.xml - kyuubi-common_2.12 + kyuubi-common_${scala.binary.version} jar Kyuubi Project Common https://kyuubi.apache.org/ - com.vladsch.flexmark - flexmark-all - test + org.apache.kyuubi + kyuubi-util-scala_${scala.binary.version} + ${project.version} @@ -88,6 +88,11 @@ runtime + + org.antlr + ST4 + + org.apache.commons commons-lang3 @@ -123,6 +128,13 @@ HikariCP + + org.apache.kyuubi + kyuubi-util-scala_${scala.binary.version} + ${project.version} + test-jar + + org.apache.hadoop hadoop-minikdc @@ -141,6 +153,12 @@ test + + org.scalatestplus + mockito-4-11_${scala.binary.version} + test + + com.google.guava failureaccess @@ -153,11 +171,23 @@ test + + org.xerial + sqlite-jdbc + test + + com.jakewharton.fliptables fliptables test + + + com.vladsch.flexmark + flexmark-all + test + diff --git a/kyuubi-common/src/main/resources/log4j2-defaults.xml b/kyuubi-common/src/main/resources/log4j2-defaults.xml index 63841959a..7a1a33235 100644 --- a/kyuubi-common/src/main/resources/log4j2-defaults.xml +++ b/kyuubi-common/src/main/resources/log4j2-defaults.xml @@ -21,7 +21,7 @@ - + diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/KyuubiSQLException.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/KyuubiSQLException.scala index a9e486fb2..570ee6d38 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/KyuubiSQLException.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/KyuubiSQLException.scala @@ -26,6 +26,7 @@ import scala.collection.JavaConverters._ import org.apache.hive.service.rpc.thrift.{TStatus, TStatusCode} import org.apache.kyuubi.Utils.stringifyException +import org.apache.kyuubi.util.reflect.DynConstructors /** * @param reason a description of the exception @@ -139,9 +140,10 @@ object KyuubiSQLException { } private def newInstance(className: String, message: String, cause: Throwable): Throwable = { try { - Class.forName(className) - .getConstructor(classOf[String], classOf[Throwable]) - .newInstance(message, cause).asInstanceOf[Throwable] + DynConstructors.builder() + .impl(className, classOf[String], classOf[Throwable]) + .buildChecked[Throwable]() + .newInstance(message, cause) } catch { case _: Exception => new RuntimeException(className + ":" + message, cause) } @@ -154,7 +156,7 @@ object KyuubiSQLException { (i1, i2, i3) } - def toCause(details: Seq[String]): Throwable = { + def toCause(details: Iterable[String]): Throwable = { var ex: Throwable = null if (details != null && details.nonEmpty) { val head = details.head @@ -170,7 +172,7 @@ object KyuubiSQLException { val lineNum = line.substring(i3 + 1).toInt new StackTraceElement(clzName, methodName, fileName, lineNum) } - ex = newInstance(exClz, msg, toCause(details.slice(length + 2, details.length))) + ex = newInstance(exClz, msg, toCause(details.slice(length + 2, details.size))) ex.setStackTrace(stackTraceElements.toArray) } ex diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/Logging.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/Logging.scala index 4944b9fcc..d6dcc8d34 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/Logging.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/Logging.scala @@ -22,9 +22,8 @@ import org.apache.logging.log4j.core.{Logger => Log4jLogger, LoggerContext} import org.apache.logging.log4j.core.config.DefaultConfiguration import org.slf4j.{Logger, LoggerFactory} import org.slf4j.bridge.SLF4JBridgeHandler -import org.slf4j.impl.StaticLoggerBinder -import org.apache.kyuubi.util.ClassUtils +import org.apache.kyuubi.util.reflect.ReflectUtils /** * Simple version of logging adopted from Apache Spark. @@ -54,12 +53,24 @@ trait Logging { } } + def debug(message: => Any, t: Throwable): Unit = { + if (logger.isDebugEnabled) { + logger.debug(message.toString, t) + } + } + def info(message: => Any): Unit = { if (logger.isInfoEnabled) { logger.info(message.toString) } } + def info(message: => Any, t: Throwable): Unit = { + if (logger.isInfoEnabled) { + logger.info(message.toString, t) + } + } + def warn(message: => Any): Unit = { if (logger.isWarnEnabled) { logger.warn(message.toString) @@ -105,16 +116,17 @@ object Logging { // This distinguishes the log4j 1.2 binding, currently // org.slf4j.impl.Log4jLoggerFactory, from the log4j 2.0 binding, currently // org.apache.logging.slf4j.Log4jLoggerFactory - val binderClass = StaticLoggerBinder.getSingleton.getLoggerFactoryClassStr - "org.slf4j.impl.Log4jLoggerFactory".equals(binderClass) + val binderClass = LoggerFactory.getILoggerFactory.getClass.getName + "org.slf4j.impl.Log4jLoggerFactory".equals( + binderClass) || "org.slf4j.impl.Reload4jLoggerFactory".equals(binderClass) } private[kyuubi] def isLog4j2: Boolean = { // This distinguishes the log4j 1.2 binding, currently // org.slf4j.impl.Log4jLoggerFactory, from the log4j 2.0 binding, currently // org.apache.logging.slf4j.Log4jLoggerFactory - val binderClass = StaticLoggerBinder.getSingleton.getLoggerFactoryClassStr - "org.apache.logging.slf4j.Log4jLoggerFactory".equals(binderClass) + "org.apache.logging.slf4j.Log4jLoggerFactory" + .equals(LoggerFactory.getILoggerFactory.getClass.getName) } /** @@ -137,7 +149,7 @@ object Logging { isInterpreter: Boolean, loggerName: String, logger: => Logger): Unit = { - if (ClassUtils.classIsLoadable("org.slf4j.bridge.SLF4JBridgeHandler")) { + if (ReflectUtils.isClassLoadable("org.slf4j.bridge.SLF4JBridgeHandler")) { // Handles configuring the JUL -> SLF4J bridge SLF4JBridgeHandler.removeHandlersForRootLogger() SLF4JBridgeHandler.install() diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala index 7283ea040..accfca4c9 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/Utils.scala @@ -21,9 +21,12 @@ import java.io._ import java.net.{Inet4Address, InetAddress, NetworkInterface} import java.nio.charset.StandardCharsets import java.nio.file.{Files, Path, Paths, StandardCopyOption} +import java.security.PrivilegedAction import java.text.SimpleDateFormat import java.util.{Date, Properties, TimeZone, UUID} +import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicLong +import java.util.concurrent.locks.Lock import scala.collection.JavaConverters._ import scala.sys.process._ @@ -143,20 +146,6 @@ object Utils extends Logging { f.delete() } - /** - * delete file in path with logging - * @param filePath path to file for deletion - * @param errorMessage message as prefix logging with error exception - */ - def deleteFile(filePath: String, errorMessage: String): Unit = { - try { - Files.delete(Paths.get(filePath)) - } catch { - case e: Exception => - error(s"$errorMessage: $filePath ", e) - } - } - /** * Create a temporary directory inside the given parent directory. The directory will be * automatically deleted when the VM shuts down. @@ -215,6 +204,14 @@ object Utils extends Logging { def currentUser: String = UserGroupInformation.getCurrentUser.getShortUserName + def doAs[T]( + proxyUser: String, + realUser: UserGroupInformation = UserGroupInformation.getCurrentUser)(f: () => T): T = { + UserGroupInformation.createProxyUser(proxyUser, realUser).doAs(new PrivilegedAction[T] { + override def run(): T = f() + }) + } + private val shortVersionRegex = """^(\d+\.\d+\.\d+)(.*)?$""".r /** @@ -235,6 +232,11 @@ object Utils extends Logging { */ val isWindows: Boolean = SystemUtils.IS_OS_WINDOWS + /** + * Whether the underlying operating system is MacOS. + */ + val isMac: Boolean = SystemUtils.IS_OS_MAC + /** * Indicates whether Kyuubi is currently running unit tests. */ @@ -401,4 +403,50 @@ object Utils extends Logging { Option(Thread.currentThread().getContextClassLoader).getOrElse(getKyuubiClassLoader) def isOnK8s: Boolean = Files.exists(Paths.get("/var/run/secrets/kubernetes.io")) + + /** + * Return a nice string representation of the exception. It will call "printStackTrace" to + * recursively generate the stack trace including the exception and its causes. + */ + def prettyPrint(e: Throwable): String = { + if (e == null) { + "" + } else { + // Use e.printStackTrace here because e.getStackTrace doesn't include the cause + val stringWriter = new StringWriter() + e.printStackTrace(new PrintWriter(stringWriter)) + stringWriter.toString + } + } + + def withLockRequired[T](lock: Lock)(block: => T): T = { + try { + lock.lock() + block + } finally { + lock.unlock() + } + } + + /** + * Try killing the process gracefully first, then forcibly if process does not exit in + * graceful period. + * + * @param process the being killed process + * @param gracefulPeriod the graceful killing period, in milliseconds + * @return the exit code if process exit normally, None if the process finally was killed + * forcibly + */ + def terminateProcess(process: java.lang.Process, gracefulPeriod: Long): Option[Int] = { + process.destroy() + if (process.waitFor(gracefulPeriod, TimeUnit.MILLISECONDS)) { + Some(process.exitValue()) + } else { + warn(s"Process does not exit after $gracefulPeriod ms, try to forcibly kill. " + + "Staging files generated by the process may be retained!") + process.destroyForcibly() + None + } + } + } diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/ConfigBuilder.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/ConfigBuilder.scala index 62f060a05..d6de40241 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/ConfigBuilder.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/ConfigBuilder.scala @@ -18,11 +18,14 @@ package org.apache.kyuubi.config import java.time.Duration +import java.util.Locale import java.util.regex.PatternSyntaxException import scala.util.{Failure, Success, Try} import scala.util.matching.Regex +import org.apache.kyuubi.util.EnumUtils._ + private[kyuubi] case class ConfigBuilder(key: String) { private[config] var _doc = "" @@ -150,7 +153,7 @@ private[kyuubi] case class ConfigBuilder(key: String) { } } - new TypedConfigBuilder(this, regexFromString(_, this.key), _.toString) + TypedConfigBuilder(this, regexFromString(_, this.key), _.toString) } } @@ -166,6 +169,21 @@ private[kyuubi] case class TypedConfigBuilder[T]( def transform(fn: T => T): TypedConfigBuilder[T] = this.copy(fromStr = s => fn(fromStr(s))) + def transformToUpperCase: TypedConfigBuilder[T] = { + transformString(_.toUpperCase(Locale.ROOT)) + } + + def transformToLowerCase: TypedConfigBuilder[T] = { + transformString(_.toLowerCase(Locale.ROOT)) + } + + private def transformString(fn: String => String): TypedConfigBuilder[T] = { + require(parent._type == "string") + this.asInstanceOf[TypedConfigBuilder[String]] + .transform(fn) + .asInstanceOf[TypedConfigBuilder[T]] + } + /** Checks if the user-provided value for the config matches the validator. */ def checkValue(validator: T => Boolean, errMsg: String): TypedConfigBuilder[T] = { transform { v => @@ -187,10 +205,35 @@ private[kyuubi] case class TypedConfigBuilder[T]( } } + /** Checks if the user-provided value for the config matches the value set of the enumeration. */ + def checkValues(enumeration: Enumeration): TypedConfigBuilder[T] = { + transform { v => + val isValid = v match { + case iter: Iterable[Any] => isValidEnums(enumeration, iter) + case name => isValidEnum(enumeration, name) + } + if (!isValid) { + val actualValueStr = v match { + case iter: Iterable[Any] => iter.mkString(",") + case value => value.toString + } + throw new IllegalArgumentException( + s"The value of ${parent.key} should be one of ${enumeration.values.mkString(", ")}," + + s" but was $actualValueStr") + } + v + } + } + /** Turns the config entry into a sequence of values of the underlying type. */ def toSequence(sp: String = ","): TypedConfigBuilder[Seq[T]] = { parent._type = "seq" - TypedConfigBuilder(parent, strToSeq(_, fromStr, sp), seqToStr(_, toStr)) + TypedConfigBuilder(parent, strToSeq(_, fromStr, sp), iterableToStr(_, toStr)) + } + + def toSet(sp: String = ",", skipBlank: Boolean = true): TypedConfigBuilder[Set[T]] = { + parent._type = "set" + TypedConfigBuilder(parent, strToSet(_, fromStr, sp, skipBlank), iterableToStr(_, toStr)) } def createOptional: OptionalConfigEntry[T] = { diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/ConfigHelpers.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/ConfigHelpers.scala index 225f1b537..525ea2ff4 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/ConfigHelpers.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/ConfigHelpers.scala @@ -17,6 +17,8 @@ package org.apache.kyuubi.config +import org.apache.commons.lang3.StringUtils + import org.apache.kyuubi.Utils object ConfigHelpers { @@ -25,7 +27,11 @@ object ConfigHelpers { Utils.strToSeq(str, sp).map(converter) } - def seqToStr[T](v: Seq[T], stringConverter: T => String): String = { - v.map(stringConverter).mkString(",") + def strToSet[T](str: String, converter: String => T, sp: String, skipBlank: Boolean): Set[T] = { + Utils.strToSeq(str, sp).filter(!skipBlank || StringUtils.isNotBlank(_)).map(converter).toSet + } + + def iterableToStr[T](v: Iterable[T], stringConverter: T => String, sp: String = ","): String = { + v.map(stringConverter).mkString(sp) } } diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala index 6ce84a70d..e52c39865 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiConf.scala @@ -42,7 +42,7 @@ case class KyuubiConf(loadSysDefault: Boolean = true) extends Logging { } if (loadSysDefault) { - val fromSysDefaults = Utils.getSystemProperties.filterKeys(_.startsWith("kyuubi.")) + val fromSysDefaults = Utils.getSystemProperties.filterKeys(_.startsWith("kyuubi.")).toMap loadFromMap(fromSysDefaults) } @@ -103,7 +103,6 @@ case class KyuubiConf(loadSysDefault: Boolean = true) extends Logging { /** unset a parameter from the configuration */ def unset(key: String): KyuubiConf = { - logDeprecationWarning(key) settings.remove(key) this } @@ -135,6 +134,31 @@ case class KyuubiConf(loadSysDefault: Boolean = true) extends Logging { getAllWithPrefix(s"$KYUUBI_BATCH_CONF_PREFIX.$normalizedBatchType", "") } + /** Get the kubernetes conf for specified kubernetes context and namespace. */ + def getKubernetesConf(context: Option[String], namespace: Option[String]): KyuubiConf = { + val conf = this.clone + context.foreach { c => + val contextConf = + getAllWithPrefix(s"$KYUUBI_KUBERNETES_CONF_PREFIX.$c", "").map { case (suffix, value) => + s"$KYUUBI_KUBERNETES_CONF_PREFIX.$suffix" -> value + } + val contextNamespaceConf = namespace.map { ns => + getAllWithPrefix(s"$KYUUBI_KUBERNETES_CONF_PREFIX.$c.$ns", "").map { + case (suffix, value) => + s"$KYUUBI_KUBERNETES_CONF_PREFIX.$suffix" -> value + } + }.getOrElse(Map.empty) + + (contextConf ++ contextNamespaceConf).map { case (key, value) => + conf.set(key, value) + } + conf.set(KUBERNETES_CONTEXT, c) + namespace.foreach(ns => conf.set(KUBERNETES_NAMESPACE, ns)) + conf + } + conf + } + /** * Retrieve key-value pairs from [[KyuubiConf]] starting with `dropped.remainder`, and put them to * the result map with the `dropped` of key being dropped. @@ -189,6 +213,8 @@ case class KyuubiConf(loadSysDefault: Boolean = true) extends Logging { s"and may be removed in the future. $comment") } } + + def isRESTEnabled: Boolean = get(FRONTEND_PROTOCOLS).contains(FrontendProtocols.REST.toString) } /** @@ -206,6 +232,7 @@ object KyuubiConf { final val KYUUBI_HOME = "KYUUBI_HOME" final val KYUUBI_ENGINE_ENV_PREFIX = "kyuubi.engineEnv" final val KYUUBI_BATCH_CONF_PREFIX = "kyuubi.batchConf" + final val KYUUBI_KUBERNETES_CONF_PREFIX = "kyuubi.kubernetes" final val USER_DEFAULTS_CONF_QUOTE = "___" private[this] val kyuubiConfEntriesUpdateLock = new Object @@ -386,12 +413,12 @@ object KyuubiConf { "") .version("1.4.0") .stringConf + .transformToUpperCase .toSequence() - .transform(_.map(_.toUpperCase(Locale.ROOT))) - .checkValue( - _.forall(FrontendProtocols.values.map(_.toString).contains), - s"the frontend protocol should be one or more of ${FrontendProtocols.values.mkString(",")}") - .createWithDefault(Seq(FrontendProtocols.THRIFT_BINARY.toString)) + .checkValues(FrontendProtocols) + .createWithDefault(Seq( + FrontendProtocols.THRIFT_BINARY.toString, + FrontendProtocols.REST.toString)) val FRONTEND_BIND_HOST: OptionalConfigEntry[String] = buildConf("kyuubi.frontend.bind.host") .doc("Hostname or IP of the machine on which to run the frontend services.") @@ -400,6 +427,16 @@ object KyuubiConf { .stringConf .createOptional + val FRONTEND_ADVERTISED_HOST: OptionalConfigEntry[String] = + buildConf("kyuubi.frontend.advertised.host") + .doc("Hostname or IP of the Kyuubi server's frontend services to publish to " + + "external systems such as the service discovery ensemble and metadata store. " + + "Use it when you want to advertise a different hostname or IP than the bind host.") + .version("1.8.0") + .serverOnly + .stringConf + .createOptional + val FRONTEND_THRIFT_BINARY_BIND_HOST: ConfigEntry[Option[String]] = buildConf("kyuubi.frontend.thrift.binary.bind.host") .doc("Hostname or IP of the machine on which to run the thrift frontend service " + @@ -444,13 +481,13 @@ object KyuubiConf { .stringConf .createOptional - val FRONTEND_THRIFT_BINARY_SSL_DISALLOWED_PROTOCOLS: ConfigEntry[Seq[String]] = + val FRONTEND_THRIFT_BINARY_SSL_DISALLOWED_PROTOCOLS: ConfigEntry[Set[String]] = buildConf("kyuubi.frontend.thrift.binary.ssl.disallowed.protocols") .doc("SSL versions to disallow for Kyuubi thrift binary frontend.") .version("1.7.0") .stringConf - .toSequence() - .createWithDefault(Seq("SSLv2", "SSLv3")) + .toSet() + .createWithDefault(Set("SSLv2", "SSLv3")) val FRONTEND_THRIFT_BINARY_SSL_INCLUDE_CIPHER_SUITES: ConfigEntry[Seq[String]] = buildConf("kyuubi.frontend.thrift.binary.ssl.include.ciphersuites") @@ -726,7 +763,7 @@ object KyuubiConf { .stringConf .createWithDefault("X-Real-IP") - val AUTHENTICATION_METHOD: ConfigEntry[Seq[String]] = buildConf("kyuubi.authentication") + val AUTHENTICATION_METHOD: ConfigEntry[Set[String]] = buildConf("kyuubi.authentication") .doc("A comma-separated list of client authentication types." + "
    " + "
  • NOSASL: raw transport.
    • " + @@ -761,18 +798,17 @@ object KyuubiConf { .version("1.0.0") .serverOnly .stringConf - .toSequence() - .transform(_.map(_.toUpperCase(Locale.ROOT))) - .checkValue( - _.forall(AuthTypes.values.map(_.toString).contains), - s"the authentication type should be one or more of ${AuthTypes.values.mkString(",")}") - .createWithDefault(Seq(AuthTypes.NONE.toString)) + .transformToUpperCase + .toSet() + .checkValues(AuthTypes) + .createWithDefault(Set(AuthTypes.NONE.toString)) val AUTHENTICATION_CUSTOM_CLASS: OptionalConfigEntry[String] = buildConf("kyuubi.authentication.custom.class") .doc("User-defined authentication implementation of " + "org.apache.kyuubi.service.authentication.PasswdAuthenticationProvider") .version("1.3.0") + .serverOnly .stringConf .createOptional @@ -788,13 +824,16 @@ object KyuubiConf { buildConf("kyuubi.authentication.ldap.url") .doc("SPACE character separated LDAP connection URL(s).") .version("1.0.0") + .serverOnly .stringConf .createOptional - val AUTHENTICATION_LDAP_BASEDN: OptionalConfigEntry[String] = - buildConf("kyuubi.authentication.ldap.base.dn") + val AUTHENTICATION_LDAP_BASE_DN: OptionalConfigEntry[String] = + buildConf("kyuubi.authentication.ldap.baseDN") + .withAlternative("kyuubi.authentication.ldap.base.dn") .doc("LDAP base DN.") - .version("1.0.0") + .version("1.7.0") + .serverOnly .stringConf .createOptional @@ -802,21 +841,129 @@ object KyuubiConf { buildConf("kyuubi.authentication.ldap.domain") .doc("LDAP domain.") .version("1.0.0") + .serverOnly + .stringConf + .createOptional + + val AUTHENTICATION_LDAP_GROUP_DN_PATTERN: OptionalConfigEntry[String] = + buildConf("kyuubi.authentication.ldap.groupDNPattern") + .doc("COLON-separated list of patterns to use to find DNs for group entities in " + + "this directory. Use %s where the actual group name is to be substituted for. " + + "For example: CN=%s,CN=Groups,DC=subdomain,DC=domain,DC=com.") + .version("1.7.0") + .serverOnly + .stringConf + .createOptional + + val AUTHENTICATION_LDAP_USER_DN_PATTERN: OptionalConfigEntry[String] = + buildConf("kyuubi.authentication.ldap.userDNPattern") + .doc("COLON-separated list of patterns to use to find DNs for users in this directory. " + + "Use %s where the actual group name is to be substituted for. " + + "For example: CN=%s,CN=Users,DC=subdomain,DC=domain,DC=com.") + .version("1.7.0") + .serverOnly .stringConf .createOptional - val AUTHENTICATION_LDAP_GUIDKEY: ConfigEntry[String] = + val AUTHENTICATION_LDAP_GROUP_FILTER: ConfigEntry[Set[String]] = + buildConf("kyuubi.authentication.ldap.groupFilter") + .doc("COMMA-separated list of LDAP Group names (short name not full DNs). " + + "For example: HiveAdmins,HadoopAdmins,Administrators") + .version("1.7.0") + .serverOnly + .stringConf + .toSet() + .createWithDefault(Set.empty) + + val AUTHENTICATION_LDAP_USER_FILTER: ConfigEntry[Set[String]] = + buildConf("kyuubi.authentication.ldap.userFilter") + .doc("COMMA-separated list of LDAP usernames (just short names, not full DNs). " + + "For example: hiveuser,impalauser,hiveadmin,hadoopadmin") + .version("1.7.0") + .serverOnly + .stringConf + .toSet() + .createWithDefault(Set.empty) + + val AUTHENTICATION_LDAP_GUID_KEY: ConfigEntry[String] = buildConf("kyuubi.authentication.ldap.guidKey") - .doc("LDAP attribute name whose values are unique in this LDAP server." + - "For example:uid or cn.") + .doc("LDAP attribute name whose values are unique in this LDAP server. " + + "For example: uid or CN.") .version("1.2.0") + .serverOnly .stringConf .createWithDefault("uid") + val AUTHENTICATION_LDAP_GROUP_MEMBERSHIP_KEY: ConfigEntry[String] = + buildConf("kyuubi.authentication.ldap.groupMembershipKey") + .doc("LDAP attribute name on the group object that contains the list of distinguished " + + "names for the user, group, and contact objects that are members of the group. " + + "For example: member, uniqueMember or memberUid") + .version("1.7.0") + .serverOnly + .stringConf + .createWithDefault("member") + + val AUTHENTICATION_LDAP_USER_MEMBERSHIP_KEY: OptionalConfigEntry[String] = + buildConf("kyuubi.authentication.ldap.userMembershipKey") + .doc("LDAP attribute name on the user object that contains groups of which the user is " + + "a direct member, except for the primary group, which is represented by the " + + "primaryGroupId. For example: memberOf") + .version("1.7.0") + .serverOnly + .stringConf + .createOptional + + val AUTHENTICATION_LDAP_GROUP_CLASS_KEY: ConfigEntry[String] = + buildConf("kyuubi.authentication.ldap.groupClassKey") + .doc("LDAP attribute name on the group entry that is to be used in LDAP group searches. " + + "For example: group, groupOfNames or groupOfUniqueNames.") + .version("1.7.0") + .serverOnly + .stringConf + .createWithDefault("groupOfNames") + + val AUTHENTICATION_LDAP_CUSTOM_LDAP_QUERY: OptionalConfigEntry[String] = + buildConf("kyuubi.authentication.ldap.customLDAPQuery") + .doc("A full LDAP query that LDAP Atn provider uses to execute against LDAP Server. " + + "If this query returns a null resultset, the LDAP Provider fails the Authentication " + + "request, succeeds if the user is part of the resultset." + + "For example: `(&(objectClass=group)(objectClass=top)(instanceType=4)(cn=Domain*))`, " + + "`(&(objectClass=person)(|(sAMAccountName=admin)" + + "(|(memberOf=CN=Domain Admins,CN=Users,DC=domain,DC=com)" + + "(memberOf=CN=Administrators,CN=Builtin,DC=domain,DC=com))))`") + .version("1.7.0") + .serverOnly + .stringConf + .createOptional + + val AUTHENTICATION_LDAP_BIND_USER: OptionalConfigEntry[String] = + buildConf("kyuubi.authentication.ldap.binddn") + .doc("The user with which to bind to the LDAP server, and search for the full domain name " + + "of the user being authenticated. This should be the full domain name of the user, and " + + "should have search access across all users in the LDAP tree. If not specified, then " + + "the user being authenticated will be used as the bind user. " + + "For example: CN=bindUser,CN=Users,DC=subdomain,DC=domain,DC=com") + .version("1.7.0") + .serverOnly + .stringConf + .createOptional + + val AUTHENTICATION_LDAP_BIND_PASSWORD: OptionalConfigEntry[String] = + buildConf("kyuubi.authentication.ldap.bindpw") + .doc("The password for the bind user, to be used to search for the full name of the " + + "user being authenticated. If the username is specified, this parameter must also be " + + "specified.") + .version("1.7.0") + .serverOnly + .stringConf + .createOptional + val AUTHENTICATION_JDBC_DRIVER: OptionalConfigEntry[String] = buildConf("kyuubi.authentication.jdbc.driver.class") .doc("Driver class name for JDBC Authentication Provider.") .version("1.6.0") + .serverOnly .stringConf .createOptional @@ -824,6 +971,7 @@ object KyuubiConf { buildConf("kyuubi.authentication.jdbc.url") .doc("JDBC URL for JDBC Authentication Provider.") .version("1.6.0") + .serverOnly .stringConf .createOptional @@ -831,6 +979,7 @@ object KyuubiConf { buildConf("kyuubi.authentication.jdbc.user") .doc("Database user for JDBC Authentication Provider.") .version("1.6.0") + .serverOnly .stringConf .createOptional @@ -838,6 +987,7 @@ object KyuubiConf { buildConf("kyuubi.authentication.jdbc.password") .doc("Database password for JDBC Authentication Provider.") .version("1.6.0") + .serverOnly .stringConf .createOptional @@ -849,6 +999,7 @@ object KyuubiConf { "The SQL statement must start with the `SELECT` clause. " + "Available placeholders are `${user}` and `${password}`.") .version("1.6.0") + .serverOnly .stringConf .createOptional @@ -887,9 +1038,10 @@ object KyuubiConf { "
  • auth-conf - authentication plus integrity and confidentiality protection. This is" + " applicable only if Kyuubi is configured to use Kerberos authentication.
") .version("1.0.0") + .serverOnly .stringConf - .checkValues(SaslQOP.values.map(_.toString)) - .transform(_.toLowerCase(Locale.ROOT)) + .checkValues(SaslQOP) + .transformToLowerCase .createWithDefault(SaslQOP.AUTH.toString) val FRONTEND_REST_BIND_HOST: ConfigEntry[Option[String]] = @@ -994,6 +1146,15 @@ object KyuubiConf { .stringConf .createOptional + val KUBERNETES_CONTEXT_ALLOW_LIST: ConfigEntry[Set[String]] = + buildConf("kyuubi.kubernetes.context.allow.list") + .doc("The allowed kubernetes context list, if it is empty," + + " there is no kubernetes context limitation.") + .version("1.8.0") + .stringConf + .toSet() + .createWithDefault(Set.empty) + val KUBERNETES_NAMESPACE: ConfigEntry[String] = buildConf("kyuubi.kubernetes.namespace") .doc("The namespace that will be used for running the kyuubi pods and find engines.") @@ -1001,6 +1162,15 @@ object KyuubiConf { .stringConf .createWithDefault("default") + val KUBERNETES_NAMESPACE_ALLOW_LIST: ConfigEntry[Set[String]] = + buildConf("kyuubi.kubernetes.namespace.allow.list") + .doc("The allowed kubernetes namespace list, if it is empty," + + " there is no kubernetes namespace limitation.") + .version("1.8.0") + .stringConf + .toSet() + .createWithDefault(Set.empty) + val KUBERNETES_MASTER: OptionalConfigEntry[String] = buildConf("kyuubi.kubernetes.master.address") .doc("The internal Kubernetes master (API server) address to be used for kyuubi.") @@ -1060,6 +1230,15 @@ object KyuubiConf { .booleanConf .createWithDefault(false) + val KUBERNETES_TERMINATED_APPLICATION_RETAIN_PERIOD: ConfigEntry[Long] = + buildConf("kyuubi.kubernetes.terminatedApplicationRetainPeriod") + .doc("The period for which the Kyuubi server retains application information after " + + "the application terminates.") + .version("1.7.1") + .timeConf + .checkValue(_ > 0, "must be positive number") + .createWithDefault(Duration.ofMinutes(5).toMillis) + // /////////////////////////////////////////////////////////////////////////////////////////////// // SQL Engine Configuration // // /////////////////////////////////////////////////////////////////////////////////////////////// @@ -1117,6 +1296,16 @@ object KyuubiConf { .timeConf .createWithDefault(0) + val ENGINE_SPARK_MAX_INITIAL_WAIT: ConfigEntry[Long] = + buildConf("kyuubi.session.engine.spark.max.initial.wait") + .doc("Max wait time for the initial connection to Spark engine. The engine will" + + " self-terminate no new incoming connection is established within this time." + + " This setting only applies at the CONNECTION share level." + + " 0 or negative means not to self-terminate.") + .version("1.8.0") + .timeConf + .createWithDefault(Duration.ofSeconds(60).toMillis) + val ENGINE_FLINK_MAIN_RESOURCE: OptionalConfigEntry[String] = buildConf("kyuubi.session.engine.flink.main.resource") .doc("The package used to create Flink SQL engine remote job. If it is undefined," + @@ -1134,6 +1323,15 @@ object KyuubiConf { .intConf .createWithDefault(1000000) + val ENGINE_FLINK_FETCH_TIMEOUT: OptionalConfigEntry[Long] = + buildConf("kyuubi.session.engine.flink.fetch.timeout") + .doc("Result fetch timeout for Flink engine. If the timeout is reached, the result " + + "fetch would be stopped and the current fetched would be returned. If no data are " + + "fetched, a TimeoutException would be thrown.") + .version("1.8.0") + .timeConf + .createOptional + val ENGINE_TRINO_MAIN_RESOURCE: OptionalConfigEntry[String] = buildConf("kyuubi.session.engine.trino.main.resource") .doc("The package used to create Trino engine remote job. If it is undefined," + @@ -1156,6 +1354,55 @@ object KyuubiConf { .stringConf .createOptional + val ENGINE_TRINO_CONNECTION_PASSWORD: OptionalConfigEntry[String] = + buildConf("kyuubi.engine.trino.connection.password") + .doc("The password used for connecting to trino cluster") + .version("1.8.0") + .stringConf + .createOptional + + val ENGINE_TRINO_CONNECTION_KEYSTORE_PATH: OptionalConfigEntry[String] = + buildConf("kyuubi.engine.trino.connection.keystore.path") + .doc("The keystore path used for connecting to trino cluster") + .version("1.8.0") + .stringConf + .createOptional + + val ENGINE_TRINO_CONNECTION_KEYSTORE_PASSWORD: OptionalConfigEntry[String] = + buildConf("kyuubi.engine.trino.connection.keystore.password") + .doc("The keystore password used for connecting to trino cluster") + .version("1.8.0") + .stringConf + .createOptional + + val ENGINE_TRINO_CONNECTION_KEYSTORE_TYPE: OptionalConfigEntry[String] = + buildConf("kyuubi.engine.trino.connection.keystore.type") + .doc("The keystore type used for connecting to trino cluster") + .version("1.8.0") + .stringConf + .createOptional + + val ENGINE_TRINO_CONNECTION_TRUSTSTORE_PATH: OptionalConfigEntry[String] = + buildConf("kyuubi.engine.trino.connection.truststore.path") + .doc("The truststore path used for connecting to trino cluster") + .version("1.8.0") + .stringConf + .createOptional + + val ENGINE_TRINO_CONNECTION_TRUSTSTORE_PASSWORD: OptionalConfigEntry[String] = + buildConf("kyuubi.engine.trino.connection.truststore.password") + .doc("The truststore password used for connecting to trino cluster") + .version("1.8.0") + .stringConf + .createOptional + + val ENGINE_TRINO_CONNECTION_TRUSTSTORE_TYPE: OptionalConfigEntry[String] = + buildConf("kyuubi.engine.trino.connection.truststore.type") + .doc("The truststore type used for connecting to trino cluster") + .version("1.8.0") + .stringConf + .createOptional + val ENGINE_TRINO_SHOW_PROGRESS: ConfigEntry[Boolean] = buildConf("kyuubi.session.engine.trino.showProgress") .doc("When true, show the progress bar and final info in the Trino engine log.") @@ -1184,6 +1431,14 @@ object KyuubiConf { .timeConf .createWithDefault(Duration.ofSeconds(15).toMillis) + val ENGINE_ALIVE_MAX_FAILURES: ConfigEntry[Int] = + buildConf("kyuubi.session.engine.alive.max.failures") + .doc("The maximum number of failures allowed for the engine.") + .version("1.8.0") + .intConf + .checkValue(_ > 0, "Must be positive") + .createWithDefault(3) + val ENGINE_ALIVE_PROBE_ENABLED: ConfigEntry[Boolean] = buildConf("kyuubi.session.engine.alive.probe.enabled") .doc("Whether to enable the engine alive probe, it true, we will create a companion thrift" + @@ -1247,6 +1502,14 @@ object KyuubiConf { .version("1.2.0") .fallbackConf(SESSION_TIMEOUT) + val SESSION_CLOSE_ON_DISCONNECT: ConfigEntry[Boolean] = + buildConf("kyuubi.session.close.on.disconnect") + .doc("Session will be closed when client disconnects from kyuubi gateway. " + + "Set this to false to have session outlive its parent connection.") + .version("1.8.0") + .booleanConf + .createWithDefault(true) + val BATCH_SESSION_IDLE_TIMEOUT: ConfigEntry[Long] = buildConf("kyuubi.batch.session.idle.timeout") .doc("Batch session idle timeout, it will be closed when it's not accessed for this duration") .version("1.6.2") @@ -1266,7 +1529,7 @@ object KyuubiConf { .timeConf .createWithDefault(Duration.ofMinutes(30L).toMillis) - val SESSION_CONF_IGNORE_LIST: ConfigEntry[Seq[String]] = + val SESSION_CONF_IGNORE_LIST: ConfigEntry[Set[String]] = buildConf("kyuubi.session.conf.ignore.list") .doc("A comma-separated list of ignored keys. If the client connection contains any of" + " them, the key and the corresponding value will be removed silently during engine" + @@ -1276,10 +1539,10 @@ object KyuubiConf { " configurations via SET syntax.") .version("1.2.0") .stringConf - .toSequence() - .createWithDefault(Nil) + .toSet() + .createWithDefault(Set.empty) - val SESSION_CONF_RESTRICT_LIST: ConfigEntry[Seq[String]] = + val SESSION_CONF_RESTRICT_LIST: ConfigEntry[Set[String]] = buildConf("kyuubi.session.conf.restrict.list") .doc("A comma-separated list of restricted keys. If the client connection contains any of" + " them, the connection will be rejected explicitly during engine bootstrap and connection" + @@ -1289,8 +1552,8 @@ object KyuubiConf { " configurations via SET syntax.") .version("1.2.0") .stringConf - .toSequence() - .createWithDefault(Nil) + .toSet() + .createWithDefault(Set.empty) val SESSION_USER_SIGN_ENABLED: ConfigEntry[Boolean] = buildConf("kyuubi.session.user.sign.enabled") @@ -1320,6 +1583,15 @@ object KyuubiConf { .booleanConf .createWithDefault(true) + val SESSION_ENGINE_STARTUP_DESTROY_TIMEOUT: ConfigEntry[Long] = + buildConf("kyuubi.session.engine.startup.destroy.timeout") + .doc("Engine startup process destroy wait time, if the process does not " + + "stop after this time, force destroy instead. This configuration only " + + s"takes effect when `${SESSION_ENGINE_STARTUP_WAIT_COMPLETION.key}=false`.") + .version("1.8.0") + .timeConf + .createWithDefault(Duration.ofSeconds(5).toMillis) + val SESSION_ENGINE_LAUNCH_ASYNC: ConfigEntry[Boolean] = buildConf("kyuubi.session.engine.launch.async") .doc("When opening kyuubi session, whether to launch the backend engine asynchronously." + @@ -1329,7 +1601,7 @@ object KyuubiConf { .booleanConf .createWithDefault(true) - val SESSION_LOCAL_DIR_ALLOW_LIST: ConfigEntry[Seq[String]] = + val SESSION_LOCAL_DIR_ALLOW_LIST: ConfigEntry[Set[String]] = buildConf("kyuubi.session.local.dir.allow.list") .doc("The local dir list that are allowed to access by the kyuubi session application. " + " End-users might set some parameters such as `spark.files` and it will " + @@ -1342,8 +1614,8 @@ object KyuubiConf { .stringConf .checkValue(dir => dir.startsWith(File.separator), "the dir should be absolute path") .transform(dir => dir.stripSuffix(File.separator) + File.separator) - .toSequence() - .createWithDefault(Nil) + .toSet() + .createWithDefault(Set.empty) val BATCH_APPLICATION_CHECK_INTERVAL: ConfigEntry[Long] = buildConf("kyuubi.batch.application.check.interval") @@ -1359,7 +1631,7 @@ object KyuubiConf { .timeConf .createWithDefault(Duration.ofMinutes(3).toMillis) - val BATCH_CONF_IGNORE_LIST: ConfigEntry[Seq[String]] = + val BATCH_CONF_IGNORE_LIST: ConfigEntry[Set[String]] = buildConf("kyuubi.batch.conf.ignore.list") .doc("A comma-separated list of ignored keys for batch conf. If the batch conf contains" + " any of them, the key and the corresponding value will be removed silently during batch" + @@ -1371,8 +1643,8 @@ object KyuubiConf { " for the Spark batch job with key `kyuubi.batchConf.spark.spark.master`.") .version("1.6.0") .stringConf - .toSequence() - .createWithDefault(Nil) + .toSet() + .createWithDefault(Set.empty) val BATCH_INTERNAL_REST_CLIENT_SOCKET_TIMEOUT: ConfigEntry[Long] = buildConf("kyuubi.batch.internal.rest.client.socket.timeout") @@ -1402,6 +1674,50 @@ object KyuubiConf { .timeConf .createWithDefault(Duration.ofSeconds(5).toMillis) + val BATCH_RESOURCE_UPLOAD_ENABLED: ConfigEntry[Boolean] = + buildConf("kyuubi.batch.resource.upload.enabled") + .internal + .doc("Whether to enable Kyuubi batch resource upload function.") + .version("1.7.1") + .booleanConf + .createWithDefault(true) + + val BATCH_SUBMITTER_ENABLED: ConfigEntry[Boolean] = + buildConf("kyuubi.batch.submitter.enabled") + .internal + .serverOnly + .doc("Batch API v2 requires batch submitter to pick the INITIALIZED batch job " + + "from metastore and submits it to Resource Manager. " + + "Note: Batch API v2 is experimental and under rapid development, this configuration " + + "is added to allow explorers conveniently testing the developing Batch v2 API, not " + + "intended exposing to end users, it may be removed in anytime.") + .version("1.8.0") + .booleanConf + .createWithDefault(false) + + val BATCH_SUBMITTER_THREADS: ConfigEntry[Int] = + buildConf("kyuubi.batch.submitter.threads") + .internal + .serverOnly + .doc("Number of threads in batch job submitter, this configuration only take effects " + + s"when ${BATCH_SUBMITTER_ENABLED.key} is enabled") + .version("1.8.0") + .intConf + .createWithDefault(16) + + val BATCH_IMPL_VERSION: ConfigEntry[String] = + buildConf("kyuubi.batch.impl.version") + .internal + .serverOnly + .doc("Batch API version, candidates: 1, 2. Only take effect when " + + s"${BATCH_SUBMITTER_ENABLED.key} is true, otherwise always use v1 implementation. " + + "Note: Batch API v2 is experimental and under rapid development, this configuration " + + "is added to allow explorers conveniently testing the developing Batch v2 API, not " + + "intended exposing to end users, it may be removed in anytime.") + .version("1.8.0") + .stringConf + .createWithDefault("1") + val SERVER_EXEC_POOL_SIZE: ConfigEntry[Int] = buildConf("kyuubi.backend.server.exec.pool.size") .doc("Number of threads in the operation execution thread pool of Kyuubi server") @@ -1459,16 +1775,6 @@ object KyuubiConf { .intConf .createWithDefault(10) - val METADATA_REQUEST_RETRY_THREADS: ConfigEntry[Int] = - buildConf("kyuubi.metadata.request.retry.threads") - .doc("Number of threads in the metadata request retry manager thread pool. The metadata" + - " store might be unavailable sometimes and the requests will fail, tolerant for this" + - " case and unblock the main thread, we support retrying the failed requests" + - " in an async way.") - .version("1.6.0") - .intConf - .createWithDefault(10) - val METADATA_REQUEST_RETRY_INTERVAL: ConfigEntry[Long] = buildConf("kyuubi.metadata.request.retry.interval") .doc("The interval to check and trigger the metadata request retry tasks.") @@ -1476,10 +1782,31 @@ object KyuubiConf { .timeConf .createWithDefault(Duration.ofSeconds(5).toMillis) - val METADATA_REQUEST_RETRY_QUEUE_SIZE: ConfigEntry[Int] = - buildConf("kyuubi.metadata.request.retry.queue.size") + val METADATA_REQUEST_ASYNC_RETRY_ENABLED: ConfigEntry[Boolean] = + buildConf("kyuubi.metadata.request.async.retry.enabled") + .doc("Whether to retry in async when metadata request failed. When true, return " + + "success response immediately even the metadata request failed, and schedule " + + "it in background until success, to tolerate long-time metadata store outages " + + "w/o blocking the submission request.") + .version("1.7.0") + .booleanConf + .createWithDefault(true) + + val METADATA_REQUEST_ASYNC_RETRY_THREADS: ConfigEntry[Int] = + buildConf("kyuubi.metadata.request.async.retry.threads") + .withAlternative("kyuubi.metadata.request.retry.threads") + .doc("Number of threads in the metadata request async retry manager thread pool. Only " + + s"take affect when ${METADATA_REQUEST_ASYNC_RETRY_ENABLED.key} is `true`.") + .version("1.6.0") + .intConf + .createWithDefault(10) + + val METADATA_REQUEST_ASYNC_RETRY_QUEUE_SIZE: ConfigEntry[Int] = + buildConf("kyuubi.metadata.request.async.retry.queue.size") + .withAlternative("kyuubi.metadata.request.retry.queue.size") .doc("The maximum queue size for buffering metadata requests in memory when the external" + - " metadata storage is down. Requests will be dropped if the queue exceeds.") + " metadata storage is down. Requests will be dropped if the queue exceeds. Only" + + s" take affect when ${METADATA_REQUEST_ASYNC_RETRY_ENABLED.key} is `true`.") .version("1.6.0") .intConf .createWithDefault(65536) @@ -1557,11 +1884,29 @@ object KyuubiConf { .checkValue(_ >= 1000, "must >= 1s if set") .createOptional + val OPERATION_QUERY_TIMEOUT_MONITOR_ENABLED: ConfigEntry[Boolean] = + buildConf("kyuubi.operation.query.timeout.monitor.enabled") + .doc("Whether to monitor timeout query timeout check on server side.") + .version("1.8.0") + .serverOnly + .internal + .booleanConf + .createWithDefault(true) + + val OPERATION_RESULT_MAX_ROWS: ConfigEntry[Int] = + buildConf("kyuubi.operation.result.max.rows") + .doc("Max rows of Spark query results. Rows exceeding the limit would be ignored. " + + "By setting this value to 0 to disable the max rows limit.") + .version("1.6.0") + .intConf + .createWithDefault(0) + val OPERATION_INCREMENTAL_COLLECT: ConfigEntry[Boolean] = buildConf("kyuubi.operation.incremental.collect") .internal .doc("When true, the executor side result will be sequentially calculated and returned to" + - " the Spark driver side.") + s" the Spark driver side. Note that, ${OPERATION_RESULT_MAX_ROWS.key} will be ignored" + + " on incremental collect mode.") .version("1.4.0") .booleanConf .createWithDefault(false) @@ -1576,16 +1921,16 @@ object KyuubiConf { .version("1.7.0") .stringConf .checkValues(Set("arrow", "thrift")) - .transform(_.toLowerCase(Locale.ROOT)) + .transformToLowerCase .createWithDefault("thrift") - val OPERATION_RESULT_MAX_ROWS: ConfigEntry[Int] = - buildConf("kyuubi.operation.result.max.rows") - .doc("Max rows of Spark query results. Rows exceeding the limit would be ignored. " + - "By setting this value to 0 to disable the max rows limit.") - .version("1.6.0") - .intConf - .createWithDefault(0) + val ARROW_BASED_ROWSET_TIMESTAMP_AS_STRING: ConfigEntry[Boolean] = + buildConf("kyuubi.operation.result.arrow.timestampAsString") + .doc("When true, arrow-based rowsets will convert columns of type timestamp to strings for" + + " transmission.") + .version("1.7.0") + .booleanConf + .createWithDefault(false) val SERVER_OPERATION_LOG_DIR_ROOT: ConfigEntry[String] = buildConf("kyuubi.operation.log.dir.root") @@ -1601,8 +1946,8 @@ object KyuubiConf { .doc(s"(deprecated) - Using kyuubi.engine.share.level instead") .version("1.0.0") .stringConf - .transform(_.toUpperCase(Locale.ROOT)) - .checkValues(ShareLevel.values.map(_.toString)) + .transformToUpperCase + .checkValues(ShareLevel) .createWithDefault(ShareLevel.USER.toString) // [ZooKeeper Data Model] @@ -1616,7 +1961,7 @@ object KyuubiConf { .doc("(deprecated) - Using kyuubi.engine.share.level.subdomain instead") .version("1.2.0") .stringConf - .transform(_.toLowerCase(Locale.ROOT)) + .transformToLowerCase .checkValue(validZookeeperSubPath.matcher(_).matches(), "must be valid zookeeper sub path.") .createOptional @@ -1676,13 +2021,15 @@ object KyuubiConf { " all the capacity of the Trino." + "
  • HIVE_SQL: specify this engine type will launch a Hive engine which can provide" + " all the capacity of the Hive Server2.
    • " + - "
  • JDBC: specify this engine type will launch a JDBC engine which can provide" + - " a MySQL protocol connector, for now we only support Doris dialect.
    • " + + "
  • JDBC: specify this engine type will launch a JDBC engine which can forward " + + " queries to the database system through the certain JDBC driver, " + + " for now, it supports Doris and Phoenix.
    • " + + "
  • CHAT: specify this engine type will launch a Chat engine.
    • " + "") .version("1.4.0") .stringConf - .transform(_.toUpperCase(Locale.ROOT)) - .checkValues(EngineType.values.map(_.toString)) + .transformToUpperCase + .checkValues(EngineType) .createWithDefault(EngineType.SPARK_SQL.toString) val ENGINE_POOL_IGNORE_SUBDOMAIN: ConfigEntry[Boolean] = @@ -1705,6 +2052,7 @@ object KyuubiConf { .doc("This parameter is introduced as a server-side parameter " + "controlling the upper limit of the engine pool.") .version("1.4.0") + .serverOnly .intConf .checkValue(s => s > 0 && s < 33, "Invalid engine pool threshold, it should be in [1, 32]") .createWithDefault(9) @@ -1718,7 +2066,7 @@ object KyuubiConf { .intConf .createWithDefault(-1) - val ENGINE_POOL_BALANCE_POLICY: ConfigEntry[String] = + val ENGINE_POOL_SELECT_POLICY: ConfigEntry[String] = buildConf("kyuubi.engine.pool.selectPolicy") .doc("The select policy of an engine from the corresponding engine pool engine for " + "a session.
      " + @@ -1727,7 +2075,7 @@ object KyuubiConf { "
    ") .version("1.7.0") .stringConf - .transform(_.toUpperCase(Locale.ROOT)) + .transformToUpperCase .checkValues(Set("RANDOM", "POLLING")) .createWithDefault("RANDOM") @@ -1751,24 +2099,24 @@ object KyuubiConf { .toSequence(";") .createWithDefault(Nil) - val ENGINE_DEREGISTER_EXCEPTION_CLASSES: ConfigEntry[Seq[String]] = + val ENGINE_DEREGISTER_EXCEPTION_CLASSES: ConfigEntry[Set[String]] = buildConf("kyuubi.engine.deregister.exception.classes") .doc("A comma-separated list of exception classes. If there is any exception thrown," + " whose class matches the specified classes, the engine would deregister itself.") .version("1.2.0") .stringConf - .toSequence() - .createWithDefault(Nil) + .toSet() + .createWithDefault(Set.empty) - val ENGINE_DEREGISTER_EXCEPTION_MESSAGES: ConfigEntry[Seq[String]] = + val ENGINE_DEREGISTER_EXCEPTION_MESSAGES: ConfigEntry[Set[String]] = buildConf("kyuubi.engine.deregister.exception.messages") .doc("A comma-separated list of exception messages. If there is any exception thrown," + " whose message or stacktrace matches the specified message list, the engine would" + " deregister itself.") .version("1.2.0") .stringConf - .toSequence() - .createWithDefault(Nil) + .toSet() + .createWithDefault(Set.empty) val ENGINE_DEREGISTER_JOB_MAX_FAILURES: ConfigEntry[Int] = buildConf("kyuubi.engine.deregister.job.max.failures") @@ -1850,12 +2198,34 @@ object KyuubiConf { .stringConf .createWithDefault("file:///tmp/kyuubi/events") + val SERVER_EVENT_KAFKA_TOPIC: OptionalConfigEntry[String] = + buildConf("kyuubi.backend.server.event.kafka.topic") + .doc("The topic of server events go for the built-in Kafka logger") + .version("1.8.0") + .serverOnly + .stringConf + .createOptional + + val SERVER_EVENT_KAFKA_CLOSE_TIMEOUT: ConfigEntry[Long] = + buildConf("kyuubi.backend.server.event.kafka.close.timeout") + .doc("Period to wait for Kafka producer of server event handlers to close.") + .version("1.8.0") + .serverOnly + .timeConf + .createWithDefault(Duration.ofMillis(5000).toMillis) + val SERVER_EVENT_LOGGERS: ConfigEntry[Seq[String]] = buildConf("kyuubi.backend.server.event.loggers") .doc("A comma-separated list of server history loggers, where session/operation etc" + " events go.
      " + s"
    • JSON: the events will be written to the location of" + s" ${SERVER_EVENT_JSON_LOG_PATH.key}
      • " + + s"
    • KAFKA: the events will be serialized in JSON format" + + s" and sent to topic of `${SERVER_EVENT_KAFKA_TOPIC.key}`." + + s" Note: For the configs of Kafka producer," + + s" please specify them with the prefix: `kyuubi.backend.server.event.kafka.`." + + s" For example, `kyuubi.backend.server.event.kafka.bootstrap.servers=127.0.0.1:9092`" + + s"
      • " + s"
    • JDBC: to be done
      • " + s"
    • CUSTOM: User-defined event handlers.
    " + " Note that: Kyuubi supports custom event handlers with the Java SPI." + @@ -1866,9 +2236,11 @@ object KyuubiConf { .version("1.4.0") .serverOnly .stringConf - .transform(_.toUpperCase(Locale.ROOT)) + .transformToUpperCase .toSequence() - .checkValue(_.toSet.subsetOf(Set("JSON", "JDBC", "CUSTOM")), "Unsupported event loggers") + .checkValue( + _.toSet.subsetOf(Set("JSON", "JDBC", "CUSTOM", "KAFKA")), + "Unsupported event loggers") .createWithDefault(Nil) @deprecated("using kyuubi.engine.spark.event.loggers instead", "1.6.0") @@ -1888,7 +2260,7 @@ object KyuubiConf { " which has a zero-arg constructor.") .version("1.3.0") .stringConf - .transform(_.toUpperCase(Locale.ROOT)) + .transformToUpperCase .toSequence() .checkValue( _.toSet.subsetOf(Set("SPARK", "JSON", "JDBC", "CUSTOM")), @@ -1950,8 +2322,23 @@ object KyuubiConf { "subclass of `EngineSecuritySecretProvider`.") .version("1.5.0") .stringConf - .createWithDefault( - "org.apache.kyuubi.service.authentication.ZooKeeperEngineSecuritySecretProviderImpl") + .transform { + case "simple" => + "org.apache.kyuubi.service.authentication.SimpleEngineSecuritySecretProviderImpl" + case "zookeeper" => + "org.apache.kyuubi.service.authentication.ZooKeeperEngineSecuritySecretProviderImpl" + case other => other + } + .createWithDefault("zookeeper") + + val SIMPLE_SECURITY_SECRET_PROVIDER_PROVIDER_SECRET: OptionalConfigEntry[String] = + buildConf("kyuubi.engine.security.secret.provider.simple.secret") + .internal + .doc("The secret key used for internal security access. Only take affects when " + + s"${ENGINE_SECURITY_SECRET_PROVIDER.key} is 'simple'") + .version("1.7.0") + .stringConf + .createOptional val ENGINE_SECURITY_CRYPTO_KEY_LENGTH: ConfigEntry[Int] = buildConf("kyuubi.engine.security.crypto.keyLength") @@ -1999,14 +2386,14 @@ object KyuubiConf { val OPERATION_PLAN_ONLY_MODE: ConfigEntry[String] = buildConf("kyuubi.operation.plan.only.mode") .doc("Configures the statement performed mode, The value can be 'parse', 'analyze', " + - "'optimize', 'optimize_with_stats', 'physical', 'execution', or 'none', " + + "'optimize', 'optimize_with_stats', 'physical', 'execution', 'lineage' or 'none', " + "when it is 'none', indicate to the statement will be fully executed, otherwise " + "only way without executing the query. different engines currently support different " + "modes, the Spark engine supports all modes, and the Flink engine supports 'parse', " + "'physical', and 'execution', other engines do not support planOnly currently.") .version("1.4.0") .stringConf - .transform(_.toUpperCase(Locale.ROOT)) + .transformToUpperCase .checkValue( mode => Set( @@ -2016,10 +2403,11 @@ object KyuubiConf { "OPTIMIZE_WITH_STATS", "PHYSICAL", "EXECUTION", + "LINEAGE", "NONE").contains(mode), "Invalid value for 'kyuubi.operation.plan.only.mode'. Valid values are" + "'parse', 'analyze', 'optimize', 'optimize_with_stats', 'physical', 'execution' and " + - "'none'.") + "'lineage', 'none'.") .createWithDefault(NoneMode.name) val OPERATION_PLAN_ONLY_OUT_STYLE: ConfigEntry[String] = @@ -2029,14 +2417,11 @@ object KyuubiConf { "of the Spark engine") .version("1.7.0") .stringConf - .transform(_.toUpperCase(Locale.ROOT)) - .checkValue( - mode => Set("PLAIN", "JSON").contains(mode), - "Invalid value for 'kyuubi.operation.plan.only.output.style'. Valid values are " + - "'plain', 'json'.") + .transformToUpperCase + .checkValues(Set("PLAIN", "JSON")) .createWithDefault(PlainStyle.name) - val OPERATION_PLAN_ONLY_EXCLUDES: ConfigEntry[Seq[String]] = + val OPERATION_PLAN_ONLY_EXCLUDES: ConfigEntry[Set[String]] = buildConf("kyuubi.operation.plan.only.excludes") .doc("Comma-separated list of query plan names, in the form of simple class names, i.e, " + "for `SET abc=xyz`, the value will be `SetCommand`. For those auxiliary plans, such as " + @@ -2046,14 +2431,21 @@ object KyuubiConf { s"See also ${OPERATION_PLAN_ONLY_MODE.key}.") .version("1.5.0") .stringConf - .toSequence() - .createWithDefault(Seq( + .toSet() + .createWithDefault(Set( "ResetCommand", "SetCommand", "SetNamespaceCommand", "UseStatement", "SetCatalogAndNamespace")) + val LINEAGE_PARSER_PLUGIN_PROVIDER: ConfigEntry[String] = + buildConf("kyuubi.lineage.parser.plugin.provider") + .doc("The provider for the Spark lineage parser plugin.") + .version("1.8.0") + .stringConf + .createWithDefault("org.apache.kyuubi.plugin.lineage.LineageParserProvider") + object OperationLanguages extends Enumeration with Logging { type OperationLanguage = Value val PYTHON, SQL, SCALA, UNKNOWN = Value @@ -2072,22 +2464,27 @@ object KyuubiConf { val OPERATION_LANGUAGE: ConfigEntry[String] = buildConf("kyuubi.operation.language") .doc("Choose a programing language for the following inputs" + - "
    • SQL: (Default) Run all following statements as SQL queries.
      • " + - "
    • SCALA: Run all following input a scala codes
    ") + "
      " + + "
    • SQL: (Default) Run all following statements as SQL queries.
      • " + + "
    • SCALA: Run all following input as scala codes
      • " + + "
    • PYTHON: (Experimental) Run all following input as Python codes with Spark engine" + + "
      • " + + "
    ") .version("1.5.0") .stringConf - .transform(_.toUpperCase(Locale.ROOT)) - .checkValues(OperationLanguages.values.map(_.toString)) + .transformToUpperCase + .checkValues(OperationLanguages) .createWithDefault(OperationLanguages.SQL.toString) - val SESSION_CONF_ADVISOR: OptionalConfigEntry[String] = + val SESSION_CONF_ADVISOR: OptionalConfigEntry[Seq[String]] = buildConf("kyuubi.session.conf.advisor") - .doc("A config advisor plugin for Kyuubi Server. This plugin can provide some custom " + + .doc("A config advisor plugin for Kyuubi Server. This plugin can provide a list of custom " + "configs for different users or session configs and overwrite the session configs before " + "opening a new session. This config value should be a subclass of " + "`org.apache.kyuubi.plugin.SessionConfAdvisor` which has a zero-arg constructor.") .version("1.5.0") .stringConf + .toSequence() .createOptional val GROUP_PROVIDER: ConfigEntry[String] = @@ -2191,14 +2588,14 @@ object KyuubiConf { val ENGINE_FLINK_MEMORY: ConfigEntry[String] = buildConf("kyuubi.engine.flink.memory") - .doc("The heap memory for the Flink SQL engine") + .doc("The heap memory for the Flink SQL engine. Only effective in yarn session mode.") .version("1.6.0") .stringConf .createWithDefault("1g") val ENGINE_FLINK_JAVA_OPTIONS: OptionalConfigEntry[String] = buildConf("kyuubi.engine.flink.java.options") - .doc("The extra Java options for the Flink SQL engine") + .doc("The extra Java options for the Flink SQL engine. Only effective in yarn session mode.") .version("1.6.0") .stringConf .createOptional @@ -2206,11 +2603,19 @@ object KyuubiConf { val ENGINE_FLINK_EXTRA_CLASSPATH: OptionalConfigEntry[String] = buildConf("kyuubi.engine.flink.extra.classpath") .doc("The extra classpath for the Flink SQL engine, for configuring the location" + - " of hadoop client jars, etc") + " of hadoop client jars, etc. Only effective in yarn session mode.") .version("1.6.0") .stringConf .createOptional + val ENGINE_FLINK_APPLICATION_JARS: OptionalConfigEntry[String] = + buildConf("kyuubi.engine.flink.application.jars") + .doc("A comma-separated list of the local jars to be shipped with the job to the cluster. " + + "For example, SQL UDF jars. Only effective in yarn application mode.") + .version("1.8.0") + .stringConf + .createOptional + val SERVER_LIMIT_CONNECTIONS_PER_USER: OptionalConfigEntry[Int] = buildConf("kyuubi.server.limit.connections.per.user") .doc("Maximum kyuubi server connections per user." + @@ -2238,17 +2643,28 @@ object KyuubiConf { .intConf .createOptional - val SERVER_LIMIT_CONNECTIONS_USER_UNLIMITED_LIST: ConfigEntry[Seq[String]] = + val SERVER_LIMIT_CONNECTIONS_USER_UNLIMITED_LIST: ConfigEntry[Set[String]] = buildConf("kyuubi.server.limit.connections.user.unlimited.list") - .doc("The maximin connections of the user in the white list will not be limited.") + .doc("The maximum connections of the user in the white list will not be limited.") .version("1.7.0") .serverOnly .stringConf - .toSequence() - .createWithDefault(Nil) + .toSet() + .createWithDefault(Set.empty) + + val SERVER_LIMIT_CONNECTIONS_USER_DENY_LIST: ConfigEntry[Set[String]] = + buildConf("kyuubi.server.limit.connections.user.deny.list") + .doc("The user in the deny list will be denied to connect to kyuubi server, " + + "if the user has configured both user.unlimited.list and user.deny.list, " + + "the priority of the latter is higher.") + .version("1.8.0") + .serverOnly + .stringConf + .toSet() + .createWithDefault(Set.empty) val SERVER_LIMIT_BATCH_CONNECTIONS_PER_USER: OptionalConfigEntry[Int] = - buildConf("kyuubi.server.batch.limit.connections.per.user") + buildConf("kyuubi.server.limit.batch.connections.per.user") .doc("Maximum kyuubi server batch connections per user." + " Any user exceeding this limit will not be allowed to connect.") .version("1.7.0") @@ -2257,7 +2673,7 @@ object KyuubiConf { .createOptional val SERVER_LIMIT_BATCH_CONNECTIONS_PER_IPADDRESS: OptionalConfigEntry[Int] = - buildConf("kyuubi.server.batch.limit.connections.per.ipaddress") + buildConf("kyuubi.server.limit.batch.connections.per.ipaddress") .doc("Maximum kyuubi server batch connections per ipaddress." + " Any user exceeding this limit will not be allowed to connect.") .version("1.7.0") @@ -2266,7 +2682,7 @@ object KyuubiConf { .createOptional val SERVER_LIMIT_BATCH_CONNECTIONS_PER_USER_IPADDRESS: OptionalConfigEntry[Int] = - buildConf("kyuubi.server.batch.limit.connections.per.user.ipaddress") + buildConf("kyuubi.server.limit.batch.connections.per.user.ipaddress") .doc("Maximum kyuubi server batch connections per user:ipaddress combination." + " Any user-ipaddress exceeding this limit will not be allowed to connect.") .version("1.7.0") @@ -2274,6 +2690,15 @@ object KyuubiConf { .intConf .createOptional + val SERVER_LIMIT_CLIENT_FETCH_MAX_ROWS: OptionalConfigEntry[Int] = + buildConf("kyuubi.server.limit.client.fetch.max.rows") + .doc("Max rows limit for getting result row set operation. If the max rows specified " + + "by client-side is larger than the limit, request will fail directly.") + .version("1.8.0") + .serverOnly + .intConf + .createOptional + val SESSION_PROGRESS_ENABLE: ConfigEntry[Boolean] = buildConf("kyuubi.operation.progress.enabled") .doc("Whether to enable the operation progress. When true," + @@ -2290,6 +2715,24 @@ object KyuubiConf { .regexConf .createOptional + val SERVER_PERIODIC_GC_INTERVAL: ConfigEntry[Long] = + buildConf("kyuubi.server.periodicGC.interval") + .doc("How often to trigger a garbage collection.") + .version("1.7.0") + .serverOnly + .timeConf + .createWithDefaultString("PT30M") + + val SERVER_ADMINISTRATORS: ConfigEntry[Set[String]] = + buildConf("kyuubi.server.administrators") + .doc("Comma-separated list of Kyuubi service administrators. " + + "We use this config to grant admin permission to any service accounts.") + .version("1.8.0") + .serverOnly + .stringConf + .toSet() + .createWithDefault(Set.empty) + val OPERATION_SPARK_LISTENER_ENABLED: ConfigEntry[Boolean] = buildConf("kyuubi.operation.spark.listener.enabled") .doc("When set to true, Spark engine registers an SQLOperationListener before executing " + @@ -2312,6 +2755,13 @@ object KyuubiConf { .stringConf .createOptional + val ENGINE_JDBC_CONNECTION_PROPAGATECREDENTIAL: ConfigEntry[Boolean] = + buildConf("kyuubi.engine.jdbc.connection.propagateCredential") + .doc("Whether to use the session's user and password to connect to database") + .version("1.8.0") + .booleanConf + .createWithDefault(false) + val ENGINE_JDBC_CONNECTION_USER: OptionalConfigEntry[String] = buildConf("kyuubi.engine.jdbc.connection.user") .doc("The user is used for connecting to server") @@ -2348,6 +2798,24 @@ object KyuubiConf { .stringConf .createOptional + val ENGINE_JDBC_INITIALIZE_SQL: ConfigEntry[Seq[String]] = + buildConf("kyuubi.engine.jdbc.initialize.sql") + .doc("SemiColon-separated list of SQL statements to be initialized in the newly created " + + "engine before queries. i.e. use `SELECT 1` to eagerly active JDBCClient.") + .version("1.8.0") + .stringConf + .toSequence(";") + .createWithDefaultString("SELECT 1") + + val ENGINE_JDBC_SESSION_INITIALIZE_SQL: ConfigEntry[Seq[String]] = + buildConf("kyuubi.engine.jdbc.session.initialize.sql") + .doc("SemiColon-separated list of SQL statements to be initialized in the newly created " + + "engine session before queries.") + .version("1.8.0") + .stringConf + .toSequence(";") + .createWithDefault(Nil) + val ENGINE_OPERATION_CONVERT_CATALOG_DATABASE_ENABLED: ConfigEntry[Boolean] = buildConf("kyuubi.engine.operation.convert.catalog.database.enabled") .doc("When set to true, The engine converts the JDBC methods of set/get Catalog " + @@ -2356,6 +2824,53 @@ object KyuubiConf { .booleanConf .createWithDefault(true) + val ENGINE_SUBMIT_TIMEOUT: ConfigEntry[Long] = + buildConf("kyuubi.engine.submit.timeout") + .doc("Period to tolerant Driver Pod ephemerally invisible after submitting. " + + "In some Resource Managers, e.g. K8s, the Driver Pod is not visible immediately " + + "after `spark-submit` is returned.") + .version("1.7.1") + .timeConf + .createWithDefaultString("PT30S") + + val ENGINE_KUBERNETES_SUBMIT_TIMEOUT: ConfigEntry[Long] = + buildConf("kyuubi.engine.kubernetes.submit.timeout") + .doc("The engine submit timeout for Kubernetes application.") + .version("1.7.2") + .fallbackConf(ENGINE_SUBMIT_TIMEOUT) + + val ENGINE_YARN_SUBMIT_TIMEOUT: ConfigEntry[Long] = + buildConf("kyuubi.engine.yarn.submit.timeout") + .doc("The engine submit timeout for YARN application.") + .version("1.7.2") + .fallbackConf(ENGINE_SUBMIT_TIMEOUT) + + object YarnUserStrategy extends Enumeration { + type YarnUserStrategy = Value + val NONE, ADMIN, OWNER = Value + } + + val YARN_USER_STRATEGY: ConfigEntry[String] = + buildConf("kyuubi.yarn.user.strategy") + .doc("Determine which user to use to construct YARN client for application management, " + + "e.g. kill application. Options:
      " + + "
    • NONE: use Kyuubi server user.
      • " + + "
    • ADMIN: use admin user configured in `kyuubi.yarn.user.admin`.
      • " + + "
    • OWNER: use session user, typically is application owner.
      • " + + "
    ") + .version("1.8.0") + .stringConf + .checkValues(YarnUserStrategy) + .createWithDefault("NONE") + + val YARN_USER_ADMIN: ConfigEntry[String] = + buildConf("kyuubi.yarn.user.admin") + .doc(s"When ${YARN_USER_STRATEGY.key} is set to ADMIN, use this admin user to " + + "construct YARN client for application management, e.g. kill application.") + .version("1.8.0") + .stringConf + .createWithDefault("yarn") + /** * Holds information about keys that have been deprecated. * @@ -2427,6 +2942,84 @@ object KyuubiConf { Map(configs.map { cfg => cfg.key -> cfg }: _*) } + val ENGINE_CHAT_MEMORY: ConfigEntry[String] = + buildConf("kyuubi.engine.chat.memory") + .doc("The heap memory for the Chat engine") + .version("1.8.0") + .stringConf + .createWithDefault("1g") + + val ENGINE_CHAT_JAVA_OPTIONS: OptionalConfigEntry[String] = + buildConf("kyuubi.engine.chat.java.options") + .doc("The extra Java options for the Chat engine") + .version("1.8.0") + .stringConf + .createOptional + + val ENGINE_CHAT_PROVIDER: ConfigEntry[String] = + buildConf("kyuubi.engine.chat.provider") + .doc("The provider for the Chat engine. Candidates:
      " + + "
    • ECHO: simply replies a welcome message.
      • " + + "
    • GPT: a.k.a ChatGPT, powered by OpenAI.
      • " + + "
    ") + .version("1.8.0") + .stringConf + .transform { + case "ECHO" | "echo" => "org.apache.kyuubi.engine.chat.provider.EchoProvider" + case "GPT" | "gpt" | "ChatGPT" => "org.apache.kyuubi.engine.chat.provider.ChatGPTProvider" + case other => other + } + .createWithDefault("ECHO") + + val ENGINE_CHAT_GPT_API_KEY: OptionalConfigEntry[String] = + buildConf("kyuubi.engine.chat.gpt.apiKey") + .doc("The key to access OpenAI open API, which could be got at " + + "https://platform.openai.com/account/api-keys") + .version("1.8.0") + .stringConf + .createOptional + + val ENGINE_CHAT_GPT_MODEL: ConfigEntry[String] = + buildConf("kyuubi.engine.chat.gpt.model") + .doc("ID of the model used in ChatGPT. Available models refer to OpenAI's " + + "[Model overview](https://platform.openai.com/docs/models/overview).") + .version("1.8.0") + .stringConf + .createWithDefault("gpt-3.5-turbo") + + val ENGINE_CHAT_EXTRA_CLASSPATH: OptionalConfigEntry[String] = + buildConf("kyuubi.engine.chat.extra.classpath") + .doc("The extra classpath for the Chat engine, for configuring the location " + + "of the SDK and etc.") + .version("1.8.0") + .stringConf + .createOptional + + val ENGINE_CHAT_GPT_HTTP_PROXY: OptionalConfigEntry[String] = + buildConf("kyuubi.engine.chat.gpt.http.proxy") + .doc("HTTP proxy url for API calling in Chat GPT engine. e.g. http://127.0.0.1:1087") + .version("1.8.0") + .stringConf + .createOptional + + val ENGINE_CHAT_GPT_HTTP_CONNECT_TIMEOUT: ConfigEntry[Long] = + buildConf("kyuubi.engine.chat.gpt.http.connect.timeout") + .doc("The timeout[ms] for establishing the connection with the Chat GPT server. " + + "A timeout value of zero is interpreted as an infinite timeout.") + .version("1.8.0") + .timeConf + .checkValue(_ >= 0, "must be 0 or positive number") + .createWithDefault(Duration.ofSeconds(120).toMillis) + + val ENGINE_CHAT_GPT_HTTP_SOCKET_TIMEOUT: ConfigEntry[Long] = + buildConf("kyuubi.engine.chat.gpt.http.socket.timeout") + .doc("The timeout[ms] for waiting for data packets after Chat GPT server " + + "connection is established. A timeout value of zero is interpreted as an infinite timeout.") + .version("1.8.0") + .timeConf + .checkValue(_ >= 0, "must be 0 or positive number") + .createWithDefault(Duration.ofSeconds(120).toMillis) + val ENGINE_JDBC_MEMORY: ConfigEntry[String] = buildConf("kyuubi.engine.jdbc.memory") .doc("The heap memory for the JDBC query engine") @@ -2483,6 +3076,15 @@ object KyuubiConf { .stringConf .createWithDefault("bin/python") + val ENGINE_SPARK_REGISTER_ATTRIBUTES: ConfigEntry[Seq[String]] = + buildConf("kyuubi.engine.spark.register.attributes") + .internal + .doc("The extra attributes to expose when registering for Spark engine.") + .version("1.8.0") + .stringConf + .toSequence() + .createWithDefault(Seq("spark.driver.memory", "spark.executor.memory")) + val ENGINE_HIVE_EVENT_LOGGERS: ConfigEntry[Seq[String]] = buildConf("kyuubi.engine.hive.event.loggers") .doc("A comma-separated list of engine history loggers, where engine/session/operation etc" + @@ -2493,7 +3095,7 @@ object KyuubiConf { "
  • CUSTOM: to be done.
    • ") .version("1.7.0") .stringConf - .transform(_.toUpperCase(Locale.ROOT)) + .transformToUpperCase .toSequence() .checkValue( _.toSet.subsetOf(Set("JSON", "JDBC", "CUSTOM")), @@ -2510,7 +3112,7 @@ object KyuubiConf { "
  • CUSTOM: to be done.
    • ") .version("1.7.0") .stringConf - .transform(_.toUpperCase(Locale.ROOT)) + .transformToUpperCase .toSequence() .checkValue( _.toSet.subsetOf(Set("JSON", "JDBC", "CUSTOM")), @@ -2538,4 +3140,23 @@ object KyuubiConf { .version("1.7.0") .timeConf .createWithDefault(Duration.ofSeconds(60).toMillis) + + val OPERATION_GET_TABLES_IGNORE_TABLE_PROPERTIES: ConfigEntry[Boolean] = + buildConf("kyuubi.operation.getTables.ignoreTableProperties") + .doc("Speed up the `GetTables` operation by returning table identities only.") + .version("1.8.0") + .booleanConf + .createWithDefault(false) + + val SERVER_LIMIT_ENGINE_CREATION: OptionalConfigEntry[Int] = + buildConf("kyuubi.server.limit.engine.startup") + .internal + .doc("The maximum engine startup concurrency of kyuubi server. Highly concurrent engine" + + " startup processes may lead to high load on the kyuubi server machine," + + " this configuration is used to limit the number of engine startup processes" + + " running at the same time to avoid it.") + .version("1.8.0") + .serverOnly + .intConf + .createOptional } diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiReservedKeys.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiReservedKeys.scala index 6036af855..592425a4b 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiReservedKeys.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/config/KyuubiReservedKeys.scala @@ -19,25 +19,33 @@ package org.apache.kyuubi.config object KyuubiReservedKeys { final val KYUUBI_CLIENT_IP_KEY = "kyuubi.client.ipAddress" + final val KYUUBI_CLIENT_VERSION_KEY = "kyuubi.client.version" final val KYUUBI_SERVER_IP_KEY = "kyuubi.server.ipAddress" final val KYUUBI_SESSION_USER_KEY = "kyuubi.session.user" final val KYUUBI_SESSION_SIGN_PUBLICKEY = "kyuubi.session.sign.publickey" final val KYUUBI_SESSION_USER_SIGN = "kyuubi.session.user.sign" final val KYUUBI_SESSION_REAL_USER_KEY = "kyuubi.session.real.user" - final val KYUUBI_SESSION_BATCH_RESOURCE_UPLOADED_KEY = "kyuubi.session.batch.resource.uploaded" final val KYUUBI_SESSION_CONNECTION_URL_KEY = "kyuubi.session.connection.url" + // default priority is 10, higher priority will be scheduled first + // when enabled metadata store priority feature + final val KYUUBI_BATCH_PRIORITY = "kyuubi.batch.priority" + final val KYUUBI_BATCH_RESOURCE_UPLOADED_KEY = "kyuubi.batch.resource.uploaded" final val KYUUBI_STATEMENT_ID_KEY = "kyuubi.statement.id" final val KYUUBI_ENGINE_ID = "kyuubi.engine.id" final val KYUUBI_ENGINE_NAME = "kyuubi.engine.name" final val KYUUBI_ENGINE_URL = "kyuubi.engine.url" final val KYUUBI_ENGINE_SUBMIT_TIME_KEY = "kyuubi.engine.submit.time" final val KYUUBI_ENGINE_CREDENTIALS_KEY = "kyuubi.engine.credentials" + final val KYUUBI_SESSION_HANDLE_KEY = "kyuubi.session.handle" final val KYUUBI_SESSION_ENGINE_LAUNCH_HANDLE_GUID = "kyuubi.session.engine.launch.handle.guid" final val KYUUBI_SESSION_ENGINE_LAUNCH_HANDLE_SECRET = "kyuubi.session.engine.launch.handle.secret" + final val KYUUBI_SESSION_ENGINE_LAUNCH_SUPPORT_RESULT = + "kyuubi.session.engine.launch.support.result" final val KYUUBI_OPERATION_SET_CURRENT_CATALOG = "kyuubi.operation.set.current.catalog" final val KYUUBI_OPERATION_GET_CURRENT_CATALOG = "kyuubi.operation.get.current.catalog" final val KYUUBI_OPERATION_SET_CURRENT_DATABASE = "kyuubi.operation.set.current.database" final val KYUUBI_OPERATION_GET_CURRENT_DATABASE = "kyuubi.operation.get.current.database" + final val KYUUBI_OPERATION_HANDLE_KEY = "kyuubi.operation.handle" } diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/EngineType.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/EngineType.scala index 88680a8c7..3d850ba14 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/EngineType.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/engine/EngineType.scala @@ -23,5 +23,5 @@ package org.apache.kyuubi.engine object EngineType extends Enumeration { type EngineType = Value - val SPARK_SQL, FLINK_SQL, TRINO, HIVE_SQL, JDBC = Value + val SPARK_SQL, FLINK_SQL, CHAT, TRINO, HIVE_SQL, JDBC = Value } diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/AbstractOperation.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/AbstractOperation.scala index 9cdd6a8f0..0a185b942 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/AbstractOperation.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/AbstractOperation.scala @@ -18,13 +18,14 @@ package org.apache.kyuubi.operation import java.util.concurrent.{Future, ScheduledExecutorService, TimeUnit} +import java.util.concurrent.locks.ReentrantLock import scala.collection.JavaConverters._ import org.apache.commons.lang3.StringUtils -import org.apache.hive.service.rpc.thrift.{TGetResultSetMetadataResp, TProgressUpdateResp, TProtocolVersion, TRowSet, TStatus, TStatusCode} +import org.apache.hive.service.rpc.thrift.{TFetchResultsResp, TGetResultSetMetadataResp, TProgressUpdateResp, TProtocolVersion, TStatus, TStatusCode} -import org.apache.kyuubi.{KyuubiSQLException, Logging} +import org.apache.kyuubi.{KyuubiSQLException, Logging, Utils} import org.apache.kyuubi.config.KyuubiConf.OPERATION_IDLE_TIMEOUT import org.apache.kyuubi.operation.FetchOrientation.FetchOrientation import org.apache.kyuubi.operation.OperationState._ @@ -36,7 +37,7 @@ abstract class AbstractOperation(session: Session) extends Operation with Loggin final protected val opType: String = getClass.getSimpleName final protected val createTime = System.currentTimeMillis() - final private val handle = OperationHandle() + protected val handle = OperationHandle() final private val operationTimeout: Long = { session.sessionManager.getConf.get(OPERATION_IDLE_TIMEOUT) } @@ -45,7 +46,11 @@ abstract class AbstractOperation(session: Session) extends Operation with Loggin private var statementTimeoutCleaner: Option[ScheduledExecutorService] = None - protected def cleanup(targetState: OperationState): Unit = state.synchronized { + private val lock: ReentrantLock = new ReentrantLock() + + protected def withLockRequired[T](block: => T): T = Utils.withLockRequired(lock)(block) + + protected def cleanup(targetState: OperationState): Unit = withLockRequired { if (!isTerminalState(state)) { setState(targetState) Option(getBackgroundHandle).foreach(_.cancel(true)) @@ -110,7 +115,7 @@ abstract class AbstractOperation(session: Session) extends Operation with Loggin info(s"Processing ${session.user}'s query[$statementId]: " + s"${state.name} -> ${newState.name}, statement:\n$redactedStatement") startTime = System.currentTimeMillis() - case ERROR | FINISHED | CANCELED | TIMEOUT => + case ERROR | FINISHED | CANCELED | TIMEOUT | CLOSED => completedTime = System.currentTimeMillis() val timeCost = s", time taken: ${(completedTime - startTime) / 1000.0} seconds" info(s"Processing ${session.user}'s query[$statementId]: " + @@ -177,7 +182,12 @@ abstract class AbstractOperation(session: Session) extends Operation with Loggin override def getResultSetMetadata: TGetResultSetMetadataResp - override def getNextRowSet(order: FetchOrientation, rowSetSize: Int): TRowSet + def getNextRowSetInternal(order: FetchOrientation, rowSetSize: Int): TFetchResultsResp + + override def getNextRowSet(order: FetchOrientation, rowSetSize: Int): TFetchResultsResp = + withLockRequired { + getNextRowSetInternal(order, rowSetSize) + } /** * convert SQL 'like' pattern to a Java regular expression. diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/FetchIterator.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/FetchIterator.scala index fdada1174..ada155887 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/FetchIterator.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/FetchIterator.scala @@ -20,7 +20,7 @@ package org.apache.kyuubi.operation /** * Borrowed from Apache Spark, see SPARK-33655 */ -sealed trait FetchIterator[A] extends Iterator[A] { +trait FetchIterator[A] extends Iterator[A] { /** * Begin a fetch block, forward from the current position. diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/Operation.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/Operation.scala index 6f496c9b8..c20a16f61 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/Operation.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/Operation.scala @@ -19,7 +19,7 @@ package org.apache.kyuubi.operation import java.util.concurrent.Future -import org.apache.hive.service.rpc.thrift.{TGetResultSetMetadataResp, TRowSet} +import org.apache.hive.service.rpc.thrift.{TFetchResultsResp, TGetResultSetMetadataResp} import org.apache.kyuubi.operation.FetchOrientation.FetchOrientation import org.apache.kyuubi.operation.log.OperationLog @@ -32,7 +32,7 @@ trait Operation { def close(): Unit def getResultSetMetadata: TGetResultSetMetadataResp - def getNextRowSet(order: FetchOrientation, rowSetSize: Int): TRowSet + def getNextRowSet(order: FetchOrientation, rowSetSize: Int): TFetchResultsResp def getSession: Session def getHandle: OperationHandle diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/OperationManager.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/OperationManager.scala index fe38263db..38dabcc1a 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/OperationManager.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/OperationManager.scala @@ -17,6 +17,8 @@ package org.apache.kyuubi.operation +import scala.collection.JavaConverters._ + import org.apache.hive.service.rpc.thrift._ import org.apache.kyuubi.KyuubiSQLException @@ -41,6 +43,8 @@ abstract class OperationManager(name: String) extends AbstractService(name) { def getOperationCount: Int = handleToOperation.size() + def allOperations(): Iterable[Operation] = handleToOperation.values().asScala + override def initialize(conf: KyuubiConf): Unit = { LogDivertAppender.initialize(skipOperationLog) super.initialize(conf) @@ -133,18 +137,22 @@ abstract class OperationManager(name: String) extends AbstractService(name) { final def getOperationNextRowSet( opHandle: OperationHandle, order: FetchOrientation, - maxRows: Int): TRowSet = { + maxRows: Int): TFetchResultsResp = { getOperation(opHandle).getNextRowSet(order, maxRows) } def getOperationLogRowSet( opHandle: OperationHandle, order: FetchOrientation, - maxRows: Int): TRowSet = { + maxRows: Int): TFetchResultsResp = { val operationLog = getOperation(opHandle).getOperationLog - operationLog.map(_.read(maxRows)).getOrElse { + val rowSet = operationLog.map(_.read(order, maxRows)).getOrElse { throw KyuubiSQLException(s"$opHandle failed to generate operation log") } + val resp = new TFetchResultsResp(new TStatus(TStatusCode.SUCCESS_STATUS)) + resp.setResults(rowSet) + resp.setHasMoreRows(false) + resp } final def removeExpiredOperations(handles: Seq[OperationHandle]): Seq[Operation] = synchronized { diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/PlanOnlyMode.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/PlanOnlyMode.scala index 3e170f05f..0407dab62 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/PlanOnlyMode.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/PlanOnlyMode.scala @@ -41,6 +41,8 @@ case object PhysicalMode extends PlanOnlyMode { val name = "physical" } case object ExecutionMode extends PlanOnlyMode { val name = "execution" } +case object LineageMode extends PlanOnlyMode { val name = "lineage" } + case object NoneMode extends PlanOnlyMode { val name = "none" } case object UnknownMode extends PlanOnlyMode { @@ -64,6 +66,7 @@ object PlanOnlyMode { case OptimizeWithStatsMode.name => OptimizeWithStatsMode case PhysicalMode.name => PhysicalMode case ExecutionMode.name => ExecutionMode + case LineageMode.name => LineageMode case NoneMode.name => NoneMode case other => UnknownMode.mode(other) } diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/Log4j12DivertAppender.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/Log4j12DivertAppender.scala index 1191e94ae..6ea853485 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/Log4j12DivertAppender.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/Log4j12DivertAppender.scala @@ -30,7 +30,7 @@ class Log4j12DivertAppender extends WriterAppender { final private val lo = Logger.getRootLogger .getAllAppenders.asScala - .find(_.isInstanceOf[ConsoleAppender]) + .find(ap => ap.isInstanceOf[ConsoleAppender] || ap.isInstanceOf[RollingFileAppender]) .map(_.asInstanceOf[Appender].getLayout) .getOrElse(new PatternLayout("%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n")) @@ -39,7 +39,7 @@ class Log4j12DivertAppender extends WriterAppender { setLayout(lo) addFilter { _: LoggingEvent => - if (OperationLog.getCurrentOperationLog == null) Filter.DENY else Filter.NEUTRAL + if (OperationLog.getCurrentOperationLog.isDefined) Filter.NEUTRAL else Filter.DENY } /** @@ -51,8 +51,7 @@ class Log4j12DivertAppender extends WriterAppender { // That should've gone into our writer. Notify the LogContext. val logOutput = writer.toString writer.reset() - val log = OperationLog.getCurrentOperationLog - if (log != null) log.write(logOutput) + OperationLog.getCurrentOperationLog.foreach(_.write(logOutput)) } } diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/Log4j2DivertAppender.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/Log4j2DivertAppender.scala index 68753cf98..d8e37a019 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/Log4j2DivertAppender.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/Log4j2DivertAppender.scala @@ -18,15 +18,18 @@ package org.apache.kyuubi.operation.log import java.io.CharArrayWriter +import java.util.concurrent.locks.ReadWriteLock import scala.collection.JavaConverters._ import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.core.{Filter, LogEvent, StringLayout} -import org.apache.logging.log4j.core.appender.{AbstractWriterAppender, ConsoleAppender, WriterManager} +import org.apache.logging.log4j.core.appender.{AbstractWriterAppender, ConsoleAppender, RollingFileAppender, WriterManager} import org.apache.logging.log4j.core.filter.AbstractFilter import org.apache.logging.log4j.core.layout.PatternLayout +import org.apache.kyuubi.util.reflect.ReflectUtils._ + class Log4j2DivertAppender( name: String, layout: StringLayout, @@ -52,22 +55,16 @@ class Log4j2DivertAppender( addFilter(new AbstractFilter() { override def filter(event: LogEvent): Filter.Result = { - if (OperationLog.getCurrentOperationLog == null) { - Filter.Result.DENY - } else { + if (OperationLog.getCurrentOperationLog.isDefined) { Filter.Result.NEUTRAL + } else { + Filter.Result.DENY } } }) - def initLayout(): StringLayout = { - LogManager.getRootLogger.asInstanceOf[org.apache.logging.log4j.core.Logger] - .getAppenders.values().asScala - .find(ap => ap.isInstanceOf[ConsoleAppender] && ap.getLayout.isInstanceOf[StringLayout]) - .map(_.getLayout.asInstanceOf[StringLayout]) - .getOrElse(PatternLayout.newBuilder().withPattern( - "%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n").build()) - } + private val writeLock = + getField[ReadWriteLock]((classOf[AbstractWriterAppender[_]], this), "readWriteLock").writeLock /** * Overrides AbstractWriterAppender.append(), which does the real logging. No need @@ -75,11 +72,15 @@ class Log4j2DivertAppender( */ override def append(event: LogEvent): Unit = { super.append(event) - // That should've gone into our writer. Notify the LogContext. - val logOutput = writer.toString - writer.reset() - val log = OperationLog.getCurrentOperationLog - if (log != null) log.write(logOutput) + writeLock.lock() + try { + // That should've gone into our writer. Notify the LogContext. + val logOutput = writer.toString + writer.reset() + OperationLog.getCurrentOperationLog.foreach(_.write(logOutput)) + } finally { + writeLock.unlock() + } } } @@ -87,15 +88,17 @@ object Log4j2DivertAppender { def initLayout(): StringLayout = { LogManager.getRootLogger.asInstanceOf[org.apache.logging.log4j.core.Logger] .getAppenders.values().asScala - .find(ap => ap.isInstanceOf[ConsoleAppender] && ap.getLayout.isInstanceOf[StringLayout]) + .find(ap => + (ap.isInstanceOf[ConsoleAppender] || ap.isInstanceOf[RollingFileAppender]) && + ap.getLayout.isInstanceOf[StringLayout]) .map(_.getLayout.asInstanceOf[StringLayout]) .getOrElse(PatternLayout.newBuilder().withPattern( - "%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n").build()) + "%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n%ex").build()) } def initialize(): Unit = { val ap = new Log4j2DivertAppender() - org.apache.logging.log4j.LogManager.getRootLogger() + org.apache.logging.log4j.LogManager.getRootLogger .asInstanceOf[org.apache.logging.log4j.core.Logger].addAppender(ap) ap.start() } diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/LogDivertAppender.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/LogDivertAppender.scala index 7d2989303..58bca992c 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/LogDivertAppender.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/LogDivertAppender.scala @@ -17,7 +17,7 @@ package org.apache.kyuubi.operation.log -import org.slf4j.impl.StaticLoggerBinder +import org.slf4j.LoggerFactory import org.apache.kyuubi.Logging @@ -30,9 +30,8 @@ object LogDivertAppender extends Logging { Log4j12DivertAppender.initialize() } else { warn(s"Unsupported SLF4J binding" + - s" ${StaticLoggerBinder.getSingleton.getLoggerFactoryClassStr}") + s" ${LoggerFactory.getILoggerFactory.getClass.getName}") } } - } } diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/OperationLog.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/OperationLog.scala index 84c4ed55c..2e133df28 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/OperationLog.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/operation/log/OperationLog.scala @@ -20,7 +20,7 @@ package org.apache.kyuubi.operation.log import java.io.{BufferedReader, IOException} import java.nio.ByteBuffer import java.nio.charset.StandardCharsets -import java.nio.file.{Files, Path, Paths} +import java.nio.file.{Files, NoSuchFileException, Path, Paths} import java.util.{ArrayList => JArrayList, List => JList} import scala.collection.JavaConverters._ @@ -29,6 +29,7 @@ import scala.collection.mutable.ListBuffer import org.apache.hive.service.rpc.thrift.{TColumn, TRow, TRowSet, TStringColumn} import org.apache.kyuubi.{KyuubiSQLException, Logging} +import org.apache.kyuubi.operation.FetchOrientation.{FETCH_FIRST, FETCH_NEXT, FetchOrientation} import org.apache.kyuubi.operation.OperationHandle import org.apache.kyuubi.session.Session import org.apache.kyuubi.util.ThriftUtils @@ -44,7 +45,7 @@ object OperationLog extends Logging { OPERATION_LOG.set(operationLog) } - def getCurrentOperationLog: OperationLog = OPERATION_LOG.get() + def getCurrentOperationLog: Option[OperationLog] = Option(OPERATION_LOG.get) def removeCurrentOperationLog(): Unit = OPERATION_LOG.remove() @@ -86,7 +87,7 @@ object OperationLog extends Logging { class OperationLog(path: Path) { private lazy val writer = Files.newBufferedWriter(path, StandardCharsets.UTF_8) - private lazy val reader = Files.newBufferedReader(path, StandardCharsets.UTF_8) + private var reader: BufferedReader = _ @volatile private var initialized: Boolean = false @@ -95,6 +96,15 @@ class OperationLog(path: Path) { private var lastSeekReadPos = 0 private var seekableReader: SeekableBufferedReader = _ + def getReader(): BufferedReader = { + if (reader == null) { + try { + reader = Files.newBufferedReader(path, StandardCharsets.UTF_8) + } catch handleFileNotFound + } + reader + } + def addExtraLog(path: Path): Unit = synchronized { try { extraReaders += Files.newBufferedReader(path, StandardCharsets.UTF_8) @@ -130,19 +140,23 @@ class OperationLog(path: Path) { val logs = new JArrayList[String] var i = 0 try { - var line: String = reader.readLine() - while ((i < lastRows || maxRows <= 0) && line != null) { - logs.add(line) + var line: String = null + do { line = reader.readLine() - i += 1 - } - (logs, i) - } catch { - case e: IOException => - val absPath = path.toAbsolutePath - val opHandle = absPath.getFileName - throw KyuubiSQLException(s"Operation[$opHandle] log file $absPath is not found", e) - } + if (line != null) { + logs.add(line) + i += 1 + } + } while ((i < lastRows || maxRows <= 0) && line != null) + } catch handleFileNotFound + (logs, i) + } + + private def handleFileNotFound: PartialFunction[Throwable, Unit] = { + case e: IOException => + val absPath = path.toAbsolutePath + val opHandle = absPath.getFileName + throw KyuubiSQLException(s"Operation[$opHandle] log file $absPath is not found", e) } private def toRowSet(logs: JList[String]): TRowSet = { @@ -152,14 +166,25 @@ class OperationLog(path: Path) { tRow } + def read(maxRows: Int): TRowSet = synchronized { + read(FETCH_NEXT, maxRows) + } + /** * Read to log file line by line * * @param maxRows maximum result number can reach + * @param order the fetch orientation of the result, can be FETCH_NEXT, FETCH_FIRST */ - def read(maxRows: Int): TRowSet = synchronized { + def read(order: FetchOrientation = FETCH_NEXT, maxRows: Int): TRowSet = synchronized { if (!initialized) return ThriftUtils.newEmptyRowSet - val (logs, lines) = readLogs(reader, maxRows, maxRows) + if (order != FETCH_NEXT && order != FETCH_FIRST) { + throw KyuubiSQLException(s"$order in operation log is not supported") + } + if (order == FETCH_FIRST) { + resetReader() + } + val (logs, lines) = readLogs(getReader(), maxRows, maxRows) var lastRows = maxRows - lines for (extraReader <- extraReaders if lastRows > 0 || maxRows <= 0) { val (extraLogs, extraRows) = readLogs(extraReader, lastRows, maxRows) @@ -170,6 +195,19 @@ class OperationLog(path: Path) { toRowSet(logs) } + private def resetReader(): Unit = { + trySafely { + if (reader != null) { + reader.close() + } + } + reader = null + closeExtraReaders() + extraReaders.clear() + extraPaths.foreach(path => + extraReaders += Files.newBufferedReader(path, StandardCharsets.UTF_8)) + } + def read(from: Int, size: Int): TRowSet = synchronized { if (!initialized) return ThriftUtils.newEmptyRowSet var pos = from @@ -195,10 +233,14 @@ class OperationLog(path: Path) { } def close(): Unit = synchronized { + if (!initialized) return + closeExtraReaders() trySafely { - reader.close() + if (reader != null) { + reader.close() + } } trySafely { writer.close() @@ -212,7 +254,7 @@ class OperationLog(path: Path) { } trySafely { - Files.delete(path) + Files.deleteIfExists(path) } } @@ -220,6 +262,7 @@ class OperationLog(path: Path) { try { f } catch { + case _: NoSuchFileException => case e: IOException => // Printing log here may cause a deadlock. The lock order of OperationLog.write // is RootLogger -> LogDivertAppender -> OperationLog. If printing log here, the diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/AbstractBackendService.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/AbstractBackendService.scala index e7c2d8365..443b35354 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/AbstractBackendService.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/AbstractBackendService.scala @@ -21,7 +21,7 @@ import java.util.concurrent.{ExecutionException, TimeoutException, TimeUnit} import scala.concurrent.CancellationException -import org.apache.hive.service.rpc.thrift.{TGetInfoType, TGetInfoValue, TGetResultSetMetadataResp, TProtocolVersion, TRowSet} +import org.apache.hive.service.rpc.thrift._ import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.operation.{OperationHandle, OperationStatus} @@ -35,6 +35,7 @@ abstract class AbstractBackendService(name: String) extends CompositeService(name) with BackendService { private lazy val timeout = conf.get(KyuubiConf.OPERATION_STATUS_POLLING_TIMEOUT) + private lazy val maxRowsLimit = conf.get(KyuubiConf.SERVER_LIMIT_CLIENT_FETCH_MAX_ROWS) override def openSession( protocol: TProtocolVersion, @@ -156,11 +157,14 @@ abstract class AbstractBackendService(name: String) queryId } - override def getOperationStatus(operationHandle: OperationHandle): OperationStatus = { + override def getOperationStatus( + operationHandle: OperationHandle, + maxWait: Option[Long]): OperationStatus = { val operation = sessionManager.operationManager.getOperation(operationHandle) if (operation.shouldRunAsync) { try { - operation.getBackgroundHandle.get(timeout, TimeUnit.MILLISECONDS) + val waitTime = maxWait.getOrElse(timeout) + operation.getBackgroundHandle.get(waitTime, TimeUnit.MILLISECONDS) } catch { case e: TimeoutException => debug(s"$operationHandle: Long polling timed out, ${e.getMessage}") @@ -197,7 +201,13 @@ abstract class AbstractBackendService(name: String) operationHandle: OperationHandle, orientation: FetchOrientation, maxRows: Int, - fetchLog: Boolean): TRowSet = { + fetchLog: Boolean): TFetchResultsResp = { + maxRowsLimit.foreach(limit => + if (maxRows > limit) { + throw new IllegalArgumentException(s"Max rows for fetching results " + + s"operation should not exceed the limit: $limit") + }) + sessionManager.operationManager .getOperation(operationHandle) .getSession diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/BackendService.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/BackendService.scala index e18411566..85df9024c 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/BackendService.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/BackendService.scala @@ -91,7 +91,9 @@ trait BackendService { foreignTable: String): OperationHandle def getQueryId(operationHandle: OperationHandle): String - def getOperationStatus(operationHandle: OperationHandle): OperationStatus + def getOperationStatus( + operationHandle: OperationHandle, + maxWait: Option[Long] = None): OperationStatus def cancelOperation(operationHandle: OperationHandle): Unit def closeOperation(operationHandle: OperationHandle): Unit def getResultSetMetadata(operationHandle: OperationHandle): TGetResultSetMetadataResp @@ -99,7 +101,7 @@ trait BackendService { operationHandle: OperationHandle, orientation: FetchOrientation, maxRows: Int, - fetchLog: Boolean): TRowSet + fetchLog: Boolean): TFetchResultsResp def sessionManager: SessionManager } diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/ServiceUtils.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/ServiceUtils.scala index d481aea77..955144af8 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/ServiceUtils.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/ServiceUtils.scala @@ -17,6 +17,10 @@ package org.apache.kyuubi.service +import java.io.{Closeable, IOException} + +import org.slf4j.Logger + object ServiceUtils { /** @@ -49,4 +53,24 @@ object ServiceUtils { userName.substring(0, indexOfDomainMatch) } } + + /** + * Close the Closeable objects and ignore any [[IOException]] or + * null pointers. Must only be used for cleanup in exception handlers. + * + * @param log the log to record problems to at debug level. Can be null. + * @param closeables the objects to close + */ + def cleanup(log: Logger, closeables: Closeable*): Unit = { + closeables.filter(_ != null).foreach { c => + try { + c.close() + } catch { + case e: IOException => + if (log != null && log.isDebugEnabled) { + log.debug(s"Exception in closing $c", e) + } + } + } + } } diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/TBinaryFrontendService.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/TBinaryFrontendService.scala index 74cf4e2e6..2f4419374 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/TBinaryFrontendService.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/TBinaryFrontendService.scala @@ -134,7 +134,7 @@ abstract class TBinaryFrontendService(name: String) keyStorePassword: String, keyStoreType: Option[String], keyStoreAlgorithm: Option[String], - disallowedSslProtocols: Seq[String], + disallowedSslProtocols: Set[String], includeCipherSuites: Seq[String]): TServerSocket = { val params = if (includeCipherSuites.nonEmpty) { @@ -163,7 +163,7 @@ abstract class TBinaryFrontendService(name: String) } } sslServerSocket.setEnabledProtocols(enabledProtocols) - info(s"SSL Server Socket enabled protocols: $enabledProtocols") + info(s"SSL Server Socket enabled protocols: ${enabledProtocols.mkString(",")}") case _ => } diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/TFrontendService.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/TFrontendService.scala index c3354cc25..1492a6af5 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/TFrontendService.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/TFrontendService.scala @@ -31,8 +31,7 @@ import org.apache.thrift.transport.TTransport import org.apache.kyuubi.{KyuubiSQLException, Logging, Utils} import org.apache.kyuubi.Utils.stringifyException -import org.apache.kyuubi.config.KyuubiConf.AUTHENTICATION_LONG_USERNAME -import org.apache.kyuubi.config.KyuubiConf.FRONTEND_CONNECTION_URL_USE_HOSTNAME +import org.apache.kyuubi.config.KyuubiConf.{AUTHENTICATION_LONG_USERNAME, FRONTEND_ADVERTISED_HOST, FRONTEND_CONNECTION_URL_USE_HOSTNAME, SESSION_CLOSE_ON_DISCONNECT} import org.apache.kyuubi.config.KyuubiReservedKeys._ import org.apache.kyuubi.operation.{FetchOrientation, OperationHandle} import org.apache.kyuubi.service.authentication.KyuubiAuthenticationFactory @@ -113,12 +112,12 @@ abstract class TFrontendService(name: String) override def connectionUrl: String = { checkInitialized() - val host = serverHost match { - case Some(h) => h // respect user's setting ahead - case None if conf.get(FRONTEND_CONNECTION_URL_USE_HOSTNAME) => + val host = (conf.get(FRONTEND_ADVERTISED_HOST), serverHost) match { + case (Some(advertisedHost), _) => advertisedHost + case (None, Some(h)) => h + case (None, None) if conf.get(FRONTEND_CONNECTION_URL_USE_HOSTNAME) => serverAddr.getCanonicalHostName - case None => - serverAddr.getHostAddress + case (None, None) => serverAddr.getHostAddress } host + ":" + actualPort @@ -526,23 +525,20 @@ abstract class TFrontendService(name: String) override def FetchResults(req: TFetchResultsReq): TFetchResultsResp = { debug(req.toString) - val resp = new TFetchResultsResp try { val operationHandle = OperationHandle(req.getOperationHandle) val orientation = FetchOrientation.getFetchOrientation(req.getOrientation) // 1 means fetching log val fetchLog = req.getFetchType == 1 val maxRows = req.getMaxRows.toInt - val rowSet = be.fetchResults(operationHandle, orientation, maxRows, fetchLog) - resp.setResults(rowSet) - resp.setHasMoreRows(false) - resp.setStatus(OK_STATUS) + be.fetchResults(operationHandle, orientation, maxRows, fetchLog) } catch { case e: Exception => error("Error fetching results: ", e) + val resp = new TFetchResultsResp resp.setStatus(KyuubiSQLException.toTStatus(e)) + resp } - resp } protected def notSupportTokenErrorStatus = { @@ -614,7 +610,14 @@ abstract class TFrontendService(name: String) if (handle != null) { info(s"Session [$handle] disconnected without closing properly, close it now") try { - be.closeSession(handle) + val needToClose = be.sessionManager.getSession(handle).conf + .getOrElse(SESSION_CLOSE_ON_DISCONNECT.key, "true").toBoolean + if (needToClose) { + be.closeSession(handle) + } else { + warn(s"Session not actually closed because configuration " + + s"${SESSION_CLOSE_ON_DISCONNECT.key} is set to false") + } } catch { case e: KyuubiSQLException => error("Failed closing session", e) diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/EngineSecuritySecretProvider.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/EngineSecuritySecretProvider.scala index 5bd9e4092..3216a43be 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/EngineSecuritySecretProvider.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/EngineSecuritySecretProvider.scala @@ -18,7 +18,8 @@ package org.apache.kyuubi.service.authentication import org.apache.kyuubi.config.KyuubiConf -import org.apache.kyuubi.config.KyuubiConf.ENGINE_SECURITY_SECRET_PROVIDER +import org.apache.kyuubi.config.KyuubiConf._ +import org.apache.kyuubi.util.reflect.DynConstructors trait EngineSecuritySecretProvider { @@ -33,11 +34,27 @@ trait EngineSecuritySecretProvider { def getSecret(): String } +class SimpleEngineSecuritySecretProviderImpl extends EngineSecuritySecretProvider { + + private var _conf: KyuubiConf = _ + + override def initialize(conf: KyuubiConf): Unit = _conf = conf + + override def getSecret(): String = { + _conf.get(SIMPLE_SECURITY_SECRET_PROVIDER_PROVIDER_SECRET).getOrElse { + throw new IllegalArgumentException( + s"${SIMPLE_SECURITY_SECRET_PROVIDER_PROVIDER_SECRET.key} must be configured " + + s"when ${ENGINE_SECURITY_SECRET_PROVIDER.key} is `simple`.") + } + } +} + object EngineSecuritySecretProvider { def create(conf: KyuubiConf): EngineSecuritySecretProvider = { - val providerClass = Class.forName(conf.get(ENGINE_SECURITY_SECRET_PROVIDER)) - val provider = providerClass.getConstructor().newInstance() - .asInstanceOf[EngineSecuritySecretProvider] + val provider = DynConstructors.builder() + .impl(conf.get(ENGINE_SECURITY_SECRET_PROVIDER)) + .buildChecked[EngineSecuritySecretProvider]() + .newInstance(conf) provider.initialize(conf) provider } diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/InternalSecurityAccessor.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/InternalSecurityAccessor.scala index 62680e6a6..afc1dde1f 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/InternalSecurityAccessor.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/InternalSecurityAccessor.scala @@ -20,6 +20,8 @@ package org.apache.kyuubi.service.authentication import javax.crypto.Cipher import javax.crypto.spec.{IvParameterSpec, SecretKeySpec} +import org.apache.hadoop.classification.VisibleForTesting + import org.apache.kyuubi.{KyuubiSQLException, Logging} import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.config.KyuubiConf._ @@ -121,4 +123,9 @@ object InternalSecurityAccessor extends Logging { def get(): InternalSecurityAccessor = { _engineSecurityAccessor } + + @VisibleForTesting + def reset(): Unit = { + _engineSecurityAccessor = null + } } diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactory.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactory.scala index 5f429fa4e..1b62f6030 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactory.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactory.scala @@ -39,7 +39,7 @@ class KyuubiAuthenticationFactory(conf: KyuubiConf, isServer: Boolean = true) ex private val authTypes = conf.get(AUTHENTICATION_METHOD).map(AuthTypes.withName) private val none = authTypes.contains(NONE) - private val noSasl = authTypes == Seq(NOSASL) + private val noSasl = authTypes == Set(NOSASL) private val kerberosEnabled = authTypes.contains(KERBEROS) private val plainAuthTypeOpt = authTypes.filterNot(_.equals(KERBEROS)) .filterNot(_.equals(NOSASL)).headOption diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/LdapAuthenticationProviderImpl.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/LdapAuthenticationProviderImpl.scala index b5e08def5..d885da55b 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/LdapAuthenticationProviderImpl.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/LdapAuthenticationProviderImpl.scala @@ -17,17 +17,26 @@ package org.apache.kyuubi.service.authentication -import javax.naming.{Context, NamingException} -import javax.naming.directory.InitialDirContext +import javax.naming.NamingException import javax.security.sasl.AuthenticationException import org.apache.commons.lang3.StringUtils +import org.apache.kyuubi.Logging import org.apache.kyuubi.config.KyuubiConf -import org.apache.kyuubi.config.KyuubiConf._ import org.apache.kyuubi.service.ServiceUtils +import org.apache.kyuubi.service.authentication.LdapAuthenticationProviderImpl.FILTER_FACTORIES +import org.apache.kyuubi.service.authentication.ldap._ +import org.apache.kyuubi.service.authentication.ldap.LdapUtils.getUserName -class LdapAuthenticationProviderImpl(conf: KyuubiConf) extends PasswdAuthenticationProvider { +class LdapAuthenticationProviderImpl( + conf: KyuubiConf, + searchFactory: DirSearchFactory = new LdapSearchFactory) + extends PasswdAuthenticationProvider with Logging { + + private val filterOpt: Option[Filter] = FILTER_FACTORIES + .map { f => f.getInstance(conf) } + .collectFirst { case Some(f: Filter) => f } /** * The authenticate method is called by the Kyuubi Server authentication layer @@ -41,47 +50,72 @@ class LdapAuthenticationProviderImpl(conf: KyuubiConf) extends PasswdAuthenticat * @throws AuthenticationException When a user is found to be invalid by the implementation */ override def authenticate(user: String, password: String): Unit = { + + val (usedBind, bindUser, bindPassword) = ( + conf.get(KyuubiConf.AUTHENTICATION_LDAP_BIND_USER), + conf.get(KyuubiConf.AUTHENTICATION_LDAP_BIND_PASSWORD)) match { + case (Some(_bindUser), Some(_bindPw)) => (true, _bindUser, _bindPw) + case _ => + // If no bind user or bind password was specified, + // we assume the user we are authenticating has the ability to search + // the LDAP tree, so we use it as the "binding" account. + // This is the way it worked before bind users were allowed in the LDAP authenticator, + // so we keep existing systems working. + (false, user, password) + } + + var search: DirSearch = null + try { + search = createDirSearch(bindUser, bindPassword) + applyFilter(search, user) + if (usedBind) { + // If we used the bind user, then we need to authenticate again, + // this time using the full user name we got during the bind process. + val username = getUserName(user) + createDirSearch(search.findUserDn(username), password) + } + } catch { + case e: NamingException => + throw new AuthenticationException( + s"Unable to find the user in the LDAP tree. ${e.getMessage}") + } finally { + ServiceUtils.cleanup(logger, search) + } + } + + @throws[AuthenticationException] + private def createDirSearch(user: String, password: String): DirSearch = { if (StringUtils.isBlank(user)) { throw new AuthenticationException(s"Error validating LDAP user, user is null" + s" or contains blank space") } - if (StringUtils.isBlank(password)) { + if (StringUtils.isBlank(password) || password.getBytes()(0) == 0) { throw new AuthenticationException(s"Error validating LDAP user, password is null" + s" or contains blank space") } - val env = new java.util.Hashtable[String, Any]() - env.put(Context.INITIAL_CONTEXT_FACTORY, "com.sun.jndi.ldap.LdapCtxFactory") - env.put(Context.SECURITY_AUTHENTICATION, "simple") - - conf.get(AUTHENTICATION_LDAP_URL).foreach(env.put(Context.PROVIDER_URL, _)) - - val domain = conf.get(AUTHENTICATION_LDAP_DOMAIN) - val u = - if (!hasDomain(user) && domain.nonEmpty) { - user + "@" + domain.get - } else { - user + val principals = LdapUtils.createCandidatePrincipals(conf, user) + val iterator = principals.iterator + while (iterator.hasNext) { + val principal = iterator.next + try { + return searchFactory.getInstance(conf, principal, password) + } catch { + case ex: AuthenticationException => if (iterator.isEmpty) throw ex } - - val guidKey = conf.get(AUTHENTICATION_LDAP_GUIDKEY) - val bindDn = conf.get(AUTHENTICATION_LDAP_BASEDN) match { - case Some(dn) => guidKey + "=" + u + "," + dn - case _ => u } + throw new AuthenticationException(s"No candidate principals for $user was found.") + } - env.put(Context.SECURITY_PRINCIPAL, bindDn) - env.put(Context.SECURITY_CREDENTIALS, password) - - try { - val ctx = new InitialDirContext(env) - ctx.close() - } catch { - case e: NamingException => - throw new AuthenticationException(s"Error validating LDAP user: $bindDn", e) - } + @throws[AuthenticationException] + private def applyFilter(client: DirSearch, user: String): Unit = filterOpt.foreach { filter => + filter.apply(client, getUserName(user)) } +} - private def hasDomain(userName: String): Boolean = ServiceUtils.indexOfDomainMatch(userName) > 0 +object LdapAuthenticationProviderImpl { + val FILTER_FACTORIES: Array[FilterFactory] = Array[FilterFactory]( + CustomQueryFilterFactory, + new ChainFilterFactory(UserSearchFilterFactory, UserFilterFactory, GroupFilterFactory)) } diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/PlainSASLServer.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/PlainSASLServer.scala index 8e84c9f81..737a6d8cd 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/PlainSASLServer.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/PlainSASLServer.scala @@ -23,7 +23,7 @@ import javax.security.auth.callback.{Callback, CallbackHandler, NameCallback, Pa import javax.security.sasl.{AuthorizeCallback, SaslException, SaslServer, SaslServerFactory} import org.apache.kyuubi.KYUUBI_VERSION -import org.apache.kyuubi.engine.SemanticVersion +import org.apache.kyuubi.util.SemanticVersion class PlainSASLServer( handler: CallbackHandler, @@ -126,10 +126,7 @@ object PlainSASLServer { } } - final private val version: Double = { - val runtimeVersion = SemanticVersion(KYUUBI_VERSION) - runtimeVersion.majorVersion + runtimeVersion.minorVersion.toDouble / 10 - } + final private val version = SemanticVersion(KYUUBI_VERSION).toDouble class SaslPlainProvider extends Provider("KyuubiSaslPlain", version, "Kyuubi Plain SASL provider") { diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/ChainFilterFactory.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/ChainFilterFactory.scala new file mode 100644 index 000000000..a5badb15d --- /dev/null +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/ChainFilterFactory.scala @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.service.authentication.ldap + +import javax.security.sasl.AuthenticationException + +import org.apache.kyuubi.config.KyuubiConf + +/** + * A factory that produces a [[Filter]] that is implemented as a chain of other filters. + * The chain of filters are created as a result of [[ChainFilterFactory#getInstance]] method call. + * The resulting object filters out all users that don't pass all chained filters. + * The filters will be applied in the order they are mentioned in the factory constructor. + */ + +class ChainFilterFactory(chainedFactories: FilterFactory*) extends FilterFactory { + override def getInstance(conf: KyuubiConf): Option[Filter] = { + val maybeFilters = chainedFactories.map(_.getInstance(conf)) + val filters = maybeFilters.flatten + if (filters.isEmpty) None else Some(new ChainFilter(filters)) + } +} + +class ChainFilter(chainedFilters: Seq[Filter]) extends Filter { + @throws[AuthenticationException] + override def apply(client: DirSearch, user: String): Unit = { + chainedFilters.foreach(_.apply(client, user)) + } +} diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/CustomQueryFilterFactory.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/CustomQueryFilterFactory.scala new file mode 100644 index 000000000..d10e6523b --- /dev/null +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/CustomQueryFilterFactory.scala @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.service.authentication.ldap + +import javax.naming.NamingException +import javax.security.sasl.AuthenticationException + +import org.apache.kyuubi.Logging +import org.apache.kyuubi.config.KyuubiConf + +/** + * A factory for a [[Filter]] based on a custom query. + *
    + * The produced filter object filters out all users that are not found in the search result + * of the query provided in Kyuubi configuration. + * + * @see [[KyuubiConf.AUTHENTICATION_LDAP_CUSTOM_LDAP_QUERY]] + */ +object CustomQueryFilterFactory extends FilterFactory { + override def getInstance(conf: KyuubiConf): Option[Filter] = + conf.get(KyuubiConf.AUTHENTICATION_LDAP_CUSTOM_LDAP_QUERY) + .map { customQuery => new CustomQueryFilter(customQuery) } +} +class CustomQueryFilter(query: String) extends Filter with Logging { + @throws[AuthenticationException] + override def apply(client: DirSearch, user: String): Unit = { + var resultList: Array[String] = null + try { + resultList = client.executeCustomQuery(query) + } catch { + case e: NamingException => + throw new AuthenticationException(s"LDAP Authentication failed for $user", e) + } + if (resultList != null) { + resultList.foreach { matchedDn => + val shortUserName = LdapUtils.getShortName(matchedDn) + info(s"") + if (shortUserName.equalsIgnoreCase(user) || matchedDn.equalsIgnoreCase(user)) { + info("Authentication succeeded based on result set from LDAP query") + return + } + } + // try a generic user search + if (query.contains("%s")) { + val userSearchQuery = query.replace("%s", user) + info("Trying with generic user search in ldap:" + userSearchQuery) + try resultList = client.executeCustomQuery(userSearchQuery) + catch { + case e: NamingException => + throw new AuthenticationException("LDAP Authentication failed for user", e) + } + if (resultList != null && resultList.length == 1) { + info("Authentication succeeded based on result from custom user search query") + return + } + } + } + info("Authentication failed based on result set from custom LDAP query") + throw new AuthenticationException( + "Authentication failed: LDAP query from property returned no data") + } +} diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/DirSearch.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/DirSearch.scala new file mode 100644 index 000000000..c1c4d5060 --- /dev/null +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/DirSearch.scala @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.service.authentication.ldap + +import java.io.Closeable +import javax.naming.NamingException + +/** + * The object used for executing queries on the Directory Service. + */ +trait DirSearch extends Closeable { + + /** + * Finds user's distinguished name. + * + * @param user username + * @return DN for the specified username + */ + @throws[NamingException] + def findUserDn(user: String): String + + /** + * Finds group's distinguished name. + * + * @param group group name or unique identifier + * @return DN for the specified group name + */ + @throws[NamingException] + def findGroupDn(group: String): String + + /** + * Verifies that specified user is a member of specified group. + * + * @param user user id or distinguished name + * @param groupDn group's DN + * @return true if the user is a member of the group, false - otherwise. + */ + @throws[NamingException] + def isUserMemberOfGroup(user: String, groupDn: String): Boolean + + /** + * Finds groups that contain the specified user. + * + * @param userDn user's distinguished name + * @return list of groups + */ + @throws[NamingException] + def findGroupsForUser(userDn: String): Array[String] + + /** + * Executes an arbitrary query. + * + * @param query any query + * @return list of names in the namespace + */ + @throws[NamingException] + def executeCustomQuery(query: String): Array[String] +} diff --git a/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/datalake/HudiOperationSuite.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/DirSearchFactory.scala similarity index 62% rename from kyuubi-server/src/test/scala/org/apache/kyuubi/operation/datalake/HudiOperationSuite.scala rename to kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/DirSearchFactory.scala index 0c507504d..2046632d8 100644 --- a/kyuubi-server/src/test/scala/org/apache/kyuubi/operation/datalake/HudiOperationSuite.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/DirSearchFactory.scala @@ -15,20 +15,25 @@ * limitations under the License. */ -package org.apache.kyuubi.operation.datalake +package org.apache.kyuubi.service.authentication.ldap + +import javax.security.sasl.AuthenticationException -import org.apache.kyuubi.WithKyuubiServer import org.apache.kyuubi.config.KyuubiConf -import org.apache.kyuubi.operation.HudiMetadataTests -import org.apache.kyuubi.tags.HudiTest -@HudiTest -class HudiOperationSuite extends WithKyuubiServer with HudiMetadataTests { - override protected val conf: KyuubiConf = { - val kyuubiConf = KyuubiConf().set(KyuubiConf.ENGINE_IDLE_TIMEOUT, 20000L) - extraConfigs.foreach { case (k, v) => kyuubiConf.set(k, v) } - kyuubiConf - } +/** + * A factory for [[DirSearch]]. + */ +trait DirSearchFactory { - override def jdbcUrl: String = getJdbcUrl + /** + * Returns an instance of [[DirSearch]]. + * + * @param conf Kyuubi configuration + * @param user username + * @param password user password + * @return instance of [[DirSearch]] + */ + @throws[AuthenticationException] + def getInstance(conf: KyuubiConf, user: String, password: String): DirSearch } diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/Filter.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/Filter.scala new file mode 100644 index 000000000..e57eddb0d --- /dev/null +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/Filter.scala @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.service.authentication.ldap + +import javax.security.sasl.AuthenticationException + +/** + * The object that filters LDAP users. + *
    + * The assumption is that this user was already authenticated by a previous bind operation. + */ +trait Filter { + + /** + * Applies this filter to the authenticated user. + * + * @param client LDAP client that will be used for execution of LDAP queries. + * @param user username + */ + @throws[AuthenticationException] + def apply(client: DirSearch, user: String): Unit +} diff --git a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/trino/api/KyuubiScalaObjectMapper.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/FilterFactory.scala similarity index 64% rename from kyuubi-server/src/main/scala/org/apache/kyuubi/server/trino/api/KyuubiScalaObjectMapper.scala rename to kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/FilterFactory.scala index 915b109b7..d85104684 100644 --- a/kyuubi-server/src/main/scala/org/apache/kyuubi/server/trino/api/KyuubiScalaObjectMapper.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/FilterFactory.scala @@ -15,15 +15,20 @@ * limitations under the License. */ -package org.apache.kyuubi.server.trino.api +package org.apache.kyuubi.service.authentication.ldap -import javax.ws.rs.ext.ContextResolver +import org.apache.kyuubi.config.KyuubiConf -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.module.scala.DefaultScalaModule - -class KyuubiScalaObjectMapper extends ContextResolver[ObjectMapper] { - private val mapper = new ObjectMapper().registerModule(DefaultScalaModule) +/** + * Factory for the filter. + */ +trait FilterFactory { - override def getContext(aClass: Class[_]): ObjectMapper = mapper + /** + * Returns an instance of the corresponding filter. + * + * @param conf Kyuubi configurations used to configure the filter. + * @return Some(filter) or None if this filter doesn't support provided set of properties + */ + def getInstance(conf: KyuubiConf): Option[Filter] } diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/GroupFilterFactory.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/GroupFilterFactory.scala new file mode 100644 index 000000000..f3048ea6f --- /dev/null +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/GroupFilterFactory.scala @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.service.authentication.ldap + +import javax.naming.NamingException +import javax.security.sasl.AuthenticationException + +import scala.collection.mutable.ArrayBuffer + +import org.apache.kyuubi.Logging +import org.apache.kyuubi.config.KyuubiConf + +object GroupFilterFactory extends FilterFactory { + override def getInstance(conf: KyuubiConf): Option[Filter] = { + val groupFilter = conf.get(KyuubiConf.AUTHENTICATION_LDAP_GROUP_FILTER) + if (groupFilter.isEmpty) { + None + } else if (conf.get(KyuubiConf.AUTHENTICATION_LDAP_USER_MEMBERSHIP_KEY).isDefined) { + Some(new UserMembershipKeyFilter(groupFilter)) + } else { + Some(new GroupMembershipKeyFilter(groupFilter)) + } + } +} + +class GroupMembershipKeyFilter(groupFilter: Set[String]) extends Filter with Logging { + + @throws[AuthenticationException] + override def apply(ldap: DirSearch, user: String): Unit = { + info(s"Authenticating user '$user' using ${classOf[GroupMembershipKeyFilter].getSimpleName})") + + var memberOf: Array[String] = null + try { + val userDn = ldap.findUserDn(user) + // Workaround for magic things on Mockito: + // unmatched invocation returns an empty list if the method return type is JList, + // but null if the method return type is Array + memberOf = Option(ldap.findGroupsForUser(userDn)).getOrElse(Array.empty) + debug(s"User $userDn member of: ${memberOf.mkString(",")}") + } catch { + case e: NamingException => + throw new AuthenticationException("LDAP Authentication failed for user", e) + } + memberOf.foreach { groupDn => + val shortName = LdapUtils.getShortName(groupDn) + if (groupFilter.exists(shortName.equalsIgnoreCase)) { + debug(s"GroupMembershipKeyFilter passes: user '$user' is a member of '$groupDn' group") + info("Authentication succeeded based on group membership") + return + } + } + info("Authentication failed based on user membership") + throw new AuthenticationException( + "Authentication failed: User not a member of specified list") + } +} + +class UserMembershipKeyFilter(groupFilter: Set[String]) extends Filter with Logging { + @throws[AuthenticationException] + override def apply(ldap: DirSearch, user: String): Unit = { + info(s"Authenticating user '$user' using $classOf[UserMembershipKeyFilter].getSimpleName") + val groupDns = new ArrayBuffer[String] + groupFilter.foreach { groupId => + try { + val groupDn = ldap.findGroupDn(groupId) + groupDns += groupDn + } catch { + case e: NamingException => + warn("Cannot find DN for group", e) + debug(s"Cannot find DN for group $groupId", e) + } + } + if (groupDns.isEmpty) { + debug(s"No DN(s) has been found for any of group(s): ${groupFilter.mkString(",")}") + throw new AuthenticationException("No DN(s) has been found for any of specified group(s)") + } + groupDns.foreach { groupDn => + try { + if (ldap.isUserMemberOfGroup(user, groupDn)) { + debug(s"UserMembershipKeyFilter passes: user '$user' is a member of '$groupDn' group") + info("Authentication succeeded based on user membership") + return + } + } catch { + case e: NamingException => + warn("Cannot match user and group", e) + debug(s"Cannot match user '$user' and group '$groupDn'", e) + } + } + throw new AuthenticationException( + s"Authentication failed: User '$user' is not a member of listed groups") + } +} diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/LdapSearch.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/LdapSearch.scala new file mode 100644 index 000000000..09dca1d5c --- /dev/null +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/LdapSearch.scala @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.service.authentication.ldap + +import javax.naming.{NamingEnumeration, NamingException} +import javax.naming.directory.{DirContext, SearchResult} + +import scala.collection.mutable.ArrayBuffer + +import org.apache.kyuubi.Logging +import org.apache.kyuubi.config.KyuubiConf + +/** + * Implements search for LDAP. + * @param conf Kyuubi configuration + * @param ctx Directory service that will be used for the queries. + */ +class LdapSearch(conf: KyuubiConf, ctx: DirContext) extends DirSearch with Logging { + + final private val baseDn = conf.get(KyuubiConf.AUTHENTICATION_LDAP_BASE_DN).orNull + final private val groupBases: Array[String] = + LdapUtils.patternsToBaseDns( + LdapUtils.parseDnPatterns(conf, KyuubiConf.AUTHENTICATION_LDAP_GROUP_DN_PATTERN)) + final private val userPatterns: Array[String] = + LdapUtils.parseDnPatterns(conf, KyuubiConf.AUTHENTICATION_LDAP_USER_DN_PATTERN) + final private val userBases: Array[String] = LdapUtils.patternsToBaseDns(userPatterns) + final private val queries: QueryFactory = new QueryFactory(conf) + + /** + * Closes this search object and releases any system resources associated + * with it. If the search object is already closed then invoking this + * method has no effect. + */ + override def close(): Unit = { + try ctx.close() + catch { + case e: NamingException => + warn("Exception when closing LDAP context:", e) + } + } + + @throws[NamingException] + override def findUserDn(user: String): String = { + var allLdapNames: Array[String] = null + if (LdapUtils.isDn(user)) { + val userBaseDn: String = LdapUtils.extractBaseDn(user) + val userRdn: String = LdapUtils.extractFirstRdn(user) + allLdapNames = execute(Array(userBaseDn), queries.findUserDnByRdn(userRdn)).getAllLdapNames + } else { + allLdapNames = findDnByPattern(userPatterns, user) + if (allLdapNames.isEmpty) { + allLdapNames = execute(userBases, queries.findUserDnByName(user)).getAllLdapNames + } + } + if (allLdapNames.length == 1) allLdapNames.head + else { + info(s"Expected exactly one user result for the user: $user, " + + s"but got ${allLdapNames.length}. Returning null") + debug("Matched users: $allLdapNames") + null + } + } + + @throws[NamingException] + private def findDnByPattern(patterns: Seq[String], name: String): Array[String] = { + for (pattern <- patterns) { + val baseDnFromPattern: String = LdapUtils.extractBaseDn(pattern) + val rdn = LdapUtils.extractFirstRdn(pattern).replaceAll("%s", name) + val names = execute(Array(baseDnFromPattern), queries.findDnByPattern(rdn)).getAllLdapNames + if (!names.isEmpty) return names + } + Array.empty + } + + @throws[NamingException] + override def findGroupDn(group: String): String = + execute(groupBases, queries.findGroupDnById(group)).getSingleLdapName + + @throws[NamingException] + override def isUserMemberOfGroup(user: String, groupDn: String): Boolean = { + val userId = LdapUtils.extractUserName(user) + execute(userBases, queries.isUserMemberOfGroup(userId, groupDn)).hasSingleResult + } + + @throws[NamingException] + override def findGroupsForUser(userDn: String): Array[String] = { + val userName = LdapUtils.extractUserName(userDn) + execute(groupBases, queries.findGroupsForUser(userName, userDn)).getAllLdapNames + } + + @throws[NamingException] + override def executeCustomQuery(query: String): Array[String] = + execute(Array(baseDn), queries.customQuery(query)).getAllLdapNamesAndAttributes + + private def execute(baseDns: Array[String], query: Query): SearchResultHandler = { + val searchResults = new ArrayBuffer[NamingEnumeration[SearchResult]] + debug(s"Executing a query: '${query.filter}' with base DNs ${baseDns.mkString(",")}") + baseDns.foreach { baseDn => + try { + val searchResult = ctx.search(baseDn, query.filter, query.controls) + if (searchResult != null) searchResults += searchResult + } catch { + case ex: NamingException => + debug( + s"Exception happened for query '${query.filter}' with base DN '$baseDn'", + ex) + } + } + new SearchResultHandler(searchResults.toArray) + } +} diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/LdapSearchFactory.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/LdapSearchFactory.scala new file mode 100644 index 000000000..e3649d359 --- /dev/null +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/LdapSearchFactory.scala @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.service.authentication.ldap + +import java.util +import javax.naming.{Context, NamingException} +import javax.naming.directory.{DirContext, InitialDirContext} +import javax.security.sasl.AuthenticationException + +import org.apache.kyuubi.Logging +import org.apache.kyuubi.config.KyuubiConf + +class LdapSearchFactory extends DirSearchFactory with Logging { + @throws[AuthenticationException] + override def getInstance(conf: KyuubiConf, principal: String, password: String): DirSearch = { + try { + val ctx = createDirContext(conf, principal, password) + new LdapSearch(conf, ctx) + } catch { + case e: NamingException => + debug(s"Could not connect to the LDAP Server: Authentication failed for $principal") + throw new AuthenticationException(s"Error validating LDAP user: $principal", e) + } + } + + @throws[NamingException] + private def createDirContext( + conf: KyuubiConf, + principal: String, + password: String): DirContext = { + val ldapUrl = conf.get(KyuubiConf.AUTHENTICATION_LDAP_URL) + val env = new util.Hashtable[String, AnyRef] + ldapUrl.foreach(env.put(Context.PROVIDER_URL, _)) + env.put(Context.INITIAL_CONTEXT_FACTORY, "com.sun.jndi.ldap.LdapCtxFactory") + env.put(Context.SECURITY_AUTHENTICATION, "simple") + env.put(Context.SECURITY_PRINCIPAL, principal) + env.put(Context.SECURITY_CREDENTIALS, password) + debug(s"Connecting using principal $principal to ldap server: ${ldapUrl.orNull}") + new InitialDirContext(env) + } +} diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/LdapUtils.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/LdapUtils.scala new file mode 100644 index 000000000..e304e96f7 --- /dev/null +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/LdapUtils.scala @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.service.authentication.ldap + +import scala.collection.mutable.ArrayBuffer + +import org.apache.kyuubi.Logging +import org.apache.kyuubi.config.{KyuubiConf, OptionalConfigEntry} +import org.apache.kyuubi.service.ServiceUtils + +/** + * Static utility methods related to LDAP authentication module. + */ +object LdapUtils extends Logging { + + /** + * Extracts a base DN from the provided distinguished name. + *
    + * Example: + *
    + * "ou=CORP,dc=mycompany,dc=com" is the base DN for "cn=user1,ou=CORP,dc=mycompany,dc=com" + * + * @param dn distinguished name + * @return base DN + */ + def extractBaseDn(dn: String): String = { + val indexOfFirstDelimiter = dn.indexOf(",") + if (indexOfFirstDelimiter > -1) { + return dn.substring(indexOfFirstDelimiter + 1) + } + null + } + + /** + * Extracts the first Relative Distinguished Name (RDN). + *
    + * Example: + *
    + * For DN "cn=user1,ou=CORP,dc=mycompany,dc=com" this method will return "cn=user1" + * + * @param dn distinguished name + * @return first RDN + */ + def extractFirstRdn(dn: String): String = dn.substring(0, dn.indexOf(",")) + + /** + * Extracts username from user DN. + *
    + * Examples: + * 
    +   * LdapUtils.extractUserName("UserName")                        = "UserName"
+   * LdapUtils.extractUserName("UserName@mycorp.com")             = "UserName"
+   * LdapUtils.extractUserName("cn=UserName,dc=mycompany,dc=com") = "UserName"
+   *
    + */ + def extractUserName(userDn: String): String = { + if (!isDn(userDn) && !hasDomain(userDn)) { + return userDn + } + val domainIdx: Int = ServiceUtils.indexOfDomainMatch(userDn) + if (domainIdx > 0) { + return userDn.substring(0, domainIdx) + } + if (userDn.contains("=")) { + return userDn.substring(userDn.indexOf("=") + 1, userDn.indexOf(",")) + } + userDn + } + + /** + * Gets value part of the first attribute in the provided RDN. + *
    + * Example: + *
    + * For RDN "cn=user1,ou=CORP" this method will return "user1" + * + * @param rdn Relative Distinguished Name + * @return value part of the first attribute + */ + def getShortName(rdn: String): String = rdn.split(",")(0).split("=")(1) + + /** + * Check for a domain part in the provided username. + *
    + * Example: + *
    + * 
    +   * LdapUtils.hasDomain("user1@mycorp.com") = true
+   * LdapUtils.hasDomain("user1")            = false
+   *
    + * + * @param userName username + * @return true if `userName` contains `@` part + */ + def hasDomain(userName: String): Boolean = { + ServiceUtils.indexOfDomainMatch(userName) > 0 + } + + /** + * Get the username part in the provided user. + *
    + * Example: + *
    + * For user "user1@mycorp.com" this method will return "user1" + * + * @param user user + * @return the username part in the provided user + */ + def getUserName(user: String): String = + if (LdapUtils.hasDomain(user)) LdapUtils.extractUserName(user) else user + + /** + * Detects DN names. + *
    + * Example: + *
    + * 
    +   * LdapUtils.isDn("cn=UserName,dc=mycompany,dc=com") = true
+   * LdapUtils.isDn("user1")                           = false
+   *
    + * + * @param name name to be checked + * @return true if the provided name is a distinguished name + */ + def isDn(name: String): Boolean = { + name.contains("=") + } + + /** + * Reads and parses DN patterns from Kyuubi configuration. + *
    + * If no patterns are provided in the configuration, then the base DN will be used. + * + * @param conf Kyuubi configuration + * @param confKey configuration key to be read + * @return a list of DN patterns + * @see [[KyuubiConf.AUTHENTICATION_LDAP_BASE_DN]] + * @see [[KyuubiConf.AUTHENTICATION_LDAP_GUID_KEY]] + * @see [[KyuubiConf.AUTHENTICATION_LDAP_GROUP_DN_PATTERN]] + * @see [[KyuubiConf.AUTHENTICATION_LDAP_USER_DN_PATTERN]] + */ + def parseDnPatterns(conf: KyuubiConf, confKey: OptionalConfigEntry[String]): Array[String] = { + val result = new ArrayBuffer[String] + conf.get(confKey).map { patternsString => + patternsString.split(":").foreach { pattern => + if (pattern.contains(",") && pattern.contains("=")) { + result += pattern + } else { + warn(s"Unexpected format for $confKey, ignoring $pattern") + } + } + }.getOrElse { + val guidAttr = conf.get(KyuubiConf.AUTHENTICATION_LDAP_GUID_KEY) + conf.get(KyuubiConf.AUTHENTICATION_LDAP_BASE_DN).foreach { defaultBaseDn => + result += s"$guidAttr=%s,$defaultBaseDn" + } + } + result.toArray + } + + private def patternToBaseDn(pattern: String): String = + if (pattern.contains("=%s")) pattern.split(",", 2)(1) else pattern + + /** + * Converts a collection of Distinguished Name patterns to a collection of base DNs. + * + * @param patterns Distinguished Name patterns + * @return a list of base DNs + * @see [[KyuubiConf.AUTHENTICATION_LDAP_GROUP_DN_PATTERN]] + * @see [[KyuubiConf.AUTHENTICATION_LDAP_USER_DN_PATTERN]] + */ + def patternsToBaseDns(patterns: Array[String]): Array[String] = { + patterns.map(patternToBaseDn) + } + + /** + * Creates a list of principals to be used for user authentication. + * + * @param conf Kyuubi configuration + * @param user username + * @return a list of user's principals + */ + def createCandidatePrincipals(conf: KyuubiConf, user: String): Array[String] = { + if (hasDomain(user) || isDn(user)) { + return Array(user) + } + conf.get(KyuubiConf.AUTHENTICATION_LDAP_DOMAIN).map { ldapDomain => + Array(user + "@" + ldapDomain) + }.getOrElse { + val userPatterns = parseDnPatterns(conf, KyuubiConf.AUTHENTICATION_LDAP_USER_DN_PATTERN) + if (userPatterns.isEmpty) { + return Array(user) + } + userPatterns.map(_.replaceAll("%s", user)) + } + } +} diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/Query.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/Query.scala new file mode 100644 index 000000000..ce9a7d472 --- /dev/null +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/Query.scala @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.service.authentication.ldap + +import java.util +import javax.naming.directory.SearchControls + +import org.stringtemplate.v4.ST + +/** + * The object that encompasses all components of a Directory Service search query. + * + * @see [[LdapSearch]] + */ +object Query { + + /** + * Creates Query Builder. + * + * @return query builder. + */ + def builder: Query.QueryBuilder = new Query.QueryBuilder + + /** + * A builder of the [[Query]]. + */ + final class QueryBuilder { + private var filterTemplate: ST = _ + private val controls: SearchControls = { + val _controls = new SearchControls + _controls.setSearchScope(SearchControls.SUBTREE_SCOPE) + _controls.setReturningAttributes(new Array[String](0)) + _controls + } + private val returningAttributes: util.List[String] = new util.ArrayList[String] + + /** + * Sets search filter template. + * + * @param filterTemplate search filter template + * @return the current instance of the builder + */ + def filter(filterTemplate: String): Query.QueryBuilder = { + this.filterTemplate = new ST(filterTemplate) + this + } + + /** + * Sets mapping between names in the search filter template and actual values. + * + * @param key marker in the search filter template. + * @param value actual value + * @return the current instance of the builder + */ + def map(key: String, value: String): Query.QueryBuilder = { + filterTemplate.add(key, value) + this + } + + /** + * Sets mapping between names in the search filter template and actual values. + * + * @param key marker in the search filter template. + * @param values array of values + * @return the current instance of the builder + */ + def map(key: String, values: Array[String]): Query.QueryBuilder = { + filterTemplate.add(key, values) + this + } + + /** + * Sets attribute that should be returned in results for the query. + * + * @param attributeName attribute name + * @return the current instance of the builder + */ + def returnAttribute(attributeName: String): Query.QueryBuilder = { + returningAttributes.add(attributeName) + this + } + + /** + * Sets the maximum number of entries to be returned as a result of the search. + *
    + * 0 indicates no limit: all entries will be returned. + * + * @param limit The maximum number of entries that will be returned. + * @return the current instance of the builder + */ + def limit(limit: Int): Query.QueryBuilder = { + controls.setCountLimit(limit) + this + } + + private def validate(): Unit = { + require(filterTemplate != null, "filter is required for LDAP search query") + } + + private def createFilter: String = filterTemplate.render + + private def updateControls(): Unit = { + if (!returningAttributes.isEmpty) controls.setReturningAttributes( + returningAttributes.toArray(new Array[String](returningAttributes.size))) + } + + /** + * Builds an instance of [[Query]]. + * + * @return configured directory service query + */ + def build: Query = { + validate() + val filter: String = createFilter + updateControls() + new Query(filter, controls) + } + } +} + +case class Query(filter: String, controls: SearchControls) diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/QueryFactory.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/QueryFactory.scala new file mode 100644 index 000000000..849006e38 --- /dev/null +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/QueryFactory.scala @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.service.authentication.ldap + +import org.apache.kyuubi.config.KyuubiConf + +/** + * A factory for common types of directory service search queries. + */ +final class QueryFactory(conf: KyuubiConf) { + private val USER_OBJECT_CLASSES = Array("person", "user", "inetOrgPerson") + + private val guidAttr = conf.get(KyuubiConf.AUTHENTICATION_LDAP_GUID_KEY) + private val groupClassAttr = conf.get(KyuubiConf.AUTHENTICATION_LDAP_GROUP_CLASS_KEY) + private val groupMembershipAttr = conf.get(KyuubiConf.AUTHENTICATION_LDAP_GROUP_MEMBERSHIP_KEY) + private val userMembershipAttrOpt = conf.get(KyuubiConf.AUTHENTICATION_LDAP_USER_MEMBERSHIP_KEY) + + /** + * Returns a query for finding Group DN based on group unique ID. + * + * @param groupId group unique identifier + * @return an instance of [[Query]] + */ + def findGroupDnById(groupId: String): Query = Query.builder + .filter("(&(objectClass=)(=))") + .map("guidAttr", guidAttr) + .map("groupClassAttr", groupClassAttr) + .map("groupID", groupId).limit(2) + .build + + /** + * Returns a query for finding user DN based on user RDN. + * + * @param userRdn user RDN + * @return an instance of [[Query]] + */ + def findUserDnByRdn(userRdn: String): Query = Query.builder + .filter("(&(|)}>)())") + .limit(2) + .map("classes", USER_OBJECT_CLASSES) + .map("userRdn", userRdn).build + + /** + * Returns a query for finding user DN based on DN pattern. + *
    + * Name of this method was derived from the original implementation of LDAP authentication. + * This method should be replaced by [[QueryFactory.findUserDnByRdn]]. + * + * @param rdn user RDN + * @return an instance of [[Query]] + */ + def findDnByPattern(rdn: String): Query = Query.builder + .filter("()") + .map("rdn", rdn) + .limit(2) + .build + + /** + * Returns a query for finding user DN based on user unique name. + * + * @param userName user unique name (uid or sAMAccountName) + * @return an instance of [[Query]] + */ + def findUserDnByName(userName: String): Query = Query.builder + .filter("(&(|)}>)" + + "(|(uid=)(sAMAccountName=)))") + .map("classes", USER_OBJECT_CLASSES) + .map("userName", userName) + .limit(2) + .build + + /** + * Returns a query for finding groups to which the user belongs. + * + * @param userName username + * @param userDn user DN + * @return an instance of [[Query]] + */ + def findGroupsForUser(userName: String, userDn: String): Query = Query.builder + .filter("(&(objectClass=)" + + "(|(=)(=)))") + .map("groupClassAttr", groupClassAttr) + .map("groupMembershipAttr", groupMembershipAttr) + .map("userName", userName) + .map("userDn", userDn) + .build + + /** + * Returns a query for checking whether specified user is a member of specified group. + * + * The query requires [[KyuubiConf.AUTHENTICATION_LDAP_USER_MEMBERSHIP_KEY]] + * configuration property to be set. + * + * @param userId user unique identifier + * @param groupDn group DN + * @return an instance of [[Query]] + * @see [[KyuubiConf.AUTHENTICATION_LDAP_USER_MEMBERSHIP_KEY]] + */ + def isUserMemberOfGroup(userId: String, groupDn: String): Query = { + require( + userMembershipAttrOpt.isDefined, + s"${KyuubiConf.AUTHENTICATION_LDAP_USER_MEMBERSHIP_KEY.key} is not configured.") + + Query.builder + .filter("(&(|)}>)" + + "(=)(=))") + .map("classes", USER_OBJECT_CLASSES) + .map("guidAttr", guidAttr) + .map("userMembershipAttr", userMembershipAttrOpt.get) + .map("userId", userId) + .map("groupDn", groupDn) + .limit(2) + .build + } + + /** + * Returns a query object created for the custom filter. + *
    + * This query is configured to return a group membership attribute as part of the search result. + * + * @param searchFilter custom search filter + * @return an instance of [[Query]] + */ + def customQuery(searchFilter: String): Query = { + val builder = Query.builder + builder.filter(searchFilter) + builder.returnAttribute(groupMembershipAttr) + builder.build + } +} diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/SearchResultHandler.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/SearchResultHandler.scala new file mode 100644 index 000000000..52d5b6a90 --- /dev/null +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/SearchResultHandler.scala @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.service.authentication.ldap + +import javax.naming.{NamingEnumeration, NamingException} +import javax.naming.directory.SearchResult + +import scala.collection.mutable.ArrayBuffer + +import org.apache.kyuubi.Logging + +/** + * The object that handles Directory Service search results. + * In most cases it converts search results into a list of names in the namespace. + */ +object SearchResultHandler { + + /** + * An interface used by [[SearchResultHandler]] for processing records of + * a [[SearchResult]] on a per-record basis. + *
    + * Implementations of this interface perform the actual work of processing each record, + * but don't need to worry about exception handling, closing underlying data structures, + * and combining results from several search requests. + * + * @see SearchResultHandler + */ + trait RecordProcessor extends (SearchResult => Boolean) { + + /** + * Implementations must implement this method to process each record in [[SearchResult]]. + * + * @param record the [[SearchResult]] to precess + * @return true to continue processing, false to stop iterating + * over search results + */ + @throws[NamingException] + override def apply(record: SearchResult): Boolean + } +} + +/** + * Constructs a search result handler object for the provided search results. + * + * @param searchResults directory service search results + */ +class SearchResultHandler(val searchResults: Array[NamingEnumeration[SearchResult]]) + extends Logging { + + /** + * Returns all entries from the search result. + * + * @return a list of names in the namespace + */ + @throws[NamingException] + def getAllLdapNames: Array[String] = { + val result = new ArrayBuffer[String] + handle { record => result += record.getNameInNamespace; true } + result.toArray + } + + /** + * Checks whether search result contains exactly one entry. + * + * @return true if the search result contains a single entry. + */ + @throws[NamingException] + def hasSingleResult: Boolean = { + val allResults = getAllLdapNames + allResults != null && allResults.length == 1 + } + + /** + * Returns a single entry from the search result. + * Throws [[NamingException]] if the search result doesn't contain exactly one entry. + * + * @return name in the namespace + */ + @throws[NamingException] + def getSingleLdapName: String = { + val allLdapNames = getAllLdapNames + if (allLdapNames.length == 1) return allLdapNames.head + throw new NamingException("Single result was expected") + } + + /** + * Returns all entries and all attributes for these entries. + * + * @return a list that includes all entries and all attributes from these entries. + */ + @throws[NamingException] + def getAllLdapNamesAndAttributes: Array[String] = { + val result = new ArrayBuffer[String] + + @throws[NamingException] + def addAllAttributeValuesToResult(values: NamingEnumeration[_]): Unit = { + while (values.hasMore) result += String.valueOf(values.next) + } + handle { record => + result += record.getNameInNamespace + val allAttributes = record.getAttributes.getAll + while (allAttributes.hasMore) { + val attribute = allAttributes.next + addAllAttributeValuesToResult(attribute.getAll) + } + true + } + result.toArray + } + + /** + * Allows for custom processing of the search results. + * + * @param processor [[SearchResultHandler.RecordProcessor]] implementation + */ + @throws[NamingException] + def handle(processor: SearchResultHandler.RecordProcessor): Unit = { + try { + searchResults.foreach { searchResult => + while (searchResult.hasMore) if (!processor.apply(searchResult.next)) return + } + } finally { + searchResults.foreach { searchResult => + try { + searchResult.close() + } catch { + case ex: NamingException => + warn("Failed to close LDAP search result", ex) + } + } + } + } +} diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/UserFilterFactory.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/UserFilterFactory.scala new file mode 100644 index 000000000..3af3c66f5 --- /dev/null +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/UserFilterFactory.scala @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.service.authentication.ldap + +import javax.security.sasl.AuthenticationException + +import org.apache.kyuubi.Logging +import org.apache.kyuubi.config.KyuubiConf + +object UserFilterFactory extends FilterFactory with Logging { + override def getInstance(conf: KyuubiConf): Option[Filter] = { + val userFilter = conf.get(KyuubiConf.AUTHENTICATION_LDAP_USER_FILTER) + if (userFilter.isEmpty) None else Some(new UserFilter(userFilter)) + } +} + +class UserFilter(_userFilter: Set[String]) extends Filter with Logging { + + lazy val userFilter: Set[String] = _userFilter.map(_.toLowerCase) + + @throws[AuthenticationException] + override def apply(ldap: DirSearch, user: String): Unit = { + info(s"Authenticating user '$user' using user filter") + val userName = LdapUtils.extractUserName(user).toLowerCase + if (!userFilter.contains(userName)) { + info("Authentication failed based on user membership") + throw new AuthenticationException( + "Authentication failed: User not a member of specified list") + } + } +} diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/UserSearchFilterFactory.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/UserSearchFilterFactory.scala new file mode 100644 index 000000000..9e8bdf364 --- /dev/null +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/service/authentication/ldap/UserSearchFilterFactory.scala @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.service.authentication.ldap + +import javax.naming.NamingException +import javax.security.sasl.AuthenticationException + +import org.apache.kyuubi.config.KyuubiConf + +/** + * A factory for a [[Filter]] that check whether provided user could be found in the directory. + *
    + * The produced filter object filters out all users that are not found in the directory. + */ +object UserSearchFilterFactory extends FilterFactory { + override def getInstance(conf: KyuubiConf): Option[Filter] = { + val groupFilter = conf.get(KyuubiConf.AUTHENTICATION_LDAP_GROUP_FILTER) + val userFilter = conf.get(KyuubiConf.AUTHENTICATION_LDAP_USER_FILTER) + if (groupFilter.isEmpty && userFilter.isEmpty) None else Some(UserSearchFilter) + } +} + +object UserSearchFilter extends Filter { + @throws[AuthenticationException] + override def apply(client: DirSearch, user: String): Unit = { + try { + val userDn = client.findUserDn(user) + // This should not be null because we were allowed to bind with this username + // safe check in case we were able to bind anonymously. + if (userDn == null) { + throw new AuthenticationException("Authentication failed: User search failed") + } + } catch { + case e: NamingException => + throw new AuthenticationException("LDAP Authentication failed for user", e) + } + } +} diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/session/AbstractSession.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/session/AbstractSession.scala index 1a8c51ccd..a9e33f5a0 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/session/AbstractSession.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/session/AbstractSession.scala @@ -233,7 +233,7 @@ abstract class AbstractSession( operationHandle: OperationHandle, orientation: FetchOrientation, maxRows: Int, - fetchLog: Boolean): TRowSet = { + fetchLog: Boolean): TFetchResultsResp = { if (fetchLog) { sessionManager.operationManager.getOperationLogRowSet(operationHandle, orientation, maxRows) } else { diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/session/Session.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/session/Session.scala index bc9f9a8f6..2cdac9f3a 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/session/Session.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/session/Session.scala @@ -17,7 +17,7 @@ package org.apache.kyuubi.session -import org.apache.hive.service.rpc.thrift.{TGetInfoType, TGetInfoValue, TGetResultSetMetadataResp, TProtocolVersion, TRowSet} +import org.apache.hive.service.rpc.thrift.{TFetchResultsResp, TGetInfoType, TGetInfoValue, TGetResultSetMetadataResp, TProtocolVersion} import org.apache.kyuubi.operation.FetchOrientation.FetchOrientation import org.apache.kyuubi.operation.OperationHandle @@ -91,7 +91,7 @@ trait Session { operationHandle: OperationHandle, orientation: FetchOrientation, maxRows: Int, - fetchLog: Boolean): TRowSet + fetchLog: Boolean): TFetchResultsResp def closeExpiredOperations(): Unit } diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/session/SessionManager.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/session/SessionManager.scala index 662ac3e58..a83335102 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/session/SessionManager.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/session/SessionManager.scala @@ -23,6 +23,7 @@ import java.util.concurrent.{ConcurrentHashMap, Future, ThreadPoolExecutor, Time import scala.collection.JavaConverters._ import scala.concurrent.duration.Duration +import scala.util.control.NonFatal import org.apache.hive.service.rpc.thrift.TProtocolVersion @@ -172,6 +173,11 @@ abstract class SessionManager(name: String) extends CompositeService(name) { execPool.getActiveCount } + def getWorkQueueSize: Int = { + assert(execPool != null) + execPool.getQueue.size() + } + private var _confRestrictList: Set[String] = _ private var _confIgnoreList: Set[String] = _ private var _batchConfIgnoreList: Set[String] = _ @@ -204,11 +210,11 @@ abstract class SessionManager(name: String) extends CompositeService(name) { key } - if (_confRestrictMatchList.exists(normalizedKey.startsWith(_)) || + if (_confRestrictMatchList.exists(normalizedKey.startsWith) || _confRestrictList.contains(normalizedKey)) { throw KyuubiSQLException(s"$normalizedKey is a restrict key according to the server-side" + s" configuration, please remove it and retry if you want to proceed") - } else if (_confIgnoreMatchList.exists(normalizedKey.startsWith(_)) || + } else if (_confIgnoreMatchList.exists(normalizedKey.startsWith) || _confIgnoreList.contains(normalizedKey)) { warn(s"$normalizedKey is a ignored key according to the server-side configuration") None @@ -223,7 +229,7 @@ abstract class SessionManager(name: String) extends CompositeService(name) { // validate whether if a batch key should be ignored def validateBatchKey(key: String, value: String): Option[(String, String)] = { - if (_batchConfIgnoreMatchList.exists(key.startsWith(_)) || _batchConfIgnoreList.contains(key)) { + if (_batchConfIgnoreMatchList.exists(key.startsWith) || _batchConfIgnoreList.contains(key)) { warn(s"$key is a ignored batch key according to the server-side configuration") None } else { @@ -260,10 +266,10 @@ abstract class SessionManager(name: String) extends CompositeService(name) { conf.get(ENGINE_EXEC_KEEPALIVE_TIME) } - _confRestrictList = conf.get(SESSION_CONF_RESTRICT_LIST).toSet - _confIgnoreList = conf.get(SESSION_CONF_IGNORE_LIST).toSet + + _confRestrictList = conf.get(SESSION_CONF_RESTRICT_LIST) + _confIgnoreList = conf.get(SESSION_CONF_IGNORE_LIST) + s"${SESSION_USER_SIGN_ENABLED.key}" - _batchConfIgnoreList = conf.get(BATCH_CONF_IGNORE_LIST).toSet + _batchConfIgnoreList = conf.get(BATCH_CONF_IGNORE_LIST) execPool = ThreadUtils.newDaemonQueuedThreadPool( poolSize, @@ -283,9 +289,9 @@ abstract class SessionManager(name: String) extends CompositeService(name) { shutdown = true val shutdownTimeout: Long = if (isServer) { - conf.get(ENGINE_EXEC_POOL_SHUTDOWN_TIMEOUT) - } else { conf.get(SERVER_EXEC_POOL_SHUTDOWN_TIMEOUT) + } else { + conf.get(ENGINE_EXEC_POOL_SHUTDOWN_TIMEOUT) } ThreadUtils.shutdown(timeoutChecker, Duration(shutdownTimeout, TimeUnit.MILLISECONDS)) @@ -302,11 +308,12 @@ abstract class SessionManager(name: String) extends CompositeService(name) { for (session <- handleToSession.values().asScala) { if (session.lastAccessTime + session.sessionIdleTimeoutThreshold <= current && session.getNoOperationTime > session.sessionIdleTimeoutThreshold) { + info(s"Closing session ${session.handle.identifier} that has been idle for more" + + s" than ${session.sessionIdleTimeoutThreshold} ms") try { closeSession(session.handle) } catch { - case e: KyuubiSQLException => - warn(s"Error closing idle session ${session.handle}", e) + case NonFatal(e) => warn(s"Error closing idle session ${session.handle}", e) } } else { session.closeExpiredOperations() diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/session/package.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/session/package.scala index 40abded98..63b17dd4d 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/session/package.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/session/package.scala @@ -25,6 +25,8 @@ package object session { val HIVECONF_PREFIX = "hiveconf:" val HIVEVAR_PREFIX = "hivevar:" val METACONF_PREFIX = "metaconf:" + val USE_CATALOG = "use:catalog" + val USE_DATABASE = "use:database" val SPARK_PREFIX = "spark." } diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/util/ClassUtils.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/util/ClassUtils.scala index bcbfdabfb..d8eda3426 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/util/ClassUtils.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/util/ClassUtils.scala @@ -17,10 +17,9 @@ package org.apache.kyuubi.util -import scala.util.Try - import org.apache.kyuubi.KyuubiException import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.util.reflect._ object ClassUtils { @@ -34,28 +33,16 @@ object ClassUtils { */ def createInstance[T](className: String, expected: Class[T], conf: KyuubiConf): T = { val classLoader = Thread.currentThread.getContextClassLoader - val cls = Class.forName(className, true, classLoader) - cls match { - case clazz if expected.isAssignableFrom(cls) => - val confConstructor = clazz.getConstructors.exists(p => { - val params = p.getParameterTypes - params.length == 1 && classOf[KyuubiConf].isAssignableFrom(params(0)) - }) - if (confConstructor) { - clazz.getConstructor(classOf[KyuubiConf]).newInstance(conf) - .asInstanceOf[T] - } else { - clazz.newInstance().asInstanceOf[T] - } - case _ => throw new KyuubiException( - s"$className must extend of ${expected.getName}") + try { + DynConstructors.builder(expected).loader(classLoader) + .impl(className, classOf[KyuubiConf]) + .impl(className) + .buildChecked[T]() + .newInstance(conf) + } catch { + case e: Exception => + throw new KyuubiException(s"$className must extend of ${expected.getName}", e) } } - /** Determines whether the provided class is loadable. */ - def classIsLoadable( - clazz: String, - cl: ClassLoader = Thread.currentThread().getContextClassLoader): Boolean = { - Try { Class.forName(clazz, false, cl) }.isSuccess - } } diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/util/JdbcUtils.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/util/JdbcUtils.scala index df72ee339..996589cb7 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/util/JdbcUtils.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/util/JdbcUtils.scala @@ -104,4 +104,13 @@ object JdbcUtils extends Logging { case _ => "(empty)" } } + + def isDuplicatedKeyDBErr(cause: Throwable): Boolean = { + val duplicatedKeyKeywords = Seq( + "duplicate key value in a unique or primary key constraint or unique index", // Derby + "Duplicate entry", // MySQL + "A UNIQUE constraint failed" // SQLite + ) + duplicatedKeyKeywords.exists(cause.getMessage.contains) + } } diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/util/KyuubiHadoopUtils.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/util/KyuubiHadoopUtils.scala index a63646d9b..4959c845d 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/util/KyuubiHadoopUtils.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/util/KyuubiHadoopUtils.scala @@ -26,24 +26,17 @@ import scala.util.{Failure, Success, Try} import org.apache.hadoop.conf.Configuration import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier import org.apache.hadoop.io.Text -import org.apache.hadoop.security.{Credentials, SecurityUtil, UserGroupInformation} +import org.apache.hadoop.security.{Credentials, SecurityUtil} import org.apache.hadoop.security.token.{Token, TokenIdentifier} import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier import org.apache.hadoop.yarn.conf.YarnConfiguration import org.apache.kyuubi.Logging import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.util.reflect.ReflectUtils._ object KyuubiHadoopUtils extends Logging { - private val subjectField = - classOf[UserGroupInformation].getDeclaredField("subject") - subjectField.setAccessible(true) - - private val tokenMapField = - classOf[Credentials].getDeclaredField("tokenMap") - tokenMapField.setAccessible(true) - def newHadoopConf( conf: KyuubiConf, loadDefaults: Boolean = true): Configuration = { @@ -81,12 +74,8 @@ object KyuubiHadoopUtils extends Logging { * Get [[Credentials#tokenMap]] by reflection as [[Credentials#getTokenMap]] is not present before * Hadoop 3.2.1. */ - def getTokenMap(credentials: Credentials): Map[Text, Token[_ <: TokenIdentifier]] = { - tokenMapField.get(credentials) - .asInstanceOf[JMap[Text, Token[_ <: TokenIdentifier]]] - .asScala - .toMap - } + def getTokenMap(credentials: Credentials): Map[Text, Token[_ <: TokenIdentifier]] = + getField[JMap[Text, Token[_ <: TokenIdentifier]]](credentials, "tokenMap").asScala.toMap def getTokenIssueDate(token: Token[_ <: TokenIdentifier]): Option[Long] = { token.decodeIdentifier() match { diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/util/KyuubiUncaughtExceptionHandler.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/util/KyuubiUncaughtExceptionHandler.scala new file mode 100644 index 000000000..69cfe207f --- /dev/null +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/util/KyuubiUncaughtExceptionHandler.scala @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.util + +import java.lang.Thread.UncaughtExceptionHandler + +import org.apache.kyuubi.Logging + +class KyuubiUncaughtExceptionHandler extends UncaughtExceptionHandler with Logging { + override def uncaughtException(t: Thread, e: Throwable): Unit = { + error(s"Uncaught exception in thread ${t.getName}", e) + } +} diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/util/NamedThreadFactory.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/util/NamedThreadFactory.scala index 89c3c96ea..3ce421e23 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/util/NamedThreadFactory.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/util/NamedThreadFactory.scala @@ -20,10 +20,17 @@ package org.apache.kyuubi.util import java.util.concurrent.ThreadFactory class NamedThreadFactory(name: String, daemon: Boolean) extends ThreadFactory { + import NamedThreadFactory._ + override def newThread(r: Runnable): Thread = { val t = new Thread(r) t.setName(name + ": Thread-" + t.getId) t.setDaemon(daemon) + t.setUncaughtExceptionHandler(kyuubiUncaughtExceptionHandler) t } } + +object NamedThreadFactory { + private[util] val kyuubiUncaughtExceptionHandler = new KyuubiUncaughtExceptionHandler +} diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/util/RowSetUtils.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/util/RowSetUtils.scala index 82417a730..f320fd902 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/util/RowSetUtils.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/util/RowSetUtils.scala @@ -18,14 +18,11 @@ package org.apache.kyuubi.util import java.nio.ByteBuffer -import java.sql.Timestamp -import java.time.{Duration, Instant, LocalDate, LocalDateTime, Period, ZoneId} +import java.time.{Instant, LocalDate, LocalDateTime, LocalTime, ZoneId} import java.time.chrono.IsoChronology -import java.time.format.DateTimeFormatter import java.time.format.DateTimeFormatterBuilder import java.time.temporal.ChronoField -import java.util.{Date, Locale, TimeZone} -import java.util.concurrent.TimeUnit +import java.util.{Date, Locale} import scala.language.implicitConversions @@ -37,24 +34,24 @@ private[kyuubi] object RowSetUtils { final private val SECOND_PER_HOUR: Long = SECOND_PER_MINUTE * 60L final private val SECOND_PER_DAY: Long = SECOND_PER_HOUR * 24L - private lazy val dateFormatter = { - createDateTimeFormatterBuilder().appendPattern("yyyy-MM-dd") - .toFormatter(Locale.US) - .withChronology(IsoChronology.INSTANCE) - } + private lazy val dateFormatter = createDateTimeFormatterBuilder() + .appendPattern("yyyy-MM-dd") + .toFormatter(Locale.US) + .withChronology(IsoChronology.INSTANCE) private lazy val legacyDateFormatter = FastDateFormat.getInstance("yyyy-MM-dd", Locale.US) - private lazy val timestampFormatter: DateTimeFormatter = { - createDateTimeFormatterBuilder().appendPattern("yyyy-MM-dd HH:mm:ss") - .appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true) - .toFormatter(Locale.US) - .withChronology(IsoChronology.INSTANCE) - } + private lazy val timeFormatter = createDateTimeFormatterBuilder() + .appendPattern("HH:mm:ss") + .appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true) + .toFormatter(Locale.US) + .withChronology(IsoChronology.INSTANCE) - private lazy val legacyTimestampFormatter = { - FastDateFormat.getInstance("yyyy-MM-dd HH:mm:ss.SSS", Locale.US) - } + private lazy val timestampFormatter = createDateTimeFormatterBuilder() + .appendPattern("yyyy-MM-dd HH:mm:ss") + .appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true) + .toFormatter(Locale.US) + .withChronology(IsoChronology.INSTANCE) private def createDateTimeFormatterBuilder(): DateTimeFormatterBuilder = { new DateTimeFormatterBuilder().parseCaseInsensitive() @@ -68,6 +65,10 @@ private[kyuubi] object RowSetUtils { dateFormatter.format(ld) } + def formatLocalTime(lt: LocalTime): String = { + timeFormatter.format(lt) + } + def formatLocalDateTime(ldt: LocalDateTime): String = { timestampFormatter.format(ldt) } @@ -77,40 +78,7 @@ private[kyuubi] object RowSetUtils { .getOrElse(timestampFormatter.format(i)) } - def formatTimestamp(t: Timestamp, timeZone: Option[ZoneId] = None): String = { - timeZone.map(zoneId => { - FastDateFormat.getInstance( - legacyTimestampFormatter.getPattern, - TimeZone.getTimeZone(zoneId), - legacyTimestampFormatter.getLocale) - .format(t) - }).getOrElse(legacyTimestampFormatter.format(t)) - } - implicit def bitSetToBuffer(bitSet: java.util.BitSet): ByteBuffer = { ByteBuffer.wrap(bitSet.toByteArray) } - - def toDayTimeIntervalString(d: Duration): String = { - var rest = d.getSeconds - var sign = "" - if (d.getSeconds < 0) { - sign = "-" - rest = -rest - } - val days = TimeUnit.SECONDS.toDays(rest) - rest %= SECOND_PER_DAY - val hours = TimeUnit.SECONDS.toHours(rest) - rest %= SECOND_PER_HOUR - val minutes = TimeUnit.SECONDS.toMinutes(rest) - val seconds = rest % SECOND_PER_MINUTE - f"$sign$days $hours%02d:$minutes%02d:$seconds%02d.${d.getNano}%09d" - } - - def toYearMonthIntervalString(d: Period): String = { - val years = d.getYears - val months = d.getMonths - val sign = if (years < 0 || months < 0) "-" else "" - s"$sign${Math.abs(years)}-${Math.abs(months)}" - } } diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/util/SignUtils.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/util/SignUtils.scala index 6f7ff18df..7fb4fde2e 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/util/SignUtils.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/util/SignUtils.scala @@ -27,7 +27,7 @@ object SignUtils { private lazy val ecKeyPairGenerator = { val g = KeyPairGenerator.getInstance(KEYPAIR_ALGORITHM_EC) - g.initialize(new ECGenParameterSpec("secp256k1"), new SecureRandom()) + g.initialize(new ECGenParameterSpec("secp521r1"), new SecureRandom()) g } diff --git a/kyuubi-common/src/main/scala/org/apache/kyuubi/util/ThreadUtils.scala b/kyuubi-common/src/main/scala/org/apache/kyuubi/util/ThreadUtils.scala index 8ce4bb2e5..76d3f416f 100644 --- a/kyuubi-common/src/main/scala/org/apache/kyuubi/util/ThreadUtils.scala +++ b/kyuubi-common/src/main/scala/org/apache/kyuubi/util/ThreadUtils.scala @@ -95,4 +95,18 @@ object ThreadUtils extends Logging { } } } + + def runInNewThread( + threadName: String, + isDaemon: Boolean = true)(body: => Unit): Unit = { + + val thread = new Thread(threadName) { + override def run(): Unit = { + body + } + } + thread.setDaemon(isDaemon) + thread.setUncaughtExceptionHandler(NamedThreadFactory.kyuubiUncaughtExceptionHandler) + thread.start() + } } diff --git a/kyuubi-common/src/test/resources/ldap/ad.example.com.ldif b/kyuubi-common/src/test/resources/ldap/ad.example.com.ldif new file mode 100644 index 000000000..68cd01d0f --- /dev/null +++ b/kyuubi-common/src/test/resources/ldap/ad.example.com.ldif @@ -0,0 +1,150 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +dn: dc=ad,dc=example,dc=com +dc: ad +objectClass: top +objectClass: domain + +dn: ou=Engineering,dc=ad,dc=example,dc=com +objectClass: top +objectClass: organizationalUnit +ou: Engineering + +dn: ou=Management,dc=ad,dc=example,dc=com +objectClass: top +objectClass: organizationalUnit +ou: Management + +dn: ou=Administration,dc=ad,dc=example,dc=com +objectClass: top +objectClass: organizationalUnit +ou: Administration + +dn: ou=Teams,dc=ad,dc=example,dc=com +objectClass: top +objectClass: organizationalUnit +ou: Teams + +dn: ou=Resources,dc=ad,dc=example,dc=com +objectClass: top +objectClass: organizationalUnit +ou: Resources + +dn: cn=Team 1,ou=Teams,dc=ad,dc=example,dc=com +objectClass: top +objectClass: groupOfNames +objectClass: microsoftSecurityPrincipal +sAMAccountName: team1 +cn: Team 1 +member: sAMAccountName=engineer1,ou=Engineering,dc=ad,dc=example,dc=com +member: sAMAccountName=manager1,ou=Management,dc=ad,dc=example,dc=com + +dn: cn=Team 2,ou=Teams,dc=ad,dc=example,dc=com +objectClass: top +objectClass: groupOfNames +objectClass: microsoftSecurityPrincipal +sAMAccountName: team2 +cn: Team 2 +member: sAMAccountName=engineer2,ou=Engineering,dc=ad,dc=example,dc=com +member: sAMAccountName=manager2,ou=Management,dc=ad,dc=example,dc=com + +dn: cn=Resource 1,ou=Resources,dc=ad,dc=example,dc=com +objectClass: top +objectClass: groupOfNames +objectClass: microsoftSecurityPrincipal +sAMAccountName: resource1 +cn: Resource 1 +member: sAMAccountName=engineer1,ou=Engineering,dc=ad,dc=example,dc=com + +dn: cn=Resource 2,ou=Resources,dc=ad,dc=example,dc=com +objectClass: top +objectClass: groupOfNames +objectClass: microsoftSecurityPrincipal +sAMAccountName: resource2 +cn: Resource 2 +member: sAMAccountName=engineer2,ou=Engineering,dc=ad,dc=example,dc=com + +dn: cn=Admins,ou=Administration,dc=ad,dc=example,dc=com +objectClass: top +objectClass: groupOfUniqueNames +objectClass: microsoftSecurityPrincipal +sAMAccountName: admins +cn: Admins +uniqueMember: sAMAccountName=admin1,ou=Administration,dc=ad,dc=example,dc=com + +dn: sAMAccountName=engineer1,ou=Engineering,dc=ad,dc=example,dc=com +objectClass: top +objectClass: person +objectClass: organizationalPerson +objectClass: inetOrgPerson +objectClass: microsoftSecurityPrincipal +sAMAccountName: engineer1 +cn: Engineer 1 +sn: Surname 1 +userPassword: engineer1-password +memberOf: cn=Team 1,ou=Teams,dc=ad,dc=example,dc=com +memberOf: cn=Resource 1,ou=Resources,dc=ad,dc=example,dc=com + +dn: sAMAccountName=engineer2,ou=Engineering,dc=ad,dc=example,dc=com +objectClass: top +objectClass: person +objectClass: organizationalPerson +objectClass: inetOrgPerson +objectClass: microsoftSecurityPrincipal +sAMAccountName: engineer2 +cn: Engineer 2 +sn: Surname 2 +userPassword: engineer2-password +memberOf: cn=Team 2,ou=Teams,dc=ad,dc=example,dc=com +memberOf: cn=Resource 2,ou=Resources,dc=ad,dc=example,dc=com + +dn: sAMAccountName=manager1,ou=Management,dc=ad,dc=example,dc=com +objectClass: top +objectClass: person +objectClass: organizationalPerson +objectClass: inetOrgPerson +objectClass: microsoftSecurityPrincipal +sAMAccountName: manager1 +cn: Manager 1 +sn: Surname 1 +userPassword: manager1-password +memberOf: cn=Team 1,ou=Teams,dc=ad,dc=example,dc=com + +dn: sAMAccountName=manager2,ou=Management,dc=ad,dc=example,dc=com +objectClass: top +objectClass: person +objectClass: organizationalPerson +objectClass: inetOrgPerson +objectClass: microsoftSecurityPrincipal +sAMAccountName: manager2 +cn: Manager 2 +sn: Surname 2 +userPassword: manager2-password +memberOf: cn=Team 2,ou=Teams,dc=ad,dc=example,dc=com + +dn: sAMAccountName=admin1,ou=Administration,dc=ad,dc=example,dc=com +objectClass: top +objectClass: person +objectClass: organizationalPerson +objectClass: inetOrgPerson +objectClass: microsoftSecurityPrincipal +sAMAccountName: admin1 +cn: Admin 1 +sn: Surname 1 +userPassword: admin1-password +memberOf: cn=Admins,ou=Administration,dc=ad,dc=example,dc=com diff --git a/kyuubi-common/src/test/resources/ldap/example.com.ldif b/kyuubi-common/src/test/resources/ldap/example.com.ldif new file mode 100644 index 000000000..f19eb2f93 --- /dev/null +++ b/kyuubi-common/src/test/resources/ldap/example.com.ldif @@ -0,0 +1,113 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +dn: ou=People,dc=example,dc=com +objectClass: top +objectClass: organizationalUnit +ou: People +description: Contains entries which describe persons (seamen) + +dn: ou=Groups,dc=example,dc=com +objectClass: top +objectClass: organizationalUnit +ou: Groups +description: Contains entries which describe groups (crews, for instance) + +dn: uid=group1,ou=Groups,dc=example,dc=com +objectClass: top +objectClass: groupOfNames +objectClass: uidObject +uid: group1 +cn: group1 +ou: Groups +member: uid=user1,ou=People,dc=example,dc=com + +dn: uid=group2,ou=Groups,dc=example,dc=com +objectClass: top +objectClass: groupOfNames +objectClass: uidObject +uid: group2 +cn: group2 +ou: Groups +member: uid=user2,ou=People,dc=example,dc=com + +dn: cn=group3,ou=Groups,dc=example,dc=com +objectClass: top +objectClass: groupOfNames +objectClass: uidObject +uid: group3 +cn: group3 +ou: Groups +member: cn=user3,ou=People,dc=example,dc=com + +dn: cn=group4,ou=Groups,dc=example,dc=com +objectClass: top +objectClass: groupOfUniqueNames +objectClass: uidObject +uid: group4 +ou: Groups +cn: group4 +uniqueMember: cn=user4,ou=People,dc=example,dc=com + +dn: uid=user1,ou=People,dc=example,dc=com +objectClass: top +objectClass: person +objectClass: organizationalPerson +objectClass: inetOrgPerson +objectClass: uidObject +givenName: Test1 +cn: Test User1 +sn: user1 +uid: user1 +userPassword: user1 + +dn: uid=user2,ou=People,dc=example,dc=com +objectClass: top +objectClass: person +objectClass: organizationalPerson +objectClass: inetOrgPerson +objectClass: uidObject +givenName: Test2 +cn: Test User2 +sn: user2 +uid: user2 +userPassword: user2 + +dn: cn=user3,ou=People,dc=example,dc=com +objectClass: top +objectClass: person +objectClass: organizationalPerson +objectClass: inetOrgPerson +objectClass: uidObject +givenName: Test3 +cn: Test User3 +sn: user3 +uid: user3 +userPassword: user3 + +dn: cn=user4,ou=People,dc=example,dc=com +objectClass: top +objectClass: person +objectClass: organizationalPerson +objectClass: inetOrgPerson +objectClass: uidObject +givenName: Test4 +cn: Test User4 +sn: user4 +uid: user4 +userPassword: user4 + diff --git a/kyuubi-common/src/test/resources/ldap/microsoft.schema.ldif b/kyuubi-common/src/test/resources/ldap/microsoft.schema.ldif new file mode 100644 index 000000000..3e3a9a5c1 --- /dev/null +++ b/kyuubi-common/src/test/resources/ldap/microsoft.schema.ldif @@ -0,0 +1,62 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +dn: cn=microsoft, ou=schema +objectclass: metaSchema +objectclass: top +cn: microsoft + +dn: ou=attributetypes, cn=microsoft, ou=schema +objectclass: organizationalUnit +objectclass: top +ou: attributetypes + +dn: m-oid=1.2.840.113556.1.4.221, ou=attributetypes, cn=microsoft, ou=schema +objectclass: metaAttributeType +objectclass: metaTop +objectclass: top +m-oid: 1.2.840.113556.1.4.221 +m-name: sAMAccountName +m-equality: caseIgnoreMatch +m-syntax: 1.3.6.1.4.1.1466.115.121.1.15 +m-singleValue: TRUE + +dn: m-oid=1.2.840.113556.1.4.222, ou=attributetypes, cn=microsoft, ou=schema +objectclass: metaAttributeType +objectclass: metaTop +objectclass: top +m-oid: 1.2.840.113556.1.4.222 +m-name: memberOf +m-equality: caseIgnoreMatch +m-syntax: 1.3.6.1.4.1.1466.115.121.1.15 +m-singleValue: FALSE + +dn: ou=objectClasses, cn=microsoft, ou=schema +objectclass: organizationalUnit +objectclass: top +ou: objectClasses + +dn: m-oid=1.2.840.113556.1.5.6, ou=objectClasses, cn=microsoft, ou=schema +objectclass: metaObjectClass +objectclass: metaTop +objectclass: top +m-oid: 1.2.840.113556.1.5.6 +m-name: microsoftSecurityPrincipal +m-supObjectClass: top +m-typeObjectClass: AUXILIARY +m-must: sAMAccountName +m-may: memberOf diff --git a/kyuubi-common/src/test/resources/log4j2-test.xml b/kyuubi-common/src/test/resources/log4j2-test.xml index bfc40dd6d..3110216c1 100644 --- a/kyuubi-common/src/test/resources/log4j2-test.xml +++ b/kyuubi-common/src/test/resources/log4j2-test.xml @@ -21,14 +21,14 @@ - + - + diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/HiveEngineTests.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/HiveEngineTests.scala index 9eb4a2440..028f755f6 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/HiveEngineTests.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/HiveEngineTests.scala @@ -140,7 +140,7 @@ trait HiveEngineTests extends HiveJDBCTestHelper { try { val meta = statement.getConnection.getMetaData var resultSet = meta.getColumns(null, null, null, null) - var resultSetBuffer = ArrayBuffer[(String, String, String, String, String)]() + val resultSetBuffer = ArrayBuffer[(String, String, String, String, String)]() while (resultSet.next()) { resultSetBuffer += Tuple5( resultSet.getString(TABLE_CAT), @@ -434,8 +434,8 @@ trait HiveEngineTests extends HiveJDBCTestHelper { val res = statement.getConnection.getMetaData.getClientInfoProperties assert(res.next()) assert(res.getString(1) === "ApplicationName") - assert(res.getInt("MAX_LEN") === 1000); - assert(!res.next()); + assert(res.getInt("MAX_LEN") === 1000) + assert(!res.next()) val connection = statement.getConnection connection.setClientInfo("ApplicationName", "test kyuubi hive jdbc") diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/KyuubiFunSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/KyuubiFunSuite.scala index 96a612aab..8d0a14c16 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/KyuubiFunSuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/KyuubiFunSuite.scala @@ -30,6 +30,7 @@ import org.scalatest.funsuite.AnyFunSuite import org.slf4j.bridge.SLF4JBridgeHandler import org.apache.kyuubi.config.internal.Tests.IS_TESTING +import org.apache.kyuubi.service.authentication.InternalSecurityAccessor trait KyuubiFunSuite extends AnyFunSuite with BeforeAndAfterAll @@ -46,6 +47,7 @@ trait KyuubiFunSuite extends AnyFunSuite override def beforeAll(): Unit = { System.setProperty(IS_TESTING.key, "true") doThreadPreAudit() + InternalSecurityAccessor.reset() super.beforeAll() } @@ -102,6 +104,7 @@ trait KyuubiFunSuite extends AnyFunSuite logger.asInstanceOf[Logger].setLevel(restoreLevels(i)) logger.asInstanceOf[Logger].get().setLevel(restoreLevels(i)) } + LogManager.getContext(false).asInstanceOf[LoggerContext].updateLoggers() } } } diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/MarkdownUtils.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/MarkdownUtils.scala new file mode 100644 index 000000000..4dbe6ea67 --- /dev/null +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/MarkdownUtils.scala @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi + +import scala.collection.mutable.ListBuffer + +import com.vladsch.flexmark.formatter.Formatter +import com.vladsch.flexmark.parser.{Parser, ParserEmulationProfile, PegdownExtensions} +import com.vladsch.flexmark.profile.pegdown.PegdownOptionsAdapter +import com.vladsch.flexmark.util.data.{MutableDataHolder, MutableDataSet} +import com.vladsch.flexmark.util.sequence.SequenceUtils.EOL + +class MarkdownBuilder { + private val buffer = new ListBuffer[String] + + /** + * append a single line + * with replacing EOL to empty string + * + * @param str single line + * @return + */ + def +=(str: String): MarkdownBuilder = { + buffer += str.stripMargin.linesIterator.mkString + this + } + + /** + * append the multiline + * with splitting EOL into single lines + * + * @param multiline multiline with line margin char + * @param marginChar margin char, default to "|" + * @return + */ + def ++=(multiline: String, marginChar: Char = '|'): MarkdownBuilder = { + buffer ++= multiline.stripMargin(marginChar).linesIterator + this + } + + /** + * append the licence + * @return + */ + def licence(): MarkdownBuilder = { + this ++= """ + | + |""" + } + + /** + * append the auto-generation hint + * @param className the full class name of agent suite + * @return + */ + def generationHint(className: String): MarkdownBuilder = { + this ++= + s""" + | + | + |""" + } + + def toMarkdown: Stream[String] = { + def createParserOptions(emulationProfile: ParserEmulationProfile): MutableDataHolder = { + PegdownOptionsAdapter.flexmarkOptions(PegdownExtensions.ALL).toMutable + .set(Parser.PARSER_EMULATION_PROFILE, emulationProfile) + } + + def createFormatterOptions( + parserOptions: MutableDataHolder, + emulationProfile: ParserEmulationProfile): MutableDataSet = { + new MutableDataSet() + .set(Parser.EXTENSIONS, Parser.EXTENSIONS.get(parserOptions)) + .set(Formatter.FORMATTER_EMULATION_PROFILE, emulationProfile) + } + + val emulationProfile = ParserEmulationProfile.COMMONMARK + val parserOptions = createParserOptions(emulationProfile) + val formatterOptions = createFormatterOptions(parserOptions, emulationProfile) + val parser = Parser.builder(parserOptions).build + val renderer = Formatter.builder(formatterOptions).build + val document = parser.parse(buffer.mkString(EOL)) + val formattedLines = new ListBuffer[String] + val formattedLinesAppendable = new Appendable { + override def append(csq: CharSequence): Appendable = { + if (csq.length() > 0) { + formattedLines.append(csq.toString) + } + this + } + + override def append(csq: CharSequence, start: Int, end: Int): Appendable = { + append(csq.toString.substring(start, end)) + } + + override def append(c: Char): Appendable = { + append(c.toString) + } + } + renderer.render(document, formattedLinesAppendable) + // trim the ending EOL appended by renderer for each line + formattedLines.toStream.map(str => + if (str.endsWith(EOL)) { + str.substring(0, str.length - 1) + } else { + str + }) + } +} + +object MarkdownBuilder { + def apply(licenced: Boolean = true, className: String = null): MarkdownBuilder = { + val builder = new MarkdownBuilder + if (licenced) { builder.licence() } + if (className != null) { builder.generationHint(className) } + builder + } +} diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/TestUtils.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/TestUtils.scala index 16a49388f..97675768a 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/TestUtils.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/TestUtils.scala @@ -14,99 +14,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.kyuubi -import java.nio.charset.StandardCharsets -import java.nio.file.{Files, Path, StandardOpenOption} import java.sql.ResultSet -import scala.collection.JavaConverters._ import scala.collection.mutable.ArrayBuffer import com.jakewharton.fliptables.FlipTable -import com.vladsch.flexmark.formatter.Formatter -import com.vladsch.flexmark.parser.{Parser, ParserEmulationProfile, PegdownExtensions} -import com.vladsch.flexmark.profile.pegdown.PegdownOptionsAdapter -import com.vladsch.flexmark.util.data.{MutableDataHolder, MutableDataSet} -import com.vladsch.flexmark.util.sequence.SequenceUtils -import org.scalatest.Assertions.{assertResult, withClue} object TestUtils { - - private def formatMarkdown(lines: ArrayBuffer[String]): ArrayBuffer[String] = { - def createParserOptions(emulationProfile: ParserEmulationProfile): MutableDataHolder = { - PegdownOptionsAdapter.flexmarkOptions(PegdownExtensions.ALL).toMutable - .set(Parser.PARSER_EMULATION_PROFILE, emulationProfile) - } - - def createFormatterOptions( - parserOptions: MutableDataHolder, - emulationProfile: ParserEmulationProfile): MutableDataSet = { - new MutableDataSet() - .set(Parser.EXTENSIONS, Parser.EXTENSIONS.get(parserOptions)) - .set(Formatter.FORMATTER_EMULATION_PROFILE, emulationProfile) - } - - val emulationProfile = ParserEmulationProfile.valueOf("COMMONMARK") - val parserOptions = createParserOptions(emulationProfile) - val formatterOptions = createFormatterOptions(parserOptions, emulationProfile) - val parser = Parser.builder(parserOptions).build - val renderer = Formatter.builder(formatterOptions).build - val document = parser.parse(lines.mkString(SequenceUtils.EOL)) - val formattedLines = new ArrayBuffer[String] - val formattedLinesAppendable = new Appendable { - override def append(csq: CharSequence): Appendable = { - if (csq.length() > 0) { - formattedLines.append(csq.toString) - } - this - } - - override def append(csq: CharSequence, start: Int, end: Int): Appendable = { - append(csq.toString.substring(start, end)) - } - - override def append(c: Char): Appendable = { - append(c.toString) - } - } - renderer.render(document, formattedLinesAppendable) - // trim the ending EOL appended by renderer for each line - formattedLines.map(str => - if (str.nonEmpty && str.endsWith(SequenceUtils.EOL)) { - str.substring(0, str.length - 1) - } else { - str - }) - } - - def verifyOutput( - markdown: Path, - newOutput: ArrayBuffer[String], - agent: String, - module: String): Unit = { - if (System.getenv("KYUUBI_UPDATE") == "1") { - val formatted = formatMarkdown(newOutput) - Files.write( - markdown, - formatted.asJava, - StandardOpenOption.CREATE, - StandardOpenOption.TRUNCATE_EXISTING) - } else { - val linesInFile = Files.readAllLines(markdown, StandardCharsets.UTF_8) - val formatted = formatMarkdown(newOutput) - linesInFile.asScala.zipWithIndex.zip(formatted).foreach { case ((str1, index), str2) => - withClue(s"$markdown out of date, as line ${index + 1} is not expected." + - " Please update doc with KYUUBI_UPDATE=1 build/mvn clean test" + - s" -pl $module -am -Pflink-provided,spark-provided,hive-provided" + - s" -DwildcardSuites=$agent") { - assertResult(str2)(str1) - } - } - } - } - def displayResultSet(resultSet: ResultSet): Unit = { if (resultSet == null) throw new NullPointerException("resultSet == null") val resultSetMetaData = resultSet.getMetaData diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/config/ConfigBuilderSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/config/ConfigBuilderSuite.scala index 4a9ade551..78429d27c 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/config/ConfigBuilderSuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/config/ConfigBuilderSuite.scala @@ -18,6 +18,7 @@ package org.apache.kyuubi.config import org.apache.kyuubi.KyuubiFunSuite +import org.apache.kyuubi.util.AssertionUtils._ class ConfigBuilderSuite extends KyuubiFunSuite { @@ -72,6 +73,33 @@ class ConfigBuilderSuite extends KyuubiFunSuite { KyuubiConf.register(sequenceConf) val kyuubiConf = KyuubiConf().set(sequenceConf.key, "kyuubi,kent") assert(kyuubiConf.get(sequenceConf) === Seq("kyuubi", "kent")) + + val stringConfUpper = ConfigBuilder("kyuubi.string.conf.upper") + .stringConf + .transformToUpperCase + .createWithDefault("Kent, Yao") + assert(stringConfUpper.key === "kyuubi.string.conf.upper") + assert(stringConfUpper.defaultVal.get === "KENT, YAO") + + val stringConfUpperSeq = ConfigBuilder("kyuubi.string.conf.upper.seq") + .stringConf + .transformToUpperCase + .toSequence() + .createWithDefault(Seq("hehe")) + assert(stringConfUpperSeq.defaultVal.get === Seq("HEHE")) + + val stringConfSet = ConfigBuilder("kyuubi.string.conf.set") + .stringConf + .toSet() + .createWithDefault(Set("hehe", "haha")) + assert(stringConfSet.defaultVal.get === Set("hehe", "haha")) + + val stringConfLower = ConfigBuilder("kyuubi.string.conf.lower") + .stringConf + .transformToLowerCase + .createWithDefault("Kent, Yao") + assert(stringConfLower.key === "kyuubi.string.conf.lower") + assert(stringConfLower.defaultVal.get === "kent, yao") } test("time config") { @@ -98,4 +126,21 @@ class ConfigBuilderSuite extends KyuubiFunSuite { val e = intercept[IllegalArgumentException](kyuubiConf.get(intConf)) assert(e.getMessage equals "'-1' in kyuubi.invalid.config is invalid. must be positive integer") } + + test("invalid config for enum") { + object TempEnum extends Enumeration { + type TempEnum = Value + val ValA, ValB = Value + } + val stringConf = ConfigBuilder("kyuubi.invalid.config.enum") + .stringConf + .checkValues(TempEnum) + .createWithDefault("ValA") + assert(stringConf.key === "kyuubi.invalid.config.enum") + assert(stringConf.defaultVal.get === "ValA") + val kyuubiConf = KyuubiConf().set(stringConf.key, "ValC") + KyuubiConf.register(stringConf) + interceptEquals[IllegalArgumentException] { kyuubiConf.get(stringConf) }( + "The value of kyuubi.invalid.config.enum should be one of ValA, ValB, but was ValC") + } } diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/config/KyuubiConfSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/config/KyuubiConfSuite.scala index f05e15d8a..39e68f0ec 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/config/KyuubiConfSuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/config/KyuubiConfSuite.scala @@ -200,4 +200,25 @@ class KyuubiConfSuite extends KyuubiFunSuite { assertResult(kSeq(1))("kyuubi.efg") assertResult(kSeq(2))("kyuubi.xyz") } + + test("KYUUBI #4843 - Support multiple kubernetes contexts and namespaces") { + val kyuubiConf = KyuubiConf(false) + kyuubiConf.set("kyuubi.kubernetes.28.master.address", "k8s://master") + kyuubiConf.set( + "kyuubi.kubernetes.28.ns1.authenticate.oauthTokenFile", + "/var/run/secrets/kubernetes.io/token.ns1") + kyuubiConf.set( + "kyuubi.kubernetes.28.ns2.authenticate.oauthTokenFile", + "/var/run/secrets/kubernetes.io/token.ns2") + + val kubernetesConf1 = kyuubiConf.getKubernetesConf(Some("28"), Some("ns1")) + assert(kubernetesConf1.get(KyuubiConf.KUBERNETES_MASTER) == Some("k8s://master")) + assert(kubernetesConf1.get(KyuubiConf.KUBERNETES_AUTHENTICATE_OAUTH_TOKEN_FILE) == + Some("/var/run/secrets/kubernetes.io/token.ns1")) + + val kubernetesConf2 = kyuubiConf.getKubernetesConf(Some("28"), Some("ns2")) + assert(kubernetesConf2.get(KyuubiConf.KUBERNETES_MASTER) == Some("k8s://master")) + assert(kubernetesConf2.get(KyuubiConf.KUBERNETES_AUTHENTICATE_OAUTH_TOKEN_FILE) == + Some("/var/run/secrets/kubernetes.io/token.ns2")) + } } diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/HiveMetadataTests.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/HiveMetadataTests.scala index fe1f5f47b..aad31d5b8 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/HiveMetadataTests.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/HiveMetadataTests.scala @@ -17,7 +17,11 @@ package org.apache.kyuubi.operation -import org.apache.kyuubi.Utils +import java.sql.{DatabaseMetaData, ResultSet, SQLException, SQLFeatureNotSupportedException} + +import scala.util.Random + +import org.apache.kyuubi.{KYUUBI_VERSION, KyuubiSQLException, Utils} import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant._ // For `hive` external catalog only @@ -98,4 +102,186 @@ trait HiveMetadataTests extends SparkMetadataTests { statement.execute(s"DROP VIEW IF EXISTS ${schemas(3)}.$view_global_test") } } + + test("audit Kyuubi Hive JDBC connection common MetaData") { + withJdbcStatement() { statement => + val metaData = statement.getConnection.getMetaData + Seq( + () => metaData.allProceduresAreCallable(), + () => metaData.getURL, + () => metaData.getUserName, + () => metaData.isReadOnly, + () => metaData.nullsAreSortedHigh, + () => metaData.nullsAreSortedLow, + () => metaData.nullsAreSortedAtStart(), + () => metaData.nullsAreSortedAtEnd(), + () => metaData.usesLocalFiles(), + () => metaData.usesLocalFilePerTable(), + () => metaData.supportsMixedCaseIdentifiers(), + () => metaData.supportsMixedCaseQuotedIdentifiers(), + () => metaData.storesUpperCaseIdentifiers(), + () => metaData.storesUpperCaseQuotedIdentifiers(), + () => metaData.storesLowerCaseIdentifiers(), + () => metaData.storesLowerCaseQuotedIdentifiers(), + () => metaData.storesMixedCaseIdentifiers(), + () => metaData.storesMixedCaseQuotedIdentifiers(), + () => metaData.nullPlusNonNullIsNull, + () => metaData.supportsConvert, + () => metaData.supportsTableCorrelationNames, + () => metaData.supportsDifferentTableCorrelationNames, + () => metaData.supportsExpressionsInOrderBy(), + () => metaData.supportsOrderByUnrelated, + () => metaData.supportsGroupByUnrelated, + () => metaData.supportsGroupByBeyondSelect, + () => metaData.supportsLikeEscapeClause, + () => metaData.supportsMultipleTransactions, + () => metaData.supportsMinimumSQLGrammar, + () => metaData.supportsCoreSQLGrammar, + () => metaData.supportsExtendedSQLGrammar, + () => metaData.supportsANSI92EntryLevelSQL, + () => metaData.supportsANSI92IntermediateSQL, + () => metaData.supportsANSI92FullSQL, + () => metaData.supportsIntegrityEnhancementFacility, + () => metaData.isCatalogAtStart, + () => metaData.supportsSubqueriesInComparisons, + () => metaData.supportsSubqueriesInExists, + () => metaData.supportsSubqueriesInIns, + () => metaData.supportsSubqueriesInQuantifieds, + // Spark support this, see https://issues.apache.org/jira/browse/SPARK-18455 + () => metaData.supportsCorrelatedSubqueries, + () => metaData.supportsOpenCursorsAcrossCommit, + () => metaData.supportsOpenCursorsAcrossRollback, + () => metaData.supportsOpenStatementsAcrossCommit, + () => metaData.supportsOpenStatementsAcrossRollback, + () => metaData.getMaxBinaryLiteralLength, + () => metaData.getMaxCharLiteralLength, + () => metaData.getMaxColumnsInGroupBy, + () => metaData.getMaxColumnsInIndex, + () => metaData.getMaxColumnsInOrderBy, + () => metaData.getMaxColumnsInSelect, + () => metaData.getMaxColumnsInTable, + () => metaData.getMaxConnections, + () => metaData.getMaxCursorNameLength, + () => metaData.getMaxIndexLength, + () => metaData.getMaxSchemaNameLength, + () => metaData.getMaxProcedureNameLength, + () => metaData.getMaxCatalogNameLength, + () => metaData.getMaxRowSize, + () => metaData.doesMaxRowSizeIncludeBlobs, + () => metaData.getMaxStatementLength, + () => metaData.getMaxStatements, + () => metaData.getMaxTableNameLength, + () => metaData.getMaxTablesInSelect, + () => metaData.getMaxUserNameLength, + () => metaData.supportsTransactionIsolationLevel(1), + () => metaData.supportsDataDefinitionAndDataManipulationTransactions, + () => metaData.supportsDataManipulationTransactionsOnly, + () => metaData.dataDefinitionCausesTransactionCommit, + () => metaData.dataDefinitionIgnoredInTransactions, + () => metaData.getColumnPrivileges("", "%", "%", "%"), + () => metaData.getTablePrivileges("", "%", "%"), + () => metaData.getBestRowIdentifier("", "%", "%", 0, true), + () => metaData.getVersionColumns("", "%", "%"), + () => metaData.getExportedKeys("", "default", ""), + () => metaData.supportsResultSetConcurrency(ResultSet.TYPE_FORWARD_ONLY, 2), + () => metaData.ownUpdatesAreVisible(ResultSet.TYPE_FORWARD_ONLY), + () => metaData.ownDeletesAreVisible(ResultSet.TYPE_FORWARD_ONLY), + () => metaData.ownInsertsAreVisible(ResultSet.TYPE_FORWARD_ONLY), + () => metaData.othersUpdatesAreVisible(ResultSet.TYPE_FORWARD_ONLY), + () => metaData.othersDeletesAreVisible(ResultSet.TYPE_FORWARD_ONLY), + () => metaData.othersInsertsAreVisible(ResultSet.TYPE_FORWARD_ONLY), + () => metaData.updatesAreDetected(ResultSet.TYPE_FORWARD_ONLY), + () => metaData.deletesAreDetected(ResultSet.TYPE_FORWARD_ONLY), + () => metaData.insertsAreDetected(ResultSet.TYPE_FORWARD_ONLY), + () => metaData.supportsNamedParameters(), + () => metaData.supportsMultipleOpenResults, + () => metaData.supportsGetGeneratedKeys, + () => metaData.getSuperTypes("", "%", "%"), + () => metaData.getSuperTables("", "%", "%"), + () => metaData.getAttributes("", "%", "%", "%"), + () => metaData.getResultSetHoldability, + () => metaData.locatorsUpdateCopy, + () => metaData.supportsStatementPooling, + () => metaData.getRowIdLifetime, + () => metaData.supportsStoredFunctionsUsingCallSyntax, + () => metaData.autoCommitFailureClosesAllResultSets, + () => metaData.getFunctionColumns("", "%", "%", "%"), + () => metaData.getPseudoColumns("", "%", "%", "%"), + () => metaData.generatedKeyAlwaysReturned).foreach { func => + val e = intercept[SQLFeatureNotSupportedException](func()) + assert(e.getMessage === "Method not supported") + } + + assert(metaData.allTablesAreSelectable) + assert(metaData.getClientInfoProperties.next) + assert(metaData.getDriverName === "Kyuubi Project Hive JDBC Client" || + metaData.getDriverName === "Kyuubi Project Hive JDBC Shaded Client") + assert(metaData.getDriverVersion === KYUUBI_VERSION) + assert( + metaData.getIdentifierQuoteString === " ", + "This method returns a space \" \" if identifier quoting is not supported") + assert(metaData.getNumericFunctions === "") + assert(metaData.getStringFunctions === "") + assert(metaData.getSystemFunctions === "") + assert(metaData.getTimeDateFunctions === "") + assert(metaData.getSearchStringEscape === "\\") + assert(metaData.getExtraNameCharacters === "") + assert(metaData.supportsAlterTableWithAddColumn()) + assert(!metaData.supportsAlterTableWithDropColumn()) + assert(metaData.supportsColumnAliasing()) + assert(metaData.supportsGroupBy) + assert(!metaData.supportsMultipleResultSets) + assert(!metaData.supportsNonNullableColumns) + assert(metaData.supportsOuterJoins) + assert(metaData.supportsFullOuterJoins) + assert(metaData.supportsLimitedOuterJoins) + assert(metaData.getSchemaTerm === "database") + assert(metaData.getProcedureTerm === "UDF") + assert(metaData.getCatalogTerm === "catalog") + assert(metaData.getCatalogSeparator === ".") + assert(metaData.supportsSchemasInDataManipulation) + assert(!metaData.supportsSchemasInProcedureCalls) + assert(metaData.supportsSchemasInTableDefinitions) + assert(!metaData.supportsSchemasInIndexDefinitions) + assert(!metaData.supportsSchemasInPrivilegeDefinitions) + assert(metaData.supportsCatalogsInDataManipulation) + assert(metaData.supportsCatalogsInProcedureCalls) + assert(metaData.supportsCatalogsInTableDefinitions) + assert(metaData.supportsCatalogsInIndexDefinitions) + assert(metaData.supportsCatalogsInPrivilegeDefinitions) + assert(!metaData.supportsPositionedDelete) + assert(!metaData.supportsPositionedUpdate) + assert(!metaData.supportsSelectForUpdate) + assert(!metaData.supportsStoredProcedures) + // This is actually supported, but hive jdbc package return false + assert(!metaData.supportsUnion) + assert(metaData.supportsUnionAll) + assert(metaData.getMaxColumnNameLength === 128) + assert(metaData.getDefaultTransactionIsolation === java.sql.Connection.TRANSACTION_NONE) + assert(!metaData.supportsTransactions) + assert(!metaData.getProcedureColumns("", "%", "%", "%").next()) + val e1 = intercept[SQLException] { + metaData.getPrimaryKeys("", "default", "src").next() + } + assert(e1.getMessage.contains(KyuubiSQLException.featureNotSupported().getMessage)) + assert(!metaData.getImportedKeys("", "default", "").next()) + + val e2 = intercept[SQLException] { + metaData.getCrossReference("", "default", "src", "", "default", "src2").next() + } + assert(e2.getMessage.contains(KyuubiSQLException.featureNotSupported().getMessage)) + assert(!metaData.getIndexInfo("", "default", "src", true, true).next()) + + assert(metaData.supportsResultSetType(new Random().nextInt())) + assert(!metaData.supportsBatchUpdates) + assert(!metaData.getUDTs(",", "%", "%", null).next()) + assert(!metaData.supportsSavepoints) + assert(!metaData.supportsResultSetHoldability(ResultSet.HOLD_CURSORS_OVER_COMMIT)) + assert(metaData.getJDBCMajorVersion === 3) + assert(metaData.getJDBCMinorVersion === 0) + assert(metaData.getSQLStateType === DatabaseMetaData.sqlStateSQL) + assert(metaData.getMaxLogicalLobSize === 0) + assert(!metaData.supportsRefCursors) + } + } } diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/HudiMetadataTests.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/HudiMetadataTests.scala deleted file mode 100644 index e6870a4e3..000000000 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/HudiMetadataTests.scala +++ /dev/null @@ -1,193 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.kyuubi.operation - -import org.apache.kyuubi.HudiSuiteMixin -import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant._ - -trait HudiMetadataTests extends HiveJDBCTestHelper with HudiSuiteMixin { - - test("get catalogs") { - withJdbcStatement() { statement => - val metaData = statement.getConnection.getMetaData - val catalogs = metaData.getCatalogs - catalogs.next() - assert(catalogs.getString(TABLE_CAT) === "spark_catalog") - assert(!catalogs.next()) - } - } - - test("get schemas") { - val dbs = Seq("db1", "db2", "db33", "db44") - val dbDflts = Seq("default", "global_temp") - - val catalog = "spark_catalog" - withDatabases(dbs: _*) { statement => - dbs.foreach(db => statement.execute(s"CREATE DATABASE IF NOT EXISTS $db")) - val metaData = statement.getConnection.getMetaData - - Seq("", "*", "%", null, ".*", "_*", "_%", ".%") foreach { pattern => - checkGetSchemas(metaData.getSchemas(catalog, pattern), dbs ++ dbDflts, catalog) - } - - Seq("db%", "db.*") foreach { pattern => - checkGetSchemas(metaData.getSchemas(catalog, pattern), dbs, catalog) - } - - Seq("db_", "db.") foreach { pattern => - checkGetSchemas(metaData.getSchemas(catalog, pattern), dbs.take(2), catalog) - } - - checkGetSchemas(metaData.getSchemas(catalog, "db1"), Seq("db1"), catalog) - checkGetSchemas(metaData.getSchemas(catalog, "db_not_exist"), Seq.empty, catalog) - } - } - - test("get tables") { - val table = "table_1_test" - val schema = "default" - val tableType = "TABLE" - - withJdbcStatement(table) { statement => - statement.execute( - s""" - | create table $table ( - | id int, - | name string, - | price double, - | ts long - | ) using $format - | options ( - | primaryKey = 'id', - | preCombineField = 'ts', - | hoodie.bootstrap.index.class = - | 'org.apache.hudi.common.bootstrap.index.NoOpBootstrapIndex' - | ) - """.stripMargin) - - val metaData = statement.getConnection.getMetaData - val rs1 = metaData.getTables(null, null, null, null) - assert(rs1.next()) - val catalogName = rs1.getString(TABLE_CAT) - assert(catalogName === "spark_catalog" || catalogName === null) - assert(rs1.getString(TABLE_SCHEM) === schema) - assert(rs1.getString(TABLE_NAME) == table) - assert(rs1.getString(TABLE_TYPE) == tableType) - assert(!rs1.next()) - - val rs2 = metaData.getTables(null, null, "table%", Array("TABLE")) - assert(rs2.next()) - assert(rs2.getString(TABLE_NAME) == table) - assert(!rs2.next()) - - val rs3 = metaData.getTables(null, "default", "*", Array("VIEW")) - assert(!rs3.next()) - } - } - - test("get columns type") { - val dataTypes = Seq( - "boolean", - "int", - "bigint", - "float", - "double", - "decimal(38,20)", - "decimal(10,2)", - "string", - "array", - "array", - "date", - "timestamp", - "struct<`X`: bigint, `Y`: double>", - "binary", - "struct<`X`: string>") - val cols = dataTypes.zipWithIndex.map { case (dt, idx) => s"c$idx" -> dt } - val (colNames, _) = cols.unzip - - val metadataCols = Seq( - "_hoodie_commit_time", - "_hoodie_commit_seqno", - "_hoodie_record_key", - "_hoodie_partition_path", - "_hoodie_file_name") - - val defaultPkCol = "uuid" - - val reservedCols = metadataCols :+ defaultPkCol - - val tableName = "hudi_get_col_operation" - val ddl = - s""" - |CREATE TABLE IF NOT EXISTS $catalog.$defaultSchema.$tableName ( - | $defaultPkCol string, - | ${cols.map { case (cn, dt) => cn + " " + dt }.mkString(",\n")} - |) - |USING hudi""".stripMargin - - withJdbcStatement(tableName) { statement => - statement.execute(ddl) - - val metaData = statement.getConnection.getMetaData - - Seq("%", null, ".*", "c.*") foreach { columnPattern => - val rowSet = metaData.getColumns(catalog, defaultSchema, tableName, columnPattern) - - import java.sql.Types._ - val expectedJavaTypes = Seq( - BOOLEAN, - INTEGER, - BIGINT, - FLOAT, - DOUBLE, - DECIMAL, - DECIMAL, - VARCHAR, - ARRAY, - ARRAY, - DATE, - TIMESTAMP, - STRUCT, - BINARY, - STRUCT) - - var pos = 0 - while (rowSet.next()) { - assert(rowSet.getString(TABLE_CAT) === catalog) - assert(rowSet.getString(TABLE_SCHEM) === defaultSchema) - assert(rowSet.getString(TABLE_NAME) === tableName) - rowSet.getString(COLUMN_NAME) match { - case name if reservedCols.contains(name) => - assert(rowSet.getInt(DATA_TYPE) === VARCHAR) - assert(rowSet.getString(TYPE_NAME) equalsIgnoreCase "STRING") - case _ => - assert(rowSet.getString(COLUMN_NAME) === colNames(pos)) - assert(rowSet.getInt(DATA_TYPE) === expectedJavaTypes(pos)) - assert(rowSet.getString(TYPE_NAME) equalsIgnoreCase dataTypes(pos)) - pos += 1 - } - } - - assert(pos === dataTypes.size, "all columns should have been verified") - } - - val rowSet = metaData.getColumns(catalog, "*", "not_exist", "not_exist") - assert(!rowSet.next()) - } - } -} diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/IcebergMetadataTests.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/IcebergMetadataTests.scala index d14224a84..814c08343 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/IcebergMetadataTests.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/IcebergMetadataTests.scala @@ -17,20 +17,24 @@ package org.apache.kyuubi.operation -import org.apache.kyuubi.IcebergSuiteMixin +import scala.collection.mutable.ListBuffer + +import org.apache.kyuubi.{IcebergSuiteMixin, SPARK_COMPILE_VERSION} import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant._ -import org.apache.kyuubi.util.SparkVersionUtil.isSparkVersionAtLeast +import org.apache.kyuubi.util.AssertionUtils._ +import org.apache.kyuubi.util.SparkVersionUtil -trait IcebergMetadataTests extends HiveJDBCTestHelper with IcebergSuiteMixin { +trait IcebergMetadataTests extends HiveJDBCTestHelper with IcebergSuiteMixin with SparkVersionUtil { test("get catalogs") { withJdbcStatement() { statement => val metaData = statement.getConnection.getMetaData val catalogs = metaData.getCatalogs - catalogs.next() - assert(catalogs.getString(TABLE_CAT) === "spark_catalog") - catalogs.next() - assert(catalogs.getString(TABLE_CAT) === catalog) + val results = ListBuffer[String]() + while (catalogs.next()) { + results += catalogs.getString(TABLE_CAT) + } + assertContains(results, "spark_catalog", catalog) } } @@ -129,7 +133,7 @@ trait IcebergMetadataTests extends HiveJDBCTestHelper with IcebergSuiteMixin { } assert(!rs1.next()) } finally { - statement.execute(s"DROP TABLE IF EXISTS $cg.$db.tbl") + statement.execute(s"DROP TABLE IF EXISTS $cg.$db.tbl PURGE") } } } @@ -153,11 +157,11 @@ trait IcebergMetadataTests extends HiveJDBCTestHelper with IcebergSuiteMixin { "date", "timestamp", // SPARK-37931 - if (isSparkVersionAtLeast("3.3")) "struct" + if (SPARK_ENGINE_RUNTIME_VERSION >= "3.3") "struct" else "struct<`X`: bigint, `Y`: double>", "binary", // SPARK-37931 - if (isSparkVersionAtLeast("3.3")) "struct" else "struct<`X`: string>") + if (SPARK_COMPILE_VERSION >= "3.3") "struct" else "struct<`X`: string>") val cols = dataTypes.zipWithIndex.map { case (dt, idx) => s"c$idx" -> dt } val (colNames, _) = cols.unzip diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/JDBCTestHelper.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/JDBCTestHelper.scala index 663fd1816..e7802f2fe 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/JDBCTestHelper.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/JDBCTestHelper.scala @@ -17,7 +17,7 @@ package org.apache.kyuubi.operation -import java.sql.{DriverManager, SQLException, Statement} +import java.sql.{DriverManager, PreparedStatement, SQLException, Statement} import java.util.Locale import org.apache.kyuubi.KyuubiFunSuite @@ -53,6 +53,7 @@ trait JDBCTestHelper extends KyuubiFunSuite { def withMultipleConnectionJdbcStatement( tableNames: String*)(fs: (Statement => Unit)*): Unit = { + info(s"Create JDBC connection using: $jdbcUrlWithConf") val connections = fs.map { _ => DriverManager.getConnection(jdbcUrlWithConf, user, password) } val statements = connections.map(_.createStatement()) @@ -75,6 +76,31 @@ trait JDBCTestHelper extends KyuubiFunSuite { } } + def withMultipleConnectionJdbcPrepareStatement( + sql: String, + tableNames: String*)(fs: (PreparedStatement => Unit)*): Unit = { + val connections = fs.map { _ => DriverManager.getConnection(jdbcUrlWithConf, user, password) } + val statements = connections.map(_.prepareStatement(sql)) + + try { + statements.zip(fs).foreach { case (s, f) => f(s) } + } finally { + tableNames.foreach { name => + if (name.toUpperCase(Locale.ROOT).startsWith("VIEW")) { + statements.head.execute(s"DROP VIEW IF EXISTS $name") + } else { + statements.head.execute(s"DROP TABLE IF EXISTS $name") + } + } + info("Closing statements") + statements.foreach(_.close()) + info("Closed statements") + info("Closing connections") + connections.foreach(_.close()) + info("Closed connections") + } + } + def withDatabases(dbNames: String*)(fs: (Statement => Unit)*): Unit = { val connections = fs.map { _ => DriverManager.getConnection(jdbcUrlWithConf, user, password) } val statements = connections.map(_.createStatement()) @@ -97,4 +123,10 @@ trait JDBCTestHelper extends KyuubiFunSuite { def withJdbcStatement(tableNames: String*)(f: Statement => Unit): Unit = { withMultipleConnectionJdbcStatement(tableNames: _*)(f) } + + def withJdbcPrepareStatement( + sql: String, + tableNames: String*)(f: PreparedStatement => Unit): Unit = { + withMultipleConnectionJdbcPrepareStatement(sql, tableNames: _*)(f) + } } diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/NoopOperation.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/NoopOperation.scala index 2d1166525..c369e00ef 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/NoopOperation.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/NoopOperation.scala @@ -21,7 +21,7 @@ import java.nio.ByteBuffer import scala.collection.JavaConverters._ -import org.apache.hive.service.rpc.thrift.{TColumn, TColumnDesc, TGetResultSetMetadataResp, TPrimitiveTypeEntry, TRowSet, TStringColumn, TTableSchema, TTypeDesc, TTypeEntry, TTypeId} +import org.apache.hive.service.rpc.thrift.{TColumn, TColumnDesc, TFetchResultsResp, TGetResultSetMetadataResp, TPrimitiveTypeEntry, TStringColumn, TTableSchema, TTypeDesc, TTypeEntry, TTypeId} import org.apache.kyuubi.KyuubiSQLException import org.apache.kyuubi.operation.FetchOrientation.FetchOrientation @@ -76,11 +76,16 @@ class NoopOperation(session: Session, shouldFail: Boolean = false) resp } - override def getNextRowSet(order: FetchOrientation, rowSetSize: Int): TRowSet = { + override def getNextRowSetInternal( + order: FetchOrientation, + rowSetSize: Int): TFetchResultsResp = { val col = TColumn.stringVal(new TStringColumn(Seq(opType).asJava, ByteBuffer.allocate(0))) val tRowSet = ThriftUtils.newEmptyRowSet tRowSet.addToColumns(col) - tRowSet + val resp = new TFetchResultsResp(OK_STATUS) + resp.setResults(tRowSet) + resp.setHasMoreRows(false) + resp } override def shouldRunAsync: Boolean = false diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/NoopOperationManager.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/NoopOperationManager.scala index 455e5d4d2..352aae905 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/NoopOperationManager.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/NoopOperationManager.scala @@ -20,7 +20,7 @@ package org.apache.kyuubi.operation import java.nio.ByteBuffer import java.util -import org.apache.hive.service.rpc.thrift.{TColumn, TRow, TRowSet, TStringColumn} +import org.apache.hive.service.rpc.thrift.{TColumn, TFetchResultsResp, TRow, TRowSet, TStatus, TStatusCode, TStringColumn} import org.apache.kyuubi.operation.FetchOrientation.FetchOrientation import org.apache.kyuubi.session.Session @@ -136,13 +136,16 @@ class NoopOperationManager extends OperationManager("noop") { override def getOperationLogRowSet( opHandle: OperationHandle, order: FetchOrientation, - maxRows: Int): TRowSet = { + maxRows: Int): TFetchResultsResp = { val logs = new util.ArrayList[String]() logs.add("test") val tColumn = TColumn.stringVal(new TStringColumn(logs, ByteBuffer.allocate(0))) val tRow = new TRowSet(0, new util.ArrayList[TRow](logs.size())) tRow.addToColumns(tColumn) - tRow + val resp = new TFetchResultsResp(new TStatus(TStatusCode.SUCCESS_STATUS)) + resp.setResults(tRow) + resp.setHasMoreRows(false) + resp } override def getQueryId(operation: Operation): String = { diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/OperationStateSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/OperationStateSuite.scala index d35ea246f..86c7e5e80 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/OperationStateSuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/OperationStateSuite.scala @@ -17,11 +17,13 @@ package org.apache.kyuubi.operation -import org.apache.hive.service.rpc.thrift.TOperationState +import org.apache.hive.service.rpc.thrift.{TOperationState, TProtocolVersion} import org.apache.hive.service.rpc.thrift.TOperationState._ import org.apache.kyuubi.{KyuubiFunSuite, KyuubiSQLException} +import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.operation.OperationState._ +import org.apache.kyuubi.session.NoopSessionManager class OperationStateSuite extends KyuubiFunSuite { test("toTOperationState") { @@ -79,4 +81,27 @@ class OperationStateSuite extends KyuubiFunSuite { assert(!OperationState.isTerminal(state)) } } + + test("kyuubi-5036 operation close should set completeTime") { + val sessionManager = new NoopSessionManager + sessionManager.initialize(KyuubiConf()) + val sHandle = sessionManager.openSession( + TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V11, + "kyuubi", + "passwd", + "localhost", + Map.empty) + val session = sessionManager.getSession(sHandle) + + val operation = new NoopOperation(session) + assert(operation.getStatus.completed == 0) + + operation.close() + val afterClose1 = operation.getStatus + assert(afterClose1.state == OperationState.CLOSED) + assert(afterClose1.completed != 0) + Thread.sleep(1000) + val afterClose2 = operation.getStatus + assert(afterClose1.completed == afterClose2.completed) + } } diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/SparkDataTypeTests.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/SparkDataTypeTests.scala index 688167703..2709bc861 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/SparkDataTypeTests.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/SparkDataTypeTests.scala @@ -19,15 +19,16 @@ package org.apache.kyuubi.operation import java.sql.{Date, Timestamp} -import org.apache.kyuubi.engine.SemanticVersion +import org.apache.kyuubi.util.SparkVersionUtil -trait SparkDataTypeTests extends HiveJDBCTestHelper { - protected lazy val SPARK_ENGINE_VERSION = sparkEngineMajorMinorVersion +trait SparkDataTypeTests extends HiveJDBCTestHelper with SparkVersionUtil { def resultFormat: String = "thrift" test("execute statement - select null") { - assume(resultFormat == "thrift" || (resultFormat == "arrow" && SPARK_ENGINE_VERSION >= "3.2")) + assume( + resultFormat == "thrift" || + (resultFormat == "arrow" && SPARK_ENGINE_RUNTIME_VERSION >= "3.2")) withJdbcStatement() { statement => val resultSet = statement.executeQuery("SELECT NULL AS col") assert(resultSet.next()) @@ -159,9 +160,10 @@ trait SparkDataTypeTests extends HiveJDBCTestHelper { } } - test("execute statement - select timestamp") { + test("execute statement - select timestamp - second") { withJdbcStatement() { statement => - val resultSet = statement.executeQuery("SELECT TIMESTAMP '2018-11-17 13:33:33' AS col") + val resultSet = statement.executeQuery( + "SELECT TIMESTAMP '2018-11-17 13:33:33' AS col") assert(resultSet.next()) assert(resultSet.getTimestamp("col") === Timestamp.valueOf("2018-11-17 13:33:33")) val metaData = resultSet.getMetaData @@ -171,13 +173,39 @@ trait SparkDataTypeTests extends HiveJDBCTestHelper { } } + test("execute statement - select timestamp - millisecond") { + withJdbcStatement() { statement => + val resultSet = statement.executeQuery( + "SELECT TIMESTAMP '2018-11-17 13:33:33.12345' AS col") + assert(resultSet.next()) + assert(resultSet.getTimestamp("col") === Timestamp.valueOf("2018-11-17 13:33:33.12345")) + val metaData = resultSet.getMetaData + assert(metaData.getColumnType(1) === java.sql.Types.TIMESTAMP) + assert(metaData.getPrecision(1) === 29) + assert(metaData.getScale(1) === 9) + } + } + + test("execute statement - select timestamp - overflow") { + withJdbcStatement() { statement => + val resultSet = statement.executeQuery( + "SELECT TIMESTAMP '2018-11-17 13:33:33.1234567' AS col") + assert(resultSet.next()) + assert(resultSet.getTimestamp("col") === Timestamp.valueOf("2018-11-17 13:33:33.123456")) + val metaData = resultSet.getMetaData + assert(metaData.getColumnType(1) === java.sql.Types.TIMESTAMP) + assert(metaData.getPrecision(1) === 29) + assert(metaData.getScale(1) === 9) + } + } + test("execute statement - select timestamp_ntz") { - assume(SPARK_ENGINE_VERSION >= "3.4") + assume(SPARK_ENGINE_RUNTIME_VERSION >= "3.4") withJdbcStatement() { statement => val resultSet = statement.executeQuery( - "SELECT make_timestamp_ntz(2022, 03, 24, 18, 08, 31.800) AS col") + "SELECT make_timestamp_ntz(2022, 03, 24, 18, 08, 31.8888) AS col") assert(resultSet.next()) - assert(resultSet.getTimestamp("col") === Timestamp.valueOf("2022-03-24 18:08:31.800")) + assert(resultSet.getTimestamp("col") === Timestamp.valueOf("2022-03-24 18:08:31.8888")) val metaData = resultSet.getMetaData assert(metaData.getColumnType(1) === java.sql.Types.TIMESTAMP) assert(metaData.getPrecision(1) === 29) @@ -186,7 +214,9 @@ trait SparkDataTypeTests extends HiveJDBCTestHelper { } test("execute statement - select daytime interval") { - assume(resultFormat == "thrift" || (resultFormat == "arrow" && SPARK_ENGINE_VERSION >= "3.3")) + assume( + resultFormat == "thrift" || + (resultFormat == "arrow" && SPARK_ENGINE_RUNTIME_VERSION >= "3.3")) withJdbcStatement() { statement => Map( "interval 1 day 1 hour -60 minutes 30 seconds" -> @@ -215,7 +245,7 @@ trait SparkDataTypeTests extends HiveJDBCTestHelper { assert(resultSet.next()) val result = resultSet.getString("col") val metaData = resultSet.getMetaData - if (SPARK_ENGINE_VERSION < "3.2") { + if (SPARK_ENGINE_RUNTIME_VERSION <= "3.1") { // for spark 3.1 and backwards assert(result === kv._2._2) assert(metaData.getPrecision(1) === Int.MaxValue) @@ -231,7 +261,9 @@ trait SparkDataTypeTests extends HiveJDBCTestHelper { } test("execute statement - select year/month interval") { - assume(resultFormat == "thrift" || (resultFormat == "arrow" && SPARK_ENGINE_VERSION >= "3.3")) + assume( + resultFormat == "thrift" || + (resultFormat == "arrow" && SPARK_ENGINE_RUNTIME_VERSION >= "3.3")) withJdbcStatement() { statement => Map( "INTERVAL 2022 YEAR" -> Tuple2("2022-0", "2022 years"), @@ -244,7 +276,7 @@ trait SparkDataTypeTests extends HiveJDBCTestHelper { assert(resultSet.next()) val result = resultSet.getString("col") val metaData = resultSet.getMetaData - if (SPARK_ENGINE_VERSION < "3.2") { + if (SPARK_ENGINE_RUNTIME_VERSION <= "3.1") { // for spark 3.1 and backwards assert(result === kv._2._2) assert(metaData.getPrecision(1) === Int.MaxValue) @@ -260,7 +292,9 @@ trait SparkDataTypeTests extends HiveJDBCTestHelper { } test("execute statement - select array") { - assume(resultFormat == "thrift" || (resultFormat == "arrow" && SPARK_ENGINE_VERSION >= "3.2")) + assume( + resultFormat == "thrift" || + (resultFormat == "arrow" && SPARK_ENGINE_RUNTIME_VERSION >= "3.2")) withJdbcStatement() { statement => val resultSet = statement.executeQuery( "SELECT array() AS col1, array(1) AS col2, array(null) AS col3") @@ -278,7 +312,9 @@ trait SparkDataTypeTests extends HiveJDBCTestHelper { } test("execute statement - select map") { - assume(resultFormat == "thrift" || (resultFormat == "arrow" && SPARK_ENGINE_VERSION >= "3.2")) + assume( + resultFormat == "thrift" || + (resultFormat == "arrow" && SPARK_ENGINE_RUNTIME_VERSION >= "3.2")) withJdbcStatement() { statement => val resultSet = statement.executeQuery( "SELECT map() AS col1, map(1, 2, 3, 4) AS col2, map(1, null) AS col3") @@ -296,7 +332,9 @@ trait SparkDataTypeTests extends HiveJDBCTestHelper { } test("execute statement - select struct") { - assume(resultFormat == "thrift" || (resultFormat == "arrow" && SPARK_ENGINE_VERSION >= "3.2")) + assume( + resultFormat == "thrift" || + (resultFormat == "arrow" && SPARK_ENGINE_RUNTIME_VERSION >= "3.2")) withJdbcStatement() { statement => val resultSet = statement.executeQuery( "SELECT struct('1', '2') AS col1," + @@ -315,15 +353,4 @@ trait SparkDataTypeTests extends HiveJDBCTestHelper { assert(metaData.getScale(2) == 0) } } - - def sparkEngineMajorMinorVersion: SemanticVersion = { - var sparkRuntimeVer = "" - withJdbcStatement() { stmt => - val result = stmt.executeQuery("SELECT version()") - assert(result.next()) - sparkRuntimeVer = result.getString(1) - assert(!result.next()) - } - SemanticVersion(sparkRuntimeVer) - } } diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/SparkMetadataTests.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/SparkMetadataTests.scala index 4faf5bba4..97099ce47 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/SparkMetadataTests.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/SparkMetadataTests.scala @@ -17,11 +17,6 @@ package org.apache.kyuubi.operation -import java.sql.{DatabaseMetaData, ResultSet, SQLException, SQLFeatureNotSupportedException} - -import scala.util.Random - -import org.apache.kyuubi.{KYUUBI_VERSION, KyuubiSQLException} import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant._ // For both `in-memory` and `hive` external catalog @@ -292,186 +287,4 @@ trait SparkMetadataTests extends HiveJDBCTestHelper { assert(typeInfo.getInt(NUM_PREC_RADIX) === 0) } } - - test("audit Kyuubi Hive JDBC connection common MetaData") { - withJdbcStatement() { statement => - val metaData = statement.getConnection.getMetaData - Seq( - () => metaData.allProceduresAreCallable(), - () => metaData.getURL, - () => metaData.getUserName, - () => metaData.isReadOnly, - () => metaData.nullsAreSortedHigh, - () => metaData.nullsAreSortedLow, - () => metaData.nullsAreSortedAtStart(), - () => metaData.nullsAreSortedAtEnd(), - () => metaData.usesLocalFiles(), - () => metaData.usesLocalFilePerTable(), - () => metaData.supportsMixedCaseIdentifiers(), - () => metaData.supportsMixedCaseQuotedIdentifiers(), - () => metaData.storesUpperCaseIdentifiers(), - () => metaData.storesUpperCaseQuotedIdentifiers(), - () => metaData.storesLowerCaseIdentifiers(), - () => metaData.storesLowerCaseQuotedIdentifiers(), - () => metaData.storesMixedCaseIdentifiers(), - () => metaData.storesMixedCaseQuotedIdentifiers(), - () => metaData.nullPlusNonNullIsNull, - () => metaData.supportsConvert, - () => metaData.supportsTableCorrelationNames, - () => metaData.supportsDifferentTableCorrelationNames, - () => metaData.supportsExpressionsInOrderBy(), - () => metaData.supportsOrderByUnrelated, - () => metaData.supportsGroupByUnrelated, - () => metaData.supportsGroupByBeyondSelect, - () => metaData.supportsLikeEscapeClause, - () => metaData.supportsMultipleTransactions, - () => metaData.supportsMinimumSQLGrammar, - () => metaData.supportsCoreSQLGrammar, - () => metaData.supportsExtendedSQLGrammar, - () => metaData.supportsANSI92EntryLevelSQL, - () => metaData.supportsANSI92IntermediateSQL, - () => metaData.supportsANSI92FullSQL, - () => metaData.supportsIntegrityEnhancementFacility, - () => metaData.isCatalogAtStart, - () => metaData.supportsSubqueriesInComparisons, - () => metaData.supportsSubqueriesInExists, - () => metaData.supportsSubqueriesInIns, - () => metaData.supportsSubqueriesInQuantifieds, - // Spark support this, see https://issues.apache.org/jira/browse/SPARK-18455 - () => metaData.supportsCorrelatedSubqueries, - () => metaData.supportsOpenCursorsAcrossCommit, - () => metaData.supportsOpenCursorsAcrossRollback, - () => metaData.supportsOpenStatementsAcrossCommit, - () => metaData.supportsOpenStatementsAcrossRollback, - () => metaData.getMaxBinaryLiteralLength, - () => metaData.getMaxCharLiteralLength, - () => metaData.getMaxColumnsInGroupBy, - () => metaData.getMaxColumnsInIndex, - () => metaData.getMaxColumnsInOrderBy, - () => metaData.getMaxColumnsInSelect, - () => metaData.getMaxColumnsInTable, - () => metaData.getMaxConnections, - () => metaData.getMaxCursorNameLength, - () => metaData.getMaxIndexLength, - () => metaData.getMaxSchemaNameLength, - () => metaData.getMaxProcedureNameLength, - () => metaData.getMaxCatalogNameLength, - () => metaData.getMaxRowSize, - () => metaData.doesMaxRowSizeIncludeBlobs, - () => metaData.getMaxStatementLength, - () => metaData.getMaxStatements, - () => metaData.getMaxTableNameLength, - () => metaData.getMaxTablesInSelect, - () => metaData.getMaxUserNameLength, - () => metaData.supportsTransactionIsolationLevel(1), - () => metaData.supportsDataDefinitionAndDataManipulationTransactions, - () => metaData.supportsDataManipulationTransactionsOnly, - () => metaData.dataDefinitionCausesTransactionCommit, - () => metaData.dataDefinitionIgnoredInTransactions, - () => metaData.getColumnPrivileges("", "%", "%", "%"), - () => metaData.getTablePrivileges("", "%", "%"), - () => metaData.getBestRowIdentifier("", "%", "%", 0, true), - () => metaData.getVersionColumns("", "%", "%"), - () => metaData.getExportedKeys("", "default", ""), - () => metaData.supportsResultSetConcurrency(ResultSet.TYPE_FORWARD_ONLY, 2), - () => metaData.ownUpdatesAreVisible(ResultSet.TYPE_FORWARD_ONLY), - () => metaData.ownDeletesAreVisible(ResultSet.TYPE_FORWARD_ONLY), - () => metaData.ownInsertsAreVisible(ResultSet.TYPE_FORWARD_ONLY), - () => metaData.othersUpdatesAreVisible(ResultSet.TYPE_FORWARD_ONLY), - () => metaData.othersDeletesAreVisible(ResultSet.TYPE_FORWARD_ONLY), - () => metaData.othersInsertsAreVisible(ResultSet.TYPE_FORWARD_ONLY), - () => metaData.updatesAreDetected(ResultSet.TYPE_FORWARD_ONLY), - () => metaData.deletesAreDetected(ResultSet.TYPE_FORWARD_ONLY), - () => metaData.insertsAreDetected(ResultSet.TYPE_FORWARD_ONLY), - () => metaData.supportsNamedParameters(), - () => metaData.supportsMultipleOpenResults, - () => metaData.supportsGetGeneratedKeys, - () => metaData.getSuperTypes("", "%", "%"), - () => metaData.getSuperTables("", "%", "%"), - () => metaData.getAttributes("", "%", "%", "%"), - () => metaData.getResultSetHoldability, - () => metaData.locatorsUpdateCopy, - () => metaData.supportsStatementPooling, - () => metaData.getRowIdLifetime, - () => metaData.supportsStoredFunctionsUsingCallSyntax, - () => metaData.autoCommitFailureClosesAllResultSets, - () => metaData.getFunctionColumns("", "%", "%", "%"), - () => metaData.getPseudoColumns("", "%", "%", "%"), - () => metaData.generatedKeyAlwaysReturned).foreach { func => - val e = intercept[SQLFeatureNotSupportedException](func()) - assert(e.getMessage === "Method not supported") - } - - assert(metaData.allTablesAreSelectable) - assert(metaData.getClientInfoProperties.next) - assert(metaData.getDriverName === "Kyuubi Project Hive JDBC Client" || - metaData.getDriverName === "Kyuubi Project Hive JDBC Shaded Client") - assert(metaData.getDriverVersion === KYUUBI_VERSION) - assert( - metaData.getIdentifierQuoteString === " ", - "This method returns a space \" \" if identifier quoting is not supported") - assert(metaData.getNumericFunctions === "") - assert(metaData.getStringFunctions === "") - assert(metaData.getSystemFunctions === "") - assert(metaData.getTimeDateFunctions === "") - assert(metaData.getSearchStringEscape === "\\") - assert(metaData.getExtraNameCharacters === "") - assert(metaData.supportsAlterTableWithAddColumn()) - assert(!metaData.supportsAlterTableWithDropColumn()) - assert(metaData.supportsColumnAliasing()) - assert(metaData.supportsGroupBy) - assert(!metaData.supportsMultipleResultSets) - assert(!metaData.supportsNonNullableColumns) - assert(metaData.supportsOuterJoins) - assert(metaData.supportsFullOuterJoins) - assert(metaData.supportsLimitedOuterJoins) - assert(metaData.getSchemaTerm === "database") - assert(metaData.getProcedureTerm === "UDF") - assert(metaData.getCatalogTerm === "catalog") - assert(metaData.getCatalogSeparator === ".") - assert(metaData.supportsSchemasInDataManipulation) - assert(!metaData.supportsSchemasInProcedureCalls) - assert(metaData.supportsSchemasInTableDefinitions) - assert(!metaData.supportsSchemasInIndexDefinitions) - assert(!metaData.supportsSchemasInPrivilegeDefinitions) - assert(metaData.supportsCatalogsInDataManipulation) - assert(metaData.supportsCatalogsInProcedureCalls) - assert(metaData.supportsCatalogsInTableDefinitions) - assert(metaData.supportsCatalogsInIndexDefinitions) - assert(metaData.supportsCatalogsInPrivilegeDefinitions) - assert(!metaData.supportsPositionedDelete) - assert(!metaData.supportsPositionedUpdate) - assert(!metaData.supportsSelectForUpdate) - assert(!metaData.supportsStoredProcedures) - // This is actually supported, but hive jdbc package return false - assert(!metaData.supportsUnion) - assert(metaData.supportsUnionAll) - assert(metaData.getMaxColumnNameLength === 128) - assert(metaData.getDefaultTransactionIsolation === java.sql.Connection.TRANSACTION_NONE) - assert(!metaData.supportsTransactions) - assert(!metaData.getProcedureColumns("", "%", "%", "%").next()) - val e1 = intercept[SQLException] { - metaData.getPrimaryKeys("", "default", "src").next() - } - assert(e1.getMessage.contains(KyuubiSQLException.featureNotSupported().getMessage)) - assert(!metaData.getImportedKeys("", "default", "").next()) - - val e2 = intercept[SQLException] { - metaData.getCrossReference("", "default", "src", "", "default", "src2").next() - } - assert(e2.getMessage.contains(KyuubiSQLException.featureNotSupported().getMessage)) - assert(!metaData.getIndexInfo("", "default", "src", true, true).next()) - - assert(metaData.supportsResultSetType(new Random().nextInt())) - assert(!metaData.supportsBatchUpdates) - assert(!metaData.getUDTs(",", "%", "%", null).next()) - assert(!metaData.supportsSavepoints) - assert(!metaData.supportsResultSetHoldability(ResultSet.HOLD_CURSORS_OVER_COMMIT)) - assert(metaData.getJDBCMajorVersion === 3) - assert(metaData.getJDBCMinorVersion === 0) - assert(metaData.getSQLStateType === DatabaseMetaData.sqlStateSQL) - assert(metaData.getMaxLogicalLobSize === 0) - assert(!metaData.supportsRefCursors) - } - } } diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/SparkQueryTests.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/SparkQueryTests.scala index e297e6281..0ac56e3bc 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/SparkQueryTests.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/SparkQueryTests.scala @@ -28,7 +28,6 @@ import org.apache.hive.service.rpc.thrift.{TExecuteStatementReq, TFetchResultsRe import org.apache.kyuubi.{KYUUBI_VERSION, Utils} import org.apache.kyuubi.config.KyuubiConf -import org.apache.kyuubi.util.SparkVersionUtil.isSparkVersionAtLeast trait SparkQueryTests extends SparkDataTypeTests with HiveJDBCTestHelper { @@ -187,7 +186,7 @@ trait SparkQueryTests extends SparkDataTypeTests with HiveJDBCTestHelper { withJdbcStatement("t") { statement => try { val assertTableOrViewNotfound: (Exception, String) => Unit = (e, tableName) => { - if (isSparkVersionAtLeast("3.4")) { + if (SPARK_ENGINE_RUNTIME_VERSION >= "3.4") { assert(e.getMessage.contains("[TABLE_OR_VIEW_NOT_FOUND]")) assert(e.getMessage.contains(s"The table or view `$tableName` cannot be found.")) } else { @@ -219,6 +218,35 @@ trait SparkQueryTests extends SparkDataTypeTests with HiveJDBCTestHelper { } } + test("kyuubi #3444: Plan only mode with lineage mode") { + + val ddl = "create table if not exists t0(a int) using parquet" + val dql = "select * from t0" + withSessionConf()(Map(KyuubiConf.OPERATION_PLAN_ONLY_MODE.key -> NoneMode.name))() { + withJdbcStatement("t0") { statement => + statement.execute(ddl) + statement.execute("SET kyuubi.operation.plan.only.mode=lineage") + val lineageParserClassName = "org.apache.kyuubi.plugin.lineage.LineageParserProvider" + + try { + val resultSet = statement.executeQuery(dql) + assert(resultSet.next()) + val actualResult = + """ + |{"inputTables":["spark_catalog.default.t0"],"outputTables":[], + |"columnLineage":[{"column":"a","originalColumns":["spark_catalog.default.t0.a"]}]} + |""".stripMargin.split("\n").mkString("") + assert(resultSet.getString(1) == actualResult) + } catch { + case e: Throwable => + assert(e.getMessage.contains(s"'$lineageParserClassName' not found")) + } finally { + statement.execute("SET kyuubi.operation.plan.only.mode=none") + } + } + } + } + test("execute simple scala code") { withJdbcStatement() { statement => statement.execute("SET kyuubi.operation.language=scala") @@ -242,7 +270,7 @@ trait SparkQueryTests extends SparkDataTypeTests with HiveJDBCTestHelper { |""".stripMargin val rs1 = statement.executeQuery(code) rs1.next() - assert(rs1.getString(1) startsWith "df: org.apache.spark.sql.DataFrame") + assert(rs1.getString(1) contains "df: org.apache.spark.sql.DataFrame") // continue val rs2 = statement.executeQuery("df.count()") @@ -283,7 +311,7 @@ trait SparkQueryTests extends SparkDataTypeTests with HiveJDBCTestHelper { |""".stripMargin val rs5 = statement.executeQuery(code2) rs5.next() - assert(rs5.getString(1) startsWith "df: org.apache.spark.sql.DataFrame") + assert(rs5.getString(1) contains "df: org.apache.spark.sql.DataFrame") // re-assign val rs6 = statement.executeQuery("result.set(df)") @@ -384,7 +412,7 @@ trait SparkQueryTests extends SparkDataTypeTests with HiveJDBCTestHelper { rs.next() // scalastyle:off println(rs.getString(1)) - // scalastyle:on + // scalastyle:on } val code1 = s"""spark.sql("add jar " + jarPath)""" @@ -392,7 +420,7 @@ trait SparkQueryTests extends SparkDataTypeTests with HiveJDBCTestHelper { statement.execute(code1) val rs = statement.executeQuery(code2) rs.next() - assert(rs.getString(1) == "x: Int = 3") + assert(rs.getString(1) contains "x: Int = 3") } } @@ -433,13 +461,13 @@ trait SparkQueryTests extends SparkDataTypeTests with HiveJDBCTestHelper { expectedFormat = "thrift") checkStatusAndResultSetFormatHint( sql = "set kyuubi.operation.result.format=arrow", - expectedFormat = "arrow") + expectedFormat = "thrift") checkStatusAndResultSetFormatHint( sql = "SELECT 1", expectedFormat = "arrow") checkStatusAndResultSetFormatHint( sql = "set kyuubi.operation.result.format=thrift", - expectedFormat = "thrift") + expectedFormat = "arrow") checkStatusAndResultSetFormatHint( sql = "set kyuubi.operation.result.format", expectedFormat = "thrift") diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/log/OperationLogSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/log/OperationLogSuite.scala index 758eeeeaf..570a8159b 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/log/OperationLogSuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/log/OperationLogSuite.scala @@ -27,7 +27,7 @@ import org.apache.hive.service.rpc.thrift.{TProtocolVersion, TRowSet} import org.apache.kyuubi.{KyuubiFunSuite, KyuubiSQLException, Utils} import org.apache.kyuubi.config.KyuubiConf -import org.apache.kyuubi.operation.OperationHandle +import org.apache.kyuubi.operation.{FetchOrientation, OperationHandle} import org.apache.kyuubi.session.NoopSessionManager import org.apache.kyuubi.util.ThriftUtils @@ -61,10 +61,10 @@ class OperationLogSuite extends KyuubiFunSuite { assert(!Files.exists(logFile)) OperationLog.setCurrentOperationLog(operationLog) - assert(OperationLog.getCurrentOperationLog === operationLog) + assert(OperationLog.getCurrentOperationLog === Some(operationLog)) OperationLog.removeCurrentOperationLog() - assert(OperationLog.getCurrentOperationLog === null) + assert(OperationLog.getCurrentOperationLog.isEmpty) operationLog.write(msg1 + "\n") assert(Files.exists(logFile)) @@ -237,6 +237,47 @@ class OperationLogSuite extends KyuubiFunSuite { } } + test("test fetchOrientation read") { + val file = Utils.createTempDir().resolve("f") + val file2 = Utils.createTempDir().resolve("extra") + val writer = Files.newBufferedWriter(file, StandardCharsets.UTF_8) + val writer2 = Files.newBufferedWriter(file2, StandardCharsets.UTF_8) + try { + 0.until(10).foreach(x => writer.write(s"$x\n")) + writer.flush() + writer.close() + 10.until(20).foreach(x => writer2.write(s"$x\n")) + writer2.flush() + writer2.close() + + def compareResult(rows: TRowSet, expected: Seq[String]): Unit = { + val res = rows.getColumns.get(0).getStringVal.getValues.asScala + assert(res.size == expected.size) + res.zip(expected).foreach { case (l, r) => + assert(l == r) + } + } + + val log = new OperationLog(file) + log.addExtraLog(file2) + // The operation log file is created externally and should be initialized actively. + log.initOperationLogIfNecessary() + + compareResult( + log.read(FetchOrientation.FETCH_NEXT, 10), + Seq("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")) + compareResult(log.read(FetchOrientation.FETCH_NEXT, 5), Seq("10", "11", "12", "13", "14")) + compareResult(log.read(FetchOrientation.FETCH_FIRST, 5), Seq("0", "1", "2", "3", "4")) + compareResult( + log.read(FetchOrientation.FETCH_NEXT, 10), + Seq("5", "6", "7", "8", "9", "10", "11", "12", "13", "14")) + compareResult(log.read(FetchOrientation.FETCH_NEXT, 10), Seq("15", "16", "17", "18", "19")) + } finally { + Utils.deleteDirectoryRecursively(file.toFile) + Utils.deleteDirectoryRecursively(file2.toFile) + } + } + test("[KYUUBI #3511] Reading an uninitialized log should return empty rowSet") { val sessionManager = new NoopSessionManager sessionManager.initialize(KyuubiConf()) @@ -297,4 +338,53 @@ class OperationLogSuite extends KyuubiFunSuite { Utils.deleteDirectoryRecursively(extraFile.toFile) } } + + test("Closing the unwritten operation log should not throw an exception") { + val sessionManager = new NoopSessionManager + sessionManager.initialize(KyuubiConf()) + val sHandle = sessionManager.openSession( + TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V10, + "kyuubi", + "passwd", + "localhost", + Map.empty) + val session = sessionManager.getSession(sHandle) + OperationLog.createOperationLogRootDirectory(session) + val oHandle = OperationHandle() + + val log = OperationLog.createOperationLog(session, oHandle) + val tRowSet = log.read(1) + assert(tRowSet == ThriftUtils.newEmptyRowSet) + // close the operation log without writing + log.close() + session.close() + } + + test("test operationLog multiple read with missing line ") { + val file = Utils.createTempDir().resolve("f") + val writer = Files.newBufferedWriter(file, StandardCharsets.UTF_8) + try { + 0.until(10).foreach(x => writer.write(s"$x\n")) + writer.flush() + writer.close() + + val log = new OperationLog(file) + // The operation log file is created externally and should be initialized actively. + log.initOperationLogIfNecessary() + + def compareResult(rows: TRowSet, expected: Seq[String]): Unit = { + val res = rows.getColumns.get(0).getStringVal.getValues.asScala + assert(res.size == expected.size) + res.zip(expected).foreach { case (l, r) => + assert(l == r) + } + } + compareResult(log.read(2), Seq("0", "1")) + compareResult(log.read(3), Seq("2", "3", "4")) + compareResult(log.read(10), Seq("5", "6", "7", "8", "9")) + } finally { + Utils.deleteDirectoryRecursively(file.toFile) + } + } + } diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/TFrontendServiceSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/TFrontendServiceSuite.scala index 28442fe62..444bfe2cc 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/TFrontendServiceSuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/TFrontendServiceSuite.scala @@ -40,8 +40,8 @@ class TFrontendServiceSuite extends KyuubiFunSuite { .set(KyuubiConf.SESSION_CHECK_INTERVAL, Duration.ofSeconds(5).toMillis) .set(KyuubiConf.SESSION_IDLE_TIMEOUT, Duration.ofSeconds(5).toMillis) .set(KyuubiConf.OPERATION_IDLE_TIMEOUT, Duration.ofSeconds(20).toMillis) - .set(KyuubiConf.SESSION_CONF_RESTRICT_LIST, Seq("spark.*")) - .set(KyuubiConf.SESSION_CONF_IGNORE_LIST, Seq("session.engine.*")) + .set(KyuubiConf.SESSION_CONF_RESTRICT_LIST, Set("spark.*")) + .set(KyuubiConf.SESSION_CONF_IGNORE_LIST, Set("session.engine.*")) private def withSessionHandle(f: (TCLIService.Iface, TSessionHandle) => Unit): Unit = { TClientTestUtils.withSessionHandle(server.frontendServices.head.connectionUrl, Map.empty)(f) @@ -115,6 +115,33 @@ class TFrontendServiceSuite extends KyuubiFunSuite { assert(service2.connectionUrl.split("\\.")(0).toInt > 0) } + test("advertised host") { + + def newService: TBinaryFrontendService = { + new TBinaryFrontendService("DummyThriftBinaryFrontendService") { + override val serverable: Serverable = new NoopTBinaryFrontendServer + override val discoveryService: Option[Service] = None + } + } + + val conf = new KyuubiConf() + .set(FRONTEND_THRIFT_BINARY_BIND_HOST.key, "localhost") + .set(FRONTEND_THRIFT_BINARY_BIND_PORT, 0) + .set(FRONTEND_ADVERTISED_HOST, "dummy.host") + val service = newService + + service.initialize(conf) + assert(service.connectionUrl.startsWith("dummy.host")) + + val service2 = newService + val conf2 = KyuubiConf() + .set(FRONTEND_THRIFT_BINARY_BIND_HOST.key, "localhost") + .set(FRONTEND_THRIFT_BINARY_BIND_PORT, 0) + + service2.initialize(conf2) + assert(service2.connectionUrl.startsWith("localhost")) + } + test("open session") { TClientTestUtils.withThriftClient(server.frontendServices.head) { client => @@ -124,7 +151,7 @@ class TFrontendServiceSuite extends KyuubiFunSuite { val resp = client.OpenSession(req) val handle = resp.getSessionHandle assert(handle != null) - assert(resp.getStatus.getStatusCode == TStatusCode.SUCCESS_STATUS) + assert(resp.getStatus.getStatusCode === TStatusCode.SUCCESS_STATUS) req.setConfiguration(Map("kyuubi.test.should.fail" -> "true").asJava) val resp1 = client.OpenSession(req) @@ -514,39 +541,37 @@ class TFrontendServiceSuite extends KyuubiFunSuite { test("close expired operations") { withSessionHandle { (client, handle) => - val req = new TCancelOperationReq() - val req1 = new TGetSchemasReq(handle) - val resp1 = client.GetSchemas(req1) + val req = new TGetSchemasReq(handle) + val resp = client.GetSchemas(req) val sessionManager = server.backendService.sessionManager val session = sessionManager .getSession(SessionHandle(handle)) .asInstanceOf[AbstractSession] var lastAccessTime = session.lastAccessTime - assert(sessionManager.getOpenSessionCount == 1) + assert(sessionManager.getOpenSessionCount === 1) assert(session.lastIdleTime > 0) - resp1.getOperationHandle - req.setOperationHandle(resp1.getOperationHandle) - val resp2 = client.CancelOperation(req) - assert(resp2.getStatus.getStatusCode === TStatusCode.SUCCESS_STATUS) - assert(sessionManager.getOpenSessionCount == 1) - assert(session.lastIdleTime == 0) - assert(lastAccessTime < session.lastAccessTime) + val cancelOpReq = new TCancelOperationReq(resp.getOperationHandle) + val cancelOpResp = client.CancelOperation(cancelOpReq) + assert(cancelOpResp.getStatus.getStatusCode === TStatusCode.SUCCESS_STATUS) + assert(sessionManager.getOpenSessionCount === 1) + assert(session.lastIdleTime === 0) + eventually(timeout(Span(60, Seconds)), interval(Span(1, Seconds))) { + assert(lastAccessTime < session.lastAccessTime) + } lastAccessTime = session.lastAccessTime eventually(timeout(Span(60, Seconds)), interval(Span(1, Seconds))) { - assert(session.lastIdleTime > lastAccessTime) + assert(lastAccessTime <= session.lastIdleTime) } - info("operation is terminated") - assert(lastAccessTime == session.lastAccessTime) - assert(sessionManager.getOpenSessionCount == 1) eventually(timeout(Span(60, Seconds)), interval(Span(1, Seconds))) { assert(session.lastAccessTime > lastAccessTime) } - assert(sessionManager.getOpenSessionCount == 0) + info("session is terminated") + assert(sessionManager.getOpenSessionCount === 0) } } @@ -562,7 +587,7 @@ class TFrontendServiceSuite extends KyuubiFunSuite { Map( "session.engine.spark.main.resource" -> "org.apahce.kyuubi.test", "session.check.interval" -> "10000")) - assert(conf.size == 1) - assert(conf("session.check.interval") == "10000") + assert(conf.size === 1) + assert(conf("session.check.interval") === "10000") } } diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/InternalSecurityAccessorSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/InternalSecurityAccessorSuite.scala index e6c4c8506..e92ac7e61 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/InternalSecurityAccessorSuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/InternalSecurityAccessorSuite.scala @@ -22,9 +22,8 @@ import org.apache.kyuubi.config.KyuubiConf class InternalSecurityAccessorSuite extends KyuubiFunSuite { private val conf = KyuubiConf() - conf.set( - KyuubiConf.ENGINE_SECURITY_SECRET_PROVIDER, - classOf[UserDefinedEngineSecuritySecretProvider].getCanonicalName) + .set(KyuubiConf.ENGINE_SECURITY_SECRET_PROVIDER, "simple") + .set(KyuubiConf.SIMPLE_SECURITY_SECRET_PROVIDER_PROVIDER_SECRET, "ENGINE____SECRET") test("test encrypt/decrypt, issue token/auth token") { Seq("AES/CBC/PKCS5PADDING", "AES/CTR/NoPadding").foreach { cipher => diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/JdbcAuthenticationProviderImplSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/JdbcAuthenticationProviderImplSuite.scala index dcbc62dfa..4642eb910 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/JdbcAuthenticationProviderImplSuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/JdbcAuthenticationProviderImplSuite.scala @@ -17,32 +17,27 @@ package org.apache.kyuubi.service.authentication -import java.sql.DriverManager import java.util.Properties import javax.security.sasl.AuthenticationException import javax.sql.DataSource import com.zaxxer.hikari.util.DriverDataSource -import org.apache.kyuubi.{KyuubiFunSuite, Utils} +import org.apache.kyuubi.KyuubiFunSuite import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.config.KyuubiConf._ import org.apache.kyuubi.util.JdbcUtils class JdbcAuthenticationProviderImplSuite extends KyuubiFunSuite { - protected val dbUser: String = "bowenliang123" - protected val dbPasswd: String = "bowenliang123@kyuubi" - protected val authDbName: String = "auth_db" - protected val dbUrl: String = s"jdbc:derby:memory:$authDbName" - protected val jdbcUrl: String = s"$dbUrl;create=true" - private val authDbDriverClz = "org.apache.derby.jdbc.AutoloadedDriver" + protected val jdbcUrl: String = "jdbc:sqlite:file:test_auth.db" + private val authDbDriverClz = "org.sqlite.JDBC" implicit private val ds: DataSource = new DriverDataSource( jdbcUrl, authDbDriverClz, new Properties, - dbUser, - dbPasswd) + null, + null) protected val authUser: String = "kyuubiuser" protected val authPasswd: String = "kyuubiuuserpassword" @@ -50,15 +45,13 @@ class JdbcAuthenticationProviderImplSuite extends KyuubiFunSuite { protected val conf: KyuubiConf = new KyuubiConf() .set(AUTHENTICATION_JDBC_DRIVER, authDbDriverClz) .set(AUTHENTICATION_JDBC_URL, jdbcUrl) - .set(AUTHENTICATION_JDBC_USER, dbUser) - .set(AUTHENTICATION_JDBC_PASSWORD, dbPasswd) .set( AUTHENTICATION_JDBC_QUERY, "SELECT 1 FROM user_auth WHERE username=${user} and passwd=${password}") override def beforeAll(): Unit = { + JdbcUtils.execute("DROP TABLE IF EXISTS user_auth")() // init db - JdbcUtils.execute(s"CREATE SCHEMA $dbUser")() JdbcUtils.execute( """CREATE TABLE user_auth ( | username VARCHAR(64) NOT NULL PRIMARY KEY, @@ -72,15 +65,6 @@ class JdbcAuthenticationProviderImplSuite extends KyuubiFunSuite { super.beforeAll() } - override def afterAll(): Unit = { - super.afterAll() - - // cleanup db - Utils.tryLogNonFatalError { - DriverManager.getConnection(s"$dbUrl;shutdown=true") - } - } - test("authenticate tests") { val providerImpl = new JdbcAuthenticationProviderImpl(conf) providerImpl.authenticate(authUser, authPasswd) @@ -144,6 +128,6 @@ class JdbcAuthenticationProviderImplSuite extends KyuubiFunSuite { val e12 = intercept[AuthenticationException] { new JdbcAuthenticationProviderImpl(_conf).authenticate(authUser, authPasswd) } - assert(e12.getCause.getMessage.contains("Column 'UNKNOWN_COLUMN' is either not in any table")) + assert(e12.getCause.getMessage.contains("no such column: unknown_column")) } } diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactorySuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactorySuite.scala index 19b89b47e..316c9b2df 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactorySuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/KyuubiAuthenticationFactorySuite.scala @@ -25,6 +25,7 @@ import org.apache.thrift.transport.TSaslServerTransport import org.apache.kyuubi.{KyuubiFunSuite, KyuubiSQLException} import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.service.authentication.PlainSASLServer.SaslPlainProvider +import org.apache.kyuubi.util.AssertionUtils._ import org.apache.kyuubi.util.KyuubiHadoopUtils class KyuubiAuthenticationFactorySuite extends KyuubiFunSuite { @@ -56,21 +57,21 @@ class KyuubiAuthenticationFactorySuite extends KyuubiFunSuite { } test("AuthType Other") { - val conf = KyuubiConf().set(KyuubiConf.AUTHENTICATION_METHOD, Seq("INVALID")) - val e = intercept[IllegalArgumentException](new KyuubiAuthenticationFactory(conf)) - assert(e.getMessage contains "the authentication type should be one or more of" + - " NOSASL,NONE,LDAP,JDBC,KERBEROS,CUSTOM") + val conf = KyuubiConf().set(KyuubiConf.AUTHENTICATION_METHOD, Set("INVALID")) + interceptEquals[IllegalArgumentException] { new KyuubiAuthenticationFactory(conf) }( + "The value of kyuubi.authentication should be one of" + + " NOSASL, NONE, LDAP, JDBC, KERBEROS, CUSTOM, but was INVALID") } test("AuthType LDAP") { - val conf = KyuubiConf().set(KyuubiConf.AUTHENTICATION_METHOD, Seq("LDAP")) + val conf = KyuubiConf().set(KyuubiConf.AUTHENTICATION_METHOD, Set("LDAP")) val authFactory = new KyuubiAuthenticationFactory(conf) authFactory.getTTransportFactory assert(Security.getProviders.exists(_.isInstanceOf[SaslPlainProvider])) } test("AuthType KERBEROS w/o keytab/principal") { - val conf = KyuubiConf().set(KyuubiConf.AUTHENTICATION_METHOD, Seq("KERBEROS")) + val conf = KyuubiConf().set(KyuubiConf.AUTHENTICATION_METHOD, Set("KERBEROS")) val factory = new KyuubiAuthenticationFactory(conf) val e = intercept[LoginException](factory.getTTransportFactory) @@ -78,11 +79,11 @@ class KyuubiAuthenticationFactorySuite extends KyuubiFunSuite { } test("AuthType is NOSASL if only NOSASL is specified") { - val conf = KyuubiConf().set(KyuubiConf.AUTHENTICATION_METHOD, Seq("NOSASL")) + val conf = KyuubiConf().set(KyuubiConf.AUTHENTICATION_METHOD, Set("NOSASL")) var factory = new KyuubiAuthenticationFactory(conf) !factory.getTTransportFactory.isInstanceOf[TSaslServerTransport.Factory] - conf.set(KyuubiConf.AUTHENTICATION_METHOD, Seq("NOSASL", "NONE")) + conf.set(KyuubiConf.AUTHENTICATION_METHOD, Set("NOSASL", "NONE")) factory = new KyuubiAuthenticationFactory(conf) factory.getTTransportFactory.isInstanceOf[TSaslServerTransport.Factory] } diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/LdapAtnProviderSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/LdapAtnProviderSuite.scala new file mode 100644 index 000000000..c3c67e421 --- /dev/null +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/LdapAtnProviderSuite.scala @@ -0,0 +1,493 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.service.authentication + +import com.unboundid.ldap.sdk.Entry + +import org.apache.kyuubi.service.authentication.ldap.{LdapAuthenticationTestCase, User} + +class LdapAtnProviderSuite extends WithLdapServer { + + override protected val ldapBaseDn: Array[String] = Array( + "dc=example,dc=com", + "cn=microsoft,ou=schema") + + private val GROUP1_NAME = "group1" + private val GROUP2_NAME = "group2" + private val GROUP3_NAME = "group3" + private val GROUP4_NAME = "group4" + + private val GROUP_ADMINS_NAME = "admins" + private val GROUP_TEAM1_NAME = "team1" + private val GROUP_TEAM2_NAME = "team2" + private val GROUP_RESOURCE1_NAME = "resource1" + private val GROUP_RESOURCE2_NAME = "resource2" + + private val USER1 = + User.useIdForPassword(id = "user1", dn = "uid=user1,ou=People,dc=example,dc=com") + + private val USER2 = + User.useIdForPassword(id = "user2", dn = "uid=user2,ou=People,dc=example,dc=com") + + private val USER3 = + User.useIdForPassword(id = "user3", dn = "cn=user3,ou=People,dc=example,dc=com") + + private val USER4 = + User.useIdForPassword(id = "user4", dn = "cn=user4,ou=People,dc=example,dc=com") + + private val ENGINEER_1 = User( + id = "engineer1", + dn = "sAMAccountName=engineer1,ou=Engineering,dc=ad,dc=example,dc=com", + password = "engineer1-password") + + private val ENGINEER_2 = User( + id = "engineer2", + dn = "sAMAccountName=engineer2,ou=Engineering,dc=ad,dc=example,dc=com", + password = "engineer2-password") + + private val MANAGER_1 = User( + id = "manager1", + dn = "sAMAccountName=manager1,ou=Management,dc=ad,dc=example,dc=com", + password = "manager1-password") + + private val MANAGER_2 = User( + id = "manager2", + dn = "sAMAccountName=manager2,ou=Management,dc=ad,dc=example,dc=com", + password = "manager2-password") + + private val ADMIN_1 = User( + id = "admin1", + dn = "sAMAccountName=admin1,ou=Administration,dc=ad,dc=example,dc=com", + password = "admin1-password") + + private var testCase: LdapAuthenticationTestCase = _ + + private def defaultBuilder = LdapAuthenticationTestCase.builder.ldapUrl(ldapUrl) + + override def beforeAll(): Unit = { + super.beforeAll() + ldapServer.add(new Entry( + "dn: dc=example,dc=com", + "dc: example", + "objectClass: top", + "objectClass: domain")) + + applyLDIF("ldap/example.com.ldif") + applyLDIF("ldap/microsoft.schema.ldif") + applyLDIF("ldap/ad.example.com.ldif") + } + + test("In-Memory LDAP server is started") { + assert(ldapServer.getListenPort > 0) + } + + test("UserBindPositiveWithShortname") { + testCase = defaultBuilder + .userDNPatterns("uid=%s,ou=People,dc=example,dc=com") + .groupDNPatterns("uid=%s,ou=Groups,dc=example,dc=com") + .build + testCase.assertAuthenticatePasses(USER1.credentialsWithId) + testCase.assertAuthenticatePasses(USER2.credentialsWithId) + } + + test("UserBindPositiveWithShortnameOldConfig") { + testCase = defaultBuilder + .baseDN("ou=People,dc=example,dc=com") + .build + testCase.assertAuthenticatePasses(USER1.credentialsWithId) + testCase.assertAuthenticatePasses(USER2.credentialsWithId) + } + + test("UserBindNegativeWithShortname") { + testCase = defaultBuilder + .userDNPatterns("uid=%s,ou=People,dc=example,dc=com") + .groupDNPatterns("uid=%s,ou=Groups,dc=example,dc=com") + .build + testCase.assertAuthenticateFailsUsingWrongPassword(USER1.credentialsWithId) + testCase.assertAuthenticateFailsUsingWrongPassword(USER2.credentialsWithId) + } + + test("UserBindNegativeWithShortnameOldConfig") { + testCase = defaultBuilder + .baseDN("ou=People,dc=example,dc=com") + .build + testCase.assertAuthenticateFailsUsingWrongPassword(USER1.credentialsWithId) + testCase.assertAuthenticateFails(USER1.dn, USER2.password) + testCase.assertAuthenticateFailsUsingWrongPassword(USER2.credentialsWithId) + } + + test("UserBindPositiveWithDN") { + testCase = defaultBuilder + .userDNPatterns("uid=%s,ou=People,dc=example,dc=com") + .groupDNPatterns("uid=%s,ou=Groups,dc=example,dc=com") + .build + testCase.assertAuthenticatePasses(USER1.credentialsWithDn) + testCase.assertAuthenticatePasses(USER2.credentialsWithDn) + } + + test("UserBindPositiveWithDNOldConfig") { + testCase = defaultBuilder + .baseDN("ou=People,dc=example,dc=com") + .build + testCase.assertAuthenticatePasses(USER1.credentialsWithDn) + testCase.assertAuthenticatePasses(USER2.credentialsWithDn) + } + + test("UserBindPositiveWithDNWrongOldConfig") { + testCase = defaultBuilder + .baseDN("ou=DummyPeople,dc=example,dc=com") + .build + testCase.assertAuthenticatePasses(USER1.credentialsWithDn) + testCase.assertAuthenticatePasses(USER2.credentialsWithDn) + } + + test("UserBindPositiveWithDNWrongConfig") { + testCase = defaultBuilder + .userDNPatterns("uid=%s,ou=DummyPeople,dc=example,dc=com") + .groupDNPatterns("uid=%s,ou=DummyGroups,dc=example,dc=com") + .build + testCase.assertAuthenticatePasses(USER1.credentialsWithDn) + testCase.assertAuthenticatePasses(USER2.credentialsWithDn) + } + + test("UserBindPositiveWithDNBlankConfig") { + testCase = defaultBuilder + .userDNPatterns(" ") + .groupDNPatterns(" ") + .build + testCase.assertAuthenticatePasses(USER1.credentialsWithDn) + testCase.assertAuthenticatePasses(USER2.credentialsWithDn) + } + + test("UserBindPositiveWithDNBlankOldConfig") { + testCase = defaultBuilder.baseDN("").build + testCase.assertAuthenticatePasses(USER1.credentialsWithDn) + testCase.assertAuthenticatePasses(USER2.credentialsWithDn) + } + + test("UserBindNegativeWithDN") { + testCase = defaultBuilder + .userDNPatterns("uid=%s,ou=People,dc=example,dc=com") + .groupDNPatterns("uid=%s,ou=Groups,dc=example,dc=com") + .build + testCase.assertAuthenticateFailsUsingWrongPassword(USER1.credentialsWithDn) + testCase.assertAuthenticateFails(USER1.dn, USER2.password) + testCase.assertAuthenticateFailsUsingWrongPassword(USER2.credentialsWithDn) + } + + test("UserBindNegativeWithDNOldConfig") { + testCase = defaultBuilder + .baseDN("ou=People,dc=example,dc=com") + .build + testCase.assertAuthenticateFailsUsingWrongPassword(USER1.credentialsWithDn) + testCase.assertAuthenticateFails(USER1.dn, USER2.password) + testCase.assertAuthenticateFailsUsingWrongPassword(USER2.credentialsWithDn) + } + + test("UserFilterPositive") { + testCase = defaultBuilder + .userDNPatterns("uid=%s,ou=People,dc=example,dc=com") + .userFilters(USER1.id) + .build + testCase.assertAuthenticatePasses(USER1.credentialsWithId) + testCase.assertAuthenticatePasses(USER1.credentialsWithDn) + testCase = defaultBuilder + .userDNPatterns("uid=%s,ou=People,dc=example,dc=com") + .userFilters(USER2.id) + .build + testCase.assertAuthenticatePasses(USER2.credentialsWithId) + testCase.assertAuthenticatePasses(USER2.credentialsWithDn) + testCase = defaultBuilder + .userDNPatterns("uid=%s,ou=People,dc=example,dc=com") + .userFilters(USER1.id, USER2.id) + .build + testCase.assertAuthenticatePasses(USER1.credentialsWithId) + testCase.assertAuthenticatePasses(USER1.credentialsWithDn) + testCase.assertAuthenticatePasses(USER2.credentialsWithId) + testCase.assertAuthenticatePasses(USER2.credentialsWithDn) + } + + test("UserFilterNegative") { + testCase = defaultBuilder + .userDNPatterns("uid=%s,ou=People,dc=example,dc=com") + .userFilters(USER2.id) + .build + testCase.assertAuthenticateFails(USER1.credentialsWithId) + testCase.assertAuthenticateFails(USER1.credentialsWithDn) + testCase = defaultBuilder + .userDNPatterns("uid=%s,ou=People,dc=example,dc=com") + .userFilters(USER1.id) + .build + testCase.assertAuthenticateFails(USER2.credentialsWithId) + testCase.assertAuthenticateFails(USER2.credentialsWithDn) + testCase = defaultBuilder + .userDNPatterns("uid=%s,ou=People,dc=example,dc=com") + .userFilters(USER3.id) + .build + testCase.assertAuthenticateFails(USER1.credentialsWithId) + testCase.assertAuthenticateFails(USER2.credentialsWithId) + } + + test("GroupFilterPositive") { + testCase = defaultBuilder + .userDNPatterns("uid=%s,ou=People,dc=example,dc=com") + .groupDNPatterns("uid=%s,ou=Groups,dc=example,dc=com") + .groupFilters(GROUP1_NAME, GROUP2_NAME) + .build + testCase.assertAuthenticatePasses(USER1.credentialsWithId) + testCase.assertAuthenticatePasses(USER1.credentialsWithDn) + testCase.assertAuthenticatePasses(USER2.credentialsWithId) + testCase.assertAuthenticatePasses(USER2.credentialsWithDn) + testCase = defaultBuilder + .userDNPatterns("uid=%s,ou=People,dc=example,dc=com") + .groupDNPatterns("uid=%s,ou=Groups,dc=example,dc=com") + .groupFilters(GROUP2_NAME) + .build + testCase.assertAuthenticatePasses(USER2.credentialsWithId) + testCase.assertAuthenticatePasses(USER2.credentialsWithDn) + } + + test("GroupFilterNegative") { + testCase = defaultBuilder + .userDNPatterns("uid=%s,ou=People,dc=example,dc=com") + .groupDNPatterns("uid=%s,ou=Groups,dc=example,dc=com") + .groupFilters(GROUP2_NAME) + .build + testCase.assertAuthenticateFails(USER1.credentialsWithId) + testCase.assertAuthenticateFails(USER1.credentialsWithDn) + testCase = defaultBuilder + .userDNPatterns("uid=%s,ou=People,dc=example,dc=com") + .groupDNPatterns("uid=%s,ou=Groups,dc=example,dc=com") + .groupFilters(GROUP1_NAME) + .build + testCase.assertAuthenticateFails(USER2.credentialsWithId) + testCase.assertAuthenticateFails(USER2.credentialsWithDn) + } + + test("UserAndGroupFilterPositive") { + testCase = defaultBuilder + .userDNPatterns("uid=%s,ou=People,dc=example,dc=com") + .groupDNPatterns("uid=%s,ou=Groups,dc=example,dc=com") + .userFilters(USER1.id, USER2.id) + .groupFilters(GROUP1_NAME, GROUP2_NAME) + .build + testCase.assertAuthenticatePasses(USER1.credentialsWithId) + testCase.assertAuthenticatePasses(USER1.credentialsWithDn) + testCase.assertAuthenticatePasses(USER2.credentialsWithId) + testCase.assertAuthenticatePasses(USER2.credentialsWithDn) + } + + test("UserAndGroupFilterNegative") { + testCase = defaultBuilder + .userDNPatterns("uid=%s,ou=People,dc=example,dc=com") + .groupDNPatterns("uid=%s,ou=Groups,dc=example,dc=com") + .userFilters(USER1.id, USER2.id) + .groupFilters(GROUP3_NAME, GROUP3_NAME) + .build + testCase.assertAuthenticateFails(USER2.credentialsWithDn) + testCase.assertAuthenticateFails(USER2.credentialsWithId) + testCase.assertAuthenticateFails(USER3.credentialsWithDn) + testCase.assertAuthenticateFails(USER3.credentialsWithId) + } + + test("CustomQueryPositive") { + testCase = defaultBuilder + .baseDN("ou=People,dc=example,dc=com") + .userDNPatterns("cn=%s,ou=People,dc=example,dc=com", "uid=%s,ou=People,dc=example,dc=com") + .groupDNPatterns("cn=%s,ou=People,dc=example,dc=com") + .customQuery(String.format("(&(objectClass=person)(|(uid=%s)(uid=%s)))", USER1.id, USER4.id)) + .build + testCase.assertAuthenticatePasses(USER1.credentialsWithId) + testCase.assertAuthenticatePasses(USER1.credentialsWithDn) + testCase.assertAuthenticatePasses(USER4.credentialsWithId) + testCase.assertAuthenticatePasses(USER4.credentialsWithDn) + testCase = defaultBuilder + .baseDN("ou=People,dc=example,dc=com") + .customQuery("(&(objectClass=person)(uid=%s))") + .build + testCase.assertAuthenticatePasses(USER1.credentialsWithId) + testCase.assertAuthenticatePasses(USER2.credentialsWithId) + } + + test("CustomQueryNegative") { + testCase = defaultBuilder + .baseDN("ou=People,dc=example,dc=com") + .customQuery("(&(objectClass=person)(cn=%s))") + .build + testCase.assertAuthenticateFails(USER2.credentialsWithDn) + testCase.assertAuthenticateFails(USER2.credentialsWithId) + } + + /** + * Test to test the LDAP Atn to use a custom LDAP query that returns + * a) A set of group DNs + * b) A combination of group(s) DN and user DN + * LDAP atn is expected to extract the members of the group using the attribute value for + * `kyuubi.authentication.ldap.userMembershipKey` + */ + test("CustomQueryWithGroupsPositive") { + testCase = defaultBuilder + .baseDN("dc=example,dc=com") + .userDNPatterns("cn=%s,ou=People,dc=example,dc=com", "uid=%s,ou=People,dc=example,dc=com") + .customQuery(s"(&(objectClass=groupOfNames)(|(cn=$GROUP1_NAME)(cn=$GROUP2_NAME)))") + .build + testCase.assertAuthenticatePasses(USER1.credentialsWithId) + testCase.assertAuthenticatePasses(USER1.credentialsWithDn) + testCase.assertAuthenticatePasses(USER2.credentialsWithId) + testCase.assertAuthenticatePasses(USER2.credentialsWithDn) + // the following test uses a query that returns a group and a user entry. + // the ldap atn should use the groupMembershipKey to identify the users for the returned group + // and the authentication should succeed for the users of that group as well as the lone user4 + // in this case + testCase = defaultBuilder + .baseDN("dc=example,dc=com") + .userDNPatterns("cn=%s,ou=People,dc=example,dc=com", "uid=%s,ou=People,dc=example,dc=com") + .customQuery( + s"(|(&(objectClass=groupOfNames)(cn=$GROUP1_NAME))(&(objectClass=person)(sn=${USER4.id})))") + .build + testCase.assertAuthenticatePasses(USER1.credentialsWithId) + testCase.assertAuthenticatePasses(USER1.credentialsWithDn) + testCase.assertAuthenticatePasses(USER4.credentialsWithId) + testCase.assertAuthenticatePasses(USER4.credentialsWithDn) + testCase = defaultBuilder + .baseDN("dc=example,dc=com") + .userDNPatterns("cn=%s,ou=People,dc=example,dc=com", "uid=%s,ou=People,dc=example,dc=com") + .groupMembershipKey("uniqueMember") + .customQuery(s"(&(objectClass=groupOfUniqueNames)(cn=$GROUP4_NAME))") + .build + testCase.assertAuthenticatePasses(USER4.credentialsWithId) + testCase.assertAuthenticatePasses(USER4.credentialsWithDn) + } + + test("CustomQueryWithGroupsNegative") { + testCase = defaultBuilder + .baseDN("dc=example,dc=com") + .userDNPatterns("cn=%s,ou=People,dc=example,dc=com", "uid=%s,ou=People,dc=example,dc=com") + .customQuery(s"(&(objectClass=groupOfNames)(|(cn=$GROUP1_NAME)(cn=$GROUP2_NAME)))") + .build + testCase.assertAuthenticateFails(USER3.credentialsWithDn) + testCase.assertAuthenticateFails(USER3.credentialsWithId) + } + + test("GroupFilterPositiveWithCustomGUID") { + testCase = defaultBuilder + .userDNPatterns("cn=%s,ou=People,dc=example,dc=com") + .groupDNPatterns("cn=%s,ou=Groups,dc=example,dc=com") + .groupFilters(GROUP3_NAME) + .guidKey("cn") + .build + testCase.assertAuthenticatePasses(USER3.credentialsWithId) + testCase.assertAuthenticatePasses(USER3.credentialsWithDn) + } + + test("GroupFilterPositiveWithCustomAttributes") { + testCase = defaultBuilder + .userDNPatterns("cn=%s,ou=People,dc=example,dc=com") + .groupDNPatterns("cn=%s,ou=Groups,dc=example,dc=com") + .groupFilters(GROUP4_NAME) + .guidKey("cn") + .groupMembershipKey("uniqueMember") + .groupClassKey("groupOfUniqueNames") + .build + testCase.assertAuthenticatePasses(USER4.credentialsWithId) + testCase.assertAuthenticatePasses(USER4.credentialsWithDn) + } + + test("DirectUserMembershipGroupFilterPositive") { + testCase = defaultBuilder + .userDNPatterns( + "sAMAccountName=%s,ou=Engineering,dc=ad,dc=example,dc=com", + "sAMAccountName=%s,ou=Management,dc=ad,dc=example,dc=com") + .groupDNPatterns( + "sAMAccountName=%s,ou=Teams,dc=ad,dc=example,dc=com", + "sAMAccountName=%s,ou=Resources,dc=ad,dc=example,dc=com") + .groupFilters(GROUP_TEAM1_NAME, GROUP_TEAM2_NAME, GROUP_RESOURCE1_NAME, GROUP_RESOURCE2_NAME) + .guidKey("sAMAccountName") + .userMembershipKey("memberOf") + .build + testCase.assertAuthenticatePasses(ENGINEER_1.credentialsWithId) + testCase.assertAuthenticatePasses(ENGINEER_2.credentialsWithId) + testCase.assertAuthenticatePasses(MANAGER_1.credentialsWithId) + testCase.assertAuthenticatePasses(MANAGER_2.credentialsWithId) + } + + test("DirectUserMembershipGroupFilterNegative") { + testCase = defaultBuilder + .userDNPatterns( + "sAMAccountName=%s,ou=Engineering,dc=ad,dc=example,dc=com", + "sAMAccountName=%s,ou=Management,dc=ad,dc=example,dc=com") + .groupDNPatterns("cn=%s,ou=Teams,dc=ad,dc=example,dc=com") + .groupFilters(GROUP_TEAM1_NAME) + .guidKey("sAMAccountName") + .userMembershipKey("memberOf") + .build + testCase.assertAuthenticateFails(ENGINEER_2.credentialsWithId) + testCase.assertAuthenticateFails(MANAGER_2.credentialsWithId) + } + + test("DirectUserMembershipGroupFilterNegativeWithoutUserBases") { + testCase = defaultBuilder + .groupDNPatterns("cn=%s,ou=Teams,dc=ad,dc=example,dc=com") + .groupFilters(GROUP_TEAM1_NAME) + .guidKey("sAMAccountName") + .userMembershipKey("memberOf") + .build + testCase.assertAuthenticateFails(ENGINEER_1.credentialsWithId) + testCase.assertAuthenticateFails(ENGINEER_2.credentialsWithId) + testCase.assertAuthenticateFails(MANAGER_1.credentialsWithId) + testCase.assertAuthenticateFails(MANAGER_2.credentialsWithId) + } + + test("DirectUserMembershipGroupFilterWithDNCredentials") { + testCase = defaultBuilder + .userDNPatterns("sAMAccountName=%s,ou=Engineering,dc=ad,dc=example,dc=com") + .groupDNPatterns("cn=%s,ou=Teams,dc=ad,dc=example,dc=com") + .groupFilters(GROUP_TEAM1_NAME) + .guidKey("sAMAccountName") + .userMembershipKey("memberOf") + .build + testCase.assertAuthenticatePasses(ENGINEER_1.credentialsWithDn) + testCase.assertAuthenticateFails(MANAGER_1.credentialsWithDn) + } + + test("DirectUserMembershipGroupFilterWithDifferentGroupClassKey") { + testCase = defaultBuilder + .userDNPatterns("sAMAccountName=%s,ou=Administration,dc=ad,dc=example,dc=com") + .groupDNPatterns("cn=%s,ou=Administration,dc=ad,dc=example,dc=com") + .groupFilters(GROUP_ADMINS_NAME).guidKey("sAMAccountName") + .userMembershipKey("memberOf") + .groupClassKey("groupOfUniqueNames") + .build + testCase.assertAuthenticatePasses(ADMIN_1.credentialsWithId) + testCase.assertAuthenticateFails(ENGINEER_1.credentialsWithId) + testCase.assertAuthenticateFails(MANAGER_1.credentialsWithDn) + } + + test("DirectUserMembershipGroupFilterNegativeWithWrongGroupClassKey") { + testCase = defaultBuilder + .userDNPatterns("sAMAccountName=%s,ou=Administration,dc=ad,dc=example,dc=com") + .groupDNPatterns("cn=%s,ou=Administration,dc=ad,dc=example,dc=com") + .groupFilters(GROUP_ADMINS_NAME).guidKey("sAMAccountName") + .userMembershipKey("memberOf") + .groupClassKey("wrongClass") + .build + testCase.assertAuthenticateFails(ADMIN_1.credentialsWithId) + testCase.assertAuthenticateFails(ENGINEER_1.credentialsWithId) + testCase.assertAuthenticateFails(MANAGER_1.credentialsWithDn) + } +} diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/LdapAuthenticationProviderImplSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/LdapAuthenticationProviderImplSuite.scala index 639411628..f10bf7ce2 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/LdapAuthenticationProviderImplSuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/LdapAuthenticationProviderImplSuite.scala @@ -17,63 +17,384 @@ package org.apache.kyuubi.service.authentication -import javax.naming.CommunicationException +import javax.naming.NamingException import javax.security.sasl.AuthenticationException +import org.mockito.ArgumentMatchers.{any, anyString, eq => mockEq, isA} +import org.mockito.Mockito._ +import org.scalatestplus.mockito.MockitoSugar.mock + import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.config.KyuubiConf._ +import org.apache.kyuubi.service.authentication.ldap.{DirSearch, DirSearchFactory, LdapSearchFactory} +import org.apache.kyuubi.service.authentication.ldap.LdapUtils.getUserName class LdapAuthenticationProviderImplSuite extends WithLdapServer { - override protected val ldapUser: String = "kentyao" - override protected val ldapUserPasswd: String = "kentyao" - private val conf = new KyuubiConf() + private var conf: KyuubiConf = _ + private var factory: DirSearchFactory = _ + private var search: DirSearch = _ + private var auth: LdapAuthenticationProviderImpl = _ - override def beforeAll(): Unit = { - super.beforeAll() + override protected def beforeEach(): Unit = { + super.beforeEach() + conf = new KyuubiConf() conf.set(AUTHENTICATION_LDAP_URL, ldapUrl) + factory = mock[DirSearchFactory] + search = mock[DirSearch] + when(factory.getInstance(any(classOf[KyuubiConf]), anyString, anyString)) + .thenReturn(search) + } + + test("authenticateGivenBlankOrNullPassword") { + Seq("", "\u0000", null).foreach { pwd => + auth = new LdapAuthenticationProviderImpl(conf, new LdapSearchFactory) + val thrown = intercept[AuthenticationException] { + auth.authenticate("user", pwd) + } + assert(thrown.getMessage.contains("is null or contains blank space")) + } + } + + test("AuthenticateNoUserOrGroupFilter") { + conf.set( + AUTHENTICATION_LDAP_USER_DN_PATTERN, + "cn=%s,ou=Users,dc=mycorp,dc=com:cn=%s,ou=PowerUsers,dc=mycorp,dc=com") + val factory = mock[DirSearchFactory] + lenient + .when(search.findUserDn("user1")) + .thenReturn("cn=user1,ou=PowerUsers,dc=mycorp,dc=com") + when(factory.getInstance(conf, "cn=user1,ou=PowerUsers,dc=mycorp,dc=com", "Blah")) + .thenReturn(search) + when(factory.getInstance(conf, "cn=user1,ou=Users,dc=mycorp,dc=com", "Blah")) + .thenThrow(classOf[AuthenticationException]) + + auth = new LdapAuthenticationProviderImpl(conf, factory) + auth.authenticate("user1", "Blah") + + verify(factory, times(2)).getInstance(isA(classOf[KyuubiConf]), anyString, mockEq("Blah")) + verify(search, atLeastOnce).close() + } + + test("AuthenticateWhenUserFilterPasses") { + conf.set(AUTHENTICATION_LDAP_USER_FILTER.key, "user1,user2") + + when(search.findUserDn("user1")).thenReturn("cn=user1,ou=PowerUsers,dc=mycorp,dc=com") + when(search.findUserDn("user2")).thenReturn("cn=user2,ou=PowerUsers,dc=mycorp,dc=com") + + authenticateUserAndCheckSearchIsClosed("user1") + authenticateUserAndCheckSearchIsClosed("user2") + } + + test("AuthenticateWhenLoginWithDomainAndUserFilterPasses") { + conf.set(AUTHENTICATION_LDAP_USER_FILTER.key, "user1") + + when(search.findUserDn("user1")).thenReturn("cn=user1,ou=PowerUsers,dc=mycorp,dc=com") + + authenticateUserAndCheckSearchIsClosed("user1@mydomain.com") + } + + test("AuthenticateWhenLoginWithDnAndUserFilterPasses") { + conf.set(AUTHENTICATION_LDAP_USER_FILTER.key, "user1") + + when(search.findUserDn("cn=user1,ou=PowerUsers,dc=mycorp,dc=com")) + .thenReturn("cn=user1,ou=PowerUsers,dc=mycorp,dc=com") + + authenticateUserAndCheckSearchIsClosed("cn=user1,ou=PowerUsers,dc=mycorp,dc=com") + } + + test("AuthenticateWhenUserSearchFails") { + conf.set(AUTHENTICATION_LDAP_USER_FILTER.key, "user1,user2") + intercept[AuthenticationException] { + when(search.findUserDn("user1")).thenReturn(null) + authenticateUserAndCheckSearchIsClosed("user1") + } + } + + test("AuthenticateWhenUserFilterFails") { + conf.set(AUTHENTICATION_LDAP_USER_FILTER.key, "user1,user2") + intercept[AuthenticationException] { + when(search.findUserDn("user3")).thenReturn("cn=user3,ou=PowerUsers,dc=mycorp,dc=com") + authenticateUserAndCheckSearchIsClosed("user3") + } + } + + test("AuthenticateWhenGroupMembershipKeyFilterPasses") { + conf.set(AUTHENTICATION_LDAP_GROUP_FILTER.key, "group1,group2") + + when(search.findUserDn("user1")).thenReturn("cn=user1,ou=PowerUsers,dc=mycorp,dc=com") + when(search.findUserDn("user2")).thenReturn("cn=user2,ou=PowerUsers,dc=mycorp,dc=com") + + when(search.findGroupsForUser("cn=user1,ou=PowerUsers,dc=mycorp,dc=com")) + .thenReturn(Array( + "cn=testGroup,ou=Groups,dc=mycorp,dc=com", + "cn=group1,ou=Groups,dc=mycorp,dc=com")) + when(search.findGroupsForUser("cn=user2,ou=PowerUsers,dc=mycorp,dc=com")) + .thenReturn(Array( + "cn=testGroup,ou=Groups,dc=mycorp,dc=com", + "cn=group2,ou=Groups,dc=mycorp,dc=com")) + + authenticateUserAndCheckSearchIsClosed("user1") + authenticateUserAndCheckSearchIsClosed("user2") + } + + test("AuthenticateWhenUserAndGroupMembershipKeyFiltersPass") { + conf.set(AUTHENTICATION_LDAP_GROUP_FILTER.key, "group1,group2") + conf.set(AUTHENTICATION_LDAP_USER_FILTER.key, "user1,user2") + + when(search.findUserDn("user1")).thenReturn("cn=user1,ou=PowerUsers,dc=mycorp,dc=com") + when(search.findUserDn("user2")).thenReturn("cn=user2,ou=PowerUsers,dc=mycorp,dc=com") + + when(search.findGroupsForUser("cn=user1,ou=PowerUsers,dc=mycorp,dc=com")) + .thenReturn(Array( + "cn=testGroup,ou=Groups,dc=mycorp,dc=com", + "cn=group1,ou=Groups,dc=mycorp,dc=com")) + when(search.findGroupsForUser("cn=user2,ou=PowerUsers,dc=mycorp,dc=com")) + .thenReturn(Array( + "cn=testGroup,ou=Groups,dc=mycorp,dc=com", + "cn=group2,ou=Groups,dc=mycorp,dc=com")) + + authenticateUserAndCheckSearchIsClosed("user1") + authenticateUserAndCheckSearchIsClosed("user2") + } + + test("AuthenticateWhenUserFilterPassesAndGroupMembershipKeyFilterFails") { + conf.set(AUTHENTICATION_LDAP_GROUP_FILTER.key, "group1,group2") + conf.set(AUTHENTICATION_LDAP_USER_FILTER.key, "user1,user2") + intercept[AuthenticationException] { + when(search.findUserDn("user1")).thenReturn("cn=user1,ou=PowerUsers,dc=mycorp,dc=com") + when(search.findGroupsForUser("cn=user1,ou=PowerUsers,dc=mycorp,dc=com")) + .thenReturn(Array( + "cn=testGroup,ou=Groups,dc=mycorp,dc=com", + "cn=OtherGroup,ou=Groups,dc=mycorp,dc=com")) + authenticateUserAndCheckSearchIsClosed("user1") + } + } + + test("AuthenticateWhenUserFilterFailsAndGroupMembershipKeyFilterPasses") { + conf.set(AUTHENTICATION_LDAP_GROUP_FILTER.key, "group3") + conf.set(AUTHENTICATION_LDAP_USER_FILTER.key, "user1,user2") + intercept[AuthenticationException] { + when(search.findUserDn("user3")).thenReturn("cn=user3,ou=PowerUsers,dc=mycorp,dc=com") + lenient.when(search.findGroupsForUser("cn=user3,ou=PowerUsers,dc=mycorp,dc=com")) + .thenReturn(Array( + "cn=testGroup,ou=Groups,dc=mycorp,dc=com", + "cn=group3,ou=Groups,dc=mycorp,dc=com")) + authenticateUserAndCheckSearchIsClosed("user3") + } + } + + test("AuthenticateWhenCustomQueryFilterPasses") { + conf.set(AUTHENTICATION_LDAP_BASE_DN, "dc=mycorp,dc=com") + conf.set( + AUTHENTICATION_LDAP_CUSTOM_LDAP_QUERY, + "(&(objectClass=person)(|(memberOf=CN=Domain Admins,CN=Users,DC=apache,DC=org)" + + "(memberOf=CN=Administrators,CN=Builtin,DC=apache,DC=org)))") + + when(search.executeCustomQuery(anyString)) + .thenReturn(Array( + "cn=user1,ou=PowerUsers,dc=mycorp,dc=com", + "cn=user2,ou=PowerUsers,dc=mycorp,dc=com")) + + authenticateUserAndCheckSearchIsClosed("user1") + } + + test("AuthenticateWhenCustomQueryFilterFailsAndUserFilterPasses") { + conf.set(AUTHENTICATION_LDAP_BASE_DN, "dc=mycorp,dc=com") + conf.set( + AUTHENTICATION_LDAP_CUSTOM_LDAP_QUERY, + "(&(objectClass=person)(|(memberOf=CN=Domain Admins,CN=Users,DC=apache,DC=org)" + + "(memberOf=CN=Administrators,CN=Builtin,DC=apache,DC=org)))") + conf.set(AUTHENTICATION_LDAP_USER_FILTER.key, "user3") + intercept[AuthenticationException] { + lenient.when(search.findUserDn("user3")).thenReturn("cn=user3,ou=PowerUsers,dc=mycorp,dc=com") + when(search.executeCustomQuery(anyString)) + .thenReturn(Array( + "cn=user1,ou=PowerUsers,dc=mycorp,dc=com", + "cn=user2,ou=PowerUsers,dc=mycorp,dc=com")) + authenticateUserAndCheckSearchIsClosed("user3") + } + } + + test("AuthenticateWhenUserMembershipKeyFilterPasses") { + conf.set(AUTHENTICATION_LDAP_BASE_DN, "dc=mycorp,dc=com") + conf.set(AUTHENTICATION_LDAP_GROUP_FILTER.key, "HIVE-USERS") + conf.set(AUTHENTICATION_LDAP_USER_MEMBERSHIP_KEY, "memberOf") + + when(search.findUserDn("user1")).thenReturn("cn=user1,ou=PowerUsers,dc=mycorp,dc=com") + + val groupDn = "cn=HIVE-USERS,ou=Groups,dc=mycorp,dc=com" + when(search.findGroupDn("HIVE-USERS")).thenReturn(groupDn) + when(search.isUserMemberOfGroup("user1", groupDn)).thenReturn(true) + + auth = new LdapAuthenticationProviderImpl(conf, factory) + auth.authenticate("user1", "Blah") + + verify(factory, times(1)).getInstance(isA(classOf[KyuubiConf]), anyString, mockEq("Blah")) + verify(search, times(1)).findGroupDn(anyString) + verify(search, times(1)).isUserMemberOfGroup(anyString, anyString) + verify(search, atLeastOnce).close() + } + + test("AuthenticateWhenUserMembershipKeyFilterFails") { + conf.set(AUTHENTICATION_LDAP_BASE_DN, "dc=mycorp,dc=com") + conf.set(AUTHENTICATION_LDAP_GROUP_FILTER.key, "HIVE-USERS") + conf.set(AUTHENTICATION_LDAP_USER_MEMBERSHIP_KEY, "memberOf") + intercept[AuthenticationException] { + when(search.findUserDn("user1")).thenReturn("cn=user1,ou=PowerUsers,dc=mycorp,dc=com") + + val groupDn = "cn=HIVE-USERS,ou=Groups,dc=mycorp,dc=com" + when(search.findGroupDn("HIVE-USERS")).thenReturn(groupDn) + when(search.isUserMemberOfGroup("user1", groupDn)).thenReturn(false) + + auth = new LdapAuthenticationProviderImpl(conf, factory) + auth.authenticate("user1", "Blah") + } + } + + test("AuthenticateWhenUserMembershipKeyFilter2x2PatternsPasses") { + conf.set(AUTHENTICATION_LDAP_GROUP_FILTER.key, "HIVE-USERS1,HIVE-USERS2") + conf.set(AUTHENTICATION_LDAP_GROUP_DN_PATTERN, "cn=%s,ou=Groups,ou=branch1,dc=mycorp,dc=com") + conf.set(AUTHENTICATION_LDAP_USER_DN_PATTERN, "cn=%s,ou=Userss,ou=branch1,dc=mycorp,dc=com") + conf.set(AUTHENTICATION_LDAP_USER_MEMBERSHIP_KEY, "memberOf") + + when(search.findUserDn("user1")).thenReturn("cn=user1,ou=PowerUsers,dc=mycorp,dc=com") + + when(search.findGroupDn("HIVE-USERS1")) + .thenReturn("cn=HIVE-USERS1,ou=Groups,ou=branch1,dc=mycorp,dc=com") + when(search.findGroupDn("HIVE-USERS2")) + .thenReturn("cn=HIVE-USERS2,ou=Groups,ou=branch1,dc=mycorp,dc=com") + + when(search.isUserMemberOfGroup( + "user1", + "cn=HIVE-USERS1,ou=Groups,ou=branch1,dc=mycorp,dc=com")) + .thenThrow(classOf[NamingException]) + when(search.isUserMemberOfGroup( + "user1", + "cn=HIVE-USERS2,ou=Groups,ou=branch1,dc=mycorp,dc=com")) + .thenReturn(true) + + auth = new LdapAuthenticationProviderImpl(conf, factory) + auth.authenticate("user1", "Blah") + + verify(factory, times(1)).getInstance(isA(classOf[KyuubiConf]), anyString, mockEq("Blah")) + verify(search, times(2)).findGroupDn(anyString) + verify(search, times(2)).isUserMemberOfGroup(anyString, anyString) + verify(search, atLeastOnce).close() } - override def afterAll(): Unit = { - super.afterAll() + // Kyuubi does not implement it + // test("AuthenticateWithBindInCredentialFilePasses") + // test("testAuthenticateWithBindInMissingCredentialFilePasses") + + test("AuthenticateWithBindUserPasses") { + val bindUser = "cn=BindUser,ou=Users,ou=branch1,dc=mycorp,dc=com" + val bindPass = "Blah" + val authFullUser = "cn=user1,ou=Users,ou=branch1,dc=mycorp,dc=com" + val authUser = "user1" + val authPass = "Blah2" + conf.set(AUTHENTICATION_LDAP_BIND_USER, bindUser) + conf.set(AUTHENTICATION_LDAP_BIND_PASSWORD, bindPass) + + when(search.findUserDn(mockEq(authUser))).thenReturn(authFullUser) + + auth = new LdapAuthenticationProviderImpl(conf, factory) + auth.authenticate(authUser, authPass) + + verify(factory, times(1)).getInstance( + isA(classOf[KyuubiConf]), + mockEq(bindUser), + mockEq(bindPass)) + verify(factory, times(1)).getInstance( + isA(classOf[KyuubiConf]), + mockEq(authFullUser), + mockEq(authPass)) + verify(search, times(1)).findUserDn(mockEq(authUser)) } - test("ldap server is started") { - assert(ldapServer.getListenPort > 0) + test("AuthenticateWithBindDomainUserPasses") { + val bindUser = "cn=BindUser,ou=Users,ou=branch1,dc=mycorp,dc=com" + val bindPass = "Blah" + val authFullUser = "cn=user1,ou=Users,ou=branch1,dc=mycorp,dc=com" + val authUser = "user1@mydomain.com" + val authPass = "Blah2" + conf.set(AUTHENTICATION_LDAP_BIND_USER, bindUser) + conf.set(AUTHENTICATION_LDAP_BIND_PASSWORD, bindPass) + + val username = getUserName(authUser) + when(search.findUserDn(mockEq(username))).thenReturn(authFullUser) + + auth = new LdapAuthenticationProviderImpl(conf, factory) + auth.authenticate(authUser, authPass) + + verify(factory, times(1)).getInstance( + isA(classOf[KyuubiConf]), + mockEq(bindUser), + mockEq(bindPass)) + verify(factory, times(1)).getInstance( + isA(classOf[KyuubiConf]), + mockEq(authFullUser), + mockEq(authPass)) + verify(search, times(1)).findUserDn(mockEq(username)) } - test("authenticate tests") { - val providerImpl = new LdapAuthenticationProviderImpl(conf) - val e1 = intercept[AuthenticationException](providerImpl.authenticate("", "")) - assert(e1.getMessage.contains("user is null")) - val e2 = intercept[AuthenticationException](providerImpl.authenticate("kyuubi", "")) - assert(e2.getMessage.contains("password is null")) + test("AuthenticateWithBindUserFailsOnAuthentication") { + val bindUser = "cn=BindUser,ou=Users,ou=branch1,dc=mycorp,dc=com" + val bindPass = "Blah" + val authFullUser = "cn=user1,ou=Users,ou=branch1,dc=mycorp,dc=com" + val authUser = "user1" + val authPass = "Blah2" + conf.set(AUTHENTICATION_LDAP_BIND_USER, bindUser) + conf.set(AUTHENTICATION_LDAP_BIND_PASSWORD, bindPass) - val user = "uid=kentyao,ou=users" - providerImpl.authenticate(user, "kentyao") - val e3 = intercept[AuthenticationException]( - providerImpl.authenticate(user, "kent")) - assert(e3.getMessage.contains(user)) - assert(e3.getCause.isInstanceOf[javax.naming.AuthenticationException]) + intercept[AuthenticationException] { + when( + factory.getInstance( + any(classOf[KyuubiConf]), + mockEq(authFullUser), + mockEq(authPass))).thenThrow(classOf[AuthenticationException]) + when(search.findUserDn(mockEq(authUser))).thenReturn(authFullUser) - conf.set(AUTHENTICATION_LDAP_BASEDN, ldapBaseDn) - val providerImpl2 = new LdapAuthenticationProviderImpl(conf) - providerImpl2.authenticate("kentyao", "kentyao") + auth = new LdapAuthenticationProviderImpl(conf, factory) + auth.authenticate(authUser, authPass) + } + } - val e4 = intercept[AuthenticationException]( - providerImpl.authenticate("kentyao", "kent")) - assert(e4.getMessage.contains(user)) + test("AuthenticateWithBindUserFailsOnGettingDn") { + val bindUser = "cn=BindUser,ou=Users,ou=branch1,dc=mycorp,dc=com" + val bindPass = "Blah" + val authUser = "user1" + val authPass = "Blah2" + conf.set(AUTHENTICATION_LDAP_BIND_USER, bindUser) + conf.set(AUTHENTICATION_LDAP_BIND_PASSWORD, bindPass) - conf.unset(AUTHENTICATION_LDAP_URL) - val providerImpl3 = new LdapAuthenticationProviderImpl(conf) - val e5 = intercept[AuthenticationException]( - providerImpl3.authenticate("kentyao", "kentyao")) + intercept[AuthenticationException] { + when(search.findUserDn(mockEq(authUser))).thenThrow(classOf[NamingException]) + auth = new LdapAuthenticationProviderImpl(conf, factory) + auth.authenticate(authUser, authPass) + } + } + + test("AuthenticateWithBindUserFailsOnBinding") { + val bindUser = "cn=BindUser,ou=Users,ou=branch1,dc=mycorp,dc=com" + val bindPass = "Blah" + val authUser = "user1" + val authPass = "Blah2" + conf.set(AUTHENTICATION_LDAP_BIND_USER, bindUser) + conf.set(AUTHENTICATION_LDAP_BIND_PASSWORD, bindPass) - assert(e5.getMessage.contains(user)) - assert(e5.getCause.isInstanceOf[CommunicationException]) + intercept[AuthenticationException] { + when(factory.getInstance(any(classOf[KyuubiConf]), mockEq(bindUser), mockEq(bindPass))) + .thenThrow(classOf[AuthenticationException]) + + auth = new LdapAuthenticationProviderImpl(conf, factory) + auth.authenticate(authUser, authPass) + } + } - conf.set(AUTHENTICATION_LDAP_DOMAIN, "kyuubi.com") - val providerImpl4 = new LdapAuthenticationProviderImpl(conf) - intercept[AuthenticationException](providerImpl4.authenticate("kentyao", "kentyao")) + private def authenticateUserAndCheckSearchIsClosed(user: String): Unit = { + auth = new LdapAuthenticationProviderImpl(conf, factory) + try auth.authenticate(user, "password doesn't matter") + finally verify(search, atLeastOnce).close() } } diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/PlainSASLHelperSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/PlainSASLHelperSuite.scala index 94a61f693..d4290a2c6 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/PlainSASLHelperSuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/PlainSASLHelperSuite.scala @@ -23,9 +23,9 @@ import org.apache.thrift.transport.{TSaslServerTransport, TSocket} import org.apache.kyuubi.{KYUUBI_VERSION, KyuubiFunSuite} import org.apache.kyuubi.config.KyuubiConf -import org.apache.kyuubi.engine.SemanticVersion import org.apache.kyuubi.service.{NoopTBinaryFrontendServer, TBinaryFrontendService} import org.apache.kyuubi.service.authentication.PlainSASLServer.SaslPlainProvider +import org.apache.kyuubi.util.SemanticVersion class PlainSASLHelperSuite extends KyuubiFunSuite { @@ -62,10 +62,6 @@ class PlainSASLHelperSuite extends KyuubiFunSuite { val saslPlainProvider = new SaslPlainProvider() assert(saslPlainProvider.containsKey("SaslServerFactory.PLAIN")) assert(saslPlainProvider.getName === "KyuubiSaslPlain") - val version: Double = { - val ver = SemanticVersion(KYUUBI_VERSION) - ver.majorVersion + ver.minorVersion.toDouble / 10 - } - assert(saslPlainProvider.getVersion === version) + assertResult(saslPlainProvider.getVersion)(SemanticVersion(KYUUBI_VERSION).toDouble) } } diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/PlainSASLServerSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/PlainSASLServerSuite.scala index 78fe3ef7a..a7f4b9535 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/PlainSASLServerSuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/PlainSASLServerSuite.scala @@ -79,9 +79,7 @@ class PlainSASLServerSuite extends KyuubiFunSuite { "NONE", "KYUUBI", map, - new CallbackHandler { - override def handle(callbacks: Array[Callback]): Unit = {} - }) + _ => {}) val e6 = intercept[SaslException](server2.evaluateResponse(res4.map(_.toByte))) assert(e6.getMessage === "Error validating the login") assert(e6.getCause.getMessage === "Authentication failed") diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/SaslQOPSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/SaslQOPSuite.scala index c48f12aa7..6cf2793d2 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/SaslQOPSuite.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/SaslQOPSuite.scala @@ -34,7 +34,7 @@ class SaslQOPSuite extends KyuubiFunSuite { val e = intercept[IllegalArgumentException](conf.get(SASL_QOP)) assert(e.getMessage === "The value of kyuubi.authentication.sasl.qop should be one of" + - " auth, auth-conf, auth-int, but was abc") + " auth, auth-int, auth-conf, but was abc") } } diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/WithLdapServer.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/WithLdapServer.scala index 0bb38684e..b31a06f20 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/WithLdapServer.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/WithLdapServer.scala @@ -17,20 +17,36 @@ package org.apache.kyuubi.service.authentication +import scala.util.Random + import com.unboundid.ldap.listener.{InMemoryDirectoryServer, InMemoryDirectoryServerConfig} +import com.unboundid.ldif.LDIFReader import org.apache.kyuubi.{KyuubiFunSuite, Utils} trait WithLdapServer extends KyuubiFunSuite { protected var ldapServer: InMemoryDirectoryServer = _ - protected val ldapBaseDn = "ou=users" - protected val ldapUser = Utils.currentUser - protected val ldapUserPasswd = "ldapPassword" + protected val ldapBaseDn: Array[String] = Array("ou=users") + protected val ldapUser: String = Utils.currentUser + protected val ldapUserPasswd: String = Random.alphanumeric.take(16).mkString protected def ldapUrl = s"ldap://localhost:${ldapServer.getListenPort}" + /** + * Apply LDIF files + * @param resource the LDIF file under classpath + */ + def applyLDIF(resource: String): Unit = { + ldapServer.applyChangesFromLDIF( + new LDIFReader(Utils.getContextOrKyuubiClassLoader.getResource(resource).openStream())) + } + override def beforeAll(): Unit = { - val config = new InMemoryDirectoryServerConfig(ldapBaseDn) + val config = new InMemoryDirectoryServerConfig(ldapBaseDn: _*) + // disable the schema so that we can apply LDIF which contains Microsoft's Active Directory + // specific definitions. + // https://myshittycode.com/2017/03/28/ + config.setSchema(null) config.addAdditionalBindCredentials(s"uid=$ldapUser,ou=users", ldapUserPasswd) ldapServer = new InMemoryDirectoryServer(config) ldapServer.startListening() diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/ldap/ChainFilterSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/ldap/ChainFilterSuite.scala new file mode 100644 index 000000000..d76611b6e --- /dev/null +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/ldap/ChainFilterSuite.scala @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.service.authentication.ldap + +import javax.security.sasl.AuthenticationException + +import org.mockito.ArgumentMatchers.{any, anyString} +import org.mockito.Mockito.{doThrow, times, verify, when} +import org.scalatestplus.mockito.MockitoSugar.mock + +import org.apache.kyuubi.KyuubiFunSuite +import org.apache.kyuubi.config.KyuubiConf + +class ChainFilterSuite extends KyuubiFunSuite { + private var conf: KyuubiConf = _ + private var filter1: Filter = _ + private var filter2: Filter = _ + private var filter3: Filter = _ + private var factory1: FilterFactory = _ + private var factory2: FilterFactory = _ + private var factory3: FilterFactory = _ + private var factory: FilterFactory = _ + private var search: DirSearch = _ + + override def beforeEach(): Unit = { + conf = new KyuubiConf() + filter1 = mock[Filter] + filter2 = mock[Filter] + filter3 = mock[Filter] + factory1 = mock[FilterFactory] + factory2 = mock[FilterFactory] + factory3 = mock[FilterFactory] + factory = new ChainFilterFactory(factory1, factory2, factory3) + search = mock[DirSearch] + super.beforeEach() + } + + test("FactoryAllNull") { + when(factory1.getInstance(any(classOf[KyuubiConf]))).thenReturn(None) + when(factory2.getInstance(any(classOf[KyuubiConf]))).thenReturn(None) + when(factory3.getInstance(any(classOf[KyuubiConf]))).thenReturn(None) + assert(factory.getInstance(conf).isEmpty) + } + + test("FactoryAllEmpty") { + val emptyFactory = new ChainFilterFactory() + assert(emptyFactory.getInstance(conf).isEmpty) + } + + test("Factory") { + when(factory1.getInstance(any(classOf[KyuubiConf]))).thenReturn(Some(filter1)) + when(factory2.getInstance(any(classOf[KyuubiConf]))).thenReturn(Some(filter2)) + when(factory3.getInstance(any(classOf[KyuubiConf]))).thenReturn(Some(filter3)) + val filter = factory.getInstance(conf).get + filter.apply(search, "User") + verify(filter1, times(1)).apply(search, "User") + verify(filter2, times(1)).apply(search, "User") + verify(filter3, times(1)).apply(search, "User") + } + + test("ApplyNegative") { + intercept[AuthenticationException] { + doThrow(classOf[AuthenticationException]) + .when(filter3) + .apply(any().asInstanceOf[DirSearch], anyString) + when(factory1.getInstance(any(classOf[KyuubiConf]))).thenReturn(Some(filter1)) + when(factory2.getInstance(any(classOf[KyuubiConf]))).thenReturn(None) + when(factory3.getInstance(any(classOf[KyuubiConf]))).thenReturn(Some(filter3)) + val filter = factory.getInstance(conf).get + filter.apply(search, "User") + } + } +} diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/ldap/CustomQueryFilterSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/ldap/CustomQueryFilterSuite.scala new file mode 100644 index 000000000..5ece4c88c --- /dev/null +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/ldap/CustomQueryFilterSuite.scala @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.service.authentication.ldap + +import javax.security.sasl.AuthenticationException + +import org.mockito.ArgumentMatchers.{eq => mockEq} +import org.mockito.Mockito.when +import org.scalatestplus.mockito.MockitoSugar.mock + +import org.apache.kyuubi.KyuubiFunSuite +import org.apache.kyuubi.config.KyuubiConf + +class CustomQueryFilterSuite extends KyuubiFunSuite { + private val USER2_DN: String = "uid=user2,ou=People,dc=example,dc=com" + private val USER1_DN: String = "uid=user1,ou=People,dc=example,dc=com" + private val CUSTOM_QUERY: String = "(&(objectClass=person)(|(uid=user1)(uid=user2)))" + + private val factory: FilterFactory = CustomQueryFilterFactory + private var conf: KyuubiConf = _ + private var search: DirSearch = _ + + override def beforeEach(): Unit = { + conf = new KyuubiConf() + search = mock[DirSearch] + super.beforeEach() + } + + test("Factory") { + conf.unset(KyuubiConf.AUTHENTICATION_LDAP_CUSTOM_LDAP_QUERY) + assert(factory.getInstance(conf).isEmpty) + conf.set(KyuubiConf.AUTHENTICATION_LDAP_CUSTOM_LDAP_QUERY, CUSTOM_QUERY) + assert(factory.getInstance(conf).isDefined) + } + + test("ApplyPositive") { + conf.set(KyuubiConf.AUTHENTICATION_LDAP_CUSTOM_LDAP_QUERY, CUSTOM_QUERY) + when(search.executeCustomQuery(mockEq(CUSTOM_QUERY))) + .thenReturn(Array(USER1_DN, USER2_DN)) + val filter: Filter = factory.getInstance(conf).get + filter.apply(search, "user1") + filter.apply(search, "user2") + } + + test("ApplyNegative") { + intercept[AuthenticationException] { + conf.set(KyuubiConf.AUTHENTICATION_LDAP_CUSTOM_LDAP_QUERY, CUSTOM_QUERY) + when(search.executeCustomQuery(mockEq(CUSTOM_QUERY))) + .thenReturn(Array(USER1_DN, USER2_DN)) + val filter: Filter = factory.getInstance(conf).get + filter.apply(search, "user3") + } + } +} diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/ldap/GroupFilterSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/ldap/GroupFilterSuite.scala new file mode 100644 index 000000000..f1e3c3581 --- /dev/null +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/ldap/GroupFilterSuite.scala @@ -0,0 +1,161 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.service.authentication.ldap + +import javax.security.sasl.AuthenticationException + +import org.mockito.ArgumentMatchers.{eq => mockEq} +import org.mockito.Mockito.{lenient, when} +import org.scalatestplus.mockito.MockitoSugar.mock + +import org.apache.kyuubi.KyuubiFunSuite +import org.apache.kyuubi.config.KyuubiConf + +class GroupFilterSuite extends KyuubiFunSuite { + private val factory: FilterFactory = GroupFilterFactory + private var conf: KyuubiConf = _ + private var search: DirSearch = _ + + override def beforeEach(): Unit = { + conf = new KyuubiConf + search = mock[DirSearch] + super.beforeEach() + } + + test("GetInstanceWhenGroupFilterIsEmpty") { + conf.unset(KyuubiConf.AUTHENTICATION_LDAP_GROUP_FILTER) + assert(factory.getInstance(conf).isEmpty) + } + + test("GetInstanceOfGroupMembershipKeyFilter") { + conf.set(KyuubiConf.AUTHENTICATION_LDAP_GROUP_FILTER.key, "G1") + val instance: Filter = factory.getInstance(conf).get + assert(instance.isInstanceOf[GroupMembershipKeyFilter]) + } + + test("GetInstanceOfUserMembershipKeyFilter") { + conf.set(KyuubiConf.AUTHENTICATION_LDAP_GROUP_FILTER.key, "G1") + conf.set(KyuubiConf.AUTHENTICATION_LDAP_USER_MEMBERSHIP_KEY, "memberof") + val instance: Filter = factory.getInstance(conf).get + assert(instance.isInstanceOf[UserMembershipKeyFilter]) + } + + test("GroupMembershipKeyFilterApplyPositive") { + conf.set(KyuubiConf.AUTHENTICATION_LDAP_GROUP_FILTER.key, "HiveUsers") + when(search.findUserDn(mockEq("user1"))) + .thenReturn("cn=user1,ou=People,dc=example,dc=com") + when(search.findUserDn(mockEq("cn=user2,dc=example,dc=com"))) + .thenReturn("cn=user2,ou=People,dc=example,dc=com") + when(search.findUserDn(mockEq("user3@mydomain.com"))) + .thenReturn("cn=user3,ou=People,dc=example,dc=com") + when(search.findGroupsForUser(mockEq("cn=user1,ou=People,dc=example,dc=com"))) + .thenReturn(Array( + "cn=SuperUsers,ou=Groups,dc=example,dc=com", + "cn=Office1,ou=Groups,dc=example,dc=com", + "cn=HiveUsers,ou=Groups,dc=example,dc=com", + "cn=G1,ou=Groups,dc=example,dc=com")) + when(search.findGroupsForUser(mockEq("cn=user2,ou=People,dc=example,dc=com"))) + .thenReturn(Array("cn=HiveUsers,ou=Groups,dc=example,dc=com")) + when(search.findGroupsForUser(mockEq("cn=user3,ou=People,dc=example,dc=com"))) + .thenReturn(Array( + "cn=HiveUsers,ou=Groups,dc=example,dc=com", + "cn=G1,ou=Groups,dc=example,dc=com", + "cn=G2,ou=Groups,dc=example,dc=com")) + val filter: Filter = factory.getInstance(conf).get + filter.apply(search, "user1") + filter.apply(search, "cn=user2,dc=example,dc=com") + filter.apply(search, "user3@mydomain.com") + } + + test("GroupMembershipKeyCaseInsensitiveFilterApplyPositive") { + conf.set(KyuubiConf.AUTHENTICATION_LDAP_GROUP_FILTER.key, "hiveusers,g1") + when(search.findUserDn(mockEq("user1"))) + .thenReturn("cn=user1,ou=People,dc=example,dc=com") + when(search.findUserDn(mockEq("cn=user2,dc=example,dc=com"))) + .thenReturn("cn=user2,ou=People,dc=example,dc=com") + when(search.findUserDn(mockEq("user3@mydomain.com"))) + .thenReturn("cn=user3,ou=People,dc=example,dc=com") + when(search.findGroupsForUser(mockEq("cn=user1,ou=People,dc=example,dc=com"))) + .thenReturn(Array( + "cn=SuperUsers,ou=Groups,dc=example,dc=com", + "cn=Office1,ou=Groups,dc=example,dc=com", + "cn=HiveUsers,ou=Groups,dc=example,dc=com", + "cn=G1,ou=Groups,dc=example,dc=com")) + when(search.findGroupsForUser(mockEq("cn=user2,ou=People,dc=example,dc=com"))) + .thenReturn(Array("cn=HiveUsers,ou=Groups,dc=example,dc=com")) + when(search.findGroupsForUser(mockEq("cn=user3,ou=People,dc=example,dc=com"))) + .thenReturn(Array( + "cn=G1,ou=Groups,dc=example,dc=com", + "cn=G2,ou=Groups,dc=example,dc=com")) + val filter: Filter = factory.getInstance(conf).get + filter.apply(search, "user1") + filter.apply(search, "cn=user2,dc=example,dc=com") + filter.apply(search, "user3@mydomain.com") + } + + test("GroupMembershipKeyCaseInsensitiveFilterApplyNegative") { + intercept[AuthenticationException] { + conf.set(KyuubiConf.AUTHENTICATION_LDAP_GROUP_FILTER.key, "hiveusers,containsg1") + lenient.when(search.findGroupsForUser(mockEq("user1"))) + .thenReturn(Array("SuperUsers", "Office1", "G1", "G2")) + val filter: Filter = factory.getInstance(conf).get + filter.apply(search, "user1") + } + } + + test("GroupMembershipKeyFilterApplyNegative") { + intercept[AuthenticationException] { + conf.set(KyuubiConf.AUTHENTICATION_LDAP_GROUP_FILTER.key, "HiveUsers") + lenient.when(search.findGroupsForUser(mockEq("user1"))) + .thenReturn(Array("SuperUsers", "Office1", "G1", "G2")) + val filter: Filter = factory.getInstance(conf).get + filter.apply(search, "user1") + } + } + + test("UserMembershipKeyFilterApplyPositiveWithUserId") { + conf.set(KyuubiConf.AUTHENTICATION_LDAP_USER_MEMBERSHIP_KEY.key, "memberOf") + conf.set(KyuubiConf.AUTHENTICATION_LDAP_GROUP_FILTER.key, "Group1,Group2") + when(search.findGroupDn("Group1")).thenReturn("cn=Group1,dc=a,dc=b") + when(search.findGroupDn("Group2")).thenReturn("cn=Group2,dc=a,dc=b") + when(search.isUserMemberOfGroup("User1", "cn=Group2,dc=a,dc=b")).thenReturn(true) + val filter: Filter = factory.getInstance(conf).get + filter.apply(search, "User1") + } + + test("UserMembershipKeyFilterApplyPositiveWithUserDn") { + conf.set(KyuubiConf.AUTHENTICATION_LDAP_USER_MEMBERSHIP_KEY.key, "memberOf") + conf.set(KyuubiConf.AUTHENTICATION_LDAP_GROUP_FILTER.key, "Group1,Group2") + when(search.findGroupDn("Group1")).thenReturn("cn=Group1,dc=a,dc=b") + when(search.findGroupDn("Group2")).thenReturn("cn=Group2,dc=a,dc=b") + when(search.isUserMemberOfGroup("cn=User1,dc=a,dc=b", "cn=Group2,dc=a,dc=b")).thenReturn(true) + val filter: Filter = factory.getInstance(conf).get + filter.apply(search, "cn=User1,dc=a,dc=b") + } + + test("UserMembershipKeyFilterApplyNegative") { + intercept[AuthenticationException] { + conf.set(KyuubiConf.AUTHENTICATION_LDAP_USER_MEMBERSHIP_KEY.key, "memberOf") + conf.set(KyuubiConf.AUTHENTICATION_LDAP_GROUP_FILTER.key, "Group1,Group2") + when(search.findGroupDn("Group1")).thenReturn("cn=Group1,dc=a,dc=b") + when(search.findGroupDn("Group2")).thenReturn("cn=Group2,dc=a,dc=b") + val filter: Filter = factory.getInstance(conf).get + filter.apply(search, "User1") + } + } +} diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/ldap/LdapAuthenticationTestCase.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/ldap/LdapAuthenticationTestCase.scala new file mode 100644 index 000000000..a06eba068 --- /dev/null +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/ldap/LdapAuthenticationTestCase.scala @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.service.authentication.ldap + +import javax.security.sasl.AuthenticationException + +import scala.collection.mutable + +import org.scalatest.Assertions._ + +import org.apache.kyuubi.config.{ConfigEntry, KyuubiConf} +import org.apache.kyuubi.service.authentication.LdapAuthenticationProviderImpl + +object LdapAuthenticationTestCase { + def builder: LdapAuthenticationTestCase.Builder = new LdapAuthenticationTestCase.Builder + + class Builder { + private val overrides: mutable.Map[ConfigEntry[_], String] = new mutable.HashMap + + var conf: KyuubiConf = _ + + def baseDN(baseDN: String): LdapAuthenticationTestCase.Builder = + setVarOnce(KyuubiConf.AUTHENTICATION_LDAP_BASE_DN, baseDN) + + def guidKey(guidKey: String): LdapAuthenticationTestCase.Builder = + setVarOnce(KyuubiConf.AUTHENTICATION_LDAP_GUID_KEY, guidKey) + + def userDNPatterns(userDNPatterns: String*): LdapAuthenticationTestCase.Builder = + setVarOnce(KyuubiConf.AUTHENTICATION_LDAP_USER_DN_PATTERN, userDNPatterns.mkString(":")) + + def userFilters(userFilters: String*): LdapAuthenticationTestCase.Builder = + setVarOnce(KyuubiConf.AUTHENTICATION_LDAP_USER_FILTER, userFilters.mkString(",")) + + def groupDNPatterns(groupDNPatterns: String*): LdapAuthenticationTestCase.Builder = + setVarOnce(KyuubiConf.AUTHENTICATION_LDAP_GROUP_DN_PATTERN, groupDNPatterns.mkString(":")) + + def groupFilters(groupFilters: String*): LdapAuthenticationTestCase.Builder = + setVarOnce(KyuubiConf.AUTHENTICATION_LDAP_GROUP_FILTER, groupFilters.mkString(",")) + + def groupClassKey(groupClassKey: String): LdapAuthenticationTestCase.Builder = + setVarOnce(KyuubiConf.AUTHENTICATION_LDAP_GROUP_CLASS_KEY, groupClassKey) + + def ldapUrl(ldapUrl: String): LdapAuthenticationTestCase.Builder = + setVarOnce(KyuubiConf.AUTHENTICATION_LDAP_URL, ldapUrl) + + def customQuery(customQuery: String): LdapAuthenticationTestCase.Builder = + setVarOnce(KyuubiConf.AUTHENTICATION_LDAP_CUSTOM_LDAP_QUERY, customQuery) + + def groupMembershipKey(groupMembershipKey: String): LdapAuthenticationTestCase.Builder = + setVarOnce(KyuubiConf.AUTHENTICATION_LDAP_GROUP_MEMBERSHIP_KEY, groupMembershipKey) + + def userMembershipKey(userMembershipKey: String): LdapAuthenticationTestCase.Builder = + setVarOnce(KyuubiConf.AUTHENTICATION_LDAP_USER_MEMBERSHIP_KEY, userMembershipKey) + + private def setVarOnce( + confVar: ConfigEntry[_], + value: String): LdapAuthenticationTestCase.Builder = { + require(!overrides.contains(confVar), s"Property $confVar has been set already") + overrides.put(confVar, value) + this + } + + def build: LdapAuthenticationTestCase = { + require(conf == null, "Test Case Builder should not be reused. Please create a new instance.") + conf = new KyuubiConf() + overrides.foreach { case (k, v) => conf.set(k.key, v) } + new LdapAuthenticationTestCase(this) + } + } +} + +final class LdapAuthenticationTestCase(builder: LdapAuthenticationTestCase.Builder) { + + private val ldapProvider = new LdapAuthenticationProviderImpl(builder.conf) + + def assertAuthenticatePasses(credentials: Credentials): Unit = + try { + ldapProvider.authenticate(credentials.user, credentials.password) + } catch { + case e: AuthenticationException => + throw new AssertionError( + s"Authentication failed for user '${credentials.user}' " + + s"with password '${credentials.password}'", + e) + } + + def assertAuthenticateFails(credentials: Credentials): Unit = { + assertAuthenticateFails(credentials.user, credentials.password) + } + + def assertAuthenticateFailsUsingWrongPassword(credentials: Credentials): Unit = { + assertAuthenticateFails(credentials.user, "not" + credentials.password) + } + + def assertAuthenticateFails(user: String, password: String): Unit = { + val e = intercept[AuthenticationException] { + ldapProvider.authenticate(user, password) + fail(s"Expected authentication to fail for $user") + } + assert(e != null) + } +} diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/ldap/LdapSearchSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/ldap/LdapSearchSuite.scala new file mode 100644 index 000000000..3bf27127b --- /dev/null +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/ldap/LdapSearchSuite.scala @@ -0,0 +1,298 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.service.authentication.ldap + +import javax.naming.{NamingEnumeration, NamingException} +import javax.naming.directory.{DirContext, SearchControls, SearchResult} + +import org.mockito.ArgumentMatchers.{any, anyString, contains, eq => mockEq} +import org.mockito.Mockito.{atLeastOnce, verify, when} +import org.scalatestplus.mockito.MockitoSugar.mock + +import org.apache.kyuubi.KyuubiFunSuite +import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.service.authentication.ldap.LdapTestUtils._ + +class LdapSearchSuite extends KyuubiFunSuite { + private var conf: KyuubiConf = _ + private var ctx: DirContext = _ + private var search: LdapSearch = _ + + override protected def beforeEach(): Unit = { + conf = new KyuubiConf() + conf.set(KyuubiConf.AUTHENTICATION_LDAP_USER_MEMBERSHIP_KEY, "memberOf") + ctx = mock[DirContext] + super.beforeEach() + } + + test("close") { + search = new LdapSearch(conf, ctx) + search.close() + verify(ctx, atLeastOnce).close() + } + + test("FindUserDnWhenUserDnPositive") { + val searchResult: NamingEnumeration[SearchResult] = + mockNamingEnumeration("CN=User1,OU=org1,DC=foo,DC=bar") + when(ctx.search(anyString, anyString, any(classOf[SearchControls]))) + .thenReturn(searchResult) + .thenThrow(classOf[NamingException]) + search = new LdapSearch(conf, ctx) + val expected: String = "CN=User1,OU=org1,DC=foo,DC=bar" + val actual: String = search.findUserDn("CN=User1,OU=org1") + assert(expected === actual) + } + + test("FindUserDnWhenUserDnNegativeDuplicates") { + val searchResult: NamingEnumeration[SearchResult] = + mockNamingEnumeration("CN=User1,OU=org1,DC=foo,DC=bar", "CN=User1,OU=org2,DC=foo,DC=bar") + when(ctx.search(anyString, anyString, any(classOf[SearchControls]))) + .thenReturn(searchResult) + search = new LdapSearch(conf, ctx) + assert(search.findUserDn("CN=User1,DC=foo,DC=bar") === null) + } + + test("FindUserDnWhenUserDnNegativeNone") { + val searchResult: NamingEnumeration[SearchResult] = mockEmptyNamingEnumeration + when(ctx.search(anyString, anyString, any(classOf[SearchControls]))) + .thenReturn(searchResult) + search = new LdapSearch(conf, ctx) + assert(search.findUserDn("CN=User1,DC=foo,DC=bar") === null) + } + + test("FindUserDnWhenUserPatternFoundBySecondPattern") { + conf.set( + KyuubiConf.AUTHENTICATION_LDAP_USER_DN_PATTERN, + "CN=%s,OU=org1,DC=foo,DC=bar:CN=%s,OU=org2,DC=foo,DC=bar") + val emptyResult: NamingEnumeration[SearchResult] = mockEmptyNamingEnumeration + val validResult: NamingEnumeration[SearchResult] = + mockNamingEnumeration("CN=User1,OU=org2,DC=foo,DC=bar") + when(ctx.search(anyString, anyString, any(classOf[SearchControls]))) + .thenReturn(emptyResult) + .thenReturn(validResult) + search = new LdapSearch(conf, ctx) + val expected: String = "CN=User1,OU=org2,DC=foo,DC=bar" + val actual: String = search.findUserDn("User1") + assert(expected === actual) + verify(ctx).search( + mockEq("OU=org1,DC=foo,DC=bar"), + contains("CN=User1"), + any(classOf[SearchControls])) + verify(ctx).search( + mockEq("OU=org2,DC=foo,DC=bar"), + contains("CN=User1"), + any(classOf[SearchControls])) + } + + test("FindUserDnWhenUserPatternFoundByFirstPattern") { + conf.set( + KyuubiConf.AUTHENTICATION_LDAP_USER_DN_PATTERN, + "CN=%s,OU=org1,DC=foo,DC=bar:CN=%s,OU=org2,DC=foo,DC=bar") + val emptyResult: NamingEnumeration[SearchResult] = mockEmptyNamingEnumeration + val validResult: NamingEnumeration[SearchResult] = + mockNamingEnumeration("CN=User1,OU=org2,DC=foo,DC=bar") + when(ctx.search(anyString, anyString, any(classOf[SearchControls]))) + .thenReturn(validResult) + .thenReturn(emptyResult) + search = new LdapSearch(conf, ctx) + val expected: String = "CN=User1,OU=org2,DC=foo,DC=bar" + val actual: String = search.findUserDn("User1") + assert(expected === actual) + verify(ctx).search( + mockEq("OU=org1,DC=foo,DC=bar"), + contains("CN=User1"), + any(classOf[SearchControls])) + } + + test("FindUserDnWhenUserPatternFoundByUniqueIdentifier") { + conf.set( + KyuubiConf.AUTHENTICATION_LDAP_USER_DN_PATTERN, + "CN=%s,OU=org1,DC=foo,DC=bar") + val validResult: NamingEnumeration[SearchResult] = + mockNamingEnumeration("CN=User1,OU=org1,DC=foo,DC=bar") + when(ctx.search(anyString, anyString, any(classOf[SearchControls]))) + .thenReturn(null) + .thenReturn(validResult) + search = new LdapSearch(conf, ctx) + val expected: String = "CN=User1,OU=org1,DC=foo,DC=bar" + val actual: String = search.findUserDn("User1") + assert(expected === actual) + verify(ctx).search( + mockEq("OU=org1,DC=foo,DC=bar"), + contains("CN=User1"), + any(classOf[SearchControls])) + verify(ctx).search( + mockEq("OU=org1,DC=foo,DC=bar"), + contains("uid=User1"), + any(classOf[SearchControls])) + } + + test("FindUserDnWhenUserPatternFoundByUniqueIdentifierNegativeNone") { + conf.set( + KyuubiConf.AUTHENTICATION_LDAP_USER_DN_PATTERN, + "CN=%s,OU=org1,DC=foo,DC=bar") + when(ctx.search(anyString, anyString, any(classOf[SearchControls]))) + .thenReturn(null) + .thenReturn(null) + search = new LdapSearch(conf, ctx) + assert(search.findUserDn("User1") === null) + } + + test("FindUserDnWhenUserPatternFoundByUniqueIdentifierNegativeMany") { + conf.set( + KyuubiConf.AUTHENTICATION_LDAP_USER_DN_PATTERN, + "CN=%s,OU=org1,DC=foo,DC=bar") + val manyResult: NamingEnumeration[SearchResult] = + mockNamingEnumeration("CN=User1,OU=org1,DC=foo,DC=bar", "CN=User12,OU=org1,DC=foo,DC=bar") + when(ctx.search(anyString, anyString, any(classOf[SearchControls]))) + .thenReturn(null) + .thenReturn(manyResult) + search = new LdapSearch(conf, ctx) + assert(search.findUserDn("User1") === null) + } + + test("FindGroupsForUser") { + conf.set( + KyuubiConf.AUTHENTICATION_LDAP_GROUP_DN_PATTERN, + "CN=%s,OU=org1,DC=foo,DC=bar") + val groupsResult: NamingEnumeration[SearchResult] = + mockNamingEnumeration("CN=Group1,OU=org1,DC=foo,DC=bar") + when( + ctx.search( + mockEq("OU=org1,DC=foo,DC=bar"), + contains("User1"), + any(classOf[SearchControls]))).thenReturn(groupsResult) + search = new LdapSearch(conf, ctx) + val expected = Array("CN=Group1,OU=org1,DC=foo,DC=bar") + val actual = search.findGroupsForUser("CN=User1,OU=org1,DC=foo,DC=bar") + assert(expected === actual) + } + + test("ExecuteCustomQuery") { + conf.set(KyuubiConf.AUTHENTICATION_LDAP_BASE_DN, "dc=example,dc=com") + val customQueryResult: NamingEnumeration[SearchResult] = mockNamingEnumeration(Array( + mockSearchResult( + "uid=group1,ou=Groups,dc=example,dc=com", + mockAttributes("member", "uid=user1,ou=People,dc=example,dc=com")), + mockSearchResult( + "uid=group2,ou=Groups,dc=example,dc=com", + mockAttributes("member", "uid=user2,ou=People,dc=example,dc=com")))) + when( + ctx.search( + mockEq("dc=example,dc=com"), + anyString, + any(classOf[SearchControls]))) + .thenReturn(customQueryResult) + search = new LdapSearch(conf, ctx) + val expected = Array( + "uid=group1,ou=Groups,dc=example,dc=com", + "uid=user1,ou=People,dc=example,dc=com", + "uid=group2,ou=Groups,dc=example,dc=com", + "uid=user2,ou=People,dc=example,dc=com") + val actual = search.executeCustomQuery("(&(objectClass=groupOfNames)(|(cn=group1)(cn=group2)))") + assert(expected.sorted === actual.sorted) + } + + test("FindGroupDnPositive") { + conf.set( + KyuubiConf.AUTHENTICATION_LDAP_GROUP_DN_PATTERN, + "CN=%s,OU=org1,DC=foo,DC=bar") + val groupDn: String = "CN=Group1" + val result: NamingEnumeration[SearchResult] = mockNamingEnumeration(groupDn) + when(ctx.search(anyString, anyString, any(classOf[SearchControls]))) + .thenReturn(result) + search = new LdapSearch(conf, ctx) + val expected: String = groupDn + val actual: String = search.findGroupDn("grp1") + assert(expected === actual) + } + + test("FindGroupDNNoResults") { + intercept[NamingException] { + conf.set( + KyuubiConf.AUTHENTICATION_LDAP_GROUP_DN_PATTERN, + "CN=%s,OU=org1,DC=foo,DC=bar") + val result: NamingEnumeration[SearchResult] = mockEmptyNamingEnumeration + when(ctx.search(anyString, anyString, any(classOf[SearchControls]))) + .thenReturn(result) + search = new LdapSearch(conf, ctx) + search.findGroupDn("anyGroup") + } + } + + test("FindGroupDNTooManyResults") { + intercept[NamingException] { + conf.set( + KyuubiConf.AUTHENTICATION_LDAP_GROUP_DN_PATTERN, + "CN=%s,OU=org1,DC=foo,DC=bar") + val result: NamingEnumeration[SearchResult] = + LdapTestUtils.mockNamingEnumeration("Result1", "Result2", "Result3") + when(ctx.search(anyString, anyString, any(classOf[SearchControls]))) + .thenReturn(result) + search = new LdapSearch(conf, ctx) + search.findGroupDn("anyGroup") + } + + } + + test("FindGroupDNWhenExceptionInSearch") { + conf.set( + KyuubiConf.AUTHENTICATION_LDAP_GROUP_DN_PATTERN, + Array("CN=%s,OU=org1,DC=foo,DC=bar", "CN=%s,OU=org2,DC=foo,DC=bar").mkString(":")) + val result: NamingEnumeration[SearchResult] = LdapTestUtils.mockNamingEnumeration("CN=Group1") + when(ctx.search(anyString, anyString, any(classOf[SearchControls]))) + .thenReturn(result) + .thenThrow(classOf[NamingException]) + search = new LdapSearch(conf, ctx) + val expected: String = "CN=Group1" + val actual: String = search.findGroupDn("grp1") + assert(expected === actual) + } + + test("IsUserMemberOfGroupWhenUserId") { + conf.set( + KyuubiConf.AUTHENTICATION_LDAP_USER_DN_PATTERN, + "CN=%s,OU=org1,DC=foo,DC=bar") + val validResult: NamingEnumeration[SearchResult] = + LdapTestUtils.mockNamingEnumeration("CN=User1") + val emptyResult: NamingEnumeration[SearchResult] = LdapTestUtils.mockEmptyNamingEnumeration + when(ctx.search(anyString, contains("(uid=usr1)"), any(classOf[SearchControls]))) + .thenReturn(validResult) + when(ctx.search(anyString, contains("(uid=usr2)"), any(classOf[SearchControls]))) + .thenReturn(emptyResult) + search = new LdapSearch(conf, ctx) + assert(search.isUserMemberOfGroup("usr1", "grp1")) + assert(!search.isUserMemberOfGroup("usr2", "grp2")) + } + + test("IsUserMemberOfGroupWhenUserDn") { + conf.set( + KyuubiConf.AUTHENTICATION_LDAP_USER_DN_PATTERN, + "CN=%s,OU=org1,DC=foo,DC=bar") + val validResult: NamingEnumeration[SearchResult] = + LdapTestUtils.mockNamingEnumeration("CN=User1") + val emptyResult: NamingEnumeration[SearchResult] = LdapTestUtils.mockEmptyNamingEnumeration + when(ctx.search(anyString, contains("(uid=User1)"), any(classOf[SearchControls]))) + .thenReturn(validResult) + when(ctx.search(anyString, contains("(uid=User2)"), any(classOf[SearchControls]))) + .thenReturn(emptyResult) + search = new LdapSearch(conf, ctx) + assert(search.isUserMemberOfGroup("CN=User1,OU=org1,DC=foo,DC=bar", "grp1")) + assert(!search.isUserMemberOfGroup("CN=User2,OU=org1,DC=foo,DC=bar", "grp2")) + } +} diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/ldap/LdapTestUtils.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/ldap/LdapTestUtils.scala new file mode 100644 index 000000000..49340f2c4 --- /dev/null +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/ldap/LdapTestUtils.scala @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.service.authentication.ldap + +import javax.naming.{NamingEnumeration, NamingException} +import javax.naming.directory._ + +import org.mockito.Mockito.when +import org.mockito.stubbing.OngoingStubbing +import org.scalatestplus.mockito.MockitoSugar + +case class NameValues(name: String, values: String*) +case class Credentials(user: String, password: String) + +case class User(dn: String, id: String, password: String) { + + def credentialsWithDn: Credentials = Credentials(dn, password) + + def credentialsWithId: Credentials = Credentials(id, password) +} + +object User { + def useIdForPassword(dn: String, id: String): User = User(dn, id, id) +} + +object LdapTestUtils extends MockitoSugar { + @throws[NamingException] + def mockEmptyNamingEnumeration: NamingEnumeration[SearchResult] = + mockNamingEnumeration(new Array[SearchResult](0)) + + @throws[NamingException] + def mockNamingEnumeration(dns: String*): NamingEnumeration[SearchResult] = + mockNamingEnumeration(mockSearchResults(dns.toArray)) + + @throws[NamingException] + def mockNamingEnumeration(searchResults: Array[SearchResult]): NamingEnumeration[SearchResult] = { + val ne = mock[NamingEnumeration[SearchResult]] + mockHasMoreMethod(ne, searchResults.length) + if (searchResults.nonEmpty) { + val mockedResults = Array(searchResults: _*) + mockNextMethod(ne, mockedResults) + } + ne + } + + @throws[NamingException] + def mockHasMoreMethod(ne: NamingEnumeration[SearchResult], length: Int): Unit = { + var hasMoreStub: OngoingStubbing[Boolean] = when(ne.hasMore) + (0 until length).foreach(_ => hasMoreStub = hasMoreStub.thenReturn(true)) + hasMoreStub.thenReturn(false) + } + + @throws[NamingException] + def mockNextMethod( + ne: NamingEnumeration[SearchResult], + searchResults: Array[SearchResult]): Unit = { + var nextStub: OngoingStubbing[SearchResult] = when(ne.next) + searchResults.foreach { searchResult => + nextStub = nextStub.thenReturn(searchResult) + } + } + + def mockSearchResults(dns: Array[String]): Array[SearchResult] = { + dns.map(mockSearchResult(_, null)) + } + + def mockSearchResult(dn: String, attributes: Attributes): SearchResult = { + val searchResult = mock[SearchResult] + when(searchResult.getNameInNamespace).thenReturn(dn) + when(searchResult.getAttributes).thenReturn(attributes) + searchResult + } + + @throws[NamingException] + def mockEmptyAttributes(): Attributes = mockAttributes() + + @throws[NamingException] + def mockAttributes(name: String, value: String): Attributes = + mockAttributes(NameValues(name, value)) + + @throws[NamingException] + def mockAttributes(name1: String, value1: String, name2: String, value2: String): Attributes = + if (name1 == name2) { + mockAttributes(NameValues(name1, value1, value2)) + } else { + mockAttributes( + NameValues(name1, value1), + NameValues(name2, value2)) + } + + @throws[NamingException] + private def mockAttributes(namedValues: NameValues*): Attributes = { + val attributes = new BasicAttributes + namedValues.foreach { namedValue => + val attr = new BasicAttribute(namedValue.name) + namedValue.values.foreach(attr.add) + attributes.put(attr) + } + attributes + } +} diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/ldap/LdapUtilsSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/ldap/LdapUtilsSuite.scala new file mode 100644 index 000000000..1ef371051 --- /dev/null +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/ldap/LdapUtilsSuite.scala @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.service.authentication.ldap + +import org.apache.kyuubi.KyuubiFunSuite +import org.apache.kyuubi.config.KyuubiConf + +class LdapUtilsSuite extends KyuubiFunSuite { + test("CreateCandidatePrincipalsForUserDn") { + val conf = new KyuubiConf() + val userDn = "cn=user1,ou=CORP,dc=mycompany,dc=com" + val expected = Array(userDn) + val actual = LdapUtils.createCandidatePrincipals(conf, userDn) + assert(actual === expected) + } + + test("CreateCandidatePrincipalsForUserWithDomain") { + val conf = new KyuubiConf() + val userWithDomain: String = "user1@mycompany.com" + val expected = Array(userWithDomain) + val actual = LdapUtils.createCandidatePrincipals(conf, userWithDomain) + assert(actual === expected) + } + + test("CreateCandidatePrincipalsLdapDomain") { + val conf = new KyuubiConf() + .set(KyuubiConf.AUTHENTICATION_LDAP_DOMAIN, "mycompany.com") + val expected = Array("user1@mycompany.com") + val actual = LdapUtils.createCandidatePrincipals(conf, "user1") + assert(actual === expected) + } + + test("CreateCandidatePrincipalsUserPatternsDefaultBaseDn") { + val conf = new KyuubiConf() + .set(KyuubiConf.AUTHENTICATION_LDAP_GUID_KEY, "sAMAccountName") + .set(KyuubiConf.AUTHENTICATION_LDAP_BASE_DN, "dc=mycompany,dc=com") + val expected = Array("sAMAccountName=user1,dc=mycompany,dc=com") + val actual = LdapUtils.createCandidatePrincipals(conf, "user1") + assert(actual === expected) + } + + test("CreateCandidatePrincipals") { + val conf = new KyuubiConf() + .set(KyuubiConf.AUTHENTICATION_LDAP_BASE_DN, "dc=mycompany,dc=com") + .set( + KyuubiConf.AUTHENTICATION_LDAP_USER_DN_PATTERN, + "cn=%s,ou=CORP1,dc=mycompany,dc=com:cn=%s,ou=CORP2,dc=mycompany,dc=com") + val expected = Array( + "cn=user1,ou=CORP1,dc=mycompany,dc=com", + "cn=user1,ou=CORP2,dc=mycompany,dc=com") + val actual = LdapUtils.createCandidatePrincipals(conf, "user1") + assert(actual.sorted === expected.sorted) + } + + test("ExtractFirstRdn") { + val dn = "cn=user1,ou=CORP1,dc=mycompany,dc=com" + val expected = "cn=user1" + val actual = LdapUtils.extractFirstRdn(dn) + assert(actual === expected) + } + + test("ExtractBaseDn") { + val dn: String = "cn=user1,ou=CORP1,dc=mycompany,dc=com" + val expected = "ou=CORP1,dc=mycompany,dc=com" + val actual = LdapUtils.extractBaseDn(dn) + assert(actual === expected) + } + + test("ExtractBaseDnNegative") { + val dn: String = "cn=user1" + assert(LdapUtils.extractBaseDn(dn) === null) + } +} diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/ldap/QueryFactorySuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/ldap/QueryFactorySuite.scala new file mode 100644 index 000000000..568009680 --- /dev/null +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/ldap/QueryFactorySuite.scala @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.service.authentication.ldap + +import org.apache.kyuubi.KyuubiFunSuite +import org.apache.kyuubi.config.KyuubiConf + +class QueryFactorySuite extends KyuubiFunSuite { + private var conf: KyuubiConf = _ + private var queries: QueryFactory = _ + + override def beforeEach(): Unit = { + conf = new KyuubiConf() + conf.set(KyuubiConf.AUTHENTICATION_LDAP_GUID_KEY, "guid") + conf.set(KyuubiConf.AUTHENTICATION_LDAP_GROUP_CLASS_KEY, "superGroups") + conf.set(KyuubiConf.AUTHENTICATION_LDAP_GROUP_MEMBERSHIP_KEY, "member") + conf.set(KyuubiConf.AUTHENTICATION_LDAP_USER_MEMBERSHIP_KEY, "partOf") + queries = new QueryFactory(conf) + super.beforeEach() + } + + test("FindGroupDnById") { + val q = queries.findGroupDnById("unique_group_id") + val expected = "(&(objectClass=superGroups)(guid=unique_group_id))" + val actual = q.filter + assert(expected === actual) + } + + test("FindUserDnByRdn") { + val q = queries.findUserDnByRdn("cn=User1") + val expected = + "(&(|(objectClass=person)(objectClass=user)(objectClass=inetOrgPerson))(cn=User1))" + val actual = q.filter + assert(expected === actual) + } + + test("FindDnByPattern") { + val q = queries.findDnByPattern("cn=User1") + val expected = "(cn=User1)" + val actual = q.filter + assert(expected === actual) + } + + test("FindUserDnByName") { + val q = queries.findUserDnByName("unique_user_id") + val expected = + "(&(|(objectClass=person)(objectClass=user)(objectClass=inetOrgPerson))" + + "(|(uid=unique_user_id)(sAMAccountName=unique_user_id)))" + val actual = q.filter + assert(expected === actual) + } + + test("FindGroupsForUser") { + val q = queries.findGroupsForUser("user_name", "user_Dn") + val expected = "(&(objectClass=superGroups)(|(member=user_Dn)(member=user_name)))" + val actual = q.filter + assert(expected === actual) + } + + test("IsUserMemberOfGroup") { + val q = queries.isUserMemberOfGroup("unique_user", "cn=MyGroup,ou=Groups,dc=mycompany,dc=com") + val expected = + "(&(|(objectClass=person)(objectClass=user)(objectClass=inetOrgPerson))" + + "(partOf=cn=MyGroup,ou=Groups,dc=mycompany,dc=com)(guid=unique_user))" + val actual = q.filter + assert(expected === actual) + } + + test("IsUserMemberOfGroupWhenMisconfigured") { + intercept[IllegalArgumentException] { + val misconfiguredQueryFactory = new QueryFactory(new KyuubiConf()) + misconfiguredQueryFactory.isUserMemberOfGroup("user", "cn=MyGroup") + } + } + + test("FindGroupDNByID") { + val q = queries.findGroupDnById("unique_group_id") + val expected = "(&(objectClass=superGroups)(guid=unique_group_id))" + val actual = q.filter + assert(expected === actual) + } +} diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/ldap/QuerySuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/ldap/QuerySuite.scala new file mode 100644 index 000000000..ffe330cce --- /dev/null +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/ldap/QuerySuite.scala @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.service.authentication.ldap + +import org.apache.kyuubi.KyuubiFunSuite + +class QuerySuite extends KyuubiFunSuite { + + test("QueryBuilderFilter") { + val q = Query.builder + .filter("test = query") + .map("uid_attr", "uid") + .map("value", "Hello!") + .build + assert("test uid=Hello! query" === q.filter) + assert(0 === q.controls.getCountLimit) + } + + test("QueryBuilderLimit") { + val q = Query.builder + .filter(",") + .map("key1", "value1") + .map("key2", "value2") + .limit(8) + .build + assert("value1,value2" === q.filter) + assert(8 === q.controls.getCountLimit) + } + + test("QueryBuilderReturningAttributes") { + val q = Query.builder + .filter("(query)") + .returnAttribute("attr1") + .returnAttribute("attr2") + .build + assert("(query)" === q.filter) + assert(Array("attr1", "attr2") === q.controls.getReturningAttributes) + } +} diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/ldap/SearchResultHandlerSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/ldap/SearchResultHandlerSuite.scala new file mode 100644 index 000000000..4e92f7f5f --- /dev/null +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/ldap/SearchResultHandlerSuite.scala @@ -0,0 +1,203 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.service.authentication.ldap + +import java.util +import javax.naming.{NamingEnumeration, NamingException} +import javax.naming.directory.SearchResult + +import scala.collection.mutable.ArrayBuffer + +import org.mockito.Mockito.{atLeastOnce, doThrow, verify} + +import org.apache.kyuubi.KyuubiFunSuite +import org.apache.kyuubi.service.authentication.ldap.LdapTestUtils._ + +class SearchResultHandlerSuite extends KyuubiFunSuite { + private var handler: SearchResultHandler = _ + + test("handle") { + val resultCollection = new MockResultCollectionBuilder() + .addSearchResultWithDns("1") + .addSearchResultWithDns("2", "3") + .build + handler = new SearchResultHandler(resultCollection) + val expected: util.List[String] = util.Arrays.asList("1", "2") + val actual: util.List[String] = new util.ArrayList[String] + handler.handle { record => + actual.add(record.getNameInNamespace) + actual.size < 2 + } + assert(expected === actual) + assertAllNamingEnumerationsClosed(resultCollection) + } + + test("GetAllLdapNamesNoRecords") { + val resultCollection = new MockResultCollectionBuilder() + .addEmptySearchResult() + .build + handler = new SearchResultHandler(resultCollection) + val actual = handler.getAllLdapNames + assert(actual.isEmpty, "ResultSet size") + assertAllNamingEnumerationsClosed(resultCollection) + } + + test("GetAllLdapNamesWithExceptionInNamingEnumerationClose") { + val resultCollection = new MockResultCollectionBuilder() + .addSearchResultWithDns("1") + .addSearchResultWithDns("2") + .build + doThrow(classOf[NamingException]).when(resultCollection.iterator.next).close() + handler = new SearchResultHandler(resultCollection) + val actual = handler.getAllLdapNames + assert(actual.length === 2, "ResultSet size") + assertAllNamingEnumerationsClosed(resultCollection) + } + + test("GetAllLdapNames") { + val objectDn1: String = "cn=a1,dc=b,dc=c" + val objectDn2: String = "cn=a2,dc=b,dc=c" + val objectDn3: String = "cn=a3,dc=b,dc=c" + val resultCollection = new MockResultCollectionBuilder() + .addSearchResultWithDns(objectDn1) + .addSearchResultWithDns(objectDn2, objectDn3) + .build + handler = new SearchResultHandler(resultCollection) + val expected = Array(objectDn1, objectDn2, objectDn3) + val actual = handler.getAllLdapNames + assert(expected.sorted === actual.sorted) + assertAllNamingEnumerationsClosed(resultCollection) + } + + test("GetAllLdapNamesAndAttributes") { + val searchResult1 = mockSearchResult( + "cn=a1,dc=b,dc=c", + mockAttributes("attr1", "attr1value1")) + val searchResult2 = mockSearchResult( + "cn=a2,dc=b,dc=c", + mockAttributes("attr1", "attr1value2", "attr2", "attr2value1")) + val searchResult3 = mockSearchResult( + "cn=a3,dc=b,dc=c", + mockAttributes("attr1", "attr1value3", "attr1", "attr1value4")) + val searchResult4 = mockSearchResult( + "cn=a4,dc=b,dc=c", + mockEmptyAttributes()) + val resultCollection = new MockResultCollectionBuilder() + .addSearchResults(searchResult1) + .addSearchResults(searchResult2, searchResult3) + .addSearchResults(searchResult4) + .build + handler = new SearchResultHandler(resultCollection) + val expected = Array( + "cn=a1,dc=b,dc=c", + "attr1value1", + "cn=a2,dc=b,dc=c", + "attr1value2", + "attr2value1", + "cn=a3,dc=b,dc=c", + "attr1value3", + "attr1value4", + "cn=a4,dc=b,dc=c") + val actual = handler.getAllLdapNamesAndAttributes + assert(expected.sorted === actual.sorted) + assertAllNamingEnumerationsClosed(resultCollection) + } + + test("HasSingleResultNoRecords") { + val resultCollection = new MockResultCollectionBuilder() + .addEmptySearchResult() + .build + handler = new SearchResultHandler(resultCollection) + assert(!handler.hasSingleResult) + assertAllNamingEnumerationsClosed(resultCollection) + } + + test("HasSingleResult") { + val resultCollection = new MockResultCollectionBuilder() + .addSearchResultWithDns("1") + .build + handler = new SearchResultHandler(resultCollection) + assert(handler.hasSingleResult) + assertAllNamingEnumerationsClosed(resultCollection) + } + + test("HasSingleResultManyRecords") { + val resultCollection = new MockResultCollectionBuilder() + .addSearchResultWithDns("1") + .addSearchResultWithDns("2") + .build + handler = new SearchResultHandler(resultCollection) + assert(!handler.hasSingleResult) + assertAllNamingEnumerationsClosed(resultCollection) + } + + test("GetSingleLdapNameNoRecords") { + intercept[NamingException] { + val resultCollection = new MockResultCollectionBuilder() + .addEmptySearchResult() + .build + handler = new SearchResultHandler(resultCollection) + try handler.getSingleLdapName + finally { + assertAllNamingEnumerationsClosed(resultCollection) + } + } + } + + test("GetSingleLdapName") { + val objectDn: String = "cn=a,dc=b,dc=c" + val resultCollection = new MockResultCollectionBuilder() + .addEmptySearchResult() + .addSearchResultWithDns(objectDn) + .build + handler = new SearchResultHandler(resultCollection) + val expected: String = objectDn + val actual: String = handler.getSingleLdapName + assert(expected === actual) + assertAllNamingEnumerationsClosed(resultCollection) + } + + private def assertAllNamingEnumerationsClosed( + resultCollection: Array[NamingEnumeration[SearchResult]]): Unit = { + for (namingEnumeration <- resultCollection) { + verify(namingEnumeration, atLeastOnce).close() + } + } +} + +class MockResultCollectionBuilder { + + val results = new ArrayBuffer[NamingEnumeration[SearchResult]] + + def addSearchResultWithDns(dns: String*): MockResultCollectionBuilder = { + results += mockNamingEnumeration(dns: _*) + this + } + + def addSearchResults(dns: SearchResult*): MockResultCollectionBuilder = { + results += mockNamingEnumeration(dns.toArray) + this + } + + def addEmptySearchResult(): MockResultCollectionBuilder = { + addSearchResults() + this + } + + def build: Array[NamingEnumeration[SearchResult]] = results.toArray +} diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/ldap/UserFilterSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/ldap/UserFilterSuite.scala new file mode 100644 index 000000000..4fc6cba49 --- /dev/null +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/ldap/UserFilterSuite.scala @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.service.authentication.ldap + +import javax.security.sasl.AuthenticationException + +import org.scalatestplus.mockito.MockitoSugar.mock + +import org.apache.kyuubi.KyuubiFunSuite +import org.apache.kyuubi.config.KyuubiConf + +class UserFilterSuite extends KyuubiFunSuite { + private val factory: FilterFactory = UserFilterFactory + private var conf: KyuubiConf = _ + private var search: DirSearch = _ + + override def beforeEach(): Unit = { + conf = new KyuubiConf() + search = mock[DirSearch] + super.beforeEach() + } + + test("Factory") { + conf.unset(KyuubiConf.AUTHENTICATION_LDAP_USER_FILTER) + assert(factory.getInstance(conf).isEmpty) + conf.set(KyuubiConf.AUTHENTICATION_LDAP_USER_FILTER.key, "User1") + assert(factory.getInstance(conf).isDefined) + } + + test("ApplyPositive") { + conf.set(KyuubiConf.AUTHENTICATION_LDAP_USER_FILTER.key, "User1,User2,uSeR3") + val filter = factory.getInstance(conf).get + filter.apply(search, "User1") + filter.apply(search, "uid=user2,ou=People,dc=example,dc=com") + filter.apply(search, "User3@mydomain.com") + } + + test("ApplyNegative") { + intercept[AuthenticationException] { + conf.set(KyuubiConf.AUTHENTICATION_LDAP_USER_FILTER.key, "User1,User2") + val filter = factory.getInstance(conf).get + filter.apply(search, "User3") + } + } +} diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/ldap/UserSearchFilterSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/ldap/UserSearchFilterSuite.scala new file mode 100644 index 000000000..1a711a6d9 --- /dev/null +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/service/authentication/ldap/UserSearchFilterSuite.scala @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.service.authentication.ldap + +import javax.naming.NamingException +import javax.security.sasl.AuthenticationException + +import org.mockito.ArgumentMatchers.anyString +import org.mockito.Mockito.when +import org.scalatestplus.mockito.MockitoSugar.mock + +import org.apache.kyuubi.KyuubiFunSuite +import org.apache.kyuubi.config.KyuubiConf + +class UserSearchFilterSuite extends KyuubiFunSuite { + private val factory: FilterFactory = UserSearchFilterFactory + private var conf: KyuubiConf = _ + private var search: DirSearch = _ + + override def beforeEach(): Unit = { + conf = new KyuubiConf() + search = mock[DirSearch] + super.beforeEach() + } + + test("FactoryWhenNoGroupOrUserFilters") { + assert(factory.getInstance(conf).isEmpty) + } + + test("FactoryWhenGroupFilter") { + conf.set(KyuubiConf.AUTHENTICATION_LDAP_GROUP_FILTER.key, "Grp1,Grp2") + assert(factory.getInstance(conf).isDefined) + } + + test("FactoryWhenUserFilter") { + conf.set(KyuubiConf.AUTHENTICATION_LDAP_USER_FILTER.key, "User1,User2") + assert(factory.getInstance(conf).isDefined) + } + + test("ApplyPositive") { + conf.set(KyuubiConf.AUTHENTICATION_LDAP_USER_FILTER.key, "User1") + val filter = factory.getInstance(conf).get + when(search.findUserDn(anyString)).thenReturn("cn=User1,ou=People,dc=example,dc=com") + filter.apply(search, "User1") + } + + test("ApplyWhenNamingException") { + intercept[AuthenticationException] { + conf.set(KyuubiConf.AUTHENTICATION_LDAP_USER_FILTER.key, "User1") + val filter = factory.getInstance(conf).get + when(search.findUserDn(anyString)).thenThrow(classOf[NamingException]) + filter.apply(search, "User3") + } + } + + test("ApplyWhenNotFound") { + intercept[AuthenticationException] { + conf.set(KyuubiConf.AUTHENTICATION_LDAP_USER_FILTER.key, "User1") + val filter = factory.getInstance(conf).get + when(search.findUserDn(anyString)).thenReturn(null) + filter.apply(search, "User3") + } + } +} diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/util/ClassUtilsSuite.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/util/ClassUtilsSuite.scala new file mode 100644 index 000000000..cda638b0d --- /dev/null +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/util/ClassUtilsSuite.scala @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.util + +import org.apache.kyuubi.KyuubiFunSuite +import org.apache.kyuubi.config.KyuubiConf + +class ClassUtilsSuite extends KyuubiFunSuite { + + private val _conf = KyuubiConf() + + test("create instance with zero-arg arg") { + val instance = ClassUtils.createInstance[SomeProvider]( + "org.apache.kyuubi.util.ProviderA", + classOf[SomeProvider], + _conf) + + assert(instance != null) + assert(instance.isInstanceOf[SomeProvider]) + assert(instance.isInstanceOf[ProviderA]) + } + + test("create instance with kyuubi conf") { + val instance = ClassUtils.createInstance[SomeProvider]( + "org.apache.kyuubi.util.ProviderB", + classOf[SomeProvider], + _conf) + assert(instance != null) + assert(instance.isInstanceOf[SomeProvider]) + assert(instance.isInstanceOf[ProviderB]) + assert(instance.asInstanceOf[ProviderB].getConf != null) + } + + test("create instance of inherited class with kyuubi conf") { + val instance = ClassUtils.createInstance[SomeProvider]( + "org.apache.kyuubi.util.ProviderC", + classOf[SomeProvider], + _conf) + assert(instance != null) + assert(instance.isInstanceOf[SomeProvider]) + assert(instance.isInstanceOf[ProviderB]) + assert(instance.isInstanceOf[ProviderC]) + assert(instance.asInstanceOf[ProviderC].getConf != null) + } + +} + +trait SomeProvider {} + +class ProviderA extends SomeProvider {} + +class ProviderB(conf: KyuubiConf) extends SomeProvider { + def getConf: KyuubiConf = conf +} + +class ProviderC(conf: KyuubiConf) extends ProviderB(conf) {} diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/util/SparkVersionUtil.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/util/SparkVersionUtil.scala index cd8409d10..ece9d53aa 100644 --- a/kyuubi-common/src/test/scala/org/apache/kyuubi/util/SparkVersionUtil.scala +++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/util/SparkVersionUtil.scala @@ -17,13 +17,19 @@ package org.apache.kyuubi.util -import org.apache.kyuubi.SPARK_COMPILE_VERSION -import org.apache.kyuubi.engine.SemanticVersion +import org.apache.kyuubi.operation.HiveJDBCTestHelper -object SparkVersionUtil { - lazy val sparkSemanticVersion: SemanticVersion = SemanticVersion(SPARK_COMPILE_VERSION) +trait SparkVersionUtil { + this: HiveJDBCTestHelper => - def isSparkVersionAtLeast(ver: String): Boolean = { - sparkSemanticVersion.isVersionAtLeast(ver) + protected lazy val SPARK_ENGINE_RUNTIME_VERSION: SemanticVersion = { + var sparkRuntimeVer = "" + withJdbcStatement() { stmt => + val result = stmt.executeQuery("SELECT version()") + assert(result.next()) + sparkRuntimeVer = result.getString(1) + assert(!result.next()) + } + SemanticVersion(sparkRuntimeVer) } } diff --git a/kyuubi-ctl/pom.xml b/kyuubi-ctl/pom.xml index aa1e8f2e4..c453cd3af 100644 --- a/kyuubi-ctl/pom.xml +++ b/kyuubi-ctl/pom.xml @@ -21,11 +21,11 @@ org.apache.kyuubi kyuubi-parent - 1.7.0-SNAPSHOT + 1.9.0-SNAPSHOT ../pom.xml - kyuubi-ctl_2.12 + kyuubi-ctl_${scala.binary.version} jar Kyuubi Project Control https://kyuubi.apache.org/ @@ -48,6 +48,11 @@ ${project.version}     + + org.apache.kyuubi + ${kyuubi-shaded-zookeeper.artifacts} + + org.apache.hadoop hadoop-client-api @@ -60,16 +65,6 @@ provided - - org.apache.curator - curator-framework - - - - org.apache.curator - curator-recipes - - com.github.scopt scopt_${scala.binary.version} @@ -86,11 +81,6 @@ ${snakeyaml.version} - - org.apache.zookeeper - zookeeper - - org.apache.kyuubi kyuubi-common_${scala.binary.version} diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/CtlConf.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/CtlConf.scala index f299a5a88..58b65582a 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/CtlConf.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/CtlConf.scala @@ -19,12 +19,11 @@ package org.apache.kyuubi.ctl import java.time.Duration -import org.apache.kyuubi.config.{ConfigBuilder, ConfigEntry, KyuubiConf, OptionalConfigEntry} +import org.apache.kyuubi.config.{ConfigEntry, OptionalConfigEntry} +import org.apache.kyuubi.config.KyuubiConf.buildConf object CtlConf { - private def buildConf(key: String): ConfigBuilder = KyuubiConf.buildConf(key) - val CTL_REST_CLIENT_BASE_URL: OptionalConfigEntry[String] = buildConf("kyuubi.ctl.rest.base.url") .doc("The REST API base URL, " + diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/RestClientFactory.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/RestClientFactory.scala index bbaa5f668..d971eec13 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/RestClientFactory.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/RestClientFactory.scala @@ -18,6 +18,8 @@ package org.apache.kyuubi.ctl import java.util.{Map => JMap} +import scala.collection.JavaConverters._ + import org.apache.commons.lang3.StringUtils import org.apache.kyuubi.KyuubiException @@ -45,7 +47,9 @@ object RestClientFactory { kyuubiRestClient: KyuubiRestClient, kyuubiInstance: String)(f: KyuubiRestClient => Unit): Unit = { val kyuubiInstanceRestClient = kyuubiRestClient.clone() - kyuubiInstanceRestClient.setHostUrls(s"http://${kyuubiInstance}") + val hostUrls = Option(kyuubiInstance).map(instance => s"http://$instance").toSeq ++ + kyuubiRestClient.getHostUrls.asScala + kyuubiInstanceRestClient.setHostUrls(hostUrls.asJava) try { f(kyuubiInstanceRestClient) } finally { diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cli/AdminControlCliArguments.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cli/AdminControlCliArguments.scala index 4bc1e1317..e015525b3 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cli/AdminControlCliArguments.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cli/AdminControlCliArguments.scala @@ -22,7 +22,7 @@ import scopt.OParser import org.apache.kyuubi.KyuubiException import org.apache.kyuubi.ctl.cmd.Command import org.apache.kyuubi.ctl.cmd.delete.AdminDeleteEngineCommand -import org.apache.kyuubi.ctl.cmd.list.AdminListEngineCommand +import org.apache.kyuubi.ctl.cmd.list.{AdminListEngineCommand, AdminListServerCommand} import org.apache.kyuubi.ctl.cmd.refresh.RefreshConfigCommand import org.apache.kyuubi.ctl.opt.{AdminCommandLine, CliConfig, ControlAction, ControlObject} @@ -37,6 +37,7 @@ class AdminControlCliArguments(args: Seq[String], env: Map[String, String] = sys cliConfig.action match { case ControlAction.LIST => cliConfig.resource match { case ControlObject.ENGINE => new AdminListEngineCommand(cliConfig) + case ControlObject.SERVER => new AdminListServerCommand(cliConfig) case _ => throw new KyuubiException(s"Invalid resource: ${cliConfig.resource}") } case ControlAction.DELETE => cliConfig.resource match { @@ -60,6 +61,12 @@ class AdminControlCliArguments(args: Seq[String], env: Map[String, String] = sys | type ${cliConfig.engineOpts.engineType} | sharelevel ${cliConfig.engineOpts.engineShareLevel} | sharesubdomain ${cliConfig.engineOpts.engineSubdomain} + | all ${cliConfig.engineOpts.all} + """.stripMargin + case ControlObject.SERVER => + s"""Parsed arguments: + | action ${cliConfig.action} + | resource ${cliConfig.resource} """.stripMargin case ControlObject.CONFIG => s"""Parsed arguments: diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cli/ControlCliArguments.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cli/ControlCliArguments.scala index 41d53b568..10bb99296 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cli/ControlCliArguments.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cli/ControlCliArguments.scala @@ -33,9 +33,9 @@ import org.apache.kyuubi.ctl.opt.{CliConfig, CommandLine, ControlAction, Control class ControlCliArguments(args: Seq[String], env: Map[String, String] = sys.env) extends ControlCliArgumentsParser with Logging { - var cliConfig: CliConfig = null + var cliConfig: CliConfig = _ - var command: Command[_] = null + var command: Command[_] = _ // Set parameters from command line arguments parse(args) @@ -112,6 +112,7 @@ class ControlCliArguments(args: Seq[String], env: Map[String, String] = sys.env) | batchType ${cliConfig.batchOpts.batchType} | batchUser ${cliConfig.batchOpts.batchUser} | batchState ${cliConfig.batchOpts.batchState} + | batchName ${cliConfig.batchOpts.batchName} | createTime ${cliConfig.batchOpts.createTime} | endTime ${cliConfig.batchOpts.endTime} | from ${cliConfig.batchOpts.from} diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/create/CreateServerCommand.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/create/CreateServerCommand.scala index 66f75fc5f..cbff0c15a 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/create/CreateServerCommand.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/create/CreateServerCommand.scala @@ -25,7 +25,8 @@ import org.apache.kyuubi.ha.HighAvailabilityConf._ import org.apache.kyuubi.ha.client.{DiscoveryClient, DiscoveryPaths, ServiceNodeInfo} import org.apache.kyuubi.ha.client.DiscoveryClientProvider.withDiscoveryClient -class CreateServerCommand(cliConfig: CliConfig) extends Command[Seq[ServiceNodeInfo]](cliConfig) { +class CreateServerCommand(cliConfig: CliConfig) + extends Command[Iterable[ServiceNodeInfo]](cliConfig) { def validate(): Unit = { if (normalizedCliConfig.resource != ControlObject.SERVER) { @@ -49,14 +50,14 @@ class CreateServerCommand(cliConfig: CliConfig) extends Command[Seq[ServiceNodeI /** * Expose Kyuubi server instance to another domain. */ - def doRun(): Seq[ServiceNodeInfo] = { + override def doRun(): Iterable[ServiceNodeInfo] = { val kyuubiConf = conf kyuubiConf.setIfMissing(HA_ADDRESSES, normalizedCliConfig.zkOpts.zkQuorum) withDiscoveryClient(kyuubiConf) { discoveryClient => val fromNamespace = DiscoveryPaths.makePath(null, kyuubiConf.get(HA_NAMESPACE)) - val toNamespace = CtlUtils.getZkNamespace(kyuubiConf, normalizedCliConfig) + val toNamespace = CtlUtils.getZkServerNamespace(kyuubiConf, normalizedCliConfig) val currentServerNodes = discoveryClient.getServiceNodesInfo(fromNamespace) val exposedServiceNodes = ListBuffer[ServiceNodeInfo]() @@ -89,7 +90,7 @@ class CreateServerCommand(cliConfig: CliConfig) extends Command[Seq[ServiceNodeI } } - def render(nodes: Seq[ServiceNodeInfo]): Unit = { + override def render(nodes: Iterable[ServiceNodeInfo]): Unit = { val title = "Created zookeeper service nodes" info(Render.renderServiceNodesInfo(title, nodes)) } diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/delete/DeleteCommand.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/delete/DeleteCommand.scala index 69479259a..113fb935c 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/delete/DeleteCommand.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/delete/DeleteCommand.scala @@ -16,15 +16,13 @@ */ package org.apache.kyuubi.ctl.cmd.delete -import scala.collection.mutable.ListBuffer - import org.apache.kyuubi.ctl.cmd.Command import org.apache.kyuubi.ctl.opt.CliConfig -import org.apache.kyuubi.ctl.util.{CtlUtils, Render, Validator} -import org.apache.kyuubi.ha.client.DiscoveryClientProvider.withDiscoveryClient +import org.apache.kyuubi.ctl.util.{Render, Validator} import org.apache.kyuubi.ha.client.ServiceNodeInfo -class DeleteCommand(cliConfig: CliConfig) extends Command[Seq[ServiceNodeInfo]](cliConfig) { +abstract class DeleteCommand(cliConfig: CliConfig) + extends Command[Iterable[ServiceNodeInfo]](cliConfig) { def validate(): Unit = { Validator.validateZkArguments(normalizedCliConfig) @@ -35,30 +33,9 @@ class DeleteCommand(cliConfig: CliConfig) extends Command[Seq[ServiceNodeInfo]]( /** * Delete zookeeper service node with specified host port. */ - def doRun(): Seq[ServiceNodeInfo] = { - withDiscoveryClient(conf) { discoveryClient => - val znodeRoot = CtlUtils.getZkNamespace(conf, normalizedCliConfig) - val hostPortOpt = - Some((normalizedCliConfig.zkOpts.host, normalizedCliConfig.zkOpts.port.toInt)) - val nodesToDelete = CtlUtils.getServiceNodes(discoveryClient, znodeRoot, hostPortOpt) - - val deletedNodes = ListBuffer[ServiceNodeInfo]() - nodesToDelete.foreach { node => - val nodePath = s"$znodeRoot/${node.nodeName}" - info(s"Deleting zookeeper service node:$nodePath") - try { - discoveryClient.delete(nodePath) - deletedNodes += node - } catch { - case e: Exception => - error(s"Failed to delete zookeeper service node:$nodePath", e) - } - } - deletedNodes - } - } + override def doRun(): Iterable[ServiceNodeInfo] - def render(nodes: Seq[ServiceNodeInfo]): Unit = { + override def render(nodes: Iterable[ServiceNodeInfo]): Unit = { val title = "Deleted zookeeper service nodes" info(Render.renderServiceNodesInfo(title, nodes)) } diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/delete/DeleteEngineCommand.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/delete/DeleteEngineCommand.scala index 7be607467..f3117a7b1 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/delete/DeleteEngineCommand.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/delete/DeleteEngineCommand.scala @@ -16,7 +16,12 @@ */ package org.apache.kyuubi.ctl.cmd.delete +import scala.collection.mutable.ListBuffer + import org.apache.kyuubi.ctl.opt.CliConfig +import org.apache.kyuubi.ctl.util.CtlUtils +import org.apache.kyuubi.ha.client.DiscoveryClientProvider.withDiscoveryClient +import org.apache.kyuubi.ha.client.ServiceNodeInfo class DeleteEngineCommand(cliConfig: CliConfig) extends DeleteCommand(cliConfig) { @@ -28,4 +33,29 @@ class DeleteEngineCommand(cliConfig: CliConfig) extends DeleteCommand(cliConfig) fail("Must specify user name for engine, please use -u or --user.") } } + + override def doRun(): Iterable[ServiceNodeInfo] = { + withDiscoveryClient(conf) { discoveryClient => + val hostPortOpt = + Some((cliConfig.zkOpts.host, cliConfig.zkOpts.port.toInt)) + val candidateNodes = CtlUtils.listZkEngineNodes(conf, normalizedCliConfig, hostPortOpt) + hostPortOpt.map { case (host, port) => + candidateNodes.filter { cn => cn.host == host && cn.port == port } + }.getOrElse(candidateNodes) + val deletedNodes = ListBuffer[ServiceNodeInfo]() + candidateNodes.foreach { node => + val engineNode = discoveryClient.getChildren(node.namespace)(0) + val nodePath = s"${node.namespace}/$engineNode" + info(s"Deleting zookeeper service node:$nodePath") + try { + discoveryClient.delete(nodePath) + deletedNodes += node + } catch { + case e: Exception => + error(s"Failed to delete zookeeper service node:$nodePath", e) + } + } + deletedNodes + } + } } diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/delete/DeleteServerCommand.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/delete/DeleteServerCommand.scala index 6debba4d5..1f4d67ee6 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/delete/DeleteServerCommand.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/delete/DeleteServerCommand.scala @@ -16,6 +16,34 @@ */ package org.apache.kyuubi.ctl.cmd.delete +import scala.collection.mutable.ListBuffer + import org.apache.kyuubi.ctl.opt.CliConfig +import org.apache.kyuubi.ctl.util.CtlUtils +import org.apache.kyuubi.ha.client.DiscoveryClientProvider.withDiscoveryClient +import org.apache.kyuubi.ha.client.ServiceNodeInfo + +class DeleteServerCommand(cliConfig: CliConfig) extends DeleteCommand(cliConfig) { + override def doRun(): Iterable[ServiceNodeInfo] = { + withDiscoveryClient(conf) { discoveryClient => + val znodeRoot = CtlUtils.getZkServerNamespace(conf, normalizedCliConfig) + val hostPortOpt = + Some((normalizedCliConfig.zkOpts.host, normalizedCliConfig.zkOpts.port.toInt)) + val nodesToDelete = CtlUtils.getServiceNodes(discoveryClient, znodeRoot, hostPortOpt) -class DeleteServerCommand(cliConfig: CliConfig) extends DeleteCommand(cliConfig) {} + val deletedNodes = ListBuffer[ServiceNodeInfo]() + nodesToDelete.foreach { node => + val nodePath = s"$znodeRoot/${node.nodeName}" + info(s"Deleting zookeeper service node:$nodePath") + try { + discoveryClient.delete(nodePath) + deletedNodes += node + } catch { + case e: Exception => + error(s"Failed to delete zookeeper service node:$nodePath", e) + } + } + deletedNodes + } + } +} diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/get/GetCommand.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/get/GetCommand.scala index d78f0b995..5b7ada27d 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/get/GetCommand.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/get/GetCommand.scala @@ -18,10 +18,11 @@ package org.apache.kyuubi.ctl.cmd.get import org.apache.kyuubi.ctl.cmd.Command import org.apache.kyuubi.ctl.opt.CliConfig -import org.apache.kyuubi.ctl.util.{CtlUtils, Render, Validator} +import org.apache.kyuubi.ctl.util.{Render, Validator} import org.apache.kyuubi.ha.client.ServiceNodeInfo -class GetCommand(cliConfig: CliConfig) extends Command[Seq[ServiceNodeInfo]](cliConfig) { +abstract class GetCommand(cliConfig: CliConfig) + extends Command[Iterable[ServiceNodeInfo]](cliConfig) { def validate(): Unit = { Validator.validateZkArguments(normalizedCliConfig) @@ -29,11 +30,9 @@ class GetCommand(cliConfig: CliConfig) extends Command[Seq[ServiceNodeInfo]](cli mergeArgsIntoKyuubiConf() } - def doRun(): Seq[ServiceNodeInfo] = { - CtlUtils.listZkServerNodes(conf, normalizedCliConfig, filterHostPort = true) - } + override def doRun(): Iterable[ServiceNodeInfo] - def render(nodes: Seq[ServiceNodeInfo]): Unit = { + override def render(nodes: Iterable[ServiceNodeInfo]): Unit = { val title = "Zookeeper service nodes" info(Render.renderServiceNodesInfo(title, nodes)) } diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/get/GetEngineCommand.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/get/GetEngineCommand.scala index 4d9101625..0d3018372 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/get/GetEngineCommand.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/get/GetEngineCommand.scala @@ -17,6 +17,8 @@ package org.apache.kyuubi.ctl.cmd.get import org.apache.kyuubi.ctl.opt.CliConfig +import org.apache.kyuubi.ctl.util.CtlUtils +import org.apache.kyuubi.ha.client.ServiceNodeInfo class GetEngineCommand(cliConfig: CliConfig) extends GetCommand(cliConfig) { @@ -28,4 +30,12 @@ class GetEngineCommand(cliConfig: CliConfig) extends GetCommand(cliConfig) { fail("Must specify user name for engine, please use -u or --user.") } } + + override def doRun(): Iterable[ServiceNodeInfo] = { + CtlUtils.listZkEngineNodes( + conf, + normalizedCliConfig, + Some((cliConfig.zkOpts.host, cliConfig.zkOpts.port.toInt))) + } + } diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/get/GetServerCommand.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/get/GetServerCommand.scala index 71b868453..744655fd9 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/get/GetServerCommand.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/get/GetServerCommand.scala @@ -17,5 +17,14 @@ package org.apache.kyuubi.ctl.cmd.get import org.apache.kyuubi.ctl.opt.CliConfig +import org.apache.kyuubi.ctl.util.CtlUtils +import org.apache.kyuubi.ha.client.ServiceNodeInfo -class GetServerCommand(cliConfig: CliConfig) extends GetCommand(cliConfig) {} +class GetServerCommand(cliConfig: CliConfig) extends GetCommand(cliConfig) { + override def doRun(): Iterable[ServiceNodeInfo] = { + CtlUtils.listZkServerNodes( + conf, + normalizedCliConfig, + Some((cliConfig.zkOpts.host, cliConfig.zkOpts.port.toInt))) + } +} diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/AdminListEngineCommand.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/AdminListEngineCommand.scala index bc0b16e67..96be5cc47 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/AdminListEngineCommand.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/AdminListEngineCommand.scala @@ -26,22 +26,24 @@ import org.apache.kyuubi.ctl.cmd.AdminCtlCommand import org.apache.kyuubi.ctl.opt.CliConfig import org.apache.kyuubi.ctl.util.Render -class AdminListEngineCommand(cliConfig: CliConfig) extends AdminCtlCommand[Seq[Engine]](cliConfig) { +class AdminListEngineCommand(cliConfig: CliConfig) + extends AdminCtlCommand[Iterable[Engine]](cliConfig) { override def validate(): Unit = {} - def doRun(): Seq[Engine] = { + override def doRun(): Iterable[Engine] = { withKyuubiRestClient(normalizedCliConfig, null, conf) { kyuubiRestClient => val adminRestApi = new AdminRestApi(kyuubiRestClient) adminRestApi.listEngines( normalizedCliConfig.engineOpts.engineType, normalizedCliConfig.engineOpts.engineShareLevel, normalizedCliConfig.engineOpts.engineSubdomain, - normalizedCliConfig.commonOpts.hs2ProxyUser).asScala + normalizedCliConfig.commonOpts.hs2ProxyUser, + normalizedCliConfig.engineOpts.all).asScala } } - def render(resp: Seq[Engine]): Unit = { + override def render(resp: Iterable[Engine]): Unit = { info(Render.renderEngineNodesInfo(resp)) } } diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/AdminListServerCommand.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/AdminListServerCommand.scala new file mode 100644 index 000000000..27471f6ad --- /dev/null +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/AdminListServerCommand.scala @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.ctl.cmd.list + +import scala.collection.JavaConverters._ + +import org.apache.kyuubi.client.AdminRestApi +import org.apache.kyuubi.client.api.v1.dto.ServerData +import org.apache.kyuubi.ctl.RestClientFactory.withKyuubiRestClient +import org.apache.kyuubi.ctl.cmd.AdminCtlCommand +import org.apache.kyuubi.ctl.opt.CliConfig +import org.apache.kyuubi.ctl.util.Render + +class AdminListServerCommand(cliConfig: CliConfig) + extends AdminCtlCommand[Iterable[ServerData]](cliConfig) { + + override def validate(): Unit = {} + + override def doRun(): Iterable[ServerData] = { + withKyuubiRestClient(normalizedCliConfig, null, conf) { kyuubiRestClient => + val adminRestApi = new AdminRestApi(kyuubiRestClient) + adminRestApi.listServers().asScala + } + } + + override def render(resp: Iterable[ServerData]): Unit = { + info(Render.renderServerNodesInfo(resp)) + } +} diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/ListBatchCommand.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/ListBatchCommand.scala index 4ce1b49b2..db781da38 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/ListBatchCommand.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/ListBatchCommand.scala @@ -46,6 +46,7 @@ class ListBatchCommand(cliConfig: CliConfig) extends Command[GetBatchesResponse] batchOpts.batchType, batchOpts.batchUser, batchOpts.batchState, + batchOpts.batchName, batchOpts.createTime, batchOpts.endTime, if (batchOpts.from < 0) 0 else batchOpts.from, diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/ListCommand.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/ListCommand.scala index 0cfeb8e4e..95399a2c7 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/ListCommand.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/ListCommand.scala @@ -18,21 +18,20 @@ package org.apache.kyuubi.ctl.cmd.list import org.apache.kyuubi.ctl.cmd.Command import org.apache.kyuubi.ctl.opt.CliConfig -import org.apache.kyuubi.ctl.util.{CtlUtils, Render, Validator} +import org.apache.kyuubi.ctl.util.{Render, Validator} import org.apache.kyuubi.ha.client.ServiceNodeInfo -class ListCommand(cliConfig: CliConfig) extends Command[Seq[ServiceNodeInfo]](cliConfig) { +abstract class ListCommand(cliConfig: CliConfig) + extends Command[Iterable[ServiceNodeInfo]](cliConfig) { def validate(): Unit = { Validator.validateZkArguments(normalizedCliConfig) mergeArgsIntoKyuubiConf() } - def doRun(): Seq[ServiceNodeInfo] = { - CtlUtils.listZkServerNodes(conf, normalizedCliConfig, filterHostPort = false) - } + override def doRun(): Iterable[ServiceNodeInfo] - def render(nodes: Seq[ServiceNodeInfo]): Unit = { + override def render(nodes: Iterable[ServiceNodeInfo]): Unit = { val title = "Zookeeper service nodes" info(Render.renderServiceNodesInfo(title, nodes)) } diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/ListEngineCommand.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/ListEngineCommand.scala index 6a78a9e97..8a26b4cc9 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/ListEngineCommand.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/ListEngineCommand.scala @@ -17,6 +17,8 @@ package org.apache.kyuubi.ctl.cmd.list import org.apache.kyuubi.ctl.opt.CliConfig +import org.apache.kyuubi.ctl.util.CtlUtils +import org.apache.kyuubi.ha.client.ServiceNodeInfo class ListEngineCommand(cliConfig: CliConfig) extends ListCommand(cliConfig) { @@ -28,4 +30,7 @@ class ListEngineCommand(cliConfig: CliConfig) extends ListCommand(cliConfig) { fail("Must specify user name for engine, please use -u or --user.") } } + + override def doRun(): Seq[ServiceNodeInfo] = + CtlUtils.listZkEngineNodes(conf, normalizedCliConfig, None) } diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/ListServerCommand.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/ListServerCommand.scala index 8c3219ece..e6c8d6ad3 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/ListServerCommand.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/ListServerCommand.scala @@ -17,5 +17,11 @@ package org.apache.kyuubi.ctl.cmd.list import org.apache.kyuubi.ctl.opt.CliConfig +import org.apache.kyuubi.ctl.util.CtlUtils +import org.apache.kyuubi.ha.client.ServiceNodeInfo -class ListServerCommand(cliConfig: CliConfig) extends ListCommand(cliConfig) {} +class ListServerCommand(cliConfig: CliConfig) extends ListCommand(cliConfig) { + override def doRun(): Iterable[ServiceNodeInfo] = { + CtlUtils.listZkServerNodes(conf, normalizedCliConfig, None) + } +} diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/ListSessionCommand.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/ListSessionCommand.scala index 7a3668876..9d1dfead4 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/ListSessionCommand.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/list/ListSessionCommand.scala @@ -26,18 +26,18 @@ import org.apache.kyuubi.ctl.cmd.Command import org.apache.kyuubi.ctl.opt.CliConfig import org.apache.kyuubi.ctl.util.Render -class ListSessionCommand(cliConfig: CliConfig) extends Command[Seq[SessionData]](cliConfig) { +class ListSessionCommand(cliConfig: CliConfig) extends Command[Iterable[SessionData]](cliConfig) { override def validate(): Unit = {} - def doRun(): Seq[SessionData] = { + override def doRun(): Iterable[SessionData] = { withKyuubiRestClient(normalizedCliConfig, null, conf) { kyuubiRestClient => val sessionRestApi = new SessionRestApi(kyuubiRestClient) sessionRestApi.listSessions.asScala } } - def render(resp: Seq[SessionData]): Unit = { + override def render(resp: Iterable[SessionData]): Unit = { info(Render.renderSessionDataListInfo(resp)) } } diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/refresh/RefreshConfigCommand.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/refresh/RefreshConfigCommand.scala index b658c0e45..1cda224df 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/refresh/RefreshConfigCommand.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/cmd/refresh/RefreshConfigCommand.scala @@ -21,7 +21,7 @@ import org.apache.kyuubi.KyuubiException import org.apache.kyuubi.client.AdminRestApi import org.apache.kyuubi.ctl.RestClientFactory.withKyuubiRestClient import org.apache.kyuubi.ctl.cmd.AdminCtlCommand -import org.apache.kyuubi.ctl.cmd.refresh.RefreshConfigCommandConfigType.{HADOOP_CONF, USER_DEFAULTS_CONF} +import org.apache.kyuubi.ctl.cmd.refresh.RefreshConfigCommandConfigType.{DENY_USERS, HADOOP_CONF, KUBERNETES_CONF, UNLIMITED_USERS, USER_DEFAULTS_CONF} import org.apache.kyuubi.ctl.opt.CliConfig import org.apache.kyuubi.ctl.util.{Tabulator, Validator} @@ -36,6 +36,9 @@ class RefreshConfigCommand(cliConfig: CliConfig) extends AdminCtlCommand[String] normalizedCliConfig.adminConfigOpts.configType match { case HADOOP_CONF => adminRestApi.refreshHadoopConf() case USER_DEFAULTS_CONF => adminRestApi.refreshUserDefaultsConf() + case KUBERNETES_CONF => adminRestApi.refreshKubernetesConf() + case UNLIMITED_USERS => adminRestApi.refreshUnlimitedUsers() + case DENY_USERS => adminRestApi.refreshDenyUsers() case configType => throw new KyuubiException(s"Invalid config type:$configType") } } @@ -48,4 +51,7 @@ class RefreshConfigCommand(cliConfig: CliConfig) extends AdminCtlCommand[String] object RefreshConfigCommandConfigType { final val HADOOP_CONF = "hadoopConf" final val USER_DEFAULTS_CONF = "userDefaultsConf" + final val KUBERNETES_CONF = "kubernetesConf" + final val UNLIMITED_USERS = "unlimitedUsers" + final val DENY_USERS = "denyUsers" } diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/opt/AdminCommandLine.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/opt/AdminCommandLine.scala index 59ad7f5fc..c02826b68 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/opt/AdminCommandLine.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/opt/AdminCommandLine.scala @@ -52,7 +52,7 @@ object AdminCommandLine extends CommonCommandLine { .text("\tDelete resources.") .action((_, c) => c.copy(action = ControlAction.DELETE)) .children( - engineCmd(builder).text("\tDelete the specified engine node for user."))) + deleteEngineCmd(builder).text("\tDelete the specified engine node for user."))) } @@ -64,7 +64,8 @@ object AdminCommandLine extends CommonCommandLine { .text("\tList information about resources.") .action((_, c) => c.copy(action = ControlAction.LIST)) .children( - engineCmd(builder).text("\tList all the engine nodes for a user"))) + listEngineCmd(builder).text("\tList the engine nodes"), + serverCmd(builder).text("\tList all the server nodes"))) } @@ -79,7 +80,7 @@ object AdminCommandLine extends CommonCommandLine { refreshConfigCmd(builder).text("\tRefresh the config with specified type."))) } - private def engineCmd(builder: OParserBuilder[CliConfig]): OParser[_, CliConfig] = { + private def deleteEngineCmd(builder: OParserBuilder[CliConfig]): OParser[_, CliConfig] = { import builder._ cmd("engine").action((_, c) => c.copy(resource = ControlObject.ENGINE)) .children( @@ -94,6 +95,29 @@ object AdminCommandLine extends CommonCommandLine { .text("The engine share level this engine belong to.")) } + private def listEngineCmd(builder: OParserBuilder[CliConfig]): OParser[_, CliConfig] = { + import builder._ + cmd("engine").action((_, c) => c.copy(resource = ControlObject.ENGINE)) + .children( + opt[String]("engine-type").abbr("et") + .action((v, c) => c.copy(engineOpts = c.engineOpts.copy(engineType = v))) + .text("The engine type this engine belong to."), + opt[String]("engine-subdomain").abbr("es") + .action((v, c) => c.copy(engineOpts = c.engineOpts.copy(engineSubdomain = v))) + .text("The engine subdomain this engine belong to."), + opt[String]("engine-share-level").abbr("esl") + .action((v, c) => c.copy(engineOpts = c.engineOpts.copy(engineShareLevel = v))) + .text("The engine share level this engine belong to."), + opt[String]("all").abbr("a") + .action((v, c) => c.copy(engineOpts = c.engineOpts.copy(all = v))) + .text("All the engine.")) + } + + private def serverCmd(builder: OParserBuilder[CliConfig]): OParser[_, CliConfig] = { + import builder._ + cmd("server").action((_, c) => c.copy(resource = ControlObject.SERVER)) + } + private def refreshConfigCmd(builder: OParserBuilder[CliConfig]): OParser[_, CliConfig] = { import builder._ cmd("config").action((_, c) => c.copy(resource = ControlObject.CONFIG)) @@ -102,6 +126,7 @@ object AdminCommandLine extends CommonCommandLine { .optional() .action((v, c) => c.copy(adminConfigOpts = c.adminConfigOpts.copy(configType = v))) .text("The valid config type can be one of the following: " + - s"$HADOOP_CONF, $USER_DEFAULTS_CONF.")) + s"$HADOOP_CONF, $USER_DEFAULTS_CONF, $KUBERNETES_CONF, " + + s"$UNLIMITED_USERS, $DENY_USERS.")) } } diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/opt/CliConfig.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/opt/CliConfig.scala index 38284c595..4ccae109c 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/opt/CliConfig.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/opt/CliConfig.scala @@ -66,6 +66,7 @@ case class BatchOpts( batchType: String = null, batchUser: String = null, batchState: String = null, + batchName: String = null, createTime: Long = 0, endTime: Long = 0, from: Int = -1, @@ -76,6 +77,7 @@ case class EngineOpts( user: String = null, engineType: String = null, engineSubdomain: String = null, - engineShareLevel: String = null) + engineShareLevel: String = null, + all: String = null) case class AdminConfigOpts(configType: String = null) diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/opt/CommandLine.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/opt/CommandLine.scala index 478c439a4..271bb06ab 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/opt/CommandLine.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/opt/CommandLine.scala @@ -222,6 +222,9 @@ object CommandLine extends CommonCommandLine { opt[String]("batchState") .action((v, c) => c.copy(batchOpts = c.batchOpts.copy(batchState = v))) .text("Batch state."), + opt[String]("batchName") + .action((v, c) => c.copy(batchOpts = c.batchOpts.copy(batchName = v))) + .text("Batch name."), opt[String]("createTime") .action((v, c) => c.copy(batchOpts = c.batchOpts.copy(createTime = diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/util/CtlUtils.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/util/CtlUtils.scala index fdcc127f1..8ce1d611a 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/util/CtlUtils.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/util/CtlUtils.scala @@ -25,48 +25,35 @@ import org.yaml.snakeyaml.Yaml import org.apache.kyuubi.KyuubiException import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.config.KyuubiConf.{ENGINE_SHARE_LEVEL, ENGINE_SHARE_LEVEL_SUBDOMAIN, ENGINE_TYPE} -import org.apache.kyuubi.ctl.opt.{CliConfig, ControlObject} +import org.apache.kyuubi.ctl.opt.CliConfig import org.apache.kyuubi.ha.client.{DiscoveryClient, DiscoveryPaths, ServiceNodeInfo} import org.apache.kyuubi.ha.client.DiscoveryClientProvider.withDiscoveryClient object CtlUtils { - private[ctl] def getZkNamespace(conf: KyuubiConf, cliConfig: CliConfig): String = { - cliConfig.resource match { - case ControlObject.SERVER => - DiscoveryPaths.makePath(null, cliConfig.zkOpts.namespace) - case ControlObject.ENGINE => - val engineType = Some(cliConfig.engineOpts.engineType) - .filter(_ != null).filter(_.nonEmpty) - .getOrElse(conf.get(ENGINE_TYPE)) - val engineSubdomain = Some(cliConfig.engineOpts.engineSubdomain) - .filter(_ != null).filter(_.nonEmpty) - .getOrElse(conf.get(ENGINE_SHARE_LEVEL_SUBDOMAIN).getOrElse("default")) - val engineShareLevel = Some(cliConfig.engineOpts.engineShareLevel) - .filter(_ != null).filter(_.nonEmpty) - .getOrElse(conf.get(ENGINE_SHARE_LEVEL)) - // The path of the engine defined in zookeeper comes from - // org.apache.kyuubi.engine.EngineRef#engineSpace - DiscoveryPaths.makePath( - s"${cliConfig.zkOpts.namespace}_" + - s"${cliConfig.zkOpts.version}_" + - s"${engineShareLevel}_${engineType}", - cliConfig.engineOpts.user, - engineSubdomain) - } + private[ctl] def getZkServerNamespace(conf: KyuubiConf, cliConfig: CliConfig): String = { + DiscoveryPaths.makePath(null, cliConfig.zkOpts.namespace) } - private[ctl] def getServiceNodes( - discoveryClient: DiscoveryClient, - znodeRoot: String, - hostPortOpt: Option[(String, Int)]): Seq[ServiceNodeInfo] = { - val serviceNodes = discoveryClient.getServiceNodesInfo(znodeRoot) - hostPortOpt match { - case Some((host, port)) => serviceNodes.filter { sn => - sn.host == host && sn.port == port - } - case _ => serviceNodes - } + private[ctl] def getZkEngineNamespaceAndSubdomain( + conf: KyuubiConf, + cliConfig: CliConfig): (String, Option[String]) = { + val engineType = Some(cliConfig.engineOpts.engineType) + .filter(_ != null).filter(_.nonEmpty) + .getOrElse(conf.get(ENGINE_TYPE)) + val engineShareLevel = Some(cliConfig.engineOpts.engineShareLevel) + .filter(_ != null).filter(_.nonEmpty) + .getOrElse(conf.get(ENGINE_SHARE_LEVEL)) + val engineSubdomain = Option(cliConfig.engineOpts.engineSubdomain) + .filter(_.nonEmpty).orElse(conf.get(ENGINE_SHARE_LEVEL_SUBDOMAIN)) + // The path of the engine defined in zookeeper comes from + // org.apache.kyuubi.engine.EngineRef#engineSpace + val rootPath = DiscoveryPaths.makePath( + s"${cliConfig.zkOpts.namespace}_" + + s"${cliConfig.zkOpts.version}_" + + s"${engineShareLevel}_${engineType}", + cliConfig.engineOpts.user) + (rootPath, engineSubdomain) } /** @@ -75,17 +62,41 @@ object CtlUtils { private[ctl] def listZkServerNodes( conf: KyuubiConf, cliConfig: CliConfig, - filterHostPort: Boolean): Seq[ServiceNodeInfo] = { - var nodes = Seq.empty[ServiceNodeInfo] + hostPortOpt: Option[(String, Int)]): Seq[ServiceNodeInfo] = { withDiscoveryClient(conf) { discoveryClient => - val znodeRoot = getZkNamespace(conf, cliConfig) - val hostPortOpt = - if (filterHostPort) { - Some((cliConfig.zkOpts.host, cliConfig.zkOpts.port.toInt)) - } else None - nodes = getServiceNodes(discoveryClient, znodeRoot, hostPortOpt) + val znodeRoot = getZkServerNamespace(conf, cliConfig) + getServiceNodes(discoveryClient, znodeRoot, hostPortOpt) } - nodes + } + + /** + * List Kyuubi engine nodes info. + */ + private[ctl] def listZkEngineNodes( + conf: KyuubiConf, + cliConfig: CliConfig, + hostPortOpt: Option[(String, Int)]): Seq[ServiceNodeInfo] = { + withDiscoveryClient(conf) { discoveryClient => + val (znodeRoot, subdomainOpt) = getZkEngineNamespaceAndSubdomain(conf, cliConfig) + val candidates = discoveryClient.getChildren(znodeRoot) + val matched = subdomainOpt match { + case Some(subdomain) => candidates.filter(_ == subdomain) + case None => candidates + } + matched.flatMap { subdomain => + getServiceNodes(discoveryClient, s"$znodeRoot/$subdomain", hostPortOpt) + } + } + } + + private[ctl] def getServiceNodes( + discoveryClient: DiscoveryClient, + znodeRoot: String, + hostPortOpt: Option[(String, Int)]): Seq[ServiceNodeInfo] = { + val serviceNodes = discoveryClient.getServiceNodesInfo(znodeRoot) + hostPortOpt.map { case (host, port) => + serviceNodes.filter { sn => sn.host == host && sn.port == port } + }.getOrElse(serviceNodes) } private[ctl] def loadYamlAsMap(cliConfig: CliConfig): JMap[String, Object] = { diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/util/Render.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/util/Render.scala index aba6df35a..92db46d88 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/util/Render.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/util/Render.scala @@ -19,21 +19,21 @@ package org.apache.kyuubi.ctl.util import scala.collection.JavaConverters._ import scala.collection.mutable.ListBuffer -import org.apache.kyuubi.client.api.v1.dto.{Batch, Engine, GetBatchesResponse, SessionData} +import org.apache.kyuubi.client.api.v1.dto.{Batch, Engine, GetBatchesResponse, ServerData, SessionData} import org.apache.kyuubi.ctl.util.DateTimeUtils._ import org.apache.kyuubi.ha.client.ServiceNodeInfo private[ctl] object Render { - def renderServiceNodesInfo(title: String, serviceNodeInfo: Seq[ServiceNodeInfo]): String = { + def renderServiceNodesInfo(title: String, serviceNodeInfo: Iterable[ServiceNodeInfo]): String = { val header = Array("Namespace", "Host", "Port", "Version") - val rows = serviceNodeInfo.sortBy(_.nodeName).map { sn => + val rows = serviceNodeInfo.toSeq.sortBy(_.nodeName).map { sn => Array(sn.namespace, sn.host, sn.port.toString, sn.version.getOrElse("")) }.toArray Tabulator.format(title, header, rows) } - def renderEngineNodesInfo(engineNodesInfo: Seq[Engine]): String = { + def renderEngineNodesInfo(engineNodesInfo: Iterable[Engine]): String = { val title = s"Engine Node List (total ${engineNodesInfo.size})" val header = Array("Namespace", "Instance", "Attributes") val rows = engineNodesInfo.map { engine => @@ -45,7 +45,20 @@ private[ctl] object Render { Tabulator.format(title, header, rows) } - def renderSessionDataListInfo(sessions: Seq[SessionData]): String = { + def renderServerNodesInfo(serverNodesInfo: Iterable[ServerData]): String = { + val title = s"Server Node List (total ${serverNodesInfo.size})" + val header = Array("Namespace", "Instance", "Attributes", "Status") + val rows = serverNodesInfo.map { server => + Array( + server.getNamespace, + server.getInstance, + server.getAttributes.asScala.map { case (k, v) => s"$k=$v" }.mkString("\n"), + server.getStatus) + }.toArray + Tabulator.format(title, header, rows) + } + + def renderSessionDataListInfo(sessions: Iterable[SessionData]): String = { val title = s"Live Session List (total ${sessions.size})" val header = Array( "Identifier", @@ -111,6 +124,9 @@ private[ctl] object Render { private def buildBatchAppInfo(batch: Batch, showDiagnostic: Boolean = true): List[String] = { val batchAppInfo = ListBuffer[String]() + batch.getBatchInfo.asScala.foreach { case (key, value) => + batchAppInfo += s"$key: $value" + } if (batch.getAppStartTime > 0) { batchAppInfo += s"App Start Time:" + s" ${millisToDateString(batch.getAppStartTime, "yyyy-MM-dd HH:mm:ss")}" diff --git a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/util/Tabulator.scala b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/util/Tabulator.scala index 704436289..70fed87f6 100644 --- a/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/util/Tabulator.scala +++ b/kyuubi-ctl/src/main/scala/org/apache/kyuubi/ctl/util/Tabulator.scala @@ -23,11 +23,11 @@ import org.apache.commons.lang3.StringUtils private[kyuubi] object Tabulator { def format(title: String, header: Array[String], rows: Array[Array[String]]): String = { val textTable = formatTextTable(header, rows) - val footer = s"${rows.size} row(s)\n" + val footer = s"${rows.length} row(s)\n" if (StringUtils.isBlank(title)) { textTable + footer } else { - val rowWidth = textTable.split("\n").head.size + val rowWidth = textTable.split("\n").head.length val titleNewLine = "\n" + StringUtils.center(title, rowWidth) + "\n" titleNewLine + textTable + footer } diff --git a/kyuubi-ctl/src/test/resources/log4j2-test.xml b/kyuubi-ctl/src/test/resources/log4j2-test.xml index bfc40dd6d..3110216c1 100644 --- a/kyuubi-ctl/src/test/resources/log4j2-test.xml +++ b/kyuubi-ctl/src/test/resources/log4j2-test.xml @@ -21,14 +21,14 @@ - + - + diff --git a/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/AdminControlCliArgumentsSuite.scala b/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/AdminControlCliArgumentsSuite.scala index afb946e92..ae7c0fa1b 100644 --- a/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/AdminControlCliArgumentsSuite.scala +++ b/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/AdminControlCliArgumentsSuite.scala @@ -63,7 +63,7 @@ class AdminControlCliArgumentsSuite extends KyuubiFunSuite with TestPrematureExi val opArgs = new AdminControlCliArguments(args) assert(opArgs.cliConfig.action === ControlAction.REFRESH) assert(opArgs.cliConfig.resource === ControlObject.CONFIG) - assert(opArgs.cliConfig.adminConfigOpts.configType === "hadoopConf") + assert(opArgs.cliConfig.adminConfigOpts.configType === HADOOP_CONF) args = Array( "refresh", @@ -72,7 +72,34 @@ class AdminControlCliArgumentsSuite extends KyuubiFunSuite with TestPrematureExi val opArgs2 = new AdminControlCliArguments(args) assert(opArgs2.cliConfig.action === ControlAction.REFRESH) assert(opArgs2.cliConfig.resource === ControlObject.CONFIG) - assert(opArgs2.cliConfig.adminConfigOpts.configType === "userDefaultsConf") + assert(opArgs2.cliConfig.adminConfigOpts.configType === USER_DEFAULTS_CONF) + + args = Array( + "refresh", + "config", + "unlimitedUsers") + val opArgs3 = new AdminControlCliArguments(args) + assert(opArgs3.cliConfig.action === ControlAction.REFRESH) + assert(opArgs3.cliConfig.resource === ControlObject.CONFIG) + assert(opArgs3.cliConfig.adminConfigOpts.configType === UNLIMITED_USERS) + + args = Array( + "refresh", + "config", + "kubernetesConf") + val opArgs4 = new AdminControlCliArguments(args) + assert(opArgs4.cliConfig.action === ControlAction.REFRESH) + assert(opArgs4.cliConfig.resource === ControlObject.CONFIG) + assert(opArgs4.cliConfig.adminConfigOpts.configType === KUBERNETES_CONF) + + args = Array( + "refresh", + "config", + "denyUsers") + val opArgs5 = new AdminControlCliArguments(args) + assert(opArgs5.cliConfig.action === ControlAction.REFRESH) + assert(opArgs5.cliConfig.resource === ControlObject.CONFIG) + assert(opArgs5.cliConfig.adminConfigOpts.configType === DENY_USERS) args = Array( "refresh", @@ -106,6 +133,13 @@ class AdminControlCliArgumentsSuite extends KyuubiFunSuite with TestPrematureExi } } + test("test list server") { + val args = Array("list", "server") + val opArgs = new AdminControlCliArguments(args) + assert(opArgs.cliConfig.action.toString === "LIST") + assert(opArgs.cliConfig.resource.toString === "SERVER") + } + test("test --help") { // scalastyle:off val helpString = @@ -121,16 +155,19 @@ class AdminControlCliArgumentsSuite extends KyuubiFunSuite with TestPrematureExi | --hs2ProxyUser The value of hive.server2.proxy.user config. | --conf Kyuubi config property pair, formatted key=value. | - |Command: list [engine] + |Command: list [engine|server] | List information about resources. |Command: list engine [options] - | List all the engine nodes for a user + | List the engine nodes | -et, --engine-type | The engine type this engine belong to. | -es, --engine-subdomain | The engine subdomain this engine belong to. | -esl, --engine-share-level | The engine share level this engine belong to. + | -a, --all All the engine. + |Command: list server + | List all the server nodes | |Command: delete [engine] | Delete resources. @@ -147,7 +184,7 @@ class AdminControlCliArgumentsSuite extends KyuubiFunSuite with TestPrematureExi | Refresh the resource. |Command: refresh config [] | Refresh the config with specified type. - | The valid config type can be one of the following: $HADOOP_CONF, $USER_DEFAULTS_CONF. + | The valid config type can be one of the following: $HADOOP_CONF, $USER_DEFAULTS_CONF, $KUBERNETES_CONF, $UNLIMITED_USERS, $DENY_USERS. | | -h, --help Show help message and exit.""".stripMargin // scalastyle:on diff --git a/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/BatchCliArgumentsSuite.scala b/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/BatchCliArgumentsSuite.scala index 7563d985a..bf8f101e0 100644 --- a/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/BatchCliArgumentsSuite.scala +++ b/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/BatchCliArgumentsSuite.scala @@ -84,7 +84,7 @@ class BatchCliArgumentsSuite extends KyuubiFunSuite with TestPrematureExit { "-f", batchYamlFile) val opArgs = new ControlCliArguments(args) - assert(opArgs.cliConfig.batchOpts.waitCompletion == true) + assert(opArgs.cliConfig.batchOpts.waitCompletion) } test("submit batch without waitForCompletion") { @@ -96,7 +96,7 @@ class BatchCliArgumentsSuite extends KyuubiFunSuite with TestPrematureExit { "--waitCompletion", "false") val opArgs = new ControlCliArguments(args) - assert(opArgs.cliConfig.batchOpts.waitCompletion == false) + assert(!opArgs.cliConfig.batchOpts.waitCompletion) } test("get/delete batch") { diff --git a/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/ControlCliArgumentsSuite.scala b/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/ControlCliArgumentsSuite.scala index 1b973c0eb..bd5b2ac45 100644 --- a/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/ControlCliArgumentsSuite.scala +++ b/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/ControlCliArgumentsSuite.scala @@ -429,6 +429,7 @@ class ControlCliArgumentsSuite extends KyuubiFunSuite with TestPrematureExit { | --batchType Batch type. | --batchUser Batch user. | --batchState Batch state. + | --batchName Batch name. | --createTime Batch create time, should be in yyyyMMddHHmmss format. | --endTime Batch end time, should be in yyyyMMddHHmmss format. | --from Specify which record to start from retrieving info. diff --git a/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/ControlCliSuite.scala b/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/ControlCliSuite.scala index d27f3ec2a..43a694a08 100644 --- a/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/ControlCliSuite.scala +++ b/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/ControlCliSuite.scala @@ -199,20 +199,23 @@ class ControlCliSuite extends KyuubiFunSuite with TestPrematureExit { } } - test("test get zk namespace for different service type") { - val arg1 = Array( + test("test get zk server namespace") { + val args = Array( "list", "server", "--zk-quorum", zkServer.getConnectString, "--namespace", namespace) - val scArgs1 = new ControlCliArguments(arg1) - assert(CtlUtils.getZkNamespace( - scArgs1.command.conf, - scArgs1.command.normalizedCliConfig) == s"/$namespace") + val scArgs = new ControlCliArguments(args) + assert( + CtlUtils.getZkServerNamespace( + scArgs.command.conf, + scArgs.command.normalizedCliConfig) === s"/$namespace") + } - val arg2 = Array( + test("test get zk engine namespace") { + val args = Array( "list", "engine", "--zk-quorum", @@ -221,9 +224,11 @@ class ControlCliSuite extends KyuubiFunSuite with TestPrematureExit { namespace, "--user", user) - val scArgs2 = new ControlCliArguments(arg2) - assert(CtlUtils.getZkNamespace(scArgs2.command.conf, scArgs2.command.normalizedCliConfig) == - s"/${namespace}_${KYUUBI_VERSION}_USER_SPARK_SQL/$user/default") + val scArgs = new ControlCliArguments(args) + val expected = (s"/${namespace}_${KYUUBI_VERSION}_USER_SPARK_SQL/$user", None) + assert(CtlUtils.getZkEngineNamespaceAndSubdomain( + scArgs.command.conf, + scArgs.command.normalizedCliConfig) === expected) } test("test list zk service nodes info") { @@ -364,8 +369,10 @@ class ControlCliSuite extends KyuubiFunSuite with TestPrematureExit { "--user", user) val scArgs1 = new ControlCliArguments(arg1) - assert(CtlUtils.getZkNamespace(scArgs1.command.conf, scArgs1.command.normalizedCliConfig) == - s"/${namespace}_${KYUUBI_VERSION}_USER_SPARK_SQL/$user/default") + val expected1 = (s"/${namespace}_${KYUUBI_VERSION}_USER_SPARK_SQL/$user", None) + assert(CtlUtils.getZkEngineNamespaceAndSubdomain( + scArgs1.command.conf, + scArgs1.command.normalizedCliConfig) === expected1) val arg2 = Array( "list", @@ -379,8 +386,10 @@ class ControlCliSuite extends KyuubiFunSuite with TestPrematureExit { "--engine-type", "FLINK_SQL") val scArgs2 = new ControlCliArguments(arg2) - assert(CtlUtils.getZkNamespace(scArgs2.command.conf, scArgs2.command.normalizedCliConfig) == - s"/${namespace}_${KYUUBI_VERSION}_USER_FLINK_SQL/$user/default") + val expected2 = (s"/${namespace}_${KYUUBI_VERSION}_USER_FLINK_SQL/$user", None) + assert(CtlUtils.getZkEngineNamespaceAndSubdomain( + scArgs2.command.conf, + scArgs2.command.normalizedCliConfig) === expected2) val arg3 = Array( "list", @@ -394,8 +403,10 @@ class ControlCliSuite extends KyuubiFunSuite with TestPrematureExit { "--engine-type", "TRINO") val scArgs3 = new ControlCliArguments(arg3) - assert(CtlUtils.getZkNamespace(scArgs3.command.conf, scArgs3.command.normalizedCliConfig) == - s"/${namespace}_${KYUUBI_VERSION}_USER_TRINO/$user/default") + val expected3 = (s"/${namespace}_${KYUUBI_VERSION}_USER_TRINO/$user", None) + assert(CtlUtils.getZkEngineNamespaceAndSubdomain( + scArgs3.command.conf, + scArgs3.command.normalizedCliConfig) === expected3) val arg4 = Array( "list", @@ -411,8 +422,10 @@ class ControlCliSuite extends KyuubiFunSuite with TestPrematureExit { "--engine-subdomain", "sub_1") val scArgs4 = new ControlCliArguments(arg4) - assert(CtlUtils.getZkNamespace(scArgs4.command.conf, scArgs4.command.normalizedCliConfig) == - s"/${namespace}_${KYUUBI_VERSION}_USER_SPARK_SQL/$user/sub_1") + val expected4 = (s"/${namespace}_${KYUUBI_VERSION}_USER_SPARK_SQL/$user", Some("sub_1")) + assert(CtlUtils.getZkEngineNamespaceAndSubdomain( + scArgs4.command.conf, + scArgs4.command.normalizedCliConfig) === expected4) val arg5 = Array( "list", @@ -430,8 +443,10 @@ class ControlCliSuite extends KyuubiFunSuite with TestPrematureExit { "--engine-subdomain", "sub_1") val scArgs5 = new ControlCliArguments(arg5) - assert(CtlUtils.getZkNamespace(scArgs5.command.conf, scArgs5.command.normalizedCliConfig) == - s"/${namespace}_1.5.0_USER_SPARK_SQL/$user/sub_1") + val expected5 = (s"/${namespace}_1.5.0_USER_SPARK_SQL/$user", Some("sub_1")) + assert(CtlUtils.getZkEngineNamespaceAndSubdomain( + scArgs5.command.conf, + scArgs5.command.normalizedCliConfig) === expected5) } test("test get zk namespace for different share level engines") { @@ -445,8 +460,10 @@ class ControlCliSuite extends KyuubiFunSuite with TestPrematureExit { "--user", user) val scArgs1 = new ControlCliArguments(arg1) - assert(CtlUtils.getZkNamespace(scArgs1.command.conf, scArgs1.command.normalizedCliConfig) == - s"/${namespace}_${KYUUBI_VERSION}_USER_SPARK_SQL/$user/default") + val expected1 = (s"/${namespace}_${KYUUBI_VERSION}_USER_SPARK_SQL/$user", None) + assert(CtlUtils.getZkEngineNamespaceAndSubdomain( + scArgs1.command.conf, + scArgs1.command.normalizedCliConfig) === expected1) val arg2 = Array( "list", @@ -460,8 +477,10 @@ class ControlCliSuite extends KyuubiFunSuite with TestPrematureExit { "--engine-share-level", "CONNECTION") val scArgs2 = new ControlCliArguments(arg2) - assert(CtlUtils.getZkNamespace(scArgs2.command.conf, scArgs2.command.normalizedCliConfig) == - s"/${namespace}_${KYUUBI_VERSION}_CONNECTION_SPARK_SQL/$user/default") + val expected2 = (s"/${namespace}_${KYUUBI_VERSION}_CONNECTION_SPARK_SQL/$user", None) + assert(CtlUtils.getZkEngineNamespaceAndSubdomain( + scArgs2.command.conf, + scArgs2.command.normalizedCliConfig) === expected2) val arg3 = Array( "list", @@ -475,8 +494,10 @@ class ControlCliSuite extends KyuubiFunSuite with TestPrematureExit { "--engine-share-level", "USER") val scArgs3 = new ControlCliArguments(arg3) - assert(CtlUtils.getZkNamespace(scArgs3.command.conf, scArgs3.command.normalizedCliConfig) == - s"/${namespace}_${KYUUBI_VERSION}_USER_SPARK_SQL/$user/default") + val expected3 = (s"/${namespace}_${KYUUBI_VERSION}_USER_SPARK_SQL/$user", None) + assert(CtlUtils.getZkEngineNamespaceAndSubdomain( + scArgs3.command.conf, + scArgs3.command.normalizedCliConfig) === expected3) val arg4 = Array( "list", @@ -490,8 +511,10 @@ class ControlCliSuite extends KyuubiFunSuite with TestPrematureExit { "--engine-share-level", "GROUP") val scArgs4 = new ControlCliArguments(arg4) - assert(CtlUtils.getZkNamespace(scArgs4.command.conf, scArgs4.command.normalizedCliConfig) == - s"/${namespace}_${KYUUBI_VERSION}_GROUP_SPARK_SQL/$user/default") + val expected4 = (s"/${namespace}_${KYUUBI_VERSION}_GROUP_SPARK_SQL/$user", None) + assert(CtlUtils.getZkEngineNamespaceAndSubdomain( + scArgs4.command.conf, + scArgs4.command.normalizedCliConfig) === expected4) val arg5 = Array( "list", @@ -505,7 +528,9 @@ class ControlCliSuite extends KyuubiFunSuite with TestPrematureExit { "--engine-share-level", "SERVER") val scArgs5 = new ControlCliArguments(arg5) - assert(CtlUtils.getZkNamespace(scArgs5.command.conf, scArgs5.command.normalizedCliConfig) == - s"/${namespace}_${KYUUBI_VERSION}_SERVER_SPARK_SQL/$user/default") + val expected5 = (s"/${namespace}_${KYUUBI_VERSION}_SERVER_SPARK_SQL/$user", None) + assert(CtlUtils.getZkEngineNamespaceAndSubdomain( + scArgs5.command.conf, + scArgs5.command.normalizedCliConfig) === expected5) } } diff --git a/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/TestPrematureExit.scala b/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/TestPrematureExit.scala index 0e4cc1302..5f8107da7 100644 --- a/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/TestPrematureExit.scala +++ b/kyuubi-ctl/src/test/scala/org/apache/kyuubi/ctl/TestPrematureExit.scala @@ -34,7 +34,7 @@ trait TestPrematureExit { /** Simple PrintStream that reads data into a buffer */ private class BufferPrintStream extends PrintStream(noOpOutputStream) { - var lineBuffer = ArrayBuffer[String]() + val lineBuffer = ArrayBuffer[String]() // scalastyle:off println override def println(line: Any): Unit = { lineBuffer += line.toString @@ -52,11 +52,11 @@ trait TestPrematureExit { @volatile var exitedCleanly = false val original = mainObject.exitFn - mainObject.exitFn = (_) => exitedCleanly = true + mainObject.exitFn = _ => exitedCleanly = true try { @volatile var exception: Exception = null val thread = new Thread { - override def run() = + override def run(): Unit = try { mainObject.main(input) } catch { diff --git a/kyuubi-events/pom.xml b/kyuubi-events/pom.xml index a8030eb83..9b30b5750 100644 --- a/kyuubi-events/pom.xml +++ b/kyuubi-events/pom.xml @@ -21,11 +21,11 @@ org.apache.kyuubi kyuubi-parent - 1.7.0-SNAPSHOT + 1.9.0-SNAPSHOT ../pom.xml - kyuubi-events_2.12 + kyuubi-events_${scala.binary.version} jar Kyuubi Project Events https://kyuubi.apache.org/ @@ -37,6 +37,17 @@ ${project.version} + + org.apache.kyuubi + kyuubi-util-scala_${scala.binary.version} + ${project.version} + + + + org.apache.kafka + kafka-clients + + org.apache.kyuubi kyuubi-common_${scala.binary.version} diff --git a/kyuubi-events/src/main/scala/org/apache/kyuubi/events/EventBus.scala b/kyuubi-events/src/main/scala/org/apache/kyuubi/events/EventBus.scala index e854e40a7..063f1719e 100644 --- a/kyuubi-events/src/main/scala/org/apache/kyuubi/events/EventBus.scala +++ b/kyuubi-events/src/main/scala/org/apache/kyuubi/events/EventBus.scala @@ -40,6 +40,8 @@ sealed trait EventBus { def register[T <: KyuubiEvent: ClassTag](eventHandler: EventHandler[T]): EventBus def registerAsync[T <: KyuubiEvent: ClassTag](eventHandler: EventHandler[T]): EventBus + + def deregisterAll(): Unit = {} } object EventBus extends Logging { @@ -68,6 +70,10 @@ object EventBus extends Logging { def registerAsync[T <: KyuubiEvent: ClassTag](et: EventHandler[T]): EventBus = defaultEventBus.registerAsync[T](et) + def deregisterAll(): Unit = synchronized { + defaultEventBus.deregisterAll() + } + private case class EventBusLive() extends EventBus { private[this] lazy val eventHandlerRegistry = new Registry private[this] lazy val asyncEventHandlerRegistry = new Registry @@ -96,6 +102,11 @@ object EventBus extends Logging { asyncEventHandlerRegistry.register(et) this } + + override def deregisterAll(): Unit = { + eventHandlerRegistry.deregisterAll() + asyncEventHandlerRegistry.deregisterAll() + } } private class Registry { @@ -122,5 +133,10 @@ object EventBus extends Logging { } yield parent clazz :: parents } + + def deregisterAll(): Unit = { + eventHandlers.values.flatten.foreach(_.close()) + eventHandlers.clear() + } } } diff --git a/kyuubi-events/src/main/scala/org/apache/kyuubi/events/EventHandlerRegister.scala b/kyuubi-events/src/main/scala/org/apache/kyuubi/events/EventHandlerRegister.scala index 6c7e0893f..f75e4be4f 100644 --- a/kyuubi-events/src/main/scala/org/apache/kyuubi/events/EventHandlerRegister.scala +++ b/kyuubi-events/src/main/scala/org/apache/kyuubi/events/EventHandlerRegister.scala @@ -51,6 +51,10 @@ trait EventHandlerRegister extends Logging { throw new KyuubiException(s"Unsupported jdbc event logger.") } + protected def createKafkaEventHandler(kyuubiConf: KyuubiConf): EventHandler[KyuubiEvent] = { + throw new KyuubiException(s"Unsupported kafka event logger.") + } + private def loadEventHandler( eventLoggerType: EventLoggerType, kyuubiConf: KyuubiConf): Seq[EventHandler[KyuubiEvent]] = { @@ -64,6 +68,9 @@ trait EventHandlerRegister extends Logging { case EventLoggerType.JDBC => createJdbcEventHandler(kyuubiConf) :: Nil + case EventLoggerType.KAFKA => + createKafkaEventHandler(kyuubiConf) :: Nil + case EventLoggerType.CUSTOM => EventHandlerLoader.loadCustom(kyuubiConf) diff --git a/kyuubi-events/src/main/scala/org/apache/kyuubi/events/EventLoggerType.scala b/kyuubi-events/src/main/scala/org/apache/kyuubi/events/EventLoggerType.scala index a029a0fc5..987982371 100644 --- a/kyuubi-events/src/main/scala/org/apache/kyuubi/events/EventLoggerType.scala +++ b/kyuubi-events/src/main/scala/org/apache/kyuubi/events/EventLoggerType.scala @@ -21,6 +21,5 @@ object EventLoggerType extends Enumeration { type EventLoggerType = Value - // TODO: Only SPARK is done now - val SPARK, JSON, JDBC, CUSTOM = Value + val SPARK, JSON, JDBC, CUSTOM, KAFKA = Value } diff --git a/kyuubi-events/src/main/scala/org/apache/kyuubi/events/handler/EventHandlerLoader.scala b/kyuubi-events/src/main/scala/org/apache/kyuubi/events/handler/EventHandlerLoader.scala index c81dcfb9b..ea4110455 100644 --- a/kyuubi-events/src/main/scala/org/apache/kyuubi/events/handler/EventHandlerLoader.scala +++ b/kyuubi-events/src/main/scala/org/apache/kyuubi/events/handler/EventHandlerLoader.scala @@ -16,40 +16,30 @@ */ package org.apache.kyuubi.events.handler -import java.util.ServiceLoader - -import scala.collection.JavaConverters._ -import scala.collection.mutable.ArrayBuffer import scala.util.{Failure, Success, Try} import org.apache.kyuubi.{Logging, Utils} import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.events.KyuubiEvent +import org.apache.kyuubi.util.reflect.ReflectUtils._ object EventHandlerLoader extends Logging { def loadCustom(kyuubiConf: KyuubiConf): Seq[EventHandler[KyuubiEvent]] = { - val providers = ArrayBuffer[CustomEventHandlerProvider]() - ServiceLoader.load( - classOf[CustomEventHandlerProvider], - Utils.getContextOrKyuubiClassLoader) - .iterator() - .asScala - .foreach(provider => providers += provider) - - providers.map { provider => - Try { - provider.create(kyuubiConf) - } match { - case Success(value) => - value - case Failure(exception) => - warn( - s"Failed to create an EventHandler by the ${provider.getClass.getName}," + - s" it will be ignored.", - exception) - null - } - }.filter(_ != null) + loadFromServiceLoader[CustomEventHandlerProvider](Utils.getContextOrKyuubiClassLoader) + .map { provider => + Try { + provider.create(kyuubiConf) + } match { + case Success(value) => + value + case Failure(exception) => + warn( + s"Failed to create an EventHandler by the ${provider.getClass.getName}," + + s" it will be ignored.", + exception) + null + } + }.filter(_ != null).toSeq } } diff --git a/kyuubi-events/src/main/scala/org/apache/kyuubi/events/handler/JsonLoggingEventHandler.scala b/kyuubi-events/src/main/scala/org/apache/kyuubi/events/handler/JsonLoggingEventHandler.scala index f6f74de9a..77d80b152 100644 --- a/kyuubi-events/src/main/scala/org/apache/kyuubi/events/handler/JsonLoggingEventHandler.scala +++ b/kyuubi-events/src/main/scala/org/apache/kyuubi/events/handler/JsonLoggingEventHandler.scala @@ -65,6 +65,17 @@ class JsonLoggingEventHandler( stream.foreach(_.hflush()) } + override def close(): Unit = { + writers.values.foreach { case (writer, stream) => + writer.flush() + stream.foreach(_.hflush()) + writer.close() + stream.foreach(_.close()) + } + writers.clear() + fs = null + } + private def getOrUpdate(event: KyuubiEvent): Logger = synchronized { val partitions = event.partitions.map(kv => s"${kv._1}=${kv._2}").mkString(Path.SEPARATOR) writers.getOrElseUpdate( @@ -108,6 +119,7 @@ class JsonLoggingEventHandler( } object JsonLoggingEventHandler { - val JSON_LOG_DIR_PERM: FsPermission = new FsPermission(Integer.parseInt("770", 8).toShort) - val JSON_LOG_FILE_PERM: FsPermission = new FsPermission(Integer.parseInt("660", 8).toShort) + private val JSON_LOG_DIR_PERM: FsPermission = new FsPermission(Integer.parseInt("770", 8).toShort) + private val JSON_LOG_FILE_PERM: FsPermission = + new FsPermission(Integer.parseInt("660", 8).toShort) } diff --git a/kyuubi-events/src/main/scala/org/apache/kyuubi/events/handler/KafkaLoggingEventHandler.scala b/kyuubi-events/src/main/scala/org/apache/kyuubi/events/handler/KafkaLoggingEventHandler.scala new file mode 100644 index 000000000..2625f167b --- /dev/null +++ b/kyuubi-events/src/main/scala/org/apache/kyuubi/events/handler/KafkaLoggingEventHandler.scala @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.events.handler + +import java.time.Duration +import java.util.Properties + +import org.apache.kafka.clients.producer.{KafkaProducer, ProducerRecord} + +import org.apache.kyuubi.Logging +import org.apache.kyuubi.config.KyuubiConf +import org.apache.kyuubi.events.KyuubiEvent +import org.apache.kyuubi.events.handler.KafkaLoggingEventHandler._ + +/** + * This event logger logs events to Kafka. + */ +class KafkaLoggingEventHandler( + topic: String, + producerConf: Iterable[(String, String)], + kyuubiConf: KyuubiConf, + closeTimeoutInMs: Long) extends EventHandler[KyuubiEvent] with Logging { + private def defaultProducerConf: Properties = { + val conf = new Properties() + conf.setProperty("key.serializer", DEFAULT_SERIALIZER_CLASS) + conf.setProperty("value.serializer", DEFAULT_SERIALIZER_CLASS) + conf + } + + private val normalizedProducerConf: Properties = { + val conf = defaultProducerConf + producerConf.foreach(p => conf.setProperty(p._1, p._2)) + conf + } + + private val kafkaProducer = new KafkaProducer[String, String](normalizedProducerConf) + + override def apply(event: KyuubiEvent): Unit = { + try { + val record = new ProducerRecord[String, String](topic, event.eventType, event.toJson) + kafkaProducer.send(record) + } catch { + case e: Exception => + error("Failed to send event in KafkaEventHandler", e) + } + } + + override def close(): Unit = { + kafkaProducer.close(Duration.ofMillis(closeTimeoutInMs)) + } +} + +object KafkaLoggingEventHandler { + private val DEFAULT_SERIALIZER_CLASS = "org.apache.kafka.common.serialization.StringSerializer" +} diff --git a/kyuubi-events/src/main/scala/org/apache/kyuubi/events/handler/package.scala b/kyuubi-events/src/main/scala/org/apache/kyuubi/events/handler/package.scala index 41cf001ed..69e1fdcee 100644 --- a/kyuubi-events/src/main/scala/org/apache/kyuubi/events/handler/package.scala +++ b/kyuubi-events/src/main/scala/org/apache/kyuubi/events/handler/package.scala @@ -18,5 +18,9 @@ package org.apache.kyuubi.events package object handler { - type EventHandler[T <: KyuubiEvent] = T => Unit + trait EventHandler[T <: KyuubiEvent] extends AutoCloseable { + def apply(event: T): Unit + + def close(): Unit = {} + } } diff --git a/kyuubi-events/src/test/resources/log4j2-test.xml b/kyuubi-events/src/test/resources/log4j2-test.xml index bfc40dd6d..3110216c1 100644 --- a/kyuubi-events/src/test/resources/log4j2-test.xml +++ b/kyuubi-events/src/test/resources/log4j2-test.xml @@ -21,14 +21,14 @@ - + - + diff --git a/kyuubi-events/src/test/scala/org/apache/kyuubi/events/EventBusSuite.scala b/kyuubi-events/src/test/scala/org/apache/kyuubi/events/EventBusSuite.scala index 9c75766da..0a8563ee4 100644 --- a/kyuubi-events/src/test/scala/org/apache/kyuubi/events/EventBusSuite.scala +++ b/kyuubi-events/src/test/scala/org/apache/kyuubi/events/EventBusSuite.scala @@ -44,29 +44,29 @@ class EventBusSuite extends KyuubiFunSuite { } test("register event handler") { - var test0EventRecievedCount = 0 - var test1EventRecievedCount = 0 - var test2EventRecievedCount = 0 - var testEventRecievedCount = 0 + var test0EventReceivedCount = 0 + var test1EventReceivedCount = 0 + var test2EventReceivedCount = 0 + var testEventReceivedCount = 0 val liveBus = EventBus() liveBus.register[Test0KyuubiEvent] { e => assert(e.content == "test0") assert(e.eventType == "test0_kyuubi") - test0EventRecievedCount += 1 + test0EventReceivedCount += 1 } liveBus.register[Test1KyuubiEvent] { e => assert(e.content == "test1") assert(e.eventType == "test1_kyuubi") - test1EventRecievedCount += 1 + test1EventReceivedCount += 1 } // scribe subclass event liveBus.register[TestKyuubiEvent] { e => assert(e.eventType == "test2_kyuubi") - test2EventRecievedCount += 1 + test2EventReceivedCount += 1 } - liveBus.register[KyuubiEvent] { e => - testEventRecievedCount += 1 + liveBus.register[KyuubiEvent] { _ => + testEventReceivedCount += 1 } class Test0Handler extends EventHandler[Test0KyuubiEvent] { @@ -77,11 +77,9 @@ class EventBusSuite extends KyuubiFunSuite { liveBus.register[Test0KyuubiEvent](new Test0Handler) - liveBus.register[Test1KyuubiEvent](new EventHandler[Test1KyuubiEvent] { - override def apply(e: Test1KyuubiEvent): Unit = { - assert(e.content == "test1") - } - }) + liveBus.register[Test1KyuubiEvent] { e => + assert(e.content == "test1") + } (1 to 10) foreach { _ => liveBus.post(Test0KyuubiEvent("test0")) @@ -92,10 +90,10 @@ class EventBusSuite extends KyuubiFunSuite { (1 to 30) foreach { _ => liveBus.post(Test2KyuubiEvent("name2", "test2")) } - assert(test0EventRecievedCount == 10) - assert(test1EventRecievedCount == 20) - assert(test2EventRecievedCount == 30) - assert(testEventRecievedCount == 60) + assert(test0EventReceivedCount == 10) + assert(test1EventReceivedCount == 20) + assert(test2EventReceivedCount == 30) + assert(testEventReceivedCount == 60) } test("register event handler for default bus") { @@ -120,7 +118,7 @@ class EventBusSuite extends KyuubiFunSuite { test("async event handler") { val countDownLatch = new CountDownLatch(4) - val count = new AtomicInteger(0); + val count = new AtomicInteger(0) class Test0Handler extends EventHandler[Test0KyuubiEvent] { override def apply(e: Test0KyuubiEvent): Unit = { Thread.sleep(10) diff --git a/kyuubi-ha/pom.xml b/kyuubi-ha/pom.xml index 8d7246eff..129f7a53d 100644 --- a/kyuubi-ha/pom.xml +++ b/kyuubi-ha/pom.xml @@ -21,11 +21,11 @@ org.apache.kyuubi kyuubi-parent - 1.7.0-SNAPSHOT + 1.9.0-SNAPSHOT ../pom.xml - kyuubi-ha_2.12 + kyuubi-ha_${scala.binary.version} jar Kyuubi Project High Availability https://kyuubi.apache.org/ @@ -38,18 +38,8 @@ - org.apache.curator - curator-framework - - - - org.apache.curator - curator-recipes - - - - org.apache.zookeeper - zookeeper + org.apache.kyuubi + ${kyuubi-shaded-zookeeper.artifacts} @@ -99,6 +89,12 @@ grpc-stub + + com.dimafeng + testcontainers-scala-scalatest_${scala.binary.version} + test + + io.etcd jetcd-launcher diff --git a/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/HighAvailabilityConf.scala b/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/HighAvailabilityConf.scala index 6052e31f5..626557008 100644 --- a/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/HighAvailabilityConf.scala +++ b/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/HighAvailabilityConf.scala @@ -21,14 +21,13 @@ import java.time.Duration import org.apache.hadoop.security.UserGroupInformation -import org.apache.kyuubi.config.{ConfigBuilder, ConfigEntry, KyuubiConf, OptionalConfigEntry} +import org.apache.kyuubi.config.{ConfigEntry, KyuubiConf, OptionalConfigEntry} +import org.apache.kyuubi.config.KyuubiConf.buildConf import org.apache.kyuubi.ha.client.AuthTypes import org.apache.kyuubi.ha.client.RetryPolicies object HighAvailabilityConf { - private def buildConf(key: String): ConfigBuilder = KyuubiConf.buildConf(key) - @deprecated("using kyuubi.ha.addresses instead", "1.6.0") val HA_ZK_QUORUM: ConfigEntry[String] = buildConf("kyuubi.ha.zookeeper.quorum") .doc("(deprecated) The connection string for the ZooKeeper ensemble") @@ -80,7 +79,7 @@ object HighAvailabilityConf { s"${AuthTypes.values.mkString("
    • ", "
    • ", "
    ")}") .version("1.3.2") .stringConf - .checkValues(AuthTypes.values.map(_.toString)) + .checkValues(AuthTypes) .createWithDefault(AuthTypes.NONE.toString) val HA_ZK_ENGINE_AUTH_TYPE: ConfigEntry[String] = @@ -89,25 +88,36 @@ object HighAvailabilityConf { s"${AuthTypes.values.mkString("
    • ", "
    • ", "
    ")}") .version("1.3.2") .stringConf - .checkValues(AuthTypes.values.map(_.toString)) + .checkValues(AuthTypes) .createWithDefault(AuthTypes.NONE.toString) + val HA_ZK_AUTH_SERVER_PRINCIPAL: OptionalConfigEntry[String] = + buildConf("kyuubi.ha.zookeeper.auth.serverPrincipal") + .doc("Kerberos principal name of ZooKeeper Server. It only takes effect when " + + "Zookeeper client's version at least 3.5.7 or 3.6.0 or applies ZOOKEEPER-1467. " + + "To use Zookeeper 3.6 client, compile Kyuubi with `-Pzookeeper-3.6`.") + .version("1.8.0") + .stringConf + .createOptional + val HA_ZK_AUTH_PRINCIPAL: ConfigEntry[Option[String]] = buildConf("kyuubi.ha.zookeeper.auth.principal") - .doc("Name of the Kerberos principal is used for ZooKeeper authentication.") + .doc("Kerberos principal name that is used for ZooKeeper authentication.") .version("1.3.2") .fallbackConf(KyuubiConf.SERVER_PRINCIPAL) - val HA_ZK_AUTH_KEYTAB: ConfigEntry[Option[String]] = buildConf("kyuubi.ha.zookeeper.auth.keytab") - .doc("Location of the Kyuubi server's keytab is used for ZooKeeper authentication.") - .version("1.3.2") - .fallbackConf(KyuubiConf.SERVER_KEYTAB) + val HA_ZK_AUTH_KEYTAB: ConfigEntry[Option[String]] = + buildConf("kyuubi.ha.zookeeper.auth.keytab") + .doc("Location of the Kyuubi server's keytab that is used for ZooKeeper authentication.") + .version("1.3.2") + .fallbackConf(KyuubiConf.SERVER_KEYTAB) - val HA_ZK_AUTH_DIGEST: OptionalConfigEntry[String] = buildConf("kyuubi.ha.zookeeper.auth.digest") - .doc("The digest auth string is used for ZooKeeper authentication, like: username:password.") - .version("1.3.2") - .stringConf - .createOptional + val HA_ZK_AUTH_DIGEST: OptionalConfigEntry[String] = + buildConf("kyuubi.ha.zookeeper.auth.digest") + .doc("The digest auth string is used for ZooKeeper authentication, like: username:password.") + .version("1.3.2") + .stringConf + .createOptional val HA_ZK_CONN_MAX_RETRIES: ConfigEntry[Int] = buildConf("kyuubi.ha.zookeeper.connection.max.retries") @@ -150,7 +160,7 @@ object HighAvailabilityConf { s" ${RetryPolicies.values.mkString("
    • ", "
    • ", "
    ")}") .version("1.0.0") .stringConf - .checkValues(RetryPolicies.values.map(_.toString)) + .checkValues(RetryPolicies) .createWithDefault(RetryPolicies.EXPONENTIAL_BACKOFF.toString) val HA_ZK_NODE_TIMEOUT: ConfigEntry[Long] = @@ -210,14 +220,14 @@ object HighAvailabilityConf { .stringConf .createOptional - val HA_ETCD_SSL_CLINET_CRT_PATH: OptionalConfigEntry[String] = + val HA_ETCD_SSL_CLIENT_CRT_PATH: OptionalConfigEntry[String] = buildConf("kyuubi.ha.etcd.ssl.client.certificate.path") .doc("Where the etcd SSL certificate file is stored.") .version("1.6.0") .stringConf .createOptional - val HA_ETCD_SSL_CLINET_KEY_PATH: OptionalConfigEntry[String] = + val HA_ETCD_SSL_CLIENT_KEY_PATH: OptionalConfigEntry[String] = buildConf("kyuubi.ha.etcd.ssl.client.key.path") .doc("Where the etcd SSL key file is stored.") .version("1.6.0") diff --git a/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/DiscoveryPaths.scala b/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/DiscoveryPaths.scala index 987a88dda..fe7ebe2ab 100644 --- a/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/DiscoveryPaths.scala +++ b/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/DiscoveryPaths.scala @@ -17,7 +17,7 @@ package org.apache.kyuubi.ha.client -import org.apache.curator.utils.ZKPaths +import org.apache.kyuubi.shaded.curator.utils.ZKPaths object DiscoveryPaths { def makePath(parent: String, firstChild: String, restChildren: String*): String = { diff --git a/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/ServiceDiscovery.scala b/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/ServiceDiscovery.scala index bdb9b12fe..a1b1466d1 100644 --- a/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/ServiceDiscovery.scala +++ b/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/ServiceDiscovery.scala @@ -60,6 +60,7 @@ abstract class ServiceDiscovery( override def start(): Unit = { discoveryClient.registerService(conf, namespace, this) + info(s"Registered $name in namespace ${_namespace}.") super.start() } diff --git a/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/etcd/EtcdDiscoveryClient.scala b/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/etcd/EtcdDiscoveryClient.scala index ad3a0550c..d979804f4 100644 --- a/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/etcd/EtcdDiscoveryClient.scala +++ b/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/etcd/EtcdDiscoveryClient.scala @@ -74,10 +74,10 @@ class EtcdDiscoveryClient(conf: KyuubiConf) extends DiscoveryClient { } else { val caPath = conf.getOption(HA_ETCD_SSL_CA_PATH.key).getOrElse( throw new IllegalArgumentException(s"${HA_ETCD_SSL_CA_PATH.key} is not defined")) - val crtPath = conf.getOption(HA_ETCD_SSL_CLINET_CRT_PATH.key).getOrElse( - throw new IllegalArgumentException(s"${HA_ETCD_SSL_CLINET_CRT_PATH.key} is not defined")) - val keyPath = conf.getOption(HA_ETCD_SSL_CLINET_KEY_PATH.key).getOrElse( - throw new IllegalArgumentException(s"${HA_ETCD_SSL_CLINET_KEY_PATH.key} is not defined")) + val crtPath = conf.getOption(HA_ETCD_SSL_CLIENT_CRT_PATH.key).getOrElse( + throw new IllegalArgumentException(s"${HA_ETCD_SSL_CLIENT_CRT_PATH.key} is not defined")) + val keyPath = conf.getOption(HA_ETCD_SSL_CLIENT_KEY_PATH.key).getOrElse( + throw new IllegalArgumentException(s"${HA_ETCD_SSL_CLIENT_KEY_PATH.key} is not defined")) val context = GrpcSslContexts.forClient() .trustManager(new File(caPath)) @@ -90,7 +90,7 @@ class EtcdDiscoveryClient(conf: KyuubiConf) extends DiscoveryClient { } } - def createClient(): Unit = { + override def createClient(): Unit = { client = buildClient() kvClient = client.getKVClient() lockClient = client.getLockClient() @@ -99,13 +99,13 @@ class EtcdDiscoveryClient(conf: KyuubiConf) extends DiscoveryClient { leaseTTL = conf.get(HighAvailabilityConf.HA_ETCD_LEASE_TIMEOUT) / 1000 } - def closeClient(): Unit = { + override def closeClient(): Unit = { if (client != null) { client.close() } } - def create(path: String, mode: String, createParent: Boolean = true): String = { + override def create(path: String, mode: String, createParent: Boolean = true): String = { // createParent can not effect here mode match { case "PERSISTENT" => kvClient.put( @@ -116,7 +116,7 @@ class EtcdDiscoveryClient(conf: KyuubiConf) extends DiscoveryClient { path } - def getData(path: String): Array[Byte] = { + override def getData(path: String): Array[Byte] = { val response = kvClient.get(ByteSequence.from(path.getBytes())).get() if (response.getKvs.isEmpty) { throw new KyuubiException(s"Key[$path] not exists in ETCD, please check it.") @@ -125,12 +125,12 @@ class EtcdDiscoveryClient(conf: KyuubiConf) extends DiscoveryClient { } } - def setData(path: String, data: Array[Byte]): Boolean = { + override def setData(path: String, data: Array[Byte]): Boolean = { val response = kvClient.put(ByteSequence.from(path.getBytes), ByteSequence.from(data)).get() response != null } - def getChildren(path: String): List[String] = { + override def getChildren(path: String): List[String] = { val kvs = kvClient.get( ByteSequence.from(path.getBytes()), GetOption.newBuilder().isPrefix(true).build()).get().getKvs @@ -142,25 +142,25 @@ class EtcdDiscoveryClient(conf: KyuubiConf) extends DiscoveryClient { } } - def pathExists(path: String): Boolean = { + override def pathExists(path: String): Boolean = { !pathNonExists(path) } - def pathNonExists(path: String): Boolean = { + override def pathNonExists(path: String): Boolean = { kvClient.get(ByteSequence.from(path.getBytes())).get().getKvs.isEmpty } - def delete(path: String, deleteChildren: Boolean = false): Unit = { + override def delete(path: String, deleteChildren: Boolean = false): Unit = { kvClient.delete( ByteSequence.from(path.getBytes()), DeleteOption.newBuilder().isPrefix(deleteChildren).build()).get() } - def monitorState(serviceDiscovery: ServiceDiscovery): Unit = { + override def monitorState(serviceDiscovery: ServiceDiscovery): Unit = { // not need with etcd } - def tryWithLock[T]( + override def tryWithLock[T]( lockPath: String, timeout: Long)(f: => T): T = { // the default unit is millis, covert to seconds. @@ -195,7 +195,7 @@ class EtcdDiscoveryClient(conf: KyuubiConf) extends DiscoveryClient { } } - def getServerHost(namespace: String): Option[(String, Int)] = { + override def getServerHost(namespace: String): Option[(String, Int)] = { // TODO: use last one because to avoid touching some maybe-crashed engines // We need a big improvement here. getServiceNodesInfo(namespace, Some(1), silent = true) match { @@ -204,7 +204,7 @@ class EtcdDiscoveryClient(conf: KyuubiConf) extends DiscoveryClient { } } - def getEngineByRefId( + override def getEngineByRefId( namespace: String, engineRefId: String): Option[(String, Int)] = { getServiceNodesInfo(namespace, silent = true) @@ -212,7 +212,7 @@ class EtcdDiscoveryClient(conf: KyuubiConf) extends DiscoveryClient { .map(data => (data.host, data.port)) } - def getServiceNodesInfo( + override def getServiceNodesInfo( namespace: String, sizeOpt: Option[Int] = None, silent: Boolean = false): Seq[ServiceNodeInfo] = { @@ -241,7 +241,7 @@ class EtcdDiscoveryClient(conf: KyuubiConf) extends DiscoveryClient { } } - def registerService( + override def registerService( conf: KyuubiConf, namespace: String, serviceDiscovery: ServiceDiscovery, @@ -267,7 +267,7 @@ class EtcdDiscoveryClient(conf: KyuubiConf) extends DiscoveryClient { } } - def deregisterService(): Unit = { + override def deregisterService(): Unit = { // close the EPHEMERAL_SEQUENTIAL node in etcd if (serviceNode != null) { if (serviceNode.lease != LEASE_NULL_VALUE) { @@ -278,7 +278,7 @@ class EtcdDiscoveryClient(conf: KyuubiConf) extends DiscoveryClient { } } - def postDeregisterService(namespace: String): Boolean = { + override def postDeregisterService(namespace: String): Boolean = { if (namespace != null) { delete(DiscoveryPaths.makePath(null, namespace), true) true @@ -287,7 +287,7 @@ class EtcdDiscoveryClient(conf: KyuubiConf) extends DiscoveryClient { } } - def createAndGetServiceNode( + override def createAndGetServiceNode( conf: KyuubiConf, namespace: String, instance: String, @@ -297,7 +297,7 @@ class EtcdDiscoveryClient(conf: KyuubiConf) extends DiscoveryClient { } @VisibleForTesting - def startSecretNode( + override def startSecretNode( createMode: String, basePath: String, initData: String, @@ -307,7 +307,7 @@ class EtcdDiscoveryClient(conf: KyuubiConf) extends DiscoveryClient { ByteSequence.from(initData.getBytes())).get() } - def getAndIncrement(path: String, delta: Int = 1): Int = { + override def getAndIncrement(path: String, delta: Int = 1): Int = { val lockPath = s"${path}_tmp_for_lock" tryWithLock(lockPath, 60 * 1000) { if (pathNonExists(path)) { @@ -358,11 +358,11 @@ class EtcdDiscoveryClient(conf: KyuubiConf) extends DiscoveryClient { client.getLeaseClient.keepAlive( leaseId, new StreamObserver[LeaseKeepAliveResponse] { - override def onNext(v: LeaseKeepAliveResponse): Unit = Unit // do nothing + override def onNext(v: LeaseKeepAliveResponse): Unit = () // do nothing - override def onError(throwable: Throwable): Unit = Unit // do nothing + override def onError(throwable: Throwable): Unit = () // do nothing - override def onCompleted(): Unit = Unit // do nothing + override def onCompleted(): Unit = () // do nothing }) client.getKVClient.put( ByteSequence.from(realPath.getBytes()), @@ -388,7 +388,7 @@ class EtcdDiscoveryClient(conf: KyuubiConf) extends DiscoveryClient { override def onError(throwable: Throwable): Unit = throw new KyuubiException(throwable.getMessage, throwable.getCause) - override def onCompleted(): Unit = Unit + override def onCompleted(): Unit = () } } diff --git a/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperACLProvider.scala b/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperACLProvider.scala index 467c323b7..87ea65c17 100644 --- a/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperACLProvider.scala +++ b/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperACLProvider.scala @@ -17,13 +17,12 @@ package org.apache.kyuubi.ha.client.zookeeper -import org.apache.curator.framework.api.ACLProvider -import org.apache.zookeeper.ZooDefs -import org.apache.zookeeper.data.ACL - import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.ha.HighAvailabilityConf import org.apache.kyuubi.ha.client.AuthTypes +import org.apache.kyuubi.shaded.curator.framework.api.ACLProvider +import org.apache.kyuubi.shaded.zookeeper.ZooDefs +import org.apache.kyuubi.shaded.zookeeper.data.ACL class ZookeeperACLProvider(conf: KyuubiConf) extends ACLProvider { diff --git a/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperClientProvider.scala b/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperClientProvider.scala index 8dd32d6b6..d0749c8d9 100644 --- a/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperClientProvider.scala +++ b/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperClientProvider.scala @@ -18,22 +18,23 @@ package org.apache.kyuubi.ha.client.zookeeper import java.io.{File, IOException} +import java.nio.charset.StandardCharsets import javax.security.auth.login.Configuration import scala.util.Random import com.google.common.annotations.VisibleForTesting -import org.apache.curator.framework.{CuratorFramework, CuratorFrameworkFactory} -import org.apache.curator.retry._ import org.apache.hadoop.security.UserGroupInformation -import org.apache.hadoop.security.token.delegation.ZKDelegationTokenSecretManager.JaasConfiguration import org.apache.kyuubi.Logging import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.ha.HighAvailabilityConf._ import org.apache.kyuubi.ha.client.{AuthTypes, RetryPolicies} import org.apache.kyuubi.ha.client.RetryPolicies._ +import org.apache.kyuubi.shaded.curator.framework.{CuratorFramework, CuratorFrameworkFactory} +import org.apache.kyuubi.shaded.curator.retry._ import org.apache.kyuubi.util.KyuubiHadoopUtils +import org.apache.kyuubi.util.reflect.DynConstructors object ZookeeperClientProvider extends Logging { @@ -65,10 +66,8 @@ object ZookeeperClientProvider extends Logging { .aclProvider(new ZookeeperACLProvider(conf)) .retryPolicy(retryPolicy) - conf.get(HA_ZK_AUTH_DIGEST) match { - case Some(anthString) => - builder.authorization("digest", anthString.getBytes("UTF-8")) - case _ => + conf.get(HA_ZK_AUTH_DIGEST).foreach { authString => + builder.authorization("digest", authString.getBytes(StandardCharsets.UTF_8)) } builder.build() @@ -103,46 +102,51 @@ object ZookeeperClientProvider extends Logging { */ @throws[Exception] def setUpZooKeeperAuth(conf: KyuubiConf): Unit = { - def setupZkAuth(): Unit = { - val keyTabFile = getKeyTabFile(conf) - val maybePrincipal = conf.get(HA_ZK_AUTH_PRINCIPAL) - val kerberized = maybePrincipal.isDefined && keyTabFile.isDefined - if (UserGroupInformation.isSecurityEnabled && kerberized) { - if (!new File(keyTabFile.get).exists()) { - throw new IOException(s"${HA_ZK_AUTH_KEYTAB.key}: $keyTabFile does not exists") + def setupZkAuth(): Unit = (conf.get(HA_ZK_AUTH_PRINCIPAL), getKeyTabFile(conf)) match { + case (Some(principal), Some(keytab)) if UserGroupInformation.isSecurityEnabled => + if (!new File(keytab).exists()) { + throw new IOException(s"${HA_ZK_AUTH_KEYTAB.key}: $keytab does not exists") } System.setProperty("zookeeper.sasl.clientconfig", "KyuubiZooKeeperClient") - var principal = maybePrincipal.get - principal = KyuubiHadoopUtils.getServerPrincipal(principal) - val jaasConf = new JaasConfiguration("KyuubiZooKeeperClient", principal, keyTabFile.get) + conf.get(HA_ZK_AUTH_SERVER_PRINCIPAL).foreach { zkServerPrincipal => + // ZOOKEEPER-1467 allows configuring SPN in client + System.setProperty("zookeeper.server.principal", zkServerPrincipal) + } + val zkClientPrincipal = KyuubiHadoopUtils.getServerPrincipal(principal) + // HDFS-16591 makes breaking change on JaasConfiguration + val jaasConf = DynConstructors.builder() + .impl( // Hadoop 3.3.5 and above + "org.apache.hadoop.security.authentication.util.JaasConfiguration", + classOf[String], + classOf[String], + classOf[String]) + .impl( // Hadoop 3.3.4 and previous + // scalastyle:off + "org.apache.hadoop.security.token.delegation.ZKDelegationTokenSecretManager$JaasConfiguration", + // scalastyle:on + classOf[String], + classOf[String], + classOf[String]) + .build[Configuration]() + .newInstance("KyuubiZooKeeperClient", zkClientPrincipal, keytab) Configuration.setConfiguration(jaasConf) - } + case _ => } - if (conf.get(HA_ENGINE_REF_ID).isEmpty - && AuthTypes.withName(conf.get(HA_ZK_AUTH_TYPE)) == AuthTypes.KERBEROS) { + if (conf.get(HA_ENGINE_REF_ID).isEmpty && + AuthTypes.withName(conf.get(HA_ZK_AUTH_TYPE)) == AuthTypes.KERBEROS) { setupZkAuth() - } else if (conf.get(HA_ENGINE_REF_ID).nonEmpty && AuthTypes - .withName(conf.get(HA_ZK_ENGINE_AUTH_TYPE)) == AuthTypes.KERBEROS) { + } else if (conf.get(HA_ENGINE_REF_ID).nonEmpty && + AuthTypes.withName(conf.get(HA_ZK_ENGINE_AUTH_TYPE)) == AuthTypes.KERBEROS) { setupZkAuth() } - } @VisibleForTesting def getKeyTabFile(conf: KyuubiConf): Option[String] = { - val zkAuthKeytab = conf.get(HA_ZK_AUTH_KEYTAB) - if (zkAuthKeytab.isDefined) { - val zkAuthKeytabPath = zkAuthKeytab.get - val relativeFileName = new File(zkAuthKeytabPath).getName - if (new File(relativeFileName).exists()) { - Some(relativeFileName) - } else { - Some(zkAuthKeytabPath) - } - } else { - None + conf.get(HA_ZK_AUTH_KEYTAB).map { fullPath => + val filename = new File(fullPath).getName + if (new File(filename).exists()) filename else fullPath } } - } diff --git a/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperDiscoveryClient.scala b/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperDiscoveryClient.scala index 1315cf029..2db7d89d6 100644 --- a/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperDiscoveryClient.scala +++ b/kyuubi-ha/src/main/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperDiscoveryClient.scala @@ -25,39 +25,25 @@ import java.util.concurrent.atomic.AtomicBoolean import scala.collection.JavaConverters._ import com.google.common.annotations.VisibleForTesting -import org.apache.curator.framework.CuratorFramework -import org.apache.curator.framework.recipes.atomic.{AtomicValue, DistributedAtomicInteger} -import org.apache.curator.framework.recipes.locks.InterProcessSemaphoreMutex -import org.apache.curator.framework.recipes.nodes.PersistentNode -import org.apache.curator.framework.state.ConnectionState -import org.apache.curator.framework.state.ConnectionState.CONNECTED -import org.apache.curator.framework.state.ConnectionState.LOST -import org.apache.curator.framework.state.ConnectionState.RECONNECTED -import org.apache.curator.framework.state.ConnectionStateListener -import org.apache.curator.retry.RetryForever -import org.apache.curator.utils.ZKPaths -import org.apache.zookeeper.CreateMode -import org.apache.zookeeper.CreateMode.PERSISTENT -import org.apache.zookeeper.KeeperException -import org.apache.zookeeper.KeeperException.NodeExistsException -import org.apache.zookeeper.WatchedEvent -import org.apache.zookeeper.Watcher - -import org.apache.kyuubi.KYUUBI_VERSION -import org.apache.kyuubi.KyuubiException -import org.apache.kyuubi.KyuubiSQLException -import org.apache.kyuubi.Logging + +import org.apache.kyuubi.{KYUUBI_VERSION, KyuubiException, KyuubiSQLException, Logging} import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.config.KyuubiReservedKeys.KYUUBI_ENGINE_ID -import org.apache.kyuubi.ha.HighAvailabilityConf.HA_ENGINE_REF_ID -import org.apache.kyuubi.ha.HighAvailabilityConf.HA_ZK_NODE_TIMEOUT -import org.apache.kyuubi.ha.HighAvailabilityConf.HA_ZK_PUBLISH_CONFIGS -import org.apache.kyuubi.ha.client.DiscoveryClient -import org.apache.kyuubi.ha.client.ServiceDiscovery -import org.apache.kyuubi.ha.client.ServiceNodeInfo -import org.apache.kyuubi.ha.client.zookeeper.ZookeeperClientProvider.buildZookeeperClient -import org.apache.kyuubi.ha.client.zookeeper.ZookeeperClientProvider.getGracefulStopThreadDelay +import org.apache.kyuubi.ha.HighAvailabilityConf.{HA_ENGINE_REF_ID, HA_ZK_NODE_TIMEOUT, HA_ZK_PUBLISH_CONFIGS} +import org.apache.kyuubi.ha.client.{DiscoveryClient, ServiceDiscovery, ServiceNodeInfo} +import org.apache.kyuubi.ha.client.zookeeper.ZookeeperClientProvider.{buildZookeeperClient, getGracefulStopThreadDelay} import org.apache.kyuubi.ha.client.zookeeper.ZookeeperDiscoveryClient.connectionChecker +import org.apache.kyuubi.shaded.curator.framework.CuratorFramework +import org.apache.kyuubi.shaded.curator.framework.recipes.atomic.{AtomicValue, DistributedAtomicInteger} +import org.apache.kyuubi.shaded.curator.framework.recipes.locks.InterProcessSemaphoreMutex +import org.apache.kyuubi.shaded.curator.framework.recipes.nodes.PersistentNode +import org.apache.kyuubi.shaded.curator.framework.state.{ConnectionState, ConnectionStateListener} +import org.apache.kyuubi.shaded.curator.framework.state.ConnectionState.{CONNECTED, LOST, RECONNECTED} +import org.apache.kyuubi.shaded.curator.retry.RetryForever +import org.apache.kyuubi.shaded.curator.utils.ZKPaths +import org.apache.kyuubi.shaded.zookeeper.{CreateMode, KeeperException, WatchedEvent, Watcher} +import org.apache.kyuubi.shaded.zookeeper.CreateMode.PERSISTENT +import org.apache.kyuubi.shaded.zookeeper.KeeperException.NodeExistsException import org.apache.kyuubi.util.ThreadUtils class ZookeeperDiscoveryClient(conf: KyuubiConf) extends DiscoveryClient { @@ -66,17 +52,17 @@ class ZookeeperDiscoveryClient(conf: KyuubiConf) extends DiscoveryClient { @volatile private var serviceNode: PersistentNode = _ private var watcher: DeRegisterWatcher = _ - def createClient(): Unit = { + override def createClient(): Unit = { zkClient.start() } - def closeClient(): Unit = { + override def closeClient(): Unit = { if (zkClient != null) { zkClient.close() } } - def create(path: String, mode: String, createParent: Boolean = true): String = { + override def create(path: String, mode: String, createParent: Boolean = true): String = { val builder = if (createParent) zkClient.create().creatingParentsIfNeeded() else zkClient.create() builder @@ -84,27 +70,27 @@ class ZookeeperDiscoveryClient(conf: KyuubiConf) extends DiscoveryClient { .forPath(path) } - def getData(path: String): Array[Byte] = { + override def getData(path: String): Array[Byte] = { zkClient.getData.forPath(path) } - def setData(path: String, data: Array[Byte]): Boolean = { + override def setData(path: String, data: Array[Byte]): Boolean = { zkClient.setData().forPath(path, data) != null } - def getChildren(path: String): List[String] = { + override def getChildren(path: String): List[String] = { zkClient.getChildren.forPath(path).asScala.toList } - def pathExists(path: String): Boolean = { + override def pathExists(path: String): Boolean = { zkClient.checkExists().forPath(path) != null } - def pathNonExists(path: String): Boolean = { + override def pathNonExists(path: String): Boolean = { zkClient.checkExists().forPath(path) == null } - def delete(path: String, deleteChildren: Boolean = false): Unit = { + override def delete(path: String, deleteChildren: Boolean = false): Unit = { if (deleteChildren) { zkClient.delete().deletingChildrenIfNeeded().forPath(path) } else { @@ -112,7 +98,7 @@ class ZookeeperDiscoveryClient(conf: KyuubiConf) extends DiscoveryClient { } } - def monitorState(serviceDiscovery: ServiceDiscovery): Unit = { + override def monitorState(serviceDiscovery: ServiceDiscovery): Unit = { zkClient .getConnectionStateListenable.addListener(new ConnectionStateListener { private val isConnected = new AtomicBoolean(false) @@ -141,7 +127,7 @@ class ZookeeperDiscoveryClient(conf: KyuubiConf) extends DiscoveryClient { }) } - def tryWithLock[T](lockPath: String, timeout: Long)(f: => T): T = { + override def tryWithLock[T](lockPath: String, timeout: Long)(f: => T): T = { var lock: InterProcessSemaphoreMutex = null try { try { @@ -189,7 +175,7 @@ class ZookeeperDiscoveryClient(conf: KyuubiConf) extends DiscoveryClient { } } - def getServerHost(namespace: String): Option[(String, Int)] = { + override def getServerHost(namespace: String): Option[(String, Int)] = { // TODO: use last one because to avoid touching some maybe-crashed engines // We need a big improvement here. getServiceNodesInfo(namespace, Some(1), silent = true) match { @@ -198,7 +184,7 @@ class ZookeeperDiscoveryClient(conf: KyuubiConf) extends DiscoveryClient { } } - def getEngineByRefId( + override def getEngineByRefId( namespace: String, engineRefId: String): Option[(String, Int)] = { getServiceNodesInfo(namespace, silent = true) @@ -206,7 +192,7 @@ class ZookeeperDiscoveryClient(conf: KyuubiConf) extends DiscoveryClient { .map(data => (data.host, data.port)) } - def getServiceNodesInfo( + override def getServiceNodesInfo( namespace: String, sizeOpt: Option[Int] = None, silent: Boolean = false): Seq[ServiceNodeInfo] = { @@ -226,7 +212,7 @@ class ZookeeperDiscoveryClient(conf: KyuubiConf) extends DiscoveryClient { info(s"Get service instance:$instance$engineIdStr and version:${version.getOrElse("")} " + s"under $namespace") ServiceNodeInfo(namespace, p, host, port, version, engineRefId, attributes) - } + }.toSeq } catch { case _: Exception if silent => Nil case e: Exception => @@ -235,7 +221,7 @@ class ZookeeperDiscoveryClient(conf: KyuubiConf) extends DiscoveryClient { } } - def registerService( + override def registerService( conf: KyuubiConf, namespace: String, serviceDiscovery: ServiceDiscovery, @@ -254,7 +240,7 @@ class ZookeeperDiscoveryClient(conf: KyuubiConf) extends DiscoveryClient { watchNode() } - def deregisterService(): Unit = { + override def deregisterService(): Unit = { // close the EPHEMERAL_SEQUENTIAL node in zk if (serviceNode != null) { try { @@ -268,7 +254,7 @@ class ZookeeperDiscoveryClient(conf: KyuubiConf) extends DiscoveryClient { } } - def postDeregisterService(namespace: String): Boolean = { + override def postDeregisterService(namespace: String): Boolean = { if (namespace != null) { try { delete(namespace, true) @@ -283,7 +269,7 @@ class ZookeeperDiscoveryClient(conf: KyuubiConf) extends DiscoveryClient { } } - def createAndGetServiceNode( + override def createAndGetServiceNode( conf: KyuubiConf, namespace: String, instance: String, @@ -293,7 +279,7 @@ class ZookeeperDiscoveryClient(conf: KyuubiConf) extends DiscoveryClient { } @VisibleForTesting - def startSecretNode( + override def startSecretNode( createMode: String, basePath: String, initData: String, @@ -305,9 +291,13 @@ class ZookeeperDiscoveryClient(conf: KyuubiConf) extends DiscoveryClient { basePath, initData.getBytes(StandardCharsets.UTF_8)) secretNode.start() + val znodeTimeout = conf.get(HA_ZK_NODE_TIMEOUT) + if (!secretNode.waitForInitialCreate(znodeTimeout, TimeUnit.MILLISECONDS)) { + throw new KyuubiException(s"Max znode creation wait time $znodeTimeout s exhausted") + } } - def getAndIncrement(path: String, delta: Int = 1): Int = { + override def getAndIncrement(path: String, delta: Int = 1): Int = { val dai = new DistributedAtomicInteger(zkClient, path, new RetryForever(1000)) var atomicVal: AtomicValue[Integer] = null do { diff --git a/kyuubi-ha/src/test/resources/log4j2-test.xml b/kyuubi-ha/src/test/resources/log4j2-test.xml index bfc40dd6d..3110216c1 100644 --- a/kyuubi-ha/src/test/resources/log4j2-test.xml +++ b/kyuubi-ha/src/test/resources/log4j2-test.xml @@ -21,14 +21,14 @@ - + - + diff --git a/kyuubi-ha/src/test/scala/org/apache/kyuubi/ha/client/DiscoveryClientTests.scala b/kyuubi-ha/src/test/scala/org/apache/kyuubi/ha/client/DiscoveryClientTests.scala index 87db340b5..9caf38646 100644 --- a/kyuubi-ha/src/test/scala/org/apache/kyuubi/ha/client/DiscoveryClientTests.scala +++ b/kyuubi-ha/src/test/scala/org/apache/kyuubi/ha/client/DiscoveryClientTests.scala @@ -135,17 +135,17 @@ trait DiscoveryClientTests extends KyuubiFunSuite { new Thread(() => { withDiscoveryClient(conf) { discoveryClient => - discoveryClient.tryWithLock(lockPath, 3000) { + discoveryClient.tryWithLock(lockPath, 10000) { lockLatch.countDown() - Thread.sleep(5000) + Thread.sleep(15000) } } }).start() withDiscoveryClient(conf) { discoveryClient => - assert(lockLatch.await(5000, TimeUnit.MILLISECONDS)) + assert(lockLatch.await(20000, TimeUnit.MILLISECONDS)) val e = intercept[KyuubiSQLException] { - discoveryClient.tryWithLock(lockPath, 2000) {} + discoveryClient.tryWithLock(lockPath, 5000) {} } assert(e.getMessage contains s"Timeout to lock on path [$lockPath]") } @@ -162,7 +162,7 @@ trait DiscoveryClientTests extends KyuubiFunSuite { test("setData method test") { withDiscoveryClient(conf) { discoveryClient => - val data = "abc"; + val data = "abc" val path = "/setData_test" discoveryClient.create(path, "PERSISTENT") discoveryClient.setData(path, data.getBytes) diff --git a/kyuubi-ha/src/test/scala/org/apache/kyuubi/ha/client/etcd/EtcdDiscoveryClientSuite.scala b/kyuubi-ha/src/test/scala/org/apache/kyuubi/ha/client/etcd/EtcdDiscoveryClientSuite.scala index 5b8855c1e..de48a3495 100644 --- a/kyuubi-ha/src/test/scala/org/apache/kyuubi/ha/client/etcd/EtcdDiscoveryClientSuite.scala +++ b/kyuubi-ha/src/test/scala/org/apache/kyuubi/ha/client/etcd/EtcdDiscoveryClientSuite.scala @@ -22,6 +22,9 @@ import java.nio.charset.StandardCharsets import scala.collection.JavaConverters._ import io.etcd.jetcd.launcher.{Etcd, EtcdCluster} +import org.scalactic.source.Position +import org.scalatest.Tag +import org.testcontainers.DockerClientFactory import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.ha.HighAvailabilityConf.{HA_ADDRESSES, HA_CLIENT_CLASS} @@ -41,25 +44,38 @@ class EtcdDiscoveryClientSuite extends DiscoveryClientTests { var conf: KyuubiConf = KyuubiConf() .set(HA_CLIENT_CLASS, classOf[EtcdDiscoveryClient].getName) + private val hasDockerEnv = DockerClientFactory.instance().isDockerAvailable + override def beforeAll(): Unit = { - etcdCluster = new Etcd.Builder() - .withNodes(2) - .build() - etcdCluster.start() - conf = new KyuubiConf() - .set(HA_CLIENT_CLASS, classOf[EtcdDiscoveryClient].getName) - .set(HA_ADDRESSES, getConnectString) + if (hasDockerEnv) { + etcdCluster = new Etcd.Builder() + .withNodes(2) + .build() + etcdCluster.start() + conf = new KyuubiConf() + .set(HA_CLIENT_CLASS, classOf[EtcdDiscoveryClient].getName) + .set(HA_ADDRESSES, getConnectString) + } super.beforeAll() } override def afterAll(): Unit = { super.afterAll() - if (etcdCluster != null) { + if (hasDockerEnv && etcdCluster != null) { etcdCluster.close() etcdCluster = null } } + override protected def test( + testName: String, + testTags: Tag*)(testFun: => Any)(implicit pos: Position): Unit = { + if (hasDockerEnv) { + super.test(testName, testTags: _*)(testFun) + } + // skip test + } + test("etcd test: set, get and delete") { withDiscoveryClient(conf) { discoveryClient => val path = "/kyuubi" diff --git a/kyuubi-ha/src/test/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperDiscoveryClientSuite.scala b/kyuubi-ha/src/test/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperDiscoveryClientSuite.scala index bbd8b94ac..dd78e1fb8 100644 --- a/kyuubi-ha/src/test/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperDiscoveryClientSuite.scala +++ b/kyuubi-ha/src/test/scala/org/apache/kyuubi/ha/client/zookeeper/ZookeeperDiscoveryClientSuite.scala @@ -25,11 +25,7 @@ import javax.security.auth.login.Configuration import scala.collection.JavaConverters._ -import org.apache.curator.framework.CuratorFrameworkFactory -import org.apache.curator.retry.ExponentialBackoffRetry import org.apache.hadoop.util.StringUtils -import org.apache.zookeeper.ZooDefs -import org.apache.zookeeper.data.ACL import org.scalatest.time.SpanSugar._ import org.apache.kyuubi.{KerberizedTestHelper, KYUUBI_VERSION} @@ -37,7 +33,13 @@ import org.apache.kyuubi.config.KyuubiConf import org.apache.kyuubi.ha.HighAvailabilityConf._ import org.apache.kyuubi.ha.client._ import org.apache.kyuubi.ha.client.DiscoveryClientProvider.withDiscoveryClient +import org.apache.kyuubi.ha.client.zookeeper.ZookeeperClientProvider._ import org.apache.kyuubi.service._ +import org.apache.kyuubi.shaded.curator.framework.CuratorFrameworkFactory +import org.apache.kyuubi.shaded.curator.retry.ExponentialBackoffRetry +import org.apache.kyuubi.shaded.zookeeper.ZooDefs +import org.apache.kyuubi.shaded.zookeeper.data.ACL +import org.apache.kyuubi.util.reflect.ReflectUtils._ import org.apache.kyuubi.zookeeper.EmbeddedZookeeper import org.apache.kyuubi.zookeeper.ZookeeperConf.ZK_CLIENT_PORT @@ -117,7 +119,7 @@ abstract class ZookeeperDiscoveryClientSuite extends DiscoveryClientTests conf.set(HA_ZK_AUTH_PRINCIPAL.key, principal) conf.set(HA_ZK_AUTH_TYPE.key, AuthTypes.KERBEROS.toString) - ZookeeperClientProvider.setUpZooKeeperAuth(conf) + setUpZooKeeperAuth(conf) val configuration = Configuration.getConfiguration val entries = configuration.getAppConfigurationEntry("KyuubiZooKeeperClient") @@ -129,9 +131,9 @@ abstract class ZookeeperDiscoveryClientSuite extends DiscoveryClientTests assert(options("useKeyTab").toString.toBoolean) conf.set(HA_ZK_AUTH_KEYTAB.key, s"${keytab.getName}") - val e = intercept[IOException](ZookeeperClientProvider.setUpZooKeeperAuth(conf)) - assert(e.getMessage === - s"${HA_ZK_AUTH_KEYTAB.key}: ${ZookeeperClientProvider.getKeyTabFile(conf)} does not exists") + val e = intercept[IOException](setUpZooKeeperAuth(conf)) + assert( + e.getMessage === s"${HA_ZK_AUTH_KEYTAB.key}: ${getKeyTabFile(conf).get} does not exists") } } @@ -155,12 +157,11 @@ abstract class ZookeeperDiscoveryClientSuite extends DiscoveryClientTests assert(service.getServiceState === ServiceState.STARTED) stopZk() - val isServerLostM = discovery.getClass.getSuperclass.getDeclaredField("isServerLost") - isServerLostM.setAccessible(true) - val isServerLost = isServerLostM.get(discovery) + val isServerLost = + getField[AtomicBoolean]((discovery.getClass.getSuperclass, discovery), "isServerLost") eventually(timeout(10.seconds), interval(100.millis)) { - assert(isServerLost.asInstanceOf[AtomicBoolean].get()) + assert(isServerLost.get()) assert(discovery.getServiceState === ServiceState.STOPPED) assert(service.getServiceState === ServiceState.STOPPED) } diff --git a/kyuubi-hive-beeline/README.md b/kyuubi-hive-beeline/README.md index ec4f86fd7..161acb99b 100644 --- a/kyuubi-hive-beeline/README.md +++ b/kyuubi-hive-beeline/README.md @@ -3,3 +3,4 @@ Aiming to make a better supported beeline for Kyuubi - Support to show launch engine log when getting KyuubiConnection(Done, available since v1.4.0-incubating) + diff --git a/kyuubi-hive-beeline/pom.xml b/kyuubi-hive-beeline/pom.xml index 76753b38d..1068a81ce 100644 --- a/kyuubi-hive-beeline/pom.xml +++ b/kyuubi-hive-beeline/pom.xml @@ -21,7 +21,7 @@ org.apache.kyuubi kyuubi-parent - 1.7.0-SNAPSHOT + 1.9.0-SNAPSHOT kyuubi-hive-beeline @@ -40,6 +40,12 @@ ${project.version}     + + org.apache.kyuubi + kyuubi-util + ${project.version} + + org.apache.hive hive-beeline @@ -115,6 +121,12 @@ commons-io + + org.mockito + mockito-core + test + + commons-lang commons-lang @@ -149,6 +161,11 @@ log4j-slf4j-impl + + org.slf4j + jul-to-slf4j + + org.apache.logging.log4j log4j-api @@ -211,6 +228,14 @@ true     + + + org.apache.maven.plugins + maven-surefire-plugin + + ${skipTests} + + target/classes target/test-classes diff --git a/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/KyuubiBeeLine.java b/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/KyuubiBeeLine.java index 7ca767148..224cbb3ce 100644 --- a/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/KyuubiBeeLine.java +++ b/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/KyuubiBeeLine.java @@ -19,22 +19,45 @@ import java.io.IOException; import java.io.InputStream; -import java.lang.reflect.Field; -import java.lang.reflect.Method; import java.sql.Driver; -import java.util.Arrays; -import java.util.Collections; -import java.util.Iterator; -import java.util.List; +import java.util.*; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; +import org.apache.hive.common.util.HiveStringUtils; +import org.apache.kyuubi.util.reflect.DynConstructors; +import org.apache.kyuubi.util.reflect.DynFields; +import org.apache.kyuubi.util.reflect.DynMethods; public class KyuubiBeeLine extends BeeLine { + + static { + try { + // We use reflection here to handle the case where users remove the + // slf4j-to-jul bridge order to route their logs to JUL. + Class bridgeClass = Class.forName("org.slf4j.bridge.SLF4JBridgeHandler"); + bridgeClass.getMethod("removeHandlersForRootLogger").invoke(null); + boolean installed = (boolean) bridgeClass.getMethod("isInstalled").invoke(null); + if (!installed) { + bridgeClass.getMethod("install").invoke(null); + } + } catch (ReflectiveOperationException cnf) { + // can't log anything yet so just fail silently + } + } + public static final String KYUUBI_BEELINE_DEFAULT_JDBC_DRIVER = "org.apache.kyuubi.jdbc.KyuubiHiveDriver"; protected KyuubiCommands commands = new KyuubiCommands(this); - private Driver defaultDriver = null; + private Driver defaultDriver; + + // copied from org.apache.hive.beeline.BeeLine + private static final int ERRNO_OK = 0; + private static final int ERRNO_ARGS = 1; + private static final int ERRNO_OTHER = 2; + + private static final String PYTHON_MODE_PREFIX = "--python-mode"; + private boolean pythonMode = false; public KyuubiBeeLine() { this(true); @@ -44,25 +67,37 @@ public KyuubiBeeLine() { public KyuubiBeeLine(boolean isBeeLine) { super(isBeeLine); try { - Field commandsField = BeeLine.class.getDeclaredField("commands"); - commandsField.setAccessible(true); - commandsField.set(this, commands); + DynFields.builder().hiddenImpl(BeeLine.class, "commands").buildChecked(this).set(commands); } catch (Throwable t) { throw new ExceptionInInitializerError("Failed to inject kyuubi commands"); } try { defaultDriver = - (Driver) - Class.forName( - KYUUBI_BEELINE_DEFAULT_JDBC_DRIVER, - true, - Thread.currentThread().getContextClassLoader()) - .newInstance(); + DynConstructors.builder() + .impl(KYUUBI_BEELINE_DEFAULT_JDBC_DRIVER) + .buildChecked() + .newInstance(); } catch (Throwable t) { throw new ExceptionInInitializerError(KYUUBI_BEELINE_DEFAULT_JDBC_DRIVER + "-missing"); } } + @Override + void usage() { + super.usage(); + output("Usage: java \" + KyuubiBeeLine.class.getCanonicalName()"); + output(" --python-mode Execute python code/script."); + } + + public boolean isPythonMode() { + return pythonMode; + } + + // Visible for testing + public void setPythonMode(boolean pythonMode) { + this.pythonMode = pythonMode; + } + /** Starts the program. */ public static void main(String[] args) throws IOException { mainWithInputRedirection(args, null); @@ -115,25 +150,37 @@ int initArgs(String[] args) { BeelineParser beelineParser; boolean connSuccessful; boolean exit; - Field exitField; + DynFields.BoundField exitField; try { - Field optionsField = BeeLine.class.getDeclaredField("options"); - optionsField.setAccessible(true); - Options options = (Options) optionsField.get(this); + Options options = + DynFields.builder() + .hiddenImpl(BeeLine.class, "options") + .buildStaticChecked() + .get(); - beelineParser = new BeelineParser(); + beelineParser = + new BeelineParser() { + @SuppressWarnings("rawtypes") + @Override + protected void processOption(String arg, ListIterator iter) throws ParseException { + if (PYTHON_MODE_PREFIX.equals(arg)) { + pythonMode = true; + } else { + super.processOption(arg, iter); + } + } + }; cl = beelineParser.parse(options, args); - Method connectUsingArgsMethod = - BeeLine.class.getDeclaredMethod( - "connectUsingArgs", BeelineParser.class, CommandLine.class); - connectUsingArgsMethod.setAccessible(true); - connSuccessful = (boolean) connectUsingArgsMethod.invoke(this, beelineParser, cl); + connSuccessful = + DynMethods.builder("connectUsingArgs") + .hiddenImpl(BeeLine.class, BeelineParser.class, CommandLine.class) + .buildChecked(this) + .invoke(beelineParser, cl); - exitField = BeeLine.class.getDeclaredField("exit"); - exitField.setAccessible(true); - exit = (boolean) exitField.get(this); + exitField = DynFields.builder().hiddenImpl(BeeLine.class, "exit").buildChecked(this); + exit = exitField.get(); } catch (ParseException e1) { output(e1.getMessage()); @@ -149,10 +196,11 @@ int initArgs(String[] args) { // no-op if the file is not present if (!connSuccessful && !exit) { try { - Method defaultBeelineConnectMethod = - BeeLine.class.getDeclaredMethod("defaultBeelineConnect", CommandLine.class); - defaultBeelineConnectMethod.setAccessible(true); - connSuccessful = (boolean) defaultBeelineConnectMethod.invoke(this, cl); + connSuccessful = + DynMethods.builder("defaultBeelineConnect") + .hiddenImpl(BeeLine.class, CommandLine.class) + .buildChecked(this) + .invoke(cl); } catch (Exception t) { error(t.getMessage()); @@ -160,6 +208,11 @@ int initArgs(String[] args) { } } + // see HIVE-19048 : InitScript errors are ignored + if (exit) { + return 1; + } + int code = 0; if (cl.getOptionValues('e') != null) { commands = Arrays.asList(cl.getOptionValues('e')); @@ -175,8 +228,7 @@ int initArgs(String[] args) { return 1; } if (!commands.isEmpty()) { - for (Iterator i = commands.iterator(); i.hasNext(); ) { - String command = i.next().toString(); + for (String command : commands) { debug(loc("executing-command", command)); if (!dispatch(command)) { code++; @@ -184,7 +236,7 @@ int initArgs(String[] args) { } try { exit = true; - exitField.set(this, exit); + exitField.set(exit); } catch (Exception e) { error(e.getMessage()); return 1; @@ -192,4 +244,59 @@ int initArgs(String[] args) { } return code; } + + // see HIVE-19048 : Initscript errors are ignored + @Override + int runInit() { + String[] initFiles = getOpts().getInitFiles(); + + // executionResult will be ERRNO_OK only if all initFiles execute successfully + int executionResult = ERRNO_OK; + boolean exitOnError = !getOpts().getForce(); + DynFields.BoundField exitField = null; + + if (initFiles != null && initFiles.length != 0) { + for (String initFile : initFiles) { + info("Running init script " + initFile); + try { + int currentResult; + try { + currentResult = + DynMethods.builder("executeFile") + .hiddenImpl(BeeLine.class, String.class) + .buildChecked(this) + .invoke(initFile); + exitField = DynFields.builder().hiddenImpl(BeeLine.class, "exit").buildChecked(this); + } catch (Exception t) { + error(t.getMessage()); + currentResult = ERRNO_OTHER; + } + + if (currentResult != ERRNO_OK) { + executionResult = currentResult; + + if (exitOnError) { + return executionResult; + } + } + } finally { + // exit beeline if there is initScript failure and --force is not set + boolean exit = exitOnError && executionResult != ERRNO_OK; + try { + exitField.set(exit); + } catch (Exception t) { + error(t.getMessage()); + return ERRNO_OTHER; + } + } + } + } + return executionResult; + } + + // see HIVE-15820: comment at the head of beeline -e + @Override + boolean dispatch(String line) { + return super.dispatch(isPythonMode() ? line : HiveStringUtils.removeComments(line)); + } } diff --git a/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/KyuubiCommands.java b/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/KyuubiCommands.java index aaa32739a..fcfee49ed 100644 --- a/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/KyuubiCommands.java +++ b/kyuubi-hive-beeline/src/main/java/org/apache/hive/beeline/KyuubiCommands.java @@ -19,10 +19,13 @@ import static org.apache.kyuubi.jdbc.hive.JdbcConnectionParams.*; +import com.google.common.annotations.VisibleForTesting; import java.io.*; +import java.nio.file.Files; import java.sql.*; import java.util.*; import org.apache.hive.beeline.logs.KyuubiBeelineInPlaceUpdateStream; +import org.apache.hive.common.util.HiveStringUtils; import org.apache.kyuubi.jdbc.hive.KyuubiStatement; import org.apache.kyuubi.jdbc.hive.Utils; import org.apache.kyuubi.jdbc.hive.logs.InPlaceUpdateStream; @@ -43,9 +46,14 @@ public boolean sql(String line) { return execute(line, false, false); } + /** For python mode, keep it as it is. */ + private String trimForNonPythonMode(String line) { + return beeLine.isPythonMode() ? line : line.trim(); + } + /** Extract and clean up the first command in the input. */ private String getFirstCmd(String cmd, int length) { - return cmd.substring(length).trim(); + return trimForNonPythonMode(cmd.substring(length)); } private String[] tokenizeCmd(String cmd) { @@ -79,10 +87,9 @@ private boolean sourceFile(String cmd) { } private boolean sourceFileInternal(File sourceFile) throws IOException { - BufferedReader reader = null; - try { - reader = new BufferedReader(new FileReader(sourceFile)); - String lines = null, extra; + try (BufferedReader reader = Files.newBufferedReader(sourceFile.toPath())) { + String lines = null; + String extra; while ((extra = reader.readLine()) != null) { if (beeLine.isComment(extra)) { continue; @@ -93,16 +100,13 @@ private boolean sourceFileInternal(File sourceFile) throws IOException { lines += "\n" + extra; } } - String[] cmds = lines.split(";"); + String[] cmds = lines.split(beeLine.getOpts().getDelimiter()); for (String c : cmds) { + c = trimForNonPythonMode(c); if (!executeInternal(c, false)) { return false; } } - } finally { - if (reader != null) { - reader.close(); - } } return true; } @@ -258,9 +262,10 @@ private boolean execute(String line, boolean call, boolean entireLineAsCommand) beeLine.handleException(e); } + line = trimForNonPythonMode(line); List cmdList = getCmdList(line, entireLineAsCommand); for (int i = 0; i < cmdList.size(); i++) { - String sql = cmdList.get(i); + String sql = trimForNonPythonMode(cmdList.get(i)); if (sql.length() != 0) { if (!executeInternal(sql, call)) { return false; @@ -276,7 +281,8 @@ private boolean execute(String line, boolean call, boolean entireLineAsCommand) * quotations. It iterates through each character in the line and checks to see if it is a ;, ', * or " */ - private List getCmdList(String line, boolean entireLineAsCommand) { + @VisibleForTesting + public List getCmdList(String line, boolean entireLineAsCommand) { List cmdList = new ArrayList(); if (entireLineAsCommand) { cmdList.add(line); @@ -352,7 +358,7 @@ private List getCmdList(String line, boolean entireLineAsCommand) { */ private void addCmdPart(List cmdList, StringBuilder command, String cmdpart) { if (cmdpart.endsWith("\\")) { - command.append(cmdpart.substring(0, cmdpart.length() - 1)).append(";"); + command.append(cmdpart, 0, cmdpart.length() - 1).append(";"); return; } else { command.append(cmdpart); @@ -417,6 +423,7 @@ private String getProperty(Properties props, String[] keys) { return null; } + @Override public boolean connect(Properties props) throws IOException { String url = getProperty( @@ -462,7 +469,7 @@ public boolean connect(Properties props) throws IOException { beeLine.info("Connecting to " + url); if (Utils.parsePropertyFromUrl(url, AUTH_PRINCIPAL) == null - || Utils.parsePropertyFromUrl(url, AUTH_KYUUBI_SERVER_PRINCIPAL) == null) { + && Utils.parsePropertyFromUrl(url, AUTH_KYUUBI_SERVER_PRINCIPAL) == null) { String urlForPrompt = url.substring(0, url.contains(";") ? url.indexOf(';') : url.length()); if (username == null) { username = beeLine.getConsoleReader().readLine("Enter username for " + urlForPrompt + ": "); @@ -484,7 +491,19 @@ public boolean connect(Properties props) throws IOException { if (!beeLine.isBeeLine()) { beeLine.updateOptsForCli(); } - beeLine.runInit(); + + // see HIVE-19048 : Initscript errors are ignored + int initScriptExecutionResult = beeLine.runInit(); + + // if execution of the init script(s) return anything other than ERRNO_OK from beeline + // exit beeline with error unless --force is set + if (initScriptExecutionResult != 0 && !beeLine.getOpts().getForce()) { + return beeLine.error("init script execution failed."); + } + + if (beeLine.getOpts().getInitFiles() != null) { + beeLine.initializeConsoleReader(null); + } beeLine.setCompletions(); beeLine.getOpts().setLastConnectedUrl(url); @@ -499,12 +518,14 @@ public boolean connect(Properties props) throws IOException { @Override public String handleMultiLineCmd(String line) throws IOException { - int[] startQuote = {-1}; Character mask = (System.getProperty("jline.terminal", "").equals("jline.UnsupportedTerminal")) ? null : jline.console.ConsoleReader.NULL_MASK; + if (!beeLine.isPythonMode()) { + line = HiveStringUtils.removeComments(line); + } while (isMultiLine(line) && beeLine.getOpts().isAllowMultiLineCommand()) { StringBuilder prompt = new StringBuilder(beeLine.getPrompt()); if (!beeLine.getOpts().isSilent()) { @@ -530,6 +551,9 @@ public String handleMultiLineCmd(String line) throws IOException { if (extra == null) { // it happens when using -f and the line of cmds does not end with ; break; } + if (!beeLine.isPythonMode()) { + extra = HiveStringUtils.removeComments(extra); + } if (!extra.isEmpty()) { line += "\n" + extra; } @@ -541,12 +565,13 @@ public String handleMultiLineCmd(String line) throws IOException { // console. Used in handleMultiLineCmd method assumes line would never be null when this method is // called private boolean isMultiLine(String line) { + line = trimForNonPythonMode(line); if (line.endsWith(beeLine.getOpts().getDelimiter()) || beeLine.isComment(line)) { return false; } // handles the case like line = show tables; --test comment List cmds = getCmdList(line, false); - return cmds.isEmpty() || !cmds.get(cmds.size() - 1).startsWith("--"); + return cmds.isEmpty() || !trimForNonPythonMode(cmds.get(cmds.size() - 1)).startsWith("--"); } static class KyuubiLogRunnable implements Runnable { diff --git a/kyuubi-hive-beeline/src/test/java/org/apache/hive/beeline/KyuubiBeeLineTest.java b/kyuubi-hive-beeline/src/test/java/org/apache/hive/beeline/KyuubiBeeLineTest.java index b144c95c6..9c7aec35a 100644 --- a/kyuubi-hive-beeline/src/test/java/org/apache/hive/beeline/KyuubiBeeLineTest.java +++ b/kyuubi-hive-beeline/src/test/java/org/apache/hive/beeline/KyuubiBeeLineTest.java @@ -19,7 +19,12 @@ package org.apache.hive.beeline; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintStream; +import org.apache.kyuubi.util.reflect.DynFields; import org.junit.Test; public class KyuubiBeeLineTest { @@ -29,4 +34,104 @@ public void testKyuubiBeelineWithoutArgs() { int result = kyuubiBeeLine.initArgs(new String[0]); assertEquals(0, result); } + + @Test + public void testKyuubiBeelineExitCodeWithoutConnection() { + KyuubiBeeLine kyuubiBeeLine = new KyuubiBeeLine(); + String scriptFile = getClass().getClassLoader().getResource("test.sql").getFile(); + + String[] args1 = {"-u", "badUrl", "-e", "show tables"}; + int result1 = kyuubiBeeLine.initArgs(args1); + assertEquals(1, result1); + + String[] args2 = {"-u", "badUrl", "-f", scriptFile}; + int result2 = kyuubiBeeLine.initArgs(args2); + assertEquals(1, result2); + + String[] args3 = {"-u", "badUrl", "-i", scriptFile}; + int result3 = kyuubiBeeLine.initArgs(args3); + assertEquals(1, result3); + } + + @Test + public void testKyuubiBeeLineCmdUsage() { + BufferPrintStream printStream = new BufferPrintStream(); + + KyuubiBeeLine kyuubiBeeLine = new KyuubiBeeLine(); + DynFields.builder() + .hiddenImpl(BeeLine.class, "outputStream") + .build(kyuubiBeeLine) + .set(printStream); + String[] args1 = {"-h"}; + kyuubiBeeLine.initArgs(args1); + String output = printStream.getOutput(); + assert output.contains("--python-mode Execute python code/script."); + } + + @Test + public void testKyuubiBeeLinePythonMode() { + KyuubiBeeLine kyuubiBeeLine = new KyuubiBeeLine(); + String[] args1 = {"-u", "badUrl", "--python-mode"}; + kyuubiBeeLine.initArgs(args1); + assertTrue(kyuubiBeeLine.isPythonMode()); + kyuubiBeeLine.setPythonMode(false); + + String[] args2 = {"--python-mode", "-f", "test.sql"}; + kyuubiBeeLine.initArgs(args2); + assertTrue(kyuubiBeeLine.isPythonMode()); + assert kyuubiBeeLine.getOpts().getScriptFile().equals("test.sql"); + kyuubiBeeLine.setPythonMode(false); + + String[] args3 = {"-u", "badUrl"}; + kyuubiBeeLine.initArgs(args3); + assertTrue(!kyuubiBeeLine.isPythonMode()); + kyuubiBeeLine.setPythonMode(false); + } + + @Test + public void testKyuubiBeelineComment() { + KyuubiBeeLine kyuubiBeeLine = new KyuubiBeeLine(); + int result = kyuubiBeeLine.initArgsFromCliVars(new String[] {"-e", "--comment show database;"}); + assertEquals(0, result); + result = kyuubiBeeLine.initArgsFromCliVars(new String[] {"-e", "--comment\n show database;"}); + assertEquals(1, result); + result = + kyuubiBeeLine.initArgsFromCliVars( + new String[] {"-e", "--comment line 1 \n --comment line 2 \n show database;"}); + assertEquals(1, result); + } + + static class BufferPrintStream extends PrintStream { + public StringBuilder stringBuilder = new StringBuilder(); + + static OutputStream noOpOutputStream = + new OutputStream() { + @Override + public void write(int b) throws IOException { + // do nothing + } + }; + + public BufferPrintStream() { + super(noOpOutputStream); + } + + public BufferPrintStream(OutputStream outputStream) { + super(noOpOutputStream); + } + + @Override + public void println(String x) { + stringBuilder.append(x).append("\n"); + } + + @Override + public void print(String x) { + stringBuilder.append(x); + } + + public String getOutput() { + return stringBuilder.toString(); + } + } } diff --git a/kyuubi-hive-beeline/src/test/java/org/apache/hive/beeline/KyuubiCommandsTest.java b/kyuubi-hive-beeline/src/test/java/org/apache/hive/beeline/KyuubiCommandsTest.java new file mode 100644 index 000000000..653d1b08f --- /dev/null +++ b/kyuubi-hive-beeline/src/test/java/org/apache/hive/beeline/KyuubiCommandsTest.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hive.beeline; + +import static org.junit.Assert.assertEquals; + +import java.io.IOException; +import java.util.List; +import jline.console.ConsoleReader; +import org.junit.Test; +import org.mockito.Mockito; + +public class KyuubiCommandsTest { + @Test + public void testParsePythonSnippets() throws IOException { + ConsoleReader reader = Mockito.mock(ConsoleReader.class); + String pythonSnippets = "for i in [1, 2, 3]:\n" + " print(i)\n"; + Mockito.when(reader.readLine()).thenReturn(pythonSnippets); + + KyuubiBeeLine beeline = new KyuubiBeeLine(); + beeline.setPythonMode(true); + beeline.setConsoleReader(reader); + KyuubiCommands commands = new KyuubiCommands(beeline); + String line = commands.handleMultiLineCmd(pythonSnippets); + + List cmdList = commands.getCmdList(line, false); + assertEquals(cmdList.size(), 1); + assertEquals(cmdList.get(0), pythonSnippets); + } + + @Test + public void testHandleMultiLineCmd() throws IOException { + ConsoleReader reader = Mockito.mock(ConsoleReader.class); + String snippets = "select 1;--comments1\nselect 2;--comments2"; + Mockito.when(reader.readLine()).thenReturn(snippets); + + KyuubiBeeLine beeline = new KyuubiBeeLine(); + beeline.setConsoleReader(reader); + beeline.setPythonMode(false); + KyuubiCommands commands = new KyuubiCommands(beeline); + String line = commands.handleMultiLineCmd(snippets); + List cmdList = commands.getCmdList(line, false); + assertEquals(cmdList.size(), 2); + assertEquals(cmdList.get(0), "select 1"); + assertEquals(cmdList.get(1), "\nselect 2"); + + // see HIVE-15820: comment at the head of beeline -e + snippets = "--comments1\nselect 2;--comments2"; + Mockito.when(reader.readLine()).thenReturn(snippets); + line = commands.handleMultiLineCmd(snippets); + cmdList = commands.getCmdList(line, false); + assertEquals(cmdList.size(), 1); + assertEquals(cmdList.get(0), "select 2"); + } +} diff --git a/kyuubi-hive-beeline/src/test/resources/test.sql b/kyuubi-hive-beeline/src/test/resources/test.sql new file mode 100644 index 000000000..c7c3ee2f9 --- /dev/null +++ b/kyuubi-hive-beeline/src/test/resources/test.sql @@ -0,0 +1,17 @@ +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +show tables; diff --git a/kyuubi-hive-jdbc-shaded/pom.xml b/kyuubi-hive-jdbc-shaded/pom.xml index 0bfe88922..174f199be 100644 --- a/kyuubi-hive-jdbc-shaded/pom.xml +++ b/kyuubi-hive-jdbc-shaded/pom.xml @@ -21,7 +21,7 @@ org.apache.kyuubi kyuubi-parent - 1.7.0-SNAPSHOT + 1.9.0-SNAPSHOT kyuubi-hive-jdbc-shaded @@ -108,10 +108,6 @@ org.apache.commons ${kyuubi.shade.packageName}.org.apache.commons - - org.apache.curator - ${kyuubi.shade.packageName}.org.apache.curator - org.apache.hive ${kyuubi.shade.packageName}.org.apache.hive @@ -120,18 +116,10 @@ org.apache.http ${kyuubi.shade.packageName}.org.apache.http - - org.apache.jute - ${kyuubi.shade.packageName}.org.apache.jute - org.apache.thrift ${kyuubi.shade.packageName}.org.apache.thrift - - org.apache.zookeeper - ${kyuubi.shade.packageName}.org.apache.zookeeper -     diff --git a/kyuubi-hive-jdbc/README.md b/kyuubi-hive-jdbc/README.md index 3210e76ac..10a0522dc 100644 --- a/kyuubi-hive-jdbc/README.md +++ b/kyuubi-hive-jdbc/README.md @@ -1,9 +1,9 @@ # Kyuubi Hive JDBC Module - Aiming to make a better supported client for Kyuubi and Spark - Add catalog to getTables meta function for DataLakes (DONE, broken in v1.3.0-incubating, fixed in v1.3.1-incubating) - Deploy to maven central (DONE, available since v1.3.0-incubating) - Create shaded jar (DONE, available since v1.4.0-incubating) - Remove Hive dependencies (DONE, available since v1.6.0-incubating) + diff --git a/kyuubi-hive-jdbc/pom.xml b/kyuubi-hive-jdbc/pom.xml index 4d9648e75..aa5e7c161 100644 --- a/kyuubi-hive-jdbc/pom.xml +++ b/kyuubi-hive-jdbc/pom.xml @@ -21,7 +21,7 @@ org.apache.kyuubi kyuubi-parent - 1.7.0-SNAPSHOT + 1.9.0-SNAPSHOT kyuubi-hive-jdbc @@ -35,6 +35,11 @@ + + org.apache.kyuubi + kyuubi-util + ${project.version} + org.apache.arrow @@ -102,24 +107,14 @@ provided - - org.apache.curator - curator-framework - - - - org.apache.curator - curator-client - - org.apache.httpcomponents httpclient - org.apache.zookeeper - zookeeper + org.apache.kyuubi + ${kyuubi-shaded-zookeeper.artifacts} @@ -171,6 +166,14 @@ + + + + true + src/main/resources + + + org.apache.maven.plugins diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/KyuubiHiveDriver.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/KyuubiHiveDriver.java index 3b874ba2e..66b797087 100644 --- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/KyuubiHiveDriver.java +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/KyuubiHiveDriver.java @@ -24,6 +24,7 @@ import java.util.jar.Attributes; import java.util.jar.Manifest; import java.util.logging.Logger; +import org.apache.commons.lang3.StringUtils; import org.apache.kyuubi.jdbc.hive.JdbcConnectionParams; import org.apache.kyuubi.jdbc.hive.KyuubiConnection; import org.apache.kyuubi.jdbc.hive.KyuubiSQLException; @@ -137,7 +138,7 @@ private Properties parseURLForPropertyInfo(String url, Properties defaults) thro host = ""; } String port = Integer.toString(params.getPort()); - if (host.equals("")) { + if (StringUtils.isEmpty(host)) { port = ""; } else if (port.equals("0") || port.equals("-1")) { port = DEFAULT_PORT; diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/JdbcColumnAttributes.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/JdbcColumnAttributes.java index 06fb39899..b0257cfff 100644 --- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/JdbcColumnAttributes.java +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/JdbcColumnAttributes.java @@ -20,7 +20,7 @@ public class JdbcColumnAttributes { public int precision = 0; public int scale = 0; - public String timeZone = ""; + public String timeZone = null; public JdbcColumnAttributes() {} diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/JdbcConnectionParams.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/JdbcConnectionParams.java index 71949b9df..bcc94e083 100644 --- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/JdbcConnectionParams.java +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/JdbcConnectionParams.java @@ -33,6 +33,7 @@ public class JdbcConnectionParams { // Client param names: + static final String CLIENT_PROTOCOL_VERSION = "clientProtocolVersion"; // Retry setting static final String RETRIES = "retries"; diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiArrowBasedResultSet.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiArrowBasedResultSet.java index c3e75c0ea..ef5008503 100644 --- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiArrowBasedResultSet.java +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiArrowBasedResultSet.java @@ -50,6 +50,7 @@ public abstract class KyuubiArrowBasedResultSet implements SQLResultSet { protected Schema arrowSchema; protected VectorSchemaRoot root; protected ArrowColumnarBatchRow row; + protected boolean timestampAsString = true; protected BufferAllocator allocator; @@ -312,11 +313,18 @@ private Object getColumnValue(int columnIndex) throws SQLException { if (wasNull) { return null; } else { - return row.get(columnIndex - 1, columnType); + JdbcColumnAttributes attributes = columnAttributes.get(columnIndex - 1); + return row.get( + columnIndex - 1, + columnType, + attributes == null ? null : attributes.timeZone, + timestampAsString); } } catch (Exception e) { - e.printStackTrace(); - throw new KyuubiSQLException("Unrecognized column type:", e); + throw new KyuubiSQLException( + String.format( + "Error getting row of type %s at column index %d", columnType, columnIndex - 1), + e); } } diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiArrowQueryResultSet.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiArrowQueryResultSet.java index 1f2af29dc..54491b2d6 100644 --- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiArrowQueryResultSet.java +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiArrowQueryResultSet.java @@ -58,9 +58,6 @@ public class KyuubiArrowQueryResultSet extends KyuubiArrowBasedResultSet { private boolean isScrollable = false; private boolean fetchFirst = false; - // TODO:(fchen) make this configurable - protected boolean convertComplexTypeToString = true; - private final TProtocolVersion protocol; public static class Builder { @@ -87,6 +84,8 @@ public static class Builder { private boolean isScrollable = false; private ReentrantLock transportLock = null; + private boolean timestampAsString = true; + public Builder(Statement statement) throws SQLException { this.statement = statement; this.connection = statement.getConnection(); @@ -153,6 +152,11 @@ public Builder setScrollable(boolean setScrollable) { return this; } + public Builder setTimestampAsString(boolean timestampAsString) { + this.timestampAsString = timestampAsString; + return this; + } + public Builder setTransportLock(ReentrantLock transportLock) { this.transportLock = transportLock; return this; @@ -189,10 +193,10 @@ protected KyuubiArrowQueryResultSet(Builder builder) throws SQLException { this.maxRows = builder.maxRows; } this.isScrollable = builder.isScrollable; + this.timestampAsString = builder.timestampAsString; this.protocol = builder.getProtocolVersion(); arrowSchema = - ArrowUtils.toArrowSchema( - columnNames, convertComplexTypeToStringType(columnTypes), columnAttributes); + ArrowUtils.toArrowSchema(columnNames, convertToStringType(columnTypes), columnAttributes); if (allocator == null) { initArrowSchemaAndAllocator(); } @@ -246,9 +250,6 @@ private void retrieveSchema() throws SQLException { metadataResp = client.GetResultSetMetadata(metadataReq); Utils.verifySuccess(metadataResp.getStatus()); - StringBuilder namesSb = new StringBuilder(); - StringBuilder typesSb = new StringBuilder(); - TTableSchema schema = metadataResp.getSchema(); if (schema == null || !schema.isSetColumns()) { // TODO: should probably throw an exception here. @@ -258,10 +259,6 @@ private void retrieveSchema() throws SQLException { List columns = schema.getColumns(); for (int pos = 0; pos < schema.getColumnsSize(); pos++) { - if (pos != 0) { - namesSb.append(","); - typesSb.append(","); - } String columnName = columns.get(pos).getColumnName(); columnNames.add(columnName); normalizedColumnNames.add(columnName.toLowerCase()); @@ -271,8 +268,7 @@ private void retrieveSchema() throws SQLException { columnAttributes.add(getColumnAttributes(primitiveTypeEntry)); } arrowSchema = - ArrowUtils.toArrowSchema( - columnNames, convertComplexTypeToStringType(columnTypes), columnAttributes); + ArrowUtils.toArrowSchema(columnNames, convertToStringType(columnTypes), columnAttributes); } catch (SQLException eS) { throw eS; // rethrow the SQLException as is } catch (Exception ex) { @@ -480,22 +476,25 @@ public boolean isClosed() { return isClosed; } - private List convertComplexTypeToStringType(List colTypes) { - if (convertComplexTypeToString) { - return colTypes.stream() - .map( - type -> { - if (type == TTypeId.ARRAY_TYPE - || type == TTypeId.MAP_TYPE - || type == TTypeId.STRUCT_TYPE) { - return TTypeId.STRING_TYPE; - } else { - return type; - } - }) - .collect(Collectors.toList()); - } else { - return colTypes; - } + /** + * 1. the complex types (map/array/struct) are always converted to string type to transport 2. if + * the user set `timestampAsString = true`, then the timestamp type will be converted to string + * type too. + */ + private List convertToStringType(List colTypes) { + return colTypes.stream() + .map( + type -> { + if ((type == TTypeId.ARRAY_TYPE + || type == TTypeId.MAP_TYPE + || type == TTypeId.STRUCT_TYPE) // complex type (map/array/struct) + // timestamp type + || (type == TTypeId.TIMESTAMP_TYPE && timestampAsString)) { + return TTypeId.STRING_TYPE; + } else { + return type; + } + }) + .collect(Collectors.toList()); } } diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiConnection.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiConnection.java index 9931dcec2..d3fbbeb6d 100644 --- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiConnection.java +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiConnection.java @@ -30,10 +30,7 @@ import java.net.UnknownHostException; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; -import java.security.AccessControlContext; -import java.security.AccessController; -import java.security.KeyStore; -import java.security.SecureRandom; +import java.security.*; import java.sql.*; import java.util.*; import java.util.Map.Entry; @@ -43,6 +40,7 @@ import javax.net.ssl.TrustManagerFactory; import javax.security.auth.Subject; import javax.security.sasl.Sasl; +import org.apache.commons.lang3.ClassUtils; import org.apache.commons.lang3.StringUtils; import org.apache.hive.service.rpc.thrift.*; import org.apache.http.HttpRequestInterceptor; @@ -106,9 +104,11 @@ public class KyuubiConnection implements SQLConnection, KyuubiLoggable { private Thread engineLogThread; private boolean engineLogInflight = true; private volatile boolean launchEngineOpCompleted = false; + private boolean launchEngineOpSupportResult = false; private String engineId = ""; private String engineName = ""; private String engineUrl = ""; + private String engineRefId = ""; private boolean isBeeLineMode; @@ -733,11 +733,24 @@ private void openSession() throws SQLException { if (sessVars.containsKey(HS2_PROXY_USER)) { openConf.put(HS2_PROXY_USER, sessVars.get(HS2_PROXY_USER)); } + String clientProtocolStr = + sessVars.getOrDefault( + CLIENT_PROTOCOL_VERSION, openReq.getClient_protocol().getValue() + ""); + TProtocolVersion clientProtocol = + TProtocolVersion.findByValue(Integer.parseInt(clientProtocolStr)); + if (clientProtocol == null) { + throw new IllegalArgumentException( + String.format( + "Unsupported Hive2 protocol version %s specified by session conf key %s", + clientProtocolStr, CLIENT_PROTOCOL_VERSION)); + } + openReq.setClient_protocol(clientProtocol); try { openConf.put("kyuubi.client.ipAddress", InetAddress.getLocalHost().getHostAddress()); } catch (UnknownHostException e) { LOG.debug("Error getting Kyuubi session local client ip address", e); } + openConf.put(Utils.KYUUBI_CLIENT_VERSION_KEY, Utils.getVersion()); openReq.setConfiguration(openConf); // Store the user name in the open request in case no non-sasl authentication @@ -770,6 +783,10 @@ private void openSession() throws SQLException { String launchEngineOpHandleSecret = openRespConf.get("kyuubi.session.engine.launch.handle.secret"); + launchEngineOpSupportResult = + Boolean.parseBoolean( + openRespConf.getOrDefault("kyuubi.session.engine.launch.support.result", "false")); + if (launchEngineOpHandleGuid != null && launchEngineOpHandleSecret != null) { try { byte[] guidBytes = Base64.getMimeDecoder().decode(launchEngineOpHandleGuid); @@ -812,11 +829,16 @@ private boolean isSaslAuthMode() { return !AUTH_SIMPLE.equalsIgnoreCase(sessConfMap.get(AUTH_TYPE)); } - private boolean isFromSubjectAuthMode() { - return isSaslAuthMode() - && hasSessionValue(AUTH_PRINCIPAL) - && AUTH_KERBEROS_AUTH_TYPE_FROM_SUBJECT.equalsIgnoreCase( - sessConfMap.get(AUTH_KERBEROS_AUTH_TYPE)); + private boolean isHadoopUserGroupInformationDoAs() { + try { + @SuppressWarnings("unchecked") + Class HadoopUserClz = + (Class) ClassUtils.getClass("org.apache.hadoop.security.User"); + Subject subject = Subject.getSubject(AccessController.getContext()); + return subject != null && !subject.getPrincipals(HadoopUserClz).isEmpty(); + } catch (ClassNotFoundException e) { + return false; + } } private boolean isKeytabAuthMode() { @@ -826,6 +848,16 @@ && hasSessionValue(AUTH_KYUUBI_CLIENT_PRINCIPAL) && hasSessionValue(AUTH_KYUUBI_CLIENT_KEYTAB); } + private boolean isFromSubjectAuthMode() { + return isSaslAuthMode() + && hasSessionValue(AUTH_PRINCIPAL) + && !hasSessionValue(AUTH_KYUUBI_CLIENT_PRINCIPAL) + && !hasSessionValue(AUTH_KYUUBI_CLIENT_KEYTAB) + && (AUTH_KERBEROS_AUTH_TYPE_FROM_SUBJECT.equalsIgnoreCase( + sessConfMap.get(AUTH_KERBEROS_AUTH_TYPE)) + || isHadoopUserGroupInformationDoAs()); + } + private boolean isTgtCacheAuthMode() { return isSaslAuthMode() && hasSessionValue(AUTH_PRINCIPAL) @@ -842,15 +874,15 @@ private boolean isKerberosAuthMode() { } private Subject createSubject() { - if (isFromSubjectAuthMode()) { + if (isKeytabAuthMode()) { + String principal = sessConfMap.get(AUTH_KYUUBI_CLIENT_PRINCIPAL); + String keytab = sessConfMap.get(AUTH_KYUUBI_CLIENT_KEYTAB); + return KerberosAuthenticationManager.getKeytabAuthentication(principal, keytab).getSubject(); + } else if (isFromSubjectAuthMode()) { AccessControlContext context = AccessController.getContext(); return Subject.getSubject(context); } else if (isTgtCacheAuthMode()) { return KerberosAuthenticationManager.getTgtCacheAuthentication().getSubject(); - } else if (isKeytabAuthMode()) { - String principal = sessConfMap.get(AUTH_KYUUBI_CLIENT_PRINCIPAL); - String keytab = sessConfMap.get(AUTH_KYUUBI_CLIENT_KEYTAB); - return KerberosAuthenticationManager.getKeytabAuthentication(principal, keytab).getSubject(); } else { // This should never happen throw new IllegalArgumentException("Unsupported auth mode"); @@ -1338,7 +1370,7 @@ public void waitLaunchEngineToComplete() throws SQLException { } private void fetchLaunchEngineResult() { - if (launchEngineOpHandle == null) return; + if (launchEngineOpHandle == null || !launchEngineOpSupportResult) return; TFetchResultsReq tFetchResultsReq = new TFetchResultsReq( @@ -1356,6 +1388,8 @@ private void fetchLaunchEngineResult() { engineName = value; } else if ("url".equals(key)) { engineUrl = value; + } else if ("refId".equals(key)) { + engineRefId = value; } } } catch (Exception e) { @@ -1374,4 +1408,8 @@ public String getEngineName() { public String getEngineUrl() { return engineUrl; } + + public String getEngineRefId() { + return engineRefId; + } } diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiDatabaseMetaData.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiDatabaseMetaData.java index f5e29f8e7..c6ab3a277 100644 --- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiDatabaseMetaData.java +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiDatabaseMetaData.java @@ -531,7 +531,7 @@ public ResultSet getProcedureColumns( @Override public String getProcedureTerm() throws SQLException { - return new String("UDF"); + return "UDF"; } @Override diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiPreparedStatement.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiPreparedStatement.java index 43c2a030b..1e53f9401 100644 --- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiPreparedStatement.java +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiPreparedStatement.java @@ -26,9 +26,7 @@ import java.sql.Timestamp; import java.sql.Types; import java.text.MessageFormat; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Scanner; import org.apache.hive.service.rpc.thrift.TCLIService; import org.apache.hive.service.rpc.thrift.TSessionHandle; @@ -81,57 +79,7 @@ public int executeUpdate() throws SQLException { /** update the SQL string with parameters set by setXXX methods of {@link PreparedStatement} */ private String updateSql(final String sql, HashMap parameters) throws SQLException { - List parts = splitSqlStatement(sql); - - StringBuilder newSql = new StringBuilder(parts.get(0)); - for (int i = 1; i < parts.size(); i++) { - if (!parameters.containsKey(i)) { - throw new KyuubiSQLException("Parameter #" + i + " is unset"); - } - newSql.append(parameters.get(i)); - newSql.append(parts.get(i)); - } - return newSql.toString(); - } - - /** - * Splits the parametered sql statement at parameter boundaries. - * - *

    taking into account ' and \ escaping. - * - *

    output for: 'select 1 from ? where a = ?' ['select 1 from ',' where a = ',''] - */ - private List splitSqlStatement(String sql) { - List parts = new ArrayList<>(); - int apCount = 0; - int off = 0; - boolean skip = false; - - for (int i = 0; i < sql.length(); i++) { - char c = sql.charAt(i); - if (skip) { - skip = false; - continue; - } - switch (c) { - case '\'': - apCount++; - break; - case '\\': - skip = true; - break; - case '?': - if ((apCount & 1) == 0) { - parts.add(sql.substring(off, i)); - off = i + 1; - } - break; - default: - break; - } - } - parts.add(sql.substring(off, sql.length())); - return parts; + return Utils.updateSql(sql, parameters); } @Override @@ -220,7 +168,7 @@ public void setObject(int parameterIndex, Object x) throws SQLException { // Can't infer a type. throw new KyuubiSQLException( MessageFormat.format( - "Can't infer the SQL type to use for an instance of {0}. Use setObject() with an explicit Types value to specify the type to use.", + "Cannot infer the SQL type to use for an instance of {0}. Use setObject() with an explicit Types value to specify the type to use.", x.getClass().getName())); } } diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiQueryResultSet.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiQueryResultSet.java index f06ada5d4..242ec7720 100644 --- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiQueryResultSet.java +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiQueryResultSet.java @@ -26,6 +26,7 @@ import org.apache.kyuubi.jdbc.hive.cli.RowSet; import org.apache.kyuubi.jdbc.hive.cli.RowSetFactory; import org.apache.kyuubi.jdbc.hive.common.HiveDecimal; +import org.apache.thrift.TException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,6 +48,7 @@ public class KyuubiQueryResultSet extends KyuubiBaseResultSet { private boolean emptyResultSet = false; private boolean isScrollable = false; private boolean fetchFirst = false; + private boolean hasMoreToFetch = false; private final TProtocolVersion protocol; @@ -223,9 +225,6 @@ private void retrieveSchema() throws SQLException { metadataResp = client.GetResultSetMetadata(metadataReq); Utils.verifySuccess(metadataResp.getStatus()); - StringBuilder namesSb = new StringBuilder(); - StringBuilder typesSb = new StringBuilder(); - TTableSchema schema = metadataResp.getSchema(); if (schema == null || !schema.isSetColumns()) { // TODO: should probably throw an exception here. @@ -235,10 +234,6 @@ private void retrieveSchema() throws SQLException { List columns = schema.getColumns(); for (int pos = 0; pos < schema.getColumnsSize(); pos++) { - if (pos != 0) { - namesSb.append(","); - typesSb.append(","); - } String columnName = columns.get(pos).getColumnName(); columnNames.add(columnName); normalizedColumnNames.add(columnName.toLowerCase()); @@ -324,25 +319,20 @@ public boolean next() throws SQLException { try { TFetchOrientation orientation = TFetchOrientation.FETCH_NEXT; if (fetchFirst) { - // If we are asked to start from begining, clear the current fetched resultset + // If we are asked to start from beginning, clear the current fetched resultset orientation = TFetchOrientation.FETCH_FIRST; fetchedRows = null; fetchedRowsItr = null; fetchFirst = false; } if (fetchedRows == null || !fetchedRowsItr.hasNext()) { - TFetchResultsReq fetchReq = new TFetchResultsReq(stmtHandle, orientation, fetchSize); - TFetchResultsResp fetchResp; - fetchResp = client.FetchResults(fetchReq); - Utils.verifySuccessWithInfo(fetchResp.getStatus()); - - TRowSet results = fetchResp.getResults(); - fetchedRows = RowSetFactory.create(results, protocol); - fetchedRowsItr = fetchedRows.iterator(); + fetchResult(orientation); } if (fetchedRowsItr.hasNext()) { row = fetchedRowsItr.next(); + } else if (hasMoreToFetch) { + fetchResult(orientation); } else { return false; } @@ -357,6 +347,18 @@ public boolean next() throws SQLException { return true; } + private void fetchResult(TFetchOrientation orientation) throws SQLException, TException { + TFetchResultsReq fetchReq = new TFetchResultsReq(stmtHandle, orientation, fetchSize); + TFetchResultsResp fetchResp; + fetchResp = client.FetchResults(fetchReq); + Utils.verifySuccessWithInfo(fetchResp.getStatus()); + hasMoreToFetch = fetchResp.isSetHasMoreRows() && fetchResp.isHasMoreRows(); + + TRowSet results = fetchResp.getResults(); + fetchedRows = RowSetFactory.create(results, protocol); + fetchedRowsItr = fetchedRows.iterator(); + } + @Override public ResultSetMetaData getMetaData() throws SQLException { if (isClosed) { diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiSQLException.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiSQLException.java index 1ac0adf04..7d26f8078 100644 --- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiSQLException.java +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiSQLException.java @@ -21,6 +21,7 @@ import java.util.ArrayList; import java.util.List; import org.apache.hive.service.rpc.thrift.TStatus; +import org.apache.kyuubi.util.reflect.DynConstructors; public class KyuubiSQLException extends SQLException { @@ -186,7 +187,10 @@ private static Throwable toStackTrace( private static Throwable newInstance(String className, String message) { try { - return (Throwable) Class.forName(className).getConstructor(String.class).newInstance(message); + return DynConstructors.builder() + .impl(className, String.class) + .buildChecked() + .newInstance(message); } catch (Exception e) { return new RuntimeException(className + ":" + message); } diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiStatement.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiStatement.java index ab7c06a55..cbe32eca6 100644 --- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiStatement.java +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/KyuubiStatement.java @@ -37,6 +37,7 @@ public class KyuubiStatement implements SQLStatement, KyuubiLoggable { public static final Logger LOG = LoggerFactory.getLogger(KyuubiStatement.class.getName()); public static final int DEFAULT_FETCH_SIZE = 1000; public static final String DEFAULT_RESULT_FORMAT = "thrift"; + public static final String DEFAULT_ARROW_TIMESTAMP_AS_STRING = "false"; private final KyuubiConnection connection; private TCLIService.Iface client; private TOperationHandle stmtHandle = null; @@ -45,7 +46,8 @@ public class KyuubiStatement implements SQLStatement, KyuubiLoggable { private int fetchSize = DEFAULT_FETCH_SIZE; private boolean isScrollableResultset = false; private boolean isOperationComplete = false; - private Map properties = new HashMap<>(); + + private Map properties = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); /** * We need to keep a reference to the result set to support the following: * statement.execute(String sql); @@ -210,9 +212,14 @@ private boolean executeWithConfOverlay(String sql, Map confOverl String resultFormat = properties.getOrDefault("__kyuubi_operation_result_format__", DEFAULT_RESULT_FORMAT); - LOG.info("kyuubi.operation.result.format: " + resultFormat); + LOG.debug("kyuubi.operation.result.format: {}", resultFormat); switch (resultFormat) { case "arrow": + boolean timestampAsString = + Boolean.parseBoolean( + properties.getOrDefault( + "__kyuubi_operation_result_arrow_timestampAsString__", + DEFAULT_ARROW_TIMESTAMP_AS_STRING)); resultSet = new KyuubiArrowQueryResultSet.Builder(this) .setClient(client) @@ -222,6 +229,7 @@ private boolean executeWithConfOverlay(String sql, Map confOverl .setFetchSize(fetchSize) .setScrollable(isScrollableResultset) .setSchema(columnNames, columnTypes, columnAttributes) + .setTimestampAsString(timestampAsString) .build(); break; default: @@ -267,9 +275,14 @@ public boolean executeAsync(String sql) throws SQLException { String resultFormat = properties.getOrDefault("__kyuubi_operation_result_format__", DEFAULT_RESULT_FORMAT); - LOG.info("kyuubi.operation.result.format: " + resultFormat); + LOG.debug("kyuubi.operation.result.format: {}", resultFormat); switch (resultFormat) { case "arrow": + boolean timestampAsString = + Boolean.parseBoolean( + properties.getOrDefault( + "__kyuubi_operation_result_arrow_timestampAsString__", + DEFAULT_ARROW_TIMESTAMP_AS_STRING)); resultSet = new KyuubiArrowQueryResultSet.Builder(this) .setClient(client) @@ -279,6 +292,7 @@ public boolean executeAsync(String sql) throws SQLException { .setFetchSize(fetchSize) .setScrollable(isScrollableResultset) .setSchema(columnNames, columnTypes, columnAttributes) + .setTimestampAsString(timestampAsString) .build(); break; default: diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/Utils.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/Utils.java index c5b197f13..135c38d8e 100644 --- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/Utils.java +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/Utils.java @@ -22,10 +22,12 @@ import java.net.InetAddress; import java.net.URI; import java.net.UnknownHostException; +import java.sql.PreparedStatement; import java.sql.SQLException; import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; +import org.apache.commons.lang3.StringUtils; import org.apache.hive.service.rpc.thrift.TStatus; import org.apache.hive.service.rpc.thrift.TStatusCode; import org.slf4j.Logger; @@ -88,6 +90,62 @@ static void verifySuccess(TStatus status, boolean withInfo) throws SQLException throw new KyuubiSQLException(status); } + /** + * Splits the parametered sql statement at parameter boundaries. + * + *

    taking into account ' and \ escaping. + * + *

    output for: 'select 1 from ? where a = ?' ['select 1 from ',' where a = ',''] + */ + static List splitSqlStatement(String sql) { + List parts = new ArrayList<>(); + int apCount = 0; + int off = 0; + boolean skip = false; + + for (int i = 0; i < sql.length(); i++) { + char c = sql.charAt(i); + if (skip) { + skip = false; + continue; + } + switch (c) { + case '\'': + apCount++; + break; + case '\\': + skip = true; + break; + case '?': + if ((apCount & 1) == 0) { + parts.add(sql.substring(off, i)); + off = i + 1; + } + break; + default: + break; + } + } + parts.add(sql.substring(off)); + return parts; + } + + /** update the SQL string with parameters set by setXXX methods of {@link PreparedStatement} */ + public static String updateSql(final String sql, HashMap parameters) + throws SQLException { + List parts = splitSqlStatement(sql); + + StringBuilder newSql = new StringBuilder(parts.get(0)); + for (int i = 1; i < parts.size(); i++) { + if (!parameters.containsKey(i)) { + throw new KyuubiSQLException("Parameter #" + i + " is unset"); + } + newSql.append(parameters.get(i)); + newSql.append(parts.get(i)); + } + return newSql.toString(); + } + public static JdbcConnectionParams parseURL(String uri) throws JdbcUriParseException, SQLException, ZooKeeperHiveClientException { return parseURL(uri, new Properties()); @@ -193,12 +251,20 @@ public static JdbcConnectionParams extractURLComponents(String uri, Properties i } } + Pattern confPattern = Pattern.compile("([^;]*)([^;]*);?"); + // parse hive conf settings String confStr = jdbcURI.getQuery(); if (confStr != null) { - Matcher confMatcher = pattern.matcher(confStr); + Matcher confMatcher = confPattern.matcher(confStr); while (confMatcher.find()) { - connParams.getHiveConfs().put(confMatcher.group(1), confMatcher.group(2)); + String connParam = confMatcher.group(1); + if (StringUtils.isNotBlank(connParam) && connParam.contains("=")) { + int symbolIndex = connParam.indexOf('='); + connParams + .getHiveConfs() + .put(connParam.substring(0, symbolIndex), connParam.substring(symbolIndex + 1)); + } } } @@ -226,6 +292,13 @@ public static JdbcConnectionParams extractURLComponents(String uri, Properties i } } } + if (!connParams.getSessionVars().containsKey(CLIENT_PROTOCOL_VERSION)) { + if (info.containsKey(CLIENT_PROTOCOL_VERSION)) { + connParams + .getSessionVars() + .put(CLIENT_PROTOCOL_VERSION, info.getProperty(CLIENT_PROTOCOL_VERSION)); + } + } // Extract user/password from JDBC connection properties if its not supplied // in the connection URL if (!connParams.getSessionVars().containsKey(AUTH_USER)) { @@ -477,4 +550,24 @@ public static String getCanonicalHostName(String hostName) { public static boolean isKyuubiOperationHint(String hint) { return KYUUBI_OPERATION_HINT_PATTERN.matcher(hint).matches(); } + + public static final String KYUUBI_CLIENT_VERSION_KEY = "kyuubi.client.version"; + private static String KYUUBI_CLIENT_VERSION; + + public static synchronized String getVersion() { + if (KYUUBI_CLIENT_VERSION == null) { + try { + Properties prop = new Properties(); + prop.load( + Utils.class + .getClassLoader() + .getResourceAsStream("org/apache/kyuubi/version.properties")); + KYUUBI_CLIENT_VERSION = prop.getProperty(KYUUBI_CLIENT_VERSION_KEY, "unknown"); + } catch (Exception e) { + LOG.error("Error getting kyuubi client version", e); + KYUUBI_CLIENT_VERSION = "unknown"; + } + } + return KYUUBI_CLIENT_VERSION; + } } diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/ZooKeeperHiveClientHelper.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/ZooKeeperHiveClientHelper.java index 349fc8dfb..948fd3334 100644 --- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/ZooKeeperHiveClientHelper.java +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/ZooKeeperHiveClientHelper.java @@ -17,27 +17,30 @@ package org.apache.kyuubi.jdbc.hive; +import com.google.common.annotations.VisibleForTesting; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Random; +import java.util.concurrent.ThreadLocalRandom; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.curator.framework.CuratorFramework; -import org.apache.curator.framework.CuratorFrameworkFactory; -import org.apache.curator.retry.ExponentialBackoffRetry; +import org.apache.kyuubi.shaded.curator.framework.CuratorFramework; +import org.apache.kyuubi.shaded.curator.framework.CuratorFrameworkFactory; +import org.apache.kyuubi.shaded.curator.retry.ExponentialBackoffRetry; class ZooKeeperHiveClientHelper { // Pattern for key1=value1;key2=value2 private static final Pattern kvPattern = Pattern.compile("([^=;]*)=([^;]*);?"); - private static String getZooKeeperNamespace(JdbcConnectionParams connParams) { + @VisibleForTesting + protected static String getZooKeeperNamespace(JdbcConnectionParams connParams) { String zooKeeperNamespace = connParams.getSessionVars().get(JdbcConnectionParams.ZOOKEEPER_NAMESPACE); if ((zooKeeperNamespace == null) || (zooKeeperNamespace.isEmpty())) { zooKeeperNamespace = JdbcConnectionParams.ZOOKEEPER_DEFAULT_NAMESPACE; } + zooKeeperNamespace = zooKeeperNamespace.replaceAll("^/+", "").replaceAll("/+$", ""); return zooKeeperNamespace; } @@ -108,7 +111,7 @@ static void configureConnParams(JdbcConnectionParams connParams) try (CuratorFramework zooKeeperClient = getZkClient(connParams)) { List serverHosts = getServerHosts(connParams, zooKeeperClient); // Now pick a server node randomly - String serverNode = serverHosts.get(new Random().nextInt(serverHosts.size())); + String serverNode = serverHosts.get(ThreadLocalRandom.current().nextInt(serverHosts.size())); updateParamsWithZKServerNode(connParams, zooKeeperClient, serverNode); } catch (Exception e) { throw new ZooKeeperHiveClientException( diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/arrow/ArrowColumnarBatchRow.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/arrow/ArrowColumnarBatchRow.java index 20ed55a1d..373867069 100644 --- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/arrow/ArrowColumnarBatchRow.java +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/arrow/ArrowColumnarBatchRow.java @@ -19,6 +19,8 @@ import java.math.BigDecimal; import java.sql.Timestamp; +import java.time.LocalDateTime; +import org.apache.arrow.vector.util.DateUtility; import org.apache.hive.service.rpc.thrift.TTypeId; import org.apache.kyuubi.jdbc.hive.common.DateUtils; import org.apache.kyuubi.jdbc.hive.common.HiveIntervalDayTime; @@ -104,7 +106,11 @@ public Object getMap(int ordinal) { throw new UnsupportedOperationException(); } - public Object get(int ordinal, TTypeId dataType) { + public Object get(int ordinal, TTypeId dataType, String timeZone, boolean timestampAsString) { + long seconds; + long milliseconds; + long microseconds; + int nanos; switch (dataType) { case BOOLEAN_TYPE: return getBoolean(ordinal); @@ -127,13 +133,19 @@ public Object get(int ordinal, TTypeId dataType) { case STRING_TYPE: return getString(ordinal); case TIMESTAMP_TYPE: - return new Timestamp(getLong(ordinal) / 1000); + if (timestampAsString) { + return Timestamp.valueOf(getString(ordinal)); + } else { + LocalDateTime localDateTime = + DateUtility.getLocalDateTimeFromEpochMicro(getLong(ordinal), timeZone); + return Timestamp.valueOf(localDateTime); + } case DATE_TYPE: return DateUtils.internalToDate(getInt(ordinal)); case INTERVAL_DAY_TIME_TYPE: - long microseconds = getLong(ordinal); - long seconds = microseconds / 1000000; - int nanos = (int) (microseconds % 1000000) * 1000; + microseconds = getLong(ordinal); + seconds = microseconds / 1_000_000; + nanos = (int) (microseconds % 1_000_000) * 1_000; return new HiveIntervalDayTime(seconds, nanos); case INTERVAL_YEAR_MONTH_TYPE: return new HiveIntervalYearMonth(getInt(ordinal)); diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/auth/HttpKerberosRequestInterceptor.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/auth/HttpKerberosRequestInterceptor.java index 278cef0b4..02d168c3f 100644 --- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/auth/HttpKerberosRequestInterceptor.java +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/auth/HttpKerberosRequestInterceptor.java @@ -65,7 +65,7 @@ protected void addHttpAuthHeader(HttpRequest httpRequest, HttpContext httpContex httpRequest.addHeader( HttpAuthUtils.AUTHORIZATION, HttpAuthUtils.NEGOTIATE + " " + kerberosAuthHeader); } catch (Exception e) { - throw new HttpException(e.getMessage(), e); + throw new HttpException(e.getMessage() == null ? "" : e.getMessage(), e); } finally { kerberosLock.unlock(); } diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/auth/HttpRequestInterceptorBase.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/auth/HttpRequestInterceptorBase.java index 9ce5a330b..42641c219 100644 --- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/auth/HttpRequestInterceptorBase.java +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/auth/HttpRequestInterceptorBase.java @@ -110,7 +110,7 @@ public void process(HttpRequest httpRequest, HttpContext httpContext) httpRequest.addHeader("Cookie", cookieHeaderKeyValues.toString()); } } catch (Exception e) { - throw new HttpException(e.getMessage(), e); + throw new HttpException(e.getMessage() == null ? "" : e.getMessage(), e); } } diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/cli/ColumnBuffer.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/cli/ColumnBuffer.java index e703cb1f0..bd5124f95 100644 --- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/cli/ColumnBuffer.java +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/cli/ColumnBuffer.java @@ -228,8 +228,9 @@ public Object get(int index) { return stringVars.get(index); case BINARY_TYPE: return binaryVars.get(index).array(); + default: + return null; } - return null; } @Override diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/common/Date.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/common/Date.java index 1b49c268a..720c7517f 100644 --- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/common/Date.java +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/common/Date.java @@ -65,6 +65,7 @@ public String toString() { return localDate.format(PRINT_FORMATTER); } + @Override public int hashCode() { return localDate.hashCode(); } @@ -164,6 +165,7 @@ public int getDayOfWeek() { } /** Return a copy of this object. */ + @Override public Object clone() { // LocalDateTime is immutable. return new Date(this.localDate); diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/common/FastHiveDecimalImpl.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/common/FastHiveDecimalImpl.java index d3dba0f7b..65f17e734 100644 --- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/common/FastHiveDecimalImpl.java +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/common/FastHiveDecimalImpl.java @@ -5182,7 +5182,6 @@ public static boolean fastRoundIntegerDown( fastResult.fastIntegerDigitCount = 0; fastResult.fastScale = 0; } else { - fastResult.fastSignum = 0; fastResult.fastSignum = fastSignum; fastResult.fastIntegerDigitCount = fastRawPrecision(fastResult); fastResult.fastScale = 0; diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/common/Timestamp.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/common/Timestamp.java index cdb6b10ce..7e02835b7 100644 --- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/common/Timestamp.java +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/common/Timestamp.java @@ -95,6 +95,7 @@ public String toString() { return localDateTime.format(PRINT_FORMATTER); } + @Override public int hashCode() { return localDateTime.hashCode(); } @@ -207,6 +208,7 @@ public int getDayOfWeek() { } /** Return a copy of this object. */ + @Override public Object clone() { // LocalDateTime is immutable. return new Timestamp(this.localDateTime); diff --git a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/common/TimestampTZUtil.java b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/common/TimestampTZUtil.java index a938e1688..be16926cb 100644 --- a/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/common/TimestampTZUtil.java +++ b/kyuubi-hive-jdbc/src/main/java/org/apache/kyuubi/jdbc/hive/common/TimestampTZUtil.java @@ -98,7 +98,7 @@ private static String handleSingleDigitHourOffset(String s) { Matcher matcher = SINGLE_DIGIT_PATTERN.matcher(s); if (matcher.find()) { int index = matcher.start() + 1; - s = s.substring(0, index) + "0" + s.substring(index, s.length()); + s = s.substring(0, index) + "0" + s.substring(index); } return s; } diff --git a/kyuubi-hive-jdbc/src/main/resources/org/apache/kyuubi/version.properties b/kyuubi-hive-jdbc/src/main/resources/org/apache/kyuubi/version.properties new file mode 100644 index 000000000..82ae50cfb --- /dev/null +++ b/kyuubi-hive-jdbc/src/main/resources/org/apache/kyuubi/version.properties @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +kyuubi.client.version = ${project.version} diff --git a/kyuubi-hive-jdbc/src/test/java/org/apache/kyuubi/jdbc/hive/TestJdbcDriver.java b/kyuubi-hive-jdbc/src/test/java/org/apache/kyuubi/jdbc/hive/TestJdbcDriver.java index 228ad00ee..efdf73092 100644 --- a/kyuubi-hive-jdbc/src/test/java/org/apache/kyuubi/jdbc/hive/TestJdbcDriver.java +++ b/kyuubi-hive-jdbc/src/test/java/org/apache/kyuubi/jdbc/hive/TestJdbcDriver.java @@ -24,6 +24,7 @@ import java.io.File; import java.io.FileWriter; import java.io.IOException; +import java.nio.file.Files; import java.util.Arrays; import java.util.Collection; import org.junit.AfterClass; @@ -67,14 +68,14 @@ public static Collection data() { public static void setUpBeforeClass() throws Exception { file = new File(System.getProperty("user.dir") + File.separator + "Init.sql"); if (!file.exists()) { - file.createNewFile(); + Files.createFile(file.toPath()); } } @AfterClass public static void cleanUpAfterClass() throws Exception { if (file != null) { - file.delete(); + Files.deleteIfExists(file.toPath()); } } diff --git a/kyuubi-hive-jdbc/src/test/java/org/apache/kyuubi/jdbc/hive/UtilsTest.java b/kyuubi-hive-jdbc/src/test/java/org/apache/kyuubi/jdbc/hive/UtilsTest.java index c890c8731..b01957b3e 100644 --- a/kyuubi-hive-jdbc/src/test/java/org/apache/kyuubi/jdbc/hive/UtilsTest.java +++ b/kyuubi-hive-jdbc/src/test/java/org/apache/kyuubi/jdbc/hive/UtilsTest.java @@ -21,9 +21,15 @@ import static org.apache.kyuubi.jdbc.hive.Utils.extractURLComponents; import static org.junit.Assert.assertEquals; +import com.google.common.collect.ImmutableMap; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Collection; +import java.util.Map; import java.util.Properties; +import java.util.regex.Pattern; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -35,23 +41,76 @@ public class UtilsTest { private String expectedPort; private String expectedCatalog; private String expectedDb; + private Map expectedHiveConf; private String uri; @Parameterized.Parameters - public static Collection data() { + public static Collection data() throws UnsupportedEncodingException { return Arrays.asList( - new String[][] { - {"localhost", "10009", null, "db", "jdbc:hive2:///db;k1=v1?k2=v2#k3=v3"}, - {"localhost", "10009", null, "default", "jdbc:hive2:///"}, - {"localhost", "10009", null, "default", "jdbc:kyuubi://"}, - {"localhost", "10009", null, "default", "jdbc:hive2://"}, - {"hostname", "10018", null, "db", "jdbc:hive2://hostname:10018/db;k1=v1?k2=v2#k3=v3"}, + new Object[][] { + { + "localhost", + "10009", + null, + "db", + new ImmutableMap.Builder().put("k2", "v2").build(), + "jdbc:hive2:///db;k1=v1?k2=v2#k3=v3" + }, + { + "localhost", + "10009", + null, + "default", + new ImmutableMap.Builder().build(), + "jdbc:hive2:///" + }, + { + "localhost", + "10009", + null, + "default", + new ImmutableMap.Builder().build(), + "jdbc:kyuubi://" + }, + { + "localhost", + "10009", + null, + "default", + new ImmutableMap.Builder().build(), + "jdbc:hive2://" + }, + { + "hostname", + "10018", + null, + "db", + new ImmutableMap.Builder().put("k2", "v2").build(), + "jdbc:hive2://hostname:10018/db;k1=v1?k2=v2#k3=v3" + }, { "hostname", "10018", "catalog", "db", + new ImmutableMap.Builder().put("k2", "v2").build(), "jdbc:hive2://hostname:10018/catalog/db;k1=v1?k2=v2#k3=v3" + }, + { + "hostname", + "10018", + "catalog", + "db", + new ImmutableMap.Builder() + .put("k2", "v2") + .put("k3", "-Xmx2g -XX:+PrintGCDetails -XX:HeapDumpPath=/heap.hprof") + .build(), + "jdbc:hive2://hostname:10018/catalog/db;k1=v1?" + + URLEncoder.encode( + "k2=v2;k3=-Xmx2g -XX:+PrintGCDetails -XX:HeapDumpPath=/heap.hprof", + StandardCharsets.UTF_8.toString()) + .replaceAll("\\+", "%20") + + "#k4=v4" } }); } @@ -61,11 +120,13 @@ public UtilsTest( String expectedPort, String expectedCatalog, String expectedDb, + Map expectedHiveConf, String uri) { this.expectedHost = expectedHost; this.expectedPort = expectedPort; this.expectedCatalog = expectedCatalog; this.expectedDb = expectedDb; + this.expectedHiveConf = expectedHiveConf; this.uri = uri; } @@ -76,5 +137,12 @@ public void testExtractURLComponents() throws JdbcUriParseException { assertEquals(Integer.parseInt(expectedPort), jdbcConnectionParams1.getPort()); assertEquals(expectedCatalog, jdbcConnectionParams1.getCatalogName()); assertEquals(expectedDb, jdbcConnectionParams1.getDbName()); + assertEquals(expectedHiveConf, jdbcConnectionParams1.getHiveConfs()); + } + + @Test + public void testGetVersion() { + Pattern pattern = Pattern.compile("^\\d+\\.\\d+\\.\\d+.*"); + assert pattern.matcher(Utils.getVersion()).matches(); } } diff --git a/kyuubi-hive-jdbc/src/test/java/org/apache/kyuubi/jdbc/hive/ZooKeeperHiveClientHelperTest.java b/kyuubi-hive-jdbc/src/test/java/org/apache/kyuubi/jdbc/hive/ZooKeeperHiveClientHelperTest.java new file mode 100644 index 000000000..d1fd78f47 --- /dev/null +++ b/kyuubi-hive-jdbc/src/test/java/org/apache/kyuubi/jdbc/hive/ZooKeeperHiveClientHelperTest.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.jdbc.hive; + +import static org.apache.kyuubi.jdbc.hive.Utils.extractURLComponents; +import static org.junit.Assert.assertEquals; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Properties; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +@RunWith(Parameterized.class) +public class ZooKeeperHiveClientHelperTest { + + private String uri; + + @Parameterized.Parameters + public static Collection data() { + return Arrays.asList( + new String[][] { + {"jdbc:hive2://hostname:10018/db;zooKeeperNamespace=zookeeper/namespace"}, + {"jdbc:hive2://hostname:10018/db;zooKeeperNamespace=/zookeeper/namespace"}, + {"jdbc:hive2://hostname:10018/db;zooKeeperNamespace=zookeeper/namespace/"}, + {"jdbc:hive2://hostname:10018/db;zooKeeperNamespace=/zookeeper/namespace/"}, + {"jdbc:hive2://hostname:10018/db;zooKeeperNamespace=///zookeeper/namespace///"} + }); + } + + public ZooKeeperHiveClientHelperTest(String uri) { + this.uri = uri; + } + + @Test + public void testGetZooKeeperNamespace() throws JdbcUriParseException { + JdbcConnectionParams jdbcConnectionParams = extractURLComponents(uri, new Properties()); + assertEquals( + "zookeeper/namespace", + ZooKeeperHiveClientHelper.getZooKeeperNamespace(jdbcConnectionParams)); + } +} diff --git a/kyuubi-hive-jdbc/src/test/resources/log4j2-test.xml b/kyuubi-hive-jdbc/src/test/resources/log4j2-test.xml index bfc40dd6d..3110216c1 100644 --- a/kyuubi-hive-jdbc/src/test/resources/log4j2-test.xml +++ b/kyuubi-hive-jdbc/src/test/resources/log4j2-test.xml @@ -21,14 +21,14 @@ - + - + diff --git a/kyuubi-metrics/pom.xml b/kyuubi-metrics/pom.xml index b8ba40f47..5a95291bd 100644 --- a/kyuubi-metrics/pom.xml +++ b/kyuubi-metrics/pom.xml @@ -21,10 +21,10 @@ org.apache.kyuubi kyuubi-parent - 1.7.0-SNAPSHOT + 1.9.0-SNAPSHOT - kyuubi-metrics_2.12 + kyuubi-metrics_${scala.binary.version} jar Kyuubi Project Metrics https://kyuubi.apache.org/ diff --git a/kyuubi-metrics/src/main/scala/org/apache/kyuubi/metrics/JsonReporterService.scala b/kyuubi-metrics/src/main/scala/org/apache/kyuubi/metrics/JsonReporterService.scala index cb0ef7404..7b172fc1e 100644 --- a/kyuubi-metrics/src/main/scala/org/apache/kyuubi/metrics/JsonReporterService.scala +++ b/kyuubi-metrics/src/main/scala/org/apache/kyuubi/metrics/JsonReporterService.scala @@ -65,7 +65,7 @@ class JsonReporterService(registry: MetricRegistry) Files.setPosixFilePermissions(tmpPath, PosixFilePermissions.fromString("rwxr--r--")) Files.move(tmpPath, reportPath, StandardCopyOption.REPLACE_EXISTING) } catch { - case NonFatal(e) => error("Error writing metrics to json file" + reportPath, e) + case NonFatal(e) => error(s"Error writing metrics to json file: $reportPath", e) } finally { if (writer != null) writer.close() } diff --git a/kyuubi-metrics/src/main/scala/org/apache/kyuubi/metrics/MetricsConf.scala b/kyuubi-metrics/src/main/scala/org/apache/kyuubi/metrics/MetricsConf.scala index daa221b78..9bc2e6324 100644 --- a/kyuubi-metrics/src/main/scala/org/apache/kyuubi/metrics/MetricsConf.scala +++ b/kyuubi-metrics/src/main/scala/org/apache/kyuubi/metrics/MetricsConf.scala @@ -19,13 +19,12 @@ package org.apache.kyuubi.metrics import java.time.Duration -import org.apache.kyuubi.config.{ConfigBuilder, ConfigEntry, KyuubiConf} +import org.apache.kyuubi.config.ConfigEntry +import org.apache.kyuubi.config.KyuubiConf.buildConf import org.apache.kyuubi.metrics.ReporterType._ object MetricsConf { - private def buildConf(key: String): ConfigBuilder = KyuubiConf.buildConf(key) - val METRICS_ENABLED: ConfigEntry[Boolean] = buildConf("kyuubi.metrics.enabled") .doc("Set to true to enable kyuubi metrics system") @@ -33,7 +32,7 @@ object MetricsConf { .booleanConf .createWithDefault(true) - val METRICS_REPORTERS: ConfigEntry[Seq[String]] = buildConf("kyuubi.metrics.reporters") + val METRICS_REPORTERS: ConfigEntry[Set[String]] = buildConf("kyuubi.metrics.reporters") .doc("A comma-separated list for all metrics reporters" + "

      " + "
    • CONSOLE - ConsoleReporter which outputs measurements to CONSOLE periodically.
      • " + @@ -44,12 +43,10 @@ object MetricsConf { "
    ") .version("1.2.0") .stringConf - .transform(_.toUpperCase()) - .toSequence() - .checkValue( - _.forall(ReporterType.values.map(_.toString).contains), - s"the reporter type should be one or more of ${ReporterType.values.mkString(",")}") - .createWithDefault(Seq(JSON.toString)) + .transformToUpperCase + .toSet() + .checkValues(ReporterType) + .createWithDefault(Set(PROMETHEUS.toString)) val METRICS_CONSOLE_INTERVAL: ConfigEntry[Long] = buildConf("kyuubi.metrics.console.interval") .doc("How often should report metrics to console") diff --git a/kyuubi-metrics/src/main/scala/org/apache/kyuubi/metrics/MetricsConstants.scala b/kyuubi-metrics/src/main/scala/org/apache/kyuubi/metrics/MetricsConstants.scala index 62c67266f..f615467f3 100644 --- a/kyuubi-metrics/src/main/scala/org/apache/kyuubi/metrics/MetricsConstants.scala +++ b/kyuubi-metrics/src/main/scala/org/apache/kyuubi/metrics/MetricsConstants.scala @@ -26,9 +26,11 @@ object MetricsConstants { final val BUFFER_POOL: String = KYUUBI + "buffer_pool" final val THREAD_STATE: String = KYUUBI + "thread_state" final val CLASS_LOADING: String = KYUUBI + "class_loading" + final val JVM: String = KYUUBI + "jvm" final val EXEC_POOL_ALIVE: String = KYUUBI + "exec.pool.threads.alive" final val EXEC_POOL_ACTIVE: String = KYUUBI + "exec.pool.threads.active" + final val EXEC_POOL_WORK_QUEUE_SIZE: String = KYUUBI + "exec.pool.work_queue.size" final private val CONN = KYUUBI + "connection." final private val THRIFT_HTTP_CONN = KYUUBI + "thrift.http.connection." diff --git a/kyuubi-metrics/src/main/scala/org/apache/kyuubi/metrics/MetricsSystem.scala b/kyuubi-metrics/src/main/scala/org/apache/kyuubi/metrics/MetricsSystem.scala index 99da1f1b0..26344ca56 100644 --- a/kyuubi-metrics/src/main/scala/org/apache/kyuubi/metrics/MetricsSystem.scala +++ b/kyuubi-metrics/src/main/scala/org/apache/kyuubi/metrics/MetricsSystem.scala @@ -67,6 +67,7 @@ class MetricsSystem extends CompositeService("MetricsSystem") { } override def initialize(conf: KyuubiConf): Unit = synchronized { + registry.registerAll(MetricsConstants.JVM, new JvmAttributeGaugeSet) registry.registerAll(MetricsConstants.GC_METRIC, new GarbageCollectorMetricSet) registry.registerAll(MetricsConstants.MEMORY_USAGE, new MemoryUsageGaugeSet) registry.registerAll( diff --git a/kyuubi-metrics/src/test/resources/log4j2-test.xml b/kyuubi-metrics/src/test/resources/log4j2-test.xml index bfc40dd6d..3110216c1 100644 --- a/kyuubi-metrics/src/test/resources/log4j2-test.xml +++ b/kyuubi-metrics/src/test/resources/log4j2-test.xml @@ -21,14 +21,14 @@ - + - + diff --git a/kyuubi-metrics/src/test/scala/org/apache/kyuubi/metrics/MetricsSystemSuite.scala b/kyuubi-metrics/src/test/scala/org/apache/kyuubi/metrics/MetricsSystemSuite.scala index 611531d73..bac20181c 100644 --- a/kyuubi-metrics/src/test/scala/org/apache/kyuubi/metrics/MetricsSystemSuite.scala +++ b/kyuubi-metrics/src/test/scala/org/apache/kyuubi/metrics/MetricsSystemSuite.scala @@ -49,7 +49,7 @@ class MetricsSystemSuite extends KyuubiFunSuite { val conf = KyuubiConf() .set(MetricsConf.METRICS_ENABLED, true) - .set(MetricsConf.METRICS_REPORTERS, Seq(ReporterType.PROMETHEUS.toString)) + .set(MetricsConf.METRICS_REPORTERS, Set(ReporterType.PROMETHEUS.toString)) .set(MetricsConf.METRICS_PROMETHEUS_PORT, 0) // random port .set(MetricsConf.METRICS_PROMETHEUS_PATH, testContextPath) val metricsSystem = new MetricsSystem() @@ -77,7 +77,7 @@ class MetricsSystemSuite extends KyuubiFunSuite { .set(MetricsConf.METRICS_ENABLED, true) .set( MetricsConf.METRICS_REPORTERS, - ReporterType.values.filterNot(_ == ReporterType.PROMETHEUS).map(_.toString).toSeq) + ReporterType.values.filterNot(_ == ReporterType.PROMETHEUS).map(_.toString)) .set(MetricsConf.METRICS_JSON_INTERVAL, Duration.ofSeconds(1).toMillis) .set(MetricsConf.METRICS_JSON_LOCATION, reportPath.toString) val metricsSystem = new MetricsSystem() diff --git a/kyuubi-rest-client/pom.xml b/kyuubi-rest-client/pom.xml index 28b1670f1..7d7e595c4 100644 --- a/kyuubi-rest-client/pom.xml +++ b/kyuubi-rest-client/pom.xml @@ -21,7 +21,7 @@ org.apache.kyuubi kyuubi-parent - 1.7.0-SNAPSHOT + 1.9.0-SNAPSHOT kyuubi-rest-client @@ -77,11 +77,22 @@ true + + org.apache.kyuubi + kyuubi-util + ${project.version} + + org.slf4j slf4j-api + + org.slf4j + jcl-over-slf4j + + org.apache.logging.log4j log4j-slf4j-impl @@ -115,6 +126,13 @@ + + + true + src/main/resources + + + net.alchim31.maven diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/AdminRestApi.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/AdminRestApi.java index da9782df5..904ecb6c9 100644 --- a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/AdminRestApi.java +++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/AdminRestApi.java @@ -17,11 +17,11 @@ package org.apache.kyuubi.client; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import org.apache.kyuubi.client.api.v1.dto.Engine; +import org.apache.kyuubi.client.api.v1.dto.OperationData; +import org.apache.kyuubi.client.api.v1.dto.ServerData; +import org.apache.kyuubi.client.api.v1.dto.SessionData; public class AdminRestApi { private KyuubiRestClient client; @@ -44,6 +44,21 @@ public String refreshUserDefaultsConf() { return this.getClient().post(path, null, client.getAuthHeader()); } + public String refreshKubernetesConf() { + String path = String.format("%s/%s", API_BASE_PATH, "refresh/kubernetes_conf"); + return this.getClient().post(path, null, client.getAuthHeader()); + } + + public String refreshUnlimitedUsers() { + String path = String.format("%s/%s", API_BASE_PATH, "refresh/unlimited_users"); + return this.getClient().post(path, null, client.getAuthHeader()); + } + + public String refreshDenyUsers() { + String path = String.format("%s/%s", API_BASE_PATH, "refresh/deny_users"); + return this.getClient().post(path, null, client.getAuthHeader()); + } + public String deleteEngine( String engineType, String shareLevel, String subdomain, String hs2ProxyUser) { Map params = new HashMap<>(); @@ -55,18 +70,59 @@ public String deleteEngine( } public List listEngines( - String engineType, String shareLevel, String subdomain, String hs2ProxyUser) { + String engineType, String shareLevel, String subdomain, String hs2ProxyUser, String all) { Map params = new HashMap<>(); params.put("type", engineType); params.put("sharelevel", shareLevel); params.put("subdomain", subdomain); params.put("hive.server2.proxy.user", hs2ProxyUser); + params.put("all", all); Engine[] result = this.getClient() .get(API_BASE_PATH + "/engine", params, Engine[].class, client.getAuthHeader()); return Arrays.asList(result); } + public List listSessions() { + return listSessions(Collections.emptyList()); + } + + public List listSessions(List users) { + Map params = new HashMap<>(); + if (users != null && !users.isEmpty()) { + params.put("users", String.join(",", users)); + } + SessionData[] result = + this.getClient() + .get(API_BASE_PATH + "/sessions", params, SessionData[].class, client.getAuthHeader()); + return Arrays.asList(result); + } + + public String closeSession(String sessionHandleStr) { + String url = String.format("%s/sessions/%s", API_BASE_PATH, sessionHandleStr); + return this.getClient().delete(url, null, client.getAuthHeader()); + } + + public List listOperations() { + OperationData[] result = + this.getClient() + .get( + API_BASE_PATH + "/operations", null, OperationData[].class, client.getAuthHeader()); + return Arrays.asList(result); + } + + public String closeOperation(String operationHandleStr) { + String url = String.format("%s/operations/%s", API_BASE_PATH, operationHandleStr); + return this.getClient().delete(url, null, client.getAuthHeader()); + } + + public List listServers() { + ServerData[] result = + this.getClient() + .get(API_BASE_PATH + "/server", null, ServerData[].class, client.getAuthHeader()); + return Arrays.asList(result); + } + private IRestClient getClient() { return this.client.getHttpClient(); } diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/BatchRestApi.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/BatchRestApi.java index a09ab562b..7d113308d 100644 --- a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/BatchRestApi.java +++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/BatchRestApi.java @@ -22,6 +22,7 @@ import java.util.Map; import org.apache.kyuubi.client.api.v1.dto.*; import org.apache.kyuubi.client.util.JsonUtils; +import org.apache.kyuubi.client.util.VersionUtils; public class BatchRestApi { @@ -36,11 +37,13 @@ public BatchRestApi(KyuubiRestClient client) { } public Batch createBatch(BatchRequest request) { + setClientVersion(request); String requestBody = JsonUtils.toJson(request); return this.getClient().post(API_BASE_PATH, requestBody, Batch.class, client.getAuthHeader()); } public Batch createBatch(BatchRequest request, File resourceFile) { + setClientVersion(request); Map multiPartMap = new HashMap<>(); multiPartMap.put("batchRequest", new MultiPart(MultiPart.MultiPartType.JSON, request)); multiPartMap.put("resourceFile", new MultiPart(MultiPart.MultiPartType.FILE, resourceFile)); @@ -60,10 +63,23 @@ public GetBatchesResponse listBatches( Long endTime, int from, int size) { + return listBatches(batchType, batchUser, batchState, null, createTime, endTime, from, size); + } + + public GetBatchesResponse listBatches( + String batchType, + String batchUser, + String batchState, + String batchName, + Long createTime, + Long endTime, + int from, + int size) { Map params = new HashMap<>(); params.put("batchType", batchType); params.put("batchUser", batchUser); params.put("batchState", batchState); + params.put("batchName", batchName); if (null != createTime && createTime > 0) { params.put("createTime", createTime); } @@ -96,4 +112,12 @@ public CloseBatchResponse deleteBatch(String batchId, String hs2ProxyUser) { private IRestClient getClient() { return this.client.getHttpClient(); } + + private void setClientVersion(BatchRequest request) { + if (request != null) { + Map newConf = new HashMap<>(request.getConf()); + newConf.put(VersionUtils.KYUUBI_CLIENT_VERSION_KEY, VersionUtils.getVersion()); + request.setConf(newConf); + } + } } diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/IRestClient.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/IRestClient.java index 66897df54..0eaffebd2 100644 --- a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/IRestClient.java +++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/IRestClient.java @@ -32,6 +32,10 @@ public interface IRestClient extends AutoCloseable { String post(String path, String body, String authHeader); + T put(String path, String body, Class type, String authHeader); + + String put(String path, String body, String authHeader); + T delete(String path, Map params, Class type, String authHeader); String delete(String path, Map params, String authHeader); diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/KyuubiRestClient.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/KyuubiRestClient.java index dbcc89b16..c83eff7e0 100644 --- a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/KyuubiRestClient.java +++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/KyuubiRestClient.java @@ -30,6 +30,8 @@ public class KyuubiRestClient implements AutoCloseable, Cloneable { private RestClientConf conf; + private List hostUrls; + private List baseUrls; private ApiVersion version; @@ -77,14 +79,20 @@ public void setHostUrls(List hostUrls) { if (hostUrls.isEmpty()) { throw new IllegalArgumentException("hostUrls cannot be blank."); } + this.hostUrls = hostUrls; List baseUrls = initBaseUrls(hostUrls, version); this.httpClient = RetryableRestClient.getRestClient(baseUrls, this.conf); } + public List getHostUrls() { + return hostUrls; + } + private KyuubiRestClient() {} private KyuubiRestClient(Builder builder) { this.version = builder.version; + this.hostUrls = builder.hostUrls; this.baseUrls = initBaseUrls(builder.hostUrls, builder.version); RestClientConf conf = new RestClientConf(); diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/OperationRestApi.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/OperationRestApi.java new file mode 100644 index 000000000..ad659a5d4 --- /dev/null +++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/OperationRestApi.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.client; + +import java.util.HashMap; +import java.util.Map; +import org.apache.kyuubi.client.api.v1.dto.*; +import org.apache.kyuubi.client.util.JsonUtils; + +public class OperationRestApi { + + private KyuubiRestClient client; + + private static final String API_BASE_PATH = "operations"; + + private OperationRestApi() {} + + public OperationRestApi(KyuubiRestClient client) { + this.client = client; + } + + public KyuubiOperationEvent getOperationEvent(String operationHandleStr) { + String path = String.format("%s/%s/event", API_BASE_PATH, operationHandleStr); + return this.getClient() + .get(path, new HashMap<>(), KyuubiOperationEvent.class, client.getAuthHeader()); + } + + public String applyOperationAction(OpActionRequest request, String operationHandleStr) { + String path = String.format("%s/%s", API_BASE_PATH, operationHandleStr); + return this.getClient().put(path, JsonUtils.toJson(request), client.getAuthHeader()); + } + + public ResultSetMetaData getResultSetMetadata(String operationHandleStr) { + String path = String.format("%s/%s/resultsetmetadata", API_BASE_PATH, operationHandleStr); + return this.getClient() + .get(path, new HashMap<>(), ResultSetMetaData.class, client.getAuthHeader()); + } + + public OperationLog getOperationLog(String operationHandleStr, int maxRows) { + String path = String.format("%s/%s/log", API_BASE_PATH, operationHandleStr); + Map params = new HashMap<>(); + params.put("maxrows", maxRows); + return this.getClient().get(path, params, OperationLog.class, client.getAuthHeader()); + } + + public ResultRowSet getNextRowSet(String operationHandleStr) { + return getNextRowSet(operationHandleStr, null, null); + } + + public ResultRowSet getNextRowSet( + String operationHandleStr, String fetchOrientation, Integer maxRows) { + String path = String.format("%s/%s/rowset", API_BASE_PATH, operationHandleStr); + Map params = new HashMap<>(); + if (fetchOrientation != null) params.put("fetchorientation", fetchOrientation); + if (maxRows != null) params.put("maxrows", maxRows); + return this.getClient().get(path, params, ResultRowSet.class, client.getAuthHeader()); + } + + private IRestClient getClient() { + return this.client.getHttpClient(); + } +} diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/RestClient.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/RestClient.java index fa3544726..e6d1d9674 100644 --- a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/RestClient.java +++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/RestClient.java @@ -27,6 +27,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.http.HttpEntity; import org.apache.http.HttpHeaders; +import org.apache.http.NoHttpResponseException; import org.apache.http.client.HttpResponseException; import org.apache.http.client.ResponseHandler; import org.apache.http.client.methods.HttpUriRequest; @@ -113,7 +114,7 @@ public T post( contentBody = new FileBody((File) payload); break; default: - throw new RuntimeException("Unsupported multi part type:" + multiPart); + throw new RuntimeException("Unsupported multi part type:" + multiPart.getType()); } entityBuilder.addPart(s, contentBody); }); @@ -125,6 +126,21 @@ public T post( return JsonUtils.fromJson(responseJson, type); } + @Override + public T put(String path, String body, Class type, String authHeader) { + String responseJson = put(path, body, authHeader); + return JsonUtils.fromJson(responseJson, type); + } + + @Override + public String put(String path, String body, String authHeader) { + RequestBuilder putRequestBuilder = RequestBuilder.put(); + if (body != null) { + putRequestBuilder.setEntity(new StringEntity(body, StandardCharsets.UTF_8)); + } + return doRequest(buildURI(path), authHeader, putRequestBuilder); + } + @Override public T delete(String path, Map params, Class type, String authHeader) { String responseJson = delete(path, params, authHeader); @@ -164,7 +180,7 @@ private String doRequest(URI uri, String authHeader, RequestBuilder requestBuild response = httpclient.execute(httpRequest, responseHandler); LOG.debug("Response: {}", response); - } catch (ConnectException | ConnectTimeoutException e) { + } catch (ConnectException | ConnectTimeoutException | NoHttpResponseException e) { // net exception can be retried by connecting to other Kyuubi server throw new RetryableKyuubiRestException("Api request failed for " + uri.toString(), e); } catch (KyuubiRestException rethrow) { diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/RetryableRestClient.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/RetryableRestClient.java index dcd052aca..d13151c2e 100644 --- a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/RetryableRestClient.java +++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/RetryableRestClient.java @@ -22,7 +22,7 @@ import java.lang.reflect.Method; import java.lang.reflect.Proxy; import java.util.List; -import java.util.Random; +import java.util.concurrent.ThreadLocalRandom; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.kyuubi.client.exception.RetryableKyuubiRestException; import org.slf4j.Logger; @@ -44,7 +44,7 @@ public class RetryableRestClient implements InvocationHandler { private RetryableRestClient(List uris, RestClientConf conf) { this.conf = conf; this.uris = uris; - this.currentUriIndex = new Random(System.currentTimeMillis()).nextInt(uris.size()); + this.currentUriIndex = ThreadLocalRandom.current().nextInt(uris.size()); newRestClient(); } diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/SessionRestApi.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/SessionRestApi.java index fbb424102..a4c3bb7ab 100644 --- a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/SessionRestApi.java +++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/SessionRestApi.java @@ -20,7 +20,8 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; -import org.apache.kyuubi.client.api.v1.dto.SessionData; +import org.apache.kyuubi.client.api.v1.dto.*; +import org.apache.kyuubi.client.util.JsonUtils; public class SessionRestApi { @@ -41,6 +42,102 @@ public List listSessions() { return Arrays.asList(result); } + public SessionHandle openSession(SessionOpenRequest sessionOpenRequest) { + return this.getClient() + .post( + API_BASE_PATH, + JsonUtils.toJson(sessionOpenRequest), + SessionHandle.class, + client.getAuthHeader()); + } + + public String closeSession(String sessionHandleStr) { + String path = String.format("%s/%s", API_BASE_PATH, sessionHandleStr); + return this.getClient().delete(path, new HashMap<>(), client.getAuthHeader()); + } + + public KyuubiSessionEvent getSessionEvent(String sessionHandleStr) { + String path = String.format("%s/%s", API_BASE_PATH, sessionHandleStr); + return this.getClient() + .get(path, new HashMap<>(), KyuubiSessionEvent.class, client.getAuthHeader()); + } + + public InfoDetail getSessionInfo(String sessionHandleStr, int infoType) { + String path = String.format("%s/%s/info/%s", API_BASE_PATH, sessionHandleStr, infoType); + return this.getClient().get(path, new HashMap<>(), InfoDetail.class, client.getAuthHeader()); + } + + public int getOpenSessionCount() { + String path = String.format("%s/count", API_BASE_PATH); + return this.getClient() + .get(path, new HashMap<>(), SessionOpenCount.class, client.getAuthHeader()) + .getOpenSessionCount(); + } + + public ExecPoolStatistic getExecPoolStatistic() { + String path = String.format("%s/execPool/statistic", API_BASE_PATH); + return this.getClient() + .get(path, new HashMap<>(), ExecPoolStatistic.class, client.getAuthHeader()); + } + + public OperationHandle executeStatement(String sessionHandleStr, StatementRequest request) { + String path = String.format("%s/%s/operations/statement", API_BASE_PATH, sessionHandleStr); + return this.getClient() + .post(path, JsonUtils.toJson(request), OperationHandle.class, client.getAuthHeader()); + } + + public OperationHandle getTypeInfo(String sessionHandleStr) { + String path = String.format("%s/%s/operations/typeInfo", API_BASE_PATH, sessionHandleStr); + return this.getClient().post(path, "", OperationHandle.class, client.getAuthHeader()); + } + + public OperationHandle getCatalogs(String sessionHandleStr) { + String path = String.format("%s/%s/operations/catalogs", API_BASE_PATH, sessionHandleStr); + return this.getClient().post(path, "", OperationHandle.class, client.getAuthHeader()); + } + + public OperationHandle getSchemas(String sessionHandleStr, GetSchemasRequest request) { + String path = String.format("%s/%s/operations/schemas", API_BASE_PATH, sessionHandleStr); + return this.getClient() + .post(path, JsonUtils.toJson(request), OperationHandle.class, client.getAuthHeader()); + } + + public OperationHandle getTables(String sessionHandleStr, GetTablesRequest request) { + String path = String.format("%s/%s/operations/tables", API_BASE_PATH, sessionHandleStr); + return this.getClient() + .post(path, JsonUtils.toJson(request), OperationHandle.class, client.getAuthHeader()); + } + + public OperationHandle getTableTypes(String sessionHandleStr) { + String path = String.format("%s/%s/operations/tableTypes", API_BASE_PATH, sessionHandleStr); + return this.getClient().post(path, "", OperationHandle.class, client.getAuthHeader()); + } + + public OperationHandle getColumns(String sessionHandleStr, GetColumnsRequest request) { + String path = String.format("%s/%s/operations/columns", API_BASE_PATH, sessionHandleStr); + return this.getClient() + .post(path, JsonUtils.toJson(request), OperationHandle.class, client.getAuthHeader()); + } + + public OperationHandle getFunctions(String sessionHandleStr, GetFunctionsRequest request) { + String path = String.format("%s/%s/operations/functions", API_BASE_PATH, sessionHandleStr); + return this.getClient() + .post(path, JsonUtils.toJson(request), OperationHandle.class, client.getAuthHeader()); + } + + public OperationHandle getPrimaryKeys(String sessionHandleStr, GetPrimaryKeysRequest request) { + String path = String.format("%s/%s/operations/primaryKeys", API_BASE_PATH, sessionHandleStr); + return this.getClient() + .post(path, JsonUtils.toJson(request), OperationHandle.class, client.getAuthHeader()); + } + + public OperationHandle getCrossReference( + String sessionHandleStr, GetCrossReferenceRequest request) { + String path = String.format("%s/%s/operations/crossReference", API_BASE_PATH, sessionHandleStr); + return this.getClient() + .post(path, JsonUtils.toJson(request), OperationHandle.class, client.getAuthHeader()); + } + private IRestClient getClient() { return this.client.getHttpClient(); } diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/Batch.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/Batch.java index 43fbf10af..b318b709d 100644 --- a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/Batch.java +++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/Batch.java @@ -17,6 +17,8 @@ package org.apache.kyuubi.client.api.v1.dto; +import java.util.Collections; +import java.util.Map; import java.util.Objects; import org.apache.commons.lang3.builder.ReflectionToStringBuilder; import org.apache.commons.lang3.builder.ToStringStyle; @@ -35,6 +37,7 @@ public class Batch { private String state; private long createTime; private long endTime; + private Map batchInfo = Collections.emptyMap(); public Batch() {} @@ -51,7 +54,8 @@ public Batch( String kyuubiInstance, String state, long createTime, - long endTime) { + long endTime, + Map batchInfo) { this.id = id; this.user = user; this.batchType = batchType; @@ -65,6 +69,7 @@ public Batch( this.state = state; this.createTime = createTime; this.endTime = endTime; + this.batchInfo = batchInfo; } public String getId() { @@ -171,6 +176,17 @@ public void setEndTime(long endTime) { this.endTime = endTime; } + public Map getBatchInfo() { + if (batchInfo == null) { + return Collections.emptyMap(); + } + return batchInfo; + } + + public void setBatchInfo(Map batchInfo) { + this.batchInfo = batchInfo; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/BatchRequest.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/BatchRequest.java index f10a8fdb5..f45821fc2 100644 --- a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/BatchRequest.java +++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/BatchRequest.java @@ -29,8 +29,8 @@ public class BatchRequest { private String resource; private String className; private String name; - private Map conf; - private List args; + private Map conf = Collections.emptyMap(); + private List args = Collections.emptyList(); public BatchRequest() {} @@ -54,8 +54,6 @@ public BatchRequest(String batchType, String resource, String className, String this.resource = resource; this.className = className; this.name = name; - this.conf = Collections.emptyMap(); - this.args = Collections.emptyList(); } public String getBatchType() { diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/Count.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/Count.java new file mode 100644 index 000000000..8f77ccd13 --- /dev/null +++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/Count.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.client.api.v1.dto; + +import java.util.Objects; +import org.apache.commons.lang3.builder.ReflectionToStringBuilder; +import org.apache.commons.lang3.builder.ToStringStyle; + +public class Count { + private Integer count; + + public Count() {} + + public Count(Integer count) { + this.count = count; + } + + public Integer getCount() { + return count; + } + + public void setCount(Integer count) { + this.count = count; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Count that = (Count) o; + return Objects.equals(getCount(), that.getCount()); + } + + @Override + public int hashCode() { + return Objects.hash(getCount()); + } + + @Override + public String toString() { + return ReflectionToStringBuilder.toString(this, ToStringStyle.JSON_STYLE); + } +} diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/ExecPoolStatistic.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/ExecPoolStatistic.java index ee8a9f007..a40811f92 100644 --- a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/ExecPoolStatistic.java +++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/ExecPoolStatistic.java @@ -24,12 +24,14 @@ public class ExecPoolStatistic { private int execPoolSize; private int execPoolActiveCount; + private int execPoolWorkQueueSize; public ExecPoolStatistic() {} - public ExecPoolStatistic(int execPoolSize, int execPoolActiveCount) { + public ExecPoolStatistic(int execPoolSize, int execPoolActiveCount, int execPoolWorkQueueSize) { this.execPoolSize = execPoolSize; this.execPoolActiveCount = execPoolActiveCount; + this.execPoolWorkQueueSize = execPoolWorkQueueSize; } public int getExecPoolSize() { @@ -48,18 +50,27 @@ public void setExecPoolActiveCount(int execPoolActiveCount) { this.execPoolActiveCount = execPoolActiveCount; } + public int getExecPoolWorkQueueSize() { + return execPoolWorkQueueSize; + } + + public void setExecPoolWorkQueueSize(int execPoolWorkQueueSize) { + this.execPoolWorkQueueSize = execPoolWorkQueueSize; + } + @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; ExecPoolStatistic that = (ExecPoolStatistic) o; return getExecPoolSize() == that.getExecPoolSize() - && getExecPoolActiveCount() == that.getExecPoolActiveCount(); + && getExecPoolActiveCount() == that.getExecPoolActiveCount() + && getExecPoolWorkQueueSize() == that.getExecPoolWorkQueueSize(); } @Override public int hashCode() { - return Objects.hash(getExecPoolSize(), getExecPoolActiveCount()); + return Objects.hash(getExecPoolSize(), getExecPoolActiveCount(), getExecPoolWorkQueueSize()); } @Override diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/KyuubiOperationEvent.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/KyuubiOperationEvent.java new file mode 100644 index 000000000..13c40eecf --- /dev/null +++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/KyuubiOperationEvent.java @@ -0,0 +1,343 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.client.api.v1.dto; + +import java.util.Map; + +public class KyuubiOperationEvent { + + private String statementId; + + private String remoteId; + + private String statement; + + private boolean shouldRunAsync; + + private String state; + + private long eventTime; + + private long createTime; + + private long startTime; + + private long completeTime; + + private Throwable exception; + + private String sessionId; + + private String sessionUser; + + private String sessionType; + + private String kyuubiInstance; + + private Map metrics; + + public KyuubiOperationEvent() {} + + public KyuubiOperationEvent( + String statementId, + String remoteId, + String statement, + boolean shouldRunAsync, + String state, + long eventTime, + long createTime, + long startTime, + long completeTime, + Throwable exception, + String sessionId, + String sessionUser, + String sessionType, + String kyuubiInstance, + Map metrics) { + this.statementId = statementId; + this.remoteId = remoteId; + this.statement = statement; + this.shouldRunAsync = shouldRunAsync; + this.state = state; + this.eventTime = eventTime; + this.createTime = createTime; + this.startTime = startTime; + this.completeTime = completeTime; + this.exception = exception; + this.sessionId = sessionId; + this.sessionUser = sessionUser; + this.sessionType = sessionType; + this.kyuubiInstance = kyuubiInstance; + this.metrics = metrics; + } + + public static KyuubiOperationEvent.KyuubiOperationEventBuilder builder() { + return new KyuubiOperationEvent.KyuubiOperationEventBuilder(); + } + + public static class KyuubiOperationEventBuilder { + private String statementId; + + private String remoteId; + + private String statement; + + private boolean shouldRunAsync; + + private String state; + + private long eventTime; + + private long createTime; + + private long startTime; + + private long completeTime; + + private Throwable exception; + + private String sessionId; + + private String sessionUser; + + private String sessionType; + + private String kyuubiInstance; + + private Map metrics; + + public KyuubiOperationEventBuilder() {} + + public KyuubiOperationEvent.KyuubiOperationEventBuilder statementId(final String statementId) { + this.statementId = statementId; + return this; + } + + public KyuubiOperationEvent.KyuubiOperationEventBuilder remoteId(final String remoteId) { + this.remoteId = remoteId; + return this; + } + + public KyuubiOperationEvent.KyuubiOperationEventBuilder statement(final String statement) { + this.statement = statement; + return this; + } + + public KyuubiOperationEvent.KyuubiOperationEventBuilder shouldRunAsync( + final boolean shouldRunAsync) { + this.shouldRunAsync = shouldRunAsync; + return this; + } + + public KyuubiOperationEvent.KyuubiOperationEventBuilder state(final String state) { + this.state = state; + return this; + } + + public KyuubiOperationEvent.KyuubiOperationEventBuilder eventTime(final long eventTime) { + this.eventTime = eventTime; + return this; + } + + public KyuubiOperationEvent.KyuubiOperationEventBuilder createTime(final long createTime) { + this.createTime = createTime; + return this; + } + + public KyuubiOperationEvent.KyuubiOperationEventBuilder startTime(final long startTime) { + this.startTime = startTime; + return this; + } + + public KyuubiOperationEvent.KyuubiOperationEventBuilder completeTime(final long completeTime) { + this.completeTime = completeTime; + return this; + } + + public KyuubiOperationEvent.KyuubiOperationEventBuilder exception(final Throwable exception) { + this.exception = exception; + return this; + } + + public KyuubiOperationEvent.KyuubiOperationEventBuilder sessionId(final String sessionId) { + this.sessionId = sessionId; + return this; + } + + public KyuubiOperationEvent.KyuubiOperationEventBuilder sessionUser(final String sessionUser) { + this.sessionUser = sessionUser; + return this; + } + + public KyuubiOperationEvent.KyuubiOperationEventBuilder sessionType(final String sessionType) { + this.sessionType = sessionType; + return this; + } + + public KyuubiOperationEvent.KyuubiOperationEventBuilder kyuubiInstance( + final String kyuubiInstance) { + this.kyuubiInstance = kyuubiInstance; + return this; + } + + public KyuubiOperationEvent.KyuubiOperationEventBuilder metrics( + final Map metrics) { + this.metrics = metrics; + return this; + } + + public KyuubiOperationEvent build() { + return new KyuubiOperationEvent( + statementId, + remoteId, + statement, + shouldRunAsync, + state, + eventTime, + createTime, + startTime, + completeTime, + exception, + sessionId, + sessionUser, + sessionType, + kyuubiInstance, + metrics); + } + } + + public String getStatementId() { + return statementId; + } + + public void setStatementId(String statementId) { + this.statementId = statementId; + } + + public String getRemoteId() { + return remoteId; + } + + public void setRemoteId(String remoteId) { + this.remoteId = remoteId; + } + + public String getStatement() { + return statement; + } + + public void setStatement(String statement) { + this.statement = statement; + } + + public boolean isShouldRunAsync() { + return shouldRunAsync; + } + + public void setShouldRunAsync(boolean shouldRunAsync) { + this.shouldRunAsync = shouldRunAsync; + } + + public String getState() { + return state; + } + + public void setState(String state) { + this.state = state; + } + + public long getEventTime() { + return eventTime; + } + + public void setEventTime(long eventTime) { + this.eventTime = eventTime; + } + + public long getCreateTime() { + return createTime; + } + + public void setCreateTime(long createTime) { + this.createTime = createTime; + } + + public long getStartTime() { + return startTime; + } + + public void setStartTime(long startTime) { + this.startTime = startTime; + } + + public long getCompleteTime() { + return completeTime; + } + + public void setCompleteTime(long completeTime) { + this.completeTime = completeTime; + } + + public Throwable getException() { + return exception; + } + + public void setException(Throwable exception) { + this.exception = exception; + } + + public String getSessionId() { + return sessionId; + } + + public void setSessionId(String sessionId) { + this.sessionId = sessionId; + } + + public String getSessionUser() { + return sessionUser; + } + + public void setSessionUser(String sessionUser) { + this.sessionUser = sessionUser; + } + + public String getSessionType() { + return sessionType; + } + + public void setSessionType(String sessionType) { + this.sessionType = sessionType; + } + + public String getKyuubiInstance() { + return kyuubiInstance; + } + + public void setKyuubiInstance(String kyuubiInstance) { + this.kyuubiInstance = kyuubiInstance; + } + + public Map getMetrics() { + return metrics; + } + + public void setMetrics(Map metrics) { + this.metrics = metrics; + } +} diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/KyuubiSessionEvent.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/KyuubiSessionEvent.java new file mode 100644 index 000000000..34d306fed --- /dev/null +++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/KyuubiSessionEvent.java @@ -0,0 +1,361 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.client.api.v1.dto; + +import java.util.Map; + +public class KyuubiSessionEvent { + + private String sessionId; + + private int clientVersion; + + private String sessionType; + + private String sessionName; + + private String remoteSessionId; + + private String engineId; + + private String user; + + private String clientIp; + + private String serverIp; + + private Map conf; + + private long eventTime; + + private long openedTime; + + private long startTime; + + private long endTime; + + private int totalOperations; + + private Throwable exception; + + public KyuubiSessionEvent() {} + + public KyuubiSessionEvent( + String sessionId, + int clientVersion, + String sessionType, + String sessionName, + String remoteSessionId, + String engineId, + String user, + String clientIp, + String serverIp, + Map conf, + long eventTime, + long openedTime, + long startTime, + long endTime, + int totalOperations, + Throwable exception) { + this.sessionId = sessionId; + this.clientVersion = clientVersion; + this.sessionType = sessionType; + this.sessionName = sessionName; + this.remoteSessionId = remoteSessionId; + this.engineId = engineId; + this.user = user; + this.clientIp = clientIp; + this.serverIp = serverIp; + this.conf = conf; + this.eventTime = eventTime; + this.openedTime = openedTime; + this.startTime = startTime; + this.endTime = endTime; + this.totalOperations = totalOperations; + this.exception = exception; + } + + public static KyuubiSessionEvent.KyuubiSessionEventBuilder builder() { + return new KyuubiSessionEvent.KyuubiSessionEventBuilder(); + } + + public static class KyuubiSessionEventBuilder { + private String sessionId; + + private int clientVersion; + + private String sessionType; + + private String sessionName; + + private String remoteSessionId; + + private String engineId; + + private String user; + + private String clientIp; + + private String serverIp; + + private Map conf; + + private long eventTime; + + private long openedTime; + + private long startTime; + + private long endTime; + + private int totalOperations; + + private Throwable exception; + + public KyuubiSessionEventBuilder() {} + + public KyuubiSessionEvent.KyuubiSessionEventBuilder sessionId(final String sessionId) { + this.sessionId = sessionId; + return this; + } + + public KyuubiSessionEvent.KyuubiSessionEventBuilder remoteSessionId( + final String remoteSessionId) { + this.remoteSessionId = remoteSessionId; + return this; + } + + public KyuubiSessionEvent.KyuubiSessionEventBuilder clientVersion(final int clientVersion) { + this.clientVersion = clientVersion; + return this; + } + + public KyuubiSessionEvent.KyuubiSessionEventBuilder sessionType(final String sessionType) { + this.sessionType = sessionType; + return this; + } + + public KyuubiSessionEvent.KyuubiSessionEventBuilder sessionName(final String sessionName) { + this.sessionName = sessionName; + return this; + } + + public KyuubiSessionEvent.KyuubiSessionEventBuilder engineId(final String engineId) { + this.engineId = engineId; + return this; + } + + public KyuubiSessionEvent.KyuubiSessionEventBuilder user(final String user) { + this.user = user; + return this; + } + + public KyuubiSessionEvent.KyuubiSessionEventBuilder clientIp(final String clientIp) { + this.clientIp = clientIp; + return this; + } + + public KyuubiSessionEvent.KyuubiSessionEventBuilder serverIp(final String serverIp) { + this.serverIp = serverIp; + return this; + } + + public KyuubiSessionEvent.KyuubiSessionEventBuilder conf(final Map conf) { + this.conf = conf; + return this; + } + + public KyuubiSessionEvent.KyuubiSessionEventBuilder eventTime(final long eventTime) { + this.eventTime = eventTime; + return this; + } + + public KyuubiSessionEvent.KyuubiSessionEventBuilder openedTime(final long openedTime) { + this.openedTime = openedTime; + return this; + } + + public KyuubiSessionEvent.KyuubiSessionEventBuilder startTime(final long startTime) { + this.startTime = startTime; + return this; + } + + public KyuubiSessionEvent.KyuubiSessionEventBuilder endTime(final long endTime) { + this.endTime = endTime; + return this; + } + + public KyuubiSessionEvent.KyuubiSessionEventBuilder totalOperations(final int totalOperations) { + this.totalOperations = totalOperations; + return this; + } + + public KyuubiSessionEvent.KyuubiSessionEventBuilder exception(final Throwable exception) { + this.exception = exception; + return this; + } + + public KyuubiSessionEvent build() { + return new KyuubiSessionEvent( + sessionId, + clientVersion, + sessionType, + sessionName, + remoteSessionId, + engineId, + user, + clientIp, + serverIp, + conf, + eventTime, + openedTime, + startTime, + endTime, + totalOperations, + exception); + } + } + + public String getSessionId() { + return sessionId; + } + + public void setSessionId(String sessionId) { + this.sessionId = sessionId; + } + + public int getClientVersion() { + return clientVersion; + } + + public void setClientVersion(int clientVersion) { + this.clientVersion = clientVersion; + } + + public String getSessionType() { + return sessionType; + } + + public void setSessionType(String sessionType) { + this.sessionType = sessionType; + } + + public String getSessionName() { + return sessionName; + } + + public void setSessionName(String sessionName) { + this.sessionName = sessionName; + } + + public String getRemoteSessionId() { + return remoteSessionId; + } + + public void setRemoteSessionId(String remoteSessionId) { + this.remoteSessionId = remoteSessionId; + } + + public String getEngineId() { + return engineId; + } + + public void setEngineId(String engineId) { + this.engineId = engineId; + } + + public String getUser() { + return user; + } + + public void setUser(String user) { + this.user = user; + } + + public String getClientIp() { + return clientIp; + } + + public void setClientIp(String clientIp) { + this.clientIp = clientIp; + } + + public String getServerIp() { + return serverIp; + } + + public void setServerIp(String serverIp) { + this.serverIp = serverIp; + } + + public Map getConf() { + return conf; + } + + public void setConf(Map conf) { + this.conf = conf; + } + + public long getEventTime() { + return eventTime; + } + + public void setEventTime(long eventTime) { + this.eventTime = eventTime; + } + + public long getOpenedTime() { + return openedTime; + } + + public void setOpenedTime(long openedTime) { + this.openedTime = openedTime; + } + + public long getStartTime() { + return startTime; + } + + public void setStartTime(long startTime) { + this.startTime = startTime; + } + + public long getEndTime() { + return endTime; + } + + public void setEndTime(long endTime) { + this.endTime = endTime; + } + + public int getTotalOperations() { + return totalOperations; + } + + public void setTotalOperations(int totalOperations) { + this.totalOperations = totalOperations; + } + + public Throwable getException() { + return exception; + } + + public void setException(Throwable exception) { + this.exception = exception; + } +} diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/OperationData.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/OperationData.java new file mode 100644 index 000000000..70c2dd3f3 --- /dev/null +++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/OperationData.java @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.client.api.v1.dto; + +import java.util.Collections; +import java.util.Map; +import java.util.Objects; +import org.apache.commons.lang3.builder.ReflectionToStringBuilder; +import org.apache.commons.lang3.builder.ToStringStyle; + +public class OperationData { + private String identifier; + private String statement; + private String state; + private Long createTime; + private Long startTime; + private Long completeTime; + private String exception; + private String sessionId; + private String sessionUser; + private String sessionType; + private String kyuubiInstance; + private Map metrics; + + public OperationData() {} + + public OperationData( + String identifier, + String statement, + String state, + Long createTime, + Long startTime, + Long completeTime, + String exception, + String sessionId, + String sessionUser, + String sessionType, + String kyuubiInstance, + Map metrics) { + this.identifier = identifier; + this.statement = statement; + this.state = state; + this.createTime = createTime; + this.startTime = startTime; + this.completeTime = completeTime; + this.exception = exception; + this.sessionId = sessionId; + this.sessionUser = sessionUser; + this.sessionType = sessionType; + this.kyuubiInstance = kyuubiInstance; + this.metrics = metrics; + } + + public String getIdentifier() { + return identifier; + } + + public void setIdentifier(String identifier) { + this.identifier = identifier; + } + + public String getStatement() { + return statement; + } + + public void setStatement(String statement) { + this.statement = statement; + } + + public String getState() { + return state; + } + + public void setState(String state) { + this.state = state; + } + + public Long getCreateTime() { + return createTime; + } + + public void setCreateTime(Long createTime) { + this.createTime = createTime; + } + + public Long getStartTime() { + return startTime; + } + + public void setStartTime(Long startTime) { + this.startTime = startTime; + } + + public Long getCompleteTime() { + return completeTime; + } + + public void setCompleteTime(Long completeTime) { + this.completeTime = completeTime; + } + + public String getException() { + return exception; + } + + public void setException(String exception) { + this.exception = exception; + } + + public String getSessionId() { + return sessionId; + } + + public void setSessionId(String sessionId) { + this.sessionId = sessionId; + } + + public String getSessionUser() { + return sessionUser; + } + + public void setSessionUser(String sessionUser) { + this.sessionUser = sessionUser; + } + + public String getSessionType() { + return sessionType; + } + + public void setSessionType(String sessionType) { + this.sessionType = sessionType; + } + + public String getKyuubiInstance() { + return kyuubiInstance; + } + + public void setKyuubiInstance(String kyuubiInstance) { + this.kyuubiInstance = kyuubiInstance; + } + + public Map getMetrics() { + if (null == metrics) { + return Collections.emptyMap(); + } + return metrics; + } + + public void setMetrics(Map metrics) { + this.metrics = metrics; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + OperationData that = (OperationData) o; + return Objects.equals(getIdentifier(), that.getIdentifier()); + } + + @Override + public int hashCode() { + return Objects.hash(getIdentifier()); + } + + @Override + public String toString() { + return ReflectionToStringBuilder.toString(this, ToStringStyle.JSON_STYLE); + } +} diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/OperationHandle.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/OperationHandle.java new file mode 100644 index 000000000..394e6c157 --- /dev/null +++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/OperationHandle.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.client.api.v1.dto; + +import java.util.Objects; +import java.util.UUID; +import org.apache.commons.lang3.builder.ReflectionToStringBuilder; +import org.apache.commons.lang3.builder.ToStringStyle; + +public class OperationHandle { + + private UUID identifier; + + public OperationHandle() {} + + public OperationHandle(UUID identifier) { + this.identifier = identifier; + } + + public UUID getIdentifier() { + return identifier; + } + + public void setIdentifier(UUID identifier) { + this.identifier = identifier; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + OperationHandle that = (OperationHandle) o; + return Objects.equals(identifier, that.identifier); + } + + @Override + public int hashCode() { + return Objects.hash(identifier); + } + + @Override + public String toString() { + return ReflectionToStringBuilder.toString(this, ToStringStyle.JSON_STYLE); + } +} diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/ServerData.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/ServerData.java new file mode 100644 index 000000000..7b68763d2 --- /dev/null +++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/ServerData.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.client.api.v1.dto; + +import java.util.Collections; +import java.util.Map; +import java.util.Objects; + +public class ServerData { + private String nodeName; + private String namespace; + private String instance; + private String host; + private int port; + private Map attributes; + private String status; + + public ServerData() {} + + public ServerData( + String nodeName, + String namespace, + String instance, + String host, + int port, + Map attributes, + String status) { + this.nodeName = nodeName; + this.namespace = namespace; + this.instance = instance; + this.host = host; + this.port = port; + this.attributes = attributes; + this.status = status; + } + + public String getNodeName() { + return nodeName; + } + + public ServerData setNodeName(String nodeName) { + this.nodeName = nodeName; + return this; + } + + public String getNamespace() { + return namespace; + } + + public ServerData setNamespace(String namespace) { + this.namespace = namespace; + return this; + } + + public String getInstance() { + return instance; + } + + public ServerData setInstance(String instance) { + this.instance = instance; + return this; + } + + public String getHost() { + return host; + } + + public ServerData setHost(String host) { + this.host = host; + return this; + } + + public int getPort() { + return port; + } + + public ServerData setPort(int port) { + this.port = port; + return this; + } + + public Map getAttributes() { + if (null == attributes) { + return Collections.emptyMap(); + } + return attributes; + } + + public ServerData setAttributes(Map attributes) { + this.attributes = attributes; + return this; + } + + public String getStatus() { + return status; + } + + public ServerData setStatus(String status) { + this.status = status; + return this; + } + + @Override + public int hashCode() { + return Objects.hash(nodeName, namespace, instance, port, attributes, status); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null || getClass() != obj.getClass()) return false; + ServerData server = (ServerData) obj; + return port == server.port + && Objects.equals(nodeName, server.nodeName) + && Objects.equals(namespace, server.namespace) + && Objects.equals(instance, server.instance) + && Objects.equals(host, server.host) + && Objects.equals(status, server.status); + } +} diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/SessionData.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/SessionData.java index bae6f39da..ae7dfdec9 100644 --- a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/SessionData.java +++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/SessionData.java @@ -31,6 +31,10 @@ public class SessionData { private Long createTime; private Long duration; private Long idleTime; + private String exception; + private String sessionType; + private String kyuubiInstance; + private String engineId; public SessionData() {} @@ -41,7 +45,11 @@ public SessionData( Map conf, Long createTime, Long duration, - Long idleTime) { + Long idleTime, + String exception, + String sessionType, + String kyuubiInstance, + String engineId) { this.identifier = identifier; this.user = user; this.ipAddr = ipAddr; @@ -49,6 +57,10 @@ public SessionData( this.createTime = createTime; this.duration = duration; this.idleTime = idleTime; + this.exception = exception; + this.sessionType = sessionType; + this.kyuubiInstance = kyuubiInstance; + this.engineId = engineId; } public String getIdentifier() { @@ -110,6 +122,38 @@ public void setIdleTime(Long idleTime) { this.idleTime = idleTime; } + public String getException() { + return exception; + } + + public void setException(String exception) { + this.exception = exception; + } + + public String getSessionType() { + return sessionType; + } + + public void setSessionType(String sessionType) { + this.sessionType = sessionType; + } + + public String getKyuubiInstance() { + return kyuubiInstance; + } + + public void setKyuubiInstance(String kyuubiInstance) { + this.kyuubiInstance = kyuubiInstance; + } + + public String getEngineId() { + return engineId; + } + + public void setEngineId(String engineId) { + this.engineId = engineId; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/SessionOpenRequest.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/SessionOpenRequest.java index 2d23aac57..06eb29e97 100644 --- a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/SessionOpenRequest.java +++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/SessionOpenRequest.java @@ -24,24 +24,14 @@ import org.apache.commons.lang3.builder.ToStringStyle; public class SessionOpenRequest { - private int protocolVersion; private Map configs; public SessionOpenRequest() {} - public SessionOpenRequest(int protocolVersion, Map configs) { - this.protocolVersion = protocolVersion; + public SessionOpenRequest(Map configs) { this.configs = configs; } - public int getProtocolVersion() { - return protocolVersion; - } - - public void setProtocolVersion(int protocolVersion) { - this.protocolVersion = protocolVersion; - } - public Map getConfigs() { if (null == configs) { return Collections.emptyMap(); @@ -58,13 +48,12 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; SessionOpenRequest that = (SessionOpenRequest) o; - return getProtocolVersion() == that.getProtocolVersion() - && Objects.equals(getConfigs(), that.getConfigs()); + return Objects.equals(getConfigs(), that.getConfigs()); } @Override public int hashCode() { - return Objects.hash(getProtocolVersion(), getConfigs()); + return Objects.hash(getConfigs()); } @Override diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/StatementRequest.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/StatementRequest.java index 436017f3c..f2dc060d5 100644 --- a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/StatementRequest.java +++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/StatementRequest.java @@ -17,6 +17,8 @@ package org.apache.kyuubi.client.api.v1.dto; +import java.util.Collections; +import java.util.Map; import java.util.Objects; import org.apache.commons.lang3.builder.ReflectionToStringBuilder; import org.apache.commons.lang3.builder.ToStringStyle; @@ -25,13 +27,20 @@ public class StatementRequest { private String statement; private boolean runAsync; private Long queryTimeout; + private Map confOverlay; public StatementRequest() {} public StatementRequest(String statement, boolean runAsync, Long queryTimeout) { + this(statement, runAsync, queryTimeout, Collections.emptyMap()); + } + + public StatementRequest( + String statement, boolean runAsync, Long queryTimeout, Map confOverlay) { this.statement = statement; this.runAsync = runAsync; this.queryTimeout = queryTimeout; + this.confOverlay = confOverlay; } public String getStatement() { @@ -58,6 +67,17 @@ public void setQueryTimeout(Long queryTimeout) { this.queryTimeout = queryTimeout; } + public Map getConfOverlay() { + if (confOverlay == null) { + return Collections.emptyMap(); + } + return confOverlay; + } + + public void setConfOverlay(Map confOverlay) { + this.confOverlay = confOverlay; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/VersionInfo.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/VersionInfo.java index 427272f41..5749c4e32 100644 --- a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/VersionInfo.java +++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/api/v1/dto/VersionInfo.java @@ -17,6 +17,8 @@ package org.apache.kyuubi.client.api.v1.dto; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; import java.util.Objects; import org.apache.commons.lang3.builder.ReflectionToStringBuilder; import org.apache.commons.lang3.builder.ToStringStyle; @@ -26,10 +28,13 @@ public class VersionInfo { public VersionInfo() {} - public VersionInfo(String version) { + // Explicitly specifies JsonProperty to be compatible if disable auto detect feature + @JsonCreator + public VersionInfo(@JsonProperty("version") String version) { this.version = version; } + @JsonProperty public String getVersion() { return version; } diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/auth/SpnegoAuthHeaderGenerator.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/auth/SpnegoAuthHeaderGenerator.java index 435a85014..c66c6465e 100644 --- a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/auth/SpnegoAuthHeaderGenerator.java +++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/auth/SpnegoAuthHeaderGenerator.java @@ -17,13 +17,13 @@ package org.apache.kyuubi.client.auth; -import java.lang.reflect.Field; -import java.lang.reflect.Method; import java.nio.charset.StandardCharsets; import java.security.PrivilegedExceptionAction; import java.util.Base64; import javax.security.auth.Subject; import org.apache.kyuubi.client.exception.KyuubiRestException; +import org.apache.kyuubi.util.reflect.DynFields; +import org.apache.kyuubi.util.reflect.DynMethods; import org.ietf.jgss.GSSContext; import org.ietf.jgss.GSSException; import org.ietf.jgss.GSSManager; @@ -61,13 +61,17 @@ public String generateAuthHeader() { private String generateToken(String server) throws Exception { Subject subject; try { - Class ugiClz = Class.forName(UGI_CLASS); - Method ugiGetCurrentUserMethod = ugiClz.getDeclaredMethod("getCurrentUser"); - Object ugiCurrentUser = ugiGetCurrentUserMethod.invoke(null); + Object ugiCurrentUser = + DynMethods.builder("getCurrentUser") + .hiddenImpl(Class.forName(UGI_CLASS)) + .buildStaticChecked() + .invoke(); LOG.debug("The user credential is {}", ugiCurrentUser); - Field ugiSubjectField = ugiCurrentUser.getClass().getDeclaredField("subject"); - ugiSubjectField.setAccessible(true); - subject = (Subject) ugiSubjectField.get(ugiCurrentUser); + subject = + DynFields.builder() + .hiddenImpl(ugiCurrentUser.getClass(), "subject") + .buildChecked(ugiCurrentUser) + .get(); } catch (ClassNotFoundException e) { // TODO do kerberos authentication using JDK class directly LOG.error("Hadoop UGI class {} is required for SPNEGO authentication.", UGI_CLASS); diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/util/BatchUtils.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/util/BatchUtils.java index 59f5967a0..f7efaad9d 100644 --- a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/util/BatchUtils.java +++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/util/BatchUtils.java @@ -20,6 +20,7 @@ import java.util.Arrays; import java.util.List; import java.util.Locale; +import org.apache.kyuubi.client.api.v1.dto.Batch; public final class BatchUtils { /** The batch has not been submitted to resource manager yet. */ @@ -40,6 +41,10 @@ public final class BatchUtils { public static List terminalBatchStates = Arrays.asList(FINISHED_STATE, ERROR_STATE, CANCELED_STATE); + public static String KYUUBI_BATCH_ID_KEY = "kyuubi.batch.id"; + + public static String KYUUBI_BATCH_DUPLICATED_KEY = "kyuubi.batch.duplicated"; + public static boolean isPendingState(String state) { return PENDING_STATE.equalsIgnoreCase(state); } @@ -55,4 +60,8 @@ public static boolean isFinishedState(String state) { public static boolean isTerminalState(String state) { return state != null && terminalBatchStates.contains(state.toUpperCase(Locale.ROOT)); } + + public static boolean isDuplicatedSubmission(Batch batch) { + return "true".equalsIgnoreCase(batch.getBatchInfo().get(KYUUBI_BATCH_DUPLICATED_KEY)); + } } diff --git a/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/util/VersionUtils.java b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/util/VersionUtils.java new file mode 100644 index 000000000..1f8cedf4b --- /dev/null +++ b/kyuubi-rest-client/src/main/java/org/apache/kyuubi/client/util/VersionUtils.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.client.util; + +import java.util.Properties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class VersionUtils { + static final Logger LOG = LoggerFactory.getLogger(VersionUtils.class); + + public static final String KYUUBI_CLIENT_VERSION_KEY = "kyuubi.client.version"; + private static String KYUUBI_CLIENT_VERSION; + + public static synchronized String getVersion() { + if (KYUUBI_CLIENT_VERSION == null) { + try { + Properties prop = new Properties(); + prop.load( + VersionUtils.class + .getClassLoader() + .getResourceAsStream("org/apache/kyuubi/version.properties")); + KYUUBI_CLIENT_VERSION = prop.getProperty(KYUUBI_CLIENT_VERSION_KEY, "unknown"); + } catch (Exception e) { + LOG.error("Error getting kyuubi client version", e); + KYUUBI_CLIENT_VERSION = "unknown"; + } + } + return KYUUBI_CLIENT_VERSION; + } +} diff --git a/kyuubi-rest-client/src/main/resources/org/apache/kyuubi/version.properties b/kyuubi-rest-client/src/main/resources/org/apache/kyuubi/version.properties new file mode 100644 index 000000000..82ae50cfb --- /dev/null +++ b/kyuubi-rest-client/src/main/resources/org/apache/kyuubi/version.properties @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +kyuubi.client.version = ${project.version} diff --git a/kyuubi-rest-client/src/test/java/org/apache/kyuubi/client/RestClientTestUtils.java b/kyuubi-rest-client/src/test/java/org/apache/kyuubi/client/RestClientTestUtils.java index 82413e2a4..1ac0278bf 100644 --- a/kyuubi-rest-client/src/test/java/org/apache/kyuubi/client/RestClientTestUtils.java +++ b/kyuubi-rest-client/src/test/java/org/apache/kyuubi/client/RestClientTestUtils.java @@ -45,35 +45,31 @@ public static CloseBatchResponse generateTestCloseBatchResp() { } public static Batch generateTestBatch(String id) { - Batch batch = - new Batch( - id, - TEST_USERNAME, - "spark", - "batch_name", - 0, - id, - null, - "RUNNING", - null, - "192.168.31.130:64573", - "RUNNING", - BATCH_CREATE_TIME, - 0); - - return batch; + return new Batch( + id, + TEST_USERNAME, + "spark", + "batch_name", + 0, + id, + null, + "RUNNING", + null, + "192.168.31.130:64573", + "RUNNING", + BATCH_CREATE_TIME, + 0, + Collections.emptyMap()); } public static BatchRequest generateTestBatchRequest() { - BatchRequest batchRequest = - new BatchRequest( - "spark", - "/MySpace/kyuubi-spark-sql-engine_2.12-1.6.0-SNAPSHOT.jar", - "org.apache.kyuubi.engine.spark.SparkSQLEngine", - "test_batch", - Collections.singletonMap("spark.driver.memory", "16m"), - Collections.emptyList()); - return batchRequest; + return new BatchRequest( + "spark", + "/MySpace/kyuubi-spark-sql-engine_2.12-1.6.0-SNAPSHOT.jar", + "org.apache.kyuubi.engine.spark.SparkSQLEngine", + "test_batch", + Collections.singletonMap("spark.driver.memory", "16m"), + Collections.emptyList()); } public static GetBatchesResponse generateTestBatchesResponse() { @@ -87,9 +83,8 @@ public static GetBatchesResponse generateTestBatchesResponse() { public static OperationLog generateTestOperationLog() { List logs = Arrays.asList( - "13:15:13.523 INFO org.apache.curator.framework.state." - + "ConnectionStateManager: State change: CONNECTED", - "13:15:13.528 INFO org.apache.kyuubi." + "engine.EngineRef: Launching engine:"); + "13:15:13.523 INFO ConnectionStateManager: State change: CONNECTED", + "13:15:13.528 INFO EngineRef: Launching engine:"); return new OperationLog(logs, 2); } } diff --git a/kyuubi-rest-client/src/test/java/org/apache/kyuubi/client/util/VersionUtilsTest.java b/kyuubi-rest-client/src/test/java/org/apache/kyuubi/client/util/VersionUtilsTest.java new file mode 100644 index 000000000..d4675f340 --- /dev/null +++ b/kyuubi-rest-client/src/test/java/org/apache/kyuubi/client/util/VersionUtilsTest.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kyuubi.client.util; + +import java.util.regex.Pattern; +import org.junit.Test; + +public class VersionUtilsTest { + + @Test + public void testGetClientVersion() { + Pattern pattern = Pattern.compile("^\\d+\\.\\d+\\.\\d+.*"); + assert pattern.matcher(VersionUtils.getVersion()).matches(); + } +} diff --git a/kyuubi-rest-client/src/test/resources/log4j2-test.xml b/kyuubi-rest-client/src/test/resources/log4j2-test.xml index 13ea5322a..2f13b5777 100644 --- a/kyuubi-rest-client/src/test/resources/log4j2-test.xml +++ b/kyuubi-rest-client/src/test/resources/log4j2-test.xml @@ -21,13 +21,13 @@ - + - + diff --git a/kyuubi-server/pom.xml b/kyuubi-server/pom.xml index 4dd89e0e6..56155a27b 100644 --- a/kyuubi-server/pom.xml +++ b/kyuubi-server/pom.xml @@ -21,10 +21,10 @@ org.apache.kyuubi kyuubi-parent - 1.7.0-SNAPSHOT + 1.9.0-SNAPSHOT - kyuubi-server_2.12 + kyuubi-server_${scala.binary.version} jar Kyuubi Project Server https://kyuubi.apache.org/ @@ -36,6 +36,12 @@ ${project.version} + + org.apache.kyuubi + kyuubi-hive-jdbc + ${project.version} + + org.apache.kyuubi kyuubi-events_${scala.binary.version} @@ -92,6 +98,11 @@ kubernetes-client + + io.fabric8 + kubernetes-httpclient-okhttp + + org.apache.hive hive-metastore @@ -221,6 +232,16 @@ jersey-media-multipart + + com.fasterxml.jackson.core + jackson-databind + + + + com.fasterxml.jackson.datatype + jackson-datatype-jdk8 + + com.zaxxer HikariCP @@ -231,11 +252,21 @@ derby + + org.xerial + sqlite-jdbc + + io.trino trino-client + + org.eclipse.jetty + jetty-proxy + + org.glassfish.jersey.test-framework jersey-test-framework-core @@ -370,8 +401,20 @@ - org.webjars - swagger-ui + org.apache.kafka + kafka-clients + + + + com.dimafeng + testcontainers-scala-scalatest_${scala.binary.version} + test + + + + com.dimafeng + testcontainers-scala-kafka_${scala.binary.version} + test @@ -406,45 +449,9 @@ test - - org.apache.spark - spark-avro_${scala.binary.version} - test - - - - org.apache.parquet - parquet-avro - test - - - - org.apache.hudi - hudi-common - test - - - - org.apache.hudi - hudi-spark-common_${scala.binary.version} - test - - - - org.apache.hudi - hudi-spark_${scala.binary.version} - test - - - - org.apache.hudi - hudi-spark3.1.x_${scala.binary.version} - test - - io.delta - delta-core_${scala.binary.version} + ${delta.artifact}_${scala.binary.version} test @@ -474,7 +481,7 @@ org.scalatestplus - mockito-4-6_${scala.binary.version} + mockito-4-11_${scala.binary.version} test @@ -533,6 +540,47 @@     + + + com.github.eirslett + frontend-maven-plugin + + web-ui + + + + install node and pnpm + + install-node-and-pnpm + + + ${webui.skip} + + + + pnpm install + + pnpm + + generate-resources + + ${webui.skip} + install + + + + pnpm run build + + pnpm + + package + + ${webui.skip} + run build + + + + target/scala-${scala.binary.version}/classes target/scala-${scala.binary.version}/test-classes diff --git a/kyuubi-server/src/main/antlr4/org/apache/kyuubi/sql/KyuubiTrinoFeBaseLexer.g4 b/kyuubi-server/src/main/antlr4/org/apache/kyuubi/sql/KyuubiTrinoFeBaseLexer.g4 index 0b9543a43..83810a073 100644 --- a/kyuubi-server/src/main/antlr4/org/apache/kyuubi/sql/KyuubiTrinoFeBaseLexer.g4 +++ b/kyuubi-server/src/main/antlr4/org/apache/kyuubi/sql/KyuubiTrinoFeBaseLexer.g4 @@ -43,6 +43,10 @@ FALSE: 'FALSE'; LIKE: 'LIKE'; IN: 'IN'; WHERE: 'WHERE'; +EXECUTE: 'EXECUTE'; +PREPARE: 'PREPARE'; +DEALLOCATE: 'DEALLOCATE'; +USING: 'USING'; ESCAPE: 'ESCAPE'; AUTO_INCREMENT: 'AUTO_INCREMENT'; @@ -97,6 +101,21 @@ SCOPE_TABLE: 'SCOPE_TABLE'; SOURCE_DATA_TYPE: 'SOURCE_DATA_TYPE'; IS_AUTOINCREMENT: 'IS_AUTOINCREMENT'; IS_GENERATEDCOLUMN: 'IS_GENERATEDCOLUMN'; +VARCHAR: 'VARCHAR'; +TINYINT: 'TINYINT'; +SMALLINT: 'SMALLINT'; +INTEGER: 'INTEGER'; +BIGINT: 'BIGINT'; +REAL: 'REAL'; +DOUBLE: 'DOUBLE'; +DECIMAL: 'DECIMAL'; +DATE: 'DATE'; +TIME: 'TIME'; +TIMESTAMP: 'TIMESTAMP'; +CAST: 'CAST'; +AS: 'AS'; +KEY_SEQ: 'KEY_SEQ'; +PK_NAME: 'PK_NAME'; fragment SEARCH_STRING_ESCAPE: '\'' '\\' '\''; @@ -108,6 +127,10 @@ STRING : '\'' ( ~'\'' | '\'\'' )* '\'' ; +STRING_MARK + : '\'' + ; + SIMPLE_COMMENT : '--' ~[\r\n]* '\r'? '\n'? -> channel(HIDDEN) ; @@ -119,6 +142,10 @@ BRACKETED_COMMENT WS : [ \r\n\t]+ -> channel(HIDDEN) ; +IDENTIFIER + : [A-Za-z_$0-9\u0080-\uFFFF]*?[A-Za-z_$\u0080-\uFFFF]+?[A-Za-z_$0-9\u0080-\uFFFF]* + ; + // Catch-all for anything we can't recognize. // We use this to be able to ignore and recover all the text // when splitting statements with DelimiterLexer diff --git a/kyuubi-server/src/main/antlr4/org/apache/kyuubi/sql/KyuubiTrinoFeBaseParser.g4 b/kyuubi-server/src/main/antlr4/org/apache/kyuubi/sql/KyuubiTrinoFeBaseParser.g4 index 590c4378d..72811e592 100644 --- a/kyuubi-server/src/main/antlr4/org/apache/kyuubi/sql/KyuubiTrinoFeBaseParser.g4 +++ b/kyuubi-server/src/main/antlr4/org/apache/kyuubi/sql/KyuubiTrinoFeBaseParser.g4 @@ -47,9 +47,27 @@ statement SOURCE_DATA_TYPE COMMA IS_AUTOINCREMENT COMMA IS_GENERATEDCOLUMN FROM SYSTEM_JDBC_COLUMNS (WHERE tableCatalogFilter? AND? tableSchemaFilter? AND? tableNameFilter? AND? colNameFilter?)? ORDER BY TABLE_CAT COMMA TABLE_SCHEM COMMA TABLE_NAME COMMA ORDINAL_POSITION #getColumns + | SELECT CAST LEFT_PAREN NULL AS VARCHAR RIGHT_PAREN TABLE_CAT COMMA + CAST LEFT_PAREN NULL AS VARCHAR RIGHT_PAREN TABLE_SCHEM COMMA + CAST LEFT_PAREN NULL AS VARCHAR RIGHT_PAREN TABLE_NAME COMMA + CAST LEFT_PAREN NULL AS VARCHAR RIGHT_PAREN COLUMN_NAME COMMA + CAST LEFT_PAREN NULL AS SMALLINT RIGHT_PAREN KEY_SEQ COMMA + CAST LEFT_PAREN NULL AS VARCHAR RIGHT_PAREN PK_NAME + WHERE FALSE #getPrimaryKeys + | EXECUTE IDENTIFIER (USING parameterList)? #execute + | PREPARE IDENTIFIER FROM statement #prepare + | DEALLOCATE PREPARE IDENTIFIER #deallocate | .*? #passThrough ; +anyStr + : ( ~',' )* + ; + +parameterList + : (TINYINT|SMALLINT|INTEGER|BIGINT|DOUBLE|REAL|DECIMAL|DATE|TIME|TIMESTAMP)? anyStr (',' (TINYINT|SMALLINT|INTEGER|BIGINT|DOUBLE|REAL|DECIMAL|DATE|TIME|TIMESTAMP)? anyStr)* + ; + tableCatalogFilter : (TABLE_CAT | TABLE_CATALOG) IS NULL #nullCatalog | (TABLE_CAT | TABLE_CATALOG) EQ catalog=STRING+ #catalogFilter diff --git a/kyuubi-server/web-ui/src/views/workload/analysis/index.vue b/kyuubi-server/src/main/resources/dist/index.html similarity index 79% rename from kyuubi-server/web-ui/src/views/workload/analysis/index.vue rename to kyuubi-server/src/main/resources/dist/index.html index 31b42d46e..ab54fc14a 100644 --- a/kyuubi-server/web-ui/src/views/workload/analysis/index.vue +++ b/kyuubi-server/src/main/resources/dist/index.html @@ -16,14 +16,13 @@ * limitations under the License. --> - - - - - + + + + + Apache Kyuubi Dashboard + + +
    This is a dummy page for development.
    + + diff --git a/kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/assets/fonts/icons.eot b/kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/assets/fonts/icons.eot deleted file mode 100644 index 33b2bb800..000000000 Binary files a/kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/assets/fonts/icons.eot and /dev/null differ diff --git a/kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/assets/fonts/icons.svg b/kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/assets/fonts/icons.svg deleted file mode 100644 index 1ee89d436..000000000 --- a/kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/assets/fonts/icons.svg +++ /dev/null @@ -1,565 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/assets/fonts/icons.ttf b/kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/assets/fonts/icons.ttf deleted file mode 100644 index ed9372f8e..000000000 Binary files a/kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/assets/fonts/icons.ttf and /dev/null differ diff --git a/kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/assets/fonts/icons.woff b/kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/assets/fonts/icons.woff deleted file mode 100644 index 8b280b98f..000000000 Binary files a/kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/assets/fonts/icons.woff and /dev/null differ diff --git a/kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/assets/fonts/icons.woff2 b/kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/assets/fonts/icons.woff2 deleted file mode 100644 index 3311d5851..000000000 Binary files a/kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/assets/fonts/icons.woff2 and /dev/null differ diff --git a/kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/environment.html b/kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/environment.html deleted file mode 100644 index 036111b1e..000000000 --- a/kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/environment.html +++ /dev/null @@ -1,59 +0,0 @@ - - - - - - - Kyuubi Server Environments - - - - - - - - -
    -
    -
    - -
    - Home - Environment - Sessions - Operations - Metrics - Thread Dump - REST API - Documentation - En -
    -
    - -
    -
    - -
    -
    Hang on...
    -

    We are looking for developers to help us build the UI.

    -
    -
    -
    - - - diff --git a/kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/icon.min.css b/kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/icon.min.css deleted file mode 100755 index 8edb4dd0d..000000000 --- a/kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/icon.min.css +++ /dev/null @@ -1,9 +0,0 @@ -/*! - * # Semantic UI 2.3.3 - Icon - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */@font-face{font-family:Icons;src:url(assets/fonts/icons.eot);src:url(assets/fonts/icons.eot?#iefix) format('embedded-opentype'),url(assets/fonts/icons.woff2) format('woff2'),url(assets/fonts/icons.woff) format('woff'),url(assets/fonts/icons.ttf) format('truetype'),url(assets/fonts/icons.svg#icons) format('svg');font-style:normal;font-weight:400;font-variant:normal;text-decoration:inherit;text-transform:none}i.icon{display:inline-block;opacity:1;margin:0 .25rem 0 0;width:1.18em;height:1em;font-family:Icons;font-style:normal;font-weight:400;text-decoration:inherit;text-align:center;speak:none;font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale;-webkit-font-smoothing:antialiased;-webkit-backface-visibility:hidden;backface-visibility:hidden}i.icon:before{background:0 0!important}i.icon.loading{height:1em;line-height:1;-webkit-animation:icon-loading 2s linear infinite;animation:icon-loading 2s linear infinite}@-webkit-keyframes icon-loading{from{-webkit-transform:rotate(0);transform:rotate(0)}to{-webkit-transform:rotate(360deg);transform:rotate(360deg)}}@keyframes icon-loading{from{-webkit-transform:rotate(0);transform:rotate(0)}to{-webkit-transform:rotate(360deg);transform:rotate(360deg)}}i.icon.hover{opacity:1!important}i.icon.active{opacity:1!important}i.emphasized.icon{opacity:1!important}i.disabled.icon{opacity:.45!important}i.fitted.icon{width:auto;margin:0!important}i.link.icon,i.link.icons{cursor:pointer;opacity:.8;-webkit-transition:opacity .1s ease;transition:opacity .1s ease}i.link.icon:hover,i.link.icons:hover{opacity:1!important}i.circular.icon{border-radius:500em!important;line-height:1!important;padding:.5em 0!important;-webkit-box-shadow:0 0 0 .1em rgba(0,0,0,.1) inset;box-shadow:0 0 0 .1em rgba(0,0,0,.1) inset;width:2em!important;height:2em!important}i.circular.inverted.icon{border:none;-webkit-box-shadow:none;box-shadow:none}i.flipped.icon,i.horizontally.flipped.icon{-webkit-transform:scale(-1,1);transform:scale(-1,1)}i.vertically.flipped.icon{-webkit-transform:scale(1,-1);transform:scale(1,-1)}i.clockwise.rotated.icon,i.right.rotated.icon,i.rotated.icon{-webkit-transform:rotate(90deg);transform:rotate(90deg)}i.counterclockwise.rotated.icon,i.left.rotated.icon{-webkit-transform:rotate(-90deg);transform:rotate(-90deg)}i.bordered.icon{line-height:1;vertical-align:baseline;width:2em;height:2em;padding:.5em 0!important;-webkit-box-shadow:0 0 0 .1em rgba(0,0,0,.1) inset;box-shadow:0 0 0 .1em rgba(0,0,0,.1) inset}i.bordered.inverted.icon{border:none;-webkit-box-shadow:none;box-shadow:none}i.inverted.bordered.icon,i.inverted.circular.icon{background-color:#1b1c1d!important;color:#fff!important}i.inverted.icon{color:#fff}i.red.icon{color:#db2828!important}i.inverted.red.icon{color:#ff695e!important}i.inverted.bordered.red.icon,i.inverted.circular.red.icon{background-color:#db2828!important;color:#fff!important}i.orange.icon{color:#f2711c!important}i.inverted.orange.icon{color:#ff851b!important}i.inverted.bordered.orange.icon,i.inverted.circular.orange.icon{background-color:#f2711c!important;color:#fff!important}i.yellow.icon{color:#fbbd08!important}i.inverted.yellow.icon{color:#ffe21f!important}i.inverted.bordered.yellow.icon,i.inverted.circular.yellow.icon{background-color:#fbbd08!important;color:#fff!important}i.olive.icon{color:#b5cc18!important}i.inverted.olive.icon{color:#d9e778!important}i.inverted.bordered.olive.icon,i.inverted.circular.olive.icon{background-color:#b5cc18!important;color:#fff!important}i.green.icon{color:#21ba45!important}i.inverted.green.icon{color:#2ecc40!important}i.inverted.bordered.green.icon,i.inverted.circular.green.icon{background-color:#21ba45!important;color:#fff!important}i.teal.icon{color:#00b5ad!important}i.inverted.teal.icon{color:#6dffff!important}i.inverted.bordered.teal.icon,i.inverted.circular.teal.icon{background-color:#00b5ad!important;color:#fff!important}i.blue.icon{color:#2185d0!important}i.inverted.blue.icon{color:#54c8ff!important}i.inverted.bordered.blue.icon,i.inverted.circular.blue.icon{background-color:#2185d0!important;color:#fff!important}i.violet.icon{color:#6435c9!important}i.inverted.violet.icon{color:#a291fb!important}i.inverted.bordered.violet.icon,i.inverted.circular.violet.icon{background-color:#6435c9!important;color:#fff!important}i.purple.icon{color:#a333c8!important}i.inverted.purple.icon{color:#dc73ff!important}i.inverted.bordered.purple.icon,i.inverted.circular.purple.icon{background-color:#a333c8!important;color:#fff!important}i.pink.icon{color:#e03997!important}i.inverted.pink.icon{color:#ff8edf!important}i.inverted.bordered.pink.icon,i.inverted.circular.pink.icon{background-color:#e03997!important;color:#fff!important}i.brown.icon{color:#a5673f!important}i.inverted.brown.icon{color:#d67c1c!important}i.inverted.bordered.brown.icon,i.inverted.circular.brown.icon{background-color:#a5673f!important;color:#fff!important}i.grey.icon{color:#767676!important}i.inverted.grey.icon{color:#dcddde!important}i.inverted.bordered.grey.icon,i.inverted.circular.grey.icon{background-color:#767676!important;color:#fff!important}i.black.icon{color:#1b1c1d!important}i.inverted.black.icon{color:#545454!important}i.inverted.bordered.black.icon,i.inverted.circular.black.icon{background-color:#1b1c1d!important;color:#fff!important}i.mini.icon,i.mini.icons{line-height:1;font-size:.4em}i.tiny.icon,i.tiny.icons{line-height:1;font-size:.5em}i.small.icon,i.small.icons{line-height:1;font-size:.75em}i.icon,i.icons{font-size:1em}i.large.icon,i.large.icons{line-height:1;vertical-align:middle;font-size:1.5em}i.big.icon,i.big.icons{line-height:1;vertical-align:middle;font-size:2em}i.huge.icon,i.huge.icons{line-height:1;vertical-align:middle;font-size:4em}i.massive.icon,i.massive.icons{line-height:1;vertical-align:middle;font-size:8em}i.icons{display:inline-block;position:relative;line-height:1}i.icons .icon{position:absolute;top:50%;left:50%;-webkit-transform:translateX(-50%) translateY(-50%);transform:translateX(-50%) translateY(-50%);margin:0;margin:0}i.icons .icon:first-child{position:static;width:auto;height:auto;vertical-align:top;-webkit-transform:none;transform:none;margin-right:.25rem}i.icons .corner.icon{top:auto;left:auto;right:0;bottom:0;-webkit-transform:none;transform:none;font-size:.45em;text-shadow:-1px -1px 0 #fff,1px -1px 0 #fff,-1px 1px 0 #fff,1px 1px 0 #fff}i.icons .top.right.corner.icon{top:0;left:auto;right:0;bottom:auto}i.icons .top.left.corner.icon{top:0;left:0;right:auto;bottom:auto}i.icons .bottom.left.corner.icon{top:auto;left:0;right:auto;bottom:0}i.icons .bottom.right.corner.icon{top:auto;left:auto;right:0;bottom:0}i.icons .inverted.corner.icon{text-shadow:-1px -1px 0 #1b1c1d,1px -1px 0 #1b1c1d,-1px 1px 0 #1b1c1d,1px 1px 0 #1b1c1d}i.icon.linkedin.in:before{content:"\f0e1"}i.icon.zoom.in:before{content:"\f00e"}i.icon.zoom.out:before{content:"\f010"}i.icon.sign.in:before{content:"\f2f6"}i.icon.in.cart:before{content:"\f218"}i.icon.log.out:before{content:"\f2f5"}i.icon.sign.out:before{content:"\f2f5"}i.icon.\35 00px:before{content:"\f26e"}i.icon.accessible.icon:before{content:"\f368"}i.icon.accusoft:before{content:"\f369"}i.icon.address.book:before{content:"\f2b9"}i.icon.address.card:before{content:"\f2bb"}i.icon.adjust:before{content:"\f042"}i.icon.adn:before{content:"\f170"}i.icon.adversal:before{content:"\f36a"}i.icon.affiliatetheme:before{content:"\f36b"}i.icon.algolia:before{content:"\f36c"}i.icon.align.center:before{content:"\f037"}i.icon.align.justify:before{content:"\f039"}i.icon.align.left:before{content:"\f036"}i.icon.align.right:before{content:"\f038"}i.icon.amazon:before{content:"\f270"}i.icon.amazon.pay:before{content:"\f42c"}i.icon.ambulance:before{content:"\f0f9"}i.icon.american.sign.language.interpreting:before{content:"\f2a3"}i.icon.amilia:before{content:"\f36d"}i.icon.anchor:before{content:"\f13d"}i.icon.android:before{content:"\f17b"}i.icon.angellist:before{content:"\f209"}i.icon.angle.double.down:before{content:"\f103"}i.icon.angle.double.left:before{content:"\f100"}i.icon.angle.double.right:before{content:"\f101"}i.icon.angle.double.up:before{content:"\f102"}i.icon.angle.down:before{content:"\f107"}i.icon.angle.left:before{content:"\f104"}i.icon.angle.right:before{content:"\f105"}i.icon.angle.up:before{content:"\f106"}i.icon.angrycreative:before{content:"\f36e"}i.icon.angular:before{content:"\f420"}i.icon.app.store:before{content:"\f36f"}i.icon.app.store.ios:before{content:"\f370"}i.icon.apper:before{content:"\f371"}i.icon.apple:before{content:"\f179"}i.icon.apple.pay:before{content:"\f415"}i.icon.archive:before{content:"\f187"}i.icon.arrow.alternate.circle.down:before{content:"\f358"}i.icon.arrow.alternate.circle.left:before{content:"\f359"}i.icon.arrow.alternate.circle.right:before{content:"\f35a"}i.icon.arrow.alternate.circle.up:before{content:"\f35b"}i.icon.arrow.circle.down:before{content:"\f0ab"}i.icon.arrow.circle.left:before{content:"\f0a8"}i.icon.arrow.circle.right:before{content:"\f0a9"}i.icon.arrow.circle.up:before{content:"\f0aa"}i.icon.arrow.down:before{content:"\f063"}i.icon.arrow.left:before{content:"\f060"}i.icon.arrow.right:before{content:"\f061"}i.icon.arrow.up:before{content:"\f062"}i.icon.arrows.alternate:before{content:"\f0b2"}i.icon.arrows.alternate.horizontal:before{content:"\f337"}i.icon.arrows.alternate.vertical:before{content:"\f338"}i.icon.assistive.listening.systems:before{content:"\f2a2"}i.icon.asterisk:before{content:"\f069"}i.icon.asymmetrik:before{content:"\f372"}i.icon.at:before{content:"\f1fa"}i.icon.audible:before{content:"\f373"}i.icon.audio.description:before{content:"\f29e"}i.icon.autoprefixer:before{content:"\f41c"}i.icon.avianex:before{content:"\f374"}i.icon.aviato:before{content:"\f421"}i.icon.aws:before{content:"\f375"}i.icon.backward:before{content:"\f04a"}i.icon.balance.scale:before{content:"\f24e"}i.icon.ban:before{content:"\f05e"}i.icon.band.aid:before{content:"\f462"}i.icon.bandcamp:before{content:"\f2d5"}i.icon.barcode:before{content:"\f02a"}i.icon.bars:before{content:"\f0c9"}i.icon.baseball.ball:before{content:"\f433"}i.icon.basketball.ball:before{content:"\f434"}i.icon.bath:before{content:"\f2cd"}i.icon.battery.empty:before{content:"\f244"}i.icon.battery.full:before{content:"\f240"}i.icon.battery.half:before{content:"\f242"}i.icon.battery.quarter:before{content:"\f243"}i.icon.battery.three.quarters:before{content:"\f241"}i.icon.bed:before{content:"\f236"}i.icon.beer:before{content:"\f0fc"}i.icon.behance:before{content:"\f1b4"}i.icon.behance.square:before{content:"\f1b5"}i.icon.bell:before{content:"\f0f3"}i.icon.bell.slash:before{content:"\f1f6"}i.icon.bicycle:before{content:"\f206"}i.icon.bimobject:before{content:"\f378"}i.icon.binoculars:before{content:"\f1e5"}i.icon.birthday.cake:before{content:"\f1fd"}i.icon.bitbucket:before{content:"\f171"}i.icon.bitcoin:before{content:"\f379"}i.icon.bity:before{content:"\f37a"}i.icon.black.tie:before{content:"\f27e"}i.icon.blackberry:before{content:"\f37b"}i.icon.blind:before{content:"\f29d"}i.icon.blogger:before{content:"\f37c"}i.icon.blogger.b:before{content:"\f37d"}i.icon.bluetooth:before{content:"\f293"}i.icon.bluetooth.b:before{content:"\f294"}i.icon.bold:before{content:"\f032"}i.icon.bolt:before{content:"\f0e7"}i.icon.bomb:before{content:"\f1e2"}i.icon.book:before{content:"\f02d"}i.icon.bookmark:before{content:"\f02e"}i.icon.bowling.ball:before{content:"\f436"}i.icon.box:before{content:"\f466"}i.icon.boxes:before{content:"\f468"}i.icon.braille:before{content:"\f2a1"}i.icon.briefcase:before{content:"\f0b1"}i.icon.btc:before{content:"\f15a"}i.icon.bug:before{content:"\f188"}i.icon.building:before{content:"\f1ad"}i.icon.bullhorn:before{content:"\f0a1"}i.icon.bullseye:before{content:"\f140"}i.icon.buromobelexperte:before{content:"\f37f"}i.icon.bus:before{content:"\f207"}i.icon.buysellads:before{content:"\f20d"}i.icon.calculator:before{content:"\f1ec"}i.icon.calendar:before{content:"\f133"}i.icon.calendar.alternate:before{content:"\f073"}i.icon.calendar.check:before{content:"\f274"}i.icon.calendar.minus:before{content:"\f272"}i.icon.calendar.plus:before{content:"\f271"}i.icon.calendar.times:before{content:"\f273"}i.icon.camera:before{content:"\f030"}i.icon.camera.retro:before{content:"\f083"}i.icon.car:before{content:"\f1b9"}i.icon.caret.down:before{content:"\f0d7"}i.icon.caret.left:before{content:"\f0d9"}i.icon.caret.right:before{content:"\f0da"}i.icon.caret.square.down:before{content:"\f150"}i.icon.caret.square.left:before{content:"\f191"}i.icon.caret.square.right:before{content:"\f152"}i.icon.caret.square.up:before{content:"\f151"}i.icon.caret.up:before{content:"\f0d8"}i.icon.cart.arrow.down:before{content:"\f218"}i.icon.cart.plus:before{content:"\f217"}i.icon.cc.amazon.pay:before{content:"\f42d"}i.icon.cc.amex:before{content:"\f1f3"}i.icon.cc.apple.pay:before{content:"\f416"}i.icon.cc.diners.club:before{content:"\f24c"}i.icon.cc.discover:before{content:"\f1f2"}i.icon.cc.jcb:before{content:"\f24b"}i.icon.cc.mastercard:before{content:"\f1f1"}i.icon.cc.paypal:before{content:"\f1f4"}i.icon.cc.stripe:before{content:"\f1f5"}i.icon.cc.visa:before{content:"\f1f0"}i.icon.centercode:before{content:"\f380"}i.icon.certificate:before{content:"\f0a3"}i.icon.chart.area:before{content:"\f1fe"}i.icon.chart.bar:before{content:"\f080"}i.icon.chart.line:before{content:"\f201"}i.icon.chart.pie:before{content:"\f200"}i.icon.check:before{content:"\f00c"}i.icon.check.circle:before{content:"\f058"}i.icon.check.square:before{content:"\f14a"}i.icon.chess:before{content:"\f439"}i.icon.chess.bishop:before{content:"\f43a"}i.icon.chess.board:before{content:"\f43c"}i.icon.chess.king:before{content:"\f43f"}i.icon.chess.knight:before{content:"\f441"}i.icon.chess.pawn:before{content:"\f443"}i.icon.chess.queen:before{content:"\f445"}i.icon.chess.rook:before{content:"\f447"}i.icon.chevron.circle.down:before{content:"\f13a"}i.icon.chevron.circle.left:before{content:"\f137"}i.icon.chevron.circle.right:before{content:"\f138"}i.icon.chevron.circle.up:before{content:"\f139"}i.icon.chevron.down:before{content:"\f078"}i.icon.chevron.left:before{content:"\f053"}i.icon.chevron.right:before{content:"\f054"}i.icon.chevron.up:before{content:"\f077"}i.icon.child:before{content:"\f1ae"}i.icon.chrome:before{content:"\f268"}i.icon.circle:before{content:"\f111"}i.icon.circle.notch:before{content:"\f1ce"}i.icon.clipboard:before{content:"\f328"}i.icon.clipboard.check:before{content:"\f46c"}i.icon.clipboard.list:before{content:"\f46d"}i.icon.clock:before{content:"\f017"}i.icon.clone:before{content:"\f24d"}i.icon.closed.captioning:before{content:"\f20a"}i.icon.cloud:before{content:"\f0c2"}i.icon.cloudscale:before{content:"\f383"}i.icon.cloudsmith:before{content:"\f384"}i.icon.cloudversify:before{content:"\f385"}i.icon.code:before{content:"\f121"}i.icon.code.branch:before{content:"\f126"}i.icon.codepen:before{content:"\f1cb"}i.icon.codiepie:before{content:"\f284"}i.icon.coffee:before{content:"\f0f4"}i.icon.cog:before{content:"\f013"}i.icon.cogs:before{content:"\f085"}i.icon.columns:before{content:"\f0db"}i.icon.comment:before{content:"\f075"}i.icon.comment.alternate:before{content:"\f27a"}i.icon.comments:before{content:"\f086"}i.icon.compass:before{content:"\f14e"}i.icon.compress:before{content:"\f066"}i.icon.connectdevelop:before{content:"\f20e"}i.icon.contao:before{content:"\f26d"}i.icon.copy:before{content:"\f0c5"}i.icon.copyright:before{content:"\f1f9"}i.icon.cpanel:before{content:"\f388"}i.icon.creative.commons:before{content:"\f25e"}i.icon.credit.card:before{content:"\f09d"}i.icon.crop:before{content:"\f125"}i.icon.crosshairs:before{content:"\f05b"}i.icon.css3:before{content:"\f13c"}i.icon.css3.alternate:before{content:"\f38b"}i.icon.cube:before{content:"\f1b2"}i.icon.cubes:before{content:"\f1b3"}i.icon.cut:before{content:"\f0c4"}i.icon.cuttlefish:before{content:"\f38c"}i.icon.d.and.d:before{content:"\f38d"}i.icon.dashcube:before{content:"\f210"}i.icon.database:before{content:"\f1c0"}i.icon.deaf:before{content:"\f2a4"}i.icon.delicious:before{content:"\f1a5"}i.icon.deploydog:before{content:"\f38e"}i.icon.deskpro:before{content:"\f38f"}i.icon.desktop:before{content:"\f108"}i.icon.deviantart:before{content:"\f1bd"}i.icon.digg:before{content:"\f1a6"}i.icon.digital.ocean:before{content:"\f391"}i.icon.discord:before{content:"\f392"}i.icon.discourse:before{content:"\f393"}i.icon.dna:before{content:"\f471"}i.icon.dochub:before{content:"\f394"}i.icon.docker:before{content:"\f395"}i.icon.dollar.sign:before{content:"\f155"}i.icon.dolly:before{content:"\f472"}i.icon.dolly.flatbed:before{content:"\f474"}i.icon.dot.circle:before{content:"\f192"}i.icon.download:before{content:"\f019"}i.icon.draft2digital:before{content:"\f396"}i.icon.dribbble:before{content:"\f17d"}i.icon.dribbble.square:before{content:"\f397"}i.icon.dropbox:before{content:"\f16b"}i.icon.drupal:before{content:"\f1a9"}i.icon.dyalog:before{content:"\f399"}i.icon.earlybirds:before{content:"\f39a"}i.icon.edge:before{content:"\f282"}i.icon.edit:before{content:"\f044"}i.icon.eject:before{content:"\f052"}i.icon.elementor:before{content:"\f430"}i.icon.ellipsis.horizontal:before{content:"\f141"}i.icon.ellipsis.vertical:before{content:"\f142"}i.icon.ember:before{content:"\f423"}i.icon.empire:before{content:"\f1d1"}i.icon.envelope:before{content:"\f0e0"}i.icon.envelope.open:before{content:"\f2b6"}i.icon.envelope.square:before{content:"\f199"}i.icon.envira:before{content:"\f299"}i.icon.eraser:before{content:"\f12d"}i.icon.erlang:before{content:"\f39d"}i.icon.ethereum:before{content:"\f42e"}i.icon.etsy:before{content:"\f2d7"}i.icon.euro.sign:before{content:"\f153"}i.icon.exchange.alternate:before{content:"\f362"}i.icon.exclamation:before{content:"\f12a"}i.icon.exclamation.circle:before{content:"\f06a"}i.icon.exclamation.triangle:before{content:"\f071"}i.icon.expand:before{content:"\f065"}i.icon.expand.arrows.alternate:before{content:"\f31e"}i.icon.expeditedssl:before{content:"\f23e"}i.icon.external.alternate:before{content:"\f35d"}i.icon.external.square.alternate:before{content:"\f360"}i.icon.eye:before{content:"\f06e"}i.icon.eye.dropper:before{content:"\f1fb"}i.icon.eye.slash:before{content:"\f070"}i.icon.facebook:before{content:"\f09a"}i.icon.facebook.f:before{content:"\f39e"}i.icon.facebook.messenger:before{content:"\f39f"}i.icon.facebook.square:before{content:"\f082"}i.icon.fast.backward:before{content:"\f049"}i.icon.fast.forward:before{content:"\f050"}i.icon.fax:before{content:"\f1ac"}i.icon.female:before{content:"\f182"}i.icon.fighter.jet:before{content:"\f0fb"}i.icon.file:before{content:"\f15b"}i.icon.file.alternate:before{content:"\f15c"}i.icon.file.archive:before{content:"\f1c6"}i.icon.file.audio:before{content:"\f1c7"}i.icon.file.code:before{content:"\f1c9"}i.icon.file.excel:before{content:"\f1c3"}i.icon.file.image:before{content:"\f1c5"}i.icon.file.pdf:before{content:"\f1c1"}i.icon.file.powerpoint:before{content:"\f1c4"}i.icon.file.video:before{content:"\f1c8"}i.icon.file.word:before{content:"\f1c2"}i.icon.film:before{content:"\f008"}i.icon.filter:before{content:"\f0b0"}i.icon.fire:before{content:"\f06d"}i.icon.fire.extinguisher:before{content:"\f134"}i.icon.firefox:before{content:"\f269"}i.icon.first.aid:before{content:"\f479"}i.icon.first.order:before{content:"\f2b0"}i.icon.firstdraft:before{content:"\f3a1"}i.icon.flag:before{content:"\f024"}i.icon.flag.checkered:before{content:"\f11e"}i.icon.flask:before{content:"\f0c3"}i.icon.flickr:before{content:"\f16e"}i.icon.flipboard:before{content:"\f44d"}i.icon.fly:before{content:"\f417"}i.icon.folder:before{content:"\f07b"}i.icon.folder.open:before{content:"\f07c"}i.icon.font:before{content:"\f031"}i.icon.font.awesome:before{content:"\f2b4"}i.icon.font.awesome.alternate:before{content:"\f35c"}i.icon.font.awesome.flag:before{content:"\f425"}i.icon.fonticons:before{content:"\f280"}i.icon.fonticons.fi:before{content:"\f3a2"}i.icon.football.ball:before{content:"\f44e"}i.icon.fort.awesome:before{content:"\f286"}i.icon.fort.awesome.alternate:before{content:"\f3a3"}i.icon.forumbee:before{content:"\f211"}i.icon.forward:before{content:"\f04e"}i.icon.foursquare:before{content:"\f180"}i.icon.free.code.camp:before{content:"\f2c5"}i.icon.freebsd:before{content:"\f3a4"}i.icon.frown:before{content:"\f119"}i.icon.futbol:before{content:"\f1e3"}i.icon.gamepad:before{content:"\f11b"}i.icon.gavel:before{content:"\f0e3"}i.icon.gem:before{content:"\f3a5"}i.icon.genderless:before{content:"\f22d"}i.icon.get.pocket:before{content:"\f265"}i.icon.gg:before{content:"\f260"}i.icon.gg.circle:before{content:"\f261"}i.icon.gift:before{content:"\f06b"}i.icon.git:before{content:"\f1d3"}i.icon.git.square:before{content:"\f1d2"}i.icon.github:before{content:"\f09b"}i.icon.github.alternate:before{content:"\f113"}i.icon.github.square:before{content:"\f092"}i.icon.gitkraken:before{content:"\f3a6"}i.icon.gitlab:before{content:"\f296"}i.icon.gitter:before{content:"\f426"}i.icon.glass.martini:before{content:"\f000"}i.icon.glide:before{content:"\f2a5"}i.icon.glide.g:before{content:"\f2a6"}i.icon.globe:before{content:"\f0ac"}i.icon.gofore:before{content:"\f3a7"}i.icon.golf.ball:before{content:"\f450"}i.icon.goodreads:before{content:"\f3a8"}i.icon.goodreads.g:before{content:"\f3a9"}i.icon.google:before{content:"\f1a0"}i.icon.google.drive:before{content:"\f3aa"}i.icon.google.play:before{content:"\f3ab"}i.icon.google.plus:before{content:"\f2b3"}i.icon.google.plus.g:before{content:"\f0d5"}i.icon.google.plus.square:before{content:"\f0d4"}i.icon.google.wallet:before{content:"\f1ee"}i.icon.graduation.cap:before{content:"\f19d"}i.icon.gratipay:before{content:"\f184"}i.icon.grav:before{content:"\f2d6"}i.icon.gripfire:before{content:"\f3ac"}i.icon.grunt:before{content:"\f3ad"}i.icon.gulp:before{content:"\f3ae"}i.icon.h.square:before{content:"\f0fd"}i.icon.hacker.news:before{content:"\f1d4"}i.icon.hacker.news.square:before{content:"\f3af"}i.icon.hand.lizard:before{content:"\f258"}i.icon.hand.paper:before{content:"\f256"}i.icon.hand.peace:before{content:"\f25b"}i.icon.hand.point.down:before{content:"\f0a7"}i.icon.hand.point.left:before{content:"\f0a5"}i.icon.hand.point.right:before{content:"\f0a4"}i.icon.hand.point.up:before{content:"\f0a6"}i.icon.hand.pointer:before{content:"\f25a"}i.icon.hand.rock:before{content:"\f255"}i.icon.hand.scissors:before{content:"\f257"}i.icon.hand.spock:before{content:"\f259"}i.icon.handshake:before{content:"\f2b5"}i.icon.hashtag:before{content:"\f292"}i.icon.hdd:before{content:"\f0a0"}i.icon.heading:before{content:"\f1dc"}i.icon.headphones:before{content:"\f025"}i.icon.heart:before{content:"\f004"}i.icon.heartbeat:before{content:"\f21e"}i.icon.hips:before{content:"\f452"}i.icon.hire.a.helper:before{content:"\f3b0"}i.icon.history:before{content:"\f1da"}i.icon.hockey.puck:before{content:"\f453"}i.icon.home:before{content:"\f015"}i.icon.hooli:before{content:"\f427"}i.icon.hospital:before{content:"\f0f8"}i.icon.hospital.symbol:before{content:"\f47e"}i.icon.hotjar:before{content:"\f3b1"}i.icon.hourglass:before{content:"\f254"}i.icon.hourglass.end:before{content:"\f253"}i.icon.hourglass.half:before{content:"\f252"}i.icon.hourglass.start:before{content:"\f251"}i.icon.houzz:before{content:"\f27c"}i.icon.html5:before{content:"\f13b"}i.icon.hubspot:before{content:"\f3b2"}i.icon.i.cursor:before{content:"\f246"}i.icon.id.badge:before{content:"\f2c1"}i.icon.id.card:before{content:"\f2c2"}i.icon.image:before{content:"\f03e"}i.icon.images:before{content:"\f302"}i.icon.imdb:before{content:"\f2d8"}i.icon.inbox:before{content:"\f01c"}i.icon.indent:before{content:"\f03c"}i.icon.industry:before{content:"\f275"}i.icon.info:before{content:"\f129"}i.icon.info.circle:before{content:"\f05a"}i.icon.instagram:before{content:"\f16d"}i.icon.internet.explorer:before{content:"\f26b"}i.icon.ioxhost:before{content:"\f208"}i.icon.italic:before{content:"\f033"}i.icon.itunes:before{content:"\f3b4"}i.icon.itunes.note:before{content:"\f3b5"}i.icon.jenkins:before{content:"\f3b6"}i.icon.joget:before{content:"\f3b7"}i.icon.joomla:before{content:"\f1aa"}i.icon.js:before{content:"\f3b8"}i.icon.js.square:before{content:"\f3b9"}i.icon.jsfiddle:before{content:"\f1cc"}i.icon.key:before{content:"\f084"}i.icon.keyboard:before{content:"\f11c"}i.icon.keycdn:before{content:"\f3ba"}i.icon.kickstarter:before{content:"\f3bb"}i.icon.kickstarter.k:before{content:"\f3bc"}i.icon.korvue:before{content:"\f42f"}i.icon.language:before{content:"\f1ab"}i.icon.laptop:before{content:"\f109"}i.icon.laravel:before{content:"\f3bd"}i.icon.lastfm:before{content:"\f202"}i.icon.lastfm.square:before{content:"\f203"}i.icon.leaf:before{content:"\f06c"}i.icon.leanpub:before{content:"\f212"}i.icon.lemon:before{content:"\f094"}i.icon.less:before{content:"\f41d"}i.icon.level.down.alternate:before{content:"\f3be"}i.icon.level.up.alternate:before{content:"\f3bf"}i.icon.life.ring:before{content:"\f1cd"}i.icon.lightbulb:before{content:"\f0eb"}i.icon.linechat:before{content:"\f3c0"}i.icon.linkify:before{content:"\f0c1"}i.icon.linkedin:before{content:"\f08c"}i.icon.linkedin.alt:before{content:"\f0e1"}i.icon.linode:before{content:"\f2b8"}i.icon.linux:before{content:"\f17c"}i.icon.lira.sign:before{content:"\f195"}i.icon.list:before{content:"\f03a"}i.icon.list.alternate:before{content:"\f022"}i.icon.list.ol:before{content:"\f0cb"}i.icon.list.ul:before{content:"\f0ca"}i.icon.location.arrow:before{content:"\f124"}i.icon.lock:before{content:"\f023"}i.icon.lock.open:before{content:"\f3c1"}i.icon.long.arrow.alternate.down:before{content:"\f309"}i.icon.long.arrow.alternate.left:before{content:"\f30a"}i.icon.long.arrow.alternate.right:before{content:"\f30b"}i.icon.long.arrow.alternate.up:before{content:"\f30c"}i.icon.low.vision:before{content:"\f2a8"}i.icon.lyft:before{content:"\f3c3"}i.icon.magento:before{content:"\f3c4"}i.icon.magic:before{content:"\f0d0"}i.icon.magnet:before{content:"\f076"}i.icon.male:before{content:"\f183"}i.icon.map:before{content:"\f279"}i.icon.map.marker:before{content:"\f041"}i.icon.map.marker.alternate:before{content:"\f3c5"}i.icon.map.pin:before{content:"\f276"}i.icon.map.signs:before{content:"\f277"}i.icon.mars:before{content:"\f222"}i.icon.mars.double:before{content:"\f227"}i.icon.mars.stroke:before{content:"\f229"}i.icon.mars.stroke.horizontal:before{content:"\f22b"}i.icon.mars.stroke.vertical:before{content:"\f22a"}i.icon.maxcdn:before{content:"\f136"}i.icon.medapps:before{content:"\f3c6"}i.icon.medium:before{content:"\f23a"}i.icon.medium.m:before{content:"\f3c7"}i.icon.medkit:before{content:"\f0fa"}i.icon.medrt:before{content:"\f3c8"}i.icon.meetup:before{content:"\f2e0"}i.icon.meh:before{content:"\f11a"}i.icon.mercury:before{content:"\f223"}i.icon.microchip:before{content:"\f2db"}i.icon.microphone:before{content:"\f130"}i.icon.microphone.slash:before{content:"\f131"}i.icon.microsoft:before{content:"\f3ca"}i.icon.minus:before{content:"\f068"}i.icon.minus.circle:before{content:"\f056"}i.icon.minus.square:before{content:"\f146"}i.icon.mix:before{content:"\f3cb"}i.icon.mixcloud:before{content:"\f289"}i.icon.mizuni:before{content:"\f3cc"}i.icon.mobile:before{content:"\f10b"}i.icon.mobile.alternate:before{content:"\f3cd"}i.icon.modx:before{content:"\f285"}i.icon.monero:before{content:"\f3d0"}i.icon.money.bill.alternate:before{content:"\f3d1"}i.icon.moon:before{content:"\f186"}i.icon.motorcycle:before{content:"\f21c"}i.icon.mouse.pointer:before{content:"\f245"}i.icon.music:before{content:"\f001"}i.icon.napster:before{content:"\f3d2"}i.icon.neuter:before{content:"\f22c"}i.icon.newspaper:before{content:"\f1ea"}i.icon.nintendo.switch:before{content:"\f418"}i.icon.node:before{content:"\f419"}i.icon.node.js:before{content:"\f3d3"}i.icon.npm:before{content:"\f3d4"}i.icon.ns8:before{content:"\f3d5"}i.icon.nutritionix:before{content:"\f3d6"}i.icon.object.group:before{content:"\f247"}i.icon.object.ungroup:before{content:"\f248"}i.icon.odnoklassniki:before{content:"\f263"}i.icon.odnoklassniki.square:before{content:"\f264"}i.icon.opencart:before{content:"\f23d"}i.icon.openid:before{content:"\f19b"}i.icon.opera:before{content:"\f26a"}i.icon.optin.monster:before{content:"\f23c"}i.icon.osi:before{content:"\f41a"}i.icon.outdent:before{content:"\f03b"}i.icon.page4:before{content:"\f3d7"}i.icon.pagelines:before{content:"\f18c"}i.icon.paint.brush:before{content:"\f1fc"}i.icon.palfed:before{content:"\f3d8"}i.icon.pallet:before{content:"\f482"}i.icon.paper.plane:before{content:"\f1d8"}i.icon.paperclip:before{content:"\f0c6"}i.icon.paragraph:before{content:"\f1dd"}i.icon.paste:before{content:"\f0ea"}i.icon.patreon:before{content:"\f3d9"}i.icon.pause:before{content:"\f04c"}i.icon.pause.circle:before{content:"\f28b"}i.icon.paw:before{content:"\f1b0"}i.icon.paypal:before{content:"\f1ed"}i.icon.pen.square:before{content:"\f14b"}i.icon.pencil.alternate:before{content:"\f303"}i.icon.percent:before{content:"\f295"}i.icon.periscope:before{content:"\f3da"}i.icon.phabricator:before{content:"\f3db"}i.icon.phoenix.framework:before{content:"\f3dc"}i.icon.phone:before{content:"\f095"}i.icon.phone.square:before{content:"\f098"}i.icon.phone.volume:before{content:"\f2a0"}i.icon.php:before{content:"\f457"}i.icon.pied.piper:before{content:"\f2ae"}i.icon.pied.piper.alternate:before{content:"\f1a8"}i.icon.pied.piper.pp:before{content:"\f1a7"}i.icon.pills:before{content:"\f484"}i.icon.pinterest:before{content:"\f0d2"}i.icon.pinterest.p:before{content:"\f231"}i.icon.pinterest.square:before{content:"\f0d3"}i.icon.plane:before{content:"\f072"}i.icon.play:before{content:"\f04b"}i.icon.play.circle:before{content:"\f144"}i.icon.playstation:before{content:"\f3df"}i.icon.plug:before{content:"\f1e6"}i.icon.plus:before{content:"\f067"}i.icon.plus.circle:before{content:"\f055"}i.icon.plus.square:before{content:"\f0fe"}i.icon.podcast:before{content:"\f2ce"}i.icon.pound.sign:before{content:"\f154"}i.icon.power.off:before{content:"\f011"}i.icon.print:before{content:"\f02f"}i.icon.product.hunt:before{content:"\f288"}i.icon.pushed:before{content:"\f3e1"}i.icon.puzzle.piece:before{content:"\f12e"}i.icon.python:before{content:"\f3e2"}i.icon.qq:before{content:"\f1d6"}i.icon.qrcode:before{content:"\f029"}i.icon.question:before{content:"\f128"}i.icon.question.circle:before{content:"\f059"}i.icon.quidditch:before{content:"\f458"}i.icon.quinscape:before{content:"\f459"}i.icon.quora:before{content:"\f2c4"}i.icon.quote.left:before{content:"\f10d"}i.icon.quote.right:before{content:"\f10e"}i.icon.random:before{content:"\f074"}i.icon.ravelry:before{content:"\f2d9"}i.icon.react:before{content:"\f41b"}i.icon.rebel:before{content:"\f1d0"}i.icon.recycle:before{content:"\f1b8"}i.icon.redriver:before{content:"\f3e3"}i.icon.reddit:before{content:"\f1a1"}i.icon.reddit.alien:before{content:"\f281"}i.icon.reddit.square:before{content:"\f1a2"}i.icon.redo:before{content:"\f01e"}i.icon.redo.alternate:before{content:"\f2f9"}i.icon.registered:before{content:"\f25d"}i.icon.rendact:before{content:"\f3e4"}i.icon.renren:before{content:"\f18b"}i.icon.reply:before{content:"\f3e5"}i.icon.reply.all:before{content:"\f122"}i.icon.replyd:before{content:"\f3e6"}i.icon.resolving:before{content:"\f3e7"}i.icon.retweet:before{content:"\f079"}i.icon.road:before{content:"\f018"}i.icon.rocket:before{content:"\f135"}i.icon.rocketchat:before{content:"\f3e8"}i.icon.rockrms:before{content:"\f3e9"}i.icon.rss:before{content:"\f09e"}i.icon.rss.square:before{content:"\f143"}i.icon.ruble.sign:before{content:"\f158"}i.icon.rupee.sign:before{content:"\f156"}i.icon.safari:before{content:"\f267"}i.icon.sass:before{content:"\f41e"}i.icon.save:before{content:"\f0c7"}i.icon.schlix:before{content:"\f3ea"}i.icon.scribd:before{content:"\f28a"}i.icon.search:before{content:"\f002"}i.icon.search.minus:before{content:"\f010"}i.icon.search.plus:before{content:"\f00e"}i.icon.searchengin:before{content:"\f3eb"}i.icon.sellcast:before{content:"\f2da"}i.icon.sellsy:before{content:"\f213"}i.icon.server:before{content:"\f233"}i.icon.servicestack:before{content:"\f3ec"}i.icon.share:before{content:"\f064"}i.icon.share.alternate:before{content:"\f1e0"}i.icon.share.alternate.square:before{content:"\f1e1"}i.icon.share.square:before{content:"\f14d"}i.icon.shekel.sign:before{content:"\f20b"}i.icon.shield.alternate:before{content:"\f3ed"}i.icon.ship:before{content:"\f21a"}i.icon.shipping.fast:before{content:"\f48b"}i.icon.shirtsinbulk:before{content:"\f214"}i.icon.shopping.bag:before{content:"\f290"}i.icon.shopping.basket:before{content:"\f291"}i.icon.shopping.cart:before{content:"\f07a"}i.icon.shower:before{content:"\f2cc"}i.icon.sign.language:before{content:"\f2a7"}i.icon.signal:before{content:"\f012"}i.icon.simplybuilt:before{content:"\f215"}i.icon.sistrix:before{content:"\f3ee"}i.icon.sitemap:before{content:"\f0e8"}i.icon.skyatlas:before{content:"\f216"}i.icon.skype:before{content:"\f17e"}i.icon.slack:before{content:"\f198"}i.icon.slack.hash:before{content:"\f3ef"}i.icon.sliders.horizontal:before{content:"\f1de"}i.icon.slideshare:before{content:"\f1e7"}i.icon.smile:before{content:"\f118"}i.icon.snapchat:before{content:"\f2ab"}i.icon.snapchat.ghost:before{content:"\f2ac"}i.icon.snapchat.square:before{content:"\f2ad"}i.icon.snowflake:before{content:"\f2dc"}i.icon.sort:before{content:"\f0dc"}i.icon.sort.alphabet.down:before{content:"\f15d"}i.icon.sort.alphabet.up:before{content:"\f15e"}i.icon.sort.amount.down:before{content:"\f160"}i.icon.sort.amount.up:before{content:"\f161"}i.icon.sort.down:before{content:"\f0dd"}i.icon.sort.numeric.down:before{content:"\f162"}i.icon.sort.numeric.up:before{content:"\f163"}i.icon.sort.up:before{content:"\f0de"}i.icon.soundcloud:before{content:"\f1be"}i.icon.space.shuttle:before{content:"\f197"}i.icon.speakap:before{content:"\f3f3"}i.icon.spinner:before{content:"\f110"}i.icon.spotify:before{content:"\f1bc"}i.icon.square:before{content:"\f0c8"}i.icon.square.full:before{content:"\f45c"}i.icon.stack.exchange:before{content:"\f18d"}i.icon.stack.overflow:before{content:"\f16c"}i.icon.star:before{content:"\f005"}i.icon.star.half:before{content:"\f089"}i.icon.staylinked:before{content:"\f3f5"}i.icon.steam:before{content:"\f1b6"}i.icon.steam.square:before{content:"\f1b7"}i.icon.steam.symbol:before{content:"\f3f6"}i.icon.step.backward:before{content:"\f048"}i.icon.step.forward:before{content:"\f051"}i.icon.stethoscope:before{content:"\f0f1"}i.icon.sticker.mule:before{content:"\f3f7"}i.icon.sticky.note:before{content:"\f249"}i.icon.stop:before{content:"\f04d"}i.icon.stop.circle:before{content:"\f28d"}i.icon.stopwatch:before{content:"\f2f2"}i.icon.strava:before{content:"\f428"}i.icon.street.view:before{content:"\f21d"}i.icon.strikethrough:before{content:"\f0cc"}i.icon.stripe:before{content:"\f429"}i.icon.stripe.s:before{content:"\f42a"}i.icon.studiovinari:before{content:"\f3f8"}i.icon.stumbleupon:before{content:"\f1a4"}i.icon.stumbleupon.circle:before{content:"\f1a3"}i.icon.subscript:before{content:"\f12c"}i.icon.subway:before{content:"\f239"}i.icon.suitcase:before{content:"\f0f2"}i.icon.sun:before{content:"\f185"}i.icon.superpowers:before{content:"\f2dd"}i.icon.superscript:before{content:"\f12b"}i.icon.supple:before{content:"\f3f9"}i.icon.sync:before{content:"\f021"}i.icon.sync.alternate:before{content:"\f2f1"}i.icon.syringe:before{content:"\f48e"}i.icon.table:before{content:"\f0ce"}i.icon.table.tennis:before{content:"\f45d"}i.icon.tablet:before{content:"\f10a"}i.icon.tablet.alternate:before{content:"\f3fa"}i.icon.tachometer.alternate:before{content:"\f3fd"}i.icon.tag:before{content:"\f02b"}i.icon.tags:before{content:"\f02c"}i.icon.tasks:before{content:"\f0ae"}i.icon.taxi:before{content:"\f1ba"}i.icon.telegram:before{content:"\f2c6"}i.icon.telegram.plane:before{content:"\f3fe"}i.icon.tencent.weibo:before{content:"\f1d5"}i.icon.terminal:before{content:"\f120"}i.icon.text.height:before{content:"\f034"}i.icon.text.width:before{content:"\f035"}i.icon.th:before{content:"\f00a"}i.icon.th.large:before{content:"\f009"}i.icon.th.list:before{content:"\f00b"}i.icon.themeisle:before{content:"\f2b2"}i.icon.thermometer:before{content:"\f491"}i.icon.thermometer.empty:before{content:"\f2cb"}i.icon.thermometer.full:before{content:"\f2c7"}i.icon.thermometer.half:before{content:"\f2c9"}i.icon.thermometer.quarter:before{content:"\f2ca"}i.icon.thermometer.three.quarters:before{content:"\f2c8"}i.icon.thumbs.down:before{content:"\f165"}i.icon.thumbs.up:before{content:"\f164"}i.icon.thumbtack:before{content:"\f08d"}i.icon.ticket.alternate:before{content:"\f3ff"}i.icon.times:before{content:"\f00d"}i.icon.times.circle:before{content:"\f057"}i.icon.tint:before{content:"\f043"}i.icon.toggle.off:before{content:"\f204"}i.icon.toggle.on:before{content:"\f205"}i.icon.trademark:before{content:"\f25c"}i.icon.train:before{content:"\f238"}i.icon.transgender:before{content:"\f224"}i.icon.transgender.alternate:before{content:"\f225"}i.icon.trash:before{content:"\f1f8"}i.icon.trash.alternate:before{content:"\f2ed"}i.icon.tree:before{content:"\f1bb"}i.icon.trello:before{content:"\f181"}i.icon.tripadvisor:before{content:"\f262"}i.icon.trophy:before{content:"\f091"}i.icon.truck:before{content:"\f0d1"}i.icon.tty:before{content:"\f1e4"}i.icon.tumblr:before{content:"\f173"}i.icon.tumblr.square:before{content:"\f174"}i.icon.tv:before{content:"\f26c"}i.icon.twitch:before{content:"\f1e8"}i.icon.twitter:before{content:"\f099"}i.icon.twitter.square:before{content:"\f081"}i.icon.typo3:before{content:"\f42b"}i.icon.uber:before{content:"\f402"}i.icon.uikit:before{content:"\f403"}i.icon.umbrella:before{content:"\f0e9"}i.icon.underline:before{content:"\f0cd"}i.icon.undo:before{content:"\f0e2"}i.icon.undo.alternate:before{content:"\f2ea"}i.icon.uniregistry:before{content:"\f404"}i.icon.universal.access:before{content:"\f29a"}i.icon.university:before{content:"\f19c"}i.icon.unlink:before{content:"\f127"}i.icon.unlock:before{content:"\f09c"}i.icon.unlock.alternate:before{content:"\f13e"}i.icon.untappd:before{content:"\f405"}i.icon.upload:before{content:"\f093"}i.icon.usb:before{content:"\f287"}i.icon.user:before{content:"\f007"}i.icon.user.circle:before{content:"\f2bd"}i.icon.user.md:before{content:"\f0f0"}i.icon.user.plus:before{content:"\f234"}i.icon.user.secret:before{content:"\f21b"}i.icon.user.times:before{content:"\f235"}i.icon.users:before{content:"\f0c0"}i.icon.ussunnah:before{content:"\f407"}i.icon.utensil.spoon:before{content:"\f2e5"}i.icon.utensils:before{content:"\f2e7"}i.icon.vaadin:before{content:"\f408"}i.icon.venus:before{content:"\f221"}i.icon.venus.double:before{content:"\f226"}i.icon.venus.mars:before{content:"\f228"}i.icon.viacoin:before{content:"\f237"}i.icon.viadeo:before{content:"\f2a9"}i.icon.viadeo.square:before{content:"\f2aa"}i.icon.viber:before{content:"\f409"}i.icon.video:before{content:"\f03d"}i.icon.vimeo:before{content:"\f40a"}i.icon.vimeo.square:before{content:"\f194"}i.icon.vimeo.v:before{content:"\f27d"}i.icon.vine:before{content:"\f1ca"}i.icon.vk:before{content:"\f189"}i.icon.vnv:before{content:"\f40b"}i.icon.volleyball.ball:before{content:"\f45f"}i.icon.volume.down:before{content:"\f027"}i.icon.volume.off:before{content:"\f026"}i.icon.volume.up:before{content:"\f028"}i.icon.vuejs:before{content:"\f41f"}i.icon.warehouse:before{content:"\f494"}i.icon.weibo:before{content:"\f18a"}i.icon.weight:before{content:"\f496"}i.icon.weixin:before{content:"\f1d7"}i.icon.whatsapp:before{content:"\f232"}i.icon.whatsapp.square:before{content:"\f40c"}i.icon.wheelchair:before{content:"\f193"}i.icon.whmcs:before{content:"\f40d"}i.icon.wifi:before{content:"\f1eb"}i.icon.wikipedia.w:before{content:"\f266"}i.icon.window.close:before{content:"\f410"}i.icon.window.maximize:before{content:"\f2d0"}i.icon.window.minimize:before{content:"\f2d1"}i.icon.window.restore:before{content:"\f2d2"}i.icon.windows:before{content:"\f17a"}i.icon.won.sign:before{content:"\f159"}i.icon.wordpress:before{content:"\f19a"}i.icon.wordpress.simple:before{content:"\f411"}i.icon.wpbeginner:before{content:"\f297"}i.icon.wpexplorer:before{content:"\f2de"}i.icon.wpforms:before{content:"\f298"}i.icon.wrench:before{content:"\f0ad"}i.icon.xbox:before{content:"\f412"}i.icon.xing:before{content:"\f168"}i.icon.xing.square:before{content:"\f169"}i.icon.y.combinator:before{content:"\f23b"}i.icon.yahoo:before{content:"\f19e"}i.icon.yandex:before{content:"\f413"}i.icon.yandex.international:before{content:"\f414"}i.icon.yelp:before{content:"\f1e9"}i.icon.yen.sign:before{content:"\f157"}i.icon.yoast:before{content:"\f2b1"}i.icon.youtube:before{content:"\f167"}i.icon.youtube.square:before{content:"\f431"}i.icon.chess.rock:before{content:"\f447"}i.icon.ordered.list:before{content:"\f0cb"}i.icon.unordered.list:before{content:"\f0ca"}i.icon.user.doctor:before{content:"\f0f0"}i.icon.shield:before{content:"\f3ed"}i.icon.puzzle:before{content:"\f12e"}i.icon.credit.card.amazon.pay:before{content:"\f42d"}i.icon.credit.card.american.express:before{content:"\f1f3"}i.icon.credit.card.diners.club:before{content:"\f24c"}i.icon.credit.card.discover:before{content:"\f1f2"}i.icon.credit.card.jcb:before{content:"\f24b"}i.icon.credit.card.mastercard:before{content:"\f1f1"}i.icon.credit.card.paypal:before{content:"\f1f4"}i.icon.credit.card.stripe:before{content:"\f1f5"}i.icon.credit.card.visa:before{content:"\f1f0"}i.icon.add.circle:before{content:"\f055"}i.icon.add.square:before{content:"\f0fe"}i.icon.add.to.calendar:before{content:"\f271"}i.icon.add.to.cart:before{content:"\f217"}i.icon.add.user:before{content:"\f234"}i.icon.add:before{content:"\f067"}i.icon.alarm.mute:before{content:"\f1f6"}i.icon.alarm:before{content:"\f0f3"}i.icon.ald:before{content:"\f2a2"}i.icon.als:before{content:"\f2a2"}i.icon.american.express.card:before{content:"\f1f3"}i.icon.american.express:before{content:"\f1f3"}i.icon.amex:before{content:"\f1f3"}i.icon.announcement:before{content:"\f0a1"}i.icon.area.chart:before{content:"\f1fe"}i.icon.area.graph:before{content:"\f1fe"}i.icon.arrow.down.cart:before{content:"\f218"}i.icon.asexual:before{content:"\f22d"}i.icon.asl.interpreting:before{content:"\f2a3"}i.icon.asl:before{content:"\f2a3"}i.icon.assistive.listening.devices:before{content:"\f2a2"}i.icon.attach:before{content:"\f0c6"}i.icon.attention:before{content:"\f06a"}i.icon.balance:before{content:"\f24e"}i.icon.bar:before{content:"\f0fc"}i.icon.bathtub:before{content:"\f2cd"}i.icon.battery.four:before{content:"\f240"}i.icon.battery.high:before{content:"\f241"}i.icon.battery.low:before{content:"\f243"}i.icon.battery.medium:before{content:"\f242"}i.icon.battery.one:before{content:"\f243"}i.icon.battery.three:before{content:"\f241"}i.icon.battery.two:before{content:"\f242"}i.icon.battery.zero:before{content:"\f244"}i.icon.birthday:before{content:"\f1fd"}i.icon.block.layout:before{content:"\f009"}i.icon.bluetooth.alternative:before{content:"\f294"}i.icon.broken.chain:before{content:"\f127"}i.icon.browser:before{content:"\f022"}i.icon.call.square:before{content:"\f098"}i.icon.call:before{content:"\f095"}i.icon.cancel:before{content:"\f00d"}i.icon.cart:before{content:"\f07a"}i.icon.cc:before{content:"\f20a"}i.icon.chain:before{content:"\f0c1"}i.icon.chat:before{content:"\f075"}i.icon.checked.calendar:before{content:"\f274"}i.icon.checkmark:before{content:"\f00c"}i.icon.circle.notched:before{content:"\f1ce"}i.icon.close:before{content:"\f00d"}i.icon.cny:before{content:"\f157"}i.icon.cocktail:before{content:"\f000"}i.icon.commenting:before{content:"\f27a"}i.icon.computer:before{content:"\f108"}i.icon.configure:before{content:"\f0ad"}i.icon.content:before{content:"\f0c9"}i.icon.deafness:before{content:"\f2a4"}i.icon.delete.calendar:before{content:"\f273"}i.icon.delete:before{content:"\f00d"}i.icon.detective:before{content:"\f21b"}i.icon.diners.club.card:before{content:"\f24c"}i.icon.diners.club:before{content:"\f24c"}i.icon.discover.card:before{content:"\f1f2"}i.icon.discover:before{content:"\f1f2"}i.icon.discussions:before{content:"\f086"}i.icon.doctor:before{content:"\f0f0"}i.icon.dollar:before{content:"\f155"}i.icon.dont:before{content:"\f05e"}i.icon.dribble:before{content:"\f17d"}i.icon.drivers.license:before{content:"\f2c2"}i.icon.dropdown:before{content:"\f0d7"}i.icon.eercast:before{content:"\f2da"}i.icon.emergency:before{content:"\f0f9"}i.icon.envira.gallery:before{content:"\f299"}i.icon.erase:before{content:"\f12d"}i.icon.eur:before{content:"\f153"}i.icon.euro:before{content:"\f153"}i.icon.eyedropper:before{content:"\f1fb"}i.icon.fa:before{content:"\f2b4"}i.icon.factory:before{content:"\f275"}i.icon.favorite:before{content:"\f005"}i.icon.feed:before{content:"\f09e"}i.icon.female.homosexual:before{content:"\f226"}i.icon.file.text:before{content:"\f15c"}i.icon.find:before{content:"\f1e5"}i.icon.first.aid:before{content:"\f0fa"}i.icon.five.hundred.pixels:before{content:"\f26e"}i.icon.fork:before{content:"\f126"}i.icon.game:before{content:"\f11b"}i.icon.gay:before{content:"\f227"}i.icon.gbp:before{content:"\f154"}i.icon.gittip:before{content:"\f184"}i.icon.google.plus.circle:before{content:"\f2b3"}i.icon.google.plus.official:before{content:"\f2b3"}i.icon.grab:before{content:"\f255"}i.icon.graduation:before{content:"\f19d"}i.icon.grid.layout:before{content:"\f00a"}i.icon.group:before{content:"\f0c0"}i.icon.h:before{content:"\f0fd"}i.icon.hand.victory:before{content:"\f25b"}i.icon.handicap:before{content:"\f193"}i.icon.hard.of.hearing:before{content:"\f2a4"}i.icon.header:before{content:"\f1dc"}i.icon.help.circle:before{content:"\f059"}i.icon.help:before{content:"\f128"}i.icon.heterosexual:before{content:"\f228"}i.icon.hide:before{content:"\f070"}i.icon.hotel:before{content:"\f236"}i.icon.hourglass.four:before{content:"\f254"}i.icon.hourglass.full:before{content:"\f254"}i.icon.hourglass.one:before{content:"\f251"}i.icon.hourglass.three:before{content:"\f253"}i.icon.hourglass.two:before{content:"\f252"}i.icon.idea:before{content:"\f0eb"}i.icon.ils:before{content:"\f20b"}i.icon.in-cart:before{content:"\f218"}i.icon.inr:before{content:"\f156"}i.icon.intergender:before{content:"\f224"}i.icon.intersex:before{content:"\f224"}i.icon.japan.credit.bureau.card:before{content:"\f24b"}i.icon.japan.credit.bureau:before{content:"\f24b"}i.icon.jcb:before{content:"\f24b"}i.icon.jpy:before{content:"\f157"}i.icon.krw:before{content:"\f159"}i.icon.lab:before{content:"\f0c3"}i.icon.law:before{content:"\f24e"}i.icon.legal:before{content:"\f0e3"}i.icon.lesbian:before{content:"\f226"}i.icon.lightning:before{content:"\f0e7"}i.icon.like:before{content:"\f004"}i.icon.line.graph:before{content:"\f201"}i.icon.linkedin.square:before{content:"\f08c"}i.icon.linkify:before{content:"\f0c1"}i.icon.lira:before{content:"\f195"}i.icon.list.layout:before{content:"\f00b"}i.icon.magnify:before{content:"\f00e"}i.icon.mail.forward:before{content:"\f064"}i.icon.mail.square:before{content:"\f199"}i.icon.mail:before{content:"\f0e0"}i.icon.male.homosexual:before{content:"\f227"}i.icon.man:before{content:"\f222"}i.icon.marker:before{content:"\f041"}i.icon.mars.alternate:before{content:"\f229"}i.icon.mars.horizontal:before{content:"\f22b"}i.icon.mars.vertical:before{content:"\f22a"}i.icon.mastercard.card:before{content:"\f1f1"}i.icon.mastercard:before{content:"\f1f1"}i.icon.microsoft.edge:before{content:"\f282"}i.icon.military:before{content:"\f0fb"}i.icon.ms.edge:before{content:"\f282"}i.icon.mute:before{content:"\f131"}i.icon.new.pied.piper:before{content:"\f2ae"}i.icon.non.binary.transgender:before{content:"\f223"}i.icon.numbered.list:before{content:"\f0cb"}i.icon.optinmonster:before{content:"\f23c"}i.icon.options:before{content:"\f1de"}i.icon.other.gender.horizontal:before{content:"\f22b"}i.icon.other.gender.vertical:before{content:"\f22a"}i.icon.other.gender:before{content:"\f229"}i.icon.payment:before{content:"\f09d"}i.icon.paypal.card:before{content:"\f1f4"}i.icon.pencil.square:before{content:"\f14b"}i.icon.photo:before{content:"\f030"}i.icon.picture:before{content:"\f03e"}i.icon.pie.chart:before{content:"\f200"}i.icon.pie.graph:before{content:"\f200"}i.icon.pied.piper.hat:before{content:"\f2ae"}i.icon.pin:before{content:"\f08d"}i.icon.plus.cart:before{content:"\f217"}i.icon.pocket:before{content:"\f265"}i.icon.point:before{content:"\f041"}i.icon.pointing.down:before{content:"\f0a7"}i.icon.pointing.left:before{content:"\f0a5"}i.icon.pointing.right:before{content:"\f0a4"}i.icon.pointing.up:before{content:"\f0a6"}i.icon.pound:before{content:"\f154"}i.icon.power.cord:before{content:"\f1e6"}i.icon.power:before{content:"\f011"}i.icon.privacy:before{content:"\f084"}i.icon.r.circle:before{content:"\f25d"}i.icon.rain:before{content:"\f0e9"}i.icon.record:before{content:"\f03d"}i.icon.refresh:before{content:"\f021"}i.icon.remove.circle:before{content:"\f057"}i.icon.remove.from.calendar:before{content:"\f272"}i.icon.remove.user:before{content:"\f235"}i.icon.remove:before{content:"\f00d"}i.icon.repeat:before{content:"\f01e"}i.icon.rmb:before{content:"\f157"}i.icon.rouble:before{content:"\f158"}i.icon.rub:before{content:"\f158"}i.icon.ruble:before{content:"\f158"}i.icon.rupee:before{content:"\f156"}i.icon.s15:before{content:"\f2cd"}i.icon.selected.radio:before{content:"\f192"}i.icon.send:before{content:"\f1d8"}i.icon.setting:before{content:"\f013"}i.icon.settings:before{content:"\f085"}i.icon.shekel:before{content:"\f20b"}i.icon.sheqel:before{content:"\f20b"}i.icon.shipping:before{content:"\f0d1"}i.icon.shop:before{content:"\f07a"}i.icon.shuffle:before{content:"\f074"}i.icon.shutdown:before{content:"\f011"}i.icon.sidebar:before{content:"\f0c9"}i.icon.signing:before{content:"\f2a7"}i.icon.signup:before{content:"\f044"}i.icon.sliders:before{content:"\f1de"}i.icon.soccer:before{content:"\f1e3"}i.icon.sort.alphabet.ascending:before{content:"\f15d"}i.icon.sort.alphabet.descending:before{content:"\f15e"}i.icon.sort.ascending:before{content:"\f0de"}i.icon.sort.content.ascending:before{content:"\f160"}i.icon.sort.content.descending:before{content:"\f161"}i.icon.sort.descending:before{content:"\f0dd"}i.icon.sort.numeric.ascending:before{content:"\f162"}i.icon.sort.numeric.descending:before{content:"\f163"}i.icon.sound:before{content:"\f025"}i.icon.spy:before{content:"\f21b"}i.icon.stripe.card:before{content:"\f1f5"}i.icon.student:before{content:"\f19d"}i.icon.talk:before{content:"\f27a"}i.icon.target:before{content:"\f140"}i.icon.teletype:before{content:"\f1e4"}i.icon.television:before{content:"\f26c"}i.icon.text.cursor:before{content:"\f246"}i.icon.text.telephone:before{content:"\f1e4"}i.icon.theme.isle:before{content:"\f2b2"}i.icon.theme:before{content:"\f043"}i.icon.thermometer:before{content:"\f2c7"}i.icon.thumb.tack:before{content:"\f08d"}i.icon.time:before{content:"\f017"}i.icon.tm:before{content:"\f25c"}i.icon.toggle.down:before{content:"\f150"}i.icon.toggle.left:before{content:"\f191"}i.icon.toggle.right:before{content:"\f152"}i.icon.toggle.up:before{content:"\f151"}i.icon.translate:before{content:"\f1ab"}i.icon.travel:before{content:"\f0b1"}i.icon.treatment:before{content:"\f0f1"}i.icon.triangle.down:before{content:"\f0d7"}i.icon.triangle.left:before{content:"\f0d9"}i.icon.triangle.right:before{content:"\f0da"}i.icon.triangle.up:before{content:"\f0d8"}i.icon.try:before{content:"\f195"}i.icon.unhide:before{content:"\f06e"}i.icon.unlinkify:before{content:"\f127"}i.icon.unmute:before{content:"\f130"}i.icon.usd:before{content:"\f155"}i.icon.user.cancel:before{content:"\f235"}i.icon.user.close:before{content:"\f235"}i.icon.user.delete:before{content:"\f235"}i.icon.user.x:before{content:"\f235"}i.icon.vcard:before{content:"\f2bb"}i.icon.video.camera:before{content:"\f03d"}i.icon.video.play:before{content:"\f144"}i.icon.visa.card:before{content:"\f1f0"}i.icon.visa:before{content:"\f1f0"}i.icon.volume.control.phone:before{content:"\f2a0"}i.icon.wait:before{content:"\f017"}i.icon.warning.circle:before{content:"\f06a"}i.icon.warning.sign:before{content:"\f071"}i.icon.warning:before{content:"\f12a"}i.icon.wechat:before{content:"\f1d7"}i.icon.wi-fi:before{content:"\f1eb"}i.icon.wikipedia:before{content:"\f266"}i.icon.winner:before{content:"\f091"}i.icon.wizard:before{content:"\f0d0"}i.icon.woman:before{content:"\f221"}i.icon.won:before{content:"\f159"}i.icon.wordpress.beginner:before{content:"\f297"}i.icon.wordpress.forms:before{content:"\f298"}i.icon.world:before{content:"\f0ac"}i.icon.write.square:before{content:"\f14b"}i.icon.x:before{content:"\f00d"}i.icon.yc:before{content:"\f23b"}i.icon.ycombinator:before{content:"\f23b"}i.icon.yen:before{content:"\f157"}i.icon.zip:before{content:"\f187"}i.icon.zoom-in:before{content:"\f00e"}i.icon.zoom-out:before{content:"\f010"}i.icon.zoom:before{content:"\f00e"}i.icon.bitbucket.square:before{content:"\f171"}i.icon.checkmark.box:before{content:"\f14a"}i.icon.circle.thin:before{content:"\f111"}i.icon.cloud.download:before{content:"\f381"}i.icon.cloud.upload:before{content:"\f382"}i.icon.compose:before{content:"\f303"}i.icon.conversation:before{content:"\f086"}i.icon.credit.card.alternative:before{content:"\f09d"}i.icon.currency:before{content:"\f3d1"}i.icon.dashboard:before{content:"\f3fd"}i.icon.diamond:before{content:"\f3a5"}i.icon.disk:before{content:"\f0a0"}i.icon.exchange:before{content:"\f362"}i.icon.external.share:before{content:"\f14d"}i.icon.external.square:before{content:"\f360"}i.icon.external:before{content:"\f35d"}i.icon.facebook.official:before{content:"\f082"}i.icon.food:before{content:"\f2e7"}i.icon.hourglass.zero:before{content:"\f253"}i.icon.level.down:before{content:"\f3be"}i.icon.level.up:before{content:"\f3bf"}i.icon.logout:before{content:"\f2f5"}i.icon.meanpath:before{content:"\f0c8"}i.icon.money:before{content:"\f3d1"}i.icon.move:before{content:"\f0b2"}i.icon.pencil:before{content:"\f303"}i.icon.protect:before{content:"\f023"}i.icon.radio:before{content:"\f192"}i.icon.remove.bookmark:before{content:"\f02e"}i.icon.resize.horizontal:before{content:"\f337"}i.icon.resize.vertical:before{content:"\f338"}i.icon.sign-in:before{content:"\f2f6"}i.icon.sign-out:before{content:"\f2f5"}i.icon.spoon:before{content:"\f2e5"}i.icon.star.half.empty:before{content:"\f089"}i.icon.star.half.full:before{content:"\f089"}i.icon.ticket:before{content:"\f3ff"}i.icon.times.rectangle:before{content:"\f410"}i.icon.write:before{content:"\f303"}i.icon.youtube.play:before{content:"\f167"}@font-face{font-family:outline-icons;src:url(assets/fonts/outline-icons.eot);src:url(assets/fonts/outline-icons.eot?#iefix) format('embedded-opentype'),url(assets/fonts/outline-icons.woff2) format('woff2'),url(assets/fonts/outline-icons.woff) format('woff'),url(assets/fonts/outline-icons.ttf) format('truetype'),url(assets/fonts/outline-icons.svg#icons) format('svg');font-style:normal;font-weight:400;font-variant:normal;text-decoration:inherit;text-transform:none}i.icon.outline{font-family:outline-icons}i.icon.address.book.outline:before{content:"\f2b9"}i.icon.address.card.outline:before{content:"\f2bb"}i.icon.arrow.alternate.circle.down.outline:before{content:"\f358"}i.icon.arrow.alternate.circle.left.outline:before{content:"\f359"}i.icon.arrow.alternate.circle.right.outline:before{content:"\f35a"}i.icon.arrow.alternate.circle.up.outline:before{content:"\f35b"}i.icon.bell.outline:before{content:"\f0f3"}i.icon.bell.slash.outline:before{content:"\f1f6"}i.icon.bookmark.outline:before{content:"\f02e"}i.icon.building.outline:before{content:"\f1ad"}i.icon.calendar.outline:before{content:"\f133"}i.icon.calendar.alternate.outline:before{content:"\f073"}i.icon.calendar.check.outline:before{content:"\f274"}i.icon.calendar.minus.outline:before{content:"\f272"}i.icon.calendar.plus.outline:before{content:"\f271"}i.icon.calendar.times.outline:before{content:"\f273"}i.icon.caret.square.down.outline:before{content:"\f150"}i.icon.caret.square.left.outline:before{content:"\f191"}i.icon.caret.square.right.outline:before{content:"\f152"}i.icon.caret.square.up.outline:before{content:"\f151"}i.icon.chart.bar.outline:before{content:"\f080"}i.icon.check.circle.outline:before{content:"\f058"}i.icon.check.square.outline:before{content:"\f14a"}i.icon.circle.outline:before{content:"\f111"}i.icon.clipboard.outline:before{content:"\f328"}i.icon.clock.outline:before{content:"\f017"}i.icon.clone.outline:before{content:"\f24d"}i.icon.closed.captioning.outline:before{content:"\f20a"}i.icon.comment.outline:before{content:"\f075"}i.icon.comment.alternate.outline:before{content:"\f27a"}i.icon.comments.outline:before{content:"\f086"}i.icon.compass.outline:before{content:"\f14e"}i.icon.copy.outline:before{content:"\f0c5"}i.icon.copyright.outline:before{content:"\f1f9"}i.icon.credit.card.outline:before{content:"\f09d"}i.icon.dot.circle.outline:before{content:"\f192"}i.icon.edit.outline:before{content:"\f044"}i.icon.envelope.outline:before{content:"\f0e0"}i.icon.envelope.open.outline:before{content:"\f2b6"}i.icon.eye.slash.outline:before{content:"\f070"}i.icon.file.outline:before{content:"\f15b"}i.icon.file.alternate.outline:before{content:"\f15c"}i.icon.file.archive.outline:before{content:"\f1c6"}i.icon.file.audio.outline:before{content:"\f1c7"}i.icon.file.code.outline:before{content:"\f1c9"}i.icon.file.excel.outline:before{content:"\f1c3"}i.icon.file.image.outline:before{content:"\f1c5"}i.icon.file.pdf.outline:before{content:"\f1c1"}i.icon.file.powerpoint.outline:before{content:"\f1c4"}i.icon.file.video.outline:before{content:"\f1c8"}i.icon.file.word.outline:before{content:"\f1c2"}i.icon.flag.outline:before{content:"\f024"}i.icon.folder.outline:before{content:"\f07b"}i.icon.folder.open.outline:before{content:"\f07c"}i.icon.frown.outline:before{content:"\f119"}i.icon.futbol.outline:before{content:"\f1e3"}i.icon.gem.outline:before{content:"\f3a5"}i.icon.hand.lizard.outline:before{content:"\f258"}i.icon.hand.paper.outline:before{content:"\f256"}i.icon.hand.peace.outline:before{content:"\f25b"}i.icon.hand.point.down.outline:before{content:"\f0a7"}i.icon.hand.point.left.outline:before{content:"\f0a5"}i.icon.hand.point.right.outline:before{content:"\f0a4"}i.icon.hand.point.up.outline:before{content:"\f0a6"}i.icon.hand.pointer.outline:before{content:"\f25a"}i.icon.hand.rock.outline:before{content:"\f255"}i.icon.hand.scissors.outline:before{content:"\f257"}i.icon.hand.spock.outline:before{content:"\f259"}i.icon.handshake.outline:before{content:"\f2b5"}i.icon.hdd.outline:before{content:"\f0a0"}i.icon.heart.outline:before{content:"\f004"}i.icon.hospital.outline:before{content:"\f0f8"}i.icon.hourglass.outline:before{content:"\f254"}i.icon.id.badge.outline:before{content:"\f2c1"}i.icon.id.card.outline:before{content:"\f2c2"}i.icon.image.outline:before{content:"\f03e"}i.icon.images.outline:before{content:"\f302"}i.icon.keyboard.outline:before{content:"\f11c"}i.icon.lemon.outline:before{content:"\f094"}i.icon.life.ring.outline:before{content:"\f1cd"}i.icon.lightbulb.outline:before{content:"\f0eb"}i.icon.list.alternate.outline:before{content:"\f022"}i.icon.map.outline:before{content:"\f279"}i.icon.meh.outline:before{content:"\f11a"}i.icon.minus.square.outline:before{content:"\f146"}i.icon.money.bill.alternate.outline:before{content:"\f3d1"}i.icon.moon.outline:before{content:"\f186"}i.icon.newspaper.outline:before{content:"\f1ea"}i.icon.object.group.outline:before{content:"\f247"}i.icon.object.ungroup.outline:before{content:"\f248"}i.icon.paper.plane.outline:before{content:"\f1d8"}i.icon.pause.circle.outline:before{content:"\f28b"}i.icon.play.circle.outline:before{content:"\f144"}i.icon.plus.square.outline:before{content:"\f0fe"}i.icon.question.circle.outline:before{content:"\f059"}i.icon.registered.outline:before{content:"\f25d"}i.icon.save.outline:before{content:"\f0c7"}i.icon.share.square.outline:before{content:"\f14d"}i.icon.smile.outline:before{content:"\f118"}i.icon.snowflake.outline:before{content:"\f2dc"}i.icon.square.outline:before{content:"\f0c8"}i.icon.star.outline:before{content:"\f005"}i.icon.star.half.outline:before{content:"\f089"}i.icon.sticky.note.outline:before{content:"\f249"}i.icon.stop.circle.outline:before{content:"\f28d"}i.icon.sun.outline:before{content:"\f185"}i.icon.thumbs.down.outline:before{content:"\f165"}i.icon.thumbs.up.outline:before{content:"\f164"}i.icon.times.circle.outline:before{content:"\f057"}i.icon.trash.alternate.outline:before{content:"\f2ed"}i.icon.user.outline:before{content:"\f007"}i.icon.user.circle.outline:before{content:"\f2bd"}i.icon.window.close.outline:before{content:"\f410"}i.icon.window.maximize.outline:before{content:"\f2d0"}i.icon.window.minimize.outline:before{content:"\f2d1"}i.icon.window.restore.outline:before{content:"\f2d2"}i.icon.disk.outline:before{content:"\f369"}i.icon.heart.empty,i.icon.star.empty{font-family:outline-icons}i.icon.heart.empty:before{content:"\f004"}i.icon.star.empty:before{content:"\f089"}@font-face{font-family:brand-icons;src:url(assets/fonts/brand-icons.eot);src:url(assets/fonts/brand-icons.eot?#iefix) format('embedded-opentype'),url(assets/fonts/brand-icons.woff2) format('woff2'),url(assets/fonts/brand-icons.woff) format('woff'),url(assets/fonts/brand-icons.ttf) format('truetype'),url(assets/fonts/brand-icons.svg#icons) format('svg');font-style:normal;font-weight:400;font-variant:normal;text-decoration:inherit;text-transform:none}i.icon.\35 00px,i.icon.accessible.icon,i.icon.accusoft,i.icon.adn,i.icon.adversal,i.icon.affiliatetheme,i.icon.algolia,i.icon.amazon,i.icon.amazon.pay,i.icon.amilia,i.icon.android,i.icon.angellist,i.icon.angrycreative,i.icon.angular,i.icon.app.store,i.icon.app.store.ios,i.icon.apper,i.icon.apple,i.icon.apple.pay,i.icon.asymmetrik,i.icon.audible,i.icon.autoprefixer,i.icon.avianex,i.icon.aviato,i.icon.aws,i.icon.bandcamp,i.icon.behance,i.icon.behance.square,i.icon.bimobject,i.icon.bitbucket,i.icon.bitcoin,i.icon.bity,i.icon.black.tie,i.icon.blackberry,i.icon.blogger,i.icon.blogger.b,i.icon.bluetooth,i.icon.bluetooth.b,i.icon.btc,i.icon.buromobelexperte,i.icon.buysellads,i.icon.cc.amazon.pay,i.icon.cc.amex,i.icon.cc.apple.pay,i.icon.cc.diners.club,i.icon.cc.discover,i.icon.cc.jcb,i.icon.cc.mastercard,i.icon.cc.paypal,i.icon.cc.stripe,i.icon.cc.visa,i.icon.centercode,i.icon.chrome,i.icon.cloudscale,i.icon.cloudsmith,i.icon.cloudversify,i.icon.codepen,i.icon.codiepie,i.icon.connectdevelop,i.icon.contao,i.icon.cpanel,i.icon.creative.commons,i.icon.css3,i.icon.css3.alternate,i.icon.cuttlefish,i.icon.d.and.d,i.icon.dashcube,i.icon.delicious,i.icon.deploydog,i.icon.deskpro,i.icon.deviantart,i.icon.digg,i.icon.digital.ocean,i.icon.discord,i.icon.discourse,i.icon.dochub,i.icon.docker,i.icon.draft2digital,i.icon.dribbble,i.icon.dribbble.square,i.icon.dropbox,i.icon.drupal,i.icon.dyalog,i.icon.earlybirds,i.icon.edge,i.icon.elementor,i.icon.ember,i.icon.empire,i.icon.envira,i.icon.erlang,i.icon.ethereum,i.icon.etsy,i.icon.expeditedssl,i.icon.facebook,i.icon.facebook.f,i.icon.facebook.messenger,i.icon.facebook.square,i.icon.firefox,i.icon.first.order,i.icon.firstdraft,i.icon.flickr,i.icon.flipboard,i.icon.fly,i.icon.font.awesome,i.icon.font.awesome.alternate,i.icon.font.awesome.flag,i.icon.fonticons,i.icon.fonticons.fi,i.icon.fort.awesome,i.icon.fort.awesome.alternate,i.icon.forumbee,i.icon.foursquare,i.icon.free.code.camp,i.icon.freebsd,i.icon.get.pocket,i.icon.gg,i.icon.gg.circle,i.icon.git,i.icon.git.square,i.icon.github,i.icon.github.alternate,i.icon.github.square,i.icon.gitkraken,i.icon.gitlab,i.icon.gitter,i.icon.glide,i.icon.glide.g,i.icon.gofore,i.icon.goodreads,i.icon.goodreads.g,i.icon.google,i.icon.google.drive,i.icon.google.play,i.icon.google.plus,i.icon.google.plus.g,i.icon.google.plus.square,i.icon.google.wallet,i.icon.gratipay,i.icon.grav,i.icon.gripfire,i.icon.grunt,i.icon.gulp,i.icon.hacker.news,i.icon.hacker.news.square,i.icon.hips,i.icon.hire.a.helper,i.icon.hooli,i.icon.hotjar,i.icon.houzz,i.icon.html5,i.icon.hubspot,i.icon.imdb,i.icon.instagram,i.icon.internet.explorer,i.icon.ioxhost,i.icon.itunes,i.icon.itunes.note,i.icon.jenkins,i.icon.joget,i.icon.joomla,i.icon.js,i.icon.js.square,i.icon.jsfiddle,i.icon.keycdn,i.icon.kickstarter,i.icon.kickstarter.k,i.icon.korvue,i.icon.laravel,i.icon.lastfm,i.icon.lastfm.square,i.icon.leanpub,i.icon.less,i.icon.linechat,i.icon.linkedin,i.icon.linkedin.alternate,i.icon.linkedin.in,i.icon.linode,i.icon.linux,i.icon.lyft,i.icon.magento,i.icon.maxcdn,i.icon.medapps,i.icon.medium,i.icon.medium.m,i.icon.medrt,i.icon.meetup,i.icon.microsoft,i.icon.mix,i.icon.mixcloud,i.icon.mizuni,i.icon.modx,i.icon.monero,i.icon.napster,i.icon.nintendo.switch,i.icon.node,i.icon.node.js,i.icon.npm,i.icon.ns8,i.icon.nutritionix,i.icon.odnoklassniki,i.icon.odnoklassniki.square,i.icon.opencart,i.icon.openid,i.icon.opera,i.icon.optin.monster,i.icon.osi,i.icon.page4,i.icon.pagelines,i.icon.palfed,i.icon.patreon,i.icon.paypal,i.icon.periscope,i.icon.phabricator,i.icon.phoenix.framework,i.icon.php,i.icon.pied.piper,i.icon.pied.piper.alternate,i.icon.pied.piper.pp,i.icon.pinterest,i.icon.pinterest.p,i.icon.pinterest.square,i.icon.playstation,i.icon.product.hunt,i.icon.pushed,i.icon.python,i.icon.qq,i.icon.quinscape,i.icon.quora,i.icon.ravelry,i.icon.react,i.icon.rebel,i.icon.reddit,i.icon.reddit.alien,i.icon.reddit.square,i.icon.redriver,i.icon.rendact,i.icon.renren,i.icon.replyd,i.icon.resolving,i.icon.rocketchat,i.icon.rockrms,i.icon.safari,i.icon.sass,i.icon.schlix,i.icon.scribd,i.icon.searchengin,i.icon.sellcast,i.icon.sellsy,i.icon.servicestack,i.icon.shirtsinbulk,i.icon.simplybuilt,i.icon.sistrix,i.icon.skyatlas,i.icon.skype,i.icon.slack,i.icon.slack.hash,i.icon.slideshare,i.icon.snapchat,i.icon.snapchat.ghost,i.icon.snapchat.square,i.icon.soundcloud,i.icon.speakap,i.icon.spotify,i.icon.stack.exchange,i.icon.stack.overflow,i.icon.staylinked,i.icon.steam,i.icon.steam.square,i.icon.steam.symbol,i.icon.sticker.mule,i.icon.strava,i.icon.stripe,i.icon.stripe.s,i.icon.studiovinari,i.icon.stumbleupon,i.icon.stumbleupon.circle,i.icon.superpowers,i.icon.supple,i.icon.telegram,i.icon.telegram.plane,i.icon.tencent.weibo,i.icon.themeisle,i.icon.trello,i.icon.tripadvisor,i.icon.tumblr,i.icon.tumblr.square,i.icon.twitch,i.icon.twitter,i.icon.twitter.square,i.icon.typo3,i.icon.uber,i.icon.uikit,i.icon.uniregistry,i.icon.untappd,i.icon.usb,i.icon.ussunnah,i.icon.vaadin,i.icon.viacoin,i.icon.viadeo,i.icon.viadeo.square,i.icon.viber,i.icon.vimeo,i.icon.vimeo.square,i.icon.vimeo.v,i.icon.vine,i.icon.vk,i.icon.vnv,i.icon.vuejs,i.icon.wechat,i.icon.weibo,i.icon.weixin,i.icon.whatsapp,i.icon.whatsapp.square,i.icon.whmcs,i.icon.wikipedia.w,i.icon.windows,i.icon.wordpress,i.icon.wordpress.simple,i.icon.wpbeginner,i.icon.wpexplorer,i.icon.wpforms,i.icon.xbox,i.icon.xing,i.icon.xing.square,i.icon.y.combinator,i.icon.yahoo,i.icon.yandex,i.icon.yandex.international,i.icon.yelp,i.icon.yoast,i.icon.youtube,i.icon.youtube.square{font-family:brand-icons} \ No newline at end of file diff --git a/kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/icon.png b/kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/icon.png deleted file mode 100644 index f0090030d..000000000 Binary files a/kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/icon.png and /dev/null differ diff --git a/kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/index.html b/kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/index.html deleted file mode 100644 index e3761df62..000000000 --- a/kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/index.html +++ /dev/null @@ -1,58 +0,0 @@ - - - - - - - Kyuubi Web UI - - - - - - - -
    -
    -
    - -
    - Home - Environment - Sessions - Operations - Metrics - Thread Dump - REST API - Documentation - En -
    -
    - -
    -
    - -
    -
    Hang on...
    -

    We are looking for developers to help us build the UI.

    -
    -
    -
    - - - diff --git a/kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/jquery-3.6.0.min.js b/kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/jquery-3.6.0.min.js deleted file mode 100644 index c4c6022f2..000000000 --- a/kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/jquery-3.6.0.min.js +++ /dev/null @@ -1,2 +0,0 @@ -/*! jQuery v3.6.0 | (c) OpenJS Foundation and other contributors | jquery.org/license */ -!function(e,t){"use strict";"object"==typeof module&&"object"==typeof module.exports?module.exports=e.document?t(e,!0):function(e){if(!e.document)throw new Error("jQuery requires a window with a document");return t(e)}:t(e)}("undefined"!=typeof window?window:this,function(C,e){"use strict";var t=[],r=Object.getPrototypeOf,s=t.slice,g=t.flat?function(e){return t.flat.call(e)}:function(e){return t.concat.apply([],e)},u=t.push,i=t.indexOf,n={},o=n.toString,v=n.hasOwnProperty,a=v.toString,l=a.call(Object),y={},m=function(e){return"function"==typeof e&&"number"!=typeof e.nodeType&&"function"!=typeof e.item},x=function(e){return null!=e&&e===e.window},E=C.document,c={type:!0,src:!0,nonce:!0,noModule:!0};function b(e,t,n){var r,i,o=(n=n||E).createElement("script");if(o.text=e,t)for(r in c)(i=t[r]||t.getAttribute&&t.getAttribute(r))&&o.setAttribute(r,i);n.head.appendChild(o).parentNode.removeChild(o)}function w(e){return null==e?e+"":"object"==typeof e||"function"==typeof e?n[o.call(e)]||"object":typeof e}var f="3.6.0",S=function(e,t){return new S.fn.init(e,t)};function p(e){var t=!!e&&"length"in e&&e.length,n=w(e);return!m(e)&&!x(e)&&("array"===n||0===t||"number"==typeof t&&0+~]|"+M+")"+M+"*"),U=new RegExp(M+"|>"),X=new RegExp(F),V=new RegExp("^"+I+"$"),G={ID:new RegExp("^#("+I+")"),CLASS:new RegExp("^\\.("+I+")"),TAG:new RegExp("^("+I+"|[*])"),ATTR:new RegExp("^"+W),PSEUDO:new RegExp("^"+F),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+M+"*(even|odd|(([+-]|)(\\d*)n|)"+M+"*(?:([+-]|)"+M+"*(\\d+)|))"+M+"*\\)|)","i"),bool:new RegExp("^(?:"+R+")$","i"),needsContext:new RegExp("^"+M+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+M+"*((?:-\\d)?\\d*)"+M+"*\\)|)(?=[^-]|$)","i")},Y=/HTML$/i,Q=/^(?:input|select|textarea|button)$/i,J=/^h\d$/i,K=/^[^{]+\{\s*\[native \w/,Z=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,ee=/[+~]/,te=new RegExp("\\\\[\\da-fA-F]{1,6}"+M+"?|\\\\([^\\r\\n\\f])","g"),ne=function(e,t){var n="0x"+e.slice(1)-65536;return t||(n<0?String.fromCharCode(n+65536):String.fromCharCode(n>>10|55296,1023&n|56320))},re=/([\0-\x1f\x7f]|^-?\d)|^-$|[^\0-\x1f\x7f-\uFFFF\w-]/g,ie=function(e,t){return t?"\0"===e?"\ufffd":e.slice(0,-1)+"\\"+e.charCodeAt(e.length-1).toString(16)+" ":"\\"+e},oe=function(){T()},ae=be(function(e){return!0===e.disabled&&"fieldset"===e.nodeName.toLowerCase()},{dir:"parentNode",next:"legend"});try{H.apply(t=O.call(p.childNodes),p.childNodes),t[p.childNodes.length].nodeType}catch(e){H={apply:t.length?function(e,t){L.apply(e,O.call(t))}:function(e,t){var n=e.length,r=0;while(e[n++]=t[r++]);e.length=n-1}}}function se(t,e,n,r){var i,o,a,s,u,l,c,f=e&&e.ownerDocument,p=e?e.nodeType:9;if(n=n||[],"string"!=typeof t||!t||1!==p&&9!==p&&11!==p)return n;if(!r&&(T(e),e=e||C,E)){if(11!==p&&(u=Z.exec(t)))if(i=u[1]){if(9===p){if(!(a=e.getElementById(i)))return n;if(a.id===i)return n.push(a),n}else if(f&&(a=f.getElementById(i))&&y(e,a)&&a.id===i)return n.push(a),n}else{if(u[2])return H.apply(n,e.getElementsByTagName(t)),n;if((i=u[3])&&d.getElementsByClassName&&e.getElementsByClassName)return H.apply(n,e.getElementsByClassName(i)),n}if(d.qsa&&!N[t+" "]&&(!v||!v.test(t))&&(1!==p||"object"!==e.nodeName.toLowerCase())){if(c=t,f=e,1===p&&(U.test(t)||z.test(t))){(f=ee.test(t)&&ye(e.parentNode)||e)===e&&d.scope||((s=e.getAttribute("id"))?s=s.replace(re,ie):e.setAttribute("id",s=S)),o=(l=h(t)).length;while(o--)l[o]=(s?"#"+s:":scope")+" "+xe(l[o]);c=l.join(",")}try{return H.apply(n,f.querySelectorAll(c)),n}catch(e){N(t,!0)}finally{s===S&&e.removeAttribute("id")}}}return g(t.replace($,"$1"),e,n,r)}function ue(){var r=[];return function e(t,n){return r.push(t+" ")>b.cacheLength&&delete e[r.shift()],e[t+" "]=n}}function le(e){return e[S]=!0,e}function ce(e){var t=C.createElement("fieldset");try{return!!e(t)}catch(e){return!1}finally{t.parentNode&&t.parentNode.removeChild(t),t=null}}function fe(e,t){var n=e.split("|"),r=n.length;while(r--)b.attrHandle[n[r]]=t}function pe(e,t){var n=t&&e,r=n&&1===e.nodeType&&1===t.nodeType&&e.sourceIndex-t.sourceIndex;if(r)return r;if(n)while(n=n.nextSibling)if(n===t)return-1;return e?1:-1}function de(t){return function(e){return"input"===e.nodeName.toLowerCase()&&e.type===t}}function he(n){return function(e){var t=e.nodeName.toLowerCase();return("input"===t||"button"===t)&&e.type===n}}function ge(t){return function(e){return"form"in e?e.parentNode&&!1===e.disabled?"label"in e?"label"in e.parentNode?e.parentNode.disabled===t:e.disabled===t:e.isDisabled===t||e.isDisabled!==!t&&ae(e)===t:e.disabled===t:"label"in e&&e.disabled===t}}function ve(a){return le(function(o){return o=+o,le(function(e,t){var n,r=a([],e.length,o),i=r.length;while(i--)e[n=r[i]]&&(e[n]=!(t[n]=e[n]))})})}function ye(e){return e&&"undefined"!=typeof e.getElementsByTagName&&e}for(e in d=se.support={},i=se.isXML=function(e){var t=e&&e.namespaceURI,n=e&&(e.ownerDocument||e).documentElement;return!Y.test(t||n&&n.nodeName||"HTML")},T=se.setDocument=function(e){var t,n,r=e?e.ownerDocument||e:p;return r!=C&&9===r.nodeType&&r.documentElement&&(a=(C=r).documentElement,E=!i(C),p!=C&&(n=C.defaultView)&&n.top!==n&&(n.addEventListener?n.addEventListener("unload",oe,!1):n.attachEvent&&n.attachEvent("onunload",oe)),d.scope=ce(function(e){return a.appendChild(e).appendChild(C.createElement("div")),"undefined"!=typeof e.querySelectorAll&&!e.querySelectorAll(":scope fieldset div").length}),d.attributes=ce(function(e){return e.className="i",!e.getAttribute("className")}),d.getElementsByTagName=ce(function(e){return e.appendChild(C.createComment("")),!e.getElementsByTagName("*").length}),d.getElementsByClassName=K.test(C.getElementsByClassName),d.getById=ce(function(e){return a.appendChild(e).id=S,!C.getElementsByName||!C.getElementsByName(S).length}),d.getById?(b.filter.ID=function(e){var t=e.replace(te,ne);return function(e){return e.getAttribute("id")===t}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n=t.getElementById(e);return n?[n]:[]}}):(b.filter.ID=function(e){var n=e.replace(te,ne);return function(e){var t="undefined"!=typeof e.getAttributeNode&&e.getAttributeNode("id");return t&&t.value===n}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n,r,i,o=t.getElementById(e);if(o){if((n=o.getAttributeNode("id"))&&n.value===e)return[o];i=t.getElementsByName(e),r=0;while(o=i[r++])if((n=o.getAttributeNode("id"))&&n.value===e)return[o]}return[]}}),b.find.TAG=d.getElementsByTagName?function(e,t){return"undefined"!=typeof t.getElementsByTagName?t.getElementsByTagName(e):d.qsa?t.querySelectorAll(e):void 0}:function(e,t){var n,r=[],i=0,o=t.getElementsByTagName(e);if("*"===e){while(n=o[i++])1===n.nodeType&&r.push(n);return r}return o},b.find.CLASS=d.getElementsByClassName&&function(e,t){if("undefined"!=typeof t.getElementsByClassName&&E)return t.getElementsByClassName(e)},s=[],v=[],(d.qsa=K.test(C.querySelectorAll))&&(ce(function(e){var t;a.appendChild(e).innerHTML="",e.querySelectorAll("[msallowcapture^='']").length&&v.push("[*^$]="+M+"*(?:''|\"\")"),e.querySelectorAll("[selected]").length||v.push("\\["+M+"*(?:value|"+R+")"),e.querySelectorAll("[id~="+S+"-]").length||v.push("~="),(t=C.createElement("input")).setAttribute("name",""),e.appendChild(t),e.querySelectorAll("[name='']").length||v.push("\\["+M+"*name"+M+"*="+M+"*(?:''|\"\")"),e.querySelectorAll(":checked").length||v.push(":checked"),e.querySelectorAll("a#"+S+"+*").length||v.push(".#.+[+~]"),e.querySelectorAll("\\\f"),v.push("[\\r\\n\\f]")}),ce(function(e){e.innerHTML="";var t=C.createElement("input");t.setAttribute("type","hidden"),e.appendChild(t).setAttribute("name","D"),e.querySelectorAll("[name=d]").length&&v.push("name"+M+"*[*^$|!~]?="),2!==e.querySelectorAll(":enabled").length&&v.push(":enabled",":disabled"),a.appendChild(e).disabled=!0,2!==e.querySelectorAll(":disabled").length&&v.push(":enabled",":disabled"),e.querySelectorAll("*,:x"),v.push(",.*:")})),(d.matchesSelector=K.test(c=a.matches||a.webkitMatchesSelector||a.mozMatchesSelector||a.oMatchesSelector||a.msMatchesSelector))&&ce(function(e){d.disconnectedMatch=c.call(e,"*"),c.call(e,"[s!='']:x"),s.push("!=",F)}),v=v.length&&new RegExp(v.join("|")),s=s.length&&new RegExp(s.join("|")),t=K.test(a.compareDocumentPosition),y=t||K.test(a.contains)?function(e,t){var n=9===e.nodeType?e.documentElement:e,r=t&&t.parentNode;return e===r||!(!r||1!==r.nodeType||!(n.contains?n.contains(r):e.compareDocumentPosition&&16&e.compareDocumentPosition(r)))}:function(e,t){if(t)while(t=t.parentNode)if(t===e)return!0;return!1},j=t?function(e,t){if(e===t)return l=!0,0;var n=!e.compareDocumentPosition-!t.compareDocumentPosition;return n||(1&(n=(e.ownerDocument||e)==(t.ownerDocument||t)?e.compareDocumentPosition(t):1)||!d.sortDetached&&t.compareDocumentPosition(e)===n?e==C||e.ownerDocument==p&&y(p,e)?-1:t==C||t.ownerDocument==p&&y(p,t)?1:u?P(u,e)-P(u,t):0:4&n?-1:1)}:function(e,t){if(e===t)return l=!0,0;var n,r=0,i=e.parentNode,o=t.parentNode,a=[e],s=[t];if(!i||!o)return e==C?-1:t==C?1:i?-1:o?1:u?P(u,e)-P(u,t):0;if(i===o)return pe(e,t);n=e;while(n=n.parentNode)a.unshift(n);n=t;while(n=n.parentNode)s.unshift(n);while(a[r]===s[r])r++;return r?pe(a[r],s[r]):a[r]==p?-1:s[r]==p?1:0}),C},se.matches=function(e,t){return se(e,null,null,t)},se.matchesSelector=function(e,t){if(T(e),d.matchesSelector&&E&&!N[t+" "]&&(!s||!s.test(t))&&(!v||!v.test(t)))try{var n=c.call(e,t);if(n||d.disconnectedMatch||e.document&&11!==e.document.nodeType)return n}catch(e){N(t,!0)}return 0":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(e){return e[1]=e[1].replace(te,ne),e[3]=(e[3]||e[4]||e[5]||"").replace(te,ne),"~="===e[2]&&(e[3]=" "+e[3]+" "),e.slice(0,4)},CHILD:function(e){return e[1]=e[1].toLowerCase(),"nth"===e[1].slice(0,3)?(e[3]||se.error(e[0]),e[4]=+(e[4]?e[5]+(e[6]||1):2*("even"===e[3]||"odd"===e[3])),e[5]=+(e[7]+e[8]||"odd"===e[3])):e[3]&&se.error(e[0]),e},PSEUDO:function(e){var t,n=!e[6]&&e[2];return G.CHILD.test(e[0])?null:(e[3]?e[2]=e[4]||e[5]||"":n&&X.test(n)&&(t=h(n,!0))&&(t=n.indexOf(")",n.length-t)-n.length)&&(e[0]=e[0].slice(0,t),e[2]=n.slice(0,t)),e.slice(0,3))}},filter:{TAG:function(e){var t=e.replace(te,ne).toLowerCase();return"*"===e?function(){return!0}:function(e){return e.nodeName&&e.nodeName.toLowerCase()===t}},CLASS:function(e){var t=m[e+" "];return t||(t=new RegExp("(^|"+M+")"+e+"("+M+"|$)"))&&m(e,function(e){return t.test("string"==typeof e.className&&e.className||"undefined"!=typeof e.getAttribute&&e.getAttribute("class")||"")})},ATTR:function(n,r,i){return function(e){var t=se.attr(e,n);return null==t?"!="===r:!r||(t+="","="===r?t===i:"!="===r?t!==i:"^="===r?i&&0===t.indexOf(i):"*="===r?i&&-1:\x20\t\r\n\f]*)[\x20\t\r\n\f]*\/?>(?:<\/\1>|)$/i;function j(e,n,r){return m(n)?S.grep(e,function(e,t){return!!n.call(e,t,e)!==r}):n.nodeType?S.grep(e,function(e){return e===n!==r}):"string"!=typeof n?S.grep(e,function(e){return-1)[^>]*|#([\w-]+))$/;(S.fn.init=function(e,t,n){var r,i;if(!e)return this;if(n=n||D,"string"==typeof e){if(!(r="<"===e[0]&&">"===e[e.length-1]&&3<=e.length?[null,e,null]:q.exec(e))||!r[1]&&t)return!t||t.jquery?(t||n).find(e):this.constructor(t).find(e);if(r[1]){if(t=t instanceof S?t[0]:t,S.merge(this,S.parseHTML(r[1],t&&t.nodeType?t.ownerDocument||t:E,!0)),N.test(r[1])&&S.isPlainObject(t))for(r in t)m(this[r])?this[r](t[r]):this.attr(r,t[r]);return this}return(i=E.getElementById(r[2]))&&(this[0]=i,this.length=1),this}return e.nodeType?(this[0]=e,this.length=1,this):m(e)?void 0!==n.ready?n.ready(e):e(S):S.makeArray(e,this)}).prototype=S.fn,D=S(E);var L=/^(?:parents|prev(?:Until|All))/,H={children:!0,contents:!0,next:!0,prev:!0};function O(e,t){while((e=e[t])&&1!==e.nodeType);return e}S.fn.extend({has:function(e){var t=S(e,this),n=t.length;return this.filter(function(){for(var e=0;e\x20\t\r\n\f]*)/i,he=/^$|^module$|\/(?:java|ecma)script/i;ce=E.createDocumentFragment().appendChild(E.createElement("div")),(fe=E.createElement("input")).setAttribute("type","radio"),fe.setAttribute("checked","checked"),fe.setAttribute("name","t"),ce.appendChild(fe),y.checkClone=ce.cloneNode(!0).cloneNode(!0).lastChild.checked,ce.innerHTML="",y.noCloneChecked=!!ce.cloneNode(!0).lastChild.defaultValue,ce.innerHTML="",y.option=!!ce.lastChild;var ge={thead:[1,"","
    "],col:[2,"","
    "],tr:[2,"","
    "],td:[3,"","
    "],_default:[0,"",""]};function ve(e,t){var n;return n="undefined"!=typeof e.getElementsByTagName?e.getElementsByTagName(t||"*"):"undefined"!=typeof e.querySelectorAll?e.querySelectorAll(t||"*"):[],void 0===t||t&&A(e,t)?S.merge([e],n):n}function ye(e,t){for(var n=0,r=e.length;n",""]);var me=/<|&#?\w+;/;function xe(e,t,n,r,i){for(var o,a,s,u,l,c,f=t.createDocumentFragment(),p=[],d=0,h=e.length;d\s*$/g;function je(e,t){return A(e,"table")&&A(11!==t.nodeType?t:t.firstChild,"tr")&&S(e).children("tbody")[0]||e}function De(e){return e.type=(null!==e.getAttribute("type"))+"/"+e.type,e}function qe(e){return"true/"===(e.type||"").slice(0,5)?e.type=e.type.slice(5):e.removeAttribute("type"),e}function Le(e,t){var n,r,i,o,a,s;if(1===t.nodeType){if(Y.hasData(e)&&(s=Y.get(e).events))for(i in Y.remove(t,"handle events"),s)for(n=0,r=s[i].length;n").attr(n.scriptAttrs||{}).prop({charset:n.scriptCharset,src:n.url}).on("load error",i=function(e){r.remove(),i=null,e&&t("error"===e.type?404:200,e.type)}),E.head.appendChild(r[0])},abort:function(){i&&i()}}});var _t,zt=[],Ut=/(=)\?(?=&|$)|\?\?/;S.ajaxSetup({jsonp:"callback",jsonpCallback:function(){var e=zt.pop()||S.expando+"_"+wt.guid++;return this[e]=!0,e}}),S.ajaxPrefilter("json jsonp",function(e,t,n){var r,i,o,a=!1!==e.jsonp&&(Ut.test(e.url)?"url":"string"==typeof e.data&&0===(e.contentType||"").indexOf("application/x-www-form-urlencoded")&&Ut.test(e.data)&&"data");if(a||"jsonp"===e.dataTypes[0])return r=e.jsonpCallback=m(e.jsonpCallback)?e.jsonpCallback():e.jsonpCallback,a?e[a]=e[a].replace(Ut,"$1"+r):!1!==e.jsonp&&(e.url+=(Tt.test(e.url)?"&":"?")+e.jsonp+"="+r),e.converters["script json"]=function(){return o||S.error(r+" was not called"),o[0]},e.dataTypes[0]="json",i=C[r],C[r]=function(){o=arguments},n.always(function(){void 0===i?S(C).removeProp(r):C[r]=i,e[r]&&(e.jsonpCallback=t.jsonpCallback,zt.push(r)),o&&m(i)&&i(o[0]),o=i=void 0}),"script"}),y.createHTMLDocument=((_t=E.implementation.createHTMLDocument("").body).innerHTML="
    ",2===_t.childNodes.length),S.parseHTML=function(e,t,n){return"string"!=typeof e?[]:("boolean"==typeof t&&(n=t,t=!1),t||(y.createHTMLDocument?((r=(t=E.implementation.createHTMLDocument("")).createElement("base")).href=E.location.href,t.head.appendChild(r)):t=E),o=!n&&[],(i=N.exec(e))?[t.createElement(i[1])]:(i=xe([e],t,o),o&&o.length&&S(o).remove(),S.merge([],i.childNodes)));var r,i,o},S.fn.load=function(e,t,n){var r,i,o,a=this,s=e.indexOf(" ");return-1").append(S.parseHTML(e)).find(r):e)}).always(n&&function(e,t){a.each(function(){n.apply(this,o||[e.responseText,t,e])})}),this},S.expr.pseudos.animated=function(t){return S.grep(S.timers,function(e){return t===e.elem}).length},S.offset={setOffset:function(e,t,n){var r,i,o,a,s,u,l=S.css(e,"position"),c=S(e),f={};"static"===l&&(e.style.position="relative"),s=c.offset(),o=S.css(e,"top"),u=S.css(e,"left"),("absolute"===l||"fixed"===l)&&-1<(o+u).indexOf("auto")?(a=(r=c.position()).top,i=r.left):(a=parseFloat(o)||0,i=parseFloat(u)||0),m(t)&&(t=t.call(e,n,S.extend({},s))),null!=t.top&&(f.top=t.top-s.top+a),null!=t.left&&(f.left=t.left-s.left+i),"using"in t?t.using.call(e,f):c.css(f)}},S.fn.extend({offset:function(t){if(arguments.length)return void 0===t?this:this.each(function(e){S.offset.setOffset(this,t,e)});var e,n,r=this[0];return r?r.getClientRects().length?(e=r.getBoundingClientRect(),n=r.ownerDocument.defaultView,{top:e.top+n.pageYOffset,left:e.left+n.pageXOffset}):{top:0,left:0}:void 0},position:function(){if(this[0]){var e,t,n,r=this[0],i={top:0,left:0};if("fixed"===S.css(r,"position"))t=r.getBoundingClientRect();else{t=this.offset(),n=r.ownerDocument,e=r.offsetParent||n.documentElement;while(e&&(e===n.body||e===n.documentElement)&&"static"===S.css(e,"position"))e=e.parentNode;e&&e!==r&&1===e.nodeType&&((i=S(e).offset()).top+=S.css(e,"borderTopWidth",!0),i.left+=S.css(e,"borderLeftWidth",!0))}return{top:t.top-i.top-S.css(r,"marginTop",!0),left:t.left-i.left-S.css(r,"marginLeft",!0)}}},offsetParent:function(){return this.map(function(){var e=this.offsetParent;while(e&&"static"===S.css(e,"position"))e=e.offsetParent;return e||re})}}),S.each({scrollLeft:"pageXOffset",scrollTop:"pageYOffset"},function(t,i){var o="pageYOffset"===i;S.fn[t]=function(e){return $(this,function(e,t,n){var r;if(x(e)?r=e:9===e.nodeType&&(r=e.defaultView),void 0===n)return r?r[i]:e[t];r?r.scrollTo(o?r.pageXOffset:n,o?n:r.pageYOffset):e[t]=n},t,e,arguments.length)}}),S.each(["top","left"],function(e,n){S.cssHooks[n]=Fe(y.pixelPosition,function(e,t){if(t)return t=We(e,n),Pe.test(t)?S(e).position()[n]+"px":t})}),S.each({Height:"height",Width:"width"},function(a,s){S.each({padding:"inner"+a,content:s,"":"outer"+a},function(r,o){S.fn[o]=function(e,t){var n=arguments.length&&(r||"boolean"!=typeof e),i=r||(!0===e||!0===t?"margin":"border");return $(this,function(e,t,n){var r;return x(e)?0===o.indexOf("outer")?e["inner"+a]:e.document.documentElement["client"+a]:9===e.nodeType?(r=e.documentElement,Math.max(e.body["scroll"+a],r["scroll"+a],e.body["offset"+a],r["offset"+a],r["client"+a])):void 0===n?S.css(e,t,i):S.style(e,t,n,i)},s,n?e:void 0,n)}})}),S.each(["ajaxStart","ajaxStop","ajaxComplete","ajaxError","ajaxSuccess","ajaxSend"],function(e,t){S.fn[t]=function(e){return this.on(t,e)}}),S.fn.extend({bind:function(e,t,n){return this.on(e,null,t,n)},unbind:function(e,t){return this.off(e,null,t)},delegate:function(e,t,n,r){return this.on(t,e,n,r)},undelegate:function(e,t,n){return 1===arguments.length?this.off(e,"**"):this.off(t,e||"**",n)},hover:function(e,t){return this.mouseenter(e).mouseleave(t||e)}}),S.each("blur focus focusin focusout resize scroll click dblclick mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave change select submit keydown keypress keyup contextmenu".split(" "),function(e,n){S.fn[n]=function(e,t){return 0 - - - - - - Kyuubi Server Metrics - - - - - - - - -
    -
    -
    - -
    - Home - Environment - Sessions - Operations - Metrics - Thread Dump - REST API - Documentation - En -
    -
    - -
    -
    - -
    -
    Hang on...
    -

    We are looking for developers to help us build the UI.

    -
    -
    -
    - - - - diff --git a/kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/operations.html b/kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/operations.html deleted file mode 100644 index e25fe9ceb..000000000 --- a/kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/operations.html +++ /dev/null @@ -1,59 +0,0 @@ - - - - - - - Title - - - - - - - - -
    -
    -
    - -
    - Home - Environment - Sessions - Operations - Metrics - Thread Dump - REST API - Documentation - En -
    -
    - -
    -
    - -
    -
    Hang on...
    -

    We are looking for developers to help us build the UI.

    -
    -
    -
    - - - diff --git a/kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/semantic.min.css b/kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/semantic.min.css deleted file mode 100644 index e2386193d..000000000 --- a/kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/semantic.min.css +++ /dev/null @@ -1,372 +0,0 @@ - /* - * # Semantic UI - 2.4.0 - * https://github.com/Semantic-Org/Semantic-UI - * http://www.semantic-ui.com/ - * - * Copyright 2014 Contributors - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */ -@import url(https://fonts.googleapis.com/css?family=Lato:400,700,400italic,700italic&subset=latin);/*! - * # Semantic UI 2.4.0 - Reset - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */*,:after,:before{-webkit-box-sizing:inherit;box-sizing:inherit}html{-webkit-box-sizing:border-box;box-sizing:border-box}input[type=email],input[type=password],input[type=search],input[type=text]{-webkit-appearance:none;-moz-appearance:none}/*! normalize.css v7.0.0 | MIT License | github.com/necolas/normalize.css */html{line-height:1.15;-ms-text-size-adjust:100%;-webkit-text-size-adjust:100%}body{margin:0}article,aside,footer,header,nav,section{display:block}h1{font-size:2em;margin:.67em 0}figcaption,figure,main{display:block}figure{margin:1em 40px}hr{-webkit-box-sizing:content-box;box-sizing:content-box;height:0;overflow:visible}pre{font-family:monospace,monospace;font-size:1em}a{background-color:transparent;-webkit-text-decoration-skip:objects}abbr[title]{border-bottom:none;text-decoration:underline;-webkit-text-decoration:underline dotted;text-decoration:underline dotted}b,strong{font-weight:inherit}b,strong{font-weight:bolder}code,kbd,samp{font-family:monospace,monospace;font-size:1em}dfn{font-style:italic}mark{background-color:#ff0;color:#000}small{font-size:80%}sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:baseline}sub{bottom:-.25em}sup{top:-.5em}audio,video{display:inline-block}audio:not([controls]){display:none;height:0}img{border-style:none}svg:not(:root){overflow:hidden}button,input,optgroup,select,textarea{font-family:sans-serif;font-size:100%;line-height:1.15;margin:0}button,input{overflow:visible}button,select{text-transform:none}[type=reset],[type=submit],button,html [type=button]{-webkit-appearance:button}[type=button]::-moz-focus-inner,[type=reset]::-moz-focus-inner,[type=submit]::-moz-focus-inner,button::-moz-focus-inner{border-style:none;padding:0}[type=button]:-moz-focusring,[type=reset]:-moz-focusring,[type=submit]:-moz-focusring,button:-moz-focusring{outline:1px dotted ButtonText}fieldset{padding:.35em .75em .625em}legend{-webkit-box-sizing:border-box;box-sizing:border-box;color:inherit;display:table;max-width:100%;padding:0;white-space:normal}progress{display:inline-block;vertical-align:baseline}textarea{overflow:auto}[type=checkbox],[type=radio]{-webkit-box-sizing:border-box;box-sizing:border-box;padding:0}[type=number]::-webkit-inner-spin-button,[type=number]::-webkit-outer-spin-button{height:auto}[type=search]{-webkit-appearance:textfield;outline-offset:-2px}[type=search]::-webkit-search-cancel-button,[type=search]::-webkit-search-decoration{-webkit-appearance:none}::-webkit-file-upload-button{-webkit-appearance:button;font:inherit}details,menu{display:block}summary{display:list-item}canvas{display:inline-block}template{display:none}[hidden]{display:none}/*! - * # Semantic UI 2.4.0 - Site - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */body,html{height:100%}html{font-size:14px}body{margin:0;padding:0;overflow-x:hidden;min-width:320px;background:#fff;font-family:Lato,'Helvetica Neue',Arial,Helvetica,sans-serif;font-size:14px;line-height:1.4285em;color:rgba(0,0,0,.87);font-smoothing:antialiased}h1,h2,h3,h4,h5{font-family:Lato,'Helvetica Neue',Arial,Helvetica,sans-serif;line-height:1.28571429em;margin:calc(2rem - .14285714em) 0 1rem;font-weight:700;padding:0}h1{min-height:1rem;font-size:2rem}h2{font-size:1.71428571rem}h3{font-size:1.28571429rem}h4{font-size:1.07142857rem}h5{font-size:1rem}h1:first-child,h2:first-child,h3:first-child,h4:first-child,h5:first-child{margin-top:0}h1:last-child,h2:last-child,h3:last-child,h4:last-child,h5:last-child{margin-bottom:0}p{margin:0 0 1em;line-height:1.4285em}p:first-child{margin-top:0}p:last-child{margin-bottom:0}a{color:#4183c4;text-decoration:none}a:hover{color:#1e70bf;text-decoration:none}::-webkit-selection{background-color:#cce2ff;color:rgba(0,0,0,.87)}::-moz-selection{background-color:#cce2ff;color:rgba(0,0,0,.87)}::selection{background-color:#cce2ff;color:rgba(0,0,0,.87)}input::-webkit-selection,textarea::-webkit-selection{background-color:rgba(100,100,100,.4);color:rgba(0,0,0,.87)}input::-moz-selection,textarea::-moz-selection{background-color:rgba(100,100,100,.4);color:rgba(0,0,0,.87)}input::selection,textarea::selection{background-color:rgba(100,100,100,.4);color:rgba(0,0,0,.87)}body ::-webkit-scrollbar{-webkit-appearance:none;width:10px;height:10px}body ::-webkit-scrollbar-track{background:rgba(0,0,0,.1);border-radius:0}body ::-webkit-scrollbar-thumb{cursor:pointer;border-radius:5px;background:rgba(0,0,0,.25);-webkit-transition:color .2s ease;transition:color .2s ease}body ::-webkit-scrollbar-thumb:window-inactive{background:rgba(0,0,0,.15)}body ::-webkit-scrollbar-thumb:hover{background:rgba(128,135,139,.8)}body .ui.inverted::-webkit-scrollbar-track{background:rgba(255,255,255,.1)}body .ui.inverted::-webkit-scrollbar-thumb{background:rgba(255,255,255,.25)}body .ui.inverted::-webkit-scrollbar-thumb:window-inactive{background:rgba(255,255,255,.15)}body .ui.inverted::-webkit-scrollbar-thumb:hover{background:rgba(255,255,255,.35)}/*! - * # Semantic UI 2.4.0 - Button - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */.ui.button{cursor:pointer;display:inline-block;min-height:1em;outline:0;border:none;vertical-align:baseline;background:#e0e1e2 none;color:rgba(0,0,0,.6);font-family:Lato,'Helvetica Neue',Arial,Helvetica,sans-serif;margin:0 .25em 0 0;padding:.78571429em 1.5em .78571429em;text-transform:none;text-shadow:none;font-weight:700;line-height:1em;font-style:normal;text-align:center;text-decoration:none;border-radius:.28571429rem;-webkit-box-shadow:0 0 0 1px transparent inset,0 0 0 0 rgba(34,36,38,.15) inset;box-shadow:0 0 0 1px transparent inset,0 0 0 0 rgba(34,36,38,.15) inset;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;-webkit-transition:opacity .1s ease,background-color .1s ease,color .1s ease,background .1s ease,-webkit-box-shadow .1s ease;transition:opacity .1s ease,background-color .1s ease,color .1s ease,background .1s ease,-webkit-box-shadow .1s ease;transition:opacity .1s ease,background-color .1s ease,color .1s ease,box-shadow .1s ease,background .1s ease;transition:opacity .1s ease,background-color .1s ease,color .1s ease,box-shadow .1s ease,background .1s ease,-webkit-box-shadow .1s ease;will-change:'';-webkit-tap-highlight-color:transparent}.ui.button:hover{background-color:#cacbcd;background-image:none;-webkit-box-shadow:0 0 0 1px transparent inset,0 0 0 0 rgba(34,36,38,.15) inset;box-shadow:0 0 0 1px transparent inset,0 0 0 0 rgba(34,36,38,.15) inset;color:rgba(0,0,0,.8)}.ui.button:hover .icon{opacity:.85}.ui.button:focus{background-color:#cacbcd;color:rgba(0,0,0,.8);background-image:''!important;-webkit-box-shadow:''!important;box-shadow:''!important}.ui.button:focus .icon{opacity:.85}.ui.active.button:active,.ui.button:active{background-color:#babbbc;background-image:'';color:rgba(0,0,0,.9);-webkit-box-shadow:0 0 0 1px transparent inset,none;box-shadow:0 0 0 1px transparent inset,none}.ui.active.button{background-color:#c0c1c2;background-image:none;-webkit-box-shadow:0 0 0 1px transparent inset;box-shadow:0 0 0 1px transparent inset;color:rgba(0,0,0,.95)}.ui.active.button:hover{background-color:#c0c1c2;background-image:none;color:rgba(0,0,0,.95)}.ui.active.button:active{background-color:#c0c1c2;background-image:none}.ui.loading.loading.loading.loading.loading.loading.button{position:relative;cursor:default;text-shadow:none!important;color:transparent!important;opacity:1;pointer-events:auto;-webkit-transition:all 0s linear,opacity .1s ease;transition:all 0s linear,opacity .1s ease}.ui.loading.button:before{position:absolute;content:'';top:50%;left:50%;margin:-.64285714em 0 0 -.64285714em;width:1.28571429em;height:1.28571429em;border-radius:500rem;border:.2em solid rgba(0,0,0,.15)}.ui.loading.button:after{position:absolute;content:'';top:50%;left:50%;margin:-.64285714em 0 0 -.64285714em;width:1.28571429em;height:1.28571429em;-webkit-animation:button-spin .6s linear;animation:button-spin .6s linear;-webkit-animation-iteration-count:infinite;animation-iteration-count:infinite;border-radius:500rem;border-color:#fff transparent transparent;border-style:solid;border-width:.2em;-webkit-box-shadow:0 0 0 1px transparent;box-shadow:0 0 0 1px transparent}.ui.labeled.icon.loading.button .icon{background-color:transparent;-webkit-box-shadow:none;box-shadow:none}@-webkit-keyframes button-spin{from{-webkit-transform:rotate(0);transform:rotate(0)}to{-webkit-transform:rotate(360deg);transform:rotate(360deg)}}@keyframes button-spin{from{-webkit-transform:rotate(0);transform:rotate(0)}to{-webkit-transform:rotate(360deg);transform:rotate(360deg)}}.ui.basic.loading.button:not(.inverted):before{border-color:rgba(0,0,0,.1)}.ui.basic.loading.button:not(.inverted):after{border-top-color:#767676}.ui.button:disabled,.ui.buttons .disabled.button,.ui.disabled.active.button,.ui.disabled.button,.ui.disabled.button:hover{cursor:default;opacity:.45!important;background-image:none!important;-webkit-box-shadow:none!important;box-shadow:none!important;pointer-events:none!important}.ui.basic.buttons .ui.disabled.button{border-color:rgba(34,36,38,.5)}.ui.animated.button{position:relative;overflow:hidden;padding-right:0!important;vertical-align:middle;z-index:1}.ui.animated.button .content{will-change:transform,opacity}.ui.animated.button .visible.content{position:relative;margin-right:1.5em}.ui.animated.button .hidden.content{position:absolute;width:100%}.ui.animated.button .hidden.content,.ui.animated.button .visible.content{-webkit-transition:right .3s ease 0s;transition:right .3s ease 0s}.ui.animated.button .visible.content{left:auto;right:0}.ui.animated.button .hidden.content{top:50%;left:auto;right:-100%;margin-top:-.5em}.ui.animated.button:focus .visible.content,.ui.animated.button:hover .visible.content{left:auto;right:200%}.ui.animated.button:focus .hidden.content,.ui.animated.button:hover .hidden.content{left:auto;right:0}.ui.vertical.animated.button .hidden.content,.ui.vertical.animated.button .visible.content{-webkit-transition:top .3s ease,-webkit-transform .3s ease;transition:top .3s ease,-webkit-transform .3s ease;transition:top .3s ease,transform .3s ease;transition:top .3s ease,transform .3s ease,-webkit-transform .3s ease}.ui.vertical.animated.button .visible.content{-webkit-transform:translateY(0);transform:translateY(0);right:auto}.ui.vertical.animated.button .hidden.content{top:-50%;left:0;right:auto}.ui.vertical.animated.button:focus .visible.content,.ui.vertical.animated.button:hover .visible.content{-webkit-transform:translateY(200%);transform:translateY(200%);right:auto}.ui.vertical.animated.button:focus .hidden.content,.ui.vertical.animated.button:hover .hidden.content{top:50%;right:auto}.ui.fade.animated.button .hidden.content,.ui.fade.animated.button .visible.content{-webkit-transition:opacity .3s ease,-webkit-transform .3s ease;transition:opacity .3s ease,-webkit-transform .3s ease;transition:opacity .3s ease,transform .3s ease;transition:opacity .3s ease,transform .3s ease,-webkit-transform .3s ease}.ui.fade.animated.button .visible.content{left:auto;right:auto;opacity:1;-webkit-transform:scale(1);transform:scale(1)}.ui.fade.animated.button .hidden.content{opacity:0;left:0;right:auto;-webkit-transform:scale(1.5);transform:scale(1.5)}.ui.fade.animated.button:focus .visible.content,.ui.fade.animated.button:hover .visible.content{left:auto;right:auto;opacity:0;-webkit-transform:scale(.75);transform:scale(.75)}.ui.fade.animated.button:focus .hidden.content,.ui.fade.animated.button:hover .hidden.content{left:0;right:auto;opacity:1;-webkit-transform:scale(1);transform:scale(1)}.ui.inverted.button{-webkit-box-shadow:0 0 0 2px #fff inset!important;box-shadow:0 0 0 2px #fff inset!important;background:transparent none;color:#fff;text-shadow:none!important}.ui.inverted.buttons .button{margin:0 0 0 -2px}.ui.inverted.buttons .button:first-child{margin-left:0}.ui.inverted.vertical.buttons .button{margin:0 0 -2px 0}.ui.inverted.vertical.buttons .button:first-child{margin-top:0}.ui.inverted.button:hover{background:#fff;-webkit-box-shadow:0 0 0 2px #fff inset!important;box-shadow:0 0 0 2px #fff inset!important;color:rgba(0,0,0,.8)}.ui.inverted.button.active,.ui.inverted.button:focus{background:#fff;-webkit-box-shadow:0 0 0 2px #fff inset!important;box-shadow:0 0 0 2px #fff inset!important;color:rgba(0,0,0,.8)}.ui.inverted.button.active:focus{background:#dcddde;-webkit-box-shadow:0 0 0 2px #dcddde inset!important;box-shadow:0 0 0 2px #dcddde inset!important;color:rgba(0,0,0,.8)}.ui.labeled.button:not(.icon){display:-webkit-inline-box;display:-ms-inline-flexbox;display:inline-flex;-webkit-box-orient:horizontal;-webkit-box-direction:normal;-ms-flex-direction:row;flex-direction:row;background:0 0!important;padding:0!important;border:none!important;-webkit-box-shadow:none!important;box-shadow:none!important}.ui.labeled.button>.button{margin:0}.ui.labeled.button>.label{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-align:center;-ms-flex-align:center;align-items:center;margin:0 0 0 -1px!important;padding:'';font-size:1em;border-color:rgba(34,36,38,.15)}.ui.labeled.button>.tag.label:before{width:1.85em;height:1.85em}.ui.labeled.button:not([class*="left labeled"])>.button{border-top-right-radius:0;border-bottom-right-radius:0}.ui.labeled.button:not([class*="left labeled"])>.label{border-top-left-radius:0;border-bottom-left-radius:0}.ui[class*="left labeled"].button>.button{border-top-left-radius:0;border-bottom-left-radius:0}.ui[class*="left labeled"].button>.label{border-top-right-radius:0;border-bottom-right-radius:0}.ui.facebook.button{background-color:#3b5998;color:#fff;text-shadow:none;background-image:none;-webkit-box-shadow:0 0 0 0 rgba(34,36,38,.15) inset;box-shadow:0 0 0 0 rgba(34,36,38,.15) inset}.ui.facebook.button:hover{background-color:#304d8a;color:#fff;text-shadow:none}.ui.facebook.button:active{background-color:#2d4373;color:#fff;text-shadow:none}.ui.twitter.button{background-color:#55acee;color:#fff;text-shadow:none;background-image:none;-webkit-box-shadow:0 0 0 0 rgba(34,36,38,.15) inset;box-shadow:0 0 0 0 rgba(34,36,38,.15) inset}.ui.twitter.button:hover{background-color:#35a2f4;color:#fff;text-shadow:none}.ui.twitter.button:active{background-color:#2795e9;color:#fff;text-shadow:none}.ui.google.plus.button{background-color:#dd4b39;color:#fff;text-shadow:none;background-image:none;-webkit-box-shadow:0 0 0 0 rgba(34,36,38,.15) inset;box-shadow:0 0 0 0 rgba(34,36,38,.15) inset}.ui.google.plus.button:hover{background-color:#e0321c;color:#fff;text-shadow:none}.ui.google.plus.button:active{background-color:#c23321;color:#fff;text-shadow:none}.ui.linkedin.button{background-color:#1f88be;color:#fff;text-shadow:none}.ui.linkedin.button:hover{background-color:#147baf;color:#fff;text-shadow:none}.ui.linkedin.button:active{background-color:#186992;color:#fff;text-shadow:none}.ui.youtube.button{background-color:red;color:#fff;text-shadow:none;background-image:none;-webkit-box-shadow:0 0 0 0 rgba(34,36,38,.15) inset;box-shadow:0 0 0 0 rgba(34,36,38,.15) inset}.ui.youtube.button:hover{background-color:#e60000;color:#fff;text-shadow:none}.ui.youtube.button:active{background-color:#c00;color:#fff;text-shadow:none}.ui.instagram.button{background-color:#49769c;color:#fff;text-shadow:none;background-image:none;-webkit-box-shadow:0 0 0 0 rgba(34,36,38,.15) inset;box-shadow:0 0 0 0 rgba(34,36,38,.15) inset}.ui.instagram.button:hover{background-color:#3d698e;color:#fff;text-shadow:none}.ui.instagram.button:active{background-color:#395c79;color:#fff;text-shadow:none}.ui.pinterest.button{background-color:#bd081c;color:#fff;text-shadow:none;background-image:none;-webkit-box-shadow:0 0 0 0 rgba(34,36,38,.15) inset;box-shadow:0 0 0 0 rgba(34,36,38,.15) inset}.ui.pinterest.button:hover{background-color:#ac0013;color:#fff;text-shadow:none}.ui.pinterest.button:active{background-color:#8c0615;color:#fff;text-shadow:none}.ui.vk.button{background-color:#4d7198;color:#fff;background-image:none;-webkit-box-shadow:0 0 0 0 rgba(34,36,38,.15) inset;box-shadow:0 0 0 0 rgba(34,36,38,.15) inset}.ui.vk.button:hover{background-color:#41648a;color:#fff}.ui.vk.button:active{background-color:#3c5876;color:#fff}.ui.button>.icon:not(.button){height:.85714286em;opacity:.8;margin:0 .42857143em 0 -.21428571em;-webkit-transition:opacity .1s ease;transition:opacity .1s ease;vertical-align:'';color:''}.ui.button:not(.icon)>.icon:not(.button):not(.dropdown){margin:0 .42857143em 0 -.21428571em}.ui.button:not(.icon)>.right.icon:not(.button):not(.dropdown){margin:0 -.21428571em 0 .42857143em}.ui[class*="left floated"].button,.ui[class*="left floated"].buttons{float:left;margin-left:0;margin-right:.25em}.ui[class*="right floated"].button,.ui[class*="right floated"].buttons{float:right;margin-right:0;margin-left:.25em}.ui.compact.button,.ui.compact.buttons .button{padding:.58928571em 1.125em .58928571em}.ui.compact.icon.button,.ui.compact.icon.buttons .button{padding:.58928571em .58928571em .58928571em}.ui.compact.labeled.icon.button,.ui.compact.labeled.icon.buttons .button{padding:.58928571em 3.69642857em .58928571em}.ui.mini.button,.ui.mini.buttons .button,.ui.mini.buttons .or{font-size:.78571429rem}.ui.tiny.button,.ui.tiny.buttons .button,.ui.tiny.buttons .or{font-size:.85714286rem}.ui.small.button,.ui.small.buttons .button,.ui.small.buttons .or{font-size:.92857143rem}.ui.button,.ui.buttons .button,.ui.buttons .or{font-size:1rem}.ui.large.button,.ui.large.buttons .button,.ui.large.buttons .or{font-size:1.14285714rem}.ui.big.button,.ui.big.buttons .button,.ui.big.buttons .or{font-size:1.28571429rem}.ui.huge.button,.ui.huge.buttons .button,.ui.huge.buttons .or{font-size:1.42857143rem}.ui.massive.button,.ui.massive.buttons .button,.ui.massive.buttons .or{font-size:1.71428571rem}.ui.icon.button,.ui.icon.buttons .button{padding:.78571429em .78571429em .78571429em}.ui.icon.button>.icon,.ui.icon.buttons .button>.icon{opacity:.9;margin:0!important;vertical-align:top}.ui.basic.button,.ui.basic.buttons .button{background:transparent none!important;color:rgba(0,0,0,.6)!important;font-weight:400;border-radius:.28571429rem;text-transform:none;text-shadow:none!important;-webkit-box-shadow:0 0 0 1px rgba(34,36,38,.15) inset;box-shadow:0 0 0 1px rgba(34,36,38,.15) inset}.ui.basic.buttons{-webkit-box-shadow:none;box-shadow:none;border:1px solid rgba(34,36,38,.15);border-radius:.28571429rem}.ui.basic.buttons .button{border-radius:0}.ui.basic.button:hover,.ui.basic.buttons .button:hover{background:#fff!important;color:rgba(0,0,0,.8)!important;-webkit-box-shadow:0 0 0 1px rgba(34,36,38,.35) inset,0 0 0 0 rgba(34,36,38,.15) inset;box-shadow:0 0 0 1px rgba(34,36,38,.35) inset,0 0 0 0 rgba(34,36,38,.15) inset}.ui.basic.button:focus,.ui.basic.buttons .button:focus{background:#fff!important;color:rgba(0,0,0,.8)!important;-webkit-box-shadow:0 0 0 1px rgba(34,36,38,.35) inset,0 0 0 0 rgba(34,36,38,.15) inset;box-shadow:0 0 0 1px rgba(34,36,38,.35) inset,0 0 0 0 rgba(34,36,38,.15) inset}.ui.basic.button:active,.ui.basic.buttons .button:active{background:#f8f8f8!important;color:rgba(0,0,0,.9)!important;-webkit-box-shadow:0 0 0 1px rgba(0,0,0,.15) inset,0 1px 4px 0 rgba(34,36,38,.15) inset;box-shadow:0 0 0 1px rgba(0,0,0,.15) inset,0 1px 4px 0 rgba(34,36,38,.15) inset}.ui.basic.active.button,.ui.basic.buttons .active.button{background:rgba(0,0,0,.05)!important;-webkit-box-shadow:''!important;box-shadow:''!important;color:rgba(0,0,0,.95)!important}.ui.basic.active.button:hover,.ui.basic.buttons .active.button:hover{background-color:rgba(0,0,0,.05)}.ui.basic.buttons .button:hover{-webkit-box-shadow:0 0 0 1px rgba(34,36,38,.35) inset,0 0 0 0 rgba(34,36,38,.15) inset inset;box-shadow:0 0 0 1px rgba(34,36,38,.35) inset,0 0 0 0 rgba(34,36,38,.15) inset inset}.ui.basic.buttons .button:active{-webkit-box-shadow:0 0 0 1px rgba(0,0,0,.15) inset,0 1px 4px 0 rgba(34,36,38,.15) inset inset;box-shadow:0 0 0 1px rgba(0,0,0,.15) inset,0 1px 4px 0 rgba(34,36,38,.15) inset inset}.ui.basic.buttons .active.button{-webkit-box-shadow:''!important;box-shadow:''!important}.ui.basic.inverted.button,.ui.basic.inverted.buttons .button{background-color:transparent!important;color:#f9fafb!important;-webkit-box-shadow:0 0 0 2px rgba(255,255,255,.5) inset!important;box-shadow:0 0 0 2px rgba(255,255,255,.5) inset!important}.ui.basic.inverted.button:hover,.ui.basic.inverted.buttons .button:hover{color:#fff!important;-webkit-box-shadow:0 0 0 2px #fff inset!important;box-shadow:0 0 0 2px #fff inset!important}.ui.basic.inverted.button:focus,.ui.basic.inverted.buttons .button:focus{color:#fff!important;-webkit-box-shadow:0 0 0 2px #fff inset!important;box-shadow:0 0 0 2px #fff inset!important}.ui.basic.inverted.button:active,.ui.basic.inverted.buttons .button:active{background-color:rgba(255,255,255,.08)!important;color:#fff!important;-webkit-box-shadow:0 0 0 2px rgba(255,255,255,.9) inset!important;box-shadow:0 0 0 2px rgba(255,255,255,.9) inset!important}.ui.basic.inverted.active.button,.ui.basic.inverted.buttons .active.button{background-color:rgba(255,255,255,.08);color:#fff;text-shadow:none;-webkit-box-shadow:0 0 0 2px rgba(255,255,255,.7) inset;box-shadow:0 0 0 2px rgba(255,255,255,.7) inset}.ui.basic.inverted.active.button:hover,.ui.basic.inverted.buttons .active.button:hover{background-color:rgba(255,255,255,.15);-webkit-box-shadow:0 0 0 2px #fff inset!important;box-shadow:0 0 0 2px #fff inset!important}.ui.basic.buttons .button{border-left:1px solid rgba(34,36,38,.15);-webkit-box-shadow:none;box-shadow:none}.ui.basic.vertical.buttons .button{border-left:none}.ui.basic.vertical.buttons .button{border-left-width:0;border-top:1px solid rgba(34,36,38,.15)}.ui.basic.vertical.buttons .button:first-child{border-top-width:0}.ui.labeled.icon.button,.ui.labeled.icon.buttons .button{position:relative;padding-left:4.07142857em!important;padding-right:1.5em!important}.ui.labeled.icon.button>.icon,.ui.labeled.icon.buttons>.button>.icon{position:absolute;height:100%;line-height:1;border-radius:0;border-top-left-radius:inherit;border-bottom-left-radius:inherit;text-align:center;margin:0;width:2.57142857em;background-color:rgba(0,0,0,.05);color:'';-webkit-box-shadow:-1px 0 0 0 transparent inset;box-shadow:-1px 0 0 0 transparent inset}.ui.labeled.icon.button>.icon,.ui.labeled.icon.buttons>.button>.icon{top:0;left:0}.ui[class*="right labeled"].icon.button{padding-right:4.07142857em!important;padding-left:1.5em!important}.ui[class*="right labeled"].icon.button>.icon{left:auto;right:0;border-radius:0;border-top-right-radius:inherit;border-bottom-right-radius:inherit;-webkit-box-shadow:1px 0 0 0 transparent inset;box-shadow:1px 0 0 0 transparent inset}.ui.labeled.icon.button>.icon:after,.ui.labeled.icon.button>.icon:before,.ui.labeled.icon.buttons>.button>.icon:after,.ui.labeled.icon.buttons>.button>.icon:before{display:block;position:absolute;width:100%;top:50%;text-align:center;-webkit-transform:translateY(-50%);transform:translateY(-50%)}.ui.labeled.icon.buttons .button>.icon{border-radius:0}.ui.labeled.icon.buttons .button:first-child>.icon{border-top-left-radius:.28571429rem;border-bottom-left-radius:.28571429rem}.ui.labeled.icon.buttons .button:last-child>.icon{border-top-right-radius:.28571429rem;border-bottom-right-radius:.28571429rem}.ui.vertical.labeled.icon.buttons .button:first-child>.icon{border-radius:0;border-top-left-radius:.28571429rem}.ui.vertical.labeled.icon.buttons .button:last-child>.icon{border-radius:0;border-bottom-left-radius:.28571429rem}.ui.fluid[class*="left labeled"].icon.button,.ui.fluid[class*="right labeled"].icon.button{padding-left:1.5em!important;padding-right:1.5em!important}.ui.button.toggle.active,.ui.buttons .button.toggle.active,.ui.toggle.buttons .active.button{background-color:#21ba45!important;-webkit-box-shadow:none!important;box-shadow:none!important;text-shadow:none;color:#fff!important}.ui.button.toggle.active:hover{background-color:#16ab39!important;text-shadow:none;color:#fff!important}.ui.circular.button{border-radius:10em}.ui.circular.button>.icon{width:1em;vertical-align:baseline}.ui.buttons .or{position:relative;width:.3em;height:2.57142857em;z-index:3}.ui.buttons .or:before{position:absolute;text-align:center;border-radius:500rem;content:'or';top:50%;left:50%;background-color:#fff;text-shadow:none;margin-top:-.89285714em;margin-left:-.89285714em;width:1.78571429em;height:1.78571429em;line-height:1.78571429em;color:rgba(0,0,0,.4);font-style:normal;font-weight:700;-webkit-box-shadow:0 0 0 1px transparent inset;box-shadow:0 0 0 1px transparent inset}.ui.buttons .or[data-text]:before{content:attr(data-text)}.ui.fluid.buttons .or{width:0!important}.ui.fluid.buttons .or:after{display:none}.ui.attached.button{position:relative;display:block;margin:0;border-radius:0;-webkit-box-shadow:0 0 0 1px rgba(34,36,38,.15)!important;box-shadow:0 0 0 1px rgba(34,36,38,.15)!important}.ui.attached.top.button{border-radius:.28571429rem .28571429rem 0 0}.ui.attached.bottom.button{border-radius:0 0 .28571429rem .28571429rem}.ui.left.attached.button{display:inline-block;border-left:none;text-align:right;padding-right:.75em;border-radius:.28571429rem 0 0 .28571429rem}.ui.right.attached.button{display:inline-block;text-align:left;padding-left:.75em;border-radius:0 .28571429rem .28571429rem 0}.ui.attached.buttons{position:relative;display:-webkit-box;display:-ms-flexbox;display:flex;border-radius:0;width:auto!important;z-index:2;margin-left:-1px;margin-right:-1px}.ui.attached.buttons .button{margin:0}.ui.attached.buttons .button:first-child{border-radius:0}.ui.attached.buttons .button:last-child{border-radius:0}.ui[class*="top attached"].buttons{margin-bottom:-1px;border-radius:.28571429rem .28571429rem 0 0}.ui[class*="top attached"].buttons .button:first-child{border-radius:.28571429rem 0 0 0}.ui[class*="top attached"].buttons .button:last-child{border-radius:0 .28571429rem 0 0}.ui[class*="bottom attached"].buttons{margin-top:-1px;border-radius:0 0 .28571429rem .28571429rem}.ui[class*="bottom attached"].buttons .button:first-child{border-radius:0 0 0 .28571429rem}.ui[class*="bottom attached"].buttons .button:last-child{border-radius:0 0 .28571429rem 0}.ui[class*="left attached"].buttons{display:-webkit-inline-box;display:-ms-inline-flexbox;display:inline-flex;margin-right:0;margin-left:-1px;border-radius:0 .28571429rem .28571429rem 0}.ui[class*="left attached"].buttons .button:first-child{margin-left:-1px;border-radius:0 .28571429rem 0 0}.ui[class*="left attached"].buttons .button:last-child{margin-left:-1px;border-radius:0 0 .28571429rem 0}.ui[class*="right attached"].buttons{display:-webkit-inline-box;display:-ms-inline-flexbox;display:inline-flex;margin-left:0;margin-right:-1px;border-radius:.28571429rem 0 0 .28571429rem}.ui[class*="right attached"].buttons .button:first-child{margin-left:-1px;border-radius:.28571429rem 0 0 0}.ui[class*="right attached"].buttons .button:last-child{margin-left:-1px;border-radius:0 0 0 .28571429rem}.ui.fluid.button,.ui.fluid.buttons{width:100%}.ui.fluid.button{display:block}.ui.two.buttons{width:100%}.ui.two.buttons>.button{width:50%}.ui.three.buttons{width:100%}.ui.three.buttons>.button{width:33.333%}.ui.four.buttons{width:100%}.ui.four.buttons>.button{width:25%}.ui.five.buttons{width:100%}.ui.five.buttons>.button{width:20%}.ui.six.buttons{width:100%}.ui.six.buttons>.button{width:16.666%}.ui.seven.buttons{width:100%}.ui.seven.buttons>.button{width:14.285%}.ui.eight.buttons{width:100%}.ui.eight.buttons>.button{width:12.5%}.ui.nine.buttons{width:100%}.ui.nine.buttons>.button{width:11.11%}.ui.ten.buttons{width:100%}.ui.ten.buttons>.button{width:10%}.ui.eleven.buttons{width:100%}.ui.eleven.buttons>.button{width:9.09%}.ui.twelve.buttons{width:100%}.ui.twelve.buttons>.button{width:8.3333%}.ui.fluid.vertical.buttons,.ui.fluid.vertical.buttons>.button{display:-webkit-box;display:-ms-flexbox;display:flex;width:auto}.ui.two.vertical.buttons>.button{height:50%}.ui.three.vertical.buttons>.button{height:33.333%}.ui.four.vertical.buttons>.button{height:25%}.ui.five.vertical.buttons>.button{height:20%}.ui.six.vertical.buttons>.button{height:16.666%}.ui.seven.vertical.buttons>.button{height:14.285%}.ui.eight.vertical.buttons>.button{height:12.5%}.ui.nine.vertical.buttons>.button{height:11.11%}.ui.ten.vertical.buttons>.button{height:10%}.ui.eleven.vertical.buttons>.button{height:9.09%}.ui.twelve.vertical.buttons>.button{height:8.3333%}.ui.black.button,.ui.black.buttons .button{background-color:#1b1c1d;color:#fff;text-shadow:none;background-image:none}.ui.black.button{-webkit-box-shadow:0 0 0 0 rgba(34,36,38,.15) inset;box-shadow:0 0 0 0 rgba(34,36,38,.15) inset}.ui.black.button:hover,.ui.black.buttons .button:hover{background-color:#27292a;color:#fff;text-shadow:none}.ui.black.button:focus,.ui.black.buttons .button:focus{background-color:#2f3032;color:#fff;text-shadow:none}.ui.black.button:active,.ui.black.buttons .button:active{background-color:#343637;color:#fff;text-shadow:none}.ui.black.active.button,.ui.black.button .active.button:active,.ui.black.buttons .active.button,.ui.black.buttons .active.button:active{background-color:#0f0f10;color:#fff;text-shadow:none}.ui.basic.black.button,.ui.basic.black.buttons .button{-webkit-box-shadow:0 0 0 1px #1b1c1d inset!important;box-shadow:0 0 0 1px #1b1c1d inset!important;color:#1b1c1d!important}.ui.basic.black.button:hover,.ui.basic.black.buttons .button:hover{background:0 0!important;-webkit-box-shadow:0 0 0 1px #27292a inset!important;box-shadow:0 0 0 1px #27292a inset!important;color:#27292a!important}.ui.basic.black.button:focus,.ui.basic.black.buttons .button:focus{background:0 0!important;-webkit-box-shadow:0 0 0 1px #2f3032 inset!important;box-shadow:0 0 0 1px #2f3032 inset!important;color:#27292a!important}.ui.basic.black.active.button,.ui.basic.black.buttons .active.button{background:0 0!important;-webkit-box-shadow:0 0 0 1px #0f0f10 inset!important;box-shadow:0 0 0 1px #0f0f10 inset!important;color:#343637!important}.ui.basic.black.button:active,.ui.basic.black.buttons .button:active{-webkit-box-shadow:0 0 0 1px #343637 inset!important;box-shadow:0 0 0 1px #343637 inset!important;color:#343637!important}.ui.buttons:not(.vertical)>.basic.black.button:not(:first-child){margin-left:-1px}.ui.inverted.black.button,.ui.inverted.black.buttons .button{background-color:transparent;-webkit-box-shadow:0 0 0 2px #d4d4d5 inset!important;box-shadow:0 0 0 2px #d4d4d5 inset!important;color:#fff}.ui.inverted.black.button.active,.ui.inverted.black.button:active,.ui.inverted.black.button:focus,.ui.inverted.black.button:hover,.ui.inverted.black.buttons .button.active,.ui.inverted.black.buttons .button:active,.ui.inverted.black.buttons .button:focus,.ui.inverted.black.buttons .button:hover{-webkit-box-shadow:none!important;box-shadow:none!important;color:#fff}.ui.inverted.black.button:hover,.ui.inverted.black.buttons .button:hover{background-color:#000}.ui.inverted.black.button:focus,.ui.inverted.black.buttons .button:focus{background-color:#000}.ui.inverted.black.active.button,.ui.inverted.black.buttons .active.button{background-color:#000}.ui.inverted.black.button:active,.ui.inverted.black.buttons .button:active{background-color:#000}.ui.inverted.black.basic.button,.ui.inverted.black.basic.buttons .button,.ui.inverted.black.buttons .basic.button{background-color:transparent;-webkit-box-shadow:0 0 0 2px rgba(255,255,255,.5) inset!important;box-shadow:0 0 0 2px rgba(255,255,255,.5) inset!important;color:#fff!important}.ui.inverted.black.basic.button:hover,.ui.inverted.black.basic.buttons .button:hover,.ui.inverted.black.buttons .basic.button:hover{-webkit-box-shadow:0 0 0 2px #000 inset!important;box-shadow:0 0 0 2px #000 inset!important;color:#fff!important}.ui.inverted.black.basic.button:focus,.ui.inverted.black.basic.buttons .button:focus{-webkit-box-shadow:0 0 0 2px #000 inset!important;box-shadow:0 0 0 2px #000 inset!important;color:#545454!important}.ui.inverted.black.basic.active.button,.ui.inverted.black.basic.buttons .active.button,.ui.inverted.black.buttons .basic.active.button{-webkit-box-shadow:0 0 0 2px #000 inset!important;box-shadow:0 0 0 2px #000 inset!important;color:#fff!important}.ui.inverted.black.basic.button:active,.ui.inverted.black.basic.buttons .button:active,.ui.inverted.black.buttons .basic.button:active{-webkit-box-shadow:0 0 0 2px #000 inset!important;box-shadow:0 0 0 2px #000 inset!important;color:#fff!important}.ui.grey.button,.ui.grey.buttons .button{background-color:#767676;color:#fff;text-shadow:none;background-image:none}.ui.grey.button{-webkit-box-shadow:0 0 0 0 rgba(34,36,38,.15) inset;box-shadow:0 0 0 0 rgba(34,36,38,.15) inset}.ui.grey.button:hover,.ui.grey.buttons .button:hover{background-color:#838383;color:#fff;text-shadow:none}.ui.grey.button:focus,.ui.grey.buttons .button:focus{background-color:#8a8a8a;color:#fff;text-shadow:none}.ui.grey.button:active,.ui.grey.buttons .button:active{background-color:#909090;color:#fff;text-shadow:none}.ui.grey.active.button,.ui.grey.button .active.button:active,.ui.grey.buttons .active.button,.ui.grey.buttons .active.button:active{background-color:#696969;color:#fff;text-shadow:none}.ui.basic.grey.button,.ui.basic.grey.buttons .button{-webkit-box-shadow:0 0 0 1px #767676 inset!important;box-shadow:0 0 0 1px #767676 inset!important;color:#767676!important}.ui.basic.grey.button:hover,.ui.basic.grey.buttons .button:hover{background:0 0!important;-webkit-box-shadow:0 0 0 1px #838383 inset!important;box-shadow:0 0 0 1px #838383 inset!important;color:#838383!important}.ui.basic.grey.button:focus,.ui.basic.grey.buttons .button:focus{background:0 0!important;-webkit-box-shadow:0 0 0 1px #8a8a8a inset!important;box-shadow:0 0 0 1px #8a8a8a inset!important;color:#838383!important}.ui.basic.grey.active.button,.ui.basic.grey.buttons .active.button{background:0 0!important;-webkit-box-shadow:0 0 0 1px #696969 inset!important;box-shadow:0 0 0 1px #696969 inset!important;color:#909090!important}.ui.basic.grey.button:active,.ui.basic.grey.buttons .button:active{-webkit-box-shadow:0 0 0 1px #909090 inset!important;box-shadow:0 0 0 1px #909090 inset!important;color:#909090!important}.ui.buttons:not(.vertical)>.basic.grey.button:not(:first-child){margin-left:-1px}.ui.inverted.grey.button,.ui.inverted.grey.buttons .button{background-color:transparent;-webkit-box-shadow:0 0 0 2px #d4d4d5 inset!important;box-shadow:0 0 0 2px #d4d4d5 inset!important;color:#fff}.ui.inverted.grey.button.active,.ui.inverted.grey.button:active,.ui.inverted.grey.button:focus,.ui.inverted.grey.button:hover,.ui.inverted.grey.buttons .button.active,.ui.inverted.grey.buttons .button:active,.ui.inverted.grey.buttons .button:focus,.ui.inverted.grey.buttons .button:hover{-webkit-box-shadow:none!important;box-shadow:none!important;color:rgba(0,0,0,.6)}.ui.inverted.grey.button:hover,.ui.inverted.grey.buttons .button:hover{background-color:#cfd0d2}.ui.inverted.grey.button:focus,.ui.inverted.grey.buttons .button:focus{background-color:#c7c9cb}.ui.inverted.grey.active.button,.ui.inverted.grey.buttons .active.button{background-color:#cfd0d2}.ui.inverted.grey.button:active,.ui.inverted.grey.buttons .button:active{background-color:#c2c4c5}.ui.inverted.grey.basic.button,.ui.inverted.grey.basic.buttons .button,.ui.inverted.grey.buttons .basic.button{background-color:transparent;-webkit-box-shadow:0 0 0 2px rgba(255,255,255,.5) inset!important;box-shadow:0 0 0 2px rgba(255,255,255,.5) inset!important;color:#fff!important}.ui.inverted.grey.basic.button:hover,.ui.inverted.grey.basic.buttons .button:hover,.ui.inverted.grey.buttons .basic.button:hover{-webkit-box-shadow:0 0 0 2px #cfd0d2 inset!important;box-shadow:0 0 0 2px #cfd0d2 inset!important;color:#fff!important}.ui.inverted.grey.basic.button:focus,.ui.inverted.grey.basic.buttons .button:focus{-webkit-box-shadow:0 0 0 2px #c7c9cb inset!important;box-shadow:0 0 0 2px #c7c9cb inset!important;color:#dcddde!important}.ui.inverted.grey.basic.active.button,.ui.inverted.grey.basic.buttons .active.button,.ui.inverted.grey.buttons .basic.active.button{-webkit-box-shadow:0 0 0 2px #cfd0d2 inset!important;box-shadow:0 0 0 2px #cfd0d2 inset!important;color:#fff!important}.ui.inverted.grey.basic.button:active,.ui.inverted.grey.basic.buttons .button:active,.ui.inverted.grey.buttons .basic.button:active{-webkit-box-shadow:0 0 0 2px #c2c4c5 inset!important;box-shadow:0 0 0 2px #c2c4c5 inset!important;color:#fff!important}.ui.brown.button,.ui.brown.buttons .button{background-color:#a5673f;color:#fff;text-shadow:none;background-image:none}.ui.brown.button{-webkit-box-shadow:0 0 0 0 rgba(34,36,38,.15) inset;box-shadow:0 0 0 0 rgba(34,36,38,.15) inset}.ui.brown.button:hover,.ui.brown.buttons .button:hover{background-color:#975b33;color:#fff;text-shadow:none}.ui.brown.button:focus,.ui.brown.buttons .button:focus{background-color:#90532b;color:#fff;text-shadow:none}.ui.brown.button:active,.ui.brown.buttons .button:active{background-color:#805031;color:#fff;text-shadow:none}.ui.brown.active.button,.ui.brown.button .active.button:active,.ui.brown.buttons .active.button,.ui.brown.buttons .active.button:active{background-color:#995a31;color:#fff;text-shadow:none}.ui.basic.brown.button,.ui.basic.brown.buttons .button{-webkit-box-shadow:0 0 0 1px #a5673f inset!important;box-shadow:0 0 0 1px #a5673f inset!important;color:#a5673f!important}.ui.basic.brown.button:hover,.ui.basic.brown.buttons .button:hover{background:0 0!important;-webkit-box-shadow:0 0 0 1px #975b33 inset!important;box-shadow:0 0 0 1px #975b33 inset!important;color:#975b33!important}.ui.basic.brown.button:focus,.ui.basic.brown.buttons .button:focus{background:0 0!important;-webkit-box-shadow:0 0 0 1px #90532b inset!important;box-shadow:0 0 0 1px #90532b inset!important;color:#975b33!important}.ui.basic.brown.active.button,.ui.basic.brown.buttons .active.button{background:0 0!important;-webkit-box-shadow:0 0 0 1px #995a31 inset!important;box-shadow:0 0 0 1px #995a31 inset!important;color:#805031!important}.ui.basic.brown.button:active,.ui.basic.brown.buttons .button:active{-webkit-box-shadow:0 0 0 1px #805031 inset!important;box-shadow:0 0 0 1px #805031 inset!important;color:#805031!important}.ui.buttons:not(.vertical)>.basic.brown.button:not(:first-child){margin-left:-1px}.ui.inverted.brown.button,.ui.inverted.brown.buttons .button{background-color:transparent;-webkit-box-shadow:0 0 0 2px #d67c1c inset!important;box-shadow:0 0 0 2px #d67c1c inset!important;color:#d67c1c}.ui.inverted.brown.button.active,.ui.inverted.brown.button:active,.ui.inverted.brown.button:focus,.ui.inverted.brown.button:hover,.ui.inverted.brown.buttons .button.active,.ui.inverted.brown.buttons .button:active,.ui.inverted.brown.buttons .button:focus,.ui.inverted.brown.buttons .button:hover{-webkit-box-shadow:none!important;box-shadow:none!important;color:#fff}.ui.inverted.brown.button:hover,.ui.inverted.brown.buttons .button:hover{background-color:#c86f11}.ui.inverted.brown.button:focus,.ui.inverted.brown.buttons .button:focus{background-color:#c16808}.ui.inverted.brown.active.button,.ui.inverted.brown.buttons .active.button{background-color:#cc6f0d}.ui.inverted.brown.button:active,.ui.inverted.brown.buttons .button:active{background-color:#a96216}.ui.inverted.brown.basic.button,.ui.inverted.brown.basic.buttons .button,.ui.inverted.brown.buttons .basic.button{background-color:transparent;-webkit-box-shadow:0 0 0 2px rgba(255,255,255,.5) inset!important;box-shadow:0 0 0 2px rgba(255,255,255,.5) inset!important;color:#fff!important}.ui.inverted.brown.basic.button:hover,.ui.inverted.brown.basic.buttons .button:hover,.ui.inverted.brown.buttons .basic.button:hover{-webkit-box-shadow:0 0 0 2px #c86f11 inset!important;box-shadow:0 0 0 2px #c86f11 inset!important;color:#d67c1c!important}.ui.inverted.brown.basic.button:focus,.ui.inverted.brown.basic.buttons .button:focus{-webkit-box-shadow:0 0 0 2px #c16808 inset!important;box-shadow:0 0 0 2px #c16808 inset!important;color:#d67c1c!important}.ui.inverted.brown.basic.active.button,.ui.inverted.brown.basic.buttons .active.button,.ui.inverted.brown.buttons .basic.active.button{-webkit-box-shadow:0 0 0 2px #cc6f0d inset!important;box-shadow:0 0 0 2px #cc6f0d inset!important;color:#d67c1c!important}.ui.inverted.brown.basic.button:active,.ui.inverted.brown.basic.buttons .button:active,.ui.inverted.brown.buttons .basic.button:active{-webkit-box-shadow:0 0 0 2px #a96216 inset!important;box-shadow:0 0 0 2px #a96216 inset!important;color:#d67c1c!important}.ui.blue.button,.ui.blue.buttons .button{background-color:#2185d0;color:#fff;text-shadow:none;background-image:none}.ui.blue.button{-webkit-box-shadow:0 0 0 0 rgba(34,36,38,.15) inset;box-shadow:0 0 0 0 rgba(34,36,38,.15) inset}.ui.blue.button:hover,.ui.blue.buttons .button:hover{background-color:#1678c2;color:#fff;text-shadow:none}.ui.blue.button:focus,.ui.blue.buttons .button:focus{background-color:#0d71bb;color:#fff;text-shadow:none}.ui.blue.button:active,.ui.blue.buttons .button:active{background-color:#1a69a4;color:#fff;text-shadow:none}.ui.blue.active.button,.ui.blue.button .active.button:active,.ui.blue.buttons .active.button,.ui.blue.buttons .active.button:active{background-color:#1279c6;color:#fff;text-shadow:none}.ui.basic.blue.button,.ui.basic.blue.buttons .button{-webkit-box-shadow:0 0 0 1px #2185d0 inset!important;box-shadow:0 0 0 1px #2185d0 inset!important;color:#2185d0!important}.ui.basic.blue.button:hover,.ui.basic.blue.buttons .button:hover{background:0 0!important;-webkit-box-shadow:0 0 0 1px #1678c2 inset!important;box-shadow:0 0 0 1px #1678c2 inset!important;color:#1678c2!important}.ui.basic.blue.button:focus,.ui.basic.blue.buttons .button:focus{background:0 0!important;-webkit-box-shadow:0 0 0 1px #0d71bb inset!important;box-shadow:0 0 0 1px #0d71bb inset!important;color:#1678c2!important}.ui.basic.blue.active.button,.ui.basic.blue.buttons .active.button{background:0 0!important;-webkit-box-shadow:0 0 0 1px #1279c6 inset!important;box-shadow:0 0 0 1px #1279c6 inset!important;color:#1a69a4!important}.ui.basic.blue.button:active,.ui.basic.blue.buttons .button:active{-webkit-box-shadow:0 0 0 1px #1a69a4 inset!important;box-shadow:0 0 0 1px #1a69a4 inset!important;color:#1a69a4!important}.ui.buttons:not(.vertical)>.basic.blue.button:not(:first-child){margin-left:-1px}.ui.inverted.blue.button,.ui.inverted.blue.buttons .button{background-color:transparent;-webkit-box-shadow:0 0 0 2px #54c8ff inset!important;box-shadow:0 0 0 2px #54c8ff inset!important;color:#54c8ff}.ui.inverted.blue.button.active,.ui.inverted.blue.button:active,.ui.inverted.blue.button:focus,.ui.inverted.blue.button:hover,.ui.inverted.blue.buttons .button.active,.ui.inverted.blue.buttons .button:active,.ui.inverted.blue.buttons .button:focus,.ui.inverted.blue.buttons .button:hover{-webkit-box-shadow:none!important;box-shadow:none!important;color:#fff}.ui.inverted.blue.button:hover,.ui.inverted.blue.buttons .button:hover{background-color:#3ac0ff}.ui.inverted.blue.button:focus,.ui.inverted.blue.buttons .button:focus{background-color:#2bbbff}.ui.inverted.blue.active.button,.ui.inverted.blue.buttons .active.button{background-color:#3ac0ff}.ui.inverted.blue.button:active,.ui.inverted.blue.buttons .button:active{background-color:#21b8ff}.ui.inverted.blue.basic.button,.ui.inverted.blue.basic.buttons .button,.ui.inverted.blue.buttons .basic.button{background-color:transparent;-webkit-box-shadow:0 0 0 2px rgba(255,255,255,.5) inset!important;box-shadow:0 0 0 2px rgba(255,255,255,.5) inset!important;color:#fff!important}.ui.inverted.blue.basic.button:hover,.ui.inverted.blue.basic.buttons .button:hover,.ui.inverted.blue.buttons .basic.button:hover{-webkit-box-shadow:0 0 0 2px #3ac0ff inset!important;box-shadow:0 0 0 2px #3ac0ff inset!important;color:#54c8ff!important}.ui.inverted.blue.basic.button:focus,.ui.inverted.blue.basic.buttons .button:focus{-webkit-box-shadow:0 0 0 2px #2bbbff inset!important;box-shadow:0 0 0 2px #2bbbff inset!important;color:#54c8ff!important}.ui.inverted.blue.basic.active.button,.ui.inverted.blue.basic.buttons .active.button,.ui.inverted.blue.buttons .basic.active.button{-webkit-box-shadow:0 0 0 2px #3ac0ff inset!important;box-shadow:0 0 0 2px #3ac0ff inset!important;color:#54c8ff!important}.ui.inverted.blue.basic.button:active,.ui.inverted.blue.basic.buttons .button:active,.ui.inverted.blue.buttons .basic.button:active{-webkit-box-shadow:0 0 0 2px #21b8ff inset!important;box-shadow:0 0 0 2px #21b8ff inset!important;color:#54c8ff!important}.ui.green.button,.ui.green.buttons .button{background-color:#21ba45;color:#fff;text-shadow:none;background-image:none}.ui.green.button{-webkit-box-shadow:0 0 0 0 rgba(34,36,38,.15) inset;box-shadow:0 0 0 0 rgba(34,36,38,.15) inset}.ui.green.button:hover,.ui.green.buttons .button:hover{background-color:#16ab39;color:#fff;text-shadow:none}.ui.green.button:focus,.ui.green.buttons .button:focus{background-color:#0ea432;color:#fff;text-shadow:none}.ui.green.button:active,.ui.green.buttons .button:active{background-color:#198f35;color:#fff;text-shadow:none}.ui.green.active.button,.ui.green.button .active.button:active,.ui.green.buttons .active.button,.ui.green.buttons .active.button:active{background-color:#13ae38;color:#fff;text-shadow:none}.ui.basic.green.button,.ui.basic.green.buttons .button{-webkit-box-shadow:0 0 0 1px #21ba45 inset!important;box-shadow:0 0 0 1px #21ba45 inset!important;color:#21ba45!important}.ui.basic.green.button:hover,.ui.basic.green.buttons .button:hover{background:0 0!important;-webkit-box-shadow:0 0 0 1px #16ab39 inset!important;box-shadow:0 0 0 1px #16ab39 inset!important;color:#16ab39!important}.ui.basic.green.button:focus,.ui.basic.green.buttons .button:focus{background:0 0!important;-webkit-box-shadow:0 0 0 1px #0ea432 inset!important;box-shadow:0 0 0 1px #0ea432 inset!important;color:#16ab39!important}.ui.basic.green.active.button,.ui.basic.green.buttons .active.button{background:0 0!important;-webkit-box-shadow:0 0 0 1px #13ae38 inset!important;box-shadow:0 0 0 1px #13ae38 inset!important;color:#198f35!important}.ui.basic.green.button:active,.ui.basic.green.buttons .button:active{-webkit-box-shadow:0 0 0 1px #198f35 inset!important;box-shadow:0 0 0 1px #198f35 inset!important;color:#198f35!important}.ui.buttons:not(.vertical)>.basic.green.button:not(:first-child){margin-left:-1px}.ui.inverted.green.button,.ui.inverted.green.buttons .button{background-color:transparent;-webkit-box-shadow:0 0 0 2px #2ecc40 inset!important;box-shadow:0 0 0 2px #2ecc40 inset!important;color:#2ecc40}.ui.inverted.green.button.active,.ui.inverted.green.button:active,.ui.inverted.green.button:focus,.ui.inverted.green.button:hover,.ui.inverted.green.buttons .button.active,.ui.inverted.green.buttons .button:active,.ui.inverted.green.buttons .button:focus,.ui.inverted.green.buttons .button:hover{-webkit-box-shadow:none!important;box-shadow:none!important;color:#fff}.ui.inverted.green.button:hover,.ui.inverted.green.buttons .button:hover{background-color:#22be34}.ui.inverted.green.button:focus,.ui.inverted.green.buttons .button:focus{background-color:#19b82b}.ui.inverted.green.active.button,.ui.inverted.green.buttons .active.button{background-color:#1fc231}.ui.inverted.green.button:active,.ui.inverted.green.buttons .button:active{background-color:#25a233}.ui.inverted.green.basic.button,.ui.inverted.green.basic.buttons .button,.ui.inverted.green.buttons .basic.button{background-color:transparent;-webkit-box-shadow:0 0 0 2px rgba(255,255,255,.5) inset!important;box-shadow:0 0 0 2px rgba(255,255,255,.5) inset!important;color:#fff!important}.ui.inverted.green.basic.button:hover,.ui.inverted.green.basic.buttons .button:hover,.ui.inverted.green.buttons .basic.button:hover{-webkit-box-shadow:0 0 0 2px #22be34 inset!important;box-shadow:0 0 0 2px #22be34 inset!important;color:#2ecc40!important}.ui.inverted.green.basic.button:focus,.ui.inverted.green.basic.buttons .button:focus{-webkit-box-shadow:0 0 0 2px #19b82b inset!important;box-shadow:0 0 0 2px #19b82b inset!important;color:#2ecc40!important}.ui.inverted.green.basic.active.button,.ui.inverted.green.basic.buttons .active.button,.ui.inverted.green.buttons .basic.active.button{-webkit-box-shadow:0 0 0 2px #1fc231 inset!important;box-shadow:0 0 0 2px #1fc231 inset!important;color:#2ecc40!important}.ui.inverted.green.basic.button:active,.ui.inverted.green.basic.buttons .button:active,.ui.inverted.green.buttons .basic.button:active{-webkit-box-shadow:0 0 0 2px #25a233 inset!important;box-shadow:0 0 0 2px #25a233 inset!important;color:#2ecc40!important}.ui.orange.button,.ui.orange.buttons .button{background-color:#f2711c;color:#fff;text-shadow:none;background-image:none}.ui.orange.button{-webkit-box-shadow:0 0 0 0 rgba(34,36,38,.15) inset;box-shadow:0 0 0 0 rgba(34,36,38,.15) inset}.ui.orange.button:hover,.ui.orange.buttons .button:hover{background-color:#f26202;color:#fff;text-shadow:none}.ui.orange.button:focus,.ui.orange.buttons .button:focus{background-color:#e55b00;color:#fff;text-shadow:none}.ui.orange.button:active,.ui.orange.buttons .button:active{background-color:#cf590c;color:#fff;text-shadow:none}.ui.orange.active.button,.ui.orange.button .active.button:active,.ui.orange.buttons .active.button,.ui.orange.buttons .active.button:active{background-color:#f56100;color:#fff;text-shadow:none}.ui.basic.orange.button,.ui.basic.orange.buttons .button{-webkit-box-shadow:0 0 0 1px #f2711c inset!important;box-shadow:0 0 0 1px #f2711c inset!important;color:#f2711c!important}.ui.basic.orange.button:hover,.ui.basic.orange.buttons .button:hover{background:0 0!important;-webkit-box-shadow:0 0 0 1px #f26202 inset!important;box-shadow:0 0 0 1px #f26202 inset!important;color:#f26202!important}.ui.basic.orange.button:focus,.ui.basic.orange.buttons .button:focus{background:0 0!important;-webkit-box-shadow:0 0 0 1px #e55b00 inset!important;box-shadow:0 0 0 1px #e55b00 inset!important;color:#f26202!important}.ui.basic.orange.active.button,.ui.basic.orange.buttons .active.button{background:0 0!important;-webkit-box-shadow:0 0 0 1px #f56100 inset!important;box-shadow:0 0 0 1px #f56100 inset!important;color:#cf590c!important}.ui.basic.orange.button:active,.ui.basic.orange.buttons .button:active{-webkit-box-shadow:0 0 0 1px #cf590c inset!important;box-shadow:0 0 0 1px #cf590c inset!important;color:#cf590c!important}.ui.buttons:not(.vertical)>.basic.orange.button:not(:first-child){margin-left:-1px}.ui.inverted.orange.button,.ui.inverted.orange.buttons .button{background-color:transparent;-webkit-box-shadow:0 0 0 2px #ff851b inset!important;box-shadow:0 0 0 2px #ff851b inset!important;color:#ff851b}.ui.inverted.orange.button.active,.ui.inverted.orange.button:active,.ui.inverted.orange.button:focus,.ui.inverted.orange.button:hover,.ui.inverted.orange.buttons .button.active,.ui.inverted.orange.buttons .button:active,.ui.inverted.orange.buttons .button:focus,.ui.inverted.orange.buttons .button:hover{-webkit-box-shadow:none!important;box-shadow:none!important;color:#fff}.ui.inverted.orange.button:hover,.ui.inverted.orange.buttons .button:hover{background-color:#ff7701}.ui.inverted.orange.button:focus,.ui.inverted.orange.buttons .button:focus{background-color:#f17000}.ui.inverted.orange.active.button,.ui.inverted.orange.buttons .active.button{background-color:#ff7701}.ui.inverted.orange.button:active,.ui.inverted.orange.buttons .button:active{background-color:#e76b00}.ui.inverted.orange.basic.button,.ui.inverted.orange.basic.buttons .button,.ui.inverted.orange.buttons .basic.button{background-color:transparent;-webkit-box-shadow:0 0 0 2px rgba(255,255,255,.5) inset!important;box-shadow:0 0 0 2px rgba(255,255,255,.5) inset!important;color:#fff!important}.ui.inverted.orange.basic.button:hover,.ui.inverted.orange.basic.buttons .button:hover,.ui.inverted.orange.buttons .basic.button:hover{-webkit-box-shadow:0 0 0 2px #ff7701 inset!important;box-shadow:0 0 0 2px #ff7701 inset!important;color:#ff851b!important}.ui.inverted.orange.basic.button:focus,.ui.inverted.orange.basic.buttons .button:focus{-webkit-box-shadow:0 0 0 2px #f17000 inset!important;box-shadow:0 0 0 2px #f17000 inset!important;color:#ff851b!important}.ui.inverted.orange.basic.active.button,.ui.inverted.orange.basic.buttons .active.button,.ui.inverted.orange.buttons .basic.active.button{-webkit-box-shadow:0 0 0 2px #ff7701 inset!important;box-shadow:0 0 0 2px #ff7701 inset!important;color:#ff851b!important}.ui.inverted.orange.basic.button:active,.ui.inverted.orange.basic.buttons .button:active,.ui.inverted.orange.buttons .basic.button:active{-webkit-box-shadow:0 0 0 2px #e76b00 inset!important;box-shadow:0 0 0 2px #e76b00 inset!important;color:#ff851b!important}.ui.pink.button,.ui.pink.buttons .button{background-color:#e03997;color:#fff;text-shadow:none;background-image:none}.ui.pink.button{-webkit-box-shadow:0 0 0 0 rgba(34,36,38,.15) inset;box-shadow:0 0 0 0 rgba(34,36,38,.15) inset}.ui.pink.button:hover,.ui.pink.buttons .button:hover{background-color:#e61a8d;color:#fff;text-shadow:none}.ui.pink.button:focus,.ui.pink.buttons .button:focus{background-color:#e10f85;color:#fff;text-shadow:none}.ui.pink.button:active,.ui.pink.buttons .button:active{background-color:#c71f7e;color:#fff;text-shadow:none}.ui.pink.active.button,.ui.pink.button .active.button:active,.ui.pink.buttons .active.button,.ui.pink.buttons .active.button:active{background-color:#ea158d;color:#fff;text-shadow:none}.ui.basic.pink.button,.ui.basic.pink.buttons .button{-webkit-box-shadow:0 0 0 1px #e03997 inset!important;box-shadow:0 0 0 1px #e03997 inset!important;color:#e03997!important}.ui.basic.pink.button:hover,.ui.basic.pink.buttons .button:hover{background:0 0!important;-webkit-box-shadow:0 0 0 1px #e61a8d inset!important;box-shadow:0 0 0 1px #e61a8d inset!important;color:#e61a8d!important}.ui.basic.pink.button:focus,.ui.basic.pink.buttons .button:focus{background:0 0!important;-webkit-box-shadow:0 0 0 1px #e10f85 inset!important;box-shadow:0 0 0 1px #e10f85 inset!important;color:#e61a8d!important}.ui.basic.pink.active.button,.ui.basic.pink.buttons .active.button{background:0 0!important;-webkit-box-shadow:0 0 0 1px #ea158d inset!important;box-shadow:0 0 0 1px #ea158d inset!important;color:#c71f7e!important}.ui.basic.pink.button:active,.ui.basic.pink.buttons .button:active{-webkit-box-shadow:0 0 0 1px #c71f7e inset!important;box-shadow:0 0 0 1px #c71f7e inset!important;color:#c71f7e!important}.ui.buttons:not(.vertical)>.basic.pink.button:not(:first-child){margin-left:-1px}.ui.inverted.pink.button,.ui.inverted.pink.buttons .button{background-color:transparent;-webkit-box-shadow:0 0 0 2px #ff8edf inset!important;box-shadow:0 0 0 2px #ff8edf inset!important;color:#ff8edf}.ui.inverted.pink.button.active,.ui.inverted.pink.button:active,.ui.inverted.pink.button:focus,.ui.inverted.pink.button:hover,.ui.inverted.pink.buttons .button.active,.ui.inverted.pink.buttons .button:active,.ui.inverted.pink.buttons .button:focus,.ui.inverted.pink.buttons .button:hover{-webkit-box-shadow:none!important;box-shadow:none!important;color:#fff}.ui.inverted.pink.button:hover,.ui.inverted.pink.buttons .button:hover{background-color:#ff74d8}.ui.inverted.pink.button:focus,.ui.inverted.pink.buttons .button:focus{background-color:#ff65d3}.ui.inverted.pink.active.button,.ui.inverted.pink.buttons .active.button{background-color:#ff74d8}.ui.inverted.pink.button:active,.ui.inverted.pink.buttons .button:active{background-color:#ff5bd1}.ui.inverted.pink.basic.button,.ui.inverted.pink.basic.buttons .button,.ui.inverted.pink.buttons .basic.button{background-color:transparent;-webkit-box-shadow:0 0 0 2px rgba(255,255,255,.5) inset!important;box-shadow:0 0 0 2px rgba(255,255,255,.5) inset!important;color:#fff!important}.ui.inverted.pink.basic.button:hover,.ui.inverted.pink.basic.buttons .button:hover,.ui.inverted.pink.buttons .basic.button:hover{-webkit-box-shadow:0 0 0 2px #ff74d8 inset!important;box-shadow:0 0 0 2px #ff74d8 inset!important;color:#ff8edf!important}.ui.inverted.pink.basic.button:focus,.ui.inverted.pink.basic.buttons .button:focus{-webkit-box-shadow:0 0 0 2px #ff65d3 inset!important;box-shadow:0 0 0 2px #ff65d3 inset!important;color:#ff8edf!important}.ui.inverted.pink.basic.active.button,.ui.inverted.pink.basic.buttons .active.button,.ui.inverted.pink.buttons .basic.active.button{-webkit-box-shadow:0 0 0 2px #ff74d8 inset!important;box-shadow:0 0 0 2px #ff74d8 inset!important;color:#ff8edf!important}.ui.inverted.pink.basic.button:active,.ui.inverted.pink.basic.buttons .button:active,.ui.inverted.pink.buttons .basic.button:active{-webkit-box-shadow:0 0 0 2px #ff5bd1 inset!important;box-shadow:0 0 0 2px #ff5bd1 inset!important;color:#ff8edf!important}.ui.violet.button,.ui.violet.buttons .button{background-color:#6435c9;color:#fff;text-shadow:none;background-image:none}.ui.violet.button{-webkit-box-shadow:0 0 0 0 rgba(34,36,38,.15) inset;box-shadow:0 0 0 0 rgba(34,36,38,.15) inset}.ui.violet.button:hover,.ui.violet.buttons .button:hover{background-color:#5829bb;color:#fff;text-shadow:none}.ui.violet.button:focus,.ui.violet.buttons .button:focus{background-color:#4f20b5;color:#fff;text-shadow:none}.ui.violet.button:active,.ui.violet.buttons .button:active{background-color:#502aa1;color:#fff;text-shadow:none}.ui.violet.active.button,.ui.violet.button .active.button:active,.ui.violet.buttons .active.button,.ui.violet.buttons .active.button:active{background-color:#5626bf;color:#fff;text-shadow:none}.ui.basic.violet.button,.ui.basic.violet.buttons .button{-webkit-box-shadow:0 0 0 1px #6435c9 inset!important;box-shadow:0 0 0 1px #6435c9 inset!important;color:#6435c9!important}.ui.basic.violet.button:hover,.ui.basic.violet.buttons .button:hover{background:0 0!important;-webkit-box-shadow:0 0 0 1px #5829bb inset!important;box-shadow:0 0 0 1px #5829bb inset!important;color:#5829bb!important}.ui.basic.violet.button:focus,.ui.basic.violet.buttons .button:focus{background:0 0!important;-webkit-box-shadow:0 0 0 1px #4f20b5 inset!important;box-shadow:0 0 0 1px #4f20b5 inset!important;color:#5829bb!important}.ui.basic.violet.active.button,.ui.basic.violet.buttons .active.button{background:0 0!important;-webkit-box-shadow:0 0 0 1px #5626bf inset!important;box-shadow:0 0 0 1px #5626bf inset!important;color:#502aa1!important}.ui.basic.violet.button:active,.ui.basic.violet.buttons .button:active{-webkit-box-shadow:0 0 0 1px #502aa1 inset!important;box-shadow:0 0 0 1px #502aa1 inset!important;color:#502aa1!important}.ui.buttons:not(.vertical)>.basic.violet.button:not(:first-child){margin-left:-1px}.ui.inverted.violet.button,.ui.inverted.violet.buttons .button{background-color:transparent;-webkit-box-shadow:0 0 0 2px #a291fb inset!important;box-shadow:0 0 0 2px #a291fb inset!important;color:#a291fb}.ui.inverted.violet.button.active,.ui.inverted.violet.button:active,.ui.inverted.violet.button:focus,.ui.inverted.violet.button:hover,.ui.inverted.violet.buttons .button.active,.ui.inverted.violet.buttons .button:active,.ui.inverted.violet.buttons .button:focus,.ui.inverted.violet.buttons .button:hover{-webkit-box-shadow:none!important;box-shadow:none!important;color:#fff}.ui.inverted.violet.button:hover,.ui.inverted.violet.buttons .button:hover{background-color:#8a73ff}.ui.inverted.violet.button:focus,.ui.inverted.violet.buttons .button:focus{background-color:#7d64ff}.ui.inverted.violet.active.button,.ui.inverted.violet.buttons .active.button{background-color:#8a73ff}.ui.inverted.violet.button:active,.ui.inverted.violet.buttons .button:active{background-color:#7860f9}.ui.inverted.violet.basic.button,.ui.inverted.violet.basic.buttons .button,.ui.inverted.violet.buttons .basic.button{background-color:transparent;-webkit-box-shadow:0 0 0 2px rgba(255,255,255,.5) inset!important;box-shadow:0 0 0 2px rgba(255,255,255,.5) inset!important;color:#fff!important}.ui.inverted.violet.basic.button:hover,.ui.inverted.violet.basic.buttons .button:hover,.ui.inverted.violet.buttons .basic.button:hover{-webkit-box-shadow:0 0 0 2px #8a73ff inset!important;box-shadow:0 0 0 2px #8a73ff inset!important;color:#a291fb!important}.ui.inverted.violet.basic.button:focus,.ui.inverted.violet.basic.buttons .button:focus{-webkit-box-shadow:0 0 0 2px #7d64ff inset!important;box-shadow:0 0 0 2px #7d64ff inset!important;color:#a291fb!important}.ui.inverted.violet.basic.active.button,.ui.inverted.violet.basic.buttons .active.button,.ui.inverted.violet.buttons .basic.active.button{-webkit-box-shadow:0 0 0 2px #8a73ff inset!important;box-shadow:0 0 0 2px #8a73ff inset!important;color:#a291fb!important}.ui.inverted.violet.basic.button:active,.ui.inverted.violet.basic.buttons .button:active,.ui.inverted.violet.buttons .basic.button:active{-webkit-box-shadow:0 0 0 2px #7860f9 inset!important;box-shadow:0 0 0 2px #7860f9 inset!important;color:#a291fb!important}.ui.purple.button,.ui.purple.buttons .button{background-color:#a333c8;color:#fff;text-shadow:none;background-image:none}.ui.purple.button{-webkit-box-shadow:0 0 0 0 rgba(34,36,38,.15) inset;box-shadow:0 0 0 0 rgba(34,36,38,.15) inset}.ui.purple.button:hover,.ui.purple.buttons .button:hover{background-color:#9627ba;color:#fff;text-shadow:none}.ui.purple.button:focus,.ui.purple.buttons .button:focus{background-color:#8f1eb4;color:#fff;text-shadow:none}.ui.purple.button:active,.ui.purple.buttons .button:active{background-color:#82299f;color:#fff;text-shadow:none}.ui.purple.active.button,.ui.purple.button .active.button:active,.ui.purple.buttons .active.button,.ui.purple.buttons .active.button:active{background-color:#9724be;color:#fff;text-shadow:none}.ui.basic.purple.button,.ui.basic.purple.buttons .button{-webkit-box-shadow:0 0 0 1px #a333c8 inset!important;box-shadow:0 0 0 1px #a333c8 inset!important;color:#a333c8!important}.ui.basic.purple.button:hover,.ui.basic.purple.buttons .button:hover{background:0 0!important;-webkit-box-shadow:0 0 0 1px #9627ba inset!important;box-shadow:0 0 0 1px #9627ba inset!important;color:#9627ba!important}.ui.basic.purple.button:focus,.ui.basic.purple.buttons .button:focus{background:0 0!important;-webkit-box-shadow:0 0 0 1px #8f1eb4 inset!important;box-shadow:0 0 0 1px #8f1eb4 inset!important;color:#9627ba!important}.ui.basic.purple.active.button,.ui.basic.purple.buttons .active.button{background:0 0!important;-webkit-box-shadow:0 0 0 1px #9724be inset!important;box-shadow:0 0 0 1px #9724be inset!important;color:#82299f!important}.ui.basic.purple.button:active,.ui.basic.purple.buttons .button:active{-webkit-box-shadow:0 0 0 1px #82299f inset!important;box-shadow:0 0 0 1px #82299f inset!important;color:#82299f!important}.ui.buttons:not(.vertical)>.basic.purple.button:not(:first-child){margin-left:-1px}.ui.inverted.purple.button,.ui.inverted.purple.buttons .button{background-color:transparent;-webkit-box-shadow:0 0 0 2px #dc73ff inset!important;box-shadow:0 0 0 2px #dc73ff inset!important;color:#dc73ff}.ui.inverted.purple.button.active,.ui.inverted.purple.button:active,.ui.inverted.purple.button:focus,.ui.inverted.purple.button:hover,.ui.inverted.purple.buttons .button.active,.ui.inverted.purple.buttons .button:active,.ui.inverted.purple.buttons .button:focus,.ui.inverted.purple.buttons .button:hover{-webkit-box-shadow:none!important;box-shadow:none!important;color:#fff}.ui.inverted.purple.button:hover,.ui.inverted.purple.buttons .button:hover{background-color:#d65aff}.ui.inverted.purple.button:focus,.ui.inverted.purple.buttons .button:focus{background-color:#d24aff}.ui.inverted.purple.active.button,.ui.inverted.purple.buttons .active.button{background-color:#d65aff}.ui.inverted.purple.button:active,.ui.inverted.purple.buttons .button:active{background-color:#cf40ff}.ui.inverted.purple.basic.button,.ui.inverted.purple.basic.buttons .button,.ui.inverted.purple.buttons .basic.button{background-color:transparent;-webkit-box-shadow:0 0 0 2px rgba(255,255,255,.5) inset!important;box-shadow:0 0 0 2px rgba(255,255,255,.5) inset!important;color:#fff!important}.ui.inverted.purple.basic.button:hover,.ui.inverted.purple.basic.buttons .button:hover,.ui.inverted.purple.buttons .basic.button:hover{-webkit-box-shadow:0 0 0 2px #d65aff inset!important;box-shadow:0 0 0 2px #d65aff inset!important;color:#dc73ff!important}.ui.inverted.purple.basic.button:focus,.ui.inverted.purple.basic.buttons .button:focus{-webkit-box-shadow:0 0 0 2px #d24aff inset!important;box-shadow:0 0 0 2px #d24aff inset!important;color:#dc73ff!important}.ui.inverted.purple.basic.active.button,.ui.inverted.purple.basic.buttons .active.button,.ui.inverted.purple.buttons .basic.active.button{-webkit-box-shadow:0 0 0 2px #d65aff inset!important;box-shadow:0 0 0 2px #d65aff inset!important;color:#dc73ff!important}.ui.inverted.purple.basic.button:active,.ui.inverted.purple.basic.buttons .button:active,.ui.inverted.purple.buttons .basic.button:active{-webkit-box-shadow:0 0 0 2px #cf40ff inset!important;box-shadow:0 0 0 2px #cf40ff inset!important;color:#dc73ff!important}.ui.red.button,.ui.red.buttons .button{background-color:#db2828;color:#fff;text-shadow:none;background-image:none}.ui.red.button{-webkit-box-shadow:0 0 0 0 rgba(34,36,38,.15) inset;box-shadow:0 0 0 0 rgba(34,36,38,.15) inset}.ui.red.button:hover,.ui.red.buttons .button:hover{background-color:#d01919;color:#fff;text-shadow:none}.ui.red.button:focus,.ui.red.buttons .button:focus{background-color:#ca1010;color:#fff;text-shadow:none}.ui.red.button:active,.ui.red.buttons .button:active{background-color:#b21e1e;color:#fff;text-shadow:none}.ui.red.active.button,.ui.red.button .active.button:active,.ui.red.buttons .active.button,.ui.red.buttons .active.button:active{background-color:#d41515;color:#fff;text-shadow:none}.ui.basic.red.button,.ui.basic.red.buttons .button{-webkit-box-shadow:0 0 0 1px #db2828 inset!important;box-shadow:0 0 0 1px #db2828 inset!important;color:#db2828!important}.ui.basic.red.button:hover,.ui.basic.red.buttons .button:hover{background:0 0!important;-webkit-box-shadow:0 0 0 1px #d01919 inset!important;box-shadow:0 0 0 1px #d01919 inset!important;color:#d01919!important}.ui.basic.red.button:focus,.ui.basic.red.buttons .button:focus{background:0 0!important;-webkit-box-shadow:0 0 0 1px #ca1010 inset!important;box-shadow:0 0 0 1px #ca1010 inset!important;color:#d01919!important}.ui.basic.red.active.button,.ui.basic.red.buttons .active.button{background:0 0!important;-webkit-box-shadow:0 0 0 1px #d41515 inset!important;box-shadow:0 0 0 1px #d41515 inset!important;color:#b21e1e!important}.ui.basic.red.button:active,.ui.basic.red.buttons .button:active{-webkit-box-shadow:0 0 0 1px #b21e1e inset!important;box-shadow:0 0 0 1px #b21e1e inset!important;color:#b21e1e!important}.ui.buttons:not(.vertical)>.basic.red.button:not(:first-child){margin-left:-1px}.ui.inverted.red.button,.ui.inverted.red.buttons .button{background-color:transparent;-webkit-box-shadow:0 0 0 2px #ff695e inset!important;box-shadow:0 0 0 2px #ff695e inset!important;color:#ff695e}.ui.inverted.red.button.active,.ui.inverted.red.button:active,.ui.inverted.red.button:focus,.ui.inverted.red.button:hover,.ui.inverted.red.buttons .button.active,.ui.inverted.red.buttons .button:active,.ui.inverted.red.buttons .button:focus,.ui.inverted.red.buttons .button:hover{-webkit-box-shadow:none!important;box-shadow:none!important;color:#fff}.ui.inverted.red.button:hover,.ui.inverted.red.buttons .button:hover{background-color:#ff5144}.ui.inverted.red.button:focus,.ui.inverted.red.buttons .button:focus{background-color:#ff4335}.ui.inverted.red.active.button,.ui.inverted.red.buttons .active.button{background-color:#ff5144}.ui.inverted.red.button:active,.ui.inverted.red.buttons .button:active{background-color:#ff392b}.ui.inverted.red.basic.button,.ui.inverted.red.basic.buttons .button,.ui.inverted.red.buttons .basic.button{background-color:transparent;-webkit-box-shadow:0 0 0 2px rgba(255,255,255,.5) inset!important;box-shadow:0 0 0 2px rgba(255,255,255,.5) inset!important;color:#fff!important}.ui.inverted.red.basic.button:hover,.ui.inverted.red.basic.buttons .button:hover,.ui.inverted.red.buttons .basic.button:hover{-webkit-box-shadow:0 0 0 2px #ff5144 inset!important;box-shadow:0 0 0 2px #ff5144 inset!important;color:#ff695e!important}.ui.inverted.red.basic.button:focus,.ui.inverted.red.basic.buttons .button:focus{-webkit-box-shadow:0 0 0 2px #ff4335 inset!important;box-shadow:0 0 0 2px #ff4335 inset!important;color:#ff695e!important}.ui.inverted.red.basic.active.button,.ui.inverted.red.basic.buttons .active.button,.ui.inverted.red.buttons .basic.active.button{-webkit-box-shadow:0 0 0 2px #ff5144 inset!important;box-shadow:0 0 0 2px #ff5144 inset!important;color:#ff695e!important}.ui.inverted.red.basic.button:active,.ui.inverted.red.basic.buttons .button:active,.ui.inverted.red.buttons .basic.button:active{-webkit-box-shadow:0 0 0 2px #ff392b inset!important;box-shadow:0 0 0 2px #ff392b inset!important;color:#ff695e!important}.ui.teal.button,.ui.teal.buttons .button{background-color:#00b5ad;color:#fff;text-shadow:none;background-image:none}.ui.teal.button{-webkit-box-shadow:0 0 0 0 rgba(34,36,38,.15) inset;box-shadow:0 0 0 0 rgba(34,36,38,.15) inset}.ui.teal.button:hover,.ui.teal.buttons .button:hover{background-color:#009c95;color:#fff;text-shadow:none}.ui.teal.button:focus,.ui.teal.buttons .button:focus{background-color:#008c86;color:#fff;text-shadow:none}.ui.teal.button:active,.ui.teal.buttons .button:active{background-color:#00827c;color:#fff;text-shadow:none}.ui.teal.active.button,.ui.teal.button .active.button:active,.ui.teal.buttons .active.button,.ui.teal.buttons .active.button:active{background-color:#009c95;color:#fff;text-shadow:none}.ui.basic.teal.button,.ui.basic.teal.buttons .button{-webkit-box-shadow:0 0 0 1px #00b5ad inset!important;box-shadow:0 0 0 1px #00b5ad inset!important;color:#00b5ad!important}.ui.basic.teal.button:hover,.ui.basic.teal.buttons .button:hover{background:0 0!important;-webkit-box-shadow:0 0 0 1px #009c95 inset!important;box-shadow:0 0 0 1px #009c95 inset!important;color:#009c95!important}.ui.basic.teal.button:focus,.ui.basic.teal.buttons .button:focus{background:0 0!important;-webkit-box-shadow:0 0 0 1px #008c86 inset!important;box-shadow:0 0 0 1px #008c86 inset!important;color:#009c95!important}.ui.basic.teal.active.button,.ui.basic.teal.buttons .active.button{background:0 0!important;-webkit-box-shadow:0 0 0 1px #009c95 inset!important;box-shadow:0 0 0 1px #009c95 inset!important;color:#00827c!important}.ui.basic.teal.button:active,.ui.basic.teal.buttons .button:active{-webkit-box-shadow:0 0 0 1px #00827c inset!important;box-shadow:0 0 0 1px #00827c inset!important;color:#00827c!important}.ui.buttons:not(.vertical)>.basic.teal.button:not(:first-child){margin-left:-1px}.ui.inverted.teal.button,.ui.inverted.teal.buttons .button{background-color:transparent;-webkit-box-shadow:0 0 0 2px #6dffff inset!important;box-shadow:0 0 0 2px #6dffff inset!important;color:#6dffff}.ui.inverted.teal.button.active,.ui.inverted.teal.button:active,.ui.inverted.teal.button:focus,.ui.inverted.teal.button:hover,.ui.inverted.teal.buttons .button.active,.ui.inverted.teal.buttons .button:active,.ui.inverted.teal.buttons .button:focus,.ui.inverted.teal.buttons .button:hover{-webkit-box-shadow:none!important;box-shadow:none!important;color:rgba(0,0,0,.6)}.ui.inverted.teal.button:hover,.ui.inverted.teal.buttons .button:hover{background-color:#54ffff}.ui.inverted.teal.button:focus,.ui.inverted.teal.buttons .button:focus{background-color:#4ff}.ui.inverted.teal.active.button,.ui.inverted.teal.buttons .active.button{background-color:#54ffff}.ui.inverted.teal.button:active,.ui.inverted.teal.buttons .button:active{background-color:#3affff}.ui.inverted.teal.basic.button,.ui.inverted.teal.basic.buttons .button,.ui.inverted.teal.buttons .basic.button{background-color:transparent;-webkit-box-shadow:0 0 0 2px rgba(255,255,255,.5) inset!important;box-shadow:0 0 0 2px rgba(255,255,255,.5) inset!important;color:#fff!important}.ui.inverted.teal.basic.button:hover,.ui.inverted.teal.basic.buttons .button:hover,.ui.inverted.teal.buttons .basic.button:hover{-webkit-box-shadow:0 0 0 2px #54ffff inset!important;box-shadow:0 0 0 2px #54ffff inset!important;color:#6dffff!important}.ui.inverted.teal.basic.button:focus,.ui.inverted.teal.basic.buttons .button:focus{-webkit-box-shadow:0 0 0 2px #4ff inset!important;box-shadow:0 0 0 2px #4ff inset!important;color:#6dffff!important}.ui.inverted.teal.basic.active.button,.ui.inverted.teal.basic.buttons .active.button,.ui.inverted.teal.buttons .basic.active.button{-webkit-box-shadow:0 0 0 2px #54ffff inset!important;box-shadow:0 0 0 2px #54ffff inset!important;color:#6dffff!important}.ui.inverted.teal.basic.button:active,.ui.inverted.teal.basic.buttons .button:active,.ui.inverted.teal.buttons .basic.button:active{-webkit-box-shadow:0 0 0 2px #3affff inset!important;box-shadow:0 0 0 2px #3affff inset!important;color:#6dffff!important}.ui.olive.button,.ui.olive.buttons .button{background-color:#b5cc18;color:#fff;text-shadow:none;background-image:none}.ui.olive.button{-webkit-box-shadow:0 0 0 0 rgba(34,36,38,.15) inset;box-shadow:0 0 0 0 rgba(34,36,38,.15) inset}.ui.olive.button:hover,.ui.olive.buttons .button:hover{background-color:#a7bd0d;color:#fff;text-shadow:none}.ui.olive.button:focus,.ui.olive.buttons .button:focus{background-color:#a0b605;color:#fff;text-shadow:none}.ui.olive.button:active,.ui.olive.buttons .button:active{background-color:#8d9e13;color:#fff;text-shadow:none}.ui.olive.active.button,.ui.olive.button .active.button:active,.ui.olive.buttons .active.button,.ui.olive.buttons .active.button:active{background-color:#aac109;color:#fff;text-shadow:none}.ui.basic.olive.button,.ui.basic.olive.buttons .button{-webkit-box-shadow:0 0 0 1px #b5cc18 inset!important;box-shadow:0 0 0 1px #b5cc18 inset!important;color:#b5cc18!important}.ui.basic.olive.button:hover,.ui.basic.olive.buttons .button:hover{background:0 0!important;-webkit-box-shadow:0 0 0 1px #a7bd0d inset!important;box-shadow:0 0 0 1px #a7bd0d inset!important;color:#a7bd0d!important}.ui.basic.olive.button:focus,.ui.basic.olive.buttons .button:focus{background:0 0!important;-webkit-box-shadow:0 0 0 1px #a0b605 inset!important;box-shadow:0 0 0 1px #a0b605 inset!important;color:#a7bd0d!important}.ui.basic.olive.active.button,.ui.basic.olive.buttons .active.button{background:0 0!important;-webkit-box-shadow:0 0 0 1px #aac109 inset!important;box-shadow:0 0 0 1px #aac109 inset!important;color:#8d9e13!important}.ui.basic.olive.button:active,.ui.basic.olive.buttons .button:active{-webkit-box-shadow:0 0 0 1px #8d9e13 inset!important;box-shadow:0 0 0 1px #8d9e13 inset!important;color:#8d9e13!important}.ui.buttons:not(.vertical)>.basic.olive.button:not(:first-child){margin-left:-1px}.ui.inverted.olive.button,.ui.inverted.olive.buttons .button{background-color:transparent;-webkit-box-shadow:0 0 0 2px #d9e778 inset!important;box-shadow:0 0 0 2px #d9e778 inset!important;color:#d9e778}.ui.inverted.olive.button.active,.ui.inverted.olive.button:active,.ui.inverted.olive.button:focus,.ui.inverted.olive.button:hover,.ui.inverted.olive.buttons .button.active,.ui.inverted.olive.buttons .button:active,.ui.inverted.olive.buttons .button:focus,.ui.inverted.olive.buttons .button:hover{-webkit-box-shadow:none!important;box-shadow:none!important;color:rgba(0,0,0,.6)}.ui.inverted.olive.button:hover,.ui.inverted.olive.buttons .button:hover{background-color:#d8ea5c}.ui.inverted.olive.button:focus,.ui.inverted.olive.buttons .button:focus{background-color:#daef47}.ui.inverted.olive.active.button,.ui.inverted.olive.buttons .active.button{background-color:#daed59}.ui.inverted.olive.button:active,.ui.inverted.olive.buttons .button:active{background-color:#cddf4d}.ui.inverted.olive.basic.button,.ui.inverted.olive.basic.buttons .button,.ui.inverted.olive.buttons .basic.button{background-color:transparent;-webkit-box-shadow:0 0 0 2px rgba(255,255,255,.5) inset!important;box-shadow:0 0 0 2px rgba(255,255,255,.5) inset!important;color:#fff!important}.ui.inverted.olive.basic.button:hover,.ui.inverted.olive.basic.buttons .button:hover,.ui.inverted.olive.buttons .basic.button:hover{-webkit-box-shadow:0 0 0 2px #d8ea5c inset!important;box-shadow:0 0 0 2px #d8ea5c inset!important;color:#d9e778!important}.ui.inverted.olive.basic.button:focus,.ui.inverted.olive.basic.buttons .button:focus{-webkit-box-shadow:0 0 0 2px #daef47 inset!important;box-shadow:0 0 0 2px #daef47 inset!important;color:#d9e778!important}.ui.inverted.olive.basic.active.button,.ui.inverted.olive.basic.buttons .active.button,.ui.inverted.olive.buttons .basic.active.button{-webkit-box-shadow:0 0 0 2px #daed59 inset!important;box-shadow:0 0 0 2px #daed59 inset!important;color:#d9e778!important}.ui.inverted.olive.basic.button:active,.ui.inverted.olive.basic.buttons .button:active,.ui.inverted.olive.buttons .basic.button:active{-webkit-box-shadow:0 0 0 2px #cddf4d inset!important;box-shadow:0 0 0 2px #cddf4d inset!important;color:#d9e778!important}.ui.yellow.button,.ui.yellow.buttons .button{background-color:#fbbd08;color:#fff;text-shadow:none;background-image:none}.ui.yellow.button{-webkit-box-shadow:0 0 0 0 rgba(34,36,38,.15) inset;box-shadow:0 0 0 0 rgba(34,36,38,.15) inset}.ui.yellow.button:hover,.ui.yellow.buttons .button:hover{background-color:#eaae00;color:#fff;text-shadow:none}.ui.yellow.button:focus,.ui.yellow.buttons .button:focus{background-color:#daa300;color:#fff;text-shadow:none}.ui.yellow.button:active,.ui.yellow.buttons .button:active{background-color:#cd9903;color:#fff;text-shadow:none}.ui.yellow.active.button,.ui.yellow.button .active.button:active,.ui.yellow.buttons .active.button,.ui.yellow.buttons .active.button:active{background-color:#eaae00;color:#fff;text-shadow:none}.ui.basic.yellow.button,.ui.basic.yellow.buttons .button{-webkit-box-shadow:0 0 0 1px #fbbd08 inset!important;box-shadow:0 0 0 1px #fbbd08 inset!important;color:#fbbd08!important}.ui.basic.yellow.button:hover,.ui.basic.yellow.buttons .button:hover{background:0 0!important;-webkit-box-shadow:0 0 0 1px #eaae00 inset!important;box-shadow:0 0 0 1px #eaae00 inset!important;color:#eaae00!important}.ui.basic.yellow.button:focus,.ui.basic.yellow.buttons .button:focus{background:0 0!important;-webkit-box-shadow:0 0 0 1px #daa300 inset!important;box-shadow:0 0 0 1px #daa300 inset!important;color:#eaae00!important}.ui.basic.yellow.active.button,.ui.basic.yellow.buttons .active.button{background:0 0!important;-webkit-box-shadow:0 0 0 1px #eaae00 inset!important;box-shadow:0 0 0 1px #eaae00 inset!important;color:#cd9903!important}.ui.basic.yellow.button:active,.ui.basic.yellow.buttons .button:active{-webkit-box-shadow:0 0 0 1px #cd9903 inset!important;box-shadow:0 0 0 1px #cd9903 inset!important;color:#cd9903!important}.ui.buttons:not(.vertical)>.basic.yellow.button:not(:first-child){margin-left:-1px}.ui.inverted.yellow.button,.ui.inverted.yellow.buttons .button{background-color:transparent;-webkit-box-shadow:0 0 0 2px #ffe21f inset!important;box-shadow:0 0 0 2px #ffe21f inset!important;color:#ffe21f}.ui.inverted.yellow.button.active,.ui.inverted.yellow.button:active,.ui.inverted.yellow.button:focus,.ui.inverted.yellow.button:hover,.ui.inverted.yellow.buttons .button.active,.ui.inverted.yellow.buttons .button:active,.ui.inverted.yellow.buttons .button:focus,.ui.inverted.yellow.buttons .button:hover{-webkit-box-shadow:none!important;box-shadow:none!important;color:rgba(0,0,0,.6)}.ui.inverted.yellow.button:hover,.ui.inverted.yellow.buttons .button:hover{background-color:#ffdf05}.ui.inverted.yellow.button:focus,.ui.inverted.yellow.buttons .button:focus{background-color:#f5d500}.ui.inverted.yellow.active.button,.ui.inverted.yellow.buttons .active.button{background-color:#ffdf05}.ui.inverted.yellow.button:active,.ui.inverted.yellow.buttons .button:active{background-color:#ebcd00}.ui.inverted.yellow.basic.button,.ui.inverted.yellow.basic.buttons .button,.ui.inverted.yellow.buttons .basic.button{background-color:transparent;-webkit-box-shadow:0 0 0 2px rgba(255,255,255,.5) inset!important;box-shadow:0 0 0 2px rgba(255,255,255,.5) inset!important;color:#fff!important}.ui.inverted.yellow.basic.button:hover,.ui.inverted.yellow.basic.buttons .button:hover,.ui.inverted.yellow.buttons .basic.button:hover{-webkit-box-shadow:0 0 0 2px #ffdf05 inset!important;box-shadow:0 0 0 2px #ffdf05 inset!important;color:#ffe21f!important}.ui.inverted.yellow.basic.button:focus,.ui.inverted.yellow.basic.buttons .button:focus{-webkit-box-shadow:0 0 0 2px #f5d500 inset!important;box-shadow:0 0 0 2px #f5d500 inset!important;color:#ffe21f!important}.ui.inverted.yellow.basic.active.button,.ui.inverted.yellow.basic.buttons .active.button,.ui.inverted.yellow.buttons .basic.active.button{-webkit-box-shadow:0 0 0 2px #ffdf05 inset!important;box-shadow:0 0 0 2px #ffdf05 inset!important;color:#ffe21f!important}.ui.inverted.yellow.basic.button:active,.ui.inverted.yellow.basic.buttons .button:active,.ui.inverted.yellow.buttons .basic.button:active{-webkit-box-shadow:0 0 0 2px #ebcd00 inset!important;box-shadow:0 0 0 2px #ebcd00 inset!important;color:#ffe21f!important}.ui.primary.button,.ui.primary.buttons .button{background-color:#2185d0;color:#fff;text-shadow:none;background-image:none}.ui.primary.button{-webkit-box-shadow:0 0 0 0 rgba(34,36,38,.15) inset;box-shadow:0 0 0 0 rgba(34,36,38,.15) inset}.ui.primary.button:hover,.ui.primary.buttons .button:hover{background-color:#1678c2;color:#fff;text-shadow:none}.ui.primary.button:focus,.ui.primary.buttons .button:focus{background-color:#0d71bb;color:#fff;text-shadow:none}.ui.primary.button:active,.ui.primary.buttons .button:active{background-color:#1a69a4;color:#fff;text-shadow:none}.ui.primary.active.button,.ui.primary.button .active.button:active,.ui.primary.buttons .active.button,.ui.primary.buttons .active.button:active{background-color:#1279c6;color:#fff;text-shadow:none}.ui.basic.primary.button,.ui.basic.primary.buttons .button{-webkit-box-shadow:0 0 0 1px #2185d0 inset!important;box-shadow:0 0 0 1px #2185d0 inset!important;color:#2185d0!important}.ui.basic.primary.button:hover,.ui.basic.primary.buttons .button:hover{background:0 0!important;-webkit-box-shadow:0 0 0 1px #1678c2 inset!important;box-shadow:0 0 0 1px #1678c2 inset!important;color:#1678c2!important}.ui.basic.primary.button:focus,.ui.basic.primary.buttons .button:focus{background:0 0!important;-webkit-box-shadow:0 0 0 1px #0d71bb inset!important;box-shadow:0 0 0 1px #0d71bb inset!important;color:#1678c2!important}.ui.basic.primary.active.button,.ui.basic.primary.buttons .active.button{background:0 0!important;-webkit-box-shadow:0 0 0 1px #1279c6 inset!important;box-shadow:0 0 0 1px #1279c6 inset!important;color:#1a69a4!important}.ui.basic.primary.button:active,.ui.basic.primary.buttons .button:active{-webkit-box-shadow:0 0 0 1px #1a69a4 inset!important;box-shadow:0 0 0 1px #1a69a4 inset!important;color:#1a69a4!important}.ui.buttons:not(.vertical)>.basic.primary.button:not(:first-child){margin-left:-1px}.ui.inverted.primary.button,.ui.inverted.primary.buttons .button{background-color:transparent;-webkit-box-shadow:0 0 0 2px #54c8ff inset!important;box-shadow:0 0 0 2px #54c8ff inset!important;color:#54c8ff}.ui.inverted.primary.button.active,.ui.inverted.primary.button:active,.ui.inverted.primary.button:focus,.ui.inverted.primary.button:hover,.ui.inverted.primary.buttons .button.active,.ui.inverted.primary.buttons .button:active,.ui.inverted.primary.buttons .button:focus,.ui.inverted.primary.buttons .button:hover{-webkit-box-shadow:none!important;box-shadow:none!important;color:#fff}.ui.inverted.primary.button:hover,.ui.inverted.primary.buttons .button:hover{background-color:#3ac0ff}.ui.inverted.primary.button:focus,.ui.inverted.primary.buttons .button:focus{background-color:#2bbbff}.ui.inverted.primary.active.button,.ui.inverted.primary.buttons .active.button{background-color:#3ac0ff}.ui.inverted.primary.button:active,.ui.inverted.primary.buttons .button:active{background-color:#21b8ff}.ui.inverted.primary.basic.button,.ui.inverted.primary.basic.buttons .button,.ui.inverted.primary.buttons .basic.button{background-color:transparent;-webkit-box-shadow:0 0 0 2px rgba(255,255,255,.5) inset!important;box-shadow:0 0 0 2px rgba(255,255,255,.5) inset!important;color:#fff!important}.ui.inverted.primary.basic.button:hover,.ui.inverted.primary.basic.buttons .button:hover,.ui.inverted.primary.buttons .basic.button:hover{-webkit-box-shadow:0 0 0 2px #3ac0ff inset!important;box-shadow:0 0 0 2px #3ac0ff inset!important;color:#54c8ff!important}.ui.inverted.primary.basic.button:focus,.ui.inverted.primary.basic.buttons .button:focus{-webkit-box-shadow:0 0 0 2px #2bbbff inset!important;box-shadow:0 0 0 2px #2bbbff inset!important;color:#54c8ff!important}.ui.inverted.primary.basic.active.button,.ui.inverted.primary.basic.buttons .active.button,.ui.inverted.primary.buttons .basic.active.button{-webkit-box-shadow:0 0 0 2px #3ac0ff inset!important;box-shadow:0 0 0 2px #3ac0ff inset!important;color:#54c8ff!important}.ui.inverted.primary.basic.button:active,.ui.inverted.primary.basic.buttons .button:active,.ui.inverted.primary.buttons .basic.button:active{-webkit-box-shadow:0 0 0 2px #21b8ff inset!important;box-shadow:0 0 0 2px #21b8ff inset!important;color:#54c8ff!important}.ui.secondary.button,.ui.secondary.buttons .button{background-color:#1b1c1d;color:#fff;text-shadow:none;background-image:none}.ui.secondary.button{-webkit-box-shadow:0 0 0 0 rgba(34,36,38,.15) inset;box-shadow:0 0 0 0 rgba(34,36,38,.15) inset}.ui.secondary.button:hover,.ui.secondary.buttons .button:hover{background-color:#27292a;color:#fff;text-shadow:none}.ui.secondary.button:focus,.ui.secondary.buttons .button:focus{background-color:#2e3032;color:#fff;text-shadow:none}.ui.secondary.button:active,.ui.secondary.buttons .button:active{background-color:#343637;color:#fff;text-shadow:none}.ui.secondary.active.button,.ui.secondary.button .active.button:active,.ui.secondary.buttons .active.button,.ui.secondary.buttons .active.button:active{background-color:#27292a;color:#fff;text-shadow:none}.ui.basic.secondary.button,.ui.basic.secondary.buttons .button{-webkit-box-shadow:0 0 0 1px #1b1c1d inset!important;box-shadow:0 0 0 1px #1b1c1d inset!important;color:#1b1c1d!important}.ui.basic.secondary.button:hover,.ui.basic.secondary.buttons .button:hover{background:0 0!important;-webkit-box-shadow:0 0 0 1px #27292a inset!important;box-shadow:0 0 0 1px #27292a inset!important;color:#27292a!important}.ui.basic.secondary.button:focus,.ui.basic.secondary.buttons .button:focus{background:0 0!important;-webkit-box-shadow:0 0 0 1px #2e3032 inset!important;box-shadow:0 0 0 1px #2e3032 inset!important;color:#27292a!important}.ui.basic.secondary.active.button,.ui.basic.secondary.buttons .active.button{background:0 0!important;-webkit-box-shadow:0 0 0 1px #27292a inset!important;box-shadow:0 0 0 1px #27292a inset!important;color:#343637!important}.ui.basic.secondary.button:active,.ui.basic.secondary.buttons .button:active{-webkit-box-shadow:0 0 0 1px #343637 inset!important;box-shadow:0 0 0 1px #343637 inset!important;color:#343637!important}.ui.buttons:not(.vertical)>.basic.primary.button:not(:first-child){margin-left:-1px}.ui.inverted.secondary.button,.ui.inverted.secondary.buttons .button{background-color:transparent;-webkit-box-shadow:0 0 0 2px #545454 inset!important;box-shadow:0 0 0 2px #545454 inset!important;color:#545454}.ui.inverted.secondary.button.active,.ui.inverted.secondary.button:active,.ui.inverted.secondary.button:focus,.ui.inverted.secondary.button:hover,.ui.inverted.secondary.buttons .button.active,.ui.inverted.secondary.buttons .button:active,.ui.inverted.secondary.buttons .button:focus,.ui.inverted.secondary.buttons .button:hover{-webkit-box-shadow:none!important;box-shadow:none!important;color:#fff}.ui.inverted.secondary.button:hover,.ui.inverted.secondary.buttons .button:hover{background-color:#616161}.ui.inverted.secondary.button:focus,.ui.inverted.secondary.buttons .button:focus{background-color:#686868}.ui.inverted.secondary.active.button,.ui.inverted.secondary.buttons .active.button{background-color:#616161}.ui.inverted.secondary.button:active,.ui.inverted.secondary.buttons .button:active{background-color:#6e6e6e}.ui.inverted.secondary.basic.button,.ui.inverted.secondary.basic.buttons .button,.ui.inverted.secondary.buttons .basic.button{background-color:transparent;-webkit-box-shadow:0 0 0 2px rgba(255,255,255,.5) inset!important;box-shadow:0 0 0 2px rgba(255,255,255,.5) inset!important;color:#fff!important}.ui.inverted.secondary.basic.button:hover,.ui.inverted.secondary.basic.buttons .button:hover,.ui.inverted.secondary.buttons .basic.button:hover{-webkit-box-shadow:0 0 0 2px #616161 inset!important;box-shadow:0 0 0 2px #616161 inset!important;color:#545454!important}.ui.inverted.secondary.basic.button:focus,.ui.inverted.secondary.basic.buttons .button:focus{-webkit-box-shadow:0 0 0 2px #686868 inset!important;box-shadow:0 0 0 2px #686868 inset!important;color:#545454!important}.ui.inverted.secondary.basic.active.button,.ui.inverted.secondary.basic.buttons .active.button,.ui.inverted.secondary.buttons .basic.active.button{-webkit-box-shadow:0 0 0 2px #616161 inset!important;box-shadow:0 0 0 2px #616161 inset!important;color:#545454!important}.ui.inverted.secondary.basic.button:active,.ui.inverted.secondary.basic.buttons .button:active,.ui.inverted.secondary.buttons .basic.button:active{-webkit-box-shadow:0 0 0 2px #6e6e6e inset!important;box-shadow:0 0 0 2px #6e6e6e inset!important;color:#545454!important}.ui.positive.button,.ui.positive.buttons .button{background-color:#21ba45;color:#fff;text-shadow:none;background-image:none}.ui.positive.button{-webkit-box-shadow:0 0 0 0 rgba(34,36,38,.15) inset;box-shadow:0 0 0 0 rgba(34,36,38,.15) inset}.ui.positive.button:hover,.ui.positive.buttons .button:hover{background-color:#16ab39;color:#fff;text-shadow:none}.ui.positive.button:focus,.ui.positive.buttons .button:focus{background-color:#0ea432;color:#fff;text-shadow:none}.ui.positive.button:active,.ui.positive.buttons .button:active{background-color:#198f35;color:#fff;text-shadow:none}.ui.positive.active.button,.ui.positive.button .active.button:active,.ui.positive.buttons .active.button,.ui.positive.buttons .active.button:active{background-color:#13ae38;color:#fff;text-shadow:none}.ui.basic.positive.button,.ui.basic.positive.buttons .button{-webkit-box-shadow:0 0 0 1px #21ba45 inset!important;box-shadow:0 0 0 1px #21ba45 inset!important;color:#21ba45!important}.ui.basic.positive.button:hover,.ui.basic.positive.buttons .button:hover{background:0 0!important;-webkit-box-shadow:0 0 0 1px #16ab39 inset!important;box-shadow:0 0 0 1px #16ab39 inset!important;color:#16ab39!important}.ui.basic.positive.button:focus,.ui.basic.positive.buttons .button:focus{background:0 0!important;-webkit-box-shadow:0 0 0 1px #0ea432 inset!important;box-shadow:0 0 0 1px #0ea432 inset!important;color:#16ab39!important}.ui.basic.positive.active.button,.ui.basic.positive.buttons .active.button{background:0 0!important;-webkit-box-shadow:0 0 0 1px #13ae38 inset!important;box-shadow:0 0 0 1px #13ae38 inset!important;color:#198f35!important}.ui.basic.positive.button:active,.ui.basic.positive.buttons .button:active{-webkit-box-shadow:0 0 0 1px #198f35 inset!important;box-shadow:0 0 0 1px #198f35 inset!important;color:#198f35!important}.ui.buttons:not(.vertical)>.basic.primary.button:not(:first-child){margin-left:-1px}.ui.negative.button,.ui.negative.buttons .button{background-color:#db2828;color:#fff;text-shadow:none;background-image:none}.ui.negative.button{-webkit-box-shadow:0 0 0 0 rgba(34,36,38,.15) inset;box-shadow:0 0 0 0 rgba(34,36,38,.15) inset}.ui.negative.button:hover,.ui.negative.buttons .button:hover{background-color:#d01919;color:#fff;text-shadow:none}.ui.negative.button:focus,.ui.negative.buttons .button:focus{background-color:#ca1010;color:#fff;text-shadow:none}.ui.negative.button:active,.ui.negative.buttons .button:active{background-color:#b21e1e;color:#fff;text-shadow:none}.ui.negative.active.button,.ui.negative.button .active.button:active,.ui.negative.buttons .active.button,.ui.negative.buttons .active.button:active{background-color:#d41515;color:#fff;text-shadow:none}.ui.basic.negative.button,.ui.basic.negative.buttons .button{-webkit-box-shadow:0 0 0 1px #db2828 inset!important;box-shadow:0 0 0 1px #db2828 inset!important;color:#db2828!important}.ui.basic.negative.button:hover,.ui.basic.negative.buttons .button:hover{background:0 0!important;-webkit-box-shadow:0 0 0 1px #d01919 inset!important;box-shadow:0 0 0 1px #d01919 inset!important;color:#d01919!important}.ui.basic.negative.button:focus,.ui.basic.negative.buttons .button:focus{background:0 0!important;-webkit-box-shadow:0 0 0 1px #ca1010 inset!important;box-shadow:0 0 0 1px #ca1010 inset!important;color:#d01919!important}.ui.basic.negative.active.button,.ui.basic.negative.buttons .active.button{background:0 0!important;-webkit-box-shadow:0 0 0 1px #d41515 inset!important;box-shadow:0 0 0 1px #d41515 inset!important;color:#b21e1e!important}.ui.basic.negative.button:active,.ui.basic.negative.buttons .button:active{-webkit-box-shadow:0 0 0 1px #b21e1e inset!important;box-shadow:0 0 0 1px #b21e1e inset!important;color:#b21e1e!important}.ui.buttons:not(.vertical)>.basic.primary.button:not(:first-child){margin-left:-1px}.ui.buttons{display:-webkit-inline-box;display:-ms-inline-flexbox;display:inline-flex;-webkit-box-orient:horizontal;-webkit-box-direction:normal;-ms-flex-direction:row;flex-direction:row;font-size:0;vertical-align:baseline;margin:0 .25em 0 0}.ui.buttons:not(.basic):not(.inverted){-webkit-box-shadow:none;box-shadow:none}.ui.buttons:after{content:".";display:block;height:0;clear:both;visibility:hidden}.ui.buttons .button{-webkit-box-flex:1;-ms-flex:1 0 auto;flex:1 0 auto;margin:0;border-radius:0;margin:0}.ui.buttons:not(.basic):not(.inverted)>.button,.ui.buttons>.ui.button:not(.basic):not(.inverted){-webkit-box-shadow:0 0 0 1px transparent inset,0 0 0 0 rgba(34,36,38,.15) inset;box-shadow:0 0 0 1px transparent inset,0 0 0 0 rgba(34,36,38,.15) inset}.ui.buttons .button:first-child{border-left:none;margin-left:0;border-top-left-radius:.28571429rem;border-bottom-left-radius:.28571429rem}.ui.buttons .button:last-child{border-top-right-radius:.28571429rem;border-bottom-right-radius:.28571429rem}.ui.vertical.buttons{display:-webkit-inline-box;display:-ms-inline-flexbox;display:inline-flex;-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column}.ui.vertical.buttons .button{display:block;float:none;width:100%;margin:0;-webkit-box-shadow:none;box-shadow:none;border-radius:0}.ui.vertical.buttons .button:first-child{border-top-left-radius:.28571429rem;border-top-right-radius:.28571429rem}.ui.vertical.buttons .button:last-child{margin-bottom:0;border-bottom-left-radius:.28571429rem;border-bottom-right-radius:.28571429rem}.ui.vertical.buttons .button:only-child{border-radius:.28571429rem}/*! - * # Semantic UI 2.4.0 - Container - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */.ui.container{display:block;max-width:100%!important}@media only screen and (max-width:767px){.ui.container{width:auto!important;margin-left:1em!important;margin-right:1em!important}.ui.grid.container{width:auto!important}.ui.relaxed.grid.container{width:auto!important}.ui.very.relaxed.grid.container{width:auto!important}}@media only screen and (min-width:768px) and (max-width:991px){.ui.container{width:723px;margin-left:auto!important;margin-right:auto!important}.ui.grid.container{width:calc(723px + 2rem)!important}.ui.relaxed.grid.container{width:calc(723px + 3rem)!important}.ui.very.relaxed.grid.container{width:calc(723px + 5rem)!important}}@media only screen and (min-width:992px) and (max-width:1199px){.ui.container{width:933px;margin-left:auto!important;margin-right:auto!important}.ui.grid.container{width:calc(933px + 2rem)!important}.ui.relaxed.grid.container{width:calc(933px + 3rem)!important}.ui.very.relaxed.grid.container{width:calc(933px + 5rem)!important}}@media only screen and (min-width:1200px){.ui.container{width:1127px;margin-left:auto!important;margin-right:auto!important}.ui.grid.container{width:calc(1127px + 2rem)!important}.ui.relaxed.grid.container{width:calc(1127px + 3rem)!important}.ui.very.relaxed.grid.container{width:calc(1127px + 5rem)!important}}.ui.text.container{font-family:Lato,'Helvetica Neue',Arial,Helvetica,sans-serif;max-width:700px!important;line-height:1.5}.ui.text.container{font-size:1.14285714rem}.ui.fluid.container{width:100%}.ui[class*="left aligned"].container{text-align:left}.ui[class*="center aligned"].container{text-align:center}.ui[class*="right aligned"].container{text-align:right}.ui.justified.container{text-align:justify;-webkit-hyphens:auto;-ms-hyphens:auto;hyphens:auto}/*! - * # Semantic UI 2.4.0 - Divider - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */.ui.divider{margin:1rem 0;line-height:1;height:0;font-weight:700;text-transform:uppercase;letter-spacing:.05em;color:rgba(0,0,0,.85);-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;-webkit-tap-highlight-color:transparent}.ui.divider:not(.vertical):not(.horizontal){border-top:1px solid rgba(34,36,38,.15);border-bottom:1px solid rgba(255,255,255,.1)}.ui.grid>.column+.divider,.ui.grid>.row>.column+.divider{left:auto}.ui.horizontal.divider{display:table;white-space:nowrap;height:auto;margin:'';line-height:1;text-align:center}.ui.horizontal.divider:after,.ui.horizontal.divider:before{content:'';display:table-cell;position:relative;top:50%;width:50%;background-repeat:no-repeat}.ui.horizontal.divider:before{background-position:right 1em top 50%}.ui.horizontal.divider:after{background-position:left 1em top 50%}.ui.vertical.divider{position:absolute;z-index:2;top:50%;left:50%;margin:0;padding:0;width:auto;height:50%;line-height:0;text-align:center;-webkit-transform:translateX(-50%);transform:translateX(-50%)}.ui.vertical.divider:after,.ui.vertical.divider:before{position:absolute;left:50%;content:'';z-index:3;border-left:1px solid rgba(34,36,38,.15);border-right:1px solid rgba(255,255,255,.1);width:0%;height:calc(100% - 1rem)}.ui.vertical.divider:before{top:-100%}.ui.vertical.divider:after{top:auto;bottom:0}@media only screen and (max-width:767px){.ui.grid .stackable.row .ui.vertical.divider,.ui.stackable.grid .ui.vertical.divider{display:table;white-space:nowrap;height:auto;margin:'';overflow:hidden;line-height:1;text-align:center;position:static;top:0;left:0;-webkit-transform:none;transform:none}.ui.grid .stackable.row .ui.vertical.divider:after,.ui.grid .stackable.row .ui.vertical.divider:before,.ui.stackable.grid .ui.vertical.divider:after,.ui.stackable.grid .ui.vertical.divider:before{position:static;left:0;border-left:none;border-right:none;content:'';display:table-cell;position:relative;top:50%;width:50%;background-repeat:no-repeat}.ui.grid .stackable.row .ui.vertical.divider:before,.ui.stackable.grid .ui.vertical.divider:before{background-position:right 1em top 50%}.ui.grid .stackable.row .ui.vertical.divider:after,.ui.stackable.grid .ui.vertical.divider:after{background-position:left 1em top 50%}}.ui.divider>.icon{margin:0;font-size:1rem;height:1em;vertical-align:middle}.ui.hidden.divider{border-color:transparent!important}.ui.hidden.divider:after,.ui.hidden.divider:before{display:none}.ui.divider.inverted,.ui.horizontal.inverted.divider,.ui.vertical.inverted.divider{color:#fff}.ui.divider.inverted,.ui.divider.inverted:after,.ui.divider.inverted:before{border-top-color:rgba(34,36,38,.15)!important;border-left-color:rgba(34,36,38,.15)!important;border-bottom-color:rgba(255,255,255,.15)!important;border-right-color:rgba(255,255,255,.15)!important}.ui.fitted.divider{margin:0}.ui.clearing.divider{clear:both}.ui.section.divider{margin-top:2rem;margin-bottom:2rem}.ui.divider{font-size:1rem}.ui.horizontal.divider:after,.ui.horizontal.divider:before{background-image:url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAABaAAAAACCAYAAACuTHuKAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAyFpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuNS1jMDE0IDc5LjE1MTQ4MSwgMjAxMy8wMy8xMy0xMjowOToxNSAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvIiB4bWxuczp4bXBNTT0iaHR0cDovL25zLmFkb2JlLmNvbS94YXAvMS4wL21tLyIgeG1sbnM6c3RSZWY9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9zVHlwZS9SZXNvdXJjZVJlZiMiIHhtcDpDcmVhdG9yVG9vbD0iQWRvYmUgUGhvdG9zaG9wIENDIChXaW5kb3dzKSIgeG1wTU06SW5zdGFuY2VJRD0ieG1wLmlpZDo1OThBRDY4OUNDMTYxMUU0OUE3NUVGOEJDMzMzMjE2NyIgeG1wTU06RG9jdW1lbnRJRD0ieG1wLmRpZDo1OThBRDY4QUNDMTYxMUU0OUE3NUVGOEJDMzMzMjE2NyI+IDx4bXBNTTpEZXJpdmVkRnJvbSBzdFJlZjppbnN0YW5jZUlEPSJ4bXAuaWlkOjU5OEFENjg3Q0MxNjExRTQ5QTc1RUY4QkMzMzMyMTY3IiBzdFJlZjpkb2N1bWVudElEPSJ4bXAuZGlkOjU5OEFENjg4Q0MxNjExRTQ5QTc1RUY4QkMzMzMyMTY3Ii8+IDwvcmRmOkRlc2NyaXB0aW9uPiA8L3JkZjpSREY+IDwveDp4bXBtZXRhPiA8P3hwYWNrZXQgZW5kPSJyIj8+VU513gAAADVJREFUeNrs0DENACAQBDBIWLGBJQby/mUcJn5sJXQmOQMAAAAAAJqt+2prAAAAAACg2xdgANk6BEVuJgyMAAAAAElFTkSuQmCC)}@media only screen and (max-width:767px){.ui.grid .stackable.row .ui.vertical.divider:after,.ui.grid .stackable.row .ui.vertical.divider:before,.ui.stackable.grid .ui.vertical.divider:after,.ui.stackable.grid .ui.vertical.divider:before{background-image:url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAABaAAAAACCAYAAACuTHuKAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAyFpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuNS1jMDE0IDc5LjE1MTQ4MSwgMjAxMy8wMy8xMy0xMjowOToxNSAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvIiB4bWxuczp4bXBNTT0iaHR0cDovL25zLmFkb2JlLmNvbS94YXAvMS4wL21tLyIgeG1sbnM6c3RSZWY9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9zVHlwZS9SZXNvdXJjZVJlZiMiIHhtcDpDcmVhdG9yVG9vbD0iQWRvYmUgUGhvdG9zaG9wIENDIChXaW5kb3dzKSIgeG1wTU06SW5zdGFuY2VJRD0ieG1wLmlpZDo1OThBRDY4OUNDMTYxMUU0OUE3NUVGOEJDMzMzMjE2NyIgeG1wTU06RG9jdW1lbnRJRD0ieG1wLmRpZDo1OThBRDY4QUNDMTYxMUU0OUE3NUVGOEJDMzMzMjE2NyI+IDx4bXBNTTpEZXJpdmVkRnJvbSBzdFJlZjppbnN0YW5jZUlEPSJ4bXAuaWlkOjU5OEFENjg3Q0MxNjExRTQ5QTc1RUY4QkMzMzMyMTY3IiBzdFJlZjpkb2N1bWVudElEPSJ4bXAuZGlkOjU5OEFENjg4Q0MxNjExRTQ5QTc1RUY4QkMzMzMyMTY3Ii8+IDwvcmRmOkRlc2NyaXB0aW9uPiA8L3JkZjpSREY+IDwveDp4bXBtZXRhPiA8P3hwYWNrZXQgZW5kPSJyIj8+VU513gAAADVJREFUeNrs0DENACAQBDBIWLGBJQby/mUcJn5sJXQmOQMAAAAAAJqt+2prAAAAAACg2xdgANk6BEVuJgyMAAAAAElFTkSuQmCC)}}/*! - * # Semantic UI 2.4.0 - Flag - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */i.flag:not(.icon){display:inline-block;width:16px;height:11px;line-height:11px;vertical-align:baseline;margin:0 .5em 0 0;text-decoration:inherit;speak:none;font-smoothing:antialiased;-webkit-backface-visibility:hidden;backface-visibility:hidden}i.flag:not(.icon):before{display:inline-block;content:'';background:url(themes/default/assets/images/flags.png) no-repeat -108px -1976px;width:16px;height:11px}i.flag.ad:before,i.flag.andorra:before{background-position:0 0}i.flag.ae:before,i.flag.uae:before,i.flag.united.arab.emirates:before{background-position:0 -26px}i.flag.af:before,i.flag.afghanistan:before{background-position:0 -52px}i.flag.ag:before,i.flag.antigua:before{background-position:0 -78px}i.flag.ai:before,i.flag.anguilla:before{background-position:0 -104px}i.flag.al:before,i.flag.albania:before{background-position:0 -130px}i.flag.am:before,i.flag.armenia:before{background-position:0 -156px}i.flag.an:before,i.flag.netherlands.antilles:before{background-position:0 -182px}i.flag.angola:before,i.flag.ao:before{background-position:0 -208px}i.flag.ar:before,i.flag.argentina:before{background-position:0 -234px}i.flag.american.samoa:before,i.flag.as:before{background-position:0 -260px}i.flag.at:before,i.flag.austria:before{background-position:0 -286px}i.flag.au:before,i.flag.australia:before{background-position:0 -312px}i.flag.aruba:before,i.flag.aw:before{background-position:0 -338px}i.flag.aland.islands:before,i.flag.ax:before{background-position:0 -364px}i.flag.az:before,i.flag.azerbaijan:before{background-position:0 -390px}i.flag.ba:before,i.flag.bosnia:before{background-position:0 -416px}i.flag.barbados:before,i.flag.bb:before{background-position:0 -442px}i.flag.bangladesh:before,i.flag.bd:before{background-position:0 -468px}i.flag.be:before,i.flag.belgium:before{background-position:0 -494px}i.flag.bf:before,i.flag.burkina.faso:before{background-position:0 -520px}i.flag.bg:before,i.flag.bulgaria:before{background-position:0 -546px}i.flag.bahrain:before,i.flag.bh:before{background-position:0 -572px}i.flag.bi:before,i.flag.burundi:before{background-position:0 -598px}i.flag.benin:before,i.flag.bj:before{background-position:0 -624px}i.flag.bermuda:before,i.flag.bm:before{background-position:0 -650px}i.flag.bn:before,i.flag.brunei:before{background-position:0 -676px}i.flag.bo:before,i.flag.bolivia:before{background-position:0 -702px}i.flag.br:before,i.flag.brazil:before{background-position:0 -728px}i.flag.bahamas:before,i.flag.bs:before{background-position:0 -754px}i.flag.bhutan:before,i.flag.bt:before{background-position:0 -780px}i.flag.bouvet.island:before,i.flag.bv:before{background-position:0 -806px}i.flag.botswana:before,i.flag.bw:before{background-position:0 -832px}i.flag.belarus:before,i.flag.by:before{background-position:0 -858px}i.flag.belize:before,i.flag.bz:before{background-position:0 -884px}i.flag.ca:before,i.flag.canada:before{background-position:0 -910px}i.flag.cc:before,i.flag.cocos.islands:before{background-position:0 -962px}i.flag.cd:before,i.flag.congo:before{background-position:0 -988px}i.flag.central.african.republic:before,i.flag.cf:before{background-position:0 -1014px}i.flag.cg:before,i.flag.congo.brazzaville:before{background-position:0 -1040px}i.flag.ch:before,i.flag.switzerland:before{background-position:0 -1066px}i.flag.ci:before,i.flag.cote.divoire:before{background-position:0 -1092px}i.flag.ck:before,i.flag.cook.islands:before{background-position:0 -1118px}i.flag.chile:before,i.flag.cl:before{background-position:0 -1144px}i.flag.cameroon:before,i.flag.cm:before{background-position:0 -1170px}i.flag.china:before,i.flag.cn:before{background-position:0 -1196px}i.flag.co:before,i.flag.colombia:before{background-position:0 -1222px}i.flag.costa.rica:before,i.flag.cr:before{background-position:0 -1248px}i.flag.cs:before,i.flag.serbia:before{background-position:0 -1274px}i.flag.cu:before,i.flag.cuba:before{background-position:0 -1300px}i.flag.cape.verde:before,i.flag.cv:before{background-position:0 -1326px}i.flag.christmas.island:before,i.flag.cx:before{background-position:0 -1352px}i.flag.cy:before,i.flag.cyprus:before{background-position:0 -1378px}i.flag.cz:before,i.flag.czech.republic:before{background-position:0 -1404px}i.flag.de:before,i.flag.germany:before{background-position:0 -1430px}i.flag.dj:before,i.flag.djibouti:before{background-position:0 -1456px}i.flag.denmark:before,i.flag.dk:before{background-position:0 -1482px}i.flag.dm:before,i.flag.dominica:before{background-position:0 -1508px}i.flag.do:before,i.flag.dominican.republic:before{background-position:0 -1534px}i.flag.algeria:before,i.flag.dz:before{background-position:0 -1560px}i.flag.ec:before,i.flag.ecuador:before{background-position:0 -1586px}i.flag.ee:before,i.flag.estonia:before{background-position:0 -1612px}i.flag.eg:before,i.flag.egypt:before{background-position:0 -1638px}i.flag.eh:before,i.flag.western.sahara:before{background-position:0 -1664px}i.flag.england:before,i.flag.gb.eng:before{background-position:0 -1690px}i.flag.er:before,i.flag.eritrea:before{background-position:0 -1716px}i.flag.es:before,i.flag.spain:before{background-position:0 -1742px}i.flag.et:before,i.flag.ethiopia:before{background-position:0 -1768px}i.flag.eu:before,i.flag.european.union:before{background-position:0 -1794px}i.flag.fi:before,i.flag.finland:before{background-position:0 -1846px}i.flag.fiji:before,i.flag.fj:before{background-position:0 -1872px}i.flag.falkland.islands:before,i.flag.fk:before{background-position:0 -1898px}i.flag.fm:before,i.flag.micronesia:before{background-position:0 -1924px}i.flag.faroe.islands:before,i.flag.fo:before{background-position:0 -1950px}i.flag.fr:before,i.flag.france:before{background-position:0 -1976px}i.flag.ga:before,i.flag.gabon:before{background-position:-36px 0}i.flag.gb:before,i.flag.uk:before,i.flag.united.kingdom:before{background-position:-36px -26px}i.flag.gd:before,i.flag.grenada:before{background-position:-36px -52px}i.flag.ge:before,i.flag.georgia:before{background-position:-36px -78px}i.flag.french.guiana:before,i.flag.gf:before{background-position:-36px -104px}i.flag.gh:before,i.flag.ghana:before{background-position:-36px -130px}i.flag.gi:before,i.flag.gibraltar:before{background-position:-36px -156px}i.flag.gl:before,i.flag.greenland:before{background-position:-36px -182px}i.flag.gambia:before,i.flag.gm:before{background-position:-36px -208px}i.flag.gn:before,i.flag.guinea:before{background-position:-36px -234px}i.flag.gp:before,i.flag.guadeloupe:before{background-position:-36px -260px}i.flag.equatorial.guinea:before,i.flag.gq:before{background-position:-36px -286px}i.flag.gr:before,i.flag.greece:before{background-position:-36px -312px}i.flag.gs:before,i.flag.sandwich.islands:before{background-position:-36px -338px}i.flag.gt:before,i.flag.guatemala:before{background-position:-36px -364px}i.flag.gu:before,i.flag.guam:before{background-position:-36px -390px}i.flag.guinea-bissau:before,i.flag.gw:before{background-position:-36px -416px}i.flag.guyana:before,i.flag.gy:before{background-position:-36px -442px}i.flag.hk:before,i.flag.hong.kong:before{background-position:-36px -468px}i.flag.heard.island:before,i.flag.hm:before{background-position:-36px -494px}i.flag.hn:before,i.flag.honduras:before{background-position:-36px -520px}i.flag.croatia:before,i.flag.hr:before{background-position:-36px -546px}i.flag.haiti:before,i.flag.ht:before{background-position:-36px -572px}i.flag.hu:before,i.flag.hungary:before{background-position:-36px -598px}i.flag.id:before,i.flag.indonesia:before{background-position:-36px -624px}i.flag.ie:before,i.flag.ireland:before{background-position:-36px -650px}i.flag.il:before,i.flag.israel:before{background-position:-36px -676px}i.flag.in:before,i.flag.india:before{background-position:-36px -702px}i.flag.indian.ocean.territory:before,i.flag.io:before{background-position:-36px -728px}i.flag.iq:before,i.flag.iraq:before{background-position:-36px -754px}i.flag.ir:before,i.flag.iran:before{background-position:-36px -780px}i.flag.iceland:before,i.flag.is:before{background-position:-36px -806px}i.flag.it:before,i.flag.italy:before{background-position:-36px -832px}i.flag.jamaica:before,i.flag.jm:before{background-position:-36px -858px}i.flag.jo:before,i.flag.jordan:before{background-position:-36px -884px}i.flag.japan:before,i.flag.jp:before{background-position:-36px -910px}i.flag.ke:before,i.flag.kenya:before{background-position:-36px -936px}i.flag.kg:before,i.flag.kyrgyzstan:before{background-position:-36px -962px}i.flag.cambodia:before,i.flag.kh:before{background-position:-36px -988px}i.flag.ki:before,i.flag.kiribati:before{background-position:-36px -1014px}i.flag.comoros:before,i.flag.km:before{background-position:-36px -1040px}i.flag.kn:before,i.flag.saint.kitts.and.nevis:before{background-position:-36px -1066px}i.flag.kp:before,i.flag.north.korea:before{background-position:-36px -1092px}i.flag.kr:before,i.flag.south.korea:before{background-position:-36px -1118px}i.flag.kuwait:before,i.flag.kw:before{background-position:-36px -1144px}i.flag.cayman.islands:before,i.flag.ky:before{background-position:-36px -1170px}i.flag.kazakhstan:before,i.flag.kz:before{background-position:-36px -1196px}i.flag.la:before,i.flag.laos:before{background-position:-36px -1222px}i.flag.lb:before,i.flag.lebanon:before{background-position:-36px -1248px}i.flag.lc:before,i.flag.saint.lucia:before{background-position:-36px -1274px}i.flag.li:before,i.flag.liechtenstein:before{background-position:-36px -1300px}i.flag.lk:before,i.flag.sri.lanka:before{background-position:-36px -1326px}i.flag.liberia:before,i.flag.lr:before{background-position:-36px -1352px}i.flag.lesotho:before,i.flag.ls:before{background-position:-36px -1378px}i.flag.lithuania:before,i.flag.lt:before{background-position:-36px -1404px}i.flag.lu:before,i.flag.luxembourg:before{background-position:-36px -1430px}i.flag.latvia:before,i.flag.lv:before{background-position:-36px -1456px}i.flag.libya:before,i.flag.ly:before{background-position:-36px -1482px}i.flag.ma:before,i.flag.morocco:before{background-position:-36px -1508px}i.flag.mc:before,i.flag.monaco:before{background-position:-36px -1534px}i.flag.md:before,i.flag.moldova:before{background-position:-36px -1560px}i.flag.me:before,i.flag.montenegro:before{background-position:-36px -1586px}i.flag.madagascar:before,i.flag.mg:before{background-position:-36px -1613px}i.flag.marshall.islands:before,i.flag.mh:before{background-position:-36px -1639px}i.flag.macedonia:before,i.flag.mk:before{background-position:-36px -1665px}i.flag.mali:before,i.flag.ml:before{background-position:-36px -1691px}i.flag.burma:before,i.flag.mm:before,i.flag.myanmar:before{background-position:-73px -1821px}i.flag.mn:before,i.flag.mongolia:before{background-position:-36px -1743px}i.flag.macau:before,i.flag.mo:before{background-position:-36px -1769px}i.flag.mp:before,i.flag.northern.mariana.islands:before{background-position:-36px -1795px}i.flag.martinique:before,i.flag.mq:before{background-position:-36px -1821px}i.flag.mauritania:before,i.flag.mr:before{background-position:-36px -1847px}i.flag.montserrat:before,i.flag.ms:before{background-position:-36px -1873px}i.flag.malta:before,i.flag.mt:before{background-position:-36px -1899px}i.flag.mauritius:before,i.flag.mu:before{background-position:-36px -1925px}i.flag.maldives:before,i.flag.mv:before{background-position:-36px -1951px}i.flag.malawi:before,i.flag.mw:before{background-position:-36px -1977px}i.flag.mexico:before,i.flag.mx:before{background-position:-72px 0}i.flag.malaysia:before,i.flag.my:before{background-position:-72px -26px}i.flag.mozambique:before,i.flag.mz:before{background-position:-72px -52px}i.flag.na:before,i.flag.namibia:before{background-position:-72px -78px}i.flag.nc:before,i.flag.new.caledonia:before{background-position:-72px -104px}i.flag.ne:before,i.flag.niger:before{background-position:-72px -130px}i.flag.nf:before,i.flag.norfolk.island:before{background-position:-72px -156px}i.flag.ng:before,i.flag.nigeria:before{background-position:-72px -182px}i.flag.ni:before,i.flag.nicaragua:before{background-position:-72px -208px}i.flag.netherlands:before,i.flag.nl:before{background-position:-72px -234px}i.flag.no:before,i.flag.norway:before{background-position:-72px -260px}i.flag.nepal:before,i.flag.np:before{background-position:-72px -286px}i.flag.nauru:before,i.flag.nr:before{background-position:-72px -312px}i.flag.niue:before,i.flag.nu:before{background-position:-72px -338px}i.flag.new.zealand:before,i.flag.nz:before{background-position:-72px -364px}i.flag.om:before,i.flag.oman:before{background-position:-72px -390px}i.flag.pa:before,i.flag.panama:before{background-position:-72px -416px}i.flag.pe:before,i.flag.peru:before{background-position:-72px -442px}i.flag.french.polynesia:before,i.flag.pf:before{background-position:-72px -468px}i.flag.new.guinea:before,i.flag.pg:before{background-position:-72px -494px}i.flag.ph:before,i.flag.philippines:before{background-position:-72px -520px}i.flag.pakistan:before,i.flag.pk:before{background-position:-72px -546px}i.flag.pl:before,i.flag.poland:before{background-position:-72px -572px}i.flag.pm:before,i.flag.saint.pierre:before{background-position:-72px -598px}i.flag.pitcairn.islands:before,i.flag.pn:before{background-position:-72px -624px}i.flag.pr:before,i.flag.puerto.rico:before{background-position:-72px -650px}i.flag.palestine:before,i.flag.ps:before{background-position:-72px -676px}i.flag.portugal:before,i.flag.pt:before{background-position:-72px -702px}i.flag.palau:before,i.flag.pw:before{background-position:-72px -728px}i.flag.paraguay:before,i.flag.py:before{background-position:-72px -754px}i.flag.qa:before,i.flag.qatar:before{background-position:-72px -780px}i.flag.re:before,i.flag.reunion:before{background-position:-72px -806px}i.flag.ro:before,i.flag.romania:before{background-position:-72px -832px}i.flag.rs:before,i.flag.serbia:before{background-position:-72px -858px}i.flag.ru:before,i.flag.russia:before{background-position:-72px -884px}i.flag.rw:before,i.flag.rwanda:before{background-position:-72px -910px}i.flag.sa:before,i.flag.saudi.arabia:before{background-position:-72px -936px}i.flag.sb:before,i.flag.solomon.islands:before{background-position:-72px -962px}i.flag.sc:before,i.flag.seychelles:before{background-position:-72px -988px}i.flag.gb.sct:before,i.flag.scotland:before{background-position:-72px -1014px}i.flag.sd:before,i.flag.sudan:before{background-position:-72px -1040px}i.flag.se:before,i.flag.sweden:before{background-position:-72px -1066px}i.flag.sg:before,i.flag.singapore:before{background-position:-72px -1092px}i.flag.saint.helena:before,i.flag.sh:before{background-position:-72px -1118px}i.flag.si:before,i.flag.slovenia:before{background-position:-72px -1144px}i.flag.jan.mayen:before,i.flag.sj:before,i.flag.svalbard:before{background-position:-72px -1170px}i.flag.sk:before,i.flag.slovakia:before{background-position:-72px -1196px}i.flag.sierra.leone:before,i.flag.sl:before{background-position:-72px -1222px}i.flag.san.marino:before,i.flag.sm:before{background-position:-72px -1248px}i.flag.senegal:before,i.flag.sn:before{background-position:-72px -1274px}i.flag.so:before,i.flag.somalia:before{background-position:-72px -1300px}i.flag.sr:before,i.flag.suriname:before{background-position:-72px -1326px}i.flag.sao.tome:before,i.flag.st:before{background-position:-72px -1352px}i.flag.el.salvador:before,i.flag.sv:before{background-position:-72px -1378px}i.flag.sy:before,i.flag.syria:before{background-position:-72px -1404px}i.flag.swaziland:before,i.flag.sz:before{background-position:-72px -1430px}i.flag.caicos.islands:before,i.flag.tc:before{background-position:-72px -1456px}i.flag.chad:before,i.flag.td:before{background-position:-72px -1482px}i.flag.french.territories:before,i.flag.tf:before{background-position:-72px -1508px}i.flag.tg:before,i.flag.togo:before{background-position:-72px -1534px}i.flag.th:before,i.flag.thailand:before{background-position:-72px -1560px}i.flag.tajikistan:before,i.flag.tj:before{background-position:-72px -1586px}i.flag.tk:before,i.flag.tokelau:before{background-position:-72px -1612px}i.flag.timorleste:before,i.flag.tl:before{background-position:-72px -1638px}i.flag.tm:before,i.flag.turkmenistan:before{background-position:-72px -1664px}i.flag.tn:before,i.flag.tunisia:before{background-position:-72px -1690px}i.flag.to:before,i.flag.tonga:before{background-position:-72px -1716px}i.flag.tr:before,i.flag.turkey:before{background-position:-72px -1742px}i.flag.trinidad:before,i.flag.tt:before{background-position:-72px -1768px}i.flag.tuvalu:before,i.flag.tv:before{background-position:-72px -1794px}i.flag.taiwan:before,i.flag.tw:before{background-position:-72px -1820px}i.flag.tanzania:before,i.flag.tz:before{background-position:-72px -1846px}i.flag.ua:before,i.flag.ukraine:before{background-position:-72px -1872px}i.flag.ug:before,i.flag.uganda:before{background-position:-72px -1898px}i.flag.um:before,i.flag.us.minor.islands:before{background-position:-72px -1924px}i.flag.america:before,i.flag.united.states:before,i.flag.us:before{background-position:-72px -1950px}i.flag.uruguay:before,i.flag.uy:before{background-position:-72px -1976px}i.flag.uz:before,i.flag.uzbekistan:before{background-position:-108px 0}i.flag.va:before,i.flag.vatican.city:before{background-position:-108px -26px}i.flag.saint.vincent:before,i.flag.vc:before{background-position:-108px -52px}i.flag.ve:before,i.flag.venezuela:before{background-position:-108px -78px}i.flag.british.virgin.islands:before,i.flag.vg:before{background-position:-108px -104px}i.flag.us.virgin.islands:before,i.flag.vi:before{background-position:-108px -130px}i.flag.vietnam:before,i.flag.vn:before{background-position:-108px -156px}i.flag.vanuatu:before,i.flag.vu:before{background-position:-108px -182px}i.flag.gb.wls:before,i.flag.wales:before{background-position:-108px -208px}i.flag.wallis.and.futuna:before,i.flag.wf:before{background-position:-108px -234px}i.flag.samoa:before,i.flag.ws:before{background-position:-108px -260px}i.flag.ye:before,i.flag.yemen:before{background-position:-108px -286px}i.flag.mayotte:before,i.flag.yt:before{background-position:-108px -312px}i.flag.south.africa:before,i.flag.za:before{background-position:-108px -338px}i.flag.zambia:before,i.flag.zm:before{background-position:-108px -364px}i.flag.zimbabwe:before,i.flag.zw:before{background-position:-108px -390px}/*! - * # Semantic UI 2.4.0 - Header - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */.ui.header{border:none;margin:calc(2rem - .14285714em) 0 1rem;padding:0 0;font-family:Lato,'Helvetica Neue',Arial,Helvetica,sans-serif;font-weight:700;line-height:1.28571429em;text-transform:none;color:rgba(0,0,0,.87)}.ui.header:first-child{margin-top:-.14285714em}.ui.header:last-child{margin-bottom:0}.ui.header .sub.header{display:block;font-weight:400;padding:0;margin:0;font-size:1rem;line-height:1.2em;color:rgba(0,0,0,.6)}.ui.header>.icon{display:table-cell;opacity:1;font-size:1.5em;padding-top:0;vertical-align:middle}.ui.header .icon:only-child{display:inline-block;padding:0;margin-right:.75rem}.ui.header>.image:not(.icon),.ui.header>img{display:inline-block;margin-top:.14285714em;width:2.5em;height:auto;vertical-align:middle}.ui.header>.image:not(.icon):only-child,.ui.header>img:only-child{margin-right:.75rem}.ui.header .content{display:inline-block;vertical-align:top}.ui.header>.image+.content,.ui.header>img+.content{padding-left:.75rem;vertical-align:middle}.ui.header>.icon+.content{padding-left:.75rem;display:table-cell;vertical-align:middle}.ui.header .ui.label{font-size:'';margin-left:.5rem;vertical-align:middle}.ui.header+p{margin-top:0}h1.ui.header{font-size:2rem}h2.ui.header{font-size:1.71428571rem}h3.ui.header{font-size:1.28571429rem}h4.ui.header{font-size:1.07142857rem}h5.ui.header{font-size:1rem}h1.ui.header .sub.header{font-size:1.14285714rem}h2.ui.header .sub.header{font-size:1.14285714rem}h3.ui.header .sub.header{font-size:1rem}h4.ui.header .sub.header{font-size:1rem}h5.ui.header .sub.header{font-size:.92857143rem}.ui.huge.header{min-height:1em;font-size:2em}.ui.large.header{font-size:1.71428571em}.ui.medium.header{font-size:1.28571429em}.ui.small.header{font-size:1.07142857em}.ui.tiny.header{font-size:1em}.ui.huge.header .sub.header{font-size:1.14285714rem}.ui.large.header .sub.header{font-size:1.14285714rem}.ui.header .sub.header{font-size:1rem}.ui.small.header .sub.header{font-size:1rem}.ui.tiny.header .sub.header{font-size:.92857143rem}.ui.sub.header{padding:0;margin-bottom:.14285714rem;font-weight:700;font-size:.85714286em;text-transform:uppercase;color:''}.ui.small.sub.header{font-size:.78571429em}.ui.sub.header{font-size:.85714286em}.ui.large.sub.header{font-size:.92857143em}.ui.huge.sub.header{font-size:1em}.ui.icon.header{display:inline-block;text-align:center;margin:2rem 0 1rem}.ui.icon.header:after{content:'';display:block;height:0;clear:both;visibility:hidden}.ui.icon.header:first-child{margin-top:0}.ui.icon.header .icon{float:none;display:block;width:auto;height:auto;line-height:1;padding:0;font-size:3em;margin:0 auto .5rem;opacity:1}.ui.icon.header .content{display:block;padding:0}.ui.icon.header .circular.icon{font-size:2em}.ui.icon.header .square.icon{font-size:2em}.ui.block.icon.header .icon{margin-bottom:0}.ui.icon.header.aligned{margin-left:auto;margin-right:auto;display:block}.ui.disabled.header{opacity:.45}.ui.inverted.header{color:#fff}.ui.inverted.header .sub.header{color:rgba(255,255,255,.8)}.ui.inverted.attached.header{background:#545454 -webkit-gradient(linear,left top,left bottom,from(transparent),to(rgba(0,0,0,.05)));background:#545454 -webkit-linear-gradient(transparent,rgba(0,0,0,.05));background:#545454 linear-gradient(transparent,rgba(0,0,0,.05));-webkit-box-shadow:none;box-shadow:none;border-color:transparent}.ui.inverted.block.header{background:#545454 -webkit-gradient(linear,left top,left bottom,from(transparent),to(rgba(0,0,0,.05)));background:#545454 -webkit-linear-gradient(transparent,rgba(0,0,0,.05));background:#545454 linear-gradient(transparent,rgba(0,0,0,.05));-webkit-box-shadow:none;box-shadow:none}.ui.inverted.block.header{border-bottom:none}.ui.red.header{color:#db2828!important}a.ui.red.header:hover{color:#d01919!important}.ui.red.dividing.header{border-bottom:2px solid #db2828}.ui.inverted.red.header{color:#ff695e!important}a.ui.inverted.red.header:hover{color:#ff5144!important}.ui.orange.header{color:#f2711c!important}a.ui.orange.header:hover{color:#f26202!important}.ui.orange.dividing.header{border-bottom:2px solid #f2711c}.ui.inverted.orange.header{color:#ff851b!important}a.ui.inverted.orange.header:hover{color:#ff7701!important}.ui.olive.header{color:#b5cc18!important}a.ui.olive.header:hover{color:#a7bd0d!important}.ui.olive.dividing.header{border-bottom:2px solid #b5cc18}.ui.inverted.olive.header{color:#d9e778!important}a.ui.inverted.olive.header:hover{color:#d8ea5c!important}.ui.yellow.header{color:#fbbd08!important}a.ui.yellow.header:hover{color:#eaae00!important}.ui.yellow.dividing.header{border-bottom:2px solid #fbbd08}.ui.inverted.yellow.header{color:#ffe21f!important}a.ui.inverted.yellow.header:hover{color:#ffdf05!important}.ui.green.header{color:#21ba45!important}a.ui.green.header:hover{color:#16ab39!important}.ui.green.dividing.header{border-bottom:2px solid #21ba45}.ui.inverted.green.header{color:#2ecc40!important}a.ui.inverted.green.header:hover{color:#22be34!important}.ui.teal.header{color:#00b5ad!important}a.ui.teal.header:hover{color:#009c95!important}.ui.teal.dividing.header{border-bottom:2px solid #00b5ad}.ui.inverted.teal.header{color:#6dffff!important}a.ui.inverted.teal.header:hover{color:#54ffff!important}.ui.blue.header{color:#2185d0!important}a.ui.blue.header:hover{color:#1678c2!important}.ui.blue.dividing.header{border-bottom:2px solid #2185d0}.ui.inverted.blue.header{color:#54c8ff!important}a.ui.inverted.blue.header:hover{color:#3ac0ff!important}.ui.violet.header{color:#6435c9!important}a.ui.violet.header:hover{color:#5829bb!important}.ui.violet.dividing.header{border-bottom:2px solid #6435c9}.ui.inverted.violet.header{color:#a291fb!important}a.ui.inverted.violet.header:hover{color:#8a73ff!important}.ui.purple.header{color:#a333c8!important}a.ui.purple.header:hover{color:#9627ba!important}.ui.purple.dividing.header{border-bottom:2px solid #a333c8}.ui.inverted.purple.header{color:#dc73ff!important}a.ui.inverted.purple.header:hover{color:#d65aff!important}.ui.pink.header{color:#e03997!important}a.ui.pink.header:hover{color:#e61a8d!important}.ui.pink.dividing.header{border-bottom:2px solid #e03997}.ui.inverted.pink.header{color:#ff8edf!important}a.ui.inverted.pink.header:hover{color:#ff74d8!important}.ui.brown.header{color:#a5673f!important}a.ui.brown.header:hover{color:#975b33!important}.ui.brown.dividing.header{border-bottom:2px solid #a5673f}.ui.inverted.brown.header{color:#d67c1c!important}a.ui.inverted.brown.header:hover{color:#c86f11!important}.ui.grey.header{color:#767676!important}a.ui.grey.header:hover{color:#838383!important}.ui.grey.dividing.header{border-bottom:2px solid #767676}.ui.inverted.grey.header{color:#dcddde!important}a.ui.inverted.grey.header:hover{color:#cfd0d2!important}.ui.left.aligned.header{text-align:left}.ui.right.aligned.header{text-align:right}.ui.center.aligned.header,.ui.centered.header{text-align:center}.ui.justified.header{text-align:justify}.ui.justified.header:after{display:inline-block;content:'';width:100%}.ui.floated.header,.ui[class*="left floated"].header{float:left;margin-top:0;margin-right:.5em}.ui[class*="right floated"].header{float:right;margin-top:0;margin-left:.5em}.ui.fitted.header{padding:0}.ui.dividing.header{padding-bottom:.21428571rem;border-bottom:1px solid rgba(34,36,38,.15)}.ui.dividing.header .sub.header{padding-bottom:.21428571rem}.ui.dividing.header .icon{margin-bottom:0}.ui.inverted.dividing.header{border-bottom-color:rgba(255,255,255,.1)}.ui.block.header{background:#f3f4f5;padding:.78571429rem 1rem;-webkit-box-shadow:none;box-shadow:none;border:1px solid #d4d4d5;border-radius:.28571429rem}.ui.tiny.block.header{font-size:.85714286rem}.ui.small.block.header{font-size:.92857143rem}.ui.block.header:not(h1):not(h2):not(h3):not(h4):not(h5):not(h6){font-size:1rem}.ui.large.block.header{font-size:1.14285714rem}.ui.huge.block.header{font-size:1.42857143rem}.ui.attached.header{background:#fff;padding:.78571429rem 1rem;margin-left:-1px;margin-right:-1px;-webkit-box-shadow:none;box-shadow:none;border:1px solid #d4d4d5}.ui.attached.block.header{background:#f3f4f5}.ui.attached:not(.top):not(.bottom).header{margin-top:0;margin-bottom:0;border-top:none;border-radius:0}.ui.top.attached.header{margin-bottom:0;border-radius:.28571429rem .28571429rem 0 0}.ui.bottom.attached.header{margin-top:0;border-top:none;border-radius:0 0 .28571429rem .28571429rem}.ui.tiny.attached.header{font-size:.85714286em}.ui.small.attached.header{font-size:.92857143em}.ui.attached.header:not(h1):not(h2):not(h3):not(h4):not(h5):not(h6){font-size:1em}.ui.large.attached.header{font-size:1.14285714em}.ui.huge.attached.header{font-size:1.42857143em}.ui.header:not(h1):not(h2):not(h3):not(h4):not(h5):not(h6){font-size:1.28571429em}/*! - * # Semantic UI 2.4.0 - Icon - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */@font-face{font-family:Icons;src:url(themes/default/assets/fonts/icons.eot);src:url(themes/default/assets/fonts/icons.eot?#iefix) format('embedded-opentype'),url(themes/default/assets/fonts/icons.woff2) format('woff2'),url(themes/default/assets/fonts/icons.woff) format('woff'),url(themes/default/assets/fonts/icons.ttf) format('truetype'),url(themes/default/assets/fonts/icons.svg#icons) format('svg');font-style:normal;font-weight:400;font-variant:normal;text-decoration:inherit;text-transform:none}i.icon{display:inline-block;opacity:1;margin:0 .25rem 0 0;width:1.18em;height:1em;font-family:Icons;font-style:normal;font-weight:400;text-decoration:inherit;text-align:center;speak:none;font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale;-webkit-font-smoothing:antialiased;-webkit-backface-visibility:hidden;backface-visibility:hidden}i.icon:before{background:0 0!important}i.icon.loading{height:1em;line-height:1;-webkit-animation:icon-loading 2s linear infinite;animation:icon-loading 2s linear infinite}@-webkit-keyframes icon-loading{from{-webkit-transform:rotate(0);transform:rotate(0)}to{-webkit-transform:rotate(360deg);transform:rotate(360deg)}}@keyframes icon-loading{from{-webkit-transform:rotate(0);transform:rotate(0)}to{-webkit-transform:rotate(360deg);transform:rotate(360deg)}}i.icon.hover{opacity:1!important}i.icon.active{opacity:1!important}i.emphasized.icon{opacity:1!important}i.disabled.icon{opacity:.45!important}i.fitted.icon{width:auto;margin:0!important}i.link.icon,i.link.icons{cursor:pointer;opacity:.8;-webkit-transition:opacity .1s ease;transition:opacity .1s ease}i.link.icon:hover,i.link.icons:hover{opacity:1!important}i.circular.icon{border-radius:500em!important;line-height:1!important;padding:.5em 0!important;-webkit-box-shadow:0 0 0 .1em rgba(0,0,0,.1) inset;box-shadow:0 0 0 .1em rgba(0,0,0,.1) inset;width:2em!important;height:2em!important}i.circular.inverted.icon{border:none;-webkit-box-shadow:none;box-shadow:none}i.flipped.icon,i.horizontally.flipped.icon{-webkit-transform:scale(-1,1);transform:scale(-1,1)}i.vertically.flipped.icon{-webkit-transform:scale(1,-1);transform:scale(1,-1)}i.clockwise.rotated.icon,i.right.rotated.icon,i.rotated.icon{-webkit-transform:rotate(90deg);transform:rotate(90deg)}i.counterclockwise.rotated.icon,i.left.rotated.icon{-webkit-transform:rotate(-90deg);transform:rotate(-90deg)}i.bordered.icon{line-height:1;vertical-align:baseline;width:2em;height:2em;padding:.5em 0!important;-webkit-box-shadow:0 0 0 .1em rgba(0,0,0,.1) inset;box-shadow:0 0 0 .1em rgba(0,0,0,.1) inset}i.bordered.inverted.icon{border:none;-webkit-box-shadow:none;box-shadow:none}i.inverted.bordered.icon,i.inverted.circular.icon{background-color:#1b1c1d!important;color:#fff!important}i.inverted.icon{color:#fff}i.red.icon{color:#db2828!important}i.inverted.red.icon{color:#ff695e!important}i.inverted.bordered.red.icon,i.inverted.circular.red.icon{background-color:#db2828!important;color:#fff!important}i.orange.icon{color:#f2711c!important}i.inverted.orange.icon{color:#ff851b!important}i.inverted.bordered.orange.icon,i.inverted.circular.orange.icon{background-color:#f2711c!important;color:#fff!important}i.yellow.icon{color:#fbbd08!important}i.inverted.yellow.icon{color:#ffe21f!important}i.inverted.bordered.yellow.icon,i.inverted.circular.yellow.icon{background-color:#fbbd08!important;color:#fff!important}i.olive.icon{color:#b5cc18!important}i.inverted.olive.icon{color:#d9e778!important}i.inverted.bordered.olive.icon,i.inverted.circular.olive.icon{background-color:#b5cc18!important;color:#fff!important}i.green.icon{color:#21ba45!important}i.inverted.green.icon{color:#2ecc40!important}i.inverted.bordered.green.icon,i.inverted.circular.green.icon{background-color:#21ba45!important;color:#fff!important}i.teal.icon{color:#00b5ad!important}i.inverted.teal.icon{color:#6dffff!important}i.inverted.bordered.teal.icon,i.inverted.circular.teal.icon{background-color:#00b5ad!important;color:#fff!important}i.blue.icon{color:#2185d0!important}i.inverted.blue.icon{color:#54c8ff!important}i.inverted.bordered.blue.icon,i.inverted.circular.blue.icon{background-color:#2185d0!important;color:#fff!important}i.violet.icon{color:#6435c9!important}i.inverted.violet.icon{color:#a291fb!important}i.inverted.bordered.violet.icon,i.inverted.circular.violet.icon{background-color:#6435c9!important;color:#fff!important}i.purple.icon{color:#a333c8!important}i.inverted.purple.icon{color:#dc73ff!important}i.inverted.bordered.purple.icon,i.inverted.circular.purple.icon{background-color:#a333c8!important;color:#fff!important}i.pink.icon{color:#e03997!important}i.inverted.pink.icon{color:#ff8edf!important}i.inverted.bordered.pink.icon,i.inverted.circular.pink.icon{background-color:#e03997!important;color:#fff!important}i.brown.icon{color:#a5673f!important}i.inverted.brown.icon{color:#d67c1c!important}i.inverted.bordered.brown.icon,i.inverted.circular.brown.icon{background-color:#a5673f!important;color:#fff!important}i.grey.icon{color:#767676!important}i.inverted.grey.icon{color:#dcddde!important}i.inverted.bordered.grey.icon,i.inverted.circular.grey.icon{background-color:#767676!important;color:#fff!important}i.black.icon{color:#1b1c1d!important}i.inverted.black.icon{color:#545454!important}i.inverted.bordered.black.icon,i.inverted.circular.black.icon{background-color:#1b1c1d!important;color:#fff!important}i.mini.icon,i.mini.icons{line-height:1;font-size:.4em}i.tiny.icon,i.tiny.icons{line-height:1;font-size:.5em}i.small.icon,i.small.icons{line-height:1;font-size:.75em}i.icon,i.icons{font-size:1em}i.large.icon,i.large.icons{line-height:1;vertical-align:middle;font-size:1.5em}i.big.icon,i.big.icons{line-height:1;vertical-align:middle;font-size:2em}i.huge.icon,i.huge.icons{line-height:1;vertical-align:middle;font-size:4em}i.massive.icon,i.massive.icons{line-height:1;vertical-align:middle;font-size:8em}i.icons{display:inline-block;position:relative;line-height:1}i.icons .icon{position:absolute;top:50%;left:50%;-webkit-transform:translateX(-50%) translateY(-50%);transform:translateX(-50%) translateY(-50%);margin:0;margin:0}i.icons .icon:first-child{position:static;width:auto;height:auto;vertical-align:top;-webkit-transform:none;transform:none;margin-right:.25rem}i.icons .corner.icon{top:auto;left:auto;right:0;bottom:0;-webkit-transform:none;transform:none;font-size:.45em;text-shadow:-1px -1px 0 #fff,1px -1px 0 #fff,-1px 1px 0 #fff,1px 1px 0 #fff}i.icons .top.right.corner.icon{top:0;left:auto;right:0;bottom:auto}i.icons .top.left.corner.icon{top:0;left:0;right:auto;bottom:auto}i.icons .bottom.left.corner.icon{top:auto;left:0;right:auto;bottom:0}i.icons .bottom.right.corner.icon{top:auto;left:auto;right:0;bottom:0}i.icons .inverted.corner.icon{text-shadow:-1px -1px 0 #1b1c1d,1px -1px 0 #1b1c1d,-1px 1px 0 #1b1c1d,1px 1px 0 #1b1c1d}i.icon.linkedin.in:before{content:"\f0e1"}i.icon.zoom.in:before{content:"\f00e"}i.icon.zoom.out:before{content:"\f010"}i.icon.sign.in:before{content:"\f2f6"}i.icon.in.cart:before{content:"\f218"}i.icon.log.out:before{content:"\f2f5"}i.icon.sign.out:before{content:"\f2f5"}i.icon.\35 00px:before{content:"\f26e"}i.icon.accessible.icon:before{content:"\f368"}i.icon.accusoft:before{content:"\f369"}i.icon.address.book:before{content:"\f2b9"}i.icon.address.card:before{content:"\f2bb"}i.icon.adjust:before{content:"\f042"}i.icon.adn:before{content:"\f170"}i.icon.adversal:before{content:"\f36a"}i.icon.affiliatetheme:before{content:"\f36b"}i.icon.algolia:before{content:"\f36c"}i.icon.align.center:before{content:"\f037"}i.icon.align.justify:before{content:"\f039"}i.icon.align.left:before{content:"\f036"}i.icon.align.right:before{content:"\f038"}i.icon.amazon:before{content:"\f270"}i.icon.amazon.pay:before{content:"\f42c"}i.icon.ambulance:before{content:"\f0f9"}i.icon.american.sign.language.interpreting:before{content:"\f2a3"}i.icon.amilia:before{content:"\f36d"}i.icon.anchor:before{content:"\f13d"}i.icon.android:before{content:"\f17b"}i.icon.angellist:before{content:"\f209"}i.icon.angle.double.down:before{content:"\f103"}i.icon.angle.double.left:before{content:"\f100"}i.icon.angle.double.right:before{content:"\f101"}i.icon.angle.double.up:before{content:"\f102"}i.icon.angle.down:before{content:"\f107"}i.icon.angle.left:before{content:"\f104"}i.icon.angle.right:before{content:"\f105"}i.icon.angle.up:before{content:"\f106"}i.icon.angrycreative:before{content:"\f36e"}i.icon.angular:before{content:"\f420"}i.icon.app.store:before{content:"\f36f"}i.icon.app.store.ios:before{content:"\f370"}i.icon.apper:before{content:"\f371"}i.icon.apple:before{content:"\f179"}i.icon.apple.pay:before{content:"\f415"}i.icon.archive:before{content:"\f187"}i.icon.arrow.alternate.circle.down:before{content:"\f358"}i.icon.arrow.alternate.circle.left:before{content:"\f359"}i.icon.arrow.alternate.circle.right:before{content:"\f35a"}i.icon.arrow.alternate.circle.up:before{content:"\f35b"}i.icon.arrow.circle.down:before{content:"\f0ab"}i.icon.arrow.circle.left:before{content:"\f0a8"}i.icon.arrow.circle.right:before{content:"\f0a9"}i.icon.arrow.circle.up:before{content:"\f0aa"}i.icon.arrow.down:before{content:"\f063"}i.icon.arrow.left:before{content:"\f060"}i.icon.arrow.right:before{content:"\f061"}i.icon.arrow.up:before{content:"\f062"}i.icon.arrows.alternate:before{content:"\f0b2"}i.icon.arrows.alternate.horizontal:before{content:"\f337"}i.icon.arrows.alternate.vertical:before{content:"\f338"}i.icon.assistive.listening.systems:before{content:"\f2a2"}i.icon.asterisk:before{content:"\f069"}i.icon.asymmetrik:before{content:"\f372"}i.icon.at:before{content:"\f1fa"}i.icon.audible:before{content:"\f373"}i.icon.audio.description:before{content:"\f29e"}i.icon.autoprefixer:before{content:"\f41c"}i.icon.avianex:before{content:"\f374"}i.icon.aviato:before{content:"\f421"}i.icon.aws:before{content:"\f375"}i.icon.backward:before{content:"\f04a"}i.icon.balance.scale:before{content:"\f24e"}i.icon.ban:before{content:"\f05e"}i.icon.band.aid:before{content:"\f462"}i.icon.bandcamp:before{content:"\f2d5"}i.icon.barcode:before{content:"\f02a"}i.icon.bars:before{content:"\f0c9"}i.icon.baseball.ball:before{content:"\f433"}i.icon.basketball.ball:before{content:"\f434"}i.icon.bath:before{content:"\f2cd"}i.icon.battery.empty:before{content:"\f244"}i.icon.battery.full:before{content:"\f240"}i.icon.battery.half:before{content:"\f242"}i.icon.battery.quarter:before{content:"\f243"}i.icon.battery.three.quarters:before{content:"\f241"}i.icon.bed:before{content:"\f236"}i.icon.beer:before{content:"\f0fc"}i.icon.behance:before{content:"\f1b4"}i.icon.behance.square:before{content:"\f1b5"}i.icon.bell:before{content:"\f0f3"}i.icon.bell.slash:before{content:"\f1f6"}i.icon.bicycle:before{content:"\f206"}i.icon.bimobject:before{content:"\f378"}i.icon.binoculars:before{content:"\f1e5"}i.icon.birthday.cake:before{content:"\f1fd"}i.icon.bitbucket:before{content:"\f171"}i.icon.bitcoin:before{content:"\f379"}i.icon.bity:before{content:"\f37a"}i.icon.black.tie:before{content:"\f27e"}i.icon.blackberry:before{content:"\f37b"}i.icon.blind:before{content:"\f29d"}i.icon.blogger:before{content:"\f37c"}i.icon.blogger.b:before{content:"\f37d"}i.icon.bluetooth:before{content:"\f293"}i.icon.bluetooth.b:before{content:"\f294"}i.icon.bold:before{content:"\f032"}i.icon.bolt:before{content:"\f0e7"}i.icon.bomb:before{content:"\f1e2"}i.icon.book:before{content:"\f02d"}i.icon.bookmark:before{content:"\f02e"}i.icon.bowling.ball:before{content:"\f436"}i.icon.box:before{content:"\f466"}i.icon.boxes:before{content:"\f468"}i.icon.braille:before{content:"\f2a1"}i.icon.briefcase:before{content:"\f0b1"}i.icon.btc:before{content:"\f15a"}i.icon.bug:before{content:"\f188"}i.icon.building:before{content:"\f1ad"}i.icon.bullhorn:before{content:"\f0a1"}i.icon.bullseye:before{content:"\f140"}i.icon.buromobelexperte:before{content:"\f37f"}i.icon.bus:before{content:"\f207"}i.icon.buysellads:before{content:"\f20d"}i.icon.calculator:before{content:"\f1ec"}i.icon.calendar:before{content:"\f133"}i.icon.calendar.alternate:before{content:"\f073"}i.icon.calendar.check:before{content:"\f274"}i.icon.calendar.minus:before{content:"\f272"}i.icon.calendar.plus:before{content:"\f271"}i.icon.calendar.times:before{content:"\f273"}i.icon.camera:before{content:"\f030"}i.icon.camera.retro:before{content:"\f083"}i.icon.car:before{content:"\f1b9"}i.icon.caret.down:before{content:"\f0d7"}i.icon.caret.left:before{content:"\f0d9"}i.icon.caret.right:before{content:"\f0da"}i.icon.caret.square.down:before{content:"\f150"}i.icon.caret.square.left:before{content:"\f191"}i.icon.caret.square.right:before{content:"\f152"}i.icon.caret.square.up:before{content:"\f151"}i.icon.caret.up:before{content:"\f0d8"}i.icon.cart.arrow.down:before{content:"\f218"}i.icon.cart.plus:before{content:"\f217"}i.icon.cc.amazon.pay:before{content:"\f42d"}i.icon.cc.amex:before{content:"\f1f3"}i.icon.cc.apple.pay:before{content:"\f416"}i.icon.cc.diners.club:before{content:"\f24c"}i.icon.cc.discover:before{content:"\f1f2"}i.icon.cc.jcb:before{content:"\f24b"}i.icon.cc.mastercard:before{content:"\f1f1"}i.icon.cc.paypal:before{content:"\f1f4"}i.icon.cc.stripe:before{content:"\f1f5"}i.icon.cc.visa:before{content:"\f1f0"}i.icon.centercode:before{content:"\f380"}i.icon.certificate:before{content:"\f0a3"}i.icon.chart.area:before{content:"\f1fe"}i.icon.chart.bar:before{content:"\f080"}i.icon.chart.line:before{content:"\f201"}i.icon.chart.pie:before{content:"\f200"}i.icon.check:before{content:"\f00c"}i.icon.check.circle:before{content:"\f058"}i.icon.check.square:before{content:"\f14a"}i.icon.chess:before{content:"\f439"}i.icon.chess.bishop:before{content:"\f43a"}i.icon.chess.board:before{content:"\f43c"}i.icon.chess.king:before{content:"\f43f"}i.icon.chess.knight:before{content:"\f441"}i.icon.chess.pawn:before{content:"\f443"}i.icon.chess.queen:before{content:"\f445"}i.icon.chess.rook:before{content:"\f447"}i.icon.chevron.circle.down:before{content:"\f13a"}i.icon.chevron.circle.left:before{content:"\f137"}i.icon.chevron.circle.right:before{content:"\f138"}i.icon.chevron.circle.up:before{content:"\f139"}i.icon.chevron.down:before{content:"\f078"}i.icon.chevron.left:before{content:"\f053"}i.icon.chevron.right:before{content:"\f054"}i.icon.chevron.up:before{content:"\f077"}i.icon.child:before{content:"\f1ae"}i.icon.chrome:before{content:"\f268"}i.icon.circle:before{content:"\f111"}i.icon.circle.notch:before{content:"\f1ce"}i.icon.clipboard:before{content:"\f328"}i.icon.clipboard.check:before{content:"\f46c"}i.icon.clipboard.list:before{content:"\f46d"}i.icon.clock:before{content:"\f017"}i.icon.clone:before{content:"\f24d"}i.icon.closed.captioning:before{content:"\f20a"}i.icon.cloud:before{content:"\f0c2"}i.icon.cloudscale:before{content:"\f383"}i.icon.cloudsmith:before{content:"\f384"}i.icon.cloudversify:before{content:"\f385"}i.icon.code:before{content:"\f121"}i.icon.code.branch:before{content:"\f126"}i.icon.codepen:before{content:"\f1cb"}i.icon.codiepie:before{content:"\f284"}i.icon.coffee:before{content:"\f0f4"}i.icon.cog:before{content:"\f013"}i.icon.cogs:before{content:"\f085"}i.icon.columns:before{content:"\f0db"}i.icon.comment:before{content:"\f075"}i.icon.comment.alternate:before{content:"\f27a"}i.icon.comments:before{content:"\f086"}i.icon.compass:before{content:"\f14e"}i.icon.compress:before{content:"\f066"}i.icon.connectdevelop:before{content:"\f20e"}i.icon.contao:before{content:"\f26d"}i.icon.copy:before{content:"\f0c5"}i.icon.copyright:before{content:"\f1f9"}i.icon.cpanel:before{content:"\f388"}i.icon.creative.commons:before{content:"\f25e"}i.icon.credit.card:before{content:"\f09d"}i.icon.crop:before{content:"\f125"}i.icon.crosshairs:before{content:"\f05b"}i.icon.css3:before{content:"\f13c"}i.icon.css3.alternate:before{content:"\f38b"}i.icon.cube:before{content:"\f1b2"}i.icon.cubes:before{content:"\f1b3"}i.icon.cut:before{content:"\f0c4"}i.icon.cuttlefish:before{content:"\f38c"}i.icon.d.and.d:before{content:"\f38d"}i.icon.dashcube:before{content:"\f210"}i.icon.database:before{content:"\f1c0"}i.icon.deaf:before{content:"\f2a4"}i.icon.delicious:before{content:"\f1a5"}i.icon.deploydog:before{content:"\f38e"}i.icon.deskpro:before{content:"\f38f"}i.icon.desktop:before{content:"\f108"}i.icon.deviantart:before{content:"\f1bd"}i.icon.digg:before{content:"\f1a6"}i.icon.digital.ocean:before{content:"\f391"}i.icon.discord:before{content:"\f392"}i.icon.discourse:before{content:"\f393"}i.icon.dna:before{content:"\f471"}i.icon.dochub:before{content:"\f394"}i.icon.docker:before{content:"\f395"}i.icon.dollar.sign:before{content:"\f155"}i.icon.dolly:before{content:"\f472"}i.icon.dolly.flatbed:before{content:"\f474"}i.icon.dot.circle:before{content:"\f192"}i.icon.download:before{content:"\f019"}i.icon.draft2digital:before{content:"\f396"}i.icon.dribbble:before{content:"\f17d"}i.icon.dribbble.square:before{content:"\f397"}i.icon.dropbox:before{content:"\f16b"}i.icon.drupal:before{content:"\f1a9"}i.icon.dyalog:before{content:"\f399"}i.icon.earlybirds:before{content:"\f39a"}i.icon.edge:before{content:"\f282"}i.icon.edit:before{content:"\f044"}i.icon.eject:before{content:"\f052"}i.icon.elementor:before{content:"\f430"}i.icon.ellipsis.horizontal:before{content:"\f141"}i.icon.ellipsis.vertical:before{content:"\f142"}i.icon.ember:before{content:"\f423"}i.icon.empire:before{content:"\f1d1"}i.icon.envelope:before{content:"\f0e0"}i.icon.envelope.open:before{content:"\f2b6"}i.icon.envelope.square:before{content:"\f199"}i.icon.envira:before{content:"\f299"}i.icon.eraser:before{content:"\f12d"}i.icon.erlang:before{content:"\f39d"}i.icon.ethereum:before{content:"\f42e"}i.icon.etsy:before{content:"\f2d7"}i.icon.euro.sign:before{content:"\f153"}i.icon.exchange.alternate:before{content:"\f362"}i.icon.exclamation:before{content:"\f12a"}i.icon.exclamation.circle:before{content:"\f06a"}i.icon.exclamation.triangle:before{content:"\f071"}i.icon.expand:before{content:"\f065"}i.icon.expand.arrows.alternate:before{content:"\f31e"}i.icon.expeditedssl:before{content:"\f23e"}i.icon.external.alternate:before{content:"\f35d"}i.icon.external.square.alternate:before{content:"\f360"}i.icon.eye:before{content:"\f06e"}i.icon.eye.dropper:before{content:"\f1fb"}i.icon.eye.slash:before{content:"\f070"}i.icon.facebook:before{content:"\f09a"}i.icon.facebook.f:before{content:"\f39e"}i.icon.facebook.messenger:before{content:"\f39f"}i.icon.facebook.square:before{content:"\f082"}i.icon.fast.backward:before{content:"\f049"}i.icon.fast.forward:before{content:"\f050"}i.icon.fax:before{content:"\f1ac"}i.icon.female:before{content:"\f182"}i.icon.fighter.jet:before{content:"\f0fb"}i.icon.file:before{content:"\f15b"}i.icon.file.alternate:before{content:"\f15c"}i.icon.file.archive:before{content:"\f1c6"}i.icon.file.audio:before{content:"\f1c7"}i.icon.file.code:before{content:"\f1c9"}i.icon.file.excel:before{content:"\f1c3"}i.icon.file.image:before{content:"\f1c5"}i.icon.file.pdf:before{content:"\f1c1"}i.icon.file.powerpoint:before{content:"\f1c4"}i.icon.file.video:before{content:"\f1c8"}i.icon.file.word:before{content:"\f1c2"}i.icon.film:before{content:"\f008"}i.icon.filter:before{content:"\f0b0"}i.icon.fire:before{content:"\f06d"}i.icon.fire.extinguisher:before{content:"\f134"}i.icon.firefox:before{content:"\f269"}i.icon.first.aid:before{content:"\f479"}i.icon.first.order:before{content:"\f2b0"}i.icon.firstdraft:before{content:"\f3a1"}i.icon.flag:before{content:"\f024"}i.icon.flag.checkered:before{content:"\f11e"}i.icon.flask:before{content:"\f0c3"}i.icon.flickr:before{content:"\f16e"}i.icon.flipboard:before{content:"\f44d"}i.icon.fly:before{content:"\f417"}i.icon.folder:before{content:"\f07b"}i.icon.folder.open:before{content:"\f07c"}i.icon.font:before{content:"\f031"}i.icon.font.awesome:before{content:"\f2b4"}i.icon.font.awesome.alternate:before{content:"\f35c"}i.icon.font.awesome.flag:before{content:"\f425"}i.icon.fonticons:before{content:"\f280"}i.icon.fonticons.fi:before{content:"\f3a2"}i.icon.football.ball:before{content:"\f44e"}i.icon.fort.awesome:before{content:"\f286"}i.icon.fort.awesome.alternate:before{content:"\f3a3"}i.icon.forumbee:before{content:"\f211"}i.icon.forward:before{content:"\f04e"}i.icon.foursquare:before{content:"\f180"}i.icon.free.code.camp:before{content:"\f2c5"}i.icon.freebsd:before{content:"\f3a4"}i.icon.frown:before{content:"\f119"}i.icon.futbol:before{content:"\f1e3"}i.icon.gamepad:before{content:"\f11b"}i.icon.gavel:before{content:"\f0e3"}i.icon.gem:before{content:"\f3a5"}i.icon.genderless:before{content:"\f22d"}i.icon.get.pocket:before{content:"\f265"}i.icon.gg:before{content:"\f260"}i.icon.gg.circle:before{content:"\f261"}i.icon.gift:before{content:"\f06b"}i.icon.git:before{content:"\f1d3"}i.icon.git.square:before{content:"\f1d2"}i.icon.github:before{content:"\f09b"}i.icon.github.alternate:before{content:"\f113"}i.icon.github.square:before{content:"\f092"}i.icon.gitkraken:before{content:"\f3a6"}i.icon.gitlab:before{content:"\f296"}i.icon.gitter:before{content:"\f426"}i.icon.glass.martini:before{content:"\f000"}i.icon.glide:before{content:"\f2a5"}i.icon.glide.g:before{content:"\f2a6"}i.icon.globe:before{content:"\f0ac"}i.icon.gofore:before{content:"\f3a7"}i.icon.golf.ball:before{content:"\f450"}i.icon.goodreads:before{content:"\f3a8"}i.icon.goodreads.g:before{content:"\f3a9"}i.icon.google:before{content:"\f1a0"}i.icon.google.drive:before{content:"\f3aa"}i.icon.google.play:before{content:"\f3ab"}i.icon.google.plus:before{content:"\f2b3"}i.icon.google.plus.g:before{content:"\f0d5"}i.icon.google.plus.square:before{content:"\f0d4"}i.icon.google.wallet:before{content:"\f1ee"}i.icon.graduation.cap:before{content:"\f19d"}i.icon.gratipay:before{content:"\f184"}i.icon.grav:before{content:"\f2d6"}i.icon.gripfire:before{content:"\f3ac"}i.icon.grunt:before{content:"\f3ad"}i.icon.gulp:before{content:"\f3ae"}i.icon.h.square:before{content:"\f0fd"}i.icon.hacker.news:before{content:"\f1d4"}i.icon.hacker.news.square:before{content:"\f3af"}i.icon.hand.lizard:before{content:"\f258"}i.icon.hand.paper:before{content:"\f256"}i.icon.hand.peace:before{content:"\f25b"}i.icon.hand.point.down:before{content:"\f0a7"}i.icon.hand.point.left:before{content:"\f0a5"}i.icon.hand.point.right:before{content:"\f0a4"}i.icon.hand.point.up:before{content:"\f0a6"}i.icon.hand.pointer:before{content:"\f25a"}i.icon.hand.rock:before{content:"\f255"}i.icon.hand.scissors:before{content:"\f257"}i.icon.hand.spock:before{content:"\f259"}i.icon.handshake:before{content:"\f2b5"}i.icon.hashtag:before{content:"\f292"}i.icon.hdd:before{content:"\f0a0"}i.icon.heading:before{content:"\f1dc"}i.icon.headphones:before{content:"\f025"}i.icon.heart:before{content:"\f004"}i.icon.heartbeat:before{content:"\f21e"}i.icon.hips:before{content:"\f452"}i.icon.hire.a.helper:before{content:"\f3b0"}i.icon.history:before{content:"\f1da"}i.icon.hockey.puck:before{content:"\f453"}i.icon.home:before{content:"\f015"}i.icon.hooli:before{content:"\f427"}i.icon.hospital:before{content:"\f0f8"}i.icon.hospital.symbol:before{content:"\f47e"}i.icon.hotjar:before{content:"\f3b1"}i.icon.hourglass:before{content:"\f254"}i.icon.hourglass.end:before{content:"\f253"}i.icon.hourglass.half:before{content:"\f252"}i.icon.hourglass.start:before{content:"\f251"}i.icon.houzz:before{content:"\f27c"}i.icon.html5:before{content:"\f13b"}i.icon.hubspot:before{content:"\f3b2"}i.icon.i.cursor:before{content:"\f246"}i.icon.id.badge:before{content:"\f2c1"}i.icon.id.card:before{content:"\f2c2"}i.icon.image:before{content:"\f03e"}i.icon.images:before{content:"\f302"}i.icon.imdb:before{content:"\f2d8"}i.icon.inbox:before{content:"\f01c"}i.icon.indent:before{content:"\f03c"}i.icon.industry:before{content:"\f275"}i.icon.info:before{content:"\f129"}i.icon.info.circle:before{content:"\f05a"}i.icon.instagram:before{content:"\f16d"}i.icon.internet.explorer:before{content:"\f26b"}i.icon.ioxhost:before{content:"\f208"}i.icon.italic:before{content:"\f033"}i.icon.itunes:before{content:"\f3b4"}i.icon.itunes.note:before{content:"\f3b5"}i.icon.jenkins:before{content:"\f3b6"}i.icon.joget:before{content:"\f3b7"}i.icon.joomla:before{content:"\f1aa"}i.icon.js:before{content:"\f3b8"}i.icon.js.square:before{content:"\f3b9"}i.icon.jsfiddle:before{content:"\f1cc"}i.icon.key:before{content:"\f084"}i.icon.keyboard:before{content:"\f11c"}i.icon.keycdn:before{content:"\f3ba"}i.icon.kickstarter:before{content:"\f3bb"}i.icon.kickstarter.k:before{content:"\f3bc"}i.icon.korvue:before{content:"\f42f"}i.icon.language:before{content:"\f1ab"}i.icon.laptop:before{content:"\f109"}i.icon.laravel:before{content:"\f3bd"}i.icon.lastfm:before{content:"\f202"}i.icon.lastfm.square:before{content:"\f203"}i.icon.leaf:before{content:"\f06c"}i.icon.leanpub:before{content:"\f212"}i.icon.lemon:before{content:"\f094"}i.icon.less:before{content:"\f41d"}i.icon.level.down.alternate:before{content:"\f3be"}i.icon.level.up.alternate:before{content:"\f3bf"}i.icon.life.ring:before{content:"\f1cd"}i.icon.lightbulb:before{content:"\f0eb"}i.icon.linechat:before{content:"\f3c0"}i.icon.linkify:before{content:"\f0c1"}i.icon.linkedin:before{content:"\f08c"}i.icon.linkedin.alt:before{content:"\f0e1"}i.icon.linode:before{content:"\f2b8"}i.icon.linux:before{content:"\f17c"}i.icon.lira.sign:before{content:"\f195"}i.icon.list:before{content:"\f03a"}i.icon.list.alternate:before{content:"\f022"}i.icon.list.ol:before{content:"\f0cb"}i.icon.list.ul:before{content:"\f0ca"}i.icon.location.arrow:before{content:"\f124"}i.icon.lock:before{content:"\f023"}i.icon.lock.open:before{content:"\f3c1"}i.icon.long.arrow.alternate.down:before{content:"\f309"}i.icon.long.arrow.alternate.left:before{content:"\f30a"}i.icon.long.arrow.alternate.right:before{content:"\f30b"}i.icon.long.arrow.alternate.up:before{content:"\f30c"}i.icon.low.vision:before{content:"\f2a8"}i.icon.lyft:before{content:"\f3c3"}i.icon.magento:before{content:"\f3c4"}i.icon.magic:before{content:"\f0d0"}i.icon.magnet:before{content:"\f076"}i.icon.male:before{content:"\f183"}i.icon.map:before{content:"\f279"}i.icon.map.marker:before{content:"\f041"}i.icon.map.marker.alternate:before{content:"\f3c5"}i.icon.map.pin:before{content:"\f276"}i.icon.map.signs:before{content:"\f277"}i.icon.mars:before{content:"\f222"}i.icon.mars.double:before{content:"\f227"}i.icon.mars.stroke:before{content:"\f229"}i.icon.mars.stroke.horizontal:before{content:"\f22b"}i.icon.mars.stroke.vertical:before{content:"\f22a"}i.icon.maxcdn:before{content:"\f136"}i.icon.medapps:before{content:"\f3c6"}i.icon.medium:before{content:"\f23a"}i.icon.medium.m:before{content:"\f3c7"}i.icon.medkit:before{content:"\f0fa"}i.icon.medrt:before{content:"\f3c8"}i.icon.meetup:before{content:"\f2e0"}i.icon.meh:before{content:"\f11a"}i.icon.mercury:before{content:"\f223"}i.icon.microchip:before{content:"\f2db"}i.icon.microphone:before{content:"\f130"}i.icon.microphone.slash:before{content:"\f131"}i.icon.microsoft:before{content:"\f3ca"}i.icon.minus:before{content:"\f068"}i.icon.minus.circle:before{content:"\f056"}i.icon.minus.square:before{content:"\f146"}i.icon.mix:before{content:"\f3cb"}i.icon.mixcloud:before{content:"\f289"}i.icon.mizuni:before{content:"\f3cc"}i.icon.mobile:before{content:"\f10b"}i.icon.mobile.alternate:before{content:"\f3cd"}i.icon.modx:before{content:"\f285"}i.icon.monero:before{content:"\f3d0"}i.icon.money.bill.alternate:before{content:"\f3d1"}i.icon.moon:before{content:"\f186"}i.icon.motorcycle:before{content:"\f21c"}i.icon.mouse.pointer:before{content:"\f245"}i.icon.music:before{content:"\f001"}i.icon.napster:before{content:"\f3d2"}i.icon.neuter:before{content:"\f22c"}i.icon.newspaper:before{content:"\f1ea"}i.icon.nintendo.switch:before{content:"\f418"}i.icon.node:before{content:"\f419"}i.icon.node.js:before{content:"\f3d3"}i.icon.npm:before{content:"\f3d4"}i.icon.ns8:before{content:"\f3d5"}i.icon.nutritionix:before{content:"\f3d6"}i.icon.object.group:before{content:"\f247"}i.icon.object.ungroup:before{content:"\f248"}i.icon.odnoklassniki:before{content:"\f263"}i.icon.odnoklassniki.square:before{content:"\f264"}i.icon.opencart:before{content:"\f23d"}i.icon.openid:before{content:"\f19b"}i.icon.opera:before{content:"\f26a"}i.icon.optin.monster:before{content:"\f23c"}i.icon.osi:before{content:"\f41a"}i.icon.outdent:before{content:"\f03b"}i.icon.page4:before{content:"\f3d7"}i.icon.pagelines:before{content:"\f18c"}i.icon.paint.brush:before{content:"\f1fc"}i.icon.palfed:before{content:"\f3d8"}i.icon.pallet:before{content:"\f482"}i.icon.paper.plane:before{content:"\f1d8"}i.icon.paperclip:before{content:"\f0c6"}i.icon.paragraph:before{content:"\f1dd"}i.icon.paste:before{content:"\f0ea"}i.icon.patreon:before{content:"\f3d9"}i.icon.pause:before{content:"\f04c"}i.icon.pause.circle:before{content:"\f28b"}i.icon.paw:before{content:"\f1b0"}i.icon.paypal:before{content:"\f1ed"}i.icon.pen.square:before{content:"\f14b"}i.icon.pencil.alternate:before{content:"\f303"}i.icon.percent:before{content:"\f295"}i.icon.periscope:before{content:"\f3da"}i.icon.phabricator:before{content:"\f3db"}i.icon.phoenix.framework:before{content:"\f3dc"}i.icon.phone:before{content:"\f095"}i.icon.phone.square:before{content:"\f098"}i.icon.phone.volume:before{content:"\f2a0"}i.icon.php:before{content:"\f457"}i.icon.pied.piper:before{content:"\f2ae"}i.icon.pied.piper.alternate:before{content:"\f1a8"}i.icon.pied.piper.pp:before{content:"\f1a7"}i.icon.pills:before{content:"\f484"}i.icon.pinterest:before{content:"\f0d2"}i.icon.pinterest.p:before{content:"\f231"}i.icon.pinterest.square:before{content:"\f0d3"}i.icon.plane:before{content:"\f072"}i.icon.play:before{content:"\f04b"}i.icon.play.circle:before{content:"\f144"}i.icon.playstation:before{content:"\f3df"}i.icon.plug:before{content:"\f1e6"}i.icon.plus:before{content:"\f067"}i.icon.plus.circle:before{content:"\f055"}i.icon.plus.square:before{content:"\f0fe"}i.icon.podcast:before{content:"\f2ce"}i.icon.pound.sign:before{content:"\f154"}i.icon.power.off:before{content:"\f011"}i.icon.print:before{content:"\f02f"}i.icon.product.hunt:before{content:"\f288"}i.icon.pushed:before{content:"\f3e1"}i.icon.puzzle.piece:before{content:"\f12e"}i.icon.python:before{content:"\f3e2"}i.icon.qq:before{content:"\f1d6"}i.icon.qrcode:before{content:"\f029"}i.icon.question:before{content:"\f128"}i.icon.question.circle:before{content:"\f059"}i.icon.quidditch:before{content:"\f458"}i.icon.quinscape:before{content:"\f459"}i.icon.quora:before{content:"\f2c4"}i.icon.quote.left:before{content:"\f10d"}i.icon.quote.right:before{content:"\f10e"}i.icon.random:before{content:"\f074"}i.icon.ravelry:before{content:"\f2d9"}i.icon.react:before{content:"\f41b"}i.icon.rebel:before{content:"\f1d0"}i.icon.recycle:before{content:"\f1b8"}i.icon.redriver:before{content:"\f3e3"}i.icon.reddit:before{content:"\f1a1"}i.icon.reddit.alien:before{content:"\f281"}i.icon.reddit.square:before{content:"\f1a2"}i.icon.redo:before{content:"\f01e"}i.icon.redo.alternate:before{content:"\f2f9"}i.icon.registered:before{content:"\f25d"}i.icon.rendact:before{content:"\f3e4"}i.icon.renren:before{content:"\f18b"}i.icon.reply:before{content:"\f3e5"}i.icon.reply.all:before{content:"\f122"}i.icon.replyd:before{content:"\f3e6"}i.icon.resolving:before{content:"\f3e7"}i.icon.retweet:before{content:"\f079"}i.icon.road:before{content:"\f018"}i.icon.rocket:before{content:"\f135"}i.icon.rocketchat:before{content:"\f3e8"}i.icon.rockrms:before{content:"\f3e9"}i.icon.rss:before{content:"\f09e"}i.icon.rss.square:before{content:"\f143"}i.icon.ruble.sign:before{content:"\f158"}i.icon.rupee.sign:before{content:"\f156"}i.icon.safari:before{content:"\f267"}i.icon.sass:before{content:"\f41e"}i.icon.save:before{content:"\f0c7"}i.icon.schlix:before{content:"\f3ea"}i.icon.scribd:before{content:"\f28a"}i.icon.search:before{content:"\f002"}i.icon.search.minus:before{content:"\f010"}i.icon.search.plus:before{content:"\f00e"}i.icon.searchengin:before{content:"\f3eb"}i.icon.sellcast:before{content:"\f2da"}i.icon.sellsy:before{content:"\f213"}i.icon.server:before{content:"\f233"}i.icon.servicestack:before{content:"\f3ec"}i.icon.share:before{content:"\f064"}i.icon.share.alternate:before{content:"\f1e0"}i.icon.share.alternate.square:before{content:"\f1e1"}i.icon.share.square:before{content:"\f14d"}i.icon.shekel.sign:before{content:"\f20b"}i.icon.shield.alternate:before{content:"\f3ed"}i.icon.ship:before{content:"\f21a"}i.icon.shipping.fast:before{content:"\f48b"}i.icon.shirtsinbulk:before{content:"\f214"}i.icon.shopping.bag:before{content:"\f290"}i.icon.shopping.basket:before{content:"\f291"}i.icon.shopping.cart:before{content:"\f07a"}i.icon.shower:before{content:"\f2cc"}i.icon.sign.language:before{content:"\f2a7"}i.icon.signal:before{content:"\f012"}i.icon.simplybuilt:before{content:"\f215"}i.icon.sistrix:before{content:"\f3ee"}i.icon.sitemap:before{content:"\f0e8"}i.icon.skyatlas:before{content:"\f216"}i.icon.skype:before{content:"\f17e"}i.icon.slack:before{content:"\f198"}i.icon.slack.hash:before{content:"\f3ef"}i.icon.sliders.horizontal:before{content:"\f1de"}i.icon.slideshare:before{content:"\f1e7"}i.icon.smile:before{content:"\f118"}i.icon.snapchat:before{content:"\f2ab"}i.icon.snapchat.ghost:before{content:"\f2ac"}i.icon.snapchat.square:before{content:"\f2ad"}i.icon.snowflake:before{content:"\f2dc"}i.icon.sort:before{content:"\f0dc"}i.icon.sort.alphabet.down:before{content:"\f15d"}i.icon.sort.alphabet.up:before{content:"\f15e"}i.icon.sort.amount.down:before{content:"\f160"}i.icon.sort.amount.up:before{content:"\f161"}i.icon.sort.down:before{content:"\f0dd"}i.icon.sort.numeric.down:before{content:"\f162"}i.icon.sort.numeric.up:before{content:"\f163"}i.icon.sort.up:before{content:"\f0de"}i.icon.soundcloud:before{content:"\f1be"}i.icon.space.shuttle:before{content:"\f197"}i.icon.speakap:before{content:"\f3f3"}i.icon.spinner:before{content:"\f110"}i.icon.spotify:before{content:"\f1bc"}i.icon.square:before{content:"\f0c8"}i.icon.square.full:before{content:"\f45c"}i.icon.stack.exchange:before{content:"\f18d"}i.icon.stack.overflow:before{content:"\f16c"}i.icon.star:before{content:"\f005"}i.icon.star.half:before{content:"\f089"}i.icon.staylinked:before{content:"\f3f5"}i.icon.steam:before{content:"\f1b6"}i.icon.steam.square:before{content:"\f1b7"}i.icon.steam.symbol:before{content:"\f3f6"}i.icon.step.backward:before{content:"\f048"}i.icon.step.forward:before{content:"\f051"}i.icon.stethoscope:before{content:"\f0f1"}i.icon.sticker.mule:before{content:"\f3f7"}i.icon.sticky.note:before{content:"\f249"}i.icon.stop:before{content:"\f04d"}i.icon.stop.circle:before{content:"\f28d"}i.icon.stopwatch:before{content:"\f2f2"}i.icon.strava:before{content:"\f428"}i.icon.street.view:before{content:"\f21d"}i.icon.strikethrough:before{content:"\f0cc"}i.icon.stripe:before{content:"\f429"}i.icon.stripe.s:before{content:"\f42a"}i.icon.studiovinari:before{content:"\f3f8"}i.icon.stumbleupon:before{content:"\f1a4"}i.icon.stumbleupon.circle:before{content:"\f1a3"}i.icon.subscript:before{content:"\f12c"}i.icon.subway:before{content:"\f239"}i.icon.suitcase:before{content:"\f0f2"}i.icon.sun:before{content:"\f185"}i.icon.superpowers:before{content:"\f2dd"}i.icon.superscript:before{content:"\f12b"}i.icon.supple:before{content:"\f3f9"}i.icon.sync:before{content:"\f021"}i.icon.sync.alternate:before{content:"\f2f1"}i.icon.syringe:before{content:"\f48e"}i.icon.table:before{content:"\f0ce"}i.icon.table.tennis:before{content:"\f45d"}i.icon.tablet:before{content:"\f10a"}i.icon.tablet.alternate:before{content:"\f3fa"}i.icon.tachometer.alternate:before{content:"\f3fd"}i.icon.tag:before{content:"\f02b"}i.icon.tags:before{content:"\f02c"}i.icon.tasks:before{content:"\f0ae"}i.icon.taxi:before{content:"\f1ba"}i.icon.telegram:before{content:"\f2c6"}i.icon.telegram.plane:before{content:"\f3fe"}i.icon.tencent.weibo:before{content:"\f1d5"}i.icon.terminal:before{content:"\f120"}i.icon.text.height:before{content:"\f034"}i.icon.text.width:before{content:"\f035"}i.icon.th:before{content:"\f00a"}i.icon.th.large:before{content:"\f009"}i.icon.th.list:before{content:"\f00b"}i.icon.themeisle:before{content:"\f2b2"}i.icon.thermometer:before{content:"\f491"}i.icon.thermometer.empty:before{content:"\f2cb"}i.icon.thermometer.full:before{content:"\f2c7"}i.icon.thermometer.half:before{content:"\f2c9"}i.icon.thermometer.quarter:before{content:"\f2ca"}i.icon.thermometer.three.quarters:before{content:"\f2c8"}i.icon.thumbs.down:before{content:"\f165"}i.icon.thumbs.up:before{content:"\f164"}i.icon.thumbtack:before{content:"\f08d"}i.icon.ticket.alternate:before{content:"\f3ff"}i.icon.times:before{content:"\f00d"}i.icon.times.circle:before{content:"\f057"}i.icon.tint:before{content:"\f043"}i.icon.toggle.off:before{content:"\f204"}i.icon.toggle.on:before{content:"\f205"}i.icon.trademark:before{content:"\f25c"}i.icon.train:before{content:"\f238"}i.icon.transgender:before{content:"\f224"}i.icon.transgender.alternate:before{content:"\f225"}i.icon.trash:before{content:"\f1f8"}i.icon.trash.alternate:before{content:"\f2ed"}i.icon.tree:before{content:"\f1bb"}i.icon.trello:before{content:"\f181"}i.icon.tripadvisor:before{content:"\f262"}i.icon.trophy:before{content:"\f091"}i.icon.truck:before{content:"\f0d1"}i.icon.tty:before{content:"\f1e4"}i.icon.tumblr:before{content:"\f173"}i.icon.tumblr.square:before{content:"\f174"}i.icon.tv:before{content:"\f26c"}i.icon.twitch:before{content:"\f1e8"}i.icon.twitter:before{content:"\f099"}i.icon.twitter.square:before{content:"\f081"}i.icon.typo3:before{content:"\f42b"}i.icon.uber:before{content:"\f402"}i.icon.uikit:before{content:"\f403"}i.icon.umbrella:before{content:"\f0e9"}i.icon.underline:before{content:"\f0cd"}i.icon.undo:before{content:"\f0e2"}i.icon.undo.alternate:before{content:"\f2ea"}i.icon.uniregistry:before{content:"\f404"}i.icon.universal.access:before{content:"\f29a"}i.icon.university:before{content:"\f19c"}i.icon.unlink:before{content:"\f127"}i.icon.unlock:before{content:"\f09c"}i.icon.unlock.alternate:before{content:"\f13e"}i.icon.untappd:before{content:"\f405"}i.icon.upload:before{content:"\f093"}i.icon.usb:before{content:"\f287"}i.icon.user:before{content:"\f007"}i.icon.user.circle:before{content:"\f2bd"}i.icon.user.md:before{content:"\f0f0"}i.icon.user.plus:before{content:"\f234"}i.icon.user.secret:before{content:"\f21b"}i.icon.user.times:before{content:"\f235"}i.icon.users:before{content:"\f0c0"}i.icon.ussunnah:before{content:"\f407"}i.icon.utensil.spoon:before{content:"\f2e5"}i.icon.utensils:before{content:"\f2e7"}i.icon.vaadin:before{content:"\f408"}i.icon.venus:before{content:"\f221"}i.icon.venus.double:before{content:"\f226"}i.icon.venus.mars:before{content:"\f228"}i.icon.viacoin:before{content:"\f237"}i.icon.viadeo:before{content:"\f2a9"}i.icon.viadeo.square:before{content:"\f2aa"}i.icon.viber:before{content:"\f409"}i.icon.video:before{content:"\f03d"}i.icon.vimeo:before{content:"\f40a"}i.icon.vimeo.square:before{content:"\f194"}i.icon.vimeo.v:before{content:"\f27d"}i.icon.vine:before{content:"\f1ca"}i.icon.vk:before{content:"\f189"}i.icon.vnv:before{content:"\f40b"}i.icon.volleyball.ball:before{content:"\f45f"}i.icon.volume.down:before{content:"\f027"}i.icon.volume.off:before{content:"\f026"}i.icon.volume.up:before{content:"\f028"}i.icon.vuejs:before{content:"\f41f"}i.icon.warehouse:before{content:"\f494"}i.icon.weibo:before{content:"\f18a"}i.icon.weight:before{content:"\f496"}i.icon.weixin:before{content:"\f1d7"}i.icon.whatsapp:before{content:"\f232"}i.icon.whatsapp.square:before{content:"\f40c"}i.icon.wheelchair:before{content:"\f193"}i.icon.whmcs:before{content:"\f40d"}i.icon.wifi:before{content:"\f1eb"}i.icon.wikipedia.w:before{content:"\f266"}i.icon.window.close:before{content:"\f410"}i.icon.window.maximize:before{content:"\f2d0"}i.icon.window.minimize:before{content:"\f2d1"}i.icon.window.restore:before{content:"\f2d2"}i.icon.windows:before{content:"\f17a"}i.icon.won.sign:before{content:"\f159"}i.icon.wordpress:before{content:"\f19a"}i.icon.wordpress.simple:before{content:"\f411"}i.icon.wpbeginner:before{content:"\f297"}i.icon.wpexplorer:before{content:"\f2de"}i.icon.wpforms:before{content:"\f298"}i.icon.wrench:before{content:"\f0ad"}i.icon.xbox:before{content:"\f412"}i.icon.xing:before{content:"\f168"}i.icon.xing.square:before{content:"\f169"}i.icon.y.combinator:before{content:"\f23b"}i.icon.yahoo:before{content:"\f19e"}i.icon.yandex:before{content:"\f413"}i.icon.yandex.international:before{content:"\f414"}i.icon.yelp:before{content:"\f1e9"}i.icon.yen.sign:before{content:"\f157"}i.icon.yoast:before{content:"\f2b1"}i.icon.youtube:before{content:"\f167"}i.icon.youtube.square:before{content:"\f431"}i.icon.chess.rock:before{content:"\f447"}i.icon.ordered.list:before{content:"\f0cb"}i.icon.unordered.list:before{content:"\f0ca"}i.icon.user.doctor:before{content:"\f0f0"}i.icon.shield:before{content:"\f3ed"}i.icon.puzzle:before{content:"\f12e"}i.icon.credit.card.amazon.pay:before{content:"\f42d"}i.icon.credit.card.american.express:before{content:"\f1f3"}i.icon.credit.card.diners.club:before{content:"\f24c"}i.icon.credit.card.discover:before{content:"\f1f2"}i.icon.credit.card.jcb:before{content:"\f24b"}i.icon.credit.card.mastercard:before{content:"\f1f1"}i.icon.credit.card.paypal:before{content:"\f1f4"}i.icon.credit.card.stripe:before{content:"\f1f5"}i.icon.credit.card.visa:before{content:"\f1f0"}i.icon.add.circle:before{content:"\f055"}i.icon.add.square:before{content:"\f0fe"}i.icon.add.to.calendar:before{content:"\f271"}i.icon.add.to.cart:before{content:"\f217"}i.icon.add.user:before{content:"\f234"}i.icon.add:before{content:"\f067"}i.icon.alarm.mute:before{content:"\f1f6"}i.icon.alarm:before{content:"\f0f3"}i.icon.ald:before{content:"\f2a2"}i.icon.als:before{content:"\f2a2"}i.icon.american.express.card:before{content:"\f1f3"}i.icon.american.express:before{content:"\f1f3"}i.icon.amex:before{content:"\f1f3"}i.icon.announcement:before{content:"\f0a1"}i.icon.area.chart:before{content:"\f1fe"}i.icon.area.graph:before{content:"\f1fe"}i.icon.arrow.down.cart:before{content:"\f218"}i.icon.asexual:before{content:"\f22d"}i.icon.asl.interpreting:before{content:"\f2a3"}i.icon.asl:before{content:"\f2a3"}i.icon.assistive.listening.devices:before{content:"\f2a2"}i.icon.attach:before{content:"\f0c6"}i.icon.attention:before{content:"\f06a"}i.icon.balance:before{content:"\f24e"}i.icon.bar:before{content:"\f0fc"}i.icon.bathtub:before{content:"\f2cd"}i.icon.battery.four:before{content:"\f240"}i.icon.battery.high:before{content:"\f241"}i.icon.battery.low:before{content:"\f243"}i.icon.battery.medium:before{content:"\f242"}i.icon.battery.one:before{content:"\f243"}i.icon.battery.three:before{content:"\f241"}i.icon.battery.two:before{content:"\f242"}i.icon.battery.zero:before{content:"\f244"}i.icon.birthday:before{content:"\f1fd"}i.icon.block.layout:before{content:"\f009"}i.icon.bluetooth.alternative:before{content:"\f294"}i.icon.broken.chain:before{content:"\f127"}i.icon.browser:before{content:"\f022"}i.icon.call.square:before{content:"\f098"}i.icon.call:before{content:"\f095"}i.icon.cancel:before{content:"\f00d"}i.icon.cart:before{content:"\f07a"}i.icon.cc:before{content:"\f20a"}i.icon.chain:before{content:"\f0c1"}i.icon.chat:before{content:"\f075"}i.icon.checked.calendar:before{content:"\f274"}i.icon.checkmark:before{content:"\f00c"}i.icon.circle.notched:before{content:"\f1ce"}i.icon.close:before{content:"\f00d"}i.icon.cny:before{content:"\f157"}i.icon.cocktail:before{content:"\f000"}i.icon.commenting:before{content:"\f27a"}i.icon.computer:before{content:"\f108"}i.icon.configure:before{content:"\f0ad"}i.icon.content:before{content:"\f0c9"}i.icon.deafness:before{content:"\f2a4"}i.icon.delete.calendar:before{content:"\f273"}i.icon.delete:before{content:"\f00d"}i.icon.detective:before{content:"\f21b"}i.icon.diners.club.card:before{content:"\f24c"}i.icon.diners.club:before{content:"\f24c"}i.icon.discover.card:before{content:"\f1f2"}i.icon.discover:before{content:"\f1f2"}i.icon.discussions:before{content:"\f086"}i.icon.doctor:before{content:"\f0f0"}i.icon.dollar:before{content:"\f155"}i.icon.dont:before{content:"\f05e"}i.icon.dribble:before{content:"\f17d"}i.icon.drivers.license:before{content:"\f2c2"}i.icon.dropdown:before{content:"\f0d7"}i.icon.eercast:before{content:"\f2da"}i.icon.emergency:before{content:"\f0f9"}i.icon.envira.gallery:before{content:"\f299"}i.icon.erase:before{content:"\f12d"}i.icon.eur:before{content:"\f153"}i.icon.euro:before{content:"\f153"}i.icon.eyedropper:before{content:"\f1fb"}i.icon.fa:before{content:"\f2b4"}i.icon.factory:before{content:"\f275"}i.icon.favorite:before{content:"\f005"}i.icon.feed:before{content:"\f09e"}i.icon.female.homosexual:before{content:"\f226"}i.icon.file.text:before{content:"\f15c"}i.icon.find:before{content:"\f1e5"}i.icon.first.aid:before{content:"\f0fa"}i.icon.five.hundred.pixels:before{content:"\f26e"}i.icon.fork:before{content:"\f126"}i.icon.game:before{content:"\f11b"}i.icon.gay:before{content:"\f227"}i.icon.gbp:before{content:"\f154"}i.icon.gittip:before{content:"\f184"}i.icon.google.plus.circle:before{content:"\f2b3"}i.icon.google.plus.official:before{content:"\f2b3"}i.icon.grab:before{content:"\f255"}i.icon.graduation:before{content:"\f19d"}i.icon.grid.layout:before{content:"\f00a"}i.icon.group:before{content:"\f0c0"}i.icon.h:before{content:"\f0fd"}i.icon.hand.victory:before{content:"\f25b"}i.icon.handicap:before{content:"\f193"}i.icon.hard.of.hearing:before{content:"\f2a4"}i.icon.header:before{content:"\f1dc"}i.icon.help.circle:before{content:"\f059"}i.icon.help:before{content:"\f128"}i.icon.heterosexual:before{content:"\f228"}i.icon.hide:before{content:"\f070"}i.icon.hotel:before{content:"\f236"}i.icon.hourglass.four:before{content:"\f254"}i.icon.hourglass.full:before{content:"\f254"}i.icon.hourglass.one:before{content:"\f251"}i.icon.hourglass.three:before{content:"\f253"}i.icon.hourglass.two:before{content:"\f252"}i.icon.idea:before{content:"\f0eb"}i.icon.ils:before{content:"\f20b"}i.icon.in-cart:before{content:"\f218"}i.icon.inr:before{content:"\f156"}i.icon.intergender:before{content:"\f224"}i.icon.intersex:before{content:"\f224"}i.icon.japan.credit.bureau.card:before{content:"\f24b"}i.icon.japan.credit.bureau:before{content:"\f24b"}i.icon.jcb:before{content:"\f24b"}i.icon.jpy:before{content:"\f157"}i.icon.krw:before{content:"\f159"}i.icon.lab:before{content:"\f0c3"}i.icon.law:before{content:"\f24e"}i.icon.legal:before{content:"\f0e3"}i.icon.lesbian:before{content:"\f226"}i.icon.lightning:before{content:"\f0e7"}i.icon.like:before{content:"\f004"}i.icon.line.graph:before{content:"\f201"}i.icon.linkedin.square:before{content:"\f08c"}i.icon.linkify:before{content:"\f0c1"}i.icon.lira:before{content:"\f195"}i.icon.list.layout:before{content:"\f00b"}i.icon.magnify:before{content:"\f00e"}i.icon.mail.forward:before{content:"\f064"}i.icon.mail.square:before{content:"\f199"}i.icon.mail:before{content:"\f0e0"}i.icon.male.homosexual:before{content:"\f227"}i.icon.man:before{content:"\f222"}i.icon.marker:before{content:"\f041"}i.icon.mars.alternate:before{content:"\f229"}i.icon.mars.horizontal:before{content:"\f22b"}i.icon.mars.vertical:before{content:"\f22a"}i.icon.mastercard.card:before{content:"\f1f1"}i.icon.mastercard:before{content:"\f1f1"}i.icon.microsoft.edge:before{content:"\f282"}i.icon.military:before{content:"\f0fb"}i.icon.ms.edge:before{content:"\f282"}i.icon.mute:before{content:"\f131"}i.icon.new.pied.piper:before{content:"\f2ae"}i.icon.non.binary.transgender:before{content:"\f223"}i.icon.numbered.list:before{content:"\f0cb"}i.icon.optinmonster:before{content:"\f23c"}i.icon.options:before{content:"\f1de"}i.icon.other.gender.horizontal:before{content:"\f22b"}i.icon.other.gender.vertical:before{content:"\f22a"}i.icon.other.gender:before{content:"\f229"}i.icon.payment:before{content:"\f09d"}i.icon.paypal.card:before{content:"\f1f4"}i.icon.pencil.square:before{content:"\f14b"}i.icon.photo:before{content:"\f030"}i.icon.picture:before{content:"\f03e"}i.icon.pie.chart:before{content:"\f200"}i.icon.pie.graph:before{content:"\f200"}i.icon.pied.piper.hat:before{content:"\f2ae"}i.icon.pin:before{content:"\f08d"}i.icon.plus.cart:before{content:"\f217"}i.icon.pocket:before{content:"\f265"}i.icon.point:before{content:"\f041"}i.icon.pointing.down:before{content:"\f0a7"}i.icon.pointing.left:before{content:"\f0a5"}i.icon.pointing.right:before{content:"\f0a4"}i.icon.pointing.up:before{content:"\f0a6"}i.icon.pound:before{content:"\f154"}i.icon.power.cord:before{content:"\f1e6"}i.icon.power:before{content:"\f011"}i.icon.privacy:before{content:"\f084"}i.icon.r.circle:before{content:"\f25d"}i.icon.rain:before{content:"\f0e9"}i.icon.record:before{content:"\f03d"}i.icon.refresh:before{content:"\f021"}i.icon.remove.circle:before{content:"\f057"}i.icon.remove.from.calendar:before{content:"\f272"}i.icon.remove.user:before{content:"\f235"}i.icon.remove:before{content:"\f00d"}i.icon.repeat:before{content:"\f01e"}i.icon.rmb:before{content:"\f157"}i.icon.rouble:before{content:"\f158"}i.icon.rub:before{content:"\f158"}i.icon.ruble:before{content:"\f158"}i.icon.rupee:before{content:"\f156"}i.icon.s15:before{content:"\f2cd"}i.icon.selected.radio:before{content:"\f192"}i.icon.send:before{content:"\f1d8"}i.icon.setting:before{content:"\f013"}i.icon.settings:before{content:"\f085"}i.icon.shekel:before{content:"\f20b"}i.icon.sheqel:before{content:"\f20b"}i.icon.shipping:before{content:"\f0d1"}i.icon.shop:before{content:"\f07a"}i.icon.shuffle:before{content:"\f074"}i.icon.shutdown:before{content:"\f011"}i.icon.sidebar:before{content:"\f0c9"}i.icon.signing:before{content:"\f2a7"}i.icon.signup:before{content:"\f044"}i.icon.sliders:before{content:"\f1de"}i.icon.soccer:before{content:"\f1e3"}i.icon.sort.alphabet.ascending:before{content:"\f15d"}i.icon.sort.alphabet.descending:before{content:"\f15e"}i.icon.sort.ascending:before{content:"\f0de"}i.icon.sort.content.ascending:before{content:"\f160"}i.icon.sort.content.descending:before{content:"\f161"}i.icon.sort.descending:before{content:"\f0dd"}i.icon.sort.numeric.ascending:before{content:"\f162"}i.icon.sort.numeric.descending:before{content:"\f163"}i.icon.sound:before{content:"\f025"}i.icon.spy:before{content:"\f21b"}i.icon.stripe.card:before{content:"\f1f5"}i.icon.student:before{content:"\f19d"}i.icon.talk:before{content:"\f27a"}i.icon.target:before{content:"\f140"}i.icon.teletype:before{content:"\f1e4"}i.icon.television:before{content:"\f26c"}i.icon.text.cursor:before{content:"\f246"}i.icon.text.telephone:before{content:"\f1e4"}i.icon.theme.isle:before{content:"\f2b2"}i.icon.theme:before{content:"\f043"}i.icon.thermometer:before{content:"\f2c7"}i.icon.thumb.tack:before{content:"\f08d"}i.icon.time:before{content:"\f017"}i.icon.tm:before{content:"\f25c"}i.icon.toggle.down:before{content:"\f150"}i.icon.toggle.left:before{content:"\f191"}i.icon.toggle.right:before{content:"\f152"}i.icon.toggle.up:before{content:"\f151"}i.icon.translate:before{content:"\f1ab"}i.icon.travel:before{content:"\f0b1"}i.icon.treatment:before{content:"\f0f1"}i.icon.triangle.down:before{content:"\f0d7"}i.icon.triangle.left:before{content:"\f0d9"}i.icon.triangle.right:before{content:"\f0da"}i.icon.triangle.up:before{content:"\f0d8"}i.icon.try:before{content:"\f195"}i.icon.unhide:before{content:"\f06e"}i.icon.unlinkify:before{content:"\f127"}i.icon.unmute:before{content:"\f130"}i.icon.usd:before{content:"\f155"}i.icon.user.cancel:before{content:"\f235"}i.icon.user.close:before{content:"\f235"}i.icon.user.delete:before{content:"\f235"}i.icon.user.x:before{content:"\f235"}i.icon.vcard:before{content:"\f2bb"}i.icon.video.camera:before{content:"\f03d"}i.icon.video.play:before{content:"\f144"}i.icon.visa.card:before{content:"\f1f0"}i.icon.visa:before{content:"\f1f0"}i.icon.volume.control.phone:before{content:"\f2a0"}i.icon.wait:before{content:"\f017"}i.icon.warning.circle:before{content:"\f06a"}i.icon.warning.sign:before{content:"\f071"}i.icon.warning:before{content:"\f12a"}i.icon.wechat:before{content:"\f1d7"}i.icon.wi-fi:before{content:"\f1eb"}i.icon.wikipedia:before{content:"\f266"}i.icon.winner:before{content:"\f091"}i.icon.wizard:before{content:"\f0d0"}i.icon.woman:before{content:"\f221"}i.icon.won:before{content:"\f159"}i.icon.wordpress.beginner:before{content:"\f297"}i.icon.wordpress.forms:before{content:"\f298"}i.icon.world:before{content:"\f0ac"}i.icon.write.square:before{content:"\f14b"}i.icon.x:before{content:"\f00d"}i.icon.yc:before{content:"\f23b"}i.icon.ycombinator:before{content:"\f23b"}i.icon.yen:before{content:"\f157"}i.icon.zip:before{content:"\f187"}i.icon.zoom-in:before{content:"\f00e"}i.icon.zoom-out:before{content:"\f010"}i.icon.zoom:before{content:"\f00e"}i.icon.bitbucket.square:before{content:"\f171"}i.icon.checkmark.box:before{content:"\f14a"}i.icon.circle.thin:before{content:"\f111"}i.icon.cloud.download:before{content:"\f381"}i.icon.cloud.upload:before{content:"\f382"}i.icon.compose:before{content:"\f303"}i.icon.conversation:before{content:"\f086"}i.icon.credit.card.alternative:before{content:"\f09d"}i.icon.currency:before{content:"\f3d1"}i.icon.dashboard:before{content:"\f3fd"}i.icon.diamond:before{content:"\f3a5"}i.icon.disk:before{content:"\f0a0"}i.icon.exchange:before{content:"\f362"}i.icon.external.share:before{content:"\f14d"}i.icon.external.square:before{content:"\f360"}i.icon.external:before{content:"\f35d"}i.icon.facebook.official:before{content:"\f082"}i.icon.food:before{content:"\f2e7"}i.icon.hourglass.zero:before{content:"\f253"}i.icon.level.down:before{content:"\f3be"}i.icon.level.up:before{content:"\f3bf"}i.icon.logout:before{content:"\f2f5"}i.icon.meanpath:before{content:"\f0c8"}i.icon.money:before{content:"\f3d1"}i.icon.move:before{content:"\f0b2"}i.icon.pencil:before{content:"\f303"}i.icon.protect:before{content:"\f023"}i.icon.radio:before{content:"\f192"}i.icon.remove.bookmark:before{content:"\f02e"}i.icon.resize.horizontal:before{content:"\f337"}i.icon.resize.vertical:before{content:"\f338"}i.icon.sign-in:before{content:"\f2f6"}i.icon.sign-out:before{content:"\f2f5"}i.icon.spoon:before{content:"\f2e5"}i.icon.star.half.empty:before{content:"\f089"}i.icon.star.half.full:before{content:"\f089"}i.icon.ticket:before{content:"\f3ff"}i.icon.times.rectangle:before{content:"\f410"}i.icon.write:before{content:"\f303"}i.icon.youtube.play:before{content:"\f167"}@font-face{font-family:outline-icons;src:url(themes/default/assets/fonts/outline-icons.eot);src:url(themes/default/assets/fonts/outline-icons.eot?#iefix) format('embedded-opentype'),url(themes/default/assets/fonts/outline-icons.woff2) format('woff2'),url(themes/default/assets/fonts/outline-icons.woff) format('woff'),url(themes/default/assets/fonts/outline-icons.ttf) format('truetype'),url(themes/default/assets/fonts/outline-icons.svg#icons) format('svg');font-style:normal;font-weight:400;font-variant:normal;text-decoration:inherit;text-transform:none}i.icon.outline{font-family:outline-icons}i.icon.address.book.outline:before{content:"\f2b9"}i.icon.address.card.outline:before{content:"\f2bb"}i.icon.arrow.alternate.circle.down.outline:before{content:"\f358"}i.icon.arrow.alternate.circle.left.outline:before{content:"\f359"}i.icon.arrow.alternate.circle.right.outline:before{content:"\f35a"}i.icon.arrow.alternate.circle.up.outline:before{content:"\f35b"}i.icon.bell.outline:before{content:"\f0f3"}i.icon.bell.slash.outline:before{content:"\f1f6"}i.icon.bookmark.outline:before{content:"\f02e"}i.icon.building.outline:before{content:"\f1ad"}i.icon.calendar.outline:before{content:"\f133"}i.icon.calendar.alternate.outline:before{content:"\f073"}i.icon.calendar.check.outline:before{content:"\f274"}i.icon.calendar.minus.outline:before{content:"\f272"}i.icon.calendar.plus.outline:before{content:"\f271"}i.icon.calendar.times.outline:before{content:"\f273"}i.icon.caret.square.down.outline:before{content:"\f150"}i.icon.caret.square.left.outline:before{content:"\f191"}i.icon.caret.square.right.outline:before{content:"\f152"}i.icon.caret.square.up.outline:before{content:"\f151"}i.icon.chart.bar.outline:before{content:"\f080"}i.icon.check.circle.outline:before{content:"\f058"}i.icon.check.square.outline:before{content:"\f14a"}i.icon.circle.outline:before{content:"\f111"}i.icon.clipboard.outline:before{content:"\f328"}i.icon.clock.outline:before{content:"\f017"}i.icon.clone.outline:before{content:"\f24d"}i.icon.closed.captioning.outline:before{content:"\f20a"}i.icon.comment.outline:before{content:"\f075"}i.icon.comment.alternate.outline:before{content:"\f27a"}i.icon.comments.outline:before{content:"\f086"}i.icon.compass.outline:before{content:"\f14e"}i.icon.copy.outline:before{content:"\f0c5"}i.icon.copyright.outline:before{content:"\f1f9"}i.icon.credit.card.outline:before{content:"\f09d"}i.icon.dot.circle.outline:before{content:"\f192"}i.icon.edit.outline:before{content:"\f044"}i.icon.envelope.outline:before{content:"\f0e0"}i.icon.envelope.open.outline:before{content:"\f2b6"}i.icon.eye.slash.outline:before{content:"\f070"}i.icon.file.outline:before{content:"\f15b"}i.icon.file.alternate.outline:before{content:"\f15c"}i.icon.file.archive.outline:before{content:"\f1c6"}i.icon.file.audio.outline:before{content:"\f1c7"}i.icon.file.code.outline:before{content:"\f1c9"}i.icon.file.excel.outline:before{content:"\f1c3"}i.icon.file.image.outline:before{content:"\f1c5"}i.icon.file.pdf.outline:before{content:"\f1c1"}i.icon.file.powerpoint.outline:before{content:"\f1c4"}i.icon.file.video.outline:before{content:"\f1c8"}i.icon.file.word.outline:before{content:"\f1c2"}i.icon.flag.outline:before{content:"\f024"}i.icon.folder.outline:before{content:"\f07b"}i.icon.folder.open.outline:before{content:"\f07c"}i.icon.frown.outline:before{content:"\f119"}i.icon.futbol.outline:before{content:"\f1e3"}i.icon.gem.outline:before{content:"\f3a5"}i.icon.hand.lizard.outline:before{content:"\f258"}i.icon.hand.paper.outline:before{content:"\f256"}i.icon.hand.peace.outline:before{content:"\f25b"}i.icon.hand.point.down.outline:before{content:"\f0a7"}i.icon.hand.point.left.outline:before{content:"\f0a5"}i.icon.hand.point.right.outline:before{content:"\f0a4"}i.icon.hand.point.up.outline:before{content:"\f0a6"}i.icon.hand.pointer.outline:before{content:"\f25a"}i.icon.hand.rock.outline:before{content:"\f255"}i.icon.hand.scissors.outline:before{content:"\f257"}i.icon.hand.spock.outline:before{content:"\f259"}i.icon.handshake.outline:before{content:"\f2b5"}i.icon.hdd.outline:before{content:"\f0a0"}i.icon.heart.outline:before{content:"\f004"}i.icon.hospital.outline:before{content:"\f0f8"}i.icon.hourglass.outline:before{content:"\f254"}i.icon.id.badge.outline:before{content:"\f2c1"}i.icon.id.card.outline:before{content:"\f2c2"}i.icon.image.outline:before{content:"\f03e"}i.icon.images.outline:before{content:"\f302"}i.icon.keyboard.outline:before{content:"\f11c"}i.icon.lemon.outline:before{content:"\f094"}i.icon.life.ring.outline:before{content:"\f1cd"}i.icon.lightbulb.outline:before{content:"\f0eb"}i.icon.list.alternate.outline:before{content:"\f022"}i.icon.map.outline:before{content:"\f279"}i.icon.meh.outline:before{content:"\f11a"}i.icon.minus.square.outline:before{content:"\f146"}i.icon.money.bill.alternate.outline:before{content:"\f3d1"}i.icon.moon.outline:before{content:"\f186"}i.icon.newspaper.outline:before{content:"\f1ea"}i.icon.object.group.outline:before{content:"\f247"}i.icon.object.ungroup.outline:before{content:"\f248"}i.icon.paper.plane.outline:before{content:"\f1d8"}i.icon.pause.circle.outline:before{content:"\f28b"}i.icon.play.circle.outline:before{content:"\f144"}i.icon.plus.square.outline:before{content:"\f0fe"}i.icon.question.circle.outline:before{content:"\f059"}i.icon.registered.outline:before{content:"\f25d"}i.icon.save.outline:before{content:"\f0c7"}i.icon.share.square.outline:before{content:"\f14d"}i.icon.smile.outline:before{content:"\f118"}i.icon.snowflake.outline:before{content:"\f2dc"}i.icon.square.outline:before{content:"\f0c8"}i.icon.star.outline:before{content:"\f005"}i.icon.star.half.outline:before{content:"\f089"}i.icon.sticky.note.outline:before{content:"\f249"}i.icon.stop.circle.outline:before{content:"\f28d"}i.icon.sun.outline:before{content:"\f185"}i.icon.thumbs.down.outline:before{content:"\f165"}i.icon.thumbs.up.outline:before{content:"\f164"}i.icon.times.circle.outline:before{content:"\f057"}i.icon.trash.alternate.outline:before{content:"\f2ed"}i.icon.user.outline:before{content:"\f007"}i.icon.user.circle.outline:before{content:"\f2bd"}i.icon.window.close.outline:before{content:"\f410"}i.icon.window.maximize.outline:before{content:"\f2d0"}i.icon.window.minimize.outline:before{content:"\f2d1"}i.icon.window.restore.outline:before{content:"\f2d2"}i.icon.disk.outline:before{content:"\f0a0"}i.icon.heart.empty,i.icon.star.empty{font-family:outline-icons}i.icon.heart.empty:before{content:"\f004"}i.icon.star.empty:before{content:"\f089"}@font-face{font-family:brand-icons;src:url(themes/default/assets/fonts/brand-icons.eot);src:url(themes/default/assets/fonts/brand-icons.eot?#iefix) format('embedded-opentype'),url(themes/default/assets/fonts/brand-icons.woff2) format('woff2'),url(themes/default/assets/fonts/brand-icons.woff) format('woff'),url(themes/default/assets/fonts/brand-icons.ttf) format('truetype'),url(themes/default/assets/fonts/brand-icons.svg#icons) format('svg');font-style:normal;font-weight:400;font-variant:normal;text-decoration:inherit;text-transform:none}i.icon.\35 00px,i.icon.accessible.icon,i.icon.accusoft,i.icon.adn,i.icon.adversal,i.icon.affiliatetheme,i.icon.algolia,i.icon.amazon,i.icon.amazon.pay,i.icon.amilia,i.icon.android,i.icon.angellist,i.icon.angrycreative,i.icon.angular,i.icon.app.store,i.icon.app.store.ios,i.icon.apper,i.icon.apple,i.icon.apple.pay,i.icon.asymmetrik,i.icon.audible,i.icon.autoprefixer,i.icon.avianex,i.icon.aviato,i.icon.aws,i.icon.bandcamp,i.icon.behance,i.icon.behance.square,i.icon.bimobject,i.icon.bitbucket,i.icon.bitcoin,i.icon.bity,i.icon.black.tie,i.icon.blackberry,i.icon.blogger,i.icon.blogger.b,i.icon.bluetooth,i.icon.bluetooth.b,i.icon.btc,i.icon.buromobelexperte,i.icon.buysellads,i.icon.cc.amazon.pay,i.icon.cc.amex,i.icon.cc.apple.pay,i.icon.cc.diners.club,i.icon.cc.discover,i.icon.cc.jcb,i.icon.cc.mastercard,i.icon.cc.paypal,i.icon.cc.stripe,i.icon.cc.visa,i.icon.centercode,i.icon.chrome,i.icon.cloudscale,i.icon.cloudsmith,i.icon.cloudversify,i.icon.codepen,i.icon.codiepie,i.icon.connectdevelop,i.icon.contao,i.icon.cpanel,i.icon.creative.commons,i.icon.css3,i.icon.css3.alternate,i.icon.cuttlefish,i.icon.d.and.d,i.icon.dashcube,i.icon.delicious,i.icon.deploydog,i.icon.deskpro,i.icon.deviantart,i.icon.digg,i.icon.digital.ocean,i.icon.discord,i.icon.discourse,i.icon.dochub,i.icon.docker,i.icon.draft2digital,i.icon.dribbble,i.icon.dribbble.square,i.icon.dropbox,i.icon.drupal,i.icon.dyalog,i.icon.earlybirds,i.icon.edge,i.icon.elementor,i.icon.ember,i.icon.empire,i.icon.envira,i.icon.erlang,i.icon.ethereum,i.icon.etsy,i.icon.expeditedssl,i.icon.facebook,i.icon.facebook.f,i.icon.facebook.messenger,i.icon.facebook.square,i.icon.firefox,i.icon.first.order,i.icon.firstdraft,i.icon.flickr,i.icon.flipboard,i.icon.fly,i.icon.font.awesome,i.icon.font.awesome.alternate,i.icon.font.awesome.flag,i.icon.fonticons,i.icon.fonticons.fi,i.icon.fort.awesome,i.icon.fort.awesome.alternate,i.icon.forumbee,i.icon.foursquare,i.icon.free.code.camp,i.icon.freebsd,i.icon.get.pocket,i.icon.gg,i.icon.gg.circle,i.icon.git,i.icon.git.square,i.icon.github,i.icon.github.alternate,i.icon.github.square,i.icon.gitkraken,i.icon.gitlab,i.icon.gitter,i.icon.glide,i.icon.glide.g,i.icon.gofore,i.icon.goodreads,i.icon.goodreads.g,i.icon.google,i.icon.google.drive,i.icon.google.play,i.icon.google.plus,i.icon.google.plus.g,i.icon.google.plus.square,i.icon.google.wallet,i.icon.gratipay,i.icon.grav,i.icon.gripfire,i.icon.grunt,i.icon.gulp,i.icon.hacker.news,i.icon.hacker.news.square,i.icon.hips,i.icon.hire.a.helper,i.icon.hooli,i.icon.hotjar,i.icon.houzz,i.icon.html5,i.icon.hubspot,i.icon.imdb,i.icon.instagram,i.icon.internet.explorer,i.icon.ioxhost,i.icon.itunes,i.icon.itunes.note,i.icon.jenkins,i.icon.joget,i.icon.joomla,i.icon.js,i.icon.js.square,i.icon.jsfiddle,i.icon.keycdn,i.icon.kickstarter,i.icon.kickstarter.k,i.icon.korvue,i.icon.laravel,i.icon.lastfm,i.icon.lastfm.square,i.icon.leanpub,i.icon.less,i.icon.linechat,i.icon.linkedin,i.icon.linkedin.alternate,i.icon.linkedin.in,i.icon.linode,i.icon.linux,i.icon.lyft,i.icon.magento,i.icon.maxcdn,i.icon.medapps,i.icon.medium,i.icon.medium.m,i.icon.medrt,i.icon.meetup,i.icon.microsoft,i.icon.mix,i.icon.mixcloud,i.icon.mizuni,i.icon.modx,i.icon.monero,i.icon.napster,i.icon.nintendo.switch,i.icon.node,i.icon.node.js,i.icon.npm,i.icon.ns8,i.icon.nutritionix,i.icon.odnoklassniki,i.icon.odnoklassniki.square,i.icon.opencart,i.icon.openid,i.icon.opera,i.icon.optin.monster,i.icon.osi,i.icon.page4,i.icon.pagelines,i.icon.palfed,i.icon.patreon,i.icon.paypal,i.icon.periscope,i.icon.phabricator,i.icon.phoenix.framework,i.icon.php,i.icon.pied.piper,i.icon.pied.piper.alternate,i.icon.pied.piper.pp,i.icon.pinterest,i.icon.pinterest.p,i.icon.pinterest.square,i.icon.playstation,i.icon.product.hunt,i.icon.pushed,i.icon.python,i.icon.qq,i.icon.quinscape,i.icon.quora,i.icon.ravelry,i.icon.react,i.icon.rebel,i.icon.reddit,i.icon.reddit.alien,i.icon.reddit.square,i.icon.redriver,i.icon.rendact,i.icon.renren,i.icon.replyd,i.icon.resolving,i.icon.rocketchat,i.icon.rockrms,i.icon.safari,i.icon.sass,i.icon.schlix,i.icon.scribd,i.icon.searchengin,i.icon.sellcast,i.icon.sellsy,i.icon.servicestack,i.icon.shirtsinbulk,i.icon.simplybuilt,i.icon.sistrix,i.icon.skyatlas,i.icon.skype,i.icon.slack,i.icon.slack.hash,i.icon.slideshare,i.icon.snapchat,i.icon.snapchat.ghost,i.icon.snapchat.square,i.icon.soundcloud,i.icon.speakap,i.icon.spotify,i.icon.stack.exchange,i.icon.stack.overflow,i.icon.staylinked,i.icon.steam,i.icon.steam.square,i.icon.steam.symbol,i.icon.sticker.mule,i.icon.strava,i.icon.stripe,i.icon.stripe.s,i.icon.studiovinari,i.icon.stumbleupon,i.icon.stumbleupon.circle,i.icon.superpowers,i.icon.supple,i.icon.telegram,i.icon.telegram.plane,i.icon.tencent.weibo,i.icon.themeisle,i.icon.trello,i.icon.tripadvisor,i.icon.tumblr,i.icon.tumblr.square,i.icon.twitch,i.icon.twitter,i.icon.twitter.square,i.icon.typo3,i.icon.uber,i.icon.uikit,i.icon.uniregistry,i.icon.untappd,i.icon.usb,i.icon.ussunnah,i.icon.vaadin,i.icon.viacoin,i.icon.viadeo,i.icon.viadeo.square,i.icon.viber,i.icon.vimeo,i.icon.vimeo.square,i.icon.vimeo.v,i.icon.vine,i.icon.vk,i.icon.vnv,i.icon.vuejs,i.icon.wechat,i.icon.weibo,i.icon.weixin,i.icon.whatsapp,i.icon.whatsapp.square,i.icon.whmcs,i.icon.wikipedia.w,i.icon.windows,i.icon.wordpress,i.icon.wordpress.simple,i.icon.wpbeginner,i.icon.wpexplorer,i.icon.wpforms,i.icon.xbox,i.icon.xing,i.icon.xing.square,i.icon.y.combinator,i.icon.yahoo,i.icon.yandex,i.icon.yandex.international,i.icon.yelp,i.icon.yoast,i.icon.youtube,i.icon.youtube.square{font-family:brand-icons}/*! - * # Semantic UI 2.4.0 - Image - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */.ui.image{position:relative;display:inline-block;vertical-align:middle;max-width:100%;background-color:transparent}img.ui.image{display:block}.ui.image img,.ui.image svg{display:block;max-width:100%;height:auto}.ui.hidden.image,.ui.hidden.images{display:none}.ui.hidden.transition.image,.ui.hidden.transition.images{display:block;visibility:hidden}.ui.images>.hidden.transition{display:inline-block;visibility:hidden}.ui.disabled.image,.ui.disabled.images{cursor:default;opacity:.45}.ui.inline.image,.ui.inline.image img,.ui.inline.image svg{display:inline-block}.ui.top.aligned.image,.ui.top.aligned.image img,.ui.top.aligned.image svg,.ui.top.aligned.images .image{display:inline-block;vertical-align:top}.ui.middle.aligned.image,.ui.middle.aligned.image img,.ui.middle.aligned.image svg,.ui.middle.aligned.images .image{display:inline-block;vertical-align:middle}.ui.bottom.aligned.image,.ui.bottom.aligned.image img,.ui.bottom.aligned.image svg,.ui.bottom.aligned.images .image{display:inline-block;vertical-align:bottom}.ui.rounded.image,.ui.rounded.image>*,.ui.rounded.images .image,.ui.rounded.images .image>*{border-radius:.3125em}.ui.bordered.image img,.ui.bordered.image svg,.ui.bordered.images .image,.ui.bordered.images img,.ui.bordered.images svg,img.ui.bordered.image{border:1px solid rgba(0,0,0,.1)}.ui.circular.image,.ui.circular.images{overflow:hidden}.ui.circular.image,.ui.circular.image>*,.ui.circular.images .image,.ui.circular.images .image>*{border-radius:500rem}.ui.fluid.image,.ui.fluid.image img,.ui.fluid.image svg,.ui.fluid.images,.ui.fluid.images img,.ui.fluid.images svg{display:block;width:100%;height:auto}.ui.avatar.image,.ui.avatar.image img,.ui.avatar.image svg,.ui.avatar.images .image,.ui.avatar.images img,.ui.avatar.images svg{margin-right:.25em;display:inline-block;width:2em;height:2em;border-radius:500rem}.ui.spaced.image{display:inline-block!important;margin-left:.5em;margin-right:.5em}.ui[class*="left spaced"].image{margin-left:.5em;margin-right:0}.ui[class*="right spaced"].image{margin-left:0;margin-right:.5em}.ui.floated.image,.ui.floated.images{float:left;margin-right:1em;margin-bottom:1em}.ui.right.floated.image,.ui.right.floated.images{float:right;margin-right:0;margin-bottom:1em;margin-left:1em}.ui.floated.image:last-child,.ui.floated.images:last-child{margin-bottom:0}.ui.centered.image,.ui.centered.images{margin-left:auto;margin-right:auto}.ui.mini.image,.ui.mini.images .image,.ui.mini.images img,.ui.mini.images svg{width:35px;height:auto;font-size:.78571429rem}.ui.tiny.image,.ui.tiny.images .image,.ui.tiny.images img,.ui.tiny.images svg{width:80px;height:auto;font-size:.85714286rem}.ui.small.image,.ui.small.images .image,.ui.small.images img,.ui.small.images svg{width:150px;height:auto;font-size:.92857143rem}.ui.medium.image,.ui.medium.images .image,.ui.medium.images img,.ui.medium.images svg{width:300px;height:auto;font-size:1rem}.ui.large.image,.ui.large.images .image,.ui.large.images img,.ui.large.images svg{width:450px;height:auto;font-size:1.14285714rem}.ui.big.image,.ui.big.images .image,.ui.big.images img,.ui.big.images svg{width:600px;height:auto;font-size:1.28571429rem}.ui.huge.image,.ui.huge.images .image,.ui.huge.images img,.ui.huge.images svg{width:800px;height:auto;font-size:1.42857143rem}.ui.massive.image,.ui.massive.images .image,.ui.massive.images img,.ui.massive.images svg{width:960px;height:auto;font-size:1.71428571rem}.ui.images{font-size:0;margin:0 -.25rem 0}.ui.images .image,.ui.images>img,.ui.images>svg{display:inline-block;margin:0 .25rem .5rem}/*! - * # Semantic UI 2.4.0 - Input - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */.ui.input{position:relative;font-weight:400;font-style:normal;display:-webkit-inline-box;display:-ms-inline-flexbox;display:inline-flex;color:rgba(0,0,0,.87)}.ui.input>input{margin:0;max-width:100%;-webkit-box-flex:1;-ms-flex:1 0 auto;flex:1 0 auto;outline:0;-webkit-tap-highlight-color:rgba(255,255,255,0);text-align:left;line-height:1.21428571em;font-family:Lato,'Helvetica Neue',Arial,Helvetica,sans-serif;padding:.67857143em 1em;background:#fff;border:1px solid rgba(34,36,38,.15);color:rgba(0,0,0,.87);border-radius:.28571429rem;-webkit-transition:border-color .1s ease,-webkit-box-shadow .1s ease;transition:border-color .1s ease,-webkit-box-shadow .1s ease;transition:box-shadow .1s ease,border-color .1s ease;transition:box-shadow .1s ease,border-color .1s ease,-webkit-box-shadow .1s ease;-webkit-box-shadow:none;box-shadow:none}.ui.input>input::-webkit-input-placeholder{color:rgba(191,191,191,.87)}.ui.input>input::-moz-placeholder{color:rgba(191,191,191,.87)}.ui.input>input:-ms-input-placeholder{color:rgba(191,191,191,.87)}.ui.disabled.input,.ui.input:not(.disabled) input[disabled]{opacity:.45}.ui.disabled.input>input,.ui.input:not(.disabled) input[disabled]{pointer-events:none}.ui.input.down input,.ui.input>input:active{border-color:rgba(0,0,0,.3);background:#fafafa;color:rgba(0,0,0,.87);-webkit-box-shadow:none;box-shadow:none}.ui.loading.loading.input>i.icon:before{position:absolute;content:'';top:50%;left:50%;margin:-.64285714em 0 0 -.64285714em;width:1.28571429em;height:1.28571429em;border-radius:500rem;border:.2em solid rgba(0,0,0,.1)}.ui.loading.loading.input>i.icon:after{position:absolute;content:'';top:50%;left:50%;margin:-.64285714em 0 0 -.64285714em;width:1.28571429em;height:1.28571429em;-webkit-animation:button-spin .6s linear;animation:button-spin .6s linear;-webkit-animation-iteration-count:infinite;animation-iteration-count:infinite;border-radius:500rem;border-color:#767676 transparent transparent;border-style:solid;border-width:.2em;-webkit-box-shadow:0 0 0 1px transparent;box-shadow:0 0 0 1px transparent}.ui.input.focus>input,.ui.input>input:focus{border-color:#85b7d9;background:#fff;color:rgba(0,0,0,.8);-webkit-box-shadow:none;box-shadow:none}.ui.input.focus>input::-webkit-input-placeholder,.ui.input>input:focus::-webkit-input-placeholder{color:rgba(115,115,115,.87)}.ui.input.focus>input::-moz-placeholder,.ui.input>input:focus::-moz-placeholder{color:rgba(115,115,115,.87)}.ui.input.focus>input:-ms-input-placeholder,.ui.input>input:focus:-ms-input-placeholder{color:rgba(115,115,115,.87)}.ui.input.error>input{background-color:#fff6f6;border-color:#e0b4b4;color:#9f3a38;-webkit-box-shadow:none;box-shadow:none}.ui.input.error>input::-webkit-input-placeholder{color:#e7bdbc}.ui.input.error>input::-moz-placeholder{color:#e7bdbc}.ui.input.error>input:-ms-input-placeholder{color:#e7bdbc!important}.ui.input.error>input:focus::-webkit-input-placeholder{color:#da9796}.ui.input.error>input:focus::-moz-placeholder{color:#da9796}.ui.input.error>input:focus:-ms-input-placeholder{color:#da9796!important}.ui.transparent.input>input{border-color:transparent!important;background-color:transparent!important;padding:0!important;-webkit-box-shadow:none!important;box-shadow:none!important;border-radius:0!important}.ui.transparent.icon.input>i.icon{width:1.1em}.ui.transparent.icon.input>input{padding-left:0!important;padding-right:2em!important}.ui.transparent[class*="left icon"].input>input{padding-left:2em!important;padding-right:0!important}.ui.transparent.inverted.input{color:#fff}.ui.transparent.inverted.input>input{color:inherit}.ui.transparent.inverted.input>input::-webkit-input-placeholder{color:rgba(255,255,255,.5)}.ui.transparent.inverted.input>input::-moz-placeholder{color:rgba(255,255,255,.5)}.ui.transparent.inverted.input>input:-ms-input-placeholder{color:rgba(255,255,255,.5)}.ui.icon.input>i.icon{cursor:default;position:absolute;line-height:1;text-align:center;top:0;right:0;margin:0;height:100%;width:2.67142857em;opacity:.5;border-radius:0 .28571429rem .28571429rem 0;-webkit-transition:opacity .3s ease;transition:opacity .3s ease}.ui.icon.input>i.icon:not(.link){pointer-events:none}.ui.icon.input>input{padding-right:2.67142857em!important}.ui.icon.input>i.icon:after,.ui.icon.input>i.icon:before{left:0;position:absolute;text-align:center;top:50%;width:100%;margin-top:-.5em}.ui.icon.input>i.link.icon{cursor:pointer}.ui.icon.input>i.circular.icon{top:.35em;right:.5em}.ui[class*="left icon"].input>i.icon{right:auto;left:1px;border-radius:.28571429rem 0 0 .28571429rem}.ui[class*="left icon"].input>i.circular.icon{right:auto;left:.5em}.ui[class*="left icon"].input>input{padding-left:2.67142857em!important;padding-right:1em!important}.ui.icon.input>input:focus~i.icon{opacity:1}.ui.labeled.input>.label{-webkit-box-flex:0;-ms-flex:0 0 auto;flex:0 0 auto;margin:0;font-size:1em}.ui.labeled.input>.label:not(.corner){padding-top:.78571429em;padding-bottom:.78571429em}.ui.labeled.input:not([class*="corner labeled"]) .label:first-child{border-top-right-radius:0;border-bottom-right-radius:0}.ui.labeled.input:not([class*="corner labeled"]) .label:first-child+input{border-top-left-radius:0;border-bottom-left-radius:0;border-left-color:transparent}.ui.labeled.input:not([class*="corner labeled"]) .label:first-child+input:focus{border-left-color:#85b7d9}.ui[class*="right labeled"].input>input{border-top-right-radius:0!important;border-bottom-right-radius:0!important;border-right-color:transparent!important}.ui[class*="right labeled"].input>input+.label{border-top-left-radius:0;border-bottom-left-radius:0}.ui[class*="right labeled"].input>input:focus{border-right-color:#85b7d9!important}.ui.labeled.input .corner.label{top:1px;right:1px;font-size:.64285714em;border-radius:0 .28571429rem 0 0}.ui[class*="corner labeled"]:not([class*="left corner labeled"]).labeled.input>input{padding-right:2.5em!important}.ui[class*="corner labeled"].icon.input:not([class*="left corner labeled"])>input{padding-right:3.25em!important}.ui[class*="corner labeled"].icon.input:not([class*="left corner labeled"])>.icon{margin-right:1.25em}.ui[class*="left corner labeled"].labeled.input>input{padding-left:2.5em!important}.ui[class*="left corner labeled"].icon.input>input{padding-left:3.25em!important}.ui[class*="left corner labeled"].icon.input>.icon{margin-left:1.25em}.ui.input>.ui.corner.label{top:1px;right:1px}.ui.input>.ui.left.corner.label{right:auto;left:1px}.ui.action.input>.button,.ui.action.input>.buttons{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-align:center;-ms-flex-align:center;align-items:center;-webkit-box-flex:0;-ms-flex:0 0 auto;flex:0 0 auto}.ui.action.input>.button,.ui.action.input>.buttons>.button{padding-top:.78571429em;padding-bottom:.78571429em;margin:0}.ui.action.input:not([class*="left action"])>input{border-top-right-radius:0!important;border-bottom-right-radius:0!important;border-right-color:transparent!important}.ui.action.input:not([class*="left action"])>.button:not(:first-child),.ui.action.input:not([class*="left action"])>.buttons:not(:first-child)>.button,.ui.action.input:not([class*="left action"])>.dropdown:not(:first-child){border-radius:0}.ui.action.input:not([class*="left action"])>.button:last-child,.ui.action.input:not([class*="left action"])>.buttons:last-child>.button,.ui.action.input:not([class*="left action"])>.dropdown:last-child{border-radius:0 .28571429rem .28571429rem 0}.ui.action.input:not([class*="left action"])>input:focus{border-right-color:#85b7d9!important}.ui[class*="left action"].input>input{border-top-left-radius:0!important;border-bottom-left-radius:0!important;border-left-color:transparent!important}.ui[class*="left action"].input>.button,.ui[class*="left action"].input>.buttons>.button,.ui[class*="left action"].input>.dropdown{border-radius:0}.ui[class*="left action"].input>.button:first-child,.ui[class*="left action"].input>.buttons:first-child>.button,.ui[class*="left action"].input>.dropdown:first-child{border-radius:.28571429rem 0 0 .28571429rem}.ui[class*="left action"].input>input:focus{border-left-color:#85b7d9!important}.ui.inverted.input>input{border:none}.ui.fluid.input{display:-webkit-box;display:-ms-flexbox;display:flex}.ui.fluid.input>input{width:0!important}.ui.mini.input{font-size:.78571429em}.ui.small.input{font-size:.92857143em}.ui.input{font-size:1em}.ui.large.input{font-size:1.14285714em}.ui.big.input{font-size:1.28571429em}.ui.huge.input{font-size:1.42857143em}.ui.massive.input{font-size:1.71428571em}/*! - * # Semantic UI 2.4.0 - Label - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */.ui.label{display:inline-block;line-height:1;vertical-align:baseline;margin:0 .14285714em;background-color:#e8e8e8;background-image:none;padding:.5833em .833em;color:rgba(0,0,0,.6);text-transform:none;font-weight:700;border:0 solid transparent;border-radius:.28571429rem;-webkit-transition:background .1s ease;transition:background .1s ease}.ui.label:first-child{margin-left:0}.ui.label:last-child{margin-right:0}a.ui.label{cursor:pointer}.ui.label>a{cursor:pointer;color:inherit;opacity:.5;-webkit-transition:.1s opacity ease;transition:.1s opacity ease}.ui.label>a:hover{opacity:1}.ui.label>img{width:auto!important;vertical-align:middle;height:2.1666em!important}.ui.label>.icon{width:auto;margin:0 .75em 0 0}.ui.label>.detail{display:inline-block;vertical-align:top;font-weight:700;margin-left:1em;opacity:.8}.ui.label>.detail .icon{margin:0 .25em 0 0}.ui.label>.close.icon,.ui.label>.delete.icon{cursor:pointer;margin-right:0;margin-left:.5em;font-size:.92857143em;opacity:.5;-webkit-transition:background .1s ease;transition:background .1s ease}.ui.label>.delete.icon:hover{opacity:1}.ui.labels>.label{margin:0 .5em .5em 0}.ui.header>.ui.label{margin-top:-.29165em}.ui.attached.segment>.ui.top.left.attached.label,.ui.bottom.attached.segment>.ui.top.left.attached.label{border-top-left-radius:0}.ui.attached.segment>.ui.top.right.attached.label,.ui.bottom.attached.segment>.ui.top.right.attached.label{border-top-right-radius:0}.ui.top.attached.segment>.ui.bottom.left.attached.label{border-bottom-left-radius:0}.ui.top.attached.segment>.ui.bottom.right.attached.label{border-bottom-right-radius:0}.ui.top.attached.label+[class*="right floated"]+*,.ui.top.attached.label:first-child+:not(.attached){margin-top:2rem!important}.ui.bottom.attached.label:first-child~:last-child:not(.attached){margin-top:0;margin-bottom:2rem!important}.ui.image.label{width:auto!important;margin-top:0;margin-bottom:0;max-width:9999px;vertical-align:baseline;text-transform:none;background:#e8e8e8;padding:.5833em .833em .5833em .5em;border-radius:.28571429rem;-webkit-box-shadow:none;box-shadow:none}.ui.image.label img{display:inline-block;vertical-align:top;height:2.1666em;margin:-.5833em .5em -.5833em -.5em;border-radius:.28571429rem 0 0 .28571429rem}.ui.image.label .detail{background:rgba(0,0,0,.1);margin:-.5833em -.833em -.5833em .5em;padding:.5833em .833em;border-radius:0 .28571429rem .28571429rem 0}.ui.tag.label,.ui.tag.labels .label{margin-left:1em;position:relative;padding-left:1.5em;padding-right:1.5em;border-radius:0 .28571429rem .28571429rem 0;-webkit-transition:none;transition:none}.ui.tag.label:before,.ui.tag.labels .label:before{position:absolute;-webkit-transform:translateY(-50%) translateX(50%) rotate(-45deg);transform:translateY(-50%) translateX(50%) rotate(-45deg);top:50%;right:100%;content:'';background-color:inherit;background-image:none;width:1.56em;height:1.56em;-webkit-transition:none;transition:none}.ui.tag.label:after,.ui.tag.labels .label:after{position:absolute;content:'';top:50%;left:-.25em;margin-top:-.25em;background-color:#fff!important;width:.5em;height:.5em;-webkit-box-shadow:0 -1px 1px 0 rgba(0,0,0,.3);box-shadow:0 -1px 1px 0 rgba(0,0,0,.3);border-radius:500rem}.ui.corner.label{position:absolute;top:0;right:0;margin:0;padding:0;text-align:center;border-color:#e8e8e8;width:4em;height:4em;z-index:1;-webkit-transition:border-color .1s ease;transition:border-color .1s ease}.ui.corner.label{background-color:transparent!important}.ui.corner.label:after{position:absolute;content:"";right:0;top:0;z-index:-1;width:0;height:0;background-color:transparent!important;border-top:0 solid transparent;border-right:4em solid transparent;border-bottom:4em solid transparent;border-left:0 solid transparent;border-right-color:inherit;-webkit-transition:border-color .1s ease;transition:border-color .1s ease}.ui.corner.label .icon{cursor:default;position:relative;top:.64285714em;left:.78571429em;font-size:1.14285714em;margin:0}.ui.left.corner.label,.ui.left.corner.label:after{right:auto;left:0}.ui.left.corner.label:after{border-top:4em solid transparent;border-right:4em solid transparent;border-bottom:0 solid transparent;border-left:0 solid transparent;border-top-color:inherit}.ui.left.corner.label .icon{left:-.78571429em}.ui.segment>.ui.corner.label{top:-1px;right:-1px}.ui.segment>.ui.left.corner.label{right:auto;left:-1px}.ui.ribbon.label{position:relative;margin:0;min-width:-webkit-max-content;min-width:-moz-max-content;min-width:max-content;border-radius:0 .28571429rem .28571429rem 0;border-color:rgba(0,0,0,.15)}.ui.ribbon.label:after{position:absolute;content:'';top:100%;left:0;background-color:transparent!important;border-style:solid;border-width:0 1.2em 1.2em 0;border-color:transparent;border-right-color:inherit;width:0;height:0}.ui.ribbon.label{left:calc(-1rem - 1.2em);margin-right:-1.2em;padding-left:calc(1rem + 1.2em);padding-right:1.2em}.ui[class*="right ribbon"].label{left:calc(100% + 1rem + 1.2em);padding-left:1.2em;padding-right:calc(1rem + 1.2em)}.ui[class*="right ribbon"].label{text-align:left;-webkit-transform:translateX(-100%);transform:translateX(-100%);border-radius:.28571429rem 0 0 .28571429rem}.ui[class*="right ribbon"].label:after{left:auto;right:0;border-style:solid;border-width:1.2em 1.2em 0 0;border-color:transparent;border-top-color:inherit}.ui.card .image>.ribbon.label,.ui.image>.ribbon.label{position:absolute;top:1rem}.ui.card .image>.ui.ribbon.label,.ui.image>.ui.ribbon.label{left:calc(.05rem - 1.2em)}.ui.card .image>.ui[class*="right ribbon"].label,.ui.image>.ui[class*="right ribbon"].label{left:calc(100% + -.05rem + 1.2em);padding-left:.833em}.ui.table td>.ui.ribbon.label{left:calc(-.78571429em - 1.2em)}.ui.table td>.ui[class*="right ribbon"].label{left:calc(100% + .78571429em + 1.2em);padding-left:.833em}.ui.attached.label,.ui[class*="top attached"].label{width:100%;position:absolute;margin:0;top:0;left:0;padding:.75em 1em;border-radius:.21428571rem .21428571rem 0 0}.ui[class*="bottom attached"].label{top:auto;bottom:0;border-radius:0 0 .21428571rem .21428571rem}.ui[class*="top left attached"].label{width:auto;margin-top:0!important;border-radius:.21428571rem 0 .28571429rem 0}.ui[class*="top right attached"].label{width:auto;left:auto;right:0;border-radius:0 .21428571rem 0 .28571429rem}.ui[class*="bottom left attached"].label{width:auto;top:auto;bottom:0;border-radius:0 .28571429rem 0 .21428571rem}.ui[class*="bottom right attached"].label{top:auto;bottom:0;left:auto;right:0;width:auto;border-radius:.28571429rem 0 .21428571rem 0}.ui.label.disabled{opacity:.5}a.ui.label:hover,a.ui.labels .label:hover{background-color:#e0e0e0;border-color:#e0e0e0;background-image:none;color:rgba(0,0,0,.8)}.ui.labels a.label:hover:before,a.ui.label:hover:before{color:rgba(0,0,0,.8)}.ui.active.label{background-color:#d0d0d0;border-color:#d0d0d0;background-image:none;color:rgba(0,0,0,.95)}.ui.active.label:before{background-color:#d0d0d0;background-image:none;color:rgba(0,0,0,.95)}a.ui.active.label:hover,a.ui.labels .active.label:hover{background-color:#c8c8c8;border-color:#c8c8c8;background-image:none;color:rgba(0,0,0,.95)}.ui.labels a.active.label:ActiveHover:before,a.ui.active.label:ActiveHover:before{background-color:#c8c8c8;background-image:none;color:rgba(0,0,0,.95)}.ui.label.visible:not(.dropdown),.ui.labels.visible .label{display:inline-block!important}.ui.label.hidden,.ui.labels.hidden .label{display:none!important}.ui.red.label,.ui.red.labels .label{background-color:#db2828!important;border-color:#db2828!important;color:#fff!important}.ui.red.labels .label:hover,a.ui.red.label:hover{background-color:#d01919!important;border-color:#d01919!important;color:#fff!important}.ui.red.corner.label,.ui.red.corner.label:hover{background-color:transparent!important}.ui.red.ribbon.label{border-color:#b21e1e!important}.ui.basic.red.label{background:none #fff!important;color:#db2828!important;border-color:#db2828!important}.ui.basic.red.labels a.label:hover,a.ui.basic.red.label:hover{background-color:#fff!important;color:#d01919!important;border-color:#d01919!important}.ui.orange.label,.ui.orange.labels .label{background-color:#f2711c!important;border-color:#f2711c!important;color:#fff!important}.ui.orange.labels .label:hover,a.ui.orange.label:hover{background-color:#f26202!important;border-color:#f26202!important;color:#fff!important}.ui.orange.corner.label,.ui.orange.corner.label:hover{background-color:transparent!important}.ui.orange.ribbon.label{border-color:#cf590c!important}.ui.basic.orange.label{background:none #fff!important;color:#f2711c!important;border-color:#f2711c!important}.ui.basic.orange.labels a.label:hover,a.ui.basic.orange.label:hover{background-color:#fff!important;color:#f26202!important;border-color:#f26202!important}.ui.yellow.label,.ui.yellow.labels .label{background-color:#fbbd08!important;border-color:#fbbd08!important;color:#fff!important}.ui.yellow.labels .label:hover,a.ui.yellow.label:hover{background-color:#eaae00!important;border-color:#eaae00!important;color:#fff!important}.ui.yellow.corner.label,.ui.yellow.corner.label:hover{background-color:transparent!important}.ui.yellow.ribbon.label{border-color:#cd9903!important}.ui.basic.yellow.label{background:none #fff!important;color:#fbbd08!important;border-color:#fbbd08!important}.ui.basic.yellow.labels a.label:hover,a.ui.basic.yellow.label:hover{background-color:#fff!important;color:#eaae00!important;border-color:#eaae00!important}.ui.olive.label,.ui.olive.labels .label{background-color:#b5cc18!important;border-color:#b5cc18!important;color:#fff!important}.ui.olive.labels .label:hover,a.ui.olive.label:hover{background-color:#a7bd0d!important;border-color:#a7bd0d!important;color:#fff!important}.ui.olive.corner.label,.ui.olive.corner.label:hover{background-color:transparent!important}.ui.olive.ribbon.label{border-color:#198f35!important}.ui.basic.olive.label{background:none #fff!important;color:#b5cc18!important;border-color:#b5cc18!important}.ui.basic.olive.labels a.label:hover,a.ui.basic.olive.label:hover{background-color:#fff!important;color:#a7bd0d!important;border-color:#a7bd0d!important}.ui.green.label,.ui.green.labels .label{background-color:#21ba45!important;border-color:#21ba45!important;color:#fff!important}.ui.green.labels .label:hover,a.ui.green.label:hover{background-color:#16ab39!important;border-color:#16ab39!important;color:#fff!important}.ui.green.corner.label,.ui.green.corner.label:hover{background-color:transparent!important}.ui.green.ribbon.label{border-color:#198f35!important}.ui.basic.green.label{background:none #fff!important;color:#21ba45!important;border-color:#21ba45!important}.ui.basic.green.labels a.label:hover,a.ui.basic.green.label:hover{background-color:#fff!important;color:#16ab39!important;border-color:#16ab39!important}.ui.teal.label,.ui.teal.labels .label{background-color:#00b5ad!important;border-color:#00b5ad!important;color:#fff!important}.ui.teal.labels .label:hover,a.ui.teal.label:hover{background-color:#009c95!important;border-color:#009c95!important;color:#fff!important}.ui.teal.corner.label,.ui.teal.corner.label:hover{background-color:transparent!important}.ui.teal.ribbon.label{border-color:#00827c!important}.ui.basic.teal.label{background:none #fff!important;color:#00b5ad!important;border-color:#00b5ad!important}.ui.basic.teal.labels a.label:hover,a.ui.basic.teal.label:hover{background-color:#fff!important;color:#009c95!important;border-color:#009c95!important}.ui.blue.label,.ui.blue.labels .label{background-color:#2185d0!important;border-color:#2185d0!important;color:#fff!important}.ui.blue.labels .label:hover,a.ui.blue.label:hover{background-color:#1678c2!important;border-color:#1678c2!important;color:#fff!important}.ui.blue.corner.label,.ui.blue.corner.label:hover{background-color:transparent!important}.ui.blue.ribbon.label{border-color:#1a69a4!important}.ui.basic.blue.label{background:none #fff!important;color:#2185d0!important;border-color:#2185d0!important}.ui.basic.blue.labels a.label:hover,a.ui.basic.blue.label:hover{background-color:#fff!important;color:#1678c2!important;border-color:#1678c2!important}.ui.violet.label,.ui.violet.labels .label{background-color:#6435c9!important;border-color:#6435c9!important;color:#fff!important}.ui.violet.labels .label:hover,a.ui.violet.label:hover{background-color:#5829bb!important;border-color:#5829bb!important;color:#fff!important}.ui.violet.corner.label,.ui.violet.corner.label:hover{background-color:transparent!important}.ui.violet.ribbon.label{border-color:#502aa1!important}.ui.basic.violet.label{background:none #fff!important;color:#6435c9!important;border-color:#6435c9!important}.ui.basic.violet.labels a.label:hover,a.ui.basic.violet.label:hover{background-color:#fff!important;color:#5829bb!important;border-color:#5829bb!important}.ui.purple.label,.ui.purple.labels .label{background-color:#a333c8!important;border-color:#a333c8!important;color:#fff!important}.ui.purple.labels .label:hover,a.ui.purple.label:hover{background-color:#9627ba!important;border-color:#9627ba!important;color:#fff!important}.ui.purple.corner.label,.ui.purple.corner.label:hover{background-color:transparent!important}.ui.purple.ribbon.label{border-color:#82299f!important}.ui.basic.purple.label{background:none #fff!important;color:#a333c8!important;border-color:#a333c8!important}.ui.basic.purple.labels a.label:hover,a.ui.basic.purple.label:hover{background-color:#fff!important;color:#9627ba!important;border-color:#9627ba!important}.ui.pink.label,.ui.pink.labels .label{background-color:#e03997!important;border-color:#e03997!important;color:#fff!important}.ui.pink.labels .label:hover,a.ui.pink.label:hover{background-color:#e61a8d!important;border-color:#e61a8d!important;color:#fff!important}.ui.pink.corner.label,.ui.pink.corner.label:hover{background-color:transparent!important}.ui.pink.ribbon.label{border-color:#c71f7e!important}.ui.basic.pink.label{background:none #fff!important;color:#e03997!important;border-color:#e03997!important}.ui.basic.pink.labels a.label:hover,a.ui.basic.pink.label:hover{background-color:#fff!important;color:#e61a8d!important;border-color:#e61a8d!important}.ui.brown.label,.ui.brown.labels .label{background-color:#a5673f!important;border-color:#a5673f!important;color:#fff!important}.ui.brown.labels .label:hover,a.ui.brown.label:hover{background-color:#975b33!important;border-color:#975b33!important;color:#fff!important}.ui.brown.corner.label,.ui.brown.corner.label:hover{background-color:transparent!important}.ui.brown.ribbon.label{border-color:#805031!important}.ui.basic.brown.label{background:none #fff!important;color:#a5673f!important;border-color:#a5673f!important}.ui.basic.brown.labels a.label:hover,a.ui.basic.brown.label:hover{background-color:#fff!important;color:#975b33!important;border-color:#975b33!important}.ui.grey.label,.ui.grey.labels .label{background-color:#767676!important;border-color:#767676!important;color:#fff!important}.ui.grey.labels .label:hover,a.ui.grey.label:hover{background-color:#838383!important;border-color:#838383!important;color:#fff!important}.ui.grey.corner.label,.ui.grey.corner.label:hover{background-color:transparent!important}.ui.grey.ribbon.label{border-color:#805031!important}.ui.basic.grey.label{background:none #fff!important;color:#767676!important;border-color:#767676!important}.ui.basic.grey.labels a.label:hover,a.ui.basic.grey.label:hover{background-color:#fff!important;color:#838383!important;border-color:#838383!important}.ui.black.label,.ui.black.labels .label{background-color:#1b1c1d!important;border-color:#1b1c1d!important;color:#fff!important}.ui.black.labels .label:hover,a.ui.black.label:hover{background-color:#27292a!important;border-color:#27292a!important;color:#fff!important}.ui.black.corner.label,.ui.black.corner.label:hover{background-color:transparent!important}.ui.black.ribbon.label{border-color:#805031!important}.ui.basic.black.label{background:none #fff!important;color:#1b1c1d!important;border-color:#1b1c1d!important}.ui.basic.black.labels a.label:hover,a.ui.basic.black.label:hover{background-color:#fff!important;color:#27292a!important;border-color:#27292a!important}.ui.basic.label{background:none #fff;border:1px solid rgba(34,36,38,.15);color:rgba(0,0,0,.87);-webkit-box-shadow:none;box-shadow:none}a.ui.basic.label:hover{text-decoration:none;background:none #fff;color:#1e70bf;-webkit-box-shadow:1px solid rgba(34,36,38,.15);box-shadow:1px solid rgba(34,36,38,.15);-webkit-box-shadow:none;box-shadow:none}.ui.basic.pointing.label:before{border-color:inherit}.ui.fluid.labels>.label,.ui.label.fluid{width:100%;-webkit-box-sizing:border-box;box-sizing:border-box}.ui.inverted.label,.ui.inverted.labels .label{color:rgba(255,255,255,.9)!important}.ui.horizontal.label,.ui.horizontal.labels .label{margin:0 .5em 0 0;padding:.4em .833em;min-width:3em;text-align:center}.ui.circular.label,.ui.circular.labels .label{min-width:2em;min-height:2em;padding:.5em!important;line-height:1em;text-align:center;border-radius:500rem}.ui.empty.circular.label,.ui.empty.circular.labels .label{min-width:0;min-height:0;overflow:hidden;width:.5em;height:.5em;vertical-align:baseline}.ui.pointing.label{position:relative}.ui.attached.pointing.label{position:absolute}.ui.pointing.label:before{background-color:inherit;background-image:inherit;border-width:none;border-style:solid;border-color:inherit}.ui.pointing.label:before{position:absolute;content:'';-webkit-transform:rotate(45deg);transform:rotate(45deg);background-image:none;z-index:2;width:.6666em;height:.6666em;-webkit-transition:background .1s ease;transition:background .1s ease}.ui.pointing.label,.ui[class*="pointing above"].label{margin-top:1em}.ui.pointing.label:before,.ui[class*="pointing above"].label:before{border-width:1px 0 0 1px;-webkit-transform:translateX(-50%) translateY(-50%) rotate(45deg);transform:translateX(-50%) translateY(-50%) rotate(45deg);top:0;left:50%}.ui[class*="bottom pointing"].label,.ui[class*="pointing below"].label{margin-top:0;margin-bottom:1em}.ui[class*="bottom pointing"].label:before,.ui[class*="pointing below"].label:before{border-width:0 1px 1px 0;top:auto;right:auto;-webkit-transform:translateX(-50%) translateY(-50%) rotate(45deg);transform:translateX(-50%) translateY(-50%) rotate(45deg);top:100%;left:50%}.ui[class*="left pointing"].label{margin-top:0;margin-left:.6666em}.ui[class*="left pointing"].label:before{border-width:0 0 1px 1px;-webkit-transform:translateX(-50%) translateY(-50%) rotate(45deg);transform:translateX(-50%) translateY(-50%) rotate(45deg);bottom:auto;right:auto;top:50%;left:0}.ui[class*="right pointing"].label{margin-top:0;margin-right:.6666em}.ui[class*="right pointing"].label:before{border-width:1px 1px 0 0;-webkit-transform:translateX(50%) translateY(-50%) rotate(45deg);transform:translateX(50%) translateY(-50%) rotate(45deg);top:50%;right:0;bottom:auto;left:auto}.ui.basic.pointing.label:before,.ui.basic[class*="pointing above"].label:before{margin-top:-1px}.ui.basic[class*="bottom pointing"].label:before,.ui.basic[class*="pointing below"].label:before{bottom:auto;top:100%;margin-top:1px}.ui.basic[class*="left pointing"].label:before{top:50%;left:-1px}.ui.basic[class*="right pointing"].label:before{top:50%;right:-1px}.ui.floating.label{position:absolute;z-index:100;top:-1em;left:100%;margin:0 0 0 -1.5em!important}.ui.mini.label,.ui.mini.labels .label{font-size:.64285714rem}.ui.tiny.label,.ui.tiny.labels .label{font-size:.71428571rem}.ui.small.label,.ui.small.labels .label{font-size:.78571429rem}.ui.label,.ui.labels .label{font-size:.85714286rem}.ui.large.label,.ui.large.labels .label{font-size:1rem}.ui.big.label,.ui.big.labels .label{font-size:1.28571429rem}.ui.huge.label,.ui.huge.labels .label{font-size:1.42857143rem}.ui.massive.label,.ui.massive.labels .label{font-size:1.71428571rem}/*! - * # Semantic UI 2.4.0 - List - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */.ui.list,ol.ui.list,ul.ui.list{list-style-type:none;margin:1em 0;padding:0 0}.ui.list:first-child,ol.ui.list:first-child,ul.ui.list:first-child{margin-top:0;padding-top:0}.ui.list:last-child,ol.ui.list:last-child,ul.ui.list:last-child{margin-bottom:0;padding-bottom:0}.ui.list .list>.item,.ui.list>.item,ol.ui.list li,ul.ui.list li{display:list-item;table-layout:fixed;list-style-type:none;list-style-position:outside;padding:.21428571em 0;line-height:1.14285714em}.ui.list>.item:after,.ui.list>.list>.item,ol.ui.list>li:first-child:after,ul.ui.list>li:first-child:after{content:'';display:block;height:0;clear:both;visibility:hidden}.ui.list .list>.item:first-child,.ui.list>.item:first-child,ol.ui.list li:first-child,ul.ui.list li:first-child{padding-top:0}.ui.list .list>.item:last-child,.ui.list>.item:last-child,ol.ui.list li:last-child,ul.ui.list li:last-child{padding-bottom:0}.ui.list .list,ol.ui.list ol,ul.ui.list ul{clear:both;margin:0;padding:.75em 0 .25em .5em}.ui.list .list>.item,ol.ui.list ol li,ul.ui.list ul li{padding:.14285714em 0;line-height:inherit}.ui.list .list>.item>i.icon,.ui.list>.item>i.icon{display:table-cell;margin:0;padding-top:0;padding-right:.28571429em;vertical-align:top;-webkit-transition:color .1s ease;transition:color .1s ease}.ui.list .list>.item>i.icon:only-child,.ui.list>.item>i.icon:only-child{display:inline-block;vertical-align:top}.ui.list .list>.item>.image,.ui.list>.item>.image{display:table-cell;background-color:transparent;margin:0;vertical-align:top}.ui.list .list>.item>.image:not(:only-child):not(img),.ui.list>.item>.image:not(:only-child):not(img){padding-right:.5em}.ui.list .list>.item>.image img,.ui.list>.item>.image img{vertical-align:top}.ui.list .list>.item>.image:only-child,.ui.list .list>.item>img.image,.ui.list>.item>.image:only-child,.ui.list>.item>img.image{display:inline-block}.ui.list .list>.item>.content,.ui.list>.item>.content{line-height:1.14285714em}.ui.list .list>.item>.icon+.content,.ui.list .list>.item>.image+.content,.ui.list>.item>.icon+.content,.ui.list>.item>.image+.content{display:table-cell;width:100%;padding:0 0 0 .5em;vertical-align:top}.ui.list .list>.item>img.image+.content,.ui.list>.item>img.image+.content{display:inline-block;width:auto}.ui.list .list>.item>.content>.list,.ui.list>.item>.content>.list{margin-left:0;padding-left:0}.ui.list .list>.item .header,.ui.list>.item .header{display:block;margin:0;font-family:Lato,'Helvetica Neue',Arial,Helvetica,sans-serif;font-weight:700;color:rgba(0,0,0,.87)}.ui.list .list>.item .description,.ui.list>.item .description{display:block;color:rgba(0,0,0,.7)}.ui.list .list>.item a,.ui.list>.item a{cursor:pointer}.ui.list .list>a.item,.ui.list>a.item{cursor:pointer;color:#4183c4}.ui.list .list>a.item:hover,.ui.list>a.item:hover{color:#1e70bf}.ui.list .list>a.item i.icon,.ui.list>a.item i.icon{color:rgba(0,0,0,.4)}.ui.list .list>.item a.header,.ui.list>.item a.header{cursor:pointer;color:#4183c4!important}.ui.list .list>.item a.header:hover,.ui.list>.item a.header:hover{color:#1e70bf!important}.ui[class*="left floated"].list{float:left}.ui[class*="right floated"].list{float:right}.ui.list .list>.item [class*="left floated"],.ui.list>.item [class*="left floated"]{float:left;margin:0 1em 0 0}.ui.list .list>.item [class*="right floated"],.ui.list>.item [class*="right floated"]{float:right;margin:0 0 0 1em}.ui.menu .ui.list .list>.item,.ui.menu .ui.list>.item{display:list-item;table-layout:fixed;background-color:transparent;list-style-type:none;list-style-position:outside;padding:.21428571em 0;line-height:1.14285714em}.ui.menu .ui.list .list>.item:before,.ui.menu .ui.list>.item:before{border:none;background:0 0}.ui.menu .ui.list .list>.item:first-child,.ui.menu .ui.list>.item:first-child{padding-top:0}.ui.menu .ui.list .list>.item:last-child,.ui.menu .ui.list>.item:last-child{padding-bottom:0}.ui.horizontal.list{display:inline-block;font-size:0}.ui.horizontal.list>.item{display:inline-block;margin-left:1em;font-size:1rem}.ui.horizontal.list:not(.celled)>.item:first-child{margin-left:0!important;padding-left:0!important}.ui.horizontal.list .list{padding-left:0;padding-bottom:0}.ui.horizontal.list .list>.item>.content,.ui.horizontal.list .list>.item>.icon,.ui.horizontal.list .list>.item>.image,.ui.horizontal.list>.item>.content,.ui.horizontal.list>.item>.icon,.ui.horizontal.list>.item>.image{vertical-align:middle}.ui.horizontal.list>.item:first-child,.ui.horizontal.list>.item:last-child{padding-top:.21428571em;padding-bottom:.21428571em}.ui.horizontal.list>.item>i.icon{margin:0;padding:0 .25em 0 0}.ui.horizontal.list>.item>.icon,.ui.horizontal.list>.item>.icon+.content{float:none;display:inline-block}.ui.list .list>.disabled.item,.ui.list>.disabled.item{pointer-events:none;color:rgba(40,40,40,.3)!important}.ui.inverted.list .list>.disabled.item,.ui.inverted.list>.disabled.item{color:rgba(225,225,225,.3)!important}.ui.list .list>a.item:hover .icon,.ui.list>a.item:hover .icon{color:rgba(0,0,0,.87)}.ui.inverted.list .list>a.item>.icon,.ui.inverted.list>a.item>.icon{color:rgba(255,255,255,.7)}.ui.inverted.list .list>.item .header,.ui.inverted.list>.item .header{color:rgba(255,255,255,.9)}.ui.inverted.list .list>.item .description,.ui.inverted.list>.item .description{color:rgba(255,255,255,.7)}.ui.inverted.list .list>a.item,.ui.inverted.list>a.item{cursor:pointer;color:rgba(255,255,255,.9)}.ui.inverted.list .list>a.item:hover,.ui.inverted.list>a.item:hover{color:#1e70bf}.ui.inverted.list .item a:not(.ui){color:rgba(255,255,255,.9)!important}.ui.inverted.list .item a:not(.ui):hover{color:#1e70bf!important}.ui.list [class*="top aligned"],.ui.list[class*="top aligned"] .content,.ui.list[class*="top aligned"] .image{vertical-align:top!important}.ui.list [class*="middle aligned"],.ui.list[class*="middle aligned"] .content,.ui.list[class*="middle aligned"] .image{vertical-align:middle!important}.ui.list [class*="bottom aligned"],.ui.list[class*="bottom aligned"] .content,.ui.list[class*="bottom aligned"] .image{vertical-align:bottom!important}.ui.link.list .item,.ui.link.list .item a:not(.ui),.ui.link.list a.item{color:rgba(0,0,0,.4);-webkit-transition:.1s color ease;transition:.1s color ease}.ui.link.list.list .item a:not(.ui):hover,.ui.link.list.list a.item:hover{color:rgba(0,0,0,.8)}.ui.link.list.list .item a:not(.ui):active,.ui.link.list.list a.item:active{color:rgba(0,0,0,.9)}.ui.link.list.list .active.item,.ui.link.list.list .active.item a:not(.ui){color:rgba(0,0,0,.95)}.ui.inverted.link.list .item,.ui.inverted.link.list .item a:not(.ui),.ui.inverted.link.list a.item{color:rgba(255,255,255,.5)}.ui.inverted.link.list.list .item a:not(.ui):hover,.ui.inverted.link.list.list a.item:hover{color:#fff}.ui.inverted.link.list.list .item a:not(.ui):active,.ui.inverted.link.list.list a.item:active{color:#fff}.ui.inverted.link.list.list .active.item a:not(.ui),.ui.inverted.link.list.list a.active.item{color:#fff}.ui.selection.list .list>.item,.ui.selection.list>.item{cursor:pointer;background:0 0;padding:.5em .5em;margin:0;color:rgba(0,0,0,.4);border-radius:.5em;-webkit-transition:.1s color ease,.1s padding-left ease,.1s background-color ease;transition:.1s color ease,.1s padding-left ease,.1s background-color ease}.ui.selection.list .list>.item:last-child,.ui.selection.list>.item:last-child{margin-bottom:0}.ui.selection.list.list>.item:hover,.ui.selection.list>.item:hover{background:rgba(0,0,0,.03);color:rgba(0,0,0,.8)}.ui.selection.list .list>.item:active,.ui.selection.list>.item:active{background:rgba(0,0,0,.05);color:rgba(0,0,0,.9)}.ui.selection.list .list>.item.active,.ui.selection.list>.item.active{background:rgba(0,0,0,.05);color:rgba(0,0,0,.95)}.ui.inverted.selection.list>.item{background:0 0;color:rgba(255,255,255,.5)}.ui.inverted.selection.list>.item:hover{background:rgba(255,255,255,.02);color:#fff}.ui.inverted.selection.list>.item:active{background:rgba(255,255,255,.08);color:#fff}.ui.inverted.selection.list>.item.active{background:rgba(255,255,255,.08);color:#fff}.ui.celled.selection.list .list>.item,.ui.celled.selection.list>.item,.ui.divided.selection.list .list>.item,.ui.divided.selection.list>.item{border-radius:0}.ui.animated.list>.item{-webkit-transition:.25s color ease .1s,.25s padding-left ease .1s,.25s background-color ease .1s;transition:.25s color ease .1s,.25s padding-left ease .1s,.25s background-color ease .1s}.ui.animated.list:not(.horizontal)>.item:hover{padding-left:1em}.ui.fitted.list:not(.selection) .list>.item,.ui.fitted.list:not(.selection)>.item{padding-left:0;padding-right:0}.ui.fitted.selection.list .list>.item,.ui.fitted.selection.list>.item{margin-left:-.5em;margin-right:-.5em}.ui.bulleted.list,ul.ui.list{margin-left:1.25rem}.ui.bulleted.list .list>.item,.ui.bulleted.list>.item,ul.ui.list li{position:relative}.ui.bulleted.list .list>.item:before,.ui.bulleted.list>.item:before,ul.ui.list li:before{-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;pointer-events:none;position:absolute;top:auto;left:auto;font-weight:400;margin-left:-1.25rem;content:'•';opacity:1;color:inherit;vertical-align:top}.ui.bulleted.list .list>a.item:before,.ui.bulleted.list>a.item:before,ul.ui.list li:before{color:rgba(0,0,0,.87)}.ui.bulleted.list .list,ul.ui.list ul{padding-left:1.25rem}.ui.horizontal.bulleted.list,ul.ui.horizontal.bulleted.list{margin-left:0}.ui.horizontal.bulleted.list>.item,ul.ui.horizontal.bulleted.list li{margin-left:1.75rem}.ui.horizontal.bulleted.list>.item:first-child,ul.ui.horizontal.bulleted.list li:first-child{margin-left:0}.ui.horizontal.bulleted.list>.item::before,ul.ui.horizontal.bulleted.list li::before{color:rgba(0,0,0,.87)}.ui.horizontal.bulleted.list>.item:first-child::before,ul.ui.horizontal.bulleted.list li:first-child::before{display:none}.ui.ordered.list,.ui.ordered.list .list,ol.ui.list,ol.ui.list ol{counter-reset:ordered;margin-left:1.25rem;list-style-type:none}.ui.ordered.list .list>.item,.ui.ordered.list>.item,ol.ui.list li{list-style-type:none;position:relative}.ui.ordered.list .list>.item:before,.ui.ordered.list>.item:before,ol.ui.list li:before{position:absolute;top:auto;left:auto;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;pointer-events:none;margin-left:-1.25rem;counter-increment:ordered;content:counters(ordered, ".") " ";text-align:right;color:rgba(0,0,0,.87);vertical-align:middle;opacity:.8}.ui.ordered.inverted.list .list>.item:before,.ui.ordered.inverted.list>.item:before,ol.ui.inverted.list li:before{color:rgba(255,255,255,.7)}.ui.ordered.list>.item[data-value],.ui.ordered.list>.list>.item[data-value]{content:attr(data-value)}ol.ui.list li[value]:before{content:attr(value)}.ui.ordered.list .list,ol.ui.list ol{margin-left:1em}.ui.ordered.list .list>.item:before,ol.ui.list ol li:before{margin-left:-2em}.ui.ordered.horizontal.list,ol.ui.horizontal.list{margin-left:0}.ui.ordered.horizontal.list .list>.item:before,.ui.ordered.horizontal.list>.item:before,ol.ui.horizontal.list li:before{position:static;margin:0 .5em 0 0}.ui.divided.list>.item{border-top:1px solid rgba(34,36,38,.15)}.ui.divided.list .list>.item{border-top:none}.ui.divided.list .item .list>.item{border-top:none}.ui.divided.list .list>.item:first-child,.ui.divided.list>.item:first-child{border-top:none}.ui.divided.list:not(.horizontal) .list>.item:first-child{border-top-width:1px}.ui.divided.bulleted.list .list,.ui.divided.bulleted.list:not(.horizontal){margin-left:0;padding-left:0}.ui.divided.bulleted.list>.item:not(.horizontal){padding-left:1.25rem}.ui.divided.ordered.list{margin-left:0}.ui.divided.ordered.list .list>.item,.ui.divided.ordered.list>.item{padding-left:1.25rem}.ui.divided.ordered.list .item .list{margin-left:0;margin-right:0;padding-bottom:.21428571em}.ui.divided.ordered.list .item .list>.item{padding-left:1em}.ui.divided.selection.list .list>.item,.ui.divided.selection.list>.item{margin:0;border-radius:0}.ui.divided.horizontal.list{margin-left:0}.ui.divided.horizontal.list>.item:not(:first-child){padding-left:.5em}.ui.divided.horizontal.list>.item:not(:last-child){padding-right:.5em}.ui.divided.horizontal.list>.item{border-top:none;border-left:1px solid rgba(34,36,38,.15);margin:0;line-height:.6}.ui.horizontal.divided.list>.item:first-child{border-left:none}.ui.divided.inverted.horizontal.list>.item,.ui.divided.inverted.list>.item,.ui.divided.inverted.list>.list{border-color:rgba(255,255,255,.1)}.ui.celled.list>.item,.ui.celled.list>.list{border-top:1px solid rgba(34,36,38,.15);padding-left:.5em;padding-right:.5em}.ui.celled.list>.item:last-child{border-bottom:1px solid rgba(34,36,38,.15)}.ui.celled.list>.item:first-child,.ui.celled.list>.item:last-child{padding-top:.21428571em;padding-bottom:.21428571em}.ui.celled.list .item .list>.item{border-width:0}.ui.celled.list .list>.item:first-child{border-top-width:0}.ui.celled.bulleted.list{margin-left:0}.ui.celled.bulleted.list .list>.item,.ui.celled.bulleted.list>.item{padding-left:1.25rem}.ui.celled.bulleted.list .item .list{margin-left:-1.25rem;margin-right:-1.25rem;padding-bottom:.21428571em}.ui.celled.ordered.list{margin-left:0}.ui.celled.ordered.list .list>.item,.ui.celled.ordered.list>.item{padding-left:1.25rem}.ui.celled.ordered.list .item .list{margin-left:0;margin-right:0;padding-bottom:.21428571em}.ui.celled.ordered.list .list>.item{padding-left:1em}.ui.horizontal.celled.list{margin-left:0}.ui.horizontal.celled.list .list>.item,.ui.horizontal.celled.list>.item{border-top:none;border-left:1px solid rgba(34,36,38,.15);margin:0;padding-left:.5em;padding-right:.5em;line-height:.6}.ui.horizontal.celled.list .list>.item:last-child,.ui.horizontal.celled.list>.item:last-child{border-bottom:none;border-right:1px solid rgba(34,36,38,.15)}.ui.celled.inverted.list>.item,.ui.celled.inverted.list>.list{border-color:1px solid rgba(255,255,255,.1)}.ui.celled.inverted.horizontal.list .list>.item,.ui.celled.inverted.horizontal.list>.item{border-color:1px solid rgba(255,255,255,.1)}.ui.relaxed.list:not(.horizontal)>.item:not(:first-child){padding-top:.42857143em}.ui.relaxed.list:not(.horizontal)>.item:not(:last-child){padding-bottom:.42857143em}.ui.horizontal.relaxed.list .list>.item:not(:first-child),.ui.horizontal.relaxed.list>.item:not(:first-child){padding-left:1rem}.ui.horizontal.relaxed.list .list>.item:not(:last-child),.ui.horizontal.relaxed.list>.item:not(:last-child){padding-right:1rem}.ui[class*="very relaxed"].list:not(.horizontal)>.item:not(:first-child){padding-top:.85714286em}.ui[class*="very relaxed"].list:not(.horizontal)>.item:not(:last-child){padding-bottom:.85714286em}.ui.horizontal[class*="very relaxed"].list .list>.item:not(:first-child),.ui.horizontal[class*="very relaxed"].list>.item:not(:first-child){padding-left:1.5rem}.ui.horizontal[class*="very relaxed"].list .list>.item:not(:last-child),.ui.horizontal[class*="very relaxed"].list>.item:not(:last-child){padding-right:1.5rem}.ui.mini.list{font-size:.78571429em}.ui.tiny.list{font-size:.85714286em}.ui.small.list{font-size:.92857143em}.ui.list{font-size:1em}.ui.large.list{font-size:1.14285714em}.ui.big.list{font-size:1.28571429em}.ui.huge.list{font-size:1.42857143em}.ui.massive.list{font-size:1.71428571em}.ui.mini.horizontal.list .list>.item,.ui.mini.horizontal.list>.item{font-size:.78571429rem}.ui.tiny.horizontal.list .list>.item,.ui.tiny.horizontal.list>.item{font-size:.85714286rem}.ui.small.horizontal.list .list>.item,.ui.small.horizontal.list>.item{font-size:.92857143rem}.ui.horizontal.list .list>.item,.ui.horizontal.list>.item{font-size:1rem}.ui.large.horizontal.list .list>.item,.ui.large.horizontal.list>.item{font-size:1.14285714rem}.ui.big.horizontal.list .list>.item,.ui.big.horizontal.list>.item{font-size:1.28571429rem}.ui.huge.horizontal.list .list>.item,.ui.huge.horizontal.list>.item{font-size:1.42857143rem}.ui.massive.horizontal.list .list>.item,.ui.massive.horizontal.list>.item{font-size:1.71428571rem}/*! - * # Semantic UI 2.4.0 - Loader - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */.ui.loader{display:none;position:absolute;top:50%;left:50%;margin:0;text-align:center;z-index:1000;-webkit-transform:translateX(-50%) translateY(-50%);transform:translateX(-50%) translateY(-50%)}.ui.loader:before{position:absolute;content:'';top:0;left:50%;width:100%;height:100%;border-radius:500rem;border:.2em solid rgba(0,0,0,.1)}.ui.loader:after{position:absolute;content:'';top:0;left:50%;width:100%;height:100%;-webkit-animation:loader .6s linear;animation:loader .6s linear;-webkit-animation-iteration-count:infinite;animation-iteration-count:infinite;border-radius:500rem;border-color:#767676 transparent transparent;border-style:solid;border-width:.2em;-webkit-box-shadow:0 0 0 1px transparent;box-shadow:0 0 0 1px transparent}@-webkit-keyframes loader{from{-webkit-transform:rotate(0);transform:rotate(0)}to{-webkit-transform:rotate(360deg);transform:rotate(360deg)}}@keyframes loader{from{-webkit-transform:rotate(0);transform:rotate(0)}to{-webkit-transform:rotate(360deg);transform:rotate(360deg)}}.ui.mini.loader:after,.ui.mini.loader:before{width:1rem;height:1rem;margin:0 0 0 -.5rem}.ui.tiny.loader:after,.ui.tiny.loader:before{width:1.14285714rem;height:1.14285714rem;margin:0 0 0 -.57142857rem}.ui.small.loader:after,.ui.small.loader:before{width:1.71428571rem;height:1.71428571rem;margin:0 0 0 -.85714286rem}.ui.loader:after,.ui.loader:before{width:2.28571429rem;height:2.28571429rem;margin:0 0 0 -1.14285714rem}.ui.large.loader:after,.ui.large.loader:before{width:3.42857143rem;height:3.42857143rem;margin:0 0 0 -1.71428571rem}.ui.big.loader:after,.ui.big.loader:before{width:3.71428571rem;height:3.71428571rem;margin:0 0 0 -1.85714286rem}.ui.huge.loader:after,.ui.huge.loader:before{width:4.14285714rem;height:4.14285714rem;margin:0 0 0 -2.07142857rem}.ui.massive.loader:after,.ui.massive.loader:before{width:4.57142857rem;height:4.57142857rem;margin:0 0 0 -2.28571429rem}.ui.dimmer .loader{display:block}.ui.dimmer .ui.loader{color:rgba(255,255,255,.9)}.ui.dimmer .ui.loader:before{border-color:rgba(255,255,255,.15)}.ui.dimmer .ui.loader:after{border-color:#fff transparent transparent}.ui.inverted.dimmer .ui.loader{color:rgba(0,0,0,.87)}.ui.inverted.dimmer .ui.loader:before{border-color:rgba(0,0,0,.1)}.ui.inverted.dimmer .ui.loader:after{border-color:#767676 transparent transparent}.ui.text.loader{width:auto!important;height:auto!important;text-align:center;font-style:normal}.ui.indeterminate.loader:after{animation-direction:reverse;-webkit-animation-duration:1.2s;animation-duration:1.2s}.ui.loader.active,.ui.loader.visible{display:block}.ui.loader.disabled,.ui.loader.hidden{display:none}.ui.inverted.dimmer .ui.mini.loader,.ui.mini.loader{width:1rem;height:1rem;font-size:.78571429em}.ui.inverted.dimmer .ui.tiny.loader,.ui.tiny.loader{width:1.14285714rem;height:1.14285714rem;font-size:.85714286em}.ui.inverted.dimmer .ui.small.loader,.ui.small.loader{width:1.71428571rem;height:1.71428571rem;font-size:.92857143em}.ui.inverted.dimmer .ui.loader,.ui.loader{width:2.28571429rem;height:2.28571429rem;font-size:1em}.ui.inverted.dimmer .ui.large.loader,.ui.large.loader{width:3.42857143rem;height:3.42857143rem;font-size:1.14285714em}.ui.big.loader,.ui.inverted.dimmer .ui.big.loader{width:3.71428571rem;height:3.71428571rem;font-size:1.28571429em}.ui.huge.loader,.ui.inverted.dimmer .ui.huge.loader{width:4.14285714rem;height:4.14285714rem;font-size:1.42857143em}.ui.inverted.dimmer .ui.massive.loader,.ui.massive.loader{width:4.57142857rem;height:4.57142857rem;font-size:1.71428571em}.ui.mini.text.loader{min-width:1rem;padding-top:1.78571429rem}.ui.tiny.text.loader{min-width:1.14285714rem;padding-top:1.92857143rem}.ui.small.text.loader{min-width:1.71428571rem;padding-top:2.5rem}.ui.text.loader{min-width:2.28571429rem;padding-top:3.07142857rem}.ui.large.text.loader{min-width:3.42857143rem;padding-top:4.21428571rem}.ui.big.text.loader{min-width:3.71428571rem;padding-top:4.5rem}.ui.huge.text.loader{min-width:4.14285714rem;padding-top:4.92857143rem}.ui.massive.text.loader{min-width:4.57142857rem;padding-top:5.35714286rem}.ui.inverted.loader{color:rgba(255,255,255,.9)}.ui.inverted.loader:before{border-color:rgba(255,255,255,.15)}.ui.inverted.loader:after{border-top-color:#fff}.ui.inline.loader{position:relative;vertical-align:middle;margin:0;left:0;top:0;-webkit-transform:none;transform:none}.ui.inline.loader.active,.ui.inline.loader.visible{display:inline-block}.ui.centered.inline.loader.active,.ui.centered.inline.loader.visible{display:block;margin-left:auto;margin-right:auto}/*! - * # Semantic UI 2.4.0 - Loader - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */.ui.placeholder{position:static;overflow:hidden;-webkit-animation:placeholderShimmer 2s linear;animation:placeholderShimmer 2s linear;-webkit-animation-iteration-count:infinite;animation-iteration-count:infinite;background-color:#fff;background-image:-webkit-gradient(linear,left top,right top,from(rgba(0,0,0,.08)),color-stop(15%,rgba(0,0,0,.15)),color-stop(30%,rgba(0,0,0,.08)));background-image:-webkit-linear-gradient(left,rgba(0,0,0,.08) 0,rgba(0,0,0,.15) 15%,rgba(0,0,0,.08) 30%);background-image:linear-gradient(to right,rgba(0,0,0,.08) 0,rgba(0,0,0,.15) 15%,rgba(0,0,0,.08) 30%);background-size:1200px 100%;max-width:30rem}@-webkit-keyframes placeholderShimmer{0%{background-position:-1200px 0}100%{background-position:1200px 0}}@keyframes placeholderShimmer{0%{background-position:-1200px 0}100%{background-position:1200px 0}}.ui.placeholder+.ui.placeholder{margin-top:2rem}.ui.placeholder+.ui.placeholder{-webkit-animation-delay:.15s;animation-delay:.15s}.ui.placeholder+.ui.placeholder+.ui.placeholder{-webkit-animation-delay:.3s;animation-delay:.3s}.ui.placeholder+.ui.placeholder+.ui.placeholder+.ui.placeholder{-webkit-animation-delay:.45s;animation-delay:.45s}.ui.placeholder+.ui.placeholder+.ui.placeholder+.ui.placeholder+.ui.placeholder{-webkit-animation-delay:.6s;animation-delay:.6s}.ui.placeholder,.ui.placeholder .image.header:after,.ui.placeholder .line,.ui.placeholder .line:after,.ui.placeholder>:before{background-color:#fff}.ui.placeholder .image:not(.header):not(.ui){height:100px}.ui.placeholder .square.image:not(.header){height:0;overflow:hidden;padding-top:100%}.ui.placeholder .rectangular.image:not(.header){height:0;overflow:hidden;padding-top:75%}.ui.placeholder .line{position:relative;height:.85714286em}.ui.placeholder .line:after,.ui.placeholder .line:before{top:100%;position:absolute;content:'';background-color:inherit}.ui.placeholder .line:before{left:0}.ui.placeholder .line:after{right:0}.ui.placeholder .line{margin-bottom:.5em}.ui.placeholder .line:after,.ui.placeholder .line:before{height:.5em}.ui.placeholder .line:not(:first-child){margin-top:.5em}.ui.placeholder .header{position:relative;overflow:hidden}.ui.placeholder .line:nth-child(1):after{width:0%}.ui.placeholder .line:nth-child(2):after{width:50%}.ui.placeholder .line:nth-child(3):after{width:10%}.ui.placeholder .line:nth-child(4):after{width:35%}.ui.placeholder .line:nth-child(5):after{width:65%}.ui.placeholder .header .line{margin-bottom:.64285714em}.ui.placeholder .header .line:after,.ui.placeholder .header .line:before{height:.64285714em}.ui.placeholder .header .line:not(:first-child){margin-top:.64285714em}.ui.placeholder .header .line:after{width:20%}.ui.placeholder .header .line:nth-child(2):after{width:60%}.ui.placeholder .image.header .line{margin-left:3em}.ui.placeholder .image.header .line:before{width:.71428571rem}.ui.placeholder .image.header:after{display:block;height:.85714286em;content:'';margin-left:3em}.ui.placeholder .header .line:first-child,.ui.placeholder .image .line:first-child,.ui.placeholder .paragraph .line:first-child{height:.01px}.ui.placeholder .header:not(:first-child):before,.ui.placeholder .image:not(:first-child):before,.ui.placeholder .paragraph:not(:first-child):before{height:1.42857143em;content:'';display:block}.ui.inverted.placeholder{background-image:-webkit-gradient(linear,left top,right top,from(rgba(255,255,255,.08)),color-stop(15%,rgba(255,255,255,.14)),color-stop(30%,rgba(255,255,255,.08)));background-image:-webkit-linear-gradient(left,rgba(255,255,255,.08) 0,rgba(255,255,255,.14) 15%,rgba(255,255,255,.08) 30%);background-image:linear-gradient(to right,rgba(255,255,255,.08) 0,rgba(255,255,255,.14) 15%,rgba(255,255,255,.08) 30%)}.ui.inverted.placeholder,.ui.inverted.placeholder .image.header:after,.ui.inverted.placeholder .line,.ui.inverted.placeholder .line:after,.ui.inverted.placeholder>:before{background-color:#1b1c1d}.ui.placeholder .full.line.line.line:after{width:0%}.ui.placeholder .very.long.line.line.line:after{width:10%}.ui.placeholder .long.line.line.line:after{width:35%}.ui.placeholder .medium.line.line.line:after{width:50%}.ui.placeholder .short.line.line.line:after{width:65%}.ui.placeholder .very.short.line.line.line:after{width:80%}.ui.fluid.placeholder{max-width:none}/*! - * # Semantic UI 2.4.0 - Rail - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */.ui.rail{position:absolute;top:0;width:300px;height:100%}.ui.left.rail{left:auto;right:100%;padding:0 2rem 0 0;margin:0 2rem 0 0}.ui.right.rail{left:100%;right:auto;padding:0 0 0 2rem;margin:0 0 0 2rem}.ui.left.internal.rail{left:0;right:auto;padding:0 0 0 2rem;margin:0 0 0 2rem}.ui.right.internal.rail{left:auto;right:0;padding:0 2rem 0 0;margin:0 2rem 0 0}.ui.dividing.rail{width:302.5px}.ui.left.dividing.rail{padding:0 2.5rem 0 0;margin:0 2.5rem 0 0;border-right:1px solid rgba(34,36,38,.15)}.ui.right.dividing.rail{border-left:1px solid rgba(34,36,38,.15);padding:0 0 0 2.5rem;margin:0 0 0 2.5rem}.ui.close.rail{width:calc(300px + 1em)}.ui.close.left.rail{padding:0 1em 0 0;margin:0 1em 0 0}.ui.close.right.rail{padding:0 0 0 1em;margin:0 0 0 1em}.ui.very.close.rail{width:calc(300px + .5em)}.ui.very.close.left.rail{padding:0 .5em 0 0;margin:0 .5em 0 0}.ui.very.close.right.rail{padding:0 0 0 .5em;margin:0 0 0 .5em}.ui.attached.left.rail,.ui.attached.right.rail{padding:0;margin:0}.ui.mini.rail{font-size:.78571429rem}.ui.tiny.rail{font-size:.85714286rem}.ui.small.rail{font-size:.92857143rem}.ui.rail{font-size:1rem}.ui.large.rail{font-size:1.14285714rem}.ui.big.rail{font-size:1.28571429rem}.ui.huge.rail{font-size:1.42857143rem}.ui.massive.rail{font-size:1.71428571rem}/*! - * # Semantic UI 2.4.0 - Reveal - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */.ui.reveal{display:inherit;position:relative!important;font-size:0!important}.ui.reveal>.visible.content{position:absolute!important;top:0!important;left:0!important;z-index:3!important;-webkit-transition:all .5s ease .1s;transition:all .5s ease .1s}.ui.reveal>.hidden.content{position:relative!important;z-index:2!important}.ui.active.reveal .visible.content,.ui.reveal:hover .visible.content{z-index:4!important}.ui.slide.reveal{position:relative!important;overflow:hidden!important;white-space:nowrap}.ui.slide.reveal>.content{display:block;width:100%;white-space:normal;float:left;margin:0;-webkit-transition:-webkit-transform .5s ease .1s;transition:-webkit-transform .5s ease .1s;transition:transform .5s ease .1s;transition:transform .5s ease .1s,-webkit-transform .5s ease .1s}.ui.slide.reveal>.visible.content{position:relative!important}.ui.slide.reveal>.hidden.content{position:absolute!important;left:0!important;width:100%!important;-webkit-transform:translateX(100%)!important;transform:translateX(100%)!important}.ui.slide.active.reveal>.visible.content,.ui.slide.reveal:hover>.visible.content{-webkit-transform:translateX(-100%)!important;transform:translateX(-100%)!important}.ui.slide.active.reveal>.hidden.content,.ui.slide.reveal:hover>.hidden.content{-webkit-transform:translateX(0)!important;transform:translateX(0)!important}.ui.slide.right.reveal>.visible.content{-webkit-transform:translateX(0)!important;transform:translateX(0)!important}.ui.slide.right.reveal>.hidden.content{-webkit-transform:translateX(-100%)!important;transform:translateX(-100%)!important}.ui.slide.right.active.reveal>.visible.content,.ui.slide.right.reveal:hover>.visible.content{-webkit-transform:translateX(100%)!important;transform:translateX(100%)!important}.ui.slide.right.active.reveal>.hidden.content,.ui.slide.right.reveal:hover>.hidden.content{-webkit-transform:translateX(0)!important;transform:translateX(0)!important}.ui.slide.up.reveal>.hidden.content{-webkit-transform:translateY(100%)!important;transform:translateY(100%)!important}.ui.slide.up.active.reveal>.visible.content,.ui.slide.up.reveal:hover>.visible.content{-webkit-transform:translateY(-100%)!important;transform:translateY(-100%)!important}.ui.slide.up.active.reveal>.hidden.content,.ui.slide.up.reveal:hover>.hidden.content{-webkit-transform:translateY(0)!important;transform:translateY(0)!important}.ui.slide.down.reveal>.hidden.content{-webkit-transform:translateY(-100%)!important;transform:translateY(-100%)!important}.ui.slide.down.active.reveal>.visible.content,.ui.slide.down.reveal:hover>.visible.content{-webkit-transform:translateY(100%)!important;transform:translateY(100%)!important}.ui.slide.down.active.reveal>.hidden.content,.ui.slide.down.reveal:hover>.hidden.content{-webkit-transform:translateY(0)!important;transform:translateY(0)!important}.ui.fade.reveal>.visible.content{opacity:1}.ui.fade.active.reveal>.visible.content,.ui.fade.reveal:hover>.visible.content{opacity:0}.ui.move.reveal{position:relative!important;overflow:hidden!important;white-space:nowrap}.ui.move.reveal>.content{display:block;float:left;white-space:normal;margin:0;-webkit-transition:-webkit-transform .5s cubic-bezier(.175,.885,.32,1) .1s;transition:-webkit-transform .5s cubic-bezier(.175,.885,.32,1) .1s;transition:transform .5s cubic-bezier(.175,.885,.32,1) .1s;transition:transform .5s cubic-bezier(.175,.885,.32,1) .1s,-webkit-transform .5s cubic-bezier(.175,.885,.32,1) .1s}.ui.move.reveal>.visible.content{position:relative!important}.ui.move.reveal>.hidden.content{position:absolute!important;left:0!important;width:100%!important}.ui.move.active.reveal>.visible.content,.ui.move.reveal:hover>.visible.content{-webkit-transform:translateX(-100%)!important;transform:translateX(-100%)!important}.ui.move.right.active.reveal>.visible.content,.ui.move.right.reveal:hover>.visible.content{-webkit-transform:translateX(100%)!important;transform:translateX(100%)!important}.ui.move.up.active.reveal>.visible.content,.ui.move.up.reveal:hover>.visible.content{-webkit-transform:translateY(-100%)!important;transform:translateY(-100%)!important}.ui.move.down.active.reveal>.visible.content,.ui.move.down.reveal:hover>.visible.content{-webkit-transform:translateY(100%)!important;transform:translateY(100%)!important}.ui.rotate.reveal>.visible.content{-webkit-transition-duration:.5s;transition-duration:.5s;-webkit-transform:rotate(0);transform:rotate(0)}.ui.rotate.reveal>.visible.content,.ui.rotate.right.reveal>.visible.content{-webkit-transform-origin:bottom right;transform-origin:bottom right}.ui.rotate.active.reveal>.visible.content,.ui.rotate.reveal:hover>.visible.content,.ui.rotate.right.active.reveal>.visible.content,.ui.rotate.right.reveal:hover>.visible.content{-webkit-transform:rotate(110deg);transform:rotate(110deg)}.ui.rotate.left.reveal>.visible.content{-webkit-transform-origin:bottom left;transform-origin:bottom left}.ui.rotate.left.active.reveal>.visible.content,.ui.rotate.left.reveal:hover>.visible.content{-webkit-transform:rotate(-110deg);transform:rotate(-110deg)}.ui.disabled.reveal:hover>.visible.visible.content{position:static!important;display:block!important;opacity:1!important;top:0!important;left:0!important;right:auto!important;bottom:auto!important;-webkit-transform:none!important;transform:none!important}.ui.disabled.reveal:hover>.hidden.hidden.content{display:none!important}.ui.reveal>.ui.ribbon.label{z-index:5}.ui.visible.reveal{overflow:visible}.ui.instant.reveal>.content{-webkit-transition-delay:0s!important;transition-delay:0s!important}.ui.reveal>.content{font-size:1rem!important}/*! - * # Semantic UI 2.4.0 - Segment - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */.ui.segment{position:relative;background:#fff;-webkit-box-shadow:0 1px 2px 0 rgba(34,36,38,.15);box-shadow:0 1px 2px 0 rgba(34,36,38,.15);margin:1rem 0;padding:1em 1em;border-radius:.28571429rem;border:1px solid rgba(34,36,38,.15)}.ui.segment:first-child{margin-top:0}.ui.segment:last-child{margin-bottom:0}.ui.vertical.segment{margin:0;padding-left:0;padding-right:0;background:none transparent;border-radius:0;-webkit-box-shadow:none;box-shadow:none;border:none;border-bottom:1px solid rgba(34,36,38,.15)}.ui.vertical.segment:last-child{border-bottom:none}.ui.inverted.segment>.ui.header{color:#fff}.ui[class*="bottom attached"].segment>[class*="top attached"].label{border-top-left-radius:0;border-top-right-radius:0}.ui[class*="top attached"].segment>[class*="bottom attached"].label{border-bottom-left-radius:0;border-bottom-right-radius:0}.ui.attached.segment:not(.top):not(.bottom)>[class*="top attached"].label{border-top-left-radius:0;border-top-right-radius:0}.ui.attached.segment:not(.top):not(.bottom)>[class*="bottom attached"].label{border-bottom-left-radius:0;border-bottom-right-radius:0}.ui.grid>.row>.ui.segment.column,.ui.grid>.ui.segment.column,.ui.page.grid.segment{padding-top:2em;padding-bottom:2em}.ui.grid.segment{margin:1rem 0;border-radius:.28571429rem}.ui.basic.table.segment{background:#fff;border:1px solid rgba(34,36,38,.15);-webkit-box-shadow:0 1px 2px 0 rgba(34,36,38,.15);box-shadow:0 1px 2px 0 rgba(34,36,38,.15)}.ui[class*="very basic"].table.segment{padding:1em 1em}.ui.placeholder.segment{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column;-webkit-box-pack:center;-ms-flex-pack:center;justify-content:center;-webkit-box-align:stretch;-ms-flex-align:stretch;align-items:stretch;max-width:initial;-webkit-animation:none;animation:none;overflow:visible;padding:1em 1em;min-height:18rem;background:#f9fafb;border-color:rgba(34,36,38,.15);-webkit-box-shadow:0 2px 25px 0 rgba(34,36,38,.05) inset;box-shadow:0 2px 25px 0 rgba(34,36,38,.05) inset}.ui.placeholder.segment .button,.ui.placeholder.segment textarea{display:block}.ui.placeholder.segment .button,.ui.placeholder.segment .field,.ui.placeholder.segment textarea,.ui.placeholder.segment>.ui.input{max-width:15rem;margin-left:auto;margin-right:auto}.ui.placeholder.segment .column .button,.ui.placeholder.segment .column .field,.ui.placeholder.segment .column textarea,.ui.placeholder.segment .column>.ui.input{max-width:15rem;margin-left:auto;margin-right:auto}.ui.placeholder.segment>.inline{-ms-flex-item-align:center;align-self:center}.ui.placeholder.segment>.inline>.button{display:inline-block;width:auto;margin:0 .35714286rem 0 0}.ui.placeholder.segment>.inline>.button:last-child{margin-right:0}.ui.piled.segment,.ui.piled.segments{margin:3em 0;-webkit-box-shadow:'';box-shadow:'';z-index:auto}.ui.piled.segment:first-child{margin-top:0}.ui.piled.segment:last-child{margin-bottom:0}.ui.piled.segment:after,.ui.piled.segment:before,.ui.piled.segments:after,.ui.piled.segments:before{background-color:#fff;visibility:visible;content:'';display:block;height:100%;left:0;position:absolute;width:100%;border:1px solid rgba(34,36,38,.15);-webkit-box-shadow:'';box-shadow:''}.ui.piled.segment:before,.ui.piled.segments:before{-webkit-transform:rotate(-1.2deg);transform:rotate(-1.2deg);top:0;z-index:-2}.ui.piled.segment:after,.ui.piled.segments:after{-webkit-transform:rotate(1.2deg);transform:rotate(1.2deg);top:0;z-index:-1}.ui[class*="top attached"].piled.segment{margin-top:3em;margin-bottom:0}.ui.piled.segment[class*="top attached"]:first-child{margin-top:0}.ui.piled.segment[class*="bottom attached"]{margin-top:0;margin-bottom:3em}.ui.piled.segment[class*="bottom attached"]:last-child{margin-bottom:0}.ui.stacked.segment{padding-bottom:1.4em}.ui.stacked.segment:after,.ui.stacked.segment:before,.ui.stacked.segments:after,.ui.stacked.segments:before{content:'';position:absolute;bottom:-3px;left:0;border-top:1px solid rgba(34,36,38,.15);background:rgba(0,0,0,.03);width:100%;height:6px;visibility:visible}.ui.stacked.segment:before,.ui.stacked.segments:before{display:none}.ui.tall.stacked.segment:before,.ui.tall.stacked.segments:before{display:block;bottom:0}.ui.stacked.inverted.segment:after,.ui.stacked.inverted.segment:before,.ui.stacked.inverted.segments:after,.ui.stacked.inverted.segments:before{background-color:rgba(0,0,0,.03);border-top:1px solid rgba(34,36,38,.35)}.ui.padded.segment{padding:1.5em}.ui[class*="very padded"].segment{padding:3em}.ui.padded.segment.vertical.segment,.ui[class*="very padded"].vertical.segment{padding-left:0;padding-right:0}.ui.compact.segment{display:table}.ui.compact.segments{display:-webkit-inline-box;display:-ms-inline-flexbox;display:inline-flex}.ui.compact.segments .segment,.ui.segments .compact.segment{display:block;-webkit-box-flex:0;-ms-flex:0 1 auto;flex:0 1 auto}.ui.circular.segment{display:table-cell;padding:2em;text-align:center;vertical-align:middle;border-radius:500em}.ui.raised.segment,.ui.raised.segments{-webkit-box-shadow:0 2px 4px 0 rgba(34,36,38,.12),0 2px 10px 0 rgba(34,36,38,.15);box-shadow:0 2px 4px 0 rgba(34,36,38,.12),0 2px 10px 0 rgba(34,36,38,.15)}.ui.segments{-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column;position:relative;margin:1rem 0;border:1px solid rgba(34,36,38,.15);-webkit-box-shadow:0 1px 2px 0 rgba(34,36,38,.15);box-shadow:0 1px 2px 0 rgba(34,36,38,.15);border-radius:.28571429rem}.ui.segments:first-child{margin-top:0}.ui.segments:last-child{margin-bottom:0}.ui.segments>.segment{top:0;bottom:0;border-radius:0;margin:0;width:auto;-webkit-box-shadow:none;box-shadow:none;border:none;border-top:1px solid rgba(34,36,38,.15)}.ui.segments:not(.horizontal)>.segment:first-child{border-top:none;margin-top:0;bottom:0;margin-bottom:0;top:0;border-radius:.28571429rem .28571429rem 0 0}.ui.segments:not(.horizontal)>.segment:last-child{top:0;bottom:0;margin-top:0;margin-bottom:0;-webkit-box-shadow:0 1px 2px 0 rgba(34,36,38,.15),none;box-shadow:0 1px 2px 0 rgba(34,36,38,.15),none;border-radius:0 0 .28571429rem .28571429rem}.ui.segments:not(.horizontal)>.segment:only-child{border-radius:.28571429rem}.ui.segments>.ui.segments{border-top:1px solid rgba(34,36,38,.15);margin:1rem 1rem}.ui.segments>.segments:first-child{border-top:none}.ui.segments>.segment+.segments:not(.horizontal){margin-top:0}.ui.horizontal.segments{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-orient:horizontal;-webkit-box-direction:normal;-ms-flex-direction:row;flex-direction:row;background-color:transparent;border-radius:0;padding:0;background-color:#fff;-webkit-box-shadow:0 1px 2px 0 rgba(34,36,38,.15);box-shadow:0 1px 2px 0 rgba(34,36,38,.15);margin:1rem 0;border-radius:.28571429rem;border:1px solid rgba(34,36,38,.15)}.ui.segments>.horizontal.segments{margin:0;background-color:transparent;border-radius:0;border:none;-webkit-box-shadow:none;box-shadow:none;border-top:1px solid rgba(34,36,38,.15)}.ui.horizontal.segments>.segment{-webkit-box-flex:1;flex:1 1 auto;-ms-flex:1 1 0px;margin:0;min-width:0;background-color:transparent;border-radius:0;border:none;-webkit-box-shadow:none;box-shadow:none;border-left:1px solid rgba(34,36,38,.15)}.ui.segments>.horizontal.segments:first-child{border-top:none}.ui.horizontal.segments>.segment:first-child{border-left:none}.ui.disabled.segment{opacity:.45;color:rgba(40,40,40,.3)}.ui.loading.segment{position:relative;cursor:default;pointer-events:none;text-shadow:none!important;color:transparent!important;-webkit-transition:all 0s linear;transition:all 0s linear}.ui.loading.segment:before{position:absolute;content:'';top:0;left:0;background:rgba(255,255,255,.8);width:100%;height:100%;border-radius:.28571429rem;z-index:100}.ui.loading.segment:after{position:absolute;content:'';top:50%;left:50%;margin:-1.5em 0 0 -1.5em;width:3em;height:3em;-webkit-animation:segment-spin .6s linear;animation:segment-spin .6s linear;-webkit-animation-iteration-count:infinite;animation-iteration-count:infinite;border-radius:500rem;border-color:#767676 rgba(0,0,0,.1) rgba(0,0,0,.1) rgba(0,0,0,.1);border-style:solid;border-width:.2em;-webkit-box-shadow:0 0 0 1px transparent;box-shadow:0 0 0 1px transparent;visibility:visible;z-index:101}@-webkit-keyframes segment-spin{from{-webkit-transform:rotate(0);transform:rotate(0)}to{-webkit-transform:rotate(360deg);transform:rotate(360deg)}}@keyframes segment-spin{from{-webkit-transform:rotate(0);transform:rotate(0)}to{-webkit-transform:rotate(360deg);transform:rotate(360deg)}}.ui.basic.segment{background:none transparent;-webkit-box-shadow:none;box-shadow:none;border:none;border-radius:0}.ui.clearing.segment:after{content:".";display:block;height:0;clear:both;visibility:hidden}.ui.red.segment:not(.inverted){border-top:2px solid #db2828!important}.ui.inverted.red.segment{background-color:#db2828!important;color:#fff!important}.ui.orange.segment:not(.inverted){border-top:2px solid #f2711c!important}.ui.inverted.orange.segment{background-color:#f2711c!important;color:#fff!important}.ui.yellow.segment:not(.inverted){border-top:2px solid #fbbd08!important}.ui.inverted.yellow.segment{background-color:#fbbd08!important;color:#fff!important}.ui.olive.segment:not(.inverted){border-top:2px solid #b5cc18!important}.ui.inverted.olive.segment{background-color:#b5cc18!important;color:#fff!important}.ui.green.segment:not(.inverted){border-top:2px solid #21ba45!important}.ui.inverted.green.segment{background-color:#21ba45!important;color:#fff!important}.ui.teal.segment:not(.inverted){border-top:2px solid #00b5ad!important}.ui.inverted.teal.segment{background-color:#00b5ad!important;color:#fff!important}.ui.blue.segment:not(.inverted){border-top:2px solid #2185d0!important}.ui.inverted.blue.segment{background-color:#2185d0!important;color:#fff!important}.ui.violet.segment:not(.inverted){border-top:2px solid #6435c9!important}.ui.inverted.violet.segment{background-color:#6435c9!important;color:#fff!important}.ui.purple.segment:not(.inverted){border-top:2px solid #a333c8!important}.ui.inverted.purple.segment{background-color:#a333c8!important;color:#fff!important}.ui.pink.segment:not(.inverted){border-top:2px solid #e03997!important}.ui.inverted.pink.segment{background-color:#e03997!important;color:#fff!important}.ui.brown.segment:not(.inverted){border-top:2px solid #a5673f!important}.ui.inverted.brown.segment{background-color:#a5673f!important;color:#fff!important}.ui.grey.segment:not(.inverted){border-top:2px solid #767676!important}.ui.inverted.grey.segment{background-color:#767676!important;color:#fff!important}.ui.black.segment:not(.inverted){border-top:2px solid #1b1c1d!important}.ui.inverted.black.segment{background-color:#1b1c1d!important;color:#fff!important}.ui[class*="left aligned"].segment{text-align:left}.ui[class*="right aligned"].segment{text-align:right}.ui[class*="center aligned"].segment{text-align:center}.ui.floated.segment,.ui[class*="left floated"].segment{float:left;margin-right:1em}.ui[class*="right floated"].segment{float:right;margin-left:1em}.ui.inverted.segment{border:none;-webkit-box-shadow:none;box-shadow:none}.ui.inverted.segment,.ui.primary.inverted.segment{background:#1b1c1d;color:rgba(255,255,255,.9)}.ui.inverted.segment .segment{color:rgba(0,0,0,.87)}.ui.inverted.segment .inverted.segment{color:rgba(255,255,255,.9)}.ui.inverted.attached.segment{border-color:#555}.ui.secondary.segment{background:#f3f4f5;color:rgba(0,0,0,.6)}.ui.secondary.inverted.segment{background:#4c4f52 -webkit-gradient(linear,left top,left bottom,from(rgba(255,255,255,.2)),to(rgba(255,255,255,.2)));background:#4c4f52 -webkit-linear-gradient(rgba(255,255,255,.2) 0,rgba(255,255,255,.2) 100%);background:#4c4f52 linear-gradient(rgba(255,255,255,.2) 0,rgba(255,255,255,.2) 100%);color:rgba(255,255,255,.8)}.ui.tertiary.segment{background:#dcddde;color:rgba(0,0,0,.6)}.ui.tertiary.inverted.segment{background:#717579 -webkit-gradient(linear,left top,left bottom,from(rgba(255,255,255,.35)),to(rgba(255,255,255,.35)));background:#717579 -webkit-linear-gradient(rgba(255,255,255,.35) 0,rgba(255,255,255,.35) 100%);background:#717579 linear-gradient(rgba(255,255,255,.35) 0,rgba(255,255,255,.35) 100%);color:rgba(255,255,255,.8)}.ui.attached.segment{top:0;bottom:0;border-radius:0;margin:0 -1px;width:calc(100% + 2px);max-width:calc(100% + 2px);-webkit-box-shadow:none;box-shadow:none;border:1px solid #d4d4d5}.ui.attached:not(.message)+.ui.attached.segment:not(.top){border-top:none}.ui[class*="top attached"].segment{bottom:0;margin-bottom:0;top:0;margin-top:1rem;border-radius:.28571429rem .28571429rem 0 0}.ui.segment[class*="top attached"]:first-child{margin-top:0}.ui.segment[class*="bottom attached"]{bottom:0;margin-top:0;top:0;margin-bottom:1rem;-webkit-box-shadow:0 1px 2px 0 rgba(34,36,38,.15),none;box-shadow:0 1px 2px 0 rgba(34,36,38,.15),none;border-radius:0 0 .28571429rem .28571429rem}.ui.segment[class*="bottom attached"]:last-child{margin-bottom:0}.ui.mini.segment,.ui.mini.segments .segment{font-size:.78571429rem}.ui.tiny.segment,.ui.tiny.segments .segment{font-size:.85714286rem}.ui.small.segment,.ui.small.segments .segment{font-size:.92857143rem}.ui.segment,.ui.segments .segment{font-size:1rem}.ui.large.segment,.ui.large.segments .segment{font-size:1.14285714rem}.ui.big.segment,.ui.big.segments .segment{font-size:1.28571429rem}.ui.huge.segment,.ui.huge.segments .segment{font-size:1.42857143rem}.ui.massive.segment,.ui.massive.segments .segment{font-size:1.71428571rem}/*! - * # Semantic UI 2.4.0 - Step - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */.ui.steps{display:-webkit-inline-box;display:-ms-inline-flexbox;display:inline-flex;-webkit-box-orient:horizontal;-webkit-box-direction:normal;-ms-flex-direction:row;flex-direction:row;-webkit-box-align:stretch;-ms-flex-align:stretch;align-items:stretch;margin:1em 0;background:'';-webkit-box-shadow:none;box-shadow:none;line-height:1.14285714em;border-radius:.28571429rem;border:1px solid rgba(34,36,38,.15)}.ui.steps:first-child{margin-top:0}.ui.steps:last-child{margin-bottom:0}.ui.steps .step{position:relative;display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-flex:1;-ms-flex:1 0 auto;flex:1 0 auto;-ms-flex-wrap:wrap;flex-wrap:wrap;-webkit-box-orient:horizontal;-webkit-box-direction:normal;-ms-flex-direction:row;flex-direction:row;vertical-align:middle;-webkit-box-align:center;-ms-flex-align:center;align-items:center;-webkit-box-pack:center;-ms-flex-pack:center;justify-content:center;margin:0 0;padding:1.14285714em 2em;background:#fff;color:rgba(0,0,0,.87);-webkit-box-shadow:none;box-shadow:none;border-radius:0;border:none;border-right:1px solid rgba(34,36,38,.15);-webkit-transition:background-color .1s ease,opacity .1s ease,color .1s ease,-webkit-box-shadow .1s ease;transition:background-color .1s ease,opacity .1s ease,color .1s ease,-webkit-box-shadow .1s ease;transition:background-color .1s ease,opacity .1s ease,color .1s ease,box-shadow .1s ease;transition:background-color .1s ease,opacity .1s ease,color .1s ease,box-shadow .1s ease,-webkit-box-shadow .1s ease}.ui.steps .step:after{display:none;position:absolute;z-index:2;content:'';top:50%;right:0;border:medium none;background-color:#fff;width:1.14285714em;height:1.14285714em;border-style:solid;border-color:rgba(34,36,38,.15);border-width:0 1px 1px 0;-webkit-transition:background-color .1s ease,opacity .1s ease,color .1s ease,-webkit-box-shadow .1s ease;transition:background-color .1s ease,opacity .1s ease,color .1s ease,-webkit-box-shadow .1s ease;transition:background-color .1s ease,opacity .1s ease,color .1s ease,box-shadow .1s ease;transition:background-color .1s ease,opacity .1s ease,color .1s ease,box-shadow .1s ease,-webkit-box-shadow .1s ease;-webkit-transform:translateY(-50%) translateX(50%) rotate(-45deg);transform:translateY(-50%) translateX(50%) rotate(-45deg)}.ui.steps .step:first-child{padding-left:2em;border-radius:.28571429rem 0 0 .28571429rem}.ui.steps .step:last-child{border-radius:0 .28571429rem .28571429rem 0}.ui.steps .step:last-child{border-right:none;margin-right:0}.ui.steps .step:only-child{border-radius:.28571429rem}.ui.steps .step .title{font-family:Lato,'Helvetica Neue',Arial,Helvetica,sans-serif;font-size:1.14285714em;font-weight:700}.ui.steps .step>.title{width:100%}.ui.steps .step .description{font-weight:400;font-size:.92857143em;color:rgba(0,0,0,.87)}.ui.steps .step>.description{width:100%}.ui.steps .step .title~.description{margin-top:.25em}.ui.steps .step>.icon{line-height:1;font-size:2.5em;margin:0 1rem 0 0}.ui.steps .step>.icon,.ui.steps .step>.icon~.content{display:block;-webkit-box-flex:0;-ms-flex:0 1 auto;flex:0 1 auto;-ms-flex-item-align:middle;align-self:middle}.ui.steps .step>.icon~.content{-webkit-box-flex:1 0 auto;-ms-flex-positive:1 0 auto;flex-grow:1 0 auto}.ui.steps:not(.vertical) .step>.icon{width:auto}.ui.steps .link.step,.ui.steps a.step{cursor:pointer}.ui.ordered.steps{counter-reset:ordered}.ui.ordered.steps .step:before{display:block;position:static;text-align:center;content:counters(ordered, ".");-ms-flex-item-align:middle;align-self:middle;margin-right:1rem;font-size:2.5em;counter-increment:ordered;font-family:inherit;font-weight:700}.ui.ordered.steps .step>*{display:block;-ms-flex-item-align:middle;align-self:middle}.ui.vertical.steps{display:-webkit-inline-box;display:-ms-inline-flexbox;display:inline-flex;-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column;overflow:visible}.ui.vertical.steps .step{-webkit-box-pack:start;-ms-flex-pack:start;justify-content:flex-start;border-radius:0;padding:1.14285714em 2em;border-right:none;border-bottom:1px solid rgba(34,36,38,.15)}.ui.vertical.steps .step:first-child{padding:1.14285714em 2em;border-radius:.28571429rem .28571429rem 0 0}.ui.vertical.steps .step:last-child{border-bottom:none;border-radius:0 0 .28571429rem .28571429rem}.ui.vertical.steps .step:only-child{border-radius:.28571429rem}.ui.vertical.steps .step:after{display:none}.ui.vertical.steps .step:after{top:50%;right:0;border-width:0 1px 1px 0}.ui.vertical.steps .step:after{display:none}.ui.vertical.steps .active.step:after{display:block}.ui.vertical.steps .step:last-child:after{display:none}.ui.vertical.steps .active.step:last-child:after{display:block}@media only screen and (max-width:767px){.ui.steps:not(.unstackable){display:-webkit-inline-box;display:-ms-inline-flexbox;display:inline-flex;overflow:visible;-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column}.ui.steps:not(.unstackable) .step{width:100%!important;-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column;border-radius:0;padding:1.14285714em 2em}.ui.steps:not(.unstackable) .step:first-child{padding:1.14285714em 2em;border-radius:.28571429rem .28571429rem 0 0}.ui.steps:not(.unstackable) .step:last-child{border-radius:0 0 .28571429rem .28571429rem}.ui.steps:not(.unstackable) .step:after{display:none!important}.ui.steps:not(.unstackable) .step .content{text-align:center}.ui.ordered.steps:not(.unstackable) .step:before,.ui.steps:not(.unstackable) .step>.icon{margin:0 0 1rem 0}}.ui.steps .link.step:hover,.ui.steps .link.step:hover::after,.ui.steps a.step:hover,.ui.steps a.step:hover::after{background:#f9fafb;color:rgba(0,0,0,.8)}.ui.steps .link.step:active,.ui.steps .link.step:active::after,.ui.steps a.step:active,.ui.steps a.step:active::after{background:#f3f4f5;color:rgba(0,0,0,.9)}.ui.steps .step.active{cursor:auto;background:#f3f4f5}.ui.steps .step.active:after{background:#f3f4f5}.ui.steps .step.active .title{color:#4183c4}.ui.ordered.steps .step.active:before,.ui.steps .active.step .icon{color:rgba(0,0,0,.85)}.ui.steps .step:after{display:block}.ui.steps .active.step:after{display:block}.ui.steps .step:last-child:after{display:none}.ui.steps .active.step:last-child:after{display:none}.ui.steps .link.active.step:hover,.ui.steps .link.active.step:hover::after,.ui.steps a.active.step:hover,.ui.steps a.active.step:hover::after{cursor:pointer;background:#dcddde;color:rgba(0,0,0,.87)}.ui.ordered.steps .step.completed:before,.ui.steps .step.completed>.icon:before{color:#21ba45}.ui.steps .disabled.step{cursor:auto;background:#fff;pointer-events:none}.ui.steps .disabled.step,.ui.steps .disabled.step .description,.ui.steps .disabled.step .title{color:rgba(40,40,40,.3)}.ui.steps .disabled.step:after{background:#fff}@media only screen and (max-width:991px){.ui[class*="tablet stackable"].steps{display:-webkit-inline-box;display:-ms-inline-flexbox;display:inline-flex;overflow:visible;-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column}.ui[class*="tablet stackable"].steps .step{-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column;border-radius:0;padding:1.14285714em 2em}.ui[class*="tablet stackable"].steps .step:first-child{padding:1.14285714em 2em;border-radius:.28571429rem .28571429rem 0 0}.ui[class*="tablet stackable"].steps .step:last-child{border-radius:0 0 .28571429rem .28571429rem}.ui[class*="tablet stackable"].steps .step:after{display:none!important}.ui[class*="tablet stackable"].steps .step .content{text-align:center}.ui[class*="tablet stackable"].ordered.steps .step:before,.ui[class*="tablet stackable"].steps .step>.icon{margin:0 0 1rem 0}}.ui.fluid.steps{display:-webkit-box;display:-ms-flexbox;display:flex;width:100%}.ui.attached.steps{width:calc(100% + 2px)!important;margin:0 -1px 0;max-width:calc(100% + 2px);border-radius:.28571429rem .28571429rem 0 0}.ui.attached.steps .step:first-child{border-radius:.28571429rem 0 0 0}.ui.attached.steps .step:last-child{border-radius:0 .28571429rem 0 0}.ui.bottom.attached.steps{margin:0 -1px 0;border-radius:0 0 .28571429rem .28571429rem}.ui.bottom.attached.steps .step:first-child{border-radius:0 0 0 .28571429rem}.ui.bottom.attached.steps .step:last-child{border-radius:0 0 .28571429rem 0}.ui.eight.steps,.ui.five.steps,.ui.four.steps,.ui.one.steps,.ui.seven.steps,.ui.six.steps,.ui.three.steps,.ui.two.steps{width:100%}.ui.eight.steps>.step,.ui.five.steps>.step,.ui.four.steps>.step,.ui.one.steps>.step,.ui.seven.steps>.step,.ui.six.steps>.step,.ui.three.steps>.step,.ui.two.steps>.step{-ms-flex-wrap:nowrap;flex-wrap:nowrap}.ui.one.steps>.step{width:100%}.ui.two.steps>.step{width:50%}.ui.three.steps>.step{width:33.333%}.ui.four.steps>.step{width:25%}.ui.five.steps>.step{width:20%}.ui.six.steps>.step{width:16.666%}.ui.seven.steps>.step{width:14.285%}.ui.eight.steps>.step{width:12.5%}.ui.mini.step,.ui.mini.steps .step{font-size:.78571429rem}.ui.tiny.step,.ui.tiny.steps .step{font-size:.85714286rem}.ui.small.step,.ui.small.steps .step{font-size:.92857143rem}.ui.step,.ui.steps .step{font-size:1rem}.ui.large.step,.ui.large.steps .step{font-size:1.14285714rem}.ui.big.step,.ui.big.steps .step{font-size:1.28571429rem}.ui.huge.step,.ui.huge.steps .step{font-size:1.42857143rem}.ui.massive.step,.ui.massive.steps .step{font-size:1.71428571rem}@font-face{font-family:Step;src:url(data:application/x-font-ttf;charset=utf-8;base64,AAEAAAAOAIAAAwBgT1MvMj3hSQEAAADsAAAAVmNtYXDQEhm3AAABRAAAAUpjdnQgBkn/lAAABuwAAAAcZnBnbYoKeDsAAAcIAAAJkWdhc3AAAAAQAAAG5AAAAAhnbHlm32cEdgAAApAAAAC2aGVhZAErPHsAAANIAAAANmhoZWEHUwNNAAADgAAAACRobXR4CykAAAAAA6QAAAAMbG9jYQA4AFsAAAOwAAAACG1heHAApgm8AAADuAAAACBuYW1lzJ0aHAAAA9gAAALNcG9zdK69QJgAAAaoAAAAO3ByZXCSoZr/AAAQnAAAAFYAAQO4AZAABQAIAnoCvAAAAIwCegK8AAAB4AAxAQIAAAIABQMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAUGZFZABA6ADoAQNS/2oAWgMLAE8AAAABAAAAAAAAAAAAAwAAAAMAAAAcAAEAAAAAAEQAAwABAAAAHAAEACgAAAAGAAQAAQACAADoAf//AAAAAOgA//8AABgBAAEAAAAAAAAAAAEGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAADpAKYABUAHEAZDwEAAQFCAAIBAmoAAQABagAAAGEUFxQDEisBFAcBBiInASY0PwE2Mh8BATYyHwEWA6QP/iAQLBD+6g8PTBAsEKQBbhAsEEwPAhYWEP4gDw8BFhAsEEwQEKUBbxAQTBAAAAH//f+xA18DCwAMABJADwABAQpDAAAACwBEFRMCESsBFA4BIi4CPgEyHgEDWXLG6MhuBnq89Lp+AV51xHR0xOrEdHTEAAAAAAEAAAABAADDeRpdXw889QALA+gAAAAAzzWYjQAAAADPNWBN//3/sQOkAwsAAAAIAAIAAAAAAAAAAQAAA1L/agBaA+gAAP/3A6QAAQAAAAAAAAAAAAAAAAAAAAMD6AAAA+gAAANZAAAAAAAAADgAWwABAAAAAwAWAAEAAAAAAAIABgATAG4AAAAtCZEAAAAAAAAAEgDeAAEAAAAAAAAANQAAAAEAAAAAAAEACAA1AAEAAAAAAAIABwA9AAEAAAAAAAMACABEAAEAAAAAAAQACABMAAEAAAAAAAUACwBUAAEAAAAAAAYACABfAAEAAAAAAAoAKwBnAAEAAAAAAAsAEwCSAAMAAQQJAAAAagClAAMAAQQJAAEAEAEPAAMAAQQJAAIADgEfAAMAAQQJAAMAEAEtAAMAAQQJAAQAEAE9AAMAAQQJAAUAFgFNAAMAAQQJAAYAEAFjAAMAAQQJAAoAVgFzAAMAAQQJAAsAJgHJQ29weXJpZ2h0IChDKSAyMDE0IGJ5IG9yaWdpbmFsIGF1dGhvcnMgQCBmb250ZWxsby5jb21mb250ZWxsb1JlZ3VsYXJmb250ZWxsb2ZvbnRlbGxvVmVyc2lvbiAxLjBmb250ZWxsb0dlbmVyYXRlZCBieSBzdmcydHRmIGZyb20gRm9udGVsbG8gcHJvamVjdC5odHRwOi8vZm9udGVsbG8uY29tAEMAbwBwAHkAcgBpAGcAaAB0ACAAKABDACkAIAAyADAAMQA0ACAAYgB5ACAAbwByAGkAZwBpAG4AYQBsACAAYQB1AHQAaABvAHIAcwAgAEAAIABmAG8AbgB0AGUAbABsAG8ALgBjAG8AbQBmAG8AbgB0AGUAbABsAG8AUgBlAGcAdQBsAGEAcgBmAG8AbgB0AGUAbABsAG8AZgBvAG4AdABlAGwAbABvAFYAZQByAHMAaQBvAG4AIAAxAC4AMABmAG8AbgB0AGUAbABsAG8ARwBlAG4AZQByAGEAdABlAGQAIABiAHkAIABzAHYAZwAyAHQAdABmACAAZgByAG8AbQAgAEYAbwBuAHQAZQBsAGwAbwAgAHAAcgBvAGoAZQBjAHQALgBoAHQAdABwADoALwAvAGYAbwBuAHQAZQBsAGwAbwAuAGMAbwBtAAAAAAIAAAAAAAAACgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAwAAAQIBAwljaGVja21hcmsGY2lyY2xlAAAAAAEAAf//AA8AAAAAAAAAAAAAAAAAAAAAADIAMgML/7EDC/+xsAAssCBgZi2wASwgZCCwwFCwBCZasARFW1ghIyEbilggsFBQWCGwQFkbILA4UFghsDhZWSCwCkVhZLAoUFghsApFILAwUFghsDBZGyCwwFBYIGYgiophILAKUFhgGyCwIFBYIbAKYBsgsDZQWCGwNmAbYFlZWRuwACtZWSOwAFBYZVlZLbACLCBFILAEJWFkILAFQ1BYsAUjQrAGI0IbISFZsAFgLbADLCMhIyEgZLEFYkIgsAYjQrIKAAIqISCwBkMgiiCKsAArsTAFJYpRWGBQG2FSWVgjWSEgsEBTWLAAKxshsEBZI7AAUFhlWS2wBCywB0MrsgACAENgQi2wBSywByNCIyCwACNCYbCAYrABYLAEKi2wBiwgIEUgsAJFY7ABRWJgRLABYC2wBywgIEUgsAArI7ECBCVgIEWKI2EgZCCwIFBYIbAAG7AwUFiwIBuwQFlZI7AAUFhlWbADJSNhRESwAWAtsAgssQUFRbABYUQtsAkssAFgICCwCUNKsABQWCCwCSNCWbAKQ0qwAFJYILAKI0JZLbAKLCC4BABiILgEAGOKI2GwC0NgIIpgILALI0IjLbALLEtUWLEHAURZJLANZSN4LbAMLEtRWEtTWLEHAURZGyFZJLATZSN4LbANLLEADENVWLEMDEOwAWFCsAorWbAAQ7ACJUKxCQIlQrEKAiVCsAEWIyCwAyVQWLEBAENgsAQlQoqKIIojYbAJKiEjsAFhIIojYbAJKiEbsQEAQ2CwAiVCsAIlYbAJKiFZsAlDR7AKQ0dgsIBiILACRWOwAUViYLEAABMjRLABQ7AAPrIBAQFDYEItsA4ssQAFRVRYALAMI0IgYLABYbUNDQEACwBCQopgsQ0FK7BtKxsiWS2wDyyxAA4rLbAQLLEBDistsBEssQIOKy2wEiyxAw4rLbATLLEEDistsBQssQUOKy2wFSyxBg4rLbAWLLEHDistsBcssQgOKy2wGCyxCQ4rLbAZLLAIK7EABUVUWACwDCNCIGCwAWG1DQ0BAAsAQkKKYLENBSuwbSsbIlktsBossQAZKy2wGyyxARkrLbAcLLECGSstsB0ssQMZKy2wHiyxBBkrLbAfLLEFGSstsCAssQYZKy2wISyxBxkrLbAiLLEIGSstsCMssQkZKy2wJCwgPLABYC2wJSwgYLANYCBDI7ABYEOwAiVhsAFgsCQqIS2wJiywJSuwJSotsCcsICBHICCwAkVjsAFFYmAjYTgjIIpVWCBHICCwAkVjsAFFYmAjYTgbIVktsCgssQAFRVRYALABFrAnKrABFTAbIlktsCkssAgrsQAFRVRYALABFrAnKrABFTAbIlktsCosIDWwAWAtsCssALADRWOwAUVisAArsAJFY7ABRWKwACuwABa0AAAAAABEPiM4sSoBFSotsCwsIDwgRyCwAkVjsAFFYmCwAENhOC2wLSwuFzwtsC4sIDwgRyCwAkVjsAFFYmCwAENhsAFDYzgtsC8ssQIAFiUgLiBHsAAjQrACJUmKikcjRyNhIFhiGyFZsAEjQrIuAQEVFCotsDAssAAWsAQlsAQlRyNHI2GwBkUrZYouIyAgPIo4LbAxLLAAFrAEJbAEJSAuRyNHI2EgsAQjQrAGRSsgsGBQWCCwQFFYswIgAyAbswImAxpZQkIjILAIQyCKI0cjRyNhI0ZgsARDsIBiYCCwACsgiophILACQ2BkI7ADQ2FkUFiwAkNhG7ADQ2BZsAMlsIBiYSMgILAEJiNGYTgbI7AIQ0awAiWwCENHI0cjYWAgsARDsIBiYCMgsAArI7AEQ2CwACuwBSVhsAUlsIBisAQmYSCwBCVgZCOwAyVgZFBYIRsjIVkjICCwBCYjRmE4WS2wMiywABYgICCwBSYgLkcjRyNhIzw4LbAzLLAAFiCwCCNCICAgRiNHsAArI2E4LbA0LLAAFrADJbACJUcjRyNhsABUWC4gPCMhG7ACJbACJUcjRyNhILAFJbAEJUcjRyNhsAYlsAUlSbACJWGwAUVjIyBYYhshWWOwAUViYCMuIyAgPIo4IyFZLbA1LLAAFiCwCEMgLkcjRyNhIGCwIGBmsIBiIyAgPIo4LbA2LCMgLkawAiVGUlggPFkusSYBFCstsDcsIyAuRrACJUZQWCA8WS6xJgEUKy2wOCwjIC5GsAIlRlJYIDxZIyAuRrACJUZQWCA8WS6xJgEUKy2wOSywMCsjIC5GsAIlRlJYIDxZLrEmARQrLbA6LLAxK4ogIDywBCNCijgjIC5GsAIlRlJYIDxZLrEmARQrsARDLrAmKy2wOyywABawBCWwBCYgLkcjRyNhsAZFKyMgPCAuIzixJgEUKy2wPCyxCAQlQrAAFrAEJbAEJSAuRyNHI2EgsAQjQrAGRSsgsGBQWCCwQFFYswIgAyAbswImAxpZQkIjIEewBEOwgGJgILAAKyCKimEgsAJDYGQjsANDYWRQWLACQ2EbsANDYFmwAyWwgGJhsAIlRmE4IyA8IzgbISAgRiNHsAArI2E4IVmxJgEUKy2wPSywMCsusSYBFCstsD4ssDErISMgIDywBCNCIzixJgEUK7AEQy6wJistsD8ssAAVIEewACNCsgABARUUEy6wLCotsEAssAAVIEewACNCsgABARUUEy6wLCotsEEssQABFBOwLSotsEIssC8qLbBDLLAAFkUjIC4gRoojYTixJgEUKy2wRCywCCNCsEMrLbBFLLIAADwrLbBGLLIAATwrLbBHLLIBADwrLbBILLIBATwrLbBJLLIAAD0rLbBKLLIAAT0rLbBLLLIBAD0rLbBMLLIBAT0rLbBNLLIAADkrLbBOLLIAATkrLbBPLLIBADkrLbBQLLIBATkrLbBRLLIAADsrLbBSLLIAATsrLbBTLLIBADsrLbBULLIBATsrLbBVLLIAAD4rLbBWLLIAAT4rLbBXLLIBAD4rLbBYLLIBAT4rLbBZLLIAADorLbBaLLIAATorLbBbLLIBADorLbBcLLIBATorLbBdLLAyKy6xJgEUKy2wXiywMiuwNistsF8ssDIrsDcrLbBgLLAAFrAyK7A4Ky2wYSywMysusSYBFCstsGIssDMrsDYrLbBjLLAzK7A3Ky2wZCywMyuwOCstsGUssDQrLrEmARQrLbBmLLA0K7A2Ky2wZyywNCuwNystsGgssDQrsDgrLbBpLLA1Ky6xJgEUKy2waiywNSuwNistsGsssDUrsDcrLbBsLLA1K7A4Ky2wbSwrsAhlsAMkUHiwARUwLQAAAEu4AMhSWLEBAY5ZuQgACABjILABI0SwAyNwsgQoCUVSRLIKAgcqsQYBRLEkAYhRWLBAiFixBgNEsSYBiFFYuAQAiFixBgFEWVlZWbgB/4WwBI2xBQBEAAA=) format('truetype'),url(data:application/font-woff;charset=utf-8;base64,d09GRgABAAAAAAoUAA4AAAAAEPQAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAABPUy8yAAABRAAAAEQAAABWPeFJAWNtYXAAAAGIAAAAOgAAAUrQEhm3Y3Z0IAAAAcQAAAAUAAAAHAZJ/5RmcGdtAAAB2AAABPkAAAmRigp4O2dhc3AAAAbUAAAACAAAAAgAAAAQZ2x5ZgAABtwAAACuAAAAtt9nBHZoZWFkAAAHjAAAADUAAAA2ASs8e2hoZWEAAAfEAAAAIAAAACQHUwNNaG10eAAAB+QAAAAMAAAADAspAABsb2NhAAAH8AAAAAgAAAAIADgAW21heHAAAAf4AAAAIAAAACAApgm8bmFtZQAACBgAAAF3AAACzcydGhxwb3N0AAAJkAAAACoAAAA7rr1AmHByZXAAAAm8AAAAVgAAAFaSoZr/eJxjYGTewTiBgZWBg6mKaQ8DA0MPhGZ8wGDIyMTAwMTAysyAFQSkuaYwOLxgeMHIHPQ/iyGKmZvBHyjMCJIDAPe9C2B4nGNgYGBmgGAZBkYGEHAB8hjBfBYGDSDNBqQZGZgYGF4w/v8PUvCCAURLMELVAwEjG8OIBwBk5AavAAB4nGNgQANGDEbM3P83gjAAELQD4XicnVXZdtNWFJU8ZHASOmSgoA7X3DhQ68qEKRgwaSrFdiEdHAitBB2kDHTkncc+62uOQrtWH/m07n09JLR0rbYsls++R1tn2DrnRhwjKn0aiGvUoZKXA6msPZZK90lc13Uvj5UMBnFdthJPSZuonSRKat3sUC7xWOsqWSdYJ+PlIFZPVZ5noAziFB5lSUQbRBuplyZJ4onjJ4kWZxAfJUkgJaMQp9LIUEI1GsRS1aFM6dCr1xNx00DKRqMedVhU90PFJ8c1p9SsA0YqVznCFevVRr4bpwMve5DEOsGzrYcxHnisfpQqkIqR6cg/dkpOlIaBVHHUoVbi6DCTX/eRTCrNQKaMYkWl7oG43f102xYxPXQ6vi5KlUaqurnOKJrt0fGogygP2cbppNzQ2fbw5RlTVKtdcbPtQGYNXErJbHSfRAAdJlLj6QFONZwCqRn1R8XZ588BEslclKo8VTKHegOZMzt7cTHtbiersnCknwcyb3Z2452HQ6dXh3/R+hdM4cxHj+Jifj5C+lBqfiJOJKVGWMzyp4YfcVcgQrkxiAsXyuBThDl0RdrZZl3jtTH2hs/5SqlhPQna6KP4fgr9TiQrHGdRo/VInM1j13Wt3GdQS7W7Fzsyr0OVIu7vCwuuM+eEYZ4WC1VfnvneBTT/Bohn/EDeNIVL+5YpSrRvm6JMu2iKCu0SVKVdNsUU7YoppmnPmmKG9h1TzNKeMzLj/8vc55H7HN7xkJv2XeSmfQ+5ad9HbtoPkJtWITdtHblpLyA3rUZu2lWjOnYEGgZpF1IVQdA0svph3Fab9UDWjDR8aWDyLmLI+upER521tcofxX914gsHcmmip7siF5viLq/bFj483e6rj5pG3bDV+MaR8jAeRnocmtBZ+c3hv+1N3S6a7jKqMugBFUwKwABl7UAC0zrbCaT1mqf48gdgXIZ4zkpDtVSfO4am7+V5X/exOfG+x+3GLrdcd3kJWdYNcmP28N9SZKrrH+UtrVQnR6wrJ49VaxhDKrwour6SlHu0tRu/KKmy8l6U1srnk5CbPYMbQlu27mGwI0xpyiUeXlOlKD3UUo6yQyxvKco84JSLC1qGxLgOdQ9qa8TpoXoYGwshhqG0vRBwSCldFd+0ynfxHqtr2Oj4xRXh6XpyEhGf4ir7UfBU10b96A7avGbdMoMpVaqn+4xPsa/b9lFZaaSOsxe3VAfXNOsaORXTT+Rr4HRvOGjdAz1UfDRBI1U1x+jGKGM0ljXl3wR0MVZ+w2jVYvs93E+dpFWsuUuY7JsT9+C0u/0q+7WcW0bW/dcGvW3kip8jMb8tCvw7B2K3ZA3UO5OBGAvIWdAYxhYmdxiug23EbfY/Jqf/34aFRXJXOxq7eerD1ZNRJXfZ8rjLTXZZ16M2R9VOGvsIjS0PN+bY4XIstsRgQbb+wf8x7gF3aVEC4NDIZZiI2nShnurh6h6rsW04VxIBds2x43QAegAuQd8cu9bzCYD13CPnLsB9cgh2yCH4lByCz8i5BfA5OQRfkEMwIIdgl5w7AA/IIXhIDsEeOQSPyNkE+JIcgq/IIYjJIUjIuQ3wmByCJ+QQfE0OwTdGrk5k/pYH2QD6zqKbQKmdGhzaOGRGrk3Y+zxY9oFFZB9aROqRkesT6lMeLPV7i0j9wSJSfzRyY0L9iQdL/dkiUn+xiNRnxpeZIymvDp7zjg7+BJfqrV4AAAAAAQAB//8AD3icY2BkAALmJUwzGEQZZBwk+RkZGBmdGJgYmbIYgMwsoGSiiLgIs5A2owg7I5uSOqOaiT2jmZE8I5gQY17C/09BQEfg3yt+fh8gvYQxD0j68DOJiQn8U+DnZxQDcQUEljLmCwBpBgbG/3//b2SOZ+Zm4GEQcuAH2sblDLSEm8FFVJhJEGgLH6OSHpMdo5EcI3Nk0bEXJ/LYqvZ82VXHGFd6pKTkyCsQwQAAq+QkqAAAeJxjYGRgYADiw5VSsfH8Nl8ZuJlfAEUYzpvO6IXQCb7///7fyLyEmRvI5WBgAokCAFb/DJAAAAB4nGNgZGBgDvqfxRDF/IKB4f935iUMQBEUwAwAi5YFpgPoAAAD6AAAA1kAAAAAAAAAOABbAAEAAAADABYAAQAAAAAAAgAGABMAbgAAAC0JkQAAAAB4nHWQy2rCQBSG//HSi0JbWui2sypKabxgN4IgWHTTbqS4LTHGJBIzMhkFX6Pv0IfpS/RZ+puMpShNmMx3vjlz5mQAXOMbAvnzxJGzwBmjnAs4Rc9ykf7Zcon8YrmMKt4sn9C/W67gAYHlKm7wwQqidM5ogU/LAlfi0nIBF+LOcpH+0XKJ3LNcxq14tXxC71muYCJSy1Xci6+BWm11FIRG1gZ12W62OnK6lYoqStxYumsTKp3KvpyrxPhxrBxPLfc89oN17Op9uJ8nvk4jlciW09yrkZ/42jX+bFc93QRtY+ZyrtVSDm2GXGm18D3jhMasuo3G3/MwgMIKW2hEvKoQBhI12jrnNppooUOaMkMyM8+KkMBFTONizR1htpIy7nPMGSW0PjNisgOP3+WRH5MC7o9ZRR+tHsYT0u6MKPOSfTns7jBrREqyTDezs9/eU2x4WpvWcNeuS511JTE8qCF5H7u1BY1H72S3Ymi7aPD95/9+AN1fhEsAeJxjYGKAAC4G7ICZgYGRiZGZMzkjNTk7N7Eomy05syg5J5WBAQBE1QZBAABLuADIUlixAQGOWbkIAAgAYyCwASNEsAMjcLIEKAlFUkSyCgIHKrEGAUSxJAGIUViwQIhYsQYDRLEmAYhRWLgEAIhYsQYBRFlZWVm4Af+FsASNsQUARAAA) format('woff')}.ui.ordered.steps .step.completed:before,.ui.steps .step.completed>.icon:before{font-family:Step;content:'\e800'}/*! - * # Semantic UI 2.4.0 - Breadcrumb - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */.ui.breadcrumb{line-height:1;display:inline-block;margin:0 0;vertical-align:middle}.ui.breadcrumb:first-child{margin-top:0}.ui.breadcrumb:last-child{margin-bottom:0}.ui.breadcrumb .divider{display:inline-block;opacity:.7;margin:0 .21428571rem 0;font-size:.92857143em;color:rgba(0,0,0,.4);vertical-align:baseline}.ui.breadcrumb a{color:#4183c4}.ui.breadcrumb a:hover{color:#1e70bf}.ui.breadcrumb .icon.divider{font-size:.85714286em;vertical-align:baseline}.ui.breadcrumb a.section{cursor:pointer}.ui.breadcrumb .section{display:inline-block;margin:0;padding:0}.ui.breadcrumb.segment{display:inline-block;padding:.78571429em 1em}.ui.breadcrumb .active.section{font-weight:700}.ui.mini.breadcrumb{font-size:.78571429rem}.ui.tiny.breadcrumb{font-size:.85714286rem}.ui.small.breadcrumb{font-size:.92857143rem}.ui.breadcrumb{font-size:1rem}.ui.large.breadcrumb{font-size:1.14285714rem}.ui.big.breadcrumb{font-size:1.28571429rem}.ui.huge.breadcrumb{font-size:1.42857143rem}.ui.massive.breadcrumb{font-size:1.71428571rem}/*! - * # Semantic UI 2.4.0 - Form - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */.ui.form{position:relative;max-width:100%}.ui.form>p{margin:1em 0}.ui.form .field{clear:both;margin:0 0 1em}.ui.form .field:last-child,.ui.form .fields:last-child .field{margin-bottom:0}.ui.form .fields .field{clear:both;margin:0}.ui.form .field>label{display:block;margin:0 0 .28571429rem 0;color:rgba(0,0,0,.87);font-size:.92857143em;font-weight:700;text-transform:none}.ui.form input:not([type]),.ui.form input[type=date],.ui.form input[type=datetime-local],.ui.form input[type=email],.ui.form input[type=file],.ui.form input[type=number],.ui.form input[type=password],.ui.form input[type=search],.ui.form input[type=tel],.ui.form input[type=text],.ui.form input[type=time],.ui.form input[type=url],.ui.form textarea{width:100%;vertical-align:top}.ui.form ::-webkit-datetime-edit,.ui.form ::-webkit-inner-spin-button{height:1.21428571em}.ui.form input:not([type]),.ui.form input[type=date],.ui.form input[type=datetime-local],.ui.form input[type=email],.ui.form input[type=file],.ui.form input[type=number],.ui.form input[type=password],.ui.form input[type=search],.ui.form input[type=tel],.ui.form input[type=text],.ui.form input[type=time],.ui.form input[type=url]{font-family:Lato,'Helvetica Neue',Arial,Helvetica,sans-serif;margin:0;outline:0;-webkit-appearance:none;tap-highlight-color:rgba(255,255,255,0);line-height:1.21428571em;padding:.67857143em 1em;font-size:1em;background:#fff;border:1px solid rgba(34,36,38,.15);color:rgba(0,0,0,.87);border-radius:.28571429rem;-webkit-box-shadow:0 0 0 0 transparent inset;box-shadow:0 0 0 0 transparent inset;-webkit-transition:color .1s ease,border-color .1s ease;transition:color .1s ease,border-color .1s ease}.ui.form textarea{margin:0;-webkit-appearance:none;tap-highlight-color:rgba(255,255,255,0);padding:.78571429em 1em;background:#fff;border:1px solid rgba(34,36,38,.15);outline:0;color:rgba(0,0,0,.87);border-radius:.28571429rem;-webkit-box-shadow:0 0 0 0 transparent inset;box-shadow:0 0 0 0 transparent inset;-webkit-transition:color .1s ease,border-color .1s ease;transition:color .1s ease,border-color .1s ease;font-size:1em;line-height:1.2857;resize:vertical}.ui.form textarea:not([rows]){height:12em;min-height:8em;max-height:24em}.ui.form input[type=checkbox],.ui.form textarea{vertical-align:top}.ui.form input.attached{width:auto}.ui.form select{display:block;height:auto;width:100%;background:#fff;border:1px solid rgba(34,36,38,.15);border-radius:.28571429rem;-webkit-box-shadow:0 0 0 0 transparent inset;box-shadow:0 0 0 0 transparent inset;padding:.62em 1em;color:rgba(0,0,0,.87);-webkit-transition:color .1s ease,border-color .1s ease;transition:color .1s ease,border-color .1s ease}.ui.form .field>.selection.dropdown{width:100%}.ui.form .field>.selection.dropdown>.dropdown.icon{float:right}.ui.form .inline.field>.selection.dropdown,.ui.form .inline.fields .field>.selection.dropdown{width:auto}.ui.form .inline.field>.selection.dropdown>.dropdown.icon,.ui.form .inline.fields .field>.selection.dropdown>.dropdown.icon{float:none}.ui.form .field .ui.input,.ui.form .fields .field .ui.input,.ui.form .wide.field .ui.input{width:100%}.ui.form .inline.field:not(.wide) .ui.input,.ui.form .inline.fields .field:not(.wide) .ui.input{width:auto;vertical-align:middle}.ui.form .field .ui.input input,.ui.form .fields .field .ui.input input{width:auto}.ui.form .eight.fields .ui.input input,.ui.form .five.fields .ui.input input,.ui.form .four.fields .ui.input input,.ui.form .nine.fields .ui.input input,.ui.form .seven.fields .ui.input input,.ui.form .six.fields .ui.input input,.ui.form .ten.fields .ui.input input,.ui.form .three.fields .ui.input input,.ui.form .two.fields .ui.input input,.ui.form .wide.field .ui.input input{-webkit-box-flex:1;-ms-flex:1 0 auto;flex:1 0 auto;width:0}.ui.form .error.message,.ui.form .success.message,.ui.form .warning.message{display:none}.ui.form .message:first-child{margin-top:0}.ui.form .field .prompt.label{white-space:normal;background:#fff!important;border:1px solid #e0b4b4!important;color:#9f3a38!important}.ui.form .inline.field .prompt,.ui.form .inline.fields .field .prompt{vertical-align:top;margin:-.25em 0 -.5em .5em}.ui.form .inline.field .prompt:before,.ui.form .inline.fields .field .prompt:before{border-width:0 0 1px 1px;bottom:auto;right:auto;top:50%;left:0}.ui.form .field.field input:-webkit-autofill{-webkit-box-shadow:0 0 0 100px ivory inset!important;box-shadow:0 0 0 100px ivory inset!important;border-color:#e5dfa1!important}.ui.form .field.field input:-webkit-autofill:focus{-webkit-box-shadow:0 0 0 100px ivory inset!important;box-shadow:0 0 0 100px ivory inset!important;border-color:#d5c315!important}.ui.form .error.error input:-webkit-autofill{-webkit-box-shadow:0 0 0 100px #fffaf0 inset!important;box-shadow:0 0 0 100px #fffaf0 inset!important;border-color:#e0b4b4!important}.ui.form ::-webkit-input-placeholder{color:rgba(191,191,191,.87)}.ui.form :-ms-input-placeholder{color:rgba(191,191,191,.87)!important}.ui.form ::-moz-placeholder{color:rgba(191,191,191,.87)}.ui.form :focus::-webkit-input-placeholder{color:rgba(115,115,115,.87)}.ui.form :focus:-ms-input-placeholder{color:rgba(115,115,115,.87)!important}.ui.form :focus::-moz-placeholder{color:rgba(115,115,115,.87)}.ui.form .error ::-webkit-input-placeholder{color:#e7bdbc}.ui.form .error :-ms-input-placeholder{color:#e7bdbc!important}.ui.form .error ::-moz-placeholder{color:#e7bdbc}.ui.form .error :focus::-webkit-input-placeholder{color:#da9796}.ui.form .error :focus:-ms-input-placeholder{color:#da9796!important}.ui.form .error :focus::-moz-placeholder{color:#da9796}.ui.form input:not([type]):focus,.ui.form input[type=date]:focus,.ui.form input[type=datetime-local]:focus,.ui.form input[type=email]:focus,.ui.form input[type=file]:focus,.ui.form input[type=number]:focus,.ui.form input[type=password]:focus,.ui.form input[type=search]:focus,.ui.form input[type=tel]:focus,.ui.form input[type=text]:focus,.ui.form input[type=time]:focus,.ui.form input[type=url]:focus{color:rgba(0,0,0,.95);border-color:#85b7d9;border-radius:.28571429rem;background:#fff;-webkit-box-shadow:0 0 0 0 rgba(34,36,38,.35) inset;box-shadow:0 0 0 0 rgba(34,36,38,.35) inset}.ui.form textarea:focus{color:rgba(0,0,0,.95);border-color:#85b7d9;border-radius:.28571429rem;background:#fff;-webkit-box-shadow:0 0 0 0 rgba(34,36,38,.35) inset;box-shadow:0 0 0 0 rgba(34,36,38,.35) inset;-webkit-appearance:none}.ui.form.success .success.message:not(:empty){display:block}.ui.form.success .compact.success.message:not(:empty){display:inline-block}.ui.form.success .icon.success.message:not(:empty){display:-webkit-box;display:-ms-flexbox;display:flex}.ui.form.warning .warning.message:not(:empty){display:block}.ui.form.warning .compact.warning.message:not(:empty){display:inline-block}.ui.form.warning .icon.warning.message:not(:empty){display:-webkit-box;display:-ms-flexbox;display:flex}.ui.form.error .error.message:not(:empty){display:block}.ui.form.error .compact.error.message:not(:empty){display:inline-block}.ui.form.error .icon.error.message:not(:empty){display:-webkit-box;display:-ms-flexbox;display:flex}.ui.form .field.error .input,.ui.form .field.error label,.ui.form .fields.error .field .input,.ui.form .fields.error .field label{color:#9f3a38}.ui.form .field.error .corner.label,.ui.form .fields.error .field .corner.label{border-color:#9f3a38;color:#fff}.ui.form .field.error input:not([type]),.ui.form .field.error input[type=date],.ui.form .field.error input[type=datetime-local],.ui.form .field.error input[type=email],.ui.form .field.error input[type=file],.ui.form .field.error input[type=number],.ui.form .field.error input[type=password],.ui.form .field.error input[type=search],.ui.form .field.error input[type=tel],.ui.form .field.error input[type=text],.ui.form .field.error input[type=time],.ui.form .field.error input[type=url],.ui.form .field.error select,.ui.form .field.error textarea,.ui.form .fields.error .field input:not([type]),.ui.form .fields.error .field input[type=date],.ui.form .fields.error .field input[type=datetime-local],.ui.form .fields.error .field input[type=email],.ui.form .fields.error .field input[type=file],.ui.form .fields.error .field input[type=number],.ui.form .fields.error .field input[type=password],.ui.form .fields.error .field input[type=search],.ui.form .fields.error .field input[type=tel],.ui.form .fields.error .field input[type=text],.ui.form .fields.error .field input[type=time],.ui.form .fields.error .field input[type=url],.ui.form .fields.error .field select,.ui.form .fields.error .field textarea{background:#fff6f6;border-color:#e0b4b4;color:#9f3a38;border-radius:'';-webkit-box-shadow:none;box-shadow:none}.ui.form .field.error input:not([type]):focus,.ui.form .field.error input[type=date]:focus,.ui.form .field.error input[type=datetime-local]:focus,.ui.form .field.error input[type=email]:focus,.ui.form .field.error input[type=file]:focus,.ui.form .field.error input[type=number]:focus,.ui.form .field.error input[type=password]:focus,.ui.form .field.error input[type=search]:focus,.ui.form .field.error input[type=tel]:focus,.ui.form .field.error input[type=text]:focus,.ui.form .field.error input[type=time]:focus,.ui.form .field.error input[type=url]:focus,.ui.form .field.error select:focus,.ui.form .field.error textarea:focus{background:#fff6f6;border-color:#e0b4b4;color:#9f3a38;-webkit-appearance:none;-webkit-box-shadow:none;box-shadow:none}.ui.form .field.error select{-webkit-appearance:menulist-button}.ui.form .field.error .ui.dropdown,.ui.form .field.error .ui.dropdown .item,.ui.form .field.error .ui.dropdown .text,.ui.form .fields.error .field .ui.dropdown,.ui.form .fields.error .field .ui.dropdown .item{background:#fff6f6;color:#9f3a38}.ui.form .field.error .ui.dropdown,.ui.form .fields.error .field .ui.dropdown{border-color:#e0b4b4!important}.ui.form .field.error .ui.dropdown:hover,.ui.form .fields.error .field .ui.dropdown:hover{border-color:#e0b4b4!important}.ui.form .field.error .ui.dropdown:hover .menu,.ui.form .fields.error .field .ui.dropdown:hover .menu{border-color:#e0b4b4}.ui.form .field.error .ui.multiple.selection.dropdown>.label,.ui.form .fields.error .field .ui.multiple.selection.dropdown>.label{background-color:#eacbcb;color:#9f3a38}.ui.form .field.error .ui.dropdown .menu .item:hover,.ui.form .fields.error .field .ui.dropdown .menu .item:hover{background-color:#fbe7e7}.ui.form .field.error .ui.dropdown .menu .selected.item,.ui.form .fields.error .field .ui.dropdown .menu .selected.item{background-color:#fbe7e7}.ui.form .field.error .ui.dropdown .menu .active.item,.ui.form .fields.error .field .ui.dropdown .menu .active.item{background-color:#fdcfcf!important}.ui.form .field.error .checkbox:not(.toggle):not(.slider) .box,.ui.form .field.error .checkbox:not(.toggle):not(.slider) label,.ui.form .fields.error .field .checkbox:not(.toggle):not(.slider) .box,.ui.form .fields.error .field .checkbox:not(.toggle):not(.slider) label{color:#9f3a38}.ui.form .field.error .checkbox:not(.toggle):not(.slider) .box:before,.ui.form .field.error .checkbox:not(.toggle):not(.slider) label:before,.ui.form .fields.error .field .checkbox:not(.toggle):not(.slider) .box:before,.ui.form .fields.error .field .checkbox:not(.toggle):not(.slider) label:before{background:#fff6f6;border-color:#e0b4b4}.ui.form .field.error .checkbox .box:after,.ui.form .field.error .checkbox label:after,.ui.form .fields.error .field .checkbox .box:after,.ui.form .fields.error .field .checkbox label:after{color:#9f3a38}.ui.form .disabled.field,.ui.form .disabled.fields .field,.ui.form .field :disabled{pointer-events:none;opacity:.45}.ui.form .field.disabled>label,.ui.form .fields.disabled>label{opacity:.45}.ui.form .field.disabled :disabled{opacity:1}.ui.loading.form{position:relative;cursor:default;pointer-events:none}.ui.loading.form:before{position:absolute;content:'';top:0;left:0;background:rgba(255,255,255,.8);width:100%;height:100%;z-index:100}.ui.loading.form:after{position:absolute;content:'';top:50%;left:50%;margin:-1.5em 0 0 -1.5em;width:3em;height:3em;-webkit-animation:form-spin .6s linear;animation:form-spin .6s linear;-webkit-animation-iteration-count:infinite;animation-iteration-count:infinite;border-radius:500rem;border-color:#767676 rgba(0,0,0,.1) rgba(0,0,0,.1) rgba(0,0,0,.1);border-style:solid;border-width:.2em;-webkit-box-shadow:0 0 0 1px transparent;box-shadow:0 0 0 1px transparent;visibility:visible;z-index:101}@-webkit-keyframes form-spin{from{-webkit-transform:rotate(0);transform:rotate(0)}to{-webkit-transform:rotate(360deg);transform:rotate(360deg)}}@keyframes form-spin{from{-webkit-transform:rotate(0);transform:rotate(0)}to{-webkit-transform:rotate(360deg);transform:rotate(360deg)}}.ui.form .required.field>.checkbox:after,.ui.form .required.field>label:after,.ui.form .required.fields.grouped>label:after,.ui.form .required.fields:not(.grouped)>.field>.checkbox:after,.ui.form .required.fields:not(.grouped)>.field>label:after{margin:-.2em 0 0 .2em;content:'*';color:#db2828}.ui.form .required.field>label:after,.ui.form .required.fields.grouped>label:after,.ui.form .required.fields:not(.grouped)>.field>label:after{display:inline-block;vertical-align:top}.ui.form .required.field>.checkbox:after,.ui.form .required.fields:not(.grouped)>.field>.checkbox:after{position:absolute;top:0;left:100%}.ui.form .inverted.segment .ui.checkbox .box,.ui.form .inverted.segment .ui.checkbox label,.ui.form .inverted.segment label,.ui.inverted.form .inline.field>label,.ui.inverted.form .inline.field>p,.ui.inverted.form .inline.fields .field>label,.ui.inverted.form .inline.fields .field>p,.ui.inverted.form .inline.fields>label,.ui.inverted.form .ui.checkbox .box,.ui.inverted.form .ui.checkbox label,.ui.inverted.form label{color:rgba(255,255,255,.9)}.ui.inverted.form input:not([type]),.ui.inverted.form input[type=date],.ui.inverted.form input[type=datetime-local],.ui.inverted.form input[type=email],.ui.inverted.form input[type=file],.ui.inverted.form input[type=number],.ui.inverted.form input[type=password],.ui.inverted.form input[type=search],.ui.inverted.form input[type=tel],.ui.inverted.form input[type=text],.ui.inverted.form input[type=time],.ui.inverted.form input[type=url]{background:#fff;border-color:rgba(255,255,255,.1);color:rgba(0,0,0,.87);-webkit-box-shadow:none;box-shadow:none}.ui.form .grouped.fields{display:block;margin:0 0 1em}.ui.form .grouped.fields:last-child{margin-bottom:0}.ui.form .grouped.fields>label{margin:0 0 .28571429rem 0;color:rgba(0,0,0,.87);font-size:.92857143em;font-weight:700;text-transform:none}.ui.form .grouped.fields .field,.ui.form .grouped.inline.fields .field{display:block;margin:.5em 0;padding:0}.ui.form .fields{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-orient:horizontal;-webkit-box-direction:normal;-ms-flex-direction:row;flex-direction:row;margin:0 -.5em 1em}.ui.form .fields>.field{-webkit-box-flex:0;-ms-flex:0 1 auto;flex:0 1 auto;padding-left:.5em;padding-right:.5em}.ui.form .fields>.field:first-child{border-left:none;-webkit-box-shadow:none;box-shadow:none}.ui.form .two.fields>.field,.ui.form .two.fields>.fields{width:50%}.ui.form .three.fields>.field,.ui.form .three.fields>.fields{width:33.33333333%}.ui.form .four.fields>.field,.ui.form .four.fields>.fields{width:25%}.ui.form .five.fields>.field,.ui.form .five.fields>.fields{width:20%}.ui.form .six.fields>.field,.ui.form .six.fields>.fields{width:16.66666667%}.ui.form .seven.fields>.field,.ui.form .seven.fields>.fields{width:14.28571429%}.ui.form .eight.fields>.field,.ui.form .eight.fields>.fields{width:12.5%}.ui.form .nine.fields>.field,.ui.form .nine.fields>.fields{width:11.11111111%}.ui.form .ten.fields>.field,.ui.form .ten.fields>.fields{width:10%}@media only screen and (max-width:767px){.ui.form .fields{-ms-flex-wrap:wrap;flex-wrap:wrap}.ui.form:not(.unstackable) .eight.fields:not(.unstackable)>.field,.ui.form:not(.unstackable) .eight.fields:not(.unstackable)>.fields,.ui.form:not(.unstackable) .five.fields:not(.unstackable)>.field,.ui.form:not(.unstackable) .five.fields:not(.unstackable)>.fields,.ui.form:not(.unstackable) .four.fields:not(.unstackable)>.field,.ui.form:not(.unstackable) .four.fields:not(.unstackable)>.fields,.ui.form:not(.unstackable) .nine.fields:not(.unstackable)>.field,.ui.form:not(.unstackable) .nine.fields:not(.unstackable)>.fields,.ui.form:not(.unstackable) .seven.fields:not(.unstackable)>.field,.ui.form:not(.unstackable) .seven.fields:not(.unstackable)>.fields,.ui.form:not(.unstackable) .six.fields:not(.unstackable)>.field,.ui.form:not(.unstackable) .six.fields:not(.unstackable)>.fields,.ui.form:not(.unstackable) .ten.fields:not(.unstackable)>.field,.ui.form:not(.unstackable) .ten.fields:not(.unstackable)>.fields,.ui.form:not(.unstackable) .three.fields:not(.unstackable)>.field,.ui.form:not(.unstackable) .three.fields:not(.unstackable)>.fields,.ui.form:not(.unstackable) .two.fields:not(.unstackable)>.field,.ui.form:not(.unstackable) .two.fields:not(.unstackable)>.fields,.ui.form:not(.unstackable) [class*="equal width"].fields:not(.unstackable)>.field,.ui[class*="equal width"].form:not(.unstackable) .fields>.field{width:100%!important;margin:0 0 1em}}.ui.form .fields .wide.field{width:6.25%;padding-left:.5em;padding-right:.5em}.ui.form .one.wide.field{width:6.25%!important}.ui.form .two.wide.field{width:12.5%!important}.ui.form .three.wide.field{width:18.75%!important}.ui.form .four.wide.field{width:25%!important}.ui.form .five.wide.field{width:31.25%!important}.ui.form .six.wide.field{width:37.5%!important}.ui.form .seven.wide.field{width:43.75%!important}.ui.form .eight.wide.field{width:50%!important}.ui.form .nine.wide.field{width:56.25%!important}.ui.form .ten.wide.field{width:62.5%!important}.ui.form .eleven.wide.field{width:68.75%!important}.ui.form .twelve.wide.field{width:75%!important}.ui.form .thirteen.wide.field{width:81.25%!important}.ui.form .fourteen.wide.field{width:87.5%!important}.ui.form .fifteen.wide.field{width:93.75%!important}.ui.form .sixteen.wide.field{width:100%!important}@media only screen and (max-width:767px){.ui.form:not(.unstackable) .fields:not(.unstackable)>.eight.wide.field,.ui.form:not(.unstackable) .fields:not(.unstackable)>.eleven.wide.field,.ui.form:not(.unstackable) .fields:not(.unstackable)>.fifteen.wide.field,.ui.form:not(.unstackable) .fields:not(.unstackable)>.five.wide.field,.ui.form:not(.unstackable) .fields:not(.unstackable)>.four.wide.field,.ui.form:not(.unstackable) .fields:not(.unstackable)>.fourteen.wide.field,.ui.form:not(.unstackable) .fields:not(.unstackable)>.nine.wide.field,.ui.form:not(.unstackable) .fields:not(.unstackable)>.seven.wide.field,.ui.form:not(.unstackable) .fields:not(.unstackable)>.six.wide.field,.ui.form:not(.unstackable) .fields:not(.unstackable)>.sixteen.wide.field,.ui.form:not(.unstackable) .fields:not(.unstackable)>.ten.wide.field,.ui.form:not(.unstackable) .fields:not(.unstackable)>.thirteen.wide.field,.ui.form:not(.unstackable) .fields:not(.unstackable)>.three.wide.field,.ui.form:not(.unstackable) .fields:not(.unstackable)>.twelve.wide.field,.ui.form:not(.unstackable) .fields:not(.unstackable)>.two.wide.field,.ui.form:not(.unstackable) .five.fields:not(.unstackable)>.field,.ui.form:not(.unstackable) .five.fields:not(.unstackable)>.fields,.ui.form:not(.unstackable) .four.fields:not(.unstackable)>.field,.ui.form:not(.unstackable) .four.fields:not(.unstackable)>.fields,.ui.form:not(.unstackable) .three.fields:not(.unstackable)>.field,.ui.form:not(.unstackable) .three.fields:not(.unstackable)>.fields,.ui.form:not(.unstackable) .two.fields:not(.unstackable)>.field,.ui.form:not(.unstackable) .two.fields:not(.unstackable)>.fields{width:100%!important}.ui.form .fields{margin-bottom:0}}.ui.form [class*="equal width"].fields>.field,.ui[class*="equal width"].form .fields>.field{width:100%;-webkit-box-flex:1;-ms-flex:1 1 auto;flex:1 1 auto}.ui.form .inline.fields{margin:0 0 1em;-webkit-box-align:center;-ms-flex-align:center;align-items:center}.ui.form .inline.fields .field{margin:0;padding:0 1em 0 0}.ui.form .inline.field>label,.ui.form .inline.field>p,.ui.form .inline.fields .field>label,.ui.form .inline.fields .field>p,.ui.form .inline.fields>label{display:inline-block;width:auto;margin-top:0;margin-bottom:0;vertical-align:baseline;font-size:.92857143em;font-weight:700;color:rgba(0,0,0,.87);text-transform:none}.ui.form .inline.fields>label{margin:.035714em 1em 0 0}.ui.form .inline.field>input,.ui.form .inline.field>select,.ui.form .inline.fields .field>input,.ui.form .inline.fields .field>select{display:inline-block;width:auto;margin-top:0;margin-bottom:0;vertical-align:middle;font-size:1em}.ui.form .inline.field>:first-child,.ui.form .inline.fields .field>:first-child{margin:0 .85714286em 0 0}.ui.form .inline.field>:only-child,.ui.form .inline.fields .field>:only-child{margin:0}.ui.form .inline.fields .wide.field{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-align:center;-ms-flex-align:center;align-items:center}.ui.form .inline.fields .wide.field>input,.ui.form .inline.fields .wide.field>select{width:100%}.ui.mini.form{font-size:.78571429rem}.ui.tiny.form{font-size:.85714286rem}.ui.small.form{font-size:.92857143rem}.ui.form{font-size:1rem}.ui.large.form{font-size:1.14285714rem}.ui.big.form{font-size:1.28571429rem}.ui.huge.form{font-size:1.42857143rem}.ui.massive.form{font-size:1.71428571rem}/*! - * # Semantic UI 2.4.0 - Grid - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */.ui.grid{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-orient:horizontal;-webkit-box-direction:normal;-ms-flex-direction:row;flex-direction:row;-ms-flex-wrap:wrap;flex-wrap:wrap;-webkit-box-align:stretch;-ms-flex-align:stretch;align-items:stretch;padding:0}.ui.grid{margin-top:-1rem;margin-bottom:-1rem;margin-left:-1rem;margin-right:-1rem}.ui.relaxed.grid{margin-left:-1.5rem;margin-right:-1.5rem}.ui[class*="very relaxed"].grid{margin-left:-2.5rem;margin-right:-2.5rem}.ui.grid+.grid{margin-top:1rem}.ui.grid>.column:not(.row),.ui.grid>.row>.column{position:relative;display:inline-block;width:6.25%;padding-left:1rem;padding-right:1rem;vertical-align:top}.ui.grid>*{padding-left:1rem;padding-right:1rem}.ui.grid>.row{position:relative;display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-orient:horizontal;-webkit-box-direction:normal;-ms-flex-direction:row;flex-direction:row;-ms-flex-wrap:wrap;flex-wrap:wrap;-webkit-box-pack:inherit;-ms-flex-pack:inherit;justify-content:inherit;-webkit-box-align:stretch;-ms-flex-align:stretch;align-items:stretch;width:100%!important;padding:0;padding-top:1rem;padding-bottom:1rem}.ui.grid>.column:not(.row){padding-top:1rem;padding-bottom:1rem}.ui.grid>.row>.column{margin-top:0;margin-bottom:0}.ui.grid>.row>.column>img,.ui.grid>.row>img{max-width:100%}.ui.grid>.ui.grid:first-child{margin-top:0}.ui.grid>.ui.grid:last-child{margin-bottom:0}.ui.aligned.grid .column>.segment:not(.compact):not(.attached),.ui.grid .aligned.row>.column>.segment:not(.compact):not(.attached){width:100%}.ui.grid .row+.ui.divider{-webkit-box-flex:1;-ms-flex-positive:1;flex-grow:1;margin:1rem 1rem}.ui.grid .column+.ui.vertical.divider{height:calc(50% - 1rem)}.ui.grid>.column:last-child>.horizontal.segment,.ui.grid>.row>.column:last-child>.horizontal.segment{-webkit-box-shadow:none;box-shadow:none}@media only screen and (max-width:767px){.ui.page.grid{width:auto;padding-left:0;padding-right:0;margin-left:0;margin-right:0}}@media only screen and (min-width:768px) and (max-width:991px){.ui.page.grid{width:auto;margin-left:0;margin-right:0;padding-left:2em;padding-right:2em}}@media only screen and (min-width:992px) and (max-width:1199px){.ui.page.grid{width:auto;margin-left:0;margin-right:0;padding-left:3%;padding-right:3%}}@media only screen and (min-width:1200px) and (max-width:1919px){.ui.page.grid{width:auto;margin-left:0;margin-right:0;padding-left:15%;padding-right:15%}}@media only screen and (min-width:1920px){.ui.page.grid{width:auto;margin-left:0;margin-right:0;padding-left:23%;padding-right:23%}}.ui.grid>.column:only-child,.ui.grid>.row>.column:only-child{width:100%}.ui[class*="one column"].grid>.column:not(.row),.ui[class*="one column"].grid>.row>.column{width:100%}.ui[class*="two column"].grid>.column:not(.row),.ui[class*="two column"].grid>.row>.column{width:50%}.ui[class*="three column"].grid>.column:not(.row),.ui[class*="three column"].grid>.row>.column{width:33.33333333%}.ui[class*="four column"].grid>.column:not(.row),.ui[class*="four column"].grid>.row>.column{width:25%}.ui[class*="five column"].grid>.column:not(.row),.ui[class*="five column"].grid>.row>.column{width:20%}.ui[class*="six column"].grid>.column:not(.row),.ui[class*="six column"].grid>.row>.column{width:16.66666667%}.ui[class*="seven column"].grid>.column:not(.row),.ui[class*="seven column"].grid>.row>.column{width:14.28571429%}.ui[class*="eight column"].grid>.column:not(.row),.ui[class*="eight column"].grid>.row>.column{width:12.5%}.ui[class*="nine column"].grid>.column:not(.row),.ui[class*="nine column"].grid>.row>.column{width:11.11111111%}.ui[class*="ten column"].grid>.column:not(.row),.ui[class*="ten column"].grid>.row>.column{width:10%}.ui[class*="eleven column"].grid>.column:not(.row),.ui[class*="eleven column"].grid>.row>.column{width:9.09090909%}.ui[class*="twelve column"].grid>.column:not(.row),.ui[class*="twelve column"].grid>.row>.column{width:8.33333333%}.ui[class*="thirteen column"].grid>.column:not(.row),.ui[class*="thirteen column"].grid>.row>.column{width:7.69230769%}.ui[class*="fourteen column"].grid>.column:not(.row),.ui[class*="fourteen column"].grid>.row>.column{width:7.14285714%}.ui[class*="fifteen column"].grid>.column:not(.row),.ui[class*="fifteen column"].grid>.row>.column{width:6.66666667%}.ui[class*="sixteen column"].grid>.column:not(.row),.ui[class*="sixteen column"].grid>.row>.column{width:6.25%}.ui.grid>[class*="one column"].row>.column{width:100%!important}.ui.grid>[class*="two column"].row>.column{width:50%!important}.ui.grid>[class*="three column"].row>.column{width:33.33333333%!important}.ui.grid>[class*="four column"].row>.column{width:25%!important}.ui.grid>[class*="five column"].row>.column{width:20%!important}.ui.grid>[class*="six column"].row>.column{width:16.66666667%!important}.ui.grid>[class*="seven column"].row>.column{width:14.28571429%!important}.ui.grid>[class*="eight column"].row>.column{width:12.5%!important}.ui.grid>[class*="nine column"].row>.column{width:11.11111111%!important}.ui.grid>[class*="ten column"].row>.column{width:10%!important}.ui.grid>[class*="eleven column"].row>.column{width:9.09090909%!important}.ui.grid>[class*="twelve column"].row>.column{width:8.33333333%!important}.ui.grid>[class*="thirteen column"].row>.column{width:7.69230769%!important}.ui.grid>[class*="fourteen column"].row>.column{width:7.14285714%!important}.ui.grid>[class*="fifteen column"].row>.column{width:6.66666667%!important}.ui.grid>[class*="sixteen column"].row>.column{width:6.25%!important}.ui.celled.page.grid{-webkit-box-shadow:none;box-shadow:none}.ui.column.grid>[class*="one wide"].column,.ui.grid>.column.row>[class*="one wide"].column,.ui.grid>.row>[class*="one wide"].column,.ui.grid>[class*="one wide"].column{width:6.25%!important}.ui.column.grid>[class*="two wide"].column,.ui.grid>.column.row>[class*="two wide"].column,.ui.grid>.row>[class*="two wide"].column,.ui.grid>[class*="two wide"].column{width:12.5%!important}.ui.column.grid>[class*="three wide"].column,.ui.grid>.column.row>[class*="three wide"].column,.ui.grid>.row>[class*="three wide"].column,.ui.grid>[class*="three wide"].column{width:18.75%!important}.ui.column.grid>[class*="four wide"].column,.ui.grid>.column.row>[class*="four wide"].column,.ui.grid>.row>[class*="four wide"].column,.ui.grid>[class*="four wide"].column{width:25%!important}.ui.column.grid>[class*="five wide"].column,.ui.grid>.column.row>[class*="five wide"].column,.ui.grid>.row>[class*="five wide"].column,.ui.grid>[class*="five wide"].column{width:31.25%!important}.ui.column.grid>[class*="six wide"].column,.ui.grid>.column.row>[class*="six wide"].column,.ui.grid>.row>[class*="six wide"].column,.ui.grid>[class*="six wide"].column{width:37.5%!important}.ui.column.grid>[class*="seven wide"].column,.ui.grid>.column.row>[class*="seven wide"].column,.ui.grid>.row>[class*="seven wide"].column,.ui.grid>[class*="seven wide"].column{width:43.75%!important}.ui.column.grid>[class*="eight wide"].column,.ui.grid>.column.row>[class*="eight wide"].column,.ui.grid>.row>[class*="eight wide"].column,.ui.grid>[class*="eight wide"].column{width:50%!important}.ui.column.grid>[class*="nine wide"].column,.ui.grid>.column.row>[class*="nine wide"].column,.ui.grid>.row>[class*="nine wide"].column,.ui.grid>[class*="nine wide"].column{width:56.25%!important}.ui.column.grid>[class*="ten wide"].column,.ui.grid>.column.row>[class*="ten wide"].column,.ui.grid>.row>[class*="ten wide"].column,.ui.grid>[class*="ten wide"].column{width:62.5%!important}.ui.column.grid>[class*="eleven wide"].column,.ui.grid>.column.row>[class*="eleven wide"].column,.ui.grid>.row>[class*="eleven wide"].column,.ui.grid>[class*="eleven wide"].column{width:68.75%!important}.ui.column.grid>[class*="twelve wide"].column,.ui.grid>.column.row>[class*="twelve wide"].column,.ui.grid>.row>[class*="twelve wide"].column,.ui.grid>[class*="twelve wide"].column{width:75%!important}.ui.column.grid>[class*="thirteen wide"].column,.ui.grid>.column.row>[class*="thirteen wide"].column,.ui.grid>.row>[class*="thirteen wide"].column,.ui.grid>[class*="thirteen wide"].column{width:81.25%!important}.ui.column.grid>[class*="fourteen wide"].column,.ui.grid>.column.row>[class*="fourteen wide"].column,.ui.grid>.row>[class*="fourteen wide"].column,.ui.grid>[class*="fourteen wide"].column{width:87.5%!important}.ui.column.grid>[class*="fifteen wide"].column,.ui.grid>.column.row>[class*="fifteen wide"].column,.ui.grid>.row>[class*="fifteen wide"].column,.ui.grid>[class*="fifteen wide"].column{width:93.75%!important}.ui.column.grid>[class*="sixteen wide"].column,.ui.grid>.column.row>[class*="sixteen wide"].column,.ui.grid>.row>[class*="sixteen wide"].column,.ui.grid>[class*="sixteen wide"].column{width:100%!important}@media only screen and (min-width:320px) and (max-width:767px){.ui.column.grid>[class*="one wide mobile"].column,.ui.grid>.column.row>[class*="one wide mobile"].column,.ui.grid>.row>[class*="one wide mobile"].column,.ui.grid>[class*="one wide mobile"].column{width:6.25%!important}.ui.column.grid>[class*="two wide mobile"].column,.ui.grid>.column.row>[class*="two wide mobile"].column,.ui.grid>.row>[class*="two wide mobile"].column,.ui.grid>[class*="two wide mobile"].column{width:12.5%!important}.ui.column.grid>[class*="three wide mobile"].column,.ui.grid>.column.row>[class*="three wide mobile"].column,.ui.grid>.row>[class*="three wide mobile"].column,.ui.grid>[class*="three wide mobile"].column{width:18.75%!important}.ui.column.grid>[class*="four wide mobile"].column,.ui.grid>.column.row>[class*="four wide mobile"].column,.ui.grid>.row>[class*="four wide mobile"].column,.ui.grid>[class*="four wide mobile"].column{width:25%!important}.ui.column.grid>[class*="five wide mobile"].column,.ui.grid>.column.row>[class*="five wide mobile"].column,.ui.grid>.row>[class*="five wide mobile"].column,.ui.grid>[class*="five wide mobile"].column{width:31.25%!important}.ui.column.grid>[class*="six wide mobile"].column,.ui.grid>.column.row>[class*="six wide mobile"].column,.ui.grid>.row>[class*="six wide mobile"].column,.ui.grid>[class*="six wide mobile"].column{width:37.5%!important}.ui.column.grid>[class*="seven wide mobile"].column,.ui.grid>.column.row>[class*="seven wide mobile"].column,.ui.grid>.row>[class*="seven wide mobile"].column,.ui.grid>[class*="seven wide mobile"].column{width:43.75%!important}.ui.column.grid>[class*="eight wide mobile"].column,.ui.grid>.column.row>[class*="eight wide mobile"].column,.ui.grid>.row>[class*="eight wide mobile"].column,.ui.grid>[class*="eight wide mobile"].column{width:50%!important}.ui.column.grid>[class*="nine wide mobile"].column,.ui.grid>.column.row>[class*="nine wide mobile"].column,.ui.grid>.row>[class*="nine wide mobile"].column,.ui.grid>[class*="nine wide mobile"].column{width:56.25%!important}.ui.column.grid>[class*="ten wide mobile"].column,.ui.grid>.column.row>[class*="ten wide mobile"].column,.ui.grid>.row>[class*="ten wide mobile"].column,.ui.grid>[class*="ten wide mobile"].column{width:62.5%!important}.ui.column.grid>[class*="eleven wide mobile"].column,.ui.grid>.column.row>[class*="eleven wide mobile"].column,.ui.grid>.row>[class*="eleven wide mobile"].column,.ui.grid>[class*="eleven wide mobile"].column{width:68.75%!important}.ui.column.grid>[class*="twelve wide mobile"].column,.ui.grid>.column.row>[class*="twelve wide mobile"].column,.ui.grid>.row>[class*="twelve wide mobile"].column,.ui.grid>[class*="twelve wide mobile"].column{width:75%!important}.ui.column.grid>[class*="thirteen wide mobile"].column,.ui.grid>.column.row>[class*="thirteen wide mobile"].column,.ui.grid>.row>[class*="thirteen wide mobile"].column,.ui.grid>[class*="thirteen wide mobile"].column{width:81.25%!important}.ui.column.grid>[class*="fourteen wide mobile"].column,.ui.grid>.column.row>[class*="fourteen wide mobile"].column,.ui.grid>.row>[class*="fourteen wide mobile"].column,.ui.grid>[class*="fourteen wide mobile"].column{width:87.5%!important}.ui.column.grid>[class*="fifteen wide mobile"].column,.ui.grid>.column.row>[class*="fifteen wide mobile"].column,.ui.grid>.row>[class*="fifteen wide mobile"].column,.ui.grid>[class*="fifteen wide mobile"].column{width:93.75%!important}.ui.column.grid>[class*="sixteen wide mobile"].column,.ui.grid>.column.row>[class*="sixteen wide mobile"].column,.ui.grid>.row>[class*="sixteen wide mobile"].column,.ui.grid>[class*="sixteen wide mobile"].column{width:100%!important}}@media only screen and (min-width:768px) and (max-width:991px){.ui.column.grid>[class*="one wide tablet"].column,.ui.grid>.column.row>[class*="one wide tablet"].column,.ui.grid>.row>[class*="one wide tablet"].column,.ui.grid>[class*="one wide tablet"].column{width:6.25%!important}.ui.column.grid>[class*="two wide tablet"].column,.ui.grid>.column.row>[class*="two wide tablet"].column,.ui.grid>.row>[class*="two wide tablet"].column,.ui.grid>[class*="two wide tablet"].column{width:12.5%!important}.ui.column.grid>[class*="three wide tablet"].column,.ui.grid>.column.row>[class*="three wide tablet"].column,.ui.grid>.row>[class*="three wide tablet"].column,.ui.grid>[class*="three wide tablet"].column{width:18.75%!important}.ui.column.grid>[class*="four wide tablet"].column,.ui.grid>.column.row>[class*="four wide tablet"].column,.ui.grid>.row>[class*="four wide tablet"].column,.ui.grid>[class*="four wide tablet"].column{width:25%!important}.ui.column.grid>[class*="five wide tablet"].column,.ui.grid>.column.row>[class*="five wide tablet"].column,.ui.grid>.row>[class*="five wide tablet"].column,.ui.grid>[class*="five wide tablet"].column{width:31.25%!important}.ui.column.grid>[class*="six wide tablet"].column,.ui.grid>.column.row>[class*="six wide tablet"].column,.ui.grid>.row>[class*="six wide tablet"].column,.ui.grid>[class*="six wide tablet"].column{width:37.5%!important}.ui.column.grid>[class*="seven wide tablet"].column,.ui.grid>.column.row>[class*="seven wide tablet"].column,.ui.grid>.row>[class*="seven wide tablet"].column,.ui.grid>[class*="seven wide tablet"].column{width:43.75%!important}.ui.column.grid>[class*="eight wide tablet"].column,.ui.grid>.column.row>[class*="eight wide tablet"].column,.ui.grid>.row>[class*="eight wide tablet"].column,.ui.grid>[class*="eight wide tablet"].column{width:50%!important}.ui.column.grid>[class*="nine wide tablet"].column,.ui.grid>.column.row>[class*="nine wide tablet"].column,.ui.grid>.row>[class*="nine wide tablet"].column,.ui.grid>[class*="nine wide tablet"].column{width:56.25%!important}.ui.column.grid>[class*="ten wide tablet"].column,.ui.grid>.column.row>[class*="ten wide tablet"].column,.ui.grid>.row>[class*="ten wide tablet"].column,.ui.grid>[class*="ten wide tablet"].column{width:62.5%!important}.ui.column.grid>[class*="eleven wide tablet"].column,.ui.grid>.column.row>[class*="eleven wide tablet"].column,.ui.grid>.row>[class*="eleven wide tablet"].column,.ui.grid>[class*="eleven wide tablet"].column{width:68.75%!important}.ui.column.grid>[class*="twelve wide tablet"].column,.ui.grid>.column.row>[class*="twelve wide tablet"].column,.ui.grid>.row>[class*="twelve wide tablet"].column,.ui.grid>[class*="twelve wide tablet"].column{width:75%!important}.ui.column.grid>[class*="thirteen wide tablet"].column,.ui.grid>.column.row>[class*="thirteen wide tablet"].column,.ui.grid>.row>[class*="thirteen wide tablet"].column,.ui.grid>[class*="thirteen wide tablet"].column{width:81.25%!important}.ui.column.grid>[class*="fourteen wide tablet"].column,.ui.grid>.column.row>[class*="fourteen wide tablet"].column,.ui.grid>.row>[class*="fourteen wide tablet"].column,.ui.grid>[class*="fourteen wide tablet"].column{width:87.5%!important}.ui.column.grid>[class*="fifteen wide tablet"].column,.ui.grid>.column.row>[class*="fifteen wide tablet"].column,.ui.grid>.row>[class*="fifteen wide tablet"].column,.ui.grid>[class*="fifteen wide tablet"].column{width:93.75%!important}.ui.column.grid>[class*="sixteen wide tablet"].column,.ui.grid>.column.row>[class*="sixteen wide tablet"].column,.ui.grid>.row>[class*="sixteen wide tablet"].column,.ui.grid>[class*="sixteen wide tablet"].column{width:100%!important}}@media only screen and (min-width:992px){.ui.column.grid>[class*="one wide computer"].column,.ui.grid>.column.row>[class*="one wide computer"].column,.ui.grid>.row>[class*="one wide computer"].column,.ui.grid>[class*="one wide computer"].column{width:6.25%!important}.ui.column.grid>[class*="two wide computer"].column,.ui.grid>.column.row>[class*="two wide computer"].column,.ui.grid>.row>[class*="two wide computer"].column,.ui.grid>[class*="two wide computer"].column{width:12.5%!important}.ui.column.grid>[class*="three wide computer"].column,.ui.grid>.column.row>[class*="three wide computer"].column,.ui.grid>.row>[class*="three wide computer"].column,.ui.grid>[class*="three wide computer"].column{width:18.75%!important}.ui.column.grid>[class*="four wide computer"].column,.ui.grid>.column.row>[class*="four wide computer"].column,.ui.grid>.row>[class*="four wide computer"].column,.ui.grid>[class*="four wide computer"].column{width:25%!important}.ui.column.grid>[class*="five wide computer"].column,.ui.grid>.column.row>[class*="five wide computer"].column,.ui.grid>.row>[class*="five wide computer"].column,.ui.grid>[class*="five wide computer"].column{width:31.25%!important}.ui.column.grid>[class*="six wide computer"].column,.ui.grid>.column.row>[class*="six wide computer"].column,.ui.grid>.row>[class*="six wide computer"].column,.ui.grid>[class*="six wide computer"].column{width:37.5%!important}.ui.column.grid>[class*="seven wide computer"].column,.ui.grid>.column.row>[class*="seven wide computer"].column,.ui.grid>.row>[class*="seven wide computer"].column,.ui.grid>[class*="seven wide computer"].column{width:43.75%!important}.ui.column.grid>[class*="eight wide computer"].column,.ui.grid>.column.row>[class*="eight wide computer"].column,.ui.grid>.row>[class*="eight wide computer"].column,.ui.grid>[class*="eight wide computer"].column{width:50%!important}.ui.column.grid>[class*="nine wide computer"].column,.ui.grid>.column.row>[class*="nine wide computer"].column,.ui.grid>.row>[class*="nine wide computer"].column,.ui.grid>[class*="nine wide computer"].column{width:56.25%!important}.ui.column.grid>[class*="ten wide computer"].column,.ui.grid>.column.row>[class*="ten wide computer"].column,.ui.grid>.row>[class*="ten wide computer"].column,.ui.grid>[class*="ten wide computer"].column{width:62.5%!important}.ui.column.grid>[class*="eleven wide computer"].column,.ui.grid>.column.row>[class*="eleven wide computer"].column,.ui.grid>.row>[class*="eleven wide computer"].column,.ui.grid>[class*="eleven wide computer"].column{width:68.75%!important}.ui.column.grid>[class*="twelve wide computer"].column,.ui.grid>.column.row>[class*="twelve wide computer"].column,.ui.grid>.row>[class*="twelve wide computer"].column,.ui.grid>[class*="twelve wide computer"].column{width:75%!important}.ui.column.grid>[class*="thirteen wide computer"].column,.ui.grid>.column.row>[class*="thirteen wide computer"].column,.ui.grid>.row>[class*="thirteen wide computer"].column,.ui.grid>[class*="thirteen wide computer"].column{width:81.25%!important}.ui.column.grid>[class*="fourteen wide computer"].column,.ui.grid>.column.row>[class*="fourteen wide computer"].column,.ui.grid>.row>[class*="fourteen wide computer"].column,.ui.grid>[class*="fourteen wide computer"].column{width:87.5%!important}.ui.column.grid>[class*="fifteen wide computer"].column,.ui.grid>.column.row>[class*="fifteen wide computer"].column,.ui.grid>.row>[class*="fifteen wide computer"].column,.ui.grid>[class*="fifteen wide computer"].column{width:93.75%!important}.ui.column.grid>[class*="sixteen wide computer"].column,.ui.grid>.column.row>[class*="sixteen wide computer"].column,.ui.grid>.row>[class*="sixteen wide computer"].column,.ui.grid>[class*="sixteen wide computer"].column{width:100%!important}}@media only screen and (min-width:1200px) and (max-width:1919px){.ui.column.grid>[class*="one wide large screen"].column,.ui.grid>.column.row>[class*="one wide large screen"].column,.ui.grid>.row>[class*="one wide large screen"].column,.ui.grid>[class*="one wide large screen"].column{width:6.25%!important}.ui.column.grid>[class*="two wide large screen"].column,.ui.grid>.column.row>[class*="two wide large screen"].column,.ui.grid>.row>[class*="two wide large screen"].column,.ui.grid>[class*="two wide large screen"].column{width:12.5%!important}.ui.column.grid>[class*="three wide large screen"].column,.ui.grid>.column.row>[class*="three wide large screen"].column,.ui.grid>.row>[class*="three wide large screen"].column,.ui.grid>[class*="three wide large screen"].column{width:18.75%!important}.ui.column.grid>[class*="four wide large screen"].column,.ui.grid>.column.row>[class*="four wide large screen"].column,.ui.grid>.row>[class*="four wide large screen"].column,.ui.grid>[class*="four wide large screen"].column{width:25%!important}.ui.column.grid>[class*="five wide large screen"].column,.ui.grid>.column.row>[class*="five wide large screen"].column,.ui.grid>.row>[class*="five wide large screen"].column,.ui.grid>[class*="five wide large screen"].column{width:31.25%!important}.ui.column.grid>[class*="six wide large screen"].column,.ui.grid>.column.row>[class*="six wide large screen"].column,.ui.grid>.row>[class*="six wide large screen"].column,.ui.grid>[class*="six wide large screen"].column{width:37.5%!important}.ui.column.grid>[class*="seven wide large screen"].column,.ui.grid>.column.row>[class*="seven wide large screen"].column,.ui.grid>.row>[class*="seven wide large screen"].column,.ui.grid>[class*="seven wide large screen"].column{width:43.75%!important}.ui.column.grid>[class*="eight wide large screen"].column,.ui.grid>.column.row>[class*="eight wide large screen"].column,.ui.grid>.row>[class*="eight wide large screen"].column,.ui.grid>[class*="eight wide large screen"].column{width:50%!important}.ui.column.grid>[class*="nine wide large screen"].column,.ui.grid>.column.row>[class*="nine wide large screen"].column,.ui.grid>.row>[class*="nine wide large screen"].column,.ui.grid>[class*="nine wide large screen"].column{width:56.25%!important}.ui.column.grid>[class*="ten wide large screen"].column,.ui.grid>.column.row>[class*="ten wide large screen"].column,.ui.grid>.row>[class*="ten wide large screen"].column,.ui.grid>[class*="ten wide large screen"].column{width:62.5%!important}.ui.column.grid>[class*="eleven wide large screen"].column,.ui.grid>.column.row>[class*="eleven wide large screen"].column,.ui.grid>.row>[class*="eleven wide large screen"].column,.ui.grid>[class*="eleven wide large screen"].column{width:68.75%!important}.ui.column.grid>[class*="twelve wide large screen"].column,.ui.grid>.column.row>[class*="twelve wide large screen"].column,.ui.grid>.row>[class*="twelve wide large screen"].column,.ui.grid>[class*="twelve wide large screen"].column{width:75%!important}.ui.column.grid>[class*="thirteen wide large screen"].column,.ui.grid>.column.row>[class*="thirteen wide large screen"].column,.ui.grid>.row>[class*="thirteen wide large screen"].column,.ui.grid>[class*="thirteen wide large screen"].column{width:81.25%!important}.ui.column.grid>[class*="fourteen wide large screen"].column,.ui.grid>.column.row>[class*="fourteen wide large screen"].column,.ui.grid>.row>[class*="fourteen wide large screen"].column,.ui.grid>[class*="fourteen wide large screen"].column{width:87.5%!important}.ui.column.grid>[class*="fifteen wide large screen"].column,.ui.grid>.column.row>[class*="fifteen wide large screen"].column,.ui.grid>.row>[class*="fifteen wide large screen"].column,.ui.grid>[class*="fifteen wide large screen"].column{width:93.75%!important}.ui.column.grid>[class*="sixteen wide large screen"].column,.ui.grid>.column.row>[class*="sixteen wide large screen"].column,.ui.grid>.row>[class*="sixteen wide large screen"].column,.ui.grid>[class*="sixteen wide large screen"].column{width:100%!important}}@media only screen and (min-width:1920px){.ui.column.grid>[class*="one wide widescreen"].column,.ui.grid>.column.row>[class*="one wide widescreen"].column,.ui.grid>.row>[class*="one wide widescreen"].column,.ui.grid>[class*="one wide widescreen"].column{width:6.25%!important}.ui.column.grid>[class*="two wide widescreen"].column,.ui.grid>.column.row>[class*="two wide widescreen"].column,.ui.grid>.row>[class*="two wide widescreen"].column,.ui.grid>[class*="two wide widescreen"].column{width:12.5%!important}.ui.column.grid>[class*="three wide widescreen"].column,.ui.grid>.column.row>[class*="three wide widescreen"].column,.ui.grid>.row>[class*="three wide widescreen"].column,.ui.grid>[class*="three wide widescreen"].column{width:18.75%!important}.ui.column.grid>[class*="four wide widescreen"].column,.ui.grid>.column.row>[class*="four wide widescreen"].column,.ui.grid>.row>[class*="four wide widescreen"].column,.ui.grid>[class*="four wide widescreen"].column{width:25%!important}.ui.column.grid>[class*="five wide widescreen"].column,.ui.grid>.column.row>[class*="five wide widescreen"].column,.ui.grid>.row>[class*="five wide widescreen"].column,.ui.grid>[class*="five wide widescreen"].column{width:31.25%!important}.ui.column.grid>[class*="six wide widescreen"].column,.ui.grid>.column.row>[class*="six wide widescreen"].column,.ui.grid>.row>[class*="six wide widescreen"].column,.ui.grid>[class*="six wide widescreen"].column{width:37.5%!important}.ui.column.grid>[class*="seven wide widescreen"].column,.ui.grid>.column.row>[class*="seven wide widescreen"].column,.ui.grid>.row>[class*="seven wide widescreen"].column,.ui.grid>[class*="seven wide widescreen"].column{width:43.75%!important}.ui.column.grid>[class*="eight wide widescreen"].column,.ui.grid>.column.row>[class*="eight wide widescreen"].column,.ui.grid>.row>[class*="eight wide widescreen"].column,.ui.grid>[class*="eight wide widescreen"].column{width:50%!important}.ui.column.grid>[class*="nine wide widescreen"].column,.ui.grid>.column.row>[class*="nine wide widescreen"].column,.ui.grid>.row>[class*="nine wide widescreen"].column,.ui.grid>[class*="nine wide widescreen"].column{width:56.25%!important}.ui.column.grid>[class*="ten wide widescreen"].column,.ui.grid>.column.row>[class*="ten wide widescreen"].column,.ui.grid>.row>[class*="ten wide widescreen"].column,.ui.grid>[class*="ten wide widescreen"].column{width:62.5%!important}.ui.column.grid>[class*="eleven wide widescreen"].column,.ui.grid>.column.row>[class*="eleven wide widescreen"].column,.ui.grid>.row>[class*="eleven wide widescreen"].column,.ui.grid>[class*="eleven wide widescreen"].column{width:68.75%!important}.ui.column.grid>[class*="twelve wide widescreen"].column,.ui.grid>.column.row>[class*="twelve wide widescreen"].column,.ui.grid>.row>[class*="twelve wide widescreen"].column,.ui.grid>[class*="twelve wide widescreen"].column{width:75%!important}.ui.column.grid>[class*="thirteen wide widescreen"].column,.ui.grid>.column.row>[class*="thirteen wide widescreen"].column,.ui.grid>.row>[class*="thirteen wide widescreen"].column,.ui.grid>[class*="thirteen wide widescreen"].column{width:81.25%!important}.ui.column.grid>[class*="fourteen wide widescreen"].column,.ui.grid>.column.row>[class*="fourteen wide widescreen"].column,.ui.grid>.row>[class*="fourteen wide widescreen"].column,.ui.grid>[class*="fourteen wide widescreen"].column{width:87.5%!important}.ui.column.grid>[class*="fifteen wide widescreen"].column,.ui.grid>.column.row>[class*="fifteen wide widescreen"].column,.ui.grid>.row>[class*="fifteen wide widescreen"].column,.ui.grid>[class*="fifteen wide widescreen"].column{width:93.75%!important}.ui.column.grid>[class*="sixteen wide widescreen"].column,.ui.grid>.column.row>[class*="sixteen wide widescreen"].column,.ui.grid>.row>[class*="sixteen wide widescreen"].column,.ui.grid>[class*="sixteen wide widescreen"].column{width:100%!important}}.ui.centered.grid,.ui.centered.grid>.row,.ui.grid>.centered.row{text-align:center;-webkit-box-pack:center;-ms-flex-pack:center;justify-content:center}.ui.centered.grid>.column:not(.aligned):not(.justified):not(.row),.ui.centered.grid>.row>.column:not(.aligned):not(.justified),.ui.grid .centered.row>.column:not(.aligned):not(.justified){text-align:left}.ui.grid>.centered.column,.ui.grid>.row>.centered.column{display:block;margin-left:auto;margin-right:auto}.ui.grid>.relaxed.row>.column,.ui.relaxed.grid>.column:not(.row),.ui.relaxed.grid>.row>.column{padding-left:1.5rem;padding-right:1.5rem}.ui.grid>[class*="very relaxed"].row>.column,.ui[class*="very relaxed"].grid>.column:not(.row),.ui[class*="very relaxed"].grid>.row>.column{padding-left:2.5rem;padding-right:2.5rem}.ui.grid .relaxed.row+.ui.divider,.ui.relaxed.grid .row+.ui.divider{margin-left:1.5rem;margin-right:1.5rem}.ui.grid [class*="very relaxed"].row+.ui.divider,.ui[class*="very relaxed"].grid .row+.ui.divider{margin-left:2.5rem;margin-right:2.5rem}.ui.padded.grid:not(.vertically):not(.horizontally){margin:0!important}[class*="horizontally padded"].ui.grid{margin-left:0!important;margin-right:0!important}[class*="vertically padded"].ui.grid{margin-top:0!important;margin-bottom:0!important}.ui.grid [class*="left floated"].column{margin-right:auto}.ui.grid [class*="right floated"].column{margin-left:auto}.ui.divided.grid:not([class*="vertically divided"])>.column:not(.row),.ui.divided.grid:not([class*="vertically divided"])>.row>.column{-webkit-box-shadow:-1px 0 0 0 rgba(34,36,38,.15);box-shadow:-1px 0 0 0 rgba(34,36,38,.15)}.ui[class*="vertically divided"].grid>.column:not(.row),.ui[class*="vertically divided"].grid>.row>.column{margin-top:1rem;margin-bottom:1rem;padding-top:0;padding-bottom:0}.ui[class*="vertically divided"].grid>.row{margin-top:0;margin-bottom:0}.ui.divided.grid:not([class*="vertically divided"])>.column:first-child,.ui.divided.grid:not([class*="vertically divided"])>.row>.column:first-child{-webkit-box-shadow:none;box-shadow:none}.ui[class*="vertically divided"].grid>.row:first-child>.column{margin-top:0}.ui.grid>.divided.row>.column{-webkit-box-shadow:-1px 0 0 0 rgba(34,36,38,.15);box-shadow:-1px 0 0 0 rgba(34,36,38,.15)}.ui.grid>.divided.row>.column:first-child{-webkit-box-shadow:none;box-shadow:none}.ui[class*="vertically divided"].grid>.row{position:relative}.ui[class*="vertically divided"].grid>.row:before{position:absolute;content:"";top:0;left:0;width:calc(100% - 2rem);height:1px;margin:0 1rem;-webkit-box-shadow:0 -1px 0 0 rgba(34,36,38,.15);box-shadow:0 -1px 0 0 rgba(34,36,38,.15)}.ui.padded.divided.grid:not(.vertically):not(.horizontally),[class*="horizontally padded"].ui.divided.grid{width:100%}.ui[class*="vertically divided"].grid>.row:first-child:before{-webkit-box-shadow:none;box-shadow:none}.ui.inverted.divided.grid:not([class*="vertically divided"])>.column:not(.row),.ui.inverted.divided.grid:not([class*="vertically divided"])>.row>.column{-webkit-box-shadow:-1px 0 0 0 rgba(255,255,255,.1);box-shadow:-1px 0 0 0 rgba(255,255,255,.1)}.ui.inverted.divided.grid:not([class*="vertically divided"])>.column:not(.row):first-child,.ui.inverted.divided.grid:not([class*="vertically divided"])>.row>.column:first-child{-webkit-box-shadow:none;box-shadow:none}.ui.inverted[class*="vertically divided"].grid>.row:before{-webkit-box-shadow:0 -1px 0 0 rgba(255,255,255,.1);box-shadow:0 -1px 0 0 rgba(255,255,255,.1)}.ui.relaxed[class*="vertically divided"].grid>.row:before{margin-left:1.5rem;margin-right:1.5rem;width:calc(100% - 3rem)}.ui[class*="very relaxed"][class*="vertically divided"].grid>.row:before{margin-left:5rem;margin-right:5rem;width:calc(100% - 5rem)}.ui.celled.grid{width:100%;margin:1em 0;-webkit-box-shadow:0 0 0 1px #d4d4d5;box-shadow:0 0 0 1px #d4d4d5}.ui.celled.grid>.row{width:100%!important;margin:0;padding:0;-webkit-box-shadow:0 -1px 0 0 #d4d4d5;box-shadow:0 -1px 0 0 #d4d4d5}.ui.celled.grid>.column:not(.row),.ui.celled.grid>.row>.column{-webkit-box-shadow:-1px 0 0 0 #d4d4d5;box-shadow:-1px 0 0 0 #d4d4d5}.ui.celled.grid>.column:first-child,.ui.celled.grid>.row>.column:first-child{-webkit-box-shadow:none;box-shadow:none}.ui.celled.grid>.column:not(.row),.ui.celled.grid>.row>.column{padding:1em}.ui.relaxed.celled.grid>.column:not(.row),.ui.relaxed.celled.grid>.row>.column{padding:1.5em}.ui[class*="very relaxed"].celled.grid>.column:not(.row),.ui[class*="very relaxed"].celled.grid>.row>.column{padding:2em}.ui[class*="internally celled"].grid{-webkit-box-shadow:none;box-shadow:none;margin:0}.ui[class*="internally celled"].grid>.row:first-child{-webkit-box-shadow:none;box-shadow:none}.ui[class*="internally celled"].grid>.row>.column:first-child{-webkit-box-shadow:none;box-shadow:none}.ui.grid>.row>[class*="top aligned"].column,.ui.grid>[class*="top aligned"].column:not(.row),.ui.grid>[class*="top aligned"].row>.column,.ui[class*="top aligned"].grid>.column:not(.row),.ui[class*="top aligned"].grid>.row>.column{-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column;vertical-align:top;-ms-flex-item-align:start!important;align-self:flex-start!important}.ui.grid>.row>[class*="middle aligned"].column,.ui.grid>[class*="middle aligned"].column:not(.row),.ui.grid>[class*="middle aligned"].row>.column,.ui[class*="middle aligned"].grid>.column:not(.row),.ui[class*="middle aligned"].grid>.row>.column{-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column;vertical-align:middle;-ms-flex-item-align:center!important;align-self:center!important}.ui.grid>.row>[class*="bottom aligned"].column,.ui.grid>[class*="bottom aligned"].column:not(.row),.ui.grid>[class*="bottom aligned"].row>.column,.ui[class*="bottom aligned"].grid>.column:not(.row),.ui[class*="bottom aligned"].grid>.row>.column{-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column;vertical-align:bottom;-ms-flex-item-align:end!important;align-self:flex-end!important}.ui.grid>.row>.stretched.column,.ui.grid>.stretched.column:not(.row),.ui.grid>.stretched.row>.column,.ui.stretched.grid>.column,.ui.stretched.grid>.row>.column{display:-webkit-inline-box!important;display:-ms-inline-flexbox!important;display:inline-flex!important;-ms-flex-item-align:stretch;align-self:stretch;-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column}.ui.grid>.row>.stretched.column>*,.ui.grid>.stretched.column:not(.row)>*,.ui.grid>.stretched.row>.column>*,.ui.stretched.grid>.column>*,.ui.stretched.grid>.row>.column>*{-webkit-box-flex:1;-ms-flex-positive:1;flex-grow:1}.ui.grid>.row>[class*="left aligned"].column.column,.ui.grid>[class*="left aligned"].column.column,.ui.grid>[class*="left aligned"].row>.column,.ui[class*="left aligned"].grid>.column,.ui[class*="left aligned"].grid>.row>.column{text-align:left;-ms-flex-item-align:inherit;align-self:inherit}.ui.grid>.row>[class*="center aligned"].column.column,.ui.grid>[class*="center aligned"].column.column,.ui.grid>[class*="center aligned"].row>.column,.ui[class*="center aligned"].grid>.column,.ui[class*="center aligned"].grid>.row>.column{text-align:center;-ms-flex-item-align:inherit;align-self:inherit}.ui[class*="center aligned"].grid{-webkit-box-pack:center;-ms-flex-pack:center;justify-content:center}.ui.grid>.row>[class*="right aligned"].column.column,.ui.grid>[class*="right aligned"].column.column,.ui.grid>[class*="right aligned"].row>.column,.ui[class*="right aligned"].grid>.column,.ui[class*="right aligned"].grid>.row>.column{text-align:right;-ms-flex-item-align:inherit;align-self:inherit}.ui.grid>.justified.column.column,.ui.grid>.justified.row>.column,.ui.grid>.row>.justified.column.column,.ui.justified.grid>.column,.ui.justified.grid>.row>.column{text-align:justify;-webkit-hyphens:auto;-ms-hyphens:auto;hyphens:auto}.ui.grid>.row>.black.column,.ui.grid>.row>.blue.column,.ui.grid>.row>.brown.column,.ui.grid>.row>.green.column,.ui.grid>.row>.grey.column,.ui.grid>.row>.olive.column,.ui.grid>.row>.orange.column,.ui.grid>.row>.pink.column,.ui.grid>.row>.purple.column,.ui.grid>.row>.red.column,.ui.grid>.row>.teal.column,.ui.grid>.row>.violet.column,.ui.grid>.row>.yellow.column{margin-top:-1rem;margin-bottom:-1rem;padding-top:1rem;padding-bottom:1rem}.ui.grid>.red.column,.ui.grid>.red.row,.ui.grid>.row>.red.column{background-color:#db2828!important;color:#fff}.ui.grid>.orange.column,.ui.grid>.orange.row,.ui.grid>.row>.orange.column{background-color:#f2711c!important;color:#fff}.ui.grid>.row>.yellow.column,.ui.grid>.yellow.column,.ui.grid>.yellow.row{background-color:#fbbd08!important;color:#fff}.ui.grid>.olive.column,.ui.grid>.olive.row,.ui.grid>.row>.olive.column{background-color:#b5cc18!important;color:#fff}.ui.grid>.green.column,.ui.grid>.green.row,.ui.grid>.row>.green.column{background-color:#21ba45!important;color:#fff}.ui.grid>.row>.teal.column,.ui.grid>.teal.column,.ui.grid>.teal.row{background-color:#00b5ad!important;color:#fff}.ui.grid>.blue.column,.ui.grid>.blue.row,.ui.grid>.row>.blue.column{background-color:#2185d0!important;color:#fff}.ui.grid>.row>.violet.column,.ui.grid>.violet.column,.ui.grid>.violet.row{background-color:#6435c9!important;color:#fff}.ui.grid>.purple.column,.ui.grid>.purple.row,.ui.grid>.row>.purple.column{background-color:#a333c8!important;color:#fff}.ui.grid>.pink.column,.ui.grid>.pink.row,.ui.grid>.row>.pink.column{background-color:#e03997!important;color:#fff}.ui.grid>.brown.column,.ui.grid>.brown.row,.ui.grid>.row>.brown.column{background-color:#a5673f!important;color:#fff}.ui.grid>.grey.column,.ui.grid>.grey.row,.ui.grid>.row>.grey.column{background-color:#767676!important;color:#fff}.ui.grid>.black.column,.ui.grid>.black.row,.ui.grid>.row>.black.column{background-color:#1b1c1d!important;color:#fff}.ui.grid>[class*="equal width"].row>.column,.ui[class*="equal width"].grid>.column:not(.row),.ui[class*="equal width"].grid>.row>.column{display:inline-block;-webkit-box-flex:1;-ms-flex-positive:1;flex-grow:1}.ui.grid>[class*="equal width"].row>.wide.column,.ui[class*="equal width"].grid>.row>.wide.column,.ui[class*="equal width"].grid>.wide.column{-webkit-box-flex:0;-ms-flex-positive:0;flex-grow:0}@media only screen and (max-width:767px){.ui.grid>[class*="mobile reversed"].row,.ui[class*="mobile reversed"].grid,.ui[class*="mobile reversed"].grid>.row{-webkit-box-orient:horizontal;-webkit-box-direction:reverse;-ms-flex-direction:row-reverse;flex-direction:row-reverse}.ui.stackable[class*="mobile reversed"],.ui[class*="mobile vertically reversed"].grid{-webkit-box-orient:vertical;-webkit-box-direction:reverse;-ms-flex-direction:column-reverse;flex-direction:column-reverse}.ui[class*="mobile reversed"].divided.grid:not([class*="vertically divided"])>.column:first-child,.ui[class*="mobile reversed"].divided.grid:not([class*="vertically divided"])>.row>.column:first-child{-webkit-box-shadow:-1px 0 0 0 rgba(34,36,38,.15);box-shadow:-1px 0 0 0 rgba(34,36,38,.15)}.ui[class*="mobile reversed"].divided.grid:not([class*="vertically divided"])>.column:last-child,.ui[class*="mobile reversed"].divided.grid:not([class*="vertically divided"])>.row>.column:last-child{-webkit-box-shadow:none;box-shadow:none}.ui.grid[class*="vertically divided"][class*="mobile vertically reversed"]>.row:first-child:before{-webkit-box-shadow:0 -1px 0 0 rgba(34,36,38,.15);box-shadow:0 -1px 0 0 rgba(34,36,38,.15)}.ui.grid[class*="vertically divided"][class*="mobile vertically reversed"]>.row:last-child:before{-webkit-box-shadow:none;box-shadow:none}.ui[class*="mobile reversed"].celled.grid>.row>.column:first-child{-webkit-box-shadow:-1px 0 0 0 #d4d4d5;box-shadow:-1px 0 0 0 #d4d4d5}.ui[class*="mobile reversed"].celled.grid>.row>.column:last-child{-webkit-box-shadow:none;box-shadow:none}}@media only screen and (min-width:768px) and (max-width:991px){.ui.grid>[class*="tablet reversed"].row,.ui[class*="tablet reversed"].grid,.ui[class*="tablet reversed"].grid>.row{-webkit-box-orient:horizontal;-webkit-box-direction:reverse;-ms-flex-direction:row-reverse;flex-direction:row-reverse}.ui[class*="tablet vertically reversed"].grid{-webkit-box-orient:vertical;-webkit-box-direction:reverse;-ms-flex-direction:column-reverse;flex-direction:column-reverse}.ui[class*="tablet reversed"].divided.grid:not([class*="vertically divided"])>.column:first-child,.ui[class*="tablet reversed"].divided.grid:not([class*="vertically divided"])>.row>.column:first-child{-webkit-box-shadow:-1px 0 0 0 rgba(34,36,38,.15);box-shadow:-1px 0 0 0 rgba(34,36,38,.15)}.ui[class*="tablet reversed"].divided.grid:not([class*="vertically divided"])>.column:last-child,.ui[class*="tablet reversed"].divided.grid:not([class*="vertically divided"])>.row>.column:last-child{-webkit-box-shadow:none;box-shadow:none}.ui.grid[class*="vertically divided"][class*="tablet vertically reversed"]>.row:first-child:before{-webkit-box-shadow:0 -1px 0 0 rgba(34,36,38,.15);box-shadow:0 -1px 0 0 rgba(34,36,38,.15)}.ui.grid[class*="vertically divided"][class*="tablet vertically reversed"]>.row:last-child:before{-webkit-box-shadow:none;box-shadow:none}.ui[class*="tablet reversed"].celled.grid>.row>.column:first-child{-webkit-box-shadow:-1px 0 0 0 #d4d4d5;box-shadow:-1px 0 0 0 #d4d4d5}.ui[class*="tablet reversed"].celled.grid>.row>.column:last-child{-webkit-box-shadow:none;box-shadow:none}}@media only screen and (min-width:992px){.ui.grid>[class*="computer reversed"].row,.ui[class*="computer reversed"].grid,.ui[class*="computer reversed"].grid>.row{-webkit-box-orient:horizontal;-webkit-box-direction:reverse;-ms-flex-direction:row-reverse;flex-direction:row-reverse}.ui[class*="computer vertically reversed"].grid{-webkit-box-orient:vertical;-webkit-box-direction:reverse;-ms-flex-direction:column-reverse;flex-direction:column-reverse}.ui[class*="computer reversed"].divided.grid:not([class*="vertically divided"])>.column:first-child,.ui[class*="computer reversed"].divided.grid:not([class*="vertically divided"])>.row>.column:first-child{-webkit-box-shadow:-1px 0 0 0 rgba(34,36,38,.15);box-shadow:-1px 0 0 0 rgba(34,36,38,.15)}.ui[class*="computer reversed"].divided.grid:not([class*="vertically divided"])>.column:last-child,.ui[class*="computer reversed"].divided.grid:not([class*="vertically divided"])>.row>.column:last-child{-webkit-box-shadow:none;box-shadow:none}.ui.grid[class*="vertically divided"][class*="computer vertically reversed"]>.row:first-child:before{-webkit-box-shadow:0 -1px 0 0 rgba(34,36,38,.15);box-shadow:0 -1px 0 0 rgba(34,36,38,.15)}.ui.grid[class*="vertically divided"][class*="computer vertically reversed"]>.row:last-child:before{-webkit-box-shadow:none;box-shadow:none}.ui[class*="computer reversed"].celled.grid>.row>.column:first-child{-webkit-box-shadow:-1px 0 0 0 #d4d4d5;box-shadow:-1px 0 0 0 #d4d4d5}.ui[class*="computer reversed"].celled.grid>.row>.column:last-child{-webkit-box-shadow:none;box-shadow:none}}@media only screen and (min-width:768px) and (max-width:991px){.ui.doubling.grid{width:auto}.ui.doubling.grid>.row,.ui.grid>.doubling.row{margin:0!important;padding:0!important}.ui.doubling.grid>.row>.column,.ui.grid>.doubling.row>.column{display:inline-block!important;padding-top:1rem!important;padding-bottom:1rem!important;-webkit-box-shadow:none!important;box-shadow:none!important;margin:0}.ui.grid>[class*="two column"].doubling.row.row>.column,.ui[class*="two column"].doubling.grid>.column:not(.row),.ui[class*="two column"].doubling.grid>.row>.column{width:100%!important}.ui.grid>[class*="three column"].doubling.row.row>.column,.ui[class*="three column"].doubling.grid>.column:not(.row),.ui[class*="three column"].doubling.grid>.row>.column{width:50%!important}.ui.grid>[class*="four column"].doubling.row.row>.column,.ui[class*="four column"].doubling.grid>.column:not(.row),.ui[class*="four column"].doubling.grid>.row>.column{width:50%!important}.ui.grid>[class*="five column"].doubling.row.row>.column,.ui[class*="five column"].doubling.grid>.column:not(.row),.ui[class*="five column"].doubling.grid>.row>.column{width:33.33333333%!important}.ui.grid>[class*="six column"].doubling.row.row>.column,.ui[class*="six column"].doubling.grid>.column:not(.row),.ui[class*="six column"].doubling.grid>.row>.column{width:33.33333333%!important}.ui.grid>[class*="seven column"].doubling.row.row>.column,.ui[class*="seven column"].doubling.grid>.column:not(.row),.ui[class*="seven column"].doubling.grid>.row>.column{width:33.33333333%!important}.ui.grid>[class*="eight column"].doubling.row.row>.column,.ui[class*="eight column"].doubling.grid>.column:not(.row),.ui[class*="eight column"].doubling.grid>.row>.column{width:25%!important}.ui.grid>[class*="nine column"].doubling.row.row>.column,.ui[class*="nine column"].doubling.grid>.column:not(.row),.ui[class*="nine column"].doubling.grid>.row>.column{width:25%!important}.ui.grid>[class*="ten column"].doubling.row.row>.column,.ui[class*="ten column"].doubling.grid>.column:not(.row),.ui[class*="ten column"].doubling.grid>.row>.column{width:20%!important}.ui.grid>[class*="eleven column"].doubling.row.row>.column,.ui[class*="eleven column"].doubling.grid>.column:not(.row),.ui[class*="eleven column"].doubling.grid>.row>.column{width:20%!important}.ui.grid>[class*="twelve column"].doubling.row.row>.column,.ui[class*="twelve column"].doubling.grid>.column:not(.row),.ui[class*="twelve column"].doubling.grid>.row>.column{width:16.66666667%!important}.ui.grid>[class*="thirteen column"].doubling.row.row>.column,.ui[class*="thirteen column"].doubling.grid>.column:not(.row),.ui[class*="thirteen column"].doubling.grid>.row>.column{width:16.66666667%!important}.ui.grid>[class*="fourteen column"].doubling.row.row>.column,.ui[class*="fourteen column"].doubling.grid>.column:not(.row),.ui[class*="fourteen column"].doubling.grid>.row>.column{width:14.28571429%!important}.ui.grid>[class*="fifteen column"].doubling.row.row>.column,.ui[class*="fifteen column"].doubling.grid>.column:not(.row),.ui[class*="fifteen column"].doubling.grid>.row>.column{width:14.28571429%!important}.ui.grid>[class*="sixteen column"].doubling.row.row>.column,.ui[class*="sixteen column"].doubling.grid>.column:not(.row),.ui[class*="sixteen column"].doubling.grid>.row>.column{width:12.5%!important}}@media only screen and (max-width:767px){.ui.doubling.grid>.row,.ui.grid>.doubling.row{margin:0!important;padding:0!important}.ui.doubling.grid>.row>.column,.ui.grid>.doubling.row>.column{padding-top:1rem!important;padding-bottom:1rem!important;margin:0!important;-webkit-box-shadow:none!important;box-shadow:none!important}.ui.grid>[class*="two column"].doubling:not(.stackable).row.row>.column,.ui[class*="two column"].doubling:not(.stackable).grid>.column:not(.row),.ui[class*="two column"].doubling:not(.stackable).grid>.row>.column{width:100%!important}.ui.grid>[class*="three column"].doubling:not(.stackable).row.row>.column,.ui[class*="three column"].doubling:not(.stackable).grid>.column:not(.row),.ui[class*="three column"].doubling:not(.stackable).grid>.row>.column{width:50%!important}.ui.grid>[class*="four column"].doubling:not(.stackable).row.row>.column,.ui[class*="four column"].doubling:not(.stackable).grid>.column:not(.row),.ui[class*="four column"].doubling:not(.stackable).grid>.row>.column{width:50%!important}.ui.grid>[class*="five column"].doubling:not(.stackable).row.row>.column,.ui[class*="five column"].doubling:not(.stackable).grid>.column:not(.row),.ui[class*="five column"].doubling:not(.stackable).grid>.row>.column{width:50%!important}.ui.grid>[class*="six column"].doubling:not(.stackable).row.row>.column,.ui[class*="six column"].doubling:not(.stackable).grid>.column:not(.row),.ui[class*="six column"].doubling:not(.stackable).grid>.row>.column{width:50%!important}.ui.grid>[class*="seven column"].doubling:not(.stackable).row.row>.column,.ui[class*="seven column"].doubling:not(.stackable).grid>.column:not(.row),.ui[class*="seven column"].doubling:not(.stackable).grid>.row>.column{width:50%!important}.ui.grid>[class*="eight column"].doubling:not(.stackable).row.row>.column,.ui[class*="eight column"].doubling:not(.stackable).grid>.column:not(.row),.ui[class*="eight column"].doubling:not(.stackable).grid>.row>.column{width:50%!important}.ui.grid>[class*="nine column"].doubling:not(.stackable).row.row>.column,.ui[class*="nine column"].doubling:not(.stackable).grid>.column:not(.row),.ui[class*="nine column"].doubling:not(.stackable).grid>.row>.column{width:33.33333333%!important}.ui.grid>[class*="ten column"].doubling:not(.stackable).row.row>.column,.ui[class*="ten column"].doubling:not(.stackable).grid>.column:not(.row),.ui[class*="ten column"].doubling:not(.stackable).grid>.row>.column{width:33.33333333%!important}.ui.grid>[class*="eleven column"].doubling:not(.stackable).row.row>.column,.ui[class*="eleven column"].doubling:not(.stackable).grid>.column:not(.row),.ui[class*="eleven column"].doubling:not(.stackable).grid>.row>.column{width:33.33333333%!important}.ui.grid>[class*="twelve column"].doubling:not(.stackable).row.row>.column,.ui[class*="twelve column"].doubling:not(.stackable).grid>.column:not(.row),.ui[class*="twelve column"].doubling:not(.stackable).grid>.row>.column{width:33.33333333%!important}.ui.grid>[class*="thirteen column"].doubling:not(.stackable).row.row>.column,.ui[class*="thirteen column"].doubling:not(.stackable).grid>.column:not(.row),.ui[class*="thirteen column"].doubling:not(.stackable).grid>.row>.column{width:33.33333333%!important}.ui.grid>[class*="fourteen column"].doubling:not(.stackable).row.row>.column,.ui[class*="fourteen column"].doubling:not(.stackable).grid>.column:not(.row),.ui[class*="fourteen column"].doubling:not(.stackable).grid>.row>.column{width:25%!important}.ui.grid>[class*="fifteen column"].doubling:not(.stackable).row.row>.column,.ui[class*="fifteen column"].doubling:not(.stackable).grid>.column:not(.row),.ui[class*="fifteen column"].doubling:not(.stackable).grid>.row>.column{width:25%!important}.ui.grid>[class*="sixteen column"].doubling:not(.stackable).row.row>.column,.ui[class*="sixteen column"].doubling:not(.stackable).grid>.column:not(.row),.ui[class*="sixteen column"].doubling:not(.stackable).grid>.row>.column{width:25%!important}}@media only screen and (max-width:767px){.ui.stackable.grid{width:auto;margin-left:0!important;margin-right:0!important}.ui.grid>.stackable.stackable.row>.column,.ui.stackable.grid>.column.grid>.column,.ui.stackable.grid>.column.row>.column,.ui.stackable.grid>.column:not(.row),.ui.stackable.grid>.row>.column,.ui.stackable.grid>.row>.wide.column,.ui.stackable.grid>.wide.column{width:100%!important;margin:0 0!important;-webkit-box-shadow:none!important;box-shadow:none!important;padding:1rem 1rem!important}.ui.stackable.grid:not(.vertically)>.row{margin:0;padding:0}.ui.container>.ui.stackable.grid>.column,.ui.container>.ui.stackable.grid>.row>.column{padding-left:0!important;padding-right:0!important}.ui.grid .ui.stackable.grid,.ui.segment:not(.vertical) .ui.stackable.page.grid{margin-left:-1rem!important;margin-right:-1rem!important}.ui.stackable.celled.grid>.column:not(.row):first-child,.ui.stackable.celled.grid>.row:first-child>.column:first-child,.ui.stackable.divided.grid>.column:not(.row):first-child,.ui.stackable.divided.grid>.row:first-child>.column:first-child{border-top:none!important}.ui.inverted.stackable.celled.grid>.column:not(.row),.ui.inverted.stackable.celled.grid>.row>.column,.ui.inverted.stackable.divided.grid>.column:not(.row),.ui.inverted.stackable.divided.grid>.row>.column{border-top:1px solid rgba(255,255,255,.1)}.ui.stackable.celled.grid>.column:not(.row),.ui.stackable.celled.grid>.row>.column,.ui.stackable.divided:not(.vertically).grid>.column:not(.row),.ui.stackable.divided:not(.vertically).grid>.row>.column{border-top:1px solid rgba(34,36,38,.15);-webkit-box-shadow:none!important;box-shadow:none!important;padding-top:2rem!important;padding-bottom:2rem!important}.ui.stackable.celled.grid>.row{-webkit-box-shadow:none!important;box-shadow:none!important}.ui.stackable.divided:not(.vertically).grid>.column:not(.row),.ui.stackable.divided:not(.vertically).grid>.row>.column{padding-left:0!important;padding-right:0!important}}@media only screen and (max-width:767px){.ui.grid.grid.grid>.row>[class*="tablet only"].column:not(.mobile),.ui.grid.grid.grid>[class*="tablet only"].column:not(.mobile),.ui.grid.grid.grid>[class*="tablet only"].row:not(.mobile),.ui[class*="tablet only"].grid.grid.grid:not(.mobile){display:none!important}.ui.grid.grid.grid>.row>[class*="computer only"].column:not(.mobile),.ui.grid.grid.grid>[class*="computer only"].column:not(.mobile),.ui.grid.grid.grid>[class*="computer only"].row:not(.mobile),.ui[class*="computer only"].grid.grid.grid:not(.mobile){display:none!important}.ui.grid.grid.grid>.row>[class*="large screen only"].column:not(.mobile),.ui.grid.grid.grid>[class*="large screen only"].column:not(.mobile),.ui.grid.grid.grid>[class*="large screen only"].row:not(.mobile),.ui[class*="large screen only"].grid.grid.grid:not(.mobile){display:none!important}.ui.grid.grid.grid>.row>[class*="widescreen only"].column:not(.mobile),.ui.grid.grid.grid>[class*="widescreen only"].column:not(.mobile),.ui.grid.grid.grid>[class*="widescreen only"].row:not(.mobile),.ui[class*="widescreen only"].grid.grid.grid:not(.mobile){display:none!important}}@media only screen and (min-width:768px) and (max-width:991px){.ui.grid.grid.grid>.row>[class*="mobile only"].column:not(.tablet),.ui.grid.grid.grid>[class*="mobile only"].column:not(.tablet),.ui.grid.grid.grid>[class*="mobile only"].row:not(.tablet),.ui[class*="mobile only"].grid.grid.grid:not(.tablet){display:none!important}.ui.grid.grid.grid>.row>[class*="computer only"].column:not(.tablet),.ui.grid.grid.grid>[class*="computer only"].column:not(.tablet),.ui.grid.grid.grid>[class*="computer only"].row:not(.tablet),.ui[class*="computer only"].grid.grid.grid:not(.tablet){display:none!important}.ui.grid.grid.grid>.row>[class*="large screen only"].column:not(.mobile),.ui.grid.grid.grid>[class*="large screen only"].column:not(.mobile),.ui.grid.grid.grid>[class*="large screen only"].row:not(.mobile),.ui[class*="large screen only"].grid.grid.grid:not(.mobile){display:none!important}.ui.grid.grid.grid>.row>[class*="widescreen only"].column:not(.mobile),.ui.grid.grid.grid>[class*="widescreen only"].column:not(.mobile),.ui.grid.grid.grid>[class*="widescreen only"].row:not(.mobile),.ui[class*="widescreen only"].grid.grid.grid:not(.mobile){display:none!important}}@media only screen and (min-width:992px) and (max-width:1199px){.ui.grid.grid.grid>.row>[class*="mobile only"].column:not(.computer),.ui.grid.grid.grid>[class*="mobile only"].column:not(.computer),.ui.grid.grid.grid>[class*="mobile only"].row:not(.computer),.ui[class*="mobile only"].grid.grid.grid:not(.computer){display:none!important}.ui.grid.grid.grid>.row>[class*="tablet only"].column:not(.computer),.ui.grid.grid.grid>[class*="tablet only"].column:not(.computer),.ui.grid.grid.grid>[class*="tablet only"].row:not(.computer),.ui[class*="tablet only"].grid.grid.grid:not(.computer){display:none!important}.ui.grid.grid.grid>.row>[class*="large screen only"].column:not(.mobile),.ui.grid.grid.grid>[class*="large screen only"].column:not(.mobile),.ui.grid.grid.grid>[class*="large screen only"].row:not(.mobile),.ui[class*="large screen only"].grid.grid.grid:not(.mobile){display:none!important}.ui.grid.grid.grid>.row>[class*="widescreen only"].column:not(.mobile),.ui.grid.grid.grid>[class*="widescreen only"].column:not(.mobile),.ui.grid.grid.grid>[class*="widescreen only"].row:not(.mobile),.ui[class*="widescreen only"].grid.grid.grid:not(.mobile){display:none!important}}@media only screen and (min-width:1200px) and (max-width:1919px){.ui.grid.grid.grid>.row>[class*="mobile only"].column:not(.computer),.ui.grid.grid.grid>[class*="mobile only"].column:not(.computer),.ui.grid.grid.grid>[class*="mobile only"].row:not(.computer),.ui[class*="mobile only"].grid.grid.grid:not(.computer){display:none!important}.ui.grid.grid.grid>.row>[class*="tablet only"].column:not(.computer),.ui.grid.grid.grid>[class*="tablet only"].column:not(.computer),.ui.grid.grid.grid>[class*="tablet only"].row:not(.computer),.ui[class*="tablet only"].grid.grid.grid:not(.computer){display:none!important}.ui.grid.grid.grid>.row>[class*="widescreen only"].column:not(.mobile),.ui.grid.grid.grid>[class*="widescreen only"].column:not(.mobile),.ui.grid.grid.grid>[class*="widescreen only"].row:not(.mobile),.ui[class*="widescreen only"].grid.grid.grid:not(.mobile){display:none!important}}@media only screen and (min-width:1920px){.ui.grid.grid.grid>.row>[class*="mobile only"].column:not(.computer),.ui.grid.grid.grid>[class*="mobile only"].column:not(.computer),.ui.grid.grid.grid>[class*="mobile only"].row:not(.computer),.ui[class*="mobile only"].grid.grid.grid:not(.computer){display:none!important}.ui.grid.grid.grid>.row>[class*="tablet only"].column:not(.computer),.ui.grid.grid.grid>[class*="tablet only"].column:not(.computer),.ui.grid.grid.grid>[class*="tablet only"].row:not(.computer),.ui[class*="tablet only"].grid.grid.grid:not(.computer){display:none!important}}.ui.menu{display:-webkit-box;display:-ms-flexbox;display:flex;margin:1rem 0;font-family:Lato,'Helvetica Neue',Arial,Helvetica,sans-serif;background:#fff;font-weight:400;border:1px solid rgba(34,36,38,.15);-webkit-box-shadow:0 1px 2px 0 rgba(34,36,38,.15);box-shadow:0 1px 2px 0 rgba(34,36,38,.15);border-radius:.28571429rem;min-height:2.85714286em}.ui.menu:after{content:'';display:block;height:0;clear:both;visibility:hidden}.ui.menu:first-child{margin-top:0}.ui.menu:last-child{margin-bottom:0}.ui.menu .menu{margin:0}.ui.menu:not(.vertical)>.menu{display:-webkit-box;display:-ms-flexbox;display:flex}.ui.menu:not(.vertical) .item{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-align:center;-ms-flex-align:center;align-items:center}.ui.menu .item{position:relative;vertical-align:middle;line-height:1;text-decoration:none;-webkit-tap-highlight-color:transparent;-webkit-box-flex:0;-ms-flex:0 0 auto;flex:0 0 auto;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;background:0 0;padding:.92857143em 1.14285714em;text-transform:none;color:rgba(0,0,0,.87);font-weight:400;-webkit-transition:background .1s ease,color .1s ease,-webkit-box-shadow .1s ease;transition:background .1s ease,color .1s ease,-webkit-box-shadow .1s ease;transition:background .1s ease,box-shadow .1s ease,color .1s ease;transition:background .1s ease,box-shadow .1s ease,color .1s ease,-webkit-box-shadow .1s ease}.ui.menu>.item:first-child{border-radius:.28571429rem 0 0 .28571429rem}.ui.menu .item:before{position:absolute;content:'';top:0;right:0;height:100%;width:1px;background:rgba(34,36,38,.1)}.ui.menu .item>a:not(.ui),.ui.menu .item>p:only-child,.ui.menu .text.item>*{-webkit-user-select:text;-moz-user-select:text;-ms-user-select:text;user-select:text;line-height:1.3}.ui.menu .item>p:first-child{margin-top:0}.ui.menu .item>p:last-child{margin-bottom:0}.ui.menu .item>i.icon{opacity:.9;float:none;margin:0 .35714286em 0 0}.ui.menu:not(.vertical) .item>.button{position:relative;top:0;margin:-.5em 0;padding-bottom:.78571429em;padding-top:.78571429em;font-size:1em}.ui.menu>.container,.ui.menu>.grid{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-align:inherit;-ms-flex-align:inherit;align-items:inherit;-webkit-box-orient:inherit;-webkit-box-direction:inherit;-ms-flex-direction:inherit;flex-direction:inherit}.ui.menu .item>.input{width:100%}.ui.menu:not(.vertical) .item>.input{position:relative;top:0;margin:-.5em 0}.ui.menu .item>.input input{font-size:1em;padding-top:.57142857em;padding-bottom:.57142857em}.ui.menu .header.item,.ui.vertical.menu .header.item{margin:0;background:'';text-transform:normal;font-weight:700}.ui.vertical.menu .item>.header:not(.ui){margin:0 0 .5em;font-size:1em;font-weight:700}.ui.menu .item>i.dropdown.icon{padding:0;float:right;margin:0 0 0 1em}.ui.menu .dropdown.item .menu{min-width:calc(100% - 1px);border-radius:0 0 .28571429rem .28571429rem;background:#fff;margin:0 0 0;-webkit-box-shadow:0 1px 3px 0 rgba(0,0,0,.08);box-shadow:0 1px 3px 0 rgba(0,0,0,.08);-webkit-box-orient:vertical!important;-webkit-box-direction:normal!important;-ms-flex-direction:column!important;flex-direction:column!important}.ui.menu .ui.dropdown .menu>.item{margin:0;text-align:left;font-size:1em!important;padding:.78571429em 1.14285714em!important;background:0 0!important;color:rgba(0,0,0,.87)!important;text-transform:none!important;font-weight:400!important;-webkit-box-shadow:none!important;box-shadow:none!important;-webkit-transition:none!important;transition:none!important}.ui.menu .ui.dropdown .menu>.item:hover{background:rgba(0,0,0,.05)!important;color:rgba(0,0,0,.95)!important}.ui.menu .ui.dropdown .menu>.selected.item{background:rgba(0,0,0,.05)!important;color:rgba(0,0,0,.95)!important}.ui.menu .ui.dropdown .menu>.active.item{background:rgba(0,0,0,.03)!important;font-weight:700!important;color:rgba(0,0,0,.95)!important}.ui.menu .ui.dropdown.item .menu .item:not(.filtered){display:block}.ui.menu .ui.dropdown .menu>.item .icon:not(.dropdown){display:inline-block;font-size:1em!important;float:none;margin:0 .75em 0 0!important}.ui.secondary.menu .dropdown.item>.menu,.ui.text.menu .dropdown.item>.menu{border-radius:.28571429rem;margin-top:.35714286em}.ui.menu .pointing.dropdown.item .menu{margin-top:.75em}.ui.inverted.menu .search.dropdown.item>.search,.ui.inverted.menu .search.dropdown.item>.text{color:rgba(255,255,255,.9)}.ui.vertical.menu .dropdown.item>.icon{float:right;content:"\f0da";margin-left:1em}.ui.vertical.menu .dropdown.item .menu{left:100%;min-width:0;margin:0;-webkit-box-shadow:0 1px 3px 0 rgba(0,0,0,.08);box-shadow:0 1px 3px 0 rgba(0,0,0,.08);border-radius:0 .28571429rem .28571429rem .28571429rem}.ui.vertical.menu .dropdown.item.upward .menu{bottom:0}.ui.vertical.menu .dropdown.item:not(.upward) .menu{top:0}.ui.vertical.menu .active.dropdown.item{border-top-right-radius:0;border-bottom-right-radius:0}.ui.vertical.menu .dropdown.active.item{-webkit-box-shadow:none;box-shadow:none}.ui.item.menu .dropdown .menu .item{width:100%}.ui.menu .item>.label{background:#999;color:#fff;margin-left:1em;padding:.3em .78571429em}.ui.vertical.menu .item>.label{background:#999;color:#fff;margin-top:-.15em;margin-bottom:-.15em;padding:.3em .78571429em}.ui.menu .item>.floating.label{padding:.3em .78571429em}.ui.menu .item>img:not(.ui){display:inline-block;vertical-align:middle;margin:-.3em 0;width:2.5em}.ui.vertical.menu .item>img:not(.ui):only-child{display:block;max-width:100%;width:auto}.ui.menu .list .item:before{background:0 0!important}.ui.vertical.sidebar.menu>.item:first-child:before{display:block!important}.ui.vertical.sidebar.menu>.item::before{top:auto;bottom:0}@media only screen and (max-width:767px){.ui.menu>.ui.container{width:100%!important;margin-left:0!important;margin-right:0!important}}@media only screen and (min-width:768px){.ui.menu:not(.secondary):not(.text):not(.tabular):not(.borderless)>.container>.item:not(.right):not(.borderless):first-child{border-left:1px solid rgba(34,36,38,.1)}}.ui.link.menu .item:hover,.ui.menu .dropdown.item:hover,.ui.menu .link.item:hover,.ui.menu a.item:hover{cursor:pointer;background:rgba(0,0,0,.03);color:rgba(0,0,0,.95)}.ui.link.menu .item:active,.ui.menu .link.item:active,.ui.menu a.item:active{background:rgba(0,0,0,.03);color:rgba(0,0,0,.95)}.ui.menu .active.item{background:rgba(0,0,0,.05);color:rgba(0,0,0,.95);font-weight:400;-webkit-box-shadow:none;box-shadow:none}.ui.menu .active.item>i.icon{opacity:1}.ui.menu .active.item:hover,.ui.vertical.menu .active.item:hover{background-color:rgba(0,0,0,.05);color:rgba(0,0,0,.95)}.ui.menu .item.disabled,.ui.menu .item.disabled:hover{cursor:default!important;background-color:transparent!important;color:rgba(40,40,40,.3)!important}.ui.menu:not(.vertical) .left.item,.ui.menu:not(.vertical) :not(.dropdown)>.left.menu{display:-webkit-box;display:-ms-flexbox;display:flex;margin-right:auto!important}.ui.menu:not(.vertical) .right.item,.ui.menu:not(.vertical) .right.menu{display:-webkit-box;display:-ms-flexbox;display:flex;margin-left:auto!important}.ui.menu .right.item::before,.ui.menu .right.menu>.item::before{right:auto;left:0}.ui.vertical.menu{display:block;-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column;background:#fff;-webkit-box-shadow:0 1px 2px 0 rgba(34,36,38,.15);box-shadow:0 1px 2px 0 rgba(34,36,38,.15)}.ui.vertical.menu .item{display:block;background:0 0;border-top:none;border-right:none}.ui.vertical.menu>.item:first-child{border-radius:.28571429rem .28571429rem 0 0}.ui.vertical.menu>.item:last-child{border-radius:0 0 .28571429rem .28571429rem}.ui.vertical.menu .item>.label{float:right;text-align:center}.ui.vertical.menu .item>i.icon{width:1.18em;float:right;margin:0 0 0 .5em}.ui.vertical.menu .item>.label+i.icon{float:none;margin:0 .5em 0 0}.ui.vertical.menu .item:before{position:absolute;content:'';top:0;left:0;width:100%;height:1px;background:rgba(34,36,38,.1)}.ui.vertical.menu .item:first-child:before{display:none!important}.ui.vertical.menu .item>.menu{margin:.5em -1.14285714em 0}.ui.vertical.menu .menu .item{background:0 0;padding:.5em 1.33333333em;font-size:.85714286em;color:rgba(0,0,0,.5)}.ui.vertical.menu .item .menu .link.item:hover,.ui.vertical.menu .item .menu a.item:hover{color:rgba(0,0,0,.85)}.ui.vertical.menu .menu .item:before{display:none}.ui.vertical.menu .active.item{background:rgba(0,0,0,.05);border-radius:0;-webkit-box-shadow:none;box-shadow:none}.ui.vertical.menu>.active.item:first-child{border-radius:.28571429rem .28571429rem 0 0}.ui.vertical.menu>.active.item:last-child{border-radius:0 0 .28571429rem .28571429rem}.ui.vertical.menu>.active.item:only-child{border-radius:.28571429rem}.ui.vertical.menu .active.item .menu .active.item{border-left:none}.ui.vertical.menu .item .menu .active.item{background-color:transparent;font-weight:700;color:rgba(0,0,0,.95)}.ui.tabular.menu{border-radius:0;-webkit-box-shadow:none!important;box-shadow:none!important;border:none;background:none transparent;border-bottom:1px solid #d4d4d5}.ui.tabular.fluid.menu{width:calc(100% + 2px)!important}.ui.tabular.menu .item{background:0 0;border-bottom:none;border-left:1px solid transparent;border-right:1px solid transparent;border-top:2px solid transparent;padding:.92857143em 1.42857143em;color:rgba(0,0,0,.87)}.ui.tabular.menu .item:before{display:none}.ui.tabular.menu .item:hover{background-color:transparent;color:rgba(0,0,0,.8)}.ui.tabular.menu .active.item{background:none #fff;color:rgba(0,0,0,.95);border-top-width:1px;border-color:#d4d4d5;font-weight:700;margin-bottom:-1px;-webkit-box-shadow:none;box-shadow:none;border-radius:.28571429rem .28571429rem 0 0!important}.ui.tabular.menu+.attached:not(.top).segment,.ui.tabular.menu+.attached:not(.top).segment+.attached:not(.top).segment{border-top:none;margin-left:0;margin-top:0;margin-right:0;width:100%}.top.attached.segment+.ui.bottom.tabular.menu{position:relative;width:calc(100% + 2px);left:-1px}.ui.bottom.tabular.menu{background:none transparent;border-radius:0;-webkit-box-shadow:none!important;box-shadow:none!important;border-bottom:none;border-top:1px solid #d4d4d5}.ui.bottom.tabular.menu .item{background:0 0;border-left:1px solid transparent;border-right:1px solid transparent;border-bottom:1px solid transparent;border-top:none}.ui.bottom.tabular.menu .active.item{background:none #fff;color:rgba(0,0,0,.95);border-color:#d4d4d5;margin:-1px 0 0 0;border-radius:0 0 .28571429rem .28571429rem!important}.ui.vertical.tabular.menu{background:none transparent;border-radius:0;-webkit-box-shadow:none!important;box-shadow:none!important;border-bottom:none;border-right:1px solid #d4d4d5}.ui.vertical.tabular.menu .item{background:0 0;border-left:1px solid transparent;border-bottom:1px solid transparent;border-top:1px solid transparent;border-right:none}.ui.vertical.tabular.menu .active.item{background:none #fff;color:rgba(0,0,0,.95);border-color:#d4d4d5;margin:0 -1px 0 0;border-radius:.28571429rem 0 0 .28571429rem!important}.ui.vertical.right.tabular.menu{background:none transparent;border-radius:0;-webkit-box-shadow:none!important;box-shadow:none!important;border-bottom:none;border-right:none;border-left:1px solid #d4d4d5}.ui.vertical.right.tabular.menu .item{background:0 0;border-right:1px solid transparent;border-bottom:1px solid transparent;border-top:1px solid transparent;border-left:none}.ui.vertical.right.tabular.menu .active.item{background:none #fff;color:rgba(0,0,0,.95);border-color:#d4d4d5;margin:0 0 0 -1px;border-radius:0 .28571429rem .28571429rem 0!important}.ui.tabular.menu .active.dropdown.item{margin-bottom:0;border-left:1px solid transparent;border-right:1px solid transparent;border-top:2px solid transparent;border-bottom:none}.ui.pagination.menu{margin:0;display:-webkit-inline-box;display:-ms-inline-flexbox;display:inline-flex;vertical-align:middle}.ui.pagination.menu .item:last-child{border-radius:0 .28571429rem .28571429rem 0}.ui.compact.menu .item:last-child{border-radius:0 .28571429rem .28571429rem 0}.ui.pagination.menu .item:last-child:before{display:none}.ui.pagination.menu .item{min-width:3em;text-align:center}.ui.pagination.menu .icon.item i.icon{vertical-align:top}.ui.pagination.menu .active.item{border-top:none;padding-top:.92857143em;background-color:rgba(0,0,0,.05);color:rgba(0,0,0,.95);-webkit-box-shadow:none;box-shadow:none}.ui.secondary.menu{background:0 0;margin-left:-.35714286em;margin-right:-.35714286em;border-radius:0;border:none;-webkit-box-shadow:none;box-shadow:none}.ui.secondary.menu .item{-ms-flex-item-align:center;align-self:center;-webkit-box-shadow:none;box-shadow:none;border:none;padding:.78571429em .92857143em;margin:0 .35714286em;background:0 0;-webkit-transition:color .1s ease;transition:color .1s ease;border-radius:.28571429rem}.ui.secondary.menu .item:before{display:none!important}.ui.secondary.menu .header.item{border-radius:0;border-right:none;background:none transparent}.ui.secondary.menu .item>img:not(.ui){margin:0}.ui.secondary.menu .dropdown.item:hover,.ui.secondary.menu .link.item:hover,.ui.secondary.menu a.item:hover{background:rgba(0,0,0,.05);color:rgba(0,0,0,.95)}.ui.secondary.menu .active.item{-webkit-box-shadow:none;box-shadow:none;background:rgba(0,0,0,.05);color:rgba(0,0,0,.95);border-radius:.28571429rem}.ui.secondary.menu .active.item:hover{-webkit-box-shadow:none;box-shadow:none;background:rgba(0,0,0,.05);color:rgba(0,0,0,.95)}.ui.secondary.inverted.menu .link.item,.ui.secondary.inverted.menu a.item{color:rgba(255,255,255,.7)!important}.ui.secondary.inverted.menu .dropdown.item:hover,.ui.secondary.inverted.menu .link.item:hover,.ui.secondary.inverted.menu a.item:hover{background:rgba(255,255,255,.08);color:#fff!important}.ui.secondary.inverted.menu .active.item{background:rgba(255,255,255,.15);color:#fff!important}.ui.secondary.item.menu{margin-left:0;margin-right:0}.ui.secondary.item.menu .item:last-child{margin-right:0}.ui.secondary.attached.menu{-webkit-box-shadow:none;box-shadow:none}.ui.vertical.secondary.menu .item:not(.dropdown)>.menu{margin:0 -.92857143em}.ui.vertical.secondary.menu .item:not(.dropdown)>.menu>.item{margin:0;padding:.5em 1.33333333em}.ui.secondary.vertical.menu>.item{border:none;margin:0 0 .35714286em;border-radius:.28571429rem!important}.ui.secondary.vertical.menu>.header.item{border-radius:0}.ui.vertical.secondary.menu .item>.menu .item{background-color:transparent}.ui.secondary.inverted.menu{background-color:transparent}.ui.secondary.pointing.menu{margin-left:0;margin-right:0;border-bottom:2px solid rgba(34,36,38,.15)}.ui.secondary.pointing.menu .item{border-bottom-color:transparent;border-bottom-style:solid;border-radius:0;-ms-flex-item-align:end;align-self:flex-end;margin:0 0 -2px;padding:.85714286em 1.14285714em;border-bottom-width:2px;-webkit-transition:color .1s ease;transition:color .1s ease}.ui.secondary.pointing.menu .header.item{color:rgba(0,0,0,.85)!important}.ui.secondary.pointing.menu .text.item{-webkit-box-shadow:none!important;box-shadow:none!important}.ui.secondary.pointing.menu .item:after{display:none}.ui.secondary.pointing.menu .dropdown.item:hover,.ui.secondary.pointing.menu .link.item:hover,.ui.secondary.pointing.menu a.item:hover{background-color:transparent;color:rgba(0,0,0,.87)}.ui.secondary.pointing.menu .dropdown.item:active,.ui.secondary.pointing.menu .link.item:active,.ui.secondary.pointing.menu a.item:active{background-color:transparent;border-color:rgba(34,36,38,.15)}.ui.secondary.pointing.menu .active.item{background-color:transparent;-webkit-box-shadow:none;box-shadow:none;border-color:#1b1c1d;font-weight:700;color:rgba(0,0,0,.95)}.ui.secondary.pointing.menu .active.item:hover{border-color:#1b1c1d;color:rgba(0,0,0,.95)}.ui.secondary.pointing.menu .active.dropdown.item{border-color:transparent}.ui.secondary.vertical.pointing.menu{border-bottom-width:0;border-right-width:2px;border-right-style:solid;border-right-color:rgba(34,36,38,.15)}.ui.secondary.vertical.pointing.menu .item{border-bottom:none;border-right-style:solid;border-right-color:transparent;border-radius:0!important;margin:0 -2px 0 0;border-right-width:2px}.ui.secondary.vertical.pointing.menu .active.item{border-color:#1b1c1d}.ui.secondary.inverted.pointing.menu{border-color:rgba(255,255,255,.1)}.ui.secondary.inverted.pointing.menu{border-width:2px;border-color:rgba(34,36,38,.15)}.ui.secondary.inverted.pointing.menu .item{color:rgba(255,255,255,.9)}.ui.secondary.inverted.pointing.menu .header.item{color:#fff!important}.ui.secondary.inverted.pointing.menu .link.item:hover,.ui.secondary.inverted.pointing.menu a.item:hover{color:rgba(0,0,0,.95)}.ui.secondary.inverted.pointing.menu .active.item{border-color:#fff;color:#fff}.ui.text.menu{background:none transparent;border-radius:0;-webkit-box-shadow:none;box-shadow:none;border:none;margin:1em -.5em}.ui.text.menu .item{border-radius:0;-webkit-box-shadow:none;box-shadow:none;-ms-flex-item-align:center;align-self:center;margin:0 0;padding:.35714286em .5em;font-weight:400;color:rgba(0,0,0,.6);-webkit-transition:opacity .1s ease;transition:opacity .1s ease}.ui.text.menu .item:before,.ui.text.menu .menu .item:before{display:none!important}.ui.text.menu .header.item{background-color:transparent;opacity:1;color:rgba(0,0,0,.85);font-size:.92857143em;text-transform:uppercase;font-weight:700}.ui.text.menu .item>img:not(.ui){margin:0}.ui.text.item.menu .item{margin:0}.ui.vertical.text.menu{margin:1em 0}.ui.vertical.text.menu:first-child{margin-top:0}.ui.vertical.text.menu:last-child{margin-bottom:0}.ui.vertical.text.menu .item{margin:.57142857em 0;padding-left:0;padding-right:0}.ui.vertical.text.menu .item>i.icon{float:none;margin:0 .35714286em 0 0}.ui.vertical.text.menu .header.item{margin:.57142857em 0 .71428571em}.ui.vertical.text.menu .item:not(.dropdown)>.menu{margin:0}.ui.vertical.text.menu .item:not(.dropdown)>.menu>.item{margin:0;padding:.5em 0}.ui.text.menu .item:hover{opacity:1;background-color:transparent}.ui.text.menu .active.item{background-color:transparent;border:none;-webkit-box-shadow:none;box-shadow:none;font-weight:400;color:rgba(0,0,0,.95)}.ui.text.menu .active.item:hover{background-color:transparent}.ui.text.pointing.menu .active.item:after{-webkit-box-shadow:none;box-shadow:none}.ui.text.attached.menu{-webkit-box-shadow:none;box-shadow:none}.ui.inverted.text.menu,.ui.inverted.text.menu .active.item,.ui.inverted.text.menu .item,.ui.inverted.text.menu .item:hover{background-color:transparent!important}.ui.fluid.text.menu{margin-left:0;margin-right:0}.ui.vertical.icon.menu{display:inline-block;width:auto}.ui.icon.menu .item{height:auto;text-align:center;color:#1b1c1d}.ui.icon.menu .item>.icon:not(.dropdown){margin:0;opacity:1}.ui.icon.menu .icon:before{opacity:1}.ui.menu .icon.item>.icon{width:auto;margin:0 auto}.ui.vertical.icon.menu .item>.icon:not(.dropdown){display:block;opacity:1;margin:0 auto;float:none}.ui.inverted.icon.menu .item{color:#fff}.ui.labeled.icon.menu{text-align:center}.ui.labeled.icon.menu .item{min-width:6em;-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column}.ui.labeled.icon.menu .item>.icon:not(.dropdown){height:1em;display:block;font-size:1.71428571em!important;margin:0 auto .5rem!important}.ui.fluid.labeled.icon.menu>.item{min-width:0}@media only screen and (max-width:767px){.ui.stackable.menu{-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column}.ui.stackable.menu .item{width:100%!important}.ui.stackable.menu .item:before{position:absolute;content:'';top:auto;bottom:0;left:0;width:100%;height:1px;background:rgba(34,36,38,.1)}.ui.stackable.menu .left.item,.ui.stackable.menu .left.menu{margin-right:0!important}.ui.stackable.menu .right.item,.ui.stackable.menu .right.menu{margin-left:0!important}.ui.stackable.menu .left.menu,.ui.stackable.menu .right.menu{-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column}}.ui.menu .red.active.item,.ui.red.menu .active.item{border-color:#db2828!important;color:#db2828!important}.ui.menu .orange.active.item,.ui.orange.menu .active.item{border-color:#f2711c!important;color:#f2711c!important}.ui.menu .yellow.active.item,.ui.yellow.menu .active.item{border-color:#fbbd08!important;color:#fbbd08!important}.ui.menu .olive.active.item,.ui.olive.menu .active.item{border-color:#b5cc18!important;color:#b5cc18!important}.ui.green.menu .active.item,.ui.menu .green.active.item{border-color:#21ba45!important;color:#21ba45!important}.ui.menu .teal.active.item,.ui.teal.menu .active.item{border-color:#00b5ad!important;color:#00b5ad!important}.ui.blue.menu .active.item,.ui.menu .blue.active.item{border-color:#2185d0!important;color:#2185d0!important}.ui.menu .violet.active.item,.ui.violet.menu .active.item{border-color:#6435c9!important;color:#6435c9!important}.ui.menu .purple.active.item,.ui.purple.menu .active.item{border-color:#a333c8!important;color:#a333c8!important}.ui.menu .pink.active.item,.ui.pink.menu .active.item{border-color:#e03997!important;color:#e03997!important}.ui.brown.menu .active.item,.ui.menu .brown.active.item{border-color:#a5673f!important;color:#a5673f!important}.ui.grey.menu .active.item,.ui.menu .grey.active.item{border-color:#767676!important;color:#767676!important}.ui.inverted.menu{border:0 solid transparent;background:#1b1c1d;-webkit-box-shadow:none;box-shadow:none}.ui.inverted.menu .item,.ui.inverted.menu .item>a:not(.ui){background:0 0;color:rgba(255,255,255,.9)}.ui.inverted.menu .item.menu{background:0 0}.ui.inverted.menu .item:before{background:rgba(255,255,255,.08)}.ui.vertical.inverted.menu .item:before{background:rgba(255,255,255,.08)}.ui.vertical.inverted.menu .menu .item,.ui.vertical.inverted.menu .menu .item a:not(.ui){color:rgba(255,255,255,.5)}.ui.inverted.menu .header.item{margin:0;background:0 0;-webkit-box-shadow:none;box-shadow:none}.ui.inverted.menu .item.disabled,.ui.inverted.menu .item.disabled:hover{color:rgba(225,225,225,.3)}.ui.inverted.menu .dropdown.item:hover,.ui.inverted.menu .link.item:hover,.ui.inverted.menu a.item:hover,.ui.link.inverted.menu .item:hover{background:rgba(255,255,255,.08);color:#fff}.ui.vertical.inverted.menu .item .menu .link.item:hover,.ui.vertical.inverted.menu .item .menu a.item:hover{background:0 0;color:#fff}.ui.inverted.menu .link.item:active,.ui.inverted.menu a.item:active{background:rgba(255,255,255,.08);color:#fff}.ui.inverted.menu .active.item{background:rgba(255,255,255,.15);color:#fff!important}.ui.inverted.vertical.menu .item .menu .active.item{background:0 0;color:#fff}.ui.inverted.pointing.menu .active.item:after{background:#3d3e3f!important;margin:0!important;-webkit-box-shadow:none!important;box-shadow:none!important;border:none!important}.ui.inverted.menu .active.item:hover{background:rgba(255,255,255,.15);color:#fff!important}.ui.inverted.pointing.menu .active.item:hover:after{background:#3d3e3f!important}.ui.floated.menu{float:left;margin:0 .5rem 0 0}.ui.floated.menu .item:last-child:before{display:none}.ui.right.floated.menu{float:right;margin:0 0 0 .5rem}.ui.inverted.menu .red.active.item,.ui.inverted.red.menu{background-color:#db2828}.ui.inverted.red.menu .item:before{background-color:rgba(34,36,38,.1)}.ui.inverted.red.menu .active.item{background-color:rgba(0,0,0,.1)!important}.ui.inverted.menu .orange.active.item,.ui.inverted.orange.menu{background-color:#f2711c}.ui.inverted.orange.menu .item:before{background-color:rgba(34,36,38,.1)}.ui.inverted.orange.menu .active.item{background-color:rgba(0,0,0,.1)!important}.ui.inverted.menu .yellow.active.item,.ui.inverted.yellow.menu{background-color:#fbbd08}.ui.inverted.yellow.menu .item:before{background-color:rgba(34,36,38,.1)}.ui.inverted.yellow.menu .active.item{background-color:rgba(0,0,0,.1)!important}.ui.inverted.menu .olive.active.item,.ui.inverted.olive.menu{background-color:#b5cc18}.ui.inverted.olive.menu .item:before{background-color:rgba(34,36,38,.1)}.ui.inverted.olive.menu .active.item{background-color:rgba(0,0,0,.1)!important}.ui.inverted.green.menu,.ui.inverted.menu .green.active.item{background-color:#21ba45}.ui.inverted.green.menu .item:before{background-color:rgba(34,36,38,.1)}.ui.inverted.green.menu .active.item{background-color:rgba(0,0,0,.1)!important}.ui.inverted.menu .teal.active.item,.ui.inverted.teal.menu{background-color:#00b5ad}.ui.inverted.teal.menu .item:before{background-color:rgba(34,36,38,.1)}.ui.inverted.teal.menu .active.item{background-color:rgba(0,0,0,.1)!important}.ui.inverted.blue.menu,.ui.inverted.menu .blue.active.item{background-color:#2185d0}.ui.inverted.blue.menu .item:before{background-color:rgba(34,36,38,.1)}.ui.inverted.blue.menu .active.item{background-color:rgba(0,0,0,.1)!important}.ui.inverted.menu .violet.active.item,.ui.inverted.violet.menu{background-color:#6435c9}.ui.inverted.violet.menu .item:before{background-color:rgba(34,36,38,.1)}.ui.inverted.violet.menu .active.item{background-color:rgba(0,0,0,.1)!important}.ui.inverted.menu .purple.active.item,.ui.inverted.purple.menu{background-color:#a333c8}.ui.inverted.purple.menu .item:before{background-color:rgba(34,36,38,.1)}.ui.inverted.purple.menu .active.item{background-color:rgba(0,0,0,.1)!important}.ui.inverted.menu .pink.active.item,.ui.inverted.pink.menu{background-color:#e03997}.ui.inverted.pink.menu .item:before{background-color:rgba(34,36,38,.1)}.ui.inverted.pink.menu .active.item{background-color:rgba(0,0,0,.1)!important}.ui.inverted.brown.menu,.ui.inverted.menu .brown.active.item{background-color:#a5673f}.ui.inverted.brown.menu .item:before{background-color:rgba(34,36,38,.1)}.ui.inverted.brown.menu .active.item{background-color:rgba(0,0,0,.1)!important}.ui.inverted.grey.menu,.ui.inverted.menu .grey.active.item{background-color:#767676}.ui.inverted.grey.menu .item:before{background-color:rgba(34,36,38,.1)}.ui.inverted.grey.menu .active.item{background-color:rgba(0,0,0,.1)!important}.ui.fitted.menu .item,.ui.fitted.menu .item .menu .item,.ui.menu .fitted.item{padding:0}.ui.horizontally.fitted.menu .item,.ui.horizontally.fitted.menu .item .menu .item,.ui.menu .horizontally.fitted.item{padding-top:.92857143em;padding-bottom:.92857143em}.ui.menu .vertically.fitted.item,.ui.vertically.fitted.menu .item,.ui.vertically.fitted.menu .item .menu .item{padding-left:1.14285714em;padding-right:1.14285714em}.ui.borderless.menu .item .menu .item:before,.ui.borderless.menu .item:before,.ui.menu .borderless.item:before{background:0 0!important}.ui.compact.menu{display:-webkit-inline-box;display:-ms-inline-flexbox;display:inline-flex;margin:0;vertical-align:middle}.ui.compact.vertical.menu{display:inline-block}.ui.compact.menu .item:last-child{border-radius:0 .28571429rem .28571429rem 0}.ui.compact.menu .item:last-child:before{display:none}.ui.compact.vertical.menu{width:auto!important}.ui.compact.vertical.menu .item:last-child::before{display:block}.ui.menu.fluid,.ui.vertical.menu.fluid{width:100%!important}.ui.item.menu,.ui.item.menu .item{width:100%;padding-left:0!important;padding-right:0!important;margin-left:0!important;margin-right:0!important;text-align:center;-webkit-box-pack:center;-ms-flex-pack:center;justify-content:center}.ui.attached.item.menu{margin:0 -1px!important}.ui.item.menu .item:last-child:before{display:none}.ui.menu.two.item .item{width:50%}.ui.menu.three.item .item{width:33.333%}.ui.menu.four.item .item{width:25%}.ui.menu.five.item .item{width:20%}.ui.menu.six.item .item{width:16.666%}.ui.menu.seven.item .item{width:14.285%}.ui.menu.eight.item .item{width:12.5%}.ui.menu.nine.item .item{width:11.11%}.ui.menu.ten.item .item{width:10%}.ui.menu.eleven.item .item{width:9.09%}.ui.menu.twelve.item .item{width:8.333%}.ui.menu.fixed{position:fixed;z-index:101;margin:0;width:100%}.ui.menu.fixed,.ui.menu.fixed .item:first-child,.ui.menu.fixed .item:last-child{border-radius:0!important}.ui.fixed.menu,.ui[class*="top fixed"].menu{top:0;left:0;right:auto;bottom:auto}.ui[class*="top fixed"].menu{border-top:none;border-left:none;border-right:none}.ui[class*="right fixed"].menu{border-top:none;border-bottom:none;border-right:none;top:0;right:0;left:auto;bottom:auto;width:auto;height:100%}.ui[class*="bottom fixed"].menu{border-bottom:none;border-left:none;border-right:none;bottom:0;left:0;top:auto;right:auto}.ui[class*="left fixed"].menu{border-top:none;border-bottom:none;border-left:none;top:0;left:0;right:auto;bottom:auto;width:auto;height:100%}.ui.fixed.menu+.ui.grid{padding-top:2.75rem}.ui.pointing.menu .item:after{visibility:hidden;position:absolute;content:'';top:100%;left:50%;-webkit-transform:translateX(-50%) translateY(-50%) rotate(45deg);transform:translateX(-50%) translateY(-50%) rotate(45deg);background:0 0;margin:.5px 0 0;width:.57142857em;height:.57142857em;border:none;border-bottom:1px solid #d4d4d5;border-right:1px solid #d4d4d5;z-index:2;-webkit-transition:background .1s ease;transition:background .1s ease}.ui.vertical.pointing.menu .item:after{position:absolute;top:50%;right:0;bottom:auto;left:auto;-webkit-transform:translateX(50%) translateY(-50%) rotate(45deg);transform:translateX(50%) translateY(-50%) rotate(45deg);margin:0 -.5px 0 0;border:none;border-top:1px solid #d4d4d5;border-right:1px solid #d4d4d5}.ui.pointing.menu .active.item:after{visibility:visible}.ui.pointing.menu .active.dropdown.item:after{visibility:hidden}.ui.pointing.menu .active.item .menu .active.item:after,.ui.pointing.menu .dropdown.active.item:after{display:none}.ui.pointing.menu .active.item:hover:after{background-color:#f2f2f2}.ui.pointing.menu .active.item:after{background-color:#f2f2f2}.ui.pointing.menu .active.item:hover:after{background-color:#f2f2f2}.ui.vertical.pointing.menu .active.item:hover:after{background-color:#f2f2f2}.ui.vertical.pointing.menu .active.item:after{background-color:#f2f2f2}.ui.vertical.pointing.menu .menu .active.item:after{background-color:#fff}.ui.attached.menu{top:0;bottom:0;border-radius:0;margin:0 -1px;width:calc(100% + 2px);max-width:calc(100% + 2px);-webkit-box-shadow:none;box-shadow:none}.ui.attached+.ui.attached.menu:not(.top){border-top:none}.ui[class*="top attached"].menu{bottom:0;margin-bottom:0;top:0;margin-top:1rem;border-radius:.28571429rem .28571429rem 0 0}.ui.menu[class*="top attached"]:first-child{margin-top:0}.ui[class*="bottom attached"].menu{bottom:0;margin-top:0;top:0;margin-bottom:1rem;-webkit-box-shadow:0 1px 2px 0 rgba(34,36,38,.15),none;box-shadow:0 1px 2px 0 rgba(34,36,38,.15),none;border-radius:0 0 .28571429rem .28571429rem}.ui[class*="bottom attached"].menu:last-child{margin-bottom:0}.ui.top.attached.menu>.item:first-child{border-radius:.28571429rem 0 0 0}.ui.bottom.attached.menu>.item:first-child{border-radius:0 0 0 .28571429rem}.ui.attached.menu:not(.tabular){border:1px solid #d4d4d5}.ui.attached.inverted.menu{border:none}.ui.attached.tabular.menu{margin-left:0;margin-right:0;width:100%}.ui.mini.menu{font-size:.78571429rem}.ui.mini.vertical.menu{width:9rem}.ui.tiny.menu{font-size:.85714286rem}.ui.tiny.vertical.menu{width:11rem}.ui.small.menu{font-size:.92857143rem}.ui.small.vertical.menu{width:13rem}.ui.menu{font-size:1rem}.ui.vertical.menu{width:15rem}.ui.large.menu{font-size:1.07142857rem}.ui.large.vertical.menu{width:18rem}.ui.huge.menu{font-size:1.21428571rem}.ui.huge.vertical.menu{width:22rem}.ui.big.menu{font-size:1.14285714rem}.ui.big.vertical.menu{width:20rem}.ui.massive.menu{font-size:1.28571429rem}.ui.massive.vertical.menu{width:25rem}/*! - * # Semantic UI 2.4.0 - Message - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */.ui.message{position:relative;min-height:1em;margin:1em 0;background:#f8f8f9;padding:1em 1.5em;line-height:1.4285em;color:rgba(0,0,0,.87);-webkit-transition:opacity .1s ease,color .1s ease,background .1s ease,-webkit-box-shadow .1s ease;transition:opacity .1s ease,color .1s ease,background .1s ease,-webkit-box-shadow .1s ease;transition:opacity .1s ease,color .1s ease,background .1s ease,box-shadow .1s ease;transition:opacity .1s ease,color .1s ease,background .1s ease,box-shadow .1s ease,-webkit-box-shadow .1s ease;border-radius:.28571429rem;-webkit-box-shadow:0 0 0 1px rgba(34,36,38,.22) inset,0 0 0 0 transparent;box-shadow:0 0 0 1px rgba(34,36,38,.22) inset,0 0 0 0 transparent}.ui.message:first-child{margin-top:0}.ui.message:last-child{margin-bottom:0}.ui.message .header{display:block;font-family:Lato,'Helvetica Neue',Arial,Helvetica,sans-serif;font-weight:700;margin:-.14285714em 0 0 0}.ui.message .header:not(.ui){font-size:1.14285714em}.ui.message p{opacity:.85;margin:.75em 0}.ui.message p:first-child{margin-top:0}.ui.message p:last-child{margin-bottom:0}.ui.message .header+p{margin-top:.25em}.ui.message .list:not(.ui){text-align:left;padding:0;opacity:.85;list-style-position:inside;margin:.5em 0 0}.ui.message .list:not(.ui):first-child{margin-top:0}.ui.message .list:not(.ui):last-child{margin-bottom:0}.ui.message .list:not(.ui) li{position:relative;list-style-type:none;margin:0 0 .3em 1em;padding:0}.ui.message .list:not(.ui) li:before{position:absolute;content:'•';left:-1em;height:100%;vertical-align:baseline}.ui.message .list:not(.ui) li:last-child{margin-bottom:0}.ui.message>.icon{margin-right:.6em}.ui.message>.close.icon{cursor:pointer;position:absolute;margin:0;top:.78575em;right:.5em;opacity:.7;-webkit-transition:opacity .1s ease;transition:opacity .1s ease}.ui.message>.close.icon:hover{opacity:1}.ui.message>:first-child{margin-top:0}.ui.message>:last-child{margin-bottom:0}.ui.dropdown .menu>.message{margin:0 -1px}.ui.visible.visible.visible.visible.message{display:block}.ui.icon.visible.visible.visible.visible.message{display:-webkit-box;display:-ms-flexbox;display:flex}.ui.hidden.hidden.hidden.hidden.message{display:none}.ui.compact.message{display:inline-block}.ui.compact.icon.message{display:-webkit-inline-box;display:-ms-inline-flexbox;display:inline-flex}.ui.attached.message{margin-bottom:-1px;border-radius:.28571429rem .28571429rem 0 0;-webkit-box-shadow:0 0 0 1px rgba(34,36,38,.15) inset;box-shadow:0 0 0 1px rgba(34,36,38,.15) inset;margin-left:-1px;margin-right:-1px}.ui.attached+.ui.attached.message:not(.top):not(.bottom){margin-top:-1px;border-radius:0}.ui.bottom.attached.message{margin-top:-1px;border-radius:0 0 .28571429rem .28571429rem;-webkit-box-shadow:0 0 0 1px rgba(34,36,38,.15) inset,0 1px 2px 0 rgba(34,36,38,.15);box-shadow:0 0 0 1px rgba(34,36,38,.15) inset,0 1px 2px 0 rgba(34,36,38,.15)}.ui.bottom.attached.message:not(:last-child){margin-bottom:1em}.ui.attached.icon.message{width:auto}.ui.icon.message{display:-webkit-box;display:-ms-flexbox;display:flex;width:100%;-webkit-box-align:center;-ms-flex-align:center;align-items:center}.ui.icon.message>.icon:not(.close){display:block;-webkit-box-flex:0;-ms-flex:0 0 auto;flex:0 0 auto;width:auto;line-height:1;vertical-align:middle;font-size:3em;opacity:.8}.ui.icon.message>.content{display:block;-webkit-box-flex:1;-ms-flex:1 1 auto;flex:1 1 auto;vertical-align:middle}.ui.icon.message .icon:not(.close)+.content{padding-left:0}.ui.icon.message .circular.icon{width:1em}.ui.floating.message{-webkit-box-shadow:0 0 0 1px rgba(34,36,38,.22) inset,0 2px 4px 0 rgba(34,36,38,.12),0 2px 10px 0 rgba(34,36,38,.15);box-shadow:0 0 0 1px rgba(34,36,38,.22) inset,0 2px 4px 0 rgba(34,36,38,.12),0 2px 10px 0 rgba(34,36,38,.15)}.ui.black.message{background-color:#1b1c1d;color:rgba(255,255,255,.9)}.ui.positive.message{background-color:#fcfff5;color:#2c662d}.ui.attached.positive.message,.ui.positive.message{-webkit-box-shadow:0 0 0 1px #a3c293 inset,0 0 0 0 transparent;box-shadow:0 0 0 1px #a3c293 inset,0 0 0 0 transparent}.ui.positive.message .header{color:#1a531b}.ui.negative.message{background-color:#fff6f6;color:#9f3a38}.ui.attached.negative.message,.ui.negative.message{-webkit-box-shadow:0 0 0 1px #e0b4b4 inset,0 0 0 0 transparent;box-shadow:0 0 0 1px #e0b4b4 inset,0 0 0 0 transparent}.ui.negative.message .header{color:#912d2b}.ui.info.message{background-color:#f8ffff;color:#276f86}.ui.attached.info.message,.ui.info.message{-webkit-box-shadow:0 0 0 1px #a9d5de inset,0 0 0 0 transparent;box-shadow:0 0 0 1px #a9d5de inset,0 0 0 0 transparent}.ui.info.message .header{color:#0e566c}.ui.warning.message{background-color:#fffaf3;color:#573a08}.ui.attached.warning.message,.ui.warning.message{-webkit-box-shadow:0 0 0 1px #c9ba9b inset,0 0 0 0 transparent;box-shadow:0 0 0 1px #c9ba9b inset,0 0 0 0 transparent}.ui.warning.message .header{color:#794b02}.ui.error.message{background-color:#fff6f6;color:#9f3a38}.ui.attached.error.message,.ui.error.message{-webkit-box-shadow:0 0 0 1px #e0b4b4 inset,0 0 0 0 transparent;box-shadow:0 0 0 1px #e0b4b4 inset,0 0 0 0 transparent}.ui.error.message .header{color:#912d2b}.ui.success.message{background-color:#fcfff5;color:#2c662d}.ui.attached.success.message,.ui.success.message{-webkit-box-shadow:0 0 0 1px #a3c293 inset,0 0 0 0 transparent;box-shadow:0 0 0 1px #a3c293 inset,0 0 0 0 transparent}.ui.success.message .header{color:#1a531b}.ui.black.message,.ui.inverted.message{background-color:#1b1c1d;color:rgba(255,255,255,.9)}.ui.red.message{background-color:#ffe8e6;color:#db2828;-webkit-box-shadow:0 0 0 1px #db2828 inset,0 0 0 0 transparent;box-shadow:0 0 0 1px #db2828 inset,0 0 0 0 transparent}.ui.red.message .header{color:#c82121}.ui.orange.message{background-color:#ffedde;color:#f2711c;-webkit-box-shadow:0 0 0 1px #f2711c inset,0 0 0 0 transparent;box-shadow:0 0 0 1px #f2711c inset,0 0 0 0 transparent}.ui.orange.message .header{color:#e7640d}.ui.yellow.message{background-color:#fff8db;color:#b58105;-webkit-box-shadow:0 0 0 1px #b58105 inset,0 0 0 0 transparent;box-shadow:0 0 0 1px #b58105 inset,0 0 0 0 transparent}.ui.yellow.message .header{color:#9c6f04}.ui.olive.message{background-color:#fbfdef;color:#8abc1e;-webkit-box-shadow:0 0 0 1px #8abc1e inset,0 0 0 0 transparent;box-shadow:0 0 0 1px #8abc1e inset,0 0 0 0 transparent}.ui.olive.message .header{color:#7aa61a}.ui.green.message{background-color:#e5f9e7;color:#1ebc30;-webkit-box-shadow:0 0 0 1px #1ebc30 inset,0 0 0 0 transparent;box-shadow:0 0 0 1px #1ebc30 inset,0 0 0 0 transparent}.ui.green.message .header{color:#1aa62a}.ui.teal.message{background-color:#e1f7f7;color:#10a3a3;-webkit-box-shadow:0 0 0 1px #10a3a3 inset,0 0 0 0 transparent;box-shadow:0 0 0 1px #10a3a3 inset,0 0 0 0 transparent}.ui.teal.message .header{color:#0e8c8c}.ui.blue.message{background-color:#dff0ff;color:#2185d0;-webkit-box-shadow:0 0 0 1px #2185d0 inset,0 0 0 0 transparent;box-shadow:0 0 0 1px #2185d0 inset,0 0 0 0 transparent}.ui.blue.message .header{color:#1e77ba}.ui.violet.message{background-color:#eae7ff;color:#6435c9;-webkit-box-shadow:0 0 0 1px #6435c9 inset,0 0 0 0 transparent;box-shadow:0 0 0 1px #6435c9 inset,0 0 0 0 transparent}.ui.violet.message .header{color:#5a30b5}.ui.purple.message{background-color:#f6e7ff;color:#a333c8;-webkit-box-shadow:0 0 0 1px #a333c8 inset,0 0 0 0 transparent;box-shadow:0 0 0 1px #a333c8 inset,0 0 0 0 transparent}.ui.purple.message .header{color:#922eb4}.ui.pink.message{background-color:#ffe3fb;color:#e03997;-webkit-box-shadow:0 0 0 1px #e03997 inset,0 0 0 0 transparent;box-shadow:0 0 0 1px #e03997 inset,0 0 0 0 transparent}.ui.pink.message .header{color:#dd238b}.ui.brown.message{background-color:#f1e2d3;color:#a5673f;-webkit-box-shadow:0 0 0 1px #a5673f inset,0 0 0 0 transparent;box-shadow:0 0 0 1px #a5673f inset,0 0 0 0 transparent}.ui.brown.message .header{color:#935b38}.ui.mini.message{font-size:.78571429em}.ui.tiny.message{font-size:.85714286em}.ui.small.message{font-size:.92857143em}.ui.message{font-size:1em}.ui.large.message{font-size:1.14285714em}.ui.big.message{font-size:1.28571429em}.ui.huge.message{font-size:1.42857143em}.ui.massive.message{font-size:1.71428571em}/*! - * # Semantic UI 2.4.0 - Table - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */.ui.table{width:100%;background:#fff;margin:1em 0;border:1px solid rgba(34,36,38,.15);-webkit-box-shadow:none;box-shadow:none;border-radius:.28571429rem;text-align:left;color:rgba(0,0,0,.87);border-collapse:separate;border-spacing:0}.ui.table:first-child{margin-top:0}.ui.table:last-child{margin-bottom:0}.ui.table td,.ui.table th{-webkit-transition:background .1s ease,color .1s ease;transition:background .1s ease,color .1s ease}.ui.table thead{-webkit-box-shadow:none;box-shadow:none}.ui.table thead th{cursor:auto;background:#f9fafb;text-align:inherit;color:rgba(0,0,0,.87);padding:.92857143em .78571429em;vertical-align:inherit;font-style:none;font-weight:700;text-transform:none;border-bottom:1px solid rgba(34,36,38,.1);border-left:none}.ui.table thead tr>th:first-child{border-left:none}.ui.table thead tr:first-child>th:first-child{border-radius:.28571429rem 0 0 0}.ui.table thead tr:first-child>th:last-child{border-radius:0 .28571429rem 0 0}.ui.table thead tr:first-child>th:only-child{border-radius:.28571429rem .28571429rem 0 0}.ui.table tfoot{-webkit-box-shadow:none;box-shadow:none}.ui.table tfoot th{cursor:auto;border-top:1px solid rgba(34,36,38,.15);background:#f9fafb;text-align:inherit;color:rgba(0,0,0,.87);padding:.78571429em .78571429em;vertical-align:middle;font-style:normal;font-weight:400;text-transform:none}.ui.table tfoot tr>th:first-child{border-left:none}.ui.table tfoot tr:first-child>th:first-child{border-radius:0 0 0 .28571429rem}.ui.table tfoot tr:first-child>th:last-child{border-radius:0 0 .28571429rem 0}.ui.table tfoot tr:first-child>th:only-child{border-radius:0 0 .28571429rem .28571429rem}.ui.table tr td{border-top:1px solid rgba(34,36,38,.1)}.ui.table tr:first-child td{border-top:none}.ui.table tbody+tbody tr:first-child td{border-top:1px solid rgba(34,36,38,.1)}.ui.table td{padding:.78571429em .78571429em;text-align:inherit}.ui.table>.icon{vertical-align:baseline}.ui.table>.icon:only-child{margin:0}.ui.table.segment{padding:0}.ui.table.segment:after{display:none}.ui.table.segment.stacked:after{display:block}@media only screen and (max-width:767px){.ui.table:not(.unstackable){width:100%}.ui.table:not(.unstackable) tbody,.ui.table:not(.unstackable) tr,.ui.table:not(.unstackable) tr>td,.ui.table:not(.unstackable) tr>th{width:auto!important;display:block!important}.ui.table:not(.unstackable){padding:0}.ui.table:not(.unstackable) thead{display:block}.ui.table:not(.unstackable) tfoot{display:block}.ui.table:not(.unstackable) tr{padding-top:1em;padding-bottom:1em;-webkit-box-shadow:0 -1px 0 0 rgba(0,0,0,.1) inset!important;box-shadow:0 -1px 0 0 rgba(0,0,0,.1) inset!important}.ui.table:not(.unstackable) tr>td,.ui.table:not(.unstackable) tr>th{background:0 0;border:none!important;padding:.25em .75em!important;-webkit-box-shadow:none!important;box-shadow:none!important}.ui.table:not(.unstackable) td:first-child,.ui.table:not(.unstackable) th:first-child{font-weight:700}.ui.definition.table:not(.unstackable) thead th:first-child{-webkit-box-shadow:none!important;box-shadow:none!important}}.ui.table td .image,.ui.table td .image img,.ui.table th .image,.ui.table th .image img{max-width:none}.ui.structured.table{border-collapse:collapse}.ui.structured.table thead th{border-left:none;border-right:none}.ui.structured.sortable.table thead th{border-left:1px solid rgba(34,36,38,.15);border-right:1px solid rgba(34,36,38,.15)}.ui.structured.basic.table th{border-left:none;border-right:none}.ui.structured.celled.table tr td,.ui.structured.celled.table tr th{border-left:1px solid rgba(34,36,38,.1);border-right:1px solid rgba(34,36,38,.1)}.ui.definition.table thead:not(.full-width) th:first-child{pointer-events:none;background:0 0;font-weight:400;color:rgba(0,0,0,.4);-webkit-box-shadow:-1px -1px 0 1px #fff;box-shadow:-1px -1px 0 1px #fff}.ui.definition.table tfoot:not(.full-width) th:first-child{pointer-events:none;background:0 0;font-weight:rgba(0,0,0,.4);color:normal;-webkit-box-shadow:1px 1px 0 1px #fff;box-shadow:1px 1px 0 1px #fff}.ui.celled.definition.table thead:not(.full-width) th:first-child{-webkit-box-shadow:0 -1px 0 1px #fff;box-shadow:0 -1px 0 1px #fff}.ui.celled.definition.table tfoot:not(.full-width) th:first-child{-webkit-box-shadow:0 1px 0 1px #fff;box-shadow:0 1px 0 1px #fff}.ui.definition.table tr td.definition,.ui.definition.table tr td:first-child:not(.ignored){background:rgba(0,0,0,.03);font-weight:700;color:rgba(0,0,0,.95);text-transform:'';-webkit-box-shadow:'';box-shadow:'';text-align:'';font-size:1em;padding-left:'';padding-right:''}.ui.definition.table thead:not(.full-width) th:nth-child(2){border-left:1px solid rgba(34,36,38,.15)}.ui.definition.table tfoot:not(.full-width) th:nth-child(2){border-left:1px solid rgba(34,36,38,.15)}.ui.definition.table td:nth-child(2){border-left:1px solid rgba(34,36,38,.15)}.ui.table td.positive,.ui.table tr.positive{-webkit-box-shadow:0 0 0 #a3c293 inset;box-shadow:0 0 0 #a3c293 inset}.ui.table td.positive,.ui.table tr.positive{background:#fcfff5!important;color:#2c662d!important}.ui.table td.negative,.ui.table tr.negative{-webkit-box-shadow:0 0 0 #e0b4b4 inset;box-shadow:0 0 0 #e0b4b4 inset}.ui.table td.negative,.ui.table tr.negative{background:#fff6f6!important;color:#9f3a38!important}.ui.table td.error,.ui.table tr.error{-webkit-box-shadow:0 0 0 #e0b4b4 inset;box-shadow:0 0 0 #e0b4b4 inset}.ui.table td.error,.ui.table tr.error{background:#fff6f6!important;color:#9f3a38!important}.ui.table td.warning,.ui.table tr.warning{-webkit-box-shadow:0 0 0 #c9ba9b inset;box-shadow:0 0 0 #c9ba9b inset}.ui.table td.warning,.ui.table tr.warning{background:#fffaf3!important;color:#573a08!important}.ui.table td.active,.ui.table tr.active{-webkit-box-shadow:0 0 0 rgba(0,0,0,.87) inset;box-shadow:0 0 0 rgba(0,0,0,.87) inset}.ui.table td.active,.ui.table tr.active{background:#e0e0e0!important;color:rgba(0,0,0,.87)!important}.ui.table tr td.disabled,.ui.table tr.disabled td,.ui.table tr.disabled:hover,.ui.table tr:hover td.disabled{pointer-events:none;color:rgba(40,40,40,.3)}@media only screen and (max-width:991px){.ui[class*="tablet stackable"].table,.ui[class*="tablet stackable"].table tbody,.ui[class*="tablet stackable"].table tr,.ui[class*="tablet stackable"].table tr>td,.ui[class*="tablet stackable"].table tr>th{width:100%!important;display:block!important}.ui[class*="tablet stackable"].table{padding:0}.ui[class*="tablet stackable"].table thead{display:block}.ui[class*="tablet stackable"].table tfoot{display:block}.ui[class*="tablet stackable"].table tr{padding-top:1em;padding-bottom:1em;-webkit-box-shadow:0 -1px 0 0 rgba(0,0,0,.1) inset!important;box-shadow:0 -1px 0 0 rgba(0,0,0,.1) inset!important}.ui[class*="tablet stackable"].table tr>td,.ui[class*="tablet stackable"].table tr>th{background:0 0;border:none!important;padding:.25em .75em;-webkit-box-shadow:none!important;box-shadow:none!important}.ui.definition[class*="tablet stackable"].table thead th:first-child{-webkit-box-shadow:none!important;box-shadow:none!important}}.ui.table [class*="left aligned"],.ui.table[class*="left aligned"]{text-align:left}.ui.table [class*="center aligned"],.ui.table[class*="center aligned"]{text-align:center}.ui.table [class*="right aligned"],.ui.table[class*="right aligned"]{text-align:right}.ui.table [class*="top aligned"],.ui.table[class*="top aligned"]{vertical-align:top}.ui.table [class*="middle aligned"],.ui.table[class*="middle aligned"]{vertical-align:middle}.ui.table [class*="bottom aligned"],.ui.table[class*="bottom aligned"]{vertical-align:bottom}.ui.table td.collapsing,.ui.table th.collapsing{width:1px;white-space:nowrap}.ui.fixed.table{table-layout:fixed}.ui.fixed.table td,.ui.fixed.table th{overflow:hidden;text-overflow:ellipsis}.ui.selectable.table tbody tr:hover,.ui.table tbody tr td.selectable:hover{background:rgba(0,0,0,.05)!important;color:rgba(0,0,0,.95)!important}.ui.inverted.table tbody tr td.selectable:hover,.ui.selectable.inverted.table tbody tr:hover{background:rgba(255,255,255,.08)!important;color:#fff!important}.ui.table tbody tr td.selectable{padding:0}.ui.table tbody tr td.selectable>a:not(.ui){display:block;color:inherit;padding:.78571429em .78571429em}.ui.selectable.table tr.error:hover,.ui.selectable.table tr:hover td.error,.ui.table tr td.selectable.error:hover{background:#ffe7e7!important;color:#943634!important}.ui.selectable.table tr.warning:hover,.ui.selectable.table tr:hover td.warning,.ui.table tr td.selectable.warning:hover{background:#fff4e4!important;color:#493107!important}.ui.selectable.table tr.active:hover,.ui.selectable.table tr:hover td.active,.ui.table tr td.selectable.active:hover{background:#e0e0e0!important;color:rgba(0,0,0,.87)!important}.ui.selectable.table tr.positive:hover,.ui.selectable.table tr:hover td.positive,.ui.table tr td.selectable.positive:hover{background:#f7ffe6!important;color:#275b28!important}.ui.selectable.table tr.negative:hover,.ui.selectable.table tr:hover td.negative,.ui.table tr td.selectable.negative:hover{background:#ffe7e7!important;color:#943634!important}.ui.attached.table{top:0;bottom:0;border-radius:0;margin:0 -1px;width:calc(100% + 2px);max-width:calc(100% + 2px);-webkit-box-shadow:none;box-shadow:none;border:1px solid #d4d4d5}.ui.attached+.ui.attached.table:not(.top){border-top:none}.ui[class*="top attached"].table{bottom:0;margin-bottom:0;top:0;margin-top:1em;border-radius:.28571429rem .28571429rem 0 0}.ui.table[class*="top attached"]:first-child{margin-top:0}.ui[class*="bottom attached"].table{bottom:0;margin-top:0;top:0;margin-bottom:1em;-webkit-box-shadow:none,none;box-shadow:none,none;border-radius:0 0 .28571429rem .28571429rem}.ui[class*="bottom attached"].table:last-child{margin-bottom:0}.ui.striped.table tbody tr:nth-child(2n),.ui.striped.table>tr:nth-child(2n){background-color:rgba(0,0,50,.02)}.ui.inverted.striped.table tbody tr:nth-child(2n),.ui.inverted.striped.table>tr:nth-child(2n){background-color:rgba(255,255,255,.05)}.ui.striped.selectable.selectable.selectable.table tbody tr.active:hover{background:#efefef!important;color:rgba(0,0,0,.95)!important}.ui.table [class*="single line"],.ui.table[class*="single line"]{white-space:nowrap}.ui.table [class*="single line"],.ui.table[class*="single line"]{white-space:nowrap}.ui.red.table{border-top:.2em solid #db2828}.ui.inverted.red.table{background-color:#db2828!important;color:#fff!important}.ui.orange.table{border-top:.2em solid #f2711c}.ui.inverted.orange.table{background-color:#f2711c!important;color:#fff!important}.ui.yellow.table{border-top:.2em solid #fbbd08}.ui.inverted.yellow.table{background-color:#fbbd08!important;color:#fff!important}.ui.olive.table{border-top:.2em solid #b5cc18}.ui.inverted.olive.table{background-color:#b5cc18!important;color:#fff!important}.ui.green.table{border-top:.2em solid #21ba45}.ui.inverted.green.table{background-color:#21ba45!important;color:#fff!important}.ui.teal.table{border-top:.2em solid #00b5ad}.ui.inverted.teal.table{background-color:#00b5ad!important;color:#fff!important}.ui.blue.table{border-top:.2em solid #2185d0}.ui.inverted.blue.table{background-color:#2185d0!important;color:#fff!important}.ui.violet.table{border-top:.2em solid #6435c9}.ui.inverted.violet.table{background-color:#6435c9!important;color:#fff!important}.ui.purple.table{border-top:.2em solid #a333c8}.ui.inverted.purple.table{background-color:#a333c8!important;color:#fff!important}.ui.pink.table{border-top:.2em solid #e03997}.ui.inverted.pink.table{background-color:#e03997!important;color:#fff!important}.ui.brown.table{border-top:.2em solid #a5673f}.ui.inverted.brown.table{background-color:#a5673f!important;color:#fff!important}.ui.grey.table{border-top:.2em solid #767676}.ui.inverted.grey.table{background-color:#767676!important;color:#fff!important}.ui.black.table{border-top:.2em solid #1b1c1d}.ui.inverted.black.table{background-color:#1b1c1d!important;color:#fff!important}.ui.one.column.table td{width:100%}.ui.two.column.table td{width:50%}.ui.three.column.table td{width:33.33333333%}.ui.four.column.table td{width:25%}.ui.five.column.table td{width:20%}.ui.six.column.table td{width:16.66666667%}.ui.seven.column.table td{width:14.28571429%}.ui.eight.column.table td{width:12.5%}.ui.nine.column.table td{width:11.11111111%}.ui.ten.column.table td{width:10%}.ui.eleven.column.table td{width:9.09090909%}.ui.twelve.column.table td{width:8.33333333%}.ui.thirteen.column.table td{width:7.69230769%}.ui.fourteen.column.table td{width:7.14285714%}.ui.fifteen.column.table td{width:6.66666667%}.ui.sixteen.column.table td{width:6.25%}.ui.table td.one.wide,.ui.table th.one.wide{width:6.25%}.ui.table td.two.wide,.ui.table th.two.wide{width:12.5%}.ui.table td.three.wide,.ui.table th.three.wide{width:18.75%}.ui.table td.four.wide,.ui.table th.four.wide{width:25%}.ui.table td.five.wide,.ui.table th.five.wide{width:31.25%}.ui.table td.six.wide,.ui.table th.six.wide{width:37.5%}.ui.table td.seven.wide,.ui.table th.seven.wide{width:43.75%}.ui.table td.eight.wide,.ui.table th.eight.wide{width:50%}.ui.table td.nine.wide,.ui.table th.nine.wide{width:56.25%}.ui.table td.ten.wide,.ui.table th.ten.wide{width:62.5%}.ui.table td.eleven.wide,.ui.table th.eleven.wide{width:68.75%}.ui.table td.twelve.wide,.ui.table th.twelve.wide{width:75%}.ui.table td.thirteen.wide,.ui.table th.thirteen.wide{width:81.25%}.ui.table td.fourteen.wide,.ui.table th.fourteen.wide{width:87.5%}.ui.table td.fifteen.wide,.ui.table th.fifteen.wide{width:93.75%}.ui.table td.sixteen.wide,.ui.table th.sixteen.wide{width:100%}.ui.sortable.table thead th{cursor:pointer;white-space:nowrap;border-left:1px solid rgba(34,36,38,.15);color:rgba(0,0,0,.87)}.ui.sortable.table thead th:first-child{border-left:none}.ui.sortable.table thead th.sorted,.ui.sortable.table thead th.sorted:hover{-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none}.ui.sortable.table thead th:after{display:none;font-style:normal;font-weight:400;text-decoration:inherit;content:'';height:1em;width:auto;opacity:.8;margin:0 0 0 .5em;font-family:Icons}.ui.sortable.table thead th.ascending:after{content:'\f0d8'}.ui.sortable.table thead th.descending:after{content:'\f0d7'}.ui.sortable.table th.disabled:hover{cursor:auto;color:rgba(40,40,40,.3)}.ui.sortable.table thead th:hover{background:rgba(0,0,0,.05);color:rgba(0,0,0,.8)}.ui.sortable.table thead th.sorted{background:rgba(0,0,0,.05);color:rgba(0,0,0,.95)}.ui.sortable.table thead th.sorted:after{display:inline-block}.ui.sortable.table thead th.sorted:hover{background:rgba(0,0,0,.05);color:rgba(0,0,0,.95)}.ui.inverted.sortable.table thead th.sorted{background:rgba(255,255,255,.15) -webkit-gradient(linear,left top,left bottom,from(transparent),to(rgba(0,0,0,.05)));background:rgba(255,255,255,.15) -webkit-linear-gradient(transparent,rgba(0,0,0,.05));background:rgba(255,255,255,.15) linear-gradient(transparent,rgba(0,0,0,.05));color:#fff}.ui.inverted.sortable.table thead th:hover{background:rgba(255,255,255,.08) -webkit-gradient(linear,left top,left bottom,from(transparent),to(rgba(0,0,0,.05)));background:rgba(255,255,255,.08) -webkit-linear-gradient(transparent,rgba(0,0,0,.05));background:rgba(255,255,255,.08) linear-gradient(transparent,rgba(0,0,0,.05));color:#fff}.ui.inverted.sortable.table thead th{border-left-color:transparent;border-right-color:transparent}.ui.inverted.table{background:#333;color:rgba(255,255,255,.9);border:none}.ui.inverted.table th{background-color:rgba(0,0,0,.15);border-color:rgba(255,255,255,.1)!important;color:rgba(255,255,255,.9)!important}.ui.inverted.table tr td{border-color:rgba(255,255,255,.1)!important}.ui.inverted.table tr td.disabled,.ui.inverted.table tr.disabled td,.ui.inverted.table tr.disabled:hover td,.ui.inverted.table tr:hover td.disabled{pointer-events:none;color:rgba(225,225,225,.3)}.ui.inverted.definition.table tfoot:not(.full-width) th:first-child,.ui.inverted.definition.table thead:not(.full-width) th:first-child{background:#fff}.ui.inverted.definition.table tr td:first-child{background:rgba(255,255,255,.02);color:#fff}.ui.collapsing.table{width:auto}.ui.basic.table{background:0 0;border:1px solid rgba(34,36,38,.15);-webkit-box-shadow:none;box-shadow:none}.ui.basic.table tfoot,.ui.basic.table thead{-webkit-box-shadow:none;box-shadow:none}.ui.basic.table th{background:0 0;border-left:none}.ui.basic.table tbody tr{border-bottom:1px solid rgba(0,0,0,.1)}.ui.basic.table td{background:0 0}.ui.basic.striped.table tbody tr:nth-child(2n){background-color:rgba(0,0,0,.05)!important}.ui[class*="very basic"].table{border:none}.ui[class*="very basic"].table:not(.sortable):not(.striped) td,.ui[class*="very basic"].table:not(.sortable):not(.striped) th{padding:''}.ui[class*="very basic"].table:not(.sortable):not(.striped) td:first-child,.ui[class*="very basic"].table:not(.sortable):not(.striped) th:first-child{padding-left:0}.ui[class*="very basic"].table:not(.sortable):not(.striped) td:last-child,.ui[class*="very basic"].table:not(.sortable):not(.striped) th:last-child{padding-right:0}.ui[class*="very basic"].table:not(.sortable):not(.striped) thead tr:first-child th{padding-top:0}.ui.celled.table tr td,.ui.celled.table tr th{border-left:1px solid rgba(34,36,38,.1)}.ui.celled.table tr td:first-child,.ui.celled.table tr th:first-child{border-left:none}.ui.padded.table th{padding-left:1em;padding-right:1em}.ui.padded.table td,.ui.padded.table th{padding:1em 1em}.ui[class*="very padded"].table th{padding-left:1.5em;padding-right:1.5em}.ui[class*="very padded"].table td{padding:1.5em 1.5em}.ui.compact.table th{padding-left:.7em;padding-right:.7em}.ui.compact.table td{padding:.5em .7em}.ui[class*="very compact"].table th{padding-left:.6em;padding-right:.6em}.ui[class*="very compact"].table td{padding:.4em .6em}.ui.small.table{font-size:.9em}.ui.table{font-size:1em}.ui.large.table{font-size:1.1em}/*! - * # Semantic UI 2.4.0 - Ad - * http://github.com/semantic-org/semantic-ui/ - * - * - * Copyright 2013 Contributors - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */.ui.ad{display:block;overflow:hidden;margin:1em 0}.ui.ad:first-child{margin:0}.ui.ad:last-child{margin:0}.ui.ad iframe{margin:0;padding:0;border:none;overflow:hidden}.ui.leaderboard.ad{width:728px;height:90px}.ui[class*="medium rectangle"].ad{width:300px;height:250px}.ui[class*="large rectangle"].ad{width:336px;height:280px}.ui[class*="half page"].ad{width:300px;height:600px}.ui.square.ad{width:250px;height:250px}.ui[class*="small square"].ad{width:200px;height:200px}.ui[class*="small rectangle"].ad{width:180px;height:150px}.ui[class*="vertical rectangle"].ad{width:240px;height:400px}.ui.button.ad{width:120px;height:90px}.ui[class*="square button"].ad{width:125px;height:125px}.ui[class*="small button"].ad{width:120px;height:60px}.ui.skyscraper.ad{width:120px;height:600px}.ui[class*="wide skyscraper"].ad{width:160px}.ui.banner.ad{width:468px;height:60px}.ui[class*="vertical banner"].ad{width:120px;height:240px}.ui[class*="top banner"].ad{width:930px;height:180px}.ui[class*="half banner"].ad{width:234px;height:60px}.ui[class*="large leaderboard"].ad{width:970px;height:90px}.ui.billboard.ad{width:970px;height:250px}.ui.panorama.ad{width:980px;height:120px}.ui.netboard.ad{width:580px;height:400px}.ui[class*="large mobile banner"].ad{width:320px;height:100px}.ui[class*="mobile leaderboard"].ad{width:320px;height:50px}.ui.mobile.ad{display:none}@media only screen and (max-width:767px){.ui.mobile.ad{display:block}}.ui.centered.ad{margin-left:auto;margin-right:auto}.ui.test.ad{position:relative;background:#545454}.ui.test.ad:after{position:absolute;top:50%;left:50%;width:100%;text-align:center;-webkit-transform:translateX(-50%) translateY(-50%);transform:translateX(-50%) translateY(-50%);content:'Ad';color:#fff;font-size:1em;font-weight:700}.ui.mobile.test.ad:after{font-size:.85714286em}.ui.test.ad[data-text]:after{content:attr(data-text)}/*! - * # Semantic UI 2.4.0 - Item - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */.ui.card,.ui.cards>.card{max-width:100%;position:relative;display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column;width:290px;min-height:0;background:#fff;padding:0;border:none;border-radius:.28571429rem;-webkit-box-shadow:0 1px 3px 0 #d4d4d5,0 0 0 1px #d4d4d5;box-shadow:0 1px 3px 0 #d4d4d5,0 0 0 1px #d4d4d5;-webkit-transition:-webkit-box-shadow .1s ease,-webkit-transform .1s ease;transition:-webkit-box-shadow .1s ease,-webkit-transform .1s ease;transition:box-shadow .1s ease,transform .1s ease;transition:box-shadow .1s ease,transform .1s ease,-webkit-box-shadow .1s ease,-webkit-transform .1s ease;z-index:''}.ui.card{margin:1em 0}.ui.card a,.ui.cards>.card a{cursor:pointer}.ui.card:first-child{margin-top:0}.ui.card:last-child{margin-bottom:0}.ui.cards{display:-webkit-box;display:-ms-flexbox;display:flex;margin:-.875em -.5em;-ms-flex-wrap:wrap;flex-wrap:wrap}.ui.cards>.card{display:-webkit-box;display:-ms-flexbox;display:flex;margin:.875em .5em;float:none}.ui.card:after,.ui.cards:after{display:block;content:' ';height:0;clear:both;overflow:hidden;visibility:hidden}.ui.cards~.ui.cards{margin-top:.875em}.ui.card>:first-child,.ui.cards>.card>:first-child{border-radius:.28571429rem .28571429rem 0 0!important;border-top:none!important}.ui.card>:last-child,.ui.cards>.card>:last-child{border-radius:0 0 .28571429rem .28571429rem!important}.ui.card>:only-child,.ui.cards>.card>:only-child{border-radius:.28571429rem!important}.ui.card>.image,.ui.cards>.card>.image{position:relative;display:block;-webkit-box-flex:0;-ms-flex:0 0 auto;flex:0 0 auto;padding:0;background:rgba(0,0,0,.05)}.ui.card>.image>img,.ui.cards>.card>.image>img{display:block;width:100%;height:auto;border-radius:inherit}.ui.card>.image:not(.ui)>img,.ui.cards>.card>.image:not(.ui)>img{border:none}.ui.card>.content,.ui.cards>.card>.content{-webkit-box-flex:1;-ms-flex-positive:1;flex-grow:1;border:none;border-top:1px solid rgba(34,36,38,.1);background:0 0;margin:0;padding:1em 1em;-webkit-box-shadow:none;box-shadow:none;font-size:1em;border-radius:0}.ui.card>.content:after,.ui.cards>.card>.content:after{display:block;content:' ';height:0;clear:both;overflow:hidden;visibility:hidden}.ui.card>.content>.header,.ui.cards>.card>.content>.header{display:block;margin:'';font-family:Lato,'Helvetica Neue',Arial,Helvetica,sans-serif;color:rgba(0,0,0,.85)}.ui.card>.content>.header:not(.ui),.ui.cards>.card>.content>.header:not(.ui){font-weight:700;font-size:1.28571429em;margin-top:-.21425em;line-height:1.28571429em}.ui.card>.content>.header+.description,.ui.card>.content>.meta+.description,.ui.cards>.card>.content>.header+.description,.ui.cards>.card>.content>.meta+.description{margin-top:.5em}.ui.card [class*="left floated"],.ui.cards>.card [class*="left floated"]{float:left}.ui.card [class*="right floated"],.ui.cards>.card [class*="right floated"]{float:right}.ui.card [class*="left aligned"],.ui.cards>.card [class*="left aligned"]{text-align:left}.ui.card [class*="center aligned"],.ui.cards>.card [class*="center aligned"]{text-align:center}.ui.card [class*="right aligned"],.ui.cards>.card [class*="right aligned"]{text-align:right}.ui.card .content img,.ui.cards>.card .content img{display:inline-block;vertical-align:middle;width:''}.ui.card .avatar img,.ui.card img.avatar,.ui.cards>.card .avatar img,.ui.cards>.card img.avatar{width:2em;height:2em;border-radius:500rem}.ui.card>.content>.description,.ui.cards>.card>.content>.description{clear:both;color:rgba(0,0,0,.68)}.ui.card>.content p,.ui.cards>.card>.content p{margin:0 0 .5em}.ui.card>.content p:last-child,.ui.cards>.card>.content p:last-child{margin-bottom:0}.ui.card .meta,.ui.cards>.card .meta{font-size:1em;color:rgba(0,0,0,.4)}.ui.card .meta *,.ui.cards>.card .meta *{margin-right:.3em}.ui.card .meta :last-child,.ui.cards>.card .meta :last-child{margin-right:0}.ui.card .meta [class*="right floated"],.ui.cards>.card .meta [class*="right floated"]{margin-right:0;margin-left:.3em}.ui.card>.content a:not(.ui),.ui.cards>.card>.content a:not(.ui){color:'';-webkit-transition:color .1s ease;transition:color .1s ease}.ui.card>.content a:not(.ui):hover,.ui.cards>.card>.content a:not(.ui):hover{color:''}.ui.card>.content>a.header,.ui.cards>.card>.content>a.header{color:rgba(0,0,0,.85)}.ui.card>.content>a.header:hover,.ui.cards>.card>.content>a.header:hover{color:#1e70bf}.ui.card .meta>a:not(.ui),.ui.cards>.card .meta>a:not(.ui){color:rgba(0,0,0,.4)}.ui.card .meta>a:not(.ui):hover,.ui.cards>.card .meta>a:not(.ui):hover{color:rgba(0,0,0,.87)}.ui.card>.button,.ui.card>.buttons,.ui.cards>.card>.button,.ui.cards>.card>.buttons{margin:0 -1px;width:calc(100% + 2px)}.ui.card .dimmer,.ui.cards>.card .dimmer{background-color:'';z-index:10}.ui.card>.content .star.icon,.ui.cards>.card>.content .star.icon{cursor:pointer;opacity:.75;-webkit-transition:color .1s ease;transition:color .1s ease}.ui.card>.content .star.icon:hover,.ui.cards>.card>.content .star.icon:hover{opacity:1;color:#ffb70a}.ui.card>.content .active.star.icon,.ui.cards>.card>.content .active.star.icon{color:#ffe623}.ui.card>.content .like.icon,.ui.cards>.card>.content .like.icon{cursor:pointer;opacity:.75;-webkit-transition:color .1s ease;transition:color .1s ease}.ui.card>.content .like.icon:hover,.ui.cards>.card>.content .like.icon:hover{opacity:1;color:#ff2733}.ui.card>.content .active.like.icon,.ui.cards>.card>.content .active.like.icon{color:#ff2733}.ui.card>.extra,.ui.cards>.card>.extra{max-width:100%;min-height:0!important;-webkit-box-flex:0;-ms-flex-positive:0;flex-grow:0;border-top:1px solid rgba(0,0,0,.05)!important;position:static;background:0 0;width:auto;margin:0 0;padding:.75em 1em;top:0;left:0;color:rgba(0,0,0,.4);-webkit-box-shadow:none;box-shadow:none;-webkit-transition:color .1s ease;transition:color .1s ease}.ui.card>.extra a:not(.ui),.ui.cards>.card>.extra a:not(.ui){color:rgba(0,0,0,.4)}.ui.card>.extra a:not(.ui):hover,.ui.cards>.card>.extra a:not(.ui):hover{color:#1e70bf}.ui.raised.card,.ui.raised.cards>.card{-webkit-box-shadow:0 0 0 1px #d4d4d5,0 2px 4px 0 rgba(34,36,38,.12),0 2px 10px 0 rgba(34,36,38,.15);box-shadow:0 0 0 1px #d4d4d5,0 2px 4px 0 rgba(34,36,38,.12),0 2px 10px 0 rgba(34,36,38,.15)}.ui.link.cards .raised.card:hover,.ui.link.raised.card:hover,.ui.raised.cards a.card:hover,a.ui.raised.card:hover{-webkit-box-shadow:0 0 0 1px #d4d4d5,0 2px 4px 0 rgba(34,36,38,.15),0 2px 10px 0 rgba(34,36,38,.25);box-shadow:0 0 0 1px #d4d4d5,0 2px 4px 0 rgba(34,36,38,.15),0 2px 10px 0 rgba(34,36,38,.25)}.ui.raised.card,.ui.raised.cards>.card{-webkit-box-shadow:0 0 0 1px #d4d4d5,0 2px 4px 0 rgba(34,36,38,.12),0 2px 10px 0 rgba(34,36,38,.15);box-shadow:0 0 0 1px #d4d4d5,0 2px 4px 0 rgba(34,36,38,.12),0 2px 10px 0 rgba(34,36,38,.15)}.ui.centered.cards{-webkit-box-pack:center;-ms-flex-pack:center;justify-content:center}.ui.centered.card{margin-left:auto;margin-right:auto}.ui.fluid.card{width:100%;max-width:9999px}.ui.cards a.card,.ui.link.card,.ui.link.cards .card,a.ui.card{-webkit-transform:none;transform:none}.ui.cards a.card:hover,.ui.link.card:hover,.ui.link.cards .card:hover,a.ui.card:hover{cursor:pointer;z-index:5;background:#fff;border:none;-webkit-box-shadow:0 1px 3px 0 #bcbdbd,0 0 0 1px #d4d4d5;box-shadow:0 1px 3px 0 #bcbdbd,0 0 0 1px #d4d4d5;-webkit-transform:translateY(-3px);transform:translateY(-3px)}.ui.cards>.red.card,.ui.red.card,.ui.red.cards>.card{-webkit-box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #db2828,0 1px 3px 0 #d4d4d5;box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #db2828,0 1px 3px 0 #d4d4d5}.ui.cards>.red.card:hover,.ui.red.card:hover,.ui.red.cards>.card:hover{-webkit-box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #d01919,0 1px 3px 0 #bcbdbd;box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #d01919,0 1px 3px 0 #bcbdbd}.ui.cards>.orange.card,.ui.orange.card,.ui.orange.cards>.card{-webkit-box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #f2711c,0 1px 3px 0 #d4d4d5;box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #f2711c,0 1px 3px 0 #d4d4d5}.ui.cards>.orange.card:hover,.ui.orange.card:hover,.ui.orange.cards>.card:hover{-webkit-box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #f26202,0 1px 3px 0 #bcbdbd;box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #f26202,0 1px 3px 0 #bcbdbd}.ui.cards>.yellow.card,.ui.yellow.card,.ui.yellow.cards>.card{-webkit-box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #fbbd08,0 1px 3px 0 #d4d4d5;box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #fbbd08,0 1px 3px 0 #d4d4d5}.ui.cards>.yellow.card:hover,.ui.yellow.card:hover,.ui.yellow.cards>.card:hover{-webkit-box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #eaae00,0 1px 3px 0 #bcbdbd;box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #eaae00,0 1px 3px 0 #bcbdbd}.ui.cards>.olive.card,.ui.olive.card,.ui.olive.cards>.card{-webkit-box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #b5cc18,0 1px 3px 0 #d4d4d5;box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #b5cc18,0 1px 3px 0 #d4d4d5}.ui.cards>.olive.card:hover,.ui.olive.card:hover,.ui.olive.cards>.card:hover{-webkit-box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #a7bd0d,0 1px 3px 0 #bcbdbd;box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #a7bd0d,0 1px 3px 0 #bcbdbd}.ui.cards>.green.card,.ui.green.card,.ui.green.cards>.card{-webkit-box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #21ba45,0 1px 3px 0 #d4d4d5;box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #21ba45,0 1px 3px 0 #d4d4d5}.ui.cards>.green.card:hover,.ui.green.card:hover,.ui.green.cards>.card:hover{-webkit-box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #16ab39,0 1px 3px 0 #bcbdbd;box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #16ab39,0 1px 3px 0 #bcbdbd}.ui.cards>.teal.card,.ui.teal.card,.ui.teal.cards>.card{-webkit-box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #00b5ad,0 1px 3px 0 #d4d4d5;box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #00b5ad,0 1px 3px 0 #d4d4d5}.ui.cards>.teal.card:hover,.ui.teal.card:hover,.ui.teal.cards>.card:hover{-webkit-box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #009c95,0 1px 3px 0 #bcbdbd;box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #009c95,0 1px 3px 0 #bcbdbd}.ui.blue.card,.ui.blue.cards>.card,.ui.cards>.blue.card{-webkit-box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #2185d0,0 1px 3px 0 #d4d4d5;box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #2185d0,0 1px 3px 0 #d4d4d5}.ui.blue.card:hover,.ui.blue.cards>.card:hover,.ui.cards>.blue.card:hover{-webkit-box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #1678c2,0 1px 3px 0 #bcbdbd;box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #1678c2,0 1px 3px 0 #bcbdbd}.ui.cards>.violet.card,.ui.violet.card,.ui.violet.cards>.card{-webkit-box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #6435c9,0 1px 3px 0 #d4d4d5;box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #6435c9,0 1px 3px 0 #d4d4d5}.ui.cards>.violet.card:hover,.ui.violet.card:hover,.ui.violet.cards>.card:hover{-webkit-box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #5829bb,0 1px 3px 0 #bcbdbd;box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #5829bb,0 1px 3px 0 #bcbdbd}.ui.cards>.purple.card,.ui.purple.card,.ui.purple.cards>.card{-webkit-box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #a333c8,0 1px 3px 0 #d4d4d5;box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #a333c8,0 1px 3px 0 #d4d4d5}.ui.cards>.purple.card:hover,.ui.purple.card:hover,.ui.purple.cards>.card:hover{-webkit-box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #9627ba,0 1px 3px 0 #bcbdbd;box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #9627ba,0 1px 3px 0 #bcbdbd}.ui.cards>.pink.card,.ui.pink.card,.ui.pink.cards>.card{-webkit-box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #e03997,0 1px 3px 0 #d4d4d5;box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #e03997,0 1px 3px 0 #d4d4d5}.ui.cards>.pink.card:hover,.ui.pink.card:hover,.ui.pink.cards>.card:hover{-webkit-box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #e61a8d,0 1px 3px 0 #bcbdbd;box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #e61a8d,0 1px 3px 0 #bcbdbd}.ui.brown.card,.ui.brown.cards>.card,.ui.cards>.brown.card{-webkit-box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #a5673f,0 1px 3px 0 #d4d4d5;box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #a5673f,0 1px 3px 0 #d4d4d5}.ui.brown.card:hover,.ui.brown.cards>.card:hover,.ui.cards>.brown.card:hover{-webkit-box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #975b33,0 1px 3px 0 #bcbdbd;box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #975b33,0 1px 3px 0 #bcbdbd}.ui.cards>.grey.card,.ui.grey.card,.ui.grey.cards>.card{-webkit-box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #767676,0 1px 3px 0 #d4d4d5;box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #767676,0 1px 3px 0 #d4d4d5}.ui.cards>.grey.card:hover,.ui.grey.card:hover,.ui.grey.cards>.card:hover{-webkit-box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #838383,0 1px 3px 0 #bcbdbd;box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #838383,0 1px 3px 0 #bcbdbd}.ui.black.card,.ui.black.cards>.card,.ui.cards>.black.card{-webkit-box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #1b1c1d,0 1px 3px 0 #d4d4d5;box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #1b1c1d,0 1px 3px 0 #d4d4d5}.ui.black.card:hover,.ui.black.cards>.card:hover,.ui.cards>.black.card:hover{-webkit-box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #27292a,0 1px 3px 0 #bcbdbd;box-shadow:0 0 0 1px #d4d4d5,0 2px 0 0 #27292a,0 1px 3px 0 #bcbdbd}.ui.one.cards{margin-left:0;margin-right:0}.ui.one.cards>.card{width:100%}.ui.two.cards{margin-left:-1em;margin-right:-1em}.ui.two.cards>.card{width:calc(50% - 2em);margin-left:1em;margin-right:1em}.ui.three.cards{margin-left:-1em;margin-right:-1em}.ui.three.cards>.card{width:calc(33.33333333% - 2em);margin-left:1em;margin-right:1em}.ui.four.cards{margin-left:-.75em;margin-right:-.75em}.ui.four.cards>.card{width:calc(25% - 1.5em);margin-left:.75em;margin-right:.75em}.ui.five.cards{margin-left:-.75em;margin-right:-.75em}.ui.five.cards>.card{width:calc(20% - 1.5em);margin-left:.75em;margin-right:.75em}.ui.six.cards{margin-left:-.75em;margin-right:-.75em}.ui.six.cards>.card{width:calc(16.66666667% - 1.5em);margin-left:.75em;margin-right:.75em}.ui.seven.cards{margin-left:-.5em;margin-right:-.5em}.ui.seven.cards>.card{width:calc(14.28571429% - 1em);margin-left:.5em;margin-right:.5em}.ui.eight.cards{margin-left:-.5em;margin-right:-.5em}.ui.eight.cards>.card{width:calc(12.5% - 1em);margin-left:.5em;margin-right:.5em;font-size:11px}.ui.nine.cards{margin-left:-.5em;margin-right:-.5em}.ui.nine.cards>.card{width:calc(11.11111111% - 1em);margin-left:.5em;margin-right:.5em;font-size:10px}.ui.ten.cards{margin-left:-.5em;margin-right:-.5em}.ui.ten.cards>.card{width:calc(10% - 1em);margin-left:.5em;margin-right:.5em}@media only screen and (max-width:767px){.ui.two.doubling.cards{margin-left:0;margin-right:0}.ui.two.doubling.cards>.card{width:100%;margin-left:0;margin-right:0}.ui.three.doubling.cards{margin-left:-1em;margin-right:-1em}.ui.three.doubling.cards>.card{width:calc(50% - 2em);margin-left:1em;margin-right:1em}.ui.four.doubling.cards{margin-left:-1em;margin-right:-1em}.ui.four.doubling.cards>.card{width:calc(50% - 2em);margin-left:1em;margin-right:1em}.ui.five.doubling.cards{margin-left:-1em;margin-right:-1em}.ui.five.doubling.cards>.card{width:calc(50% - 2em);margin-left:1em;margin-right:1em}.ui.six.doubling.cards{margin-left:-1em;margin-right:-1em}.ui.six.doubling.cards>.card{width:calc(50% - 2em);margin-left:1em;margin-right:1em}.ui.seven.doubling.cards{margin-left:-1em;margin-right:-1em}.ui.seven.doubling.cards>.card{width:calc(33.33333333% - 2em);margin-left:1em;margin-right:1em}.ui.eight.doubling.cards{margin-left:-1em;margin-right:-1em}.ui.eight.doubling.cards>.card{width:calc(33.33333333% - 2em);margin-left:1em;margin-right:1em}.ui.nine.doubling.cards{margin-left:-1em;margin-right:-1em}.ui.nine.doubling.cards>.card{width:calc(33.33333333% - 2em);margin-left:1em;margin-right:1em}.ui.ten.doubling.cards{margin-left:-1em;margin-right:-1em}.ui.ten.doubling.cards>.card{width:calc(33.33333333% - 2em);margin-left:1em;margin-right:1em}}@media only screen and (min-width:768px) and (max-width:991px){.ui.two.doubling.cards{margin-left:0;margin-right:0}.ui.two.doubling.cards>.card{width:100%;margin-left:0;margin-right:0}.ui.three.doubling.cards{margin-left:-1em;margin-right:-1em}.ui.three.doubling.cards>.card{width:calc(50% - 2em);margin-left:1em;margin-right:1em}.ui.four.doubling.cards{margin-left:-1em;margin-right:-1em}.ui.four.doubling.cards>.card{width:calc(50% - 2em);margin-left:1em;margin-right:1em}.ui.five.doubling.cards{margin-left:-1em;margin-right:-1em}.ui.five.doubling.cards>.card{width:calc(33.33333333% - 2em);margin-left:1em;margin-right:1em}.ui.six.doubling.cards{margin-left:-1em;margin-right:-1em}.ui.six.doubling.cards>.card{width:calc(33.33333333% - 2em);margin-left:1em;margin-right:1em}.ui.eight.doubling.cards{margin-left:-1em;margin-right:-1em}.ui.eight.doubling.cards>.card{width:calc(33.33333333% - 2em);margin-left:1em;margin-right:1em}.ui.eight.doubling.cards{margin-left:-.75em;margin-right:-.75em}.ui.eight.doubling.cards>.card{width:calc(25% - 1.5em);margin-left:.75em;margin-right:.75em}.ui.nine.doubling.cards{margin-left:-.75em;margin-right:-.75em}.ui.nine.doubling.cards>.card{width:calc(25% - 1.5em);margin-left:.75em;margin-right:.75em}.ui.ten.doubling.cards{margin-left:-.75em;margin-right:-.75em}.ui.ten.doubling.cards>.card{width:calc(20% - 1.5em);margin-left:.75em;margin-right:.75em}}@media only screen and (max-width:767px){.ui.stackable.cards{display:block!important}.ui.stackable.cards .card:first-child{margin-top:0!important}.ui.stackable.cards>.card{display:block!important;height:auto!important;margin:1em 1em;padding:0!important;width:calc(100% - 2em)!important}}.ui.cards>.card{font-size:1em}/*! - * # Semantic UI 2.4.0 - Comment - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */.ui.comments{margin:1.5em 0;max-width:650px}.ui.comments:first-child{margin-top:0}.ui.comments:last-child{margin-bottom:0}.ui.comments .comment{position:relative;background:0 0;margin:.5em 0 0;padding:.5em 0 0;border:none;border-top:none;line-height:1.2}.ui.comments .comment:first-child{margin-top:0;padding-top:0}.ui.comments .comment .comments{margin:0 0 .5em .5em;padding:1em 0 1em 1em}.ui.comments .comment .comments:before{position:absolute;top:0;left:0}.ui.comments .comment .comments .comment{border:none;border-top:none;background:0 0}.ui.comments .comment .avatar{display:block;width:2.5em;height:auto;float:left;margin:.2em 0 0}.ui.comments .comment .avatar img,.ui.comments .comment img.avatar{display:block;margin:0 auto;width:100%;height:100%;border-radius:.25rem}.ui.comments .comment>.content{display:block}.ui.comments .comment>.avatar~.content{margin-left:3.5em}.ui.comments .comment .author{font-size:1em;color:rgba(0,0,0,.87);font-weight:700}.ui.comments .comment a.author{cursor:pointer}.ui.comments .comment a.author:hover{color:#1e70bf}.ui.comments .comment .metadata{display:inline-block;margin-left:.5em;color:rgba(0,0,0,.4);font-size:.875em}.ui.comments .comment .metadata>*{display:inline-block;margin:0 .5em 0 0}.ui.comments .comment .metadata>:last-child{margin-right:0}.ui.comments .comment .text{margin:.25em 0 .5em;font-size:1em;word-wrap:break-word;color:rgba(0,0,0,.87);line-height:1.3}.ui.comments .comment .actions{font-size:.875em}.ui.comments .comment .actions a{cursor:pointer;display:inline-block;margin:0 .75em 0 0;color:rgba(0,0,0,.4)}.ui.comments .comment .actions a:last-child{margin-right:0}.ui.comments .comment .actions a.active,.ui.comments .comment .actions a:hover{color:rgba(0,0,0,.8)}.ui.comments>.reply.form{margin-top:1em}.ui.comments .comment .reply.form{width:100%;margin-top:1em}.ui.comments .reply.form textarea{font-size:1em;height:12em}.ui.collapsed.comments,.ui.comments .collapsed.comment,.ui.comments .collapsed.comments{display:none}.ui.threaded.comments .comment .comments{margin:-1.5em 0 -1em 1.25em;padding:3em 0 2em 2.25em;-webkit-box-shadow:-1px 0 0 rgba(34,36,38,.15);box-shadow:-1px 0 0 rgba(34,36,38,.15)}.ui.minimal.comments .comment .actions{opacity:0;position:absolute;top:0;right:0;left:auto;-webkit-transition:opacity .2s ease;transition:opacity .2s ease;-webkit-transition-delay:.1s;transition-delay:.1s}.ui.minimal.comments .comment>.content:hover>.actions{opacity:1}.ui.mini.comments{font-size:.78571429rem}.ui.tiny.comments{font-size:.85714286rem}.ui.small.comments{font-size:.92857143rem}.ui.comments{font-size:1rem}.ui.large.comments{font-size:1.14285714rem}.ui.big.comments{font-size:1.28571429rem}.ui.huge.comments{font-size:1.42857143rem}.ui.massive.comments{font-size:1.71428571rem}/*! - * # Semantic UI 2.4.0 - Feed - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */.ui.feed{margin:1em 0}.ui.feed:first-child{margin-top:0}.ui.feed:last-child{margin-bottom:0}.ui.feed>.event{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-orient:horizontal;-webkit-box-direction:normal;-ms-flex-direction:row;flex-direction:row;width:100%;padding:.21428571rem 0;margin:0;background:0 0;border-top:none}.ui.feed>.event:first-child{border-top:0;padding-top:0}.ui.feed>.event:last-child{padding-bottom:0}.ui.feed>.event>.label{display:block;-webkit-box-flex:0;-ms-flex:0 0 auto;flex:0 0 auto;width:2.5em;height:auto;-ms-flex-item-align:stretch;align-self:stretch;text-align:left}.ui.feed>.event>.label .icon{opacity:1;font-size:1.5em;width:100%;padding:.25em;background:0 0;border:none;border-radius:none;color:rgba(0,0,0,.6)}.ui.feed>.event>.label img{width:100%;height:auto;border-radius:500rem}.ui.feed>.event>.label+.content{margin:.5em 0 .35714286em 1.14285714em}.ui.feed>.event>.content{display:block;-webkit-box-flex:1;-ms-flex:1 1 auto;flex:1 1 auto;-ms-flex-item-align:stretch;align-self:stretch;text-align:left;word-wrap:break-word}.ui.feed>.event:last-child>.content{padding-bottom:0}.ui.feed>.event>.content a{cursor:pointer}.ui.feed>.event>.content .date{margin:-.5rem 0 0;padding:0;font-weight:400;font-size:1em;font-style:normal;color:rgba(0,0,0,.4)}.ui.feed>.event>.content .summary{margin:0;font-size:1em;font-weight:700;color:rgba(0,0,0,.87)}.ui.feed>.event>.content .summary img{display:inline-block;width:auto;height:10em;margin:-.25em .25em 0 0;border-radius:.25em;vertical-align:middle}.ui.feed>.event>.content .user{display:inline-block;font-weight:700;margin-right:0;vertical-align:baseline}.ui.feed>.event>.content .user img{margin:-.25em .25em 0 0;width:auto;height:10em;vertical-align:middle}.ui.feed>.event>.content .summary>.date{display:inline-block;float:none;font-weight:400;font-size:.85714286em;font-style:normal;margin:0 0 0 .5em;padding:0;color:rgba(0,0,0,.4)}.ui.feed>.event>.content .extra{margin:.5em 0 0;background:0 0;padding:0;color:rgba(0,0,0,.87)}.ui.feed>.event>.content .extra.images img{display:inline-block;margin:0 .25em 0 0;width:6em}.ui.feed>.event>.content .extra.text{padding:0;border-left:none;font-size:1em;max-width:500px;line-height:1.4285em}.ui.feed>.event>.content .meta{display:inline-block;font-size:.85714286em;margin:.5em 0 0;background:0 0;border:none;border-radius:0;-webkit-box-shadow:none;box-shadow:none;padding:0;color:rgba(0,0,0,.6)}.ui.feed>.event>.content .meta>*{position:relative;margin-left:.75em}.ui.feed>.event>.content .meta>:after{content:'';color:rgba(0,0,0,.2);top:0;left:-1em;opacity:1;position:absolute;vertical-align:top}.ui.feed>.event>.content .meta .like{color:'';-webkit-transition:.2s color ease;transition:.2s color ease}.ui.feed>.event>.content .meta .like:hover .icon{color:#ff2733}.ui.feed>.event>.content .meta .active.like .icon{color:#ef404a}.ui.feed>.event>.content .meta>:first-child{margin-left:0}.ui.feed>.event>.content .meta>:first-child::after{display:none}.ui.feed>.event>.content .meta a,.ui.feed>.event>.content .meta>.icon{cursor:pointer;opacity:1;color:rgba(0,0,0,.5);-webkit-transition:color .1s ease;transition:color .1s ease}.ui.feed>.event>.content .meta a:hover,.ui.feed>.event>.content .meta a:hover .icon,.ui.feed>.event>.content .meta>.icon:hover{color:rgba(0,0,0,.95)}.ui.small.feed{font-size:.92857143rem}.ui.feed{font-size:1rem}.ui.large.feed{font-size:1.14285714rem}/*! - * # Semantic UI 2.4.0 - Item - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */.ui.items>.item{display:-webkit-box;display:-ms-flexbox;display:flex;margin:1em 0;width:100%;min-height:0;background:0 0;padding:0;border:none;border-radius:0;-webkit-box-shadow:none;box-shadow:none;-webkit-transition:-webkit-box-shadow .1s ease;transition:-webkit-box-shadow .1s ease;transition:box-shadow .1s ease;transition:box-shadow .1s ease,-webkit-box-shadow .1s ease;z-index:''}.ui.items>.item a{cursor:pointer}.ui.items{margin:1.5em 0}.ui.items:first-child{margin-top:0!important}.ui.items:last-child{margin-bottom:0!important}.ui.items>.item:after{display:block;content:' ';height:0;clear:both;overflow:hidden;visibility:hidden}.ui.items>.item:first-child{margin-top:0}.ui.items>.item:last-child{margin-bottom:0}.ui.items>.item>.image{position:relative;-webkit-box-flex:0;-ms-flex:0 0 auto;flex:0 0 auto;display:block;float:none;margin:0;padding:0;max-height:'';-ms-flex-item-align:top;align-self:top}.ui.items>.item>.image>img{display:block;width:100%;height:auto;border-radius:.125rem;border:none}.ui.items>.item>.image:only-child>img{border-radius:0}.ui.items>.item>.content{display:block;-webkit-box-flex:1;-ms-flex:1 1 auto;flex:1 1 auto;background:0 0;margin:0;padding:0;-webkit-box-shadow:none;box-shadow:none;font-size:1em;border:none;border-radius:0}.ui.items>.item>.content:after{display:block;content:' ';height:0;clear:both;overflow:hidden;visibility:hidden}.ui.items>.item>.image+.content{min-width:0;width:auto;display:block;margin-left:0;-ms-flex-item-align:top;align-self:top;padding-left:1.5em}.ui.items>.item>.content>.header{display:inline-block;margin:-.21425em 0 0;font-family:Lato,'Helvetica Neue',Arial,Helvetica,sans-serif;font-weight:700;color:rgba(0,0,0,.85)}.ui.items>.item>.content>.header:not(.ui){font-size:1.28571429em}.ui.items>.item [class*="left floated"]{float:left}.ui.items>.item [class*="right floated"]{float:right}.ui.items>.item .content img{-ms-flex-item-align:middle;align-self:middle;width:''}.ui.items>.item .avatar img,.ui.items>.item img.avatar{width:'';height:'';border-radius:500rem}.ui.items>.item>.content>.description{margin-top:.6em;max-width:auto;font-size:1em;line-height:1.4285em;color:rgba(0,0,0,.87)}.ui.items>.item>.content p{margin:0 0 .5em}.ui.items>.item>.content p:last-child{margin-bottom:0}.ui.items>.item .meta{margin:.5em 0 .5em;font-size:1em;line-height:1em;color:rgba(0,0,0,.6)}.ui.items>.item .meta *{margin-right:.3em}.ui.items>.item .meta :last-child{margin-right:0}.ui.items>.item .meta [class*="right floated"]{margin-right:0;margin-left:.3em}.ui.items>.item>.content a:not(.ui){color:'';-webkit-transition:color .1s ease;transition:color .1s ease}.ui.items>.item>.content a:not(.ui):hover{color:''}.ui.items>.item>.content>a.header{color:rgba(0,0,0,.85)}.ui.items>.item>.content>a.header:hover{color:#1e70bf}.ui.items>.item .meta>a:not(.ui){color:rgba(0,0,0,.4)}.ui.items>.item .meta>a:not(.ui):hover{color:rgba(0,0,0,.87)}.ui.items>.item>.content .favorite.icon{cursor:pointer;opacity:.75;-webkit-transition:color .1s ease;transition:color .1s ease}.ui.items>.item>.content .favorite.icon:hover{opacity:1;color:#ffb70a}.ui.items>.item>.content .active.favorite.icon{color:#ffe623}.ui.items>.item>.content .like.icon{cursor:pointer;opacity:.75;-webkit-transition:color .1s ease;transition:color .1s ease}.ui.items>.item>.content .like.icon:hover{opacity:1;color:#ff2733}.ui.items>.item>.content .active.like.icon{color:#ff2733}.ui.items>.item .extra{display:block;position:relative;background:0 0;margin:.5rem 0 0;width:100%;padding:0 0 0;top:0;left:0;color:rgba(0,0,0,.4);-webkit-box-shadow:none;box-shadow:none;-webkit-transition:color .1s ease;transition:color .1s ease;border-top:none}.ui.items>.item .extra>*{margin:.25rem .5rem .25rem 0}.ui.items>.item .extra>[class*="right floated"]{margin:.25rem 0 .25rem .5rem}.ui.items>.item .extra:after{display:block;content:' ';height:0;clear:both;overflow:hidden;visibility:hidden}.ui.items>.item>.image:not(.ui){width:175px}@media only screen and (min-width:768px) and (max-width:991px){.ui.items>.item{margin:1em 0}.ui.items>.item>.image:not(.ui){width:150px}.ui.items>.item>.image+.content{display:block;padding:0 0 0 1em}}@media only screen and (max-width:767px){.ui.items:not(.unstackable)>.item{-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column;margin:2em 0}.ui.items:not(.unstackable)>.item>.image{display:block;margin-left:auto;margin-right:auto}.ui.items:not(.unstackable)>.item>.image,.ui.items:not(.unstackable)>.item>.image>img{max-width:100%!important;width:auto!important;max-height:250px!important}.ui.items:not(.unstackable)>.item>.image+.content{display:block;padding:1.5em 0 0}}.ui.items>.item>.image+[class*="top aligned"].content{-ms-flex-item-align:start;align-self:flex-start}.ui.items>.item>.image+[class*="middle aligned"].content{-ms-flex-item-align:center;align-self:center}.ui.items>.item>.image+[class*="bottom aligned"].content{-ms-flex-item-align:end;align-self:flex-end}.ui.relaxed.items>.item{margin:1.5em 0}.ui[class*="very relaxed"].items>.item{margin:2em 0}.ui.divided.items>.item{border-top:1px solid rgba(34,36,38,.15);margin:0;padding:1em 0}.ui.divided.items>.item:first-child{border-top:none;margin-top:0!important;padding-top:0!important}.ui.divided.items>.item:last-child{margin-bottom:0!important;padding-bottom:0!important}.ui.relaxed.divided.items>.item{margin:0;padding:1.5em 0}.ui[class*="very relaxed"].divided.items>.item{margin:0;padding:2em 0}.ui.items a.item:hover,.ui.link.items>.item:hover{cursor:pointer}.ui.items a.item:hover .content .header,.ui.link.items>.item:hover .content .header{color:#1e70bf}.ui.items>.item{font-size:1em}@media only screen and (max-width:767px){.ui.unstackable.items>.item>.image,.ui.unstackable.items>.item>.image>img{width:125px!important}}/*! - * # Semantic UI 2.4.0 - Statistic - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */.ui.statistic{display:-webkit-inline-box;display:-ms-inline-flexbox;display:inline-flex;-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column;margin:1em 0;max-width:auto}.ui.statistic+.ui.statistic{margin:0 0 0 1.5em}.ui.statistic:first-child{margin-top:0}.ui.statistic:last-child{margin-bottom:0}.ui.statistics{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-align:start;-ms-flex-align:start;align-items:flex-start;-ms-flex-wrap:wrap;flex-wrap:wrap}.ui.statistics>.statistic{display:-webkit-inline-box;display:-ms-inline-flexbox;display:inline-flex;-webkit-box-flex:0;-ms-flex:0 1 auto;flex:0 1 auto;-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column;margin:0 1.5em 1em;max-width:auto}.ui.statistics{display:-webkit-box;display:-ms-flexbox;display:flex;margin:1em -1.5em -1em}.ui.statistics:after{display:block;content:' ';height:0;clear:both;overflow:hidden;visibility:hidden}.ui.statistics:first-child{margin-top:0}.ui.statistic>.value,.ui.statistics .statistic>.value{font-family:Lato,'Helvetica Neue',Arial,Helvetica,sans-serif;font-size:4rem;font-weight:400;line-height:1em;color:#1b1c1d;text-transform:uppercase;text-align:center}.ui.statistic>.label,.ui.statistics .statistic>.label{font-family:Lato,'Helvetica Neue',Arial,Helvetica,sans-serif;font-size:1em;font-weight:700;color:rgba(0,0,0,.87);text-transform:uppercase;text-align:center}.ui.statistic>.label~.value,.ui.statistics .statistic>.label~.value{margin-top:0}.ui.statistic>.value~.label,.ui.statistics .statistic>.value~.label{margin-top:0}.ui.statistic>.value .icon,.ui.statistics .statistic>.value .icon{opacity:1;width:auto;margin:0}.ui.statistic>.text.value,.ui.statistics .statistic>.text.value{line-height:1em;min-height:2em;font-weight:700;text-align:center}.ui.statistic>.text.value+.label,.ui.statistics .statistic>.text.value+.label{text-align:center}.ui.statistic>.value img,.ui.statistics .statistic>.value img{max-height:3rem;vertical-align:baseline}.ui.ten.statistics{margin:0 0 -1em}.ui.ten.statistics .statistic{min-width:10%;margin:0 0 1em}.ui.nine.statistics{margin:0 0 -1em}.ui.nine.statistics .statistic{min-width:11.11111111%;margin:0 0 1em}.ui.eight.statistics{margin:0 0 -1em}.ui.eight.statistics .statistic{min-width:12.5%;margin:0 0 1em}.ui.seven.statistics{margin:0 0 -1em}.ui.seven.statistics .statistic{min-width:14.28571429%;margin:0 0 1em}.ui.six.statistics{margin:0 0 -1em}.ui.six.statistics .statistic{min-width:16.66666667%;margin:0 0 1em}.ui.five.statistics{margin:0 0 -1em}.ui.five.statistics .statistic{min-width:20%;margin:0 0 1em}.ui.four.statistics{margin:0 0 -1em}.ui.four.statistics .statistic{min-width:25%;margin:0 0 1em}.ui.three.statistics{margin:0 0 -1em}.ui.three.statistics .statistic{min-width:33.33333333%;margin:0 0 1em}.ui.two.statistics{margin:0 0 -1em}.ui.two.statistics .statistic{min-width:50%;margin:0 0 1em}.ui.one.statistics{margin:0 0 -1em}.ui.one.statistics .statistic{min-width:100%;margin:0 0 1em}.ui.horizontal.statistic{-webkit-box-orient:horizontal;-webkit-box-direction:normal;-ms-flex-direction:row;flex-direction:row;-webkit-box-align:center;-ms-flex-align:center;align-items:center}.ui.horizontal.statistics{-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column;margin:0;max-width:none}.ui.horizontal.statistics .statistic{-webkit-box-orient:horizontal;-webkit-box-direction:normal;-ms-flex-direction:row;flex-direction:row;-webkit-box-align:center;-ms-flex-align:center;align-items:center;max-width:none;margin:1em 0}.ui.horizontal.statistic>.text.value,.ui.horizontal.statistics>.statistic>.text.value{min-height:0!important}.ui.horizontal.statistic>.value .icon,.ui.horizontal.statistics .statistic>.value .icon{width:1.18em}.ui.horizontal.statistic>.value,.ui.horizontal.statistics .statistic>.value{display:inline-block;vertical-align:middle}.ui.horizontal.statistic>.label,.ui.horizontal.statistics .statistic>.label{display:inline-block;vertical-align:middle;margin:0 0 0 .75em}.ui.red.statistic>.value,.ui.red.statistics .statistic>.value,.ui.statistics .red.statistic>.value{color:#db2828}.ui.orange.statistic>.value,.ui.orange.statistics .statistic>.value,.ui.statistics .orange.statistic>.value{color:#f2711c}.ui.statistics .yellow.statistic>.value,.ui.yellow.statistic>.value,.ui.yellow.statistics .statistic>.value{color:#fbbd08}.ui.olive.statistic>.value,.ui.olive.statistics .statistic>.value,.ui.statistics .olive.statistic>.value{color:#b5cc18}.ui.green.statistic>.value,.ui.green.statistics .statistic>.value,.ui.statistics .green.statistic>.value{color:#21ba45}.ui.statistics .teal.statistic>.value,.ui.teal.statistic>.value,.ui.teal.statistics .statistic>.value{color:#00b5ad}.ui.blue.statistic>.value,.ui.blue.statistics .statistic>.value,.ui.statistics .blue.statistic>.value{color:#2185d0}.ui.statistics .violet.statistic>.value,.ui.violet.statistic>.value,.ui.violet.statistics .statistic>.value{color:#6435c9}.ui.purple.statistic>.value,.ui.purple.statistics .statistic>.value,.ui.statistics .purple.statistic>.value{color:#a333c8}.ui.pink.statistic>.value,.ui.pink.statistics .statistic>.value,.ui.statistics .pink.statistic>.value{color:#e03997}.ui.brown.statistic>.value,.ui.brown.statistics .statistic>.value,.ui.statistics .brown.statistic>.value{color:#a5673f}.ui.grey.statistic>.value,.ui.grey.statistics .statistic>.value,.ui.statistics .grey.statistic>.value{color:#767676}.ui.inverted.statistic .value,.ui.inverted.statistics .statistic>.value{color:#fff}.ui.inverted.statistic .label,.ui.inverted.statistics .statistic>.label{color:rgba(255,255,255,.9)}.ui.inverted.red.statistic>.value,.ui.inverted.red.statistics .statistic>.value,.ui.statistics .inverted.red.statistic>.value{color:#ff695e}.ui.inverted.orange.statistic>.value,.ui.inverted.orange.statistics .statistic>.value,.ui.statistics .inverted.orange.statistic>.value{color:#ff851b}.ui.inverted.yellow.statistic>.value,.ui.inverted.yellow.statistics .statistic>.value,.ui.statistics .inverted.yellow.statistic>.value{color:#ffe21f}.ui.inverted.olive.statistic>.value,.ui.inverted.olive.statistics .statistic>.value,.ui.statistics .inverted.olive.statistic>.value{color:#d9e778}.ui.inverted.green.statistic>.value,.ui.inverted.green.statistics .statistic>.value,.ui.statistics .inverted.green.statistic>.value{color:#2ecc40}.ui.inverted.teal.statistic>.value,.ui.inverted.teal.statistics .statistic>.value,.ui.statistics .inverted.teal.statistic>.value{color:#6dffff}.ui.inverted.blue.statistic>.value,.ui.inverted.blue.statistics .statistic>.value,.ui.statistics .inverted.blue.statistic>.value{color:#54c8ff}.ui.inverted.violet.statistic>.value,.ui.inverted.violet.statistics .statistic>.value,.ui.statistics .inverted.violet.statistic>.value{color:#a291fb}.ui.inverted.purple.statistic>.value,.ui.inverted.purple.statistics .statistic>.value,.ui.statistics .inverted.purple.statistic>.value{color:#dc73ff}.ui.inverted.pink.statistic>.value,.ui.inverted.pink.statistics .statistic>.value,.ui.statistics .inverted.pink.statistic>.value{color:#ff8edf}.ui.inverted.brown.statistic>.value,.ui.inverted.brown.statistics .statistic>.value,.ui.statistics .inverted.brown.statistic>.value{color:#d67c1c}.ui.inverted.grey.statistic>.value,.ui.inverted.grey.statistics .statistic>.value,.ui.statistics .inverted.grey.statistic>.value{color:#dcddde}.ui[class*="left floated"].statistic{float:left;margin:0 2em 1em 0}.ui[class*="right floated"].statistic{float:right;margin:0 0 1em 2em}.ui.floated.statistic:last-child{margin-bottom:0}.ui.mini.statistic>.value,.ui.mini.statistics .statistic>.value{font-size:1.5rem!important}.ui.mini.horizontal.statistic>.value,.ui.mini.horizontal.statistics .statistic>.value{font-size:1.5rem!important}.ui.mini.statistic>.text.value,.ui.mini.statistics .statistic>.text.value{font-size:1rem!important}.ui.tiny.statistic>.value,.ui.tiny.statistics .statistic>.value{font-size:2rem!important}.ui.tiny.horizontal.statistic>.value,.ui.tiny.horizontal.statistics .statistic>.value{font-size:2rem!important}.ui.tiny.statistic>.text.value,.ui.tiny.statistics .statistic>.text.value{font-size:1rem!important}.ui.small.statistic>.value,.ui.small.statistics .statistic>.value{font-size:3rem!important}.ui.small.horizontal.statistic>.value,.ui.small.horizontal.statistics .statistic>.value{font-size:2rem!important}.ui.small.statistic>.text.value,.ui.small.statistics .statistic>.text.value{font-size:1rem!important}.ui.statistic>.value,.ui.statistics .statistic>.value{font-size:4rem!important}.ui.horizontal.statistic>.value,.ui.horizontal.statistics .statistic>.value{font-size:3rem!important}.ui.statistic>.text.value,.ui.statistics .statistic>.text.value{font-size:2rem!important}.ui.large.statistic>.value,.ui.large.statistics .statistic>.value{font-size:5rem!important}.ui.large.horizontal.statistic>.value,.ui.large.horizontal.statistics .statistic>.value{font-size:4rem!important}.ui.large.statistic>.text.value,.ui.large.statistics .statistic>.text.value{font-size:2.5rem!important}.ui.huge.statistic>.value,.ui.huge.statistics .statistic>.value{font-size:6rem!important}.ui.huge.horizontal.statistic>.value,.ui.huge.horizontal.statistics .statistic>.value{font-size:5rem!important}.ui.huge.statistic>.text.value,.ui.huge.statistics .statistic>.text.value{font-size:2.5rem!important}/*! - * # Semantic UI 2.4.0 - Accordion - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */.ui.accordion,.ui.accordion .accordion{max-width:100%}.ui.accordion .accordion{margin:1em 0 0;padding:0}.ui.accordion .accordion .title,.ui.accordion .title{cursor:pointer}.ui.accordion .title:not(.ui){padding:.5em 0;font-family:Lato,'Helvetica Neue',Arial,Helvetica,sans-serif;font-size:1em;color:rgba(0,0,0,.87)}.ui.accordion .accordion .title~.content,.ui.accordion .title~.content{display:none}.ui.accordion:not(.styled) .accordion .title~.content:not(.ui),.ui.accordion:not(.styled) .title~.content:not(.ui){margin:'';padding:.5em 0 1em}.ui.accordion:not(.styled) .title~.content:not(.ui):last-child{padding-bottom:0}.ui.accordion .accordion .title .dropdown.icon,.ui.accordion .title .dropdown.icon{display:inline-block;float:none;opacity:1;width:1.25em;height:1em;margin:0 .25rem 0 0;padding:0;font-size:1em;-webkit-transition:opacity .1s ease,-webkit-transform .1s ease;transition:opacity .1s ease,-webkit-transform .1s ease;transition:transform .1s ease,opacity .1s ease;transition:transform .1s ease,opacity .1s ease,-webkit-transform .1s ease;vertical-align:baseline;-webkit-transform:none;transform:none}.ui.accordion.menu .item .title{display:block;padding:0}.ui.accordion.menu .item .title>.dropdown.icon{float:right;margin:.21425em 0 0 1em;-webkit-transform:rotate(180deg);transform:rotate(180deg)}.ui.accordion .ui.header .dropdown.icon{font-size:1em;margin:0 .25rem 0 0}.ui.accordion .accordion .active.title .dropdown.icon,.ui.accordion .active.title .dropdown.icon{-webkit-transform:rotate(90deg);transform:rotate(90deg)}.ui.accordion.menu .item .active.title>.dropdown.icon{-webkit-transform:rotate(90deg);transform:rotate(90deg)}.ui.styled.accordion{width:600px}.ui.styled.accordion,.ui.styled.accordion .accordion{border-radius:.28571429rem;background:#fff;-webkit-box-shadow:0 1px 2px 0 rgba(34,36,38,.15),0 0 0 1px rgba(34,36,38,.15);box-shadow:0 1px 2px 0 rgba(34,36,38,.15),0 0 0 1px rgba(34,36,38,.15)}.ui.styled.accordion .accordion .title,.ui.styled.accordion .title{margin:0;padding:.75em 1em;color:rgba(0,0,0,.4);font-weight:700;border-top:1px solid rgba(34,36,38,.15);-webkit-transition:background .1s ease,color .1s ease;transition:background .1s ease,color .1s ease}.ui.styled.accordion .accordion .title:first-child,.ui.styled.accordion>.title:first-child{border-top:none}.ui.styled.accordion .accordion .content,.ui.styled.accordion .content{margin:0;padding:.5em 1em 1.5em}.ui.styled.accordion .accordion .content{padding:0;padding:.5em 1em 1.5em}.ui.styled.accordion .accordion .active.title,.ui.styled.accordion .accordion .title:hover,.ui.styled.accordion .active.title,.ui.styled.accordion .title:hover{background:0 0;color:rgba(0,0,0,.87)}.ui.styled.accordion .accordion .active.title,.ui.styled.accordion .accordion .title:hover{background:0 0;color:rgba(0,0,0,.87)}.ui.styled.accordion .active.title{background:0 0;color:rgba(0,0,0,.95)}.ui.styled.accordion .accordion .active.title{background:0 0;color:rgba(0,0,0,.95)}.ui.accordion .accordion .active.content,.ui.accordion .active.content{display:block}.ui.fluid.accordion,.ui.fluid.accordion .accordion{width:100%}.ui.inverted.accordion .title:not(.ui){color:rgba(255,255,255,.9)}@font-face{font-family:Accordion;src:url(data:application/x-font-ttf;charset=utf-8;base64,AAEAAAALAIAAAwAwT1MvMggjB5AAAAC8AAAAYGNtYXAPfOIKAAABHAAAAExnYXNwAAAAEAAAAWgAAAAIZ2x5Zryj6HgAAAFwAAAAyGhlYWT/0IhHAAACOAAAADZoaGVhApkB5wAAAnAAAAAkaG10eAJuABIAAAKUAAAAGGxvY2EAjABWAAACrAAAAA5tYXhwAAgAFgAAArwAAAAgbmFtZfC1n04AAALcAAABPHBvc3QAAwAAAAAEGAAAACAAAwIAAZAABQAAAUwBZgAAAEcBTAFmAAAA9QAZAIQAAAAAAAAAAAAAAAAAAAABEAAAAAAAAAAAAAAAAAAAAABAAADw2gHg/+D/4AHgACAAAAABAAAAAAAAAAAAAAAgAAAAAAACAAAAAwAAABQAAwABAAAAFAAEADgAAAAKAAgAAgACAAEAIPDa//3//wAAAAAAIPDZ//3//wAB/+MPKwADAAEAAAAAAAAAAAAAAAEAAf//AA8AAQAAAAAAAAAAAAIAADc5AQAAAAABAAAAAAAAAAAAAgAANzkBAAAAAAEAAAAAAAAAAAACAAA3OQEAAAAAAQASAEkAtwFuABMAADc0PwE2FzYXFh0BFAcGJwYvASY1EgaABQgHBQYGBQcIBYAG2wcGfwcBAQcECf8IBAcBAQd/BgYAAAAAAQAAAEkApQFuABMAADcRNDc2MzIfARYVFA8BBiMiJyY1AAUGBwgFgAYGgAUIBwYFWwEACAUGBoAFCAcFgAYGBQcAAAABAAAAAQAAqWYls18PPPUACwIAAAAAAM/9o+4AAAAAz/2j7gAAAAAAtwFuAAAACAACAAAAAAAAAAEAAAHg/+AAAAIAAAAAAAC3AAEAAAAAAAAAAAAAAAAAAAAGAAAAAAAAAAAAAAAAAQAAAAC3ABIAtwAAAAAAAAAKABQAHgBCAGQAAAABAAAABgAUAAEAAAAAAAIAAAAAAAAAAAAAAAAAAAAAAAAADgCuAAEAAAAAAAEADAAAAAEAAAAAAAIADgBAAAEAAAAAAAMADAAiAAEAAAAAAAQADABOAAEAAAAAAAUAFgAMAAEAAAAAAAYABgAuAAEAAAAAAAoANABaAAMAAQQJAAEADAAAAAMAAQQJAAIADgBAAAMAAQQJAAMADAAiAAMAAQQJAAQADABOAAMAAQQJAAUAFgAMAAMAAQQJAAYADAA0AAMAAQQJAAoANABaAHIAYQB0AGkAbgBnAFYAZQByAHMAaQBvAG4AIAAxAC4AMAByAGEAdABpAG4AZ3JhdGluZwByAGEAdABpAG4AZwBSAGUAZwB1AGwAYQByAHIAYQB0AGkAbgBnAEYAbwBuAHQAIABnAGUAbgBlAHIAYQB0AGUAZAAgAGIAeQAgAEkAYwBvAE0AbwBvAG4ALgADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA) format('truetype'),url(data:application/font-woff;charset=utf-8;base64,d09GRk9UVE8AAASwAAoAAAAABGgAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAABDRkYgAAAA9AAAAS0AAAEtFpovuE9TLzIAAAIkAAAAYAAAAGAIIweQY21hcAAAAoQAAABMAAAATA984gpnYXNwAAAC0AAAAAgAAAAIAAAAEGhlYWQAAALYAAAANgAAADb/0IhHaGhlYQAAAxAAAAAkAAAAJAKZAedobXR4AAADNAAAABgAAAAYAm4AEm1heHAAAANMAAAABgAAAAYABlAAbmFtZQAAA1QAAAE8AAABPPC1n05wb3N0AAAEkAAAACAAAAAgAAMAAAEABAQAAQEBB3JhdGluZwABAgABADr4HAL4GwP4GAQeCgAZU/+Lix4KABlT/4uLDAeLa/iU+HQFHQAAAHkPHQAAAH4RHQAAAAkdAAABJBIABwEBBw0PERQZHnJhdGluZ3JhdGluZ3UwdTF1MjB1RjBEOXVGMERBAAACAYkABAAGAQEEBwoNVp38lA78lA78lA77lA773Z33bxWLkI2Qj44I9xT3FAWOj5CNkIuQi4+JjoePiI2Gi4YIi/uUBYuGiYeHiIiHh4mGi4aLho2Ijwj7FPcUBYeOiY+LkAgO+92L5hWL95QFi5CNkI6Oj4+PjZCLkIuQiY6HCPcU+xQFj4iNhouGi4aJh4eICPsU+xQFiIeGiYaLhouHjYePiI6Jj4uQCA74lBT4lBWLDAoAAAAAAwIAAZAABQAAAUwBZgAAAEcBTAFmAAAA9QAZAIQAAAAAAAAAAAAAAAAAAAABEAAAAAAAAAAAAAAAAAAAAABAAADw2gHg/+D/4AHgACAAAAABAAAAAAAAAAAAAAAgAAAAAAACAAAAAwAAABQAAwABAAAAFAAEADgAAAAKAAgAAgACAAEAIPDa//3//wAAAAAAIPDZ//3//wAB/+MPKwADAAEAAAAAAAAAAAAAAAEAAf//AA8AAQAAAAEAADfYOJZfDzz1AAsCAAAAAADP/aPuAAAAAM/9o+4AAAAAALcBbgAAAAgAAgAAAAAAAAABAAAB4P/gAAACAAAAAAAAtwABAAAAAAAAAAAAAAAAAAAABgAAAAAAAAAAAAAAAAEAAAAAtwASALcAAAAAUAAABgAAAAAADgCuAAEAAAAAAAEADAAAAAEAAAAAAAIADgBAAAEAAAAAAAMADAAiAAEAAAAAAAQADABOAAEAAAAAAAUAFgAMAAEAAAAAAAYABgAuAAEAAAAAAAoANABaAAMAAQQJAAEADAAAAAMAAQQJAAIADgBAAAMAAQQJAAMADAAiAAMAAQQJAAQADABOAAMAAQQJAAUAFgAMAAMAAQQJAAYADAA0AAMAAQQJAAoANABaAHIAYQB0AGkAbgBnAFYAZQByAHMAaQBvAG4AIAAxAC4AMAByAGEAdABpAG4AZ3JhdGluZwByAGEAdABpAG4AZwBSAGUAZwB1AGwAYQByAHIAYQB0AGkAbgBnAEYAbwBuAHQAIABnAGUAbgBlAHIAYQB0AGUAZAAgAGIAeQAgAEkAYwBvAE0AbwBvAG4ALgADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA) format('woff');font-weight:400;font-style:normal}.ui.accordion .accordion .title .dropdown.icon,.ui.accordion .title .dropdown.icon{font-family:Accordion;line-height:1;-webkit-backface-visibility:hidden;backface-visibility:hidden;font-weight:400;font-style:normal;text-align:center}.ui.accordion .accordion .title .dropdown.icon:before,.ui.accordion .title .dropdown.icon:before{content:'\f0da'}/*! - * # Semantic UI 2.4.0 - Checkbox - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */.ui.checkbox{position:relative;display:inline-block;-webkit-backface-visibility:hidden;backface-visibility:hidden;outline:0;vertical-align:baseline;font-style:normal;min-height:17px;font-size:1rem;line-height:17px;min-width:17px}.ui.checkbox input[type=checkbox],.ui.checkbox input[type=radio]{cursor:pointer;position:absolute;top:0;left:0;opacity:0!important;outline:0;z-index:3;width:17px;height:17px}.ui.checkbox .box,.ui.checkbox label{cursor:auto;position:relative;display:block;padding-left:1.85714em;outline:0;font-size:1em}.ui.checkbox .box:before,.ui.checkbox label:before{position:absolute;top:0;left:0;width:17px;height:17px;content:'';background:#fff;border-radius:.21428571rem;-webkit-transition:border .1s ease,opacity .1s ease,-webkit-transform .1s ease,-webkit-box-shadow .1s ease;transition:border .1s ease,opacity .1s ease,-webkit-transform .1s ease,-webkit-box-shadow .1s ease;transition:border .1s ease,opacity .1s ease,transform .1s ease,box-shadow .1s ease;transition:border .1s ease,opacity .1s ease,transform .1s ease,box-shadow .1s ease,-webkit-transform .1s ease,-webkit-box-shadow .1s ease;border:1px solid #d4d4d5}.ui.checkbox .box:after,.ui.checkbox label:after{position:absolute;font-size:14px;top:0;left:0;width:17px;height:17px;text-align:center;opacity:0;color:rgba(0,0,0,.87);-webkit-transition:border .1s ease,opacity .1s ease,-webkit-transform .1s ease,-webkit-box-shadow .1s ease;transition:border .1s ease,opacity .1s ease,-webkit-transform .1s ease,-webkit-box-shadow .1s ease;transition:border .1s ease,opacity .1s ease,transform .1s ease,box-shadow .1s ease;transition:border .1s ease,opacity .1s ease,transform .1s ease,box-shadow .1s ease,-webkit-transform .1s ease,-webkit-box-shadow .1s ease}.ui.checkbox label,.ui.checkbox+label{color:rgba(0,0,0,.87);-webkit-transition:color .1s ease;transition:color .1s ease}.ui.checkbox+label{vertical-align:middle}.ui.checkbox .box:hover::before,.ui.checkbox label:hover::before{background:#fff;border-color:rgba(34,36,38,.35)}.ui.checkbox label:hover,.ui.checkbox+label:hover{color:rgba(0,0,0,.8)}.ui.checkbox .box:active::before,.ui.checkbox label:active::before{background:#f9fafb;border-color:rgba(34,36,38,.35)}.ui.checkbox .box:active::after,.ui.checkbox label:active::after{color:rgba(0,0,0,.95)}.ui.checkbox input:active~label{color:rgba(0,0,0,.95)}.ui.checkbox input:focus~.box:before,.ui.checkbox input:focus~label:before{background:#fff;border-color:#96c8da}.ui.checkbox input:focus~.box:after,.ui.checkbox input:focus~label:after{color:rgba(0,0,0,.95)}.ui.checkbox input:focus~label{color:rgba(0,0,0,.95)}.ui.checkbox input:checked~.box:before,.ui.checkbox input:checked~label:before{background:#fff;border-color:rgba(34,36,38,.35)}.ui.checkbox input:checked~.box:after,.ui.checkbox input:checked~label:after{opacity:1;color:rgba(0,0,0,.95)}.ui.checkbox input:not([type=radio]):indeterminate~.box:before,.ui.checkbox input:not([type=radio]):indeterminate~label:before{background:#fff;border-color:rgba(34,36,38,.35)}.ui.checkbox input:not([type=radio]):indeterminate~.box:after,.ui.checkbox input:not([type=radio]):indeterminate~label:after{opacity:1;color:rgba(0,0,0,.95)}.ui.checkbox input:checked:focus~.box:before,.ui.checkbox input:checked:focus~label:before,.ui.checkbox input:not([type=radio]):indeterminate:focus~.box:before,.ui.checkbox input:not([type=radio]):indeterminate:focus~label:before{background:#fff;border-color:#96c8da}.ui.checkbox input:checked:focus~.box:after,.ui.checkbox input:checked:focus~label:after,.ui.checkbox input:not([type=radio]):indeterminate:focus~.box:after,.ui.checkbox input:not([type=radio]):indeterminate:focus~label:after{color:rgba(0,0,0,.95)}.ui.read-only.checkbox,.ui.read-only.checkbox label{cursor:default}.ui.checkbox input[disabled]~.box:after,.ui.checkbox input[disabled]~label,.ui.disabled.checkbox .box:after,.ui.disabled.checkbox label{cursor:default!important;opacity:.5;color:#000}.ui.checkbox input.hidden{z-index:-1}.ui.checkbox input.hidden+label{cursor:pointer;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none}.ui.radio.checkbox{min-height:15px}.ui.radio.checkbox .box,.ui.radio.checkbox label{padding-left:1.85714em}.ui.radio.checkbox .box:before,.ui.radio.checkbox label:before{content:'';-webkit-transform:none;transform:none;width:15px;height:15px;border-radius:500rem;top:1px;left:0}.ui.radio.checkbox .box:after,.ui.radio.checkbox label:after{border:none;content:''!important;width:15px;height:15px;line-height:15px}.ui.radio.checkbox .box:after,.ui.radio.checkbox label:after{top:1px;left:0;width:15px;height:15px;border-radius:500rem;-webkit-transform:scale(.46666667);transform:scale(.46666667);background-color:rgba(0,0,0,.87)}.ui.radio.checkbox input:focus~.box:before,.ui.radio.checkbox input:focus~label:before{background-color:#fff}.ui.radio.checkbox input:focus~.box:after,.ui.radio.checkbox input:focus~label:after{background-color:rgba(0,0,0,.95)}.ui.radio.checkbox input:indeterminate~.box:after,.ui.radio.checkbox input:indeterminate~label:after{opacity:0}.ui.radio.checkbox input:checked~.box:before,.ui.radio.checkbox input:checked~label:before{background-color:#fff}.ui.radio.checkbox input:checked~.box:after,.ui.radio.checkbox input:checked~label:after{background-color:rgba(0,0,0,.95)}.ui.radio.checkbox input:focus:checked~.box:before,.ui.radio.checkbox input:focus:checked~label:before{background-color:#fff}.ui.radio.checkbox input:focus:checked~.box:after,.ui.radio.checkbox input:focus:checked~label:after{background-color:rgba(0,0,0,.95)}.ui.slider.checkbox{min-height:1.25rem}.ui.slider.checkbox input{width:3.5rem;height:1.25rem}.ui.slider.checkbox .box,.ui.slider.checkbox label{padding-left:4.5rem;line-height:1rem;color:rgba(0,0,0,.4)}.ui.slider.checkbox .box:before,.ui.slider.checkbox label:before{display:block;position:absolute;content:'';border:none!important;left:0;z-index:1;top:.4rem;background-color:rgba(0,0,0,.05);width:3.5rem;height:.21428571rem;-webkit-transform:none;transform:none;border-radius:500rem;-webkit-transition:background .3s ease;transition:background .3s ease}.ui.slider.checkbox .box:after,.ui.slider.checkbox label:after{background:#fff -webkit-gradient(linear,left top,left bottom,from(transparent),to(rgba(0,0,0,.05)));background:#fff -webkit-linear-gradient(transparent,rgba(0,0,0,.05));background:#fff linear-gradient(transparent,rgba(0,0,0,.05));position:absolute;content:''!important;opacity:1;z-index:2;border:none;-webkit-box-shadow:0 1px 2px 0 rgba(34,36,38,.15),0 0 0 1px rgba(34,36,38,.15) inset;box-shadow:0 1px 2px 0 rgba(34,36,38,.15),0 0 0 1px rgba(34,36,38,.15) inset;width:1.5rem;height:1.5rem;top:-.25rem;left:0;-webkit-transform:none;transform:none;border-radius:500rem;-webkit-transition:left .3s ease;transition:left .3s ease}.ui.slider.checkbox input:focus~.box:before,.ui.slider.checkbox input:focus~label:before{background-color:rgba(0,0,0,.15);border:none}.ui.slider.checkbox .box:hover,.ui.slider.checkbox label:hover{color:rgba(0,0,0,.8)}.ui.slider.checkbox .box:hover::before,.ui.slider.checkbox label:hover::before{background:rgba(0,0,0,.15)}.ui.slider.checkbox input:checked~.box,.ui.slider.checkbox input:checked~label{color:rgba(0,0,0,.95)!important}.ui.slider.checkbox input:checked~.box:before,.ui.slider.checkbox input:checked~label:before{background-color:#545454!important}.ui.slider.checkbox input:checked~.box:after,.ui.slider.checkbox input:checked~label:after{left:2rem}.ui.slider.checkbox input:focus:checked~.box,.ui.slider.checkbox input:focus:checked~label{color:rgba(0,0,0,.95)!important}.ui.slider.checkbox input:focus:checked~.box:before,.ui.slider.checkbox input:focus:checked~label:before{background-color:#000!important}.ui.toggle.checkbox{min-height:1.5rem}.ui.toggle.checkbox input{width:3.5rem;height:1.5rem}.ui.toggle.checkbox .box,.ui.toggle.checkbox label{min-height:1.5rem;padding-left:4.5rem;color:rgba(0,0,0,.87)}.ui.toggle.checkbox label{padding-top:.15em}.ui.toggle.checkbox .box:before,.ui.toggle.checkbox label:before{display:block;position:absolute;content:'';z-index:1;-webkit-transform:none;transform:none;border:none;top:0;background:rgba(0,0,0,.05);-webkit-box-shadow:none;box-shadow:none;width:3.5rem;height:1.5rem;border-radius:500rem}.ui.toggle.checkbox .box:after,.ui.toggle.checkbox label:after{background:#fff -webkit-gradient(linear,left top,left bottom,from(transparent),to(rgba(0,0,0,.05)));background:#fff -webkit-linear-gradient(transparent,rgba(0,0,0,.05));background:#fff linear-gradient(transparent,rgba(0,0,0,.05));position:absolute;content:''!important;opacity:1;z-index:2;border:none;-webkit-box-shadow:0 1px 2px 0 rgba(34,36,38,.15),0 0 0 1px rgba(34,36,38,.15) inset;box-shadow:0 1px 2px 0 rgba(34,36,38,.15),0 0 0 1px rgba(34,36,38,.15) inset;width:1.5rem;height:1.5rem;top:0;left:0;border-radius:500rem;-webkit-transition:background .3s ease,left .3s ease;transition:background .3s ease,left .3s ease}.ui.toggle.checkbox input~.box:after,.ui.toggle.checkbox input~label:after{left:-.05rem;-webkit-box-shadow:0 1px 2px 0 rgba(34,36,38,.15),0 0 0 1px rgba(34,36,38,.15) inset;box-shadow:0 1px 2px 0 rgba(34,36,38,.15),0 0 0 1px rgba(34,36,38,.15) inset}.ui.toggle.checkbox input:focus~.box:before,.ui.toggle.checkbox input:focus~label:before{background-color:rgba(0,0,0,.15);border:none}.ui.toggle.checkbox .box:hover::before,.ui.toggle.checkbox label:hover::before{background-color:rgba(0,0,0,.15);border:none}.ui.toggle.checkbox input:checked~.box,.ui.toggle.checkbox input:checked~label{color:rgba(0,0,0,.95)!important}.ui.toggle.checkbox input:checked~.box:before,.ui.toggle.checkbox input:checked~label:before{background-color:#2185d0!important}.ui.toggle.checkbox input:checked~.box:after,.ui.toggle.checkbox input:checked~label:after{left:2.15rem;-webkit-box-shadow:0 1px 2px 0 rgba(34,36,38,.15),0 0 0 1px rgba(34,36,38,.15) inset;box-shadow:0 1px 2px 0 rgba(34,36,38,.15),0 0 0 1px rgba(34,36,38,.15) inset}.ui.toggle.checkbox input:focus:checked~.box,.ui.toggle.checkbox input:focus:checked~label{color:rgba(0,0,0,.95)!important}.ui.toggle.checkbox input:focus:checked~.box:before,.ui.toggle.checkbox input:focus:checked~label:before{background-color:#0d71bb!important}.ui.fitted.checkbox .box,.ui.fitted.checkbox label{padding-left:0!important}.ui.fitted.toggle.checkbox{width:3.5rem}.ui.fitted.slider.checkbox{width:3.5rem}@font-face{font-family:Checkbox;src:url(data:application/x-font-ttf;charset=utf-8;base64,AAEAAAALAIAAAwAwT1MvMg8SBD8AAAC8AAAAYGNtYXAYVtCJAAABHAAAAFRnYXNwAAAAEAAAAXAAAAAIZ2x5Zn4huwUAAAF4AAABYGhlYWQGPe1ZAAAC2AAAADZoaGVhB30DyAAAAxAAAAAkaG10eBBKAEUAAAM0AAAAHGxvY2EAmgESAAADUAAAABBtYXhwAAkALwAAA2AAAAAgbmFtZSC8IugAAAOAAAABknBvc3QAAwAAAAAFFAAAACAAAwMTAZAABQAAApkCzAAAAI8CmQLMAAAB6wAzAQkAAAAAAAAAAAAAAAAAAAABEAAAAAAAAAAAAAAAAAAAAABAAADoAgPA/8AAQAPAAEAAAAABAAAAAAAAAAAAAAAgAAAAAAADAAAAAwAAABwAAQADAAAAHAADAAEAAAAcAAQAOAAAAAoACAACAAIAAQAg6AL//f//AAAAAAAg6AD//f//AAH/4xgEAAMAAQAAAAAAAAAAAAAAAQAB//8ADwABAAAAAAAAAAAAAgAANzkBAAAAAAEAAAAAAAAAAAACAAA3OQEAAAAAAQAAAAAAAAAAAAIAADc5AQAAAAABAEUAUQO7AvgAGgAAARQHAQYjIicBJjU0PwE2MzIfAQE2MzIfARYVA7sQ/hQQFhcQ/uMQEE4QFxcQqAF2EBcXEE4QAnMWEP4UEBABHRAXFhBOEBCoAXcQEE4QFwAAAAABAAABbgMlAkkAFAAAARUUBwYjISInJj0BNDc2MyEyFxYVAyUQEBf9SRcQEBAQFwK3FxAQAhJtFxAQEBAXbRcQEBAQFwAAAAABAAAASQMlA24ALAAAARUUBwYrARUUBwYrASInJj0BIyInJj0BNDc2OwE1NDc2OwEyFxYdATMyFxYVAyUQEBfuEBAXbhYQEO4XEBAQEBfuEBAWbhcQEO4XEBACEm0XEBDuFxAQEBAX7hAQF20XEBDuFxAQEBAX7hAQFwAAAQAAAAIAAHRSzT9fDzz1AAsEAAAAAADRsdR3AAAAANGx1HcAAAAAA7sDbgAAAAgAAgAAAAAAAAABAAADwP/AAAAEAAAAAAADuwABAAAAAAAAAAAAAAAAAAAABwQAAAAAAAAAAAAAAAIAAAAEAABFAyUAAAMlAAAAAAAAAAoAFAAeAE4AcgCwAAEAAAAHAC0AAQAAAAAAAgAAAAAAAAAAAAAAAAAAAAAAAAAOAK4AAQAAAAAAAQAIAAAAAQAAAAAAAgAHAGkAAQAAAAAAAwAIADkAAQAAAAAABAAIAH4AAQAAAAAABQALABgAAQAAAAAABgAIAFEAAQAAAAAACgAaAJYAAwABBAkAAQAQAAgAAwABBAkAAgAOAHAAAwABBAkAAwAQAEEAAwABBAkABAAQAIYAAwABBAkABQAWACMAAwABBAkABgAQAFkAAwABBAkACgA0ALBDaGVja2JveABDAGgAZQBjAGsAYgBvAHhWZXJzaW9uIDIuMABWAGUAcgBzAGkAbwBuACAAMgAuADBDaGVja2JveABDAGgAZQBjAGsAYgBvAHhDaGVja2JveABDAGgAZQBjAGsAYgBvAHhSZWd1bGFyAFIAZQBnAHUAbABhAHJDaGVja2JveABDAGgAZQBjAGsAYgBvAHhGb250IGdlbmVyYXRlZCBieSBJY29Nb29uLgBGAG8AbgB0ACAAZwBlAG4AZQByAGEAdABlAGQAIABiAHkAIABJAGMAbwBNAG8AbwBuAC4AAAADAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA) format('truetype')}.ui.checkbox .box:after,.ui.checkbox label:after{font-family:Checkbox}.ui.checkbox input:checked~.box:after,.ui.checkbox input:checked~label:after{content:'\e800'}.ui.checkbox input:indeterminate~.box:after,.ui.checkbox input:indeterminate~label:after{font-size:12px;content:'\e801'}/*! - * # Semantic UI 2.4.0 - Dimmer - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */.dimmable:not(body){position:relative}.ui.dimmer{display:none;position:absolute;top:0!important;left:0!important;width:100%;height:100%;text-align:center;vertical-align:middle;padding:1em;background-color:rgba(0,0,0,.85);opacity:0;line-height:1;-webkit-animation-fill-mode:both;animation-fill-mode:both;-webkit-animation-duration:.5s;animation-duration:.5s;-webkit-transition:background-color .5s linear;transition:background-color .5s linear;-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column;-webkit-box-align:center;-ms-flex-align:center;align-items:center;-webkit-box-pack:center;-ms-flex-pack:center;justify-content:center;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;will-change:opacity;z-index:1000}.ui.dimmer>.content{-webkit-user-select:text;-moz-user-select:text;-ms-user-select:text;user-select:text;color:#fff}.ui.segment>.ui.dimmer{border-radius:inherit!important}.ui.dimmer:not(.inverted)::-webkit-scrollbar-track{background:rgba(255,255,255,.1)}.ui.dimmer:not(.inverted)::-webkit-scrollbar-thumb{background:rgba(255,255,255,.25)}.ui.dimmer:not(.inverted)::-webkit-scrollbar-thumb:window-inactive{background:rgba(255,255,255,.15)}.ui.dimmer:not(.inverted)::-webkit-scrollbar-thumb:hover{background:rgba(255,255,255,.35)}.animating.dimmable:not(body),.dimmed.dimmable:not(body){overflow:hidden}.dimmed.dimmable>.ui.animating.dimmer,.dimmed.dimmable>.ui.visible.dimmer,.ui.active.dimmer{display:-webkit-box;display:-ms-flexbox;display:flex;opacity:1}.ui.disabled.dimmer{width:0!important;height:0!important}.dimmed.dimmable>.ui.animating.legacy.dimmer,.dimmed.dimmable>.ui.visible.legacy.dimmer,.ui.active.legacy.dimmer{display:block}.ui[class*="top aligned"].dimmer{-webkit-box-pack:start;-ms-flex-pack:start;justify-content:flex-start}.ui[class*="bottom aligned"].dimmer{-webkit-box-pack:end;-ms-flex-pack:end;justify-content:flex-end}.ui.page.dimmer{position:fixed;-webkit-transform-style:'';transform-style:'';-webkit-perspective:2000px;perspective:2000px;-webkit-transform-origin:center center;transform-origin:center center}body.animating.in.dimmable,body.dimmed.dimmable{overflow:hidden}body.dimmable>.dimmer{position:fixed}.blurring.dimmable>:not(.dimmer){-webkit-filter:blur(0) grayscale(0);filter:blur(0) grayscale(0);-webkit-transition:.8s -webkit-filter ease;transition:.8s -webkit-filter ease;transition:.8s filter ease;transition:.8s filter ease,.8s -webkit-filter ease}.blurring.dimmed.dimmable>:not(.dimmer){-webkit-filter:blur(5px) grayscale(.7);filter:blur(5px) grayscale(.7)}.blurring.dimmable>.dimmer{background-color:rgba(0,0,0,.6)}.blurring.dimmable>.inverted.dimmer{background-color:rgba(255,255,255,.6)}.ui.dimmer>.top.aligned.content>*{vertical-align:top}.ui.dimmer>.bottom.aligned.content>*{vertical-align:bottom}.ui.inverted.dimmer{background-color:rgba(255,255,255,.85)}.ui.inverted.dimmer>.content>*{color:#fff}.ui.simple.dimmer{display:block;overflow:hidden;opacity:1;width:0%;height:0%;z-index:-100;background-color:rgba(0,0,0,0)}.dimmed.dimmable>.ui.simple.dimmer{overflow:visible;opacity:1;width:100%;height:100%;background-color:rgba(0,0,0,.85);z-index:1}.ui.simple.inverted.dimmer{background-color:rgba(255,255,255,0)}.dimmed.dimmable>.ui.simple.inverted.dimmer{background-color:rgba(255,255,255,.85)}/*! - * # Semantic UI 2.4.0 - Dropdown - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */.ui.dropdown{cursor:pointer;position:relative;display:inline-block;outline:0;text-align:left;-webkit-transition:width .1s ease,-webkit-box-shadow .1s ease;transition:width .1s ease,-webkit-box-shadow .1s ease;transition:box-shadow .1s ease,width .1s ease;transition:box-shadow .1s ease,width .1s ease,-webkit-box-shadow .1s ease;-webkit-tap-highlight-color:transparent}.ui.dropdown .menu{cursor:auto;position:absolute;display:none;outline:0;top:100%;min-width:-webkit-max-content;min-width:-moz-max-content;min-width:max-content;margin:0;padding:0 0;background:#fff;font-size:1em;text-shadow:none;text-align:left;-webkit-box-shadow:0 2px 3px 0 rgba(34,36,38,.15);box-shadow:0 2px 3px 0 rgba(34,36,38,.15);border:1px solid rgba(34,36,38,.15);border-radius:.28571429rem;-webkit-transition:opacity .1s ease;transition:opacity .1s ease;z-index:11;will-change:transform,opacity}.ui.dropdown .menu>*{white-space:nowrap}.ui.dropdown>input:not(.search):first-child,.ui.dropdown>select{display:none!important}.ui.dropdown>.dropdown.icon{position:relative;width:auto;font-size:.85714286em;margin:0 0 0 1em}.ui.dropdown .menu>.item .dropdown.icon{width:auto;float:right;margin:0 0 0 1em}.ui.dropdown .menu>.item .dropdown.icon+.text{margin-right:1em}.ui.dropdown>.text{display:inline-block;-webkit-transition:none;transition:none}.ui.dropdown .menu>.item{position:relative;cursor:pointer;display:block;border:none;height:auto;text-align:left;border-top:none;line-height:1em;color:rgba(0,0,0,.87);padding:.78571429rem 1.14285714rem!important;font-size:1rem;text-transform:none;font-weight:400;-webkit-box-shadow:none;box-shadow:none;-webkit-touch-callout:none}.ui.dropdown .menu>.item:first-child{border-top-width:0}.ui.dropdown .menu .item>[class*="right floated"],.ui.dropdown>.text>[class*="right floated"]{float:right!important;margin-right:0!important;margin-left:1em!important}.ui.dropdown .menu .item>[class*="left floated"],.ui.dropdown>.text>[class*="left floated"]{float:left!important;margin-left:0!important;margin-right:1em!important}.ui.dropdown .menu .item>.flag.floated,.ui.dropdown .menu .item>.icon.floated,.ui.dropdown .menu .item>.image.floated,.ui.dropdown .menu .item>img.floated{margin-top:0}.ui.dropdown .menu>.header{margin:1rem 0 .75rem;padding:0 1.14285714rem;color:rgba(0,0,0,.85);font-size:.78571429em;font-weight:700;text-transform:uppercase}.ui.dropdown .menu>.divider{border-top:1px solid rgba(34,36,38,.1);height:0;margin:.5em 0}.ui.dropdown.dropdown .menu>.input{width:auto;display:-webkit-box;display:-ms-flexbox;display:flex;margin:1.14285714rem .78571429rem;min-width:10rem}.ui.dropdown .menu>.header+.input{margin-top:0}.ui.dropdown .menu>.input:not(.transparent) input{padding:.5em 1em}.ui.dropdown .menu>.input:not(.transparent) .button,.ui.dropdown .menu>.input:not(.transparent) .icon,.ui.dropdown .menu>.input:not(.transparent) .label{padding-top:.5em;padding-bottom:.5em}.ui.dropdown .menu>.item>.description,.ui.dropdown>.text>.description{float:right;margin:0 0 0 1em;color:rgba(0,0,0,.4)}.ui.dropdown .menu>.message{padding:.78571429rem 1.14285714rem;font-weight:400}.ui.dropdown .menu>.message:not(.ui){color:rgba(0,0,0,.4)}.ui.dropdown .menu .menu{top:0!important;left:100%;right:auto;margin:0 0 0 -.5em!important;border-radius:.28571429rem!important;z-index:21!important}.ui.dropdown .menu .menu:after{display:none}.ui.dropdown>.text>.flag,.ui.dropdown>.text>.icon,.ui.dropdown>.text>.image,.ui.dropdown>.text>.label,.ui.dropdown>.text>img{margin-top:0}.ui.dropdown .menu>.item>.flag,.ui.dropdown .menu>.item>.icon,.ui.dropdown .menu>.item>.image,.ui.dropdown .menu>.item>.label,.ui.dropdown .menu>.item>img{margin-top:0}.ui.dropdown .menu>.item>.flag,.ui.dropdown .menu>.item>.icon,.ui.dropdown .menu>.item>.image,.ui.dropdown .menu>.item>.label,.ui.dropdown .menu>.item>img,.ui.dropdown>.text>.flag,.ui.dropdown>.text>.icon,.ui.dropdown>.text>.image,.ui.dropdown>.text>.label,.ui.dropdown>.text>img{margin-left:0;float:none;margin-right:.78571429rem}.ui.dropdown .menu>.item>.image,.ui.dropdown .menu>.item>img,.ui.dropdown>.text>.image,.ui.dropdown>.text>img{display:inline-block;vertical-align:top;width:auto;margin-top:-.5em;margin-bottom:-.5em;max-height:2em}.ui.dropdown .ui.menu>.item:before,.ui.menu .ui.dropdown .menu>.item:before{display:none}.ui.menu .ui.dropdown .menu .active.item{border-left:none}.ui.buttons>.ui.dropdown:last-child .menu,.ui.menu .right.dropdown.item .menu,.ui.menu .right.menu .dropdown:last-child .menu{left:auto;right:0}.ui.label.dropdown .menu{min-width:100%}.ui.dropdown.icon.button>.dropdown.icon{margin:0}.ui.button.dropdown .menu{min-width:100%}.ui.selection.dropdown{cursor:pointer;word-wrap:break-word;line-height:1em;white-space:normal;outline:0;-webkit-transform:rotateZ(0);transform:rotateZ(0);min-width:14em;min-height:2.71428571em;background:#fff;display:inline-block;padding:.78571429em 2.1em .78571429em 1em;color:rgba(0,0,0,.87);-webkit-box-shadow:none;box-shadow:none;border:1px solid rgba(34,36,38,.15);border-radius:.28571429rem;-webkit-transition:width .1s ease,-webkit-box-shadow .1s ease;transition:width .1s ease,-webkit-box-shadow .1s ease;transition:box-shadow .1s ease,width .1s ease;transition:box-shadow .1s ease,width .1s ease,-webkit-box-shadow .1s ease}.ui.selection.dropdown.active,.ui.selection.dropdown.visible{z-index:10}select.ui.dropdown{height:38px;padding:.5em;border:1px solid rgba(34,36,38,.15);visibility:visible}.ui.selection.dropdown>.delete.icon,.ui.selection.dropdown>.dropdown.icon,.ui.selection.dropdown>.search.icon{cursor:pointer;position:absolute;width:auto;height:auto;line-height:1.21428571em;top:.78571429em;right:1em;z-index:3;margin:-.78571429em;padding:.91666667em;opacity:.8;-webkit-transition:opacity .1s ease;transition:opacity .1s ease}.ui.compact.selection.dropdown{min-width:0}.ui.selection.dropdown .menu{overflow-x:hidden;overflow-y:auto;-webkit-backface-visibility:hidden;backface-visibility:hidden;-webkit-overflow-scrolling:touch;border-top-width:0!important;width:auto;outline:0;margin:0 -1px;min-width:calc(100% + 2px);width:calc(100% + 2px);border-radius:0 0 .28571429rem .28571429rem;-webkit-box-shadow:0 2px 3px 0 rgba(34,36,38,.15);box-shadow:0 2px 3px 0 rgba(34,36,38,.15);-webkit-transition:opacity .1s ease;transition:opacity .1s ease}.ui.selection.dropdown .menu:after,.ui.selection.dropdown .menu:before{display:none}.ui.selection.dropdown .menu>.message{padding:.78571429rem 1.14285714rem}@media only screen and (max-width:767px){.ui.selection.dropdown .menu{max-height:8.01428571rem}}@media only screen and (min-width:768px){.ui.selection.dropdown .menu{max-height:10.68571429rem}}@media only screen and (min-width:992px){.ui.selection.dropdown .menu{max-height:16.02857143rem}}@media only screen and (min-width:1920px){.ui.selection.dropdown .menu{max-height:21.37142857rem}}.ui.selection.dropdown .menu>.item{border-top:1px solid #fafafa;padding:.78571429rem 1.14285714rem!important;white-space:normal;word-wrap:normal}.ui.selection.dropdown .menu>.hidden.addition.item{display:none}.ui.selection.dropdown:hover{border-color:rgba(34,36,38,.35);-webkit-box-shadow:none;box-shadow:none}.ui.selection.active.dropdown{border-color:#96c8da;-webkit-box-shadow:0 2px 3px 0 rgba(34,36,38,.15);box-shadow:0 2px 3px 0 rgba(34,36,38,.15)}.ui.selection.active.dropdown .menu{border-color:#96c8da;-webkit-box-shadow:0 2px 3px 0 rgba(34,36,38,.15);box-shadow:0 2px 3px 0 rgba(34,36,38,.15)}.ui.selection.dropdown:focus{border-color:#96c8da;-webkit-box-shadow:none;box-shadow:none}.ui.selection.dropdown:focus .menu{border-color:#96c8da;-webkit-box-shadow:0 2px 3px 0 rgba(34,36,38,.15);box-shadow:0 2px 3px 0 rgba(34,36,38,.15)}.ui.selection.visible.dropdown>.text:not(.default){font-weight:400;color:rgba(0,0,0,.8)}.ui.selection.active.dropdown:hover{border-color:#96c8da;-webkit-box-shadow:0 2px 3px 0 rgba(34,36,38,.15);box-shadow:0 2px 3px 0 rgba(34,36,38,.15)}.ui.selection.active.dropdown:hover .menu{border-color:#96c8da;-webkit-box-shadow:0 2px 3px 0 rgba(34,36,38,.15);box-shadow:0 2px 3px 0 rgba(34,36,38,.15)}.ui.active.selection.dropdown>.dropdown.icon,.ui.visible.selection.dropdown>.dropdown.icon{opacity:'';z-index:3}.ui.active.selection.dropdown{border-bottom-left-radius:0!important;border-bottom-right-radius:0!important}.ui.active.empty.selection.dropdown{border-radius:.28571429rem!important;-webkit-box-shadow:none!important;box-shadow:none!important}.ui.active.empty.selection.dropdown .menu{border:none!important;-webkit-box-shadow:none!important;box-shadow:none!important}.ui.search.dropdown{min-width:''}.ui.search.dropdown>input.search{background:none transparent!important;border:none!important;-webkit-box-shadow:none!important;box-shadow:none!important;cursor:text;top:0;left:1px;width:100%;outline:0;-webkit-tap-highlight-color:rgba(255,255,255,0);padding:inherit}.ui.search.dropdown>input.search{position:absolute;z-index:2}.ui.search.dropdown>.text{cursor:text;position:relative;left:1px;z-index:3}.ui.search.selection.dropdown>input.search{line-height:1.21428571em;padding:.67857143em 2.1em .67857143em 1em}.ui.search.selection.dropdown>span.sizer{line-height:1.21428571em;padding:.67857143em 2.1em .67857143em 1em;display:none;white-space:pre}.ui.search.dropdown.active>input.search,.ui.search.dropdown.visible>input.search{cursor:auto}.ui.search.dropdown.active>.text,.ui.search.dropdown.visible>.text{pointer-events:none}.ui.active.search.dropdown input.search:focus+.text .flag,.ui.active.search.dropdown input.search:focus+.text .icon{opacity:.45}.ui.active.search.dropdown input.search:focus+.text{color:rgba(115,115,115,.87)!important}.ui.search.dropdown .menu{overflow-x:hidden;overflow-y:auto;-webkit-backface-visibility:hidden;backface-visibility:hidden;-webkit-overflow-scrolling:touch}@media only screen and (max-width:767px){.ui.search.dropdown .menu{max-height:8.01428571rem}}@media only screen and (min-width:768px){.ui.search.dropdown .menu{max-height:10.68571429rem}}@media only screen and (min-width:992px){.ui.search.dropdown .menu{max-height:16.02857143rem}}@media only screen and (min-width:1920px){.ui.search.dropdown .menu{max-height:21.37142857rem}}.ui.multiple.dropdown{padding:.22619048em 2.1em .22619048em .35714286em}.ui.multiple.dropdown .menu{cursor:auto}.ui.multiple.search.dropdown,.ui.multiple.search.dropdown>input.search{cursor:text}.ui.multiple.dropdown>.label{-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;display:inline-block;vertical-align:top;white-space:normal;font-size:1em;padding:.35714286em .78571429em;margin:.14285714rem .28571429rem .14285714rem 0;-webkit-box-shadow:0 0 0 1px rgba(34,36,38,.15) inset;box-shadow:0 0 0 1px rgba(34,36,38,.15) inset}.ui.multiple.dropdown .dropdown.icon{margin:'';padding:''}.ui.multiple.dropdown>.text{position:static;padding:0;max-width:100%;margin:.45238095em 0 .45238095em .64285714em;line-height:1.21428571em}.ui.multiple.dropdown>.label~input.search{margin-left:.14285714em!important}.ui.multiple.dropdown>.label~.text{display:none}.ui.multiple.search.dropdown>.text{display:inline-block;position:absolute;top:0;left:0;padding:inherit;margin:.45238095em 0 .45238095em .64285714em;line-height:1.21428571em}.ui.multiple.search.dropdown>.label~.text{display:none}.ui.multiple.search.dropdown>input.search{position:static;padding:0;max-width:100%;margin:.45238095em 0 .45238095em .64285714em;width:2.2em;line-height:1.21428571em}.ui.inline.dropdown{cursor:pointer;display:inline-block;color:inherit}.ui.inline.dropdown .dropdown.icon{margin:0 .21428571em 0 .21428571em;vertical-align:baseline}.ui.inline.dropdown>.text{font-weight:700}.ui.inline.dropdown .menu{cursor:auto;margin-top:.21428571em;border-radius:.28571429rem}.ui.dropdown .menu .active.item{background:0 0;font-weight:700;color:rgba(0,0,0,.95);-webkit-box-shadow:none;box-shadow:none;z-index:12}.ui.dropdown .menu>.item:hover{background:rgba(0,0,0,.05);color:rgba(0,0,0,.95);z-index:13}.ui.loading.dropdown>i.icon{height:1em!important}.ui.loading.selection.dropdown>i.icon{padding:1.5em 1.28571429em!important}.ui.loading.dropdown>i.icon:before{position:absolute;content:'';top:50%;left:50%;margin:-.64285714em 0 0 -.64285714em;width:1.28571429em;height:1.28571429em;border-radius:500rem;border:.2em solid rgba(0,0,0,.1)}.ui.loading.dropdown>i.icon:after{position:absolute;content:'';top:50%;left:50%;-webkit-box-shadow:0 0 0 1px transparent;box-shadow:0 0 0 1px transparent;margin:-.64285714em 0 0 -.64285714em;width:1.28571429em;height:1.28571429em;-webkit-animation:dropdown-spin .6s linear;animation:dropdown-spin .6s linear;-webkit-animation-iteration-count:infinite;animation-iteration-count:infinite;border-radius:500rem;border-color:#767676 transparent transparent;border-style:solid;border-width:.2em}.ui.loading.dropdown.button>i.icon:after,.ui.loading.dropdown.button>i.icon:before{display:none}@-webkit-keyframes dropdown-spin{from{-webkit-transform:rotate(0);transform:rotate(0)}to{-webkit-transform:rotate(360deg);transform:rotate(360deg)}}@keyframes dropdown-spin{from{-webkit-transform:rotate(0);transform:rotate(0)}to{-webkit-transform:rotate(360deg);transform:rotate(360deg)}}.ui.default.dropdown:not(.button)>.text,.ui.dropdown:not(.button)>.default.text{color:rgba(191,191,191,.87)}.ui.default.dropdown:not(.button)>input:focus~.text,.ui.dropdown:not(.button)>input:focus~.default.text{color:rgba(115,115,115,.87)}.ui.loading.dropdown>.text{-webkit-transition:none;transition:none}.ui.dropdown .loading.menu{display:block;visibility:hidden;z-index:-1}.ui.dropdown>.loading.menu{left:0!important;right:auto!important}.ui.dropdown>.menu .loading.menu{left:100%!important;right:auto!important}.ui.dropdown .menu .selected.item,.ui.dropdown.selected{background:rgba(0,0,0,.03);color:rgba(0,0,0,.95)}.ui.dropdown>.filtered.text{visibility:hidden}.ui.dropdown .filtered.item{display:none!important}.ui.dropdown.error,.ui.dropdown.error>.default.text,.ui.dropdown.error>.text{color:#9f3a38}.ui.selection.dropdown.error{background:#fff6f6;border-color:#e0b4b4}.ui.selection.dropdown.error:hover{border-color:#e0b4b4}.ui.dropdown.error>.menu,.ui.dropdown.error>.menu .menu{border-color:#e0b4b4}.ui.dropdown.error>.menu>.item{color:#9f3a38}.ui.multiple.selection.error.dropdown>.label{border-color:#e0b4b4}.ui.dropdown.error>.menu>.item:hover{background-color:#fff2f2}.ui.dropdown.error>.menu .active.item{background-color:#fdcfcf}.ui.dropdown>.clear.dropdown.icon{opacity:.8;-webkit-transition:opacity .1s ease;transition:opacity .1s ease}.ui.dropdown>.clear.dropdown.icon:hover{opacity:1}.ui.disabled.dropdown,.ui.dropdown .menu>.disabled.item{cursor:default;pointer-events:none;opacity:.45}.ui.dropdown .menu{left:0}.ui.dropdown .menu .right.menu,.ui.dropdown .right.menu>.menu{left:100%!important;right:auto!important;border-radius:.28571429rem!important}.ui.dropdown>.left.menu{left:auto!important;right:0!important}.ui.dropdown .menu .left.menu,.ui.dropdown>.left.menu .menu{left:auto;right:100%;margin:0 -.5em 0 0!important;border-radius:.28571429rem!important}.ui.dropdown .item .left.dropdown.icon,.ui.dropdown .left.menu .item .dropdown.icon{width:auto;float:left;margin:0}.ui.dropdown .item .left.dropdown.icon,.ui.dropdown .left.menu .item .dropdown.icon{width:auto;float:left;margin:0}.ui.dropdown .item .left.dropdown.icon+.text,.ui.dropdown .left.menu .item .dropdown.icon+.text{margin-left:1em;margin-right:0}.ui.upward.dropdown>.menu{top:auto;bottom:100%;-webkit-box-shadow:0 0 3px 0 rgba(0,0,0,.08);box-shadow:0 0 3px 0 rgba(0,0,0,.08);border-radius:.28571429rem .28571429rem 0 0}.ui.dropdown .upward.menu{top:auto!important;bottom:0!important}.ui.simple.upward.active.dropdown,.ui.simple.upward.dropdown:hover{border-radius:.28571429rem .28571429rem 0 0!important}.ui.upward.dropdown.button:not(.pointing):not(.floating).active{border-radius:.28571429rem .28571429rem 0 0}.ui.upward.selection.dropdown .menu{border-top-width:1px!important;border-bottom-width:0!important;-webkit-box-shadow:0 -2px 3px 0 rgba(0,0,0,.08);box-shadow:0 -2px 3px 0 rgba(0,0,0,.08)}.ui.upward.selection.dropdown:hover{-webkit-box-shadow:0 0 2px 0 rgba(0,0,0,.05);box-shadow:0 0 2px 0 rgba(0,0,0,.05)}.ui.active.upward.selection.dropdown{border-radius:0 0 .28571429rem .28571429rem!important}.ui.upward.selection.dropdown.visible{-webkit-box-shadow:0 0 3px 0 rgba(0,0,0,.08);box-shadow:0 0 3px 0 rgba(0,0,0,.08);border-radius:0 0 .28571429rem .28571429rem!important}.ui.upward.active.selection.dropdown:hover{-webkit-box-shadow:0 0 3px 0 rgba(0,0,0,.05);box-shadow:0 0 3px 0 rgba(0,0,0,.05)}.ui.upward.active.selection.dropdown:hover .menu{-webkit-box-shadow:0 -2px 3px 0 rgba(0,0,0,.08);box-shadow:0 -2px 3px 0 rgba(0,0,0,.08)}.ui.dropdown .scrolling.menu,.ui.scrolling.dropdown .menu{overflow-x:hidden;overflow-y:auto}.ui.scrolling.dropdown .menu{overflow-x:hidden;overflow-y:auto;-webkit-backface-visibility:hidden;backface-visibility:hidden;-webkit-overflow-scrolling:touch;min-width:100%!important;width:auto!important}.ui.dropdown .scrolling.menu{position:static;overflow-y:auto;border:none;-webkit-box-shadow:none!important;box-shadow:none!important;border-radius:0!important;margin:0!important;min-width:100%!important;width:auto!important;border-top:1px solid rgba(34,36,38,.15)}.ui.dropdown .scrolling.menu>.item.item.item,.ui.scrolling.dropdown .menu .item.item.item{border-top:none}.ui.dropdown .scrolling.menu .item:first-child,.ui.scrolling.dropdown .menu .item:first-child{border-top:none}.ui.dropdown>.animating.menu .scrolling.menu,.ui.dropdown>.visible.menu .scrolling.menu{display:block}@media all and (-ms-high-contrast:none){.ui.dropdown .scrolling.menu,.ui.scrolling.dropdown .menu{min-width:calc(100% - 17px)}}@media only screen and (max-width:767px){.ui.dropdown .scrolling.menu,.ui.scrolling.dropdown .menu{max-height:10.28571429rem}}@media only screen and (min-width:768px){.ui.dropdown .scrolling.menu,.ui.scrolling.dropdown .menu{max-height:15.42857143rem}}@media only screen and (min-width:992px){.ui.dropdown .scrolling.menu,.ui.scrolling.dropdown .menu{max-height:20.57142857rem}}@media only screen and (min-width:1920px){.ui.dropdown .scrolling.menu,.ui.scrolling.dropdown .menu{max-height:20.57142857rem}}.ui.simple.dropdown .menu:after,.ui.simple.dropdown .menu:before{display:none}.ui.simple.dropdown .menu{position:absolute;display:block;overflow:hidden;top:-9999px!important;opacity:0;width:0;height:0;-webkit-transition:opacity .1s ease;transition:opacity .1s ease}.ui.simple.active.dropdown,.ui.simple.dropdown:hover{border-bottom-left-radius:0!important;border-bottom-right-radius:0!important}.ui.simple.active.dropdown>.menu,.ui.simple.dropdown:hover>.menu{overflow:visible;width:auto;height:auto;top:100%!important;opacity:1}.ui.simple.dropdown:hover>.menu>.item:hover>.menu,.ui.simple.dropdown>.menu>.item:active>.menu{overflow:visible;width:auto;height:auto;top:0!important;left:100%!important;opacity:1}.ui.simple.disabled.dropdown:hover .menu{display:none;height:0;width:0;overflow:hidden}.ui.simple.visible.dropdown>.menu{display:block}.ui.fluid.dropdown{display:block;width:100%;min-width:0}.ui.fluid.dropdown>.dropdown.icon{float:right}.ui.floating.dropdown .menu{left:0;right:auto;-webkit-box-shadow:0 2px 4px 0 rgba(34,36,38,.12),0 2px 10px 0 rgba(34,36,38,.15)!important;box-shadow:0 2px 4px 0 rgba(34,36,38,.12),0 2px 10px 0 rgba(34,36,38,.15)!important;border-radius:.28571429rem!important}.ui.floating.dropdown>.menu{margin-top:.5em!important;border-radius:.28571429rem!important}.ui.pointing.dropdown>.menu{top:100%;margin-top:.78571429rem;border-radius:.28571429rem}.ui.pointing.dropdown>.menu:after{display:block;position:absolute;pointer-events:none;content:'';visibility:visible;-webkit-transform:rotate(45deg);transform:rotate(45deg);width:.5em;height:.5em;-webkit-box-shadow:-1px -1px 0 0 rgba(34,36,38,.15);box-shadow:-1px -1px 0 0 rgba(34,36,38,.15);background:#fff;z-index:2}.ui.pointing.dropdown>.menu:after{top:-.25em;left:50%;margin:0 0 0 -.25em}.ui.top.left.pointing.dropdown>.menu{top:100%;bottom:auto;left:0;right:auto;margin:1em 0 0}.ui.top.left.pointing.dropdown>.menu{top:100%;bottom:auto;left:0;right:auto;margin:1em 0 0}.ui.top.left.pointing.dropdown>.menu:after{top:-.25em;left:1em;right:auto;margin:0;-webkit-transform:rotate(45deg);transform:rotate(45deg)}.ui.top.right.pointing.dropdown>.menu{top:100%;bottom:auto;right:0;left:auto;margin:1em 0 0}.ui.top.pointing.dropdown>.left.menu:after,.ui.top.right.pointing.dropdown>.menu:after{top:-.25em;left:auto!important;right:1em!important;margin:0;-webkit-transform:rotate(45deg);transform:rotate(45deg)}.ui.left.pointing.dropdown>.menu{top:0;left:100%;right:auto;margin:0 0 0 1em}.ui.left.pointing.dropdown>.menu:after{top:1em;left:-.25em;margin:0;-webkit-transform:rotate(-45deg);transform:rotate(-45deg)}.ui.left:not(.top):not(.bottom).pointing.dropdown>.left.menu{left:auto!important;right:100%!important;margin:0 1em 0 0}.ui.left:not(.top):not(.bottom).pointing.dropdown>.left.menu:after{top:1em;left:auto;right:-.25em;margin:0;-webkit-transform:rotate(135deg);transform:rotate(135deg)}.ui.right.pointing.dropdown>.menu{top:0;left:auto;right:100%;margin:0 1em 0 0}.ui.right.pointing.dropdown>.menu:after{top:1em;left:auto;right:-.25em;margin:0;-webkit-transform:rotate(135deg);transform:rotate(135deg)}.ui.bottom.pointing.dropdown>.menu{top:auto;bottom:100%;left:0;right:auto;margin:0 0 1em}.ui.bottom.pointing.dropdown>.menu:after{top:auto;bottom:-.25em;right:auto;margin:0;-webkit-transform:rotate(-135deg);transform:rotate(-135deg)}.ui.bottom.pointing.dropdown>.menu .menu{top:auto!important;bottom:0!important}.ui.bottom.left.pointing.dropdown>.menu{left:0;right:auto}.ui.bottom.left.pointing.dropdown>.menu:after{left:1em;right:auto}.ui.bottom.right.pointing.dropdown>.menu{right:0;left:auto}.ui.bottom.right.pointing.dropdown>.menu:after{left:auto;right:1em}.ui.pointing.upward.dropdown .menu,.ui.top.pointing.upward.dropdown .menu{top:auto!important;bottom:100%!important;margin:0 0 .78571429rem;border-radius:.28571429rem}.ui.pointing.upward.dropdown .menu:after,.ui.top.pointing.upward.dropdown .menu:after{top:100%!important;bottom:auto!important;-webkit-box-shadow:1px 1px 0 0 rgba(34,36,38,.15);box-shadow:1px 1px 0 0 rgba(34,36,38,.15);margin:-.25em 0 0}.ui.right.pointing.upward.dropdown:not(.top):not(.bottom) .menu{top:auto!important;bottom:0!important;margin:0 1em 0 0}.ui.right.pointing.upward.dropdown:not(.top):not(.bottom) .menu:after{top:auto!important;bottom:0!important;margin:0 0 1em 0;-webkit-box-shadow:-1px -1px 0 0 rgba(34,36,38,.15);box-shadow:-1px -1px 0 0 rgba(34,36,38,.15)}.ui.left.pointing.upward.dropdown:not(.top):not(.bottom) .menu{top:auto!important;bottom:0!important;margin:0 0 0 1em}.ui.left.pointing.upward.dropdown:not(.top):not(.bottom) .menu:after{top:auto!important;bottom:0!important;margin:0 0 1em 0;-webkit-box-shadow:-1px -1px 0 0 rgba(34,36,38,.15);box-shadow:-1px -1px 0 0 rgba(34,36,38,.15)}@font-face{font-family:Dropdown;src:url(data:application/font-woff;charset=utf-8;base64,d09GRgABAAAAAAVgAA8AAAAACFAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAABGRlRNAAABWAAAABwAAAAchGgaq0dERUYAAAF0AAAAHAAAAB4AJwAPT1MvMgAAAZAAAABDAAAAVnW4TJdjbWFwAAAB1AAAAEsAAAFS8CcaqmN2dCAAAAIgAAAABAAAAAQAEQFEZ2FzcAAAAiQAAAAIAAAACP//AANnbHlmAAACLAAAAQoAAAGkrRHP9WhlYWQAAAM4AAAAMAAAADYPK8YyaGhlYQAAA2gAAAAdAAAAJANCAb1obXR4AAADiAAAACIAAAAiCBkAOGxvY2EAAAOsAAAAFAAAABQBnAIybWF4cAAAA8AAAAAfAAAAIAEVAF5uYW1lAAAD4AAAATAAAAKMFGlj5HBvc3QAAAUQAAAARgAAAHJoedjqd2ViZgAABVgAAAAGAAAABrO7W5UAAAABAAAAANXulPUAAAAA1r4hgAAAAADXu2Q1eNpjYGRgYOABYjEgZmJgBEIOIGYB8xgAA/YAN3jaY2BktGOcwMDKwMI4jTGNgYHBHUp/ZZBkaGFgYGJgZWbACgLSXFMYHFT/fLjFeOD/AQY9xjMMbkBhRpAcAN48DQYAeNpjYGBgZoBgGQZGBhDwAfIYwXwWBgMgzQGETAwMqn8+8H649f8/lHX9//9b7Pzf+fWgusCAkY0BzmUE6gHpQwGMDMMeAACbxg7SAAARAUQAAAAB//8AAnjadZBPSsNAGMXfS+yMqYgOhpSuSlKadmUhiVEhEMQzFF22m17BbbvzCh5BXCUn6EG8gjeQ4DepwYo4i+/ffL95j4EDA+CFC7jQuKyIeVHrI3wkleq9F7XrSInKteOeHdda8bOoaeepSc00NWPz/LRec9G8GabyGtEdF7h19z033GAMTK7zbM42xNEZpzYof0RtQ5CUHAQJ73OtVyutc+3b7Ou//b8XNlsPx3jgjUifABdhEohKJJL5iM5p39uqc7X1+sRQSqmGrUVhlsJ4lpmEUVwyT8SUYtg0P9DyNzPADDs+tjrGV6KRCRfsui3eHcL4/p8ZXvfMlcnEU+CLv7hDykOP+AKTPTxbAAB42mNgZGBgAGKuf5KP4vltvjLIMzGAwLV9ig0g+vruFFMQzdjACOJzMIClARh0CTJ42mNgZGBgPPD/AJD8wgAEjA0MjAyogAMAbOQEAQAAAAC7ABEAAAAAAKoAAAH0AAABgAAAAUAACAFAAAgAwAAXAAAAAAAAACoAKgAqADIAbACGAKAAugDSeNpjYGRgYOBkUGFgYgABEMkFhAwM/xn0QAIADdUBdAB42qWQvUoDQRSFv3GjaISUQaymSmGxJoGAsRC0iPYLsU50Y6IxrvlRtPCJJKUPIBb+PIHv4EN4djKuKAqCDHfmu+feOdwZoMCUAJNbAlYUMzaUlM14jjxbngOq7HnOia89z1Pk1vMCa9x7ztPkzfMyJbPj+ZGi6Xp+omxuPD+zaD7meaFg7mb8GrBqHmhwxoAxlm0uiRkpP9X5m26pKRoMxTGR1D49Dv/Yb/91o6l8qL6eu5n2hZQzn68utR9m3FU2cB4t9cdSLG2utI+44Eh/P9bqKO+oJ/WxmXssj77YkrjasZQD6SFddythk3Wtzrf+UF2p076Udla1VNzsERP3kkjVRKel7mp1udXYcHtZSlV7RfmJe1GiFWveluaeKD5/MuJcSk8Tpm/vvwPIbmJleNpjYGKAAFYG7ICTgYGRiZGZkYWRlZGNkZ2Rg5GTLT2nsiDDEEIZsZfmZRqZujmDaDcDAxcI7WIOpS2gtCWUdgQAZkcSmQAAAAFblbO6AAA=) format('woff');font-weight:400;font-style:normal}.ui.dropdown>.dropdown.icon{font-family:Dropdown;line-height:1;height:1em;width:1.23em;-webkit-backface-visibility:hidden;backface-visibility:hidden;font-weight:400;font-style:normal;text-align:center}.ui.dropdown>.dropdown.icon{width:auto}.ui.dropdown>.dropdown.icon:before{content:'\f0d7'}.ui.dropdown .menu .item .dropdown.icon:before{content:'\f0da'}.ui.dropdown .item .left.dropdown.icon:before,.ui.dropdown .left.menu .item .dropdown.icon:before{content:"\f0d9"}.ui.vertical.menu .dropdown.item>.dropdown.icon:before{content:"\f0da"}.ui.dropdown>.clear.icon:before{content:"\f00d"}/*! - * # Semantic UI 2.4.0 - Video - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */.ui.embed{position:relative;max-width:100%;height:0;overflow:hidden;background:#dcddde;padding-bottom:56.25%}.ui.embed embed,.ui.embed iframe,.ui.embed object{position:absolute;border:none;width:100%;height:100%;top:0;left:0;margin:0;padding:0}.ui.embed>.embed{display:none}.ui.embed>.placeholder{position:absolute;cursor:pointer;top:0;left:0;display:block;width:100%;height:100%;background-color:radial-gradient(transparent 45%,rgba(0,0,0,.3))}.ui.embed>.icon{cursor:pointer;position:absolute;top:0;left:0;width:100%;height:100%;z-index:2}.ui.embed>.icon:after{position:absolute;top:0;left:0;width:100%;height:100%;z-index:3;content:'';background:-webkit-radial-gradient(transparent 45%,rgba(0,0,0,.3));background:radial-gradient(transparent 45%,rgba(0,0,0,.3));opacity:.5;-webkit-transition:opacity .5s ease;transition:opacity .5s ease}.ui.embed>.icon:before{position:absolute;top:50%;left:50%;z-index:4;-webkit-transform:translateX(-50%) translateY(-50%);transform:translateX(-50%) translateY(-50%);color:#fff;font-size:6rem;text-shadow:0 2px 10px rgba(34,36,38,.2);-webkit-transition:opacity .5s ease,color .5s ease;transition:opacity .5s ease,color .5s ease;z-index:10}.ui.embed .icon:hover:after{background:-webkit-radial-gradient(transparent 45%,rgba(0,0,0,.3));background:radial-gradient(transparent 45%,rgba(0,0,0,.3));opacity:1}.ui.embed .icon:hover:before{color:#fff}.ui.active.embed>.icon,.ui.active.embed>.placeholder{display:none}.ui.active.embed>.embed{display:block}.ui.square.embed{padding-bottom:100%}.ui[class*="4:3"].embed{padding-bottom:75%}.ui[class*="16:9"].embed{padding-bottom:56.25%}.ui[class*="21:9"].embed{padding-bottom:42.85714286%}/*! - * # Semantic UI 2.4.0 - Modal - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */.ui.modal{position:absolute;display:none;z-index:1001;text-align:left;background:#fff;border:none;-webkit-box-shadow:1px 3px 3px 0 rgba(0,0,0,.2),1px 3px 15px 2px rgba(0,0,0,.2);box-shadow:1px 3px 3px 0 rgba(0,0,0,.2),1px 3px 15px 2px rgba(0,0,0,.2);-webkit-transform-origin:50% 25%;transform-origin:50% 25%;-webkit-box-flex:0;-ms-flex:0 0 auto;flex:0 0 auto;border-radius:.28571429rem;-webkit-user-select:text;-moz-user-select:text;-ms-user-select:text;user-select:text;will-change:top,left,margin,transform,opacity}.ui.modal>.icon:first-child+*,.ui.modal>:first-child:not(.icon){border-top-left-radius:.28571429rem;border-top-right-radius:.28571429rem}.ui.modal>:last-child{border-bottom-left-radius:.28571429rem;border-bottom-right-radius:.28571429rem}.ui.modal>.close{cursor:pointer;position:absolute;top:-2.5rem;right:-2.5rem;z-index:1;opacity:.8;font-size:1.25em;color:#fff;width:2.25rem;height:2.25rem;padding:.625rem 0 0 0}.ui.modal>.close:hover{opacity:1}.ui.modal>.header{display:block;font-family:Lato,'Helvetica Neue',Arial,Helvetica,sans-serif;background:#fff;margin:0;padding:1.25rem 1.5rem;-webkit-box-shadow:none;box-shadow:none;color:rgba(0,0,0,.85);border-bottom:1px solid rgba(34,36,38,.15)}.ui.modal>.header:not(.ui){font-size:1.42857143rem;line-height:1.28571429em;font-weight:700}.ui.modal>.content{display:block;width:100%;font-size:1em;line-height:1.4;padding:1.5rem;background:#fff}.ui.modal>.image.content{display:-webkit-box;display:-ms-flexbox;display:flex;-webkit-box-orient:horizontal;-webkit-box-direction:normal;-ms-flex-direction:row;flex-direction:row}.ui.modal>.content>.image{display:block;-webkit-box-flex:0;-ms-flex:0 1 auto;flex:0 1 auto;width:'';-ms-flex-item-align:top;align-self:top}.ui.modal>[class*="top aligned"]{-ms-flex-item-align:top;align-self:top}.ui.modal>[class*="middle aligned"]{-ms-flex-item-align:middle;align-self:middle}.ui.modal>[class*=stretched]{-ms-flex-item-align:stretch;align-self:stretch}.ui.modal>.content>.description{display:block;-webkit-box-flex:1;-ms-flex:1 0 auto;flex:1 0 auto;min-width:0;-ms-flex-item-align:top;align-self:top}.ui.modal>.content>.icon+.description,.ui.modal>.content>.image+.description{-webkit-box-flex:0;-ms-flex:0 1 auto;flex:0 1 auto;min-width:'';width:auto;padding-left:2em}.ui.modal>.content>.image>i.icon{margin:0;opacity:1;width:auto;line-height:1;font-size:8rem}.ui.modal>.actions{background:#f9fafb;padding:1rem 1rem;border-top:1px solid rgba(34,36,38,.15);text-align:right}.ui.modal .actions>.button{margin-left:.75em}@media only screen and (max-width:767px){.ui.modal{width:95%;margin:0}}@media only screen and (min-width:768px){.ui.modal{width:88%;margin:0}}@media only screen and (min-width:992px){.ui.modal{width:850px;margin:0}}@media only screen and (min-width:1200px){.ui.modal{width:900px;margin:0}}@media only screen and (min-width:1920px){.ui.modal{width:950px;margin:0}}@media only screen and (max-width:991px){.ui.modal>.header{padding-right:2.25rem}.ui.modal>.close{top:1.0535rem;right:1rem;color:rgba(0,0,0,.87)}}@media only screen and (max-width:767px){.ui.modal>.header{padding:.75rem 1rem!important;padding-right:2.25rem!important}.ui.modal>.content{display:block;padding:1rem!important}.ui.modal>.close{top:.5rem!important;right:.5rem!important}.ui.modal .image.content{-webkit-box-orient:vertical;-webkit-box-direction:normal;-ms-flex-direction:column;flex-direction:column}.ui.modal .content>.image{display:block;max-width:100%;margin:0 auto!important;text-align:center;padding:0 0 1rem!important}.ui.modal>.content>.image>i.icon{font-size:5rem;text-align:center}.ui.modal .content>.description{display:block;width:100%!important;margin:0!important;padding:1rem 0!important;-webkit-box-shadow:none;box-shadow:none}.ui.modal>.actions{padding:1rem 1rem 0!important}.ui.modal .actions>.button,.ui.modal .actions>.buttons{margin-bottom:1rem}}.ui.inverted.dimmer>.ui.modal{-webkit-box-shadow:1px 3px 10px 2px rgba(0,0,0,.2);box-shadow:1px 3px 10px 2px rgba(0,0,0,.2)}.ui.basic.modal{background-color:transparent;border:none;border-radius:0;-webkit-box-shadow:none!important;box-shadow:none!important;color:#fff}.ui.basic.modal>.actions,.ui.basic.modal>.content,.ui.basic.modal>.header{background-color:transparent}.ui.basic.modal>.header{color:#fff}.ui.basic.modal>.close{top:1rem;right:1.5rem}.ui.inverted.dimmer>.basic.modal{color:rgba(0,0,0,.87)}.ui.inverted.dimmer>.ui.basic.modal>.header{color:rgba(0,0,0,.85)}.ui.legacy.modal,.ui.legacy.page.dimmer>.ui.modal{top:50%;left:50%}.ui.legacy.page.dimmer>.ui.scrolling.modal,.ui.page.dimmer>.ui.scrolling.legacy.modal,.ui.top.aligned.dimmer>.ui.legacy.modal,.ui.top.aligned.legacy.page.dimmer>.ui.modal{top:auto}@media only screen and (max-width:991px){.ui.basic.modal>.close{color:#fff}}.ui.loading.modal{display:block;visibility:hidden;z-index:-1}.ui.active.modal{display:block}.modals.dimmer[class*="top aligned"] .modal{margin:5vh auto}@media only screen and (max-width:767px){.modals.dimmer[class*="top aligned"] .modal{margin:1rem auto}}.legacy.modals.dimmer[class*="top aligned"]{padding-top:5vh}@media only screen and (max-width:767px){.legacy.modals.dimmer[class*="top aligned"]{padding-top:1rem}}.scrolling.dimmable.dimmed{overflow:hidden}.scrolling.dimmable>.dimmer{-webkit-box-pack:start;-ms-flex-pack:start;justify-content:flex-start}.scrolling.dimmable.dimmed>.dimmer{overflow:auto;-webkit-overflow-scrolling:touch}.scrolling.dimmable>.dimmer{position:fixed}.modals.dimmer .ui.scrolling.modal{margin:1rem auto}.scrolling.undetached.dimmable.dimmed{overflow:auto;-webkit-overflow-scrolling:touch}.scrolling.undetached.dimmable.dimmed>.dimmer{overflow:hidden}.scrolling.undetached.dimmable .ui.scrolling.modal{position:absolute;left:50%;margin-top:1rem!important}.ui.modal .scrolling.content{max-height:calc(70vh);overflow:auto}.ui.fullscreen.modal{width:95%!important;left:0!important;margin:1em auto}.ui.fullscreen.scrolling.modal{left:0!important}.ui.fullscreen.modal>.header{padding-right:2.25rem}.ui.fullscreen.modal>.close{top:1.0535rem;right:1rem;color:rgba(0,0,0,.87)}.ui.modal{font-size:1rem}.ui.mini.modal>.header:not(.ui){font-size:1.3em}@media only screen and (max-width:767px){.ui.mini.modal{width:95%;margin:0}}@media only screen and (min-width:768px){.ui.mini.modal{width:35.2%;margin:0}}@media only screen and (min-width:992px){.ui.mini.modal{width:340px;margin:0}}@media only screen and (min-width:1200px){.ui.mini.modal{width:360px;margin:0}}@media only screen and (min-width:1920px){.ui.mini.modal{width:380px;margin:0}}.ui.small.modal>.header:not(.ui){font-size:1.3em}@media only screen and (max-width:767px){.ui.tiny.modal{width:95%;margin:0}}@media only screen and (min-width:768px){.ui.tiny.modal{width:52.8%;margin:0}}@media only screen and (min-width:992px){.ui.tiny.modal{width:510px;margin:0}}@media only screen and (min-width:1200px){.ui.tiny.modal{width:540px;margin:0}}@media only screen and (min-width:1920px){.ui.tiny.modal{width:570px;margin:0}}.ui.small.modal>.header:not(.ui){font-size:1.3em}@media only screen and (max-width:767px){.ui.small.modal{width:95%;margin:0}}@media only screen and (min-width:768px){.ui.small.modal{width:70.4%;margin:0}}@media only screen and (min-width:992px){.ui.small.modal{width:680px;margin:0}}@media only screen and (min-width:1200px){.ui.small.modal{width:720px;margin:0}}@media only screen and (min-width:1920px){.ui.small.modal{width:760px;margin:0}}.ui.large.modal>.header{font-size:1.6em}@media only screen and (max-width:767px){.ui.large.modal{width:95%;margin:0}}@media only screen and (min-width:768px){.ui.large.modal{width:88%;margin:0}}@media only screen and (min-width:992px){.ui.large.modal{width:1020px;margin:0}}@media only screen and (min-width:1200px){.ui.large.modal{width:1080px;margin:0}}@media only screen and (min-width:1920px){.ui.large.modal{width:1140px;margin:0}}/*! - * # Semantic UI 2.4.0 - Nag - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */.ui.nag{display:none;opacity:.95;position:relative;top:0;left:0;z-index:999;min-height:0;width:100%;margin:0;padding:.75em 1em;background:#555;-webkit-box-shadow:0 1px 2px 0 rgba(0,0,0,.2);box-shadow:0 1px 2px 0 rgba(0,0,0,.2);font-size:1rem;text-align:center;color:rgba(0,0,0,.87);border-radius:0 0 .28571429rem .28571429rem;-webkit-transition:.2s background ease;transition:.2s background ease}a.ui.nag{cursor:pointer}.ui.nag>.title{display:inline-block;margin:0 .5em;color:#fff}.ui.nag>.close.icon{cursor:pointer;opacity:.4;position:absolute;top:50%;right:1em;font-size:1em;margin:-.5em 0 0;color:#fff;-webkit-transition:opacity .2s ease;transition:opacity .2s ease}.ui.nag:hover{background:#555;opacity:1}.ui.nag .close:hover{opacity:1}.ui.overlay.nag{position:absolute;display:block}.ui.fixed.nag{position:fixed}.ui.bottom.nag,.ui.bottom.nags{border-radius:.28571429rem .28571429rem 0 0;top:auto;bottom:0}.ui.inverted.nag,.ui.inverted.nags .nag{background-color:#f3f4f5;color:rgba(0,0,0,.85)}.ui.inverted.nag .close,.ui.inverted.nag .title,.ui.inverted.nags .nag .close,.ui.inverted.nags .nag .title{color:rgba(0,0,0,.4)}.ui.nags .nag{border-radius:0!important}.ui.nags .nag:last-child{border-radius:0 0 .28571429rem .28571429rem}.ui.bottom.nags .nag:last-child{border-radius:.28571429rem .28571429rem 0 0}/*! - * # Semantic UI 2.4.0 - Popup - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */.ui.popup{display:none;position:absolute;top:0;right:0;min-width:-webkit-min-content;min-width:-moz-min-content;min-width:min-content;z-index:1900;border:1px solid #d4d4d5;line-height:1.4285em;max-width:250px;background:#fff;padding:.833em 1em;font-weight:400;font-style:normal;color:rgba(0,0,0,.87);border-radius:.28571429rem;-webkit-box-shadow:0 2px 4px 0 rgba(34,36,38,.12),0 2px 10px 0 rgba(34,36,38,.15);box-shadow:0 2px 4px 0 rgba(34,36,38,.12),0 2px 10px 0 rgba(34,36,38,.15)}.ui.popup>.header{padding:0;font-family:Lato,'Helvetica Neue',Arial,Helvetica,sans-serif;font-size:1.14285714em;line-height:1.2;font-weight:700}.ui.popup>.header+.content{padding-top:.5em}.ui.popup:before{position:absolute;content:'';width:.71428571em;height:.71428571em;background:#fff;-webkit-transform:rotate(45deg);transform:rotate(45deg);z-index:2;-webkit-box-shadow:1px 1px 0 0 #bababc;box-shadow:1px 1px 0 0 #bababc}[data-tooltip]{position:relative}[data-tooltip]:before{pointer-events:none;position:absolute;content:'';font-size:1rem;width:.71428571em;height:.71428571em;background:#fff;-webkit-transform:rotate(45deg);transform:rotate(45deg);z-index:2;-webkit-box-shadow:1px 1px 0 0 #bababc;box-shadow:1px 1px 0 0 #bababc}[data-tooltip]:after{pointer-events:none;content:attr(data-tooltip);position:absolute;text-transform:none;text-align:left;white-space:nowrap;font-size:1rem;border:1px solid #d4d4d5;line-height:1.4285em;max-width:none;background:#fff;padding:.833em 1em;font-weight:400;font-style:normal;color:rgba(0,0,0,.87);border-radius:.28571429rem;-webkit-box-shadow:0 2px 4px 0 rgba(34,36,38,.12),0 2px 10px 0 rgba(34,36,38,.15);box-shadow:0 2px 4px 0 rgba(34,36,38,.12),0 2px 10px 0 rgba(34,36,38,.15);z-index:1}[data-tooltip]:not([data-position]):before{top:auto;right:auto;bottom:100%;left:50%;background:#fff;margin-left:-.07142857rem;margin-bottom:.14285714rem}[data-tooltip]:not([data-position]):after{left:50%;-webkit-transform:translateX(-50%);transform:translateX(-50%);bottom:100%;margin-bottom:.5em}[data-tooltip]:after,[data-tooltip]:before{pointer-events:none;visibility:hidden}[data-tooltip]:before{opacity:0;-webkit-transform:rotate(45deg) scale(0)!important;transform:rotate(45deg) scale(0)!important;-webkit-transform-origin:center top;transform-origin:center top;-webkit-transition:all .1s ease;transition:all .1s ease}[data-tooltip]:after{opacity:1;-webkit-transform-origin:center bottom;transform-origin:center bottom;-webkit-transition:all .1s ease;transition:all .1s ease}[data-tooltip]:hover:after,[data-tooltip]:hover:before{visibility:visible;pointer-events:auto}[data-tooltip]:hover:before{-webkit-transform:rotate(45deg) scale(1)!important;transform:rotate(45deg) scale(1)!important;opacity:1}[data-tooltip]:after,[data-tooltip][data-position="bottom center"]:after,[data-tooltip][data-position="top center"]:after{-webkit-transform:translateX(-50%) scale(0)!important;transform:translateX(-50%) scale(0)!important}[data-tooltip]:hover:after,[data-tooltip][data-position="bottom center"]:hover:after{-webkit-transform:translateX(-50%) scale(1)!important;transform:translateX(-50%) scale(1)!important}[data-tooltip][data-position="left center"]:after,[data-tooltip][data-position="right center"]:after{-webkit-transform:translateY(-50%) scale(0)!important;transform:translateY(-50%) scale(0)!important}[data-tooltip][data-position="left center"]:hover:after,[data-tooltip][data-position="right center"]:hover:after{-webkit-transform:translateY(-50%) scale(1)!important;transform:translateY(-50%) scale(1)!important}[data-tooltip][data-position="bottom left"]:after,[data-tooltip][data-position="bottom right"]:after,[data-tooltip][data-position="top left"]:after,[data-tooltip][data-position="top right"]:after{-webkit-transform:scale(0)!important;transform:scale(0)!important}[data-tooltip][data-position="bottom left"]:hover:after,[data-tooltip][data-position="bottom right"]:hover:after,[data-tooltip][data-position="top left"]:hover:after,[data-tooltip][data-position="top right"]:hover:after{-webkit-transform:scale(1)!important;transform:scale(1)!important}[data-tooltip][data-inverted]:before{-webkit-box-shadow:none!important;box-shadow:none!important}[data-tooltip][data-inverted]:before{background:#1b1c1d}[data-tooltip][data-inverted]:after{background:#1b1c1d;color:#fff;border:none;-webkit-box-shadow:none;box-shadow:none}[data-tooltip][data-inverted]:after .header{background-color:none;color:#fff}[data-position="top center"][data-tooltip]:after{top:auto;right:auto;left:50%;bottom:100%;-webkit-transform:translateX(-50%);transform:translateX(-50%);margin-bottom:.5em}[data-position="top center"][data-tooltip]:before{top:auto;right:auto;bottom:100%;left:50%;background:#fff;margin-left:-.07142857rem;margin-bottom:.14285714rem}[data-position="top left"][data-tooltip]:after{top:auto;right:auto;left:0;bottom:100%;margin-bottom:.5em}[data-position="top left"][data-tooltip]:before{top:auto;right:auto;bottom:100%;left:1em;margin-left:-.07142857rem;margin-bottom:.14285714rem}[data-position="top right"][data-tooltip]:after{top:auto;left:auto;right:0;bottom:100%;margin-bottom:.5em}[data-position="top right"][data-tooltip]:before{top:auto;left:auto;bottom:100%;right:1em;margin-left:-.07142857rem;margin-bottom:.14285714rem}[data-position="bottom center"][data-tooltip]:after{bottom:auto;right:auto;left:50%;top:100%;-webkit-transform:translateX(-50%);transform:translateX(-50%);margin-top:.5em}[data-position="bottom center"][data-tooltip]:before{bottom:auto;right:auto;top:100%;left:50%;margin-left:-.07142857rem;margin-top:.14285714rem}[data-position="bottom left"][data-tooltip]:after{left:0;top:100%;margin-top:.5em}[data-position="bottom left"][data-tooltip]:before{bottom:auto;right:auto;top:100%;left:1em;margin-left:-.07142857rem;margin-top:.14285714rem}[data-position="bottom right"][data-tooltip]:after{right:0;top:100%;margin-top:.5em}[data-position="bottom right"][data-tooltip]:before{bottom:auto;left:auto;top:100%;right:1em;margin-left:-.14285714rem;margin-top:.07142857rem}[data-position="left center"][data-tooltip]:after{right:100%;top:50%;margin-right:.5em;-webkit-transform:translateY(-50%);transform:translateY(-50%)}[data-position="left center"][data-tooltip]:before{right:100%;top:50%;margin-top:-.14285714rem;margin-right:-.07142857rem}[data-position="right center"][data-tooltip]:after{left:100%;top:50%;margin-left:.5em;-webkit-transform:translateY(-50%);transform:translateY(-50%)}[data-position="right center"][data-tooltip]:before{left:100%;top:50%;margin-top:-.07142857rem;margin-left:.14285714rem}[data-position~=bottom][data-tooltip]:before{background:#fff;-webkit-box-shadow:-1px -1px 0 0 #bababc;box-shadow:-1px -1px 0 0 #bababc}[data-position="left center"][data-tooltip]:before{background:#fff;-webkit-box-shadow:1px -1px 0 0 #bababc;box-shadow:1px -1px 0 0 #bababc}[data-position="right center"][data-tooltip]:before{background:#fff;-webkit-box-shadow:-1px 1px 0 0 #bababc;box-shadow:-1px 1px 0 0 #bababc}[data-position~=top][data-tooltip]:before{background:#fff}[data-inverted][data-position~=bottom][data-tooltip]:before{background:#1b1c1d;-webkit-box-shadow:-1px -1px 0 0 #bababc;box-shadow:-1px -1px 0 0 #bababc}[data-inverted][data-position="left center"][data-tooltip]:before{background:#1b1c1d;-webkit-box-shadow:1px -1px 0 0 #bababc;box-shadow:1px -1px 0 0 #bababc}[data-inverted][data-position="right center"][data-tooltip]:before{background:#1b1c1d;-webkit-box-shadow:-1px 1px 0 0 #bababc;box-shadow:-1px 1px 0 0 #bababc}[data-inverted][data-position~=top][data-tooltip]:before{background:#1b1c1d}[data-position~=bottom][data-tooltip]:before{-webkit-transform-origin:center bottom;transform-origin:center bottom}[data-position~=bottom][data-tooltip]:after{-webkit-transform-origin:center top;transform-origin:center top}[data-position="left center"][data-tooltip]:before{-webkit-transform-origin:top center;transform-origin:top center}[data-position="left center"][data-tooltip]:after{-webkit-transform-origin:right center;transform-origin:right center}[data-position="right center"][data-tooltip]:before{-webkit-transform-origin:right center;transform-origin:right center}[data-position="right center"][data-tooltip]:after{-webkit-transform-origin:left center;transform-origin:left center}.ui.popup{margin:0}.ui.top.popup{margin:0 0 .71428571em}.ui.top.left.popup{-webkit-transform-origin:left bottom;transform-origin:left bottom}.ui.top.center.popup{-webkit-transform-origin:center bottom;transform-origin:center bottom}.ui.top.right.popup{-webkit-transform-origin:right bottom;transform-origin:right bottom}.ui.left.center.popup{margin:0 .71428571em 0 0;-webkit-transform-origin:right 50%;transform-origin:right 50%}.ui.right.center.popup{margin:0 0 0 .71428571em;-webkit-transform-origin:left 50%;transform-origin:left 50%}.ui.bottom.popup{margin:.71428571em 0 0}.ui.bottom.left.popup{-webkit-transform-origin:left top;transform-origin:left top}.ui.bottom.center.popup{-webkit-transform-origin:center top;transform-origin:center top}.ui.bottom.right.popup{-webkit-transform-origin:right top;transform-origin:right top}.ui.bottom.center.popup:before{margin-left:-.30714286em;top:-.30714286em;left:50%;right:auto;bottom:auto;-webkit-box-shadow:-1px -1px 0 0 #bababc;box-shadow:-1px -1px 0 0 #bababc}.ui.bottom.left.popup{margin-left:0}.ui.bottom.left.popup:before{top:-.30714286em;left:1em;right:auto;bottom:auto;margin-left:0;-webkit-box-shadow:-1px -1px 0 0 #bababc;box-shadow:-1px -1px 0 0 #bababc}.ui.bottom.right.popup{margin-right:0}.ui.bottom.right.popup:before{top:-.30714286em;right:1em;bottom:auto;left:auto;margin-left:0;-webkit-box-shadow:-1px -1px 0 0 #bababc;box-shadow:-1px -1px 0 0 #bababc}.ui.top.center.popup:before{top:auto;right:auto;bottom:-.30714286em;left:50%;margin-left:-.30714286em}.ui.top.left.popup{margin-left:0}.ui.top.left.popup:before{bottom:-.30714286em;left:1em;top:auto;right:auto;margin-left:0}.ui.top.right.popup{margin-right:0}.ui.top.right.popup:before{bottom:-.30714286em;right:1em;top:auto;left:auto;margin-left:0}.ui.left.center.popup:before{top:50%;right:-.30714286em;bottom:auto;left:auto;margin-top:-.30714286em;-webkit-box-shadow:1px -1px 0 0 #bababc;box-shadow:1px -1px 0 0 #bababc}.ui.right.center.popup:before{top:50%;left:-.30714286em;bottom:auto;right:auto;margin-top:-.30714286em;-webkit-box-shadow:-1px 1px 0 0 #bababc;box-shadow:-1px 1px 0 0 #bababc}.ui.bottom.popup:before{background:#fff}.ui.left.center.popup:before,.ui.right.center.popup:before{background:#fff}.ui.top.popup:before{background:#fff}.ui.inverted.bottom.popup:before{background:#1b1c1d}.ui.inverted.left.center.popup:before,.ui.inverted.right.center.popup:before{background:#1b1c1d}.ui.inverted.top.popup:before{background:#1b1c1d}.ui.popup>.ui.grid:not(.padded){width:calc(100% + 1.75rem);margin:-.7rem -.875rem}.ui.loading.popup{display:block;visibility:hidden;z-index:-1}.ui.animating.popup,.ui.visible.popup{display:block}.ui.visible.popup{-webkit-transform:translateZ(0);transform:translateZ(0);-webkit-backface-visibility:hidden;backface-visibility:hidden}.ui.basic.popup:before{display:none}.ui.wide.popup{max-width:350px}.ui[class*="very wide"].popup{max-width:550px}@media only screen and (max-width:767px){.ui.wide.popup,.ui[class*="very wide"].popup{max-width:250px}}.ui.fluid.popup{width:100%;max-width:none}.ui.inverted.popup{background:#1b1c1d;color:#fff;border:none;-webkit-box-shadow:none;box-shadow:none}.ui.inverted.popup .header{background-color:none;color:#fff}.ui.inverted.popup:before{background-color:#1b1c1d;-webkit-box-shadow:none!important;box-shadow:none!important}.ui.flowing.popup{max-width:none}.ui.mini.popup{font-size:.78571429rem}.ui.tiny.popup{font-size:.85714286rem}.ui.small.popup{font-size:.92857143rem}.ui.popup{font-size:1rem}.ui.large.popup{font-size:1.14285714rem}.ui.huge.popup{font-size:1.42857143rem}/*! - * # Semantic UI 2.4.0 - Progress Bar - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */.ui.progress{position:relative;display:block;max-width:100%;border:none;margin:1em 0 2.5em;-webkit-box-shadow:none;box-shadow:none;background:rgba(0,0,0,.1);padding:0;border-radius:.28571429rem}.ui.progress:first-child{margin:0 0 2.5em}.ui.progress:last-child{margin:0 0 1.5em}.ui.progress .bar{display:block;line-height:1;position:relative;width:0%;min-width:2em;background:#888;border-radius:.28571429rem;-webkit-transition:width .1s ease,background-color .1s ease;transition:width .1s ease,background-color .1s ease}.ui.progress .bar>.progress{white-space:nowrap;position:absolute;width:auto;font-size:.92857143em;top:50%;right:.5em;left:auto;bottom:auto;color:rgba(255,255,255,.7);text-shadow:none;margin-top:-.5em;font-weight:700;text-align:left}.ui.progress>.label{position:absolute;width:100%;font-size:1em;top:100%;right:auto;left:0;bottom:auto;color:rgba(0,0,0,.87);font-weight:700;text-shadow:none;margin-top:.2em;text-align:center;-webkit-transition:color .4s ease;transition:color .4s ease}.ui.indicating.progress[data-percent^="1"] .bar,.ui.indicating.progress[data-percent^="2"] .bar{background-color:#d95c5c}.ui.indicating.progress[data-percent^="3"] .bar{background-color:#efbc72}.ui.indicating.progress[data-percent^="4"] .bar,.ui.indicating.progress[data-percent^="5"] .bar{background-color:#e6bb48}.ui.indicating.progress[data-percent^="6"] .bar{background-color:#ddc928}.ui.indicating.progress[data-percent^="7"] .bar,.ui.indicating.progress[data-percent^="8"] .bar{background-color:#b4d95c}.ui.indicating.progress[data-percent^="100"] .bar,.ui.indicating.progress[data-percent^="9"] .bar{background-color:#66da81}.ui.indicating.progress[data-percent^="1"] .label,.ui.indicating.progress[data-percent^="2"] .label{color:rgba(0,0,0,.87)}.ui.indicating.progress[data-percent^="3"] .label{color:rgba(0,0,0,.87)}.ui.indicating.progress[data-percent^="4"] .label,.ui.indicating.progress[data-percent^="5"] .label{color:rgba(0,0,0,.87)}.ui.indicating.progress[data-percent^="6"] .label{color:rgba(0,0,0,.87)}.ui.indicating.progress[data-percent^="7"] .label,.ui.indicating.progress[data-percent^="8"] .label{color:rgba(0,0,0,.87)}.ui.indicating.progress[data-percent^="100"] .label,.ui.indicating.progress[data-percent^="9"] .label{color:rgba(0,0,0,.87)}.ui.indicating.progress[data-percent="1"] .bar,.ui.indicating.progress[data-percent="2"] .bar,.ui.indicating.progress[data-percent="3"] .bar,.ui.indicating.progress[data-percent="4"] .bar,.ui.indicating.progress[data-percent="5"] .bar,.ui.indicating.progress[data-percent="6"] .bar,.ui.indicating.progress[data-percent="7"] .bar,.ui.indicating.progress[data-percent="8"] .bar,.ui.indicating.progress[data-percent="9"] .bar{background-color:#d95c5c}.ui.indicating.progress[data-percent="1"] .label,.ui.indicating.progress[data-percent="2"] .label,.ui.indicating.progress[data-percent="3"] .label,.ui.indicating.progress[data-percent="4"] .label,.ui.indicating.progress[data-percent="5"] .label,.ui.indicating.progress[data-percent="6"] .label,.ui.indicating.progress[data-percent="7"] .label,.ui.indicating.progress[data-percent="8"] .label,.ui.indicating.progress[data-percent="9"] .label{color:rgba(0,0,0,.87)}.ui.indicating.progress.success .label{color:#1a531b}.ui.progress.success .bar{background-color:#21ba45!important}.ui.progress.success .bar,.ui.progress.success .bar::after{-webkit-animation:none!important;animation:none!important}.ui.progress.success>.label{color:#1a531b}.ui.progress.warning .bar{background-color:#f2c037!important}.ui.progress.warning .bar,.ui.progress.warning .bar::after{-webkit-animation:none!important;animation:none!important}.ui.progress.warning>.label{color:#794b02}.ui.progress.error .bar{background-color:#db2828!important}.ui.progress.error .bar,.ui.progress.error .bar::after{-webkit-animation:none!important;animation:none!important}.ui.progress.error>.label{color:#912d2b}.ui.active.progress .bar{position:relative;min-width:2em}.ui.active.progress .bar::after{content:'';opacity:0;position:absolute;top:0;left:0;right:0;bottom:0;background:#fff;border-radius:.28571429rem;-webkit-animation:progress-active 2s ease infinite;animation:progress-active 2s ease infinite}@-webkit-keyframes progress-active{0%{opacity:.3;width:0}100%{opacity:0;width:100%}}@keyframes progress-active{0%{opacity:.3;width:0}100%{opacity:0;width:100%}}.ui.disabled.progress{opacity:.35}.ui.disabled.progress .bar,.ui.disabled.progress .bar::after{-webkit-animation:none!important;animation:none!important}.ui.inverted.progress{background:rgba(255,255,255,.08);border:none}.ui.inverted.progress .bar{background:#888}.ui.inverted.progress .bar>.progress{color:#f9fafb}.ui.inverted.progress>.label{color:#fff}.ui.inverted.progress.success>.label{color:#21ba45}.ui.inverted.progress.warning>.label{color:#f2c037}.ui.inverted.progress.error>.label{color:#db2828}.ui.progress.attached{background:0 0;position:relative;border:none;margin:0}.ui.progress.attached,.ui.progress.attached .bar{display:block;height:.2rem;padding:0;overflow:hidden;border-radius:0 0 .28571429rem .28571429rem}.ui.progress.attached .bar{border-radius:0}.ui.progress.top.attached,.ui.progress.top.attached .bar{top:0;border-radius:.28571429rem .28571429rem 0 0}.ui.progress.top.attached .bar{border-radius:0}.ui.card>.ui.attached.progress,.ui.segment>.ui.attached.progress{position:absolute;top:auto;left:0;bottom:100%;width:100%}.ui.card>.ui.bottom.attached.progress,.ui.segment>.ui.bottom.attached.progress{top:100%;bottom:auto}.ui.red.progress .bar{background-color:#db2828}.ui.red.inverted.progress .bar{background-color:#ff695e}.ui.orange.progress .bar{background-color:#f2711c}.ui.orange.inverted.progress .bar{background-color:#ff851b}.ui.yellow.progress .bar{background-color:#fbbd08}.ui.yellow.inverted.progress .bar{background-color:#ffe21f}.ui.olive.progress .bar{background-color:#b5cc18}.ui.olive.inverted.progress .bar{background-color:#d9e778}.ui.green.progress .bar{background-color:#21ba45}.ui.green.inverted.progress .bar{background-color:#2ecc40}.ui.teal.progress .bar{background-color:#00b5ad}.ui.teal.inverted.progress .bar{background-color:#6dffff}.ui.blue.progress .bar{background-color:#2185d0}.ui.blue.inverted.progress .bar{background-color:#54c8ff}.ui.violet.progress .bar{background-color:#6435c9}.ui.violet.inverted.progress .bar{background-color:#a291fb}.ui.purple.progress .bar{background-color:#a333c8}.ui.purple.inverted.progress .bar{background-color:#dc73ff}.ui.pink.progress .bar{background-color:#e03997}.ui.pink.inverted.progress .bar{background-color:#ff8edf}.ui.brown.progress .bar{background-color:#a5673f}.ui.brown.inverted.progress .bar{background-color:#d67c1c}.ui.grey.progress .bar{background-color:#767676}.ui.grey.inverted.progress .bar{background-color:#dcddde}.ui.black.progress .bar{background-color:#1b1c1d}.ui.black.inverted.progress .bar{background-color:#545454}.ui.tiny.progress{font-size:.85714286rem}.ui.tiny.progress .bar{height:.5em}.ui.small.progress{font-size:.92857143rem}.ui.small.progress .bar{height:1em}.ui.progress{font-size:1rem}.ui.progress .bar{height:1.75em}.ui.large.progress{font-size:1.14285714rem}.ui.large.progress .bar{height:2.5em}.ui.big.progress{font-size:1.28571429rem}.ui.big.progress .bar{height:3.5em}/*! - * # Semantic UI 2.4.0 - Rating - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */.ui.rating{display:-webkit-inline-box;display:-ms-inline-flexbox;display:inline-flex;white-space:nowrap;vertical-align:baseline}.ui.rating:last-child{margin-right:0}.ui.rating .icon{padding:0;margin:0;text-align:center;font-weight:400;font-style:normal;-webkit-box-flex:1;-ms-flex:1 0 auto;flex:1 0 auto;cursor:pointer;width:1.25em;height:auto;-webkit-transition:opacity .1s ease,background .1s ease,text-shadow .1s ease,color .1s ease;transition:opacity .1s ease,background .1s ease,text-shadow .1s ease,color .1s ease}.ui.rating .icon{background:0 0;color:rgba(0,0,0,.15)}.ui.rating .active.icon{background:0 0;color:rgba(0,0,0,.85)}.ui.rating .icon.selected,.ui.rating .icon.selected.active{background:0 0;color:rgba(0,0,0,.87)}.ui.star.rating .icon{width:1.25em;height:auto;background:0 0;color:rgba(0,0,0,.15);text-shadow:none}.ui.star.rating .active.icon{background:0 0!important;color:#ffe623!important;text-shadow:0 -1px 0 #ddc507,-1px 0 0 #ddc507,0 1px 0 #ddc507,1px 0 0 #ddc507!important}.ui.star.rating .icon.selected,.ui.star.rating .icon.selected.active{background:0 0!important;color:#fc0!important;text-shadow:0 -1px 0 #e6a200,-1px 0 0 #e6a200,0 1px 0 #e6a200,1px 0 0 #e6a200!important}.ui.heart.rating .icon{width:1.4em;height:auto;background:0 0;color:rgba(0,0,0,.15);text-shadow:none!important}.ui.heart.rating .active.icon{background:0 0!important;color:#ff6d75!important;text-shadow:0 -1px 0 #cd0707,-1px 0 0 #cd0707,0 1px 0 #cd0707,1px 0 0 #cd0707!important}.ui.heart.rating .icon.selected,.ui.heart.rating .icon.selected.active{background:0 0!important;color:#ff3000!important;text-shadow:0 -1px 0 #aa0101,-1px 0 0 #aa0101,0 1px 0 #aa0101,1px 0 0 #aa0101!important}.ui.disabled.rating .icon{cursor:default}.ui.rating.selected .active.icon{opacity:1}.ui.rating .icon.selected,.ui.rating.selected .icon.selected{opacity:1}.ui.mini.rating{font-size:.78571429rem}.ui.tiny.rating{font-size:.85714286rem}.ui.small.rating{font-size:.92857143rem}.ui.rating{font-size:1rem}.ui.large.rating{font-size:1.14285714rem}.ui.huge.rating{font-size:1.42857143rem}.ui.massive.rating{font-size:2rem}@font-face{font-family:Rating;src:url(data:application/x-font-ttf;charset=utf-8;base64,AAEAAAALAIAAAwAwT1MvMggjCBsAAAC8AAAAYGNtYXCj2pm8AAABHAAAAKRnYXNwAAAAEAAAAcAAAAAIZ2x5ZlJbXMYAAAHIAAARnGhlYWQBGAe5AAATZAAAADZoaGVhA+IB/QAAE5wAAAAkaG10eCzgAEMAABPAAAAAcGxvY2EwXCxOAAAUMAAAADptYXhwACIAnAAAFGwAAAAgbmFtZfC1n04AABSMAAABPHBvc3QAAwAAAAAVyAAAACAAAwIAAZAABQAAAUwBZgAAAEcBTAFmAAAA9QAZAIQAAAAAAAAAAAAAAAAAAAABEAAAAAAAAAAAAAAAAAAAAABAAADxZQHg/+D/4AHgACAAAAABAAAAAAAAAAAAAAAgAAAAAAACAAAAAwAAABQAAwABAAAAFAAEAJAAAAAgACAABAAAAAEAIOYF8AbwDfAj8C7wbvBw8Irwl/Cc8SPxZf/9//8AAAAAACDmAPAE8AzwI/Au8G7wcPCH8JfwnPEj8WT//f//AAH/4xoEEAYQAQ/sD+IPow+iD4wPgA98DvYOtgADAAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAH//wAPAAEAAAAAAAAAAAACAAA3OQEAAAAAAQAAAAAAAAAAAAIAADc5AQAAAAABAAAAAAAAAAAAAgAANzkBAAAAAAIAAP/tAgAB0wAKABUAAAEvAQ8BFwc3Fyc3BQc3Jz8BHwEHFycCALFPT7GAHp6eHoD/AHAWW304OH1bFnABGRqgoBp8sFNTsHyyOnxYEnFxElh8OgAAAAACAAD/7QIAAdMACgASAAABLwEPARcHNxcnNwUxER8BBxcnAgCxT0+xgB6enh6A/wA4fVsWcAEZGqCgGnywU1OwfLIBHXESWHw6AAAAAQAA/+0CAAHTAAoAAAEvAQ8BFwc3Fyc3AgCxT0+xgB6enh6AARkaoKAafLBTU7B8AAAAAAEAAAAAAgABwAArAAABFA4CBzEHDgMjIi4CLwEuAzU0PgIzMh4CFz4DMzIeAhUCAAcMEgugBgwMDAYGDAwMBqALEgwHFyg2HhAfGxkKChkbHxAeNigXAS0QHxsZCqAGCwkGBQkLBqAKGRsfEB42KBcHDBILCxIMBxcoNh4AAAAAAgAAAAACAAHAACsAWAAAATQuAiMiDgIHLgMjIg4CFRQeAhcxFx4DMzI+Aj8BPgM1DwEiFCIGMTAmIjQjJy4DNTQ+AjMyHgIfATc+AzMyHgIVFA4CBwIAFyg2HhAfGxkKChkbHxAeNigXBwwSC6AGDAwMBgYMDAwGoAsSDAdbogEBAQEBAaIGCgcEDRceEQkREA4GLy8GDhARCREeFw0EBwoGAS0eNigXBwwSCwsSDAcXKDYeEB8bGQqgBgsJBgUJCwagChkbHxA+ogEBAQGiBg4QEQkRHhcNBAcKBjQ0BgoHBA0XHhEJERAOBgABAAAAAAIAAcAAMQAAARQOAgcxBw4DIyIuAi8BLgM1ND4CMzIeAhcHFwc3Jzc+AzMyHgIVAgAHDBILoAYMDAwGBgwMDAagCxIMBxcoNh4KFRMSCC9wQLBwJwUJCgkFHjYoFwEtEB8bGQqgBgsJBgUJCwagChkbHxAeNigXAwUIBUtAoMBAOwECAQEXKDYeAAABAAAAAAIAAbcAKgAAEzQ3NjMyFxYXFhcWFzY3Njc2NzYzMhcWFRQPAQYjIi8BJicmJyYnJicmNQAkJUARExIQEAsMCgoMCxAQEhMRQCUkQbIGBwcGsgMFBQsKCQkGBwExPyMkBgYLCgkKCgoKCQoLBgYkIz8/QawFBawCBgUNDg4OFRQTAAAAAQAAAA0B2wHSACYAABM0PwI2FzYfAhYVFA8BFxQVFAcGByYvAQcGByYnJjU0PwEnJjUAEI9BBQkIBkCPEAdoGQMDBgUGgIEGBQYDAwEYaAcBIwsCFoEMAQEMgRYCCwYIZJABBQUFAwEBAkVFAgEBAwUFAwOQZAkFAAAAAAIAAAANAdsB0gAkAC4AABM0PwI2FzYfAhYVFA8BFxQVFAcmLwEHBgcmJyY1ND8BJyY1HwEHNxcnNy8BBwAQj0EFCQgGQI8QB2gZDAUGgIEGBQYDAwEYaAc/WBVsaxRXeDY2ASMLAhaBDAEBDIEWAgsGCGSQAQUNAQECRUUCAQEDBQUDA5BkCQURVXg4OHhVEW5uAAABACMAKQHdAXwAGgAANzQ/ATYXNh8BNzYXNh8BFhUUDwEGByYvASY1IwgmCAwLCFS8CAsMCCYICPUIDAsIjgjSCwkmCQEBCVS7CQEBCSYJCg0H9gcBAQePBwwAAAEAHwAfAXMBcwAsAAA3ND8BJyY1ND8BNjMyHwE3NjMyHwEWFRQPARcWFRQPAQYjIi8BBwYjIi8BJjUfCFRUCAgnCAwLCFRUCAwLCCcICFRUCAgnCAsMCFRUCAsMCCcIYgsIVFQIDAsIJwgIVFQICCcICwwIVFQICwwIJwgIVFQICCcIDAAAAAACAAAAJQFJAbcAHwArAAA3NTQ3NjsBNTQ3NjMyFxYdATMyFxYdARQHBiMhIicmNTczNTQnJiMiBwYdAQAICAsKJSY1NCYmCQsICAgIC/7tCwgIW5MWFR4fFRZApQsICDc0JiYmJjQ3CAgLpQsICAgIC8A3HhYVFRYeNwAAAQAAAAcBbgG3ACEAADcRNDc2NzYzITIXFhcWFREUBwYHBiMiLwEHBiMiJyYnJjUABgUKBgYBLAYGCgUGBgUKBQcOCn5+Cg4GBgoFBicBcAoICAMDAwMICAr+kAoICAQCCXl5CQIECAgKAAAAAwAAACUCAAFuABgAMQBKAAA3NDc2NzYzMhcWFxYVFAcGBwYjIicmJyY1MxYXFjMyNzY3JicWFRQHBiMiJyY1NDcGBzcUFxYzMjc2NTQ3NjMyNzY1NCcmIyIHBhUABihDREtLREMoBgYoQ0RLS0RDKAYlJjk5Q0M5OSYrQREmJTU1JSYRQSuEBAQGBgQEEREZBgQEBAQGJBkayQoKQSgoKChBCgoKCkEoJycoQQoKOiMjIyM6RCEeIjUmJSUmNSIeIUQlBgQEBAQGGBIRBAQGBgQEGhojAAAABQAAAAkCAAGJACwAOABRAGgAcAAANzQ3Njc2MzIXNzYzMhcWFxYXFhcWFxYVFDEGBwYPAQYjIicmNTQ3JicmJyY1MxYXNyYnJjU0NwYHNxQXFjMyNzY1NDc2MzI3NjU0JyYjIgcGFRc3Njc2NyYnNxYXFhcWFRQHBgcGBwYjPwEWFRQHBgcABitBQU0ZGhADBQEEBAUFBAUEBQEEHjw8Hg4DBQQiBQ0pIyIZBiUvSxYZDg4RQSuEBAQGBgQEEREZBgQEBAQGJBkaVxU9MzQiIDASGxkZEAYGCxQrODk/LlACFxYlyQsJQycnBRwEAgEDAwIDAwIBAwUCNmxsNhkFFAMFBBUTHh8nCQtKISgSHBsfIh4hRCUGBAQEBAYYEhEEBAYGBAQaGiPJJQUiIjYzISASGhkbCgoKChIXMRsbUZANCyghIA8AAAMAAAAAAbcB2wA5AEoAlAAANzU0NzY7ATY3Njc2NzY3Njc2MzIXFhcWFRQHMzIXFhUUBxYVFAcUFRQHFgcGKwEiJyYnJisBIicmNTcUFxYzMjc2NTQnJiMiBwYVFzMyFxYXFhcWFxYXFhcWOwEyNTQnNjc2NTQnNjU0JyYnNjc2NTQnJisBNDc2NTQnJiMGBwYHBgcGBwYHBgcGBwYHBgcGBwYrARUACwoQTgodEQ4GBAMFBgwLDxgTEwoKDjMdFhYOAgoRARkZKCUbGxsjIQZSEAoLJQUFCAcGBQUGBwgFBUkJBAUFBAQHBwMDBwcCPCUjNwIJBQUFDwMDBAkGBgsLDmUODgoJGwgDAwYFDAYQAQUGAwQGBgYFBgUGBgQJSbcPCwsGJhUPCBERExMMCgkJFBQhGxwWFR4ZFQoKFhMGBh0WKBcXBgcMDAoLDxIHBQYGBQcIBQYGBQgSAQEBAQICAQEDAgEULwgIBQoLCgsJDhQHCQkEAQ0NCg8LCxAdHREcDQ4IEBETEw0GFAEHBwUECAgFBQUFAgO3AAADAAD/2wG3AbcAPABNAJkAADc1NDc2OwEyNzY3NjsBMhcWBxUWFRQVFhUUBxYVFAcGKwEWFRQHBgcGIyInJicmJyYnJicmJyYnIyInJjU3FBcWMzI3NjU0JyYjIgcGFRczMhcWFxYXFhcWFxYXFhcWFxYXFhcWFzI3NjU0JyY1MzI3NjU0JyYjNjc2NTQnNjU0JyYnNjU0JyYrASIHIgcGBwYHBgcGIwYrARUACwoQUgYhJRsbHiAoGRkBEQoCDhYWHTMOCgoTExgPCwoFBgIBBAMFDhEdCk4QCgslBQUIBwYFBQYHCAUFSQkEBgYFBgUGBgYEAwYFARAGDAUGAwMIGwkKDg5lDgsLBgYJBAMDDwUFBQkCDg4ZJSU8AgcHAwMHBwQEBQUECbe3DwsKDAwHBhcWJwIWHQYGExYKChUZHhYVHRoiExQJCgsJDg4MDAwNBg4WJQcLCw+kBwUGBgUHCAUGBgUIpAMCBQYFBQcIBAUHBwITBwwTExERBw0OHBEdHRALCw8KDQ0FCQkHFA4JCwoLCgUICBgMCxUDAgEBAgMBAQG3AAAAAQAAAA0A7gHSABQAABM0PwI2FxEHBgcmJyY1ND8BJyY1ABCPQQUJgQYFBgMDARhoBwEjCwIWgQwB/oNFAgEBAwUFAwOQZAkFAAAAAAIAAAAAAgABtwAqAFkAABM0NzYzMhcWFxYXFhc2NzY3Njc2MzIXFhUUDwEGIyIvASYnJicmJyYnJjUzFB8BNzY1NCcmJyYnJicmIyIHBgcGBwYHBiMiJyYnJicmJyYjIgcGBwYHBgcGFQAkJUARExIQEAsMCgoMCxAQEhMRQCUkQbIGBwcGsgMFBQsKCQkGByU1pqY1BgYJCg4NDg0PDhIRDg8KCgcFCQkFBwoKDw4REg4PDQ4NDgoJBgYBMT8jJAYGCwoJCgoKCgkKCwYGJCM/P0GsBQWsAgYFDQ4ODhUUEzA1oJ82MBcSEgoLBgcCAgcHCwsKCQgHBwgJCgsLBwcCAgcGCwoSEhcAAAACAAAABwFuAbcAIQAoAAA3ETQ3Njc2MyEyFxYXFhURFAcGBwYjIi8BBwYjIicmJyY1PwEfAREhEQAGBQoGBgEsBgYKBQYGBQoFBw4Kfn4KDgYGCgUGJZIZef7cJwFwCggIAwMDAwgICv6QCggIBAIJeXkJAgQICAoIjRl0AWP+nQAAAAABAAAAJQHbAbcAMgAANzU0NzY7ATU0NzYzMhcWHQEUBwYrASInJj0BNCcmIyIHBh0BMzIXFh0BFAcGIyEiJyY1AAgIC8AmJjQ1JiUFBQgSCAUFFhUfHhUWHAsICAgIC/7tCwgIQKULCAg3NSUmJiU1SQgFBgYFCEkeFhUVFh43CAgLpQsICAgICwAAAAIAAQANAdsB0gAiAC0AABM2PwI2MzIfAhYXFg8BFxYHBiMiLwEHBiMiJyY/AScmNx8CLwE/AS8CEwEDDJBABggJBUGODgIDCmcYAgQCCAMIf4IFBgYEAgEZaQgC7hBbEgINSnkILgEBJggCFYILC4IVAggICWWPCgUFA0REAwUFCo9lCQipCTBmEw1HEhFc/u0AAAADAAAAAAHJAbcAFAAlAHkAADc1NDc2OwEyFxYdARQHBisBIicmNTcUFxYzMjc2NTQnJiMiBwYVFzU0NzYzNjc2NzY3Njc2NzY3Njc2NzY3NjMyFxYXFhcWFxYXFhUUFRQHBgcGBxQHBgcGBzMyFxYVFAcWFRYHFgcGBxYHBgcjIicmJyYnJiciJyY1AAUGB1MHBQYGBQdTBwYFJQUFCAcGBQUGBwgFBWQFBQgGDw8OFAkFBAQBAQMCAQIEBAYFBw4KCgcHBQQCAwEBAgMDAgYCAgIBAU8XEBAQBQEOBQUECwMREiYlExYXDAwWJAoHBQY3twcGBQUGB7cIBQUFBQgkBwYFBQYHCAUGBgUIJLcHBQYBEBATGQkFCQgGBQwLBgcICQUGAwMFBAcHBgYICQQEBwsLCwYGCgIDBAMCBBEQFhkSDAoVEhAREAsgFBUBBAUEBAcMAQUFCAAAAAADAAD/2wHJAZIAFAAlAHkAADcUFxYXNxY3Nj0BNCcmBycGBwYdATc0NzY3FhcWFRQHBicGJyY1FzU0NzY3Fjc2NzY3NjcXNhcWBxYXFgcWBxQHFhUUBwYHJxYXFhcWFRYXFhcWFRQVFAcGBwYHBgcGBwYnBicmJyYnJicmJyYnJicmJyYnJiciJyY1AAUGB1MHBQYGBQdTBwYFJQUFCAcGBQUGBwgFBWQGBQcKJBYMDBcWEyUmEhEDCwQFBQ4BBRAQEBdPAQECAgIGAgMDAgEBAwIEBQcHCgoOBwUGBAQCAQIDAQEEBAUJFA4PDwYIBQWlBwYFAQEBBwQJtQkEBwEBAQUGB7eTBwYEAQEEBgcJBAYBAQYECZS4BwYEAgENBwUCBgMBAQEXEyEJEhAREBcIDhAaFhEPAQEFAgQCBQELBQcKDAkIBAUHCgUGBwgDBgIEAQEHBQkIBwUMCwcECgcGCRoREQ8CBgQIAAAAAQAAAAEAAJth57dfDzz1AAsCAAAAAADP/GODAAAAAM/8Y4MAAP/bAgAB2wAAAAgAAgAAAAAAAAABAAAB4P/gAAACAAAAAAACAAABAAAAAAAAAAAAAAAAAAAAHAAAAAAAAAAAAAAAAAEAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAdwAAAHcAAACAAAjAZMAHwFJAAABbgAAAgAAAAIAAAACAAAAAgAAAAEAAAACAAAAAW4AAAHcAAAB3AABAdwAAAHcAAAAAAAAAAoAFAAeAEoAcACKAMoBQAGIAcwCCgJUAoICxgMEAzoDpgRKBRgF7AYSBpgG2gcgB2oIGAjOAAAAAQAAABwAmgAFAAAAAAACAAAAAAAAAAAAAAAAAAAAAAAAAA4ArgABAAAAAAABAAwAAAABAAAAAAACAA4AQAABAAAAAAADAAwAIgABAAAAAAAEAAwATgABAAAAAAAFABYADAABAAAAAAAGAAYALgABAAAAAAAKADQAWgADAAEECQABAAwAAAADAAEECQACAA4AQAADAAEECQADAAwAIgADAAEECQAEAAwATgADAAEECQAFABYADAADAAEECQAGAAwANAADAAEECQAKADQAWgByAGEAdABpAG4AZwBWAGUAcgBzAGkAbwBuACAAMQAuADAAcgBhAHQAaQBuAGdyYXRpbmcAcgBhAHQAaQBuAGcAUgBlAGcAdQBsAGEAcgByAGEAdABpAG4AZwBGAG8AbgB0ACAAZwBlAG4AZQByAGEAdABlAGQAIABiAHkAIABJAGMAbwBNAG8AbwBuAC4AAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==) format('truetype'),url(data:application/font-woff;charset=utf-8;base64,d09GRk9UVE8AABcUAAoAAAAAFswAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAABDRkYgAAAA9AAAEuEAABLho6TvIE9TLzIAABPYAAAAYAAAAGAIIwgbY21hcAAAFDgAAACkAAAApKPambxnYXNwAAAU3AAAAAgAAAAIAAAAEGhlYWQAABTkAAAANgAAADYBGAe5aGhlYQAAFRwAAAAkAAAAJAPiAf1obXR4AAAVQAAAAHAAAABwLOAAQ21heHAAABWwAAAABgAAAAYAHFAAbmFtZQAAFbgAAAE8AAABPPC1n05wb3N0AAAW9AAAACAAAAAgAAMAAAEABAQAAQEBB3JhdGluZwABAgABADr4HAL4GwP4GAQeCgAZU/+Lix4KABlT/4uLDAeLZviU+HQFHQAAAP0PHQAAAQIRHQAAAAkdAAAS2BIAHQEBBw0PERQZHiMoLTI3PEFGS1BVWl9kaW5zeH2Ch4xyYXRpbmdyYXRpbmd1MHUxdTIwdUU2MDB1RTYwMXVFNjAydUU2MDN1RTYwNHVFNjA1dUYwMDR1RjAwNXVGMDA2dUYwMEN1RjAwRHVGMDIzdUYwMkV1RjA2RXVGMDcwdUYwODd1RjA4OHVGMDg5dUYwOEF1RjA5N3VGMDlDdUYxMjN1RjE2NHVGMTY1AAACAYkAGgAcAgABAAQABwAKAA0AVgCWAL0BAgGMAeQCbwLwA4cD5QR0BQMFdgZgB8MJkQtxC7oM2Q1jDggOmRAYEZr8lA78lA78lA77lA74lPetFftFpTz3NDz7NPtFcfcU+xBt+0T3Mt73Mjht90T3FPcQBfuU+0YV+wRRofcQMOP3EZ3D9wXD+wX3EXkwM6H7EPsExQUO+JT3rRX7RaU89zQ8+zT7RXH3FPsQbftE9zLe9zI4bfdE9xT3EAX7lPtGFYuLi/exw/sF9xF5MDOh+xD7BMUFDviU960V+0WlPPc0PPs0+0Vx9xT7EG37RPcy3vcyOG33RPcU9xAFDviU98EVi2B4ZG5wCIuL+zT7NAV7e3t7e4t7i3ube5sI+zT3NAVupniyi7aL3M3N3Iu2i7J4pm6mqLKetovci81JizoIDviU98EVi9xJzTqLYItkeHBucKhknmCLOotJSYs6i2CeZKhwCIuL9zT7NAWbe5t7m4ubi5ubm5sI9zT3NAWopp6yi7YIME0V+zb7NgWKioqKiouKi4qMiowI+zb3NgV6m4Ghi6OLubCwuYuji6GBm3oIule6vwWbnKGVo4u5i7Bmi12Lc4F1ensIDviU98EVi2B4ZG5wCIuL+zT7NAV7e3t7e4t7i3ube5sI+zT3NAVupniyi7aL3M3N3Iuni6WDoX4IXED3BEtL+zT3RPdU+wTLssYFl46YjZiL3IvNSYs6CA6L98UVi7WXrKOio6Otl7aLlouXiZiHl4eWhZaEloSUhZKFk4SShZKEkpKSkZOSkpGUkZaSCJaSlpGXj5iPl42Wi7aLrX+jc6N0l2qLYYthdWBgYAj7RvtABYeIh4mGi4aLh42Hjgj7RvdABYmNiY2Hj4iOhpGDlISUhZWFlIWVhpaHmYaYiZiLmAgOZ4v3txWLkpCPlo0I9yOgzPcWBY6SkI+Ri5CLkIePhAjL+xb3I3YFlomQh4uEi4aJh4aGCCMmpPsjBYuKi4mLiIuHioiJiImIiIqHi4iLh4yHjQj7FM/7FUcFh4mHioiLh4uIjImOiY6KjouPi4yLjYyOCKP3IyPwBYaQiZCLjwgOZ4v3txWLkpCPlo0I9yOgzPcWBY6SkI+Ri5CLkIePhAjL+xb3I3YFlomQh4uEi4aJh4aGCCMmpPsjBYuKi4mLiIuCh4aDi4iLh4yHjQj7FM/7FUcFh4mHioiLh4uIjImOiY6KjouPi4yLjYyOCKP3IyPwBYaQiZCLjwjKeRXjN3b7DfcAxPZSd/cN4t/7DJ1V9wFV+wEFDq73ZhWLk42RkZEIsbIFkZCRjpOLkouSiJCGCN8291D3UAWQkJKOkouTi5GIkYYIsWQFkYaNhIuEi4OJhYWFCPuJ+4kFhYWFiYOLhIuEjYaRCPsi9yIFhZCJkouSCA77AartFYuSjpKQkAjf3zffBYaQiJKLk4uSjpKQkAiysgWRkJGOk4uSi5KIkIYI3zff3wWQkJKOk4uSi5KIkIYIsmQFkIaOhIuEi4OIhIaGCDc33zcFkIaOhIuEi4OIhYaFCGRkBYaGhIiEi4OLhI6GkAg33zc3BYaGhIiEi4OLhY6FkAhksgWGkYiRi5MIDvtLi8sVi/c5BYuSjpKQkJCQko6SiwiVi4vCBYuul6mkpKSkqpiui66LqX6kcqRymG2LaAiLVJSLBZKLkoiQhpCGjoSLhAiL+zkFi4OIhYaGhoWEiYSLCPuniwWEi4SNhpGGkIiRi5MI5vdUFfcni4vCBYufhJx8mn2ZepJ3i3aLeoR9fX18g3qLdwiLVAUO+yaLshWL+AQFi5GNkY+RjpCQj5KNj42PjI+LCPfAiwWPi4+Kj4mRiZCHj4aPhY2Fi4UIi/wEBYuEiYWHhoeGhoeFiIiKhoqHi4GLhI6EkQj7EvcN+xL7DQWEhYOIgouHi4eLh42EjoaPiJCHkImRi5IIDov3XRWLko2Rj5Kltq+vuKW4pbuZvYu9i7t9uHG4ca9npWCPhI2Fi4SLhYmEh4RxYGdoXnAIXnFbflmLWYtbmF6lXqZnrnG2h5KJkouRCLCLFaRkq2yxdLF0tH+4i7iLtJexorGiq6qksm64Z61goZZ3kXaLdItnfm1ycnJybX9oiwhoi22XcqRypH6pi6+LopGglp9gdWdpbl4I9xiwFYuHjIiOiI6IjoqPi4+LjoyOjo2OjY6Lj4ubkJmXl5eWmZGbi4+LjoyOjo2OjY6LjwiLj4mOiY6IjYiNh4tzi3eCenp6eoJ3i3MIDov3XRWLko2Sj5GouK+utqW3pbqYvouci5yJnIgIm6cFjY6NjI+LjIuNi42JjYqOio+JjomOiY6KjomOiY6JjoqNioyKjomMiYuHi4qLiouLCHdnbVVjQ2NDbVV3Zwh9cgWJiIiJiIuJi36SdJiIjYmOi46LjY+UlJlvl3KcdJ90oHeie6WHkYmSi5IIsIsVqlq0Z711CKGzBXqXfpqCnoKdhp6LoIuikaCWn2B1Z2luXgj3GLAVi4eMiI6IjoiOio+Lj4uOjI6OjY6NjouPi5uQmZeXl5aZkZuLj4uOjI6OjY6NjouPCIuPiY6JjoiNiI2Hi3OLd4J6enp6gneLcwji+10VoLAFtI+wmK2hrqKnqKKvdq1wp2uhCJ2rBZ1/nHycepx6mHqWeY+EjYWLhIuEiYWHhIR/gH1+fG9qaXJmeWV5Y4Jhiwi53BXb9yQFjIKMg4uEi3CDc3x1fHV3fHOBCA6L1BWL90sFi5WPlJKSkpKTj5aLCNmLBZKPmJqepJaZlZeVlY+Qj5ONl42WjpeOmI+YkZWTk5OSk46Vi5uLmYiYhZiFlIGSfgiSfo55i3WLeYd5gXgIvosFn4uchJl8mn2Seot3i3qGfIJ9jYSLhYuEi3yIfoR+i4eLh4uHi3eGen99i3CDdnt8CHt8dYNwiwhmiwV5i3mNeY95kHeRc5N1k36Ph4sIOYsFgIuDjoSShJKHlIuVCLCdFYuGjIePiI+Hj4mQi5CLj42Pj46OjY+LkIuQiZCIjoePh42Gi4aLh4mHh4eIioaLhgjUeRWUiwWNi46Lj4qOi4+KjYqOi4+Kj4mQio6KjYqNio+Kj4mQio6KjIqzfquEpIsIrosFr4uemouri5CKkYqQkY6QkI6SjpKNkouSi5KJkoiRlZWQlouYi5CKkImRiZGJj4iOCJGMkI+PlI+UjZKLkouViJODk4SSgo+CiwgmiwWLlpCalJ6UnpCbi5aLnoiYhJSFlH+QeYuGhoeDiYCJf4h/h3+IfoWBg4KHh4SCgH4Ii4qIiYiGh4aIh4mIiIiIh4eGh4aHh4eHiIiHiIeHiIiHiIeKh4mIioiLCIKLi/tLBQ6L90sVi/dLBYuVj5OSk5KSk46WiwjdiwWPi5iPoZOkk6CRnZCdj56Nn4sIq4sFpougg5x8m3yTd4txCIuJBZd8kHuLd4uHi4eLh5J+jn6LfIuEi4SJhZR9kHyLeot3hHp8fH19eoR3iwhYiwWVeI95i3mLdIh6hH6EfoKBfoV+hX2He4uBi4OPg5KFkYaTh5SHlYiTipOKk4qTiJMIiZSIkYiPgZSBl4CaeKR+moSPCD2LBYCLg4+EkoSSh5SLlQiw9zgVi4aMh4+Ij4ePiZCLkIuPjY+Pjo6Nj4uQi5CJkIiOh4+HjYaLhouHiYeHh4iKhouGCNT7OBWUiwWOi46Kj4mPio+IjoiPh4+IjoePiI+Hj4aPho6HjoiNiI6Hj4aOho6Ii4qWfpKDj4YIk4ORgY5+j36OgI1/jYCPg5CGnYuXj5GUkpSOmYuei5aGmoKfgp6GmouWCPCLBZSLlI+SkpOTjpOLlYuSiZKHlIeUho+Fi46PjY+NkY2RjJCLkIuYhpaBlY6RjZKLkgiLkomSiJKIkoaQhY6MkIyRi5CLm4aXgpOBkn6Pe4sIZosFcotrhGN9iouIioaJh4qHiomKiYqIioaKh4mHioiKiYuHioiLh4qIi4mLCIKLi/tLBQ77lIv3txWLkpCPlo0I9yOgzPcWBY6SkI+RiwiL/BL7FUcFh4mHioiLh4uIjImOiY6KjouPi4yLjYyOCKP3IyPwBYaQiZCLjwgOi/fFFYu1l6yjoqOjrZe2i5aLl4mYh5eHloWWhJaElIWShZOEkoWShJKSkpGTkpKRlJGWkgiWkpaRl4+Yj5eNlou2i61/o3OjdJdqi2GLYXVgYGAI+0b7QAWHiIeJhouGi4eNh44I+0b3QAWJjYmNh4+IjoaRg5SElIWVhZSFlYaWh5mGmImYi5gIsIsVi2ucaa9oCPc6+zT3OvczBa+vnK2Lq4ubiZiHl4eXhpSFkoSSg5GCj4KQgo2CjYONgYuBi4KLgIl/hoCGgIWChAiBg4OFhISEhYaFhoaIhoaJhYuFi4aNiJCGkIaRhJGEkoORgZOCkoCRgJB/kICNgosIgYuBi4OJgomCiYKGgoeDhYSEhYSGgod/h3+Jfot7CA77JouyFYv4BAWLkY2Rj5GOkJCPko2PjY+Mj4sI98CLBY+Lj4qPiZGJkIePho+FjYWLhQiL/AQFi4SJhYeGh4aGh4WIiIqGioeLgYuEjoSRCPsS9w37EvsNBYSFg4iCi4eLh4uHjYSOho+IkIeQiZGLkgiwkxX3JvchpHL3DfsIi/f3+7iLi/v3BQ5ni8sVi/c5BYuSjpKQkJCQko6Siwj3VIuLwgWLrpippKSkpKmYrouvi6l+pHKkcpdti2gIi0IFi4aKhoeIh4eHiYaLCHmLBYaLh42Hj4eOipCLkAiL1AWLn4OcfZp9mXqSdot3i3qEfX18fIR6i3cIi1SniwWSi5KIkIaQho6Ei4QIi/s5BYuDiIWGhoaFhImEiwj7p4sFhIuEjYaRhpCIkYuTCA5njPe6FYyQkI6UjQj3I6DM9xYFj5KPj5GLkIuQh4+ECMv7FvcjdgWUiZCIjYaNhoiFhYUIIyak+yMFjIWKhomHiYiIiYaLiIuHjIeNCPsUz/sVRwWHiYeKiIuHi4eNiY6Jj4uQjJEIo/cjI/AFhZGJkY2QCPeB+z0VnILlW3rxiJ6ZmNTS+wydgpxe54v7pwUOZ4vCFYv3SwWLkI2Pjo+Pjo+NkIsI3osFkIuPiY6Ij4eNh4uGCIv7SwWLhomHh4eIh4eKhosIOIsFhouHjIePiI+Jj4uQCLCvFYuGjIePh46IkImQi5CLj42Pjo6PjY+LkIuQiZCIjoePh42Gi4aLhomIh4eIioaLhgjvZxWL90sFi5CNj46Oj4+PjZCLj4ySkJWWlZaVl5SXmJuVl5GRjo6OkI6RjZCNkIyPjI6MkY2TCIySjJGMj4yPjZCOkY6RjpCPjo6Pj42Qi5SLk4qSiZKJkYiPiJCIjoiPho6GjYeMhwiNh4yGjIaMhYuHi4iLiIuHi4eLg4uEiYSJhImFiYeJh4mFh4WLioqJiomJiIqJiokIi4qKiIqJCNqLBZqLmIWWgJaAkH+LfIt6hn2Af46DjYSLhIt9h36Cf4+Bi3+HgImAhYKEhI12hnmAfgh/fXiDcosIZosFfot+jHyOfI5/joOOg41/j32Qc5N8j4SMhouHjYiOh4+Jj4uQCA5ni/c5FYuGjYaOiI+Hj4mQiwjeiwWQi4+Njo+Pjo2Qi5AIi/dKBYuQiZCHjoiPh42Giwg4iwWGi4eJh4eIiImGi4YIi/tKBbD3JhWLkIyPj4+OjpCNkIuQi4+Jj4iOh42Hi4aLhomHiIeHh4eKhouGi4aMiI+Hj4qPi5AI7/snFYv3SwWLkI2Qj46Oj4+NkIuSi5qPo5OZkJePk46TjZeOmo6ajpiMmIsIsIsFpIueg5d9ln6Qeol1koSRgo2Aj4CLgIeAlH+Pfot9i4WJhIiCloCQfIt7i3yFfoGACICAfoZ8iwg8iwWMiIyJi4mMiYyJjYmMiIyKi4mPhI2GjYeNh42GjYOMhIyEi4SLhouHi4iLiYuGioYIioWKhomHioeJh4iGh4eIh4aIh4iFiISJhImDioKLhouHjYiPh4+Ij4iRiJGJkIqPCIqPipGKkomTipGKj4qOiZCJkYiQiJCIjoWSgZZ+nIKXgZaBloGWhJGHi4aLh42HjwiIjomQi48IDviUFPiUFYsMCgAAAAADAgABkAAFAAABTAFmAAAARwFMAWYAAAD1ABkAhAAAAAAAAAAAAAAAAAAAAAEQAAAAAAAAAAAAAAAAAAAAAEAAAPFlAeD/4P/gAeAAIAAAAAEAAAAAAAAAAAAAACAAAAAAAAIAAAADAAAAFAADAAEAAAAUAAQAkAAAACAAIAAEAAAAAQAg5gXwBvAN8CPwLvBu8HDwivCX8JzxI/Fl//3//wAAAAAAIOYA8ATwDPAj8C7wbvBw8Ifwl/Cc8SPxZP/9//8AAf/jGgQQBhABD+wP4g+jD6IPjA+AD3wO9g62AAMAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAf//AA8AAQAAAAEAAJrVlLJfDzz1AAsCAAAAAADP/GODAAAAAM/8Y4MAAP/bAgAB2wAAAAgAAgAAAAAAAAABAAAB4P/gAAACAAAAAAACAAABAAAAAAAAAAAAAAAAAAAAHAAAAAAAAAAAAAAAAAEAAAACAAAAAgAAAAIAAAACAAAAAgAAAAIAAAACAAAAAdwAAAHcAAACAAAjAZMAHwFJAAABbgAAAgAAAAIAAAACAAAAAgAAAAEAAAACAAAAAW4AAAHcAAAB3AABAdwAAAHcAAAAAFAAABwAAAAAAA4ArgABAAAAAAABAAwAAAABAAAAAAACAA4AQAABAAAAAAADAAwAIgABAAAAAAAEAAwATgABAAAAAAAFABYADAABAAAAAAAGAAYALgABAAAAAAAKADQAWgADAAEECQABAAwAAAADAAEECQACAA4AQAADAAEECQADAAwAIgADAAEECQAEAAwATgADAAEECQAFABYADAADAAEECQAGAAwANAADAAEECQAKADQAWgByAGEAdABpAG4AZwBWAGUAcgBzAGkAbwBuACAAMQAuADAAcgBhAHQAaQBuAGdyYXRpbmcAcgBhAHQAaQBuAGcAUgBlAGcAdQBsAGEAcgByAGEAdABpAG4AZwBGAG8AbgB0ACAAZwBlAG4AZQByAGEAdABlAGQAIABiAHkAIABJAGMAbwBNAG8AbwBuAC4AAwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==) format('woff');font-weight:400;font-style:normal}.ui.rating .icon{font-family:Rating;line-height:1;-webkit-backface-visibility:hidden;backface-visibility:hidden;font-weight:400;font-style:normal;text-align:center}.ui.rating .icon:before{content:'\f005'}.ui.rating .active.icon:before{content:'\f005'}.ui.star.rating .icon:before{content:'\f005'}.ui.star.rating .active.icon:before{content:'\f005'}.ui.star.rating .partial.icon:before{content:'\f006'}.ui.star.rating .partial.icon{content:'\f005'}.ui.heart.rating .icon:before{content:'\f004'}.ui.heart.rating .active.icon:before{content:'\f004'}/*! - * # Semantic UI 2.4.0 - Search - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */.ui.search{position:relative}.ui.search>.prompt{margin:0;outline:0;-webkit-appearance:none;-webkit-tap-highlight-color:rgba(255,255,255,0);text-shadow:none;font-style:normal;font-weight:400;line-height:1.21428571em;padding:.67857143em 1em;font-size:1em;background:#fff;border:1px solid rgba(34,36,38,.15);color:rgba(0,0,0,.87);-webkit-box-shadow:0 0 0 0 transparent inset;box-shadow:0 0 0 0 transparent inset;-webkit-transition:background-color .1s ease,color .1s ease,border-color .1s ease,-webkit-box-shadow .1s ease;transition:background-color .1s ease,color .1s ease,border-color .1s ease,-webkit-box-shadow .1s ease;transition:background-color .1s ease,color .1s ease,box-shadow .1s ease,border-color .1s ease;transition:background-color .1s ease,color .1s ease,box-shadow .1s ease,border-color .1s ease,-webkit-box-shadow .1s ease}.ui.search .prompt{border-radius:500rem}.ui.search .prompt~.search.icon{cursor:pointer}.ui.search>.results{display:none;position:absolute;top:100%;left:0;-webkit-transform-origin:center top;transform-origin:center top;white-space:normal;text-align:left;text-transform:none;background:#fff;margin-top:.5em;width:18em;border-radius:.28571429rem;-webkit-box-shadow:0 2px 4px 0 rgba(34,36,38,.12),0 2px 10px 0 rgba(34,36,38,.15);box-shadow:0 2px 4px 0 rgba(34,36,38,.12),0 2px 10px 0 rgba(34,36,38,.15);border:1px solid #d4d4d5;z-index:998}.ui.search>.results>:first-child{border-radius:.28571429rem .28571429rem 0 0}.ui.search>.results>:last-child{border-radius:0 0 .28571429rem .28571429rem}.ui.search>.results .result{cursor:pointer;display:block;overflow:hidden;font-size:1em;padding:.85714286em 1.14285714em;color:rgba(0,0,0,.87);line-height:1.33;border-bottom:1px solid rgba(34,36,38,.1)}.ui.search>.results .result:last-child{border-bottom:none!important}.ui.search>.results .result .image{float:right;overflow:hidden;background:0 0;width:5em;height:3em;border-radius:.25em}.ui.search>.results .result .image img{display:block;width:auto;height:100%}.ui.search>.results .result .image+.content{margin:0 6em 0 0}.ui.search>.results .result .title{margin:-.14285714em 0 0;font-family:Lato,'Helvetica Neue',Arial,Helvetica,sans-serif;font-weight:700;font-size:1em;color:rgba(0,0,0,.85)}.ui.search>.results .result .description{margin-top:0;font-size:.92857143em;color:rgba(0,0,0,.4)}.ui.search>.results .result .price{float:right;color:#21ba45}.ui.search>.results>.message{padding:1em 1em}.ui.search>.results>.message .header{font-family:Lato,'Helvetica Neue',Arial,Helvetica,sans-serif;font-size:1rem;font-weight:700;color:rgba(0,0,0,.87)}.ui.search>.results>.message .description{margin-top:.25rem;font-size:1em;color:rgba(0,0,0,.87)}.ui.search>.results>.action{display:block;border-top:none;background:#f3f4f5;padding:.92857143em 1em;color:rgba(0,0,0,.87);font-weight:700;text-align:center}.ui.search>.prompt:focus{border-color:rgba(34,36,38,.35);background:#fff;color:rgba(0,0,0,.95)}.ui.loading.search .input>i.icon:before{position:absolute;content:'';top:50%;left:50%;margin:-.64285714em 0 0 -.64285714em;width:1.28571429em;height:1.28571429em;border-radius:500rem;border:.2em solid rgba(0,0,0,.1)}.ui.loading.search .input>i.icon:after{position:absolute;content:'';top:50%;left:50%;margin:-.64285714em 0 0 -.64285714em;width:1.28571429em;height:1.28571429em;-webkit-animation:button-spin .6s linear;animation:button-spin .6s linear;-webkit-animation-iteration-count:infinite;animation-iteration-count:infinite;border-radius:500rem;border-color:#767676 transparent transparent;border-style:solid;border-width:.2em;-webkit-box-shadow:0 0 0 1px transparent;box-shadow:0 0 0 1px transparent}.ui.category.search>.results .category .result:hover,.ui.search>.results .result:hover{background:#f9fafb}.ui.search .action:hover{background:#e0e0e0}.ui.category.search>.results .category.active{background:#f3f4f5}.ui.category.search>.results .category.active>.name{color:rgba(0,0,0,.87)}.ui.category.search>.results .category .result.active,.ui.search>.results .result.active{position:relative;border-left-color:rgba(34,36,38,.1);background:#f3f4f5;-webkit-box-shadow:none;box-shadow:none}.ui.search>.results .result.active .title{color:rgba(0,0,0,.85)}.ui.search>.results .result.active .description{color:rgba(0,0,0,.85)}.ui.disabled.search{cursor:default;pointer-events:none;opacity:.45}.ui.search.selection .prompt{border-radius:.28571429rem}.ui.search.selection>.icon.input>.remove.icon{pointer-events:none;position:absolute;left:auto;opacity:0;color:'';top:0;right:0;-webkit-transition:color .1s ease,opacity .1s ease;transition:color .1s ease,opacity .1s ease}.ui.search.selection>.icon.input>.active.remove.icon{cursor:pointer;opacity:.8;pointer-events:auto}.ui.search.selection>.icon.input:not([class*="left icon"])>.icon~.remove.icon{right:1.85714em}.ui.search.selection>.icon.input>.remove.icon:hover{opacity:1;color:#db2828}.ui.category.search .results{width:28em}.ui.category.search .results.animating,.ui.category.search .results.visible{display:table}.ui.category.search>.results .category{display:table-row;background:#f3f4f5;-webkit-box-shadow:none;box-shadow:none;-webkit-transition:background .1s ease,border-color .1s ease;transition:background .1s ease,border-color .1s ease}.ui.category.search>.results .category:last-child{border-bottom:none}.ui.category.search>.results .category:first-child .name+.result{border-radius:0 .28571429rem 0 0}.ui.category.search>.results .category:last-child .result:last-child{border-radius:0 0 .28571429rem 0}.ui.category.search>.results .category>.name{display:table-cell;text-overflow:ellipsis;width:100px;white-space:nowrap;background:0 0;font-family:Lato,'Helvetica Neue',Arial,Helvetica,sans-serif;font-size:1em;padding:.4em 1em;font-weight:700;color:rgba(0,0,0,.4);border-bottom:1px solid rgba(34,36,38,.1)}.ui.category.search>.results .category .results{display:table-cell;background:#fff;border-left:1px solid rgba(34,36,38,.15);border-bottom:1px solid rgba(34,36,38,.1)}.ui.category.search>.results .category .result{border-bottom:1px solid rgba(34,36,38,.1);-webkit-transition:background .1s ease,border-color .1s ease;transition:background .1s ease,border-color .1s ease;padding:.85714286em 1.14285714em}.ui[class*="left aligned"].search>.results{right:auto;left:0}.ui[class*="right aligned"].search>.results{right:0;left:auto}.ui.fluid.search .results{width:100%}.ui.mini.search{font-size:.78571429em}.ui.small.search{font-size:.92857143em}.ui.search{font-size:1em}.ui.large.search{font-size:1.14285714em}.ui.big.search{font-size:1.28571429em}.ui.huge.search{font-size:1.42857143em}.ui.massive.search{font-size:1.71428571em}@media only screen and (max-width:767px){.ui.search .results{max-width:calc(100vw - 2rem)}}/*! - * # Semantic UI 2.4.0 - Shape - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */.ui.shape{position:relative;vertical-align:top;display:inline-block;-webkit-perspective:2000px;perspective:2000px;-webkit-transition:left .6s ease-in-out,width .6s ease-in-out,height .6s ease-in-out,-webkit-transform .6s ease-in-out;transition:left .6s ease-in-out,width .6s ease-in-out,height .6s ease-in-out,-webkit-transform .6s ease-in-out;transition:transform .6s ease-in-out,left .6s ease-in-out,width .6s ease-in-out,height .6s ease-in-out;transition:transform .6s ease-in-out,left .6s ease-in-out,width .6s ease-in-out,height .6s ease-in-out,-webkit-transform .6s ease-in-out}.ui.shape .sides{-webkit-transform-style:preserve-3d;transform-style:preserve-3d}.ui.shape .side{opacity:1;width:100%;margin:0!important;-webkit-backface-visibility:hidden;backface-visibility:hidden}.ui.shape .side{display:none}.ui.shape .side *{-webkit-backface-visibility:visible!important;backface-visibility:visible!important}.ui.cube.shape .side{min-width:15em;height:15em;padding:2em;background-color:#e6e6e6;color:rgba(0,0,0,.87);-webkit-box-shadow:0 0 2px rgba(0,0,0,.3);box-shadow:0 0 2px rgba(0,0,0,.3)}.ui.cube.shape .side>.content{width:100%;height:100%;display:table;text-align:center;-webkit-user-select:text;-moz-user-select:text;-ms-user-select:text;user-select:text}.ui.cube.shape .side>.content>div{display:table-cell;vertical-align:middle;font-size:2em}.ui.text.shape.animating .sides{position:static}.ui.text.shape .side{white-space:nowrap}.ui.text.shape .side>*{white-space:normal}.ui.loading.shape{position:absolute;top:-9999px;left:-9999px}.ui.shape .animating.side{position:absolute;top:0;left:0;display:block;z-index:100}.ui.shape .hidden.side{opacity:.6}.ui.shape.animating .sides{position:absolute}.ui.shape.animating .sides{-webkit-transition:left .6s ease-in-out,width .6s ease-in-out,height .6s ease-in-out,-webkit-transform .6s ease-in-out;transition:left .6s ease-in-out,width .6s ease-in-out,height .6s ease-in-out,-webkit-transform .6s ease-in-out;transition:transform .6s ease-in-out,left .6s ease-in-out,width .6s ease-in-out,height .6s ease-in-out;transition:transform .6s ease-in-out,left .6s ease-in-out,width .6s ease-in-out,height .6s ease-in-out,-webkit-transform .6s ease-in-out}.ui.shape.animating .side{-webkit-transition:opacity .6s ease-in-out;transition:opacity .6s ease-in-out}.ui.shape .active.side{display:block}/*! - * # Semantic UI 2.4.0 - Sidebar - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */.ui.sidebar{position:fixed;top:0;left:0;-webkit-backface-visibility:hidden;backface-visibility:hidden;-webkit-transition:none;transition:none;will-change:transform;-webkit-transform:translate3d(0,0,0);transform:translate3d(0,0,0);visibility:hidden;-webkit-overflow-scrolling:touch;height:100%!important;max-height:100%;border-radius:0!important;margin:0!important;overflow-y:auto!important;z-index:102}.ui.sidebar>*{-webkit-backface-visibility:hidden;backface-visibility:hidden}.ui.left.sidebar{right:auto;left:0;-webkit-transform:translate3d(-100%,0,0);transform:translate3d(-100%,0,0)}.ui.right.sidebar{right:0!important;left:auto!important;-webkit-transform:translate3d(100%,0,0);transform:translate3d(100%,0,0)}.ui.bottom.sidebar,.ui.top.sidebar{width:100%!important;height:auto!important}.ui.top.sidebar{top:0!important;bottom:auto!important;-webkit-transform:translate3d(0,-100%,0);transform:translate3d(0,-100%,0)}.ui.bottom.sidebar{top:auto!important;bottom:0!important;-webkit-transform:translate3d(0,100%,0);transform:translate3d(0,100%,0)}.pushable{height:100%;overflow-x:hidden;padding:0!important}body.pushable{background:#545454!important}.pushable:not(body){-webkit-transform:translate3d(0,0,0);transform:translate3d(0,0,0)}.pushable:not(body)>.fixed,.pushable:not(body)>.pusher:after,.pushable:not(body)>.ui.sidebar{position:absolute}.pushable>.fixed{position:fixed;-webkit-backface-visibility:hidden;backface-visibility:hidden;-webkit-transition:-webkit-transform .5s ease;transition:-webkit-transform .5s ease;transition:transform .5s ease;transition:transform .5s ease,-webkit-transform .5s ease;will-change:transform;z-index:101}.pushable>.pusher{position:relative;-webkit-backface-visibility:hidden;backface-visibility:hidden;overflow:hidden;min-height:100%;-webkit-transition:-webkit-transform .5s ease;transition:-webkit-transform .5s ease;transition:transform .5s ease;transition:transform .5s ease,-webkit-transform .5s ease;z-index:2}body.pushable>.pusher{background:#fff}.pushable>.pusher{background:inherit}.pushable>.pusher:after{position:fixed;top:0;right:0;content:'';background-color:rgba(0,0,0,.4);overflow:hidden;opacity:0;-webkit-transition:opacity .5s;transition:opacity .5s;will-change:opacity;z-index:1000}.ui.sidebar.menu .item{border-radius:0!important}.pushable>.pusher.dimmed:after{width:100%!important;height:100%!important;opacity:1!important}.ui.animating.sidebar{visibility:visible}.ui.visible.sidebar{visibility:visible;-webkit-transform:translate3d(0,0,0);transform:translate3d(0,0,0)}.ui.left.visible.sidebar,.ui.right.visible.sidebar{-webkit-box-shadow:0 0 20px rgba(34,36,38,.15);box-shadow:0 0 20px rgba(34,36,38,.15)}.ui.bottom.visible.sidebar,.ui.top.visible.sidebar{-webkit-box-shadow:0 0 20px rgba(34,36,38,.15);box-shadow:0 0 20px rgba(34,36,38,.15)}.ui.visible.left.sidebar~.fixed,.ui.visible.left.sidebar~.pusher{-webkit-transform:translate3d(260px,0,0);transform:translate3d(260px,0,0)}.ui.visible.right.sidebar~.fixed,.ui.visible.right.sidebar~.pusher{-webkit-transform:translate3d(-260px,0,0);transform:translate3d(-260px,0,0)}.ui.visible.top.sidebar~.fixed,.ui.visible.top.sidebar~.pusher{-webkit-transform:translate3d(0,36px,0);transform:translate3d(0,36px,0)}.ui.visible.bottom.sidebar~.fixed,.ui.visible.bottom.sidebar~.pusher{-webkit-transform:translate3d(0,-36px,0);transform:translate3d(0,-36px,0)}.ui.visible.left.sidebar~.ui.visible.right.sidebar~.fixed,.ui.visible.left.sidebar~.ui.visible.right.sidebar~.pusher,.ui.visible.right.sidebar~.ui.visible.left.sidebar~.fixed,.ui.visible.right.sidebar~.ui.visible.left.sidebar~.pusher{-webkit-transform:translate3d(0,0,0);transform:translate3d(0,0,0)}.ui.thin.left.sidebar,.ui.thin.right.sidebar{width:150px}.ui[class*="very thin"].left.sidebar,.ui[class*="very thin"].right.sidebar{width:60px}.ui.left.sidebar,.ui.right.sidebar{width:260px}.ui.wide.left.sidebar,.ui.wide.right.sidebar{width:350px}.ui[class*="very wide"].left.sidebar,.ui[class*="very wide"].right.sidebar{width:475px}.ui.visible.thin.left.sidebar~.fixed,.ui.visible.thin.left.sidebar~.pusher{-webkit-transform:translate3d(150px,0,0);transform:translate3d(150px,0,0)}.ui.visible[class*="very thin"].left.sidebar~.fixed,.ui.visible[class*="very thin"].left.sidebar~.pusher{-webkit-transform:translate3d(60px,0,0);transform:translate3d(60px,0,0)}.ui.visible.wide.left.sidebar~.fixed,.ui.visible.wide.left.sidebar~.pusher{-webkit-transform:translate3d(350px,0,0);transform:translate3d(350px,0,0)}.ui.visible[class*="very wide"].left.sidebar~.fixed,.ui.visible[class*="very wide"].left.sidebar~.pusher{-webkit-transform:translate3d(475px,0,0);transform:translate3d(475px,0,0)}.ui.visible.thin.right.sidebar~.fixed,.ui.visible.thin.right.sidebar~.pusher{-webkit-transform:translate3d(-150px,0,0);transform:translate3d(-150px,0,0)}.ui.visible[class*="very thin"].right.sidebar~.fixed,.ui.visible[class*="very thin"].right.sidebar~.pusher{-webkit-transform:translate3d(-60px,0,0);transform:translate3d(-60px,0,0)}.ui.visible.wide.right.sidebar~.fixed,.ui.visible.wide.right.sidebar~.pusher{-webkit-transform:translate3d(-350px,0,0);transform:translate3d(-350px,0,0)}.ui.visible[class*="very wide"].right.sidebar~.fixed,.ui.visible[class*="very wide"].right.sidebar~.pusher{-webkit-transform:translate3d(-475px,0,0);transform:translate3d(-475px,0,0)}.ui.overlay.sidebar{z-index:102}.ui.left.overlay.sidebar{-webkit-transform:translate3d(-100%,0,0);transform:translate3d(-100%,0,0)}.ui.right.overlay.sidebar{-webkit-transform:translate3d(100%,0,0);transform:translate3d(100%,0,0)}.ui.top.overlay.sidebar{-webkit-transform:translate3d(0,-100%,0);transform:translate3d(0,-100%,0)}.ui.bottom.overlay.sidebar{-webkit-transform:translate3d(0,100%,0);transform:translate3d(0,100%,0)}.animating.ui.overlay.sidebar,.ui.visible.overlay.sidebar{-webkit-transition:-webkit-transform .5s ease;transition:-webkit-transform .5s ease;transition:transform .5s ease;transition:transform .5s ease,-webkit-transform .5s ease}.ui.visible.left.overlay.sidebar{-webkit-transform:translate3d(0,0,0);transform:translate3d(0,0,0)}.ui.visible.right.overlay.sidebar{-webkit-transform:translate3d(0,0,0);transform:translate3d(0,0,0)}.ui.visible.top.overlay.sidebar{-webkit-transform:translate3d(0,0,0);transform:translate3d(0,0,0)}.ui.visible.bottom.overlay.sidebar{-webkit-transform:translate3d(0,0,0);transform:translate3d(0,0,0)}.ui.visible.overlay.sidebar~.fixed,.ui.visible.overlay.sidebar~.pusher{-webkit-transform:none!important;transform:none!important}.ui.push.sidebar{-webkit-transition:-webkit-transform .5s ease;transition:-webkit-transform .5s ease;transition:transform .5s ease;transition:transform .5s ease,-webkit-transform .5s ease;z-index:102}.ui.left.push.sidebar{-webkit-transform:translate3d(-100%,0,0);transform:translate3d(-100%,0,0)}.ui.right.push.sidebar{-webkit-transform:translate3d(100%,0,0);transform:translate3d(100%,0,0)}.ui.top.push.sidebar{-webkit-transform:translate3d(0,-100%,0);transform:translate3d(0,-100%,0)}.ui.bottom.push.sidebar{-webkit-transform:translate3d(0,100%,0);transform:translate3d(0,100%,0)}.ui.visible.push.sidebar{-webkit-transform:translate3d(0,0,0);transform:translate3d(0,0,0)}.ui.uncover.sidebar{-webkit-transform:translate3d(0,0,0);transform:translate3d(0,0,0);z-index:1}.ui.visible.uncover.sidebar{-webkit-transform:translate3d(0,0,0);transform:translate3d(0,0,0);-webkit-transition:-webkit-transform .5s ease;transition:-webkit-transform .5s ease;transition:transform .5s ease;transition:transform .5s ease,-webkit-transform .5s ease}.ui.slide.along.sidebar{z-index:1}.ui.left.slide.along.sidebar{-webkit-transform:translate3d(-50%,0,0);transform:translate3d(-50%,0,0)}.ui.right.slide.along.sidebar{-webkit-transform:translate3d(50%,0,0);transform:translate3d(50%,0,0)}.ui.top.slide.along.sidebar{-webkit-transform:translate3d(0,-50%,0);transform:translate3d(0,-50%,0)}.ui.bottom.slide.along.sidebar{-webkit-transform:translate3d(0,50%,0);transform:translate3d(0,50%,0)}.ui.animating.slide.along.sidebar{-webkit-transition:-webkit-transform .5s ease;transition:-webkit-transform .5s ease;transition:transform .5s ease;transition:transform .5s ease,-webkit-transform .5s ease}.ui.visible.slide.along.sidebar{-webkit-transform:translate3d(0,0,0);transform:translate3d(0,0,0)}.ui.slide.out.sidebar{z-index:1}.ui.left.slide.out.sidebar{-webkit-transform:translate3d(50%,0,0);transform:translate3d(50%,0,0)}.ui.right.slide.out.sidebar{-webkit-transform:translate3d(-50%,0,0);transform:translate3d(-50%,0,0)}.ui.top.slide.out.sidebar{-webkit-transform:translate3d(0,50%,0);transform:translate3d(0,50%,0)}.ui.bottom.slide.out.sidebar{-webkit-transform:translate3d(0,-50%,0);transform:translate3d(0,-50%,0)}.ui.animating.slide.out.sidebar{-webkit-transition:-webkit-transform .5s ease;transition:-webkit-transform .5s ease;transition:transform .5s ease;transition:transform .5s ease,-webkit-transform .5s ease}.ui.visible.slide.out.sidebar{-webkit-transform:translate3d(0,0,0);transform:translate3d(0,0,0)}.ui.scale.down.sidebar{-webkit-transition:-webkit-transform .5s ease;transition:-webkit-transform .5s ease;transition:transform .5s ease;transition:transform .5s ease,-webkit-transform .5s ease;z-index:102}.ui.left.scale.down.sidebar{-webkit-transform:translate3d(-100%,0,0);transform:translate3d(-100%,0,0)}.ui.right.scale.down.sidebar{-webkit-transform:translate3d(100%,0,0);transform:translate3d(100%,0,0)}.ui.top.scale.down.sidebar{-webkit-transform:translate3d(0,-100%,0);transform:translate3d(0,-100%,0)}.ui.bottom.scale.down.sidebar{-webkit-transform:translate3d(0,100%,0);transform:translate3d(0,100%,0)}.ui.scale.down.left.sidebar~.pusher{-webkit-transform-origin:75% 50%;transform-origin:75% 50%}.ui.scale.down.right.sidebar~.pusher{-webkit-transform-origin:25% 50%;transform-origin:25% 50%}.ui.scale.down.top.sidebar~.pusher{-webkit-transform-origin:50% 75%;transform-origin:50% 75%}.ui.scale.down.bottom.sidebar~.pusher{-webkit-transform-origin:50% 25%;transform-origin:50% 25%}.ui.animating.scale.down>.visible.ui.sidebar{-webkit-transition:-webkit-transform .5s ease;transition:-webkit-transform .5s ease;transition:transform .5s ease;transition:transform .5s ease,-webkit-transform .5s ease}.ui.animating.scale.down.sidebar~.pusher,.ui.visible.scale.down.sidebar~.pusher{display:block!important;width:100%;height:100%;overflow:hidden!important}.ui.visible.scale.down.sidebar{-webkit-transform:translate3d(0,0,0);transform:translate3d(0,0,0)}.ui.visible.scale.down.sidebar~.pusher{-webkit-transform:scale(.75);transform:scale(.75)}/*! - * # Semantic UI 2.4.0 - Sticky - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */.ui.sticky{position:static;-webkit-transition:none;transition:none;z-index:800}.ui.sticky.bound{position:absolute;left:auto;right:auto}.ui.sticky.fixed{position:fixed;left:auto;right:auto}.ui.sticky.bound.top,.ui.sticky.fixed.top{top:0;bottom:auto}.ui.sticky.bound.bottom,.ui.sticky.fixed.bottom{top:auto;bottom:0}.ui.native.sticky{position:-webkit-sticky;position:-moz-sticky;position:-ms-sticky;position:-o-sticky;position:sticky}/*! - * # Semantic UI 2.4.0 - Tab - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */.ui.tab{display:none}.ui.tab.active,.ui.tab.open{display:block}.ui.tab.loading{position:relative;overflow:hidden;display:block;min-height:250px}.ui.tab.loading *{position:relative!important;left:-10000px!important}.ui.tab.loading.segment:before,.ui.tab.loading:before{position:absolute;content:'';top:100px;left:50%;margin:-1.25em 0 0 -1.25em;width:2.5em;height:2.5em;border-radius:500rem;border:.2em solid rgba(0,0,0,.1)}.ui.tab.loading.segment:after,.ui.tab.loading:after{position:absolute;content:'';top:100px;left:50%;margin:-1.25em 0 0 -1.25em;width:2.5em;height:2.5em;-webkit-animation:button-spin .6s linear;animation:button-spin .6s linear;-webkit-animation-iteration-count:infinite;animation-iteration-count:infinite;border-radius:500rem;border-color:#767676 transparent transparent;border-style:solid;border-width:.2em;-webkit-box-shadow:0 0 0 1px transparent;box-shadow:0 0 0 1px transparent}/*! - * # Semantic UI 2.4.0 - Transition - * http://github.com/semantic-org/semantic-ui/ - * - * - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */.transition{-webkit-animation-iteration-count:1;animation-iteration-count:1;-webkit-animation-duration:.3s;animation-duration:.3s;-webkit-animation-timing-function:ease;animation-timing-function:ease;-webkit-animation-fill-mode:both;animation-fill-mode:both}.animating.transition{-webkit-backface-visibility:hidden;backface-visibility:hidden;visibility:visible!important}.loading.transition{position:absolute;top:-99999px;left:-99999px}.hidden.transition{display:none;visibility:hidden}.visible.transition{display:block!important;visibility:visible!important}.disabled.transition{-webkit-animation-play-state:paused;animation-play-state:paused}.looping.transition{-webkit-animation-iteration-count:infinite;animation-iteration-count:infinite}.transition.browse{-webkit-animation-duration:.5s;animation-duration:.5s}.transition.browse.in{-webkit-animation-name:browseIn;animation-name:browseIn}.transition.browse.left.out,.transition.browse.out{-webkit-animation-name:browseOutLeft;animation-name:browseOutLeft}.transition.browse.right.out{-webkit-animation-name:browseOutRight;animation-name:browseOutRight}@-webkit-keyframes browseIn{0%{-webkit-transform:scale(.8) translateZ(0);transform:scale(.8) translateZ(0);z-index:-1}10%{-webkit-transform:scale(.8) translateZ(0);transform:scale(.8) translateZ(0);z-index:-1;opacity:.7}80%{-webkit-transform:scale(1.05) translateZ(0);transform:scale(1.05) translateZ(0);opacity:1;z-index:999}100%{-webkit-transform:scale(1) translateZ(0);transform:scale(1) translateZ(0);z-index:999}}@keyframes browseIn{0%{-webkit-transform:scale(.8) translateZ(0);transform:scale(.8) translateZ(0);z-index:-1}10%{-webkit-transform:scale(.8) translateZ(0);transform:scale(.8) translateZ(0);z-index:-1;opacity:.7}80%{-webkit-transform:scale(1.05) translateZ(0);transform:scale(1.05) translateZ(0);opacity:1;z-index:999}100%{-webkit-transform:scale(1) translateZ(0);transform:scale(1) translateZ(0);z-index:999}}@-webkit-keyframes browseOutLeft{0%{z-index:999;-webkit-transform:translateX(0) rotateY(0) rotateX(0);transform:translateX(0) rotateY(0) rotateX(0)}50%{z-index:-1;-webkit-transform:translateX(-105%) rotateY(35deg) rotateX(10deg) translateZ(-10px);transform:translateX(-105%) rotateY(35deg) rotateX(10deg) translateZ(-10px)}80%{opacity:1}100%{z-index:-1;-webkit-transform:translateX(0) rotateY(0) rotateX(0) translateZ(-10px);transform:translateX(0) rotateY(0) rotateX(0) translateZ(-10px);opacity:0}}@keyframes browseOutLeft{0%{z-index:999;-webkit-transform:translateX(0) rotateY(0) rotateX(0);transform:translateX(0) rotateY(0) rotateX(0)}50%{z-index:-1;-webkit-transform:translateX(-105%) rotateY(35deg) rotateX(10deg) translateZ(-10px);transform:translateX(-105%) rotateY(35deg) rotateX(10deg) translateZ(-10px)}80%{opacity:1}100%{z-index:-1;-webkit-transform:translateX(0) rotateY(0) rotateX(0) translateZ(-10px);transform:translateX(0) rotateY(0) rotateX(0) translateZ(-10px);opacity:0}}@-webkit-keyframes browseOutRight{0%{z-index:999;-webkit-transform:translateX(0) rotateY(0) rotateX(0);transform:translateX(0) rotateY(0) rotateX(0)}50%{z-index:1;-webkit-transform:translateX(105%) rotateY(35deg) rotateX(10deg) translateZ(-10px);transform:translateX(105%) rotateY(35deg) rotateX(10deg) translateZ(-10px)}80%{opacity:1}100%{z-index:1;-webkit-transform:translateX(0) rotateY(0) rotateX(0) translateZ(-10px);transform:translateX(0) rotateY(0) rotateX(0) translateZ(-10px);opacity:0}}@keyframes browseOutRight{0%{z-index:999;-webkit-transform:translateX(0) rotateY(0) rotateX(0);transform:translateX(0) rotateY(0) rotateX(0)}50%{z-index:1;-webkit-transform:translateX(105%) rotateY(35deg) rotateX(10deg) translateZ(-10px);transform:translateX(105%) rotateY(35deg) rotateX(10deg) translateZ(-10px)}80%{opacity:1}100%{z-index:1;-webkit-transform:translateX(0) rotateY(0) rotateX(0) translateZ(-10px);transform:translateX(0) rotateY(0) rotateX(0) translateZ(-10px);opacity:0}}.drop.transition{-webkit-transform-origin:top center;transform-origin:top center;-webkit-animation-duration:.4s;animation-duration:.4s;-webkit-animation-timing-function:cubic-bezier(.34,1.61,.7,1);animation-timing-function:cubic-bezier(.34,1.61,.7,1)}.drop.transition.in{-webkit-animation-name:dropIn;animation-name:dropIn}.drop.transition.out{-webkit-animation-name:dropOut;animation-name:dropOut}@-webkit-keyframes dropIn{0%{opacity:0;-webkit-transform:scale(0);transform:scale(0)}100%{opacity:1;-webkit-transform:scale(1);transform:scale(1)}}@keyframes dropIn{0%{opacity:0;-webkit-transform:scale(0);transform:scale(0)}100%{opacity:1;-webkit-transform:scale(1);transform:scale(1)}}@-webkit-keyframes dropOut{0%{opacity:1;-webkit-transform:scale(1);transform:scale(1)}100%{opacity:0;-webkit-transform:scale(0);transform:scale(0)}}@keyframes dropOut{0%{opacity:1;-webkit-transform:scale(1);transform:scale(1)}100%{opacity:0;-webkit-transform:scale(0);transform:scale(0)}}.transition.fade.in{-webkit-animation-name:fadeIn;animation-name:fadeIn}.transition[class*="fade up"].in{-webkit-animation-name:fadeInUp;animation-name:fadeInUp}.transition[class*="fade down"].in{-webkit-animation-name:fadeInDown;animation-name:fadeInDown}.transition[class*="fade left"].in{-webkit-animation-name:fadeInLeft;animation-name:fadeInLeft}.transition[class*="fade right"].in{-webkit-animation-name:fadeInRight;animation-name:fadeInRight}.transition.fade.out{-webkit-animation-name:fadeOut;animation-name:fadeOut}.transition[class*="fade up"].out{-webkit-animation-name:fadeOutUp;animation-name:fadeOutUp}.transition[class*="fade down"].out{-webkit-animation-name:fadeOutDown;animation-name:fadeOutDown}.transition[class*="fade left"].out{-webkit-animation-name:fadeOutLeft;animation-name:fadeOutLeft}.transition[class*="fade right"].out{-webkit-animation-name:fadeOutRight;animation-name:fadeOutRight}@-webkit-keyframes fadeIn{0%{opacity:0}100%{opacity:1}}@keyframes fadeIn{0%{opacity:0}100%{opacity:1}}@-webkit-keyframes fadeInUp{0%{opacity:0;-webkit-transform:translateY(10%);transform:translateY(10%)}100%{opacity:1;-webkit-transform:translateY(0);transform:translateY(0)}}@keyframes fadeInUp{0%{opacity:0;-webkit-transform:translateY(10%);transform:translateY(10%)}100%{opacity:1;-webkit-transform:translateY(0);transform:translateY(0)}}@-webkit-keyframes fadeInDown{0%{opacity:0;-webkit-transform:translateY(-10%);transform:translateY(-10%)}100%{opacity:1;-webkit-transform:translateY(0);transform:translateY(0)}}@keyframes fadeInDown{0%{opacity:0;-webkit-transform:translateY(-10%);transform:translateY(-10%)}100%{opacity:1;-webkit-transform:translateY(0);transform:translateY(0)}}@-webkit-keyframes fadeInLeft{0%{opacity:0;-webkit-transform:translateX(10%);transform:translateX(10%)}100%{opacity:1;-webkit-transform:translateX(0);transform:translateX(0)}}@keyframes fadeInLeft{0%{opacity:0;-webkit-transform:translateX(10%);transform:translateX(10%)}100%{opacity:1;-webkit-transform:translateX(0);transform:translateX(0)}}@-webkit-keyframes fadeInRight{0%{opacity:0;-webkit-transform:translateX(-10%);transform:translateX(-10%)}100%{opacity:1;-webkit-transform:translateX(0);transform:translateX(0)}}@keyframes fadeInRight{0%{opacity:0;-webkit-transform:translateX(-10%);transform:translateX(-10%)}100%{opacity:1;-webkit-transform:translateX(0);transform:translateX(0)}}@-webkit-keyframes fadeOut{0%{opacity:1}100%{opacity:0}}@keyframes fadeOut{0%{opacity:1}100%{opacity:0}}@-webkit-keyframes fadeOutUp{0%{opacity:1;-webkit-transform:translateY(0);transform:translateY(0)}100%{opacity:0;-webkit-transform:translateY(5%);transform:translateY(5%)}}@keyframes fadeOutUp{0%{opacity:1;-webkit-transform:translateY(0);transform:translateY(0)}100%{opacity:0;-webkit-transform:translateY(5%);transform:translateY(5%)}}@-webkit-keyframes fadeOutDown{0%{opacity:1;-webkit-transform:translateY(0);transform:translateY(0)}100%{opacity:0;-webkit-transform:translateY(-5%);transform:translateY(-5%)}}@keyframes fadeOutDown{0%{opacity:1;-webkit-transform:translateY(0);transform:translateY(0)}100%{opacity:0;-webkit-transform:translateY(-5%);transform:translateY(-5%)}}@-webkit-keyframes fadeOutLeft{0%{opacity:1;-webkit-transform:translateX(0);transform:translateX(0)}100%{opacity:0;-webkit-transform:translateX(5%);transform:translateX(5%)}}@keyframes fadeOutLeft{0%{opacity:1;-webkit-transform:translateX(0);transform:translateX(0)}100%{opacity:0;-webkit-transform:translateX(5%);transform:translateX(5%)}}@-webkit-keyframes fadeOutRight{0%{opacity:1;-webkit-transform:translateX(0);transform:translateX(0)}100%{opacity:0;-webkit-transform:translateX(-5%);transform:translateX(-5%)}}@keyframes fadeOutRight{0%{opacity:1;-webkit-transform:translateX(0);transform:translateX(0)}100%{opacity:0;-webkit-transform:translateX(-5%);transform:translateX(-5%)}}.flip.transition.in,.flip.transition.out{-webkit-animation-duration:.6s;animation-duration:.6s}.horizontal.flip.transition.in{-webkit-animation-name:horizontalFlipIn;animation-name:horizontalFlipIn}.horizontal.flip.transition.out{-webkit-animation-name:horizontalFlipOut;animation-name:horizontalFlipOut}.vertical.flip.transition.in{-webkit-animation-name:verticalFlipIn;animation-name:verticalFlipIn}.vertical.flip.transition.out{-webkit-animation-name:verticalFlipOut;animation-name:verticalFlipOut}@-webkit-keyframes horizontalFlipIn{0%{-webkit-transform:perspective(2000px) rotateY(-90deg);transform:perspective(2000px) rotateY(-90deg);opacity:0}100%{-webkit-transform:perspective(2000px) rotateY(0);transform:perspective(2000px) rotateY(0);opacity:1}}@keyframes horizontalFlipIn{0%{-webkit-transform:perspective(2000px) rotateY(-90deg);transform:perspective(2000px) rotateY(-90deg);opacity:0}100%{-webkit-transform:perspective(2000px) rotateY(0);transform:perspective(2000px) rotateY(0);opacity:1}}@-webkit-keyframes verticalFlipIn{0%{-webkit-transform:perspective(2000px) rotateX(-90deg);transform:perspective(2000px) rotateX(-90deg);opacity:0}100%{-webkit-transform:perspective(2000px) rotateX(0);transform:perspective(2000px) rotateX(0);opacity:1}}@keyframes verticalFlipIn{0%{-webkit-transform:perspective(2000px) rotateX(-90deg);transform:perspective(2000px) rotateX(-90deg);opacity:0}100%{-webkit-transform:perspective(2000px) rotateX(0);transform:perspective(2000px) rotateX(0);opacity:1}}@-webkit-keyframes horizontalFlipOut{0%{-webkit-transform:perspective(2000px) rotateY(0);transform:perspective(2000px) rotateY(0);opacity:1}100%{-webkit-transform:perspective(2000px) rotateY(90deg);transform:perspective(2000px) rotateY(90deg);opacity:0}}@keyframes horizontalFlipOut{0%{-webkit-transform:perspective(2000px) rotateY(0);transform:perspective(2000px) rotateY(0);opacity:1}100%{-webkit-transform:perspective(2000px) rotateY(90deg);transform:perspective(2000px) rotateY(90deg);opacity:0}}@-webkit-keyframes verticalFlipOut{0%{-webkit-transform:perspective(2000px) rotateX(0);transform:perspective(2000px) rotateX(0);opacity:1}100%{-webkit-transform:perspective(2000px) rotateX(-90deg);transform:perspective(2000px) rotateX(-90deg);opacity:0}}@keyframes verticalFlipOut{0%{-webkit-transform:perspective(2000px) rotateX(0);transform:perspective(2000px) rotateX(0);opacity:1}100%{-webkit-transform:perspective(2000px) rotateX(-90deg);transform:perspective(2000px) rotateX(-90deg);opacity:0}}.scale.transition.in{-webkit-animation-name:scaleIn;animation-name:scaleIn}.scale.transition.out{-webkit-animation-name:scaleOut;animation-name:scaleOut}@-webkit-keyframes scaleIn{0%{opacity:0;-webkit-transform:scale(.8);transform:scale(.8)}100%{opacity:1;-webkit-transform:scale(1);transform:scale(1)}}@keyframes scaleIn{0%{opacity:0;-webkit-transform:scale(.8);transform:scale(.8)}100%{opacity:1;-webkit-transform:scale(1);transform:scale(1)}}@-webkit-keyframes scaleOut{0%{opacity:1;-webkit-transform:scale(1);transform:scale(1)}100%{opacity:0;-webkit-transform:scale(.9);transform:scale(.9)}}@keyframes scaleOut{0%{opacity:1;-webkit-transform:scale(1);transform:scale(1)}100%{opacity:0;-webkit-transform:scale(.9);transform:scale(.9)}}.transition.fly{-webkit-animation-duration:.6s;animation-duration:.6s;-webkit-transition-timing-function:cubic-bezier(.215,.61,.355,1);transition-timing-function:cubic-bezier(.215,.61,.355,1)}.transition.fly.in{-webkit-animation-name:flyIn;animation-name:flyIn}.transition[class*="fly up"].in{-webkit-animation-name:flyInUp;animation-name:flyInUp}.transition[class*="fly down"].in{-webkit-animation-name:flyInDown;animation-name:flyInDown}.transition[class*="fly left"].in{-webkit-animation-name:flyInLeft;animation-name:flyInLeft}.transition[class*="fly right"].in{-webkit-animation-name:flyInRight;animation-name:flyInRight}.transition.fly.out{-webkit-animation-name:flyOut;animation-name:flyOut}.transition[class*="fly up"].out{-webkit-animation-name:flyOutUp;animation-name:flyOutUp}.transition[class*="fly down"].out{-webkit-animation-name:flyOutDown;animation-name:flyOutDown}.transition[class*="fly left"].out{-webkit-animation-name:flyOutLeft;animation-name:flyOutLeft}.transition[class*="fly right"].out{-webkit-animation-name:flyOutRight;animation-name:flyOutRight}@-webkit-keyframes flyIn{0%{opacity:0;-webkit-transform:scale3d(.3,.3,.3);transform:scale3d(.3,.3,.3)}20%{-webkit-transform:scale3d(1.1,1.1,1.1);transform:scale3d(1.1,1.1,1.1)}40%{-webkit-transform:scale3d(.9,.9,.9);transform:scale3d(.9,.9,.9)}60%{opacity:1;-webkit-transform:scale3d(1.03,1.03,1.03);transform:scale3d(1.03,1.03,1.03)}80%{-webkit-transform:scale3d(.97,.97,.97);transform:scale3d(.97,.97,.97)}100%{opacity:1;-webkit-transform:scale3d(1,1,1);transform:scale3d(1,1,1)}}@keyframes flyIn{0%{opacity:0;-webkit-transform:scale3d(.3,.3,.3);transform:scale3d(.3,.3,.3)}20%{-webkit-transform:scale3d(1.1,1.1,1.1);transform:scale3d(1.1,1.1,1.1)}40%{-webkit-transform:scale3d(.9,.9,.9);transform:scale3d(.9,.9,.9)}60%{opacity:1;-webkit-transform:scale3d(1.03,1.03,1.03);transform:scale3d(1.03,1.03,1.03)}80%{-webkit-transform:scale3d(.97,.97,.97);transform:scale3d(.97,.97,.97)}100%{opacity:1;-webkit-transform:scale3d(1,1,1);transform:scale3d(1,1,1)}}@-webkit-keyframes flyInUp{0%{opacity:0;-webkit-transform:translate3d(0,1500px,0);transform:translate3d(0,1500px,0)}60%{opacity:1;-webkit-transform:translate3d(0,-20px,0);transform:translate3d(0,-20px,0)}75%{-webkit-transform:translate3d(0,10px,0);transform:translate3d(0,10px,0)}90%{-webkit-transform:translate3d(0,-5px,0);transform:translate3d(0,-5px,0)}100%{-webkit-transform:translate3d(0,0,0);transform:translate3d(0,0,0)}}@keyframes flyInUp{0%{opacity:0;-webkit-transform:translate3d(0,1500px,0);transform:translate3d(0,1500px,0)}60%{opacity:1;-webkit-transform:translate3d(0,-20px,0);transform:translate3d(0,-20px,0)}75%{-webkit-transform:translate3d(0,10px,0);transform:translate3d(0,10px,0)}90%{-webkit-transform:translate3d(0,-5px,0);transform:translate3d(0,-5px,0)}100%{-webkit-transform:translate3d(0,0,0);transform:translate3d(0,0,0)}}@-webkit-keyframes flyInDown{0%{opacity:0;-webkit-transform:translate3d(0,-1500px,0);transform:translate3d(0,-1500px,0)}60%{opacity:1;-webkit-transform:translate3d(0,25px,0);transform:translate3d(0,25px,0)}75%{-webkit-transform:translate3d(0,-10px,0);transform:translate3d(0,-10px,0)}90%{-webkit-transform:translate3d(0,5px,0);transform:translate3d(0,5px,0)}100%{-webkit-transform:none;transform:none}}@keyframes flyInDown{0%{opacity:0;-webkit-transform:translate3d(0,-1500px,0);transform:translate3d(0,-1500px,0)}60%{opacity:1;-webkit-transform:translate3d(0,25px,0);transform:translate3d(0,25px,0)}75%{-webkit-transform:translate3d(0,-10px,0);transform:translate3d(0,-10px,0)}90%{-webkit-transform:translate3d(0,5px,0);transform:translate3d(0,5px,0)}100%{-webkit-transform:none;transform:none}}@-webkit-keyframes flyInLeft{0%{opacity:0;-webkit-transform:translate3d(1500px,0,0);transform:translate3d(1500px,0,0)}60%{opacity:1;-webkit-transform:translate3d(-25px,0,0);transform:translate3d(-25px,0,0)}75%{-webkit-transform:translate3d(10px,0,0);transform:translate3d(10px,0,0)}90%{-webkit-transform:translate3d(-5px,0,0);transform:translate3d(-5px,0,0)}100%{-webkit-transform:none;transform:none}}@keyframes flyInLeft{0%{opacity:0;-webkit-transform:translate3d(1500px,0,0);transform:translate3d(1500px,0,0)}60%{opacity:1;-webkit-transform:translate3d(-25px,0,0);transform:translate3d(-25px,0,0)}75%{-webkit-transform:translate3d(10px,0,0);transform:translate3d(10px,0,0)}90%{-webkit-transform:translate3d(-5px,0,0);transform:translate3d(-5px,0,0)}100%{-webkit-transform:none;transform:none}}@-webkit-keyframes flyInRight{0%{opacity:0;-webkit-transform:translate3d(-1500px,0,0);transform:translate3d(-1500px,0,0)}60%{opacity:1;-webkit-transform:translate3d(25px,0,0);transform:translate3d(25px,0,0)}75%{-webkit-transform:translate3d(-10px,0,0);transform:translate3d(-10px,0,0)}90%{-webkit-transform:translate3d(5px,0,0);transform:translate3d(5px,0,0)}100%{-webkit-transform:none;transform:none}}@keyframes flyInRight{0%{opacity:0;-webkit-transform:translate3d(-1500px,0,0);transform:translate3d(-1500px,0,0)}60%{opacity:1;-webkit-transform:translate3d(25px,0,0);transform:translate3d(25px,0,0)}75%{-webkit-transform:translate3d(-10px,0,0);transform:translate3d(-10px,0,0)}90%{-webkit-transform:translate3d(5px,0,0);transform:translate3d(5px,0,0)}100%{-webkit-transform:none;transform:none}}@-webkit-keyframes flyOut{20%{-webkit-transform:scale3d(.9,.9,.9);transform:scale3d(.9,.9,.9)}50%,55%{opacity:1;-webkit-transform:scale3d(1.1,1.1,1.1);transform:scale3d(1.1,1.1,1.1)}100%{opacity:0;-webkit-transform:scale3d(.3,.3,.3);transform:scale3d(.3,.3,.3)}}@keyframes flyOut{20%{-webkit-transform:scale3d(.9,.9,.9);transform:scale3d(.9,.9,.9)}50%,55%{opacity:1;-webkit-transform:scale3d(1.1,1.1,1.1);transform:scale3d(1.1,1.1,1.1)}100%{opacity:0;-webkit-transform:scale3d(.3,.3,.3);transform:scale3d(.3,.3,.3)}}@-webkit-keyframes flyOutUp{20%{-webkit-transform:translate3d(0,10px,0);transform:translate3d(0,10px,0)}40%,45%{opacity:1;-webkit-transform:translate3d(0,-20px,0);transform:translate3d(0,-20px,0)}100%{opacity:0;-webkit-transform:translate3d(0,2000px,0);transform:translate3d(0,2000px,0)}}@keyframes flyOutUp{20%{-webkit-transform:translate3d(0,10px,0);transform:translate3d(0,10px,0)}40%,45%{opacity:1;-webkit-transform:translate3d(0,-20px,0);transform:translate3d(0,-20px,0)}100%{opacity:0;-webkit-transform:translate3d(0,2000px,0);transform:translate3d(0,2000px,0)}}@-webkit-keyframes flyOutDown{20%{-webkit-transform:translate3d(0,-10px,0);transform:translate3d(0,-10px,0)}40%,45%{opacity:1;-webkit-transform:translate3d(0,20px,0);transform:translate3d(0,20px,0)}100%{opacity:0;-webkit-transform:translate3d(0,-2000px,0);transform:translate3d(0,-2000px,0)}}@keyframes flyOutDown{20%{-webkit-transform:translate3d(0,-10px,0);transform:translate3d(0,-10px,0)}40%,45%{opacity:1;-webkit-transform:translate3d(0,20px,0);transform:translate3d(0,20px,0)}100%{opacity:0;-webkit-transform:translate3d(0,-2000px,0);transform:translate3d(0,-2000px,0)}}@-webkit-keyframes flyOutRight{20%{opacity:1;-webkit-transform:translate3d(20px,0,0);transform:translate3d(20px,0,0)}100%{opacity:0;-webkit-transform:translate3d(-2000px,0,0);transform:translate3d(-2000px,0,0)}}@keyframes flyOutRight{20%{opacity:1;-webkit-transform:translate3d(20px,0,0);transform:translate3d(20px,0,0)}100%{opacity:0;-webkit-transform:translate3d(-2000px,0,0);transform:translate3d(-2000px,0,0)}}@-webkit-keyframes flyOutLeft{20%{opacity:1;-webkit-transform:translate3d(-20px,0,0);transform:translate3d(-20px,0,0)}100%{opacity:0;-webkit-transform:translate3d(2000px,0,0);transform:translate3d(2000px,0,0)}}@keyframes flyOutLeft{20%{opacity:1;-webkit-transform:translate3d(-20px,0,0);transform:translate3d(-20px,0,0)}100%{opacity:0;-webkit-transform:translate3d(2000px,0,0);transform:translate3d(2000px,0,0)}}.transition.slide.in,.transition[class*="slide down"].in{-webkit-animation-name:slideInY;animation-name:slideInY;-webkit-transform-origin:top center;transform-origin:top center}.transition[class*="slide up"].in{-webkit-animation-name:slideInY;animation-name:slideInY;-webkit-transform-origin:bottom center;transform-origin:bottom center}.transition[class*="slide left"].in{-webkit-animation-name:slideInX;animation-name:slideInX;-webkit-transform-origin:center right;transform-origin:center right}.transition[class*="slide right"].in{-webkit-animation-name:slideInX;animation-name:slideInX;-webkit-transform-origin:center left;transform-origin:center left}.transition.slide.out,.transition[class*="slide down"].out{-webkit-animation-name:slideOutY;animation-name:slideOutY;-webkit-transform-origin:top center;transform-origin:top center}.transition[class*="slide up"].out{-webkit-animation-name:slideOutY;animation-name:slideOutY;-webkit-transform-origin:bottom center;transform-origin:bottom center}.transition[class*="slide left"].out{-webkit-animation-name:slideOutX;animation-name:slideOutX;-webkit-transform-origin:center right;transform-origin:center right}.transition[class*="slide right"].out{-webkit-animation-name:slideOutX;animation-name:slideOutX;-webkit-transform-origin:center left;transform-origin:center left}@-webkit-keyframes slideInY{0%{opacity:0;-webkit-transform:scaleY(0);transform:scaleY(0)}100%{opacity:1;-webkit-transform:scaleY(1);transform:scaleY(1)}}@keyframes slideInY{0%{opacity:0;-webkit-transform:scaleY(0);transform:scaleY(0)}100%{opacity:1;-webkit-transform:scaleY(1);transform:scaleY(1)}}@-webkit-keyframes slideInX{0%{opacity:0;-webkit-transform:scaleX(0);transform:scaleX(0)}100%{opacity:1;-webkit-transform:scaleX(1);transform:scaleX(1)}}@keyframes slideInX{0%{opacity:0;-webkit-transform:scaleX(0);transform:scaleX(0)}100%{opacity:1;-webkit-transform:scaleX(1);transform:scaleX(1)}}@-webkit-keyframes slideOutY{0%{opacity:1;-webkit-transform:scaleY(1);transform:scaleY(1)}100%{opacity:0;-webkit-transform:scaleY(0);transform:scaleY(0)}}@keyframes slideOutY{0%{opacity:1;-webkit-transform:scaleY(1);transform:scaleY(1)}100%{opacity:0;-webkit-transform:scaleY(0);transform:scaleY(0)}}@-webkit-keyframes slideOutX{0%{opacity:1;-webkit-transform:scaleX(1);transform:scaleX(1)}100%{opacity:0;-webkit-transform:scaleX(0);transform:scaleX(0)}}@keyframes slideOutX{0%{opacity:1;-webkit-transform:scaleX(1);transform:scaleX(1)}100%{opacity:0;-webkit-transform:scaleX(0);transform:scaleX(0)}}.transition.swing{-webkit-animation-duration:.8s;animation-duration:.8s}.transition[class*="swing down"].in{-webkit-animation-name:swingInX;animation-name:swingInX;-webkit-transform-origin:top center;transform-origin:top center}.transition[class*="swing up"].in{-webkit-animation-name:swingInX;animation-name:swingInX;-webkit-transform-origin:bottom center;transform-origin:bottom center}.transition[class*="swing left"].in{-webkit-animation-name:swingInY;animation-name:swingInY;-webkit-transform-origin:center right;transform-origin:center right}.transition[class*="swing right"].in{-webkit-animation-name:swingInY;animation-name:swingInY;-webkit-transform-origin:center left;transform-origin:center left}.transition.swing.out,.transition[class*="swing down"].out{-webkit-animation-name:swingOutX;animation-name:swingOutX;-webkit-transform-origin:top center;transform-origin:top center}.transition[class*="swing up"].out{-webkit-animation-name:swingOutX;animation-name:swingOutX;-webkit-transform-origin:bottom center;transform-origin:bottom center}.transition[class*="swing left"].out{-webkit-animation-name:swingOutY;animation-name:swingOutY;-webkit-transform-origin:center right;transform-origin:center right}.transition[class*="swing right"].out{-webkit-animation-name:swingOutY;animation-name:swingOutY;-webkit-transform-origin:center left;transform-origin:center left}@-webkit-keyframes swingInX{0%{-webkit-transform:perspective(1000px) rotateX(90deg);transform:perspective(1000px) rotateX(90deg);opacity:0}40%{-webkit-transform:perspective(1000px) rotateX(-30deg);transform:perspective(1000px) rotateX(-30deg);opacity:1}60%{-webkit-transform:perspective(1000px) rotateX(15deg);transform:perspective(1000px) rotateX(15deg)}80%{-webkit-transform:perspective(1000px) rotateX(-7.5deg);transform:perspective(1000px) rotateX(-7.5deg)}100%{-webkit-transform:perspective(1000px) rotateX(0);transform:perspective(1000px) rotateX(0)}}@keyframes swingInX{0%{-webkit-transform:perspective(1000px) rotateX(90deg);transform:perspective(1000px) rotateX(90deg);opacity:0}40%{-webkit-transform:perspective(1000px) rotateX(-30deg);transform:perspective(1000px) rotateX(-30deg);opacity:1}60%{-webkit-transform:perspective(1000px) rotateX(15deg);transform:perspective(1000px) rotateX(15deg)}80%{-webkit-transform:perspective(1000px) rotateX(-7.5deg);transform:perspective(1000px) rotateX(-7.5deg)}100%{-webkit-transform:perspective(1000px) rotateX(0);transform:perspective(1000px) rotateX(0)}}@-webkit-keyframes swingInY{0%{-webkit-transform:perspective(1000px) rotateY(-90deg);transform:perspective(1000px) rotateY(-90deg);opacity:0}40%{-webkit-transform:perspective(1000px) rotateY(30deg);transform:perspective(1000px) rotateY(30deg);opacity:1}60%{-webkit-transform:perspective(1000px) rotateY(-17.5deg);transform:perspective(1000px) rotateY(-17.5deg)}80%{-webkit-transform:perspective(1000px) rotateY(7.5deg);transform:perspective(1000px) rotateY(7.5deg)}100%{-webkit-transform:perspective(1000px) rotateY(0);transform:perspective(1000px) rotateY(0)}}@keyframes swingInY{0%{-webkit-transform:perspective(1000px) rotateY(-90deg);transform:perspective(1000px) rotateY(-90deg);opacity:0}40%{-webkit-transform:perspective(1000px) rotateY(30deg);transform:perspective(1000px) rotateY(30deg);opacity:1}60%{-webkit-transform:perspective(1000px) rotateY(-17.5deg);transform:perspective(1000px) rotateY(-17.5deg)}80%{-webkit-transform:perspective(1000px) rotateY(7.5deg);transform:perspective(1000px) rotateY(7.5deg)}100%{-webkit-transform:perspective(1000px) rotateY(0);transform:perspective(1000px) rotateY(0)}}@-webkit-keyframes swingOutX{0%{-webkit-transform:perspective(1000px) rotateX(0);transform:perspective(1000px) rotateX(0)}40%{-webkit-transform:perspective(1000px) rotateX(-7.5deg);transform:perspective(1000px) rotateX(-7.5deg)}60%{-webkit-transform:perspective(1000px) rotateX(17.5deg);transform:perspective(1000px) rotateX(17.5deg)}80%{-webkit-transform:perspective(1000px) rotateX(-30deg);transform:perspective(1000px) rotateX(-30deg);opacity:1}100%{-webkit-transform:perspective(1000px) rotateX(90deg);transform:perspective(1000px) rotateX(90deg);opacity:0}}@keyframes swingOutX{0%{-webkit-transform:perspective(1000px) rotateX(0);transform:perspective(1000px) rotateX(0)}40%{-webkit-transform:perspective(1000px) rotateX(-7.5deg);transform:perspective(1000px) rotateX(-7.5deg)}60%{-webkit-transform:perspective(1000px) rotateX(17.5deg);transform:perspective(1000px) rotateX(17.5deg)}80%{-webkit-transform:perspective(1000px) rotateX(-30deg);transform:perspective(1000px) rotateX(-30deg);opacity:1}100%{-webkit-transform:perspective(1000px) rotateX(90deg);transform:perspective(1000px) rotateX(90deg);opacity:0}}@-webkit-keyframes swingOutY{0%{-webkit-transform:perspective(1000px) rotateY(0);transform:perspective(1000px) rotateY(0)}40%{-webkit-transform:perspective(1000px) rotateY(7.5deg);transform:perspective(1000px) rotateY(7.5deg)}60%{-webkit-transform:perspective(1000px) rotateY(-10deg);transform:perspective(1000px) rotateY(-10deg)}80%{-webkit-transform:perspective(1000px) rotateY(30deg);transform:perspective(1000px) rotateY(30deg);opacity:1}100%{-webkit-transform:perspective(1000px) rotateY(-90deg);transform:perspective(1000px) rotateY(-90deg);opacity:0}}@keyframes swingOutY{0%{-webkit-transform:perspective(1000px) rotateY(0);transform:perspective(1000px) rotateY(0)}40%{-webkit-transform:perspective(1000px) rotateY(7.5deg);transform:perspective(1000px) rotateY(7.5deg)}60%{-webkit-transform:perspective(1000px) rotateY(-10deg);transform:perspective(1000px) rotateY(-10deg)}80%{-webkit-transform:perspective(1000px) rotateY(30deg);transform:perspective(1000px) rotateY(30deg);opacity:1}100%{-webkit-transform:perspective(1000px) rotateY(-90deg);transform:perspective(1000px) rotateY(-90deg);opacity:0}}.transition.zoom.in{-webkit-animation-name:zoomIn;animation-name:zoomIn}.transition.zoom.out{-webkit-animation-name:zoomOut;animation-name:zoomOut}@-webkit-keyframes zoomIn{0%{opacity:1;-webkit-transform:scale(0);transform:scale(0)}100%{opacity:1;-webkit-transform:scale(1);transform:scale(1)}}@keyframes zoomIn{0%{opacity:1;-webkit-transform:scale(0);transform:scale(0)}100%{opacity:1;-webkit-transform:scale(1);transform:scale(1)}}@-webkit-keyframes zoomOut{0%{opacity:1;-webkit-transform:scale(1);transform:scale(1)}100%{opacity:1;-webkit-transform:scale(0);transform:scale(0)}}@keyframes zoomOut{0%{opacity:1;-webkit-transform:scale(1);transform:scale(1)}100%{opacity:1;-webkit-transform:scale(0);transform:scale(0)}}.flash.transition{-webkit-animation-duration:750ms;animation-duration:750ms;-webkit-animation-name:flash;animation-name:flash}.shake.transition{-webkit-animation-duration:750ms;animation-duration:750ms;-webkit-animation-name:shake;animation-name:shake}.bounce.transition{-webkit-animation-duration:750ms;animation-duration:750ms;-webkit-animation-name:bounce;animation-name:bounce}.tada.transition{-webkit-animation-duration:750ms;animation-duration:750ms;-webkit-animation-name:tada;animation-name:tada}.pulse.transition{-webkit-animation-duration:.5s;animation-duration:.5s;-webkit-animation-name:pulse;animation-name:pulse}.jiggle.transition{-webkit-animation-duration:750ms;animation-duration:750ms;-webkit-animation-name:jiggle;animation-name:jiggle}.transition.glow{-webkit-animation-duration:2s;animation-duration:2s;-webkit-animation-timing-function:cubic-bezier(.19,1,.22,1);animation-timing-function:cubic-bezier(.19,1,.22,1)}.transition.glow{-webkit-animation-name:glow;animation-name:glow}@-webkit-keyframes flash{0%,100%,50%{opacity:1}25%,75%{opacity:0}}@keyframes flash{0%,100%,50%{opacity:1}25%,75%{opacity:0}}@-webkit-keyframes shake{0%,100%{-webkit-transform:translateX(0);transform:translateX(0)}10%,30%,50%,70%,90%{-webkit-transform:translateX(-10px);transform:translateX(-10px)}20%,40%,60%,80%{-webkit-transform:translateX(10px);transform:translateX(10px)}}@keyframes shake{0%,100%{-webkit-transform:translateX(0);transform:translateX(0)}10%,30%,50%,70%,90%{-webkit-transform:translateX(-10px);transform:translateX(-10px)}20%,40%,60%,80%{-webkit-transform:translateX(10px);transform:translateX(10px)}}@-webkit-keyframes bounce{0%,100%,20%,50%,80%{-webkit-transform:translateY(0);transform:translateY(0)}40%{-webkit-transform:translateY(-30px);transform:translateY(-30px)}60%{-webkit-transform:translateY(-15px);transform:translateY(-15px)}}@keyframes bounce{0%,100%,20%,50%,80%{-webkit-transform:translateY(0);transform:translateY(0)}40%{-webkit-transform:translateY(-30px);transform:translateY(-30px)}60%{-webkit-transform:translateY(-15px);transform:translateY(-15px)}}@-webkit-keyframes tada{0%{-webkit-transform:scale(1);transform:scale(1)}10%,20%{-webkit-transform:scale(.9) rotate(-3deg);transform:scale(.9) rotate(-3deg)}30%,50%,70%,90%{-webkit-transform:scale(1.1) rotate(3deg);transform:scale(1.1) rotate(3deg)}40%,60%,80%{-webkit-transform:scale(1.1) rotate(-3deg);transform:scale(1.1) rotate(-3deg)}100%{-webkit-transform:scale(1) rotate(0);transform:scale(1) rotate(0)}}@keyframes tada{0%{-webkit-transform:scale(1);transform:scale(1)}10%,20%{-webkit-transform:scale(.9) rotate(-3deg);transform:scale(.9) rotate(-3deg)}30%,50%,70%,90%{-webkit-transform:scale(1.1) rotate(3deg);transform:scale(1.1) rotate(3deg)}40%,60%,80%{-webkit-transform:scale(1.1) rotate(-3deg);transform:scale(1.1) rotate(-3deg)}100%{-webkit-transform:scale(1) rotate(0);transform:scale(1) rotate(0)}}@-webkit-keyframes pulse{0%{-webkit-transform:scale(1);transform:scale(1);opacity:1}50%{-webkit-transform:scale(.9);transform:scale(.9);opacity:.7}100%{-webkit-transform:scale(1);transform:scale(1);opacity:1}}@keyframes pulse{0%{-webkit-transform:scale(1);transform:scale(1);opacity:1}50%{-webkit-transform:scale(.9);transform:scale(.9);opacity:.7}100%{-webkit-transform:scale(1);transform:scale(1);opacity:1}}@-webkit-keyframes jiggle{0%{-webkit-transform:scale3d(1,1,1);transform:scale3d(1,1,1)}30%{-webkit-transform:scale3d(1.25,.75,1);transform:scale3d(1.25,.75,1)}40%{-webkit-transform:scale3d(.75,1.25,1);transform:scale3d(.75,1.25,1)}50%{-webkit-transform:scale3d(1.15,.85,1);transform:scale3d(1.15,.85,1)}65%{-webkit-transform:scale3d(.95,1.05,1);transform:scale3d(.95,1.05,1)}75%{-webkit-transform:scale3d(1.05,.95,1);transform:scale3d(1.05,.95,1)}100%{-webkit-transform:scale3d(1,1,1);transform:scale3d(1,1,1)}}@keyframes jiggle{0%{-webkit-transform:scale3d(1,1,1);transform:scale3d(1,1,1)}30%{-webkit-transform:scale3d(1.25,.75,1);transform:scale3d(1.25,.75,1)}40%{-webkit-transform:scale3d(.75,1.25,1);transform:scale3d(.75,1.25,1)}50%{-webkit-transform:scale3d(1.15,.85,1);transform:scale3d(1.15,.85,1)}65%{-webkit-transform:scale3d(.95,1.05,1);transform:scale3d(.95,1.05,1)}75%{-webkit-transform:scale3d(1.05,.95,1);transform:scale3d(1.05,.95,1)}100%{-webkit-transform:scale3d(1,1,1);transform:scale3d(1,1,1)}}@-webkit-keyframes glow{0%{background-color:#fcfcfd}30%{background-color:#fff6cd}100%{background-color:#fcfcfd}}@keyframes glow{0%{background-color:#fcfcfd}30%{background-color:#fff6cd}100%{background-color:#fcfcfd}} \ No newline at end of file diff --git a/kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/semantic.min.js b/kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/semantic.min.js deleted file mode 100644 index b574823be..000000000 --- a/kyuubi-server/src/main/resources/org/apache/kyuubi/ui/static/semantic.min.js +++ /dev/null @@ -1,11 +0,0 @@ - /* - * # Semantic UI - 2.4.1 - * https://github.com/Semantic-Org/Semantic-UI - * http://www.semantic-ui.com/ - * - * Copyright 2014 Contributors - * Released under the MIT license - * http://opensource.org/licenses/MIT - * - */ -!function(p,h,v,b){p.site=p.fn.site=function(e){var s,l,i=(new Date).getTime(),o=[],t=e,n="string"==typeof t,c=[].slice.call(arguments,1),u=p.isPlainObject(e)?p.extend(!0,{},p.site.settings,e):p.extend({},p.site.settings),a=u.namespace,d=u.error,r="module-"+a,f=p(v),m=this,g=f.data(r);return s={initialize:function(){s.instantiate()},instantiate:function(){s.verbose("Storing instance of site",s),g=s,f.data(r,s)},normalize:function(){s.fix.console(),s.fix.requestAnimationFrame()},fix:{console:function(){s.debug("Normalizing window.console"),console!==b&&console.log!==b||(s.verbose("Console not available, normalizing events"),s.disable.console()),void 0!==console.group&&void 0!==console.groupEnd&&void 0!==console.groupCollapsed||(s.verbose("Console group not available, normalizing events"),h.console.group=function(){},h.console.groupEnd=function(){},h.console.groupCollapsed=function(){}),void 0===console.markTimeline&&(s.verbose("Mark timeline not available, normalizing events"),h.console.markTimeline=function(){})},consoleClear:function(){s.debug("Disabling programmatic console clearing"),h.console.clear=function(){}},requestAnimationFrame:function(){s.debug("Normalizing requestAnimationFrame"),h.requestAnimationFrame===b&&(s.debug("RequestAnimationFrame not available, normalizing event"),h.requestAnimationFrame=h.requestAnimationFrame||h.mozRequestAnimationFrame||h.webkitRequestAnimationFrame||h.msRequestAnimationFrame||function(e){setTimeout(e,0)})}},moduleExists:function(e){return p.fn[e]!==b&&p.fn[e].settings!==b},enabled:{modules:function(e){var n=[];return e=e||u.modules,p.each(e,function(e,t){s.moduleExists(t)&&n.push(t)}),n}},disabled:{modules:function(e){var n=[];return e=e||u.modules,p.each(e,function(e,t){s.moduleExists(t)||n.push(t)}),n}},change:{setting:function(o,a,e,r){e="string"==typeof e?"all"===e?u.modules:[e]:e||u.modules,r=r===b||r,p.each(e,function(e,t){var n,i=!s.moduleExists(t)||(p.fn[t].settings.namespace||!1);s.moduleExists(t)&&(s.verbose("Changing default setting",o,a,t),p.fn[t].settings[o]=a,r&&i&&0<(n=p(":data(module-"+i+")")).length&&(s.verbose("Modifying existing settings",n),n[t]("setting",o,a)))})},settings:function(i,e,o){e="string"==typeof e?[e]:e||u.modules,o=o===b||o,p.each(e,function(e,t){var n;s.moduleExists(t)&&(s.verbose("Changing default setting",i,t),p.extend(!0,p.fn[t].settings,i),o&&a&&0<(n=p(":data(module-"+a+")")).length&&(s.verbose("Modifying existing settings",n),n[t]("setting",i)))})}},enable:{console:function(){s.console(!0)},debug:function(e,t){e=e||u.modules,s.debug("Enabling debug for modules",e),s.change.setting("debug",!0,e,t)},verbose:function(e,t){e=e||u.modules,s.debug("Enabling verbose debug for modules",e),s.change.setting("verbose",!0,e,t)}},disable:{console:function(){s.console(!1)},debug:function(e,t){e=e||u.modules,s.debug("Disabling debug for modules",e),s.change.setting("debug",!1,e,t)},verbose:function(e,t){e=e||u.modules,s.debug("Disabling verbose debug for modules",e),s.change.setting("verbose",!1,e,t)}},console:function(e){if(e){if(g.cache.console===b)return void s.error(d.console);s.debug("Restoring console function"),h.console=g.cache.console}else s.debug("Disabling console function"),g.cache.console=h.console,h.console={clear:function(){},error:function(){},group:function(){},groupCollapsed:function(){},groupEnd:function(){},info:function(){},log:function(){},markTimeline:function(){},warn:function(){}}},destroy:function(){s.verbose("Destroying previous site for",f),f.removeData(r)},cache:{},setting:function(e,t){if(p.isPlainObject(e))p.extend(!0,u,e);else{if(t===b)return u[e];u[e]=t}},internal:function(e,t){if(p.isPlainObject(e))p.extend(!0,s,e);else{if(t===b)return s[e];s[e]=t}},debug:function(){u.debug&&(u.performance?s.performance.log(arguments):(s.debug=Function.prototype.bind.call(console.info,console,u.name+":"),s.debug.apply(console,arguments)))},verbose:function(){u.verbose&&u.debug&&(u.performance?s.performance.log(arguments):(s.verbose=Function.prototype.bind.call(console.info,console,u.name+":"),s.verbose.apply(console,arguments)))},error:function(){s.error=Function.prototype.bind.call(console.error,console,u.name+":"),s.error.apply(console,arguments)},performance:{log:function(e){var t,n;u.performance&&(n=(t=(new Date).getTime())-(i||t),i=t,o.push({Element:m,Name:e[0],Arguments:[].slice.call(e,1)||"","Execution Time":n})),clearTimeout(s.performance.timer),s.performance.timer=setTimeout(s.performance.display,500)},display:function(){var e=u.name+":",n=0;i=!1,clearTimeout(s.performance.timer),p.each(o,function(e,t){n+=t["Execution Time"]}),e+=" "+n+"ms",(console.group!==b||console.table!==b)&&0")},fields:function(e){var n=F();return F.each(e,function(e,t){n=n.add(h.get.field(t))}),n},validation:function(n){var i,o;return!!c&&(F.each(c,function(e,t){o=t.identifier||e,h.get.field(o)[0]==n[0]&&(t.identifier=o,i=t)}),i||!1)},value:function(e){var t=[];return t.push(e),h.get.values.call(v,t)[e]},values:function(e){var t=F.isArray(e)?h.get.fields(e):n,c={};return t.each(function(e,t){var n=F(t),i=(n.prop("type"),n.prop("name")),o=n.val(),a=n.is(f.checkbox),r=n.is(f.radio),s=-1!==i.indexOf("[]"),l=!!a&&n.is(":checked");i&&(s?(i=i.replace("[]",""),c[i]||(c[i]=[]),a?l?c[i].push(o||!0):c[i].push(!1):c[i].push(o)):r?c[i]!==D&&0!=c[i]||(c[i]=!!l&&(o||!0)):c[i]=a?!!l&&(o||!0):o)}),c}},has:{field:function(e){return h.verbose("Checking for existence of a field with identifier",e),"string"!=typeof(e=h.escape.string(e))&&h.error(s.identifier,e),0"}),F(n+="")},prompt:function(e){return F("
    ").addClass("ui basic red pointing prompt label").html(e[0])}},rules:{empty:function(e){return!(e===D||""===e||F.isArray(e)&&0===e.length)},checked:function(){return 0=t},length:function(e,t){return e!==D&&e.length>=t},exactLength:function(e,t){return e!==D&&e.length==t},maxLength:function(e,t){return e!==D&&e.length<=t},match:function(e,t){var n;F(this);return 0=t)},exactCount:function(e,t){return 0==t?""===e:1==t?""!==e&&-1===e.search(","):e.split(",").length==t},maxCount:function(e,t){return 0!=t&&(1==t?-1===e.search(","):e.split(",").length<=t)}}}}(jQuery,window,document),function(S,k,e,T){"use strict";k=void 0!==k&&k.Math==Math?k:"undefined"!=typeof self&&self.Math==Math?self:Function("return this")(),S.fn.accordion=function(a){var v,r=S(this),b=(new Date).getTime(),y=[],x=a,C="string"==typeof x,w=[].slice.call(arguments,1);k.requestAnimationFrame||k.mozRequestAnimationFrame||k.webkitRequestAnimationFrame||k.msRequestAnimationFrame;return r.each(function(){var e,c,u=S.isPlainObject(a)?S.extend(!0,{},S.fn.accordion.settings,a):S.extend({},S.fn.accordion.settings),d=u.className,t=u.namespace,f=u.selector,s=u.error,n="."+t,i="module-"+t,o=r.selector||"",m=S(this),g=m.find(f.title),p=m.find(f.content),l=this,h=m.data(i);c={initialize:function(){c.debug("Initializing",m),c.bind.events(),u.observeChanges&&c.observeChanges(),c.instantiate()},instantiate:function(){h=c,m.data(i,c)},destroy:function(){c.debug("Destroying previous instance",m),m.off(n).removeData(i)},refresh:function(){g=m.find(f.title),p=m.find(f.content)},observeChanges:function(){"MutationObserver"in k&&((e=new MutationObserver(function(e){c.debug("DOM tree modified, updating selector cache"),c.refresh()})).observe(l,{childList:!0,subtree:!0}),c.debug("Setting up mutation observer",e))},bind:{events:function(){c.debug("Binding delegated events"),m.on(u.on+n,f.trigger,c.event.click)}},event:{click:function(){c.toggle.call(this)}},toggle:function(e){var t=e!==T?"number"==typeof e?g.eq(e):S(e).closest(f.title):S(this).closest(f.title),n=t.next(p),i=n.hasClass(d.animating),o=n.hasClass(d.active),a=o&&!i,r=!o&&i;c.debug("Toggling visibility of content",t),a||r?u.collapsible?c.close.call(t):c.debug("Cannot close accordion content collapsing is disabled"):c.open.call(t)},open:function(e){var t=e!==T?"number"==typeof e?g.eq(e):S(e).closest(f.title):S(this).closest(f.title),n=t.next(p),i=n.hasClass(d.animating);n.hasClass(d.active)||i?c.debug("Accordion already open, skipping",n):(c.debug("Opening accordion content",t),u.onOpening.call(n),u.onChanging.call(n),u.exclusive&&c.closeOthers.call(t),t.addClass(d.active),n.stop(!0,!0).addClass(d.animating),u.animateChildren&&(S.fn.transition!==T&&m.transition("is supported")?n.children().transition({animation:"fade in",queue:!1,useFailSafe:!0,debug:u.debug,verbose:u.verbose,duration:u.duration}):n.children().stop(!0,!0).animate({opacity:1},u.duration,c.resetOpacity)),n.slideDown(u.duration,u.easing,function(){n.removeClass(d.animating).addClass(d.active),c.reset.display.call(this),u.onOpen.call(this),u.onChange.call(this)}))},close:function(e){var t=e!==T?"number"==typeof e?g.eq(e):S(e).closest(f.title):S(this).closest(f.title),n=t.next(p),i=n.hasClass(d.animating),o=n.hasClass(d.active);!o&&!(!o&&i)||o&&i||(c.debug("Closing accordion content",n),u.onClosing.call(n),u.onChanging.call(n),t.removeClass(d.active),n.stop(!0,!0).addClass(d.animating),u.animateChildren&&(S.fn.transition!==T&&m.transition("is supported")?n.children().transition({animation:"fade out",queue:!1,useFailSafe:!0,debug:u.debug,verbose:u.verbose,duration:u.duration}):n.children().stop(!0,!0).animate({opacity:0},u.duration,c.resetOpacity)),n.slideUp(u.duration,u.easing,function(){n.removeClass(d.animating).removeClass(d.active),c.reset.display.call(this),u.onClose.call(this),u.onChange.call(this)}))},closeOthers:function(e){var t,n,i,o=e!==T?g.eq(e):S(this).closest(f.title),a=o.parents(f.content).prev(f.title),r=o.closest(f.accordion),s=f.title+"."+d.active+":visible",l=f.content+"."+d.active+":visible";i=u.closeNested?(t=r.find(s).not(a)).next(p):(t=r.find(s).not(a),n=r.find(l).find(s).not(a),(t=t.not(n)).next(p)),0 adjusting invoked element"),c=c.closest(o.checkbox),s.refresh())}},setup:function(){s.set.initialLoad(),s.is.indeterminate()?(s.debug("Initial value is indeterminate"),s.indeterminate()):s.is.checked()?(s.debug("Initial value is checked"),s.check()):(s.debug("Initial value is unchecked"),s.uncheck()),s.remove.initialLoad()},refresh:function(){u=c.children(o.label),d=c.children(o.input),f=d[0]},hide:{input:function(){s.verbose("Modifying z-index to be unselectable"),d.addClass(t.hidden)}},show:{input:function(){s.verbose("Modifying z-index to be selectable"),d.removeClass(t.hidden)}},observeChanges:function(){"MutationObserver"in A&&((e=new MutationObserver(function(e){s.debug("DOM tree modified, updating selector cache"),s.refresh()})).observe(h,{childList:!0,subtree:!0}),s.debug("Setting up mutation observer",e))},attachEvents:function(e,t){var n=T(e);t=T.isFunction(s[t])?s[t]:s.toggle,0").insertAfter(d),s.debug("Creating label",u))}},has:{label:function(){return 0 .ui.dimmer",content:".ui.dimmer > .content, .ui.dimmer > .content > .center"},template:{dimmer:function(){return S("
    ").attr("class","ui dimmer")}}}}(jQuery,window,document),function(Y,Z,K,J){"use strict";Z=void 0!==Z&&Z.Math==Math?Z:"undefined"!=typeof self&&self.Math==Math?self:Function("return this")(),Y.fn.dropdown=function(M){var L,V=Y(this),N=Y(K),H=V.selector||"",U="ontouchstart"in K.documentElement,W=(new Date).getTime(),B=[],Q=M,X="string"==typeof Q,$=[].slice.call(arguments,1);return V.each(function(n){var e,t,i,o,a,r,s,g,p=Y.isPlainObject(M)?Y.extend(!0,{},Y.fn.dropdown.settings,M):Y.extend({},Y.fn.dropdown.settings),h=p.className,c=p.message,l=p.fields,v=p.keys,b=p.metadata,u=p.namespace,d=p.regExp,y=p.selector,f=p.error,m=p.templates,x="."+u,C="module-"+u,w=Y(this),S=Y(p.context),k=w.find(y.text),T=w.find(y.search),A=w.find(y.sizer),R=w.find(y.input),P=w.find(y.icon),E=0").html(o).attr("data-"+b.value,t).attr("data-"+b.text,t).addClass(h.addition).addClass(h.item),p.hideAdditions&&i.addClass(h.hidden),n=n===J?i:n.add(i),g.verbose("Creating user choices for value",t,i))}),n)},userLabels:function(e){var t=g.get.userValues();t&&(g.debug("Adding user labels",t),Y.each(t,function(e,t){g.verbose("Adding custom user value"),g.add.label(t,t)}))},menu:function(){F=Y("
    ").addClass(h.menu).appendTo(w)},sizer:function(){A=Y("").addClass(h.sizer).insertAfter(T)}},search:function(e){e=e!==J?e:g.get.query(),g.verbose("Searching for query",e),g.has.minCharacters(e)?g.filter(e):g.hide()},select:{firstUnfiltered:function(){g.verbose("Selecting first non-filtered element"),g.remove.selectedItem(),O.not(y.unselectable).not(y.addition+y.hidden).eq(0).addClass(h.selected)},nextAvailable:function(e){var t=(e=e.eq(0)).nextAll(y.item).not(y.unselectable).eq(0),n=e.prevAll(y.item).not(y.unselectable).eq(0);0").addClass(h.search).prop("autocomplete","off").insertBefore(k)),g.is.multiple()&&g.is.searchSelection()&&!g.has.sizer()&&g.create.sizer(),p.allowTab&&g.set.tabbable()},select:function(){var e=g.get.selectValues();g.debug("Dropdown initialized on a select",e),w.is("select")&&(R=w),0").attr("class",R.attr("class")).addClass(h.selection).addClass(h.dropdown).html(m.dropdown(e)).insertBefore(R),R.hasClass(h.multiple)&&!1===R.prop("multiple")&&(g.error(f.missingMultiple),R.prop("multiple",!0)),R.is("[multiple]")&&g.set.multiple(),R.prop("disabled")&&(g.debug("Disabling dropdown"),w.addClass(h.disabled)),R.removeAttr("class").detach().prependTo(w)),g.refresh()},menu:function(e){F.html(m.menu(e,l)),O=F.find(y.item)},reference:function(){g.debug("Dropdown behavior was called on select, replacing with closest dropdown"),w=w.parent(y.dropdown),I=w.data(C),z=w.get(0),g.refresh(),g.setup.returnedObject()},returnedObject:function(){var e=V.slice(0,n),t=V.slice(n+1);V=e.add(w).add(t)}},refresh:function(){g.refreshSelectors(),g.refreshData()},refreshItems:function(){O=F.find(y.item)},refreshSelectors:function(){g.verbose("Refreshing selector cache"),k=w.find(y.text),T=w.find(y.search),R=w.find(y.input),P=w.find(y.icon),E=0 modified, recreating menu");var n=!1;Y.each(e,function(e,t){if(Y(t.target).is("select")||Y(t.addedNodes).is("select"))return n=!0}),n&&(g.disconnect.selectObserver(),g.refresh(),g.setup.select(),g.set.selected(),g.observe.select())}},menu:{mutation:function(e){var t=e[0],n=t.addedNodes?Y(t.addedNodes[0]):Y(!1),i=t.removedNodes?Y(t.removedNodes[0]):Y(!1),o=n.add(i),a=o.is(y.addition)||0t.name?1:-1}),g.debug("Retrieved and sorted values from select",o)):g.debug("Retrieved values from select",o),o},activeItem:function(){return O.filter("."+h.active)},selectedItem:function(){var e=O.not(y.unselectable).filter("."+h.selected);return 0=p.maxSelections?(g.debug("Maximum selection count reached"),p.useLabels&&(O.addClass(h.filtered),g.add.message(c.maxSelections)),!0):(g.verbose("No longer at maximum selection count"),g.remove.message(),g.remove.filteredItem(),g.is.searchSelection()&&g.filterItems(),!1))}},restore:{defaults:function(){g.clear(),g.restore.defaultText(),g.restore.defaultValue()},defaultText:function(){var e=g.get.defaultText();e===g.get.placeholderText?(g.debug("Restoring default placeholder text",e),g.set.placeholderText(e)):(g.debug("Restoring default text",e),g.set.text(e))},placeholderText:function(){g.set.placeholderText()},defaultValue:function(){var e=g.get.defaultValue();e!==J&&(g.debug("Restoring default value",e),""!==e?(g.set.value(e),g.set.selected()):(g.remove.activeItem(),g.remove.selectedItem()))},labels:function(){p.allowAdditions&&(p.useLabels||(g.error(f.labels),p.useLabels=!0),g.debug("Restoring selected values"),g.create.userLabels()),g.check.maxSelections()},selected:function(){g.restore.values(),g.is.multiple()?(g.debug("Restoring previously selected values and labels"),g.restore.labels()):g.debug("Restoring previously selected values")},values:function(){g.set.initialLoad(),p.apiSettings&&p.saveRemoteData&&g.get.remoteValues()?g.restore.remoteValues():g.set.selected(),g.remove.initialLoad()},remoteValues:function(){var e=g.get.remoteValues();g.debug("Recreating selected from session data",e),e&&(g.is.single()?Y.each(e,function(e,t){g.set.text(t)}):Y.each(e,function(e,t){g.add.label(e,t)}))}},read:{remoteData:function(e){var t;if(Z.Storage!==J)return(t=sessionStorage.getItem(e))!==J&&t;g.error(f.noStorage)}},save:{defaults:function(){g.save.defaultText(),g.save.placeholderText(),g.save.defaultValue()},defaultValue:function(){var e=g.get.value();g.verbose("Saving default value as",e),w.data(b.defaultValue,e)},defaultText:function(){var e=g.get.text();g.verbose("Saving default text as",e),w.data(b.defaultText,e)},placeholderText:function(){var e;!1!==p.placeholder&&k.hasClass(h.placeholder)&&(e=g.get.text(),g.verbose("Saving placeholder text as",e),w.data(b.placeholderText,e))},remoteData:function(e,t){Z.Storage!==J?(g.verbose("Saving remote data to session storage",t,e),sessionStorage.setItem(t,e)):g.error(f.noStorage)}},clear:function(){g.is.multiple()&&p.useLabels?g.remove.labels():(g.remove.activeItem(),g.remove.selectedItem()),g.set.placeholderText(),g.clearValue()},clearValue:function(){g.set.value("")},scrollPage:function(e,t){var n,i,o=t||g.get.selectedItem(),a=o.closest(y.menu),r=a.outerHeight(),s=a.scrollTop(),l=O.eq(0).outerHeight(),c=Math.floor(r/l),u=(a.prop("scrollHeight"),"up"==e?s-l*c:s+l*c),d=O.not(y.unselectable);i="up"==e?d.index(o)-c:d.index(o)+c,0<(n=("up"==e?0<=i:i").addClass(h.label).attr("data-"+b.value,a).html(m.label(a,t)),i=p.onLabelCreate.call(i,a,t),g.has.label(e)?g.debug("User selection already exists, skipping",a):(p.label.variation&&i.addClass(p.label.variation),!0===n?(g.debug("Animating in label",i),i.addClass(h.hidden).insertBefore(o).transition(p.label.transition,p.label.duration)):(g.debug("Adding selection label",i),i.insertBefore(o)))},message:function(e){var t=F.children(y.message),n=p.templates.message(g.add.variables(e));0").html(n).addClass(h.message).appendTo(F)},optionValue:function(e){var t=g.escape.value(e);0").prop("value",t).addClass(h.addition).html(e).appendTo(R),g.verbose("Adding user addition as an