Migrate to using common-files and build tools image for linting (#799)

* Migrate to using build image

* Remove license files

* Fix up CONTRIBUTION

* Update to only ignore out directory at the root

.gitignore

@@ -1,2 +1,3 @@
 .idea/
-
+# Contains the built artifacts
+/out/

BUGS-AND-FEATURE-REQUESTS.md

@@ -0,0 +1,10 @@
+# Bugs and Feature Requests
+
+You can report bugs and feature requests to the Istio team in one of three places:
+
+- [Product Bugs and Feature Requests](https://github.com/istio/istio/issues)
+- [Documentation Bugs and Feature Requests](https://github.com/istio/istio.io/issues)
+- [Community and Governance Issues](https://github.com/istio/community/issues)
+
+For security vulnerabilities, please don't report a bug (which is public) and instead follow
+[these procedures](https://istio.io/about/security-vulnerabilities/).

LICENSE

@@ -1,4 +1,5 @@
-                                 Apache License
+
+                                Apache License
                            Version 2.0, January 2004
                         http://www.apache.org/licenses/
 
@@ -186,7 +187,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright {yyyy} {name of copyright owner}
+   Copyright 2016-2020 Istio Authors
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

Makefile

@@ -1,8 +1,69 @@
-img := gcr.io/istio-testing/build-tools:2019-10-24T14-05-17
-docker := docker run -e -t -i --sig-proxy=true --rm -v $(shell pwd):/foo -w /foo $(img)
+# WARNING: DO NOT EDIT, THIS FILE IS PROBABLY A COPY
+#
+# The original version of this file is located in the https://github.com/istio/common-files repo.
+# If you're looking at this file in a different repo and want to make a change, please go to the
+# common-files repo, make the change there and check it in. Then come back to this repo and run
+# "make update-common".
 
-lint:
-	@$(docker) prow/community-lint.sh
+# Copyright Istio Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
-test:
-	go test ./...
+SHELL := /bin/bash
+
+# allow optional per-repo overrides
+-include Makefile.overrides.mk
+
+# Set the environment variable BUILD_WITH_CONTAINER to use a container
+# to build the repo. The only dependencies in this mode are to have make and
+# docker. If you'd rather build with a local tool chain instead, you'll need to
+# figure out all the tools you need in your environment to make that work.
+export BUILD_WITH_CONTAINER ?= 0
+
+ifeq ($(BUILD_WITH_CONTAINER),1)
+
+# An export free of arguments in a Makefile places all variables in the Makefile into the
+# environment. This is needed to allow overrides from Makefile.overrides.mk.
+export
+
+RUN = ./common/scripts/run.sh
+
+MAKE_DOCKER = $(RUN) make --no-print-directory -e -f Makefile.core.mk
+
+%:
+	@$(MAKE_DOCKER) $@
+
+default:
+	@$(MAKE_DOCKER)
+
+shell:
+	@$(RUN) /bin/bash
+
+.PHONY: default shell
+
+else
+
+# If we are not in build container, we need a workaround to get environment properly set
+# Write to file, then include
+$(shell mkdir -p out)
+$(shell $(shell pwd)/common/scripts/setup_env.sh envfile > out/.env)
+include out/.env
+# An export free of arguments in a Makefile places all variables in the Makefile into the
+# environment. This behavior may be surprising to many that use shell often, which simply
+# displays the existing environment
+export
+
+export GOBIN ?= $(GOPATH)/bin
+include Makefile.core.mk
+
+endif

Makefile.core.mk

@@ -0,0 +1,26 @@
+# Copyright Istio Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# lint can  run all lint-all targets minus lint-markdown which is run by the community-lint.sh
+lint: lint-dockerfiles lint-scripts lint-yaml lint-helm lint-copyright-banner lint-go lint-python lint-sass lint-typescript lint-protos lint-licenses
+	@prow/community-lint.sh
+
+test:
+	go test ./...
+
+gen: fmt
+
+fmt: format-go tidy-go
+
+include common/Makefile.common.mk

Makefile.overrides.mk

@@ -0,0 +1,14 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# this repo is on the container plan by default
+BUILD_WITH_CONTAINER ?= 1

SUPPORT.md

@@ -0,0 +1,7 @@
+# Support
+
+Here are some resources to help you understand and use Istio:
+
+- For in-depth information about how to use Istio, visit [istio.io](https://istio.io)
+- To ask questions and get assistance from our community, visit [discuss.istio.io](https://discuss.istio.io)
+- To learn how to participate in our overall community, visit [our community page](https://istio.io/about/community)

common/.commonfiles.sha

@@ -0,0 +1 @@
+0b03d2e90b025edbf5b6676542c7fa76d86f7b2f

common/Makefile.common.mk

@@ -0,0 +1,117 @@
+# WARNING: DO NOT EDIT, THIS FILE IS PROBABLY A COPY
+#
+# The original version of this file is located in the https://github.com/istio/common-files repo.
+# If you're looking at this file in a different repo and want to make a change, please go to the
+# common-files repo, make the change there and check it in. Then come back to this repo and run
+# "make update-common".
+
+# Copyright Istio Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FINDFILES=find . \( -path ./common-protos -o -path ./.git -o -path ./out -o -path ./.github -o -path ./licenses -o -path ./vendor \) -prune -o -type f
+XARGS = xargs -0 -r
+
+lint-dockerfiles:
+	@${FINDFILES} -name 'Dockerfile*' -print0 | ${XARGS} hadolint -c ./common/config/.hadolint.yml
+
+lint-scripts:
+	@${FINDFILES} -name '*.sh' -print0 | ${XARGS} shellcheck
+
+lint-yaml:
+	@${FINDFILES} \( -name '*.yml' -o -name '*.yaml' \) -not -exec grep -q -e "{{" {} \; -print0 | ${XARGS} yamllint -c ./common/config/.yamllint.yml
+
+lint-helm:
+	@${FINDFILES} -name 'Chart.yaml' -print0 | ${XARGS} -L 1 dirname | xargs -r helm lint --strict
+
+lint-copyright-banner:
+	@${FINDFILES} \( -name '*.go' -o -name '*.cc' -o -name '*.h' -o -name '*.proto' -o -name '*.py' -o -name '*.sh' \) \( ! \( -name '*.gen.go' -o -name '*.pb.go' -o -name '*_pb2.py' \) \) -print0 |\
+		${XARGS} common/scripts/lint_copyright_banner.sh
+
+fix-copyright-banner:
+	@${FINDFILES} \( -name '*.go' -o -name '*.cc' -o -name '*.h' -o -name '*.proto' -o -name '*.py' -o -name '*.sh' \) \( ! \( -name '*.gen.go' -o -name '*.pb.go' -o -name '*_pb2.py' \) \) -print0 |\
+		${XARGS} common/scripts/fix_copyright_banner.sh
+
+lint-go:
+	@${FINDFILES} -name '*.go' \( ! \( -name '*.gen.go' -o -name '*.pb.go' \) \) -print0 | ${XARGS} common/scripts/lint_go.sh
+
+lint-python:
+	@${FINDFILES} -name '*.py' \( ! \( -name '*_pb2.py' \) \) -print0 | ${XARGS} autopep8 --max-line-length 160 --exit-code -d
+
+lint-markdown:
+	@${FINDFILES} -name '*.md' -print0 | ${XARGS} mdl --ignore-front-matter --style common/config/mdl.rb
+
+lint-links:
+	@${FINDFILES} -name '*.md' -print0 | ${XARGS} awesome_bot --skip-save-results --allow_ssl --allow-timeout --allow-dupe --allow-redirect --white-list ${MARKDOWN_LINT_ALLOWLIST}
+
+lint-sass:
+	@${FINDFILES} -name '*.scss' -print0 | ${XARGS} sass-lint -c common/config/sass-lint.yml --verbose
+
+lint-typescript:
+	@${FINDFILES} -name '*.ts' -print0 | ${XARGS} tslint -c common/config/tslint.json
+
+lint-licenses:
+	@if test -d licenses; then license-lint --config common/config/license-lint.yml; fi
+
+lint-all: lint-dockerfiles lint-scripts lint-yaml lint-helm lint-copyright-banner lint-go lint-python lint-markdown lint-sass lint-typescript lint-protos lint-licenses
+
+tidy-go:
+	@find -name go.mod -execdir go mod tidy \;
+
+mod-download-go:
+	@-GOFLAGS="-mod=readonly" find -name go.mod -execdir go mod download \;
+# go mod tidy is needed with Golang 1.16+ as go mod download affects go.sum
+# https://github.com/golang/go/issues/43994
+	@find -name go.mod -execdir go mod tidy \;
+
+format-go: tidy-go
+	@${FINDFILES} -name '*.go' \( ! \( -name '*.gen.go' -o -name '*.pb.go' \) \) -print0 | ${XARGS} common/scripts/format_go.sh
+
+format-python:
+	@${FINDFILES} -name '*.py' -print0 | ${XARGS} autopep8 --max-line-length 160 --aggressive --aggressive -i
+
+dump-licenses: mod-download-go
+	@license-lint --config common/config/license-lint.yml --report
+
+dump-licenses-csv: mod-download-go
+	@license-lint --config common/config/license-lint.yml --csv
+
+mirror-licenses: mod-download-go
+	@rm -fr licenses
+	@license-lint --mirror
+
+TMP := $(shell mktemp -d -u)
+UPDATE_BRANCH ?= "master"
+
+update-common:
+	@mkdir -p $(TMP)
+	@git clone -q --depth 1 --single-branch --branch $(UPDATE_BRANCH) https://github.com/istio/common-files $(TMP)/common-files
+	@cd $(TMP)/common-files ; git rev-parse HEAD >files/common/.commonfiles.sha
+	@rm -fr common
+	@CONTRIB_OVERRIDE=$(shell grep -l "refer to Istio" CONTRIBUTING.md)
+	if [ "$(CONTRIB_OVERRIDE)" != "CONTRIBUTING.md" ]; then\
+		rm $(TMP)/common-files/files/CONTRIBUTING.md;\
+	fi
+	@rm -fr $(TMP)/common-files
+
+check-clean-repo:
+	@common/scripts/check_clean_repo.sh
+
+tidy-docker:
+	@docker image prune --all --force --filter="label=io.istio.repo=https://github.com/istio/tools" --filter="label!=io.istio.version=$(IMAGE_VERSION)"
+
+# help works by looking over all Makefile includes matching `target: ## comment` regex and outputting them
+help: ## Show this help
+	@egrep -h '^[a-zA-Z_\.-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort  | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}'
+
+.PHONY: lint-dockerfiles lint-scripts lint-yaml lint-copyright-banner lint-go lint-python lint-helm lint-markdown lint-sass lint-typescript lint-protos lint-all format-go format-python format-protos update-common lint-licenses dump-licenses dump-licenses-csv check-clean-repo tidy-docker help tidy-go mod-download-go

common/config/.golangci-format.yml

@@ -0,0 +1,59 @@
+# WARNING: DO NOT EDIT, THIS FILE IS PROBABLY A COPY
+#
+# The original version of this file is located in the https://github.com/istio/common-files repo.
+# If you're looking at this file in a different repo and want to make a change, please go to the
+# common-files repo, make the change there and check it in. Then come back to this repo and run
+# "make update-common".
+
+service:
+  # When updating this, also update the version stored in docker/build-tools/Dockerfile in the istio/tools repo.
+  golangci-lint-version: 1.49.x # use the fixed version to not introduce new linters unexpectedly
+run:
+  # timeout for analysis, e.g. 30s, 5m, default is 1m
+  deadline: 20m
+  build-tags:
+  - integ
+  - integfuzz
+  # which dirs to skip: they won't be analyzed;
+  # can use regexp here: generated.*, regexp is applied on full path;
+  # default value is empty list, but next dirs are always skipped independently
+  # from this option's value:
+  #   	vendor$, third_party$, testdata$, examples$, Godeps$, builtin$
+  skip-dirs:
+  - genfiles$
+  - vendor$
+
+  # which files to skip: they will be analyzed, but issues from them
+  # won't be reported. Default value is empty list, but there is
+  # no need to include all autogenerated files, we confidently recognize
+  # autogenerated files. If it's not please let us know.
+  skip-files:
+  - ".*\\.pb\\.go"
+  - ".*\\.gen\\.go"
+
+linters:
+  disable-all: true
+  enable:
+  - goimports
+  - gofumpt
+  - gci
+  fast: false
+
+linters-settings:
+  gci:
+    sections:
+      - standard # Captures all standard packages if they do not match another section.
+      - default # Contains all imports that could not be matched to another section type.
+      - prefix(istio.io/) # Groups all imports with the specified Prefix.
+  goimports:
+    # put imports beginning with prefix after 3rd-party packages;
+    # it's a comma-separated list of prefixes
+    local-prefixes: istio.io/
+
+issues:
+
+  # Maximum issues count per one linter. Set to 0 to disable. Default is 50.
+  max-per-linter: 0
+
+  # Maximum count of issues with the same text. Set to 0 to disable. Default is 3.
+  max-same-issues: 0