🎉 pop uds-proxy into github-existence

.gitignore

@@ -0,0 +1,6 @@
+/uds-proxy
+/test_server
+coverage*
+.idea/
+uds-proxy.iml
+uds-proxy.pid

Makefile

@@ -0,0 +1,119 @@
+
+BINARY := uds-proxy
+
+VERSION := $(shell git describe --tags | cut -dv -f2)
+DOCKER_IMAGE := schnoddelbotz/uds-proxy
+LDFLAGS := -X github.com/schnoddelbotz/uds-proxy/proxy.AppVersion=$(VERSION) -w
+
+TEST_SOCKET   := $(PWD)/uds-proxy-test.socket
+TEST_PIDFILE  := uds-proxy.pid
+TEST_PROMETHEUS_PORT := 18080
+TEST_DOCKERIZED_SOCKET_DIR := $(PWD)/udsproxy_docker_test
+TEST_DOCKERIZED_SOCKET := $(TEST_DOCKERIZED_SOCKET_DIR)/uds-proxy-docker.sock
+
+IS_MAC := $(shell test "Darwin" = "`uname -s`" && echo 1)
+USE_DOCKER := $(shell test $(IS_MAC) || echo _docker)
+
+
+build: $(BINARY)
+
+$(BINARY): cmd/uds-proxy/main.go proxy/*.go
+	env go build -ldflags='-w -s $(LDFLAGS)' ./cmd/uds-proxy
+
+test_server: cmd/test_server/main.go proxy_test_server/server.go
+	go build ./cmd/test_server
+
+zip: $(BINARY)
+	zip $(BINARY)_$(GOOS)-$(GOARCH)_$(VERSION).zip $(BINARY)
+
+release: test realclean
+	env GOOS=linux  GOARCH=amd64 make clean zip
+	env GOOS=darwin GOARCH=amd64 make clean zip
+
+
+run_proxy: $(BINARY)
+	-./$(BINARY) -socket $(TEST_SOCKET) -pid-file $(TEST_PIDFILE) \
+		-prometheus-port :$(TEST_PROMETHEUS_PORT) -no-log-timestamps $(EXTRA_ARGS)
+
+run_proxy_docker:
+	@echo "run_proxy_docker: Wait for docker-composed uds-proxy to come up; or use 'make docker_run'"
+	# preparing mount directory for dockerized uds-proxy socket
+	mkdir -p $(TEST_DOCKERIZED_SOCKET_DIR)
+	chmod 777 $(TEST_DOCKERIZED_SOCKET_DIR)
+
+run_test_server: test_server
+	./test_server
+
+run_test_server_docker:
+	echo "INFO: run_test_server_docker is a noop, it's always started via docker-compose"
+
+
+test: test_go_unit test_go_functional
+
+test_go_unit:
+	go test -ldflags='-w -s $(LDFLAGS)' -tags=unit ./proxy_test
+
+test_go_functional:
+	go test -tags=functional ./proxy_test
+
+test_integration:
+	# test_integration: same as monitoring test, but without Prometheus and Grafana
+	make -j4 compose_up_no_metrics run_proxy$(USE_DOCKER) run_test_server$(USE_DOCKER) run_some_requests$(USE_DOCKER)
+
+
+coverage: clean
+	go test -coverprofile=coverage.out -coverpkg="github.com/schnoddelbotz/uds-proxy/proxy" \
+		-tags="functional unit" -ldflags='-w -s $(LDFLAGS)' ./proxy_test
+	go tool cover -html=coverage.out
+
+monitoring_test:
+	# visit Grafana on http://localhost:3000/ ... (re-)run_some_requests ... and ctrl-c to quit
+	make -j2 compose_build compose_pull
+	make -j4 compose_up run_proxy$(USE_DOCKER) run_test_server$(USE_DOCKER) run_some_requests$(USE_DOCKER)
+
+run_some_requests:
+	sh monitoring/paint_graphs.sh $(TEST_SOCKET) http://localhost:25777
+
+run_some_requests_docker:
+	sh monitoring/paint_graphs.sh $(TEST_DOCKERIZED_SOCKET) http://testserver:44555 use-sudo
+
+
+docker_image: clean
+	docker build -f uds-proxy.Dockerfile -t $(DOCKER_IMAGE):$(VERSION) -t $(DOCKER_IMAGE):latest .
+
+docker_image_push:
+	docker push $(DOCKER_IMAGE)
+
+docker_run:
+	@if [ $(IS_MAC) ]; then echo "Mounting UDS does not work on Mac. Use 'make run_proxy' instead"; exit 1; fi
+	docker run --rm -it -p28080:28080 -v$(TEST_DOCKERIZED_SOCKET_DIR):/tmp $(DOCKER_IMAGE) $(EXTRA_ARGS)
+
+
+compose_build:
+	docker-compose build
+
+compose_pull:
+	docker-compose pull
+
+compose_up:
+	docker-compose up --force-recreate
+
+compose_up_no_metrics:
+	docker-compose up --force-recreate udsproxy testserver
+
+
+grafana_dump_udsproxy_dashboard:
+	DASH_COPY_1_UID=$(shell curl -s 'localhost:3000/api/search' | jq -r '.[] | select(.id==5) | .uid'); \
+	curl -s localhost:3000/api/dashboards/uid/$$DASH_COPY_1_UID | \
+		jq '.dashboard|.id=null|.title="uds-proxy stats"|.uid="ups"' \
+		> monitoring/grafana/dashboards/uds-proxy.json
+	git diff monitoring/grafana/dashboards/uds-proxy.json
+
+
+clean:
+	rm -f $(BINARY) $(TEST_SOCKET) $(TEST_PIDFILE) test_server coverage*
+
+realclean: clean
+	-docker-compose down --volumes
+	-docker-compose rm -f
+	rm -rf $(BINARY)*.zip $(TEST_DOCKERIZED_SOCKET_DIR)

README.md

@@ -0,0 +1,177 @@
+# uds-proxy
+uds-proxy provides a UNIX domain socket and forwards traffic to HTTP(S) remotes
+through a customizable connection pool (i.e. using persistent connections).
+
+## what for? why? how?
+Interacting with microservices often involves communication overhead: Every contact
+with another service may involve DNS lookups and establishment of a TCP connection
+plus, most likely, a HTTPS handshake.
+
+This overhead can be costly and especially hard to circumvent for legacy applications -- thus uds-proxy.
+
+uds-proxy creates a UNIX domain socket and forwards communication to one or more
+remote web servers. In a way, uds-proxy aims a bit at reducing application/API complexity by
+providing a generic and simple solution for connection pooling.
+
+uds-proxy is implemented in Go, so it runs as native application on any
+OS supporting Go and UNIX domain sockets (i.e. not on Windows). Critical
+performance metrics of uds-proxy (request latencies, response codes...)
+and Go process statistics are exposed through Prometheus client library.
+
+## building / installing uds-proxy
+
+Building requires a local Go 1.11+ installation:
+
+```bash
+go get -v github.com/schnoddelbotz/uds-proxy/cmd/uds-proxy
+```
+
+... or just grab a [uds-proxy binary release](https://github.com/schnoddelbotz/uds-proxy/releases).
+
+See [usage-example-for-an-https-endpoint](#usage-example-for-an-https-endpoint) for Docker usage.
+
+To start uds-proxy at system boot, create e.g. a systemd unit.
+Don't try to run uds-proxy as root. It won't start.
+
+## usage
+
+```
+Usage of ./uds-proxy:
+  -client-timeout int
+      http client connection timeout [ms] for proxy requests (default 5000)
+  -idle-timeout int
+      connection timeout [ms] for idle backend connections (default 90000)
+  -max-conns-per-host int
+      maximum number of connections per backend host (default 20)
+  -max-idle-conns int
+      maximum number of idle HTTP(S) connections (default 100)
+  -max-idle-conns-per-host int
+      maximum number of idle conns per backend (default 25)
+  -no-log-timestamps
+      disable timestamps in log messages
+  -pid-file string
+      pid file to use, none if empty
+  -prometheus-port string
+      Prometheus monitoring port, e.g. :18080
+  -remote-https
+      remote uses https://
+  -socket string
+      path of socket to create
+  -socket-read-timeout int
+      read timeout [ms] for -socket (default 5000)
+  -socket-write-timeout int
+      write timeout [ms] for -socket (default 5000)
+  -version
+      print uds-proxy version
+```
+
+## monitoring / testing / development
+
+Clone this repository and check the [Makefile](Makefile) targets.
+
+Most relevant `make` targets:
+
+- `make monitoring_test` spins up Prometheus, grafana and uds-proxy using Docker and
+  starts another uds-proxy instance locally (outside Docker, on Mac only). The uds-proxy instances will be
+  scraped by dockerized Prometheus and Grafana will provide dashboards.
+  See [monitoring/README.md](monitoring/README.md) for details.
+- `make run_proxy` starts a local uds-proxy instance for testing purposes.
+  `TEST_SOCKET` environment variable controls socket location, defaults
+  to `uds-proxy-test.socket`.
+- `make test` runs unit and functional tests from [proxy_test](proxy_test) directory.
+- `make coverage` generates code test coverage statistics.
+- `make test_integration` starts a local uds-proxy and runs some proxied-vs-non-proxied perf tests.
+- `make realclean` removes leftovers from tests or builds.
+
+### usage example for an HTTPS endpoint
+Start the proxy:
+
+```bash
+uds-proxy -socket /tmp/proxied-svc.sock -prometheus-port :28080 -remote-https
+```
+
+Docker users:
+
+```bash
+mkdir -p /tmp/mysock_dir
+docker run --rm -it -p28080:28080 -v/tmp/mysock_dir:/tmp schnoddelbotz/uds-proxy
+```
+
+For both cases, metrics should be available at http://localhost:28080/metrics while uds-proxy is running.
+
+#### using bash / curl
+```bash
+# without uds-proxy, you would...
+time curl -I https://www.google.com/
+
+# with uds-proxy, always ...
+# a) talk through socket and
+# b) use http:// and let `-remote-https` ensure https is used to connect to remote hosts
+time curl -I --unix-socket /tmp/proxied-svc.sock http://www.google.com/
+# ... or using socket provided by dockerized uds-proxy:
+time curl -I --unix-socket /tmp/mysock_dir/uds-proxy-docker.sock http://www.google.com/
+```
+
+#### using php / curl
+```php
+<?php
+// without uds-proxy
+$ch = curl_init();
+curl_setopt($ch, CURLOPT_URL, "https://www.google.com/");
+curl_exec($ch);
+
+// with uds-proxy
+$ch = curl_init();
+curl_setopt($ch, CURLOPT_URL, "http://www.google.com/");
+curl_setopt($ch, CURLOPT_UNIX_SOCKET_PATH, "/tmp/proxied-svc.sock");
+curl_exec($ch);
+```
+
+### further socket testing
+
+Mac's (i.e. BSD's) netcat allows to talk to unix domain sockets.
+It can be used to e.g. ensure correct behaviour of uds-proxy's
+`-socket-(read|write)-timeout` options. Simply use `nc -U /path/to/uds-proxy.sock`.
+
+## todo ...
+
+- fix heatmap
+- fix/drop sudo nobody for dockerized tests
+- fixme: add option [-dont-follow-redirects](https://stackoverflow.com/questions/23297520/how-can-i-make-the-go-http-client-not-follow-redirects-automatically)
+- add a circuit breaker and or exponential backoff?
+- cleanup: wrap handleProxyRequest into logging handler
+- travis-ci + github release push
+- level-based stdout/err logging https://awesome-go.com/#logging
+- example systemd unit
+- log request sizes?
+- use/fix 'make test' in Dockerfile
+- sock umask / cli opt
+- support magic uds request headers...?
+  - X-udsproxy-timeout: 250ms
+  - X-udsproxy-debug: true
+
+## links
+
+- https://godoc.org/gotest.tools/assert
+- https://golang.org/pkg/net/#hdr-Name_Resolution
+- https://stackoverflow.com/questions/17948827/reusing-http-connections-in-golang
+- https://medium.com/@povilasve/go-advanced-tips-tricks-a872503ac859
+- https://github.com/bouk/monkey/blob/master/monkey_test.go
+- https://github.com/prometheus/client_golang/blob/master/prometheus/examples_test.go
+- https://github.com/prometheus/client_golang/blob/master/prometheus/promhttp/instrument_server.go
+
+## alternatives
+
+- for Python and Redis, use a [redis.py connection pool](https://github.com/andymccurdy/redis-py#connection-pools)
+- for Python and HTTP, requests library will just do it
+- for Redis and PHP, [phpredis](https://github.com/phpredis/phpredis) supports experimental persistent connections
+- a potentially more sophisticated solution can be found in
+  [this TCP vs UDS speed comparison stackoverflow thread](https://stackoverflow.com/questions/14973942/performance-tcp-loopback-connection-vs-unix-domain-socket):
+  [Speedus](http://speedus.torusware.com/) intercepts relevant system calls, which avoids
+  need for any code changes. However, if I understood correctly, Speedus only helps if
+  services actually sit on the same host system.
+
+
+## license
+
+MIT

cmd/test_server/main.go

@@ -0,0 +1,17 @@
+package main
+
+import (
+	"flag"
+
+	"github.com/schnoddelbotz/uds-proxy/proxy_test_server"
+)
+
+func main() {
+	var port string
+	flag.StringVar(&port, "port", ":25777", "fake webserver tcp port")
+	// could add flag to exit after N requests?
+	// could add flag to exit after N seconds?
+	flag.Parse()
+
+	proxy_test_server.RunUpstreamFakeServer(port, false)
+}

cmd/uds-proxy/main.go

@@ -0,0 +1,37 @@
+package main
+
+import (
+	"flag"
+	"os"
+
+	"github.com/schnoddelbotz/uds-proxy/proxy"
+)
+
+func main() {
+	var args proxy.CliArgs
+
+	if os.Getuid() == 0 {
+		println("uds-proxy is refusing to run as root user")
+		os.Exit(1)
+	}
+
+	flag.BoolVar(&args.NoLogTimeStamps, "no-log-timestamps", false, "disable timestamps in log messages")
+	flag.BoolVar(&args.PrintVersion, "version", false, "print uds-proxy version")
+	flag.BoolVar(&args.RemoteHTTPS, "remote-https", false, "remote uses https://")
+
+	flag.IntVar(&args.MaxConnsPerHost, "max-conns-per-host", 20, "maximum number of connections per backend host")
+	flag.IntVar(&args.MaxIdleConns, "max-idle-conns", 100, "maximum number of idle HTTP(S) connections")
+	flag.IntVar(&args.MaxIdleConnsPerHost, "max-idle-conns-per-host", 25, "maximum number of idle conns per backend")
+	flag.IntVar(&args.ClientTimeout, "client-timeout", 5000, "http client connection timeout [ms] for proxy requests")
+	flag.IntVar(&args.IdleConnTimeout, "idle-timeout", 90000, "connection timeout [ms] for idle backend connections")
+	flag.IntVar(&args.SocketReadTimeout, "socket-read-timeout", 5000, "read timeout [ms] for -socket")
+	flag.IntVar(&args.SocketWriteTimeout, "socket-write-timeout", 5000, "write timeout [ms] for -socket")
+
+	flag.StringVar(&args.PidFile, "pid-file", "", "pid file to use, none if empty")
+	flag.StringVar(&args.SocketPath, "socket", os.Getenv("UDS_PROXY_SOCKET"), "path of socket to create")
+	flag.StringVar(&args.PrometheusPort, "prometheus-port", "", "Prometheus monitoring port, e.g. :18080")
+
+	flag.Parse()
+
+	proxy.NewProxyInstance(args).Run()
+}