godev: create godev submodule and content package

The godev submodule contains code necessary to run the telemetry
services we plan to host on GCP. The submodule structure will keep
packages that rely on third party dependencies separate from the
telemetry collection code which may eventually be depended on by gopls
and the go command.

The content package implements a basic web serving framework with support
for markdown, go templates, and typescript files.

Change-Id: If3da206d858bd969a681d29d1d9ad829b31211da
Reviewed-on: https://go-review.googlesource.com/c/telemetry/+/495755
Reviewed-by: Hyang-Ah Hana Kim <[email protected]>
TryBot-Result: Gopher Robot <[email protected]>
Run-TryBot: Jamal Carvalho <[email protected]>

Author: jamalc

doc.go

@@ -0,0 +1 @@
+package telemetry

go.mod

@@ -0,0 +1,3 @@
+module golang.org/x/telemetry
+
+go 1.20

go.sum

@@ -0,0 +1,17 @@
+module golang.org/x/telemetry/godev
+
+go 1.20
+
+require (
+	github.com/evanw/esbuild v0.17.19
+	github.com/google/go-cmp v0.5.9
+	github.com/yuin/goldmark v1.5.4
+	golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1
+)
+
+require gopkg.in/yaml.v2 v2.4.0 // indirect
+
+require (
+	github.com/yuin/goldmark-meta v1.1.0
+	golang.org/x/sys v0.8.0 // indirect
+)

godev/go.mod

@@ -0,0 +1,17 @@
+github.com/evanw/esbuild v0.17.19 h1:JdzNCvfFEoUCXKHhdP326Vn2mhCu8PybXeBDHaSRyWo=
+github.com/evanw/esbuild v0.17.19/go.mod h1:iINY06rn799hi48UqEnaQvVfZWe6W9bET78LbvN8VWk=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/yuin/goldmark v1.5.4 h1:2uY/xC0roWy8IBEGLgB1ywIoEJFGmRrX21YQcvGZzjU=
+github.com/yuin/goldmark v1.5.4/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+github.com/yuin/goldmark-meta v1.1.0 h1:pWw+JLHGZe8Rk0EGsMVssiNb/AaPMHfSRszZeUeiOUc=
+github.com/yuin/goldmark-meta v1.1.0/go.mod h1:U4spWENafuA7Zyg+Lj5RqK/MF+ovMYtBvXi1lBb2VP0=
+golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc=
+golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=

godev/go.sum

@@ -0,0 +1,312 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package content implements a basic web serving framework.
+//
+// # Content Server
+//
+// A content server is an http.Handler that serves requests from a file system.
+// Use Server(fsys) to create a new content server.
+//
+// The server is defined primarily by the content of its file system fsys,
+// which holds files to be served. It renders markdown files and golang
+// templates into HTML and transforms TypeScript into JavaScript.
+//
+// # Page Rendering
+//
+// A request for a path like "/page" will search the file system for
+// "page.md", "page.html", "page/index.md", and "page/index.html" and
+// render HTML output for the first file found.
+//
+// Partial templates with the extension ".tmpl" at the root of the file system
+// and in the same directory as the requested page are included in the
+// html/template execution step to allow for sharing and composing logic from
+// multiple templates.
+package content
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"html/template"
+	"io/fs"
+	"net/http"
+	"path"
+	"strconv"
+	"strings"
+
+	"github.com/yuin/goldmark"
+	meta "github.com/yuin/goldmark-meta"
+	"github.com/yuin/goldmark/extension"
+	"github.com/yuin/goldmark/parser"
+	"github.com/yuin/goldmark/renderer/html"
+	"golang.org/x/exp/slog"
+)
+
+// contentServer serves requests for a given file system. It can also render
+// templates and transform TypeScript into JavaScript.
+type contentServer struct {
+	fsys     fs.FS
+	fserv    http.Handler
+	handlers map[string]handlerFunc
+}
+
+type handler struct {
+	path string
+	fn   handlerFunc
+}
+
+type handlerFunc func(http.ResponseWriter, *http.Request, fs.FS) error
+
+// Server returns a handler that serves HTTP requests with the contents
+// of the file system rooted at fsys. For requests to a path without an
+// extension, the server will search fsys for markdown or html templates
+// first by appending .md, .html, /index.md, and /index.html to the
+// requested url path.
+//
+// The default behavior of looking for templates within fsys can be overriden
+// by using an optional set of content handlers.
+//
+// For example, a server can be constructed for a file system with a single
+// template, “index.html“, in a directory, “content“, and a handler:
+//
+//	 s := content.Server(os.DirFS("content"),
+//		 content.Handler("/", func(w http.ReponseWriter, _ *http.Request, fsys fs.FS) error {
+//		 	 return content.Template(w, fsys, "index.html", nil, http.StatusOK)
+//		 }))
+//
+// or without a handler:
+//
+//	content.Server(os.DirFS("content"))
+//
+// Both examples will render the template index.html for requests to "/".
+func Server(fsys fs.FS, handlers ...*handler) http.Handler {
+	fserv := http.FileServer(http.FS(fsys))
+	hs := make(map[string]handlerFunc)
+	for _, h := range handlers {
+		if _, ok := hs[h.path]; ok {
+			panic("multiple registrations for " + h.path)
+		}
+		hs[h.path] = h.fn
+	}
+	return &contentServer{fsys, fserv, hs}
+}
+
+func Handler(path string, h handlerFunc) *handler {
+	return &handler{path, h}
+}
+
+func (c *contentServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if len(r.URL.Path) > 255 {
+		Error(w, r, errors.New("url too long"), http.StatusBadRequest)
+		return
+	}
+
+	if handler, ok := c.handlers[r.URL.Path]; ok {
+		err := handler(w, r, c.fsys)
+		if err != nil {
+			Error(w, r, err, http.StatusInternalServerError)
+		}
+		return
+	}
+
+	ext := path.Ext(r.URL.Path)
+	if ext == ".md" || ext == ".html" {
+		http.Redirect(w, r, strings.TrimSuffix(r.URL.Path, ext), http.StatusMovedPermanently)
+		return
+	}
+
+	filepath, info, err := stat(c.fsys, r.URL.Path)
+	if errors.Is(err, fs.ErrNotExist) {
+		Error(w, r, errors.New("page not found"), http.StatusNotFound)
+		return
+	}
+	if err == nil {
+		if strings.HasSuffix(r.URL.Path, "/index") {
+			http.Redirect(w, r, strings.TrimSuffix(r.URL.Path, "/index"), http.StatusMovedPermanently)
+			return
+		}
+		switch path.Ext(filepath) {
+		case ".html":
+			err = Template(w, c.fsys, filepath, nil, http.StatusOK)
+		case ".md":
+			err = markdown(w, c.fsys, filepath, http.StatusOK)
+		case ".ts":
+			err = script(w, c.fsys, filepath, info)
+		default:
+			c.fserv.ServeHTTP(w, r)
+		}
+	}
+	if err != nil {
+		Error(w, r, err, http.StatusInternalServerError)
+	}
+}
+
+// Template executes a template response.
+func Template(w http.ResponseWriter, fsys fs.FS, tmplPath string, data any, code int) error {
+	patterns, err := tmplPatterns(fsys, tmplPath)
+	if err != nil {
+		return err
+	}
+	patterns = append(patterns, tmplPath)
+	tmpl, err := template.ParseFS(fsys, patterns...)
+	if err != nil {
+		return err
+	}
+	name := path.Base(tmplPath)
+	var buf bytes.Buffer
+	if err := tmpl.ExecuteTemplate(&buf, name, data); err != nil {
+		return err
+	}
+	if code != 0 {
+		w.WriteHeader(code)
+	}
+	w.Header().Set("Content-Type", "text/html")
+	w.Header().Set("Content-Length", strconv.Itoa(buf.Len()))
+	if _, err := w.Write(buf.Bytes()); err != nil {
+		return err
+	}
+	return nil
+}
+
+// JSON encodes data as JSON response with a status code.
+func JSON(w http.ResponseWriter, data any, code int) error {
+	var buf bytes.Buffer
+	if err := json.NewEncoder(&buf).Encode(data); err != nil {
+		return err
+	}
+	if code != 0 {
+		w.WriteHeader(code)
+	}
+	w.Header().Set("Content-Type", "application/json")
+	w.Header().Set("Content-Length", strconv.Itoa(buf.Len()))
+	if _, err := w.Write(buf.Bytes()); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Text formats data as a text response with a status code.
+func Text(w http.ResponseWriter, data any, code int) error {
+	var buf bytes.Buffer
+	if _, err := fmt.Fprint(&buf, data); err != nil {
+		return err
+	}
+	if code != 0 {
+		w.WriteHeader(code)
+	}
+	w.Header().Set("Content-Type", "text/plain")
+	w.Header().Set("Content-Length", strconv.Itoa(buf.Len()))
+	if _, err := w.Write(buf.Bytes()); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Error writes an error as an HTTP response with a status code.
+func Error(w http.ResponseWriter, req *http.Request, err error, code int) {
+	if code == http.StatusInternalServerError {
+		http.Error(w, http.StatusText(http.StatusInternalServerError), code)
+	} else {
+		http.Error(w, err.Error(), code)
+	}
+	slog.Error("request error",
+		slog.String("method", req.Method),
+		slog.String("uri", req.RequestURI),
+		slog.Int("status", code),
+		slog.String("error", err.Error()),
+	)
+}
+
+// markdown renders a markdown template as html.
+func markdown(w http.ResponseWriter, fsys fs.FS, tmplPath string, code int) error {
+	markdown, err := fs.ReadFile(fsys, tmplPath)
+	if err != nil {
+		return err
+	}
+	md := goldmark.New(
+		goldmark.WithParserOptions(
+			parser.WithHeadingAttribute(),
+			parser.WithAutoHeadingID(),
+		),
+		goldmark.WithRendererOptions(
+			html.WithUnsafe(),
+			html.WithXHTML(),
+		),
+		goldmark.WithExtensions(
+			extension.GFM,
+			extension.NewTypographer(),
+			meta.Meta,
+		),
+	)
+	var content bytes.Buffer
+	ctx := parser.NewContext()
+	if err := md.Convert(markdown, &content, parser.WithContext(ctx)); err != nil {
+		return err
+	}
+	data := meta.Get(ctx)
+	if data == nil {
+		data = map[string]interface{}{}
+	}
+	data["Content"] = template.HTML(content.String())
+	if _, ok := data["Template"]; !ok {
+		data["Template"] = "base.html"
+	}
+	return Template(w, fsys, data["Template"].(string), data, code)
+}
+
+// script serves TypeScript code tranformed into JavaScript.
+func script(w http.ResponseWriter, fsys fs.FS, filepath string, info fs.FileInfo) error {
+	data, err := fs.ReadFile(fsys, filepath)
+	if err != nil {
+		return err
+	}
+	output := esbuild(data)
+	w.Header().Set("Content-Type", "text/javascript")
+	w.Header().Set("Content-Length", strconv.Itoa(output.Len()))
+	w.Header().Set("Last-Modified", info.ModTime().Format("Mon, 02 Jan 2006 15:04:05 GMT"))
+	if _, err := w.Write(output.Bytes()); err != nil {
+		return err
+	}
+	return nil
+}
+
+// stat trys to coerce a urlPath into an openable file then returns the
+// file path and file info.
+func stat(fsys fs.FS, urlPath string) (string, fs.FileInfo, error) {
+	cleanPath := path.Clean(strings.TrimPrefix(urlPath, "/"))
+	ext := path.Ext(cleanPath)
+	filePaths := []string{cleanPath}
+	if ext == "" || ext == "." {
+		md := cleanPath + ".md"
+		html := cleanPath + ".html"
+		indexMD := path.Join(cleanPath, "index.md")
+		indexHTML := path.Join(cleanPath, "index.html")
+		filePaths = []string{md, html, indexMD, indexHTML, cleanPath}
+	}
+	var p string
+	var stat fs.FileInfo
+	var err error
+	for _, p = range filePaths {
+		if stat, err = fs.Stat(fsys, p); err == nil {
+			break
+		}
+	}
+	return p, stat, err
+}
+
+// tmplPatters generates a slice of file patterns to use in template.ParseFS.
+func tmplPatterns(fsys fs.FS, tmplPath string) ([]string, error) {
+	var patterns []string
+	globs := []string{"*.tmpl", path.Join(path.Dir(tmplPath), "*.tmpl")}
+	for _, g := range globs {
+		matches, err := fs.Glob(fsys, g)
+		if err != nil {
+			return nil, err
+		}
+		patterns = append(patterns, matches...)
+	}
+	return patterns, nil
+}