initial commit

Author: daehee

.gitignore

@@ -0,0 +1,3 @@
+.idea/
+feeds/
+tmp/

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Daehee Park
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1 @@
+# cvebaser

cmd/cvebaser/main.go

@@ -0,0 +1,61 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"time"
+
+	"github.com/cvebase/cvebaser"
+	"github.com/gobwas/cli"
+)
+
+func main() {
+	cli.Main(cli.Commands{
+		"lint": new(lintCommand),
+	})
+}
+
+type lintCommand struct {
+	commit   string
+	repoPath string
+}
+
+func (l *lintCommand) DefineFlags(fs *flag.FlagSet) {
+	fs.StringVar(&l.commit,
+		"c", l.commit,
+		"commit hash",
+	)
+	fs.StringVar(&l.repoPath,
+		"r", l.repoPath,
+		"path to cvebase.com repo",
+	)
+	// TODO add concurrency option
+}
+
+func (l *lintCommand) Run(_ context.Context, _ []string) error {
+	repo, err := cvebaser.NewRepo(l.repoPath, &cvebaser.GitOpts{})
+	if err != nil {
+		return err
+	}
+
+	start := time.Now()
+	if l.commit != "" {
+		err = repo.LintCommit(l.commit)
+		if err != nil {
+			return err
+		}
+	} else {
+		err = repo.LintAll(20)
+		if err != nil {
+			return err
+		}
+	}
+	duration := time.Since(start)
+
+	// TODO print number of files modified
+
+	fmt.Printf("\n--- %.1f seconds ---\n", duration.Seconds())
+
+	return nil
+}

cve.go

@@ -0,0 +1,76 @@
+package cvebaser
+
+import (
+	"fmt"
+	"path"
+	"strconv"
+	"strings"
+
+	"github.com/daehee/nvd"
+)
+
+type CVE struct {
+	CVEID    string   `json:"-" yaml:"id"`
+	Pocs     []string `json:"pocs,omitempty" yaml:"pocs,omitempty"`
+	Courses  []string `json:"courses,omitempty" yaml:"courses,omitempty"`
+	Writeups []string `json:"writeups,omitempty" yaml:"writeups,omitempty"`
+	Advisory string   `json:"advisory,omitempty" yaml:"-"`
+}
+
+func (m *CVE) dedupeSort() {
+	m.Pocs = sortUniqStrings(m.Pocs)
+	m.Writeups = sortUniqStrings(m.Writeups)
+	m.Courses = sortUniqStrings(m.Courses)
+}
+
+// isValidCVESubPath checks if cve file is placed in correct year and sequence sub-directories.
+func isValidCVESubPath(cveID, path string) bool {
+	// Truncate path to slice containing relative path
+	splitPath := strings.Split(path, "/")
+	// ../../../../cvebase.com/cve/2018/xxx/CVE-2018-0142.md ->
+	// [2018, xxx, CVE-2018-0142.md]
+	splitPath = splitPath[len(splitPath)-3:]
+
+	validPath, err := cveSubPath(cveID)
+	if err != nil {
+		return false
+	}
+	splitValid := strings.Split(validPath, "/")
+
+	// Compare equality of slice values
+	for i, v := range splitPath {
+		if v != splitValid[i] {
+			return false
+		}
+	}
+
+	return true
+}
+
+// cvePathToRelPath truncates cve filepath to relative path starting with year subdirectory
+func cvePathToRelPath(p string) string {
+	// ../../../../cvebase.com/cve/2018/0xxx/CVE-2018-0142.md ->
+	// 2018/0xxx/CVE-2018-0142.md
+	splitPath := strings.Split(p, "/")
+	return strings.Join(splitPath[len(splitPath)-3:], "/")
+}
+
+// cveSubPath converts a CVE ID to cve relative path starting with year subdirectory
+func cveSubPath(cveID string) (string, error) {
+	year, sequence := nvd.ParseCVEID(cveID)
+	seqDir, err := cveSeqDir(sequence)
+	if err != nil {
+		return "", fmt.Errorf("error parsing %s sequence to dir: %v", cveID, err)
+	}
+	return path.Join(strconv.Itoa(year), seqDir, fmt.Sprintf("%s.md", cveID)), nil
+}
+
+// cveSeqDir converts a cve sequence number to a "x"-padded sequence directory name
+func cveSeqDir(seq int) (string, error) {
+	seqStr := nvd.PadCVESequence(seq)
+	subDir := seqStr[:len(seqStr)-3]
+	if len(subDir) < 1 {
+		return "", fmt.Errorf("CVE sequence invalid: %s -> %s", seqStr, subDir)
+	}
+	return fmt.Sprintf("%sxxx", subDir), nil
+}

cve_test.go

@@ -0,0 +1,67 @@
+package cvebaser
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCVE_DedupeSort(t *testing.T) {
+	tests := []struct {
+		cve     CVE
+		wantCVE CVE
+	}{
+		{
+			CVE{
+				CVEID:    "CVE-2020-14882",
+				Pocs:     []string{"https://github.com", "https://github.com", "https://github.com", "https://exploit-db.com"},
+				Writeups: []string{"https://github.com", "https://github.com", "https://github.com", "https://exploit-db.com"},
+				Courses:  []string{"https://github.com", "https://github.com", "https://github.com", "https://exploit-db.com"},
+				Advisory: "lorem ipsum dolor",
+			},
+			CVE{
+				CVEID:    "CVE-2020-14882",
+				Pocs:     []string{"https://exploit-db.com", "https://github.com"},
+				Writeups: []string{"https://exploit-db.com", "https://github.com"},
+				Courses:  []string{"https://exploit-db.com", "https://github.com"},
+				Advisory: "lorem ipsum dolor",
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		tt.cve.dedupeSort()
+		assert.EqualValues(t, tt.wantCVE.Pocs, tt.cve.Pocs)
+		assert.EqualValues(t, tt.wantCVE.Writeups, tt.cve.Writeups)
+		assert.EqualValues(t, tt.wantCVE.Courses, tt.cve.Courses)
+	}
+}
+
+func TestCVESubPath(t *testing.T) {
+	want := "2020/14xxx/CVE-2020-14882.md"
+	got, err := cveSubPath("CVE-2020-14882")
+	assert.NoError(t, err)
+	assert.Equal(t, want, got)
+}
+
+func TestIsValidCVEDirPath(t *testing.T) {
+	got := isValidCVESubPath("CVE-2016-0974", "../../../../cvebase.com/cve/2016/0xxx/CVE-2016-0974.md")
+	assert.True(t, got)
+}
+
+func TestCVESeqDir(t *testing.T) {
+	tests := []struct {
+		sequence int
+		want     string
+	}{
+		{974, "0xxx"},
+		{14882, "14xxx"},
+		{97, "0xxx"},
+	}
+
+	for _, tt := range tests {
+		got, err := cveSeqDir(tt.sequence)
+		assert.NoError(t, err)
+		assert.Equal(t, got, tt.want)
+	}
+}

file.go

@@ -0,0 +1,138 @@
+package cvebaser
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/gohugoio/hugo/parser/pageparser"
+	"gopkg.in/yaml.v3"
+)
+
+// ParseCVEMDFile reads markdown file contents containing YAML and markdown
+// and returns CVE data struct
+func ParseCVEMDFile(reader io.Reader) (cve CVE, err error) {
+	pf, err := pageparser.ParseFrontMatterAndContent(reader)
+	if err != nil {
+		return cve, err
+	}
+	fm, err := yaml.Marshal(pf.FrontMatter)
+	if err != nil {
+		return cve, err
+	}
+	err = yaml.Unmarshal(fm, &cve)
+	if err != nil {
+		return cve, err
+	}
+	cve.Advisory = string(pf.Content)
+	return
+}
+
+// ParseResearcherMDFile reads markdown file contents containing YAML and markdown
+// and returns Researcher data struct
+func ParseResearcherMDFile(reader io.Reader) (researcher Researcher, err error) {
+	pf, err := pageparser.ParseFrontMatterAndContent(reader)
+	if err != nil {
+		return researcher, err
+	}
+	fm, err := yaml.Marshal(pf.FrontMatter)
+	if err != nil {
+		return researcher, err
+	}
+	err = yaml.Unmarshal(fm, &researcher)
+	if err != nil {
+		return researcher, err
+	}
+	researcher.Bio = string(pf.Content)
+	return
+}
+
+// ParseMDFile reads markdown file contents containing YAML and markdown
+// and returns either CVE or Researcher data struct
+func ParseMDFile(r io.Reader, tPtr interface{}) error {
+	pf, err := pageparser.ParseFrontMatterAndContent(r)
+	if err != nil {
+		return fmt.Errorf("error parsing front matter: %v", err)
+	}
+	fm, err := yaml.Marshal(pf.FrontMatter)
+	if err != nil {
+		return fmt.Errorf("error marshaling yaml: %v", err)
+	}
+	err = yaml.Unmarshal(fm, tPtr)
+	if err != nil {
+		return fmt.Errorf("erorr unmarshaling yaml: %v", err)
+	}
+
+	// Set field value for markdown text depending on CVE or Researcher
+	switch t := tPtr.(type) {
+	case *CVE:
+		t.Advisory = string(pf.Content)
+	case *Researcher:
+		t.Bio = string(pf.Content)
+	default:
+		return fmt.Errorf("unknown type: %+v", tPtr)
+	}
+
+	return nil
+}
+
+const yamlDelimLf = "---\n"
+
+func CompileToFile(
+	f *os.File,
+	path string,
+	t interface{}, /* CVE or Researcher to marshal */
+) error {
+	// Configure yaml encoding for custom indent spacing
+	var d bytes.Buffer
+	yamlEncoder := yaml.NewEncoder(&d)
+	// go-yaml v3 now defaults to 4 spaces, so manually set to 2
+	yamlEncoder.SetIndent(2)
+	err := yamlEncoder.Encode(t)
+	if err != nil {
+		return fmt.Errorf("error marshaling yaml to %s", path)
+	}
+	yamlEncoder.Close()
+
+	// Lead with marshaling first so that
+	// if fails doesn't error with a pre-maturely truncated file
+	// d, err := yaml.Marshal(t)
+
+	// TODO Check if file contents have changed before writing, otherwise return early
+
+	// Clear file for writing
+	f.Truncate(0)
+	f.Seek(0, 0)
+
+	_, err = f.WriteString(yamlDelimLf)
+	if err != nil {
+		return fmt.Errorf("error writing to %s: %v", path, err)
+	}
+	_, err = f.Write(d.Bytes())
+	if err != nil {
+		return fmt.Errorf("error writing to %s: %v", path, err)
+	}
+	_, err = f.WriteString(yamlDelimLf)
+	if err != nil {
+		return fmt.Errorf("error writing to %s: %v", path, err)
+	}
+
+	// Write markdown content from struct field depending on type CVE or Researcher
+	switch t := t.(type) {
+	case CVE:
+		_, err = f.WriteString(t.Advisory)
+		if err != nil {
+			return fmt.Errorf("error writing to %s: %v", path, err)
+		}
+	case Researcher:
+		_, err = f.WriteString(t.Bio)
+		if err != nil {
+			return fmt.Errorf("error writing to %s: %v", path, err)
+		}
+	default:
+		return fmt.Errorf("unknown type: %+v", t)
+	}
+
+	return nil
+}