V2 HdrHistogram Log format (v1.3) (#35)

* [add] V2 HdrHistogram Log format (v1.3)

* [fix] Added nolint to the examples

* [fix] Fixed error message typo on minimum length on bytes for 7,8,and 9 bytelen

* [fix] Fixed ExampleHistogram_PercentilesPrint

Author: filipecosta90

.gitignore

@@ -14,8 +14,12 @@ coverage.txt
 # Test binary, built with `go test -c`
 *.test
 
+# Test example output
+example.logV2.hlog
+
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
 
 # Dependency directories (remove the comment below to include it)
 # vendor/
+

Makefile

@@ -7,6 +7,7 @@ GOTEST=$(GOCMD) test
 GOGET=$(GOCMD) get
 GOMOD=$(GOCMD) mod
 GOFMT=$(GOCMD) fmt
+GODOC=godoc
 
 .PHONY: all test coverage
 all: test coverage
@@ -36,3 +37,6 @@ test: get fmt
 coverage: get test
 	$(GOTEST) -race -coverprofile=coverage.txt -covermode=atomic .
 
+godoc:
+	$(GODOC)
+

example_hdr_test.go

@@ -1,13 +1,65 @@
 package hdrhistogram_test
 
 import (
+	"fmt"
 	"github.com/HdrHistogram/hdrhistogram-go"
 	"os"
 )
 
+// This latency Histogram could be used to track and analyze the counts of
+// observed integer values between 0 us and 30000000 us ( 30 secs )
+// while maintaining a value precision of 4 significant digits across that range,
+// translating to a value resolution of :
+//   - 1 microsecond up to 10 milliseconds,
+//   - 100 microsecond (or better) from 10 milliseconds up to 10 seconds,
+//   - 300 microsecond (or better) from 10 seconds up to 30 seconds,
+// nolint
+func ExampleNew() {
+	lH := hdrhistogram.New(1, 30000000, 4)
+	input := []int64{
+		459876, 669187, 711612, 816326, 931423, 1033197, 1131895, 2477317,
+		3964974, 12718782,
+	}
+
+	for _, sample := range input {
+		lH.RecordValue(sample)
+	}
+
+	fmt.Printf("Percentile 50: %d\n", lH.ValueAtQuantile(50.0))
+
+	// Output:
+	// Percentile 50: 931423
+}
+
+// This latency Histogram could be used to track and analyze the counts of
+// observed integer values between 0 us and 30000000 us ( 30 secs )
+// while maintaining a value precision of 3 significant digits across that range,
+// translating to a value resolution of :
+//   - 1 microsecond up to 1 millisecond,
+//   - 1 millisecond (or better) up to one second,
+//   - 1 second (or better) up to it's maximum tracked value ( 30 seconds ).
+// nolint
+func ExampleHistogram_RecordValue() {
+	lH := hdrhistogram.New(1, 30000000, 3)
+	input := []int64{
+		459876, 669187, 711612, 816326, 931423, 1033197, 1131895, 2477317,
+		3964974, 12718782,
+	}
+
+	for _, sample := range input {
+		lH.RecordValue(sample)
+	}
+
+	fmt.Printf("Percentile 50: %d\n", lH.ValueAtQuantile(50.0))
+
+	// Output:
+	// Percentile 50: 931839
+}
+
 // The following example details the creation of an histogram used to track
 // and analyze the counts of observed integer values between 0 us and 30000000 us ( 30 secs )
 // and the printing of the percentile output format
+// nolint
 func ExampleHistogram_PercentilesPrint() {
 	lH := hdrhistogram.New(1, 30000000, 3)
 	input := []int64{

example_log_writer_test.go

@@ -0,0 +1,114 @@
+package hdrhistogram_test
+
+import (
+	"bytes"
+	"fmt"
+	hdrhistogram "github.com/HdrHistogram/hdrhistogram-go"
+	"io/ioutil"
+)
+
+// The log format encodes into a single file, multiple histograms with optional shared meta data.
+// The following example showcases reading a log file into a slice of histograms
+// nolint
+func ExampleNewHistogramLogReader() {
+	dat, _ := ioutil.ReadFile("./test/tagged-Log.logV2.hlog")
+	r := bytes.NewReader(dat)
+
+	// Create a histogram log reader
+	reader := hdrhistogram.NewHistogramLogReader(r)
+	var histograms []*hdrhistogram.Histogram = make([]*hdrhistogram.Histogram, 0)
+
+	// Read all histograms in the file
+	for hist, err := reader.NextIntervalHistogram(); hist != nil && err == nil; hist, err = reader.NextIntervalHistogram() {
+		histograms = append(histograms, hist)
+	}
+	fmt.Printf("Read a total of %d histograms\n", len(histograms))
+
+	min := reader.RangeObservedMin()
+	max := reader.RangeObservedMax()
+	sigdigits := 3
+	overallHistogram := hdrhistogram.New(min, max, sigdigits)
+
+	//// We can then merge all histograms into one and retrieve overall metrics
+	for _, hist := range histograms {
+		overallHistogram.Merge(hist)
+	}
+	fmt.Printf("Overall count: %d samples\n", overallHistogram.TotalCount())
+	fmt.Printf("Overall Percentile 50: %d\n", overallHistogram.ValueAtQuantile(50.0))
+
+	// Output:
+	// Read a total of 42 histograms
+	// Overall count: 32290 samples
+	// Overall Percentile 50: 344319
+
+}
+
+// The log format encodes into a single file, multiple histograms with optional shared meta data.
+// The following example showcases writing multiple histograms into a log file and then
+// processing them again to confirm a proper encode-decode flow
+// nolint
+func ExampleNewHistogramLogWriter() {
+	var buff bytes.Buffer
+
+	// Create a histogram log writer to write to a bytes.Buffer
+	writer := hdrhistogram.NewHistogramLogWriter(&buff)
+
+	writer.OutputLogFormatVersion()
+	writer.OutputStartTime(0)
+	writer.OutputLegend()
+
+	// Lets create 3 distinct histograms to exemply the logwriter features
+	// each one with a time-frame of 60 secs ( 60000 ms )
+	hist1 := hdrhistogram.New(1, 30000000, 3)
+	hist1.SetStartTimeMs(0)
+	hist1.SetEndTimeMs(60000)
+	for _, sample := range []int64{10, 20, 30, 40} {
+		hist1.RecordValue(sample)
+	}
+	hist2 := hdrhistogram.New(1, 3000, 3)
+	hist1.SetStartTimeMs(60001)
+	hist1.SetEndTimeMs(120000)
+	for _, sample := range []int64{50, 70, 80, 60} {
+		hist2.RecordValue(sample)
+	}
+	hist3 := hdrhistogram.New(1, 30000, 3)
+	hist1.SetStartTimeMs(120001)
+	hist1.SetEndTimeMs(180000)
+	for _, sample := range []int64{90, 100} {
+		hist3.RecordValue(sample)
+	}
+	writer.OutputIntervalHistogram(hist1)
+	writer.OutputIntervalHistogram(hist2)
+	writer.OutputIntervalHistogram(hist3)
+
+	ioutil.WriteFile("example.logV2.hlog", buff.Bytes(), 0644)
+
+	// read check
+	// Lets read all again and confirm that the total sample count is 10
+	dat, _ := ioutil.ReadFile("example.logV2.hlog")
+	r := bytes.NewReader(dat)
+
+	// Create a histogram log reader
+	reader := hdrhistogram.NewHistogramLogReader(r)
+	var histograms []*hdrhistogram.Histogram = make([]*hdrhistogram.Histogram, 0)
+
+	// Read all histograms in the file
+	for hist, err := reader.NextIntervalHistogram(); hist != nil && err == nil; hist, err = reader.NextIntervalHistogram() {
+		histograms = append(histograms, hist)
+	}
+	fmt.Printf("Read a total of %d histograms\n", len(histograms))
+
+	min := reader.RangeObservedMin()
+	max := reader.RangeObservedMax()
+	sigdigits := 3
+	overallHistogram := hdrhistogram.New(min, max, sigdigits)
+
+	//// We can then merge all histograms into one and retrieve overall metrics
+	for _, hist := range histograms {
+		overallHistogram.Merge(hist)
+	}
+	fmt.Printf("Overall count: %d samples\n", overallHistogram.TotalCount())
+	// Output:
+	// Read a total of 3 histograms
+	// Overall count: 10 samples
+}

hdr.go

@@ -40,6 +40,33 @@ type Histogram struct {
 	countsLen                   int32
 	totalCount                  int64
 	counts                      []int64
+	startTimeMs                 int64
+	endTimeMs                   int64
+	tag                         string
+}
+
+func (h *Histogram) Tag() string {
+	return h.tag
+}
+
+func (h *Histogram) SetTag(tag string) {
+	h.tag = tag
+}
+
+func (h *Histogram) EndTimeMs() int64 {
+	return h.endTimeMs
+}
+
+func (h *Histogram) SetEndTimeMs(endTimeMs int64) {
+	h.endTimeMs = endTimeMs
+}
+
+func (h *Histogram) StartTimeMs() int64 {
+	return h.startTimeMs
+}
+
+func (h *Histogram) SetStartTimeMs(startTimeMs int64) {
+	h.startTimeMs = startTimeMs
 }
 
 // New returns a new Histogram instance capable of tracking values in the given
@@ -93,6 +120,9 @@ func New(minValue, maxValue int64, sigfigs int) *Histogram {
 		countsLen:                   countsLen,
 		totalCount:                  0,
 		counts:                      make([]int64, countsLen),
+		startTimeMs:                 0,
+		endTimeMs:                   0,
+		tag:                         "",
 	}
 }