Merge pull request #436 from pixlise/fixes

Fixes

Author: pnemere

api/dataimport/sdfToRSI/scanSDF.go

@@ -115,6 +115,12 @@ func scanSDF(sdfPath string) ([]EventEntry, error) {
 			refs = append(refs, EventEntry{Line: lineNo, What: "science", Value: "end"})
 		}
 
+		// For scans that are prematurely ended, we may not get the above, so treat the scan log printing out
+		// less points than actual as a science end!
+		if isPrematureEnd(lineData) {
+			refs = append(refs, EventEntry{Line: lineNo, What: "science", Value: "end"})
+		}
+
 		if strings.Contains(lineData, "Open the Dust Cover\"") {
 			refs = append(refs, EventEntry{Line: lineNo, What: "dust-cover", Value: "opening"})
 		}
@@ -142,6 +148,37 @@ func scanSDF(sdfPath string) ([]EventEntry, error) {
 	return refs, nil
 }
 
+func isPrematureEnd(lineData string) bool {
+	// Example line:
+	// Planned Points:         2339   Actual:    761
+	plannedToken := "Planned Points:"
+	actualToken := "Actual:"
+
+	plannedPtsIdx := strings.Index(lineData, plannedToken)
+	actualPtsIdx := strings.Index(lineData, actualToken)
+
+	if plannedPtsIdx > -1 && actualPtsIdx > plannedPtsIdx {
+		plannedPtsIdx += len(plannedToken)
+
+		// Check further... does the point count differ?
+		plannedPtsStr := strings.Trim(lineData[plannedPtsIdx:actualPtsIdx], " \t")
+
+		actualPtsIdx += len(actualToken)
+		actualPtsStr := strings.Trim(lineData[actualPtsIdx:], " \r\n\t")
+
+		plannedPts, err := strconv.Atoi(plannedPtsStr)
+		if err == nil {
+			actualPts, err := strconv.Atoi(actualPtsStr)
+			if err == nil {
+				// Both are ints, check!
+				return plannedPts > actualPts
+			}
+		}
+	}
+
+	return false
+}
+
 func readRTT(tok string) (int64, error) {
 	tok = strings.Trim(tok, "\t ")
 	if len(tok) <= 0 {

api/dataimport/sdfToRSI/scanSDF_test.go

@@ -49,3 +49,16 @@ func Example_scanSDF() {
 	// 276000: science: 'end'
 	//
 }
+
+func Example_isPrematureEnd() {
+	fmt.Printf("end: %v\n", isPrematureEnd("     Planned Points:         2339   Actual:    761"))
+	fmt.Printf("end: %v\n", isPrematureEnd("     Planned Points:         2339   Actual:    761 "))
+	fmt.Printf("end: %v\n", isPrematureEnd("     Planned Points:         2339   Actual:    761\n"))
+	fmt.Printf("end: %v\n", isPrematureEnd("     Planned Points:         2339   Actual:    7611"))
+
+	// Output:
+	// end: true
+	// end: true
+	// end: true
+	// end: false
+}

api/dataimport/sdfToRSI/sdfToRSI.go

@@ -157,17 +157,24 @@ func sdfToRSI(sdfPath string, rtt int64, startLine int, endLine int, outPath str
 		out := strings.Builder{}
 		err = nil
 		if tok == "gv" {
+			sliSpotlistFilenameToken := "Filename token: \"_MCC_SLI_SpotList_BF\""
 			if state != "gv" {
 				// NOTE: we ignore gv until we find startTok on the line - we then expect/read gv lines until they stop coming
-				startTok := "Filename token: \"_MCC_SLI_SpotList_BF\""
-
-				if strings.HasSuffix(lineData, startTok) {
+				if strings.HasSuffix(lineData, sliSpotlistFilenameToken) {
 					state = "gv" // expect gv from now
 					continue
 				} else {
 					// We're not interested in this gv
 					continue
 				}
+			} else {
+				// We're already in state=gv so reading lines for MCC_SLI_SpotList_BF already...
+				// but there's a chance it's the start of another file name. Check for this
+				startTok := "Filename token: \""
+				if (strings.Contains(lineData, startTok)) && !strings.HasSuffix(lineData, sliSpotlistFilenameToken) {
+					state = "" // yep we've ended our state reading, stop here
+					continue
+				}
 			}
 
 			err = processGV(lineNo, line, lineData, sclk, rtt, pmc, &out)

api/dataimport/sdfToRSI/sdfToRSI_test.go

@@ -9,7 +9,7 @@ import (
 	"path/filepath"
 )
 
-func Example_ConvertSDFtoRSI() {
+func Example_sdfToRSI_ConvertSDFtoRSI() {
 	ensureSDFRawExists()
 
 	fmt.Printf("mkdir worked: %v\n", os.MkdirAll("./output", 0777) == nil) // other than 0777 fails in unit tests :(
@@ -25,6 +25,20 @@ func Example_ConvertSDFtoRSI() {
 	// [RSI-208536069.csv HK-208536069.csv RSI-208601602.csv HK-208601602.csv], [208536069 208601602]: <nil>
 }
 
+func Example_sdfToRSI_ConvertSDFtoRSI_EndingPrematurely() {
+	fmt.Printf("mkdir worked: %v\n", os.MkdirAll("./output", 0777) == nil) // other than 0777 fails in unit tests :(
+	wd, err := os.Getwd()
+	fmt.Printf("Getwd: %v\n", err == nil)
+	p := filepath.Join(wd, "output")
+	files, rtts, err := ConvertSDFtoRSIs("./test-data/sdf_raw_premature_end.txt", p)
+	fmt.Printf("%v, %v: %v\n", files, rtts, err)
+
+	// Output:
+	// mkdir worked: true
+	// Getwd: true
+	// [RSI-453.csv HK-453.csv RSI-453.csv HK-453.csv], [453 453]: <nil>
+}
+
 func ensureSDFRawExists() {
 	// If it's not here, unzip it
 	_, err := os.Stat("./test-data/sdf_raw.txt")

api/dataimport/sdfToRSI/test-data/sdf_raw_premature_end.txt

@@ -58,7 +58,13 @@ func logHandlerErrors(err error, log logger.ILogger, w http.ResponseWriter, r *h
 	case errorwithstatus.Error:
 		// We can retrieve the status here and write out a specific
 		// HTTP status code.
-		log.Errorf("Request: %v (%v), Result: status=%v, error=%v", r.URL, r.Method, e.Status(), e)
+		// Memoisation spams this a lot, we can switch this to debug level for those
+		err := fmt.Sprintf("Request: %v (%v), Result: status=%v, error=%v", r.URL, r.Method, e.Status(), e)
+		if strings.HasPrefix(r.URL.String(), "/memoise?key=") && r.Method == "GET" {
+			log.Debugf(err)
+		} else {
+			log.Errorf(err)
+		}
 		http.Error(w, e.Error(), e.Status())
 	default:
 		log.Errorf("Request: %v (%v), Result: status=%v, error=%v", r.URL, r.Method, http.StatusInternalServerError, e)

api/router/helpers.go

@@ -576,7 +576,6 @@ func (c *APIClient) GetQuantColumnAsMap(quantId string, columnName string, detec
 	columnType := protos.Quantification_QT_FLOAT
 	for i, l := range quant.Data.Labels {
 		if l == columnName {
-			fmt.Println(l)
 			// We've found our column!
 			columnIdx = i
 			columnType = quant.Data.Types[i]