Skip to content

Commit b18a900

Browse files
mahmednabil109mahmednabil109
committed
Add log query endpoint
- add GET `log` endpoint to query logs - support text search, date and log level filtering - add integ test for the log query endpoint Signed-off-by: Mohamed Abokammer <[email protected]>
1 parent c5baa17 commit b18a900

File tree

7 files changed

+586
-37
lines changed

7 files changed

+586
-37
lines changed

‎cmds/admin_server/main.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import (
88
"syscall"
99

1010
"github.com/linuxboot/contest/cmds/admin_server/server"
11-
mongoAdapter "github.com/linuxboot/contest/cmds/admin_server/storage/mongo"
11+
mongoStorage "github.com/linuxboot/contest/cmds/admin_server/storage/mongo"
1212
"github.com/linuxboot/contest/pkg/logging"
1313
"github.com/linuxboot/contest/pkg/xcontext/bundles/logrusctx"
1414
"github.com/linuxboot/contest/pkg/xcontext/logger"
@@ -51,7 +51,7 @@ func main() {
5151
exitWithError(err, 1)
5252
}
5353

54-
storage, err := mongoAdapter.NewMongoStorage(*flagDBURI)
54+
storage, err := mongoStorage.NewMongoStorage(*flagDBURI)
5555
if err != nil {
5656
exitWithError(err, 1)
5757
}

‎cmds/admin_server/server/server.go

Lines changed: 75 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,66 @@ import (
55
"fmt"
66
"net/http"
77
"os"
8+
"time"
89

910
"github.com/gin-gonic/gin"
1011
"github.com/linuxboot/contest/cmds/admin_server/storage"
1112
"github.com/linuxboot/contest/pkg/xcontext"
1213
)
1314

15+
var (
16+
MaxPageSize uint = 100
17+
DefaultPage uint = 0
18+
)
19+
20+
type Query struct {
21+
Text *string `form:"text"`
22+
LogLevel *string `form:"logLevel"`
23+
StartDate *time.Time `form:"startDate" time_format:"2006-01-02T15:04:05.000Z07:00"`
24+
EndDate *time.Time `form:"endDate" time_format:"2006-01-02T15:04:05.000Z07:00"`
25+
PageSize *uint `form:"pageSize"`
26+
Page *uint `form:"page"`
27+
}
28+
29+
// toStorageQurey returns a storage Query and populates the required fields
30+
func (q *Query) ToStorageQuery() storage.Query {
31+
storageQuery := storage.Query{
32+
Page: DefaultPage,
33+
PageSize: MaxPageSize,
34+
}
35+
36+
storageQuery.Text = q.Text
37+
storageQuery.LogLevel = q.LogLevel
38+
storageQuery.StartDate = q.StartDate
39+
storageQuery.EndDate = q.EndDate
40+
41+
if q.Page != nil {
42+
storageQuery.Page = *q.Page
43+
}
44+
45+
if q.PageSize != nil && *q.PageSize < MaxPageSize {
46+
storageQuery.PageSize = *q.PageSize
47+
}
48+
49+
return storageQuery
50+
}
51+
52+
type Log struct {
53+
JobID uint64 `json:"jobID"`
54+
LogData string `json:"logData"`
55+
Date time.Time `json:"date"`
56+
LogLevel string `json:"logLevel"`
57+
}
58+
59+
func (l *Log) ToStorageLog() storage.Log {
60+
return storage.Log{
61+
JobID: l.JobID,
62+
LogData: l.LogData,
63+
Date: l.Date,
64+
LogLevel: l.LogLevel,
65+
}
66+
}
67+
1468
type RouteHandler struct {
1569
storage storage.Storage
1670
}
@@ -22,14 +76,14 @@ func (r *RouteHandler) status(c *gin.Context) {
2276

2377
// addLog inserts a new log entry inside the database
2478
func (r *RouteHandler) addLog(c *gin.Context) {
25-
var log storage.Log
79+
var log Log
2680
if err := c.Bind(&log); err != nil {
27-
c.JSON(http.StatusBadRequest, gin.H{"status": "err", "msg": "bad formatted log"})
81+
c.JSON(http.StatusBadRequest, gin.H{"status": "err", "msg": "badly formatted log"})
2882
fmt.Fprintf(os.Stderr, "Err while binding request body %v", err)
2983
return
3084
}
3185

32-
err := r.storage.StoreLog(log)
86+
err := r.storage.StoreLog(log.ToStorageLog())
3387
if err != nil {
3488
switch {
3589
case errors.Is(err, storage.ErrInsert):
@@ -45,13 +99,31 @@ func (r *RouteHandler) addLog(c *gin.Context) {
4599
c.JSON(http.StatusOK, gin.H{"status": "ok"})
46100
}
47101

102+
// geLogs gets logs form the db based on the filters
103+
func (r *RouteHandler) getLogs(c *gin.Context) {
104+
var query Query
105+
if err := c.BindQuery(&query); err != nil {
106+
c.JSON(http.StatusBadRequest, gin.H{"status": "err", "msg": fmt.Sprintf("bad formatted query %v", err)})
107+
return
108+
}
109+
110+
result, err := r.storage.GetLogs(query.ToStorageQuery())
111+
if err != nil {
112+
c.JSON(http.StatusInternalServerError, gin.H{"status": "err", "msg": "error while getting the logs"})
113+
return
114+
}
115+
116+
c.JSON(http.StatusOK, result)
117+
}
118+
48119
func initRouter(ctx xcontext.Context, rh RouteHandler) *gin.Engine {
49120

50121
r := gin.New()
51122
r.Use(gin.Logger())
52123

53124
r.GET("/status", rh.status)
54125
r.POST("/log", rh.addLog)
126+
r.GET("/log", rh.getLogs)
55127

56128
return r
57129
}

‎cmds/admin_server/storage/mongo/mongo.go

Lines changed: 91 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,12 @@ import (
44
"context"
55
"fmt"
66
"os"
7+
"strings"
78
"time"
89

910
"github.com/linuxboot/contest/cmds/admin_server/storage"
11+
"go.mongodb.org/mongo-driver/bson"
12+
"go.mongodb.org/mongo-driver/bson/primitive"
1013
"go.mongodb.org/mongo-driver/mongo"
1114
"go.mongodb.org/mongo-driver/mongo/options"
1215
)
@@ -53,15 +56,59 @@ func NewMongoStorage(uri string) (storage.Storage, error) {
5356
}, nil
5457
}
5558

56-
func (s *MongoStorage) StoreLog(log storage.Log) error {
57-
logEntry := Log{
58-
log,
59+
func toMongoQuery(query storage.Query) bson.D {
60+
q := bson.D{}
61+
62+
if query.Text != nil {
63+
q = append(q, bson.E{
64+
Key: "logdata",
65+
Value: bson.M{
66+
"$regex": primitive.Regex{Pattern: *query.Text, Options: "ig"},
67+
},
68+
})
69+
}
70+
71+
if query.StartDate != nil && query.EndDate != nil {
72+
q = append(q, bson.E{
73+
Key: "date",
74+
Value: bson.M{
75+
"$gte": primitive.NewDateTimeFromTime(*query.StartDate),
76+
"$lte": primitive.NewDateTimeFromTime(*query.EndDate),
77+
},
78+
})
79+
} else if query.StartDate != nil {
80+
q = append(q, bson.E{
81+
Key: "date",
82+
Value: bson.M{"$gte": primitive.NewDateTimeFromTime(*query.StartDate)},
83+
})
84+
} else if query.EndDate != nil {
85+
q = append(q, bson.E{
86+
Key: "date",
87+
Value: bson.M{"$lte": primitive.NewDateTimeFromTime(*query.EndDate)},
88+
})
5989
}
6090

91+
if query.LogLevel != nil && *query.LogLevel != "" {
92+
levels := strings.Split(*query.LogLevel, ",")
93+
q = append(q,
94+
bson.E{
95+
Key: "loglevel",
96+
Value: bson.M{
97+
"$in": levels,
98+
},
99+
},
100+
)
101+
}
102+
103+
return q
104+
}
105+
106+
func (s *MongoStorage) StoreLog(log storage.Log) error {
107+
61108
ctx, cancel := context.WithTimeout(context.Background(), DefaultConnectionTimeout)
62109
defer cancel()
63110

64-
_, err := s.collection.InsertOne(ctx, logEntry)
111+
_, err := s.collection.InsertOne(ctx, log)
65112
if err != nil {
66113
// for better debugging
67114
fmt.Fprintf(os.Stderr, "Error while inserting into the db: %v", err)
@@ -70,15 +117,48 @@ func (s *MongoStorage) StoreLog(log storage.Log) error {
70117
return nil
71118
}
72119

73-
func (s *MongoStorage) GetLogs(query storage.Query) ([]storage.Log, error) {
74-
// TODO
75-
return nil, nil
120+
func (s *MongoStorage) GetLogs(query storage.Query) (*storage.Result, error) {
121+
122+
q := toMongoQuery(query)
123+
124+
//get the count of the logs
125+
ctx, cancel := context.WithTimeout(context.Background(), DefaultConnectionTimeout)
126+
defer cancel()
127+
count, err := s.collection.CountDocuments(ctx, q)
128+
if err != nil {
129+
fmt.Fprintf(os.Stderr, "Error while performing count query: %v", err)
130+
return nil, storage.ErrQuery
131+
}
132+
133+
opts := options.Find()
134+
opts.SetSkip(int64(int(query.PageSize) * int(query.Page)))
135+
opts.SetLimit(int64(query.PageSize))
136+
137+
ctx, cancel = context.WithTimeout(context.Background(), DefaultConnectionTimeout)
138+
defer cancel()
139+
cur, err := s.collection.Find(ctx, q, opts)
140+
if err != nil {
141+
fmt.Fprintf(os.Stderr, "Error while querying logs from db: %v", err)
142+
return nil, storage.ErrQuery
143+
}
144+
145+
var logs []storage.Log
146+
ctx, cancel = context.WithTimeout(context.Background(), DefaultConnectionTimeout)
147+
defer cancel()
148+
err = cur.All(ctx, &logs)
149+
if err != nil {
150+
fmt.Fprintf(os.Stderr, "Error while reading query result from db: %v", err)
151+
return nil, storage.ErrQuery
152+
}
153+
154+
return &storage.Result{
155+
Logs: logs,
156+
Count: uint64(count),
157+
Page: query.Page,
158+
PageSize: query.PageSize,
159+
}, nil
76160
}
77161

78162
func (s *MongoStorage) Close() error {
79163
return s.dbClient.Disconnect(context.Background())
80164
}
81-
82-
type Log struct {
83-
storage.Log `json:"log"`
84-
}

‎cmds/admin_server/storage/storage.go

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,17 @@ import (
88
var (
99
ErrReadOnlyStorage = errors.New("error read only storage")
1010
ErrInsert = errors.New("error inserting into the db")
11+
ErrConstructQuery = errors.New("error forming db query from api query")
12+
ErrQuery = errors.New("error querying from the database")
13+
)
14+
15+
var (
16+
DefaultTimestampFormat = "2006-01-02T15:04:05.000Z07:00"
1117
)
1218

1319
type Storage interface {
1420
StoreLog(Log) error
15-
GetLogs(Query) ([]Log, error)
21+
GetLogs(Query) (*Result, error)
1622

1723
Close() error
1824
}
@@ -27,9 +33,18 @@ type Log struct {
2733

2834
// Query defines the different options to filter with
2935
type Query struct {
30-
Text string
31-
LogLevel string
32-
StartDate string
33-
EndDate string
34-
Page int
36+
Text *string
37+
LogLevel *string
38+
StartDate *time.Time
39+
EndDate *time.Time
40+
PageSize uint
41+
Page uint
42+
}
43+
44+
//Result defines the expected result returned from the db
45+
type Result struct {
46+
Logs []Log `json:"logs"`
47+
Count uint64 `json:"count"`
48+
Page uint `json:"page"`
49+
PageSize uint `json:"pageSize"`
3550
}

‎tests/integ/admin_server/flags_test.go

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
//go:build integration
2+
// +build integration
3+
4+
package test
5+
6+
import "flag"
7+
8+
var (
9+
flagAdminEndpoint = flag.String("adminServer", "http://adminserver:8000/log", "admin server log push endpoint")
10+
flagMongoEndpoint = flag.String("mongoDBURI", "mongodb://mongostorage:27017", "mongodb URI")
11+
flagOperationTimeout = flag.Uint("operationTimeout", 10, "operation timeout in seconds")
12+
)

0 commit comments

Comments
 (0)