Skip to content

Commit a0ed149

Browse files
lostlevelslostlevels
committed
Create new deviceData indexes, hint, and remove "type" field from indexes
that don't need it.
1 parent fb72260 commit a0ed149

File tree

3 files changed

+19
-57
lines changed

3 files changed

+19
-57
lines changed

‎data/store/mongo/mongo_data_set.go‎

Lines changed: 0 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -39,15 +39,6 @@ func (d *DataSetRepository) EnsureIndexes() error {
3939
Options: options.Index().
4040
SetName("UserIdTypeWeighted_v2"),
4141
},
42-
{
43-
Keys: bson.D{
44-
{Key: "origin.id", Value: 1},
45-
{Key: "deletedTime", Value: -1},
46-
{Key: "_active", Value: 1},
47-
},
48-
Options: options.Index().
49-
SetName("OriginId"),
50-
},
5142
{
5243
Keys: bson.D{
5344
{Key: "uploadId", Value: 1},
@@ -56,7 +47,6 @@ func (d *DataSetRepository) EnsureIndexes() error {
5647
SetUnique(true).
5748
SetName("UniqueUploadId"),
5849
},
59-
// TODO UploadId and UniqueUploadId is always unique in deviceDataSets so remove one or the other.
6050
{
6151
Keys: bson.D{
6252
{Key: "uploadId", Value: 1},
@@ -70,7 +60,6 @@ func (d *DataSetRepository) EnsureIndexes() error {
7060
Keys: bson.D{
7161
{Key: "_userId", Value: 1},
7262
{Key: "client.name", Value: 1},
73-
{Key: "type", Value: 1},
7463
{Key: "createdTime", Value: -1},
7564
},
7665
Options: options.Index().
@@ -83,7 +72,6 @@ func (d *DataSetRepository) EnsureIndexes() error {
8372
Keys: bson.D{
8473
{Key: "_userId", Value: 1},
8574
{Key: "deviceId", Value: 1},
86-
{Key: "type", Value: 1},
8775
{Key: "createdTime", Value: -1},
8876
},
8977
Options: options.Index().
@@ -92,18 +80,6 @@ func (d *DataSetRepository) EnsureIndexes() error {
9280
{Key: "_active", Value: true},
9381
}),
9482
},
95-
{
96-
Keys: bson.D{
97-
{Key: "_userId", Value: 1},
98-
{Key: "_active", Value: 1},
99-
{Key: "client.name", Value: 1},
100-
},
101-
Options: options.Index().
102-
SetPartialFilterExpression(bson.D{
103-
{Key: "client.name", Value: bson.D{{Key: "$exists", Value: true}}},
104-
}).
105-
SetName("ClientName"),
106-
},
10783
})
10884
}
10985

‎data/store/mongo/mongo_datum.go‎

Lines changed: 8 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -91,16 +91,6 @@ func (d *DatumRepository) EnsureIndexes() error {
9191
},
9292
}),
9393
},
94-
{
95-
Keys: bson.D{
96-
{Key: "origin.id", Value: 1},
97-
{Key: "type", Value: 1},
98-
{Key: "deletedTime", Value: -1},
99-
{Key: "_active", Value: 1},
100-
},
101-
Options: options.Index().
102-
SetName("OriginId"),
103-
},
10494
{
10595
Keys: bson.D{
10696
{Key: "_userId", Value: 1},
@@ -128,8 +118,6 @@ func (d *DatumRepository) EnsureIndexes() error {
128118
Keys: bson.D{
129119
{Key: "_userId", Value: 1},
130120
{Key: "deviceId", Value: 1},
131-
{Key: "type", Value: 1},
132-
{Key: "_active", Value: 1},
133121
{Key: "_deduplicator.hash", Value: 1},
134122
},
135123
Options: options.Index().
@@ -138,7 +126,7 @@ func (d *DatumRepository) EnsureIndexes() error {
138126
{Key: "_deduplicator.hash", Value: bson.D{{Key: "$exists", Value: true}}},
139127
{Key: "deviceId", Value: bson.D{{Key: "$exists", Value: true}}},
140128
}).
141-
SetName("DeduplicatorHash"),
129+
SetName("DeduplicatorHashNoType"),
142130
},
143131
})
144132
}
@@ -168,7 +156,8 @@ func (d *DatumRepository) CreateDataSetData(ctx context.Context, dataSet *upload
168156
datum.SetDataSetID(dataSet.UploadID)
169157
datum.SetCreatedTime(&timestamp)
170158
datum.SetModifiedTime(&timestamp)
171-
datum.SetModifiedTime(&timestamp)
159+
}
160+
for _, datum := range dataSetData {
172161
insertData = append(insertData, mongo.NewInsertOneModel().SetDocument(datum))
173162
}
174163

@@ -255,7 +244,7 @@ func (d *DatumRepository) ArchiveDataSetData(ctx context.Context, dataSet *uploa
255244
}
256245
opts := options.Update()
257246
if hasOriginID {
258-
opts.SetHint("OriginId")
247+
opts.SetHint("UserIdOriginId")
259248
}
260249
changeInfo, err := d.UpdateMany(ctx, selector, d.ConstructUpdate(set, unset), opts)
261250
if err != nil {
@@ -299,7 +288,7 @@ func (d *DatumRepository) DeleteDataSetData(ctx context.Context, dataSet *upload
299288
}
300289
opts := options.Update()
301290
if hasOriginID {
302-
opts.SetHint("OriginId")
291+
opts.SetHint("UserIdOriginId")
303292
}
304293
changeInfo, err := d.UpdateMany(ctx, selector, d.ConstructUpdate(set, unset), opts)
305294
if err != nil {
@@ -332,7 +321,7 @@ func (d *DatumRepository) DestroyDeletedDataSetData(ctx context.Context, dataSet
332321
selector["deletedTime"] = bson.M{"$exists": true}
333322
opts := options.Delete()
334323
if hasOriginID {
335-
opts.SetHint("OriginId")
324+
opts.SetHint("UserIdOriginId")
336325
}
337326
changeInfo, err := d.DeleteMany(ctx, selector, opts)
338327
if err != nil {
@@ -407,7 +396,7 @@ func (d *DatumRepository) ArchiveDeviceDataUsingHashesFromDataSet(ctx context.Co
407396
"modifiedTime": timestamp,
408397
}
409398
unset := bson.M{}
410-
opts := options.Update().SetHint("DeduplicatorHash")
399+
opts := options.Update().SetHint("DeduplicatorHashNoType")
411400
updateInfo, err = d.UpdateMany(ctx, selector, d.ConstructUpdate(set, unset), opts)
412401
}
413402

@@ -535,7 +524,6 @@ func validateAndTranslateSelectors(selectors *data.Selectors) (filter bson.M, ha
535524
if selector.ID != nil {
536525
selectorIDs = append(selectorIDs, *selector.ID)
537526
} else if selector.Origin != nil && selector.Origin.ID != nil {
538-
hasOriginID = true
539527
selectorOriginIDs = append(selectorOriginIDs, *selector.Origin.ID)
540528
}
541529
}
@@ -746,8 +734,7 @@ func (d *DatumRepository) populateLastUpload(ctx context.Context, userId string,
746734

747735
findOptions := options.Find().SetProjection(bson.M{"_id": 0, "modifiedTime": 1, "createdTime": 1})
748736
if lowerTimeBound, err := time.Parse(time.RFC3339, LowerTimeIndexRaw); err == nil && timeMin.After(lowerTimeBound) {
749-
// has blocking sort, but more selective so usually performs better.
750-
findOptions.SetHint("UserIdActiveTypeTimeModifiedTime")
737+
findOptions.SetHint("TestUserIdActiveTypeModifiedTimeTime")
751738
}
752739
findOptions.SetLimit(1)
753740
findOptions.SetSort(bson.D{{Key: "modifiedTime", Value: -1}})

‎data/store/mongo/mongo_test.go‎

Lines changed: 11 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -332,16 +332,19 @@ var _ = Describe("Mongo", func() {
332332
}),
333333
}),
334334
MatchFields(IgnoreExtras, Fields{
335-
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("origin.id", "type", "-deletedTime", "_active")),
336-
"Name": Equal("OriginId"),
335+
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("_userId", "origin.id", "-deletedTime", "_active")),
336+
"Name": Equal("UserIdOriginId"),
337+
"PartialFilterExpression": Equal(bson.D{
338+
{Key: "origin.id", Value: bson.D{{Key: "$exists", Value: true}}},
339+
}),
337340
}),
338341
MatchFields(IgnoreExtras, Fields{
339342
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("uploadId", "type", "-deletedTime", "_active")),
340343
"Name": Equal("UploadId"),
341344
}),
342345
MatchFields(IgnoreExtras, Fields{
343-
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("_userId", "deviceId", "type", "_active", "_deduplicator.hash")),
344-
"Name": Equal("DeduplicatorHash"),
346+
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("_userId", "deviceId", "_deduplicator.hash")),
347+
"Name": Equal("DeduplicatorHashNoType"),
345348
"PartialFilterExpression": Equal(bson.D{
346349
{Key: "_active", Value: true},
347350
{Key: "_deduplicator.hash", Value: bson.D{{Key: "$exists", Value: true}}},
@@ -364,15 +367,11 @@ var _ = Describe("Mongo", func() {
364367
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("_id")),
365368
}),
366369
MatchFields(IgnoreExtras, Fields{
367-
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("_userId", "_active", "type", "-time")),
370+
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("_userId", "_active", "-time")),
368371
"Name": Equal("UserIdTypeWeighted_v2"),
369372
}),
370373
MatchFields(IgnoreExtras, Fields{
371-
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("origin.id", "type", "-deletedTime", "_active")),
372-
"Name": Equal("OriginId"),
373-
}),
374-
MatchFields(IgnoreExtras, Fields{
375-
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("uploadId", "type", "-deletedTime", "_active")),
374+
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("uploadId", "-deletedTime", "_active")),
376375
"Name": Equal("UploadId"),
377376
}),
378377
MatchFields(IgnoreExtras, Fields{
@@ -381,14 +380,14 @@ var _ = Describe("Mongo", func() {
381380
"Name": Equal("UniqueUploadId"),
382381
}),
383382
MatchFields(IgnoreExtras, Fields{
384-
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("_userId", "client.name", "type", "-createdTime")),
383+
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("_userId", "client.name", "-createdTime")),
385384
"Name": Equal("ListUserDataSets"),
386385
"PartialFilterExpression": Equal(bson.D{
387386
{Key: "_active", Value: true},
388387
}),
389388
}),
390389
MatchFields(IgnoreExtras, Fields{
391-
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("_userId", "deviceId", "type", "-createdTime")),
390+
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("_userId", "deviceId", "-createdTime")),
392391
"Name": Equal("ListUserDataSetsDeviceId"),
393392
"PartialFilterExpression": Equal(bson.D{
394393
{Key: "_active", Value: true},

0 commit comments

Comments
 (0)