Skip to content

Commit

Permalink
Create new OriginId index, hint, and remove "type" field from indexes
Browse files Browse the repository at this point in the history 
that don't need it.
  • Loading branch information
@lostlevels
lostlevels committed Sep 12, 2024
1 parent fb72260 commit a45308d
Show file tree
Hide file tree
Showing 2 changed files with 54 additions and 25 deletions.
55 changes: 34 additions & 21 deletions data/store/mongo/mongo_datum.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,16 +91,16 @@ func (d *DatumRepository) EnsureIndexes() error {
},
}),
},
{
Keys: bson.D{
{Key: "origin.id", Value: 1},
{Key: "type", Value: 1},
{Key: "deletedTime", Value: -1},
{Key: "_active", Value: 1},
},
Options: options.Index().
SetName("OriginId"),
},
// {
// Keys: bson.D{
// {Key: "origin.id", Value: 1},
// {Key: "type", Value: 1},
// {Key: "deletedTime", Value: -1},
// {Key: "_active", Value: 1},
// },
// Options: options.Index().
// SetName("OriginId"),
// },
{
Keys: bson.D{
{Key: "_userId", Value: 1},
Expand All @@ -124,12 +124,26 @@ func (d *DatumRepository) EnsureIndexes() error {
Options: options.Index().
SetName("UploadId"),
},
// {
// Keys: bson.D{
// {Key: "_userId", Value: 1},
// {Key: "deviceId", Value: 1},
// {Key: "type", Value: 1},
// {Key: "_active", Value: 1},
// {Key: "_deduplicator.hash", Value: 1},
// },
// Options: options.Index().
// SetPartialFilterExpression(bson.D{
// {Key: "_active", Value: true},
// {Key: "_deduplicator.hash", Value: bson.D{{Key: "$exists", Value: true}}},
// {Key: "deviceId", Value: bson.D{{Key: "$exists", Value: true}}},
// }).
// SetName("DeduplicatorHash"),
// },
{
Keys: bson.D{
{Key: "_userId", Value: 1},
{Key: "deviceId", Value: 1},
{Key: "type", Value: 1},
{Key: "_active", Value: 1},
{Key: "_deduplicator.hash", Value: 1},
},
Options: options.Index().
Expand All @@ -138,7 +152,7 @@ func (d *DatumRepository) EnsureIndexes() error {
{Key: "_deduplicator.hash", Value: bson.D{{Key: "$exists", Value: true}}},
{Key: "deviceId", Value: bson.D{{Key: "$exists", Value: true}}},
}).
SetName("DeduplicatorHash"),
SetName("DeduplicatorHashNoType"),
},
})
}
Expand Down Expand Up @@ -168,7 +182,8 @@ func (d *DatumRepository) CreateDataSetData(ctx context.Context, dataSet *upload
datum.SetDataSetID(dataSet.UploadID)
datum.SetCreatedTime(&timestamp)
datum.SetModifiedTime(&timestamp)
datum.SetModifiedTime(&timestamp)
}
for _, datum := range dataSetData {
insertData = append(insertData, mongo.NewInsertOneModel().SetDocument(datum))
}

Expand Down Expand Up @@ -255,7 +270,7 @@ func (d *DatumRepository) ArchiveDataSetData(ctx context.Context, dataSet *uploa
}
opts := options.Update()
if hasOriginID {
opts.SetHint("OriginId")
opts.SetHint("UserIdOriginId")
}
changeInfo, err := d.UpdateMany(ctx, selector, d.ConstructUpdate(set, unset), opts)
if err != nil {
Expand Down Expand Up @@ -299,7 +314,7 @@ func (d *DatumRepository) DeleteDataSetData(ctx context.Context, dataSet *upload
}
opts := options.Update()
if hasOriginID {
opts.SetHint("OriginId")
opts.SetHint("UserIdOriginId")
}
changeInfo, err := d.UpdateMany(ctx, selector, d.ConstructUpdate(set, unset), opts)
if err != nil {
Expand Down Expand Up @@ -332,7 +347,7 @@ func (d *DatumRepository) DestroyDeletedDataSetData(ctx context.Context, dataSet
selector["deletedTime"] = bson.M{"$exists": true}
opts := options.Delete()
if hasOriginID {
opts.SetHint("OriginId")
opts.SetHint("UserIdOriginId")
}
changeInfo, err := d.DeleteMany(ctx, selector, opts)
if err != nil {
Expand Down Expand Up @@ -407,7 +422,7 @@ func (d *DatumRepository) ArchiveDeviceDataUsingHashesFromDataSet(ctx context.Co
"modifiedTime": timestamp,
}
unset := bson.M{}
opts := options.Update().SetHint("DeduplicatorHash")
opts := options.Update().SetHint("DeduplicatorHashNoType")
updateInfo, err = d.UpdateMany(ctx, selector, d.ConstructUpdate(set, unset), opts)
}

Expand Down Expand Up @@ -535,7 +550,6 @@ func validateAndTranslateSelectors(selectors *data.Selectors) (filter bson.M, ha
if selector.ID != nil {
selectorIDs = append(selectorIDs, *selector.ID)
} else if selector.Origin != nil && selector.Origin.ID != nil {
hasOriginID = true
selectorOriginIDs = append(selectorOriginIDs, *selector.Origin.ID)
}
}
Expand Down Expand Up @@ -746,8 +760,7 @@ func (d *DatumRepository) populateLastUpload(ctx context.Context, userId string,

findOptions := options.Find().SetProjection(bson.M{"_id": 0, "modifiedTime": 1, "createdTime": 1})
if lowerTimeBound, err := time.Parse(time.RFC3339, LowerTimeIndexRaw); err == nil && timeMin.After(lowerTimeBound) {
// has blocking sort, but more selective so usually performs better.
findOptions.SetHint("UserIdActiveTypeTimeModifiedTime")
findOptions.SetHint("TestUserIdActiveTypeModifiedTimeTime")
}
findOptions.SetLimit(1)
findOptions.SetSort(bson.D{{Key: "modifiedTime", Value: -1}})
Expand Down
24 changes: 20 additions & 4 deletions data/store/mongo/mongo_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -331,17 +331,33 @@ var _ = Describe("Mongo", func() {
{Key: "time", Value: bson.D{{Key: "$gt", Value: primitive.NewDateTimeFromTime(lowerTimeIndex)}}},
}),
}),
// MatchFields(IgnoreExtras, Fields{
// "Key": Equal(storeStructuredMongoTest.MakeKeySlice("origin.id", "type", "-deletedTime", "_active")),
// "Name": Equal("OriginId"),
// }),
MatchFields(IgnoreExtras, Fields{
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("origin.id", "type", "-deletedTime", "_active")),
"Name": Equal("OriginId"),
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("_userId", "origin.id", "-deletedTime", "_active")),
"Name": Equal("UserIdOriginId"),
"PartialFilterExpression": Equal(bson.D{
{Key: "origin.id", Value: bson.D{{Key: "$exists", Value: true}}},
}),
}),
MatchFields(IgnoreExtras, Fields{
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("uploadId", "type", "-deletedTime", "_active")),
"Name": Equal("UploadId"),
}),
// MatchFields(IgnoreExtras, Fields{
// "Key": Equal(storeStructuredMongoTest.MakeKeySlice("_userId", "deviceId", "type", "_active", "_deduplicator.hash")),
// "Name": Equal("DeduplicatorHash"),
// "PartialFilterExpression": Equal(bson.D{
// {Key: "_active", Value: true},
// {Key: "_deduplicator.hash", Value: bson.D{{Key: "$exists", Value: true}}},
// {Key: "deviceId", Value: bson.D{{Key: "$exists", Value: true}}},
// }),
// }),
MatchFields(IgnoreExtras, Fields{
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("_userId", "deviceId", "type", "_active", "_deduplicator.hash")),
"Name": Equal("DeduplicatorHash"),
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("_userId", "deviceId", "_deduplicator.hash")),
"Name": Equal("DeduplicatorHashNoType"),
"PartialFilterExpression": Equal(bson.D{
{Key: "_active", Value: true},
{Key: "_deduplicator.hash", Value: bson.D{{Key: "$exists", Value: true}}},
Expand Down

0 comments on commit a45308d

Please sign in to comment.