Skip to content

Commit

Permalink
Create new OriginId index and remove type from indexes that don't need
Browse files Browse the repository at this point in the history 
it.
  • Loading branch information
@lostlevels
lostlevels committed Jun 23, 2024
1 parent 164a4bc commit 2973153
Show file tree
Hide file tree
Showing 3 changed files with 71 additions and 30 deletions.
11 changes: 11 additions & 0 deletions data/deduplicator/deduplicator/data_set_delete_origin.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ package deduplicator

import (
"context"
"slices"
"strings"

"github.com/tidepool-org/platform/data"
dataStore "github.com/tidepool-org/platform/data/store"
Expand Down Expand Up @@ -79,6 +81,15 @@ func (d *DataSetDeleteOrigin) AddData(ctx context.Context, repository dataStore.
if err := repository.DeleteDataSetData(ctx, dataSet, selectors); err != nil {
return err
}
// Even though doing an unordered bulkwrite in AddData there is some benefit in sorting this for our origin.id index
slices.SortFunc(dataSetData, func(a, b data.Datum) int {
originA, originB := a.GetOrigin(), b.GetOrigin()
if originA != nil && originA.ID != nil && originB != nil && originB.ID != nil {
return strings.Compare(pointer.ToString(originA.ID), pointer.ToString(originB.ID))
}
timeA, timeB := a.GetTime(), b.GetTime()
return pointer.ToTime(timeA).Compare(pointer.ToTime(timeB))
})
if err := d.Base.AddData(ctx, repository, dataSet, dataSetData); err != nil {
return err
}
Expand Down
54 changes: 36 additions & 18 deletions data/store/mongo/mongo_datum.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,16 +92,16 @@ func (d *DatumRepository) EnsureIndexes() error {
},
}),
},
{
Keys: bson.D{
{Key: "origin.id", Value: 1},
{Key: "type", Value: 1},
{Key: "deletedTime", Value: -1},
{Key: "_active", Value: 1},
},
Options: options.Index().
SetName("OriginId"),
},
// {
// Keys: bson.D{
// {Key: "origin.id", Value: 1},
// {Key: "type", Value: 1},
// {Key: "deletedTime", Value: -1},
// {Key: "_active", Value: 1},
// },
// Options: options.Index().
// SetName("OriginId"),
// },
{
Keys: bson.D{
{Key: "_userId", Value: 1},
Expand All @@ -125,12 +125,26 @@ func (d *DatumRepository) EnsureIndexes() error {
Options: options.Index().
SetName("UploadId"),
},
// {
// Keys: bson.D{
// {Key: "_userId", Value: 1},
// {Key: "deviceId", Value: 1},
// {Key: "type", Value: 1},
// {Key: "_active", Value: 1},
// {Key: "_deduplicator.hash", Value: 1},
// },
// Options: options.Index().
// SetPartialFilterExpression(bson.D{
// {Key: "_active", Value: true},
// {Key: "_deduplicator.hash", Value: bson.D{{Key: "$exists", Value: true}}},
// {Key: "deviceId", Value: bson.D{{Key: "$exists", Value: true}}},
// }).
// SetName("DeduplicatorHash"),
// },
{
Keys: bson.D{
{Key: "_userId", Value: 1},
{Key: "deviceId", Value: 1},
{Key: "type", Value: 1},
{Key: "_active", Value: 1},
{Key: "_deduplicator.hash", Value: 1},
},
Options: options.Index().
Expand All @@ -139,7 +153,7 @@ func (d *DatumRepository) EnsureIndexes() error {
{Key: "_deduplicator.hash", Value: bson.D{{Key: "$exists", Value: true}}},
{Key: "deviceId", Value: bson.D{{Key: "$exists", Value: true}}},
}).
SetName("DeduplicatorHash"),
SetName("DeduplicatorHashNoType"),
},
})
}
Expand Down Expand Up @@ -233,7 +247,7 @@ func (d *DatumRepository) ArchiveDataSetData(ctx context.Context, dataSet *uploa
if err := validateDataSet(dataSet); err != nil {
return err
}
selector, _, err := validateAndTranslateSelectors(selectors)
selector, hasOriginID, err := validateAndTranslateSelectors(selectors)
if err != nil {
return err
}
Expand All @@ -256,7 +270,11 @@ func (d *DatumRepository) ArchiveDataSetData(ctx context.Context, dataSet *uploa
"archivedDatasetId": 1,
"modifiedUserId": 1,
}
changeInfo, err := d.UpdateMany(ctx, selector, d.ConstructUpdate(set, unset))
opts := options.Update()
if hasOriginID {
opts.SetHint("UserIdOriginId")
}
changeInfo, err := d.UpdateMany(ctx, selector, d.ConstructUpdate(set, unset), opts)
if err != nil {
logger.WithError(err).Error("Unable to archive data set data")
return fmt.Errorf("unable to archive data set data: %w", err)
Expand Down Expand Up @@ -299,7 +317,7 @@ func (d *DatumRepository) DeleteDataSetData(ctx context.Context, dataSet *upload
}
opts := options.Update()
if hasOriginID {
opts.SetHint("OriginId")
opts.SetHint("UserIdOriginId")
}
changeInfo, err := d.UpdateMany(ctx, selector, d.ConstructUpdate(set, unset), opts)
if err != nil {
Expand Down Expand Up @@ -332,7 +350,7 @@ func (d *DatumRepository) DestroyDeletedDataSetData(ctx context.Context, dataSet
selector["deletedTime"] = bson.M{"$exists": true}
opts := options.Delete()
if hasOriginID {
opts.SetHint("OriginId")
opts.SetHint("UserIdOriginId")
}
changeInfo, err := d.DeleteMany(ctx, selector, opts)
if err != nil {
Expand Down Expand Up @@ -410,7 +428,7 @@ func (d *DatumRepository) ArchiveDeviceDataUsingHashesFromDataSet(ctx context.Co
"modifiedTime": timestamp,
}
unset := bson.M{}
opts := options.Update().SetHint("DeduplicatorHash")
opts := options.Update().SetHint("DeduplicatorHashNoType")
updateInfo, err = d.UpdateMany(ctx, selector, d.ConstructUpdate(set, unset), opts)
}

Expand Down
36 changes: 24 additions & 12 deletions data/store/mongo/mongo_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -547,9 +547,8 @@ var _ = Describe("Mongo", func() {
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("_id")),
}),
MatchFields(IgnoreExtras, Fields{
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("_userId", "_active", "type", "-time")),
"Background": Equal(true),
"Name": Equal("UserIdTypeWeighted_v2"),
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("_userId", "_active", "type", "-time")),
"Name": Equal("UserIdTypeWeighted_v2"),
}),
MatchFields(IgnoreExtras, Fields{
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("_userId", "_active", "type", "time", "modifiedTime")),
Expand All @@ -565,20 +564,33 @@ var _ = Describe("Mongo", func() {
{Key: "time", Value: bson.D{{Key: "$gt", Value: primitive.NewDateTimeFromTime(lowerTimeIndex)}}},
}),
}),
// MatchFields(IgnoreExtras, Fields{
// "Key": Equal(storeStructuredMongoTest.MakeKeySlice("origin.id", "type", "-deletedTime", "_active")),
// "Name": Equal("OriginId"),
// }),
MatchFields(IgnoreExtras, Fields{
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("origin.id", "type", "-deletedTime", "_active")),
"Background": Equal(true),
"Name": Equal("OriginId"),
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("_userId", "origin.id", "-deletedTime", "_active")),
"Name": Equal("UserIdOriginId"),
"PartialFilterExpression": Equal(bson.D{
{Key: "origin.id", Value: bson.D{{Key: "$exists", Value: true}}},
}),
}),
MatchFields(IgnoreExtras, Fields{
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("uploadId", "type", "-deletedTime", "_active")),
"Background": Equal(true),
"Name": Equal("UploadId"),
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("uploadId", "type", "-deletedTime", "_active")),
"Name": Equal("UploadId"),
}),
// MatchFields(IgnoreExtras, Fields{
// "Key": Equal(storeStructuredMongoTest.MakeKeySlice("_userId", "deviceId", "type", "_active", "_deduplicator.hash")),
// "Name": Equal("DeduplicatorHash"),
// "PartialFilterExpression": Equal(bson.D{
// {Key: "_active", Value: true},
// {Key: "_deduplicator.hash", Value: bson.D{{Key: "$exists", Value: true}}},
// {Key: "deviceId", Value: bson.D{{Key: "$exists", Value: true}}},
// }),
// }),
MatchFields(IgnoreExtras, Fields{
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("_userId", "deviceId", "type", "_active", "_deduplicator.hash")),
"Background": Equal(true),
"Name": Equal("DeduplicatorHash"),
"Key": Equal(storeStructuredMongoTest.MakeKeySlice("_userId", "deviceId", "_deduplicator.hash")),
"Name": Equal("DeduplicatorHashNoType"),
"PartialFilterExpression": Equal(bson.D{
{Key: "_active", Value: true},
{Key: "_deduplicator.hash", Value: bson.D{{Key: "$exists", Value: true}}},
Expand Down

0 comments on commit 2973153

Please sign in to comment.