Skip to content

Commit

Permalink
Merge pull request #54 from pbs/bugfix/batching_deletes
Browse files Browse the repository at this point in the history 
Deleting parameters in batches of 10 due to limitation in AWS API
  • Loading branch information
Yousif Akbar authored Feb 16, 2021
2 parents 878992c + e8e7a80 commit 074a080
Show file tree
Hide file tree
Showing 2 changed files with 283 additions and 19 deletions.
64 changes: 45 additions & 19 deletions internal/gorson/io/io.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -212,38 +212,64 @@ func find(slice []*string, val *string) (int, bool) {
}

// deleteFromParameterStore deletes parameters at a given path from parameter store
func deleteFromParameterStore(parameters []string, path util.ParameterStorePath, client ssmiface.SSMAPI) ([]string, error) {
func deleteFromParameterStore(parameters []string, path util.ParameterStorePath, client ssmiface.SSMAPI) (deletedParams []string, err error) {
deletedParams = []string{}

fullPathParameters := make([]*string, len(parameters))
for idx, parameter := range parameters {
fullPathParameter := fmt.Sprintf("%s%s", path.String(), parameter)
fullPathParameters[idx] = &fullPathParameter
}

deleteParametersInput := ssm.DeleteParametersInput{
Names: fullPathParameters,
var chunkSize int
var chunkedParameters [][]*string

for {
if len(fullPathParameters) == 0 {
break
}

if len(fullPathParameters) < 10 {
chunkSize = len(fullPathParameters)
} else {
chunkSize = 10
}

chunkedParameters = append(chunkedParameters, fullPathParameters[0:chunkSize])
fullPathParameters = fullPathParameters[chunkSize:]
}

output, err := client.DeleteParameters(&deleteParametersInput)
for _, params := range chunkedParameters {
deleteParametersInput := ssm.DeleteParametersInput{
Names: params,
}

output, err := client.DeleteParameters(&deleteParametersInput)

if len(output.DeletedParameters) != len(parameters) {
fmt.Println("Some parameters failed to delete:")
for _, parameter := range fullPathParameters {
idx, found := find(output.DeletedParameters, parameter)
if !found {
fmt.Println(*output.DeletedParameters[idx])
if err != nil {
fmt.Println(err)
}

if len(output.DeletedParameters) != len(params) {
fmt.Println("Some parameters failed to delete:")
for _, parameter := range params {
_, found := find(output.DeletedParameters, parameter)
if !found {
fmt.Println(parameter)
}
}
}
}
if len(output.InvalidParameters) != 0 {
fmt.Println("Some parameters failed to delete due to being invalid:")
for _, invalidParameter := range output.InvalidParameters {
fmt.Println(*invalidParameter)

if len(output.InvalidParameters) != 0 {
fmt.Println("Some parameters failed to delete due to being invalid:")
for _, invalidParameter := range output.InvalidParameters {
fmt.Println(*invalidParameter)
}
}
}

deletedParams := []string{}
for _, deletedParam := range output.DeletedParameters {
deletedParams = append(deletedParams, *deletedParam)
for _, deletedParam := range output.DeletedParameters {
deletedParams = append(deletedParams, *deletedParam)
}
}

return deletedParams, err
Expand Down
238 changes: 238 additions & 0 deletions internal/gorson/io/io_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -381,6 +381,244 @@ func TestDeleteDeltaFromParameterStore(t *testing.T) {
"/path/paramTwo",
},
},
// Twenty things to delete
{
FileParams: map[string]string{},
GetParamsRetVal: mockedGetParametersByPathReturnPair{
Resp: ssm.GetParametersByPathOutput{
Parameters: []*ssm.Parameter{
{
Name: aws.String("paramOne"),
Value: aws.String("valueOne"),
},
{
Name: aws.String("paramTwo"),
Value: aws.String("valueTwo"),
},
{
Name: aws.String("paramThree"),
Value: aws.String("valueThree"),
},
{
Name: aws.String("paramFour"),
Value: aws.String("valueFour"),
},
{
Name: aws.String("paramFive"),
Value: aws.String("valueFive"),
},
{
Name: aws.String("paramSix"),
Value: aws.String("valueSix"),
},
{
Name: aws.String("paramSeven"),
Value: aws.String("valueSeven"),
},
{
Name: aws.String("paramEight"),
Value: aws.String("valueEight"),
},
{
Name: aws.String("paramNine"),
Value: aws.String("valueNine"),
},
{
Name: aws.String("paramTen"),
Value: aws.String("valueTen"),
},
{
Name: aws.String("paramEleven"),
Value: aws.String("valueEleven"),
},
{
Name: aws.String("paramTwelve"),
Value: aws.String("valueTwelve"),
},
{
Name: aws.String("paramThirteen"),
Value: aws.String("valueThirteen"),
},
{
Name: aws.String("paramFourteen"),
Value: aws.String("valueFourteen"),
},
{
Name: aws.String("paramFifteen"),
Value: aws.String("valueFifteen"),
},
{
Name: aws.String("paramSixteen"),
Value: aws.String("valueSixteen"),
},
{
Name: aws.String("paramSeventeen"),
Value: aws.String("valueSeventeen"),
},
{
Name: aws.String("paramEighteen"),
Value: aws.String("valueEighteen"),
},
{
Name: aws.String("paramNineteen"),
Value: aws.String("valueNineteen"),
},
{
Name: aws.String("paramTwenty"),
Value: aws.String("valueTwenty"),
},
},
},
Err: nil,
},
DeleteSuccessful: true,
Expected: []string{
"/path/paramOne",
"/path/paramTwo",
"/path/paramThree",
"/path/paramFour",
"/path/paramFive",
"/path/paramSix",
"/path/paramSeven",
"/path/paramEight",
"/path/paramNine",
"/path/paramTen",
"/path/paramEleven",
"/path/paramTwelve",
"/path/paramThirteen",
"/path/paramFourteen",
"/path/paramFifteen",
"/path/paramSixteen",
"/path/paramSeventeen",
"/path/paramEighteen",
"/path/paramNineteen",
"/path/paramTwenty",
},
},
// Twenty-two things to delete
{
FileParams: map[string]string{},
GetParamsRetVal: mockedGetParametersByPathReturnPair{
Resp: ssm.GetParametersByPathOutput{
Parameters: []*ssm.Parameter{
{
Name: aws.String("paramOne"),
Value: aws.String("valueOne"),
},
{
Name: aws.String("paramTwo"),
Value: aws.String("valueTwo"),
},
{
Name: aws.String("paramThree"),
Value: aws.String("valueThree"),
},
{
Name: aws.String("paramFour"),
Value: aws.String("valueFour"),
},
{
Name: aws.String("paramFive"),
Value: aws.String("valueFive"),
},
{
Name: aws.String("paramSix"),
Value: aws.String("valueSix"),
},
{
Name: aws.String("paramSeven"),
Value: aws.String("valueSeven"),
},
{
Name: aws.String("paramEight"),
Value: aws.String("valueEight"),
},
{
Name: aws.String("paramNine"),
Value: aws.String("valueNine"),
},
{
Name: aws.String("paramTen"),
Value: aws.String("valueTen"),
},
{
Name: aws.String("paramEleven"),
Value: aws.String("valueEleven"),
},
{
Name: aws.String("paramTwelve"),
Value: aws.String("valueTwelve"),
},
{
Name: aws.String("paramThirteen"),
Value: aws.String("valueThirteen"),
},
{
Name: aws.String("paramFourteen"),
Value: aws.String("valueFourteen"),
},
{
Name: aws.String("paramFifteen"),
Value: aws.String("valueFifteen"),
},
{
Name: aws.String("paramSixteen"),
Value: aws.String("valueSixteen"),
},
{
Name: aws.String("paramSeventeen"),
Value: aws.String("valueSeventeen"),
},
{
Name: aws.String("paramEighteen"),
Value: aws.String("valueEighteen"),
},
{
Name: aws.String("paramNineteen"),
Value: aws.String("valueNineteen"),
},
{
Name: aws.String("paramTwenty"),
Value: aws.String("valueTwenty"),
},
{
Name: aws.String("paramTwentyOne"),
Value: aws.String("valueTwentyOne"),
},
{
Name: aws.String("paramTwentyTwo"),
Value: aws.String("valueTwentyTwo"),
},
},
},
Err: nil,
},
DeleteSuccessful: true,
Expected: []string{
"/path/paramOne",
"/path/paramTwo",
"/path/paramThree",
"/path/paramFour",
"/path/paramFive",
"/path/paramSix",
"/path/paramSeven",
"/path/paramEight",
"/path/paramNine",
"/path/paramTen",
"/path/paramEleven",
"/path/paramTwelve",
"/path/paramThirteen",
"/path/paramFourteen",
"/path/paramFifteen",
"/path/paramSixteen",
"/path/paramSeventeen",
"/path/paramEighteen",
"/path/paramNineteen",
"/path/paramTwenty",
"/path/paramTwentyOne",
"/path/paramTwentyTwo",
},
},
}

path := util.NewParameterStorePath("/path/")
Expand Down

0 comments on commit 074a080

Please sign in to comment.