From 056f4facddc9eae6bf78c2469b4395534d3c3f82 Mon Sep 17 00:00:00 2001 From: Varsius <42544122+Varsius@users.noreply.github.com> Date: Tue, 4 Feb 2025 13:19:41 +0100 Subject: [PATCH] acpq: distribute minquota overwrites proportionally to available capacity --- .../datamodel/apply_computed_project_quota.go | 43 +++++-- .../apply_computed_project_quota_test.go | 117 ++++++++++++++++++ 2 files changed, 153 insertions(+), 7 deletions(-) diff --git a/internal/datamodel/apply_computed_project_quota.go b/internal/datamodel/apply_computed_project_quota.go index 49d563d7..f5e63aa2 100644 --- a/internal/datamodel/apply_computed_project_quota.go +++ b/internal/datamodel/apply_computed_project_quota.go @@ -364,7 +364,7 @@ func acpqComputeQuotas(stats map[limes.AvailabilityZone]clusterAZAllocationStats } } } - target.EnforceConstraints(constraints, allAZsInOrder, isProjectResourceID, isAZAware) + target.EnforceConstraints(stats, constraints, allAZsInOrder, isProjectResourceID, isAZAware) target.TryFulfillDesired(stats, cfg, allowsQuotaOvercommit) // phase 3: try granting desired_quota @@ -382,7 +382,7 @@ func acpqComputeQuotas(stats map[limes.AvailabilityZone]clusterAZAllocationStats target[az][resourceID].Desired = desiredQuota } } - target.EnforceConstraints(constraints, allAZsInOrder, isProjectResourceID, isAZAware) + target.EnforceConstraints(stats, constraints, allAZsInOrder, isProjectResourceID, isAZAware) target.TryFulfillDesired(stats, cfg, allowsQuotaOvercommit) // phase 4: try granting additional "any" quota until sum of all quotas is ProjectBaseQuota @@ -408,7 +408,7 @@ func acpqComputeQuotas(stats map[limes.AvailabilityZone]clusterAZAllocationStats if resInfo.Topology != liquid.AZSeparatedResourceTopology && !slices.Contains(allAZsInOrder, limes.AvailabilityZoneAny) { allAZsInOrder = append(allAZsInOrder, limes.AvailabilityZoneAny) } - target.EnforceConstraints(constraints, allAZsInOrder, isProjectResourceID, isAZAware) + target.EnforceConstraints(stats, constraints, allAZsInOrder, isProjectResourceID, isAZAware) target.TryFulfillDesired(stats, cfg, allowsQuotaOvercommit) } @@ -417,10 +417,14 @@ func acpqComputeQuotas(stats map[limes.AvailabilityZone]clusterAZAllocationStats // After increasing Desired, but before increasing Allocated, this decreases // Desired in order to fit into project-local quota constraints. -func (target acpqGlobalTarget) EnforceConstraints(constraints map[db.ProjectResourceID]projectLocalQuotaConstraints, allAZs []limes.AvailabilityZone, isProjectResourceID map[db.ProjectResourceID]struct{}, isAZAware bool) { +func (target acpqGlobalTarget) EnforceConstraints(stats map[limes.AvailabilityZone]clusterAZAllocationStats, constraints map[db.ProjectResourceID]projectLocalQuotaConstraints, allAZs []limes.AvailabilityZone, isProjectResourceID map[db.ProjectResourceID]struct{}, isAZAware bool) { + // TODO: quota is distributed for all resources without updating capacity in between + // how to handle the case that the first resource uses all capacity in some AZ? for resourceID, c := range constraints { // raise Allocated as necessary to fulfil minimum quota if c.MinQuota != nil && *c.MinQuota > 0 { + // phase 1: distribute quota proportionally to desire in AZs that have capacity + // if there is sufficient capacity, all quota required additionally will be assigned in this phase totalAllocated := uint64(0) desireScalePerAZ := make(map[limes.AvailabilityZone]uint64) for _, az := range allAZs { @@ -430,14 +434,39 @@ func (target acpqGlobalTarget) EnforceConstraints(constraints map[db.ProjectReso } t := target[az][resourceID] totalAllocated += t.Allocated - desireScalePerAZ[az] = *c.MinQuota * max(1, subtractOrZero(t.Desired, t.Allocated)) + if stats[az].Capacity > 0 { + desireScalePerAZ[az] = *c.MinQuota * max(1, subtractOrZero(t.Desired, t.Allocated)) + } } - extraAllocatedPerAZ := liquidapi.DistributeFairly(subtractOrZero(*c.MinQuota, totalAllocated), desireScalePerAZ) + missingQuota := subtractOrZero(*c.MinQuota, totalAllocated) + extraAllocatedPerAZ := liquidapi.DistributeFairly(missingQuota, desireScalePerAZ) for _, az := range allAZs { if az == limes.AvailabilityZoneAny && isAZAware { continue } - target[az][resourceID].Allocated += extraAllocatedPerAZ[az] + extraAllocated := min(extraAllocatedPerAZ[az], stats[az].Capacity) + target[az][resourceID].Allocated += extraAllocated + missingQuota -= extraAllocated + } + + // phase 2: if not enough quota could be assigned due to capacity constraints, + // iniate second distribution round with distribution proportionally to the available capacity. + // Since min quota should be enforced, more quota than available capacity may be distributed + if missingQuota > 0 { + capacityScalePerAZ := make(map[limes.AvailabilityZone]uint64) + for _, az := range allAZs { + if az == limes.AvailabilityZoneAny && isAZAware { + continue + } + capacityScalePerAZ[az] = *c.MinQuota * stats[az].Capacity // TODO: Possible overflow? + } + extraAllocatedPerAZ := liquidapi.DistributeFairly(missingQuota, capacityScalePerAZ) + for _, az := range allAZs { + if az == limes.AvailabilityZoneAny && isAZAware { + continue + } + target[az][resourceID].Allocated += extraAllocatedPerAZ[az] + } } } diff --git a/internal/datamodel/apply_computed_project_quota_test.go b/internal/datamodel/apply_computed_project_quota_test.go index 79369320..7e312399 100644 --- a/internal/datamodel/apply_computed_project_quota_test.go +++ b/internal/datamodel/apply_computed_project_quota_test.go @@ -715,6 +715,123 @@ func TestMinQuotaConstraintRespectsAZAwareCapacityDistribution(t *testing.T) { 402: {Allocated: 0}, }, }, liquid.ResourceInfo{Topology: liquid.AZAwareResourceTopology}) + + // Multiple AZs with capacity. + // Sufficient total capacity for quota demand. + // Distribute quota w.r.t. available capacity + input = map[limes.AvailabilityZone]clusterAZAllocationStats{ + "az-one": { + Capacity: 0, + ProjectStats: map[db.ProjectResourceID]projectAZAllocationStats{ + 401: {}, + 402: {}, + }, + }, + "az-two": { + Capacity: 1, // Capacity available here as well + ProjectStats: map[db.ProjectResourceID]projectAZAllocationStats{ + 401: {}, + 402: {}, + }, + }, + "az-three": { + Capacity: 10, + ProjectStats: map[db.ProjectResourceID]projectAZAllocationStats{ + 401: {}, + 402: {}, + }, + }, + "any": { + Capacity: 0, + ProjectStats: map[db.ProjectResourceID]projectAZAllocationStats{ + 401: {}, + 402: {}, + }, + }, + } + + constraints = map[db.ProjectResourceID]projectLocalQuotaConstraints{ + 401: {MinQuota: p2u64(3)}, + 402: {MinQuota: p2u64(5)}, + } + + expectACPQResult(t, input, cfg, constraints, acpqGlobalTarget{ + "az-one": { + 401: {Allocated: 0}, + 402: {Allocated: 0}, + }, + "az-two": { + 401: {Allocated: 1}, + 402: {Allocated: 1}, + }, + "az-three": { + 401: {Allocated: 2}, + 402: {Allocated: 4}, + }, + "any": { + 401: {Allocated: 0}, + 402: {Allocated: 0}, + }, + }, liquid.ResourceInfo{Topology: liquid.AZAwareResourceTopology}) + + // Multiple AZs with capacity. + // Total capacity can not fully satisfy quota demand. + // Distribute quota to fulfill min quota constraint ignoring capacity limits. + // Distribute proportional to available capacity. + input = map[limes.AvailabilityZone]clusterAZAllocationStats{ + "az-one": { + Capacity: 0, + ProjectStats: map[db.ProjectResourceID]projectAZAllocationStats{ + 401: {}, + 402: {}, + }, + }, + "az-two": { + Capacity: 1, + ProjectStats: map[db.ProjectResourceID]projectAZAllocationStats{ + 401: {}, + 402: {}, + }, + }, + "az-three": { + Capacity: 2, + ProjectStats: map[db.ProjectResourceID]projectAZAllocationStats{ + 401: {}, + 402: {}, + }, + }, + "any": { + Capacity: 0, + ProjectStats: map[db.ProjectResourceID]projectAZAllocationStats{ + 401: {}, + 402: {}, + }, + }, + } + + constraints = map[db.ProjectResourceID]projectLocalQuotaConstraints{ + 401: {MinQuota: p2u64(3)}, + 402: {MinQuota: p2u64(6)}, + } + + expectACPQResult(t, input, cfg, constraints, acpqGlobalTarget{ + "az-one": { + 401: {Allocated: 0}, + 402: {Allocated: 0}, + }, + "az-two": { + 401: {Allocated: 1}, + 402: {Allocated: 2}, + }, + "az-three": { + 401: {Allocated: 2}, + 402: {Allocated: 4}, + }, + "any": { + 401: {Allocated: 0}, + 402: {Allocated: 0}, + }, + }, liquid.ResourceInfo{Topology: liquid.AZAwareResourceTopology}) } // Shortcut to avoid repetition in projectAZAllocationStats literals.