Skip to content

Commit

Permalink
Add details to <failure> element describing which assertion failed
Browse files Browse the repository at this point in the history
  • Loading branch information
SarahFrench committed Jan 15, 2025
1 parent 0f9cb40 commit 1437843
Show file tree
Hide file tree
Showing 3 changed files with 57 additions and 9 deletions.
47 changes: 44 additions & 3 deletions internal/command/junit/junit.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,11 @@ import (
"github.com/hashicorp/terraform/internal/configs/configload"
"github.com/hashicorp/terraform/internal/moduletest"
"github.com/hashicorp/terraform/internal/tfdiags"
"github.com/zclconf/go-cty/cty"
)

var (
failedTestSummary = "Test assertion failed"
)

// TestJUnitXMLFile produces a JUnit XML file at the conclusion of a test
Expand Down Expand Up @@ -206,11 +211,20 @@ func junitXMLTestReport(suite *moduletest.Suite, sources map[string][]byte) ([]b
// why the test was skipped?
}
case moduletest.Fail:
var diagsStr strings.Builder
var failedAssertion tfdiags.Diagnostic
for _, diag := range run.Diagnostics {
// Find the diag resulting from a failed assertion
if diag.Description().Summary == failedTestSummary {
diagsStr.WriteString(format.DiagnosticPlain(diag, sources, 80))
failedAssertion = diag
break
}
}
body := getFailBody(run, failedAssertion)
testCase.Failure = &withMessage{
Message: "Test run failed",
// FIXME: What's a useful thing to report in the body
// here? A summary of the statuses from all of the
// checkable objects in the configuration?
Body: body,
}
case moduletest.Error:
var diagsStr strings.Builder
Expand Down Expand Up @@ -248,6 +262,33 @@ func junitXMLTestReport(suite *moduletest.Suite, sources map[string][]byte) ([]b
return buf.Bytes(), nil
}

func getFailBody(run *moduletest.Run, diag tfdiags.Diagnostic) string {
testCtx := diag.FromExpr()

// Identify which assertion failed out of multiple assertions in a run block
var failedIndex int
var found bool
for i, assertion := range run.Config.CheckRules {
condition, diag := assertion.Condition.Value(testCtx.EvalContext)
if diag.HasErrors() {
return ""
}

if condition.RawEquals(cty.BoolVal(false)) {
failedIndex = i + 1 // index 1
found = true
break
}
}

if found {
return fmt.Sprintf("Test failed on assertion %d of %d", failedIndex, len(run.Config.CheckRules))
}

// Unhandled case
return ""
}

func suiteFilesAsSortedList(files map[string]*moduletest.File) []*moduletest.File {
fileNames := make([]string, len(files))
i := 0
Expand Down
Original file line number Diff line number Diff line change
@@ -1,17 +1,16 @@
<?xml version="1.0" encoding="UTF-8"?><testsuites>
<testsuite name="main.tftest.hcl" tests="2" skipped="0" failures="1" errors="0">
<testcase name="failing_assertion" classname="main.tftest.hcl" time="TIME_REDACTED" timestamp="TIMESTAMP_REDACTED">
<failure message="Test run failed"></failure>
<failure message="Test run failed"><![CDATA[Test failed on assertion 2 of 3]]></failure>
<system-err><![CDATA[
Error: Test assertion failed
on main.tftest.hcl line 3, in run "failing_assertion":
3: condition = local.number < 0
on main.tftest.hcl line 7, in run "failing_assertion":
7: condition = local.number < 0
├────────────────
│ local.number is 10
local variable 'number' has a value greater than zero, so this assertion will
fail
local variable 'number' has a value greater than zero, so assertion 2 will fail
]]></system-err>
</testcase>
<testcase name="passing_assertion" classname="main.tftest.hcl" time="TIME_REDACTED" timestamp="TIMESTAMP_REDACTED"></testcase>
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,15 @@
run "failing_assertion" {
assert {
condition = local.number == 10
error_message = "assertion 1 should pass"
}
assert {
condition = local.number < 0
error_message = "local variable 'number' has a value greater than zero, so this assertion will fail"
error_message = "local variable 'number' has a value greater than zero, so assertion 2 will fail"
}
assert {
condition = local.number == 10
error_message = "assertion 3 should pass"
}
}

Expand Down

0 comments on commit 1437843

Please sign in to comment.