terraform/command/views/test.go

374 lines
12 KiB
Go
Raw Normal View History

package views
import (
"encoding/xml"
"fmt"
"io/ioutil"
"sort"
"strings"
"github.com/hashicorp/terraform/command/arguments"
"github.com/hashicorp/terraform/command/format"
"github.com/hashicorp/terraform/internal/moduletest"
"github.com/hashicorp/terraform/internal/terminal"
"github.com/hashicorp/terraform/tfdiags"
"github.com/mitchellh/colorstring"
)
// Test is the view interface for the "terraform test" command.
type Test interface {
// Results presents the given test results.
Results(map[string]*moduletest.Suite) tfdiags.Diagnostics
// Diagnostics is for reporting warnings or errors that occurred with the
// mechanics of running tests. For this command in particular, some
// errors are considered to be test failures rather than mechanism failures,
// and so those will be reported via Results rather than via Diagnostics.
Diagnostics(tfdiags.Diagnostics)
}
// NewTest returns an implementation of Test configured to respect the
// settings described in the given arguments.
func NewTest(base *View, args arguments.TestOutput) Test {
return &testHuman{
streams: base.streams,
showDiagnostics: base.Diagnostics,
colorize: base.colorize,
junitXMLFile: args.JUnitXMLFile,
}
}
type testHuman struct {
// This is the subset of functionality we need from the base view.
streams *terminal.Streams
showDiagnostics func(diags tfdiags.Diagnostics)
colorize *colorstring.Colorize
// If junitXMLFile is not empty then results will be written to
// the given file path in addition to the usual output.
junitXMLFile string
}
func (v *testHuman) Results(results map[string]*moduletest.Suite) tfdiags.Diagnostics {
var diags tfdiags.Diagnostics
// FIXME: Due to how this prototype command evolved concurrently with
// establishing the idea of command views, the handling of JUnit output
// as part of the "human" view rather than as a separate view in its
// own right is a little odd and awkward. We should refactor this
// prior to making "terraform test" a real supported command to make
// it be structured more like the other commands that use the views
// package.
v.humanResults(results)
if v.junitXMLFile != "" {
moreDiags := v.junitXMLResults(results, v.junitXMLFile)
diags = diags.Append(moreDiags)
}
return diags
}
func (v *testHuman) Diagnostics(diags tfdiags.Diagnostics) {
if len(diags) == 0 {
return
}
v.showDiagnostics(diags)
}
func (v *testHuman) humanResults(results map[string]*moduletest.Suite) {
failCount := 0
width := v.streams.Stderr.Columns()
suiteNames := make([]string, 0, len(results))
for suiteName := range results {
suiteNames = append(suiteNames, suiteName)
}
sort.Strings(suiteNames)
for _, suiteName := range suiteNames {
suite := results[suiteName]
componentNames := make([]string, 0, len(suite.Components))
for componentName := range suite.Components {
componentNames = append(componentNames, componentName)
}
for _, componentName := range componentNames {
component := suite.Components[componentName]
assertionNames := make([]string, 0, len(component.Assertions))
for assertionName := range component.Assertions {
assertionNames = append(assertionNames, assertionName)
}
sort.Strings(assertionNames)
for _, assertionName := range assertionNames {
assertion := component.Assertions[assertionName]
fullName := fmt.Sprintf("%s.%s.%s", suiteName, componentName, assertionName)
if strings.HasPrefix(componentName, "(") {
// parenthesis-prefixed components are placeholders that
// the test harness generates to represent problems that
// prevented checking any assertions at all, so we'll
// just hide them and show the suite name.
fullName = suiteName
}
headingExtra := fmt.Sprintf("%s (%s)", fullName, assertion.Description)
switch assertion.Outcome {
case moduletest.Failed:
// Failed means that the assertion was successfully
// excecuted but that the assertion condition didn't hold.
v.eprintRuleHeading("yellow", "Failed", headingExtra)
case moduletest.Error:
// Error means that the system encountered an unexpected
// error when trying to evaluate the assertion.
v.eprintRuleHeading("red", "Error", headingExtra)
default:
// We don't do anything for moduletest.Passed or
// moduletest.Skipped. Perhaps in future we'll offer a
// -verbose option to include information about those.
continue
}
failCount++
if len(assertion.Message) > 0 {
dispMsg := format.WordWrap(assertion.Message, width)
v.streams.Eprintln(dispMsg)
}
if len(assertion.Diagnostics) > 0 {
// We'll do our own writing of the diagnostics in this
// case, rather than using v.Diagnostics, because we
// specifically want all of these diagnostics to go to
// Stderr along with all of the other output we've
// generated.
for _, diag := range assertion.Diagnostics {
diagStr := format.Diagnostic(diag, nil, v.colorize, width)
v.streams.Eprint(diagStr)
}
}
}
}
}
if failCount > 0 {
// If we've printed at least one failure then we'll have printed at
// least one horizontal rule across the terminal, and so we'll balance
// that with another horizontal rule.
if width > 1 {
rule := strings.Repeat("─", width-1)
v.streams.Eprintln(v.colorize.Color("[dark_gray]" + rule))
}
}
if failCount == 0 {
if len(results) > 0 {
// This is not actually an error, but it's convenient if all of our
// result output goes to the same stream for when this is running in
// automation that might be gathering this output via a pipe.
v.streams.Eprint(v.colorize.Color("[bold][green]Success![reset] All of the test assertions passed.\n\n"))
} else {
v.streams.Eprint(v.colorize.Color("[bold][yellow]No tests defined.[reset] This module doesn't have any test suites to run.\n\n"))
}
}
// Try to flush any buffering that might be happening. (This isn't always
// successful, depending on what sort of fd Stderr is connected to.)
v.streams.Stderr.File.Sync()
}
func (v *testHuman) junitXMLResults(results map[string]*moduletest.Suite, filename string) tfdiags.Diagnostics {
var diags tfdiags.Diagnostics
// "JUnit XML" is a file format that has become a de-facto standard for
// test reporting tools but that is not formally specified anywhere, and
// so each producer and consumer implementation unfortunately tends to
// differ in certain ways from others.
// With that in mind, this is a best effort sort of thing aimed at being
// broadly compatible with various consumers, but it's likely that
// some consumers will present these results better than others.
// This implementation is based mainly on the pseudo-specification of the
// format curated here, based on the Jenkins parser implementation:
// https://llg.cubic.org/docs/junit/
// An "Outcome" represents one of the various XML elements allowed inside
// a testcase element to indicate the test outcome.
type Outcome struct {
Message string `xml:"message,omitempty"`
}
// TestCase represents an individual test case as part of a suite. Note
// that a JUnit XML incorporates both the "component" and "assertion"
// levels of our model: we pretend that component is a class name and
// assertion is a method name in order to match with the Java-flavored
// expectations of JUnit XML, which are hopefully close enough to get
// a test result rendering that's useful to humans.
type TestCase struct {
AssertionName string `xml:"name"`
ComponentName string `xml:"classname"`
// These fields represent the different outcomes of a TestCase. Only one
// of these should be populated in each TestCase; this awkward
// structure is just to make this play nicely with encoding/xml's
// expecatations.
Skipped *Outcome `xml:"skipped,omitempty"`
Error *Outcome `xml:"error,omitempty"`
Failure *Outcome `xml:"failure,omitempty"`
Stderr string `xml:"system-out,omitempty"`
}
// TestSuite represents an individual test suite, of potentially many
// in a JUnit XML document.
type TestSuite struct {
Name string `xml:"name"`
TotalCount int `xml:"tests"`
SkippedCount int `xml:"skipped"`
ErrorCount int `xml:"errors"`
FailureCount int `xml:"failures"`
Cases []*TestCase `xml:"testcase"`
}
// TestSuites represents the root element of the XML document.
type TestSuites struct {
XMLName struct{} `xml:"testsuites"`
ErrorCount int `xml:"errors"`
FailureCount int `xml:"failures"`
TotalCount int `xml:"tests"`
Suites []*TestSuite `xml:"testsuite"`
}
xmlSuites := TestSuites{}
suiteNames := make([]string, 0, len(results))
for suiteName := range results {
suiteNames = append(suiteNames, suiteName)
}
sort.Strings(suiteNames)
for _, suiteName := range suiteNames {
suite := results[suiteName]
xmlSuite := &TestSuite{
Name: suiteName,
}
xmlSuites.Suites = append(xmlSuites.Suites, xmlSuite)
componentNames := make([]string, 0, len(suite.Components))
for componentName := range suite.Components {
componentNames = append(componentNames, componentName)
}
for _, componentName := range componentNames {
component := suite.Components[componentName]
assertionNames := make([]string, 0, len(component.Assertions))
for assertionName := range component.Assertions {
assertionNames = append(assertionNames, assertionName)
}
sort.Strings(assertionNames)
for _, assertionName := range assertionNames {
assertion := component.Assertions[assertionName]
xmlSuites.TotalCount++
xmlSuite.TotalCount++
xmlCase := &TestCase{
ComponentName: componentName,
AssertionName: assertionName,
}
xmlSuite.Cases = append(xmlSuite.Cases, xmlCase)
switch assertion.Outcome {
case moduletest.Pending:
// We represent "pending" cases -- cases blocked by
// upstream errors -- as if they were "skipped" in JUnit
// terms, because we didn't actually check them and so
// can't say whether they succeeded or not.
xmlSuite.SkippedCount++
xmlCase.Skipped = &Outcome{
Message: assertion.Message,
}
case moduletest.Failed:
xmlSuites.FailureCount++
xmlSuite.FailureCount++
xmlCase.Failure = &Outcome{
Message: assertion.Message,
}
case moduletest.Error:
xmlSuites.ErrorCount++
xmlSuite.ErrorCount++
xmlCase.Error = &Outcome{
Message: assertion.Message,
}
// We'll also include the diagnostics in the "stderr"
// portion of the output, so they'll hopefully be visible
// in a test log viewer in JUnit-XML-Consuming CI systems.
var buf strings.Builder
for _, diag := range assertion.Diagnostics {
diagStr := format.DiagnosticPlain(diag, nil, 68)
buf.WriteString(diagStr)
}
xmlCase.Stderr = buf.String()
}
}
}
}
xmlOut, err := xml.MarshalIndent(&xmlSuites, "", " ")
if err != nil {
// If marshalling fails then that's a bug in the code above,
// because we should always be producing a value that is
// accepted by encoding/xml.
panic(fmt.Sprintf("invalid values to marshal as JUnit XML: %s", err))
}
err = ioutil.WriteFile(filename, xmlOut, 0644)
if err != nil {
diags = diags.Append(tfdiags.Sourceless(
tfdiags.Error,
"Failed to write JUnit XML file",
fmt.Sprintf(
"Could not create %s to record the test results in JUnit XML format: %s.",
filename,
err,
),
))
}
return diags
}
func (v *testHuman) eprintRuleHeading(color, prefix, extra string) {
const lineCell string = "─"
textLen := len(prefix) + len(": ") + len(extra)
spacingLen := 2
leftLineLen := 3
rightLineLen := 0
width := v.streams.Stderr.Columns()
if (textLen + spacingLen + leftLineLen) < (width - 1) {
// (we allow an extra column at the end because some terminals can't
// print in the final column without wrapping to the next line)
rightLineLen = width - (textLen + spacingLen + leftLineLen) - 1
}
colorCode := "[" + color + "]"
// We'll prepare what we're going to print in memory first, so that we can
// send it all to stderr in one write in case other programs are also
// concurrently trying to write to the terminal for some reason.
var buf strings.Builder
buf.WriteString(v.colorize.Color(colorCode + strings.Repeat(lineCell, leftLineLen)))
buf.WriteByte(' ')
buf.WriteString(v.colorize.Color("[bold]" + colorCode + prefix + ":"))
buf.WriteByte(' ')
buf.WriteString(extra)
if rightLineLen > 0 {
buf.WriteByte(' ')
buf.WriteString(v.colorize.Color(colorCode + strings.Repeat(lineCell, rightLineLen)))
}
v.streams.Eprintln(buf.String())
}