From b802db75d73a14a2458113525246555caecb107c Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Thu, 23 Dec 2021 09:10:59 -0800 Subject: [PATCH] build: Build and run e2etest as part of the release build pipeline This uses the decoupled build and run strategy to run the e2etests so that we can arrange to run the tests against the real release packages produced elsewhere in this workflow, rather than ones generated just in time by the test harness. The modifications to make-archive.sh here make it more consistent with its originally-intended purpose of producing a harness for testing "real" release executables. Our earlier compromise of making it include its own terraform executable came from a desire to use that script as part of manual cross-platform testing when we weren't yet set up to support automation of those tests as we're doing here. That does mean, however, that the terraform-e2etest package content must be combined with content from a terraform release package in order to produce a valid contest for running the tests. We use a single job to cross-compile the test harness for all of the supported platforms, because that build is relatively fast and so not worth the overhead of matrix build, but then use a matrix build to actually run the tests so that we can run them in a worker matching the target platform. We currently have access only to amd64 (x64) runners in GitHub Actions and so for the moment this process is limited only to the subset of our supported platforms which use that architecture. --- .github/workflows/build.yml | 209 +++++++++++++++++- internal/command/e2etest/main_test.go | 25 ++- internal/command/e2etest/make-archive.sh | 13 +- internal/command/e2etest/primary_test.go | 4 +- internal/command/e2etest/provider_dev_test.go | 8 + .../command/e2etest/provider_plugin_test.go | 8 + .../command/e2etest/providers_tamper_test.go | 16 +- .../e2etest/provisioner_plugin_test.go | 8 + 8 files changed, 269 insertions(+), 22 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c2f2092ba..ee7dc5369 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -28,11 +28,22 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 # Need all commits and tags to find a reasonable version number - - name: Decide version number - id: get-product-version + - name: Git Describe + id: git-describe run: | git describe --first-parent - echo "::set-output name=product-version::$(git describe --first-parent)" + echo "::set-output name=raw-version::$(git describe --first-parent)" + - name: Decide version number + id: get-product-version + shell: bash + env: + RAW_VERSION: ${{ steps.git-describe.outputs.raw-version }} + run: | + echo "::set-output name=product-version::${RAW_VERSION#v}" + - name: Report chosen version number + run: | + [ -n "${{steps.get-product-version.outputs.product-version}}" ] + echo "::notice title=Terraform CLI Version::${{ steps.get-product-version.outputs.product-version }}" get-go-version: name: "Determine Go toolchain version" @@ -207,3 +218,195 @@ jobs: tags: | docker.io/hashicorp/${{env.repo}}:${{env.version}} 986891699432.dkr.ecr.us-east-1.amazonaws.com/hashicorp/${{env.repo}}:${{env.version}} + + e2etest-build: + name: Build e2etest for ${{ matrix.goos }}_${{ matrix.goarch }} + needs: ["get-go-version"] + runs-on: ubuntu-latest + strategy: + matrix: + # We build test harnesses only for the v1.0 Compatibility Promises + # supported platforms. Even within that set, we can only run on + # architectures for which we have GitHub Actions runners available, + # which is currently only amd64 (x64). + # TODO: GitHub Actions does support _self-hosted_ arm and arm64 + # runners, so we could potentially run some ourselves to run our + # tests there, but at the time of writing there is no documented + # support for darwin_arm64 (macOS on Apple Silicon). + include: + - {goos: "darwin", goarch: "amd64"} + #- {goos: "darwin", goarch: "arm64"} + - {goos: "windows", goarch: "amd64"} + - {goos: "linux", goarch: "amd64"} + #- {goos: "linux", goarch: "arm"} + #- {goos: "linux", goarch: "arm64"} + fail-fast: false + + env: + build_script: ./internal/command/e2etest/make-archive.sh + + steps: + - uses: actions/checkout@v2 + + - name: Install Go toolchain + uses: actions/setup-go@v2 + with: + go-version: ${{ needs.get-go-version.outputs.go-version }} + + - name: Build test harness package + env: + GOOS: ${{ matrix.goos }} + GOARCH: ${{ matrix.goarch }} + run: | + bash ./internal/command/e2etest/make-archive.sh + + - uses: actions/upload-artifact@v2 + with: + name: terraform-e2etest_${{ matrix.goos }}_${{ matrix.goarch }}.zip + path: internal/command/e2etest/build/terraform-e2etest_${{ matrix.goos }}_${{ matrix.goarch }}.zip + if-no-files-found: error + + e2etest-linux: + name: e2etest for linux_${{ matrix.goarch }} + runs-on: ubuntu-latest + needs: + - get-product-version + - build + - e2etest-build + + strategy: + matrix: + include: + - {goarch: "amd64"} + #- {goarch: "arm64"} + #- {goarch: "arm"} + fail-fast: false + + env: + os: linux + arch: ${{ matrix.goarch }} + version: ${{needs.get-product-version.outputs.product-version}} + + steps: + # NOTE: This intentionally _does not_ check out the source code + # for the commit/tag we're building, because by now we should + # have everything we need in the combination of CLI release package + # and e2etest package for this platform. (This helps ensure that we're + # really testing the release package and not inadvertently testing a + # fresh build from source.) + - name: "Download e2etest package" + uses: actions/download-artifact@v2 + id: e2etestpkg + with: + name: terraform-e2etest_${{ env.os }}_${{ env.arch }}.zip + path: . + - name: "Download Terraform CLI package" + uses: actions/download-artifact@v2 + id: clipkg + with: + name: terraform_${{env.version}}_${{ env.os }}_${{ env.arch }}.zip + path: . + - name: Extract packages + run: | + unzip "./terraform-e2etest_${{ env.os }}_${{ env.arch }}.zip" + unzip "./terraform_${{env.version}}_${{ env.os }}_${{ env.arch }}.zip" + - name: Run E2E Tests + run: | + TF_ACC=1 ./e2etest -test.v + + e2etest-darwin: + name: e2etest for darwin_${{ matrix.goarch }} + runs-on: macos-latest + needs: + - get-product-version + - build-darwin + - e2etest-build + + strategy: + matrix: + include: + - {goarch: "amd64"} + #- {goarch: "arm64"} + fail-fast: false + + env: + os: darwin + arch: ${{ matrix.goarch }} + version: ${{needs.get-product-version.outputs.product-version}} + + steps: + # NOTE: This intentionally _does not_ check out the source code + # for the commit/tag we're building, because by now we should + # have everything we need in the combination of CLI release package + # and e2etest package for this platform. (This helps ensure that we're + # really testing the release package and not inadvertently testing a + # fresh build from source.) + - name: "Download e2etest package" + uses: actions/download-artifact@v2 + id: e2etestpkg + with: + name: terraform-e2etest_${{ env.os }}_${{ env.arch }}.zip + path: . + - name: "Download Terraform CLI package" + uses: actions/download-artifact@v2 + id: clipkg + with: + name: terraform_${{env.version}}_${{ env.os }}_${{ env.arch }}.zip + path: . + - name: Extract packages + run: | + unzip "./terraform-e2etest_${{ env.os }}_${{ env.arch }}.zip" + unzip "./terraform_${{env.version}}_${{ env.os }}_${{ env.arch }}.zip" + - name: Run E2E Tests + run: | + TF_ACC=1 ./e2etest -test.v + + e2etest-windows: + name: e2etest for windows_${{ matrix.goarch }} + runs-on: windows-latest + needs: + - get-product-version + - build + - e2etest-build + + strategy: + matrix: + include: + - {goarch: "amd64"} + fail-fast: false + + env: + os: windows + arch: ${{ matrix.goarch }} + version: ${{needs.get-product-version.outputs.product-version}} + + steps: + # NOTE: This intentionally _does not_ check out the source code + # for the commit/tag we're building, because by now we should + # have everything we need in the combination of CLI release package + # and e2etest package for this platform. (This helps ensure that we're + # really testing the release package and not inadvertently testing a + # fresh build from source.) + - name: "Download e2etest package" + uses: actions/download-artifact@v2 + id: e2etestpkg + with: + name: terraform-e2etest_${{ env.os }}_${{ env.arch }}.zip + path: . + - name: "Download Terraform CLI package" + uses: actions/download-artifact@v2 + id: clipkg + with: + name: terraform_${{env.version}}_${{ env.os }}_${{ env.arch }}.zip + path: . + - name: Extract packages + shell: pwsh + run: | + Expand-Archive -LiteralPath 'terraform-e2etest_${{ env.os }}_${{ env.arch }}.zip' -DestinationPath '.' + Expand-Archive -LiteralPath 'terraform_${{env.version}}_${{ env.os }}_${{ env.arch }}.zip' -DestinationPath '.' + - name: Run E2E Tests + env: + TF_ACC: 1 + shell: cmd + run: | + e2etest.exe -test.v diff --git a/internal/command/e2etest/main_test.go b/internal/command/e2etest/main_test.go index 01e20a982..3c9ba5a5e 100644 --- a/internal/command/e2etest/main_test.go +++ b/internal/command/e2etest/main_test.go @@ -11,6 +11,18 @@ import ( var terraformBin string +// canRunGoBuild is a short-term compromise to account for the fact that we +// have a small number of tests that work by building helper programs using +// "go build" at runtime, but we can't do that in our isolated test mode +// driven by the make-archive.sh script. +// +// FIXME: Rework this a bit so that we build the necessary helper programs +// (test plugins, etc) as part of the initial suite setup, and in the +// make-archive.sh script, so that we can run all of the tests in both +// situations with the tests just using the executable already built for +// them, as we do for terraformBin. +var canRunGoBuild bool + func TestMain(m *testing.M) { teardown := setup() code := m.Run() @@ -21,10 +33,10 @@ func TestMain(m *testing.M) { func setup() func() { if terraformBin != "" { // this is pre-set when we're running in a binary produced from - // the make-archive.sh script, since that builds a ready-to-go - // binary into the archive. However, we do need to turn it into - // an absolute path so that we can find it when we change the - // working directory during tests. + // the make-archive.sh script, since that is for testing an + // executable obtained from a real release package. However, we do + // need to turn it into an absolute path so that we can find it + // when we change the working directory during tests. var err error terraformBin, err = filepath.Abs(terraformBin) if err != nil { @@ -38,6 +50,11 @@ func setup() func() { // Make the executable available for use in tests terraformBin = tmpFilename + // Tests running in the ad-hoc testing mode are allowed to use "go build" + // and similar to produce other test executables. + // (See the comment on this variable's declaration for more information.) + canRunGoBuild = true + return func() { os.Remove(tmpFilename) } diff --git a/internal/command/e2etest/make-archive.sh b/internal/command/e2etest/make-archive.sh index 8fabe2f7a..040633b5d 100755 --- a/internal/command/e2etest/make-archive.sh +++ b/internal/command/e2etest/make-archive.sh @@ -13,9 +13,12 @@ # and then executed as follows: # set TF_ACC=1 # ./e2etest.exe -# Since the test archive includes both the test fixtures and the compiled -# terraform executable along with this test program, the result is -# self-contained and does not require a local Go compiler on the target system. +# +# Because separated e2etest harnesses are intended for testing against "real" +# release executables, the generated archives don't include a copy of +# the Terraform executable. Instead, the caller of the tests must retrieve +# and extract a release package into the working directory before running +# the e2etest executable, so that "e2etest" can find and execute it. set +euo pipefail @@ -33,10 +36,6 @@ mkdir -p "$OUTDIR" # We need the test fixtures available when we run the tests. cp -r testdata "$OUTDIR/testdata" -# Bundle a copy of our binary so the target system doesn't need the go -# compiler installed. -go build -o "$OUTDIR/terraform$GOEXE" github.com/hashicorp/terraform - # Build the test program go test -o "$OUTDIR/e2etest$GOEXE" -c -ldflags "-X github.com/hashicorp/terraform/internal/command/e2etest.terraformBin=./terraform$GOEXE" github.com/hashicorp/terraform/internal/command/e2etest diff --git a/internal/command/e2etest/primary_test.go b/internal/command/e2etest/primary_test.go index 4081d2d4f..de4ad95b2 100644 --- a/internal/command/e2etest/primary_test.go +++ b/internal/command/e2etest/primary_test.go @@ -204,13 +204,13 @@ func TestPrimaryChdirOption(t *testing.T) { } gotOutput := state.RootModule().OutputValues["cwd"] - wantOutputValue := cty.StringVal(tf.Path()) // path.cwd returns the original path, because path.root is how we get the overridden path + wantOutputValue := cty.StringVal(filepath.ToSlash(tf.Path())) // path.cwd returns the original path, because path.root is how we get the overridden path if gotOutput == nil || !wantOutputValue.RawEquals(gotOutput.Value) { t.Errorf("incorrect value for cwd output\ngot: %#v\nwant Value: %#v", gotOutput, wantOutputValue) } gotOutput = state.RootModule().OutputValues["root"] - wantOutputValue = cty.StringVal(tf.Path("subdir")) // path.root is a relative path, but the text fixture uses abspath on it. + wantOutputValue = cty.StringVal(filepath.ToSlash(tf.Path("subdir"))) // path.root is a relative path, but the text fixture uses abspath on it. if gotOutput == nil || !wantOutputValue.RawEquals(gotOutput.Value) { t.Errorf("incorrect value for root output\ngot: %#v\nwant Value: %#v", gotOutput, wantOutputValue) } diff --git a/internal/command/e2etest/provider_dev_test.go b/internal/command/e2etest/provider_dev_test.go index 8c52c2909..1aac10bcd 100644 --- a/internal/command/e2etest/provider_dev_test.go +++ b/internal/command/e2etest/provider_dev_test.go @@ -18,6 +18,14 @@ import ( // we normally do, so they can just overwrite the same local executable // in-place to iterate faster. func TestProviderDevOverrides(t *testing.T) { + if !canRunGoBuild { + // We're running in a separate-build-then-run context, so we can't + // currently execute this test which depends on being able to build + // new executable at runtime. + // + // (See the comment on canRunGoBuild's declaration for more information.) + t.Skip("can't run without building a new provider executable") + } t.Parallel() tf := e2e.NewBinary(terraformBin, "testdata/provider-dev-override") diff --git a/internal/command/e2etest/provider_plugin_test.go b/internal/command/e2etest/provider_plugin_test.go index 585818b1b..c8ac7fe3f 100644 --- a/internal/command/e2etest/provider_plugin_test.go +++ b/internal/command/e2etest/provider_plugin_test.go @@ -13,6 +13,14 @@ import ( // TestProviderProtocols verifies that Terraform can execute provider plugins // with both supported protocol versions. func TestProviderProtocols(t *testing.T) { + if !canRunGoBuild { + // We're running in a separate-build-then-run context, so we can't + // currently execute this test which depends on being able to build + // new executable at runtime. + // + // (See the comment on canRunGoBuild's declaration for more information.) + t.Skip("can't run without building a new provider executable") + } t.Parallel() tf := e2e.NewBinary(terraformBin, "testdata/provider-plugin") diff --git a/internal/command/e2etest/providers_tamper_test.go b/internal/command/e2etest/providers_tamper_test.go index 03026354a..0c285c1f2 100644 --- a/internal/command/e2etest/providers_tamper_test.go +++ b/internal/command/e2etest/providers_tamper_test.go @@ -41,12 +41,16 @@ func TestProviderTampering(t *testing.T) { seedDir := tf.WorkDir() const providerVersion = "3.1.0" // must match the version in the fixture config - pluginDir := ".terraform/providers/registry.terraform.io/hashicorp/null/" + providerVersion + "/" + getproviders.CurrentPlatform.String() - pluginExe := pluginDir + "/terraform-provider-null_v" + providerVersion + "_x5" + pluginDir := filepath.Join(".terraform", "providers", "registry.terraform.io", "hashicorp", "null", providerVersion, getproviders.CurrentPlatform.String()) + pluginExe := filepath.Join(pluginDir, "terraform-provider-null_v"+providerVersion+"_x5") if getproviders.CurrentPlatform.OS == "windows" { pluginExe += ".exe" // ugh } + // filepath.Join here to make sure we get the right path separator + // for whatever OS we're running these tests on. + providerCacheDir := filepath.Join(".terraform", "providers") + t.Run("cache dir totally gone", func(t *testing.T) { tf := e2e.NewBinary(terraformBin, seedDir) defer tf.Close() @@ -61,7 +65,7 @@ func TestProviderTampering(t *testing.T) { if err == nil { t.Fatalf("unexpected plan success\nstdout:\n%s", stdout) } - if want := `registry.terraform.io/hashicorp/null: there is no package for registry.terraform.io/hashicorp/null 3.1.0 cached in .terraform/providers`; !strings.Contains(stderr, want) { + if want := `registry.terraform.io/hashicorp/null: there is no package for registry.terraform.io/hashicorp/null 3.1.0 cached in ` + providerCacheDir; !strings.Contains(stderr, want) { t.Errorf("missing expected error message\nwant substring: %s\ngot:\n%s", want, stderr) } if want := `terraform init`; !strings.Contains(stderr, want) { @@ -128,7 +132,7 @@ func TestProviderTampering(t *testing.T) { if err == nil { t.Fatalf("unexpected plan success\nstdout:\n%s", stdout) } - if want := `registry.terraform.io/hashicorp/null: the cached package for registry.terraform.io/hashicorp/null 3.1.0 (in .terraform/providers) does not match any of the checksums recorded in the dependency lock file`; !strings.Contains(stderr, want) { + if want := `registry.terraform.io/hashicorp/null: the cached package for registry.terraform.io/hashicorp/null 3.1.0 (in ` + providerCacheDir + `) does not match any of the checksums recorded in the dependency lock file`; !strings.Contains(stderr, want) { t.Errorf("missing expected error message\nwant substring: %s\ngot:\n%s", want, stderr) } if want := `terraform init`; !strings.Contains(stderr, want) { @@ -237,7 +241,7 @@ func TestProviderTampering(t *testing.T) { if err == nil { t.Fatalf("unexpected apply success\nstdout:\n%s", stdout) } - if want := `registry.terraform.io/hashicorp/null: there is no package for registry.terraform.io/hashicorp/null 3.1.0 cached in .terraform/providers`; !strings.Contains(stderr, want) { + if want := `registry.terraform.io/hashicorp/null: there is no package for registry.terraform.io/hashicorp/null 3.1.0 cached in ` + providerCacheDir; !strings.Contains(stderr, want) { t.Errorf("missing expected error message\nwant substring: %s\ngot:\n%s", want, stderr) } }) @@ -260,7 +264,7 @@ func TestProviderTampering(t *testing.T) { if err == nil { t.Fatalf("unexpected apply success\nstdout:\n%s", stdout) } - if want := `registry.terraform.io/hashicorp/null: the cached package for registry.terraform.io/hashicorp/null 3.1.0 (in .terraform/providers) does not match any of the checksums recorded in the dependency lock file`; !strings.Contains(stderr, want) { + if want := `registry.terraform.io/hashicorp/null: the cached package for registry.terraform.io/hashicorp/null 3.1.0 (in ` + providerCacheDir + `) does not match any of the checksums recorded in the dependency lock file`; !strings.Contains(stderr, want) { t.Errorf("missing expected error message\nwant substring: %s\ngot:\n%s", want, stderr) } }) diff --git a/internal/command/e2etest/provisioner_plugin_test.go b/internal/command/e2etest/provisioner_plugin_test.go index 4220df574..4ee75f9ce 100644 --- a/internal/command/e2etest/provisioner_plugin_test.go +++ b/internal/command/e2etest/provisioner_plugin_test.go @@ -12,6 +12,14 @@ import ( // TestProvisionerPlugin is a test that terraform can execute a 3rd party // provisioner plugin. func TestProvisionerPlugin(t *testing.T) { + if !canRunGoBuild { + // We're running in a separate-build-then-run context, so we can't + // currently execute this test which depends on being able to build + // new executable at runtime. + // + // (See the comment on canRunGoBuild's declaration for more information.) + t.Skip("can't run without building a new provisioner executable") + } t.Parallel() // This test reaches out to releases.hashicorp.com to download the