diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c2f2092ba..ee7dc5369 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -28,11 +28,22 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 # Need all commits and tags to find a reasonable version number - - name: Decide version number - id: get-product-version + - name: Git Describe + id: git-describe run: | git describe --first-parent - echo "::set-output name=product-version::$(git describe --first-parent)" + echo "::set-output name=raw-version::$(git describe --first-parent)" + - name: Decide version number + id: get-product-version + shell: bash + env: + RAW_VERSION: ${{ steps.git-describe.outputs.raw-version }} + run: | + echo "::set-output name=product-version::${RAW_VERSION#v}" + - name: Report chosen version number + run: | + [ -n "${{steps.get-product-version.outputs.product-version}}" ] + echo "::notice title=Terraform CLI Version::${{ steps.get-product-version.outputs.product-version }}" get-go-version: name: "Determine Go toolchain version" @@ -207,3 +218,195 @@ jobs: tags: | docker.io/hashicorp/${{env.repo}}:${{env.version}} 986891699432.dkr.ecr.us-east-1.amazonaws.com/hashicorp/${{env.repo}}:${{env.version}} + + e2etest-build: + name: Build e2etest for ${{ matrix.goos }}_${{ matrix.goarch }} + needs: ["get-go-version"] + runs-on: ubuntu-latest + strategy: + matrix: + # We build test harnesses only for the v1.0 Compatibility Promises + # supported platforms. Even within that set, we can only run on + # architectures for which we have GitHub Actions runners available, + # which is currently only amd64 (x64). + # TODO: GitHub Actions does support _self-hosted_ arm and arm64 + # runners, so we could potentially run some ourselves to run our + # tests there, but at the time of writing there is no documented + # support for darwin_arm64 (macOS on Apple Silicon). + include: + - {goos: "darwin", goarch: "amd64"} + #- {goos: "darwin", goarch: "arm64"} + - {goos: "windows", goarch: "amd64"} + - {goos: "linux", goarch: "amd64"} + #- {goos: "linux", goarch: "arm"} + #- {goos: "linux", goarch: "arm64"} + fail-fast: false + + env: + build_script: ./internal/command/e2etest/make-archive.sh + + steps: + - uses: actions/checkout@v2 + + - name: Install Go toolchain + uses: actions/setup-go@v2 + with: + go-version: ${{ needs.get-go-version.outputs.go-version }} + + - name: Build test harness package + env: + GOOS: ${{ matrix.goos }} + GOARCH: ${{ matrix.goarch }} + run: | + bash ./internal/command/e2etest/make-archive.sh + + - uses: actions/upload-artifact@v2 + with: + name: terraform-e2etest_${{ matrix.goos }}_${{ matrix.goarch }}.zip + path: internal/command/e2etest/build/terraform-e2etest_${{ matrix.goos }}_${{ matrix.goarch }}.zip + if-no-files-found: error + + e2etest-linux: + name: e2etest for linux_${{ matrix.goarch }} + runs-on: ubuntu-latest + needs: + - get-product-version + - build + - e2etest-build + + strategy: + matrix: + include: + - {goarch: "amd64"} + #- {goarch: "arm64"} + #- {goarch: "arm"} + fail-fast: false + + env: + os: linux + arch: ${{ matrix.goarch }} + version: ${{needs.get-product-version.outputs.product-version}} + + steps: + # NOTE: This intentionally _does not_ check out the source code + # for the commit/tag we're building, because by now we should + # have everything we need in the combination of CLI release package + # and e2etest package for this platform. (This helps ensure that we're + # really testing the release package and not inadvertently testing a + # fresh build from source.) + - name: "Download e2etest package" + uses: actions/download-artifact@v2 + id: e2etestpkg + with: + name: terraform-e2etest_${{ env.os }}_${{ env.arch }}.zip + path: . + - name: "Download Terraform CLI package" + uses: actions/download-artifact@v2 + id: clipkg + with: + name: terraform_${{env.version}}_${{ env.os }}_${{ env.arch }}.zip + path: . + - name: Extract packages + run: | + unzip "./terraform-e2etest_${{ env.os }}_${{ env.arch }}.zip" + unzip "./terraform_${{env.version}}_${{ env.os }}_${{ env.arch }}.zip" + - name: Run E2E Tests + run: | + TF_ACC=1 ./e2etest -test.v + + e2etest-darwin: + name: e2etest for darwin_${{ matrix.goarch }} + runs-on: macos-latest + needs: + - get-product-version + - build-darwin + - e2etest-build + + strategy: + matrix: + include: + - {goarch: "amd64"} + #- {goarch: "arm64"} + fail-fast: false + + env: + os: darwin + arch: ${{ matrix.goarch }} + version: ${{needs.get-product-version.outputs.product-version}} + + steps: + # NOTE: This intentionally _does not_ check out the source code + # for the commit/tag we're building, because by now we should + # have everything we need in the combination of CLI release package + # and e2etest package for this platform. (This helps ensure that we're + # really testing the release package and not inadvertently testing a + # fresh build from source.) + - name: "Download e2etest package" + uses: actions/download-artifact@v2 + id: e2etestpkg + with: + name: terraform-e2etest_${{ env.os }}_${{ env.arch }}.zip + path: . + - name: "Download Terraform CLI package" + uses: actions/download-artifact@v2 + id: clipkg + with: + name: terraform_${{env.version}}_${{ env.os }}_${{ env.arch }}.zip + path: . + - name: Extract packages + run: | + unzip "./terraform-e2etest_${{ env.os }}_${{ env.arch }}.zip" + unzip "./terraform_${{env.version}}_${{ env.os }}_${{ env.arch }}.zip" + - name: Run E2E Tests + run: | + TF_ACC=1 ./e2etest -test.v + + e2etest-windows: + name: e2etest for windows_${{ matrix.goarch }} + runs-on: windows-latest + needs: + - get-product-version + - build + - e2etest-build + + strategy: + matrix: + include: + - {goarch: "amd64"} + fail-fast: false + + env: + os: windows + arch: ${{ matrix.goarch }} + version: ${{needs.get-product-version.outputs.product-version}} + + steps: + # NOTE: This intentionally _does not_ check out the source code + # for the commit/tag we're building, because by now we should + # have everything we need in the combination of CLI release package + # and e2etest package for this platform. (This helps ensure that we're + # really testing the release package and not inadvertently testing a + # fresh build from source.) + - name: "Download e2etest package" + uses: actions/download-artifact@v2 + id: e2etestpkg + with: + name: terraform-e2etest_${{ env.os }}_${{ env.arch }}.zip + path: . + - name: "Download Terraform CLI package" + uses: actions/download-artifact@v2 + id: clipkg + with: + name: terraform_${{env.version}}_${{ env.os }}_${{ env.arch }}.zip + path: . + - name: Extract packages + shell: pwsh + run: | + Expand-Archive -LiteralPath 'terraform-e2etest_${{ env.os }}_${{ env.arch }}.zip' -DestinationPath '.' + Expand-Archive -LiteralPath 'terraform_${{env.version}}_${{ env.os }}_${{ env.arch }}.zip' -DestinationPath '.' + - name: Run E2E Tests + env: + TF_ACC: 1 + shell: cmd + run: | + e2etest.exe -test.v diff --git a/internal/command/e2etest/main_test.go b/internal/command/e2etest/main_test.go index 01e20a982..3c9ba5a5e 100644 --- a/internal/command/e2etest/main_test.go +++ b/internal/command/e2etest/main_test.go @@ -11,6 +11,18 @@ import ( var terraformBin string +// canRunGoBuild is a short-term compromise to account for the fact that we +// have a small number of tests that work by building helper programs using +// "go build" at runtime, but we can't do that in our isolated test mode +// driven by the make-archive.sh script. +// +// FIXME: Rework this a bit so that we build the necessary helper programs +// (test plugins, etc) as part of the initial suite setup, and in the +// make-archive.sh script, so that we can run all of the tests in both +// situations with the tests just using the executable already built for +// them, as we do for terraformBin. +var canRunGoBuild bool + func TestMain(m *testing.M) { teardown := setup() code := m.Run() @@ -21,10 +33,10 @@ func TestMain(m *testing.M) { func setup() func() { if terraformBin != "" { // this is pre-set when we're running in a binary produced from - // the make-archive.sh script, since that builds a ready-to-go - // binary into the archive. However, we do need to turn it into - // an absolute path so that we can find it when we change the - // working directory during tests. + // the make-archive.sh script, since that is for testing an + // executable obtained from a real release package. However, we do + // need to turn it into an absolute path so that we can find it + // when we change the working directory during tests. var err error terraformBin, err = filepath.Abs(terraformBin) if err != nil { @@ -38,6 +50,11 @@ func setup() func() { // Make the executable available for use in tests terraformBin = tmpFilename + // Tests running in the ad-hoc testing mode are allowed to use "go build" + // and similar to produce other test executables. + // (See the comment on this variable's declaration for more information.) + canRunGoBuild = true + return func() { os.Remove(tmpFilename) } diff --git a/internal/command/e2etest/make-archive.sh b/internal/command/e2etest/make-archive.sh index 8fabe2f7a..040633b5d 100755 --- a/internal/command/e2etest/make-archive.sh +++ b/internal/command/e2etest/make-archive.sh @@ -13,9 +13,12 @@ # and then executed as follows: # set TF_ACC=1 # ./e2etest.exe -# Since the test archive includes both the test fixtures and the compiled -# terraform executable along with this test program, the result is -# self-contained and does not require a local Go compiler on the target system. +# +# Because separated e2etest harnesses are intended for testing against "real" +# release executables, the generated archives don't include a copy of +# the Terraform executable. Instead, the caller of the tests must retrieve +# and extract a release package into the working directory before running +# the e2etest executable, so that "e2etest" can find and execute it. set +euo pipefail @@ -33,10 +36,6 @@ mkdir -p "$OUTDIR" # We need the test fixtures available when we run the tests. cp -r testdata "$OUTDIR/testdata" -# Bundle a copy of our binary so the target system doesn't need the go -# compiler installed. -go build -o "$OUTDIR/terraform$GOEXE" github.com/hashicorp/terraform - # Build the test program go test -o "$OUTDIR/e2etest$GOEXE" -c -ldflags "-X github.com/hashicorp/terraform/internal/command/e2etest.terraformBin=./terraform$GOEXE" github.com/hashicorp/terraform/internal/command/e2etest diff --git a/internal/command/e2etest/primary_test.go b/internal/command/e2etest/primary_test.go index 4081d2d4f..de4ad95b2 100644 --- a/internal/command/e2etest/primary_test.go +++ b/internal/command/e2etest/primary_test.go @@ -204,13 +204,13 @@ func TestPrimaryChdirOption(t *testing.T) { } gotOutput := state.RootModule().OutputValues["cwd"] - wantOutputValue := cty.StringVal(tf.Path()) // path.cwd returns the original path, because path.root is how we get the overridden path + wantOutputValue := cty.StringVal(filepath.ToSlash(tf.Path())) // path.cwd returns the original path, because path.root is how we get the overridden path if gotOutput == nil || !wantOutputValue.RawEquals(gotOutput.Value) { t.Errorf("incorrect value for cwd output\ngot: %#v\nwant Value: %#v", gotOutput, wantOutputValue) } gotOutput = state.RootModule().OutputValues["root"] - wantOutputValue = cty.StringVal(tf.Path("subdir")) // path.root is a relative path, but the text fixture uses abspath on it. + wantOutputValue = cty.StringVal(filepath.ToSlash(tf.Path("subdir"))) // path.root is a relative path, but the text fixture uses abspath on it. if gotOutput == nil || !wantOutputValue.RawEquals(gotOutput.Value) { t.Errorf("incorrect value for root output\ngot: %#v\nwant Value: %#v", gotOutput, wantOutputValue) } diff --git a/internal/command/e2etest/provider_dev_test.go b/internal/command/e2etest/provider_dev_test.go index 8c52c2909..1aac10bcd 100644 --- a/internal/command/e2etest/provider_dev_test.go +++ b/internal/command/e2etest/provider_dev_test.go @@ -18,6 +18,14 @@ import ( // we normally do, so they can just overwrite the same local executable // in-place to iterate faster. func TestProviderDevOverrides(t *testing.T) { + if !canRunGoBuild { + // We're running in a separate-build-then-run context, so we can't + // currently execute this test which depends on being able to build + // new executable at runtime. + // + // (See the comment on canRunGoBuild's declaration for more information.) + t.Skip("can't run without building a new provider executable") + } t.Parallel() tf := e2e.NewBinary(terraformBin, "testdata/provider-dev-override") diff --git a/internal/command/e2etest/provider_plugin_test.go b/internal/command/e2etest/provider_plugin_test.go index 585818b1b..c8ac7fe3f 100644 --- a/internal/command/e2etest/provider_plugin_test.go +++ b/internal/command/e2etest/provider_plugin_test.go @@ -13,6 +13,14 @@ import ( // TestProviderProtocols verifies that Terraform can execute provider plugins // with both supported protocol versions. func TestProviderProtocols(t *testing.T) { + if !canRunGoBuild { + // We're running in a separate-build-then-run context, so we can't + // currently execute this test which depends on being able to build + // new executable at runtime. + // + // (See the comment on canRunGoBuild's declaration for more information.) + t.Skip("can't run without building a new provider executable") + } t.Parallel() tf := e2e.NewBinary(terraformBin, "testdata/provider-plugin") diff --git a/internal/command/e2etest/providers_tamper_test.go b/internal/command/e2etest/providers_tamper_test.go index 03026354a..0c285c1f2 100644 --- a/internal/command/e2etest/providers_tamper_test.go +++ b/internal/command/e2etest/providers_tamper_test.go @@ -41,12 +41,16 @@ func TestProviderTampering(t *testing.T) { seedDir := tf.WorkDir() const providerVersion = "3.1.0" // must match the version in the fixture config - pluginDir := ".terraform/providers/registry.terraform.io/hashicorp/null/" + providerVersion + "/" + getproviders.CurrentPlatform.String() - pluginExe := pluginDir + "/terraform-provider-null_v" + providerVersion + "_x5" + pluginDir := filepath.Join(".terraform", "providers", "registry.terraform.io", "hashicorp", "null", providerVersion, getproviders.CurrentPlatform.String()) + pluginExe := filepath.Join(pluginDir, "terraform-provider-null_v"+providerVersion+"_x5") if getproviders.CurrentPlatform.OS == "windows" { pluginExe += ".exe" // ugh } + // filepath.Join here to make sure we get the right path separator + // for whatever OS we're running these tests on. + providerCacheDir := filepath.Join(".terraform", "providers") + t.Run("cache dir totally gone", func(t *testing.T) { tf := e2e.NewBinary(terraformBin, seedDir) defer tf.Close() @@ -61,7 +65,7 @@ func TestProviderTampering(t *testing.T) { if err == nil { t.Fatalf("unexpected plan success\nstdout:\n%s", stdout) } - if want := `registry.terraform.io/hashicorp/null: there is no package for registry.terraform.io/hashicorp/null 3.1.0 cached in .terraform/providers`; !strings.Contains(stderr, want) { + if want := `registry.terraform.io/hashicorp/null: there is no package for registry.terraform.io/hashicorp/null 3.1.0 cached in ` + providerCacheDir; !strings.Contains(stderr, want) { t.Errorf("missing expected error message\nwant substring: %s\ngot:\n%s", want, stderr) } if want := `terraform init`; !strings.Contains(stderr, want) { @@ -128,7 +132,7 @@ func TestProviderTampering(t *testing.T) { if err == nil { t.Fatalf("unexpected plan success\nstdout:\n%s", stdout) } - if want := `registry.terraform.io/hashicorp/null: the cached package for registry.terraform.io/hashicorp/null 3.1.0 (in .terraform/providers) does not match any of the checksums recorded in the dependency lock file`; !strings.Contains(stderr, want) { + if want := `registry.terraform.io/hashicorp/null: the cached package for registry.terraform.io/hashicorp/null 3.1.0 (in ` + providerCacheDir + `) does not match any of the checksums recorded in the dependency lock file`; !strings.Contains(stderr, want) { t.Errorf("missing expected error message\nwant substring: %s\ngot:\n%s", want, stderr) } if want := `terraform init`; !strings.Contains(stderr, want) { @@ -237,7 +241,7 @@ func TestProviderTampering(t *testing.T) { if err == nil { t.Fatalf("unexpected apply success\nstdout:\n%s", stdout) } - if want := `registry.terraform.io/hashicorp/null: there is no package for registry.terraform.io/hashicorp/null 3.1.0 cached in .terraform/providers`; !strings.Contains(stderr, want) { + if want := `registry.terraform.io/hashicorp/null: there is no package for registry.terraform.io/hashicorp/null 3.1.0 cached in ` + providerCacheDir; !strings.Contains(stderr, want) { t.Errorf("missing expected error message\nwant substring: %s\ngot:\n%s", want, stderr) } }) @@ -260,7 +264,7 @@ func TestProviderTampering(t *testing.T) { if err == nil { t.Fatalf("unexpected apply success\nstdout:\n%s", stdout) } - if want := `registry.terraform.io/hashicorp/null: the cached package for registry.terraform.io/hashicorp/null 3.1.0 (in .terraform/providers) does not match any of the checksums recorded in the dependency lock file`; !strings.Contains(stderr, want) { + if want := `registry.terraform.io/hashicorp/null: the cached package for registry.terraform.io/hashicorp/null 3.1.0 (in ` + providerCacheDir + `) does not match any of the checksums recorded in the dependency lock file`; !strings.Contains(stderr, want) { t.Errorf("missing expected error message\nwant substring: %s\ngot:\n%s", want, stderr) } }) diff --git a/internal/command/e2etest/provisioner_plugin_test.go b/internal/command/e2etest/provisioner_plugin_test.go index 4220df574..4ee75f9ce 100644 --- a/internal/command/e2etest/provisioner_plugin_test.go +++ b/internal/command/e2etest/provisioner_plugin_test.go @@ -12,6 +12,14 @@ import ( // TestProvisionerPlugin is a test that terraform can execute a 3rd party // provisioner plugin. func TestProvisionerPlugin(t *testing.T) { + if !canRunGoBuild { + // We're running in a separate-build-then-run context, so we can't + // currently execute this test which depends on being able to build + // new executable at runtime. + // + // (See the comment on canRunGoBuild's declaration for more information.) + t.Skip("can't run without building a new provisioner executable") + } t.Parallel() // This test reaches out to releases.hashicorp.com to download the