From b2c0ccdf9610e40e5795d73ffd796d6ee8848cdf Mon Sep 17 00:00:00 2001 From: Martin Atkins Date: Tue, 8 Sep 2020 16:44:55 -0700 Subject: [PATCH] internal/getproviders: Allow PackageMeta to carry acceptable hashes The "acceptable hashes" for a package is a set of hashes that the upstream source considers to be good hashes for checking whether future installs of the same provider version are considered to match this one. Because the acceptable hashes are a package authentication concern and they already need to be known (at least in part) to implement the authenticators, here we add AcceptableHashes as an optional extra method that an authenticator can implement. Because these are hashes chosen by the upstream system, the caller must make its own determination about their trustworthiness. The result of authentication is likely to be an input to that, for example by distrusting hashes produced by an authenticator that succeeds but doesn't report having validated anything. --- .../filesystem_mirror_source_test.go | 4 + internal/getproviders/http_mirror_source.go | 2 +- .../getproviders/http_mirror_source_test.go | 13 ++- internal/getproviders/mock_source.go | 2 +- .../getproviders/package_authentication.go | 108 +++++++++++++++++- .../package_authentication_test.go | 14 +-- internal/getproviders/registry_client.go | 6 +- internal/getproviders/registry_source_test.go | 25 +++- internal/getproviders/types.go | 23 ++++ 9 files changed, 175 insertions(+), 22 deletions(-) diff --git a/internal/getproviders/filesystem_mirror_source_test.go b/internal/getproviders/filesystem_mirror_source_test.go index b526c8bc4..58013a44b 100644 --- a/internal/getproviders/filesystem_mirror_source_test.go +++ b/internal/getproviders/filesystem_mirror_source_test.go @@ -140,6 +140,10 @@ func TestFilesystemMirrorSourcePackageMeta(t *testing.T) { if diff := cmp.Diff(want, got); diff != "" { t.Errorf("incorrect result\n%s", diff) } + + if gotHashes := got.AcceptableHashes(); len(gotHashes) != 0 { + t.Errorf("wrong acceptable hashes\ngot: %#v\nwant: none", gotHashes) + } }) t.Run("unavailable platform", func(t *testing.T) { source := NewFilesystemMirrorSource("testdata/filesystem-mirror") diff --git a/internal/getproviders/http_mirror_source.go b/internal/getproviders/http_mirror_source.go index 578c9ed7a..ada5287ff 100644 --- a/internal/getproviders/http_mirror_source.go +++ b/internal/getproviders/http_mirror_source.go @@ -225,7 +225,7 @@ func (s *HTTPMirrorSource) PackageMeta(provider addrs.Provider, version Version, // A network mirror might not provide any hashes at all, in which case // the package has no source-defined authentication whatsoever. if len(archiveMeta.Hashes) > 0 { - ret.Authentication = NewPackageHashAuthentication(archiveMeta.Hashes) + ret.Authentication = NewPackageHashAuthentication(target, archiveMeta.Hashes) } return ret, nil diff --git a/internal/getproviders/http_mirror_source_test.go b/internal/getproviders/http_mirror_source_test.go index 013672fbd..4e01fc520 100644 --- a/internal/getproviders/http_mirror_source_test.go +++ b/internal/getproviders/http_mirror_source_test.go @@ -124,11 +124,21 @@ func TestHTTPMirrorSource(t *testing.T) { Location: PackageHTTPURL(httpServer.URL + "/terraform.io/test/exists/terraform-provider-test_v1.0.0_tos_m68k.zip"), Authentication: packageHashAuthentication{ RequiredHash: "h1:placeholder-hash", + ValidHashes: []string{"h1:placeholder-hash", "h0:unacceptable-hash"}, + Platform: Platform{"tos", "m68k"}, }, } if diff := cmp.Diff(want, got); diff != "" { t.Errorf("wrong result\n%s", diff) } + + gotHashes := got.AcceptableHashes() + wantHashes := map[Platform][]string{ + tosPlatform: {"h1:placeholder-hash", "h0:unacceptable-hash"}, + } + if diff := cmp.Diff(wantHashes, gotHashes); diff != "" { + t.Errorf("wrong acceptable hashes\n%s", diff) + } }) t.Run("PackageMeta for a version that exists and has no hash", func(t *testing.T) { version := MustParseVersion("1.0.1") @@ -238,7 +248,8 @@ func testHTTPMirrorSourceHandler(resp http.ResponseWriter, req *http.Request) { "tos_m68k": { "url": "terraform-provider-test_v1.0.0_tos_m68k.zip", "hashes": [ - "h1:placeholder-hash" + "h1:placeholder-hash", + "h0:unacceptable-hash" ] } } diff --git a/internal/getproviders/mock_source.go b/internal/getproviders/mock_source.go index 9390ac95b..1565777bf 100644 --- a/internal/getproviders/mock_source.go +++ b/internal/getproviders/mock_source.go @@ -206,7 +206,7 @@ func FakeInstallablePackageMeta(provider addrs.Provider, version Version, protoc // knows what the future holds?) Filename: fmt.Sprintf("terraform-provider-%s_%s_%s.zip", provider.Type, version.String(), target.String()), - Authentication: NewArchiveChecksumAuthentication(checksum), + Authentication: NewArchiveChecksumAuthentication(target, checksum), } return meta, close, nil } diff --git a/internal/getproviders/package_authentication.go b/internal/getproviders/package_authentication.go index a87a624ac..0a0ef2203 100644 --- a/internal/getproviders/package_authentication.go +++ b/internal/getproviders/package_authentication.go @@ -5,9 +5,7 @@ import ( "crypto/sha256" "encoding/hex" "fmt" - "io" "log" - "os" "strings" "golang.org/x/crypto/openpgp" @@ -47,6 +45,32 @@ func (t *PackageAuthenticationResult) String() string { }[t.result] } +// SignedByHashiCorp returns whether the package was authenticated as signed +// by HashiCorp. +func (t *PackageAuthenticationResult) SignedByHashiCorp() bool { + if t == nil { + return false + } + if t.result == officialProvider { + return true + } + + return false +} + +// SignedByAnyParty returns whether the package was authenticated as signed +// by either HashiCorp or by a third-party. +func (t *PackageAuthenticationResult) SignedByAnyParty() bool { + if t == nil { + return false + } + if t.result == officialProvider || t.result == partnerProvider || t.result == communityProvider { + return true + } + + return false +} + // ThirdPartySigned returns whether the package was authenticated as signed by a party // other than HashiCorp. func (t *PackageAuthenticationResult) ThirdPartySigned() bool { @@ -88,6 +112,44 @@ type PackageAuthentication interface { AuthenticatePackage(localLocation PackageLocation) (*PackageAuthenticationResult, error) } +// PackageAuthenticationHashes is an optional interface implemented by +// PackageAuthentication implementations that are able to return a set of +// hashes they would consider valid if a given PackageLocation referred to +// a package that matched that hash string. +// +// This can be used to record a set of acceptable hashes for a particular +// package in a lock file so that future install operations can determine +// whether the package has changed since its initial installation. +type PackageAuthenticationHashes interface { + PackageAuthentication + + // AcceptableHashes returns a set of hash strings that this authenticator + // would accept as valid, grouped by platform. The order of the items + // in each of the slices is not significant, and may contain duplicates + // that are also not significant. + // + // This method's result should only be used to create a "lock" for a + // particular provider if an earlier call to AuthenticatePackage for + // the corresponding package succeeded. A caller might choose to apply + // differing levels of trust for the acceptable hashes depending on + // the authentication result: a "verified checksum" result only checked + // that the downloaded package matched what the source claimed, which + // could be considered to be less trustworthy than a check that includes + // verifying a signature from the origin registry, depending on what the + // hashes are going to be used for. + // + // Hashes are returned as strings with hashing scheme prefixes like "h1:" + // to record which hashing scheme the hash was created with. + // Implementations of PackageAuthenticationHashes may return multiple + // hashes with different schemes, which means that all of them are equally + // acceptable. + // + // Authenticators that don't use hashes as their authentication procedure + // will either not implement this interface or will have an implementation + // that returns an empty result. + AcceptableHashes() map[Platform][]string +} + type packageAuthenticationAll []PackageAuthentication // PackageAuthenticationAll combines several authentications together into a @@ -98,6 +160,13 @@ type packageAuthenticationAll []PackageAuthentication // // The returned result is from the last authentication, so callers should // take care to order the authentications such that the strongest is last. +// +// The returned object also implements the AcceptableHashes method from +// interface PackageAuthenticationHashes, returning the hashes from the +// last of the given checks that indicates at least one acceptable hash, +// or no hashes at all if none of the constituents indicate any. The result +// may therefore be incomplete if there is more than one check that can provide +// hashes and they disagree about which hashes are acceptable. func PackageAuthenticationAll(checks ...PackageAuthentication) PackageAuthentication { return packageAuthenticationAll(checks) } @@ -114,8 +183,28 @@ func (checks packageAuthenticationAll) AuthenticatePackage(localLocation Package return authResult, nil } +func (checks packageAuthenticationAll) AcceptableHashes() map[Platform][]string { + // The elements of checks are expected to be ordered so that the strongest + // one is later in the list, so we'll visit them in reverse order and + // take the first one that implements the interface and returns a non-empty + // result. + for i := len(checks) - 1; i >= 0; i-- { + check, ok := checks[i].(PackageAuthenticationHashes) + if !ok { + continue + } + allHashes := check.AcceptableHashes() + if len(allHashes) > 0 { + return allHashes + } + } + return nil +} + type packageHashAuthentication struct { RequiredHash string + ValidHashes []string + Platform Platform } // NewPackageHashAuthentication returns a PackageAuthentication implementation @@ -126,10 +215,12 @@ type packageHashAuthentication struct { // The PreferredHash function will select which of the given hashes is // considered by Terraform to be the strongest verification, and authentication // succeeds as long as that chosen hash matches. -func NewPackageHashAuthentication(validHashes []string) PackageAuthentication { +func NewPackageHashAuthentication(platform Platform, validHashes []string) PackageAuthentication { requiredHash := PreferredHash(validHashes) return packageHashAuthentication{ RequiredHash: requiredHash, + ValidHashes: validHashes, + Platform: platform, } } @@ -152,7 +243,14 @@ func (a packageHashAuthentication) AuthenticatePackage(localLocation PackageLoca return nil, fmt.Errorf("provider package doesn't match the expected checksum %q", a.RequiredHash) } +func (a packageHashAuthentication) AcceptableHashes() map[Platform][]string { + return map[Platform][]string{ + a.Platform: a.ValidHashes, + } +} + type archiveHashAuthentication struct { + Platform Platform WantSHA256Sum [sha256.Size]byte } @@ -169,8 +267,8 @@ type archiveHashAuthentication struct { // NewPackageHashAuthentication is preferable to use when possible because // it uses the newer hashing scheme (implemented by function Hash) that // can work with both packed and unpacked provider packages. -func NewArchiveChecksumAuthentication(wantSHA256Sum [sha256.Size]byte) PackageAuthentication { - return archiveHashAuthentication{wantSHA256Sum} +func NewArchiveChecksumAuthentication(platform Platform, wantSHA256Sum [sha256.Size]byte) PackageAuthentication { + return archiveHashAuthentication{platform, wantSHA256Sum} } func (a archiveHashAuthentication) AuthenticatePackage(localLocation PackageLocation) (*PackageAuthenticationResult, error) { diff --git a/internal/getproviders/package_authentication_test.go b/internal/getproviders/package_authentication_test.go index 5b734183d..683eda677 100644 --- a/internal/getproviders/package_authentication_test.go +++ b/internal/getproviders/package_authentication_test.go @@ -109,7 +109,7 @@ func TestPackageHashAuthentication_success(t *testing.T) { "h1:qjsREM4DqEWECD43FcPqddZ9oxCG+IaMTxvWPciS05g=", } - auth := NewPackageHashAuthentication(wantHashes) + auth := NewPackageHashAuthentication(Platform{"linux", "amd64"}, wantHashes) result, err := auth.AuthenticatePackage(location) wantResult := PackageAuthenticationResult{result: verifiedChecksum} @@ -143,9 +143,9 @@ func TestPackageHashAuthentication_failure(t *testing.T) { for name, test := range tests { t.Run(name, func(t *testing.T) { - // Empty expected hash, either because we'll error before we + // Invalid expected hash, either because we'll error before we // reach it, or we want to force a checksum mismatch. - auth := NewPackageHashAuthentication([]string{"h1:invalid"}) + auth := NewPackageHashAuthentication(Platform{"linux", "amd64"}, []string{"h1:invalid"}) result, err := auth.AuthenticatePackage(test.location) if result != nil { @@ -172,7 +172,7 @@ func TestArchiveChecksumAuthentication_success(t *testing.T) { 0x5a, 0x79, 0x2a, 0xde, 0x97, 0x11, 0xf5, 0x01, } - auth := NewArchiveChecksumAuthentication(wantSHA256Sum) + auth := NewArchiveChecksumAuthentication(Platform{"linux", "amd64"}, wantSHA256Sum) result, err := auth.AuthenticatePackage(location) wantResult := PackageAuthenticationResult{result: verifiedChecksum} @@ -194,11 +194,11 @@ func TestArchiveChecksumAuthentication_failure(t *testing.T) { }{ "missing file": { PackageLocalArchive("testdata/no-package-here.zip"), - "open testdata/no-package-here.zip: no such file or directory", + "failed to compute checksum for testdata/no-package-here.zip: lstat testdata/no-package-here.zip: no such file or directory", }, "checksum mismatch": { PackageLocalArchive("testdata/filesystem-mirror/registry.terraform.io/hashicorp/null/terraform-provider-null_2.1.0_linux_amd64.zip"), - "archive has incorrect SHA-256 checksum 4fb39849f2e138eb16a18ba0c682635d781cb8c3b25901dd5a792ade9711f501 (expected 0000000000000000000000000000000000000000000000000000000000000000)", + "archive has incorrect checksum zh:4fb39849f2e138eb16a18ba0c682635d781cb8c3b25901dd5a792ade9711f501 (expected zh:0000000000000000000000000000000000000000000000000000000000000000)", }, "invalid location": { PackageLocalDir("testdata/filesystem-mirror/tfe.example.com/AwesomeCorp/happycloud/0.1.0-alpha.2/darwin_amd64"), @@ -210,7 +210,7 @@ func TestArchiveChecksumAuthentication_failure(t *testing.T) { t.Run(name, func(t *testing.T) { // Zero expected checksum, either because we'll error before we // reach it, or we want to force a checksum mismatch - auth := NewArchiveChecksumAuthentication([sha256.Size]byte{0}) + auth := NewArchiveChecksumAuthentication(Platform{"linux", "amd64"}, [sha256.Size]byte{0}) result, err := auth.AuthenticatePackage(test.location) if result != nil { diff --git a/internal/getproviders/registry_client.go b/internal/getproviders/registry_client.go index 54c8a18c0..1b36abf8b 100644 --- a/internal/getproviders/registry_client.go +++ b/internal/getproviders/registry_client.go @@ -275,6 +275,10 @@ func (c *registryClient) PackageMeta(provider addrs.Provider, version Version, t } } + if body.OS != target.OS || body.Arch != target.Arch { + return PackageMeta{}, fmt.Errorf("registry response to request for %s archive has incorrect target %s", target, Platform{body.OS, body.Arch}) + } + downloadURL, err := url.Parse(body.DownloadURL) if err != nil { return PackageMeta{}, fmt.Errorf("registry response includes invalid download URL: %s", err) @@ -351,7 +355,7 @@ func (c *registryClient) PackageMeta(provider addrs.Provider, version Version, t ret.Authentication = PackageAuthenticationAll( NewMatchingChecksumAuthentication(document, body.Filename, checksum), - NewArchiveChecksumAuthentication(checksum), + NewArchiveChecksumAuthentication(ret.TargetPlatform, checksum), NewSignatureAuthentication(document, signature, keys), ) diff --git a/internal/getproviders/registry_source_test.go b/internal/getproviders/registry_source_test.go index bbdc22387..232354528 100644 --- a/internal/getproviders/registry_source_test.go +++ b/internal/getproviders/registry_source_test.go @@ -111,11 +111,12 @@ func TestSourcePackageMeta(t *testing.T) { defer close() tests := []struct { - provider string - version string - os, arch string - want PackageMeta - wantErr string + provider string + version string + os, arch string + want PackageMeta + wantHashes map[Platform][]string + wantErr string }{ // These test cases are relying on behaviors of the fake provider // registry server implemented in client_test.go. @@ -138,7 +139,7 @@ func TestSourcePackageMeta(t *testing.T) { "happycloud_1.2.0.zip", [32]byte{30: 0xf0, 31: 0x0d}, ), - NewArchiveChecksumAuthentication([32]byte{30: 0xf0, 31: 0x0d}), + NewArchiveChecksumAuthentication(Platform{"linux", "amd64"}, [32]byte{30: 0xf0, 31: 0x0d}), NewSignatureAuthentication( []byte("000000000000000000000000000000000000000000000000000000000000f00d happycloud_1.2.0.zip\n"), []byte("GPG signature"), @@ -148,6 +149,11 @@ func TestSourcePackageMeta(t *testing.T) { ), ), }, + map[Platform][]string{ + {"linux", "amd64"}: { + "zh:000000000000000000000000000000000000000000000000000000000000f00d", + }, + }, ``, }, { @@ -155,6 +161,7 @@ func TestSourcePackageMeta(t *testing.T) { "1.2.0", "nonexist", "amd64", PackageMeta{}, + nil, `provider example.com/awesomesauce/happycloud 1.2.0 is not available for nonexist_amd64`, }, { @@ -162,6 +169,7 @@ func TestSourcePackageMeta(t *testing.T) { "1.2.0", "linux", "amd64", PackageMeta{}, + nil, `host not.example.com does not offer a Terraform provider registry`, }, { @@ -169,6 +177,7 @@ func TestSourcePackageMeta(t *testing.T) { "1.2.0", "linux", "amd64", PackageMeta{}, + nil, `host too-new.example.com does not support the provider registry protocol required by this Terraform version, but may be compatible with a different Terraform version`, }, { @@ -176,6 +185,7 @@ func TestSourcePackageMeta(t *testing.T) { "1.2.0", "linux", "amd64", PackageMeta{}, + nil, `could not query provider registry for fails.example.com/awesomesauce/happycloud: the request failed after 2 attempts, please try again later: Get "http://placeholder-origin/fails-immediately/awesomesauce/happycloud/1.2.0/download/linux/amd64": EOF`, }, } @@ -220,6 +230,9 @@ func TestSourcePackageMeta(t *testing.T) { if diff := cmp.Diff(test.want, got, cmpOpts); diff != "" { t.Errorf("wrong result\n%s", diff) } + if diff := cmp.Diff(test.wantHashes, got.AcceptableHashes()); diff != "" { + t.Errorf("wrong AcceptableHashes result\n%s", diff) + } }) } diff --git a/internal/getproviders/types.go b/internal/getproviders/types.go index 2fa0c81af..9c5e6285c 100644 --- a/internal/getproviders/types.go +++ b/internal/getproviders/types.go @@ -244,6 +244,29 @@ func (m PackageMeta) PackedFilePath(baseDir string) string { return PackedFilePathForPackage(baseDir, m.Provider, m.Version, m.TargetPlatform) } +// AcceptableHashes returns a set of hashes (grouped by target platform) that +// could be recorded for comparison to future results for the same provider +// version, to implement a "trust on first use" scheme. +// +// Callers of this method should typically also verify the package using the +// object in the Authentication field, and consider how much trust to give +// the result of this method depending on the authentication result: an +// unauthenticated result or one that only verified a checksum could be +// considered less trustworthy than one that checked the package against +// a signature provided by the origin registry. +// +// The AcceptableHashes result is actually provided by the object in the +// Authentication field. AcceptableHashes therefore returns an empty result +// for a PackageMeta that has no authentication object, or has one that does +// not make use of hashes. +func (m PackageMeta) AcceptableHashes() map[Platform][]string { + auth, ok := m.Authentication.(PackageAuthenticationHashes) + if !ok { + return nil + } + return auth.AcceptableHashes() +} + // PackageLocation represents a location where a provider distribution package // can be obtained. A value of this type contains one of the following // concrete types: PackageLocalArchive, PackageLocalDir, or PackageHTTPURL.