diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f28c2b53f..386b37e90 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -57,9 +57,7 @@ jobs: export GITHUB_TOKEN="${RELEEN_GITHUB_TOKEN}" set -x - go test --run '(using_kiln|baking_a_tile|generating_release_notes|updating_)' \ - -v --timeout 24h --tags acceptance \ - github.com/pivotal-cf/kiln/internal/acceptance/workflows + go test -v --timeout 24h --tags acceptance github.com/pivotal-cf/kiln/internal/acceptance/workflows git reset --hard HEAD diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d9ba07886..82b970299 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -56,6 +56,4 @@ jobs: export GITHUB_TOKEN="${RELEEN_GITHUB_TOKEN}" set -x - go test --run '(using_kiln|baking_a_tile|generating_release_notes|updating_)' \ - -v --timeout 15m --tags acceptance \ - github.com/pivotal-cf/kiln/internal/acceptance/workflows + go test -v --timeout 15m --tags acceptance github.com/pivotal-cf/kiln/internal/acceptance/workflows diff --git a/go.mod b/go.mod index be91e7800..a9da223ad 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,6 @@ require ( github.com/aws/aws-sdk-go v1.44.325 github.com/blang/semver/v4 v4.0.0 github.com/cloudfoundry/bosh-cli v6.4.1+incompatible - github.com/cloudfoundry/bosh-utils v0.0.384 github.com/cppforlife/go-patch v0.2.0 github.com/crhntr/bijection v0.0.0-20230628013949-46b5c800bc70 github.com/crhntr/yamlutil v0.0.0-20230524174859-4e4388958875 @@ -50,13 +49,11 @@ require ( github.com/Masterminds/semver v1.5.0 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 // indirect - github.com/VividCortex/ewma v1.2.0 // indirect github.com/acomagu/bufpipe v1.0.4 // indirect github.com/bmatcuk/doublestar v1.3.4 // indirect github.com/charlievieth/fs v0.0.3 // indirect - github.com/cheggaaa/pb/v3 v3.1.4 // indirect github.com/cloudflare/circl v1.3.3 // indirect - github.com/cloudfoundry-community/go-uaa v0.3.1 // indirect + github.com/cloudfoundry/bosh-utils v0.0.384 // indirect github.com/cloudfoundry/go-socks5 v0.0.0-20180221174514-54f73bdb8a8e // indirect github.com/cloudfoundry/socks5-proxy v0.2.96 // indirect github.com/containerd/containerd v1.7.3 // indirect diff --git a/go.sum b/go.sum index 33a343b5f..4258751d1 100644 --- a/go.sum +++ b/go.sum @@ -92,8 +92,6 @@ github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95/go.mod h1:EjA github.com/StackExchange/wmi v0.0.0-20180725035823-b12b22c5341f/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= -github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= -github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/acomagu/bufpipe v1.0.4 h1:e3H4WUzM3npvo5uv95QuJM3cQspFNtFBzvJ2oNjKIDQ= github.com/acomagu/bufpipe v1.0.4/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= @@ -122,15 +120,12 @@ github.com/charlievieth/fs v0.0.3/go.mod h1:hD4sRzto1Hw8zCua76tNVKZxaeZZr1RiKftj github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= github.com/cheggaaa/pb v1.0.28/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s= github.com/cheggaaa/pb/v3 v3.0.5/go.mod h1:X1L61/+36nz9bjIsrDU52qHKOQukUQe2Ge+YvGuquCw= -github.com/cheggaaa/pb/v3 v3.1.4 h1:DN8j4TVVdKu3WxVwcRKu0sG00IIU6FewoABZzXbRQeo= -github.com/cheggaaa/pb/v3 v3.1.4/go.mod h1:6wVjILNBaXMs8c21qRiaUM8BR82erfgau1DQ4iUXmSA= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs= github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= -github.com/cloudfoundry-community/go-uaa v0.3.1 h1:qmiTqE8Be3zJoL2wcwddVnbRNMDhOrURZbWOUQ2ibJQ= github.com/cloudfoundry-community/go-uaa v0.3.1/go.mod h1:m3JOryy7cx+7QLxuwB+bXuAx5AUJ3W9XhRUBu6Eih0Q= github.com/cloudfoundry/bosh-agent v2.340.0+incompatible/go.mod h1:7UvVn5vc/d6icLrBx6GhBlpSMwe2+x1C2A7x4TbPhiU= github.com/cloudfoundry/bosh-cli v6.4.1+incompatible h1:n5/+NIF9QxvGINOrjh6DmO+GTen78MoCj5+LU9L8bR4= diff --git a/internal/acceptance/README.md b/internal/acceptance/README.md index 577315386..be0bd3266 100644 --- a/internal/acceptance/README.md +++ b/internal/acceptance/README.md @@ -30,43 +30,6 @@ These are written in Go and use [godog](https://github.com/cucumber/godog) (a Cu go test -v --tags acceptance --timeout=1h github.com/pivotal-cf/kiln/internal/acceptance/workflows ``` -#### ⚠️ The `caching_compiled_releases` Test Does Not Run in CI ⚠️ -This test requires SSH access to an Ops Manager. -Deploying an Ops Manager for an open source repo is not secure. -This one test should be run against a non-production Ops Manager or from a VMware developer's machine on the internal network using a Toolsmiths environment. - -
-Instructions to test with a Toolsmiths deployed Ops Manager -
- -Ensure you have the [Smith CLI](https://github.com/pivotal/smith) properly installed and you are logged in. - -PPE team members may execute the AWS environment setup expressions in the script. -Non-ppe-team-members may ask us for temporary credentials [generated here](https://console.aws.amazon.com/iam/home#/users/kiln_acceptance_tests?section=security_credentials). -Note the credential created on 2022-08-08 (id ending in "QOV") should not be deleted. It is stored in vault. - -```bash -## START Setup -eval "$(smith claim -p us_2_12)" -eval "$(smith bosh)" -eval "$(smith om)" -export OM_PRIVATE_KEY="$(cat $(echo "${BOSH_ALL_PROXY}" | awk -F= '{print $2}'))" - -# AWS environment setup -export AWS_ACCESS_KEY_ID="$(vault read --field=aws_access_key_id runway_concourse/ppe-ci/kiln-acceptance-tests-s3)" -export AWS_SECRET_ACCESS_KEY="$(vault read --field=aws_secret_access_key runway_concourse/ppe-ci/kiln-acceptance-tests-s3)" - -export GITHUB_TOKEN="$(gh auth status --show-token 2>&1 | grep Token | awk '{print $NF}')" - -# optional -export CGO_ENABLED=0 -## END Setup - -# Run the caching_compiled_releases test -go test --run caching_compiled_releases -v --tags acceptance --timeout=1h github.com/pivotal-cf/kiln/internal/acceptance/workflows -``` -
- ## Contributing Please follow existing style and make sure the acceptance unit tests both in the workflows and in the scenario package pass. diff --git a/internal/acceptance/workflows/acceptance_test.go b/internal/acceptance/workflows/acceptance_test.go index 70c663ed8..a23c94fbe 100644 --- a/internal/acceptance/workflows/acceptance_test.go +++ b/internal/acceptance/workflows/acceptance_test.go @@ -36,12 +36,6 @@ func Test_baking_a_tile(t *testing.T) { setupAndRunFeatureTest(t) } -func Test_caching_compiled_releases(t *testing.T) { - setupAndRunFeatureTest(t, - scenario.InitializeCacheCompiledReleases, - ) -} - func Test_generating_release_notes(t *testing.T) { setupAndRunFeatureTest(t, scenario.InitializeGitHub) } diff --git a/internal/acceptance/workflows/caching_compiled_releases.feature b/internal/acceptance/workflows/caching_compiled_releases.feature deleted file mode 100644 index 606041825..000000000 --- a/internal/acceptance/workflows/caching_compiled_releases.feature +++ /dev/null @@ -1,35 +0,0 @@ -Feature: As a robot, I want to cache compiled releases - Scenario: it stores compiled releases in an S3 bucket - Given I have a "hello-tile" repository checked out at v0.1.2 - And the environment variable "GITHUB_TOKEN" is set - And the environment variable "OM_USERNAME" is set - And the environment variable "OM_PASSWORD" is set - And the environment variable "OM_TARGET" is set - And the environment variable "OM_PRIVATE_KEY" is set - And the environment variable "BOSH_ALL_PROXY" is set - And the environment variable "AWS_ACCESS_KEY_ID" is set - And the environment variable "AWS_SECRET_ACCESS_KEY" is set - And I remove all the objects in the bucket "hello-tile-releases" - And I invoke kiln - | fetch | - | --variable=github_token="${GITHUB_TOKEN}" | - And I invoke kiln - | bake | - | --version=0.1.2 | - And I upload, configure, and apply the tile - And I add a compiled s3 release-source "hello-tile-releases" to the Kilnfile - And I set the stemcell version in the lock to match the one used for the tile - When I invoke kiln - | cache-compiled-releases | - | --upload-target-id=hello-tile-releases | - | --name=hello | - | --variable=github_token="${GITHUB_TOKEN}" | - And the repository has no fetched releases - And I invoke kiln - | fetch | - | --variable=github_token="${GITHUB_TOKEN}" | - And I invoke kiln - | bake | - | --version=0.1.2 | - Then a Tile is created - And the Tile only contains compiled releases diff --git a/internal/acceptance/workflows/scenario/initialize.go b/internal/acceptance/workflows/scenario/initialize.go index 1f5007f6c..2bc1cd08f 100644 --- a/internal/acceptance/workflows/scenario/initialize.go +++ b/internal/acceptance/workflows/scenario/initialize.go @@ -27,57 +27,6 @@ func initializeAWS(ctx scenarioContext) { ctx.Step(regexp.MustCompile(`^I remove all the objects in the bucket "([^"]+)"$`), iRemoveAllTheObjectsInBucket) } -// InitializeCacheCompiledReleases requires environment configuration to interact with a Tanzu Ops Manager. -// -// # Environment -// -// - AWS_ACCESS_KEY_ID: credentials with access to an empty S3 bucket where the release will be cached -// - AWS_SECRET_ACCESS_KEY: credentials with access to empty an S3 bucket where the release will be cached -// - BOSH_ALL_PROXY: this environment variable is required by the BOSH client used in Kiln. To see how to construct it see [BOSH CLI Tunneling]: https://bosh.io/docs/cli-tunnel/ -// - OM_TARGET: should be set to a url like https://pcf.example.com -// - OM_USERNAME: should be set to the Ops Manager username -// - OM_PASSWORD: should be set to the Ops Manager password -// - OM_PRIVATE_KEY: should be set with a private key in PEM format that can be used to ssh to the ops manager -// -// ## Debugging -// -// The AWS credentials are the default environment variables for the AWS CLI. -// Note you can change the bucket used for testing by changing the value in the feature file. -// So you can check if they will work by invoking the following command: -// -// aws s3 ls s3://hello-tile-releases -// -// The OM_TARGET, OM_USERNAME, and OM_PASSWORD behave as they would with the OM CLI. -// You can ensure they are correct by running any om command. For example: -// -// om staged-products -// -// Note, where the scenario uses the om CLI, the command inherits the parent's environment. -// So if needed you can set OM_SKIP_SSL_VALIDATION and other om environment variables. -// -// OM_PRIVATE_KEY is not a standard om environment variable; it is used by kiln not OM. -// To ensure it works you can execute: -// -// echo "${OM_PRIVATE_KEY}" > /tmp/om.key -// chmod 0400 /tmp/om.key -// ssh -i /tmp/om.key "ubuntu@pcf.example.com" -func InitializeCacheCompiledReleases(ctx *godog.ScenarioContext) { - initializeCacheCompiledReleases(ctx) -} - -func initializeCacheCompiledReleases(ctx scenarioContext) { - ctx.Before(func(ctx context.Context, sc *godog.Scenario) (context.Context, error) { - _, _, err := loadS3Credentials() - if err != nil { - return ctx, err - } - return loadEnvironment(ctx) - }) - ctx.Step(regexp.MustCompile(`^I add a compiled s3 release-source "([^"]*)" to the Kilnfile$`), iAddACompiledSReleaseSourceToTheKilnfile) - ctx.Step(regexp.MustCompile(`^I set the stemcell version in the lock to match the one used for the tile$`), iSetTheStemcellVersionInTheLockToMatchTheOneUsedForTheTile) - ctx.Step(regexp.MustCompile(`^I upload, configure, and apply the tile$`), iUploadConfigureAndApplyTheTile) -} - func InitializeEnv(ctx *godog.ScenarioContext) { initializeEnv(ctx) } func initializeEnv(ctx scenarioContext) { ctx.Step(regexp.MustCompile(`^the environment variable "([^"]+)" is set$`), theEnvironmentVariableIsSet) diff --git a/internal/acceptance/workflows/scenario/initialize_test.go b/internal/acceptance/workflows/scenario/initialize_test.go index 131d62eb7..826906bca 100644 --- a/internal/acceptance/workflows/scenario/initialize_test.go +++ b/internal/acceptance/workflows/scenario/initialize_test.go @@ -20,9 +20,6 @@ func TestInitialize(t *testing.T) { t.Run("AWS", func(t *testing.T) { initializeAWS(newFakeScenarioContext(t)) }) - t.Run("CacheCompiledReleases", func(t *testing.T) { - initializeCacheCompiledReleases(newFakeScenarioContext(t)) - }) t.Run("Env", func(t *testing.T) { initializeEnv(newFakeScenarioContext(t)) }) diff --git a/internal/acceptance/workflows/scenario/ops_manager_environment.go b/internal/acceptance/workflows/scenario/ops_manager_environment.go deleted file mode 100644 index 426de269d..000000000 --- a/internal/acceptance/workflows/scenario/ops_manager_environment.go +++ /dev/null @@ -1,129 +0,0 @@ -package scenario - -import ( - "context" - "fmt" - "net/url" - "os" - "os/exec" - "strings" -) - -type opsManagerEnvironment struct { - OpsManagerPrivateKey string - OpsManager struct { - URL string - Password string - Username string - } - AvailabilityZones []string - ServiceSubnetName string -} - -func (env *opsManagerEnvironment) loadFromEnvironmentVariables() error { - var err error - env.OpsManager.URL, err = loadEnvironmentVariable("OM_TARGET", "") - if err != nil { - return err - } - env.OpsManager.Username, err = loadEnvironmentVariable("OM_USERNAME", "") - if err != nil { - return err - } - env.OpsManager.Password, err = loadEnvironmentVariable("OM_PASSWORD", "") - if err != nil { - return err - } - boshAllProxy, err := loadEnvironmentVariable("BOSH_ALL_PROXY", "") - if err != nil { - return err - } - _, loadOmPrivateKeyErr := loadEnvironmentVariable("OM_PRIVATE_KEY", "") - if loadOmPrivateKeyErr != nil { - privateKey, err := readPrivateKeyFromBOSHAllProxyURL(boshAllProxy) - if err != nil { - return err - } - err = os.Setenv("OM_PRIVATE_KEY", privateKey) - if err != nil { - return err - } - } - return nil -} - -func (env *opsManagerEnvironment) fetchNetworksAndAvailabilityZones(ctx context.Context) (context.Context, error) { - var directorConfig struct { - AvailabilityZoneConfiguration []struct { - Name string `yaml:"name"` - } `yaml:"az-configuration"` - NetworkConfiguration struct { - Networks []struct { - Name string `yaml:"name"` - } `yaml:"networks"` - } `yaml:"networks-configuration"` - } - ctx, err := runAndParseStdoutAsYAML(ctx, - exec.Command("om", "--skip-ssl-validation", "staged-director-config", "--no-redact"), - &directorConfig, - ) - if err != nil { - return ctx, err - } - for _, az := range directorConfig.AvailabilityZoneConfiguration { - env.AvailabilityZones = append(env.AvailabilityZones, az.Name) - } - for _, network := range directorConfig.NetworkConfiguration.Networks { - if !strings.HasSuffix(network.Name, "-services-subnet") { - continue - } - env.ServiceSubnetName = network.Name - break - } - return ctx, err -} - -func fetchAssociatedStemcellVersion(ctx context.Context, productID string) (string, error) { - var stemcellAssociations struct { - Products []struct { - ID string `yaml:"identifier"` - DeployedStemcells []struct { - Version string `yaml:"version"` - } `yaml:"deployed_stemcells"` - } `yaml:"products"` - } - var err error - _, err = runAndParseStdoutAsYAML(ctx, - exec.Command("om", "curl", "--silent", "--path", "/api/v0/stemcell_associations"), - &stemcellAssociations, - ) - if err != nil { - return "", err - } - for _, p := range stemcellAssociations.Products { - if p.ID != productID { - continue - } - for _, s := range p.DeployedStemcells { - return s.Version, nil - } - } - return "", fmt.Errorf("no stemcells found on ops manager") -} - -func readPrivateKeyFromBOSHAllProxyURL(boshAllProxy string) (string, error) { - const failedToSetOMPrivateKey = "failed to set OM_PRIVATE_KEY from BOSH_ALL_PROXY: %w" - u, err := url.Parse(boshAllProxy) - if err != nil { - return "", fmt.Errorf(failedToSetOMPrivateKey, err) - } - keyPath := u.Query().Get("private-key") - if keyPath == "" { - return "", fmt.Errorf(failedToSetOMPrivateKey, fmt.Errorf(`url did not have "private-key"`)) - } - keyBytes, err := os.ReadFile(keyPath) - if err != nil { - return "", fmt.Errorf(failedToSetOMPrivateKey, err) - } - return string(keyBytes), nil -} diff --git a/internal/acceptance/workflows/scenario/ops_manager_environment_test.go b/internal/acceptance/workflows/scenario/ops_manager_environment_test.go deleted file mode 100644 index 71b90c85b..000000000 --- a/internal/acceptance/workflows/scenario/ops_manager_environment_test.go +++ /dev/null @@ -1,23 +0,0 @@ -package scenario - -import ( - "context" - "testing" -) - -// Test_fetchAssociatedStemcellVersion will fail if you do not have a tile uploaded to the target ops manager -// unset OM_TARGET when you want this test not to run. -func Test_fetchAssociatedStemcellVersion(t *testing.T) { - _, err := loadEnvironmentVariable("OM_TARGET", "OM_TARGET not set") - if err != nil { - t.Skip("OM_TARGET not set") - } - ctx := configureStandardFileDescriptors(context.Background()) - v, err := fetchAssociatedStemcellVersion(ctx, "hello") - if err != nil { - t.Error(err) - } - if v == "" { - t.Errorf("expected a version") - } -} diff --git a/internal/acceptance/workflows/scenario/shared_state.go b/internal/acceptance/workflows/scenario/shared_state.go index 85ecfea47..d08176e65 100644 --- a/internal/acceptance/workflows/scenario/shared_state.go +++ b/internal/acceptance/workflows/scenario/shared_state.go @@ -9,8 +9,6 @@ import ( "os/exec" "path/filepath" "strconv" - - "gopkg.in/yaml.v2" ) // key represents the type of the context key for shared values between steps @@ -119,29 +117,6 @@ func loadGithubToken(ctx context.Context) (context.Context, error) { return context.WithValue(ctx, githubTokenKey, token), nil } -func environment(ctx context.Context) (opsManagerEnvironment, error) { - return contextValue[opsManagerEnvironment](ctx, environmentKey, "ops manager environment") -} - -func loadEnvironment(ctx context.Context) (context.Context, error) { - _, err := environment(ctx) - if err == nil { - return ctx, nil - } - - var omEnv opsManagerEnvironment - err = omEnv.loadFromEnvironmentVariables() - if err != nil { - return ctx, err - } - ctx, err = omEnv.fetchNetworksAndAvailabilityZones(ctx) - if err != nil { - return ctx, err - } - - return context.WithValue(ctx, environmentKey, omEnv), nil -} - type standardFileDescriptors [3]*bytes.Buffer const ( @@ -204,22 +179,3 @@ func runAndLogOnError(ctx context.Context, cmd *exec.Cmd, requireSuccess bool) ( } return ctx, nil } - -func runAndParseStdoutAsYAML(ctx context.Context, cmd *exec.Cmd, d any) (context.Context, error) { - var stdout, stderr bytes.Buffer - fds := ctx.Value(standardFileDescriptorsKey).(standardFileDescriptors) - cmd.Stdout = io.MultiWriter(&stdout, fds[1]) - cmd.Stderr = io.MultiWriter(&stderr, fds[2]) - runErr := cmd.Run() - ctx = setLastCommandStatus(ctx, cmd.ProcessState) - if runErr != nil { - _, _ = io.Copy(os.Stdout, &stdout) - _, _ = io.Copy(os.Stdout, &stderr) - return ctx, runErr - } - err := yaml.Unmarshal(stdout.Bytes(), d) - if err != nil { - return ctx, err - } - return ctx, nil -} diff --git a/internal/acceptance/workflows/scenario/step_funcs_ops_manager.go b/internal/acceptance/workflows/scenario/step_funcs_ops_manager.go deleted file mode 100644 index 9a0af83a6..000000000 --- a/internal/acceptance/workflows/scenario/step_funcs_ops_manager.go +++ /dev/null @@ -1,70 +0,0 @@ -package scenario - -import ( - "context" - "os/exec" - - "github.com/pivotal-cf/kiln/pkg/cargo" -) - -func iUploadConfigureAndApplyTheTile(ctx context.Context) (context.Context, error) { - env, err := environment(ctx) - if err != nil { - return ctx, err - } - tilePath, err := defaultFilePathForTile(ctx) - if err != nil { - return ctx, err - } - version, err := tileVersion(ctx) - if err != nil { - return ctx, err - } - - ctx, err = runAndLogOnError(ctx, exec.Command("om", "--skip-ssl-validation", "upload-product", "--product", tilePath), true) - if err != nil { - return ctx, err - } - ctx, err = runAndLogOnError(ctx, exec.Command("om", "--skip-ssl-validation", "stage-product", "--product-name", "hello", "--product-version", version), true) - if err != nil { - return ctx, err - } - ctx, err = runAndLogOnError(ctx, exec.Command("om", "--skip-ssl-validation", "configure-product", - "--config", "scenario/fixtures/hello-product-config.yml", - "--var", "subnet="+env.ServiceSubnetName, - "--var", "az="+env.AvailabilityZones[0], - ), true) - if err != nil { - return ctx, err - } - ctx, err = runAndLogOnError(ctx, exec.Command("om", "--skip-ssl-validation", "apply-changes"), true) - if err != nil { - return ctx, err - } - - return ctx, nil -} - -func iSetTheStemcellVersionInTheLockToMatchTheOneUsedForTheTile(ctx context.Context) (context.Context, error) { - lockPath, err := kilnfileLockPath(ctx) - if err != nil { - return ctx, err - } - - var lock cargo.KilnfileLock - err = loadFileAsYAML(lockPath, &lock) - if err != nil { - return ctx, err - } - - lock.Stemcell.Version, err = fetchAssociatedStemcellVersion(ctx, "hello") - if err != nil { - return ctx, err - } - - err = saveAsYAML(lockPath, lock) - if err != nil { - return ctx, err - } - return ctx, nil -} diff --git a/internal/acceptance/workflows/scenario/step_funcs_tile_source_code.go b/internal/acceptance/workflows/scenario/step_funcs_tile_source_code.go index 3216dc780..bfe3e1271 100644 --- a/internal/acceptance/workflows/scenario/step_funcs_tile_source_code.go +++ b/internal/acceptance/workflows/scenario/step_funcs_tile_source_code.go @@ -12,7 +12,6 @@ import ( "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" - "github.com/pivotal-cf/kiln/internal/component" "github.com/pivotal-cf/kiln/pkg/cargo" ) @@ -150,41 +149,6 @@ func theRepositoryHasNoFetchedReleases(ctx context.Context) error { return nil } -func iAddACompiledSReleaseSourceToTheKilnfile(ctx context.Context, bucketName string) error { - keyID, accessKey, err := loadS3Credentials() - if err != nil { - return err - } - kfPath, err := kilnfilePath(ctx) - if err != nil { - return err - } - - var kf cargo.Kilnfile - err = loadFileAsYAML(kfPath, &kf) - if err != nil { - return err - } - - for _, rs := range kf.ReleaseSources { - if rs.Bucket == bucketName { - return nil - } - } - - kf.ReleaseSources = append(kf.ReleaseSources, cargo.ReleaseSourceConfig{ - Type: component.ReleaseSourceTypeS3, - Bucket: bucketName, - PathTemplate: "{{.Name}}-{{.Version}}-{{.StemcellOS}}-{{.StemcellVersion}}.tgz", - Region: "us-west-1", - Publishable: true, - AccessKeyId: keyID, - SecretAccessKey: accessKey, - }) - - return saveAsYAML(kfPath, kf) -} - func iSetTheKilnfileStemcellVersionConstraint(ctx context.Context, versionConstraint string) error { spcePath, err := kilnfilePath(ctx) if err != nil { diff --git a/internal/acceptance/workflows/scenario/step_funcs_tile_test.go b/internal/acceptance/workflows/scenario/step_funcs_tile_test.go index 5d22da588..f97e053fe 100644 --- a/internal/acceptance/workflows/scenario/step_funcs_tile_test.go +++ b/internal/acceptance/workflows/scenario/step_funcs_tile_test.go @@ -19,7 +19,7 @@ func Test_theLockSpecifiesVersionForRelease(t *testing.T) { t.Run("it matches the release version", func(t *testing.T) { ctx, please := setup(t) - err := theLockSpecifiesVersionForRelease(ctx, "0.2.3", "hello-release") + err := theLockSpecifiesVersionForRelease(ctx, "0.1.5", "hello-release") please.Expect(err).NotTo(HaveOccurred()) }) diff --git a/internal/acceptance/workflows/scenario/utilities.go b/internal/acceptance/workflows/scenario/utilities.go index 6d3897d65..3041d743b 100644 --- a/internal/acceptance/workflows/scenario/utilities.go +++ b/internal/acceptance/workflows/scenario/utilities.go @@ -49,17 +49,6 @@ func closeAndIgnoreErr(c io.Closer) { _ = c.Close() } -func loadEnvironmentVariable(variableName, errorHelpMessage string) (string, error) { - v := os.Getenv(variableName) - if v == "" { - if errorHelpMessage == "" { - return "", fmt.Errorf("%s is not set", variableName) - } - return "", fmt.Errorf("%s is not set (%s)", variableName, errorHelpMessage) - } - return v, nil -} - func loadFileAsYAML(filePath string, v any) error { kfBuf, err := os.ReadFile(filePath) if err != nil { @@ -88,18 +77,6 @@ func saveAsYAML(filePath string, v any) error { return err } -func loadS3Credentials() (keyID, accessKey string, err error) { - keyID, err = loadEnvironmentVariable("AWS_ACCESS_KEY_ID", "required for s3 release source to cache releases") - if err != nil { - return - } - accessKey, err = loadEnvironmentVariable("AWS_SECRET_ACCESS_KEY", "required for s3 release source to cache releases") - if err != nil { - return - } - return -} - func getGithubTokenFromCLI() (string, error) { cmd := exec.Command("gh", "auth", "status", "--show-token") var out bytes.Buffer diff --git a/internal/acceptance/workflows/using_kiln.feature b/internal/acceptance/workflows/using_kiln.feature index a0477f147..d2ba13460 100644 --- a/internal/acceptance/workflows/using_kiln.feature +++ b/internal/acceptance/workflows/using_kiln.feature @@ -29,7 +29,6 @@ Feature: As a developer, I want the Kiln CLI to be usable Examples: | command | | bake | - | cache-compiled-releases | | fetch | | find-release-version | | find-stemcell-version | diff --git a/internal/commands/cache_compiled_releases.go b/internal/commands/cache_compiled_releases.go deleted file mode 100644 index 3fdf52282..000000000 --- a/internal/commands/cache_compiled_releases.go +++ /dev/null @@ -1,426 +0,0 @@ -package commands - -import ( - "crypto/sha1" - "crypto/sha256" - "errors" - "fmt" - "io" - "log" - "os" - "path/filepath" - - boshdir "github.com/cloudfoundry/bosh-cli/director" - "github.com/go-git/go-billy/v5" - "github.com/go-git/go-billy/v5/osfs" - "github.com/pivotal-cf/jhanda" - "github.com/pivotal-cf/om/api" - "gopkg.in/yaml.v2" - - "github.com/pivotal-cf/kiln/internal/commands/flags" - "github.com/pivotal-cf/kiln/internal/component" - "github.com/pivotal-cf/kiln/internal/om" - "github.com/pivotal-cf/kiln/pkg/cargo" -) - -//counterfeiter:generate -o ./fakes/ops_manager_release_cache_source.go --fake-name OpsManagerReleaseCacheSource . OpsManagerReleaseCacheSource -//counterfeiter:generate -o ./fakes/release_storage.go --fake-name ReleaseStorage . ReleaseStorage - -type ( - OpsManagerReleaseCacheSource interface { - om.GetBoshEnvironmentAndSecurityRootCACertificateProvider - GetStagedProductManifest(guid string) (string, error) - GetStagedProductByName(productName string) (api.StagedProductsFindOutput, error) - } - - ReleaseStorage interface { - component.ReleaseSource - UploadRelease(spec cargo.BOSHReleaseTarballSpecification, file io.Reader) (cargo.BOSHReleaseTarballLock, error) - } -) - -type CacheCompiledReleases struct { - Options struct { - flags.Standard - om.ClientConfiguration - - UploadTargetID string ` long:"upload-target-id" required:"true" description:"the ID of the release source where the built release will be uploaded"` - ReleasesDir string `short:"rd" long:"releases-directory" default:"releases" description:"path to a directory to download releases into"` - Name string `short:"n" long:"name" default:"cf" description:"name of the tile"` // TODO: parse from base.yml - } - - Logger *log.Logger - FS billy.Filesystem - - ReleaseSourceAndCache func(kilnfile cargo.Kilnfile, targetID string) (ReleaseStorage, error) - OpsManager func(om.ClientConfiguration) (OpsManagerReleaseCacheSource, error) - Director func(om.ClientConfiguration, om.GetBoshEnvironmentAndSecurityRootCACertificateProvider) (boshdir.Director, error) -} - -func NewCacheCompiledReleases() *CacheCompiledReleases { - cmd := &CacheCompiledReleases{ - FS: osfs.New(""), - Logger: log.Default(), - } - cmd.ReleaseSourceAndCache = func(kilnfile cargo.Kilnfile, targetID string) (ReleaseStorage, error) { - releaseSource, err := component.NewReleaseSourceRepo(kilnfile, cmd.Logger).FindByID(targetID) - if err != nil { - return nil, err - } - releaseCache, ok := releaseSource.(ReleaseStorage) - if !ok { - return nil, fmt.Errorf("unsupported release source type %T: it does not implement the required methods", releaseSource) - } - return releaseCache, nil - } - cmd.OpsManager = func(conf om.ClientConfiguration) (OpsManagerReleaseCacheSource, error) { - return conf.API() - } - cmd.Director = om.BoshDirector - return cmd -} - -func (cmd *CacheCompiledReleases) WithLogger(logger *log.Logger) *CacheCompiledReleases { - if logger == nil { - logger = log.New(io.Discard, "", 0) - } - cmd.Logger = logger - return cmd -} - -func (cmd *CacheCompiledReleases) Execute(args []string) error { - _, err := flags.LoadWithDefaultFilePaths(&cmd.Options, args, cmd.FS.Stat) - if err != nil { - return err - } - - kilnfile, lock, err := cmd.Options.LoadKilnfiles(cmd.FS, nil) - if err != nil { - return fmt.Errorf("failed to load kilnfiles: %w", err) - } - - omAPI, deploymentName, stagedStemcellOS, stagedStemcellVersion, err := cmd.fetchProductDeploymentData() - if err != nil { - return err - } - - if stagedStemcellOS != lock.Stemcell.OS || stagedStemcellVersion != lock.Stemcell.Version { - return fmt.Errorf( - "staged stemcell (%s %s) and lock stemcell (%s %s) do not match", - stagedStemcellOS, stagedStemcellVersion, - lock.Stemcell.OS, lock.Stemcell.Version, - ) - } - - releaseStore, err := cmd.ReleaseSourceAndCache(kilnfile, cmd.Options.UploadTargetID) - if err != nil { - return fmt.Errorf("failed to configure release source: %w", err) - } - - var ( - releasesToExport []cargo.BOSHReleaseTarballLock - releasesUpdatedFromCache = false - ) - for _, rel := range lock.Releases { - remote, err := releaseStore.GetMatchedRelease(cargo.BOSHReleaseTarballSpecification{ - Name: rel.Name, - Version: rel.Version, - StemcellOS: lock.Stemcell.OS, - StemcellVersion: lock.Stemcell.Version, - }) - if err != nil { - if !component.IsErrNotFound(err) { - return fmt.Errorf("failed check for matched release: %w", err) - } - releasesToExport = append(releasesToExport, cargo.BOSHReleaseTarballLock{ - Name: rel.Name, - Version: rel.Version, - StemcellOS: lock.Stemcell.OS, - StemcellVersion: lock.Stemcell.Version, - }) - continue - } - - cmd.Logger.Printf("found %s/%s in %s\n", rel.Name, rel.Version, remote.RemoteSource) - - sum, err := cmd.downloadAndComputeSHA(releaseStore, remote) - if err != nil { - cmd.Logger.Printf("unable to get hash sum for %s", remote.ReleaseSlug()) - continue - } - remote.SHA1 = sum - - releasesUpdatedFromCache = true - err = updateLock(lock, remote, cmd.Options.UploadTargetID) - if err != nil { - return fmt.Errorf("failed to update lock file: %w", err) - } - } - - switch len(releasesToExport) { - case 0: - cmd.Logger.Print("cache already contains releases matching constraint\n") - if releasesUpdatedFromCache { - err = cmd.Options.Standard.SaveKilnfileLock(cmd.FS, lock) - if err != nil { - return err - } - - cmd.Logger.Printf("DON'T FORGET TO MAKE A COMMIT AND PR\n") - - return nil - } - case 1: - cmd.Logger.Printf("1 release needs to be exported and cached\n") - default: - cmd.Logger.Printf("%d releases need to be exported and cached\n", len(releasesToExport)) - } - - for _, rel := range releasesToExport { - cmd.Logger.Printf("\t%s compiled with %s not found in cache\n", rel.ReleaseSlug(), rel.StemcellSlug()) - } - - bosh, err := cmd.Director(cmd.Options.ClientConfiguration, omAPI) - if err != nil { - return err - } - - deployment, err := bosh.FindDeployment(deploymentName) - if err != nil { - return err - } - - cmd.Logger.Printf("exporting from bosh deployment %s\n", deploymentName) - - err = cmd.FS.MkdirAll(cmd.Options.ReleasesDir, 0o777) - if err != nil { - return fmt.Errorf("failed to create release directory: %w", err) - } - - for _, rel := range releasesToExport { - releaseSlug := rel.ReleaseSlug() - stemcellSlug := boshdir.NewOSVersionSlug(stagedStemcellOS, stagedStemcellVersion) - - hasRelease, err := hasRequiredCompiledPackages(bosh, rel.ReleaseSlug(), stemcellSlug) - if err != nil { - if !errors.Is(err, errNoPackages) { - return fmt.Errorf("failed to find release %s: %w", rel.ReleaseSlug(), err) - } - cmd.Logger.Printf("%s does not have any packages\n", rel) - } - if !hasRelease { - return fmt.Errorf("%[1]s compiled with %[2]s is not found on bosh director (it might have been uploaded as a compiled release and the director can't recompile it for the compilation target %[2]s)", releaseSlug, stemcellSlug) - } - - newRemote, err := cmd.cacheRelease(bosh, releaseStore, deployment, releaseSlug, stemcellSlug) - if err != nil { - cmd.Logger.Printf("\tfailed to cache release %s for %s: %s\n", releaseSlug, stemcellSlug, err) - continue - } - - err = updateLock(lock, newRemote, cmd.Options.UploadTargetID) - if err != nil { - return fmt.Errorf("failed to lock release %s: %w", rel.Name, err) - } - } - - err = cmd.Options.Standard.SaveKilnfileLock(cmd.FS, lock) - if err != nil { - return err - } - - cmd.Logger.Printf("DON'T FORGET TO MAKE A COMMIT AND PR\n") - - return nil -} - -var errNoPackages = errors.New("release has no packages") - -// hasRequiredCompiledPackages implementation is copied from the boshdir.DirectorImpl HasRelease method. It adds the check -// for the length of the packages slice to allow for BOSH releases that do not have any packages. One example of a BOSH -// release without packages is https://github.com/cloudfoundry/bosh-dns-aliases-release. -func hasRequiredCompiledPackages(d boshdir.Director, releaseSlug boshdir.ReleaseSlug, stemcell boshdir.OSVersionSlug) (bool, error) { - release, err := d.FindRelease(releaseSlug) - if err != nil { - return false, err - } - - pkgs, err := release.Packages() - if err != nil { - return false, err - } - - if len(pkgs) == 0 { - return true, errNoPackages - } - - for _, pkg := range pkgs { - for _, compiledPkg := range pkg.CompiledPackages { - if compiledPkg.Stemcell == stemcell { - return true, nil - } - } - } - - return false, nil -} - -func (cmd *CacheCompiledReleases) fetchProductDeploymentData() (_ OpsManagerReleaseCacheSource, deploymentName, stemcellOS, stemcellVersion string, _ error) { - omAPI, err := cmd.OpsManager(cmd.Options.ClientConfiguration) - if err != nil { - return nil, "", "", "", err - } - - stagedProduct, err := omAPI.GetStagedProductByName(cmd.Options.Name) - if err != nil { - return nil, "", "", "", err - } - - stagedManifest, err := omAPI.GetStagedProductManifest(stagedProduct.Product.GUID) - if err != nil { - return nil, "", "", "", err - } - - var manifest struct { - Name string `yaml:"name"` - Stemcells []struct { - OS string `yaml:"os"` - Version string `yaml:"version"` - } `yaml:"stemcells"` - } - - if err := yaml.Unmarshal([]byte(stagedManifest), &manifest); err != nil { - return nil, "", "", "", err - } - - if len(manifest.Stemcells) == 0 { - return nil, "", "", "", errors.New("manifest stemcell not set") - } - stagedStemcell := manifest.Stemcells[0] - - return omAPI, manifest.Name, stagedStemcell.OS, stagedStemcell.Version, nil -} - -func (cmd *CacheCompiledReleases) cacheRelease(bosh boshdir.Director, rc ReleaseStorage, deployment boshdir.Deployment, releaseSlug boshdir.ReleaseSlug, stemcellSlug boshdir.OSVersionSlug) (cargo.BOSHReleaseTarballLock, error) { - cmd.Logger.Printf("\texporting %s\n", releaseSlug) - result, err := deployment.ExportRelease(releaseSlug, stemcellSlug, nil) - if err != nil { - return cargo.BOSHReleaseTarballLock{}, err - } - - cmd.Logger.Printf("\tdownloading %s\n", releaseSlug) - releaseFilePath, _, sha1sum, err := cmd.saveReleaseLocally(bosh, cmd.Options.ReleasesDir, releaseSlug, stemcellSlug, result) - if err != nil { - return cargo.BOSHReleaseTarballLock{}, err - } - - cmd.Logger.Printf("\tuploading %s\n", releaseSlug) - remoteRelease, err := cmd.uploadLocalRelease(cargo.BOSHReleaseTarballSpecification{ - Name: releaseSlug.Name(), - Version: releaseSlug.Version(), - StemcellOS: stemcellSlug.OS(), - StemcellVersion: stemcellSlug.Version(), - }, releaseFilePath, rc) - if err != nil { - return cargo.BOSHReleaseTarballLock{}, err - } - - remoteRelease.SHA1 = sha1sum - - return remoteRelease, nil -} - -func updateLock(lock cargo.KilnfileLock, release cargo.BOSHReleaseTarballLock, targetID string) error { - for index, releaseLock := range lock.Releases { - if release.Name != releaseLock.Name { - continue - } - - checksum := release.SHA1 - if releaseLock.RemoteSource == targetID { - checksum = releaseLock.SHA1 - } - - lock.Releases[index] = cargo.BOSHReleaseTarballLock{ - Name: release.Name, - Version: release.Version, - RemoteSource: release.RemoteSource, - RemotePath: release.RemotePath, - SHA1: checksum, - } - return nil - } - return fmt.Errorf("existing release not found in Kilnfile.lock") -} - -func (cmd *CacheCompiledReleases) uploadLocalRelease(spec cargo.BOSHReleaseTarballSpecification, fp string, uploader ReleaseStorage) (cargo.BOSHReleaseTarballLock, error) { - f, err := cmd.FS.Open(fp) - if err != nil { - return cargo.BOSHReleaseTarballLock{}, err - } - defer closeAndIgnoreError(f) - return uploader.UploadRelease(spec, f) -} - -func (cmd *CacheCompiledReleases) saveReleaseLocally(director boshdir.Director, relDir string, releaseSlug boshdir.ReleaseSlug, stemcellSlug boshdir.OSVersionSlug, res boshdir.ExportReleaseResult) (string, string, string, error) { - fileName := fmt.Sprintf("%s-%s-%s-%s.tgz", releaseSlug.Name(), releaseSlug.Version(), stemcellSlug.OS(), stemcellSlug.Version()) - filePath := filepath.Join(relDir, fileName) - - f, err := cmd.FS.Create(filePath) - if err != nil { - return "", "", "", err - } - defer closeAndIgnoreError(f) - - sha256sum := sha256.New() - sha1sum := sha1.New() - - w := io.MultiWriter(f, sha256sum, sha1sum) - - err = director.DownloadResourceUnchecked(res.BlobstoreID, w) - if err != nil { - _ = os.Remove(filePath) - return "", "", "", err - } - - sha256sumString := fmt.Sprintf("%x", sha256sum.Sum(nil)) - sha1sumString := fmt.Sprintf("%x", sha1sum.Sum(nil)) - - if sum := fmt.Sprintf("sha256:%s", sha256sumString); sum != res.SHA1 { - return "", "", "", fmt.Errorf("checksums do not match got %q but expected %q", sum, res.SHA1) - } - - return filePath, sha256sumString, sha1sumString, nil -} - -func (cmd *CacheCompiledReleases) downloadAndComputeSHA(cache component.ReleaseSource, remote cargo.BOSHReleaseTarballLock) (string, error) { - if remote.SHA1 != "" { - return remote.SHA1, nil - } - - tmpdir, err := os.MkdirTemp("/tmp", "kiln") - if err != nil { - return "", fmt.Errorf("failed to create temporary directory: %w", err) - } - defer func() { - err = os.RemoveAll(tmpdir) - if err != nil { - cmd.Logger.Printf("unable to delete '%s': %s", tmpdir, err) - } - }() - - comp, err := cache.DownloadRelease(tmpdir, remote) - if err != nil { - return "", fmt.Errorf("failed to download release: %s", err) - } - - return comp.Lock.SHA1, nil -} - -func (cmd *CacheCompiledReleases) Usage() jhanda.Usage { - return jhanda.Usage{ - Description: "Downloads compiled bosh releases from an Tanzu Ops Manager bosh director and then uploads them to a bucket", - ShortDescription: "Cache compiled releases", - Flags: cmd.Options, - } -} diff --git a/internal/commands/cache_compiled_releases_test.go b/internal/commands/cache_compiled_releases_test.go deleted file mode 100644 index 86479f01b..000000000 --- a/internal/commands/cache_compiled_releases_test.go +++ /dev/null @@ -1,608 +0,0 @@ -package commands_test - -import ( - "bytes" - "crypto/sha256" - "fmt" - "io" - "log" - "testing" - - "github.com/cloudfoundry/bosh-cli/director" - boshdirFakes "github.com/cloudfoundry/bosh-cli/director/directorfakes" - "github.com/go-git/go-billy/v5/memfs" - . "github.com/onsi/gomega" - "github.com/pivotal-cf/jhanda" - - "github.com/pivotal-cf/kiln/internal/commands" - "github.com/pivotal-cf/kiln/internal/commands/fakes" - "github.com/pivotal-cf/kiln/internal/component" - "github.com/pivotal-cf/kiln/internal/om" - "github.com/pivotal-cf/kiln/pkg/cargo" -) - -var _ jhanda.Command = (*commands.CacheCompiledReleases)(nil) - -func TestNewCacheCompiledReleases(t *testing.T) { - please := NewWithT(t) - cmd := commands.NewCacheCompiledReleases() - please.Expect(cmd).NotTo(BeNil()) - please.Expect(cmd.Logger).NotTo(BeNil()) - please.Expect(cmd.FS).NotTo(BeNil()) - please.Expect(cmd.ReleaseSourceAndCache).NotTo(BeNil()) - please.Expect(cmd.OpsManager).NotTo(BeNil()) - please.Expect(cmd.Director).NotTo(BeNil()) -} - -type cacheCompiledReleasesTestData struct { - cmd *commands.CacheCompiledReleases - - bosh *boshdirFakes.FakeDirector - deployment *boshdirFakes.FakeDeployment - opsManager *fakes.OpsManagerReleaseCacheSource - output *bytes.Buffer - releaseStorage *fakes.ReleaseStorage -} - -const ( - releaseInBlobstore = `lemon-release-buffer` -) - -func newCacheCompiledReleasesTestData(t *testing.T, kf cargo.Kilnfile, kl cargo.KilnfileLock, stagedStemcellVersion string) cacheCompiledReleasesTestData { - t.Helper() - - fs := memfs.New() - - if err := fsWriteYAML(fs, "Kilnfile", kf); err != nil { - t.Fatal(err) - } - if err := fsWriteYAML(fs, "Kilnfile.lock", kl); err != nil { - t.Fatal(err) - } - - var output bytes.Buffer - logger := log.New(&output, "", 0) - - releaseStorage := new(fakes.ReleaseStorage) - releaseStorage.GetMatchedReleaseCalls(func(spec cargo.BOSHReleaseTarballSpecification) (cargo.BOSHReleaseTarballLock, error) { - switch spec.Lock() { - case cargo.BOSHReleaseTarballLock{Name: "orange", Version: "1.0.0", StemcellOS: "alpine", StemcellVersion: "9.0.0"}: - return cargo.BOSHReleaseTarballLock{ - Name: "orange", Version: "1.0.0", - SHA1: "fake-checksum", - RemoteSource: "cached-compiled-releases", - RemotePath: "orange-1.0.0-alpine-9.0.0", - }, nil - case cargo.BOSHReleaseTarballLock{Name: "banana", Version: "2.0.0", StemcellOS: "alpine", StemcellVersion: "9.0.0"}: - return cargo.BOSHReleaseTarballLock{ - Name: "banana", Version: "2.0.0", - SHA1: "fake-checksum", - RemoteSource: "cached-compiled-releases", - RemotePath: "banana-2.0.0-alpine-9.0.0", - }, nil - case cargo.BOSHReleaseTarballLock{Name: "lemon", Version: "3.0.0", StemcellOS: "alpine", StemcellVersion: "9.0.0"}, - cargo.BOSHReleaseTarballLock{Name: "banana", Version: "2.0.0", StemcellOS: "alpine", StemcellVersion: "8.0.0"}: - return cargo.BOSHReleaseTarballLock{}, component.ErrNotFound - } - - panic(fmt.Sprintf("unexpected spec %#v", spec)) - }) - - opsManager := new(fakes.OpsManagerReleaseCacheSource) - opsManager.GetStagedProductManifestReturns(fmt.Sprintf(`{"name": "cf-some-id", "stemcells": [{"os": "alpine", "version": %q}]}`, stagedStemcellVersion), nil) - - deployment := new(boshdirFakes.FakeDeployment) - bosh := new(boshdirFakes.FakeDirector) - bosh.FindDeploymentReturns(deployment, nil) - - cmd := commands.CacheCompiledReleases{ - FS: fs, - Logger: logger, - ReleaseSourceAndCache: func(kilnfile cargo.Kilnfile, targetID string) (commands.ReleaseStorage, error) { - return releaseStorage, nil - }, - OpsManager: func(configuration om.ClientConfiguration) (commands.OpsManagerReleaseCacheSource, error) { - return opsManager, nil - }, - Director: func(configuration om.ClientConfiguration, provider om.GetBoshEnvironmentAndSecurityRootCACertificateProvider) (director.Director, error) { - return bosh, nil - }, - } - return cacheCompiledReleasesTestData{ - cmd: &cmd, - - bosh: bosh, - output: &output, - deployment: deployment, - opsManager: opsManager, - releaseStorage: releaseStorage, - } -} - -func TestCacheCompiledReleases_Execute_all_releases_are_already_compiled(t *testing.T) { - please := NewWithT(t) - - // setup - - test := newCacheCompiledReleasesTestData(t, cargo.Kilnfile{ - ReleaseSources: []cargo.ReleaseSourceConfig{ - { - ID: "compiled-releases", - }, - }, - }, cargo.KilnfileLock{ - Releases: []cargo.BOSHReleaseTarballLock{ - { - Name: "banana", - Version: "2.0.0", - RemoteSource: "compiled-releases", - RemotePath: "banana-2.0.0-alpine-9.0.0", - SHA1: "fake-checksum", - }, - }, - Stemcell: cargo.Stemcell{ - OS: "alpine", - Version: "9.0.0", - }, - }, "9.0.0") - - // run - - err := test.cmd.Execute([]string{ - "--upload-target-id", "compiled-releases", - }) - - // check - - please.Expect(test.output.String()).To(ContainSubstring("cache already contains releases")) - please.Expect(err).NotTo(HaveOccurred()) -} - -func TestCacheCompiledReleases_Execute_all_releases_are_already_cached(t *testing.T) { - please := NewWithT(t) - - // setup - - test := newCacheCompiledReleasesTestData(t, cargo.Kilnfile{ - ReleaseSources: []cargo.ReleaseSourceConfig{ - { - ID: "compiled-releases", - }, - }, - }, cargo.KilnfileLock{ - Releases: []cargo.BOSHReleaseTarballLock{ - { - Name: "orange", - Version: "1.0.0", - - RemoteSource: "new-releases", - RemotePath: "orange-1.0.0", - - SHA1: "fake-checksum", - }, - }, - Stemcell: cargo.Stemcell{ - OS: "alpine", - Version: "9.0.0", - }, - }, "9.0.0") - - // run - - err := test.cmd.Execute([]string{ - "--upload-target-id", "compiled-releases", - }) - - // check - - please.Expect(test.output.String()).To(ContainSubstring("cache already contains releases")) - please.Expect(err).NotTo(HaveOccurred()) - - var updatedKilnfile cargo.KilnfileLock - please.Expect(fsReadYAML(test.cmd.FS, "Kilnfile.lock", &updatedKilnfile)).NotTo(HaveOccurred()) - please.Expect(updatedKilnfile.Releases).To(ContainElement(cargo.BOSHReleaseTarballLock{ - Name: "orange", Version: "1.0.0", - SHA1: "fake-checksum", - RemoteSource: "cached-compiled-releases", - RemotePath: "orange-1.0.0-alpine-9.0.0", - })) -} - -// this test covers -// - an export, download, upload, and lock of a non-cached release -// - an update the kilnfile with a non-locked release cached in the database -// (the release is cached on s3 but not set in the lock file) -// - ignoring a locked and cached release -// (the release is cached on the s3 bucket and the lock already has that value in it) -func TestCacheCompiledReleases_Execute_when_one_release_is_cached_another_is_already_compiled_and_another_is_already_locked(t *testing.T) { - // setup - - test := newCacheCompiledReleasesTestData(t, cargo.Kilnfile{ - ReleaseSources: []cargo.ReleaseSourceConfig{ - { - ID: "cached-compiled-releases", - Publishable: true, - PathTemplate: "{{.Release}}-{{.Version}}.tgz", - }, - { - ID: "new-releases", - Publishable: false, - PathTemplate: "{{.Release}}-{{.Version}}.tgz", - }, - }, - }, cargo.KilnfileLock{ - Releases: []cargo.BOSHReleaseTarballLock{ - { - Name: "orange", - Version: "1.0.0", - - RemoteSource: "new-releases", - RemotePath: "orange-1.0.0", - - SHA1: "fake-checksum", - }, - { - Name: "banana", - Version: "2.0.0", - - RemoteSource: "cached-compiled-releases", - RemotePath: "banana-2.0.0-alpine-9.0.0", - - SHA1: "fake-checksum", - }, - { - Name: "lemon", - Version: "3.0.0", - - RemoteSource: "new-releases", - RemotePath: "lemon-3.0.0", - - SHA1: "fake-checksum", - }, - }, - Stemcell: cargo.Stemcell{ - OS: "alpine", - Version: "9.0.0", - }, - }, "9.0.0") - - test.deployment.ExportReleaseReturns(director.ExportReleaseResult{ - BlobstoreID: "some-blob-id", - SHA1: fmt.Sprintf("sha256:%x", sha256.Sum256([]byte(releaseInBlobstore))), - }, nil) - test.bosh.DownloadResourceUncheckedCalls(func(_ string, writer io.Writer) error { - _, _ = writer.Write([]byte(releaseInBlobstore)) - return nil - }) - test.bosh.FindReleaseStub = func(slug director.ReleaseSlug) (director.Release, error) { - switch slug.Name() { - default: - panic(fmt.Errorf("FindReleaseStub input not handled: %#v", slug)) - case "lemon": - return &boshdirFakes.FakeRelease{ - PackagesStub: func() ([]director.Package, error) { - return []director.Package{{CompiledPackages: []director.CompiledPackage{{Stemcell: director.NewOSVersionSlug("alpine", "9.0.0")}}}}, nil - }, - }, nil - } - } - - var uploadedRelease bytes.Buffer - test.releaseStorage.UploadReleaseCalls(func(_ cargo.BOSHReleaseTarballSpecification, reader io.Reader) (cargo.BOSHReleaseTarballLock, error) { - _, _ = io.Copy(&uploadedRelease, reader) - return cargo.BOSHReleaseTarballLock{ - Name: "lemon", Version: "3.0.0", - - RemoteSource: "cached-compiled-releases", - RemotePath: "lemon-3.0.0-alpine-9.0.0", - SHA1: "012ed191f1d07c14bbcbbc0423d0de1c56757348", - }, nil - }) - test.releaseStorage.DownloadReleaseReturns(component.Local{}, fmt.Errorf("SO MUCH NOTHING")) - - // run - - err := test.cmd.Execute([]string{ - "--upload-target-id", "cached-compiled-releases", - }) - - // check - please := NewWithT(t) - please.Expect(err).NotTo(HaveOccurred()) - please.Expect(test.releaseStorage.GetMatchedReleaseCallCount()).To(Equal(3)) - please.Expect(test.bosh.DownloadResourceUncheckedCallCount()).To(Equal(1)) - - requestedID, _ := test.bosh.DownloadResourceUncheckedArgsForCall(0) - please.Expect(requestedID).To(Equal("some-blob-id")) - - please.Expect(test.output.String()).To(ContainSubstring("1 release needs to be exported and cached")) - please.Expect(test.output.String()).To(ContainSubstring("lemon/3.0.0 compiled with alpine/9.0.0 not found in cache")) - please.Expect(test.output.String()).To(ContainSubstring("exporting from bosh deployment cf-some-id")) - please.Expect(test.output.String()).To(ContainSubstring("exporting lemon")) - please.Expect(test.output.String()).To(ContainSubstring("downloading lemon")) - please.Expect(test.output.String()).To(ContainSubstring("uploading lemon")) - please.Expect(test.output.String()).To(ContainSubstring("DON'T FORGET TO MAKE A COMMIT AND PR")) - - please.Expect(uploadedRelease.String()).To(Equal(releaseInBlobstore)) - - var updatedKilnfile cargo.KilnfileLock - please.Expect(fsReadYAML(test.cmd.FS, "Kilnfile.lock", &updatedKilnfile)).NotTo(HaveOccurred()) - please.Expect(updatedKilnfile.Releases).To(ContainElement(cargo.BOSHReleaseTarballLock{ - Name: "lemon", - Version: "3.0.0", - SHA1: "012ed191f1d07c14bbcbbc0423d0de1c56757348", - RemoteSource: "cached-compiled-releases", - RemotePath: "lemon-3.0.0-alpine-9.0.0", - })) -} - -// this test covers -// - when a release is compiled with a stemcell that is not the one in the Kilnfile.lock (aka the compilation target) -// - the deployment succeeds because the stemcell major lines are the same/compatible -// - export release returns a broken bosh release because we requested the wrong compilation target and the director didn't have the source code necessarily to re-compile against the requested stemcell -// - (ideally bosh export-release should return an error but in this case it doesn't so we are just checking for a release with the correct stemcell before downloading a bad one) -func TestCacheCompiledReleases_Execute_when_a_release_is_not_compiled_with_the_correct_stemcell(t *testing.T) { - please := NewWithT(t) - - // setup - - test := newCacheCompiledReleasesTestData(t, cargo.Kilnfile{ - ReleaseSources: []cargo.ReleaseSourceConfig{ - { - ID: "cached-compiled-releases", - Publishable: true, - PathTemplate: "{{.Release}}-{{.Version}}.tgz", - }, - { - ID: "new-releases", - Publishable: false, - PathTemplate: "{{.Release}}-{{.Version}}.tgz", - }, - }, - }, cargo.KilnfileLock{ - Releases: []cargo.BOSHReleaseTarballLock{ - { - Name: "banana", - Version: "2.0.0", - - RemoteSource: "cached-compiled-releases", - RemotePath: "banana-2.0.0-alpine-5.5.5", - - SHA1: "fake-checksum", - }, - }, - Stemcell: cargo.Stemcell{ - OS: "alpine", - Version: "8.0.0", - }, - }, "8.0.0") - - test.bosh.FindReleaseStub = func(slug director.ReleaseSlug) (director.Release, error) { - switch slug.Name() { - default: - panic(fmt.Errorf("FindReleaseStub input not handled: %#v", slug)) - case "banana": - return &boshdirFakes.FakeRelease{ - PackagesStub: func() ([]director.Package, error) { - return make([]director.Package, 1), nil - }, - }, nil - } - } - - // run - - err := test.cmd.Execute([]string{ - "--upload-target-id", "cached-compiled-releases", - }) - - // check - - please.Expect(err).To(MatchError(ContainSubstring("not found on bosh director"))) - - please.Expect(test.bosh.DownloadResourceUncheckedCallCount()).To(Equal(0)) - please.Expect(test.bosh.HasReleaseCallCount()).To(Equal(0)) - please.Expect(test.bosh.FindReleaseCallCount()).To(Equal(1)) - - { - requestedReleaseSlug := test.bosh.FindReleaseArgsForCall(0) - please.Expect(requestedReleaseSlug.Name()).To(Equal("banana")) - please.Expect(requestedReleaseSlug.Version()).To(Equal("2.0.0")) - } - - please.Expect(test.output.String()).To(ContainSubstring("1 release needs to be exported and cached")) - please.Expect(test.output.String()).To(ContainSubstring("banana/2.0.0 compiled with alpine/8.0.0 not found in cache")) - please.Expect(test.output.String()).To(ContainSubstring("exporting from bosh deployment cf-some-id")) - please.Expect(test.output.String()).NotTo(ContainSubstring("exporting lemon")) - please.Expect(test.output.String()).NotTo(ContainSubstring("DON'T FORGET TO MAKE A COMMIT AND PR")) - - var updatedKilnfile cargo.KilnfileLock - please.Expect(fsReadYAML(test.cmd.FS, "Kilnfile.lock", &updatedKilnfile)).NotTo(HaveOccurred()) - please.Expect(updatedKilnfile.Releases).To(ContainElement(cargo.BOSHReleaseTarballLock{ - Name: "banana", - Version: "2.0.0", - - RemoteSource: "cached-compiled-releases", - RemotePath: "banana-2.0.0-alpine-5.5.5", - - SHA1: "fake-checksum", - }), "it should not override the in-correct element in the Kilnfile.lock") -} - -// this test covers -// - when a release does not contain packages -func TestCacheCompiledReleases_Execute_when_a_release_has_no_packages(t *testing.T) { - please := NewWithT(t) - - // setup - test := newCacheCompiledReleasesTestData(t, cargo.Kilnfile{ - ReleaseSources: []cargo.ReleaseSourceConfig{ - { - ID: "cached-compiled-releases", - Publishable: true, - PathTemplate: "{{.Release}}-{{.Version}}.tgz", - }, - { - ID: "new-releases", - Publishable: false, - PathTemplate: "{{.Release}}-{{.Version}}.tgz", - }, - }, - Releases: []cargo.BOSHReleaseTarballSpecification{ - { - Name: "banana", - }, - }, - }, cargo.KilnfileLock{ - Releases: []cargo.BOSHReleaseTarballLock{ - { - Name: "banana", - Version: "2.0.0", - - RemoteSource: "cached-compiled-releases", - RemotePath: "banana-2.0.0-alpine-5.5.5", - - SHA1: "fake-checksum", - }, - }, - Stemcell: cargo.Stemcell{ - OS: "alpine", - Version: "8.0.0", - }, - }, "8.0.0") - - test.deployment.ExportReleaseReturns(director.ExportReleaseResult{SHA1: "sha256:7dd4f2f077e449b47215359e8020c0b6c81e184d2c614486246cb8f70cac7a70"}, nil) - test.bosh.DownloadResourceUncheckedCalls(func(_ string, writer io.Writer) error { - _, _ = writer.Write([]byte("greetings")) - return nil - }) - test.bosh.FindReleaseStub = func(slug director.ReleaseSlug) (director.Release, error) { - switch slug.Name() { - default: - panic(fmt.Errorf("FindReleaseStub input not handled: %#v", slug)) - case "banana": - return &boshdirFakes.FakeRelease{ - PackagesStub: func() ([]director.Package, error) { - return make([]director.Package, 0), nil - }, - }, nil - } - } - test.releaseStorage.UploadReleaseStub = func(spec cargo.BOSHReleaseTarballSpecification, reader io.Reader) (cargo.BOSHReleaseTarballLock, error) { - l := spec.Lock() - l.RemotePath = "BANANA.tgz" - l.RemoteSource = "BASKET" - return l, nil - } - test.releaseStorage.DownloadReleaseReturns(component.Local{}, fmt.Errorf("SO MUCH NOTHING")) - // run - - err := test.cmd.Execute([]string{ - "--upload-target-id", "cached-compiled-releases", - }) - - // check - - please.Expect(test.bosh.DownloadResourceUncheckedCallCount()).To(Equal(1)) - please.Expect(test.bosh.HasReleaseCallCount()).To(Equal(0)) - please.Expect(test.bosh.FindReleaseCallCount()).To(Equal(1)) - - { - requestedReleaseSlug := test.bosh.FindReleaseArgsForCall(0) - please.Expect(requestedReleaseSlug.Name()).To(Equal("banana")) - please.Expect(requestedReleaseSlug.Version()).To(Equal("2.0.0")) - } - - please.Expect(test.output.String()).To(ContainSubstring("1 release needs to be exported and cached")) - please.Expect(test.output.String()).To(ContainSubstring("banana/2.0.0 compiled with alpine/8.0.0 not found in cache")) - please.Expect(test.output.String()).To(ContainSubstring("exporting from bosh deployment cf-some-id")) - please.Expect(test.output.String()).To(ContainSubstring("oes not have any packages")) - please.Expect(test.output.String()).To(ContainSubstring("exporting banana")) - - var updatedKilnfile cargo.KilnfileLock - please.Expect(fsReadYAML(test.cmd.FS, "Kilnfile.lock", &updatedKilnfile)).NotTo(HaveOccurred()) - please.Expect(updatedKilnfile.Releases).To(ContainElement(cargo.BOSHReleaseTarballLock{ - Name: "banana", - Version: "2.0.0", - - RemoteSource: "BASKET", - RemotePath: "BANANA.tgz", - - SHA1: "fake-checksum", - }), "it should not override the in-correct element in the Kilnfile.lock") - - please.Expect(err).NotTo(HaveOccurred()) - - please.Expect(test.output.String()).To(ContainSubstring("DON'T FORGET TO MAKE A COMMIT AND PR")) -} - -func TestCacheCompiledReleases_Execute_staged_and_lock_stemcells_are_not_the_same(t *testing.T) { - please := NewWithT(t) - - // setup - - initialLock := cargo.KilnfileLock{ - Releases: []cargo.BOSHReleaseTarballLock{ - { - Name: "orange", - Version: "1.0.0", - - RemoteSource: "new-releases", - RemotePath: "orange-1.0.0", - - SHA1: "fake-checksum", - }, - { - Name: "banana", - Version: "2.0.0", - - RemoteSource: "cached-compiled-releases", - RemotePath: "banana-2.0.0-alpine-9.0.0", - - SHA1: "fake-checksum", - }, - { - Name: "lemon", - Version: "3.0.0", - SHA1: "fake-checksum", - RemoteSource: "new-releases", - RemotePath: "lemon-3.0.0", - }, - }, - Stemcell: cargo.Stemcell{ - OS: "alpine", - Version: "9.0.0", - }, - } - - test := newCacheCompiledReleasesTestData(t, cargo.Kilnfile{ - ReleaseSources: []cargo.ReleaseSourceConfig{ - { - ID: "cached-compiled-releases", - Publishable: true, - PathTemplate: "{{.Release}}-{{.Version}}.tgz", - }, - { - ID: "new-releases", - Publishable: false, - PathTemplate: "{{.Release}}-{{.Version}}.tgz", - }, - }, - }, initialLock, "9.0.1") - - // run - - err := test.cmd.Execute([]string{ - "--upload-target-id", "cached-compiled-releases", - }) - - // check - - please.Expect(test.releaseStorage.GetMatchedReleaseCallCount()).To(Equal(0)) - please.Expect(test.bosh.DownloadResourceUncheckedCallCount()).To(Equal(0)) - please.Expect(err).To(MatchError(Equal("staged stemcell (alpine 9.0.1) and lock stemcell (alpine 9.0.0) do not match"))) - - var updatedLock cargo.KilnfileLock - please.Expect(fsReadYAML(test.cmd.FS, "Kilnfile.lock", &updatedLock)).NotTo(HaveOccurred()) - please.Expect(updatedLock).To(Equal(initialLock)) -} diff --git a/internal/commands/fakes/ops_manager_release_cache_source.go b/internal/commands/fakes/ops_manager_release_cache_source.go deleted file mode 100644 index ab6ca789e..000000000 --- a/internal/commands/fakes/ops_manager_release_cache_source.go +++ /dev/null @@ -1,336 +0,0 @@ -// Code generated by counterfeiter. DO NOT EDIT. -package fakes - -import ( - "sync" - - "github.com/pivotal-cf/kiln/internal/commands" - "github.com/pivotal-cf/om/api" -) - -type OpsManagerReleaseCacheSource struct { - GetBoshEnvironmentStub func() (api.GetBoshEnvironmentOutput, error) - getBoshEnvironmentMutex sync.RWMutex - getBoshEnvironmentArgsForCall []struct { - } - getBoshEnvironmentReturns struct { - result1 api.GetBoshEnvironmentOutput - result2 error - } - getBoshEnvironmentReturnsOnCall map[int]struct { - result1 api.GetBoshEnvironmentOutput - result2 error - } - GetSecurityRootCACertificateStub func() (string, error) - getSecurityRootCACertificateMutex sync.RWMutex - getSecurityRootCACertificateArgsForCall []struct { - } - getSecurityRootCACertificateReturns struct { - result1 string - result2 error - } - getSecurityRootCACertificateReturnsOnCall map[int]struct { - result1 string - result2 error - } - GetStagedProductByNameStub func(string) (api.StagedProductsFindOutput, error) - getStagedProductByNameMutex sync.RWMutex - getStagedProductByNameArgsForCall []struct { - arg1 string - } - getStagedProductByNameReturns struct { - result1 api.StagedProductsFindOutput - result2 error - } - getStagedProductByNameReturnsOnCall map[int]struct { - result1 api.StagedProductsFindOutput - result2 error - } - GetStagedProductManifestStub func(string) (string, error) - getStagedProductManifestMutex sync.RWMutex - getStagedProductManifestArgsForCall []struct { - arg1 string - } - getStagedProductManifestReturns struct { - result1 string - result2 error - } - getStagedProductManifestReturnsOnCall map[int]struct { - result1 string - result2 error - } - invocations map[string][][]interface{} - invocationsMutex sync.RWMutex -} - -func (fake *OpsManagerReleaseCacheSource) GetBoshEnvironment() (api.GetBoshEnvironmentOutput, error) { - fake.getBoshEnvironmentMutex.Lock() - ret, specificReturn := fake.getBoshEnvironmentReturnsOnCall[len(fake.getBoshEnvironmentArgsForCall)] - fake.getBoshEnvironmentArgsForCall = append(fake.getBoshEnvironmentArgsForCall, struct { - }{}) - stub := fake.GetBoshEnvironmentStub - fakeReturns := fake.getBoshEnvironmentReturns - fake.recordInvocation("GetBoshEnvironment", []interface{}{}) - fake.getBoshEnvironmentMutex.Unlock() - if stub != nil { - return stub() - } - if specificReturn { - return ret.result1, ret.result2 - } - return fakeReturns.result1, fakeReturns.result2 -} - -func (fake *OpsManagerReleaseCacheSource) GetBoshEnvironmentCallCount() int { - fake.getBoshEnvironmentMutex.RLock() - defer fake.getBoshEnvironmentMutex.RUnlock() - return len(fake.getBoshEnvironmentArgsForCall) -} - -func (fake *OpsManagerReleaseCacheSource) GetBoshEnvironmentCalls(stub func() (api.GetBoshEnvironmentOutput, error)) { - fake.getBoshEnvironmentMutex.Lock() - defer fake.getBoshEnvironmentMutex.Unlock() - fake.GetBoshEnvironmentStub = stub -} - -func (fake *OpsManagerReleaseCacheSource) GetBoshEnvironmentReturns(result1 api.GetBoshEnvironmentOutput, result2 error) { - fake.getBoshEnvironmentMutex.Lock() - defer fake.getBoshEnvironmentMutex.Unlock() - fake.GetBoshEnvironmentStub = nil - fake.getBoshEnvironmentReturns = struct { - result1 api.GetBoshEnvironmentOutput - result2 error - }{result1, result2} -} - -func (fake *OpsManagerReleaseCacheSource) GetBoshEnvironmentReturnsOnCall(i int, result1 api.GetBoshEnvironmentOutput, result2 error) { - fake.getBoshEnvironmentMutex.Lock() - defer fake.getBoshEnvironmentMutex.Unlock() - fake.GetBoshEnvironmentStub = nil - if fake.getBoshEnvironmentReturnsOnCall == nil { - fake.getBoshEnvironmentReturnsOnCall = make(map[int]struct { - result1 api.GetBoshEnvironmentOutput - result2 error - }) - } - fake.getBoshEnvironmentReturnsOnCall[i] = struct { - result1 api.GetBoshEnvironmentOutput - result2 error - }{result1, result2} -} - -func (fake *OpsManagerReleaseCacheSource) GetSecurityRootCACertificate() (string, error) { - fake.getSecurityRootCACertificateMutex.Lock() - ret, specificReturn := fake.getSecurityRootCACertificateReturnsOnCall[len(fake.getSecurityRootCACertificateArgsForCall)] - fake.getSecurityRootCACertificateArgsForCall = append(fake.getSecurityRootCACertificateArgsForCall, struct { - }{}) - stub := fake.GetSecurityRootCACertificateStub - fakeReturns := fake.getSecurityRootCACertificateReturns - fake.recordInvocation("GetSecurityRootCACertificate", []interface{}{}) - fake.getSecurityRootCACertificateMutex.Unlock() - if stub != nil { - return stub() - } - if specificReturn { - return ret.result1, ret.result2 - } - return fakeReturns.result1, fakeReturns.result2 -} - -func (fake *OpsManagerReleaseCacheSource) GetSecurityRootCACertificateCallCount() int { - fake.getSecurityRootCACertificateMutex.RLock() - defer fake.getSecurityRootCACertificateMutex.RUnlock() - return len(fake.getSecurityRootCACertificateArgsForCall) -} - -func (fake *OpsManagerReleaseCacheSource) GetSecurityRootCACertificateCalls(stub func() (string, error)) { - fake.getSecurityRootCACertificateMutex.Lock() - defer fake.getSecurityRootCACertificateMutex.Unlock() - fake.GetSecurityRootCACertificateStub = stub -} - -func (fake *OpsManagerReleaseCacheSource) GetSecurityRootCACertificateReturns(result1 string, result2 error) { - fake.getSecurityRootCACertificateMutex.Lock() - defer fake.getSecurityRootCACertificateMutex.Unlock() - fake.GetSecurityRootCACertificateStub = nil - fake.getSecurityRootCACertificateReturns = struct { - result1 string - result2 error - }{result1, result2} -} - -func (fake *OpsManagerReleaseCacheSource) GetSecurityRootCACertificateReturnsOnCall(i int, result1 string, result2 error) { - fake.getSecurityRootCACertificateMutex.Lock() - defer fake.getSecurityRootCACertificateMutex.Unlock() - fake.GetSecurityRootCACertificateStub = nil - if fake.getSecurityRootCACertificateReturnsOnCall == nil { - fake.getSecurityRootCACertificateReturnsOnCall = make(map[int]struct { - result1 string - result2 error - }) - } - fake.getSecurityRootCACertificateReturnsOnCall[i] = struct { - result1 string - result2 error - }{result1, result2} -} - -func (fake *OpsManagerReleaseCacheSource) GetStagedProductByName(arg1 string) (api.StagedProductsFindOutput, error) { - fake.getStagedProductByNameMutex.Lock() - ret, specificReturn := fake.getStagedProductByNameReturnsOnCall[len(fake.getStagedProductByNameArgsForCall)] - fake.getStagedProductByNameArgsForCall = append(fake.getStagedProductByNameArgsForCall, struct { - arg1 string - }{arg1}) - stub := fake.GetStagedProductByNameStub - fakeReturns := fake.getStagedProductByNameReturns - fake.recordInvocation("GetStagedProductByName", []interface{}{arg1}) - fake.getStagedProductByNameMutex.Unlock() - if stub != nil { - return stub(arg1) - } - if specificReturn { - return ret.result1, ret.result2 - } - return fakeReturns.result1, fakeReturns.result2 -} - -func (fake *OpsManagerReleaseCacheSource) GetStagedProductByNameCallCount() int { - fake.getStagedProductByNameMutex.RLock() - defer fake.getStagedProductByNameMutex.RUnlock() - return len(fake.getStagedProductByNameArgsForCall) -} - -func (fake *OpsManagerReleaseCacheSource) GetStagedProductByNameCalls(stub func(string) (api.StagedProductsFindOutput, error)) { - fake.getStagedProductByNameMutex.Lock() - defer fake.getStagedProductByNameMutex.Unlock() - fake.GetStagedProductByNameStub = stub -} - -func (fake *OpsManagerReleaseCacheSource) GetStagedProductByNameArgsForCall(i int) string { - fake.getStagedProductByNameMutex.RLock() - defer fake.getStagedProductByNameMutex.RUnlock() - argsForCall := fake.getStagedProductByNameArgsForCall[i] - return argsForCall.arg1 -} - -func (fake *OpsManagerReleaseCacheSource) GetStagedProductByNameReturns(result1 api.StagedProductsFindOutput, result2 error) { - fake.getStagedProductByNameMutex.Lock() - defer fake.getStagedProductByNameMutex.Unlock() - fake.GetStagedProductByNameStub = nil - fake.getStagedProductByNameReturns = struct { - result1 api.StagedProductsFindOutput - result2 error - }{result1, result2} -} - -func (fake *OpsManagerReleaseCacheSource) GetStagedProductByNameReturnsOnCall(i int, result1 api.StagedProductsFindOutput, result2 error) { - fake.getStagedProductByNameMutex.Lock() - defer fake.getStagedProductByNameMutex.Unlock() - fake.GetStagedProductByNameStub = nil - if fake.getStagedProductByNameReturnsOnCall == nil { - fake.getStagedProductByNameReturnsOnCall = make(map[int]struct { - result1 api.StagedProductsFindOutput - result2 error - }) - } - fake.getStagedProductByNameReturnsOnCall[i] = struct { - result1 api.StagedProductsFindOutput - result2 error - }{result1, result2} -} - -func (fake *OpsManagerReleaseCacheSource) GetStagedProductManifest(arg1 string) (string, error) { - fake.getStagedProductManifestMutex.Lock() - ret, specificReturn := fake.getStagedProductManifestReturnsOnCall[len(fake.getStagedProductManifestArgsForCall)] - fake.getStagedProductManifestArgsForCall = append(fake.getStagedProductManifestArgsForCall, struct { - arg1 string - }{arg1}) - stub := fake.GetStagedProductManifestStub - fakeReturns := fake.getStagedProductManifestReturns - fake.recordInvocation("GetStagedProductManifest", []interface{}{arg1}) - fake.getStagedProductManifestMutex.Unlock() - if stub != nil { - return stub(arg1) - } - if specificReturn { - return ret.result1, ret.result2 - } - return fakeReturns.result1, fakeReturns.result2 -} - -func (fake *OpsManagerReleaseCacheSource) GetStagedProductManifestCallCount() int { - fake.getStagedProductManifestMutex.RLock() - defer fake.getStagedProductManifestMutex.RUnlock() - return len(fake.getStagedProductManifestArgsForCall) -} - -func (fake *OpsManagerReleaseCacheSource) GetStagedProductManifestCalls(stub func(string) (string, error)) { - fake.getStagedProductManifestMutex.Lock() - defer fake.getStagedProductManifestMutex.Unlock() - fake.GetStagedProductManifestStub = stub -} - -func (fake *OpsManagerReleaseCacheSource) GetStagedProductManifestArgsForCall(i int) string { - fake.getStagedProductManifestMutex.RLock() - defer fake.getStagedProductManifestMutex.RUnlock() - argsForCall := fake.getStagedProductManifestArgsForCall[i] - return argsForCall.arg1 -} - -func (fake *OpsManagerReleaseCacheSource) GetStagedProductManifestReturns(result1 string, result2 error) { - fake.getStagedProductManifestMutex.Lock() - defer fake.getStagedProductManifestMutex.Unlock() - fake.GetStagedProductManifestStub = nil - fake.getStagedProductManifestReturns = struct { - result1 string - result2 error - }{result1, result2} -} - -func (fake *OpsManagerReleaseCacheSource) GetStagedProductManifestReturnsOnCall(i int, result1 string, result2 error) { - fake.getStagedProductManifestMutex.Lock() - defer fake.getStagedProductManifestMutex.Unlock() - fake.GetStagedProductManifestStub = nil - if fake.getStagedProductManifestReturnsOnCall == nil { - fake.getStagedProductManifestReturnsOnCall = make(map[int]struct { - result1 string - result2 error - }) - } - fake.getStagedProductManifestReturnsOnCall[i] = struct { - result1 string - result2 error - }{result1, result2} -} - -func (fake *OpsManagerReleaseCacheSource) Invocations() map[string][][]interface{} { - fake.invocationsMutex.RLock() - defer fake.invocationsMutex.RUnlock() - fake.getBoshEnvironmentMutex.RLock() - defer fake.getBoshEnvironmentMutex.RUnlock() - fake.getSecurityRootCACertificateMutex.RLock() - defer fake.getSecurityRootCACertificateMutex.RUnlock() - fake.getStagedProductByNameMutex.RLock() - defer fake.getStagedProductByNameMutex.RUnlock() - fake.getStagedProductManifestMutex.RLock() - defer fake.getStagedProductManifestMutex.RUnlock() - copiedInvocations := map[string][][]interface{}{} - for key, value := range fake.invocations { - copiedInvocations[key] = value - } - return copiedInvocations -} - -func (fake *OpsManagerReleaseCacheSource) recordInvocation(key string, args []interface{}) { - fake.invocationsMutex.Lock() - defer fake.invocationsMutex.Unlock() - if fake.invocations == nil { - fake.invocations = map[string][][]interface{}{} - } - if fake.invocations[key] == nil { - fake.invocations[key] = [][]interface{}{} - } - fake.invocations[key] = append(fake.invocations[key], args) -} - -var _ commands.OpsManagerReleaseCacheSource = new(OpsManagerReleaseCacheSource) diff --git a/internal/commands/generate_osm_manifest.go b/internal/commands/generate_osm_manifest.go index 6438bb041..e7bdb141e 100644 --- a/internal/commands/generate_osm_manifest.go +++ b/internal/commands/generate_osm_manifest.go @@ -35,6 +35,13 @@ type OSM struct { } } +//counterfeiter:generate -o ./fakes/release_storage.go --fake-name ReleaseStorage . ReleaseStorage + +type ReleaseStorage interface { + component.ReleaseSource + UploadRelease(spec cargo.BOSHReleaseTarballSpecification, file io.Reader) (cargo.BOSHReleaseTarballLock, error) +} + func NewOSM(outLogger *log.Logger, rs component.ReleaseSource) *OSM { if rs == nil { rs = component.NewBOSHIOReleaseSource(cargo.ReleaseSourceConfig{}, "", outLogger) diff --git a/internal/om/client.go b/internal/om/client.go deleted file mode 100644 index 136ab56c9..000000000 --- a/internal/om/client.go +++ /dev/null @@ -1,230 +0,0 @@ -package om - -import ( - "context" - "crypto/tls" - "crypto/x509" - "errors" - "fmt" - "io" - "log" - "net" - "net/http" - "net/url" - "os" - "time" - - boshdir "github.com/cloudfoundry/bosh-cli/director" - boshuaa "github.com/cloudfoundry/bosh-cli/uaa" - boshhttpclient "github.com/cloudfoundry/bosh-utils/httpclient" - boshlog "github.com/cloudfoundry/bosh-utils/logger" - "github.com/pivotal-cf/om/api" - "github.com/pivotal-cf/om/network" - "golang.org/x/crypto/ssh" -) - -type ClientConfiguration struct { - Target string `long:"om-target" env:"OM_TARGET"` - Username string `long:"om-username" env:"OM_USERNAME"` - Password string `long:"om-password" env:"OM_PASSWORD"` - CACert string `long:"om-ca-cert" env:"OM_CA_CERT"` - PrivateKey string `long:"om-private-key" env:"OM_PRIVATE_KEY"` - - SkipSSLValidation bool `long:"om-skip-ssl-validation" env:"OM_SKIP_SSL_VALIDATION" default:"false"` -} - -func (conf ClientConfiguration) API() (api.Api, error) { - var ( - // OM_CONNECT_TIMEOUT - connectTimeout = 10 * time.Second - - // OM_REQUEST_TIMEOUT - requestTimeout = 1800 * time.Second - - unauthenticatedClient, authedClient, unauthenticatedProgressClient, authedProgressClient interface { - Do(*http.Request) (*http.Response, error) - } - ) - - var err error - unauthenticatedClient, err = network.NewUnauthenticatedClient(conf.Target, conf.SkipSSLValidation, conf.CACert, connectTimeout, requestTimeout) - if err != nil { - return api.Api{}, err - } - - authedClient, err = network.NewOAuthClient(conf.Target, conf.Username, conf.Password, "", "", conf.SkipSSLValidation, conf.CACert, connectTimeout, requestTimeout) - if err != nil { - return api.Api{}, err - } - - unauthenticatedProgressClient = network.NewProgressClient(unauthenticatedClient, os.Stderr) - authedProgressClient = network.NewProgressClient(authedClient, os.Stderr) - - return api.New(api.ApiInput{ - Client: authedClient, - UnauthedClient: unauthenticatedClient, - ProgressClient: authedProgressClient, - UnauthedProgressClient: unauthenticatedProgressClient, - Logger: log.New(os.Stderr, "", 0), - }), nil -} - -type GetBoshEnvironmentAndSecurityRootCACertificateProvider interface { - GetSecurityRootCACertificate() (string, error) - GetBoshEnvironment() (api.GetBoshEnvironmentOutput, error) -} - -func BoshDirector(omConfig ClientConfiguration, omAPI GetBoshEnvironmentAndSecurityRootCACertificateProvider) (boshdir.Director, error) { - omPrivateKey, err := ssh.ParsePrivateKey([]byte(omConfig.PrivateKey)) - if err != nil { - return nil, fmt.Errorf("failed to parse private key from OM_PRIVATE_KEY: %s", err) - } - - target, err := url.Parse(omConfig.Target) - if err != nil { - return nil, fmt.Errorf("failed to parse om target: %s", err) - } - serverURL := target.Hostname() + ":22" - - hostKey, err := insecureGetHostKey(omPrivateKey, serverURL) - if err != nil { - return nil, fmt.Errorf("failed to get om ssh host key: %s", err) - } - - socksClient, err := ssh.Dial("tcp", serverURL, &ssh.ClientConfig{ - Timeout: 30 * time.Second, - User: "ubuntu", - HostKeyCallback: ssh.FixedHostKey(hostKey), - Auth: []ssh.AuthMethod{ssh.PublicKeys(omPrivateKey)}, - }) - if err != nil { - return nil, fmt.Errorf("failed to connect to ops manager: %s", err) - } - - boshRootCA, err := omAPI.GetSecurityRootCACertificate() - if err != nil { - return nil, err - } - - pool := x509.NewCertPool() - if !pool.AppendCertsFromPEM([]byte(boshRootCA)) { - return nil, fmt.Errorf("failed to append ops manager root CA") - } - - httpClient := &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{ - ClientCAs: pool, - InsecureSkipVerify: true, - }, - DialContext: func(_ context.Context, network, addr string) (net.Conn, error) { - return socksClient.Dial(network, addr) - }, - TLSHandshakeTimeout: 30 * time.Second, - DisableKeepAlives: false, - }, - } - - boshEnv, err := omAPI.GetBoshEnvironment() - if err != nil { - return nil, fmt.Errorf("failed to get bosh env: %s", err) - } - - directorConfig, err := boshdir.NewConfigFromURL(boshEnv.Environment) - if err != nil { - return nil, err - } - - boshLogger := boshlog.NewLogger(boshlog.LevelError) - boshFactory := boshdir.NewFactory(boshLogger) - - boshDirectorURL := fmt.Sprintf("https://%s:%d", directorConfig.Host, directorConfig.Port) - - unAuthedDirector := boshdir.NewClient(boshDirectorURL, boshhttpclient.NewHTTPClient(httpClient, boshLogger), - boshdir.NewNoopTaskReporter(), boshdir.NewNoopFileReporter(), boshLogger, - ) - - info, err := unAuthedDirector.Info() - if err != nil { - return nil, fmt.Errorf("could not get basic director info: %s", err) - } - - uaaClientFactory := boshuaa.NewFactory(boshLogger) - - uaaURL, err := getAuthURLFromInfo(info) - if err != nil { - return nil, fmt.Errorf("could not get basic director info: %s", err) - } - - uaaConfig, err := boshuaa.NewConfigFromURL(uaaURL) - if err != nil { - return nil, err - } - - uaaConfig.CACert = boshRootCA - uaaConfig.Client = boshEnv.Client - uaaConfig.ClientSecret = boshEnv.ClientSecret - - uaa, err := uaaClientFactory.New(uaaConfig) - if err != nil { - return nil, fmt.Errorf("could not build uaa auth from director info: %s", err) - } - - directorConfig.TokenFunc = boshuaa.NewClientTokenSession(uaa).TokenFunc - directorConfig.CACert = uaaConfig.CACert - - dir, err := boshFactory.New(directorConfig, boshdir.NewNoopTaskReporter(), boshdir.NewNoopFileReporter()) - if err != nil { - return nil, err - } - - return dir, nil -} - -// insecureGetHostKey just returns the key returned by the host and does not -// attempt to ensure the key is from whom it says it is from. This is what the -// bosh CLI does, so it seems to be secure enough. -func insecureGetHostKey(signer ssh.Signer, serverURL string) (ssh.PublicKey, error) { - publicKeyChannel := make(chan ssh.PublicKey, 1) - defer close(publicKeyChannel) - - dialErrorChannel := make(chan error) - defer close(dialErrorChannel) - - clientConfig := &ssh.ClientConfig{ - Timeout: time.Minute, - User: "ubuntu", - HostKeyCallback: func(_ string, _ net.Addr, key ssh.PublicKey) error { - publicKeyChannel <- key - return nil - }, - Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)}, - } - - go func() { - conn, err := ssh.Dial("tcp", serverURL, clientConfig) - if err != nil { - publicKeyChannel <- nil - dialErrorChannel <- err - return - } - defer closeAndIgnoreError(conn) - dialErrorChannel <- nil - }() - - return <-publicKeyChannel, <-dialErrorChannel -} - -func getAuthURLFromInfo(info boshdir.InfoResp) (string, error) { - v, ok := info.Auth.Options["url"] - if !ok { - return "", errors.New("missing uaa auth url") - } - s, ok := v.(string) - if !ok { - return "", errors.New("uaa auth url has the wrong type") - } - return s, nil -} - -func closeAndIgnoreError(c io.Closer) { _ = c.Close() } diff --git a/main.go b/main.go index 81c7663e8..75e3ff751 100644 --- a/main.go +++ b/main.go @@ -107,8 +107,6 @@ func main() { commandSet["find-stemcell-version"] = commands.NewFindStemcellVersion(outLogger, pivnetService) - commandSet["cache-compiled-releases"] = commands.NewCacheCompiledReleases().WithLogger(outLogger) - commandSet["validate"] = commands.NewValidate(osfs.New("")) commandSet["release-notes"], err = commands.NewReleaseNotesCommand() if err != nil { diff --git a/pkg/cargo/opsman/resource_config.go b/pkg/cargo/opsman/resource_config.go deleted file mode 100644 index 0bc1fefc8..000000000 --- a/pkg/cargo/opsman/resource_config.go +++ /dev/null @@ -1,14 +0,0 @@ -package opsman - -type ResourceConfig struct { - Name string - Instances ResourceConfigInstances -} - -type ResourceConfigInstances struct { - Value int -} - -func (rci ResourceConfigInstances) IsAutomatic() bool { - return rci.Value < 0 -} diff --git a/pkg/cargo/opsman/stemcell.go b/pkg/cargo/opsman/stemcell.go deleted file mode 100644 index e2e6feab7..000000000 --- a/pkg/cargo/opsman/stemcell.go +++ /dev/null @@ -1,10 +0,0 @@ -package opsman - -type Stemcell struct { - File string - Hypervisor string - Infrastructure string - Name string - OS string - Version string -}