buildpack.yml.erb

<% buildpacks = {
  'apt' => {
    'stacks' => %w(cflinuxfs3 cflinuxfs4),
    'non_lts' => true,
  },
  'binary' => {
    'stacks' => %w(cflinuxfs3 cflinuxfs4 windows),
    'product_slug' => 'binary-buildpack',
    'skip_docker_start' => true,
  },
  'dotnet-core' => {
    'stacks' => %w(cflinuxfs3 cflinuxfs4),
    'product_slug' => 'dotnet-core-buildpack',
    'compute_instance_count' => 3
  },
  'go' => {
    'stacks' => %w(cflinuxfs3 cflinuxfs4),
    'product_slug' => 'go-buildpack',
    'compute_instance_count' => 3
  },
  'hwc' => {
    'stacks' => %w(windows),
    'skip_brats' => true,
    'product_slug' => 'hwc-buildpack',
    'non_lts' => true,
  },
  'nginx' => {
    'stacks' => %w(cflinuxfs3 cflinuxfs4),
    'product_slug' => 'nginx-buildpack',
  },
  'nodejs' => {
    'stacks' => %w(cflinuxfs3 cflinuxfs4),
    'product_slug' => 'nodejs-buildpack',
  },
  'php' => {
    'stacks' => %w(cflinuxfs3 cflinuxfs4),
    'product_slug' => 'php-buildpack',
    'compile_extensions' => true,
  },
  'python' => {
    'stacks' => %w(cflinuxfs3 cflinuxfs4),
    'product_slug' => 'python-buildpack',
  },
  'r' => {
    'stacks' => %w(cflinuxfs3 cflinuxfs4),
    'product_slug' => 'r-buildpack',
    'skip_brats' => true,
    'non_lts' => true,
    'skip_docker_start' => true,
  },
  'ruby' => {
    'stacks' => %w(cflinuxfs3 cflinuxfs4),
    'product_slug' => 'ruby-buildpack',
    'compute_instance_count' => 3
  },
  'staticfile' => {
    'stacks' => %w(cflinuxfs3 cflinuxfs4),
    'product_slug' => 'staticfile-buildpack',
  }
} %>
<% tas_version =  '4.0' %>
<% tas_pool_name = 'tas_4' %>
<% low_nodes_lts = ['ruby', 'python', 'dotnet-core', 'php'] %>
---
resource_types:
  - name: cron
    type: docker-image
    source:
      repository: cfbuildpacks/cron-resource

  - name: shepherd
    type: registry-image
    source:
      repository: us-west2-docker.pkg.dev/shepherd-268822/shepherd2/concourse-resource
      tag: v1

  - name: pool
    type: registry-image
    source:
      repository: concourse/pool-resource
      tag: 1.1.3

resources: ############################################################################################################
## Git ##

  - name: buildpack-develop
    type: git
    webhook_token: ob0aigh3
    source:
      uri: git@github.com:<%= organization %>/<%= language %>-buildpack.git
      private_key: ((cloudfoundry-<%= language %>-buildpack-deploy-key.private_key))
      branch: develop
      ignore_paths:
        - VERSION
        - CHANGELOG

  - name: cf-deployment
    type: git
    source:
      uri: https://github.com/cloudfoundry/cf-deployment

  - name: buildpack-new-version
    type: git
    source:
      uri: git@github.com:<%= organization %>/<%= language %>-buildpack.git
      private_key: ((cloudfoundry-<%= language %>-buildpack-deploy-key.private_key))
      branch: develop
      paths:
        - VERSION
        - CHANGELOG

  - name: buildpack-master
    type: git
    source:
      uri: git@github.com:<%= organization %>/<%= language %>-buildpack.git
      private_key: ((cloudfoundry-<%= language %>-buildpack-deploy-key.private_key))
      branch: master

  - name: buildpacks-ci
    type: git
    source:
      uri: https://github.com/cloudfoundry/buildpacks-ci
      branch: master

  - name: libbuildpack
    type: git
    webhook_token: ob0aigh3
    source:
      uri: https://github.com/cloudfoundry/libbuildpack.git
<% if buildpacks[language]['compile_extensions'] %>
  - name: compile-extensions
    type: git
    source:
      uri: https://github.com/cloudfoundry/compile-extensions.git
<% end %>

<% if language == 'hwc' %>
  - name: buildpack-bosh-release
    type: git
    source:
      uri: git@github.com:<%= organization %>/<%= language %>-buildpack-release.git
      private_key: ((cloudfoundry-<%= language %>-buildpack-bosh-release-deploy-key.private_key))
      branch: master
<% end %>
  - name: env-repo
    type: git
    source:
      uri: git@github.com:cloudfoundry/buildpacks-envs
      branch: master
      private_key: '((cloudfoundry-buildpacks-envs-deploy-key.private_key))'


  ## Github Releases ##

  - name: buildpack-github-release
    type: github-release
    source:
      user: {{buildpacks-github-org}}
      repository: <%= language %>-buildpack
      access_token: ((buildpacks-github-token))

<% if language == 'hwc' %>
  - name: buildpack-bosh-release-github-release
    type: github-release
    source:
      user: {{buildpacks-github-org}}
      repository: <%= language %>-buildpack-release
      access_token: ((buildpacks-github-token))
<% end %>

<% if buildpacks[language]['compile_extensions'] %>
  - name: buildpack-packager
    type: github-release
    source:
      user: {{buildpacks-github-org}}
      repository: buildpack-packager
      access_token: ((buildpacks-github-token))
<% end %>

  ## S3 Buckets ##

<% buildpacks[language]['stacks'].each do |stack| %>
  - name: pivotal-buildpack-<%= stack %>
    type: s3
    source:
      bucket: {{buildpack-release-candidates-bucket}}
      regexp: <%= language%>/<%= language %>_buildpack-<%= stack %>-v(.*).zip
      access_key_id: ((svc-buildpacks-aws-bp-ci-access-key))
      secret_access_key: ((svc-buildpacks-aws-bp-ci-secret-key))
      aws_role_arn: ((svc-buildpacks-aws-bp-ci-assume-role-arn))

  - name: pivotal-buildpack-cached-<%= stack %>
    type: s3
    source:
      bucket: {{buildpack-release-candidates-bucket}}
      regexp: <%= language%>/<%= language %>_buildpack-cached-<%= stack %>-v(.*).zip
      access_key_id: ((svc-buildpacks-aws-bp-ci-access-key))
      secret_access_key: ((svc-buildpacks-aws-bp-ci-secret-key))
      aws_role_arn: ((svc-buildpacks-aws-bp-ci-assume-role-arn))
<% end %>

<% if language.to_s == 'hwc' || language.to_s == 'binary' %>
  - name: pivotal-buildpack-cached-any
    type: s3
    source:
      bucket: {{buildpack-release-candidates-bucket}}
      regexp: <%= language%>/<%= language %>_buildpack-cached-v(.*).zip
      access_key_id: ((svc-buildpacks-aws-bp-ci-access-key))
      secret_access_key: ((svc-buildpacks-aws-bp-ci-secret-key))
      aws_role_arn: ((svc-buildpacks-aws-bp-ci-assume-role-arn))

  - name: pivotal-buildpack-any
    type: s3
    source:
      bucket: {{buildpack-release-candidates-bucket}}
      regexp: <%= language%>/<%= language %>_buildpack-v(.*).zip
      access_key_id: ((svc-buildpacks-aws-bp-ci-access-key))
      secret_access_key: ((svc-buildpacks-aws-bp-ci-secret-key))
      aws_role_arn: ((svc-buildpacks-aws-bp-ci-assume-role-arn))
<% end %>

  ## Resource Pools ##

  - name: shepherd-cf-environment
    type: shepherd
    source:
      url: https://v2.shepherd.run
      service-account-key: ((shepherd-buildpacks-service-account-key))
      lease:
        namespace: buildpacks
        pool:
          namespace: official
          name: cfd
      compatibility-mode: environments-app

<% unless buildpacks[language]['non_lts'] %>
  - name: shepherd-tas-<%= tas_version %>-environment
    type: shepherd
    source:
      url: https://v2.shepherd.run
      service-account-key: ((shepherd-buildpacks-service-account-key))
      lease:
        namespace: buildpacks
        pool:
          namespace: buildpacks
          name: <%= tas_pool_name %>
      compatibility-mode: environments-app
<% end %>

jobs: ################################################################################################################
<% if language == "php" %>
  - name: ensure-manifest-has-modules
    serial: true
    public: true
    plan:
      - in_parallel:
        - get: buildpacks-ci
        - get: buildpack
          resource: buildpack-develop
          trigger: true
      - do:
        - task: check-modules
          file: buildpacks-ci/tasks/check-php-modules-in-manifest/task.yml
<% end %>
  - name: specs-unit-develop
    serial: true
    public: true
    plan:
      - do:
        - in_parallel:
          - get: buildpacks-ci
          - get: buildpack
            resource: buildpack-develop
<% if language == "php" %>
            passed: [ ensure-manifest-has-modules ]
<% end %>
            trigger: true
        - do:
          - task: ginkgo
            file: buildpacks-ci/tasks/run-buildpack-unit-specs/task.yml
<% if language == "php" %>
            privileged: true
            params:
              DOCKER_START: true
<% end %>

  - name: detect-new-version-and-upload-artifacts
    serial: true
    public: true
    plan:
      - in_parallel:
        - get: buildpacks-ci
        - get: buildpack
          resource: buildpack-master
          trigger: true
<% bucket_stacks = buildpacks[language]['stacks']
   bucket_stacks = bucket_stacks + ['any'] if language.to_s == 'hwc' || language.to_s == 'binary'
   bucket_stacks.each do |stack| %> #change bucket_stacks to buildpacks[language]['stacks'] when removing any_stack
        - get: pivotal-buildpack-cached-<%= stack %>
        - get: pivotal-buildpack-<%= stack %>
<% end %>
      - do:
        - in_parallel:
<% bucket_stacks = buildpacks[language]['stacks']
   bucket_stacks = bucket_stacks + ['any'] if language.to_s == 'hwc' || language.to_s == 'binary'
   bucket_stacks.each do |stack| %> #change bucket_stacks to buildpacks[language]['stacks'] when removing any_stack
          - do:
            - task: detect-and-upload-<%= stack %>
              file: buildpacks-ci/tasks/detect-and-upload/task.yml
              params:
                GIT_REPO_ORG: <%= organization %>
                BUILDPACK_NAME: <%= language %>-buildpack
                RUBYGEM_MIRROR: {{rubygem-mirror}}
                CF_STACK: <%= stack %>
              input_mapping:
                pivotal-buildpack: pivotal-buildpack-<%= stack %>
                pivotal-buildpack-cached: pivotal-buildpack-cached-<%= stack %>
              output_mapping:
                buildpack-artifacts: buildpack-artifacts-<%= stack %>
            - put: pivotal-buildpack-<%= stack %>
              params:
                file: buildpack-artifacts-<%= stack %>/uncached/*_buildpack*-v*.zip
            - put: pivotal-buildpack-cached-<%= stack %>
              params:
                file: buildpack-artifacts-<%= stack %>/cached/*_buildpack-cached*-v*.zip
<% end %>
  - name: ship-it
    serial: true
    public: true
    plan:
      - in_parallel:
        - get: buildpacks-ci
        <%
        all_stacks = buildpacks[language]['stacks']
        all_stacks = all_stacks + ['any'] if language.to_s == 'hwc' || language.to_s == 'binary'
        all_stacks.each_with_index do |stack, ndx|
        %>
        - get: pivotal-buildpack-cached-<%= stack %>
          passed:
            - detect-new-version-and-upload-artifacts
        - get: pivotal-buildpack-<%= stack %>
          passed:
            - detect-new-version-and-upload-artifacts
        <% end %>
        - get: buildpack
          passed:
            - detect-new-version-and-upload-artifacts
          resource: buildpack-master
      - task: check-tag-not-already-added
        file: buildpacks-ci/tasks/check-tag-not-already-added/task.yml
      - put: buildpack-master
        params:
          repository: buildpack
          tag: buildpack/VERSION
          tag_prefix: v
  - name: buildpack-to-github
    serial: true
    public: true
    plan:
      - in_parallel:
        - get: buildpacks-ci
        <% buildpacks[language]['stacks'].each_with_index do |stack, ndx| %>
        - get: uncached-buildpack-for-stack<%= ndx.to_s %>
          resource: pivotal-buildpack-<%= stack %>
          passed: [ ship-it ]
        <% end %>
        - get: buildpack
          resource: buildpack-master
          passed: [ ship-it ]
          trigger: true
      - task: finalize-buildpack
        file: buildpacks-ci/tasks/finalize-buildpack/task.yml
      - put: buildpack-github-release
        params:
          name: buildpack-artifacts/tag
          tag: buildpack-artifacts/tag
          body: buildpack-artifacts/RECENT_CHANGES
          globs:
            - buildpack-artifacts/*-buildpack*-v*.zip
            - buildpack-artifacts/*-buildpack*-v*.zip.SHA256SUM.txt

<% relevant_stacks = buildpacks[language]['stacks'] %>
<% relevant_stacks.each do |stack| %>
  - name: specs-edge-integration-develop-<%= stack %>
    serial: true
    public: true
    plan:
      - in_parallel:
        - put: environment
          resource: shepherd-cf-environment
          params:
            action: create
            duration: 6h
            resource: shepherd-cf-environment
            description: |
              Running <%= language %>-buildpack specs-edge-integration-develop-<%= stack %> job.
          timeout: 6h
        - get: buildpacks-ci
        - get: env-repo
        - get: buildpack
          resource: buildpack-develop
          trigger: true
          passed:
          - specs-unit-develop
        - get: repo
          resource: buildpack-develop
        - get: cf-deployment
      - task: redeploy
        file: buildpacks-ci/tasks/cf/redeploy/task.yml
        input_mapping:
          ci: buildpacks-ci
          lock: environment
        params:
          SCALE_DIEGO_CELLS: true
<% if stack == "cflinuxfs3" %>
          ADD_CFLINUXFS3_STACK: true
<% end %>
<% if buildpacks[language]['stacks'].include?('windows')%>
          DEPLOY_WINDOWS_CELL: true
<% end %>
      - do:
        - task: create-cf-space
          attempts: 5
          file: buildpacks-ci/tasks/cf/create-space/task.yml
          input_mapping:
            ci: buildpacks-ci
            lock: environment
          params:
            DOMAIN: 'cf-app.com'
            ORG: pivotal
    <% unless buildpacks[language]['skip_brats'] %>
    <% if buildpacks[language]['stacks'].include?('cflinuxfs4')%>
        - task: configure-brats-test-config
          config:
            platform: linux
            image_resource:
              type: docker-image
              source:
                repository: cfbuildpacks/ci
            inputs:
            - name: buildpack
            outputs:
            - name: buildpack
            run:
              dir: ""
              path: bash
              args:
              - -c
              - |
                #!/bin/bash
                set -e
                cd buildpack
                contents="$(jq '.stack = "<%= stack %>"' config.json)"
                echo -E "${contents}" > config.json
                echo -e "config.json modified to:\n $(cat config.json)"
      <% end %>
        - task: brats-<%= stack %>
          file: buildpacks-ci/tasks/<%= language.to_s == 'php' ? 'run-bp-brats-jammy' : 'run-bp-brats' %>/task.yml
          input_mapping: {cf-space: space}
          attempts: 5
          params:
            CF_STACK: <%= stack %>
            GINKGO_ATTEMPTS: 4
            GINKGO_NODES: 3
          <% end %>
        - task: integration-test-<%= stack %>
          file: buildpacks-ci/tasks/run-buildpack-integration-specs/task.yml
          input_mapping: {cf-space: space}
          params:
            CF_STACK: <%= stack %>
            GINKGO_ATTEMPTS: 4
            GINKGO_NODES: 3
            <% if language == 'php' %>
            COMPOSER_GITHUB_OAUTH_TOKEN: '((composer-github-oauth-token))'
            <% end %>
            <% if buildpacks[language]['skip_docker_start'] %>
            SKIP_DOCKER_START: true
            <% else %>
          privileged: true
            <% end %>
        ensure:
          task: delete-cf-space
          file: buildpacks-ci/tasks/delete-cf-space/task.yml
          input_mapping: {cf-space: space}
    on_success:
      put: environment
      resource: shepherd-cf-environment
      params:
        action: release
        resource: environment
<% end %>

  - name: specs-edge-shared-develop
    serial: true
    public: true
    plan:
      - do:
        - get: buildpacks-ci
        - get: repo
          resource: buildpack-develop
          trigger: true
          passed:
<% relevant_stacks = buildpacks[language]['stacks'] %>
<% relevant_stacks.each do |stack| %>
          - specs-edge-integration-develop-<%= stack %>
<% end %>
        on_success:
          task: github-set-status-success
          file: buildpacks-ci/tasks/set-status-success/task.yml
          params:
            GITHUB_ACCESS_TOKEN: ((buildpacks-github-token))
            GITHUB_REPO: <%= organization %>/<%= language %>-buildpack
            GITHUB_STATUS_CONTEXT: 'buildpacks-ci/edge-develop'
            GITHUB_STATUS_DESCRIPTION: 'Buildpacks CI edge develop spec passed'
            PIPELINE_URI: {{buildpacks-ci-pipeline-uri}}

<% unless buildpacks[language]['non_lts'] %>
  - name: specs-lts-integration-develop
    serial: true
    public: true
    plan:
      - in_parallel:
        - put: environment
          resource: shepherd-tas-<%= tas_version %>-environment
          params:
            action: create
            duration: 6h
            resource: shepherd-tas-<%= tas_version %>-environment
            description: |
              Running <%= language %>-buildpack specs-lts-integration-develop job.
          timeout: 6h
        - get: buildpacks-ci
        - get: buildpack
          resource: buildpack-develop
          trigger: true
          passed:
          - specs-unit-develop
        - get: repo
          resource: buildpack-develop
      - task: configure-pas
        file: buildpacks-ci/tasks/configure-pas/task.yml
      <% if buildpacks[language]['compute_instance_count'] %>
        params:
          COMPUTE_INSTANCE_COUNT: <%= buildpacks[language]['compute_instance_count'] %>
      <% end %>
      - do:
        - task: create-cf-space
          file: buildpacks-ci/tasks/create-cf-space-toolsmiths/task.yml
          params:
            ORG: pivotal
        - task: ginkgo-cflinuxfs3
          file: buildpacks-ci/tasks/run-buildpack-integration-specs/task.yml
          # python tests on lts are flaky for yet-to-be-determined reasons.
          <% if language == 'python' %>
          attempts: 5
          <% end %>
          params:
            CF_STACK: cflinuxfs3
            GINKGO_ATTEMPTS: <%= language.to_s == 'php' ? 10 : 4 %>
            GINKGO_NODES: <%= low_nodes_lts.include?(language.to_s) ? 2 : 3 %>
            SKIP_WINDOWS_TESTS: true
            <% if language == 'php' %>
            COMPOSER_GITHUB_OAUTH_TOKEN: '((composer-github-oauth-token))'
            <% end %>
<% if buildpacks[language]['skip_docker_start'] %>
            SKIP_DOCKER_START: true
<% else %>
          privileged: true
<% end %>
<% unless buildpacks[language]['skip_brats'] %>
        - task: brats-cflinuxfs3
          attempts: 5
          file: buildpacks-ci/tasks/<%= language.to_s == 'php' ? 'run-bp-brats-jammy' : 'run-bp-brats' %>/task.yml
          params:
            CF_STACK: cflinuxfs3
            GINKGO_ATTEMPTS: <%= language.to_s == 'php' ? 10 : 4 %>
            GINKGO_NODES: 3
<% end %>
        ensure:
          task: delete-cf-space
          file: buildpacks-ci/tasks/delete-cf-space/task.yml
    ensure:
      put: environment
      resource: shepherd-tas-<%= tas_version %>-environment
      params:
        action: release
        resource: environment

  - name: specs-lts-gcp-develop
    serial: true
    public: true
    plan:
      - do:
        - get: buildpacks-ci
        - get: repo
          resource: buildpack-develop
          trigger: true
          passed:
          - specs-lts-integration-develop
        on_success:
          task: github-set-status-success
          file: buildpacks-ci/tasks/set-status-success/task.yml
          params:
            GITHUB_ACCESS_TOKEN: ((buildpacks-github-token))
            GITHUB_REPO: <%= organization %>/<%= language %>-buildpack
            GITHUB_STATUS_CONTEXT: 'buildpacks-ci/lts-develop'
            GITHUB_STATUS_DESCRIPTION: 'Buildpacks CI lts develop spec passed'
            PIPELINE_URI: {{buildpacks-ci-pipeline-uri}}
<% end %>
  - name: buildpack-to-master
    serial: true
    public: true
    plan:
      - in_parallel:
        - get: buildpacks-ci
        - get: repo
          trigger: true
          resource: buildpack-new-version
      - task: merge-to-master
        file: buildpacks-ci/tasks/buildpack-to-master/task.yml
        params:
          GITHUB_ACCESS_TOKEN: ((buildpacks-github-token))
          GITHUB_REPO: <%= organization %>/<%= language %>-buildpack
          GITHUB_STATUS_CONTEXT: 'buildpacks-ci/passed-dev-specs'
          GITHUB_STATUS_DESCRIPTION: 'Buildpacks CI develop specs passed'
          PIPELINE_URI: {{buildpacks-ci-pipeline-uri}}

  - name: update-libbuildpack
    serial: true
    public: true
    plan:
      - in_parallel:
        - get: buildpacks-ci
        - get: libbuildpack
          trigger: true
        - get: buildpack
          resource: buildpack-develop
      - task: update-libbuildpack
        file: buildpacks-ci/tasks/update-libbuildpack/task.yml
        <% if language == "php" %>
        privileged: true
        <% end %>
        params:
          <% if language == 'php' %>
          DOCKER_START: true
          <% end %>
          LANGUAGE: <%= language %>
      - put: buildpack-develop
        params:
          repository: buildpack
          rebase: true
<% if buildpacks[language]['compile_extensions'] %>
  - name: update-compile-extensions
    serial: true
    public: true
    plan:
      - in_parallel:
        - get: buildpacks-ci
        - get: compile-extensions
          trigger: true
        - get: buildpack
          resource: buildpack-develop
      - task: update-compile-extensions
        file: buildpacks-ci/tasks/update-compile-extensions/task.yml
        params: { RUBYGEM_MIRROR: {{rubygem-mirror}}  }
      - put: buildpack-develop
        params:
          repository: buildpack-artifacts
          rebase: true
  - name: update-buildpack-packager
    serial: true
    public: true
    plan:
      - in_parallel:
        - get: buildpacks-ci
        - get: gem
          resource: buildpack-packager
          trigger: true
        - get: repo-with-gemfile
          resource: buildpack-develop
      - task: update-buildpack-packager
        file: buildpacks-ci/tasks/update-gem-in-gemfile/task.yml
        params:
          RUBYGEM_MIRROR: {{rubygem-mirror}}
          GEM_NAME: buildpack-packager
          GEM_GIT_REPOSITORY: {{buildpack-packager-git-uri-public}}
          GEMFILE_NAME: cf.Gemfile
      - put: buildpack-develop
        params:
          repository: repo-with-gemfile-artifacts
          rebase: true
<% end %>

<% if language == 'hwc' %>
  - name: update-buildpack-bosh-release
    serial: true
    public: true
    plan:
      - in_parallel:
        - get: buildpacks-ci
        - get: release
          resource: buildpack-bosh-release
        <% buildpacks[language]['stacks'].each_with_index do |stack, i| %>
        - get: buildpack-zip-stack<%= i %>
          resource: pivotal-buildpack-<%= stack %>
          trigger: true
          passed: [ ship-it ]
        <% end %>
      - do:
        - task: create-bosh-release
          file: buildpacks-ci/tasks/create-bosh-release/task.yml
          params:
            ACCESS_KEY_ID: ((buildpacks-cloudfoundry-org-aws-access-key-id))
            SECRET_ACCESS_KEY: ((buildpacks-cloudfoundry-org-aws-secret-access-key))
            LANGUAGE: "hwc"
            RELEASE_NAME: hwc-buildpack
            RELEASE_DIR: release
        - put: buildpack-bosh-release
          params:
            repository: release-artifacts
            rebase: true
            tag: version/version
        - put: buildpack-bosh-release-github-release
          params:
            name: version/version
            tag: version/version
<% end %>