Initial check-in for the packaging pipeline (#221)

.pipelines/Build-Official.yml

@@ -86,187 +86,37 @@ extends:
     - stage: macos
       displayName: macOS - build and sign
       jobs:
-      - template: /.pipelines/templates/mac.yml@self
+      - template: /.pipelines/templates/mac-build.yml@self
         parameters:
-          Architecture: x64
-      - template: /.pipelines/templates/mac.yml@self
+          architecture: x64
+      - template: /.pipelines/templates/mac-build.yml@self
         parameters:
-          Architecture: arm64
+          architecture: arm64
 
     - stage: linux
       displayName: linux - build and sign
       jobs:
-      - template: /.pipelines/templates/linux.yml@self
+      - template: /.pipelines/templates/linux-build.yml@self
         parameters:
-          Architecture: x64
-      - template: /.pipelines/templates/linux.yml@self
+          architecture: x64
+      - template: /.pipelines/templates/linux-build.yml@self
         parameters:
-          Architecture: arm64
+          architecture: arm64
 
     - stage: windows
       displayName: windows - build and sign
       jobs:
-      - template: /.pipelines/templates/windows.yml@self
+      - template: /.pipelines/templates/windows-build.yml@self
         parameters:
-          Architecture: x64
-      - template: /.pipelines/templates/windows.yml@self
+          architecture: x86
+      - template: /.pipelines/templates/windows-build.yml@self
         parameters:
-          Architecture: arm64
+          architecture: x64
+      - template: /.pipelines/templates/windows-build.yml@self
+        parameters:
+          architecture: arm64
 
     - stage: module
       displayName: module - build and sign
       jobs:
-      - job: build
-        displayName: Build AIShell module
-        condition: succeeded()
-        pool:
-          type: linux
-        variables:
-        - name: runCodesignValidationInjection
-          value: false
-        - name: NugetSecurityAnalysisWarningLevel
-          value: none
-        - name: DOTNET_NOLOGO
-          value: 1
-        - group: DotNetPrivateBuildAccess
-        - name: ob_outputDirectory
-          value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT'
-        - name: repoRoot
-          value: $(Build.SourcesDirectory)\ProjectMercury
-        - name: ob_sdl_sbom_enabled
-          value: false
-        - name: ob_sdl_codeSignValidation_enabled
-          value: false
-        - name: ob_sdl_binskim_enabled
-          value: true
-        - name: ob_sdl_tsa_configFile
-          value: $(repoRoot)\.config\tsaoptions.json
-        #CodeQL tasks added manually to workaround signing failures
-        - name: ob_sdl_codeql_compiled_enabled
-          value: false
-
-        steps:
-        - checkout: self
-          clean: true
-          env:
-            ob_restore_phase: true
-
-        - pwsh: |
-            Get-ChildItem Env:
-          displayName: Capture environment variables
-          env:
-            ob_restore_phase: true
-
-        - template: /.pipelines/templates/update-nuget-config.yml@self
-          parameters:
-            repoRoot: $(repoRoot)
-
-        # Add CodeQL Init task right before your 'Build' step.
-        - task: CodeQL3000Init@0
-          env:
-            ob_restore_phase: true
-          inputs:
-            Enabled: true
-            Language: csharp
-
-        - pwsh: |
-            Import-Module $(repoRoot)/build.psm1 -Force
-            Install-Dotnet
-
-            $runtime = 'linux-x64'
-            Write-Verbose "Building with Runtime: $runtime" -Verbose
-            Start-Build -Runtime $runtime -Configuration Release -Clean -Verbose
-
-            $outputJson = '$(repoRoot)/_build_output_.json'
-            if (Test-Path $outputJson) {
-              $result = Get-Content $outputJson | ConvertFrom-Json
-              Write-Verbose "Module path: $($result.Module)" -Verbose
-
-              $vstsCommandString = "vso[task.setvariable variable=ModuleDir]$($result.Module)"
-              Write-Host ("sending " + $vstsCommandString)
-              Write-Host "##$vstsCommandString"
-            }
-          displayName: 'Build AIShell module'
-          env:
-            ob_restore_phase: true
-
-        # Add CodeQL Finalize task right after your 'Build' step.
-        - task: CodeQL3000Finalize@0
-          env:
-            ob_restore_phase: true
-
-        - pwsh: |
-            Get-ChildItem $(ModuleDir) -Recurse
-          displayName: "List AIShell module files"
-          env:
-            ob_restore_phase: true
-
-        - task: CopyFiles@2
-          displayName: "Upload unsigned module files"
-          env:
-            ob_restore_phase: true
-          inputs:
-            SourceFolder: $(ModuleDir)
-            Contents: '**/*'
-            TargetFolder: $(ob_outputDirectory)
-
-        - template: /.pipelines/templates/finalize.yml@self
-
-      - job: sign
-        displayName: Sign AIShell module
-        condition: succeeded()
-        dependsOn: build
-        pool:
-          type: windows
-        variables:
-        - name: runCodesignValidationInjection
-          value: false
-        - name: NugetSecurityAnalysisWarningLevel
-          value: none
-        - name: DOTNET_NOLOGO
-          value: 1
-        - group: DotNetPrivateBuildAccess
-        - group: certificate_logical_to_actual
-        - name: ob_outputDirectory
-          value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT'
-        - name: repoRoot
-          value: $(Build.SourcesDirectory)\ProjectMercury
-        - name: ob_sdl_codeSignValidation_enabled
-          value: false
-        - name: ob_sdl_binskim_enabled
-          value: false
-        - name: ob_sdl_tsa_configFile
-          value: $(repoRoot)\.config\tsaoptions.json
-        - name: ob_sdl_codeql_compiled_enabled
-          value: false
-        - name: ob_sdl_sbom_packageName
-          value: 'AIShell.Module'
-
-        steps:
-        - checkout: self
-          clean: true
-          env:
-            ob_restore_phase: true
-
-        - pwsh: |
-            Get-ChildItem Env:
-          displayName: Capture environment variables
-          env:
-            ob_restore_phase: true
-
-        - task: DownloadPipelineArtifact@2
-          inputs:
-            artifact: drop_module_build
-            path: $(Pipeline.Workspace)/drop_module_build
-          displayName: Download build
-
-        - pwsh: |
-            Get-ChildItem -Path $(Pipeline.Workspace)/drop_module_build -Recurse
-          displayName: 'List downloaded files'
-
-        - template: /.pipelines/templates/sign-files.yml@self
-          parameters:
-            appPath: '$(Pipeline.Workspace)/drop_module_build'
-            repoRoot: '$(repoRoot)'
-
-        - template: /.pipelines/templates/finalize.yml@self
+      - template: /.pipelines/templates/module-build.yml@self

.pipelines/Package-Official.yml

@@ -0,0 +1,147 @@
+trigger: none
+
+parameters: # parameters are shown up in ADO UI in a build queue time
+  - name: ForceAzureBlobDelete
+    displayName: Delete Azure Blob
+    type: string
+    values:
+    - true
+    - false
+    default: false
+  - name: 'debug'
+    displayName: 'Enable debug output'
+    type: boolean
+    default: false
+
+variables:
+  - name: CDP_DEFINITION_BUILD_COUNT
+    value: $[counter('', 0)] # needed for onebranch.pipeline.version task
+  - name: system.debug
+    value: ${{ parameters.debug }}
+  - name: ENABLE_PRS_DELAYSIGN
+    value: 1
+  - name: ROOT
+    value: $(Build.SourcesDirectory)
+  - name: ForceAzureBlobDelete
+    value: ${{ parameters.ForceAzureBlobDelete }}
+  - name: NUGET_XMLDOC_MODE
+    value: none
+  - name: nugetMultiFeedWarnLevel
+    value: none
+  - name: NugetSecurityAnalysisWarningLevel
+    value: none
+  - name: skipNugetSecurityAnalysis
+    value: true
+  - name: ob_outputDirectory
+    value: '$(Build.ArtifactStagingDirectory)/ONEBRANCH_ARTIFACT'
+  - name: WindowsContainerImage
+    value: 'onebranch.azurecr.io/windows/ltsc2019/vse2022:latest' # Docker image which is used to build the project
+  - name: LinuxContainerImage
+    value: mcr.microsoft.com/onebranch/cbl-mariner/build:2.0
+  - group: mscodehub-feed-read-general
+  - group: mscodehub-feed-read-akv
+  - name: branchCounterKey
+    value: $[format('{0:yyyyMMdd}-{1}', pipeline.startTime,variables['Build.SourceBranch'])]
+  - name: branchCounter
+    value: $[counter(variables['branchCounterKey'], 1)]
+
+resources:
+  pipelines:
+    - pipeline: AIShellBuildPipeline
+      source: 'AIShell-Build-Official'
+      trigger:
+        branches:
+          include:
+            - main
+            - obp-*
+            - release-*
+
+  repositories:
+    - repository: templates
+      type: git
+      name: OneBranch.Pipelines/GovernedTemplates
+      ref: refs/heads/main
+
+extends:
+  template: v2/OneBranch.Official.CrossPlat.yml@templates
+  parameters:
+    cloudvault:
+      enabled: false
+    featureFlags:
+      linuxEsrpSigning: true
+    globalSdl:
+      disableLegacyManifest: true
+      # disabled Armorty as we dont have any ARM templates to scan. It fails on some sample ARM templates.
+      armory:
+        enabled: false
+      sbom:
+        enabled: true
+      compiled:
+        enabled: false
+      cg:
+        enabled: true
+        ignoreDirectories: 'docs,shell,test,tools'
+      asyncSdl:
+        enabled: true
+        forStages: ['build']
+        credscan:
+          enabled: true
+          scanFolder:  $(Build.SourcesDirectory)
+        binskim:
+          enabled: false
+        apiscan:
+          enabled: false
+        tsaOptionsFile: .config\tsaoptions.json
+
+    stages:
+    - stage: prep
+      jobs:
+      - template: /.pipelines/templates/check-Azure-container.yml@self
+
+    - stage: mac
+      dependsOn: [prep]
+      jobs:
+      - template: /.pipelines/templates/mac-package.yml@self
+        parameters:
+          architecture: x64
+      - template: /.pipelines/templates/mac-package.yml@self
+        parameters:
+          architecture: arm64
+
+    - stage: windows
+      dependsOn: [prep]
+      jobs:
+      - template: /.pipelines/templates/windows-package.yml@self
+        parameters:
+          architecture: x86
+      - template: /.pipelines/templates/windows-package.yml@self
+        parameters:
+          architecture: x64
+      - template: /.pipelines/templates/windows-package.yml@self
+        parameters:
+          architecture: arm64
+
+    - stage: linux
+      dependsOn: [prep]
+      jobs:
+      - template: /.pipelines/templates/linux-package.yml@self
+        parameters:
+          architecture: x64
+      - template: /.pipelines/templates/linux-package.yml@self
+        parameters:
+          architecture: arm64
+
+    - stage: module
+      dependsOn: [prep]
+      jobs:
+      - template: /.pipelines/templates/module-package.yml@self
+
+    - stage: nupkg
+      dependsOn: [prep]
+      jobs:
+      - template: /.pipelines/templates/nupkg-package.yml@self
+
+    - stage: upload
+      dependsOn: [mac_package, windows_package, linux_package, module, nupkg]
+      jobs:
+      - template: /.pipelines/templates/upload-to-Azure.yml@self