From b8b8dc0d746315ae55037534d31965784ad9ceea Mon Sep 17 00:00:00 2001 From: Garrett Campbell Date: Mon, 5 Feb 2024 11:21:06 -0500 Subject: [PATCH 1/8] update pipelines to use template --- azure-pipelines.yml | 47 +++++++++++++++++---------- build/prerelease.yml | 75 ++++++++++++++++++++++++++++---------------- 2 files changed, 78 insertions(+), 44 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 12232288..39d2fbd6 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -4,14 +4,14 @@ name: $(Date:yyyyMMdd)$(Rev:.r).0-$(SourceBranchName) parameters: - - name: SignTypeOverride - displayName: Signing type override - type: string - default: default - values: - - default - - test - - real +- name: SignTypeOverride + displayName: Signing type override + type: string + default: default + values: + - default + - test + - real pr: - main @@ -19,17 +19,12 @@ pr: trigger: branches: include: - - main + - main tags: include: - - v* - -pool: - name: VSEngSS-MicroBuild2019-1ES + - v* variables: - # If the user didn't override the signing type, then only real-sign on tags or - # the main branch. ${{ if ne(parameters.SignTypeOverride, 'default') }}: SignType: ${{ parameters.SignTypeOverride }} ${{ if and(eq(parameters.SignTypeOverride, 'default'), or(startsWith(variables['Build.SourceBranch'], 'refs/tags'), eq(variables['Build.SourceBranchName'], 'main'))) }}: @@ -37,5 +32,23 @@ variables: ${{ if and(eq(parameters.SignTypeOverride, 'default'), not(or(startsWith(variables['Build.SourceBranch'], 'refs/tags'), eq(variables['Build.SourceBranchName'], 'main')))) }}: SignType: test -jobs: - - template: build/build.yml +resources: + repositories: + - repository: MicroBuildTemplate + type: git + name: 1ESPipelineTemplates/MicroBuildTemplate + ref: refs/tags/release + +extends: + template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate + parameters: + pool: + name: VSEngSS-MicroBuild2022-1ES + sdl: + sourceAnalysisPool: VSEngSS-MicroBuild2022-1ES + customBuildTags: + - ES365AIMigrationTooling + stages: + - stage: stage + jobs: + - template: /build/build.yml@self diff --git a/build/prerelease.yml b/build/prerelease.yml index 2cfb3896..87627d8f 100644 --- a/build/prerelease.yml +++ b/build/prerelease.yml @@ -8,35 +8,56 @@ name: $(Date:yyMMdd)$(Rev:rrr) trigger: none pr: none schedules: - - cron: "0 2 * * *" - displayName: Daily 2 AM - branches: - include: - - main +- cron: "0 2 * * *" + displayName: Daily 2 AM + branches: + include: + - main variables: SignType: real -jobs: - - template: ./build.yml - parameters: - prerelease: true - - job: PublishExtensions - displayName: Publish extensions - dependsOn: Build +resources: + repositories: + - repository: MicroBuildTemplate + type: git + name: 1ESPipelineTemplates/MicroBuildTemplate + ref: refs/tags/release + +extends: + template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate + parameters: pool: - name: "AzurePipelines-EO" - steps: - - checkout: none - - download: current - artifact: extension-vsixes - - task: NodeTool@0 - displayName: Use Node 16.x - inputs: - versionSpec: 16.x - - script: npm install --global @vscode/vsce - displayName: Install vsce - - script: for f in $(Pipeline.Workspace)/extension-vsixes/*.vsix; do vsce publish --packagePath $f; done - displayName: Publish vscode-arduino - env: - VSCE_PAT: $(vscePat) + name: AzurePipelines-EO + image: AzurePipelinesWindows2022compliantGPT + os: windows + sdl: + sourceAnalysisPool: + name: AzurePipelines-EO + image: AzurePipelinesWindows2022compliantGPT + os: windows + customBuildTags: + - ES365AIMigrationTooling + stages: + - stage: stage + jobs: + - template: /build/build.yml@self + parameters: + prerelease: true + - job: PublishExtensions + displayName: Publish extensions + dependsOn: Build + steps: + - checkout: none + - download: current + artifact: extension-vsixes + - task: NodeTool@0 + displayName: Use Node 16.x + inputs: + versionSpec: 16.x + - script: npm install --global @vscode/vsce + displayName: Install vsce + - script: for f in $(Pipeline.Workspace)/extension-vsixes/*.vsix; do vsce publish --packagePath $f; done + displayName: Publish vscode-arduino + env: + VSCE_PAT: $(vscePat) \ No newline at end of file From 680b94075837167955646611e1c5b8e45c0c06c4 Mon Sep 17 00:00:00 2001 From: Garrett Campbell Date: Mon, 5 Feb 2024 11:25:37 -0500 Subject: [PATCH 2/8] update official pipeline --- build/build.yml | 287 ++++++++++++++++++++++-------------------------- 1 file changed, 129 insertions(+), 158 deletions(-) diff --git a/build/build.yml b/build/build.yml index a2e82192..13eef5e3 100644 --- a/build/build.yml +++ b/build/build.yml @@ -2,163 +2,134 @@ # Licensed under the MIT License. parameters: - - name: prerelease - type: boolean - default: false +- name: prerelease + type: boolean + default: false jobs: - - job: Build - pool: - name: VSEngSS-MicroBuild2019-1ES - variables: - # MicroBuild requires TeamName to be set. - TeamName: C++ Cross Platform and Cloud - steps: - - task: MicroBuildSigningPlugin@3 - displayName: Install MicroBuild Signing - inputs: - signType: $(SignType) - zipSources: false - # MicroBuild signing will always fail on public PRs. - condition: ne(variables['Build.Reason'], 'PullRequest') - - # Run these scanners first so that they don't detect issues in dependencies. - # Failures won't break the build until "Check for compliance errors" step. - - task: CredScan@3 - displayName: Run CredScan - inputs: - toolMajorVersion: V2 - - task: PoliCheck@2 - displayName: Run PoliCheck - inputs: - targetType: F - targetArgument: $(Build.SourcesDirectory) - - # Node 14 matches the version of Node used by VS Code when this was written, - # but it should be updated when VS Code updates its Node version. - - task: NodeTool@0 - displayName: Use Node 16.x - inputs: - versionSpec: 16.x - - # Override the patch version if this is a pre-release build. - - ${{ if parameters.prerelease }}: - - pwsh: node -e "p=require('./package.json');p.version=p.version.replace(/\.\d+$/,'.'+$(Build.BuildNumber));require('fs').writeFileSync('./package.json',JSON.stringify(p,undefined,2))" - - - script: npm install --global gulp node-gyp @vscode/vsce - displayName: Install global dependencies - - script: npm install - displayName: Install project dependencies - - - task: ComponentGovernanceComponentDetection@0 - displayName: Detect components - - task: notice@0 - displayName: Generate NOTICE file - inputs: - outputfile: $(Build.SourcesDirectory)/NOTICE.txt - condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) - - - script: gulp tslint - displayName: Check for linting errors - - script: gulp genAikey - displayName: Use production AI key - condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags')) - # Pack the extension now even though it's unsigned so that we ignore files - # from .vscodeignore. This will reduce load on the signing server later and - # ensure we only attempt to sign shipping files. - - ${{ if parameters.prerelease }}: - - script: node build/package.js --pre-release - displayName: Build and pack extension - - ${{ else }}: - - script: node build/package.js - displayName: Build and pack extension - - # Extract the VSIXes, sign what we can, then pack it back up and publish it. - - pwsh: | - $path = Join-Path $Env:TEMP "7z-installer.exe" - Invoke-WebRequest https://www.7-zip.org/a/7z2201-x64.exe -OutFile $path - Start-Process -FilePath $path -Args "/S" -Verb RunAs -Wait - Remove-Item $path - Echo "##vso[task.prependpath]$Env:PROGRAMFILES\7-Zip\" - displayName: Install 7zip - - pwsh: Get-ChildItem out\vsix | Foreach-Object { 7z x $_.FullName -o$(Build.StagingDirectory)\vscode-arduino\$($_.BaseName) } - displayName: Extract extension for signing - - task: NuGetToolInstaller@1 - displayName: Install NuGet - - task: NuGetAuthenticate@0 - displayName: Authenticate NuGet - - script: nuget restore .\build\SignFiles.proj -PackagesDirectory .\build\packages - displayName: Restore MicroBuild Core - # MicroBuild signing will always fail on public PRs. - condition: ne(variables['Build.Reason'], 'PullRequest') - - task: MSBuild@1 - displayName: Sign files - inputs: - solution: .\build\SignFiles.proj - msbuildArguments: /p:SignType=$(SignType) - # MicroBuild signing will always fail on public PRs. - condition: ne(variables['Build.Reason'], 'PullRequest') - - pwsh: | - Get-ChildItem -Directory $(Build.StagingDirectory)\vscode-arduino | Foreach-Object { 7z a ($_.FullName + ".vsix") ($_.FullName + "\*") -tzip } - New-Item -Path $(Build.StagingDirectory)\vscode-arduino\vsix -ItemType Directory - Get-Item $(Build.StagingDirectory)\vscode-arduino\*.vsix | Move-Item -Destination $(Build.StagingDirectory)\vscode-arduino\vsix - displayName: Pack signed files - - task: MSBuild@1 - displayName: Sign VSIXes - inputs: - solution: .\build\SignVsix.proj - msbuildArguments: /p:SignType=$(SignType) - # MicroBuild signing will always fail on public PRs. - condition: ne(variables['Build.Reason'], 'PullRequest') - - publish: $(Build.StagingDirectory)\vscode-arduino\vsix - artifact: extension-vsixes - displayName: Publish extension VSIXes as artifact - - # Install the Arduino IDE and run tests. - - script: curl -LO https://downloads.arduino.cc/arduino-1.8.19-windows.zip - displayName: Download Arduino IDE - - script: >- - node build/checkHash.js arduino-1.8.19-windows.zip - c704a821089eab2588f1deae775916219b1517febd1dd574ff29958dca873945 - displayName: Verify Arduino IDE - - task: ExtractFiles@1 - displayName: Extract Arduino IDE - inputs: - archiveFilePatterns: arduino-1.8.19-windows.zip - destinationFolder: arduino-ide - - script: "echo ##vso[task.prependpath]$(Build.SourcesDirectory)\\arduino-ide\\arduino-1.8.19" - displayName: Add Arduino IDE to PATH - - script: npm test --silent - displayName: Run tests - - - task: PostAnalysis@2 - displayName: Check for compliance errors - # To avoid spirious warnings about missing logs, explicitly declare what we scanned. - inputs: - CredScan: true - PoliCheck: true - - # Trust Services Automation (TSA) can automatically open bugs for compliance issues. - # https://www.1eswiki.com/wiki/Trust_Services_Automation_(TSA) - - task: TSAUpload@2 - displayName: Upload logs to TSA - inputs: - GdnPublishTsaOnboard: true - GdnPublishTsaConfigFile: $(Build.SourcesDirectory)\build\tsa.gdntsa - # Don't open bugs for PR builds - condition: ne(variables['Build.Reason'], 'PullRequest') - - - task: GitHubRelease@0 - displayName: Publish to GitHub - inputs: - gitHubConnection: embeddedbot - repositoryName: microsoft/vscode-arduino - action: create - target: $(Build.SourceVersion) - tagSource: auto - assets: $(Build.StagingDirectory)\vscode-arduino\vsix\*.vsix - isPreRelease: $[contains(variables['Build.SourceBranch'], '-rc')] - condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags')) - - - task: MicroBuildCleanup@1 - displayName: Clean up MicroBuild +- job: Build + pool: + name: VSEngSS-MicroBuild2022-1ES + variables: + TeamName: C++ Cross Platform and Cloud + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish extension VSIXes as artifact' + targetPath: $(Build.StagingDirectory)\vscode-arduino\vsix + artifactName: extension-vsixes + steps: + - task: MicroBuildSigningPlugin@3 + displayName: Install MicroBuild Signing + inputs: + signType: $(SignType) + zipSources: false + condition: ne(variables['Build.Reason'], 'PullRequest') + - task: CredScan@3 + displayName: Run CredScan + inputs: + toolMajorVersion: V2 + - task: PoliCheck@2 + displayName: Run PoliCheck + inputs: + targetType: F + targetArgument: $(Build.SourcesDirectory) + - task: NodeTool@0 + displayName: Use Node 16.x + inputs: + versionSpec: 16.x + - ${{ if parameters.prerelease }}: + - pwsh: node -e "p=require('./package.json');p.version=p.version.replace(/\.\d+$/,'.'+$(Build.BuildNumber));require('fs').writeFileSync('./package.json',JSON.stringify(p,undefined,2))" + - script: npm install --global gulp node-gyp @vscode/vsce + displayName: Install global dependencies + - script: npm install + displayName: Install project dependencies + - task: ComponentGovernanceComponentDetection@0 + displayName: Detect components + - task: notice@0 + displayName: Generate NOTICE file + inputs: + outputfile: $(Build.SourcesDirectory)/NOTICE.txt + condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) + - script: gulp tslint + displayName: Check for linting errors + - script: gulp genAikey + displayName: Use production AI key + condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags')) + - ${{ if parameters.prerelease }}: + - script: node build/package.js --pre-release + displayName: Build and pack extension + - ${{ else }}: + - script: node build/package.js + displayName: Build and pack extension + - pwsh: | + $path = Join-Path $Env:TEMP "7z-installer.exe" + Invoke-WebRequest https://www.7-zip.org/a/7z2201-x64.exe -OutFile $path + Start-Process -FilePath $path -Args "/S" -Verb RunAs -Wait + Remove-Item $path + Echo "##vso[task.prependpath]$Env:PROGRAMFILES\7-Zip\" + displayName: Install 7zip + - pwsh: Get-ChildItem out\vsix | Foreach-Object { 7z x $_.FullName -o$(Build.StagingDirectory)\vscode-arduino\$($_.BaseName) } + displayName: Extract extension for signing + - task: NuGetToolInstaller@1 + displayName: Install NuGet + - task: NuGetAuthenticate@0 + displayName: Authenticate NuGet + - script: nuget restore .\build\SignFiles.proj -PackagesDirectory .\build\packages + displayName: Restore MicroBuild Core + condition: ne(variables['Build.Reason'], 'PullRequest') + - task: MSBuild@1 + displayName: Sign files + inputs: + solution: .\build\SignFiles.proj + msbuildArguments: /p:SignType=$(SignType) + condition: ne(variables['Build.Reason'], 'PullRequest') + - pwsh: | + Get-ChildItem -Directory $(Build.StagingDirectory)\vscode-arduino | Foreach-Object { 7z a ($_.FullName + ".vsix") ($_.FullName + "\*") -tzip } + New-Item -Path $(Build.StagingDirectory)\vscode-arduino\vsix -ItemType Directory + Get-Item $(Build.StagingDirectory)\vscode-arduino\*.vsix | Move-Item -Destination $(Build.StagingDirectory)\vscode-arduino\vsix + displayName: Pack signed files + - task: MSBuild@1 + displayName: Sign VSIXes + inputs: + solution: .\build\SignVsix.proj + msbuildArguments: /p:SignType=$(SignType) + condition: ne(variables['Build.Reason'], 'PullRequest') + - script: curl -LO https://downloads.arduino.cc/arduino-1.8.19-windows.zip + displayName: Download Arduino IDE + - script: >- + node build/checkHash.js arduino-1.8.19-windows.zip c704a821089eab2588f1deae775916219b1517febd1dd574ff29958dca873945 + displayName: Verify Arduino IDE + - task: ExtractFiles@1 + displayName: Extract Arduino IDE + inputs: + archiveFilePatterns: arduino-1.8.19-windows.zip + destinationFolder: arduino-ide + - script: "echo ##vso[task.prependpath]$(Build.SourcesDirectory)\\arduino-ide\\arduino-1.8.19" + displayName: Add Arduino IDE to PATH + - script: npm test --silent + displayName: Run tests + - task: PostAnalysis@2 + displayName: Check for compliance errors + inputs: + CredScan: true + PoliCheck: true + - task: TSAUpload@2 + displayName: Upload logs to TSA + inputs: + GdnPublishTsaOnboard: true + GdnPublishTsaConfigFile: $(Build.SourcesDirectory)\build\tsa.gdntsa + condition: ne(variables['Build.Reason'], 'PullRequest') + - task: GitHubRelease@0 + displayName: Publish to GitHub + inputs: + gitHubConnection: embeddedbot + repositoryName: microsoft/vscode-arduino + action: create + target: $(Build.SourceVersion) + tagSource: auto + assets: $(Build.StagingDirectory)\vscode-arduino\vsix\*.vsix + isPreRelease: $[contains(variables['Build.SourceBranch'], '-rc')] + condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags')) + - task: MicroBuildCleanup@1 + displayName: Clean up MicroBuild \ No newline at end of file From 650f90d0bb67d9db5ad301d98d7ac1e75696ec33 Mon Sep 17 00:00:00 2001 From: Garrett Campbell Date: Mon, 5 Feb 2024 11:41:10 -0500 Subject: [PATCH 3/8] don't package unsupported version --- build/package.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/package.js b/build/package.js index daf0562b..fb34852c 100644 --- a/build/package.js +++ b/build/package.js @@ -10,7 +10,7 @@ const flags = argv.slice(2).join(" "); // Taken from https://code.visualstudio.com/api/working-with-extensions/publishing-extension#platformspecific-extensions const platforms = [ "win32-x64", - "win32-ia32", + // "win32-ia32", This is no longer supported by vscode based on the link above. "win32-arm64", "linux-x64", "linux-arm64", From 2266531ec1446c04ed13d3b44722e7b8130c6152 Mon Sep 17 00:00:00 2001 From: Garrett Campbell Date: Mon, 5 Feb 2024 12:16:30 -0500 Subject: [PATCH 4/8] disable spotbugs --- azure-pipelines.yml | 2 ++ build/prerelease.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 39d2fbd6..c04d7462 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -46,6 +46,8 @@ extends: name: VSEngSS-MicroBuild2022-1ES sdl: sourceAnalysisPool: VSEngSS-MicroBuild2022-1ES + spotBugs: + enabled: false # Turn this off, this isn't java. customBuildTags: - ES365AIMigrationTooling stages: diff --git a/build/prerelease.yml b/build/prerelease.yml index 87627d8f..fba8077a 100644 --- a/build/prerelease.yml +++ b/build/prerelease.yml @@ -36,6 +36,8 @@ extends: name: AzurePipelines-EO image: AzurePipelinesWindows2022compliantGPT os: windows + spotBugs: + enabled: false # Turn this off, this isn't java. customBuildTags: - ES365AIMigrationTooling stages: From fa96eb728c3e001ce2cad529884bc8e742bbde69 Mon Sep 17 00:00:00 2001 From: Garrett Campbell Date: Mon, 5 Feb 2024 12:45:53 -0500 Subject: [PATCH 5/8] remove credscan and policheck --- build/build.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/build/build.yml b/build/build.yml index 13eef5e3..9f10227a 100644 --- a/build/build.yml +++ b/build/build.yml @@ -25,15 +25,6 @@ jobs: signType: $(SignType) zipSources: false condition: ne(variables['Build.Reason'], 'PullRequest') - - task: CredScan@3 - displayName: Run CredScan - inputs: - toolMajorVersion: V2 - - task: PoliCheck@2 - displayName: Run PoliCheck - inputs: - targetType: F - targetArgument: $(Build.SourcesDirectory) - task: NodeTool@0 displayName: Use Node 16.x inputs: From 3979776c97ed51c994f3a4a0ff25c13ca03141fd Mon Sep 17 00:00:00 2001 From: Garrett Campbell Date: Mon, 5 Feb 2024 17:13:36 -0500 Subject: [PATCH 6/8] have the sbom be built for source directory --- build/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/build/build.yml b/build/build.yml index 9f10227a..2806c7a8 100644 --- a/build/build.yml +++ b/build/build.yml @@ -18,6 +18,7 @@ jobs: displayName: 'Publish extension VSIXes as artifact' targetPath: $(Build.StagingDirectory)\vscode-arduino\vsix artifactName: extension-vsixes + sbomBuildDropPath: $(Build.SourcesDirectory) steps: - task: MicroBuildSigningPlugin@3 displayName: Install MicroBuild Signing From 006210ff9befddfa00cc5fd0e6bce175e1c08d55 Mon Sep 17 00:00:00 2001 From: Garrett Campbell Date: Wed, 7 Feb 2024 14:33:54 -0500 Subject: [PATCH 7/8] fix tsa --- build/build.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/build/build.yml b/build/build.yml index 2806c7a8..8e71e2d6 100644 --- a/build/build.yml +++ b/build/build.yml @@ -13,6 +13,12 @@ jobs: variables: TeamName: C++ Cross Platform and Cloud templateContext: + sdl: + - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + tsa: + enabled: true + GdnPublishTsaOnboard: true + GdnPublishTsaConfigFile: $(Build.SourcesDirectory)\build\tsa.gdntsa outputs: - output: pipelineArtifact displayName: 'Publish extension VSIXes as artifact' @@ -106,11 +112,6 @@ jobs: inputs: CredScan: true PoliCheck: true - - task: TSAUpload@2 - displayName: Upload logs to TSA - inputs: - GdnPublishTsaOnboard: true - GdnPublishTsaConfigFile: $(Build.SourcesDirectory)\build\tsa.gdntsa condition: ne(variables['Build.Reason'], 'PullRequest') - task: GitHubRelease@0 displayName: Publish to GitHub From 3457117408eb17bac0b58a3b8a53908130c52228 Mon Sep 17 00:00:00 2001 From: Garrett Campbell Date: Wed, 7 Feb 2024 14:36:13 -0500 Subject: [PATCH 8/8] remove - --- build/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/build.yml b/build/build.yml index 8e71e2d6..0f480489 100644 --- a/build/build.yml +++ b/build/build.yml @@ -14,7 +14,7 @@ jobs: TeamName: C++ Cross Platform and Cloud templateContext: sdl: - - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: tsa: enabled: true GdnPublishTsaOnboard: true