Skip to content

Commit e94ab04

Browse files
author
Victoria Hall
committed
official, public, e2e builds
1 parent c458fca commit e94ab04

File tree

5 files changed

+455
-73
lines changed

5 files changed

+455
-73
lines changed

‎.github/workflows/ci_e2e_workflow.yml

+156-44
Original file line numberDiff line numberDiff line change
@@ -1,49 +1,161 @@
1-
pr:
2-
branches:
3-
include:
4-
- dev
5-
paths:
6-
exclude:
7-
- '**.md'
1+
# This workflow will install Python dependencies and run end to end tests with single version of Python
2+
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
83

9-
trigger:
10-
branches:
11-
include:
12-
- dev
13-
paths:
14-
exclude:
15-
- '**.md'
4+
name: CI E2E tests
165

17-
resources:
18-
repositories:
19-
- repository: 1es
20-
type: git
21-
name: 1ESPipelineTemplates/1ESPipelineTemplates
22-
ref: refs/tags/release
6+
on:
7+
workflow_dispatch:
8+
inputs:
9+
archive_webhost_logging:
10+
description: "For debugging purposes, archive test webhost logs"
11+
required: false
12+
default: "false"
13+
push:
14+
branches: [dev, main, release/*]
15+
pull_request:
16+
branches: [dev, main, release/*]
17+
schedule:
18+
# Monday to Friday 3 AM CST build
19+
# * is a special character in YAML so you have to quote this string
20+
- cron: "0 8 * * 1,2,3,4,5"
2321

24-
schedules:
25-
- cron: '0 0 * * MON'
26-
displayName: At 12:00 AM, only on Monday
27-
branches:
28-
include:
29-
- dev
30-
always: true
22+
jobs:
23+
build:
24+
name: "Python E2E CI Run"
25+
runs-on: ubuntu-latest
26+
strategy:
27+
fail-fast: false
28+
matrix:
29+
python-version: [3.7, 3.8, 3.9, "3.10", "3.11"]
30+
# Currently runs FWPC E2E tests, classic E2E tests, and DefBindings E2E tests.
31+
# To run tests from another script, add the script name to this matrix
32+
test-type: [fwpc-e2e-tests, e2e-tests, deferred-bindings-e2e-tests]
33+
permissions: read-all
34+
steps:
35+
- name: Checkout code.
36+
uses: actions/checkout@v4
37+
- name: Set up Python ${{ matrix.python-version }}
38+
uses: actions/setup-python@v5
39+
with:
40+
python-version: ${{ matrix.python-version }}
41+
- name: Set up Dotnet 8.0.x
42+
uses: actions/setup-dotnet@v4
43+
with:
44+
dotnet-version: "8.0.x"
45+
- name: Install dependencies and the worker
46+
run: |
47+
retry() {
48+
local -r -i max_attempts="$1"; shift
49+
local -r cmd="$@"
50+
local -i attempt_num=1
51+
until $cmd
52+
do
53+
if (( attempt_num == max_attempts ))
54+
then
55+
echo "Attempt $attempt_num failed and there are no more attempts left!"
56+
return 1
57+
else
58+
echo "Attempt $attempt_num failed! Trying again in $attempt_num seconds..."
59+
sleep 1
60+
fi
61+
done
62+
}
3163
32-
extends:
33-
template: v1/1ES.Official.PipelineTemplate.yml@1es
34-
parameters:
35-
pool:
36-
name: 1es-pool-azfunc
37-
image: 1es-windows-2022
38-
os: windows
64+
python -m pip install --upgrade pip
65+
python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U azure-functions --pre
66+
python -m pip install -U -e .[dev]
67+
68+
if [[ "${{ matrix.python-version }}" != "3.7" ]]; then
69+
python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple --pre -U -e .[test-http-v2]
70+
fi
71+
if [[ "${{ matrix.python-version }}" != "3.7" && "${{ matrix.python-version }}" != "3.8" ]]; then
72+
python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple --pre -U -e .[test-deferred-bindings]
73+
fi
3974

40-
stages:
41-
- stage: Build
42-
displayName: 'Build extension and samples'
43-
jobs:
44-
- job: Build
45-
steps:
46-
- task: UseDotNet@2
47-
displayName: 'Install .NET Core SDK 6 LTS'
48-
inputs:
49-
version: '6.x'
75+
# Retry a couple times to avoid certificate issue
76+
retry 5 python setup.py build
77+
retry 5 python setup.py webhost --branch-name=dev
78+
retry 5 python setup.py extension
79+
mkdir logs
80+
- name: Grant execute permission
81+
run: |
82+
chmod +x .github/Scripts/${{ matrix.test-type }}.sh
83+
- name: Running 3.7 ${{ matrix.test-type }}
84+
if: matrix.python-version == 3.7
85+
env:
86+
AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString37 }}
87+
AzureWebJobsCosmosDBConnectionString: ${{ secrets.LinuxCosmosDBConnectionString37 }}
88+
AzureWebJobsEventHubConnectionString: ${{ secrets.LinuxEventHubConnectionString37 }}
89+
AzureWebJobsServiceBusConnectionString: ${{ secrets.LinuxServiceBusConnectionString37 }}
90+
AzureWebJobsSqlConnectionString: ${{ secrets.LinuxSqlConnectionString37 }}
91+
AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString37 }}
92+
AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString37 }}
93+
ARCHIVE_WEBHOST_LOGS: ${{ github.event.inputs.archive_webhost_logging }}
94+
run: |
95+
.github/Scripts/${{ matrix.test-type }}.sh
96+
- name: Running 3.8 ${{ matrix.test-type }}
97+
if: matrix.python-version == 3.8
98+
env:
99+
AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString38 }}
100+
AzureWebJobsCosmosDBConnectionString: ${{ secrets.LinuxCosmosDBConnectionString38 }}
101+
AzureWebJobsEventHubConnectionString: ${{ secrets.LinuxEventHubConnectionString38 }}
102+
AzureWebJobsServiceBusConnectionString: ${{ secrets.LinuxServiceBusConnectionString38 }}
103+
AzureWebJobsSqlConnectionString: ${{ secrets.LinuxSqlConnectionString38 }}
104+
AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString38 }}
105+
AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString38 }}
106+
ARCHIVE_WEBHOST_LOGS: ${{ github.event.inputs.archive_webhost_logging }}
107+
run: |
108+
.github/Scripts/${{ matrix.test-type }}.sh
109+
- name: Running 3.9 ${{ matrix.test-type }}
110+
if: matrix.python-version == 3.9
111+
env:
112+
AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString39 }}
113+
AzureWebJobsCosmosDBConnectionString: ${{ secrets.LinuxCosmosDBConnectionString39 }}
114+
AzureWebJobsEventHubConnectionString: ${{ secrets.LinuxEventHubConnectionString39 }}
115+
AzureWebJobsServiceBusConnectionString: ${{ secrets.LinuxServiceBusConnectionString39 }}
116+
AzureWebJobsSqlConnectionString: ${{ secrets.LinuxSqlConnectionString39 }}
117+
AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString39 }}
118+
AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString39 }}
119+
ARCHIVE_WEBHOST_LOGS: ${{ github.event.inputs.archive_webhost_logging }}
120+
run: |
121+
.github/Scripts/${{ matrix.test-type }}.sh
122+
- name: Running 3.10 ${{ matrix.test-type }}
123+
if: matrix.python-version == 3.10
124+
env:
125+
AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString310 }}
126+
AzureWebJobsCosmosDBConnectionString: ${{ secrets.LinuxCosmosDBConnectionString310 }}
127+
AzureWebJobsEventHubConnectionString: ${{ secrets.LinuxEventHubConnectionString310 }}
128+
AzureWebJobsServiceBusConnectionString: ${{ secrets.LinuxServiceBusConnectionString310 }}
129+
AzureWebJobsSqlConnectionString: ${{ secrets.LinuxSqlConnectionString310 }}
130+
AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString310 }}
131+
AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString310 }}
132+
ARCHIVE_WEBHOST_LOGS: ${{ github.event.inputs.archive_webhost_logging }}
133+
run: |
134+
.github/Scripts/${{ matrix.test-type }}.sh
135+
- name: Running 3.11 ${{ matrix.test-type }}
136+
if: matrix.python-version == 3.11
137+
env:
138+
AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString311 }}
139+
AzureWebJobsCosmosDBConnectionString: ${{ secrets.LinuxCosmosDBConnectionString311 }}
140+
AzureWebJobsEventHubConnectionString: ${{ secrets.LinuxEventHubConnectionString311 }}
141+
AzureWebJobsServiceBusConnectionString: ${{ secrets.LinuxServiceBusConnectionString311 }}
142+
AzureWebJobsSqlConnectionString: ${{ secrets.LinuxSqlConnectionString311 }}
143+
AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString311 }}
144+
AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString311 }}
145+
ARCHIVE_WEBHOST_LOGS: ${{ github.event.inputs.archive_webhost_logging }}
146+
run: |
147+
.github/Scripts/${{ matrix.test-type }}.sh
148+
- name: Codecov
149+
uses: codecov/codecov-action@v3
150+
with:
151+
file: ./coverage.xml # optional
152+
flags: unittests # optional
153+
name: codecov # optional
154+
fail_ci_if_error: false # optional (default = false)
155+
- name: Publish Logs to Artifact
156+
if: failure()
157+
uses: actions/upload-artifact@v4
158+
with:
159+
name: Test WebHost Logs ${{ github.run_id }} ${{ matrix.python-version }}
160+
path: logs/*.log
161+
if-no-files-found: ignore

‎eng/ci/official-build.yml

+33-29
Original file line numberDiff line numberDiff line change
@@ -1,32 +1,27 @@
11
trigger:
2+
batch: true
23
branches:
34
include:
4-
- dev
5-
paths:
6-
exclude:
7-
- '**.md'
5+
- dev
6+
- release/*
87

98
# CI only, does not trigger on PRs.
109
pr: none
1110

12-
schedules:
13-
- cron: '0 0 * * MON'
14-
displayName: At 12:00 AM, only on Monday
15-
branches:
16-
include:
17-
- main
18-
always: true
19-
2011
resources:
2112
repositories:
22-
- repository: 1es
23-
type: git
24-
name: 1ESPipelineTemplates/1ESPipelineTemplates
25-
ref: refs/tags/release
26-
- repository: eng
27-
type: git
28-
name: engineering
29-
ref: refs/tags/release
13+
- repository: 1es
14+
type: git
15+
name: 1ESPipelineTemplates/1ESPipelineTemplates
16+
ref: refs/tags/release
17+
- repository: eng
18+
type: git
19+
name: engineering
20+
ref: refs/tags/release
21+
22+
variables:
23+
- template: ci/variables/build.yml@eng
24+
- template: ci/variables/cfs.yml@eng
3025

3126
extends:
3227
template: v1/1ES.Official.PipelineTemplate.yml@1es
@@ -37,12 +32,21 @@ extends:
3732
os: windows
3833

3934
stages:
40-
- stage: BuildDotNet
41-
displayName: 'Build DotNet'
42-
jobs:
43-
- job: Build
44-
steps:
45-
- task: UseDotNet@2
46-
displayName: 'Install .NET Core SDK 6 LTS'
47-
inputs:
48-
version: '6.x'
35+
- stage: Build
36+
37+
jobs:
38+
- template: /eng/ci/templates/official/jobs/build-artifacts.yml@self
39+
40+
- stage: TestWindows
41+
dependsOn: Build
42+
43+
jobs:
44+
- template: /eng/ci/templates/official/jobs/run-e2e-tests-windows.yml@self
45+
46+
- stage: TestLinux
47+
dependsOn:
48+
- Build
49+
- TestWindows
50+
51+
jobs:
52+
- template: /eng/ci/templates/official/jobs/run-e2e-tests-linux.yml@self

‎eng/ci/public-build.yml

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
trigger:
2+
batch: true
3+
branches:
4+
include:
5+
- dev
6+
7+
pr:
8+
branches:
9+
include:
10+
- dev
11+
12+
resources:
13+
repositories:
14+
- repository: 1es
15+
type: git
16+
name: 1ESPipelineTemplates/1ESPipelineTemplates
17+
ref: refs/tags/release
18+
19+
extends:
20+
template: v1/1ES.Unofficial.PipelineTemplate.yml@1es
21+
parameters:
22+
pool:
23+
name: 1es-pool-azfunc-public
24+
image: 1es-windows-2022
25+
os: windows

‎eng/templates/official/jobs/build-artifacts.yml

+80
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
jobs:
2+
- job: "Build"
3+
displayName: 'Build python worker'
4+
5+
templateContext:
6+
outputParentDirectory: $(Build.ArtifactStagingDirectory)
7+
outputs:
8+
- output: pipelineArtifact
9+
targetPath: $(Build.ArtifactStagingDirectory)
10+
artifactName: 'drop'
11+
- output: nuget
12+
condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/dev'), eq(variables['UPLOADPACKAGETOPRERELEASEFEED'], true))
13+
useDotNetTask: false
14+
packagesToPush: '$(Build.ArtifactStagingDirectory)/**/*.nupkg;!$(Build.ArtifactStagingDirectory)/**/*.symbols.nupkg'
15+
packageParentPath: '$(Build.ArtifactStagingDirectory)'
16+
publishVstsFeed: 'e6a70c92-4128-439f-8012-382fe78d6396/f37f760c-aebd-443e-9714-ce725cd427df'
17+
nuGetFeedType: 'internal'
18+
allowPackageConflicts: true
19+
20+
pool:
21+
name: 1es-pool-azfunc
22+
image: 1es-windows-2022
23+
os: windows
24+
25+
variables:
26+
${{ if contains(variables['Build.SourceBranch'], '/tags/' ) }}:
27+
isTagTemp: true
28+
isTag: $[variables.isTagTemp]
29+
30+
strategy:
31+
matrix:
32+
Python37V4:
33+
pythonVersion: '3.7'
34+
workerPath: $(PROD_V4_WORKER_PY)
35+
Python38V4:
36+
pythonVersion: '3.8'
37+
workerPath: $(PROD_V4_WORKER_PY)
38+
Python39V4:
39+
pythonVersion: '3.9'
40+
workerPath: $(PROD_V4_WORKER_PY)
41+
Python310V4:
42+
pythonVersion: '3.10'
43+
workerPath: $(PROD_V4_WORKER_PY)
44+
Python311V4:
45+
pythonVersion: '3.11'
46+
workerPath: $(PROD_V4_WORKER_PY)
47+
48+
steps:
49+
- template: pack/templates/win_env_gen.yml
50+
displayName: 'Build Windows x64'
51+
parameters:
52+
pythonVersion: '$(pythonVersion)'
53+
workerPath: '$(workerPath)'
54+
architecture: 'x64'
55+
artifactName: '$(pythonVersion)_WINDOWS_X64'
56+
- template: pack/templates/win_env_gen.yml
57+
displayName: 'Build Windows x86'
58+
parameters:
59+
pythonVersion: '$(pythonVersion)'
60+
workerPath: '$(workerPath)'
61+
architecture: 'x86'
62+
artifactName: '$(pythonVersion)_WINDOWS_x86'
63+
- template: pack/templates/nix_env_gen.yml
64+
displayName: 'Build Linux x64'
65+
parameters:
66+
pythonVersion: '$(pythonVersion)'
67+
workerPath: '$(workerPath)'
68+
artifactName: '$(pythonVersion)_LINUX_X64'
69+
- template: pack/templates/nix_env_gen.yml
70+
displayName: 'Build OSX X64'
71+
parameters:
72+
pythonVersion: '$(pythonVersion)'
73+
workerPath: '$(workerPath)'
74+
artifactName: '$(pythonVersion)_OSX_X64'
75+
- template: pack/templates/macos_64_env_gen.yml
76+
displayName: 'Build OSX ARM64'
77+
parameters:
78+
pythonVersion: '$(pythonVersion)'
79+
workerPath: '$(workerPath)'
80+
artifactName: '$(pythonVersion)_OSX_ARM64'

0 commit comments

Comments
 (0)