Skip to content

Commit c244569

Browse files
hallvictoriaVictoria Hallmanvkaur
authored
build: update to 1ES PT (#1503)
* code mirror template for worker * converting to 1ES pipeline * basic 1ES pipeline * basic format changed 1ES pipeline * same template as openai pipeline * official build template * official, public, e2e builds * yml formatting * tests typo * formatting & feedback * removed nuget internal reference * removed official folder * feedback, updating public build * updating build cmd and dotnet version --------- Co-authored-by: Victoria Hall <[email protected]> Co-authored-by: Manvir Kaur <[email protected]>
1 parent 255fc24 commit c244569

File tree

7 files changed

+366
-3
lines changed

7 files changed

+366
-3
lines changed

‎.github/workflows/ci_e2e_workflow.yml

+3-3
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ jobs:
6464
python -m pip install --upgrade pip
6565
python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U azure-functions --pre
6666
python -m pip install -U -e .[dev]
67-
67+
6868
if [[ "${{ matrix.python-version }}" != "3.7" ]]; then
6969
python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple --pre -U -e .[test-http-v2]
7070
fi
@@ -79,7 +79,7 @@ jobs:
7979
mkdir logs
8080
- name: Grant execute permission
8181
run: |
82-
chmod +x .github/Scripts/${{ matrix.test-type }}.sh
82+
chmod +x .github/Scripts/${{ matrix.test-type }}.sh
8383
- name: Running 3.7 ${{ matrix.test-type }}
8484
if: matrix.python-version == 3.7
8585
env:
@@ -158,4 +158,4 @@ jobs:
158158
with:
159159
name: Test WebHost Logs ${{ github.run_id }} ${{ matrix.python-version }}
160160
path: logs/*.log
161-
if-no-files-found: ignore
161+
if-no-files-found: ignore

‎eng/ci/code-mirror.yml

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
trigger:
2+
branches:
3+
include:
4+
- dev
5+
- release/*
6+
7+
resources:
8+
repositories:
9+
- repository: eng
10+
type: git
11+
name: engineering
12+
ref: refs/tags/release
13+
14+
variables:
15+
- template: ci/variables/cfs.yml@eng
16+
17+
extends:
18+
template: ci/code-mirror.yml@eng

‎eng/ci/official-build.yml

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
trigger:
2+
batch: true
3+
branches:
4+
include:
5+
- dev
6+
- release/*
7+
8+
# CI only, does not trigger on PRs.
9+
pr: none
10+
11+
schedules:
12+
- cron: '0 0 * * MON'
13+
displayName: At 12:00 AM, only on Monday
14+
branches:
15+
include:
16+
- dev
17+
always: true
18+
19+
resources:
20+
repositories:
21+
- repository: 1es
22+
type: git
23+
name: 1ESPipelineTemplates/1ESPipelineTemplates
24+
ref: refs/tags/release
25+
- repository: eng
26+
type: git
27+
name: engineering
28+
ref: refs/tags/release
29+
30+
variables:
31+
- template: ci/variables/build.yml@eng
32+
- template: ci/variables/cfs.yml@eng
33+
34+
extends:
35+
template: v1/1ES.Official.PipelineTemplate.yml@1es
36+
parameters:
37+
pool:
38+
name: 1es-pool-azfunc
39+
image: 1es-windows-2022
40+
os: windows
41+
42+
stages:
43+
- stage: Build
44+
jobs:
45+
- template: /eng/ci/templates/official/jobs/build-artifacts.yml@self
46+
47+
- stage: RunTests
48+
dependsOn: Build
49+
jobs:
50+
- template: /eng/ci/templates/official/jobs/ci-e2e-tests.yml@self

‎eng/ci/public-build.yml

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
trigger:
2+
batch: true
3+
branches:
4+
include:
5+
- dev
6+
7+
pr:
8+
branches:
9+
include:
10+
- dev
11+
12+
schedules:
13+
- cron: '0 0 * * MON'
14+
displayName: At 12:00 AM, only on Monday
15+
branches:
16+
include:
17+
- dev
18+
always: true
19+
20+
resources:
21+
repositories:
22+
- repository: 1es
23+
type: git
24+
name: 1ESPipelineTemplates/1ESPipelineTemplates
25+
ref: refs/tags/release
26+
27+
extends:
28+
template: v1/1ES.Unofficial.PipelineTemplate.yml@1es
29+
parameters:
30+
pool:
31+
name: 1es-pool-azfunc-public
32+
image: 1es-windows-2022
33+
os: windows
34+
35+
stages:
36+
- stage: Build
37+
jobs:
38+
- template: /eng/ci/templates/jobs/build.yml@self

‎eng/templates/jobs/build.yml

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
jobs:
2+
- job: "Build"
3+
displayName: 'Build python worker'
4+
5+
pool:
6+
name: 1es-pool-azfunc-public
7+
image: 1es-windows-2022
8+
os: windows
9+
10+
steps:
11+
- task: UsePythonVersion@0
12+
inputs:
13+
versionSpec: "3.11"
14+
- bash: |
15+
python --version
16+
displayName: 'Check python version'
17+
- bash: |
18+
python -m venv .env
19+
.env\Scripts\Activate.ps1
20+
python -m pip install --upgrade pip==23.0
21+
python -m pip install .
22+
displayName: 'Build python worker'

‎eng/templates/official/jobs/build-artifacts.yml

+80
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
jobs:
2+
- job: "Build"
3+
displayName: "Build python worker"
4+
5+
templateContext:
6+
outputParentDirectory: $(Build.ArtifactStagingDirectory)
7+
outputs:
8+
- output: pipelineArtifact
9+
targetPath: $(Build.ArtifactStagingDirectory)
10+
artifactName: "drop"
11+
- output: nuget
12+
condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/dev'), eq(variables['UPLOADPACKAGETOPRERELEASEFEED'], true))
13+
useDotNetTask: false
14+
packagesToPush: "$(Build.ArtifactStagingDirectory)/**/*.nupkg;!$(Build.ArtifactStagingDirectory)/**/*.symbols.nupkg"
15+
packageParentPath: "$(Build.ArtifactStagingDirectory)"
16+
publishVstsFeed: "e6a70c92-4128-439f-8012-382fe78d6396/f37f760c-aebd-443e-9714-ce725cd427df"
17+
nuGetFeedType: "internal"
18+
allowPackageConflicts: true
19+
20+
pool:
21+
name: 1es-pool-azfunc
22+
image: 1es-windows-2022
23+
os: windows
24+
25+
variables:
26+
${{ if contains(variables['Build.SourceBranch'], '/tags/' ) }}:
27+
isTagTemp: true
28+
isTag: $[variables.isTagTemp]
29+
30+
strategy:
31+
matrix:
32+
Python37V4:
33+
pythonVersion: "3.7"
34+
workerPath: $(PROD_V4_WORKER_PY)
35+
Python38V4:
36+
pythonVersion: "3.8"
37+
workerPath: $(PROD_V4_WORKER_PY)
38+
Python39V4:
39+
pythonVersion: "3.9"
40+
workerPath: $(PROD_V4_WORKER_PY)
41+
Python310V4:
42+
pythonVersion: "3.10"
43+
workerPath: $(PROD_V4_WORKER_PY)
44+
Python311V4:
45+
pythonVersion: "3.11"
46+
workerPath: $(PROD_V4_WORKER_PY)
47+
48+
steps:
49+
- template: pack/templates/win_env_gen.yml
50+
displayName: "Build Windows x64"
51+
parameters:
52+
pythonVersion: "$(pythonVersion)"
53+
workerPath: "$(workerPath)"
54+
architecture: "x64"
55+
artifactName: "$(pythonVersion)_WINDOWS_X64"
56+
- template: pack/templates/win_env_gen.yml
57+
displayName: "Build Windows x86"
58+
parameters:
59+
pythonVersion: "$(pythonVersion)"
60+
workerPath: "$(workerPath)"
61+
architecture: "x86"
62+
artifactName: "$(pythonVersion)_WINDOWS_x86"
63+
- template: pack/templates/nix_env_gen.yml
64+
displayName: "Build Linux x64"
65+
parameters:
66+
pythonVersion: "$(pythonVersion)"
67+
workerPath: "$(workerPath)"
68+
artifactName: "$(pythonVersion)_LINUX_X64"
69+
- template: pack/templates/nix_env_gen.yml
70+
displayName: "Build OSX X64"
71+
parameters:
72+
pythonVersion: "$(pythonVersion)"
73+
workerPath: "$(workerPath)"
74+
artifactName: "$(pythonVersion)_OSX_X64"
75+
- template: pack/templates/macos_64_env_gen.yml
76+
displayName: "Build OSX ARM64"
77+
parameters:
78+
pythonVersion: "$(pythonVersion)"
79+
workerPath: "$(workerPath)"
80+
artifactName: "$(pythonVersion)_OSX_ARM64"

‎eng/templates/official/jobs/ci-e2e-tests.yml

+155
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
jobs:
2+
- job: "TestPython"
3+
displayName: "Run Python E2E Tests"
4+
5+
pool:
6+
name: 1es-pool-azfunc
7+
image: 1es-ubuntu-22.04
8+
os: linux
9+
10+
strategy:
11+
matrix:
12+
python-37-fwpc:
13+
PYTHON_VERSION: '3.7'
14+
TEST_TYPE: 'fwpc-e2e-tests'
15+
python-37-e2e:
16+
PYTHON_VERSION: '3.7'
17+
TEST_TYPE: 'e2e-tests'
18+
python-37-deferred-bindings:
19+
PYTHON_VERSION: '3.7'
20+
TEST_TYPE: 'deferred-bindings-e2e-tests'
21+
python-38-fwpc:
22+
PYTHON_VERSION: '3.8'
23+
TEST_TYPE: 'fwpc-e2e-tests'
24+
python-38-e2e:
25+
PYTHON_VERSION: '3.8'
26+
TEST_TYPE: 'e2e-tests'
27+
python-38-deferred-bindings:
28+
PYTHON_VERSION: '3.8'
29+
TEST_TYPE: 'deferred-bindings-e2e-tests'
30+
python-39-fwpc:
31+
PYTHON_VERSION: '3.9'
32+
TEST_TYPE: 'fwpc-e2e-tests'
33+
python-39-e2e:
34+
PYTHON_VERSION: '3.9'
35+
TEST_TYPE: 'e2e-tests'
36+
python-39-deferred-bindings:
37+
PYTHON_VERSION: '3.9'
38+
TEST_TYPE: 'deferred-bindings-e2e-tests'
39+
python-310-fwpc:
40+
PYTHON_VERSION: '3.10'
41+
TEST_TYPE: 'fwpc-e2e-tests'
42+
python-310-e2e:
43+
PYTHON_VERSION: '3.10'
44+
TEST_TYPE: 'e2e-tests'
45+
python-310-deferred-bindings:
46+
PYTHON_VERSION: '3.10'
47+
TEST_TYPE: 'deferred-bindings-e2e-tests'
48+
python-311-fwpc:
49+
PYTHON_VERSION: '3.11'
50+
TEST_TYPE: 'fwpc-e2e-tests'
51+
python-311-e2e:
52+
PYTHON_VERSION: '3.11'
53+
TEST_TYPE: 'e2e-tests'
54+
python-311-deferred-bindings:
55+
PYTHON_VERSION: '3.11'
56+
TEST_TYPE: 'deferred-bindings-e2e-tests'
57+
steps:
58+
- task: UsePythonVersion@0
59+
inputs:
60+
versionSpec: $(PYTHON_VERSION)
61+
- task: UseDotNet@2
62+
displayName: 'Install .NET 8'
63+
inputs:
64+
version: 8.0.x
65+
- bash: |
66+
python -m pip install --upgrade pip
67+
python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple -U azure-functions --pre
68+
python -m pip install -U -e .[dev]
69+
70+
if [[ "${{ PYTHON_VERSION }}" != "3.7" ]]; then
71+
python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple --pre -U -e .[test-http-v2]
72+
fi
73+
if [[ "${{ PYTHON_VERSION }}" != "3.7" && "${{ PYTHON_VERSION }}" != "3.8" ]]; then
74+
python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple --pre -U -e .[test-deferred-bindings]
75+
fi
76+
77+
# Retry a couple times to avoid certificate issue
78+
retry 5 python setup.py build
79+
retry 5 python setup.py webhost --branch-name=dev
80+
retry 5 python setup.py extension
81+
mkdir logs
82+
displayName: 'Install dependencies and the worker'
83+
- bash: |
84+
chmod +x .github/Scripts/${{ TEST_TYPE }}.sh
85+
displayName: 'Grant execute permission'
86+
- bash: |
87+
if [[ "${{ PYTHON_VERSION }}" == "3.7" ]]; then
88+
.github/Scripts/${{ TEST_TYPE }}.sh
89+
fi
90+
env:
91+
AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString37 }}
92+
AzureWebJobsCosmosDBConnectionString: ${{ secrets.LinuxCosmosDBConnectionString37 }}
93+
AzureWebJobsEventHubConnectionString: ${{ secrets.LinuxEventHubConnectionString37 }}
94+
AzureWebJobsServiceBusConnectionString: ${{ secrets.LinuxServiceBusConnectionString37 }}
95+
AzureWebJobsSqlConnectionString: ${{ secrets.LinuxSqlConnectionString37 }}
96+
AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString37 }}
97+
AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString37 }}
98+
ARCHIVE_WEBHOST_LOGS: ${{ github.event.inputs.archive_webhost_logging }}
99+
displayName: "Running 3.7 $(TEST_TYPE) tests"
100+
- bash: |
101+
if [[ "${{ PYTHON_VERSION }}" == "3.8" ]]; then
102+
.github/Scripts/${{ TEST_TYPE }}.sh
103+
fi
104+
env:
105+
AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString38 }}
106+
AzureWebJobsCosmosDBConnectionString: ${{ secrets.LinuxCosmosDBConnectionString38 }}
107+
AzureWebJobsEventHubConnectionString: ${{ secrets.LinuxEventHubConnectionString38 }}
108+
AzureWebJobsServiceBusConnectionString: ${{ secrets.LinuxServiceBusConnectionString38 }}
109+
AzureWebJobsSqlConnectionString: ${{ secrets.LinuxSqlConnectionString38 }}
110+
AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString38 }}
111+
AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString38 }}
112+
ARCHIVE_WEBHOST_LOGS: ${{ github.event.inputs.archive_webhost_logging }}
113+
displayName: "Running 3.8 $(TEST_TYPE) tests"
114+
- bash: |
115+
if [[ "${{ PYTHON_VERSION }}" == "3.9" ]]; then
116+
.github/Scripts/${{ TEST_TYPE }}.sh
117+
fi
118+
env:
119+
AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString39 }}
120+
AzureWebJobsCosmosDBConnectionString: ${{ secrets.LinuxCosmosDBConnectionString39 }}
121+
AzureWebJobsEventHubConnectionString: ${{ secrets.LinuxEventHubConnectionString39 }}
122+
AzureWebJobsServiceBusConnectionString: ${{ secrets.LinuxServiceBusConnectionString39 }}
123+
AzureWebJobsSqlConnectionString: ${{ secrets.LinuxSqlConnectionString39 }}
124+
AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString39 }}
125+
AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString39 }}
126+
ARCHIVE_WEBHOST_LOGS: ${{ github.event.inputs.archive_webhost_logging }}
127+
displayName: "Running 3.9 $(TEST_TYPE) tests"
128+
- bash: |
129+
if [[ "${{ PYTHON_VERSION }}" == "3.10" ]]; then
130+
.github/Scripts/${{ TEST_TYPE }}.sh
131+
fi
132+
env:
133+
AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString310 }}
134+
AzureWebJobsCosmosDBConnectionString: ${{ secrets.LinuxCosmosDBConnectionString310 }}
135+
AzureWebJobsEventHubConnectionString: ${{ secrets.LinuxEventHubConnectionString310 }}
136+
AzureWebJobsServiceBusConnectionString: ${{ secrets.LinuxServiceBusConnectionString310 }}
137+
AzureWebJobsSqlConnectionString: ${{ secrets.LinuxSqlConnectionString310 }}
138+
AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString310 }}
139+
AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString310 }}
140+
ARCHIVE_WEBHOST_LOGS: ${{ github.event.inputs.archive_webhost_logging }}
141+
displayName: "Running 3.10 $(TEST_TYPE) tests"
142+
- bash: |
143+
if [[ "${{ PYTHON_VERSION }}" == "3.11" ]]; then
144+
.github/Scripts/${{ TEST_TYPE }}.sh
145+
fi
146+
env:
147+
AzureWebJobsStorage: ${{ secrets.LinuxStorageConnectionString311 }}
148+
AzureWebJobsCosmosDBConnectionString: ${{ secrets.LinuxCosmosDBConnectionString311 }}
149+
AzureWebJobsEventHubConnectionString: ${{ secrets.LinuxEventHubConnectionString311 }}
150+
AzureWebJobsServiceBusConnectionString: ${{ secrets.LinuxServiceBusConnectionString311 }}
151+
AzureWebJobsSqlConnectionString: ${{ secrets.LinuxSqlConnectionString311 }}
152+
AzureWebJobsEventGridTopicUri: ${{ secrets.LinuxEventGridTopicUriString311 }}
153+
AzureWebJobsEventGridConnectionKey: ${{ secrets.LinuxEventGridConnectionKeyString311 }}
154+
ARCHIVE_WEBHOST_LOGS: ${{ github.event.inputs.archive_webhost_logging }}
155+
displayName: "Running 3.11 $(TEST_TYPE) tests"

0 commit comments

Comments
 (0)