Skip to content

Commit eabe8b2

Browse files
Merge branch 'main' into rishav-feat-parser
2 parents 23804d6 + 2796693 commit eabe8b2

File tree

200 files changed

+8077
-2598
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

200 files changed

+8077
-2598
lines changed
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
#
2+
# This local action serves two purposes:
3+
#
4+
# 1. For core workflows like pull.yml and push.yml,
5+
# it is uses to check that the checked in copy of generated code
6+
# matches what the current submodule version of smithy-dafny generates.
7+
# This is important to ensure whenever someone changes the models
8+
# or needs to regenerate to pick up smithy-dafny improvements,
9+
# they don't have to deal with unpleasant surprises.
10+
#
11+
# 2. For workflows that check compatibility with other Dafny versions,
12+
# such as nightly_dafny.yml, it is necessary to regenerate the code
13+
# for that version of Dafny first.
14+
# This is ultimately because some of the code smithy-dafny generates
15+
# is tightly coupled to what code Dafny itself generates.
16+
# A concrete example is that Dafny 4.3 added TypeDescriptors
17+
# as parameters when constructing datatypes like Option and Result.
18+
#
19+
# This is why this is a composite action instead of a reusable workflow:
20+
# the latter executes in a separate runner environment,
21+
# but here we need to actually overwrite the generated code in place
22+
# so that subsequent steps can work correctly.
23+
#
24+
# This action assumes that the given version of Dafny and .NET 6.0.x
25+
# have already been set up, since they are used to format generated code.
26+
#
27+
# Note that recursively generating code doesn't currently work in this repo
28+
# with the version of the mpl pinned by the submodule,
29+
# because the SharedMakefileV2.mk in it doesn't work with newer versions of smithy-dafny.
30+
# Therefore by default we don't recursively regenerate code
31+
# (accomplished by setting the POLYMORPH_DEPENDENCIES environment variable to "").
32+
# If `update-and-regenerate-mpl` is true, we first pull the latest mpl,
33+
# which is necessary both for Makefile compatibility and so we can regenerate mpl code
34+
# for compatibility with newer versions of Dafny.
35+
#
36+
37+
name: "Polymorph code generation"
38+
description: "Regenerates code using smithy-dafny, and optionally checks that the result matches the checked in state"
39+
inputs:
40+
dafny:
41+
description: "The Dafny version to run"
42+
required: true
43+
type: string
44+
library:
45+
description: "Name of the library to regenerate code for"
46+
required: true
47+
type: string
48+
diff-generated-code:
49+
description: "Diff regenerated code against committed state"
50+
required: true
51+
type: boolean
52+
update-and-regenerate-mpl:
53+
description: "Locally update MPL to the tip of master and regenerate its code too"
54+
required: false
55+
default: false
56+
type: boolean
57+
runs:
58+
using: "composite"
59+
steps:
60+
- name: Update MPL submodule locally if requested
61+
if: inputs.update-and-regenerate-mpl == 'true'
62+
working-directory: submodules/MaterialProviders
63+
shell: bash
64+
run: |
65+
git checkout main
66+
git pull
67+
git submodule update --init --recursive
68+
69+
- name: Update top-level project.properties file in MPL
70+
if: inputs.update-and-regenerate-mpl == 'true'
71+
shell: bash
72+
working-directory: submodules/MaterialProviders
73+
run: |
74+
make generate_properties_file
75+
76+
# Update the project.properties file so that we pick up the right runtimes etc.,
77+
# in cases where inputs.dafny is different from the current value in that file.
78+
- name: Generate smithy-dafny-project.properties file
79+
if: inputs.update-and-regenerate-mpl == 'true'
80+
env:
81+
DAFNY_VERSION: ${{ inputs.dafny }}
82+
shell: bash
83+
run: |
84+
make generate_properties_file
85+
86+
- name: Update top-level project.properties file
87+
if: inputs.update-and-regenerate-mpl == 'true'
88+
shell: bash
89+
run: |
90+
awk -F= '!a[$1]++' smithy-dafny-project.properties project.properties > merged.properties
91+
mv merged.properties project.properties
92+
cat project.properties
93+
94+
- name: Don't regenerate dependencies unless requested
95+
id: dependencies
96+
shell: bash
97+
run: |
98+
echo "PROJECT_DEPENDENCIES=${{ inputs.update-and-regenerate-mpl != 'true' && 'PROJECT_DEPENDENCIES=' || '' }}" >> $GITHUB_OUTPUT
99+
100+
- name: Regenerate Dafny code using smithy-dafny
101+
# Unfortunately Dafny codegen doesn't work on Windows:
102+
# https://github.com/smithy-lang/smithy-dafny/issues/317
103+
if: runner.os != 'Windows'
104+
working-directory: ./${{ inputs.library }}
105+
shell: bash
106+
run: |
107+
make polymorph_dafny ${{ steps.dependencies.outputs.PROJECT_DEPENDENCIES }}
108+
109+
- name: Set up prettier in MPL
110+
if: inputs.update-and-regenerate-mpl == 'true'
111+
shell: bash
112+
# Annoyingly, prettier has to be installed in each library individually.
113+
# And this is only necessary or even possible if we've updated the mpl submodule.
114+
run: |
115+
make -C submodules/MaterialProviders/AwsCryptographyPrimitives setup_prettier
116+
make -C submodules/MaterialProviders/AwsCryptographicMaterialProviders setup_prettier
117+
make -C submodules/MaterialProviders/ComAmazonawsKms setup_prettier
118+
make -C submodules/MaterialProviders/ComAmazonawsDynamodb setup_prettier
119+
120+
- name: Regenerate Java code using smithy-dafny
121+
# npx seems to be unavailable on Windows GHA runners,
122+
# so we don't regenerate Java code on them either.
123+
if: runner.os != 'Windows'
124+
working-directory: ./${{ inputs.library }}
125+
shell: bash
126+
# smithy-dafny also formats generated code itself now,
127+
# so prettier is a necessary dependency.
128+
run: |
129+
make setup_prettier
130+
make polymorph_java ${{ steps.dependencies.outputs.PROJECT_DEPENDENCIES }}
131+
132+
- name: Regenerate .NET code using smithy-dafny
133+
working-directory: ./${{ inputs.library }}
134+
shell: bash
135+
run: |
136+
make polymorph_dotnet ${{ steps.dependencies.outputs.PROJECT_DEPENDENCIES }}
137+
138+
- name: Check regenerated code against commited code
139+
# Composite action inputs seem to not actually support booleans properly for some reason
140+
if: inputs.diff-generated-code == 'true'
141+
working-directory: ./${{ inputs.library }}
142+
shell: bash
143+
run: |
144+
make check_polymorph_diff

.github/workflows/ci_codegen.yml

+56
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
# This workflow regenerates code using smithy-dafny and checks that the output matches what's checked in.
2+
name: Library Code Generation
3+
on:
4+
pull_request:
5+
push:
6+
branches:
7+
- main
8+
9+
jobs:
10+
code-generation:
11+
strategy:
12+
fail-fast: false
13+
matrix:
14+
library:
15+
[
16+
DynamoDbEncryption,
17+
TestVectors
18+
]
19+
# Note dotnet is only used for formatting generated code
20+
# in this workflow
21+
dotnet-version: ["6.0.x"]
22+
os: [ubuntu-latest]
23+
runs-on: ${{ matrix.os }}
24+
defaults:
25+
run:
26+
shell: bash
27+
env:
28+
DOTNET_CLI_TELEMETRY_OPTOUT: 1
29+
DOTNET_NOLOGO: 1
30+
steps:
31+
- name: Support longpaths
32+
run: |
33+
git config --global core.longpaths true
34+
35+
- uses: actions/checkout@v3
36+
with:
37+
submodules: recursive
38+
39+
# Only used to format generated code
40+
# and to translate version strings such as "nightly-latest"
41+
# to an actual DAFNY_VERSION.
42+
- name: Setup Dafny
43+
uses: dafny-lang/[email protected]
44+
with:
45+
dafny-version: 4.2.0
46+
47+
- name: Setup .NET Core SDK ${{ matrix.dotnet-version }}
48+
uses: actions/setup-dotnet@v4
49+
with:
50+
dotnet-version: ${{ matrix.dotnet-version }}
51+
52+
- uses: ./.github/actions/polymorph_codegen
53+
with:
54+
dafny: ${{ env.DAFNY_VERSION }}
55+
library: ${{ matrix.library }}
56+
diff-generated-code: true

.github/workflows/ci_examples_java.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ jobs:
3232
matrix:
3333
java-version: [ 8, 11, 16, 17 ]
3434
os: [
35-
macos-latest
35+
macos-12
3636
]
3737
runs-on: ${{ matrix.os }}
3838
permissions:

.github/workflows/ci_examples_net.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ jobs:
1818
]
1919
dotnet-version: [ '6.0.x' ]
2020
os: [
21-
macos-latest,
21+
macos-12,
2222
]
2323
runs-on: ${{ matrix.os }}
2424
permissions:

.github/workflows/ci_test_java.yml

+10-1
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ jobs:
3434
]
3535
java-version: [ 8, 11, 16, 17 ]
3636
os: [
37-
macos-latest
37+
macos-12
3838
]
3939
runs-on: ${{ matrix.os }}
4040
permissions:
@@ -58,6 +58,15 @@ jobs:
5858
# A && B || C is the closest thing to an if .. then ... else ... or ?: expression the GitHub Actions syntax supports.
5959
dafny-version: ${{ (github.event_name == 'schedule' || inputs.nightly) && 'nightly-latest' || '4.2.0' }}
6060

61+
- name: Regenerate code using smithy-dafny if necessary
62+
if: ${{ inputs.nightly }}
63+
uses: ./.github/actions/polymorph_codegen
64+
with:
65+
dafny: ${{ env.DAFNY_VERSION }}
66+
library: ${{ matrix.library }}
67+
diff-generated-code: false
68+
update-and-regenerate-mpl: true
69+
6170
- name: Setup Java ${{ matrix.java-version }}
6271
uses: actions/setup-java@v4
6372
with:

.github/workflows/ci_test_net.yml

+10-1
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ jobs:
3535
]
3636
dotnet-version: [ '6.0.x' ]
3737
os: [
38-
macos-latest,
38+
macos-12,
3939
ubuntu-latest,
4040
windows-latest
4141
]
@@ -65,6 +65,15 @@ jobs:
6565
# A && B || C is the closest thing to an if .. then ... else ... or ?: expression the GitHub Actions syntax supports.
6666
dafny-version: ${{ (github.event_name == 'schedule' || inputs.nightly) && 'nightly-latest' || '4.2.0' }}
6767

68+
- name: Regenerate code using smithy-dafny if necessary
69+
if: ${{ inputs.nightly }}
70+
uses: ./.github/actions/polymorph_codegen
71+
with:
72+
dafny: ${{ env.DAFNY_VERSION }}
73+
library: ${{ matrix.library }}
74+
diff-generated-code: false
75+
update-and-regenerate-mpl: true
76+
6877
- name: Download Dependencies
6978
working-directory: ./${{ matrix.library }}
7079
run: make setup_net

.github/workflows/ci_todos.yml

+3-3
Original file line numberDiff line numberDiff line change
@@ -9,16 +9,16 @@ on:
99

1010
jobs:
1111
findTodos:
12-
runs-on: macos-latest
12+
runs-on: macos-12
1313
steps:
1414
- uses: actions/checkout@v3
1515

1616
- name: Check TODOs in code
1717
shell: bash
1818
# TODOs may be committed as long as the same line contains a link to a Github Issue or refers to a CrypTool SIM.
1919
run: |
20-
ALL_TODO_COUNT=$( { grep -r "TODO" . --exclude-dir=./submodules --exclude-dir=./.git --exclude=./.github/workflows/ci_todos.yml || true; } | wc -l)
21-
GOOD_TODO_COUNT=$( { grep -r "TODO.*\(github.com\/.*issues.*\/[1-9][0-9]*\|CrypTool-[1-9][0-9]*\)" . --exclude-dir=./submodules --exclude-dir=./.git --exclude=./.github/workflows/ci_todos.yml || true; } | wc -l)
20+
ALL_TODO_COUNT=$( { grep -r "TODO" . --exclude-dir=./TestVectors/runtimes --exclude-dir=./submodules --exclude-dir=./.git --exclude=./.github/workflows/ci_todos.yml || true; } | wc -l)
21+
GOOD_TODO_COUNT=$( { grep -r "TODO.*\(github.com\/.*issues.*\/[1-9][0-9]*\|CrypTool-[1-9][0-9]*\)" . --exclude-dir=./submodules --exclude-dir=./.git --exclude-dir=./TestVectors/runtimes --exclude=./.github/workflows/ci_todos.yml || true; } | wc -l)
2222
if [ "$ALL_TODO_COUNT" != "$GOOD_TODO_COUNT" ]; then
2323
exit 1;
2424
fi

.github/workflows/ci_tv_verification.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ jobs:
3535
DDBEncryption
3636
]
3737
os: [
38-
macos-latest,
38+
macos-12,
3939
]
4040
runs-on: ${{ matrix.os }}
4141
steps:

.github/workflows/ci_verification.yml

+12-2
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ jobs:
2828
# Don't run the nightly build on forks
2929
if: github.event_name != 'schedule' || github.repository_owner == 'aws'
3030
strategy:
31+
fail-fast: false
3132
matrix:
3233
# Break up verification between namespaces over multiple
3334
# actions to take advantage of parallelization
@@ -38,7 +39,7 @@ jobs:
3839
StructuredEncryption
3940
]
4041
os: [
41-
macos-latest,
42+
macos-12,
4243
]
4344
runs-on: ${{ matrix.os }}
4445
steps:
@@ -51,8 +52,17 @@ jobs:
5152
with:
5253
# A && B || C is the closest thing to an if .. then ... else ... or ?: expression the GitHub Actions syntax supports.
5354
dafny-version: ${{ (github.event_name == 'schedule' || inputs.nightly) && 'nightly-latest' || '4.2.0' }}
55+
56+
- name: Regenerate code using smithy-dafny if necessary
57+
if: ${{ inputs.nightly }}
58+
uses: ./.github/actions/polymorph_codegen
59+
with:
60+
dafny: ${{ env.DAFNY_VERSION }}
61+
library: DynamoDbEncryption
62+
diff-generated-code: false
63+
update-and-regenerate-mpl: true
5464

55-
- name: Verify ${{ matrix.library }} Dafny code
65+
- name: Verify ${{ matrix.service }} Dafny code
5666
shell: bash
5767
working-directory: ./DynamoDbEncryption
5868
run: |

.github/workflows/sem_ver.yml

+53
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
# This workflow tests the installation of semantic release
2+
name: Semantic Release Test Installation
3+
4+
on:
5+
pull_request:
6+
7+
jobs:
8+
semantic-release:
9+
runs-on: macos-12
10+
permissions:
11+
id-token: write
12+
contents: read
13+
steps:
14+
- name: Support longpaths on Git checkout
15+
run: |
16+
git config --global core.longpaths true
17+
- uses: actions/checkout@v4
18+
with:
19+
submodules: recursive
20+
21+
# We need access to the role that is able to get CI Bot Creds
22+
- name: Configure AWS Credentials for Release
23+
uses: aws-actions/configure-aws-credentials@v4
24+
with:
25+
aws-region: us-west-2
26+
role-to-assume: arn:aws:iam::587316601012:role/GitHub-CI-CI-Bot-Credential-Access-Role-us-west-2
27+
role-session-name: CI_Bot_Release
28+
29+
- name: Upgrade Node
30+
uses: actions/setup-node@v4
31+
with:
32+
node-version: 21
33+
34+
# Use AWS Secrets Manger GHA to retrieve CI Bot Creds
35+
- name: Get CI Bot Creds Secret
36+
uses: aws-actions/aws-secretsmanager-get-secrets@v2
37+
with:
38+
secret-ids: Github/aws-crypto-tools-ci-bot
39+
parse-json-secrets: true
40+
41+
# Log in as the CI Bot
42+
- name: Log in as CI Bot
43+
run: |
44+
echo ${{env.GITHUB_AWS_CRYPTO_TOOLS_CI_BOT_ESDK_RELEASE_TOKEN}} > token.txt
45+
gh auth login --with-token < token.txt
46+
rm token.txt
47+
gh auth status
48+
49+
# Test to see if we can setup semantic release
50+
- name: Test Semantic Release Installation
51+
uses: actions/checkout@v4
52+
- run: |
53+
make setup_semantic_release

0 commit comments

Comments
 (0)