From 85c14e5c72f5e28237a2f1ed2c9acf22c953ba79 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 9 Mar 2023 19:22:02 +0100 Subject: [PATCH 1/6] Upgrade minimum go version to 1.20 --- .github/workflows/check-easyjson.yml | 2 +- .github/workflows/check-go-dependencies-task.yml | 2 +- .github/workflows/check-go-task.yml | 2 +- .github/workflows/check-i18n-task.yml | 2 +- .github/workflows/check-markdown-task.yml | 2 +- .github/workflows/check-mkdocs-task.yml | 2 +- .github/workflows/check-protobuf-task.yml | 2 +- .github/workflows/deploy-cobra-mkdocs-versioned-poetry.yml | 2 +- .github/workflows/i18n-weekly-pull.yaml | 2 +- .github/workflows/test-go-task.yml | 2 +- DistTasks.yml | 2 +- docs/CONTRIBUTING.md | 2 +- go.mod | 2 +- 13 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/check-easyjson.yml b/.github/workflows/check-easyjson.yml index e3b29864ac6..ae0e0b7b8f2 100644 --- a/.github/workflows/check-easyjson.yml +++ b/.github/workflows/check-easyjson.yml @@ -2,7 +2,7 @@ name: Check easyjson generated files env: # See: https://github.com/actions/setup-go/tree/main#supported-version-syntax - GO_VERSION: "1.19" + GO_VERSION: "1.20" # See: https://docs.github.com/en/actions/reference/events-that-trigger-workflows on: diff --git a/.github/workflows/check-go-dependencies-task.yml b/.github/workflows/check-go-dependencies-task.yml index 7f130f5d695..35e5800c764 100644 --- a/.github/workflows/check-go-dependencies-task.yml +++ b/.github/workflows/check-go-dependencies-task.yml @@ -3,7 +3,7 @@ name: Check Go Dependencies env: # See: https://github.com/actions/setup-go/tree/main#supported-version-syntax - GO_VERSION: "1.19" + GO_VERSION: "1.20" # See: https://docs.github.com/actions/using-workflows/events-that-trigger-workflows on: diff --git a/.github/workflows/check-go-task.yml b/.github/workflows/check-go-task.yml index 2fae7ee7845..a90c7b34e7b 100644 --- a/.github/workflows/check-go-task.yml +++ b/.github/workflows/check-go-task.yml @@ -3,7 +3,7 @@ name: Check Go env: # See: https://github.com/actions/setup-go/tree/main#supported-version-syntax - GO_VERSION: "1.19" + GO_VERSION: "1.20" # See: https://docs.github.com/en/actions/reference/events-that-trigger-workflows on: diff --git a/.github/workflows/check-i18n-task.yml b/.github/workflows/check-i18n-task.yml index 273efe06ac8..3f38036fbd5 100644 --- a/.github/workflows/check-i18n-task.yml +++ b/.github/workflows/check-i18n-task.yml @@ -2,7 +2,7 @@ name: Check Internationalization env: # See: https://github.com/actions/setup-go/tree/main#supported-version-syntax - GO_VERSION: "1.19" + GO_VERSION: "1.20" # See: https://docs.github.com/en/actions/reference/events-that-trigger-workflows on: diff --git a/.github/workflows/check-markdown-task.yml b/.github/workflows/check-markdown-task.yml index 900a8dfa4cb..617258ba717 100644 --- a/.github/workflows/check-markdown-task.yml +++ b/.github/workflows/check-markdown-task.yml @@ -3,7 +3,7 @@ name: Check Markdown env: # See: https://github.com/actions/setup-go/tree/main#supported-version-syntax - GO_VERSION: "1.19" + GO_VERSION: "1.20" # See: https://docs.github.com/en/actions/reference/events-that-trigger-workflows on: diff --git a/.github/workflows/check-mkdocs-task.yml b/.github/workflows/check-mkdocs-task.yml index 98c3fb612d2..29143946800 100644 --- a/.github/workflows/check-mkdocs-task.yml +++ b/.github/workflows/check-mkdocs-task.yml @@ -3,7 +3,7 @@ name: Check Website env: # See: https://github.com/actions/setup-go/tree/main#supported-version-syntax - GO_VERSION: "1.19" + GO_VERSION: "1.20" # See: https://github.com/actions/setup-python/tree/main#available-versions-of-python PYTHON_VERSION: "3.9" diff --git a/.github/workflows/check-protobuf-task.yml b/.github/workflows/check-protobuf-task.yml index 80f082baeff..35f6dbd54f2 100644 --- a/.github/workflows/check-protobuf-task.yml +++ b/.github/workflows/check-protobuf-task.yml @@ -2,7 +2,7 @@ name: Check Protocol Buffers env: # See: https://github.com/actions/setup-go/tree/main#supported-version-syntax - GO_VERSION: "1.19" + GO_VERSION: "1.20" # See: https://docs.github.com/en/actions/reference/events-that-trigger-workflows on: diff --git a/.github/workflows/deploy-cobra-mkdocs-versioned-poetry.yml b/.github/workflows/deploy-cobra-mkdocs-versioned-poetry.yml index ee9230e9ae2..815e2371871 100644 --- a/.github/workflows/deploy-cobra-mkdocs-versioned-poetry.yml +++ b/.github/workflows/deploy-cobra-mkdocs-versioned-poetry.yml @@ -3,7 +3,7 @@ name: Deploy Website env: # See: https://github.com/actions/setup-go/tree/main#supported-version-syntax - GO_VERSION: "1.19" + GO_VERSION: "1.20" # See: https://github.com/actions/setup-python/tree/main#available-versions-of-python PYTHON_VERSION: "3.9" diff --git a/.github/workflows/i18n-weekly-pull.yaml b/.github/workflows/i18n-weekly-pull.yaml index 306af7f4917..472e26bc0a6 100644 --- a/.github/workflows/i18n-weekly-pull.yaml +++ b/.github/workflows/i18n-weekly-pull.yaml @@ -2,7 +2,7 @@ name: i18n-weekly-pull env: # See: https://github.com/actions/setup-go/tree/main#supported-version-syntax - GO_VERSION: "1.19" + GO_VERSION: "1.20" COVERAGE_ARTIFACT: coverage-data on: diff --git a/.github/workflows/test-go-task.yml b/.github/workflows/test-go-task.yml index e13f50efef0..83f231c8330 100644 --- a/.github/workflows/test-go-task.yml +++ b/.github/workflows/test-go-task.yml @@ -3,7 +3,7 @@ name: Test Go env: # See: https://github.com/actions/setup-go/tree/main#supported-version-syntax - GO_VERSION: "1.19" + GO_VERSION: "1.20" COVERAGE_ARTIFACT: coverage-data # See: https://docs.github.com/en/free-pro-team@latest/actions/reference/events-that-trigger-workflows diff --git a/DistTasks.yml b/DistTasks.yml index ebc0db6424a..802bbbceb1d 100644 --- a/DistTasks.yml +++ b/DistTasks.yml @@ -19,7 +19,7 @@ version: "3" vars: CONTAINER: "docker.elastic.co/beats-dev/golang-crossbuild" - GO_VERSION: "1.19.3" + GO_VERSION: "1.20.1" tasks: Windows_32bit: diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index e9dc2e1d8ea..73ccee0998e 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -70,7 +70,7 @@ submitting a PR: To build the Arduino CLI from sources you need the following tools to be available in your local environment: -- [Go][1] version 1.19 or later +- [Go][1] version 1.20 or later - [Taskfile][2] to help you run the most common tasks from the command line If you want to run integration tests you will also need: diff --git a/go.mod b/go.mod index 76735088430..c91df414f95 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/arduino/arduino-cli -go 1.19 +go 1.20 // We must use this fork until https://github.com/mailru/easyjson/pull/372 is merged replace github.com/mailru/easyjson => github.com/cmaglie/easyjson v0.8.1 From c2364be2f6d2c5ad5b4771a4060879b17c2f4baa Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 9 Mar 2023 19:24:44 +0100 Subject: [PATCH 2/6] Updated license cache --- .../go/golang.org/x/crypto/curve25519.dep.yml | 63 +++++++++++++++++++ .../internal/field.dep.yml} | 6 +- 2 files changed, 66 insertions(+), 3 deletions(-) create mode 100644 .licenses/go/golang.org/x/crypto/curve25519.dep.yml rename .licenses/go/golang.org/x/crypto/{internal/alias.dep.yml => curve25519/internal/field.dep.yml} (94%) diff --git a/.licenses/go/golang.org/x/crypto/curve25519.dep.yml b/.licenses/go/golang.org/x/crypto/curve25519.dep.yml new file mode 100644 index 00000000000..e7809657fe5 --- /dev/null +++ b/.licenses/go/golang.org/x/crypto/curve25519.dep.yml @@ -0,0 +1,63 @@ +--- +name: golang.org/x/crypto/curve25519 +version: v0.7.0 +type: go +summary: Package curve25519 provides an implementation of the X25519 function, which + performs scalar multiplication on the elliptic curve known as Curve25519. +homepage: https://pkg.go.dev/golang.org/x/crypto/curve25519 +license: bsd-3-clause +licenses: +- sources: crypto@v0.7.0/LICENSE + text: | + Copyright (c) 2009 The Go Authors. All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +- sources: crypto@v0.7.0/PATENTS + text: | + Additional IP Rights Grant (Patents) + + "This implementation" means the copyrightable works distributed by + Google as part of the Go project. + + Google hereby grants to You a perpetual, worldwide, non-exclusive, + no-charge, royalty-free, irrevocable (except as stated in this section) + patent license to make, have made, use, offer to sell, sell, import, + transfer and otherwise run, modify and propagate the contents of this + implementation of Go, where such license applies only to those patent + claims, both currently owned or controlled by Google and acquired in + the future, licensable by Google that are necessarily infringed by this + implementation of Go. This grant does not include claims that would be + infringed only as a consequence of further modification of this + implementation. If you or your agent or exclusive licensee institute or + order or agree to the institution of patent litigation against any + entity (including a cross-claim or counterclaim in a lawsuit) alleging + that this implementation of Go or any code incorporated within this + implementation of Go constitutes direct or contributory patent + infringement, or inducement of patent infringement, then any patent + rights granted to you under this License for this implementation of Go + shall terminate as of the date such litigation is filed. +notices: [] diff --git a/.licenses/go/golang.org/x/crypto/internal/alias.dep.yml b/.licenses/go/golang.org/x/crypto/curve25519/internal/field.dep.yml similarity index 94% rename from .licenses/go/golang.org/x/crypto/internal/alias.dep.yml rename to .licenses/go/golang.org/x/crypto/curve25519/internal/field.dep.yml index 55037fbb5ec..133d6683cf8 100644 --- a/.licenses/go/golang.org/x/crypto/internal/alias.dep.yml +++ b/.licenses/go/golang.org/x/crypto/curve25519/internal/field.dep.yml @@ -1,9 +1,9 @@ --- -name: golang.org/x/crypto/internal/alias +name: golang.org/x/crypto/curve25519/internal/field version: v0.7.0 type: go -summary: Package alias implements memory aliasing tests. -homepage: https://pkg.go.dev/golang.org/x/crypto/internal/alias +summary: Package field implements fast arithmetic modulo 2^255-19. +homepage: https://pkg.go.dev/golang.org/x/crypto/curve25519/internal/field license: bsd-3-clause licenses: - sources: crypto@v0.7.0/LICENSE From 6ec13e76abc1229a69f9b1a6db5424476cd99e9f Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Thu, 9 Mar 2023 23:54:12 +0100 Subject: [PATCH 3/6] Implemented code coverage in integration tests --- .github/workflows/test-go-task.yml | 16 ++++++++++++++-- .gitignore | 1 + Taskfile.yml | 9 +++++++-- internal/integrationtest/arduino-cli.go | 5 +++++ 4 files changed, 27 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-go-task.yml b/.github/workflows/test-go-task.yml index 83f231c8330..7449b8ee9b2 100644 --- a/.github/workflows/test-go-task.yml +++ b/.github/workflows/test-go-task.yml @@ -110,6 +110,15 @@ jobs: run: | export GO_TEST_PACKAGE="github.com/arduino/arduino-cli/internal/integrationtest/${{ matrix.tests }}" task go:integration-test + mv coverage_integration.txt coverage_integration_${{ matrix.operating-system }}_${{ matrix.tests }}.txt + + - name: Upload coverage data to workflow artifact + uses: actions/upload-artifact@v3 + with: + if-no-files-found: error + name: ${{ env.COVERAGE_ARTIFACT }} + path: | + ./coverage_integration_*.txt test: needs: run-determination @@ -162,7 +171,9 @@ jobs: coverage-upload: runs-on: ubuntu-latest - needs: test + needs: + - test + - test-integration steps: - name: Checkout repository uses: actions/checkout@v3 @@ -176,6 +187,7 @@ jobs: with: files: > ./coverage_unit.txt, - ./coverage_legacy.txt + ./coverage_legacy.txt, + ./coverage_integration_*.txt flags: unit fail_ci_if_error: ${{ github.repository == 'arduino/arduino-cli' }} diff --git a/.gitignore b/.gitignore index 70751f882c6..924c5301545 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ /wiki .idea coverage_*.txt +coverage_data __pycache__ venv .pytest_cache diff --git a/Taskfile.yml b/Taskfile.yml index 380b0898867..9a4c4ed45e2 100755 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -48,7 +48,7 @@ tasks: desc: Build the Go code dir: '{{default "./" .GO_MODULE_PATH}}' cmds: - - go build -v {{.LDFLAGS}} + - go build -v {{default "" .EXTRA_FLAGS}} {{.LDFLAGS}} # Source: https://github.com/arduino/tooling-project-assets/blob/main/workflow-templates/assets/deploy-cobra-mkdocs-versioned-poetry/Taskfile.yml go:cli-docs: @@ -110,10 +110,14 @@ tasks: desc: Run the Go-based integration tests deps: - task: go:build + vars: + EXTRA_FLAGS: "-covermode=atomic" dir: '{{default "./" .GO_MODULE_PATH}}' cmds: - | - go test \ + rm -fr coverage_data + mkdir coverage_data + INTEGRATION_GOCOVERDIR={{ .ROOT_DIR }}/coverage_data go test \ -v \ -short \ {{ .GO_TEST_PACKAGE }} \ @@ -122,6 +126,7 @@ tasks: -coverprofile=coverage_unit.txt \ {{default .DEFAULT_INTEGRATIONTEST_GO_PACKAGES .GO_PACKAGES}} \ {{.TEST_LDFLAGS}} + go tool covdata textfmt -i=coverage_data -o coverage_integration.txt # Source: https://github.com/arduino/tooling-project-assets/blob/main/workflow-templates/assets/check-go-task/Taskfile.yml go:vet: diff --git a/internal/integrationtest/arduino-cli.go b/internal/integrationtest/arduino-cli.go index 92bce9e85bc..6bb78a695ff 100644 --- a/internal/integrationtest/arduino-cli.go +++ b/internal/integrationtest/arduino-cli.go @@ -185,6 +185,11 @@ func (cli *ArduinoCLI) convertEnvForExecutils(env map[string]string) []string { for k, v := range env { envVars = append(envVars, fmt.Sprintf("%s=%s", k, v)) } + + // Proxy code-coverage related env vars + if gocoverdir := os.Getenv("INTEGRATION_GOCOVERDIR"); gocoverdir != "" { + envVars = append(envVars, "GOCOVERDIR="+gocoverdir) + } return envVars } From 3154ebbde599050ddcc0d3ed1a1c9d08f3e0ffa4 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 10 Mar 2023 00:46:34 +0100 Subject: [PATCH 4/6] Use gocovmerge to merge all coverage artifacts --- .github/workflows/test-go-task.yml | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-go-task.yml b/.github/workflows/test-go-task.yml index 7449b8ee9b2..88fe8258245 100644 --- a/.github/workflows/test-go-task.yml +++ b/.github/workflows/test-go-task.yml @@ -177,17 +177,26 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v3 + + - name: Install Go + uses: actions/setup-go@v3 + with: + go-version: ${{ env.GO_VERSION }} + + - name: Install gocovmerge + run: go install github.com/wadey/gocovmerge@b5bfa59 + - name: Download coverage data artifact uses: actions/download-artifact@v3 with: name: ${{ env.COVERAGE_ARTIFACT }} + - name: Merge all code coverage artifacts + run: gocovmerge coverage*.txt > coverage.txt + - name: Send unit tests coverage to Codecov uses: codecov/codecov-action@v3 with: - files: > - ./coverage_unit.txt, - ./coverage_legacy.txt, - ./coverage_integration_*.txt + files: ./coverage.txt flags: unit fail_ci_if_error: ${{ github.repository == 'arduino/arduino-cli' }} From 27ac5fd97d297e9433ead12d2ade4381d7a8197f Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Fri, 10 Mar 2023 10:12:11 +0100 Subject: [PATCH 5/6] Fixed build error on macOS dist The error was due to an increased git security constraints ... go: downloading github.com/xanzy/ssh-agent v0.2.1 go: downloading gopkg.in/warnings.v0 v0.1.2 error obtaining VCS status: exit status 128 Use -buildvcs=false to disable VCS stamping. Error: failed building for darwin/amd64: exit status 1 failed building for darwin/amd64: exit status 1 task: Failed to run task "dist:macOS_64bit": exit status 1 To fix this I followed the suggestion here https://github.com/elastic/golang-crossbuild/issues/232 --- DistTasks.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/DistTasks.yml b/DistTasks.yml index 802bbbceb1d..af5536e9e8a 100644 --- a/DistTasks.yml +++ b/DistTasks.yml @@ -201,11 +201,12 @@ tasks: desc: Builds Mac OS X 64 bit binaries dir: "{{.DIST_DIR}}" cmds: + # "git config safe.directory" is required until this is fixed https://github.com/elastic/golang-crossbuild/issues/232 - | docker run -v `pwd`/..:/home/build -w /home/build \ -e CGO_ENABLED=1 \ {{.CONTAINER}}:{{.CONTAINER_TAG}} \ - --build-cmd "{{.BUILD_COMMAND}}" \ + --build-cmd "git config --global --add safe.directory /home/build && {{.BUILD_COMMAND}}" \ -p "{{.BUILD_PLATFORM}}" tar cz -C {{.PLATFORM_DIR}} {{.PROJECT_NAME}} -C ../.. LICENSE.txt -f {{.PACKAGE_NAME}} @@ -235,11 +236,12 @@ tasks: desc: Builds Mac OS X ARM64 binaries dir: "{{.DIST_DIR}}" cmds: + # "git config safe.directory" is required until this is fixed https://github.com/elastic/golang-crossbuild/issues/232 - | docker run -v `pwd`/..:/home/build -w /home/build \ -e CGO_ENABLED=1 \ {{.CONTAINER}}:{{.CONTAINER_TAG}} \ - --build-cmd "{{.BUILD_COMMAND}}" \ + --build-cmd "git config --global --add safe.directory /home/build && {{.BUILD_COMMAND}}" \ -p "{{.BUILD_PLATFORM}}" tar cz -C {{.PLATFORM_DIR}} {{.PROJECT_NAME}} -C ../.. LICENSE.txt -f {{.PACKAGE_NAME}} From eacd81c5b35b381a24f6949de1c6720b5089c718 Mon Sep 17 00:00:00 2001 From: Cristian Maglie Date: Sat, 11 Mar 2023 16:51:23 +0100 Subject: [PATCH 6/6] Before killing the daemon, try a graceful shutdown Otherwise the process will not outut the coverage data. --- internal/integrationtest/arduino-cli.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/internal/integrationtest/arduino-cli.go b/internal/integrationtest/arduino-cli.go index 6bb78a695ff..efbe717b02b 100644 --- a/internal/integrationtest/arduino-cli.go +++ b/internal/integrationtest/arduino-cli.go @@ -70,6 +70,7 @@ type ArduinoCLI struct { path *paths.Path t *require.Assertions proc *executils.Process + stdIn io.WriteCloser cliEnvVars map[string]string cliConfigPath *paths.Path stagingDir *paths.Path @@ -122,7 +123,12 @@ func NewArduinoCliWithinEnvironment(env *Environment, config *ArduinoCLIConfig) func (cli *ArduinoCLI) CleanUp() { if cli.proc != nil { cli.daemonConn.Close() - cli.proc.Kill() + cli.stdIn.Close() + proc := cli.proc + go func() { + time.Sleep(time.Second) + proc.Kill() + }() cli.proc.Wait() } } @@ -251,10 +257,11 @@ func (cli *ArduinoCLI) StartDaemon(verbose bool) string { cli.t.NoError(err) stderr, err := cliProc.StderrPipe() cli.t.NoError(err) - _, err = cliProc.StdinPipe() + stdIn, err := cliProc.StdinPipe() cli.t.NoError(err) cli.t.NoError(cliProc.Start()) + cli.stdIn = stdIn cli.proc = cliProc cli.daemonAddr = "127.0.0.1:50051"