Skip to content

Commit 4917a48

Browse files
author
vinay-lanka
committed
fixup! Added tests
2 parents 96a2df7 + 763b4a7 commit 4917a48

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

52 files changed

+687
-274
lines changed

Diff for: ‎.github/tools/fetch_athena_stats.sh

+93
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
#!/usr/bin/env bash
2+
3+
# This script performs the following:
4+
# 1. Run the query, use jq to capture the QueryExecutionId, and then capture that into bash variable
5+
# 2. Wait for the query to finish running (240 seconds).
6+
# 3. Get the results.
7+
# 4. Json data points struct build
8+
9+
# Expected env variables are:
10+
# AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY for accessing AWS resources
11+
# AWS_ATHENA_SOURCE_TABLE
12+
# AWS_ATHENA_OUTPUT_LOCATION
13+
# GITHUB_REPOSITORY
14+
15+
set -euo pipefail
16+
17+
! read -r -d '' query << EOM
18+
select
19+
replace(url_extract_path("d.url"), '/arduino-cli/arduino-cli_', '') as flavor,
20+
count("id") as gauge
21+
from ${AWS_ATHENA_SOURCE_TABLE}
22+
where "d.url" like 'https://downloads.arduino.cc/arduino-cli/arduino-cli_%'
23+
and "d.url" not like '%latest%' -- exclude latest redirect
24+
and "d.url" not like '%alpha%' -- exclude early alpha releases
25+
and "d.url" not like '%.tar.bz2%' -- exclude very old releases archive formats
26+
group by 1
27+
EOM
28+
29+
queryExecutionId=$(
30+
aws athena start-query-execution \
31+
--query-string "${query}" \
32+
--query-execution-context "Database=demo_books" \
33+
--result-configuration "OutputLocation=${AWS_ATHENA_OUTPUT_LOCATION}" \
34+
--region us-east-1 | jq -r ".QueryExecutionId"
35+
)
36+
37+
echo "QueryExecutionId is ${queryExecutionId}"
38+
for i in $(seq 1 120); do
39+
queryState=$( aws athena get-query-execution \
40+
--query-execution-id "${queryExecutionId}" \
41+
--region us-east-1 | jq -r ".QueryExecution.Status.State"
42+
);
43+
44+
if [[ "${queryState}" == "SUCCEEDED" ]]; then
45+
break;
46+
fi;
47+
48+
echo "QueryExecutionId ${queryExecutionId} - state is ${queryState}"
49+
50+
if [[ "${queryState}" == "FAILED" ]]; then
51+
exit 1;
52+
fi;
53+
54+
sleep 2
55+
done
56+
57+
echo "Query succeeded. Processing data"
58+
queryResult=$( aws athena get-query-results \
59+
--query-execution-id "${queryExecutionId}" \
60+
--region us-east-1 | jq --compact-output
61+
);
62+
63+
! read -r -d '' jsonTemplate << EOM
64+
{
65+
"type": "gauge",
66+
"name": "arduino.downloads.total",
67+
"value": "%s",
68+
"host": "${GITHUB_REPOSITORY}",
69+
"tags": [
70+
"version:%s",
71+
"os:%s",
72+
"arch:%s",
73+
"cdn:downloads.arduino.cc",
74+
"project:arduino-cli"
75+
]
76+
},
77+
EOM
78+
79+
datapoints="["
80+
for row in $(echo "${queryResult}" | jq 'del(.ResultSet.Rows[0])' | jq -r '.ResultSet.Rows[] | .Data' --compact-output); do
81+
value=$(jq -r ".[1].VarCharValue" <<< "${row}")
82+
tag=$(jq -r ".[0].VarCharValue" <<< "${row}")
83+
# Some splitting to obtain 0.6.0, Windows, 32bit elements from string 0.6.0_Windows_32bit.zip
84+
split=($(echo "$tag" | tr '_' '\n'))
85+
if [[ ${#split[@]} -ne 3 ]]; then
86+
continue
87+
fi
88+
archSplit=($(echo "${split[2]}" | tr '.' '\n'))
89+
datapoints+=$(printf "${jsonTemplate}" "${value}" "${split[0]}" "${split[1]}" "${archSplit[0]}")
90+
done
91+
datapoints="${datapoints::-1}]"
92+
93+
echo "::set-output name=result::$(jq --compact-output <<< "${datapoints}")"

Diff for: ‎.github/workflows/arduino-stats.yaml

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
name: arduino-stats
2+
3+
on:
4+
schedule:
5+
# run every day at 12:30:00
6+
- cron: '30 12 * * *'
7+
8+
jobs:
9+
push-stats:
10+
runs-on: ubuntu-latest
11+
12+
steps:
13+
- name: Checkout
14+
uses: actions/checkout@v1
15+
16+
- name: Fetch downloads count form Arduino CDN using AWS Athena
17+
id: fetch
18+
env:
19+
AWS_ACCESS_KEY_ID: ${{ secrets.STATS_AWS_ACCESS_KEY_ID }}
20+
AWS_SECRET_ACCESS_KEY: ${{ secrets.STATS_AWS_SECRET_ACCESS_KEY }}
21+
AWS_ATHENA_SOURCE_TABLE: ${{ secrets.STATS_AWS_ATHENA_SOURCE_TABLE }}
22+
AWS_ATHENA_OUTPUT_LOCATION: ${{ secrets.STATS_AWS_ATHENA_OUTPUT_LOCATION }}
23+
GITHUB_REPOSITORY: ${{ github.repository }}
24+
run: |
25+
# Fetch jq 1.6 as VM has only 1.5 ATM
26+
wget -q https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 -O jq
27+
chmod +x jq
28+
PATH=${{ github.workspace }}:$PATH
29+
.github/tools/fetch_athena_stats.sh
30+
31+
- name: Send metrics
32+
uses: masci/datadog@v1
33+
with:
34+
api-key: ${{ secrets.DD_API_KEY }}
35+
# Metrics input expects YAML but JSON will work just right.
36+
metrics: ${{steps.fetch.outputs.result}}
37+
38+
- name: Report failure
39+
if: failure()
40+
uses: masci/datadog@v1
41+
with:
42+
api-key: ${{ secrets.DD_API_KEY }}
43+
events: |
44+
- title: "Arduino CLI stats failing"
45+
text: "Stats collection failed"
46+
alert_type: "error"
47+
host: ${{ github.repository }}
48+
tags:
49+
- "project:arduino-cli"
50+
- "cdn:downloads.arduino.cc"

Diff for: ‎.github/workflows/github-stats.yaml

+90
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
name: github-stats
2+
3+
on:
4+
schedule:
5+
# run every 30 minutes
6+
- cron: '*/30 * * * *'
7+
8+
jobs:
9+
push-stats:
10+
runs-on: ubuntu-latest
11+
12+
steps:
13+
- name: Fetch downloads count
14+
id: fetch
15+
uses: actions/[email protected]
16+
with:
17+
github-token: ${{github.token}}
18+
script: |
19+
let metrics = []
20+
21+
// Get a list of releases
22+
const opts = github.repos.listReleases.endpoint.merge({
23+
...context.repo
24+
})
25+
const releases = await github.paginate(opts)
26+
27+
// Get download stats for every release
28+
for (const rel of releases) {
29+
// Names for assets are like `arduino-cli_0.4.0_Linux_32bit.tar.gz`,
30+
// we'll use this later to split the asset file name more easily
31+
const baseName = `arduino-cli_${rel.name}_`
32+
33+
// Get a list of assets for this release
34+
const opts = github.repos.listAssetsForRelease.endpoint.merge({
35+
...context.repo,
36+
release_id: rel.id
37+
})
38+
const assets = await github.paginate(opts)
39+
40+
for (const asset of assets) {
41+
// Ignore files that are not arduino-cli packages
42+
if (!asset.name.startsWith(baseName)) {
43+
continue
44+
}
45+
46+
// Strip the base and remove file extension to get `Linux_32bit`
47+
systemArch = asset.name.replace(baseName, "").split(".")[0].split("_")
48+
49+
// Add a metric object to the list of gathered metrics
50+
metrics.push({
51+
"type": "gauge",
52+
"name": "arduino.downloads.total",
53+
"value": asset.download_count,
54+
"host": "${{ github.repository }}",
55+
"tags": [
56+
`version:${rel.name}`,
57+
`os:${systemArch[0]}`,
58+
`arch:${systemArch[1]}`,
59+
"cdn:github.com",
60+
"project:arduino-cli"
61+
]
62+
})
63+
}
64+
}
65+
66+
// The action will put whatever we return from this function in
67+
// `outputs.result`, JSON encoded. So we just return the array
68+
// of objects and GitHub will do the rest.
69+
return metrics
70+
71+
- name: Send metrics
72+
uses: masci/datadog@v1
73+
with:
74+
api-key: ${{ secrets.DD_API_KEY }}
75+
# Metrics input expects YAML but JSON will work just right.
76+
metrics: ${{steps.fetch.outputs.result}}
77+
78+
- name: Report failure
79+
if: failure()
80+
uses: masci/datadog@v1
81+
with:
82+
api-key: ${{ secrets.DD_API_KEY }}
83+
events: |
84+
- title: "Arduino CLI stats failing"
85+
text: "Stats collection failed"
86+
alert_type: "error"
87+
host: ${{ github.repository }}
88+
tags:
89+
- "project:arduino-cli"
90+
- "cdn:github.com"

Diff for: ‎.github/workflows/nightly.yaml

+13
Original file line numberDiff line numberDiff line change
@@ -33,3 +33,16 @@ jobs:
3333
PLUGIN_BUCKET: ${{ secrets.DOWNLOADS_BUCKET }}
3434
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
3535
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
36+
37+
- name: Report failure
38+
if: failure()
39+
uses: masci/datadog@v1
40+
with:
41+
api-key: ${{ secrets.DD_API_KEY }}
42+
events: |
43+
- title: "Arduino CLI nighly build failed"
44+
text: "Nightly build worfklow has failed"
45+
alert_type: "error"
46+
host: ${{ github.repository }}
47+
tags:
48+
- "project:arduino-cli"

Diff for: ‎.github/workflows/release.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ name: release
33
on:
44
push:
55
tags:
6-
- '[0-9].[0-9].[0-9]*'
6+
- '[0-9]+.[0-9]+.[0-9]+*'
77

88
jobs:
99

Diff for: ‎arduino/cores/packagemanager/download.go

+4-5
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ package packagemanager
1717

1818
import (
1919
"fmt"
20-
"net/http"
2120

2221
"github.com/arduino/arduino-cli/arduino/cores"
2322
"go.bug.st/downloader"
@@ -101,16 +100,16 @@ func (pm *PackageManager) FindPlatformReleaseDependencies(item *PlatformReferenc
101100

102101
// DownloadToolRelease downloads a ToolRelease. If the tool is already downloaded a nil Downloader
103102
// is returned.
104-
func (pm *PackageManager) DownloadToolRelease(tool *cores.ToolRelease, downloaderHeaders http.Header) (*downloader.Downloader, error) {
103+
func (pm *PackageManager) DownloadToolRelease(tool *cores.ToolRelease, config *downloader.Config) (*downloader.Downloader, error) {
105104
resource := tool.GetCompatibleFlavour()
106105
if resource == nil {
107106
return nil, fmt.Errorf("tool not available for your OS")
108107
}
109-
return resource.Download(pm.DownloadDir, downloaderHeaders)
108+
return resource.Download(pm.DownloadDir, config)
110109
}
111110

112111
// DownloadPlatformRelease downloads a PlatformRelease. If the platform is already downloaded a
113112
// nil Downloader is returned.
114-
func (pm *PackageManager) DownloadPlatformRelease(platform *cores.PlatformRelease, downloaderHeaders http.Header) (*downloader.Downloader, error) {
115-
return platform.Resource.Download(pm.DownloadDir, downloaderHeaders)
113+
func (pm *PackageManager) DownloadPlatformRelease(platform *cores.PlatformRelease, config *downloader.Config) (*downloader.Downloader, error) {
114+
return platform.Resource.Download(pm.DownloadDir, config)
116115
}

Diff for: ‎arduino/libraries/librariesmanager/download.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,8 @@ import (
2525
var LibraryIndexURL, _ = url.Parse("https://downloads.arduino.cc/libraries/library_index.json")
2626

2727
// UpdateIndex downloads the libraries index file from Arduino repository.
28-
func (lm *LibrariesManager) UpdateIndex() (*downloader.Downloader, error) {
28+
func (lm *LibrariesManager) UpdateIndex(config *downloader.Config) (*downloader.Downloader, error) {
2929
lm.IndexFile.Parent().MkdirAll()
3030
// TODO: Download from gzipped URL index
31-
return downloader.Download(lm.IndexFile.String(), LibraryIndexURL.String(), downloader.NoResume)
31+
return downloader.DownloadWithConfig(lm.IndexFile.String(), LibraryIndexURL.String(), *config, downloader.NoResume)
3232
}

Diff for: ‎arduino/resources/helpers.go

+2-5
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ package resources
1717

1818
import (
1919
"fmt"
20-
"net/http"
2120
"os"
2221

2322
"github.com/arduino/go-paths-helper"
@@ -44,7 +43,7 @@ func (r *DownloadResource) IsCached(downloadDir *paths.Path) (bool, error) {
4443
}
4544

4645
// Download a DownloadResource.
47-
func (r *DownloadResource) Download(downloadDir *paths.Path, downloaderHeaders http.Header) (*downloader.Downloader, error) {
46+
func (r *DownloadResource) Download(downloadDir *paths.Path, config *downloader.Config) (*downloader.Downloader, error) {
4847
cached, err := r.TestLocalArchiveIntegrity(downloadDir)
4948
if err != nil {
5049
return nil, fmt.Errorf("testing local archive integrity: %s", err)
@@ -72,7 +71,5 @@ func (r *DownloadResource) Download(downloadDir *paths.Path, downloaderHeaders h
7271
return nil, fmt.Errorf("getting archive file info: %s", err)
7372
}
7473

75-
downloadConfig := downloader.Config{
76-
RequestHeaders: downloaderHeaders}
77-
return downloader.DownloadWithConfig(path.String(), r.URL, downloadConfig)
74+
return downloader.DownloadWithConfig(path.String(), r.URL, *config)
7875
}

Diff for: ‎arduino/resources/helpers_test.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525

2626
"github.com/arduino/go-paths-helper"
2727
"github.com/stretchr/testify/require"
28+
"go.bug.st/downloader"
2829
)
2930

3031
type EchoHandler struct{}
@@ -52,7 +53,7 @@ func TestDownloadApplyUserAgentHeaderUsingConfig(t *testing.T) {
5253
URL: srv.URL,
5354
}
5455

55-
d, err := r.Download(tmp, http.Header{"User-Agent": []string{goldUserAgentValue}})
56+
d, err := r.Download(tmp, &downloader.Config{RequestHeaders: http.Header{"User-Agent": []string{goldUserAgentValue}}})
5657
require.NoError(t, err)
5758
err = d.Run()
5859
require.NoError(t, err)

Diff for: ‎arduino/resources/resources_test.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,11 @@ package resources
1818
import (
1919
"crypto"
2020
"encoding/hex"
21-
"net/http"
2221
"testing"
2322

2423
"github.com/arduino/go-paths-helper"
2524
"github.com/stretchr/testify/require"
25+
"go.bug.st/downloader"
2626
)
2727

2828
func TestDownloadAndChecksums(t *testing.T) {
@@ -42,7 +42,7 @@ func TestDownloadAndChecksums(t *testing.T) {
4242
require.NoError(t, err)
4343

4444
downloadAndTestChecksum := func() {
45-
d, err := r.Download(tmp, http.Header{})
45+
d, err := r.Download(tmp, &downloader.Config{})
4646
require.NoError(t, err)
4747
err = d.Run()
4848
require.NoError(t, err)
@@ -58,7 +58,7 @@ func TestDownloadAndChecksums(t *testing.T) {
5858
downloadAndTestChecksum()
5959

6060
// Download with cached file
61-
d, err := r.Download(tmp, http.Header{})
61+
d, err := r.Download(tmp, &downloader.Config{})
6262
require.NoError(t, err)
6363
require.Nil(t, d)
6464

Diff for: ‎cli/board/board.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ func NewCommand() *cobra.Command {
3030
Example: " # Lists all connected boards.\n" +
3131
" " + os.Args[0] + " board list\n\n" +
3232
" # Attaches a sketch to a board.\n" +
33-
" " + os.Args[0] + " board attach serial:///dev/tty/ACM0 mySketch",
33+
" " + os.Args[0] + " board attach serial:///dev/ttyACM0 mySketch",
3434
}
3535

3636
boardCommand.AddCommand(initAttachCommand())

Diff for: ‎cli/board/list.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ func initListCommand() *cobra.Command {
3535
listCommand := &cobra.Command{
3636
Use: "list",
3737
Short: "List connected boards.",
38-
Long: "Detects and displays a list of connected boards to the current computer.",
38+
Long: "Detects and displays a list of boards connected to the current computer.",
3939
Example: " " + os.Args[0] + " board list --timeout 10s",
4040
Args: cobra.NoArgs,
4141
Run: runListCommand,

0 commit comments

Comments
 (0)