Merge branch 'master' into arduifine

Author: d-a-v

Diff for: .dependabot/config.yml

@@ -19,6 +19,6 @@ indent_size = 4
 indent_style = space
 indent_size = 2
 
-[*.sh]
-indent_style = tab
-indent_size = 4
+[*.{bash,sh}]
+indent_size = 2
+indent_style = space

Diff for: .editorconfig

@@ -1,9 +1,16 @@
-# TODO: move this to pyproject.toml when supported: https://gitlab.com/pycqa/flake8/merge_requests/245
+# Source: https://github.com/arduino/tooling-project-assets/blob/main/workflow-templates/assets/check-python/.flake8
+# See: https://flake8.pycqa.org/en/latest/user/configuration.html
+# The code style defined in this file is the official standardized style to be used in all Arduino tooling projects and
+# should not be modified.
 
 [flake8]
-max-line-length = 120
+doctests = True
+per-file-ignores =
+    test/test_upload_mock.py:E501
 ignore =
     E741,
-    # W503 and W504 are mutually exclusive, so one or the other must be ignored.
-    # PEP 8 recommends line break before, so we keep W504.
+    # W503 and W504 are mutually exclusive. PEP 8 recommends line break before.
     W503
+max-complexity = 10
+max-line-length = 120
+select = E,W,F,C,N

Diff for: .flake8

@@ -0,0 +1,13 @@
+# See: https://docs.github.com/en/code-security/supply-chain-security/configuration-options-for-dependency-updates#about-the-dependabotyml-file
+version: 2
+
+updates:
+  # Configure check for outdated GitHub Actions actions in workflows.
+  # Source: https://github.com/arduino/tooling-project-assets/blob/main/workflow-templates/assets/dependabot/README.md
+  # See: https://docs.github.com/en/code-security/supply-chain-security/keeping-your-actions-up-to-date-with-dependabot
+  - package-ecosystem: github-actions
+    directory: / # Check the repository's workflows under /.github/workflows/
+    schedule:
+      interval: daily
+    labels:
+      - "topic: infrastructure"

Diff for: .github/dependabot.yml

@@ -0,0 +1,21 @@
+# Used by the "Sync Labels" workflow
+# See: https://github.com/Financial-Times/github-label-sync#label-config-file
+
+- name: "architecture: arm64"
+  color: ff00ff
+  description: Specific to 64 bit ARM host architecture
+- name: "architecture: armv6"
+  color: ff00ff
+  description: Specific to hosts using an ARMv6 core
+- name: "architecture: armv7"
+  color: ff00ff
+  description: Specific to hosts using an ARMv7 core
+- name: "topic: CLI"
+  color: "00ffff"
+  description: Related to the command line interface
+- name: "topic: gRPC"
+  color: "00ffff"
+  description: Related to the gRPC interface
+- name: "topic: packaging"
+  color: "00ffff"
+  description: Related to the release distribution package

Diff for: .github/label-configuration-files/labels.yml

@@ -0,0 +1,135 @@
+import boto3
+import semver
+import os
+import logging
+import uuid
+import time
+
+
+# logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
+log = logging.getLogger()
+logging.getLogger("boto3").setLevel(logging.CRITICAL)
+logging.getLogger("botocore").setLevel(logging.CRITICAL)
+logging.getLogger("urllib3").setLevel(logging.CRITICAL)
+
+
+def execute(client, statement, dest_s3_output_location):
+    log.info("execute query: {} dumping in {}".format(statement, dest_s3_output_location))
+    result = client.start_query_execution(
+        QueryString=statement,
+        ClientRequestToken=str(uuid.uuid4()),
+        ResultConfiguration={
+            "OutputLocation": dest_s3_output_location,
+        },
+    )
+    execution_id = result["QueryExecutionId"]
+    log.info("wait for query {} completion".format(execution_id))
+    wait_for_query_execution_completion(client, execution_id)
+    log.info("operation successful")
+    return execution_id
+
+
+def wait_for_query_execution_completion(client, query_execution_id):
+    query_ended = False
+    while not query_ended:
+        query_execution = client.get_query_execution(QueryExecutionId=query_execution_id)
+        state = query_execution["QueryExecution"]["Status"]["State"]
+        if state == "SUCCEEDED":
+            query_ended = True
+        elif state in ["FAILED", "CANCELLED"]:
+            raise BaseException(
+                "query failed or canceled: {}".format(query_execution["QueryExecution"]["Status"]["StateChangeReason"])
+            )
+        else:
+            time.sleep(1)
+
+
+def valid(key):
+    split = key.split("_")
+    if len(split) < 1:
+        return False
+    try:
+        semver.parse(split[0])
+    except ValueError:
+        return False
+    return True
+
+
+def get_results(client, execution_id):
+    results_paginator = client.get_paginator("get_query_results")
+    results_iter = results_paginator.paginate(QueryExecutionId=execution_id, PaginationConfig={"PageSize": 1000})
+    res = {}
+    for results_page in results_iter:
+        for row in results_page["ResultSet"]["Rows"][1:]:
+            # Loop through the JSON objects
+            key = row["Data"][0]["VarCharValue"]
+            if valid(key):
+                res[key] = row["Data"][1]["VarCharValue"]
+
+    return res
+
+
+def convert_data(data):
+    result = []
+    for key, value in data.items():
+        # 0.18.0_macOS_64bit.tar.gz
+        split_key = key.split("_")
+        if len(split_key) != 3:
+            continue
+        (version, os_version, arch) = split_key
+        arch_split = arch.split(".")
+        if len(arch_split) < 1:
+            continue
+        arch = arch_split[0]
+        if len(arch) > 10:
+            # This can't be an architecture really.
+            # It's an ugly solution but works for now so deal with it.
+            continue
+        repo = os.environ["GITHUB_REPOSITORY"].split("/")[1]
+        result.append(
+            {
+                "type": "gauge",
+                "name": "arduino.downloads.total",
+                "value": value,
+                "host": os.environ["GITHUB_REPOSITORY"],
+                "tags": [
+                    f"version:{version}",
+                    f"os:{os_version}",
+                    f"arch:{arch}",
+                    "cdn:downloads.arduino.cc",
+                    f"project:{repo}",
+                ],
+            }
+        )
+
+    return result
+
+
+if __name__ == "__main__":
+    DEST_S3_OUTPUT = os.environ["AWS_ATHENA_OUTPUT_LOCATION"]
+    AWS_ATHENA_SOURCE_TABLE = os.environ["AWS_ATHENA_SOURCE_TABLE"]
+
+    session = boto3.session.Session(region_name="us-east-1")
+    athena_client = session.client("athena")
+
+    # Load all partitions before querying downloads
+    execute(athena_client, f"MSCK REPAIR TABLE {AWS_ATHENA_SOURCE_TABLE};", DEST_S3_OUTPUT)
+
+    query = f"""SELECT replace(json_extract_scalar(url_decode(url_decode(querystring)),
+'$.data.url'), 'https://downloads.arduino.cc/arduino-cli/arduino-cli_', '')
+AS flavor, count(json_extract(url_decode(url_decode(querystring)),'$')) AS gauge
+FROM {AWS_ATHENA_SOURCE_TABLE}
+WHERE json_extract_scalar(url_decode(url_decode(querystring)),'$.data.url')
+LIKE 'https://downloads.arduino.cc/arduino-cli/arduino-cli_%'
+AND json_extract_scalar(url_decode(url_decode(querystring)),'$.data.url')
+NOT LIKE '%latest%' -- exclude latest redirect
+AND json_extract_scalar(url_decode(url_decode(querystring)),'$.data.url')
+NOT LIKE '%alpha%' -- exclude early alpha releases
+AND json_extract_scalar(url_decode(url_decode(querystring)),'$.data.url')
+NOT LIKE '%.tar.bz2%' -- exclude very old releases archive formats
+group by 1 ;"""
+    exec_id = execute(athena_client, query, DEST_S3_OUTPUT)
+    results = get_results(athena_client, exec_id)
+    result_json = convert_data(results)
+
+    print(f"::set-output name=result::{result_json}")

Diff for: .github/tools/fetch_athena_stats.py

@@ -2,8 +2,10 @@ name: arduino-stats
 
 on:
   schedule:
-    # run every day at 12:30:00
-    - cron: "30 12 * * *"
+    # run every day at 07:00 AM, 03:00 PM and 11:00 PM
+    - cron: "0 7,15,23 * * *"
+  workflow_dispatch:
+  repository_dispatch:
 
 jobs:
   push-stats:
@@ -16,6 +18,10 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v2
 
+      - uses: actions/setup-python@v2
+        with:
+          python-version: "3.x"
+
       - name: Fetch downloads count form Arduino CDN using AWS Athena
         id: fetch
         env:
@@ -25,11 +31,8 @@ jobs:
           AWS_ATHENA_OUTPUT_LOCATION: ${{ secrets.STATS_AWS_ATHENA_OUTPUT_LOCATION }}
           GITHUB_REPOSITORY: ${{ github.repository }}
         run: |
-          # Fetch jq 1.6 as VM has only 1.5 ATM
-          wget -q https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 -O jq
-          chmod +x jq
-          PATH="${{ github.workspace }}:$PATH"
-          .github/tools/fetch_athena_stats.sh
+          pip install boto3 semver
+          python .github/tools/fetch_athena_stats.py
 
       - name: Send metrics
         uses: masci/datadog@v1