ci: test sqlsmith on release (#13072)

everpcpc · web-flow · commit c228a20eec91 · 2023-09-29T11:59:30.000+08:00
diff --git a/.github/actions/publish_binary/action.yml b/.github/actions/publish_binary/action.yml
@@ -8,7 +8,7 @@ inputs:
     description: "Release target"
     required: true
   category:
-    description: "Release default / hdfs"
+    description: "Release default/hdfs/testsuite"
     required: false
     default: default
 
@@ -26,6 +26,9 @@ runs:
           hdfs)
             publish_name="databend-hdfs-${{ inputs.version }}-${{ inputs.target }}"
             ;;
+          testsuite)
+            publish_name="databend-testsuite-${{ inputs.version }}-${{ inputs.target }}"
+            ;;
           *)
             echo "Unknown release category {{ inputs.category }}"
             exit 1
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -187,7 +187,7 @@ jobs:
         with:
           sha: ${{ github.sha }}
           target: ${{ steps.target.outputs.target }}
-          artifacts: sqllogictests,metactl,meta,query
+          artifacts: sqllogictests,sqlsmith,metactl,meta,query
           upload: false
       - name: Basic Sqllogic Test
         if: matrix.arch == 'x86_64'
@@ -198,29 +198,35 @@ jobs:
           mkdir -p target/release
           cp ./target/${{ steps.target.outputs.target }}/release/databend-{meta,query,sqllogictests} ./target/release/
           bash ./scripts/ci/ci-run-sqllogic-tests.sh base
-      - name: Pack binaries
+      - name: Pack Binaries
         run: |
           target=${{ steps.target.outputs.target }}
           version=${{ needs.create_release.outputs.version }}
           mkdir -p release/${target}/{bin,configs,systemd,scripts}
-          cp ./target/${target}/release/databend-* release/${target}/bin/
+          cp ./target/${target}/release/databend-{query,meta,metactl} release/${target}/bin/
           rm -f release/${target}/bin/*.d
           cp ./scripts/distribution/systemd/databend-* release/${target}/systemd/
           cp ./scripts/distribution/configs/databend-* release/${target}/configs/
           cp ./scripts/distribution/release-readme.txt release/${target}/readme.txt
           cp -r ./scripts/distribution/local-scripts/* release/${target}/scripts/
           cp -r ./scripts/distribution/package-scripts/* release/${target}/scripts/
           tar -C ./release/${target} -czvf databend-${version}-${target}.tar.gz bin configs systemd scripts readme.txt
-      - name: generate sha256sums
+          sha256sum databend-${version}-${target}.tar.gz >> sha256-${version}-${target}.txt
+      - name: Pack Testsuite
+        if: steps.target.outputs.target == 'x86_64-unknown-linux-gnu'
         run: |
           target=${{ steps.target.outputs.target }}
           version=${{ needs.create_release.outputs.version }}
-          sha256sum databend-${version}-${target}.tar.gz >> sha256-${version}-${target}.txt
+          mkdir -p release/testsuite/bin
+          cp -r ./tests/sqllogictests/suites ./release/testsuite/
+          cp ./target/${target}/release/databend-{sqllogictests,sqlsmith} release/testsuite/bin/
+          tar -C ./release/testsuite -czvf databend-testsuite-${version}-${target}.tar.gz bin suites
+          sha256sum databend-testsuite-${version}-${target}.tar.gz >> sha256-testsuite-${version}-${target}.txt
       - name: post sha256
         uses: actions/upload-artifact@v3
         with:
           name: sha256sums
-          path: sha256-${{ needs.create_release.outputs.version }}-${{ steps.target.outputs.target }}.txt
+          path: sha256-*.txt
           retention-days: 1
       - name: Publish Binaries
         uses: ./.github/actions/publish_binary
@@ -233,6 +239,15 @@ jobs:
         with:
           version: ${{ needs.create_release.outputs.version }}
           target: ${{ steps.target.outputs.target }}
+      - name: Publish Testsuite
+        if: steps.target.outputs.target == 'x86_64-unknown-linux-gnu'
+        uses: ./.github/actions/publish_binary
+        env:
+          GH_TOKEN: ${{ github.token }}
+        with:
+          version: ${{ needs.create_release.outputs.version }}
+          target: ${{ steps.target.outputs.target }}
+          category: testsuite
 
   hdfs:
     runs-on: [self-hosted, X64, Linux, 16c32g, gcp]
@@ -283,16 +298,12 @@ jobs:
           cp -r ./scripts/distribution/local-scripts/* release/${target}/scripts/
           cp -r ./scripts/distribution/package-scripts/* release/${target}/scripts/
           tar -C ./release/${target} -czvf databend-hdfs-${version}-${target}.tar.gz bin configs systemd scripts readme.txt
-      - name: generate sha256sums
-        run: |
-          target=${{ steps.target.outputs.target }}
-          version=${{ needs.create_release.outputs.version }}
           sha256sum databend-hdfs-${version}-${target}.tar.gz >> sha256-hdfs-${version}-${target}.txt
       - name: post sha256
         uses: actions/upload-artifact@v3
         with:
           name: sha256sums
-          path: sha256-hdfs-${{ needs.create_release.outputs.version }}-${{ steps.target.outputs.target }}.txt
+          path: sha256-*.txt
           retention-days: 1
       - name: Publish Binaries
         uses: ./.github/actions/publish_binary
@@ -307,23 +318,6 @@ jobs:
           target: ${{ steps.target.outputs.target }}
           category: hdfs
 
-  testsuites:
-    runs-on: ubuntu-latest
-    needs: create_release
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-      - name: Get the version
-        id: get_version
-        run: echo "VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT
-      - name: Upload to github release
-        env:
-          GH_TOKEN: ${{ github.token }}
-        run: |
-          version=${{ needs.create_release.outputs.version }}
-          tar -C ./tests/sqllogictests -czvf testsuites-${version}.tar.gz suites
-          gh release upload ${version} testsuites-${version}.tar.gz --clobber
-
   docker_combined:
     name: docker combined
     runs-on: ubuntu-latest
@@ -559,6 +553,35 @@ jobs:
       version: ${{ needs.create_release.outputs.version }}
       runner_provider: aws
 
+  sqlsmith:
+    needs: [create_release, linux]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Download artifact for release
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          version=${{ needs.create_release.outputs.version }}
+          target=x86_64-unknown-linux-gnu
+          mkdir -p ./distro/
+          mkdir -p ./target/release/
+          gh release download ${version} --pattern "databend-${version}-${target}.tar.gz" --dir distro/
+          gh release download ${version} --pattern "databend-testsuite-${version}-${target}.tar.gz" --dir distro/
+          tar x -C ./target/release -f ./distro/databend-${version}-${target}.tar.gz --strip-components 1 bin/
+          tar x -C ./target/release -f ./distro/databend-testsuite-${version}-${target}.tar.gz --strip-components 1 bin/
+          chmod +x ./target/release/databend-*
+      - name: Run sqlsmith
+        timeout-minutes: 30
+        shell: bash
+        run: |
+          bash ./scripts/ci/ci-run-sqlsmith-tests.sh
+      - name: Upload failure
+        if: failure()
+        uses: ./.github/actions/artifact_failure
+        with:
+          name: test-sqlsmith
+
   bindings_python:
     needs: create_release
     uses: ./.github/workflows/bindings.python.yml
diff --git a/scripts/build/build-debug.sh b/scripts/build/build-debug.sh
@@ -8,5 +8,5 @@ SCRIPT_PATH="$(cd "$(dirname "$0")" >/dev/null 2>&1 && pwd)"
 cd "$SCRIPT_PATH/../.." || exit
 
 echo "Build(DEBUG) start..."
-cargo build --bin=databend-query --bin=databend-meta --bin=databend-metactl --bin=open-sharing --bin=databend-query-oss --bin=databend-meta-oss
+cargo build --bin=databend-query --bin=databend-meta --bin=databend-metactl --bin=databend-sqllogictests --bin=databend-sqlsmith
 echo "All done..."
diff --git a/scripts/ci/ci-run-sqlsmith-tests.sh b/scripts/ci/ci-run-sqlsmith-tests.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+# Copyright 2020-2021 The Databend Authors.
+# SPDX-License-Identifier: Apache-2.0.
+
+set -e
+
+echo "Starting standalone DatabendQuery and DatabendMeta"
+./scripts/ci/deploy/databend-query-standalone.sh
+
+BUILD_PROFILE=${BUILD_PROFILE:-debug}
+
+echo 'Starting databend-sqlsmith tests...'
+nohup target/${BUILD_PROFILE}/databend-sqlsmith
diff --git a/src/tests/sqlsmith/Cargo.toml b/src/tests/sqlsmith/Cargo.toml
@@ -29,5 +29,5 @@ common-io = { path = "../../common/io" }
 common-sql = { path = "../../query/sql" }
 
 [[bin]]
-name = "sqlsmith"
+name = "databend-sqlsmith"
 path = "src/bin/main.rs"
