stackabletech · stackable-bot · Feb 14, 2023
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -35,15 +35,15 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install protobuf-compiler
-      - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
         with:
           submodules: recursive
       - uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # renovate: tag=v1.0.7
         with:
           profile: minimal
           toolchain: "1.65.0"
           override: true
-      - uses: Swatinem/rust-cache@6720f05bc48b77f96918929a9019fb2203ff71f8 # tag=v2.0.0
+      - uses: Swatinem/rust-cache@359a70e43a0bb8a13953b04a90f76428b4959bb6 # v2.2.0
         with:
           key: udeps
       - uses: actions-rs/cargo@844f36862e911db73fe0815f00a4a2602c279505 # renovate: tag=v1.0.3
@@ -111,18 +111,18 @@ jobs:
     continue-on-error: ${{ matrix.checks == 'advisories' }}
 
     steps:
-      - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
         with:
           submodules: recursive
-      - uses: EmbarkStudios/cargo-deny-action@7257a18a9c2fe3f92b85d41ae473520dff953c97 # tag=v1.3.2
+      - uses: EmbarkStudios/cargo-deny-action@8a8607bd8e2b3a514d5a40174cc7c55b229d9ba7 # v1.4.0
         with:
           command: check ${{ matrix.checks }}
 
   run_rustfmt:
     name: Run Rustfmt
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
         with:
           submodules: recursive
       - uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # renovate: tag=v1.0.7
@@ -144,7 +144,7 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install protobuf-compiler
-      - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
         with:
           submodules: recursive
       - uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # renovate: tag=v1.0.7
@@ -153,7 +153,7 @@ jobs:
           toolchain: "1.65.0"
           components: clippy
           override: true
-      - uses: Swatinem/rust-cache@6720f05bc48b77f96918929a9019fb2203ff71f8 # tag=v2.0.0
+      - uses: Swatinem/rust-cache@359a70e43a0bb8a13953b04a90f76428b4959bb6 # v2.2.0
         with:
           key: clippy
       - name: Run clippy action to produce annotations
@@ -178,7 +178,7 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install protobuf-compiler
-      - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
         with:
           submodules: recursive
       - uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # renovate: tag=v1.0.7
@@ -187,7 +187,7 @@ jobs:
           toolchain: "1.65.0"
           components: rustfmt
           override: true
-      - uses: Swatinem/rust-cache@6720f05bc48b77f96918929a9019fb2203ff71f8 # tag=v2.0.0
+      - uses: Swatinem/rust-cache@359a70e43a0bb8a13953b04a90f76428b4959bb6 # v2.2.0
         with:
           key: doc
       - uses: actions-rs/cargo@844f36862e911db73fe0815f00a4a2602c279505 # renovate: tag=v1.0.3
@@ -203,15 +203,15 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install protobuf-compiler
-      - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
         with:
           submodules: recursive
       - uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # renovate: tag=v1.0.7
         with:
           profile: minimal
           toolchain: "1.65.0"
           override: true
-      - uses: Swatinem/rust-cache@6720f05bc48b77f96918929a9019fb2203ff71f8 # tag=v2.0.0
+      - uses: Swatinem/rust-cache@359a70e43a0bb8a13953b04a90f76428b4959bb6 # v2.2.0
         with:
           key: test
       - uses: actions-rs/cargo@844f36862e911db73fe0815f00a4a2602c279505 # renovate: tag=v1.0.3
@@ -226,10 +226,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
         with:
           submodules: recursive
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4
         with:
           python-version: '3.10'
       - name: Install jinja2-cli
@@ -240,7 +240,7 @@ jobs:
         run: git diff --exit-code
       - name: Git Diff showed uncommitted changes
         if: ${{ failure() }}
-        uses: actions/github-script@d50f485531ba88479582bc2da03ff424389af5c1 # tag=v6
+        uses: actions/github-script@d556feaca394842dc55e4734bf3bb9f685482fa0 # v6
         with:
           script: |
             core.setFailed('Committed README are not up to date, please make sure to apply them to the templated partials, and re-commit!')
@@ -264,11 +264,11 @@ jobs:
           sudo apt-get update
           sudo apt-get install protobuf-compiler
       - name: Checkout
-        uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
         with:
           submodules: recursive
       - name: Set up Helm
-        uses: azure/setup-helm@b5b231a831f96336bbfeccc1329990f0005c5bb1 # tag=v3.3
+        uses: azure/setup-helm@5119fcb9089d432beecbf79bb2c7915207344b78 # v3.5
         with:
           version: v3.6.2
       - name: Set up cargo
@@ -278,7 +278,7 @@ jobs:
           toolchain: "1.65.0"
           override: true
       - name: Set up rust-cache
-        uses: Swatinem/rust-cache@6720f05bc48b77f96918929a9019fb2203ff71f8 # tag=v2.0.0
+        uses: Swatinem/rust-cache@359a70e43a0bb8a13953b04a90f76428b4959bb6 # v2.2.0
         with:
           key: charts
       - name: Regenerate charts
@@ -287,7 +287,7 @@ jobs:
         run: git diff --exit-code
       - name: Git Diff showed uncommitted changes
         if: ${{ failure() }}
-        uses: actions/github-script@7dff1a87643417cf3b95bb10b29f4c4bc60d8ebd # tag=v6
+        uses: actions/github-script@d556feaca394842dc55e4734bf3bb9f685482fa0 # v6
         with:
           script: |
             core.setFailed('Committed charts were not up to date, please regenerate and re-commit!')
@@ -324,10 +324,10 @@ jobs:
           sudo apt-get update
           sudo apt-get install protobuf-compiler
       - name: Checkout
-        uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
         with:
           submodules: recursive
-      - uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5 # tag=v4
+      - uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4
         if: ${{ github.event_name == 'pull_request' }}
       - uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # renovate: tag=v1.0.7
         with:

diff --git a/.github/workflows/daily_security.yml b/.github/workflows/daily_security.yml
@@ -14,7 +14,7 @@ jobs:
   audit:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
       - uses: actions-rs/audit-check@35b7b53b1e25b55642157ac01b4adceb5b9ebef3 # renovate: tag=v1.2.0
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/reviewdog.yaml b/.github/workflows/reviewdog.yaml
@@ -13,16 +13,16 @@ jobs:
   actionlint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2
-      - uses: reviewdog/action-actionlint@fe5c2db6029881e0b5634d7cb0d119b0c16b6922 # tag=v1.32.0
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+      - uses: reviewdog/action-actionlint@b6feb003955cad286985c42e7047f4567a798f3f # v1.36.0
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
 
   flake8:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2
-      - uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5 # tag=v4
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+      - uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4
         with:
           python-version: "3.9"
       - uses: reviewdog/action-flake8@b6435e67f0cfda225b9e0c9283cfb7ea7c551bdb # tag=v3.6.0
@@ -32,15 +32,15 @@ jobs:
   hadolint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2
-      - uses: reviewdog/action-hadolint@55be5d2c4b0b80d439247b128a9ded3747f92a29 # tag=v1.33.0
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+      - uses: reviewdog/action-hadolint@141ffd8d2f0b75e6fc7c87341331985448b62aa4 # v1.34.1
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
 
   markdownlint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
       - uses: reviewdog/action-markdownlint@97e3df02fe1573d505a7b268046a44fa5409cbc3 # tag=v0.9.0
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
@@ -49,15 +49,15 @@ jobs:
   shellcheck:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2
-      - uses: reviewdog/action-shellcheck@66c9a47bf02255b250284a82251cb4cadf5043f5 # tag=v1.15.0
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
+      - uses: reviewdog/action-shellcheck@f52d78284b4941028952125a10c76676c3d456eb # v1.17.0
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
 
   yamllint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2
+      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
       - uses: reviewdog/action-yamllint@8c429dfe4fc47b1ce1fa99a64e94693880d5dc30 # tag=v1.6.1
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.gitignore b/.gitignore
@@ -12,4 +12,6 @@ target/
 Cargo.nix
 crate-hashes.json
 result
-image.tar
+image.tar
+
+tilt_options.json
diff --git a/Makefile b/Makefile
@@ -73,3 +73,6 @@ regenerate-charts: chart-clean compile-chart
 build: regenerate-charts helm-package docker-build
 
 publish: build docker-publish helm-publish
+
+run-dev:
+	nix run -f. tilt -- up --port 5441
diff --git a/Tiltfile b/Tiltfile
@@ -0,0 +1,36 @@
+# If tilt_options.json exists read it and load the default_registry value from it
+settings = read_json('tilt_options.json', default={})
+registry = settings.get('default_registry', 'docker.stackable.tech/sandbox')
+
+# Configure default registry either read from config file above, or with default value of "docker.stackable.tech/sandbox"
+default_registry(registry)
+
+meta = read_json('nix/meta.json')
+operator_name = meta['operator']['name']
+
+custom_build(
+    registry + '/' + operator_name,
+    'nix shell -f . crate2nix -c crate2nix generate && nix-build . -A docker --argstr dockerName "${EXPECTED_REGISTRY}/' + operator_name + '" && ./result/load-image | docker load',
+    deps=['rust', 'Cargo.toml', 'Cargo.lock', 'default.nix', "nix", 'build.rs', 'vendor'],
+    # ignore=['result*', 'Cargo.nix', 'target', *.yaml],
+    outputs_image_ref_to='result/ref',
+)
+
+# Load the latest CRDs from Nix
+watch_file('result')
+if os.path.exists('result'):
+   k8s_yaml('result/crds.yaml')
+
+# Exclude stale CRDs from Helm chart, and apply the rest
+helm_crds, helm_non_crds = filter_yaml(
+   helm(
+      'deploy/helm/' + operator_name,
+      name=operator_name,
+      set=[
+         'image.repository=' + registry + '/' + operator_name,
+      ],
+   ),
+   api_version = "^apiextensions\\.k8s\\.io/.*$",
+   kind = "^CustomResourceDefinition$",
+)
+k8s_yaml(helm_non_crds)
diff --git a/default.nix b/default.nix
@@ -0,0 +1,68 @@
+{ sources ? import ./nix/sources.nix # managed by https://github.com/nmattia/niv
+, nixpkgs ? sources.nixpkgs
+, pkgs ? import nixpkgs {}
+, cargo ? import ./Cargo.nix {
+    inherit nixpkgs pkgs; release = false;
+    defaultCrateOverrides = pkgs.defaultCrateOverrides // {
+      prost-build = attrs: {
+        buildInputs = [ pkgs.protobuf ];
+      };
+      tonic-reflection = attrs: {
+        buildInputs = [ pkgs.rustfmt ];
+      };
+    };
+  }
+, meta ? pkgs.lib.importJSON ./nix/meta.json
+, dockerName ? "docker.stackable.tech/sandbox/${meta.operator.name}"
+, dockerTag ? null
+}:
+rec {
+  build = cargo.allWorkspaceMembers;
+  entrypoint = build+"/bin/stackable-${meta.operator.name}";
+  crds = pkgs.runCommand "${meta.operator.name}-crds.yaml" {}
+  ''
+    ${entrypoint} crd > $out
+  '';
+
+  dockerImage = pkgs.dockerTools.streamLayeredImage {
+    name = dockerName;
+    tag = dockerTag;
+    contents = [ pkgs.bashInteractive pkgs.coreutils pkgs.util-linuxMinimal ];
+    config = {
+    Env =
+      let
+        fileRefVars = {
+          PRODUCT_CONFIG = deploy/config-spec/properties.yaml;
+        };
+      in pkgs.lib.concatLists (pkgs.lib.mapAttrsToList (env: path: pkgs.lib.optional (pkgs.lib.pathExists path) "${env}=${path}") fileRefVars);
+      Entrypoint = [ entrypoint ];
+      Cmd = [ "run" ];
+    };
+  };
+  docker = pkgs.linkFarm "listener-operator-docker" [
+    {
+      name = "load-image";
+      path = dockerImage;
+    }
+    {
+      name = "ref";
+      path = pkgs.writeText "${dockerImage.name}-image-tag" "${dockerImage.imageName}:${dockerImage.imageTag}";
+    }
+    {
+      name = "image-repo";
+      path = pkgs.writeText "${dockerImage.name}-repo" dockerImage.imageName;
+    }
+    {
+      name = "image-tag";
+      path = pkgs.writeText "${dockerImage.name}-tag" dockerImage.imageTag;
+    }
+    {
+      name = "crds.yaml";
+      path = crds;
+    }
+  ];
+
+  # need to use vendored crate2nix because of https://github.com/kolloch/crate2nix/issues/264
+  crate2nix = import sources.crate2nix {};
+  tilt = pkgs.tilt;
+}
diff --git a/nix/meta.json b/nix/meta.json
@@ -0,0 +1 @@
+{"operator": {"name": "spark-k8s-operator", "pretty_string": "Apache Spark-on-Kubernetes", "product_string": "spark-k8s", "url": "stackabletech/spark-k8s-operator.git"}}
diff --git a/nix/sources.json b/nix/sources.json
@@ -0,0 +1,26 @@
+{
+    "crate2nix": {
+        "branch": "master",
+        "description": "nix build file generator for rust crates",
+        "homepage": "",
+        "owner": "kolloch",
+        "repo": "crate2nix",
+        "rev": "45fc83132c8c91c77a1cd61fe0c945411d1edba8",
+        "sha256": "00w9qza56xh9m98308wzfbjf135gpz6sf7b5yfharvmax1k46c5q",
+        "type": "tarball",
+        "url": "https://github.com/kolloch/crate2nix/archive/45fc83132c8c91c77a1cd61fe0c945411d1edba8.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    },
+    "nixpkgs": {
+        "branch": "nixpkgs-unstable",
+        "description": "Nix Packages collection",
+        "homepage": "",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "747927516efcb5e31ba03b7ff32f61f6d47e7d87",
+        "sha256": "1s4xabv59r99z8vd74w3r84kkxwqggqir3b0nh3ma04mni0m40gf",
+        "type": "tarball",
+        "url": "https://github.com/NixOS/nixpkgs/archive/747927516efcb5e31ba03b7ff32f61f6d47e7d87.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    }
+}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		{"operator": {"name": "spark-k8s-operator", "pretty_string": "Apache Spark-on-Kubernetes", "product_string": "spark-k8s", "url": "stackabletech/spark-k8s-operator.git"}}