diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 36fe7ffb..bdf23f22 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -35,7 +35,7 @@ jobs: run: | sudo apt-get update sudo apt-get install protobuf-compiler - - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2 + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 with: submodules: recursive - uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # renovate: tag=v1.0.7 @@ -43,7 +43,7 @@ jobs: profile: minimal toolchain: "1.65.0" override: true - - uses: Swatinem/rust-cache@6720f05bc48b77f96918929a9019fb2203ff71f8 # tag=v2.0.0 + - uses: Swatinem/rust-cache@359a70e43a0bb8a13953b04a90f76428b4959bb6 # v2.2.0 with: key: udeps - uses: actions-rs/cargo@844f36862e911db73fe0815f00a4a2602c279505 # renovate: tag=v1.0.3 @@ -111,10 +111,10 @@ jobs: continue-on-error: ${{ matrix.checks == 'advisories' }} steps: - - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2 + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 with: submodules: recursive - - uses: EmbarkStudios/cargo-deny-action@7257a18a9c2fe3f92b85d41ae473520dff953c97 # tag=v1.3.2 + - uses: EmbarkStudios/cargo-deny-action@8a8607bd8e2b3a514d5a40174cc7c55b229d9ba7 # v1.4.0 with: command: check ${{ matrix.checks }} @@ -122,7 +122,7 @@ jobs: name: Run Rustfmt runs-on: ubuntu-latest steps: - - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2 + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 with: submodules: recursive - uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # renovate: tag=v1.0.7 @@ -144,7 +144,7 @@ jobs: run: | sudo apt-get update sudo apt-get install protobuf-compiler - - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2 + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 with: submodules: recursive - uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # renovate: tag=v1.0.7 @@ -153,7 +153,7 @@ jobs: toolchain: "1.65.0" components: clippy override: true - - uses: Swatinem/rust-cache@6720f05bc48b77f96918929a9019fb2203ff71f8 # tag=v2.0.0 + - uses: Swatinem/rust-cache@359a70e43a0bb8a13953b04a90f76428b4959bb6 # v2.2.0 with: key: clippy - name: Run clippy action to produce annotations @@ -178,7 +178,7 @@ jobs: run: | sudo apt-get update sudo apt-get install protobuf-compiler - - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2 + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 with: submodules: recursive - uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # renovate: tag=v1.0.7 @@ -187,7 +187,7 @@ jobs: toolchain: "1.65.0" components: rustfmt override: true - - uses: Swatinem/rust-cache@6720f05bc48b77f96918929a9019fb2203ff71f8 # tag=v2.0.0 + - uses: Swatinem/rust-cache@359a70e43a0bb8a13953b04a90f76428b4959bb6 # v2.2.0 with: key: doc - uses: actions-rs/cargo@844f36862e911db73fe0815f00a4a2602c279505 # renovate: tag=v1.0.3 @@ -203,7 +203,7 @@ jobs: run: | sudo apt-get update sudo apt-get install protobuf-compiler - - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2 + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 with: submodules: recursive - uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # renovate: tag=v1.0.7 @@ -211,7 +211,7 @@ jobs: profile: minimal toolchain: "1.65.0" override: true - - uses: Swatinem/rust-cache@6720f05bc48b77f96918929a9019fb2203ff71f8 # tag=v2.0.0 + - uses: Swatinem/rust-cache@359a70e43a0bb8a13953b04a90f76428b4959bb6 # v2.2.0 with: key: test - uses: actions-rs/cargo@844f36862e911db73fe0815f00a4a2602c279505 # renovate: tag=v1.0.3 @@ -226,10 +226,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2 + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 with: submodules: recursive - - uses: actions/setup-python@v4 + - uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4 with: python-version: '3.10' - name: Install jinja2-cli @@ -240,7 +240,7 @@ jobs: run: git diff --exit-code - name: Git Diff showed uncommitted changes if: ${{ failure() }} - uses: actions/github-script@d50f485531ba88479582bc2da03ff424389af5c1 # tag=v6 + uses: actions/github-script@d556feaca394842dc55e4734bf3bb9f685482fa0 # v6 with: script: | core.setFailed('Committed README are not up to date, please make sure to apply them to the templated partials, and re-commit!') @@ -264,11 +264,11 @@ jobs: sudo apt-get update sudo apt-get install protobuf-compiler - name: Checkout - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2 + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 with: submodules: recursive - name: Set up Helm - uses: azure/setup-helm@b5b231a831f96336bbfeccc1329990f0005c5bb1 # tag=v3.3 + uses: azure/setup-helm@5119fcb9089d432beecbf79bb2c7915207344b78 # v3.5 with: version: v3.6.2 - name: Set up cargo @@ -278,7 +278,7 @@ jobs: toolchain: "1.65.0" override: true - name: Set up rust-cache - uses: Swatinem/rust-cache@6720f05bc48b77f96918929a9019fb2203ff71f8 # tag=v2.0.0 + uses: Swatinem/rust-cache@359a70e43a0bb8a13953b04a90f76428b4959bb6 # v2.2.0 with: key: charts - name: Regenerate charts @@ -287,7 +287,7 @@ jobs: run: git diff --exit-code - name: Git Diff showed uncommitted changes if: ${{ failure() }} - uses: actions/github-script@7dff1a87643417cf3b95bb10b29f4c4bc60d8ebd # tag=v6 + uses: actions/github-script@d556feaca394842dc55e4734bf3bb9f685482fa0 # v6 with: script: | core.setFailed('Committed charts were not up to date, please regenerate and re-commit!') @@ -324,10 +324,10 @@ jobs: sudo apt-get update sudo apt-get install protobuf-compiler - name: Checkout - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2 + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 with: submodules: recursive - - uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5 # tag=v4 + - uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4 if: ${{ github.event_name == 'pull_request' }} - uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # renovate: tag=v1.0.7 with: diff --git a/.github/workflows/daily_security.yml b/.github/workflows/daily_security.yml index b609d366..06b5d462 100644 --- a/.github/workflows/daily_security.yml +++ b/.github/workflows/daily_security.yml @@ -14,7 +14,7 @@ jobs: audit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2 + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 - uses: actions-rs/audit-check@35b7b53b1e25b55642157ac01b4adceb5b9ebef3 # renovate: tag=v1.2.0 with: token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/reviewdog.yaml b/.github/workflows/reviewdog.yaml index c91d5eac..fbc81952 100644 --- a/.github/workflows/reviewdog.yaml +++ b/.github/workflows/reviewdog.yaml @@ -13,16 +13,16 @@ jobs: actionlint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2 - - uses: reviewdog/action-actionlint@fe5c2db6029881e0b5634d7cb0d119b0c16b6922 # tag=v1.32.0 + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + - uses: reviewdog/action-actionlint@b6feb003955cad286985c42e7047f4567a798f3f # v1.36.0 with: github_token: ${{ secrets.GITHUB_TOKEN }} flake8: runs-on: ubuntu-latest steps: - - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2 - - uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5 # tag=v4 + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + - uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4 with: python-version: "3.9" - uses: reviewdog/action-flake8@b6435e67f0cfda225b9e0c9283cfb7ea7c551bdb # tag=v3.6.0 @@ -32,15 +32,15 @@ jobs: hadolint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2 - - uses: reviewdog/action-hadolint@55be5d2c4b0b80d439247b128a9ded3747f92a29 # tag=v1.33.0 + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + - uses: reviewdog/action-hadolint@141ffd8d2f0b75e6fc7c87341331985448b62aa4 # v1.34.1 with: github_token: ${{ secrets.GITHUB_TOKEN }} markdownlint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2 + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 - uses: reviewdog/action-markdownlint@97e3df02fe1573d505a7b268046a44fa5409cbc3 # tag=v0.9.0 with: github_token: ${{ secrets.GITHUB_TOKEN }} @@ -49,15 +49,15 @@ jobs: shellcheck: runs-on: ubuntu-latest steps: - - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2 - - uses: reviewdog/action-shellcheck@66c9a47bf02255b250284a82251cb4cadf5043f5 # tag=v1.15.0 + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + - uses: reviewdog/action-shellcheck@f52d78284b4941028952125a10c76676c3d456eb # v1.17.0 with: github_token: ${{ secrets.GITHUB_TOKEN }} yamllint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # tag=v3.0.2 + - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 - uses: reviewdog/action-yamllint@8c429dfe4fc47b1ce1fa99a64e94693880d5dc30 # tag=v1.6.1 with: github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 132b42c3..44b2140f 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,6 @@ target/ Cargo.nix crate-hashes.json result -image.tar \ No newline at end of file +image.tar + +tilt_options.json \ No newline at end of file diff --git a/Makefile b/Makefile index b6a03c81..f2645f94 100644 --- a/Makefile +++ b/Makefile @@ -73,3 +73,6 @@ regenerate-charts: chart-clean compile-chart build: regenerate-charts helm-package docker-build publish: build docker-publish helm-publish + +run-dev: + nix run -f. tilt -- up --port 5441 \ No newline at end of file diff --git a/Tiltfile b/Tiltfile new file mode 100644 index 00000000..375a77e9 --- /dev/null +++ b/Tiltfile @@ -0,0 +1,36 @@ +# If tilt_options.json exists read it and load the default_registry value from it +settings = read_json('tilt_options.json', default={}) +registry = settings.get('default_registry', 'docker.stackable.tech/sandbox') + +# Configure default registry either read from config file above, or with default value of "docker.stackable.tech/sandbox" +default_registry(registry) + +meta = read_json('nix/meta.json') +operator_name = meta['operator']['name'] + +custom_build( + registry + '/' + operator_name, + 'nix shell -f . crate2nix -c crate2nix generate && nix-build . -A docker --argstr dockerName "${EXPECTED_REGISTRY}/' + operator_name + '" && ./result/load-image | docker load', + deps=['rust', 'Cargo.toml', 'Cargo.lock', 'default.nix', "nix", 'build.rs', 'vendor'], + # ignore=['result*', 'Cargo.nix', 'target', *.yaml], + outputs_image_ref_to='result/ref', +) + +# Load the latest CRDs from Nix +watch_file('result') +if os.path.exists('result'): + k8s_yaml('result/crds.yaml') + +# Exclude stale CRDs from Helm chart, and apply the rest +helm_crds, helm_non_crds = filter_yaml( + helm( + 'deploy/helm/' + operator_name, + name=operator_name, + set=[ + 'image.repository=' + registry + '/' + operator_name, + ], + ), + api_version = "^apiextensions\\.k8s\\.io/.*$", + kind = "^CustomResourceDefinition$", +) +k8s_yaml(helm_non_crds) diff --git a/default.nix b/default.nix new file mode 100644 index 00000000..1cf848c2 --- /dev/null +++ b/default.nix @@ -0,0 +1,68 @@ +{ sources ? import ./nix/sources.nix # managed by https://github.com/nmattia/niv +, nixpkgs ? sources.nixpkgs +, pkgs ? import nixpkgs {} +, cargo ? import ./Cargo.nix { + inherit nixpkgs pkgs; release = false; + defaultCrateOverrides = pkgs.defaultCrateOverrides // { + prost-build = attrs: { + buildInputs = [ pkgs.protobuf ]; + }; + tonic-reflection = attrs: { + buildInputs = [ pkgs.rustfmt ]; + }; + }; + } +, meta ? pkgs.lib.importJSON ./nix/meta.json +, dockerName ? "docker.stackable.tech/sandbox/${meta.operator.name}" +, dockerTag ? null +}: +rec { + build = cargo.allWorkspaceMembers; + entrypoint = build+"/bin/stackable-${meta.operator.name}"; + crds = pkgs.runCommand "${meta.operator.name}-crds.yaml" {} + '' + ${entrypoint} crd > $out + ''; + + dockerImage = pkgs.dockerTools.streamLayeredImage { + name = dockerName; + tag = dockerTag; + contents = [ pkgs.bashInteractive pkgs.coreutils pkgs.util-linuxMinimal ]; + config = { + Env = + let + fileRefVars = { + PRODUCT_CONFIG = deploy/config-spec/properties.yaml; + }; + in pkgs.lib.concatLists (pkgs.lib.mapAttrsToList (env: path: pkgs.lib.optional (pkgs.lib.pathExists path) "${env}=${path}") fileRefVars); + Entrypoint = [ entrypoint ]; + Cmd = [ "run" ]; + }; + }; + docker = pkgs.linkFarm "listener-operator-docker" [ + { + name = "load-image"; + path = dockerImage; + } + { + name = "ref"; + path = pkgs.writeText "${dockerImage.name}-image-tag" "${dockerImage.imageName}:${dockerImage.imageTag}"; + } + { + name = "image-repo"; + path = pkgs.writeText "${dockerImage.name}-repo" dockerImage.imageName; + } + { + name = "image-tag"; + path = pkgs.writeText "${dockerImage.name}-tag" dockerImage.imageTag; + } + { + name = "crds.yaml"; + path = crds; + } + ]; + + # need to use vendored crate2nix because of https://github.com/kolloch/crate2nix/issues/264 + crate2nix = import sources.crate2nix {}; + tilt = pkgs.tilt; +} diff --git a/nix/meta.json b/nix/meta.json new file mode 100644 index 00000000..54787a2b --- /dev/null +++ b/nix/meta.json @@ -0,0 +1 @@ +{"operator": {"name": "spark-k8s-operator", "pretty_string": "Apache Spark-on-Kubernetes", "product_string": "spark-k8s", "url": "stackabletech/spark-k8s-operator.git"}} diff --git a/nix/sources.json b/nix/sources.json new file mode 100644 index 00000000..6dc1f0e6 --- /dev/null +++ b/nix/sources.json @@ -0,0 +1,26 @@ +{ + "crate2nix": { + "branch": "master", + "description": "nix build file generator for rust crates", + "homepage": "", + "owner": "kolloch", + "repo": "crate2nix", + "rev": "45fc83132c8c91c77a1cd61fe0c945411d1edba8", + "sha256": "00w9qza56xh9m98308wzfbjf135gpz6sf7b5yfharvmax1k46c5q", + "type": "tarball", + "url": "https://github.com/kolloch/crate2nix/archive/45fc83132c8c91c77a1cd61fe0c945411d1edba8.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "nixpkgs": { + "branch": "nixpkgs-unstable", + "description": "Nix Packages collection", + "homepage": "", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "747927516efcb5e31ba03b7ff32f61f6d47e7d87", + "sha256": "1s4xabv59r99z8vd74w3r84kkxwqggqir3b0nh3ma04mni0m40gf", + "type": "tarball", + "url": "https://github.com/NixOS/nixpkgs/archive/747927516efcb5e31ba03b7ff32f61f6d47e7d87.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + } +} diff --git a/nix/sources.nix b/nix/sources.nix new file mode 100644 index 00000000..9a01c8ac --- /dev/null +++ b/nix/sources.nix @@ -0,0 +1,194 @@ +# This file has been generated by Niv. + +let + + # + # The fetchers. fetch_ fetches specs of type . + # + + fetch_file = pkgs: name: spec: + let + name' = sanitizeName name + "-src"; + in + if spec.builtin or true then + builtins_fetchurl { inherit (spec) url sha256; name = name'; } + else + pkgs.fetchurl { inherit (spec) url sha256; name = name'; }; + + fetch_tarball = pkgs: name: spec: + let + name' = sanitizeName name + "-src"; + in + if spec.builtin or true then + builtins_fetchTarball { name = name'; inherit (spec) url sha256; } + else + pkgs.fetchzip { name = name'; inherit (spec) url sha256; }; + + fetch_git = name: spec: + let + ref = + if spec ? ref then spec.ref else + if spec ? branch then "refs/heads/${spec.branch}" else + if spec ? tag then "refs/tags/${spec.tag}" else + abort "In git source '${name}': Please specify `ref`, `tag` or `branch`!"; + submodules = if spec ? submodules then spec.submodules else false; + submoduleArg = + let + nixSupportsSubmodules = builtins.compareVersions builtins.nixVersion "2.4" >= 0; + emptyArgWithWarning = + if submodules == true + then + builtins.trace + ( + "The niv input \"${name}\" uses submodules " + + "but your nix's (${builtins.nixVersion}) builtins.fetchGit " + + "does not support them" + ) + {} + else {}; + in + if nixSupportsSubmodules + then { inherit submodules; } + else emptyArgWithWarning; + in + builtins.fetchGit + ({ url = spec.repo; inherit (spec) rev; inherit ref; } // submoduleArg); + + fetch_local = spec: spec.path; + + fetch_builtin-tarball = name: throw + ''[${name}] The niv type "builtin-tarball" is deprecated. You should instead use `builtin = true`. + $ niv modify ${name} -a type=tarball -a builtin=true''; + + fetch_builtin-url = name: throw + ''[${name}] The niv type "builtin-url" will soon be deprecated. You should instead use `builtin = true`. + $ niv modify ${name} -a type=file -a builtin=true''; + + # + # Various helpers + # + + # https://github.com/NixOS/nixpkgs/pull/83241/files#diff-c6f540a4f3bfa4b0e8b6bafd4cd54e8bR695 + sanitizeName = name: + ( + concatMapStrings (s: if builtins.isList s then "-" else s) + ( + builtins.split "[^[:alnum:]+._?=-]+" + ((x: builtins.elemAt (builtins.match "\\.*(.*)" x) 0) name) + ) + ); + + # The set of packages used when specs are fetched using non-builtins. + mkPkgs = sources: system: + let + sourcesNixpkgs = + import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) { inherit system; }; + hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath; + hasThisAsNixpkgsPath = == ./.; + in + if builtins.hasAttr "nixpkgs" sources + then sourcesNixpkgs + else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then + import {} + else + abort + '' + Please specify either (through -I or NIX_PATH=nixpkgs=...) or + add a package called "nixpkgs" to your sources.json. + ''; + + # The actual fetching function. + fetch = pkgs: name: spec: + + if ! builtins.hasAttr "type" spec then + abort "ERROR: niv spec ${name} does not have a 'type' attribute" + else if spec.type == "file" then fetch_file pkgs name spec + else if spec.type == "tarball" then fetch_tarball pkgs name spec + else if spec.type == "git" then fetch_git name spec + else if spec.type == "local" then fetch_local spec + else if spec.type == "builtin-tarball" then fetch_builtin-tarball name + else if spec.type == "builtin-url" then fetch_builtin-url name + else + abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}"; + + # If the environment variable NIV_OVERRIDE_${name} is set, then use + # the path directly as opposed to the fetched source. + replace = name: drv: + let + saneName = stringAsChars (c: if isNull (builtins.match "[a-zA-Z0-9]" c) then "_" else c) name; + ersatz = builtins.getEnv "NIV_OVERRIDE_${saneName}"; + in + if ersatz == "" then drv else + # this turns the string into an actual Nix path (for both absolute and + # relative paths) + if builtins.substring 0 1 ersatz == "/" then /. + ersatz else /. + builtins.getEnv "PWD" + "/${ersatz}"; + + # Ports of functions for older nix versions + + # a Nix version of mapAttrs if the built-in doesn't exist + mapAttrs = builtins.mapAttrs or ( + f: set: with builtins; + listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set)) + ); + + # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/lists.nix#L295 + range = first: last: if first > last then [] else builtins.genList (n: first + n) (last - first + 1); + + # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L257 + stringToCharacters = s: map (p: builtins.substring p 1 s) (range 0 (builtins.stringLength s - 1)); + + # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L269 + stringAsChars = f: s: concatStrings (map f (stringToCharacters s)); + concatMapStrings = f: list: concatStrings (map f list); + concatStrings = builtins.concatStringsSep ""; + + # https://github.com/NixOS/nixpkgs/blob/8a9f58a375c401b96da862d969f66429def1d118/lib/attrsets.nix#L331 + optionalAttrs = cond: as: if cond then as else {}; + + # fetchTarball version that is compatible between all the versions of Nix + builtins_fetchTarball = { url, name ? null, sha256 }@attrs: + let + inherit (builtins) lessThan nixVersion fetchTarball; + in + if lessThan nixVersion "1.12" then + fetchTarball ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; })) + else + fetchTarball attrs; + + # fetchurl version that is compatible between all the versions of Nix + builtins_fetchurl = { url, name ? null, sha256 }@attrs: + let + inherit (builtins) lessThan nixVersion fetchurl; + in + if lessThan nixVersion "1.12" then + fetchurl ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; })) + else + fetchurl attrs; + + # Create the final "sources" from the config + mkSources = config: + mapAttrs ( + name: spec: + if builtins.hasAttr "outPath" spec + then abort + "The values in sources.json should not have an 'outPath' attribute" + else + spec // { outPath = replace name (fetch config.pkgs name spec); } + ) config.sources; + + # The "config" used by the fetchers + mkConfig = + { sourcesFile ? if builtins.pathExists ./sources.json then ./sources.json else null + , sources ? if isNull sourcesFile then {} else builtins.fromJSON (builtins.readFile sourcesFile) + , system ? builtins.currentSystem + , pkgs ? mkPkgs sources system + }: rec { + # The sources, i.e. the attribute set of spec name to spec + inherit sources; + + # The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers + inherit pkgs; + }; + +in +mkSources (mkConfig {}) // { __functor = _: settings: mkSources (mkConfig settings); }