Skip to content

[Merged by Bors] - Merging and validation of the configuration refactored #223

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 13 commits into from
Closed

[Merged by Bors] - Merging and validation of the configuration refactored #223

Show file tree
Hide file tree
Changes from 10 commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
631aa0e
Allow fragments for job, driver, and executor configurations
siegfriedweber Mar 10, 2023
13dff68
Reorganize imports
siegfriedweber Mar 30, 2023
fed4572
Fix bug in configuration merge
siegfriedweber Mar 30, 2023
21bef7a
Make volume mounts and node selector optional
siegfriedweber Mar 30, 2023
512146a
Remove unused error variants
siegfriedweber Mar 30, 2023
5413cf6
Upgrade stackable-operator to version 0.38.0
siegfriedweber Mar 30, 2023
6abfa4a
Upgrade clap
siegfriedweber Mar 30, 2023
a03f344
Update changelog
siegfriedweber Mar 30, 2023
b5ccdb7
Use volume mounts from the merged configuration
siegfriedweber Mar 30, 2023
661fc61
Fix Clippy warning
siegfriedweber Mar 30, 2023
6a170f2
Use separate indexes for test steps
siegfriedweber Mar 31, 2023
aaa3cf0
Wait 10 seconds in all tests steps for MinIO to start up
siegfriedweber Mar 31, 2023
68cfb03
Use distinct names for SecretClasses in the tests
siegfriedweber Mar 31, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,12 @@ All notable changes to this project will be documented in this file.

- Revert openshift settings ([#207])
- BUGFIX: assign service account to history pods ([#207])
- Merging and validation of the configuration refactored ([#223])
- `operator-rs` `0.36.0` → `0.38.0` ([#223])

[#207]: https://github.com/stackabletech/spark-k8s-operator/pull/207
[#217]: https://github.com/stackabletech/spark-k8s-operator/pull/217
[#223]: https://github.com/stackabletech/spark-k8s-operator/pull/223

## [23.1.0] - 2023-01-23

Expand Down
8 changes: 4 additions & 4 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

55 changes: 42 additions & 13 deletions deploy/helm/spark-k8s-operator/crds/crds.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,9 +87,13 @@ spec:
nullable: true
properties:
affinity:
nullable: true
default:
podAffinity: null
podAntiAffinity: null
nodeAffinity: null
nodeSelector: null
properties:
node_affinity:
nodeAffinity:
description: Node affinity is a group of node affinity scheduling rules.
nullable: true
properties:
Expand Down Expand Up @@ -217,10 +221,10 @@ spec:
- nodeSelectorTerms
type: object
type: object
node_selector:
nodeSelector:
nullable: true
type: object
pod_affinity:
podAffinity:
description: Pod affinity is a group of inter pod affinity scheduling rules.
nullable: true
properties:
Expand Down Expand Up @@ -390,7 +394,7 @@ spec:
type: object
type: array
type: object
pod_anti_affinity:
podAntiAffinity:
description: Pod anti affinity is a group of inter pod anti affinity scheduling rules.
nullable: true
properties:
Expand Down Expand Up @@ -562,7 +566,14 @@ spec:
type: object
type: object
resources:
nullable: true
default:
memory:
limit: null
runtimeLimits: {}
cpu:
min: null
max: null
storage: {}
properties:
cpu:
default:
Expand Down Expand Up @@ -704,9 +715,13 @@ spec:
nullable: true
properties:
affinity:
nullable: true
default:
podAffinity: null
podAntiAffinity: null
nodeAffinity: null
nodeSelector: null
properties:
node_affinity:
nodeAffinity:
description: Node affinity is a group of node affinity scheduling rules.
nullable: true
properties:
Expand Down Expand Up @@ -834,10 +849,10 @@ spec:
- nodeSelectorTerms
type: object
type: object
node_selector:
nodeSelector:
nullable: true
type: object
pod_affinity:
podAffinity:
description: Pod affinity is a group of inter pod affinity scheduling rules.
nullable: true
properties:
Expand Down Expand Up @@ -1007,7 +1022,7 @@ spec:
type: object
type: array
type: object
pod_anti_affinity:
podAntiAffinity:
description: Pod anti affinity is a group of inter pod anti affinity scheduling rules.
nullable: true
properties:
Expand Down Expand Up @@ -1189,7 +1204,14 @@ spec:
nullable: true
type: object
resources:
nullable: true
default:
memory:
limit: null
runtimeLimits: {}
cpu:
min: null
max: null
storage: {}
properties:
cpu:
default:
Expand Down Expand Up @@ -1253,7 +1275,14 @@ spec:
nullable: true
properties:
resources:
nullable: true
default:
memory:
limit: null
runtimeLimits: {}
cpu:
min: null
max: null
storage: {}
properties:
cpu:
default:
Expand Down
2 changes: 1 addition & 1 deletion rust/crd/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ version = "0.0.0-dev"
publish = false

[dependencies]
stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag="0.36.0" }
stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag="0.38.0" }

semver = "1.0"
serde = "1.0"
Expand Down
48 changes: 24 additions & 24 deletions rust/crd/src/history.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,33 +1,33 @@
use crate::affinity::history_affinity;
use crate::constants::*;
use stackable_operator::commons::affinity::StackableAffinity;
use stackable_operator::commons::product_image_selection::{ProductImage, ResolvedProductImage};
use stackable_operator::commons::resources::{
CpuLimitsFragment, MemoryLimitsFragment, NoRuntimeLimitsFragment,
};
use stackable_operator::commons::s3::S3BucketDef;
use stackable_operator::config::fragment::ValidationError;
use stackable_operator::k8s_openapi::apimachinery::pkg::api::resource::Quantity;
use stackable_operator::kube::runtime::reflector::ObjectRef;
use stackable_operator::kube::ResourceExt;
use stackable_operator::product_config::types::PropertyNameKind;
use stackable_operator::product_config::ProductConfigManager;
use stackable_operator::product_config_utils::{
transform_all_roles_to_config, validate_all_roles_and_groups_config, Configuration,
ValidatedRoleConfigByPropertyKind,
};
use stackable_operator::role_utils::{Role, RoleGroupRef};
use crate::{affinity::history_affinity, constants::*};

use std::collections::{BTreeMap, HashMap};

use serde::{Deserialize, Serialize};
use snafu::{ResultExt, Snafu};
use stackable_operator::{
commons::resources::{NoRuntimeLimits, Resources, ResourcesFragment},
config::{fragment, fragment::Fragment, merge::Merge},
};
use stackable_operator::{
commons::{
affinity::StackableAffinity,
product_image_selection::{ProductImage, ResolvedProductImage},
resources::{
CpuLimitsFragment, MemoryLimitsFragment, NoRuntimeLimits, NoRuntimeLimitsFragment,
Resources, ResourcesFragment,
},
s3::S3BucketDef,
},
config::{
fragment,
fragment::{Fragment, ValidationError},
merge::Merge,
},
k8s_openapi::apimachinery::pkg::api::resource::Quantity,
kube::CustomResource,
kube::{runtime::reflector::ObjectRef, ResourceExt},
product_config::{types::PropertyNameKind, ProductConfigManager},
product_config_utils::{
transform_all_roles_to_config, validate_all_roles_and_groups_config, Configuration,
ValidatedRoleConfigByPropertyKind,
},
role_utils::{Role, RoleGroupRef},
schemars::{self, JsonSchema},
};
use strum::Display;
Expand Down Expand Up @@ -91,7 +91,7 @@ impl SparkHistoryServer {
conf_role.merge(&conf_defaults);
conf_rolegroup.merge(&conf_role);

fragment::validate(conf_defaults).context(FragmentValidationFailureSnafu)
fragment::validate(conf_rolegroup).context(FragmentValidationFailureSnafu)
}

pub fn replicas(&self, rolegroup_ref: &RoleGroupRef<Self>) -> Option<i32> {
Expand Down
Loading