example-sparkapp-image.yaml

---
apiVersion: spark.stackable.tech/v1alpha1
kind: SparkApplication
metadata:
  name: example-sparkapp-image
  namespace: default
spec:
  version: "1.0"
  image: docker.stackable.tech/stackable/ny-tlc-report:0.1.0 # <1>
  sparkImage:
    productVersion: 3.5.0
  mode: cluster
  mainApplicationFile: local:///stackable/spark/jobs/ny_tlc_report.py # <2>
  args:
    - "--input 's3a://nyc-tlc/trip data/yellow_tripdata_2021-07.csv'" # <3>
  deps:
    requirements:
      - tabulate==0.8.9 # <4>
  sparkConf: # <5>
    "spark.hadoop.fs.s3a.aws.credentials.provider": "org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider"
  job:
    config:
      resources:
        cpu:
          min: "1"
          max: "1"
        memory:
          limit: "1Gi"
  driver:
    config:
      resources:
        cpu:
          min: "1"
          max: "1500m"
        memory:
          limit: "1Gi"
  executor:
    replicas: 3
    config:
      resources:
        cpu:
          min: "1"
          max: "4"
        memory:
          limit: "2Gi"