ny-tlc-report-image.yaml

---
apiVersion: spark.stackable.tech/v1alpha1
kind: SparkApplication
metadata:
  name: spark-ny-image
  namespace: default
spec:
  version: "1.0"
  # everything under /jobs will be copied to /stackable/spark/jobs
  image: docker.stackable.tech/stackable/ny-tlc-report:0.1.0
  sparkImage: docker.stackable.tech/stackable/spark-k8s:3.5.0-stackable0.0.0-dev
  sparkImagePullPolicy: IfNotPresent
  mode: cluster
  mainApplicationFile: local:///stackable/spark/jobs/ny_tlc_report.py
  args:
    - "--input 's3a://my-bucket/yellow_tripdata_2021-07.csv'"
  deps:
    requirements:
      - tabulate==0.8.9
  s3connection:
    inline:
      host: test-minio
      port: 9000
      accessStyle: Path
  sparkConf:
    spark.hadoop.fs.s3a.aws.credentials.provider: "org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider"
  executor:
    replicas: 3