refactor(glue-alpha): Refactored glue-alpha L2 CDK construct RFC 0497 (#32521)

### Issue # (if applicable)

Implementation of [RFC
0497](https://github.com/aws/aws-cdk-rfcs/blob/main/text/0497-glue-l2-construct.md)

### Reason for this change

Refactored glue-alpha construct to enforce validations by contract and
interfaces, improve developer experience, and adhere to best practices.
[Related PR with merge conflicts and
history](mjanardhan#12)

### Description of changes

Refactored from a single Job class to a pattern of inheritance that
removes the need for synth-time validations and sets best practice
defaults. Allows for overriding language and Glue versions where
applicable, and other job-type specific parameters.

The existing Job and Job Executable monoliths have been decomposed into
Job Type and Language specific classes that implement and extend an
abstract Job parent class. Developers will be able to see mandatory and
optional parameters that apply just to their selected job type and
language, rather than having to reference documentation and examples or
find out during synth or deploy time that they've selected the wrong
configuration.

BREAKING CHANGE: Developers must refactor their existing Job
instantiation method calls to choose the right job type and language,
and use the new constants static values to define the associated Job
configuration settings. See the RFC and/or new README for examples.

### Description of how you validated changes

Increased unit test coverage to > 90%, consulted with Glue service team
on best practices and sane defaults, updated integration tests.

### Checklist
- [X] My code adheres to the [CONTRIBUTING
GUIDE](https://github.com/aws/aws-cdk/blob/main/CONTRIBUTING.md) and
[DESIGN
GUIDELINES](https://github.com/aws/aws-cdk/blob/main/docs/DESIGN_GUIDELINES.md)

----

*By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache-2.0 license*

---------

Co-authored-by: Janardhan (Janny) Molumuri <[email protected]>
Co-authored-by: GZ <[email protected]>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

Author: 4 people

packages/@aws-cdk/aws-glue-alpha/README.md

@@ -51,15 +51,15 @@
     "docs-public-apis:@aws-cdk/aws-glue-alpha.ITable",
     "docs-public-apis:@aws-cdk/aws-glue-alpha.ITable.tableArn",
     "docs-public-apis:@aws-cdk/aws-glue-alpha.ITable.tableName",
-    "props-default-doc:@aws-cdk/aws-glue-alpha.PythonRayExecutableProps.runtime",
-    "props-default-doc:@aws-cdk/aws-glue-alpha.PythonShellExecutableProps.runtime",
-    "props-default-doc:@aws-cdk/aws-glue-alpha.PythonSparkJobExecutableProps.runtime",
     "docs-public-apis:@aws-cdk/aws-glue-alpha.S3TableProps",
-    "props-default-doc:@aws-cdk/aws-glue-alpha.ScalaJobExecutableProps.runtime",
     "docs-public-apis:@aws-cdk/aws-glue-alpha.TableAttributes",
     "docs-public-apis:@aws-cdk/aws-glue-alpha.TableAttributes.tableArn",
     "docs-public-apis:@aws-cdk/aws-glue-alpha.TableAttributes.tableName",
     "docs-public-apis:@aws-cdk/aws-glue-alpha.TableBaseProps",
-    "docs-public-apis:@aws-cdk/aws-glue-alpha.TableProps"
+    "docs-public-apis:@aws-cdk/aws-glue-alpha.TableProps",
+    "docs-public-apis:@aws-cdk/aws-glue-alpha.PredicateLogical",
+    "no-unused-type:@aws-cdk/aws-glue-alpha.ExecutionClass",
+    "no-unused-type:@aws-cdk/aws-glue-alpha.JobLanguage",
+    "no-unused-type:@aws-cdk/aws-glue-alpha.JobType"
   ]
 }

packages/@aws-cdk/aws-glue-alpha/awslint.json

@@ -0,0 +1,313 @@
+/**
+ * The type of predefined worker that is allocated when a job runs.
+ *
+ * If you need to use a WorkerType that doesn't exist as a static member, you
+ * can instantiate a `WorkerType` object, e.g: `WorkerType.of('other type')`
+ */
+export enum WorkerType {
+  /**
+   * Standard Worker Type
+   * 4 vCPU, 16 GB of memory and a 50GB disk, and 2 executors per worker.
+  */
+  STANDARD = 'Standard',
+
+  /**
+   * G.1X Worker Type
+   * 1 DPU (4 vCPU, 16 GB of memory, 64 GB disk), and provides 1 executor per worker. Suitable for memory-intensive jobs.
+   */
+  G_1X = 'G.1X',
+
+  /**
+   * G.2X Worker Type
+   * 2 DPU (8 vCPU, 32 GB of memory, 128 GB disk), and provides 1 executor per worker. Suitable for memory-intensive jobs.
+   */
+  G_2X = 'G.2X',
+
+  /**
+   * G.4X Worker Type
+   * 4 DPU (16 vCPU, 64 GB of memory, 256 GB disk), and provides 1 executor per worker.
+   * We recommend this worker type for jobs whose workloads contain your most demanding transforms,
+   * aggregations, joins, and queries. This worker type is available only for AWS Glue version 3.0 or later jobs.
+  */
+  G_4X = 'G.4X',
+
+  /**
+   * G.8X Worker Type
+   * 8 DPU (32 vCPU, 128 GB of memory, 512 GB disk), and provides 1 executor per worker. We recommend this worker
+   * type for jobs whose workloads contain your most demanding transforms, aggregations, joins, and queries.
+   * This worker type is available only for AWS Glue version 3.0 or later jobs.
+  */
+  G_8X = 'G.8X',
+
+  /**
+   * G.025X Worker Type
+   * 0.25 DPU (2 vCPU, 4 GB of memory, 64 GB disk), and provides 1 executor per worker. Suitable for low volume streaming jobs.
+   */
+  G_025X = 'G.025X',
+
+  /**
+   * Z.2X Worker Type
+   */
+  Z_2X = 'Z.2X',
+}
+
+/**
+ * The number of workers of a defined workerType that are allocated when a job runs.
+ *
+ * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-api-jobs-job.html
+ */
+
+/**
+ * Job states emitted by Glue to CloudWatch Events.
+ *
+ * @see https://docs.aws.amazon.com/AmazonCloudWatch/latest/events/EventTypes.html#glue-event-types for more information.
+ */
+export enum JobState {
+  /**
+   * State indicating job run succeeded
+   */
+  SUCCEEDED = 'SUCCEEDED',
+
+  /**
+   * State indicating job run failed
+   */
+  FAILED = 'FAILED',
+
+  /**
+   * State indicating job run timed out
+   */
+  TIMEOUT = 'TIMEOUT',
+
+  /**
+   * State indicating job is starting
+   */
+  STARTING = 'STARTING',
+
+  /**
+   * State indicating job is running
+   */
+  RUNNING = 'RUNNING',
+
+  /**
+   * State indicating job is stopping
+   */
+  STOPPING = 'STOPPING',
+
+  /**
+   * State indicating job stopped
+   */
+  STOPPED = 'STOPPED',
+}
+
+/**
+ * The Glue CloudWatch metric type.
+ *
+ * @see https://docs.aws.amazon.com/glue/latest/dg/monitoring-awsglue-with-cloudwatch-metrics.html
+ */
+export enum MetricType {
+  /**
+   * A value at a point in time.
+   */
+  GAUGE = 'gauge',
+
+  /**
+   * An aggregate number.
+   */
+  COUNT = 'count',
+}
+
+/**
+ * The ExecutionClass whether the job is run with a standard or flexible execution class.
+ *
+ * @see https://docs.aws.amazon.com/glue/latest/dg/aws-glue-api-jobs-job.html#aws-glue-api-jobs-job-Job
+ * @see https://docs.aws.amazon.com/glue/latest/dg/add-job.html
+ */
+export enum ExecutionClass {
+  /**
+   * The flexible execution class is appropriate for time-insensitive jobs whose start
+   * and completion times may vary.
+   */
+  FLEX = 'FLEX',
+
+  /**
+   * The standard execution class is ideal for time-sensitive workloads that require fast job
+   * startup and dedicated resources.
+   */
+  STANDARD = 'STANDARD',
+}
+
+/**
+ * AWS Glue version determines the versions of Apache Spark and Python that are available to the job.
+ *
+ * @see https://docs.aws.amazon.com/glue/latest/dg/add-job.html.
+ */
+export enum GlueVersion {
+  /**
+   * Glue version using Spark 2.2.1 and Python 2.7
+   */
+  V0_9 = '0.9',
+
+  /**
+   * Glue version using Spark 2.4.3, Python 2.7 and Python 3.6
+   */
+  V1_0 = '1.0',
+
+  /**
+   * Glue version using Spark 2.4.3 and Python 3.7
+   */
+  V2_0 = '2.0',
+
+  /**
+   * Glue version using Spark 3.1.1 and Python 3.7
+   */
+  V3_0 = '3.0',
+
+  /**
+   * Glue version using Spark 3.3.0 and Python 3.10
+   */
+  V4_0 = '4.0',
+
+  /**
+   * Glue version using Spark 3.3.0 and Python 3.10
+   */
+  V5_0 = '5.0',
+
+}
+
+/**
+ * Runtime language of the Glue job
+ */
+export enum JobLanguage {
+  /**
+   * Scala
+   */
+  SCALA = 'scala',
+
+  /**
+   * Python
+   */
+  PYTHON = 'python',
+}
+
+/**
+ * Python version
+ */
+export enum PythonVersion {
+  /**
+   * Python 2 (the exact version depends on GlueVersion and JobCommand used)
+   */
+  TWO = '2',
+
+  /**
+   * Python 3 (the exact version depends on GlueVersion and JobCommand used)
+   */
+  THREE = '3',
+
+  /**
+   * Python 3.9 (the exact version depends on GlueVersion and JobCommand used)
+   */
+  THREE_NINE = '3.9',
+
+}
+
+/**
+ * AWS Glue runtime determines the runtime engine of the job.
+ *
+ */
+export enum Runtime {
+  /**
+   * Runtime for a Glue for Ray 2.4.
+   */
+  RAY_TWO_FOUR = 'Ray2.4',
+}
+
+/**
+ * The job type.
+ *
+ * If you need to use a JobType that doesn't exist as a static member, you
+ * can instantiate a `JobType` object, e.g: `JobType.of('other name')`.
+ */
+export enum JobType {
+  /**
+   * Command for running a Glue Spark job.
+   */
+  ETL = 'glueetl',
+
+  /**
+   * Command for running a Glue Spark streaming job.
+   */
+  STREAMING = 'gluestreaming',
+
+  /**
+   * Command for running a Glue python shell job.
+   */
+  PYTHON_SHELL = 'pythonshell',
+
+  /**
+   * Command for running a Glue Ray job.
+   */
+  RAY = 'glueray',
+
+}
+
+/**
+ * The number of AWS Glue data processing units (DPUs) that can be allocated when this job runs. A DPU is a relative measure of processing power that consists of 4 vCPUs of compute capacity and 16 GB of memory.
+ */
+export enum MaxCapacity {
+
+  /**
+   * DPU value of 1/16th
+   */
+  DPU_1_16TH = 0.0625,
+
+  /**
+   * DPU value of 1
+   */
+  DPU_1 = 1,
+}
+
+/*
+ * Represents the logical operator for combining multiple conditions in the Glue Trigger API.
+ */
+export enum PredicateLogical {
+  /**
+  * All conditions must be true for the predicate to be true.
+  */
+  AND = 'AND',
+
+  /**
+  * At least one condition must be true for the predicate to be true.
+  */
+  ANY = 'ANY',
+}
+
+/**
+ * Represents the logical operator for evaluating a single condition in the Glue Trigger API.
+ */
+export enum ConditionLogicalOperator {
+  /** The condition is true if the values are equal. */
+  EQUALS = 'EQUALS',
+}
+
+/**
+ * Represents the state of a crawler for a condition in the Glue Trigger API.
+ */
+export enum CrawlerState {
+  /** The crawler is currently running. */
+  RUNNING = 'RUNNING',
+
+  /** The crawler is in the process of being cancelled. */
+  CANCELLING = 'CANCELLING',
+
+  /** The crawler has been cancelled. */
+  CANCELLED = 'CANCELLED',
+
+  /** The crawler has completed its operation successfully. */
+  SUCCEEDED = 'SUCCEEDED',
+
+  /** The crawler has failed to complete its operation. */
+  FAILED = 'FAILED',
+
+  /** The crawler encountered an error during its operation. */
+  ERROR = 'ERROR',
+}

packages/@aws-cdk/aws-glue-alpha/lib/constants.ts

@@ -6,11 +6,22 @@ export * from './data-format';
 export * from './data-quality-ruleset';
 export * from './database';
 export * from './external-table';
-export * from './job';
-export * from './job-executable';
 export * from './s3-table';
 export * from './schema';
 export * from './security-configuration';
 export * from './storage-parameter';
+export * from './constants';
+export * from './jobs/job';
+export * from './jobs/pyspark-etl-job';
+export * from './jobs/pyspark-flex-etl-job';
+export * from './jobs/pyspark-streaming-job';
+export * from './jobs/python-shell-job';
+export * from './jobs/ray-job';
+export * from './jobs/scala-spark-etl-job';
+export * from './jobs/scala-spark-flex-etl-job';
+export * from './jobs/scala-spark-streaming-job';
+export * from './jobs/spark-ui-utils';
 export * from './table-base';
 export * from './table-deprecated';
+export * from './triggers/workflow';
+export * from './triggers/trigger-options';