Skip to content

Commit f22b3d7

Browse files
ramnivasramnivas
authored
Simplify migration and make it extensible (#1444)
Our migration tests have used strings in Rust files (for exo and expected SQL), which required string escaping, prefix stripping, and other modifications. This made it difficult to read and extend the test code (for example, consider `MigrationScope`s). Also, we relied on exact string matches, which required paying too much attention to the precise format. This refactoring: 1. Move .exo and .sql files outside of rust code 2. Use `sqlparser` to match without considering the format Note that in some cases, `sqlparser` doesn't parse valid syntax, so we rely on exact string comparison as a fallback/optimization. We have already contributed a couple of changes (apache/datafusion-sqlparser-rs#1608, apache/datafusion-sqlparser-rs#1610) and will continue to do so to remove this reliance.
1 parent dd447c5 commit f22b3d7

File tree

156 files changed

+1679
-1860
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

156 files changed

+1679
-1860
lines changed

‎Cargo.lock

Lines changed: 11 additions & 7 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎crates/postgres-subsystem/postgres-core-model/Cargo.toml

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,17 @@ exo-sql = { path = "../../../libs/exo-sql" }
1313
core-plugin-interface = { path = "../../core-subsystem/core-plugin-interface" }
1414

1515
[dev-dependencies]
16-
stripmargin = "0.1.1"
1716
tokio.workspace = true
1817
wasm-bindgen-test.workspace = true
1918
bincode.workspace = true
2019
exo-sql = { path = "../../../libs/exo-sql", features = ["bigdecimal"] }
20+
sqlparser = { git = "https://github.com/exograph/datafusion-sqlparser-rs.git", branch = "optional-columns-foreign-reference" }
21+
colored.workspace = true
22+
wildmatch.workspace = true
2123

2224
builder = { path = "../../builder" }
2325
postgres-builder = { path = "../postgres-builder" }
2426

2527
[lib]
2628
doctest = false
29+

‎crates/postgres-subsystem/postgres-core-model/src/migration-test-data/.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
*.actual.sql

‎crates/postgres-subsystem/postgres-core-model/src/migration-test-data/add-field/new/src/index.exo

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
@postgres
2+
module ConcertModule {
3+
type Concert {
4+
@pk id: Int = autoIncrement()
5+
title: String
6+
published: Boolean
7+
}
8+
}

‎crates/postgres-subsystem/postgres-core-model/src/migration-test-data/add-field/old/src/index.exo

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
@postgres
2+
module ConcertModule {
3+
type Concert {
4+
@pk id: Int = autoIncrement()
5+
title: String
6+
}
7+
}

‎crates/postgres-subsystem/postgres-core-model/src/migration-test-data/add-field/scope-all-schemas/down.sql

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
-- ALTER TABLE "concerts" DROP COLUMN "published";

‎crates/postgres-subsystem/postgres-core-model/src/migration-test-data/add-field/scope-all-schemas/new.sql

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
CREATE TABLE "concerts" (
2+
"id" SERIAL PRIMARY KEY,
3+
"title" TEXT NOT NULL,
4+
"published" BOOLEAN NOT NULL
5+
);

‎crates/postgres-subsystem/postgres-core-model/src/migration-test-data/add-field/scope-all-schemas/old.sql

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
CREATE TABLE "concerts" (
2+
"id" SERIAL PRIMARY KEY,
3+
"title" TEXT NOT NULL
4+
);

‎crates/postgres-subsystem/postgres-core-model/src/migration-test-data/add-field/scope-all-schemas/up.sql

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
ALTER TABLE "concerts" ADD "published" BOOLEAN NOT NULL;

‎crates/postgres-subsystem/postgres-core-model/src/migration-test-data/add-indices-non-public-schemas/new/src/index.exo

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
@postgres
2+
module ConcertModule {
3+
@table(schema="c")
4+
type Concert {
5+
@pk id: Int = autoIncrement()
6+
@index title: String
7+
@index venue: Venue
8+
}
9+
10+
@table(schema="v")
11+
type Venue {
12+
@pk id: Int = autoIncrement()
13+
@index name: String
14+
concerts: Set<Concert>?
15+
}
16+
}

‎crates/postgres-subsystem/postgres-core-model/src/migration-test-data/add-indices-non-public-schemas/old/src/index.exo

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
@postgres
2+
module ConcertModule {
3+
type Concert {
4+
@pk id: Int = autoIncrement()
5+
title: String
6+
venue: Venue
7+
}
8+
9+
type Venue {
10+
@pk id: Int = autoIncrement()
11+
name: String
12+
concerts: Set<Concert>?
13+
}
14+
}

‎crates/postgres-subsystem/postgres-core-model/src/migration-test-data/add-indices-non-public-schemas/scope-all-schemas/down.sql

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
-- DROP TABLE "c"."concerts" CASCADE;
2+
3+
-- DROP TABLE "v"."venues" CASCADE;
4+
5+
CREATE TABLE "concerts" (
6+
"id" SERIAL PRIMARY KEY,
7+
"title" TEXT NOT NULL,
8+
"venue_id" INT NOT NULL
9+
);
10+
11+
CREATE TABLE "venues" (
12+
"id" SERIAL PRIMARY KEY,
13+
"name" TEXT NOT NULL
14+
);
15+
16+
-- DROP SCHEMA "c" CASCADE;
17+
18+
-- DROP SCHEMA "v" CASCADE;
19+
20+
ALTER TABLE "concerts" ADD CONSTRAINT "concerts_venue_id_fk" FOREIGN KEY ("venue_id") REFERENCES "venues";
21+

‎crates/postgres-subsystem/postgres-core-model/src/migration-test-data/add-indices-non-public-schemas/scope-all-schemas/new.sql

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
CREATE SCHEMA "c";
2+
3+
CREATE SCHEMA "v";
4+
5+
CREATE TABLE "c"."concerts" (
6+
"id" SERIAL PRIMARY KEY,
7+
"title" TEXT NOT NULL,
8+
"venue_id" INT NOT NULL
9+
);
10+
11+
CREATE TABLE "v"."venues" (
12+
"id" SERIAL PRIMARY KEY,
13+
"name" TEXT NOT NULL
14+
);
15+
16+
ALTER TABLE "c"."concerts" ADD CONSTRAINT "c_concerts_venue_id_fk" FOREIGN KEY ("venue_id") REFERENCES "v"."venues";
17+
18+
CREATE INDEX "concert_title_idx" ON "c"."concerts" ("title");
19+
20+
CREATE INDEX "concert_venue_idx" ON "c"."concerts" ("venue_id");
21+
22+
CREATE INDEX "venue_name_idx" ON "v"."venues" ("name");
23+

‎crates/postgres-subsystem/postgres-core-model/src/migration-test-data/add-indices-non-public-schemas/scope-all-schemas/old.sql

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
CREATE TABLE "concerts" (
2+
"id" SERIAL PRIMARY KEY,
3+
"title" TEXT NOT NULL,
4+
"venue_id" INT NOT NULL
5+
);
6+
7+
CREATE TABLE "venues" (
8+
"id" SERIAL PRIMARY KEY,
9+
"name" TEXT NOT NULL
10+
);
11+
12+
ALTER TABLE "concerts" ADD CONSTRAINT "concerts_venue_id_fk" FOREIGN KEY ("venue_id") REFERENCES "venues";
13+

‎crates/postgres-subsystem/postgres-core-model/src/migration-test-data/add-indices-non-public-schemas/scope-all-schemas/up.sql

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
CREATE SCHEMA "c";
2+
3+
CREATE SCHEMA "v";
4+
5+
-- DROP TABLE "concerts" CASCADE;
6+
7+
-- DROP TABLE "venues" CASCADE;
8+
9+
CREATE TABLE "c"."concerts" (
10+
"id" SERIAL PRIMARY KEY,
11+
"title" TEXT NOT NULL,
12+
"venue_id" INT NOT NULL
13+
);
14+
15+
CREATE TABLE "v"."venues" (
16+
"id" SERIAL PRIMARY KEY,
17+
"name" TEXT NOT NULL
18+
);
19+
20+
ALTER TABLE "c"."concerts" ADD CONSTRAINT "c_concerts_venue_id_fk" FOREIGN KEY ("venue_id") REFERENCES "v"."venues";
21+
22+
CREATE INDEX "concert_title_idx" ON "c"."concerts" ("title");
23+
24+
CREATE INDEX "concert_venue_idx" ON "c"."concerts" ("venue_id");
25+
26+
CREATE INDEX "venue_name_idx" ON "v"."venues" ("name");
27+

‎crates/postgres-subsystem/postgres-core-model/src/migration-test-data/add-indices/new/src/index.exo

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
@postgres
2+
module ConcertModule {
3+
type Concert {
4+
@pk id: Int = autoIncrement()
5+
@index title: String
6+
@index venue: Venue
7+
}
8+
type Venue {
9+
@pk id: Int = autoIncrement()
10+
@index name: String
11+
concerts: Set<Concert>?
12+
}
13+
}

‎crates/postgres-subsystem/postgres-core-model/src/migration-test-data/add-indices/old/src/index.exo

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
@postgres
2+
module ConcertModule {
3+
type Concert {
4+
@pk id: Int = autoIncrement()
5+
title: String
6+
venue: Venue
7+
}
8+
type Venue {
9+
@pk id: Int = autoIncrement()
10+
name: String
11+
concerts: Set<Concert>?
12+
}
13+
}

‎crates/postgres-subsystem/postgres-core-model/src/migration-test-data/add-indices/scope-all-schemas/down.sql

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
DROP INDEX "concert_title_idx";
2+
3+
DROP INDEX "concert_venue_idx";
4+
5+
DROP INDEX "venue_name_idx";
6+

‎crates/postgres-subsystem/postgres-core-model/src/migration-test-data/add-indices/scope-all-schemas/new.sql

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
CREATE TABLE "concerts" (
2+
"id" SERIAL PRIMARY KEY,
3+
"title" TEXT NOT NULL,
4+
"venue_id" INT NOT NULL
5+
);
6+
7+
CREATE TABLE "venues" (
8+
"id" SERIAL PRIMARY KEY,
9+
"name" TEXT NOT NULL
10+
);
11+
12+
ALTER TABLE "concerts" ADD CONSTRAINT "concerts_venue_id_fk" FOREIGN KEY ("venue_id") REFERENCES "venues";
13+
14+
CREATE INDEX "concert_title_idx" ON "concerts" ("title");
15+
16+
CREATE INDEX "concert_venue_idx" ON "concerts" ("venue_id");
17+
18+
CREATE INDEX "venue_name_idx" ON "venues" ("name");
19+

‎crates/postgres-subsystem/postgres-core-model/src/migration-test-data/add-indices/scope-all-schemas/old.sql

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
CREATE TABLE "concerts" (
2+
"id" SERIAL PRIMARY KEY,
3+
"title" TEXT NOT NULL,
4+
"venue_id" INT NOT NULL
5+
);
6+
7+
CREATE TABLE "venues" (
8+
"id" SERIAL PRIMARY KEY,
9+
"name" TEXT NOT NULL
10+
);
11+
12+
ALTER TABLE "concerts" ADD CONSTRAINT "concerts_venue_id_fk" FOREIGN KEY ("venue_id") REFERENCES "venues";
13+

‎crates/postgres-subsystem/postgres-core-model/src/migration-test-data/add-indices/scope-all-schemas/up.sql

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
CREATE INDEX "concert_title_idx" ON "concerts" ("title");
2+
3+
CREATE INDEX "concert_venue_idx" ON "concerts" ("venue_id");
4+
5+
CREATE INDEX "venue_name_idx" ON "venues" ("name");
6+

‎crates/postgres-subsystem/postgres-core-model/src/migration-test-data/add-model/new/src/index.exo

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
@postgres
2+
module ConcertModule {
3+
type Concert {
4+
@pk id: Int = autoIncrement()
5+
title: String
6+
published: Boolean
7+
}
8+
}

‎crates/postgres-subsystem/postgres-core-model/src/migration-test-data/add-model/old/src/index.exo

Whitespace-only changes.

‎crates/postgres-subsystem/postgres-core-model/src/migration-test-data/add-model/scope-all-schemas/down.sql

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
-- DROP TABLE "concerts" CASCADE;

‎crates/postgres-subsystem/postgres-core-model/src/migration-test-data/add-model/scope-all-schemas/new.sql

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
CREATE TABLE "concerts" (
2+
"id" SERIAL PRIMARY KEY,
3+
"title" TEXT NOT NULL,
4+
"published" BOOLEAN NOT NULL
5+
);

‎crates/postgres-subsystem/postgres-core-model/src/migration-test-data/add-model/scope-all-schemas/old.sql

Whitespace-only changes.

‎crates/postgres-subsystem/postgres-core-model/src/migration-test-data/add-model/scope-all-schemas/up.sql

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
CREATE TABLE "concerts" (
2+
"id" SERIAL PRIMARY KEY,
3+
"title" TEXT NOT NULL,
4+
"published" BOOLEAN NOT NULL
5+
);

‎crates/postgres-subsystem/postgres-core-model/src/migration-test-data/add-relation-and-related-model/new/src/index.exo

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
2+
@postgres
3+
module ConcertModule {
4+
type Concert {
5+
@pk id: Int = autoIncrement()
6+
title: String
7+
venue: Venue
8+
}
9+
type Venue {
10+
@pk id: Int = autoIncrement()
11+
name: String
12+
concerts: Set<Concert>?
13+
}
14+
}

‎crates/postgres-subsystem/postgres-core-model/src/migration-test-data/add-relation-and-related-model/old/src/index.exo

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
@postgres
2+
module ConcertModule {
3+
type Concert {
4+
@pk id: Int = autoIncrement()
5+
title: String
6+
}
7+
}

‎crates/postgres-subsystem/postgres-core-model/src/migration-test-data/add-relation-and-related-model/scope-all-schemas/down.sql

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
-- ALTER TABLE "concerts" DROP COLUMN "venue_id";
2+
3+
-- DROP TABLE "venues" CASCADE;
4+

‎crates/postgres-subsystem/postgres-core-model/src/migration-test-data/add-relation-and-related-model/scope-all-schemas/new.sql

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
CREATE TABLE "concerts" (
2+
"id" SERIAL PRIMARY KEY,
3+
"title" TEXT NOT NULL,
4+
"venue_id" INT NOT NULL
5+
);
6+
7+
CREATE TABLE "venues" (
8+
"id" SERIAL PRIMARY KEY,
9+
"name" TEXT NOT NULL
10+
);
11+
12+
ALTER TABLE "concerts" ADD CONSTRAINT "concerts_venue_id_fk" FOREIGN KEY ("venue_id") REFERENCES "venues";
13+

‎crates/postgres-subsystem/postgres-core-model/src/migration-test-data/add-relation-and-related-model/scope-all-schemas/old.sql

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
CREATE TABLE "concerts" (
2+
"id" SERIAL PRIMARY KEY,
3+
"title" TEXT NOT NULL
4+
);
5+

‎crates/postgres-subsystem/postgres-core-model/src/migration-test-data/add-relation-and-related-model/scope-all-schemas/up.sql

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
ALTER TABLE "concerts" ADD "venue_id" INT NOT NULL;
2+
3+
CREATE TABLE "venues" (
4+
"id" SERIAL PRIMARY KEY,
5+
"name" TEXT NOT NULL
6+
);
7+
8+
ALTER TABLE "concerts" ADD CONSTRAINT "concerts_venue_id_fk" FOREIGN KEY ("venue_id") REFERENCES "venues";
9+

0 commit comments

Comments
 (0)