tests: create table from source: dbt

test/canary-environment/macros/create_loadgen_source.sql

@@ -8,9 +8,5 @@
 -- by the Apache License, Version 2.0.
 
 {% macro create_loadgen_source(name) %}
-FROM KAFKA CONNECTION kafka_connection (TOPIC 'datagen_demo_snowflakeschema_{{ name.table }}')
-KEY FORMAT BYTES
-VALUE FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY CONNECTION csr_connection
-INCLUDE TIMESTAMP as kafka_timestamp
-ENVELOPE UPSERT ;
+FROM KAFKA CONNECTION kafka_connection (TOPIC 'datagen_demo_snowflakeschema_{{ name.table }}');
 {% endmacro %}

test/canary-environment/models/loadgen/product_category_tbl.sql

@@ -0,0 +1,19 @@
+-- Copyright Materialize, Inc. and contributors. All rights reserved.
+--
+-- Use of this software is governed by the Business Source License
+-- included in the LICENSE file at the root of this repository.
+--
+-- As of the Change Date specified in that file, in accordance with
+-- the Business Source License, use of this software will be governed
+-- by the Apache License, Version 2.0.
+
+-- depends_on: {{ ref('product_category') }}
+{{ config(
+    materialized='source_table'
+) }}
+FROM SOURCE {{ ref('product_category') }}
+(REFERENCE "datagen_demo_snowflakeschema_product_category")
+KEY FORMAT BYTES
+VALUE FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY CONNECTION csr_connection
+INCLUDE TIMESTAMP as kafka_timestamp
+ENVELOPE UPSERT;

test/canary-environment/models/loadgen/product_tbl.sql

@@ -0,0 +1,19 @@
+-- Copyright Materialize, Inc. and contributors. All rights reserved.
+--
+-- Use of this software is governed by the Business Source License
+-- included in the LICENSE file at the root of this repository.
+--
+-- As of the Change Date specified in that file, in accordance with
+-- the Business Source License, use of this software will be governed
+-- by the Apache License, Version 2.0.
+
+-- depends_on: {{ ref('product') }}
+{{ config(
+    materialized='source_table'
+) }}
+FROM SOURCE {{ ref('product') }}
+(REFERENCE "datagen_demo_snowflakeschema_product")
+KEY FORMAT BYTES
+VALUE FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY CONNECTION csr_connection
+INCLUDE TIMESTAMP as kafka_timestamp
+ENVELOPE UPSERT;

test/canary-environment/models/loadgen/sales_product_product_category.sql

@@ -6,17 +6,17 @@
 -- As of the Change Date specified in that file, in accordance with
 -- the Business Source License, use of this software will be governed
 -- by the Apache License, Version 2.0.
--- depends_on: {{ ref('sales') }}
--- depends_on: {{ ref('product') }}
--- depends_on: {{ ref('product_category') }}
+-- depends_on: {{ ref('sales_tbl') }}
+-- depends_on: {{ ref('product_tbl') }}
+-- depends_on: {{ ref('product_category_tbl') }}
 
 {{ config(materialized='materialized_view', cluster="qa_canary_environment_compute", indexes=[{'default': True}]) }}
 
 SELECT
     count(*) AS count_star,
-    count(distinct sales.key) As count_distinct_loadgen_sales_key,
+    count(distinct sales_tbl.key) As count_distinct_loadgen_sales_key,
     count(distinct product_id) AS count_distinct_product_id,
     count(distinct category_id) AS count_distinct_category_id
-FROM      {{ source('loadgen','sales') }}
-LEFT JOIN {{ source('loadgen','product') }} USING (product_id)
-LEFT JOIN {{ source('loadgen','product_category') }} USING (category_id)
+FROM      {{ source('loadgen','sales_tbl') }}
+LEFT JOIN {{ source('loadgen','product_tbl') }} USING (product_id)
+LEFT JOIN {{ source('loadgen','product_category_tbl') }} USING (category_id)

test/canary-environment/models/loadgen/sales_tbl.sql

@@ -0,0 +1,19 @@
+-- Copyright Materialize, Inc. and contributors. All rights reserved.
+--
+-- Use of this software is governed by the Business Source License
+-- included in the LICENSE file at the root of this repository.
+--
+-- As of the Change Date specified in that file, in accordance with
+-- the Business Source License, use of this software will be governed
+-- by the Apache License, Version 2.0.
+
+-- depends_on: {{ ref('sales') }}
+{{ config(
+    materialized='source_table'
+) }}
+FROM SOURCE {{ ref('sales') }}
+(REFERENCE "datagen_demo_snowflakeschema_sales")
+KEY FORMAT BYTES
+VALUE FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY CONNECTION csr_connection
+INCLUDE TIMESTAMP as kafka_timestamp
+ENVELOPE UPSERT;

test/canary-environment/models/loadgen/schema.yml

@@ -21,6 +21,9 @@ sources:
           - makes_progress
       - name: product
       - name: product_category
+      - name: product_tbl
+      - name: product_category_tbl
+      - name: sales_tbl
 
 
 models:

test/canary-environment/models/mysql_cdc/mysql_cdc.sql

@@ -9,4 +9,3 @@
 
 {{ config(materialized='source', cluster='qa_canary_environment_storage') }}
 FROM MYSQL CONNECTION mysql
-FOR TABLES (public.people, public.relationships)

test/canary-environment/models/mysql_cdc/mysql_people.sql

@@ -0,0 +1,15 @@
+-- Copyright Materialize, Inc. and contributors. All rights reserved.
+--
+-- Use of this software is governed by the Business Source License
+-- included in the LICENSE file at the root of this repository.
+--
+-- As of the Change Date specified in that file, in accordance with
+-- the Business Source License, use of this software will be governed
+-- by the Apache License, Version 2.0.
+
+-- depends_on: {{ ref('mysql_cdc') }}
+{{ config(
+    materialized='source_table'
+) }}
+FROM SOURCE {{ ref('mysql_cdc') }}
+(REFERENCE "public"."people")

test/canary-environment/models/mysql_cdc/mysql_relationships.sql

@@ -0,0 +1,15 @@
+-- Copyright Materialize, Inc. and contributors. All rights reserved.
+--
+-- Use of this software is governed by the Business Source License
+-- included in the LICENSE file at the root of this repository.
+--
+-- As of the Change Date specified in that file, in accordance with
+-- the Business Source License, use of this software will be governed
+-- by the Apache License, Version 2.0.
+
+-- depends_on: {{ ref('mysql_cdc') }}
+{{ config(
+    materialized='source_table'
+) }}
+FROM SOURCE {{ ref('mysql_cdc') }}
+(REFERENCE "public"."relationships")

test/canary-environment/models/mysql_cdc/mysql_wmr.sql

@@ -12,13 +12,15 @@
 -- to ensure the result updates frequently
 
 -- depends_on: {{ ref('mysql_cdc') }}
+-- depends_on: {{ ref('mysql_people') }}
+-- depends_on: {{ ref('mysql_relationships') }}
 {{ config(materialized='materialized_view', cluster="qa_canary_environment_compute", indexes=[{'default': True}]) }}
 
 WITH MUTUALLY RECURSIVE
     symm (a int, b int) AS (
-        SELECT a, b FROM {{ source('mysql_cdc','relationships') }}
+        SELECT a, b FROM {{ source('mysql_cdc', 'mysql_relationships') }}
         UNION
-        SELECT b, a FROM {{ source('mysql_cdc','relationships') }}
+        SELECT b, a FROM {{ source('mysql_cdc', 'mysql_relationships') }}
     ),
     candidates (a int, b int, degree int) AS (
         SELECT a, b, 1 FROM symm
@@ -32,6 +34,6 @@ WITH MUTUALLY RECURSIVE
     )
 SELECT DISTINCT a_people.name AS a_name, b_people.name AS b_name, degree
 FROM reach
-LEFT JOIN {{ source('mysql_cdc','people') }} AS a_people ON (a = a_people.id)
-LEFT JOIN {{ source('mysql_cdc','people') }} AS b_people ON (b = b_people.id)
+LEFT JOIN {{ source('mysql_cdc', 'mysql_people') }} AS a_people ON (a = a_people.id)
+LEFT JOIN {{ source('mysql_cdc', 'mysql_people') }} AS b_people ON (b = b_people.id)
 WHERE degree >= 2

test/canary-environment/models/mysql_cdc/schema.yml

@@ -16,10 +16,10 @@ sources:
       - name: mysql_cdc_progress
         tests:
           - makes_progress
-      - name: people
+      - name: mysql_people
         tests:
           - makes_progress
-      - name: relationships
+      - name: mysql_relationships
         tests:
           - makes_progress
 

test/canary-environment/models/pg_cdc/pg_cdc.sql

@@ -10,4 +10,3 @@
 {{ config(materialized='source', cluster='qa_canary_environment_storage') }}
 FROM POSTGRES
 CONNECTION pg (PUBLICATION 'mz_source')
-FOR TABLES (people, relationships)

test/canary-environment/models/pg_cdc/pg_people.sql

@@ -0,0 +1,15 @@
+-- Copyright Materialize, Inc. and contributors. All rights reserved.
+--
+-- Use of this software is governed by the Business Source License
+-- included in the LICENSE file at the root of this repository.
+--
+-- As of the Change Date specified in that file, in accordance with
+-- the Business Source License, use of this software will be governed
+-- by the Apache License, Version 2.0.
+
+-- depends_on: {{ ref('pg_cdc') }}
+{{ config(
+    materialized='source_table'
+) }}
+FROM SOURCE {{ ref('pg_cdc') }}
+(REFERENCE "people")

test/canary-environment/models/pg_cdc/pg_relationships.sql

@@ -0,0 +1,15 @@
+-- Copyright Materialize, Inc. and contributors. All rights reserved.
+--
+-- Use of this software is governed by the Business Source License
+-- included in the LICENSE file at the root of this repository.
+--
+-- As of the Change Date specified in that file, in accordance with
+-- the Business Source License, use of this software will be governed
+-- by the Apache License, Version 2.0.
+
+-- depends_on: {{ ref('pg_cdc') }}
+{{ config(
+    materialized='source_table'
+) }}
+FROM SOURCE {{ ref('pg_cdc') }}
+(REFERENCE "relationships")

test/canary-environment/models/pg_cdc/wmr.sql → test/canary-environment/models/pg_cdc/pg_wmr.sql

@@ -12,13 +12,15 @@
 -- to ensure the result updates frequently
 
 -- depends_on: {{ ref('pg_cdc') }}
+-- depends_on: {{ ref('pg_people') }}
+-- depends_on: {{ ref('pg_relationships') }}
 {{ config(materialized='materialized_view', cluster="qa_canary_environment_compute", indexes=[{'default': True}]) }}
 
 WITH MUTUALLY RECURSIVE
     symm (a int, b int) AS (
-        SELECT a, b FROM {{ source('pg_cdc','relationships') }}
+        SELECT a, b FROM {{ source('pg_cdc','pg_relationships') }}
         UNION
-        SELECT b, a FROM {{ source('pg_cdc','relationships') }}
+        SELECT b, a FROM {{ source('pg_cdc','pg_relationships') }}
     ),
     candidates (a int, b int, degree int) AS (
         SELECT a, b, 1 FROM symm
@@ -32,6 +34,6 @@ WITH MUTUALLY RECURSIVE
     )
 SELECT DISTINCT a_people.name AS a_name, b_people.name AS b_name, degree
 FROM reach
-LEFT JOIN {{ source('pg_cdc','people') }} AS a_people ON (a = a_people.id)
-LEFT JOIN {{ source('pg_cdc','people') }} AS b_people ON (b = b_people.id)
+LEFT JOIN {{ source('pg_cdc','pg_people') }} AS a_people ON (a = a_people.id)
+LEFT JOIN {{ source('pg_cdc','pg_people') }} AS b_people ON (b = b_people.id)
 WHERE degree >= 2

test/canary-environment/models/pg_cdc/schema.yml

@@ -16,14 +16,14 @@ sources:
       - name: pg_cdc_progress
         tests:
           - makes_progress
-      - name: people
+      - name: pg_people
         tests:
           - makes_progress
-      - name: relationships
+      - name: pg_relationships
         tests:
           - makes_progress
 
 models:
-    - name: wmr
+    - name: pg_wmr
       tests:
         - makes_progress

test/canary-environment/models/tpch/schema.yml

@@ -16,16 +16,11 @@ sources:
       - name: tpch_progress
         tests:
           - makes_progress
-      - name: customer
-      - name: lineitem
-      - name: nation
-      - name: orders
+      - name: tpch_customer
+      - name: tpch_lineitem
+      - name: tpch_orders
         tests:
           - makes_progress
-      - name: part
-      - name: partsupp
-      - name: region
-      - name: supplier
 
 models:
     - name: tpch_q01

test/canary-environment/models/tpch/tpch.sql

@@ -10,4 +10,3 @@
 {{ config(materialized='source', cluster='qa_canary_environment_storage') }}
 FROM LOAD GENERATOR
 TPCH (SCALE FACTOR 1, TICK INTERVAL '0.1s')
-FOR ALL TABLES

test/canary-environment/models/tpch/tpch_customer.sql

@@ -0,0 +1,15 @@
+-- Copyright Materialize, Inc. and contributors. All rights reserved.
+--
+-- Use of this software is governed by the Business Source License
+-- included in the LICENSE file at the root of this repository.
+--
+-- As of the Change Date specified in that file, in accordance with
+-- the Business Source License, use of this software will be governed
+-- by the Apache License, Version 2.0.
+
+-- depends_on: {{ ref('tpch') }}
+{{ config(
+    materialized='source_table'
+) }}
+FROM SOURCE {{ ref('tpch') }}
+(REFERENCE "customer")

test/canary-environment/models/tpch/tpch_lineitem.sql

@@ -0,0 +1,15 @@
+-- Copyright Materialize, Inc. and contributors. All rights reserved.
+--
+-- Use of this software is governed by the Business Source License
+-- included in the LICENSE file at the root of this repository.
+--
+-- As of the Change Date specified in that file, in accordance with
+-- the Business Source License, use of this software will be governed
+-- by the Apache License, Version 2.0.
+
+-- depends_on: {{ ref('tpch') }}
+{{ config(
+    materialized='source_table'
+) }}
+FROM SOURCE {{ ref('tpch') }}
+(REFERENCE "lineitem")

test/canary-environment/models/tpch/tpch_orders.sql

@@ -0,0 +1,15 @@
+-- Copyright Materialize, Inc. and contributors. All rights reserved.
+--
+-- Use of this software is governed by the Business Source License
+-- included in the LICENSE file at the root of this repository.
+--
+-- As of the Change Date specified in that file, in accordance with
+-- the Business Source License, use of this software will be governed
+-- by the Apache License, Version 2.0.
+
+-- depends_on: {{ ref('tpch') }}
+{{ config(
+    materialized='source_table'
+) }}
+FROM SOURCE {{ ref('tpch') }}
+(REFERENCE "orders")

test/canary-environment/models/tpch/tpch_q01.sql

@@ -10,6 +10,7 @@
 -- TPC-H Query #Q18, chosen for the workload for its GROUP BY clause
 
 -- depends_on: {{ ref('tpch') }}
+-- depends_on: {{ ref('tpch_lineitem') }}
 {{ config(materialized='materialized_view', cluster='qa_canary_environment_compute', indexes=[{'default': True}]) }}
 
 SELECT
@@ -24,7 +25,7 @@ SELECT
     avg(l_discount) AS avg_disc,
     count(*) AS count_order
 FROM
-    {{ source('tpch','lineitem') }}
+    {{ source('tpch', 'tpch_lineitem') }}
 WHERE
     l_shipdate <= DATE '1998-12-01' - INTERVAL '60' day
 GROUP BY

test/canary-environment/models/tpch/tpch_q18.sql

@@ -12,6 +12,9 @@
 -- to ensure the result updates frequently
 
 -- depends_on: {{ ref('tpch') }}
+-- depends_on: {{ ref('tpch_customer') }}
+-- depends_on: {{ ref('tpch_orders') }}
+-- depends_on: {{ ref('tpch_lineitem') }}
 {{ config(materialized='materialized_view', cluster='qa_canary_environment_compute', indexes=[{'default': True}]) }}
 
 SELECT
@@ -22,15 +25,15 @@ SELECT
     o_totalprice,
     sum(l_quantity)
 FROM
-    {{ source('tpch','customer') }} ,
-    {{ source('tpch','orders') }} ,
-    {{ source('tpch','lineitem') }}
+    {{ source('tpch','tpch_customer') }} ,
+    {{ source('tpch','tpch_orders') }} ,
+    {{ source('tpch','tpch_lineitem') }}
 WHERE
     o_orderkey IN (
         SELECT
             l_orderkey
         FROM
-            {{ source('tpch','lineitem') }}
+            {{ source('tpch','tpch_lineitem') }}
         GROUP BY
             l_orderkey having
                 sum(l_quantity) > 250