diff --git a/scenarios/cdp_campaign_management/README.md b/scenarios/cdp_campaign_management/README.md index 9a83e3db..e1a84af8 100644 --- a/scenarios/cdp_campaign_management/README.md +++ b/scenarios/cdp_campaign_management/README.md @@ -131,7 +131,7 @@ The `cdp_campaigns_${ps_id}` is automatically created and the following tables a ### master_activations -| time | journey_id | activation_step_id | syndication_id | activation_name | schedule_type | schedule_option | timezone | connection_id | all_columns | step_id | stage_name | stage_no | state | created_at | updated_at | journey_name | connector_type | +| time | journey_id | activation_id | syndication_id | activation_name | schedule_type | schedule_option | timezone | connection_id | all_columns | step_id | stage_name | stage_no | state | created_at | updated_at | journey_name | connector_type | | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | | 1701927703 | 32805 | 51925 | 240504 | Welcome Mail | daily | 1:00:00 | Asia/Tokyo | 301955 | FALSE | eaae03ce_fc23_4945_aa7a_4717c25ea9b4 | 1 | 0 | launched | 2023-12-07T05:41:43.283Z | 2023-12-07T15:18:23.687Z | Purchase Journey | Marketo | | 1701927703 | 32805 | 51926 | 240505 | Next Step Mail | daily | 1:00:00 | Asia/Tokyo | 301955 | FALSE | 2a83381f_f044_4da8_b65b_d87bd1c493c5 | 1 | 0 | launched | 2023-12-07T05:41:43.283Z | 2023-12-07T15:18:23.687Z | Purchase Journey | Marketo | @@ -154,7 +154,7 @@ This table is the output of the activation history from Journey Orchestration. T #### table example -| time | *member_id | activation_step_id | syndication_id | activation_type | activation_name | cv_name | utm_campaign | utm_medium | utm_source | utm_content | utm_term | utm_connector | +| time | *member_id | activation_id | syndication_id | activation_type | activation_name | cv_name | utm_campaign | utm_medium | utm_source | utm_content | utm_term | utm_connector | | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |--- | | 1701705887 | ffdae365-24b7-4d69-ae54-05340ee5c57f | 50659 | 237814 | journeyActivationStep | to_td | PURCHASE | td_plazma12 | email | treasuredatajp | control | | mailchimp | @@ -166,7 +166,7 @@ This table is the campaign click history. This table is output in the process of #### table example -| time | db_name | table_name | *member_id | activation_step_id | cv_name | utm_campaign | utm_medium | utm_source | utm_content | utm_term | utm_connector | +| time | db_name | table_name | *member_id | activation_id | cv_name | utm_campaign | utm_medium | utm_source | utm_content | utm_term | utm_connector | | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | | 1695535786 | cdp_audience_507568 | behavior_behv_website | e05c10e0-1943-4afe-b799-92ef9f24ed9f | 21 | SUBSCRIBE | td_plazma2022summerinv_link1 | email | nikkei_bpdmp | welcome_mail | nikkei | marketo | @@ -190,17 +190,17 @@ We can union `activations` and `clicks` and `conversions` to create a conversion | table_name | | | | | | | | | | | | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| activations | time | user_id | | activation_step_id | | | | | | | -| clicks | time | user_id | cv_name | activation_step_id | utm_campaign | utm_medium | utm_source | utm_content | utm_connector | utm_term | +| activations | time | user_id | | activation_id | | | | | | | +| clicks | time | user_id | cv_name | activation_id | utm_campaign | utm_medium | utm_source | utm_content | utm_connector | utm_term | | conversions | time | user_id | cv_name | | | | | | | | -- `activations` and `clicks` table are unioned by `user_id` and `activation_step_id`. +- `activations` and `clicks` table are unioned by `user_id` and `activation_id`. - `clicks` and `conversions` table are unioned by `user_id` and `cv_name`. Hence, we can create an `activation -> click -> conversion` journey for each user and each conversion. #### table example -| time | type | member_id | activation_step_id | cv_name | utm_campaign | utm_medium | utm_source | utm_content | utm_connector | cv_flg | val | revenue | time_hour_from_activation | +| time | type | member_id | activation_id | cv_name | utm_campaign | utm_medium | utm_source | utm_content | utm_connector | cv_flg | val | revenue | time_hour_from_activation | | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | | 1700605034 | Activation | 007a5c3d-1355-4352-af1d-440f2d803f90 | 1928 | DOWNLOAD | td_14947 | push | pushcode | | | 0 | 0 | 0 | | | 1700606147 | Click | 007a5c3d-1355-4352-af1d-440f2d803f90 | 1928 | DOWNLOAD | td_14947 | push | pushcode | | | 0 | 0 | 0 | 46.3 | @@ -218,7 +218,7 @@ This table is the result of the calculation of acquired revenue per campaign by #### table example -| time | date | cv_time | *member_id | cv_id | position | time_hour_to_cv | time_hour_to_next | time_hour_from_activation | type | click_type | activation_step_id | utm_source | utm_medium | utm_campaign | utm_content | utm_connector | cv_name | size_journey | size_cv_session | size_middle_click | is_within_cv_session | revenue | acquired_person_last_click_model | acquired_revenue_last_click_model | acquired_person_first_click_model | acquired_revenue_first_click_model | acquired_person_session_model | acquired_revenue_session_model | +| time | date | cv_time | *member_id | cv_id | position | time_hour_to_cv | time_hour_to_next | time_hour_from_activation | type | click_type | activation_id | utm_source | utm_medium | utm_campaign | utm_content | utm_connector | cv_name | size_journey | size_cv_session | size_middle_click | is_within_cv_session | revenue | acquired_person_last_click_model | acquired_revenue_last_click_model | acquired_person_first_click_model | acquired_revenue_first_click_model | acquired_person_session_model | acquired_revenue_session_model | | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | | 1700606147 | 2023-11-22 | 1702167880 | 007a5c3d-1355-4352-af1d-440f2d803f90 | 02a0ca2ff7412557b1a617d5855de2a1 | 1 | 433 | 337 | | Click | First Click | 1928 | pushcode | push | td_14947 | | | DOWNLOAD | 3 | 2 | 1 | 0 | 0 | 0 | 0 | 1 | 20000 | 0 | 0 | | 1701822271 | 2023-12-06 | 1702167880 | 007a5c3d-1355-4352-af1d-440f2d803f90 | 02a0ca2ff7412557b1a617d5855de2a1 | 2 | 96 | 48 | | Click | Middle Click | 51418 | diamondrm | email | td_plazma15 | | | DOWNLOAD | 3 | 2 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0.5 | 10000 | @@ -245,7 +245,7 @@ This table is the result of the calculation of acquired revenue per campaign by #### table example -| time | date | activation_step_id | is_internal_campaign_click | type | utm_source | utm_medium | utm_campaign | utm_content | utm_connector | cv_name | cnt_activations | cnt_clicks | cnt_clicks_related_conversion | acquired_person_last_click_model | acquired_person_first_click_model | acquired_person_session_model | acquired_revenue_last_click_model | acquired_revenue_first_click_model | acquired_revenue_session_model | size_journey | cnt_cv_id | +| time | date | activation_id | is_internal_campaign_click | type | utm_source | utm_medium | utm_campaign | utm_content | utm_connector | cv_name | cnt_activations | cnt_clicks | cnt_clicks_related_conversion | acquired_person_last_click_model | acquired_person_first_click_model | acquired_person_session_model | acquired_revenue_last_click_model | acquired_revenue_first_click_model | acquired_revenue_session_model | size_journey | cnt_cv_id | | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | | 1701788400 | 2023-12-06 | 51397 | internal | Click | treasuredatajp | email | td_newsletter20200327 | | | DOWNLOAD | 366 | 1636 | 740 | 289 | 199 | 345.1166667 | 5780000 | 3980000 | 6902333.333 | 2161 | 740 | | 1701788400 | 2023-12-06 | 51418 | internal | Click | diamondrm | email | td_plazma15 | | | DOWNLOAD | 0 | 1597 | 723 | 150 | 404 | 336.95 | 3000000 | 8080000 | 6739000 | 2076 | 723 | @@ -279,23 +279,23 @@ It is effective to visualize the above indicators on a dashboard. The above dash ### existing_campaigns -This table extracts all utm parameters present in the `clicks` table. It lists unique campaigns with the following 5 values, but since `cv_name`, `activation_step_id` often do not contain values, it is effectively a combination of 3. +This table extracts all utm parameters present in the `clicks` table. It lists unique campaigns with the following 5 values, but since `cv_name`, `activation_id` often do not contain values, it is effectively a combination of 3. - utm_source - utm_medium - utm_campaign - cv_name -- activation_step_id +- activation_id #### table example -| time | exists_in_daily_activations | utm_source | utm_medium | utm_campaign | cv_name | activation_step_id | utm_content | utm_connector | utm_term | date_first_appeared | date_last_appeared | cnt | +| time | exists_in_daily_activations | utm_source | utm_medium | utm_campaign | cv_name | activation_id | utm_content | utm_connector | utm_term | date_first_appeared | date_last_appeared | cnt | | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | | 1702771200 | 0 | treasuredatajp | email | td_dac-webinar-thx | COST_REDUCTION | 1926 | | | COST_REDUCTION | 2023-12-17 | 2023-12-17 | 1 | | 1703635200 | 1 | treasuredatajp | email | td_plazma2022summerinv_03 | SUBSCRIBE | 57189 | | | SUBSCRIBE | 2023-12-27 | 2023-12-27 | 1 | | 1589500800 | 0 | google | cpc | company_summer_sale_2019 | | | | | | 2020-05-15 | 2020-05-15 | 1 | -`exists_in_daily_activations` is 1 if the `activation_step_id` is from an activation within this parent segment, 0 otherwise or with no value, meaning it is an external campaign. +`exists_in_daily_activations` is 1 if the `activation_id` is from an activation within this parent segment, 0 otherwise or with no value, meaning it is an external campaign. By editing/adding and uploading the value of each utm parameter in this table, the retrofitted parameter values can be used when measuring effectiveness. Refer to [master_campaigns_tables](#master_campaigns_tables) for how to do this. @@ -565,7 +565,7 @@ Required options depend on the value of `is_audience_table`. Specify the source of the master_campaigns table that the user will be uploading. By setting this table, the utm parameter of the clicks table can be retrofitted. > [!IMPORTANT] -> The goal for setting up this table is when you want to retrofit the `cv_name` or `activation_step_id` ( or `utm_term`, `utm_content`, `utm_connector`). These values can be set for each combination of `utm_source`, `utm_medium`, and `utm_campaign`. So you can't retrofit the `utm_source`, `utm_medium`, and `utm_campaign`. +> The goal for setting up this table is when you want to retrofit the `cv_name` or `activation_id` ( or `utm_term`, `utm_content`, `utm_connector`). These values can be set for each combination of `utm_source`, `utm_medium`, and `utm_campaign`. So you can't retrofit the `utm_source`, `utm_medium`, and `utm_campaign`. The master_campaigns table will be based on the [existing_campaigns](#existing_campaigns) output from this WF, with the utm parameter values of the campaigns you wish to edit or add edited. @@ -593,7 +593,7 @@ The required table columns for the master_campaigns table are as follows. It mus | utm_medium | Y | The same value as the record in the original clicks table. | | utm_campaign | Y | The same value as the record in the original clicks table. | | cv_name | Y | Setting a value will be priority reflected when measuring effectiveness. Unless this value is set, it is not subject to effectiveness measurement. | -| activation_step_id | | Setting a value will be priority reflected when measuring effectiveness. Specify if the campaign link is tied to an activation. | +| activation_id | | Setting a value will be priority reflected when measuring effectiveness. Specify if the campaign link is tied to an activation. | | utm_content | | Setting a value will be priority reflected when measuring effectiveness. | | utm_connector | | Setting a value will be priority reflected when measuring effectiveness. | | utm_term | | Setting a value will be priority reflected when measuring effectiveness. | @@ -602,7 +602,7 @@ The required table columns for the master_campaigns table are as follows. It mus ###### Records in the original clicks table -| utm_source | utm_medium | utm_campaign | cv_name | activation_step_id | utm_content | utm_connector | +| utm_source | utm_medium | utm_campaign | cv_name | activation_id | utm_content | utm_connector | | --- | --- | --- | --- | --- | --- | --- | | google | cpc | company_summer_sale_2019 | | | A | td | | sfmc | email | mnt_helmet_abandoned_cart | | | A | td | @@ -610,7 +610,7 @@ The required table columns for the master_campaigns table are as follows. It mus ###### Records in the master_campaigns table -| utm_source | utm_medium | utm_campaign | cv_name | activation_step_id | utm_content | utm_connector | +| utm_source | utm_medium | utm_campaign | cv_name | activation_id | utm_content | utm_connector | | --- | --- | --- | --- | --- | --- | --- | | google | cpc | company_summer_sale_2019 | PURCHASE | 1234 | | | | sfmc | email | mnt_helmet_abandoned_cart | DOWNLOAD | 5678 | B | marketo | @@ -618,7 +618,7 @@ The required table columns for the master_campaigns table are as follows. It mus ###### Records in the clicks table after merge -| utm_source | utm_medium | utm_campaign | cv_name | activation_step_id | utm_content | utm_connector | +| utm_source | utm_medium | utm_campaign | cv_name | activation_id | utm_content | utm_connector | | --- | --- | --- | --- | --- | --- | --- | | google | cpc | company_summer_sale_2019 | PURCHASE | 1234 | A | td | | sfmc | email | mnt_helmet_abandoned_cart | DOWNLOAD | 5678 | B | marketo | @@ -868,7 +868,7 @@ This will create a `clicks -> conversions` journey. This allows us to calculate #### example of existing_campaigns table - | utm_source | utm_medium | utm_campaign | cv_name | activation_step_id | utm_content | utm_connector | utm_term | date_first_appeared | date_last_appeared | cnt | time | + | utm_source | utm_medium | utm_campaign | cv_name | activation_id | utm_content | utm_connector | utm_term | date_first_appeared | date_last_appeared | cnt | time | | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | | treasuredatajp | email | newsletter20220112 | | | plazmaarticle | marketo | | 2022-01-11 | 2023-04-27 | 118 | 1641859200 | | treasuredatajp | email | nurture_general | | | awareness_em4 | marketo | | 2022-01-11 | 2023-12-05 | 85 | 1641859200 | @@ -877,7 +877,7 @@ This will create a `clicks -> conversions` journey. This allows us to calculate #### example of master_campaigns_0 table - | utm_source | utm_medium | utm_campaign | cv_name | activation_step_id | utm_content | utm_connector | utm_term | + | utm_source | utm_medium | utm_campaign | cv_name | activation_id | utm_content | utm_connector | utm_term | | --- | --- | --- | --- | --- | --- | --- | --- | | treasuredatajp | email | newsletter20220112 | SUBSCRIBE | | plazmaarticle | marketo | | | treasuredatajp | email | nurture_general | SUBSCRIBE | | awareness_em4 | marketo | |2024-01-09 | 263 | 1641859200 | diff --git a/scenarios/cdp_campaign_management/common/system_settings.yaml b/scenarios/cdp_campaign_management/common/system_settings.yaml index 7d72ee98..91f48c85 100644 --- a/scenarios/cdp_campaign_management/common/system_settings.yaml +++ b/scenarios/cdp_campaign_management/common/system_settings.yaml @@ -1,5 +1,7 @@ parallel: true td: + timezone: UTC + valuation_target: 'click' mta: session_model: allowable_time_to_cv: 24*10 diff --git a/scenarios/cdp_campaign_management/common/user_settings_sample.yaml b/scenarios/cdp_campaign_management/common/user_settings_sample.yaml index 5ac3379b..6f55c062 100644 --- a/scenarios/cdp_campaign_management/common/user_settings_sample.yaml +++ b/scenarios/cdp_campaign_management/common/user_settings_sample.yaml @@ -1,5 +1,6 @@ td: timezone: JST + # valuation_target: 'activation' ps: - 507568 @@ -7,8 +8,8 @@ td: 507568: td_client_id activations_tables: - # 507568: - # scan_journey_tables: true + 507568: + scan_journey_tables: true clicks_tables: 507568: diff --git a/scenarios/cdp_campaign_management/main_incremental_ingest.dig b/scenarios/cdp_campaign_management/main_incremental_ingest.dig index d6f5145b..c162b127 100644 --- a/scenarios/cdp_campaign_management/main_incremental_ingest.dig +++ b/scenarios/cdp_campaign_management/main_incremental_ingest.dig @@ -16,13 +16,12 @@ _export: ps_id: ${td.ps} _do: _export: - user_timezone: "${typeof td.timezone === 'undefined' ? 'UTC' : td.timezone}" td: database: ${td.base_db_name}_${ps_id} user_id: ${td.user_id[ps_id]} +show_timezone: - echo>: "Timezone: ${user_timezone}" + echo>: "Timezone: ${td.timezone}" +prepare_dbs: td_ddl>: diff --git a/scenarios/cdp_campaign_management/main_initial_ingest.dig b/scenarios/cdp_campaign_management/main_initial_ingest.dig index cc276836..28dc94df 100644 --- a/scenarios/cdp_campaign_management/main_initial_ingest.dig +++ b/scenarios/cdp_campaign_management/main_initial_ingest.dig @@ -11,13 +11,12 @@ _export: ps_id: ${td.ps} _do: _export: - user_timezone: "${typeof td.timezone === 'undefined' ? 'UTC' : td.timezone}" td: database: ${td.base_db_name}_${ps_id} user_id: ${td.user_id[ps_id]} +show_timezone: - echo>: "Timezone: ${user_timezone}" + echo>: "Timezone: ${td.timezone}" +prepare_dbs: td_ddl>: diff --git a/scenarios/cdp_campaign_management/py_scripts/ingest_activations_queries.py b/scenarios/cdp_campaign_management/py_scripts/ingest_activations_queries.py index 24d2def0..9123183f 100644 --- a/scenarios/cdp_campaign_management/py_scripts/ingest_activations_queries.py +++ b/scenarios/cdp_campaign_management/py_scripts/ingest_activations_queries.py @@ -39,7 +39,7 @@ def run( input_tables.append(tbl.name) res = client.query( - f'SELECT journey_id, syndication_id, activation_step_id, activation_name, stage_no, step_id FROM {input_db}.{input_table_master_activations} WHERE syndication_id is not NULL' + f'SELECT journey_id, syndication_id, activation_id, activation_name, stage_no, step_id FROM {input_db}.{input_table_master_activations} WHERE syndication_id is not NULL AND stage_no is not NULL' ) df_1 = pd.DataFrame(**res) print(f"Searched {len(df_1)} activations. Create {math.ceil(len(df_1)/MAX_UNIONS)} queries in the process.") @@ -52,7 +52,7 @@ def run( journey_id = row['journey_id'] journey_table = 'journey_%s' % (journey_id) syndication_id = row['syndication_id'] - activation_step_id = row['activation_step_id'] + activation_id = row['activation_id'] activation_name = row['activation_name'] stage_no = row['stage_no'] step_id = row['step_id'] @@ -65,7 +65,7 @@ def run( continue inner_qry = ' '.join([ - f"SELECT \"{column_id}\" AS session_time, t2.{user_id}, t1.cdp_customer_id, '{syndication_id}' AS syndication_id, '{activation_step_id}' AS activation_step_id, '{activation_name}' AS activation_name", + f"SELECT \"{column_id}\" AS session_time, t2.{user_id}, t1.cdp_customer_id, '{syndication_id}' AS syndication_id, '{activation_id}' AS activation_id, '{activation_name}' AS activation_name", f"FROM {cdp_audience_db}.{journey_table} t1", f"LEFT OUTER JOIN {cdp_audience_db}.{input_table_customers} t2", f"ON t1.cdp_customer_id = t2.cdp_customer_id", @@ -77,7 +77,7 @@ def run( if not journey_reentry_history_table in input_tables: inner_qry += ' '.join([ f" UNION ALL", - f"SELECT \"{column_id}\" AS session_time, t2.{user_id}, t1.cdp_customer_id, '{syndication_id}' AS syndication_id, '{activation_step_id}' AS activation_step_id, '{activation_name}' AS activation_name", + f"SELECT \"{column_id}\" AS session_time, t2.{user_id}, t1.cdp_customer_id, '{syndication_id}' AS syndication_id, '{activation_id}' AS activation_id, '{activation_name}' AS activation_name", f"FROM {cdp_audience_db}.{journey_reentry_history_table} t1", f"LEFT OUTER JOIN {cdp_audience_db}.{input_table_customers} t2", f"ON t1.cdp_customer_id = t2.cdp_customer_id", @@ -89,7 +89,7 @@ def run( if journey_jump_history_table in input_tables: inner_qry += ' '.join([ f" UNION ALL", - f"SELECT \"{column_id}\" AS session_time, t2.{user_id}, t1.cdp_customer_id, '{syndication_id}' AS syndication_id, '{activation_step_id}' AS activation_step_id, '{activation_name}' AS activation_name", + f"SELECT \"{column_id}\" AS session_time, t2.{user_id}, t1.cdp_customer_id, '{syndication_id}' AS syndication_id, '{activation_id}' AS activation_id, '{activation_name}' AS activation_name", f"FROM {cdp_audience_db}.{journey_jump_history_table} t1", f"LEFT OUTER JOIN {cdp_audience_db}.{input_table_customers} t2", f"ON t1.cdp_customer_id = t2.cdp_customer_id", @@ -101,7 +101,7 @@ def run( f"SELECT", f"s2.time_finished AS time", f",s1.{user_id}", - f",s1.activation_step_id", + f",s1.activation_id", f",s1.syndication_id", f",'journeyActivationStep' AS activation_type", f",{journey_id} AS journey_id", @@ -120,7 +120,7 @@ def run( f"AND s1.syndication_id = s2.syndication_id", f"LEFT OUTER JOIN (", f"SELECT", - f"activation_step_id", + f"activation_id", f",MAX_BY(cv_name,time) AS cv_name", f",MAX_BY(utm_campaign,time) AS utm_campaign", f",MAX_BY(utm_medium,time) AS utm_medium", @@ -131,7 +131,7 @@ def run( f"FROM (", f"SELECT", f"time", - f",CAST(activation_step_id AS VARCHAR) AS activation_step_id", + f",CAST(activation_id AS VARCHAR) AS activation_id", f",cv_name", f",utm_campaign", f",utm_medium", @@ -143,7 +143,7 @@ def run( f"UNION ALL", f"SELECT", f"time", - f",CAST(activation_step_id AS VARCHAR) AS activation_step_id", + f",CAST(activation_id AS VARCHAR) AS activation_id", f",cv_name", f",utm_campaign", f",utm_medium", @@ -155,7 +155,7 @@ def run( f")", f"GROUP BY 1", f") s3", - f"ON s1.activation_step_id = s3.activation_step_id", + f"ON s1.activation_id = s3.activation_id", ]) if qry_union_all == '': diff --git a/scenarios/cdp_campaign_management/queries/create_activations.sql b/scenarios/cdp_campaign_management/queries/create_activations.sql index afb068f9..5c47fdfc 100644 --- a/scenarios/cdp_campaign_management/queries/create_activations.sql +++ b/scenarios/cdp_campaign_management/queries/create_activations.sql @@ -2,6 +2,7 @@ DROP TABLE IF EXISTS ${dest_db}.${dest_table}; CREATE TABLE IF NOT EXISTS ${dest_db}.${dest_table} ( time bigint ,${user_id} varchar + ,activation_id varchar ,activation_step_id varchar ,syndication_id varchar ,journey_id varchar diff --git a/scenarios/cdp_campaign_management/queries/create_clicks.sql b/scenarios/cdp_campaign_management/queries/create_clicks.sql index db17a787..460f439c 100644 --- a/scenarios/cdp_campaign_management/queries/create_clicks.sql +++ b/scenarios/cdp_campaign_management/queries/create_clicks.sql @@ -5,7 +5,7 @@ CREATE TABLE IF NOT EXISTS ${dest_db}.${dest_table} ( ,db_name varchar ,table_name varchar ,${user_id} varchar - ,activation_step_id varchar + ,activation_id varchar ,cv_name varchar ,utm_campaign varchar ,utm_medium varchar diff --git a/scenarios/cdp_campaign_management/queries/create_conversion_journeys.sql b/scenarios/cdp_campaign_management/queries/create_conversion_journeys.sql index 4685dcb0..2b2816e1 100644 --- a/scenarios/cdp_campaign_management/queries/create_conversion_journeys.sql +++ b/scenarios/cdp_campaign_management/queries/create_conversion_journeys.sql @@ -4,7 +4,7 @@ CREATE TABLE IF NOT EXISTS ${dest_db}.${dest_table} ( ,time_s varchar ,type varchar ,${user_id} varchar - ,activation_step_id varchar + ,activation_id varchar ,cv_name varchar ,cv_flg int ,val double diff --git a/scenarios/cdp_campaign_management/queries/create_daily_activations.sql b/scenarios/cdp_campaign_management/queries/create_daily_activations.sql index 57745465..bc09cdaf 100644 --- a/scenarios/cdp_campaign_management/queries/create_daily_activations.sql +++ b/scenarios/cdp_campaign_management/queries/create_daily_activations.sql @@ -2,7 +2,7 @@ DROP TABLE IF EXISTS ${dest_db}.${dest_table}; CREATE TABLE IF NOT EXISTS ${dest_db}.${dest_table} ( time bigint ,date varchar - ,activation_step_id varchar + ,activation_id varchar ,cv_name varchar ,utm_source varchar ,utm_medium varchar diff --git a/scenarios/cdp_campaign_management/queries/create_daily_clicks.sql b/scenarios/cdp_campaign_management/queries/create_daily_clicks.sql index a46a509b..b4137d65 100644 --- a/scenarios/cdp_campaign_management/queries/create_daily_clicks.sql +++ b/scenarios/cdp_campaign_management/queries/create_daily_clicks.sql @@ -2,7 +2,7 @@ DROP TABLE IF EXISTS ${dest_db}.${dest_table}; CREATE TABLE IF NOT EXISTS ${dest_db}.${dest_table} ( time bigint ,date varchar - ,activation_step_id varchar + ,activation_id varchar ,cv_name varchar ,utm_source varchar ,utm_medium varchar diff --git a/scenarios/cdp_campaign_management/queries/create_daily_mta_conversion_journeys.sql b/scenarios/cdp_campaign_management/queries/create_daily_mta_conversion_journeys.sql index ab74e2c0..8b460801 100644 --- a/scenarios/cdp_campaign_management/queries/create_daily_mta_conversion_journeys.sql +++ b/scenarios/cdp_campaign_management/queries/create_daily_mta_conversion_journeys.sql @@ -2,7 +2,7 @@ DROP TABLE IF EXISTS ${dest_db}.${dest_table}; CREATE TABLE IF NOT EXISTS ${dest_db}.${dest_table} ( time bigint ,date varchar - ,activation_step_id varchar varchar + ,activation_id varchar varchar ,is_internal_campaign_click varchar ,type varchar ,utm_source varchar diff --git a/scenarios/cdp_campaign_management/queries/create_master_activations.sql b/scenarios/cdp_campaign_management/queries/create_master_activations.sql index 917c1ed2..0af63df3 100644 --- a/scenarios/cdp_campaign_management/queries/create_master_activations.sql +++ b/scenarios/cdp_campaign_management/queries/create_master_activations.sql @@ -2,6 +2,9 @@ DROP TABLE IF EXISTS ${dest_db}.${dest_table}; CREATE TABLE IF NOT EXISTS ${dest_db}.${dest_table}( time bigint ,journey_id varchar + ,segment_id varchar + ,activation_type varchar + ,activation_id varchar ,activation_step_id varchar ,syndication_id varchar ,activation_name varchar diff --git a/scenarios/cdp_campaign_management/queries/create_master_campaigns.sql b/scenarios/cdp_campaign_management/queries/create_master_campaigns.sql index feee2e53..0181daaf 100644 --- a/scenarios/cdp_campaign_management/queries/create_master_campaigns.sql +++ b/scenarios/cdp_campaign_management/queries/create_master_campaigns.sql @@ -1,7 +1,7 @@ DROP TABLE IF EXISTS ${dest_db}.${dest_table}; CREATE TABLE IF NOT EXISTS ${dest_db}.${dest_table} ( time bigint - ,activation_step_id varchar + ,activation_id varchar ,cv_name varchar ,utm_campaign varchar ,utm_medium varchar diff --git a/scenarios/cdp_campaign_management/queries/create_mta_conversion_journeys.sql b/scenarios/cdp_campaign_management/queries/create_mta_conversion_journeys.sql index 371d0a84..badb71e3 100644 --- a/scenarios/cdp_campaign_management/queries/create_mta_conversion_journeys.sql +++ b/scenarios/cdp_campaign_management/queries/create_mta_conversion_journeys.sql @@ -10,7 +10,7 @@ CREATE TABLE IF NOT EXISTS ${dest_db}.${dest_table} ( ,time_hour_from_activation double ,type varchar ,click_type varchar - ,activation_step_id varchar + ,activation_id varchar ,utm_source varchar ,utm_medium varchar ,utm_campaign varchar diff --git a/scenarios/cdp_campaign_management/queries/de_duplicate_master_campaings.sql b/scenarios/cdp_campaign_management/queries/de_duplicate_master_campaings.sql index e831c48f..b22d5cda 100644 --- a/scenarios/cdp_campaign_management/queries/de_duplicate_master_campaings.sql +++ b/scenarios/cdp_campaign_management/queries/de_duplicate_master_campaings.sql @@ -1,5 +1,5 @@ SELECT - activation_step_id + activation_id ,utm_campaign ,utm_medium ,utm_source diff --git a/scenarios/cdp_campaign_management/queries/ingest_activations.sql b/scenarios/cdp_campaign_management/queries/ingest_activations.sql index a713eb00..e02880c8 100644 --- a/scenarios/cdp_campaign_management/queries/ingest_activations.sql +++ b/scenarios/cdp_campaign_management/queries/ingest_activations.sql @@ -6,6 +6,10 @@ WITH tbl_base_activations AS ,cdp_customer_id ,t1.syndication_id ,COALESCE(type,'segment') AS activation_type + ,CASE type + WHEN 'journeyActivationStep' THEN activation_step_id + ELSE segment_id + END AS activation_id ,segment_id ,activation_step_id ,activation_name @@ -44,6 +48,7 @@ WITH tbl_base_activations AS SELECT time ,${user_id} + ,s1.activation_id ,s1.activation_step_id ,syndication_id ,segment_id @@ -61,7 +66,7 @@ SELECT FROM tbl_base_activations s1 LEFT OUTER JOIN ( SELECT - activation_step_id + activation_id ,MAX_BY(cv_name,time) AS cv_name ,MAX_BY(utm_campaign,time) AS utm_campaign ,MAX_BY(utm_medium,time) AS utm_medium @@ -72,7 +77,7 @@ LEFT OUTER JOIN ( FROM ( SELECT time - ,CAST(activation_step_id AS VARCHAR) AS activation_step_id + ,CAST(activation_id AS VARCHAR) AS activation_id ,cv_name ,utm_campaign ,utm_medium @@ -84,7 +89,7 @@ LEFT OUTER JOIN ( UNION ALL SELECT time - ,CAST(activation_step_id AS VARCHAR) AS activation_step_id + ,CAST(activation_id AS VARCHAR) AS activation_id ,cv_name ,utm_campaign ,utm_medium @@ -96,4 +101,4 @@ LEFT OUTER JOIN ( ) GROUP BY 1 ) s2 -ON s1.activation_step_id = s2.activation_step_id +ON s1.activation_id = s2.activation_id diff --git a/scenarios/cdp_campaign_management/queries/ingest_clicks.sql b/scenarios/cdp_campaign_management/queries/ingest_clicks.sql index 1b2df7a0..603678ff 100644 --- a/scenarios/cdp_campaign_management/queries/ingest_clicks.sql +++ b/scenarios/cdp_campaign_management/queries/ingest_clicks.sql @@ -3,7 +3,7 @@ SELECT ${distinct} ,db_name ,table_name ,${user_column} - ,COALESCE(CAST(t3.activation_step_id AS VARCHAR), CAST(t1.activation_step_id AS VARCHAR)) AS activation_step_id + ,COALESCE(CAST(t3.activation_id AS VARCHAR), CAST(t1.activation_id AS VARCHAR)) AS activation_id ,COALESCE(t3.cv_name, t1.cv_name) AS cv_name ,COALESCE(t3.utm_campaign, t1.utm_campaign) AS utm_campaign ,COALESCE(t3.utm_medium, t1.utm_medium) AS utm_medium @@ -18,7 +18,7 @@ FROM ,'${input_db}' AS db_name ,'${input_table}' AS table_name ,${user_column_inner} - ,url_extract_parameter(${url_column}, '${td.utm_names.utm_id}') as activation_step_id + ,url_extract_parameter(${url_column}, '${td.utm_names.utm_id}') as activation_id ,url_extract_parameter(${url_column}, '${td.utm_names.utm_campaign}') as utm_campaign ,url_extract_parameter(${url_column}, '${td.utm_names.utm_medium}') as utm_medium ,url_extract_parameter(${url_column}, '${td.utm_names.utm_source}') as utm_source @@ -40,7 +40,7 @@ LEFT OUTER JOIN ( utm_source ,utm_medium ,utm_campaign - ,MAX_BY(CAST(activation_step_id AS VARCHAR), time) AS activation_step_id + ,MAX_BY(CAST(activation_id AS VARCHAR), time) AS activation_id ,MAX_BY(cv_name, time) AS cv_name ,MAX_BY(utm_connector, time) AS utm_connector ,MAX_BY(utm_content,time) AS utm_content @@ -61,7 +61,7 @@ SELECT ${distinct} ,db_name ,table_name ,${user_column} - ,COALESCE(CAST(t3.activation_step_id AS VARCHAR), CAST(t1.activation_step_id AS VARCHAR)) AS activation_step_id + ,COALESCE(CAST(t3.activation_id AS VARCHAR), CAST(t1.activation_id AS VARCHAR)) AS activation_id ,COALESCE(t3.cv_name, t1.cv_name) AS cv_name ,COALESCE(t3.utm_campaign, t1.utm_campaign) AS utm_campaign ,COALESCE(t3.utm_medium, t1.utm_medium) AS utm_medium @@ -76,7 +76,7 @@ FROM ,'${input_db}' AS db_name ,'${input_table}' AS table_name ,${user_column_inner} - ,url_extract_parameter(${url_column}, '${td.utm_names.utm_id}') as activation_step_id + ,url_extract_parameter(${url_column}, '${td.utm_names.utm_id}') as activation_id ,url_extract_parameter(${url_column}, '${td.utm_names.utm_campaign}') as utm_campaign ,url_extract_parameter(${url_column}, '${td.utm_names.utm_medium}') as utm_medium ,url_extract_parameter(${url_column}, '${td.utm_names.utm_source}') as utm_source @@ -98,7 +98,7 @@ LEFT OUTER JOIN ( utm_source ,utm_medium ,utm_campaign - ,CAST(activation_step_id AS VARCHAR) AS activation_step_id + ,CAST(activation_id AS VARCHAR) AS activation_id ,MAX_BY(cv_name, time) AS cv_name ,MAX_BY(utm_connector, time) AS utm_connector ,MAX_BY(utm_content,time) AS utm_content @@ -111,5 +111,5 @@ ON ( t1.utm_campaign = t3.utm_campaign AND t1.utm_medium = t3.utm_medium AND t1.utm_source = t3.utm_source - AND t1.activation_step_id = t3.activation_step_id + AND t1.activation_id = t3.activation_id ) diff --git a/scenarios/cdp_campaign_management/queries/ingest_conversion_journeys.sql b/scenarios/cdp_campaign_management/queries/ingest_conversion_journeys.sql index 46e77bcb..7ea599ed 100644 --- a/scenarios/cdp_campaign_management/queries/ingest_conversion_journeys.sql +++ b/scenarios/cdp_campaign_management/queries/ingest_conversion_journeys.sql @@ -1,12 +1,12 @@ SELECT * - ,ROUND((time-LAG(time)OVER(PARTITION BY ${user_id}, activation_step_id ORDER BY time))/3600.0,1) AS time_hour_from_activation + ,ROUND((time-LAG(time)OVER(PARTITION BY ${user_id}, activation_id ORDER BY time))/3600.0,1) AS time_hour_from_activation FROM ( SELECT time ,'Activation' AS type ,${user_id} - ,CAST(activation_step_id AS VARCHAR) AS activation_step_id + ,CAST(activation_id AS VARCHAR) AS activation_id ,cv_name ,utm_campaign ,utm_medium @@ -26,7 +26,7 @@ FROM ( time ,'Click' AS type ,${user_id} - ,CAST(activation_step_id AS VARCHAR) AS activation_step_id + ,CAST(activation_id AS VARCHAR) AS activation_id ,cv_name ,utm_campaign ,utm_medium @@ -46,7 +46,7 @@ FROM ( time ,'Conversion' AS type ,${user_id} - ,NULL AS activation_step_id + ,NULL AS activation_id ,cv_name ,NULL AS utm_campaign ,NULL AS utm_medium diff --git a/scenarios/cdp_campaign_management/queries/ingest_daily_activations.sql b/scenarios/cdp_campaign_management/queries/ingest_daily_activations.sql index 4118f640..b6bfe050 100644 --- a/scenarios/cdp_campaign_management/queries/ingest_daily_activations.sql +++ b/scenarios/cdp_campaign_management/queries/ingest_daily_activations.sql @@ -1,7 +1,7 @@ SELECT - TD_DATE_TRUNC('day', time, '${user_timezone}') AS time - ,TD_TIME_STRING(time, 'd!', '${user_timezone}') AS date - ,activation_step_id + TD_DATE_TRUNC('day', time, '${td.timezone}') AS time + ,TD_TIME_STRING(time, 'd!', '${td.timezone}') AS date + ,activation_id ,cv_name ,utm_source ,utm_medium diff --git a/scenarios/cdp_campaign_management/queries/ingest_daily_activations_info.sql b/scenarios/cdp_campaign_management/queries/ingest_daily_activations_info.sql index 53ff2432..5256c27c 100644 --- a/scenarios/cdp_campaign_management/queries/ingest_daily_activations_info.sql +++ b/scenarios/cdp_campaign_management/queries/ingest_daily_activations_info.sql @@ -11,8 +11,8 @@ WITH tbl_jobs AS ,tbl_act_hst AS ( SELECT - TD_TIME_PARSE(JSON_EXTRACT_SCALAR(exe, '$.createdAt'), '${user_timezone}') AS time - ,TD_TIME_PARSE(JSON_EXTRACT_SCALAR(exe, '$.finishedAt'), '${user_timezone}') AS time_finished + TD_TIME_PARSE(JSON_EXTRACT_SCALAR(exe, '$.createdAt'), '${td.timezone}') AS time + ,TD_TIME_PARSE(JSON_EXTRACT_SCALAR(exe, '$.finishedAt'), '${td.timezone}') AS time_finished ,id AS syndication_id ,name AS activation_name ,JSON_EXTRACT_SCALAR(exe, '$.workflowId') AS workflow_id @@ -25,7 +25,7 @@ WITH tbl_jobs AS ${td.monitoring.db.cdp_monitoring}.${td.monitoring.tables.activations} CROSS JOIN UNNEST(CAST(JSON_PARSE(executions) AS ARRAY(JSON))) AS t(exe) WHERE ps_id = '${ps_id}' - AND TD_TIME_RANGE(TD_TIME_PARSE(JSON_EXTRACT_SCALAR(exe, '$.createdAt'), '${user_timezone}'), ${time_from}, ${time_to}) + AND TD_TIME_RANGE(TD_TIME_PARSE(JSON_EXTRACT_SCALAR(exe, '$.createdAt'), '${td.timezone}'), ${time_from}, ${time_to}) ) SELECT act_hst.* diff --git a/scenarios/cdp_campaign_management/queries/ingest_daily_clicks.sql b/scenarios/cdp_campaign_management/queries/ingest_daily_clicks.sql index e27e1fec..5420dfaf 100644 --- a/scenarios/cdp_campaign_management/queries/ingest_daily_clicks.sql +++ b/scenarios/cdp_campaign_management/queries/ingest_daily_clicks.sql @@ -1,7 +1,7 @@ SELECT - TD_DATE_TRUNC('day', time, '${user_timezone}') AS time - ,TD_TIME_STRING(time, 'd!', '${user_timezone}') AS date - ,activation_step_id + TD_DATE_TRUNC('day', time, '${td.timezone}') AS time + ,TD_TIME_STRING(time, 'd!', '${td.timezone}') AS date + ,activation_id ,cv_name ,utm_source ,utm_medium diff --git a/scenarios/cdp_campaign_management/queries/ingest_daily_conversions.sql b/scenarios/cdp_campaign_management/queries/ingest_daily_conversions.sql index 812dcd63..ad7af9ea 100644 --- a/scenarios/cdp_campaign_management/queries/ingest_daily_conversions.sql +++ b/scenarios/cdp_campaign_management/queries/ingest_daily_conversions.sql @@ -1,6 +1,6 @@ SELECT - TD_DATE_TRUNC('day', time, '${user_timezone}') AS time - ,TD_TIME_STRING(time, 'd!', '${user_timezone}') AS date + TD_DATE_TRUNC('day', time, '${td.timezone}') AS time + ,TD_TIME_STRING(time, 'd!', '${td.timezone}') AS date ,cv_name ,COUNT(1) AS cnt ,SUM(val) AS val diff --git a/scenarios/cdp_campaign_management/queries/ingest_daily_mta.sql b/scenarios/cdp_campaign_management/queries/ingest_daily_mta.sql index 92359bac..354b6be5 100644 --- a/scenarios/cdp_campaign_management/queries/ingest_daily_mta.sql +++ b/scenarios/cdp_campaign_management/queries/ingest_daily_mta.sql @@ -4,9 +4,9 @@ FROM SELECT COALESCE(act.time,clk.time,mta.time) AS time ,COALESCE(act.date,clk.date,mta.date) AS date - ,COALESCE(act.activation_step_id,clk.activation_step_id,mta.activation_step_id) AS activation_step_id + ,COALESCE(act.activation_id,clk.activation_id,mta.activation_id) AS activation_id ,CASE - WHEN COALESCE(act.activation_step_id,clk.activation_step_id,mta.activation_step_id) IN (SELECT activation_step_id FROM ${td.tables.master_activations}) THEN 'internal' + WHEN COALESCE(act.activation_id,clk.activation_id,mta.activation_id) IN (SELECT activation_id FROM ${td.tables.master_activations}) THEN 'internal' WHEN mta.type='Conversion' THEN 'internal' ELSE 'external' END AS is_internal_campaign_click @@ -32,12 +32,12 @@ FROM ,COALESCE(cnt_cv_id,0) AS cnt_cv_id FROM ${td.tables.daily_activations} act FULL OUTER JOIN ${td.tables.daily_clicks} clk - ON act.time = clk.time AND act.activation_step_id = clk.activation_step_id + ON act.time = clk.time AND act.activation_id = clk.activation_id FULL OUTER JOIN ( SELECT - TD_DATE_TRUNC('day',time, '${user_timezone}') AS time - ,TD_TIME_STRING(time, 'd!', '${user_timezone}') AS date - ,activation_step_id + TD_DATE_TRUNC('day',time, '${td.timezone}') AS time + ,TD_TIME_STRING(time, 'd!', '${td.timezone}') AS date + ,activation_id ,type ,utm_source ,utm_medium @@ -58,10 +58,10 @@ FROM ,COUNT(DISTINCT cv_id) AS cnt_cv_id FROM ${td.tables.mta_conversion_journeys} GROUP BY 1,2,3,4,5,6,7,8,9,10 - ORDER BY date, activation_step_id + ORDER BY date, activation_id ) mta - ON clk.time = mta.time AND clk.activation_step_id = mta.activation_step_id + ON clk.time = mta.time AND clk.activation_id = mta.activation_id ) WHERE TD_TIME_RANGE(time, ${time_from}, ${time_to}) -- AND type <> 'Conversion' --- ORDER BY cv_name, time DESC, activation_step_id \ No newline at end of file +-- ORDER BY cv_name, time DESC, activation_id \ No newline at end of file diff --git a/scenarios/cdp_campaign_management/queries/ingest_daily_mta_conversion_journeys.sql b/scenarios/cdp_campaign_management/queries/ingest_daily_mta_conversion_journeys.sql index c39c73d1..6e6c57ea 100644 --- a/scenarios/cdp_campaign_management/queries/ingest_daily_mta_conversion_journeys.sql +++ b/scenarios/cdp_campaign_management/queries/ingest_daily_mta_conversion_journeys.sql @@ -4,9 +4,9 @@ FROM SELECT COALESCE(act.time,clk.time,mta.time) AS time ,COALESCE(act.date,clk.date,mta.date) AS date - ,COALESCE(act.activation_step_id,clk.activation_step_id,mta.activation_step_id) AS activation_step_id + ,COALESCE(act.activation_id,clk.activation_id,mta.activation_id) AS activation_id ,CASE - WHEN COALESCE(act.activation_step_id,clk.activation_step_id,mta.activation_step_id) IN (SELECT activation_step_id FROM ${td.tables.master_activations}) THEN 'internal' + WHEN COALESCE(act.activation_id,clk.activation_id,mta.activation_id) IN (SELECT activation_id FROM ${td.tables.master_activations}) THEN 'internal' WHEN mta.type='Conversion' THEN 'internal' ELSE 'external' END AS is_internal_campaign_click @@ -32,12 +32,12 @@ FROM ,COALESCE(cnt_cv_id,0) AS cnt_cv_id FROM ${td.tables.daily_activations} act FULL OUTER JOIN ${td.tables.daily_clicks} clk - ON act.time = clk.time AND act.activation_step_id = clk.activation_step_id + ON act.time = clk.time AND act.activation_id = clk.activation_id FULL OUTER JOIN ( SELECT - TD_DATE_TRUNC('day',time, '${user_timezone}') AS time - ,TD_TIME_STRING(time, 'd!', '${user_timezone}') AS date - ,activation_step_id + TD_DATE_TRUNC('day',time, '${td.timezone}') AS time + ,TD_TIME_STRING(time, 'd!', '${td.timezone}') AS date + ,activation_id ,type ,utm_source ,utm_medium @@ -58,9 +58,9 @@ FROM ,COUNT(DISTINCT cv_id) AS cnt_cv_id FROM ${td.tables.mta_conversion_journeys} GROUP BY 1,2,3,4,5,6,7,8,9,10 - ORDER BY date, activation_step_id + ORDER BY date, activation_id ) mta - ON clk.time = mta.time AND clk.activation_step_id = mta.activation_step_id + ON clk.time = mta.time AND clk.activation_id = mta.activation_id ) WHERE TD_TIME_RANGE(time, ${time_from}, ${time_to}) --- ORDER BY cv_name, time DESC, activation_step_id \ No newline at end of file +-- ORDER BY cv_name, time DESC, activation_id \ No newline at end of file diff --git a/scenarios/cdp_campaign_management/queries/ingest_existing_campaigns.sql b/scenarios/cdp_campaign_management/queries/ingest_existing_campaigns.sql index 51fa033f..8346d106 100644 --- a/scenarios/cdp_campaign_management/queries/ingest_existing_campaigns.sql +++ b/scenarios/cdp_campaign_management/queries/ingest_existing_campaigns.sql @@ -1,8 +1,8 @@ SELECT IF( - COALESCE(activation_step_id IN ( + COALESCE(activation_id IN ( SELECT DISTINCT - activation_step_id + activation_id FROM daily_activations ),false) ,1,0) AS exists_in_daily_activations @@ -10,7 +10,7 @@ SELECT ,utm_medium ,utm_campaign ,cv_name - ,activation_step_id + ,activation_id ,MAX_BY(utm_content,time) AS utm_content ,MAX_BY(utm_connector,time) AS utm_connector ,MAX_BY(utm_term,time) AS utm_term diff --git a/scenarios/cdp_campaign_management/queries/ingest_journeys.sql b/scenarios/cdp_campaign_management/queries/ingest_journeys.sql index 51daab9e..2ffc67f9 100644 --- a/scenarios/cdp_campaign_management/queries/ingest_journeys.sql +++ b/scenarios/cdp_campaign_management/queries/ingest_journeys.sql @@ -1,5 +1,5 @@ SELECT - TD_TIME_PARSE(JSON_EXTRACT_SCALAR(JSON_PARSE(attributes), '$.createdAt'),'${user_timezone}') AS time + TD_TIME_PARSE(JSON_EXTRACT_SCALAR(JSON_PARSE(attributes), '$.createdAt'),'${td.timezone}') AS time ,JSON_EXTRACT_SCALAR(JSON_PARSE(attributes), '$.audienceId') AS audience_id ,id ,JSON_EXTRACT_SCALAR(JSON_PARSE(attributes), '$.name') AS name @@ -14,7 +14,7 @@ FROM ${td.monitoring.db.cdp_monitoring}.${td.monitoring.tables.entities} WHERE type = 'journey' AND JSON_EXTRACT_SCALAR(JSON_PARSE(attributes), '$.audienceId') = '${ps_id}' AND TD_TIME_RANGE( - TD_TIME_PARSE(JSON_EXTRACT_SCALAR(JSON_PARSE(attributes), '$.createdAt'),'${user_timezone}') + TD_TIME_PARSE(JSON_EXTRACT_SCALAR(JSON_PARSE(attributes), '$.createdAt'),'${td.timezone}') ,${time_from} ,${time_to} ) \ No newline at end of file diff --git a/scenarios/cdp_campaign_management/queries/ingest_master_activations.sql b/scenarios/cdp_campaign_management/queries/ingest_master_activations.sql index c9f7bb24..308ab6cc 100644 --- a/scenarios/cdp_campaign_management/queries/ingest_master_activations.sql +++ b/scenarios/cdp_campaign_management/queries/ingest_master_activations.sql @@ -1,37 +1,54 @@ WITH tbl_act AS ( SELECT - journey_id - ,id AS activation_step_id - ,JSON_EXTRACT_SCALAR(JSON_PARSE(attributes), '$.syndicationId') AS syndication_id - ,JSON_EXTRACT_SCALAR(JSON_PARSE(attributes), '$.activationParams.name') AS activation_name - ,JSON_EXTRACT_SCALAR(JSON_PARSE(attributes), '$.activationParams.scheduleType') AS schedule_type - ,JSON_EXTRACT_SCALAR(JSON_PARSE(attributes), '$.activationParams.scheduleOption') AS schedule_option - ,JSON_EXTRACT_SCALAR(JSON_PARSE(attributes), '$.activationParams.timezone') AS timezone - ,JSON_EXTRACT_SCALAR(JSON_PARSE(attributes), '$.activationParams.connectionId') AS connection_id - ,JSON_EXTRACT_SCALAR(JSON_PARSE(attributes), '$.activationParams.allColumns') AS all_columns - FROM ${td.monitoring.db.cdp_monitoring}.${td.monitoring.tables.journey_activation} + TD_TIME_PARSE(createdat,'${td.timezone}') AS time + ,audienceid AS ps_id + ,IF(j_act.syndication_id IS NULL, segmentid, activation_step_id ) AS activation_id + ,segmentid AS segment_id + ,name AS activation_name + ,id AS syndication_id + ,activation_step_id + ,IF(j_act.syndication_id IS NULL, 'segment', 'journeyActivationStep' ) AS activation_type + ,scheduletype AS schedule_type + ,scheduleoption AS schedule_option + ,timezone + ,createdat + ,updatedat + ,journey_id + ,connectionid AS connection_id + FROM ${td.monitoring.db.cdp_monitoring}.${td.monitoring.tables.activations} act + LEFT OUTER JOIN + ( + SELECT + id AS activation_step_id + ,JSON_EXTRACT_SCALAR(JSON_PARSE(attributes),'$.syndicationId') AS syndication_id + ,journey_id + FROM ${td.monitoring.db.cdp_monitoring}.${td.monitoring.tables.journey_activation} + ) j_act + ON act.id = j_act.syndication_id + WHERE ps_id = '${ps_id}' + AND TD_TIME_RANGE(TD_TIME_PARSE(createdat,'${td.timezone}'), ${time_from}, ${time_to}) ) , tbl_step_ids AS( SELECT - time - ,REPLACE(step_id, '-', '_') AS step_id + -- time + REPLACE(step_id, '-', '_') AS step_id ,JSON_EXTRACT_SCALAR(jsn, '$.journeyActivationStepId') AS activation_step_id ,JSON_EXTRACT_SCALAR(jsn, '$.type') AS type ,stage_name ,stage_no - ,ps_id + -- ,ps_id ,state - ,created_at - ,updated_at + -- ,created_at + -- ,updated_at ,journey_name FROM ( SELECT elm - ,idx-1 AS stage_no + ,CAST(idx-1 AS INTEGER) AS stage_no ,TD_TIME_PARSE( JSON_EXTRACT_SCALAR(JSON_PARSE(attributes), '$.createdAt') - ,'${user_timezone}' + ,'${td.timezone}' ) AS time ,JSON_EXTRACT_SCALAR(elm, '$.name') AS stage_name ,JSON_EXTRACT_SCALAR(JSON_PARSE(attributes), '$.audienceId') AS ps_id @@ -61,22 +78,102 @@ WITH tbl_act AS ( ON conn_d.name = conn.name ) - SELECT - step_ids.time - ,act.* + act.* + ,conn.connector_type ,step_ids.step_id ,step_ids.stage_name ,step_ids.stage_no ,step_ids.state - ,step_ids.created_at - ,step_ids.updated_at + -- ,step_ids.created_at + -- ,step_ids.updated_at ,step_ids.journey_name - ,conn.connector_type - FROM tbl_act act -LEFT OUTER JOIN tbl_step_ids step_ids -ON act.activation_step_id = step_ids.activation_step_id -AND step_ids.ps_id = '${ps_id}' +FROM tbl_act act LEFT OUTER JOIN tbl_conn conn ON CAST(act.connection_id AS VARCHAR) = CAST(conn.connection_id AS VARCHAR) -WHERE TD_TIME_RANGE(step_ids.time, ${time_from}, ${time_to}) +LEFT OUTER JOIN tbl_step_ids step_ids +ON act.activation_step_id = step_ids.activation_step_id + + +-- WITH tbl_act AS ( +-- SELECT +-- journey_id +-- ,id AS activation_step_id +-- ,JSON_EXTRACT_SCALAR(JSON_PARSE(attributes), '$.syndicationId') AS syndication_id +-- ,JSON_EXTRACT_SCALAR(JSON_PARSE(attributes), '$.activationParams.name') AS activation_name +-- ,JSON_EXTRACT_SCALAR(JSON_PARSE(attributes), '$.activationParams.scheduleType') AS schedule_type +-- ,JSON_EXTRACT_SCALAR(JSON_PARSE(attributes), '$.activationParams.scheduleOption') AS schedule_option +-- ,JSON_EXTRACT_SCALAR(JSON_PARSE(attributes), '$.activationParams.timezone') AS timezone +-- ,JSON_EXTRACT_SCALAR(JSON_PARSE(attributes), '$.activationParams.connectionId') AS connection_id +-- ,JSON_EXTRACT_SCALAR(JSON_PARSE(attributes), '$.activationParams.allColumns') AS all_columns +-- FROM ${td.monitoring.db.cdp_monitoring}.${td.monitoring.tables.journey_activation} +-- ) +-- , tbl_step_ids AS( +-- SELECT +-- time +-- ,REPLACE(step_id, '-', '_') AS step_id +-- ,JSON_EXTRACT_SCALAR(jsn, '$.journeyActivationStepId') AS activation_step_id +-- ,JSON_EXTRACT_SCALAR(jsn, '$.type') AS type +-- ,stage_name +-- ,stage_no +-- ,ps_id +-- ,state +-- ,created_at +-- ,updated_at +-- ,journey_name +-- FROM +-- ( +-- SELECT +-- elm +-- ,idx-1 AS stage_no +-- ,TD_TIME_PARSE( +-- JSON_EXTRACT_SCALAR(JSON_PARSE(attributes), '$.createdAt') +-- ,'${td.timezone}' +-- ) AS time +-- ,JSON_EXTRACT_SCALAR(elm, '$.name') AS stage_name +-- ,JSON_EXTRACT_SCALAR(JSON_PARSE(attributes), '$.audienceId') AS ps_id +-- ,JSON_EXTRACT_SCALAR(JSON_PARSE(attributes), '$.state') AS state +-- ,JSON_EXTRACT_SCALAR(JSON_PARSE(attributes), '$.createdAt') AS created_at +-- ,JSON_EXTRACT_SCALAR(JSON_PARSE(attributes), '$.updatedAt') AS updated_at +-- ,JSON_EXTRACT_SCALAR(JSON_PARSE(attributes), '$.name') AS journey_name +-- FROM ${td.monitoring.db.cdp_monitoring}.${td.monitoring.tables.journey_summary} +-- CROSS JOIN UNNEST(CAST(JSON_EXTRACT(JSON_PARSE(attributes), '$.journeyStages') AS ARRAY(JSON))) WITH ORDINALITY AS t(elm, idx) +-- ) +-- CROSS JOIN UNNEST( +-- MAP_KEYS(CAST(JSON_EXTRACT(elm, '$.steps') AS MAP(VARCHAR, JSON))) +-- , MAP_VALUES(CAST(JSON_EXTRACT(elm, '$.steps') AS MAP(VARCHAR, JSON))) +-- ) AS t(step_id, jsn) +-- WHERE JSON_EXTRACT_SCALAR(jsn, '$.type') = 'Activation' +-- ) +-- ,tbl_conn AS ( +-- SELECT +-- id AS connection_id +-- ,IF( +-- STARTS_WITH(conn.url,'{') +-- ,JSON_EXTRACT_SCALAR(JSON_PARSE(conn.url), '$.type') +-- ,SPLIT_PART(conn.url, '://', 1) +-- ) AS connector_type +-- FROM ${td.monitoring.db.basic_monitoring}.${td.monitoring.tables.connections_details} conn_d +-- JOIN ${td.monitoring.db.basic_monitoring}.${td.monitoring.tables.connections} conn +-- ON conn_d.name = conn.name +-- ) + + +-- SELECT +-- step_ids.time +-- ,act.* +-- ,step_ids.step_id +-- ,step_ids.stage_name +-- ,step_ids.stage_no +-- ,step_ids.state +-- ,step_ids.created_at +-- ,step_ids.updated_at +-- ,step_ids.journey_name +-- ,conn.connector_type +-- FROM tbl_act act +-- LEFT OUTER JOIN tbl_step_ids step_ids +-- ON act.activation_step_id = step_ids.activation_step_id +-- AND step_ids.ps_id = '${ps_id}' +-- LEFT OUTER JOIN tbl_conn conn +-- ON CAST(act.connection_id AS VARCHAR) = CAST(conn.connection_id AS VARCHAR) +-- WHERE TD_TIME_RANGE(step_ids.time, ${time_from}, ${time_to}) diff --git a/scenarios/cdp_campaign_management/queries/ingest_mta_conversion_journeys.sql b/scenarios/cdp_campaign_management/queries/ingest_mta_conversion_journeys.sql index 72c58d5f..30bbfee7 100644 --- a/scenarios/cdp_campaign_management/queries/ingest_mta_conversion_journeys.sql +++ b/scenarios/cdp_campaign_management/queries/ingest_mta_conversion_journeys.sql @@ -11,7 +11,7 @@ WITH tbl_cv_history AS ( SELECT cv_id - ,ROW_NUMBER()OVER(PARTITION BY raw_data.cv_name, raw_data.${user_id}, raw_data.time, activation_step_id, type ORDER BY cv_id) AS cv_order + ,ROW_NUMBER()OVER(PARTITION BY raw_data.cv_name, raw_data.${user_id}, raw_data.time, activation_id, type ORDER BY cv_id) AS cv_order ,cv_history.time AS cv_time ,(cv_history.time - raw_data.time)/3600 AS time_hour_to_cv , raw_data.* @@ -21,7 +21,7 @@ WITH tbl_cv_history AS AND raw_data.cv_name = cv_history.cv_name WHERE raw_data.time <= cv_history.time AND raw_data.${user_id} <= cv_history.${user_id} - AND type <> 'Activation' + AND type <> '${valuation_target=="activation" ? "Click" : "Activation"}' ) WHERE cv_order = 1 ) @@ -55,7 +55,7 @@ WITH tbl_cv_history AS END AS click_type ,type ,${user_id} - ,activation_step_id + ,activation_id ,utm_source ,utm_medium ,utm_campaign @@ -80,7 +80,7 @@ SELECT ,time_hour_from_activation ,type ,click_type - ,activation_step_id + ,activation_id ,utm_source ,utm_medium ,utm_campaign diff --git a/scenarios/cdp_campaign_management/write_to_gsheet.dig b/scenarios/cdp_campaign_management/write_to_gsheet.dig index febbba0f..59ba4be1 100644 --- a/scenarios/cdp_campaign_management/write_to_gsheet.dig +++ b/scenarios/cdp_campaign_management/write_to_gsheet.dig @@ -17,7 +17,7 @@ _parallel: true +export_master_activations: td>: - query: SELECT * FROM ${td.tables.master_activations} ORDER BY journey_id,activation_step_id + query: SELECT * FROM ${td.tables.master_activations} ORDER BY journey_id,activation_id result_connection: ${gsheet.result_connection} result_settings: spreadsheet_folder: ${gsheet.sheet_folder} @@ -59,7 +59,7 @@ +export_mta: td>: - query: SELECT TD_TIME_STRING(time,'d!','${user_timezone}') AS date,* FROM ${td.tables.mta_conversion_journeys} ORDER BY cv_name, ${user_id}, cv_id, position LIMIT 1000 + query: SELECT TD_TIME_STRING(time,'d!','${td.timezone}') AS date,* FROM ${td.tables.mta_conversion_journeys} ORDER BY cv_name, ${user_id}, cv_id, position LIMIT 1000 result_connection: ${gsheet.result_connection} result_settings: spreadsheet_folder: ${gsheet.sheet_folder} @@ -73,7 +73,7 @@ +export_daily_mta: td>: - query: SELECT * FROM ${td.tables.daily_mta_conversion_journeys} ORDER BY cv_name, time DESC, activation_step_id + query: SELECT * FROM ${td.tables.daily_mta_conversion_journeys} ORDER BY cv_name, time DESC, activation_id result_connection: ${gsheet.result_connection} result_settings: spreadsheet_folder: ${gsheet.sheet_folder}