custom_transformations.py
# Databricks notebook source
from pyspark.sql import DataFrame
secret_name = "jdbc-dynamicscrm-secret"
additional_secret_names = "empower-d365-nonprod-$$-connectionstring-part"
num_of_dynamics_clients = 0
ddu_log_db = "empower_ddu"
def transform_account() -> DataFrame:
key_columns = "id"
mode = "upsert"
query = """
select 'c9381589-d31c-4efe-8978-8d3449eb1f0f' as id
,'John Doe Jr' as name
"""
df = spark.sql(query)
return df, key_columns, mode
def transform_contact() -> DataFrame:
key_columns = "id"
mode = "upsert"
query = """
select '916fa482-6e09-ef11-9f89-000d3a1b9638' as id
,'Doe' as LastName
"""
df = spark.sql(query)
return df, key_columns, mode
def transform_cr34c_ddu_customer() -> DataFrame:
key_columns = "id"
mode = "upsert"
query = """
SELECT
rowguid as id,
customerid as cr34c_customerid,
firstname as cr34c_firstname,
CONCAT(
LEFT(emailaddress, 1),
REPEAT('*', INSTR(emailaddress, '@') - 2),
SUBSTR(emailaddress, INSTR(emailaddress, '@'))
) as cr34c_emailaddress
FROM
emp_demo_dv.demo_aw_01_silver.customer
"""
df = spark.sql(query)
return df, key_columns, mode
Updated 3 months ago