custom_transformations.py

# Databricks notebook source
from pyspark.sql import DataFrame
secret_name = "jdbc-dynamicscrm-secret"
additional_secret_names = "empower-d365-nonprod-$$-connectionstring-part"
num_of_dynamics_clients = 0

ddu_log_db = "empower_ddu"

def transform_account() -> DataFrame:
    key_columns = "id"
    mode = "upsert"
    query = """
    select 'c9381589-d31c-4efe-8978-8d3449eb1f0f' as id
          ,'John Doe Jr' as name
    """

    df = spark.sql(query)
    return df, key_columns, mode

def transform_contact() -> DataFrame:
    key_columns = "id"
    mode = "upsert"
    query = """
    select '916fa482-6e09-ef11-9f89-000d3a1b9638' as id
          ,'Doe' as LastName
    """

    df = spark.sql(query)
    return df, key_columns, mode

def transform_cr34c_ddu_customer() -> DataFrame:
    key_columns = "id"
    mode = "upsert"
    query = """
SELECT
    rowguid as id,
    customerid as cr34c_customerid,
    firstname as cr34c_firstname,
    CONCAT(
        LEFT(emailaddress, 1),
        REPEAT('*', INSTR(emailaddress, '@') - 2),
        SUBSTR(emailaddress, INSTR(emailaddress, '@'))
    ) as cr34c_emailaddress
FROM
    emp_demo_dv.demo_aw_01_silver.customer
        """

    df = spark.sql(query)
    return df, key_columns, mode