Modern Data Lakehouse Architecture: Medallion Design Pattern

from pyspark.sql import functions as F

def transform_bronze_to_silver(bronze_df):
    """
    Cleanses raw JSON events from Bronze and writes to Silver:
    1. Extracts and casts typed fields
    2. Drops exact and partial event duplicates
    3. Enforces non-null primary keys
    """
    return (
        bronze_df
        .filter(F.col("event_payload").isNotNull())
        .select(
            F.col("event_id").cast("string").alias("event_id"),
            F.col("timestamp").cast("timestamp").alias("event_time"),
            F.from_json("event_payload", schema).alias("payload")
        )
        .select("event_id", "event_time", "payload.*")
        .dropDuplicates(["event_id"])
        .filter(F.col("event_id").isNotNull())
    )