Skip to main content

easyfabric.loaders.merge

logging​

reduce​

DeltaTable​

DataFrame​

abs​

coalesce​

col​

concat_ws​

expr​

lit​

max​

row_number​

LongType​

StringType​

Window​

MSG_MERGE_COLUMNS_TO_EXCLUDE​

MSG_MERGE_COLUMNS_TO_UPDATE​

MSG_MERGE_CONDITION​

MSG_MERGE_DELETE_FILTER_INFO​

MSG_MERGE_DELETE_TABLE​

MSG_MERGE_DIRECT_LOAD​

MSG_MERGE_EMPTY_SOURCE​

MSG_MERGE_EXECUTE​

MSG_MERGE_GET_TABLE​

MSG_MERGE_JOIN_STRATEGY​

MSG_MERGE_MID_SILVER_NOTEBOOK​

MSG_MERGE_ROW_COUNT​

MSG_MERGE_SILVERQUERY_SIZE​

MSG_MERGE_SOURCE_DEST_LAKEHOUSE​

MSG_MERGE_SOURCE_ROWS​

MSG_MERGE_TIMESTAMP_STATUS​

MSG_MERGE_WRITE_HISTORY​

MSG_MERGE_WRITE_SILVER​

ConfigManager​

initialize_config​

LoadConfig​

TableConfig​

build_join_condition​

build_join_delcondition​

build_update_condition​

build_update_set​

check_for_duplicate_keys_dataframe​

get_df_size​

get_spark​

remove_items_from_list​

segment_record​

segment_unchanged​

union_with_schema_alignment​

spark_operation_with_retries​

spark​

source_alias​

target_alias​

full_load_silver​

def full_load_silver(table_config: TableConfig,
                     config_manager: ConfigManager = None,
                     columns_to_exclude: list[str] = None)

Fully replace silver his table with source data from the bronze lakehouse into the silver lakehouse.

Arguments:

  • table_config TableConfig - Configuration for the table to be merged.
  • config_manager ConfigManager - Configuration manager instance.
  • columns_to_exclude list[str], optional - Columns to exclude from the merge. Defaults to None.

load_dataframe_silver​

def load_dataframe_silver(df_input: DataFrame,
                          table_config: TableConfig,
                          config_manager: ConfigManager = None)

merge_source_into_target​

def merge_source_into_target(table_config: TableConfig,
                             config_manager: ConfigManager = None,
                             columns_to_exclude: list[str] = None)

Merges source data from the bronze lakehouse into the silver lakehouse.

Arguments:

  • table_config TableConfig - Configuration for the table to be merged.
  • config_manager ConfigManager - Configuration manager instance.
  • columns_to_exclude list[str], optional - Columns to exclude from the merge. Defaults to None.

bronze_into_silver​

def bronze_into_silver(table_config: TableConfig,
                       config_manager: ConfigManager = None,
                       columns_to_exclude: list[str] = None)

Merges source data from the bronze lakehouse into the silver lakehouse.

Arguments:

  • table_config TableConfig - Configuration for the table to be merged.
  • config_manager ConfigManager - Configuration manager instance.
  • columns_to_exclude list[str], optional - Columns to exclude from the merge. Defaults to None.

load_silver_table_from_bronze​

def load_silver_table_from_bronze(table_config: TableConfig,
                                  config_manager: ConfigManager = None,
                                  columns_to_exclude: list[str] = None)

Loads the silver table directly from the bronze lakehouse without merging.

Arguments:

  • table_config TableConfig - Configuration for the table to be loaded.
  • config_manager ConfigManager - Configuration manager instance.
  • columns_to_exclude list[str], optional - Columns to exclude from the load. Defaults to None.

merge_loadconfig_into_target​

def merge_loadconfig_into_target(df_source: DataFrame,
                                 load_config: LoadConfig,
                                 table_config: TableConfig,
                                 config_manager: ConfigManager = None)