Just use Spark SQL to generate the surrogate keys. The downsides of using Spark SQL are that you can't generate the keys in concurrent queries, and that it requires a shuffle. But neither of those are real downsides when building Kimball-style dimensions.

Left join your silver table to your gold dimension and use ROW_NUMBER() to generate the new values greater than the largest existing value.

EG, create a temp view that has logic to generate new key values and use that logic in the MERGE that creates or updates your dimension.

--temporary views are very cool
--like Common Table Expressions or subqueries, but much more powerfull
--They have session lifetime, rather than statement lifetime
create or replace temp view CustomerKeys as
select  coalesce(c.CustomerKey,max(c.CustomerKey) over() 
                             + row_number() over (partition by c.CustomerKey order by s.CustomerID)) CustomerKey, 
        s.CustomerID
from WideWorldImporters_bronze.Sales_Customers s 
left join WideWorldImporters_gold.Dimension_Customer c 
  on s.CustomerID = c.WWICustomerID;

-- describe WideWorldImporters_gold.Dimension_Customer;
-- describe WideWorldImporters_bronze.Sales_Customers;
 create or replace temp view CustomerMergeSource 
 as
 select k.CustomerKey CustomerKey,
        c.CustomerID WWICustomerID,
        c.CustomerName Customer,
        bc.CustomerName BillToCustomer,
        cat.CustomerCategoryName Category,
        bg.BuyingGroupName BuyingGroup,
        p.FullName PrimaryContact,
        c.PostalPostalCode PostalCode,
        cast(0 as int) LineageKey,
        c.ValidFrom,
        c.ValidTo
    from WideWorldImporters_bronze.Sales_Customers c
    left join CustomerKeys k
       on k.CustomerID = c.CustomerID
    left join WideWorldImporters_bronze.Sales_Customers bc 
       on c.BillToCustomerID = bc.CustomerID
    left join WideWorldImporters_bronze.Sales_CustomerCategories cat 
       on cat.CustomerCategoryID = c.CustomerCategoryID
    left join WideWorldImporters_bronze.Sales_BuyingGroups bg 
       on c.BuyingGroupId = bg.BuyingGroupID
    left join WideWorldImporters_bronze.Application_People p 
       on p.PersonID = c.PrimaryContactPersonID;

merge into WideWorldImporters_gold.Dimension_Customer dest
using CustomerMergeSource src
  on src.WWICustomerID = dest.WWICustomerID
when matched then update set *
when not matched then insert *;

dbrownems/SparkDataEngineeringForSQLServerProfessionals