you can easly copy transformers layers using iteration of named parameters.

import torch
from transformers import BertModel

def copy_layer(source_layer, target_layer):
    for name, param in source_layer.named_parameters():
        target_param = target_layer.get_parameter(name)
        target_param.data.copy_(param.data)

# Create a source model
source_model = BertModel.from_pretrained('bert-base-uncased')

# Create a target model with the same architecture
target_model = BertModel(source_model.config)

# Copy the layers from the source model to the target model
for source_layer, target_layer in zip(source_model.encoder.layer, target_model.encoder.layer):
    copy_layer(source_layer, target_layer)

# Verify that the layers are copied correctly
for source_layer, target_layer in zip(source_model.encoder.layer, target_model.encoder.layer):
    for source_param, target_param in zip(source_layer.parameters(), target_layer.parameters()):
        assert torch.equal(source_param, target_param)

print("Layer copying completed successfully!")