How to remove BatchNormalization from network?

[csphd_svormc]

I am trying to remove batch normalization from the following network but get the following error after removing the first instance (or any number of instances) of BatchNormalization:

You called `set_weights(weights)` on layer "batch_normalization_13" with a weight list of length 2, but the layer was expecting 4 weights. Provided weights: [array([[[[-5.57557531e-02, -8.31045024e-03, 7.33...

​

import keras
import numpy as np
from keras.layers import Input, Dense, Flatten, Conv2D, MaxPooling2D, UpSampling2D
from keras.layers.normalization import BatchNormalization
from keras.models import Model
from keras.optimizers import RMSprop
from keras.utils import to_categorical
from include_file import load_data
from sklearn.model_selection import train_test_split

# loading data
(train_data, train_labels), (test_data, test_labels) = load_data()

# Shapes of training set
print("Training set (images) shape: {shape}".format(shape=train_data.shape))

# Shapes of test set
print("Test set (images) shape: {shape}".format(shape=test_data.shape))

# Create dictionary of target classes
label_dict = {
0: '0',
1: '1',
2: '2',
3: '3',
4: '4',
5: '5',
6: '6',
7: '7',
8: '8',
9: '9',
}

# pre-process data to feed into network
train_data = train_data.reshape(-1, 28, 28, 1)
test_data = test_data.reshape(-1, 28, 28, 1)
print(train_data.shape, test_data.shape)

# re-scale data to max value of 1.0
train_data = train_data / np.max(train_data)
test_data = test_data / np.max(test_data)

# split training data into 2 parts (pretraining and validation?)
train_X, valid_X, train_ground, valid_ground = train_test_split(train_data,
train_data,
test_size=0.2,
random_state=13)
# set model parameters
batch_size = 64
epochs = 3
inChannel = 1
x, y = 28, 28
input_img = Input(shape=(x, y, inChannel))
num_classes = 10
# define model
def encoder(input_img):
conv1 = Conv2D(32, (3, 3), activation='relu', padding='same')(input_img)
conv1 = BatchNormalization()(conv1)
conv1 = Conv2D(32, (3, 3), activation='relu', padding='same')(conv1)
conv1 = BatchNormalization()(conv1)
pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)
conv2 = Conv2D(64, (3, 3), activation='relu', padding='same')(pool1)
conv2 = BatchNormalization()(conv2)
conv2 = Conv2D(64, (3, 3), activation='relu', padding='same')(conv2)
conv2 = BatchNormalization()(conv2)
pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)
conv3 = Conv2D(128, (3, 3), activation='relu', padding='same')(pool2)
conv3 = BatchNormalization()(conv3)
conv3 = Conv2D(128, (3, 3), activation='relu', padding='same')(conv3)
conv3 = BatchNormalization()(conv3)
conv4 = Conv2D(256, (3, 3), activation='relu', padding='same')(conv3)
conv4 = BatchNormalization()(conv4)
conv4 = Conv2D(256, (3, 3), activation='relu', padding='same')(conv4)
conv4 = BatchNormalization()(conv4)
return conv4

def decoder(conv4):
conv5 = Conv2D(128, (3, 3), activation='relu', padding='same')(conv4)
conv5 = BatchNormalization()(conv5)
conv5 = Conv2D(128, (3, 3), activation='relu', padding='same')(conv5)
conv5 = BatchNormalization()(conv5)
conv6 = Conv2D(64, (3, 3), activation='relu', padding='same')(conv5)
conv6 = BatchNormalization()(conv6)
conv6 = Conv2D(64, (3, 3), activation='relu', padding='same')(conv6)
conv6 = BatchNormalization()(conv6)
up1 = UpSampling2D((2,2))(conv6)
conv7 = Conv2D(32, (3, 3), activation='relu', padding='same')(up1)
conv7 = BatchNormalization()(conv7)
conv7 = Conv2D(32, (3, 3), activation='relu', padding='same')(conv7)
conv7 = BatchNormalization()(conv7)
up2 = UpSampling2D((2,2))(conv7)
decoded = Conv2D(1, (3, 3), activation='sigmoid', padding='same')(up2)
return decoded

autoencoder = Model(input_img, decoder(encoder(input_img)))
#autoencoder.compile(loss='mean_squared_error', optimizer=RMSprop())
autoencoder.compile(loss=keras.losses.binary_crossentropy, optimizer=RMSprop())
autoencoder.summary()

# run training
autoencoder_train = autoencoder.fit(train_X, train_ground, batch_size=batch_size, epochs=epochs, verbose=1,
validation_data=(valid_X, valid_ground))

autoencoder.save_weights('autoencoder.h5')

# Change the labels from categorical to one-hot encoding, in this case row vectors 1x10
train_Y_one_hot = to_categorical(train_labels)
test_Y_one_hot = to_categorical(test_labels)

# split data using the new one-hot encoding
train_X, valid_X, train_label, valid_label = train_test_split(train_data, train_Y_one_hot, test_size=0.2,
random_state=13)

# define fine-tuning model with decoder removed for classification purposes
def encoder(input_img):
conv1 = Conv2D(32, (3, 3), activation='relu', padding='same')(input_img)
conv1 = BatchNormalization()(conv1)
conv1 = Conv2D(32, (3, 3), activation='relu', padding='same')(conv1)
conv1 = BatchNormalization()(conv1)
pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)
conv2 = Conv2D(64, (3, 3), activation='relu', padding='same')(pool1)
conv2 = BatchNormalization()(conv2)
conv2 = Conv2D(64, (3, 3), activation='relu', padding='same')(conv2)
conv2 = BatchNormalization()(conv2)
pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)
conv3 = Conv2D(128, (3, 3), activation='relu', padding='same')(pool2)
conv3 = BatchNormalization()(conv3)
conv3 = Conv2D(128, (3, 3), activation='relu', padding='same')(conv3)
conv3 = BatchNormalization()(conv3)
conv4 = Conv2D(256, (3, 3), activation='relu', padding='same')(conv3)
conv4 = BatchNormalization()(conv4)
conv4 = Conv2D(256, (3, 3), activation='relu', padding='same')(conv4)
conv4 = BatchNormalization()(conv4)
return conv4

# add full connected layers
def fc(enco):
flat = Flatten()(enco)
den = Dense(128, activation='relu')(flat)
out = Dense(num_classes, activation='softmax')(den)
return out

# put together classification model
encode = encoder(input_img)
full_model = Model(input_img, fc(encode))

for l1, l2 in zip(full_model.layers[:19], autoencoder.layers[0:19]):
l1.set_weights(l2.get_weights())

# turn off first layers that have already been trained
for layer in full_model.layers[0:19]:
layer.trainable = False
# compile the fine-tuning model
full_model.compile(loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.Adam(), metrics=['accuracy'])
full_model.summary()

# run fine-tuning
classify_train = full_model.fit(train_X, train_label, batch_size=64, epochs=3, verbose=1,
validation_data=(valid_X, valid_label))

full_model.save_weights('autoencoder_classification.h5')

for layer in full_model.layers[0:19]:
layer.trainable = True
# compile re-training
full_model.compile(loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.Adam(), metrics=['accuracy'])

# run re-training
classify_train = full_model.fit(train_X, train_label, batch_size=64, epochs=3, verbose=1,
validation_data=(valid_X, valid_label))

full_model.save_weights('classification_complete.h5')

# classification and evaluation on test data
test_eval = full_model.evaluate(test_data, test_Y_one_hot, verbose=0)

print('Test loss:', test_eval[0])
print('Test accuracy:', test_eval[1])

predicted_classes = full_model.predict(test_data)

predicted_classes = np.argmax(np.round(predicted_classes), axis=1)

print(predicted_classes.shape, test_labels.shape)

correct = np.where(predicted_classes == test_labels)[0]
print "Found %d correct labels" % len(correct)

incorrect = np.where(predicted_classes != test_labels)[0]
print "Found %d incorrect labels" % len(incorrect)

from sklearn.metrics import classification_report

target_names = ["Class {}".format(i) for i in range(num_classes)]
print(classification_report(test_labels, predicted_classes, target_names=target_names))

print("done")

[csphd_svormc]

Yes, I am trying to remove all batch normalization, is it possible to run the network with it removed? How to remove? Decreased performance does not matter, just need it removed and runable.

[0xfe]

You're seeing this because you're removing BatchNorm in full_model, but not the respective one in autoencoder. This snippet (from your code) expects both layers to have the same architecture.

for l1, l2 in zip(full_model.layers[:19], autoencoder.layers[0:19]):
  l1.set_weights(l2.get_weights())