|
-
- from keras.utils import to_categorical
- import matplotlib.pyplot as plt
- from collections import defaultdict
- from keras.models import load_model
- import numpy as np
- import keras
- from keras.models import Model
- import os
- os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
- os.environ["CUDA_VISIBLE_DEVICES"] = "1"
- os.environ["TF_CPP_MIN_LOG_LEVEL"] = "2"
-
-
-
- def load_data(path='mnist.npz'):
- """Loads the MNIST dataset.
-
- # Arguments
- path: path where to cache the dataset locally
- (relative to ~/.keras/datasets).
-
- # Returns
- Tuple of Numpy arrays: `(x_train, y_train), (x_test, y_test)`.
- """
-
- f = np.load(path)
- x_train, y_train = f['x_train'], f['y_train']
- x_test, y_test = f['x_test'], f['y_test']
- f.close()
- return (x_train, y_train), (x_test, y_test)
-
-
- def get_boundary(neuron_value, number_of_neuron):
-
- max_value = np.max(neuron_value, axis=0)
- min_value = np.min(neuron_value, axis=0)
- boundary = []
- for i in range(number_of_neuron):
- dic = {"max": max_value[i], "min": min_value[i]}
- boundary.append(dic)
- print("------神经元边界计算------")
- # print(boundary)
- return boundary
-
- def nbcov_and_snacov_one(neuron_value, number_of_neuron, boundary):
- """
- 计算样本集的神经元边界覆盖率NBCov和强神经元激活覆盖率SNACov
- :param neuron_value:
- :param number_of_neuron:
- :param boundary:
- :return: NBCov, SNACov
- """
- # print("------开始计算神经元边界覆盖率和强神经元激活覆盖率------")
- count_upper = 0
- count_lower = 0
- for i in range(number_of_neuron):
- upper_flag, lower_flag = 0, 0
- if neuron_value[i] > boundary[i]["max"] and upper_flag == 0:
- count_upper += 1
- upper_flag = 1
- elif neuron_value[i] < boundary[i]["min"] and lower_flag == 0:
- count_lower += 1
- lower_flag = 1
- if upper_flag == 1 and lower_flag == 1:
- break
- # print("------计算神经元边界覆盖率和强神经元激活覆盖率结束------")
- return (count_upper + count_lower) / (2 * number_of_neuron), count_upper / number_of_neuron
-
-
-
-
-
- def nbcov_and_snacov(neuron_value, number_of_neuron, boundary):
-
- print("------开始计算神经元边界覆盖率和强神经元激活覆盖率------")
- count_upper = 0
- count_lower = 0
- for i in range(number_of_neuron):
- upper_flag, lower_flag = 0, 0
- for example in neuron_value:
- if example[i] > boundary[i]["max"] and upper_flag == 0:
- count_upper += 1
- upper_flag = 1
- elif example[i] < boundary[i]["min"] and lower_flag == 0:
- count_lower += 1
- lower_flag = 1
- if upper_flag == 1 and lower_flag == 1:
- break
- print("------计算神经元边界覆盖率和强神经元激活覆盖率结束------")
- return (count_upper + count_lower) / (2 * number_of_neuron), count_upper / number_of_neuron
-
-
-
- def init_coverage_tables(model1):
- model_layer_dict1 = defaultdict(bool)
- init_dict(model1, model_layer_dict1)
- return model_layer_dict1
-
- def init_dict(model, model_layer_dict):
- for layer in model.layers:
- if 'flatten' in layer.name or 'input' in layer.name:
- continue
- for index in range(layer.output_shape[-1]):
- model_layer_dict[(layer.name, index)] = False
-
-
- def neuron_covered(model_layer_dict):
- covered_neurons = len([v for v in model_layer_dict.values() if v])
- total_neurons = len(model_layer_dict)
- return covered_neurons, total_neurons, covered_neurons / float(total_neurons)
-
-
- def scale(intermediate_layer_output, rmax=1, rmin=0):
- X_std = (intermediate_layer_output - intermediate_layer_output.min()) / (
- intermediate_layer_output.max() - intermediate_layer_output.min())
- X_scaled = X_std * (rmax - rmin) + rmin
- return X_scaled
-
-
- def update_coverage(input_data, model, model_layer_dict, threshold=0):
- layer_names = [layer.name for layer in model.layers if
- 'flatten' not in layer.name and 'input' not in layer.name]
-
- intermediate_layer_model = Model(inputs=model.input,
- outputs=[model.get_layer(layer_name).output for layer_name in layer_names])
- intermediate_layer_outputs = intermediate_layer_model.predict(input_data)
-
- for i, intermediate_layer_output in enumerate(intermediate_layer_outputs):
- scaled = scale(intermediate_layer_output[0])
- for num_neuron in range(scaled.shape[-1]):
- if np.mean(scaled[..., num_neuron]) > threshold and not model_layer_dict[(layer_names[i], num_neuron)]:
- model_layer_dict[(layer_names[i], num_neuron)] = True
-
-
-
- def plot_coverage(input_data, model):
- # plt.figure(figsize=(20, 20))
- # layer_names = [layer.name for layer in model.layers if
- # 'flatten' not in layer.name and 'input' not in layer.name]
- layer_names = [layer.name for layer in model.layers if
- 'conv' in layer.name or 'pool' in layer.name]
- intermediate_layer_model = Model(inputs=model.input,
- outputs=[model.get_layer(layer_name).output for layer_name in layer_names])
- intermediate_layer_outputs = intermediate_layer_model.predict(input_data)
- cc = [[] for x in range(len(layer_names))]
- c_change = [[] for x in range(len(layer_names))]
- max_neurons = 0
- number_neurons = []
- # print('index:', m)
- for i, intermediate_layer_output in enumerate(intermediate_layer_outputs):
- scaled = scale(intermediate_layer_output[0])
- number_neurons.append(scaled.shape[2])
- if scaled.shape[2] >= max_neurons:
- max_neurons = scaled.shape[2]
- for i, intermediate_layer_output in enumerate(intermediate_layer_outputs):
- scaled = scale(intermediate_layer_output[0])
- # print(layer_names[i])
- b=''
- d=[]
- add_index = (max_neurons - number_neurons[i])/2
- for num_neuron in range(scaled.shape[-1]):
-
- d.append(np.mean(scaled[..., num_neuron]))
- g = str(np.mean(scaled[..., num_neuron]))
- b += g+'\n'
- # print(np.mean(scaled[..., num_neuron]))
- # if np.mean(scaled[..., num_neuron]) > 0.1 and not model_layer_dict[(layer_names[i], num_neuron)]:
- # model_layer_dict[(layer_names[i], num_neuron)] = True
- d = list(d)
- # print(d)
- plt.scatter(i, num_neuron + add_index, color='none', edgecolors='black', s=5)
- cc[i] = d.index(max(d))
- # print(cc[i])
- # plt.title('input label: {}'.format(np.argmax(model.predict(input_data))))
- # plt.title('LeNet5:Benign Input')
- scale_ls = range(len(layer_names))
- # index_ls = layer_names
- index_ls = ['layers' + str(i) for i in range(len(layer_names))]
- plt.yticks([])
- plt.xticks([])
- plt.scatter(i, cc[i] + add_index, color='r', edgecolors='r', s=5)
- plt.text(i, cc[i] + add_index, str(cc[i]), color='r')
- c_change[i] = cc[i] + add_index
- # plt.xticks(scale_ls, index_ls, rotation=70)
- for l in range(len(layer_names) - 1):
- # print(c_change[l])
- plt.plot([l, l+1], [c_change[l], c_change[l+1]], color='r')
- plt.savefig("/data0/BigPlatform/ZJPlatform/007_DeepTesting/001-Demo/Plot_Path_MNIST/"+'path.png', bbox_inches='tight')
- plt.close()
- # plt.show()
-
- def coverage_one(input_data, model):
- get_value = []
- layer_names = [layer.name for layer in model.layers if
- 'flatten' not in layer.name and 'input' not in layer.name]
-
- intermediate_layer_model = Model(inputs=model.input,
- outputs=[model.get_layer(layer_name).output for layer_name in layer_names])
- intermediate_layer_outputs = intermediate_layer_model.predict(input_data)
-
- for i, intermediate_layer_output in enumerate(intermediate_layer_outputs):
- scaled = scale(intermediate_layer_output[0])
- for num_neuron in range(scaled.shape[-1]):
- get_value.append(np.mean(scaled[..., num_neuron]))
- # if np.mean(scaled[..., num_neuron]) > threshold and not model_layer_dict[(layer_names[i], num_neuron)]:
- # model_layer_dict[(layer_names[i], num_neuron)] = True
- return get_value
-
- def coverage(data, model):
- values = []
- for j in range(data.shape[0]):
- input_data = data[j:j+1]
- values.append(coverage_one(input_data, model))
- return values
-
-
- #%%
- path = '/data0/BigPlatform/ZJPlatform/007_DeepTesting/000-Dataset/datasets/mnist.npz'
- (x_train, y_train), (x_test, y_test) = load_data(path)
- y_train = to_categorical(y_train, num_classes=10)
- y_test = to_categorical(y_test, num_classes=10)
- x_train = np.expand_dims(x_train, axis=3)
- x_test = np.expand_dims(x_test, axis=3)
- x_train, x_test = x_train/255.0, x_test/255.0
- model = load_model('/data0/BigPlatform/ZJPlatform/007_DeepTesting/001-Demo/AlexNet_MNIST.h5')
- bounds = np.load('/data0/BigPlatform/ZJPlatform/007_DeepTesting/001-Demo/AlexNet_MNIST_Train_Bounds.npy')
- model_layer_dict1 = init_coverage_tables(model)
- #%%
- batch = 64
- x_batch = x_test[:batch]
- x = x_test[0:1]
- plot_coverage(x, model)
- boundary = get_boundary(bounds, len(model_layer_dict1))
- update_coverage(x, model, model_layer_dict1)
- nc = neuron_covered(model_layer_dict1)[0] / len(model_layer_dict1)
- print('nc:', nc)
- nbc, snac = nbcov_and_snacov_one(coverage_one(x, model), len(model_layer_dict1), boundary)
- print('nbc', nbc)
- print('snac', snac)
- nbc_batch, snac_batch = nbcov_and_snacov(coverage(x_batch, model), len(model_layer_dict1), boundary)
- print('nbc_batch', nbc_batch)
- print('snac', snac_batch)
|