尝试用卷积AE和卷积VAE做无监督检测，思路如下：
1.先用正常样本训练AE或VAE
2.输入测试集给AE或VAE，获得重构的测试集数据。
3.计算重构的数据和原始数据的误差，如果误差大于某一个阈值，则此测试样本为一样。
对于数据集的描述如下：
本数据集一共有10100个样本，每个样本是1行48列的向量，为了让它变成矩阵，自己在末尾补了一个0，将其转变成7*7的矩阵。前8000个是正常样本。后2100个中，前300个是正常样本，之后的1800个中包括6种异常时间序列，每种异常时间序列包括300个样本。
VAE的代码如下：
#https://blog.csdn.net/wyx100/article/details/80647379
'''This script demonstrates how to build a variational autoencoder
with Keras and deconvolution layers.
使用Keras和反卷积层建立变分自编码器演示脚本
# Reference
- Auto-Encoding Variational Bayes
自动编码变分贝叶斯
https://arxiv.org/abs/1312.6114
'''
from __future__ import print_function import numpy as np
import matplotlib.pyplot as plt
from scipy.stats import norm
from pandas import read_csv
from keras.layers import Input, Dense, Lambda, Flatten, Reshape
from keras.layers import Conv2D, Conv2DTranspose
from keras.models import Model
from keras import backend as K
from keras import metrics
import xlwt
from keras.datasets import mnist
from matplotlib import pyplot
import numpy
# input image dimensions
# 输入图像维度
img_rows, img_cols, img_chns = 7, 7, 1
dimension_image=7
# number of convolutional filters to use
# 使用的卷积过滤器数量
filters = 64
# convolution kernel size
# 卷积核大小
num_conv = 3 batch_size = 50
if K.image_data_format() == 'channels_first':
original_img_size = (img_chns, img_rows, img_cols)
else:
original_img_size = (img_rows, img_cols, img_chns)
latent_dim = 2
intermediate_dim = 128
epsilon_std = 1.0
epochs = 100 x = Input(shape=original_img_size)
conv_1 = Conv2D(img_chns,
kernel_size=(2, 2),
padding='same', activation='relu')(x)
conv_2 = Conv2D(filters,
kernel_size=(2, 2),
padding='same', activation='relu',
strides=(2, 2))(conv_1)
conv_3 = Conv2D(filters,
kernel_size=num_conv,
padding='same', activation='relu',
strides=1)(conv_2)
conv_4 = Conv2D(filters,
kernel_size=num_conv,
padding='same', activation='relu',
strides=1)(conv_3)
flat = Flatten()(conv_4)
hidden = Dense(intermediate_dim, activation='relu')(flat) z_mean = Dense(latent_dim)(hidden)
z_log_var = Dense(latent_dim)(hidden) def sampling(args):
z_mean, z_log_var = args
epsilon = K.random_normal(shape=(K.shape(z_mean)[0], latent_dim),
mean=0., stddev=epsilon_std)
return z_mean + K.exp(z_log_var) * epsilon # note that "output_shape" isn't necessary with the TensorFlow backend
# so you could write `Lambda(sampling)([z_mean, z_log_var])`
# 注意，“output_shape”对于TensorFlow后端不是必需的。因此可以编写Lambda(sampling)([z_mean, z_log_var])`
z = Lambda(sampling, output_shape=(latent_dim,))([z_mean, z_log_var]) # we instantiate these layers separately so as to reuse them later
# 分别实例化这些层，以便在以后重用它们。
number=4
decoder_hid = Dense(intermediate_dim, activation='relu')
decoder_upsample = Dense(filters * number * number, activation='relu') if K.image_data_format() == 'channels_first':
output_shape = (batch_size, filters, number, number)
else:
output_shape = (batch_size, number, number, filters) decoder_reshape = Reshape(output_shape[1:])
decoder_deconv_1 = Conv2DTranspose(filters,
kernel_size=num_conv,
padding='same',
strides=1,
activation='relu')
decoder_deconv_2 = Conv2DTranspose(filters,
kernel_size=num_conv,
padding='same',
strides=1,
activation='relu')
if K.image_data_format() == 'channels_first':
output_shape = (batch_size, filters, 13, 13)
else:
output_shape = (batch_size,13, 13, filters)
decoder_deconv_3_upsamp = Conv2DTranspose(filters,
kernel_size=(3, 3),
strides=(2, 2),
padding='valid',
activation='relu')
decoder_mean_squash = Conv2D(img_chns,
kernel_size=3,
padding='valid',
activation='sigmoid') hid_decoded = decoder_hid(z)
up_decoded = decoder_upsample(hid_decoded)
reshape_decoded = decoder_reshape(up_decoded)
deconv_1_decoded = decoder_deconv_1(reshape_decoded)
deconv_2_decoded = decoder_deconv_2(deconv_1_decoded)
x_decoded_relu = decoder_deconv_3_upsamp(deconv_2_decoded)
x_decoded_mean_squash = decoder_mean_squash(x_decoded_relu) # instantiate VAE model
# 实例化VAE模型
vae = Model(x, x_decoded_mean_squash)
# Compute VAE loss
# 计算VAE损失
xent_loss = img_rows * img_cols * metrics.binary_crossentropy(
K.flatten(x),
K.flatten(x_decoded_mean_squash))
kl_loss = - 0.5 * K.sum(1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1)
vae_loss = K.mean(xent_loss + kl_loss)
vae.add_loss(vae_loss)
vae.compile(optimizer='Adam')
vae.summary() dataset = read_csv('randperm_zerone_Dataset.csv')
values = dataset.values
XY= values
n_train_hours1 =7000
n_train_hours3 =8000
x_train=XY[:n_train_hours1,:]
x_valid =XY[n_train_hours1:n_train_hours3, :]
x_test =XY[n_train_hours3:, :]
x_train=x_train.reshape(-1,dimension_image,dimension_image,1)
x_valid=x_valid.reshape(-1,dimension_image,dimension_image,1)
x_test=x_test.reshape(-1,dimension_image,dimension_image,1) history=vae.fit(x_train,
shuffle=True,
epochs=epochs,
batch_size=batch_size,
validation_data=(x_valid, None))
pyplot.plot(history.history['loss'], label='train')
pyplot.plot(history.history['val_loss'], label='valid')
pyplot.legend()
pyplot.show() # 建立一个潜在空间输入模型
encoder = Model(x, z_mean)
# 在潜在空间中显示数字类的2D图
x_test_encoded = encoder.predict(x_test, batch_size=batch_size)
plt.figure(figsize=(6, 6))
plt.scatter(x_test_encoded[:, 0], x_test_encoded[:, 1])
plt.show() Reconstructed_train = vae.predict(x_train)
Reconstructed_valid = vae.predict(x_valid)
Reconstructed_test = vae.predict(x_test)
ReconstructedData1=np.vstack((Reconstructed_train,Reconstructed_valid))
ReconstructedData2=np.vstack((ReconstructedData1,Reconstructed_test))
ReconstructedData3=ReconstructedData2.reshape((ReconstructedData2.shape[0], -1)) numpy.savetxt("ReconstructedData.csv", ReconstructedData3, delimiter=',')
AE代码如下
from keras.layers import Input, Dense, Conv2D, MaxPooling2D, UpSampling2D
from keras.models import Model
from keras import backend as K
import numpy as np
from pandas import read_csv
from matplotlib import pyplot
import numpy dimension_image=7
input_img = Input(shape=(dimension_image, dimension_image, 1)) # adapt this if using `channels_first` image data format
x = Conv2D(16, (3, 3), activation='relu', padding='same')(input_img)
x = MaxPooling2D((2, 2), padding='same')(x)
x = Conv2D(8, (3, 3), activation='relu', padding='same')(x)
x = MaxPooling2D((2, 2), padding='same')(x)
x = Conv2D(8, (3, 3), activation='relu', padding='same')(x)
encoded = MaxPooling2D((2, 2), padding='same')(x) # at this point the representation is (4, 4, 8) i.e. 128-dimensional
x = Conv2D(8, (3, 3), activation='relu', padding='same')(encoded)
x = UpSampling2D((2, 2))(x)
x = Conv2D(8, (3, 3), activation='relu', padding='same')(x)
x = UpSampling2D((2, 2))(x)
x = Conv2D(16, (3, 3), activation='relu', padding='same')(x)
x = UpSampling2D((2, 2))(x)
decoded = Conv2D(1, (2, 2), activation='sigmoid')(x) autoencoder = Model(input_img, decoded)
autoencoder.compile(optimizer='adadelta', loss='binary_crossentropy')
autoencoder.summary() dataset = read_csv('randperm_zerone_Dataset.csv')
values = dataset.values
XY= values
n_train_hours1 =7000
n_train_hours3 =8000
x_train=XY[:n_train_hours1,:]
x_valid =XY[n_train_hours1:n_train_hours3, :]
x_test =XY[n_train_hours3:, :]
x_train=x_train.reshape(-1,dimension_image,dimension_image,1)
x_valid=x_valid.reshape(-1,dimension_image,dimension_image,1)
x_test=x_test.reshape(-1,dimension_image,dimension_image,1) history=autoencoder.fit(x_train, x_train,
epochs=200,
batch_size=32,
shuffle=True,
validation_data=(x_valid, x_valid))
pyplot.plot(history.history['loss'], label='train')
pyplot.plot(history.history['val_loss'], label='valid')
pyplot.legend()
pyplot.show()
Reconstructed_train = autoencoder.predict(x_train)
Reconstructed_valid = autoencoder.predict(x_valid)
Reconstructed_test = autoencoder.predict(x_test)
ReconstructedData1=np.vstack((Reconstructed_train,Reconstructed_valid))
ReconstructedData2=np.vstack((ReconstructedData1,Reconstructed_test))
ReconstructedData3=ReconstructedData2.reshape((ReconstructedData2.shape[0], -1)) numpy.savetxt("ReconstructedData.csv", ReconstructedData3, delimiter=',')
至于数据集，正在上传到百度文库，以后更新