TensorFlow使用记录 (五）： 激活函数和初始化方式

ReLU

\begin{equation}
ReLU(z) = max(0, z)
\end{equation}

tf.nn.relu

import matplotlib.pyplot as plt

import numpy as np

def relu(z):

    return np.maximum(0, z)

z = np.linspace(-5, 5, 200)

plt.plot(z, relu(z), "r--", linewidth=2)

props = dict(facecolor='black', shrink=0.1)

plt.annotate('ReLU', xytext=(-3.5, 0.5), xy=(-5, 0.1), arrowprops=props, fontsize=14, ha="center")

plt.title("ReLU activation function", fontsize=14)

plt.plot([-5, 5], [0, 0], 'k-')

plt.plot([0, 0], [-0.5, 4.2], 'k-')

plt.grid(True)

plt.axis([-5, 5, -0.5, 4.2])

plt.tight_layout()

plt.show()

leaky ReLU

\begin{equation}
LeakyReLU_{\alpha}(z) = max(\alpha z, z)
\end{equation}

nn.leaky_relu

import matplotlib.pyplot as plt

import numpy as np

def leaky_relu(z, alpha=0.01):

    return np.maximum(alpha*z, z)

z = np.linspace(-5, 5, 200)

plt.plot(z, leaky_relu(z, 0.05), "b-", linewidth=2)

plt.plot([-5, 5], [0, 0], 'k-')

plt.plot([0, 0], [-0.5, 4.2], 'k-')

plt.grid(True)

props = dict(facecolor='black', shrink=0.1)

plt.annotate('Leak', xytext=(-3.5, 0.5), xy=(-5, -0.2), arrowprops=props, fontsize=14, ha="center")

plt.title("Leaky ReLU activation function", fontsize=14)

plt.axis([-5, 5, -0.5, 4.2])

plt.tight_layout()

plt.show()

ELU

\begin{equation}
\label{b}
ELU(z)=
\begin{cases}
\alpha(e^{z} - 1) & if\ z < 0 \\
z & if\ z\ge 0
\end{cases}
\end{equation}

tf.nn.elu

import matplotlib.pyplot as plt

import numpy as np

def elu(z, alpha=1):

    return np.where(z < 0, alpha * (np.exp(z) - 1), z)

z = np.linspace(-5, 5, 200)

plt.plot(z, elu(z), "g-", linewidth=2)

plt.plot([-5, 5], [0, 0], 'k-')

plt.plot([-5, 5], [-1, -1], 'k--')

plt.plot([0, 0], [-2.2, 3.2], 'k-')

plt.grid(True)

plt.title(r"ELU activation function ($\alpha=1$)", fontsize=14)

plt.axis([-5, 5, -2.2, 3.2])

plt.tight_layout()

plt.show()

Relu6

\begin{equation}
ReLU6(z) =  min(max(z, 0), 6)
\end{equation}

tf.nn.relu6

Swish

\begin{equation}
Swish(z) =  z*sigmoid(\beta z)
\end{equation}

def swish(x, b = 1):

    return x * tf.nn.sigmoid(b * x)

import matplotlib.pyplot as plt

import numpy as np

def swish(z, b=1):

    return z/(1+np.exp(-b*z))

z = np.linspace(-5, 5, 200)

plt.plot(z, swish(z), "g--", linewidth=2)

plt.plot([-5, 5], [0, 0], 'k-')

plt.plot([0, 0], [-0.5, 5.2], 'k-')

plt.grid(True)

plt.title(r"Swish activation function", fontsize=14)

plt.axis([-5, 5, -0.5, 5.2])

plt.tight_layout()

plt.show()

简单训练 MNIST

import tensorflow as tf

def Swish(features):

    return features*tf.nn.sigmoid(features)

# 1. create data

from tensorflow.examples.tutorials.mnist import input_data

mnist = input_data.read_data_sets('../MNIST_data', one_hot=True)

X = tf.placeholder(tf.float32, shape=(None, 784), name='X')

y = tf.placeholder(tf.int32, shape=(None), name='y')

# 2. define network

with tf.name_scope('dnn'):

    hidden1 = tf.layers.dense(X, 300, activation=Swish, name='hidden1')

    hidden2 = tf.layers.dense(hidden1, 100, activation=Swish, name='hidden2')

    logits = tf.layers.dense(hidden2, 10, name='output')

    # prob = tf.layers.dense(hidden2, 10, tf.nn.softmax, name='prob')

# 3. define loss

with tf.name_scope('loss'):

    # tf.losses.sparse_softmax_cross_entropy() label is not one_hot and dtype is int*

    # xentropy = tf.losses.sparse_softmax_cross_entropy(labels=tf.argmax(y, axis=1), logits=logits)

    # tf.nn.sparse_softmax_cross_entropy_with_logits() label is not one_hot and dtype is int*

    # xentropy = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=tf.argmax(y, axis=1), logits=logits)

    # loss = tf.reduce_mean(xentropy)

    loss = tf.losses.softmax_cross_entropy(onehot_labels=y, logits=logits) # label is one_hot

# 4. define optimizer

learning_rate = 0.01

with tf.name_scope('train'):

    optimizer_op = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss)

with tf.name_scope('eval'):

    correct = tf.nn.in_top_k(logits, tf.argmax(y, axis=1), 1) # 目标是否在前K个预测中, label's dtype is int*

    accuracy = tf.reduce_mean(tf.cast(correct, tf.float32))

# 5. initialize

init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())

saver = tf.train.Saver()

# 5. train & test

n_epochs = 20

n_batches = 50

batch_size = 50

with tf.Session() as sess:

    sess.run(init_op)

    for epoch in range(n_epochs):

        for iteration in range(mnist.train.num_examples // batch_size):

            X_batch, y_batch = mnist.train.next_batch(batch_size)

            sess.run(optimizer_op, feed_dict={X: X_batch, y: y_batch})

        acc_train = accuracy.eval(feed_dict={X: X_batch, y: y_batch}) # 最后一个 batch 的 accuracy

        acc_test = accuracy.eval(feed_dict={X: mnist.test.images, y: mnist.test.labels})

        loss_test = loss.eval(feed_dict={X: mnist.test.images, y: mnist.test.labels})

        print(epoch, "Train accuracy:", acc_train, "Test accuracy:", acc_test, "Test loss:", loss_test)

    save_path = saver.save(sess, "./my_model_final.ckpt")

# with tf.Session() as sess:

#     sess.run(init_op)

#     saver.restore(sess, "./my_model_final.ckpt")

#     acc_test = accuracy.eval(feed_dict={X: mnist.test.images, y: mnist.test.labels})

#     loss_test = loss.eval(feed_dict={X: mnist.test.images, y: mnist.test.labels})

#     print("Test accuracy:", acc_test, ", Test loss:", loss_test)

"""

tf.sigmoid        0.9062

tf.tanh           0.9611

tf.relu           0.9713

tf.nn.leaky_relu  0.9674

tf.nn.elu         0.9613

Swish             0.9605

"""

初始化方式

tensorflow\python\ops\init_ops.py

zeros_initializer = Zeros

ones_initializer = Ones

constant_initializer = Constant

random_uniform_initializer = RandomUniform

random_normal_initializer = RandomNormal

truncated_normal_initializer = TruncatedNormal

uniform_unit_scaling_initializer = UniformUnitScaling

variance_scaling_initializer = VarianceScaling

glorot_uniform_initializer = GlorotUniform

glorot_normal_initializer = GlorotNormal

orthogonal_initializer = Orthogonal

identity_initializer = Identity

convolutional_delta_orthogonal = ConvolutionDeltaOrthogonal

convolutional_orthogonal_1d = ConvolutionOrthogonal1D

convolutional_orthogonal_2d = ConvolutionOrthogonal2D

convolutional_orthogonal_3d = ConvolutionOrthogonal3D

random_uniform_initializer

生成均匀分布的随机数

__init__(

    minval=0,

    maxval=None,

    seed=None,

    dtype=tf.dtypes.float32

)

random_normal_initializer

生成标准正态分布的随机数

__init__(

    mean=0.0,

    stddev=1.0,

    seed=None,

    dtype=tf.dtypes.float32

)

truncated_normal_initializer

生成截断正态分布的随机数，参数同 random_normal_initializer

uniform_unit_scaling_initializer

和均匀分布差不多，只是这个初始化方法不需要指定最小最大值，是通过计算出来的。参数为（factor=1.0, seed=None, dtype=dtypes.float32)

max_val = math.sqrt(3 / input_size) * self.factor

variance_scaling_initializer

综合体

__init__(self,

         scale=1.0,

         mode="fan_in",

         distribution="truncated_normal",

         seed=None,

         dtype=dtypes.float32)

scale = self.scaleif self.mode == "fan_in":

  scale /= max(1., fan_in)

elif self.mode == "fan_out":

  scale /= max(1., fan_out)

else:

  scale /= max(1., (fan_in + fan_out) / 2.)

if self.distribution == "normal" or self.distribution == "truncated_normal":

  # constant taken from scipy.stats.truncnorm.std(a=-2, b=2, loc=0., scale=1.)

  stddev = math.sqrt(scale) / .87962566103423978

  return random_ops.truncated_normal(

      shape, 0.0, stddev, dtype, seed=self.seed)

elif self.distribution == "untruncated_normal":

  stddev = math.sqrt(scale)

  return random_ops.random_normal(shape, 0.0, stddev, dtype, seed=self.seed)

else:

  limit = math.sqrt(3.0 * scale)

  return random_ops.random_uniform(

      shape, -limit, limit, dtype, seed=self.seed)

glorot_uniform_initializer & glorot_normal_initializer

标准的 Xavier 初始化方式

limit = sqrt(6 / (fan_in + fan_out))

stddev = sqrt(2 / (fan_in + fan_out))