动手学深度学习(一)——线性回归从零开始

文章作者:Tyan
博客:noahsnail.com  |  CSDN  |  简书

注:本文为李沐大神的《动手学深度学习》的课程笔记!

参考资料

1
2
3
# 导入mxnet的ndarray, autograd
from mxnet import autograd
from mxnet import ndarray as nd

创建数据集

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
# 训练数据的维度
num_inputs = 2
# 训练数据的样本数量
num_examples = 1000
# 实际的权重w
true_w = [2, -3.4]
# 实际的偏置b
true_b = 4.2
# 随机生成均值为0, 方差为1, 服从正态分布的训练数据X,
X = nd.random_normal(shape=(num_examples, num_inputs))
# 根据X, w, b生成对应的输出y
y = true_w[0] * X[:, 0] + true_w[1] * X[:, 1] + true_b
# 给y加上随机噪声
y += 0.01 * nd.random_normal(shape=y.shape)

数据可视化

1
print(X[0], y[0])
(
[ 1.16307867  0.48380461]
<NDArray 2 @cpu(0)>, 
[ 4.87962484]
<NDArray 1 @cpu(0)>)
1
2
3
4
5
6
%matplotlib inline
import matplotlib.pyplot as plt
# 绘制数据的散点图
plt.scatter(X[:, 1].asnumpy(), y.asnumpy())
plt.show()

Figure 1

数据读取

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
import random
# 训练时的批数据大小
batch_size = 10
# 通过yield进行数据读取
def data_iter():
# 产生样本的索引
idx = list(range(num_examples))
# 将索引随机打乱
random.shuffle(idx)
# 迭代一个epoch, xrange循环时效率比range更高
for i in xrange(0, num_examples, batch_size):
# 依次取出样本的索引, 这种实现方式在num_examples/batch_size不能整除时也适用
j = nd.array(idx[i:min((i + batch_size), num_examples)])
# 根据提供的索引取元素
yield nd.take(X, j), nd.take(y, j)
1
2
3
4
5
6
7
8
9
10
11
12
# 查看data_iter是否是generator函数
from inspect import isgeneratorfunction
print isgeneratorfunction(data_iter)
# data_iter类似于类的定义, 而data_iter()相当于一个类的实例, 当然是匿名实例
import types
print isinstance(data_iter(), types.GeneratorType)
# 读取数据测试
for data, label in data_iter():
print(data, label)
break
True
True
(
[[ 1.18770552 -0.46362698]
 [-3.15577412  2.19352984]
 [-0.45067298 -0.96665388]
 [ 0.05416773 -1.21203637]
 [-1.49418294 -1.61555624]
 [-0.93778831 -1.69338322]
 [ 0.91439158  1.31797135]
 [ 0.82403505  0.33020774]
 [-0.19660901  1.13431609]
 [ 0.15364595  1.01133049]]
<NDArray 10x2 @cpu(0)>, 
[ 8.17057896 -9.57918072  6.58949089  8.41831684  6.69815683  8.08473206
  1.54548573  4.73358202 -0.0632825   1.06603777]
<NDArray 10 @cpu(0)>)

初始化模型参数

1
2
3
4
5
6
7
8
9
10
# 随机初始化权重w
w = nd.random_normal(shape=(num_inputs, 1))
# 偏置b初始化为0
b = nd.zeros((1,))
# w, b放入list里
params = [w, b]
# 需要计算反向传播, 添加自动求导
for param in params:
param.attach_grad()

定义模型

1
2
3
4
# 定义运算y = w * x + b
def net(X):
# 向量, 矩阵乘用dot
return nd.dot(X, w) + b

损失函数

1
2
3
4
5
# 定义平方损失
def square_loss(yhat, y):
# 注意这里我们把y变形成yhat的形状来避免矩阵形状的自动转换
# loss为预测值减去真实值
return (yhat - y.reshape(yhat.shape)) ** 2

优化

1
2
3
4
5
6
# 定义随机梯度下降法
def SGD(params, lr):
# 对参数进行梯度下降
for param in params:
# 这样写不会创建新的param, 而是会写在原来的param里, 新的param没有梯度
param[:] = param - lr * param.grad

数据可视化

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
# 模型函数
def real_fn(X):
return true_w[0] * X[:, 0] - true_w[1] * X[:, 1] + true_b
# 绘制损失随训练迭代次数变化的折线图,以及预测值和真实值的散点图
def plot(losses, X, sample_size=100):
xs = list(range(len(losses)))
# 绘制两个子图
fig, (ax1, ax2) = plt.subplots(1, 2)
# 子图一设置标题
ax1.set_title('Loss during training')
# 绘制loss图像, 蓝色实线
ax1.plot(xs, losses, '-b')
# 子图二设置标题
ax2.set_title('Estimated vs Real Function')
# 绘制预测值, 蓝色的小圈
ax2.plot(X[:sample_size, 0].asnumpy(), net(X[:sample_size, :]).asnumpy(), 'ob', label = 'Estimated')
# 绘制实际值, 绿色的星号
ax2.plot(X[:sample_size, 0].asnumpy(), real_fn(X[:sample_size, :]).asnumpy(), '*g', label = 'Real Value')
# 绘制图例
ax2.legend()
# 显示图像
plt.show()

训练

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
# 定义训练的迭代周期
epochs = 5
# 定义学习率
learning_rate = 0.01
# 迭代次数
niter = 0
# 保存loss
losses = []
# 移动平均损失(加权)
moving_loss = 0
# 指数平滑系数
smoothing_constant = 0.01
# 训练
for epoch in xrange(epochs):
# 总的loss
total_loss = 0
# 迭代训练
for data, label in data_iter():
# 记录梯度
with autograd.record():
# 计算预测值
output = net(data)
# 计算loss
loss = square_loss(output, label)
# 根据loss进行反向传播计算梯度
loss.backward()
# 使用随机梯度下降求解(BSGD)
SGD(params, learning_rate)
# 计算总的loss
total_loss += nd.sum(loss).asscalar()
# 记录每读取一个数据点后,损失的移动平均值的变化
# 迭代次数加一
niter += 1
# 计算当前损失
current_loss = nd.mean(loss).asscalar()
# 计算移动平均损失,指数平滑方法
moving_loss = (1 - smoothing_constant) * moving_loss + smoothing_constant * current_loss
# 计算估计损失
est_loss = moving_loss / (1 - (1 - smoothing_constant) ** niter)
# 输出迭代信息
if (niter + 1) % 100 == 0:
# 保存估计损失
losses.append(est_loss)
print 'Epoch %s, batch %s. Moving average of loss: %s. Average loss: %f' % (epoch, niter, est_loss, total_loss / num_examples)
plot(losses, X)
Epoch 0, batch 99. Moving average of loss: 0.378590331091. Average loss: 0.625015

Epoch 0

Epoch 1, batch 199. Moving average of loss: 0.10108379838. Average loss: 0.000099

Epoch 1

Epoch 2, batch 299. Moving average of loss: 0.033726038259. Average loss: 0.000099

Epoch 2

Epoch 3, batch 399. Moving average of loss: 0.0120152144263. Average loss: 0.000099

Epoch 3

Epoch 4, batch 499. Moving average of loss: 0.00441111205064. Average loss: 0.000101

Epoch 4

1
2
3
4
print w
print true_w
print b
print true_b
[[ 1.99982905]
 [-3.40232825]]
<NDArray 2x1 @cpu(0)>
[2, -3.4]

[ 4.20024347]
<NDArray 1 @cpu(0)>
4.2

其他学习率

1
2
3
4
5
6
7
learning_rate = 0.001
Epoch 0, batch 99. Moving average of loss: 4.20676625843. Average loss: 5.549237
Epoch 1, batch 199. Moving average of loss: 1.1782055765. Average loss: 0.098550
Epoch 2, batch 299. Moving average of loss: 0.393321947036. Average loss: 0.001857
Epoch 3, batch 399. Moving average of loss: 0.13944143045. Average loss: 0.000127
Epoch 4, batch 499. Moving average of loss: 0.0505110244825. Average loss: 0.000096
1
2
3
4
5
6
7
learning_rate = 0.1
Epoch 0, batch 99. Moving average of loss: 3.79341099229e+13. Average loss: 26080307360862.457031
Epoch 1, batch 199. Moving average of loss: 1.7174457145e+28. Average loss: 15303785876879711197739352064.000000
Epoch 2, batch 299. Moving average of loss: nan. Average loss: nan
Epoch 3, batch 399. Moving average of loss: nan. Average loss: nan
Epoch 4, batch 499. Moving average of loss: nan. Average loss: nan
1
2
3
4
5
6
7
learning_rate = 1
Epoch 0, batch 99. Moving average of loss: nan. Average loss: nan
Epoch 1, batch 199. Moving average of loss: nan. Average loss: nan
Epoch 2, batch 299. Moving average of loss: nan. Average loss: nan
Epoch 3, batch 399. Moving average of loss: nan. Average loss: nan
Epoch 4, batch 499. Moving average of loss: nan. Average loss: nan

代码地址

https://github.com/SnailTyan/gluon-practice-code

如果有收获,可以请我喝杯咖啡!