线性回归

1. 线性回归的基本要素

2. 线性回归模型从零开始的实现

3. 线性回归模型使用pytorch的简洁实现

线性回归的基本要素

优化函数 - 随机梯度下降

• (i)初始化模型参数，一般来说使用随机初始化；

• (ii)我们在数据上迭代多次，通过在负梯度方向移动参数来更新每个参数。

矢量计算

1. 向量相加的一种方法是，将这两个向量按元素逐一做标量加法。

2. 向量相加的另一种方法是，将这两个向量直接做矢量加法。

import torch
import time

# init variable a, b as 1000 dimension vector
n = 1000
a = torch.ones(n)
b = torch.ones(n)
# define a timer class to record time
class Timer(object):
"""Record multiple running times."""
def __init__(self):
self.times = []
self.start()

def start(self):
# start the timer
self.start_time = time.time()

def stop(self):
# stop the timer and record time into a list
self.times.append(time.time() - self.start_time)
return self.times[-1]

def avg(self):
# calculate the average and return
return sum(self.times)/len(self.times)

def sum(self):
# return the sum of recorded time
return sum(self.times)


timer = Timer()
c = torch.zeros(n)
for i in range(n):
c[i] = a[i] + b[i]
'%.5f sec' % timer.stop()



timer.start()
d = a + b
'%.5f sec' % timer.stop()



线性回归模型从零开始的实现

# import packages and modules
%matplotlib inline
import torch
from IPython import display
from matplotlib import pyplot as plt
import numpy as np
import random

print(torch.__version__)



生成数据集

# set input feature number
num_inputs = 2
# set example number
num_examples = 1000

# set true weight and bias in order to generate corresponded label
true_w = [2, -3.4]
true_b = 4.2

features = torch.randn(num_examples, num_inputs,
dtype=torch.float32)
labels = true_w[0] * features[:, 0] + true_w[1] * features[:, 1] + true_b
labels += torch.tensor(np.random.normal(0, 0.01, size=labels.size()),
dtype=torch.float32)


使用图像来展示生成的数据

plt.scatter(features[:, 1].numpy(), labels.numpy(), 1);


features = torch.randn(num_examples, num_inputs,
dtype=torch.float32)
print(features)

[-1.6370,  1.6305],
[-0.1965,  0.8613],
...,
[-0.9776,  0.0575],
[ 1.9371, -0.1497],
[-0.1417, -1.0046]])


读取数据集

def data_iter(batch_size, features, labels):
num_examples = len(features)
indices = list(range(num_examples))
random.shuffle(indices)  # random read 10 samples
for i in range(0, num_examples, batch_size):
j = torch.LongTensor(indices[i: min(i + batch_size, num_examples)]) # the last time may be not enough for a whole batch
yield  features.index_select(0, j), labels.index_select(0, j)

batch_size = 10
for X, y in data_iter(batch_size, features, labels):
print(X, '\n', y)
break

[ 0.5206, -0.2726],
[-0.6639,  0.9716],
[ 2.7164, -0.6513],
[-1.0642,  1.9331],
[-2.2240, -0.3616],
[-0.9094,  0.6691],
[-0.2991,  0.2488],
[ 1.8312,  0.2209],
[ 0.2833, -1.1672]])
tensor([6.9694, 6.0005, 9.5797, 0.6944, 4.1964, 6.8519, 2.5178, 4.4217, 5.4679,
9.9754])


初始化模型参数

w = torch.tensor(np.random.normal(0, 0.01, (num_inputs, 1)), dtype=torch.float32)
b = torch.zeros(1, dtype=torch.float32)



定义模型

def linreg(X, w, b):


定义损失函数

$l^{(i)}(\mathbf{w}, b) = \frac{1}{2} \left(\hat{y}^{(i)} - y^{(i)}\right)^2,$

def squared_loss(y_hat, y):
return (y_hat - y.view(y_hat.size())) ** 2 / 2


定义优化函数

$(\mathbf{w},b) \leftarrow (\mathbf{w},b) - \frac{\eta}{|\mathcal{B}|} \sum_{i \in \mathcal{B}} \partial_{(\mathbf{w},b)} l^{(i)}(\mathbf{w},b)$

def sgd(params, lr, batch_size):
for param in params:
param.data -= lr * param.grad / batch_size


训练

# super parameters init
lr = 0.03
num_epochs = 5

net = linreg
loss = squared_loss

# training
for epoch in range(num_epochs):  # training repeats num_epochs times
# in each epoch, all the samples in dataset will be used once

# X is the feature and y is the label of a batch sample
for X, y in data_iter(batch_size, features, labels):
l = loss(net(X, w, b), y).sum()
# calculate the gradient of batch sample loss
l.backward()
# using small batch random gradient descent to iter model parameters
sgd([w, b], lr, batch_size)
# reset parameter gradient
train_l = loss(net(features, w, b), labels)
print('epoch %d, loss %f' % (epoch + 1, train_l.mean().item()))

epoch 2, loss 7.521966
epoch 3, loss 7.550967
epoch 4, loss 7.542496
epoch 5, loss 7.535208


线性回归模型使用pytorch的简洁实现

import torch
from torch import nn
import numpy as np
torch.manual_seed(1)
torch.set_default_tensor_type('torch.FloatTensor')


生成数据集

num_examples = 1000

true_w = [2, -3.4]
true_b = 4.2

features = torch.tensor(np.random.normal(0, 1, (num_examples, num_inputs)), dtype=torch.float)
labels = true_w[0] * features[:, 0] + true_w[1] * features[:, 1] + true_b
labels += torch.tensor(np.random.normal(0, 0.01, size=labels.size()), dtype=torch.float)


读取数据集

import torch.utils.data as Data

batch_size = 10

# combine featues and labels of dataset
dataset = Data.TensorDataset(features, labels)

# put dataset into DataLoader
dataset=dataset,            # torch TensorDataset format
batch_size=batch_size,      # mini batch size
shuffle=True,               # whether shuffle the data or not
)
for X, y in data_iter:
print(X, '\n', y)
break

[-0.1495, -1.6520],
[-0.3280,  0.2594],
[-0.4857, -1.2976],
[ 1.8603,  0.4539],
[-0.3628,  0.0064],
[ 1.3235, -0.3536],
[-2.3426, -0.5968],
[-0.6290, -0.2948],
[-0.0787,  0.2180]])
tensor([7.0088, 9.5071, 2.6718, 7.6535, 6.3802, 3.4601, 8.0475, 1.5223, 3.9682,
3.2977])


定义模型

class LinearNet(nn.Module):
def __init__(self, n_feature):
super(LinearNet, self).__init__()      # call father function to init
self.linear = nn.Linear(n_feature, 1)  # function prototype: torch.nn.Linear(in_features, out_features, bias=True)

def forward(self, x):
y = self.linear(x)
return y

net = LinearNet(num_inputs)
# ways to init a multilayer network
# method one
net = nn.Sequential(
nn.Linear(num_inputs, 1)
# other layers can be added here
)

# method two
net = nn.Sequential()

# method three
from collections import OrderedDict
net = nn.Sequential(OrderedDict([
('linear', nn.Linear(num_inputs, 1))
# ......
]))



初始化模型参数

from torch.nn import init

init.normal_(net[0].weight, mean=0.0, std=0.01)
init.constant_(net[0].bias, val=0.0)  # or you can use net[0].bias.data.fill_(0) to modify it directly

for param in net.parameters():
print(param)

Parameter containing:


定义损失函数

loss = nn.MSELoss()    # nn built-in squared loss function
# function prototype: torch.nn.MSELoss(size_average=None, reduce=None, reduction='mean')


定义优化函数

import torch.optim as optim

optimizer = optim.SGD(net.parameters(), lr=0.03)   # built-in random gradient descent function
print(optimizer)  # function prototype: torch.optim.SGD(params, lr=, momentum=0, dampening=0, weight_decay=0, nesterov=False)

Parameter Group 0
dampening: 0
lr: 0.03
momentum: 0
nesterov: False
weight_decay: 0
)


训练

num_epochs = 3
for epoch in range(1, num_epochs + 1):
for X, y in data_iter:
output = net(X)
l = loss(output, y.view(-1, 1))
l.backward()
optimizer.step()
print('epoch %d, loss: %f' % (epoch, l.item()))
# result comparision
dense = net[0]
print(true_w, dense.weight.data)
print(true_b, dense.bias.data)

epoch 2, loss: 0.000097
epoch 3, loss: 0.000079