Dive Into Deep Learning

$ source /opt/anaconda/bin/activate root
(base) $ conda create --name d2l python=3.8 -y
Collecting package metadata (current_repodata.json): done
Solving environment: done
...
Preparing transaction: done
Verifying transaction: done
Executing transaction: done
>#
># To activate this environment, use
>#
>#     $ conda activate d2l
>#
># To deactivate an active environment, use
>#
>#     $ conda deactivate

$ conda activate d2l
(d2l) $

(d2l) $ pip install mxnet==1.7.0.post1
Collecting mxnet==1.7.0.post1
  Downloading mxnet-1.7.0.post1-py2.py3-none-manylinux2014_x86_64.whl (55.0 MB)
...
Installing collected packages: urllib3, idna, chardet, requests, numpy, graphviz, mxnet
Successfully installed chardet-4.0.0 graphviz-0.8.4 idna-2.10 mxnet-1.7.0.post1 numpy-1.21.0 requests-2.25.1 urllib3-1.26.6
(d2l) $ pip install torch torchvision
Collecting torch
  Downloading torch-1.9.0-cp38-cp38-manylinux1_x86_64.whl (831.4 MB)
...
Installing collected packages: typing-extensions, torch, pillow, torchvision
Successfully installed pillow-8.3.0 torch-1.9.0 torchvision-0.10.0 typing-extensions-3.10.0.0

from mxnet import autograd, np, npx
npx.set_np()
x = np.arange(4.0)

x.attach_grad()
with autograd.record():
    y = 2 * np.dot(x, x)
y.backward()
x.grad

import torch
x = torch.arange(4.0)
x.requires_grad_(True)
y = 2 * torch.dot(x, x)
y.backward()
x.grad

lr = 0.03
num_epochs = 3
net = linreg
loss = squared_loss

for epoch in range(num_epochs):
    for X, y in data_iter(batch_size, features, labels):
        with autograd.record():
            l = loss(net(X, w, b), y)
        l.backward()
        sgd([w, b], lr, batch_size)
    train_l = loss(net(features, w, b), labels)
    print(f'epoch {epoch + 1}, loss {float(train_l.mean()):f}')

lr = 0.03
num_epochs = 3
net = linreg
loss = squared_loss

for epoch in range(num_epochs):
    for X, y in data_iter(batch_size, features, labels):
        l = loss(net(X, w, b), y)
        l.sum().backward()
        sgd([w, b], lr, batch_size)
    with torch.no_grad():
        train_l = loss(net(features, w, b), labels)
        print(f'epoch {epoch + 1}, loss {float(train_l.mean()):f}')

from mxnet import autograd, gluon, np, npx

# ...

dataset = gluon.data.ArrayDataset(features, labels)
data_iter = gluon.data.DataLoader(dataset, batch_size, shuffle=True)

from mxnet.gluon import nn
net = nn.Sequential()
# Gluon 会自动根据之后的数据调整网络大小
# 因为我们数据是 [x1, x2]
# 所以之后 Gluon 模型里参数就是 [w1, w2, b]
# 下面的 1 表示输出的节点数，我们只有一个 y
net.add(nn.Dense(1))

from mxnet import init
# 这里的初始化实际上被留到了知道网络实际大小之后再进行
net.initialize(init.Normal(sigma=0.01))

loss = gluon.loss.L2Loss()
trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.03})
num_epochs = 3
for epoch in range(num_epochs):
    for X, y in data_iter:
        with autograd.record():
            l = loss(net(X), y)
        l.backward()
        trainer.step(batch_size)
    l = loss(net(features), labels)
    print(f'epoch {epoch + 1}, loss {l.mean().asnumpy():f}')

# 获取参数
w = net[0].weight.data()
b = net[0].bias.data()

import torch
from torch.utils import data

# ...

dataset = data.TensorDataset(features, labels)
data_iter = data.DataLoader(dataset, batch_size, shuffle=True)

from torch import nn
net = nn.Sequential(nn.Linear(2, 1))

net[0].weight.data.normal_(0, 0.01) # [w1, w2]
net[0].bias.data.fill_(0)           # [b]

loss = nn.MSELoss()
trainer = torch.optim.SGD(net.parameters(), lr=0.03)
num_epochs = 3
for epoch in range(num_epochs):
    for X, y in data_iter:
        l = loss(net(X), y)
        trainer.zero_grad()
        l.backward()
        trainer.step()
    l = loss(net(features), labels)
    print(f'epoch {epoch + 1}, loss {l:f}')

# 获取参数
w = net[0].weight.data()
b = net[0].bias.data()

net = nn.Sequential()
net.add(nn.Dense(256, activation='relu'), nn.Dense(10))
net.initialize(init.Normal(sigma=0.01))

net = nn.Sequential(nn.Flatten(), nn.Linear(784, 256), nn.ReLU(),
                    nn.Linear(256, 10))

def init_weights(m):
    if type(m) == nn.Linear:
        nn.init.normal_(m.weight, std=0.01)

net.apply(init_weights);

# MXNet
trainer = gluon.Trainer(net.collect_params(), 'sgd', {
    'learning_rate': lr,
    'wd': wd})
# 取消常数项的 weight decay
net.collect_params('.*bias').setattr('wd_mult', 0)
# PyTorch
# 只给 wight 添加 weight decay
trainer = torch.optim.SGD([
    {
        "params": net[0].weight,
        'weight_decay': wd
    }, {
        "params": net[0].bias
    }], lr=lr)

# MXNet
mask = np.random.uniform(0, 1, X.shape) > dropout
return mask.astype(np.float32) * X / (1.0 - dropout)
# PyTorch
mask = (torch.rand(X.shape) > dropout).float()
return mask * X / (1.0 - dropout)

# MXNet
net = nn.Sequential()
net.add(
    nn.Dense(256, activation="relu"),
    nn.Dropout(dropout1), nn.Dense(256, activation="relu"),
    nn.Dropout(dropout2), nn.Dense(10))
net.initialize(init.Normal(sigma=0.01))

# PyTorch
net = nn.Sequential(
    nn.Flatten(), nn.Linear(784, 256), nn.ReLU(),
    nn.Dropout(dropout1), nn.Linear(256, 256), nn.ReLU(),
    nn.Dropout(dropout2), nn.Linear(256, 10))
def init_weights(m):
    if type(m) == nn.Linear:
        nn.init.normal_(m.weight, std=0.01)
net.apply(init_weights);

# MXNet
net.save_parameters('mlp.params')
clone = YourModel()
clone.load_parameters('mlp.params')
# PyTorch
torch.save(net.state_dict(), 'mlp.params')
clone = YourModel()
clone.load_state_dict(torch.load('mlp.params'))
clone.eval()

# MXNet
X = np.ones((2, 3), ctx=npx.gpu(0))
X.copyto(npx.cpu())
X.as_in_ctx(npx.gpu(0)) # 若已处于设备则不复制
# PyTorch
X = torch.ones(2, 3, device=torch.device('cuda'))
Z = X.cpu()
X.cuda(0) # 若已处于设备则不复制

# MXNet
net.initialize(ctx=npx.gpu())
# PyTorch
net = net.to(device=torch.cuda())

# MXNet
# nn.Conv2D(out_channel_count, kernel_size, ...)
conv2d = nn.Conv2D(1, kernel_size=3, padding=(2, 1))
# PyTorch
# nn.Conv2d(in_channel_count, out_channel_count, ...)
conv2d = nn.Conv2d(1, 1, kernel_size=3, padding=1)

# MXNet
# 还有 nn.AvgPool2D
nn.MaxPool2D(pool_size, strides=2, padding=(1, 2))
# PyTorch
# 还有 nn.AvgPool2d
nn.MaxPool2d(pool_size, stride=2, padding=(1, 2))