迁移经验汇总¶

import torch
from torch import nn

class LeNet(nn.Module):
    def __init__(self, num_classes=10):
        super(LeNet, self).__init__()
        self.num_classes = num_classes
        self.features = nn.Sequential(
            nn.Conv2d(1, 6, 3, stride=1, padding=1),
            nn.ReLU(),
            nn.MaxPool2d(2, 2),
            nn.Conv2d(6, 16, 5, stride=1, padding=0),
            nn.ReLU(),
            nn.MaxPool2d(2, 2)
        )
        self.fc = nn.Sequential(
            nn.Linear(400, 120),
            nn.Linear(120, 84), nn.Linear(84, num_classes))

    def forward(self, inputs):
        x = self.features(inputs)
        x = torch.flatten(x, 1)
        x = self.fc(x)
        return x

import paddle
from paddle import nn

class LeNet(nn.Layer):
    def __init__(self, num_classes=10):
        super(LeNet, self).__init__()
        self.num_classes = num_classes
        self.features = nn.Sequential(
            nn.Conv2D(1, 6, 3, stride=1, padding=1),
            nn.ReLU(),
            nn.MaxPool2D(2, 2),
            nn.Conv2D(6, 16, 5, stride=1, padding=0),
            nn.ReLU(),
            nn.MaxPool2D(2, 2)
        )
        self.fc = nn.Sequential(
            nn.Linear(400, 120),
            nn.Linear(120, 84), nn.Linear(84, num_classes))

    def forward(self, inputs):
        x = self.features(inputs)
        x = paddle.flatten(x, 1)
        x = self.fc(x)
        return x

from torchvision.models import mobilenet_v3_small

model = mobilenet_v3_small()
torch.save(model.state_dict(), "mobilenet_v3_small.pth")

import numpy as np
import torch
import paddle

def torch2paddle():
    torch_path = "./data/mobilenet_v3_small.pth"
    paddle_path = "./data/mv3_small_paddle.pdparams"
    torch_state_dict = torch.load(torch_path)
    fc_names = ["classifier"]
    paddle_state_dict = {}
    for k in torch_state_dict:
        if "num_batches_tracked" in k:  # 飞桨中无此参数，无需保存
            continue
        v = torch_state_dict[k].detach().cpu().numpy()
        flag = [i in k for i in fc_names]
        if any(flag) and "weight" in k:
            new_shape = [1, 0] + list(range(2, v.ndim))
            print(
                f"name: {k}, ori shape: {v.shape}, new shape: {v.transpose(new_shape).shape}"
            )
            v = v.transpose(new_shape)   # 转置 Linear 层的 weight 参数
        # 将 torch.nn.BatchNorm2d 的参数名称改成 paddle.nn.BatchNorm2D 对应的参数名称
        k = k.replace("running_var", "_variance")
        k = k.replace("running_mean", "_mean")
        # 添加到飞桨权重字典中
        paddle_state_dict[k] = v
    paddle.save(paddle_state_dict, paddle_path)

>>> from torchvision.models import mobilenet_v3_small
>>> model = mobilenet_v3_small()
>>> torch_dict = model.state_dict()
>>> torch_dict.keys()
odict_keys(['features.0.0.weight', 'features.0.1.weight', 'features.0.1.bias', 'features.0.1.running_mean', 'features.0.1.running_var', 'features.0.1.num_batches_tracked', 'features.1.block.0.0.weight', 'features.1.block.0.1.weight', 'features.1.block.0.1.bias', 'features.1.block.0.1.running_mean', 'features.1.block.0.1.running_var', 'features.1.block.0.1.num_batches_tracked', 'features.1.block.1.fc1.weight', 'features.1.block.1.fc1.bias', 'features.1.block.1.fc2.weight', 'features.1.block.1.fc2.bias', 'features.1.block.2.0.weight', 'features.1.block.2.1.weight', 'features.1.block.2.1.bias', 'features.1.block.2.1.running_mean', 'features.1.block.2.1.running_var', 'features.1.block.2.1.num_batches_tracked', 'features.2.block.0.0.weight', 'features.2.block.0.1.weight', 'features.2.block.0.1.bias', 'features.2.block.0.1.running_mean', 'features.2.block.0.1.running_var', 'features.2.block.0.1.num_batches_tracked', 'features.2.block.1.0.weight', 'features.2.block.1.1.weight', 'features.2.block.1.1.bias', 'features.2.block.1.1.running_mean', 'features.2.block.1.1.running_var', 'features.2.block.1.1.num_batches_tracked', 'features.2.block.2.0.weight', 'features.2.block.2.1.weight', 'features.2.block.2.1.bias', 'features.2.block.2.1.running_mean', 'features.2.block.2.1.running_var', 'features.2.block.2.1.num_batches_tracked', 'features.3.block.0.0.weight', 'features.3.block.0.1.weight', 'features.3.block.0.1.bias', 'features.3.block.0.1.running_mean', 'features.3.block.0.1.running_var', 'features.3.block.0.1.num_batches_tracked', 'features.3.block.1.0.weight', 'features.3.block.1.1.weight', 'features.3.block.1.1.bias', 'features.3.block.1.1.running_mean', 'features.3.block.1.1.running_var', 'features.3.block.1.1.num_batches_tracked', 'features.3.block.2.0.weight', 'features.3.block.2.1.weight', 'features.3.block.2.1.bias', 'features.3.block.2.1.running_mean', 'features.3.block.2.1.running_var', 'features.3.block.2.1.num_batches_tracked', 'features.4.block.0.0.weight', 'features.4.block.0.1.weight', 'features.4.block.0.1.bias', 'features.4.block.0.1.running_mean', 'features.4.block.0.1.running_var', 'features.4.block.0.1.num_batches_tracked', 'features.4.block.1.0.weight', 'features.4.block.1.1.weight', 'features.4.block.1.1.bias', 'features.4.block.1.1.running_mean', 'features.4.block.1.1.running_var', 'features.4.block.1.1.num_batches_tracked', 'features.4.block.2.fc1.weight', 'features.4.block.2.fc1.bias', 'features.4.block.2.fc2.weight', 'features.4.block.2.fc2.bias', 'features.4.block.3.0.weight', 'features.4.block.3.1.weight', 'features.4.block.3.1.bias', 'features.4.block.3.1.running_mean', 'features.4.block.3.1.running_var', 'features.4.block.3.1.num_batches_tracked', 'features.5.block.0.0.weight', 'features.5.block.0.1.weight', 'features.5.block.0.1.bias', 'features.5.block.0.1.running_mean', 'features.5.block.0.1.running_var', 'features.5.block.0.1.num_batches_tracked', 'features.5.block.1.0.weight', 'features.5.block.1.1.weight', 'features.5.block.1.1.bias', 'features.5.block.1.1.running_mean', 'features.5.block.1.1.running_var', 'features.5.block.1.1.num_batches_tracked', 'features.5.block.2.fc1.weight', 'features.5.block.2.fc1.bias', 'features.5.block.2.fc2.weight', 'features.5.block.2.fc2.bias', 'features.5.block.3.0.weight', 'features.5.block.3.1.weight', 'features.5.block.3.1.bias', 'features.5.block.3.1.running_mean', 'features.5.block.3.1.running_var', 'features.5.block.3.1.num_batches_tracked', 'features.6.block.0.0.weight', 'features.6.block.0.1.weight', 'features.6.block.0.1.bias', 'features.6.block.0.1.running_mean', 'features.6.block.0.1.running_var', 'features.6.block.0.1.num_batches_tracked', 'features.6.block.1.0.weight', 'features.6.block.1.1.weight', 'features.6.block.1.1.bias', 'features.6.block.1.1.running_mean', 'features.6.block.1.1.running_var', 'features.6.block.1.1.num_batches_tracked', 'features.6.block.2.fc1.weight', 'features.6.block.2.fc1.bias', 'features.6.block.2.fc2.weight', 'features.6.block.2.fc2.bias', 'features.6.block.3.0.weight', 'features.6.block.3.1.weight', 'features.6.block.3.1.bias', 'features.6.block.3.1.running_mean', 'features.6.block.3.1.running_var', 'features.6.block.3.1.num_batches_tracked', 'features.7.block.0.0.weight', 'features.7.block.0.1.weight', 'features.7.block.0.1.bias', 'features.7.block.0.1.running_mean', 'features.7.block.0.1.running_var', 'features.7.block.0.1.num_batches_tracked', 'features.7.block.1.0.weight', 'features.7.block.1.1.weight', 'features.7.block.1.1.bias', 'features.7.block.1.1.running_mean', 'features.7.block.1.1.running_var', 'features.7.block.1.1.num_batches_tracked', 'features.7.block.2.fc1.weight', 'features.7.block.2.fc1.bias', 'features.7.block.2.fc2.weight', 'features.7.block.2.fc2.bias', 'features.7.block.3.0.weight', 'features.7.block.3.1.weight', 'features.7.block.3.1.bias', 'features.7.block.3.1.running_mean', 'features.7.block.3.1.running_var', 'features.7.block.3.1.num_batches_tracked', 'features.8.block.0.0.weight', 'features.8.block.0.1.weight', 'features.8.block.0.1.bias', 'features.8.block.0.1.running_mean', 'features.8.block.0.1.running_var', 'features.8.block.0.1.num_batches_tracked', 'features.8.block.1.0.weight', 'features.8.block.1.1.weight', 'features.8.block.1.1.bias', 'features.8.block.1.1.running_mean', 'features.8.block.1.1.running_var', 'features.8.block.1.1.num_batches_tracked', 'features.8.block.2.fc1.weight', 'features.8.block.2.fc1.bias', 'features.8.block.2.fc2.weight', 'features.8.block.2.fc2.bias', 'features.8.block.3.0.weight', 'features.8.block.3.1.weight', 'features.8.block.3.1.bias', 'features.8.block.3.1.running_mean', 'features.8.block.3.1.running_var', 'features.8.block.3.1.num_batches_tracked', 'features.9.block.0.0.weight', 'features.9.block.0.1.weight', 'features.9.block.0.1.bias', 'features.9.block.0.1.running_mean', 'features.9.block.0.1.running_var', 'features.9.block.0.1.num_batches_tracked', 'features.9.block.1.0.weight', 'features.9.block.1.1.weight', 'features.9.block.1.1.bias', 'features.9.block.1.1.running_mean', 'features.9.block.1.1.running_var', 'features.9.block.1.1.num_batches_tracked', 'features.9.block.2.fc1.weight', 'features.9.block.2.fc1.bias', 'features.9.block.2.fc2.weight', 'features.9.block.2.fc2.bias', 'features.9.block.3.0.weight', 'features.9.block.3.1.weight', 'features.9.block.3.1.bias', 'features.9.block.3.1.running_mean', 'features.9.block.3.1.running_var', 'features.9.block.3.1.num_batches_tracked', 'features.10.block.0.0.weight', 'features.10.block.0.1.weight', 'features.10.block.0.1.bias', 'features.10.block.0.1.running_mean', 'features.10.block.0.1.running_var', 'features.10.block.0.1.num_batches_tracked', 'features.10.block.1.0.weight', 'features.10.block.1.1.weight', 'features.10.block.1.1.bias', 'features.10.block.1.1.running_mean', 'features.10.block.1.1.running_var', 'features.10.block.1.1.num_batches_tracked', 'features.10.block.2.fc1.weight', 'features.10.block.2.fc1.bias', 'features.10.block.2.fc2.weight', 'features.10.block.2.fc2.bias', 'features.10.block.3.0.weight', 'features.10.block.3.1.weight', 'features.10.block.3.1.bias', 'features.10.block.3.1.running_mean', 'features.10.block.3.1.running_var', 'features.10.block.3.1.num_batches_tracked', 'features.11.block.0.0.weight', 'features.11.block.0.1.weight', 'features.11.block.0.1.bias', 'features.11.block.0.1.running_mean', 'features.11.block.0.1.running_var', 'features.11.block.0.1.num_batches_tracked', 'features.11.block.1.0.weight', 'features.11.block.1.1.weight', 'features.11.block.1.1.bias', 'features.11.block.1.1.running_mean', 'features.11.block.1.1.running_var', 'features.11.block.1.1.num_batches_tracked', 'features.11.block.2.fc1.weight', 'features.11.block.2.fc1.bias', 'features.11.block.2.fc2.weight', 'features.11.block.2.fc2.bias', 'features.11.block.3.0.weight', 'features.11.block.3.1.weight', 'features.11.block.3.1.bias', 'features.11.block.3.1.running_mean', 'features.11.block.3.1.running_var', 'features.11.block.3.1.num_batches_tracked', 'features.12.0.weight', 'features.12.1.weight', 'features.12.1.bias', 'features.12.1.running_mean', 'features.12.1.running_var', 'features.12.1.num_batches_tracked', 'classifier.0.weight', 'classifier.0.bias', 'classifier.3.weight', 'classifier.3.bias'])

- 必选项：损失值`loss`, 训练耗时`batch_cost`, 数据读取耗时`reader_cost`。
- 建议项：当前`epoch`, 当前迭代次数`iter`，学习率(`lr`), 准确率(`acc`)等。

def train_one_epoch(model,
                    criterion,
                    optimizer,
                    data_loader,
                    epoch,
                    print_freq):
    model.train()
    # 读取数据总用时
    train_reader_cost = 0.0
    # 训练总用时
    train_run_cost = 0.0
    # 总样本数
    total_samples = 0
    # 准确率
    acc = 0.0
    # 读取数据开始时间
    reader_start = time.time()
    batch_past = 0

    for batch_idx, (image, target) in enumerate(data_loader):
        # 读取数据用时
        train_reader_cost += time.time() - reader_start
        # 记录训练开始的时间
        train_start = time.time()
        # 前向传播
        output = model(image)
        loss = criterion(output, target)
        # 反向传播
        loss.backward()
        # 更新参数
        optimizer.step()
        optimizer.clear_grad()
        # 训练用时
        train_run_cost += time.time() - train_start
        # 记录训练准确率
        acc = utils.accuracy(output, target).item()
        total_samples += image.shape[0]
        batch_past += 1

        if batch_idx > 0 and batch_idx % print_freq == 0:
            # 每隔 print_freq 输出一次
            msg = "[Epoch {}, iter: {}] acc: {:.5f}, lr: {:.5f}, loss: {:.5f}, avg_reader_cost: {:.5f} sec, avg_batch_cost: {:.5f} sec, avg_samples: {}, avg_ips: {:.5f} images/sec.".format(
                epoch, batch_idx, acc / batch_past,
                optimizer.get_lr(),
                loss.item(), train_reader_cost / batch_past,
                (train_reader_cost + train_run_cost) / batch_past,
                total_samples / batch_past,
                total_samples / (train_reader_cost + train_run_cost))
            # 仅需在第一个 device 上 log
            if paddle.distributed.get_rank() <= 0:
                print(msg)
            sys.stdout.flush()
            train_reader_cost = 0.0
            train_run_cost = 0.0
            total_samples = 0
            acc = 0.0
            batch_past = 0

        reader_start = time.time()

train_sampler = paddle.io.RandomSampler(dataset)
train_batch_sampler = paddle.io.BatchSampler(
    sampler=train_sampler, batch_size=args.batch_size)

train_batch_sampler = paddle.io.DistributedBatchSampler(
        dataset=dataset,
        batch_size=args.batch_size,
        shuffle=True,
        drop_last=False
    )

# 如果当前节点用于训练的设备数量大于 1，则初始化并行训练环境
if paddle.distributed.get_world_size() > 1:
    paddle.distributed.init_parallel_env()

if paddle.distributed.get_world_size() > 1:
    model = paddle.DataParallel(model)

if paddle.distributed.get_rank() == 0:
    paddle.save(params_state_dict, model_path + ".pdparams")

def main(args):
    if args.output_dir:
        utils.mkdir(args.output_dir)

    print(args)

    device = paddle.set_device(args.device)

    # 多卡训练环境初始化
    if paddle.distributed.get_world_size() > 1:
        paddle.distributed.init_parallel_env()

    # 数据读取定义
    train_dir = os.path.join(args.data_path, 'train')
    val_dir = os.path.join(args.data_path, 'val')
    dataset, dataset_test, train_sampler, test_sampler = load_data(
        train_dir, val_dir, args)
    train_batch_sampler = train_sampler
    data_loader = paddle.io.DataLoader(
        dataset=dataset,
        num_workers=args.workers,
        return_list=True,
        batch_sampler=train_batch_sampler)
    test_batch_sampler = paddle.io.BatchSampler(
        sampler=test_sampler, batch_size=args.batch_size)
    data_loader_test = paddle.io.DataLoader(
        dataset_test,
        batch_sampler=test_batch_sampler,
        num_workers=args.workers)

    print("Creating model")
    model = paddlevision.models.__dict__[args.model](
        pretrained=args.pretrained)

    # 损失函数定义
    criterion = nn.CrossEntropyLoss()

    lr_scheduler = paddle.optimizer.lr.StepDecay(
        args.lr, step_size=args.lr_step_size, gamma=args.lr_gamma)

    # 优化器定义
    opt_name = args.opt.lower()
    if opt_name == 'sgd':
        optimizer = paddle.optimizer.Momentum(
            learning_rate=lr_scheduler,
            momentum=args.momentum,
            parameters=model.parameters(),
            weight_decay=args.weight_decay)
    elif opt_name == 'rmsprop':
        optimizer = paddle.optimizer.RMSprop(
            learning_rate=lr_scheduler,
            momentum=args.momentum,
            parameters=model.parameters(),
            weight_decay=args.weight_decay,
            eps=0.0316,
            alpha=0.9)
    else:
        raise RuntimeError(
            "Invalid optimizer {}. Only SGD and RMSprop are supported.".format(
                args.opt))

    # 从 checkpoints 恢复训练
    if args.resume:
        layer_state_dict = paddle.load(os.path.join(args.resume, '.pdparams'))
        model.set_state_dict(layer_state_dict)
        opt_state_dict = paddle.load(os.path.join(args.resume, '.pdopt'))
        optimizer.load_state_dict(opt_state_dict)

    # 多卡训练模型初始化
    if paddle.distributed.get_world_size() > 1:
        model = paddle.DataParallel(model)

    if args.test_only and paddle.distributed.get_rank() == 0:
        top1 = evaluate(model, criterion, data_loader_test, device=device)
        return top1

    print("Start training")
    start_time = time.time()
    best_top1 = 0.0

    # 从 start_epoch 开始训练 epochs 轮
    for epoch in range(args.start_epoch, args.epochs):
        train_one_epoch(model, criterion, optimizer, data_loader, device,
                        epoch, args.print_freq)
        lr_scheduler.step()
        # 保存模型及优化器参数，仅需在 0 号卡上保存
        if paddle.distributed.get_rank() == 0:
            top1 = evaluate(model, criterion, data_loader_test, device=device)
            best_top1 = max(best_top1, top1)
            if args.output_dir:
                paddle.save(model.state_dict(),
                            os.path.join(args.output_dir,
                                         'model_{}.pdparams'.format(epoch)))
                paddle.save(optimizer.state_dict(),
                            os.path.join(args.output_dir,
                                         'model_{}.pdopt'.format(epoch)))
                paddle.save(model.state_dict(),
                            os.path.join(args.output_dir, 'latest.pdparams'))
                paddle.save(optimizer.state_dict(),
                            os.path.join(args.output_dir, 'latest.pdopt'))

    total_time = time.time() - start_time
    total_time_str = str(datetime.timedelta(seconds=int(total_time)))
    print('Training time {}'.format(total_time_str))
    return best_top1

a = paddle.range(3)
c = paddle.ones([3])
b = a>1
# 会增加引用的一种写法
c[b] = 0
# 修改后
paddle.where(b, paddle.zeros(c.shape), c)

cross_entropy_loss = paddle.nn.CrossEntropyLoss()
loss = cross_entropy_loss(pred, gt)
# 会导致内存泄漏的操作
loss_total += loss
# 修改后
loss_total += loss.numpy() # 如果可以转化为 numpy
loss_total += loss.detach().clone() # 如果需要持续使用 tensor