ML2023Spring-hw2 Bossline 0.83448

Homework 2: Phoneme Classification

• Solve a classification problem with deep neural networks (DNNs).

• Understand recursive neural networks (RNNs).

Download Data

• libriphone/train_split.txt: training metadata

• libriphone/train_labels: training labels

• libriphone/test_split.txt: testing metadata

• libriphone/feat/train/*.pt: training feature

• libriphone/feat/test/*.pt: testing feature

Notes: if the google drive link is dead, you can download the data directly from [Kaggle](https://www.kaggle.com/c/ml2023spring-hw2/data "Kaggle") and upload it to the workspace.

# !pip install --upgrade gdown

# # Main link
# # 如果你已经从其他地方下载了数据集，则不需要执行该代码块
# !gdown --id '1qzCRnywKh30mTbWUEjXuNT2isOCAPdO1' --output libriphone.zip

# !unzip -q libriphone.zip
# !ls libriphone

Some Utility Functions

import numpy as np
import torch
import random

def same_seeds(seed):
    random.seed(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    if torch.cuda.is_available():
        torch.cuda.manual_seed(seed)
        torch.cuda.manual_seed_all(seed)
    torch.backends.cudnn.benchmark = False
    torch.backends.cudnn.deterministic = True

import os
import torch
from tqdm import tqdm

def load_feat(path):
    feat = torch.load(path)	# 导入音频文件
    return feat

def shift(x, n):
    # 平移数据
    if n < 0:
        left = x[0].repeat(-n, 1)
        right = x[:n]
    elif n > 0:
        right = x[-1].repeat(n, 1)
        left = x[n:]
    else:
        return x

    return torch.cat((left, right), dim=0)

def concat_feat(x, concat_n):
    assert concat_n % 2 == 1 # n must be odd
    if concat_n < 2:
        return x
    seq_len, feature_dim = x.size(0), x.size(1)
    x = x.repeat(1, concat_n)
    x = x.view(seq_len, concat_n, feature_dim).permute(1, 0, 2) # concat_n, seq_len, feature_dim
    mid = (concat_n // 2)
    for r_idx in range(1, mid+1):
        x[mid + r_idx, :] = shift(x[mid + r_idx], r_idx)
        x[mid - r_idx, :] = shift(x[mid - r_idx], -r_idx)

    return x.permute(1, 0, 2).view(seq_len, concat_n * feature_dim)

def preprocess_data(split, feat_dir, phone_path, concat_nframes, train_ratio=0.8, random_seed=1213):
    class_num = 41 # NOTE: pre-computed, should not need change

    if split == 'train' or split == 'val':
        mode = 'train'
    elif split == 'test':
        mode = 'test'
    else:
        raise ValueError('Invalid \'split\' argument for dataset: PhoneDataset!')

    label_dict = {}
    if mode == 'train':
        for line in open(os.path.join(phone_path, f'{mode}_labels.txt')).readlines():
            line = line.strip('\n').split(' ')
            label_dict[line[0]] = [int(p) for p in line[1:]]

        # split training and validation data
        usage_list = open(os.path.join(phone_path, 'train_split.txt')).readlines()
        random.seed(random_seed)
        random.shuffle(usage_list)
        train_len = int(len(usage_list) * train_ratio)
        usage_list = usage_list[:train_len] if split == 'train' else usage_list[train_len:]

    elif mode == 'test':
        usage_list = open(os.path.join(phone_path, 'test_split.txt')).readlines()

    usage_list = [line.strip('\n') for line in usage_list]
    print('[Dataset] - # phone classes: ' + str(class_num) + ', number of utterances for ' + split + ': ' + str(len(usage_list)))

    max_len = 3000000
    X = torch.empty(max_len, 39 * concat_nframes)
    if mode == 'train':
        y = torch.empty(max_len, dtype=torch.long)

    idx = 0
    for i, fname in tqdm(enumerate(usage_list)):
        feat = load_feat(os.path.join(feat_dir, mode, f'{fname}.pt'))
        cur_len = len(feat)
        feat = concat_feat(feat, concat_nframes)
        if mode == 'train':
            label = torch.LongTensor(label_dict[fname])

        X[idx: idx + cur_len, :] = feat
        if mode == 'train':
            y[idx: idx + cur_len] = label

        idx += cur_len

    X = X[:idx, :]
    if mode == 'train':
        y = y[:idx]

    print(f'[INFO] {split} set')
    print(X.shape)
    if mode == 'train':
        print(y.shape)
        return X, y
    else:
        return X

Dataset

import torch
from torch.utils.data import Dataset

class LibriDataset(Dataset):
    def __init__(self, X, y=None):
        self.data = X
        if y is not None:
            self.label = torch.LongTensor(y)
        else:
            self.label = None

    def __getitem__(self, idx):
        if self.label is not None:
            return self.data[idx], self.label[idx]
        else:
            return self.data[idx]

    def __len__(self):
        return len(self.data)

Model

import torch.nn as nn

class BasicBlock(nn.Module):
    def __init__(self, input_dim, output_dim):
        super(BasicBlock, self).__init__()

        # TODO: apply batch normalization and dropout for strong baseline.
        # Reference: https://pytorch.org/docs/stable/generated/torch.nn.BatchNorm1d.html (batch normalization)
        #       https://pytorch.org/docs/stable/generated/torch.nn.Dropout.html (dropout)
        self.block = nn.Sequential(
            nn.Linear(input_dim, output_dim),
            nn.BatchNorm1d(output_dim),
            nn.ReLU(),
            # 在此处增加 nn.Dropout()
            nn.Dropout(p=0.15)
        )

    def forward(self, x):
        x = self.block(x)
        return x

class Classifier(nn.Module):
    def __init__(self, input_dim, output_dim=41, hidden_layers=1, hidden_dim=256):
        super(Classifier, self).__init__()

        self.fc = nn.Sequential(
            BasicBlock(input_dim, hidden_dim),
            # 在函数的调用中，一个 * 表示将一个序列展开为单独的位置参数，这一行代码是列表推导，最终的表现是重复生成多个 hidden layer
            #（原来的整段代码实际上生成了 hidden_layers+1 个隐藏层，所以我修改了一下代码，让其符合定义）
            *[BasicBlock(hidden_dim, hidden_dim) for _ in range(hidden_layers-1)],
            nn.Linear(hidden_dim, output_dim)
        )

    def forward(self, x):
        x = self.fc(x)
        return x

# TODO: 做 Boss baseline 再取消注释
# class Classifier(nn.Module):
#     def __init__(self, input_dim, output_dim=41, hidden_layers=1, hidden_dim=256):
#         super(Classifier, self).__init__()

#         # TODO: 此时模型超参数在这里修改
#         # Create BiLSTM
#         self.input_size = 39    # 这一项是RNN的"input_dim"，RNN需要对"单"个数据进行处理
#         self.hidden_size = 512  # 这一项是RNN的"hidden_dim"
#         self.num_layers = 6     # 这一项是RNN的"hidden_layers"
#         self.rnn = nn.LSTM(input_size=self.input_size, hidden_size=self.hidden_size, num_layers=self.num_layers, batch_first=True, dropout=0.3, bidirectional=True)

#         # 后接全连接层
#         self.fc = nn.Sequential(
#             # 修改成 2 * self.hidden_size 的原因是因为LSTM()中的bidirectional设置为了True，这表示使用Bi（双向）LSTM模型，所以需要修改输入维度以匹配
#             BasicBlock(2 * self.hidden_size, hidden_dim),
#             # 在函数的调用中，一个 * 表示将一个序列展开为单独的位置参数，这一行代码是列表推导，最终的表现是重复生成多个 hidden layer
#             #（原来的整段代码实际上生成了 hidden_layers+1 个隐藏层，所以我修改了一下代码，让其符合定义）
# #             *[BasicBlock(hidden_dim, hidden_dim) for _ in range(hidden_layers-1)],
#             nn.Linear(hidden_dim, output_dim)
#         )

#     def forward(self, x):
#         # 通过RNN层，得到输出和最后一个隐藏状态，注意输出的shape
#         # x.shape: (batch_size, seq_len, RNN_input_size)
#         x, _ = self.rnn(x)  # => (batch_size, seq_len, RNN_hidden_size)

#         # 取最后一个时间步的输出作为分类的输入
#         x = x[:, -1]        # => (batch_size, RNN_hidden_size)

#         # 通过线性层，得到最终的分类结果
#         x = self.fc(x)      # => (batch_size, labels)

        return x

Hyper-parameters

# Data prarameters
# TODO: change the value of "concat_nframes" for medium baseline
concat_nframes = 21 #61      # the number of frames to concat with, n must be odd (total 2k+1 = n frames)
train_ratio = 0.95           # the ratio of data used for training, the rest will be used for validation

# Training parameters
seed = 1213                  # random seed
batch_size = 512             # batch size
num_epoch = 15               # the number of training epoch
learning_rate =  1e-3        # learning rate
model_path = './model.ckpt'  # the path where the checkpoint will be saved

# Model parameters
# TODO: change the value of "hidden_layers" or "hidden_dim" for medium baseline
input_dim = 39 * concat_nframes  # the input dim of the model, you should not change the value
hidden_layers = 6            # the number of hidden layers
hidden_dim = 512             # the hidden dim

''' 以下是为了完成 report 所添加的代码 '''
# TODO: 完成 report 后注释下面所有代码
# 提前输出模型参数数量，以便调整网络架构
total_params = (
    (input_dim+1) * hidden_dim +
    (hidden_dim + 1) * hidden_dim * (hidden_layers - 1) +
    (hidden_dim + 1) * 41
)
print(f'Total params: {total_params}')

def get_dest_dim(input_dim, output_dim, hidden_layers, dest_hidden_layers, hidden_dim):
    '''获取目标网络隐藏层的维度（总参数量接近于原网络）'''
    # 计算一元二次方程的系数 a,b,c
    a = dest_hidden_layers - 1  # a = l_d - 1
    b = input_dim + output_dim + dest_hidden_layers  #  b = i + o + l_d
    c = - (hidden_layers - 1) * (hidden_dim ** 2) - (input_dim + output_dim + hidden_layers) * hidden_dim  # c = - (l - 1) * (d ** 2) - (i + o + l) * d

    # 计算分子中的平方根部分，即 b^2-4ac
    sqrt_part = (b ** 2) - 4 * a * c

    # 计算两个解，一个是加号，一个是减号，即(-b±√(b^2-4ac))/(2a)
    d_d_plus = (-b + sqrt_part**(0.5)) / (2 * a)
    d_d_minus = (-b - sqrt_part**(0.5)) / (2 * a)

    # 返回两个解的元组
    return (d_d_plus, d_d_minus)

# 设置你想要的目标网络隐藏层数量
dest_hidden_layers = 2

# 获取对应的维数
dest_hidden_dim, _ = get_dest_dim(input_dim, 41, hidden_layers, dest_hidden_layers, hidden_dim)
print(f"若将隐藏层网络层数改为: {dest_hidden_layers}，则维数应当改为: {round(dest_hidden_dim)}",)

Total params: 1754153
若将隐藏层网络层数改为: 2，则维数应当改为: 962

Dataloader

from torch.utils.data import DataLoader
import gc

same_seeds(seed)
device = 'cuda' if torch.cuda.is_available() else 'cpu'
print(f'DEVICE: {device}')

# Preprocess data
train_X, train_y = preprocess_data(split='train', feat_dir='./libriphone/feat', phone_path='./libriphone', concat_nframes=concat_nframes, train_ratio=train_ratio, random_seed=seed)
val_X, val_y = preprocess_data(split='val', feat_dir='./libriphone/feat', phone_path='./libriphone', concat_nframes=concat_nframes, train_ratio=train_ratio, random_seed=seed)

# Get dataset
train_set = LibriDataset(train_X, train_y)
val_set = LibriDataset(val_X, val_y)

# Remove raw feature to save memory
del train_X, train_y, val_X, val_y
gc.collect()

# Get dataloader
train_loader = DataLoader(train_set, batch_size=batch_size, shuffle=True)
val_loader = DataLoader(val_set, batch_size=batch_size, shuffle=False)

DEVICE: cuda
[Dataset] - # phone classes: 41, number of utterances for train: 3257

3257it [00:03, 1026.81it/s]

[INFO] train set
torch.Size([2007632, 819])
torch.Size([2007632])
[Dataset] - # phone classes: 41, number of utterances for val: 172

172it [00:00, 991.69it/s]

[INFO] val set
torch.Size([109162, 819])
torch.Size([109162])

Training

## For plotting learning curve
from torch.utils.tensorboard import SummaryWriter

writer = SummaryWriter() # Tensorboard 画图，结果存储在 ./runs 中

RESUME = False  # 是否导入模型继续跑（在你不小心中断了内核后）

# Create a model, and put it on the device specified.
model = Classifier(input_dim=input_dim, hidden_layers=hidden_layers, hidden_dim=hidden_dim).to(device)

if RESUME:
    model.load_state_dict(torch.load(model_path, map_location='cuda'))

# Define a loss function, and optimizer
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)

# Create a learning rate scheduler
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', factor=0.8, patience=5, threshold=0.05) # 5 轮没有优化（增长率 < threshold)就令 lr *= factor

best_acc = 0.0
for epoch in range(num_epoch):
    train_acc = 0.0
    train_loss = 0.0
    val_acc = 0.0
    val_loss = 0.0

    # ---------- Training ----------
    # Make sure the model is in train mode before training.
    model.train()

    for i, batch in enumerate(tqdm(train_loader)):

        # A batch consists of features data and corresponding labels.
        features, labels = batch  # feature.shape: (batch_size, seq_len * input_size)

        # Forward the data. (Make sure data and model are on the same device.)
        features = features.to(device)
        # TODO: RNN则取消注释下行
        #features = features.view(-1, concat_nframes, 39).to(device) # feature.shape: (batch_size, seq_len, input_size)
        labels = labels.to(device)

        outputs = model(features) # (batch_size, labels)

        # Calculate the cross-entropy loss.
        # We don't need to apply softmax before computing cross-entropy as it is done automatically.
        loss = criterion(outputs, labels)

        # Gradients stored in the parameters in the previous step should be cleared out first.
        optimizer.zero_grad()

        # Compute the gradients for parameters.
        loss.backward()

        # Update the parameters with computed gradients.
        optimizer.step()

        # Get the index of the class with the highest probability
        _, train_pred = torch.max(outputs, 1)

        # Compute the accuracy for current batch.
        train_acc += (train_pred.detach() == labels.detach()).sum().item()
        train_loss += loss.item()

    # ---------- Validation ----------
    # Make sure the model is in eval mode so that some modules like dropout are disabled and work normally.
    model.eval()

    # We don't need gradient in validation.
    # Using torch.no_grad() accelerates the forward process.
    with torch.no_grad():
        for i, batch in enumerate(tqdm(val_loader)):
            features, labels = batch
            features = features.to(device)
            # TODO: RNN则取消注释下行
            #features = features.view(-1, concat_nframes, 39).to(device)
            labels = labels.to(device)
            outputs = model(features)

            # We can still compute the loss (but not the gradient).
            loss = criterion(outputs, labels)

            # Get the index of the class with the highest probability
            _, val_pred = torch.max(outputs, 1)

            # Compute the accuracy for current batch.
            val_acc += (val_pred.cpu() == labels.cpu()).sum().item()
            val_loss += loss.item()

    # Record the accuracy and lr.
    writer.add_scalar('Acc/train', train_acc/len(train_set), epoch)
    writer.add_scalar('Acc/valid', val_acc/len(val_set), epoch)
    writer.add_scalar('lr', optimizer.state_dict()['param_groups'][0]['lr'], epoch)

    # Print the information.
    print(f'[{epoch+1:03d}/{num_epoch:03d}] Train Acc: {train_acc/len(train_set):3.5f} Loss: {train_loss/len(train_loader):3.5f} | Val Acc: {val_acc/len(val_set):3.5f} loss: {val_loss/len(val_loader):3.5f}')

    # If the model improves, save a checkpoint at this epoch
    if val_acc > best_acc:
        best_acc = val_acc
        torch.save(model.state_dict(), model_path)
        print(f'saving model with acc {best_acc/len(val_set):.5f}')

    print(f"{epoch+1} lr: {optimizer.state_dict()['param_groups'][0]['lr']}")

    # Update learning rate based on best loss
    # Modify step() according to your scheduler
    scheduler.step(val_acc/len(val_set))

print(f'saving model with acc {best_acc/len(val_set):.5f}')

100%|██████████████████████████████████████████████████████████| 3922/3922 [00:28<00:00, 137.37it/s]
100%|████████████████████████████████████████████████████████████| 214/214 [00:01<00:00, 200.25it/s]

[001/015] Train Acc: 0.60141 Loss: 1.29663 | Val Acc: 0.66148 loss: 1.07313
saving model with acc 0.66148
1 lr: 0.001

100%|██████████████████████████████████████████████████████████| 3922/3922 [00:26<00:00, 146.95it/s]
100%|████████████████████████████████████████████████████████████| 214/214 [00:01<00:00, 200.57it/s]

[002/015] Train Acc: 0.65867 Loss: 1.08783 | Val Acc: 0.68738 loss: 0.98166
saving model with acc 0.68738
2 lr: 0.001

100%|██████████████████████████████████████████████████████████| 3922/3922 [00:26<00:00, 146.91it/s]
100%|████████████████████████████████████████████████████████████| 214/214 [00:01<00:00, 195.51it/s]

[003/015] Train Acc: 0.68086 Loss: 1.01169 | Val Acc: 0.69976 loss: 0.93793
saving model with acc 0.69976
3 lr: 0.001

100%|██████████████████████████████████████████████████████████| 3922/3922 [00:26<00:00, 147.15it/s]
100%|████████████████████████████████████████████████████████████| 214/214 [00:01<00:00, 200.29it/s]

[004/015] Train Acc: 0.69364 Loss: 0.96641 | Val Acc: 0.70617 loss: 0.91173
saving model with acc 0.70617
4 lr: 0.001

100%|██████████████████████████████████████████████████████████| 3922/3922 [00:26<00:00, 146.43it/s]
100%|████████████████████████████████████████████████████████████| 214/214 [00:01<00:00, 200.32it/s]

[005/015] Train Acc: 0.70306 Loss: 0.93281 | Val Acc: 0.71353 loss: 0.88949
saving model with acc 0.71353
5 lr: 0.001

100%|██████████████████████████████████████████████████████████| 3922/3922 [00:26<00:00, 147.36it/s]
100%|████████████████████████████████████████████████████████████| 214/214 [00:01<00:00, 206.16it/s]

[006/015] Train Acc: 0.71054 Loss: 0.90720 | Val Acc: 0.71871 loss: 0.87763
saving model with acc 0.71871
6 lr: 0.001

100%|██████████████████████████████████████████████████████████| 3922/3922 [00:26<00:00, 147.66it/s]
100%|████████████████████████████████████████████████████████████| 214/214 [00:01<00:00, 200.11it/s]

[007/015] Train Acc: 0.71565 Loss: 0.88763 | Val Acc: 0.72075 loss: 0.86572
saving model with acc 0.72075
7 lr: 0.001

100%|██████████████████████████████████████████████████████████| 3922/3922 [00:26<00:00, 147.42it/s]
100%|████████████████████████████████████████████████████████████| 214/214 [00:01<00:00, 201.18it/s]

[008/015] Train Acc: 0.72075 Loss: 0.87027 | Val Acc: 0.72478 loss: 0.85883
saving model with acc 0.72478
8 lr: 0.001

100%|██████████████████████████████████████████████████████████| 3922/3922 [00:26<00:00, 147.44it/s]
100%|████████████████████████████████████████████████████████████| 214/214 [00:01<00:00, 201.20it/s]

[009/015] Train Acc: 0.72452 Loss: 0.85762 | Val Acc: 0.72654 loss: 0.85334
saving model with acc 0.72654
9 lr: 0.001

100%|██████████████████████████████████████████████████████████| 3922/3922 [00:26<00:00, 147.10it/s]
100%|████████████████████████████████████████████████████████████| 214/214 [00:01<00:00, 200.62it/s]

[010/015] Train Acc: 0.73125 Loss: 0.83234 | Val Acc: 0.73024 loss: 0.84064
saving model with acc 0.73024
10 lr: 0.0008

100%|██████████████████████████████████████████████████████████| 3922/3922 [00:26<00:00, 147.85it/s]
100%|████████████████████████████████████████████████████████████| 214/214 [00:01<00:00, 206.59it/s]

[011/015] Train Acc: 0.73514 Loss: 0.81942 | Val Acc: 0.73024 loss: 0.83678
saving model with acc 0.73024
11 lr: 0.0008

100%|██████████████████████████████████████████████████████████| 3922/3922 [00:26<00:00, 147.95it/s]
100%|████████████████████████████████████████████████████████████| 214/214 [00:01<00:00, 201.13it/s]

[012/015] Train Acc: 0.73723 Loss: 0.81176 | Val Acc: 0.73299 loss: 0.83120
saving model with acc 0.73299
12 lr: 0.0008

100%|██████████████████████████████████████████████████████████| 3922/3922 [00:26<00:00, 148.23it/s]
100%|████████████████████████████████████████████████████████████| 214/214 [00:01<00:00, 200.75it/s]

[013/015] Train Acc: 0.73926 Loss: 0.80464 | Val Acc: 0.73271 loss: 0.82860
13 lr: 0.0008

100%|██████████████████████████████████████████████████████████| 3922/3922 [00:26<00:00, 147.10it/s]
100%|████████████████████████████████████████████████████████████| 214/214 [00:01<00:00, 195.51it/s]

[014/015] Train Acc: 0.74099 Loss: 0.79797 | Val Acc: 0.73405 loss: 0.82708
saving model with acc 0.73405
14 lr: 0.0008

100%|██████████████████████████████████████████████████████████| 3922/3922 [00:26<00:00, 147.28it/s]
100%|████████████████████████████████████████████████████████████| 214/214 [00:01<00:00, 201.14it/s]

[015/015] Train Acc: 0.74299 Loss: 0.79146 | Val Acc: 0.73589 loss: 0.82056
saving model with acc 0.73589
15 lr: 0.0008
saving model with acc 0.73589

%reload_ext tensorboard
%tensorboard --logdir=./runs/

ERROR: Failed to launch TensorBoard (exited with 1).
Contents of stderr:
TensorFlow installation not found - running with reduced feature set.

NOTE: Using experimental fast data loading logic. To disable, pass
    "--load_fast=false" and report issues on GitHub. More details:
    https://github.com/tensorflow/tensorboard/issues/4784

Address already in use
Port 6006 is in use by another program. Either identify and stop that program, or start the server with a different port.

del train_set, val_set
del train_loader, val_loader
gc.collect()

25

Testing

# load data
test_X = preprocess_data(split='test', feat_dir='./libriphone/feat', phone_path='./libriphone', concat_nframes=concat_nframes)
test_set = LibriDataset(test_X, None)
test_loader = DataLoader(test_set, batch_size=batch_size, shuffle=False)

[Dataset] - # phone classes: 41, number of utterances for test: 857

857it [00:00, 1057.77it/s]

[INFO] test set
torch.Size([527364, 819])

# load model
model = Classifier(input_dim=input_dim, hidden_layers=hidden_layers, hidden_dim=hidden_dim).to(device)
model.load_state_dict(torch.load(model_path))

<All keys matched successfully>

pred = np.array([], dtype=np.int32)

model.eval()
with torch.no_grad():
    for i, batch in enumerate(tqdm(test_loader)):
        features = batch
        features = features.to(device)
#         features = features.view(-1, concat_nframes, 39).to(device)

        outputs = model(features)

        _, test_pred = torch.max(outputs, 1) # get the index of the class with the highest probability
        pred = np.concatenate((pred, test_pred.cpu().numpy()), axis=0)

100%|██████████████████████████████████████████████████████████| 1031/1031 [00:02<00:00, 378.35it/s]

with open('prediction.csv', 'w') as f:
    f.write('Id,Class\n')
    for i, y in enumerate(pred):
        f.write('{},{}\n'.format(i, y))