6. Loss function

6.1. L1

Computes the mean absolute error.

import torch
import torch.nn as nn

criterion = nn.L1Loss()

outputs = torch.tensor([[0.9, 0.8, 0.7]], requires_grad=True)
labels = torch.tensor([[1.0, 0.9, 0.8]], dtype=torch.float)
loss = criterion(outputs, labels)

print('outputs: ', outputs)
print('labels: ', labels)
print('loss: ', loss)

outputs:  tensor([[0.9000, 0.8000, 0.7000]], requires_grad=True)
labels:  tensor([[1.0000, 0.9000, 0.8000]])
loss:  tensor(0.1000, grad_fn=<L1LossBackward>)

6.2. Smooth L1

import torch
import torch.nn as nn

criterion = nn.SmoothL1Loss()

outputs = torch.tensor([[0.9, 0.8, 0.7]], requires_grad=True)
labels = torch.tensor([[1.0, 0.9, 0.8]], dtype=torch.float)
loss = criterion(outputs, labels)

print('outputs: ', outputs)
print('labels: ', labels)
print('loss: ', loss)

outputs:  tensor([[0.9000, 0.8000, 0.7000]], requires_grad=True)
labels:  tensor([[1.0000, 0.9000, 0.8000]])
loss:  tensor(0.0050, grad_fn=<SmoothL1LossBackward>)

6.3. Mean-Squared Error

criterion = nn.MSELoss()

outputs = torch.tensor([[0.9, 0.8, 0.7]], requires_grad=True)
labels = torch.tensor([[1.0, 0.9, 0.8]], dtype=torch.float)
loss = criterion(outputs, labels)

print('outputs: ', outputs)
print('labels: ', labels)
print('loss: ', loss)

outputs:  tensor([[0.9000, 0.8000, 0.7000]], requires_grad=True)
labels:  tensor([[1.0000, 0.9000, 0.8000]])
loss:  tensor(0.0100, grad_fn=<MseLossBackward>)

6.4. Cross-Entropy

import math

criterion = nn.CrossEntropyLoss()

probs = [math.log(p) for p in [0.9, 0.8, 0.7]]
outputs = torch.tensor([probs], requires_grad=True)
labels = torch.tensor([0], dtype=torch.long)
loss = criterion(outputs, labels)

print('outputs: ', outputs)
print('labels: ', labels)
print('loss: ', loss)

outputs:  tensor([[-0.1054, -0.2231, -0.3567]], requires_grad=True)
labels:  tensor([0])
loss:  tensor(0.9808, grad_fn=<NllLossBackward>)

6.5. Negative Log-Likelihood

criterion = nn.NLLLoss()

probs = [math.log(p) for p in [0.9, 0.8, 0.7]]
outputs = torch.tensor([probs], requires_grad=True)
labels = torch.tensor([0], dtype=torch.long)
loss = criterion(outputs, labels)

print('outputs: ', outputs)
print('labels: ', labels)
print('loss: ', loss)

outputs:  tensor([[-0.1054, -0.2231, -0.3567]], requires_grad=True)
labels:  tensor([0])
loss:  tensor(0.1054, grad_fn=<NllLossBackward>)

6.6. Poisson Negative Log-Likelihood

criterion = nn.PoissonNLLLoss()

probs = [math.log(p) for p in [0.9, 0.8, 0.7]]
outputs = torch.tensor([probs], requires_grad=True)
labels = torch.tensor([0], dtype=torch.long)
loss = criterion(outputs, labels)

print('outputs: ', outputs)
print('labels: ', labels)
print('loss: ', loss)

outputs:  tensor([[-0.1054, -0.2231, -0.3567]], requires_grad=True)
labels:  tensor([0])
loss:  tensor(0.8000, grad_fn=<MeanBackward0>)

6.7. Kullback-Leibler Divergence

criterion = nn.KLDivLoss(reduction='batchmean')

probs = [math.log(p) for p in [0.9, 0.8, 0.7]]
outputs = torch.tensor([probs], requires_grad=True)
labels = torch.tensor([0], dtype=torch.float)
loss = criterion(outputs, labels)

print('outputs: ', outputs)
print('labels: ', labels)
print('loss: ', loss)

outputs:  tensor([[-0.1054, -0.2231, -0.3567]], requires_grad=True)
labels:  tensor([0.])
loss:  tensor(0., grad_fn=<DivBackward0>)

6.8. Binary Cross-Entropy

sigmoid = nn.Sigmoid()
criterion = nn.BCELoss()

probs = [math.log(p) for p in [0.9, 0.8, 0.7]]
outputs = torch.tensor([probs], requires_grad=True)
labels = torch.tensor([[0.9, 0.8, 0.7]], dtype=torch.float)
loss = criterion(sigmoid(outputs), labels)

print('outputs: ', outputs)
print('labels: ', labels)
print('loss: ', loss)

outputs:  tensor([[-0.1054, -0.2231, -0.3567]], requires_grad=True)
labels:  tensor([[0.9000, 0.8000, 0.7000]])
loss:  tensor(0.7611, grad_fn=<BinaryCrossEntropyBackward>)

6.9. Binary Cross-Entropy with Logits

criterion = nn.BCEWithLogitsLoss()

probs = [math.log(p) for p in [0.9, 0.8, 0.7]]
outputs = torch.tensor([probs], requires_grad=True)
labels = torch.tensor([[0.9, 0.8, 0.7]], dtype=torch.float)
loss = criterion(outputs, labels)

print('outputs: ', outputs)
print('labels: ', labels)
print('loss: ', loss)

outputs:  tensor([[-0.1054, -0.2231, -0.3567]], requires_grad=True)
labels:  tensor([[0.9000, 0.8000, 0.7000]])
loss:  tensor(0.7611, grad_fn=<BinaryCrossEntropyWithLogitsBackward>)

6.10. Margin Ranking

criterion = nn.MarginRankingLoss()

x1 = torch.tensor([[0.5, 0.5, 0.5]])
x2 = torch.tensor([[0.4, 0.6, 0.3]])
y = torch.tensor([[1]])

criterion(x1, x2, y)

tensor(0.0333)

criterion = nn.MarginRankingLoss()

x1 = torch.tensor([[0.5, 0.5, 0.5]])
x2 = torch.tensor([[0.4, 0.6, 0.3]])
y = torch.tensor([[-1]])

criterion(x1, x2, y)

tensor(0.1000)

6.11. Soft Margin

criterion = nn.SoftMarginLoss()

x1 = torch.tensor([[0.5, 0.5, 0.5]])
x2 = torch.tensor([[0.4, 0.6, 0.3]])

criterion(x1, x2)

tensor(0.5912)