Table of Contents

AI deep learning

see also:

Introduction

Neural networks

Basic steps

Tensors

def myRelu (x :torch.tensor) -> torch.tensor #takes a tensor as input and outputs a tensor
    return torch.maximum(torch.tensor(0), x) #this converts any negative value to zero
 
def mySigmoid (x :torch.tensor) -> torch.tensor
    return 1 / (1+torch.exp(-x))

NumPy

  • NumPy ONLY works on the CPU and NOT on a GPU!
    • move tensor to CPU BEFORE moving to a NumPy array via: tensor.cpu().numpy()

PyTorch

PyTorch workflow

data import and cleansing

initial set up code

import torch
from torch import nn #nn contains all of pyTorch modules for neural networks
import matplotlib.pyplot as plt # allows visualisation

torch.__version__ #check version

get data into tensor

option 1. create linear data to test
weight = 0.7 # gradient of y = mx + c
bias = 0.3 # Y intercept at x= 0 ie. c
start = 0
end=1
step=0.02
X = torch.arange(start,end,step).unsqueeze(dim=1) #features
y = weight * X + bias #output labels
option 2. Import data
split data into training data and test data
train_split = int(0.8 * len(X) )
X_train, y_train = X[:train_split], y[:train_split]
X_test, y_test = X[train_split:], y[train_split:]
# could make a more random split by using scikit learn train test split (see in machine learning page)
visualise data
def plot_predictions(train_data=X_train, train_labels=y_train, test_data=X_test, test_labels=y_test, predictions=None):

plt.figure(figsize=10,7))
plt.scatter(train_data,train_labels,c="b",s=4,label="Training data") #c is color, b = blue, s is size

plt.scatter(test_data,test_labels,c="g",s=4,label="Testing data") #c is color, g = green, s is size

if predictions is not NONE:
   plt.scatter(test_data, predictions, c="r",s=4,label="Predictions") #c is color, g = green, s is size)
   
plt.legend(prop={"size": 14});

plot_predictions();

move data to correct device

X_train, y_train = X_train.to(device), y_train.to(device)
X_test, y_test = X_test.to(device), y_test.to(device)

build model

# create linear regression  model
from torch import nn

class LinearRegressionModel(nn.Module): #almost everything in PyTorch inherits from nn.Module
  def __init__(self):
    super().__init__()
    self.weights = nn.parameter(torch.randn(1,
                                             requires_grad=True,
                                             dtype=torch.float))
    self.bias = nn.parameter(torch.randn(1,
                                             requires_grad=True,
                                             dtype=torch.float))
    #forward method to define computation
    def forward(self,x:torch.Tensor) -> torch.Tensor: # x is input data eg. training data
        return self.weights * x + self.bias # create random values for weights and bias and then return internal result using the linear regression formula
    #by use of torch.optim it will aim to get as close as possible to best fit values for these using two algorithms behind the scenes:
      #gradient descent - hence requires_grad=True and then uses torch.autograd
      #backpropagation

create the model

model_0 = LinearRegressionModel()

create loss and optimizer functions

# Create the loss function
loss_fn = nn.L1Loss() # MAE loss is same as L1Loss

# Create the optimizer
optimizer = torch.optim.SGD(params=model_0.parameters(), # parameters of target model to optimize
                            lr=0.001) # learning rate (how much the optimizer should change parameters at each step, higher=more (less gives more accuracy but takes more epochs to get there)

fitting model to data - step-wise training and evaluating

torch.manual_seed(42)
torch.cuda.manual_seed(42) 

# Set the number of epochs (how many times the model will pass over the training data)
epochs = 2000

# Create empty loss lists to track values
train_loss_values = []
test_loss_values = []
epoch_count = []

for epoch in range(epochs):
    ### Training

    # Put model in training mode (this is the default state of a model)
    model_0.train()

    # 1. Forward pass on train data using the forward() method inside 
    y_pred = model_0(X_train)
    # print(y_pred)

    # 2. Calculate the loss (how different are our models predictions to the ground truth)
    loss = loss_fn(y_pred, y_train)
 
    # 3. Zero grad of the optimizer
    optimizer.zero_grad()

    # 4. Loss backwards
    loss.backward()

    # 5. Progress the optimizer
    optimizer.step()
    
    ### Testing

    # Put the model in evaluation mode - turns off various settings not needed in evaluation mode
    model_0.eval()

    with torch.inference_mode(): // turn off gradient checking as that is only needed in training mode and is similar to, but faster than torch.no_grad()
      # 1. Forward pass on test data
      test_pred = model_0(X_test)

      # 2. Calculate loss on test data
      test_loss = loss_fn(test_pred, y_test.type(torch.float)) # predictions come in torch.float datatype, so comparisons need to be done with tensors of the same type

      # Print out what's happening but need to convert the tensor values to numpy array values and need to get to CPU if using GPU
      if epoch % 10 == 0:
            epoch_count.append(epoch)
            train_loss_values.append(loss.detach().cpu().numpy())
            test_loss_values.append(test_loss.detach().cpu().numpy())
            print(f"Epoch: {epoch} | MAE Train Loss: {loss} | MAE Test Loss: {test_loss} ")
            print(model_0.state_dict())

improve model

save trained model

load trained model

# Instantiate a new instance of our model (this will be instantiated with random weights)
loaded_model_0 = LinearRegressionModel()

# Load the state_dict of our saved model (this will update the new instance of our model with trained weights)
loaded_model_0.load_state_dict(torch.load(f=MODEL_SAVE_PATH))

# 1. Put the loaded model into evaluation mode
loaded_model_0.eval()

# 2. Use the inference mode context manager to make predictions
with torch.inference_mode():
    loaded_model_preds = loaded_model_0(X_test) # perform a forward pass on the test data with the loaded model
    
# Compare previous model predictions with loaded model predictions (these should be the same)
y_preds == loaded_model_preds

Data classification or grouping with PyTorch

input data

choosing a classification model

  • use tensorflow playground to experiment with various layer architectures - how many layers and neurons / layer works best for your data type

Computer vision with PyTorch and Convolutional Neural Networks (CNN)

Training your own medical small language model