From 3b87b8bd271ade1622149516ae14f94b7ff20dd5 Mon Sep 17 00:00:00 2001 From: Tanner Robison Date: Wed, 17 Jun 2026 18:14:20 -0700 Subject: neurobench & memtorch --- .gitignore | 29 +++ Learning_notes/NeuroBench/system track.md | 1 - README.md | 3 + neurobench_testing/README.md | 8 + neurobench_testing/examples/gsc/ANN.py | 49 ++++ neurobench_testing/examples/gsc/GSC_tutorial.ipynb | 273 +++++++++++++++++++++ neurobench_testing/examples/gsc/SNN.py | 17 ++ neurobench_testing/examples/gsc/benchmark_ann.py | 70 ++++++ neurobench_testing/examples/gsc/benchmark_snn.py | 53 ++++ neurobench_testing/examples/gsc/model_data/m5_ann | Bin 0 -> 116267 bytes .../examples/gsc/model_data/s2s_gsc_snntorch | Bin 0 -> 591111 bytes .../gsc/model_data/s2s_gsc_snntorch_1/data.pkl | Bin 0 -> 2149 bytes .../gsc/model_data/s2s_gsc_snntorch_1/version | 1 + neurobench_testing/examples/gsc/train_ANN.py | 112 +++++++++ neurobench_testing/examples/gsc/train_SNN.py | 107 ++++++++ neurobench_testing/memTorch_testing.py | 109 ++++++++ neurobench_testing/neurobench_testing.py | 90 +++++++ neurobench_testing/requirements.txt | 13 + 18 files changed, 934 insertions(+), 1 deletion(-) create mode 100644 neurobench_testing/README.md create mode 100644 neurobench_testing/examples/gsc/ANN.py create mode 100644 neurobench_testing/examples/gsc/GSC_tutorial.ipynb create mode 100644 neurobench_testing/examples/gsc/SNN.py create mode 100644 neurobench_testing/examples/gsc/benchmark_ann.py create mode 100644 neurobench_testing/examples/gsc/benchmark_snn.py create mode 100644 neurobench_testing/examples/gsc/model_data/m5_ann create mode 100644 neurobench_testing/examples/gsc/model_data/s2s_gsc_snntorch create mode 100644 neurobench_testing/examples/gsc/model_data/s2s_gsc_snntorch_1/data.pkl create mode 100644 neurobench_testing/examples/gsc/model_data/s2s_gsc_snntorch_1/version create mode 100644 neurobench_testing/examples/gsc/train_ANN.py create mode 100644 neurobench_testing/examples/gsc/train_SNN.py create mode 100644 neurobench_testing/memTorch_testing.py create mode 100644 neurobench_testing/neurobench_testing.py create mode 100644 neurobench_testing/requirements.txt diff --git a/.gitignore b/.gitignore index dd33554..9127869 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,30 @@ .obsidian +# Environments +.venv/ +venv/ +env/ + +# Python cache files +__pycache__/ +*.pyc + +neurobench_testing/memtorch/ +neurobench_testing/.venv/ + +**/MemTorch/ +**/memtorch/ + +# Block dataset folders +**/data/ +**/datasets/ + +# Block large data files +*.zip +*.tar.gz +*.tgz +*.rar +*.npy +*.npz +*.csv +*.h5 +*.hdf5 diff --git a/Learning_notes/NeuroBench/system track.md b/Learning_notes/NeuroBench/system track.md index 5aa7dad..43529de 100644 --- a/Learning_notes/NeuroBench/system track.md +++ b/Learning_notes/NeuroBench/system track.md @@ -1,6 +1,5 @@ ## metrics **Correctness:** must be measured to verify the validity of the solution. No thresholds are imposed so the benchmark leaderboard must be analysed to evaluate correctness - efficiency trade offs of solutions. - **Timing:** measurements can be either sample throughput or execution time depending on the task. **Efficiency:** blah blah blah diff --git a/README.md b/README.md index afa6b13..7bef651 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,5 @@ # benchmark_suite_IREU A benchmark suite for memristive and memcapacitive computing systems + +`git clone https://github.com/vaila-uoi/memtorch.git` + `cd memtorch & python setup.py install.` diff --git a/neurobench_testing/README.md b/neurobench_testing/README.md new file mode 100644 index 0000000..7fc7003 --- /dev/null +++ b/neurobench_testing/README.md @@ -0,0 +1,8 @@ +#Set Up + +``` +pip install -r requirements.txt + +git clone https://github.com/vaila-uoi/memtorch.git +python setup.py install +``` diff --git a/neurobench_testing/examples/gsc/ANN.py b/neurobench_testing/examples/gsc/ANN.py new file mode 100644 index 0000000..7607833 --- /dev/null +++ b/neurobench_testing/examples/gsc/ANN.py @@ -0,0 +1,49 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim +import torchaudio +import sys + +from tqdm import tqdm + +class M5(nn.Module): + def __init__(self, n_input=1, n_output=35, stride=16, n_channel=32): + super().__init__() + self.conv1 = nn.Conv1d(n_input, n_channel, kernel_size=80, stride=stride) + self.bn1 = nn.BatchNorm1d(n_channel) + self.pool1 = nn.MaxPool1d(4) + self.conv2 = nn.Conv1d(n_channel, n_channel, kernel_size=3) + self.bn2 = nn.BatchNorm1d(n_channel) + self.pool2 = nn.MaxPool1d(4) + self.conv3 = nn.Conv1d(n_channel, 2 * n_channel, kernel_size=3) + self.bn3 = nn.BatchNorm1d(2 * n_channel) + self.pool3 = nn.MaxPool1d(4) + self.conv4 = nn.Conv1d(2 * n_channel, 2 * n_channel, kernel_size=3) + self.bn4 = nn.BatchNorm1d(2 * n_channel) + self.pool4 = nn.MaxPool1d(4) + self.fc1 = nn.Linear(2 * n_channel, n_output) + + # these need to be different ReLU objects so that they can be individually hooked + self.act1 = nn.ReLU() + self.act2 = nn.ReLU() + self.act3 = nn.ReLU() + self.act4 = nn.ReLU() + + def forward(self, x): + x = self.conv1(x) + x = self.act1(self.bn1(x)) + x = self.pool1(x) + x = self.conv2(x) + x = self.act2(self.bn2(x)) + x = self.pool2(x) + x = self.conv3(x) + x = self.act3(self.bn3(x)) + x = self.pool3(x) + x = self.conv4(x) + x = self.act4(self.bn4(x)) + x = self.pool4(x) + x = F.avg_pool1d(x, x.shape[-1]) + x = x.permute(0, 2, 1) + x = self.fc1(x) + return F.log_softmax(x, dim=2) \ No newline at end of file diff --git a/neurobench_testing/examples/gsc/GSC_tutorial.ipynb b/neurobench_testing/examples/gsc/GSC_tutorial.ipynb new file mode 100644 index 0000000..c7a65de --- /dev/null +++ b/neurobench_testing/examples/gsc/GSC_tutorial.ipynb @@ -0,0 +1,273 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "yGm4fad3M-Sr" + }, + "source": [ + "# Google Speech Commands Benchmark Tutorial\n", + "\n", + "This tutorial aims to provide an insight on how the NeuroBench framework is organized and how you can use it to benchmark your own models!\n", + "\n", + "## About Google Speech Commands (GSC):\n", + "Google Speech Commands is a keyword spotting dataset. Voice commands represent a natural and easily accessible modality for human-machine interaction. Keyword detection, in particular, is frequently employed in edge devices that operate in always-listening, wake-up situations, where it triggers more computationally demanding processes such as automatic speech recognition. Keyword spotting finds application in activating voice assistants, speech data mining, audio indexing, and phone call routing. Given that it generally operates in always-on and battery-powered edge scenarios, keyword detection represents a pertinent benchmark for energy-efficient neuromorphic solutions.\n", + "### Dataset:\n", + "The GSC dataset (V2) is a commonly used dataset in assessing the performance of keyword spotting algorithms. The dataset consists of 105,829 1 second utterances of 35 different words from 2,618 distinct speakers. The data is encoded as linear 16-bit, single-channel, pulse code modulated values, at a 16 kHz sampling frequency.\n", + "\n", + "### Benchmark Task:\n", + "The task is to classify keywords from the GSC dataset test split, after training using the train and val splits." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First we will import the relevant libraries. These include the dataset, pre- and post-processors, model wrapper, and benchmark object." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "lqtM6XbMM_hO" + }, + "outputs": [], + "source": [ + "import torch\n", + "# import the dataloader\n", + "from torch.utils.data import DataLoader\n", + "\n", + "# import the dataset, preprocessors and postprocessors you want to use\n", + "from neurobench.datasets import SpeechCommands\n", + "from neurobench.metrics.static import Footprint\n", + "from neurobench.processors.preprocessors import S2SPreProcessor\n", + "from neurobench.processors.postprocessors import ChooseMaxCount\n", + "\n", + "# import the NeuroBench wrapper to wrap the snnTorch model for usage in the NeuroBench framework\n", + "from neurobench.models import SNNTorchModel\n", + "# import the benchmark class\n", + "from neurobench.benchmarks import Benchmark\n", + "\n", + "from neurobench.metrics.workload import (\n", + " ActivationSparsity,\n", + " SynapticOperations,\n", + " ClassificationAccuracy\n", + ")\n", + "from neurobench.metrics.static import (\n", + " Footprint,\n", + " ConnectionSparsity,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "R7HMjVPX7LZh" + }, + "source": [ + "For this tutorial, we will make use of a simple feedforward SNN, written using snnTorch." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "r0yYDNRZ7UxY" + }, + "outputs": [], + "source": [ + "from torch import nn\n", + "import snntorch as snn\n", + "from snntorch import surrogate\n", + "\n", + "beta = 0.9\n", + "spike_grad = surrogate.fast_sigmoid()\n", + "net = nn.Sequential(\n", + " nn.Flatten(),\n", + " nn.Linear(20, 256),\n", + " snn.Leaky(beta=beta, spike_grad=spike_grad, init_hidden=True),\n", + " nn.Linear(256, 256),\n", + " snn.Leaky(beta=beta, spike_grad=spike_grad, init_hidden=True),\n", + " nn.Linear(256, 256),\n", + " snn.Leaky(beta=beta, spike_grad=spike_grad, init_hidden=True),\n", + " nn.Linear(256, 35),\n", + " snn.Leaky(beta=beta, spike_grad=spike_grad, init_hidden=True, output=True),\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "VNIgTfvuOMe-" + }, + "source": [ + "To get started, we will load our desired dataset in a dataloader:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "chZeyUTAOQ6B" + }, + "outputs": [], + "source": [ + "# data in repo root dir\n", + "test_set = SpeechCommands(path=\"../../data/speech_commands/\", subset=\"testing\")\n", + "\n", + "test_set_loader = DataLoader(test_set, batch_size=500, shuffle=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "GTB808RoNXqL" + }, + "source": [ + "Here, we are loading a pre-trained model. The model is wrapped in the SNNTorchModel wrapper, which includes boilerplate inference code and interfaces with the top-level Benchmark class." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "x4jOfnt6OeIH" + }, + "outputs": [], + "source": [ + "net.load_state_dict(torch.load(\"model_data/s2s_gsc_snntorch\", map_location=torch.device('cpu')))\n", + "\n", + "# Wrap our net in the SNNTorchModel wrapper\n", + "model = SNNTorchModel(net)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "UfRfdvXvOqRP" + }, + "source": [ + "Specify any pre-processors and post-processors you want to use. These will be applied to your data before feeding into the model, and to the output spikes respectively.\n", + "Here, we are using the Speech2Spikes pre-processor to convert the keyword audio data to spikes, and the choose_max_count post-processor which returns a classification based on the neuron with the greatest number of spikes." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "3GHY8vTROwzP" + }, + "outputs": [], + "source": [ + "preprocessors = [S2SPreProcessor()]\n", + "postprocessors = [ChooseMaxCount()]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "o9doNsI0O0Jl" + }, + "source": [ + "Next specify the metrics which you want to calculate. The metrics include static metrics, which are computed before any model inference, and workload metrics, which show inference results.\n", + "\n", + "- Footprint: Bytes used to store the model parameters and buffers.\n", + "- Connection sparsity: Proportion of zero weights in the model.\n", + "- Classification accuracy: Accuracy of keyword predictions.\n", + "- Activation sparsity: Proportion of zero activations, averaged over all neurons, timesteps, and samples.\n", + "- Synaptic operations: Number of weight-activation operations, averaged over keyword samples.\n", + " - Effective MACs: Number of non-zero multiply-accumulate synops, where the activations are not spikes with values -1 or 1.\n", + " - Effective ACs: Number of non-zero accumulate synops, where the activations are -1 or 1 only.\n", + " - Dense: Total zero and non-zero synops." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "sDUczVTkPOsQ" + }, + "outputs": [], + "source": [ + "static_metrics = [Footprint, ConnectionSparsity]\n", + "workload_metrics = [ClassificationAccuracy, ActivationSparsity, SynapticOperations]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "KXQYfiJpPTZb" + }, + "source": [ + "Next, we instantiate the benchmark. We pass the model, the dataloader, the preprocessors, the postprocessor and the list of the static and data metrics which we want to measure:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "U0_N96ADPeO5" + }, + "outputs": [], + "source": [ + "benchmark = Benchmark(model, test_set_loader, preprocessors, postprocessors, [static_metrics, workload_metrics])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6ytLJ-dUPp0b" + }, + "source": [ + "Now, let's run the benchmark and print our results!" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Ldww7kiYPsU2" + }, + "outputs": [], + "source": [ + "results = benchmark.run()\n", + "print(results)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Expected output:\n", + "{'footprint': 583900, 'connection_sparsity': 0.0,\n", + "'classification_accuracy': 0.8484325295196562, 'activation_sparsity': 0.9675956131759854, \n", + "'synaptic_operations': {'Effective_MACs': 0.0, 'Effective_ACs': 3556689.9895502045, 'Dense': 29336955.0}}" + ] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.6" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/neurobench_testing/examples/gsc/SNN.py b/neurobench_testing/examples/gsc/SNN.py new file mode 100644 index 0000000..1642e28 --- /dev/null +++ b/neurobench_testing/examples/gsc/SNN.py @@ -0,0 +1,17 @@ +from torch import nn +import snntorch as snn +from snntorch import surrogate + +beta = 0.9 +spike_grad = surrogate.fast_sigmoid() +net = nn.Sequential( + nn.Flatten(), + nn.Linear(20, 256), + snn.Leaky(beta=beta, spike_grad=spike_grad, init_hidden=True), + nn.Linear(256, 256), + snn.Leaky(beta=beta, spike_grad=spike_grad, init_hidden=True), + nn.Linear(256, 256), + snn.Leaky(beta=beta, spike_grad=spike_grad, init_hidden=True), + nn.Linear(256, 35), + snn.Leaky(beta=beta, spike_grad=spike_grad, init_hidden=True, output=True), +) \ No newline at end of file diff --git a/neurobench_testing/examples/gsc/benchmark_ann.py b/neurobench_testing/examples/gsc/benchmark_ann.py new file mode 100644 index 0000000..6f71a1e --- /dev/null +++ b/neurobench_testing/examples/gsc/benchmark_ann.py @@ -0,0 +1,70 @@ +import os +import torch + +from torch.utils.data import DataLoader +import torchaudio + +from neurobench.datasets import SpeechCommands + +from neurobench.models import TorchModel +from neurobench.benchmarks import Benchmark + +from neurobench.processors.abstract import NeuroBenchPreProcessor, NeuroBenchPostProcessor + +from neurobench.metrics.workload import ( + ActivationSparsity, + SynapticOperations, + ClassificationAccuracy +) +from neurobench.metrics.static import ( + Footprint, + ConnectionSparsity, +) + +from ANN import M5 + +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + +file_path = os.path.dirname(os.path.abspath(__file__)) +model_path = os.path.join(file_path, "model_data/m5_ann") +data_dir = os.path.join(file_path, "../../data/speech_commands") # data in repo root dir + +test_set = SpeechCommands(path=data_dir, subset="testing") + +test_set_loader = DataLoader(test_set, batch_size=500, shuffle=True) + +net = M5() +net.load_state_dict(torch.load(model_path, map_location=device)) + +class resample(NeuroBenchPreProcessor): + def __init__(self): + self.resample = torchaudio.transforms.Resample(orig_freq=16000, new_freq=8000).to(device) + + def __call__(self, dataset): + inputs = dataset[0].permute(0, 2, 1) + inputs = self.resample(inputs) + return (inputs, dataset[1]) + +preprocessors = [resample()] + +class convert_to_label(NeuroBenchPostProcessor): + + def __call__(self, output): + return output.argmax(dim=-1).squeeze() + +postprocessors = [convert_to_label()] + +## Define model ## +model = TorchModel(net) + +static_metrics = [Footprint, ConnectionSparsity] +workload_metrics = [ClassificationAccuracy, ActivationSparsity, SynapticOperations] + +benchmark = Benchmark(model, test_set_loader, preprocessors, postprocessors, [static_metrics, workload_metrics]) +results = benchmark.run(device=device) +print(results) + +# Results: +# {'Footprint': 109228, 'ConnectionSparsity': 0.0, +# 'ClassificationAccuracy': 0.8653339412687909, 'ActivationSparsity': 0.3854464619019532, +# 'SynapticOperations': {'Effective_MACs': 1728071.1701953658, 'Effective_ACs': 0.0, 'Dense': 1880256.0}} \ No newline at end of file diff --git a/neurobench_testing/examples/gsc/benchmark_snn.py b/neurobench_testing/examples/gsc/benchmark_snn.py new file mode 100644 index 0000000..48bea78 --- /dev/null +++ b/neurobench_testing/examples/gsc/benchmark_snn.py @@ -0,0 +1,53 @@ +import os +import torch + +from torch.utils.data import DataLoader + +from neurobench.datasets import SpeechCommands +from neurobench.processors.preprocessors import S2SPreProcessor +from neurobench.processors.postprocessors import ChooseMaxCount + +from neurobench.models import SNNTorchModel +from neurobench.benchmarks import Benchmark + +from neurobench.metrics.workload import ( + ActivationSparsity, + SynapticOperations, + ClassificationAccuracy +) +from neurobench.metrics.static import ( + Footprint, + ConnectionSparsity, +) + +from SNN import net + +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + +file_path = os.path.dirname(os.path.abspath(__file__)) +model_path = os.path.join(file_path, "model_data/s2s_gsc_snntorch") +data_dir = os.path.join(file_path, "../../data/speech_commands") # data in repo root dir + +test_set = SpeechCommands(path=data_dir, subset="testing") + +test_set_loader = DataLoader(test_set, batch_size=500, shuffle=True) + +net.load_state_dict(torch.load(model_path, map_location=torch.device('cpu'))) + +## Define model ## +model = SNNTorchModel(net) + +preprocessors = [S2SPreProcessor(device=device)] +postprocessors = [ChooseMaxCount()] + +static_metrics = [Footprint, ConnectionSparsity] +workload_metrics = [ClassificationAccuracy, ActivationSparsity, SynapticOperations] + +benchmark = Benchmark(model, test_set_loader, preprocessors, postprocessors, [static_metrics, workload_metrics]) +results = benchmark.run(device=device) +print(results) + +# Results: +# {'Footprint': 583900, 'ConnectionSparsity': 0.0, +# 'ClassificationAccuracy': 0.85633802969095, 'ActivationSparsity': 0.9668664144456199, +# 'SynapticOperations': {'Effective_MACs': 0.0, 'Effective_ACs': 3289834.3206724217, 'Dense': 29030400.0}} \ No newline at end of file diff --git a/neurobench_testing/examples/gsc/model_data/m5_ann b/neurobench_testing/examples/gsc/model_data/m5_ann new file mode 100644 index 0000000..305c4b8 Binary files /dev/null and b/neurobench_testing/examples/gsc/model_data/m5_ann differ diff --git a/neurobench_testing/examples/gsc/model_data/s2s_gsc_snntorch b/neurobench_testing/examples/gsc/model_data/s2s_gsc_snntorch new file mode 100644 index 0000000..4ed8229 Binary files /dev/null and b/neurobench_testing/examples/gsc/model_data/s2s_gsc_snntorch differ diff --git a/neurobench_testing/examples/gsc/model_data/s2s_gsc_snntorch_1/data.pkl b/neurobench_testing/examples/gsc/model_data/s2s_gsc_snntorch_1/data.pkl new file mode 100644 index 0000000..e2303dd Binary files /dev/null and b/neurobench_testing/examples/gsc/model_data/s2s_gsc_snntorch_1/data.pkl differ diff --git a/neurobench_testing/examples/gsc/model_data/s2s_gsc_snntorch_1/version b/neurobench_testing/examples/gsc/model_data/s2s_gsc_snntorch_1/version new file mode 100644 index 0000000..00750ed --- /dev/null +++ b/neurobench_testing/examples/gsc/model_data/s2s_gsc_snntorch_1/version @@ -0,0 +1 @@ +3 diff --git a/neurobench_testing/examples/gsc/train_ANN.py b/neurobench_testing/examples/gsc/train_ANN.py new file mode 100644 index 0000000..55c5ba5 --- /dev/null +++ b/neurobench_testing/examples/gsc/train_ANN.py @@ -0,0 +1,112 @@ +import torch +import numpy as np +import torch.optim as optim +from tqdm import tqdm +from torch.utils.data import DataLoader +import torchaudio + +import torch.nn.functional as F + + +from neurobench.datasets import SpeechCommands + +from ANN import M5 + +BATCH_SIZE = 256 +NUM_WORKERS = 8 +EPOCHS = 50 + +# Check if GPU is available +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") +print(device) + +# Load the dataset +data_dir = "../../../data/speech_commands/" +train_set = SpeechCommands(path=data_dir, subset="training") +val_set = SpeechCommands(path=data_dir, subset="validation") +test_set = SpeechCommands(path=data_dir, subset="testing") + +# Create the dataloaders +train_loader = DataLoader(train_set, batch_size=BATCH_SIZE, shuffle=True) +val_loader = DataLoader(val_set, batch_size=BATCH_SIZE, shuffle=True) +test_loader = DataLoader(test_set, batch_size=BATCH_SIZE, shuffle=True) + +# Model +model = M5() +model.to(device) + +transform = torchaudio.transforms.Resample(orig_freq=16000, new_freq=8000) +optimizer = optim.Adam(model.parameters(), lr=0.01, weight_decay=0.0001) +scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=20, gamma=0.1) + +def get_likely_index(tensor): + # find most likely label index for each element in the batch + return tensor.argmax(dim=-1) + +def number_of_correct(pred, target): + # count number of correct predictions + return pred.squeeze().eq(target).sum().item() + +def train(model, epoch, log_interval): + model.train() + for batch_idx, (data, target) in enumerate(train_loader): + + data = data.permute(0, 2, 1).to(device) + target = target.to(device) + + # apply transform and model on whole batch directly on device + data = transform(data) + output = model(data) + + # negative log-likelihood for a tensor of size (batch x 1 x n_output) + loss = F.nll_loss(output.squeeze(), target) + + optimizer.zero_grad() + loss.backward() + optimizer.step() + + # print training stats + if batch_idx % log_interval == 0: + print(f"Train Epoch: {epoch} [{batch_idx * len(data)}/{len(train_loader.dataset)} ({100. * batch_idx / len(train_loader):.0f}%)]\tLoss: {loss.item():.6f}") + + +def validate(model, epoch): + model.eval() + correct = 0 + for batch_idx, (data, target) in enumerate(val_loader): + data = data.permute(0, 2, 1).to(device) + target = target.to(device) + + # apply transform and model on whole batch directly on device + data = transform(data) + output = model(data) + + pred = get_likely_index(output) + correct += number_of_correct(pred, target) + + return correct / len(val_loader.dataset) + +# Training Start +best_acc = 0 +for epoch in range(EPOCHS): + print(f"Epoch {epoch}:") + model.train() + train(model, epoch, log_interval=20) + + val_acc = [] + # validate + val_acc.append(validate(model, epoch)) + + print(f"Validation Accuracy: {np.mean(val_acc) * 100:.2f}%") + + if np.mean(val_acc) > best_acc: + print("New Best Validation Accuracy. Saving...") + best_acc = np.mean(val_acc) + torch.save(model.state_dict(), "model_data/m5_ann") + + scheduler.step() + + print(f"---------------------\n") + +# Load the weights into the network for inference +model.load_state_dict(torch.load("model_data/m5_ann")) \ No newline at end of file diff --git a/neurobench_testing/examples/gsc/train_SNN.py b/neurobench_testing/examples/gsc/train_SNN.py new file mode 100644 index 0000000..4c98cbe --- /dev/null +++ b/neurobench_testing/examples/gsc/train_SNN.py @@ -0,0 +1,107 @@ +import torch +import numpy as np +import snntorch.functional as func +import snntorch.surrogate as surrogate +import snntorch.utils as utils + +from tqdm import tqdm +from torch.utils.data import DataLoader + +from neurobench.datasets import SpeechCommands +from neurobench.preprocessing import S2SPreProcessor +from neurobench.postprocessing import choose_max_count + +from SNN import net + +BATCH_SIZE = 5 +NUM_WORKERS = 8 +EPOCHS = 100 + +# Check if GPU is available +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") +print(device) + +# Load the dataset +data_dir = "../../../data/speech_commands/" +train_set = SpeechCommands(path=data_dir, subset="training") +val_set = SpeechCommands(path=data_dir, subset="validation") +test_set = SpeechCommands(path=data_dir, subset="testing") + +s2s = S2SPreProcessor() + +# Create the dataloaders +train_loader = DataLoader(train_set, batch_size=BATCH_SIZE, shuffle=True) +val_loader = DataLoader(val_set, batch_size=BATCH_SIZE, shuffle=True) +test_loader = DataLoader(test_set, batch_size=BATCH_SIZE, shuffle=True) + +# Different parameters to Speech2Spikes make different shapes of data +tmp_dat = torch.stack([train_set[0][0], train_set[1][0]]) +tmp_label = torch.stack([train_set[0][1], train_set[1][1]]) +tmp_out = s2s((tmp_dat, tmp_label)) +num_steps = tmp_out[0].shape[1] +num_feat = tmp_out[0].shape[2] + +# The SNNTorch forward pass +def forward_pass(net, data, num_steps): + spk_rec = [] + utils.reset(net) + for step in range(num_steps): + spk_out, _ = net(data[:, step, ...]) + spk_rec.append(spk_out) + return torch.stack(spk_rec) + +# Send network to device +net.to(device) +optimizer = torch.optim.Adam(net.parameters(), lr=0.001, betas=(0.9, 0.999)) +loss_fn = func.mse_count_loss(correct_rate=0.25, incorrect_rate=0.025) + +# Training Start +best_acc = 0 +for epoch in range(EPOCHS): + print(f"Epoch {epoch}:") + train_loss = [] + train_acc = [] + net.train() + for batch in tqdm(iter(train_loader)): + events, targets = s2s(batch) + events = events.to(device) + targets = targets.to(device) + + spk_rec = forward_pass(net, events, num_steps) + loss_val = loss_fn(spk_rec, targets) + + train_loss.append(loss_val.item()) + train_acc.append(func.accuracy_rate(spk_rec, targets)) + + optimizer.zero_grad() + loss_val.backward() + optimizer.step() + + print(f"Train Loss: {np.mean(train_loss):.3f}") + print(f"Train Accuracy: {np.mean(train_acc) * 100:.2f}%") + + val_loss = [] + val_acc = [] + net.eval() + for batch in tqdm(iter(val_loader)): + events, targets = s2s(batch) + events = events.to(device) + targets = targets.to(device) + + spk_rec = forward_pass(net, events, num_steps) + + val_loss.append(loss_fn(spk_rec, targets).item()) + val_acc.append(func.accuracy_rate(spk_rec, targets)) + + print(f"Validation Loss: {np.mean(val_loss):.3f}") + print(f"Validation Accuracy: {np.mean(val_acc) * 100:.2f}%") + + if np.mean(val_acc) > best_acc: + print("New Best Validation Accuracy. Saving...") + best_acc = np.mean(val_acc) + torch.save(net.state_dict(), "model_data/s2s_gsc_snntorch") + + print(f"---------------------\n") + +# Load the weights into the network for inference +net.load_state_dict(torch.load("model_data/s2s_gsc_snntorch")) diff --git a/neurobench_testing/memTorch_testing.py b/neurobench_testing/memTorch_testing.py new file mode 100644 index 0000000..a44fd98 --- /dev/null +++ b/neurobench_testing/memTorch_testing.py @@ -0,0 +1,109 @@ +from memtorch import memristor +import torch +import torch.nn as nn +import snntorch as snn + +from torch.utils.data import DataLoader + +from neurobench.models import SNNTorchModel +from neurobench.benchmarks import Benchmark, benchmark +from neurobench.datasets import SpeechCommands + +from neurobench.metrics.workload import ( + ActivationSparsity, + SynapticOperations, + ClassificationAccuracy, +) + +from neurobench.metrics.static import ( + Footprint, + ConnectionSparsity, +) + +from neurobench.processors.preprocessors import S2SPreProcessor +from neurobench.processors.postprocessors import ChooseMaxCount + +import copy +from memtorch.mn.Module import patch_model +from memtorch.map.Parameter import naive_map +from memtorch.bh.memristor import VTEAM + +beta = 0.9 + +class SimpleSNN(nn.Module): + def __init__(self): + super().__init__() + + #standard layers + self.fc1 = nn.Linear(20, 128) + self.fc2 = nn.Linear(128, 35) + + #Spiking neurons + self.lif1 = snn.Leaky(beta=beta, init_hidden=True) + self.lif2 = snn.Leaky(beta=beta, init_hidden=True, output=True) + + def forward(self, x): + x = x.view(x.size(0), -1) + + cur1 = self.fc1(x) + spk1 = self.lif1(cur1) + + cur2 = self.fc2(spk1) + spk2, mem2 = self.lif2(cur2) + + return spk2, mem2 + +device = torch.device("cpu") +net = SimpleSNN().to(device) + +#memristor patch +reference_memristor = VTEAM() + +print("Patching model to memristive crossbar") +patched_net = patch_model( + copy.deepcopy(net), + memristor_model=reference_memristor, + memristor_model_params={}, + mapping_routine=naive_map, + transistor=True, + ADC_resolution=8, + use_bindings=False +) + +model = SNNTorchModel(patched_net) + +static_metrics = [Footprint, ConnectionSparsity] +workload_metrics = [ActivationSparsity, SynapticOperations, ClassificationAccuracy] + +# data loader here maybe?? +test_set = SpeechCommands(path="data/SpeechCommands/", subset="testing") +test_set_loader = DataLoader(test_set, batch_size=50, shuffle=True) + +pre_processor = [S2SPreProcessor(device=device)] +post_processor = [ChooseMaxCount()] + +# print("Layer 1 Beta:", net[2].beta) + +benchmark = Benchmark( + model, + test_set_loader, + pre_processor, + post_processor, + [static_metrics, workload_metrics] +) + +results = benchmark.run() +print(results) + +#energy calculations +ENERGY_PER_MAC = 0.9e-12 +ENERGY_PER_AC = 0.1e-12 + +macs = results['SynapticOperations']['Effective_MACs'] +acs = results['SynapticOperations']['Effective_ACs'] + +total_energy = (macs * ENERGY_PER_MAC) + (acs * ENERGY_PER_AC) + +print("Energy Report:") +print(f"Total Operations: {macs} MACS, {acs} acs ") +print(f"Calculated Energy cost: {total_energy} joules per batch") diff --git a/neurobench_testing/neurobench_testing.py b/neurobench_testing/neurobench_testing.py new file mode 100644 index 0000000..338562d --- /dev/null +++ b/neurobench_testing/neurobench_testing.py @@ -0,0 +1,90 @@ +import torch +from torch.utils.data import DataLoader + +from neurobench.datasets import SpeechCommands +from neurobench.processors.preprocessors import S2SPreProcessor +from neurobench.processors.postprocessors import ChooseMaxCount +from neurobench.models import NeuroBenchModel + +from neurobench.models import SNNTorchModel + +from neurobench.metrics.workload import ( + ActivationSparsity, + SynapticOperations, + ClassificationAccuracy, +) + +from neurobench.metrics.static import ( + Footprint, + ConnectionSparsity, +) + +from neurobench.benchmarks import Benchmark + +from torch import nn +import snntorch as snn +from snntorch import surrogate + +beta = 0.5 #this does nothing with the current model +device = torch.device("cpu") +spike_grad = surrogate.fast_sigmoid() +net = nn.Sequential( + nn.Flatten(), + nn.Linear(20, 256), + snn.Leaky(beta=beta, spike_grad=spike_grad, init_hidden=True), + nn.Linear(256, 256), + snn.Leaky(beta=beta, spike_grad=spike_grad, init_hidden=True), + nn.Linear(256, 256), + snn.Leaky(beta=beta, spike_grad=spike_grad, init_hidden=True), + nn.Linear(256, 35), + snn.Leaky(beta=beta, spike_grad=spike_grad, init_hidden=True, output=True), +) + +test_set = SpeechCommands(path="data/SpeechCommands/", subset="testing") +test_set_loader = DataLoader(test_set, batch_size=500, shuffle=True) + +net.load_state_dict(torch.load("examples/gsc/model_data/s2s_gsc_snntorch", map_location=device)) +print("weight before noise: ", net[1].weight[0][0].item()) + +#adds noise for 'simulating' memristors +# not perfect ik but just testing it out +noise = 0.05 +with torch.no_grad(): + for param in net.parameters(): + param.add_(torch.randn_like(param) * noise) + +print("weight after noise: ", net[1].weight[0][0].item()) + +model = SNNTorchModel(net) + +preprocessors = [S2SPreProcessor(device=device)] +postprocessors = [ChooseMaxCount()] + +static_metrics = [Footprint, ConnectionSparsity] +workload_metrics = [ClassificationAccuracy, ActivationSparsity, SynapticOperations] + +print("Layer 1 Beta:", net[2].beta) +print("Layer 2 Beta:", net[4].beta) + +benchmark = Benchmark( + model, + test_set_loader, + preprocessors, + postprocessors, + [static_metrics, workload_metrics] +) +results = benchmark.run() +print(results) + +# Energy calulations +ENERGY_PER_MAC = 0.9e-12 +ENERGY_PER_AC = 0.1e-12 + +macs = results['SynapticOperations']['Effective_MACs'] +acs = results['SynapticOperations']['Effective_ACs'] + +total_energy = (macs * ENERGY_PER_MAC) + (acs * ENERGY_PER_AC) + +print("Energy Report:") +print(f"Total Operations: {macs} MACS, {acs} acs ") +print(f"Calculated Energy cost: {total_energy} joules per batch") diff --git a/neurobench_testing/requirements.txt b/neurobench_testing/requirements.txt new file mode 100644 index 0000000..66dd5cf --- /dev/null +++ b/neurobench_testing/requirements.txt @@ -0,0 +1,13 @@ +torch +torchvision +torchaudio +neurobench +snntorch +tonic +matplotlib +seaborn +lmfit +scikit-learn +ipython +numpy<2.0.0 +pandas>=2.2.0,<3.0.0 -- cgit v1.2.3