diff options
Diffstat (limited to 'neurobench_testing')
17 files changed, 0 insertions, 1141 deletions
diff --git a/neurobench_testing/README.md b/neurobench_testing/README.md deleted file mode 100644 index 60d3488..0000000 --- a/neurobench_testing/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# Set Up - -all packages needed (maybe some extra oopsie) are listed in the requirement.txt file -except for memtorch which needs to be cloned and compiled locally on linux systems, -I am not sure for windows or mac. - -``` -python3 -m venv .venv -pip install -r requirements.txt - -git clone --recursive https://github.com/coreylammie/MemTorch -python setup.py install -``` - -the memtorch testing file is building off of the example neurobench using memtorch -to simulate the model on a memristive crossbar. diff --git a/neurobench_testing/custom_memristor_model.py b/neurobench_testing/custom_memristor_model.py deleted file mode 100644 index eb91b99..0000000 --- a/neurobench_testing/custom_memristor_model.py +++ /dev/null @@ -1,36 +0,0 @@ -import torch -import memtorch -from memtorch.bh.memristor.Memristor import Memristor -from memtorch.utils import clip, convert_range #idk if ill need this - -class MemtorchMemristor(Memristor): - def __init__( - self, - k_off = 1.0, # switching rate for off state - k_on = -1.0, # switching rate for on state - alpha_off = 5, # exponent controlling nonlinearity - alpha_on = 5, # exponent controlling nonlinearity - i_off = 0.5e-3, # threshhold current to trigger off state - i_on = 0.5e-3, # threshold current to trigger on state - r_on = 1e3, # maximum resistance - r_off = 10e3, # minimum resistance - p = 2, # window function exponent - **kwargs - ): - #initializing base memristor class - super(MemtorchMemristor, self).__init__(r_off=r_off, r_on=r_on, **kwargs) - - # hyper parameters - self.k_off = k_off - self.k_on = k_on - self.alpha_off = alpha_off - self.alpha_on = alpha_on - self.i_on = i_on - self.i_off = i_off - self.p = p - - #state variables - self.w = 0.5 - self.g = 1/self.r_on - - def window() diff --git a/neurobench_testing/examples/gsc/ANN.py b/neurobench_testing/examples/gsc/ANN.py deleted file mode 100644 index 7607833..0000000 --- a/neurobench_testing/examples/gsc/ANN.py +++ /dev/null @@ -1,49 +0,0 @@ -import torch -import torch.nn as nn -import torch.nn.functional as F -import torch.optim as optim -import torchaudio -import sys - -from tqdm import tqdm - -class M5(nn.Module): - def __init__(self, n_input=1, n_output=35, stride=16, n_channel=32): - super().__init__() - self.conv1 = nn.Conv1d(n_input, n_channel, kernel_size=80, stride=stride) - self.bn1 = nn.BatchNorm1d(n_channel) - self.pool1 = nn.MaxPool1d(4) - self.conv2 = nn.Conv1d(n_channel, n_channel, kernel_size=3) - self.bn2 = nn.BatchNorm1d(n_channel) - self.pool2 = nn.MaxPool1d(4) - self.conv3 = nn.Conv1d(n_channel, 2 * n_channel, kernel_size=3) - self.bn3 = nn.BatchNorm1d(2 * n_channel) - self.pool3 = nn.MaxPool1d(4) - self.conv4 = nn.Conv1d(2 * n_channel, 2 * n_channel, kernel_size=3) - self.bn4 = nn.BatchNorm1d(2 * n_channel) - self.pool4 = nn.MaxPool1d(4) - self.fc1 = nn.Linear(2 * n_channel, n_output) - - # these need to be different ReLU objects so that they can be individually hooked - self.act1 = nn.ReLU() - self.act2 = nn.ReLU() - self.act3 = nn.ReLU() - self.act4 = nn.ReLU() - - def forward(self, x): - x = self.conv1(x) - x = self.act1(self.bn1(x)) - x = self.pool1(x) - x = self.conv2(x) - x = self.act2(self.bn2(x)) - x = self.pool2(x) - x = self.conv3(x) - x = self.act3(self.bn3(x)) - x = self.pool3(x) - x = self.conv4(x) - x = self.act4(self.bn4(x)) - x = self.pool4(x) - x = F.avg_pool1d(x, x.shape[-1]) - x = x.permute(0, 2, 1) - x = self.fc1(x) - return F.log_softmax(x, dim=2)
\ No newline at end of file diff --git a/neurobench_testing/examples/gsc/GSC_tutorial.ipynb b/neurobench_testing/examples/gsc/GSC_tutorial.ipynb deleted file mode 100644 index c7a65de..0000000 --- a/neurobench_testing/examples/gsc/GSC_tutorial.ipynb +++ /dev/null @@ -1,273 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "id": "yGm4fad3M-Sr" - }, - "source": [ - "# Google Speech Commands Benchmark Tutorial\n", - "\n", - "This tutorial aims to provide an insight on how the NeuroBench framework is organized and how you can use it to benchmark your own models!\n", - "\n", - "## About Google Speech Commands (GSC):\n", - "Google Speech Commands is a keyword spotting dataset. Voice commands represent a natural and easily accessible modality for human-machine interaction. Keyword detection, in particular, is frequently employed in edge devices that operate in always-listening, wake-up situations, where it triggers more computationally demanding processes such as automatic speech recognition. Keyword spotting finds application in activating voice assistants, speech data mining, audio indexing, and phone call routing. Given that it generally operates in always-on and battery-powered edge scenarios, keyword detection represents a pertinent benchmark for energy-efficient neuromorphic solutions.\n", - "### Dataset:\n", - "The GSC dataset (V2) is a commonly used dataset in assessing the performance of keyword spotting algorithms. The dataset consists of 105,829 1 second utterances of 35 different words from 2,618 distinct speakers. The data is encoded as linear 16-bit, single-channel, pulse code modulated values, at a 16 kHz sampling frequency.\n", - "\n", - "### Benchmark Task:\n", - "The task is to classify keywords from the GSC dataset test split, after training using the train and val splits." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "First we will import the relevant libraries. These include the dataset, pre- and post-processors, model wrapper, and benchmark object." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "lqtM6XbMM_hO" - }, - "outputs": [], - "source": [ - "import torch\n", - "# import the dataloader\n", - "from torch.utils.data import DataLoader\n", - "\n", - "# import the dataset, preprocessors and postprocessors you want to use\n", - "from neurobench.datasets import SpeechCommands\n", - "from neurobench.metrics.static import Footprint\n", - "from neurobench.processors.preprocessors import S2SPreProcessor\n", - "from neurobench.processors.postprocessors import ChooseMaxCount\n", - "\n", - "# import the NeuroBench wrapper to wrap the snnTorch model for usage in the NeuroBench framework\n", - "from neurobench.models import SNNTorchModel\n", - "# import the benchmark class\n", - "from neurobench.benchmarks import Benchmark\n", - "\n", - "from neurobench.metrics.workload import (\n", - " ActivationSparsity,\n", - " SynapticOperations,\n", - " ClassificationAccuracy\n", - ")\n", - "from neurobench.metrics.static import (\n", - " Footprint,\n", - " ConnectionSparsity,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "R7HMjVPX7LZh" - }, - "source": [ - "For this tutorial, we will make use of a simple feedforward SNN, written using snnTorch." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "r0yYDNRZ7UxY" - }, - "outputs": [], - "source": [ - "from torch import nn\n", - "import snntorch as snn\n", - "from snntorch import surrogate\n", - "\n", - "beta = 0.9\n", - "spike_grad = surrogate.fast_sigmoid()\n", - "net = nn.Sequential(\n", - " nn.Flatten(),\n", - " nn.Linear(20, 256),\n", - " snn.Leaky(beta=beta, spike_grad=spike_grad, init_hidden=True),\n", - " nn.Linear(256, 256),\n", - " snn.Leaky(beta=beta, spike_grad=spike_grad, init_hidden=True),\n", - " nn.Linear(256, 256),\n", - " snn.Leaky(beta=beta, spike_grad=spike_grad, init_hidden=True),\n", - " nn.Linear(256, 35),\n", - " snn.Leaky(beta=beta, spike_grad=spike_grad, init_hidden=True, output=True),\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "VNIgTfvuOMe-" - }, - "source": [ - "To get started, we will load our desired dataset in a dataloader:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "chZeyUTAOQ6B" - }, - "outputs": [], - "source": [ - "# data in repo root dir\n", - "test_set = SpeechCommands(path=\"../../data/speech_commands/\", subset=\"testing\")\n", - "\n", - "test_set_loader = DataLoader(test_set, batch_size=500, shuffle=True)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "GTB808RoNXqL" - }, - "source": [ - "Here, we are loading a pre-trained model. The model is wrapped in the SNNTorchModel wrapper, which includes boilerplate inference code and interfaces with the top-level Benchmark class." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "x4jOfnt6OeIH" - }, - "outputs": [], - "source": [ - "net.load_state_dict(torch.load(\"model_data/s2s_gsc_snntorch\", map_location=torch.device('cpu')))\n", - "\n", - "# Wrap our net in the SNNTorchModel wrapper\n", - "model = SNNTorchModel(net)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "UfRfdvXvOqRP" - }, - "source": [ - "Specify any pre-processors and post-processors you want to use. These will be applied to your data before feeding into the model, and to the output spikes respectively.\n", - "Here, we are using the Speech2Spikes pre-processor to convert the keyword audio data to spikes, and the choose_max_count post-processor which returns a classification based on the neuron with the greatest number of spikes." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "3GHY8vTROwzP" - }, - "outputs": [], - "source": [ - "preprocessors = [S2SPreProcessor()]\n", - "postprocessors = [ChooseMaxCount()]" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "o9doNsI0O0Jl" - }, - "source": [ - "Next specify the metrics which you want to calculate. The metrics include static metrics, which are computed before any model inference, and workload metrics, which show inference results.\n", - "\n", - "- Footprint: Bytes used to store the model parameters and buffers.\n", - "- Connection sparsity: Proportion of zero weights in the model.\n", - "- Classification accuracy: Accuracy of keyword predictions.\n", - "- Activation sparsity: Proportion of zero activations, averaged over all neurons, timesteps, and samples.\n", - "- Synaptic operations: Number of weight-activation operations, averaged over keyword samples.\n", - " - Effective MACs: Number of non-zero multiply-accumulate synops, where the activations are not spikes with values -1 or 1.\n", - " - Effective ACs: Number of non-zero accumulate synops, where the activations are -1 or 1 only.\n", - " - Dense: Total zero and non-zero synops." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "sDUczVTkPOsQ" - }, - "outputs": [], - "source": [ - "static_metrics = [Footprint, ConnectionSparsity]\n", - "workload_metrics = [ClassificationAccuracy, ActivationSparsity, SynapticOperations]" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "KXQYfiJpPTZb" - }, - "source": [ - "Next, we instantiate the benchmark. We pass the model, the dataloader, the preprocessors, the postprocessor and the list of the static and data metrics which we want to measure:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "U0_N96ADPeO5" - }, - "outputs": [], - "source": [ - "benchmark = Benchmark(model, test_set_loader, preprocessors, postprocessors, [static_metrics, workload_metrics])" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "6ytLJ-dUPp0b" - }, - "source": [ - "Now, let's run the benchmark and print our results!" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "Ldww7kiYPsU2" - }, - "outputs": [], - "source": [ - "results = benchmark.run()\n", - "print(results)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Expected output:\n", - "{'footprint': 583900, 'connection_sparsity': 0.0,\n", - "'classification_accuracy': 0.8484325295196562, 'activation_sparsity': 0.9675956131759854, \n", - "'synaptic_operations': {'Effective_MACs': 0.0, 'Effective_ACs': 3556689.9895502045, 'Dense': 29336955.0}}" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.6" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/neurobench_testing/examples/gsc/SNN.py b/neurobench_testing/examples/gsc/SNN.py deleted file mode 100644 index 1642e28..0000000 --- a/neurobench_testing/examples/gsc/SNN.py +++ /dev/null @@ -1,17 +0,0 @@ -from torch import nn -import snntorch as snn -from snntorch import surrogate - -beta = 0.9 -spike_grad = surrogate.fast_sigmoid() -net = nn.Sequential( - nn.Flatten(), - nn.Linear(20, 256), - snn.Leaky(beta=beta, spike_grad=spike_grad, init_hidden=True), - nn.Linear(256, 256), - snn.Leaky(beta=beta, spike_grad=spike_grad, init_hidden=True), - nn.Linear(256, 256), - snn.Leaky(beta=beta, spike_grad=spike_grad, init_hidden=True), - nn.Linear(256, 35), - snn.Leaky(beta=beta, spike_grad=spike_grad, init_hidden=True, output=True), -)
\ No newline at end of file diff --git a/neurobench_testing/examples/gsc/benchmark_ann.py b/neurobench_testing/examples/gsc/benchmark_ann.py deleted file mode 100644 index 6f71a1e..0000000 --- a/neurobench_testing/examples/gsc/benchmark_ann.py +++ /dev/null @@ -1,70 +0,0 @@ -import os -import torch - -from torch.utils.data import DataLoader -import torchaudio - -from neurobench.datasets import SpeechCommands - -from neurobench.models import TorchModel -from neurobench.benchmarks import Benchmark - -from neurobench.processors.abstract import NeuroBenchPreProcessor, NeuroBenchPostProcessor - -from neurobench.metrics.workload import ( - ActivationSparsity, - SynapticOperations, - ClassificationAccuracy -) -from neurobench.metrics.static import ( - Footprint, - ConnectionSparsity, -) - -from ANN import M5 - -device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - -file_path = os.path.dirname(os.path.abspath(__file__)) -model_path = os.path.join(file_path, "model_data/m5_ann") -data_dir = os.path.join(file_path, "../../data/speech_commands") # data in repo root dir - -test_set = SpeechCommands(path=data_dir, subset="testing") - -test_set_loader = DataLoader(test_set, batch_size=500, shuffle=True) - -net = M5() -net.load_state_dict(torch.load(model_path, map_location=device)) - -class resample(NeuroBenchPreProcessor): - def __init__(self): - self.resample = torchaudio.transforms.Resample(orig_freq=16000, new_freq=8000).to(device) - - def __call__(self, dataset): - inputs = dataset[0].permute(0, 2, 1) - inputs = self.resample(inputs) - return (inputs, dataset[1]) - -preprocessors = [resample()] - -class convert_to_label(NeuroBenchPostProcessor): - - def __call__(self, output): - return output.argmax(dim=-1).squeeze() - -postprocessors = [convert_to_label()] - -## Define model ## -model = TorchModel(net) - -static_metrics = [Footprint, ConnectionSparsity] -workload_metrics = [ClassificationAccuracy, ActivationSparsity, SynapticOperations] - -benchmark = Benchmark(model, test_set_loader, preprocessors, postprocessors, [static_metrics, workload_metrics]) -results = benchmark.run(device=device) -print(results) - -# Results: -# {'Footprint': 109228, 'ConnectionSparsity': 0.0, -# 'ClassificationAccuracy': 0.8653339412687909, 'ActivationSparsity': 0.3854464619019532, -# 'SynapticOperations': {'Effective_MACs': 1728071.1701953658, 'Effective_ACs': 0.0, 'Dense': 1880256.0}}
\ No newline at end of file diff --git a/neurobench_testing/examples/gsc/benchmark_snn.py b/neurobench_testing/examples/gsc/benchmark_snn.py deleted file mode 100644 index 48bea78..0000000 --- a/neurobench_testing/examples/gsc/benchmark_snn.py +++ /dev/null @@ -1,53 +0,0 @@ -import os -import torch - -from torch.utils.data import DataLoader - -from neurobench.datasets import SpeechCommands -from neurobench.processors.preprocessors import S2SPreProcessor -from neurobench.processors.postprocessors import ChooseMaxCount - -from neurobench.models import SNNTorchModel -from neurobench.benchmarks import Benchmark - -from neurobench.metrics.workload import ( - ActivationSparsity, - SynapticOperations, - ClassificationAccuracy -) -from neurobench.metrics.static import ( - Footprint, - ConnectionSparsity, -) - -from SNN import net - -device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - -file_path = os.path.dirname(os.path.abspath(__file__)) -model_path = os.path.join(file_path, "model_data/s2s_gsc_snntorch") -data_dir = os.path.join(file_path, "../../data/speech_commands") # data in repo root dir - -test_set = SpeechCommands(path=data_dir, subset="testing") - -test_set_loader = DataLoader(test_set, batch_size=500, shuffle=True) - -net.load_state_dict(torch.load(model_path, map_location=torch.device('cpu'))) - -## Define model ## -model = SNNTorchModel(net) - -preprocessors = [S2SPreProcessor(device=device)] -postprocessors = [ChooseMaxCount()] - -static_metrics = [Footprint, ConnectionSparsity] -workload_metrics = [ClassificationAccuracy, ActivationSparsity, SynapticOperations] - -benchmark = Benchmark(model, test_set_loader, preprocessors, postprocessors, [static_metrics, workload_metrics]) -results = benchmark.run(device=device) -print(results) - -# Results: -# {'Footprint': 583900, 'ConnectionSparsity': 0.0, -# 'ClassificationAccuracy': 0.85633802969095, 'ActivationSparsity': 0.9668664144456199, -# 'SynapticOperations': {'Effective_MACs': 0.0, 'Effective_ACs': 3289834.3206724217, 'Dense': 29030400.0}}
\ No newline at end of file diff --git a/neurobench_testing/examples/gsc/model_data/m5_ann b/neurobench_testing/examples/gsc/model_data/m5_ann Binary files differdeleted file mode 100644 index 305c4b8..0000000 --- a/neurobench_testing/examples/gsc/model_data/m5_ann +++ /dev/null diff --git a/neurobench_testing/examples/gsc/model_data/s2s_gsc_snntorch b/neurobench_testing/examples/gsc/model_data/s2s_gsc_snntorch Binary files differdeleted file mode 100644 index 4ed8229..0000000 --- a/neurobench_testing/examples/gsc/model_data/s2s_gsc_snntorch +++ /dev/null diff --git a/neurobench_testing/examples/gsc/model_data/s2s_gsc_snntorch_1/data.pkl b/neurobench_testing/examples/gsc/model_data/s2s_gsc_snntorch_1/data.pkl Binary files differdeleted file mode 100644 index e2303dd..0000000 --- a/neurobench_testing/examples/gsc/model_data/s2s_gsc_snntorch_1/data.pkl +++ /dev/null diff --git a/neurobench_testing/examples/gsc/model_data/s2s_gsc_snntorch_1/version b/neurobench_testing/examples/gsc/model_data/s2s_gsc_snntorch_1/version deleted file mode 100644 index 00750ed..0000000 --- a/neurobench_testing/examples/gsc/model_data/s2s_gsc_snntorch_1/version +++ /dev/null @@ -1 +0,0 @@ -3 diff --git a/neurobench_testing/examples/gsc/train_ANN.py b/neurobench_testing/examples/gsc/train_ANN.py deleted file mode 100644 index 55c5ba5..0000000 --- a/neurobench_testing/examples/gsc/train_ANN.py +++ /dev/null @@ -1,112 +0,0 @@ -import torch -import numpy as np -import torch.optim as optim -from tqdm import tqdm -from torch.utils.data import DataLoader -import torchaudio - -import torch.nn.functional as F - - -from neurobench.datasets import SpeechCommands - -from ANN import M5 - -BATCH_SIZE = 256 -NUM_WORKERS = 8 -EPOCHS = 50 - -# Check if GPU is available -device = torch.device("cuda" if torch.cuda.is_available() else "cpu") -print(device) - -# Load the dataset -data_dir = "../../../data/speech_commands/" -train_set = SpeechCommands(path=data_dir, subset="training") -val_set = SpeechCommands(path=data_dir, subset="validation") -test_set = SpeechCommands(path=data_dir, subset="testing") - -# Create the dataloaders -train_loader = DataLoader(train_set, batch_size=BATCH_SIZE, shuffle=True) -val_loader = DataLoader(val_set, batch_size=BATCH_SIZE, shuffle=True) -test_loader = DataLoader(test_set, batch_size=BATCH_SIZE, shuffle=True) - -# Model -model = M5() -model.to(device) - -transform = torchaudio.transforms.Resample(orig_freq=16000, new_freq=8000) -optimizer = optim.Adam(model.parameters(), lr=0.01, weight_decay=0.0001) -scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=20, gamma=0.1) - -def get_likely_index(tensor): - # find most likely label index for each element in the batch - return tensor.argmax(dim=-1) - -def number_of_correct(pred, target): - # count number of correct predictions - return pred.squeeze().eq(target).sum().item() - -def train(model, epoch, log_interval): - model.train() - for batch_idx, (data, target) in enumerate(train_loader): - - data = data.permute(0, 2, 1).to(device) - target = target.to(device) - - # apply transform and model on whole batch directly on device - data = transform(data) - output = model(data) - - # negative log-likelihood for a tensor of size (batch x 1 x n_output) - loss = F.nll_loss(output.squeeze(), target) - - optimizer.zero_grad() - loss.backward() - optimizer.step() - - # print training stats - if batch_idx % log_interval == 0: - print(f"Train Epoch: {epoch} [{batch_idx * len(data)}/{len(train_loader.dataset)} ({100. * batch_idx / len(train_loader):.0f}%)]\tLoss: {loss.item():.6f}") - - -def validate(model, epoch): - model.eval() - correct = 0 - for batch_idx, (data, target) in enumerate(val_loader): - data = data.permute(0, 2, 1).to(device) - target = target.to(device) - - # apply transform and model on whole batch directly on device - data = transform(data) - output = model(data) - - pred = get_likely_index(output) - correct += number_of_correct(pred, target) - - return correct / len(val_loader.dataset) - -# Training Start -best_acc = 0 -for epoch in range(EPOCHS): - print(f"Epoch {epoch}:") - model.train() - train(model, epoch, log_interval=20) - - val_acc = [] - # validate - val_acc.append(validate(model, epoch)) - - print(f"Validation Accuracy: {np.mean(val_acc) * 100:.2f}%") - - if np.mean(val_acc) > best_acc: - print("New Best Validation Accuracy. Saving...") - best_acc = np.mean(val_acc) - torch.save(model.state_dict(), "model_data/m5_ann") - - scheduler.step() - - print(f"---------------------\n") - -# Load the weights into the network for inference -model.load_state_dict(torch.load("model_data/m5_ann"))
\ No newline at end of file diff --git a/neurobench_testing/examples/gsc/train_SNN.py b/neurobench_testing/examples/gsc/train_SNN.py deleted file mode 100644 index 4c98cbe..0000000 --- a/neurobench_testing/examples/gsc/train_SNN.py +++ /dev/null @@ -1,107 +0,0 @@ -import torch -import numpy as np -import snntorch.functional as func -import snntorch.surrogate as surrogate -import snntorch.utils as utils - -from tqdm import tqdm -from torch.utils.data import DataLoader - -from neurobench.datasets import SpeechCommands -from neurobench.preprocessing import S2SPreProcessor -from neurobench.postprocessing import choose_max_count - -from SNN import net - -BATCH_SIZE = 5 -NUM_WORKERS = 8 -EPOCHS = 100 - -# Check if GPU is available -device = torch.device("cuda" if torch.cuda.is_available() else "cpu") -print(device) - -# Load the dataset -data_dir = "../../../data/speech_commands/" -train_set = SpeechCommands(path=data_dir, subset="training") -val_set = SpeechCommands(path=data_dir, subset="validation") -test_set = SpeechCommands(path=data_dir, subset="testing") - -s2s = S2SPreProcessor() - -# Create the dataloaders -train_loader = DataLoader(train_set, batch_size=BATCH_SIZE, shuffle=True) -val_loader = DataLoader(val_set, batch_size=BATCH_SIZE, shuffle=True) -test_loader = DataLoader(test_set, batch_size=BATCH_SIZE, shuffle=True) - -# Different parameters to Speech2Spikes make different shapes of data -tmp_dat = torch.stack([train_set[0][0], train_set[1][0]]) -tmp_label = torch.stack([train_set[0][1], train_set[1][1]]) -tmp_out = s2s((tmp_dat, tmp_label)) -num_steps = tmp_out[0].shape[1] -num_feat = tmp_out[0].shape[2] - -# The SNNTorch forward pass -def forward_pass(net, data, num_steps): - spk_rec = [] - utils.reset(net) - for step in range(num_steps): - spk_out, _ = net(data[:, step, ...]) - spk_rec.append(spk_out) - return torch.stack(spk_rec) - -# Send network to device -net.to(device) -optimizer = torch.optim.Adam(net.parameters(), lr=0.001, betas=(0.9, 0.999)) -loss_fn = func.mse_count_loss(correct_rate=0.25, incorrect_rate=0.025) - -# Training Start -best_acc = 0 -for epoch in range(EPOCHS): - print(f"Epoch {epoch}:") - train_loss = [] - train_acc = [] - net.train() - for batch in tqdm(iter(train_loader)): - events, targets = s2s(batch) - events = events.to(device) - targets = targets.to(device) - - spk_rec = forward_pass(net, events, num_steps) - loss_val = loss_fn(spk_rec, targets) - - train_loss.append(loss_val.item()) - train_acc.append(func.accuracy_rate(spk_rec, targets)) - - optimizer.zero_grad() - loss_val.backward() - optimizer.step() - - print(f"Train Loss: {np.mean(train_loss):.3f}") - print(f"Train Accuracy: {np.mean(train_acc) * 100:.2f}%") - - val_loss = [] - val_acc = [] - net.eval() - for batch in tqdm(iter(val_loader)): - events, targets = s2s(batch) - events = events.to(device) - targets = targets.to(device) - - spk_rec = forward_pass(net, events, num_steps) - - val_loss.append(loss_fn(spk_rec, targets).item()) - val_acc.append(func.accuracy_rate(spk_rec, targets)) - - print(f"Validation Loss: {np.mean(val_loss):.3f}") - print(f"Validation Accuracy: {np.mean(val_acc) * 100:.2f}%") - - if np.mean(val_acc) > best_acc: - print("New Best Validation Accuracy. Saving...") - best_acc = np.mean(val_acc) - torch.save(net.state_dict(), "model_data/s2s_gsc_snntorch") - - print(f"---------------------\n") - -# Load the weights into the network for inference -net.load_state_dict(torch.load("model_data/s2s_gsc_snntorch")) diff --git a/neurobench_testing/kaevin_memristor.py b/neurobench_testing/kaevin_memristor.py deleted file mode 100644 index b4a4364..0000000 --- a/neurobench_testing/kaevin_memristor.py +++ /dev/null @@ -1,134 +0,0 @@ -import numpy as np -from scipy.integrate import solve_ivp - -# TEAM (Threshold Adaptive Memristor) model as a reusable class -class TEAMMemristor: - - def __init__( - self, - k_off=1, # switching rate for off-state - k_on=-1, # switching rate for on-state - alpha_off=5, # exponent controlling nonlinearity when switching off - alpha_on=5, # exponent controlling nonlinearity when switching on - i_off=0.5e-3, # threshold current to trigger off state switching - i_on=-0.5e-3, # threshold current to trigger on state switching - g_on=1/1e3, # maximum conductance (1 kohm = 1 ms) - g_off=1/10e3, # minimum conductance (10 kohm = 0.1 ms) - w_init=0.5, # initial state variable (0=off, 1=on) - p=2 # window function exponent - ): - self.k_off = k_off - self.k_on = k_on - self.alpha_off = alpha_off - self.alpha_on = alpha_on - self.i_off = i_off - self.i_on = i_on - self.G_on = G_on - self.G_off = G_off - self.w_init = w_init - self.p = p - - def set_state(self, w): - # Update initial condition for next simulation - self.w_init = np.clip(w, 0, 1) - - def window(self, w, i): - # Nonlinear window function: reduces switching rate near the boundaries - w = np.clip(w, 0.0, 1.0) - if i >= 0: - return 1 - w**(2*self.p) # switching off: slower near w=1 - else: - return 1 - (1-w)**(2*self.p) # switching on: slower near w=0 - - def conductance(self, w): - # Linear interpolation between off and on conductance based on state w - w = np.clip(w, 0.0, 1.0) - return self.G_off + w*(self.G_on - self.G_off) - - def dw_dt(self, w, i): - # TEAM state dynamics: dw/dt depends on current magnitude and direction - w = np.clip(w, 0.0, 1.0) - - if i >= self.i_off: # positive current above threshold = switch off - dw = ( - self.k_off - * ((i/self.i_off)-1)**self.alpha_off - * self.window(w, i) - ) - elif i <= self.i_on: # negative current below threshold = switch on - dw = ( - self.k_on - * (((-i)/abs(self.i_on))-1)**self.alpha_on - * self.window(w, i) - ) - else: # between thresholds = no switching - dw = 0.0 - - # Enforce physical bounds: prevent state from leaving [0,1] - if w <= 0 and dw < 0: - dw = 0 - if w >= 1 and dw > 0: - dw = 0 - - return dw - - def simulate(self, - freq=1, # excitation frequency (Hz) - V_amp=1.5, # sinusoid amplitude (V) - cycles=3): # number of periods to simulate - # Solve the memristor ODE for given frequency and voltage amplitude - - def voltage(t): # sinusoidal excitation signal - return V_amp*np.sin(2*np.pi*freq*t) - - def ode(t, y): # dy/dt: current through memristor - w = y[0] - v = voltage(t) - G = self.conductance(w) - i = G*v # Ohm's law: i = G*v - return [self.dw_dt(w, i)] - - T = 1/freq # period - t_end = cycles*T - t_eval = np.linspace(0, t_end, 10000) # dense time grid for smooth curves - - # Solve with RK45, small max step - sol = solve_ivp( - ode, - [0, t_end], - [self.w_init], - t_eval=t_eval, - method='RK45', - max_step=T/1000, # max step keeps resolution within one period - rtol=1e-8, - atol=1e-10 - ) - - raw_w = sol.y[0] - - # Check if numerical solver violated physical bounds - eps = 1e-6 - if np.any(raw_w < -eps) or np.any(raw_w > 1+eps): - print( - "WARNING: solver left bounds " - f"min={raw_w.min():.12f}, " - f"max={raw_w.max():.12f}" - ) - - w = np.clip(raw_w, 0, 1) # enforce bounds just in case - t = sol.t - v = voltage(t) - G = self.conductance(w) - i = G*v # current throughout simulation - - return t, w, v, i - - def resistance(self, w): - # Compute resistance as reciprocal of conductance - return 1/self.conductance(w) - - def reset(self): - # Reset state to default initial condition - self.w_init = 0.5 - - diff --git a/neurobench_testing/memTorch_testing.py b/neurobench_testing/memTorch_testing.py deleted file mode 100644 index 1992e05..0000000 --- a/neurobench_testing/memTorch_testing.py +++ /dev/null @@ -1,169 +0,0 @@ -import sys -import torch -import torch.nn as nn -import snntorch as snn -from snntorch import surrogate - -from torch.utils.data import DataLoader, Subset - -from neurobench.models import SNNTorchModel -from neurobench.benchmarks import Benchmark, benchmark -from neurobench.datasets import SpeechCommands -from neurobench.metrics.workload import ( - ActivationSparsity, - SynapticOperations, - ClassificationAccuracy, -) - -from neurobench.metrics.static import ( - Footprint, - ConnectionSparsity, -) - -from neurobench.processors.preprocessors import S2SPreProcessor -from neurobench.processors.postprocessors import ChooseMaxCount - -import copy -from memtorch.mn.Module import patch_model -from memtorch.map.Parameter import naive_map -from memtorch.bh.memristor import VTEAM -from memtorch.map.Input import naive_scale - -beta = 0.9 -device = torch.device("cpu") -spike_grad = surrogate.fast_sigmoid() -net = nn.Sequential( - nn.Flatten(), - nn.Linear(20, 256), - snn.Leaky(beta=beta, spike_grad=spike_grad, init_hidden=True), - nn.Linear(256, 256), - snn.Leaky(beta=beta, spike_grad=spike_grad, init_hidden=True), - nn.Linear(256, 256), - snn.Leaky(beta=beta, spike_grad=spike_grad, init_hidden=True), - nn.Linear(256, 35), - snn.Leaky(beta=beta, spike_grad=spike_grad, init_hidden=True, output=True), -) - -vteam_params = { - 'time_series_resolution': 1e-3, - 'r_on': 50, - 'r_off': 1000, -} - -net.load_state_dict(torch.load("examples/gsc/model_data/s2s_gsc_snntorch", map_location=device)) - -patched_net = patch_model( - copy.deepcopy(net), - memristor_model=VTEAM, - memristor_model_params=vteam_params, - module_parameters_to_patch=[torch.nn.Linear], - mapping_routine=naive_map, - transistor=True, - tile_shape=(128, 128), - max_input_voltage=0.3, - scaling_routine=naive_scale, - ADC_resolution=16, - use_bindings=True, - verbose=True, -) - -static_metrics = [Footprint, ConnectionSparsity] -workload_metrics = [ActivationSparsity, SynapticOperations, ClassificationAccuracy] - -test_set = SpeechCommands(path="data/SpeechCommands/", subset="testing") - -#shorten data set so I can actually run it lol -tiny_indices = list(range(10)) -tiny_test_set = Subset(test_set, tiny_indices) -test_set_loader = DataLoader(tiny_test_set, batch_size=16, shuffle=True) - -pre_processor = [S2SPreProcessor(device=device)] -post_processor = [ChooseMaxCount()] - -model = SNNTorchModel(net) -benchmark = Benchmark( - model, - test_set_loader, - pre_processor, - post_processor, - [static_metrics, workload_metrics] -) - -print("\n --Checking signal strength--") -dummy_input = torch.randn(2, 20).to(device) - -try: - raw_signal = patched_net(dummy_input) - - if isinstance(raw_signal, tuple) and len(raw_signal) > 1: - voltage_signal = raw_signal[0] - print("Checking Neuron voltages") - else: - voltage_signal = raw_signal - print("Checking RAW OUTPUTS") - - - print(f"Signal Max: {raw_signal.max().item():.8f}") - print(f"Signal Min: {raw_signal.min().item():.8f}") - print(f"Signal Mean: {raw_signal.mean().item():.8f}") - -except Exception as e: - print("Error getting signal:", e) - -sys.exit() - -results = benchmark.run() -print("\n\n----- IDEAL BENCHMARK -----") -print(f"Footprint: {results['Footprint']}") -print(f"Connection Sparsity: {results['ConnectionSparsity']}") -print(f"Activation Sparsity: {results['ActivationSparsity']}") -print(f"Synaptic Operations: {results['SynapticOperations']}") -print(f"Classification Accuracy: {results['ClassificationAccuracy']}\n") - -#energy calculations -ENERGY_PER_MAC = 0.9e-12 -ENERGY_PER_AC = 0.1e-12 - -macs = results['SynapticOperations']['Effective_MACs'] -acs = results['SynapticOperations']['Effective_ACs'] - -total_energy = (macs * ENERGY_PER_MAC) + (acs * ENERGY_PER_AC) - -print("Energy Report:") -print(f"Total Operations: {macs} MACS, {acs} acs ") -print(f"Calculated Energy cost: {total_energy} joules per batch\n\n") - - -model = SNNTorchModel(patched_net) -benchmark = Benchmark( - model, - test_set_loader, - pre_processor, - post_processor, - [static_metrics, workload_metrics] -) - - -with torch.no_grad(): #makes sure its in inference mode - #otherwise you get memory leaks : ( - results = benchmark.run() - -print("\n\n----- MEMRISTOR BENCHMARK -----") -print(f"Footprint: {results['Footprint']}") -print(f"Connection Sparsity: {results['ConnectionSparsity']}") -print(f"Activation Sparsity: {results['ActivationSparsity']}") -print(f"Synaptic Operations: {results['SynapticOperations']}") -print(f"Classification Accuracy: {results['ClassificationAccuracy']}") - -#energy calculations -ENERGY_PER_MAC = 0.9e-12 -ENERGY_PER_AC = 0.1e-12 - -macs = results['SynapticOperations']['Effective_MACs'] -acs = results['SynapticOperations']['Effective_ACs'] - -total_energy = (macs * ENERGY_PER_MAC) + (acs * ENERGY_PER_AC) - -print("Energy Report:") -print(f"Total Operations: {macs} MACS, {acs} acs ") -print(f"Calculated Energy cost: {total_energy} joules per batch") diff --git a/neurobench_testing/neurobench_testing.py b/neurobench_testing/neurobench_testing.py deleted file mode 100644 index 338562d..0000000 --- a/neurobench_testing/neurobench_testing.py +++ /dev/null @@ -1,90 +0,0 @@ -import torch -from torch.utils.data import DataLoader - -from neurobench.datasets import SpeechCommands -from neurobench.processors.preprocessors import S2SPreProcessor -from neurobench.processors.postprocessors import ChooseMaxCount -from neurobench.models import NeuroBenchModel - -from neurobench.models import SNNTorchModel - -from neurobench.metrics.workload import ( - ActivationSparsity, - SynapticOperations, - ClassificationAccuracy, -) - -from neurobench.metrics.static import ( - Footprint, - ConnectionSparsity, -) - -from neurobench.benchmarks import Benchmark - -from torch import nn -import snntorch as snn -from snntorch import surrogate - -beta = 0.5 #this does nothing with the current model -device = torch.device("cpu") -spike_grad = surrogate.fast_sigmoid() -net = nn.Sequential( - nn.Flatten(), - nn.Linear(20, 256), - snn.Leaky(beta=beta, spike_grad=spike_grad, init_hidden=True), - nn.Linear(256, 256), - snn.Leaky(beta=beta, spike_grad=spike_grad, init_hidden=True), - nn.Linear(256, 256), - snn.Leaky(beta=beta, spike_grad=spike_grad, init_hidden=True), - nn.Linear(256, 35), - snn.Leaky(beta=beta, spike_grad=spike_grad, init_hidden=True, output=True), -) - -test_set = SpeechCommands(path="data/SpeechCommands/", subset="testing") -test_set_loader = DataLoader(test_set, batch_size=500, shuffle=True) - -net.load_state_dict(torch.load("examples/gsc/model_data/s2s_gsc_snntorch", map_location=device)) -print("weight before noise: ", net[1].weight[0][0].item()) - -#adds noise for 'simulating' memristors -# not perfect ik but just testing it out -noise = 0.05 -with torch.no_grad(): - for param in net.parameters(): - param.add_(torch.randn_like(param) * noise) - -print("weight after noise: ", net[1].weight[0][0].item()) - -model = SNNTorchModel(net) - -preprocessors = [S2SPreProcessor(device=device)] -postprocessors = [ChooseMaxCount()] - -static_metrics = [Footprint, ConnectionSparsity] -workload_metrics = [ClassificationAccuracy, ActivationSparsity, SynapticOperations] - -print("Layer 1 Beta:", net[2].beta) -print("Layer 2 Beta:", net[4].beta) - -benchmark = Benchmark( - model, - test_set_loader, - preprocessors, - postprocessors, - [static_metrics, workload_metrics] -) -results = benchmark.run() -print(results) - -# Energy calulations -ENERGY_PER_MAC = 0.9e-12 -ENERGY_PER_AC = 0.1e-12 - -macs = results['SynapticOperations']['Effective_MACs'] -acs = results['SynapticOperations']['Effective_ACs'] - -total_energy = (macs * ENERGY_PER_MAC) + (acs * ENERGY_PER_AC) - -print("Energy Report:") -print(f"Total Operations: {macs} MACS, {acs} acs ") -print(f"Calculated Energy cost: {total_energy} joules per batch") diff --git a/neurobench_testing/requirements.txt b/neurobench_testing/requirements.txt deleted file mode 100644 index 193c357..0000000 --- a/neurobench_testing/requirements.txt +++ /dev/null @@ -1,14 +0,0 @@ -torch -torchvision -torchaudio -neurobench -snntorch -tonic -matplotlib -seaborn -lmfit -scikit-learn -ipython -numpy<2.0.0 -pandas>=2.2.0,<3.0.0 -torchcodec |