1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
|
import os
import torch
from torch.utils.data import DataLoader
import torchaudio
from neurobench.datasets import SpeechCommands
from neurobench.models import TorchModel
from neurobench.benchmarks import Benchmark
from neurobench.processors.abstract import NeuroBenchPreProcessor, NeuroBenchPostProcessor
from neurobench.metrics.workload import (
ActivationSparsity,
SynapticOperations,
ClassificationAccuracy
)
from neurobench.metrics.static import (
Footprint,
ConnectionSparsity,
)
from ANN import M5
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
file_path = os.path.dirname(os.path.abspath(__file__))
model_path = os.path.join(file_path, "model_data/m5_ann")
data_dir = os.path.join(file_path, "../../data/speech_commands") # data in repo root dir
test_set = SpeechCommands(path=data_dir, subset="testing")
test_set_loader = DataLoader(test_set, batch_size=500, shuffle=True)
net = M5()
net.load_state_dict(torch.load(model_path, map_location=device))
class resample(NeuroBenchPreProcessor):
def __init__(self):
self.resample = torchaudio.transforms.Resample(orig_freq=16000, new_freq=8000).to(device)
def __call__(self, dataset):
inputs = dataset[0].permute(0, 2, 1)
inputs = self.resample(inputs)
return (inputs, dataset[1])
preprocessors = [resample()]
class convert_to_label(NeuroBenchPostProcessor):
def __call__(self, output):
return output.argmax(dim=-1).squeeze()
postprocessors = [convert_to_label()]
## Define model ##
model = TorchModel(net)
static_metrics = [Footprint, ConnectionSparsity]
workload_metrics = [ClassificationAccuracy, ActivationSparsity, SynapticOperations]
benchmark = Benchmark(model, test_set_loader, preprocessors, postprocessors, [static_metrics, workload_metrics])
results = benchmark.run(device=device)
print(results)
# Results:
# {'Footprint': 109228, 'ConnectionSparsity': 0.0,
# 'ClassificationAccuracy': 0.8653339412687909, 'ActivationSparsity': 0.3854464619019532,
# 'SynapticOperations': {'Effective_MACs': 1728071.1701953658, 'Effective_ACs': 0.0, 'Dense': 1880256.0}}
|
