1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
|
import os
import torch
from torch.utils.data import DataLoader
from neurobench.datasets import SpeechCommands
from neurobench.processors.preprocessors import S2SPreProcessor
from neurobench.processors.postprocessors import ChooseMaxCount
from neurobench.models import SNNTorchModel
from neurobench.benchmarks import Benchmark
from neurobench.metrics.workload import (
ActivationSparsity,
SynapticOperations,
ClassificationAccuracy
)
from neurobench.metrics.static import (
Footprint,
ConnectionSparsity,
)
from SNN import net
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
file_path = os.path.dirname(os.path.abspath(__file__))
model_path = os.path.join(file_path, "model_data/s2s_gsc_snntorch")
data_dir = os.path.join(file_path, "../../data/speech_commands") # data in repo root dir
test_set = SpeechCommands(path=data_dir, subset="testing")
test_set_loader = DataLoader(test_set, batch_size=500, shuffle=True)
net.load_state_dict(torch.load(model_path, map_location=torch.device('cpu')))
## Define model ##
model = SNNTorchModel(net)
preprocessors = [S2SPreProcessor(device=device)]
postprocessors = [ChooseMaxCount()]
static_metrics = [Footprint, ConnectionSparsity]
workload_metrics = [ClassificationAccuracy, ActivationSparsity, SynapticOperations]
benchmark = Benchmark(model, test_set_loader, preprocessors, postprocessors, [static_metrics, workload_metrics])
results = benchmark.run(device=device)
print(results)
# Results:
# {'Footprint': 583900, 'ConnectionSparsity': 0.0,
# 'ClassificationAccuracy': 0.85633802969095, 'ActivationSparsity': 0.9668664144456199,
# 'SynapticOperations': {'Effective_MACs': 0.0, 'Effective_ACs': 3289834.3206724217, 'Dense': 29030400.0}}
|
