Master

03_Running/ResNetInferenceInteractive.ipynb

@@ -301,9 +301,9 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python3.7 (ImageCalc)",
+   "display_name": "Python 3",
    "language": "python",
-   "name": "imagecalc"
+   "name": "python3"
   },
   "language_info": {
    "codemirror_mode": {

03_Running/ResNetInferenceV2.ipynb

@@ -0,0 +1,172 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Load library"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from importlib import reload\n",
+    "import RunningResNetLibrary as RTA\n",
+    "from IPython.display import clear_output, display\n",
+    "import os\n",
+    "import subprocess"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Run Program"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "This involves creating an audio buffer that we continually update with information from the microphone, then creating an image and running it through the neural net. This happens as fast as possible over and over again."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def DoStuff(Input,Probablity):\n",
+    "    clear_output(wait=True)\n",
+    "    print(Input,Probablity)\n",
+    "    return;\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def FullOut(predicted,prob,classes):\n",
+    "    clear_output(wait=True)\n",
+    "    print(classes[predicted[0,0]], prob[0,0])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Dogs tensor(0.5714, grad_fn=<SelectBackward>)\n",
+      "Stopping!\n",
+      "Stopped and Done!\n"
+     ]
+    }
+   ],
+   "source": [
+    "RTA.RunTheSystem(TargetTime=20,ModelPath=\"../models/CatDogResNetNew.pth\",CallBackFunction=DoStuff,CallBack2=FullOut)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 104,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "<module 'RunningResNetLibrary' from '/Users/d.goedicke/Desktop/CoctailParty/RealtimeAudioClassification/03_Running/RunningResNetLibrary.py'>"
+      ]
+     },
+     "execution_count": 104,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "reload(RTA)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Loading all relevant data.\n"
+     ]
+    },
+    {
+     "ename": "KeyError",
+     "evalue": "'resolution'",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
+      "\u001b[0;32m<ipython-input-10-7a9696bc83e3>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mRTA\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mRunTheSystem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mTargetTime\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m15\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mModelPath\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"../models/UrbanSoundsClean\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mCallBackFunction\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mDoStuff\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mCallBack2\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mFullOut\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
+      "\u001b[0;32m~/GitRepos/RealtimeAudioClassification/03_Running/RunningResNetLibrary.py\u001b[0m in \u001b[0;36mRunTheSystem\u001b[0;34m(TargetTime, ModelPath, CallBackFunction, CallBack2)\u001b[0m\n\u001b[1;32m    160\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mRunTheSystem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mTargetTime\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m30\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mModelPath\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"../models/UrbanResNet.pth\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mCallBackFunction\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mDoStuff\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mCallBack2\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    161\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Loading all relevant data.\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 162\u001b[0;31m     \u001b[0mStartAudio\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mModelPath\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mModelPath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    163\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Starting Running\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    164\u001b[0m     \u001b[0mt0\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;32m~/GitRepos/RealtimeAudioClassification/03_Running/RunningResNetLibrary.py\u001b[0m in \u001b[0;36mStartAudio\u001b[0;34m(ModelPath)\u001b[0m\n\u001b[1;32m     66\u001b[0m     \u001b[0;32mglobal\u001b[0m \u001b[0mstream\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m     \u001b[0mModelData\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mModelPath\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mmap_location\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'cpu'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 68\u001b[0;31m     \u001b[0mInput_Resolution\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mModelData\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'resolution'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     69\u001b[0m     \u001b[0mSpectrumVariables\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mModelData\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'SpectrumVariables'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     70\u001b[0m     \u001b[0mclasses\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mModelData\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'classes'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+      "\u001b[0;31mKeyError\u001b[0m: 'resolution'"
+     ]
+    }
+   ],
+   "source": [
+    "RTA.RunTheSystem(TargetTime=15,ModelPath=\"../models/UrbanSoundsClean\",CallBackFunction=DoStuff,CallBack2=FullOut)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python3.6 workshop",
+   "language": "python",
+   "name": "workshop"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

03_Running/RunningResNetLibrary.py

@@ -0,0 +1,172 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+## Load library
+import pyaudio
+import librosa
+import numpy as np
+from numpy_ringbuffer import RingBuffer
+import matplotlib.pyplot as plt
+import pyaudio
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torchvision import datasets, transforms, models
+
+import torchvision
+import time
+from  numpy_ringbuffer import RingBuffer
+from torch.autograd import Variable
+from threading import Thread
+from time import sleep
+import cv2
+import time
+import pickle
+from IPython.display import clear_output, display
+#import rtmidi
+from IPython.display import Image 
+import os
+
+
+model=None
+classes=None
+ringBuffer=None
+Input_Resolution=None
+SamplingRate =48000
+SpectrumVariables=None
+ringBuffer = RingBuffer(28672*2)
+pa = None
+stream = None
+RunningAverageSlow={}
+RunningAverageFast={}
+timer = 0
+
+
+def SmoothingFunction(PredictedClassName, Probablity,CallBackFunction):
+    global RunningAverageSlow
+    global RunningAverageFast
+    if(not PredictedClassName in RunningAverageSlow):
+        RunningAverageSlow[PredictedClassName]=0
+    if(not PredictedClassName in RunningAverageFast):
+        RunningAverageFast[PredictedClassName]=0
+
+    RunningAverageFast[PredictedClassName]=RunningAverageFast[PredictedClassName]*0.7+0.3*Probablity
+    RunningAverageSlow[PredictedClassName]=RunningAverageSlow[PredictedClassName]*0.9+0.1*Probablity
+    if(RunningAverageFast[PredictedClassName]-RunningAverageSlow[PredictedClassName])>0.25 and Probablity>2.0:
+        CallBackFunction(PredictedClassName,Probablity)
+        return
+    #print(RunningAverageSlow)
+    CallBackFunction('None',Probablity)
+
+def StartAudio(ModelPath="../models/CatDogResNet.pth"):
+    global model
+    global SpectrumVariables
+    global classes
+    global Input_Resolution
+    global stream
+    ModelData = torch.load(ModelPath,map_location='cpu')
+    Input_Resolution = ModelData['resolution']
+    SpectrumVariables = ModelData['SpectrumVariables']
+    classes = ModelData['classes']
+    foundAModel=False
+    if ModelData['modelType']=="resnet18":
+        model = models.resnet18()
+        model.fc = nn.Linear(512, len(classes))
+        foundAModel=True
+    if not foundAModel:
+        print("Could not find requested Model. Please provide a network structure for model:",ModelData['modelType'])
+        exit()
+    model.load_state_dict (ModelData['model'])
+    model.cpu()
+    model.eval()
+    print("Opening Audio Channel")
+    cv2.startWindowThread() 
+    pa = pyaudio.PyAudio()
+    stream = pa.open(format=pyaudio.paFloat32,
+                     channels=1,
+                     rate=SamplingRate,
+                     output=False,
+                     input=True,
+                     stream_callback=callback)
+    stream.start_stream()
+
+def callback(in_data, frame_count, time_info, flag):
+    audio_data = np.frombuffer(in_data, dtype=np.float32)
+    ringBuffer.extend(audio_data)
+    return None, pyaudio.paContinue
+
+def infere_Class_Type(CallBack,CallBack2):
+    if(not ringBuffer.is_full):
+        return
+    N_FFT=SpectrumVariables["N_FFT"]
+    HOP_LENGTH= SpectrumVariables["HOP_LENGTH"]
+    FMIN=SpectrumVariables["FMIN"]
+    FMAX=SpectrumVariables["FMAX"]
+    N_MELS=SpectrumVariables["N_MELS"]
+    POWER=SpectrumVariables["POWER"]      
+    mel_spec_power = librosa.feature.melspectrogram(np.array(ringBuffer), sr=SamplingRate, n_fft=N_FFT,
+                                                hop_length=HOP_LENGTH,
+                                                n_mels=N_MELS, power=POWER,
+                                               fmin=FMIN,fmax=FMAX)
+    mel_spec_db = librosa.power_to_db(mel_spec_power, ref=np.max)
+    image=mel_spec_db[0:Input_Resolution,0:Input_Resolution]
+    image = mel_spec_db; # convert to float
+    image -= image.min() # ensure the minimal value is 0.0
+    image /= image.max() # maximum value in image is now 1.0
+    image*=256
+    img = image.astype(np.uint8)
+    colerPic = cv2.applyColorMap(img, cv2.COLORMAP_BONE)
+    if(int(np.floor(colerPic.shape[1]/Input_Resolution))<0):
+        return 0
+    OutputImage = cv2.resize(colerPic[:,-Input_Resolution:,:],(Input_Resolution,Input_Resolution))
+    if(OutputImage.shape[1]<Input_Resolution):
+        return 0
+    imagesTensor = transforms.Compose(
+    [transforms.ToPILImage(),
+     transforms.ToTensor(),
+     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])(OutputImage)
+    imagesTensor = Variable(imagesTensor, requires_grad=False)
+    testImages = imagesTensor.unsqueeze(0)
+    outputs = model(testImages)
+    outputs = F.softmax(outputs)
+    prob, predicted = torch.topk(outputs,len(classes))
+    #print(predicted[:2],prob[:2])
+    if(not CallBack2==None):
+        CallBack2(predicted,prob,classes)
+    else:
+        predicted=predicted[0].numpy()
+        prob=prob[0].detach().numpy()
+        SmoothingFunction(classes[predicted[0]],prob[0],CallBack)
+
+def StopAudio():
+    global pa
+    global stream
+    time.sleep(1)
+    stream.close()
+    cv2.destroyAllWindows()
+
+def DoStuff(Input,Probablity):
+    #print("I heard a "+str(Input)+'with'+str(Probablity))
+    global timer
+    clear_output(wait=True)
+    print(Input,timer)
+    if(timer>0):
+        timer-=1
+    if(timer<=1):
+        os.system("say I think I heard a "+str(Input))
+        print("I heard a "+str(Input))
+        timer=50
+
+def RunTheSystem(TargetTime=30,ModelPath = "../models/UrbanResNet.pth",CallBackFunction=DoStuff,CallBack2=None):
+    print("Loading all relevant data.")
+    StartAudio(ModelPath=ModelPath)
+    print("Starting Running")
+    t0 = time.time()
+    while stream.is_active():
+        infere_Class_Type(CallBackFunction,CallBack2)
+        if (TargetTime>0 )and ((time.time()-t0)>=TargetTime):
+            break
+    print("Stopping!")
+    StopAudio()
+    print("Stopped and Done!")
+

04_ProcessingData/Transforming Datasets.ipynb

@@ -117,7 +117,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.7.3"
+   "version": "3.7.1"
   }
  },
  "nbformat": 4,