diff --git a/README.md b/README.md index 7330e70ee..cad047078 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -jasper-client +jasper-client - Snowboy Hot Word Detection ============= [![Build Status](https://travis-ci.org/jasperproject/jasper-client.svg?branch=master)](https://travis-ci.org/jasperproject/jasper-client) [![Coverage Status](https://img.shields.io/coveralls/jasperproject/jasper-client.svg)](https://coveralls.io/r/jasperproject/jasper-client) [![Codacy Badge](https://www.codacy.com/project/badge/3a50e1bc2261419894d76b7e2c1ac694)](https://www.codacy.com/app/jasperproject/jasper-client) @@ -7,6 +7,38 @@ Client code for the Jasper voice computing platform. Jasper is an open source pl Learn more at [jasperproject.github.io](http://jasperproject.github.io/), where we have assembly and installation instructions, as well as extensive documentation. For the relevant disk image, please visit [SourceForge](http://sourceforge.net/projects/jasperproject/). +## The differences with the main project + +The problem with stt online, it's the big lack of PriVAcy !!! + +In jasper by adding stt_passive_engine in profile.yml you can choose the stt who gonna listen to you til you said "Jasper !". +Of course you have to choose a not online stt, if you mind about your PriVAcy... + +That's good But ! +- You can't easily choose a wake up word. +- Due to my accent (I think), when i said "Jasper"... doesn't work all the time... + +To address these points, i decide to use "Snowboy Hot Word Detection" [https://snowboy.kitt.ai/](https://snowboy.kitt.ai/) as "stt passive engine". +- Define and train your own hotword +- High accuracy, +- Low latency and no internet needed +- Small memory footprint and cross-platform support + +## Installation + +- Follow the python installation -> [https://github.com/Kitt-AI/snowboy](https://github.com/Kitt-AI/snowboy) +- copy _snowboydetect.so (generated in project snowboy/swig/Python) in jasper/client/snowboy +- create your model -> [https://snowboy.kitt.ai/](https://snowboy.kitt.ai/) +- copy your model under the name "model.pmdl" in /jasper/client/snowboy) +- And here we go !!! + +## TODO + +- Remove "Persona" in the code +- Remove all the dependance in the code stt_passive_engine +- Too much latence in active listenning. +- Re add gmail notification + ## Contributing If you'd like to contribute to Jasper, please read through our **[Contributing Guide](CONTRIBUTING.md)**, which outlines the philosophies to preserve, tests to run, and more. We highly recommend reading through this guide before writing any code. diff --git a/client/conversation.py b/client/conversation.py index 6b2fab1a0..07a13cc5f 100644 --- a/client/conversation.py +++ b/client/conversation.py @@ -1,49 +1,56 @@ # -*- coding: utf-8-*- import logging +import signal +import os from notifier import Notifier from brain import Brain - +from snowboy import snowboydecoder class Conversation(object): - def __init__(self, persona, mic, profile): + def __init__(self, mic, profile): self._logger = logging.getLogger(__name__) - self.persona = persona self.mic = mic self.profile = profile self.brain = Brain(mic, profile) self.notifier = Notifier(profile) + self.interrupted = False + + def signal_handler(self, signal, frame): + self.interrupted = True + + def interrupt_callback(self): + return self.interrupted + + def startListenningActively(self): + threshold = None + self._logger.debug("Started to listen actively with threshold: %r", + threshold) + input = self.mic.activeListenToAllOptions(threshold) + self._logger.debug("Stopped to listen actively with threshold: %r", + threshold) + print("i'm here now") + if input: + self.brain.query(input) + else: + self.mic.say("Pardon?") def handleForever(self): """ Delegates user input to the handling function when activated. """ - self._logger.info("Starting to handle conversation with keyword '%s'.", - self.persona) - while True: - # Print notifications until empty - notifications = self.notifier.getAllNotifications() - for notif in notifications: - self._logger.info("Received notification: '%s'", str(notif)) - - self._logger.debug("Started listening for keyword '%s'", - self.persona) - threshold, transcribed = self.mic.passiveListen(self.persona) - self._logger.debug("Stopped listening for keyword '%s'", - self.persona) - - if not transcribed or not threshold: - self._logger.info("Nothing has been said or transcribed.") - continue - self._logger.info("Keyword '%s' has been said!", self.persona) - - self._logger.debug("Started to listen actively with threshold: %r", - threshold) - input = self.mic.activeListenToAllOptions(threshold) - self._logger.debug("Stopped to listen actively with threshold: %r", - threshold) + self._logger.info("Starting to handle conversation") + + TOP_DIR = os.path.dirname(os.path.abspath(__file__)) + MODEL_FILE = os.path.join(TOP_DIR, "snowboy/model.pmdl") + + signal.signal(signal.SIGINT, self.signal_handler) + detector = snowboydecoder.HotwordDetector(MODEL_FILE, sensitivity=0.5) + print('Listening... Press Ctrl+C to exit') + + # main loop + detector.start(detected_callback=self.startListenningActively, + interrupt_check=self.interrupt_callback, + sleep_time=0.03) - if input: - self.brain.query(input) - else: - self.mic.say("Pardon?") + detector.terminate() \ No newline at end of file diff --git a/client/mic.py b/client/mic.py index 401cddbd6..169b7bce6 100644 --- a/client/mic.py +++ b/client/mic.py @@ -16,19 +16,16 @@ class Mic: speechRec = None speechRec_persona = None - def __init__(self, speaker, passive_stt_engine, active_stt_engine): + def __init__(self, speaker, active_stt_engine): """ Initiates the pocketsphinx instance. Arguments: speaker -- handles platform-independent audio output - passive_stt_engine -- performs STT while Jasper is in passive listen - mode acive_stt_engine -- performs STT while Jasper is in active listen mode """ self._logger = logging.getLogger(__name__) self.speaker = speaker - self.passive_stt_engine = passive_stt_engine self.active_stt_engine = active_stt_engine self._logger.info("Initializing PyAudio. ALSA/Jack error messages " + "that pop up during this process are normal and " + @@ -86,103 +83,6 @@ def fetchThreshold(self): return THRESHOLD - def passiveListen(self, PERSONA): - """ - Listens for PERSONA in everyday sound. Times out after LISTEN_TIME, so - needs to be restarted. - """ - - THRESHOLD_MULTIPLIER = 1.8 - RATE = 16000 - CHUNK = 1024 - - # number of seconds to allow to establish threshold - THRESHOLD_TIME = 1 - - # number of seconds to listen before forcing restart - LISTEN_TIME = 10 - - # prepare recording stream - stream = self._audio.open(format=pyaudio.paInt16, - channels=1, - rate=RATE, - input=True, - frames_per_buffer=CHUNK) - - # stores the audio data - frames = [] - - # stores the lastN score values - lastN = [i for i in range(30)] - - # calculate the long run average, and thereby the proper threshold - for i in range(0, RATE / CHUNK * THRESHOLD_TIME): - - data = stream.read(CHUNK) - frames.append(data) - - # save this data point as a score - lastN.pop(0) - lastN.append(self.getScore(data)) - average = sum(lastN) / len(lastN) - - # this will be the benchmark to cause a disturbance over! - THRESHOLD = average * THRESHOLD_MULTIPLIER - - # save some memory for sound data - frames = [] - - # flag raised when sound disturbance detected - didDetect = False - - # start passively listening for disturbance above threshold - for i in range(0, RATE / CHUNK * LISTEN_TIME): - - data = stream.read(CHUNK) - frames.append(data) - score = self.getScore(data) - - if score > THRESHOLD: - didDetect = True - break - - # no use continuing if no flag raised - if not didDetect: - print "No disturbance detected" - stream.stop_stream() - stream.close() - return (None, None) - - # cutoff any recording before this disturbance was detected - frames = frames[-20:] - - # otherwise, let's keep recording for few seconds and save the file - DELAY_MULTIPLIER = 1 - for i in range(0, RATE / CHUNK * DELAY_MULTIPLIER): - - data = stream.read(CHUNK) - frames.append(data) - - # save the audio data - stream.stop_stream() - stream.close() - - with tempfile.NamedTemporaryFile(mode='w+b') as f: - wav_fp = wave.open(f, 'wb') - wav_fp.setnchannels(1) - wav_fp.setsampwidth(pyaudio.get_sample_size(pyaudio.paInt16)) - wav_fp.setframerate(RATE) - wav_fp.writeframes(''.join(frames)) - wav_fp.close() - f.seek(0) - # check if PERSONA was said - transcribed = self.passive_stt_engine.transcribe(f) - - if any(PERSONA in phrase for phrase in transcribed): - return (THRESHOLD, PERSONA) - - return (False, transcribed) - def activeListen(self, THRESHOLD=None, LISTEN=True, MUSIC=False): """ Records until a second of silence or times out after 12 seconds diff --git a/client/modules/MPDControl.py b/client/modules/MPDControl.py index 54f479cf8..dfafbcaa8 100644 --- a/client/modules/MPDControl.py +++ b/client/modules/MPDControl.py @@ -78,7 +78,6 @@ def __init__(self, PERSONA, mic, mpdwrapper): music_stt_engine = mic.active_stt_engine.get_instance('music', phrases) self.mic = Mic(mic.speaker, - mic.passive_stt_engine, music_stt_engine) def delegateInput(self, input): diff --git a/client/snowboy/__init__.py b/client/snowboy/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/client/snowboy/demo.py b/client/snowboy/demo.py new file mode 100644 index 000000000..305b8b624 --- /dev/null +++ b/client/snowboy/demo.py @@ -0,0 +1,35 @@ +import snowboydecoder +import sys +import signal + +interrupted = False + + +def signal_handler(signal, frame): + global interrupted + interrupted = True + + +def interrupt_callback(): + global interrupted + return interrupted + +if len(sys.argv) == 1: + print("Error: need to specify model name") + print("Usage: python demo.py your.model") + sys.exit(-1) + +model = sys.argv[1] + +# capture SIGINT signal, e.g., Ctrl+C +signal.signal(signal.SIGINT, signal_handler) + +detector = snowboydecoder.HotwordDetector(model, sensitivity=0.5) +print('Listening... Press Ctrl+C to exit') + +# main loop +detector.start(detected_callback=snowboydecoder.play_audio_file, + interrupt_check=interrupt_callback, + sleep_time=0.03) + +detector.terminate() diff --git a/client/snowboy/resources/common.res b/client/snowboy/resources/common.res new file mode 100644 index 000000000..0e267f5eb Binary files /dev/null and b/client/snowboy/resources/common.res differ diff --git a/client/snowboy/resources/ding.wav b/client/snowboy/resources/ding.wav new file mode 100644 index 000000000..79346e069 Binary files /dev/null and b/client/snowboy/resources/ding.wav differ diff --git a/client/snowboy/resources/dong.wav b/client/snowboy/resources/dong.wav new file mode 100644 index 000000000..426596b29 Binary files /dev/null and b/client/snowboy/resources/dong.wav differ diff --git a/client/snowboy/snowboydecoder.py b/client/snowboy/snowboydecoder.py new file mode 100644 index 000000000..77a20b941 --- /dev/null +++ b/client/snowboy/snowboydecoder.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python + +import collections +import pyaudio +import snowboydetect +import time +import wave +import os +import logging + +logging.basicConfig() +logger = logging.getLogger("snowboy") +logger.setLevel(logging.INFO) +TOP_DIR = os.path.dirname(os.path.abspath(__file__)) + +RESOURCE_FILE = os.path.join(TOP_DIR, "resources/common.res") +DETECT_DING = os.path.join(TOP_DIR, "resources/ding.wav") +DETECT_DONG = os.path.join(TOP_DIR, "resources/dong.wav") + + +class RingBuffer(object): + """Ring buffer to hold audio from PortAudio""" + def __init__(self, size = 4096): + self._buf = collections.deque(maxlen=size) + + def extend(self, data): + """Adds data to the end of buffer""" + self._buf.extend(data) + + def get(self): + """Retrieves data from the beginning of buffer and clears it""" + tmp = ''.join(self._buf) + self._buf.clear() + return tmp + + +def play_audio_file(fname=DETECT_DING): + """Simple callback function to play a wave file. By default it plays + a Ding sound. + + :param str fname: wave file name + :return: None + """ + ding_wav = wave.open(fname, 'rb') + ding_data = ding_wav.readframes(ding_wav.getnframes()) + audio = pyaudio.PyAudio() + stream_out = audio.open( + format=audio.get_format_from_width(ding_wav.getsampwidth()), + channels=ding_wav.getnchannels(), + rate=ding_wav.getframerate(), input=False, output=True) + stream_out.start_stream() + stream_out.write(ding_data) + time.sleep(0.2) + stream_out.stop_stream() + stream_out.close() + audio.terminate() + + +class HotwordDetector(object): + """ + Snowboy decoder to detect whether a keyword specified by `decoder_model` + exists in a microphone input stream. + + :param decoder_model: decoder model file path, a string or a list of strings + :param resource: resource file path. + :param sensitivity: decoder sensitivity, a float of a list of floats. + The bigger the value, the more senstive the + decoder. If an empty list is provided, then the + default sensitivity in the model will be used. + :param audio_gain: multiply input volume by this factor. + """ + def __init__(self, decoder_model, + resource=RESOURCE_FILE, + sensitivity=[], + audio_gain=1): + + def audio_callback(in_data, frame_count, time_info, status): + self.ring_buffer.extend(in_data) + play_data = chr(0) * len(in_data) + return play_data, pyaudio.paContinue + + tm = type(decoder_model) + ts = type(sensitivity) + if tm is not list: + decoder_model = [decoder_model] + if ts is not list: + sensitivity = [sensitivity] + model_str = ",".join(decoder_model) + + self.detector = snowboydetect.SnowboyDetect( + resource_filename=resource, model_str=model_str) + self.detector.SetAudioGain(audio_gain) + self.num_hotwords = self.detector.NumHotwords() + + if len(decoder_model) > 1 and len(sensitivity) == 1: + sensitivity = sensitivity*self.num_hotwords + if len(sensitivity) != 0: + assert self.num_hotwords == len(sensitivity), \ + "number of hotwords in decoder_model (%d) and sensitivity " \ + "(%d) does not match" % (self.num_hotwords, len(sensitivity)) + sensitivity_str = ",".join([str(t) for t in sensitivity]) + if len(sensitivity) != 0: + self.detector.SetSensitivity(sensitivity_str); + + self.ring_buffer = RingBuffer( + self.detector.NumChannels() * self.detector.SampleRate() * 5) + self.audio = pyaudio.PyAudio() + self.stream_in = self.audio.open( + input=True, output=False, + format=self.audio.get_format_from_width( + self.detector.BitsPerSample() / 8), + channels=self.detector.NumChannels(), + rate=self.detector.SampleRate(), + frames_per_buffer=2048, + stream_callback=audio_callback) + + + def start(self, detected_callback=play_audio_file, + interrupt_check=lambda: False, + sleep_time=0.03): + """ + Start the voice detector. For every `sleep_time` second it checks the + audio buffer for triggering keywords. If detected, then call + corresponding function in `detected_callback`, which can be a single + function (single model) or a list of callback functions (multiple + models). Every loop it also calls `interrupt_check` -- if it returns + True, then breaks from the loop and return. + + :param detected_callback: a function or list of functions. The number of + items must match the number of models in + `decoder_model`. + :param interrupt_check: a function that returns True if the main loop + needs to stop. + :param float sleep_time: how much time in second every loop waits. + :return: None + """ + if interrupt_check(): + logger.debug("detect voice return") + return + + tc = type(detected_callback) + if tc is not list: + detected_callback = [detected_callback] + if len(detected_callback) == 1 and self.num_hotwords > 1: + detected_callback *= self.num_hotwords + + assert self.num_hotwords == len(detected_callback), \ + "Error: hotwords in your models (%d) do not match the number of " \ + "callbacks (%d)" % (self.num_hotwords, len(detected_callback)) + + logger.debug("detecting...") + + while True: + if interrupt_check(): + logger.debug("detect voice break") + break + data = self.ring_buffer.get() + if len(data) == 0: + time.sleep(sleep_time) + continue + + ans = self.detector.RunDetection(data) + if ans == -1: + logger.warning("Error initializing streams or reading audio data") + elif ans > 0: + message = "Keyword " + str(ans) + " detected at time: " + message += time.strftime("%Y-%m-%d %H:%M:%S", + time.localtime(time.time())) + logger.info(message) + callback = detected_callback[ans-1] + if callback is not None: + callback() + + logger.debug("finished.") + + def terminate(self): + """ + Terminate audio stream. Users cannot call start() again to detect. + :return: None + """ + self.stream_in.stop_stream() + self.stream_in.close() + self.audio.terminate() diff --git a/client/snowboy/snowboydetect.py b/client/snowboy/snowboydetect.py new file mode 100644 index 000000000..a1393ff24 --- /dev/null +++ b/client/snowboy/snowboydetect.py @@ -0,0 +1 @@ +../../swig/Python/snowboydetect.py \ No newline at end of file diff --git a/jasper.py b/jasper.py index 4e7f28edf..e5d50f199 100755 --- a/jasper.py +++ b/jasper.py @@ -87,12 +87,6 @@ def __init__(self): "to '%s'", stt_engine_slug) stt_engine_class = stt.get_engine_by_slug(stt_engine_slug) - try: - slug = self.config['stt_passive_engine'] - stt_passive_engine_class = stt.get_engine_by_slug(slug) - except KeyError: - stt_passive_engine_class = stt_engine_class - try: tts_engine_slug = self.config['tts_engine'] except KeyError: @@ -103,7 +97,6 @@ def __init__(self): # Initialize Mic self.mic = Mic(tts_engine_class.get_instance(), - stt_passive_engine_class.get_passive_instance(), stt_engine_class.get_active_instance()) def run(self): @@ -114,7 +107,7 @@ def run(self): salutation = "How can I be of service?" self.mic.say(salutation) - conversation = Conversation("JASPER", self.mic, self.config) + conversation = Conversation(self.mic, self.config) conversation.handleForever() if __name__ == "__main__":