Commit 12665d59 authored by Joline Hellström's avatar Joline Hellström
Browse files

cobra working, issues with gpt3? Not listening to when stop talking yet

parent c238ec56
Pipeline #84898 failed with stage
in 2 minutes and 54 seconds
......@@ -2,6 +2,7 @@
import rclpy
import os
from rclpy.node import Node
from rclpy.executors import MultiThreadedExecutor
from lhw_interfaces.msg import Audio, Response
from lhw_interfaces.srv import VoiceDetectionResponse
import whisper
......@@ -9,17 +10,19 @@ import queue
import numpy as np
from std_msgs.msg import String
import json
from functools import partial
class Whisper(Node):
def __init__(self):
super().__init__('whisper')
self.model = whisper.load_model("base")
self.audio_buff_sub = self.create_subscription(Audio, 'microphone_buffer', self.audio_cb, 10)
"""self.voice_detection_client = self.create_client(VoiceDetectionResponse, 'voice_detection_service')
self.voice_detection_client = self.create_client(VoiceDetectionResponse, 'voice_detection_service')
while not self.voice_detection_client.wait_for_service(timeout_sec=1.0):
self.get_logger().info('service not available, waiting again...')
self.req = VoiceDetectionResponse.Request()"""
self.req = VoiceDetectionResponse.Request()
#self.response_pub = self.create_publisher(String, 'text', 10)
......@@ -29,68 +32,47 @@ class Whisper(Node):
self.total_float_data = np.array([])
self.words_heard = ""
self.timer = 0
def send_voice_detection_request(self, audio_data):
self.req.data = audio_data
#print(self.req)
self.future = self.voice_detection_client.call_async(self.req)
#rclpy.spin_until_future_complete(self, self.future)
print(self.future.result())
return self.future.result()
def audio_cb(self, audio_msg):
#result = self.send_to_model(audio_msg)
#self.words_heard += result[0]
#self.total_float_data = self.total_float_data.astype(np.float32, order="C")/32768.0
#result = self.model.transcribe(self.total_float_data)
response = Response()
#response.recognized_text = self.words_heard
print("in audio cb")
cobra_result = self.send_voice_detection_request(audio_msg.data)
print("efter request")
if cobra_result:
if cobra_result.mean_prob > 0.8:
print(cobra_result.mean_prob)
model_result = self.send_to_model(audio_msg)
response.recognized_text = model_result
self.response_pub.publish(response)
else:
print(cobra_result.mean_prob)
"""
if self.words_heard:
if self.words_heard.strip() not in self.ignore_words:
if ".." not in response.recognized_text:
print(response.recognized_text)
self.response_pub.publish(response)
self.words_heard = ""
self.timer = 0
#else:
#self.total_float_data = np.concatenate((self.total_float_data, result[1]))
"""
def send_to_model(self, audio_msg):
data_s16 = np.frombuffer(audio_msg.data, dtype=np.int16, count=len(audio_msg.data)//2, offset=0)
#print("in audio cb")
self.req.data = audio_msg.data
self.future = self.voice_detection_client.call_async(self.req)
self.future.add_done_callback(partial(self.check_if_speaking, audio_data=audio_msg.data))
def check_if_speaking(self, future, audio_data):
cobra_result = self.future.result()
if cobra_result.mean_prob > 0.5:
#print("in if", cobra_result.mean_prob)
model_result = self.send_to_model(audio_data)
response = Response()
response.recognized_text = model_result
self.response_pub.publish(response)
else:
pass#print("if else", cobra_result.mean_prob)
def send_to_model(self, audio_data):
data_s16 = np.frombuffer(audio_data, dtype=np.int16, count=len(audio_data)//2, offset=0)
float_data = data_s16.astype(np.float32, order="C")/32768.0
result = self.model.transcribe(float_data)
return result["text"]#, float_data
return result["text"]
def main(args=None):
rclpy.init(args=args)
whisper_node = Whisper()
"""executor = MultiThreadedExecutor(num_threads=2)
rclpy.spin(whisper_node)
print(1)
executor.add_node(whisper_node)
executor.spin()"""
rclpy.spin(whisper_node)
whisper_node.destroy_node()
rclpy.shutdown()
if __name__ == '__main__':
main()
......
ros2 service call /voice_detection_service lhw_interfaces/srv/VoiceDetectionResponse "{data:[0, 0, 0, 0, 254, 255, 5, 0, 248, 255, 10, 0, 251, 255, 239, 255, 88, 1, 179, 1, 139, 0, 59, 255, 147, 253, 251, 252, 233, 252, 234, 251, 206, 250, 196, 250, 42, 250, 143, 248, 220, 247, 241, 247, 44, 248, 165, 248, 150, 249, 100, 250, 182, 250, 112, 251, 125, 252, 32, 253, 1, 254, 190, 255, 236, 1, 79, 4, 171, 5, 169, 5, 25, 6, 174, 6, 115, 6, 184, 6, 223, 6, 69, 6, 49, 5, 107, 4, 118, 4, 135, 3, 204, 1, 104, 0, 198, 254, 243, 252, 51, 252, 252, 251, 219, 250, 196, 249, 29, 249, 155, 248, 40, 248, 176, 247, 119, 247, 254, 247, 92, 248, 86, 249, 48, 251, 120, 252, 133, 253, 110, 254, 201, 255, 72, 1, 34, 2, 124, 3, 236, 5, 218, 7, 181, 8, 128, 9, 204, 9, 22, 9, 223, 8, 128, 8, 180, 6, 113, 5, 152, 5, 62, 5, 43, 4, 17, 3, 151, 1, 16, 0, 7, 255, 102, 254, 15, 253, 137, 251, 105, 251, 156, 251, 117, 251, 238, 250, 86, 250, 138, 249, 28, 249, 57, 249, 33, 249, 9, 249, 124, 249, 224, 249, 223, 249, 211, 250, 57, 252, 221, 252, 55, 254, 233, 255, 37, 1, 73, 2, 60, 3, 208, 3, 241, 3, 58, 4, 223, 4, 105, 5, 124, 6, 133, 7, 182, 7, 229, 7, 181, 7, 107, 7, 253, 6, 84, 6, 2, 5, 232, 3, 69, 3, 112, 2, 102, 1, 45, 0, 15, 255, 132, 254, 176, 254, 157, 254, 155, 253, 210, 252, 207, 252, 166, 252, 93, 252, 92, 252, 64, 252, 221, 251, 143, 251, 70, 251, 128, 251, 152, 252, 6, 253, 220, 252, 182, 252, 152, 253, 33, 255, 121, 255, 46, 0, 85, 1, 14, 2, 47, 2, 201, 1, 196, 1, 224, 1, 87, 1, 45, 1, 184, 1, 195, 1, 152, 1, 95, 1, 218, 0, 13, 0, 112, 255, 35, 255, 252, 254, 244, 254, 78, 254, 171, 253, 186, 253, 199, 253, 90, 253, 187, 252, 20, 253, 170, 253, 254, 253, 0, 254, 240, 254, 237, 255, 41, 0, 65, 1, 166, 1, 38, 1, 142, 0, 206, 0, 160, 1, 135, 1, 186, 1, 83, 2, 49, 2, 203, 1, 188, 1, 184, 1, 80, 1, 43, 1, 203, 0, 64, 0, 84, 0, 100, 255, 133, 253, 221, 251, 216, 250, 219, 249, 136, 248, 138, 248, 158, 248, 40, 248, 231, 247, 190, 247, 5, 248, 143, 248, 102, 249, 178, 249, 121, 250, 214, 251, 120, 253, 123, 255, 39, 1, 118, 2, 41, 3, 129, 4, 9, 5, 199, 4, 47, 5, 153, 5, 133, 5, 117, 5, 11, 6, 190, 6, 23, 7, 119, 7, 57, 7, 86, 6, 51, 6, 181, 5, 188, 4, 39, 4, 56, 3, 134, 1, 240, 255, 247, 254, 5, 254, 50, 253, 247, 252, 211, 252, 51, 252, 202, 251, 22, 251, 45, 251, 9, 252, 34, 252, 77, 252, 0, 0, 0, 0, 254, 255, 5, 0, 248, 255, 10, 0, 251, 255, 239, 255, 88, 1, 179, 1, 139, 0, 59, 255, 147, 253, 251, 252, 233, 252, 234, 251, 206, 250, 196, 250, 42, 250, 143, 248, 220, 247, 241, 247, 44, 248, 165, 248, 150, 249, 100, 250, 182, 250, 112, 251, 125, 252, 32, 253, 1, 254, 190, 255, 236, 1, 79, 4, 171, 5, 169, 5, 25, 6, 174, 6, 115, 6, 184, 6, 223, 6, 69, 6, 49, 5, 107, 4, 118, 4, 135, 3, 204, 1, 104, 0, 198, 254, 243, 252, 51, 252, 252, 251, 219, 250, 196, 249, 29, 249, 155, 248, 40, 248, 176, 247, 119, 247, 254, 247, 92, 248, 86, 249, 48, 251, 120, 252, 133, 253, 110, 254, 201, 255, 72, 1, 34, 2, 124, 3, 236, 5, 218, 7, 181, 8, 128, 9, 204, 9, 22, 9, 223, 8, 128, 8, 180, 6, 113, 5, 152, 5, 62, 5, 43, 4, 17, 3, 151, 1, 16, 0, 7, 255, 102, 254, 15, 253, 137, 251, 105, 251, 156, 251, 117, 251, 238, 250, 86, 250, 138, 249, 28, 249, 57, 249, 33, 249, 9, 249, 124, 249, 224, 249, 223, 249, 211, 250, 57, 252, 221, 252, 55, 254, 233, 255, 37, 1, 73, 2, 60, 3, 208, 3, 241, 3, 58, 4, 223, 4, 105, 5, 124, 6, 133, 7, 182, 7, 229, 7, 181, 7, 107, 7, 253, 6, 84, 6, 2, 5, 232, 3, 69, 3, 112, 2, 102, 1, 45, 0, 15, 255, 132, 254, 176, 254, 157, 254, 155, 253, 210, 252, 207, 252, 166, 252, 93, 252, 92, 252, 64, 252, 221, 251, 143, 251, 70, 251, 128, 251, 152, 252, 6, 253, 220, 252, 182, 252, 152, 253, 33, 255, 121, 255, 46, 0, 85, 1, 14, 2, 47, 2, 201, 1, 196, 1, 224, 1, 87, 1, 45, 1, 184, 1, 195, 1, 152, 1, 95, 1, 218, 0, 13, 0, 112, 255, 35, 255, 252, 254, 244, 254, 78, 254, 171, 253, 186, 253, 199, 253, 90, 253, 187, 252, 20, 253, 170, 253, 254, 253, 0, 254, 240, 254, 237, 255, 41, 0, 65, 1, 166, 1, 38, 1, 142, 0, 206, 0, 160, 1, 135, 1, 186, 1, 83, 2, 49, 2, 203, 1, 188, 1, 184, 1, 80, 1, 43, 1, 203, 0, 64, 0, 84, 0, 100, 255, 133, 253, 221, 251, 216, 250, 219, 249, 136, 248, 138, 248, 158, 248, 40, 248, 231, 247, 190, 247, 5, 248, 143, 248, 102, 249, 178, 249, 121, 250, 214, 251, 120, 253, 123, 255, 39, 1, 118, 2, 41, 3, 129, 4, 9, 5, 199, 4, 47, 5, 153, 5, 133, 5, 117, 5, 11, 6, 190, 6, 23, 7, 119, 7, 57, 7, 86, 6, 51, 6, 181, 5, 188, 4, 39, 4, 56, 3, 134, 1, 240, 255, 247, 254, 5, 254, 50, 253, 247, 252, 211, 252, 51, 252, 202, 251, 22, 251, 45, 251, 9, 252, 34, 252, 77, 252, 169,122]}"
\ No newline at end of file
......@@ -63,25 +63,21 @@ class VoiceDetectionService(Node):
def audio_cb(self, audio_msg_req, response):
start = 0
prob = []
audio_array = []
start = 0
for i in range(0, len(audio_msg_req.data), 2):
audio_array.append((audio_msg_req.data[i+1]<<8) + audio_msg_req.data[i])
for frame in range(512, len(audio_array), 512):
voice_probability = self.cobra.process(audio_array[start:frame])
voice_probability = self.cobra.process(audio_array[start: frame])
prob.append(voice_probability)
start = frame
response.mean_prob = statistics.mean(prob)
response.max_prob = max(prob)
response.min_prob = min(prob)
response.std = statistics.stdev(prob)
print(response)
return response
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment