Skip to content

Instantly share code, notes, and snippets.

@ehzawad

ehzawad/app.py

Created June 18, 2023 08:30
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save ehzawad/a460a62af5bedf08fdff37ee105d1d37 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save ehzawad/a460a62af5bedf08fdff37ee105d1d37 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
app.py
from flask import Flask, render_template, request as rqst, jsonify
from flask_socketio import SocketIO, emit
import sys
from flask_cors import CORS
import argparse
import queue
import sys
import json
import asyncio
import sounddevice as sd
from urllib.request import urlopen
from urllib import request, parse
import urllib.error
from vosk import Model, KaldiRecognizer
from engineio.payload import Payload
Payload.max_decode_packets = 1000
samplerate = 8000
model = Model(model_path="/home/gns/vosk-api/model/")
dump_fn = None
rec = KaldiRecognizer(model, samplerate)
q = queue.Queue()
def int_or_str(text):
"""Helper function for argument parsing."""
try:
return int(text)
except ValueError:
return text
def callback(indata, frames, time, status):
"""This is called (from a separate thread) for each audio block."""
if status:
print(status, file=sys.stderr)
q.put(bytes(indata))
app = Flask(__name__)
socketio = SocketIO(app)
@app.route('/')
def index():
return render_template('index.html')
@app.route('/message')
def message():
output = rqst.args['output']
print(output)
#output="নামজারী সেবা প্রাপ্তির সময়"
sender_data = '{"sender":"default","message":"'+output+'","cli": "12346798","metadata":"bn"}'
post_data = sender_data.encode('utf-8')
print(post_data)
req = request.Request("http://192.168.10.44:5004/webhooks/rest/webhook", data=post_data)
post_resp = request.urlopen(req)
ai_resp = json.loads(post_resp.read())
print(ai_resp)
#for tts request
bot_response='No response'
if(len(ai_resp)>0):
bot_response=ai_resp[0]["text"]
#url="http://192.168.10.42:5002/api/tts?text=%E0%A6%86%E0%A6%AA%E0%A6%A8%E0%A6%BE%E0%A6%95%E0%A7%87%20%E0%A6%A7%E0%A6%A8%E0%A7%8D%E0%A6%AF%E0%A6%AC%E0%A6%BE%E0%A6%A6%2C%20%2C%20%E0%A6%86%E0%A6%AA%E0%A6%A8%E0%A6%BE%E0%A6%95%E0%A7%87%20%E0%A6%95%E0%A6%BF%E0%A6%AD%E0%A6%BE%E0%A6%AC%E0%A7%87%20%E0%A6%B8%E0%A6%B9%E0%A6%BE%E0%A7%9F%E0%A6%A4%E0%A6%BE%20%E0%A6%95%E0%A6%B0%E0%A6%A4%E0%A7%87%20%E0%A6%AA%E0%A6%BE%E0%A6%B0%E0%A6%BF&speaker_id=Romana&style_wav=&language_id="
br = urllib.parse.quote(bot_response)
url="http://192.168.10.42:5002/api/tts?text="+str(br)+"&speaker_id=Romana&style_wav=&language_id="
req = request.Request(url)
post_resp = request.urlopen(req)
#ai_resp = json.loads(post_resp.read())
print(post_resp)
return post_resp
return jsonify(ai_resp)
@socketio.on('audio')
def handle_audio(data):
try:
if rec.AcceptWaveform(data):
kaldiresponse=rec.Result()
# print(kaldiresponse)
kr = json.loads(kaldiresponse)
if(len(kr['text'])>0):
print(f"TEXT Full Result-------------> {kr['partial']}")
socketio.emit('kaldiresponse', kr['text'])
else:
print("Partial")
kaldiresponse=rec.PartialResult()
# print(kaldiresponse)
kr = json.loads(kaldiresponse)
if(len(kr['partial'])>0):
print(f"TEXT -------------> {kr['partial']}")
socketio.emit('kaldiresponse', kr['partial'])
except Exception as e:
print("No data")
print('ERROR: '+ str(e))
# print('received audio: ' + str(data))
if __name__ == '__main__':
socketio.run(app, port=5025, allow_unsafe_werkzeug=True)
@ehzawad

ehzawad commented Jun 18, 2023 •
edited
Loading

Copy link
Copy Markdown
Author 
<!-- <script src="https://cdn.socket.io/3.1.3/socket.io.min.js"></script> -->

<!DOCTYPE html>
<html>
    <head>
        <meta charset="UTF-8">
        <title>WebRTC Kaldi ASR</title>
        <!-- <link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css"
              integrity_no="sha384-MCw98/SFnGE8fJT3GXwEOngsV7Zt27NXFoaoApmYm81iuXoPkFOJwJ8ERdknLPMO" crossorigin="anonymous"> -->
        <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.0/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-9ndCyUaIbzAi2FUVXJi0CjmCapSmO7SnpJef0486qhLnuZ2cdeRhO02iuK6FUUVM" crossorigin="anonymous">
        <script src="https://code.jquery.com/jquery-3.3.1.slim.min.js"></script>
        <link rel="icon" href="data:,">
    </head>
<body>
    <h1 class="text-center">Online Decoding Using kaldi</h1>
    <div class="form-floating mt-4">
        <textarea class="form-control mx-auto" placeholder="Leave a comment here" id="output" style="height: 300px; width: 50%; "></textarea>
    </div>
    <div class="container text-center mt-3">
        <button class="btn btn-outline-success mx-auto btn-lg" style="width: 300px;" id="start" onclick="startRecording()">Start</button>
        <button class="btn btn-outline-danger mx-auto btn-lg" style="width: 300px;" id="stop" onclick="stopRecording()">Stop</button>
        <label id="disconnect">0</label>
    </div>

<audio id="audio" hidden controls></audio>

<script src="https://cdn.socket.io/3.1.3/socket.io.min.js"></script>
<script>
disconnect=0;
var socket = io.connect('http://127.0.0.1:5025');
socket.on('kaldiresponse', function(data){
    console.log("kaldiresponse")
    //document.getElementById('output').innerText=data
    $("#output").val(data);

    
   });

socket.on('disconnect', function(data){
    disconnect+=1;
    document.getElementById('disconnect').innerText=disconnect;
    console.log("disconnect:");
   });

var audioCtx;
var source;
var stream;
var audio1="nn";
let buffer = []

var ws = null;
var audiotrack = null;
var source = null;
var processor = null;

var audioElement = document.getElementById('audio')

 function floatTo16BitPCM(input) {
  let i      = input.length;
  let output = new Int16Array(i);
  while (i--) {
    let s     = Math.max(-1, Math.min(1, input[i]));
    output[i] = (s < 0 ? s * 0x8000 : s * 0x7FFF);
  }
  return output;
}


function int16ToFloat32BitPCM(input) {
  let i      = input.length;
  let output = new Float32Array(i);
  while (i--) {
    let int   = input[i];
    output[i] = (int >= 0x8000) ? -(0x10000 - int) / 0x8000 : int / 0x7FFF;
  }
  return output;
}
var sendcounter=0
function wsSender() {
    console.log("INSIDE WsSender")
    // 16000(sr) * 2(bytes) = 640 * 2(bytes) * (1000(millisecond) / 40(interval))
    this.handlerInterval = setInterval(() => {
        // audio package length 1280 bytes (640 * 2(bytes))
        console.log("buffer len " + buffer.length)
        var audioData = buffer.splice(0, 3000)
        if (audioData.length > 0) {
            aud = floatTo16BitPCM(audioData)
            socket.emit("audio", aud)
            
        }
    }, 150)  // must less than 40ms
}

counter = 0
const handleSuccess = function (stream) {
    audioElement.srcObject = stream
    const context = new AudioContext();
    source = context.createMediaStreamSource(stream);
    // createScriptProcessor bufferSize = 0 means let Browser select the best size, e.g. 2048
    processor = context.createScriptProcessor(0, 1, 1);
    audiotrack = stream.getAudioTracks()[0];

    source.connect(processor);
    processor.connect(context.destination);

    processor.onaudioprocess = function (e) {
        // console.log(new Date().getTime() / 1000)
        // console.log('counter ' + counter)
        counter++
        // audioElement.play()
        // Do something with the data, e.g. convert it to WAV

        let sourceAudioBuffer = e.inputBuffer;
        // console.log(sourceAudioBuffer);

        // `sourceAudioBuffer` is an AudioBuffer instance of the source audio
        // at the original sample rate.
        const DESIRED_SAMPLE_RATE = 8000;
        const offlineCtx = new OfflineAudioContext(sourceAudioBuffer.numberOfChannels, sourceAudioBuffer.duration * DESIRED_SAMPLE_RATE, DESIRED_SAMPLE_RATE);
        const cloneBuffer = offlineCtx.createBuffer(sourceAudioBuffer.numberOfChannels, sourceAudioBuffer.length, sourceAudioBuffer.sampleRate);
        // Copy the source data into the offline AudioBuffer
        for (let channel = 0; channel < sourceAudioBuffer.numberOfChannels; channel++) {
            cloneBuffer.copyToChannel(sourceAudioBuffer.getChannelData(channel), channel);
        }
        // Play it from the beginning.
        const source = offlineCtx.createBufferSource();
        source.buffer = cloneBuffer;
        source.connect(offlineCtx.destination);
        offlineCtx.oncomplete = function (e) {
            // `resampledAudioBuffer` contains an AudioBuffer resampled at 16000Hz.
            // use resampled.getChannelData(x) to get an Float32Array for channel x.
            const resampledAudioBuffer = e.renderedBuffer;
            // console.log(resampledAudioBuffer);
            // convert to  int16 buffer array
            buffer.push(...resampledAudioBuffer.getChannelData(0))
        }
        offlineCtx.startRendering();
        source.start(0);
    };
};

function startRecording() {
    if (audio1 !== 'nn') {
        audio1.pause();
        audio1.currentTime = 0;
    }

    navigator.mediaDevices.getUserMedia({
        audio: {channelCount: 1, sampleRate: 8000},
        video: false
    }).then(handleSuccess);

    setTimeout(() => {
        wsSender();
    }, 1000);

    // setTimeout(() => {
    //     startRecording();
    // }, 6000)

    // console.log('record time ' + new Date().getTime() / 1000)
    // console.log('record');
}

function httpGet(theUrl,data)
{
    var xmlHttp = new XMLHttpRequest();
    xmlHttp.open( "GET", theUrl, false ); // false for synchronous request
    xmlHttp.send( null );
    return xmlHttp.responseText;

}

async function stopRecording() {

    if (null != audiotrack) {
        console.log('audiotrack.stop');
        audiotrack.stop();
        audiotrack = null;
    }
    if (null != source) {
        source.disconnect();
    }
    if (null != processor) {
        processor.disconnect();
    }

    if (null != this.handlerInterval) {
        clearInterval(this.handlerInterval)
    }
    console.log('stop time ' + new Date().getTime() / 1000);
    console.log('stop');

    var output=$("#output").val();
    console.log(output.length)
    if(output.length==0){
        return
    }

    var response = await fetch('http://127.0.0.1:5025/message?output=' + encodeURIComponent(output));
    var audioBlob = await response.blob();

    // Create a URL for the blob
    var url = URL.createObjectURL(audioBlob);

    // Create a new Audio object and play the audio
    audio1 = new Audio(url);
    audio1.autoplay = true;

    // audio1.addEventListener('loadeddata', function() {
    //   console.log('Audio data has loaded.');
    // }, false);

    // audio1.addEventListener('canplay', function() {
    //   console.log('Audio can start playing now.');
    //   audio1.play();
    // }, false);
    $("#output").val("");
    // var res=httpGet('http://127.0.0.1:5025/message?output='+output,output);

   
}
</script>

</body>
</html>

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment