first commit

app.py

@@ -0,0 +1,113 @@
+from flask import Flask, render_template, jsonify
+from flask_socketio import SocketIO
+
+import os
+import speech_recognition as sr
+import question
+import imageio
+import threading
+import time
+
+app = Flask(__name__)
+socketio = SocketIO(app)
+
+current_directory = os.path.dirname(os.path.realpath(__file__))
+
+intents_and_question = question.intents_and_question
+video_urls = question.video_urls
+
+micPermission = True
+
+class SharedData:
+
+    def __init__(self):
+
+        self.micPermission = micPermission
+        #self.pause_event = threading.Event()
+        #self.listening_state = True
+
+
+def get_intent(text):
+    lowercase_text = text.lower()
+    matching_intent = [intent for intent, questions in intents_and_question.items() if any(q in lowercase_text for q in questions)]
+    return matching_intent[0] if matching_intent else 'no_answer'
+
+
+def get_video_response(intent):
+    return video_urls.get(intent)
+
+
+def calc_video_length(url):
+    try:
+        reader = imageio.get_reader(url)
+        duration = reader.get_meta_data()['duration']
+        reader.close()
+        return duration
+    except Exception as e:
+        return "error"
+    
+
+def recognize_and_emit(shared_data):
+
+    recognizer = sr.Recognizer()
+
+    if shared_data.micPermission is True:
+        with sr.Microphone() as source:
+            while True:
+                try:
+                    
+                    #shared_data.pause_event.wait()
+                    print("Listening ....")
+                    audio_data = recognizer.record(source, duration = 3.5)
+                    #audio_data = recognizer.listen(source, timeout=3)
+                    print("Recognizing your text .........")
+                    text = recognizer.recognize_google(audio_data)
+
+                
+                    intent = get_intent(text.strip())
+                    video_url = get_video_response(intent)
+
+                    length = calc_video_length(video_url)
+
+                    #shared_data.length = length
+
+                    
+
+                    socketio.emit('text_received', {'text': text, 'intent': intent, 'video_url': video_url})
+                    print("Recognized Text:", text)
+
+                    shared_data.micPermission = False
+                    print("Going to sleep...")
+                    source = None
+                    time.sleep(length)
+                    shared_data.micPermission = True
+                    recognize_and_emit(shared_data)
+
+                   
+                    
+                except sr.UnknownValueError:
+                    print("Speech Recognition could not understand audio")
+                except sr.RequestError as e:
+                    print(f"Could not request {e}")
+
+
+
+@app.route('/')
+def index():
+    return render_template('index.html')
+
+
+@socketio.on('start_listening')
+def start_listening(data):
+    
+    shared_data = SharedData()
+    recognition_thread = threading.Thread(target=recognize_and_emit, args=(shared_data,))
+    recognition_thread.start()
+
+   
+    #recognition_thread.join()  # Wait for the recognition thread to finish
+
+   
+
+if __name__ == '__main__':
+    socketio.run(app, debug=True)

question.py

@@ -0,0 +1,209 @@
+import os
+
+
+current_directory = os.path.dirname(os.path.realpath(__file__))
+
+intents_and_question = {
+
+        "greet":  [
+
+            "hey",
+            "hello",
+            "Hello?",
+            "hello?",
+            "hi",
+            "hello there",
+            "good morning",
+            "good evening",
+            "moin",
+            "hey there",
+            "let's go",
+            "hey dude",
+            "goodmorning",
+            "goodevening",
+            "good afternoon"
+        ],
+    
+
+    
+        "appointment_booking": [
+
+            "How can I schedule an appointment",
+            "I need to book an appointment with a doctor",
+            "what's the procedure for booking an appointment",
+            "hello, I'd like to schedule an appointment.",
+            "book an appointment",
+            "appointment",
+            "need an appointment"
+        ],
+
+
+        "doctor_information": [
+
+            "tell me more about dr. abc",
+            "what's the specialty of dr. abc",
+            "give me the contact details for dr. abc"
+            "doctor",
+            "contact doctor",
+            "doctor contact",
+            "doctor detail",
+            "detail doctor"
+
+        ],
+
+
+        "department_information": [
+
+            "which departments are available in the hospital",
+            "can you provide details about the cardiology department",
+            "tell me more about the pediatric department",
+            "department",
+            "which department",
+            "department info"
+
+        ],
+
+
+        "visiting_hours":[
+
+            "what are the visiting hours for patients",
+            "when can I visit a patient in the hospital",
+            "is there a specific time for visiting hours",
+            "visiting time",
+            "visit",
+            "time",
+            "time to visit",
+            "visit time",
+            "hospital visit",
+            "visit hospital"
+        ],
+
+
+        "location_and_directions":[
+
+            "how do I get to the hospital",
+            "can you provide me with directions to your location",
+            "where is the hospital situated",
+            "direction",
+            "direction hospital",
+            "hospital direction",
+            "location",
+            "hospital location"
+        ],
+
+
+        "billing_and_insurance": [
+
+            "how do I pay my medical bills",
+            "do you accept my insurance",
+            "what's the billing process for a hospital stay",
+            "bills",
+            "insurance",
+            "pay bill",
+        ],
+
+
+        "medical_records": [
+            "how can I access my medical records",
+            "request my medical history",
+            "I need a copy of my lab results",
+            "medical records",
+            "medical record",
+            "record",
+            "records",
+            "record medical",
+            "results",
+            "lab results",
+            "result copy",
+            "copy result"
+        ],
+
+
+        "emergency_services": [
+            "what should I do in case of a medical emergency",
+            "how do I contact the hospital in an emergency",
+            "tell me about your emergency services",
+            "emergency",
+            "emergency service"
+        ],
+
+
+        "covid19_information": [
+            "what safety measures are in place due to COVID-19",
+            "is it safe to visit the hospital during the pandemic",
+            "do you offer COVID-19 testing or vaccinations",
+            "covid19"
+
+        ],
+
+
+        "feedback_and_complaints": [
+            "I want to provide feedback about my experience",
+            "how can I file a complaint about a staff member",
+            "share my thoughts on my recent visit",
+            "feedback",
+            "complaints",
+            "complain",
+            "share thoughts",
+            "experiance"
+        ],
+
+
+        "general_information": [
+            "tell me more about the hospital",
+            "what services do you offer",
+            "is there a cafeteria in the hospital",
+            "information",
+            "cafeteria",
+            "services",
+            "services offered",
+            "offered services"
+        ],
+
+        "diabetes_information": [
+
+            "diabetes",
+            "what precautions are to be taken in case of diabetes",
+            "precautions",
+            "case of diabetes",
+            "precautions taken"
+
+        ],
+
+        
+}
+
+
+video_urls = {
+
+            "greet": "static/greet.mp4",
+            "appointment_booking": "static/appointment_booking.mp4",
+            "doctor_information": "static/doctor_information.mp4",
+            "department_information": "static/department_information.mp4",
+            "visiting_hours": "static/visiting_hours.mp4",
+            "location_and_directions": "static/location_and_directions.mp4",
+            "billing_and_insurance": "static/billing_and_insurance.mp4",
+            "medical_records": "static/medical_records.mp4",
+            "emergency_services": "static/emergency_services.mp4",
+            "covid19_information": "static/covid19_information.mp4",
+            "feedback_and_complaints": "static/feedback_and_complaints.mp4",
+            "thank_you": "static/thank_you.mp4",
+            "general_information": "static/general_information.mp4",
+            "diabetes_information": "static/diabetes_information.mp4",
+            "no_answer": "static/no_answer.mp4", 
+        
+        
+        }
+
+
+def get_video_response(intent):
+
+    return video_urls.get(intent)
+
+
+
+#print(get_video_response('greet'))
+
+
+
+

templates/index.html

@@ -0,0 +1,95 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>CONVO</title>
+
+    
+    <style>
+        body {
+            margin: 0;
+            display: flex;
+            align-items: center;
+            justify-content: center;
+            height: 100vh;
+            flex-direction: column;
+        }
+
+        h1 {
+            margin-bottom: 20px;
+            color: rgb(11, 22, 53);
+            
+        }
+
+        #videoPlayer {
+            display: block;
+            margin: auto;
+        }
+    </style>
+    
+    <script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/3.0.4/socket.io.js"></script>
+</head>
+
+<body>
+   
+    <h1>CONVO</h1>
+    <video id="videoPlayer" width="800" height="800" autoplay="true" controls></video>
+
+    <script>
+        var socket = io.connect('http://' + document.domain + ':' + location.port);
+
+        // Sample video URL for continuous loop
+        var sampleVideoUrl = "{{ url_for('static', filename='Friendly_new.mp4') }}";
+
+        // Flag to indicate whether to play the sample video
+        var playSampleVideo = true;
+
+        startListening();
+
+        playVideo(sampleVideoUrl, true);
+
+        
+        function startListening() {
+            socket.emit('start_listening', {});
+        }
+
+        // Play the sample video in a loop when the page loads
+        playVideo(sampleVideoUrl, true);
+
+        socket.on('text_received', function(data) {
+            //var outputTextarea = document.getElementById('output');
+            //outputTextarea.value = 'Text: ' + data.text + '\nIntent: ' + data.intent;
+
+            // Stop playing the sample video
+            playSampleVideo = false;
+
+            // Play the answer video
+            playVideo(data.video_url, false, function() {
+                
+                playSampleVideo = true;
+                playVideo(sampleVideoUrl, true);
+            });
+        });
+
+        function playVideo(videoUrl, loop, onEndedCallback) {
+            var videoPlayer = document.getElementById('videoPlayer');
+
+            //videoPlayer.controls = false;
+            videoPlayer.src = videoUrl;
+            videoPlayer.loop = loop;
+            
+            
+            
+
+            videoPlayer.play();
+
+            if (onEndedCallback) {
+                videoPlayer.onended = onEndedCallback;
+            }
+
+            videoPlayer.play();
+        }
+    </script>
+</body>
+</html>