Click here to Skip to main content
15,868,009 members
Please Sign up or sign in to vote.
0.00/5 (No votes)
As a part of my project involving conversion of sign language to english , I developed .tflite model and did predictions successfully using opencv.

Now i want to implement web application for this . On the web page , there will be live streaming going on and frames will be predicted and the corresponding predicted text has to be displayed on the top of page
This camera.py will take image from live streaming and predict it and return the modified image along with the predicted sequence to app.py.

Now the problem is how do i continously return these both(frame and the sequence) to index.html

What I have tried:

My app.py code is as follows
from flask import Flask, render_template, Response
from camera import Video
import time
sequence = ''
app=Flask(__name__)

print("before gen",time.perf_counter())
def gen(camera):
    while True:
        frame,sequence=camera.get_frame()
        print(sequence)
        sequence = sequence.upper()
        yield(b'--frame\r\n'
            b'Content-Type:  image/jpeg\r\n\r\n' + frame  + b'\r\n')

@app.route('/video')
def video():
    return Response(gen(Video()),mimetype='multipart/x-mixed-replace; boundary=frame') 
    
@app.route('/')
def index():
    return render_template('index.html' , sequence = sequence)

app.run(debug=True)

The camera.py code is as follows
import os
import time
import numpy as np
import tensorflow.lite as tflite
import cv2
# Disable tensorflow compilation warnings
os.environ['TF_CPP_MIN_LOG_LEVEL']='3'
import tensorflow as tf

class Video(object):
    def __init__(self):
        self.video=cv2.VideoCapture(0)
        self.model()
    def __del__(self):
        self.video.release()
    def model(self):
        with open("logs/trained_labels.txt", 'r') as f:
            self.labels =  [line.strip() for line in f.readlines()]
        self.interpreter = tflite.Interpreter(model_path="model_unquant.tflite")
        self.interpreter.allocate_tensors()
        self.input_details = self.interpreter.get_input_details()
        self.input_index = self.input_details[0]['index']
        self.c= 0
        self.res, self.score= '', 0.0
        self.i = 0
        self.mem = ''
        self.consecutive = 0
        self.sequence = ''
    def predict(self,image_data):
        #print(image_data.shape)
        input_data = np.expand_dims(image_data, axis=0).astype(np.float32)  # expand to 4-dim
        #print(input_data)
        self.interpreter.set_tensor(self.input_index,input_data)
        self.interpreter.invoke()
        output_details = self.interpreter.get_output_details()
        output_data = self.interpreter.get_tensor(output_details[0]['index'])
        output_data = np.squeeze(output_data)
        top_k = output_data.argsort()[-len(output_data):][::-1]
        a = []
        for i in top_k:
            sign = self.labels[i]
            score = output_data[i]
            sign = self.labels[i]
            score = output_data[i]
            a.append((sign,score))
            a = sorted(a, key = lambda x: x[1],reverse=True)
        return a[0][0], a[0][1]
    def get_frame(self):
        print("getframe",time.perf_counter())
        ret,frame=self.video.read()
        img = cv2.flip(frame, 1)    
        if ret:
            x1, y1, x2, y2 = 200, 100, 424, 324
            img_cropped = img[y1:y2, x1:x2]
            self.c+= 1
            #image_data = cv2.imencode('.jpg', img_cropped)[1].tostring() 
            if self.i == 4:
                res_tmp, self.score= self.predict(img_cropped)
                self.res= res_tmp
                self.i = 0
                if self.mem == self.res:
                    self.consecutive += 1
                else:
                    self.consecutive = 0
                if self.consecutive == 2 and self.res not in ['nothing']:
                    if self.res== 'space':
                        self.sequence += ' '
                    elif self.res== 'del':
                        self.sequence = self.sequence[:-1]
                    else:
                        self.sequence += self.res
                    self.consecutive = 0
            print("after getframe",time.perf_counter())
            self.i += 1
            cv2.putText(img, '%s' % (self.res.upper()), (250,400), cv2.FONT_HERSHEY_SIMPLEX, 3, (255,255,255), 4)
            cv2.putText(img, '%.3f' % (float(self.score)), (200,450), cv2.FONT_HERSHEY_SIMPLEX, 1, (255,255,255))
            self.mem = self.res
            cv2.rectangle(img, (x1, y1), (x2, y2), (18,31,150), 3)

        ret,jpg=cv2.imencode('.jpg',img)
        return jpg.tobytes(),self.sequence.upper()


the model and labels files are in the following link
https://www.dropbox.com/sh/nej0cvmo2w34g1a/AAAYeEd4A6f4VzcdyTVP30Hia?dl=0

These 2 fields (image and sequence )should be keep on updating as long as the application is running. How do i send these both at a time to html page. how do i use both response and render_template keywords at a time. Please help me..
Posted

This content, along with any associated source code and files, is licensed under The Code Project Open License (CPOL)



CodeProject, 20 Bay Street, 11th Floor Toronto, Ontario, Canada M5J 2N8 +1 (416) 849-8900