diff --git a/api/exam.pt b/api/exam.pt deleted file mode 100644 index aa344f4..0000000 Binary files a/api/exam.pt and /dev/null differ diff --git a/api/inference.py b/api/inference.py new file mode 100644 index 0000000..b4c26f3 --- /dev/null +++ b/api/inference.py @@ -0,0 +1,46 @@ +from ultralytics import YOLO +import cv2 +import os +current_dir = os.path.dirname(os.path.abspath(__file__)) +det_model = YOLO(os.path.join(current_dir, "yolov8n-pose.pt")) +cls_model = YOLO(os.path.join(current_dir, "yolov8n-cls.pt")) +# det_model = YOLO('yolov8n-pose.pt') +# cls_model = YOLO('yolov8n-cls.pt') +class_name = ['normal', 'raise_hand', 'speak', 'stand', 'turn_head', 'use_phone'] + +def infer(image): + det_results = det_model(image, conf=0.5, iou=0.25) + for r in det_results: + if len(r) == 0: + return 'leave' + box = r.boxes.xyxy + if len(box) == 1: + crop_image = image[int(box[0][1]):int(box[0][3]), int(box[0][0]):int(box[0][2])] + cls_results = cls_model(crop_image) + return class_name[cls_results[0].probs.top1] + else: + return 'many_humans' + +# image_path = os.path.join(current_dir, "frames/video1_1.png") +# result = infer(image_path) +# print(result) + +cap = cv2.VideoCapture(0) +if cap.isOpened(): + while True: + ret, frame = cap.read() + if ret: + result = infer(frame) + print(result) + # 设置要添加的文本内容和位置 + org = (50, 50) # 文本起始位置 + font = cv2.FONT_HERSHEY_SIMPLEX + font_scale = 1 + color = (255, 0, 0) # 文本颜色，BGR格式 + thickness = 2 # 文本字体粗细 + + # 添加文本到图像 + cv2.putText(frame, result, org, font, font_scale, color, thickness, cv2.LINE_AA) + cv2.imshow('test.png', frame) + cv2.waitKey(1000) + diff --git a/api/main.py b/api/main.py index edb4b9b..55654c9 100644 --- a/api/main.py +++ b/api/main.py @@ -1,9 +1,8 @@ from flask import Flask, request, jsonify +from flask_cors import CORS from pymongo import MongoClient from ultralytics import YOLO -from PIL import Image import base64 -from io import BytesIO from bson import ObjectId import hashlib import json @@ -11,25 +10,30 @@ import os from flask import Flask, request from flask_socketio import SocketIO, emit, join_room, leave_room import time +import numpy as np +import cv2 app = Flask(__name__) + client = MongoClient("mongodb://localhost:27019/") db = client["back"] usercollection = db["users"] zuobicollection = db["zuobi"] # sockitIo解决跨域问题 app.config["SECRET_KEY"] = "secret!" +CORS(app, resources={r"/socket.io/*": {"origins": "*"}}) # 允许所有来源的请求 socketio = SocketIO(app, cors_allowed_origins="*") +# CORS(app) # 存储连接的客户端的房间号 clients = [] adminSid = "" # 初始化考试状态 studentStatus = {} current_dir = os.path.dirname(os.path.abspath(__file__)) -weights = os.path.join(current_dir, "exam.pt") -print(89744, weights) -model = YOLO(weights) -frame_rate_divider = 300 # 设置帧率除数 +det_model = YOLO(os.path.join(current_dir, "yolov8n-pose.pt")) +cls_model = YOLO(os.path.join(current_dir, "yolov8n-cls.pt")) +class_name = ["normal", "raise_hand", "speak", "stand", "turn_head", "use_phone"] +frame_rate_divider = 90 # 设置帧率除数 frame_count = 0 # 初始化帧计数器 @@ -43,7 +47,7 @@ def dealStudentStatus(): dealStudentStatus() -@socketio.on("connect", namespace="/ws/video") +@socketio.on("connect") def on_connect(): global clients global adminSid @@ -53,11 +57,10 @@ def on_connect(): isAdmin = request.args.get("isAdmin") if isAdmin == "1": adminSid = sid - print(87874, adminSid) clients.append({"id": id, "sid": sid, "isAdmin": isAdmin}) -@socketio.on("disconnect", namespace="/ws/video") +@socketio.on("disconnect") def on_disconnect(): global clients for item in clients: @@ -66,19 +69,16 @@ def on_disconnect(): print(f"Client disconnected with id {item['id']}") -@socketio.on("video", namespace="/ws/video") +@socketio.on("video") def handle_video_frame(message): # 广播视频帧给老师 global adminSid global studentStatus - global model global frame_rate_divider global frame_count - # print(f"Received video frame from {message['userId']}") - socketio.emit( - "teacherVideo" + message["userId"], message, to=adminSid, namespace="/ws/video" - ) + frame_count += 1 # 更新帧计数器 + frame_count %= frame_rate_divider # 更新帧计数器 if frame_count % frame_rate_divider == 0: # 这里省略了实际的base64数据 base64_str = message["data"] @@ -86,56 +86,80 @@ def handle_video_frame(message): _, img_data = base64_str.split(",") img_data = base64.b64decode(img_data) # 将字节流转换为PIL图像对象 - image = Image.open(BytesIO(img_data)) - results = model.predict(source=image, iou=0.5, conf=0.25) - det = results[0] - checkVedioResult(message, det) - frame_count += 1 # 更新帧计数器 - frame_count %= frame_rate_divider # 更新帧计数器 - # print(99777,frame_count) - # if studentStatus[message["userId"]]: - # print(f"学生{message['userId']}已作弊", f"{studentStatus[message['userId']]}") + # 将字节数据转换为NumPy数组 + np_data = np.frombuffer(img_data, dtype=np.uint8) + # 使用cv2.imdecode将数组解码为图像 + image = cv2.imdecode(np_data, cv2.IMREAD_COLOR) + checkVedioResult(message, image) + socketio.emit( + "teacherVideo" + message["userId"], message, to=adminSid + ) -def checkVedioResult(message, det): + +def checkVedioResult(message, image): # 如果有有效的检测结果 # {"cheating": "疑似作弊", "good": "良好", "normal": "正常"} - if det is not None and len(det): - results = [] # 初始化结果列表 - for res in det.boxes: - for box in res: - # 提前计算并转换数据类型 - class_id = int(box.cls.cpu()) - bbox = box.xyxy.cpu().squeeze().tolist() - bbox = [int(coord) for coord in bbox] # 转换边界框坐标为整数 - result = { - "bbox": bbox, # 边界框 - "score": box.conf.cpu().squeeze().item(), # 置信度 - "class_id": class_id, # 类别ID - } - results.append(result) # 将结果添加到列表 - zuobiItem = { - "userId": message["userId"], - "msg": class_id, - "type": class_id, - "create_at": time.strftime( - "%Y-%m-%d %H:%M:%S", time.localtime(time.time()) - ), - "update_at": time.strftime( - "%Y-%m-%d %H:%M:%S", time.localtime(time.time()) - ), - "isExit": 1, - } - addzuobi(zuobiItem) - - -@socketio.on("talk", namespace="/ws/video") + global det_model + global cls_model + global class_name + global clients + global adminSid + det_results = det_model(image,conf=0.5, iou=0.25) + type = "" + for r in det_results: + if len(r) == 0: + type = "leave" + break + # print(57841,r.boxes) + box = r.boxes.xyxy + if len(box) == 1: + crop_image = image[ + int(box[0][1]) : int(box[0][3]), int(box[0][0]) : int(box[0][2]) + ] + # cv2.imshow('test.png', crop_image) + # cv2.waitKey(0) + cls_results = cls_model(crop_image) + type = class_name[cls_results[0].probs.top1] + else: + type = "many_humans" + message["type"] = type + socketio.emit( + "teacherTalk" + message["userId"], + message, + to=adminSid, + ) + sid = "" + for item in clients: + if item["id"] == message["userId"]: + sid = item["sid"] + socketio.emit( + "studentMsg" + message["userId"], message, to=sid + ) + # print(97444, type) + zuobiItem = { + "userId": message["userId"], + "msg": message["data"], + "type": type, + "create_at": time.strftime( + "%Y-%m-%d %H:%M:%S", time.localtime(time.time()) + ), + "update_at": time.strftime( + "%Y-%m-%d %H:%M:%S", time.localtime(time.time()) + ), + "isExit": 1, + } + addzuobi(zuobiItem) + + +@socketio.on("talk") def handle_talk(message): # 说话实时传输给老师 global adminSid # print(f"Received video frame from {message['userId']}") + message["type"] = "talk" socketio.emit( - "teacherTalk" + message["userId"], message, to=adminSid, namespace="/ws/video" + "teacherTalk" + message["userId"], message, to=adminSid ) zuobiItem = { "userId": message["userId"], @@ -148,13 +172,13 @@ def handle_talk(message): addzuobi(zuobiItem) -@socketio.on("sendMsg", namespace="/ws/video") +@socketio.on("sendMsg") def handle_msg(message): # 说话实时传输给老师 global adminSid # print(f"Received video frame from {message['userId']}") socketio.emit( - "teacherMsg" + message["userId"], message, to=adminSid, namespace="/ws/video" + "teacherMsg" + message["userId"], message, to=adminSid ) zuobiItem = { "userId": message["userId"], @@ -167,14 +191,15 @@ def handle_msg(message): addzuobi(zuobiItem) -@socketio.on("answerMsg", namespace="/ws/video") +@socketio.on("answerMsg") def handle_answer_msg(message): sid = "" for item in clients: if item["id"] == message["userId"]: sid = item["sid"] + message["type"] = "answer" socketio.emit( - "studentMsg" + message["userId"], message, to=sid, namespace="/ws/video" + "studentMsg" + message["userId"], message, to=sid ) zuobiItem = { "userId": message["userId"], @@ -208,9 +233,9 @@ def initData(): "nianji": "", "banji": "", "xueshengxingming": "老师", - "xuehao": "0000", + "xuehao": "admin", "kaoshengtupian": "", - "mima": md5_encrypt("0000"), + "mima": md5_encrypt("admin"), "chengji": "", "kaoshileixing": "", "kaoshikemu": "", @@ -476,4 +501,4 @@ def getzuobi(): if __name__ == "__main__": initData() # app.run(debug=True) - socketio.run(app, debug=True) + socketio.run(app, host='0.0.0.0', port=5000, debug=True) diff --git a/api/yolov8n-cls.pt b/api/yolov8n-cls.pt new file mode 100644 index 0000000..2616bbf Binary files /dev/null and b/api/yolov8n-cls.pt differ diff --git a/api/yolov8n-pose.pt b/api/yolov8n-pose.pt new file mode 100644 index 0000000..f41b11f Binary files /dev/null and b/api/yolov8n-pose.pt differ diff --git a/front/public/config.json b/front/public/config.json deleted file mode 100644 index 0f21a96..0000000 --- a/front/public/config.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "title": "可视化预警平台", - "jianceduixiang1path": "@/assets/img/jianceduixiang1.gif", - "jianceduixiang2path": "@/assets/img/jianceduixiang2.jpg", - "jianceduixiang3path": "@/assets/img/jianceduixiang3.png", - "jianceduixiang4path": "@/assets/img/jianceduixiang4.png", - "jianceduixiang5path": "@/assets/img/jianceduixiang5.gif", - "jianceduixiang6path": "@/assets/img/jianceduixiang6.gif" -} diff --git a/front/src/assets/css/base.css b/front/src/assets/css/base.css index 955ddcd..0650964 100644 --- a/front/src/assets/css/base.css +++ b/front/src/assets/css/base.css @@ -4,4 +4,7 @@ html,body { } .el-form-item__content{ width: 250px; +} +.mx-4{ +margin: 8px 8px; } \ No newline at end of file diff --git a/front/src/component/table.vue b/front/src/component/table.vue index 1f54f2c..d0df75b 100644 --- a/front/src/component/table.vue +++ b/front/src/component/table.vue @@ -1,5 +1,5 @@