début pose agent

Modules/PoseEstimation/pose_agent.py

@@ -0,0 +1,74 @@
+from Modules.PoseEstimation.pose_estimator import calculate_angle, joints_id_dict, model
+from langchain.tools import tool
+from langchain.agents import AgentExecutor, create_tool_calling_agent
+from langchain_core.prompts import ChatPromptTemplate
+from langchain_core.messages import HumanMessage
+from langchain_mistralai.chat_models import ChatMistralAI
+
+# If api_key is not passed, default behavior is to use the `MISTRAL_API_KEY` environment variable.
+llm = ChatMistralAI(model='mistral-large-latest', api_key="i5jSJkCFNGKfgIztloxTMjfckiFbYBj4")
+
+@tool
+def compute_right_knee_angle(pose: list) -> float:
+
+    """
+    Computes the knee angle.
+
+    Args:
+        pose (list): list of keypoints
+
+    Returns:
+        knee_angle (float): knee angle
+    """
+
+    right_hip = pose[joints_id_dict['right_hip']]
+    right_knee = pose[joints_id_dict['right_knee']]
+    right_ankle = pose[joints_id_dict['right_ankle']]
+
+    knee_angle = calculate_angle(right_hip, right_knee, right_ankle)
+
+    print(knee_angle)
+
+    return str(knee_angle)
+
+@tool
+def get_keypoints_from_path(video_path: str):
+    """
+    Get keypoints from a video.
+
+    Args:
+        video_path (str): path to the video
+        model (YOLO): model to use
+
+    Returns:
+        keypoints (list): list of keypoints
+    """
+
+    keypoints = []
+    results = model(video_path, save=True, show_conf=False, show_boxes=False)
+    for frame in results:
+        tensor = frame.keypoints.xy[0]
+        keypoints.append(tensor.tolist())
+
+    return keypoints
+
+
+
+tools = [compute_right_knee_angle]
+
+prompt = ChatPromptTemplate.from_messages(
+    [
+        (
+            "system",
+            "You are a helpful assistant. Make sure to use the compute_right_knee_angle tool for information.",
+        ),
+        ("placeholder", "{chat_history}"),
+        ("human", "{input}"),
+        ("placeholder", "{agent_scratchpad}"),
+    ]
+)
+
+# Construct the Tools agent
+agent = create_tool_calling_agent(llm, tools, prompt)
+
+agent_executor = AgentExecutor(agent=agent, tools=tools, verbose=True)

Modules/PoseEstimation/pose_estimator.py

@@ -31,8 +31,6 @@ def get_keypoints_from_keypoints(model, video_path):
 
     return keypoints
 
-keypoints = get_keypoints_from_keypoints(model, '../../data/pose/squat.mp4')
-
 def calculate_angle(a, b, c):
 
     """
@@ -112,5 +110,4 @@ def moving_average(data, window_size):
     for i in range(len(data) - window_size + 1):
         avg.append(sum(data[i:i + window_size]) / window_size)
 
-    return avg
-
+    return avg

app.py

@@ -7,6 +7,8 @@ from dotenv import load_dotenv
 load_dotenv() # load .env api keys 
 import os
 mistral_api_key = os.getenv("MISTRAL_API_KEY")
+from Modules.PoseEstimation import pose_estimator
+from utils import save_uploaded_file
 
 st.set_page_config(layout="wide", initial_sidebar_state="collapsed")
 # Create two columns
@@ -59,12 +61,17 @@ with col2:
     if video_uploaded is None:
         video_uploaded = ask_video.file_uploader("Choose a video file", type=["mp4", "ogg", "webm"])
     if video_uploaded:
+        video_uploaded = save_uploaded_file(video_uploaded)
         ask_video.empty()
-        with st.spin("Processing video"):
-            pass # TO DO
         _left, mid, _right = st.columns(3)
         with mid:
             st.video(video_uploaded)
+            apply_pose = st.button("Apply Pose Estimation")
+
+        if apply_pose:
+            with st.spinner("Processing video"):
+                keypoints = pose_estimator.get_keypoints_from_keypoints(pose_estimator.model, video_uploaded)
+            
 
     st.subheader("Graph Displayer")
         # TO DO

utils.py

@@ -0,0 +1,12 @@
+import streamlit as st
+import os
+
+def save_uploaded_file(uploaded_file):
+    try:
+        file_path = os.path.join('uploaded', uploaded_file.name)
+        with open(file_path, 'wb') as f:
+            f.write(uploaded_file.getvalue())
+        return file_path
+    except Exception as e:
+        st.error(f"Error: {e}")
+        return None