update

app.py

@@ -26,7 +26,7 @@ from numpy.typing import NDArray
     iris_mask_blur - final iris edge blur
 '''
 
-def process_images(image,base_image,
+def process_images(image,base_image,order,
                    double_check_offset_center,center_index,
                    draw_mediapipe_mesh,z_multiply=0.8,draw_mediapipe_angle=False,draw_hozizontal_line=False,draw_vertical_line=False,draw_faceratio_line=False,
         progress=gr.Progress(track_tqdm=True)):
@@ -118,9 +118,9 @@ def process_images(image,base_image,
    
     if draw_hozizontal_line:
         for cordinates in h_cordinates:
-            print(cordinates)
+            #print(cordinates)
             points = normalized_to_pixel(cordinates,w,h)
-            print(points)
+            #print(points)
             plot_points(cv2_image,points[:2],False,5,(255,0,0),3)#last one is middle point on horizontal
         
             
@@ -179,6 +179,7 @@ def process_images(image,base_image,
     im_with_pose = cv2_image
     mediapipe_text = None
 
+    
     def face_landmarker_result_to_angle_label(face_landmarker_result,order="yxz"):
         if len(face_landmarker_result.facial_transformation_matrixes)>0:
             
@@ -195,16 +196,16 @@ def process_images(image,base_image,
             r = R.from_matrix(rotation_matrix)
             euler_angles = r.as_euler(order, degrees=True)
             #label = f"Media pipe {order}-Euler Angles [x,y,z] (degrees): [{euler_angles[1]:.2f},{euler_angles[0]:.2f},{euler_angles[2]:.2f}]"
-            label = f"[x:{euler_angles[1]:.2f},y:{-euler_angles[0]:.2f},z:{-euler_angles[2]:.2f}]"
+            label = f"[{order[0]}:{euler_angles[0]:.2f},{order[1]}:{-euler_angles[1]:.2f},{order[2]}:{-euler_angles[2]:.2f}]"
             
             return label,rotation_matrix,scaled_translation_vector
         
     if first_landmarker_result != None:
-        mediapipe_first_text,_,_ = face_landmarker_result_to_angle_label(first_landmarker_result)
+        mediapipe_first_text,_,_ = face_landmarker_result_to_angle_label(first_landmarker_result,order)
     else:
         mediapipe_first_text = ""
         
-    mediapipe_second_text,rotation_matrix,scaled_translation_vector = face_landmarker_result_to_angle_label(face_landmarker_result)
+    mediapipe_second_text,rotation_matrix,scaled_translation_vector = face_landmarker_result_to_angle_label(face_landmarker_result,order)
 
     rotation_vector, _ = cv2.Rodrigues(rotation_matrix)
     translation_vector =  scaled_translation_vector
@@ -222,9 +223,9 @@ def process_images(image,base_image,
     
         r = R.from_matrix(rotation_matrix)
         euler_angles = r.as_euler("yxz", degrees=False)
-        print(r.as_euler("yxz", degrees=True))
-        draw_cordinate1=rotate_point_euler((0,0,-100),[-euler_angles[1],euler_angles[0],euler_angles[2]],"yzx")
-        draw_cordinate2=rotate_point_euler((0,0,-200),[-euler_angles[1],euler_angles[0],euler_angles[2]],"yzx")
+        #print(r.as_euler("yxz", degrees=True))
+        draw_cordinate1=rotate_point_euler((0,0,-100),[-euler_angles[1],euler_angles[0],euler_angles[2]],"yxz")
+        draw_cordinate2=rotate_point_euler((0,0,-200),[-euler_angles[1],euler_angles[0],euler_angles[2]],"yxz")
         
         plot_points(im_with_pose,[root_cordinate[:2]+draw_cordinate1[:2],root_cordinate[:2]+draw_cordinate2[:2],root_cordinate[:2]],False,5,(0,128,0),3,(0,255,0))
 
@@ -233,9 +234,9 @@ def process_images(image,base_image,
     face_ratio_infos = []
    
 
-    print("landmark",[landmarks[37],landmarks[267]])
-    print("numpy",np.array([landmarks[37],landmarks[267]]))
-    print("mean",np.mean(np.array([landmarks[37],landmarks[267]]),axis=0))
+    #print("landmark",[landmarks[37],landmarks[267]])
+    #print("numpy",np.array([landmarks[37],landmarks[267]]))
+    #print("mean",np.mean(np.array([landmarks[37],landmarks[267]]),axis=0))
     v_cordinates=[
     ["philtrum",landmarks[175],landmarks[13],np.mean((landmarks[164],landmarks[2]),axis=0).tolist()],
     ["straight",landmarks[175],landmarks[94],landmarks[9]],
@@ -258,7 +259,7 @@ def process_images(image,base_image,
     return cv2.cvtColor(im_with_pose,cv2.COLOR_BGR2RGB),mediapipe_first_text,mediapipe_second_text,z_angle_text,y_ratio_text,x_ratio_text,z_angle_detail,y_ratio_detail,x_ratio_detail,face_ratio_info
 
 
-
+#deprecated
 def find_nearest_weighted_euclidean_2d(target_angles_full, all_angles_full, weights):
     target_angles = target_angles_full[:5]  # 最初の3つの角度を使用
     all_angles = all_angles_full[:, :5]  # 最初の3列を使用
@@ -268,8 +269,13 @@ def find_nearest_weighted_euclidean_2d(target_angles_full, all_angles_full, weig
     nearest_index = np.argmin(distances)
     return nearest_index, all_angles_full[nearest_index]
 
+import math
 from mp_estimate import estimate_horizontal_points ,estimate_vertical_points,estimate_rotations_v2
-def find_angles(image):
+
+import joblib
+stacking8_model = joblib.load(f"models/stacking8.joblib")
+cached_models = {}
+def find_angles(image,order):
     if image is None:
          raise gr.Error("need image")
     cv2_image = pil_to_bgr_image(image)
@@ -280,6 +286,7 @@ def find_angles(image):
     features_text = estimate_rotations_v2(face_landmarker_result)
     features_value_origin = [float(value) for value in features_text.split(",")]
     features_value = features_value_origin.copy()
+    print("features x-angle",math.degrees(features_value[3])-90)
     #print(features_value)
     #weights = np.array([0.2, 0.2,0.3,0.3])
 
@@ -296,31 +303,67 @@ def find_angles(image):
         [np.hstack([features_value[ 3:5],features_value[ 6:-x_ratios]])]
         #[features_value[:-x_ratios]]
     ]
-    import joblib
-    def estimate(model_path,scaler_path,features_values):
-        scalers = joblib.load("models/"+scaler_path)
+
+
+    from scipy.spatial.transform import Rotation as R
+    def flatten_for(lst):
+        return [round(item, 3) for sublist in lst for item in sublist]
+    def change_euler_order(orderd_array,from_order,to_order,degrees=True):
+         r = R.from_euler(from_order,orderd_array,degrees=degrees)
+         result = r.as_euler(to_order,degrees=degrees)
+         return np.round(result,2).tolist()
+
+    def load_joblib(path):
+        if path in cached_models:
+            return cached_models[path]
+        else:
+             model = joblib.load(path)
+             cached_models[path] = model
+             return model
+        
+    def estimate(model_path,scaler_path,features_values,multi=True):
+        
+        scalers = load_joblib("models/"+scaler_path)
         if not isinstance(scalers,list):
             scalers=(scalers,scalers,scalers)
         for i,scaler  in enumerate(scalers):
-            print(i,scaler)
+            #print(i,scaler)
             features_values[i] = scaler.transform(features_values[i].copy())
 
 
         result_preds=[]
-        models = joblib.load("models/"+model_path)
-        for i,model in enumerate(models):
-            y_pred = model.predict(features_values[i])
-            result_preds.append(y_pred.round(2))
-        return result_preds
+        models = load_joblib("models/"+model_path)
+        
+        if multi: 
+            for i,model in enumerate(models):
+                y_pred = model.predict(features_values[i])
+                result_preds.append(y_pred.round(2))
+            result_preds=flatten_for(result_preds)    
+            yxz =[result_preds[1],result_preds[0],result_preds[2]]
+        else:
+              result_preds=models.predict(features_values[0])
+              result_preds=flatten_for(result_preds)
+              #yxz=flatten_for(yxz)
+              #yxz =[yxz[1],yxz[0],yxz[2]]
+                
+        #zyx = change_euler_order(yxz,"yxz","zyx")
+        #return [round(zyx[2],2),round(zyx[1],2),round(zyx[0],2)]#
+        return result_preds # yxz-orderd x,y,z
+    
+
+
+    
     def estimate2(model_key,features_values):
         model_path=f"models/{model_key}.joblib"
         scaler_path=f"models/{model_key}_scaler.joblib"
         polynomial_path=f"models/{model_key}_polynomial_features.joblib"
         selectkbest_path=f"models/{model_key}_selectkbest.joblib"
-        model = joblib.load(model_path)
-        scaler = joblib.load(scaler_path)
-        polynomial = joblib.load(polynomial_path)
-        selectkbest = joblib.load(selectkbest_path)
+
+        model = load_joblib(model_path)
+        scaler = load_joblib(scaler_path)
+        polynomial = load_joblib(polynomial_path)
+        selectkbest = load_joblib(selectkbest_path)
+
         result_preds=[]
         for i in range(3):
             x = polynomial[i].transform(features_values[i].copy())
@@ -328,7 +371,20 @@ def find_angles(image):
             x = scaler[i].transform(x)
             y_pred = model[i].predict(x)
             result_preds.append(y_pred.round(2))
-        return result_preds
+        return result_preds # yxz-orderd x,y,z
+    import onnxruntime as ort
+    def estimate3(model_key,features_values):
+        model_path=f"models/{model_key}.onnx"
+        ort_session = ort.InferenceSession(model_path)
+       
+        #result_preds=[]
+        #result_preds=models.predict(features_values[0])
+        #result_preds=flatten_for(result_preds)
+        input_name = ort_session.get_inputs()[0].name
+        input_data = features_values.astype(np.float32)
+        result_preds = ort_session.run(None, {input_name: input_data})
+        #print((result_preds))
+        return result_preds[0] # yxz-orderd x,y,z
 
 
     #short_result = estimate('linear-svr-xyz_5.joblib','linear-svr-xyz_5_scaler.joblib',features_values)
@@ -359,30 +415,57 @@ def find_angles(image):
     middle_result2 = estimate(f'{e1_key}.joblib',f'{e1_key}_scaler.joblib',features_values.copy())
     e1_key="lgbm-optimizer_90_random"
     long_result2 = estimate(f'{e1_key}.joblib',f'{e1_key}_scaler.joblib',features_values.copy())
-    def flatten_for(lst):
-        return [round(item, 3) for sublist in lst for item in sublist]
-        
+
+    e1_key="etr_90"
+    long_result3 = estimate(f'{e1_key}.joblib',f'{e1_key}_scaler.joblib',features_values.copy(),False)
+    #long_result3 = estimate3(e1_key,np.array([features_value]))#single 
+    #long_result3 = flatten_for(long_result3)
+    #long_result3 = long_result2
     def average(values):
         flat_values=[]
         for value in values:
              flat_values += [flatten_for(value)]
-        print(np.mean(flat_values,axis=0))
+        #print(np.mean(flat_values,axis=0))
 
     import average
     data={
          "hgbr-15":flatten_for(short_result),
          "hgbr-45":flatten_for(middle_result),
          "hgbr-90":flatten_for(long_result),
-         "lgbm-15dart":flatten_for(short_result2a),
-         "lgbm-15":flatten_for(short_result2),
-         "lgbm-45":flatten_for(middle_result2),
-         "lgbm-90":flatten_for(long_result2),
+         "lgbm-15dart":(short_result2a),
+         "lgbm-15":(short_result2),
+         "lgbm-45":(middle_result2),
+         "lgbm-90":(long_result2),
     }
+
+    
+    stack_x = short_result2a+short_result2+middle_result2+long_result2+flatten_for(short_result)+flatten_for(middle_result)+flatten_for(long_result)+long_result3
+    
+    #average_data=estimate3("stacking8",np.array([stack_x]))#onnx not 
+    average_data=stacking8_model.predict(np.array([stack_x]))
+    
+    #change order
+    
+    #all data train with yxz-order x,y,z
+    def yxz_xyz_to_yxz(euler):
+         return [euler[1],euler[0],euler[2]]
+    
+    average_data = change_euler_order(yxz_xyz_to_yxz(flatten_for(average_data)),"yxz",order)
+    short_result = change_euler_order(yxz_xyz_to_yxz(flatten_for(short_result)),"yxz",order)
+    middle_result = change_euler_order(yxz_xyz_to_yxz(flatten_for(middle_result)),"yxz",order)
+    long_result = change_euler_order(yxz_xyz_to_yxz(flatten_for(long_result)),"yxz",order)
+    short_result2a = change_euler_order(yxz_xyz_to_yxz(short_result2a),"yxz",order)
+    short_result2 = change_euler_order(yxz_xyz_to_yxz(short_result2),"yxz",order)
+    middle_result2 = change_euler_order(yxz_xyz_to_yxz(middle_result2),"yxz",order)
+    long_result2 = change_euler_order(yxz_xyz_to_yxz(long_result2),"yxz",order)
+    long_result3 = change_euler_order(yxz_xyz_to_yxz(long_result3),"yxz",order)
+
     #print(data)
-    average_data=average.analyze_3d_data(data.values())
-    print(average_data)
+    #average_data=average.analyze_3d_data(data.values())
+    #print(average_data)
     #average((short_result,middle_result,long_result,short_result2a,short_result2,middle_result2,long_result2))
-    return average_data['trimmed_mean'],flatten_for(short_result),flatten_for(middle_result),flatten_for(long_result),flatten_for(short_result2a),flatten_for(short_result2),flatten_for(middle_result2),flatten_for(long_result2)
+    return average_data,short_result,middle_result,long_result,(short_result2a),(short_result2),(middle_result2),(long_result2),long_result3
+    #return average_data['trimmed_mean'],flatten_for(short_result),flatten_for(middle_result),flatten_for(long_result),(short_result2a),(short_result2),(middle_result2),(long_result2)
 
 css="""
 #col-left {
@@ -426,7 +509,8 @@ with gr.Blocks(css=css, elem_id="demo-container") as demo:
                     with gr.Row(elem_id="prompt-container",  equal_height=False):
                         with gr.Row():
                             btn = gr.Button("Head-Pose Estimate", elem_id="run_button",variant="primary")
-                    
+                            order = gr.Dropdown(label="Order",value="xyz",choices=["xyz","xzy","yxz","yzx","zxy","zyx"],info="returened array order is same as label")
+                            
                     
                         
                     with gr.Accordion(label="Advanced Settings", open=True):
@@ -435,7 +519,7 @@ with gr.Blocks(css=css, elem_id="demo-container") as demo:
                        
                         with gr.Row( equal_height=True):
                             
-                            double_check = gr.Checkbox(label="Double Check",value=True,info="move center and detect again(usually more accurate).recommend choose 195") 
+                            double_check = gr.Checkbox(label="Double Check",value=True,info="move center-index and detect again(usually more accurate).recommend choose 195") 
                             center_index = gr.Slider(info="center-index",
                             label="Center-index",
                             minimum=0,
@@ -460,8 +544,8 @@ with gr.Blocks(css=css, elem_id="demo-container") as demo:
                 with gr.Column():
                     result_image = gr.Image(height=760,label="Result", elem_id="output-animation",image_mode='RGB')
                     with gr.Row( equal_height=True):
-                        mediapipe_last_text = gr.Textbox(label="2nd or last mediapipe result(yzx-eulder[x,y,z])")
-                        mediapipe_first_text = gr.Textbox(label="first mediapipe result(yzx-eulder[x,y,z])")
+                        mediapipe_last_text = gr.Textbox(label=f"2nd or last mediapipe result",)
+                        mediapipe_first_text = gr.Textbox(label=f"first mediapipe result")
                     
                     with gr.Row( equal_height=True):
                         z_angle_text = gr.Textbox(label="Z angle by horizontal-line",info="start with 0,exactly Z-Angle")
@@ -474,23 +558,25 @@ with gr.Blocks(css=css, elem_id="demo-container") as demo:
                             x_ratio_detail = gr.TextArea(label="X-ratio detail",value="")
                     with gr.Row( equal_height=True):
                         face_ratio_info = gr.Text(label="Face Ratio",info="Average philtrum:1.82(std 0.13),straight:0.82(std 0.04),face:0.91(std 0.02),r-eyes:0.86(std 0.03),r-contour:0.77(std 0.05),l-eyes:0.86(std 0.03),l-contour:0.75(std 0.05),lips:1.43(std 0.16),mouth-eye:1.21(std 0.07)")
-                    gr.HTML("<h5>For Rotation sometime differenct to mediapipe's result(Especially X usually minus 4-7)</h5>")
-                    bt_test = gr.Button("Model-Estimate")
-                    gr.HTML("<p>YXZ-Euler [x,y,z] hgbr is stable,lgbm is accurate(dart is more).trimmed works well on small angles</p>")  
+                    gr.HTML("<h5>For Rotation sometime differenct to mediapipe's result</h5>")
+                    with gr.Row( equal_height=True):  
+                        bt_test = gr.Button("Estimate by Models")
+                        average_result = gr.Text(label="stacking")
+                    gr.HTML("<p>number is max training angle,usually stacking is works well.slow because of etr</p>")  
                     with gr.Row( equal_height=True):  
-                            average_result = gr.Text(label="trimmed-mean")
                             short_result = gr.Text(label="hgbr-15")
                             middle_result = gr.Text(label="hgbr-45")
                             long_result = gr.Text(label="hgbr-90")
+                            long_result3 = gr.Text(label="etr-90")
                     with gr.Row( equal_height=True):  
                             short_result2a = gr.Text(label="lgbm-15dart")
                             short_result2 = gr.Text(label="lgbm-15")
                             middle_result2 = gr.Text(label="lgbm-45")
                             long_result2 = gr.Text(label="lgbm-90")
                             #,
-                            bt_test.click(fn=find_angles,inputs=image,outputs=[average_result,short_result,middle_result,long_result,short_result2a,short_result2,middle_result2,long_result2])
+                            bt_test.click(fn=find_angles,inputs=[image,order],outputs=[average_result,short_result,middle_result,long_result,short_result2a,short_result2,middle_result2,long_result2,long_result3])
 
-    btn.click(fn=process_images, inputs=[image,base_image,
+    btn.click(fn=process_images, inputs=[image,base_image,order,
                                          double_check,center_index,
                                          draw_mediapipe_mesh,z_multiply,draw_mediapipe_angle,draw_hozizontal_line,draw_vertical_line,draw_faceratio_line,
                                          ],outputs=[result_image,mediapipe_first_text,mediapipe_last_text,z_angle_text,y_ratio_text,x_ratio_text,z_angle_detail_text,y_ratio_detail,x_ratio_detail,face_ratio_info] ,api_name='infer')

demo_header.html

@@ -12,7 +12,8 @@
             Accurate detection in MediaPipe requires correct positioning, but I don't know the exact position needed. This is a known <a href="https://github.com/google-ai-edge/mediapipe/issues/4759">issue</a><br>
             I start to test hgbr and lgbm models,this estimate result help to make aligned image.<br>
             center index see <a href="https://github.com/google-ai-edge/mediapipe/blob/a908d668c730da128dfa8d9f6bd25d519d006692/mediapipe/modules/face_geometry/data/canonical_face_model_uv_visualization.png">mediapipe face index image</a><br>
-            Recently Choose center index and mediapipe-estimate + models trained with face-features extracted by mediapipe
+            Recently Choose center index and mediapipe-estimate + models trained with face-features extracted by mediapipe<br>
+            I'll share model and dataset soon.
         </p>
     </div>
     

mp_estimate.py

@@ -114,9 +114,7 @@ def get_feature_ratios_cordinate(face_landmarks,ratios=feature_ratios_indices):
 def ratios_cordinates(cordinates):
     
     distance_a = calculate_distance(cordinates[0],cordinates[1])
-    print(distance_a)
     distance_b = calculate_distance(cordinates[-2],cordinates[-1])
-    print(distance_b)
     if distance_a == 0 or distance_b == 0:
         return 0
     else: