进一步优化

.gitignore

@@ -7,6 +7,7 @@
 # Precompiled Headers
 *.gch
 *.pch
+*.pyc
 
 # Compiled Dynamic libraries
 *.so
@@ -37,3 +38,40 @@ CMakeSettings.json
 out/
 bin*/
 make*/
+libs/MVS/texture_output/texture_0.png
+libs/MVS/texture_output/texture_1.png
+libs/MVS/texture_output/texture_2.png
+libs/MVS/texture_output/texture_3.png
+libs/MVS/texture_output/texture_4.png
+libs/MVS/texture_output/texture_5.png
+libs/MVS/texture_output/texture_6.png
+libs/MVS/texture_output/texture_7.png
+libs/MVS/texture_output/texture_8.png
+libs/MVS/texture_output/texture_9.png
+libs/MVS/texture_output/texture_10.png
+libs/MVS/texture_output/texture_11.png
+libs/MVS/texture_output/texture_12.png
+libs/MVS/texture_output/texture_13.png
+libs/MVS/texture_output/texture_14.png
+libs/MVS/texture_output/texture_15.png
+libs/MVS/texture_output/texture_16.png
+libs/MVS/texture_output/texture_17.png
+libs/MVS/texture_output/texture_18.png
+libs/MVS/texture_output/texture_19.png
+libs/MVS/texture_output/texture_20.png
+libs/MVS/texture_output/texture_21.png
+libs/MVS/texture_output/texture_22.png
+libs/MVS/texture_output/texture_23.png
+libs/MVS/texture_output/texture_24.png
+libs/MVS/texture_output/texture_25.png
+libs/MVS/texture_output/texture_26.png
+libs/MVS/texture_output/texture_27.png
+libs/MVS/texture_output/texture_28.png
+libs/MVS/texture_output/texture_29.png
+libs/MVS/texture_output/texture_30.png
+libs/MVS/texture_output/texture_31.png
+libs/MVS/texture_output/texture_32.png
+libs/MVS/texture_output/texture_33.png
+libs/MVS/texture_output/texture_34.png
+libs/MVS/texture_output/texture_35.png
+texture_output/texture_0.png

fill_all_empty_faces_v1.2.py

@@ -323,7 +323,7 @@ def compute_regions_face_colors(
     return regions_face_color
 
 
-def update_uv_map_and_indices(
+def update_uv_map_and_indices2(
     uv_map: torch.Tensor,
     uvs: torch.Tensor,
     face_uv_indices: torch.Tensor,
@@ -394,6 +394,75 @@ def update_uv_map_and_indices(
 
     return new_uv_map, uvs_updated, face_uv_indices
 
+def update_uv_map_and_indices(
+    uv_map: torch.Tensor,
+    uvs: torch.Tensor,
+    face_uv_indices: torch.Tensor,
+    regions: List[Tuple[set, set]],
+    regions_face_color: Dict[int, torch.Tensor],
+    device: str
+) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+    """
+    根据计算得到的区域颜色，更新UV贴图及对应的UV坐标，并批量更新face_uv_indices。
+    """
+    total_regions = len(regions_face_color)
+    
+    # 1. 在CPU上创建新的UV贴图
+    new_height = int(uv_map.shape[0] + uv_map.shape[0] / 2)
+    new_width = uv_map.shape[1]
+    
+    # 将原始UV贴图移到CPU
+    uv_map_cpu = uv_map.cpu().numpy()
+    
+    # 创建新的UV贴图（在CPU上）
+    new_uv_map_cpu = np.zeros((new_height, new_width, 3), dtype=np.uint8)
+    new_uv_map_cpu[:uv_map.shape[0], :, :] = uv_map_cpu
+    
+    # 2. 在CPU上创建颜色块
+    grid_size = new_width // 3
+    for i, (region_index, color) in enumerate(regions_face_color.items()):
+        all_c = i // grid_size
+        all_r = i % grid_size
+        
+        # 计算3x3颜色块的位置
+        for dr in range(3):
+            for dc in range(3):
+                r = all_r * 3 + dr
+                c = all_c * 3 + dc + uv_map.shape[0]  # 偏移到下半部分
+                
+                if r < new_width and c < new_height:
+                    new_uv_map_cpu[c, r, :] = color.cpu().numpy()
+    
+    # 3. 更新UV坐标
+    uvs[:, 1] = uvs[:, 1] * (2 / 3) + 1 / 3
+    
+    # 计算新的UV坐标
+    new_uvs = []
+    for i, (region_index, _) in enumerate(sorted(regions_face_color.items(), key=lambda x: x[0])):
+        r = (i % grid_size) * 3 + 1
+        c = uv_map.shape[0] + (i // grid_size) * 3 + 1
+        
+        u_new = r / (new_width - 1)
+        v_new = (new_height - 1 - c) / (new_height - 1)
+        new_uvs.append([u_new, v_new])
+    
+    new_uvs_tensor = torch.tensor(new_uvs, device=device)
+    uvs_updated = torch.cat([uvs, new_uvs_tensor], dim=0)
+    uv_coordinates_start = uvs_updated.shape[0] - total_regions
+    
+    # 4. 批量更新face_uv_indices
+    for i, (region_index, _) in enumerate(sorted(regions_face_color.items(), key=lambda x: x[0])):
+        region_faces_indexes = torch.tensor(list(regions[region_index][0]), device=device)
+        face_uv_indices[region_faces_indexes] = torch.full(
+            (1, 3), uv_coordinates_start + i, 
+            device=device, dtype=face_uv_indices.dtype
+        )
+    
+    # 将结果转回GPU
+    new_uv_map = torch.from_numpy(new_uv_map_cpu).to(device)
+    
+    return new_uv_map, uvs_updated, face_uv_indices
+
 def group_regions_by_y_axis(
     regions: List[Tuple[set, set]],
     vertices: torch.Tensor,

libs/MVS/Scene.h

@@ -65,6 +65,7 @@ public:
 	std::map<std::string, std::unordered_set<int>> visible_faces_map;
 	std::map<std::string, std::unordered_set<int>> edge_faces_map;
 	std::map<std::string, std::unordered_set<int>> delete_edge_faces_map;
+	std::map<std::string, std::unordered_set<int>> face_normal_visible_map;
 	std::unordered_set<int> face_visible_relative;
 
 public:
@@ -163,6 +164,7 @@ public:
                          std::unordered_set<int>& face_visible_relative, 
 						 std::map<std::string, std::unordered_set<int>>& edge_faces_map,
 						 std::map<std::string, std::unordered_set<int>>& delete_edge_faces_map,
+						 std::map<std::string, std::unordered_set<int>>& face_normal_visible_map,		
                          std::string& basePath);
 
 	// Mesh texturing
@@ -176,6 +178,7 @@ public:
 	bool is_face_visible_relative(int face_index);
 	bool is_face_edge(const std::string& image_name, int face_index);
 	bool is_face_delete_edge(const std::string& image_name, int face_index);
+	bool is_face_normal_visible_map(const std::string& image_name, int face_index);
 	
 	void SegmentMeshBasedOnCurvature(Mesh::FaceIdxArr& regionMap, float curvatureThreshold);
 

libs/MVS/mask_face_occlusion.py

@@ -4,7 +4,8 @@ import os
 import numpy as np
 from scipy.spatial.transform import Rotation
 import sys
-sys.path.append("/home/algo/Documents/openMVS/openMVS/libs/MVS/utils")
+# sys.path.append("/home/algo/Documents/openMVS/openMVS/libs/MVS/utils")
+sys.path.append("/root/code/openMVS/libs/MVS/utils")
 from colmap_loader import read_cameras_text, read_images_text, read_int_text, write_int_text, read_indices_from_file
 # from get_pose_matrix import get_w2c
 import argparse
@@ -33,6 +34,7 @@ from typing import Dict, List, Set
 import struct
 import math
 # import os
+from pathlib import Path
 
 CameraModel = collections.namedtuple(
     "CameraModel", ["model_id", "model_name", "num_params"]
@@ -1350,14 +1352,88 @@ class ModelProcessor:
         
         face_visible = v0_visible | v1_visible | v2_visible
         
+        # ============ 新增：法线夹角过滤 ============
+        # 1. 获取面片法线（向量化计算，性能更好）
+        if not hasattr(self, 'face_normals_tensor'):
+            # 从mesh获取顶点和面片索引
+            vertices = np.asarray(self.mesh.vertices)  # 形状: (V, 3)
+            triangles = np.asarray(self.mesh.triangles)  # 形状: (F, 3)
+            
+            # 向量化计算法线
+            # 获取所有三角形的顶点
+            v0 = vertices[triangles[:, 0]]  # 形状: (F, 3)
+            v1 = vertices[triangles[:, 1]]  # 形状: (F, 3)
+            v2 = vertices[triangles[:, 2]]  # 形状: (F, 3)
+            
+            # 计算边向量
+            edge1 = v1 - v0
+            edge2 = v2 - v0
+            
+            # 向量化叉积
+            # np.cross支持批量计算
+            normals = np.cross(edge1, edge2)
+            
+            # 计算法线长度
+            norms = np.linalg.norm(normals, axis=1, keepdims=True)
+            
+            # 避免除零
+            norms[norms < 1e-6] = 1.0
+            
+            # 归一化
+            normals = normals / norms
+            
+            # 将退化三角形的法线设为默认值
+            mask = norms[:, 0] < 1e-6
+            if np.any(mask):
+                normals[mask] = np.array([0.0, 0.0, 1.0])
+            
+            # 转换为GPU张量
+            self.face_normals_tensor = torch.tensor(normals, device=self.device, dtype=torch.float32)
+        
+        # 2. 计算摄像机方向（从面片中心指向摄像机）
+        # 获取面片顶点坐标
+        v0_pos = self.vertices_tensor[v0_indices]
+        v1_pos = self.vertices_tensor[v1_indices]
+        v2_pos = self.vertices_tensor[v2_indices]
+        
+        # 计算面片中心
+        face_centers = (v0_pos + v1_pos + v2_pos) / 3.0
+        
+        # 计算摄像机方向向量
+        camera_pos = torch.tensor(eye, device=self.device, dtype=torch.float32)
+        if len(camera_pos.shape) == 1:
+            camera_pos = camera_pos.unsqueeze(0)  # 从(3)变为(1,3)
+        
+        # 方向：从面片中心指向摄像机
+        view_dirs = camera_pos - face_centers
+        view_dirs = view_dirs / torch.norm(view_dirs, dim=1, keepdim=True)
+        
+        # 3. 计算法线与摄像机方向的夹角
+        face_normals = self.face_normals_tensor
+        # 归一化法线
+        face_normals = face_normals / torch.norm(face_normals, dim=1, keepdim=True)
+        
+        # 计算点积（余弦值）
+        cos_angles = torch.sum(face_normals * view_dirs, dim=1)
+        
+        # 计算角度（弧度转角度）
+        angles_deg = torch.acos(torch.clamp(cos_angles, -1.0, 1.0)) * 180.0 / torch.pi
+        
+        # 4. 创建法线可见性掩码（夹角 ≤ 45度）
+        normal_visible_mask = angles_deg <= 40.0
+        
+        # 5. 结合遮挡可见性和法线可见性
+        face_normal_visible = face_visible & normal_visible_mask
+        # ============ 法线夹角过滤结束 ============
+
         # 使用与CPU版本相同的后续处理
-        shrunk_visibility = self._shrink_face_visibility(face_visible.cpu().numpy(), 6)
+        shrunk_visibility = self._shrink_face_visibility(face_visible.cpu().numpy(), 12)
-        expanded_visibility = self._expand_face_visibility(face_visible.cpu().numpy(), 30)
+        expanded_visibility = self._expand_face_visibility(face_visible.cpu().numpy(), 0)
-        shrunk_visibility2 = self._shrink_face_visibility(face_visible.cpu().numpy(), 50)
+        shrunk_visibility2 = self._shrink_face_visibility(face_visible.cpu().numpy(), 0)
         expanded_edge = expanded_visibility & ~shrunk_visibility2
         delete_edge = face_visible.cpu().numpy() & ~shrunk_visibility
         
-        return shrunk_visibility, expanded_edge, delete_edge
+        return shrunk_visibility, expanded_edge, delete_edge, face_normal_visible
     
     """
     def _gen_depth_image_gpu(self, cam_data, render):
@@ -1388,6 +1464,7 @@ class ModelProcessor:
         visible_faces_dict = {}
         edge_faces_dict = {}
         delete_edge_faces_dict = {}
+        face_normal_visible_dict = {}
         
         total_start = time.time()
         
@@ -1413,13 +1490,14 @@ class ModelProcessor:
             #    continue
             
             start_time = time.time()
-            face_visibility, face_edge, face_delete_edge = self._flag_model_gpu(camera_data)
+            face_visibility, face_edge, face_delete_edge, face_normal_visible = self._flag_model_gpu(camera_data)
             processing_time = time.time() - start_time
             
             visible_faces = np.where(face_visibility)[0].tolist()
             visible_faces_dict[img_name] = visible_faces
             edge_faces_dict[img_name] = np.where(face_edge)[0].tolist()
             delete_edge_faces_dict[img_name] = np.where(face_delete_edge)[0].tolist()
+            face_normal_visible_dict[img_name] = np.where(face_normal_visible.cpu().numpy())[0].tolist()
             
             print(f"图像 {img_name} 处理完成，耗时: {processing_time:.2f}秒，可见面数量{len(visible_faces)}")
         
@@ -1427,12 +1505,13 @@ class ModelProcessor:
         print(f"所有图像处理完成，总耗时: {total_time:.2f}秒")
         print(f"平均每张图像耗时: {total_time/len(images):.2f}秒")
         
-        self.save_occlusion_data(visible_faces_dict, edge_faces_dict, delete_edge_faces_dict, self.asset_dir)
+        self.save_occlusion_data(visible_faces_dict, edge_faces_dict, delete_edge_faces_dict, face_normal_visible_dict, self.asset_dir)
         
         return {
             "result1": visible_faces_dict, 
             "result2": edge_faces_dict, 
-            "result3": delete_edge_faces_dict
+            "result3": delete_edge_faces_dict, 
+            "result4": face_normal_visible_dict
         }
             
     #"""
@@ -1926,64 +2005,6 @@ class ModelProcessor:
         depth = render.render_to_depth_image(z_in_view_space=True)
         return np.asarray(depth)
     
-    def sort_vertices(vertices_original):
-        return sorted(
-        (v for v in vertices_original),
-        key=lambda v: (v.co.x, v.co.y, v.co.z)
-        )
-    
-    def _flag_model(self, camera_data, face_points):
-        """标记可见顶点"""
-        vertex_visible = []
-        vertex_occlusion = []
-        depth_images = []
-
-        render = o3d.visualization.rendering.OffscreenRenderer(camera_data['width'], camera_data['height'])
-        
-        material = o3d.visualization.rendering.MaterialRecord()
-        render.scene.add_geometry("mesh", self.mesh, material)
-
-        # 生成深度图
-        depth_image = self._gen_depth_image(camera_data, render)
-
-        # 计算可见性
-        R = self.qvec2rotmat(camera_data['qvec']).T
-        eye = -R @ camera_data['tvec']
-        # eye = camera_data['tvec']
-        # final_visible_list, final_occlusion_list, final_vertex_difference_list = self._compute_vertex_in_frustum(
-        final_visible_list, final_occlusion_list = self._compute_vertex_in_frustum(
-            camera_data['fx'], camera_data['fy'],
-            camera_data['cx'], camera_data['cy'],
-            R, eye,
-            camera_data['height'], camera_data['width'],
-            depth_image,camera_data['qvec'], camera_data['tvec']
-        )
-        print("_flag_model", len(final_occlusion_list), len(self.mesh.vertices), len(self.mesh.vertex_colors))
-
-        # 获取三角形面片数组
-        triangles = np.asarray(self.mesh.triangles)
-        
-        face_visible_bitmap = np.zeros(len(triangles), dtype=bool)
-        
-        # 遍历所有面片
-        for face_idx, face in enumerate(triangles):
-            v0, v1, v2 = face
-            
-            face_visible_bitmap[face_idx] = any([ # any all
-                final_visible_list[v0],
-                final_visible_list[v1],
-                final_visible_list[v2]
-            ]) 
-
-        shrunk_visibility = self._shrink_face_visibility(face_visible_bitmap, 6) # 6 10
-
-        expanded_visibility = self._expand_face_visibility(face_visible_bitmap, 30)
-        shrunk_visibility2 = self._shrink_face_visibility(face_visible_bitmap, 50)
-        expanded_edge = expanded_visibility & ~shrunk_visibility2
-        delete_edge = face_visible_bitmap & ~shrunk_visibility
-        
-        return shrunk_visibility, expanded_edge, delete_edge
-    
     def _flag_contour(self, camera_data, face_points):
         """标记可见顶点"""
         vertex_visible = []
@@ -2073,97 +2094,10 @@ class ModelProcessor:
         face_visible_bitmap = np.ones(len(triangles), dtype=bool)  # 临时填充
         return face_visible_bitmap, face_edge
 
-    """
-    def _mask_face_occlusion(self):
-        # 读取相机数据
-        cameras = read_cameras_text(os.path.join(self.pose_path, "cameras.txt"))
-        images = read_images_text(os.path.join(self.pose_path, "images.txt"))
-        # cameras = read_cameras_text(os.path.join(self.pose_path, "backup_cameras.txt"))
-        # images = read_images_text(os.path.join(self.pose_path, "backup_images.txt"))
-
-        face_points_sorted_path = os.path.join(self.pose_path, "face_points_sorted.txt")
-        print("face_points_sorted_path=", face_points_sorted_path)
-        #face_points = read_int_text(face_points_sorted_path)
-        face_points = read_indices_from_file(face_points_sorted_path)
-        # face_points = {}
-        camera_data = {}
-
-        for img in images.values():
-            if self.mask_image == img.name[:-4]:
-                camera = cameras[img.camera_id]
-                camera_data = {
-                    "qvec": img.qvec,
-                    "tvec": img.tvec,
-                    "fx": camera.params[0],
-                    "fy": camera.params[1],
-                    "cx": camera.params[2],
-                    "cy": camera.params[3],
-                    "width": camera.width,
-                    "height": camera.height,
-                    "name": img.name[:-4]
-                }
-
-        # print(face_points)
-        self._flag_model(camera_data, face_points)
-    """
-
-    def _mask_occlusion(self):
-        # 读取相机数据
-        cameras = read_cameras_text(os.path.join(self.pose_path, "cameras.txt"))
-        images = read_images_text(os.path.join(self.pose_path, "images.txt"))
-
-        camera_data = {}
-
-        countour_faces_dict = {}
-        visible_faces_dict = {}
-        edge_faces_dict = {}
-        delete_edge_faces_dict = {}
-
-        total_start = time.time()
-
-        n = 0
-        for img in images.values():
-            camera = cameras[img.camera_id]
-            camera_data = {
-                "qvec": img.qvec,
-                "tvec": img.tvec,
-                "fx": camera.params[0],
-                "fy": camera.params[1],
-                "cx": camera.params[2],
-                "cy": camera.params[3],
-                "width": camera.width,
-                "height": camera.height,
-                "name": img.name[:-4]
-            }
-            img_name = img.name[:-4]       
-            # if (img_name!="73_8" and img_name!="52_8" and img_name!="62_8"):
-            # if (img_name!="52_8" and img_name!="62_8"):
-            # if (img_name!="52_8"):
-            #    continue
-
-            start_time = time.time()
-            face_visibility, face_edge, face_delete_edge = self._flag_model(camera_data, None)
-            processing_time = time.time() - start_time
-            
-            visible_faces = np.where(face_visibility)[0].tolist()
-            visible_faces_dict[img.name[:-4]] = visible_faces
-            edge_faces_dict[img.name[:-4]] = np.where(face_edge)[0].tolist()
-            delete_edge_faces_dict[img.name[:-4]] = np.where(face_delete_edge)[0].tolist()
-            n += 1
-            
-            print(f"图像={img_name}，耗时={processing_time:.2f}秒，可见面数={len(visible_faces)}")    
-
-        total_time = time.time() - total_start
-        print(f"所有图像处理完成，总耗时: {total_time:.2f}秒")
-        print(f"平均每张图像耗时: {total_time/len(images):.2f}秒")
-
-        self.save_occlusion_data(visible_faces_dict, edge_faces_dict, delete_edge_faces_dict, self.asset_dir)
-
-        return {"result1": visible_faces_dict, "result2": edge_faces_dict, "result3": delete_edge_faces_dict}
-
     def save_occlusion_data(self, result1: Dict[str, List[int]], 
                         result2: Dict[str, List[int]], 
                         result3: Dict[str, List[int]],
+                        result4: Dict[str, List[int]], 
                         base_path: str) -> None:
         """
         保存遮挡数据到文件
@@ -2175,6 +2109,8 @@ class ModelProcessor:
             base_path: 基础文件路径
         """
         
+        os.makedirs(base_path, exist_ok = True)
+
         print(f"save_occlusion_data {base_path}, {len(result1)}, {len(result2)}, {len(result3)}")
         
         # 处理返回的可见面字典 - 转换为图像名到面编号集合的映射
@@ -2197,9 +2133,15 @@ class ModelProcessor:
         for image_name, face_list in result3.items():
             delete_edge_faces_map[image_name] = set(face_list)
 
+        face_normal_visible_map: Dict[str, Set[int]] = {}
+        for image_name, face_list in result4.items():
+            face_normal_visible_map[image_name] = set(face_list)
+
         # 保存 visible_faces_map
         try:
-            with open(base_path + "/_visible_faces_map.txt", "w", encoding='utf-8') as map_file:
+            file_name = "_visible_faces_map.txt"
+            file_path = Path(base_path) / file_name
+            with open(file_path, "w", encoding='utf-8') as map_file:
                 for image_name, face_set in visible_faces_map.items():
                     # 写入图像名称和所有面ID，用空格分隔
                     line = image_name + " " + " ".join(str(face) for face in face_set) + "\n"
@@ -2209,7 +2151,9 @@ class ModelProcessor:
         
         # 保存 face_visible_relative
         try:
-            with open(base_path + "/_face_visible_relative.txt", "w", encoding='utf-8') as relative_file:
+            file_name = "_face_visible_relative.txt"
+            file_path = Path(base_path) / file_name
+            with open(file_path, "w", encoding='utf-8') as relative_file:
                 for face in face_visible_relative:
                     relative_file.write(str(face) + "\n")
         except IOError as e:
@@ -2217,7 +2161,9 @@ class ModelProcessor:
         
         # 保存 edge_faces_map
         try:
-            with open(base_path + "/_edge_faces_map.txt", "w", encoding='utf-8') as map_file2:
+            file_name = "_edge_faces_map.txt"
+            file_path = Path(base_path) / file_name
+            with open(file_path, "w", encoding='utf-8') as map_file2:
                 for image_name, face_set in edge_faces_map.items():
                     line = image_name + " " + " ".join(str(face) for face in face_set) + "\n"
                     map_file2.write(line)
@@ -2226,13 +2172,26 @@ class ModelProcessor:
         
         # 保存 delete_edge_faces_map
         try:
-            with open(base_path + "/_delete_edge_faces_map.txt", "w", encoding='utf-8') as map_file3:
+            file_name = "_delete_edge_faces_map.txt"
+            file_path = Path(base_path) / file_name
+            with open(file_path, "w", encoding='utf-8') as map_file3:
                 for image_name, face_set in delete_edge_faces_map.items():
                     line = image_name + " " + " ".join(str(face) for face in face_set) + "\n"
                     map_file3.write(line)
         except IOError as e:
             print(f"Error writing delete_edge_faces_map file: {e}")
 
+        # 保存 face_normal_visible_map
+        try:
+            file_name = "_face_normal_visible_map.txt"
+            file_path = Path(base_path) / file_name
+            with open(file_path, "w", encoding='utf-8') as map_file4:
+                for image_name, face_set in face_normal_visible_map.items():
+                    line = image_name + " " + " ".join(str(face) for face in face_set) + "\n"
+                    map_file4.write(line)
+        except IOError as e:
+            print(f"Error writing delete_edge_faces_map file: {e}")
+
     def process(self):
 
         print("process")