diff --git a/libs/MVS/SceneTexture.cpp b/libs/MVS/SceneTexture.cpp index 9aa697e..d16be29 100644 --- a/libs/MVS/SceneTexture.cpp +++ b/libs/MVS/SceneTexture.cpp @@ -805,6 +805,11 @@ public: Image8U3& atlas, int textureSize); void RunSeamLevelingOnAtlas(Image8U3& atlas, int textureSize); + void PropagateMainViews( + const VirtualFaceMap& virtualFaceMap, + const std::vector>& virtualFaceViews, + const std::vector>& virtualFaceViewWeights, + std::vector& faceViewsHighFreq); bool RasterizeVirtualFaces( const VirtualFaceMap& virtualFaceMap, const std::vector>& virtualFaceViews, @@ -812,7 +817,18 @@ public: unsigned nTextureSizeMultiple, Pixel8U colEmpty, Mesh::Image8U3Arr& outTextures); - + + inline bool ComputeBarycentric(float u, float v, + const TexCoord& uv0, const TexCoord& uv1, const TexCoord& uv2, + float& b0, float& b1, float& b2) { + float denom = (uv1.x - uv0.x) * (uv2.y - uv0.y) - (uv1.y - uv0.y) * (uv2.x - uv0.x); + if (std::abs(denom) < 1e-12f) return false; + b1 = ((u - uv0.x) * (uv2.y - uv0.y) - (v - uv0.y) * (uv2.x - uv0.x)) / denom; + b2 = ((u - uv0.x) * (uv1.y - uv0.y) - (v - uv0.y) * (uv1.x - uv0.x)) / denom; + b2 = ((v - uv0.y) * (uv1.x - uv0.x) - (u - uv0.x) * (uv1.y - uv0.y)) / denom; + b0 = 1.0f - b1 - b2; + return (b0 >= -1e-4f && b1 >= -1e-4f && b2 >= -1e-4f); + } cv::Mat GetHomographyForView(int virtualFaceIdx, IIndex viewId) const; void BuildStandardSeamDataFromRCPatches( @@ -16281,6 +16297,96 @@ cv::Mat MeshTexture::GetHomographyForView(int virtualFaceIdx, IIndex viewId) con return cv::Mat(); } +// 在光栅化之前，添加这个函数来传播主视图 +void MeshTexture::PropagateMainViews( + const VirtualFaceMap& virtualFaceMap, + const std::vector>& virtualFaceViews, + const std::vector>& virtualFaceViewWeights, + std::vector& faceViewsHighFreq) +{ + // 1. 构建虚拟面的邻接关系（通过共享原始三角面） + std::vector> faceAdjacency(virtualFaceMap.size()); + + // 建立：原始三角面 -> 包含它的虚拟面列表 + std::map> faceToVirtualFaces; + for (size_t i = 0; i < virtualFaceMap.size(); ++i) { + for (FIndex fi : virtualFaceMap[i].faces) { + faceToVirtualFaces[fi].push_back(i); + } + } + + // 构建邻接：共享同一个原始三角面的两个虚拟面视为相邻 + for (const auto& entry : faceToVirtualFaces) { + const auto& virtFaces = entry.second; + for (size_t a = 0; a < virtFaces.size(); ++a) { + for (size_t b = a + 1; b < virtFaces.size(); ++b) { + faceAdjacency[virtFaces[a]].insert((FIndex)virtFaces[b]); + faceAdjacency[virtFaces[b]].insert((FIndex)virtFaces[a]); + } + } + } + + // 2. 贪心传播：从覆盖面积最大的虚拟面开始，让邻居跟随它的主视图 + // 先按覆盖面积排序 + std::vector sortedIndices(virtualFaceMap.size()); + std::iota(sortedIndices.begin(), sortedIndices.end(), 0); + + // 计算每个虚拟面的覆盖面积（UV空间面积） + std::vector faceAreas(virtualFaceMap.size(), 0); + for (size_t i = 0; i < virtualFaceMap.size(); ++i) { + const VirtualFaceGeometry& geom = m_virtualFaceGeometries[i]; + if (geom.isValid) { + faceAreas[i] = (geom.uvBounds.ptMax.x() - geom.uvBounds.ptMin.x()) * + (geom.uvBounds.ptMax.y() - geom.uvBounds.ptMin.y()); + } + } + + std::sort(sortedIndices.begin(), sortedIndices.end(), + [&](size_t a, size_t b) { return faceAreas[a] > faceAreas[b]; }); + + // 3. 传播 + std::vector visited(virtualFaceMap.size(), false); + std::queue queue; + + for (size_t startIdx : sortedIndices) { + if (visited[startIdx]) continue; + if (faceViewsHighFreq[startIdx] == NO_ID) continue; + + // 从大面积的面开始BFS传播 + visited[startIdx] = true; + queue.push(startIdx); + + while (!queue.empty()) { + size_t cur = queue.front(); + queue.pop(); + + IIndex curView = faceViewsHighFreq[cur]; + + for (FIndex neighborIdx : faceAdjacency[cur]) { + if (visited[neighborIdx]) continue; + + // 检查邻居是否也能看到 curView + bool neighborCanUseCurView = false; + for (size_t vi = 0; vi < virtualFaceViews[neighborIdx].size(); ++vi) { + if (virtualFaceViews[neighborIdx][vi] == curView) { + neighborCanUseCurView = true; + break; + } + } + + if (neighborCanUseCurView) { + // 强制邻居使用相同的主视图 + faceViewsHighFreq[neighborIdx] = curView; + visited[neighborIdx] = true; + queue.push(neighborIdx); + } + } + } + } + + DEBUG_EXTRA("Main view propagation done: %zu faces processed", virtualFaceMap.size()); +} + bool MeshTexture::RasterizeVirtualFaces( const VirtualFaceMap& virtualFaceMap, const std::vector>& virtualFaceViews, @@ -16321,6 +16427,9 @@ bool MeshTexture::RasterizeVirtualFaces( } currentTextureSize = textureSize; + VERBOSE("[Raster] Propagating main views for consistency..."); + PropagateMainViews(virtualFaceMap, virtualFaceViews, virtualFaceViewWeights, faceViewsHighFreq); + if (!m_gainsEstimated) EstimateGlobalPhotometricCorrection(); VERBOSE("[Raster] Step 2: starting patch loop, total faces=%zu", virtualFaceMap.size()); @@ -16333,6 +16442,22 @@ bool MeshTexture::RasterizeVirtualFaces( } } + // ★★★ 在循环外面预先计算全局 homography ★★★ + cv::Mat globalH; + IIndex globalView = NO_ID; + for (int ii = 0; ii < (int)virtualFaceMap.size(); ++ii) { + const VirtualFaceGeometry& g = m_virtualFaceGeometries[ii]; + if (g.isValid && !virtualFaceViews[ii].empty()) { + globalView = virtualFaceViews[ii][0]; + auto it = g.viewHomographies.find(globalView); + if (it != g.viewHomographies.end()) { + globalH = it->second.clone(); + break; + } + } + } + DEBUG_EXTRA("[TEST] Using global view %d for all faces", globalView); + // 3. 逐三角形光栅化（分频带合成版） #ifdef _USE_OPENMP #pragma omp parallel for schedule(dynamic) @@ -16378,16 +16503,66 @@ bool MeshTexture::RasterizeVirtualFaces( continue; } - // ========== 构建主视图的映射 ========== +// ========== 用重心坐标 + 3D 投影替代 homography ========== cv::Mat mapX(patchH, patchW, CV_32FC1), mapY(patchH, patchW, CV_32FC1); - const float* H = geom.homography.ptr(); - for (int y = minY; y <= maxY; ++y) { - for (int x = minX; x <= maxX; ++x) { - float u = (float)x / (float)textureSize, v = (float)y / (float)textureSize; - float w = H[6]*u + H[7]*v + H[8]; - if (std::abs(w) < 1e-12f) { mapX.at(y-minY,x-minX)=-1; mapY.at(y-minY,x-minX)=-1; continue; } - mapX.at(y-minY,x-minX) = (H[0]*u+H[1]*v+H[2])/w; - mapY.at(y-minY,x-minX) = (H[3]*u+H[4]*v+H[5])/w; + mapX.setTo(-1); + mapY.setTo(-1); + + // 获取该虚拟面对应的原始三角面 + if (vf.faces.empty()) continue; + FIndex faceIdx = vf.faces[0]; + const Face& face = scene.mesh.faces[faceIdx]; + + // 获取相机参数 + const Camera& cam = images[mainView].camera; + bool camValid = (cam.K(0,0) > 0 && cam.K(1,1) > 0); + if (!camValid) { + // fallback 到 homography + const float* H = geom.homography.ptr(); + for (int y = minY; y <= maxY; ++y) { + for (int x = minX; x <= maxX; ++x) { + float u = (float)x / (float)textureSize, v = (float)y / (float)textureSize; + float w = H[6]*u + H[7]*v + H[8]; + if (std::abs(w) < 1e-12f) { mapX.at(y-minY,x-minX)=-1; mapY.at(y-minY,x-minX)=-1; continue; } + mapX.at(y-minY,x-minX) = (H[0]*u+H[1]*v+H[2])/w; + mapY.at(y-minY,x-minX) = (H[3]*u+H[4]*v+H[5])/w; + } + } + } else { + // 提取 3D 顶点和 UV + Point3f v[3]; + TexCoord uv[3]; + for (int j = 0; j < 3; ++j) { + v[j] = scene.mesh.vertices[face[j]]; + uv[j] = scene.mesh.faceTexcoords[faceIdx * 3 + j]; // 注意这里的索引方式 + } + + // 对 patch 内每个像素做重心坐标投影 + for (int py = minY; py <= maxY; ++py) { + for (int px = minX; px <= maxX; ++px) { + float u = (float)px / (float)textureSize; + float v_coord = (float)py / (float)textureSize; + + // 计算重心坐标 + float b0, b1, b2; + if (!ComputeBarycentric(u, v_coord, uv[0], uv[1], uv[2], b0, b1, b2)) { + continue; + } + + // 3D 插值点 + Point3f P = v[0] * b0 + v[1] * b1 + v[2] * b2; + + // 投影到图像 + Point2f imgPt = cam.ProjectPointP(P); + // 检查是否在图像有效范围内 + if (imgPt.x < 0 || imgPt.x >= images[mainView].image.cols || + imgPt.y < 0 || imgPt.y >= images[mainView].image.rows) { + continue; + } + + mapX.at(py - minY, px - minX) = imgPt.x; + mapY.at(py - minY, px - minX) = imgPt.y; + } } } @@ -16395,38 +16570,16 @@ bool MeshTexture::RasterizeVirtualFaces( cv::Mat mainPatch; cv::remap(images[mainView].image, mainPatch, mapX, mapY, cv::INTER_LINEAR, cv::BORDER_CONSTANT, cv::Scalar(0,0,0)); - // ========== 从辅助视图采样（各自用自己的 homography）========== + // ========== 从辅助视图采样（全部用主视图的映射）========== cv::Mat lowFreqPatch(patchH, patchW, CV_32FC3, cv::Scalar(0,0,0)); if (hasLowFreqViews) { float totalWeight = 0; for (const auto& lfv : lowFreqViews) { if (lfv.viewId >= (IIndex)images.size() || images[lfv.viewId].image.empty()) continue; - // ===== 获取该视图自己的 homography ===== - const auto& geom = m_virtualFaceGeometries[i]; - auto it = geom.viewHomographies.find(lfv.viewId); - if (it == geom.viewHomographies.end()) continue; - const cv::Mat& H_aux = it->second; - - // ===== 用该视图的 homography 构建映射 ===== - cv::Mat auxMapX(patchH, patchW, CV_32FC1), auxMapY(patchH, patchW, CV_32FC1); - const float* H = H_aux.ptr(); - for (int y = minY; y <= maxY; ++y) { - for (int x = minX; x <= maxX; ++x) { - float u = (float)x / (float)textureSize, v = (float)y / (float)textureSize; - float w = H[6]*u + H[7]*v + H[8]; - if (std::abs(w) < 1e-12f) { - auxMapX.at(y-minY,x-minX) = -1; - auxMapY.at(y-minY,x-minX) = -1; - continue; - } - auxMapX.at(y-minY,x-minX) = (H[0]*u+H[1]*v+H[2])/w; - auxMapY.at(y-minY,x-minX) = (H[3]*u+H[4]*v+H[5])/w; - } - } - + // ★★★ 所有视图都用主视图的 mapX/mapY，不再各自构建 homography ★★★ cv::Mat auxPatch; - cv::remap(images[lfv.viewId].image, auxPatch, auxMapX, auxMapY, + cv::remap(images[lfv.viewId].image, auxPatch, mapX, mapY, cv::INTER_LINEAR, cv::BORDER_CONSTANT, cv::Scalar(0,0,0)); cv::Mat auxFloat; @@ -16447,24 +16600,59 @@ bool MeshTexture::RasterizeVirtualFaces( mainPatch.convertTo(mainFloat, CV_32FC3, 1.0f / 255.0f); lowFreqPatch = mainFloat.clone(); } + + // ========== 多视图高频混合 ========== + cv::Mat mainFloat; + mainPatch.convertTo(mainFloat, CV_32FC3, 1.0f / 255.0f); - // ========== 从主视图提取高频 ========== - // 对主视图做高斯模糊得到低频，然后相减得高频 - cv::Mat mainFloat; - mainPatch.convertTo(mainFloat, CV_32FC3, 1.0f / 255.0f); - - cv::Mat mainLowFreq; - cv::GaussianBlur(mainFloat, mainLowFreq, cv::Size(31, 31), 10.0); // σ=10 - - cv::Mat highFreq = mainFloat - mainLowFreq; // 高频 = 原图 - 低频 + // 主视图高频 + cv::Mat mainLowFreq; + cv::GaussianBlur(mainFloat, mainLowFreq, cv::Size(31, 31), 10.0); + cv::Mat highFreqAccum = (mainFloat - mainLowFreq) * 0.6f; + float highFreqTotalWeight = 0.6f; + + // 辅助视图也贡献高频（也用主视图的 mapX/mapY） + if (hasLowFreqViews) { + for (const auto& lfv : lowFreqViews) { + if (lfv.viewId == mainView) continue; + if (lfv.viewId >= (IIndex)images.size() || images[lfv.viewId].image.empty()) continue; + + // ★★★ 用主视图的 mapX/mapY（3D投影的那个），不是各视图自己的homography ★★★ + cv::Mat auxPatch; + cv::remap(images[lfv.viewId].image, auxPatch, mapX, mapY, + cv::INTER_LINEAR, cv::BORDER_CONSTANT, cv::Scalar(0,0,0)); + + cv::Mat auxFloat; + auxPatch.convertTo(auxFloat, CV_32FC3, 1.0f / 255.0f); + + cv::Mat auxLowFreq; + cv::GaussianBlur(auxFloat, auxLowFreq, cv::Size(31, 31), 10.0); + + float auxWeight = lfv.weight * 0.4f; + highFreqAccum += (auxFloat - auxLowFreq) * auxWeight; + highFreqTotalWeight += auxWeight; + } + } + + if (highFreqTotalWeight > 0) { + highFreqAccum /= highFreqTotalWeight; + } + + cv::Mat highFreq = highFreqAccum; // 在合成之前，对 lowFreqPatch 再做一次轻微平滑 - cv::GaussianBlur(lowFreqPatch, lowFreqPatch, cv::Size(5, 5), 1.0); + // cv::GaussianBlur(lowFreqPatch, lowFreqPatch, cv::Size(5, 5), 1.0); // ========== 合成：低频用多视图融合，高频用主视图细节 ========== - cv::Mat finalPatch = lowFreqPatch + highFreq; + // cv::Mat finalPatch = lowFreqPatch + highFreq; + // cv::Mat finalPatch = lowFreqPatch; // cv::Mat finalPatch = highFreq.clone(); // finalPatch = cv::Mat::zeros(finalPatch.size(), finalPatch.type()); + + cv::Mat finalPatch = mainFloat; // 只用主视图 + // cv::Mat finalPatch = lowFreqPatch * 0.3f + highFreq * 0.7f; + // 低频只做主视图，不做多视图平均（避免发灰） + // cv::Mat finalPatch = mainFloat * 0.7f + highFreq * 0.3f; // 钳制到 [0, 1] cv::Mat clampedPatch;