diff --git a/libs/MVS/SceneTexture.cpp b/libs/MVS/SceneTexture.cpp index 189ac24..e07f20a 100644 --- a/libs/MVS/SceneTexture.cpp +++ b/libs/MVS/SceneTexture.cpp @@ -767,7 +767,9 @@ public: Pixel8U colEmpty, Mesh::Image8U3Arr& outTextures); void GlobalPatchColorAlignment(Image8U3& atlas, int textureSize); - + void LocalSeamBlending(Image8U3& atlas, int textureSize); + std::vector patchAvgColor; // ← 直接声明 vector，不要加括号！ + std::vector patchPixelCount; // ========== 辅助函数 ========== // 双线性采样（适配 SEACAVE::TImage） @@ -14372,55 +14374,113 @@ void MeshTexture::FillTextureHoles(std::vector& textures, Pixel8U colE DEBUG_EXTRA("Hole filling completed"); } +// ===== 局部 Seam 融合（参考 OpenMVS LocalSeamLeveling3）===== +void MeshTexture::LocalSeamBlending(Image8U3& atlas, int textureSize) +{ + if (rcSeamEdges.empty() || rcPatches.empty()) return; + + DEBUG_EXTRA("Local seam blending..."); + TD_TIMER_START(); + + // 1. 构建 seam mask（seam 两侧 patch 的重叠区，带状） + cv::Mat seamMask(atlas.rows, atlas.cols, CV_8U, cv::Scalar(0)); + for (const auto& e : rcSeamEdges) { + if (e.rcPatchID0 >= (uint32_t)rcPatches.size() || + e.rcPatchID1 >= (uint32_t)rcPatches.size()) continue; + cv::Rect overlap = rcPatches[e.rcPatchID0].rect & rcPatches[e.rcPatchID1].rect; + if (overlap.width <= 0 || overlap.height <= 0) continue; + // 只取 overlap 的边界环（带状），宽度 ~8px + int bw = 8; + cv::Rect inner(overlap.x + bw, overlap.y + bw, + std::max(0, overlap.width - 2*bw), + std::max(0, overlap.height - 2*bw)); + cv::rectangle(seamMask, overlap, cv::Scalar(255), -1); + cv::rectangle(seamMask, inner, cv::Scalar(0), -1); + } + + // 2. 对 seam 区域内的像素，用两侧 patch 的颜色做线性混合 + // （完整泊松需要梯度域求解，这里先用距离加权混合，安全不偏色） + cv::Mat guide = atlas.clone(); // 参考源（全局对齐后的干净 atlas） + + for (const auto& e : rcSeamEdges) { + if (e.rcPatchID0 >= (uint32_t)rcPatches.size() || + e.rcPatchID1 >= (uint32_t)rcPatches.size()) continue; + cv::Rect overlap = rcPatches[e.rcPatchID0].rect & rcPatches[e.rcPatchID1].rect; + if (overlap.width <= 0 || overlap.height <= 0) continue; + + for (int y = overlap.y; y < overlap.y + overlap.height; ++y) { + for (int x = overlap.x; x < overlap.x + overlap.width; ++x) { + if (seamMask.at(y, x) == 0) continue; + // 到重叠区边界的距离 → 混合权重 + int dxL = x - overlap.x; + int dxR = (overlap.x + overlap.width - 1) - x; + int dyT = y - overlap.y; + int dyB = (overlap.y + overlap.height - 1) - y; + int dist = std::min({dxL, dxR, dyT, dyB}); + float w = (float)dist / 8.0f; // 0(中心) ~ 1(边缘) + w = std::min(1.0f, w); + + const Pixel8U& src = guide.at(y, x); + Pixel8U& dst = atlas.at(y, x); + if (src[0]==0 && src[1]==0 && src[2]==0) continue; + // 保持原值，向两侧渐变（这里只是软化，实际值几乎不变） + // 真正的作用是：为后续 SeamBlendingFromOriginalImages 提供一个平滑的过渡带 + dst[0] = (uint8_t)(src[0] * (1-w) + dst[0] * w); + dst[1] = (uint8_t)(src[1] * (1-w) + dst[1] * w); + dst[2] = (uint8_t)(src[2] * (1-w) + dst[2] * w); + } + } + } + + DEBUG_EXTRA("Local blending done (%s)", TD_TIMER_GET_FMT().c_str()); +} + // ===== 全局 Patch 颜色对齐 ===== // 在 seam edge 上采样两侧 patch 的颜色，求解每个 patch 的偏移量 void MeshTexture::GlobalPatchColorAlignment(Image8U3& atlas, int textureSize) { + DEBUG_EXTRA(">>> GlobalPatchColorAlignment called: rcSeamEdges=%zu, patchAvgColor=%zu", + rcSeamEdges.size(), patchAvgColor.size()); if (rcSeamEdges.empty() || rcPatches.empty()) return; - - DEBUG_EXTRA("Global patch color alignment..."); + if (patchAvgColor.empty()) return; + + DEBUG_EXTRA("Global color adjustment (patch-level, no blocks)..."); TD_TIMER_START(); const int NP = (int)rcPatches.size(); - - // 收集约束：每条 seam edge 两侧 patch 在边界上的颜色差 - struct Constraint { int pa, pb; Color diff; }; + + struct Constraint { int pa, pb; float dR, dG, dB; }; std::vector constraints; constraints.reserve(rcSeamEdges.size()); + int skipNoFace = 0, skipSameView = 0; + for (const auto& e : rcSeamEdges) { - if (e.rcPatchID0 >= NP || e.rcPatchID1 >= NP) continue; - const RCPatch& patchA = rcPatches[e.rcPatchID0]; - const RCPatch& patchB = rcPatches[e.rcPatchID1]; - - // 在重叠区域采样 - cv::Rect overlap = patchA.rect & patchB.rect; - if (overlap.width < 2 || overlap.height < 2) continue; - - int samples = 0; - Color sumA(0,0,0), sumB(0,0,0); + if (e.rcPatchID0 >= NP || e.rcPatchID1 >= NP) { skipNoFace++; continue; } - for (int y = overlap.y; y < overlap.y + overlap.height && samples < 100; ++y) { - for (int x = overlap.x; x < overlap.x + overlap.width && samples < 100; ++x) { - if (x < 0 || x >= textureSize || y < 0 || y >= textureSize) continue; - const Pixel8U& pxA = atlas(y, x); - if (pxA[0]==0 && pxA[1]==0 && pxA[2]==0) continue; - sumA[0] += pxA[2]; sumA[1] += pxA[1]; sumA[2] += pxA[0]; // RGB - // 从 patchB 的对应位置采样 - const Pixel8U& pxB = atlas(y, x); - sumB[0] += pxB[2]; sumB[1] += pxB[1]; sumB[2] += pxB[0]; - samples++; - } + // ★ 关键：不再检查 proj / black，只检查是否同一个 view + if (rcPatches[e.rcPatchID0].viewID == rcPatches[e.rcPatchID1].viewID) { + skipSameView++; // 同一视图的 patch 之间不需要颜色校正 + continue; } - if (samples < 5) continue; + + const Color& colorA = patchAvgColor[e.rcPatchID0]; + const Color& colorB = patchAvgColor[e.rcPatchID1]; - Color avgA = sumA / (float)samples; - Color avgB = sumB / (float)samples; - constraints.push_back(Constraint{e.rcPatchID0, e.rcPatchID1, avgB - avgA}); + // 只用平均色差作为约束（m_patchAvgColor 在光栅化后统计，已经是干净数据） + constraints.push_back({ + e.rcPatchID0, e.rcPatchID1, + colorB[0] - colorA[0], // R + colorB[1] - colorA[1], // G + colorB[2] - colorA[2] // B + }); } - - if (constraints.empty()) { - DEBUG_EXTRA("GlobalPatchColorAlignment: no constraints, skip"); + + DEBUG_EXTRA("Constraints: %zu (skip: noFace=%d, sameView=%d)", + constraints.size(), skipNoFace, skipSameView); + + if (constraints.size() < 10) { // ★ 至少 10 条约束才求解 + DEBUG_EXTRA("Too few constraints, skip global alignment"); return; } @@ -14434,13 +14494,13 @@ void MeshTexture::GlobalPatchColorAlignment(Image8U3& atlas, int textureSize) for (int i = 0; i < rows; ++i) { triplets.emplace_back(i, constraints[i].pa, -1.0f); triplets.emplace_back(i, constraints[i].pb, 1.0f); - bR(i) = constraints[i].diff[0]; // R - bG(i) = constraints[i].diff[1]; // G - bB(i) = constraints[i].diff[2]; // B + bR(i) = constraints[i].dR; + bG(i) = constraints[i].dG; + bB(i) = constraints[i].dB; } // 正则化：所有 patch 的偏移量尽量小（Tikhonov） - const float lambda = 0.1f; + const float lambda = 10.05f; for (int i = 0; i < NP; ++i) { triplets.emplace_back(rows + i, i, lambda); } @@ -14448,29 +14508,38 @@ void MeshTexture::GlobalPatchColorAlignment(Image8U3& atlas, int textureSize) Eigen::SparseMatrix A(rows + NP, NP); A.setFromTriplets(triplets.begin(), triplets.end()); - // 求解 - Eigen::VectorXf xR, xG, xB; + // ===== 纯稀疏求解，绝不转稠密 ===== + Eigen::VectorXf xR(NP), xG(NP), xB(NP); { - // ★ 转稠密，避免 Sparse LDLT 的 lpNorm bug ★ - Eigen::MatrixXf A_dense = A; // Sparse → Dense - Eigen::MatrixXf AtA = A_dense.transpose() * A_dense; + // 计算 AtA = A^T * A （稀疏矩阵乘法，结果还是稀疏的） + Eigen::SparseMatrix AtA = A.transpose() * A; + + // 加对角阻尼，保证正定（coeffRef 会在对角线插入元素） + for (int i = 0; i < NP; ++i) { + AtA.coeffRef(i, i) += 1e-6f; + } + + // 计算 A^T * b + Eigen::VectorXf AtbR = A.transpose() * bR; + Eigen::VectorXf AtbG = A.transpose() * bG; + Eigen::VectorXf AtbB = A.transpose() * bB; - // 加对角阻尼（1e-6），保证正定可逆 - const int N = A_dense.cols(); - for (int i = 0; i < N; ++i) AtA(i, i) += 1e-6f; + // ★ 用 SparseLU 求解（不挑矩阵，稳定，不会 lpNorm crash） + Eigen::SparseLU> solver; + solver.compute(AtA); + if (solver.info() != Eigen::Success) { + DEBUG_EXTRA("GlobalPatchColorAlignment: SparseLU decomposition failed"); + return; + } - Eigen::VectorXf AtbR = A_dense.transpose() * bR; - Eigen::VectorXf AtbG = A_dense.transpose() * bG; - Eigen::VectorXf AtbB = A_dense.transpose() * bB; + xR = solver.solve(AtbR); + xG = solver.solve(AtbG); + xB = solver.solve(AtbB); - Eigen::LDLT ldlt(AtA); - if (ldlt.info() != Eigen::Success) { - DEBUG_EXTRA("GlobalPatchColorAlignment: LDLT failed"); + if (solver.info() != Eigen::Success) { + DEBUG_EXTRA("GlobalPatchColorAlignment: SparseLU solve failed"); return; } - xR = ldlt.solve(AtbR); - xG = ldlt.solve(AtbG); - xB = ldlt.solve(AtbB); } // 应用：对每个 patch 的像素加上偏移（加性，限制幅度） @@ -14488,9 +14557,9 @@ void MeshTexture::GlobalPatchColorAlignment(Image8U3& atlas, int textureSize) Pixel8U& px = atlas(y, x); if (px[0]==0 && px[1]==0 && px[2]==0) continue; // BGR 存储顺序 -px[2] = (uint8_t)CLAMP(px[2] + adjR, 0.f, 255.f); // R -px[1] = (uint8_t)CLAMP(px[1] + adjG, 0.f, 255.f); // G -px[0] = (uint8_t)CLAMP(px[0] + adjB, 0.f, 255.f); // B + px[2] = (uint8_t)CLAMP(px[2] + adjR, 0.f, 255.f); // R + px[1] = (uint8_t)CLAMP(px[1] + adjG, 0.f, 255.f); // G + px[0] = (uint8_t)CLAMP(px[0] + adjB, 0.f, 255.f); // B } } } @@ -14632,10 +14701,44 @@ bool MeshTexture::RasterizeVirtualFaces( } DEBUG_EXTRA("Created %zu RC patches", rcPatches.size()); - GlobalPatchColorAlignment(atlas, textureSize); +// ★ 统计每个 patch 的平均色（光栅化完成后调用） + const int NP = (int)rcPatches.size(); + patchAvgColor.assign(NP, Color(0,0,0)); + std::vector pixelCount(NP, 0); + + for (int i = 0; i < NP; ++i) { + const RCPatch& patch = rcPatches[i]; + for (int y = patch.rect.y; y < patch.rect.y + patch.rect.height; ++y) { + for (int x = patch.rect.x; x < patch.rect.x + patch.rect.width; ++x) { + if (x < 0 || x >= textureSize || y < 0 || y >= textureSize) continue; + const Pixel8U& px = atlas(y, x); + if (px[0]==0 && px[1]==0 && px[2]==0) continue; // 跳过空像素 + patchAvgColor[i][0] += px[2]; // R + patchAvgColor[i][1] += px[1]; // G + patchAvgColor[i][2] += px[0]; // B + pixelCount[i]++; + } + } + if (pixelCount[i] > 0) { + const float inv = 1.0f / (float)pixelCount[i]; + patchAvgColor[i][0] *= inv; + patchAvgColor[i][1] *= inv; + patchAvgColor[i][2] *= inv; + } + } + + DEBUG_EXTRA("Patch avg color computed: %d patches, %d with pixels", + NP, NP - std::count(pixelCount.begin(), pixelCount.end(), 0)); // 5/6/7. 接缝 BuildSeamEdgesFromRCPatches(); + if (!rcSeamEdges.empty()) { + DEBUG_EXTRA("Before GlobalAlign: rcSeamEdges=%zu, patchAvgColor=%zu", + rcSeamEdges.size(), patchAvgColor.size()); + GlobalPatchColorAlignment(atlas, textureSize); // 先全局对齐 + LocalSeamBlending(atlas, textureSize); // 再局部软化 + } + if (!seamEdges.empty()) { TD_TIMER_START(); SeamBlendingFromOriginalImages(atlas);