From 7a6887349efd05e968bbb67eb7fd67766ecf5ce7 Mon Sep 17 00:00:00 2001 From: hesuicong Date: Mon, 4 May 2026 17:23:00 +0800 Subject: [PATCH] =?UTF-8?q?=E8=BF=9B=E4=B8=80=E6=AD=A5=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- libs/MVS/SceneTexture.cpp | 1814 +++++++------------------------------ 1 file changed, 347 insertions(+), 1467 deletions(-) diff --git a/libs/MVS/SceneTexture.cpp b/libs/MVS/SceneTexture.cpp index 4dc1c9d..d64493f 100644 --- a/libs/MVS/SceneTexture.cpp +++ b/libs/MVS/SceneTexture.cpp @@ -192,6 +192,14 @@ struct PatchQualityInfo { std::vector faceQualities; }; + +// 在函数外部定义 PixelSample +struct PixelSample { + cv::Vec3f color; // 颜色（BGR格式） + float weight; // 权重 + IIndex viewIdx; // 视图索引 +}; + std::vector patchQualityInfos; struct MeshTexture { @@ -333,6 +341,11 @@ struct MeshTexture { }; typedef cList TexturePatchArr; + struct FaceScore { + MVS::IIndex view; + float score; + }; + // used to optimize texture patches struct SeamVertex { struct Patch { @@ -527,72 +540,15 @@ public: const Mesh::TexCoordArr& existingTexcoords, // 添加已有UV参数 const Mesh::TexIndexArr& existingTexindices // 添加已有纹理索引参数 ); - - struct PixelSample { - cv::Vec3f color; // BGR 颜色 - float weight; // 权重 - IIndex viewIdx; // 视图索引 - }; - - struct FaceViewScore { - IIndex viewIdx; // 视图索引 - float score; // 视图得分 - float quality; // 视图质量 - }; - - struct EnhancedPixelSample { - cv::Vec3f color; // BGR颜色 - float weight; // 权重 - IIndex viewIdx; // 视图索引 - float viewQuality; // 视图质量评分 - float resolution; // 分辨率 - float viewAngle; // 视角角度 - - EnhancedPixelSample(const cv::Vec3f& c, float w, IIndex v, float q, float r, float a) - : color(c), weight(w), viewIdx(v), viewQuality(q), resolution(r), viewAngle(a) {} - - // 计算样本的综合质量 - float GetOverallQuality() const { - return viewQuality * resolution * viewAngle; - } - }; - - // 在 MeshTexture 结构体声明中添加以下函数 - void BuildFaceNeighbors(std::vector>& faceNeighbors) const; - void OptimizeViewSelectionWithGraphCut(LabelArr& faceLabels, const std::vector>& faceNeighbors); - std::vector SelectHighQualityAuxiliaryViews(FIndex faceID, IIndex primaryView, const LabelArr& faceLabels, const IIndexArr& views); - void ProcessFaceForTextureSampling(FIndex faceID, IIndex viewIdx, const Mesh::TexCoordArr& texcoords, int textureSize, - std::vector>& pixelSamples, int& totalSamples, bool isPrimary); - Pixel8U FuseColorsIntelligently(const std::vector& samples, int pixelIdx, int textureSize, - const std::vector>& allPixelSamples); - void ApplyAdaptiveColorCorrection(Image8U3& texture, Pixel8U colEmpty, float strength); - void ApplyEdgePreservingSharpening(Image8U3& texture, float strength, Pixel8U colEmpty); - Mesh::Image8U3Arr GenerateTextureAtlasWithEnhancedConsistency( - const LabelArr& faceLabels, - const IIndexArr& views, - const Mesh::TexCoordArr& sourceTexcoords, - unsigned nTextureSizeMultiple, - Pixel8U colEmpty, - float fSharpnessWeight); - void ProcessFaceForTexture( - FIndex faceID, - IIndex viewIdx, - const Mesh::TexCoordArr& texcoords, - int textureSize, - std::vector>& pixelSamples, - int& sampleCounter, - bool isPrimaryView); - void ApplyBoundarySmoothing( - Image8U3& texture, - Pixel8U colEmpty, - int textureSize); - void ApplyBalancedColorCorrection(Image8U3& texture, Pixel8U colEmpty, float strength); - void ApplyConservativeSharpening(Image8U3& texture, float strength, Pixel8U colEmpty); bool GenerateTextureWithViewConsistency( bool bGlobalSeamLeveling, bool bLocalSeamLeveling, unsigned nTextureSizeMultiple, unsigned nRectPackingHeuristic, Pixel8U colEmpty, float fSharpnessWeight, int maxTextureSize, const String& basename, bool bOriginFaceview, Scene* pScene); + void OptimizeViewSelectionWithGraphCut(std::vector& faceScores, + const std::vector>& faceNeighbors, + float fRatioDataSmoothness); + float ComputeFaceViewScore(FIndex faceID, IIndex viewID, float fOutlierThreshold); bool ValidateSeamDataForLeveling(); void CheckMemoryIntegrity(); void RebuildComponentMapping(); @@ -13704,27 +13660,25 @@ bool MeshTexture::TextureWithExistingUV( return false; } - // 注意：这里故意不检查 existingTextures 是否为空 - // 因为这个函数的核心是从原始图像重新采样，不是从现有纹理复制 - DEBUG_EXTRA("Processing %zu faces with existing UV data", scene.mesh.faces.size()); + if (existingTextures.empty()) { + VERBOSE("error: no existing texture data provided"); + // return false; + } - // 2. 重新为每个面选择最佳视图 + DEBUG_EXTRA("Processing %zu faces with existing texture data", scene.mesh.faces.size()); + + // 2. 为每个面选择最佳视图（从原始图像，而不是已有纹理） FaceDataViewArr facesDatas; if (!ListCameraFaces(facesDatas, fOutlierThreshold, nIgnoreMaskLabel, views, false)) { return false; } - // 3. 为每个面分配最佳视图（使用改进的算法） + // 3. 为每个面分配最佳视图 LabelArr faceLabels(scene.mesh.faces.size()); - - // 步骤1：计算相邻面关系 - std::vector> faceNeighbors(scene.mesh.faces.size()); - - // 为每个面选择最佳视图 - for (FIndex idxFace = 0; idxFace < scene.mesh.faces.size(); ++idxFace) { + FOREACH(idxFace, scene.mesh.faces) { const FaceDataArr& faceDatas = facesDatas[idxFace]; if (faceDatas.empty()) { - faceLabels[idxFace] = 0; + faceLabels[idxFace] = 0; // 无视图可用 continue; } @@ -13732,7 +13686,7 @@ bool MeshTexture::TextureWithExistingUV( float bestQuality = -1; IIndex bestView = NO_ID; for (const FaceData& data : faceDatas) { - if (!data.bInvalidFacesRelative && data.quality > bestQuality) { + if (data.quality > bestQuality && !data.bInvalidFacesRelative) { bestQuality = data.quality; bestView = data.idxView; } @@ -13741,18 +13695,21 @@ bool MeshTexture::TextureWithExistingUV( faceLabels[idxFace] = (bestView != NO_ID) ? (bestView + 1) : 0; } - // 步骤2：使用图割优化视图选择 - BuildFaceNeighbors(faceNeighbors); - OptimizeViewSelectionWithGraphCut(faceLabels, faceNeighbors); - - // 4. 生成纹理图集（使用改进的版本） - Mesh::Image8U3Arr generatedTextures = GenerateTextureAtlasWithEnhancedConsistency( - // faceLabels, views, existingTexcoords, - faceLabels, views, scene.mesh.faceTexcoords, + // 4. 生成纹理图集 + Mesh::Image8U3Arr generatedTextures = GenerateTextureAtlasWith3DBridge( + faceLabels, views, existingTextures, existingTexcoords, existingTexindices, nTextureSizeMultiple, colEmpty, fSharpnessWeight ); if (!generatedTextures.empty()) { + // 检查颜色通道 + CheckColorChannels(generatedTextures[0], "生成的纹理"); + + // 检查源纹理颜色通道 + if (!existingTextures.empty()) { + CheckColorChannels(existingTextures[0], "源纹理"); + } + // 保存纹理 scene.mesh.texturesDiffuse = std::move(generatedTextures); @@ -13766,7 +13723,7 @@ bool MeshTexture::TextureWithExistingUV( scene.mesh.faceTexindices.Release(); } - DEBUG_EXTRA("Generated %zu textures from existing UV data", + DEBUG_EXTRA("Generated %zu textures from existing data", scene.mesh.texturesDiffuse.size()); return true; } @@ -13775,1338 +13732,182 @@ bool MeshTexture::TextureWithExistingUV( return false; } -void MeshTexture::ApplyAdaptiveColorCorrection(Image8U3& texture, Pixel8U colEmpty, float strength) { - if (strength <= 0) return; - - cv::Mat& textureMat = (cv::Mat&)texture; +void MeshTexture::FillTextureGaps(Image8U3& texture, Pixel8U colEmpty) { + if (texture.empty()) return; - // 统计非背景像素 - cv::Vec3d sum(0, 0, 0); - int count = 0; + cv::Mat textureMat = (cv::Mat&)texture; + // 创建掩码 + cv::Mat mask = cv::Mat::zeros(texture.rows, texture.cols, CV_8UC1); for (int y = 0; y < texture.rows; ++y) { for (int x = 0; x < texture.cols; ++x) { - Pixel8U pixel = texture(y, x); - if (pixel != colEmpty) { - sum[0] += pixel.r; // R - sum[1] += pixel.g; // G - sum[2] += pixel.b; // B - count++; + if (texture(y, x) != colEmpty) { + mask.at(y, x) = 255; } } } - if (count == 0) return; - - cv::Vec3d mean = sum / count; - double avgGray = (mean[0] + mean[1] + mean[2]) / 3.0; - - // 计算增益因子 - double gainR = 1.0, gainG = 1.0, gainB = 1.0; - if (avgGray > 0) { - gainR = avgGray / mean[0]; - gainG = avgGray / mean[1]; - gainB = avgGray / mean[2]; - } - - // 限制调整幅度 - gainR = std::max(0.9, std::min(gainR, 1.1)); - gainG = std::max(0.9, std::min(gainG, 1.1)); - gainB = std::max(0.9, std::min(gainB, 1.1)); - - DEBUG_EXTRA("全局颜色校正: 平均颜色 R=%.1f, G=%.1f, B=%.1f", mean[0], mean[1], mean[2]); - DEBUG_EXTRA("全局颜色校正: 增益因子 R=%.3f, G=%.3f, B=%.3f", gainR, gainG, gainB); + // 使用图像修复算法填充空隙 + cv::Mat inpainted; + cv::inpaint(textureMat, mask, inpainted, 3, cv::INPAINT_NS); - // 应用颜色校正 + // 只替换空区域 for (int y = 0; y < texture.rows; ++y) { for (int x = 0; x < texture.cols; ++x) { - Pixel8U& pixel = texture(y, x); - if (pixel == colEmpty) continue; - - // 线性混合原始颜色和校正颜色 - double r = pixel.r * (1.0 - strength) + pixel.r * gainR * strength; - double g = pixel.g * (1.0 - strength) + pixel.g * gainG * strength; - double b = pixel.b * (1.0 - strength) + pixel.b * gainB * strength; - - pixel.r = (unsigned char)cv::saturate_cast(r); - pixel.g = (unsigned char)cv::saturate_cast(g); - pixel.b = (unsigned char)cv::saturate_cast(b); + if (texture(y, x) == colEmpty) { + cv::Vec3b color = inpainted.at(y, x); + Pixel8U pixel; + pixel.b = color[0]; + pixel.g = color[1]; + pixel.r = color[2]; + texture(y, x) = pixel; + } } } } -void MeshTexture::ApplyEdgePreservingSharpening(Image8U3& texture, float strength, Pixel8U colEmpty) { - if (strength <= 0) return; +void MeshTexture::ApplyColorConsistencyOptimization( + Image8U3& texture, + const std::vector>& pixelSamples, + int textureSize, + Pixel8U colEmpty) +{ + if (pixelSamples.empty()) return; - cv::Mat& textureMat = (cv::Mat&)texture; + cv::Mat textureMat = (cv::Mat&)texture; - // 使用轻微的高斯模糊 - cv::Mat blurred; - cv::GaussianBlur(textureMat, blurred, cv::Size(3, 3), 0.5); + // 创建临时的颜色缓冲区 + cv::Mat3f newColor(texture.rows, texture.cols, cv::Vec3f(0, 0, 0)); + cv::Mat1f newWeight(texture.rows, texture.cols, 0.0f); - // 计算细节层 - cv::Mat detail = textureMat.clone(); - for (int y = 0; y < texture.rows; ++y) { - for (int x = 0; x < texture.cols; ++x) { - Pixel8U pixel = texture(y, x); - if (pixel == colEmpty) { - detail.at(y, x) = cv::Vec3b(colEmpty.b, colEmpty.g, colEmpty.r); - continue; + // 计算邻域颜色一致性 + const int kernelSize = 3; + const int halfKernel = kernelSize / 2; + + for (int y = halfKernel; y < texture.rows - halfKernel; ++y) { + for (int x = halfKernel; x < texture.cols - halfKernel; ++x) { + int pixelIdx = y * textureSize + x; + const auto& samples = pixelSamples[pixelIdx]; + + if (samples.empty()) continue; + + // 收集邻域颜色 + std::vector neighborColors; + for (int dy = -halfKernel; dy <= halfKernel; ++dy) { + for (int dx = -halfKernel; dx <= halfKernel; ++dx) { + int nx = x + dx; + int ny = y + dy; + int neighborIdx = ny * textureSize + nx; + + if (!pixelSamples[neighborIdx].empty()) { + // 获取当前像素颜色 + Pixel8U pixel = texture(ny, nx); + if (pixel != colEmpty) { + neighborColors.push_back(cv::Vec3f(pixel.b, pixel.g, pixel.r)); + } + } + } } - cv::Vec3b& detailPixel = detail.at(y, x); - cv::Vec3b blurPixel = blurred.at(y, x); + if (neighborColors.size() < 3) continue; // 需要足够的邻域信息 - // 计算细节 - detailPixel[0] = cv::saturate_cast(detailPixel[0] - blurPixel[0] + 128); - detailPixel[1] = cv::saturate_cast(detailPixel[1] - blurPixel[1] + 128); - detailPixel[2] = cv::saturate_cast(detailPixel[2] - blurPixel[2] + 128); - } - } - - // 应用锐化 - float alpha = 0.5f * strength; - for (int y = 0; y < texture.rows; ++y) { - for (int x = 0; x < texture.cols; ++x) { - Pixel8U pixel = texture(y, x); - if (pixel == colEmpty) continue; + // 计算颜色统计 + cv::Vec3f mean(0, 0, 0); + for (const auto& color : neighborColors) { + mean += color; + } + mean /= (float)neighborColors.size(); - cv::Vec3b& pixelVec = textureMat.at(y, x); - cv::Vec3b detailPixel = detail.at(y, x); + // 计算方差 + cv::Vec3f variance(0, 0, 0); + for (const auto& color : neighborColors) { + cv::Vec3f diff = color - mean; + variance[0] += diff[0] * diff[0]; + variance[1] += diff[1] * diff[1]; + variance[2] += diff[2] * diff[2]; + } + variance /= (float)neighborColors.size(); - pixelVec[0] = cv::saturate_cast(pixelVec[0] + (detailPixel[0] - 128) * alpha); - pixelVec[1] = cv::saturate_cast(pixelVec[1] + (detailPixel[1] - 128) * alpha); - pixelVec[2] = cv::saturate_cast(pixelVec[2] + (detailPixel[2] - 128) * alpha); + // 获取当前像素颜色 + Pixel8U currentPixel = texture(y, x); + cv::Vec3f currentColor(currentPixel.b, currentPixel.g, currentPixel.r); + + // 计算颜色调整 + const float alpha = 0.3f; // 调整强度 + cv::Vec3f adjustedColor = currentColor; + + // 如果颜色差异较大，向邻域均值调整 + cv::Vec3f diff = currentColor - mean; + float diffMag = std::sqrt(diff[0]*diff[0] + diff[1]*diff[1] + diff[2]*diff[2]); + + if (diffMag > 10.0f) { // 颜色差异阈值 + adjustedColor = currentColor * (1.0f - alpha) + mean * alpha; + } + + // 应用调整 + Pixel8U newPixel; + newPixel.r = (unsigned char)cv::saturate_cast(adjustedColor[2]); + newPixel.g = (unsigned char)cv::saturate_cast(adjustedColor[1]); + newPixel.b = (unsigned char)cv::saturate_cast(adjustedColor[0]); + + texture(y, x) = newPixel; } } } -// 新增辅助函数实现 -void MeshTexture::BuildFaceNeighbors(std::vector>& faceNeighbors) const { - const Mesh& mesh = scene.mesh; - const int numFaces = (int)mesh.faces.size(); - - faceNeighbors.resize(numFaces); +Mesh::Image8U3Arr MeshTexture::GenerateTextureAtlasWith3DBridge( + const LabelArr& faceLabels, + const IIndexArr& views, + const Mesh::Image8U3Arr& sourceTextures, + const Mesh::TexCoordArr& sourceTexcoords, + const Mesh::TexIndexArr& sourceTexindices, + unsigned nTextureSizeMultiple, + Pixel8U colEmpty, + float fSharpnessWeight) +{ + DEBUG_EXTRA("GenerateTextureAtlasWith3DBridge - 使用3D几何坐标和多视图融合"); - // 构建边到面的映射 - std::unordered_map> edgeFaces; + // 定义INVALID_INDEX常量 + const IIndex INVALID_INDEX = (IIndex)-1; - for (int fid = 0; fid < numFaces; ++fid) { - const Face& face = mesh.faces[fid]; - - // 三条边 - for (int i = 0; i < 3; ++i) { - int v1 = face[i]; - int v2 = face[(i + 1) % 3]; - - // 确保边有序 - if (v1 > v2) std::swap(v1, v2); - - uint64_t edgeKey = ((uint64_t)v1 << 32) | v2; - edgeFaces[edgeKey].push_back(fid); - } + // 1. 分析外部UV布局 + AABB2f uvBounds(true); + FOREACH(i, scene.mesh.faceTexcoords) { + const TexCoord& uv = scene.mesh.faceTexcoords[i]; + uvBounds.InsertFull(uv); } - // 构建相邻面关系 - for (const auto& pair : edgeFaces) { - const std::vector& faceList = pair.second; - if (faceList.size() >= 2) { - // 边的相邻面 - for (size_t i = 0; i < faceList.size(); ++i) { - for (size_t j = i + 1; j < faceList.size(); ++j) { - int fid1 = faceList[i]; - int fid2 = faceList[j]; - - faceNeighbors[fid1].push_back(fid2); - faceNeighbors[fid2].push_back(fid1); - } + // 确保UV在[0,1]范围内 + if (uvBounds.ptMin.x() < 0 || uvBounds.ptMin.y() < 0 || + uvBounds.ptMax.x() > 1 || uvBounds.ptMax.y() > 1) { + DEBUG_EXTRA("UV超出[0,1]范围，进行归一化"); + uvBounds = AABB2f(true); + for (size_t i = 0; i < scene.mesh.faceTexcoords.size(); i += 3) { + for (int v = 0; v < 3; ++v) { + const TexCoord& uv = scene.mesh.faceTexcoords[i + v]; + uvBounds.InsertFull(uv); } } } - // 去重 - for (int fid = 0; fid < numFaces; ++fid) { - auto& neighbors = faceNeighbors[fid]; - std::sort(neighbors.begin(), neighbors.end()); - neighbors.erase(std::unique(neighbors.begin(), neighbors.end()), neighbors.end()); - } + // 计算纹理尺寸 + const float uvWidth = uvBounds.ptMax.x() - uvBounds.ptMin.x(); + const float uvHeight = uvBounds.ptMax.y() - uvBounds.ptMin.y(); + const int textureSize = ComputeOptimalTextureSize(uvWidth, uvHeight, nTextureSizeMultiple); - DEBUG_EXTRA("相邻面关系构建完成: 共 %d 个面", numFaces); -} - -void MeshTexture::OptimizeViewSelectionWithGraphCut(LabelArr& faceLabels, const std::vector>& faceNeighbors) { - DEBUG_EXTRA("优化视图选择以减少色块"); + // 创建目标纹理 + Mesh::Image8U3Arr textures; + Image8U3& textureAtlas = textures.emplace_back(textureSize, textureSize); - if (faceLabels.empty() || faceNeighbors.empty()) return; + // 使用统一的背景色设置 + DEBUG_EXTRA("设置背景色: RGB(%d,%d,%d)", colEmpty.r, colEmpty.g, colEmpty.b); + cv::Scalar cvEmpty(colEmpty.b, colEmpty.g, colEmpty.r); + textureAtlas.setTo(cvEmpty); - const float smoothnessWeight = 1.0f; // 平滑项权重 - const float dataWeight = 0.5f; // 数据项权重 - - int iterationCount = 0; - int changedCount = 0; - - do { - changedCount = 0; - iterationCount++; - - // 并行处理每个面 - #ifdef _USE_OPENMP - #pragma omp parallel for schedule(dynamic) reduction(+:changedCount) - #endif - for (int_t idxFace = 0; idxFace < (int_t)scene.mesh.faces.size(); ++idxFace) { - FIndex faceID = (FIndex)idxFace; - Label currentLabel = faceLabels[faceID]; - - if (currentLabel == 0) continue; - - IIndex currentView = currentLabel - 1; - float currentEnergy = 0.0f; - - // 平滑项：相邻面的视图一致性 - float smoothEnergy = 0.0f; - int neighborCount = 0; - - for (int neighborID : faceNeighbors[faceID]) { - if (neighborID < 0 || neighborID >= (int)faceLabels.size()) continue; - - Label neighborLabel = faceLabels[neighborID]; - if (neighborLabel == 0) continue; - - if (currentLabel != neighborLabel) { - smoothEnergy += 1.0f; // 视图不同增加能量 - } - neighborCount++; - } - - if (neighborCount > 0) { - smoothEnergy /= neighborCount; - currentEnergy += smoothnessWeight * smoothEnergy; - } - - // 尝试相邻面的视图 - std::vector