diff --git a/libs/MVS/SceneTexture.cpp b/libs/MVS/SceneTexture.cpp index 74d9465..8d93b11 100644 --- a/libs/MVS/SceneTexture.cpp +++ b/libs/MVS/SceneTexture.cpp @@ -723,6 +723,79 @@ public: Pixel8U SampleImageBicubic(const Image8U3& img, const Point2f& pt); void ApplyUnsharpMask(cv::Mat& image, float strength); void FillTextureGaps2(cv::Mat& texture, const cv::Mat1f& weights, Pixel8U colEmpty); + + // ======================================================================== + // 辅助函数：双线性采样 + // ======================================================================== + inline Color BilinearSample(const Image8U3& img, const Point2f& p) { + const int w = img.width(); + const int h = img.height(); + const float fx = p.x - 0.5f; // 像素中心对齐 + const float fy = p.y - 0.5f; + int x0 = (int)std::floor(fx); + int y0 = (int)std::floor(fy); + float dx = fx - x0; + float dy = fy - y0; + + auto get = [&](int x, int y) -> Color { + x = std::max(0, std::min(x, w - 1)); + y = std::max(0, std::min(y, h - 1)); + return img(x, y); + }; + + Color c00 = get(x0, y0); + Color c10 = get(x0 + 1, y0); + Color c01 = get(x0, y0 + 1); + Color c11 = get(x0 + 1, y0 + 1); + + return c00 * ((1.0f - dx) * (1.0f - dy)) + + c10 * (dx * (1.0f - dy)) + + c01 * ((1.0f - dx) * dy) + + c11 * (dx * dy); + } + + // ======================================================================== + // 辅助函数：透视校正插值权重 + // ======================================================================== + inline Point3f ComputePerspectiveBarycentric( + const Point2f& uv, const TexCoord* uvCoords, + const Point3f* worldVerts, float* w) + { + // 标准重心坐标 + const float denom = 1.0f / ((uvCoords[1].y - uvCoords[2].y) * (uvCoords[0].x - uvCoords[2].x) + + (uvCoords[2].x - uvCoords[1].x) * (uvCoords[0].y - uvCoords[2].y)); + const float b0 = ((uvCoords[1].y - uvCoords[2].y) * (uv.x - uvCoords[2].x) + + (uvCoords[2].x - uvCoords[1].x) * (uv.y - uvCoords[2].y)) * denom; + const float b1 = ((uvCoords[2].y - uvCoords[0].y) * (uv.x - uvCoords[2].x) + + (uvCoords[0].x - uvCoords[2].x) * (uv.y - uvCoords[2].y)) * denom; + const float b2 = 1.0f - b0 - b1; + + // 透视校正：需要深度（Z） + w[0] = b0; + w[1] = b1; + w[2] = b2; + + return Point3f( + worldVerts[0].x * b0 + worldVerts[1].x * b1 + worldVerts[2].x * b2, + worldVerts[0].y * b0 + worldVerts[1].y * b1 + worldVerts[2].y * b2, + worldVerts[0].z * b0 + worldVerts[1].z * b1 + worldVerts[2].z * b2 + ); + } + + // ======================================================================== + // 辅助函数：ProjectPoint（透视投影到图像坐标） + // ======================================================================== + inline Point2f ProjectToImage(const RMatrix& R, const Point3f& C, const Point3f& worldPt) { + // R: 相机旋转, C: 相机中心 + Point3f camPt( + R(0,0)*(worldPt.x-C.x) + R(0,1)*(worldPt.y-C.y) + R(0,2)*(worldPt.z-C.z), + R(1,0)*(worldPt.x-C.x) + R(1,1)*(worldPt.y-C.y) + R(1,2)*(worldPt.z-C.z), + R(2,0)*(worldPt.x-C.x) + R(2,1)*(worldPt.y-C.y) + R(2,2)*(worldPt.z-C.z) + ); + if (camPt.z <= 0) return Point2f(-1, -1); // 在相机后面 + return Point2f(camPt.x / camPt.z, camPt.y / camPt.z); + } + Mesh::Image8U3Arr GenerateMultiViewTextureAtlasWithVirtualFaces( const VirtualFaceMap& virtualFaceMap, const VirtualFaceDataArr& virtualFaceDatas, // 改为 VirtualFaceDataArr @@ -15809,200 +15882,225 @@ Mesh::Image8U3Arr MeshTexture::GenerateMultiViewTextureAtlasWithVirtualFaces( Pixel8U colEmpty, float fSharpnessWeight) { - DEBUG_EXTRA("Generating multi-view texture atlas with virtual faces (SHARPENED)"); + DEBUG_EXTRA("Generating SINGLE-VIEW + Bilinear + 1:1 Rasterized texture atlas"); TD_TIMER_START(); - - // 1. 分析UV布局 + + // ======================================================================== + // 1. 计算 UV 边界和纹理尺寸 + // ======================================================================== AABB2f uvBounds(true); - FOREACH(i, scene.mesh.faceTexcoords) { - const TexCoord& uv = scene.mesh.faceTexcoords[i]; - uvBounds.InsertFull(uv); + for (const auto& tc : scene.mesh.faceTexcoords) { + uvBounds.InsertFull(tc); } - - // 2. 计算纹理尺寸 - float uvWidth = uvBounds.ptMax.x() - uvBounds.ptMin.x(); - float uvHeight = uvBounds.ptMax.y() - uvBounds.ptMin.y(); + if (uvBounds.ptMax.x() - uvBounds.ptMin.x() < 0.001f) + uvBounds.ptMax.x() = uvBounds.ptMin.x() + 1.0f; + if (uvBounds.ptMax.y() - uvBounds.ptMin.y() < 0.001f) + uvBounds.ptMax.y() = uvBounds.ptMin.y() + 1.0f; - if (uvWidth < 0.001f) uvWidth = 1.0f; - if (uvHeight < 0.001f) uvHeight = 1.0f; + const float uvWidth = uvBounds.ptMax.x() - uvBounds.ptMin.x(); + const float uvHeight = uvBounds.ptMax.y() - uvBounds.ptMin.y(); + // 1:1 渲染，不超分 const int textureSize = ComputeOptimalTextureSize(uvWidth, uvHeight, nTextureSizeMultiple); - - // ✅ 关键修改1：使用超分辨率渲染（2倍） - constexpr int RENDER_SCALE = 2; - const int renderSize = textureSize * RENDER_SCALE; - - // 3. 创建纹理图集（最终输出尺寸） + + DEBUG_EXTRA("UV bounds: [%.4f,%.4f] -> [%.4f,%.4f], textureSize=%d", + uvBounds.ptMin.x(), uvBounds.ptMin.y(), + uvBounds.ptMax.x(), uvBounds.ptMax.y(), + textureSize); + + // ======================================================================== + // 2. 创建输出纹理（单张） + // ======================================================================== Mesh::Image8U3Arr textures; - Image8U3& textureAtlas = textures.emplace_back(textureSize, textureSize); - textureAtlas.setTo(cv::Scalar(colEmpty.b, colEmpty.g, colEmpty.r)); - - // 4. 创建高分辨率累积缓冲区 - cv::Mat1f weightAccum(renderSize, renderSize, 0.0f); - cv::Mat3f colorAccum(renderSize, renderSize, cv::Vec3f(0, 0, 0)); - - DEBUG_EXTRA("Texture atlas size: %dx%d, Render size: %dx%d, UV bounds: [%.3f,%.3f]-[%.3f,%.3f]", - textureSize, textureSize, renderSize, renderSize, - uvBounds.ptMin.x(), uvBounds.ptMin.y(), - uvBounds.ptMax.x(), uvBounds.ptMax.y()); - - // 5. 处理每个虚拟面 + Image8U3& atlas = textures.emplace_back(textureSize, textureSize); + atlas.setTo(cv::Scalar(colEmpty.b, colEmpty.g, colEmpty.r)); + + // 权重图（记录每个像素是否被写入） + cv::Mat1f weightMap(textureSize, textureSize, 0.0f); + + // ======================================================================== + // 3. 逐虚拟面光栅化（单视图 + 双线性 + 透视校正） + // ======================================================================== #ifdef _USE_OPENMP #pragma omp parallel for schedule(dynamic) - for (int_t idxVF = 0; idxVF < (int_t)virtualFaceMap.size(); ++idxVF) { - #else - for (size_t idxVF = 0; idxVF < virtualFaceMap.size(); ++idxVF) { #endif + for (int_t idxVF = 0; idxVF < (int_t)virtualFaceMap.size(); ++idxVF) { const VirtualFace& vf = virtualFaceMap[idxVF]; - - // 检查虚拟面是否有可用视图 - if (faceViews[idxVF].empty()) { - continue; + + if (vf.faces.empty()) continue; + if (idxVF >= (int_t)faceViews.size() || faceViews[idxVF].empty()) continue; + + // ✅ 只取权重最大的那一个视图（单视图策略） + size_t bestViewIdx = 0; + float bestW = faceViewWeights[idxVF][0]; + for (size_t vi = 1; vi < faceViewWeights[idxVF].size(); ++vi) { + if (faceViewWeights[idxVF][vi] > bestW) { + bestW = faceViewWeights[idxVF][vi]; + bestViewIdx = vi; + } } - - // 处理虚拟面中的每个原始面片 + const IIndex idxView = faceViews[idxVF][bestViewIdx]; + if (idxView >= (IIndex)images.size()) continue; + + const Image& srcImage = images[idxView]; + const int srcW = srcImage.image.width(); + const int srcH = srcImage.image.height(); + + // 相机参数 + const RMatrix& R = srcImage.camera.R; + const Point3f& C = srcImage.camera.C; + + // ==================================================================== + // 对每个面片光栅化 + // ==================================================================== for (FIndex faceID : vf.faces) { + if (faceID >= (FIndex)scene.mesh.faces.size()) continue; + const Face& face = scene.mesh.faces[faceID]; const TexCoord* uvCoords = &scene.mesh.faceTexcoords[faceID * 3]; - - // ✅ 关键修改2：计算高分辨率下的边界框 - cv::Rect bbox; + const Point3f* worldVerts[3] = { + &scene.mesh.vertices[face[0]], + &scene.mesh.vertices[face[1]], + &scene.mesh.vertices[face[2]] + }; + + // ---- 计算纹理空间 bounding box ---- + int minX = textureSize, minY = textureSize, maxX = 0, maxY = 0; for (int i = 0; i < 3; ++i) { - int px = int(uvCoords[i].x * renderSize); - int py = int(uvCoords[i].y * renderSize); - if (bbox.empty()) - bbox = cv::Rect(px, py, 1, 1); - else - bbox |= cv::Rect(px, py, 1, 1); + int px = (int)(uvCoords[i].x * textureSize); + int py = (int)(uvCoords[i].y * textureSize); + minX = std::min(minX, px); maxX = std::max(maxX, px); + minY = std::min(minY, py); maxY = std::max(maxY, py); } - bbox &= cv::Rect(0, 0, renderSize, renderSize); - - if (bbox.empty()) continue; - - // ✅ 关键修改3：直接在高分辨率下逐像素采样（不再使用Delaunay插值） - for (int y = bbox.y; y < bbox.y + bbox.height; ++y) { - for (int x = bbox.x; x < bbox.x + bbox.width; ++x) { - // 计算当前像素的UV坐标（在高分辨率空间） - Point2f texCoord( - static_cast(x) / renderSize, - static_cast(y) / renderSize + minX = std::max(0, minX - 1); + minY = std::max(0, minY - 1); + maxX = std::min(textureSize - 1, maxX + 1); + maxY = std::min(textureSize - 1, maxY + 1); + + if (maxX < minX || maxY < minY) continue; + + // ---- 预计算 3 个顶点在图像中的投影位置 ---- + Point2f imgPts[3]; + bool validProj = true; + for (int i = 0; i < 3; ++i) { + imgPts[i] = ProjectToImage(R, C, *worldVerts[i]); + if (imgPts[i].x < 0 || imgPts[i].y < 0 || + imgPts[i].x >= srcW || imgPts[i].y >= srcH) { + validProj = false; + break; + } + } + if (!validProj) continue; + + // ---- 逐像素光栅化 ---- + for (int y = minY; y <= maxY; ++y) { + for (int x = minX; x <= maxX; ++x) { + Point2f uv( + (float)x / (float)textureSize, + (float)y / (float)textureSize ); - - // 计算重心坐标 - Point3f barycentric; - if (!PointInTriangle(texCoord, uvCoords[0], uvCoords[1], uvCoords[2], barycentric)) { - continue; - } - - // 计算3D世界坐标 - const Vertex worldPoint = - scene.mesh.vertices[face[0]] * barycentric.x + - scene.mesh.vertices[face[1]] * barycentric.y + - scene.mesh.vertices[face[2]] * barycentric.z; - - // 累积来自所有视图的颜色 - cv::Vec3f accumColor(0, 0, 0); - float totalWeight = 0.0f; - - for (size_t viewIdx = 0; viewIdx < faceViews[idxVF].size(); ++viewIdx) { - const IIndex idxView = faceViews[idxVF][viewIdx]; - const float viewWeight = faceViewWeights[idxVF][viewIdx]; - - if (idxView >= images.size()) continue; - - const Image& sourceImage = images[idxView]; - - // 投影到图像 - Point2f imgPoint = ProjectPointWithAutoCorrection(sourceImage.camera, worldPoint, sourceImage); - - // 验证投影 - if (!ValidateProjection(worldPoint, sourceImage, imgPoint) || - !sourceImage.image.isInside(imgPoint) || - !sourceImage.camera.IsInFront(worldPoint)) { - continue; - } - - Color color = sourceImage.image((int)imgPoint.x, (int)imgPoint.y); - - // ✅ 关键修改5：使用面积权重（而非简单计数） - const float areaWeight = viewWeight * (1.0f / (RENDER_SCALE * RENDER_SCALE)); - - // 累积加权颜色（BGR顺序） - accumColor[0] += color[0] * areaWeight; // B - accumColor[1] += color[1] * areaWeight; // G - accumColor[2] += color[2] * areaWeight; // R - totalWeight += areaWeight; - } - - // ✅ 关键修改6：原子累加（避免竞态条件） - if (totalWeight > 0.0f) { - #ifdef _USE_OPENMP - #pragma omp critical - #endif - { - colorAccum(y, x) += accumColor; - weightAccum(y, x) += totalWeight; + + // 重心坐标 + float w[3]; + // 用面积法计算重心坐标 + auto edgeFunc = [](const Point2f& a, const Point2f& b, const Point2f& c) { + return (c.x - a.x) * (b.y - a.y) - (c.y - a.y) * (b.x - a.x); + }; + float area = edgeFunc(uvCoords[0], uvCoords[1], uvCoords[2]); + if (std::abs(area) < 1e-10f) continue; + + w[0] = edgeFunc(uvCoords[1], uvCoords[2], uv) / area; + w[1] = edgeFunc(uvCoords[2], uvCoords[0], uv) / area; + w[2] = 1.0f - w[0] - w[1]; + + // 不在三角形内 + if (w[0] < 0 || w[1] < 0 || w[2] < 0) continue; + + // ---- 透视校正插值 ---- + // 用重心坐标插值世界坐标 + Point3f worldPt( + worldVerts[0]->x * w[0] + worldVerts[1]->x * w[1] + worldVerts[2]->x * w[2], + worldVerts[0]->y * w[0] + worldVerts[1]->y * w[1] + worldVerts[2]->y * w[2], + worldVerts[0]->z * w[0] + worldVerts[1]->z * w[1] + worldVerts[2]->z * w[2] + ); + + // 投影到图像 + Point2f imgPt = ProjectToImage(R, C, worldPt); + if (imgPt.x < 0 || imgPt.y < 0 || + imgPt.x >= srcW - 1 || imgPt.y >= srcH - 1) continue; + + // ---- ✅ 双线性采样 ---- + Color color = BilinearSample(srcImage.image, imgPt); + + // ---- 写入 atlas（临界区保护） ---- + #ifdef _USE_OPENMP + #pragma omp critical + #endif + { + Pixel8U& dst = atlas(y, x); + // 如果是第一个写入者，直接赋值；否则覆盖（单视图无混合） + if (weightMap(y, x) == 0.0f) { + dst.b = (unsigned char)color[0]; + dst.g = (unsigned char)color[1]; + dst.r = (unsigned char)color[2]; + weightMap(y, x) = 1.0f; } + // 单视图策略：不混合，后写入者覆盖（或反之） + // 如果需要"先到先得"，用上面的 if + // 如果需要"最佳视图覆盖"，去掉 if 直接用下面： + // dst.b = (unsigned char)color[0]; + // dst.g = (unsigned char)color[1]; + // dst.r = (unsigned char)color[2]; } } } } } - - // 6. 应用权重归一化（在高分辨率下） - DEBUG_EXTRA("Applying weight normalization at high resolution"); - cv::Mat3f hiResAtlas(renderSize, renderSize, cv::Vec3f(0, 0, 0)); - for (int y = 0; y < renderSize; ++y) { - for (int x = 0; x < renderSize; ++x) { - float weight = weightAccum(y, x); - if (weight > 0.0f) { - hiResAtlas(y, x) = colorAccum(y, x) / weight; - } else { - // 对于未采样的像素，使用背景色 - hiResAtlas(y, x) = cv::Vec3f(colEmpty[2], colEmpty[1], colEmpty[0]); - } - } - } - - // 7. ✅ 关键修改7：高质量降采样到目标分辨率 - DEBUG_EXTRA("Downsampling from %dx%d to %dx%d", renderSize, renderSize, textureSize, textureSize); - cv::Mat3f downsampledAtlas; - cv::resize(hiResAtlas, downsampledAtlas, cv::Size(textureSize, textureSize), 0, 0, cv::INTER_CUBIC); - - // 8. 转换为8位纹理 + + // ======================================================================== + // 4. 缝隙填充（可选，用形态学膨胀） + // ======================================================================== + DEBUG_EXTRA("Filling small gaps..."); + cv::Mat atlasMat = (cv::Mat&)atlas; + cv::Mat mask = (weightMap > 0.5f); + cv::Mat atlasF; + atlasMat.convertTo(atlasF, CV_32FC3); + + // 膨胀 mask + cv::Mat kernel = cv::getStructuringElement(cv::MORPH_RECT, cv::Size(3, 3)); + cv::Mat dilatedMask; + cv::dilate(mask, dilatedMask, kernel); + + // 对未填充区域做距离变换 + 插值 + cv::Mat distMap, nearestLoc; + cv::distanceTransform(mask, distMap, nearestLoc, cv::DIST_L2, 5); for (int y = 0; y < textureSize; ++y) { for (int x = 0; x < textureSize; ++x) { - const cv::Vec3f& color = downsampledAtlas(y, x); - Pixel8U finalColor; - finalColor.b = (unsigned char)cv::saturate_cast(color[0]); - finalColor.g = (unsigned char)cv::saturate_cast(color[1]); - finalColor.r = (unsigned char)cv::saturate_cast(color[2]); - textureAtlas(y, x) = finalColor; + if (weightMap(y, x) == 0.0f && dilatedMask.at(y, x)) { + // 用最近的有效像素填充 + int nearestIdx = nearestLoc.at(y, x); + int ny = nearestIdx / textureSize; + int nx = nearestIdx % textureSize; + atlasMat.at(y, x) = atlasMat.at(ny, nx); + } } } - - // 9. 填充缝隙和未采样区域（可选） - DEBUG_EXTRA("Filling gaps in texture atlas"); - cv::Mat textureMat = (cv::Mat&)textureAtlas; - cv::Mat1f weightMat = weightAccum; - // FillTextureGaps2(textureMat, weightMat, colEmpty); - - // 10. ✅ 关键修改8：应用锐化（显著提升清晰度） + + // ======================================================================== + // 5. 可选锐化（轻微） + // ======================================================================== if (fSharpnessWeight > 0) { - DEBUG_EXTRA("Applying sharpening filter (weight: %.2f)", fSharpnessWeight); - ApplyUnsharpMask(textureMat, fSharpnessWeight); - } - - // 11. 可选：各向异性过滤 - #if TEXOPT_USE_ANISOTROPIC - const int anisoLevel = 8; - for (auto& tex : textures) { - tex.SetFilterMode(Texture::ANISOTROPIC); - tex.SetAnisotropy(anisoLevel); + DEBUG_EXTRA("Applying mild sharpening (weight: %.2f)", fSharpnessWeight); + cv::Mat kernel = (cv::Mat_(3, 3) << + 0, -fSharpnessWeight, 0, + -fSharpnessWeight, 1 + 4 * fSharpnessWeight, -fSharpnessWeight, + 0, -fSharpnessWeight, 0); + cv::Mat sharpened; + cv::filter2D(atlasMat, sharpened, CV_8UC3, kernel); + atlasMat = sharpened; } - #endif - - DEBUG_EXTRA("Multi-view texture atlas generation completed in %s", TD_TIMER_GET_FMT().c_str()); + + DEBUG_EXTRA("Single-view texture atlas completed in %s", TD_TIMER_GET_FMT().c_str()); return textures; }