import yaml import oss2 from tqdm import tqdm import os import numpy as np import os import argparse # import bpy import sys import open3d as o3d from pathlib import Path import requests import json import shutil from config import url_get_oss_suffix_by_orderId from general import transform_save_bpy from general import read_models_from_json from compute_print_net import get_oss_client def download_transform_save_by_json(json_name, workdir, oss_config): layout_data = download_datas_by_json(json_name, workdir, oss_config) original_obj_pid_dir = workdir transform_save_bpy(layout_data, original_obj_pid_dir) class DataTransfer: ''' 数据传输类 ''' def __init__(self, local_path: str, oss_path: str, oss_client: oss2.Bucket): ''' local_path: 本地输出路径 oss_path: oss路径 oss_client: oss客户端 ''' self.local_path = local_path self.oss_path = oss_path.lstrip('/') self.oss_client = oss_client # self.description = description # @log_execution(self.description) def download_data(self): """ 从 OSS 下载数据到本地,保持原有目录结构 """ # 列出所有对象 objects = [] prefix = self.oss_path.lstrip('/') # 移除开头的 '/' 以匹配 OSS 格式 for obj in oss2.ObjectIterator(self.oss_client, prefix=prefix): if obj.key != prefix: # 跳过目录本身 objects.append(obj.key) # 下载所有文件,添加进度条 for obj_key in tqdm(objects, desc="下载进度"): if obj_key.endswith('/'): continue if "printId" in obj_key: continue # 计算相对路径 rel_path = obj_key[len(prefix):].lstrip('/') # 构建本地完整路径 local_path = os.path.join(self.local_path, rel_path) # 创建必要的目录 os.makedirs(os.path.dirname(local_path), exist_ok=True) # 下载文件 self.oss_client.get_object_to_file(obj_key, local_path) print("download_data local_path=" + local_path) order_id: str pid: str model_height: str def download_data_rename_json(self, json_model_info): """ 从 OSS 下载数据到本地,保持原有目录结构 """ # 列出所有对象 objects = [] prefix = self.oss_path.lstrip('/') # 移除开头的 '/' 以匹配 OSS 格式 for obj in oss2.ObjectIterator(self.oss_client, prefix=prefix): if obj.key != prefix: # 跳过目录本身 objects.append(obj.key) # 下载所有文件,添加进度条 for obj_key in tqdm(objects, desc="下载进度"): if obj_key.endswith('/'): continue if "printId" in obj_key: continue # 计算相对路径 rel_path = obj_key[len(prefix):].lstrip('/') file_dir, file_name = os.path.split(rel_path) file_base, file_ext = os.path.splitext(file_name) # 根据文件后缀名进行重命名 if file_ext.lower() in ['.mtl', '.jpg', '.jpeg', '.png']: # 对于.mtl和图片文件,在原名前加order_id new_file_name = f"{json_model_info.order_id}_{file_name}" # new_file_name = file_name elif file_ext.lower() == '.obj': # 对于.obj文件,完全重命名 new_file_name = f"{json_model_info.obj_name}" else: # 其他文件类型保持原名 new_file_name = file_name print("new_file_name=", new_file_name) # 构建新的相对路径 if file_dir: # 如果有子目录 new_rel_path = os.path.join(file_dir, new_file_name) else: new_rel_path = new_file_name # 构建本地完整路径 local_path = os.path.join(self.local_path, new_rel_path) # 创建必要的目录 os.makedirs(os.path.dirname(local_path), exist_ok=True) # 下载文件 self.oss_client.get_object_to_file(obj_key, local_path) if file_ext == '.obj': # 10MB以上 try: # 使用临时文件避免内存问题 [8](@ref) temp_path = local_path + '.tmp' with open(local_path, 'r', encoding='utf-8') as f_in, \ open(temp_path, 'w', encoding='utf-8') as f_out: mtllib_modified = False for line in f_in: if not mtllib_modified and line.strip().startswith('mtllib '): parts = line.split(' ', 1) if len(parts) > 1: old_mtl_name = parts[1].strip() new_mtl_name = f"{json_model_info.order_id}_{old_mtl_name}" f_out.write(f"mtllib {new_mtl_name}\n") mtllib_modified = True continue f_out.write(line) os.replace(temp_path, local_path) # 原子性替换 except IOError as e: print(f"处理大文件 {local_path} 时出错: {e}") if os.path.exists(temp_path): os.remove(temp_path) # 优化后的.obj文件处理逻辑 if file_ext == '.mtl': try: # 使用更高效的文件读取方式 [6,8](@ref) with open(local_path, 'r', encoding='utf-8') as f: content = f.read() # 使用字符串方法直接查找和替换,避免不必要的循环 [9](@ref) lines = content.split('\n') mtllib_modified = False for i, line in enumerate(lines): stripped_line = line.strip() if not mtllib_modified and stripped_line.startswith('map_Kd '): # 更高效的分割方式 [9](@ref) parts = line.split(' ', 1) if len(parts) > 1: old_name = parts[1].strip() new_name = f"{json_model_info.order_id}_{old_name}" lines[i] = f"map_Kd {new_name}" mtllib_modified = True # print(f"已更新材质库引用: {old_name} -> {new_name}") break # 找到第一个后立即退出 # 批量写入,减少I/O操作 [6](@ref) with open(local_path, 'w', encoding='utf-8') as f: f.write('\n'.join(lines)) except IOError as e: print(f"处理文件 {local_path} 时出错: {e}") except UnicodeDecodeError as e: print(f"文件编码错误 {local_path}: {e}") print(f"下载文件: {obj_key} -> {local_path}") def download_data_rename_batch(self, batch_model_info): """ 从 OSS 下载数据到本地,保持原有目录结构 """ # 列出所有对象 objects = [] prefix = self.oss_path.lstrip('/') # 移除开头的 '/' 以匹配 OSS 格式 prefix_exists = False for obj in oss2.ObjectIterator(self.oss_client, prefix=prefix): prefix_exists = True if obj.key != prefix: # 跳过目录本身 objects.append(obj.key) # print(f"obj.key={obj.key}") if not prefix_exists: print(f"前缀 '{prefix}' 下没有找到任何文件或目录。") return False else: print(f"前缀 '{prefix}' 存在,共找到 {len(objects)} 个对象。") # 下载所有文件,添加进度条 for obj_key in tqdm(objects, desc="下载进度"): if obj_key.endswith('/'): # print("下载 endswith('/'") continue if "printId" in obj_key: # print(f"下载 in obj_key") continue # 计算相对路径 rel_path = obj_key[len(prefix):].lstrip('/') file_dir, file_name = os.path.split(rel_path) file_base, file_ext = os.path.splitext(file_name) # 根据文件后缀名进行重命名 if file_ext.lower() in ['.mtl', '.jpg', '.jpeg', '.png']: # 对于.mtl和图片文件,在原名前加order_id new_file_name = f"{batch_model_info.order_id}_{file_name}" # new_file_name = file_name elif file_ext.lower() == '.obj': # 对于.obj文件,完全重命名 new_file_name = f"{batch_model_info.order_id}_{batch_model_info.pid}_P{batch_model_info.print_order_id}_{batch_model_info.model_size}{file_ext}" else: # 其他文件类型保持原名 new_file_name = file_name # 构建新的相对路径 if file_dir: # 如果有子目录 new_rel_path = os.path.join(file_dir, new_file_name) else: new_rel_path = new_file_name # 构建本地完整路径 local_path = os.path.join(self.local_path, new_rel_path) # 创建必要的目录 os.makedirs(os.path.dirname(local_path), exist_ok=True) # 下载文件 self.oss_client.get_object_to_file(obj_key, local_path) if file_ext == '.obj': # 10MB以上 try: # 使用临时文件避免内存问题 [8](@ref) temp_path = local_path + '.tmp' with open(local_path, 'r', encoding='utf-8') as f_in, \ open(temp_path, 'w', encoding='utf-8') as f_out: mtllib_modified = False for line in f_in: if not mtllib_modified and line.strip().startswith('mtllib '): parts = line.split(' ', 1) if len(parts) > 1: old_mtl_name = parts[1].strip() new_mtl_name = f"{batch_model_info.order_id}_{old_mtl_name}" f_out.write(f"mtllib {new_mtl_name}\n") mtllib_modified = True print("len(parts) > 1") continue f_out.write(line) os.replace(temp_path, local_path) # 原子性替换 except IOError as e: print(f"处理大文件 {local_path} 时出错: {e}") if os.path.exists(temp_path): os.remove(temp_path) # 优化后的.obj文件处理逻辑 if file_ext == '.mtl': try: # 使用更高效的文件读取方式 [6,8](@ref) with open(local_path, 'r', encoding='utf-8') as f: content = f.read() # 使用字符串方法直接查找和替换,避免不必要的循环 [9](@ref) lines = content.split('\n') mtllib_modified = False for i, line in enumerate(lines): stripped_line = line.strip() if not mtllib_modified and stripped_line.startswith('map_Kd '): # 更高效的分割方式 [9](@ref) parts = line.split(' ', 1) if len(parts) > 1: old_name = parts[1].strip() new_name = f"{batch_model_info.order_id}_{old_name}" lines[i] = f"map_Kd {new_name}" mtllib_modified = True # print(f"已更新材质库引用: {old_name} -> {new_name}") break # 找到第一个后立即退出 # 批量写入,减少I/O操作 [6](@ref) with open(local_path, 'w', encoding='utf-8') as f: f.write('\n'.join(lines)) except IOError as e: print(f"处理文件 {local_path} 时出错: {e}") except UnicodeDecodeError as e: print(f"文件编码错误 {local_path}: {e}") print(f"下载文件: {obj_key} -> {local_path}") return True def download_single_file(self): """ 下载单个文件从OSS到本地 """ # 确保本地目录存在 os.makedirs(os.path.dirname(self.local_path), exist_ok=True) # 直接下载文件 try: self.oss_client.get_object_to_file(self.oss_path, self.local_path) print(f"文件已下载到: {self.local_path}") except oss2.exceptions.NoSuchKey: print(f"OSS文件不存在: {self.oss_path}") def upload_data(self): ''' 上传数据到OSS ''' # 检测本地路径是否存在 if not os.path.exists(self.local_path): raise FileNotFoundError(f"本地路径不存在: {self.local_path}") # 判断本地路径是文件还是目录 if os.path.isfile(self.local_path): local_suffix = Path(self.local_path).suffix oss_suffix = Path(self.oss_path).suffix if oss_suffix and oss_suffix != local_suffix: # 后缀名不一致,上传到指定文件夹下的同名文件 oss_dir = os.path.dirname(self.oss_path) oss_target_path = os.path.join(oss_dir, os.path.basename(self.local_path)) else: # 后缀名一致,上传到指定OSS路径 oss_target_path = self.oss_path # 上传文件 self.oss_client.put_object_from_file(oss_target_path, self.local_path) print(f"文件已上传到: {oss_target_path}") elif os.path.isdir(self.local_path): oss_suffix = Path(self.oss_path).suffix if oss_suffix: raise ValueError("不能将目录上传到具有后缀名的OSS路径。") # 遍历本地目录并上传 for root, dirs, files in os.walk(self.local_path): for file in files: local_file_path = os.path.join(root, file) relative_path = os.path.relpath(local_file_path, self.local_path) oss_file_path = os.path.join(self.oss_path, relative_path).replace("\\", "/") # 创建必要的目录 oss_dir = os.path.dirname(oss_file_path) # 上传文件 self.oss_client.put_object_from_file(oss_file_path, local_file_path) print(f"文件已上传到: {oss_file_path}") else: raise ValueError(f"无效的本地路径类型: {self.local_path}") def get_api(url): try: response = requests.get(url) response.raise_for_status() # 检查请求是否成功 response = json.loads(response.text) if response.get("code") != 1000: raise Exception(f"Error fetching URL {url}: {response.get('message')}") else: return response except requests.exceptions.RequestException as e: raise Exception(f"Error fetching URL {url}: {e}") def download_data_by_json(model_info, workdir, oss_client ): try: pid = model_info.pid model_height = model_info.model_height target_dir = f"{workdir}" url = f"{url_get_oss_suffix_by_orderId}{model_info.order_id}" res = requests.get(url) data = res.json()["data"] # print("datas=",data) data = data.replace("/init_obj", "") print("target_dir=", target_dir) download_textures = DataTransfer(target_dir, f"objs/download/print/{pid}/{data}/{model_height}/", oss_client) download_textures.download_data_rename_json(model_info) # 下载后检查目标文件夹是否为空 if os.path.exists(target_dir) and not os.listdir(target_dir): shutil.rmtree(target_dir) print(f"下载后检查发现目标文件夹为空,已删除: {target_dir}") except Exception as e: print(f"下载失败: {pid}, 错误: {str(e)}") pass def download_datas_by_json(pid_file, workdir, oss_config): oss_client = get_oss_client(oss_config) json_path = os.path.join(workdir, f"{pid_file}.json") # 读取所有PID list_model_info, json_data = read_models_from_json(json_path) print(f"从文件读取了 {len(list_model_info)} 个PID") # 批量下载 for model_info in list_model_info: print(f"开始下载PID: {model_info}") download_data_by_json(model_info, args.workdir, oss_client) return json_data if __name__ == "__main__": script_args = [] try: separator_index = sys.argv.index("--") script_args = sys.argv[separator_index + 1:] except ValueError: pass parser = argparse.ArgumentParser() parser.add_argument("--pid_file", type=str, required=True, help="批次号, 也是json文件名") parser.add_argument("--workdir", type=str, required=True, help="json文件所在的目录") parser.add_argument("--oss_config", type=str, required=True, help="run.yaml所在的目录") args = parser.parse_args(script_args) download_transform_save_by_json(args.pid_file, args.workdir, args.oss_config) # blender --background --python download_print_out.py -- --pid_file {your_batch_id} --workdir {your_batch_dir} --oss_config {your_yaml_dir/run.yaml}