diff --git a/doc/install.txt b/doc/install.txt new file mode 100644 index 0000000..8cfb0f6 --- /dev/null +++ b/doc/install.txt @@ -0,0 +1,13 @@ +pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple +pip config set install.trusted-host pypi.tuna.tsinghua.edu.cn + +python -m pip install --upgrade pip +pip install oss2 redis pillow numpy opencv-python bpy tqdm pyautogui psutil pywin32 pymysql + + +config +set bin="C:\Program Files\Capturing Reality\RealityCapture\RealityCapture.exe" +%bin% -disableOnlineCommunication -setInstanceName %pid% +%bin% -disableOnlineCommunication -delegateTo %pid% +%bin% -set "appCacheLocation=ProjectFolder" + diff --git a/doc/step.text b/doc/step.text new file mode 100644 index 0000000..f68ca32 --- /dev/null +++ b/doc/step.text @@ -0,0 +1,5 @@ +old version + +init: + + '"C:\\Program Files\\Capturing Reality\\RealityCapture\\RealityCapture.exe" -disableOnlineCommunication -set \"sfmEnableCameraPrior=False\" -set \"sfmMaxFeaturesPerMpx=20000\" -set \"sfmMaxFeaturesPerImage=200000\" -set \"sfmImagesOverlap=High\" -set \"sfmMaxFeatureReprojectionError=1\" -addFolder "D:\\{pid}\\photo1" -addFolder "D:\\{pid}\\photo2" -importControlPointsMeasurements "D:\\{pid}\\{pid}..controlPoints.csv" -align -save' \ No newline at end of file diff --git a/libs/common.py b/libs/common.py index 2b4d052..e5094d1 100644 --- a/libs/common.py +++ b/libs/common.py @@ -1,4 +1,4 @@ -import redis,sys,os,re +import redis,sys,os,re,oss2 import platform import xml.etree.ElementTree as ET if platform.system() == 'Windows': @@ -140,4 +140,58 @@ def changeRcprojFile(pid): # 保存修改后的XML文件 tree.write(file_path) - return True \ No newline at end of file + return True +#修改 rcproj 文件中的 controlpoints 文件的引用 +def changeRcprojControlpointsFile(pid): + psid = libs.getPSid(pid) + # 解析XML文件 + file_path = os.path.join(config.workdir, str(pid), f'{pid}.rcproj') + #判断文件是否存在 + if not os.path.exists(file_path): + return False + #下载指定的psid的 points文件到本地 + flag = down_points_from_oss(pid,psid) + if flag == False: + return False + tree = ET.parse(file_path) + root = tree.getroot() + # 遍历所有的reconstructions节点 + for controlpoints in root.findall('controlpoints'): + #修改 controlpoints 标签内容里 双引号的内容 + controlpoints.set('fileName',f'controlpoints_{psid}.dat') + # 保存修改后的XML文件 + tree.write(file_path) + return True + + +def down_points_from_oss(pid,psid): + # 根据前缀获取文件列表 + prefix = f'points/{psid}/' + filelist = oss2.ObjectIteratorV2(config.oss_bucket, prefix=prefix) + flag = False + for file in filelist: + filename = file.key.split('/')[-1] + if filename.endswith('.dat'): + # print('正在下载:', file.key) + localfile = os.path.join(config.workdir,str(pid), filename) + config.oss_bucket.get_object_to_file(file.key, localfile) + flag = True + return flag + +#判断oss上是否存在指定的controlpoints文件 +def isExistControlPointsOss(pid): + psid = libs.getPSid(pid) + filePath = f'points/{psid}/controlpoints_{psid}.dat' + #判断oss上是否存在 + if config.oss_bucket.object_exists(filePath): + return True + else: + return False + +#将本地的controlpoints文件上传到oss上 +def uploadControlPointsOss(pid): + psid = libs.getPSid(pid) + filePath = f'points/{psid}/controlpoints_{psid}.dat' + localfile = os.path.join(config.workdir,str(pid), f'{str(pid)}_wait/controlpoints0.dat') + #进行上传 + config.oss_bucket.put_object_from_file(filePath, localfile) diff --git a/libs/libs_db.py b/libs/libs_db.py index 59572c7..7b96304 100644 --- a/libs/libs_db.py +++ b/libs/libs_db.py @@ -39,9 +39,6 @@ def add_task(data): #判断是否是昆山教学的,是的话优先级设置为默认 if data["psid"] == '85': sql = f'insert into tasks (task_type, task_key,studio_id) values ("{data["task_type"]}", "{data["task_key"]}","{data["psid"]}")' - elif data["psid"] == '1': - #实验室的订单走分布式处理 - sql = f'insert into task_distributed (task_type, task_key, priority,created_at,studio_id) values ("{data["task_type"]}", "{data["task_key"]}", 1,"{time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())}","{data["psid"]}")' else: sql = f'insert into tasks (task_type, task_key, priority,studio_id) values ("{data["task_type"]}", "{data["task_key"]}", 1,"{data["psid"]}")' # print(f'sql: {sql}') @@ -50,6 +47,18 @@ def add_task(data): except Exception as e: print(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行add_task({data})异常: {str(e)}") +#新增任务到分布式处理任务表中 {"task_type":key,"task_key":pid,"psid":psid} +def add_task_distributed(data): + try: + with pymysqlAlias() as conn: + cursor = conn.cursor() + sql = f'insert into task_distributed (task_type, task_key, priority,created_at,studio_id) values ("{data["task_type"]}", "{data["task_key"]}", "{data["priority"]}","{time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())}","{data["psid"]}")' + cursor.execute(sql) + conn.commit() + except Exception as e: + print(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行add_task_distributed({data})异常: {str(e)}") + + # 开始任务 def start_task(data): try: @@ -130,6 +139,23 @@ def change_to_new_make_psid(psid): cursor.execute(sql) conn.commit() print(f'修改影棚{psid}为走新的建模系统') + except Exception as e: + print(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行change_to_new_make_psid()异常: {str(e)}") + return "error" + + +#判断影棚走分布式的建模系统 还是走原来的建模系统 +def isStudioConfigDistribute(psid): + try: + with pymysqlAlias() as conn: + cursor = conn.cursor() + sql = f'select count(*) from studio_config_distribute where studio_id = {psid}' + cursor.execute(sql) + result = cursor.fetchone() + if result[0] == 0: + return False + else: + return True except Exception as e: print(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行change_to_new_make_psid()异常: {str(e)}") return "error" \ No newline at end of file diff --git a/main_step1.py b/main_step1.py index c6ea175..17e39cc 100644 --- a/main_step1.py +++ b/main_step1.py @@ -74,6 +74,7 @@ def detect_markers(psid, pid): print(cmd) cmd = shlex.split(cmd) res = subprocess.run(cmd) + time.sleep(3) fix_region() print(f'{time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())} pid: {pid} 定位点检测完成, 共费时{libs.diff_time(start_time)}') diff --git a/main_step2.py b/main_step2.py index 47fcde8..603dd33 100644 --- a/main_step2.py +++ b/main_step2.py @@ -27,37 +27,67 @@ def make3d(pid): simplify_value = 1000000 * libs.getHeadCount(pid) add_photo3 = ' ' - if common.task_need_photo3(pid): + #判断是否存在photo3 + if os.path.exists(os.path.join(config.workdir, pid, 'photo3')): add_photo3 = ' -addFolder "' + os.path.join(config.workdir, pid, 'photo3') + '" -align -align ' if get_rcver() == 1: # old version + + #判断oss 上是否存在 controlpoints 文件 + isExistPoint = common.isExistControlPointsOss(pid) cmd = f'{config.rcbin} {config.r1["init"]} \ - -addFolder "{os.path.join(config.workdir, pid, "photo1")}" -addFolder "{os.path.join(config.workdir, pid, "photo2")}" {add_photo3} \ - -importControlPointsMeasurements "{os.path.join(config.workdir, pid, f"{pid}.controlPoints.csv")}" \ - -align -save "{os.path.join(config.workdir, pid, f"{pid}_wait.rcproj")}"' + -addFolder "{os.path.join(config.workdir, pid, "photo1")}" -addFolder "{os.path.join(config.workdir, pid, "photo2")}" {add_photo3} \ + -importControlPointsMeasurements "{os.path.join(config.workdir, pid, f"{pid}.controlPoints.csv")}" \ + -align -save' + + #不存在point文件的时候就要用自动点击的方式 + if isExistPoint == False: + cmd = cmd + f' "{os.path.join(config.workdir, pid, f"{pid}_wait.rcproj")}"' + else: + cmd = cmd + f' "{os.path.join(config.workdir, pid, f"{pid}.rcproj")}" -quit' print(cmd) cmd = shlex.split(cmd) res = subprocess.run(cmd) - print(f'{time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())} pid: {pid} 定位点导入完成') - time.sleep(3) - # defind_distance - #死循环阻塞获取 - while True: - print("循环阻塞开始") + time.sleep(2) + + if isExistPoint == False: + print(f'{time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())} pid: {pid} 定位点导入完成') time.sleep(3) - #判断是否有 pid_1 的的值 - print(pid+"_1") - if redisLocal.lpos('model:auto_distance',pid+"_1") == None: - continue - shutil.move(os.path.join(config.workdir, pid, f'{pid}_wait.rcproj'), os.path.join(config.workdir, pid, f'{pid}.rcproj')) - print(f'{time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())} pid: {pid} 定义定位点距离完成') - #最后处理掉redis中的值 - redisLocal.lrem('model:auto_distance', 0, pid+"_1") - break - print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),"循环阻塞结束,开始建模") + # defind_distance + #死循环阻塞获取 + while True: + print("循环阻塞开始") + time.sleep(3) + #判断是否有 pid_1 的的值 + print(pid+"_1") + if redisLocal.lpos('model:auto_distance',pid+"_1") == None: + continue + shutil.move(os.path.join(config.workdir, pid, f'{pid}_wait.rcproj'), os.path.join(config.workdir, pid, f'{pid}.rcproj')) + print(f'{time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())} pid: {pid} 定义定位点距离完成') + #将 controlpoints_0.dat 文件拷贝到 oss 上作为公共的使用 + common.uploadControlPointsOss(pid) + + #最后处理掉redis中的值 + redisLocal.lrem('model:auto_distance', 0, pid+"_1") + break + print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),"循环阻塞结束,开始建模") + else: + #修改rcproj文件 controlpoints fileName 的引入 + flag = common.changeRcprojControlpointsFile(pid) + if flag == False: + print(f'{time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())} pid: {pid} 获取及修改controlpoints失败') + return + + print("使用公共point文件的方式进行建模") #区域的设置 建模 - cmd = f'{config.rcbin} {config.r1["init"]} -load "{os.path.join(config.workdir, pid, f"{pid}.rcproj")}" -update \ + #update + cmdSmall = "-align" + #使用公共point文件的时候,就不能使用update , 而是要用 align + if isExistPoint == True: + cmdSmall = "-align" + #{config.r1["init"]} + cmd = f'{config.rcbin} -load "{os.path.join(config.workdir, pid, f"{pid}.rcproj")}" {cmdSmall} \ -set "sfmEnableCameraPrior=True" -align -set "sfmEnableCameraPrior=False" -align -setReconstructionRegion "{os.path.join(config.workdir, pid, f"{pid}.rcbox")}" \ -mvs -modelSelectMaximalConnectedComponent -modelInvertSelection -modelRemoveSelectedTriangles -closeHoles -clean -simplify {simplify_value} -smooth -unwrap -calculateTexture -renameModel {pid} -save "{os.path.join(config.workdir, pid, f"{pid}.rcproj")}" -quit' print(cmd) @@ -141,7 +171,7 @@ def step2(pid,task_distributed_id=""): print('step2 执行完,开始执行step3') os.system(f'python d:\\make2\\main_step3.py {pid}') main_service_db.update_task_distributed({"id":task_distributed_id,"status":2,"finished_at":time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())}) - #main_service_db.update_task_distributed_detail({"task_distributed_id":task_distributed_id,"finished_at":time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())}) + main_service_db.update_task_distributed_detail({"task_distributed_id":task_distributed_id,"finished_at":time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())}) #return def main(pid): diff --git a/old_step2.py b/old_step2.py new file mode 100644 index 0000000..cc75b34 --- /dev/null +++ b/old_step2.py @@ -0,0 +1,68 @@ +import os, sys, time, shutil, subprocess, shlex, json +import xml.etree.ElementTree as ET +import platform +if platform.system() == 'Windows': + sys.path.append('e:\\libs\\') + #sys.path.append('libs') +else: + sys.path.append('/data/deploy/make3d/make2/libs/') + +import config, libs, libs_db,common + + +def old_step2(pid): + cmd = f'"C:\\Program Files\\Capturing Reality\\RealityCapture\\RealityCapture.exe" -disableOnlineCommunication -set \"sfmEnableCameraPrior=False\" -set \"sfmMaxFeaturesPerMpx=20000\" -set \"sfmMaxFeaturesPerImage=200000\" -set \"sfmImagesOverlap=High\" -set \"sfmMaxFeatureReprojectionError=1\" -addFolder "D:\\{pid}\\photo1" -addFolder "D:\\{pid}\\photo2" -importControlPointsMeasurements "D:\\{pid}\\{pid}.controlPoints.csv" -align -save "D:\\{pid}\\{pid}.rcproj" -quit' + + cmd = shlex.split(cmd) + res = subprocess.run(cmd) + time.sleep(2) + + #修改rcproj文件 + flag = common.changeRcprojControlpointsFile(pid) + if flag == False: + print(f'{time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())} pid: {pid} point文件不存在') + exit() + + simplify_value = 1000000 * libs.getHeadCount(pid) + + cmd = f'"C:\\Program Files\\Capturing Reality\\RealityCapture\\RealityCapture.exe" -load "D:\\{pid}\\{pid}.rcproj" -set "sfmEnableCameraPrior=True" -align -set "sfmEnableCameraPrior=False" -align -setReconstructionRegion "D:\\{pid}\\{pid}.rcbox" -mvs -modelSelectMaximalConnectedComponent -modelInvertSelection -modelRemoveSelectedTriangles -closeHoles -clean -simplify {simplify_value} -smooth -unwrap -calculateTexture -renameModel {pid} -save "{os.path.join(config.workdir, pid, f"{pid}.rcproj")}" -quit' + + cmd = shlex.split(cmd) + res = subprocess.run(cmd) + + time.sleep(3) + #修改rcproj文件 + flag = common.changeRcprojFile(pid) + if flag == False: + print(f'{time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())} pid: {pid} rcproj文件不存在') + exit() + + #创建指定文件夹 + if not os.path.exists(os.path.join(config.workdir, pid, "output")): + os.makedirs(os.path.join(config.workdir, pid, "output")) + #执行导出 + cmd = f'{config.rcbin} -load "{os.path.join(config.workdir, pid, f"{pid}.rcproj")}"\ + -exportModel "{pid}" "{os.path.join(config.workdir, pid, "output", f"{pid}.obj")}" "d:\\make2\\config\\ModelExportParams102.xml" -quit' + print(cmd) + cmd = shlex.split(cmd) + res = subprocess.run(cmd) + + #阻塞判断是否导出完成 + while True: + #判断 output 目录下是否存在 三个文件 + files = os.listdir(os.path.join(config.workdir, pid, "output")) + if len(files) >= 3: + break + #暂时 step2 step3 一起连续执行 + print('step2 执行完,开始执行step3') + os.system(f'python d:\\make2\\main_step3.py {pid}') + +if __name__ == '__main__': + pid = '0' + if len(sys.argv) == 1: + exit() + else: + pids = sys.argv[1].split(',') + for pid in pids: + old_step2(str(pid)) + exit() diff --git a/timer/get_task_to_db.py b/timer/get_task_to_db.py index 98939c0..66ddb01 100644 --- a/timer/get_task_to_db.py +++ b/timer/get_task_to_db.py @@ -34,8 +34,15 @@ def readTask(key): #psid = getPSid(pid) if key == "make10": key = "make" - print("走新的建模系统插入",key,pid,psid) - libs_db.add_task({"task_type":key,"task_key":pid,"psid":psid}) + + #默认走原来的数据表tasks,现在要切入一些数据过来走分布式建模系统 + taskData = {"task_type":key,"task_key":pid,"psid":psid,"priority":1} + if libs_db.isStudioConfigDistribute(psid): + print("走分布式建模系统插入",key,pid,psid) + libs_db.add_task_distributed(taskData) + else: + print("走新的建模系统不是分布式插入",key,pid,psid) + libs_db.add_task(taskData) #程序主入口 if __name__ == '__main__': diff --git a/tools/push_cmd.py b/tools/push_cmd.py index b694df1..07b4413 100644 --- a/tools/push_cmd.py +++ b/tools/push_cmd.py @@ -1,6 +1,10 @@ -import redis, os, sys,requests -sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -import config, libs +import redis, os, sys,requests,platform +if platform.system() == 'Windows': + sys.path.append('e:\\libs\\') + #sys.path.append('libs') +else: + sys.path.append('/data/deploy/make3d/make2/libs/') +import config, libs,libs_db def main(cmd, order_id): if cmd == 'print': #order_id @@ -25,6 +29,23 @@ def main(cmd, order_id): print(f'已推送{order_id}到{key}, 当前队列长度:{r.llen(key)}') def cmd(cmdName,pid): + if pid == "view": + key = "" + if cmdName == 'print': #order_id + key = 'model:printOrder' + elif cmdName == 'repair': #order_id + key = 'model:IndependentRepairTeamcheckGLBQueue' + elif cmdName == 'make3d': #pid + key = 'model:make' + elif cmdName == 'make3d10': #pid + key = 'model:make10' + elif cmdName == 'foot': # print_id + key = 'model:foot' + for i in r.lrange(key, 0, -1): + print(i) + print(f'当前{key}队列长度:{r.llen(key)}') + exit() + res = requests.get(f'https://mp.api.suwa3d.com/api/infoQuery/infoByPid?pid={pid}&cmd={cmdName}') #获取res 的数据 res = res.json() @@ -32,6 +53,7 @@ def cmd(cmdName,pid): print("查询失败",res) exit(1) data = res["data"] + if cmdName == 'print' or cmdName == 'repair': #需要去吃查询pid 对应的order_id #遍历data @@ -40,7 +62,7 @@ def cmd(cmdName,pid): tempData = data[i] if len(tempData) == 1: #pid 只有对应的一笔数据就可以直接插入了 - main(cmdName, tempData[0]["order_id"]) + main(cmdName, str(tempData[0]["order_id"])) else: while True: try: @@ -49,7 +71,7 @@ def cmd(cmdName,pid): order_id = int(input("pid-"+i+"对应有多个数据,请从下列中选择正确的order_id填入\r\n"+tempDataStrings+"\r\n->")) # 在这里处理用户输入的整数 print("输入的是: ", order_id) - main(cmdName, order_id) + main(cmdName, str(order_id)) break except ValueError: print("输入不满足要求,请重新输入") @@ -61,8 +83,9 @@ def cmd(cmdName,pid): #获取 pid 对应的数据 tempData = data[i] if len(tempData) == 1: + print("tempData",tempData) #pid 只有对应的一笔数据就可以直接插入了 - main(cmdName, tempData[0]["print_id"]) + main(cmdName, str(tempData[0]["print_id"])) else: while True: try: @@ -71,7 +94,7 @@ def cmd(cmdName,pid): printId = int(input("pid-"+i+"对应有多个数据,请从下列中选择正确的print_id填入\r\n"+tempDataStrings+"\r\n->")) # 在这里处理用户输入的整数 print("输入的是: ", printId) - main(cmdName, printId) + main(cmdName, str(printId)) break except ValueError: print("输入不满足要求,请重新输入") @@ -95,8 +118,8 @@ if __name__ == '__main__': else: print('用法:python push_cmd.py ') exit(1) - r = config.redis_local - + r = config.redis_remote + #pid 可能是多个的,用逗号分隔,查询的时候接口应该也要支持多个的 if cmd == 'make3d' or cmd == 'make3d10': main(cmdName, pid)