分布式任务调整

install.txt

@@ -2,7 +2,7 @@ pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple
 pip config set install.trusted-host pypi.tuna.tsinghua.edu.cn
 
 python -m pip install --upgrade pip
-pip install oss2 redis MySQLdb pillow numpy opencv-python bpy tqdm pyautogui psutil pywin32 pymysql
+pip install oss2 redis pillow numpy opencv-python bpy tqdm pyautogui psutil pywin32 pymysql
 
 
 config

libs/common.py

@@ -1,5 +1,6 @@
 import redis,sys,os,re
 import platform
+import xml.etree.ElementTree as ET
 if platform.system() == 'Windows':
     #sys.path.append('e:\\libs\\')
     sys.path.append('libs')
@@ -111,3 +112,32 @@ def change_rcbox_s(pid,new_value):
         #重新写入进去
         with open(rcbox_path, 'w') as f:
             f.write(new_content)
+
+#修改rcproj文件,删除没有模型的component,保留最多model 的component
+def changeRcprojFile(pid):
+    # 解析XML文件
+    file_path = os.path.join(config.workdir, pid, f'{pid}.rcproj')
+    #判断文件是否存在
+    if not os.path.exists(file_path):
+        return False
+    tree = ET.parse(file_path)
+    root = tree.getroot()
+    # 遍历所有的reconstructions节点
+    for reconstruction in root.findall('reconstructions'):
+        for component in reconstruction.findall('component'):
+            if component.find('model') == None:
+                reconstruction.remove(component)
+                continue
+        # 获取所有包含model标签的component节点
+        components_with_model = [component for component in reconstruction.findall('component') if component.find('model') is not None]
+        print(components_with_model)
+        # 如果包含model标签的component节点数量大于1，则按照model数量降序排序
+        if len(components_with_model) > 1:
+            components_with_model.sort(key=lambda x: len(x.findall('model')), reverse=False)
+
+        for i in range(len(components_with_model)-1):
+            reconstruction.remove(components_with_model[i])
+
+    # 保存修改后的XML文件
+    tree.write(file_path)
+    return True

logic/logic_main_service.py

@@ -22,29 +22,30 @@ def get_task_distributed():
             result = main_service_db.db_task_distributed("status = 1 order by priority desc,created_at asc limit 1 for update")
             if result:
                 #获取需要执行的步骤
-                next_step = need_run_stepx(result["id"])
-                if next_step == "no" or next_step == "error":
-                    print("获取需要执行的步骤 next_step",next_step)
-                    return next_step
+                #next_step = need_run_stepx(result["id"])
+                next_step_result = need_run_step_no_step1()
+                if next_step_result == "no" or next_step_result == "error":
+                    print("获取需要执行的步骤 next_step",next_step_result)
+                    return next_step_result
                 #非R11 R12 的主机在执行step2的时候,需要判断当前模型是否需要高精模或者photo3参与建模，如果是的话，该主机不执行这一步
-                if next_step == "step2":
-                    if common.task_need_high_model_or_photo3(result["task_key"]):
-                        print(f'模型{result["task_key"]}需要高精模或者photo3参与建模,该主机{hostname}不执行step2')
+                if next_step_result["run_step"] == "step2":
+                    if common.task_need_high_model_or_photo3(next_step_result["task_key"]):
+                        print(f'模型{next_step_result["task_key"]}需要高精模或者photo3参与建模,该主机{hostname}不执行step2')
                         return "no"
-                taskData = {"hostname":hostname,"run_step":next_step,"task_distributed_id":result["id"],"task_key":result["task_key"]}
-                flagRes = update_main_and_add_detail(taskData)
+                #taskData = next_step_result #{"hostname":hostname,"run_step":next_step,"task_distributed_id":result["id"],"task_key":result["task_key"]}
+                flagRes = update_main_and_add_detail(next_step_result)
                 if flagRes == "error":
                     print(f'出现错误,有可能是多个进程获取同一个任务了')
                     return "error"
-                print(f'任务ID-{taskData["task_key"]}- "执行{taskData["run_step"]}"')
-                return taskData
+                print(f'任务ID-{next_step_result["task_key"]}- "执行{next_step_result["run_step"]}"')
+                return next_step_result
             else:
                 return 'no_data'
         else:
             #R11 R12的主机 可以执行step1 2 3 的任务
             #如果R11 R12的主机目前没有正在执行step2,则优先处理step2,
             # print("次数",is_run_stepx_nums("step2"))
-            if is_run_stepx_nums("step2") < 2:
+            if is_run_stepx_nums("step2") < 0:
                 resultData = need_run_step2()
                 if resultData != "no": 
                     resultData["hostname"] = hostname
@@ -188,6 +189,27 @@ def need_run_step_no_step2():
         logging.error(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行need_run_step_no_step2()异常: {str(e)}")
         return 'error'
 
+#查询出哪些任务需要执行非step1
+def need_run_step_no_step1():
+    try:
+        result = main_service_db.db_task_distributed_list("finished_at is null order by priority desc,created_at asc for update")
+        #判断是否有值
+        if len(result) == 0:
+            return "no"
+        #遍历循环哪笔需要执行step2
+        for row in result:
+            #判断是否有正在执行的step2
+            xstep = need_run_stepx(row["id"])
+            #print("查询非step1的任务列表",xstep,row["id"])
+            if xstep == "step2" or xstep == "step3":
+                #没有正在执行的step1,则返回该任务
+                return {"hostname":hostname,"run_step":xstep,"task_distributed_id":row["id"],"task_key":row["task_key"]}
+        return "no"
+    except Exception as e:
+        print(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行need_run_step_no_step1()异常: {str(e)}")
+        logging.error(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行need_run_step_no_step1()异常: {str(e)}")
+        return 'error'
+            
 #更新主表和插入明细表的步骤
 def update_main_and_add_detail(data):
     if data["run_step"] == "step1":

main_service.py

@@ -8,14 +8,19 @@ if platform.system() == 'Windows':
     #本地测试
 else:
     sys.path.append('/data/deploy/make3d/make2/libs/')
-import main_service_db
+import main_service_db,config
 if __name__ == '__main__':
+     if len(sys.argv) == 1:
+        print(sys.argv[1])
+        os.system(f'python main_step1.py {sys.argv[1]}')
+    else:
         #循环值守
+        redisLocal = config.redis_local
         while True:
             data = logic_main_service.get_task_distributed()
             #判断data数据类型
             if isinstance(data, str):
-            print("没有可执行的任务 sleep 3s")
+                print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),',没有可执行的任务 sleep 3s')
                 time.sleep(3)
                 continue
             else:
@@ -26,7 +31,7 @@ if __name__ == '__main__':
                     # main_service_db.update_task_distributed_detail({"task_distributed_id":data["task_distributed_id"],"step":"step1","finished_at":time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())})
 
                     #生产线上用
-                main_step1.step1(data["task_key"], experience=False, makeloop=False,task_distributed_id=data['task_distributed_id'])
+                    main_step1.step1(data["task_key"], experience=False, makeloop=True,task_distributed_id=data['task_distributed_id'])
                 elif data["run_step"] == "step2":
                     # 本地测试分布运行的用
                     # time.sleep(15)
@@ -34,6 +39,9 @@ if __name__ == '__main__':
                     # main_service_db.update_task_distributed_detail({"task_distributed_id":data["task_distributed_id"],"step":"step2","finished_at":time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())})
 
                     #生产线上用
+                    #移除 redis 里的自动点击的队列
+                    redisLocal.lrem("model:auto_distance",0,data["task_key"]+"_0")
+                    redisLocal.lrem("model:auto_distance",0,data["task_key"]+"_1")
                     main_step2.step2(data["task_key"], data['task_distributed_id'])
                 elif data["run_step"] == "step3":
                     # 本地测试分布运行的用

main_step1.py

@@ -196,6 +196,8 @@ def step1(pid, experience=False, makeloop=True,task_distributed_id="",isNoColorT
     else:
         #分布式服务执行完后,需要更新任务状态,更新字表的finished_at字段
         main_service_db.update_task_distributed_detail({"task_distributed_id":task_distributed_id,"finished_at":time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())})
+        print("step1完成,休息6s")
+        time.sleep(4)
         return
         
 

main_step2.py

@@ -6,7 +6,7 @@ if platform.system() == 'Windows':
 else:
     sys.path.append('/data/deploy/make3d/make2/libs/')
 import config, libs, libs_db,common,main_service_db
-
+redisLocal = config.redis_local
 def load_model(pid):
     cmd = f'{config.rcbin} {config.r1["init"]} -load "{os.path.join(config.workdir, pid, f"{pid}.rcproj")}"'
     print(cmd)
@@ -39,27 +39,66 @@ def make3d(pid):
         cmd = shlex.split(cmd)
         res = subprocess.run(cmd)
         print(f'{time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())} pid: {pid} 定位点导入完成')
-
+        time.sleep(3)
         # defind_distance
         #死循环阻塞获取
         while True:
+            print("循环阻塞开始")
             time.sleep(3)
             #判断是否有 pid_1 的的值
-            if redisLocal.sismember('model:auto_distance', pid+"_1") == False:
+            print(pid+"_1")
+            if redisLocal.lpos('model:auto_distance',pid+"_1") == None:
                 continue
-            #如果存在就移除掉并退出死循环
-            redisLocal.lrem('model:auto_distance', 0, pid+"_0")
-            break
-
             shutil.move(os.path.join(config.workdir, pid, f'{pid}_wait.rcproj'), os.path.join(config.workdir, pid, f'{pid}.rcproj'))
             print(f'{time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())} pid: {pid} 定义定位点距离完成')
-        #2023-10-27为了解决老版本使用step1 的 重建区域框的问题，这里加入了 -set "sfmEnableCameraPrior=True" -align -set "sfmEnableCameraPrior=False" align 使相机的对齐线统一向下后，再进行重建区域的设置
+            #最后处理掉redis中的值
+            redisLocal.lrem('model:auto_distance', 0, pid+"_1")
+            break
+        print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),"循环阻塞结束,开始建模")
+        
+        #区域的设置 建模
         cmd = f'{config.rcbin} {config.r1["init"]} -load "{os.path.join(config.workdir, pid, f"{pid}.rcproj")}" -update \
             -set "sfmEnableCameraPrior=True" -align -set "sfmEnableCameraPrior=False" -align -setReconstructionRegion "{os.path.join(config.workdir, pid, f"{pid}.rcbox")}" \
-            -mvs -modelSelectMaximalConnectedComponent -modelInvertSelection -modelRemoveSelectedTriangles -closeHoles -clean -simplify {simplify_value} -smooth -unwrap -calculateTexture -renameModel {pid} -exportModel "{pid}" "{os.path.join(config.workdir, pid, "output", f"{pid}.obj")}" "d:\\make2\\config\\ModelExportParams102.xml" -quit'
+            -mvs -modelSelectMaximalConnectedComponent -modelInvertSelection -modelRemoveSelectedTriangles -closeHoles -clean -simplify {simplify_value} -smooth -unwrap -calculateTexture -renameModel {pid} -save "{os.path.join(config.workdir, pid, f"{pid}.rcproj")}" -quit'
         print(cmd)
         cmd = shlex.split(cmd)
         res = subprocess.run(cmd)
+        
+        time.sleep(3)
+        #修改rcproj文件
+        flag = common.changeRcprojFile(pid)
+        if flag == False:
+            print(f'{time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())} pid: {pid} rcproj文件不存在')
+            return
+        #保存在导出
+        
+        #创建指定文件夹
+        if not os.path.exists(os.path.join(config.workdir, pid, "output")):
+            os.makedirs(os.path.join(config.workdir, pid, "output"))
+        #执行导出
+        cmd = f'{config.rcbin} -load "{os.path.join(config.workdir, pid, f"{pid}.rcproj")}"\
+        -exportModel "{pid}" "{os.path.join(config.workdir, pid, "output", f"{pid}.obj")}" "d:\\make2\\config\\ModelExportParams102.xml" -quit'
+        print(cmd)
+        cmd = shlex.split(cmd)
+        res = subprocess.run(cmd)
+        
+        #阻塞判断是否导出完成
+        while True:
+            #判断 output 目录下是否存在 三个文件
+            files = os.listdir(os.path.join(config.workdir, pid, "output"))
+            if len(files) >= 3:
+                break
+        
+        
+        #2023-10-27为了解决老版本使用step1 的 重建区域框的问题，这里加入了 -set "sfmEnableCameraPrior=True" -align -set "sfmEnableCameraPrior=False" align 使相机的对齐线统一向下后，再进行重建区域的设置
+        # cmd = f'{config.rcbin} {config.r1["init"]} -load "{os.path.join(config.workdir, pid, f"{pid}.rcproj")}" -update \
+        #     -set "sfmEnableCameraPrior=True" -align -set "sfmEnableCameraPrior=False" -align -setReconstructionRegion "{os.path.join(config.workdir, pid, f"{pid}.rcbox")}" \
+        #     -mvs -modelSelectMaximalConnectedComponent -renameModel {pid}  -modelInvertSelection -modelRemoveSelectedTriangles -closeHoles -clean -simplify {simplify_value} -smooth -unwrap -calculateTexture  -save "{os.path.join(config.workdir, pid, f"{pid}.rcproj")}" -exportModel "{pid}" "{os.path.join(config.workdir, pid, "output", f"{pid}.obj")}" "d:\\make2\\config\\ModelExportParams102.xml" -quit'
+        # print(cmd)
+        # cmd = shlex.split(cmd)
+        # res = subprocess.run(cmd)
+        
+            
     else: # new version
         #判断是否要进行高精模
         if common.task_need_high_model(pid):
@@ -99,13 +138,13 @@ def step2(pid,task_distributed_id=""):
         os.system(f'python d:\\make2\\main_step3.py {pid}')
     else:
         #暂时 step2 step3 一起连续执行
+        print('step2 执行完,开始执行step3')
         os.system(f'python d:\\make2\\main_step3.py {pid}')
         main_service_db.update_task_distributed({"id":task_distributed_id,"status":2,"finished_at":time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())})
         #main_service_db.update_task_distributed_detail({"task_distributed_id":task_distributed_id,"finished_at":time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())})
-        return
+        #return
 
 def main(pid):
-    redisLocal = config.redis_local
     if pid == '0':
         while True:
             # 取本地mysql队列任务，完成第二步的建模任务

main_step3.py

@@ -61,6 +61,7 @@ def base_fix(pid):
     start_time = time.time()
     # 统一文件名规则
     def fix_filename(pid):
+        texture_filename_end = ""
         if os.path.exists(os.path.join(config.workdir, pid, 'output', f'{pid}.jpg')):
             print(f'{time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())} pid: {pid} 已经是最新文件名规则，无需处理')
             return
@@ -214,7 +215,8 @@ def step3(pid,task_distributed_id=""):
         
         #更新主表的status 和 finished_at
         main_service_db.update_task_distributed({"id":task_distributed_id,"status":2,"finished_at":time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())})
-        return
+        print("step3 已执行完成")
+        #return
 
 def main(pid):
     if pid == '0':

test.py

@@ -1,8 +1,9 @@
 import os, sys, time, shutil, subprocess, shlex, json
+import xml.etree.ElementTree as ET
 import platform
 if platform.system() == 'Windows':
-    #sys.path.append('e:\\libs\\')
-    sys.path.append('libs')
+    sys.path.append('e:\\libs\\')
+    #sys.path.append('libs')
 else:
     sys.path.append('/data/deploy/make3d/make2/libs/')
     
@@ -11,9 +12,26 @@ import config, libs, libs_db,common
 
 if __name__ == '__main__':
     
-    #common.copy_remote_directory("172.31.1.11","D:\\7831","E:\\")
-    # config.oss_bucket.delete_object(f'test/test_delete')
-    # #删除oss 上的文件夹里的内容
-    # object_list = oss2.ObjectIterator(config.oss_bucket, prefix='test/test_delete/')
-    # result = config.oss_bucket.batch_delete_objects([obj.key for obj in object_list])
-    print(libs_db.is_new_make_psid(1))
+    # 解析XML文件
+    pid = "112322"
+    file_path = os.path.join(config.workdir, pid, f'{pid}.rcproj')
+    tree = ET.parse(file_path)
+    root = tree.getroot()
+    # 遍历所有的reconstructions节点
+    for reconstruction in root.findall('reconstructions'):
+        for component in reconstruction.findall('component'):
+            if component.find('model') == None:
+                reconstruction.remove(component)
+                continue
+        # 获取所有包含model标签的component节点
+        components_with_model = [component for component in reconstruction.findall('component') if component.find('model') is not None]
+        print(components_with_model)
+        # 如果包含model标签的component节点数量大于1，则按照model数量降序排序
+        if len(components_with_model) > 1:
+            components_with_model.sort(key=lambda x: len(x.findall('model')), reverse=False)
+
+        for i in range(len(components_with_model)-1):
+            reconstruction.remove(components_with_model[i])
+
+    # 保存修改后的XML文件
+    tree.write(file_path)

timer/get_task_to_db.py

@@ -1,18 +1,18 @@
-import platform,sys,redis,time
+import platform,sys,redis,time,requests,json
 if platform.system() == 'Windows':
     sys.path.append('e:\\libs\\')
 else:
     sys.path.append('/data/deploy/make3d/make2/libs/')
 
 import config,libs,libs_db
-
+r = redis.Redis(host="106.14.158.208",password="kcV2000",port=6379,db=6)
 def getPSid(pid):
-    res = request.get("https://mp.api.suwa3d.com/api/customerP3dLog/photoStudio",params={"pid":pid})
+    res = requests.get("https://mp.api.suwa3d.com/api/customerP3dLog/photoStudio",params={"pid":pid})
     res = json.loads(res.text)
     return str(res['data'])
 
-def readTask(r,key):
-    print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())+"-读取队列-"+key)
+def readTask(key):
+    print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())+"-读取队列-"+'model:'+key)
     if r.llen('model:'+key) == 0:
         print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())+"-没有查询到任务在"+key+"队列中")
         return
@@ -23,25 +23,30 @@ def readTask(r,key):
         print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())+"-读取的PID为空")
         return
     pid = pid.decode('utf-8')
+    psid = getPSid(pid)
     #判断是否走新的建模系统
-    if libs_db.is_new_make_psid(pid) == False:
+    if libs_db.is_new_make_psid(psid) == False:
         #如果不是走新的建模系统就塞回原来的队列
         print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())+"-该任务不是走新的建模任务，塞回原来的队列-"+pid)
         r.rpush('model:'+key,pid)
         return
     #新的建模系统
-    psid = getPSid(pid)
+    #psid = getPSid(pid)
+    if key == "make10":
+        key = "make"
+    print("走新的建模系统插入",key,pid,psid)
     libs_db.add_task({"task_type":key,"task_key":pid,"psid":psid})
    
 #程序主入口
 if __name__ == '__main__':
-    r = redis.Redis(host="106.14.158.208",password="kcV2000",port=6379,db=6)
+    
+    #print(r.llen('model:make10'))
     #开启死循环
     while True:
         print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())+"-读取redis中的任务")
         # 3D相册建模队列 model:make_experience 
         # model:make  model:make10
-        readTask(r,'make10')
-        # time.sleep(2)
-        readTask(r,'make')
-        time.sleep(2)
+        readTask('make10')
+        time.sleep(10)
+        readTask('make')
+        time.sleep(10)

tools/auto_distance.py

@@ -5,8 +5,8 @@ if platform.system() == 'Windows':
     sys.path.append('e:\\libs\\')
 else:
     sys.path.append('/data/deploy/make3d/make2/libs/')
-import config, libs
-
+import config, libs,libs_db
+redisLocal = config.redis_local
 def find_and_maximize_window(window_title):
     windows = []
     win32gui.EnumWindows(lambda hwnd, windows: windows.append(hwnd), windows)
@@ -14,10 +14,13 @@ def find_and_maximize_window(window_title):
     for hwnd in windows:
         if win32gui.IsWindowVisible(hwnd):
             if window_title in win32gui.GetWindowText(hwnd):
-                print(f'found {window_title} hwnd:{hwnd}')
-                # win32gui.ShowWindow(hwnd, win32con.SW_MAXIMIZE)
-                win32gui.SetForegroundWindow(hwnd)
                 pid = win32gui.GetWindowText(hwnd).split('wait')[0].split(' ')[0].split('-')[0].split('*')[0].split('_')[0]
+                #判断是否已经存在了,已经存在就不在处理了
+                if redisLocal.lpos('model:auto_distance',pid+"_0") != None or redisLocal.lpos('model:auto_distance',pid+"_1") != None:  #pid+"_0" in result or pid+"_1" in result:
+                    return '0', 0, 0, 0, 0
+                win32gui.SetForegroundWindow(hwnd)
+                win32gui.ShowWindow(hwnd, win32con.SW_MAXIMIZE)
+                print(f'found {window_title} hwnd:{hwnd}')
                 left, top, right, bottom = win32gui.GetWindowRect(hwnd)
                 return pid, left, top, right, bottom
     return '0', 0, 0, 0, 0
@@ -119,9 +122,12 @@ def defind_distance(pid, left, top, right, bottom):
     # ag.click()
 
     get_defineDistances(pid, left, top, right, bottom)
-
+    time.sleep(2)
     ag.hotkey('ctrl', 's') # save project
+    print("点击保存按钮")
+    time.sleep(6)
     ag.hotkey('alt', 'f4') # close project
+    print("点击关闭按钮")
     print(f'{time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())} {pid} 定义定位点距离完成')
     
     #更新 r.lpush('model:auto_distance', pid+"_0") 
@@ -130,9 +136,10 @@ def defind_distance(pid, left, top, right, bottom):
     redisLocal.lpush('model:auto_distance', pid+"_1")
     #去掉pid+"_0"的队列
     redisLocal.lrem('model:auto_distance', 0, pid+"_0")
+    time.sleep(5)
 
 def main():
-    redisLocal = config.redis_local
+    
     while True:
         time.sleep(1)
         title = "wait"
@@ -140,11 +147,6 @@ def main():
         if pid == '0':
             pass
         else:
-            
-            #判断是否已经存在了,已经存在就不在处理了
-            if redisLocal.sismember('model:auto_distance', pid+"_0") or redisLocal.sismember('model:auto_distance', pid+"_1"):
-                continue
-
             redisLocal.lpush('model:auto_distance', pid+"_0")
             
             print(f'{time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())} 找到{pid}的定位点距离定义窗口，开始定位点距离定义...')

tools/gen_xmps.py

@@ -81,7 +81,7 @@ def main(pid, lock=False):
         {libs.get_defineDistances(psid)} -update -align -align {config.r2["setRegion"]} \
         {exportxmp} \
         -exportReconstructionRegion "{os.path.join(config.workdir, pid, f"{pid}.rcbox")}" \
-        -save "{os.path.join(config.workdir, pid, f"{pid}.rcproj")}" -quit'
+        -save "{os.path.join(config.workdir, pid, f"{pid}.rcproj")}"'
     print(cmd)
     cmd = shlex.split(cmd)
     res = subprocess.run(cmd)