数据库代码抽离，增加影棚ID

libs/common.py

@@ -0,0 +1,96 @@
+import redis,sys,os
+import platform
+if platform.system() == 'Windows':
+    #sys.path.append('e:\\libs\\')
+    sys.path.append('libs')
+else:
+    sys.path.append('/data/deploy/make3d/make2/libs/')
+import config,libs
+
+#判断模型是需要高精模 或者是 需要photo3 参与
+def task_need_high_model_or_photo3(pid):
+    resHigh = task_need_high_model(pid)
+    resPhoto3 = task_need_photo3(pid)
+    if resHigh or resPhoto3:
+        return True
+    else:
+        return False
+
+#判断是否需要高精模
+def task_need_high_model(pid):
+    redis_conn = config.redis_local_common
+    #判断在redis中是否有高精模和 需要photo3 参与的 task
+    if redis_conn.sismember("calculateHighModel",pid):
+        return True
+    #判断是否需要高精模
+    if redis_conn.sismember("calculateHighModel_no",pid):
+        return False
+
+
+    #判断是否需要高精模
+    if libs.get_ps_type(pid) == 1:
+        if libs.aliyun_face(pid):
+            #calulate_type = 'calculateHighModel'
+            redis_conn.sadd("calculateHighModel",pid)
+            return True
+        else:
+            redis_conn.sadd("calculateHighModel_no",pid)
+    return False
+
+#判断是否需要photo3参与建模
+def task_need_photo3(pid):
+    redis_conn = config.redis_local_common
+    #判断在redis中是否有高精模和 需要photo3 参与的 task
+    if redis_conn.sismember("photo3",pid):
+        return True
+    #判断是否需要photo3参与建模
+    if redis_conn.sismember("photo3_no",pid):
+        return False
+    if os.path.exists(os.path.join(config.workdir, pid, 'photo3')):
+        redis_conn.sadd("photo3",pid)
+        return True
+    else:
+        redis_conn.sadd("photo3_no",pid)
+    return False
+
+#拷贝远程主机上的指定目录到本地指定目录
+def copy_remote_directory(remote_host, remote_path, local_path):
+    # 建立 SSH 连接
+    ssh = paramiko.SSHClient()
+    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+    ssh.connect(remote_host, username='your_username', password='your_password')
+
+    # 创建 SFTP 客户端
+    sftp = ssh.open_sftp()
+
+    # 获取远程目录下的所有文件/子目录
+    file_list = sftp.listdir(remote_path)
+
+    # 遍历远程目录中的每个文件/子目录
+    for file_name in file_list:
+        remote_file_path = os.path.join(remote_path, file_name)
+        local_file_path = os.path.join(local_path, file_name)
+
+        # 判断当前项是文件还是目录
+        if sftp.stat(remote_file_path).st_isdir():
+            # 如果是目录，递归调用函数进行拷贝
+            os.makedirs(local_file_path, exist_ok=True)
+            copy_remote_directory(remote_host, remote_file_path, local_file_path)
+        else:
+            # 如果是文件，直接拷贝到指定目录
+            sftp.get(remote_file_path, local_file_path)
+
+    # 关闭 SFTP 客户端和 SSH 连接
+    sftp.close()
+    ssh.close()
+
+#移除redis中的高精模和 需要photo3 参与的 task
+# def remove_redis_high_model_or_photo3(pid):
+#     redis_conn = config.redis_local_common
+#     redis_conn.srem("calculateHighModel",pid)
+#     redis_conn.srem("photo3",pid)
+
+
+# if __name__ == '__main__':
+#     redis_conn = config.redis_local_common
+#     print(redis_conn.sismember("photo3_no","1"))

libs/config.py

@@ -127,4 +127,7 @@ task_run_timeout = {
         "step2":60*12,
         "step3":60*12,
     },
-}
+}
+
+high_host = ["R11","R12"]
+low_host  = ["R13","R14","R15","R16","R17"]

libs/main_service_db.py

@@ -0,0 +1,188 @@
+# mysql数据库常用任务函数封装
+import pymysql, socket, time
+import config
+import logging
+logging.basicConfig(filename='task_distributed_error.log', level=logging.ERROR)
+#公共连接库
+def pymysqlAlias():
+    return pymysql.connect(
+            host=config.mysql_local['host'],
+            port=config.mysql_local['port'],
+            user=config.mysql_local['user'],
+            password=config.mysql_local['password'],
+            db=config.mysql_local['db'],
+            charset=config.mysql_local['charset'],)
+
+#查询 task_distributed
+def db_task_distributed(where):
+    try:
+        with pymysqlAlias() as conn:
+                cursor = conn.cursor(pymysql.cursors.DictCursor)
+                sql = 'select * from task_distributed where 1=1'
+                if where:
+                    sql += f' and {where}'
+                
+                cursor.execute(sql)
+                result = cursor.fetchone()
+                # 关闭游标和连接
+                ##cursor.close()
+                #conn.close()
+                return result
+    except Exception as e:
+        print(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行db_task_distributed()异常: {str(e)}")
+        logging.error(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行db_task_distributed()异常: {str(e)}")
+        return 'error'
+
+def db_task_distributed_list(where):
+    try:
+        with pymysqlAlias() as conn:
+                cursor = conn.cursor(pymysql.cursors.DictCursor)
+                sql = 'select * from task_distributed where 1=1'
+                if where:
+                    sql += f' and {where}'
+                
+                cursor.execute(sql)
+                result = cursor.fetchall()
+                # 关闭游标和连接
+                ##cursor.close()
+                #conn.close()
+                return result
+    except Exception as e:
+        print(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行db_task_distributed_list()异常: {str(e)}")
+        logging.error(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行db_task_distributed_list()异常: {str(e)}")
+        return 'error'
+
+#查询 task_distributed_detail
+def db_task_distributed_detail(where):
+    try:
+        with pymysqlAlias() as conn:
+                cursor = conn.cursor(pymysql.cursors.DictCursor)
+                sql = 'select * from task_distributed_detail where 1=1'
+                if where:
+                    sql += f' and {where}'
+                
+                cursor.execute(sql)
+                result = cursor.fetchone()
+                # 关闭游标和连接
+                #cursor.close()
+                #conn.close()
+                return result
+    except Exception as e:
+        print(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行db_task_distributed_detail()异常: {str(e)}")
+        logging.error(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行db_task_distributed_detail()异常: {str(e)}")
+        return 'error'
+
+#查询指定条件下的数量 task_distributed_detail
+def db_task_distributed_detail_count(where):
+    try:
+        with pymysqlAlias() as conn:
+                cursor = conn.cursor(pymysql.cursors.DictCursor)
+                sql = 'select count(*) as nums from task_distributed_detail where 1=1'
+                if where:
+                    sql += f' and {where}'
+                
+                cursor.execute(sql)
+                result = cursor.fetchone()
+                # 关闭游标和连接
+                #cursor.close()
+                #conn.close()
+                return result["nums"]
+    except Exception as e:
+        print(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行db_task_distributed_detail_count()异常: {str(e)}")
+        logging.error(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行db_task_distributed_detail_count()异常: {str(e)}")
+        return 'error'
+
+
+# 在task_distributed_detail插入明细步骤
+def add_task_distributed_detail(data):
+    try:
+        with pymysqlAlias() as conn:
+            cursor = conn.cursor()
+            sql = f'insert into task_distributed_detail (task_distributed_id,step,hostname,started_at) values ("{data["task_distributed_id"]}", "{data["step"]}","{data["hostname"]}","{time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())}")'
+            cursor.execute(sql)
+            conn.commit()
+            return "ok"
+    except Exception as e:
+        print(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行add_task_distributed_detail({data})异常: {str(e)}")
+        logging.error(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行add_task_distributed_detail({data})异常: {str(e)}")
+        return "error"
+
+# 更新 task_distributed 主表
+def update_task_distributed(data):
+    try:
+        with pymysqlAlias() as conn:
+            cursor = conn.cursor()
+
+            sql = f'update task_distributed set '
+            #判断要更新哪些字段
+            if "status" in data:
+                sql += f'status = "{data["status"]}",'
+            
+            if "hostname" in data:
+                sql += f'hostname = "{data["hostname"]}",'
+            
+            if "step_last" in data:
+                sql += f'step_last = "{data["step_last"]}",'
+            
+            if "priority" in data:
+                sql += f'priority = "{data["priority"]}",'
+            
+            if "started_at" in data:
+                sql += f'started_at = "{data["started_at"]}",'
+
+            if "finished_at" in data:
+                sql += f'finished_at = "{data["finished_at"]}",'
+            
+
+            #去掉 sql 最右边的逗号
+            sql = sql.rstrip(',')
+
+
+            sql += f' where 1=1 '
+            #条件要放在最后面
+            if "id" in data:
+                sql += f' and id = "{data["id"]}"'
+            
+            if "task_key" in data:
+                sql += f' and task_type = "{data["task_key"]}" and status != 2'
+
+            #sql = f'update task_distributed set status = "{data["status"]}",updated_at = "{now()}" where id = "{data["id"]}"'
+            # print(f'sql: {sql}')
+            cursor.execute(sql)
+            conn.commit()
+    except Exception as e:
+        print(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行update_task_distributed({data})异常: {str(e)}")
+        logging.error(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行update_task_distributed({data})异常: {str(e)}")
+        return "error"
+
+# 更新 task_distributed_detail 主表
+def update_task_distributed_detail(data):
+    try:
+        with pymysqlAlias() as conn:
+            cursor = conn.cursor()
+
+            sql = f'update task_distributed_detail set '
+            #判断要更新哪些字段
+            if "finished_at" in data:
+                sql += f'finished_at = "{data["finished_at"]}"'
+            
+            if "step" in data:
+                sql += f',step = "{data["step"]}"'
+            
+            if "hostname" in data:
+                sql += f',hostname = "{data["hostname"]}"'
+            
+            #where 条件
+            sql += f' where 1=1 '
+            if "task_distributed_id" in data:
+                sql += f' and task_distributed_id = "{data["task_distributed_id"]}"'
+            
+            if "step" in data:
+                sql += f' and step = "{data["step"]}"'
+
+            cursor.execute(sql)
+            conn.commit()
+    except Exception as e:
+        print(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行update_task_distributed_detail({data})异常: {str(e)}")
+        logging.error(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行update_task_distributed_detail({data})异常: {str(e)}")
+        return "error"