10 changed files with 527 additions and 327 deletions
@ -0,0 +1,96 @@ |
|||||||
|
import redis,sys,os |
||||||
|
import platform |
||||||
|
if platform.system() == 'Windows': |
||||||
|
#sys.path.append('e:\\libs\\') |
||||||
|
sys.path.append('libs') |
||||||
|
else: |
||||||
|
sys.path.append('/data/deploy/make3d/make2/libs/') |
||||||
|
import config,libs |
||||||
|
|
||||||
|
#判断模型是需要高精模 或者是 需要photo3 参与 |
||||||
|
def task_need_high_model_or_photo3(pid): |
||||||
|
resHigh = task_need_high_model(pid) |
||||||
|
resPhoto3 = task_need_photo3(pid) |
||||||
|
if resHigh or resPhoto3: |
||||||
|
return True |
||||||
|
else: |
||||||
|
return False |
||||||
|
|
||||||
|
#判断是否需要高精模 |
||||||
|
def task_need_high_model(pid): |
||||||
|
redis_conn = config.redis_local_common |
||||||
|
#判断在redis中是否有高精模和 需要photo3 参与的 task |
||||||
|
if redis_conn.sismember("calculateHighModel",pid): |
||||||
|
return True |
||||||
|
#判断是否需要高精模 |
||||||
|
if redis_conn.sismember("calculateHighModel_no",pid): |
||||||
|
return False |
||||||
|
|
||||||
|
|
||||||
|
#判断是否需要高精模 |
||||||
|
if libs.get_ps_type(pid) == 1: |
||||||
|
if libs.aliyun_face(pid): |
||||||
|
#calulate_type = 'calculateHighModel' |
||||||
|
redis_conn.sadd("calculateHighModel",pid) |
||||||
|
return True |
||||||
|
else: |
||||||
|
redis_conn.sadd("calculateHighModel_no",pid) |
||||||
|
return False |
||||||
|
|
||||||
|
#判断是否需要photo3参与建模 |
||||||
|
def task_need_photo3(pid): |
||||||
|
redis_conn = config.redis_local_common |
||||||
|
#判断在redis中是否有高精模和 需要photo3 参与的 task |
||||||
|
if redis_conn.sismember("photo3",pid): |
||||||
|
return True |
||||||
|
#判断是否需要photo3参与建模 |
||||||
|
if redis_conn.sismember("photo3_no",pid): |
||||||
|
return False |
||||||
|
if os.path.exists(os.path.join(config.workdir, pid, 'photo3')): |
||||||
|
redis_conn.sadd("photo3",pid) |
||||||
|
return True |
||||||
|
else: |
||||||
|
redis_conn.sadd("photo3_no",pid) |
||||||
|
return False |
||||||
|
|
||||||
|
#拷贝远程主机上的指定目录到本地指定目录 |
||||||
|
def copy_remote_directory(remote_host, remote_path, local_path): |
||||||
|
# 建立 SSH 连接 |
||||||
|
ssh = paramiko.SSHClient() |
||||||
|
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) |
||||||
|
ssh.connect(remote_host, username='your_username', password='your_password') |
||||||
|
|
||||||
|
# 创建 SFTP 客户端 |
||||||
|
sftp = ssh.open_sftp() |
||||||
|
|
||||||
|
# 获取远程目录下的所有文件/子目录 |
||||||
|
file_list = sftp.listdir(remote_path) |
||||||
|
|
||||||
|
# 遍历远程目录中的每个文件/子目录 |
||||||
|
for file_name in file_list: |
||||||
|
remote_file_path = os.path.join(remote_path, file_name) |
||||||
|
local_file_path = os.path.join(local_path, file_name) |
||||||
|
|
||||||
|
# 判断当前项是文件还是目录 |
||||||
|
if sftp.stat(remote_file_path).st_isdir(): |
||||||
|
# 如果是目录,递归调用函数进行拷贝 |
||||||
|
os.makedirs(local_file_path, exist_ok=True) |
||||||
|
copy_remote_directory(remote_host, remote_file_path, local_file_path) |
||||||
|
else: |
||||||
|
# 如果是文件,直接拷贝到指定目录 |
||||||
|
sftp.get(remote_file_path, local_file_path) |
||||||
|
|
||||||
|
# 关闭 SFTP 客户端和 SSH 连接 |
||||||
|
sftp.close() |
||||||
|
ssh.close() |
||||||
|
|
||||||
|
#移除redis中的高精模和 需要photo3 参与的 task |
||||||
|
# def remove_redis_high_model_or_photo3(pid): |
||||||
|
# redis_conn = config.redis_local_common |
||||||
|
# redis_conn.srem("calculateHighModel",pid) |
||||||
|
# redis_conn.srem("photo3",pid) |
||||||
|
|
||||||
|
|
||||||
|
# if __name__ == '__main__': |
||||||
|
# redis_conn = config.redis_local_common |
||||||
|
# print(redis_conn.sismember("photo3_no","1")) |
||||||
@ -0,0 +1,188 @@ |
|||||||
|
# mysql数据库常用任务函数封装 |
||||||
|
import pymysql, socket, time |
||||||
|
import config |
||||||
|
import logging |
||||||
|
logging.basicConfig(filename='task_distributed_error.log', level=logging.ERROR) |
||||||
|
#公共连接库 |
||||||
|
def pymysqlAlias(): |
||||||
|
return pymysql.connect( |
||||||
|
host=config.mysql_local['host'], |
||||||
|
port=config.mysql_local['port'], |
||||||
|
user=config.mysql_local['user'], |
||||||
|
password=config.mysql_local['password'], |
||||||
|
db=config.mysql_local['db'], |
||||||
|
charset=config.mysql_local['charset'],) |
||||||
|
|
||||||
|
#查询 task_distributed |
||||||
|
def db_task_distributed(where): |
||||||
|
try: |
||||||
|
with pymysqlAlias() as conn: |
||||||
|
cursor = conn.cursor(pymysql.cursors.DictCursor) |
||||||
|
sql = 'select * from task_distributed where 1=1' |
||||||
|
if where: |
||||||
|
sql += f' and {where}' |
||||||
|
|
||||||
|
cursor.execute(sql) |
||||||
|
result = cursor.fetchone() |
||||||
|
# 关闭游标和连接 |
||||||
|
##cursor.close() |
||||||
|
#conn.close() |
||||||
|
return result |
||||||
|
except Exception as e: |
||||||
|
print(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行db_task_distributed()异常: {str(e)}") |
||||||
|
logging.error(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行db_task_distributed()异常: {str(e)}") |
||||||
|
return 'error' |
||||||
|
|
||||||
|
def db_task_distributed_list(where): |
||||||
|
try: |
||||||
|
with pymysqlAlias() as conn: |
||||||
|
cursor = conn.cursor(pymysql.cursors.DictCursor) |
||||||
|
sql = 'select * from task_distributed where 1=1' |
||||||
|
if where: |
||||||
|
sql += f' and {where}' |
||||||
|
|
||||||
|
cursor.execute(sql) |
||||||
|
result = cursor.fetchall() |
||||||
|
# 关闭游标和连接 |
||||||
|
##cursor.close() |
||||||
|
#conn.close() |
||||||
|
return result |
||||||
|
except Exception as e: |
||||||
|
print(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行db_task_distributed_list()异常: {str(e)}") |
||||||
|
logging.error(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行db_task_distributed_list()异常: {str(e)}") |
||||||
|
return 'error' |
||||||
|
|
||||||
|
#查询 task_distributed_detail |
||||||
|
def db_task_distributed_detail(where): |
||||||
|
try: |
||||||
|
with pymysqlAlias() as conn: |
||||||
|
cursor = conn.cursor(pymysql.cursors.DictCursor) |
||||||
|
sql = 'select * from task_distributed_detail where 1=1' |
||||||
|
if where: |
||||||
|
sql += f' and {where}' |
||||||
|
|
||||||
|
cursor.execute(sql) |
||||||
|
result = cursor.fetchone() |
||||||
|
# 关闭游标和连接 |
||||||
|
#cursor.close() |
||||||
|
#conn.close() |
||||||
|
return result |
||||||
|
except Exception as e: |
||||||
|
print(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行db_task_distributed_detail()异常: {str(e)}") |
||||||
|
logging.error(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行db_task_distributed_detail()异常: {str(e)}") |
||||||
|
return 'error' |
||||||
|
|
||||||
|
#查询指定条件下的数量 task_distributed_detail |
||||||
|
def db_task_distributed_detail_count(where): |
||||||
|
try: |
||||||
|
with pymysqlAlias() as conn: |
||||||
|
cursor = conn.cursor(pymysql.cursors.DictCursor) |
||||||
|
sql = 'select count(*) as nums from task_distributed_detail where 1=1' |
||||||
|
if where: |
||||||
|
sql += f' and {where}' |
||||||
|
|
||||||
|
cursor.execute(sql) |
||||||
|
result = cursor.fetchone() |
||||||
|
# 关闭游标和连接 |
||||||
|
#cursor.close() |
||||||
|
#conn.close() |
||||||
|
return result["nums"] |
||||||
|
except Exception as e: |
||||||
|
print(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行db_task_distributed_detail_count()异常: {str(e)}") |
||||||
|
logging.error(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行db_task_distributed_detail_count()异常: {str(e)}") |
||||||
|
return 'error' |
||||||
|
|
||||||
|
|
||||||
|
# 在task_distributed_detail插入明细步骤 |
||||||
|
def add_task_distributed_detail(data): |
||||||
|
try: |
||||||
|
with pymysqlAlias() as conn: |
||||||
|
cursor = conn.cursor() |
||||||
|
sql = f'insert into task_distributed_detail (task_distributed_id,step,hostname,started_at) values ("{data["task_distributed_id"]}", "{data["step"]}","{data["hostname"]}","{time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())}")' |
||||||
|
cursor.execute(sql) |
||||||
|
conn.commit() |
||||||
|
return "ok" |
||||||
|
except Exception as e: |
||||||
|
print(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行add_task_distributed_detail({data})异常: {str(e)}") |
||||||
|
logging.error(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行add_task_distributed_detail({data})异常: {str(e)}") |
||||||
|
return "error" |
||||||
|
|
||||||
|
# 更新 task_distributed 主表 |
||||||
|
def update_task_distributed(data): |
||||||
|
try: |
||||||
|
with pymysqlAlias() as conn: |
||||||
|
cursor = conn.cursor() |
||||||
|
|
||||||
|
sql = f'update task_distributed set ' |
||||||
|
#判断要更新哪些字段 |
||||||
|
if "status" in data: |
||||||
|
sql += f'status = "{data["status"]}",' |
||||||
|
|
||||||
|
if "hostname" in data: |
||||||
|
sql += f'hostname = "{data["hostname"]}",' |
||||||
|
|
||||||
|
if "step_last" in data: |
||||||
|
sql += f'step_last = "{data["step_last"]}",' |
||||||
|
|
||||||
|
if "priority" in data: |
||||||
|
sql += f'priority = "{data["priority"]}",' |
||||||
|
|
||||||
|
if "started_at" in data: |
||||||
|
sql += f'started_at = "{data["started_at"]}",' |
||||||
|
|
||||||
|
if "finished_at" in data: |
||||||
|
sql += f'finished_at = "{data["finished_at"]}",' |
||||||
|
|
||||||
|
|
||||||
|
#去掉 sql 最右边的逗号 |
||||||
|
sql = sql.rstrip(',') |
||||||
|
|
||||||
|
|
||||||
|
sql += f' where 1=1 ' |
||||||
|
#条件要放在最后面 |
||||||
|
if "id" in data: |
||||||
|
sql += f' and id = "{data["id"]}"' |
||||||
|
|
||||||
|
if "task_key" in data: |
||||||
|
sql += f' and task_type = "{data["task_key"]}" and status != 2' |
||||||
|
|
||||||
|
#sql = f'update task_distributed set status = "{data["status"]}",updated_at = "{now()}" where id = "{data["id"]}"' |
||||||
|
# print(f'sql: {sql}') |
||||||
|
cursor.execute(sql) |
||||||
|
conn.commit() |
||||||
|
except Exception as e: |
||||||
|
print(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行update_task_distributed({data})异常: {str(e)}") |
||||||
|
logging.error(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行update_task_distributed({data})异常: {str(e)}") |
||||||
|
return "error" |
||||||
|
|
||||||
|
# 更新 task_distributed_detail 主表 |
||||||
|
def update_task_distributed_detail(data): |
||||||
|
try: |
||||||
|
with pymysqlAlias() as conn: |
||||||
|
cursor = conn.cursor() |
||||||
|
|
||||||
|
sql = f'update task_distributed_detail set ' |
||||||
|
#判断要更新哪些字段 |
||||||
|
if "finished_at" in data: |
||||||
|
sql += f'finished_at = "{data["finished_at"]}"' |
||||||
|
|
||||||
|
if "step" in data: |
||||||
|
sql += f',step = "{data["step"]}"' |
||||||
|
|
||||||
|
if "hostname" in data: |
||||||
|
sql += f',hostname = "{data["hostname"]}"' |
||||||
|
|
||||||
|
#where 条件 |
||||||
|
sql += f' where 1=1 ' |
||||||
|
if "task_distributed_id" in data: |
||||||
|
sql += f' and task_distributed_id = "{data["task_distributed_id"]}"' |
||||||
|
|
||||||
|
if "step" in data: |
||||||
|
sql += f' and step = "{data["step"]}"' |
||||||
|
|
||||||
|
cursor.execute(sql) |
||||||
|
conn.commit() |
||||||
|
except Exception as e: |
||||||
|
print(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行update_task_distributed_detail({data})异常: {str(e)}") |
||||||
|
logging.error(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行update_task_distributed_detail({data})异常: {str(e)}") |
||||||
|
return "error" |
||||||
@ -0,0 +1,18 @@ |
|||||||
|
import os, sys, time, shutil, subprocess, shlex, json |
||||||
|
import platform |
||||||
|
if platform.system() == 'Windows': |
||||||
|
#sys.path.append('e:\\libs\\') |
||||||
|
sys.path.append('libs') |
||||||
|
else: |
||||||
|
sys.path.append('/data/deploy/make3d/make2/libs/') |
||||||
|
|
||||||
|
import config, libs, libs_db,common |
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__': |
||||||
|
|
||||||
|
common.copy_remote_directory("172.31.1.11","D:\\7831","E:\\") |
||||||
|
# config.oss_bucket.delete_object(f'test/test_delete') |
||||||
|
# #删除oss 上的文件夹里的内容 |
||||||
|
# object_list = oss2.ObjectIterator(config.oss_bucket, prefix='test/test_delete/') |
||||||
|
# result = config.oss_bucket.batch_delete_objects([obj.key for obj in object_list]) |
||||||
Loading…
Reference in new issue