10 changed files with 527 additions and 327 deletions
@ -0,0 +1,96 @@
@@ -0,0 +1,96 @@
|
||||
import redis,sys,os |
||||
import platform |
||||
if platform.system() == 'Windows': |
||||
#sys.path.append('e:\\libs\\') |
||||
sys.path.append('libs') |
||||
else: |
||||
sys.path.append('/data/deploy/make3d/make2/libs/') |
||||
import config,libs |
||||
|
||||
#判断模型是需要高精模 或者是 需要photo3 参与 |
||||
def task_need_high_model_or_photo3(pid): |
||||
resHigh = task_need_high_model(pid) |
||||
resPhoto3 = task_need_photo3(pid) |
||||
if resHigh or resPhoto3: |
||||
return True |
||||
else: |
||||
return False |
||||
|
||||
#判断是否需要高精模 |
||||
def task_need_high_model(pid): |
||||
redis_conn = config.redis_local_common |
||||
#判断在redis中是否有高精模和 需要photo3 参与的 task |
||||
if redis_conn.sismember("calculateHighModel",pid): |
||||
return True |
||||
#判断是否需要高精模 |
||||
if redis_conn.sismember("calculateHighModel_no",pid): |
||||
return False |
||||
|
||||
|
||||
#判断是否需要高精模 |
||||
if libs.get_ps_type(pid) == 1: |
||||
if libs.aliyun_face(pid): |
||||
#calulate_type = 'calculateHighModel' |
||||
redis_conn.sadd("calculateHighModel",pid) |
||||
return True |
||||
else: |
||||
redis_conn.sadd("calculateHighModel_no",pid) |
||||
return False |
||||
|
||||
#判断是否需要photo3参与建模 |
||||
def task_need_photo3(pid): |
||||
redis_conn = config.redis_local_common |
||||
#判断在redis中是否有高精模和 需要photo3 参与的 task |
||||
if redis_conn.sismember("photo3",pid): |
||||
return True |
||||
#判断是否需要photo3参与建模 |
||||
if redis_conn.sismember("photo3_no",pid): |
||||
return False |
||||
if os.path.exists(os.path.join(config.workdir, pid, 'photo3')): |
||||
redis_conn.sadd("photo3",pid) |
||||
return True |
||||
else: |
||||
redis_conn.sadd("photo3_no",pid) |
||||
return False |
||||
|
||||
#拷贝远程主机上的指定目录到本地指定目录 |
||||
def copy_remote_directory(remote_host, remote_path, local_path): |
||||
# 建立 SSH 连接 |
||||
ssh = paramiko.SSHClient() |
||||
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) |
||||
ssh.connect(remote_host, username='your_username', password='your_password') |
||||
|
||||
# 创建 SFTP 客户端 |
||||
sftp = ssh.open_sftp() |
||||
|
||||
# 获取远程目录下的所有文件/子目录 |
||||
file_list = sftp.listdir(remote_path) |
||||
|
||||
# 遍历远程目录中的每个文件/子目录 |
||||
for file_name in file_list: |
||||
remote_file_path = os.path.join(remote_path, file_name) |
||||
local_file_path = os.path.join(local_path, file_name) |
||||
|
||||
# 判断当前项是文件还是目录 |
||||
if sftp.stat(remote_file_path).st_isdir(): |
||||
# 如果是目录,递归调用函数进行拷贝 |
||||
os.makedirs(local_file_path, exist_ok=True) |
||||
copy_remote_directory(remote_host, remote_file_path, local_file_path) |
||||
else: |
||||
# 如果是文件,直接拷贝到指定目录 |
||||
sftp.get(remote_file_path, local_file_path) |
||||
|
||||
# 关闭 SFTP 客户端和 SSH 连接 |
||||
sftp.close() |
||||
ssh.close() |
||||
|
||||
#移除redis中的高精模和 需要photo3 参与的 task |
||||
# def remove_redis_high_model_or_photo3(pid): |
||||
# redis_conn = config.redis_local_common |
||||
# redis_conn.srem("calculateHighModel",pid) |
||||
# redis_conn.srem("photo3",pid) |
||||
|
||||
|
||||
# if __name__ == '__main__': |
||||
# redis_conn = config.redis_local_common |
||||
# print(redis_conn.sismember("photo3_no","1")) |
||||
@ -0,0 +1,188 @@
@@ -0,0 +1,188 @@
|
||||
# mysql数据库常用任务函数封装 |
||||
import pymysql, socket, time |
||||
import config |
||||
import logging |
||||
logging.basicConfig(filename='task_distributed_error.log', level=logging.ERROR) |
||||
#公共连接库 |
||||
def pymysqlAlias(): |
||||
return pymysql.connect( |
||||
host=config.mysql_local['host'], |
||||
port=config.mysql_local['port'], |
||||
user=config.mysql_local['user'], |
||||
password=config.mysql_local['password'], |
||||
db=config.mysql_local['db'], |
||||
charset=config.mysql_local['charset'],) |
||||
|
||||
#查询 task_distributed |
||||
def db_task_distributed(where): |
||||
try: |
||||
with pymysqlAlias() as conn: |
||||
cursor = conn.cursor(pymysql.cursors.DictCursor) |
||||
sql = 'select * from task_distributed where 1=1' |
||||
if where: |
||||
sql += f' and {where}' |
||||
|
||||
cursor.execute(sql) |
||||
result = cursor.fetchone() |
||||
# 关闭游标和连接 |
||||
##cursor.close() |
||||
#conn.close() |
||||
return result |
||||
except Exception as e: |
||||
print(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行db_task_distributed()异常: {str(e)}") |
||||
logging.error(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行db_task_distributed()异常: {str(e)}") |
||||
return 'error' |
||||
|
||||
def db_task_distributed_list(where): |
||||
try: |
||||
with pymysqlAlias() as conn: |
||||
cursor = conn.cursor(pymysql.cursors.DictCursor) |
||||
sql = 'select * from task_distributed where 1=1' |
||||
if where: |
||||
sql += f' and {where}' |
||||
|
||||
cursor.execute(sql) |
||||
result = cursor.fetchall() |
||||
# 关闭游标和连接 |
||||
##cursor.close() |
||||
#conn.close() |
||||
return result |
||||
except Exception as e: |
||||
print(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行db_task_distributed_list()异常: {str(e)}") |
||||
logging.error(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行db_task_distributed_list()异常: {str(e)}") |
||||
return 'error' |
||||
|
||||
#查询 task_distributed_detail |
||||
def db_task_distributed_detail(where): |
||||
try: |
||||
with pymysqlAlias() as conn: |
||||
cursor = conn.cursor(pymysql.cursors.DictCursor) |
||||
sql = 'select * from task_distributed_detail where 1=1' |
||||
if where: |
||||
sql += f' and {where}' |
||||
|
||||
cursor.execute(sql) |
||||
result = cursor.fetchone() |
||||
# 关闭游标和连接 |
||||
#cursor.close() |
||||
#conn.close() |
||||
return result |
||||
except Exception as e: |
||||
print(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行db_task_distributed_detail()异常: {str(e)}") |
||||
logging.error(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行db_task_distributed_detail()异常: {str(e)}") |
||||
return 'error' |
||||
|
||||
#查询指定条件下的数量 task_distributed_detail |
||||
def db_task_distributed_detail_count(where): |
||||
try: |
||||
with pymysqlAlias() as conn: |
||||
cursor = conn.cursor(pymysql.cursors.DictCursor) |
||||
sql = 'select count(*) as nums from task_distributed_detail where 1=1' |
||||
if where: |
||||
sql += f' and {where}' |
||||
|
||||
cursor.execute(sql) |
||||
result = cursor.fetchone() |
||||
# 关闭游标和连接 |
||||
#cursor.close() |
||||
#conn.close() |
||||
return result["nums"] |
||||
except Exception as e: |
||||
print(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行db_task_distributed_detail_count()异常: {str(e)}") |
||||
logging.error(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行db_task_distributed_detail_count()异常: {str(e)}") |
||||
return 'error' |
||||
|
||||
|
||||
# 在task_distributed_detail插入明细步骤 |
||||
def add_task_distributed_detail(data): |
||||
try: |
||||
with pymysqlAlias() as conn: |
||||
cursor = conn.cursor() |
||||
sql = f'insert into task_distributed_detail (task_distributed_id,step,hostname,started_at) values ("{data["task_distributed_id"]}", "{data["step"]}","{data["hostname"]}","{time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())}")' |
||||
cursor.execute(sql) |
||||
conn.commit() |
||||
return "ok" |
||||
except Exception as e: |
||||
print(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行add_task_distributed_detail({data})异常: {str(e)}") |
||||
logging.error(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行add_task_distributed_detail({data})异常: {str(e)}") |
||||
return "error" |
||||
|
||||
# 更新 task_distributed 主表 |
||||
def update_task_distributed(data): |
||||
try: |
||||
with pymysqlAlias() as conn: |
||||
cursor = conn.cursor() |
||||
|
||||
sql = f'update task_distributed set ' |
||||
#判断要更新哪些字段 |
||||
if "status" in data: |
||||
sql += f'status = "{data["status"]}",' |
||||
|
||||
if "hostname" in data: |
||||
sql += f'hostname = "{data["hostname"]}",' |
||||
|
||||
if "step_last" in data: |
||||
sql += f'step_last = "{data["step_last"]}",' |
||||
|
||||
if "priority" in data: |
||||
sql += f'priority = "{data["priority"]}",' |
||||
|
||||
if "started_at" in data: |
||||
sql += f'started_at = "{data["started_at"]}",' |
||||
|
||||
if "finished_at" in data: |
||||
sql += f'finished_at = "{data["finished_at"]}",' |
||||
|
||||
|
||||
#去掉 sql 最右边的逗号 |
||||
sql = sql.rstrip(',') |
||||
|
||||
|
||||
sql += f' where 1=1 ' |
||||
#条件要放在最后面 |
||||
if "id" in data: |
||||
sql += f' and id = "{data["id"]}"' |
||||
|
||||
if "task_key" in data: |
||||
sql += f' and task_type = "{data["task_key"]}" and status != 2' |
||||
|
||||
#sql = f'update task_distributed set status = "{data["status"]}",updated_at = "{now()}" where id = "{data["id"]}"' |
||||
# print(f'sql: {sql}') |
||||
cursor.execute(sql) |
||||
conn.commit() |
||||
except Exception as e: |
||||
print(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行update_task_distributed({data})异常: {str(e)}") |
||||
logging.error(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行update_task_distributed({data})异常: {str(e)}") |
||||
return "error" |
||||
|
||||
# 更新 task_distributed_detail 主表 |
||||
def update_task_distributed_detail(data): |
||||
try: |
||||
with pymysqlAlias() as conn: |
||||
cursor = conn.cursor() |
||||
|
||||
sql = f'update task_distributed_detail set ' |
||||
#判断要更新哪些字段 |
||||
if "finished_at" in data: |
||||
sql += f'finished_at = "{data["finished_at"]}"' |
||||
|
||||
if "step" in data: |
||||
sql += f',step = "{data["step"]}"' |
||||
|
||||
if "hostname" in data: |
||||
sql += f',hostname = "{data["hostname"]}"' |
||||
|
||||
#where 条件 |
||||
sql += f' where 1=1 ' |
||||
if "task_distributed_id" in data: |
||||
sql += f' and task_distributed_id = "{data["task_distributed_id"]}"' |
||||
|
||||
if "step" in data: |
||||
sql += f' and step = "{data["step"]}"' |
||||
|
||||
cursor.execute(sql) |
||||
conn.commit() |
||||
except Exception as e: |
||||
print(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行update_task_distributed_detail({data})异常: {str(e)}") |
||||
logging.error(f"{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())} 执行update_task_distributed_detail({data})异常: {str(e)}") |
||||
return "error" |
||||
@ -0,0 +1,18 @@
@@ -0,0 +1,18 @@
|
||||
import os, sys, time, shutil, subprocess, shlex, json |
||||
import platform |
||||
if platform.system() == 'Windows': |
||||
#sys.path.append('e:\\libs\\') |
||||
sys.path.append('libs') |
||||
else: |
||||
sys.path.append('/data/deploy/make3d/make2/libs/') |
||||
|
||||
import config, libs, libs_db,common |
||||
|
||||
|
||||
if __name__ == '__main__': |
||||
|
||||
common.copy_remote_directory("172.31.1.11","D:\\7831","E:\\") |
||||
# config.oss_bucket.delete_object(f'test/test_delete') |
||||
# #删除oss 上的文件夹里的内容 |
||||
# object_list = oss2.ObjectIterator(config.oss_bucket, prefix='test/test_delete/') |
||||
# result = config.oss_bucket.batch_delete_objects([obj.key for obj in object_list]) |
||||
Loading…
Reference in new issue