make2/apps/process_photos.py

#!/usr/bin/python

import os, time, redis, piexif, oss2
from threading import Thread
from PIL import Image
from ftplib import FTP

def get_maxnum(path):
    maxnum = 0
    for file in os.listdir(path):
        if len(file.split('_')) > 1:
            num = int(file.split('_')[-1].split('.')[0])
        else:
            num = 1
        
        if num > maxnum:
            maxnum = num
    print('最大文件序号: ' + str(maxnum))
    return maxnum

def fix_exif(exif_dict):
    exif_dict['Exif'][piexif.ExifIFD.MakerNote] = b''
    exif_dict['Exif'][piexif.ExifIFD.UserComment] = b''
    # exif_dict['0th'][piexif.ImageIFD.Make] = 'Huawei'
    # exif_dict['0th'][piexif.ImageIFD.Model] = 'Nexus 6P'
    # exif_dict['0th'][piexif.ImageIFD.Software] = 'sux V7'
    exif_dict['0th'][piexif.ImageIFD.ImageWidth] = 4056
    exif_dict['0th'][piexif.ImageIFD.ImageLength] = 3040
    if exif_dict['Exif'].get(piexif.ExifIFD.DigitalZoomRatio, 'havenothiskey') != 'havenothiskey':
        del(exif_dict['Exif'][piexif.ExifIFD.DigitalZoomRatio])
    if exif_dict['Exif'].get(piexif.ExifIFD.FocalLengthIn35mmFilm, 'havenothiskey') != 'havenothiskey':
        del(exif_dict['Exif'][piexif.ExifIFD.FocalLengthIn35mmFilm])
    if not exif_dict['0th'].get(piexif.ImageIFD.Orientation) is None:
        del(exif_dict['0th'][piexif.ImageIFD.Orientation])

    exif_dict['GPS'] = {}
    exif_bytes = piexif.dump(exif_dict)
    return exif_bytes

def orientation_optimize(pathname, filename, pid, thread_num, ftp_client, orientation=180, optimize=80):
    try:
        img = Image.open(pathname)
        exif = piexif.load(img.info['exif'])
        exif_bytes = fix_exif(exif)

        device_id = filename.split('_')[0]
        index = filename.split('_')[-1].split('.')[0]

        newfile = pathname.split('_')[0] + '_' + pathname.split('_')[-1]

        img.rotate(orientation).save(newfile, optimize=True, quality=optimize, exif=exif_bytes)
        print('Thread ' + thread_num + ': 照片预处理: ' + pathname + ' -> ' + newfile + ' 待处理队列：' + str(r.llen('photos_queue')))
        os.remove(pathname)

        for method in upload_methods:
            if method == 'oss':
                if index == '1':
                    photo = 'photos/' + pid + '/photo1/' + device_id + '_' + index + filename[-4:]
                else:
                    photo = 'photos/' + pid + '/photo2/' + device_id + '_' + index + filename[-4:]
                oss_client.put_object_from_file(photo, newfile)
            elif method == 'ftp':
                ftp_client.storbinary('STOR ' + filename, open(newfile, 'rb'))
            elif method == 'ssh':
                os.system('scp ' + newfile + ' ' + ssh_host + ':Downloads/' + pid + '/')
            else:
                pass

        # os.remove(newfile)

    except Exception as e:
        print(e)
        # 如果文件存在删除它
        if os.path.exists(pathname): os.remove(pathname)

def process_photos(thread_num):
    while True:
        pathname = r.lpop('photos_queue')
        if pathname is None:
            time.sleep(1)
            continue
        pathname = pathname.decode()
        filename = pathname.split('/')[-1]
        pid = pathname.split('/')[-2]
        if pid not in ds:
            for method in upload_methods:
                if method == 'oss':
                    pass
                elif method == 'ftp':
                    try:
                       ftp_client.mkd(pid)
                    except Exception as e:
                        print(e)
                    ftp_client.cwd(pid)
                elif method == 'ssh':
                    os.system('ssh ' + ssh_host + ' "mkdir -p Downloads/' + pid + '"')
                else:
                    pass
            ds[pid] = {}

        orientation_optimize(pathname, filename, pid, thread_num, ftp_client)

if __name__ == '__main__':
    ds = {}
    r = redis.Redis(host='127.0.0.1', port=6379, db=0)
    
    ftp_client =  FTP('192.168.88.177', 'photo', 'ph2008')

    ssh_host = 'ds'
    
    AccessKeyId = 'LTAI5tSReWm8hz7dSYxxth8f'
    AccessKeySecret = '8ywTDF9upPAtvgXtLKALY2iMYHIxdS'
    Endpoint = 'oss-cn-shanghai.aliyuncs.com'
    Bucket = 'suwa3d-securedata'
    oss_client = oss2.Bucket(oss2.Auth(AccessKeyId, AccessKeySecret), Endpoint, Bucket)

    upload_methods = ('oss', '')
        
    threads = []
    thread_nums = 5
    for i in range(thread_nums):
        t = Thread(target=process_photos, args=(str(i)))
        threads.append(t)
        t.start()