多进程 | python实现多任务进程

python实现多任务进程

以下内容主要阐述了python实现多进程的两种方式、进程间如何通信、进程池以及一个多进程案例：批量复制文件。

Show You the Code:

# 实现多进程
# 1.使用process类
import multiprocessing


def process(index):
    print(f'Process:{index}')


if __name__ == "__main__":
    for i in range(5):
        p = multiprocessing.Process(target=process, args=(i,))
        p.start()


# 2.继承process类
import time
from multiprocessing import Process


class MyProcess(Process):
    def __init__(self, loop):
        Process.__init__()
        self.loop = loop

    def run(self):
        for count in range(self.loop):
            time.sleep(1)
            print(f'Pid:{self.pid} LoopCount:{count}')


if __name__ == '__main__':
    for i in range(2, 5):
        p = MyProcess(i)
        p.start()


# 进程间的通信
# 1、Queue-队列 先进先出
from multiprocessing import Queue
import multiprocessing


def download(p):  # download the data
    lst = [11, 22, 33, 44]
    for item in lst:
        p.put(item)
    print("successfully downloaded data!")


def save_data(p):
    lst = []
    while True:
        data = p.get()
        lst.append(data)
        if p.empty():
            break
    print(lst)
    print("successfully saved data!")


def main():
    p1 = Queue()

    t1 = multiprocessing.Process(target=download, args=(p1,))
    t2 = multiprocessing.Process(target=save_data, args=(p1,))

    t1.start()
    t2.start()


if __name__ == "__main__":
    main()


# 2、共享全局变量不适用于多进程编程
import multiprocessing

a = 1  # 全局变量


def demo1():
    global a
    a += 1


def demo2():
    print(a)


def main():
    t1 = multiprocessing.Process(target=demo1)
    t2 = multiprocessing.Process(target=demo2)

    t1.start()
    t2.start()


if __name__ == "__main__":
    main()


# 进程池之间的通信
from multiprocessing import Pool
import os, time, random


def worker(a):
    t_start = time.time()
    print('%s开始执行，进程号为%d' % (a, os.getpid()))

    time.sleep(random.random() * 2)
    t_stop = time.time()
    print(a, '执行完成，耗时%0.2f' % (t_stop - t_start))


if __name__ == '__main__':
    po = Pool(3)  # 定义一个进程池
    for i in range(0, 10):
        po.apply_async(worker, (i,))  # 向进程池中添加worker任务

    print("--start--")
    po.close()

    po.join()
    print("--end--")


# 案例：文件批量复制
import os
import time
import multiprocessing


def copy_file(Q, oldfolderName, newfolderName, file_name):
    # 文件复制,不需要返回
    time.sleep(0.5)
    # print('\r从%s文件夹复制到%s文件夹的%s文件'%(oldfolderName,newfolderName,file_name),end='')

    old_file = open(oldfolderName + '/' + file_name, 'rb')  # 待复制文件
    content = old_file.read()
    old_file.close()

    new_file = open(newfolderName + '/' + file_name, 'wb')  # 复制出的新文件
    new_file.write(content)
    new_file.close()

    Q.put(file_name)  # 向Q队列中添加文件


def main():
    oldfolderName = input('请输入要复制的文件夹名字:')  # 步骤1获取要复制文件夹的名字(可以手动创建,也可以通过代码创建,这里我们手动创建)
    newfolderName = oldfolderName + '复件'
    # 步骤二 创建一个新的文件夹
    if not os.path.exists(newfolderName):
        os.mkdir(newfolderName)

    filenames = os.listdir(oldfolderName)  # 3.获取文件夹里面所有待复制的文件名
    # print(filenames)

    pool = multiprocessing.Pool(5)  # 4.创建进程池

    Q = multiprocessing.Manager().Queue()  # 创建队列,进行通信
    for file_name in filenames:
        pool.apply_async(copy_file, args=(Q, oldfolderName, newfolderName, file_name))  # 5.向进程池添加任务
    pool.close()

    copy_file_num = 0
    file_count = len(filenames)
    # 不知道什么时候完成,所以定义一个死循环
    while True:
        file_name = Q.get()
        print(f"{file_name}已经复制！")
        copy_file_num += 1
        time.sleep(0.2)
        print('\r拷贝进度%.2f %%' % (copy_file_num * 100 / file_count), end='')  # 做一个拷贝进度条

        if copy_file_num >= file_count:
            break


if __name__ == '__main__':
    main()