Python IO密集型任务、计算密集型任务，以及多线程、多进程

对于IO密集型任务：

• 直接执行用时：10.0333秒
• 多线程执行用时：4.0156秒
• 多进程执行用时：5.0182秒

说明多线程适合IO密集型任务。

对于计算密集型任务

• 直接执行用时：10.0273秒
• 多线程执行用时：13.247秒
• 多进程执行用时：6.8377秒

说明多进程适合计算密集型任务。

#coding=utf-8

import sys

import multiprocessing

import time

import threading

# 定义全局变量Queue

g_queue = multiprocessing.Queue()

def init_queue():

    print("init g_queue start")

    while not g_queue.empty():

        g_queue.get()

    for _index in range(10):

        g_queue.put(_index)

    print("init g_queue end")

    return

# 定义一个IO密集型任务：利用time.sleep()

def task_io(task_id):

    print("IOTask[%s] start" % task_id)

    while not g_queue.empty():

        time.sleep(1)

        try:

            data = g_queue.get(block=True, timeout=1)

            print("IOTask[%s] get data: %s" % (task_id, data))

        except Exception as excep:

            print("IOTask[%s] error: %s" % (task_id, str(excep)))

    print("IOTask[%s] end" % task_id)

    return

g_search_list = list(range(10000))

# 定义一个计算密集型任务：利用一些复杂加减乘除、列表查找等

def task_cpu(task_id):

    print("CPUTask[%s] start" % task_id)

    while not g_queue.empty():

        count = 0

        for i in range(10000):

            count += pow(3*2, 3*2) if i in g_search_list else 0

        try:

            data = g_queue.get(block=True, timeout=1)

            print("CPUTask[%s] get data: %s" % (task_id, data))

        except Exception as excep:

            print("CPUTask[%s] error: %s" % (task_id, str(excep)))

    print("CPUTask[%s] end" % task_id)

    return task_id

if __name__ == '__main__':

    print("cpu count:", multiprocessing.cpu_count(), "\n")

    print(u"========== 直接执行IO密集型任务 ==========")

    init_queue()

    time_0 = time.time()

    task_io(0)

    print(u"结束：", time.time() - time_0, "\n")

    print("========== 多线程执行IO密集型任务 ==========")

    init_queue()

    time_0 = time.time()

    thread_list = [threading.Thread(target=task_io, args=(i,)) for i in range(10)]

    for t in thread_list:

        t.start()

    for t in thread_list:

        if t.is_alive():

            t.join()

    print("结束：", time.time() - time_0, "\n")

    print("========== 多进程执行IO密集型任务 ==========")

    init_queue()

    time_0 = time.time()

    process_list = [multiprocessing.Process(target=task_io, args=(i,)) for i in range(multiprocessing.cpu_count())]

    for p in process_list:

        p.start()

    for p in process_list:

        if p.is_alive():

            p.join()

    print("结束：", time.time() - time_0, "\n")

    print("========== 直接执行CPU密集型任务 ==========")

    init_queue()

    time_0 = time.time()

    task_cpu(0)

    print("结束：", time.time() - time_0, "\n")

    print("========== 多线程执行CPU密集型任务 ==========")

    init_queue()

    time_0 = time.time()

    thread_list = [threading.Thread(target=task_cpu, args=(i,)) for i in range(10)]

    for t in thread_list:

        t.start()

    for t in thread_list:

        if t.is_alive():

            t.join()

    print("结束：", time.time() - time_0, "\n")

    print("========== 多进程执行cpu密集型任务 ==========")

    init_queue()

    time_0 = time.time()

    process_list = [multiprocessing.Process(target=task_cpu, args=(i,)) for i in range(multiprocessing.cpu_count())]

    for p in process_list:

        p.start()

    for p in process_list:

        if p.is_alive():

            p.join()

    print("结束：", time.time() - time_0, "\n")

参考：https://zhuanlan.zhihu.com/p/24283040