python多线程比单线程处理时间长，请问该如何改进？

Z时代
2024-01-10
分类：技术分享

各位大佬好，我用如下的代码进行文本之间的相似度计算（其中相似度计算的代码未附），主要是多线程这块计算时间比单线程所用时间多了将近一半时间，我的文本量很大，想高效处理，请问为何我使用多线程还是如此之慢，以下代码该如何改进？`

import threading   
data3 =data2[:10]
start_time=time.time()
for item in data3:
    title = item['autn:content']['DOCUMENT']['DRETITLE']['$'].strip().replace(' ','')
    content = item['autn:content']['DOCUMENT']['DRECONTENT']['$'].strip().replace('\n','').replace(' ','')
    list1 = [0] * (len(data_all)+1)
    new_data =    title + ' ' + content
    if len(data_all)>=10:
        dis = int(len(data_all)/10)
        j = 1
        t_obj=[]
        for i in range(10):
            if i ==9:
                t=threading.Thread(target=deal_many_data,args=("Thread-1",j,len(data_all)+1,))
            else:
                t=threading.Thread(target=deal_many_data,args=("Thread-1",j,j+dis,))
                j = j + dis
            t_obj.append(t)
            t.setDaemon(True)  #把当前线程设置为守护线程，一定在start前设
            t.start()
        for t in t_obj:
            t.join()
            print(time.time()-start_time)
def deal_many_data(threadName,s,e):
    for key,v in data_all.items():
        if int(key) >= e:
            break
        if int(key)>=s:
            sim_all = 0
            count = 0
            sim3 = 0 #判断前两个是否相似不相似则跳出
            if len(data_all[key]['content']) >= 10:
                newlist = random.sample(list(range(0,len(data_all[key]['content']))),10)
            else:
                newlist = list(range(0,len(data_all[key]['content'])))
            for index in newlist:
                data = data_all[key]['title'][index] + data_all[key]['content'][index]
                sim = sentence_similarity(data,new_data)
                if index <= 2:
                    if sim < 0.55:
                        sim3 += 1
                        if sim3 > 1:
                            list1[int(key)] = 0
                sim_all += sim
                count += 1
                list1[int(key)] = sim_all / countprint ("%s: %s" % ( threadName, key )) `