Python多线程如何保证线程同步？

Z时代
2024-01-10
分类：技术分享

Python多线程如何保证线程同步？

在我使用多线程的过程中，返回的结果有很多重复和缺少，我个人认为是因为线程资源不同步导致的，请问如何修改代码实现同步，和不要出现重复？是使用生产消费模型吗？

下面是代码，部分：

import json
import time
import requests
from queue import Queue
from concurrent.futures import ThreadPoolExecutor, as_completed
pool = ThreadPoolExecutor(100)
def get_email():
email_queue = Queue()
with open('email.txt', 'r') as file_object:
for line in file_object:
email_queue.put(line.strip())
return email_queue
def get_proxy(retry=3):
start = 0
while start <= retry:
res = requests.get("someurl")
dic_info = res.text
dic_info = json.loads(dic_info)
status_code = dic_info['code']
if status_code != '0':
start += 1
time.sleep(0.5)
continue
data = dic_info['obj']
ip = data[0]["ip"]
port = data[0]["port"]
ip_port = 'http://{}:{}'.format(ip,port)
return {'http': ip_port}
return None
def check(email, proxy):
headers = {
'authority': 'unite.nike.com',
'pragma': 'no-cache',
'cache-control': 'no-cache',
'user-agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36',
'content-type': 'text/plain;charset=UTF-8',
'accept': '*/*',
'origin': 'https://www.baidu.com',
'sec-fetch-site': 'same-site',
'sec-fetch-mode': 'cors',
'sec-fetch-dest': 'empty',
'referer': 'https://www.baidu.com/gb/register',
'accept-language': 'zh,en;q=0.9,en-US;q=0.8,zh-CN;q=0.7'
}
params = (
('appVersion', '786'),
('experienceVersion', '786'),
('uxid', 'com.nike.commerce.nikedotcom.web'),
('locale', 'en_GB'),
('backendEnvironment', 'identity'),
('browser', 'Google Inc.'),
('os', 'undefined'),
('mobile', 'false'),
('native', 'false')
)
url = 'someurl'
data = f'{"emailAddress": "{email}"}'
try:
response = requests.post(url=url, params=params, headers=headers, data=data, proxies=proxy, timeout=8)
if response.status_code == 204:
print(f'未注册 {email}')
return f'未注册 {email}'
elif response.status_code == 409:
print(f'已注册 {email}')
return f'已注册 {email}'
else:
print(f'链接错误 {email}')
return f'链接错误 {email}'
except requests.exceptions.RequestException:
print(f'连接失败 {email}')
return f'连接失败 {email}'
def save(result):
with open('log.txt', 'a+') as file_object:
file_object.write(result + 'n')
def main():
queue = get_email()
while queue.empty != True:
email = queue.get()
proxy = get_proxy()
tasks = [pool.submit(check, email, proxy)]
for task in as_completed(tasks):
check_result = task.result()
save(check_result)
if __name__ == '__main__':main()