wordpress后台管理界面地址网站优化seo
这几天写了个爬虫，爬取的数据比较多。一直挂在服务器上跑，后面发现启动十个进程总会运行着某几个进程挂掉，导致数据采集工作比较延后。
后面重新改进了一下，从日志中读取当前进程断点，继续爬取。
用了一个笨方法解决进程挂掉的问题。就是每半个小时关闭所有进程，重新从断点开始，当然各位有好的方法可以留言告诉我。
脚本使用方法直接运行 run.py
run.py
#-*- coding:utf-8 -*-
import os,time
import re
#检查sqlpwd.py的进程，并关闭进程
def kill():
lines = os.popen('ps -e|grep sqlpwd.py').readlines()
pa = re.compile(r'(\d+) ')
for line in lines:
result = re.findall(pa,line)
print(result[0])
os.popen('kill -9 '+str(result[0]))
def check():
with open('log.txt','r') as f:
lines = f.readlines()
for line in lines:
if 'FinishAll' in line:
return True
return False
'''
如果已经进行爬取了，则直接从上次扫描的地方开始
设置每半个小时重新扫描，解决之前扫描过程中进程掉的问题
'''
def runpwd():
if os.path.exists('log.txt') == False:
os.system('python sqlpwd.py 0 &')
time.sleep(18000)
kill()
while True:
print('-------------启动进程-------------------')
os.system('python sqlpwd.py 1 &')
time.sleep(60)
print('-------------结束进程-------------------')
kill()
if check() == True:
print('所有信息采集完成')
break
else:
pass
if __name__ == '__main__':
runpwd()
sqlpwd.py
#-*- coding:utf-8 -*-
import requests
import re
import random
from bs4 import BeautifulSoup
from multiprocessing import Process,Pool,Lock
import multiprocessing
import time
import sys
import logging
reload(sys)
sys.setdefaultencoding('utf-8')
def get_pwd(j,start,end):
logger = logging.getLogger(__name__)
logger.setLevel(level = logging.INFO)
handler = logging.FileHandler("log.txt")
handler.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s - %(message)s')
handler.setFormatter(formatter)
console = logging.StreamHandler()
console.setLevel(logging.INFO)
logger.addHandler(handler)
logger.addHandler(console)
user_agent=['Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.87 Safari/537.36',
'Mozilla/5.0 (X11; U; Linux x86_64; zh-CN; rv:1.9.2.10) Gecko/20100922 Ubuntu/10.10 (maverick) Firefox/3.6.10',
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101 Safari/537.36',
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71 Safari/537.1 LBBROWSER',
'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E; QQBrowser/7.0.3698.400)',
]
headers={
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Encoding': 'gzip, deflate, sdch',
'Accept-Language': 'zh-CN,zh;q=0.8',
'User-Agent': user_agent[random.randint(0,5)]
}
#print(start,end)
logger.info('进程'+str(j)+'任务从第'+str(start)+'页采集到'+str(end)+'页')
for i in range(start,end):
url = 'https://www.mysql-password.com/database/'+str(i)
#print(url)
logger.info(url)
try:
r = requests.get(url,headers=headers)
r.encoding = 'utf8'
html = r.text
soup = BeautifulSoup(html,'lxml')
except Exception as e:
print('进程'+str(j)+'异常：原因如下')
print(e)
#log.write('进程'+str(j)+'异常：原因如下'+str(e)+'\n')
logger.info('进程'+str(j)+'异常：原因如下'+str(e))
time.sleep(10)
continue
filename = 'password'+str(j)+'.txt'
try:
for tag in soup.find('tbody').find_all('tr'):
ss = str(tag.find_all('td')[2].string)
#print(ss)
with open(filename,'a') as f:
f.write(ss+'\n')
except Exception as e:
print('进程'+str(j)+'异常：原因如下')
print(e)
#log.write('进程'+str(j)+'异常：原因如下'+str(e)+'\n')
logger.info('进程'+str(j)+'异常：原因如下'+str(e))
time.sleep(10)
continue
#print('进程'+str(j)+'采集完成第'+str(i)+'页')
logger.info('进程'+str(j)+'采集完成第'+str(i)+'页')
def check():
num = []
mul = []
with open('log.txt','r') as f:
lines = f.readlines()
pa = re.compile(r'https://www.mysql-password.com/database/(\d+)')
for line in lines:
result = re.findall(pa,line)
if len(result) != 0 :
#print(result)
num.append(int(result[0]))
m = 10
start = 20000
cha = int(((180000-start)/m))
for j in range(0,m):
numList = []
info = []
end = 20000 + cha * (j+1)
for n in num:
if n > start and n < end:
numList.append(n)
print (str(j)+','+str(numList[-1])+','+str(end)+',')
if numList[-1] + 1 == end:
start = end + 1
continue
else:
info.append(j)
info.append(numList[-1])
info.append(end)
mul.append(info)
start = end + 1
return mul
def run(type):
if type == 0:
m = 10
pool = multiprocessing.Pool(m)
#start是开始爬行的页数， 下面写得是20000 180000 意思是从20000页爬到180000页，根据实际情况修改
start = 20000
cha = int(((180000-start)/m))
for j in range(0,m):
end = 20000 + cha * (j+1)
pool.apply_async(get_pwd,args=(j,start,end,))
#print(start,end)
start = end + 1
pool.close()
pool.join()
else:
mul = check()
#print(mul)
if len(mul) == 0:
print('所有进程都爬取完成')
logger.info('FinishAll')
else:
pool = multiprocessing.Pool(len(mul))
for mu in mul:
pool.apply_async(get_pwd,args=(mu[0],mu[1],mu[2],))
pool.close()
pool.join()
if __name__ == '__main__':
runtype = int(sys.argv[1])
run(runtype)