在 https://www.cnblogs.com/mengyu/p/6759671.html 的启示下，解决了乱码问题，在此向作者表示感谢。
至此，困扰我几天的乱码问题和退出问题都解决了，只是处理速度上有点遗憾。
下面是新代码，较原代码多出了两句 rsp.encoding = 'gb18030' 这是解决中文乱码问题的关键。
#------------------------------------------------------------------------------------ # 理想论坛爬虫1.07，用于爬取主贴再爬子贴，数据存到文件里，再由insertDB.py读取插DB # 退出问题，乱码问题至此解决，只是目前速度上还是遗憾点 # 2018年4月26日 #------------------------------------------------------------------------------------ from bs4 import BeautifulSoup import requests import threading import re import time import datetime import os import json import colorama from colorama import Fore, Back, Style colorama.init()user_agent='Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36' headers={'User-Agent':user_agent}# 存储数据文件的目录 folder=""# 主帖数组 topics=[]#------------------------------------ # 在论坛页中寻找主贴 # pageUrl:论坛页url #------------------------------------ def findTopics(pageUrl):print("\n开始读取页面"+pageUrl+"的帖子");try:rsp=requests.get(pageUrl,headers=headers)rsp.encoding = 'gb18030' #解决中文乱码问题的关键soup= BeautifulSoup(rsp.text,'html.parser',from_encoding='gb2312')for tbodys in soup.find_all('tbody'):pageCount=1url='none'title='none'for spans in tbodys.find_all('span',class_="forumdisplay"):for link in spans.find_all('a'):if link and link.get("href"): url="http://www.55188.com/"+link.get("href")title=link.textfor spans in tbodys.find_all('span',class_="threadpages"):for link in spans.find_all('a'):pageCount=link.textif url!='none' and title!='none':topic={'pageCount':pageCount,'url':url,'title':title}#print("topic="+str(topic)) topics.append(topic)#print("读取页面"+pageUrl+"的帖子完毕");except Exception as e:log("findTopics出现异常:"+str(e),'red')#------------------------------------ # 以不同颜色在控制台输出文字 # pageUrl:论坛页url #------------------------------------ def log(text,color):if color=='red':print(Fore.RED + text+ Style.RESET_ALL)elif color=='green':print(Fore.GREEN + text+ Style.RESET_ALL)else:print(text)#------------------------------------ # 找到并保存帖子的细节 # index:序号,url:地址,title:标题 #------------------------------------ def saveTopicDetail(index,url,title):infos=[] # 找到的子贴信息while(len(infos)==0):try:rsp=requests.get(url,headers=headers)rsp.encoding = 'gb18030' #解决中文乱码问题的关键soup= BeautifulSoup(rsp.text,'html.parser',from_encoding='gb2312')session = requests.session()session.keep_alive = Falsefor divs in soup.find_all('div',class_="postinfo"):# 用正则表达式将多个空白字符替换成一个空格RE = re.compile(r'(\s+)')line=RE.sub(" ",divs.text)arr=line.split(' ')arrLength=len(arr)if arrLength==7:info={'楼层':arr[1],'作者':arr[2].replace('只看：',''),'日期':arr[4],'时间':arr[5],'title':title,'url':url}infos.append(info);elif arrLength==8:info={'楼层':arr[1],'作者':arr[2].replace('只看：',''),'日期':arr[5],'时间':arr[6],'title':title,'url':url}infos.append(info);#存文件filename=folder+"/"+str(index)+'.json'with open(filename,'w',encoding='utf-8') as fObj:json.dump(infos,fObj)except Exception as e:log("findTopicDetail访问"+url+"时出现异常:"+str(e),'red')time.sleep(5); # 如果出现异常，休息五秒后再试continue;#------------------------------------ # 入口函数 # start:起始页,end:终止页 #------------------------------------ def main(start,end):# 创建目录currTime=time.strftime('%H_%M_%S',time.localtime(time.time()))global folderfolder="./"+currTimeos.makedirs(folder)print("目录"+folder+"创建完成")# 获取主贴print('\n将从以下页面获取主贴:');for i in range(start,end+1): pageUrl='http://www.55188.com/forum-8-'+str(i)+'.html' # 这个页是论坛页，即第1页，第2页等 findTopics(pageUrl);n=len(topics)log("共读取到:"+str(n)+"个主贴",'green')# 获取主贴及其子贴finalTopics=[]index=0for topic in topics:end=int(topic['pageCount'])+1title=topic['title']for i in range(1,end):pattern='-(\d+)-(\d+)-(\d+)'newUrl=re.sub(pattern,lambda m:'-'+m.group(1)+'-'+str(i)+'-'+m.group(3),topic['url'])#print(newUrl) newTopic={'index':index,'url':newUrl,'title':title}finalTopics.append(newTopic)index=index+1n=len(finalTopics)log("共读取到:"+str(n)+"个帖子",'green')# 遍历finalTopicsfor newTopic in finalTopics:saveTopicDetail(newTopic['index'],newTopic['url'],newTopic['title']);# 开始 main(1,1)
与之配套把数据进库的insert.py的代码还是没变
# 读取理想论坛爬虫1.06生成的数据，然后写入DB import pymysql import time import datetime import os import json# 数据库插值 def insertDB(sqls):conn=pymysql.connect(host='127.0.0.1',user='root',passwd='12345678',db='test',charset='utf8')sum=0;# 插入成功总数for sql in sqls: try:count=conn.query(sql) #单条是否成功 except Exception as e:print("sql'"+sql+"'出现异常:"+str(e))continue;if count==0:print(sql+'插入记录失败'); sum+=countconn.commit()conn.close()return sum# 入口函数 def main(folder):allinfos=[]for filename in os.listdir(folder):filePathname=folder+"/"+filenamewith open(filePathname,'r',encoding='utf-8') as fObj:infos=json.load(fObj)#print(infos) allinfos.extend(infos)sqls=[]for info in allinfos:sql="insert into test.topic13(floor,author,tdate,ttime,addtime,url,title) values ('"+info['楼层']+"','"+info['作者']+"','"+info['日期']+"','"+info['时间']+"',"+"now(),'"+info['url']+"','"+info['title']+"' "+" )"sqls.append(sql)print("将向数据库插入"+str(len(sqls))+"条记录")retval=insertDB(sqls)print("已向数据库插入"+str(retval)+"条记录")# 开始 main("./17_17_51")
数据库里中文正常了：
2018年4月26日17点49分