成语大全网 - 汉语词典 - python爬虫 源码

python爬虫 源码

import os,requests

from bs4 import BeautifulSoup

headers ={

'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:75.0) Gecko/20100101 Firefox/75.0'

}

for i in range(105,200):

try:

url = '/web201605/herodetail/' + str(i) +'.shtml'

response = requests.get(url,headers)

response.encoding = 'gbk'

soup = BeautifulSoup(response.text,'html.parser')

# skill_name = soup.find('p','skill-name')

# skill_desc = soup.find('p','skill-desc')

# print(skill_name.text)

# print(skill_desc.text)

name = soup.find("h2", "cover-name").text

# print(name)

story = soup.find('div', 'pop-bd').text

if story =='\n':

print("\n没有%d%s的故事!"%(i,name))

else:

story_ = story.replace('。' ,'。\n' )

story_ = story.replace('\n' ,'\t>>>' )

print(story_[0:30]+"...")

# os.mkdir('C:\\Users\\Crystal\\Desktop\\英雄故事2')

# os.mkdir('C:\\Users\\28459\\Desktop\\测试\\')

os.chdir('C:\\Users\\28459\\Desktop\\测试\\')

open('%s'%name + '.txt' ,'w').write(story_)

print('%d%s的故事已保存!'%(i,name))

print()

except AttributeError:

print("\n没有编号为%d的英雄!"%i)