一个简单的demo，python爬虫，其实是以前的存货，很久很久没有写爬虫了，渣渣更渣了啊！
爬取财富中文网，2020年《财富》中国500强排行榜相关数据，数据都在网页源码里，结构也比较清晰，基本上一个请求页面可以搞定所有数据，一个老哥要的数据，用来做数据分析！
新人可以用来练手的网站，这里给出参考demo，仅供参考和学习使用！
fake_useragent库本地使用
来自于吾爱破解网站（ID：jxt441621944）上的分享，fake_useragent库也是本渣渣比较喜欢使用的一个库，比较方便吧，好用倒也说不上，看着用吧，fake_useragent库本地使用方法，这里给大家整理和打包了一下！
UserAgent就是用户代{过}{滤}理，是一串字符串，相当于是浏览器的身份证明，在写爬虫的时候频繁更换请求头中的UserAgent可以避免触发反爬机制（配合代{过}{滤}理IP食用更佳）。
fake_useragent就是可以获得一个随机的用户代{过}{滤}理的库。
打包地址获取 关注微信公众号：二爷记 后台回复 “UA”
fake_useragent库总共250条UA!
UA库文件：fake_ua.txt
py调用文件：fakeua.py
-
#ua库使用
-
#微信：huguo00289
-
# -*- coding: UTF
-8 -*-
-
-
import random
-
-
with open(
'fake_ua.txt',
'r') as f:
-
fake_ua = [fua.strip()
for fua in f.readlines()]
-
-
print(random.choice(fake_ua))
python爬虫2020年《财富》中国500强排行榜数据爬取demo
没啥好说的，照着抄吧，本渣渣也是抄的的，不懂的尽量百度
也可以加本渣渣微信：huguo00289
交流探讨学习！
-
#财富爬取
-
#微信：huguo00289
-
# -*- coding: UTF
-8 -*-
-
import requests,random
-
from lxml
import etree
-
import xlsxwriter
-
-
-
class Httprequest(object):
-
ua_list = [
-
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.163 Safari/535.1',
-
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36Chrome 17.0',
-
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',
-
'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:6.0) Gecko/20100101 Firefox/6.0Firefox 4.0.1',
-
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1',
-
'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',
-
'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',
-
'Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11',
-
]
-
@property #把方法变成属性的装饰器
-
def random_headers(self):
-
return {
-
'User-Agent': random.choice(self.ua_list)
-
}
-
-
-
class Get_data(Httprequest):
-
def __init__(self):
-
self.url=
"http://www.fortunechina.com/fortune500/c/2020-07/27/content_369925.htm"
-
self.murl=
"http://www.fortunechina.com/fortune500/c/2020-05/18/content_365275.htm"
-
-
def get_data(self):
-
html=requests.get(self.url,headers=self.random_headers,timeout=
5).content.decode(
'utf-8')
-
#
print(html)
-
req = etree.HTML(html)
-
rankings=req.xpath(
'//table[@class="wt-table"]/tbody/tr/td[1]/text()')
-
last_rankings=req.xpath(
'//table[@class="wt-table"]/tbody/tr/td[2]/text()')
-
companys=req.xpath(
'//table[@class="wt-table"]/tbody/tr/td[3]/a/text()')
-
incomes=req.xpath(
'//table[@class="wt-table"]/tbody/tr/td[4]/text()')
-
profits=req.xpath(
'//table[@class="wt-table"]/tbody/tr/td[5]/text()')
-
#
print(ranking)
-
#
print(last_ranking)
-
#
print(company)
-
#
print(income)
-
#
print(profit)
-
data_list=[]
-
for ranking,last_ranking,company,income,profit in zip(
-
rankings, last_rankings, companys, incomes, profits
-
):
-
data = [
-
ranking,last_ranking,company,income,profit
-
]
-
-
print(data)
-
data_list.
append(data)
-
print(
'\n')
-
-
self.write_to_xlsx(data_list)
-
-
-
-
def write_to_xlsx(self, data_list):
-
workbook = xlsxwriter.Workbook(
'{}_search_results.xlsx'.format(
"2020年《财富》中国500强排行榜")) # 创建一个Excel文件
-
worksheet = workbook.add_worksheet(
"2020年《财富》中国500强排行榜")
-
title = [
'排名',
'上年排名',
'公司名称（中文）',
'营业收入（百万元）',
'利润（百万元）'] # 表格title
-
worksheet.write_row(
'A1', title)
-
for index, data in enumerate(data_list):
-
# content = content.rstrip()
-
# keyword, rank, include_num, chart_url, title, game_id, company_num, long_words_num = data
-
num0 = str(index +
2)
-
row =
'A' + num0
-
# data = [name, size, game_id]
-
worksheet.write_row(row, data)
-
workbook.
close()
-
-
-
def get_mdata(self):
-
html=requests.get(self.murl,headers=self.random_headers,timeout=
5).content.decode(
'utf-8')
-
#
print(html)
-
req = etree.HTML(html)
-
rankings=req.xpath(
'//table[@class="wt-table"]/tbody/tr/td[1]/text()')
-
companys=req.xpath(
'//table[@class="wt-table"]/tbody/tr/td[2]/a/text()')
-
incomes=req.xpath(
'//table[@class="wt-table"]/tbody/tr/td[3]/text()')
-
profits=req.xpath(
'//table[@class="wt-table"]/tbody/tr/td[4]/text()')
-
data_list=[]
-
for ranking,company,income,profit in zip(
-
rankings,companys, incomes, profits
-
):
-
data = [
-
ranking,company,income,profit
-
]
-
-
print(data)
-
data_list.
append(data)
-
print(
'\n')
-
-
self.write_to_mxlsx(data_list)
-
-
-
def write_to_mxlsx(self, data_list):
-
workbook = xlsxwriter.Workbook(
'{}_search_results.xlsx'.format(
"2020年《财富》美国500强排行榜")) # 创建一个Excel文件
-
worksheet = workbook.add_worksheet(
"2020年《财富》美国500强排行榜")
-
title = [
'排名',
'公司名称（中文）',
'营业收入（百万美元）',
'利润（百万美元）'] # 表格title
-
worksheet.write_row(
'A1', title)
-
for index, data in enumerate(data_list):
-
# content = content.rstrip()
-
# keyword, rank, include_num, chart_url, title, game_id, company_num, long_words_num = data
-
num0 = str(index +
2)
-
row =
'A' + num0
-
# data = [name, size, game_id]
-
worksheet.write_row(row, data)
-
workbook.
close()
-
-
-
if __name__==
"__main__":
-
spider=Get_data()
-
#spider.get_data()
-
spider.get_mdata()
-
-
微信公众号：二爷记
不定时分享python源码及工具
转载：https://blog.csdn.net/minge89/article/details/109507834
查看评论