爬虫-扇贝字典

import requests
from lxml import etree
for i in range(1,4):
    url = 'https://www.shanbay.com/wordlist/110521/232414/?page={}'.format(i)
    headers = {'user-agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.131 Safari/537.36' }
    response = requests.get(url,headers=headers).text
    # with open('shanbei.html','w',encoding='utf-8') as f:
    #     f.write(response)
    tree = etree.HTML(response)
    tr_list = tree.xpath('.//tr[@class="row"]')
    # print(len(tr_list))
    for tr in tr_list:
        word = tr.xpath('.//strong/text()')[0]
        # print(word)
        translate = tr.xpath('./td[@class="span10"]/text()')[0]
        infor = word + ':  ' + translate + '\n'
        with open('shanbei.txt','a',encoding='utf-8') as fp:
            fp.write(infor)