流程很清晰：
1.分析拉勾页面
2.发现拉勾下面招聘页面是用Ajax写的，抓包试了以下很简单：
招聘块
3.通过代码获取到json
4.通过json里面的一个值positionId拼接得出详情页面的网址
5.通过分析详情页的HTML获取到职位要求
6.将所有的职业要求添加到一个字符串当中，并将无用的词语去掉（比如'职位要求'这四个字这样的）
7.使用jieba分析出高频词汇
8.最后制作出职业技能画像
中间遇到了很多问题，比如PC端的网页不好分析，字符串不在一个标签当中，而且拉勾的headers很迷，然后我抓取详情页面就用的手机端的抓取的，结果刚出了一个错误，我用了一个try之后，我的IP就被禁掉了
IP被禁
没办法啊，只能去获取IP代理去了，代码先写到这儿
# -*- coding: UTF-8 -*-
import requests,re,json
from bs4 import BeautifulSoup
import sys
reload(sys)
sys.setdefaultencoding('utf8')
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)#去掉SSL警告
def get_detail_page(detail_url):
mobile_headers = {
'User-Agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Mobile Safari/537.36',
'Cookie': 'JSESSIONID=15A9FE2A6A2CC09A2FB61021BF8E8086; '
'user_trace_token=20170501124201-1adf364d88864075b61dde9bdd5871ea; '
'LGUID=20170501124202-850be946-2e28-11e7-b43c-5254005c3644; '
'index_location_city=%E5%8C%97%E4%BA%AC; TG-TRACK-CODE=index_search;'
' SEARCH_ID=0a596428cb014d3bab7284f879e214f0; Hm_lvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1493613734;'
' Hm_lpvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1493622592;'
' LGRID=20170501150939-247d4c29-2e3d-11e7-8a78-525400f775ce;'
' _ga=GA1.2.1933438823.1493613734'
}
detail_response = requests.get(detail_url,headers=mobile_headers,verify=False)
detail_soup = BeautifulSoup(detail_response.content,'html.parser')
detail_page_dd = detail_soup.find('div',attrs={'class': 'content'})
try:
return detail_page_dd.text
except Exception,e:
print "这页没找到对应标签"
finally:
pass
def get_all_text(data):
all_text = ''
for i in data['content']['positionResult']['result']:
#解析json，获取详情页的拼接地址
detail_url_pinjie = 'https://m.lagou.com/jobs/'
detail_url = detail_url_pinjie + str(i['positionId']) + '.html'
print detail_url
all_text += get_detail_page(detail_url)
print all_text
return all_text
def get_page():
url = 'https://www.lagou.com/jobs/positionAjax.json'
for pn in range(1,30):
post_data = {
'city': '北京',
'needAddtionalResult': 'false',
'first': 'true',
'pn': str(pn),
'kd': 'python'
}
headers = {
'Host': 'www.lagou.com',
'Referer': 'https://www.lagou.com/jobs/list_python爬虫',
'User-Agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Mobile Safari/537.36',
'Cookie': 'JSESSIONID=15A9FE2A6A2CC09A2FB61021BF8E8086; '
'user_trace_token=20170501124201-1adf364d88864075b61dde9bdd5871ea; '
'LGUID=20170501124202-850be946-2e28-11e7-b43c-5254005c3644; '
'index_location_city=%E5%8C%97%E4%BA%AC; TG-TRACK-CODE=index_search;'
' SEARCH_ID=0a596428cb014d3bab7284f879e214f0; Hm_lvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1493613734;'
' Hm_lpvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1493622592;'
' LGRID=20170501150939-247d4c29-2e3d-11e7-8a78-525400f775ce;'
' _ga=GA1.2.1933438823.1493613734'
}
response = requests.post(url=url, verify=False, params=post_data, headers=headers)
data = json.loads(response.content)
print response.content
get_all_text(data)
get_page()