百度地图POI数据爬取，突破百度地图API爬取数目“400条“的限制11。

# -*- coding: utf-8 -*- 
import sys
import codecs
import urllib
import urllib2
import re
import pandas as pd

reload(sys)
sys.setdefaultencoding(‘utf-8‘)

user_agent = ‘Mozilla/5.0 (Windows; U; Windows NT 5.2)‘
headers = {‘User-Agent‘:user_agent}

inpath = ‘POI总表.csv‘
path = unicode(inpath, ‘utf-8‘)
poi_list = pd.read_csv(path)
api_key="" #输入AK

for poi in range(len(poi_list)):
outpath = ‘+str(poi)+‘_‘+poi_list.iloc[poi, 1] + ‘.csv‘
path = unicode(outpath, ‘utf-8‘)
with open(path, ‘w‘) as f:
f.write(codecs.BOM_UTF8)
f.write(‘id,name,lat,lng\n‘)
i=1
j=1
try:
for i in range(int(poi_list.iloc[poi,4])/10+2):
url = "http://api.map.baidu.com/place/v2/search?q="+poi_list.iloc[poi,3]+"&page_size=10&page_num="
url = url+str(i-1) + "&scope=1®ion=%E6%AD%A6%E7%A9%B4&city_limit=true&coord_type=1&output=json&ak="
+api_key # ak需输入申请的ak 
request = urllib2.Request(url, headers=headers)
response = urllib2.urlopen(request)
content = response.read().decode("utf-8")
# print content
reg = r‘"name":"([\d\D]*?)",[\d\D]*?"lat":([\d\D]*?),[\d\D]*?"lng":([\d\D]*?)},‘
pattern = re.compile(reg)
items = re.findall(pattern,content)
for item in items:
# print "name:",item[0]
# print "lat:",item[1]
# print "lng:",item[2]
savestr = str(j)+‘,‘+str(item[0])+‘,‘+str(item[1])+‘,‘+str(item[2])
j += 1
f.write(savestr)
except urllib2.URLError, e:
if hasattr(e, "code"):
print e.code
if hasattr(e, ‘reason‘):
print e.reason
finally:
f.close()

# -*- coding:utf-8 -*-
import json
import codecs
import os
import urllib2
import sys
import time

reload(sys)
sys.setdefaultencoding(‘utf8‘)


class BaiDuPOI(object):
def __init__(self, itemy, loc):
self.itemy = itemy
self.loc = loc

def urls(self):
api_key = baidu_api
urls = []
for pages in range(0, 20):
url = ‘http://api.map.baidu.com/place/v2/search?query=‘ + self.itemy + ‘&bounds=‘ + self.loc + ‘&page_size=20&page_num=‘ + str(
pages) + ‘&output=json&ak=‘ + api_key
urls.append(url)
return urls

def baidu_search(self):
json_sel = []
for url in self.urls():
json_obj = urllib2.urlopen(url)
data = json.load(json_obj)
for item in data[‘results‘]:
jname = item["name"]
jlat = item["location"]["lat"]
jlng = item["location"]["lng"]
js_sel = jname + ‘,‘ + str(jlat) + ‘,‘ + str(jlng)
json_sel.append(js_sel)
return json_sel


class LocaDiv(object):
def __init__(self, loc_all):
self.loc_all = loc_all

def lat_all(self):
lat_sw = float(self.loc_all.split(‘,‘)[0])
lat_ne = float(self.loc_all.split(‘,‘)[2])
lat_list = []
for i in range(0, int((lat_ne - lat_sw + 0.0001) / 0.1)): # 0.1为网格大小，可更改
lat_list.append(lat_sw + 0.1 * i) # 0.05
lat_list.append(lat_ne)
return lat_list

def lng_all(self):
lng_sw = float(self.loc_all.split(‘,‘)[1])
lng_ne = float(self.loc_all.split(‘,‘)[3])
lng_list = []
for i in range(0, int((lng_ne - lng_sw + 0.0001) / 0.1)): # 0.1为网格大小，可更改
lng_list.append(lng_sw + 0.1 * i) # 0.1为网格大小，可更改
lng_list.append(lng_ne)
return lng_list

def ls_com(self):
l1 = self.lat_all()
l2 = self.lng_all()
ab_list = []
for i in range(0, len(l1)):
a = str(l1[i])
for i2 in range(0, len(l2)):
b = str(l2[i2])
ab = a + ‘,‘ + b
ab_list.append(ab)
return ab_list

def ls_row(self):
l1 = self.lat_all()
l2 = self.lng_all()
ls_com_v = self.ls_com()
ls = []
for n in range(0, len(l1) - 1):
for i in range(0 + len(l1) * n, len(l2) + (len(l2)) * n - 1):
a = ls_com_v[i]
b = ls_com_v[i + len(l2) + 1]
ab = a + ‘,‘ + b
ls.append(ab)
return ls


if __name__ == ‘__main__‘:
# ak
baidu_api ="" # 这里填入你的百度API的ak
print "开始爬取数据，请稍等..."
start_time = time.time()
loc = LocaDiv(‘29.8255, 115.367400, 30.2194, 115.8287‘)
locs_to_use = loc.ls_row()

for loc_to_use in locs_to_use:
par = BaiDuPOI(u‘购物‘, loc_to_use) # 请修改爬取的类别
a = par.baidu_search()
doc = open(‘zhengfujigou.csv‘, ‘a+‘)
doc.write(codecs.BOM_UTF8)
for ax in a:
doc.write(ax)
doc.write(‘\n‘)
doc.close()
end_time = time.time()
print "购物爬取完毕，用时%.2f秒" % (end_time - start_time)