通过Python、BeautifulSoup爬取Gitee热门开源项目

pip install requests

pip install beautifulsoup4

pip install pymongo

# -*- coding: utf-8 -*-
# __author__ : "初一丶" 公众号:程序员共成长
# __time__ : 2018/8/22 18:51
# __file__ : spider_mayun.py
# 导入相关库
import requests
from bs4 import BeautifulSoup
import pymongo

"""
通过分析页面url 查询不同语言的热门信息是有language这个参数决定的
"""
# language = ‘java‘
language = ‘python‘
domain = ‘https://gitee.com‘
uri = ‘/explore/starred?lang=%s‘ % language
url = domain + uri

# 用户代理
user_agent = ‘Mozilla/5.0 (Macintosh;Intel Mac OS X 10_12_6) ‘             ‘AppleWebKit/537.36(KHTML, like Gecko) ‘             ‘Chrome/67.0.3396.99Safari/537.36‘

# 构建header
header = {‘User_Agent‘: user_agent}
# 获取页面源代码
html = requests.get(url, headers=header).text
# 获取Beautiful对象
soup = BeautifulSoup(html)

# 热门类型分类 今日热门 本周热门 data-tab标签来区分当日热门和本周热门
hot_type = [‘today-trending‘, ‘week-trending‘]

# divs = soup.find_all(‘div‘, class_=‘ui tab active‘)
# 创建热门列表
hot_gitee = []
for i in hot_type:
   # 通过热门标签 查询该热门下的数据
   divs = soup.find_all(‘div‘, attrs={‘data-tab‘: i})
   divs = divs[0].select(‘div.row‘)
   for div in divs:
       gitee = {}
       a_content = div.select(‘div.sixteen > h3 > a‘)
       div_content = div.select(‘div.project-desc‘)
       # 项目描述
       script = div_content[0].string
       # title属性
       title = a_content[0][‘title‘]
       arr = title.split(‘/‘)
       # 作者名字
       author_name = arr[0]
       # 项目名字
       project_name = arr[1]
       # 项目url
       href = domain + a_content[0][‘href‘]
       # 进入热门项目子页面
       child_page = requests.get(href, headers=header).text
       child_soup = BeautifulSoup(child_page)
       child_div = child_soup.find(‘div‘, class_=‘ui small secondary pointing menu‘)
       """
           

               HTTPS
               SSH
           
       """
       a_arr = child_div.findAll(‘a‘)
       # git http下载链接
       http_url = a_arr[0][‘data-url‘]
       # git ssh下载链接
       ssh_url = a_arr[1][‘data-url‘]

       gitee[‘project_name‘] = project_name
       gitee[‘author_name‘] = author_name
       gitee[‘href‘] = href
       gitee[‘script‘] = script
       gitee[‘http_url‘] = http_url
       gitee[‘ssh_url‘] = ssh_url
       gitee[‘hot_type‘] = i

       # 连接mongo
       hot_gitee.append(gitee)

print(hot_gitee)

# 链接mongo参数
HOST, PORT, DB, TABLE = ‘127.0.0.1‘, 27017, ‘spider‘, ‘gitee‘
# 创建链接
client = pymongo.MongoClient(host=HOST, port=PORT)
# 选定库
db = client[DB]
tables = db[TABLE]
# 插入mongo库
tables.insert_many(hot_gitee)

[{
 ‘project_name‘: ‘IncetOps‘,
 ‘author_name‘: ‘staugur‘,
 ‘href‘: ‘https://gitee.com/staugur/IncetOps‘,
 ‘script‘: ‘基于Inception，一个审计、执行、回滚、统计sql的开源系统‘,
 ‘http_url‘: ‘https://gitee.com/staugur/IncetOps.git‘,
 ‘ssh_url‘: ‘git@gitee.com:staugur/IncetOps.git‘,
 ‘hot_type‘: ‘today-trending‘
}, {
 ‘project_name‘: ‘cms‘,
 ‘author_name‘: ‘dlg_center‘,
 ‘href‘: ‘https://gitee.com/dlg_center/cms‘,
 ‘script‘: None,
 ‘http_url‘: ‘https://gitee.com/dlg_center/cms.git‘,
 ‘ssh_url‘: ‘git@gitee.com:dlg_center/cms.git‘,
 ‘hot_type‘: ‘today-trending‘
}, {
 ‘project_name‘: ‘WebsiteAccount‘,
 ‘author_name‘: ‘张聪‘,
 ‘href‘: ‘https://gitee.com/crazy_zhangcong/WebsiteAccount‘,
 ‘script‘: ‘各种问答平台账号注册‘,
 ‘http_url‘: ‘https://gitee.com/crazy_zhangcong/WebsiteAccount.git‘,
 ‘ssh_url‘: ‘git@gitee.com:crazy_zhangcong/WebsiteAccount.git‘,
 ‘hot_type‘: ‘today-trending‘
}, {
 ‘project_name‘: ‘chain‘,
 ‘author_name‘: ‘何全‘,
 ‘href‘: ‘https://gitee.com/hequan2020/chain‘,
 ‘script‘: ‘linux 云主机 管理系统，包含 CMDB,webssh登录、命令执行、异步执行shell/python/yml等。持续更...‘,
 ‘http_url‘: ‘https://gitee.com/hequan2020/chain.git‘,
 ‘ssh_url‘: ‘git@gitee.com:hequan2020/chain.git‘,
 ‘hot_type‘: ‘today-trending‘
}, {
 ‘project_name‘: ‘Lepus‘,
 ‘author_name‘: ‘茹憶。‘,
 ‘href‘: ‘https://gitee.com/ruzuojun/Lepus‘,
 ‘script‘: ‘简洁、直观、强大的开源企业级数据库监控系统，MySQL/Oracle/MongoDB/Redis一站式监控，让数据库监控更简...‘,
 ‘http_url‘: ‘https://gitee.com/ruzuojun/Lepus.git‘,
 ‘ssh_url‘: ‘git@gitee.com:ruzuojun/Lepus.git‘,
 ‘hot_type‘: ‘today-trending‘
}, {
 ‘project_name‘: ‘AutoLink‘,
 ‘author_name‘: ‘苦叶子‘,
 ‘href‘: ‘https://gitee.com/lym51/AutoLink‘,
 ‘script‘: ‘AutoLink是一个开源Web IDE自动化测试集成解决方案‘,
 ‘http_url‘: ‘https://gitee.com/lym51/AutoLink.git‘,
 ‘ssh_url‘: ‘git@gitee.com:lym51/AutoLink.git‘,
 ‘hot_type‘: ‘week-trending‘
}, {
 ‘project_name‘: ‘PornHubBot‘,
 ‘author_name‘: ‘xiyouMc‘,
 ‘href‘: ‘https://gitee.com/xiyouMc/pornhubbot‘,
 ‘script‘: ‘全球最大成人网站PornHub爬虫 （Scrapy、MongoDB） 一天500w的数据‘,
 ‘http_url‘: ‘https://gitee.com/xiyouMc/pornhubbot.git‘,
 ‘ssh_url‘: ‘git@gitee.com:xiyouMc/pornhubbot.git‘,
 ‘hot_type‘: ‘week-trending‘
}, {
 ‘project_name‘: ‘wph_opc‘,
 ‘author_name‘: ‘万屏汇‘,
 ‘href‘: ‘https://gitee.com/wph_it/wph_opc‘,
 ‘script‘: None,
 ‘http_url‘: ‘https://gitee.com/wph_it/wph_opc.git‘,
 ‘ssh_url‘: ‘git@gitee.com:wph_it/wph_opc.git‘,
 ‘hot_type‘: ‘week-trending‘
}, {
 ‘project_name‘: ‘WebsiteAccount‘,
 ‘author_name‘: ‘张聪‘,
 ‘href‘: ‘https://gitee.com/crazy_zhangcong/WebsiteAccount‘,
 ‘script‘: ‘各种问答平台账号注册‘,
 ‘http_url‘: ‘https://gitee.com/crazy_zhangcong/WebsiteAccount.git‘,
 ‘ssh_url‘: ‘git@gitee.com:crazy_zhangcong/WebsiteAccount.git‘,
 ‘hot_type‘: ‘week-trending‘
}, {
 ‘project_name‘: ‘information27‘,
 ‘author_name‘: ‘印妈妈‘,
 ‘href‘: ‘https://gitee.com/itcastyinqiaoyin/information27‘,
 ‘script‘: None,
 ‘http_url‘: ‘https://gitee.com/itcastyinqiaoyin/information27.git‘,
 ‘ssh_url‘: ‘git@gitee.com:itcastyinqiaoyin/information27.git‘,
 ‘hot_type‘: ‘week-trending‘
}]