Python 爬虫实战(1)-爬取百度图片

2019-10-05

原创字数统计: 564 | 阅读时长: 2分

爬虫实战，按搜索关键字爬取百度图片。

简单的爬虫框架(爬取单个页面)

import requests

def getHTMLText(url):
    try:
        r.requests.get(url)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.text
    except:
        return "产生异常"

if __name__ == "__main__":
    url = "https://www.baidu.com"
    print(getHTMLText(url))

爬取单张图片

import requests
import os
kv = {'user-agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36'}
url = "https://img.bizhiku.net/uploads/2019/0924/eew40o0keon.jpg?x-oss-process=style/w_auto-h_280"
root = "E://pic//"   #保存路径
path = root + url.split('/')[-1]+'.jpg'  #文件名，如果url后缀有.jpg就不用加.jpg
try:
    if not os.path.exists(root):
        os.mkdir(root)
    if not os.path.exists(path):
        r = requests.get(url, headers=kv)
        with open(path, 'wb') as f:
            f.write(r.content)
            f.close()
            print('文件成功保存')
    else:
        print('文件已存在')
except:
    print("爬取失败")

批量按关键字爬取百度图片(单页)

import re
import requests
import random
import ssl
ssl.create_default_https_context = ssl._create_unverified_context  #验证SSL

def find_img(html):
    tag = '"objURL":"(.*?)"'             #核心在这里，正则匹配出符合标准的URL
    imgre = re.compile(tag)
    img_url_list = imgre.findall(html)
    return img_url_list

def getHTMLText(url,ua):         #获取HTML文本
    try:
        r = requests.get(url,headers=ua)
        r.encoding = r.apparent_encoding
        return r.text
    except:
        return "产生异常"

def save_img(img_list,word):       #保存文件
    root = "E://pic//"  #保存路径
    i = 1
    for img_url in img_list:
        try:
            pic = requests.get(img_url,timeout=10)  #设置timeout，避免等待
            print('正在下载'+word+str(i)+'.jpg...')
        except requests.exceptions.ConnectionError:
            print('URL错误')
            continue
        path = root + word + str(i)+'.jpg'    #文件命名为: 关键字 + 序列号（从1开始）.jpg
        # print(path)
        i = i+1
        fp = open(path,'wb')
        fp.write(pic.content)
        fp.close()

if __name__ == "__main__":
    baidu_img = 'http://image.baidu.com/search/index?tn=baiduimage&ps=1&ct=201326592&lm=-1&cl=2&nc=1&ie=utf-8&word='   #百度图片
    keyword = input('请输入要搜索的关键词： ')
    url = baidu_img + keyword    #这个URL是可以分析出来的
    # User-Agent
    ua = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36'}
    temp = getHTMLText(url, ua)
    if temp != "产生异常":
        # print(find_img(temp))
        imgs = find_img(temp)
        save_img(imgs,keyword)