Fork me on GitHub

Python 爬虫实战(2)-批量下载多页图片

原创 字数统计: 1.8k   |   阅读时长: 10分

20191005013849.png

爬虫实战，批量下载多页图片。

这次选取的是 http://www.bizhi88.com

Ps: 这是一个免费下载图片的网站哟 (＾Ｕ＾)ノ~ＹＯ

QQ截图20191005233906.png

不同于百度图片的后缀直接是汉字关键词,这个网站的URL格式是：http://www.bizhi88.com/s + 关键词序号:
比如搜索框搜索星空，就会跳转到http://www.bizhi88.com/s/26/,就是星空关键词的第一页。
翻到第二页会发现，除了首页，URL后缀就都是序号.html

QQ截图201005234924.png

分析到这里，大概就知道该怎么做了。
最重要的其实还是正则表达式，这里推荐一个在线测试网站：**http://tool.oschina.net/regex/#** ,先拿图片网站的代码去试一试：

20191005235658.png

看起来没啥问题，实际用的时候，问题就现出来了。

QQ截图20191005225318.png

Q1：一个标签里有两个重复的URL

解决办法：用set保存

20191005235925.png

Q2：正则表达式有缺陷(新手太弱)

解决办法：URL都向合法格式靠拢(符合规则的图片格式肯定都一样)

QQ截图20191005231331.png

这里通过URL长度排除不合法的

20191006000612.png

完整代码

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
import re
import requests
import random
import ssl
ssl.create_default_https_context = ssl._create_unverified_context  #验证SSL

def find_img(html):
    tag = '[a-zA-z]+://[^\\s]*[a-zA-z]'             #核心在这里，正则匹配出符合标准的URL
    # \\s其实是\s,转义
    imgre = re.compile(tag)
    img_url_list = imgre.findall(html)
    return set(img_url_list)

def getHTMLText(url,ua):         #获取HTML文本
    try:
        r = requests.get(url,headers=ua)
        r.encoding = r.apparent_encoding
        return r.text
    except:
        return "产生异常"

def save_img(img_list):       #保存文件
    root = "E://pic//"  #保存路径
    for img_url in img_list:
        try:
            if len(img_url) > 100:   #弥补之前正则式的缺陷
                pic = requests.get(img_url,timeout=10)  #设置timeout，避免等待
                print('正在下载：'+img_url[32:60]+'.jpg...')
            else:
                continue
        except requests.exceptions.ConnectionError:
            print('URL错误')
            continue
        path = root  + img_url[32:60] + '.jpg'  #文件命名为: url的32到60位.jpg
        # print(path)
        fp = open(path,'wb')
        fp.write(pic.content)
        fp.close()

if __name__ == "__main__":
    aotourl = 'http://www.bizhi88.com/s/26/'
    url = aotourl    #这个URL是可以分析出来的
    # User-Agent
    ua = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36'}
    """ temp = getHTMLText(url, ua) #这是下载第一页的
    if temp != "产生异常":
        # print(find_img(temp))
        imgs = find_img(temp)
        save_img(imgs) """
    i = 2
    for i in range(5):  #先下到第5页,不想下太多
        url = aotourl + str(i) + '.html'
        temp = getHTMLText(url, ua)
        if temp != "产生异常":
            imgs = find_img(temp)
            save_img(imgs)

这个爬虫只是方便下载图片，适当取用，理性爬取ヾ(◍°∇°◍)ﾉﾞ，另外，福利请拿好：
代码1：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
# 下载某个分组的所有图组
from bs4 import BeautifulSoup
import requests
import re
import os
import time

def getHTMLText(url,head):         #获取HTML文本
    try:
        r = requests.get(url,headers=head)
        r.encoding = r.apparent_encoding
        # print(r.status_code)
        return r.text
    except:
        return "产生异常"

def getKeygroup(html):
    soup = BeautifulSoup(html, "html.parser")
    list = []
    for link in soup.find_all('a'):
        temp = str(link.get('href'))
        if len(temp) == 39:
            list.append(temp)
        else:
            continue
    return list

def setmenu(path):
    if os.path.exists(path):
        os.chdir(path)
    else:
        os.mkdir(path)
        os.chdir(path)

def getperpage(list):
    head = {
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',
        'referer': 'https://www.baidu.com/s?ie=utf-8&f=8&rsv_bp=1&tn=baidu&wd=%E7%88%AC%E8%99%AB%E8%AE%BE%E7%BD%AEReferer&oq=mm1%2526lt%253B1%25E5%258F%258D%25E7%2588%25AC&rsv_pq=980badde01100064&rsv_t=940dMaGp%2Bql0SY6yvTJc4wfiibW6O9b%2B1r%2FJuiTLVvc7Fp9hT5jcAIfG0tk&rqlang=cn&rsv_enter=1&rsv_dl=tb&inputT=3890&rsv_sug3=162&rsv_sug1=108&rsv_sug7=100&rsv_n=2&rsv_sug2=0&rsv_sug4=3890'
    }

    path = "E://pic//"
    for img_group in list:
        time.sleep(1)
        r = requests.get(img_group,headers=head, timeout=10)
        soup = BeautifulSoup(r.text, "html.parser")
        num = soup.select('span.page-ch')[0].get_text()[2:4]
        groupname = path + img_group[30:34]
        setmenu(groupname)
        i = 2
        for i in range(int(num)):
            # time.sleep(1)
            img_index = img_group[:34]+'_'+str(i)+'.html'
            img = requests.get(img_index,headers=head, timeout=10)
            img_soup = BeautifulSoup(img.text, 'html.parser')
            img_url = img_soup.select('.content-pic > a > img:nth-of-type(1)')
            if len(img_url) != 0:
                img_r_url = img_soup.select('.content-pic > a > img:nth-of-type(1)')[0].get('src')
                img_get = requests.get(img_r_url,headers=head,timeout=10)
                print('正在下载：'+img_r_url)
                fp = open(groupname + '//'+str(i) + '.jpg','wb')
                fp.write(img_get.content)
                fp.close
            else:
                continue


if __name__ == "__main__":

    head = {
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',
        'referer': 'https://www.baidu.com/s?ie=utf-8&f=8&rsv_bp=1&tn=baidu&wd=%E7%88%AC%E8%99%AB%E8%AE%BE%E7%BD%AEReferer&oq=mm1%2526lt%253B1%25E5%258F%258D%25E7%2588%25AC&rsv_pq=980badde01100064&rsv_t=940dMaGp%2Bql0SY6yvTJc4wfiibW6O9b%2B1r%2FJuiTLVvc7Fp9hT5jcAIfG0tk&rqlang=cn&rsv_enter=1&rsv_dl=tb&inputT=3890&rsv_sug3=162&rsv_sug1=108&rsv_sug7=100&rsv_n=2&rsv_sug2=0&rsv_sug4=3890'
    }
    # 支持后缀是：xinggan,xiaohua,mingxing,
    url = 'https://www.mm131.net/xinggan'
    r = getHTMLText(url, head)
    tp = getKeygroup(r)
    getperpage(tp)
    i = 2
    for i in range(3):
        #xiaohua 这里就改成 /list_2_  , mingxing 就改成 /list_5_
        more_url = url + '/list_6_' + str(i) + '.html'
        r = getHTMLText(more_url, head)
        tp = getKeygroup(r)
        getperpage(tp)

代码2：

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
# 下载某个搜索词的图组
# 注意用time.sleep(),下太快会被封ip
from bs4 import BeautifulSoup
import requests
import re
import os
import time

def getHTMLText(url,head):         #获取HTML文本
    try:
        r = requests.get(url,headers=head)
        r.encoding = r.apparent_encoding
        # print(r.status_code)
        return r.text
    except:
        return "产生异常"

def getKeygroup(html):
    soup = BeautifulSoup(html, "html.parser")
    list = []
    # 据说这是万能匹配 URL 的正则
    tag = r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+'
    imgre = re.compile(tag)
    # 从浏览器copy的selector
    for link in soup.select('body > div:nth-child(2) > div.pleft > div.listbox > ul > li'):
        img_url_list = imgre.findall(str(link))
        # print(img_url_list[0])
        list.append(img_url_list[0])
    return list

def setmenu(path):
    if os.path.exists(path):
        os.chdir(path)
    else:
        os.mkdir(path)
        os.chdir(path)

def getperpage(list):
    head = {
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',
        'referer': 'https://www.baidu.com/s?ie=utf-8&f=8&rsv_bp=1&tn=baidu&wd=%E7%88%AC%E8%99%AB%E8%AE%BE%E7%BD%AEReferer&oq=mm1%2526lt%253B1%25E5%258F%258D%25E7%2588%25AC&rsv_pq=980badde01100064&rsv_t=940dMaGp%2Bql0SY6yvTJc4wfiibW6O9b%2B1r%2FJuiTLVvc7Fp9hT5jcAIfG0tk&rqlang=cn&rsv_enter=1&rsv_dl=tb&inputT=3890&rsv_sug3=162&rsv_sug1=108&rsv_sug7=100&rsv_n=2&rsv_sug2=0&rsv_sug4=3890'
    }

    path = "E://pic//"
    for img_group in list:
        time.sleep(1)
        r = requests.get(img_group,headers=head, timeout=10)
        soup = BeautifulSoup(r.text, "html.parser")
        num = soup.select('span.page-ch')[0].get_text()[2:4]
        groupname = path + img_group[30:34]
        setmenu(groupname)
        i = 2
        for i in range(int(num)):
            time.sleep(1)
            img_index = img_group[:34]+'_'+str(i)+'.html'
            img = requests.get(img_index,headers=head, timeout=10)
            img_soup = BeautifulSoup(img.text, 'html.parser')
            img_url = img_soup.select('.content-pic > a > img:nth-of-type(1)')
            if len(img_url) != 0:
                img_r_url = img_soup.select('.content-pic > a > img:nth-of-type(1)')[0].get('src')
                img_get = requests.get(img_r_url,headers=head,timeout=10)
                print('正在下载：'+img_r_url)
                fp = open(groupname + '//'+str(i) + '.jpg','wb')
                fp.write(img_get.content)
                fp.close
            else:
                continue


if __name__ == "__main__":

    head = {
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',
        'referer': 'https://www.baidu.com/s?ie=utf-8&f=8&rsv_bp=1&tn=baidu&wd=%E7%88%AC%E8%99%AB%E8%AE%BE%E7%BD%AEReferer&oq=mm1%2526lt%253B1%25E5%258F%258D%25E7%2588%25AC&rsv_pq=980badde01100064&rsv_t=940dMaGp%2Bql0SY6yvTJc4wfiibW6O9b%2B1r%2FJuiTLVvc7Fp9hT5jcAIfG0tk&rqlang=cn&rsv_enter=1&rsv_dl=tb&inputT=3890&rsv_sug3=162&rsv_sug1=108&rsv_sug7=100&rsv_n=2&rsv_sug2=0&rsv_sug4=3890'
    }

    url = 'https://www.mm131.net/search/?key=%E6%A7%BC%BA&page='
    i = 1
    while i<=3:
        more_url = url + str(i)
        r = getHTMLText(more_url, head)
        tp = getKeygroup(r)
        getperpage(tp)

(会不会用看造化.jpg)

timooadhufg.jpg

保重身体！！！ 罒ω罒

赞

赏杯Java？٩(๑^o^๑)۶

创作不易，感谢支持！

扫一扫，分享到微信

微信分享二维码

Show Tags:

    缺失模块。
    1、请确保node版本大于6.2
    2、在博客根目录（注意不是yilia根目录）执行以下命令：
    npm i hexo-generator-json-content --save

    3、在根目录_config.yml里添加配置： 

      jsonContent:
    meta: false
    pages: false
    posts:
      title: true
      date: true
      path: true
      text: false
      raw: false
      content: false
      slug: false
      updated: false
      comments: false
      link: false
      permalink: false
      excerpt: false
      categories: false
      tags: true

<br>我必须承认<br><br>生命中大部分时光是属于孤独的，<br><br>努力成长<br><br>是在孤独里可以进行的最好的游戏。