ShuoGG's blog

宅技术

0%

go-爬点Pixiv画师图

发表于

背景

  • 刚好需要某个画师的插画, 故写了个简单无需登录的爬图工具
  • 根据Pixiv画师ID, 爬完直接保存在当前目录的

使用

1
2
3
4
5
6
7
// Fetch完后自动建一个画师ID的目录然后图片就放里面
cli := PixivClient{
    Cli:       http.Client{},
    Proxy:     defaultProxy(), // 不需要代理则nil即可
}
ctx, _ := context.WithTimeout(context.Background(), time.Second*600)
cli.Fetch(ctx, "4462", -1) // 画师ID:4462

完整实现

  • 代码很简单, 就100多行
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
package pixiv

import (
    "context"
    "encoding/json"
    "fmt"
    "golang.org/x/net/proxy"
    "io"
    "io/ioutil"
    "math/rand"
    "net/http"
    "os"
    "strconv"
    "strings"
    "sync"
    "time"
)

type PixivClient struct {
    Cli       http.Client
    Proxy     proxy.Dialer // 不为nil则走proxy
    RandSleep bool         // 随机sleep, 防止爬太快
}

// limit为-1则无限
func (pcli PixivClient) Fetch(ctx context.Context, userId string, limit int) {
    wg := sync.WaitGroup{}
    tasksDone := make(chan struct{})
    illusts, err := pcli.ListIllustByUser(userId)
    if err != nil {
        return
    }
    for i, illust := range illusts {
        if i > limit && limit != -1 {
            break
        }
        wg.Add(1)
        go func(imgUrl, imgId string) {
            defer wg.Done()
            if pcli.RandSleep {
                randSleep(time.Second * 5)
            }
            err := pcli.fetchOne(userId, imgUrl, imgId)
            if err != nil {
                fmt.Printf("fetchOne(%s, %s): %s\n", userId, imgId, err.Error())
                return
            }
            fmt.Printf("%s finish!\n", imgId)
        }(illust.ImageUrls.Large, strconv.Itoa(illust.Id))
    }
    go func() {
        wg.Wait()
        tasksDone <- struct{}{}
    }()
    select {
    case <-tasksDone:
    case <-ctx.Done():
        fmt.Println(ctx.Err())
    }
}

type illustInfo struct {
    Id        int `json:"id"`
    ImageUrls struct {
        Large string `json:"large"`
    } `json:"image_urls"`
}

func (pcli PixivClient) ListIllustByUser(userId string) ([]illustInfo, error) {
    type pagination struct {
        Pages   int `json:"pages"`
        Current int `json:"current"`
        PerPage int `json:"per_page"`
        Total   int `json:"total"`
    }
    type imjad struct {
        Response   []illustInfo `json:"response"`
        Pagination pagination   `json:"pagination"`
    }
    pcli.Cli.Transport = nil // 国内的api无需代理
    getFn := func(pageNo int) (*imjad, error) {
        u := fmt.Sprintf("https://api.imjad.cn/pixiv/v1/?type=member_illust&id=%s&page=%d", userId, pageNo)
        req, _ := http.NewRequest("GET", u, nil)
        req.Header.Set("user-agent",
            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36")
        resp, err := pcli.Cli.Do(req)
        if err != nil {
            return nil, err
        }
        data, err := ioutil.ReadAll(resp.Body)
        if err != nil {
            return nil, err
        }
        var im imjad
        err = json.Unmarshal(data, &im)
        if err != nil {
            return nil, err
        }
        return &im, nil
    }
    im, err := getFn(1)
    if err != nil {
        return nil, err
    }
    illusts := make([]illustInfo, 0)
    for p := 1; p <= im.Pagination.Pages; p++ {
        im, err := getFn(p)
        if err != nil {
            return nil, err
        }
        illusts = append(illusts, im.Response...)
    }
    return illusts, nil
}

func (pcli PixivClient) fetchOne(userId, imgUrl, imgId string) error {
    u := imgUrl
    cli := pcli.Cli
    if pcli.Proxy != nil {
        cli.Transport = pcli.socks5Transport()
    }
    req, _ := http.NewRequest("GET", u, nil)
    req.Header.Set("user-agent",
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36")
    req.Header.Set("Referer", "http://www.pixiv.net/")
    resp, err := cli.Do(req)
    if err != nil {
        return err
    }
    _ = os.Mkdir(userId, os.ModePerm)
    idx := strings.LastIndexFunc(imgUrl, func(r rune) bool {
        if r == '/' {
            return true
        }
        return false
    })
    f, err := os.Create(fmt.Sprintf("%s/%s", userId, imgUrl[idx+1:]))
    if err != nil {
        return err
    }
    defer func() { _ = f.Close() }()
    if _, err = io.Copy(f, resp.Body); err != nil {
        return err
    }
    return nil
}

func (pcli PixivClient) socks5Transport() *http.Transport {
    httpTransport := &http.Transport{}
    httpTransport.Dial = pcli.Proxy.Dial
    return httpTransport
}

func randSleep(max time.Duration) {
    t := rand.Int63n(int64(max))
    time.Sleep(time.Duration(t))
}

func defaultProxy() proxy.Dialer {
    d, err := proxy.SOCKS5("tcp", "127.0.0.1:1080", nil, proxy.Direct)
    if err != nil {
        panic(err)
    }
    return d
}