用 Golang 爬取豆瓣电影 Top250 列表

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84

package main

import (
	"fmt"
	"io"
	"net/http"
	"os"
	"strconv"
	"strings"

	"github.com/PuerkitoBio/goquery"
	"github.com/WeiZhang555/tabwriter"
)

var config struct {
	UserAgent string
}

func init() {
	config.UserAgent = `Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:75.0) Gecko/20100101 Firefox/75.0`
}

// Item ...
type Item struct {
	Titles []string
	Score  float64
	Link   string
	Type   string
	Quote  string
}

// page starts from 1
func scrape(page int) []*Item {
	offset := (page - 1) * 25
	u := fmt.Sprintf(`https://movie.douban.com/top250?start=%d`, offset)
	req, err := http.NewRequest(http.MethodGet, u, nil)
	if err != nil {
		panic(err)
	}
	req.Header.Set(`User-Agent`, config.UserAgent)
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		io.Copy(os.Stderr, resp.Body)
		panic(resp.Status)
	}
	doc, err := goquery.NewDocumentFromReader(resp.Body)
	if err != nil {
		panic(err)
	}

	var items []*Item
	doc.Find(`#content .article ol.grid_view > li`).Each(func(i int, s *goquery.Selection) {
		var item Item
		a := s.Find(`.hd > a`)
		item.Link, _ = a.Attr(`href`)
		for _, t := range strings.Split(a.Text(), `/`) {
			item.Titles = append(item.Titles, strings.TrimSpace(t))
		}
		item.Type = strings.TrimSpace(strings.Split(s.Find(`.bd > p`).Text(), "\n")[2])
		item.Score, _ = strconv.ParseFloat(s.Find(`.bd .rating_num`).Text(), 64)
		item.Quote = s.Find(`.bd .quote span`).Text()
		items = append(items, &item)
	})
	return items
}

func main() {
	var items []*Item
	for i := 1; i <= 10; i++ {
		items = append(items, scrape(i)...)
	}
	w := tabwriter.NewWriter(os.Stdout, 8, 8, 3, ' ', 0)
	defer w.Flush()
	for _, item := range items {
		fmt.Fprintf(w,
			"%s\t%.1f\t%s\t%s\t%s\n",
			item.Titles[0], item.Score, item.Link, item.Type, item.Quote,
		)
	}
}

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128

package main

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"sync"
	"time"
)

type Item struct {
	Filename string `json:"fileName"`
	FileID   string `json:"fileId"`
	FileSize int64  `json:"fileSize"`
}

var downloadPrefix = `https://189.ly93.cc/`
var shareCode = `bMZRzufYvUnu`

func main() {
	action := `https://189.ly93.cc/TianYi.php`
	values := url.Values{}
	values.Set(`page`, `1`)
	values.Set(`limit`, `300`) // 拉取全部
	values.Set(`c`, `list`)
	values.Set(`accessToken`, ``)
	values.Set(`shareCode`, shareCode)
	values.Set(`fileId`, ``)
	values.Set(`accessCode`, ``)
	rsp, err := http.PostForm(action, values)
	if err != nil {
		panic(err)
	}
	defer rsp.Body.Close()

	var r struct {
		Code int     `json:"code"`
		Data []*Item `json:"data"`
	}

	bom := [3]byte{}
	io.ReadFull(rsp.Body, bom[:])

	if err := json.NewDecoder(rsp.Body).Decode(&r); err != nil {
		panic(err)
	}

	if r.Code != 0 {
		panic(`code != 0`)
	}

	// 所有待下载的链接放这里面
	ch := make(chan *Item, len(r.Data))
	defer close(ch)

	for _, d := range r.Data {
		ch <- d
	}

	// 启动下载线程
	const nWorkers = 20
	wg := &sync.WaitGroup{}
	for i := 0; i < nWorkers; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for {
				select {
				case item := <-ch:
					if download(item) {
						continue
					}
					ch <- item
					time.Sleep(time.Second)
				case <-time.After(time.Second * 3):
					log.Printf("no item, exiting")
					return
				}
			}

		}()
	}
	wg.Wait()
}

func download(item *Item) bool {
	downloadURL := downloadPrefix + shareCode + "/" + item.FileID
	stat, err := os.Stat(item.Filename)
	if err == nil {
		if stat.Size() == item.FileSize {
			fmt.Printf("Download complete: %s\n", item.Filename)
			return true
		}
	}

	c := http.Client{
		CheckRedirect: func(req *http.Request, via []*http.Request) error {
			return http.ErrUseLastResponse
		},
	}
	rsp, err := c.Get(downloadURL)
	if err != nil {
		log.Printf("get link error: %v", err)
		return false
	}
	defer rsp.Body.Close()
	if rsp.StatusCode != 301 && rsp.StatusCode != 302 {
		log.Printf("code %d != 301/2, %s %s\n", rsp.StatusCode, downloadURL, item.Filename)
		io.Copy(log.Writer(), rsp.Body)
		return false
	}
	u := rsp.Header.Get(`Location`)
	cmd := exec.Command(`wget`, `--continue`, `-O`, item.Filename, u)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	fmt.Println(cmd.String())
	if err := cmd.Run(); err != nil {
		log.Printf("%v (%s)\n", err, item.Filename)
		return false
	}
	return true
}