用 Golang 爬取豆瓣电影 Top250 列表

写了段代码把豆瓣电影 Top250 的列表给爬下来了，方便搜索下载。

当前的列表如下（按评分从高至低排序）：

完整 Go 语言代码（gist）：

package main

import (
	"fmt"
	"io"
	"net/http"
	"os"
	"strconv"
	"strings"

	"github.com/PuerkitoBio/goquery"
	"github.com/WeiZhang555/tabwriter"
)

var config struct {
	UserAgent string
}

func init() {
	config.UserAgent = `Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:75.0) Gecko/20100101 Firefox/75.0`
}

// Item ...
type Item struct {
	Titles []string
	Score  float64
	Link   string
	Type   string
	Quote  string
}

// page starts from 1
func scrape(page int) []*Item {
	offset := (page - 1) * 25
	u := fmt.Sprintf(`https://movie.douban.com/top250?start=%d`, offset)
	req, err := http.NewRequest(http.MethodGet, u, nil)
	if err != nil {
		panic(err)
	}
	req.Header.Set(`User-Agent`, config.UserAgent)
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		panic(err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		io.Copy(os.Stderr, resp.Body)
		panic(resp.Status)
	}
	doc, err := goquery.NewDocumentFromReader(resp.Body)
	if err != nil {
		panic(err)
	}

	var items []*Item
	doc.Find(`#content .article ol.grid_view > li`).Each(func(i int, s *goquery.Selection) {
		var item Item
		a := s.Find(`.hd > a`)
		item.Link, _ = a.Attr(`href`)
		for _, t := range strings.Split(a.Text(), `/`) {
			item.Titles = append(item.Titles, strings.TrimSpace(t))
		}
		item.Type = strings.TrimSpace(strings.Split(s.Find(`.bd > p`).Text(), "\n")[2])
		item.Score, _ = strconv.ParseFloat(s.Find(`.bd .rating_num`).Text(), 64)
		item.Quote = s.Find(`.bd .quote span`).Text()
		items = append(items, &item)
	})
	return items
}

func main() {
	var items []*Item
	for i := 1; i <= 10; i++ {
		items = append(items, scrape(i)...)
	}
	w := tabwriter.NewWriter(os.Stdout, 8, 8, 3, ' ', 0)
	defer w.Flush()
	for _, item := range items {
		fmt.Fprintf(w,
			"%s\t%.1f\t%s\t%s\t%s\n",
			item.Titles[0], item.Score, item.Link, item.Type, item.Quote,
		)
	}
}

更新：

推上有人分享了全部电影的单个下载链接： 但是不能批量下载，于是我就再写了一段代码，转换成直接下载链接后用 wget 在服务器上下载：

代码下载：top250_2.go。

package main

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"sync"
	"time"
)

type Item struct {
	Filename string `json:"fileName"`
	FileID   string `json:"fileId"`
	FileSize int64  `json:"fileSize"`
}

var downloadPrefix = `https://189.ly93.cc/`
var shareCode = `bMZRzufYvUnu`

func main() {
	action := `https://189.ly93.cc/TianYi.php`
	values := url.Values{}
	values.Set(`page`, `1`)
	values.Set(`limit`, `300`) // 拉取全部
	values.Set(`c`, `list`)
	values.Set(`accessToken`, ``)
	values.Set(`shareCode`, shareCode)
	values.Set(`fileId`, ``)
	values.Set(`accessCode`, ``)
	rsp, err := http.PostForm(action, values)
	if err != nil {
		panic(err)
	}
	defer rsp.Body.Close()

	var r struct {
		Code int     `json:"code"`
		Data []*Item `json:"data"`
	}

	bom := [3]byte{}
	io.ReadFull(rsp.Body, bom[:])

	if err := json.NewDecoder(rsp.Body).Decode(&r); err != nil {
		panic(err)
	}

	if r.Code != 0 {
		panic(`code != 0`)
	}

	// 所有待下载的链接放这里面
	ch := make(chan *Item, len(r.Data))
	defer close(ch)

	for _, d := range r.Data {
		ch <- d
	}

	// 启动下载线程
	const nWorkers = 20
	wg := &sync.WaitGroup{}
	for i := 0; i < nWorkers; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for {
				select {
				case item := <-ch:
					if download(item) {
						continue
					}
					ch <- item
					time.Sleep(time.Second)
				case <-time.After(time.Second * 3):
					log.Printf("no item, exiting")
					return
				}
			}

		}()
	}
	wg.Wait()
}

func download(item *Item) bool {
	downloadURL := downloadPrefix + shareCode + "/" + item.FileID
	stat, err := os.Stat(item.Filename)
	if err == nil {
		if stat.Size() == item.FileSize {
			fmt.Printf("Download complete: %s\n", item.Filename)
			return true
		}
	}

	c := http.Client{
		CheckRedirect: func(req *http.Request, via []*http.Request) error {
			return http.ErrUseLastResponse
		},
	}
	rsp, err := c.Get(downloadURL)
	if err != nil {
		log.Printf("get link error: %v", err)
		return false
	}
	defer rsp.Body.Close()
	if rsp.StatusCode != 301 && rsp.StatusCode != 302 {
		log.Printf("code %d != 301/2, %s %s\n", rsp.StatusCode, downloadURL, item.Filename)
		io.Copy(log.Writer(), rsp.Body)
		return false
	}
	u := rsp.Header.Get(`Location`)
	cmd := exec.Command(`wget`, `--continue`, `-O`, item.Filename, u)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	fmt.Println(cmd.String())
	if err := cmd.Run(); err != nil {
		log.Printf("%v (%s)\n", err, item.Filename)
		return false
	}
	return true
}