后端Redis 存储

---

package main

import (
	"log"

	"github.com/gocolly/colly"
	"github.com/gocolly/colly/queue"
	"github.com/gocolly/redisstorage"
)

func main() {
	urls := []string{
		"http://httpbin.org/",
		"http://httpbin.org/ip",
		"http://httpbin.org/cookies/set?a=b&c=d",
		"http://httpbin.org/cookies",
	}

	c := colly.NewCollector()

	// 创建一个redis db
	storage := &redisstorage.Storage{
		Address:  "127.0.0.1:6379",
		Password: "",
		DB:       0,
		Prefix:   "httpbin_test",
	}

	// 添加存储到收集器
	err := c.SetStorage(storage)
	if err != nil {
		panic(err)
	}

	// 删除收集器之前的数据
	if err := storage.Clear(); err != nil {
		log.Fatal(err)
	}

	// 关闭 redis client
	defer storage.Client.Close()

	// 用redis存储器 创建一个新的请求队列
	q, _ := queue.New(2, storage)

	c.OnResponse(func(r *colly.Response) {
		log.Println("Cookies:", c.Cookies(r.Request.URL.String()))
	})

	// 添加URL到队列
	for _, u := range urls {
		q.AddURL(u)
	}
	// 消费请求
	q.Run(c)
}