基于Go语言实现的代理池
/ Api列表/api/proxy?protocol=[http, https] 所有proxy/api/proxy/count proxy数量/api/proxy/random 随机一个proxy// Validator
const (
VerifyUrl = "http://httpbin.org/get"
VerifyHttpsUrl = "https://httpbin.org/get"
TimeOut = 10 * time.Second
ValidTimeOut = 10 * time.Second
ValidCount = 3
)
// 系统每5分钟会重新验证一次数据库中的代理,并更延迟,
// 当池中代理小于 MinPoolNum 时才会重新爬取
const NextValidTime = 5 * time.Minute
const MinPoolNum = 50
// API port
const APIPORT = ":8888"
系统会以设定time定时检查代理,不通过直接抛弃
latency = 三次验证的平均值
func YourProvider(ch chan<- *Proxy) {
...
ch <- &proxy
...
}
schedule/pool.go:52 CrawlerJob
...
providers := []func(ch chan<- *proxy.Proxy){
proxy.Data5uProvider,
proxy.A2uProvider,
proxy.LiuLiuProvider,
YourProvider,
}
...
Apache License 2.0. For more details, please read the LICENSE file.
Content type
Image
Digest
Size
7.1 MB
Last updated
over 7 years ago
docker pull zzde/proxypool