[{"_1":2,"_175":-5,"_176":-5},"loaderData",{"_3":4,"_17":18,"_25":26,"_150":151},"root",{"_5":-5,"_6":-7,"_7":8,"_9":-7,"_10":11,"_12":-7,"_13":14,"_15":16},"currentUser","primaryEmail","siteVerification","6LeIxAcTAAAAAJcZVRqyHh71UMIEGNQ_MXjiZKhI","toast","enableDev",false,"incidentStatus","theme","system","contrast","normal","routes/_layout",{"_19":-5,"_20":21,"_22":21,"_23":11,"_24":21},"username","isAskGordonEnabled",true,"isUsingNewTemplate","showMinimalLanding","showHoneypotTrap","routes/_layout.r.$namespace.$reponame",{"_27":28,"_29":-5,"_30":11,"_31":11,"_32":33,"_34":35,"_36":-5,"_37":11,"_38":39,"_88":-5,"_89":90,"_139":-5,"_140":11,"_141":21,"_142":11,"_143":-5,"_144":11,"_145":-5,"_146":147,"_148":91,"_149":-7},"canonicalUrl","https://hub.docker.com/r/ckeyer/proxypool/tags","mcpServer","canSeeDockerfile","canSeeAutobuilds","namespace","ckeyer","reponame","proxypool","cloudBuildSource","isAutobuildsAvailable","repo",{"_40":41,"_42":43,"_45":46,"_47":48,"_49":11,"_50":51,"_52":53,"_54":55,"_57":35,"_32":33,"_58":59,"_60":44,"_61":62,"_63":62,"_64":65,"_66":-5,"_67":68,"_69":70,"_71":11,"_72":33,"_73":74,"_79":21,"_80":81,"_85":33,"_86":87},"categories",[],"content_types",[44],"image","date_registered","2016-12-09T16:25:17.026556Z","description","proxy pool","is_private","last_updated","2018-05-26T11:21:32.378014Z","last_modified","2024-10-16T13:48:34.145251Z","media_types",[56],"application/vnd.docker.container.image.v1+json","name","pull_count",1790,"repository_type","star_count",1,"status","status_description","active","source","collaborator_count",0,"full_description","# IPProxyPool\nIPProxyPool代理池项目，提供代理ip。支持py2和py3两个版本。\n### 我的新书[《Python爬虫开发与项目实战》](https://item.jd.com/12206762.html)出版了,喜欢的话可以看一下[样章](http://pan.baidu.com/s/1hrWEOYg)\n
\n详细使用方式，请看我的博客:\nhttp://www.cnblogs.com/qiyeboy/p/5693128.html\n
\n最近正在为IPProxyPool添加二级代理，方便调度。大家可以关注我的公众号，更新我会及时通知。\n
\n\n#### 我的微信公众号:\n\n![](qiye2.jpg)\n
\n希望大家提供更多的代理网站，现在爬取的好用的代理ip还是太少。\n
\n同时感谢[super1-chen](https://github.com/super1-chen),[fancoo](https://github.com/fancoo),[Leibnizhu](https://github.com/Leibnizhu)对项目的贡献。\n
\n\n## 项目依赖\n\n#### Ubuntu,debian\n\n1.安装sqlite数据库(一般系统内置):\napt-get install sqlite3\n
\n2.安装requests,chardet,web.py,gevent psutil:\npip install requests chardet web.py sqlalchemy gevent psutil\n
\n3.安装lxml:\napt-get install python-lxml\n
\n注意：\n\n* python3下的是pip3\n* 有时候使用的gevent版本过低会出现自动退出情况，请使用pip install gevent --upgrade更新)\n* 在python3中安装web.py，不能使用pip，直接下载py3版本的[源码](https://codeload.github.com/webpy/webpy/zip/py3)进行安装\n\n#### Windows\n\n1.下载[sqlite](http://www.sqlite.org/download.html),路径添加到环境变量\n
\n2.安装requests,chardet,web.py,gevent:\npip install requests chardet web.py sqlalchemy gevent\n
\n3.安装lxml:\npip install lxml或者下载[lxml windows版](https://pypi.python.org/pypi/lxml/)\n
\n注意：\n\n* python3下的是pip3\n* 有时候使用的gevent版本过低会出现自动退出情况，请使用pip install gevent --upgrade更新)\n* 在python3中安装web.py，不能使用pip，直接下载py3版本的[源码](https://codeload.github.com/webpy/webpy/zip/py3)进行安装\n\n#### 扩展说明\n\n本项目默认数据库是sqlite，但是采用sqlalchemy的ORM模型，通过预留接口可以拓展使用MySQL，MongoDB等数据库。\n配置方法：\n
\n1.MySQL配置\n```\n第一步：首先安装MySQL数据库并启动\n第二步：安装MySQLdb或者pymysql(推荐)\n第三步：在config.py文件中配置DB_CONFIG。如果安装的是MySQLdb模块，配置如下：\n DB_CONFIG={\n 'DB_CONNECT_TYPE':'sqlalchemy',\n 'DB_CONNECT_STRING':'mysql+mysqldb://root:root@localhost/proxy?charset=utf8'\n }\n 如果安装的是pymysql模块，配置如下：\n DB_CONFIG={\n 'DB_CONNECT_TYPE':'sqlalchemy',\n 'DB_CONNECT_STRING':'mysql+pymysql://root:root@localhost/proxy?charset=utf8'\n }\n```\nsqlalchemy下的DB_CONNECT_STRING参考[支持数据库](http://docs.sqlalchemy.org/en/latest/core/engines.html#supported-databases)，理论上使用这种配置方式不只是适配MySQL，sqlalchemy支持的数据库都可以，但是仅仅测试过MySQL。\n
\n2.MongoDB配置\n```\n第一步：首先安装MongoDB数据库并启动\n第二步：安装pymongo模块\n第三步：在config.py文件中配置DB_CONFIG。配置类似如下：\n DB_CONFIG={\n 'DB_CONNECT_TYPE':'pymongo',\n 'DB_CONNECT_STRING':'mongodb://localhost:27017/'\n }\n```\n由于sqlalchemy并不支持MongoDB,因此额外添加了pymongo模式，DB_CONNECT_STRING参考pymongo的连接字符串。\n\n##### 注意\n\n如果大家想拓展其他数据库，可以直接继承db下ISqlHelper类，实现其中的方法，具体实现参考我的代码，然后在DataStore中导入类即可。\n```\ntry:\n if DB_CONFIG['DB_CONNECT_TYPE'] == 'pymongo':\n from db.MongoHelper import MongoHelper as SqlHelper\n else:\n from db.SqlHelper import SqlHelper as SqlHelper\n sqlhelper = SqlHelper()\n sqlhelper.init_db()\nexcept Exception,e:\n raise Con_DB_Fail\n```\n有感兴趣的朋友，可以将Redis的实现方式添加进来。\n\n\n## 如何使用\n\n### 本地使用\n\n将项目目录clone到当前文件夹\n\n$ git clone \n\n切换工程目录\n\n```\n$ cd IPProxyPool\n```\n\n运行脚本\n\n```\npython IPProxy.py\n```\n成功运行后，打印信息\n```\nIPProxyPool----->>>>>>>>beginning\nhttp://0.0.0.0:8000/\nIPProxyPool----->>>>>>>>db exists ip:0\nIPProxyPool----->>>>>>>>now ip num < MINNUM,start crawling...\nIPProxyPool----->>>>>>>>Success ip num :134,Fail ip num:7882\n```\n\n### 使用docker容器\n\n$ git clone \n\n切换工程目录\n\n```\n$ cd IPProxys\n$ docker build -t ipproxy .\n$ docker run --name ipProxyPool -d -p 8080:8080 ipproxy\n```\n\n## API 使用方法\n\n#### 第一种模式\n```\nGET /\n```\n这种模式用于查询代理ip数据，同时加入评分机制，返回数据的顺序是按照评分由高到低，速度由快到慢制定的。\n\n#### 参数 \n\n| Name | Type | Description |\n| ----| ---- | ---- |\n| types | int | 0: 高匿,1:匿名,2 透明 |\n| protocol | int | 0: http, 1 https, 2 http/https |\n| count | int | 数量 |\n| country | str | 取值为国内, 国外 |\n| area | str | 地区 |\n\n\n\n#### 例子\n\n##### IPProxys默认端口为8000,端口可以在config.py中配置。\n\n##### 如果是在本机上测试：\n\n1.获取5个ip地址在中国的高匿代理：http://127.0.0.1:8000/?types=0&count=5&country=国内\n
\n2.响应为JSON格式，按照评分由高到低，响应速度由高到低的顺序，返回数据：\n
\n```\n[[\"122.226.189.55\", 138, 10], [\"183.61.236.54\", 3128, 10], [\"61.132.241.109\", 808, 10], [\"183.61.236.53\", 3128, 10], [\"122.227.246.102\", 808, 10]]\n```\n
\n以[\"122.226.189.55\", 138, 10]为例，第一个元素是ip,第二个元素是port，第三个元素是分值score。\n\n```\nimport requests\nimport json\nr = requests.get('http://127.0.0.1:8000/?types=0&count=5&country=国内')\nip_ports = json.loads(r.text)\nprint ip_ports\nip = ip_ports[0][0]\nport = ip_ports[0][1]\nproxies={\n 'http':'http://%s:%s'%(ip,port),\n 'https':'http://%s:%s'%(ip,port)\n}\nr = requests.get('http://ip.chinaz.com/',proxies=proxies)\nr.encoding='utf-8'\nprint r.text\n```\n#### 第二种模式\n```\nGET /delete\n```\n这种模式用于方便用户根据自己的需求删除代理ip数据\n\n#### 参数 \n\n| Name | Type | Description |\n| ----| ---- | ---- |\n| ip | str | 类似192.168.1.1 |\n| port | int | 类似 80 |\n| types | int | 0: 高匿,1:匿名,2 透明 |\n| protocol | int | 0: http, 1 https, 2 http/https |\n| count | int | 数量 |\n| country | str | 取值为国内, 国外 |\n| area | str | 地区 |\n\n大家可以根据指定以上一种或几种方式删除数据。\n\n#### 例子\n\n##### 如果是在本机上测试：\n\n1.删除ip为120.92.3.127的代理：http://127.0.0.1:8000/delete?ip=120.92.3.127\n
\n2.响应为JSON格式，返回删除的结果为成功,失败或者返回删除的个数,类似如下的效果：\n[\"deleteNum\", \"ok\"]或者[\"deleteNum\", 1]\n```\nimport requests\nr = requests.get('http://127.0.0.1:8000/delete?ip=120.92.3.127')\nprint r.text\n```\n## config.py参数配置\n```\n#parserList是网址解析规则表,大家可以将发现的代理网址,将提取规则添加到其中,方便爬虫的爬取。\nparserList = [\n {\n 'urls': ['http://www.66ip.cn/%s.html' % n for n in ['index'] + list(range(2, 12))],\n 'type': 'xpath',\n 'pattern': \".//*[@id='main']/div/div[1]/table/tr[position()>1]\",\n 'position': {'ip': './td[1]', 'port': './td[2]', 'type': './td[4]', 'protocol': ''}\n },\n \n ......\n \n \n {\n 'urls': ['http://www.cnproxy.com/proxy%s.html' % i for i in range(1, 11)],\n 'type': 'module',\n 'moduleName': 'CnproxyPraser',\n 'pattern': r'(\\d+\\.\\d+\\.\\d+\\.\\d+)(HTTP|SOCKS4)\\s*',\n 'position': {'ip': 0, 'port': 1, 'type': -1, 'protocol': 2}\n }\n]\n\n#数据库的配置\n\nDB_CONFIG = {\n\n 'DB_CONNECT_TYPE': 'sqlalchemy', # 'pymongo'sqlalchemy;redis\n # 'DB_CONNECT_STRING':'mongodb://localhost:27017/'\n 'DB_CONNECT_STRING': 'sqlite:///' + os.path.dirname(__file__) + '/data/proxy.db'\n # DB_CONNECT_STRING : 'mysql+mysqldb://root:root@localhost/proxy?charset=utf8'\n\n # 'DB_CONNECT_TYPE': 'redis', # 'pymongo'sqlalchemy;redis\n # 'DB_CONNECT_STRING': 'redis://localhost:6379/8',\n\n}\n#THREADNUM为gevent pool的协程数目\nTHREADNUM = 5\n\n#API_PORT为API web服务器的端口\nAPI_PORT = 8000\n\n#爬虫爬取和检测ip的设置条件\n#不需要检测ip是否已经存在，因为会定时清理\n# UPDATE_TIME:每半个小时检测一次是否有代理ip失效\nUPDATE_TIME = 30 * 60 \n\n# 当有效的ip值小于MINNUM时需要启动爬虫进行爬取\nMINNUM = 50 \n\n# socket超时\nTIMEOUT = 5 \n\n\n\n\n#爬虫下载网页的重试次数\nRETRY_TIME = 3\n\n\n#USER_AGENTS 随机头信息,用来突破爬取网站的反爬虫\n\nUSER_AGENTS = [\n \"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)\",\n \"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)\",\n \"Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)\",\n \"Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)\",\n \"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)\",\n \"Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)\",\n \"Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)\",\n ]\n#默认给抓取的ip分配20分,每次连接失败,减一分,直到分数全部扣完从数据库中删除\nDEFAULT_SCORE=10\n\n#CHECK_PROXY变量是为了用户自定义检测代理的函数,，默认是CHECK_PROXY={'function':'checkProxy'}。\n#现在使用检测的网址是httpbin.org,但是即使ip通过了验证和检测\n#也只能说明通过此代理ip可以到达httpbin.org,但是不一定能到达用户爬取的网址\n#因此在这个地方用户可以自己添加检测函数,我以百度为访问网址尝试一下\n#大家可以看一下Validator.py文件中的baidu_check函数和detect_proxy函数就会明白\n\nCHECK_PROXY={'function':'checkProxy'}#{'function':'baidu_check'}\n```\n## TODO\n1.添加squid代理，简化爬虫配置\n
\n\n\n## 更新进度\n-----------------------------2017-4-6----------------------------\n
\n1.更新评分机制。\n
\n* 之前的评分机制是刚添加进来每个代理ip为0分，每隔半个小时检测一次，检测之后依然有效则加分，无效则删除。\n* 现在的评分机制是每个新的代理ip分配10分,每隔半个小时检测一次，检测之后依然有效则分数不变，无效则分数减一,直至为0删除,可以避免由于检测网站不稳定导致的误删。\n\n2.用户可以自定义检测函数,在config.py的CHECK_PROXY变量中可以配置。\n```\nCHECK_PROXY变量是为了用户自定义检测代理的函数，默认是CHECK_PROXY={'function':'checkProxy'}\n现在使用检测的网址是httpbin.org,但是即使ip通过了验证和检测\n也只能说明通过此代理ip可以到达httpbin.org,但是不一定能到达用户爬取的网址\n因此在这个地方用户可以自己添加检测函数,我以百度为访问网址尝试一下\n大家可以看一下Validator.py文件中的baidu_check函数和detect_proxy函数就会明白。\n\nCHECK_PROXY={'function':'baidu_check'}\n```\n3.经过大家的共同努力,彻底解决了僵死进程的问题。\n\n-----------------------------2017-1-16----------------------------\n
\n1.将py2和py3版本合并，并且兼容\n
\n2.修复pymongo查询bug\n
\n-----------------------------2017-1-11----------------------------\n
\n1.使用httpbin.org检测代理ip的高匿性\n
\n2.使用国内和国外作为country的查询条件\n
\n3.修改types和protocol参数，一定要注意protocol的使用，试试访问http://www.baidu.com和https://www.baidu.com\n
\n4.美化代码风格\n
\n-----------------------------2016-12-11----------------------------\n####大规模重构，主要包括以下几个方面：\n1.使用多进程+协程的方式，将爬取和验证的效率提高了50倍以上，可以在几分钟之内获取所有的有效IP\n
\n2.使用web.py作为API服务器，重构HTTP接口\n
\n3.增加Mysql,MongoDB等数据库的适配\n
\n4.增加了三个代理网站\n
\n5.增加评分机制，评比稳定的ip\n
\n6.支持python3\n
\n-----------------------------2016-11-24----------------------------\n
\n1.增加chardet识别网页编码\n
\n2.突破66ip.cn反爬限制\n
\n-----------------------------2016-10-27----------------------------\n
\n1.增加对代理的检测，测试是否能真正访问到网址，实现代理\n
\n2.添加通过正则表达式和加载插件解析网页的方式\n
\n3.又增加一个新的代理网站\n
\n\n-----------------------------2016-7-20----------------------------\n
\n1.修复bug ,将数据库进行压缩\n
\n","has_starred","hub_user","immutable_tags_settings",{"_75":11,"_76":77},"enabled","rules",[78],".*","is_automated","permissions",{"_82":21,"_83":11,"_84":11},"read","write","admin","user","storage_size",358654740,"orgData","recentTags",[91,129],{"_92":44,"_93":94,"_95":96,"_97":98,"_99":100,"_50":118,"_119":94,"_120":33,"_121":56,"_57":122,"_123":124,"_125":109,"_126":118,"_127":65,"_128":21},"content_type","creator",252838,"full_size",50071911,"id",6722627,"images",[101],{"_102":103,"_104":105,"_106":107,"_108":109,"_110":111,"_112":113,"_114":107,"_115":-5,"_116":96,"_63":65,"_117":-5},"architecture","amd64","digest","sha256:9efeec2b3acf8b118208dee5b4a3eade961ae162900f3551682452029fc68d1e","features","","last_pulled","2026-10-10T18:27:34.378890968Z","last_pushed","2018-05-26T11:21:32Z","os","linux","os_features","os_version","size","variant","2018-05-26T11:21:31.918028Z","last_updater","last_updater_username","media_type","latest","repository",1103864,"tag_last_pulled","tag_last_pushed","tag_status","v2",{"_92":44,"_93":94,"_95":130,"_97":131,"_99":132,"_50":137,"_119":94,"_120":33,"_121":56,"_57":138,"_123":124,"_125":135,"_126":137,"_127":65,"_128":21},50025746,14829119,[133],{"_102":103,"_104":134,"_106":107,"_108":135,"_110":136,"_112":113,"_114":107,"_115":-5,"_116":130,"_63":65,"_117":-5},"sha256:b7d56e7f38b53cb1e08c7a807b10688a8bb0676cc96c8c019f0b922b3f45af6c","2026-10-05T11:36:03.790376174Z","2018-05-13T16:42:18Z","2018-05-13T16:42:17.850478Z","add_Dockerfile","sbxKitMetadata","isPotentialSpam","isHubSbxKitsEnabled","showSbxLogoOverride","sbxTrademarkText","displaySecurityMetadata","tagAttestations","tagsHaveAttestations",[],"selectedTag","attestationsUrl","routes/_layout.r.$namespace.$reponame.tags",{"_38":152,"_159":160,"_146":173,"_174":11},{"_40":153,"_42":154,"_45":46,"_47":48,"_49":11,"_50":51,"_52":53,"_54":155,"_57":35,"_32":33,"_58":59,"_60":44,"_61":62,"_63":62,"_64":65,"_66":-5,"_67":68,"_69":70,"_71":11,"_72":33,"_73":156,"_79":21,"_80":158,"_85":33,"_86":87},[],[44],[56],{"_75":11,"_76":157},[78],{"_82":21,"_83":11,"_84":11},"tags",{"_161":162,"_163":-5,"_164":-5,"_165":166},"count",2,"next","previous","results",[167,170],{"_92":44,"_93":94,"_95":96,"_97":98,"_99":168,"_50":118,"_119":94,"_120":33,"_121":56,"_57":122,"_123":124,"_125":109,"_126":118,"_127":65,"_128":21},[169],{"_102":103,"_104":105,"_106":107,"_108":109,"_110":111,"_112":113,"_114":107,"_115":-5,"_116":96,"_63":65,"_117":-5},{"_92":44,"_93":94,"_95":130,"_97":131,"_99":171,"_50":137,"_119":94,"_120":33,"_121":56,"_57":138,"_123":124,"_125":135,"_126":137,"_127":65,"_128":21},[172],{"_102":103,"_104":134,"_106":107,"_108":135,"_110":136,"_112":113,"_114":107,"_115":-5,"_116":130,"_63":65,"_117":-5},[],"canDeleteTags","actionData","errors"]

ckeyer/proxypool

Tags for ckeyer/proxypool