Skip to content

Commit

Permalink
fixed https proxy not working, added proxy pool
Browse files Browse the repository at this point in the history
  • Loading branch information
@Albresky
Albresky committed Oct 14, 2022
1 parent adee939 commit 8897e7d
Show file tree
Hide file tree
Showing 2 changed files with 52 additions and 39 deletions.
62 changes: 34 additions & 28 deletions src/main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@


import argparse
import logging
from time import sleep

import requests
Expand All @@ -24,65 +25,70 @@
class MomoFlash:
def __init__(self, url='', times=30, delay=5):
self.proxyFetcher = ProxyFetcher()
self.freeProxysGen = None
self.freeProxysGen = []
self.proxys = []
self.shareTargetTimes = times
self.sharedTimes = 0
self.proxyNum = 30
self.delay = delay
self.url = url

self.trials = 0
self.genProxys()

def getProxys(self):
for proxy in self.freeProxysGen:
self.proxys.append(proxy)
for proxyPool in self.freeProxysGen:
for proxy in proxyPool:
self.proxys.append(proxy)

def genProxys(self):
print('\n开始获取代理IP...\n')
if len(self.proxys) < self.proxyNum:
self.freeProxysGen = self.proxyFetcher.freeProxy02()
self.getProxys()
if len(self.proxys) < self.proxyNum:
self.freeProxysGen = self.proxyFetcher.freeProxy03()
self.getProxys()
if len(self.proxys) < self.proxyNum:
self.freeProxysGen = self.proxyFetcher.freeProxy05()
self.getProxys()
if len(self.proxys) < self.proxyNum:
self.freeProxysGen = self.proxyFetcher.freeProxy06()
self.getProxys()
if len(self.proxys) < self.proxyNum:
self.freeProxysGen = self.proxyFetcher.freeProxy07()
self.getProxys()
self.freeProxysGen.append(self.proxyFetcher.freeProxy02())
self.freeProxysGen.append(self.proxyFetcher.freeProxy03())
self.freeProxysGen.append(self.proxyFetcher.freeProxy05())
self.freeProxysGen.append(self.proxyFetcher.freeProxy06())
self.freeProxysGen.append(self.proxyFetcher.freeProxy07())
self.freeProxysGen.append(self.proxyFetcher.freeProxy09())
self.freeProxysGen.append(self.proxyFetcher.freeProxy10())
self.getProxys()
print('\n代理IP获取完毕，共获取{}个代理IP\n'.format(len(self.proxys)))

def momo_flash(self):
print("\n开始刷墨墨背单词分享链接...\n")
print("当前代理IP池数量：{}\n".format(len(self.proxys)))
print("当前刷新延迟：{}ms\n".format(self.delay))
for url in self.url:
if len(self.proxys) < self.shareTargetTimes:
self.genProxys()
self.sharedTimes = 0
for i in range(self.shareTargetTimes):
self.trials = 0
while self.sharedTimes < self.shareTargetTimes:
if len(self.proxys) == 0:
self.genProxys()
try:
self.trials += 1
nowProxy = {"https": "https://{}".format(self.proxys[-1]),
"http": "http://{}".format(self.proxys[-1])}
print("[{}]当前代理IP：{}".format(self.trials, nowProxy))
resp = requests.get(url=url, headers=utils.Headers.header(),
proxies={"http": "http://{}".format(self.proxys[-1])}, verify=False)
proxies=nowProxy, verify=False, timeout=1)
if resp.status_code == 200:
print(resp.text)
self.sharedTimes += 1
print('成功分享第{}次，using proxy[{}]'.format(self.sharedTimes, self.proxys[-1]))
print('成功分享第{}次，using proxy[{}]\n'.format(self.sharedTimes, self.proxys[-1]))
else:
print('分享失败，using proxy[{}]'.format(self.proxys[-1]))
print('分享失败， 代理超时=>[{}]\n'.format(self.proxys[-1]))
except Exception as e:
print('分享失败，代理失效=>[{}]\n'.format(self.proxys[-1]))
logging.debug(e)
finally:
self.proxys.pop()
sleep(self.delay)
except Exception as e:
print(e)
print('*' * 30 + '\n分享结束，共分享{}次\n'.format(self.sharedTimes))
print("成功【{}】次，当前任务：【{}】".format(self.sharedTimes, url))

print('*' * 30 + '\n分享{}结束，共分享{}次\n'.format(url, self.sharedTimes))


if __name__ == '__main__':
print("请不要使用VPN等代理工具，否则可能会导致代理IP获取失败\n")
momoFlash = MomoFlash(args.url, args.num, args.delay)
# momoFlash = MomoFlash(["http://httpbin.org/ip"], 10, 1)
momoFlash.momo_flash()
input('按任意键退出...')
29 changes: 18 additions & 11 deletions src/proxyFetcher.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,9 +59,12 @@ def freeProxy03():
for url in target_urls:
tree = WebRequest().get(url).tree
for tr in tree.xpath("//table[@class='active']//tr")[1:]:
ip = "".join(tr.xpath('./td[1]/text()')).strip()
port = "".join(tr.xpath('./td[2]/text()')).strip()
yield "%s:%s" % (ip, port)
protocol = "".join(tr.xpath('./td[4]/text()')).strip()
if protocol == "HTTP,HTTPS":
ip = "".join(tr.xpath('./td[1]/text()')).strip()
port = "".join(tr.xpath('./td[2]/text()')).strip()
yield "%s:%s" % (ip, port)


@staticmethod
def freeProxy04():
Expand Down Expand Up @@ -111,9 +114,10 @@ def freeProxy06():
resp_text = WebRequest().get(url).text
for each in resp_text.split("\n"):
json_info = json.loads(each)
if json_info.get("country") == "CN":
yield "%s:%s" % (json_info.get("host", ""), json_info.get("port", ""))
# if json_info.get("country") == "CN":
yield "%s:%s" % (json_info.get("host", ""), json_info.get("port", ""))
except Exception as e:
pass
print(e)

@staticmethod
Expand Down Expand Up @@ -150,9 +154,12 @@ def freeProxy09(page_count=1):
@staticmethod
def freeProxy10():
""" 89免费代理 """
r = WebRequest().get("https://www.89ip.cn/index_1.html", timeout=10)
proxies = re.findall(
r'<td.*?>[\s\S]*?(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})[\s\S]*?</td>[\s\S]*?<td.*?>[\s\S]*?(\d+)[\s\S]*?</td>',
r.text)
for proxy in proxies:
yield ':'.join(proxy)

for i in range(21)[1:]:
url='http://www.89ip.cn/index_{}.html'.format(i)
r = WebRequest().get(url, timeout=10)
proxies = re.findall(r'<td.*?>[\s\S]*?(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})[\s\S]*?</td>[\s\S]*?<td.*?>[\s\S]*?(\d+)[\s\S]*?</td>',r.text)
if len(proxies)==0:
continue
for proxy in proxies:
yield ':'.join(proxy)

0 comments on commit 8897e7d

Please sign in to comment.