Skip to content

Commit

Permalink
shuffle the proxy list during during task
Browse files Browse the repository at this point in the history
  • Loading branch information
sgrtye committed Oct 8, 2024
1 parent 0677015 commit 1f5dd03
Showing 1 changed file with 9 additions and 3 deletions.
12 changes: 9 additions & 3 deletions novel/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,6 @@ def load_proxies():
if len(proxies) == 0:
raise Exception("No available proxy")

random.shuffle(proxies)

else:
raise Exception("Proxy server not responding")

Expand All @@ -74,13 +72,17 @@ def load_proxies():

def load_books():
books = []

with open(BOOK_PATH, "r") as file:
lines = file.readlines()

for line in lines:
if line.startswith("//"):
continue

number, name = line.strip().split(":")
books.append((number, name))

return books


Expand All @@ -93,6 +95,7 @@ def load_cache():
book_name = set(name for _, name in books)
book_name_previous = set(name + "previous" for _, name in books)
dict_copy = titles.copy()

for book in dict_copy.keys():
if book not in book_name and book not in book_name_previous:
del titles[book]
Expand Down Expand Up @@ -136,7 +139,7 @@ def do_GET(self):
self.end_headers()

payload = {
name: (titles.get(name, 'Unknown'), BOOK_URL.replace("BOOK_ID", id))
name: (titles.get(name, "Unknown"), BOOK_URL.replace("BOOK_ID", id))
for id, name in books
}
response = json.dumps(payload)
Expand Down Expand Up @@ -187,6 +190,9 @@ def start_api_server():

try:
while True:
if j == len(proxies) - 1 and random.randint(0, 9) == 0:
random.shuffle(proxies)

for index in range(len(proxies)):
j = (j + 1) % len(proxies)

Expand Down

0 comments on commit 1f5dd03

Please sign in to comment.