wlc.py

#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
This bot is used for checking external links found at the wiki.

masti's personalized version for polish projects

Call: python pwb.py masti/wlc.py -ignore:401 -ignore:403 -ignore:451 -ignore:500 -ignore:503 -ignore:429 -talk -pt:0 -talk -start:'!'

It checks several pages at once, with a limit set by the config variable
max_external_links, which defaults to 50.

The bot won't change any wiki pages, it will only report dead links such that
people can fix or remove the links themselves.

The bot will store all links found dead in a .dat file in the deadlinks
subdirectory. To avoid the removing of links which are only temporarily
unavailable, the bot ONLY reports links which were reported dead at least
two times, with a time lag of at least one week. Such links will be logged to a
.txt file in the deadlinks subdirectory.

The .txt file uses wiki markup and so it may be useful to post it on the
wiki and then exclude that page from subsequent runs. For example if the
page is named Broken Links, exclude it with '-titleregexnot:^Broken Links$'

After running the bot and waiting for at least one week, you can re-check those
pages where dead links were found, using the -repeat parameter.

In addition to the logging step, it is possible to automatically report dead
links to the talk page of the article where the link was found. To use this
feature, set report_dead_links_on_talk = True in your user-config.py, or
specify "-talk" on the command line. Adding "-notalk" switches this off
irrespective of the configuration variable.

When a link is found alive, it will be removed from the .dat file.

These command line parameters can be used to specify which pages to work on:

&params;

-repeat      Work on all pages were dead links were found before. This is
             useful to confirm that the links are dead after some time (at
             least one week), which is required before the script will report
             the problem.

-namespace   Only process templates in the namespace with the given number or
             name. This parameter may be used multiple times.

-xml         Should be used instead of a simple page fetching method from
             pagegenerators.py for performance and load issues

-xmlstart    Page to start with when using an XML dump

-ignore      HTTP return codes to ignore. Can be provided several times :
                -ignore:401 -ignore:500

Furthermore, the following command line parameters are supported:

-talk        Overrides the report_dead_links_on_talk config variable, enabling
             the feature.

-notalk      Overrides the report_dead_links_on_talk config variable, disabling
             the feature.

-day         Do not report broken link if the link is there only since
             x days or less. If not set, the default is 7 days.

The following config variables are supported:

max_external_links        - The maximum number of web pages that should be
                            loaded simultaneously. You should change this
                            according to your Internet connection speed.
                            Be careful: if it is set too high, the script
                            might get socket errors because your network
                            is congested, and will then think that the page
                            is offline.

report_dead_links_on_talk - If set to true, causes the script to report dead
                            links on the article's talk page if (and ONLY if)
                            the linked page has been unavailable at least two
                            times during a timespan of at least one week.

weblink_dead_days         - sets the timespan (default: one week) after which
                            a dead link will be reported

Syntax examples:
    python pwb.py weblinkchecker -start:!
        Loads all wiki pages in alphabetical order using the Special:Allpages
        feature.

    python pwb.py weblinkchecker -start:Example_page
        Loads all wiki pages using the Special:Allpages feature, starting at
        "Example page"

    python pwb.py weblinkchecker -weblink:www.example.org
        Loads all wiki pages that link to www.example.org

    python pwb.py weblinkchecker Example page
        Only checks links found in the wiki page "Example page"

    python pwb.py weblinkchecker -repeat
        Loads all wiki pages where dead links were found during a prior run
"""
#
# (C) Daniel Herding, 2005
# (C) Pywikibot team, 2005-2017
#
# Distributed under the terms of the MIT license.
#
from __future__ import absolute_import, unicode_literals

import codecs
import datetime
import pickle
import re
import socket
import sys
import threading
import time

from functools import partial
from time import sleep
from warnings import warn

try:
    import memento_client
except ImportError as e:
    memento_client = e

import pywikibot

from pywikibot import (
    comms, i18n, config, pagegenerators, textlib, weblib, config2,
)

from pywikibot.bot import ExistingPageBot, SingleSiteBot
from pywikibot.pagegenerators import (
    XMLDumpPageGenerator as _XMLDumpPageGenerator,
)
from pywikibot.tools import deprecated
from pywikibot.tools.formatter import color_format

import requests

if sys.version_info[0] > 2:
    import http.client as httplib
    import urllib.parse as urlparse
    import urllib.request as urllib

    basestring = (str, )
    unicode = str
else:
    import httplib
    import urllib
    import urlparse

docuReplacements = {
    '&params;': pagegenerators.parameterHelp
}

ignorelist = [
    # Officially reserved for testing, documentation, etc. in
    # https://tools.ietf.org/html/rfc2606#page-2
    # top-level domains:
    re.compile(r'.*[\./@]test(/.*)?'),
    re.compile(r'.*[\./@]example(/.*)?'),
    re.compile(r'.*[\./@]invalid(/.*)?'),
    re.compile(r'.*[\./@]localhost(/.*)?'),
    # second-level domains:
    re.compile(r'.*[\./@]example\.com(/.*)?'),
    re.compile(r'.*[\./@]example\.net(/.*)?'),
    re.compile(r'.*[\./@]example\.org(/.*)?'),

    # Other special cases
    re.compile(r'.*[\./@]berlinonline\.de(/.*)?'),
    # above entry to be manually fixed per request at
    # [[de:Benutzer:BLueFiSH.as/BZ]]
    # bot can't handle their redirects:

    # bot rejected on the site, already archived
    re.compile(r'.*[\./@]web\.archive\.org(/.*)?'),

    # Ignore links containing * in domain name
    # as they are intentionally fake
    re.compile(r'https?\:\/\/\*(/.*)?'),
    # masti's collected exceptions
    re.compile('.*[\./@]gso\.gbv\.de(/.*)?'),  # bot somehow can't handle their redirects
    re.compile('.*[\./@]berlinonline\.de(/.*)?'),  # a de: user wants to fix them by hand and doesn't want them to be deleted, see [[de:Benutzer:BLueFiSH.as/BZ]].
    re.compile('.*[\./@]bodo\.kommune\.no(/.*)?'),  # bot can't handle their redirects
    re.compile('.*[\./@]jpl\.nasa\.gov(/.*)?'),  # bot rejected on the site
    re.compile('.*[\./@]itis\.gov(/.*)?'),  # bot rejected on the site
    re.compile('.*[\./@]cev\.lu(/.*)?'),  # bot rejected on the site
    re.compile('.*[\./@]science\.ksc\.nasa\.gov(/.*)?'),  # very slow response resulting in bot error
    re.compile('.*[\./@]britannica\.com(/.*)?'),  # HTTP redirect loop
    re.compile('.*[\./@]archive\.is(/.*)?'),  # bot rejected on the site (masti)
    re.compile('.*[\./@]archive\.today(/.*)?'),  # bot rejected on the site  (masti)
    re.compile('.*[\./@]archive\.org(/.*)?'),  # bot rejected on the site (masti)
    re.compile('.*[\./@]krs-online\.com\.pl(/.*)?'),  # bot rejected on the site  (masti)
    re.compile('.*[\./@]europarl\.europa\.eu/meps(/.*)?'),  # links redirected  (masti)
    re.compile('.*[\./@]discogs\.com(/.*)?'),  # bot rejected on the site  (masti)
    re.compile('.*[\./@]ted\.europa\.eu(/.*)?'),  # bot rejected on the site  (masti)
    re.compile('.*[\./@]depatisnet\.dpma\.de(/.*)?'),  # bot rejected on the site  (masti)
    re.compile('.*[\./@]dzieje\.pl(/.*)?'),  # bot rejected on the site  (masti)
    re.compile('.*[\./@]deon\.pl(/.*)?'),  # bot rejected on the site  (masti)
    re.compile('.*[\./@]president\.gov\.ua(/.*)?'),  # redirect loop  (masti)
    re.compile('.*[\./@]rada\.gov\.ua(/.*)?'),  # bot rejected on the site  (masti)
    re.compile('.*[\./@]ak\.org\.pl/download/zywoty_swietych\.pdf(/.*)?'),  # well known missing doc  (masti)
    re.compile('.*[\./@]demographia\.com/db-worldua.pdf(/.*)?'),  # well known missing doc  (masti)
    re.compile('.*[\./@]gutenberg\.org(/.*)?'),  # bot rejected on site  (masti)
    re.compile('.*[\./@]census\.gov(/.*)?'),  # bot rejected on site  (masti)
    re.compile('.*[\./@]carfolio\.com(/.*)?'),  # site very slow timeouts  (masti)
    re.compile('.*[\./@]geonames\.usgs\.gov(/.*)?'),  # site very slow timeouts  (masti)
    re.compile('.*[\./@]stratigraphy\.org(/.*)?'),  # site very slow timeouts  (masti)
    re.compile('.*[\./@]youtube\.com/(/.*)?'),  # bot rejected on site  (masti)
    re.compile('.*[\./@]www\.uj\.edu\.pl/documents/10172/24c02901-aecc-4d79-b067-6ab2fa71fb00'),  # bot rejected on site  (masti)
    re.compile('.*[\./@]dlastudenta\.pl(/.*)?'),  # bot rejected on site  (masti)
    re.compile('.*[\./@]automobile-catalog\.com(/.*)?'),  # bot rejected on site  (masti)
    re.compile('.*[\./@]basketball-players\.pointafter\.com(/.*)?'),  # bot rejected on site  (masti)
    re.compile('.*[\./@]nature\.com(/.*)?'),  # bot rejected on site  (masti)
    re.compile('.*[\./@]gamespot\.com(/.*)?'),  # bot rejected on site  (masti)
    re.compile('.*[\./@]www\.muzeumsg\.pl/images/Publikacje_1918_1939/70\.J\.Prochwicz\.pdf(/.*)?'),  # false postive  (masti)
    re.compile('.*[\./@]oxfordmusiconline\.com\/subscriber/(/.*)?'),  # paywall  (masti)
    re.compile('.*[\./@]independent\.ie(/.*)?'),  # bot rejected on site  (masti)
    re.compile('.*[\./@]usosweb\.amu\.edu\.pl(/.*)?'),  # HTTP redirect loop on site  (masti)
    re.compile('.*[\./@]www\.ibdb\.com(/.*)?'), #site automatically redirectong (masti)
    re.compile('.*[\./@]www\.ft\.dk/Folketinget/findMedlem(/.*)?'), # bot rejected on site  (masti)
    re.compile('.*[\./@]nba\.com(/.*)?'),  # bot rejected on site  (masti)
    re.compile('.*[\./@]dlastudenta\.pl(/.*)?'),  # bot rejected on site  (masti)
    re.compile('.*[\./@]automobile-catalog\.com(/.*)?'),  # bot rejected on site  (masti)
    re.compile('.*[\./@]kadra\.pl(/.*)?'),  # bot rejected on site  (masti)
    re.compile('.*[\./@]d-nb\.info(/.*)?'),  # bot rejected on site  (masti)
    re.compile('.*[\./@]allafrica\.com(/.*)?'),  # bot rejected on site  (masti)
    re.compile('.*[\./@]osnews\.pl(/.*)?'),  # bot rejected on site  (masti)
    re.compile('.*[\./@]2lo\.traugutt\.net(/.*)?'),  # bot rejected on site  (masti)
    re.compile('.*[\./@]dzieje\.pl(/.*)?'),  # bot rejected on site  (masti)
    re.compile('.*[\./@]heilsbronn\.de(/.*)?'),  # bot rejected on site  (masti)
    re.compile('.*[\./@]heimenkirch\.de(/.*)?'),  # bot rejected on site  (masti)
    re.compile('.*[\./@]ibdb\.com(/.*)?'),  # redirect  (masti)
    re.compile('.*[\./@]parlamento\.pt(/.*)?'),  # slow response  (masti)
    re.compile('.*[\./@]rottentomatoes\.com(/.*)?'),  # bot rejected on site  (masti)
    re.compile('.*[\./@]ft\.dk(/.*)?'),  # bot rejected on site  (masti)
    re.compile('.*[\./@]tygodnikpowszechny\.pl(/.*)?'),  # bot redirect loop  (masti)
    re.compile('.*[\./@]przewodnik-katolicki\.pl(/.*)?'),  # bot rejected on site  (masti)
    re.compile('.*[\./@]seriea\.pl(/.*)?'),  # slow response  (masti)
    re.compile('.*[\./@]shmetro\.com(/.*)?'),  # slow response  (masti)
    re.compile('.*[\./@]the-sports\.org(/.*)?'),  # slow response  (masti)
    re.compile('.*[\./@]pzhl\.org\.pl/files/absolwencisms\.doc'),  # slow response (masti)
    re.compile('.*[\./@]bs\.sejm\.gov\.pl'),  # slow response (masti)
    re.compile('.*[\./@]metacritic\.com'),  # false positive (masti)
    re.compile('.*[\./@]plantes-botanique\.org'),  # false positive (masti)
    re.compile('.*[\./@]niezalezna\.pl'),  # false positive (masti)
    re.compile('.*[\./@]doi\.org'),  # false positive (masti)
    re.compile('.*[\./@]rateyourmusic\.com'),  # false positive (masti)
    re.compile('.*[\./@]portalsamorzadowy\.pl'),  # false positive (masti)
    re.compile('.*[\./@]ifpicr\.cz'),  # false positive (masti)
    re.compile('.*[\./@]pnf\.pl'),  # false positive (masti)
    re.compile('.*[\./@]ira\.art\.pl'),  # false positive (masti)
    re.compile('.*[\./@]pointstreak\.com'),  # false positive (masti)
    re.compile('.*[\./@]postal-codes\.findthedata\.com'),  # false positive (masti)
    re.compile('.*[\./@]parlamentarny\.pl'),  # bot rejected on site (masti)
    re.compile('.*[\./@]otz\.de/web/zgt/politik/detail/-/specific/Fusionen-im-Altenburger-Land-Kreis-Greiz-und-Saalfeld-Rudolstadt-nun-moeglich-1908536788'),  # false positive (masti)
    re.compile('.*[\./@]soccerbase\.com'),  # bot rejected on site (masti)
    re.compile('.*[\./@]usnews\.com'),  # bot rejected on site (masti)
    re.compile('.*[\./@]archive\.fo'),  # redirect t archive.is (masti)
    re.compile('.*[\./@]researchgate\.net'),  # bot rejected on site (masti)
    re.compile('.*[\./@]pe2014\.pkw\.gov\.pl'),  # bot rejected on site (masti)
    re.compile('.*[\./@]raciborz\.pl'),  # bot rejected on site (masti)
    re.compile('.*[\./@]encysol\.pl'),  # wrong URLs (masti)
    re.compile('.*[\./@]formula3\.co'),  # bot rejected on site (masti)
    re.compile('.*[\./@]senate\.gov'),  # bot rejected on site (masti)
    re.compile('.*[\./@]rektor\.us\.edu\.pl'),  # bot rejected on site (masti)
    re.compile('.*[\./@]upjp2\.edu\.pl'),  # bot rejected on site (masti)
    re.compile('.*[\./@]kyivpost\.com'),  # bot rejected on site (masti)
    re.compile('.*[\./@]inserbia\.info'),  # bot rejected on site (masti)
    re.compile('.*[\./@]plymouthherald\.co\.uk'),  # bot rejected on site (masti)
    re.compile('.*[\./@]aei\.org'),  # bot rejected on site (masti)
    re.compile('.*[\./@]sns\.org\.rs'),  # bot rejected on site (masti)
    re.compile('.*[\./@]sdlp\.ie'),  # bot rejected on site (masti)
    re.compile('.*[\./@]emmelle\.it'),  # bot rejected on site (masti)
    re.compile('.*[\./@]trm\.md'),  # bot rejected on site (masti)
    re.compile('.*[\./@]frontnational\.com'),  # bot rejected on site (masti)
    re.compile('.*[\./@]rzeszow-news\.pl'),  # bot rejected on site (masti)
    re.compile('.*[\./@]tiger\.edu\.pl'),  # bot rejected on site (masti)
    re.compile('.*[\./@]gazetagazeta\.com'),  # bot rejected on site (masti)
    re.compile('.*[\./@]hfhrpol\.waw\.pl'),  # bot rejected on site (masti)
    re.compile('.*[\./@]mae\.ro'),  # bot rejected on site (masti)
    re.compile('.*[\./@]jusbrasil\.com\.br'),  # bot rejected on site (masti)
    re.compile('.*[\./@]hertfordshiremercury\.co\.uk'),  # bot rejected on site (masti)
    re.compile('.*[\./@]hfhr\.org\.pl'),  # bot rejected on site (masti)
    re.compile('.*[\./@]scholar\.google\.pl'),  # bot rejected on site (masti)
    re.compile('.*[\./@]raciborz\.com\.pl'),  # bot rejected on site (masti)
    re.compile('.*[\./@]harku\.ee'),  # bot rejected on site (masti)
    re.compile('.*[\./@]pcworld\.com'),  # bot rejected on site (masti)
    re.compile('.*[\./@]the-athenaeum\.org'),  # bot rejected on site (masti)
    re.compile('.*[\./@]diecezja\.rzeszow\.pl'),  # bot rejected on site (masti)
    re.compile('.*[\./@]wielcy\.pl'),  # bot rejected on site (masti)
    re.compile('.*[\./@]planespotters\.net'),  # bot rejected on site (masti)
    re.compile('.*[\./@]sbc\.org\.pl/dlibra'),  # bot rejected on site (masti)
    re.compile('.*[\./@]spoilertv\.pl'),  # bot rejected on site (masti)
    re.compile('.*[\./@]rain-tree\.com'),  # bot rejected on site (masti)
    re.compile('.*[\./@]epolotsk\.com'),  # bot rejected on site (masti)
    re.compile('.*[\./@]gminawilkow\.pl'),  # bot rejected on site (masti)
    re.compile('.*[\./@]keanemusic\.com'),  # bot rejected on site (masti)
    re.compile('.*[\./@]apa\.org'),  # redirect loop (masti)
    re.compile('.*[\./@]interlude\.hk'),  # timeouts (masti)
    re.compile('.*[\./@]ochtrup\.de'),  # bot rejected on site (masti)
    re.compile('.*[\./@]stadtlohn\.de'),  # bot rejected on site (masti)
    re.compile('.*[\./@]switzenhausen\.eu'),  # bot rejected on site (masti)
    re.compile('.*[\./@]kanalbydgoski\.pl'),  # bot rejected on site (masti)
    re.compile('.*[\./@]mpkolsztyn\.pl'),  # bot rejected on site (masti)
    re.compile('.*[\./@]rehden\.de'),  # bot rejected on site (masti)
    re.compile('.*[\./@]horneburg\.de'),  # bot rejected on site (masti)
    re.compile('.*[\./@]zdw\.lublin\.pl'),  # bot rejected on site (masti)
    re.compile('.*[\./@]torun\.pl'),  # bot rejected on site (masti)
    re.compile('.*[\./@]dovidka\.com\.ua'),  # bot rejected on site (masti)
    re.compile('.*[\./@]minorplanetcenter\.net'),  # bot rejected on site (masti)
    re.compile('.*[\./@]minorplanetcenter\.org'),  # bot rejected on site (masti)
    re.compile('.*[\./@]tablicerejestracyjne\.pl'),  # bot rejected on site (masti)
    re.compile('.*[\./@]leparisien\.fr'),  # bot rejected on site (masti)
    re.compile('.*[\./@]msz\.gov\.pl'),  # bot rejected on site (masti)
    re.compile('.*[\./@]usps\.com'),  # bot rejected on site (masti)
    re.compile('.*[\./@]norden\.org'),  # slow site (masti)
    re.compile('.*[\./@]tulospalvelu\.vaalit\.fi'),  # slow site (masti)
    re.compile('.*[\./@]nasipolitici\.cz'),  # slow site (masti)
    re.compile('.*[\./@]senat\.ro'),  # slow site (masti)
    re.compile('.*[\./@]euroferroviarios\.net'),  # bot rejected on site (masti)
    re.compile('.*[\./@]rozklad\.zdkium\.walbrzych\.pl'),  # bot rejected on site (masti)
    re.compile('.*[\./@]wysokosc\.mapa\.info\.pl'),  # bot rejected on site (masti)
    re.compile('.*[\./@]contraloria\.gob\.pa'),  # bot rejected on site (masti)
    re.compile('.*[\./@]rsssf\.com'),  # bot rejected on site (masti)
    re.compile('.*[\./@]stare-babice\.pl/sites/default/files/attachment/ludnosc_w_podziale_na_miejscowosci_2010_2015.pdf'),  # bot rejected on site (masti)
    re.compile('.*[\./@]mpu\.bydgoszcz\.pl'),  # bot rejected on site (masti)
    re.compile('.*[\./@]boxrec\.com'),  # bot rejected on site (masti)
    re.compile('.*[\./@]leyendablanca\.galeon\.com'),  # bot rejected on site (masti)
    re.compile('.*[\./@]journals\.indexcopernicus\.com'),  # slow site (masti)
    re.compile('.*[\./@]pingzhen\.tycg\.gov\.tw'),  # bot rejected on site (masti)
    re.compile('.*[\./@]demografia\.stat\.gov\.pl/bazademografia/Tables\.aspx'),  # bot rejected on site (masti)
    re.compile('.*[\./@]dlibra\.umcs\.lublin\.pl/dlibra/plain-content\?id=3251'),  # bot rejected on site (masti)
    re.compile('.*[\./@]spsarnow\.pl'),  # bot rejected on site (masti)
    re.compile('.*[\./@]kloczew\.eurzad\.eu'),  # bot rejected on site (masti)
    re.compile('.*[\./@]skisprungschanzen\.com'),  # bot rejected on site (masti)
    re.compile('.*[\./@]esbl\.ee/biograafia'),  # bot rejected on site (masti)
    re.compile('.*[\./@]gimnazjum\.bystrzyca\.eu'),  # bot rejected on site (masti)
    re.compile('.*[\./@]liceum1\.bystrzyca\.eu'),  # bot rejected on site (masti)
    re.compile('.*[\./@]zso\.bystrzyca\.eu'),  # bot rejected on site (masti)
    re.compile('.*[\./@]genealogics\.org/getperson\.php'),  # bot rejected on site (masti)
    re.compile('.*[\./@]orzeczenia\.nsa\.gov\.pl/doc/'),  # bot rejected on site (masti)
    re.compile('.*[\./@]cbssy.sy/new%20web%20site/General_census/census_2004/NH/'),  # bot rejected on site (masti)
    re.compile('.*[\./@]dziennikustaw\.gov\.pl/DU'),  # bot rejected on site (masti)    http://dziennikustaw.gov.pl/du
    re.compile('.*[\./@]dziennikustaw\.gov\.pl/du'),  # bot rejected on site (masti)    http://dziennikustaw.gov.pl/du
    re.compile('.*[\./@]bpi\.co\.uk/bpi-awards/'),  # bot rejected on site (masti)
    re.compile('.*[\./@]bpi\.co\.uk/brit-certified/'),  # bot rejected on site (masti)
    re.compile('.*[\./@]lazarus\.elte\.hu/hun/digkonyv/topo/3felmeres\.htm'),  # bot rejected on site (masti)
    re.compile('.*[\./@]nztop40\.co\.nz/index.php/chart/singles'),  # bot rejected on site (masti)
    re.compile('.*[\./@]amt-franzburg-richtenberg\.de'),  # bot rejected on site (masti)
    re.compile('.*[\./@]historialomzy\.pl/orzel-kolno/'),  # bot rejected on site (masti)
    re.compile('.*[\./@]arquidiocesisdesucre\.org\.bo'),  # bot rejected on site (masti)
    re.compile('.*[\./@]catholic-hierarchy\.org/'),  # bot rejected on site (masti)
    re.compile('.*[\./@]catholic-hierarchy\.org/bishop'),  # bot rejected on site (masti)
    re.compile('.*[\./@]catholic-hierarchy\.org/diocese'),  # bot rejected on site (masti)
    re.compile('.*[\./@]tokarczuk\.wydawnictwoliterackie\.pl'),  # bot rejected on site (masti)
    re.compile('.*[\./@]edziennik\.poznan\.uw\.gov\.pl'),  # bot rejected on site (masti)
    re.compile('.*[\./@]transport\.gov\.mg'),  # bot rejected on site (masti)
    re.compile('.*[\./@]eco\.gov\.az/en/67-hydrometeorology'),  # bot rejected on site (masti)
    re.compile('.*[\./@]get-ligaen\.stats\.pointstreak\.com/scoreboard.html'),  # bot rejected on site (masti)
    re.compile('.*[\./@]sbc\.org\.pl/publication/11793'),  # bot rejected on site (masti)
    re.compile('.*[\./@]eosielsko\.pl'),  # bot rejected on site (masti)
    re.compile('.*[\./@]svencionys\.lt/index.php?3819840680'),  # bot rejected on site (masti)
    re.compile('.*[\./@]halama\.pl'),  # bot rejected on site (masti)
    re.compile('.*[\./@]kszosiatkowka\.pl'),  # bot rejected on site (masti)
    re.compile('.*[\./@]metro\.gov\.az'),  # bot rejected on site (masti)
    re.compile('.*[\./@]mini\.sl\.se/sv/travelplanner'),  # bot rejected on site (masti)
    re.compile('.*[\./@]noweskalmierzyce\.pl/pl/strona/parki-krajobrazowe'),  # bot rejected on site (masti)
    re.compile('.*[\./@]muzeumgdansk\.pl/o-muzeum-gdanska/zarzad-i-rada-muzeum-gdanska'),  # bot rejected on site (masti)
    re.compile('.*[\./@]yg-life\.com/archives'),  # bot rejected on site (masti)
    re.compile('.*[\./@]ostpommern\.de/kr-regenwalde.php'),  # bot rejected on site (masti)
    re.compile('.*[\./@]ostpommern\.de/kr-schlawe.php'),  # bot rejected on site (masti)
    re.compile('.*[\./@]vled\.co\.il'),  # bot rejected on site (masti)
    re.compile('.*[\./@]monitorpolski\.gov.pl'),  # bot rejected on site (masti)
    re.compile('.*[\./@]militaryarchitecture\.com'),  # bot rejected on site (masti)
    re.compile('.*[\./@]gazetapolska\.pl'),  # bot rejected on site (masti)
    re.compile('.*[\./@]panstwo\.net'),  # bot rejected on site (masti)
    re.compile('.*[\./@]earlparkindiana\.com'),  # bot rejected on site (masti)
    re.compile('.*[\./@]monitorpolski\.gov\.pl/M1950052058801.pdf'),  # bot rejected on site (masti)
    re.compile('.*[\./@]monitorpolski\.gov\.pl/MP/1946/s/26/43'),  # bot rejected on site (masti)
    re.compile('.*[\./@]monitorpolski\.gov\.pl/MP/1947/s/74/490'),  # bot rejected on site (masti)
    re.compile('.*[\./@]monitorpolski\.gov\.pl/MP/1947/s/79/528'),  # bot rejected on site (masti)
    re.compile('.*[\./@]monitorpolski\.gov\.pl/MP/1951/s/66/860'),  # bot rejected on site (masti)
    re.compile('.*[\./@]monitorpolski\.gov\.pl/mp/1949/s/44/592'),  # bot rejected on site (masti)
    re.compile('.*[\./@]eredivisiestats\.nl/topscorers.php'),  # bot rejected on site (masti)
    re.compile('.*[\./@]mtr\.com\.hk/en/customer/services/system_map.html'),  # bot rejected on site (masti)
    re.compile('.*[\./@]dlibra\.umcs\.lublin\.pl/dlibra/plain-content?id=3251'),  # bot rejected on site (masti)
    re.compile('.*[\./@]fenditton\.org'),  # bot rejected on site (masti)
    re.compile('.*[\./@]jura-pilica\.com/?rezerwat-ruskie-gory-,388'),  # bot rejected on site (masti)
    re.compile('.*[\./@]da\.katowice\.pl/lux-ex-silesia'),  # bot rejected on site (masti)
    re.compile('.*[\./@]biblioteka\.nama-hatta\.pl'),  # bot rejected on site (masti)
    re.compile('.*[\./@]cruxgaliciae\.org'),  # bot rejected on site (masti)
    re.compile('.*[\./@]russiavolley\.com'),  # bot rejected on site (masti)
    re.compile('.*[\./@]pzbs\.pl/regulaminy-stale/137-regulamin-klasyfikacyjny'),  # bot rejected on site (masti)
    re.compile('.*[\./@]annalubanska\.pl'),  # bot rejected on site (masti)
    re.compile('.*[\./@]kuriergalicyjski\.com/historia/zabytki/3123-tajna-apteka'),  # bot rejected on site (masti)
    re.compile('.*[\./@]dioceseofscranton\.org'),  # bot rejected on site (masti)
    re.compile('.*[\./@]kelseyserwa\.com'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]cafonline\\.com/en-us/competitions/32ndeditionoftotalafricacupofnations/MatchDetails?MatchId=c8WFJCFnBOuM7mR%2feYEFkCmdq3y59q4uIqqQwH7I4XBdCUMKpVuT5gHSHovlxfKL'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]cafonline\.com/Portals/0/President/ranking%20tirage%20PDF%20English.pdf'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]cafonline\.com/Portals/0/Total%20AFCON%202016/Qualifiers%20CAN%202019%20-%20matches.pdf'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]cafonline\.com/en-us/competitions/32ndeditionoftotalafricacupofnations/MatchDetails?MatchId=c8WFJCFnBOuM7mR%2feYEFkCmdq3y59q4uIqqQwH7I4XBdCUMKpVuT5gHSHovlxfKL'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]cafonline\.com/Portals/0/President/ranking%20tirage%20PDF%20English.pdf'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]cafonline\.com/Portals/0/Total%20AFCON%202016/Qualifiers%20CAN%202019%20-%20matches.pdf'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]cafonline\.com/Portals/0/glo%20caf%202014/Draw%20Procedure%20-%20FT%20AFCON%202017%20---.pdf'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]cafonline\.com/Portals/0/glo%20caf%202014/Final%20Ranking%20AFCON%20FT,%20Gabon%202017%20FT%20FT.pdf'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]cafonline\.com/en-US/NewsCenter/News/NewsDetails?id=7Q1Us%2BTZ%2Bi03aalfg76fmw%3D%3D'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]cafonline\.com/en-US/NewsCenter/News/NewsDetails?id=rHQkXwbJ/qnlkT0kYVKcMg%3D%3D'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]cafonline\.com/en-US/NewsCenter/News/NewsDetails?id=7hxNqK1bVNLnCtOlOfU1ZA%3D%3D'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]cafonline\.com/en-US/NewsCenter/News/NewsDetails?id=tjUi4YBkLWPBNKHA%2B7kBJg%3D%3D'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]cafonline\.com/en-US/NewsCenter/News/NewsDetails?id=auMqtAj3SdstqcMlrNnjPQ%3D%3D'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]cafonline\.com/en-US/NewsCenter/News/NewsDetails?id=vHiSLG/k2NKtlLQi4VfGCA%3D%3D'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]cafonline\.com/en-US/NewsCenter/News/NewsDetails?id=8OD0yxG/y9dts7Ih8e/JqA%3D%3D'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]cafonline\.com/en-US/NewsCenter/News/NewsDetails?id=PEhc3UzJyA5sc0oWvJWcag%3D%3D'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]cafonline\.com/en-US/NewsCenter/News/NewsDetails?id=KXDfhHRQfmo848MmjaimQA%3D%3D'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]cafonline\.com/en-us/competitions/31theditionofafricancupofnationtotal,gabon2017'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]cafonline\.com/en-us/competitions/32ndeditionoftotalafricacupofnations'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]cafonline\.com/en-us/competitions/africanwomenchampionship,cameroon2016'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]cafonline\.com/en-us/competitions/allafricagamesmencongo2015'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]cafonline\.com/en-us/competitions/orangeafricacupofnations,equatorialguinea2015'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]cafonline\.com/en-us/competitions/qcan2017.aspx'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]cafonline\.com/en-us/competitions/qcan2017'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]cafonline\.com/en-us/competitions/tn9thafricanwomenchampionship-namibia/news.aspx/NewsDetails?id=FiwOlHoESQWLBCXDsqhA7w%3D%3D'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]cafonline\.com/en-us/memberassociations/f%C3%A9d%C3%A9rationgabonaisedefootball'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]cafonline\.com/en-us/competitions/orangeafricacupofnations,equatorialguinea2015.aspx'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]mogiel\.net/POL/history/polhist.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]age.ne\.jp/x/sas/alljapan_jump_lh_men_results20171105.pdf'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]age.ne\.jp/x/sas/96th_alljapan_j_nh-l2018results.pdf'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]age.ne\.jp/x/sas/96th_alljapan_j_nh-m2018results.pdf'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]sputniknews\.com/society/201803251062883119-israel-sculptor-death-meisler'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]galeriabwa.bydgoszcz\.pl/wystawa/milosz-matwijewicz-moj-malowany-swiat'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]zagraevsky\.com/vsmz5.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]zagraevsky\.com/democracy_engl.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]zagraevsky\.com/sloboda_book1.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]zagraevsky\.com/vsmz11.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]zagraevsky\.com/vsmz2.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]zagraevsky\.com/vsmz7.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]www3.put.poznan\.pl/jubileusz/honoriscausa'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]anst.gov\.ro/documente/documente/0993-1016%20National%20Federations%20-%20Ski%20-%20Biathlon%20.pdf'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]nicesport\.pl/sportyzimowe/109610/ps-w-planicy-214-metrow-muellera'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]bpi.co.uk/award/ - wszystkie podstrony'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]przystanekplanszowka\.pl/2015/07/instrukcja-neuroshima-epub-mobi.html'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]przystanekplanszowka\.pl/2012/09/k2-wyrok.html'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]przystanekplanszowka\.pl/2012/10/dominion-rozdarte-krolewstwo-wyrok.html'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]neonmuzeum\.org'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]delipark\.pl'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]cityofshoreacres.us'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]shantymen\.pl'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]atlasryb.online/opis_ryby.php?id= (wszystkie numery po "id")'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]histpol.pl.ua/ru/biblioteka/ukazatel-po-nazvaniyam?id=491'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]histpol.pl.ua/ru/gosudarstvennoe-upravlenie/sudebnye-i-pravookhranitelnye-organy-pravo?id=1759'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]histpol.pl.ua/ru/kultura/pechatnye-izdaniya/gazety?id=2366'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]pttk.gubin.com\.pl/luz/wycieczki.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]czyczy\.pl/2014/jadrowa/wlk-brytania-sellafield-troche-rzeczywistych-danych-kosztach-rozbiorki-elektrowni-jadrowej'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]czyczy\.pl/2012/jadrowa/litwa-energetyka-jadrowa-polegla-w-referendum'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]czyczy\.pl/mapa'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]ethnologue\.com/subgroups/b2-1'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]ethnologue\.com/subgroups/australian'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]ethnologue\.com/subgroups/tangale'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]ethnologue\.com/subgroups/germanic'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]ethnologue\.com/language/[a-z]{3}'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]ethnologue\.com/country/CF'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]ethnologue\.com/country/CL'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gwz.bielsko\.pl'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]dsm.psychiatryonline\.org//book.aspx?bookid=22'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.linyi\.gov\.cn/'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.vismaskiclassics\.com/standings_total?personID=3421292'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.polishmusic\.ca/skok/cds/polskie/grupy/a/2plus1/2plus1\.html'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.polishmusic\.ca/skok/cds/polskie/grupy/r/roma/roma\.html'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.polishmusic\.ca/skok/cds/polskie/grupy/s/smerfy/smerfy\.html'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.polishmusic\.ca/skok/cds/polskie/grupy/s/szczesni/szczesni\.html'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]canadianpoetry\.org/2016/06/28/widow-of-the-rock/#thewidowoftherock'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]pgeenergiaciepla\.pl/'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]cieplodlatrojmiasta\.pl/'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.cityofandalusia\.com/'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.anfp\.cl/'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]zulawy\.infopl\.info/index\.php/pndg/gstegna/drewnica'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.brutalism\.com/content/welicoruss-and-the-story-behind-skirts-and-make-up'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.brutalism\.com/content/hyperial-sceptical-vision'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.brutalism\.com/content/anima-damnata-atrocious-disfigurement-of-the-redeemers-corpse-at-the-graveyard-of-humanity'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.vysna-jedlova\.sk/'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.rezeknesbiblioteka\.lv/index\.php?option=com_content&view=article&id=376:apsveicam-vladimirs-nikonovs&catid=163:par-izstadem-cb&Itemid=104'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.ujfeherto\.hu/'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]www\.kulanu-party\.co\.il/'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.the-athenaeum\.org/art/list\.php?m=a&s=tu&aid='),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.the-athenaeum\.org/art/by_artist\.php?Artist_ID=426'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.the-athenaeum\.org/art/detail\.php?ID= +wszystkie ID'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.the-athenaeum\.org/art/list\.php?m=o&s=du&oid=1\.&f=a&fa=11380'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.the-athenaeum\.org/art/list\.php?m=o&s=du&oid=1\.&f=a&fa=3453'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.the-athenaeum\.org/art/list\.php?s=tu&m=a&aid=428&p=3'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.the-athenaeum\.org/people/detail\.php?ID='),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]cmi2\.yale\.edu/ym/archive/artists/jamespeale/artist\.html'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]erc24\.com/'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]erc24\.com/archives/16292'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gorzowianin\.com/wiadomosc/10529-tvp-wybuduje-w-koncu-siedzibe\.html'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gorzowianin\.com/wiadomosc/10584-zamiast-kary-dwa-nowe-linki\.html'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gorzowianin\.com/wiadomosc/11301-powstanie-przystanek-kolejowy-gorzow-zachod-zdjecia\.html'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gorzowianin\.com/wiadomosc/7775-radio-go-juz-ruszylo-w-rytmie-hitow-tylko-na-1017-fm\.html'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gorzowianin\.com/wiadomosc/9607-z-centrum-na-zawarcie-to-bedzie-wielki-korek\.html'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gorzowianin\.com/wiadomosc/9746-pociag-do-berlina-coraz-bardziej-popularny\.html'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gminajozefow\.pl/soltysi-2/'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gminajozefow\.pl/wyniki-konsultacji-spolecznych/'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gminajozefow\.pl/zapraszamy-do-udzialu-w-konsultacjach-spolecznych/'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gminajozefow\.pl/zawiadomienie-o-sesji-rady-gminy-2/'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]speedwayresults\.com'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.maius\.uj\.edu\.pl'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.monsourdelrosario\.com/'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]krknews\.pl/zobaczyc-caly-swiat-swietna-akcja-krakowem-barany-wytepione-ruchu-wideo/'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.jablunkov\.cz/ic/index\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.antiqueadvertising\.com/pics/lucky\.html'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]antiqueadvertising\.com/price-guide/'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]www\.pcworld\.com/article/253200/googles_project_glass_teases_augmented_reality_glasses\.html'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.ceausescu\.org/ceausescu_texts/revolution/trial-eng\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]www\.netcarshow\.com/volkswagen/2018-tiguan_allspace/'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]www\.netcarshow\.com/ford/2017-fiesta/'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]www\.netcarshow\.com/opel/2018-crossland_x/'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.vilani\.lv/'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.cavanheritage\.ie/Default\.aspx?StructureID_str=2'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.villefort-cevennes\.com/'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.vilalba\.gal/'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.vaulnaveys-le-bas\.fr/'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]www\.maltauncovered\.com/valletta-capital-city/'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]snooker\.org/res/index\.asp?event=281 -> http://www\.snooker\.org/res/index\.asp?event=281'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]sl\.se/ficktid/vinter/h22ny\.pdf'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]pbn\.nauka\.gov\.pl/sedno-webapp/persons/969455/Tomasz_Maszczyk'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]pbn\.nauka\.gov\.pl/sedno-webapp/search?searchCategory=WORK&filter\.globalKey=tomasz%20bajerowski&search\.x=44&search\.y=12&filter\.title&filter\.inJournal&filter\.inBook&filter\.contributor&filter\.contributorRole&filter\.publicationYearFrom&filter\.publicationYearTo&filter\.workType&filter\.affInstitution&sortField=RELEVANCE&sortAscending=true&pageSize=PAGE_SIZE_10'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.songfacts\.com/detail\.php?id='),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]www\.lincolnshire\.org/lincolnshire-sausage/'),  # bot rejected on site (masti, szoltys)
# gcatholic.org
    re.compile('.*[\./@]gcatholic\.org/dioceses/former/t0136\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/former/t1686\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/goaa0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/country/AR-province\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/country/CR-province\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/country/CU-province\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/country/IE\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/country/PY-province\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/country/SS\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/country/UA-province\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/algi0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/amad0\.htm#3222'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/anta0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/anto0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/antw0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/areq0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/ayac0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/bang2\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/barq0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/barr0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/bele0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/bogo0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/bans0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/buca0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/cala1\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/caph0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/cara0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/caro1\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/cart0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/casc0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/cast0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/celj0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/chan0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/cili0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/ciud0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/coch0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/coro0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/cuma0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/curi0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/cuzc0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/falk0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/goaa0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/guay0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/hany0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/hels0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/honi0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/huan0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/ibag0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/ivan0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/king1\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/koln0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/lase0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/lisb0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/ljub0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/loya0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/luts0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/lviv1\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/maca1\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/malt0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/mani0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/mani1\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/mara0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/mars2\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/melf0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/meri0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/moun0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/nass0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/ndja0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/neth0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/nico0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/npam0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/pana0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/paup0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/phil1\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/phno0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/pmor0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/popa0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/port0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/priz0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/pvil0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/pyon0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/quit0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/raba0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/rome0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/sucr0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/suva0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/taio0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/tara2\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/tern1\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/tong0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/truj0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/tsin0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/tuba0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/tunj0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/utre0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/vale1\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/vill3\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/viln0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/wall0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/yamo0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/yung3\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/zcru0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/zdom0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/zfed0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/zjua1\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/zpau0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/ztia1\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/ztia3\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/former/vlad0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/toronto/pr-we\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/nunciature/nunc034\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/hierarchy/data/cardL13-5\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/amma0\.htm '),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/done0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/odes1\.htm '),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/pari2\.htm#50444'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/kolo0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/muka0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/soka0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/zjos7\.htm#57002'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/conference/018\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/angr0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/bere0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/falk0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/funa0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/honi0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/koln0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/kyrg0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/ljub0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/luts1\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/lviv1\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/mars2\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/neth0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/priz0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/pyon0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/raro0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/taio0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/tong0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/wloc0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/yamo0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/former/t0895\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/data/rite-Rt\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/agan0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/pape0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/winn0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/yoko0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/organizations/card\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/orders/index\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/angr0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/anti0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/cypr0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/lisb0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/luts0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/ndja0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/priz0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/diocese/wanh0\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/former/t1090\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/former/t1091\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/former/t1093\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/former/t1094\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/former/t1095\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/former/t1108\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/former/t1109\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/former/t1110\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/former/t1111\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/former/t1113\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/former/t2060\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/dioceses/nunciature/org220\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/hierarchy/data/bishops-X\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/hierarchy/data/officials-M\.htm'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]gcatholic\.org/orders/053\.htm'),  # bot rejected on site (masti, szoltys)
# Ze słownikiem Kopalińskiego podobna sytuacja jak z Gcatholikiem, tzn\. część linków zgłoszona prawidłowo, część nieprawidłowo\. Nieprawidłowe zgłoszenia to:
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/6DEC11B84950D1F9412565BA0021C94C\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/02162EB3B37F6455412565B70004B0F9\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/0CE83B0EDC7B72F2C12565DB0064BCBF\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/18D2145DAFA7FBD3C12565BE001644D4\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/C9CF9AFC8175D311412565CC007B6AE5\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/21BBD727783F862B412565CD0051A57E\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/246F68861BD49548412565BA003434EE\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/255DF08B813660F5412565BA0036F441\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/27192AAB9A0B2452412565B70039BB8A\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/27DC83AA85CAF6D5C125656F001C75B0\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/286890E66B322A6B412565BA0022F4CB\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/2D64302975D79C3AC125658C005F8974\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/2E48A299DC6EB482C125658B0074E140\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/2ED46894AB6C6A27C12565E70063258C\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/6FCBE3EE599F3EC3C12565B60006269D\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/650218F9F3D03517412565CB0072C82A\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/2FD5655E3C91FBCD412565D30056733A\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/2FFEDBF26AC72CCF412565B60051565A\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/33EF8940386A2E0AC12565EE005AE531\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/3BD9AF807D7613D2412565BA0022CCA0\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/3E3CFA773D4AABF3C12565BD006673CB\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/40650EF20C762479412565BF00294622\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/97EC1A11607283AE412565CC004763B7\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/489E074C12B30A35412565CD004BECDB\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/4AB76064E593F011C125658C0063B83F\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/4AB9FF36C825C18FC125657C0081D067\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/4AD308518E458BA8412565B8001C110B\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/994E28BF6FC733A9412565BA00289ACA\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/B9A4FD7AD8EC09F5412565BA002D5036\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/55ADB4D52C9A3ECBC12565BE0043D580\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/5774F597D0942D31412565CB0059A735\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/578F60BA759FF4A4412565BA002D82B8\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/7A9AECCA5EC4A591C12565BD0057FAD5\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/73D68C65F2C33E6BC12565BD003BCD92\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/7290491D99D06D0A412565D3004C6CD1\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/8061E4B2EE139D5AC12565EE006E6A0A\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/844BEF3539CCC370C12565EF0046B5C6\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/D6420FD8CDF9C6AF412565AF0078C733\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/84C8DEF5AB6694F2412565CD005DB310\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/93E2A52FDA53B491C12565BD003B5FB3\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/944B03D3BB6696F6C12565BD00540FE8\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/9CC69A2A9E99A48CC12565B5007CD020\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/9BB6A4284E15C4EEC12565E700674E8B\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/BB932EB1B605294A412565B80010EED6\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/AE5ABDF9E893388EC12565EE005D855D\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/ADE096011D04FB9DC12565BD0057383C\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/C1489C254BDB29D1412565B800092026\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/CEA41203C7084B3BC12565BE0039CDBC\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/D14910AA76E13D8A412565D4004AEEE8\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/D0EEBBD933631C13C12565D800472319\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/DCC4BF7D21C6A5D3412565B8001B743A\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/DF34B55637518E1D412565B7003E98AD\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/E8B75264E4BDA91E412565CD004B610F\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/F9267F8EF0DE1437C12565DA00556B95\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.slownik-online\.pl/kopalinski/FB880FDFC5A8A9A0412565BD0035BFA4\.php'),  # bot rejected on site (masti, szoltys)
    re.compile('.*[\./@]\.swr\.de/swraktuell/rp/ludwigshafen/entscheidung-in-ludwigshafen-spd-frau-jutta-steinruck-gewinnt-ob-wahl/-/id=1652/did=20434688/nid=1652/elild0/index\.html'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]\.forumakademickie\.pl/fa/2013/02/chalasinscy/'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]\.linkedin\.com/in/krzysztof-lisek-7a6259bb/'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]\.hej\.mielec\.pl/miasto2/repoe/art548,publiczne-gimnazjum-w-wadowicach-gornych-z-imieniem-leszka-deptuly-ten-dzien-zapisze-sie-w-historii\\.html'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]\.piw\.pl/indeks-autorow/ferry-luc'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]archinea\.pl/biblioteka-uniwersytetu-warszawskiego-marek-budzynski-zbigniew-badowski/'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]archinea\.pl/sad-najwyzszy-w-warszawie-marek-budzynski-zbigniew-badowski/'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]\.ebelchatow\.pl/content/nie-plus-plus-polska-razem'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]\.msp\.gov\.pl/pl/media/aktualnosci/31579,Zmiany-w-kierownictwie-MSP\\.html'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]uniaeuropejska.org/marek-safjan-ponownie-sedzia-tsue/'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]\.ck-czestochowa\.pl/wyszukiwarka-grobow/szukaj'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]\.lameziainstrada\.com/politica/politiche-2018-matteo-salvini-eletto-senatore-in-calabria-furgiuele-deputato'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]\.bipraciborz\.pl/bip/dokumenty-akcja-wyszukaj-idkategorii-39906'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]\.niccolorinaldi\.it/chi-sono/biografia\.html'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]\.segodnya\.ua/politics/pnews/olga-bogomolec-sobiraetsya-ballotirovatsya-v-prezidenty-ukrainy-505598\.html'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]\.federnuoto\.it/federazione/federazione-news/item/40079-barelli-eletto-alla-camera\.html'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]\.pip\.gov\.pl/pl/wiadomosci/69784,roman-giedrojc-glownym-inspektorem-pracy\.html'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]\.movimentocinquestelle\.it/'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]\.forumakademickie\.pl/aktualnosci/2011/1/5/765/jak-ck-wybierano/'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]\.forumakademickie\.pl/fa/2015/07-08/kronika-wydarzen/odzew-w-sprawie-bez-odzewu/'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]pl.linkedin\.com/pub/sabina-nowosielska/51/308/5a7'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]ksi.home\.pl/archiwaprzelomu/obrazy/AP-6-1-1-42_23\.PDF'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]ksi.home\.pl/archiwaprzelomu/obrazy/AP-6-1-1-44_39\.PDF'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]ksi.home\.pl/archiwaprzelomu/obrazy/AP-6-1-1-44_40\.PDF'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]\.wloclawek\.info\.pl/nowosci,wiadomosci_wloclawek_i_region,1,1,tadeusz_dubicki_nowym_rektorem_p,16036\.html'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]\.europeanvoice\.com/folder/theswedishpresidencyoftheeu/124.aspx?artid=65305'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]\.swr\.de/report/presse/-/id=1197424/nid=1197424/did=2918594/1wxuzhj/index\.html'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]\.pism\.pl/publications/bulletin/no-55-905'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]\.stowarzyszeniekongreskobiet\.pl/pl-PL/text/o_nas/rada_programowa'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]\.shellenicparliament\.gr/en/Vouleftes/Diatelesantes-Vouleftes-Apo-Ti-Metapolitefsi-Os-Simera'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www\.mojamongolia\.com/moj-zyciorys'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]zgryglas\.pl/o-mnie'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]uniwersytetradom\.pl/art/display_article\.php?id=1178'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]uniwersytetradom\.pl/art/display_article\.php?id=1986'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]dziennikzbrojny\.pl/aktualnosci/news,1,2155,aktualnosci-z-polski,robert-kupiecki-wiceministrem-on'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www\.zbigniewkonwinski\.pl'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www\.terzobinario\.it/elezioni-alessandro-battilocchio-eletto-alla-camera/131593'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]elk\.gmina\.pl/nauczmy-sie-na-pamiec-tego-kraju-gminne-obchody-2-i-3-maja'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]przegladlubartowski\.pl/informacje/6762/wybory-2014-wyniki-wyborow-wojtow-i-radnych'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www\.forumakademickie\.pl/fa/2015/06/jak-stracic-prestiz'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www\.tubawyszkowa\.pl/aktualnosci/czytaj/4863/Kandydaci-KWW-Kukiz-15-pod-szczesliwa-siodemka'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www\.kulturalna\.warszawa\.pl/nagroda-literacka,1,10564\.html?locale=pl_PL'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www\.newyorktheatreguide\.com/news/jl09/bacchae555183\.htm'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www\.fiedler\.pl/sub,pl,arkady-radoslaw-fiedler\.html'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www\.fiedler\.pl'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www\.uniaeuropejska\.org/nominacje-do-nagrod-mep-awards'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www\.kedzierzynkozle\.pl/portal/index\.php?t=200&id=35673'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www\.wrp\.pl/prof-dr-hab-eberhard-makosz-1932-2018'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]actmedia\.eu/daily/cristian-diaconescu-was-appointment-as-foreign-affairs-minister/37811'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www\.bielecki\.pl'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www\.frithjof-schmidt\.de'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www\.odg\.mi\.it/node/30222'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www\.businesseurope\.eu/history-organisation'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]sledzinska-katarasinska\.pl/o-mnie'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www\.ireneuszras\.pl'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www\.forumakademickie\.pl/fa/2014/04/kurczewscy'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]wielkopolanie\.zhr\.pl/rozkazy/L4_2010\.pdf'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www\.jacektomczak\.pl'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www\.jadwigarotnicka\.pl'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]obywatelenauki\.pl/2014/02/wiecej-dobrej-nauki-nowa-akcja-prof-janusza-bujnickiego'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www\.faktyoswiecim\.pl/fakty/aktualnosci/15778-oswiecim-to-pewne-janusz-chwierut-wygral-w-pierwszej-turze'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www\.jerzymaslowski\.pl'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www\.jerzymaslowski\.pl/wp-content/uploads/2009/02/jmm5\.jpg'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www\.prowincja\.com\.pl/autorzy/Jerzy-Wcisla,25'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]jerzymalecki\.pl'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www\.mfa\.gov\.pl/pl/aktualnosci/wiadomosci/nominacje_dla_nowych_ambasadorow_rp'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]dlibra\.bg\.ajd\.czest\.pl:8080/Content/855/Kultura_fizyczna_9\.-57\.pdf'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]nicesport\.pl/sportyzimowe/140101/fc-w-rasnovie-znamy-liste-uczestnikow'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]nicesport\.pl/sportyzimowe/138514/pk-w-engelbergu-schmitt-wygrywa-serie-probna-nowy-rekord-grecji'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]nicesport\.pl/sportyzimowe/142712/ps-w-oberstdorfie-seria-probna-dla-kranjca-kolejny-rekord-bulgarii'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]nicesport\.pl/sportyzimowe/105016/mitz-mistrzem-szwecji'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/conference/'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/romancuria/'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/nunciature/nunc017\.htm#18000'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.nba-allstar\.com/legends/rosters\.htm '),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]wcsp\.science\.kew\.org/prepareChecklist\.do?checklist=selected_families%40%40222100820181252697'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/churches/'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/diocese/'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/hierarchy/data/card'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former//t1946\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/bagn0\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/iles0\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/micr0\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t3398\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t3383\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t3381\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t2075\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t2065\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t2063\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t2022\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1987\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1984\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1981\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1980\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1979\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1978\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1977\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1976\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1975\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1974\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1971\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1970\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1969\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1967\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1966\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1965\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1964\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1962\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1960\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1959\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1958\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1957\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1956\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1952\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1951\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1949\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1947\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1945\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1943\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1941\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1940\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1938\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1937\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1936\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1935\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1934\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1931\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1928\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1926\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1923\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1911\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1910\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1908\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1907\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1906\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1905\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1904\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1903\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1902\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1901\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1900\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1897\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1896\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1895\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1893\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0036\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0037\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0038\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0039\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0040\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0041\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0042\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0045\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0047\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0049\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0050\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0059\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0060\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0061\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0150\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0156\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0164\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0166\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0167\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0170\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0171\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0174\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0181\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0187\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0212\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0216\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0219\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0221\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0229\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0239\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0241\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0248\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0255\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0256\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0267\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0270\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0273\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0279\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0283\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0284\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0287\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0291\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0296\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0305\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0308\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0311\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0312\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0315\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0316\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0317\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0318\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0320\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0323\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0324\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0328\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0338\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0340\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0341\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0344\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0351\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0352\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0355\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0356\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0406\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0476\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0666\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0670\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0672\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0682\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0688\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0805\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0806\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0869\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0872\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0874\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0875\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0876\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0878\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0879\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0880\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0881\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0885\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0887\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0888\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0892\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0896\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0899\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0901\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0902\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0906\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0907\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0909\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0943\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t0944\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1040\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1302\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1418\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1419\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1425\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1428\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1433\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1434\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1435\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1437\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1438\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1439\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1441\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1445\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1447\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1449\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1450\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1451\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1452\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1453\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1454\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1577\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1578\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1585\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1589\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1597\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1602\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1605\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1606\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1609\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1617\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1622\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1623\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1627\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1632\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1633\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1635\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1641\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1644\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1647\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1651\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1652\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1679\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1682\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1683\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1690\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1691\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1693\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1694\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1696\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1703\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1704\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1706\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1710\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1712\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1716\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1718\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1719\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1723\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1724\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1725\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1727\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1733\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1738\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1799\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1826\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1831\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1833\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1835\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1836\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1838\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/former/t1839\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/hierarchy/data/bishops-BAR\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/hierarchy/data/bishops-BR\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/hierarchy/data/bishops-BU\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/hierarchy/data/bishops-CR\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/hierarchy/data/bishops-DEC\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/hierarchy/data/bishops-DEM\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/hierarchy/data/bishops-DI\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/hierarchy/data/bishops-FI\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/hierarchy/data/bishops-GL\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/hierarchy/data/bishops-GR\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/hierarchy/data/bishops-H\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/hierarchy/data/bishops-K\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/hierarchy/data/bishops-KI\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/hierarchy/data/bishops-KR\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/hierarchy/data/bishops-L\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/hierarchy/data/bishops-LO\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/hierarchy/data/bishops-N\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/hierarchy/data/bishops-O\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/hierarchy/data/bishops-P\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/hierarchy/data/bishops-PE\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/hierarchy/data/bishops-PF\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/hierarchy/data/bishops-PL\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/hierarchy/data/bishops-PR\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/hierarchy/data/bishops-RO\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/hierarchy/data/bishops-SH\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/hierarchy/data/bishops-SK\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/hierarchy/data/bishops-SU\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/hierarchy/data/bishops-TI\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/hierarchy/data/bishops-VONH\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/hierarchy/data/bishops-VONS\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/hierarchy/data/bishops-W\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/hierarchy/data/bishops-WI\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/hierarchy/data/bishops-X\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/hierarchy/data/bishops-Z\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/nunciature/nunc076\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/nunciature/org202\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/nunciature/org204\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/nunciature/org207\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/nunciature/org210\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/romancuria/d60\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/romancuria/d91\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/dioceses/romancuria/d9e\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/hierarchy/cardinals-title-c2\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/hierarchy/cardinals-title-c3\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/hierarchy/data/officials-C\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/hierarchy/data/officials-S\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/orders/006\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/orders/009\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/orders/040\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www\.gcatholic\.org/orders/237\.htm'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]entsyklopeedia\.ee'),  # bot rejected on site (masti, Szoltys)
    re.compile('.*[\./@]www.msp.gov.pl/pl/media/aktualnosci/31579,Zmiany-w-kierownictwie-MSP.html'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www.ebelchatow.pl/content/nie-plus-plus-polska-razem'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www.bipraciborz.pl/bip/dokumenty-akcja-wyszukaj-idkategorii-39906'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www.leggiperme.it/?p=495'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]uniaeuropejska.org/antysemickie-hasa-w-wgierskim-parlamencie/'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www.matica.hr/knjige/autor/369/'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www.segodnya.ua/politics/pnews/olga-bogomolec-sobiraetsya-ballotirovatsya-v-prezidenty-ukrainy-505598.html'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www.federnuoto.it/federazione/federazione-news/item/40079-barelli-eletto-alla-camera.html'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www.independent.co.uk/news/world/europe/mariano-rajoy-latest-spain-election-pedro-sanchez-premier-basque-national-party-a8378101.html'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www.independent.co.uk/opinion/commentators/denis-macshane-britain-can-help-to-shape-a-new-europe-481214.html'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www.pip.gov.pl/pl/wiadomosci/69784,roman-giedrojc-glownym-inspektorem-pracy.html'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www.forumakademickie.pl/aktualnosci/2011/1/5/765/jak-ck-wybierano/'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www.forumakademickie.pl/fa/2015/07-08/kronika-wydarzen/odzew-w-sprawie-bez-odzewu/'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www.independent.co.uk/news/world/europe/mariano-rajoy-latest-spain-election-pedro-sanchez-premier-basque-national-party-a8378101.html'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www.wloclawek.info.pl/nowosci,wiadomosci_wloclawek_i_region,1,1,tadeusz_dubicki_nowym_rektorem_p,16036.html'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www.pism.pl/publications/bulletin/no-55-905'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]glencteresa.pl/'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www.europeanvoice.com/folder/theswedishpresidencyoftheeu/124.aspx?artid=65305'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www.mojabibliotekamazurska.pl/biblioteka/ukazaly_sie_02_03.html'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www.ordynacka.pl/2017/04/22/zakonczyl-sie-vii-kongres-ordynackiej/'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]uniwersytetradom.pl/art/display_article.php?id=1178'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]uniwersytetradom.pl/art/display_article.php?id=1986'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www.ptp.org.pl/modules.php?name=News&file=article&sid=25'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]zeglarski.info/artykuly/zmarl-kapitan-zygrfyd-zyga-perlicki/'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www.gopsusports.com/sports/m-fenc/spec-rel/032402aaa.html'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]www.independent.co.uk/news/world/europe/silvio-berlusconis-heir-angelino-alfano-forms-new-party-in-italy-8943520.html'),  # bot rejected on site (masti, Elfhelm)
    re.compile('.*[\./@]allaboutmusic\.pl'),  # bot rejected on site (masti, edk)

]



def _get_closest_memento_url(url, when=None, timegate_uri=None):
    """Get most recent memento for url."""
    if isinstance(memento_client, ImportError):
        raise memento_client

    if not when:
        when = datetime.datetime.now()

    mc = memento_client.MementoClient()
    if timegate_uri:
        mc.timegate_uri = timegate_uri

    retry_count = 0
    while retry_count <= config2.max_retries:
        try:
            memento_info = mc.get_memento_info(url, when)
            break
        except requests.ConnectionError as e:
            error = e
            retry_count += 1
            sleep(config2.retry_wait)
    else:
        raise error

    mementos = memento_info.get('mementos')
    if not mementos:
        raise Exception(
            'mementos not found for {0} via {1}'.format(url, timegate_uri))
    if 'closest' not in mementos:
        raise Exception(
            'closest memento not found for {0} via {1}'.format(
                url, timegate_uri))
    if 'uri' not in mementos['closest']:
        raise Exception(
            'closest memento uri not found for {0} via {1}'.format(
                url, timegate_uri))
    return mementos['closest']['uri'][0]


def get_archive_url(url):
    """Get archive URL."""
    try:
        archive = _get_closest_memento_url(
            url,
            timegate_uri='http://web.archive.org/web/')
    except Exception:
        archive = _get_closest_memento_url(
            url,
            timegate_uri='http://timetravel.mementoweb.org/webcite/timegate/')
        pass

    # FIXME: Hack for T167463: Use https instead of http for archive.org links
    if archive.startswith('http://web.archive.org'):
        archive = archive.replace('http://', 'https://', 1)
    return archive

def citeArchivedLink(link, text):
        #look if link is in cite template with non empty archive param
        #or link itself is an archive
        # return True in this cases

        temppars = textlib.extract_templates_and_params(text,remove_disabled_parts=True, strip=True)

        for (t,p) in temppars:
            if t.lower().startswith("cytuj"):
                #pywikibot.output(u'T:%s\nP:%s' % (t,p))
                arch = False
                urlin = False
                if 'archiwum' in p.keys():
                    if p['archiwum'].startswith(link):
                        #skip archive links
                        pywikibot.output('[%s] citeArchivedLink is archive:%s' % (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),link))
                        return(True)
                    arch = len(p['archiwum'])
                    #if arch:
                    #    pywikibot.output('[%s] citeArchivedLink archive found:%s' % (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),link))
                if 'url' in p.keys():
                    urlin = p['url'].startswith(link)
                    #pywikibot.output('[%s] citeArchivedLink link found:%s' % (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),link))
                if arch and urlin:
                    pywikibot.output('[%s] citeArchivedLink link archived:%s' % (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),link))
                    return(True)
        return(False)

'''
def citeArchivedLink(link, text):
        """
        check if link is within {{cytuj...}} template with filled archiwum= field or within this field
	conditions on link removal:
	    link in url/tytuł field + archiwum not empty
	    link in archiwum field
        return True if link archived

        adapted from: m-removedeadlinktemplates.py
        """
        citetempR = re.compile(ur'(?P<citetemplate>\{\{[cC]ytuj.*?\|[^}]*?\}\})')
        urlfieldR = re.compile(ur'(url|tytuł)\s*?=(?P<url>[^\|\}]*)')
        archfieldR = re.compile(ur'archiwum\s*?=\s*?(?P<arch>[^\|\}]*)')
        archived = False
        
        cites = citetempR.finditer(text)
        for c in cites:
            citetemplate = c.group('citetemplate').strip()
            try:
                urlfield = re.search(urlfieldR,citetemplate).group('url').strip()
            except AttributeError:
                continue
            #pywikibot.output(u'URL:%s' % urlfield) 
            try:
                archfield = re.search(archfieldR,citetemplate).group('arch').strip()
            except AttributeError:
                continue
            if link in urlfield:
                if (len(archfield) > 0):
                    archived = True
                    #pywikibot.input('link [%s] found in:%s' % (link,c.group('citetemplate')))

        return(archived)
'''

def weblinksIn(text, withoutBracketed=False, onlyBracketed=False):
    """
    Yield web links from text.

    TODO: move to textlib
    """
    text = textlib.removeDisabledParts(text)

    # Ignore links in fullurl template
    text = re.sub(r'{{\s?fullurl:.[^}]*}}', '', text)
    # TODO search for links within cite with filled-in archiwum parameter
    

    # MediaWiki parses templates before parsing external links. Thus, there
    # might be a | or a } directly after a URL which does not belong to
    # the URL itself.

    # First, remove the curly braces of inner templates:
    nestedTemplateR = re.compile(r'{{([^}]*?){{(.*?)}}(.*?)}}')
    while nestedTemplateR.search(text):
        text = nestedTemplateR.sub(r'{{\1 \2 \3}}', text)

    # Then blow up the templates with spaces so that the | and }} will not
    # be regarded as part of the link:.
    templateWithParamsR = re.compile(r'{{([^}]*?[^ ])\|([^ ][^}]*?)}}',
                                     re.DOTALL)
    while templateWithParamsR.search(text):
        text = templateWithParamsR.sub(r'{{ \1 | \2 }}', text)

    # Add <blank> at the end of a template
    # URL as last param of multiline template would not be correct
    text = text.replace('}}', ' }}')

    # Remove HTML comments in URLs as well as URLs in HTML comments.
    # Also remove text inside nowiki links etc.
    text = textlib.removeDisabledParts(text)
    #linkR = textlib.compileLinkR(withoutBracketed, onlyBracketed)
    #linkR = re.compile(ur'(?P<url>http[s]?:(\/\/[^:\s\?]+?)(\??[^\s;<>\"\|\)]*))(?:[\]\s\.:;,<>\"\|\)])')
    linkR = re.compile(ur'(?m)(?P<url>http[s]?:(\/\/[^\s\?]+?)(\??[^\s<\|\}\]]*))(?:[\]\s\.<\|\}])')
    for m in linkR.finditer(text):
        if m.group('url'):
            #pywikibot.output('URL to YIELD:%s' % m.group('url'))
            if not citeArchivedLink(m.group('url'),text):
                yield m.group('url')
            else:
                #test output
                pywikibot.output('[%s] WebLinksIn: link skipped:%s' % (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),m.group('url')))
        #else:
        #    yield m.group('urlb')


XmlDumpPageGenerator = partial(
    _XMLDumpPageGenerator, text_predicate=weblinksIn)


class NotAnURLError(BaseException):

    """The link is not an URL."""


@deprecated('requests')
class LinkChecker(object):

    """
    Check links.

    Given a HTTP URL, tries to load the page from the Internet and checks if it
    is still online.

    Returns a (boolean, string) tuple saying if the page is online and
    including a status reason.

    Per-domain user-agent faking is not supported in this deprecated class.

    Warning: Also returns false if your Internet connection isn't working
    correctly! (This will give a Socket Error)

    """

    def __init__(self, url, redirectChain=[], serverEncoding=None,
                 HTTPignore=[]):
        """
        Constructor.

        redirectChain is a list of redirects which were resolved by
        resolveRedirect(). This is needed to detect redirect loops.
        """
        self.url = url
        self.serverEncoding = serverEncoding

        fake_ua_config = config.fake_user_agent_default.get(
            'weblinkchecker', False)
        if fake_ua_config and isinstance(fake_ua_config, str):
            user_agent = fake_ua_config
        elif fake_ua_config:
            user_agent = comms.http.fake_user_agent()
        else:
            user_agent = comms.http.user_agent()
        self.header = {
            'user-agent': user_agent,
            'Accept': 'text/xml,application/xml,application/xhtml+xml,'
                      'text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5',
            'Accept-Language': 'de-de,de;q=0.8,en-us;q=0.5,en;q=0.3',
            'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
            'Keep-Alive': '30',
            'Connection': 'keep-alive',
        }
        self.redirectChain = redirectChain + [url]
        self.changeUrl(url)
        self.HTTPignore = HTTPignore

    def getConnection(self):
        """Get a connection."""
        if self.scheme == 'http':
            return httplib.HTTPConnection(self.host)
        elif self.scheme == 'https':
            return httplib.HTTPSConnection(self.host)
        else:
            raise NotAnURLError(self.url)

    def getEncodingUsedByServer(self):
        """Get encodung used by server."""
        if not self.serverEncoding:
            try:
                pywikibot.output(
                    u'Contacting server %s to find out its default encoding...'
                    % self.host)
                conn = self.getConnection()
                conn.request('HEAD', '/', None, self.header)
                self.response = conn.getresponse()
                self.readEncodingFromResponse(self.response)
            except:
                pass
            if not self.serverEncoding:
                # TODO: We might also load a page, then check for an encoding
                # definition in a HTML meta tag.
                pywikibot.output('Error retrieving server\'s default charset. '
                                 'Using ISO 8859-1.')
                # most browsers use ISO 8859-1 (Latin-1) as the default.
                self.serverEncoding = 'iso8859-1'
        return self.serverEncoding

    def readEncodingFromResponse(self, response):
        """Read encoding from response."""
        if not self.serverEncoding:
            try:
                ct = response.getheader('Content-Type')
                charsetR = re.compile('charset=(.+)')
                charset = charsetR.search(ct).group(1)
                self.serverEncoding = charset
            except:
                pass

    def changeUrl(self, url):
        """Change url."""
        self.url = url
        # we ignore the fragment
        (self.scheme, self.host, self.path, self.query,
         self.fragment) = urlparse.urlsplit(self.url)
        if not self.path:
            self.path = '/'
        if self.query:
            self.query = '?' + self.query
        self.protocol = url.split(':', 1)[0]
        # check if there are non-ASCII characters inside path or query, and if
        # so, encode them in an encoding that hopefully is the right one.
        try:
            self.path.encode('ascii')
            self.query.encode('ascii')
        except UnicodeEncodeError:
            encoding = self.getEncodingUsedByServer()
            self.path = unicode(urllib.quote(self.path.encode(encoding)))
            self.query = unicode(urllib.quote(self.query.encode(encoding),
                                              '=&'))

    def resolveRedirect(self, useHEAD=False):
        """
        Return the redirect target URL as a string, if it is a HTTP redirect.

        If useHEAD is true, uses the HTTP HEAD method, which saves bandwidth
        by not downloading the body. Otherwise, the HTTP GET method is used.

        @rtype: unicode or None
        """
        conn = self.getConnection()
        try:
            if useHEAD:
                conn.request('HEAD', '%s%s' % (self.path, self.query), None,
                             self.header)
            else:
                conn.request('GET', '%s%s' % (self.path, self.query), None,
                             self.header)
            self.response = conn.getresponse()
            # read the server's encoding, in case we need it later
            self.readEncodingFromResponse(self.response)
        except httplib.BadStatusLine:
            # Some servers don't seem to handle HEAD requests properly,
            # e.g. http://www.radiorus.ru/ which is running on a very old
            # Apache server. Using GET instead works on these (but it uses
            # more bandwidth).
            if useHEAD:
                return self.resolveRedirect(useHEAD=False)
            else:
                raise
        if self.response.status >= 300 and self.response.status <= 399:
            # to debug, print response.getheaders()
            redirTarget = self.response.getheader('Location')
            if redirTarget:
                try:
                    redirTarget.encode('ascii')
                except UnicodeError:
                    redirTarget = redirTarget.decode(
                        self.getEncodingUsedByServer())
                if redirTarget.startswith('http://') or \
                   redirTarget.startswith('https://'):
                    self.changeUrl(redirTarget)
                    return True
                elif redirTarget.startswith('/'):
                    self.changeUrl(u'%s://%s%s'
                                   % (self.protocol, self.host, redirTarget))
                    return True
                else:  # redirect to relative position
                    # cut off filename
                    directory = self.path[:self.path.rindex('/') + 1]
                    # handle redirect to parent directory
                    while redirTarget.startswith('../'):
                        redirTarget = redirTarget[3:]
                        # some servers redirect to .. although we are already
                        # in the root directory; ignore this.
                        if directory != '/':
                            # change /foo/bar/ to /foo/
                            directory = directory[:-1]
                            directory = directory[:directory.rindex('/') + 1]
                    self.changeUrl('%s://%s%s%s'
                                   % (self.protocol, self.host, directory,
                                      redirTarget))
                    return True
        else:
            return False  # not a redirect

    def check(self, useHEAD=False):
        """
        Return True and the server status message if the page is alive.

        @rtype: tuple of (bool, unicode)
        """
        try:
            wasRedirected = self.resolveRedirect(useHEAD=useHEAD)
        except UnicodeError as error:
            return False, u'Encoding Error: %s (%s)' % (
                error.__class__.__name__, error)
        except httplib.error as error:
            return False, u'HTTP Error: %s' % error.__class__.__name__
        except socket.error as error:
            # https://docs.python.org/2/library/socket.html :
            # socket.error :
            # The accompanying value is either a string telling what went
            # wrong or a pair (errno, string) representing an error
            # returned by a system call, similar to the value
            # accompanying os.error
            if isinstance(error, basestring):
                msg = error
            else:
                try:
                    msg = error[1]
                except IndexError:
                    pywikibot.output('### DEBUG information for T57282')
                    raise IndexError(type(error))
            # TODO: decode msg. On Linux, it's encoded in UTF-8.
            # How is it encoded in Windows? Or can we somehow just
            # get the English message?
            return False, u'Socket Error: %s' % repr(msg)
        if wasRedirected:
            if self.url in self.redirectChain:
                if useHEAD:
                    # Some servers don't seem to handle HEAD requests properly,
                    # which leads to a cyclic list of redirects.
                    # We simply start from the beginning, but this time,
                    # we don't use HEAD, but GET requests.
                    redirChecker = LinkChecker(
                        self.redirectChain[0],
                        serverEncoding=self.serverEncoding,
                        HTTPignore=self.HTTPignore)
                    return redirChecker.check(useHEAD=False)
                else:
                    urlList = ['[%s]' % url
                               for url in self.redirectChain + [self.url]]
                    return (False,
                            u'HTTP Redirect Loop: %s' % ' -> '.join(urlList))
            elif len(self.redirectChain) >= 19:
                if useHEAD:
                    # Some servers don't seem to handle HEAD requests properly,
                    # which leads to a long (or infinite) list of redirects.
                    # We simply start from the beginning, but this time,
                    # we don't use HEAD, but GET requests.
                    redirChecker = LinkChecker(
                        self.redirectChain[0],
                        serverEncoding=self.serverEncoding,
                        HTTPignore=self.HTTPignore)
                    return redirChecker.check(useHEAD=False)
                else:
                    urlList = ['[%s]' % url
                               for url in self.redirectChain + [self.url]]
                    return (False,
                            u'Long Chain of Redirects: %s'
                            % ' -> '.join(urlList))
            else:
                redirChecker = LinkChecker(self.url, self.redirectChain,
                                           self.serverEncoding,
                                           HTTPignore=self.HTTPignore)
                return redirChecker.check(useHEAD=useHEAD)
        else:
            try:
                conn = self.getConnection()
            except httplib.error as error:
                return False, u'HTTP Error: %s' % error.__class__.__name__
            try:
                conn.request('GET', '%s%s'
                             % (self.path, self.query), None, self.header)
            except socket.error as error:
                return False, u'Socket Error: %s' % repr(error[1])
            try:
                self.response = conn.getresponse()
            except Exception as error:
                return False, u'Error: %s' % error
            # read the server's encoding, in case we need it later
            self.readEncodingFromResponse(self.response)
            # site down if the server status is between 400 and 499
            alive = self.response.status not in range(400, 500)
            if self.response.status in self.HTTPignore:
                alive = False
            return alive, '%s %s' % (self.response.status,
                                     self.response.reason)


class LinkCheckThread(threading.Thread):

    """A thread responsible for checking one URL.

    After checking the page, it will die.
    """

    def __init__(self, page, url, history, HTTPignore, day):
        """Constructor."""
        threading.Thread.__init__(self)
        self.page = page
        self.url = url
        self.history = history
        self.header = {
            'Accept': 'text/xml,application/xml,application/xhtml+xml,'
                      'text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5',
            'Accept-Language': 'de-de,de;q=0.8,en-us;q=0.5,en;q=0.3',
            'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
            'Keep-Alive': '30',
            'Connection': 'keep-alive',
        }
        # identification for debugging purposes
        self.setName((u'%s - %s' % (page.title(), url)).encode('utf-8',
                                                               'replace'))
        self.HTTPignore = HTTPignore
        self._use_fake_user_agent = config.fake_user_agent_default.get(
            'weblinkchecker', False)
        self.day = day

    def run(self):
        """Run the bot."""
        ok = False
        exception = False
        ignore = False
        try:
            header = self.header
            r = comms.http.fetch(
                self.url, headers=header,
                use_fake_user_agent=self._use_fake_user_agent)
        except requests.exceptions.InvalidURL:
            exception = True
            message = u'Podany link nie jest prawidłowym adresem URL'
            #message = i18n.twtranslate(self.page.site,
            #                           'weblinkchecker-badurl_msg',
            #                           {'URL': self.url})
        except:
            pywikibot.output('[%s] Exception while processing URL %s in page %s'
                             % (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),self.url, self.page.title()))
            exception = True
            message = 'Exception while connecting.'
            #raise
            #return

        if not exception:
            #test output
            pywikibot.output('[%s] HTTP status:%s in [%s - %s]' % (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),r.status,self.page.title(),self.url)) 

            #if (r.status == requests.codes.ok and str(r.status) not in self.HTTPignore):
            if r.status == requests.codes.ok:
                #pywikibot.output(u'[%s] CODE [%s] OK in [%s - %s]' % (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),r.status,self.page.title(),self.url))
                ok = True
            elif r.status in self.HTTPignore:
                ignore = True
                #test output
                #pywikibot.output(u'[%s] CODE [%s] ignored in [%s - %s]' % (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),r.status,self.page.title(),self.url))
            else:
                #test output
                #pywikibot.output(u'[%s] CODE [%s] rejected in [%s - %s]' % (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),r.status,self.page.title(),self.url))
                message = '{0}'.format(r.status)
        if ok:
            if self.history.setLinkAlive(self.url):
                pywikibot.output('[%s] *Link to %s in [[%s]] is back alive.'
                                 % (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),self.url, self.page.title()))
        elif ignore:
            pywikibot.output(u'[%s] CODE [%s] ignored in [%s - %s]' % (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),r.status,self.page.title(),self.url))
        else:
            pywikibot.output('[%s] *[[%s]] links to %s - %s.'
                             % (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), self.page.title(), self.url, message))
            self.history.setLinkDead(self.url, message, self.page,
                                     config.weblink_dead_days)


class History(object):

    """
    Store previously found dead links.

    The URLs are dictionary keys, and
    values are lists of tuples where each tuple represents one time the URL was
    found dead. Tuples have the form (title, date, error) where title is the
    wiki page where the URL was found, date is an instance of time, and error
    is a string with error code and message.

    We assume that the first element in the list represents the first time we
    found this dead link, and the last element represents the last time.

    Example:

    dict = {
        'https://www.example.org/page': [
            ('WikiPageTitle', DATE, '404: File not found'),
            ('WikiPageName2', DATE, '404: File not found'),
        ]

    """

    def __init__(self, reportThread, site=None):
        """Constructor."""
        self.reportThread = reportThread
        if not site:
            self.site = pywikibot.Site()
        else:
            self.site = site
        self.semaphore = threading.Semaphore()
        self.datfilename = pywikibot.config.datafilepath(
            'deadlinks', 'deadlinks-%s-%s.dat' % (self.site.family.name,
                                                  self.site.code))
        # Count the number of logged links, so that we can insert captions
        # from time to time
        self.logCount = 0
        try:
            with open(self.datfilename, 'rb') as datfile:
                self.historyDict = pickle.load(datfile)
        except (IOError, EOFError):
            # no saved history exists yet, or history dump broken
            self.historyDict = {}

    def log(self, url, error, containingPage, archiveURL):
        """Log an error report to a text file in the deadlinks subdirectory."""
        if archiveURL:
            errorReport = u'* %s ([%s archiwum])\n' % (url, archiveURL)
        else:
            errorReport = u'* %s\n' % url
        for (pageTitle, date, error) in self.historyDict[url]:
            # ISO 8601 formulation
            isoDate = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(date))
            errorReport += "** In [[%s]] on %s, %s\n" % (pageTitle, isoDate,
                                                         error)
        pywikibot.output(u"** Logging link for deletion.")
        txtfilename = pywikibot.config.datafilepath('deadlinks',
                                                    'results-%s-%s.txt'
                                                    % (self.site.family.name,
                                                       self.site.lang))
        txtfile = codecs.open(txtfilename, 'a', 'utf-8')
        self.logCount += 1
        if self.logCount % 30 == 0:
            # insert a caption
            txtfile.write('=== %s ===\n' % containingPage.title()[:3])
        txtfile.write(errorReport)
        txtfile.close()

        if self.reportThread and not containingPage.isTalkPage():
            self.reportThread.report(url, errorReport, containingPage,
                                     archiveURL)

    def setLinkDead(self, url, error, page, weblink_dead_days):
        """Add the fact that the link was found dead to the .dat file."""
        #test output
        #pywikibot.output('setLinkDead: SEM acquire [%s][%s][%s]' % (url,page.title(),error))
        self.semaphore.acquire()
        #test output
        #pywikibot.output('[%s] setLinkDead: SEM acc DONE [%s][%s][%s]' % (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), url, page.title(), error))
        now = time.time()
        if url in self.historyDict:
            timeSinceFirstFound = now - self.historyDict[url][0][1]
            timeSinceLastFound = now - self.historyDict[url][-1][1]
            archiveURL = None
            # if the last time we found this dead link is less than an hour
            # ago, we won't save it in the history this time.
            if timeSinceLastFound > 60 * 60:
                self.historyDict[url].append((page.title(), now, error))
            # if the first time we found this link longer than x day ago
            # (default is a week), it should probably be fixed or removed.
            # We'll list it in a file so that it can be removed manually.
            if timeSinceFirstFound > 60 * 60 * 24 * weblink_dead_days:
                # search for archived page
                # skip memento search
                """
                try:
                    #test output
                    pywikibot.output('setlinkDead: Memento Archive [%s]' % url)
                    archiveURL = get_archive_url(url)
                except Exception as e:
                    pywikibot.warning(
                        'get_closest_memento_url({0}) failed: {1}'.format(
                            url, e))
                    archiveURL = None
                """
                if archiveURL is None:
                    #test output
                    pywikibot.output('[%s] setlinkDead: InternetArchive [%s]' % (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), url))
                    try:
                        archiveURL = weblib.getInternetArchiveURL(url)
                    except:
                        pywikibot.output('[%s] EXCEPTION setlinkDead: InternetArchive [%s]' % (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"), url))
                        pass
                if archiveURL is None:
                    #test output
                    pywikibot.output('[%s] setlinkDead: WebCitation [%s]' % (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),url))
                    try:
                        archiveURL = weblib.getWebCitationURL(url)
                    except:
                        pywikibot.output('[%s] EXCEPTION setlinkDead: WebCitation [%s]' % (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),url))
                        pass
                #test output
                pywikibot.output('[%s] setlinkDead: ArchiveLink received [%s]' % (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),archiveURL))
                self.log(url, error, page, archiveURL)
        else:
            self.historyDict[url] = [(page.title(), now, error)]
        #test output
        #pywikibot.output('[%s] setlinkDead: SEM release [%s][%s][%s]' % (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),url,page.title(),error))
        self.semaphore.release()
        #test output
        #pywikibot.output('[%s] setlinkDead: SEM rel DONE [%s][%s][%s]' % (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),url,page.title(),error))

    def setLinkAlive(self, url):
        """
        Record that the link is now alive.

        If link was previously found dead, remove it from the .dat file.

        @return: True if previously found dead, else returns False.
        """
        if url in self.historyDict:
            #test output
            #pywikibot.output('[%s] setLinkAlive: SEM acquire [%s]' % (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),url))
            self.semaphore.acquire()
            try:
                del self.historyDict[url]
            except KeyError:
                # Not sure why this can happen, but I guess we can ignore this.
                pass
            #test output
            #pywikibot.output('[%s] setLinkAlive: SEM release [%s]' % (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),url))
            self.semaphore.release()
            return True
        else:
            return False

    def save(self):
        """Save the .dat file to disk."""
        #test output
        pywikibot.output('PICKLING at %s' % datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
        with open(self.datfilename, 'wb') as f:
            pickle.dump(self.historyDict, f, protocol=config.pickle_protocol)


class DeadLinkReportThread(threading.Thread):

    """
    A Thread that is responsible for posting error reports on talk pages.

    There is only one DeadLinkReportThread, and it is using a semaphore to make
    sure that two LinkCheckerThreads can not access the queue at the same time.
    """

    def __init__(self):
        """Constructor."""
        threading.Thread.__init__(self)
        self.semaphore = threading.Semaphore()
        self.queue = []
        self.finishing = False
        self.killed = False

    def report(self, url, errorReport, containingPage, archiveURL):
        """Report error on talk page of the page containing the dead link."""
        self.semaphore.acquire()
        self.queue.append((url, errorReport, containingPage, archiveURL))
        self.semaphore.release()

    def shutdown(self):
        """Finish thread."""
        self.finishing = True

    def kill(self):
        """Kill thread."""
        # TODO: remove if unneeded
        self.killed = True

    def run(self):
        """Run thread."""
        while not self.killed:
            if len(self.queue) == 0:
                if self.finishing:
                    break
                else:
                    time.sleep(0.1)
            else:
                self.semaphore.acquire()
                (url, errorReport, containingPage, archiveURL) = self.queue[0]
                #test output
                #pywikibot.output('[%s] DeadLinkReportThread: SEM acquire [%s][%s][%s]' % (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),url,containingPage.title(),errorReport))
                self.queue = self.queue[1:]
                talkPage = containingPage.toggleTalkPage()
                pywikibot.output(color_format(
                    '{lightaqua}** Reporting dead link on {0}...{default}',
                    talkPage.title(asLink=True)))
                try:
                    content = talkPage.get() + "\n"
                    if url in content:
                        pywikibot.output(color_format(
                            '{lightaqua}** Dead link seems to have already '
                            'been reported on {0}{default}',
                            talkPage.title(asLink=True)))
                        #test output
                        #pywikibot.output('[%s] DeadLinkReportThread: SEM release [%s][%s][%s]' % (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),url,containingPage.title(),errorReport))
                        self.semaphore.release()
                        continue
                except (pywikibot.NoPage, pywikibot.IsRedirectPage):
                    content = u''

                if archiveURL:
                # removed original code using i18n

                #    archiveMsg = u'\n' + \
                #                 i18n.twtranslate(containingPage.site,
                #                                  'weblinkchecker-archive_msg',
                #                                  {'URL': archiveURL})
                    archiveMsg = archiveURL
                else:
                    archiveMsg = u''
                # The caption will default to "Dead link". But if there is
                # already such a caption, we'll use "Dead link 2",
                # "Dead link 3", etc.
                caption = u'Martwy link'
                #caption = i18n.twtranslate(containingPage.site,
                #                           'weblinkchecker-caption')
                i = 1
                count = u''
                # Check if there is already such a caption on the talk page.
                #while re.search('= *%s%s *=' % (caption, count),
                #                content) is not None:
                #    i += 1
                #    count = u' ' + str(i)
                #caption += count
                #content += '== %s ==\n\n%s\n\n%s%s\n--~~~~' % \
                #           (caption,
                #            i18n.twtranslate(containingPage.site,
                #                             'weblinkchecker-report'),
                #            errorReport,
                #            archiveMsg)
                # new code: use polish template
                content += u'{{Martwy link dyskusja\n | link=' + errorReport + u'\n | IA=' + archiveMsg + u'\n}}'

                comment = u'[[%s#%s|→]] %s' % \
                          (talkPage.title(), caption, u'Robot zgłasza niedostępny link zewnętrzny')
                           #i18n.twtranslate(containingPage.site,
                           #                 'weblinkchecker-summary'))
                try:
                    talkPage.put(content, comment)
                except pywikibot.SpamfilterError as error:
                    pywikibot.output(color_format(
                        '{lightaqua}** SpamfilterError while trying to '
                        'change {0}: {1}{default}',
                        talkPage.title(asLink=True), error.url))
                #test output
                #pywikibot.output('[%s] DeadLinkReportThread: SEM release [%s][%s][%s]' % (datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),url,containingPage.title(),errorReport))
                self.semaphore.release()


class WeblinkCheckerRobot(SingleSiteBot, ExistingPageBot):

    """
    Bot which will search for dead weblinks.

    It uses several LinkCheckThreads at once to process pages from generator.
    """

    def __init__(self, generator, HTTPignore=None, day=7, site=True):
        """Constructor."""
        super(WeblinkCheckerRobot, self).__init__(
            generator=generator, site=site)

        if config.report_dead_links_on_talk:
            pywikibot.log("Starting talk page thread")
            reportThread = DeadLinkReportThread()
            # thread dies when program terminates
            # reportThread.setDaemon(True)
            reportThread.start()
        else:
            reportThread = None
        self.history = History(reportThread, site=self.site)
        if HTTPignore is None:
            self.HTTPignore = []
        else:
            self.HTTPignore = HTTPignore
        self.day = day

    def treat_page(self):
        """Process one page."""
        page = self.current_page
        text = page.get()

        """report  page.title and time"""
        #now = datetime.datetime.now()
        try:
            pywikibot.output(u'P:%s >>>%s' % (page.title(), datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
        except:
            pass

        for url in weblinksIn(text):
            ignoreUrl = False
            for ignoreR in ignorelist:
                if ignoreR.match(url):
                    ignoreUrl = True
            if not ignoreUrl:
                # Limit the number of threads started at the same time. Each
                # thread will check one page, then die.
                #test output
                pywikibot.output(u'STARTING thread #%i' % threading.activeCount())
                while threading.activeCount() >= config.max_external_links:
                    #test output
                    pywikibot.output(u'WAIT %s for thread #%i' % (config.retry_wait,threading.activeCount()))
                    time.sleep(config.retry_wait)
                thread = LinkCheckThread(page, url, self.history,
                                         self.HTTPignore, self.day)
                # thread dies when program terminates
                thread.setDaemon(True)
                try:
                    thread.start()
                except threading.ThreadError:
                    pywikibot.warning(
                        "Can't start a new thread.\nPlease decrease "
                        "max_external_links in your user-config.py or use\n"
                        "'-max_external_links:' option with a smaller value. "
                        "Default is 50.")
                    raise
                except request.data:
                    pywikibot.warning('*** NO DATA received')
                    raise


def RepeatPageGenerator():
    """Generator for pages in History."""
    history = History(None)
    pageTitles = set()
    for value in history.historyDict.values():
        for entry in value:
            pageTitles.add(entry[0])
    for pageTitle in sorted(pageTitles):
        page = pywikibot.Page(pywikibot.Site(), pageTitle)
        yield page


def countLinkCheckThreads():
    """
    Count LinkCheckThread threads.

    @return: number of LinkCheckThread threads
    @rtype: int
    """
    i = 0
    for thread in threading.enumerate():
        if isinstance(thread, LinkCheckThread):
            i += 1
    return i


@deprecated('requests')
def check(url):
    """DEPRECATED: Use requests instead. Perform a check on URL."""
    c = LinkChecker(url)
    return c.check()


def main(*args):
    """
    Process command line arguments and invoke bot.

    If args is an empty list, sys.argv is used.

    @param args: command line arguments
    @type args: list of unicode
    """
    gen = None
    xmlFilename = None
    HTTPignore = []

    if isinstance(memento_client, ImportError):
        warn('memento_client not imported: %s' % memento_client, ImportWarning)

    # Process global args and prepare generator args parser
    local_args = pywikibot.handle_args(args)
    genFactory = pagegenerators.GeneratorFactory()

    for arg in local_args:
        if arg == '-talk':
            config.report_dead_links_on_talk = True
        elif arg == '-notalk':
            config.report_dead_links_on_talk = False
        elif arg == '-repeat':
            gen = RepeatPageGenerator()
        elif arg.startswith('-ignore:'):
            HTTPignore.append(int(arg[8:]))
        elif arg.startswith('-day:'):
            config.weblink_dead_days = int(arg[5:])
        elif arg.startswith('-xmlstart'):
            if len(arg) == 9:
                xmlStart = pywikibot.input(
                    u'Please enter the dumped article to start with:')
            else:
                xmlStart = arg[10:]
        elif arg.startswith('-xml'):
            if len(arg) == 4:
                xmlFilename = i18n.input('pywikibot-enter-xml-filename')
            else:
                xmlFilename = arg[5:]
        else:
            genFactory.handleArg(arg)

    if xmlFilename:
        try:
            xmlStart
        except NameError:
            xmlStart = None
        gen = XmlDumpPageGenerator(xmlFilename, xmlStart,
                                   genFactory.namespaces)

    if not gen:
        gen = genFactory.getCombinedGenerator()
    if gen:
        if not genFactory.nopreload:
            # fetch at least 240 pages simultaneously from the wiki, but more
            # if a high thread number is set.
            pageNumber = max(240, config.max_external_links * 2)
            gen = pagegenerators.PreloadingGenerator(gen, groupsize=pageNumber)
        gen = pagegenerators.RedirectFilterPageGenerator(gen)
        bot = WeblinkCheckerRobot(gen, HTTPignore, config.weblink_dead_days)
        try:
            bot.run()
        finally:
            waitTime = 0
            # Don't wait longer than 30 seconds for threads to finish.
            while countLinkCheckThreads() > 0 and waitTime < 30:
                try:
                    pywikibot.output(u"Waiting for remaining %i threads to "
                                     u"finish, please wait..."
                                     % countLinkCheckThreads())
                    # wait 1 second
                    time.sleep(1)
                    waitTime += 1
                except KeyboardInterrupt:
                    pywikibot.output(u'Interrupted.')
                    break
            if countLinkCheckThreads() > 0:
                pywikibot.output(u'Remaining %i threads will be killed.'
                                 % countLinkCheckThreads())
                # Threads will die automatically because they are daemonic.
            if bot.history.reportThread:
                bot.history.reportThread.shutdown()
                # wait until the report thread is shut down; the user can
                # interrupt it by pressing CTRL-C.
                try:
                    while bot.history.reportThread.isAlive():
                        time.sleep(0.1)
                except KeyboardInterrupt:
                    pywikibot.output(u'Report thread interrupted.')
                    bot.history.reportThread.kill()
            pywikibot.output(u'Saving history...')
            bot.history.save()
        return True
    else:
        pywikibot.bot.suggest_help(missing_generator=True)
        return False


if __name__ == "__main__":
    main()