-
Notifications
You must be signed in to change notification settings - Fork 1
/
.htaccess
112 lines (105 loc) · 2.96 KB
/
.htaccess
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
Options All -Indexes
RewriteEngine on
# Block Transparent Proxies
# RewriteCond %{HTTP:VIA} !^$ [OR]
# RewriteCond %{HTTP:FORWARDED} !^$ [OR]
# RewriteCond %{HTTP:USERAGENT_VIA} !^$ [OR]
# RewriteCond %{HTTP:X_FORWARDED_FOR} !^$ [OR]
# RewriteCond %{HTTP:PROXY_CONNECTION} !^$ [OR]
# RewriteCond %{HTTP:XPROXY_CONNECTION} !^$ [OR]
# RewriteCond %{HTTP:HTTP_PC_REMOTE_ADDR} !^$ [OR]
# RewriteCond %{HTTP:HTTP_CLIENT_IP} !^$
# RewriteRule ^(.*)$ – [F]
# Block Command-line & Frameworks
SetEnvIfNoCase User-Agent ^$ scraper_bot
SetEnvIfNoCase User-Agent "^httplib" scraper_bot
SetEnvIfNoCase User-Agent "^Curl" scraper_bot
SetEnvIfNoCase User-Agent "^Wget" scraper_bot
SetEnvIfNoCase User-Agent "^libwww-perl" scraper_bot
SetEnvIfNoCase User-Agent "HttpClient" scraper_bot
SetEnvIfNoCase User-Agent "^libcurl" scraper_bot
SetEnvIfNoCase User-Agent "^Snoopy" scraper_bot
SetEnvIfNoCase User-Agent "^Ruby" scraper_bot
SetEnvIfNoCase User-Agent "pycurl" scraper_bot
# Other
SetEnvIfNoCase User-Agent "Aboundex" scraper_bot
<Limit GET POST HEAD>
order allow,deny
allow from all
# Amazon EC2
deny from 23.20.0.0/14
deny from 46.51.128.0/17
deny from 46.137.0.0/16
deny from 50.16.0.0/14
deny from 50.112.0.0/16
deny from 54.224.0.0/11
deny from 67.202.0.0/18
deny from 72.44.32.0/19
deny from 75.101.128.0/17
deny from 79.125.0.0/17
deny from 96.127.0.0/17
deny from 103.4.8.0/21
deny from 107.20.0.0/14
deny from 122.248.192.0/18
deny from 174.129.0.0/16
deny from 175.41.128.0/17
deny from 176.32.64.0/18
deny from 176.34.0.0/16
deny from 177.71.128.0/17
deny from 184.72.0.0/15
deny from 184.169.128.0/17
deny from 204.236.128.0/17
deny from 216.182.224.0/20
# Aggressively Block Proxies
deny from 38.100.19.8/29
deny from 188.165.192.0/18
deny from 38.100.21.0/24
deny from 38.100.41.64/26
deny from 38.105.71.0/25
deny from 38.105.83.0/27
deny from 38.112.21.140/30
deny from 38.118.42.32/29
deny from 65.213.208.128/27
deny from 65.222.176.96/27
deny from 65.222.185.72/29
deny from 74.117.56.0/21
deny from 108.171.240.0/20
deny from 173.224.208.0/20
deny from 199.71.212.0/22
deny from 199.119.200.0/21
deny from 208.87.240.0/22
deny from 216.24.192.0/20
deny from 216.99.144.0/20
deny from 216.104.32.0/20
deny from 61.206.125.0/24
deny from 62.171.194.0/23
deny from 75.126.0.0/16
deny from 80.33.0.0/16
deny from 80.58.0.0/16
deny from 81.12.0.0/17
deny from 83.16.154.152/29
deny from 85.10.219.104/29
deny from 85.92.130.0/24
deny from 85.185.0.0/16
deny from 88.198.241.104/29
deny from 88.198.252.144/29
deny from 145.253.239.8/29
deny from 150.188.0.0/15
deny from 193.164.131.0/24
deny from 194.112.195.202
deny from 198.145.112.128/25
deny from 198.145.182.0/26
deny from 200.30.64.0/20
deny from 200.43.108.0/24
deny from 200.75.128.0/20
deny from 200.126.112.0/20
deny from 200.172.222.0/26
deny from 200.202.192.0/18
deny from 200.210.0.0/16
deny from 203.160.0.0/23
deny from 207.44.128.0/17
deny from 207.210.192.0/18
deny from 208.110.68.144/29
deny from 96.47.225.0/24
Deny from env=scraper_bot
</Limit>