-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- 
-
-
All reports end up in a private Slack to protect people's privacy. The reports can be read by all team members of MyCrypto and they can decide whether the report makes it to the blacklist.
-The database was created in 2017 by #TeamMEW when trying to find a solution to the ethereum scams. Of course grouping all the scams won't make them go away, but it will make identifying them and taking them down easier. It is now being maintained by the MyCrypto team.
diff --git a/_layouts/index.html b/_layouts/index.html deleted file mode 100644 index a575b141..00000000 --- a/_layouts/index.html +++ /dev/null @@ -1,24 +0,0 @@ - - -
Sorry, but this record cannot be found.
\ No newline at end of file diff --git a/_layouts/redirect.html b/_layouts/redirect.html deleted file mode 100644 index 88eac067..00000000 --- a/_layouts/redirect.html +++ /dev/null @@ -1,16 +0,0 @@ - -| Category | -Subcategory | -Status | -Title | -Info | -
|---|
![](/img/" + url.name.toLowerCase().replace()
" + url.name + "![](/img/" + scams[i].subcategory.toLowerCase().replace(/\s/g, )
" + scams[i].subcategory;
- } else {
- var subcategory = scams[i].subcategory;
- if (!(icon_warnings.includes(subcategory))) {
- icon_warnings.push(subcategory);
- debug("Warning! No subcategory icon found for %s",subcategory);
- }
- }
- } else {
- var subcategory = ' None';
- }
- if (scams[i].name.length > 40) {
- scams[i].name = scams[i].name.substring(0, 40) + '...';
- }
- table += "
');
- } else {
- template = template.replace("{{ scam.screenshot }}", '');
- }
- } else {
- template = template.replace("{{ scam.googlethreat }}", '');
- template = template.replace("{{ scam.screenshot }}", '');
- }
- actions_text += ' Improve';
- template = template.replace("{{ scam.actions }}", 'This page was built in ' + ((new Date()).getTime() - startTime) + 'ms, and last updated at ' + dateFormat(getCache().updated, "UTC:mmm dd yyyy, HH:MM") + ' UTC
'); - res.send(default_template.replace('{{ content }}', template)); - }); - } else { - debug("Warning: No Google Safe Browsing API key found"); - res.send(default_template.replace('{{ content }}', template)); - } - }); - - app.get('/domain/:domain/', function(req, res) { // Serve /domain/This page was built in ' + ((new Date()).getTime() - startTime) + 'ms, and last updated at ' + dateFormat(getCache().updated, "UTC:mmm dd yyyy, HH:MM") + ' UTC
'); - }); - } else { - template = template.replace("{{ neutral.virustotal }}", " Could not pull data from VirusTotal"); - template = template.replace("{{ neutral.phishtank }}", " Could not pull data from Phishtank"); - template = template.replace("{{ page.built }}", 'This page was built in ' + ((new Date()).getTime() - startTime) + 'ms, and last updated at ' + dateFormat(getCache().updated, "UTC:mmm dd yyyy, HH:MM") + ' UTC
'); - debug("Warning: No VirusTotal API key found"); - } - } - - // VERIFIED DOMAIN PAGES - if(typeof verified != "undefined"){ - let template = fs.readFileSync('./_layouts/verifieddomain.html', 'utf8'); - template = template.replace("{{ verified.name }}", verified.name); - template = template.replace("{{ verified.notification }}", '') - template = template.replace("{{ verified.googlethreat }}", "Google Safe Browsing Status: {{ verified.googlethreat }}This page was built in ' + ((new Date()).getTime() - startTime) + 'ms, and last updated at ' + dateFormat(getCache().updated, "UTC:mmm dd yyyy, HH:MM") + ' UTC
'); - }); - } else { - template = template.replace("{{ verified.virustotal }}", " Could not pull data from VirusTotal"); - template = template.replace("{{ verified.phishtank }}", " Could not pull data from Phishtank"); - template = template.replace("{{ page.built }}", 'This page was built in ' + ((new Date()).getTime() - startTime) + 'ms, and last updated at ' + dateFormat(getCache().updated, "UTC:mmm dd yyyy, HH:MM") + ' UTC
'); - } - } - - // SCAM DOMAIN PAGES - if(typeof scam != "undefined"){ - let template = fs.readFileSync('./_layouts/scamdomain.html', 'utf8'); - var actions_text = ""; - template = template.replace("{{ scam.id }}", scam.id); - template = template.replace("{{ scam.name }}", scam.name); - template = template.replace("{{ scam.tip }}", " Security Tip(s):');
- } else {
- template = template.replace("{{ scam.screenshot }}", '');
- }
- } else {
- template = template.replace("{{ scam.googlethreat }}", '');
- template = template.replace("{{ scam.screenshot }}", '');
- }
- actions_text += ' Improve';
- template = template.replace("{{ scam.actions }}", 'This page was built in ' + ((new Date()).getTime() - startTime) + 'ms, and last updated at ' + dateFormat(getCache().updated, "UTC:mmm dd yyyy, HH:MM") + ' UTC
'); - }); - } else { - template = template.replace("{{ scam.virustotal }}", " Could not pull data from VirusTotal"); - template = template.replace("{{ scam.phishtank }}", " Could not pull data from Phishtank"); - template = template.replace("{{ page.built }}", 'This page was built in ' + ((new Date()).getTime() - startTime) + 'ms, and last updated at ' + dateFormat(getCache().updated, "UTC:mmm dd yyyy, HH:MM") + ' UTC
'); - debug("Warning: No VirusTotal API key found"); - } - } - - }); - - app.get('/ip/:ip/', function(req, res) { // Serve /ip/' + util.inspect(db.read()) + '')) + }, { + label: 'View config', + click: () => mainWindow.loadURL("data:text/html;charset=utf-8," + encodeURI('
' + util.inspect(config) + '')) + }, { + label: 'Manually spawn update', + click: () => { + esdb.update(); + mainWindow.loadURL("data:text/html;charset=utf-8," + encodeURI('Spawned update process successfully.')); + } + }] + }]); + Menu.setApplicationMenu(windowMenu); + mainWindow.loadURL('http://localhost:' + config.port); + mainWindow.once('ready-to-show', () => { + mainWindow.maximize(); + mainWindow.show(); + mainWindow.focus(); + }); + let handled = false; + app.on('window-all-closed', app.quit); + app.on('before-quit', (event) => { + event.preventDefault(); + mainWindow.removeAllListeners('close'); + if(!handled) { + handled = true; + console.log("Cleaning up..."); + db.exitHandler(); + console.log("Exited."); + mainWindow.close(); + process.exit(); + } + }); +}); \ No newline at end of file diff --git a/src/scripts/update.js b/src/scripts/update.js new file mode 100644 index 00000000..87cf9171 --- /dev/null +++ b/src/scripts/update.js @@ -0,0 +1,38 @@ +process.env.UV_THREADPOOL_SIZE = 128; +const debug = require('debug')('update'); +const path = require('path'); +const Scam = require('../classes/scam.class'); +const serialijse = require("serialijse"); +const fs = require('fs-extra'); +const config = require('../utils/config'); + +serialijse.declarePersistable(Scam); +if(!process.send) throw new Error("This script can only run as a child process"); + +process.once('close', () => process.exit(1)); + +(async () => { + const cacheExists = await fs.pathExists('./cache.db'); + if(!cacheExists) throw new Error("No cache file found"); + const cacheFile = await fs.readFile('./cache.db','utf8'); + + debug("Updating scams..."); + + await Promise.all(serialijse.deserialize(cacheFile).scams.reverse().filter(scam => scam.howRecent() > config.interval.cacheExpiration).map(async scam => { + if(config.lookups.HTTP.enabled) await scam.getStatus(); + if(config.lookups.DNS.IP.enabled) await scam.getIP(); + if(config.lookups.DNS.NS.enabled) await scam.getNameservers(); + + process.send({ + url: scam.url, + name: scam.name, + ip: scam.ip, + nameservers: scam.nameservers, + status: scam.status, + statusCode: scam.statusCode, + updated: Date.now() + }); + })); + + debug("Done updating!"); +})(); diff --git a/src/utils/abusereport.js b/src/utils/abusereport.js new file mode 100644 index 00000000..eb353955 --- /dev/null +++ b/src/utils/abusereport.js @@ -0,0 +1,25 @@ +module.exports = (scam) => { + let abusereport = ""; + abusereport += "I would like to inform you of suspicious activities at the domain " + scam.getHostname(); + if ('ip' in scam && scam.ip) { + abusereport += " located at IP address " + scam['ip'] + "."; + } else { + abusereport += "."; + } + if ('subcategory' in scam && scam.subcategory == "MyEtherWallet") { + abusereport += "The domain is impersonating MyEtherWallet.com, a website where people can create Ethereum wallets (a cryptocurrency like Bitcoin)."; + } else if ('subcategory' in scam && scam.subcategory == "MyCrypto") { + abusereport += "The domain is impersonating MyCrypto.com, a website where people can create Ethereum wallets (a cryptocurrency like Bitcoin)."; + } else if ('subcategory' in scam && scam.subcategory == "Classic Ether Wallet") { + abusereport += "The domain is impersonating classicetherwallet.com, a website where people can create Ethereum Classic wallets (a cryptocurrency like Bitcoin)."; + } else if ('category' in scam && scam.category == "Fake ICO") { + abusereport += "The domain is impersonating a website where an ICO is being held (initial coin offering, like an initial public offering but it's for cryptocurrencies)."; + } + if ('category' in scam && scam.category == "Phishing") { + abusereport += "\r\n\r\nThe attackers wish to steal funds by using phishing to get the victim's private keys (passwords to a wallet) and using them to send funds to their own wallets."; + } else if ('category' in scam && scam.category == "Fake ICO") { + abusereport += "\r\n\r\nThe attackers wish to steal funds by cloning the real website and changing the ethereum address so people will send funds to the attackers' address instead of the real address."; + } + abusereport += "\r\n\r\nPlease shut down this domain so further attacks will be prevented."; + return abusereport; +} \ No newline at end of file diff --git a/src/utils/config.js b/src/utils/config.js new file mode 100644 index 00000000..9bfd14e2 --- /dev/null +++ b/src/utils/config.js @@ -0,0 +1,36 @@ +const fs = require('fs'); +const dns = require('graceful-dns'); +const debug = require('debug')('config'); + +if (!fs.existsSync('./config.json')) { + module.exports = { + manual: false, + announcement: null, + port: 5111, + interval: { + cacheExpiration: -1, + cacheRenewCheck: -1, + databasePersist: -1 + }, + apiKeys: { + Google_SafeBrowsing: undefined, + Github_WebHook: undefined, + VirusTotal: undefined + }, + autoPull: { enabled: false }, + lookups: { + DNS: { + IP: { enabled: false }, + NS: { enabled: false } + }, + HTTP: { enabled: false } + } + } +} else { + const config = JSON.parse(fs.readFileSync('./config.json','utf8')); + config.manual = true; + if(!config.apiKeys.Google_SafeBrowsing) debug("Warning: No Google SafeBrowsing API key found"); + if(!config.apiKeys.VirusTotal) debug("Warning: No VirusTotal API key found"); + if(config.lookups.DNS.servers.length > 0) dns.setServers(config.lookups.DNS.servers); + module.exports = config; +} \ No newline at end of file diff --git a/src/utils/db.js b/src/utils/db.js new file mode 100644 index 00000000..bd3db47e --- /dev/null +++ b/src/utils/db.js @@ -0,0 +1,93 @@ +const fs = require('fs-extra'); +const yaml = require('js-yaml'); +const url = require('url'); +const path = require('path'); +const config = require('./config'); +const serialijse = require("serialijse"); +const createDictionary = require('array-object-dictionary'); +const Scam = require('../classes/scam.class'); +const debug = require('debug')('db'); + +serialijse.declarePersistable(Scam); + +const db = { + scams: [], + verified: [], + index: { + featured: [], + blacklist: [], + whitelist: [], + whitelistAddresses: [], + addresses: [], + ips: [], + inactives: [], + actives: [] + } +}; + +const readEntries = async () => { + debug("Reading entries..."); + const scamsFile = await fs.readFile(path.join(__dirname, '../../_data/scams.yaml'),'utf8'); + const verifiedFile = await fs.readFile(path.join(__dirname, '../../_data/legit_urls.yaml'),'utf8'); + const cacheExists = await fs.pathExists('./cache.db'); + if(!cacheExists) { + yaml.safeLoad(scamsFile).map(entry => new Scam(entry)).forEach(entry => db.scams.push(entry)); + yaml.safeLoad(verifiedFile).forEach(entry => db.verified.push(entry)); + } else { + const cacheFile = await fs.readFile('./cache.db','utf8'); + Object.assign(db,serialijse.deserialize(cacheFile)); + yaml.safeLoad(scamsFile).filter(entry => !db.scams.find(scam => scam.url == entry.url)).map(entry => new Scam(entry)).forEach(entry => db.scams.push(entry)); + yaml.safeLoad(verifiedFile).filter(entry => !db.verified.find(verified => verified.url == entry.url)).forEach(entry => db.verified.push(entry)); + } +} + +const updateIndex = async () => { + debug("Updating index..."); + const scamDictionary = createDictionary(db.scams); + const verifiedDictionary = createDictionary(db.verified); + + db.index.featured = db.verified.filter(entry => entry.featured).sort((a,b) => a.name.localeCompare(b.name)); + db.index.blacklist = [...db.scams.map(entry => entry.getHostname().replace('www.','')),...db.scams.map(entry => entry.getHostname().replace('www.','')),...Object.keys(scamDictionary.ip || {})]; + db.index.whitelist = [...db.verified.map(entry => url.parse(entry.url).hostname.replace('www.','')),...db.verified.map(entry => 'www.' + url.parse(entry.url).hostname.replace('www.',''))]; + db.index.whitelistAddresses = (verifiedDictionary.addresses || []); + db.index.addresses = (scamDictionary.addresses || []); + db.index.ips = (scamDictionary.ip || []); + db.index.inactives = db.scams.filter(scam => scam.status !== 'Active'); + db.index.actives = db.scams.filter(scam => scam.status === 'Active'); +} + +const exitHandler = () => { + console.log("Cleaning up..."); + fs.writeFileSync('./cache.db',serialijse.serialize(db)); + console.log("Exited."); +} + +module.exports.init = async () => { + await readEntries(); + await updateIndex(); + await module.exports.persist(); + if(config.interval.databasePersist > 0) setInterval(module.exports.persist,config.interval.databasePersist); + process.stdin.resume(); + process.once('beforeExit', exitHandler); + process.once('SIGINT', exitHandler); + process.once('SIGTERM', exitHandler); +} + +module.exports.read = () => db; + +module.exports.write = (url,data) => { + const scam = db.scams.find(scam => scam.url == url); + Object.keys(data).forEach(key => scam[key] = data[key]); + updateIndex(); +} + +module.exports.persist = async () => { + debug("Persisting cache..."); + await fs.writeFile('./cache.db',serialijse.serialize(db)); +} + +module.exports.exitHandler = exitHandler; + +module.exports.readEntries = readEntries; + +module.exports.updateIndex = updateIndex; \ No newline at end of file diff --git a/src/utils/github.js b/src/utils/github.js new file mode 100644 index 00000000..2f2e49c7 --- /dev/null +++ b/src/utils/github.js @@ -0,0 +1,45 @@ +const request = require('request'); +const download = require('download'); +const crypto = require('crypto'); +const config = require('./config'); +const db = require('./db'); +const debug = require('debug')('github'); + +const pullDataFiles = async () => { + debug("Pulling data files..."); + await download("https://raw.githubusercontent.com/" + config.autoPull.repository.author + "/" + config.autoPull.repository.name + "/" + config.autoPull.repository.branch + "/_data/scams.yaml", "_data"); + await download("https://raw.githubusercontent.com/" + config.autoPull.repository.author + "/" + config.autoPull.repository.name + "/" + config.autoPull.repository.branch + "/_data/legit_urls.yaml", "_data"); + await download("https://raw.githubusercontent.com/" + config.autoPull.repository.author + "/" + config.autoPull.repository.name + "/" + config.autoPull.repository.branch + "/_data/twitter.json", "_data"); + debug("Done"); +} + +module.exports.webhook = async (req,res) => { + if(!config.apiKeys.Github_WebHook) { + debug("Warning: Incoming Github Webhook attempt - no secret found in config"); + res.status(403).end(); + } else if(!('x-hub-signature' in req.headers)) { + debug("Warning: Incoming Github Webhook attempt without x-hub-signature header"); + res.status(403).end(); + } else { + const githubSig = Buffer.from(req.headers['x-hub-signature']); + const localSig = Buffer.from("sha1=" + crypto.createHmac("sha1", config.apiKeys.Github_WebHook).update(req.rawBody).digest("hex")); + if(crypto.timingSafeEqual(githubSig,localSig)) { + debug("Valid incoming Github webhook!"); + await pullDataFiles(); + await db.readEntries(); + await db.updateIndex(); + await db.persist(); + res.status(200).end(); + } else { + debug("Warning: Invalid Github webhook attempt"); + res.status(403).end(); + } + } +} + +module.exports.pullData = async () => { + await pullDataFiles(); + await db.readEntries(); + await db.updateIndex(); + await db.persist(); +} \ No newline at end of file diff --git a/src/utils/lookup.js b/src/utils/lookup.js new file mode 100644 index 00000000..e1f402a6 --- /dev/null +++ b/src/utils/lookup.js @@ -0,0 +1,93 @@ +const request = require('request'); +const config = require('./config'); +const debug = require('debug')('lookup'); +const Bottleneck = require('bottleneck'); + +const limiter = new Bottleneck({ + minTime: config.lookups.HTTP.minTime, + maxConcurrent: config.lookups.HTTP.maxConcurrent +}); + +module.exports.lookup = limiter.wrap(url => { + return new Promise(resolve => { + debug('Requesting ' + url + '...'); + request({ + url: url, + timeout: config.lookups.HTTP.timeoutAfter, + followAllRedirects: true, + maxRedirects: 5 + }, (err, response, body) => { + if(err) resolve(undefined); + else resolve(response); + }); + }); +}); + +module.exports.getURLScan = (url) => { + return new Promise((resolve, reject) => { + request('https://urlscan.io/api/v1/search/?q=domain%3A' + url, { json: true }, (err, response, body) => { + if(err) { + reject(err); + } else { + resolve(body) + } + }); + }); +} + +module.exports.getGoogleSafeBrowsing = (url) => { + return new Promise((resolve,reject) => { + debug("Google SafeBrowsing: %o",url); + request({ + url: 'https://safebrowsing.googleapis.com/v4/threatMatches:find?key=' + encodeURIComponent(config.apiKeys.Google_SafeBrowsing), + method: 'POST', + headers: { + "Content-Type": "application/json" + }, + json: { + client: { + clientId: "EtherScamDB", + clientVersion: "3.0.0" + }, + threatInfo: { + threatTypes: ["THREAT_TYPE_UNSPECIFIED", "MALWARE", "SOCIAL_ENGINEERING", "UNWANTED_SOFTWARE", "POTENTIALLY_HARMFUL_APPLICATION"], + platformTypes: ["PLATFORM_TYPE_UNSPECIFIED","WINDOWS","LINUX","ANDROID","OSX","IOS","ANY_PLATFORM","ALL_PLATFORMS","CHROME"], + threatEntryTypes: ["THREAT_ENTRY_TYPE_UNSPECIFIED", "URL", "EXECUTABLE"], + threatEntries: [{ + url: url + }] + } + } + }, (err, response, body) => { + debug("%s returned %s %o",url,((response || {}).statusCode || -1),body); + if(err) { + reject(err); + } else if(response.statusCode != 200) { + reject("Google SafeBrowsing returned an invalid status code"); + } else if(body && body.matches && body.matches[0]) { + resolve(body.matches[0]); + } else { + resolve(false); + } + }); + }); +} + +module.exports.getVirusTotal = (url) => { + return new Promise((resolve,reject) => { + request({ + uri: 'https://www.virustotal.com/vtapi/v2/url/report?apikey=' + config.VirusTotal_API_Key + '&resource=http://' + url, + json: true + }, (err, response, body) => { + if(err) { + reject(err); + } else if(response.statusCode != 200) { + reject("VirusTotal returned an invalid status code"); + } else if(body.response_code == 0) { + reject("VirusTotal returned an invalid internal status code"); + } else { + resolve(body); + } + }); + }); +} \ No newline at end of file diff --git a/src/utils/router.js b/src/utils/router.js new file mode 100644 index 00000000..e8f878b2 --- /dev/null +++ b/src/utils/router.js @@ -0,0 +1,251 @@ +const express = require('express'); +const db = require('./db'); +const generateAbuseReport = require('./abusereport'); +const checkForPhishing = require('eth-phishing-detect'); +const dateFormat = require('dateformat'); +const url = require('url'); +const config = require('./config'); +const github = require('./github'); +const router = express.Router(); +const isIpPrivate = require('private-ip'); +const {getGoogleSafeBrowsing,getURLScan} = require('./lookup'); +const debug = require('debug')('router') + +/* Homepage */ +router.get('/(/|index.html)?', (req, res) => res.render('index')); + +/* FAQ page */ +router.get('/faq/', (req, res) => res.render('faq')); + +/* API documentation page */ +router.get('/api/', (req, res) => res.render('api')); + +/* Report pages */ +router.get('/report/', (req, res) => res.render('report')); + +router.get('/report/domain/:domain', (req, res) => res.render('report', { + domain: req.params.domain +})); + +router.get('/report/address/:address', (req, res) => res.render('report', { + address: req.params.address +})); + +/* IP pages */ +router.get('/ip/:ip', async (req, res) => { + const entry = db.read() + console.log(JSON.stringify(entry.index.ips, null, 2)) + await res.render('ip', { + ip: req.params.ip, + isPrivate: isIpPrivate(req.params.ip), + related: (entry.index.ips[req.params.ip] || []) + }) +}); + +/* Address pages */ +router.get('/address/:address', async (req, res) => { + const entry = await db.read() + if(entry.index.whitelistAddresses[req.params.address]) {6 + res.render('address', { + address: req.params.address, + related: (entry.index.whitelistAddresses[req.params.address]), + type: "verified" + }) + } + else if(entry.index.addresses[req.params.address]) { + res.render('address', { + address: req.params.address, + related: (entry.index.addresses[req.params.address] || []), + type: "scam" + }) + } + else { + res.render('address', { + address: req.params.address, + related: (entry.index.addresses[req.params.address] || []), + type: "neutral" + }) + } +}); + +/* (dev) Add scam page */ +router.get('/add/', (req,res) => { + const {NODE_ENV} = process.env; + if(NODE_ENV === 'development') res.render('add'); + else res.send(403).end(); +}); + +/* Domain pages */ +router.get('/domain/:url', async (req, res) => { + const startTime = Date.now(); + const {hostname} = url.parse('http://' + req.params.url.replace('http://','').replace('https://')); + const scamEntry = db.read().scams.find(scam => scam.getHostname() == hostname); + const verifiedEntry = db.read().verified.find(verified => url.parse(verified.url).hostname == hostname); + + const urlScan = await getURLScan(hostname); + let googleSafeBrowsing = undefined; + let virusTotal = undefined; + + if((scamEntry || !verifiedEntry) && config.apiKeys.Google_SafeBrowsing) googleSafeBrowsing = await getGoogleSafeBrowsing(hostname); + if((scamEntry || !verifiedEntry) && config.apiKeys.VirusTotal) virusTotal = await virusTotal(hostname); + + if(verifiedEntry) res.render('domain', { type: 'verified', result: verifiedEntry, domain: hostname, urlScan: urlScan, metamask: false, googleSafeBrowsing: googleSafeBrowsing, virusTotal: virusTotal, startTime: startTime, dateFormat: dateFormat }); + else if(scamEntry) res.render('domain', { type: 'scam', result: scamEntry, domain: hostname, urlScan: urlScan, metamask: checkForPhishing(hostname), googleSafeBrowsing: googleSafeBrowsing, virusTotal: virusTotal, startTime: startTime, dateFormat: dateFormat, abuseReport: generateAbuseReport(scamEntry) }); + else res.render('domain', { type: 'neutral', domain: hostname, result: false, urlScan: urlScan, metamask: checkForPhishing(hostname), googleSafeBrowsing: googleSafeBrowsing, virusTotal: virusTotal, addresses: [], startTime: startTime }); +}); + +/* Scams index */ +router.get('/scams/:page?/:sorting?/', (req, res) => { + const MAX_RESULTS_PER_PAGE = 30; + const scamList = []; + let scams = [...db.read().scams].reverse(); + let index = [0,MAX_RESULTS_PER_PAGE]; + + if(req.params.page && req.params.page != 'all' && (!isFinite(parseInt(req.params.page)) || isNaN(parseInt(req.params.page)) || parseInt(req.params.page) < 1)) { + res.status(404).render('404'); + } else { + if (req.params.sorting == 'oldest') scams = db.read().scams; + else if (req.params.sorting == 'status') scams = [...db.read().scams].sort((a,b) => (a.status || '').localeCompare(b.status || '')); + else if (req.params.sorting == 'category') scams = [...db.read().scams].sort((a,b) => (a.category || '').localeCompare(b.category || '')); + else if (req.params.sorting == 'subcategory') scams = [...db.read().scams].sort((a,b) => (a.subcategory || '').localeCompare(b.subcategory || '')); + else if (req.params.sorting == 'name') scams = [...db.read().scams].sort((a,b) => a.getHostname().localeCompare(b.getHostname())); + + if (req.params.page == "all") index = [0,scams.length-1]; + else if(req.params.page) index = [(req.params.page-1) * MAX_RESULTS_PER_PAGE,(req.params.page * MAX_RESULTS_PER_PAGE)]; + + for (var i = index[0]; i <= index[1]; i++) { + if (scams.hasOwnProperty(i) === false) continue; + scamList.push(scams[i]); + } + + res.render('scams', { + 'page': req.params.page, + 'sorting': req.params.sorting, + 'total': scams.length.toLocaleString('en-US'), + 'active': Object.keys(scams.filter(scam => scam.status === 'Active')).length.toLocaleString('en-US'), + 'total_addresses': Object.keys(db.read().index.addresses).length.toLocaleString('en-US'), + 'inactive': Object.keys(scams.filter(scam => scam.status === 'Inactive')).length.toLocaleString('en-US'), + 'scams': scamList, + 'MAX_RESULTS_PER_PAGE': MAX_RESULTS_PER_PAGE, + 'scamsLength': scams.length + }); + } +}); + +/* Search pages */ +router.get('/search/', (req, res) => res.render('search', { featured: db.read().index.featured })); + +/* RSS */ +router.get('/rss/', (req, res) => res.render('rss', { scams: db.read().scams })); + +/* API middleware */ +router.use('/api/:type?/:domain?/', (req,res,next) => { + res.header('Access-Control-Allow-Origin', '*'); + next(); +}); + +router.get('/api/scams', (req, res) => res.json({ success: true, result: db.read().scams })); +router.get('/api/addresses', (req, res) => res.json({ success: true, result: db.read().index.addresses })); +router.get('/api/ips', (req, res) => res.json({ success: true, result: db.read().index.ips })); +router.get('/api/verified', (req, res) => res.json({ success: true, result: db.read().verified })); +router.get('/api/inactives', (req, res) => res.json({ success: true, result: db.read().index.inactives })); +router.get('/api/actives', (req, res) => res.json({ success: true, result: db.read().index.actives })); +router.get('/api/blacklist', (req, res) => res.json(db.read().index.blacklist)); +router.get('/api/whitelist', (req, res) => res.json(db.read().index.whitelist)); +router.get('/api/abusereport/:domain', (req, res) => { + const result = db.read().scams.find(scam => scam.getHostname() == url.parse(req.params.domain).hostname || scam.url.replace(/(^\w+:|^)\/\//, '') == req.params.domain); + if (result) res.json({ success: false, message: "URL wasn't found"}); + else res.send({ success: true, result: generateAbuseReport(result)}); +}); +router.get('/api/check/:search', (req,res) => { + if (/^0x?[0-9A-Fa-f]{40,42}$/.test(req.params.search)) { + /* Searched for an ethereum address */ + const whitelistAddresses = Object.keys(db.read().index.whitelistAddresses).filter(address => req.params.search.toLowerCase() === address.toLowerCase()); + const blacklistAddresses = Object.keys(db.read().index.addresses).filter(address => req.params.search.toLowerCase() === address.toLowerCase()); + if(whitelistAddresses.length > 0) { + res.json({ + success: true, + result: 'whitelisted', + type: 'address', + entries: whitelistAddresses.map(address => db.read().index.whitelistAddresses[address]) + }); + } else if(blacklistAddresses.length > 0) { + res.json({ + success: true, + result: 'blocked', + type: 'address', + entries: blacklistAddresses.map(address => db.read().index.addresses[address]) + }); + } else { + res.json({ + success: true, + result: 'neutral', + type: 'address', + entries: [] + }); + } + } else if(/[-a-zA-Z0-9@:%._\+~#=]{2,256}\.[a-z]{2,6}\b([-a-zA-Z0-9@:%_\+.~#?&//=]*)/.test(req.params.search)) { + /* Searched for a domain */ + const whitelistURL = db.read().verified.find(entry => (url.parse(req.params.search).hostname || req.params.search) === (url.parse(entry.url).hostname)); + const blacklistURL = db.read().scams.find(entry => (url.parse(req.params.search).hostname || req.params.search) === entry.getHostname()); + if(whitelistURL) { + res.json({ + success: true, + result: 'verified', + type: 'domain', + entries: [whitelistURL] + }); + } else if(blacklistURL) { + res.json({ + success: true, + result: 'blocked', + type: 'domain', + entries: [blacklistURL] + }); + } else { + res.json({ + success: true, + result: 'neutral', + type: 'domain', + entries: [] + }); + } + } else if(/^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$|^(([a-zA-Z]|[a-zA-Z][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z]|[A-Za-z][A-Za-z0-9\-]*[A-Za-z0-9])$|^\s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?\s*$/.test(req.params.search)) { + /* Searched for an ip address */ + const blacklistIP = Object.keys(db.read().index.ips).filter(ip => req.params.search.toLowerCase() === ip.toLowerCase()); + if(blacklistIP.length > 0) { + res.json({ + success: true, + result: 'blocked', + type: 'ip', + entries: blacklistIP + }); + } else { + res.json({ + success: true, + result: 'neutral', + type: 'ip', + entries: [] + }); + } + } else { + res.json({ + success: false, + message: "Incorrect search type (must be ethereum address / ip address / URL)" + }); + } +}); + +/* Incoming Github webhook attempt */ +router.post('/update/', (req, res) => { + req.rawBody = ''; + req.setEncoding('utf8'); + req.on('data', chunk => req.rawBody += chunk); + req.on('end', () => github.webhook(req,res)); +}); + +/* Safe redirect pages */ +router.get('/redirect/:url', (req,res) => res.render('redirect', { url: req.params.url })); + + +module.exports = router; diff --git a/src/utils/writeConfig.js b/src/utils/writeConfig.js new file mode 100644 index 00000000..1c6754bd --- /dev/null +++ b/src/utils/writeConfig.js @@ -0,0 +1,69 @@ +const fs = require('fs-extra'); + +module.exports = async (options) => { + let httpMinTime = null; + let httpMaxConcurrent = null; + let httpTimeoutAfter = null; + let dnsServers = []; + + if(options['http-bottleneck'] == 'fast') { + httpMinTime = 0; + httpMaxConcurrent = 200; + } else if(options['http-bottleneck'] == 'regular') { + httpMinTime = 100; + httpMaxConcurrent = 20; + } else if(options['http-bottleneck'] == 'slow') { + httpMinTime = 500; + httpMaxConcurrent = 5; + } + + if(options['dns-servers'] == 'cloudflare') dnsServers = ['1.1.1.1','1.0.0.1']; + else if(options['dns-servers'] == 'google') dnsServers = ['8.8.8.8','8.8.4.4']; + else if(options['dns-servers'] == 'opendns') dnsServers = ['208.67.222.222','208.67.220.220']; + + if(options['http-timeout'] == 'highly-accurate') httpTimeoutAfter = null; + else if(options['http-timeout'] == 'accurate') httpTimeoutAfter = 15*1000; + else if(options['http-timeout'] == 'less-accurate') httpTimeoutAfter = 5*1000; + + const config = { + port: parseInt(options.port), + announcement: null, + interval: { + cacheExpiration: 1000 * 60 * parseInt(options['cache-expiration']), + cacheRenewCheck: 1000 * 60 * parseInt(options['cache-renewal']), + databasePersist: 1000 * parseInt(options['database-persist']) + }, + apiKeys: { + Google_SafeBrowsing: options['google-safebrowsing'] || null, + Github_WebHook: null, + VirusTotal: options.virustotal || null + }, + autoPull: { + enabled: (options.pull == "on"), + interval: 1000 * 60 * 2, + repository: { + author: "MrLuit", + name: "EtherScamDB", + branch: "master" + } + }, + lookups: { + DNS: { + servers: dnsServers, + IP: { + enabled: (options.mode === "safe" || options.mode == "full") + }, + NS: { + enabled: (options.mode === "safe" || options.mode == "full") + } + }, + HTTP: { + enabled: (options.mode == "full"), + minTime: httpMinTime, + maxConcurrent: httpMaxConcurrent, + timeoutAfter: httpTimeoutAfter + } + } + } + await fs.writeJson('./config.json',config,{ spaces: 4 }); +} \ No newline at end of file diff --git a/src/views/pages/404.ejs b/src/views/pages/404.ejs new file mode 100644 index 00000000..8244c862 --- /dev/null +++ b/src/views/pages/404.ejs @@ -0,0 +1,18 @@ + + + + <% include ../partials/header.ejs %> + + + + <% include ../partials/navbar.ejs %> + + <% include ../partials/footer.ejs %> + + \ No newline at end of file diff --git a/src/views/pages/add.ejs b/src/views/pages/add.ejs new file mode 100644 index 00000000..18e85822 --- /dev/null +++ b/src/views/pages/add.ejs @@ -0,0 +1,54 @@ + + + + <% include ../partials/header.ejs %> + + +
+ ++
+ ++
+ ++
+ ++
+ ++
+ ++
+ ++
+ ++
+ ++
+ ++
Because your app was launched into production mode, the configuration process cannot be started. Please relaunch into development mode or manually create the config.json
+ <% } else if(done) { %> +Your preferred configuration was saved successfully. Please restart this application for changes to apply.
+ <% } else { %> + + <% } %> +This page was built in <%= Date.now()-startTime %>ms
+<% } else if(result.updated == "0") { %> +This page was built in <%= Date.now()-startTime %>ms, and never updated
+<% } else { %> +This page was built in <%= Date.now()-startTime %>ms, and last updated at <%= dateFormat(result.updated, "UTC:mmm dd yyyy, HH:MM") %> UTC
+<% } %> +All reports end up in a private Slack to protect people's privacy. The reports can be read by all team members of MyCrypto and they can decide whether the report makes it to the blacklist.
+The database was created in 2017 when trying to find a solution to the ethereum scams. Of course grouping all the scams won't make them go away, but it will make identifying them and taking them down easier. It is now being maintained by members of the MyCrypto team.
+
+ .replace(/ /g, "") %>.png)
<%= entry.name %>
+
+
+
+
+
.replace(/\s/g, "") %>.png)
<%= scam.subcategory %>