Add VPN list and applied-privacy subnet list

graphicsmagick
Cadence Ember 3 years ago
parent db364721d8
commit 271afec9db
No known key found for this signature in database
GPG Key ID: 128B99B1B74A6412
  1. 5
      package-lock.json
  2. 1
      package.json
  3. 2
      scripts/process_log.js
  4. 54
      src/lib/quota/get_identifier.js
  5. 4
      src/lib/quota/index.js

5
package-lock.json generated

@ -2175,6 +2175,11 @@
"is-extglob": "^2.1.1"
}
},
"is-in-subnet": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/is-in-subnet/-/is-in-subnet-3.1.0.tgz",
"integrity": "sha512-UpHSoZ+S53dhiM5q48atdahm6GmaTFlZaelKTdtBQeSihWZdToE7b8hlTzWVVLghntGlCK97c1JTIdzV5Q+Vcw=="
},
"is-number": {
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",

@ -20,6 +20,7 @@
"cookie": "^0.4.1",
"feed": "git+https://git.sr.ht/~cadence/nodejs-feed#3dde82f8296d7a6f5659323e497e0c684f03ab71",
"get-stream": "^5.1.0",
"is-in-subnet": "^3.1.0",
"mixin-deep": "^2.0.1",
"node-dir": "^0.1.17",
"node-fetch": "^2.6.0",

@ -175,7 +175,7 @@ reader.on("line", line => {
if (kind) {
kinds[kind]++
dateCollection.add(kind, dateObject.getTime())
if (kind === "api") ips.add(parsed.ip)
ips.add(parsed.ip)
}
})

@ -1,25 +1,53 @@
const {request} = require("../utils/request")
const {log} = require("pinski/util/common")
const constants = require("../constants.js")
const {createChecker} = require("is-in-subnet")
let addresses = []
let addressSet = new Set()
let subnets = []
let checker = createChecker([])
request("https://check.torproject.org/torbulkexitlist").text().then(text => {
const lines = text.split("\n").filter(l => l)
addresses = addresses.concat(lines)
log(`Loaded Tor exit node list (${addresses.length} total)`, "spam")
})
function getList(url, description) {
return request(url).text().then(text => {
// this is decently fast, but if you have ideas for optimisation, please go for it
let d = Date.now()
const lines = text.split("\n").filter(l => l && l[0] !== "#")
subnets = subnets.concat(lines.filter(l => l.includes("/")))
addresses = addresses.concat(lines.filter(l => !l.includes("/")))
log(`Loaded ${description} (entries: ${lines.length}) (${Date.now()-d} ms)`, "spam")
})
}
/*
request("https://meta.bibliogram.art/ip_proxy_list.txt").text().then(text => {
const lines = text.split("\n").filter(l => l)
addresses = addresses.concat(lines)
log(`Loaded Bibliogram proxy list (${addresses.length} total)`, "spam")
if (constants.quota.enabled) {
Promise.all([
getList("https://check.torproject.org/torbulkexitlist", "Tor exit node list"),
getList("https://meta.bibliogram.art/quota-list/vpn-ipv4.txt", "VPN IPv4 list"),
getList("https://meta.bibliogram.art/quota-list/applied-privacy.txt", "applied-privacy.net subnets")
]).then(() => {
let d = Date.now()
checker = createChecker(subnets)
addressSet = new Set(addresses.values())
log(`Created subnet checker (${Date.now()-d} ms)`, "spam")
})
*/
}
function getIdentifier(address) {
if (addresses.includes(address)) return "proxy"
else return address
let d = Date.now()
const result = (() => {
try {
if (address == undefined) return "missing"
else if (checker(address)) return "proxy"
else if (addressSet.has(address)) return "proxy"
else return address
} catch (e) {
// not a valid IP address, or some error like that
console.error(e)
throw e
}
})()
console.log(`identified ${address} -> ${result} in ${Date.now()-d} ms`)
return result
}
module.exports.getIdentifier = getIdentifier

@ -20,7 +20,7 @@ function remaining(req) {
if (!constants.quota.enabled) return Infinity // sure.
const ip = getIPFromReq(req)
const identifier = String(getIdentifier(ip))
const identifier = getIdentifier(ip)
const remaining = limiter.remaining(identifier)
if (constants.quota.track) {
@ -34,7 +34,7 @@ function add(req, count) {
if (!constants.quota.enabled) return Infinity // why not.
const ip = getIPFromReq(req)
const identifier = String(getIdentifier(ip))
const identifier = getIdentifier(ip)
return limiter.add(identifier, count)
}

Loading…
Cancel
Save