#!/usr/bin/env python3 """ Handle DDOS by barring bad IPs temporarily. Country lookups use the external geoip3 command (output: "CC IP"). Any lookup result matching IGNORE_PATTERN is never blocked. Single IPs are added to the ipset ddos_ips_min, subnets to ddos_nets_min. Without ipset, falls back to "ip route add prohibit". """ import argparse import fcntl import ipaddress import os import re import shutil import subprocess import sys import time sys.path.insert(0, "/usr/local/maintenance") try: import config except ImportError: config = None try: import frob # it's frob, not frob.py except ImportError: frob = None GEOIP_CMD = "/usr/local/afribin/geoip3" # Equivalent of: grep -Ev "ZA|__ 100\.|__ 10\.|127.0.0.1|__ 102\.|PT|0.0.0.0" # applied to the geoip3 output line ("CC IP"). A match means: never block. IGNORE_PATTERN = r"ZA|__ 100\.|__ 10\.|127.0.0.1|__ 102\.|PT|0.0.0.0" IPSET_IPS = "ddos_ips_min" IPSET_NETS = "ddos_nets_min" SUCURI = ( # Sucuri proxy that does actually cache very well: "192.124.249.0/24", "192.88.134.0/24", "192.88.135.0/24", "185.93.228.0/24", "185.93.229.0/24", "185.93.230.0/24", "192.161.0.0/24", "192.81.128.0/22", "154.127.61.210", "154.66.197.69", "197.85.187.18", "197.85.186.185", "66.248.200.0/22", "185.93.228.0/22", "192.88.134.0/23", "207.46.0.0/16", "197.242.144.0/21", "41.76.208.0/21", "197.242.152.0/21", "41.76.212.0/21", "154.0.0.0/16", ) CLOUDFLARE = ( # Cloudflare and mail.faxfx.biz get a free pass: "199.27.128.0/21", "173.245.48.0/20", "103.21.244.0/22", "103.22.200.0/22", "103.31.4.0/22", "141.101.64.0/18", "108.162.192.0/18", "190.93.240.0/20", "188.114.96.0/20", "197.234.240.0/22", "198.41.128.0/17", "162.158.0.0/15", "104.16.0.0/12", "196.25.125.70", "131.0.72.0/22", "172.64.0.0/13", ) BASE_WHITELIST = ( "127.0.0.0/8", "100.64.0.0/10", "10.0.0.0/8", "192.168.0.0/16", "169.254.0.0/16", "172.16.0.0/12", "196.30.224.202", "127.2.1.1", "197.242.144.0/21", "41.76.208.0/21", "197.242.152.0/21", "41.76.212.0/21", "154.0.0.0/16", "102.182.0.0/16", "165.73.0.0/16", "41.76.208.209", ) # Networks we block whenever load is high (value is ignored). # e.g. bingbot ranges: '65.52.0.0/14', '207.46.0.0/16', '157.56.0.0/14', ... NETWORK_BLOCKLIST = { } # -------------------------------------------------------------------------- # Helpers # -------------------------------------------------------------------------- def run_capture(argv): """Run a command and return its stdout ('' if it can't run).""" try: return subprocess.run(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, check=False).stdout except OSError: return "" class IPList: """IPv4 hosts and networks with subnet-aware membership tests.""" def __init__(self, addresses=()): self.hosts = set() self.networks = [] self.extend(addresses) @staticmethod def parse(addr): """Accept 1.2.3.4, 1.2.3.0/24, 1.2.3.0/255.255.255.0 or shorthand 1. / 1.2. / 1.2.3.""" addr = addr.strip() if addr.endswith("."): dots = addr.count(".") addr = addr + ".".join(["0"] * (4 - dots)) + "/" + str(dots * 8) return ipaddress.IPv4Network(addr, strict=False) def _covers(self, net): if net.prefixlen == 32 and net.network_address in self.hosts: return True # subnet_of() needs Python 3.7+; check both ends of the range instead return any(net.network_address in n and net.broadcast_address in n for n in self.networks) def __contains__(self, addr): try: return self._covers(self.parse(addr)) except ValueError: return False def append(self, addr): try: net = self.parse(addr) except ValueError: print(f"# ignoring invalid whitelist entry {addr!r}", file=sys.stderr) return if self._covers(net): return if net.prefixlen == 32: self.hosts.add(net.network_address) else: self.networks.append(net) def extend(self, addresses): for addr in addresses: self.append(addr) def __iter__(self): for host in self.hosts: yield str(host) for net in self.networks: yield str(net) def __str__(self): return " ".join(self) class GeoLookup: """Country lookups via the external geoip3 command, with caching.""" def __init__(self, cmd, ignore_pattern, country_whitelist): self.cmd = cmd self.ignore_re = re.compile(ignore_pattern) self.country_whitelist = {c.upper() for c in country_whitelist} self.cache = {} self.available = bool(cmd) and os.access(cmd, os.X_OK) def lookup(self, ip): """Return geoip3 output, e.g. 'US 35.246.237.40', or '' on failure.""" if ip in self.cache: return self.cache[ip] if not self.available: return "" try: # Equivalent of: echo | /usr/local/geoip3 r = subprocess.run([self.cmd], input=ip + "\n", stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True, timeout=5) lines = r.stdout.strip().splitlines() line = lines[0].strip() if lines else "" except (subprocess.TimeoutExpired, OSError) as e: print(f"# geoip3 lookup failed for {ip}: {e}", file=sys.stderr) return "" if line: if len(self.cache) > 50000: self.cache.clear() self.cache[ip] = line return line def should_ignore(self, target): """Return (ignore, geoip_line). Subnets are looked up by their network address.""" ip = target.split("/", 1)[0] line = self.lookup(ip) if not line: return False, "" if self.ignore_re.search(line): return True, line if line.split()[0].upper() in self.country_whitelist: return True, line return False, line def have_ipset(): """True if ipset is installed and both target sets exist.""" if not shutil.which("ipset"): return False r = subprocess.run(["ipset", "list", "-n"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) if r.returncode != 0: return False sets = set(r.stdout.split()) missing = [s for s in (IPSET_IPS, IPSET_NETS) if s not in sets] if missing: print(f"# ipset found but missing set(s): {', '.join(missing)}", file=sys.stderr) return False return True def local_addresses(): return re.findall(r"\binet\s+([0-9.]+)", run_capture(["ip", "-4", "-o", "addr", "show"])) def read_connections(testfile=""): """Yield (local_port, remote_ip) for IPv4 connections from netstat -ntu (or ss -ntu).""" if testfile: with open(testfile) as f: lines = f.read().splitlines() local_col, remote_col = 3, 4 elif shutil.which("netstat"): lines = run_capture(["netstat", "-ntu"]).splitlines() local_col, remote_col = 3, 4 else: lines = run_capture(["ss", "-ntu"]).splitlines() local_col, remote_col = 4, 5 for line in lines: bits = line.split() if len(bits) <= remote_col: continue rhost = bits[remote_col].rpartition(":")[0].strip("[]") if rhost.startswith("::ffff:"): rhost = rhost[7:] try: ipaddress.IPv4Address(rhost) except ValueError: continue # header line, IPv6, etc. yield bits[local_col].rpartition(":")[2], rhost def run_actions(actions, no_act): """actions: list of (argv, comment, quiet).""" for argv, comment, quiet in actions: print(" ".join(argv) + (f" # {comment}" if comment else "")) if no_act: continue dev = subprocess.DEVNULL if quiet else None try: subprocess.run(argv, stdout=dev, stderr=dev, check=False) except OSError as e: print(f"# failed to run {argv[0]}: {e}", file=sys.stderr) # -------------------------------------------------------------------------- # Main # -------------------------------------------------------------------------- def main(opts): only_ports = {str(p) for p in opts.only_ports} exclude_ports = {str(p) for p in (opts.exclude_ports or [3306])} idle_seconds = opts.idle * 60 blocklist = {} if (opts.no_robots_blacklist or opts.load == 0) else dict(NETWORK_BLOCKLIST) next_antibruteforce = time.time() if opts.antibruteforce else 0 use_ipset = opts.ipset or have_ipset() if use_ipset: def block_cmd(target, is_net): return ["ipset", "-!", "add", IPSET_NETS if is_net else IPSET_IPS, target] unblock_cmd = None # ipset entries expire via the set's own timeout else: def block_cmd(target, is_net): return ["ip", "route", "add", "prohibit", target] def unblock_cmd(target): return ["ip", "route", "del", "prohibit", target] # Delete leftover prohibit routes ip_route_ls = run_capture(["ip", "route", "ls"]).splitlines() for line in ip_route_ls: if "prohibit" in line: run_actions([(["ip", "route", "del"] + line.split(), "", False)], opts.no_act) # Build the whitelist whitelist = IPList(BASE_WHITELIST) country_whitelist = [] for arg in opts.whitelist: if len(arg) == 2 and arg.isalpha(): country_whitelist.append(arg) else: whitelist.append(arg) whitelist_config, skip_bruteforce, skip_ddos = True, False, False if config: whitelist_config = config.option("whitelist") skip_bruteforce = config.option("skip_bruteforce") skip_ddos = config.option("skip_ddos") if whitelist_config and isinstance(whitelist_config, str): for ipaddr in re.split(r"[\s,;]+", whitelist_config): if ipaddr: whitelist.append(ipaddr) try: with open("/etc/resolv.conf") as f: for m in re.finditer(r"nameserver\s+([0-9.]+)", f.read()): whitelist.append(m.group(1)) except OSError: pass whitelist.extend(local_addresses()) # Immediate neighbours are cool for line in ip_route_ls: m = re.match(r"([0-9.]{7,16}/\d{1,2})", line) if m: whitelist.append(m.group(1)) whitelist.extend(CLOUDFLARE) whitelist.extend(SUCURI) ssh_client = os.environ.get("SSH_CLIENT", "").split() if ssh_client and ssh_client[0].count(".") == 3: whitelist.append(ssh_client[0]) geo = GeoLookup(opts.geoip_cmd, opts.ignore_pattern, country_whitelist) if skip_ddos: print("skip_ddos is defined on this system") return queuelen = int((opts.time + opts.sleep - 1) / opts.sleep) print(f"# {time.strftime('%Y-%m-%d %H:%M:%S')} {sys.argv[0]}") print(f"# When load above -l {opts.load},\n" f"# checking every -s {opts.sleep} and\n" f"# blocking for -t {opts.sleep * queuelen} seconds, when there are over\n" f"# -c {opts.connections} connections per IP or\n" f"# -C {opts.connections24} connections per /24") print("# Ports allowed: " + " ".join(f"-p {p}" for p in sorted(only_ports))) print("# Ports excluded: " + " ".join(f"-P {p}" for p in sorted(exclude_ports))) print(f"# Block method: {'ipset ' + IPSET_IPS + ' / ' + IPSET_NETS if use_ipset else 'ip route prohibit'}") print(f"# GeoIP: {opts.geoip_cmd if geo.available else 'NOT AVAILABLE - country filter disabled'}") print(f"# GeoIP ignore pattern: {opts.ignore_pattern}") if country_whitelist: print(f"# Country whitelist: {' '.join(country_whitelist)}") print(f"# Network whitelist: {whitelist}") print(f"# Network blocklist: {' '.join(blocklist)}") sys.stdout.flush() all_blocked = set() unblock_queue = [] # list of lists of unblock actions unblock_queue_ips = [] # list of lists of blocked targets last_act = time.time() while True: with open("/proc/loadavg") as f: load = float(f.read().split()[0]) actions, unblock_actions, blocked_now = [], [], [] max_count, max_ip, net_max_count, net_max_ip = 0, "", 0, "" # Anti-brute-force runs regardless of load if not skip_bruteforce and next_antibruteforce and time.time() >= next_antibruteforce: next_antibruteforce = time.time() + opts.antibruteforce * 60 if frob: frob.do_antibruteforce([], seconds=60) if load <= opts.load: if opts.verbose > 1: print(f"Load of {load} ok") else: ipcount, netcount = {}, {} for lport, ip in read_connections(opts.testfile): if only_ports and lport not in only_ports: continue if lport in exclude_ports: continue net24 = ".".join(ip.split(".")[:3]) + ".0/24" ipcount[ip] = ipcount.get(ip, 0) + 1 netcount[net24] = netcount.get(net24, 0) + 1 for is_net, counts, limit, quiet in ( (False, ipcount, opts.connections, False), (True, netcount, opts.connections24, False), (True, blocklist, 0, True)): for target, count in counts.items(): if is_net: if count > net_max_count: net_max_count, net_max_ip = count, target elif count > max_count: max_count, max_ip = count, target if target in whitelist or target in all_blocked: continue if count < limit: continue # Only call geoip3 for actual candidates ignore, geo_line = geo.should_ignore(target) if ignore: if opts.verbose: print(f"# not blocking {target} ({count} connections): {geo_line}") continue comment = "" if quiet else f"{count} connections {geo_line}".strip() actions.append((block_cmd(target, is_net), comment, quiet)) if unblock_cmd: unblock_actions.append((unblock_cmd(target), "unblock", quiet)) blocked_now.append(target) all_blocked.add(target) if count and not quiet and frob: frob.send_syslog("ddos", f"blocking {target}, {count} connections {geo_line}") unblock_queue.insert(0, unblock_actions) unblock_queue_ips.insert(0, blocked_now) if len(unblock_queue) > queuelen: actions = unblock_queue.pop() + actions for target in unblock_queue_ips.pop(): all_blocked.discard(target) if actions: print(f"# {time.strftime('%Y-%m-%d %H:%M:%S')}") run_actions(actions, opts.no_act) last_act = time.time() if max_count and opts.verbose: print(f"Max: {max_count} from {max_ip}, and {net_max_count} from {net_max_ip}") sys.stdout.flush() if opts.once: break if idle_seconds and time.time() - last_act > idle_seconds: print(f"# Exiting on idle of {opts.idle} minutes") break time.sleep(opts.sleep) def parse_args(): p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) p.add_argument("whitelist", nargs="*", help="Extra whitelist IPs/subnets, or 2-letter country codes") p.add_argument("-v", "--verbose", action="count", default=0, help="Be verbose") p.add_argument("-n", "--no-act", action="store_true", help="Don't run commands") p.add_argument("-s", "--sleep", type=int, default=30, help="Seconds between checks (30)") p.add_argument("-c", "--connections", type=int, default=50, help="Connections per IP to block at (50)") p.add_argument("-C", "--connections24", type=int, default=200, help="Connections per /24 to block at (200)") p.add_argument("-t", "--time", type=int, default=600, help="Seconds to retain a block (600)") p.add_argument("-l", "--load", type=float, default=6, help="Only block when load is above this (6)") p.add_argument("-p", "--port", dest="only_ports", type=int, action="append", default=[], help="Only consider this local port (repeatable)") p.add_argument("-P", "--exclude-port", dest="exclude_ports", type=int, action="append", default=[], help="Exclude this local port (repeatable; default 3306)") p.add_argument("-i", "--idle", type=int, default=0, help="Exit after this many idle minutes (no blocking)") p.add_argument("-R", "--no-robots-blacklist", action="store_true", help="Don't apply the network blocklist on high load") p.add_argument("--test", dest="testfile", default="", help="Read netstat output from a test file") p.add_argument("-f", "--antibruteforce", type=int, default=4, help="Run anti-brute-force every n minutes") p.add_argument("--ipset", action="store_true", help="Use ipset blocks no matter what") p.add_argument("--geoip-cmd", default=GEOIP_CMD, help=f"GeoIP lookup command ({GEOIP_CMD})") p.add_argument("--ignore-pattern", default=IGNORE_PATTERN, help="Regex on geoip output; matches are never blocked") p.add_argument("--once", action="store_true", help="Run a single check and exit") return p.parse_args() if __name__ == "__main__": options = parse_args() # Exclusive lock on this script so only one copy runs lockme = open(os.path.realpath(__file__), "r") try: fcntl.flock(lockme, fcntl.LOCK_EX | fcntl.LOCK_NB) except OSError: sys.stderr.write(f"{sys.argv[0]} is already running on this machine\n") sys.exit(1) try: main(options) except KeyboardInterrupt: pass finally: lockme.close()