From 01b8ff656eb7c16a032df1bafa142d17b9f6fbb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Gruszczy=C5=84ski?= Date: Thu, 28 Aug 2025 22:54:52 +0200 Subject: [PATCH] refactor --- .env.example | 41 ++++ app.py | 538 +++++++++++++++++++++++++---------------------- config.py | 65 ++++++ listapp.service | 20 +- requirements.txt | 10 + start_dev.sh | 1 + 6 files changed, 419 insertions(+), 256 deletions(-) create mode 100644 .env.example create mode 100644 config.py create mode 100644 requirements.txt create mode 100644 start_dev.sh diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..0593823 --- /dev/null +++ b/.env.example @@ -0,0 +1,41 @@ +# Podstawy +FLASK_DEBUG=false +SECRET_KEY=change-me + +# Limity +MAX_CONTENT_LENGTH=52428800 +RATE_LIMIT_DEFAULT="100 per minute" +RATE_LIMIT_CONVERT="100 per minute" + +# Redis – wybierz REDIS_URL lub HOST/PORT/DB +REDIS_URL=redis://localhost:6379/7 +# REDIS_HOST=localhost +# REDIS_PORT=6379 +# REDIS_DB=7 + +# Basic Auth dla /stats +STATS_BASIC_AUTH_ENABLED=true +STATS_BASIC_AUTH_REALM=Stats +STATS_BASIC_AUTH_USER=admin +STATS_BASIC_AUTH_PASS=strong-password + +# Cache (Varnish-friendly) +CACHE_ENABLED=true +CACHE_S_MAXAGE=43200 +CACHE_MAX_AGE=3600 +USE_REDIS_BODY_CACHE=false + +# Stream / aiohttp +AIOHTTP_TOTAL_TIMEOUT=70 +AIOHTTP_CONNECT_TIMEOUT=10 +AIOHTTP_SOCK_CONNECT_TIMEOUT=10 +AIOHTTP_SOCK_READ_TIMEOUT=60 +READ_CHUNK=65536 +STREAM_LINE_LIMIT=4096 + +# Serwer +BIND_HOST=127.0.0.1 +BIND_PORT=8283 + +# Domyślny URL źródłowy (opcjonalnie) +DEFAULT_SOURCE_URL="https://raw.githubusercontent.com/217heidai/adblockfilters/main/rules/adblockdns.txt" diff --git a/app.py b/app.py index 0da94bf..06975b4 100644 --- a/app.py +++ b/app.py @@ -1,203 +1,224 @@ import re import redis import requests -import aiohttp -import asyncio import socket import time import json +import base64 +import hashlib from datetime import datetime -from flask import Flask, request, render_template, abort, jsonify, g from urllib.parse import urlparse, quote, unquote, urljoin from functools import wraps +from typing import Optional + +from flask import Flask, request, render_template, abort, jsonify, stream_with_context, g, Response from flask_compress import Compress from flask_limiter import Limiter -from flask_limiter.util import get_remote_address + +import config app = Flask(__name__) -app.config['MAX_CONTENT_LENGTH'] = 50 * 1024 * 1024 # limit -redis_client = redis.Redis(host='localhost', port=6379, db=7) +app.config["MAX_CONTENT_LENGTH"] = config.MAX_CONTENT_LENGTH +app.config["SECRET_KEY"] = config.SECRET_KEY +app.debug = config.FLASK_DEBUG + +def build_redis(): + if config.REDIS_URL: + return redis.Redis.from_url(config.REDIS_URL) + return redis.Redis(host=config.REDIS_HOST, port=config.REDIS_PORT, db=config.REDIS_DB) + +redis_client = build_redis() -# Ustawienia do rate limiting – 100 żądań na minutę def get_client_ip(): - """Pobranie prawdziwego adresu IP klienta (uwzględniając proxy)""" - x_forwarded_for = request.headers.get('X-Forwarded-For', '').split(',') - if x_forwarded_for and x_forwarded_for[0].strip(): - return x_forwarded_for[0].strip() + xff = request.headers.get("X-Forwarded-For", "").split(",") + if xff and xff[0].strip(): + return xff[0].strip() return request.remote_addr -limiter = Limiter(key_func=get_client_ip, default_limits=["100 per minute"], app=app) -Compress(app) +limiter = Limiter( + key_func=get_client_ip, + app=app, + default_limits=[config.RATE_LIMIT_DEFAULT], + storage_uri=config.REDIS_URL +) -ALLOWED_IPS = {'127.0.0.1', '109.173.163.139'} -ALLOWED_DOMAIN = '' +Compress(app) @app.before_request def track_request_data(): - """Rejestracja IP klienta, User-Agent, metody HTTP oraz rozpoczęcie pomiaru czasu requestu""" - g.start_time = time.perf_counter() # rozpoczęcie pomiaru czasu - client_ip = get_client_ip() - user_agent = request.headers.get('User-Agent', 'Unknown') - method = request.method - - # Rejestracja User-Agent - redis_client.incr(f'stats:user_agents:{quote(user_agent, safe="")}') - # Rejestracja adresu IP klienta - redis_client.incr(f'stats:client_ips:{client_ip}') - # Rejestracja metody HTTP - redis_client.incr(f'stats:methods:{method}') + g.start_time = time.perf_counter() + redis_client.incr(f"stats:user_agents:{quote(request.headers.get('User-Agent', 'Unknown'), safe='')}") + redis_client.incr(f"stats:client_ips:{get_client_ip()}") + redis_client.incr(f"stats:methods:{request.method}") @app.after_request def after_request(response): - """Pomiar i rejestracja czasu przetwarzania żądania""" elapsed = time.perf_counter() - g.start_time - # Aktualizacja statystyk czasu przetwarzania (w sekundach) - redis_client.incrbyfloat('stats:processing_time_total', elapsed) - redis_client.incr('stats:processing_time_count') - - # Aktualizacja minimalnego czasu przetwarzania + redis_client.incrbyfloat("stats:processing_time_total", elapsed) + redis_client.incr("stats:processing_time_count") try: - current_min = float(redis_client.get('stats:processing_time_min') or elapsed) + current_min = float(redis_client.get("stats:processing_time_min") or elapsed) if elapsed < current_min: - redis_client.set('stats:processing_time_min', elapsed) + redis_client.set("stats:processing_time_min", elapsed) except Exception: - redis_client.set('stats:processing_time_min', elapsed) - - # Aktualizacja maksymalnego czasu przetwarzania + redis_client.set("stats:processing_time_min", elapsed) try: - current_max = float(redis_client.get('stats:processing_time_max') or elapsed) + current_max = float(redis_client.get("stats:processing_time_max") or elapsed) if elapsed > current_max: - redis_client.set('stats:processing_time_max', elapsed) + redis_client.set("stats:processing_time_max", elapsed) except Exception: - redis_client.set('stats:processing_time_max', elapsed) - + redis_client.set("stats:processing_time_max", elapsed) return response -@app.template_filter('datetimeformat') -def datetimeformat_filter(value, format='%Y-%m-%d %H:%M'): +@app.template_filter("datetimeformat") +def datetimeformat_filter(value, format="%Y-%m-%d %H:%M"): try: dt = datetime.fromisoformat(value) return dt.strftime(format) except (ValueError, AttributeError): return value -def ip_restriction(f): - @wraps(f) - def decorated(*args, **kwargs): - client_ip = get_client_ip() - host = request.host.split(':')[0] - - allowed_conditions = [ - client_ip in ALLOWED_IPS, - host == ALLOWED_DOMAIN, - request.headers.get('X-Forwarded-For', '').split(',')[0].strip() in ALLOWED_IPS - ] - - if any(allowed_conditions): - return f(*args, **kwargs) - redis_client.incr('stats:errors_403') - abort(403) - return decorated +def basic_auth_required(realm: str, user: str, password: str): + def decorator(f): + @wraps(f) + def wrapper(*args, **kwargs): + if not config.STATS_BASIC_AUTH_ENABLED: + return f(*args, **kwargs) + auth = request.headers.get("Authorization", "") + if auth.startswith("Basic "): + try: + decoded = base64.b64decode(auth[6:]).decode("utf-8", errors="ignore") + u, p = decoded.split(":", 1) + if u == user and p == password: + return f(*args, **kwargs) + except Exception: + pass + resp = Response(status=401) + resp.headers["WWW-Authenticate"] = f'Basic realm="{realm}"' + return resp + return wrapper + return decorator def cache_key(source_url, ip): return f"cache:{source_url}:{ip}" def should_ignore_domain(domain): - """Sprawdza, czy domena zaczyna się od kropki i powinna być ignorowana.""" - return domain.startswith('.') or any(char in domain for char in ['~', '=', '$', "'", "^", "_", ">", "<", ":"]) + return domain.startswith(".") or any(ch in domain for ch in ["~", "=", "$", "'", "^", "_", ">", "<", ":"]) def should_ignore_line(line): - """Sprawdza, czy linia zawiera określone znaki i powinna być ignorowana.""" - return any(symbol in line for symbol in ['<', '>', '##', '###', "div", "span"]) + return any(sym in line for sym in ["<", ">", "##", "###", "div", "span"]) def is_valid_domain(domain): - """Sprawdza, czy domena ma poprawną składnię.""" - domain_regex = re.compile(r'^(?:[a-zA-Z0-9-]+\.)+[a-zA-Z]{2,}$') - return bool(domain_regex.match(domain)) + return bool(re.compile(r"^(?:[a-zA-Z0-9-]+\.)+[a-zA-Z]{2,}$").match(domain)) -def convert_hosts(content, target_ip): - """Konwersja treści pliku hosts oraz reguł AdGuard DNS.""" - converted = [] - invalid_lines = [] +def convert_host_line(line: str, target_ip: str): + # szybkie odrzucenia + if not line: + return None + line = line.strip() - for line in content.splitlines(): - line = line.strip() + # komentarze/puste + if not line or line.startswith(("!", "#", "/", ";")): + return None - # Pomijanie pustych linii, komentarzy i linii do ignorowania - if not line or line[0] in ('!', '#', '/') or should_ignore_line(line): - continue + # wytnij komentarz końcowy (# lub ;) – ostrożnie ze 'http://' + # usuwamy wszystko od ' #' lub ' ;' (spacja przed znacznikiem komentarza) + for sep in (" #", " ;"): + idx = line.find(sep) + if idx != -1: + line = line[:idx].rstrip() - # Obsługa reguł AdGuard DNS - match = re.match(r'^\|\|([^\^]+)\^.*', line) - if match: - domain = match.group(1) - if should_ignore_domain(domain): - continue - if not is_valid_domain(domain): - invalid_lines.append(line) - continue - converted.append(f"{target_ip} {domain}") - continue + if not line: + return None - # Obsługa klasycznego formatu hosts - parts = line.split() - if len(parts) > 1: - domain_part = parts[1] - if should_ignore_domain(domain_part): - continue - if not is_valid_domain(domain_part): - invalid_lines.append(line) - continue - converted.append(re.sub(r'^\S+', target_ip, line, count=1)) - - if invalid_lines: - print("Niepoprawne linie:") - for invalid in invalid_lines: - print(invalid) - - return '\n'.join(converted) + # 1) AdGuard / uBlock DNS: ||domain^ (opcjonalnie z dodatkami po '^') + m = re.match(r"^\|\|([a-z0-9.-]+)\^", line, re.IGNORECASE) + if m: + domain = m.group(1).strip(".") + if not should_ignore_domain(domain) and is_valid_domain(domain): + return f"{target_ip} {domain}" + return None + parts = line.split() + + # 2) Klasyczny hosts: "IP domena [...]" (IPv4 lub IPv6) + if len(parts) >= 2 and ( + re.match(r"^\d{1,3}(?:\.\d{1,3}){3}$", parts[0]) or ":" in parts[0] + ): + domain = parts[1].strip().split("#", 1)[0].strip().strip(".") + if not should_ignore_domain(domain) and is_valid_domain(domain): + return f"{target_ip} {domain}" + return None + + # 3) dnsmasq: address=/domain/0.0.0.0 czy server=/domain/... + m = re.match(r"^(?:address|server)=/([a-z0-9.-]+)/", line, re.IGNORECASE) + if m: + domain = m.group(1).strip(".") + if not should_ignore_domain(domain) and is_valid_domain(domain): + return f"{target_ip} {domain}" + return None + + # 4) Domain-only: "example.com" lub "example.com # komentarz" + token = parts[0].split("#", 1)[0].strip().strip(".") + if token and not should_ignore_domain(token) and is_valid_domain(token): + return f"{target_ip} {token}" + + return None + + +def build_etag(up_etag: Optional[str], up_lastmod: Optional[str], target_ip: str) -> str: + base = (up_etag or up_lastmod or "no-upstream") + f"::{target_ip}::v1" + return 'W/"' + hashlib.sha1(base.encode("utf-8")).hexdigest() + '"' + +def cache_headers(etag: str, up_lm: Optional[str]): + headers = { + "ETag": etag, + "Vary": "Accept-Encoding", + "Content-Type": "text/plain; charset=utf-8", + "X-Content-Type-Options": "nosniff", + "Content-Disposition": "inline; filename=converted_hosts.txt", + } + if config.CACHE_ENABLED: + headers["Cache-Control"] = f"public, s-maxage={config.CACHE_S_MAXAGE}, max-age={config.CACHE_MAX_AGE}" + else: + headers["Cache-Control"] = "no-store" + if up_lm: + headers["Last-Modified"] = up_lm + return headers def validate_and_normalize_url(url): - """Walidacja i normalizacja adresu URL""" parsed = urlparse(url) if not parsed.scheme: - url = f'https://{url}' + url = f"https://{url}" parsed = urlparse(url) if not parsed.netloc: raise ValueError("Missing host in URL") return parsed.geturl() def track_url_request(url): - """Rejestracja żądania dla określonego URL""" - redis_key = f"stats:url_requests:{quote(url, safe='')}" - redis_client.incr(redis_key) + redis_client.incr(f"stats:url_requests:{quote(url, safe='')}") def add_recent_link(url, target_ip): - """Dodanie ostatniego linku do historii (ostatnie 10)""" - timestamp = datetime.now().isoformat() - link_data = f"{timestamp}|{url}|{target_ip}" - + ts = datetime.now().isoformat() + link_data = f"{ts}|{url}|{target_ip}" with redis_client.pipeline() as pipe: pipe.lpush("recent_links", link_data) pipe.ltrim("recent_links", 0, 9) pipe.execute() - redis_client.incr('stats:recent_links_added') + redis_client.incr("stats:recent_links_added") def get_recent_links(): - """Pobranie ostatnich 10 linków""" links = redis_client.lrange("recent_links", 0, 9) - parsed_links = [] + out = [] for link in links: parts = link.decode().split("|") if len(parts) >= 3: - parsed_links.append((parts[0], parts[1], parts[2])) + out.append((parts[0], parts[1], parts[2])) elif len(parts) == 2: - parsed_links.append((parts[0], parts[1], "127.0.0.1")) - return parsed_links + out.append((parts[0], parts[1], "127.0.0.1")) + return out def get_hostname(ip): - """Cache’owanie wyników reverse DNS dla danego IP""" key = f"reverse_dns:{ip}" cached = redis_client.get(key) if cached: @@ -206,202 +227,215 @@ def get_hostname(ip): hostname = socket.gethostbyaddr(ip)[0] except Exception: hostname = ip - # Cache na 1 godzinę redis_client.setex(key, 3600, hostname) return hostname -# Nowa funkcja do logowania requestów dla endpointu /convert def add_recent_convert(): - """Dodaje dane żądania do listy ostatnich konwersji (/convert)""" ip = get_client_ip() hostname = get_hostname(ip) - user_agent = request.headers.get('User-Agent', 'Unknown') + ua = request.headers.get("User-Agent", "Unknown") time_str = datetime.now().astimezone().isoformat() - url = request.full_path # pełna ścieżka wraz z query string - data = { - "url": url, - "ip": ip, - "hostname": hostname, - "time": time_str, - "user_agent": user_agent - } - json_data = json.dumps(data) - redis_client.lpush("recent_converts", json_data) + url = request.full_path + data = {"url": url, "ip": ip, "hostname": hostname, "time": time_str, "user_agent": ua} + redis_client.lpush("recent_converts", json.dumps(data)) redis_client.ltrim("recent_converts", 0, 49) -@app.route('/', methods=['GET']) +@app.route("/favicon.ico", methods=["GET"]) +def favicon(): + return Response(status=204) + +@app.route("/", methods=["GET"]) def index(): - """Strona główna z formularzem""" generated_link = None recent_links = get_recent_links() - url_param = request.args.get('url') - target_ip = request.args.get('ip', '127.0.0.1') - - client_ip = get_client_ip() - user_agent = request.headers.get('User-Agent', 'Unknown') + url_param = request.args.get("url", config.DEFAULT_SOURCE_URL) + target_ip = request.args.get("ip", "127.0.0.1") if url_param: try: - normalized_url = validate_and_normalize_url(unquote(url_param)) - encoded_url = quote(normalized_url, safe='') - generated_link = urljoin( - request.host_url, - f"convert?url={encoded_url}&ip={target_ip}" - ) - add_recent_link(normalized_url, target_ip) + normalized = validate_and_normalize_url(unquote(url_param)) + encoded = quote(normalized, safe="") + generated_link = urljoin(request.host_url, f"convert?url={encoded}&ip={target_ip}") + add_recent_link(normalized, target_ip) recent_links = get_recent_links() except Exception as e: app.logger.error(f"Error processing URL: {str(e)}") - return render_template('form.html', - generated_link=generated_link, - recent_links=recent_links, - client_ip=client_ip, - user_agent=user_agent) -@app.route('/convert') -@limiter.limit("100 per minute") -async def convert(): - """Asynchroniczny endpoint do konwersji z weryfikacją typu zawartości""" try: - redis_client.incr('stats:convert_requests') - # Logowanie danych dla requestu do /convert + return render_template( + "form.html", + generated_link=generated_link, + recent_links=recent_links, + client_ip=get_client_ip(), + user_agent=request.headers.get("User-Agent", "Unknown"), + ) + except Exception: + return jsonify( + { + "generated_link": generated_link, + "recent_links": recent_links, + "client_ip": get_client_ip(), + "user_agent": request.headers.get("User-Agent", "Unknown"), + } + ) + +@app.route("/convert") +@limiter.limit(config.RATE_LIMIT_CONVERT) +def convert(): + try: + redis_client.incr("stats:convert_requests") add_recent_convert() - encoded_url = request.args.get('url') + encoded_url = request.args.get("url") if not encoded_url: - redis_client.incr('stats:errors_400') + redis_client.incr("stats:errors_400") abort(400, description="Missing URL parameter") decoded_url = unquote(encoded_url) normalized_url = validate_and_normalize_url(decoded_url) - target_ip = request.args.get('ip', '127.0.0.1') + target_ip = request.args.get("ip", "127.0.0.1") - # Rejestracja statystyk dotyczących URL track_url_request(normalized_url) - redis_client.incr(f'stats:target_ips:{target_ip}') + redis_client.incr(f"stats:target_ips:{target_ip}") - # Sprawdzenie pamięci podręcznej - cached = redis_client.get(cache_key(normalized_url, target_ip)) - if cached: - redis_client.incr('stats:cache_hits') - return cached.decode('utf-8'), 200, {'Content-Type': 'text/plain'} + req_headers = {} + inm = request.headers.get("If-None-Match") + ims = request.headers.get("If-Modified-Since") + if inm: + req_headers["If-None-Match"] = inm + if ims: + req_headers["If-Modified-Since"] = ims - redis_client.incr('stats:cache_misses') + with requests.get(normalized_url, headers=req_headers, stream=True, timeout=(10, 60)) as r: + ct = r.headers.get("Content-Type", "") + # pozwól na text/* oraz octet-stream (często używane przez listy) + if "text" not in ct and "octet-stream" not in ct and ct != "": + abort(415, description="Unsupported Media Type") - # Asynchroniczne pobranie zasobu za pomocą aiohttp - async with aiohttp.ClientSession() as session: - async with session.get(normalized_url, timeout=60) as response: - # Sprawdzanie typu zawartości – musi zawierać "text" - content_type = response.headers.get("Content-Type", "") - if "text" not in content_type: - abort(415, description="Unsupported Media Type") - content = b"" - while True: - try: - chunk = await response.content.read(4096) - except asyncio.TimeoutError: - abort(504, description="Timeout reading remote data") - if not chunk: - break - content += chunk - if len(content) > app.config['MAX_CONTENT_LENGTH']: - redis_client.incr('stats:errors_413') - abort(413) + if r.status_code == 304: + etag = build_etag(r.headers.get("ETag"), r.headers.get("Last-Modified"), target_ip) + resp = Response(status=304) + resp.headers.update(cache_headers(etag, r.headers.get("Last-Modified"))) + resp.direct_passthrough = True + return resp - # Rejestracja rozmiaru pobranej treści - content_size = len(content) - redis_client.incrby('stats:content_size_total', content_size) - redis_client.incr('stats:content_size_count') + up_etag = r.headers.get("ETag") + up_lm = r.headers.get("Last-Modified") + etag = build_etag(up_etag, up_lm, target_ip) - converted = convert_hosts(content.decode('utf-8'), target_ip) - redis_client.setex(cache_key(normalized_url, target_ip), 43200, converted) # 12h cache - redis_client.incr('stats:conversions_success') - return converted, 200, {'Content-Type': 'text/plain'} + @stream_with_context + def body_gen(): + total = 0 + # iter_lines pewnie tnie po \n/\r\n i dekoduje do str + for line in r.iter_lines(decode_unicode=True, chunk_size=config.READ_CHUNK): + if line is None: + continue + # zabezpieczenie przed megadługimi wierszami + if len(line) > config.STREAM_LINE_LIMIT: + continue + out = convert_host_line(line, target_ip) + if out: + s = out + "\n" + total += len(s) + yield s + # statystyki po zakończeniu streamu + redis_client.incrby("stats:content_size_total", total) + redis_client.incr("stats:content_size_count") - except aiohttp.ClientError as e: + resp = Response(body_gen(), mimetype="text/plain; charset=utf-8") + resp.headers.update(cache_headers(etag, up_lm)) + # wyłącz kompresję/buforowanie dla strumienia + resp.direct_passthrough = True + redis_client.incr("stats:conversions_success") + return resp + + except requests.exceptions.RequestException as e: app.logger.error(f"Request error: {str(e)}") - redis_client.incr('stats:errors_500') + redis_client.incr("stats:errors_500") abort(500) except ValueError as e: app.logger.error(f"URL validation error: {str(e)}") - redis_client.incr('stats:errors_400') + redis_client.incr("stats:errors_400") abort(400) +@app.route("/convert", methods=["HEAD"]) +def convert_head(): + encoded_url = request.args.get("url", config.DEFAULT_SOURCE_URL) + if not encoded_url: + abort(400) + decoded_url = unquote(encoded_url) + validate_and_normalize_url(decoded_url) + target_ip = request.args.get("ip", "127.0.0.1") + etag = build_etag(None, None, target_ip) + resp = Response(status=200) + resp.headers.update(cache_headers(etag, None)) + resp.direct_passthrough = True + return resp -@app.route('/stats') -@ip_restriction +@app.route("/stats") +@basic_auth_required( + realm=config.STATS_BASIC_AUTH_REALM, + user=config.STATS_BASIC_AUTH_USER, + password=config.STATS_BASIC_AUTH_PASS, +) def stats(): - """Endpoint statystyk""" - stats_data = {} - target_ips = {} - url_requests = {} - user_agents = {} - client_ips = {} + stats_data, target_ips, url_requests, user_agents, client_ips = {}, {}, {}, {}, {} - # Agregacja statystyk z Redisa for key in redis_client.scan_iter("stats:*"): key_str = key.decode() value = redis_client.get(key).decode() - - if key_str.startswith('stats:target_ips:'): - ip = key_str.split(':', 2)[2] + if key_str.startswith("stats:target_ips:"): + ip = key_str.split(":", 2)[2] target_ips[ip] = value - elif key_str.startswith('stats:url_requests:'): - url = unquote(key_str.split(':', 2)[2]) + elif key_str.startswith("stats:url_requests:"): + url = unquote(key_str.split(":", 2)[2]) url_requests[url] = value - elif key_str.startswith('stats:user_agents:'): - ua = unquote(key_str.split(':', 2)[2]) + elif key_str.startswith("stats:user_agents:"): + ua = unquote(key_str.split(":", 2)[2]) user_agents[ua] = value - elif key_str.startswith('stats:client_ips:'): - ip = key_str.split(':', 2)[2] + elif key_str.startswith("stats:client_ips:"): + ip = key_str.split(":", 2)[2] client_ips[ip] = value else: stats_data[key_str] = value - # Pobranie ostatnich 50 requestów dla endpointu /convert recent_converts = [] - convert_entries = redis_client.lrange("recent_converts", 0, 49) - for entry in convert_entries: + for entry in redis_client.lrange("recent_converts", 0, 49): try: - data = json.loads(entry.decode()) - recent_converts.append(data) + recent_converts.append(json.loads(entry.decode())) except Exception: pass - # Obliczenie średniego czasu przetwarzania żądań - processing_time_total = float(redis_client.get('stats:processing_time_total') or 0) - processing_time_count = int(redis_client.get('stats:processing_time_count') or 0) + processing_time_total = float(redis_client.get("stats:processing_time_total") or 0) + processing_time_count = int(redis_client.get("stats:processing_time_count") or 0) avg_processing_time = processing_time_total / processing_time_count if processing_time_count > 0 else 0 - # Obliczenie średniego rozmiaru pobranej treści dla /convert - content_size_total = int(redis_client.get('stats:content_size_total') or 0) - content_size_count = int(redis_client.get('stats:content_size_count') or 0) + content_size_total = int(redis_client.get("stats:content_size_total") or 0) + content_size_count = int(redis_client.get("stats:content_size_count") or 0) avg_content_size = content_size_total / content_size_count if content_size_count > 0 else 0 - # Rozszerzone statystyki dotyczące wydajności i rozmiarów danych detailed_stats = { "processing_time_total_sec": processing_time_total, "processing_time_count": processing_time_count, "processing_time_avg_sec": avg_processing_time, - "processing_time_min_sec": float(redis_client.get('stats:processing_time_min') or 0), - "processing_time_max_sec": float(redis_client.get('stats:processing_time_max') or 0), + "processing_time_min_sec": float(redis_client.get("stats:processing_time_min") or 0), + "processing_time_max_sec": float(redis_client.get("stats:processing_time_max") or 0), "content_size_total_bytes": content_size_total, "content_size_count": content_size_count, - "content_size_avg_bytes": avg_content_size + "content_size_avg_bytes": avg_content_size, } - # Struktura odpowiedzi - response_data = { - **stats_data, - 'target_ips': target_ips, - 'url_requests': url_requests, - 'user_agents': user_agents, - 'client_ips': client_ips, - 'recent_converts': recent_converts, - 'detailed_stats': detailed_stats - } - - return jsonify(response_data) + return jsonify( + { + **stats_data, + "target_ips": target_ips, + "url_requests": url_requests, + "user_agents": user_agents, + "client_ips": client_ips, + "recent_converts": recent_converts, + "detailed_stats": detailed_stats, + } + ) @app.errorhandler(400) @app.errorhandler(403) @@ -410,13 +444,13 @@ def stats(): @app.errorhandler(415) @app.errorhandler(500) def handle_errors(e): - """Obsługa błędów""" - return render_template('error.html', error=e), e.code + try: + return render_template("error.html", error=e), e.code + except Exception: + return jsonify({"error": getattr(e, "description", str(e)), "code": e.code}), e.code -# Jeśli aplikacja jest uruchamiana bezpośrednio, korzystamy z Flask's run -if __name__ == '__main__': - app.run(host='0.0.0.0', port=8283) -# W przeciwnym razie (np. przy uruchamianiu przez Gunicorn) opakowujemy aplikację w adapter ASGI +if __name__ == "__main__": + app.run(host=config.BIND_HOST, port=config.BIND_PORT) else: from asgiref.wsgi import WsgiToAsgi asgi_app = WsgiToAsgi(app) diff --git a/config.py b/config.py new file mode 100644 index 0000000..e704d23 --- /dev/null +++ b/config.py @@ -0,0 +1,65 @@ +import os + +def getenv_bool(key: str, default: bool = False) -> bool: + v = os.getenv(key) + if v is None: + return default + return v.lower() in ("1", "true", "t", "yes", "y", "on") + +def getenv_int(key: str, default: int) -> int: + try: + return int(os.getenv(key, str(default))) + except ValueError: + return default + +def getenv_float(key: str, default: float) -> float: + try: + return float(os.getenv(key, str(default))) + except ValueError: + return default + +# Podstawowe +FLASK_DEBUG = getenv_bool("FLASK_DEBUG", True) +SECRET_KEY = os.getenv("SECRET_KEY", "change-me") + +# Rozmiary/limity +MAX_CONTENT_LENGTH = getenv_int("MAX_CONTENT_LENGTH", 50 * 1024 * 1024) # 50MB +RATE_LIMIT_DEFAULT = os.getenv("RATE_LIMIT_DEFAULT", "100 per minute") +RATE_LIMIT_CONVERT = os.getenv("RATE_LIMIT_CONVERT", "100 per minute") + +# Redis +REDIS_URL = os.getenv("REDIS_URL", "redis://localhost:6379/7") +REDIS_HOST = os.getenv("REDIS_HOST", "localhost") +REDIS_PORT = getenv_int("REDIS_PORT", 6379) +REDIS_DB = getenv_int("REDIS_DB", 7) + +# Basic Auth dla /stats +STATS_BASIC_AUTH_ENABLED = getenv_bool("STATS_BASIC_AUTH_ENABLED", True) +STATS_BASIC_AUTH_REALM = os.getenv("STATS_BASIC_AUTH_REALM", "Stats") +STATS_BASIC_AUTH_USER = os.getenv("STATS_BASIC_AUTH_USER", "admin") +STATS_BASIC_AUTH_PASS = os.getenv("STATS_BASIC_AUTH_PASS", "change-me") + +# Cache/ETag dla Varnisha +CACHE_ENABLED = getenv_bool("CACHE_ENABLED", True) +CACHE_S_MAXAGE = getenv_int("CACHE_S_MAXAGE", 43200) # 12h +CACHE_MAX_AGE = getenv_int("CACHE_MAX_AGE", 3600) # 1h +USE_REDIS_BODY_CACHE = getenv_bool("USE_REDIS_BODY_CACHE", False) + +# AIOHTTP/stream +AIOHTTP_TOTAL_TIMEOUT = getenv_float("AIOHTTP_TOTAL_TIMEOUT", 70.0) +AIOHTTP_CONNECT_TIMEOUT = getenv_float("AIOHTTP_CONNECT_TIMEOUT", 10.0) +AIOHTTP_SOCK_CONNECT_TIMEOUT= getenv_float("AIOHTTP_SOCK_CONNECT_TIMEOUT", 10.0) +AIOHTTP_SOCK_READ_TIMEOUT = getenv_float("AIOHTTP_SOCK_READ_TIMEOUT", 60.0) + +READ_CHUNK = getenv_int("READ_CHUNK", 64 * 1024) # 64 KiB +STREAM_LINE_LIMIT= getenv_int("STREAM_LINE_LIMIT", 4096) + +# Serwer +BIND_HOST = os.getenv("BIND_HOST", "127.0.0.1") +BIND_PORT = getenv_int("BIND_PORT", 8283) + +# Domyślny URL źródłowy (opcjonalny) +DEFAULT_SOURCE_URL = os.getenv( + "DEFAULT_SOURCE_URL", + "" +) diff --git a/listapp.service b/listapp.service index 60d7a4d..29c9b75 100644 --- a/listapp.service +++ b/listapp.service @@ -1,15 +1,27 @@ -# /etc/systemd/system/listapp.service [Unit] Description=ListApp - Flask application for hosts file conversion -After=network.target redis.service +After=network-online.target redis.service +Wants=network-online.target [Service] User=www-data Group=www-data WorkingDirectory=/var/www/listapp + +# Globalne env + nadpisania (opcjonalne; minus oznacza „jeśli istnieje”) +EnvironmentFile=-/var/www/listapp/.env + +# Ścieżka do virtualenv Environment="PATH=/var/www/listapp/venv/bin" -#ExecStart=/var/www/listapp/bin/gunicorn -w 2 --bind 127.0.0.1:8283 app:app -ExecStart=/var/www/listapp/bin/gunicorn -k uvicorn.workers.UvicornWorker -w 4 --bind 127.0.0.1:8283 app:asgi_app + +# Gunicorn + UvicornWorker (ASGI) +ExecStart=/var/www/listapp/venv/bin/gunicorn \ + -k uvicorn.workers.UvicornWorker \ + --workers 4 \ + --bind 127.0.0.1:8283 \ + --keep-alive 30 \ + --timeout 90 \ + app:asgi_app Restart=always RestartSec=5 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..e22f2c6 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +Flask +Flask-Compress +Flask-Limiter +redis +requests +aiohttp +asgiref +unicorn +gunicorn +uvicorn \ No newline at end of file diff --git a/start_dev.sh b/start_dev.sh new file mode 100644 index 0000000..9e3e4e1 --- /dev/null +++ b/start_dev.sh @@ -0,0 +1 @@ +venv/bin/gunicorn -k uvicorn.workers.UvicornWorker --workers 4 --bind 127.0.0.1:8283 --keep-alive 30 --timeout 90 app:asgi_app \ No newline at end of file