---

fault_detector.py

# -*- coding: utf-8 -*- # Copyright Cloud Linux GmbH & Cloud Linux Software, Inc 2010-2021 All Rights Reserved # # Licensed under CLOUD LINUX LICENSE AGREEMENT # http://cloudlinux.com/docs/LICENSE.TXT """ This module contains FaultDetector class, aimed to track throttling for incoming requests """ import collections import logging import time from datetime import datetime, timedelta from threading import RLock from typing import Optional, Tuple, Any from xray.internal.constants import drop_after, check_period, throttling_threshold class FaultDetector: """ Fault Detector class """ _MAX_ENTRIES = 10000 # Per-UID sub-limit on how many entries a single SO_PEERCRED uid may occupy # in the shared throttling_stat dict. The request-init path is reachable by # any local uid (the agent socket is world-rw by design), so a per-UID quota # prevents one uid from flooding distinct PIDs and starving other tenants' # entries within the global _MAX_ENTRIES cap. 1000 is ~10x above realistic # concurrent PHP worker counts per user (PHP-FPM pm.max_children) yet only # 1/10 of the global cap, so a single uid's flood can never block another. _MAX_ENTRIES_PER_UID = 1000 def __init__(self): self.logger = logging.getLogger('fault_detector') self.throttling_stat = dict() # per-UID bookkeeping: number of live entries owned by each uid, and the # owning uid for each key (kept in lockstep with throttling_stat so that # expiry/consumption can decrement the right uid's count). self._uid_entry_count = collections.defaultdict(int) self._entry_uid = dict() self.lock = RLock() self.time_marker_file = '/usr/share/alt-php-xray/flush.latest' def __call__(self, t_key: Any, cpu_value: int) -> Tuple[bool, int]: """ Perform fault detection """ with self.lock: try: saved_cpu, _ = self.throttling_stat[t_key] except KeyError: hitting_limits, throttled_time = False, 0 else: hitting_limits, throttled_time = self._was_throttled_since( saved_cpu, cpu_value) self._flush_entry(t_key) return hitting_limits, throttled_time @property def timestamp(self) -> int: """ Current Unix timestamp """ return int(time.time()) @property def _drop_after(self) -> timedelta: """ Drop after value as datetime.timedelta """ return self.timedelta_in_minutes(drop_after) @property def _check_period(self) -> timedelta: """ Check period value as datetime.timedelta """ return self.timedelta_in_minutes(check_period) @property def should_flush(self) -> bool: """ If throttling_stat should be flushed right now regarding check period value """ try: with open(self.time_marker_file) as latest_info: latest_flush_time = int(latest_info.read().strip()) return self.current_delta(latest_flush_time) > self._check_period except (OSError, ValueError): with self.lock: self._update_latest_flush() return False @staticmethod def timedelta_in_minutes(mins: int) -> timedelta: """ Get timedelta object for given number of minutes """ return timedelta(minutes=mins) def current_delta(self, time_stamp: int) -> timedelta: """ Calculate timedelta for given time_stamp from current timestamp """ def _cast(ts: int) -> datetime: """ Make a datetime object from ts (int) """ return datetime.fromtimestamp(ts) return _cast(self.timestamp) - _cast(time_stamp) def save(self, key: Any, cpu: int, uid: Optional[int] = None) -> None: """ Save throttling entry for given key. :param uid: SO_PEERCRED uid of the connecting peer (when known). A per-UID quota is enforced so one uid's request-init flood cannot saturate the shared cap and starve other tenants. """ stamp = self.timestamp with self.lock: is_new = key not in self.throttling_stat # Per-UID quota: only new keys count against a uid's quota; updating # an existing entry never costs a new slot. Dropping a NEW entry for # an over-quota uid leaves room in the global cap for other uids. if is_new and uid is not None \ and self._uid_entry_count.get(uid, 0) >= self._MAX_ENTRIES_PER_UID: self.logger.warning('throttling_stat per-uid quota (%d) reached for ' 'uid=%s, dropping entry', self._MAX_ENTRIES_PER_UID, uid) return if is_new and len(self.throttling_stat) >= self._MAX_ENTRIES: self.logger.warning('throttling_stat at capacity (%d), dropping entry', len(self.throttling_stat)) return self.logger.debug('Adding new throttling entry %s: (%i, %i)', key, cpu, stamp) self.throttling_stat[key] = (cpu, stamp) if is_new: # Track ownership so the uid's count can be decremented when the # entry is consumed (request-end) or expired (flush). self._entry_uid[key] = uid if uid is not None: self._uid_entry_count[uid] += 1 self.logger.debug('Throttling entries total %i', len(self.throttling_stat)) def _flush_entry(self, key: Any) -> None: """ Remove throttling entry by given key """ with self.lock: self.logger.debug('Abandon throttling entry %s', key) existed = key in self.throttling_stat self.throttling_stat.pop(key, 0) if existed: # release the per-UID slot held by this entry uid = self._entry_uid.pop(key, None) if uid is not None and self._uid_entry_count.get(uid, 0) > 0: self._uid_entry_count[uid] -= 1 if self._uid_entry_count[uid] <= 0: del self._uid_entry_count[uid] self.logger.debug('Throttling entries total %i', len(self.throttling_stat)) def _was_throttled_since(self, initial_value: int, current_value: int) -> Tuple[bool, int]: """ Check current throttling time of given lve_id vs given initial_value Return: tuple(fact of throttling, throttling time in ms) """ # initial values of throttling time are in nanoseconds! spent_time = (current_value - initial_value) // 1000000 # cast to ms throttling_fact = spent_time > throttling_threshold self.logger.debug('Throttling checked with %i: %s, %i', throttling_threshold, throttling_fact, spent_time) return throttling_fact, spent_time if spent_time > 0 else 0 def flush(self) -> None: """ Clear throttling stat dict """ if self.should_flush: with self.lock: self.logger.debug('Flush started, total entries %i', len(self.throttling_stat)) for key in self._expired_entries(): self._flush_entry(key) self._update_latest_flush() self.logger.debug('Flush finished, total entries %i', len(self.throttling_stat)) def _update_latest_flush(self) -> None: """ Write current timestamp into time marker file """ try: with open(self.time_marker_file, 'w') as latest_info: latest_info.write(str(self.timestamp)) except OSError: self.logger.error('Failed to save timestamp of latest flush') def _expired_entries(self) -> list: """ Collect expired throttling stat entries keys regarding the drop after value """ expired = list() for key, value in self.throttling_stat.items(): _, timestamp = value _delta = self.current_delta(timestamp) if _delta > self._drop_after: self.logger.debug('Expired entry detected %i: %s elapsed', key, _delta) expired.append(key) return expired

---

docroot: /home/induvedam/public_html

---

curdir: /opt/cloudlinux/venv/lib/python3.11/site-packages/xray/agent

	.		105
	..		4096
X	__init__.py		231
	__pycache__		138
X	daemon.py		48000
X	executor.py		2195
X	fault_detector.py		8807