魔のkyo的BLOG

import threading
import traceback
import time
import sys
import os
from functools import wraps

class WatchDog(threading.Thread):

    def __init__(self, timeout=10, echo=False):
        super(WatchDog, self).__init__()
        self.timeout = timeout
        self.echo = echo
        self.last_kicked_ts = time.time()
        self.lock = threading.Lock()
        self.thread_id = threading.currentThread().ident
        self.terminated = False
        self.setDaemon(True)
        self.start()

    def terminate(self):
        self.terminated = True
        self.join(self.timeout)

    def kick(self):
        self.lock.acquire()
        self.last_kicked_ts = time.time()
        self.lock.release()

    def bark(self):
        formated_frame_stack = self._get_formated_frame_stack()
        if self.echo:
            print("!!!!! WATCH DOG FAILURE TRIGGERED !!!!!\n" + formated_frame_stack, flush=True)
        pid = os.getpid()
        os.kill(pid, 2) # 通知进程退出
        time.sleep(5)   # 等待5秒
        os.kill(pid, 9) # 发送强制退出

    def run(self):
        while not self.terminated:
            ts = time.time()
            self.lock.acquire()
            is_timeout = ts - self.last_kicked_ts > self.timeout
            self.lock.release()
            if is_timeout:
                self.bark()
            n = int(max(self.timeout / 3, 1))
            for i in range(n*10):
                time.sleep(0.1)
                if self.terminated:
                    break

    @staticmethod
    def _get_thread(tid):
        for t in threading.enumerate():
            if t.ident == tid:
                return t
        return None

    @staticmethod
    def _get_frame_stack(tid):
        for thread_id, stack in sys._current_frames().items():
            if thread_id == tid:
                return stack
        return None

    def _get_formated_frame_stack(self):
        info = []
        th = self._get_thread(self.thread_id)
        stack = self._get_frame_stack(self.thread_id)
        info.append('%s thead_id=%d' % (th.name, self.thread_id))
        for filename, lineno, _, line in traceback.extract_stack(stack):
            info.append('    at %s(%s:%d)' % (line, filename[filename.rfind(os.path.sep) + 1:], lineno))
        return '\n'.join(info)


def watch_dog(timeout=10, echo=False):
    def inner(func):
        def wrapper(*args, **kw):
            dog = WatchDog(timeout=timeout, echo=echo)
            ret = func(*args, **kw)
            dog.terminate()
            return ret
        return wrapper
    return inner

@watch_dog(timeout=3, echo=True)
def func():
    # do something
    time.sleep(5)

def main():
    func()

if __name__ == '__main__':
    main()

def main():
    watch_dog = WatchDog(timeout=3, echo=True)

    # main loop
    # while True:
    for i in range(0, 10):
        time.sleep(i)
        print(f"i={i}")
        watch_dog.kick()
    watch_dog.terminated()


if __name__ == '__main__':
    main()

import multiprocessing

def main():
    ...

if __name__ == '__main__':
    RETRY_TIMES = 5
    for i in range(RETRY_TIMES): # 重试次数
        child_process = multiprocessing.Process(target=main)
        child_process.start()
        child_process.join()

        if child_process.exitcode == 0:
            print("子进程正常结束")
            exit(0)
        elif i < RETRY_TIMES - 1:
            print("子进程异常结束，即将重试")
        else:
            print("子进程异常结束，重试超次")
            exit(child_process.exitcode)

# -*- coding: utf-8 -*-
import threading
import time
import queue
from dataclasses import dataclass, field
from typing import Any
import signal
import traceback
import os


@dataclass(order=True)
class ScheduledItem:
    time_sec: float
    cb: Any = field(compare=False)


class Driver:
    def __init__(self):
        self._scheduled = queue.PriorityQueue()
        self._scheduled_every_lock = threading.Lock()
        self._scheduled_every = []
        self._callbacks_lock = threading.Lock()
        self._callbacks = {}
        self._async_queue = queue.Queue()
        self._epoch_sec = time.time()
        self._debug_epoch = None
        self._last_epoch = None
        self._sigterm_received = False

    # 得到driver内的当前时间
    def get_epoch(self):
        return self._epoch_sec

    # 设置调试时间，设置了调试时间driver内将不更新当前时间而采用调试时间
    def set_debug_epoch(self, epoch):
        self._debug_epoch = epoch

    # 主循环
    # wait_sync_interval 每趟循环的时间间隔，如果为0相当于占用一个core
    # intercept_signals  拦截并处理信号，默认处理CTRL+C和kill，收到时退出主循环
    def run(self, wait_sync_interval=0, intercept_signals=(signal.SIGINT, signal.SIGTERM)):
        if intercept_signals:
            self.intercept_sigterm(intercept_signals)
        while not self.sigterm_received():
            self.run_step(wait_sync_interval)

    # 执行一趟主逻辑，一般放在主循环中执行
    def run_step(self, wait_sync_interval=0):
        if self._debug_epoch:
            self._epoch_sec = self._debug_epoch
        else:
            self._epoch_sec = time.time()
            if self._last_epoch is not None:
                if self._epoch_sec - self._last_epoch < wait_sync_interval:
                    t = wait_sync_interval - (self._epoch_sec - self._last_epoch)
                    time.sleep(t)
                    self._epoch_sec = time.time()
        self._last_epoch = self._epoch_sec

        self._do_async()
        self._do_schedule()
        self._do_schedule_every()
        
    # 计划单次定时任务
    def schedule(self, cb, time_sec):
        self._scheduled.put_nowait( ScheduledItem(time_sec, cb) )

    # 计划重复任务
    def schedule_every(self, cb, interval_sec):
        self._scheduled_every_lock.acquire()
        self._scheduled_every.append( { "next_sec":self._epoch_sec+interval_sec, "interval":interval_sec, "cb":cb } )
        self._scheduled_every_lock.release()

    # 增加消息接收者
    def add_receiver(self, topic_or_type, cb):
        self._callbacks_lock.acquire()
        if topic_or_type not in self._callbacks:
            self._callbacks[topic_or_type] = set()
        self._callbacks[topic_or_type].add(cb)
        self._callbacks_lock.release()
        return cb

    # 删除消息接收者
    def remove_receiver(self, topic_or_type, cb):
        self._callbacks_lock.acquire()
        if topic_or_type in self._callbacks:
            if cb in self._callbacks[topic_or_type]:
                self._callbacks[topic_or_type].remove(cb)
        self._callbacks_lock.release()

    # 同步发送消息
    def send(self, obj, topic=None):
        if topic == None:
            topic = type(obj)
        cbs = []
        self._callbacks_lock.acquire()
        if topic in self._callbacks.keys():
            cbs = list(self._callbacks[topic])
        self._callbacks_lock.release()
        for cb in cbs:
            cb(obj)

    # 异步发送消息
    def send_async(self, obj, topic=None):
        self._async_queue.put_nowait( (obj, topic) )

    def _do_async(self):
        while not self._async_queue.empty():
            self.send(*self._async_queue.get_nowait())

    def _do_schedule(self):
        i = 0
        while not self._scheduled.empty():
            item = self._scheduled.get_nowait()
            if item.time_sec > self._epoch_sec:
                self._scheduled.put_nowait(item)
                break
            item.cb(self._epoch_sec)

    def _do_schedule_every(self):
        cbs = []
        self._scheduled_every_lock.acquire()
        for o in self._scheduled_every:
            while self._epoch_sec >= o["next_sec"]:
                cbs.append(o["cb"])
                o["next_sec"] += o["interval"]
        self._scheduled_every_lock.release()
        for cb in cbs:
            cb(self._epoch_sec)

    # 拦截终止信号
    def intercept_sigterm(self, intercept_signals):
        for sig in intercept_signals:
            signal.signal(sig, bind(Driver._on_signal, self))

    # 是否收到了终止信号
    def sigterm_received(self):
        return self._sigterm_received

    def _on_signal(self, signum, stack):
        print(f"Signal #{signum} received, Traceback:")
        for filename, lineno, _, line in traceback.extract_stack(stack):
            print('    at %s(%s:%d)' % (line, filename[filename.rfind(os.path.sep) + 1:], lineno))
        self._sigterm_received = True


def bind(func, *args, **kw):
    return lambda *_args, **_kw: func(*args, *_args, **kw, **_kw)


driver = Driver()


if __name__ == '__main__':
    print("initialize")

    driver.schedule(lambda epoch: print(f"Do sth. one time"), driver.get_epoch()+3)
    driver.schedule_every(lambda epoch: print(f"Do sth. regularly"), 1)
    driver.schedule(lambda _: os.system(f"kill {os.getpid()}"), driver.get_epoch()+10)
    driver.run(0.1)

    print("finalize")