Задачи помнят процесс-владельца: при старте панель больше не объявляет прерванными задачи, которые ведёт другой живой процесс
Этот коммит содержится в:
@@ -139,6 +139,31 @@ def test_orphaned_operations_are_closed_on_startup(data_dirs):
|
|||||||
assert core.active_operations_for("web01") == []
|
assert core.active_operations_for("web01") == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_interrupt_leaves_operations_of_other_live_processes(data_dirs):
|
||||||
|
"""Копии по расписанию идут в отдельной службе: перезапуск панели не должен объявлять их прерванными."""
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
import core
|
||||||
|
|
||||||
|
sleeper = subprocess.Popen(["sleep", "30"])
|
||||||
|
finished = subprocess.Popen(["sleep", "0.1"])
|
||||||
|
finished_owner = core.process_owner(finished.pid)
|
||||||
|
finished.wait()
|
||||||
|
try:
|
||||||
|
scheduled = core.new_operation("backup", "Резервная копия web01 по расписанию", vm_name="web01")
|
||||||
|
core.update_operation(scheduled, status="running", owner=core.process_owner(sleeper.pid))
|
||||||
|
dead = core.new_operation("backup", "Резервная копия db01", vm_name="db01")
|
||||||
|
core.update_operation(dead, status="running", owner=finished_owner)
|
||||||
|
legacy = core.new_operation("backup", "Задача старой версии без владельца", vm_name="db01")
|
||||||
|
core.update_operation(legacy, status="running", owner=None)
|
||||||
|
assert core.interrupt_orphaned_operations() == 2
|
||||||
|
assert core.read_operation(scheduled["id"])["status"] == "running"
|
||||||
|
assert core.read_operation(dead["id"])["interrupted"] and core.read_operation(legacy["id"])["interrupted"]
|
||||||
|
finally:
|
||||||
|
sleeper.kill()
|
||||||
|
sleeper.wait()
|
||||||
|
|
||||||
|
|
||||||
def test_snapshot_names_with_spaces_are_listed():
|
def test_snapshot_names_with_spaces_are_listed():
|
||||||
from features import snapshots
|
from features import snapshots
|
||||||
|
|
||||||
|
|||||||
+23
-2
@@ -278,9 +278,27 @@ def list_operations(limit: int = 25) -> list[dict[str, Any]]:
|
|||||||
return operations
|
return operations
|
||||||
|
|
||||||
|
|
||||||
|
def process_owner(pid: int | None = None) -> dict[str, Any] | None:
|
||||||
|
"""Who runs an operation: pid plus its start time and boot id, so a reused pid is not mistaken for it."""
|
||||||
|
pid = pid or os.getpid()
|
||||||
|
try:
|
||||||
|
stat = Path(f"/proc/{pid}/stat").read_text()
|
||||||
|
boot = Path("/proc/sys/kernel/random/boot_id").read_text().strip()
|
||||||
|
except OSError:
|
||||||
|
return None
|
||||||
|
return {"pid": pid, "start": stat.rsplit(")", 1)[1].split()[19], "boot": boot}
|
||||||
|
|
||||||
|
|
||||||
|
def owner_alive(owner: Any) -> bool:
|
||||||
|
if not isinstance(owner, dict) or not isinstance(owner.get("pid"), int):
|
||||||
|
return False
|
||||||
|
return process_owner(owner["pid"]) == owner
|
||||||
|
|
||||||
|
|
||||||
def interrupt_orphaned_operations() -> int:
|
def interrupt_orphaned_operations() -> int:
|
||||||
"""Operations run in threads of the panel process: after a restart the ones still marked
|
"""Operations run in threads of the panel process: after a restart the ones still marked
|
||||||
queued/running will never finish. Close them so they do not block the next attempt."""
|
queued/running will never finish. Close them so they do not block the next attempt.
|
||||||
|
Operations of another live process (scheduled backups run from a timer) are left alone."""
|
||||||
if not OPERATIONS_DIR.exists():
|
if not OPERATIONS_DIR.exists():
|
||||||
return 0
|
return 0
|
||||||
count = 0
|
count = 0
|
||||||
@@ -291,6 +309,9 @@ def interrupt_orphaned_operations() -> int:
|
|||||||
continue
|
continue
|
||||||
if data.get("status") not in ("queued", "running"):
|
if data.get("status") not in ("queued", "running"):
|
||||||
continue
|
continue
|
||||||
|
owner = data.get("owner")
|
||||||
|
if owner_alive(owner) and owner["pid"] != os.getpid():
|
||||||
|
continue
|
||||||
data.pop("log_tail", None)
|
data.pop("log_tail", None)
|
||||||
message = "Прервано: панель перезапускалась во время операции. Проверьте состояние машины и запустите действие снова."
|
message = "Прервано: панель перезапускалась во время операции. Проверьте состояние машины и запустите действие снова."
|
||||||
update_operation(data, status="error", progress=100, message=message, finished_at=utc_now(), interrupted=True)
|
update_operation(data, status="error", progress=100, message=message, finished_at=utc_now(), interrupted=True)
|
||||||
@@ -334,7 +355,7 @@ def update_operation(operation: dict[str, Any], **changes: Any) -> None:
|
|||||||
|
|
||||||
|
|
||||||
def new_operation(kind: str, title: str, message: str = "Операция поставлена в очередь", **meta: Any) -> dict[str, Any]:
|
def new_operation(kind: str, title: str, message: str = "Операция поставлена в очередь", **meta: Any) -> dict[str, Any]:
|
||||||
operation = {"id": str(uuid.uuid4()), "type": kind, "title": title, "status": "queued", "progress": 0, "message": message, "created_at": utc_now(), "updated_at": utc_now(), "created_by": AUTH_USER}
|
operation = {"id": str(uuid.uuid4()), "type": kind, "title": title, "status": "queued", "progress": 0, "message": message, "created_at": utc_now(), "updated_at": utc_now(), "created_by": AUTH_USER, "owner": process_owner()}
|
||||||
operation.update(meta)
|
operation.update(meta)
|
||||||
write_operation(operation)
|
write_operation(operation)
|
||||||
append_operation_log(operation["id"], "Операция поставлена в очередь.")
|
append_operation_log(operation["id"], "Операция поставлена в очередь.")
|
||||||
|
|||||||
Ссылка в новой задаче
Block a user