Merge branch 'feature/downloads' into claude/production-system-image-smcxyr

# Conflicts:
#	tests/dev_server.py
#	web/app.py
#	web/static/app.css
Этот коммит содержится в:
Claude
2026-09-25 15:02:19 +00:00
родитель 0e4b1783ec 4fd04be588
Коммит a43f96883e
15 изменённых файлов: 1130 добавлений и 2 удалений
+2
Просмотреть файл
@@ -91,6 +91,8 @@ def fake_run_cmd(cmd, timeout=12, **_kwargs):
return fake_result(":0")
if cmd[:2] == ["virsh", "pool-list"]:
return fake_result(" Name State Autostart\n---\n virtuality-images active yes")
if cmd[:2] == ["virsh", "pool-refresh"]:
return fake_result(f"Pool {cmd[-1]} refreshed")
if cmd[:2] == ["virsh", "net-list"]:
return fake_result(" Name State Autostart Persistent\n---\n virtuality-nat active yes yes")
if cmd[:2] == ["systemctl", "is-active"]:
+11
Просмотреть файл
@@ -132,6 +132,17 @@ def seed(root: Path) -> None:
for line in ("Машина работает — отправляем команду выключения.", "Машина выключена.", "Настройки машины сохранены в vm.xml.", "Копируем диск vda (1 из 1): 100%", "Машина запущена снова."):
app.append_operation_log(op_id, line)
# Идущие скачивания — видны на /iso и /disk-images (в каталоге Ubuntu 26.04 помечена «Скачивается»).
for kind, url, catalog_id, title, progress, message, total, done in (
("iso", "https://software-download.microsoft.com/download/pr/windows-server-2025-eval-x64.iso", None, "Скачивание windows-server-2025-eval-x64.iso", 37, "1,9 ГБ из 5,2 ГБ · 24 МБ/с · осталось 2 мин", 5_583_457_280, 2_065_879_193),
("disk", "https://cloud-images.ubuntu.com/resolute/current/resolute-server-cloudimg-amd64.img", "ubuntu-26.04", "Скачивание Ubuntu Server 26.04 LTS", 62, "412 МБ из 663 МБ · 31 МБ/с · осталось 8 с", 695_205_888, 431_027_650),
):
filename = url.rsplit("/", 1)[-1]
target = (dirs["iso"] if kind == "iso" else dirs["disk-images"]) / filename
op = core.new_operation("download", title, download_kind=kind, url=url, filename=filename, target_path=str(target), page="/iso" if kind == "iso" else "/disk-images", catalog_id=catalog_id, total_bytes=total, downloaded_bytes=done, cancel_requested=False)
core.update_operation(op, status="running", progress=progress, message=message, started_at=core.utc_now())
core.append_operation_log(op["id"], f"Ссылка: {url}")
if __name__ == "__main__":
import uvicorn
+3
Просмотреть файл
@@ -0,0 +1,3 @@
5861314d7fccb39c2192173240eab44fa35ca66426201ca2acd0630a6258dd51 *resolute-server-cloudimg-amd64.img
f69162950f235e3cdbbad33f1f912d1a504be90d8a37d002c735d6f3e3882265 *resolute-server-cloudimg-arm64.img
e0361263a9f568c21a59e75c3b7810bc5867866e5bd48f499f8b2e6ab723f059 *resolute-server-cloudimg-amd64.vmdk
+3
Просмотреть файл
@@ -0,0 +1,3 @@
# generated by debian cloud team
1412d62c2d55ae2432df6d22c36b9edf4512a59a43bb4eadb1bd3a5ef201154a3052d83ffae704d7079b50388e61992013348e4bb4dee45b3c2a266e2b296135 debian-13-genericcloud-amd64.qcow2
27c03aab0b5ee1f5918bfaa94821d2a1500fc171ffeb0ae159e7d18662f49171551f397683004434096e4006380fa607577b3c90fafa1eceb3a92702ebd49a15 debian-13-genericcloud-arm64.qcow2
+1
Просмотреть файл
@@ -0,0 +1 @@
2ae30006299896d225c9d4de36e9a7c3d047c9b4a5793efe3a6422ec5040e8ee5f7d71326aa9922ffa446fbd11ed6cd1a8d0af4659b3d00233d3fea3550d2d78
+381
Просмотреть файл
@@ -0,0 +1,381 @@
"""Скачивание образов по ссылке и из каталога: парсер контрольных сумм,
проверка ссылок, фоновая загрузка с локального http-сервера, отмена."""
import hashlib
import lzma
import threading
import time
from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from types import SimpleNamespace
from urllib.parse import unquote
import pytest
import app # noqa: F401 — подключает features.downloads к приложению
import catalog
import core
import features.downloads as downloads
FIXTURES = Path(__file__).resolve().parent / "fixtures"
MB = 1024 * 1024
SLOW_TOTAL = 4 * MB
class Handler(SimpleHTTPRequestHandler):
"""Раздаёт файлы из каталога www; /slow.iso отдаёт 4 МБ по 64 КБ с паузами, /redirect.iso — перенаправляет."""
root = ""
def __init__(self, *args, **kwargs):
super().__init__(*args, directory=self.root, **kwargs)
def log_message(self, *args): # noqa: D102 — тише в тестах
pass
def do_GET(self):
if self.path == "/redirect.iso":
self.send_response(302)
self.send_header("Location", "/small.iso")
self.end_headers()
return
if self.path == "/slow.iso":
self.send_response(200)
self.send_header("Content-Type", "application/octet-stream")
self.send_header("Content-Length", str(SLOW_TOTAL))
self.end_headers()
try:
for _ in range(SLOW_TOTAL // (64 * 1024)):
self.wfile.write(b"\0" * 64 * 1024)
self.wfile.flush()
time.sleep(0.05)
except (BrokenPipeError, ConnectionResetError):
pass
return
super().do_GET()
@pytest.fixture()
def www(tmp_path):
root = tmp_path / "www"
root.mkdir()
(root / "small.iso").write_bytes(b"ISO" * 5000)
handler = type("TestHandler", (Handler,), {"root": str(root)})
server = ThreadingHTTPServer(("127.0.0.1", 0), handler)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
yield SimpleNamespace(root=root, url=f"http://127.0.0.1:{server.server_port}")
server.shutdown()
server.server_close()
def wait_for(operation_id: str, timeout: float = 20.0) -> dict:
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
operation = core.read_operation(operation_id)
if operation and operation.get("status") in ("success", "error"):
return operation
time.sleep(0.05)
raise AssertionError("операция не завершилась вовремя")
def wait_until(predicate, timeout: float = 10.0) -> None:
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if predicate():
return
time.sleep(0.05)
raise AssertionError("условие не наступило вовремя")
# ---------------------------------------------------------------- parsers & formatting
def test_parse_checksums_ubuntu_star_format():
text = (FIXTURES / "SHA256SUMS").read_text()
assert downloads.parse_checksums(text, "resolute-server-cloudimg-amd64.img") == "5861314d7fccb39c2192173240eab44fa35ca66426201ca2acd0630a6258dd51"
assert downloads.parse_checksums(text, "resolute-server-cloudimg-arm64.img").startswith("f6916295")
assert downloads.parse_checksums(text, "missing.img") is None
def test_parse_checksums_debian_two_spaces_and_comments():
text = (FIXTURES / "SHA512SUMS").read_text()
assert downloads.parse_checksums(text, "debian-13-genericcloud-amd64.qcow2").startswith("1412d62c2d55")
assert downloads.parse_checksums(text, "debian-13-genericcloud-arm64.qcow2").startswith("27c03aab0b5e")
def test_parse_checksums_alpine_bare_hash_and_bsd_style():
bare = (FIXTURES / "nocloud_alpine-3.23.0-x86_64-uefi-cloudinit-r0.qcow2.sha512").read_text()
assert downloads.parse_checksums(bare, "anything.qcow2") == bare.strip().lower()
bsd = "SHA256 (disk.qcow2) = ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789\n"
assert downloads.parse_checksums(bsd, "disk.qcow2") == "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"
assert downloads.parse_checksums(bsd, "other.qcow2") is None
assert downloads.parse_checksums("not a checksum file\n", "disk.qcow2") is None
def test_human_size_and_eta():
assert downloads.human_size(0) == "0 Б"
assert downloads.human_size(1023) == "1023 Б"
assert downloads.human_size(1536) == "1,5 КБ"
assert downloads.human_size(245 * MB) == "245 МБ"
assert downloads.human_size(int(3.1 * 1024 * MB)) == "3,1 ГБ"
assert downloads.human_size(10 * MB) == "10 МБ"
assert downloads.human_eta(45) == "45 с"
assert downloads.human_eta(244) == "4 мин"
assert downloads.human_eta(3700) == "1 ч 2 мин"
assert downloads.human_eta(7200) == "2 ч"
def test_progress_message():
assert downloads.progress_message(245 * MB, int(3.1 * 1024 * MB), 12 * MB) == "245 МБ из 3,1 ГБ · 12 МБ/с · осталось 4 мин"
assert downloads.progress_message(245 * MB, None, 12 * MB) == "245 МБ · 12 МБ/с"
assert downloads.progress_message(0, 100 * MB, None) == "0 Б из 100 МБ"
# ---------------------------------------------------------------- validation
@pytest.mark.parametrize("url", ["", "ftp://example.org/x.iso", "file:///etc/passwd", "javascript:alert(1)", "http://", "http:///x.iso", "http://example.org/a b.iso"])
def test_validate_url_rejects(url):
with pytest.raises(downloads.DownloadError):
downloads.validate_url(url)
def test_filename_from_url_sanitises_and_checks_suffix():
assert downloads.filename_from_url("https://h.example/a%20b%20(1).ISO?x=1#frag", "iso") == "a-b-_1.iso"
assert downloads.filename_from_url("https://h.example/dir/debian-13.qcow2", "disk") == "debian-13.qcow2"
assert downloads.filename_from_url("https://h.example/rpi/2026-01-01-raspios.img.xz", "disk") == "2026-01-01-raspios.img.xz"
assert downloads.filename_from_url("https://h.example/../../etc.iso", "iso") == "etc.iso"
for url, kind in (("https://h.example/", "iso"), ("https://h.example/setup.exe", "iso"), ("https://h.example/x.iso", "disk"), ("https://h.example/x.qcow2", "iso")):
with pytest.raises(downloads.DownloadError):
downloads.filename_from_url(url, kind)
def test_start_download_refuses_existing_file(data_dirs):
(data_dirs["iso"] / "small.iso").write_bytes(b"x")
with pytest.raises(downloads.DownloadError, match="уже есть"):
downloads.start_download("http://127.0.0.1:9/small.iso", "iso")
def test_start_download_checks_free_space(data_dirs, monkeypatch):
monkeypatch.setattr(downloads.shutil, "disk_usage", lambda path: SimpleNamespace(free=1000, total=1000, used=0))
with pytest.raises(downloads.DownloadError, match="мало места"):
downloads.start_download("http://127.0.0.1:9/small.iso", "iso")
def test_image_candidates():
assert downloads.image_candidates("a.img.xz") == ["a.img.xz", "a.img", "a.qcow2"]
assert downloads.image_candidates("a.img") == ["a.img", "a.qcow2"]
assert downloads.image_candidates("a.qcow2") == ["a.qcow2"]
# ---------------------------------------------------------------- background download
def test_download_iso_success_refreshes_pool(data_dirs, www, monkeypatch):
calls = []
monkeypatch.setattr(downloads, "run_cmd", lambda cmd, timeout=12: calls.append(cmd) or {"ok": True, "stdout": "", "stderr": ""})
operation = downloads.start_download(f"{www.url}/small.iso", "iso")
assert operation["type"] == "download" and operation["download_kind"] == "iso"
done = wait_for(operation["id"])
assert done["status"] == "success", done
assert done["progress"] == 100
assert done["total_bytes"] == 15000 and done["downloaded_bytes"] == 15000
assert (data_dirs["iso"] / "small.iso").read_bytes() == b"ISO" * 5000
assert not (data_dirs["iso"] / ".small.iso.part").exists()
assert ["virsh", "pool-refresh", "virtuality-iso"] in calls
assert "Готово" in done["message"]
def test_download_follows_redirect(data_dirs, www):
operation = downloads.start_download(f"{www.url}/redirect.iso", "iso")
done = wait_for(operation["id"])
assert done["status"] == "success", done
assert (data_dirs["iso"] / "redirect.iso").stat().st_size == 15000
assert "Перенаправление" in done["log_tail"]
def test_download_missing_file_fails(data_dirs, www):
operation = downloads.start_download(f"{www.url}/nope.iso", "iso")
done = wait_for(operation["id"])
assert done["status"] == "error"
assert "404" in done["message"]
assert not list(data_dirs["iso"].iterdir())
def test_download_refuses_when_disk_fills_up(data_dirs, www, monkeypatch):
real = downloads.free_space_ok
seen = []
def fake(directory, needed):
seen.append(needed)
return real(directory, needed) if len(seen) == 1 else (False, 42)
monkeypatch.setattr(downloads, "free_space_ok", fake)
operation = downloads.start_download(f"{www.url}/small.iso", "iso")
done = wait_for(operation["id"])
assert done["status"] == "error"
assert "мало места" in done["message"] and "42 Б" in done["message"]
assert not list(data_dirs["iso"].iterdir())
def test_download_cancel(data_dirs, www, logged_in):
operation = downloads.start_download(f"{www.url}/slow.iso", "iso")
wait_until(lambda: (core.read_operation(operation["id"]) or {}).get("status") == "running")
wait_until(lambda: (data_dirs["iso"] / ".slow.iso.part").exists())
response = logged_in.post(f"/api/operations/{operation['id']}/cancel")
assert response.status_code == 200 and response.json()["ok"] is True
done = wait_for(operation["id"])
assert done["status"] == "error"
assert done["message"] == "Скачивание отменено"
assert done["cancelled"] is True and done["cancel_requested"] is True
assert not (data_dirs["iso"] / ".slow.iso.part").exists()
assert not (data_dirs["iso"] / "slow.iso").exists()
assert not downloads.cancel_requested(operation["id"])
def test_catalog_download_verifies_checksum(data_dirs, www):
payload = b"QCOW2 image bytes" * 100
(www.root / "disk.qcow2").write_bytes(payload)
(www.root / "SHA256SUMS").write_text(f"{hashlib.sha256(payload).hexdigest()} *disk.qcow2\n")
operation = downloads.start_download(f"{www.url}/disk.qcow2", "disk", sums_url=f"{www.url}/SHA256SUMS", algo="sha256", catalog_id="test")
done = wait_for(operation["id"])
assert done["status"] == "success", done
assert "совпала" in done["log_tail"]
assert (data_dirs["disk_images"] / "disk.qcow2").read_bytes() == payload
def test_catalog_download_checksum_mismatch_deletes_file(data_dirs, www):
(www.root / "disk.qcow2").write_bytes(b"corrupted" * 100)
(www.root / "SHA512SUMS").write_text(f"{'0' * 128} disk.qcow2\n")
operation = downloads.start_download(f"{www.url}/disk.qcow2", "disk", sums_url=f"{www.url}/SHA512SUMS", algo="sha512")
done = wait_for(operation["id"])
assert done["status"] == "error"
assert "Контрольная сумма не совпала" in done["message"]
assert not list(data_dirs["disk_images"].iterdir())
def test_catalog_download_fails_without_sums_entry(data_dirs, www):
(www.root / "disk.qcow2").write_bytes(b"data")
(www.root / "SHA256SUMS").write_text(f"{'a' * 64} *other.qcow2\n")
operation = downloads.start_download(f"{www.url}/disk.qcow2", "disk", sums_url=f"{www.url}/SHA256SUMS", algo="sha256")
done = wait_for(operation["id"])
assert done["status"] == "error"
assert "нет записи" in done["message"]
assert not list(data_dirs["disk_images"].iterdir())
def test_disk_download_extracts_xz_and_starts_conversion(data_dirs, www, monkeypatch):
raw = bytes(range(256)) * 256
(www.root / "pi.img.xz").write_bytes(lzma.compress(raw))
started = []
monkeypatch.setattr(app, "start_disk_convert_operation", lambda path: started.append(path) or {"id": "conv-1"})
operation = downloads.start_download(f"{www.url}/pi.img.xz", "disk")
done = wait_for(operation["id"])
assert done["status"] == "success", done
assert (data_dirs["disk_images"] / "pi.img").read_bytes() == raw
assert not (data_dirs["disk_images"] / "pi.img.xz").exists()
assert started == [data_dirs["disk_images"] / "pi.img"]
assert done["convert_operation_id"] == "conv-1"
assert "qcow2" in done["message"]
def test_qcow2_download_needs_no_conversion(data_dirs, www, monkeypatch):
(www.root / "ready.qcow2").write_bytes(b"q" * 10)
monkeypatch.setattr(app, "start_disk_convert_operation", lambda path: None)
done = wait_for(downloads.start_download(f"{www.url}/ready.qcow2", "disk")["id"])
assert done["status"] == "success" and "convert_operation_id" not in done
assert done["result_path"] == str(data_dirs["disk_images"] / "ready.qcow2")
# ---------------------------------------------------------------- routes
def test_url_route_starts_download(logged_in, data_dirs, www):
response = logged_in.post("/downloads/url", data={"url": f"{www.url}/small.iso", "kind": "iso"}, follow_redirects=False)
assert response.status_code == 303
location = unquote(response.headers["location"])
assert location.startswith("/iso?download_message=") and "small.iso" in location
running = downloads.active_downloads("iso")
assert len(running) == 1 and running[0]["filename"] == "small.iso"
assert wait_for(running[0]["id"])["status"] == "success"
assert (data_dirs["iso"] / "small.iso").exists()
def test_url_route_rejects_bad_links(logged_in, data_dirs):
response = logged_in.post("/downloads/url", data={"url": "ftp://example.org/x.iso", "kind": "iso"}, follow_redirects=False)
assert response.status_code == 303
assert "download_error=" in response.headers["location"] and response.headers["location"].startswith("/iso?")
response = logged_in.post("/downloads/url", data={"url": "https://example.org/setup.exe", "kind": "disk"}, follow_redirects=False)
assert response.headers["location"].startswith("/disk-images?download_error=")
assert not core.running_operations("download")
def test_url_route_requires_login(client):
assert client.post("/downloads/url", data={"url": "https://example.org/x.iso"}, follow_redirects=False).headers["location"] == "/login"
assert client.post("/api/operations/x/cancel").status_code == 401
def test_catalog_route(logged_in, data_dirs, www, monkeypatch):
payload = b"debian" * 1000
(www.root / "debian-13-genericcloud-amd64.qcow2").write_bytes(payload)
(www.root / "SHA512SUMS").write_text(f"{hashlib.sha512(payload).hexdigest()} debian-13-genericcloud-amd64.qcow2\n")
entry = {"id": "debian-13", "title": "Debian 13", "hint": "x", "login": "debian", "arches": {"x86_64": {"url": f"{www.url}/debian-13-genericcloud-amd64.qcow2", "sums": f"{www.url}/SHA512SUMS", "algo": "sha512"}}}
monkeypatch.setattr(catalog, "CLOUD_IMAGES", [entry])
monkeypatch.setattr(downloads, "host_arch", lambda: "x86_64")
monkeypatch.setattr(app, "start_disk_convert_operation", lambda path: None)
assert unquote(logged_in.post("/downloads/catalog/nope", follow_redirects=False).headers["location"]).startswith("/disk-images?download_error=Такого образа нет")
response = logged_in.post("/downloads/catalog/debian-13", follow_redirects=False)
assert response.status_code == 303 and "download_message=" in response.headers["location"]
page = logged_in.get("/disk-images").text
assert "Скачивается" in page
running = downloads.active_downloads("disk")
assert running[0]["catalog_id"] == "debian-13"
assert wait_for(running[0]["id"])["status"] == "success"
assert (data_dirs["disk_images"] / "debian-13-genericcloud-amd64.qcow2").read_bytes() == payload
page = logged_in.get("/disk-images").text
assert "Загружен" in page and "/vm/create?image=debian-13-genericcloud-amd64.qcow2" in page
# второй раз — файл уже есть
response = logged_in.post("/downloads/catalog/debian-13", follow_redirects=False)
assert "download_error=" in response.headers["location"]
def test_catalog_route_skips_missing_arch(logged_in, data_dirs, monkeypatch):
monkeypatch.setattr(catalog, "CLOUD_IMAGES", [{"id": "only-arm", "title": "ARM only", "arches": {"aarch64": {"url": "https://example.org/a.qcow2"}}}])
monkeypatch.setattr(downloads, "host_arch", lambda: "x86_64")
location = unquote(logged_in.post("/downloads/catalog/only-arm", follow_redirects=False).headers["location"])
assert "не выпускается для процессора x86_64" in location
assert downloads.catalog_entries() == []
assert "готовых образов в каталоге нет" in logged_in.get("/disk-images").text
def test_cancel_api_rules(logged_in, data_dirs):
assert logged_in.post("/api/operations/00000000-0000-0000-0000-000000000000/cancel").status_code == 404
other = core.new_operation("vm_create", "Создание VM x")
response = logged_in.post(f"/api/operations/{other['id']}/cancel")
assert response.status_code == 400 and "только скачивание" in response.json()["error"]
finished = core.new_operation("download", "Скачивание x", download_kind="iso", filename="x.iso")
core.finish_operation(finished, True, "Готово")
assert logged_in.post(f"/api/operations/{finished['id']}/cancel").status_code == 409
running = core.new_operation("download", "Скачивание y", download_kind="disk", filename="y.qcow2")
core.update_operation(running, status="running")
response = logged_in.post(f"/api/operations/{running['id']}/cancel")
assert response.status_code == 200 and response.json()["operation"]["cancel_requested"] is True
assert downloads.cancel_requested(running["id"])
downloads.clear_cancel(running["id"])
def test_cancel_form_redirects_to_page(logged_in, data_dirs):
running = core.new_operation("download", "Скачивание y", download_kind="disk", filename="y.qcow2")
core.update_operation(running, status="running")
response = logged_in.post(f"/downloads/{running['id']}/cancel", follow_redirects=False)
assert response.status_code == 303
assert unquote(response.headers["location"]).startswith("/disk-images?download_message=Останавливаем скачивание y.qcow2")
assert unquote(logged_in.post("/downloads/nope/cancel", follow_redirects=False).headers["location"]).startswith("/iso?download_error=")
downloads.clear_cancel(running["id"])
def test_pages_show_download_cards(logged_in, data_dirs):
iso_page = logged_in.get("/iso").text
assert "Скачать по ссылке" in iso_page and 'name="kind" value="iso"' in iso_page
assert "Скачиваем на сервер" not in iso_page
disk_page = logged_in.get("/disk-images").text
assert "Каталог готовых систем" in disk_page and "Рекомендуем" in disk_page and 'name="kind" value="disk"' in disk_page
running = core.new_operation("download", "Скачивание big.iso", download_kind="iso", filename="big.iso", url="https://example.org/big.iso", total_bytes=None)
core.update_operation(running, status="running", progress=0, message="Подключаемся к серверу…")
iso_page = logged_in.get("/iso?download_message=Скачиваем").text
assert "Скачиваем на сервер" in iso_page and "big.iso" in iso_page and "indeterminate" in iso_page
assert f"/downloads/{running['id']}/cancel" in iso_page and "Скачиваем</span>" in iso_page
assert "big.iso" not in logged_in.get("/disk-images").text