Files
grotti/grotti/print_queue.py
T

1595 lines
65 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Druckwarteschlange: startet den nächsten Job, sobald ein Drucker frei ist.
Abläufe (Drucken → Abkühlen → Schieben, optional mehrfach) bleiben running,
bis die letzte Wiederholung durch ist.
"""
from __future__ import annotations
import threading
import time
from datetime import datetime, timezone
from grotti import acm, eject, jobs, logutil, moonraker, notify, store
from grotti.db import connect
from grotti.errors import GrottiError
from grotti import time_window
WINDOW_WAIT_MSG = "Wartet auf Zeitfenster…"
from grotti.events_bus import publish as publish_live
log = logutil.get("grotti.queue")
def _notify_queue():
publish_live("queue", {})
FREE = {"standby", "complete", "cancelled"}
BUSY = {"printing", "paused", "heating", "error"}
# deutlich unter Klippers idle_timeout (600 s)
KEEPALIVE_S = 120
# Nach Start ohne print_active: Upload/MQTT/LeviQ brauchen Zeit, aber nicht ewig
PRINT_START_TIMEOUT_S = 600
# Upload-Worker weg (typisch Grotti-Neustart mitten im Hochladen).
# Nach erfolgreichem Senden ist die Phase await_print — dort gilt das nicht,
# sonst wird ein noch anlaufender Druck als „Neustart“ neu eingereiht.
SIMPLE_START_STUCK_S = 45
# Anycubic meldet kurz cancelled/standby bei LeviQ und Filament-Resume.
TERMINAL_CONFIRM_HITS = 3
AWAIT_CANCEL_HITS = 3
def _ablauf_needs_motors(item):
"""True wenn nach dem aktuellen Schritt noch Cool/Sweep kommt (kein M84 im Upload)."""
steps = item.get("steps") or []
try:
idx = int(item.get("step_index") or 0)
except (TypeError, ValueError):
idx = 0
for step in steps[idx + 1 :]:
if isinstance(step, dict) and step.get("type") in ("cool", "sweep"):
return True
return False
def _cool_wait_text(live, cool):
"""Ohne „ · “: die Oberfläche zeigt nur das letzte „ · “-Stück der Meldung."""
waiting = []
done = []
for label, key, limit_key in (("Düse", "nozzle", "nozzle_max"), ("Bett", "bed", "bed_max")):
value = live.get(key)
limit = float(cool[limit_key])
if not isinstance(value, (int, float)):
waiting.append("%s ? → ≤ %.0f °C" % (label, limit))
elif float(value) > limit:
waiting.append("%s %.0f → ≤ %.0f °C" % (label, value, limit))
else:
done.append("%s %.0f °C ok" % (label, value))
text = "wartet auf " + " und ".join(waiting) if waiting else "gleich kühl genug"
if done:
text += ", " + ", ".join(done)
return text
def _parse_steps(row):
steps = row.get("steps") if isinstance(row, dict) else None
if isinstance(steps, str):
import json
try:
steps = json.loads(steps)
except json.JSONDecodeError:
steps = []
return steps if isinstance(steps, list) else []
def _has_ablauf(row):
return bool(_parse_steps(row))
class PrintQueueDispatcher:
def __init__(self, database_url, cfg, interval=8, watcher=None):
self.database_url = database_url
self.cfg = cfg
self.interval = max(4, int(interval))
self.watcher = watcher
self._stop = threading.Event()
self._thread = None
self._keepalive = {}
self._cool_park = {}
self._workers = {}
self._worker_lock = threading.Lock()
def start(self):
if self._thread and self._thread.is_alive():
return
self._thread = threading.Thread(
target=self._loop,
name="grotti-print-queue",
daemon=True,
)
self._thread.start()
def stop(self):
self._stop.set()
def _loop(self):
while not self._stop.is_set():
try:
self.tick()
except Exception as exc:
log.warning("Druck-Queue: %s", exc)
self._stop.wait(self.interval)
def _live_map(self, printers):
if self.watcher:
cached = self.watcher.by_host()
age = self.watcher.age_s()
if cached and age is not None and age <= 30:
return cached
return {row["host"]: row for row in moonraker.snapshot(printers)}
def _auto_enabled(self):
return bool((self.cfg or {}).get("auto_queue", True))
def _worker_busy(self, item_id):
with self._worker_lock:
thread = self._workers.get(item_id)
return bool(thread and thread.is_alive())
def _item_phase_age_s(self, item):
"""Alter von phase_started_at, sonst updated_at (für hängende Starts nach Neustart)."""
started = item.get("phase_started_at")
if started:
if getattr(started, "tzinfo", None) is None:
started = started.replace(tzinfo=timezone.utc)
return (datetime.now(timezone.utc) - started).total_seconds()
updated = item.get("updated_at")
if updated:
if getattr(updated, "tzinfo", None) is None:
updated = updated.replace(tzinfo=timezone.utc)
return (datetime.now(timezone.utc) - updated).total_seconds()
return 0
def _auto_retry_enabled(self):
return bool((self.cfg or {}).get("auto_retry", True))
def _retryable_message(self, message):
text = (message or "").lower()
if "abgebrochen" in text:
return False
if "druck abgebrochen" in text:
return False
return True
def _try_auto_retry(self, conn, item, message):
"""Ein automatischer Neuversuch; sonst False. Nicht nach Abbruch."""
if not self._auto_retry_enabled():
return False
if not self._retryable_message(message):
return False
retries = int(item.get("retry_count") or 0)
if retries >= 1:
return False
store.update_print_queue(
conn,
item["id"],
{
"status": "waiting",
"message": "Nochmal (%s)" % (message or "Fehler"),
"retry_count": retries + 1,
"phase": "",
"phase_started_at": None,
"host": item.get("host") or "",
},
)
return True
def _terminal_hits(self, item_id, kind="end"):
key = "%s:%s" % (item_id, kind)
if not hasattr(self, "_term_hits"):
self._term_hits = {}
self._term_hits[key] = self._term_hits.get(key, 0) + 1
return self._term_hits[key]
def _clear_terminal_hits(self, item_id, kind=None):
if not hasattr(self, "_term_hits"):
return
if kind:
self._term_hits.pop("%s:%s" % (item_id, kind), None)
return
for key in list(self._term_hits):
if key.startswith("%s:" % item_id):
self._term_hits.pop(key, None)
def _fail_stuck_simple_start(self, conn, item, printer):
host = printer["host"]
msg = "Start abgebrochen (Neustart)"
if self._try_auto_retry(conn, item, msg):
self._event(
conn, host, printer["name"], "queue",
"%s: Nochmal — %s" % (printer["name"], msg),
item.get("filename") or "",
)
return
store.update_print_queue(
conn,
item["id"],
{
"status": "error",
"message": msg,
"phase": "",
"phase_started_at": None,
},
)
self._event(
conn,
host,
printer["name"],
"queue",
"%s: Start abgebrochen (Neustart)" % printer["name"],
item.get("filename") or "",
)
def _spawn_worker(self, item_id, target, *args):
with self._worker_lock:
existing = self._workers.get(item_id)
if existing and existing.is_alive():
return False
def run():
try:
target(*args)
except Exception as exc:
log.warning("Queue-Worker %s: %s", item_id, exc)
finally:
_notify_queue()
with self._worker_lock:
self._workers.pop(item_id, None)
thread = threading.Thread(
target=run,
name="grotti-queue-job-%s" % item_id,
daemon=True,
)
self._workers[item_id] = thread
thread.start()
return True
def tick(self):
try:
self._tick_once()
finally:
_notify_queue()
def _tick_once(self):
with connect(self.database_url) as conn:
printers = store.list_printers(conn)
if not printers:
conn.commit()
return
lives = self._live_map(printers)
# Zuerst laufende Abläufe weiterführen; diese Hosts sind für „wenn frei“ belegt.
ablauf_hosts = set()
for printer in printers:
host = printer["host"]
live = lives.get(host) or {}
running = store.running_print_on_host(conn, host)
if running and _has_ablauf(running):
ablauf_hosts.add(host)
# Langer Schritt läuft schon im Worker — nicht doppelt anstoßen.
if self._worker_busy(running["id"]):
continue
self._advance_ablauf(conn, running, printer, live)
conn.commit()
if not self._auto_enabled():
conn.commit()
return
# Alle freien Drucker in diesem Tick anstoßen — nicht einer alle 8 s.
for printer in printers:
host = printer["host"]
if host in ablauf_hosts:
continue
if printer.get("locked"):
continue
live = lives.get(host) or {}
state = (live.get("state") or "").lower()
if live.get("ok") is False or state in ("offline", ""):
continue
# Hängender error (z. B. alter Fehlstart) blockiert sonst „wenn frei“ —
# räumen wie beim Sofort-Start, Slice macht G28 selbst.
if state == "error":
if not store.next_waiting_print(conn, host=host, family=printer.get("family") or ""):
continue
live = self._try_clear_stuck_error(conn, printer, live)
lives[host] = live
state = (live.get("state") or "").lower()
if state in BUSY:
# Laufenden Einfach-Druck mitziehen. Sonst bleibt await_print
# stehen, bis der Drucker frei ist — und der Start gilt als Timeout,
# obwohl er längst druckt.
running = store.running_print_on_host(conn, host)
if running and not _has_ablauf(running):
self._advance_simple(conn, running, printer, live)
conn.commit()
continue
if state not in FREE:
continue
running = store.running_print_on_host(conn, host)
if running:
if not _has_ablauf(running):
if (
(running.get("phase") or "").lower() == "starting"
and not self._worker_busy(running["id"])
and self._item_phase_age_s(running) > SIMPLE_START_STUCK_S
):
self._fail_stuck_simple_start(conn, running, printer)
else:
self._advance_simple(conn, running, printer, live)
conn.commit()
continue
if store.bed_needs_clear(conn, host):
continue
waiting = store.next_waiting_print(conn, host=host, family=printer.get("family") or "")
if not waiting:
continue
sw = printer.get("start_window") or {}
if sw and not time_window.allowed_now(sw):
if (waiting.get("message") or "") != WINDOW_WAIT_MSG:
store.update_print_queue(conn, waiting["id"], {"message": WINDOW_WAIT_MSG})
continue
est_s = 0
if waiting.get("file_id"):
file_row = store.get_file_row(conn, waiting["file_id"])
if file_row:
est_s = float(file_row.get("estimated_s") or 0)
if sw and not time_window.job_fits_overnight(sw, est_s):
if (waiting.get("message") or "") != WINDOW_WAIT_MSG:
store.update_print_queue(conn, waiting["id"], {"message": WINDOW_WAIT_MSG})
continue
# Nächster passender / leere Map: ACE aus Spulen bauen oder warten
if not _has_ablauf(waiting):
waiting = self._ensure_ace_map(conn, waiting, host, live)
if not waiting:
continue
claimed = store.claim_print_queue(conn, waiting["id"], host)
if not claimed:
continue
conn.commit()
ablauf_hosts.add(host)
if _has_ablauf(claimed):
self._begin_ablauf(conn, claimed, printer, live)
else:
self._start_simple(conn, claimed, printer)
conn.commit()
def _ensure_ace_map(self, conn, waiting, host, live):
"""Bei ACE-Drucker und leerer Map: aus Spulen zuordnen oder Wartegrund setzen."""
slots = (live or {}).get("slots") or []
if not slots:
return waiting
ace = waiting.get("ace_map") or []
if isinstance(ace, str):
import json
try:
ace = json.loads(ace)
except json.JSONDecodeError:
ace = []
if acm.normalize_mapping(ace if isinstance(ace, list) else []):
return waiting
file_id = waiting.get("file_id")
if not file_id:
return waiting
mapping, reason = store.auto_ace_map(conn, host, file_id)
if reason:
if (waiting.get("wait_reason") or "") != reason or (waiting.get("message") or "") != reason:
store.update_print_queue(
conn,
waiting["id"],
{"wait_reason": reason, "message": reason},
)
return None
if not mapping:
# ACE am Drucker, aber keine ACE-Spulen in Grotti → nicht raten
reason = "ACE-Zuordnung fehlt (keine passende Spule)"
if (waiting.get("wait_reason") or "") != reason:
store.update_print_queue(
conn,
waiting["id"],
{"wait_reason": reason, "message": reason},
)
return None
store.update_print_queue(conn, waiting["id"], {"ace_map": mapping, "wait_reason": "", "message": ""})
waiting = dict(waiting)
waiting["ace_map"] = mapping
waiting["wait_reason"] = ""
return waiting
def _start_simple(self, conn, item, printer):
"""Upload/Start im Worker — mehrere freie Drucker parallel."""
host = printer["host"]
item_id = item["id"]
ace = item.get("ace_map") or []
if isinstance(ace, str):
import json
try:
ace = json.loads(ace)
except json.JSONDecodeError:
ace = []
ace_list = ace if isinstance(ace, list) else []
store.update_print_queue(
conn,
item_id,
{
"status": "running",
"host": host,
"phase": "starting",
"phase_started_at": "now",
"message": "Wird gestartet…",
},
)
conn.commit()
printer_copy = dict(printer)
file_id = item["file_id"]
override = bool(item.get("override_machine"))
spool_id = item.get("spool_id")
bed_leveling = bool(item.get("bed_leveling"))
def work():
with connect(self.database_url) as wconn:
try:
result = jobs.send_file(
wconn,
self.cfg,
file_id,
host,
True,
override,
spool_id,
prepared=False,
ace_map=ace_list,
bed_leveling=bed_leveling,
)
name = (result or {}).get("filename") or ""
# Abbruch während Upload: Status nicht überschreiben, Druck stoppen.
current = store.get_print_queue_row(wconn, item_id)
if current and (current.get("status") or "") in ("cancelled", "dismissed"):
try:
moonraker.command(host, "cancel")
except Exception:
pass
return
oid = item.get("order_item_id")
if oid:
from grotti import orders
orders.attach_order_item_to_live_run(wconn, host, name, oid)
live_run = store.find_active_run(wconn, host, name)
fields = {
"status": "running",
"message": "Datei ist auf %s. Druckstart kann etwa eine Minute dauern."
% printer_copy["name"],
"host": host,
"phase": "await_print",
"phase_started_at": "now",
}
if live_run:
fields["run_id"] = live_run["id"]
store.update_print_queue(wconn, item_id, fields)
self._event(
wconn,
host,
printer_copy["name"],
"ablauf",
"%s startet %s" % (printer_copy["name"], name),
name,
)
wconn.commit()
except GrottiError as exc:
wconn.rollback()
store.update_print_queue(
wconn,
item_id,
{"status": "error", "message": exc.message, "host": host, "phase": ""},
)
self._event(
wconn,
host,
printer_copy["name"],
"queue",
"%s Queue-Fehler: %s" % (printer_copy["name"], exc.message),
"",
)
wconn.commit()
except Exception as exc:
wconn.rollback()
store.update_print_queue(
wconn,
item_id,
{
"status": "error",
"message": str(exc) or "Start fehlgeschlagen.",
"host": host,
"phase": "",
},
)
self._event(
wconn,
host,
printer_copy["name"],
"queue",
"%s Queue-Fehler: %s" % (printer_copy["name"], exc),
"",
)
wconn.commit()
self._spawn_worker(item_id, work)
def _advance_simple(self, conn, item, printer, live):
"""Einfach-Druck: running bleiben bis Moonraker fertig/abgebrochen."""
phase = (item.get("phase") or "").lower()
state = (live.get("state") or "").lower()
if phase in ("starting", "await_print"):
if state in ("printing", "paused", "heating") or live.get("print_active"):
self._clear_terminal_hits(item["id"])
store.update_print_queue(
conn,
item["id"],
{
"phase": "print_active",
"phase_started_at": "now",
"message": "Druckt…",
},
)
return
if phase == "await_print" and state == "cancelled":
hits = self._terminal_hits(item["id"], "await_cancel")
if hits >= AWAIT_CANCEL_HITS:
self._clear_terminal_hits(item["id"])
store.update_print_queue(
conn,
item["id"],
{
"status": "error",
"message": "Druck abgebrochen.",
"phase": "",
"phase_started_at": None,
},
)
return
if phase == "await_print" and self._print_start_timed_out(item):
store.update_print_queue(
conn,
item["id"],
{
"status": "error",
"message": "Druckstart-Timeout: Drucker hat nach %d min nicht angefangen."
% (PRINT_START_TIMEOUT_S // 60),
"phase": "",
"phase_started_at": None,
},
)
return
# complete/standby vom vorherigen Auftrag: noch warten, nicht „Fertig.“
return
if phase != "print_active":
return
if state in ("printing", "paused", "heating"):
self._clear_terminal_hits(item["id"])
return
if state == "error":
store.update_print_queue(
conn,
item["id"],
{
"status": "error",
"message": (live.get("message") or "Druckerfehler").strip()[:200],
"phase": "",
"phase_started_at": None,
},
)
return
# Nur complete = fertig. cancelled/standby erst nach Entprellen.
if state == "complete":
self._clear_terminal_hits(item["id"])
store.update_print_queue(
conn,
item["id"],
{
"status": "done",
"message": "Fertig.",
"phase": "",
"phase_started_at": None,
},
)
self._event(
conn,
printer["host"],
printer["name"],
"fertig",
"%s fertig: %s" % (printer["name"], item.get("filename") or "Druck"),
item.get("filename") or "",
)
return
if state in ("cancelled", "standby"):
hits = self._terminal_hits(item["id"], "end")
if hits < TERMINAL_CONFIRM_HITS:
return
live2 = live
try:
live2 = moonraker.query(printer["host"])
except Exception:
pass
state2 = (live2.get("state") or "").lower()
if state2 in ("printing", "paused", "heating") or live2.get("print_active"):
self._clear_terminal_hits(item["id"])
return
self._clear_terminal_hits(item["id"])
if state2 == "complete":
store.update_print_queue(
conn,
item["id"],
{
"status": "done",
"message": "Fertig.",
"phase": "",
"phase_started_at": None,
},
)
self._event(
conn,
printer["host"],
printer["name"],
"fertig",
"%s fertig: %s" % (printer["name"], item.get("filename") or "Druck"),
item.get("filename") or "",
)
return
# Echter Abbruch — nicht „Fertig.“ und kein Auto-Retry.
store.update_print_queue(
conn,
item["id"],
{
"status": "cancelled",
"message": "Druck abgebrochen.",
"phase": "",
"phase_started_at": None,
},
)
self._event(
conn,
printer["host"],
printer["name"],
"abbruch",
"%s: Druck abgebrochen (%s)"
% (printer["name"], item.get("filename") or "ohne Datei"),
item.get("filename") or "",
)
def _begin_ablauf(self, conn, item, printer, live):
try:
eject.require_kobra_3(printer)
except GrottiError as exc:
self._fail_ablauf(conn, item, printer, exc.message)
return
store.update_print_queue(
conn,
item["id"],
{
"step_index": 0,
"repeat_done": 0,
"phase": "",
"phase_started_at": None,
"message": self._ablauf_message(item, "startet…"),
},
)
item = store.get_print_queue_row(conn, item["id"]) or item
steps = _parse_steps(item)
self._event(
conn,
printer["host"],
printer["name"],
"ablauf",
"%s Ablauf startet: %s (%s, %d×)" % (
printer["name"],
item.get("filename") or "",
" → ".join(eject.step_label(step) for step in steps),
max(1, int(item.get("repeat_total") or 1)),
),
item.get("filename") or "",
)
self._run_current_step(conn, item, printer, live)
def _try_clear_stuck_error(self, conn, printer, live):
"""Hängenden print_stats=error räumen (Idle, kein aktiver Druck)."""
state = (live.get("state") or "").lower()
if state != "error":
return live
host = printer["host"]
before = live.get("message") or "Fehlerzustand"
try:
moonraker.clear_print_error(host)
fresh = moonraker.query(host)
except Exception as exc:
self._event(conn, host, printer["name"], "ablauf",
"%s: hängenden Fehler nicht geräumt (%s)" % (printer["name"], exc), "")
return live
if (fresh.get("state") or "").lower() != "error":
self._event(conn, host, printer["name"], "ablauf",
"%s: hängenden Fehler geräumt („%s“)" % (printer["name"], before), "")
return fresh
def _advance_ablauf(self, conn, item, printer, live):
host = printer["host"]
state = (live.get("state") or "").lower()
if live.get("ok") is False or state in ("", "offline"):
return
steps = _parse_steps(item)
if not steps:
store.update_print_queue(conn, item["id"], {"status": "done", "message": "Fertig.", "phase": ""})
return
step_index = int(item.get("step_index") or 0)
if step_index < 0 or step_index >= len(steps):
self._finish_or_repeat(conn, item, printer, live)
return
step = steps[step_index]
kind = (step.get("type") or "").lower()
phase = (item.get("phase") or "").lower()
# Hängender Fehler zwischen Schritten / vor Start: räumen und weiter.
# Nur mitten im Druck (print_active / printing nach Start) ist error ein Abbruch.
if state == "error" and phase not in ("starting", "printing", "print_active"):
live = self._try_clear_stuck_error(conn, printer, live)
state = (live.get("state") or "").lower()
if state == "error":
self._fail_ablauf(
conn,
item,
printer,
"Drucker im Fehlerzustand — auf der Wand „Fehler löschen“, dann Ablauf neu einreihen.",
)
return
if kind == "print":
if phase in ("starting", "printing"):
# Start angenommen / Upload lief — warten auf print_active oder Timeout.
if state == "error":
self._fail_ablauf(
conn,
item,
printer,
live.get("message") or "Druckerfehler beim Start.",
)
return
if state == "cancelled":
# Anycubic/GoKlipper meldet cancelled kurz beim Start/LeviQ —
# frisch abfragen; wenn schon wieder Druck → nicht abbrechen.
try:
live = moonraker.query(printer["host"])
state = (live.get("state") or "").lower()
except Exception:
pass
if state == "cancelled":
self._fail_ablauf(conn, item, printer, "Druck abgebrochen.")
return
# Nach Grotti-Neustart mitten im Upload: Datei läuft schon → übernehmen.
want = self._step_filename(conn, step)
live_name = (live.get("filename") or "").replace("\\", "/").rsplit("/", 1)[-1]
if want and live_name and (
live_name == want or live_name.endswith(want) or want.endswith(live_name)
):
if state in ("printing", "paused", "heating"):
store.update_print_queue(
conn,
item["id"],
{
"phase": "print_active",
"message": self._ablauf_message(item, "druckt…"),
},
)
return
# Nie bei complete/standby weiter: nach Repeat zeigt Moonraker oft
# noch complete vom vorherigen Durchlauf (gleiche Datei) → sonst
# springt der Ablauf sofort auf Abkühlen, während LeviQ/Druck startet.
if state in ("printing", "paused", "heating"):
store.update_print_queue(
conn,
item["id"],
{
"phase": "print_active",
"message": self._ablauf_message(item, "druckt…"),
},
)
return
# Worker tot (typisch Grotti-Neustart): nicht als Fehler, sondern erneut starten.
# Gesamt-Timeout bleibt über phase_started_at des ersten Versuchs.
if phase == "starting" and not self._worker_busy(item["id"]):
started = item.get("phase_started_at")
age = 0
if started:
if getattr(started, "tzinfo", None) is None:
started = started.replace(tzinfo=timezone.utc)
age = (datetime.now(timezone.utc) - started).total_seconds()
if age > 45:
if self._print_start_timed_out(item):
self._fail_ablauf(
conn,
item,
printer,
"Druckstart-Timeout: Drucker hat nach %d min nicht angefangen."
% (PRINT_START_TIMEOUT_S // 60),
)
return
if state in BUSY:
return
self._step_event(
conn, item, printer,
"Start-Worker weg — Upload/Start erneut",
)
self._start_print_step(
conn, item, printer, step, keep_phase_started=True
)
return
if self._print_start_timed_out(item):
self._fail_ablauf(
conn,
item,
printer,
"Druckstart-Timeout: Drucker hat nach %d min nicht angefangen."
% (PRINT_START_TIMEOUT_S // 60),
)
return
# Fortschritt in der Meldung, damit „lädt ewig“ nicht stumm wirkt.
if phase == "starting":
started = item.get("phase_started_at")
age_m = 0
if started:
if getattr(started, "tzinfo", None) is None:
started = started.replace(tzinfo=timezone.utc)
age_m = int((datetime.now(timezone.utc) - started).total_seconds() // 60)
hint = "lädt und startet…"
if age_m >= 1:
hint = "lädt und startet… (%d min, LeviQ/Upload)" % age_m
store.update_print_queue(
conn,
item["id"],
{"message": self._ablauf_message(item, hint)},
)
return
if phase != "print_active":
# Druck noch nicht gestartet (z. B. nach Repeat)
if state in BUSY:
return
self._start_print_step(conn, item, printer, step)
return
if state == "printing" or state == "paused" or state == "heating":
store.update_print_queue(
conn,
item["id"],
{
"phase": "print_active",
"message": self._ablauf_message(item, "druckt…"),
},
)
return
if state == "cancelled":
try:
live = moonraker.query(printer["host"])
state = (live.get("state") or "").lower()
except Exception:
pass
if state in ("printing", "paused", "heating") or live.get("print_active"):
store.update_print_queue(
conn,
item["id"],
{
"phase": "print_active",
"message": self._ablauf_message(item, "druckt…"),
},
)
return
if state == "cancelled":
self._fail_ablauf(conn, item, printer, "Druck abgebrochen.")
return
if state == "error":
self._fail_ablauf(
conn,
item,
printer,
live.get("message") or "Druckerfehler während des Drucks.",
)
return
# Nur complete = Druckende. standby/cancelled entprellen (Filament-Resume).
if phase == "print_active" and state == "complete":
self._clear_terminal_hits(item["id"])
self._next_step(conn, item, printer, live)
return
if phase == "print_active" and state in ("cancelled", "standby"):
hits = self._terminal_hits(item["id"], "ablauf_end")
if hits < TERMINAL_CONFIRM_HITS:
return
try:
live = moonraker.query(printer["host"])
state = (live.get("state") or "").lower()
except Exception:
pass
if state in ("printing", "paused", "heating") or live.get("print_active"):
self._clear_terminal_hits(item["id"])
return
self._clear_terminal_hits(item["id"])
if state == "complete":
self._next_step(conn, item, printer, live)
return
if state == "cancelled":
self._fail_ablauf(conn, item, printer, "Druck abgebrochen.")
return
# standby nach Entprellen: Job wirklich zu Ende
self._next_step(conn, item, printer, live)
return
# Idle schon während des Drucks verlängern — sonst greift M84/Default zu früh.
if phase == "print_active" and _ablauf_needs_motors(item):
last = self._keepalive.get(item["id"], 0)
if time.monotonic() - last >= KEEPALIVE_S:
try:
moonraker.gcode_script(
printer["host"], eject.idle_timeout_script(), timeout=15
)
self._keepalive[item["id"]] = time.monotonic()
except Exception:
pass
return
if kind == "cool":
if state in ("printing", "paused", "heating"):
return
if phase != "cooling":
self._start_cool_step(conn, item, printer, step)
return
self._poll_cool(conn, item, printer, live, step)
return
if kind == "sweep":
if state in ("printing", "paused", "heating"):
return
if phase == "sweeping":
# Worker macht _next_step selbst. FREE hier weiterzuschalten war ein
# Race. Nach Grotti-Neustart ist der Thread weg → erneut schieben.
if self._worker_busy(item["id"]):
return
self._step_event(conn, item, printer, "Schieben erneut (Worker war weg)")
self._start_sweep_step(conn, item, printer, step)
return
self._start_sweep_step(conn, item, printer, step)
return
if kind == "check":
# Altes 3×3 (probe_count/mesh_*): aus der Kette nehmen — Timeout ohne Netz.
if store._is_legacy_mesh_check(step):
self._step_event(conn, item, printer, "altes 3×3-Prüfen übersprungen")
self._next_step(conn, item, printer, live)
return
if state == "printing":
return
if phase == "checking":
# Worker läuft noch — warten. Nach Grotti-Neustart ist der Thread weg:
# sonst hängt „tastet Mitte…“ für immer.
if self._worker_busy(item["id"]):
return
self._step_event(conn, item, printer, "Tasten erneut (Worker war weg)")
self._start_check_step(conn, item, printer, step)
return
self._start_check_step(conn, item, printer, step)
return
self._fail_ablauf(conn, item, printer, "Unbekannter Schritt: %s" % kind)
def _print_start_timed_out(self, item):
started = item.get("phase_started_at")
if not started:
return False
if getattr(started, "tzinfo", None) is None:
started = started.replace(tzinfo=timezone.utc)
age = (datetime.now(timezone.utc) - started).total_seconds()
return age > PRINT_START_TIMEOUT_S
def _step_filename(self, conn, step):
try:
file_id = int((step or {}).get("file_id") or 0)
except (TypeError, ValueError):
return ""
if not file_id:
return ""
row = store.get_file_row(conn, file_id)
return (row or {}).get("name") or ""
def _start_print_step(self, conn, item, printer, step, keep_phase_started=False):
try:
normalized = eject.normalize_print(step)
except GrottiError as exc:
self._fail_ablauf(conn, item, printer, exc.message)
return
if self._worker_busy(item["id"]):
return
# Vor dem Upload markieren — sonst Doppelstart nach Grotti-Neustart.
fields = {
"phase": "starting",
"file_id": normalized["file_id"],
"message": self._ablauf_message(item, "lädt und startet…"),
}
if not keep_phase_started:
fields["phase_started_at"] = "now"
store.update_print_queue(conn, item["id"], fields)
conn.commit()
item_id = item["id"]
printer_copy = dict(printer)
step_copy = dict(normalized)
def work():
with connect(self.database_url) as wconn:
try:
result = jobs.send_file(
wconn,
self.cfg,
step_copy["file_id"],
printer_copy["host"],
True,
bool(step_copy.get("override_machine")),
step_copy.get("spool_id"),
prepared=False,
ace_map=step_copy.get("ace_map") or [],
bed_leveling=bool(step_copy.get("bed_leveling")),
# Slice-Ende M84 → unhomed; Abräumen braucht die Motoren.
keep_motors=_ablauf_needs_motors(item),
)
name = (result or {}).get("filename") or ""
row = store.get_print_queue_row(wconn, item_id) or item
if (row.get("status") or "") in ("cancelled", "dismissed"):
try:
moonraker.command(printer_copy["host"], "cancel")
except Exception:
pass
return
fields = {
"phase": "printing",
"phase_started_at": "now",
"file_id": step_copy["file_id"],
"message": self._ablauf_message(row, "druckt %s" % name),
}
live_run = store.find_active_run(wconn, printer_copy["host"], name)
if live_run:
fields["run_id"] = live_run["id"]
store.update_print_queue(wconn, item_id, fields)
self._step_event(wconn, row, printer_copy, "druckt %s" % name)
wconn.commit()
except GrottiError as exc:
wconn.rollback()
row = store.get_print_queue_row(wconn, item_id) or item
self._fail_ablauf(wconn, row, printer_copy, exc.message)
wconn.commit()
except Exception as exc:
wconn.rollback()
row = store.get_print_queue_row(wconn, item_id) or item
self._fail_ablauf(wconn, row, printer_copy, str(exc) or "Druckstart fehlgeschlagen.")
wconn.commit()
self._spawn_worker(item_id, work)
def _try_cool_park(self, host, item_id, force=False):
"""Kopf zur Bettmitte. Merkt Erfolg/Fehler in self._cool_park.
Nach Slice-M84 oft unhomed: letzte XYZ per SET_KINEMATIC (Present), dann Park.
"""
state = self._cool_park.setdefault(item_id, {"ok": False, "tries": 0, "error": ""})
if state["ok"] and not force:
return state
if state["tries"] >= 3 and not force:
return state
state["tries"] += 1
try:
live = moonraker.query(host)
homed = (live.get("homed_axes") or "").lower()
if not all(axis in homed for axis in "xyz"):
script, _z = eject.restore_position_script(live, {"z_push": 2})
moonraker.gcode_script(host, script, timeout=15)
live = moonraker.query(host)
before_z = eject.cool_park_z_now(live)
lift = eject.cool_park_lift_mm(live)
moonraker.gcode_script(host, eject.cool_park_script(live), timeout=60)
after = moonraker.query(host)
state["before_z"] = before_z
state["lift_mm"] = lift
if not eject.cool_parked(after, before_z=before_z, lift_mm=lift):
pos = after.get("position") or []
z = float(pos[2]) if len(pos) >= 3 and isinstance(pos[2], (int, float)) else None
state["ok"] = False
state["error"] = (
"Kühl-Z-Hub fehlgeschlagen (Z %s mm nach +%.0f mm, max %.0f) — kein Schieben."
% (
("%.1f" % z) if z is not None else "?",
lift,
eject.cool_park_z_max(live),
)
)
return state
state["ok"] = True
state["error"] = ""
except GrottiError as exc:
state["ok"] = False
state["error"] = exc.message
except Exception as exc:
state["ok"] = False
state["error"] = eject.explain_printer_error(str(exc)) or str(exc)
return state
def _start_cool_step(self, conn, item, printer, step):
# Park (bis 60 s G-code) läuft im Worker, damit tick() nicht blockiert.
host = printer["host"]
cool = eject.normalize_cool(step)
item_id = item["id"]
printer_copy = dict(printer)
item_copy = dict(item)
def work():
with connect(self.database_url) as wconn:
try:
moonraker.gcode_script(host, eject.cool_script(cool), timeout=30)
try:
moonraker.gcode_script(host, eject.idle_timeout_script(cool), timeout=15)
except Exception:
pass
park = self._try_cool_park(host, item_id, force=True)
self._keepalive[item_id] = time.monotonic()
park_note = ""
if not park["ok"] and park["error"]:
park_note = " — Mitte fehlgeschlagen: %s" % park["error"]
elif park["ok"]:
park_note = " · Düse Mitte"
row = store.get_print_queue_row(wconn, item_id) or item_copy
store.update_print_queue(
wconn,
item_id,
{
"message": self._ablauf_message(row, "kühlt%s" % park_note),
"phase": "cooling",
"phase_started_at": datetime.now(timezone.utc),
},
)
wconn.commit()
except GrottiError as exc:
self._fail_ablauf(wconn, item_copy, printer_copy, "Abkühlen: %s" % exc.message)
wconn.commit()
except Exception as exc:
self._fail_ablauf(wconn, item_copy, printer_copy, "Abkühlen: %s" % exc)
wconn.commit()
store.update_print_queue(
conn,
item_id,
{
"message": self._ablauf_message(
item,
"kühlt ab (Bett ≤ %.0f °C, Düse ≤ %.0f °C)"
% (cool["bed_max"], cool["nozzle_max"]),
),
"phase": "cooling",
"phase_started_at": datetime.now(timezone.utc),
},
)
self._step_event(
conn, item, printer,
"kühlt ab bis Bett ≤ %.0f °C, Düse ≤ %.0f °C" % (cool["bed_max"], cool["nozzle_max"]),
)
self._spawn_worker(item_id, work)
def _poll_cool(self, conn, item, printer, live, step):
cool = eject.normalize_cool(step)
started = item.get("phase_started_at")
if started:
if getattr(started, "tzinfo", None) is None:
started = started.replace(tzinfo=timezone.utc)
age = (datetime.now(timezone.utc) - started).total_seconds()
if age > cool["timeout_min"] * 60:
self._cool_park.pop(item["id"], None)
self._fail_ablauf(
conn,
item,
printer,
"Abkühlen abgebrochen nach %d min." % cool["timeout_min"],
)
return
# Nachziehen zur Mitte, falls Start-Park scheiterte oder Position noch am Rand.
park = self._cool_park.get(item["id"]) or {}
if not eject.cool_parked(live, before_z=park.get("before_z"), lift_mm=park.get("lift_mm")):
park = self._try_cool_park(printer["host"], item["id"])
if park["ok"]:
# frische Live-Position kommt im nächsten Tick
pass
last = self._keepalive.get(item["id"], 0)
if time.monotonic() - last >= KEEPALIVE_S:
try:
moonraker.gcode_script(
printer["host"],
eject.idle_timeout_script(cool) + "\nG4 P10",
timeout=15,
)
self._keepalive[item["id"]] = time.monotonic()
except Exception:
pass
if eject.cool_ready(live, cool):
park = self._cool_park.get(item["id"]) or {}
parked = eject.cool_parked(
live, before_z=park.get("before_z"), lift_mm=park.get("lift_mm")
)
if not parked:
park = self._try_cool_park(printer["host"], item["id"], force=True)
after = moonraker.query(printer["host"])
parked = eject.cool_parked(
after, before_z=park.get("before_z"), lift_mm=park.get("lift_mm")
)
if not parked:
self._cool_park.pop(item["id"], None)
self._fail_ablauf(
conn,
item,
printer,
park.get("error")
or "Abgekühlt, aber Kühl-Z-Hub fehlgeschlagen — kein Schieben.",
)
return
self._keepalive.pop(item["id"], None)
self._cool_park.pop(item["id"], None)
self._step_event(
conn, item, printer,
"abgekühlt (Bett %s °C, Düse %s °C)" % (
"%.0f" % live["bed"] if isinstance(live.get("bed"), (int, float)) else "?",
"%.0f" % live["nozzle"] if isinstance(live.get("nozzle"), (int, float)) else "?",
),
)
self._next_step(conn, item, printer, live)
return
park = self._cool_park.get(item["id"]) or {}
wait = _cool_wait_text(live, cool)
parked_now = eject.cool_parked(
live, before_z=park.get("before_z"), lift_mm=park.get("lift_mm")
)
# Oberfläche (phaseText) zeigt nur das letzte „ · “-Stück — Warte-Text zuletzt.
if not parked_now and park.get("error"):
wait = "Mitte fehlgeschlagen: %s · %s" % (park["error"], wait)
elif parked_now:
wait = "Mitte · %s" % wait
store.update_print_queue(
conn,
item["id"],
{"message": self._ablauf_message(item, wait)},
)
def _start_sweep_step(self, conn, item, printer, step):
host = printer["host"]
try:
eject.require_kobra_3(printer)
sweep = eject.normalize_sweep(step)
self._ensure_position(conn, item, printer, sweep)
store.update_print_queue(
conn,
item["id"],
{
"phase": "sweeping",
"phase_started_at": "now",
"message": self._ablauf_message(item, "räumt ab…"),
},
)
self._step_event(conn, item, printer, "schiebt die Platte frei")
conn.commit()
except GrottiError as exc:
self._fail_ablauf(conn, item, printer, exc.message)
return
except Exception as exc:
self._fail_ablauf(conn, item, printer, "Schieben: %s" % exc)
return
item_id = item["id"]
printer_copy = dict(printer)
sweep_copy = dict(sweep)
def work():
with connect(self.database_url) as wconn:
try:
# 1) hoch + hinten 2) Z noch klar? 3) langsam senken 4) Bahn
moonraker.gcode_script(
host, eject.sweep_go_back_high_script(sweep_copy), timeout=60
)
eject.assert_clear_before_lower(moonraker.query(host), sweep_copy)
moonraker.gcode_script(
host, eject.sweep_lower_to_push_script(sweep_copy), timeout=60
)
eject.assert_at_push_height(moonraker.query(host), sweep_copy)
moonraker.gcode_script(host, eject.sweep_script(sweep_copy), timeout=180)
store.set_bed_needs_clear(wconn, host, False, "Ablauf Schieben")
row = store.get_print_queue_row(wconn, item_id) or item
self._step_event(wconn, row, printer_copy, "Platte freigeschoben")
self._next_step(wconn, row, printer_copy, {"state": "standby", "ok": True})
wconn.commit()
except GrottiError as exc:
wconn.rollback()
row = store.get_print_queue_row(wconn, item_id) or item
self._fail_ablauf(wconn, row, printer_copy, exc.message)
wconn.commit()
except moonraker.PrinterError as exc:
wconn.rollback()
row = store.get_print_queue_row(wconn, item_id) or item
self._fail_ablauf(wconn, row, printer_copy, eject.explain_printer_error(exc.message))
wconn.commit()
except Exception as exc:
wconn.rollback()
row = store.get_print_queue_row(wconn, item_id) or item
self._fail_ablauf(wconn, row, printer_copy, "Schieben: %s" % exc)
wconn.commit()
self._spawn_worker(item_id, work)
def _ensure_position(self, conn, item, printer, sweep, allow_low_z=False):
"""Wenn unhomed (typisch nach M84): letzte XYZ setzen, sonst weiter."""
host = printer["host"]
live = moonraker.query(host)
homed = (live.get("homed_axes") or "").lower()
if all(axis in homed for axis in "xyz"):
return
script, z = eject.restore_position_script(
live, sweep or {"z_push": 2}, allow_low_z=allow_low_z
)
moonraker.gcode_script(host, script, timeout=15)
homed = (moonraker.query(host).get("homed_axes") or "").lower()
if not all(axis in homed for axis in "xyz"):
raise GrottiError("Position nach Leerlauf/M84 ließ sich nicht wiederherstellen.", "homed")
self._step_event(
conn, item, printer,
"Motoren waren aus (M84/Idle), letzte Position übernommen (Z %.1f mm)" % z,
)
def _start_check_step(self, conn, item, printer, step):
host = printer["host"]
try:
eject.require_kobra_3(printer)
check = eject.normalize_check(step)
self._ensure_position(conn, item, printer, {"z_push": 2})
live_z = None
pos = (moonraker.query(host).get("position") or [])
if len(pos) >= 3 and isinstance(pos[2], (int, float)):
live_z = float(pos[2])
store.update_print_queue(
conn,
item["id"],
{
"phase": "checking",
"phase_started_at": "now",
"message": self._ablauf_message(item, "tastet Mitte…"),
},
)
self._step_event(conn, item, printer, "tastet Bettmitte")
conn.commit()
except GrottiError as exc:
self._fail_ablauf(conn, item, printer, exc.message)
return
except Exception as exc:
self._fail_ablauf(conn, item, printer, "Tasten: %s" % exc)
return
item_id = item["id"]
printer_copy = dict(printer)
check_copy = dict(check)
live_z_copy = live_z
def work():
with connect(self.database_url) as wconn:
try:
timeout_s = max(30, int(check_copy["timeout_min"]) * 60)
t0 = time.time()
moonraker.gcode_script(
host,
eject.check_script(check_copy, current_z=live_z_copy),
timeout=timeout_s,
)
z = moonraker.latest_probe_result_z(host, after_time=t0)
z_src = "result"
if z is None:
live = moonraker.query(host)
z, z_src = eject.check_z_from_live(live)
if z is None:
raise GrottiError(
"Nach dem Tasten fehlt Result is z= (probe-Objekt auf Kobra 3 leer).",
"probe",
)
if hasattr(eject, "check_lift_script"):
moonraker.gcode_script(host, eject.check_lift_script(check_copy), timeout=30)
limit = float(check_copy["max_delta_mm"])
if z > limit:
raise GrottiError(
"Restteil? Mitte tastet Z %.2f mm (Grenze %.1f mm)%s."
% (z, limit, " · %s" % z_src if z_src else ""),
"probe",
)
row = store.get_print_queue_row(wconn, item_id) or item
store.update_print_queue(
wconn,
item_id,
{"message": self._ablauf_message(row, "Mitte frei (Z %.2f mm)" % z)},
)
self._step_event(wconn, row, printer_copy, "Mitte frei (Z %.2f mm)" % z)
self._next_step(wconn, row, printer_copy, {"state": "standby", "ok": True})
wconn.commit()
except GrottiError as exc:
wconn.rollback()
row = store.get_print_queue_row(wconn, item_id) or item
self._fail_ablauf(wconn, row, printer_copy, exc.message)
wconn.commit()
except moonraker.PrinterError as exc:
wconn.rollback()
row = store.get_print_queue_row(wconn, item_id) or item
self._fail_ablauf(wconn, row, printer_copy, eject.explain_printer_error(exc.message))
wconn.commit()
except Exception as exc:
wconn.rollback()
row = store.get_print_queue_row(wconn, item_id) or item
self._fail_ablauf(wconn, row, printer_copy, "Tasten: %s" % exc)
wconn.commit()
self._spawn_worker(item_id, work)
def _next_step(self, conn, item, printer, live):
steps = _parse_steps(item)
nxt = int(item.get("step_index") or 0) + 1
if nxt >= len(steps):
self._finish_or_repeat(conn, item, printer, live)
return
store.update_print_queue(
conn,
item["id"],
{
"step_index": nxt,
"phase": "",
"phase_started_at": None,
"message": self._ablauf_message(
{**item, "step_index": nxt},
eject.step_label(steps[nxt]) or "nächster Schritt",
),
},
)
refreshed = store.get_print_queue_row(conn, item["id"]) or item
self._run_current_step(conn, refreshed, printer, live)
def _finish_or_repeat(self, conn, item, printer, live):
done = int(item.get("repeat_done") or 0) + 1
total = max(1, int(item.get("repeat_total") or 1))
if done < total:
self._event(
conn,
printer["host"],
printer["name"],
"ablauf",
"%s Ablauf: Durchlauf %d/%d fertig, startet %d/%d" % (printer["name"], done, total, done + 1, total),
item.get("filename") or "",
)
store.update_print_queue(
conn,
item["id"],
{
"repeat_done": done,
"step_index": 0,
"phase": "",
"phase_started_at": None,
"message": self._ablauf_message(
{**item, "repeat_done": done, "step_index": 0},
"Wiederholung %d/%d" % (done + 1, total),
),
},
)
refreshed = store.get_print_queue_row(conn, item["id"]) or item
self._run_current_step(conn, refreshed, printer, live)
return
store.update_print_queue(
conn,
item["id"],
{
"status": "done",
"repeat_done": done,
"phase": "",
"phase_started_at": None,
"message": "Ablauf fertig (%d×)." % total,
},
)
self._event(
conn,
printer["host"],
printer["name"],
"ablauf",
"%s Ablauf fertig (%d×)." % (printer["name"], total),
item.get("filename") or "",
)
def _run_current_step(self, conn, item, printer, live):
steps = _parse_steps(item)
idx = int(item.get("step_index") or 0)
if idx < 0 or idx >= len(steps):
self._finish_or_repeat(conn, item, printer, live)
return
step = steps[idx]
kind = (step.get("type") or "").lower()
state = (live.get("state") or "").lower()
if kind == "print":
if state in BUSY:
return
self._start_print_step(conn, item, printer, step)
elif kind == "cool":
self._start_cool_step(conn, item, printer, step)
elif kind == "sweep":
self._start_sweep_step(conn, item, printer, step)
elif kind == "check":
self._start_check_step(conn, item, printer, step)
else:
self._fail_ablauf(conn, item, printer, "Unbekannter Schritt: %s" % kind)
def _fail_ablauf(self, conn, item, printer, message):
if self._try_auto_retry(conn, item, message):
self._event(
conn,
printer["host"],
printer["name"],
"queue",
"%s Nochmal — %s" % (printer["name"], message),
item.get("filename") or "",
)
return
store.update_print_queue(
conn,
item["id"],
{
"status": "error",
"message": message,
"phase": "",
"phase_started_at": None,
},
)
self._event(
conn,
printer["host"],
printer["name"],
"queue",
"%s Ablauf gestoppt: %s" % (printer["name"], message),
item.get("filename") or "",
)
def _ablauf_message(self, item, suffix):
steps = _parse_steps(item)
total = max(1, int(item.get("repeat_total") or 1))
done = int(item.get("repeat_done") or 0)
idx = int(item.get("step_index") or 0)
label = eject.step_label(steps[idx]) if 0 <= idx < len(steps) else ""
if total > 1:
head = "%d/%d" % (done + 1, total)
if label:
return "%s · %s · %s" % (head, label, suffix)
return "%s · %s" % (head, suffix)
if label:
return "%s · %s" % (label, suffix)
return suffix
def _event(self, conn, host, printer, kind, message, filename):
notify.record(conn, self.cfg, host, printer, kind, message, filename, dedupe_s=30)
def _step_event(self, conn, item, printer, text):
self._event(
conn,
printer["host"],
printer["name"],
"ablauf",
"%s Ablauf %s" % (printer["name"], self._ablauf_message(item, text)),
item.get("filename") or "",
)