Grotti 3.6.0: Lauf-Identität nach Filament-Pause, robustere Queue, Projekte/Aufträge list-first und API-Polish.

This commit is contained in:
Dominic Gundel committed 2026-10-25 08:24:24 +01:00
1 parent d1eb87ba25
commit 2b1e36634a
53 files changed
+4254 -636

No files matched your search

+4 -1
View File
@@ -45,7 +45,10 @@ def _yesterday_stats(conn):
def build_text(conn, watcher_latest=None, printers=None):
stats = _yesterday_stats(conn)
waiting = len([j for j in store.list_print_queue(conn, limit=200) if j.get("status") == "waiting"])
waiting = len([
j for j in store.list_print_queue(conn, limit=200)["items"]
if j.get("status") == "waiting"
])
eta = jetzt_mod.farm_eta([
{"remain": (live or {}).get("remain") or 0}
for live in (watcher_latest or [])
+1 -1
View File
@@ -111,7 +111,7 @@ def build(conn, watcher_latest, printers=None, send_jobs=None):
"SELECT DISTINCT host FROM print_queue WHERE status = 'running'"
).fetchall()
}
queue = store.list_print_queue(conn, limit=80)
queue = store.list_print_queue(conn, limit=80)["items"]
waiting = [j for j in queue if j.get("status") == "waiting"]
errors = [j for j in queue if j.get("status") == "error"]
running_q = [j for j in queue if j.get("status") == "running"]
+520 -93
View File
@@ -5,7 +5,7 @@ import urllib.error
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
from grotti import acm, catalog, eject, library, moonraker, store, usage
from grotti import acm, catalog, eject, library, logutil, moonraker, store, usage
from grotti.cost import energy_cost
from grotti.errors import GrottiError
from grotti.parse import content_has_ace_trailer, prepare_rinkhals_gcode
@@ -39,12 +39,12 @@ def _ace_map_of(sent):
return []
def _targets_from_ace(conn, host, ace):
def _targets_from_ace(conn, host, ace, ended_at=None):
targets = []
for item in ace:
slot = int(item["ams_index"]) + 1
row = store.spool_at_slot(conn, host, slot)
if row:
if row and _spool_valid_for_end(row, ended_at):
targets.append(row)
return targets
@@ -55,6 +55,380 @@ def unix_to_dt(value):
return datetime.fromtimestamp(float(value), timezone.utc)
# Anycubic schließt bei Filament-Pause den History-Job (cancelled) und öffnet
# beim Fortsetzen eine neue Job-ID. Zusammengelegt wird nur mit Beweis:
# print_duration läuft weiter und die Lücke ist klein, oder der Watcher
# hat paused→printing gesehen. Ein Neustart mit Dauer nahe 0 bleibt getrennt.
RESUME_GAP_S = 5
RESUME_DURATION_TOL_S = 30
RESUME_FRESH_MAX_S = 60
# Abwärtskompatibel für alte Aufrufe / Docs.
RESUME_JOIN_S = RESUME_GAP_S
LIVE_DEDUCT_FALLBACK_S = 600
SYNC_LOCK_KEY = 748291053
def _as_utc(value):
if value is None:
return None
if getattr(value, "tzinfo", None) is None:
return value.replace(tzinfo=timezone.utc)
return value
def history_shell_is_empty(run_or_job):
"""History-Job ohne Ende und ohne Dauer. Nie als eigenen Lauf speichern."""
if not run_or_job:
return False
duration = float(
run_or_job.get("duration_s")
or run_or_job.get("print_duration")
or run_or_job.get("total_duration")
or 0
)
ended = run_or_job.get("ended_at") or run_or_job.get("end_time")
return duration <= 0 and not ended
def is_filament_resume(
prev_end,
new_start,
prev_duration_s=None,
new_duration_s=None,
watcher_resume=False,
gap_s=RESUME_GAP_S,
):
"""True nur mit Beweis: Dauer fortgesetzt + kleine Lücke, oder Watcher-Signal.
Aufruf mit zwei Zeitpunkten (alt): nur Lücke — gilt nicht mehr als Resume.
"""
prev_end = _as_utc(prev_end)
new_start = _as_utc(new_start)
if prev_end is None or new_start is None:
return False
gap = (new_start - prev_end).total_seconds()
if not (-30 <= gap <= max(float(gap_s), RESUME_GAP_S)):
# Watcher-Signal: Lücke darf länger sein (Pause vor dem Resume-Ende).
if not watcher_resume or gap > 24 * 3600 or gap < -30:
return False
return True
if watcher_resume:
return True
# Zwei-Argument-Aufruf (Tests/alt): ohne Dauer kein Beweis → kein Resume.
if prev_duration_s is None and new_duration_s is None:
return False
prev_d = float(prev_duration_s or 0)
new_d = float(new_duration_s or 0)
# Frischer Neustart: Dauer beginnt nahe 0 — nie zusammenlegen.
if new_d <= RESUME_FRESH_MAX_S:
return False
if new_d < max(0.0, prev_d - RESUME_DURATION_TOL_S):
return False
return True
def undo_deduction(conn, run):
"""Spulenabzug eines Laufs zurücknehmen. Abschnitte bleiben."""
if not run or not run.get("deducted"):
return False
rows = conn.execute(
"SELECT spool_id, weight_g FROM run_spools WHERE run_id = %s",
(run["id"],),
).fetchall()
for row in rows:
grams = float(row["weight_g"] or 0)
if grams and row["spool_id"]:
conn.execute(
"UPDATE spools SET remaining_g = remaining_g + %s WHERE id = %s",
(grams, row["spool_id"]),
)
conn.execute("DELETE FROM run_spools WHERE run_id = %s", (run["id"],))
conn.execute(
"UPDATE runs SET deducted = false, filament_cost = 0 WHERE id = %s",
(run["id"],),
)
return True
def fold_filament_resume(
conn,
host,
filename,
job_id,
start_dt,
print_duration_s=0,
watcher_resume=False,
):
"""Fortsetzung nach Filament-Pause auf den bestehenden Lauf legen.
Gibt die Lauf-ID zurück, deren moonraker_job_id jetzt job_id ist.
Leere Hüllen (Dauer 0, kein Ende) werden vorher übersprungen — nicht hier.
"""
pred = store.find_resume_predecessor(
conn, host, filename, job_id, start_dt, watcher_resume=watcher_resume
)
if not pred:
return None
if not is_filament_resume(
pred.get("ended_at"),
start_dt,
prev_duration_s=pred.get("duration_s"),
new_duration_s=print_duration_s,
watcher_resume=watcher_resume,
):
return None
other = conn.execute(
"""
SELECT * FROM runs
WHERE printer_host = %s AND moonraker_job_id = %s AND id <> %s
""",
(host, str(job_id), pred["id"]),
).fetchone()
store.remember_run_job(conn, host, pred.get("moonraker_job_id"), pred["id"], replace=True)
undo_deduction(conn, pred)
kept = None
if other:
undo_deduction(conn, other)
conn.execute(
"UPDATE events SET run_id = %s WHERE run_id = %s",
(pred["id"], other["id"]),
)
if not (pred.get("verdict") or "").strip() and (other.get("verdict") or "").strip():
conn.execute(
"""
UPDATE runs
SET verdict = %s, verdict_reason = %s, pieces_ok = %s, pieces_bad = %s
WHERE id = %s
""",
(
other.get("verdict") or "",
other.get("verdict_reason") or "",
other.get("pieces_ok") or 0,
other.get("pieces_bad") or 0,
pred["id"],
),
)
kept = {
"state": other.get("state") or "completed",
"ended_at": other.get("ended_at"),
"duration_s": float(other.get("duration_s") or 0),
"weight_used_g": float(other.get("weight_used_g") or 0),
"weight_source": other.get("weight_source") or "",
"filament_used_mm": float(other.get("filament_used_mm") or 0),
"filament_total_mm": float(other.get("filament_total_mm") or 0),
"progress_last": float(other.get("progress_last") or 0),
}
for table in ("run_samples", "run_segments"):
conn.execute(
"UPDATE %s SET run_id = %%s WHERE run_id = %%s" % table,
(pred["id"], other["id"]),
)
try:
conn.execute(
"""
DELETE FROM run_labels AS src
WHERE src.run_id = %s
AND EXISTS (
SELECT 1 FROM run_labels AS dst
WHERE dst.run_id = %s AND dst.object_name = src.object_name
)
""",
(other["id"], pred["id"]),
)
conn.execute(
"UPDATE run_labels SET run_id = %s WHERE run_id = %s",
(pred["id"], other["id"]),
)
except Exception:
logutil.get("grotti.jobs").warning(
"Etiketten von Lauf %s nicht übernommen.", other["id"]
)
conn.execute("DELETE FROM run_spools WHERE run_id = %s", (other["id"],))
store.move_run_jobs(conn, host, other["id"], pred["id"])
# Job-ID freigeben, sonst Unique (printer_host, moonraker_job_id) beim Pred.
conn.execute(
"UPDATE runs SET moonraker_job_id = %s WHERE id = %s",
("live:merged:%s" % other["id"], other["id"]),
)
conn.execute("DELETE FROM runs WHERE id = %s", (other["id"],))
# Der Druck läuft weiter. Das Ende kommt vom Folgejob.
conn.execute(
"""
UPDATE runs
SET moonraker_job_id = %s,
ended_at = NULL,
state = CASE
WHEN %s IN ('in_progress', 'printing', 'paused') THEN %s
WHEN state IN ('cancelled', 'interrupted', 'error') THEN 'in_progress'
ELSE state
END
WHERE id = %s
""",
(
str(job_id),
(kept or {}).get("state") or "in_progress",
(kept or {}).get("state") or "in_progress",
pred["id"],
),
)
store.remember_run_job(conn, host, job_id, pred["id"], replace=True)
store.set_printer_active_run(conn, host, pred["id"])
if kept and kept.get("ended_at"):
conn.execute(
"""
UPDATE runs SET
state = %s,
ended_at = COALESCE(%s, ended_at),
duration_s = GREATEST(COALESCE(duration_s, 0), %s),
weight_used_g = GREATEST(COALESCE(weight_used_g, 0), %s),
weight_source = CASE WHEN %s <> '' THEN %s ELSE weight_source END,
filament_used_mm = GREATEST(COALESCE(filament_used_mm, 0), %s),
filament_total_mm = CASE WHEN %s > 0 THEN %s ELSE filament_total_mm END,
progress_last = GREATEST(COALESCE(progress_last, 0), %s),
deducted = false
WHERE id = %s
""",
(
kept["state"],
kept["ended_at"],
kept["duration_s"],
kept["weight_used_g"],
kept["weight_source"],
kept["weight_source"],
kept["filament_used_mm"],
kept["filament_total_mm"],
kept["filament_total_mm"],
kept["progress_last"],
pred["id"],
),
)
elif print_duration_s:
conn.execute(
"""
UPDATE runs
SET duration_s = GREATEST(COALESCE(duration_s, 0), %s),
deducted = false
WHERE id = %s
""",
(float(print_duration_s), pred["id"]),
)
return pred["id"]
def history_in_order(items):
"""Älteste zuerst. Der abgebrochene Teil muss stehen, bevor die Fortsetzung kommt."""
return sorted(
items or [],
key=lambda job: (float(job.get("start_time") or 0), str(job.get("job_id") or "")),
)
def manual_merge_runs(conn, keep_id, absorb_id, confirm=False):
"""Zwei Läufe manuell verbinden. keep überlebt, absorb wird übernommen."""
if not confirm:
raise GrottiError("Verbinden braucht eine Bestätigung.", "bestaetigung")
keep = store.get_run_row(conn, keep_id)
absorb = store.get_run_row(conn, absorb_id)
if not keep or not absorb:
raise GrottiError("Der Lauf fehlt.", "lauf", 404)
if keep["id"] == absorb["id"]:
raise GrottiError("Gleicher Lauf.", "lauf")
if (keep.get("printer_host") or "") != (absorb.get("printer_host") or ""):
raise GrottiError("Nur Läufe desselben Druckers.", "drucker")
if (keep.get("filename") or "") != (absorb.get("filename") or ""):
raise GrottiError("Nur Läufe derselben Datei.", "datei")
# Früherer Start bleibt der überlebende Lauf.
if keep.get("started_at") and absorb.get("started_at") and absorb["started_at"] < keep["started_at"]:
keep, absorb = absorb, keep
host = keep["printer_host"]
undo_deduction(conn, keep)
undo_deduction(conn, absorb)
snapshot = {
"keep_id": keep["id"],
"absorb_id": absorb["id"],
"absorb_job_id": absorb.get("moonraker_job_id"),
}
for table in ("run_samples", "run_segments", "run_spools", "events"):
conn.execute(
"UPDATE %s SET run_id = %%s WHERE run_id = %%s" % table,
(keep["id"], absorb["id"]),
)
try:
conn.execute(
"""
DELETE FROM run_labels AS src
WHERE src.run_id = %s
AND EXISTS (
SELECT 1 FROM run_labels AS dst
WHERE dst.run_id = %s AND dst.object_name = src.object_name
)
""",
(absorb["id"], keep["id"]),
)
conn.execute(
"UPDATE run_labels SET run_id = %s WHERE run_id = %s",
(keep["id"], absorb["id"]),
)
except Exception:
pass
store.move_run_jobs(conn, host, absorb["id"], keep["id"])
job_id = absorb.get("moonraker_job_id") or keep.get("moonraker_job_id")
if job_id and not str(job_id).startswith("live:"):
store.remember_run_job(conn, host, job_id, keep["id"], replace=True)
# Job-ID am absorb freigeben, sonst Unique beim UPDATE auf keep.
conn.execute(
"UPDATE runs SET moonraker_job_id = %s WHERE id = %s",
("live:merged:%s" % absorb["id"], absorb["id"]),
)
absorb_terminal = (absorb.get("state") or "") in TERMINAL
keep_open = (keep.get("state") or "") in ("in_progress", "printing", "paused")
new_state = absorb["state"] if absorb_terminal and not keep_open else (
absorb["state"] if not keep_open else keep["state"]
)
if keep_open or (absorb.get("state") or "") in ("in_progress", "printing", "paused"):
new_state = absorb["state"] if (absorb.get("state") or "") in ("in_progress", "printing", "paused") else "in_progress"
ended = None
else:
ended = absorb.get("ended_at") or keep.get("ended_at")
conn.execute(
"""
UPDATE runs SET
moonraker_job_id = COALESCE(NULLIF(%s, ''), moonraker_job_id),
state = %s,
ended_at = %s,
duration_s = GREATEST(COALESCE(duration_s, 0), COALESCE(%s, 0)),
weight_used_g = GREATEST(COALESCE(weight_used_g, 0), COALESCE(%s, 0)),
filament_used_mm = GREATEST(COALESCE(filament_used_mm, 0), COALESCE(%s, 0)),
progress_last = GREATEST(COALESCE(progress_last, 0), COALESCE(%s, 0)),
deducted = false
WHERE id = %s
""",
(
str(job_id or ""),
new_state,
ended,
float(absorb.get("duration_s") or 0),
float(absorb.get("weight_used_g") or 0),
float(absorb.get("filament_used_mm") or 0),
float(absorb.get("progress_last") or 0),
keep["id"],
),
)
try:
conn.execute(
"UPDATE print_queue SET run_id = %s WHERE run_id = %s",
(keep["id"], absorb["id"]),
)
except Exception:
pass
conn.execute("DELETE FROM runs WHERE id = %s", (absorb["id"],))
store.set_printer_active_run(
conn, host, keep["id"] if new_state in ("in_progress", "printing", "paused") else None
)
fresh = store.get_run_row(conn, keep["id"])
return {"run": store.run_out(fresh), "undo": snapshot}
def choose_spool(conn, host, material, hinted_id, ended_at=None):
"""Spule für History-Abzug. Ohne Hint nur Rollen, die vor Druckende existierten."""
at_host = [
@@ -196,12 +570,13 @@ def apply_deduction(conn, run, energy, metadata=None):
store.close_open_segments(conn, run["id"], None, run.get("filament_used_mm") or 0)
segments = store.list_run_segments(conn, run["id"])
ended = run.get("ended_at")
entries = []
targets = _targets_from_ace(conn, run["printer_host"], ace) if ace else []
targets = _targets_from_ace(conn, run["printer_host"], ace, ended_at=ended) if ace else []
if not targets and run.get("spool_id"):
spool = store.get_spool_row(conn, run["spool_id"])
if spool:
if spool and _spool_valid_for_end(spool, ended):
targets = [spool]
# Abschnitte ohne ACE-Map: externe Rollen nacheinander
if segments and not ace:
@@ -291,8 +666,10 @@ def apply_deduction(conn, run, energy, metadata=None):
filament = sum(item["cost"] for item in entries)
samples = store.list_run_samples(conn, run["id"])
power = energy_cost(run["duration_s"], energy, samples)
# run_spools vor deducted=true, sonst bleibt die Akte leer bei Abbruch dazwischen
store.replace_run_spools(conn, run["id"], entries)
weight_source = source or (
"estimate" if any(e["is_estimate"] for e in entries) else "measured"
)
# Zuerst deducted=true — nur der Gewinner schreibt Akte und zieht ab.
marked = store.mark_deducted(
conn,
run["id"],
@@ -300,11 +677,11 @@ def apply_deduction(conn, run, energy, metadata=None):
filament,
power,
weight_used_g=used_total,
weight_source=source or ("estimate" if any(e["is_estimate"] for e in entries) else "measured"),
weight_source=weight_source,
)
if not marked:
store.replace_run_spools(conn, run["id"], [])
return False
store.replace_run_spools(conn, run["id"], entries)
for item in entries:
store.deduct_spool(conn, item["spool_id"], item["weight_g"])
store.ensure_run_chargen(conn, run["id"], used_total)
@@ -318,99 +695,149 @@ def apply_deduction(conn, run, energy, metadata=None):
return True
def sync_history(conn, cfg):
def _sync_one_job(conn, cfg, printer, job, watcher_resume=False):
"""Einen History-Job verarbeiten. Gibt 1 zurück wenn abgezogen wurde."""
job_id = str(job.get("job_id") or "")
if not job_id or str(job_id).startswith("live:"):
return 0
filename = job.get("filename") or ""
start_dt = unix_to_dt(job.get("start_time"))
end_dt = unix_to_dt(job.get("end_time"))
duration_s = float(job.get("print_duration") or job.get("total_duration") or 0)
state = job.get("status") or ""
# Leere Hülle: nie speichern. Sobald Dauer da ist, gilt die Beweis-Regel.
if history_shell_is_empty({
"duration_s": duration_s,
"ended_at": end_dt,
"print_duration": duration_s,
"end_time": end_dt,
}) and state not in TERMINAL:
return 0
if store.history_job_already_folded(conn, printer["host"], filename, job_id, end_dt):
return 0
if store.history_job_retired(conn, printer["host"], job_id):
return 0
fold_filament_resume(
conn,
printer["host"],
filename,
job_id,
start_dt,
print_duration_s=duration_s,
watcher_resume=watcher_resume,
)
metadata = job.get("metadata") or {}
planned = weight_of(metadata)
material = material_of(metadata)
used_mm = usage.filament_used_mm(job)
total_mm = usage.filament_total_mm(metadata, job)
estimated = float(metadata.get("estimated_time") or 0)
sent = store.latest_send(conn, printer["host"], filename)
known = store.file_by_name(conn, filename)
file_id = (sent or {}).get("file_id") or (known or {}).get("id")
project_id = (sent or {}).get("project_id") or (known or {}).get("project_id")
hinted = (sent or {}).get("spool_id") or (known or {}).get("spool_id")
ended_at = end_dt
if not sent and not hinted:
spool = None
else:
spool = choose_spool(conn, printer["host"], material, hinted, ended_at=ended_at)
terminal = state in TERMINAL
live = store.find_live_run(conn, printer["host"], filename)
progress = float((live or {}).get("progress_last") or 0)
if used_mm <= 0 and live:
used_mm = float(live.get("filament_used_mm") or 0)
if total_mm <= 0 and live:
total_mm = float(live.get("filament_total_mm") or 0)
if planned <= 0 and live:
planned = float(live.get("weight_planned_g") or 0)
objects = (live or {}).get("objects") or []
if isinstance(objects, str):
try:
objects = json.loads(objects)
except json.JSONDecodeError:
objects = []
used_g, source = usage.resolve_used_grams(
planned, used_mm, total_mm, progress, state, duration_s
)
row = store.upsert_run(conn, {
"printer_host": printer["host"],
"printer_name": printer["name"],
"filename": filename,
"file_id": file_id,
"project_id": project_id,
"spool_id": spool["id"] if spool else None,
"state": state,
"started_at": start_dt,
"ended_at": ended_at,
"duration_s": duration_s,
"weight_g": planned,
"weight_planned_g": planned,
"weight_used_g": used_g,
"weight_source": source,
"filament_used_mm": used_mm,
"filament_total_mm": total_mm,
"progress_last": progress,
"estimated_s": estimated,
"energy_cost": energy_cost(duration_s, cfg["energy"]),
"moonraker_job_id": job_id,
"needs_spool": bool(terminal and used_g > 0 and not spool),
"material_type": material,
"message": (job.get("message") or "") if state == "error" else "",
"objects": objects,
})
store.adopt_live_run(conn, printer["host"], filename, row["id"])
row = store.get_run_row(conn, row["id"])
if not terminal:
store.set_printer_active_run(conn, printer["host"], row["id"])
else:
active = store.find_live_run(conn, printer["host"])
if not active:
ended = row.get("ended_at") or ended_at
store.ensure_bed_after_print(
conn, printer["host"], ended_at=ended, message=filename or state
)
store.set_printer_active_run(conn, printer["host"], None)
if apply_deduction(conn, row, cfg["energy"], metadata=metadata):
return 1
return 0
def deduct_stale_live_runs(conn, cfg, older_than_s=LIVE_DEDUCT_FALLBACK_S):
"""Live-Läufe ohne History nach Frist abziehen (Fertig-Pfad ohne History)."""
rows = store.list_stale_live_runs(conn, older_than_s=older_than_s)
n = 0
for row in rows:
if apply_deduction(conn, row, cfg["energy"]):
n += 1
return n
def sync_history(conn, cfg, host=None, watcher_resume=False):
"""History abgleichen. Serialisiert über Session-Advisory-Lock."""
printers = store.list_printers(conn)
if host:
printers = [p for p in printers if p.get("host") == host]
def fetch(printer):
return printer, moonraker.history(printer["host"])
with ThreadPoolExecutor(max_workers=max(1, len(printers))) as pool:
fetched = list(pool.map(fetch, printers))
with ThreadPoolExecutor(max_workers=max(1, len(printers) or 1)) as pool:
fetched = list(pool.map(fetch, printers)) if printers else []
collected = []
for printer, items in fetched:
for job in items:
for job in history_in_order(items):
collected.append((printer, job))
deducted = 0
for printer, job in collected:
job_id = str(job.get("job_id") or "")
if not job_id or str(job_id).startswith("live:"):
continue
filename = job.get("filename") or ""
metadata = job.get("metadata") or {}
planned = weight_of(metadata)
material = material_of(metadata)
state = job.get("status") or ""
used_mm = usage.filament_used_mm(job)
total_mm = usage.filament_total_mm(metadata, job)
estimated = float(metadata.get("estimated_time") or 0)
sent = store.latest_send(conn, printer["host"], filename)
known = store.file_by_name(conn, filename)
file_id = (sent or {}).get("file_id") or (known or {}).get("id")
project_id = (sent or {}).get("project_id") or (known or {}).get("project_id")
hinted = (sent or {}).get("spool_id") or (known or {}).get("spool_id")
ended_at = unix_to_dt(job.get("end_time"))
# Ohne Send-Eintrag keine Rate — sonst zieht eine neue Spule alle alten Jobs ab.
if not sent and not hinted:
spool = None
else:
spool = choose_spool(conn, printer["host"], material, hinted, ended_at=ended_at)
terminal = state in TERMINAL
live = store.find_live_run(conn, printer["host"], filename)
progress = float((live or {}).get("progress_last") or 0)
if used_mm <= 0 and live:
used_mm = float(live.get("filament_used_mm") or 0)
if total_mm <= 0 and live:
total_mm = float(live.get("filament_total_mm") or 0)
if planned <= 0 and live:
planned = float(live.get("weight_planned_g") or 0)
objects = (live or {}).get("objects") or []
if isinstance(objects, str):
try:
objects = json.loads(objects)
except json.JSONDecodeError:
objects = []
used_g, source = usage.resolve_used_grams(
planned, used_mm, total_mm, progress, state, job.get("print_duration") or 0
)
row = store.upsert_run(conn, {
"printer_host": printer["host"],
"printer_name": printer["name"],
"filename": filename,
"file_id": file_id,
"project_id": project_id,
"spool_id": spool["id"] if spool else None,
"state": state,
"started_at": unix_to_dt(job.get("start_time")),
"ended_at": unix_to_dt(job.get("end_time")),
"duration_s": float(job.get("print_duration") or job.get("total_duration") or 0),
"weight_g": planned,
"weight_planned_g": planned,
"weight_used_g": used_g,
"weight_source": source,
"filament_used_mm": used_mm,
"filament_total_mm": total_mm,
"progress_last": progress,
"estimated_s": estimated,
"energy_cost": energy_cost(job.get("print_duration") or job.get("total_duration") or 0, cfg["energy"]),
"moonraker_job_id": job_id,
"needs_spool": bool(terminal and used_g > 0 and not spool),
"material_type": material,
"message": (job.get("message") or "") if state == "error" else "",
"objects": objects,
})
# Live sofort auf History hängen — auch solange noch in_progress (eine Protokoll-Zeile)
store.adopt_live_run(conn, printer["host"], filename, row["id"])
row = store.get_run_row(conn, row["id"])
if terminal:
# Nachzug nicht während eines neuen Live-Drucks — sonst „Platte belegt“ zu Unrecht
active = store.find_live_run(conn, printer["host"])
if not active:
ended = row.get("ended_at") or unix_to_dt(job.get("end_time"))
store.ensure_bed_after_print(
conn, printer["host"], ended_at=ended, message=filename or state
)
if apply_deduction(conn, row, cfg["energy"], metadata=metadata):
deducted += 1
conn.execute("SELECT pg_advisory_lock(%s)", (SYNC_LOCK_KEY,))
try:
for printer, job in collected:
deducted += _sync_one_job(
conn, cfg, printer, job, watcher_resume=watcher_resume
)
deducted += deduct_stale_live_runs(conn, cfg)
finally:
conn.execute("SELECT pg_advisory_unlock(%s)", (SYNC_LOCK_KEY,))
return {"gelesen": len(collected), "abgezogen": deducted}
+16
View File
@@ -0,0 +1,16 @@
-- Jede Moonraker-Job-ID bleibt einem Lauf zugeordnet, auch nach dem Zusammenlegen.
CREATE TABLE IF NOT EXISTS run_jobs (
printer_host text NOT NULL,
job_id text NOT NULL,
run_id bigint NOT NULL REFERENCES runs (id) ON DELETE CASCADE,
PRIMARY KEY (printer_host, job_id)
);
CREATE INDEX IF NOT EXISTS run_jobs_run ON run_jobs (run_id);
INSERT INTO run_jobs (printer_host, job_id, run_id)
SELECT printer_host, moonraker_job_id, id
FROM runs
WHERE moonraker_job_id <> ''
AND moonraker_job_id NOT LIKE 'live:%'
ON CONFLICT DO NOTHING;
+6
View File
@@ -0,0 +1,6 @@
-- Last active run per printer (survives Grotti restart during pause)
ALTER TABLE printers
ADD COLUMN IF NOT EXISTS active_run_id bigint REFERENCES runs (id) ON DELETE SET NULL;
CREATE INDEX IF NOT EXISTS printers_active_run ON printers (active_run_id)
WHERE active_run_id IS NOT NULL;
+10
View File
@@ -0,0 +1,10 @@
-- Queue item points at the run it started. At most one running job per host.
ALTER TABLE print_queue
ADD COLUMN IF NOT EXISTS run_id bigint REFERENCES runs (id) ON DELETE SET NULL;
CREATE INDEX IF NOT EXISTS print_queue_run ON print_queue (run_id)
WHERE run_id IS NOT NULL;
CREATE UNIQUE INDEX IF NOT EXISTS print_queue_one_running_per_host
ON print_queue (host)
WHERE status = 'running' AND host <> '';
+174 -30
View File
@@ -34,6 +34,9 @@ PRINT_START_TIMEOUT_S = 600
# Nach erfolgreichem Senden ist die Phase await_print — dort gilt das nicht,
# sonst wird ein noch anlaufender Druck als „Neustart“ neu eingereiht.
SIMPLE_START_STUCK_S = 45
# Anycubic meldet kurz cancelled/standby bei LeviQ und Filament-Resume.
TERMINAL_CONFIRM_HITS = 3
AWAIT_CANCEL_HITS = 3
def _ablauf_needs_motors(item):
@@ -151,10 +154,20 @@ class PrintQueueDispatcher:
def _auto_retry_enabled(self):
return bool((self.cfg or {}).get("auto_retry", True))
def _retryable_message(self, message):
text = (message or "").lower()
if "abgebrochen" in text:
return False
if "druck abgebrochen" in text:
return False
return True
def _try_auto_retry(self, conn, item, message):
"""Ein automatischer Neuversuch; sonst False."""
"""Ein automatischer Neuversuch; sonst False. Nicht nach Abbruch."""
if not self._auto_retry_enabled():
return False
if not self._retryable_message(message):
return False
retries = int(item.get("retry_count") or 0)
if retries >= 1:
return False
@@ -172,6 +185,23 @@ class PrintQueueDispatcher:
)
return True
def _terminal_hits(self, item_id, kind="end"):
key = "%s:%s" % (item_id, kind)
if not hasattr(self, "_term_hits"):
self._term_hits = {}
self._term_hits[key] = self._term_hits.get(key, 0) + 1
return self._term_hits[key]
def _clear_terminal_hits(self, item_id, kind=None):
if not hasattr(self, "_term_hits"):
return
if kind:
self._term_hits.pop("%s:%s" % (item_id, kind), None)
return
for key in list(self._term_hits):
if key.startswith("%s:" % item_id):
self._term_hits.pop(key, None)
def _fail_stuck_simple_start(self, conn, item, printer):
host = printer["host"]
msg = "Start abgebrochen (Neustart)"
@@ -277,6 +307,13 @@ class PrintQueueDispatcher:
lives[host] = live
state = (live.get("state") or "").lower()
if state in BUSY:
# Laufenden Einfach-Druck mitziehen. Sonst bleibt await_print
# stehen, bis der Drucker frei ist — und der Start gilt als Timeout,
# obwohl er längst druckt.
running = store.running_print_on_host(conn, host)
if running and not _has_ablauf(running):
self._advance_simple(conn, running, printer, live)
conn.commit()
continue
if state not in FREE:
continue
@@ -418,26 +455,31 @@ class PrintQueueDispatcher:
bed_leveling=bed_leveling,
)
name = (result or {}).get("filename") or ""
# Abbruch während Upload: Status nicht überschreiben, Druck stoppen.
current = store.get_print_queue_row(wconn, item_id)
if current and (current.get("status") or "") in ("cancelled", "dismissed"):
try:
moonraker.command(host, "cancel")
except Exception:
pass
return
oid = item.get("order_item_id")
if oid:
from grotti import orders
orders.attach_order_item_to_live_run(wconn, host, name, oid)
# await_print: Datei ist raus, Drucker meldet printing oft erst
# nach einer Minute. Nicht bei starting lassen — sonst gilt nach
# 45 s der Neustart-Retry und der zweite Start trifft „druckt schon“.
store.update_print_queue(
wconn,
item_id,
{
"status": "running",
"message": "Datei ist auf %s. Druckstart kann etwa eine Minute dauern."
% printer_copy["name"],
"host": host,
"phase": "await_print",
"phase_started_at": "now",
},
)
live_run = store.find_active_run(wconn, host, name)
fields = {
"status": "running",
"message": "Datei ist auf %s. Druckstart kann etwa eine Minute dauern."
% printer_copy["name"],
"host": host,
"phase": "await_print",
"phase_started_at": "now",
}
if live_run:
fields["run_id"] = live_run["id"]
store.update_print_queue(wconn, item_id, fields)
self._event(
wconn,
host,
@@ -493,6 +535,7 @@ class PrintQueueDispatcher:
state = (live.get("state") or "").lower()
if phase in ("starting", "await_print"):
if state in ("printing", "paused", "heating") or live.get("print_active"):
self._clear_terminal_hits(item["id"])
store.update_print_queue(
conn,
item["id"],
@@ -503,6 +546,21 @@ class PrintQueueDispatcher:
},
)
return
if phase == "await_print" and state == "cancelled":
hits = self._terminal_hits(item["id"], "await_cancel")
if hits >= AWAIT_CANCEL_HITS:
self._clear_terminal_hits(item["id"])
store.update_print_queue(
conn,
item["id"],
{
"status": "error",
"message": "Druck abgebrochen.",
"phase": "",
"phase_started_at": None,
},
)
return
if phase == "await_print" and self._print_start_timed_out(item):
store.update_print_queue(
conn,
@@ -521,6 +579,7 @@ class PrintQueueDispatcher:
if phase != "print_active":
return
if state in ("printing", "paused", "heating"):
self._clear_terminal_hits(item["id"])
return
if state == "error":
store.update_print_queue(
@@ -534,7 +593,9 @@ class PrintQueueDispatcher:
},
)
return
if state in FREE or state in ("complete", "cancelled"):
# Nur complete = fertig. cancelled/standby erst nach Entprellen.
if state == "complete":
self._clear_terminal_hits(item["id"])
store.update_print_queue(
conn,
item["id"],
@@ -553,6 +614,61 @@ class PrintQueueDispatcher:
"%s fertig: %s" % (printer["name"], item.get("filename") or "Druck"),
item.get("filename") or "",
)
return
if state in ("cancelled", "standby"):
hits = self._terminal_hits(item["id"], "end")
if hits < TERMINAL_CONFIRM_HITS:
return
live2 = live
try:
live2 = moonraker.query(printer["host"])
except Exception:
pass
state2 = (live2.get("state") or "").lower()
if state2 in ("printing", "paused", "heating") or live2.get("print_active"):
self._clear_terminal_hits(item["id"])
return
self._clear_terminal_hits(item["id"])
if state2 == "complete":
store.update_print_queue(
conn,
item["id"],
{
"status": "done",
"message": "Fertig.",
"phase": "",
"phase_started_at": None,
},
)
self._event(
conn,
printer["host"],
printer["name"],
"fertig",
"%s fertig: %s" % (printer["name"], item.get("filename") or "Druck"),
item.get("filename") or "",
)
return
# Echter Abbruch — nicht „Fertig.“ und kein Auto-Retry.
store.update_print_queue(
conn,
item["id"],
{
"status": "cancelled",
"message": "Druck abgebrochen.",
"phase": "",
"phase_started_at": None,
},
)
self._event(
conn,
printer["host"],
printer["name"],
"abbruch",
"%s: Druck abgebrochen (%s)"
% (printer["name"], item.get("filename") or "ohne Datei"),
item.get("filename") or "",
)
def _begin_ablauf(self, conn, item, printer, live):
try:
@@ -790,9 +906,31 @@ class PrintQueueDispatcher:
live.get("message") or "Druckerfehler während des Drucks.",
)
return
# Erst weiter, wenn der Job wirklich gelaufen ist — sonst Race
# (noch standby/complete vom vorherigen Auftrag).
if phase == "print_active" and state in ("complete", "standby"):
# Nur complete = Druckende. standby/cancelled entprellen (Filament-Resume).
if phase == "print_active" and state == "complete":
self._clear_terminal_hits(item["id"])
self._next_step(conn, item, printer, live)
return
if phase == "print_active" and state in ("cancelled", "standby"):
hits = self._terminal_hits(item["id"], "ablauf_end")
if hits < TERMINAL_CONFIRM_HITS:
return
try:
live = moonraker.query(printer["host"])
state = (live.get("state") or "").lower()
except Exception:
pass
if state in ("printing", "paused", "heating") or live.get("print_active"):
self._clear_terminal_hits(item["id"])
return
self._clear_terminal_hits(item["id"])
if state == "complete":
self._next_step(conn, item, printer, live)
return
if state == "cancelled":
self._fail_ablauf(conn, item, printer, "Druck abgebrochen.")
return
# standby nach Entprellen: Job wirklich zu Ende
self._next_step(conn, item, printer, live)
return
# Idle schon während des Drucks verlängern — sonst greift M84/Default zu früh.
@@ -912,16 +1050,22 @@ class PrintQueueDispatcher:
)
name = (result or {}).get("filename") or ""
row = store.get_print_queue_row(wconn, item_id) or item
store.update_print_queue(
wconn,
item_id,
{
"phase": "printing",
"phase_started_at": "now",
"file_id": step_copy["file_id"],
"message": self._ablauf_message(row, "druckt %s" % name),
},
)
if (row.get("status") or "") in ("cancelled", "dismissed"):
try:
moonraker.command(printer_copy["host"], "cancel")
except Exception:
pass
return
fields = {
"phase": "printing",
"phase_started_at": "now",
"file_id": step_copy["file_id"],
"message": self._ablauf_message(row, "druckt %s" % name),
}
live_run = store.find_active_run(wconn, printer_copy["host"], name)
if live_run:
fields["run_id"] = live_run["id"]
store.update_print_queue(wconn, item_id, fields)
self._step_event(wconn, row, printer_copy, "druckt %s" % name)
wconn.commit()
except GrottiError as exc:
+805 -57
View File
File diff suppressed because it is too large. Load diff
+1 -1
View File
@@ -1,3 +1,3 @@
"""Version von Grotti (SemVer). Bei jeder Änderung CHANGELOG.md mitführen."""
__version__ = "3.5.2"
__version__ = "3.6.0"
+149 -70
View File
@@ -16,6 +16,21 @@ SAMPLE_EVERY_S = 30
SAMPLE_HEAT_S = 10
def contact_gap_keeps_run(prev_filename, next_filename, next_state):
"""Nach einem Kontaktverlust bleibt der Lauf, solange kein anderer Auftrag läuft.
Ein Timeout ist kein Druckende. Ohne Dateiname gibt es keinen Beweis
für einen Wechsel, der offene Lauf bleibt.
"""
if next_state not in ("printing", "paused"):
return False
prev_filename = prev_filename or ""
next_filename = next_filename or ""
if not prev_filename or not next_filename:
return True
return prev_filename == next_filename
class StateWatcher:
def __init__(self, database_url, cfg, interval=10):
self.database_url = database_url
@@ -23,6 +38,7 @@ class StateWatcher:
self.interval = max(5, int(interval))
self._prev = {}
self._offline_hits = {}
self._cancel_hits = {}
self.latest = []
self._lock = threading.Lock()
self._updated_at = 0.0
@@ -95,40 +111,35 @@ class StateWatcher:
if offline:
hits = self._offline_hits.get(host, 0) + 1
self._offline_hits[host] = hits
if hits >= 2 and not prev_offline:
# Ein Aussetzer ist noch kein Offline. Erst der zweite Poll.
if hits < 2:
return
if not prev_offline:
self._emit(conn, host, name, "offline", "%s ist offline." % name, filename)
# Nach Crash/Shutdown (503/gklib) bleibt sonst „druckt“ im Protokoll.
if hits >= 2:
dangling = store.find_live_run(conn, host)
if dangling:
closed = store.complete_open_run(
conn, dangling["id"], state="interrupted"
)
if closed:
label = dangling.get("filename") or filename or "offline"
self._emit(
conn,
host,
name,
"abbruch",
"%s: Druck unterbrochen (Drucker offline)." % name,
label,
closed["id"],
)
try:
from grotti import jobs
jobs.apply_deduction(conn, closed, self.cfg["energy"])
except Exception as exc:
log.warning("Watcher Abzug offline %s: %s", host, exc)
self._prev[host] = {"state": "offline", "offline": True, "filename": filename}
# Kontakt weg ist kein Druckende. Der letzte echte Zustand bleibt
# gemerkt, damit fertig/Abbruch/Fehler beim nächsten Poll greifen.
remembered = prev_state if prev_state and prev_state != "offline" else ""
self._prev[host] = {
"state": remembered,
"offline": True,
"heating": bool(prev.get("heating")),
"filename": prev.get("filename") or filename or "",
"spool_warn": bool(prev.get("spool_warn")) if remembered == "printing" else False,
"run_id": prev.get("run_id"),
}
return
self._offline_hits[host] = 0
file_changed = False
if prev_offline and state in ("printing", "paused"):
file_changed = self._release_replaced_run(conn, host, name, filename)
starting = (
bool(prev_state)
and prev_state not in ("printing", "paused")
and state == "printing"
(
bool(prev_state)
and prev_state not in ("printing", "paused")
and state == "printing"
)
or (file_changed and state == "printing")
)
if starting:
# Aufheiz-Proben behalten: Live-Lauf gleicher Datei nicht verwerfen
@@ -154,9 +165,21 @@ class StateWatcher:
force = starting or (heating and not prev.get("heating"))
run = self._track_print(conn, host, name, filename, live, state, force_sample=force)
run_id = run["id"] if run else None
if run_id:
store.set_printer_active_run(conn, host, run_id)
if state not in ("cancelled", "standby"):
self._cancel_hits[host] = 0
if prev_offline:
self._emit(conn, host, name, "offline", "%s ist wieder erreichbar." % name, filename, run_id)
if file_changed and state == "printing":
self._emit(
conn, host, name, "start",
"%s druckt %s" % (name, filename or "einen Auftrag"),
filename, run_id,
)
self._ensure_start_segment(conn, host, filename, live, run_id)
elif starting:
self._emit(
conn, host, name, "start",
@@ -177,9 +200,16 @@ class StateWatcher:
self._emit(conn, host, name, "start", "%s pausiert." % name, filename, run_id)
elif prev_state == "paused" and state == "printing":
self._emit(conn, host, name, "start", "%s druckt weiter." % name, filename, run_id)
try:
jobs.sync_history(conn, self.cfg, host=host, watcher_resume=True)
except Exception as exc:
log.warning("Watcher Abgleich Resume %s: %s", host, exc)
# Nach Fold: aktiven History-Lauf weiter tracken.
open_run = store.find_active_run(conn, host, filename)
if open_run:
run_id = open_run["id"]
store.set_printer_active_run(conn, host, run_id)
elif prev_state == "printing" and state == "complete":
# Live-Übergang: immer belegen (ensure_bed_after_print ohne Endzeit
# würde nach Quittierung eines älteren Drucks stehen bleiben).
store.set_bed_needs_clear(conn, host, True, filename or "fertig")
self._emit(
conn,
@@ -191,53 +221,72 @@ class StateWatcher:
run_id or (prev.get("run_id")),
)
try:
from grotti import jobs
jobs.sync_history(conn, self.cfg)
jobs.sync_history(conn, self.cfg, host=host)
except Exception as exc:
log.warning("Watcher Abgleich %s: %s", host, exc)
# History oft ohne Endzeit (interrupted) — Live trotzdem schließen.
# Live schließen ohne Abzug — History bucht; Fallback nach Frist.
dangling = store.find_live_run(conn, host)
if dangling:
closed = store.complete_open_run(conn, dangling["id"], state="completed")
if closed:
try:
from grotti import jobs
jobs.apply_deduction(conn, closed, self.cfg["energy"])
except Exception as exc:
log.warning("Watcher Abzug %s: %s", host, exc)
store.complete_open_run(conn, dangling["id"], state="completed")
store.set_printer_active_run(conn, host, None)
elif state in ("cancelled", "standby"):
# Live-Lauf schließen: normaler Abbruch, oder nach Offline/Neustart
# (prev wird „offline“, der Übergang paused→standby geht sonst verloren).
dangling = store.find_live_run(conn, host)
from_active = prev_state in ("printing", "paused")
recover = bool(dangling) and (prev_offline or not prev_state)
if from_active or recover:
hits = self._cancel_hits.get(host, 0) + 1
self._cancel_hits[host] = hits
# Anycubic flackert kurz cancelled/standby während Pause/Resume —
# erst nach 3 Polls aufgeben. Nach Offline-Lücke sofort.
need_hits = 1 if recover else 3
if (from_active or recover) and hits >= need_hits:
file_label = filename or (dangling or {}).get("filename") or state
store.set_bed_needs_clear(conn, host, True, file_label)
store.abandon_live_runs(conn, host)
self._emit(
conn,
host,
name,
"abbruch",
"%s: Druck abgebrochen (%s)" % (
name, file_label if file_label != state else "ohne Datei",
),
file_label if file_label != state else "",
run_id or prev.get("run_id") or (dangling or {}).get("id"),
)
try:
from grotti import jobs
jobs.sync_history(conn, self.cfg)
jobs.sync_history(conn, self.cfg, host=host)
except Exception as exc:
log.warning("Watcher Abgleich %s: %s", host, exc)
# Nach Sync evtl. schon Fortsetzung — dann nicht aufgeben.
still = store.find_live_run(conn, host) or store.find_open_history_run(
conn, host, filename or (dangling or {}).get("filename") or ""
)
if still and (still.get("state") or "") in ("in_progress", "printing", "paused"):
self._cancel_hits[host] = 0
run_id = still["id"]
store.set_printer_active_run(conn, host, run_id)
else:
store.set_bed_needs_clear(conn, host, True, file_label)
store.abandon_live_runs(conn, host)
store.set_printer_active_run(conn, host, None)
self._emit(
conn,
host,
name,
"abbruch",
"%s: Druck abgebrochen (%s)" % (
name, file_label if file_label != state else "ohne Datei",
),
file_label if file_label != state else "",
run_id or prev.get("run_id") or (dangling or {}).get("id"),
)
# prev-Zustand behalten, bis die Hits reichen.
if hits < 3 and from_active:
self._prev[host] = {
"state": prev_state,
"offline": False,
"heating": heating,
"filename": prev.get("filename") or filename,
"spool_warn": bool(prev.get("spool_warn")),
"run_id": run_id or prev.get("run_id"),
}
return
elif state == "error" and prev_state != "error":
if prev_state == "printing" or prev_state == "paused":
store.set_bed_needs_clear(conn, host, True, filename or "Fehler")
try:
jobs.sync_history(conn, self.cfg, host=host)
except Exception as exc:
log.warning("Watcher Abgleich %s: %s", host, exc)
store.abandon_live_runs(conn, host)
store.set_printer_active_run(conn, host, None)
msg = (live.get("message") or "").strip()
if not msg or msg.lower() in ("error", "fehler"):
msg = "Ursache unbekannt — in Mainsail prüfen"
@@ -251,18 +300,10 @@ class StateWatcher:
filename,
run_id or prev.get("run_id"),
)
try:
from grotti import jobs
jobs.sync_history(conn, self.cfg)
except Exception as exc:
log.warning("Watcher Abgleich %s: %s", host, exc)
elif state == "error":
if (live.get("nozzle_target") or 0) > 0 or (live.get("bed_target") or 0) > 0:
self._cool_heaters(host)
elif state == "complete" and (not prev_state or prev_offline):
# Erster Blick nach Neustart: nur mit Endzeit des letzten Laufs,
# sonst wird bereits Quittiertes wieder „Platte belegt“.
ended = conn.execute(
"""
SELECT ended_at FROM runs
@@ -280,6 +321,10 @@ class StateWatcher:
ended_at=ended["ended_at"] if ended else None,
message=filename or "fertig",
)
try:
jobs.sync_history(conn, self.cfg, host=host)
except Exception as exc:
log.warning("Watcher Abgleich %s: %s", host, exc)
spool_warn = bool(prev.get("spool_warn"))
if state == "printing" and not spool_warn:
@@ -294,6 +339,40 @@ class StateWatcher:
"run_id": run_id or (prev.get("run_id") if state in ("printing", "paused") or heating else None),
}
def _release_replaced_run(self, conn, host, name, filename):
"""Anderer Auftrag nach der Lücke: den offenen Lauf loslassen.
History zuerst, damit Proben am echten Job hängen. Bleibt der alte
Live-Lauf, wird er beendet ohne Abzug — der History-Abgleich bucht
das Filament, sobald der Job da ist.
"""
existing = store.find_live_run(conn, host)
if not existing:
return False
if contact_gap_keeps_run(existing.get("filename") or "", filename, "printing"):
return False
try:
jobs.sync_history(conn, self.cfg)
except Exception as exc:
log.warning("Watcher Abgleich nach Offline %s: %s", host, exc)
existing = store.find_live_run(conn, host)
if not existing:
return True
if contact_gap_keeps_run(existing.get("filename") or "", filename, "printing"):
return False
label = existing.get("filename") or "ohne Datei"
store.abandon_live_runs(conn, host)
self._emit(
conn,
host,
name,
"abbruch",
"%s: Druck unterbrochen (%s)" % (name, label),
label,
existing["id"],
)
return True
def _should_sample(self, conn, host, live, state):
"""Proben bei Druck/Pause und beim Aufheizen mit Bezug zu einem Auftrag."""
if state in ("printing", "paused"):