Added batch process and test
This commit is contained in:
@@ -54,7 +54,7 @@ def save(form: dict) -> None:
|
||||
"sftp_host", "sftp_port", "sftp_user", "sftp_auth_method",
|
||||
"sftp_password", "sftp_remote_path",
|
||||
"calibre_url", "calibre_user", "calibre_pass",
|
||||
"local_work_dir", "scheduler_interval_minutes",
|
||||
"local_work_dir", "scheduler_interval_minutes", "sync_batch_size",
|
||||
]
|
||||
for key in keys:
|
||||
if key in form and form[key] is not None:
|
||||
|
||||
@@ -60,6 +60,7 @@ async def dashboard(request: Request):
|
||||
runs = [dict(r) for r in db.get_recent_runs(10)]
|
||||
zips = [dict(z) for z in db.get_recent_zips(20)]
|
||||
interval = int(db.get_setting("scheduler_interval_minutes", "0") or "0")
|
||||
batch_size = int(db.get_setting("sync_batch_size", "0") or "0")
|
||||
return templates.TemplateResponse(
|
||||
"index.html",
|
||||
{
|
||||
@@ -70,6 +71,7 @@ async def dashboard(request: Request):
|
||||
"sync_running": sync.is_running(),
|
||||
"next_run": next_run_time(),
|
||||
"interval": interval,
|
||||
"batch_size": batch_size,
|
||||
},
|
||||
)
|
||||
|
||||
@@ -116,6 +118,7 @@ async def save_settings(
|
||||
calibre_pass: str = Form(""),
|
||||
local_work_dir: str = Form("/tmp/calibresync"),
|
||||
scheduler_interval_minutes: str = Form("0"),
|
||||
sync_batch_size: str = Form("0"),
|
||||
):
|
||||
config.save({
|
||||
"sftp_host": sftp_host,
|
||||
@@ -130,12 +133,13 @@ async def save_settings(
|
||||
"calibre_pass": calibre_pass,
|
||||
"local_work_dir": local_work_dir,
|
||||
"scheduler_interval_minutes": scheduler_interval_minutes,
|
||||
"sync_batch_size": sync_batch_size,
|
||||
})
|
||||
_reschedule_auto_sync()
|
||||
return RedirectResponse("/settings?saved=1", status_code=303)
|
||||
|
||||
|
||||
# --- Sync trigger ---
|
||||
# --- Sync triggers ---
|
||||
|
||||
@app.post("/sync")
|
||||
async def trigger_sync(background_tasks: BackgroundTasks):
|
||||
@@ -145,6 +149,14 @@ async def trigger_sync(background_tasks: BackgroundTasks):
|
||||
return RedirectResponse("/?started=1", status_code=303)
|
||||
|
||||
|
||||
@app.post("/sync/test")
|
||||
async def trigger_test_sync(background_tasks: BackgroundTasks):
|
||||
if sync.is_running():
|
||||
return RedirectResponse("/?already_running=1", status_code=303)
|
||||
background_tasks.add_task(sync.run_sync, 1)
|
||||
return RedirectResponse("/?test_started=1", status_code=303)
|
||||
|
||||
|
||||
# --- JSON status API ---
|
||||
|
||||
@app.get("/api/status")
|
||||
|
||||
@@ -150,6 +150,8 @@ tr:hover td { background: rgba(255,255,255,0.02); }
|
||||
}
|
||||
.btn-primary { background: var(--accent); color: #fff; }
|
||||
.btn-primary:hover { background: var(--accent-hover); }
|
||||
.btn-secondary { background: transparent; color: var(--accent); border: 1px solid var(--accent); }
|
||||
.btn-secondary:hover { background: rgba(99,102,241,0.1); }
|
||||
.btn-disabled { background: var(--border); color: var(--muted); cursor: not-allowed; }
|
||||
|
||||
/* Forms */
|
||||
|
||||
@@ -18,7 +18,13 @@ def is_running() -> bool:
|
||||
return _running
|
||||
|
||||
|
||||
def run_sync() -> None:
|
||||
def run_sync(limit: int | None = None) -> None:
|
||||
"""
|
||||
Process all unprocessed remote zips in chunks.
|
||||
|
||||
limit: if set, process at most this many zips (used by test mode).
|
||||
If None, processes all unprocessed zips in batches of sync_batch_size.
|
||||
"""
|
||||
global _running
|
||||
if not _lock.acquire(blocking=False):
|
||||
log.warning("Sync already running, skipping")
|
||||
@@ -38,11 +44,31 @@ def run_sync() -> None:
|
||||
log.info("Listing remote zips at %s@%s:%s", cfg.sftp.user, cfg.sftp.host, cfg.sftp.remote_path)
|
||||
new_zips = sftp_module.list_new_zips(cfg.sftp)
|
||||
counters["zips_found"] = len(new_zips)
|
||||
|
||||
# Test mode: cap at the explicit limit
|
||||
if limit is not None:
|
||||
new_zips = new_zips[:limit]
|
||||
|
||||
counters["zips_new"] = len(new_zips)
|
||||
|
||||
if not new_zips:
|
||||
log.info("No new zips to process")
|
||||
db.finish_sync_run(run_id, status="success", **counters)
|
||||
return
|
||||
|
||||
# Determine chunk size; 0 means process everything in one chunk
|
||||
batch_size = int(db.get_setting("sync_batch_size", "0") or "0")
|
||||
if batch_size <= 0:
|
||||
batch_size = len(new_zips)
|
||||
|
||||
total_batches = -(-len(new_zips) // batch_size) # ceiling division
|
||||
client = CalibreClient(cfg.calibre)
|
||||
|
||||
for remote_zip in new_zips:
|
||||
for batch_num, i in enumerate(range(0, len(new_zips), batch_size), start=1):
|
||||
chunk = new_zips[i : i + batch_size]
|
||||
log.info("Batch %d/%d — processing %d zip(s)", batch_num, total_batches, len(chunk))
|
||||
|
||||
for remote_zip in chunk:
|
||||
zip_status = "success"
|
||||
zip_error = None
|
||||
local_zip = None
|
||||
@@ -69,9 +95,11 @@ def run_sync() -> None:
|
||||
extractor.cleanup(local_zip)
|
||||
db.mark_zip_processed(remote_zip.remote_path, remote_zip.file_size, zip_status, zip_error)
|
||||
|
||||
log.info("Batch %d/%d done", batch_num, total_batches)
|
||||
|
||||
db.finish_sync_run(run_id, status="success", **counters)
|
||||
log.info(
|
||||
"Sync done. Zips: %d, Uploaded: %d, Skipped: %d, Errors: %d",
|
||||
"Sync complete. Total zips: %d, Uploaded: %d, Skipped: %d, Errors: %d",
|
||||
counters["zips_new"], counters["books_uploaded"],
|
||||
counters["books_skipped"], counters["books_errored"],
|
||||
)
|
||||
|
||||
+12
-1
@@ -18,11 +18,22 @@
|
||||
<button class="btn btn-primary">Run Sync Now</button>
|
||||
{% endif %}
|
||||
</form>
|
||||
{% if not sync_running %}
|
||||
<form method="post" action="/sync/test" style="display:inline">
|
||||
<button class="btn btn-secondary">Test (1 zip)</button>
|
||||
</form>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
{% if batch_size > 0 %}
|
||||
<p class="muted small" style="margin-bottom:1.5rem">Batch size: {{ batch_size }} zips per chunk — full sync processes all unprocessed files.</p>
|
||||
{% endif %}
|
||||
|
||||
{% if request.query_params.get("started") %}
|
||||
<div class="alert alert-success">Sync started in background.</div>
|
||||
<div class="alert alert-success">Sync started — processing all unprocessed archives{% if batch_size > 0 %} in batches of {{ batch_size }}{% endif %}.</div>
|
||||
{% endif %}
|
||||
{% if request.query_params.get("test_started") %}
|
||||
<div class="alert alert-success">Test sync started — processing 1 archive.</div>
|
||||
{% endif %}
|
||||
{% if request.query_params.get("already_running") %}
|
||||
<div class="alert alert-warning">A sync is already running.</div>
|
||||
|
||||
@@ -113,6 +113,15 @@
|
||||
placeholder="0">
|
||||
<p class="muted small">Set to 0 to disable automatic sync. Changes take effect immediately on save. Examples: 60 = hourly, 1440 = daily.</p>
|
||||
</div>
|
||||
|
||||
<div class="form-row">
|
||||
<label for="sync_batch_size">Batch size (zips per chunk)</label>
|
||||
<input id="sync_batch_size" name="sync_batch_size" type="number"
|
||||
min="0" step="1" style="width:8rem"
|
||||
value="{{ s.get('sync_batch_size','0') }}"
|
||||
placeholder="0">
|
||||
<p class="muted small">Each sync run processes <strong>all</strong> unprocessed files, but works through them in chunks of this size to limit temp disk usage. Set to 0 to process all at once.</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<div class="form-actions">
|
||||
|
||||
Reference in New Issue
Block a user