diff --git a/config.py b/config.py index 9dace51..315dd51 100644 --- a/config.py +++ b/config.py @@ -54,7 +54,7 @@ def save(form: dict) -> None: "sftp_host", "sftp_port", "sftp_user", "sftp_auth_method", "sftp_password", "sftp_remote_path", "calibre_url", "calibre_user", "calibre_pass", - "local_work_dir", "scheduler_interval_minutes", + "local_work_dir", "scheduler_interval_minutes", "sync_batch_size", ] for key in keys: if key in form and form[key] is not None: diff --git a/main.py b/main.py index 02f3191..0540338 100644 --- a/main.py +++ b/main.py @@ -60,6 +60,7 @@ async def dashboard(request: Request): runs = [dict(r) for r in db.get_recent_runs(10)] zips = [dict(z) for z in db.get_recent_zips(20)] interval = int(db.get_setting("scheduler_interval_minutes", "0") or "0") + batch_size = int(db.get_setting("sync_batch_size", "0") or "0") return templates.TemplateResponse( "index.html", { @@ -70,6 +71,7 @@ async def dashboard(request: Request): "sync_running": sync.is_running(), "next_run": next_run_time(), "interval": interval, + "batch_size": batch_size, }, ) @@ -116,6 +118,7 @@ async def save_settings( calibre_pass: str = Form(""), local_work_dir: str = Form("/tmp/calibresync"), scheduler_interval_minutes: str = Form("0"), + sync_batch_size: str = Form("0"), ): config.save({ "sftp_host": sftp_host, @@ -130,12 +133,13 @@ async def save_settings( "calibre_pass": calibre_pass, "local_work_dir": local_work_dir, "scheduler_interval_minutes": scheduler_interval_minutes, + "sync_batch_size": sync_batch_size, }) _reschedule_auto_sync() return RedirectResponse("/settings?saved=1", status_code=303) -# --- Sync trigger --- +# --- Sync triggers --- @app.post("/sync") async def trigger_sync(background_tasks: BackgroundTasks): @@ -145,6 +149,14 @@ async def trigger_sync(background_tasks: BackgroundTasks): return RedirectResponse("/?started=1", status_code=303) +@app.post("/sync/test") +async def trigger_test_sync(background_tasks: BackgroundTasks): + if sync.is_running(): + return RedirectResponse("/?already_running=1", status_code=303) + background_tasks.add_task(sync.run_sync, 1) + return RedirectResponse("/?test_started=1", status_code=303) + + # --- JSON status API --- @app.get("/api/status") diff --git a/static/style.css b/static/style.css index 296a736..98a2204 100644 --- a/static/style.css +++ b/static/style.css @@ -150,6 +150,8 @@ tr:hover td { background: rgba(255,255,255,0.02); } } .btn-primary { background: var(--accent); color: #fff; } .btn-primary:hover { background: var(--accent-hover); } +.btn-secondary { background: transparent; color: var(--accent); border: 1px solid var(--accent); } +.btn-secondary:hover { background: rgba(99,102,241,0.1); } .btn-disabled { background: var(--border); color: var(--muted); cursor: not-allowed; } /* Forms */ diff --git a/sync.py b/sync.py index 254caa9..bac4d2a 100644 --- a/sync.py +++ b/sync.py @@ -18,7 +18,13 @@ def is_running() -> bool: return _running -def run_sync() -> None: +def run_sync(limit: int | None = None) -> None: + """ + Process all unprocessed remote zips in chunks. + + limit: if set, process at most this many zips (used by test mode). + If None, processes all unprocessed zips in batches of sync_batch_size. + """ global _running if not _lock.acquire(blocking=False): log.warning("Sync already running, skipping") @@ -38,40 +44,62 @@ def run_sync() -> None: log.info("Listing remote zips at %s@%s:%s", cfg.sftp.user, cfg.sftp.host, cfg.sftp.remote_path) new_zips = sftp_module.list_new_zips(cfg.sftp) counters["zips_found"] = len(new_zips) + + # Test mode: cap at the explicit limit + if limit is not None: + new_zips = new_zips[:limit] + counters["zips_new"] = len(new_zips) + if not new_zips: + log.info("No new zips to process") + db.finish_sync_run(run_id, status="success", **counters) + return + + # Determine chunk size; 0 means process everything in one chunk + batch_size = int(db.get_setting("sync_batch_size", "0") or "0") + if batch_size <= 0: + batch_size = len(new_zips) + + total_batches = -(-len(new_zips) // batch_size) # ceiling division client = CalibreClient(cfg.calibre) - for remote_zip in new_zips: - zip_status = "success" - zip_error = None - local_zip = None - try: - local_zip = sftp_module.download(cfg.sftp, remote_zip, str(work_dir / "downloads")) - books = extractor.extract(local_zip, work_dir / "extracted") + for batch_num, i in enumerate(range(0, len(new_zips), batch_size), start=1): + chunk = new_zips[i : i + batch_size] + log.info("Batch %d/%d — processing %d zip(s)", batch_num, total_batches, len(chunk)) - for book in books: - status = client.upload(book, zip_source=remote_zip.remote_path) - if status == "uploaded": - counters["books_uploaded"] += 1 - elif status == "skipped_duplicate": - counters["books_skipped"] += 1 - else: - counters["books_errored"] += 1 + for remote_zip in chunk: + zip_status = "success" + zip_error = None + local_zip = None + try: + local_zip = sftp_module.download(cfg.sftp, remote_zip, str(work_dir / "downloads")) + books = extractor.extract(local_zip, work_dir / "extracted") - extractor.cleanup(work_dir / "extracted" / local_zip.stem) - except Exception as e: - log.error("Error processing %s: %s", remote_zip.remote_path, e) - zip_status = "error" - zip_error = str(e) - finally: - if local_zip and local_zip.exists(): - extractor.cleanup(local_zip) - db.mark_zip_processed(remote_zip.remote_path, remote_zip.file_size, zip_status, zip_error) + for book in books: + status = client.upload(book, zip_source=remote_zip.remote_path) + if status == "uploaded": + counters["books_uploaded"] += 1 + elif status == "skipped_duplicate": + counters["books_skipped"] += 1 + else: + counters["books_errored"] += 1 + + extractor.cleanup(work_dir / "extracted" / local_zip.stem) + except Exception as e: + log.error("Error processing %s: %s", remote_zip.remote_path, e) + zip_status = "error" + zip_error = str(e) + finally: + if local_zip and local_zip.exists(): + extractor.cleanup(local_zip) + db.mark_zip_processed(remote_zip.remote_path, remote_zip.file_size, zip_status, zip_error) + + log.info("Batch %d/%d done", batch_num, total_batches) db.finish_sync_run(run_id, status="success", **counters) log.info( - "Sync done. Zips: %d, Uploaded: %d, Skipped: %d, Errors: %d", + "Sync complete. Total zips: %d, Uploaded: %d, Skipped: %d, Errors: %d", counters["zips_new"], counters["books_uploaded"], counters["books_skipped"], counters["books_errored"], ) diff --git a/templates/index.html b/templates/index.html index 49c07cb..e787b14 100644 --- a/templates/index.html +++ b/templates/index.html @@ -18,11 +18,22 @@ {% endif %} + {% if not sync_running %} +
+ {% endif %} +{% if batch_size > 0 %} +Batch size: {{ batch_size }} zips per chunk — full sync processes all unprocessed files.
+{% endif %} {% if request.query_params.get("started") %} -Set to 0 to disable automatic sync. Changes take effect immediately on save. Examples: 60 = hourly, 1440 = daily.
+ +Each sync run processes all unprocessed files, but works through them in chunks of this size to limit temp disk usage. Set to 0 to process all at once.
+