Added batch process and test

config.py

@@ -54,7 +54,7 @@ def save(form: dict) -> None:
         "sftp_host", "sftp_port", "sftp_user", "sftp_auth_method",
         "sftp_password", "sftp_remote_path",
         "calibre_url", "calibre_user", "calibre_pass",
-        "local_work_dir", "scheduler_interval_minutes",
+        "local_work_dir", "scheduler_interval_minutes", "sync_batch_size",
     ]
     for key in keys:
         if key in form and form[key] is not None:

main.py

@@ -60,6 +60,7 @@ async def dashboard(request: Request):
     runs = [dict(r) for r in db.get_recent_runs(10)]
     zips = [dict(z) for z in db.get_recent_zips(20)]
     interval = int(db.get_setting("scheduler_interval_minutes", "0") or "0")
+    batch_size = int(db.get_setting("sync_batch_size", "0") or "0")
     return templates.TemplateResponse(
         "index.html",
         {
@@ -70,6 +71,7 @@ async def dashboard(request: Request):
             "sync_running": sync.is_running(),
             "next_run": next_run_time(),
             "interval": interval,
+            "batch_size": batch_size,
         },
     )
 
@@ -116,6 +118,7 @@ async def save_settings(
     calibre_pass: str = Form(""),
     local_work_dir: str = Form("/tmp/calibresync"),
     scheduler_interval_minutes: str = Form("0"),
+    sync_batch_size: str = Form("0"),
 ):
     config.save({
         "sftp_host": sftp_host,
@@ -130,12 +133,13 @@ async def save_settings(
         "calibre_pass": calibre_pass,
         "local_work_dir": local_work_dir,
         "scheduler_interval_minutes": scheduler_interval_minutes,
+        "sync_batch_size": sync_batch_size,
     })
     _reschedule_auto_sync()
     return RedirectResponse("/settings?saved=1", status_code=303)
 
 
-# --- Sync trigger ---
+# --- Sync triggers ---
 
 @app.post("/sync")
 async def trigger_sync(background_tasks: BackgroundTasks):
@@ -145,6 +149,14 @@ async def trigger_sync(background_tasks: BackgroundTasks):
     return RedirectResponse("/?started=1", status_code=303)
 
 
+@app.post("/sync/test")
+async def trigger_test_sync(background_tasks: BackgroundTasks):
+    if sync.is_running():
+        return RedirectResponse("/?already_running=1", status_code=303)
+    background_tasks.add_task(sync.run_sync, 1)
+    return RedirectResponse("/?test_started=1", status_code=303)
+
+
 # --- JSON status API ---
 
 @app.get("/api/status")

static/style.css

@@ -150,6 +150,8 @@ tr:hover td { background: rgba(255,255,255,0.02); }
 }
 .btn-primary { background: var(--accent); color: #fff; }
 .btn-primary:hover { background: var(--accent-hover); }
+.btn-secondary { background: transparent; color: var(--accent); border: 1px solid var(--accent); }
+.btn-secondary:hover { background: rgba(99,102,241,0.1); }
 .btn-disabled { background: var(--border); color: var(--muted); cursor: not-allowed; }
 
 /* Forms */

sync.py

@@ -18,7 +18,13 @@ def is_running() -> bool:
     return _running
 
 
-def run_sync() -> None:
+def run_sync(limit: int | None = None) -> None:
+    """
+    Process all unprocessed remote zips in chunks.
+
+    limit: if set, process at most this many zips (used by test mode).
+           If None, processes all unprocessed zips in batches of sync_batch_size.
+    """
     global _running
     if not _lock.acquire(blocking=False):
         log.warning("Sync already running, skipping")
@@ -38,40 +44,62 @@ def run_sync() -> None:
         log.info("Listing remote zips at %s@%s:%s", cfg.sftp.user, cfg.sftp.host, cfg.sftp.remote_path)
         new_zips = sftp_module.list_new_zips(cfg.sftp)
         counters["zips_found"] = len(new_zips)
+
+        # Test mode: cap at the explicit limit
+        if limit is not None:
+            new_zips = new_zips[:limit]
+
         counters["zips_new"] = len(new_zips)
 
+        if not new_zips:
+            log.info("No new zips to process")
+            db.finish_sync_run(run_id, status="success", **counters)
+            return
+
+        # Determine chunk size; 0 means process everything in one chunk
+        batch_size = int(db.get_setting("sync_batch_size", "0") or "0")
+        if batch_size <= 0:
+            batch_size = len(new_zips)
+
+        total_batches = -(-len(new_zips) // batch_size)  # ceiling division
         client = CalibreClient(cfg.calibre)
 
-        for remote_zip in new_zips:
+        for batch_num, i in enumerate(range(0, len(new_zips), batch_size), start=1):
-            zip_status = "success"
+            chunk = new_zips[i : i + batch_size]
-            zip_error = None
+            log.info("Batch %d/%d — processing %d zip(s)", batch_num, total_batches, len(chunk))
-            local_zip = None
-            try:
-                local_zip = sftp_module.download(cfg.sftp, remote_zip, str(work_dir / "downloads"))
-                books = extractor.extract(local_zip, work_dir / "extracted")
 
-                for book in books:
+            for remote_zip in chunk:
-                    status = client.upload(book, zip_source=remote_zip.remote_path)
+                zip_status = "success"
-                    if status == "uploaded":
+                zip_error = None
-                        counters["books_uploaded"] += 1
+                local_zip = None
-                    elif status == "skipped_duplicate":
+                try:
-                        counters["books_skipped"] += 1
+                    local_zip = sftp_module.download(cfg.sftp, remote_zip, str(work_dir / "downloads"))
-                    else:
+                    books = extractor.extract(local_zip, work_dir / "extracted")
-                        counters["books_errored"] += 1
 
-                extractor.cleanup(work_dir / "extracted" / local_zip.stem)
+                    for book in books:
-            except Exception as e:
+                        status = client.upload(book, zip_source=remote_zip.remote_path)
-                log.error("Error processing %s: %s", remote_zip.remote_path, e)
+                        if status == "uploaded":
-                zip_status = "error"
+                            counters["books_uploaded"] += 1
-                zip_error = str(e)
+                        elif status == "skipped_duplicate":
-            finally:
+                            counters["books_skipped"] += 1
-                if local_zip and local_zip.exists():
+                        else:
-                    extractor.cleanup(local_zip)
+                            counters["books_errored"] += 1
-                db.mark_zip_processed(remote_zip.remote_path, remote_zip.file_size, zip_status, zip_error)
+
+                    extractor.cleanup(work_dir / "extracted" / local_zip.stem)
+                except Exception as e:
+                    log.error("Error processing %s: %s", remote_zip.remote_path, e)
+                    zip_status = "error"
+                    zip_error = str(e)
+                finally:
+                    if local_zip and local_zip.exists():
+                        extractor.cleanup(local_zip)
+                    db.mark_zip_processed(remote_zip.remote_path, remote_zip.file_size, zip_status, zip_error)
+
+            log.info("Batch %d/%d done", batch_num, total_batches)
 
         db.finish_sync_run(run_id, status="success", **counters)
         log.info(
-            "Sync done. Zips: %d, Uploaded: %d, Skipped: %d, Errors: %d",
+            "Sync complete. Total zips: %d, Uploaded: %d, Skipped: %d, Errors: %d",
             counters["zips_new"], counters["books_uploaded"],
             counters["books_skipped"], counters["books_errored"],
         )

templates/index.html

@@ -18,11 +18,22 @@
         <button class="btn btn-primary">Run Sync Now</button>
       {% endif %}
     </form>
+    {% if not sync_running %}
+    <form method="post" action="/sync/test" style="display:inline">
+      <button class="btn btn-secondary">Test (1 zip)</button>
+    </form>
+    {% endif %}
   </div>
 </div>
+{% if batch_size > 0 %}
+<p class="muted small" style="margin-bottom:1.5rem">Batch size: {{ batch_size }} zips per chunk — full sync processes all unprocessed files.</p>
+{% endif %}
 
 {% if request.query_params.get("started") %}
-  <div class="alert alert-success">Sync started in background.</div>
+  <div class="alert alert-success">Sync started — processing all unprocessed archives{% if batch_size > 0 %} in batches of {{ batch_size }}{% endif %}.</div>
+{% endif %}
+{% if request.query_params.get("test_started") %}
+  <div class="alert alert-success">Test sync started — processing 1 archive.</div>
 {% endif %}
 {% if request.query_params.get("already_running") %}
   <div class="alert alert-warning">A sync is already running.</div>

templates/settings.html

@@ -113,6 +113,15 @@
              placeholder="0">
       <p class="muted small">Set to 0 to disable automatic sync. Changes take effect immediately on save. Examples: 60 = hourly, 1440 = daily.</p>
     </div>
+
+    <div class="form-row">
+      <label for="sync_batch_size">Batch size (zips per chunk)</label>
+      <input id="sync_batch_size" name="sync_batch_size" type="number"
+             min="0" step="1" style="width:8rem"
+             value="{{ s.get('sync_batch_size','0') }}"
+             placeholder="0">
+      <p class="muted small">Each sync run processes <strong>all</strong> unprocessed files, but works through them in chunks of this size to limit temp disk usage. Set to 0 to process all at once.</p>
+    </div>
   </section>
 
   <div class="form-actions">