sync errors

sftp.py

@@ -79,14 +79,21 @@ def test_connection(cfg: SFTPConfig) -> tuple[bool, str]:
         return False, str(e)
 
 
-def list_new_zips(cfg: SFTPConfig) -> list[RemoteZip]:
+def list_new_zips(cfg: SFTPConfig, max_results: int | None = None) -> list[RemoteZip]:
     transport = _make_transport(cfg)
     sftp = paramiko.SFTPClient.from_transport(transport)
     try:
         log.info("Walking remote directory tree from %s ...", cfg.remote_path)
-        all_zips = _walk_zips(sftp, cfg.remote_path)
-        new_zips = [z for z in all_zips if not db.is_zip_processed(z.remote_path)]
-        log.info("Remote: %d zip(s) total, %d new", len(all_zips), len(new_zips))
+        new_zips: list[RemoteZip] = []
+        total_seen = 0
+        for zip_info in _walk_zips_iter(sftp, cfg.remote_path):
+            total_seen += 1
+            if not db.is_zip_processed(zip_info.remote_path):
+                new_zips.append(zip_info)
+                if max_results and len(new_zips) >= max_results:
+                    log.info("Reached limit of %d — stopping walk early", max_results)
+                    break
+        log.info("Remote walk done: %d zip(s) seen, %d new", total_seen, len(new_zips))
         return new_zips
     finally:
         sftp.close()
@@ -108,14 +115,13 @@ def download(cfg: SFTPConfig, remote_zip: RemoteZip, dest_dir: str) -> Path:
     return local_path
 
 
-def _walk_zips(sftp: paramiko.SFTPClient, remote_dir: str) -> list[RemoteZip]:
-    results: list[RemoteZip] = []
+def _walk_zips_iter(sftp: paramiko.SFTPClient, remote_dir: str):
     log.info("Listing %s ...", remote_dir)
     try:
         entries = sftp.listdir_attr(remote_dir)
     except IOError as e:
         log.warning("Cannot list %s: %s", remote_dir, e)
-        return results
+        return
 
     subdirs = []
     zips_here = 0
@@ -124,10 +130,9 @@ def _walk_zips(sftp: paramiko.SFTPClient, remote_dir: str) -> list[RemoteZip]:
         if stat.S_ISDIR(entry.st_mode):
             subdirs.append(full_path)
         elif entry.filename.lower().endswith(".zip"):
-            results.append(RemoteZip(remote_path=full_path, file_size=entry.st_size or 0))
+            yield RemoteZip(remote_path=full_path, file_size=entry.st_size or 0)
             zips_here += 1
 
     log.info("  %s: %d entries, %d zip(s), %d subdir(s)", remote_dir, len(entries), zips_here, len(subdirs))
     for subdir in subdirs:
-        results.extend(_walk_zips(sftp, subdir))
-    return results
+        yield from _walk_zips_iter(sftp, subdir)