Before Handing an Agent a .zip: Dropping .git Cut 29 MB to 15 MB, and 28 Secret Warnings Turned Out to Be Noise

#!/usr/bin/env python3
"""agzip.py: Build a .zip for the agent only after checking what goes in."""
import argparse, fnmatch, hashlib, os, re, sys, zipfile
 
SKIP_DIRS = {".git", "node_modules", ".next", "dist", "build", ".venv",
             "__pycache__", ".wrangler", ".turbo", ".cache"}
SKIP_FILES = [".env", ".env.*", "*.pem", "*.key", "id_rsa*", "*credentials*.json",
              ".DS_Store", "*.log"]
KEEP_FILES = [".env.example", ".env.sample"]  # let these through: they only name the variables
SECRET_RES = {
    "github-token": re.compile(r"gh[pousr]_[A-Za-z0-9]{30,}"),
    "aws-access-key": re.compile(r"AKIA[0-9A-Z]{16}"),
    "google-api-key": re.compile(r"AIza[0-9A-Za-z_\-]{35}"),
    "private-key": re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----"),
    "generic-secret": re.compile(r"(?i)(secret|token|password)\s*[:=]\s*['\"][^'\"\s]{16,}['\"]"),
}
# drop obvious dummy values from warnings (widen this too far and real keys slip by)
PLACEHOLDER = re.compile(r"(?i)(x{4,}|your_|<[^>]+>|\$\{|example|dummy|changeme|_test_|\.\.\.)")
TEXT_LIMIT = 1_000_000
 
def line_id(line):
    return hashlib.sha256(line.strip().encode()).hexdigest()[:8]
 
def skipped_file(name):
    if name in KEEP_FILES:
        return False
    return any(fnmatch.fnmatch(name, p) for p in SKIP_FILES)
 
def dir_bytes(path):
    return sum(os.path.getsize(os.path.join(r, f)) for r, _, fs in os.walk(path)
               for f in fs if not os.path.islink(os.path.join(r, f)))
 
def collect(src):
    kept, dropped, links = [], {"dirs": 0, "files": 0, "bytes": 0}, []
    for root, dirs, files in os.walk(src, followlinks=False):
        keep = []
        for d in sorted(dirs):
            full = os.path.join(root, d)
            if os.path.islink(full):          # a link may point outside the folder, so leave it out
                links.append(os.path.relpath(full, src))
            elif d in SKIP_DIRS:
                dropped["dirs"] += 1; dropped["bytes"] += dir_bytes(full)
            else:
                keep.append(d)
        dirs[:] = keep
        for f in sorted(files):
            full = os.path.join(root, f); rel = os.path.relpath(full, src)
            if os.path.islink(full):
                links.append(rel)
            elif skipped_file(f):
                dropped["files"] += 1
            else:
                kept.append((rel, os.path.getsize(full)))
    return kept, dropped, links
 
def scan(src, kept):
    hits = []
    for rel, size in kept:
        if size > TEXT_LIMIT:
            continue
        try:
            with open(os.path.join(src, rel), encoding="utf-8") as fh:
                for n, line in enumerate(fh, 1):
                    for kind, rx in SECRET_RES.items():
                        m = rx.search(line)
                        if m and not PLACEHOLDER.search(m.group(0)):
                            hits.append((rel, n, kind, line_id(line)))
        except (UnicodeDecodeError, OSError):
            continue                          # binaries are out of scope for the regexes
    return hits
 
def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("src"); ap.add_argument("out")
    ap.add_argument("--max-mb", type=float, default=20.0)
    ap.add_argument("--ack", action="append", default=[],
                    help="a line you checked by eye and judged a false positive, as path:id (id is the 8 digits shown in the warning)")
    a = ap.parse_args()
    kept, dropped, links = collect(a.src)
    hits = [h for h in scan(a.src, kept) if f"{h[0]}:{h[3]}" not in a.ack]
    print(f"files={len(kept)} raw={sum(s for _, s in kept)/1e6:.2f}MB "
          f"dropped_dirs={dropped['dirs']}({dropped['bytes']/1e6:.2f}MB) "
          f"dropped_files={dropped['files']} symlinks={len(links)}")
    for rel, n, kind, lid in hits:
        print(f"SECRET? {rel}:{n} [{kind}] ack={rel}:{lid}")   # never print the value itself
    if hits:
        print("ABORT: lines that look like secrets were found"); return 2
    with zipfile.ZipFile(a.out, "w", zipfile.ZIP_DEFLATED, compresslevel=6) as z:
        manifest = ["path\tbytes\tsha256_8"]
        for rel, size in kept:
            zname = rel.replace(os.sep, "/")        # keep "/" inside the zip on Windows too
            data = open(os.path.join(a.src, rel), "rb").read()
            manifest.append(f"{zname}\t{size}\t{hashlib.sha256(data).hexdigest()[:8]}")
            z.writestr(zipfile.ZipInfo(zname, (2026, 1, 1, 0, 0, 0)), data, zipfile.ZIP_DEFLATED)
        z.writestr("_MANIFEST.tsv", "\n".join(manifest) + "\n")
    mb = os.path.getsize(a.out) / 1e6
    print(f"zip={mb:.2f}MB")
    if mb > a.max_mb:
        print(f"ABORT: over {a.max_mb}MB"); return 3
    return 0
 
if __name__ == "__main__":
    sys.exit(main())