Mockup

.gitignore

@@ -7,10 +7,19 @@ data/cdx-index/*.tmp
 
 # Raw scraped HTML pages
 data/pilot/html/
+data/pilot/html-recent/
+data/pilot/html-recent-test/
+
+# Transient pipeline stdout logs
+data/pilot/*.out
+runs/**/*.out
 
 # Large pilot manifests (small derived CSVs/summaries are kept)
 data/pilot/*-manifest*.tsv
 
+# Scratch / working artifacts (not deliverables)
+scratchpad/
+
 # === Python / OS cruft ===
 __pycache__/
 *.pyc

code/09-extract-prices.py

@@ -293,7 +293,10 @@ def process_file(filepath):
     result["year"] = year
     result["month"] = month
     result["title"] = extract_title(html)
-    result["file_path"] = str(filepath.relative_to(BASE_DIR))
+    try:
+        result["file_path"] = str(Path(filepath).resolve().relative_to(BASE_DIR))
+    except ValueError:
+        result["file_path"] = str(filepath)
 
     rating, review_count = extract_rating(html)
     result["rating"] = rating if rating is not None else ""
@@ -303,7 +306,17 @@ def process_file(filepath):
 
 
 def main():
+    import argparse
+    global HTML_DIR, OUTPUT, ERRORS_LOG
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--html-dir", type=Path, default=HTML_DIR)
+    ap.add_argument("--output", type=Path, default=OUTPUT)
+    ap.add_argument("--errors-log", type=Path, default=ERRORS_LOG)
+    a = ap.parse_args()
+    HTML_DIR, OUTPUT, ERRORS_LOG = a.html_dir, a.output, a.errors_log
+
     print("Extracting prices from downloaded HTML...")
+    print(f"  Input: {HTML_DIR}")
 
     html_files = sorted(HTML_DIR.rglob("*.html"))
     print(f"  Found {len(html_files):,} HTML files")

code/12-panel-ipi.py

@@ -196,17 +196,25 @@ def main():
         # Initialize at base quarter = 100
         index = {base_q: 100.0}
 
-        # Chain forward from base
+        # Chain forward from base. Gap-tolerant: the matched-model relative at a
+        # quarter is p_t / p_{gig's previous observed quarter}, which already spans
+        # any gap, so we chain from the most recent quarter already in the index
+        # rather than requiring the immediately-preceding grid quarter. This keeps a
+        # category alive across coverage gaps (e.g. translation is empty in 2019Q2–Q4
+        # right after the base but has 18 solid transitions over 2020Q1–2024Q3; the
+        # old "prev grid quarter must be in index" rule silently discarded all of it).
         forward_qs = [q for q in all_quarters if q >= base_q]
+        last_q = base_q
         for i in range(1, len(forward_qs)):
             q = forward_qs[i]
             if q in quarter_relatives and len(quarter_relatives[q]) >= 3:
                 geo_mean = np.exp(np.mean(np.log(quarter_relatives[q])))
-                prev_q = forward_qs[i-1]
-                if prev_q in index:
-                    index[q] = index[prev_q] * geo_mean
+                index[q] = index[last_q] * geo_mean
+                last_q = q
 
-        # Chain backward from base
+        # Chain backward from base. Kept strict (immediately-preceding grid quarter
+        # must already be in the index) so pre-base historical levels — which feed
+        # the paper's peak/start figures — are unchanged by this fix.
         backward_qs = [q for q in reversed(all_quarters) if q <= base_q]
         for i in range(1, len(backward_qs)):
             q = backward_qs[i]

code/13-recent-manifest.py

@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+"""
+Step 13: Build a recent-window download manifest to extend the IPI into the
+trailing 12 months (true "past year").
+
+The 500-seller pilot was sampled for long histories and goes sparse after 2024Q4.
+To compute a genuine last-12-months per-category index we need gigs that are
+actually archived in the recent quarters. This script selects, per category,
+gigs that have:
+  - >= 2 distinct quarters of coverage in the window (2024Q3 onward), AND
+  - >= 1 snapshot in the trailing window (2025Q3..2026Q2),
+so each selected gig yields a chainable price path reaching into the recent year.
+
+For each selected gig we keep one snapshot per month (latest in the month),
+across the full window, so the matched-model index can chain quarter-to-quarter.
+
+Input:  data/cdx-index/gig-pages-classified.tsv  (6 GB; streamed once)
+        (or a pre-filtered TSV via --prefiltered: category,timestamp,original,urlkey)
+Output: data/pilot/recent-manifest.tsv  (timestamp, original, category, gig_id, month)
+"""
+
+import argparse
+import csv
+import subprocess
+import sys
+from collections import defaultdict
+from pathlib import Path
+
+BASE_DIR = Path(__file__).resolve().parent.parent
+CLASSIFIED = BASE_DIR / "data" / "cdx-index" / "gig-pages-classified.tsv"
+OUTPUT = BASE_DIR / "data" / "pilot" / "recent-manifest.tsv"
+
+WINDOW_START = "202407"          # 2024Q3 — anchor quarters for chaining
+TRAILING_QUARTERS = {"2025Q3", "2025Q4", "2026Q1", "2026Q2"}
+# Categories too thin in the recent window to support an index (see 06f census).
+SKIP_CATEGORIES = {"uncategorized", "data_entry", "data_analysis"}
+
+
+def quarter(ts):
+    y, m = int(ts[:4]), int(ts[4:6])
+    return f"{y}Q{(m - 1) // 3 + 1}"
+
+
+def gig_id(urlkey):
+    # urlkey looks like "com,fiverr)/seller/slug?sorted&query".
+    # Key by seller/slug only (drop query string) so the same gig merges
+    # across snapshots regardless of tracking params — matches the price
+    # pipeline, which keys by (seller, slug) from the URL path.
+    tail = urlkey.split(")/", 1)[1] if ")/" in urlkey else urlkey
+    return tail.split("?", 1)[0]
+
+
+def iter_rows(prefiltered):
+    """Yield (category, timestamp, original, urlkey) over the recent window."""
+    if prefiltered:
+        with open(prefiltered) as f:
+            for line in f:
+                p = line.rstrip("\n").split("\t")
+                if len(p) >= 4:
+                    yield p[0], p[1], p[2], p[3]
+        return
+    # Stream the classified file directly (status 200, recent window only).
+    with open(CLASSIFIED) as f:
+        next(f, None)  # header
+        for line in f:
+            p = line.rstrip("\n").split("\t")
+            if len(p) < 7:
+                continue
+            ts, status, cat = p[1], p[3], p[6]
+            if status != "200" or ts[:6] < WINDOW_START:
+                continue
+            yield cat, ts, p[2], p[0]
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--prefiltered", type=Path, default=None,
+                    help="Optional pre-filtered TSV: category,timestamp,original,urlkey")
+    args = ap.parse_args()
+
+    print("Building recent-window manifest...")
+    # gig -> {"cat":, "quarters": set, "months": {YYYYMM: (timestamp, original)}}
+    gigs = {}
+    n = 0
+    for cat, ts, original, urlkey in iter_rows(args.prefiltered):
+        n += 1
+        if cat in SKIP_CATEGORIES:
+            continue
+        gid = gig_id(urlkey)
+        g = gigs.get(gid)
+        if g is None:
+            g = gigs[gid] = {"cat": cat, "quarters": set(), "months": {}}
+        g["quarters"].add(quarter(ts))
+        ym = ts[:6]
+        # keep latest timestamp within the month
+        prev = g["months"].get(ym)
+        if prev is None or ts > prev[0]:
+            g["months"][ym] = (ts, original)
+    print(f"  Scanned {n:,} snapshots; {len(gigs):,} distinct gigs in window")
+
+    # Select gigs with multi-quarter coverage reaching the trailing window.
+    selected = {gid: g for gid, g in gigs.items()
+                if len(g["quarters"]) >= 2 and (g["quarters"] & TRAILING_QUARTERS)}
+
+    per_cat = defaultdict(int)
+    for g in selected.values():
+        per_cat[g["cat"]] += 1
+    print(f"  Selected {len(selected):,} gigs:")
+    for c in sorted(per_cat, key=lambda x: -per_cat[x]):
+        print(f"    {c:<16}{per_cat[c]:>7,} gigs")
+
+    OUTPUT.parent.mkdir(parents=True, exist_ok=True)
+    rows = 0
+    with open(OUTPUT, "w", newline="") as f:
+        w = csv.writer(f, delimiter="\t")
+        w.writerow(["timestamp", "original", "category", "gig_id", "month"])
+        for gid, g in selected.items():
+            for ym, (ts, original) in sorted(g["months"].items()):
+                w.writerow([ts, original, g["cat"], gid, ym])
+                rows += 1
+    print(f"  Wrote {rows:,} snapshot rows -> {OUTPUT}")
+    print(f"  (~{rows} downloads; avg {rows/max(len(selected),1):.1f} snapshots/gig)")
+
+
+if __name__ == "__main__":
+    main()