GSoC Module A: week 5 : feat(harvester): add git diff retrieval pipeline (stacked on top of #986)

.gitignore

@@ -85,3 +85,5 @@ tmp/
 cres/*
 ### Local project management tooling
 project management scripts/
+
+.harvester_cache/

application/tests/harvester_test/diff_normalizer_test.py

@@ -0,0 +1,117 @@
+import unittest
+from datetime import datetime
+
+from application.utils.harvester.diff_normalizer import (
+    DiffNormalizer,
+)
+
+from application.utils.harvester.models import (
+    DiffBlock,
+)
+
+
+DIFF_METADATA = {
+    "repository": "OWASP/ASVS",
+    "commit_sha": "abc123",
+    "committed_at": datetime(2026, 1, 1),
+}
+
+
+class DiffNormalizerTests(unittest.TestCase):
+    def test_whitespace_normalization(self):
+        normalizer = DiffNormalizer()
+
+        blocks = [
+            DiffBlock(
+                file_path="README.md",
+                added_lines=[
+                    "     Hello      World      ",
+                    "\t\tTabs\t\tEverywhere\t",
+                    "",
+                    "     ",
+                    "Unicode\u00a0Space",
+                    "Mix\t of\t tabs   and    spaces",
+                    "   Multiple     words      together   ",
+                    "\u00a0\u00a0Leading unicode spaces\u00a0",
+                    "   ## Authentication   ",
+                    "   -   Use MFA   ",
+                    "   `inline code`   ",
+                    "   **Important**   ",
+                ],
+                **DIFF_METADATA,
+            )
+        ]
+
+        result = normalizer.normalize(blocks)
+
+        self.assertEqual(
+            result[0].added_lines,
+            [
+                "Hello World",
+                "Tabs Everywhere",
+                "Unicode Space",
+                "Mix of tabs and spaces",
+                "Multiple words together",
+                "Leading unicode spaces",
+                "## Authentication",
+                "- Use MFA",
+                "`inline code`",
+                "**Important**",
+            ],
+        )
+
+    def test_remove_empty_lines(self):
+        normalizer = DiffNormalizer()
+
+        blocks = [
+            DiffBlock(
+                file_path="README.md",
+                added_lines=[
+                    "",
+                    "   ",
+                    "Hello",
+                ],
+                **DIFF_METADATA,
+            )
+        ]
+
+        result = normalizer.normalize(blocks)
+
+        self.assertEqual(
+            result[0].added_lines,
+            [
+                "Hello",
+            ],
+        )
+
+    def test_multiple_blocks(self):
+        normalizer = DiffNormalizer()
+
+        blocks = [
+            DiffBlock(
+                file_path="a.md",
+                added_lines=["  One  "],
+                **DIFF_METADATA,
+            ),
+            DiffBlock(
+                file_path="b.md",
+                added_lines=["  Two  "],
+                **DIFF_METADATA,
+            ),
+        ]
+
+        result = normalizer.normalize(blocks)
+
+        self.assertEqual(
+            result[0].added_lines,
+            ["One"],
+        )
+
+        self.assertEqual(
+            result[1].added_lines,
+            ["Two"],
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

application/tests/harvester_test/diff_parser_test.py

@@ -0,0 +1,103 @@
+from datetime import UTC, datetime
+import unittest
+
+from application.utils.harvester.diff_parser import (
+    DiffParser,
+)
+
+TEST_REPOSITORY = "OWASP/ASVS"
+TEST_COMMIT_SHA = "abc123"
+TEST_COMMITTED_AT = datetime.now(UTC)
+
+
+class DiffParserTests(unittest.TestCase):
+    def test_single_file_diff(self):
+        parser = DiffParser()
+
+        diff = """diff --git a/test.md b/test.md
+--- a/test.md
++++ b/test.md
+@@
+-old
++new
++another
+"""
+
+        blocks = parser.parse(
+            diff,
+            repository=TEST_REPOSITORY,
+            commit_sha=TEST_COMMIT_SHA,
+            committed_at=TEST_COMMITTED_AT,
+        )
+
+        self.assertEqual(len(blocks), 1)
+
+        self.assertEqual(
+            blocks[0].file_path,
+            "test.md",
+        )
+
+        self.assertEqual(
+            blocks[0].added_lines,
+            [
+                "new",
+                "another",
+            ],
+        )
+
+        self.assertEqual(blocks[0].repository, TEST_REPOSITORY)
+        self.assertEqual(blocks[0].commit_sha, TEST_COMMIT_SHA)
+        self.assertEqual(blocks[0].committed_at, TEST_COMMITTED_AT)
+
+    def test_multiple_files(self):
+        parser = DiffParser()
+
+        diff = """diff --git a/a.md b/a.md
+@@
++one
+diff --git a/b.md b/b.md
+@@
++two
+"""
+
+        blocks = parser.parse(
+            diff,
+            repository=TEST_REPOSITORY,
+            commit_sha=TEST_COMMIT_SHA,
+            committed_at=TEST_COMMITTED_AT,
+        )
+
+        self.assertEqual(len(blocks), 2)
+
+        self.assertEqual(blocks[0].file_path, "a.md")
+        self.assertEqual(blocks[1].file_path, "b.md")
+
+        self.assertEqual(blocks[0].repository, TEST_REPOSITORY)
+        self.assertEqual(blocks[1].repository, TEST_REPOSITORY)
+
+    def test_deleted_lines_are_ignored(self):
+        parser = DiffParser()
+
+        diff = """diff --git a/test.md b/test.md
+@@
+-old
++new
+"""
+
+        blocks = parser.parse(
+            diff,
+            repository=TEST_REPOSITORY,
+            commit_sha=TEST_COMMIT_SHA,
+            committed_at=TEST_COMMITTED_AT,
+        )
+
+        self.assertEqual(
+            blocks[0].added_lines,
+            [
+                "new",
+            ],
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

application/tests/harvester_test/diff_pipeline_test.py

@@ -0,0 +1,73 @@
+from datetime import UTC, datetime
+import subprocess
+import time
+import unittest
+import os
+
+from application.utils.harvester.diff_normalizer import DiffNormalizer
+from application.utils.harvester.diff_parser import DiffParser
+from application.utils.harvester.diff_retriever import DiffRetriever
+from application.utils.harvester.git_repository_client import GitRepositoryClient
+
+
+class DiffPipelineBenchmark(unittest.TestCase):
+    """
+    Simple benchmark to ensure the complete diff pipeline remains fast.
+
+    This is not intended as a strict performance benchmark, only as a
+    regression guard against accidental slowdowns.
+    """
+
+    def test_pipeline_benchmark(self):
+
+        if os.getenv("OPENCRE_RUN_NETWORK_TESTS") != "1":
+            self.skipTest("Network benchmark disabled")
+
+        client = GitRepositoryClient(
+            "OWASP",
+            "ASVS",
+            "master",
+        )
+        client.sync()
+
+        head_commit = client.get_current_commit_sha()
+
+        previous_commit = subprocess.run(
+            [
+                "git",
+                "-C",
+                str(client.get_local_path()),
+                "rev-parse",
+                "HEAD~1",
+            ],
+            check=True,
+            capture_output=True,
+            text=True,
+            timeout=300,
+        ).stdout.strip()
+
+        retriever = DiffRetriever(client)
+        parser = DiffParser()
+        normalizer = DiffNormalizer()
+
+        start = time.perf_counter()
+
+        diff = retriever.get_diff(
+            previous_commit,
+            head_commit,
+        )
+
+        blocks = parser.parse(
+            diff,
+            repository="OWASP/ASVS",
+            commit_sha=head_commit,
+            committed_at=datetime.now(UTC),
+        )
+
+        normalizer.normalize(blocks)
+
+        elapsed = time.perf_counter() - start
+
+        print(f"\nPipeline took {elapsed:.3f}s")
+
+        self.assertLess(elapsed, 5)