Architectural change

.github/workflows/ci.yml

@@ -0,0 +1,36 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  lint:
+    name: Lint (ruff)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - run: pip install ruff
+      - run: ruff check scripts/ tests/
+      - run: ruff format --check scripts/ tests/
+
+  test:
+    name: Test (Python ${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dev dependencies
+        run: pip install -e ".[dev]"
+      - name: Run tests
+        run: pytest tests/ -v --tb=short

.github/workflows/publish.yml

@@ -0,0 +1,39 @@
+name: Publish to PyPI
+
+on:
+  push:
+    tags:
+      - "v*"
+
+jobs:
+  build:
+    name: Build distribution
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Install hatch
+        run: pip install hatch
+      - name: Build wheel and sdist
+        run: hatch build
+      - uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    name: Publish to PyPI
+    needs: build
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write  # required for trusted publishing (OIDC)
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

Makefile

@@ -0,0 +1,33 @@
+.PHONY: test lint format typecheck check install-dev
+
+# Install dev dependencies (requires pip)
+install-dev:
+	pip install -e ".[dev]"
+
+# Run all unit tests with verbose output
+test:
+	pytest tests/ -v
+
+# Run tests with coverage report
+test-cov:
+	pytest tests/ -v --tb=short --cov=token_reducer --cov-report=term-missing
+
+# Lint: check for errors and style issues
+lint:
+	ruff check scripts/ tests/
+
+# Format: auto-fix lint issues and reformat code
+format:
+	ruff check --fix scripts/ tests/
+	ruff format scripts/ tests/
+
+# Type check with mypy (strict mode)
+typecheck:
+	mypy scripts/token_reducer/ --strict
+
+# Run pyright for additional static analysis
+pyright:
+	pyright scripts/token_reducer/
+
+# Run all checks (lint + typecheck + tests) — use before committing
+check: lint typecheck test

pyproject.toml

@@ -0,0 +1,85 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "token-reducer"
+version = "0.1.0"
+description = "Local-first context slimming pipeline for Claude Code — cuts token usage without losing signal."
+readme = "README.md"
+license = { file = "LICENSE" }
+requires-python = ">=3.11"
+dependencies = [
+    "typer[all]>=0.12.0",
+    "pydantic>=2.0.0",
+]
+
+[project.optional-dependencies]
+ml = [
+    "sentence-transformers>=3.0.0",
+    "numpy>=1.26.0,<2.0.0",
+    "hnswlib>=0.8.0",
+    "faiss-cpu>=1.8.0",
+]
+ast = [
+    "tree-sitter>=0.21.0",
+    "tree-sitter-python>=0.21.0",
+    "tree-sitter-javascript>=0.21.0",
+    "tree-sitter-typescript>=0.21.0",
+    "tree-sitter-java>=0.21.0",
+    "tree-sitter-go>=0.21.0",
+    "tree-sitter-rust>=0.21.0",
+    "tree-sitter-c>=0.21.0",
+    "tree-sitter-cpp>=0.21.0",
+    "tree-sitter-ruby>=0.21.0",
+]
+full = ["token-reducer[ml,ast]"]
+dev = [
+    "pytest>=8.0",
+    "ruff>=0.4",
+    "mypy>=1.10",
+    "pyright>=1.1",
+]
+
+[project.scripts]
+token-reducer = "token_reducer.cli:main"
+
+[tool.hatch.build.targets.wheel]
+packages = ["scripts/token_reducer"]
+
+[tool.hatch.build.targets.sdist]
+include = ["scripts/token_reducer", "README.md", "LICENSE", "CHANGELOG.md"]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["scripts"]
+
+[tool.ruff]
+target-version = "py311"
+line-length = 100
+
+[tool.ruff.lint]
+select = ["E", "F", "W", "I", "UP", "B", "C4", "SIM"]
+ignore = [
+    "E501",   # line too long — ruff format handles this
+    "SIM108", # ternary operator — often less readable
+]
+
+[tool.ruff.lint.per-file-ignores]
+"tests/*" = ["SIM"]
+
+[tool.mypy]
+python_version = "3.11"
+strict = true
+warn_return_any = true
+warn_unused_ignores = true
+plugins = ["pydantic.mypy"]
+
+[[tool.mypy.overrides]]
+module = ["tree_sitter", "tree_sitter_*", "sentence_transformers", "hnswlib", "faiss"]
+ignore_missing_imports = true
+
+[tool.pydantic-mypy]
+init_forbid_extra = true
+init_typed = true
+warn_required_dynamic_aliases = true

scripts/token_reducer/chunker.py

@@ -557,6 +557,7 @@ def extract_function_calls(text: str) -> list[str]:
 def resolve_import_to_file(import_path: str, source_file: str, indexed_files: set[str]) -> str | None:
     """Attempt to resolve an import path to an actual indexed file."""
     source_dir = Path(source_file).parent
+    dotted_import_as_path = import_path.replace(".", "/")
 
     # Common resolution strategies
     candidates = [
@@ -568,8 +569,8 @@ def resolve_import_to_file(import_path: str, source_file: str, indexed_files: se
         str(source_dir / import_path / "index.ts"),
         str(source_dir / import_path / "index.js"),
         # Convert dot notation to path
-        str(source_dir / import_path.replace(".", "/") + ".py"),
-        str(source_dir / import_path.replace(".", "/") + ".ts"),
+        str(source_dir / f"{dotted_import_as_path}.py"),
+        str(source_dir / f"{dotted_import_as_path}.ts"),
     ]
 
     for candidate in candidates: