fix up python packaging

.gitignore

@@ -13,8 +13,9 @@ a.out
 .#*
 dist/
 build/
-pandacan.egg-info/
+*.egg-info/
 obj/
+panda/fw/
 examples/output.csv
 .DS_Store
 .vscode*

Dockerfile

@@ -7,9 +7,9 @@ ENV PATH="$WORKDIR/.venv/bin:$PATH"
 WORKDIR $WORKDIR
 
 # deps install
-COPY pyproject.toml __init__.py setup.sh $WORKDIR
-RUN mkdir -p $WORKDIR/python/ $WORKDIR/board/body/ $WORKDIR/board/jungle/ && \
-    touch $WORKDIR/__init__.py $WORKDIR/board/__init__.py $WORKDIR/board/body/__init__.py $WORKDIR/board/jungle/__init__.py
+COPY pyproject.toml setup.py setup.sh $WORKDIR
+RUN mkdir -p $WORKDIR/panda/body/ $WORKDIR/panda/jungle/ && \
+    touch $WORKDIR/panda/__init__.py $WORKDIR/panda/body/__init__.py $WORKDIR/panda/jungle/__init__.py
 RUN apt-get update && apt-get install -y --no-install-recommends sudo && DEBIAN_FRONTEND=noninteractive $WORKDIR/setup.sh
 
 # second pass for the opendbc moving tag

MANIFEST.in

@@ -0,0 +1,4 @@
+include SConstruct SConscript
+graft board
+prune board/obj
+global-exclude __pycache__ *.pyc *.o *.os *.d *.elf *.bin *.so

README.md

@@ -7,7 +7,7 @@ panda speaks CAN and CAN FD, and it runs on the [STM32H725](https://www.st.com/r
 ```
 .
 ├── board           # Code that runs on the STM32
-├── python          # Python userspace library for interfacing with the panda
+├── panda           # Python userspace library for interfacing with the panda
 ├── tests           # Tests for panda
 ├── scripts         # Miscellaneous used for panda development and debugging
 ├── examples        # Example scripts for using a panda in a car
@@ -50,7 +50,7 @@ cd panda
 ./test.sh
 ```
 
-See [the Panda class](https://github.com/commaai/panda/blob/master/python/__init__.py) for how to interact with the panda.
+See [the Panda class](https://github.com/commaai/panda/blob/master/panda/__init__.py) for how to interact with the panda.
 
 For example, to receive CAN messages:
 ``` python

SConscript

@@ -27,7 +27,7 @@ def objcopy(source, target, env, for_signature):
 def get_version(builder, build_type):
   try:
     git = subprocess.check_output(["git", "rev-parse", "--short=8", "HEAD"], encoding='utf8').strip()
-  except subprocess.CalledProcessError:
+  except (subprocess.CalledProcessError, OSError):
     git = "unknown"
   return f"{builder}-{git}-{build_type}"
 
@@ -110,7 +110,7 @@ def build_project(project_name, project, main, extra_flags):
     "./board/crypto/sha.c",
     "./board/bootstub.c",
   ])
-  bs_env.Objcopy(f"./board/obj/bootstub.{project_name}.bin", bs_elf)
+  bs_env.Objcopy(f"./panda/fw/bootstub.{project_name}.bin", bs_elf)
 
   # Build + sign main (aka app)
   main_elf = env.Program(f"{project_dir}/main.elf", [
@@ -119,7 +119,7 @@ def build_project(project_name, project, main, extra_flags):
   ], LINKFLAGS=[f"-Wl,--section-start,.isr_vector={project['APP_START_ADDRESS']}"] + flags)
   main_bin = env.Objcopy(f"{project_dir}/main.bin", main_elf)
   sign_py = File(f"./board/crypto/sign.py").srcnode().relpath
-  env.Command(f"./board/obj/{project_name}.bin.signed", main_bin, f"SETLEN=1 {sign_py} $SOURCE $TARGET {cert_fn}")
+  env.Command(f"./panda/fw/{project_name}.bin.signed", main_bin, f"SETLEN=1 {sign_py} $SOURCE $TARGET {cert_fn}")
 
 
 
@@ -138,6 +138,7 @@ base_project_h7 = {
 }
 
 # Common autogenerated includes
+os.makedirs("board/obj", exist_ok=True)
 with open("board/obj/gitversion.h", "w") as f:
   version = get_version(BUILDER, BUILD_TYPE)
   f.write(f'extern const uint8_t gitversion[{len(version)+1}];\n')
@@ -156,7 +157,7 @@ def version_hash(path):
   with open(path, "rb") as f:
     # Normalize line endings on Windows
     return int.from_bytes(hashlib.sha256(f.read().replace(b'\r', b'')).digest()[:4], 'little')
-hh, ch, jh = version_hash("board/health.h"), version_hash(os.path.join(opendbc.INCLUDE_PATH, "opendbc/safety/can.h")), version_hash("board/jungle/jungle_health.h")
+hh, ch, jh = version_hash("panda/health.h"), version_hash(os.path.join(opendbc.INCLUDE_PATH, "opendbc/safety/can.h")), version_hash("panda/jungle/jungle_health.h")
 common_flags += [f"-DHEALTH_PACKET_VERSION=0x{hh:08X}U", f"-DCAN_PACKET_VERSION_HASH=0x{ch:08X}U",
                  f"-DJUNGLE_HEALTH_PACKET_VERSION=0x{jh:08X}U"]
 
@@ -172,5 +173,6 @@ build_project("panda_jungle_h7", base_project_h7, "./board/jungle/main.c", flags
 # body fw
 build_project("body_h7", base_project_h7, "./board/body/main.c", ["-DPANDA_BODY"])
 
-# test files
-SConscript('tests/libpanda/SConscript')
+# test files (not present in the package sdist)
+if os.path.exists(File('tests/libpanda/SConscript').srcnode().abspath):
+  SConscript('tests/libpanda/SConscript')

board/body/can.h

@@ -4,7 +4,7 @@
 #include <stdint.h>
 
 #include "board/can.h"
-#include "board/health.h"
+#include "panda/health.h"
 #include "board/body/boards/board_declarations.h"
 #include "board/drivers/drivers.h"
 #include "opendbc/safety/declarations.h"

board/body/flash.py

@@ -8,12 +8,12 @@
 BODY_DIR = os.path.dirname(os.path.realpath(__file__))
 BOARD_DIR = os.path.abspath(os.path.join(BODY_DIR, ".."))
 REPO_ROOT = os.path.abspath(os.path.join(BOARD_DIR, ".."))
-DEFAULT_FIRMWARE = os.path.join(BOARD_DIR, "obj", "body_h7.bin.signed")
+DEFAULT_FIRMWARE = os.path.join(REPO_ROOT, "panda", "fw", "body_h7.bin.signed")
 
 
 def build_body() -> None:
   subprocess.check_call(
-    f"scons -C {REPO_ROOT} -j$(nproc) board/obj/body_h7.bin.signed",
+    f"scons -C {REPO_ROOT} -j$(nproc) panda/fw/body_h7.bin.signed",
     shell=True,
   )
 

board/debug/dfu_util_h7.sh

@@ -5,7 +5,7 @@ DFU_UTIL="dfu-util"
 
 scons -u -j$(nproc)
 
-PYTHONPATH=.. python3 -c "from python import Panda; Panda().reset(enter_bootstub=True); Panda().reset(enter_bootloader=True)" || true
+PYTHONPATH=.. python3 -c "from panda import Panda; Panda().reset(enter_bootstub=True); Panda().reset(enter_bootloader=True)" || true
 sleep 1
 $DFU_UTIL -d 0483:df11 -a 0 -s 0x08020000 -D obj/panda_h7.bin.signed
 $DFU_UTIL -d 0483:df11 -a 0 -s 0x08000000:leave -D obj/bootstub.panda_h7.bin

board/drivers/drivers.h

@@ -1,7 +1,7 @@
 #pragma once
 
 #include "board/can.h"
-#include "board/health.h"
+#include "panda/health.h"
 #include "board/crc.h"
 #ifdef STM32H7
 #include "board/stm32h7/lladc_declarations.h"

board/flash.py

@@ -12,7 +12,7 @@
   parser.add_argument("--all", action="store_true", help="Recover all Panda devices")
   args = parser.parse_args()
 
-  subprocess.check_call(f"scons -C {board_path}/.. -j$(nproc) {board_path}", shell=True)
+  subprocess.check_call(f"scons -C {board_path}/.. -j$(nproc) panda/fw", shell=True)
 
   if args.all:
     serials = Panda.list()

board/jungle/flash.py

@@ -12,7 +12,7 @@
   parser.add_argument("--all", action="store_true", help="Recover all panda jungle devices")
   args = parser.parse_args()
 
-  subprocess.check_call(f"scons -C {board_path}/.. -u -j$(nproc) .", shell=True)
+  subprocess.check_call(f"scons -C {board_path}/../.. -j$(nproc) panda/fw", shell=True)
 
   if args.all:
     serials = PandaJungle.list()

board/jungle/main.c

@@ -10,8 +10,8 @@
 #include "board/early_init.h"
 #include "board/provision.h"
 
-#include "board/health.h"
-#include "board/jungle/jungle_health.h"
+#include "panda/health.h"
+#include "panda/jungle/jungle_health.h"
 
 #include "board/drivers/can_common.h"
 

board/jungle/recover.py

@@ -13,7 +13,7 @@
   parser.add_argument("--all", action="store_true", help="Recover all panda jungle devices")
   args = parser.parse_args()
 
-  subprocess.check_call(f"scons -C {board_path}/.. -u -j$(nproc) .", shell=True)
+  subprocess.check_call(f"scons -C {board_path}/../.. -j$(nproc) panda/fw", shell=True)
 
   serials = PandaJungle.list() if args.all else [None]
   for s in serials:

board/main.c

@@ -12,7 +12,7 @@
 
 #include "opendbc/safety/safety.h"
 
-#include "board/health.h"
+#include "panda/health.h"
 
 #include "board/drivers/can_common.h"
 

board/recover.py

@@ -8,7 +8,7 @@
 board_path = os.path.dirname(os.path.realpath(__file__))
 
 if __name__ == "__main__":
-  subprocess.check_call(f"scons -C {board_path}/.. -j$(nproc) {board_path}", shell=True)
+  subprocess.check_call(f"scons -C {board_path}/.. -j$(nproc) panda/fw", shell=True)
 
   for s in Panda.list():
     with Panda(serial=s) as p:

python/__init__.py → panda/__init__.py

@@ -15,9 +15,9 @@
 from opendbc.car.structs import CarParams
 
 from .base import BaseHandle
-from .constants import BASEDIR, FW_PATH, McuType, compute_version_hash
+from .constants import BASEDIR, FW_PATH, McuType, USBPACKET_MAX_SIZE, compute_version_hash  # noqa: F401
 from .dfu import PandaDFU
-from .spi import PandaSpiHandle, PandaSpiException, PandaProtocolMismatch
+from .spi import PandaSpiHandle, PandaSpiException, PandaProtocolMismatch, STBootloaderSPIHandle  # noqa: F401
 from .usb import PandaUsbHandle
 from .utils import logger
 
@@ -137,8 +137,8 @@ class Panda:
   HW_TYPE_BODY = b'\xb1'
 
   CAN_PACKET_VERSION = compute_version_hash(os.path.join(opendbc.INCLUDE_PATH, "opendbc/safety/can.h"))
-  HEALTH_PACKET_VERSION = compute_version_hash(os.path.join(BASEDIR, "board/health.h"))
-  HEALTH_STRUCT = _parse_c_struct(os.path.join(BASEDIR, "board/health.h"), "health_t")
+  HEALTH_PACKET_VERSION = compute_version_hash(os.path.join(os.path.dirname(__file__), "health.h"))
+  HEALTH_STRUCT = _parse_c_struct(os.path.join(os.path.dirname(__file__), "health.h"), "health_t")
   CAN_HEALTH_STRUCT = struct.Struct("<BIBBBBBBBBIIIIIIIHHBBBIIII")
 
   H7_DEVICES = [HW_TYPE_RED_PANDA, HW_TYPE_TRES, HW_TYPE_CUATRO, HW_TYPE_BODY]
@@ -805,3 +805,8 @@ def force_relay_drive(self, intercept_relay_drive, ignition_relay_drive):
   def read_som_gpio(self) -> bool:
     r = self._handle.controlRead(Panda.REQUEST_IN, 0xc6, 0, 0, 1)
     return r[0] == 1
+
+
+from .serial import PandaSerial  # noqa: F401
+from .jungle import PandaJungle, PandaJungleDFU  # noqa: F401
+from .body import PandaBody  # noqa: F401

python/constants.py → panda/constants.py

@@ -4,7 +4,7 @@
 from typing import NamedTuple
 
 BASEDIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../")
-FW_PATH = os.path.join(BASEDIR, "board/obj/")
+FW_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "fw/")
 
 def compute_version_hash(filepath):
   with open(filepath, "rb") as f:

board/jungle/__init__.py → panda/jungle/__init__.py

@@ -4,10 +4,9 @@
 from functools import wraps
 
 from panda import Panda, PandaDFU
-from panda.python.constants import McuType, compute_version_hash
+from panda.constants import FW_PATH, McuType, compute_version_hash
 
 BASEDIR = os.path.dirname(os.path.realpath(__file__))
-FW_PATH = os.path.join(BASEDIR, "../obj/")
 
 
 def ensure_jungle_health_packet_version(fn):

pyproject.toml

@@ -1,5 +1,5 @@
 [project]
-name = "pandacan"
+name = "panda"
 version = "0.0.10"
 description = "Code powering the comma.ai panda"
 readme = "README.md"
@@ -35,24 +35,22 @@ dev = [
 ]
 
 [build-system]
-requires = ["setuptools>=61", "wheel"]
+requires = [
+  "setuptools>=64",
+  # setup.py builds the firmware into the wheel (not supported on Windows)
+  "scons; platform_system != 'Windows'",
+  "pycryptodome >= 3.9.8; platform_system != 'Windows'",
+  "opendbc @ git+https://github.com/commaai/opendbc.git@master ; platform_system != 'Windows'",
+  "gcc-arm-none-eabi @ git+https://github.com/commaai/dependencies.git@release-gcc-arm-none-eabi#subdirectory=gcc-arm-none-eabi ; platform_system != 'Windows'",
+]
 build-backend = "setuptools.build_meta"
 
-[tool.setuptools]
-packages = [
-  "panda",
-  "panda.python",
-  "panda.board",
-  "panda.board.body",
-  "panda.board.jungle",
-]
+[tool.setuptools.packages.find]
+include = ["panda", "panda.*"]
 
 [tool.setuptools.package-data]
-"panda.board" = ["health.h"]
-"panda.board.jungle" = ["jungle_health.h"]
-
-[tool.setuptools.package-dir]
-panda = "."
+panda = ["health.h", "fw/*"]
+"panda.jungle" = ["jungle_health.h"]
 
 # https://beta.ruff.rs/docs/configuration/#using-pyprojecttoml
 [tool.ruff]
@@ -69,6 +67,7 @@ flake8-implicit-str-concat.allow-multiline=false
 
 [tool.pytest.ini_options]
 addopts = "-Werror --strict-config --strict-markers --maxprocesses=8 -nauto --durations=10 --ignore-glob='*.sh' --ignore=tests/som --ignore=tests/hitl"
+pythonpath = ["."]
 python_files = "test_*.py"
 testpaths = [
   "tests/"

scripts/benchmark.py

@@ -6,8 +6,11 @@
 import cProfile
 from contextlib import contextmanager
 
+import sys
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
 from panda import Panda, PandaDFU
-from panda.tests.hitl.helpers import get_random_can_messages
+from tests.hitl.helpers import get_random_can_messages
 
 
 PROFILE = "PROFILE" in os.environ

scripts/test_canfd.py

@@ -1,12 +1,15 @@
 #!/usr/bin/env python3
 import os
+import sys
 import time
 import random
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
 from collections import defaultdict
 from opendbc.car.structs import CarParams
 from panda import Panda, calculate_checksum, DLC_TO_LEN
 from panda import PandaJungle
-from panda.tests.hitl.helpers import time_many_sends
+from tests.hitl.helpers import time_many_sends
 
 H7_HW_TYPES = [Panda.HW_TYPE_RED_PANDA, Panda.HW_TYPE_RED_PANDA_V2]
 JUNGLE_SERIAL = os.getenv("JUNGLE")

setup.py

@@ -0,0 +1,19 @@
+import os
+import platform
+import subprocess
+
+from setuptools import setup
+from setuptools.command.build_py import build_py
+
+
+class BuildPyWithFirmware(build_py):
+  def run(self):
+    # Build the firmware into panda/fw/ so it ships with the package.
+    # Skipped for editable installs, where scons is run directly,
+    # and on Windows, where the toolchain isn't available.
+    if not getattr(self, "editable_mode", False) and platform.system() != "Windows":
+      subprocess.check_call(["scons", f"-j{os.cpu_count()}", "panda/fw"], cwd=os.path.dirname(os.path.abspath(__file__)))
+    super().run()
+
+
+setup(cmdclass={"build_py": BuildPyWithFirmware})

tests/hitl/3_usb.py

@@ -4,7 +4,7 @@
 
 from opendbc.car.structs import CarParams
 from panda import Panda
-from panda.tests.hitl.helpers import time_many_sends
+from tests.hitl.helpers import time_many_sends
 
 pytestmark = [
   pytest.mark.test_panda_types((Panda.HW_TYPE_RED_PANDA, ))

tests/hitl/4_can_loopback.py

@@ -7,7 +7,7 @@
 from collections import defaultdict
 
 from opendbc.car.structs import CarParams
-from panda.tests.hitl.helpers import time_many_sends, get_random_can_messages, clear_can_buffers
+from tests.hitl.helpers import time_many_sends, get_random_can_messages, clear_can_buffers
 
 @flaky(max_runs=3, min_passes=1)
 @pytest.mark.timeout(35)