Reorganize Python worker around shared package and app registry

.github/workflows/ci.yml

@@ -205,7 +205,6 @@ jobs:
         run: |
           docker build \
             -f ${{ matrix.dockerfile }} \
-            --build-arg PROJECT_NAME=${{ matrix.project }} \
             --build-arg SDK_VERSION=${PROJECT_IMAGE_SDK_VERSION} \
             -t ${PROJECT_IMAGE_TAG} .
       - name: Create Docker network
@@ -219,6 +218,8 @@ jobs:
             --network omes-project-net \
             -p 7233:7233 \
             ${PROJECT_IMAGE_TAG} \
+            run-worker \
+            --app ${{ matrix.project }} \
             --run-id ${PROJECT_RUN_ID} \
             --embedded-server-address 0.0.0.0:7233 &
       - name: Wait for embedded server
@@ -234,7 +235,10 @@ jobs:
             --run-id ${PROJECT_RUN_ID} \
             --server-address ${PROJECT_WORKER_CONTAINER}:7233 \
             --iterations 1 \
-            --connect-timeout 30s
+            --connect-timeout 30s \
+            --option language=${{ matrix.sdk }} \
+            --option project-name=${{ matrix.project }} \
+            --option prebuilt-project-dir=/app/workers/python/prepared
       - name: Print project worker logs
         if: failure()
         run: |

README.md

@@ -218,9 +218,9 @@ Writing a project should be fairly similar to writing a Temporal sample, and req
 familiarity with the project harness interface.
 
 To get started:
-- projects should be written at `workers/<lang>/projects/tests/<name>`, this is a path convention expected
-  by Omes when building your project
-- use the example `helloworld/` project as a reference to get started
+- projects should be written as apps under `workers/<lang>/apps/<name>`
+- register the app in `workers/<lang>/apps/registry.py`
+- use the example `workers/python/apps/helloworld/` app as a reference to get started
 - take a look at `workers/python/harness/src/harness/__init__.py` as an entrypoint to the harness and 
   how it works
 
@@ -246,7 +246,7 @@ To run a project:
 ```sh
 go run ./cmd run-worker \
   --language python \
-  --project-name helloworld \
+  --app helloworld \
   --run-id local-project-test \
   --server-address <your server address>
 ```
@@ -272,7 +272,7 @@ go run ./cmd run-scenario-with-worker \
   --scenario project \
   --iterations 1 \
   --language python \
-  --project-name helloworld \
+  --app helloworld \
   --run-id local-project-test \
   --embedded-server \
   --option language=python \
@@ -284,34 +284,39 @@ To run a project via Docker:
 ```sh
 docker build \
   -f dockerfiles/python.Dockerfile \
-  --build-arg PROJECT_NAME=helloworld \
   --build-arg SDK_VERSION=v1.25.0 \
-  -t omes-python-project-helloworld .
+  -t omes-python .
 
 docker network create omes-project-net
 
 docker run -d --rm \
   --name omes-python-project-worker \
   --network omes-project-net \
-  omes-python-project-helloworld \
+  omes-python \
+  run-worker \
+  --app helloworld \
   --run-id local-project-test \
   --embedded-server-address 0.0.0.0:7233
 
 docker run --rm \
   --network omes-project-net \
-  omes-python-project-helloworld \
+  omes-python \
   run-scenario \
   --scenario project \
   --run-id local-project-test \
   --server-address omes-python-project-worker:7233 \
   --iterations 1 \
-  --connect-timeout 30s
+  --connect-timeout 30s \
+  --option language=python \
+  --option project-name=helloworld \
+  --option prebuilt-project-dir=/app/workers/python/prepared
 
 docker stop omes-python-project-worker
 docker network rm omes-project-net
 ```
 
-This docker workflow it is not yet wired into `go run ./cmd/dev build-worker-image`.
+The Python image is a single prepared worker package. Select the app at runtime
+with `--app` for workers and `--option project-name=...` for project scenarios.
 
 ### ThroughputStress
 

dockerfiles/entrypoint.sh

@@ -1,27 +1,17 @@
 #!/bin/sh
 set -eu
 
-if [ "${1:-}" = "run-scenario" ]; then
-  shift
-  if [ -n "${OMES_PROJECT_NAME:-}" ] && [ -n "${OMES_PROJECT_PREBUILT_DIR:-}" ]; then
-    exec /app/temporal-omes run-scenario "$@" \
-      --option "language=${OMES_WORKER_LANGUAGE}" \
-      --option "prebuilt-project-dir=${OMES_PROJECT_PREBUILT_DIR}"
-  fi
-  exec /app/temporal-omes run-scenario "$@"
-fi
-
-if [ "${1:-}" = "run-worker" ]; then
-  shift
-fi
-
-if [ -n "${OMES_PROJECT_NAME:-}" ]; then
-  exec /app/temporal-omes run-worker \
-    --language "${OMES_WORKER_LANGUAGE}" \
-    --project-name "${OMES_PROJECT_NAME}" \
-    --dir-name "${OMES_PROJECT_PREPARED_DIR}" \
-    "$@"
-fi
+# This image normally behaves like the other worker images: callers pass worker
+# flags and we add run-worker plus the prepared Python package defaults. Explicit
+# omes subcommands pass through so the same image can run project scenarios.
+case "${1:-}" in
+  run-scenario|cleanup-scenario|list-scenarios|prepare-worker|run-scenario-with-worker|completion|help)
+    exec /app/temporal-omes "$@"
+    ;;
+  run-worker)
+    shift
+    ;;
+esac
 
 exec /app/temporal-omes run-worker \
   --language "${OMES_WORKER_LANGUAGE}" \

dockerfiles/python.Dockerfile

@@ -40,7 +40,6 @@ COPY go.mod go.sum ./
 RUN CGO_ENABLED=0 /usr/local/go/bin/go build -o temporal-omes ./cmd
 
 ARG SDK_VERSION
-ARG PROJECT_NAME=""
 
 # Optional SDK dir to copy, defaults to unimportant file
 ARG SDK_DIR=.gitignore
@@ -49,18 +48,12 @@ COPY ${SDK_DIR} ./repo
 # Copy the worker files
 COPY workers/python ./workers/python
 
-# Build the worker or project runner
-RUN if [ -n "$PROJECT_NAME" ]; then \
-      CGO_ENABLED=0 ./temporal-omes prepare-worker --language python --project-name "$PROJECT_NAME" --dir-name "project-build-runner-$PROJECT_NAME" --version "$SDK_VERSION" ; \
-    else \
-      CGO_ENABLED=0 ./temporal-omes prepare-worker --language python --dir-name prepared --version "$SDK_VERSION" ; \
-    fi
+# Build one prepared package that can run any Python app via --app.
+RUN CGO_ENABLED=0 ./temporal-omes prepare-worker --language python --dir-name prepared --version "$SDK_VERSION"
 
-# Copy the CLI and built worker to a distroless "run" container
+# Copy the CLI and built worker to a run container
 FROM --platform=linux/$TARGETARCH python:3.11-slim-bullseye
 
-ARG PROJECT_NAME=""
-
 COPY --from=uv /uv /uvx /bin/
 COPY --from=build /app/temporal-omes /app/temporal-omes
 COPY --from=build /app/workers/python /app/workers/python
@@ -70,8 +63,5 @@ RUN chmod +x /app/entrypoint.sh
 ENV UV_NO_SYNC=1 UV_FROZEN=1 UV_OFFLINE=1
 ENV OMES_WORKER_LANGUAGE=python
 ENV OMES_PREPARED_DIR=prepared
-ENV OMES_PROJECT_NAME=$PROJECT_NAME
-ENV OMES_PROJECT_PREPARED_DIR=project-build-runner-${PROJECT_NAME}
-ENV OMES_PROJECT_PREBUILT_DIR=/app/workers/python/projects/tests/${PROJECT_NAME}/project-build-runner-${PROJECT_NAME}
 
 ENTRYPOINT ["/app/entrypoint.sh"]

scenarios/project/build.go

@@ -13,10 +13,9 @@ import (
 // buildProject builds a project test program for the given language.
 func buildProject(ctx context.Context, repoRoot string, p projectScenarioOptions, logger *zap.SugaredLogger) (sdkbuild.Program, error) {
 	b := workers.Builder{
-		DirName:     fmt.Sprintf("project-build-runner-%s", p.projectName),
-		SdkOptions:  p.sdkOpts,
-		ProjectName: p.projectName,
-		Logger:      logger,
+		DirName:    fmt.Sprintf("project-build-runner-%s", p.projectName),
+		SdkOptions: p.sdkOpts,
+		Logger:     logger,
 	}
 
 	baseDir := workers.BaseDir(repoRoot, p.sdkOpts.Language)

scenarios/project/project.go

@@ -22,8 +22,8 @@ func init() {
 	loadgen.MustRegisterScenario(loadgen.Scenario{
 		Description: `Run a self-contained project test. Builds (or loads) the project program, spawns a project-server, and drives iterations via gRPC.
   Required: --option language=<lang>
-  One of:   --option project-name=<name>  (build from source; optional --option version=<version> override)
-            --option prebuilt-project-dir=<path>  (use pre-built project dir)
+  Required: --option project-name=<name>  (app entrypoint; optional --option version=<version> override)
+  Optional: --option prebuilt-project-dir=<path>  (use pre-built root worker package)
   Optional: --option project-config-file=<path>   (project-specific JSON config)
   Optional: --option project-server-ready-timeout=<duration> (timeout to connect to project-server, default 15s)
   See README.md ("Project" section) for local usage examples and current limitations.`,
@@ -71,7 +71,7 @@ func (e *projectScenarioExecutor) Run(ctx context.Context, info loadgen.Scenario
 	serverCtx, serverCancel := context.WithCancel(ctx)
 	defer serverCancel()
 
-	serverCmd, err := startProjectProcess(serverCtx, prog, info.Logger, opts.sdkOpts.Language, port)
+	serverCmd, err := startProjectProcess(serverCtx, prog, info.Logger, opts.sdkOpts.Language, opts.projectName, port)
 	if err != nil {
 		return fmt.Errorf("failed to spawn project server: %w", err)
 	}
@@ -119,25 +119,22 @@ func (e *projectScenarioExecutor) validate(info loadgen.ScenarioInfo) (projectSc
 
 	projectName := info.ScenarioOptions["project-name"]
 	prebuiltDir := info.ScenarioOptions["prebuilt-project-dir"]
-	if projectName == "" && prebuiltDir == "" {
-		return opts, fmt.Errorf("either --option project-name or --option prebuilt-project-dir is required")
-	}
-	if projectName != "" && prebuiltDir != "" {
-		return opts, fmt.Errorf("cannot specify both project-name and prebuilt-project-dir")
+	if projectName == "" {
+		return opts, fmt.Errorf("--option project-name=<name> is required")
 	}
+	opts.projectName = projectName
 
 	if prebuiltDir != "" {
 		abs, err := filepath.Abs(prebuiltDir)
 		if err != nil {
 			return opts, fmt.Errorf("failed to resolve prebuilt-project-dir: %w", err)
 		}
 		opts.prebuiltDir = abs
-	} else {
-		opts.projectName = projectName
-		version := info.ScenarioOptions["version"]
-		if version != "" {
-			opts.sdkOpts.Version = version
-		}
+	}
+
+	version := info.ScenarioOptions["version"]
+	if version != "" && opts.prebuiltDir == "" {
+		opts.sdkOpts.Version = version
 	}
 
 	if configPath := info.ScenarioOptions["project-config-file"]; configPath != "" {
@@ -170,11 +167,11 @@ func findAvailablePort() (int, error) {
 	return port, nil
 }
 
-func startProjectProcess(ctx context.Context, prog sdkbuild.Program, logger *zap.SugaredLogger, lang clioptions.Language, port int) (*exec.Cmd, error) {
+func startProjectProcess(ctx context.Context, prog sdkbuild.Program, logger *zap.SugaredLogger, lang clioptions.Language, appName string, port int) (*exec.Cmd, error) {
 	var args []string
 	// Python needs module name
 	if lang == clioptions.LangPython {
-		args = append(args, "main")
+		args = append(args, "apps.registry", "--app", appName)
 	}
 	args = append(args, "project-server", "--port", strconv.Itoa(port))
 	cmd, err := prog.NewCommand(ctx, args...)

scenarios/project/project_test.go

@@ -37,15 +37,14 @@ func TestValidateLimitedPythonSupport(t *testing.T) {
 	require.EqualError(t, err, "project scenario is currently limited to Python, got go")
 }
 
-func TestValidateRejectsConflictingProjectSources(t *testing.T) {
+func TestValidateRequiresProjectNameWithPrebuilt(t *testing.T) {
 	_, err := (&projectScenarioExecutor{}).validate(loadgen.ScenarioInfo{
 		ScenarioOptions: map[string]string{
 			"language":             "python",
-			"project-name":         "helloworld",
-			"prebuilt-project-dir": "workers/python/projects/tests/project-build-helloworld",
+			"prebuilt-project-dir": "workers/python/project-build-runner-helloworld",
 		},
 	})
-	require.EqualError(t, err, "cannot specify both project-name and prebuilt-project-dir")
+	require.EqualError(t, err, "--option project-name=<name> is required")
 }
 
 func TestPythonHelloWorldSourceBuild(t *testing.T) {
@@ -75,7 +74,6 @@ func runProjectScenario(
 	if usePrebuilt {
 		prog, err = buildProject(ctx, info.RootPath, opts, info.Logger)
 		require.NoError(t, err)
-		info.ScenarioOptions["project-name"] = ""
 		info.ScenarioOptions["prebuilt-project-dir"] = prog.Dir()
 	}
 
@@ -195,9 +193,8 @@ func startProjectWorker(
 	require.NotEmpty(t, opts.projectName)
 
 	builder := workers.Builder{
-		ProjectName: opts.projectName,
-		SdkOptions:  opts.sdkOpts,
-		Logger:      info.Logger.Named(fmt.Sprintf("%s-worker-builder", opts.sdkOpts.Language)),
+		SdkOptions: opts.sdkOpts,
+		Logger:     info.Logger.Named(fmt.Sprintf("%s-worker-builder", opts.sdkOpts.Language)),
 	}
 
 	// If we have a prebuilt program, use it
@@ -208,6 +205,7 @@ func startProjectWorker(
 
 	runner := &workers.Runner{
 		Builder:                  builder,
+		AppName:                  opts.projectName,
 		TaskQueueName:            loadgen.TaskQueueForRun(info.RunID),
 		GracefulShutdownDuration: 5 * time.Second,
 		ScenarioID: clioptions.ScenarioID{

workers/python/apps/__init__.py

@@ -0,0 +1 @@
+

workers/python/apps/helloworld/__init__.py

@@ -0,0 +1 @@
+

workers/python/projects/tests/helloworld/app.py → workers/python/apps/helloworld/app.py

@@ -2,7 +2,6 @@
 
 from temporalio.client import Client
 from temporalio.worker import Worker
-from workflow import HelloWorldWorkflow
 
 from harness import (
     App,
@@ -12,13 +11,7 @@
     default_client_factory,
 )
 
-
-def app() -> App:
-    return App(
-        worker=build_worker,
-        client_factory=default_client_factory,
-        project=ProjectHandlers(execute=execute_project_iteration),
-    )
+from .workflow import HelloWorldWorkflow
 
 
 def build_worker(client: Client, context: WorkerContext) -> Worker:
@@ -41,3 +34,10 @@ async def execute_project_iteration(
     )
     result = await handle.result()
     print(result)
+
+
+app = App(
+    worker=build_worker,
+    client_factory=default_client_factory,
+    project=ProjectHandlers(execute=execute_project_iteration),
+)

workers/python/apps/registry.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+import argparse
+import sys
+from collections.abc import Sequence
+
+from apps.helloworld.app import app as helloworld_app
+from apps.worker.app import app as worker_app
+from harness import App, run
+
+DEFAULT_APP_NAME = "worker"
+
+registry: dict[str, App] = {
+    DEFAULT_APP_NAME: worker_app,
+    "helloworld": helloworld_app,
+}
+
+
+def main(argv: Sequence[str] | None = None) -> None:
+    parser = argparse.ArgumentParser(add_help=False, allow_abbrev=False)
+    parser.add_argument("--app", default=DEFAULT_APP_NAME)
+    args, remaining = parser.parse_known_args(sys.argv[1:] if argv is None else argv)
+
+    app = registry.get(args.app)
+    if app is None:
+        raise SystemExit(f"unknown Python worker app {args.app!r}")
+
+    run(app, remaining)
+
+
+if __name__ == "__main__":
+    main()

workers/python/apps/worker/__init__.py

@@ -0,0 +1 @@
+