jobs.run_sweep

jobs.run_sweep(
    cli,
    job_set,
    *,
    registry=None,
    session_id=None,
    manage_status=True,
    remote=False,
    api_key=None,
    endpoint=None,
    batch_remote=False,
    target=None,
    batch_no_wait=False,
    poll_interval=10,
    batch_timeout=None,
    auto_ingest=True,
    collision_policy='fail',
    on_complete=None,
    stop_on_failure=True,
    dry_run=False,
    quiet=False,
    jfr=None,
    enable_profiler=False,
    stream_output=False,
    bottle=None,
    bottle_dir=None,
    bottle_omit_jshd=False,
)

Execute all jobs in a JobSet.

This is a convenience function that handles the common case of running all jobs in a sweep with optional progress output and registry tracking.

Parameters

Name	Type	Description	Default
cli	JoshCLI	JoshCLI instance to use for execution.	required
job_set	JobSet	Expanded jobs to run.	required
registry	RunRegistry \| None	Optional RunRegistry for automatic run tracking. If provided along with session_id, runs are automatically recorded.	`None`
session_id	str \| None	Session ID for registry tracking (required if registry provided).	`None`
manage_status	bool	If True (default) and registry/session_id provided, automatically manage session status lifecycle: - Set status to “running” at start - Set status to “completed” if all jobs succeed - Set status to “failed” if any job fails or on exception Set to False for manual status control (e.g., API use cases).	`True`
remote	bool	If True, use run_remote() for cloud execution.	`False`
api_key	str \| None	Josh Cloud API key (required if remote=True).	`None`
endpoint	str \| None	Custom Josh Cloud endpoint URL.	`None`
batch_remote	bool	If True, use batch_remote() for MinIO-staged execution. Mutually exclusive with `remote`.	`False`
target	str \| None	Target profile name (required if batch_remote=True).	`None`
batch_no_wait	bool	If True, dispatch all jobs with `--no-wait` then poll for completion (async mode). If False (default), each job blocks until the JAR finishes polling internally (blocking mode).	`False`
poll_interval	int	Seconds between poll attempts in async mode (default: 10).	`10`
batch_timeout	int \| None	Overall timeout in seconds per job for async polling.	`None`
auto_ingest	bool	If True (default), call `ingest_results()` after each successful batch job to load CSVs from S3 into the registry.	`True`
on_complete	Callable[[ExpandedJob, Any], None] \| None	Optional callback invoked after each job completes. Signature: callback(job, result) -> None. Called after registry recording (if enabled). Use for progress reporting, logging, etc.	`None`
stop_on_failure	bool	If True (default), stop on first failure and raise SweepExecutionError with details about the failed job. If False, continue running remaining jobs and return partial results.	`True`
dry_run	bool	If True, print plan without executing.	`False`
quiet	bool	If True, suppress progress output.	`False`
jfr	JfrConfig \| None	Optional JFR profiling configuration. When provided, each job gets its own recording file with the run_hash in the filename.	`None`
enable_profiler	bool	Enable Josh evaluation profiler (–enable-profiler).	`False`
stream_output	bool	If True, stream JAR stdout/stderr to the terminal in real time while still capturing them in CLIResult.	`False`
bottle	str \| None	Optional bottling mode — see `joshpy.bottle.BOTTLE_MODES`. `"receipt"` bottles every job (pass or fail) with no registry required: a self-contained archive per run holding the rendered config, resolved inputs, the exact JAR command, and exit code — for callers who want a portable, inspectable record of what joshpy did without adopting `RunRegistry`.	`None`
bottle_dir	Path \| None	Directory for bottle archives (default: `./bottles/`).	`None`
bottle_omit_jshd	bool	If True, skip copying `.jshd` data files into bottle archives (default: False).	`False`

Returns

Name	Type	Description
	SweepResult	SweepResult with all job outcomes.

Raises

Name	Type	Description
	SweepExecutionError	If stop_on_failure=True and a job fails. Contains the failed job, CLI result with exit code/stderr, job index, and count of jobs that succeeded before the failure.
	ValueError	If remote=True but api_key is not provided.
	ValueError	If registry provided but session_id is not.

Examples

>>> from joshpy.jobs import run_sweep, JobExpander, JobConfig
>>> from joshpy.cli import JoshCLI
>>> from joshpy.registry import RunRegistry
>>> cli = JoshCLI()
>>> config = JobConfig(...)
>>> job_set = JobExpander().expand(config)
>>> registry = RunRegistry("experiment.duckdb")
>>> session_id = registry.create_session(config=config)

>>> # Register job configs
>>> for job in job_set:
...     registry.register_run(
...         session_id=session_id,
...         run_hash=job.run_hash,
...         josh_path=str(job.source_path),
...         config_content=job.config_content,
...         file_mappings=job.file_mappings,
...         parameters=job.parameters,
...     )

>>> # Run with automatic tracking (status managed automatically)
>>> results = run_sweep(cli, job_set, registry=registry, session_id=session_id)
>>> print(f"Completed: {results.succeeded} succeeded, {results.failed} failed")

>>> # Or run remotely on Josh Cloud:
>>> results = run_sweep(
...     cli, job_set,
...     registry=registry,
...     session_id=session_id,
...     remote=True,
...     api_key="your-api-key",
... )

>>> # Or without registry (just execute, no tracking):
>>> results = run_sweep(cli, job_set)