jobs.run_sweep

jobs.run_sweep(
    cli,
    job_set,
    *,
    registry=None,
    session_id=None,
    manage_status=True,
    remote=False,
    api_key=None,
    endpoint=None,
    batch_remote=False,
    target=None,
    batch_no_wait=False,
    poll_interval=10,
    batch_timeout=None,
    auto_ingest=True,
    collision_policy='fail',
    on_complete=None,
    stop_on_failure=True,
    dry_run=False,
    quiet=False,
    jfr=None,
    enable_profiler=False,
    stream_output=False,
    bottle=None,
    bottle_dir=None,
    bottle_omit_jshd=False,
)

Execute all jobs in a JobSet.

This is a convenience function that handles the common case of running all jobs in a sweep with optional progress output and registry tracking.

Parameters

Name	Type	Description	Default
cli	JoshCLI	JoshCLI instance to use for execution.	required
job_set	JobSet	Expanded jobs to run.	required
registry	RunRegistry | None	Optional RunRegistry for automatic run tracking. If provided along with session_id, runs are automatically recorded.	None
session_id	str | None	Session ID for registry tracking (required if registry provided).	None
manage_status	bool	If True (default) and registry/session_id provided, automatically manage session status lifecycle: - Set status to “running” at start - Set status to “completed” if all jobs succeed - Set status to “failed” if any job fails or on exception Set to False for manual status control (e.g., API use cases).	True
remote	bool	If True, use run_remote() for cloud execution.	False
api_key	str | None	Josh Cloud API key (required if remote=True).	None
endpoint	str | None	Custom Josh Cloud endpoint URL.	None
batch_remote	bool	If True, use batch_remote() for MinIO-staged execution. Mutually exclusive with remote.	False
target	str | None	Target profile name (required if batch_remote=True).	None
batch_no_wait	bool	If True, dispatch all jobs with --no-wait then poll for completion (async mode). If False (default), each job blocks until the JAR finishes polling internally (blocking mode).	False
poll_interval	int	Seconds between poll attempts in async mode (default: 10).	10
batch_timeout	int | None	Overall timeout in seconds per job for async polling.	None
auto_ingest	bool	If True (default), call ingest_results() after each successful batch job to load CSVs from S3 into the registry.	True
on_complete	Callable[[ExpandedJob, Any], None] | None	Optional callback invoked after each job completes. Signature: callback(job, result) -> None. Called after registry recording (if enabled). Use for progress reporting, logging, etc.	None
stop_on_failure	bool	If True (default), stop on first failure and raise SweepExecutionError with details about the failed job. If False, continue running remaining jobs and return partial results.	True
dry_run	bool	If True, print plan without executing.	False
quiet	bool	If True, suppress progress output.	False
jfr	JfrConfig | None	Optional JFR profiling configuration. When provided, each job gets its own recording file with the run_hash in the filename.	None
enable_profiler	bool	Enable Josh evaluation profiler (–enable-profiler).	False
stream_output	bool	If True, stream JAR stdout/stderr to the terminal in real time while still capturing them in CLIResult.	False

Returns

Name	Type	Description
	SweepResult	SweepResult with all job outcomes.

Raises

Name	Type	Description
	SweepExecutionError	If stop_on_failure=True and a job fails. Contains the failed job, CLI result with exit code/stderr, job index, and count of jobs that succeeded before the failure.
	ValueError	If remote=True but api_key is not provided.
	ValueError	If registry provided but session_id is not.

Examples

>>> from joshpy.jobs import run_sweep, JobExpander, JobConfig
>>> from joshpy.cli import JoshCLI
>>> from joshpy.registry import RunRegistry
>>> cli = JoshCLI()
>>> config = JobConfig(...)
>>> job_set = JobExpander().expand(config)
>>> registry = RunRegistry("experiment.duckdb")
>>> session_id = registry.create_session(config=config)

>>> # Register job configs
>>> for job in job_set:
...     registry.register_run(
...         session_id=session_id,
...         run_hash=job.run_hash,
...         josh_path=str(job.source_path),
...         config_content=job.config_content,
...         file_mappings=job.file_mappings,
...         parameters=job.parameters,
...     )

>>> # Run with automatic tracking (status managed automatically)
>>> results = run_sweep(cli, job_set, registry=registry, session_id=session_id)
>>> print(f"Completed: {results.succeeded} succeeded, {results.failed} failed")

>>> # Or run remotely on Josh Cloud:
>>> results = run_sweep(
...     cli, job_set,
...     registry=registry,
...     session_id=session_id,
...     remote=True,
...     api_key="your-api-key",
... )

>>> # Or without registry (just execute, no tracking):
>>> results = run_sweep(cli, job_set)