cmd_queue.cli_boilerplate module

This file defines a helper scriptconfig base config that can be used to help make cmd_queue CLIs so cmd_queue options are standardized and present at the top level.

CommandLine

xdoctest -m cmd_queue.cli_boilerplate __doc__:0

Example

>>> from cmd_queue.cli_boilerplate import CMDQueueConfig
>>> import scriptconfig as scfg
>>> import rich
>>> #
>>> class MyQueueCLI(CMDQueueConfig):
>>>     'A custom CLI that includes the cmd-queue boilerplate'
>>>     my_input_file = scfg.Value(None, help='some custom param')
>>>     my_num_steps = scfg.Value(3, help='some custom param')
>>>     is_small = scfg.Value(False, help='some custom param')
>>>     my_output_file = scfg.Value(None, help='some custom param')
>>> #
>>> def my_cli_main(cmdline=1, **kwargs):
>>>     config = MyQueueCLI.cli(cmdline=cmdline, data=kwargs)
>>>     rich.print('config = {}'.format(ub.urepr(config, nl=1)))
>>>     queue = config.create_queue()
>>>     #
>>>     ###
>>>     # Custom code to submit jobs to the queue
>>>     #
>>>     job0 = queue.submit(f'echo "processing input file: {config.my_input_file}"', name='ROOT-INPUT-JOB')
>>>     #
>>>     independent_outputs = []
>>>     for idx in range(config.my_num_steps):
>>>         job_t1 = queue.submit(f'echo "tree {idx}.S"', depends=[job0], name=f'jobname{idx}.1')
>>>         if not config.is_small:
>>>             job_t2 = queue.submit(f'echo "tree {idx}.SL"', depends=[job_t1], name=f'jobname{idx}.2')
>>>             job_t3 = queue.submit(f'echo "tree {idx}.SR"', depends=[job_t2], name=f'jobname{idx}.3')
>>>             job_t4 = queue.submit(f'echo "tree {idx}.SRR"', depends=[job_t3], name=f'jobname{idx}.4')
>>>             job_t5 = queue.submit(f'echo "tree {idx}.SRL"', depends=[job_t3], name=f'jobname{idx}.5')
>>>             job_t6 = queue.submit(f'echo "tree {idx}.T"', depends=[job_t4, job_t5], name=f'jobname{idx}.6')
>>>             job_t7 = queue.submit(f'echo "tree {idx}.SLT"', depends=[job_t2], name=f'jobname{idx}.7')
>>>             independent_outputs.extend([job_t6, job_t2])
>>>         else:
>>>             independent_outputs.extend([job_t1])
>>>     #
>>>     queue.submit(f'echo "processing output file: {config.my_output_file}"', depends=independent_outputs, name='FINAL-OUTPUT-JOB')
>>>     ###
>>>     #
>>>     config.run_queue(queue)
>>> #
>>> # Show what happens when you use the serial backend
>>> print('-------------------')
>>> print('--- DEMO SERIAL ---')
>>> print('-------------------')
>>> my_cli_main(
>>>     cmdline=0,
>>>     run=0,
>>>     print_queue=1,
>>>     print_commands=1,
>>>     backend='serial'
>>> )
>>> # Show what happens when you use the tmux backend
>>> print('-----------------')
>>> print('--- DEMO TMUX ---')
>>> print('-----------------')
>>> my_cli_main(
>>>     cmdline=0,
>>>     run=0,
>>>     print_queue=0,
>>>     is_small=True,
>>>     my_num_steps=0,
>>>     print_commands=1,
>>>     backend='tmux'
>>> )
>>> # Show what happens when you use the slurm backend
>>> print('------------------')
>>> print('--- DEMO SLURM ---')
>>> print('------------------')
>>> my_cli_main(
>>>     cmdline=0,
>>>     run=0, backend='slurm',
>>>     print_commands=1,
>>>     print_queue=False,
>>>     slurm_options='''
>>>         partition: 'general-gpu'
>>>         account: 'default'
>>>         ntasks: 1
>>>         gres: 'gpu:1'
>>>         cpus_per_task: 4
>>>     '''
>>> )
>>> # xdoctest: +REQUIRES(--run)
>>> # Actually run with the defaults
>>> print('----------------')
>>> print('--- DEMO RUN ---')
>>> print('----------------')
>>> my_cli_main(cmdline=0, run=1, print_queue=0, print_commands=0)

class cmd_queue.cli_boilerplate.CMDQueueConfig(*args, **kwargs)

Bases: DataConfig

A helper to carry around the common boilerplate for cmd-queue CLI’s. The general usage is that you will inherit from this class and define config options your CLI cares about, however they must not overload any of the options specified here.

Usage will be to call CMDQueueConfig.create_queue() to initialize a queue based on these options, and then execute it with CMDQueueConfig.run_queue(). In this way you do not need to worry about this specific boilerplate when writing your application. See cmd_queue.cli_boilerplate __doc__:0 for example usage.

Valid options: []

Parameters:

• *args – positional arguments for this data config

• **kwargs – keyword arguments for this data config

create_queue(**kwargs)

Create an empty queue based on options specified in this config

Parameters:

**kwargs – extra args passed to cmd_queue.Queue.create

Returns:

cmd_queue.Queue

run_queue(queue, print_kwargs=None, **kwargs)

Execute a queue with options based on this config.

Parameters:

• queue (cmd_queue.Queue) – queue to run / report

• print_kwargs (None | Dict)

default = {'backend': <Value('tmux')>, 'other_session_handler': <Value('ask')>, 'print_commands': <Value('auto')>, 'print_queue': <Value('auto')>, 'queue_name': <Value(None)>, 'run': <Value(False)>, 'slurm_options': <Value(None)>, 'tmux_workers': <Value(8)>, 'virtualenv_cmd': <Value(None)>, 'with_textual': <Value('auto')>}

normalize()