diff --git a/MANIFEST b/MANIFEST deleted file mode 100644 index b3c2352..0000000 --- a/MANIFEST +++ /dev/null @@ -1,3 +0,0 @@ -README -setup.py -shell.py diff --git a/README b/README index 2d49280..0d64abe 100644 --- a/README +++ b/README @@ -1,12 +1,12 @@ author ====== Todd Mueller -firstnamelastname@common.google.mail.domain +firstname@firstnamelastname.com https://github.com/microlinux/python-shell version ======= -1.2-20130408 +2.0 history ======= @@ -21,13 +21,20 @@ history Added pid to results. 20130408 1.2 Various clean ups and minor internal changes. Removed compressed version, mostly laziness. -20130408 1.3b Added stdin support. +20130802 1.3b Added stdin support. Reworked function param checks. - -install -======= -python setup.py install - +20160926 2.0 Big changes. + Re-factored and cleaned up. + stdout/stderr now returned separately. + stdout/stderr now returned as strings. + namedtuple command field now 'cmd'. + namedtuple runtime field now in milliseconds. + timeout retval now 254. + Python exception retval now 255. + Got rid of setup script/files. + Added 'test' function for . . . testing. + I'm three years older. + truth ===== This program is free software: you can redistribute it and/or modify it @@ -54,46 +61,52 @@ return results in a normalized manner. command() returns a namedtuple and multi_command() returns a namedtuple generator: -result.command command -result.pid pid of process (None on exception) +result.cmd command executed +result.pid pid of process result.retval retval -result.runtime runtime down to ms (None on exception) -result.output output split by line, stripped of vertical and - horizontal whitespace +result.runtime runtime in milliseconds +result.stdout stdout stripped of vertical and horizontal + whitespace +result.stderr stderr stripped of vertical and horizontal + whitespace -retval = -9 on command timeout retval = 127 on command not found -retval = 257 on exception +retval = 254 on command timeout +retval = 255 on Python exception -On retval -9 or 127 a standard description is returned as output. On retval -257, format_exc() is returned. Otherwise stdout + stderr is returned. +On retval 127 or 254 a standard description is returned in stderr. On retval +255, traceback.format_exc() is returned in stderr. ------------------------------------------------------------------- -command( command, timeout=10, stdin=None) ------------------------------------------------------------------- +---------------------------------------------------------------- +command( command, timeout=10, stdin=None) +---------------------------------------------------------------- Executes a command with a timeout. Returns the result as a namedtuple. -result = command('whoami', 2) +result = command('whoami') -print result.command +print +print result.cmd print result.pid print result.retval print result.runtime -print result.output +print result.stdout +print result.stderr ----------------------------------------------------------------------- -multi_command( commands, timeout=10, workers=4, +multi_command( commands, timeout=10, workers=4, stdins=None ----------------------------------------------------------------------- Executes commands concurrently with individual timeouts in a pool of workers. Length of stdins must match commands, empty indexes must contain None. Returns ordered results as a namedtuple generator. -results = multi_command(['whoami', 'uptime'], 2) +results = multi_command(['whoami', 'uptime']) for result in results: - print result.command - print result.pid - print result.retval - print result.runtime - print result.output + print + print result.cmd + print result.pid + print result.retval + print result.runtime + print result.stdout + print result.stderr diff --git a/dist/shell-1.3b.tar.gz b/dist/shell-1.3b.tar.gz deleted file mode 100644 index 77c03f6..0000000 Binary files a/dist/shell-1.3b.tar.gz and /dev/null differ diff --git a/setup.py b/setup.py deleted file mode 100644 index 6d35008..0000000 --- a/setup.py +++ /dev/null @@ -1,3 +0,0 @@ -from distutils.core import setup - -setup(name='shell', version='1.3b', py_modules=['shell']) \ No newline at end of file diff --git a/shell.py b/shell.py index ecdcab8..33f2c12 100644 --- a/shell.py +++ b/shell.py @@ -1,182 +1,200 @@ -""" shell +"""Functions for executing one shell command and multiple, concurrent shell commands. -version 1.3b -https://github.com/microlinux/python-shell +Repository: + https://github.com/microlinux/python-shell -Module for easily executing shell commands and retreiving their results in a -normalized manner. +Author: + Todd Mueller -See https://github.com/microlinux/python-shell/blob/master/README. - -This program is free software: you can redistribute it and/or modify it -under the terms of the GNU General Public License as published by the Free -Software Foundation, version 3. See https://www.gnu.org/licenses/. +License: + This program is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the + Free Software Foundation, version 3. See https://www.gnu.org/licenses/. """ -from collections import namedtuple -from itertools import imap -from multiprocessing import Pool -from shlex import split -from subprocess import PIPE, Popen, STDOUT -from threading import Timer -from time import time -from traceback import format_exc - -""" shell.strip_command_output() - -Strips horizontal and vertical whitespace from a list of lines. Returns the -result as a list. - -@param [str] lines -@return [str] lines +import collections +import multiprocessing +import shlex +import subprocess +import threading +import time +import traceback + +CommandResult = collections.namedtuple('CommandResult', ['cmd', 'pid', 'retval', 'runtime', 'stdout', 'stderr']) +"""Namedtuple representing the result of a shell command. + +Fields: + cmd (str): command executed + pid (int): process id + retval (int): return value + runtime (int): runtime in msecs + stdout (str): stdout buffer, horizontal/vertical whitespace stripped + stderr (str): stderr buffer, horizontal/vertical whitespace stripped """ -def strip_command_output(lines): - if len(lines) > 0: - for i in range(len(lines)): - lines[i] = lines[i].strip() - - while lines and not lines[-1]: - lines.pop() - - while lines and not lines[0]: - lines.pop(0) - - return lines - -""" shell.parse_command_result() - -Parses a command result into a namedtuple. Returns the namedtuple. -@param [str] command, [int] pid, [int] retval, [float] runtime/secs, - [list] output - -@return [str] command, [int] pid, [int] retval, - [float] runtime/secs, [list] output - -""" -def parse_command_result(result): - return namedtuple('ResultTuple', ['command', 'pid', 'retval', 'runtime', - 'output'])._make([result[0], result[1], result[2], - result[3], result[4]]) +def stripper(string): + """Sexily strips horizontal and vertical whitespace from a string. -""" shell.command_worker() + Args: + string (str): string to strip -Executes a command with a timeout. Returns the result as a list. + Returns: + str: stripped string -@param [str] command, [int|float] secs before timeout, [mixed] stdin + """ -@return [str] command, [int] pid, [int] retval, - [float] runtime/secs, [list] output + lines = map(str.strip, string.splitlines()) -""" -def command_worker(command): - kill = lambda this_proc: this_proc.kill() - output = [] - pid = None - retval = None - runtime = None - start = time() - - try: - proc = Popen(split('%s' % str(command[0])), stdout=PIPE, stderr=STDOUT, - stdin=PIPE) - timer = Timer(command[1], kill, [proc]) - - timer.start() - output = proc.communicate(command[2])[0].splitlines() - timer.cancel() - - runtime = round(time() - start, 3) - except OSError: - retval = 127 - output = ['command not found'] - except Exception, e: - retval = 257 - output = format_exc().splitlines() - finally: - if retval == None: - if proc.returncode == -9: - output = ['command timed out'] - - pid = proc.pid - retval = proc.returncode - - return [command[0], pid, retval, runtime, strip_command_output(output)] - -""" shell.command() - -Executes a command with a timeout. Returns the result as a namedtuple. - -@param command -@param secs before timeout (10) -@param stdin (None) - -@return [str] command, [int] pid, [int] retval, - [float] runtime/secs, [list] output - -""" -def command(command, timeout=10, stdin=None): - if not isinstance(command, str): - raise TypeError('command is not a string') - - if not isinstance(timeout, int) and not isinstance(timeout, float): - raise TypeError('timeout is not an integer or float') - - return parse_command_result(command_worker([command, timeout, stdin])) - -""" shell.multi_command() - -Executes commands concurrently with individual timeouts in a pool of workers. -Length of stdins must match commands, empty indexes must contain None. Returns -ordered results as a namedtuple generator. - -@param [str] commands -@param secs before individual timeout (10) -@param max workers (4) -@param [mixed] stdins (None) - -@return [str] command, [int] pid, [int] retval, - [float] runtime/secs, [list] output - -""" -def multi_command(commands, timeout=10, workers=4, stdins=None): - if not isinstance(commands, list): - raise TypeError('commands is not a list') + while lines and not lines[-1]: + lines.pop() - for i in range(len(commands)): - if not isinstance(commands[i], str): - raise TypeError('commands[%s] is not a string' % i) + while lines and not lines[0]: + lines.pop(0) - if not isinstance(timeout, int) and not isinstance(timeout, float): - raise TypeError('timeout is not an integer or float') + return '\n'.join(lines) - if not isinstance(workers, int): - raise TypeError('workers is not an integer') +def command(cmd, timeout=10, stdin=None): + """Executes one shell command and returns the result. - if stdins and not isinstance(stdins, list): - raise TypeError('stdins is not a list') - elif stdins and len(stdins) != len(commands): - raise ValueError('length of stdins does not match commands') + Args: + cmd (str): command to execute + timeout (int): timeout in seconds (10) + stdin (str): input (None) - count = len(commands) + Returns: + CommandResult: result of command - if workers > count: - workers = count + """ - for i in range(count): - if stdins: - stdin = stdins[i] - else: - stdin = None + kill = lambda this_proc: this_proc.kill() + pid = None + retval = None + runtime = None + stderr = None + stdout = None - commands[i] = [commands[i], timeout, stdin] + try: + start = time.time() * 1000.0 + proc = subprocess.Popen(shlex.split(cmd), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + stdin=subprocess.PIPE, + close_fds=True, + shell=False) - pool = Pool(processes=workers) - results = pool.imap(command_worker, commands) + timer = threading.Timer(timeout, kill, [proc]) - pool.close() - pool.join() + timer.start() + stdout, stderr = map(stripper, proc.communicate(input=stdin)) + timer.cancel() - return imap(parse_command_result, results) \ No newline at end of file + runtime = int((time.time() * 1000.0) - start) + except OSError: + retval = 127 + stderr = 'command not found' + except: + retval = 255 + stderr = traceback.format_exc() + finally: + if retval == None: + if proc.returncode == -9: + retval = 254 + stderr = 'command timed out' + else: + retval = proc.returncode + + pid = proc.pid + + return CommandResult(cmd, pid, retval, runtime, stdout, stderr) + +def command_wrapper(args): + """Wrapper used my multi_command() to pass command() arguments as a list.""" + + return command(*args) + +def multi_command(cmds, timeout=10, stdins=None, workers=4): + """Executes multiple shell commands concurrently and returns the results. + + If stdins is given, its length must match that of cmds. If commands do + not require input, their corresponding stdin field must be None. + + Args: + cmds (list): commands to execute + timeout (int): timeout in seconds per command (10) + workers (int): max concurrency (4) + stdins (list): inputs (None) + + Returns: + CommandResult generator: results of commands in given order + + """ + + num_cmds = len(cmds) + + if stdins != None: + if len(stdins) != num_cmds: + raise RuntimeError("stdins length doesn't match cmds length") + + if workers > num_cmds: + workers = num_cmds + + for i in xrange(num_cmds): + try: + stdin = stdins[i] + except: + stdin = None + + cmds[i] = (cmds[i], timeout, stdin) + + pool = multiprocessing.Pool(processes=workers) + results = pool.imap(command_wrapper, cmds) + + pool.close() + pool.join() + + return results + +def test(single_cmds=['ls -l', 'uptime', 'blargh'], + multi_cmds=['ls -l', 'uptime', 'ping -c3 -i1 -W1 google.com']): + """Demonstrates command and multi_command functionality. + + Args: + single_cmds (list): commands to run one at a time + multi_cmds (list): commands to run concurrently + + """ + + print + print '-----------------------' + print 'Running single commands' + print '-----------------------' + + for cmd in single_cmds: + result = command(cmd) + print + print 'cmd: %s' % result.cmd + print 'pid: %s' % result.pid + print 'ret: %s' % result.retval + print 'run: %s' % result.runtime + print 'out:' + print result.stdout + print 'err:' + print result.stderr + + print + print '-------------------------' + print 'Running multiple commands' + print '-------------------------' + + for result in multi_command(multi_cmds): + print + print 'cmd: %s' % result.cmd + print 'pid: %s' % result.pid + print 'ret: %s' % result.retval + print 'run: %s' % result.runtime + print 'out:' + print result.stdout + print 'err:' + print result.stderr diff --git a/test/command.py b/test/command.py deleted file mode 100755 index 10f5910..0000000 --- a/test/command.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/python - -from shell import command - -print -print 'Execute a command that succeeds' -print '-------------------------------' -result = command('ls -l', 2) -print 'result.command : %s' % result.command -print 'result.pid : %s' % result.pid -print 'result.retval : %s' % result.retval -print 'result.runtime : %s' % result.runtime -print 'result.output : %s' % result.output -print -print 'Execute a command that doesn\'t exist' -print '------------------------------------' -result = command('/usr/bin/blurple -f opt', 2) -print 'result.command : %s' % result.command -print 'result.pid : %s' % result.pid -print 'result.retval : %s' % result.retval -print 'result.runtime : %s' % result.runtime -print 'result.output : %s' % result.output -print -print 'Execute a command that times out' -print '--------------------------------' -result = command('ping -c5 -i1 -W1 -q google.com', 2) -print 'result.command : %s' % result.command -print 'result.pid : %s' % result.pid -print 'result.retval : %s' % result.retval -print 'result.runtime : %s' % result.runtime -print 'result.output : %s' % result.output -print \ No newline at end of file diff --git a/test/hosts.csv b/test/hosts.csv deleted file mode 100644 index 5701bc8..0000000 --- a/test/hosts.csv +++ /dev/null @@ -1,100 +0,0 @@ -call.com -called.com -came.com -can.com -cannot.com -cant.com -car.com -care.com -carried.com -cars.com -case.com -cases.com -cause.com -cent.com -center.com -central.com -century.com -certain.com -certainly.com -chance.com -change.com -changes.com -character.com -charge.com -chief.com -child.com -children.com -choice.com -christian.com -church.com -city.com -class.com -clear.com -clearly.com -close.com -closed.com -club.com -co.com -cold.com -college.com -color.com -come.com -comes.com -coming.com -committee.com -common.com -communist.com -community.com -company.com -complete.com -completely.com -concerned.com -conditions.com -congress.com -consider.com -considered.com -continued.com -control.com -corner.com -corps.com -cost.com -costs.com -could.com -couldnt.com -countries.com -country.com -county.com -couple.com -course.com -court.com -covered.com -cut.com -daily.com -dark.com -data.com -day.com -days.com -de.com -dead.com -deal.com -death.com -decided.com -decision.com -deep.com -defense.com -degree.com -democratic.com -department.com -described.com -design.com -designed.com -determined.com -developed.com -development.com -did.com -didnt.com -difference.com -different.com -difficult.com -direct.com \ No newline at end of file diff --git a/test/multi_command.py b/test/multi_command.py deleted file mode 100755 index 0d8473d..0000000 --- a/test/multi_command.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/python - -from sys import exit - -from shell import multi_command - -commands = [] -i = 0 - -print -print 'Execute three commands with varying results' -print '-------------------------------------------' -results = multi_command(['ls -l', '/usr/bin/blurple -f opt', - 'ping -c5 -i1 -W1 -q google.com'], 2) -for result in results: - print 'result%s.command: %s' % (i, result.command) - print 'result%s.pid : %s' % (i, result.pid) - print 'result%s.retval : %s' % (i, result.retval) - print 'result%s.runtime: %s' % (i, result.runtime) - print 'result%s.output : %s' % (i, result.output) - print - i = i + 1 - -""" remove this if you want to continue, will use 150M-200M of RAM """ -exit() - -hosts = open('hosts.csv', 'r') -for host in hosts: - commands.append('ping -c3 -i1 -W1 -q %s' % host.strip()) -hosts.close() - -print 'Execute 100 pings with 100 workers' -print '----------------------------------' -results = multi_command(commands, 5, 100) -for result in results: - print result.command - print result.pid - print result.retval - print result.runtime - print result.output - print \ No newline at end of file