From fca2020219e77544d0c76ab6d576a62f670ac29c Mon Sep 17 00:00:00 2001 From: microlinux Date: Fri, 2 Aug 2013 20:16:53 -0500 Subject: [PATCH 01/16] fix merge fail . . . --- README | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README b/README index 2d49280..2f94c7a 100644 --- a/README +++ b/README @@ -6,7 +6,7 @@ https://github.com/microlinux/python-shell version ======= -1.2-20130408 +1.3b history ======= From 6cb32937bd407085f764dbcd66ff639a40fa1de8 Mon Sep 17 00:00:00 2001 From: microlinux Date: Fri, 2 Aug 2013 20:20:30 -0500 Subject: [PATCH 02/16] README fix --- README | 2 +- dist/shell-1.3b.tar.gz | Bin 3155 -> 3155 bytes 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/README b/README index 2f94c7a..5b4e0ee 100644 --- a/README +++ b/README @@ -21,7 +21,7 @@ history Added pid to results. 20130408 1.2 Various clean ups and minor internal changes. Removed compressed version, mostly laziness. -20130408 1.3b Added stdin support. +20130802 1.3b Added stdin support. Reworked function param checks. install diff --git a/dist/shell-1.3b.tar.gz b/dist/shell-1.3b.tar.gz index 77c03f661dfb17e4aa9a98177aa89e50c0c12c05..c10dc3f3f4794d181635b20a5bf2b815e911dd7e 100644 GIT binary patch delta 2778 zcmV<03MKW^7}FRAABzYG%UY2JCV$&;>x+W=Ql~MR*oLjN2n4~Pb}5OMeI2WD zaJbipmnXwLa=iBqAd@K-gzTmF|NS72cH=!X|2G9$=>PiHU-mC9UtE0^{eM4saC`SHE96FXNEyCv?Gzp|dPq#-jed&qB(BV)PaFSR_l!erD1WJW17NcJMl&={y!; zF|yad*YW?G^Ru5{pMRzJ{}d7be{ytkSolBO1>pbv<9_4+-vX060UCeFi^DLE+(jHO z0_LJSyM7Emk+V$GM4lcTEO@%iW-cHegxnKxz@zN`AX&pE(Y}r+&U+nD%$_4h#T94C zWg3fhJ~~7zIO-2D9uYz=BA#*@5WxaQC3Efk5atInLe2vo@`$F4M2z_aLIz>?DH9Q_ zNqNZPEG08Q5bGX^;+2!w0WTU?;D|;(3FG(hl*S~5se_FLXTA^9JOYbE1(P8H69~vs&w?HSO!E81_~q4`v2%9$mi&Hpb8~h%e)})L1PQ_{d(Si-9wq??vcR68 zQMyKEUZ3AQe+l?!|Gs#2F@8%R_Pn?lU!LFIk{4GuXQNlVm4er;Pto+wR_L`J;c2g3&3&FoODdr-l3jx@q6xLFkQuH zH3IneFydHGl-jn~J@e2%^eVhslbQr1O;kSkaKLmp0t)=@63&hp218&#lRAtkL&=*5|cp%6MskK$BNBdxyAHgcv7B6qnl&EDlnkr ztDMMjt|EKgB5|Q;P~aR0P~n58=k)>xGOby(cWKV-lAMAA)O#2tLYm&e7GMdbC4dN# zK&&|UJX|Xn#Qup7Oo$4au!X4?>+_> z=YQJ!qzZ;wCo0ZSySWQjsCE?$lG?aXv}MO}t-FBw?6KObWlTff?vY{FaS{Q?B2}NN z4uaoQ2aLk1W7-9(gIdXtp%X2VYN#;oW8>0C6r{XZWg9|{j(Rl@GIW&sq|)D`F`Vl# z51w$dc$p9^!u(0;lSe*>aGVusu!*Nh9DfJs{DyJYdBvgyEPx@+D}+v&Laa@_Wl0Jw zOj1mae1`XcLb2Bwo-MZ;ClRIh2H#KW9gKsAoDkY939DS!4_&StfTdte{n)44jB1@Y`4q%vi~G`AI%M{44< z9}(4;wer1q}z>o6+QuGrh2)WHZ6`F(@Kv%o0F|aP-xxL7UZQISYnYGAXUb)4UKXTrwOFkbw59Rg>dK4r2~eA2i^o3ddI8>4 zK{&NN8pxqN*=ubyQ6@-sYT6PN{TF#UIV2deVHfcQD-6EJ2>=-i7oX>rC|CU?V-F#mS&}98P!8G4qE7 z>d|!Ct=f-0q^@?g<$OU4<&E_1quY5Sef#WmA=75V zC0&h!YzWv?1=-BoAk7z(*Sl%SA87JtDtld_!JaM}fyU3DB4 z{5s{$UC2f}78KF7keR?_p`#^5_OtSUf0-N1iZb7@+?XwO%x$iOmd&O5v zVk9Vj1sJarru?Rpo0a;Mo~jQJ0ut7?amP)nd2ojlcazKt4*{x^+zK3jXBAt0jDB@}N*tvitU!iNyjdwY1&IqL4f& zzzM6CQfRL1i^b=W@|TbY{1Ry3rVn|hEL;=Oty63%f$W033fjFjR6~`p@R98&-^=Mn zbby3}|AivNCJ#metNE#!lG$l)a^ups z>&Og)@#zT`NVb8a6JE!?aH8Lk0=<3VFevCt07kfgr gOQc6?l2M52pU!lCr{4b)lM)O$2t2N$EC5gd00zfy@c;k- delta 2774 zcmV;{3Muu|7}FRAABzYGmRFGmCV#)Kz9^_Kby}l|ZP-eSKoAUSmy+1*>&V^63i{)3 zhL3$oiqpmk+QSZDXqOxghx6jhtmTpg!T!KKoE_|aDd+>7o*r-gHLy1r4o*+|$A>4w zlf6E?oSg2Fnk-iyO9j@(5YF9PPG zJG*`iKasOc(?p&f94vUc%w{ei9)#QzaloVO;UHPVCegl*C(e5vP|ThqXW$O^hf3%D ze>u*Q%QP12eAdhiFfI{7E+d|D8W6z(MkRCY{1D~`GeRx`9`cB$j6{t21VRU44=EE7 ztVwyu;w&XIKoIMb(*ZLzS>T99J_+OZ@RY_Rg{gy$1?RpG(>wypL`CeF7g>Y{LNbEM zEJ)=Bq8G%H-OqhVxWvXaN+e6O#7)*#4UUrva23b+30Nrsu`Q4kJkAAUM1RKfbVUW+ zXzMs59>_v4AFNsMSqApVQJ9mkMNEL; zZn9axJz{Q4LSbJV0blaTY^~5=!tTzU*`2(MfhtXv1dS-kp`~odz`5&3C^$vaQ^?CW z0mhazMc!9D2*`|~f6TL>M}Gj5{CYY5>H5vsIlp>Kem%dvJ--^i{WoBO1YwrFXPOQV zlYj$RU{BB}T_ZEEFK%D_1o-FwxqNjweoG(*y}TS>UEJM~m)E!CoZOt>jxS%ld3AnE zZr@z;V%>bZnD+ z1Q`ee^Uy!^D!gKoodhIJSU&i0z;rkQ3jF@#h|WT(AFiJ>$I+X*W(6V7z&)O0Ow|K+ z9%G||XAtUe9tXgpOiU%6E7TmgAbaqb%5A%LskuUmrFtNrHu0T2o0AR&Ab-`&z;eOq zlY}Xr!AwBKz!IL1>;T9X_y;uH0PYbE7MM_vp}5pxMr8jx@qd8{Q9*N@{m+KnXtiOPx1KVF-;KtN^hSd|jECTH{&RiZ zjZp91r{Lm3d!JOnQ0qj+Ichg|;R@BRf^e>& z;8>*UQ`JH6o9cj3SanRhKy^?n`6+awMN$nF#(ivD`iO#*7prVT$bZpMujWC9j#8gg z`g<~la~WQ&R^!CNSGSK1;x*GDdCbj?NP-nrbucN6shuUre|xI! z3iEqodOW`<&Or2A4S%jSj3!qbGDcS)HD*^EHiq~AQvchO_pA5+28YK*|8FqF`v2tg zu&MvQ2Fyhqs@y+IFcM#WA*lJzl?hmF_ zLT(a&(!&0>b$zqY2&F6rqe_sbWVsx)#PxPJI8!cP^Ku}Kt3_(1kS&exR##q>Pk`DK zTRZkq*MAA{rV7ES?ae?A?a5wiqj@qxvQyKJsOZ1o)aElvi- z<8Zozj+s9!P>-h5ZqUMjkg*Vc-k8b0Q z^ndNM(}hfX4VQE^4zeL&NBtqtZh&@)M+S)^N7Fp=DRftzp<^iI0#SlO3Rwi#Ov-X< z!)X^-cGYoI@avQ}cOe_`SWraQLS_P!g^rdK+1H}$kwdjwH&Ds--GvabFybrsVQbvw zzFahA+tU)E3)_(G5mu)*Jg!qKw^%n0HhXR@`6Q=y8lbe+zK3jW))j~iavAei5tzW zKhSC3c5^H9D_gI8+-o)uTX!JyWe*{w5BBh+bJX3zK{+3g+uSWb#q+A`eWvGqh7(pT zrO;g27mLp#(G;Un8mzLnFB=l}@` z{|iNkO&*L!G!MIfHn8@(u|bKhG1V=n&@+!H(JbFkA>(93iV&` zpRx@3pdMuWnCY#mfyN%lgS-dRi3;`ntX3Y+HJ;hkj*j%`R`XLcC9~7q Date: Mon, 26 Sep 2016 21:14:46 -0500 Subject: [PATCH 03/16] Update shell.py --- shell.py | 309 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 165 insertions(+), 144 deletions(-) diff --git a/shell.py b/shell.py index ecdcab8..c670a5d 100644 --- a/shell.py +++ b/shell.py @@ -1,182 +1,203 @@ -""" shell +"""Functions for executing one shell command and multiple, concurrent shell commands. -version 1.3b -https://github.com/microlinux/python-shell +Repository: + https://github.com/microlinux/python-shell -Module for easily executing shell commands and retreiving their results in a -normalized manner. +Author: + Todd Mueller + +Version: + 2.0 20160926 -See https://github.com/microlinux/python-shell/blob/master/README. - -This program is free software: you can redistribute it and/or modify it -under the terms of the GNU General Public License as published by the Free -Software Foundation, version 3. See https://www.gnu.org/licenses/. +License: + This program is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the + Free Software Foundation, version 3. See https://www.gnu.org/licenses/. """ -from collections import namedtuple -from itertools import imap -from multiprocessing import Pool -from shlex import split -from subprocess import PIPE, Popen, STDOUT -from threading import Timer -from time import time -from traceback import format_exc - -""" shell.strip_command_output() - -Strips horizontal and vertical whitespace from a list of lines. Returns the -result as a list. -@param [str] lines - -@return [str] lines +import collections +import multiprocessing +import shlex +import subprocess +import threading +import time +import traceback + +CommandResult = collections.namedtuple('CommandResult', ['cmd', 'pid', 'retval', 'runtime', 'stdout', 'stderr']) +"""Namedtuple representing the result of a shell command. + +Fields: + cmd (str): command executed + pid (int): process id + retval (int): return value + runtime (int): runtime in msecs + stdout (str): stdout buffer, horizontal/vertical whitespace stripped + stderr (str): stderr buffer, horizontal/vertical whitespace stripped """ -def strip_command_output(lines): - if len(lines) > 0: - for i in range(len(lines)): - lines[i] = lines[i].strip() - - while lines and not lines[-1]: - lines.pop() - while lines and not lines[0]: - lines.pop(0) +def stripper(string): + """Sexily strips horizontal and vertical whitespace from a string. - return lines + Args: + string (str): string to strip -""" shell.parse_command_result() + Returns: + str: stripped string -Parses a command result into a namedtuple. Returns the namedtuple. + """ -@param [str] command, [int] pid, [int] retval, [float] runtime/secs, - [list] output + lines = map(str.strip, string.splitlines()) -@return [str] command, [int] pid, [int] retval, - [float] runtime/secs, [list] output + while lines and not lines[-1]: + lines.pop() -""" -def parse_command_result(result): - return namedtuple('ResultTuple', ['command', 'pid', 'retval', 'runtime', - 'output'])._make([result[0], result[1], result[2], - result[3], result[4]]) + while lines and not lines[0]: + lines.pop(0) -""" shell.command_worker() + return '\n'.join(lines) -Executes a command with a timeout. Returns the result as a list. +def command(cmd, timeout=10, stdin=None): + """Executes one shell command and returns the result. -@param [str] command, [int|float] secs before timeout, [mixed] stdin + Args: + cmd (str): command to execute + timeout (int): timeout in seconds (10) + stdin (str): input (None) -@return [str] command, [int] pid, [int] retval, - [float] runtime/secs, [list] output + Returns: + CommandResult: result of command -""" -def command_worker(command): - kill = lambda this_proc: this_proc.kill() - output = [] - pid = None - retval = None - runtime = None - start = time() - - try: - proc = Popen(split('%s' % str(command[0])), stdout=PIPE, stderr=STDOUT, - stdin=PIPE) - timer = Timer(command[1], kill, [proc]) - - timer.start() - output = proc.communicate(command[2])[0].splitlines() - timer.cancel() - - runtime = round(time() - start, 3) - except OSError: - retval = 127 - output = ['command not found'] - except Exception, e: - retval = 257 - output = format_exc().splitlines() - finally: - if retval == None: - if proc.returncode == -9: - output = ['command timed out'] - - pid = proc.pid - retval = proc.returncode - - return [command[0], pid, retval, runtime, strip_command_output(output)] - -""" shell.command() - -Executes a command with a timeout. Returns the result as a namedtuple. - -@param command -@param secs before timeout (10) -@param stdin (None) - -@return [str] command, [int] pid, [int] retval, - [float] runtime/secs, [list] output + """ -""" -def command(command, timeout=10, stdin=None): - if not isinstance(command, str): - raise TypeError('command is not a string') + kill = lambda this_proc: this_proc.kill() + pid = None + retval = None + runtime = None + stderr = None + stdout = None - if not isinstance(timeout, int) and not isinstance(timeout, float): - raise TypeError('timeout is not an integer or float') + try: + start = time.time() * 1000.0 + proc = subprocess.Popen(shlex.split(cmd), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + stdin=subprocess.PIPE, + close_fds=True, + shell=False) - return parse_command_result(command_worker([command, timeout, stdin])) + timer = threading.Timer(timeout, kill, [proc]) -""" shell.multi_command() + timer.start() + stdout, stderr = map(stripper, proc.communicate(input=stdin)) + timer.cancel() -Executes commands concurrently with individual timeouts in a pool of workers. -Length of stdins must match commands, empty indexes must contain None. Returns -ordered results as a namedtuple generator. + runtime = int((time.time() * 1000.0) - start) + except OSError: + retval = 127 + stderr = 'command not found' + except: + retval = 255 + stderr = traceback.format_exc() + finally: + if retval == None: + if proc.returncode == -9: + retval = 254 + stderr = 'command timed out' + else: + retval = proc.returncode -@param [str] commands -@param secs before individual timeout (10) -@param max workers (4) -@param [mixed] stdins (None) + pid = proc.pid -@return [str] command, [int] pid, [int] retval, - [float] runtime/secs, [list] output - -""" -def multi_command(commands, timeout=10, workers=4, stdins=None): - if not isinstance(commands, list): - raise TypeError('commands is not a list') + return CommandResult(cmd, pid, retval, runtime, stdout, stderr) - for i in range(len(commands)): - if not isinstance(commands[i], str): - raise TypeError('commands[%s] is not a string' % i) +def command_wrapper(args): + """Wrapper used my multi_command to pass command arguments as a list.""" - if not isinstance(timeout, int) and not isinstance(timeout, float): - raise TypeError('timeout is not an integer or float') + return command(*args) - if not isinstance(workers, int): - raise TypeError('workers is not an integer') +def multi_command(cmds, timeout=10, stdins=None, workers=4): + """Executes multiple shell commands concurrently and returns the results. - if stdins and not isinstance(stdins, list): - raise TypeError('stdins is not a list') - elif stdins and len(stdins) != len(commands): - raise ValueError('length of stdins does not match commands') + If stdins is given, its length must match that of cmds. If commands do + not require input, their corresponding stdin field must be None. - count = len(commands) + Args: + cmds (list): commands to execute + timeout (int): timeout in seconds per command (10) + workers (int): max concurrency (4) + stdins (list): inputs (None) - if workers > count: - workers = count + Returns: + CommandResult generator: results of commands in given order - for i in range(count): - if stdins: - stdin = stdins[i] - else: - stdin = None + """ - commands[i] = [commands[i], timeout, stdin] + num_cmds = len(cmds) - pool = Pool(processes=workers) - results = pool.imap(command_worker, commands) + if stdins != None: + if len(stdins) != num_cmds: + raise RuntimeError("stdins length doesn't match cmds length") - pool.close() - pool.join() + if workers > num_cmds: + workers = num_cmds - return imap(parse_command_result, results) \ No newline at end of file + for i in xrange(num_cmds): + try: + stdin = stdins[i] + except: + stdin = None + + cmds[i] = (cmds[i], timeout, stdin) + + pool = multiprocessing.Pool(processes=workers) + results = pool.imap(command_wrapper, cmds) + + pool.close() + pool.join() + + return results + +def test(single_cmds=['ls -l', 'uptime', 'blargh'], + multi_cmds=['ls -l', 'uptime', 'ping -c3 -i1 -W1 google.com']): + """Demonstrates command and multi_command functionality. + + Args: + single_cmds (list): commands to run one at a time + multi_cmds (list): commands to run concurrently + + """ + + print + print '-----------------------' + print 'Running single commands' + print '-----------------------' + + for cmd in single_cmds: + result = command(cmd) + print + print 'cmd: %s' % result.cmd + print 'pid: %s' % result.pid + print 'ret: %s' % result.retval + print 'run: %s' % result.runtime + print 'out:' + print result.stdout + print 'err:' + print result.stderr + + print + print '-------------------------' + print 'Running multiple commands' + print '-------------------------' + + for result in multi_command(multi_cmds): + print + print 'cmd: %s' % result.cmd + print 'pid: %s' % result.pid + print 'ret: %s' % result.retval + print 'run: %s' % result.runtime + print 'out:' + print result.stdout + print 'err:' + print result.stderr From 653a126b187a7b3eec3bee4f9794bed27b025f48 Mon Sep 17 00:00:00 2001 From: Todd Date: Mon, 26 Sep 2016 21:15:22 -0500 Subject: [PATCH 04/16] Delete setup.py --- setup.py | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 setup.py diff --git a/setup.py b/setup.py deleted file mode 100644 index 6d35008..0000000 --- a/setup.py +++ /dev/null @@ -1,3 +0,0 @@ -from distutils.core import setup - -setup(name='shell', version='1.3b', py_modules=['shell']) \ No newline at end of file From 244aff3e7e102d417c89a94604f08c9356328a60 Mon Sep 17 00:00:00 2001 From: Todd Date: Mon, 26 Sep 2016 21:15:39 -0500 Subject: [PATCH 05/16] Delete MANIFEST --- MANIFEST | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 MANIFEST diff --git a/MANIFEST b/MANIFEST deleted file mode 100644 index b3c2352..0000000 --- a/MANIFEST +++ /dev/null @@ -1,3 +0,0 @@ -README -setup.py -shell.py From 0e60e9b43c7d79f5f5556181bb5a3b815c83824b Mon Sep 17 00:00:00 2001 From: Todd Date: Mon, 26 Sep 2016 21:32:01 -0500 Subject: [PATCH 06/16] Update README --- README | 59 ++++++++++++++++++++++++++++++++++------------------------ 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/README b/README index 5b4e0ee..ce42b1e 100644 --- a/README +++ b/README @@ -23,11 +23,18 @@ history Removed compressed version, mostly laziness. 20130802 1.3b Added stdin support. Reworked function param checks. - -install -======= -python setup.py install - +20160926 2.0 Big changes. + Re-factored and cleaned up. + stdout/stderr now returned separately. + stdout/stderr now returned as strings. + namedtuple command field now 'cmd'. + namedtuple runtime field now in milliseconds. + timeout retval now 254. + Python exception retval now 255. + Got rid of setup script/files. + Added 'test' function for . . . testing. + I'm three years older. + truth ===== This program is free software: you can redistribute it and/or modify it @@ -54,32 +61,35 @@ return results in a normalized manner. command() returns a namedtuple and multi_command() returns a namedtuple generator: -result.command command -result.pid pid of process (None on exception) +result.cmd command executed +result.pid pid of process result.retval retval -result.runtime runtime down to ms (None on exception) -result.output output split by line, stripped of vertical and - horizontal whitespace +result.runtime runtime in milliseconds +result.stdout stdout stripped of vertical and horizontal + whitespace +result.stderr stderr stripped of vertical and horizontal + whitespace -retval = -9 on command timeout retval = 127 on command not found -retval = 257 on exception +retval = 254 on command timeout +retval = 255 on Python exception -On retval -9 or 127 a standard description is returned as output. On retval -257, format_exc() is returned. Otherwise stdout + stderr is returned. +On retval 127 or 254 a standard description is returned in stderr. On retval +255, traceback.format_exc() is returned in stderr. ------------------------------------------------------------------- -command( command, timeout=10, stdin=None) ------------------------------------------------------------------- +---------------------------------------------------------------- +command( command, timeout=10, stdin=None) +---------------------------------------------------------------- Executes a command with a timeout. Returns the result as a namedtuple. result = command('whoami', 2) -print result.command +print result.cmd print result.pid print result.retval print result.runtime -print result.output +print result.stdout +print result.stderr ----------------------------------------------------------------------- multi_command( commands, timeout=10, workers=4, @@ -92,8 +102,9 @@ ordered results as a namedtuple generator. results = multi_command(['whoami', 'uptime'], 2) for result in results: - print result.command - print result.pid - print result.retval - print result.runtime - print result.output + print result.cmd + print result.pid + print result.retval + print result.runtime + print result.stdout + print result.stderr From e97bbec1608d35c559a59b1334ad52f330502455 Mon Sep 17 00:00:00 2001 From: Todd Date: Mon, 26 Sep 2016 21:32:52 -0500 Subject: [PATCH 07/16] Update shell.py --- shell.py | 1 - 1 file changed, 1 deletion(-) diff --git a/shell.py b/shell.py index c670a5d..9fe6a99 100644 --- a/shell.py +++ b/shell.py @@ -15,7 +15,6 @@ Free Software Foundation, version 3. See https://www.gnu.org/licenses/. """ - import collections import multiprocessing import shlex From 63bf594c5caf33b8073d1cc383dee012fa4f73c2 Mon Sep 17 00:00:00 2001 From: Todd Date: Mon, 26 Sep 2016 21:42:01 -0500 Subject: [PATCH 08/16] Update shell.py --- shell.py | 1 + 1 file changed, 1 insertion(+) diff --git a/shell.py b/shell.py index 9fe6a99..c670a5d 100644 --- a/shell.py +++ b/shell.py @@ -15,6 +15,7 @@ Free Software Foundation, version 3. See https://www.gnu.org/licenses/. """ + import collections import multiprocessing import shlex From 9a903c025321e8cfa059f3454f3ec9faa1da2cd4 Mon Sep 17 00:00:00 2001 From: Todd Date: Mon, 26 Sep 2016 21:42:24 -0500 Subject: [PATCH 09/16] Update shell.py --- shell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shell.py b/shell.py index c670a5d..428a015 100644 --- a/shell.py +++ b/shell.py @@ -7,7 +7,7 @@ Todd Mueller Version: - 2.0 20160926 + 2.0 20160926 License: This program is free software: you can redistribute it and/or modify it From 635ac5d41920217caf1145dc0412f3cd9048f535 Mon Sep 17 00:00:00 2001 From: Todd Date: Mon, 26 Sep 2016 21:55:55 -0500 Subject: [PATCH 10/16] Delete shell-1.3b.tar.gz --- dist/shell-1.3b.tar.gz | Bin 3155 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 dist/shell-1.3b.tar.gz diff --git a/dist/shell-1.3b.tar.gz b/dist/shell-1.3b.tar.gz deleted file mode 100644 index c10dc3f3f4794d181635b20a5bf2b815e911dd7e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3155 zcmV-Z46O4XiwFqITKrK019NC)Y-}wtE;C{-bYXG;?HX%u8#mIQ7tnv;6(}U(C`SHE96FXNEyCv?Gzp|dPq#-jed z&qB(BV)PaFSR_l!erD1WJW17NcJMl&={y!;F|yad*YW?G^Ru5{pMRzJ{}KOxa&&T7 z_&?kQ;Q#&Oe&he&0;uv$XQaRx$G%TqXJ8UcICCy!8bKrosQ&%Qi^DLE+(jHO0_LJS zyM7Emk+V$GM4lcTEO@%iW-cHegxnKxz@zN`AX&pE(Y}r+&U+nD%$_4h#T94CWg3fh zJ~~7zIO-2D9uYz=BA#*@5WxaQC3Efk5atInLe2vo@`$F4M2z_aLIz>?DH9Q_NqNZP zEG08Q5bGX^;*~?{kVz& z8F>V<=aG@U7S84lH%toyEF}@hginlJ^gv894b~)}A9w^)X5$}}t`BFHJDGx_;222~ zi*yV9RV?12C1jC@R|-j}pds+Xymvs-F=Ec`P1`7uEX@)(Sz9$YPAb4f9OEBgr3A#b zKvM8H7mN`Z&(jqZaFea$jCddm!F;f0!DkuRBTrE-2OyL%_W2wrr%o36On|k47MTbo ziRbF&m&-Tg7Zx!Ag1gRU0r!ZxB?*OnaRhwGC$qIee*wEYw`OI6)eGXssIWDeQ6Kk-77}*QJo|A3XQNyMYX{?5B;f=qd;pqOhQ|a_1BXG{ zz?NPKW)BV~@VKZzvy~SMIH}`N^m^C?IKmYa`J02Ufzf~%46_gcUGiK&;Pto+wR_L` zJ;c2g3&3&FoODdr-l3jx@q6xLFkQuHH3IneFydHGl-jn~J@e2%^eVhsfWF$qfgqcS z$OaxXOA2g?kOuq%90(9243Cak)aeqvLLweqR6h7{z;rkQ3jFTlh|WT(AFiKM$I+X* zW(6Tn!9AW~Ow|K+9%G||rx5CJ9tXgpOiU%6OVk{=AiMXN%4xfHskuUmrFtNrHu0T2 zod+=m1Uulzs|e$9_)+TMfeBazEQer$417gejiEOhCoJ5}uFj z0LT*f2Q=IO?hy_am{5xGOby(cWKV-lAMAA)O#2tLYm&e7GMdbC4dN#K&&|UJX|Xn z#Qup7Oo$4au!X4?>+_>=i2+E3Wi!I zD$Y^6xeHgQb`=bg+PF}(Wyf)?yMX%avD&L;Ohex8kzvs}(%+*ooa-kOzkW|{@YV+SD4=$)8qL?aSEc}YH+n- zG`ZT4F}nJwF}vEZF}(k``roF$-@N}fI6SuX|Iw5F3D*B7{igo^CNLLqsA_;L#jM%G z6dLk&D5-QQ)N)#mdg!3odn4*7>J@Kmp9qr}6U zT0k|9vyfkMQ7QIX&StfTdte{n)44jB1@Y`4q%vi~G`AI%M{44<9NVr|Y=;F@o-^vbvvWmR zXJnua&Q&$8O(ySj*4F7NL&6zZu4&OG^E$XFaEaQH5=`PzzgRrWN@>(`%C!;VDNX#O z+l_e@J^^N?dbyf5Esh=2N{>F9ldVTkXx-Bmi3O^@*fvc5^%ti}or7{wpW zq<&E_1quY5Sef#WmA=75VC0&h!YzWv?1=-BoAk7z(*Sl%SA87Qr=>vYgs*+69(fbsQD^I_1q>$VNOC6w$SinZRVB zqa{W5wdi`}P_5RDRC0BDE<`Mh_{#m*8h5!b7tPuBv_$CKHfDQ-)u|1S>(t7v){TSB zc|?Q2sub=pOGsNmenW8GPj|IXc(KnAW&g+1LVa7}@6Ep^Oou7sA&rTVlBE>$c8u5Dr@D1HSPuM?*HrjwhM`jnok z4-f(p*0yoSO{#fthZA@6^k*A_I_=JltxoL5k@~8>fM^8Com$$i=1p7g^IBY=)J0|F z1Oc;ZrL_-d6h9p6 zoDaxt?v|h8S=IHv(DS~)39FV;Xs+yw#pjXomyiei5@_J24|%36Toci)Q*0@L?1H=s z+PyVYLzS@bk?kkn%jrgRfP{qqg(Acz4@M)Jhg}<3d)?TeMAw+=mQ(1N2b5@*Z>SLR zj-W~%HK<>^}p3x zdUUJ#shN`5X>M}k(zol#41@9M2^L7Ufuj>;Jz6qX%z-M_TMO`di3-Cb)yL`N|Nk{I zC+OYvVZey8656Z5E3y`U7WZMT%I++_2~pn(P&XrO@x t8fc(_1{!Fffd(3Apn(P&XrO@x8fc(_1{!Fffd>8r{0}^?qAUPV0061*Bm4jW From 4f4734ef8374276e49c37eeae55904de7afd914f Mon Sep 17 00:00:00 2001 From: Todd Date: Mon, 26 Sep 2016 21:56:15 -0500 Subject: [PATCH 11/16] Delete command.py --- test/command.py | 32 -------------------------------- 1 file changed, 32 deletions(-) delete mode 100755 test/command.py diff --git a/test/command.py b/test/command.py deleted file mode 100755 index 10f5910..0000000 --- a/test/command.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/python - -from shell import command - -print -print 'Execute a command that succeeds' -print '-------------------------------' -result = command('ls -l', 2) -print 'result.command : %s' % result.command -print 'result.pid : %s' % result.pid -print 'result.retval : %s' % result.retval -print 'result.runtime : %s' % result.runtime -print 'result.output : %s' % result.output -print -print 'Execute a command that doesn\'t exist' -print '------------------------------------' -result = command('/usr/bin/blurple -f opt', 2) -print 'result.command : %s' % result.command -print 'result.pid : %s' % result.pid -print 'result.retval : %s' % result.retval -print 'result.runtime : %s' % result.runtime -print 'result.output : %s' % result.output -print -print 'Execute a command that times out' -print '--------------------------------' -result = command('ping -c5 -i1 -W1 -q google.com', 2) -print 'result.command : %s' % result.command -print 'result.pid : %s' % result.pid -print 'result.retval : %s' % result.retval -print 'result.runtime : %s' % result.runtime -print 'result.output : %s' % result.output -print \ No newline at end of file From 19d92e1245452faeaf67b086249c8be7984e08c6 Mon Sep 17 00:00:00 2001 From: Todd Date: Mon, 26 Sep 2016 21:56:24 -0500 Subject: [PATCH 12/16] Delete hosts.csv --- test/hosts.csv | 100 ------------------------------------------------- 1 file changed, 100 deletions(-) delete mode 100644 test/hosts.csv diff --git a/test/hosts.csv b/test/hosts.csv deleted file mode 100644 index 5701bc8..0000000 --- a/test/hosts.csv +++ /dev/null @@ -1,100 +0,0 @@ -call.com -called.com -came.com -can.com -cannot.com -cant.com -car.com -care.com -carried.com -cars.com -case.com -cases.com -cause.com -cent.com -center.com -central.com -century.com -certain.com -certainly.com -chance.com -change.com -changes.com -character.com -charge.com -chief.com -child.com -children.com -choice.com -christian.com -church.com -city.com -class.com -clear.com -clearly.com -close.com -closed.com -club.com -co.com -cold.com -college.com -color.com -come.com -comes.com -coming.com -committee.com -common.com -communist.com -community.com -company.com -complete.com -completely.com -concerned.com -conditions.com -congress.com -consider.com -considered.com -continued.com -control.com -corner.com -corps.com -cost.com -costs.com -could.com -couldnt.com -countries.com -country.com -county.com -couple.com -course.com -court.com -covered.com -cut.com -daily.com -dark.com -data.com -day.com -days.com -de.com -dead.com -deal.com -death.com -decided.com -decision.com -deep.com -defense.com -degree.com -democratic.com -department.com -described.com -design.com -designed.com -determined.com -developed.com -development.com -did.com -didnt.com -difference.com -different.com -difficult.com -direct.com \ No newline at end of file From 807dee3d23d10d95d8b4c02acde1c828cdd1144a Mon Sep 17 00:00:00 2001 From: Todd Date: Mon, 26 Sep 2016 21:56:32 -0500 Subject: [PATCH 13/16] Delete multi_command.py --- test/multi_command.py | 41 ----------------------------------------- 1 file changed, 41 deletions(-) delete mode 100755 test/multi_command.py diff --git a/test/multi_command.py b/test/multi_command.py deleted file mode 100755 index 0d8473d..0000000 --- a/test/multi_command.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/python - -from sys import exit - -from shell import multi_command - -commands = [] -i = 0 - -print -print 'Execute three commands with varying results' -print '-------------------------------------------' -results = multi_command(['ls -l', '/usr/bin/blurple -f opt', - 'ping -c5 -i1 -W1 -q google.com'], 2) -for result in results: - print 'result%s.command: %s' % (i, result.command) - print 'result%s.pid : %s' % (i, result.pid) - print 'result%s.retval : %s' % (i, result.retval) - print 'result%s.runtime: %s' % (i, result.runtime) - print 'result%s.output : %s' % (i, result.output) - print - i = i + 1 - -""" remove this if you want to continue, will use 150M-200M of RAM """ -exit() - -hosts = open('hosts.csv', 'r') -for host in hosts: - commands.append('ping -c3 -i1 -W1 -q %s' % host.strip()) -hosts.close() - -print 'Execute 100 pings with 100 workers' -print '----------------------------------' -results = multi_command(commands, 5, 100) -for result in results: - print result.command - print result.pid - print result.retval - print result.runtime - print result.output - print \ No newline at end of file From 6dcb8d5e30c274b6f5b5504a5f1e6a9e9be786ff Mon Sep 17 00:00:00 2001 From: Todd Date: Mon, 26 Sep 2016 22:02:46 -0500 Subject: [PATCH 14/16] Update README --- README | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/README b/README index ce42b1e..0d64abe 100644 --- a/README +++ b/README @@ -1,12 +1,12 @@ author ====== Todd Mueller -firstnamelastname@common.google.mail.domain +firstname@firstnamelastname.com https://github.com/microlinux/python-shell version ======= -1.3b +2.0 history ======= @@ -64,10 +64,10 @@ generator: result.cmd command executed result.pid pid of process result.retval retval -result.runtime runtime in milliseconds -result.stdout stdout stripped of vertical and horizontal +result.runtime runtime in milliseconds +result.stdout stdout stripped of vertical and horizontal whitespace -result.stderr stderr stripped of vertical and horizontal +result.stderr stderr stripped of vertical and horizontal whitespace retval = 127 on command not found @@ -78,12 +78,13 @@ On retval 127 or 254 a standard description is returned in stderr. On retval 255, traceback.format_exc() is returned in stderr. ---------------------------------------------------------------- -command( command, timeout=10, stdin=None) +command( command, timeout=10, stdin=None) ---------------------------------------------------------------- Executes a command with a timeout. Returns the result as a namedtuple. -result = command('whoami', 2) +result = command('whoami') +print print result.cmd print result.pid print result.retval @@ -92,16 +93,17 @@ print result.stdout print result.stderr ----------------------------------------------------------------------- -multi_command( commands, timeout=10, workers=4, +multi_command( commands, timeout=10, workers=4, stdins=None ----------------------------------------------------------------------- Executes commands concurrently with individual timeouts in a pool of workers. Length of stdins must match commands, empty indexes must contain None. Returns ordered results as a namedtuple generator. -results = multi_command(['whoami', 'uptime'], 2) +results = multi_command(['whoami', 'uptime']) for result in results: + print print result.cmd print result.pid print result.retval From 198fb568370283276aa686d9e0b011c12bb3475e Mon Sep 17 00:00:00 2001 From: Todd Date: Mon, 26 Sep 2016 22:57:24 -0500 Subject: [PATCH 15/16] Update shell.py --- shell.py | 283 +++++++++++++++++++++++++++---------------------------- 1 file changed, 140 insertions(+), 143 deletions(-) diff --git a/shell.py b/shell.py index 428a015..40e3859 100644 --- a/shell.py +++ b/shell.py @@ -1,18 +1,15 @@ """Functions for executing one shell command and multiple, concurrent shell commands. Repository: - https://github.com/microlinux/python-shell + https://github.com/microlinux/python-shell Author: - Todd Mueller - -Version: - 2.0 20160926 + Todd Mueller License: - This program is free software: you can redistribute it and/or modify it - under the terms of the GNU General Public License as published by the - Free Software Foundation, version 3. See https://www.gnu.org/licenses/. + This program is free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by the + Free Software Foundation, version 3. See https://www.gnu.org/licenses/. """ @@ -28,176 +25,176 @@ """Namedtuple representing the result of a shell command. Fields: - cmd (str): command executed - pid (int): process id - retval (int): return value - runtime (int): runtime in msecs - stdout (str): stdout buffer, horizontal/vertical whitespace stripped - stderr (str): stderr buffer, horizontal/vertical whitespace stripped + cmd (str): command executed + pid (int): process id + retval (int): return value + runtime (int): runtime in msecs + stdout (str): stdout buffer, horizontal/vertical whitespace stripped + stderr (str): stderr buffer, horizontal/vertical whitespace stripped """ def stripper(string): - """Sexily strips horizontal and vertical whitespace from a string. + """Sexily strips horizontal and vertical whitespace from a string. - Args: - string (str): string to strip + Args: + string (str): string to strip - Returns: - str: stripped string + Returns: + str: stripped string - """ + """ - lines = map(str.strip, string.splitlines()) + lines = map(str.strip, string.splitlines()) - while lines and not lines[-1]: - lines.pop() + while lines and not lines[-1]: + lines.pop() - while lines and not lines[0]: - lines.pop(0) + while lines and not lines[0]: + lines.pop(0) - return '\n'.join(lines) + return '\n'.join(lines) def command(cmd, timeout=10, stdin=None): - """Executes one shell command and returns the result. + """Executes one shell command and returns the result. - Args: - cmd (str): command to execute - timeout (int): timeout in seconds (10) - stdin (str): input (None) + Args: + cmd (str): command to execute + timeout (int): timeout in seconds (10) + stdin (str): input (None) - Returns: - CommandResult: result of command + Returns: + CommandResult: result of command """ - kill = lambda this_proc: this_proc.kill() - pid = None - retval = None - runtime = None - stderr = None - stdout = None - - try: - start = time.time() * 1000.0 - proc = subprocess.Popen(shlex.split(cmd), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - stdin=subprocess.PIPE, - close_fds=True, - shell=False) - - timer = threading.Timer(timeout, kill, [proc]) - - timer.start() - stdout, stderr = map(stripper, proc.communicate(input=stdin)) - timer.cancel() - - runtime = int((time.time() * 1000.0) - start) - except OSError: - retval = 127 - stderr = 'command not found' - except: - retval = 255 - stderr = traceback.format_exc() - finally: - if retval == None: - if proc.returncode == -9: - retval = 254 - stderr = 'command timed out' - else: - retval = proc.returncode - - pid = proc.pid - - return CommandResult(cmd, pid, retval, runtime, stdout, stderr) + kill = lambda this_proc: this_proc.kill() + pid = None + retval = None + runtime = None + stderr = None + stdout = None + + try: + start = time.time() * 1000.0 + proc = subprocess.Popen(shlex.split(cmd), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + stdin=subprocess.PIPE, + close_fds=True, + shell=False) + + timer = threading.Timer(timeout, kill, [proc]) + + timer.start() + stdout, stderr = map(stripper, proc.communicate(input=stdin)) + timer.cancel() + + runtime = int((time.time() * 1000.0) - start) + except OSError: + retval = 127 + stderr = 'command not found' + except: + retval = 255 + stderr = traceback.format_exc() + finally: + if retval == None: + if proc.returncode == -9: + retval = 254 + stderr = 'command timed out' + else: + retval = proc.returncode + + pid = proc.pid + + return CommandResult(cmd, pid, retval, runtime, stdout, stderr) def command_wrapper(args): - """Wrapper used my multi_command to pass command arguments as a list.""" + """Wrapper used my multi_command to pass command arguments as a list.""" - return command(*args) + return command(*args) def multi_command(cmds, timeout=10, stdins=None, workers=4): - """Executes multiple shell commands concurrently and returns the results. + """Executes multiple shell commands concurrently and returns the results. - If stdins is given, its length must match that of cmds. If commands do - not require input, their corresponding stdin field must be None. + If stdins is given, its length must match that of cmds. If commands do + not require input, their corresponding stdin field must be None. - Args: - cmds (list): commands to execute - timeout (int): timeout in seconds per command (10) - workers (int): max concurrency (4) - stdins (list): inputs (None) + Args: + cmds (list): commands to execute + timeout (int): timeout in seconds per command (10) + workers (int): max concurrency (4) + stdins (list): inputs (None) - Returns: - CommandResult generator: results of commands in given order + Returns: + CommandResult generator: results of commands in given order - """ + """ - num_cmds = len(cmds) + num_cmds = len(cmds) - if stdins != None: - if len(stdins) != num_cmds: - raise RuntimeError("stdins length doesn't match cmds length") + if stdins != None: + if len(stdins) != num_cmds: + raise RuntimeError("stdins length doesn't match cmds length") - if workers > num_cmds: - workers = num_cmds + if workers > num_cmds: + workers = num_cmds - for i in xrange(num_cmds): - try: - stdin = stdins[i] - except: - stdin = None + for i in xrange(num_cmds): + try: + stdin = stdins[i] + except: + stdin = None - cmds[i] = (cmds[i], timeout, stdin) + cmds[i] = (cmds[i], timeout, stdin) - pool = multiprocessing.Pool(processes=workers) - results = pool.imap(command_wrapper, cmds) + pool = multiprocessing.Pool(processes=workers) + results = pool.imap(command_wrapper, cmds) - pool.close() - pool.join() + pool.close() + pool.join() - return results + return results def test(single_cmds=['ls -l', 'uptime', 'blargh'], multi_cmds=['ls -l', 'uptime', 'ping -c3 -i1 -W1 google.com']): - """Demonstrates command and multi_command functionality. - - Args: - single_cmds (list): commands to run one at a time - multi_cmds (list): commands to run concurrently - - """ - - print - print '-----------------------' - print 'Running single commands' - print '-----------------------' - - for cmd in single_cmds: - result = command(cmd) - print - print 'cmd: %s' % result.cmd - print 'pid: %s' % result.pid - print 'ret: %s' % result.retval - print 'run: %s' % result.runtime - print 'out:' - print result.stdout - print 'err:' - print result.stderr - - print - print '-------------------------' - print 'Running multiple commands' - print '-------------------------' - - for result in multi_command(multi_cmds): - print - print 'cmd: %s' % result.cmd - print 'pid: %s' % result.pid - print 'ret: %s' % result.retval - print 'run: %s' % result.runtime - print 'out:' - print result.stdout - print 'err:' - print result.stderr + """Demonstrates command and multi_command functionality. + + Args: + single_cmds (list): commands to run one at a time + multi_cmds (list): commands to run concurrently + + """ + + print + print '-----------------------' + print 'Running single commands' + print '-----------------------' + + for cmd in single_cmds: + result = command(cmd) + print + print 'cmd: %s' % result.cmd + print 'pid: %s' % result.pid + print 'ret: %s' % result.retval + print 'run: %s' % result.runtime + print 'out:' + print result.stdout + print 'err:' + print result.stderr + + print + print '-------------------------' + print 'Running multiple commands' + print '-------------------------' + + for result in multi_command(multi_cmds): + print + print 'cmd: %s' % result.cmd + print 'pid: %s' % result.pid + print 'ret: %s' % result.retval + print 'run: %s' % result.runtime + print 'out:' + print result.stdout + print 'err:' + print result.stderr From 57c6bd4e1bd57ca199004cc8f08dd50a325b48d2 Mon Sep 17 00:00:00 2001 From: Todd Date: Fri, 14 Oct 2016 18:30:11 -0500 Subject: [PATCH 16/16] Update shell.py --- shell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shell.py b/shell.py index 40e3859..33f2c12 100644 --- a/shell.py +++ b/shell.py @@ -110,7 +110,7 @@ def command(cmd, timeout=10, stdin=None): return CommandResult(cmd, pid, retval, runtime, stdout, stderr) def command_wrapper(args): - """Wrapper used my multi_command to pass command arguments as a list.""" + """Wrapper used my multi_command() to pass command() arguments as a list.""" return command(*args)