From b3c67490325e19db88b0ae93f4bb2f57092296df Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 00:43:59 -0700 Subject: [PATCH 001/138] whitespace formatting --- mansnip | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/mansnip b/mansnip index 0cfd903..cb1fd80 100755 --- a/mansnip +++ b/mansnip @@ -34,6 +34,7 @@ term_indent = False indent_window = [] stack_guess = [] stack_indent = [] +last_stack_guess = [] # There's lots of let's say "creative" ways to format man pages, so # we have a plain-text version in case our sophisticated searching @@ -104,11 +105,23 @@ for line_num, line in enumerate(man_input): while not len(buf[0]): buf = buf[1:] while not len(buf[-1]): buf = buf[:-1] - stack_print = stack_start[0] - if len(stack_start) > 2: - stack_print += spacer + spacer.join(stack_start[1:-1]) + stack_small = stack_start[:] + for i in range(0, min(len(last_stack_guess),len(stack_start))): + if last_stack_guess[i] == stack_start[i]: + stack_small = stack_small[1:] - print("{:<6}{}\n{}\n".format(buf_start, stack_print, '\n'.join(buf))) + last_stack_guess = stack_start[:] + + if len(stack_small) > 1: + stack_print = stack_small[0] + if len(stack_small) > 2: + stack_print += spacer + spacer.join(stack_small[1:-1]) + stack_print += "\n" + else: + stack_print = '' + buf[0] = re.sub('^\s{6}', '', buf[0]) + + print("{:<6}{}{}\n".format(buf_start, stack_print, '\n'.join(buf))) buf = [] term_indent = False From 0e7c079b03fd2d24ddbf1e813136d0b7d3d39ccb Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 00:52:21 -0700 Subject: [PATCH 002/138] optional indent length --- README.md | 17 +++++++++-------- mansnip | 8 +++++--- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 5d20241..7b82249 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ This is insane. Why is this basic functionality not there? Why can't I just do something like this: $ mansnip bash declare - 3812 + 3812 SHELL BUILTIN COMMANDS declare [-aAfFgilnrtux] [-p] [name[=value] ...] typeset [-aAfFgilnrtux] [-p] [name[=value] ...] Declare variables and/or give them attributes. If no names are given then display the @@ -48,7 +48,7 @@ Why can't I just do something like this: Well, now you can! (ok that was a little obvious) -You even get the line number for further investigation. +You even get the line number and context for further investigation. Pssttt ... it's also [on pypi](https://pypi.org/project/mansnip-kristopolous/): `pip3 install mansnip-kristopolous` @@ -71,24 +71,25 @@ Anyway, so inside the document source that leads to the pretty man page, there i Alright, what do those things mean? You can see that in `man 7 man` or actually, $ mansnip 7 man .TP .PD .B .P .I - 55 + 55 DESCRIPTION + Fonts .B Bold - 61 - .I Italics + 61 .I Italics - 97 + 97 Normal paragraphs .P Same as .PP (begin a new paragraph). - 124 + 124 Indented paragraph macros .TP i Begin paragraph with hanging tag. The tag is given on the next line, but its re‐ sults are like those of the .IP command. - 147 + 147 Miscellaneous macros .PD d Set inter-paragraph vertical distance to d (if omitted, d=0.4v); does not cause a break. + Hrmm, well that's a problem. It's effectively just a stylesheet. In fact, the format doesn't look fundamentally different than it did in [UNIX v0 in 1970](https://github.com/DoctorWkt/pdp7-unix/blob/master/man/stat.1). A mere 2 years after [Engelbart demoed a prototype hypertext](https://en.wikipedia.org/wiki/The_Mother_of_All_Demos) system, this is well before the [semantic web](https://en.wikipedia.org/wiki/Semantic_Web). "But wait," you say. "If I do `man --html bash` and then wait for the glacially slow [groff HTML post-processor](http://git.savannah.gnu.org/cgit/groff.git/tree/src/devices/grohtml) I do indeed get links!" diff --git a/mansnip b/mansnip index cb1fd80..5307406 100755 --- a/mansnip +++ b/mansnip @@ -42,6 +42,8 @@ last_stack_guess = [] man_input_plain = re.sub('(.\x08)', '', man_input).split('\n') man_input = man_input.split('\n') +rs = 5 if len(man_input) < 1e4 else 7 + for line_num, line in enumerate(man_input): line = line.strip('\n') indent = re.match('^(\s*)', line).end() @@ -99,7 +101,7 @@ for line_num, line in enumerate(man_input): elif term_indent: if indent > 1 and (indent < term_indent or (not is_def and indent == term_indent)): - spacer = '\n' + ' ' * 6 + spacer = '\n' + ' ' * rs if (len(buf) == 2 and indent == term_indent and line_def) or len(buf) > 2: while not len(buf[0]): buf = buf[1:] @@ -119,9 +121,9 @@ for line_num, line in enumerate(man_input): stack_print += "\n" else: stack_print = '' - buf[0] = re.sub('^\s{6}', '', buf[0]) + buf[0] = re.sub('^\s{%d}' % rs, '', buf[0]) - print("{:<6}{}{}\n".format(buf_start, stack_print, '\n'.join(buf))) + print(("{:<%d}{}{}\n" % rs).format(buf_start, stack_print, '\n'.join(buf))) buf = [] term_indent = False From f7b50368c71f8c279ed637bdb271a94b11b1f6d5 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 01:11:15 -0700 Subject: [PATCH 003/138] cleanup --- mansnip | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/mansnip b/mansnip index 5307406..ac359b5 100755 --- a/mansnip +++ b/mansnip @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -import sys, re, os +import sys, re, os, logging # The bastard manpage for less, has a number of things like this # @@ -14,6 +14,8 @@ import sys, re, os # wrapper to man as opposed to just parsing # +logging.basicConfig(level=(os.environ.get('LOGLEVEL') or 'info').upper()) + # This allows us to do mansnip or just cutoff = 3 if sys.argv[1].isnumeric() else 2 os.environ['MAN_KEEP_FORMATTING'] = '1' @@ -22,8 +24,15 @@ man_input = os.popen(cmd).read() pack = sys.argv[cutoff:] +opts = '|'.join(pack) # here's our utilization of the formatting (_) -my_re = '^\s*({})([\s_].*$|$)'.format('|'.join(pack)) +# +# ffmpeg uses [ at times (see hwaccel). +# the second part is for the long options, sometimes (git config) +# specified by commas +# +my_re = '^\s*(({})([\s_\[].*|, .*|)|-.*({}))$'.format(opts, opts) +logging.debug(my_re) is_def = False line_def = False @@ -50,7 +59,6 @@ for line_num, line in enumerate(man_input): indent_window = indent_window[-2:] + [indent] if len(line): - #stack_indent = stack_indent[-2:] + [indent] while len(stack_indent) and indent <= stack_indent[-1]: stack_indent.pop() stack_guess.pop() @@ -59,8 +67,6 @@ for line_num, line in enumerate(man_input): stack_indent.append(indent) stack_guess.append(man_input[line_num].strip()) - #print("{:4}| {:12}\n{}".format(indent, ' '.join([str(x) for x in stack_indent]), '\n'.join(stack_guess))) - if not len(buf): res = re.match(my_re, line) or re.match(my_re, man_input_plain[line_num]) if res: @@ -91,9 +97,6 @@ for line_num, line in enumerate(man_input): elif indent_window[0] == indent and indent_window[1] == 0: line_def = True - #print(indent_window, is_def, line_def) - #print("<{}>".format(res[1])) - buf.append(line) buf_start = line_num stack_start = stack_guess[:] From be14daf32fa8bb8760e654f0355a14bb957edeb2 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 01:13:06 -0700 Subject: [PATCH 004/138] version bump --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index aa6f5d6..1bf1d48 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setuptools.setup( name="mansnip-kristopolous", - version="0.2", + version="0.3", scripts=["mansnip"], author="Chris McKenzie", author_email="kristopolous@yahoo.com", From dc938c17daac9ec7fcfbad0600f5267fa0f71238 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 01:32:21 -0700 Subject: [PATCH 005/138] updating the commenting --- mansnip | 46 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/mansnip b/mansnip index ac359b5..29b4baf 100755 --- a/mansnip +++ b/mansnip @@ -14,7 +14,7 @@ import sys, re, os, logging # wrapper to man as opposed to just parsing # -logging.basicConfig(level=(os.environ.get('LOGLEVEL') or 'info').upper()) +logging.basicConfig(level=(os.environ.get('LOGLEVEL') or 'critical').upper()) # This allows us to do mansnip or just cutoff = 3 if sys.argv[1].isnumeric() else 2 @@ -25,21 +25,22 @@ man_input = os.popen(cmd).read() pack = sys.argv[cutoff:] opts = '|'.join(pack) -# here's our utilization of the formatting (_) # # ffmpeg uses [ at times (see hwaccel). # the second part is for the long options, sometimes (git config) # specified by commas # my_re = '^\s*(({})([\s_\[].*|, .*|)|-.*({}))$'.format(opts, opts) -logging.debug(my_re) +logging.info(my_re) is_def = False line_def = False -buf = [] -buf_start = False stack_start = False term_indent = False + +buf_start = False +buf = [] + indent_window = [] stack_guess = [] stack_indent = [] @@ -51,14 +52,25 @@ last_stack_guess = [] man_input_plain = re.sub('(.\x08)', '', man_input).split('\n') man_input = man_input.split('\n') +# We try to output something nice and readable. If the input is a +# lot of lines, say zshall (all of zsh), then we set aside a bit of +# space for the indentation. rs = 5 if len(man_input) < 1e4 else 7 for line_num, line in enumerate(man_input): line = line.strip('\n') + + # Establish the "indent", this is crucial to how essentially + # everything works. indent = re.match('^(\s*)', line).end() indent_window = indent_window[-2:] + [indent] if len(line): + logging.debug(line) + + # Our nice stack guessing system essentially keeps a stack + # of the indents and then based on the current indent either + # pushes or pops on to the stack. while len(stack_indent) and indent <= stack_indent[-1]: stack_indent.pop() stack_guess.pop() @@ -68,7 +80,11 @@ for line_num, line in enumerate(man_input): stack_guess.append(man_input[line_num].strip()) if not len(buf): + + # See if our ansi escaped or plain text version has what we are looking for. + # yes that makes it a bit slower, but not as slow as doing it manually. res = re.match(my_re, line) or re.match(my_re, man_input_plain[line_num]) + if res: # This is sheer frantic handwaving for things like this (From bash) # @@ -106,10 +122,19 @@ for line_num, line in enumerate(man_input): if indent > 1 and (indent < term_indent or (not is_def and indent == term_indent)): spacer = '\n' + ' ' * rs if (len(buf) == 2 and indent == term_indent and line_def) or len(buf) > 2: - - while not len(buf[0]): buf = buf[1:] - while not len(buf[-1]): buf = buf[:-1] - + # This is a lot of fancy formatting. We want the + # vertical whitespace between the heading and snippet to be consistent. + # If we simply removed the empty lines then it would dump the interstitial + # (as in, in the middle of our content) empty lines. so we want to get rid + # of leading and trailing new lines. + # + # So we join the array, strip out the trailing new lines, then split it up + # again. eh, it's fine. + # + buf = '\n'.join(buf).strip('\n').split('\n') + + # If we show the same breadcrumb every time it gets kinda laborious and + # repetitive so we cleverly hide the redundancy. stack_small = stack_start[:] for i in range(0, min(len(last_stack_guess),len(stack_start))): if last_stack_guess[i] == stack_start[i]: @@ -117,6 +142,9 @@ for line_num, line in enumerate(man_input): last_stack_guess = stack_start[:] + # Now we have to figure out how to print it out to make it look right. + # Sometimes it can be inline. The %/fmt double formatting trick comes + # in handle here. if len(stack_small) > 1: stack_print = stack_small[0] if len(stack_small) > 2: From 1a81f0fe5b27c47efc6a20f58d481c454f5294ef Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 01:46:15 -0700 Subject: [PATCH 006/138] filtering of filler words --- mansnip | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/mansnip b/mansnip index 29b4baf..2035ac1 100755 --- a/mansnip +++ b/mansnip @@ -46,10 +46,15 @@ stack_guess = [] stack_indent = [] last_stack_guess = [] +# Words we can just leave out of the breadcrumb. +filler_terms = ['DESCRIPTION','OPTIONS'] + +clean_ansi = lambda w: re.sub('(.\x08)', '', w) + # There's lots of let's say "creative" ways to format man pages, so # we have a plain-text version in case our sophisticated searching # method fails. -man_input_plain = re.sub('(.\x08)', '', man_input).split('\n') +man_input_plain = clean_ansi(man_input).split('\n') man_input = man_input.split('\n') # We try to output something nice and readable. If the input is a @@ -142,6 +147,10 @@ for line_num, line in enumerate(man_input): last_stack_guess = stack_start[:] + if len(stack_small) > 1: + if clean_ansi(stack_small[0]) in filler_terms: + stack_small = stack_small[1:] + # Now we have to figure out how to print it out to make it look right. # Sometimes it can be inline. The %/fmt double formatting trick comes # in handle here. From 0dd0492555449cd8491f977f30c820635770b06a Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 01:49:49 -0700 Subject: [PATCH 007/138] partial fix of #2 --- mansnip | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mansnip b/mansnip index 2035ac1..6110597 100755 --- a/mansnip +++ b/mansnip @@ -30,7 +30,7 @@ opts = '|'.join(pack) # the second part is for the long options, sometimes (git config) # specified by commas # -my_re = '^\s*(({})([\s_\[].*|, .*|)|-.*({}))$'.format(opts, opts) +my_re = '^\s*(({})([\s_\[].*|, .*|)|-.*\s({}))$'.format(opts, opts) logging.info(my_re) is_def = False From e838b9e89b58e8bec09fce756a30a05f2f93dccf Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 02:02:21 -0700 Subject: [PATCH 008/138] fixes #1 and #2 --- mansnip | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mansnip b/mansnip index 6110597..798dfa3 100755 --- a/mansnip +++ b/mansnip @@ -124,7 +124,9 @@ for line_num, line in enumerate(man_input): term_indent = indent elif term_indent: - if indent > 1 and (indent < term_indent or (not is_def and indent == term_indent)): + if (indent == 0 and len(line)) or ( + indent > 1 and (indent < term_indent or (not is_def and indent == term_indent) + )): spacer = '\n' + ' ' * rs if (len(buf) == 2 and indent == term_indent and line_def) or len(buf) > 2: # This is a lot of fancy formatting. We want the From d5893beb1b00f3c4e8b21b6eb6390bdbadfe6cf4 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 02:17:59 -0700 Subject: [PATCH 009/138] implementing multi-line single-level breadcrumbs --- mansnip | 52 ++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 48 insertions(+), 4 deletions(-) diff --git a/mansnip b/mansnip index 798dfa3..2d61e85 100755 --- a/mansnip +++ b/mansnip @@ -51,22 +51,28 @@ filler_terms = ['DESCRIPTION','OPTIONS'] clean_ansi = lambda w: re.sub('(.\x08)', '', w) +# # There's lots of let's say "creative" ways to format man pages, so # we have a plain-text version in case our sophisticated searching # method fails. +# man_input_plain = clean_ansi(man_input).split('\n') man_input = man_input.split('\n') +# # We try to output something nice and readable. If the input is a # lot of lines, say zshall (all of zsh), then we set aside a bit of # space for the indentation. +# rs = 5 if len(man_input) < 1e4 else 7 for line_num, line in enumerate(man_input): line = line.strip('\n') + # # Establish the "indent", this is crucial to how essentially # everything works. + # indent = re.match('^(\s*)', line).end() indent_window = indent_window[-2:] + [indent] @@ -76,9 +82,40 @@ for line_num, line in enumerate(man_input): # Our nice stack guessing system essentially keeps a stack # of the indents and then based on the current indent either # pushes or pops on to the stack. - while len(stack_indent) and indent <= stack_indent[-1]: - stack_indent.pop() - stack_guess.pop() + + # + # This one is kinda tricky. If we are looking for -z in say, zshall + # + # zcompile [ -U ] [ -z | -k ] [ -R | -M ] file [ name ... ] + # zcompile -ca [ -m ] [ -R | -M ] file [ name ... ] + # zcompile -t file [ name ... ] + # -z + # + # Is what we want. So what we want is a longer version of what we currently have + # + # But it gets trickier since we suppress empty lines in this algorithm. + # + # r Same as fc -e -. + # + # read [ -rszpqAclneE ] [ -t [ num ] ] [ -k [ num ] ] [ -d delim ] + # [ -u n ] [ name[?prompt] ] [ name ... ] + # + # Also from zsh, will be interpreted as a group, so we have to just this once + # use our real indent tracker to catch it. + # + if len(stack_indent) and indent == stack_indent[-1] and indent_window[-2] != 0: + # + # Historically string growing like this was always dog, I don't know + # if it's still super slow. Mostly of the time we won't actually be using + # it. Since we aren't doing a stream, we could just reconstruct it by + # storing "pointers" (indexes here) ... sometime in the future. + # + stack_guess[-1] += '\n' + man_input[line_num] + + else: + while len(stack_indent) and indent <= stack_indent[-1]: + stack_indent.pop() + stack_guess.pop() if not len(stack_indent) or indent > stack_indent[-1]: stack_indent.append(indent) @@ -91,6 +128,7 @@ for line_num, line in enumerate(man_input): res = re.match(my_re, line) or re.match(my_re, man_input_plain[line_num]) if res: + # # This is sheer frantic handwaving for things like this (From bash) # # declare [-aAfFgilnrtux] [-p] [name[=value] ...] @@ -108,7 +146,9 @@ for line_num, line in enumerate(man_input): if indent_window[0] > indent and indent_window[1] == 0: is_def = True + # # From man 7 man we get things like this: + # # .I Italics # # .IB Italics alternating with bold @@ -129,6 +169,7 @@ for line_num, line in enumerate(man_input): )): spacer = '\n' + ' ' * rs if (len(buf) == 2 and indent == term_indent and line_def) or len(buf) > 2: + # # This is a lot of fancy formatting. We want the # vertical whitespace between the heading and snippet to be consistent. # If we simply removed the empty lines then it would dump the interstitial @@ -140,8 +181,10 @@ for line_num, line in enumerate(man_input): # buf = '\n'.join(buf).strip('\n').split('\n') + # # If we show the same breadcrumb every time it gets kinda laborious and # repetitive so we cleverly hide the redundancy. + # stack_small = stack_start[:] for i in range(0, min(len(last_stack_guess),len(stack_start))): if last_stack_guess[i] == stack_start[i]: @@ -152,10 +195,11 @@ for line_num, line in enumerate(man_input): if len(stack_small) > 1: if clean_ansi(stack_small[0]) in filler_terms: stack_small = stack_small[1:] - + # # Now we have to figure out how to print it out to make it look right. # Sometimes it can be inline. The %/fmt double formatting trick comes # in handle here. + # if len(stack_small) > 1: stack_print = stack_small[0] if len(stack_small) > 2: From dba096fa03f0c5efa8fdba833b3c43e4ace8d2cd Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 02:32:44 -0700 Subject: [PATCH 010/138] getting some lsof use cases --- mansnip | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/mansnip b/mansnip index 2d61e85..6aa47fe 100755 --- a/mansnip +++ b/mansnip @@ -20,17 +20,19 @@ logging.basicConfig(level=(os.environ.get('LOGLEVEL') or 'critical').upper()) cutoff = 3 if sys.argv[1].isnumeric() else 2 os.environ['MAN_KEEP_FORMATTING'] = '1' cmd = '/usr/bin/man {}'.format(' '.join(sys.argv[1:cutoff])) +logging.info(cmd) man_input = os.popen(cmd).read() pack = sys.argv[cutoff:] opts = '|'.join(pack) # -# ffmpeg uses [ at times (see hwaccel). -# the second part is for the long options, sometimes (git config) -# specified by commas +# lsof uses +|- syntax, that's what the very fun (\+\||) is for +# Then comes the term expansion and a few trailing characters. +# ffmpeg uses [ at times (see hwaccel), many things use ',' to show a second option. +# the second part is for the long options, sometimes (git config) specified by commas # -my_re = '^\s*(({})([\s_\[].*|, .*|)|-.*\s({}))$'.format(opts, opts) +my_re = '^\s*((\+\||)({})([\s_\[].*|, .*|)|-.*\s({}))$'.format(opts, opts) logging.info(my_re) is_def = False From d3dd3e3d9d0f08155de45c180338b358c48c481c Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 02:35:47 -0700 Subject: [PATCH 011/138] bumping the version for pypi --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1bf1d48..c3af0ce 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setuptools.setup( name="mansnip-kristopolous", - version="0.3", + version="0.3.1", scripts=["mansnip"], author="Chris McKenzie", author_email="kristopolous@yahoo.com", From f0f8cc41bf892b618491f04dc552f80222ab539f Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 02:56:24 -0700 Subject: [PATCH 012/138] adding an image --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 7b82249..9cf1b67 100644 --- a/README.md +++ b/README.md @@ -111,3 +111,8 @@ Man pages are ridiculously consistent as far as non-semantically structured text Thanks. enjoy. +#### Bonus round! + +Here's a rather fancy demo for looking for the documentation for every "-z" option in all of the zsh man pages (zshall) + +![zshall for all](http://i.9ol.es/mansnip.webp) From e4d535deb9e304c510201dbd49ecbdef9a4374bd Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 03:02:32 -0700 Subject: [PATCH 013/138] adding an image --- README.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 9cf1b67..7eb7f22 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,10 @@ You even get the line number and context for further investigation. Pssttt ... it's also [on pypi](https://pypi.org/project/mansnip-kristopolous/): `pip3 install mansnip-kristopolous` +Here's a rather fancy demo for looking for the documentation for every "-z" option in all of the zsh man pages (zshall) + +![zshall for all](http://i.9ol.es/mansnip.webp) + ## Why hasn't this existed forever? Man pages don't really encode a lot of semantic detail. The format is pretty old. There's been a number of attempted replacements, such as [GNU info](https://www.gnu.org/software/texinfo/manual/info-stnd/) and [BSD mdoc](https://mandoc.bsd.lv/) (`man 7 mandoc_mdoc`) but the ones you use on your system are probably just the traditional boring old man files. Ah, inertia. @@ -111,8 +115,3 @@ Man pages are ridiculously consistent as far as non-semantically structured text Thanks. enjoy. -#### Bonus round! - -Here's a rather fancy demo for looking for the documentation for every "-z" option in all of the zsh man pages (zshall) - -![zshall for all](http://i.9ol.es/mansnip.webp) From 39fea60b642338bd1279f5b44cf9d2b6527f0bde Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 03:07:27 -0700 Subject: [PATCH 014/138] adding an image --- README.md | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 7eb7f22..55d8f76 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,10 @@ ### The Problem -Some man pages can be very unwieldy and hard to navigate. - -Let's say you were looking for the syntax of the built-in `declare` command inside of `bash(1)`: +Some man pages can be very unwieldy and hard to navigate. Let's say you were looking for the syntax of the built-in `declare` command inside of `bash(1)`: > I fire up `man bash` and just simply search for the string "declare" with the "/" command! Easy peasy! -Welllll, here's the problem: +Here's the problem: $ man bash | grep -c declare 34 @@ -46,15 +44,15 @@ Why can't I just do something like this: ... $ -Well, now you can! (ok that was a little obvious) +Well, now you can! -You even get the line number and context for further investigation. +Watch how mansnip shows everything with a "-z" option in the 25,888 lines of the zsh man pages (zshall) -Pssttt ... it's also [on pypi](https://pypi.org/project/mansnip-kristopolous/): `pip3 install mansnip-kristopolous` +![zshall for all](http://i.9ol.es/mansnip.webp) -Here's a rather fancy demo for looking for the documentation for every "-z" option in all of the zsh man pages (zshall) +But wait, there's more! You'll also get the line number and hierarchical context totally free! -![zshall for all](http://i.9ol.es/mansnip.webp) +You'll even get to install it easily [through pypi](https://pypi.org/project/mansnip-kristopolous/): `pip3 install mansnip-kristopolous` ## Why hasn't this existed forever? From 743450d20859c20bfe0adc929fec323dd1ca7b39 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 03:18:24 -0700 Subject: [PATCH 015/138] updating the commenting --- README.md | 25 +++++-------------------- 1 file changed, 5 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 55d8f76..dfd647d 100644 --- a/README.md +++ b/README.md @@ -25,26 +25,11 @@ This is insane. Why is this basic functionality not there? ## Introducting Mansnip! -Why can't I just do something like this: - - $ mansnip bash declare - 3812 SHELL BUILTIN COMMANDS - declare [-aAfFgilnrtux] [-p] [name[=value] ...] - typeset [-aAfFgilnrtux] [-p] [name[=value] ...] - Declare variables and/or give them attributes. If no names are given then display the - values of variables. The -p option will display the attributes and values of each - name. When -p is used with name arguments, additional options, other than -f and -F, - are ignored. When -p is supplied without name arguments, it will display the at‐ - tributes and values of all variables having the attributes specified by the additional - options. If no other options are supplied with -p, declare will display the at‐ - tributes and values of all shell variables. The -f option will restrict the display - to shell functions. The -F option inhibits the display of function definitions; only - the function name and attributes are printed. If the extdebug shell option is enabled - using shopt, the source file name and line number where each name is defined are dis‐ - ... - $ - -Well, now you can! +Introducing the revolutionary way to navigate through manpages, only with mansnip. + +![zshall for all](http://i.9ol.es/msfade.webp) + +Now you to can just zip through documentation, saving precious time you can use to write github readmes like you're trying to sell ginsu steak knives. Watch how mansnip shows everything with a "-z" option in the 25,888 lines of the zsh man pages (zshall) From f74580b70ddc65f6c6efdda7e469f31d83dc1335 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 03:24:14 -0700 Subject: [PATCH 016/138] updating the commenting --- README.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index dfd647d..79596da 100644 --- a/README.md +++ b/README.md @@ -29,15 +29,20 @@ Introducing the revolutionary way to navigate through manpages, only with mansni ![zshall for all](http://i.9ol.es/msfade.webp) -Now you to can just zip through documentation, saving precious time you can use to write github readmes like you're trying to sell ginsu steak knives. +**With mansnip** you'll just zip through documentation, saving precious time you can use to write github readmes like you're trying to sell ginsu steak knives. -Watch how mansnip shows everything with a "-z" option in the 25,888 lines of the zsh man pages (zshall) +Watch how mansnip shows everything with a "-z" option in the 25,888 lines of the zsh man pages. ![zshall for all](http://i.9ol.es/mansnip.webp) But wait, there's more! You'll also get the line number and hierarchical context totally free! -You'll even get to install it easily [through pypi](https://pypi.org/project/mansnip-kristopolous/): `pip3 install mansnip-kristopolous` +You'll even get to install it easily [through pypi](https://pypi.org/project/mansnip-kristopolous/). + +Here's how to install `pip3 install mansnip-kristopolous` + + ＦＡＤＥ ＴＯ ＢＬＡＣＫ + ＥＮＤ ＳＣＥＮＥ ## Why hasn't this existed forever? From 1346f50ac467df61393479f8eaa448c6b564e913 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 03:24:44 -0700 Subject: [PATCH 017/138] updating the commenting --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 79596da..8474a94 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,7 @@ You'll even get to install it easily [through pypi](https://pypi.org/project/man Here's how to install `pip3 install mansnip-kristopolous` ＦＡＤＥ ＴＯ ＢＬＡＣＫ + ＥＮＤ ＳＣＥＮＥ ## Why hasn't this existed forever? From 94a91a8a6c4ce9a80654e61935dbba1c2a8710b4 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 03:25:14 -0700 Subject: [PATCH 018/138] updating the commenting --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8474a94..f8924a7 100644 --- a/README.md +++ b/README.md @@ -41,9 +41,9 @@ You'll even get to install it easily [through pypi](https://pypi.org/project/man Here's how to install `pip3 install mansnip-kristopolous` - ＦＡＤＥ ＴＯ ＢＬＡＣＫ + ＦＡＤＥ ＴＯ ＢＬＡＣＫ - ＥＮＤ ＳＣＥＮＥ + ＥＮＤ ＳＣＥＮＥ ## Why hasn't this existed forever? From 638a5b5bc52a87f70736cafa05ce2b05181f30af Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 03:45:00 -0700 Subject: [PATCH 019/138] updating the commenting --- README.md | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index f8924a7..83bf4da 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,8 @@ -### The Problem +**Don't you hate** wasting your time navigating through manpages with the clunky search tools of less? -Some man pages can be very unwieldy and hard to navigate. Let's say you were looking for the syntax of the built-in `declare` command inside of `bash(1)`: +Try to find the usage of "declare" built-in in the bash man page and you'll need to incrementally slodge through 34 results using the 'n' key, which here stands for 'nope, nope, nope'. -> I fire up `man bash` and just simply search for the string "declare" with the "/" command! Easy peasy! - -Here's the problem: - - $ man bash | grep -c declare - 34 - -You'll need to incrementally slodge through 34 of them using the 'n' key, which here stands for 'nope, nope, nope'. - -This is the best optimization I know of: +Even the methods of experts struggle to fix these basic problems. $ man bash -N turn line numbering on @@ -19,13 +10,11 @@ This is the best optimization I know of: &[enter] turn off the filtered view g(number) go to that line -There you go. I remember navigating in the 1980s as well. - -This is insane. Why is this basic functionality not there? +Let's finally leave the 1980s ways of stumbling through manuals behind us. Let's say goodbye to these problems once and for all! ## Introducting Mansnip! -Introducing the revolutionary way to navigate through manpages, only with mansnip. +Mansnip is a revolutionary way to navigate through manpages, a tool that no terminal should be without! ![zshall for all](http://i.9ol.es/msfade.webp) @@ -39,7 +28,7 @@ But wait, there's more! You'll also get the line number and hierarchical context You'll even get to install it easily [through pypi](https://pypi.org/project/mansnip-kristopolous/). -Here's how to install `pip3 install mansnip-kristopolous` +Here's how do it `pip3 install mansnip-kristopolous`. Act now, servers are standing by. ＦＡＤＥ ＴＯ ＢＬＡＣＫ From e27071782e84d1835ed5363d7dc831144adb344b Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 03:46:20 -0700 Subject: [PATCH 020/138] updating the commenting --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 83bf4da..802dbb1 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ **Don't you hate** wasting your time navigating through manpages with the clunky search tools of less? -Try to find the usage of "declare" built-in in the bash man page and you'll need to incrementally slodge through 34 results using the 'n' key, which here stands for 'nope, nope, nope'. +Ever try to find the usage of things like the "declare" built-in in the bash(1) man page only to incrementally slodge through 34 results using the 'n' key going 'nope, nope, nope'. -Even the methods of experts struggle to fix these basic problems. +Even the methods of experts struggle to fix these problems. $ man bash -N turn line numbering on @@ -10,7 +10,7 @@ Even the methods of experts struggle to fix these basic problems. &[enter] turn off the filtered view g(number) go to that line -Let's finally leave the 1980s ways of stumbling through manuals behind us. Let's say goodbye to these problems once and for all! +Let's finally leave the 1980s ways of stumbling through manuals behind us and say goodbye to these problems once and for all! ## Introducting Mansnip! From b2925fa3416af54de06b3f1d0d601fa2b49d3eeb Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 03:46:52 -0700 Subject: [PATCH 021/138] updating the commenting --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 802dbb1..aefe133 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ **Don't you hate** wasting your time navigating through manpages with the clunky search tools of less? -Ever try to find the usage of things like the "declare" built-in in the bash(1) man page only to incrementally slodge through 34 results using the 'n' key going 'nope, nope, nope'. +Ever try to find the usage of things like the "declare" built-in in the `bash(1)` man page only to incrementally slodge through 34 results using the 'n' key going 'nope, nope, nope'? Even the methods of experts struggle to fix these problems. From fc4d42a0ea8e3789a4abc8243720545bd1209f0f Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 03:47:32 -0700 Subject: [PATCH 022/138] updating the commenting --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index aefe133..0f58335 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ **Don't you hate** wasting your time navigating through manpages with the clunky search tools of less? -Ever try to find the usage of things like the "declare" built-in in the `bash(1)` man page only to incrementally slodge through 34 results using the 'n' key going 'nope, nope, nope'? +Ever try to find things like the "declare" built-in in the `bash(1)` man page only to incrementally slodge through 34 results using the 'n' key going 'nope, nope, nope'? Even the methods of experts struggle to fix these problems. From 4b3ed6b19e3bd4bad989b4eefd4126f87579c6c2 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 03:49:39 -0700 Subject: [PATCH 023/138] updating the commenting --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0f58335..661fcaa 100644 --- a/README.md +++ b/README.md @@ -30,9 +30,9 @@ You'll even get to install it easily [through pypi](https://pypi.org/project/man Here's how do it `pip3 install mansnip-kristopolous`. Act now, servers are standing by. - ＦＡＤＥ ＴＯ ＢＬＡＣＫ +ＦＡＤＥ ＴＯ ＢＬＡＣＫ - ＥＮＤ ＳＣＥＮＥ +ＥＮＤ ＳＣＥＮＥ ## Why hasn't this existed forever? From e4e3cd81b884f8ef2df2990deaff5fb540c1541d Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 04:16:44 -0700 Subject: [PATCH 024/138] the old way --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 661fcaa..2638113 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ Ever try to find things like the "declare" built-in in the `bash(1)` man page only to incrementally slodge through 34 results using the 'n' key going 'nope, nope, nope'? +![the old way](http://i.9ol.es/animate.gif) + Even the methods of experts struggle to fix these problems. $ man bash @@ -16,7 +18,7 @@ Let's finally leave the 1980s ways of stumbling through manuals behind us and sa Mansnip is a revolutionary way to navigate through manpages, a tool that no terminal should be without! -![zshall for all](http://i.9ol.es/msfade.webp) +![mansnip is amazing](http://i.9ol.es/msfade.webp) **With mansnip** you'll just zip through documentation, saving precious time you can use to write github readmes like you're trying to sell ginsu steak knives. From 16ba9422af41b98086f27b8c77a2ba29d5725121 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 04:18:16 -0700 Subject: [PATCH 025/138] the old way --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2638113..9434a15 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,9 @@ But wait, there's more! You'll also get the line number and hierarchical context You'll even get to install it easily [through pypi](https://pypi.org/project/mansnip-kristopolous/). -Here's how do it `pip3 install mansnip-kristopolous`. Act now, servers are standing by. +Here's how do it `pip3 install mansnip-kristopolous`. + +Act now, servers are standing by. ＦＡＤＥ ＴＯ ＢＬＡＣＫ From 523836d1f12cd42aa228085633c89b3b3b56abfa Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 04:19:51 -0700 Subject: [PATCH 026/138] the old way --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9434a15..e8e16e8 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Even the methods of experts struggle to fix these problems. &[enter] turn off the filtered view g(number) go to that line -Let's finally leave the 1980s ways of stumbling through manuals behind us and say goodbye to these problems once and for all! +Now you'll leave the 1980s ways of stumbling through manuals behind you and say goodbye to these problems once and for all! ## Introducting Mansnip! From a7fff9cd67491877090e4ce7f722d6d702d6d7f5 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 04:22:15 -0700 Subject: [PATCH 027/138] the old way --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e8e16e8..890f56d 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -**Don't you hate** wasting your time navigating through manpages with the clunky search tools of less? +**Don't you hate** wasting time navigating through manpages with the clunky search tools of less? Ever try to find things like the "declare" built-in in the `bash(1)` man page only to incrementally slodge through 34 results using the 'n' key going 'nope, nope, nope'? @@ -12,7 +12,7 @@ Even the methods of experts struggle to fix these problems. &[enter] turn off the filtered view g(number) go to that line -Now you'll leave the 1980s ways of stumbling through manuals behind you and say goodbye to these problems once and for all! +Soon you'll leave the 1980s ways of stumbling through manuals behind you and say goodbye to these problems once and for all! ## Introducting Mansnip! From e11326c534beb32a009eea7fd5be97ff4dbbb3e0 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 04:27:28 -0700 Subject: [PATCH 028/138] the old way --- mansnip | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mansnip b/mansnip index 6aa47fe..f34c313 100755 --- a/mansnip +++ b/mansnip @@ -1,7 +1,7 @@ #!/usr/bin/env python3 import sys, re, os, logging -# The bastard manpage for less, has a number of things like this +# less and others have a number of things like this # # -hn or --max-back-scroll=n # Specifies a maximum number of lines to scroll backward. If it From 20c91b6c4918722ec91330086727b48b0cc3981e Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 04:33:54 -0700 Subject: [PATCH 029/138] pitch --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 890f56d..0694540 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -**Don't you hate** wasting time navigating through manpages with the clunky search tools of less? +**Don't you hate** wasting time navigating through manpages with those leading pager's clunky search tools? -Ever try to find things like the "declare" built-in in the `bash(1)` man page only to incrementally slodge through 34 results using the 'n' key going 'nope, nope, nope'? +Ever try to find things like the "declare" built-in in the `bash(1)` man page only to incrementally slodge through 34 results using the 'n' key going 'nope, nope, nope'? ![the old way](http://i.9ol.es/animate.gif) @@ -12,6 +12,8 @@ Even the methods of experts struggle to fix these problems. &[enter] turn off the filtered view g(number) go to that line +**Now there's finally a solution!** + Soon you'll leave the 1980s ways of stumbling through manuals behind you and say goodbye to these problems once and for all! ## Introducting Mansnip! From 8af03c11e1768970ef82e158b533b6b1a28a9930 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 04:35:50 -0700 Subject: [PATCH 030/138] pitch --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 0694540..2e63d7a 100644 --- a/README.md +++ b/README.md @@ -12,9 +12,9 @@ Even the methods of experts struggle to fix these problems. &[enter] turn off the filtered view g(number) go to that line -**Now there's finally a solution!** +Stop wasting time with the 1980s ways of stumbling through manuals behind you and say goodbye to these problems once and for all! -Soon you'll leave the 1980s ways of stumbling through manuals behind you and say goodbye to these problems once and for all! +**Now there's finally a solution!** ## Introducting Mansnip! From 6c91c9d162b0da0d8668bc30c8ed724a254ce34c Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 04:36:22 -0700 Subject: [PATCH 031/138] pitch --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2e63d7a..4a4d6fb 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Even the methods of experts struggle to fix these problems. &[enter] turn off the filtered view g(number) go to that line -Stop wasting time with the 1980s ways of stumbling through manuals behind you and say goodbye to these problems once and for all! +Stop wasting time with the 1980s ways of stumbling through manual and say goodbye to these problems once and for all! **Now there's finally a solution!** From a03e14116fd47dd271f670b2f8e61ebbacffe363 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 04:36:50 -0700 Subject: [PATCH 032/138] pitch --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4a4d6fb..ccdb625 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Even the methods of experts struggle to fix these problems. &[enter] turn off the filtered view g(number) go to that line -Stop wasting time with the 1980s ways of stumbling through manual and say goodbye to these problems once and for all! +Stop wasting time with the 1980s ways of manually stumbling through manuals and say goodbye to these problems once and for all! **Now there's finally a solution!** From af796d8b525d7090d6bb01da7620d83a06fc441e Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 04:37:34 -0700 Subject: [PATCH 033/138] pitch --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ccdb625..e953362 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ Even the methods of experts struggle to fix these problems. Stop wasting time with the 1980s ways of manually stumbling through manuals and say goodbye to these problems once and for all! -**Now there's finally a solution!** +**Finally there's a better way!** ## Introducting Mansnip! From fef339229b97bc91f0ba5ec73f979dde9dc1ea3f Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 04:38:52 -0700 Subject: [PATCH 034/138] bumping version to matcy pypi readme --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c3af0ce..22e0d40 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setuptools.setup( name="mansnip-kristopolous", - version="0.3.1", + version="0.3.2", scripts=["mansnip"], author="Chris McKenzie", author_email="kristopolous@yahoo.com", From 90dace1fb833462e7f64cf9af01c591ac0ccd5f0 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 04:50:11 -0700 Subject: [PATCH 035/138] pitch --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e953362..298b9e9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -**Don't you hate** wasting time navigating through manpages with those leading pager's clunky search tools? +**Don't you hate** wasting time navigating through manpages with the leading pager's clunky search tools? -Ever try to find things like the "declare" built-in in the `bash(1)` man page only to incrementally slodge through 34 results using the 'n' key going 'nope, nope, nope'? +Ever try to find things like the "declare" built-in in the `bash(1)` manpage only to incrementally slodge through 34 results using the 'n' key going 'nope, nope, nope'? ![the old way](http://i.9ol.es/animate.gif) @@ -24,7 +24,7 @@ Mansnip is a revolutionary way to navigate through manpages, a tool that no term **With mansnip** you'll just zip through documentation, saving precious time you can use to write github readmes like you're trying to sell ginsu steak knives. -Watch how mansnip shows everything with a "-z" option in the 25,888 lines of the zsh man pages. +Watch how mansnip shows everything with a "-z" option in the 25,888 lines of the [zshall manpage](http://gsp.com/cgi-bin/man.cgi?section=1&topic=zshall). ![zshall for all](http://i.9ol.es/mansnip.webp) From f59e1e69ca746e604135d961dd06e5ea117d5a3e Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 04:54:35 -0700 Subject: [PATCH 036/138] moving the background aside --- README.md | 60 +-------------------------------------------------- background.md | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 59 deletions(-) create mode 100644 background.md diff --git a/README.md b/README.md index 298b9e9..eeb10ee 100644 --- a/README.md +++ b/README.md @@ -40,62 +40,4 @@ Act now, servers are standing by. ＥＮＤ ＳＣＥＮＥ -## Why hasn't this existed forever? - -Man pages don't really encode a lot of semantic detail. The format is pretty old. There's been a number of attempted replacements, such as [GNU info](https://www.gnu.org/software/texinfo/manual/info-stnd/) and [BSD mdoc](https://mandoc.bsd.lv/) (`man 7 mandoc_mdoc`) but the ones you use on your system are probably just the traditional boring old man files. Ah, inertia. - - -Being old, it's primarily concerned with formatting and not any kind of meta-information. And boy can it format! Try using the `groffer(1)` tool and do something like `groffer git-config`. You'll hopefully get a very beautiful PDF popping up on your screen, excellent for printing out and keeping in a 3-ring binder next to your Rolodex and FAX machine. Did you know groff is in a lineage that goes back to the [1964 RUNOFF](https://en.wikipedia.org/wiki/TYPSET_and_RUNOFF) program on MIT's IBM 7094 CTSS? A mere year after Licklider's [Intergalactic Computer Network](https://en.wikipedia.org/wiki/Intergalactic_Computer_Network) memo leading to what *you* probably call "the internet" these days. - -Anyway, so inside the document source that leads to the pretty man page, there is next to no indication whether something has any special meaning. Let's go back to `bash(1)` and specifically the source lines for "declare" and "typeset": - - .TP - \fBdeclare\fP [\fB\-aAfFgilnrtux\fP] [\fB\-p\fP] [\fIname\fP[=\fIvalue\fP] ...] - .PD 0 - .TP - \fBtypeset\fP [\fB\-aAfFgilnrtux\fP] [\fB\-p\fP] [\fIname\fP[=\fIvalue\fP] ...] - .PD - -Alright, what do those things mean? You can see that in `man 7 man` or actually, - - $ mansnip 7 man .TP .PD .B .P .I - 55 DESCRIPTION - Fonts - .B Bold - - 61 .I Italics - - 97 Normal paragraphs - .P Same as .PP (begin a new paragraph). - - 124 Indented paragraph macros - .TP i Begin paragraph with hanging tag. The tag is given on the next line, but its re‐ - sults are like those of the .IP command. - - 147 Miscellaneous macros - .PD d Set inter-paragraph vertical distance to d (if omitted, d=0.4v); does not cause a - break. - - - -Hrmm, well that's a problem. It's effectively just a stylesheet. In fact, the format doesn't look fundamentally different than it did in [UNIX v0 in 1970](https://github.com/DoctorWkt/pdp7-unix/blob/master/man/stat.1). A mere 2 years after [Engelbart demoed a prototype hypertext](https://en.wikipedia.org/wiki/The_Mother_of_All_Demos) system, this is well before the [semantic web](https://en.wikipedia.org/wiki/Semantic_Web). - -"But wait," you say. "If I do `man --html bash` and then wait for the glacially slow [groff HTML post-processor](http://git.savannah.gnu.org/cgit/groff.git/tree/src/devices/grohtml) I do indeed get links!" - -Yes, the only two semantic concessions given in the modern man format is `.SH` which is for sections and `.TH` for the title and ... that's ... it. - -"Well let's fix this!" you say! - -A third time? lol, ok, have fun. Might work. For now we're stuck with these. - -"So there's no surefire easy way to find these snippets then, you can only guess?" - -Uh, yep. - - -### It's not that terrible though! - -Man pages are ridiculously consistent as far as non-semantically structured text goes. This tool usually works fine and if it doesn't, file a bug and go hunting and pecking like you usually do. I'll fix it eventually. - -Thanks. enjoy. - +[background](background.md) diff --git a/background.md b/background.md new file mode 100644 index 0000000..de14ea1 --- /dev/null +++ b/background.md @@ -0,0 +1,59 @@ +## Why hasn't this existed forever? + +Man pages don't really encode a lot of semantic detail. The format is pretty old. There's been a number of attempted replacements, such as [GNU info](https://www.gnu.org/software/texinfo/manual/info-stnd/) and [BSD mdoc](https://mandoc.bsd.lv/) (`man 7 mandoc_mdoc`) but the ones you use on your system are probably just the traditional boring old man files. Ah, inertia. + + +Being old, it's primarily concerned with formatting and not any kind of meta-information. And boy can it format! Try using the `groffer(1)` tool and do something like `groffer git-config`. You'll hopefully get a very beautiful PDF popping up on your screen, excellent for printing out and keeping in a 3-ring binder next to your Rolodex and FAX machine. Did you know groff is in a lineage that goes back to the [1964 RUNOFF](https://en.wikipedia.org/wiki/TYPSET_and_RUNOFF) program on MIT's IBM 7094 CTSS? A mere year after Licklider's [Intergalactic Computer Network](https://en.wikipedia.org/wiki/Intergalactic_Computer_Network) memo leading to what *you* probably call "the internet" these days. + +Anyway, so inside the document source that leads to the pretty man page, there is next to no indication whether something has any special meaning. Let's go back to `bash(1)` and specifically the source lines for "declare" and "typeset": + + .TP + \fBdeclare\fP [\fB\-aAfFgilnrtux\fP] [\fB\-p\fP] [\fIname\fP[=\fIvalue\fP] ...] + .PD 0 + .TP + \fBtypeset\fP [\fB\-aAfFgilnrtux\fP] [\fB\-p\fP] [\fIname\fP[=\fIvalue\fP] ...] + .PD + +Alright, what do those things mean? You can see that in `man 7 man` or actually, + + $ mansnip 7 man .TP .PD .B .P .I + 55 DESCRIPTION + Fonts + .B Bold + + 61 .I Italics + + 97 Normal paragraphs + .P Same as .PP (begin a new paragraph). + + 124 Indented paragraph macros + .TP i Begin paragraph with hanging tag. The tag is given on the next line, but its re‐ + sults are like those of the .IP command. + + 147 Miscellaneous macros + .PD d Set inter-paragraph vertical distance to d (if omitted, d=0.4v); does not cause a + break. + + + +Hrmm, well that's a problem. It's effectively just a stylesheet. In fact, the format doesn't look fundamentally different than it did in [UNIX v0 in 1970](https://github.com/DoctorWkt/pdp7-unix/blob/master/man/stat.1). A mere 2 years after [Engelbart demoed a prototype hypertext](https://en.wikipedia.org/wiki/The_Mother_of_All_Demos) system, this is well before the [semantic web](https://en.wikipedia.org/wiki/Semantic_Web). + +"But wait," you say. "If I do `man --html bash` and then wait for the glacially slow [groff HTML post-processor](http://git.savannah.gnu.org/cgit/groff.git/tree/src/devices/grohtml) I do indeed get links!" + +Yes, the only two semantic concessions given in the modern man format is `.SH` which is for sections and `.TH` for the title and ... that's ... it. + +"Well let's fix this!" you say! + +A third time? lol, ok, have fun. Might work. For now we're stuck with these. + +"So there's no surefire easy way to find these snippets then, you can only guess?" + +Uh, yep. + + +### It's not that terrible though! + +Man pages are ridiculously consistent as far as non-semantically structured text goes. This tool usually works fine and if it doesn't, file a bug and go hunting and pecking like you usually do. I'll fix it eventually. + +Thanks. enjoy. + From 730e0eac482c51db534f1815c305cfacbdd75c59 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 04:55:46 -0700 Subject: [PATCH 037/138] moving the background aside --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index eeb10ee..a1e3ff1 100644 --- a/README.md +++ b/README.md @@ -40,4 +40,6 @@ Act now, servers are standing by. ＥＮＤ ＳＣＥＮＥ -[background](background.md) +--- + +Want more? Here's some [background](background.md) From d0d62ae3a5d1b1912b6e24ae611a3241e620712b Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 04:57:45 -0700 Subject: [PATCH 038/138] more --- background.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/background.md b/background.md index de14ea1..fa665ee 100644 --- a/background.md +++ b/background.md @@ -57,3 +57,7 @@ Man pages are ridiculously consistent as far as non-semantically structured text Thanks. enjoy. +--- + +**Still** not satisfied? The [source is commented](mansnip) but seriously, that's the last stop. + From 2d0f6c2e3021e0c3ef0856c7e6c6738339535c1d Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 05:02:11 -0700 Subject: [PATCH 039/138] more --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a1e3ff1..6e4a67c 100644 --- a/README.md +++ b/README.md @@ -30,9 +30,13 @@ Watch how mansnip shows everything with a "-z" option in the 25,888 lines of the But wait, there's more! You'll also get the line number and hierarchical context totally free! -You'll even get to install it easily [through pypi](https://pypi.org/project/mansnip-kristopolous/). +*We're still not finished yet!* -Here's how do it `pip3 install mansnip-kristopolous`. +You'll also get to install it easily [through pypi](https://pypi.org/project/mansnip-kristopolous/). + +Here's how do it + +`$ pip3 install mansnip-kristopolous`. Act now, servers are standing by. From 942ebc4df5b8f3ac6089a3a921e0c54210d4860f Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 05:02:55 -0700 Subject: [PATCH 040/138] more --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6e4a67c..57410c8 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ You'll also get to install it easily [through pypi](https://pypi.org/project/man Here's how do it -`$ pip3 install mansnip-kristopolous`. +`$ pip3 install mansnip-kristopolous` Act now, servers are standing by. From fe7da07eea3a2a722ef72b56e01aebe7beefdeda Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 05:24:31 -0700 Subject: [PATCH 041/138] more --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 57410c8..7ecbd2f 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,16 @@ **Don't you hate** wasting time navigating through manpages with the leading pager's clunky search tools? -Ever try to find things like the "declare" built-in in the `bash(1)` manpage only to incrementally slodge through 34 results using the 'n' key going 'nope, nope, nope'? +Ever try to find things like the "declare" built-in in `bash(1)` only to incrementally slodge through 34 results using the 'n' key going 'nope, nope, nope'? ![the old way](http://i.9ol.es/animate.gif) Even the methods of experts struggle to fix these problems. - $ man bash - -N turn line numbering on - &declare show all the results for "declare", scan them (remember the line number for which one you want) - &[enter] turn off the filtered view - g(number) go to that line + * `man bash ` + * `-N ` turn line numbering on + * `&declare ` show all the results for "declare", scan them (remember the line number for which one you want) + * `&[enter] ` turn off the filtered view + * `g(number)` go to that line Stop wasting time with the 1980s ways of manually stumbling through manuals and say goodbye to these problems once and for all! From 22d058d851de959f849f879391b7312b1e580c11 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 05:33:24 -0700 Subject: [PATCH 042/138] more --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7ecbd2f..140ec1e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ **Don't you hate** wasting time navigating through manpages with the leading pager's clunky search tools? -Ever try to find things like the "declare" built-in in `bash(1)` only to incrementally slodge through 34 results using the 'n' key going 'nope, nope, nope'? +Ever try to find things like the "declare" built-in in `bash(1)` only to incrementally slodge through the results using the 'n' key going 'nope, nope, nope'? ![the old way](http://i.9ol.es/animate.gif) From 516e3437643e87f2e20753d52311e108cfd9e873 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 05:38:35 -0700 Subject: [PATCH 043/138] more --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 140ec1e..03d8aff 100644 --- a/README.md +++ b/README.md @@ -20,8 +20,15 @@ Stop wasting time with the 1980s ways of manually stumbling through manuals and Mansnip is a revolutionary way to navigate through manpages, a tool that no terminal should be without! +It will intelligently search through a man page and return the results we all are looking for. +Simply use it the way you use man, at the command line, followed by the search term(s) you are looking for. + +See how I can immediately find `bash(1)'s` declare using mansnip in milliseconds. + ![mansnip is amazing](http://i.9ol.es/msfade.webp) +Mansnip works on any manpage. + **With mansnip** you'll just zip through documentation, saving precious time you can use to write github readmes like you're trying to sell ginsu steak knives. Watch how mansnip shows everything with a "-z" option in the 25,888 lines of the [zshall manpage](http://gsp.com/cgi-bin/man.cgi?section=1&topic=zshall). From 33d63c594d13ac9ada682d378f8d533dbab4022f Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 05:40:26 -0700 Subject: [PATCH 044/138] more --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 03d8aff..903ee57 100644 --- a/README.md +++ b/README.md @@ -20,10 +20,10 @@ Stop wasting time with the 1980s ways of manually stumbling through manuals and Mansnip is a revolutionary way to navigate through manpages, a tool that no terminal should be without! -It will intelligently search through a man page and return the results we all are looking for. +It will intelligently search through a manpage and return the results we all are looking for. Simply use it the way you use man, at the command line, followed by the search term(s) you are looking for. -See how I can immediately find `bash(1)'s` declare using mansnip in milliseconds. +Watch how I can immediately find `bash(1)'s` declare using mansnip in milliseconds. ![mansnip is amazing](http://i.9ol.es/msfade.webp) From 4dd994225d7af441f1449133142eb000683b1c52 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 05:41:41 -0700 Subject: [PATCH 045/138] more --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 903ee57..697cd91 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ Mansnip is a revolutionary way to navigate through manpages, a tool that no term It will intelligently search through a manpage and return the results we all are looking for. Simply use it the way you use man, at the command line, followed by the search term(s) you are looking for. -Watch how I can immediately find `bash(1)'s` declare using mansnip in milliseconds. +Watch how I can immediately find `bash(1)'s` declare using mansnip without any extra effort: ![mansnip is amazing](http://i.9ol.es/msfade.webp) From f73f3903d4c289a0c2b9e3e94848de704edf1142 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 05:42:52 -0700 Subject: [PATCH 046/138] more --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 697cd91..3e18e72 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Stop wasting time with the 1980s ways of manually stumbling through manuals and Mansnip is a revolutionary way to navigate through manpages, a tool that no terminal should be without! It will intelligently search through a manpage and return the results we all are looking for. -Simply use it the way you use man, at the command line, followed by the search term(s) you are looking for. +Simply use it the way you use man, at the command line, followed by the your search term(s). Watch how I can immediately find `bash(1)'s` declare using mansnip without any extra effort: From ab320a92dac28e11730f47ef72f615e5d93f09ff Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 05:45:35 -0700 Subject: [PATCH 047/138] more --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3e18e72..f5273ca 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Stop wasting time with the 1980s ways of manually stumbling through manuals and Mansnip is a revolutionary way to navigate through manpages, a tool that no terminal should be without! -It will intelligently search through a manpage and return the results we all are looking for. +It will intelligently search through a manpage and return only the relevant results as self-contained browsable sections. Simply use it the way you use man, at the command line, followed by the your search term(s). Watch how I can immediately find `bash(1)'s` declare using mansnip without any extra effort: From f3dea69b37ff88c8e84da862499e7e3233181da9 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 05:47:57 -0700 Subject: [PATCH 048/138] more --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f5273ca..106cbf4 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Stop wasting time with the 1980s ways of manually stumbling through manuals and Mansnip is a revolutionary way to navigate through manpages, a tool that no terminal should be without! It will intelligently search through a manpage and return only the relevant results as self-contained browsable sections. -Simply use it the way you use man, at the command line, followed by the your search term(s). +Simply use it the way you use man, at the command line, followed by your search term(s). Watch how I can immediately find `bash(1)'s` declare using mansnip without any extra effort: From 41d084f86069a664d388d0da0bdc8f0aff856842 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 05:50:41 -0700 Subject: [PATCH 049/138] more --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 106cbf4..4d8fa0c 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ Mansnip works on any manpage. **With mansnip** you'll just zip through documentation, saving precious time you can use to write github readmes like you're trying to sell ginsu steak knives. -Watch how mansnip shows everything with a "-z" option in the 25,888 lines of the [zshall manpage](http://gsp.com/cgi-bin/man.cgi?section=1&topic=zshall). +Watch how mansnip shows everything with a "-z" option in the 25,888 lines of the [zshall manpage](http://gsp.com/cgi-bin/man.cgi?section=1&topic=zshall) on a single screen, all at once, in an easy-to-read manner. ![zshall for all](http://i.9ol.es/mansnip.webp) From b2b6e73274387f9e0de0aeb5e40605b9731341dc Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 13:51:08 -0700 Subject: [PATCH 050/138] smaller lead --- README.md | 10 +--------- background.md | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 4d8fa0c..ce46382 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,9 @@ **Don't you hate** wasting time navigating through manpages with the leading pager's clunky search tools? -Ever try to find things like the "declare" built-in in `bash(1)` only to incrementally slodge through the results using the 'n' key going 'nope, nope, nope'? +Ever try to find things like the "declare" built-in in `bash(1)` only to slodge through the results using the 'n' key going 'nope, nope, nope'? ![the old way](http://i.9ol.es/animate.gif) -Even the methods of experts struggle to fix these problems. - - * `man bash ` - * `-N ` turn line numbering on - * `&declare ` show all the results for "declare", scan them (remember the line number for which one you want) - * `&[enter] ` turn off the filtered view - * `g(number)` go to that line - Stop wasting time with the 1980s ways of manually stumbling through manuals and say goodbye to these problems once and for all! **Finally there's a better way!** diff --git a/background.md b/background.md index fa665ee..8c15f83 100644 --- a/background.md +++ b/background.md @@ -1,3 +1,17 @@ +## Pre-history + +Here were the two methods before: + + * "search with a leading space" and + + + * `man bash ` + * `-N ` turn line numbering on + * `&declare ` show all the results for "declare", scan them (remember the line number for which one you want) + * `&[enter] ` turn off the filtered view + * `g(number)` go to that line + + ## Why hasn't this existed forever? Man pages don't really encode a lot of semantic detail. The format is pretty old. There's been a number of attempted replacements, such as [GNU info](https://www.gnu.org/software/texinfo/manual/info-stnd/) and [BSD mdoc](https://mandoc.bsd.lv/) (`man 7 mandoc_mdoc`) but the ones you use on your system are probably just the traditional boring old man files. Ah, inertia. From 08bca27f91dd53352d8f8bbe839be15a5ca0bda5 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 13:52:15 -0700 Subject: [PATCH 051/138] phrasing --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ce46382..b9d3352 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Watch how I can immediately find `bash(1)'s` declare using mansnip without any e Mansnip works on any manpage. -**With mansnip** you'll just zip through documentation, saving precious time you can use to write github readmes like you're trying to sell ginsu steak knives. +**With mansnip** you'll just zip through documentation, saving precious time so you can write github readmes like you're trying to sell ginsu steak knives. Watch how mansnip shows everything with a "-z" option in the 25,888 lines of the [zshall manpage](http://gsp.com/cgi-bin/man.cgi?section=1&topic=zshall) on a single screen, all at once, in an easy-to-read manner. From bef0452bc5acad4a7055a41bcfaf452f3199814f Mon Sep 17 00:00:00 2001 From: chris mckenzie Date: Tue, 21 Apr 2020 15:01:32 -0700 Subject: [PATCH 052/138] Formatting --- background.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/background.md b/background.md index 8c15f83..9990c23 100644 --- a/background.md +++ b/background.md @@ -1,13 +1,14 @@ ## Pre-history -Here were the two methods before: +Here were the two methods of searching before I made this tool: - * "search with a leading space" and + * "search with a leading space" +And when using less as the pager * `man bash ` * `-N ` turn line numbering on - * `&declare ` show all the results for "declare", scan them (remember the line number for which one you want) + * `&declare ` filter and show all the results for "declare", scan them (remember the line number for which one you want) * `&[enter] ` turn off the filtered view * `g(number)` go to that line From d5957f4ab484e768dbe710095d84f04ebf82b05d Mon Sep 17 00:00:00 2001 From: chris mckenzie Date: Tue, 21 Apr 2020 15:02:29 -0700 Subject: [PATCH 053/138] Formatting --- background.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/background.md b/background.md index 9990c23..5a5da4d 100644 --- a/background.md +++ b/background.md @@ -2,7 +2,7 @@ Here were the two methods of searching before I made this tool: - * "search with a leading space" + * Search with a leading space, sometimes " -t" with two spaces, sometimes with one, etc... Very imprecise And when using less as the pager From 43ed95c69f698c5b19586262a255b8c050c51fa1 Mon Sep 17 00:00:00 2001 From: chris mckenzie Date: Tue, 21 Apr 2020 15:03:55 -0700 Subject: [PATCH 054/138] Formatting --- background.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/background.md b/background.md index 5a5da4d..aa8e098 100644 --- a/background.md +++ b/background.md @@ -13,9 +13,11 @@ And when using less as the pager * `g(number)` go to that line -## Why hasn't this existed forever? +## Why hasn't something like mansnip existed forever? -Man pages don't really encode a lot of semantic detail. The format is pretty old. There's been a number of attempted replacements, such as [GNU info](https://www.gnu.org/software/texinfo/manual/info-stnd/) and [BSD mdoc](https://mandoc.bsd.lv/) (`man 7 mandoc_mdoc`) but the ones you use on your system are probably just the traditional boring old man files. Ah, inertia. +It is pretty obvious and I was surprised myself to find nothing. + +One of the reasons is man pages don't really encode a lot of semantic detail. The format is pretty old. There's been a number of attempted replacements, such as [GNU info](https://www.gnu.org/software/texinfo/manual/info-stnd/) and [BSD mdoc](https://mandoc.bsd.lv/) (`man 7 mandoc_mdoc`) but the ones you use on your system are probably just the traditional boring old man files. Ah, inertia. Being old, it's primarily concerned with formatting and not any kind of meta-information. And boy can it format! Try using the `groffer(1)` tool and do something like `groffer git-config`. You'll hopefully get a very beautiful PDF popping up on your screen, excellent for printing out and keeping in a 3-ring binder next to your Rolodex and FAX machine. Did you know groff is in a lineage that goes back to the [1964 RUNOFF](https://en.wikipedia.org/wiki/TYPSET_and_RUNOFF) program on MIT's IBM 7094 CTSS? A mere year after Licklider's [Intergalactic Computer Network](https://en.wikipedia.org/wiki/Intergalactic_Computer_Network) memo leading to what *you* probably call "the internet" these days. From e988b6249d03c71dbe9b3ca7ee144aa0edb36445 Mon Sep 17 00:00:00 2001 From: chris mckenzie Date: Tue, 21 Apr 2020 15:09:44 -0700 Subject: [PATCH 055/138] Update background.md --- background.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/background.md b/background.md index aa8e098..8065b14 100644 --- a/background.md +++ b/background.md @@ -12,6 +12,13 @@ And when using less as the pager * `&[enter] ` turn off the filtered view * `g(number)` go to that line +I've been using these older workflows for 20+ years really without questioning it. It always felt a little primitive so I went looking for the more sophisticated way. + +I thought "surely there's like an xpath system here where I can do some * queries or display a tree hierarchy... I'm clearly just doing this in a bonehead manner" + +I looked and there wasn't. + +Shocking. So I built one. ## Why hasn't something like mansnip existed forever? From f3d3c1ac8dc65990c21f11d9f3e28a57737f4a49 Mon Sep 17 00:00:00 2001 From: chris mckenzie Date: Tue, 21 Apr 2020 15:11:32 -0700 Subject: [PATCH 056/138] Updates --- background.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/background.md b/background.md index 8065b14..5c68038 100644 --- a/background.md +++ b/background.md @@ -16,9 +16,11 @@ I've been using these older workflows for 20+ years really without questioning i I thought "surely there's like an xpath system here where I can do some * queries or display a tree hierarchy... I'm clearly just doing this in a bonehead manner" -I looked and there wasn't. +I looked for these and they didn't exist. The regex and cycle method was the best way there was. -Shocking. So I built one. +Shocking. + +So I built something better. ## Why hasn't something like mansnip existed forever? From 3ea743c83683778677ccabf17429f22369df6281 Mon Sep 17 00:00:00 2001 From: chris mckenzie Date: Tue, 21 Apr 2020 15:12:25 -0700 Subject: [PATCH 057/138] Updates --- background.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/background.md b/background.md index 5c68038..ad086ac 100644 --- a/background.md +++ b/background.md @@ -1,6 +1,6 @@ ## Pre-history -Here were the two methods of searching before I made this tool: +Here were my two methods of searching before I made this tool: * Search with a leading space, sometimes " -t" with two spaces, sometimes with one, etc... Very imprecise From 0b9b7a9938cf122ec525e884fee8b8b31545a95b Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 16:09:47 -0700 Subject: [PATCH 058/138] phrasing --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index b9d3352..c0b40d1 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Ever try to find things like the "declare" built-in in `bash(1)` only to slodge ![the old way](http://i.9ol.es/animate.gif) -Stop wasting time with the 1980s ways of manually stumbling through manuals and say goodbye to these problems once and for all! +Stop wasting time with the old way of manually stumbling through manuals. Say goodbye to these problems once and for all! **Finally there's a better way!** @@ -12,10 +12,10 @@ Stop wasting time with the 1980s ways of manually stumbling through manuals and Mansnip is a revolutionary way to navigate through manpages, a tool that no terminal should be without! -It will intelligently search through a manpage and return only the relevant results as self-contained browsable sections. +It intelligently searches through manpages and outputs the snippets relevant to your query as self-contained browsable sections. Simply use it the way you use man, at the command line, followed by your search term(s). -Watch how I can immediately find `bash(1)'s` declare using mansnip without any extra effort: +Watch how mansnip can immediately find `bash(1)'s` declare without any extra effort: ![mansnip is amazing](http://i.9ol.es/msfade.webp) From f62c10b6eb6e456192369354e69b1a10ec2a95f6 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 21 Apr 2020 16:11:27 -0700 Subject: [PATCH 059/138] phrasing --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index c0b40d1..c0033d6 100644 --- a/README.md +++ b/README.md @@ -21,9 +21,9 @@ Watch how mansnip can immediately find `bash(1)'s` declare without any extra eff Mansnip works on any manpage. -**With mansnip** you'll just zip through documentation, saving precious time so you can write github readmes like you're trying to sell ginsu steak knives. +**With mansnip** you'll just zip through documentation, saving precious time so you can write GitHub readmes like you're trying to sell Ginsu steak knives. -Watch how mansnip shows everything with a "-z" option in the 25,888 lines of the [zshall manpage](http://gsp.com/cgi-bin/man.cgi?section=1&topic=zshall) on a single screen, all at once, in an easy-to-read manner. +See how mansnip obediently shows everything with a "-z" option in the 25,888 lines of the [zshall manpage](http://gsp.com/cgi-bin/man.cgi?section=1&topic=zshall) on a single screen, all at once, in an easy-to-read manner. ![zshall for all](http://i.9ol.es/mansnip.webp) From db74f281abd4d5d5c78cf638b7462464801ed351 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Wed, 22 Apr 2020 00:28:15 -0700 Subject: [PATCH 060/138] fixes #4 --- mansnip | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/mansnip b/mansnip index f34c313..7247cf0 100755 --- a/mansnip +++ b/mansnip @@ -17,9 +17,30 @@ import sys, re, os, logging logging.basicConfig(level=(os.environ.get('LOGLEVEL') or 'critical').upper()) # This allows us to do mansnip or just -cutoff = 3 if sys.argv[1].isnumeric() else 2 -os.environ['MAN_KEEP_FORMATTING'] = '1' -cmd = '/usr/bin/man {}'.format(' '.join(sys.argv[1:cutoff])) +try: + if len(sys.argv) < 3: + raise Exception + + cutoff = 3 if sys.argv[1].isnumeric() else 2 + os.environ['MAN_KEEP_FORMATTING'] = '1' + cmd = '/usr/bin/man {}'.format(' '.join(sys.argv[1:cutoff])) +except: + from textwrap import dedent + print(dedent("""\ + ✂ mansnip usage ✂ + Mansnip works much like man does only it takes query strings after the + page you're looking to read. The syntax is generally: + + $ mansnip [ section ] page [ query0 query1 ... queryN ] + + For instance, if you want to find out what say, the xzv and f options + in tar do you can do + + $ mansnip tar -x -z -v -f + + Have fun.""")) + sys.exit(-1) + logging.info(cmd) man_input = os.popen(cmd).read() From ec651fab9dcce0ea6d869dd40baf7c85c448bbaf Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Wed, 22 Apr 2020 00:31:24 -0700 Subject: [PATCH 061/138] moving the actual execution into the exception block, silly me --- mansnip | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mansnip b/mansnip index 7247cf0..e8ab3a5 100755 --- a/mansnip +++ b/mansnip @@ -24,6 +24,9 @@ try: cutoff = 3 if sys.argv[1].isnumeric() else 2 os.environ['MAN_KEEP_FORMATTING'] = '1' cmd = '/usr/bin/man {}'.format(' '.join(sys.argv[1:cutoff])) + logging.info(cmd) + man_input = os.popen(cmd).read() + except: from textwrap import dedent print(dedent("""\ @@ -41,9 +44,6 @@ except: Have fun.""")) sys.exit(-1) -logging.info(cmd) -man_input = os.popen(cmd).read() - pack = sys.argv[cutoff:] opts = '|'.join(pack) From 171bbab6d9006321403a346ff534c51a64fe4f93 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Wed, 22 Apr 2020 02:33:43 -0700 Subject: [PATCH 062/138] trying to address cygwin --- mansnip | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/mansnip b/mansnip index e8ab3a5..bb23972 100755 --- a/mansnip +++ b/mansnip @@ -16,18 +16,22 @@ import sys, re, os, logging logging.basicConfig(level=(os.environ.get('LOGLEVEL') or 'critical').upper()) -# This allows us to do mansnip or just try: if len(sys.argv) < 3: - raise Exception + raise Exception("Not enough params") + # This allows us to do mansnip or just cutoff = 3 if sys.argv[1].isnumeric() else 2 os.environ['MAN_KEEP_FORMATTING'] = '1' - cmd = '/usr/bin/man {}'.format(' '.join(sys.argv[1:cutoff])) + + # Some people use cygwin + # You may gladly throw me in python prison for this line... + cmd = ' '.join(['/usr/bin/man' if os.path.exists('/usr/bin') else 'man'] + sys.argv[1:cutoff]) logging.info(cmd) + man_input = os.popen(cmd).read() -except: +except Exception as ex: from textwrap import dedent print(dedent("""\ ✂ mansnip usage ✂ @@ -42,6 +46,7 @@ except: $ mansnip tar -x -z -v -f Have fun.""")) + logging.info(ex) sys.exit(-1) pack = sys.argv[cutoff:] From c729cafae13c16e343e243aef1df36fe59dc4574 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Wed, 22 Apr 2020 02:35:44 -0700 Subject: [PATCH 063/138] bugfixes --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 22e0d40..e079878 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setuptools.setup( name="mansnip-kristopolous", - version="0.3.2", + version="0.3.3", scripts=["mansnip"], author="Chris McKenzie", author_email="kristopolous@yahoo.com", From 03f33af65a653c2e9e0f572f9ba4160e8ece4290 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Wed, 22 Apr 2020 05:32:35 -0700 Subject: [PATCH 064/138] adding a fun image --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index c0033d6..49d5f25 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,8 @@ Here's how do it Act now, servers are standing by. +![mansnip](http://i.9ol.es/man1.jpg) + ＦＡＤＥ ＴＯ ＢＬＡＣＫ ＥＮＤ ＳＣＥＮＥ From 7448c03c56839a9cf1c9f5775d6a8a11318ec326 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Wed, 22 Apr 2020 23:47:35 -0700 Subject: [PATCH 065/138] adding a fix to find things in mmcli --- mansnip | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mansnip b/mansnip index bb23972..62d2d13 100755 --- a/mansnip +++ b/mansnip @@ -56,9 +56,10 @@ opts = '|'.join(pack) # lsof uses +|- syntax, that's what the very fun (\+\||) is for # Then comes the term expansion and a few trailing characters. # ffmpeg uses [ at times (see hwaccel), many things use ',' to show a second option. +# mmcli uses = a lot of the times # the second part is for the long options, sometimes (git config) specified by commas # -my_re = '^\s*((\+\||)({})([\s_\[].*|, .*|)|-.*\s({}))$'.format(opts, opts) +my_re = '^\s*((\+\||)({})([\s_\[=].*|, .*|)|-.*\s({}))$'.format(opts, opts) logging.info(my_re) is_def = False From 77e38e80de6241cb21c391772b2fc697d06012b3 Mon Sep 17 00:00:00 2001 From: chris mckenzie Date: Thu, 23 Apr 2020 02:19:55 -0700 Subject: [PATCH 066/138] adding the promo video --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 49d5f25..ba901e8 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,10 @@ +

+ +[![Video](http://img.youtube.com/vi/3GT1J-ejM3Q/0.jpg)](http://www.youtube.com/watch?v=3GT1J-ejM3Q) + +> "As seen on YouTube!" +

+ **Don't you hate** wasting time navigating through manpages with the leading pager's clunky search tools? Ever try to find things like the "declare" built-in in `bash(1)` only to slodge through the results using the 'n' key going 'nope, nope, nope'? From 2128d1b586d3d79c3dbafaed79d15fd40ef8e640 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Thu, 23 Apr 2020 02:33:38 -0700 Subject: [PATCH 067/138] yt image --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ba901e8..21c74e8 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,12 @@

-[![Video](http://img.youtube.com/vi/3GT1J-ejM3Q/0.jpg)](http://www.youtube.com/watch?v=3GT1J-ejM3Q) +[![Video](http://i.9ol.es/vid.jpg)](http://www.youtube.com/watch?v=3GT1J-ejM3Q) -> "As seen on YouTube!" +> "As seen on YouTube!" (1min 45sec)

+---- + **Don't you hate** wasting time navigating through manpages with the leading pager's clunky search tools? Ever try to find things like the "declare" built-in in `bash(1)` only to slodge through the results using the 'n' key going 'nope, nope, nope'? From 8c77224446afc37ec589832c74582f262de902e5 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Thu, 23 Apr 2020 02:33:51 -0700 Subject: [PATCH 068/138] yt image --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 21c74e8..ae901ac 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![Video](http://i.9ol.es/vid.jpg)](http://www.youtube.com/watch?v=3GT1J-ejM3Q) -> "As seen on YouTube!" (1min 45sec) +> "As seen on YouTube!" (click image, it's only 1min 45sec)

---- From 7b1b798495150a1e5c7f965666c1ce41d99c055d Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Sat, 25 Apr 2020 18:33:31 -0700 Subject: [PATCH 069/138] adding more debug strings --- mansnip | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mansnip b/mansnip index bb23972..16ba5d0 100755 --- a/mansnip +++ b/mansnip @@ -156,6 +156,7 @@ for line_num, line in enumerate(man_input): res = re.match(my_re, line) or re.match(my_re, man_input_plain[line_num]) if res: + logging.info("matched: {}".format(line)) # # This is sheer frantic handwaving for things like this (From bash) # @@ -192,10 +193,12 @@ for line_num, line in enumerate(man_input): term_indent = indent elif term_indent: + # If our indent is zero and there's some text (subsection heading) if (indent == 0 and len(line)) or ( indent > 1 and (indent < term_indent or (not is_def and indent == term_indent) )): spacer = '\n' + ' ' * rs + logging.info(len(buf)) if (len(buf) == 2 and indent == term_indent and line_def) or len(buf) > 2: # # This is a lot of fancy formatting. We want the From 6e9bb66685e79b9a59c435522500b75e53a27fbb Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Sat, 25 Apr 2020 18:36:53 -0700 Subject: [PATCH 070/138] testing plan --- tests/readme.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 tests/readme.md diff --git a/tests/readme.md b/tests/readme.md new file mode 100644 index 0000000..4bc7a0b --- /dev/null +++ b/tests/readme.md @@ -0,0 +1,11 @@ +The testing system here is as follows: + +test file is: + \w+.test + +test format is + [ arguments on the first line ] + [ ... expected output ... ] + +In order to make the line numbers match we are injecting +a MANWIDTH=50 into all the executions From d68ef08f65142801e2937d5770de9a6cc0c9b803 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Sat, 25 Apr 2020 19:10:23 -0700 Subject: [PATCH 071/138] cheap tester --- tests/tester.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 tests/tester.py diff --git a/tests/tester.py b/tests/tester.py new file mode 100644 index 0000000..8fadfcb --- /dev/null +++ b/tests/tester.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +import glob +import os + +os.environ['MANWIDTH'] = '50' +for path in glob.glob("*.test"): + with open(path, 'r') as f: + expected = f.readlines() + command = expected[0] + result = expected[1:] + + cmd = "../mansnip " + command + res = os.popen(cmd).read() + if res == expected: + print(cmd + "passed") + else: + print(cmd + "failed") + + From ca6ff9bbd80d507235f41cb46a43e1e8a9f257e6 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Sat, 25 Apr 2020 19:20:03 -0700 Subject: [PATCH 072/138] more work on the tester --- tests/readme.md | 2 +- tests/tester.py | 25 +++++++++++++++++++++++-- 2 files changed, 24 insertions(+), 3 deletions(-) mode change 100644 => 100755 tests/tester.py diff --git a/tests/readme.md b/tests/readme.md index 4bc7a0b..cd407f5 100644 --- a/tests/readme.md +++ b/tests/readme.md @@ -8,4 +8,4 @@ test format is [ ... expected output ... ] In order to make the line numbers match we are injecting -a MANWIDTH=50 into all the executions +a MANWIDTH=250 into all the executions diff --git a/tests/tester.py b/tests/tester.py old mode 100644 new mode 100755 index 8fadfcb..60e45e2 --- a/tests/tester.py +++ b/tests/tester.py @@ -1,12 +1,33 @@ #!/usr/bin/env python3 import glob import os +import sys +import re +import json + +os.environ['MANWIDTH'] = '250' +if len(sys.argv) > 1: + print("making test for arguments") + args = sys.argv[1:] + fname = re.sub(' ', '_', ' '.join(args)) + ".test" + if os.path.exists(fname): + print("I'm not going to overwrite {}. Please remove it then rerun this.".format(fname)) + sys.exit(-1) + + with open(fname, 'w') as f: + print("Creating {}".format(fname)) + f.write("{}\n".format(json.dumps(args))) + command = ' '.join(args) + res = os.popen("../mansnip " + command).read() + print(res) + f.write(res) + sys.exit(0) + -os.environ['MANWIDTH'] = '50' for path in glob.glob("*.test"): with open(path, 'r') as f: expected = f.readlines() - command = expected[0] + command = ' '.join(json.loads(expected[0])) result = expected[1:] cmd = "../mansnip " + command From 0684952ab663bc96e062df8287d674f051526cc7 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Sat, 25 Apr 2020 19:51:34 -0700 Subject: [PATCH 073/138] working test system --- tests/bash_declare.test | 27 +++++++++++++++++++++ tests/tester.py | 53 ++++++++++++++++++++++++++++------------- tests/testlist.txt | 1 + 3 files changed, 65 insertions(+), 16 deletions(-) create mode 100644 tests/bash_declare.test create mode 100644 tests/testlist.txt diff --git a/tests/bash_declare.test b/tests/bash_declare.test new file mode 100644 index 0000000..2d10671 --- /dev/null +++ b/tests/bash_declare.test @@ -0,0 +1,27 @@ +2397 SSHHEELLLL BBUUIILLTTIINN CCOOMMMMAANNDDSS + ddeeccllaarree [--aaAAffFFggiillnnrrttuuxx] [--pp] [_n_a_m_e[=_v_a_l_u_e] ...] + ttyyppeesseett [--aaAAffFFggiillnnrrttuuxx] [--pp] [_n_a_m_e[=_v_a_l_u_e] ...] + Declare variables and/or give them attributes. If no _n_a_m_es are given then display the values of variables. The --pp option will display the attributes and values of each _n_a_m_e. When --pp is used with _n_a_m_e arguments, additional op‐ + tions, other than --ff and --FF, are ignored. When --pp is supplied without _n_a_m_e arguments, it will display the attributes and values of all variables having the attributes specified by the additional options. If no other options are + supplied with --pp, ddeeccllaarree will display the attributes and values of all shell variables. The --ff option will restrict the display to shell functions. The --FF option inhibits the display of function definitions; only the function + name and attributes are printed. If the eexxttddeebbuugg shell option is enabled using sshhoopptt, the source file name and line number where each _n_a_m_e is defined are displayed as well. The --FF option implies --ff. The --gg option forces vari‐ + ables to be created or modified at the global scope, even when ddeeccllaarree is executed in a shell function. It is ignored in all other cases. The following options can be used to restrict output to variables with the specified at‐ + tribute or to give variables attributes: + --aa Each _n_a_m_e is an indexed array variable (see AArrrraayyss above). + --AA Each _n_a_m_e is an associative array variable (see AArrrraayyss above). + --ff Use function names only. + --ii The variable is treated as an integer; arithmetic evaluation (see AARRIITTHHMMEETTIICC EEVVAALLUUAATTIIOONN above) is performed when the variable is assigned a value. + --ll When the variable is assigned a value, all upper-case characters are converted to lower-case. The upper-case attribute is disabled. + --nn Give each _n_a_m_e the _n_a_m_e_r_e_f attribute, making it a name reference to another variable. That other variable is defined by the value of _n_a_m_e. All references, assignments, and attribute modifications to _n_a_m_e, except those + using or changing the --nn attribute itself, are performed on the variable referenced by _n_a_m_e's value. The nameref attribute cannot be applied to array variables. + --rr Make _n_a_m_es readonly. These names cannot then be assigned values by subsequent assignment statements or unset. + --tt Give each _n_a_m_e the _t_r_a_c_e attribute. Traced functions inherit the DDEEBBUUGG and RREETTUURRNN traps from the calling shell. The trace attribute has no special meaning for variables. + --uu When the variable is assigned a value, all lower-case characters are converted to upper-case. The lower-case attribute is disabled. + --xx Mark _n_a_m_es for export to subsequent commands via the environment. + + Using `+' instead of `-' turns off the attribute instead, with the exceptions that ++aa and ++AA may not be used to destroy array variables and ++rr will not remove the readonly attribute. When used in a function, ddeeccllaarree and ttyyppeesseett + make each _n_a_m_e local, as with the llooccaall command, unless the --gg option is supplied. If a variable name is followed by =_v_a_l_u_e, the value of the variable is set to _v_a_l_u_e. When using --aa or --AA and the compound assignment syntax to + create array variables, additional attributes do not take effect until subsequent assignments. The return value is 0 unless an invalid option is encountered, an attempt is made to define a function using ``-f foo=bar'', an at‐ + tempt is made to assign a value to a readonly variable, an attempt is made to assign a value to an array variable without using the compound assignment syntax (see AArrrraayyss above), one of the _n_a_m_e_s is not a valid shell variable + name, an attempt is made to turn off readonly status for a readonly variable, an attempt is made to turn off array status for an array variable, or an attempt is made to display a non-existent function with --ff. + diff --git a/tests/tester.py b/tests/tester.py index 60e45e2..aa6cd88 100755 --- a/tests/tester.py +++ b/tests/tester.py @@ -1,40 +1,61 @@ #!/usr/bin/env python3 -import glob import os import sys import re -import json +import tempfile os.environ['MANWIDTH'] = '250' +tempdir = None +def params_to_fname(params): + return re.sub(' ', '_', params) + ".test" + +def store_results(path, expected, actual): + global tempdir + if not tempdir: + tempdir = tempfile.mkdtemp() + print("Using {} for failures".format(tempdir)) + + with open("{}/{}-expected".format(tempdir,path), 'w') as f: + f.write(expected) + + with open("{}/{}-actual".format(tempdir,path), 'w') as f: + f.write(actual) + if len(sys.argv) > 1: print("making test for arguments") args = sys.argv[1:] - fname = re.sub(' ', '_', ' '.join(args)) + ".test" + fname = params_to_fname(' '.join(args)) + if os.path.exists(fname): print("I'm not going to overwrite {}. Please remove it then rerun this.".format(fname)) sys.exit(-1) + with open('testlist.txt', 'a') as f: + f.write("{}\n".format(' '.join(args))) + with open(fname, 'w') as f: print("Creating {}".format(fname)) - f.write("{}\n".format(json.dumps(args))) command = ' '.join(args) res = os.popen("../mansnip " + command).read() - print(res) f.write(res) sys.exit(0) +with open('testlist.txt', 'r') as f: + testList = f.read().splitlines() + + for test in testList: + fname = params_to_fname(test) + + with open(fname, 'r') as f: + expected = f.read() -for path in glob.glob("*.test"): - with open(path, 'r') as f: - expected = f.readlines() - command = ' '.join(json.loads(expected[0])) - result = expected[1:] + cmd = "../mansnip " + test + actual = os.popen(cmd).read() - cmd = "../mansnip " + command - res = os.popen(cmd).read() - if res == expected: - print(cmd + "passed") - else: - print(cmd + "failed") + if actual == expected: + print(cmd + " passed") + else: + print(cmd + " failed") + store_results(path, expected, actual) diff --git a/tests/testlist.txt b/tests/testlist.txt new file mode 100644 index 0000000..a8daeb1 --- /dev/null +++ b/tests/testlist.txt @@ -0,0 +1 @@ +bash declare From eb6810a79a6b7a3d1899081c12397d25f73d8b56 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Sat, 25 Apr 2020 19:54:11 -0700 Subject: [PATCH 074/138] adding control flow for the testing --- tests/tester.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/tester.py b/tests/tester.py index aa6cd88..6ab4469 100755 --- a/tests/tester.py +++ b/tests/tester.py @@ -44,6 +44,14 @@ def store_results(path, expected, actual): testList = f.read().splitlines() for test in testList: + if test == 'stop': + print("asked to stop") + sys.exit(0) + + if test[0] == '#': + print("skipping {}".format(test)) + continue + fname = params_to_fname(test) with open(fname, 'r') as f: From e1d9e4302ebed77e71ee5c34e49f99b6cbcc0689 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Sat, 25 Apr 2020 19:54:26 -0700 Subject: [PATCH 075/138] updating the documentation --- tests/readme.md | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/tests/readme.md b/tests/readme.md index cd407f5..fa17fb0 100644 --- a/tests/readme.md +++ b/tests/readme.md @@ -3,9 +3,19 @@ The testing system here is as follows: test file is: \w+.test -test format is - [ arguments on the first line ] - [ ... expected output ... ] +To make new tests run this: -In order to make the line numbers match we are injecting -a MANWIDTH=250 into all the executions + ./tester.py (mansnip args) + +Then the args get appened to the testlist.txt + +To actually run the tests execute the following: + + ./tester.py (no args) + +And it will run through everything inside of testlist.txt + +Notes: + + You can comment out a test with # and you can have the test system stop + by inserting a "stop" after a test on its own line. From e7193b66f0770ed5b86cea0fa55912740e86e8e3 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Sat, 25 Apr 2020 20:02:19 -0700 Subject: [PATCH 076/138] better output --- tests/tester.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/tester.py b/tests/tester.py index 6ab4469..6a82fe8 100755 --- a/tests/tester.py +++ b/tests/tester.py @@ -37,6 +37,7 @@ def store_results(path, expected, actual): print("Creating {}".format(fname)) command = ' '.join(args) res = os.popen("../mansnip " + command).read() + print(res) f.write(res) sys.exit(0) @@ -61,9 +62,9 @@ def store_results(path, expected, actual): actual = os.popen(cmd).read() if actual == expected: - print(cmd + " passed") + print(test + " passed") else: - print(cmd + " failed") + print(test + " failed") store_results(path, expected, actual) From 1cb2c45c7f864b84cffcb02458868fb1fda6b7a7 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Sat, 25 Apr 2020 20:02:42 -0700 Subject: [PATCH 077/138] adding another test --- tests/testlist.txt | 1 + tests/zshall_-z.test | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 tests/zshall_-z.test diff --git a/tests/testlist.txt b/tests/testlist.txt index a8daeb1..d53d527 100644 --- a/tests/testlist.txt +++ b/tests/testlist.txt @@ -1 +1,2 @@ bash declare +zshall -z diff --git a/tests/zshall_-z.test b/tests/zshall_-z.test new file mode 100644 index 0000000..30efde3 --- /dev/null +++ b/tests/zshall_-z.test @@ -0,0 +1,32 @@ +1371 CCOONNDDIITTIIOONNAALL EEXXPPRREESSSSIIOONNSS + --zz _s_t_r_i_n_g + true if length of _s_t_r_i_n_g is zero. + +5777 SSHHEELLLL BBUUIILLTTIINN CCOOMMMMAANNDDSS + pprriinntt [ --aabbccDDiillmmnnNNooOOppPPrrssSSzz ] [ --uu _n ] [ --ff _f_o_r_m_a_t ] [ --CC _c_o_l_s ] + [ --vv _n_a_m_e ] [ --xxXX _t_a_b_s_t_o_p ] [ --RR [ --eenn ]] [ _a_r_g ... ] + --zz Push the arguments onto the editing buffer stack, separated by spaces. + + If any of `--mm', `--oo' or `--OO' are used in combination with `--ff' and there are no arguments (after the removal process in the case of `--mm') then nothing is printed. + +5839 rreeaadd [ --rrsszzppqqAAccllnneeEE ] [ --tt [ _n_u_m ] ] [ --kk [ _n_u_m ] ] [ --dd _d_e_l_i_m ] + [ --uu _n ] [ _n_a_m_e[??_p_r_o_m_p_t] ] [ _n_a_m_e ... ] + --zz Read one entry from the editor buffer stack and assign it to the first _n_a_m_e, without word splitting. Text is pushed onto the stack with `pprriinntt --zz' or with ppuusshh--lliinnee from the line editor (see _z_s_h_z_l_e(1)). This flag is ig‐ + nored when the --kk or --qq flags are present. + +6382 zzccoommppiillee [ --UU ] [ --zz | --kk ] [ --RR | --MM ] _f_i_l_e [ _n_a_m_e ... ] + zzccoommppiillee --ccaa [ --mm ] [ --RR | --MM ] _f_i_l_e [ _n_a_m_e ... ] + zzccoommppiillee --tt _f_i_l_e [ _n_a_m_e ... ] + --zz These options are used when the compiled file contains functions which are to be autoloaded. If --zz is given, the function will be autoloaded as if the KKSSHH__AAUUTTOOLLOOAADD option is _n_o_t set, even if it is set at the time the com‐ + piled file is read, while if the --kk is given, the function will be loaded as if KKSSHH__AAUUTTOOLLOOAADD _i_s set. These options also take precedence over any --kk or --zz options specified to the aauuttoollooaadd builtin. If neither of these op‐ + tions is given, the function will be loaded as determined by the setting of the KKSSHH__AAUUTTOOLLOOAADD option at the time the compiled file is read. + + These options may also appear as many times as necessary between the listed _n_a_m_es to specify the loading style of all following functions, up to the next --kk or --zz. + + The created file always contains two versions of the compiled format, one for big-endian machines and one for small-endian machines. The upshot of this is that the compiled file is machine independent and if it is read or + mapped, only one half of the file is actually used (and mapped). + +11596 OOPPTTIIOONN FFLLAAGGSS + SSiimmppllee FFllaaggss + --zz Names of suspended jobs. + From cf1ed0f363f5e5e1aeb619b5bfba38d409ef0d7e Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Sat, 25 Apr 2020 20:17:33 -0700 Subject: [PATCH 078/138] Adding another test from the readmes --- tests/7_man_.TP_.PD_.B_.P_.I.test | 11 +++++++++++ tests/testlist.txt | 1 + 2 files changed, 12 insertions(+) create mode 100644 tests/7_man_.TP_.PD_.B_.P_.I.test diff --git a/tests/7_man_.TP_.PD_.B_.P_.I.test b/tests/7_man_.TP_.PD_.B_.P_.I.test new file mode 100644 index 0000000..a3f3534 --- /dev/null +++ b/tests/7_man_.TP_.PD_.B_.P_.I.test @@ -0,0 +1,11 @@ +46 FFoonnttss + ..BB Bold + +52 ..II Italics + +79 NNoorrmmaall ppaarraaggrraapphhss + ..PP Same as ..PPPP (begin a new paragraph). + +112 MMiisscceellllaanneeoouuss mmaaccrrooss + ..PPDD _d Set inter-paragraph vertical distance to d (if omitted, d=0.4v); does not cause a break. + diff --git a/tests/testlist.txt b/tests/testlist.txt index d53d527..11b5a45 100644 --- a/tests/testlist.txt +++ b/tests/testlist.txt @@ -1,2 +1,3 @@ bash declare zshall -z +7 man .TP .PD .B .P .I From 65deb1c5acbdfd2384257bb1dc638885c58834b9 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Sat, 25 Apr 2020 20:24:35 -0700 Subject: [PATCH 079/138] adding the test case for -E in lsof --- tests/lsof_-E.test | 17 +++++++++++++++++ tests/testlist.txt | 1 + 2 files changed, 18 insertions(+) create mode 100644 tests/lsof_-E.test diff --git a/tests/lsof_-E.test b/tests/lsof_-E.test new file mode 100644 index 0000000..f1ed6e1 --- /dev/null +++ b/tests/lsof_-E.test @@ -0,0 +1,17 @@ +174 ++||--EE ++EE specifies that Linux pipe, Linux UNIX socket and Linux pseudoterminal files should be displayed with endpoint information and the files of the endpoints should also be displayed. Note: UNIX socket file endpoint information + is only available when the compile flags line of --vv output contains HASUXSOCKEPT, and psudoterminal endpoint information is only available when the compile flags line contains HASPTYEPT. + + Pipe endpoint information is displayed in the NAME column in the form ``_P_I_D_,_c_m_d_,_F_D_m_o_d_e'', where _P_I_D is the endpoint process ID; _c_m_d is the endpoint process command; _F_D is the endpoint file's descriptor; and _m_o_d_e is the endpoint + file's access mode. + + Pseudoterminal endpoint information is displayed in the NAME column as ``->/dev/pts_m_i_n _P_I_D_,_c_m_d_,_F_D_m_o_d_e'' or ``_P_I_D_,_c_m_d_,_F_D_m_o_d_e''. The first form is for a master device; the second, for a slave device. _m_i_n is a slave device's mi‐ + nor device number; and _P_I_D_, _c_m_d_, _F_D and _m_o_d_e are the same as with pipe endpoint information. Note: psudoterminal endpoint information is only available when the compile flags line of --VV output contains HASPTYEPT. + + UNIX socket file endpoint information is displayed in the NAME column in the form + ``type=_T_Y_P_E ->INO=_I_N_O_D_E _P_I_D_,_c_m_d_,_F_D_m_o_d_e'', where _T_Y_P_E is the socket type; _I_N_O_D_E is the i-node number of the connected socket; and _P_I_D_, _c_m_d_, _F_D and _m_o_d_e are the same as with pipe endpoint information. Note: UNIX socket file end‐ + point information is available only when the compile flags line of --vv output contains HASUXSOCKEPT. + + Multiple occurrences of this information can appear in a file's NAME column. + + --EE specfies that Linux pipe and Linux UNIX socket files should be displayed with endpoint information, but not the files of the endpoints. + diff --git a/tests/testlist.txt b/tests/testlist.txt index 11b5a45..48924dd 100644 --- a/tests/testlist.txt +++ b/tests/testlist.txt @@ -1,3 +1,4 @@ bash declare zshall -z 7 man .TP .PD .B .P .I +lsof -E From 932c1a85702c1c8ee6c2510fafcf2489757a4111 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Sat, 25 Apr 2020 20:28:35 -0700 Subject: [PATCH 080/138] testing the ffmpeg style docs --- tests/ffmpeg_-c.test | 18 ++++++++++++++++++ tests/testlist.txt | 1 + 2 files changed, 19 insertions(+) create mode 100644 tests/ffmpeg_-c.test diff --git a/tests/ffmpeg_-c.test b/tests/ffmpeg_-c.test new file mode 100644 index 0000000..5963d64 --- /dev/null +++ b/tests/ffmpeg_-c.test @@ -0,0 +1,18 @@ +435 MMaaiinn ooppttiioonnss + --cc[[::_s_t_r_e_a_m___s_p_e_c_i_f_i_e_r]] _c_o_d_e_c ((_i_n_p_u_t_/_o_u_t_p_u_t_,_p_e_r_-_s_t_r_e_a_m)) + --ccooddeecc[[::_s_t_r_e_a_m___s_p_e_c_i_f_i_e_r]] _c_o_d_e_c ((_i_n_p_u_t_/_o_u_t_p_u_t_,_p_e_r_-_s_t_r_e_a_m)) + Select an encoder (when used before an output file) or a decoder (when used before an input file) for one or more streams. _c_o_d_e_c is the name of a decoder/encoder or a special value "copy" (output only) to indicate that the stream is + not to be re-encoded. + + For example + + ffmpeg -i INPUT -map 0 -c:v libx264 -c:a copy OUTPUT + + encodes all video streams with libx264 and copies all audio streams. + + For each stream, the last matching "c" option is applied, so + + ffmpeg -i INPUT -map 0 -c copy -c:v:1 libx264 -c:a:137 libvorbis OUTPUT + + will copy all the streams except the second video, which will be encoded with libx264, and the 138th audio, which will be encoded with libvorbis. + diff --git a/tests/testlist.txt b/tests/testlist.txt index 48924dd..f97a6bb 100644 --- a/tests/testlist.txt +++ b/tests/testlist.txt @@ -2,3 +2,4 @@ bash declare zshall -z 7 man .TP .PD .B .P .I lsof -E +ffmpeg -c From 5f449b71fd3f41d20ee2ae793e95d2602ce1e597 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Sat, 25 Apr 2020 20:41:11 -0700 Subject: [PATCH 081/138] commenting the regex --- mansnip | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/mansnip b/mansnip index 3f7c644..c461d6b 100755 --- a/mansnip +++ b/mansnip @@ -52,14 +52,24 @@ except Exception as ex: pack = sys.argv[cutoff:] opts = '|'.join(pack) -# -# lsof uses +|- syntax, that's what the very fun (\+\||) is for -# Then comes the term expansion and a few trailing characters. -# ffmpeg uses [ at times (see hwaccel), many things use ',' to show a second option. -# mmcli uses = a lot of the times -# the second part is for the long options, sometimes (git config) specified by commas -# -my_re = '^\s*((\+\||)({})([\s_\[=].*|, .*|)|-.*\s({}))$'.format(opts, opts) +my_re = ''.join([ + '^\s*', + '(', + # lsof uses +|- syntax, + '(\+\||)', + + # the term itself + '({})', + + # ffmpeg uses [ at times (see hwaccel), many things use ',' to show a second option. + # mmcli uses = a lot of the times + '([\s_\[=].*|, .*|)', + '|', + # this is for the long options, sometimes (git config) specified by commas + '-.*\s({})' + ')$' + ]).format(opts, opts) + logging.info(my_re) is_def = False From 3fec1a1b66817783c818a9b5c8108046395a718c Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Sat, 25 Apr 2020 21:00:27 -0700 Subject: [PATCH 082/138] matching long-args with args --- mansnip | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mansnip b/mansnip index c461d6b..2704bb3 100755 --- a/mansnip +++ b/mansnip @@ -66,7 +66,8 @@ my_re = ''.join([ '([\s_\[=].*|, .*|)', '|', # this is for the long options, sometimes (git config) specified by commas - '-.*\s({})' + '-.*\s({})', + '([\s_\[=].*|)', ')$' ]).format(opts, opts) From 884fa4fc9755fbadf8757498fd79b0964f7fb285 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Sat, 25 Apr 2020 21:00:46 -0700 Subject: [PATCH 083/138] adding new tests --- tests/ls_--ignore.test | 3 +++ tests/testlist.txt | 1 + 2 files changed, 4 insertions(+) create mode 100644 tests/ls_--ignore.test diff --git a/tests/ls_--ignore.test b/tests/ls_--ignore.test new file mode 100644 index 0000000..332ae50 --- /dev/null +++ b/tests/ls_--ignore.test @@ -0,0 +1,3 @@ +93 --II, ----iiggnnoorree=_P_A_T_T_E_R_N + do not list implied entries matching shell PATTERN + diff --git a/tests/testlist.txt b/tests/testlist.txt index f97a6bb..ec4ae46 100644 --- a/tests/testlist.txt +++ b/tests/testlist.txt @@ -3,3 +3,4 @@ zshall -z 7 man .TP .PD .B .P .I lsof -E ffmpeg -c +ls --ignore From 722f8aa58f24a3420da1252276c5ba14ccf784c1 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Sat, 25 Apr 2020 21:01:39 -0700 Subject: [PATCH 084/138] cleaner output --- tests/tester.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/tester.py b/tests/tester.py index 6a82fe8..3c2630d 100755 --- a/tests/tester.py +++ b/tests/tester.py @@ -62,9 +62,9 @@ def store_results(path, expected, actual): actual = os.popen(cmd).read() if actual == expected: - print(test + " passed") + print("PASSED " + test) else: - print(test + " failed") + print("!! FAILED " + test) store_results(path, expected, actual) From b08fa1ac5ea38210e496cc87394c0a56731b306a Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Sat, 25 Apr 2020 21:15:46 -0700 Subject: [PATCH 085/138] testing for adjacent terms --- tests/ls_--inode_--ignore.test | 6 ++++++ tests/testlist.txt | 1 + 2 files changed, 7 insertions(+) create mode 100644 tests/ls_--inode_--ignore.test diff --git a/tests/ls_--inode_--ignore.test b/tests/ls_--inode_--ignore.test new file mode 100644 index 0000000..39f401f --- /dev/null +++ b/tests/ls_--inode_--ignore.test @@ -0,0 +1,6 @@ +90 --ii, ----iinnooddee + print the index number of each file + +93 --II, ----iiggnnoorree=_P_A_T_T_E_R_N + do not list implied entries matching shell PATTERN + diff --git a/tests/testlist.txt b/tests/testlist.txt index ec4ae46..f247144 100644 --- a/tests/testlist.txt +++ b/tests/testlist.txt @@ -4,3 +4,4 @@ zshall -z lsof -E ffmpeg -c ls --ignore +ls --inode --ignore From 0315c5d8063839ffe743aeba02f88282e9bbf310 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Sat, 25 Apr 2020 21:16:12 -0700 Subject: [PATCH 086/138] changing the processing flow to accomodate for adjacent matches --- mansnip | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/mansnip b/mansnip index 2704bb3..897acdb 100755 --- a/mansnip +++ b/mansnip @@ -67,9 +67,11 @@ my_re = ''.join([ '|', # this is for the long options, sometimes (git config) specified by commas '-.*\s({})', + + # and kinda our same capture from above '([\s_\[=].*|)', ')$' - ]).format(opts, opts) + ]).format(opts, opts) logging.info(my_re) @@ -106,9 +108,13 @@ man_input = man_input.split('\n') # rs = 5 if len(man_input) < 1e4 else 7 -for line_num, line in enumerate(man_input): - line = line.strip('\n') +line_num = -1 +while line_num + 1 < len(man_input): + line_num += 1 + + line = man_input[line_num].strip('\n') + logging.info(line_num) # # Establish the "indent", this is crucial to how essentially # everything works. @@ -254,6 +260,9 @@ for line_num, line in enumerate(man_input): print(("{:<%d}{}{}\n" % rs).format(buf_start, stack_print, '\n'.join(buf))) + # if this was the line we decided to reset everything on then we should actually process + # it again because it could ALSO be the first line of a new match. + line_num -= 1 buf = [] term_indent = False is_def = False From 7525c127a84c91efbeb9b674fe81d1d0dec1da54 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Sat, 25 Apr 2020 21:18:01 -0700 Subject: [PATCH 087/138] markdown updates --- tests/readme.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tests/readme.md b/tests/readme.md index fa17fb0..8761381 100644 --- a/tests/readme.md +++ b/tests/readme.md @@ -1,21 +1,19 @@ The testing system here is as follows: -test file is: - \w+.test - To make new tests run this: - ./tester.py (mansnip args) + $ ./tester.py (mansnip args) +The test file is automatically created Then the args get appened to the testlist.txt To actually run the tests execute the following: - ./tester.py (no args) + $ ./tester.py (no args) And it will run through everything inside of testlist.txt Notes: - You can comment out a test with # and you can have the test system stop - by inserting a "stop" after a test on its own line. + * You can comment out a test with # and you can have the test system stop + * by inserting a "stop" after a test on its own line. From ccd02ebca2b8f02a7342d0c426b702126d4ae652 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Sat, 25 Apr 2020 21:27:15 -0700 Subject: [PATCH 088/138] fixing the storing of broken tests --- tests/tester.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tester.py b/tests/tester.py index 3c2630d..bfde91f 100755 --- a/tests/tester.py +++ b/tests/tester.py @@ -65,6 +65,6 @@ def store_results(path, expected, actual): print("PASSED " + test) else: print("!! FAILED " + test) - store_results(path, expected, actual) + store_results(fname, expected, actual) From f660143b01839a18db3528a88272fa8dd062819d Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Sat, 25 Apr 2020 21:27:31 -0700 Subject: [PATCH 089/138] Fixing the erroneous output for one of the tests --- tests/7_man_.TP_.PD_.B_.P_.I.test | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/7_man_.TP_.PD_.B_.P_.I.test b/tests/7_man_.TP_.PD_.B_.P_.I.test index a3f3534..37c0a86 100644 --- a/tests/7_man_.TP_.PD_.B_.P_.I.test +++ b/tests/7_man_.TP_.PD_.B_.P_.I.test @@ -6,6 +6,9 @@ 79 NNoorrmmaall ppaarraaggrraapphhss ..PP Same as ..PPPP (begin a new paragraph). +96 IInnddeenntteedd ppaarraaggrraapphh mmaaccrrooss + ..TTPP _i Begin paragraph with hanging tag. The tag is given on the next line, but its results are like those of the ..IIPP command. + 112 MMiisscceellllaanneeoouuss mmaaccrrooss ..PPDD _d Set inter-paragraph vertical distance to d (if omitted, d=0.4v); does not cause a break. From 22799071a3ce2c69043ed6b436cf7648798c61b2 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Sat, 25 Apr 2020 21:34:21 -0700 Subject: [PATCH 090/138] moving to a json format for arg parsing --- tests/tester.py | 7 +++++-- tests/testlist.txt | 15 ++++++++------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/tests/tester.py b/tests/tester.py index bfde91f..e6d881e 100755 --- a/tests/tester.py +++ b/tests/tester.py @@ -3,6 +3,7 @@ import sys import re import tempfile +import json os.environ['MANWIDTH'] = '250' tempdir = None @@ -31,7 +32,7 @@ def store_results(path, expected, actual): sys.exit(-1) with open('testlist.txt', 'a') as f: - f.write("{}\n".format(' '.join(args))) + f.write(json.dumps(args) + "\n") with open(fname, 'w') as f: print("Creating {}".format(fname)) @@ -44,7 +45,9 @@ def store_results(path, expected, actual): with open('testlist.txt', 'r') as f: testList = f.read().splitlines() - for test in testList: + for testraw in testList: + test = ' '.join(json.loads(testraw)) + if test == 'stop': print("asked to stop") sys.exit(0) diff --git a/tests/testlist.txt b/tests/testlist.txt index f247144..c8ac95d 100644 --- a/tests/testlist.txt +++ b/tests/testlist.txt @@ -1,7 +1,8 @@ -bash declare -zshall -z -7 man .TP .PD .B .P .I -lsof -E -ffmpeg -c -ls --ignore -ls --inode --ignore +["bash", "declare"] +["zshall", "-z"] +["7", "man", ".TP", ".PD", ".B", ".P", ".I"] +["lsof", "-E"] +["ffmpeg", "-c"] +["ls", "--ignore"] +["ls", "--inode", "--ignore"] +["bash", "-z string"] From 903317f31bb4ace8d0f238b4a457e39f8fb3c5ee Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Sat, 25 Apr 2020 21:34:47 -0700 Subject: [PATCH 091/138] fixing #6 bash "-z string" --- mansnip | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mansnip b/mansnip index 897acdb..36ac7da 100755 --- a/mansnip +++ b/mansnip @@ -114,7 +114,6 @@ while line_num + 1 < len(man_input): line = man_input[line_num].strip('\n') - logging.info(line_num) # # Establish the "indent", this is crucial to how essentially # everything works. @@ -217,7 +216,7 @@ while line_num + 1 < len(man_input): )): spacer = '\n' + ' ' * rs logging.info(len(buf)) - if (len(buf) == 2 and indent == term_indent and line_def) or len(buf) > 2: + if (len(buf) == 2 and indent == term_indent and line_def) or len(buf) > 1: # # This is a lot of fancy formatting. We want the # vertical whitespace between the heading and snippet to be consistent. From 40dd89f4001dfd1650cf9208f6df597acf106b58 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Sat, 25 Apr 2020 21:42:17 -0700 Subject: [PATCH 092/138] adding tests for #6 "bash '-z string'" --- tests/bash_-z_string.test | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 tests/bash_-z_string.test diff --git a/tests/bash_-z_string.test b/tests/bash_-z_string.test new file mode 100644 index 0000000..4fb8aea --- /dev/null +++ b/tests/bash_-z_string.test @@ -0,0 +1,4 @@ +1311 CCOONNDDIITTIIOONNAALL EEXXPPRREESSSSIIOONNSS + --zz _s_t_r_i_n_g + True if the length of _s_t_r_i_n_g is zero. + From e549dec8af22c7cfef90557f109164d9fb2ac572 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Sat, 25 Apr 2020 21:42:46 -0700 Subject: [PATCH 093/138] moving over the subprocess to preserve arglist --- tests/readme.md | 2 +- tests/tester.py | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/tests/readme.md b/tests/readme.md index 8761381..79537b2 100644 --- a/tests/readme.md +++ b/tests/readme.md @@ -5,7 +5,7 @@ To make new tests run this: $ ./tester.py (mansnip args) The test file is automatically created -Then the args get appened to the testlist.txt +Then the args get appened to the testlist.txt. They are stored in JSON to preserve the arglist To actually run the tests execute the following: diff --git a/tests/tester.py b/tests/tester.py index e6d881e..c36e132 100755 --- a/tests/tester.py +++ b/tests/tester.py @@ -3,6 +3,7 @@ import sys import re import tempfile +import subprocess import json os.environ['MANWIDTH'] = '250' @@ -36,8 +37,10 @@ def store_results(path, expected, actual): with open(fname, 'w') as f: print("Creating {}".format(fname)) - command = ' '.join(args) - res = os.popen("../mansnip " + command).read() + + p = subprocess.Popen(['../mansnip'] + args, stdout=subprocess.PIPE) + res = p.communicate()[0].decode("utf-8") + print(res) f.write(res) sys.exit(0) @@ -46,7 +49,8 @@ def store_results(path, expected, actual): testList = f.read().splitlines() for testraw in testList: - test = ' '.join(json.loads(testraw)) + testList = json.loads(testraw) + test = ' '.join(testList) if test == 'stop': print("asked to stop") @@ -61,8 +65,8 @@ def store_results(path, expected, actual): with open(fname, 'r') as f: expected = f.read() - cmd = "../mansnip " + test - actual = os.popen(cmd).read() + p = subprocess.Popen(['../mansnip'] + testList, stdout=subprocess.PIPE) + actual = p.communicate()[0].decode("utf-8") if actual == expected: print("PASSED " + test) From 8e736e27da47896aa381957ea55bcf63dfa02bd3 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Sat, 25 Apr 2020 22:14:06 -0700 Subject: [PATCH 094/138] version bump for pypi --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e079878..1fbcfa8 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setuptools.setup( name="mansnip-kristopolous", - version="0.3.3", + version="0.3.4", scripts=["mansnip"], author="Chris McKenzie", author_email="kristopolous@yahoo.com", From a5f63bc9ec1898a97f506350757b849bd0b20614 Mon Sep 17 00:00:00 2001 From: chris mckenzie Date: Sun, 26 Apr 2020 00:20:35 -0700 Subject: [PATCH 095/138] Formatting --- tests/readme.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/readme.md b/tests/readme.md index 79537b2..ee7f954 100644 --- a/tests/readme.md +++ b/tests/readme.md @@ -4,8 +4,9 @@ To make new tests run this: $ ./tester.py (mansnip args) -The test file is automatically created -Then the args get appened to the testlist.txt. They are stored in JSON to preserve the arglist +The test file is automatically created. + +Then the args get appened to the `testlist.txt`. They are stored in JSON to preserve the arglist To actually run the tests execute the following: @@ -15,5 +16,5 @@ And it will run through everything inside of testlist.txt Notes: - * You can comment out a test with # and you can have the test system stop - * by inserting a "stop" after a test on its own line. + * You can comment out a test with # + * you can have the test system stop by inserting a "stop" after a test on its own line. From a82622f88bd04117a1c6fd5a877479d2973116a8 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Sun, 26 Apr 2020 17:14:31 -0700 Subject: [PATCH 096/138] Fixing the subheading issue --- mansnip | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mansnip b/mansnip index 36ac7da..712dd56 100755 --- a/mansnip +++ b/mansnip @@ -78,7 +78,7 @@ logging.info(my_re) is_def = False line_def = False stack_start = False -term_indent = False +term_indent = None buf_start = False buf = [] @@ -122,7 +122,6 @@ while line_num + 1 < len(man_input): indent_window = indent_window[-2:] + [indent] if len(line): - logging.debug(line) # Our nice stack guessing system essentially keeps a stack # of the indents and then based on the current indent either @@ -209,7 +208,8 @@ while line_num + 1 < len(man_input): stack_start = stack_guess[:] term_indent = indent - elif term_indent: + elif term_indent != None: + logging.info(line) # If our indent is zero and there's some text (subsection heading) if (indent == 0 and len(line)) or ( indent > 1 and (indent < term_indent or (not is_def and indent == term_indent) @@ -263,7 +263,7 @@ while line_num + 1 < len(man_input): # it again because it could ALSO be the first line of a new match. line_num -= 1 buf = [] - term_indent = False + term_indent = None is_def = False else: # Once our formatting goes in we have to reset it From 3eb594d93a2e1ea34e083d2dab69caca6438d6e9 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Sun, 26 Apr 2020 17:14:51 -0700 Subject: [PATCH 097/138] adding a test for subheadings --- tests/bash_CONDITIONAL.test | 93 +++++++++++++++++++++++++++++++++++++ tests/testlist.txt | 1 + 2 files changed, 94 insertions(+) create mode 100644 tests/bash_CONDITIONAL.test diff --git a/tests/bash_CONDITIONAL.test b/tests/bash_CONDITIONAL.test new file mode 100644 index 0000000..76ee234 --- /dev/null +++ b/tests/bash_CONDITIONAL.test @@ -0,0 +1,93 @@ +1246 CCOONNDDIITTIIOONNAALL EEXXPPRREESSSSIIOONNSS + Conditional expressions are used by the [[[[ compound command and the tteesstt and [[ builtin commands to test file attributes and perform string and arithmetic comparisons. The tteesstt abd [[ commands determine their behavior based on the number + of arguments; see the descriptions of those commands for any other command-specific actions. + + Expressions are formed from the following unary or binary primaries. BBaasshh handles several filenames specially when they are used in expressions. If the operating system on which bbaasshh is running provides these special files, bash will + use them; otherwise it will emulate them internally with this behavior: If any _f_i_l_e argument to one of the primaries is of the form _/_d_e_v_/_f_d_/_n, then file descriptor _n is checked. If the _f_i_l_e argument to one of the primaries is one of + _/_d_e_v_/_s_t_d_i_n, _/_d_e_v_/_s_t_d_o_u_t, or _/_d_e_v_/_s_t_d_e_r_r, file descriptor 0, 1, or 2, respectively, is checked. + + Unless otherwise specified, primaries that operate on files follow symbolic links and operate on the target of the link, rather than the link itself. + + When used with [[[[, the << and >> operators sort lexicographically using the current locale. The tteesstt command sorts using ASCII ordering. + + --aa _f_i_l_e + True if _f_i_l_e exists. + --bb _f_i_l_e + True if _f_i_l_e exists and is a block special file. + --cc _f_i_l_e + True if _f_i_l_e exists and is a character special file. + --dd _f_i_l_e + True if _f_i_l_e exists and is a directory. + --ee _f_i_l_e + True if _f_i_l_e exists. + --ff _f_i_l_e + True if _f_i_l_e exists and is a regular file. + --gg _f_i_l_e + True if _f_i_l_e exists and is set-group-id. + --hh _f_i_l_e + True if _f_i_l_e exists and is a symbolic link. + --kk _f_i_l_e + True if _f_i_l_e exists and its ``sticky'' bit is set. + --pp _f_i_l_e + True if _f_i_l_e exists and is a named pipe (FIFO). + --rr _f_i_l_e + True if _f_i_l_e exists and is readable. + --ss _f_i_l_e + True if _f_i_l_e exists and has a size greater than zero. + --tt _f_d True if file descriptor _f_d is open and refers to a terminal. + --uu _f_i_l_e + True if _f_i_l_e exists and its set-user-id bit is set. + --ww _f_i_l_e + True if _f_i_l_e exists and is writable. + --xx _f_i_l_e + True if _f_i_l_e exists and is executable. + --GG _f_i_l_e + True if _f_i_l_e exists and is owned by the effective group id. + --LL _f_i_l_e + True if _f_i_l_e exists and is a symbolic link. + --NN _f_i_l_e + True if _f_i_l_e exists and has been modified since it was last read. + --OO _f_i_l_e + True if _f_i_l_e exists and is owned by the effective user id. + --SS _f_i_l_e + True if _f_i_l_e exists and is a socket. + _f_i_l_e_1 --eeff _f_i_l_e_2 + True if _f_i_l_e_1 and _f_i_l_e_2 refer to the same device and inode numbers. + _f_i_l_e_1 -nntt _f_i_l_e_2 + True if _f_i_l_e_1 is newer (according to modification date) than _f_i_l_e_2, or if _f_i_l_e_1 exists and _f_i_l_e_2 does not. + _f_i_l_e_1 -oott _f_i_l_e_2 + True if _f_i_l_e_1 is older than _f_i_l_e_2, or if _f_i_l_e_2 exists and _f_i_l_e_1 does not. + --oo _o_p_t_n_a_m_e + True if the shell option _o_p_t_n_a_m_e is enabled. See the list of options under the description of the --oo option to the sseett builtin below. + --vv _v_a_r_n_a_m_e + True if the shell variable _v_a_r_n_a_m_e is set (has been assigned a value). + --RR _v_a_r_n_a_m_e + True if the shell variable _v_a_r_n_a_m_e is set and is a name reference. + --zz _s_t_r_i_n_g + True if the length of _s_t_r_i_n_g is zero. + _s_t_r_i_n_g + --nn _s_t_r_i_n_g + True if the length of _s_t_r_i_n_g is non-zero. + + _s_t_r_i_n_g_1 ==== _s_t_r_i_n_g_2 + _s_t_r_i_n_g_1 == _s_t_r_i_n_g_2 + True if the strings are equal. == should be used with the tteesstt command for POSIX conformance. When used with the [[[[ command, this performs pattern matching as described above (CCoommppoouunndd CCoommmmaannddss). + + _s_t_r_i_n_g_1 !!== _s_t_r_i_n_g_2 + True if the strings are not equal. + + _s_t_r_i_n_g_1 << _s_t_r_i_n_g_2 + True if _s_t_r_i_n_g_1 sorts before _s_t_r_i_n_g_2 lexicographically. + + _s_t_r_i_n_g_1 >> _s_t_r_i_n_g_2 + True if _s_t_r_i_n_g_1 sorts after _s_t_r_i_n_g_2 lexicographically. + + _a_r_g_1 OOPP _a_r_g_2 + OOPP is one of --eeqq, --nnee, --lltt, --llee, --ggtt, or --ggee. These arithmetic binary operators return true if _a_r_g_1 is equal to, not equal to, less than, less than or equal to, greater than, or greater than or equal to _a_r_g_2, respectively. _A_r_g_1 + and _a_r_g_2 may be positive or negative integers. When used with the [[[[ command, _A_r_g_1 and _A_r_g_2 are evaluated as arithmetic expressions (see AARRIITTHHMMEETTIICC EEVVAALLUUAATTIIOONN above). + +2950 SSHHEELLLL BBUUIILLTTIINN CCOOMMMMAANNDDSS + tteesstt _e_x_p_r + [[ _e_x_p_r ]] + CCOONNDDIITTIIOONNAALL EEXXPPRREESSSSIIOONNSS. tteesstt does not accept any options, nor does it accept and ignore an argument of ---- as signifying the end of options. + diff --git a/tests/testlist.txt b/tests/testlist.txt index c8ac95d..896bbfc 100644 --- a/tests/testlist.txt +++ b/tests/testlist.txt @@ -6,3 +6,4 @@ ["ls", "--ignore"] ["ls", "--inode", "--ignore"] ["bash", "-z string"] +["bash", "CONDITIONAL"] From ccc2a60244e735133d9a4d779784b82689a0fb04 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Sun, 26 Apr 2020 17:42:43 -0700 Subject: [PATCH 098/138] bumping the pypi version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 1fbcfa8..2872b48 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setuptools.setup( name="mansnip-kristopolous", - version="0.3.4", + version="0.3.5", scripts=["mansnip"], author="Chris McKenzie", author_email="kristopolous@yahoo.com", From be9a207b42991ecf648aac5633ed9ffea4deb09a Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Sun, 26 Apr 2020 23:21:34 -0700 Subject: [PATCH 099/138] background info --- background.md | 3 ++- mansnip | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/background.md b/background.md index ad086ac..3664582 100644 --- a/background.md +++ b/background.md @@ -81,7 +81,8 @@ Uh, yep. Man pages are ridiculously consistent as far as non-semantically structured text goes. This tool usually works fine and if it doesn't, file a bug and go hunting and pecking like you usually do. I'll fix it eventually. -Thanks. enjoy. +Want even more background? Try `mansnip roff HISTORY`. + --- diff --git a/mansnip b/mansnip index 712dd56..ca3d775 100755 --- a/mansnip +++ b/mansnip @@ -78,6 +78,13 @@ logging.info(my_re) is_def = False line_def = False stack_start = False + +# +# We want to be able to detect None versus 0 without the hassles +# of False == 0 (which I know can be avoided, sure, ok fine, but +# why introduce the possibility of bugs when you can easily avoid +# that possibility?) +# term_indent = None buf_start = False @@ -108,6 +115,15 @@ man_input = man_input.split('\n') # rs = 5 if len(man_input) < 1e4 else 7 +# +# Essentially we want to reduce our chances of false positives. +# Because there's so many variations we can't do a single-pass +# catch-all for everything. But what we can do is cast a somewhat +# wide net with the regexs and then do a second-pass rejection +# test with this function, essentially just looking for prose. +def random_word_test(line): + pass + line_num = -1 while line_num + 1 < len(man_input): line_num += 1 From 860325178e6288c02bbef316678d5deddce10b31 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Sun, 3 May 2020 16:44:39 -0700 Subject: [PATCH 100/138] restructuring --- {tests => testing}/readme.md | 0 {tests => testing}/tester.py | 0 {tests => testing}/testlist.txt | 0 {tests => testing/tests}/7_man_.TP_.PD_.B_.P_.I.test | 0 {tests => testing/tests}/bash_-z_string.test | 0 {tests => testing/tests}/bash_CONDITIONAL.test | 0 {tests => testing/tests}/bash_declare.test | 0 {tests => testing/tests}/ffmpeg_-c.test | 0 {tests => testing/tests}/ls_--ignore.test | 0 {tests => testing/tests}/ls_--inode_--ignore.test | 0 {tests => testing/tests}/lsof_-E.test | 0 {tests => testing/tests}/zshall_-z.test | 0 12 files changed, 0 insertions(+), 0 deletions(-) rename {tests => testing}/readme.md (100%) rename {tests => testing}/tester.py (100%) rename {tests => testing}/testlist.txt (100%) rename {tests => testing/tests}/7_man_.TP_.PD_.B_.P_.I.test (100%) rename {tests => testing/tests}/bash_-z_string.test (100%) rename {tests => testing/tests}/bash_CONDITIONAL.test (100%) rename {tests => testing/tests}/bash_declare.test (100%) rename {tests => testing/tests}/ffmpeg_-c.test (100%) rename {tests => testing/tests}/ls_--ignore.test (100%) rename {tests => testing/tests}/ls_--inode_--ignore.test (100%) rename {tests => testing/tests}/lsof_-E.test (100%) rename {tests => testing/tests}/zshall_-z.test (100%) diff --git a/tests/readme.md b/testing/readme.md similarity index 100% rename from tests/readme.md rename to testing/readme.md diff --git a/tests/tester.py b/testing/tester.py similarity index 100% rename from tests/tester.py rename to testing/tester.py diff --git a/tests/testlist.txt b/testing/testlist.txt similarity index 100% rename from tests/testlist.txt rename to testing/testlist.txt diff --git a/tests/7_man_.TP_.PD_.B_.P_.I.test b/testing/tests/7_man_.TP_.PD_.B_.P_.I.test similarity index 100% rename from tests/7_man_.TP_.PD_.B_.P_.I.test rename to testing/tests/7_man_.TP_.PD_.B_.P_.I.test diff --git a/tests/bash_-z_string.test b/testing/tests/bash_-z_string.test similarity index 100% rename from tests/bash_-z_string.test rename to testing/tests/bash_-z_string.test diff --git a/tests/bash_CONDITIONAL.test b/testing/tests/bash_CONDITIONAL.test similarity index 100% rename from tests/bash_CONDITIONAL.test rename to testing/tests/bash_CONDITIONAL.test diff --git a/tests/bash_declare.test b/testing/tests/bash_declare.test similarity index 100% rename from tests/bash_declare.test rename to testing/tests/bash_declare.test diff --git a/tests/ffmpeg_-c.test b/testing/tests/ffmpeg_-c.test similarity index 100% rename from tests/ffmpeg_-c.test rename to testing/tests/ffmpeg_-c.test diff --git a/tests/ls_--ignore.test b/testing/tests/ls_--ignore.test similarity index 100% rename from tests/ls_--ignore.test rename to testing/tests/ls_--ignore.test diff --git a/tests/ls_--inode_--ignore.test b/testing/tests/ls_--inode_--ignore.test similarity index 100% rename from tests/ls_--inode_--ignore.test rename to testing/tests/ls_--inode_--ignore.test diff --git a/tests/lsof_-E.test b/testing/tests/lsof_-E.test similarity index 100% rename from tests/lsof_-E.test rename to testing/tests/lsof_-E.test diff --git a/tests/zshall_-z.test b/testing/tests/zshall_-z.test similarity index 100% rename from tests/zshall_-z.test rename to testing/tests/zshall_-z.test From b4c19d90251416abe579d8d619efca3907fe13df Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Sun, 3 May 2020 17:44:49 -0700 Subject: [PATCH 101/138] adding a background --- testing/tester.py | 7 +++++-- toolchain.md | 48 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 toolchain.md diff --git a/testing/tester.py b/testing/tester.py index c36e132..b384240 100755 --- a/testing/tester.py +++ b/testing/tester.py @@ -3,6 +3,7 @@ import sys import re import tempfile +import time import subprocess import json @@ -62,14 +63,16 @@ def store_results(path, expected, actual): fname = params_to_fname(test) - with open(fname, 'r') as f: + with open("tests/" + fname, 'r') as f: expected = f.read() + start = time.time() p = subprocess.Popen(['../mansnip'] + testList, stdout=subprocess.PIPE) actual = p.communicate()[0].decode("utf-8") + runtime = "{:#.3g}".format(time.time() - start) if actual == expected: - print("PASSED " + test) + print(runtime + " PASSED " + test) else: print("!! FAILED " + test) store_results(fname, expected, actual) diff --git a/toolchain.md b/toolchain.md new file mode 100644 index 0000000..177f251 --- /dev/null +++ b/toolchain.md @@ -0,0 +1,48 @@ +These are my notes on the rather complex assortment of tools surrounding manpages and what they do. + +groff will take man pages as input, but not in the gzip format that you find on modern unix systems ... it will ingest these files just fine, because the language syntax doesn't use a stack machine and unrecognized syntax is just interpreted as content. This means a bunch of binary crap will still be "processed". For instance, you can do `groff random_file.jpg` and get output. It never fails, take that Turing! + +Now if instead we take this from man 7 roff + + cat file | ... | preproc | ... | troff options | postproc + +and try to make a real example of it ... I'm going to use lsmod(8), one of my favorite man pages, 74 words including a copyright and a notice that it's maintained by more than one person. + +Isn't that hilarious? Yes, it's quite funny. Anyway, + +Let's just do this to make our life easier for now. + + $ cp /usr/share/man/man8/lsmod.8.gz /tmp/ + $ cd /tmp + $ gzip -cd lsmod.8.gz | groff - + +Hey, stuff ... it looks like maybe that will be something. + +And now we'll find out that this is actually a thing. + + $ gzip -cd lsmod.8.gz | groff -X - + +Wow, that's hideous. Not even anti-aliasing. Did you know that all of +X used to look basically just like this? Look up xmail. + +Oh heck the internet is useless these days, all these results suck. + +Ok try xvidtune instead. There we go, that's the stuff. If X didn't come out of MIT nobody would have taken this thing seriously, it's crazy. + +Anyway, you may also notice that the `groff -X` formatting has all the white-space of 18th century literature, let's fix that. + +There's a tool that tries to make this sane called grog(1). + + + + + + +groffer +grog + +troff ditroff + +groff is the GNU roff + +man From 3fdbf9653c1091a8d2984dd971b62b142a4cc67b Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Sun, 3 Jan 2021 15:27:12 -0800 Subject: [PATCH 102/138] found a bug for wget pages. --- mansnip | 55 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/mansnip b/mansnip index ca3d775..d9ec53c 100755 --- a/mansnip +++ b/mansnip @@ -52,28 +52,36 @@ except Exception as ex: pack = sys.argv[cutoff:] opts = '|'.join(pack) -my_re = ''.join([ - '^\s*', - '(', - # lsof uses +|- syntax, - '(\+\||)', - # the term itself - '({})', +def matcher(term): + return ''.join([ + '^\s*', + '(', + # lsof uses +|- syntax, + '(\+\||)', - # ffmpeg uses [ at times (see hwaccel), many things use ',' to show a second option. - # mmcli uses = a lot of the times - '([\s_\[=].*|, .*|)', - '|', - # this is for the long options, sometimes (git config) specified by commas - '-.*\s({})', + # the term itself + '({})', - # and kinda our same capture from above - '([\s_\[=].*|)', - ')$' - ]).format(opts, opts) + # ffmpeg uses [ at times (see hwaccel), many things use ',' to show a second option. + # mmcli uses = a lot of the times + '([\s_\[=].*|, .*|)', + '|', + # this is for the long options, sometimes (git config) specified by commas + '-.*\s({})', -logging.info(my_re) + # and kinda our same capture from above + '([\s_\[=].*|)', + ')$' + ]).format(term, term) + +# the thing we are genuinely looking for +my_re = matcher(opts) + +# a general purpose getopts re +getopts_re = matcher('\w*') + +logging.info("The re for this search: {}".format(my_re)) is_def = False line_def = False @@ -188,7 +196,7 @@ while line_num + 1 < len(man_input): res = re.match(my_re, line) or re.match(my_re, man_input_plain[line_num]) if res: - logging.info("matched: {}".format(line)) + logging.info("matched : {}".format(line)) # # This is sheer frantic handwaving for things like this (From bash) # @@ -225,13 +233,18 @@ while line_num + 1 < len(man_input): term_indent = indent elif term_indent != None: - logging.info(line) + logging.info("consider: {}".format(line)) # If our indent is zero and there's some text (subsection heading) if (indent == 0 and len(line)) or ( indent > 1 and (indent < term_indent or (not is_def and indent == term_indent) )): spacer = '\n' + ' ' * rs - logging.info(len(buf)) + logging.info("Current rows: {}".format( len(buf)) ) + + #print(line) + res = re.match(getopts_re, line) or re.match(getopts_re, man_input_plain[line_num]) + #print(res) + if (len(buf) == 2 and indent == term_indent and line_def) or len(buf) > 1: # # This is a lot of fancy formatting. We want the From c895baa2d2dfd87dce760e177f2c9e2af477ce48 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Sun, 3 Jan 2021 17:52:55 -0800 Subject: [PATCH 103/138] wget use case --- mansnip | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/mansnip b/mansnip index d9ec53c..8270877 100755 --- a/mansnip +++ b/mansnip @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -import sys, re, os, logging +import sys, re, os, logging, pdb # less and others have a number of things like this # @@ -68,7 +68,8 @@ def matcher(term): '([\s_\[=].*|, .*|)', '|', # this is for the long options, sometimes (git config) specified by commas - '-.*\s({})', + #'-.*\s({})', + '-.*({})', # and kinda our same capture from above '([\s_\[=].*|)', @@ -79,12 +80,13 @@ def matcher(term): my_re = matcher(opts) # a general purpose getopts re -getopts_re = matcher('\w*') +getopts_simple_re = '^\s+(-{1,2}\w+\s?)+\s*$' logging.info("The re for this search: {}".format(my_re)) is_def = False line_def = False +multiline_def = False stack_start = False # @@ -234,16 +236,37 @@ while line_num + 1 < len(man_input): elif term_indent != None: logging.info("consider: {}".format(line)) + + # This tests if we have a format like in wget, something like + # + # -r + # --recursive + # Turn on recursive retrieving. The default maximum depth is 5. + # + # -l depth + # --level=depth + # Specify recursion maximum depth level depth. + # + # In this case we're ok with not increasing indentation as long as we match + # a general getopts RE + + is_multiline_def = re.match(getopts_simple_re, line) or re.match(getopts_simple_re, man_input_plain[line_num]) + + #print(is_multiline_def, '\n\t', "[{}]".format(line), '\n\t', "[{}]".format(man_input_plain[line_num]), '\n\t') # If our indent is zero and there's some text (subsection heading) if (indent == 0 and len(line)) or ( - indent > 1 and (indent < term_indent or (not is_def and indent == term_indent) + indent > 1 and ( + indent < term_indent or ( + # If we have the same indentation but we've determined it's a + # "multiline" definition. + indent == term_indent and not ( + is_def or is_multiline_def + ) + ) )): spacer = '\n' + ' ' * rs logging.info("Current rows: {}".format( len(buf)) ) - #print(line) - res = re.match(getopts_re, line) or re.match(getopts_re, man_input_plain[line_num]) - #print(res) if (len(buf) == 2 and indent == term_indent and line_def) or len(buf) > 1: # From 07d31525add09da659328ad630a36180d0197ab6 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Sun, 3 Jan 2021 17:54:40 -0800 Subject: [PATCH 104/138] adding a wget test --- testing/testlist.txt | 1 + testing/wget_-r.test | 9 +++++++++ 2 files changed, 10 insertions(+) create mode 100644 testing/wget_-r.test diff --git a/testing/testlist.txt b/testing/testlist.txt index 896bbfc..2800aa8 100644 --- a/testing/testlist.txt +++ b/testing/testlist.txt @@ -7,3 +7,4 @@ ["ls", "--inode", "--ignore"] ["bash", "-z string"] ["bash", "CONDITIONAL"] +["wget", "-r"] diff --git a/testing/wget_-r.test b/testing/wget_-r.test new file mode 100644 index 0000000..96a90f0 --- /dev/null +++ b/testing/wget_-r.test @@ -0,0 +1,9 @@ +888 FFTTPP OOppttiioonnss + ----nnoo--rreemmoovvee--lliissttiinngg + --rr so the file will be overwritten. + +938 RReeccuurrssiivvee RReettrriieevvaall OOppttiioonnss + --rr + ----rreeccuurrssiivvee + Turn on recursive retrieving. The default maximum depth is 5. + From 895f364aefc8eb2a9dcc8556cb5ef5120d197734 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Sun, 3 Jan 2021 20:20:54 -0800 Subject: [PATCH 105/138] tests are made in the wrong place --- testing/tester.py | 2 +- testing/{ => tests}/wget_-r.test | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename testing/{ => tests}/wget_-r.test (100%) diff --git a/testing/tester.py b/testing/tester.py index b384240..1773fb5 100755 --- a/testing/tester.py +++ b/testing/tester.py @@ -27,7 +27,7 @@ def store_results(path, expected, actual): if len(sys.argv) > 1: print("making test for arguments") args = sys.argv[1:] - fname = params_to_fname(' '.join(args)) + fname = "tests/" + params_to_fname(' '.join(args)) if os.path.exists(fname): print("I'm not going to overwrite {}. Please remove it then rerun this.".format(fname)) diff --git a/testing/wget_-r.test b/testing/tests/wget_-r.test similarity index 100% rename from testing/wget_-r.test rename to testing/tests/wget_-r.test From 03ca042148ef8a67c35509ef4d9a81d0da78933f Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Sun, 3 Jan 2021 20:40:56 -0800 Subject: [PATCH 106/138] Adding a way for tests to suppress line numbers to have more test longevity --- mansnip | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mansnip b/mansnip index 8270877..6cb78e6 100755 --- a/mansnip +++ b/mansnip @@ -15,6 +15,7 @@ import sys, re, os, logging, pdb # logging.basicConfig(level=(os.environ.get('LOGLEVEL') or 'critical').upper()) +show_line_numbers = 'MANSNIP_TESTING_NOLINES' not in os.environ try: if len(sys.argv) < 3: @@ -309,6 +310,10 @@ while line_num + 1 < len(man_input): stack_print = '' buf[0] = re.sub('^\s{%d}' % rs, '', buf[0]) + # This makes the tests more portable + if not show_line_numbers: + buf_start = -1 + print(("{:<%d}{}{}\n" % rs).format(buf_start, stack_print, '\n'.join(buf))) # if this was the line we decided to reset everything on then we should actually process From bcabd69adb0959218408fe38f9140d0544979d38 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Sun, 3 Jan 2021 20:42:17 -0800 Subject: [PATCH 107/138] Adding a way to do single unit testing and list tests --- testing/tester.py | 52 +++++++++++++++++++++++++++++++---------------- 1 file changed, 35 insertions(+), 17 deletions(-) diff --git a/testing/tester.py b/testing/tester.py index 1773fb5..442d84a 100755 --- a/testing/tester.py +++ b/testing/tester.py @@ -8,7 +8,10 @@ import json os.environ['MANWIDTH'] = '250' +os.environ['MANSNIP_TESTING_NOLINES'] = '1' + tempdir = None +test_filter = None def params_to_fname(params): return re.sub(' ', '_', params) + ".test" @@ -24,27 +27,40 @@ def store_results(path, expected, actual): with open("{}/{}-actual".format(tempdir,path), 'w') as f: f.write(actual) -if len(sys.argv) > 1: - print("making test for arguments") - args = sys.argv[1:] - fname = "tests/" + params_to_fname(' '.join(args)) - - if os.path.exists(fname): - print("I'm not going to overwrite {}. Please remove it then rerun this.".format(fname)) - sys.exit(-1) +if len(sys.argv) == 2: + with open('testlist.txt', 'r') as f: + testList = f.read().splitlines() - with open('testlist.txt', 'a') as f: - f.write(json.dumps(args) + "\n") + for testraw in testList: + testList = json.loads(testraw) + test = ' '.join(testList) + print(test) - with open(fname, 'w') as f: - print("Creating {}".format(fname)) + sys.exit(0) - p = subprocess.Popen(['../mansnip'] + args, stdout=subprocess.PIPE) - res = p.communicate()[0].decode("utf-8") +if len(sys.argv) > 2: + args = sys.argv[1:] + test_filter = ' '.join(args) + fname = "tests/" + params_to_fname(test_filter) - print(res) - f.write(res) - sys.exit(0) + if os.path.exists(fname): + sys.stderr.write("I'm not going to overwrite {}. If you want to replace it then remove it and this.\n".format(fname)) + sys.stderr.write("Running single test.\n") + + else: + sys.stderr.write("making test for arguments\n") + with open('testlist.txt', 'a') as f: + f.write(json.dumps(args) + "\n") + + with open(fname, 'w') as f: + print("Creating {}".format(fname)) + + p = subprocess.Popen(['../mansnip'] + args, stdout=subprocess.PIPE) + res = p.communicate()[0].decode("utf-8") + + print(res) + f.write(res) + sys.exit(0) with open('testlist.txt', 'r') as f: testList = f.read().splitlines() @@ -52,6 +68,8 @@ def store_results(path, expected, actual): for testraw in testList: testList = json.loads(testraw) test = ' '.join(testList) + if test_filter and test_filter != test: + continue if test == 'stop': print("asked to stop") From 0fa0a057356ed2224b8ae5bda2e1bd900a8b04b9 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Sun, 3 Jan 2021 20:42:46 -0800 Subject: [PATCH 108/138] updating tests to be linenumber free --- testing/tests/7_man_.TP_.PD_.B_.P_.I.test | 10 +++++----- testing/tests/bash_-z_string.test | 2 +- testing/tests/bash_CONDITIONAL.test | 4 ++-- testing/tests/bash_declare.test | 2 +- testing/tests/ffmpeg_-c.test | 2 +- testing/tests/ls_--ignore.test | 2 +- testing/tests/ls_--inode_--ignore.test | 4 ++-- testing/tests/lsof_-E.test | 2 +- testing/tests/wget_-r.test | 4 ++-- testing/tests/zshall_-z.test | 10 +++++----- 10 files changed, 21 insertions(+), 21 deletions(-) diff --git a/testing/tests/7_man_.TP_.PD_.B_.P_.I.test b/testing/tests/7_man_.TP_.PD_.B_.P_.I.test index 37c0a86..a82f988 100644 --- a/testing/tests/7_man_.TP_.PD_.B_.P_.I.test +++ b/testing/tests/7_man_.TP_.PD_.B_.P_.I.test @@ -1,14 +1,14 @@ -46 FFoonnttss +-1 FFoonnttss ..BB Bold -52 ..II Italics +-1 ..II Italics -79 NNoorrmmaall ppaarraaggrraapphhss +-1 NNoorrmmaall ppaarraaggrraapphhss ..PP Same as ..PPPP (begin a new paragraph). -96 IInnddeenntteedd ppaarraaggrraapphh mmaaccrrooss +-1 IInnddeenntteedd ppaarraaggrraapphh mmaaccrrooss ..TTPP _i Begin paragraph with hanging tag. The tag is given on the next line, but its results are like those of the ..IIPP command. -112 MMiisscceellllaanneeoouuss mmaaccrrooss +-1 MMiisscceellllaanneeoouuss mmaaccrrooss ..PPDD _d Set inter-paragraph vertical distance to d (if omitted, d=0.4v); does not cause a break. diff --git a/testing/tests/bash_-z_string.test b/testing/tests/bash_-z_string.test index 4fb8aea..3151514 100644 --- a/testing/tests/bash_-z_string.test +++ b/testing/tests/bash_-z_string.test @@ -1,4 +1,4 @@ -1311 CCOONNDDIITTIIOONNAALL EEXXPPRREESSSSIIOONNSS +-1 CCOONNDDIITTIIOONNAALL EEXXPPRREESSSSIIOONNSS --zz _s_t_r_i_n_g True if the length of _s_t_r_i_n_g is zero. diff --git a/testing/tests/bash_CONDITIONAL.test b/testing/tests/bash_CONDITIONAL.test index 76ee234..b779f8e 100644 --- a/testing/tests/bash_CONDITIONAL.test +++ b/testing/tests/bash_CONDITIONAL.test @@ -1,4 +1,4 @@ -1246 CCOONNDDIITTIIOONNAALL EEXXPPRREESSSSIIOONNSS +-1 CCOONNDDIITTIIOONNAALL EEXXPPRREESSSSIIOONNSS Conditional expressions are used by the [[[[ compound command and the tteesstt and [[ builtin commands to test file attributes and perform string and arithmetic comparisons. The tteesstt abd [[ commands determine their behavior based on the number of arguments; see the descriptions of those commands for any other command-specific actions. @@ -86,7 +86,7 @@ OOPP is one of --eeqq, --nnee, --lltt, --llee, --ggtt, or --ggee. These arithmetic binary operators return true if _a_r_g_1 is equal to, not equal to, less than, less than or equal to, greater than, or greater than or equal to _a_r_g_2, respectively. _A_r_g_1 and _a_r_g_2 may be positive or negative integers. When used with the [[[[ command, _A_r_g_1 and _A_r_g_2 are evaluated as arithmetic expressions (see AARRIITTHHMMEETTIICC EEVVAALLUUAATTIIOONN above). -2950 SSHHEELLLL BBUUIILLTTIINN CCOOMMMMAANNDDSS +-1 SSHHEELLLL BBUUIILLTTIINN CCOOMMMMAANNDDSS tteesstt _e_x_p_r [[ _e_x_p_r ]] CCOONNDDIITTIIOONNAALL EEXXPPRREESSSSIIOONNSS. tteesstt does not accept any options, nor does it accept and ignore an argument of ---- as signifying the end of options. diff --git a/testing/tests/bash_declare.test b/testing/tests/bash_declare.test index 2d10671..0a76348 100644 --- a/testing/tests/bash_declare.test +++ b/testing/tests/bash_declare.test @@ -1,4 +1,4 @@ -2397 SSHHEELLLL BBUUIILLTTIINN CCOOMMMMAANNDDSS +-1 SSHHEELLLL BBUUIILLTTIINN CCOOMMMMAANNDDSS ddeeccllaarree [--aaAAffFFggiillnnrrttuuxx] [--pp] [_n_a_m_e[=_v_a_l_u_e] ...] ttyyppeesseett [--aaAAffFFggiillnnrrttuuxx] [--pp] [_n_a_m_e[=_v_a_l_u_e] ...] Declare variables and/or give them attributes. If no _n_a_m_es are given then display the values of variables. The --pp option will display the attributes and values of each _n_a_m_e. When --pp is used with _n_a_m_e arguments, additional op‐ diff --git a/testing/tests/ffmpeg_-c.test b/testing/tests/ffmpeg_-c.test index 5963d64..83aeee1 100644 --- a/testing/tests/ffmpeg_-c.test +++ b/testing/tests/ffmpeg_-c.test @@ -1,4 +1,4 @@ -435 MMaaiinn ooppttiioonnss +-1 MMaaiinn ooppttiioonnss --cc[[::_s_t_r_e_a_m___s_p_e_c_i_f_i_e_r]] _c_o_d_e_c ((_i_n_p_u_t_/_o_u_t_p_u_t_,_p_e_r_-_s_t_r_e_a_m)) --ccooddeecc[[::_s_t_r_e_a_m___s_p_e_c_i_f_i_e_r]] _c_o_d_e_c ((_i_n_p_u_t_/_o_u_t_p_u_t_,_p_e_r_-_s_t_r_e_a_m)) Select an encoder (when used before an output file) or a decoder (when used before an input file) for one or more streams. _c_o_d_e_c is the name of a decoder/encoder or a special value "copy" (output only) to indicate that the stream is diff --git a/testing/tests/ls_--ignore.test b/testing/tests/ls_--ignore.test index 332ae50..23d541c 100644 --- a/testing/tests/ls_--ignore.test +++ b/testing/tests/ls_--ignore.test @@ -1,3 +1,3 @@ -93 --II, ----iiggnnoorree=_P_A_T_T_E_R_N +-1 --II, ----iiggnnoorree=_P_A_T_T_E_R_N do not list implied entries matching shell PATTERN diff --git a/testing/tests/ls_--inode_--ignore.test b/testing/tests/ls_--inode_--ignore.test index 39f401f..0964eaa 100644 --- a/testing/tests/ls_--inode_--ignore.test +++ b/testing/tests/ls_--inode_--ignore.test @@ -1,6 +1,6 @@ -90 --ii, ----iinnooddee +-1 --ii, ----iinnooddee print the index number of each file -93 --II, ----iiggnnoorree=_P_A_T_T_E_R_N +-1 --II, ----iiggnnoorree=_P_A_T_T_E_R_N do not list implied entries matching shell PATTERN diff --git a/testing/tests/lsof_-E.test b/testing/tests/lsof_-E.test index f1ed6e1..9e1c981 100644 --- a/testing/tests/lsof_-E.test +++ b/testing/tests/lsof_-E.test @@ -1,4 +1,4 @@ -174 ++||--EE ++EE specifies that Linux pipe, Linux UNIX socket and Linux pseudoterminal files should be displayed with endpoint information and the files of the endpoints should also be displayed. Note: UNIX socket file endpoint information +-1 ++||--EE ++EE specifies that Linux pipe, Linux UNIX socket and Linux pseudoterminal files should be displayed with endpoint information and the files of the endpoints should also be displayed. Note: UNIX socket file endpoint information is only available when the compile flags line of --vv output contains HASUXSOCKEPT, and psudoterminal endpoint information is only available when the compile flags line contains HASPTYEPT. Pipe endpoint information is displayed in the NAME column in the form ``_P_I_D_,_c_m_d_,_F_D_m_o_d_e'', where _P_I_D is the endpoint process ID; _c_m_d is the endpoint process command; _F_D is the endpoint file's descriptor; and _m_o_d_e is the endpoint diff --git a/testing/tests/wget_-r.test b/testing/tests/wget_-r.test index 96a90f0..840a709 100644 --- a/testing/tests/wget_-r.test +++ b/testing/tests/wget_-r.test @@ -1,8 +1,8 @@ -888 FFTTPP OOppttiioonnss +-1 FFTTPP OOppttiioonnss ----nnoo--rreemmoovvee--lliissttiinngg --rr so the file will be overwritten. -938 RReeccuurrssiivvee RReettrriieevvaall OOppttiioonnss +-1 RReeccuurrssiivvee RReettrriieevvaall OOppttiioonnss --rr ----rreeccuurrssiivvee Turn on recursive retrieving. The default maximum depth is 5. diff --git a/testing/tests/zshall_-z.test b/testing/tests/zshall_-z.test index 30efde3..a310371 100644 --- a/testing/tests/zshall_-z.test +++ b/testing/tests/zshall_-z.test @@ -1,20 +1,20 @@ -1371 CCOONNDDIITTIIOONNAALL EEXXPPRREESSSSIIOONNSS +-1 CCOONNDDIITTIIOONNAALL EEXXPPRREESSSSIIOONNSS --zz _s_t_r_i_n_g true if length of _s_t_r_i_n_g is zero. -5777 SSHHEELLLL BBUUIILLTTIINN CCOOMMMMAANNDDSS +-1 SSHHEELLLL BBUUIILLTTIINN CCOOMMMMAANNDDSS pprriinntt [ --aabbccDDiillmmnnNNooOOppPPrrssSSzz ] [ --uu _n ] [ --ff _f_o_r_m_a_t ] [ --CC _c_o_l_s ] [ --vv _n_a_m_e ] [ --xxXX _t_a_b_s_t_o_p ] [ --RR [ --eenn ]] [ _a_r_g ... ] --zz Push the arguments onto the editing buffer stack, separated by spaces. If any of `--mm', `--oo' or `--OO' are used in combination with `--ff' and there are no arguments (after the removal process in the case of `--mm') then nothing is printed. -5839 rreeaadd [ --rrsszzppqqAAccllnneeEE ] [ --tt [ _n_u_m ] ] [ --kk [ _n_u_m ] ] [ --dd _d_e_l_i_m ] +-1 rreeaadd [ --rrsszzppqqAAccllnneeEE ] [ --tt [ _n_u_m ] ] [ --kk [ _n_u_m ] ] [ --dd _d_e_l_i_m ] [ --uu _n ] [ _n_a_m_e[??_p_r_o_m_p_t] ] [ _n_a_m_e ... ] --zz Read one entry from the editor buffer stack and assign it to the first _n_a_m_e, without word splitting. Text is pushed onto the stack with `pprriinntt --zz' or with ppuusshh--lliinnee from the line editor (see _z_s_h_z_l_e(1)). This flag is ig‐ nored when the --kk or --qq flags are present. -6382 zzccoommppiillee [ --UU ] [ --zz | --kk ] [ --RR | --MM ] _f_i_l_e [ _n_a_m_e ... ] +-1 zzccoommppiillee [ --UU ] [ --zz | --kk ] [ --RR | --MM ] _f_i_l_e [ _n_a_m_e ... ] zzccoommppiillee --ccaa [ --mm ] [ --RR | --MM ] _f_i_l_e [ _n_a_m_e ... ] zzccoommppiillee --tt _f_i_l_e [ _n_a_m_e ... ] --zz These options are used when the compiled file contains functions which are to be autoloaded. If --zz is given, the function will be autoloaded as if the KKSSHH__AAUUTTOOLLOOAADD option is _n_o_t set, even if it is set at the time the com‐ @@ -26,7 +26,7 @@ The created file always contains two versions of the compiled format, one for big-endian machines and one for small-endian machines. The upshot of this is that the compiled file is machine independent and if it is read or mapped, only one half of the file is actually used (and mapped). -11596 OOPPTTIIOONN FFLLAAGGSS +-1 OOPPTTIIOONN FFLLAAGGSS SSiimmppllee FFllaaggss --zz Names of suspended jobs. From 43b5079fb6585b0973d7594f066e514510258c46 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Sun, 3 Jan 2021 21:13:38 -0800 Subject: [PATCH 109/138] new version of the zshall test --- testing/tests/zshall_-z.test | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/tests/zshall_-z.test b/testing/tests/zshall_-z.test index a310371..0cb2fc4 100644 --- a/testing/tests/zshall_-z.test +++ b/testing/tests/zshall_-z.test @@ -11,14 +11,14 @@ -1 rreeaadd [ --rrsszzppqqAAccllnneeEE ] [ --tt [ _n_u_m ] ] [ --kk [ _n_u_m ] ] [ --dd _d_e_l_i_m ] [ --uu _n ] [ _n_a_m_e[??_p_r_o_m_p_t] ] [ _n_a_m_e ... ] - --zz Read one entry from the editor buffer stack and assign it to the first _n_a_m_e, without word splitting. Text is pushed onto the stack with `pprriinntt --zz' or with ppuusshh--lliinnee from the line editor (see _z_s_h_z_l_e(1)). This flag is ig‐ + --zz Read one entry from the editor buffer stack and assign it to the first _n_a_m_e, without word splitting. Text is pushed onto the stack with `pprriinntt --zz' or with ppuusshh--lliinnee from the line editor (see _z_s_h_z_l_e(1)). This flag is ig‐ nored when the --kk or --qq flags are present. -1 zzccoommppiillee [ --UU ] [ --zz | --kk ] [ --RR | --MM ] _f_i_l_e [ _n_a_m_e ... ] zzccoommppiillee --ccaa [ --mm ] [ --RR | --MM ] _f_i_l_e [ _n_a_m_e ... ] zzccoommppiillee --tt _f_i_l_e [ _n_a_m_e ... ] - --zz These options are used when the compiled file contains functions which are to be autoloaded. If --zz is given, the function will be autoloaded as if the KKSSHH__AAUUTTOOLLOOAADD option is _n_o_t set, even if it is set at the time the com‐ - piled file is read, while if the --kk is given, the function will be loaded as if KKSSHH__AAUUTTOOLLOOAADD _i_s set. These options also take precedence over any --kk or --zz options specified to the aauuttoollooaadd builtin. If neither of these op‐ + --zz These options are used when the compiled file contains functions which are to be autoloaded. If --zz is given, the function will be autoloaded as if the KKSSHH__AAUUTTOOLLOOAADD option is _n_o_t set, even if it is set at the time the com‐ + piled file is read, while if the --kk is given, the function will be loaded as if KKSSHH__AAUUTTOOLLOOAADD _i_s set. These options also take precedence over any --kk or --zz options specified to the aauuttoollooaadd builtin. If neither of these op‐ tions is given, the function will be loaded as determined by the setting of the KKSSHH__AAUUTTOOLLOOAADD option at the time the compiled file is read. These options may also appear as many times as necessary between the listed _n_a_m_es to specify the loading style of all following functions, up to the next --kk or --zz. From c6981b88b7f6fb8dd0183cbd9f685ffd6bbbbbb5 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Sun, 3 Jan 2021 21:14:09 -0800 Subject: [PATCH 110/138] fixing the zshall bug introduced with the wget feature --- mansnip | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/mansnip b/mansnip index 6cb78e6..074d142 100755 --- a/mansnip +++ b/mansnip @@ -194,7 +194,7 @@ while line_num + 1 < len(man_input): if not len(buf): - # See if our ansi escaped or plain text version has what we are looking for. + # See if our ANSI escaped or plain text version has what we are looking for. # yes that makes it a bit slower, but not as slow as doing it manually. res = re.match(my_re, line) or re.match(my_re, man_input_plain[line_num]) @@ -233,25 +233,38 @@ while line_num + 1 < len(man_input): buf.append(line) buf_start = line_num stack_start = stack_guess[:] + is_multiline_def = True term_indent = indent elif term_indent != None: logging.info("consider: {}".format(line)) + # # This tests if we have a format like in wget, something like # - # -r - # --recursive - # Turn on recursive retrieving. The default maximum depth is 5. + # -r + # --recursive + # Turn on recursive retrieving. The default maximum depth is 5. # - # -l depth - # --level=depth - # Specify recursion maximum depth level depth. + # -l depth + # --level=depth + # Specify recursion maximum depth level depth. # # In this case we're ok with not increasing indentation as long as we match - # a general getopts RE - - is_multiline_def = re.match(getopts_simple_re, line) or re.match(getopts_simple_re, man_input_plain[line_num]) + # + # What is this boolean flippin' nonsense then? We allow essentially the following + # + # --A <- These three should be chained. + # --B + # --C + # blah blah <- Once we reach this indent further + # blah blah <- chaining should be prohibited + # + # --D <- This should be excluded. + # + if is_multiline_def: + if not (re.match(getopts_simple_re, line) or re.match(getopts_simple_re, man_input_plain[line_num])): + is_multiline_def = False #print(is_multiline_def, '\n\t', "[{}]".format(line), '\n\t', "[{}]".format(man_input_plain[line_num]), '\n\t') # If our indent is zero and there's some text (subsection heading) From d3deeb50f5fa4fc90217ad447e946469164d8de4 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Mon, 4 Jan 2021 00:48:26 -0800 Subject: [PATCH 111/138] version bump for pypi --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2872b48..62db584 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setuptools.setup( name="mansnip-kristopolous", - version="0.3.5", + version="0.3.6", scripts=["mansnip"], author="Chris McKenzie", author_email="kristopolous@yahoo.com", From 3c0f1e4928194caed413df48e482d759f6ad8171 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Mon, 4 Jan 2021 00:53:20 -0800 Subject: [PATCH 112/138] fixing the awk bug described in #7 --- mansnip | 2 +- testing/testlist.txt | 1 + testing/tests/awk_-f.test | 4 ++++ 3 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 testing/tests/awk_-f.test diff --git a/mansnip b/mansnip index 074d142..c210fc0 100755 --- a/mansnip +++ b/mansnip @@ -81,7 +81,7 @@ def matcher(term): my_re = matcher(opts) # a general purpose getopts re -getopts_simple_re = '^\s+(-{1,2}\w+\s?)+\s*$' +getopts_simple_re = '^\s+(-{1,2}\w+\s?\w*)+\s*$' logging.info("The re for this search: {}".format(my_re)) diff --git a/testing/testlist.txt b/testing/testlist.txt index 2800aa8..7f109b0 100644 --- a/testing/testlist.txt +++ b/testing/testlist.txt @@ -8,3 +8,4 @@ ["bash", "-z string"] ["bash", "CONDITIONAL"] ["wget", "-r"] +["awk", "-f"] diff --git a/testing/tests/awk_-f.test b/testing/tests/awk_-f.test new file mode 100644 index 0000000..01fee99 --- /dev/null +++ b/testing/tests/awk_-f.test @@ -0,0 +1,4 @@ +-1 --ff _p_r_o_g_r_a_m_-_f_i_l_e + ----ffiillee _p_r_o_g_r_a_m_-_f_i_l_e + Read the AWK program source from the file _p_r_o_g_r_a_m_-_f_i_l_e, instead of from the first command line argument. Multiple --ff (or ----ffiillee) options may be used. + From d5de9e84fa709d3dc23dccadecd9e5089ca0bc2d Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Mon, 4 Jan 2021 00:56:02 -0800 Subject: [PATCH 113/138] cleaner output --- testing/tester.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/tester.py b/testing/tester.py index 442d84a..777cb86 100755 --- a/testing/tester.py +++ b/testing/tester.py @@ -87,7 +87,7 @@ def store_results(path, expected, actual): start = time.time() p = subprocess.Popen(['../mansnip'] + testList, stdout=subprocess.PIPE) actual = p.communicate()[0].decode("utf-8") - runtime = "{:#.3g}".format(time.time() - start) + runtime = "{:#.3f}".format(time.time() - start) if actual == expected: print(runtime + " PASSED " + test) From af0b3145149100180fc5faf09390c8821adfdc67 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Mon, 4 Jan 2021 01:02:05 -0800 Subject: [PATCH 114/138] testing cleanup --- testing/tester.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/tester.py b/testing/tester.py index 777cb86..9496a92 100755 --- a/testing/tester.py +++ b/testing/tester.py @@ -19,7 +19,7 @@ def store_results(path, expected, actual): global tempdir if not tempdir: tempdir = tempfile.mkdtemp() - print("Using {} for failures".format(tempdir)) + print(" Using {} for failures".format(tempdir)) with open("{}/{}-expected".format(tempdir,path), 'w') as f: f.write(expected) @@ -90,9 +90,9 @@ def store_results(path, expected, actual): runtime = "{:#.3f}".format(time.time() - start) if actual == expected: - print(runtime + " PASSED " + test) + print(runtime + " OK " + test) else: - print("!! FAILED " + test) store_results(fname, expected, actual) + print(runtime + " !! FAILED " + test) From 5af69d9838dcdbd296a48cf7257ef87f10efc75f Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Fri, 8 Jan 2021 13:20:28 -0800 Subject: [PATCH 115/138] RHEL fix #7 --- mansnip | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mansnip b/mansnip index c210fc0..19a54ff 100755 --- a/mansnip +++ b/mansnip @@ -109,7 +109,7 @@ last_stack_guess = [] # Words we can just leave out of the breadcrumb. filler_terms = ['DESCRIPTION','OPTIONS'] -clean_ansi = lambda w: re.sub('(.\x08)', '', w) +clean_ansi = lambda w: re.sub('(.\x08|\x1B\[\d*m)', '', w) # # There's lots of let's say "creative" ways to format man pages, so From b2409618e601a233b0b6566f2f297f0895f9e7b4 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 13 Dec 2022 09:09:26 -0800 Subject: [PATCH 116/138] updating urls --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index ae901ac..66105da 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Ever try to find things like the "declare" built-in in `bash(1)` only to slodge through the results using the 'n' key going 'nope, nope, nope'? -![the old way](http://i.9ol.es/animate.gif) +![the old way](https://9ol.es/animate.gif) Stop wasting time with the old way of manually stumbling through manuals. Say goodbye to these problems once and for all! @@ -26,7 +26,7 @@ Simply use it the way you use man, at the command line, followed by your search Watch how mansnip can immediately find `bash(1)'s` declare without any extra effort: -![mansnip is amazing](http://i.9ol.es/msfade.webp) +![mansnip is amazing](https://9ol.es/msfade.webp) Mansnip works on any manpage. @@ -34,7 +34,7 @@ Mansnip works on any manpage. See how mansnip obediently shows everything with a "-z" option in the 25,888 lines of the [zshall manpage](http://gsp.com/cgi-bin/man.cgi?section=1&topic=zshall) on a single screen, all at once, in an easy-to-read manner. -![zshall for all](http://i.9ol.es/mansnip.webp) +![zshall for all](https://9ol.es/mansnip.webp) But wait, there's more! You'll also get the line number and hierarchical context totally free! @@ -48,7 +48,7 @@ Here's how do it Act now, servers are standing by. -![mansnip](http://i.9ol.es/man1.jpg) +![mansnip](https://9ol.es/man1.jpg) ＦＡＤＥ ＴＯ ＢＬＡＣＫ From ebd13d4e8e1b03c93b537cb6d5e2dec356014369 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 13 Dec 2022 09:10:56 -0800 Subject: [PATCH 117/138] updating links --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 66105da..d937eed 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ Simply use it the way you use man, at the command line, followed by your search Watch how mansnip can immediately find `bash(1)'s` declare without any extra effort: -![mansnip is amazing](https://9ol.es/msfade.webp) +![mansnip is amazing](https://9ol.es/msfade.png) Mansnip works on any manpage. @@ -34,7 +34,7 @@ Mansnip works on any manpage. See how mansnip obediently shows everything with a "-z" option in the 25,888 lines of the [zshall manpage](http://gsp.com/cgi-bin/man.cgi?section=1&topic=zshall) on a single screen, all at once, in an easy-to-read manner. -![zshall for all](https://9ol.es/mansnip.webp) +![zshall for all](https://9ol.es/mansnip.png) But wait, there's more! You'll also get the line number and hierarchical context totally free! From 382f3a1c5258f43b5b3aef198466e695640ecd63 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Tue, 13 Dec 2022 09:12:03 -0800 Subject: [PATCH 118/138] updating links --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d937eed..e4cb876 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@

-[![Video](http://i.9ol.es/vid.jpg)](http://www.youtube.com/watch?v=3GT1J-ejM3Q) +[![Video](https://9ol.es/vid.jpg)](http://www.youtube.com/watch?v=3GT1J-ejM3Q) > "As seen on YouTube!" (click image, it's only 1min 45sec)

From 4be965ecbc1d371b6f5c0a79b59a08a6f81b644e Mon Sep 17 00:00:00 2001 From: elig0n <31196036+elig0n@users.noreply.github.com> Date: Sun, 30 Jun 2024 20:54:47 +0300 Subject: [PATCH 119/138] Fix regexp strings --- mansnip | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/mansnip b/mansnip index 19a54ff..f557097 100755 --- a/mansnip +++ b/mansnip @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/python import sys, re, os, logging, pdb # less and others have a number of things like this @@ -56,24 +56,24 @@ opts = '|'.join(pack) def matcher(term): return ''.join([ - '^\s*', - '(', + r'^\s*', + r'(', # lsof uses +|- syntax, - '(\+\||)', + r'(\+\||)', # the term itself - '({})', + r'({})', # ffmpeg uses [ at times (see hwaccel), many things use ',' to show a second option. # mmcli uses = a lot of the times - '([\s_\[=].*|, .*|)', + r'([\s_\[=].*|, .*|)', '|', # this is for the long options, sometimes (git config) specified by commas #'-.*\s({})', - '-.*({})', + r'-.*({})', # and kinda our same capture from above - '([\s_\[=].*|)', + r'([\s_\[=].*|)', ')$' ]).format(term, term) @@ -81,7 +81,7 @@ def matcher(term): my_re = matcher(opts) # a general purpose getopts re -getopts_simple_re = '^\s+(-{1,2}\w+\s?\w*)+\s*$' +getopts_simple_re = r'^\s+(-{1,2}\w+\s?\w*)+\s*$' logging.info("The re for this search: {}".format(my_re)) @@ -109,7 +109,7 @@ last_stack_guess = [] # Words we can just leave out of the breadcrumb. filler_terms = ['DESCRIPTION','OPTIONS'] -clean_ansi = lambda w: re.sub('(.\x08|\x1B\[\d*m)', '', w) +clean_ansi = lambda w: re.sub(r'(.\x08|\x1B\[\d*m)', '', w) # # There's lots of let's say "creative" ways to format man pages, so @@ -145,7 +145,7 @@ while line_num + 1 < len(man_input): # Establish the "indent", this is crucial to how essentially # everything works. # - indent = re.match('^(\s*)', line).end() + indent = re.match(r'^(\s*)', line).end() indent_window = indent_window[-2:] + [indent] if len(line): @@ -321,7 +321,7 @@ while line_num + 1 < len(man_input): stack_print += "\n" else: stack_print = '' - buf[0] = re.sub('^\s{%d}' % rs, '', buf[0]) + buf[0] = re.sub(r'^\s{%d}' % rs, '', buf[0]) # This makes the tests more portable if not show_line_numbers: From 36ab6f50759d2a235f8e440da38fc410a17be714 Mon Sep 17 00:00:00 2001 From: elig0n <31196036+elig0n@users.noreply.github.com> Date: Sun, 30 Jun 2024 20:59:53 +0300 Subject: [PATCH 120/138] Fix shebang --- mansnip | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mansnip b/mansnip index f557097..421661f 100755 --- a/mansnip +++ b/mansnip @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 import sys, re, os, logging, pdb # less and others have a number of things like this From 2a998e868ad814ec175a2617978d430ea210beb5 Mon Sep 17 00:00:00 2001 From: elig0n <31196036+elig0n@users.noreply.github.com> Date: Sun, 30 Jun 2024 21:02:43 +0300 Subject: [PATCH 121/138] regexped more strings --- mansnip | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mansnip b/mansnip index 421661f..ec960eb 100755 --- a/mansnip +++ b/mansnip @@ -67,14 +67,14 @@ def matcher(term): # ffmpeg uses [ at times (see hwaccel), many things use ',' to show a second option. # mmcli uses = a lot of the times r'([\s_\[=].*|, .*|)', - '|', + r'|', # this is for the long options, sometimes (git config) specified by commas #'-.*\s({})', r'-.*({})', # and kinda our same capture from above r'([\s_\[=].*|)', - ')$' + r')$' ]).format(term, term) # the thing we are genuinely looking for From 0c2266e1d04eb7cc23fceab19d4688efe933b8a7 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Mon, 21 Apr 2025 09:55:01 -0700 Subject: [PATCH 122/138] version bump: --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 62db584..a146e8c 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setuptools.setup( name="mansnip-kristopolous", - version="0.3.6", + version="0.4.0", scripts=["mansnip"], author="Chris McKenzie", author_email="kristopolous@yahoo.com", From 08870f869641bf9aef8467b342b615881153dc1f Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Thu, 17 Jul 2025 05:37:50 -0700 Subject: [PATCH 123/138] adding an LLM option --- mansnip | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mansnip b/mansnip index ec960eb..a150482 100755 --- a/mansnip +++ b/mansnip @@ -15,7 +15,7 @@ import sys, re, os, logging, pdb # logging.basicConfig(level=(os.environ.get('LOGLEVEL') or 'critical').upper()) -show_line_numbers = 'MANSNIP_TESTING_NOLINES' not in os.environ +show_line_numbers = 'MANSNIP_LLM' not in os.environ try: if len(sys.argv) < 3: @@ -325,9 +325,9 @@ while line_num + 1 < len(man_input): # This makes the tests more portable if not show_line_numbers: - buf_start = -1 - - print(("{:<%d}{}{}\n" % rs).format(buf_start, stack_print, '\n'.join(buf))) + print(clean_ansi("{}{}\n".format(stack_print, '\n'.join(buf)))) + else: + print(("{:<%d}{}{}\n" % rs).format(buf_start, stack_print, '\n'.join(buf))) # if this was the line we decided to reset everything on then we should actually process # it again because it could ALSO be the first line of a new match. From 40b325cd0ae44dbcf827fb6ef0bb87720ba6eff1 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Thu, 17 Jul 2025 05:42:56 -0700 Subject: [PATCH 124/138] more llm support --- mansnip | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mansnip b/mansnip index a150482..c4c9d05 100755 --- a/mansnip +++ b/mansnip @@ -32,6 +32,9 @@ try: man_input = os.popen(cmd).read() + if not show_line_numbers: + print(f"Excerpts from man {sys.argv[1]}:") + except Exception as ex: from textwrap import dedent print(dedent("""\ @@ -136,6 +139,7 @@ def random_word_test(line): pass line_num = -1 + while line_num + 1 < len(man_input): line_num += 1 From 960d03b34ac4b81d3ad5dbed6fa43fd0670f7135 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Thu, 17 Jul 2025 07:30:46 -0700 Subject: [PATCH 125/138] more llm stuff --- mansnip | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/mansnip b/mansnip index c4c9d05..c8f4960 100755 --- a/mansnip +++ b/mansnip @@ -49,6 +49,11 @@ except Exception as ex: $ mansnip tar -x -z -v -f + But wait! There's more! We are in the world of LLM so you can do + MANSNIP_LLM=1 mansnip tar -x -z -v -f + + and then get a snapshot that you can just dump into your context window + Have fun.""")) logging.info(ex) sys.exit(-1) @@ -321,7 +326,10 @@ while line_num + 1 < len(man_input): if len(stack_small) > 1: stack_print = stack_small[0] if len(stack_small) > 2: - stack_print += spacer + spacer.join(stack_small[1:-1]) + if not show_line_numbers: + stack_print += '#' * spacer + else: + stack_print += spacer + spacer.join(stack_small[1:-1]) stack_print += "\n" else: stack_print = '' From 3646c9f1b51806ea551cc03c038e28148f3c5e8d Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Fri, 18 Jul 2025 04:17:07 -0700 Subject: [PATCH 126/138] adding better llm formatting --- mansnip | 46 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/mansnip b/mansnip index c8f4960..48dc1c6 100755 --- a/mansnip +++ b/mansnip @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -import sys, re, os, logging, pdb +import sys, re, os, logging, pdb, math, subprocess # less and others have a number of things like this # @@ -15,7 +15,7 @@ import sys, re, os, logging, pdb # logging.basicConfig(level=(os.environ.get('LOGLEVEL') or 'critical').upper()) -show_line_numbers = 'MANSNIP_LLM' not in os.environ +llm = 'MANSNIP_LLM' in os.environ try: if len(sys.argv) < 3: @@ -27,12 +27,23 @@ try: # Some people use cygwin # You may gladly throw me in python prison for this line... - cmd = ' '.join(['/usr/bin/man' if os.path.exists('/usr/bin') else 'man'] + sys.argv[1:cutoff]) + cmd = ['/usr/bin/man' if os.path.exists('/usr/bin') else 'man'] + sys.argv[1:cutoff] logging.info(cmd) - man_input = os.popen(cmd).read() + env = os.environ.copy() + if llm: + env["MANWIDTH"] = "8192" - if not show_line_numbers: + proc = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + text=True + ) + man_input, stderr = proc.communicate() + + if llm: print(f"Excerpts from man {sys.argv[1]}:") except Exception as ex: @@ -275,7 +286,6 @@ while line_num + 1 < len(man_input): if not (re.match(getopts_simple_re, line) or re.match(getopts_simple_re, man_input_plain[line_num])): is_multiline_def = False - #print(is_multiline_def, '\n\t', "[{}]".format(line), '\n\t', "[{}]".format(man_input_plain[line_num]), '\n\t') # If our indent is zero and there's some text (subsection heading) if (indent == 0 and len(line)) or ( indent > 1 and ( @@ -326,17 +336,33 @@ while line_num + 1 < len(man_input): if len(stack_small) > 1: stack_print = stack_small[0] if len(stack_small) > 2: - if not show_line_numbers: - stack_print += '#' * spacer - else: + if not llm: stack_print += spacer + spacer.join(stack_small[1:-1]) stack_print += "\n" else: stack_print = '' buf[0] = re.sub(r'^\s{%d}' % rs, '', buf[0]) + if llm: + for x in range(len(buf)): + i = buf[x] + pre = '' + leading = len(i) - len(i.lstrip()) + #print("(" + i[:50]) + if leading < 2 and len(i) > 2: + pre = '# ' + elif leading < 8 and len(i) > 5: + pre = '## ' + else: + pre = ' ' * math.floor(leading/8) + buf[x] = pre + re.sub(r'\s+', ' ', i[leading:]) + + if len(stack_print) > 0: + stack_print = "# " + stack_print + + # This makes the tests more portable - if not show_line_numbers: + if llm: print(clean_ansi("{}{}\n".format(stack_print, '\n'.join(buf)))) else: print(("{:<%d}{}{}\n" % rs).format(buf_start, stack_print, '\n'.join(buf))) From 17dab1fe3fffe1c80a6cd056c8760673c74e151c Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Fri, 18 Jul 2025 04:35:18 -0700 Subject: [PATCH 127/138] version bump --- README.md | 21 +++++++++++++++++++++ setup.py | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e4cb876..295747d 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,24 @@ +### woahhh updates! + +So this handy tool has been updated for the llm era. You can mansnip into your context window by setting an environment variable like this: + +```bash +$ MANSNIP_LLM=1 mansnip ... +``` + +This will do a variety of things (try it yourself) that optimize for minimal token-length when using an llm. + +```bash +$ man bash | token-count +73392 +$ mansnip bash complete | token-count +2908 +$ MANSNIP_LLM=1 mansnip bash complete | token-count +1624 +``` + +That's a 98% reduction! Sweet. +

[![Video](https://9ol.es/vid.jpg)](http://www.youtube.com/watch?v=3GT1J-ejM3Q) diff --git a/setup.py b/setup.py index a146e8c..dc817b2 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setuptools.setup( name="mansnip-kristopolous", - version="0.4.0", + version="0.5.0", scripts=["mansnip"], author="Chris McKenzie", author_email="kristopolous@yahoo.com", From e31388fd98755ce557e61e96656b533c5c4495cd Mon Sep 17 00:00:00 2001 From: chris mckenzie Date: Fri, 18 Jul 2025 04:39:54 -0700 Subject: [PATCH 128/138] Update README.md --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 295747d..8fa247c 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -### woahhh updates! +## Woahhh updates for the AI era! So this handy tool has been updated for the llm era. You can mansnip into your context window by setting an environment variable like this: @@ -19,6 +19,8 @@ $ MANSNIP_LLM=1 mansnip bash complete | token-count That's a 98% reduction! Sweet. +## My classic 2020 pitch below! +

[![Video](https://9ol.es/vid.jpg)](http://www.youtube.com/watch?v=3GT1J-ejM3Q) From f8bd4bbc826d3dbb15fbae13e6919dbc329395ce Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Fri, 18 Jul 2025 05:05:00 -0700 Subject: [PATCH 129/138] updating readme --- LICENSE => LICENSE.MIT | 0 README.md | 9 +++++---- 2 files changed, 5 insertions(+), 4 deletions(-) rename LICENSE => LICENSE.MIT (100%) diff --git a/LICENSE b/LICENSE.MIT similarity index 100% rename from LICENSE rename to LICENSE.MIT diff --git a/README.md b/README.md index 8fa247c..f4a6b49 100644 --- a/README.md +++ b/README.md @@ -9,14 +9,15 @@ $ MANSNIP_LLM=1 mansnip ... This will do a variety of things (try it yourself) that optimize for minimal token-length when using an llm. ```bash -$ man bash | token-count +$ man bash | token-count # whole page 73392 -$ mansnip bash complete | token-count +$ man bash | grep -C 3 complete | token-count # naive approach with a bunch of garage input +8833 +$ mansnip bash complete | token-count # mansnip without llm feature 2908 -$ MANSNIP_LLM=1 mansnip bash complete | token-count +$ MANSNIP_LLM=1 mansnip bash complete | token-count # with new llm compaction! 1624 ``` - That's a 98% reduction! Sweet. ## My classic 2020 pitch below! From 3f8181bf26f86185503f5cde148463ee679c329c Mon Sep 17 00:00:00 2001 From: chris mckenzie Date: Fri, 18 Jul 2025 05:06:36 -0700 Subject: [PATCH 130/138] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f4a6b49..3a5989a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -## Woahhh updates for the AI era! +## Updates for the AI era! -So this handy tool has been updated for the llm era. You can mansnip into your context window by setting an environment variable like this: +You can now intelligently mansnip into your context window by setting an environment variable like this: ```bash $ MANSNIP_LLM=1 mansnip ... From 1048a4041b9510168f416b965172a746e550839e Mon Sep 17 00:00:00 2001 From: chris mckenzie Date: Fri, 18 Jul 2025 05:07:02 -0700 Subject: [PATCH 131/138] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3a5989a..857d8d1 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ This will do a variety of things (try it yourself) that optimize for minimal tok ```bash $ man bash | token-count # whole page 73392 -$ man bash | grep -C 3 complete | token-count # naive approach with a bunch of garage input +$ man bash | grep -C 3 complete | token-count # naive approach with a bunch of garbage input 8833 $ mansnip bash complete | token-count # mansnip without llm feature 2908 From 9a64369e4cba3fe7cb0e6a5e9c602cbefc42d8c6 Mon Sep 17 00:00:00 2001 From: chris mckenzie Date: Sat, 19 Jul 2025 11:57:20 -0700 Subject: [PATCH 132/138] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 857d8d1..e17b634 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,8 @@ $ MANSNIP_LLM=1 mansnip ... This will do a variety of things (try it yourself) that optimize for minimal token-length when using an llm. +Compare various approaches for finding the documentation for bash's complete command: + ```bash $ man bash | token-count # whole page 73392 From 2dcc3b0da5ffcf1bb58e8ff8c75b383861851cc6 Mon Sep 17 00:00:00 2001 From: chris mckenzie Date: Sat, 19 Jul 2025 11:58:57 -0700 Subject: [PATCH 133/138] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index e17b634..564d25b 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,8 @@ $ MANSNIP_LLM=1 mansnip bash complete | token-count # with new llm compaction! ``` That's a 98% reduction! Sweet. +Just `pip install mansnip-kristopolous` + ## My classic 2020 pitch below!

From 362429b5cb597062664e3c41de29658d599678cd Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Fri, 10 Oct 2025 14:25:36 -0700 Subject: [PATCH 134/138] reformatting things to make it mcp-able --- mansnip | 381 -------------------------------------------------- mansnip.py | 401 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 401 insertions(+), 381 deletions(-) delete mode 100755 mansnip create mode 100755 mansnip.py diff --git a/mansnip b/mansnip deleted file mode 100755 index 48dc1c6..0000000 --- a/mansnip +++ /dev/null @@ -1,381 +0,0 @@ -#!/usr/bin/env python3 -import sys, re, os, logging, pdb, math, subprocess - -# less and others have a number of things like this -# -# -hn or --max-back-scroll=n -# Specifies a maximum number of lines to scroll backward. If it -# is necessary to scroll backward more than n lines, the screen is -# repainted in a forward direction instead. (If the terminal does -# not have the ability to scroll backward, -h0 is implied.) -# -# Because of this the only *real* reliable system is to look at the formatting, -# which normally gets stripped so we have to be a bit more clever and be a full -# wrapper to man as opposed to just parsing -# - -logging.basicConfig(level=(os.environ.get('LOGLEVEL') or 'critical').upper()) -llm = 'MANSNIP_LLM' in os.environ - -try: - if len(sys.argv) < 3: - raise Exception("Not enough params") - - # This allows us to do mansnip or just - cutoff = 3 if sys.argv[1].isnumeric() else 2 - os.environ['MAN_KEEP_FORMATTING'] = '1' - - # Some people use cygwin - # You may gladly throw me in python prison for this line... - cmd = ['/usr/bin/man' if os.path.exists('/usr/bin') else 'man'] + sys.argv[1:cutoff] - logging.info(cmd) - - env = os.environ.copy() - if llm: - env["MANWIDTH"] = "8192" - - proc = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=env, - text=True - ) - man_input, stderr = proc.communicate() - - if llm: - print(f"Excerpts from man {sys.argv[1]}:") - -except Exception as ex: - from textwrap import dedent - print(dedent("""\ - ✂ mansnip usage ✂ - Mansnip works much like man does only it takes query strings after the - page you're looking to read. The syntax is generally: - - $ mansnip [ section ] page [ query0 query1 ... queryN ] - - For instance, if you want to find out what say, the xzv and f options - in tar do you can do - - $ mansnip tar -x -z -v -f - - But wait! There's more! We are in the world of LLM so you can do - MANSNIP_LLM=1 mansnip tar -x -z -v -f - - and then get a snapshot that you can just dump into your context window - - Have fun.""")) - logging.info(ex) - sys.exit(-1) - -pack = sys.argv[cutoff:] - -opts = '|'.join(pack) - -def matcher(term): - return ''.join([ - r'^\s*', - r'(', - # lsof uses +|- syntax, - r'(\+\||)', - - # the term itself - r'({})', - - # ffmpeg uses [ at times (see hwaccel), many things use ',' to show a second option. - # mmcli uses = a lot of the times - r'([\s_\[=].*|, .*|)', - r'|', - # this is for the long options, sometimes (git config) specified by commas - #'-.*\s({})', - r'-.*({})', - - # and kinda our same capture from above - r'([\s_\[=].*|)', - r')$' - ]).format(term, term) - -# the thing we are genuinely looking for -my_re = matcher(opts) - -# a general purpose getopts re -getopts_simple_re = r'^\s+(-{1,2}\w+\s?\w*)+\s*$' - -logging.info("The re for this search: {}".format(my_re)) - -is_def = False -line_def = False -multiline_def = False -stack_start = False - -# -# We want to be able to detect None versus 0 without the hassles -# of False == 0 (which I know can be avoided, sure, ok fine, but -# why introduce the possibility of bugs when you can easily avoid -# that possibility?) -# -term_indent = None - -buf_start = False -buf = [] - -indent_window = [] -stack_guess = [] -stack_indent = [] -last_stack_guess = [] - -# Words we can just leave out of the breadcrumb. -filler_terms = ['DESCRIPTION','OPTIONS'] - -clean_ansi = lambda w: re.sub(r'(.\x08|\x1B\[\d*m)', '', w) - -# -# There's lots of let's say "creative" ways to format man pages, so -# we have a plain-text version in case our sophisticated searching -# method fails. -# -man_input_plain = clean_ansi(man_input).split('\n') -man_input = man_input.split('\n') - -# -# We try to output something nice and readable. If the input is a -# lot of lines, say zshall (all of zsh), then we set aside a bit of -# space for the indentation. -# -rs = 5 if len(man_input) < 1e4 else 7 - -# -# Essentially we want to reduce our chances of false positives. -# Because there's so many variations we can't do a single-pass -# catch-all for everything. But what we can do is cast a somewhat -# wide net with the regexs and then do a second-pass rejection -# test with this function, essentially just looking for prose. -def random_word_test(line): - pass - -line_num = -1 - -while line_num + 1 < len(man_input): - line_num += 1 - - line = man_input[line_num].strip('\n') - - # - # Establish the "indent", this is crucial to how essentially - # everything works. - # - indent = re.match(r'^(\s*)', line).end() - indent_window = indent_window[-2:] + [indent] - - if len(line): - - # Our nice stack guessing system essentially keeps a stack - # of the indents and then based on the current indent either - # pushes or pops on to the stack. - - # - # This one is kinda tricky. If we are looking for -z in say, zshall - # - # zcompile [ -U ] [ -z | -k ] [ -R | -M ] file [ name ... ] - # zcompile -ca [ -m ] [ -R | -M ] file [ name ... ] - # zcompile -t file [ name ... ] - # -z - # - # Is what we want. So what we want is a longer version of what we currently have - # - # But it gets trickier since we suppress empty lines in this algorithm. - # - # r Same as fc -e -. - # - # read [ -rszpqAclneE ] [ -t [ num ] ] [ -k [ num ] ] [ -d delim ] - # [ -u n ] [ name[?prompt] ] [ name ... ] - # - # Also from zsh, will be interpreted as a group, so we have to just this once - # use our real indent tracker to catch it. - # - if len(stack_indent) and indent == stack_indent[-1] and indent_window[-2] != 0: - # - # Historically string growing like this was always dog, I don't know - # if it's still super slow. Mostly of the time we won't actually be using - # it. Since we aren't doing a stream, we could just reconstruct it by - # storing "pointers" (indexes here) ... sometime in the future. - # - stack_guess[-1] += '\n' + man_input[line_num] - - else: - while len(stack_indent) and indent <= stack_indent[-1]: - stack_indent.pop() - stack_guess.pop() - - if not len(stack_indent) or indent > stack_indent[-1]: - stack_indent.append(indent) - stack_guess.append(man_input[line_num].strip()) - - if not len(buf): - - # See if our ANSI escaped or plain text version has what we are looking for. - # yes that makes it a bit slower, but not as slow as doing it manually. - res = re.match(my_re, line) or re.match(my_re, man_input_plain[line_num]) - - if res: - logging.info("matched : {}".format(line)) - # - # This is sheer frantic handwaving for things like this (From bash) - # - # declare [-aAfFgilnrtux] [-p] [name[=value] ...] - # typeset [-aAfFgilnrtux] [-p] [name[=value] ...] - # - # Surely, if I search for "declare" this is what I want, but it - # it breaks our classic rules so instead we try a number of - # imperfet guesses. - # - # The first one is back-searching the indent margins. Generally - # there's a space before we see this and then some end of a - # previous block that was indented further. soooo yeah we - # look for that. - # - if indent_window[0] > indent and indent_window[1] == 0: - is_def = True - - # - # From man 7 man we get things like this: - # - # .I Italics - # - # .IB Italics alternating with bold - # - # .IR Italics alternating with Roman - # - elif indent_window[0] == indent and indent_window[1] == 0: - line_def = True - - buf.append(line) - buf_start = line_num - stack_start = stack_guess[:] - is_multiline_def = True - term_indent = indent - - elif term_indent != None: - logging.info("consider: {}".format(line)) - - # - # This tests if we have a format like in wget, something like - # - # -r - # --recursive - # Turn on recursive retrieving. The default maximum depth is 5. - # - # -l depth - # --level=depth - # Specify recursion maximum depth level depth. - # - # In this case we're ok with not increasing indentation as long as we match - # - # What is this boolean flippin' nonsense then? We allow essentially the following - # - # --A <- These three should be chained. - # --B - # --C - # blah blah <- Once we reach this indent further - # blah blah <- chaining should be prohibited - # - # --D <- This should be excluded. - # - if is_multiline_def: - if not (re.match(getopts_simple_re, line) or re.match(getopts_simple_re, man_input_plain[line_num])): - is_multiline_def = False - - # If our indent is zero and there's some text (subsection heading) - if (indent == 0 and len(line)) or ( - indent > 1 and ( - indent < term_indent or ( - # If we have the same indentation but we've determined it's a - # "multiline" definition. - indent == term_indent and not ( - is_def or is_multiline_def - ) - ) - )): - spacer = '\n' + ' ' * rs - logging.info("Current rows: {}".format( len(buf)) ) - - - if (len(buf) == 2 and indent == term_indent and line_def) or len(buf) > 1: - # - # This is a lot of fancy formatting. We want the - # vertical whitespace between the heading and snippet to be consistent. - # If we simply removed the empty lines then it would dump the interstitial - # (as in, in the middle of our content) empty lines. so we want to get rid - # of leading and trailing new lines. - # - # So we join the array, strip out the trailing new lines, then split it up - # again. eh, it's fine. - # - buf = '\n'.join(buf).strip('\n').split('\n') - - # - # If we show the same breadcrumb every time it gets kinda laborious and - # repetitive so we cleverly hide the redundancy. - # - stack_small = stack_start[:] - for i in range(0, min(len(last_stack_guess),len(stack_start))): - if last_stack_guess[i] == stack_start[i]: - stack_small = stack_small[1:] - - last_stack_guess = stack_start[:] - - if len(stack_small) > 1: - if clean_ansi(stack_small[0]) in filler_terms: - stack_small = stack_small[1:] - # - # Now we have to figure out how to print it out to make it look right. - # Sometimes it can be inline. The %/fmt double formatting trick comes - # in handle here. - # - if len(stack_small) > 1: - stack_print = stack_small[0] - if len(stack_small) > 2: - if not llm: - stack_print += spacer + spacer.join(stack_small[1:-1]) - stack_print += "\n" - else: - stack_print = '' - buf[0] = re.sub(r'^\s{%d}' % rs, '', buf[0]) - - if llm: - for x in range(len(buf)): - i = buf[x] - pre = '' - leading = len(i) - len(i.lstrip()) - #print("(" + i[:50]) - if leading < 2 and len(i) > 2: - pre = '# ' - elif leading < 8 and len(i) > 5: - pre = '## ' - else: - pre = ' ' * math.floor(leading/8) - buf[x] = pre + re.sub(r'\s+', ' ', i[leading:]) - - if len(stack_print) > 0: - stack_print = "# " + stack_print - - - # This makes the tests more portable - if llm: - print(clean_ansi("{}{}\n".format(stack_print, '\n'.join(buf)))) - else: - print(("{:<%d}{}{}\n" % rs).format(buf_start, stack_print, '\n'.join(buf))) - - # if this was the line we decided to reset everything on then we should actually process - # it again because it could ALSO be the first line of a new match. - line_num -= 1 - buf = [] - term_indent = None - is_def = False - else: - # Once our formatting goes in we have to reset it - if is_def and indent > term_indent: - is_def = False - - buf.append(line) diff --git a/mansnip.py b/mansnip.py new file mode 100755 index 0000000..2cf608a --- /dev/null +++ b/mansnip.py @@ -0,0 +1,401 @@ +#!/usr/bin/env python3 +import sys, re, os, logging, math, subprocess + +# less and others have a number of things like this +# +# -hn or --max-back-scroll=n +# Specifies a maximum number of lines to scroll backward. If it +# is necessary to scroll backward more than n lines, the screen is +# repainted in a forward direction instead. (If the terminal does +# not have the ability to scroll backward, -h0 is implied.) +# +# Because of this the only *real* reliable system is to look at the formatting, +# which normally gets stripped so we have to be a bit more clever and be a full +# wrapper to man as opposed to just parsing +# + +logging.basicConfig(level=(os.environ.get('LOGLEVEL') or 'critical').upper()) + +def mansnip(page: str, query: list, environ: dict): + buf = [] + llm = 'MANSNIP_LLM' in environ + + try: + # Some people use cygwin + # You may gladly throw me in python prison for this line... + cmd = ['/usr/bin/man' if os.path.exists('/usr/bin') else 'man'] + sys.argv[1:cutoff] + logging.info(cmd) + + if llm: + env["MANWIDTH"] = "8192" + + proc = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=environ, + text=True + ) + man_input, stderr = proc.communicate() + + if llm: + buf.append(f"Excerpts from man {page}:") + + except Exception as ex: + from textwrap import dedent + print(dedent("""\ + ✂ mansnip usage ✂ + Mansnip works much like man does only it takes query strings after the + page you're looking to read. The syntax is generally: + + $ mansnip [ section ] page [ query0 query1 ... queryN ] + + For instance, if you want to find out what say, the xzv and f options + in tar do you can do + + $ mansnip tar -x -z -v -f + + But wait! There's more! We are in the world of LLM so you can do + MANSNIP_LLM=1 mansnip tar -x -z -v -f + + and then get a snapshot that you can just dump into your context window + + Have fun.""")) + logging.info(ex) + sys.exit(-1) + + opts = '|'.join(query) + + def matcher(term): + return ''.join([ + r'^\s*', + r'(', + # lsof uses +|- syntax, + r'(\+\||)', + + # the term itself + r'({})', + + # ffmpeg uses [ at times (see hwaccel), many things use ',' to show a second option. + # mmcli uses = a lot of the times + r'([\s_\[=].*|, .*|)', + r'|', + # this is for the long options, sometimes (git config) specified by commas + #'-.*\s({})', + r'-.*({})', + + # and kinda our same capture from above + r'([\s_\[=].*|)', + r')$' + ]).format(term, term) + + # the thing we are genuinely looking for + my_re = matcher(opts) + + # a general purpose getopts re + getopts_simple_re = r'^\s+(-{1,2}\w+\s?\w*)+\s*$' + + logging.info("The re for this search: {}".format(my_re)) + + is_def = False + line_def = False + multiline_def = False + stack_start = False + + # + # We want to be able to detect None versus 0 without the hassles + # of False == 0 (which I know can be avoided, sure, ok fine, but + # why introduce the possibility of bugs when you can easily avoid + # that possibility?) + # + term_indent = None + + buf_start = False + buf = [] + + indent_window = [] + stack_guess = [] + stack_indent = [] + last_stack_guess = [] + + # Words we can just leave out of the breadcrumb. + filler_terms = ['DESCRIPTION','OPTIONS'] + + clean_ansi = lambda w: re.sub(r'(.\x08|\x1B\[\d*m)', '', w) + + # + # There's lots of let's say "creative" ways to format man pages, so + # we have a plain-text version in case our sophisticated searching + # method fails. + # + man_input_plain = clean_ansi(man_input).split('\n') + man_input = man_input.split('\n') + + # + # We try to output something nice and readable. If the input is a + # lot of lines, say zshall (all of zsh), then we set aside a bit of + # space for the indentation. + # + rs = 5 if len(man_input) < 1e4 else 7 + + # + # Essentially we want to reduce our chances of false positives. + # Because there's so many variations we can't do a single-pass + # catch-all for everything. But what we can do is cast a somewhat + # wide net with the regexs and then do a second-pass rejection + # test with this function, essentially just looking for prose. + def random_word_test(line): + pass + + line_num = -1 + + while line_num + 1 < len(man_input): + line_num += 1 + + line = man_input[line_num].strip('\n') + + # + # Establish the "indent", this is crucial to how essentially + # everything works. + # + indent = re.match(r'^(\s*)', line).end() + indent_window = indent_window[-2:] + [indent] + + if len(line): + + # Our nice stack guessing system essentially keeps a stack + # of the indents and then based on the current indent either + # pushes or pops on to the stack. + + # + # This one is kinda tricky. If we are looking for -z in say, zshall + # + # zcompile [ -U ] [ -z | -k ] [ -R | -M ] file [ name ... ] + # zcompile -ca [ -m ] [ -R | -M ] file [ name ... ] + # zcompile -t file [ name ... ] + # -z + # + # Is what we want. So what we want is a longer version of what we currently have + # + # But it gets trickier since we suppress empty lines in this algorithm. + # + # r Same as fc -e -. + # + # read [ -rszpqAclneE ] [ -t [ num ] ] [ -k [ num ] ] [ -d delim ] + # [ -u n ] [ name[?prompt] ] [ name ... ] + # + # Also from zsh, will be interpreted as a group, so we have to just this once + # use our real indent tracker to catch it. + # + if len(stack_indent) and indent == stack_indent[-1] and indent_window[-2] != 0: + # + # Historically string growing like this was always dog, I don't know + # if it's still super slow. Mostly of the time we won't actually be using + # it. Since we aren't doing a stream, we could just reconstruct it by + # storing "pointers" (indexes here) ... sometime in the future. + # + stack_guess[-1] += '\n' + man_input[line_num] + + else: + while len(stack_indent) and indent <= stack_indent[-1]: + stack_indent.pop() + stack_guess.pop() + + if not len(stack_indent) or indent > stack_indent[-1]: + stack_indent.append(indent) + stack_guess.append(man_input[line_num].strip()) + + if not len(buf): + + # See if our ANSI escaped or plain text version has what we are looking for. + # yes that makes it a bit slower, but not as slow as doing it manually. + res = re.match(my_re, line) or re.match(my_re, man_input_plain[line_num]) + + if res: + logging.info("matched : {}".format(line)) + # + # This is sheer frantic handwaving for things like this (From bash) + # + # declare [-aAfFgilnrtux] [-p] [name[=value] ...] + # typeset [-aAfFgilnrtux] [-p] [name[=value] ...] + # + # Surely, if I search for "declare" this is what I want, but it + # it breaks our classic rules so instead we try a number of + # imperfet guesses. + # + # The first one is back-searching the indent margins. Generally + # there's a space before we see this and then some end of a + # previous block that was indented further. soooo yeah we + # look for that. + # + if indent_window[0] > indent and indent_window[1] == 0: + is_def = True + + # + # From man 7 man we get things like this: + # + # .I Italics + # + # .IB Italics alternating with bold + # + # .IR Italics alternating with Roman + # + elif indent_window[0] == indent and indent_window[1] == 0: + line_def = True + + buf.append(line) + buf_start = line_num + stack_start = stack_guess[:] + is_multiline_def = True + term_indent = indent + + elif term_indent != None: + logging.info("consider: {}".format(line)) + + # + # This tests if we have a format like in wget, something like + # + # -r + # --recursive + # Turn on recursive retrieving. The default maximum depth is 5. + # + # -l depth + # --level=depth + # Specify recursion maximum depth level depth. + # + # In this case we're ok with not increasing indentation as long as we match + # + # What is this boolean flippin' nonsense then? We allow essentially the following + # + # --A <- These three should be chained. + # --B + # --C + # blah blah <- Once we reach this indent further + # blah blah <- chaining should be prohibited + # + # --D <- This should be excluded. + # + if is_multiline_def: + if not (re.match(getopts_simple_re, line) or re.match(getopts_simple_re, man_input_plain[line_num])): + is_multiline_def = False + + # If our indent is zero and there's some text (subsection heading) + if (indent == 0 and len(line)) or ( + indent > 1 and ( + indent < term_indent or ( + # If we have the same indentation but we've determined it's a + # "multiline" definition. + indent == term_indent and not ( + is_def or is_multiline_def + ) + ) + )): + spacer = '\n' + ' ' * rs + logging.info("Current rows: {}".format( len(buf)) ) + + + if (len(buf) == 2 and indent == term_indent and line_def) or len(buf) > 1: + # + # This is a lot of fancy formatting. We want the + # vertical whitespace between the heading and snippet to be consistent. + # If we simply removed the empty lines then it would dump the interstitial + # (as in, in the middle of our content) empty lines. so we want to get rid + # of leading and trailing new lines. + # + # So we join the array, strip out the trailing new lines, then split it up + # again. eh, it's fine. + # + buf = '\n'.join(buf).strip('\n').split('\n') + + # + # If we show the same breadcrumb every time it gets kinda laborious and + # repetitive so we cleverly hide the redundancy. + # + stack_small = stack_start[:] + for i in range(0, min(len(last_stack_guess),len(stack_start))): + if last_stack_guess[i] == stack_start[i]: + stack_small = stack_small[1:] + + last_stack_guess = stack_start[:] + + if len(stack_small) > 1: + if clean_ansi(stack_small[0]) in filler_terms: + stack_small = stack_small[1:] + # + # Now we have to figure out how to print it out to make it look right. + # Sometimes it can be inline. The %/fmt double formatting trick comes + # in handle here. + # + if len(stack_small) > 1: + stack_print = stack_small[0] + if len(stack_small) > 2: + if not llm: + stack_print += spacer + spacer.join(stack_small[1:-1]) + stack_print += "\n" + else: + stack_print = '' + buf[0] = re.sub(r'^\s{%d}' % rs, '', buf[0]) + + if llm: + for x in range(len(buf)): + i = buf[x] + pre = '' + leading = len(i) - len(i.lstrip()) + + if leading < 2 and len(i) > 2: + pre = '# ' + elif leading < 8 and len(i) > 5: + pre = '## ' + else: + pre = ' ' * math.floor(leading/8) + buf[x] = pre + re.sub(r'\s+', ' ', i[leading:]) + + if len(stack_print) > 0: + stack_print = "# " + stack_print + + + # This makes the tests more portable + if llm: + print(clean_ansi("{}{}\n".format(stack_print, '\n'.join(buf)))) + else: + print(("{:<%d}{}{}\n" % rs).format(buf_start, stack_print, '\n'.join(buf))) + + # if this was the line we decided to reset everything on then we should actually process + # it again because it could ALSO be the first line of a new match. + line_num -= 1 + buf = [] + term_indent = None + is_def = False + else: + # Once our formatting goes in we have to reset it + if is_def and indent > term_indent: + is_def = False + + buf.append(line) + +if __name__ == '__main__': + logging.basicConfig(level=(os.environ.get('LOGLEVEL') or 'critical').upper()) + llm = 'MANSNIP_LLM' in os.environ + + if len(sys.argv) < 3: + raise Exception("Not enough params") + + local_env = os.environ.copy() + + # This allows us to do mansnip or just + cutoff = 3 if sys.argv[1].isnumeric() else 2 + local_env['MAN_KEEP_FORMATTING'] = '1' + + # Some people use cygwin + # You may gladly throw me in python prison for this line... + cmd = ['/usr/bin/man' if os.path.exists('/usr/bin') else 'man'] + sys.argv[1:cutoff] + logging.info(cmd) + + env = os.environ.copy() + if llm: + local_env["MANWIDTH"] = "8192" + + page = sys.argv[1:cutoff] + query = sys.argv[cutoff:] + mansnip(page, query, env) + From 000c08230d6ae199958c9dcfdc434c68bdb48ce4 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Fri, 10 Oct 2025 15:05:58 -0700 Subject: [PATCH 135/138] updates --- .envrc | 1 + .gitignore | 2 ++ mansnip.py | 4 ++-- mcp-server.py | 59 ++++++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 1 + 5 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 .envrc create mode 100644 mcp-server.py create mode 100644 requirements.txt diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..175de89 --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +layout python diff --git a/.gitignore b/.gitignore index 9adf187..dca8826 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ build dist mansnip_kristopolous.egg-info pyupload +__pycache__ +.direnv diff --git a/mansnip.py b/mansnip.py index 2cf608a..9c52f2d 100755 --- a/mansnip.py +++ b/mansnip.py @@ -14,9 +14,9 @@ # wrapper to man as opposed to just parsing # -logging.basicConfig(level=(os.environ.get('LOGLEVEL') or 'critical').upper()) - def mansnip(page: str, query: list, environ: dict): + logging.basicConfig(level=(environ.get('LOGLEVEL') or 'critical').upper()) + buf = [] llm = 'MANSNIP_LLM' in environ diff --git a/mcp-server.py b/mcp-server.py new file mode 100644 index 0000000..6340538 --- /dev/null +++ b/mcp-server.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 + +import asyncio +import json +import sys +from mcp.server import Server +from mcp.server.stdio import stdio_server +from mcp.types import Tool, TextContent +import mansnip +page='bash' +query='complete' +print(mansnip.mansnip(page, query, {'LOGLEVEL': 'debug', 'MANSNIP_LLM': 1})) +sys.exit(0) + +app = Server("manpage-query") + +@app.list_tools() +async def list_tools() -> list[Tool]: + return [ + Tool( + name="manpage_query", + description="return a contextual snippet from a manpage", + inputSchema={ + "type": "object", + "properties": { + "manpage": { + "type": "string", + "description": "man page to retrieve" + }, + "query": { + "type": "string", + "description": "term to look up" + } + }, + "required": ["manpage", "query"] + } + ) + ] + +@app.call_tool() +async def call_tool(name: str, arguments: dict) -> list[TextContent]: + if name == "manpage_query": + page = arguments.get("manpage") + query = arguments.get("query") + res = mansnip.mansnip(page, query, {'MANSNIP_LLM': 1}) + return [TextContent(type="text", text=res)] + + raise ValueError(f"Unknown tool: {name}") + +async def main(): + async with stdio_server() as (read_stream, write_stream): + await app.run( + read_stream, + write_stream, + app.create_initialization_options() + ) + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..6664c6d --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +mcp From 5c6351e285f3cf42d5cf42e04f026f5448b66488 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Fri, 10 Oct 2025 18:04:14 -0700 Subject: [PATCH 136/138] cleanup --- mansnip.py | 44 ++++++++++++++++++++------------------------ mcp-server.py | 10 +++++++--- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/mansnip.py b/mansnip.py index 9c52f2d..26c1707 100755 --- a/mansnip.py +++ b/mansnip.py @@ -14,20 +14,22 @@ # wrapper to man as opposed to just parsing # -def mansnip(page: str, query: list, environ: dict): +def mansnip(section: str, page: str, query: list, environ: dict): logging.basicConfig(level=(environ.get('LOGLEVEL') or 'critical').upper()) buf = [] llm = 'MANSNIP_LLM' in environ + res_buf = [] + environ['MAN_KEEP_FORMATTING'] = '1' try: # Some people use cygwin # You may gladly throw me in python prison for this line... - cmd = ['/usr/bin/man' if os.path.exists('/usr/bin') else 'man'] + sys.argv[1:cutoff] + cmd = ['/usr/bin/man' if os.path.exists('/usr/bin') else 'man'] + list(filter(None, [section, page])) logging.info(cmd) if llm: - env["MANWIDTH"] = "8192" + environ["MANWIDTH"] = "8192" proc = subprocess.Popen( cmd, @@ -61,7 +63,7 @@ def mansnip(page: str, query: list, environ: dict): and then get a snapshot that you can just dump into your context window Have fun.""")) - logging.info(ex) + logging.info("unhandled", exc_info=True) sys.exit(-1) opts = '|'.join(query) @@ -70,7 +72,7 @@ def matcher(term): return ''.join([ r'^\s*', r'(', - # lsof uses +|- syntax, + # lsof uses +|- syntax, r'(\+\||)', # the term itself @@ -353,12 +355,11 @@ def random_word_test(line): if len(stack_print) > 0: stack_print = "# " + stack_print - # This makes the tests more portable if llm: - print(clean_ansi("{}{}\n".format(stack_print, '\n'.join(buf)))) + res_buf.append(clean_ansi("{}{}\n".format(stack_print, '\n'.join(buf)))) else: - print(("{:<%d}{}{}\n" % rs).format(buf_start, stack_print, '\n'.join(buf))) + res_buf.append(("{:<%d}{}{}\n" % rs).format(buf_start, stack_print, '\n'.join(buf))) # if this was the line we decided to reset everything on then we should actually process # it again because it could ALSO be the first line of a new match. @@ -373,29 +374,24 @@ def random_word_test(line): buf.append(line) + return "\n".join(res_buf) + if __name__ == '__main__': logging.basicConfig(level=(os.environ.get('LOGLEVEL') or 'critical').upper()) - llm = 'MANSNIP_LLM' in os.environ if len(sys.argv) < 3: raise Exception("Not enough params") - local_env = os.environ.copy() - # This allows us to do mansnip or just - cutoff = 3 if sys.argv[1].isnumeric() else 2 - local_env['MAN_KEEP_FORMATTING'] = '1' - - # Some people use cygwin - # You may gladly throw me in python prison for this line... - cmd = ['/usr/bin/man' if os.path.exists('/usr/bin') else 'man'] + sys.argv[1:cutoff] - logging.info(cmd) + if sys.argv[1].isnumeric(): + section = sys.argv[1] + page = sys.argv[2] + query = sys.argv[3:] + else: + section = '' + page = sys.argv[1] + query = sys.argv[1:] env = os.environ.copy() - if llm: - local_env["MANWIDTH"] = "8192" - - page = sys.argv[1:cutoff] - query = sys.argv[cutoff:] - mansnip(page, query, env) + print(mansnip(section, page, query, env)) diff --git a/mcp-server.py b/mcp-server.py index 6340538..c581f50 100644 --- a/mcp-server.py +++ b/mcp-server.py @@ -9,7 +9,7 @@ import mansnip page='bash' query='complete' -print(mansnip.mansnip(page, query, {'LOGLEVEL': 'debug', 'MANSNIP_LLM': 1})) +print(mansnip.mansnip("", page, query, {'LOGLEVEL': 'debug', 'MANSNIP_LLM': 1})) sys.exit(0) app = Server("manpage-query") @@ -23,6 +23,10 @@ async def list_tools() -> list[Tool]: inputSchema={ "type": "object", "properties": { + "section": { + "type": "string", + "description": "man page section if needed for disambiguation" + }, "manpage": { "type": "string", "description": "man page to retrieve" @@ -38,11 +42,11 @@ async def list_tools() -> list[Tool]: ] @app.call_tool() -async def call_tool(name: str, arguments: dict) -> list[TextContent]: +async def call_tool(section: str, name: str, arguments: dict) -> list[TextContent]: if name == "manpage_query": page = arguments.get("manpage") query = arguments.get("query") - res = mansnip.mansnip(page, query, {'MANSNIP_LLM': 1}) + res = mansnip.mansnip(section, page, query, {'MANSNIP_LLM': 1}) return [TextContent(type="text", text=res)] raise ValueError(f"Unknown tool: {name}") From 2c968343fd91641cff778acb8fca8f9e404effbb Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Fri, 10 Oct 2025 18:41:00 -0700 Subject: [PATCH 137/138] mcp work --- mansnip.py | 7 +++++-- mcp-server.py | 7 ++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/mansnip.py b/mansnip.py index 26c1707..00b99eb 100755 --- a/mansnip.py +++ b/mansnip.py @@ -21,6 +21,9 @@ def mansnip(section: str, page: str, query: list, environ: dict): llm = 'MANSNIP_LLM' in environ res_buf = [] + for n in environ.keys(): + environ[n] = str(environ[n]) + environ['MAN_KEEP_FORMATTING'] = '1' try: # Some people use cygwin @@ -63,7 +66,7 @@ def mansnip(section: str, page: str, query: list, environ: dict): and then get a snapshot that you can just dump into your context window Have fun.""")) - logging.info("unhandled", exc_info=True) + logging.error("unhandled", exc_info=True) sys.exit(-1) opts = '|'.join(query) @@ -377,7 +380,7 @@ def random_word_test(line): return "\n".join(res_buf) if __name__ == '__main__': - logging.basicConfig(level=(os.environ.get('LOGLEVEL') or 'critical').upper()) + logging.basicConfig(level=(os.environ.get('LOGLEVEL') or 'warning').upper()) if len(sys.argv) < 3: raise Exception("Not enough params") diff --git a/mcp-server.py b/mcp-server.py index c581f50..280a6ce 100644 --- a/mcp-server.py +++ b/mcp-server.py @@ -7,10 +7,6 @@ from mcp.server.stdio import stdio_server from mcp.types import Tool, TextContent import mansnip -page='bash' -query='complete' -print(mansnip.mansnip("", page, query, {'LOGLEVEL': 'debug', 'MANSNIP_LLM': 1})) -sys.exit(0) app = Server("manpage-query") @@ -42,8 +38,9 @@ async def list_tools() -> list[Tool]: ] @app.call_tool() -async def call_tool(section: str, name: str, arguments: dict) -> list[TextContent]: +async def call_tool(name: str, arguments: dict) -> list[TextContent]: if name == "manpage_query": + section = arguments.get("section") or "" page = arguments.get("manpage") query = arguments.get("query") res = mansnip.mansnip(section, page, query, {'MANSNIP_LLM': 1}) From 5a4784738377a20e86cb673db5eaa525a5c9fed4 Mon Sep 17 00:00:00 2001 From: Chris McKenzie Date: Fri, 10 Oct 2025 18:42:07 -0700 Subject: [PATCH 138/138] adding the mcp server announcement --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 564d25b..9f82819 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,8 @@ $ MANSNIP_LLM=1 mansnip bash complete | token-count # with new llm compaction! ``` That's a 98% reduction! Sweet. +There's also a ready-to-go MCP server for it in `mcp-server.py`. + Just `pip install mansnip-kristopolous` ## My classic 2020 pitch below!