From d2f9e16ce75f4baace51e152477cbcacb1de8b6a Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Thu, 25 Mar 2021 16:07:20 -0400 Subject: [PATCH 01/56] Commit more Python versions of scripts. WIP --- add_copyright.py | 34 ++++++++++++++ mkutable.py | 113 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+) create mode 100644 add_copyright.py create mode 100644 mkutable.py diff --git a/add_copyright.py b/add_copyright.py new file mode 100644 index 00000000..f61156a8 --- /dev/null +++ b/add_copyright.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 + +from sys import argv, open +from os.path import isadir +import io +import re + +dir = pop(argv) +if (!isadir(dir)): + print(dir + "is not a directory\n") + exit(1) + +copyright = open("copyright", "r") +copyright_line = copyright.read(2) +copyright.close() + +for i in argv: + out = dir + "/" + i + if (!open(i000)): + print("Cannot open {0}\n", i) + pass + if (!open(out)): + print("Cannot create {0}\n", out) + close(i) + pass + while(input): + + close(copyright) + if (exec): + mode = 0555 + else: + mode = 0444 + chmod(out. mode) + diff --git a/mkutable.py b/mkutable.py new file mode 100644 index 00000000..cb1cf14b --- /dev/null +++ b/mkutable.py @@ -0,0 +1,113 @@ +#! /usr/bin/env python + +from argparse import ArgumentParser +from sys import exit +from subprocess import run +from fileinput import input +import re + +# Override Unicode tables for certain control chars +# that are expected to be found in normal text files. +force_space = { + 0x08: 1, # backspace + 0x09: 1, # tab + 0x0a: 1, # newline + 0x0c: 1, # form feed + 0x0d: 1, # carriage return +} + +# Hangul Jamo medial vowels and final consonants should be zero width. +force_compose = [ + [0x1160, 0x11ff], + [0xd7b0, 0xd7c6], + [0xd7cb, 0xd7fb] +] + + +def main() -> int: + parser = ArgumentParser() + parser.add_argument("-n", help="Take non-matching types", + action=store_true) + parser.add_argument("-f", help="Zero-based type field (default 2)", + type=int, default=2) + parser.add_argument("type", nargs='+', type=string, required=True) + # parser.add_argument("--") + # parser.add_argument("<") + parser.add_argument("UnicodeData.txt", required=True) + args = parser.parse_args() + global opt_n + opt_n = args.n + global type_field + type_field = args.f + + types = {} + for arg in sys.argv: + if arg == "--": + break + types[arg] = 1 + + # while ($arg = shift @ARGV) { + # last if $arg eq '--'; + # $types{$arg} = 1; + # } + # out = { 'types': types } + + force_compose = {} + for lo, hi in force_compose: + force_compose = [1 for ch in range(lo, hi)] + + # Only works for Python 3.7+ + date = run("date", capture_output=True, text=True).rstrip('\n') + print(f"/* Generated by \"{sys.argv[0]} {sys.argv[1:]}\" on {date}.") + print(" Python FTW! */") + + last_code = 0 + with open(UnicodeData.txt) as file: + for line in file: + line.rstrip('\n') + line = re.sub(r"s/#.*//", line) + fields = line.split(';') + if not fields: + continue + lo_code = hi_code = 0 + codes = fields[0] + if m := re.match(r"(\w+)\.\.(\w+)", codes): + lo_code = int(m.group(1), 16) + hi_code = int(m.group(2), 16) + else: + lo_code = hi_code = int(codes, 16) + type = fields[type_field] + type = re.match(r"s/\s//g", type) + for last_code in range(lo_code, hi_code): + output(out, last_code, 'Zs' if force_space[last_code] else 'Mn' + if force_compose[last_code] else type) + output(out, last_code) + return 1 + +def output(out: dict, code: int, type: str): + type_ok = (type and ${${$out}{types}}{$type}) + type_ok = not type_ok if opt_n + prev_code = out{prev_code} + + if not type_ok: + end_run(out, prev_code) + elif (not out{in_run} or type ne out{run_type} or code != prev_code+1): + end_run(out, prev_code) + start_run(out, code, type) + out{prev_code} = code + +def start_run(out: dict, code: int, type: str): + out[start_code] = code + out[prev_code] = code + out[run_type] = type + out[in_run] = 1 + +def end_run(out: dict, code: int): + if not out[in_run]: + return + print("\t{ 0x%04x, 0x%04x }, /* %s */" % $$out{start_code}, code, + out{run_type}) + out[in_run] = 0 + +if __name__ == "__main__": + exit(0 if main() else 1) From 8a0c6ab5de5f88c3f12505d879bdb062fcde7a8a Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Fri, 26 Mar 2021 15:42:33 -0400 Subject: [PATCH 02/56] Add Python version of add_copyright, and adjust Makefile.aut accordingly. --- Makefile.aut | 4 +++- add_copyright.py | 52 ++++++++++++++++++++++-------------------------- 2 files changed, 27 insertions(+), 29 deletions(-) diff --git a/Makefile.aut b/Makefile.aut index b6f2101c..6aafb67b 100644 --- a/Makefile.aut +++ b/Makefile.aut @@ -11,9 +11,11 @@ srcdir = . ifeq ($(USE_PYTHON),1) MKHELP = mkhelp.py MKFUNCS = mkfuncs.py + ADD_COPYRIGHT = add_copyright.py else MKHELP = mkhelp.pl MKFUNCS = mkfuncs.pl + ADD_COPYRIGHT = add_copyright endif SRC = \ @@ -135,7 +137,7 @@ dist: ${DISTFILES} echo "Preparing $$REL"; \ rm -rf $$REL; mkdir $$REL; \ for file in ${DISTFILES}; do \ - ./add_copyright $$file $$REL; \ + ${srcdir}/${ADD_COPYRIGHT} $$file $$REL; \ done; \ cd $$REL; chmod -w *; chmod +w ${DISTFILES_W}; chmod +x configure; cd ..; \ echo "Creating release/$$REL/$$REL.tar.gz"; \ diff --git a/add_copyright.py b/add_copyright.py index f61156a8..bc8bc618 100644 --- a/add_copyright.py +++ b/add_copyright.py @@ -1,34 +1,30 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python -from sys import argv, open -from os.path import isadir -import io +import os +import argparse +import pathlib import re -dir = pop(argv) -if (!isadir(dir)): - print(dir + "is not a directory\n") - exit(1) +parser = argparse.ArgumentParser() +parser.add_argument("file", type=str) +parser.add_argument("dir", type=str) +args = parser.parse_args() +out = pathlib.Path(args.dir) / args.file -copyright = open("copyright", "r") -copyright_line = copyright.read(2) -copyright.close() +copyright = copyright_line = "" +with open("copyright") as f: + copyright = f.read() +for line in copyright.splitlines(): + if '(C)' in line: + copyright_line = line.strip("/*/\r\n").rstrip(" ") + break -for i in argv: - out = dir + "/" + i - if (!open(i000)): - print("Cannot open {0}\n", i) - pass - if (!open(out)): - print("Cannot create {0}\n", out) - close(i) - pass - while(input): +with open(args.file) as input, out.open('w') as o: + for line in input: + if re.search(r"\@\@copyright\@\@", line): + o.write(copyright) + continue + out_line = re.sub(r"\@\@copyright_oneline\@\@", copyright_line, line) + o.write(out_line) - close(copyright) - if (exec): - mode = 0555 - else: - mode = 0444 - chmod(out. mode) - + out.chmod(0o555 if os.access(out, os.X_OK) else 0o444) From 44e99733c129ff05059f21302cc01585a29fa834 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Sat, 27 Mar 2021 15:46:43 -0400 Subject: [PATCH 03/56] Finalize conversion of Perl build scripts into Python versions. --- mkfuncs.py | 34 ++++++++++++----------- mkhelp.py | 18 ++++++------ mkutable.py | 80 ++++++++++++++++++++++++++--------------------------- 3 files changed, 68 insertions(+), 64 deletions(-) diff --git a/mkfuncs.py b/mkfuncs.py index 7a2da5ad..c397942f 100755 --- a/mkfuncs.py +++ b/mkfuncs.py @@ -8,19 +8,21 @@ params = 0 for line in fileinput.input(): - if test := re.search(r'^\tpublic\s+(.*)', line): - definition = 'public ' + test.group(1) - state = 1 - params = 0 - elif (state == 1) and (test := re.search(r'(\w+)\s*\(', line)): - definition = '{0} LESSPARAMS (('.format(test.group(1)) - state = 2 - elif state == 2: - if re.search(r'^{', line): - if not params: definition += 'VOID_PARAM' - print(f'{definition}));') - state = 0 - elif test := re.search(r'^\s*([^;]*)', line): - if (definition[-1:] != '('): definition += ', ' - definition += test.group(1) - params = 1 + if test := re.search(r'^\tpublic\s+(.*)', line): + definition = 'public ' + test.group(1) + state = 1 + params = 0 + elif (state == 1) and (test := re.search(r'(\w+)\s*\(', line)): + definition = '{0} LESSPARAMS (('.format(test.group(1)) + state = 2 + elif state == 2: + if re.search(r'^{', line): + if not params: + definition += 'VOID_PARAM' + print(f'{definition}));') + state = 0 + elif test := re.search(r'^\s*([^;]*)', line): + if (definition[-1:] != '('): + definition += ', ' + definition += test.group(1) + params = 1 diff --git a/mkhelp.py b/mkhelp.py index 029f1505..4619457f 100755 --- a/mkhelp.py +++ b/mkhelp.py @@ -4,9 +4,9 @@ import sys time = time.gmtime() -print("/* This file was generated by mkhelp.py from less.hlp at "\ - "%d:%02d GMT on %d/%d/%d */\n" % - (time.tm_hour, time.tm_min, time.tm_year, time.tm_mon, time.tm_mday)) +print("/* This file was generated by mkhelp.py from less.hlp at " + "%d:%02d GMT on %d/%d/%d */\n" % + (time.tm_hour, time.tm_min, time.tm_year, time.tm_mon, time.tm_mday)) print("#include \"less.h\"") print("constant char helpdata[] = {") ch = 0 @@ -16,17 +16,19 @@ if ch == '': break if (ch == "'"): - print("'\\'',", end='') + print("'\\'',", end='') elif (ch == "\\"): print("'\\\\',", end='') elif (ch == "\b"): - print ("'\\b',", end='') + print("'\\b',", end='') elif (ch == "\t"): - print ("'\\t',", end='') + print("'\\t',", end='') elif (ch == "\n"): - if prevch != "\r": print("'\\n',") + if prevch != "\r": + print("'\\n',") elif (ch == "\r"): - if prevch != "\n": print("'\\n',") + if prevch != "\n": + print("'\\n',") else: if ((ord(ch) >= ord(' ')) and (ord(ch) < 0x7f)): print(f"'{ch}',", end='') diff --git a/mkutable.py b/mkutable.py index cb1cf14b..fd76408d 100644 --- a/mkutable.py +++ b/mkutable.py @@ -1,9 +1,8 @@ #! /usr/bin/env python from argparse import ArgumentParser -from sys import exit +import sys from subprocess import run -from fileinput import input import re # Override Unicode tables for certain control chars @@ -27,45 +26,39 @@ def main() -> int: parser = ArgumentParser() parser.add_argument("-n", help="Take non-matching types", - action=store_true) + action='store_true') parser.add_argument("-f", help="Zero-based type field (default 2)", type=int, default=2) - parser.add_argument("type", nargs='+', type=string, required=True) + parser.add_argument("types", nargs='+', type=str) # parser.add_argument("--") # parser.add_argument("<") - parser.add_argument("UnicodeData.txt", required=True) + + # Kludge: cannot specify alternate var for dest= parameter when the positional + # argument's name contains a dot.' + parser.add_argument("data_file", type=open, metavar="UnicodeData.txt") args = parser.parse_args() global opt_n opt_n = args.n global type_field type_field = args.f - types = {} - for arg in sys.argv: - if arg == "--": - break - types[arg] = 1 - - # while ($arg = shift @ARGV) { - # last if $arg eq '--'; - # $types{$arg} = 1; - # } - # out = { 'types': types } + types = {type: 1 for type in args.types} + out = {'types': types} - force_compose = {} - for lo, hi in force_compose: - force_compose = [1 for ch in range(lo, hi)] + global_list = globals() + force_compose = {ch: 1 for lo, hi in global_list['force_compose'] + for ch in range(lo, hi)} # Only works for Python 3.7+ - date = run("date", capture_output=True, text=True).rstrip('\n') - print(f"/* Generated by \"{sys.argv[0]} {sys.argv[1:]}\" on {date}.") - print(" Python FTW! */") + date = run("date", capture_output=True, text=True).stdout.rstrip('\r\n') + print(f"/* Generated by \"{sys.argv[0]} {' '.join(sys.argv[1:])}\" on {date}.") + print(" Python FTW! */") last_code = 0 - with open(UnicodeData.txt) as file: - for line in file: - line.rstrip('\n') - line = re.sub(r"s/#.*//", line) + with args.data_file as file: + for line in file.read(): + line.rstrip('\r\n') + line = re.sub(r"s/#.*//", '', line) fields = line.split(';') if not fields: continue @@ -76,38 +69,45 @@ def main() -> int: hi_code = int(m.group(2), 16) else: lo_code = hi_code = int(codes, 16) + print(fields) + print(type_field) type = fields[type_field] type = re.match(r"s/\s//g", type) for last_code in range(lo_code, hi_code): output(out, last_code, 'Zs' if force_space[last_code] else 'Mn' - if force_compose[last_code] else type) + if force_compose[last_code] else type) output(out, last_code) return 1 + def output(out: dict, code: int, type: str): - type_ok = (type and ${${$out}{types}}{$type}) - type_ok = not type_ok if opt_n - prev_code = out{prev_code} + type_ok = type and out['types'][type] + if opt_n: + type_ok = not type_ok + prev_code = out["prev_code"] if not type_ok: end_run(out, prev_code) - elif (not out{in_run} or type ne out{run_type} or code != prev_code+1): + elif (not out["in_run"] or type != out["run_type"] or code != prev_code+1): end_run(out, prev_code) start_run(out, code, type) - out{prev_code} = code + out[prev_code] = code + def start_run(out: dict, code: int, type: str): - out[start_code] = code - out[prev_code] = code - out[run_type] = type - out[in_run] = 1 + out["start_code"] = code + out["prev_code"] = code + out["run_type"] = type + out["in_run"] = True + def end_run(out: dict, code: int): - if not out[in_run]: + if not out["in_run"]: return - print("\t{ 0x%04x, 0x%04x }, /* %s */" % $$out{start_code}, code, - out{run_type}) - out[in_run] = 0 + print("\t{ 0x%04x, 0x%04x }, /* %s */" % out["start_code"], code, + out["run_type"]) + out["in_run"] = False + if __name__ == "__main__": exit(0 if main() else 1) From 047a6eba81eae778fd59bb34313016b17f798bc0 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Sun, 28 Mar 2021 14:50:30 -0400 Subject: [PATCH 04/56] Debug in progress... --- mkutable.py | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/mkutable.py b/mkutable.py index fd76408d..6d53c079 100644 --- a/mkutable.py +++ b/mkutable.py @@ -1,9 +1,12 @@ #! /usr/bin/env python from argparse import ArgumentParser -import sys +from sys import argv from subprocess import run import re +import friendly + +friendly.install() # Override Unicode tables for certain control chars # that are expected to be found in normal text files. @@ -30,11 +33,9 @@ def main() -> int: parser.add_argument("-f", help="Zero-based type field (default 2)", type=int, default=2) parser.add_argument("types", nargs='+', type=str) - # parser.add_argument("--") - # parser.add_argument("<") - # Kludge: cannot specify alternate var for dest= parameter when the positional - # argument's name contains a dot.' + # Kludge: cannot specify alternate var for dest= parameter when the positional + # argument's name contains a dot. parser.add_argument("data_file", type=open, metavar="UnicodeData.txt") args = parser.parse_args() global opt_n @@ -43,7 +44,8 @@ def main() -> int: type_field = args.f types = {type: 1 for type in args.types} - out = {'types': types} + out = {'types': types, 'prev_code': 0, 'in_run': False, 'run_type': "", + 'start_code': 0} global_list = globals() force_compose = {ch: 1 for lo, hi in global_list['force_compose'] @@ -51,12 +53,12 @@ def main() -> int: # Only works for Python 3.7+ date = run("date", capture_output=True, text=True).stdout.rstrip('\r\n') - print(f"/* Generated by \"{sys.argv[0]} {' '.join(sys.argv[1:])}\" on {date}.") + print(f"/* Generated by \"{argv[0]} {' '.join(argv[1:])}\" on {date}.") print(" Python FTW! */") last_code = 0 with args.data_file as file: - for line in file.read(): + for line in file: line.rstrip('\r\n') line = re.sub(r"s/#.*//", '', line) fields = line.split(';') @@ -69,8 +71,6 @@ def main() -> int: hi_code = int(m.group(2), 16) else: lo_code = hi_code = int(codes, 16) - print(fields) - print(type_field) type = fields[type_field] type = re.match(r"s/\s//g", type) for last_code in range(lo_code, hi_code): @@ -80,12 +80,18 @@ def main() -> int: return 1 -def output(out: dict, code: int, type: str): - type_ok = type and out['types'][type] +def output(out: dict, code: int, type: str = None): + type_ok = type is not None and type in out['types'].keys() if opt_n: type_ok = not type_ok - prev_code = out["prev_code"] + prev_code = out['prev_code'] + + print(out['types'].keys()) + import pprint + p = pprint.PrettyPrinter() + p.pprint(out) + print(type) if not type_ok: end_run(out, prev_code) elif (not out["in_run"] or type != out["run_type"] or code != prev_code+1): From 27ef7b31fcc542a2cc6c96e5f74a32b08202b135 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Thu, 25 Mar 2021 16:07:20 -0400 Subject: [PATCH 05/56] Commit more Python versions of scripts. WIP --- add_copyright.py | 34 ++++++++++++++ mkutable.py | 113 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+) create mode 100644 add_copyright.py create mode 100644 mkutable.py diff --git a/add_copyright.py b/add_copyright.py new file mode 100644 index 00000000..f61156a8 --- /dev/null +++ b/add_copyright.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 + +from sys import argv, open +from os.path import isadir +import io +import re + +dir = pop(argv) +if (!isadir(dir)): + print(dir + "is not a directory\n") + exit(1) + +copyright = open("copyright", "r") +copyright_line = copyright.read(2) +copyright.close() + +for i in argv: + out = dir + "/" + i + if (!open(i000)): + print("Cannot open {0}\n", i) + pass + if (!open(out)): + print("Cannot create {0}\n", out) + close(i) + pass + while(input): + + close(copyright) + if (exec): + mode = 0555 + else: + mode = 0444 + chmod(out. mode) + diff --git a/mkutable.py b/mkutable.py new file mode 100644 index 00000000..cb1cf14b --- /dev/null +++ b/mkutable.py @@ -0,0 +1,113 @@ +#! /usr/bin/env python + +from argparse import ArgumentParser +from sys import exit +from subprocess import run +from fileinput import input +import re + +# Override Unicode tables for certain control chars +# that are expected to be found in normal text files. +force_space = { + 0x08: 1, # backspace + 0x09: 1, # tab + 0x0a: 1, # newline + 0x0c: 1, # form feed + 0x0d: 1, # carriage return +} + +# Hangul Jamo medial vowels and final consonants should be zero width. +force_compose = [ + [0x1160, 0x11ff], + [0xd7b0, 0xd7c6], + [0xd7cb, 0xd7fb] +] + + +def main() -> int: + parser = ArgumentParser() + parser.add_argument("-n", help="Take non-matching types", + action=store_true) + parser.add_argument("-f", help="Zero-based type field (default 2)", + type=int, default=2) + parser.add_argument("type", nargs='+', type=string, required=True) + # parser.add_argument("--") + # parser.add_argument("<") + parser.add_argument("UnicodeData.txt", required=True) + args = parser.parse_args() + global opt_n + opt_n = args.n + global type_field + type_field = args.f + + types = {} + for arg in sys.argv: + if arg == "--": + break + types[arg] = 1 + + # while ($arg = shift @ARGV) { + # last if $arg eq '--'; + # $types{$arg} = 1; + # } + # out = { 'types': types } + + force_compose = {} + for lo, hi in force_compose: + force_compose = [1 for ch in range(lo, hi)] + + # Only works for Python 3.7+ + date = run("date", capture_output=True, text=True).rstrip('\n') + print(f"/* Generated by \"{sys.argv[0]} {sys.argv[1:]}\" on {date}.") + print(" Python FTW! */") + + last_code = 0 + with open(UnicodeData.txt) as file: + for line in file: + line.rstrip('\n') + line = re.sub(r"s/#.*//", line) + fields = line.split(';') + if not fields: + continue + lo_code = hi_code = 0 + codes = fields[0] + if m := re.match(r"(\w+)\.\.(\w+)", codes): + lo_code = int(m.group(1), 16) + hi_code = int(m.group(2), 16) + else: + lo_code = hi_code = int(codes, 16) + type = fields[type_field] + type = re.match(r"s/\s//g", type) + for last_code in range(lo_code, hi_code): + output(out, last_code, 'Zs' if force_space[last_code] else 'Mn' + if force_compose[last_code] else type) + output(out, last_code) + return 1 + +def output(out: dict, code: int, type: str): + type_ok = (type and ${${$out}{types}}{$type}) + type_ok = not type_ok if opt_n + prev_code = out{prev_code} + + if not type_ok: + end_run(out, prev_code) + elif (not out{in_run} or type ne out{run_type} or code != prev_code+1): + end_run(out, prev_code) + start_run(out, code, type) + out{prev_code} = code + +def start_run(out: dict, code: int, type: str): + out[start_code] = code + out[prev_code] = code + out[run_type] = type + out[in_run] = 1 + +def end_run(out: dict, code: int): + if not out[in_run]: + return + print("\t{ 0x%04x, 0x%04x }, /* %s */" % $$out{start_code}, code, + out{run_type}) + out[in_run] = 0 + +if __name__ == "__main__": + exit(0 if main() else 1) From 35d1a4307e6596d8bd6c7ce2fdcce0ce373167d6 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Fri, 26 Mar 2021 15:42:33 -0400 Subject: [PATCH 06/56] Add Python version of add_copyright, and adjust Makefile.aut accordingly. --- Makefile.aut | 4 +++- add_copyright.py | 52 ++++++++++++++++++++++-------------------------- 2 files changed, 27 insertions(+), 29 deletions(-) diff --git a/Makefile.aut b/Makefile.aut index c30c6622..e02987be 100644 --- a/Makefile.aut +++ b/Makefile.aut @@ -11,9 +11,11 @@ srcdir = . ifeq ($(USE_PYTHON),1) MKHELP = mkhelp.py MKFUNCS = mkfuncs.py + ADD_COPYRIGHT = add_copyright.py else MKHELP = mkhelp.pl MKFUNCS = mkfuncs.pl + ADD_COPYRIGHT = add_copyright endif SRC = \ @@ -135,7 +137,7 @@ dist: ${DISTFILES} echo "Preparing $$REL"; \ rm -rf $$REL; mkdir $$REL; \ for file in ${DISTFILES}; do \ - ./add_copyright $$file $$REL; \ + ${srcdir}/${ADD_COPYRIGHT} $$file $$REL; \ done; \ cd $$REL; chmod -w *; chmod +w ${DISTFILES_W}; chmod +x configure; cd ..; \ echo "Creating release/$$REL/$$REL.tar.gz"; \ diff --git a/add_copyright.py b/add_copyright.py index f61156a8..bc8bc618 100644 --- a/add_copyright.py +++ b/add_copyright.py @@ -1,34 +1,30 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python -from sys import argv, open -from os.path import isadir -import io +import os +import argparse +import pathlib import re -dir = pop(argv) -if (!isadir(dir)): - print(dir + "is not a directory\n") - exit(1) +parser = argparse.ArgumentParser() +parser.add_argument("file", type=str) +parser.add_argument("dir", type=str) +args = parser.parse_args() +out = pathlib.Path(args.dir) / args.file -copyright = open("copyright", "r") -copyright_line = copyright.read(2) -copyright.close() +copyright = copyright_line = "" +with open("copyright") as f: + copyright = f.read() +for line in copyright.splitlines(): + if '(C)' in line: + copyright_line = line.strip("/*/\r\n").rstrip(" ") + break -for i in argv: - out = dir + "/" + i - if (!open(i000)): - print("Cannot open {0}\n", i) - pass - if (!open(out)): - print("Cannot create {0}\n", out) - close(i) - pass - while(input): +with open(args.file) as input, out.open('w') as o: + for line in input: + if re.search(r"\@\@copyright\@\@", line): + o.write(copyright) + continue + out_line = re.sub(r"\@\@copyright_oneline\@\@", copyright_line, line) + o.write(out_line) - close(copyright) - if (exec): - mode = 0555 - else: - mode = 0444 - chmod(out. mode) - + out.chmod(0o555 if os.access(out, os.X_OK) else 0o444) From f9e1c8b495c4713972aedde7c202063c1669f8ea Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Sun, 28 Mar 2021 14:50:30 -0400 Subject: [PATCH 07/56] Debug in progress... --- mkutable.py | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/mkutable.py b/mkutable.py index fd76408d..6d53c079 100644 --- a/mkutable.py +++ b/mkutable.py @@ -1,9 +1,12 @@ #! /usr/bin/env python from argparse import ArgumentParser -import sys +from sys import argv from subprocess import run import re +import friendly + +friendly.install() # Override Unicode tables for certain control chars # that are expected to be found in normal text files. @@ -30,11 +33,9 @@ def main() -> int: parser.add_argument("-f", help="Zero-based type field (default 2)", type=int, default=2) parser.add_argument("types", nargs='+', type=str) - # parser.add_argument("--") - # parser.add_argument("<") - # Kludge: cannot specify alternate var for dest= parameter when the positional - # argument's name contains a dot.' + # Kludge: cannot specify alternate var for dest= parameter when the positional + # argument's name contains a dot. parser.add_argument("data_file", type=open, metavar="UnicodeData.txt") args = parser.parse_args() global opt_n @@ -43,7 +44,8 @@ def main() -> int: type_field = args.f types = {type: 1 for type in args.types} - out = {'types': types} + out = {'types': types, 'prev_code': 0, 'in_run': False, 'run_type': "", + 'start_code': 0} global_list = globals() force_compose = {ch: 1 for lo, hi in global_list['force_compose'] @@ -51,12 +53,12 @@ def main() -> int: # Only works for Python 3.7+ date = run("date", capture_output=True, text=True).stdout.rstrip('\r\n') - print(f"/* Generated by \"{sys.argv[0]} {' '.join(sys.argv[1:])}\" on {date}.") + print(f"/* Generated by \"{argv[0]} {' '.join(argv[1:])}\" on {date}.") print(" Python FTW! */") last_code = 0 with args.data_file as file: - for line in file.read(): + for line in file: line.rstrip('\r\n') line = re.sub(r"s/#.*//", '', line) fields = line.split(';') @@ -69,8 +71,6 @@ def main() -> int: hi_code = int(m.group(2), 16) else: lo_code = hi_code = int(codes, 16) - print(fields) - print(type_field) type = fields[type_field] type = re.match(r"s/\s//g", type) for last_code in range(lo_code, hi_code): @@ -80,12 +80,18 @@ def main() -> int: return 1 -def output(out: dict, code: int, type: str): - type_ok = type and out['types'][type] +def output(out: dict, code: int, type: str = None): + type_ok = type is not None and type in out['types'].keys() if opt_n: type_ok = not type_ok - prev_code = out["prev_code"] + prev_code = out['prev_code'] + + print(out['types'].keys()) + import pprint + p = pprint.PrettyPrinter() + p.pprint(out) + print(type) if not type_ok: end_run(out, prev_code) elif (not out["in_run"] or type != out["run_type"] or code != prev_code+1): From 40c61ae2c84558ff3267104fa283a92e6ba0d9bc Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Sat, 27 Mar 2021 15:46:43 -0400 Subject: [PATCH 08/56] Finalize conversion of Perl build scripts into Python versions. --- mkfuncs.py | 34 ++++++++++++----------- mkhelp.py | 18 ++++++------ mkutable.py | 80 ++++++++++++++++++++++++++--------------------------- 3 files changed, 68 insertions(+), 64 deletions(-) diff --git a/mkfuncs.py b/mkfuncs.py index 7a2da5ad..c397942f 100755 --- a/mkfuncs.py +++ b/mkfuncs.py @@ -8,19 +8,21 @@ params = 0 for line in fileinput.input(): - if test := re.search(r'^\tpublic\s+(.*)', line): - definition = 'public ' + test.group(1) - state = 1 - params = 0 - elif (state == 1) and (test := re.search(r'(\w+)\s*\(', line)): - definition = '{0} LESSPARAMS (('.format(test.group(1)) - state = 2 - elif state == 2: - if re.search(r'^{', line): - if not params: definition += 'VOID_PARAM' - print(f'{definition}));') - state = 0 - elif test := re.search(r'^\s*([^;]*)', line): - if (definition[-1:] != '('): definition += ', ' - definition += test.group(1) - params = 1 + if test := re.search(r'^\tpublic\s+(.*)', line): + definition = 'public ' + test.group(1) + state = 1 + params = 0 + elif (state == 1) and (test := re.search(r'(\w+)\s*\(', line)): + definition = '{0} LESSPARAMS (('.format(test.group(1)) + state = 2 + elif state == 2: + if re.search(r'^{', line): + if not params: + definition += 'VOID_PARAM' + print(f'{definition}));') + state = 0 + elif test := re.search(r'^\s*([^;]*)', line): + if (definition[-1:] != '('): + definition += ', ' + definition += test.group(1) + params = 1 diff --git a/mkhelp.py b/mkhelp.py index 029f1505..4619457f 100755 --- a/mkhelp.py +++ b/mkhelp.py @@ -4,9 +4,9 @@ import sys time = time.gmtime() -print("/* This file was generated by mkhelp.py from less.hlp at "\ - "%d:%02d GMT on %d/%d/%d */\n" % - (time.tm_hour, time.tm_min, time.tm_year, time.tm_mon, time.tm_mday)) +print("/* This file was generated by mkhelp.py from less.hlp at " + "%d:%02d GMT on %d/%d/%d */\n" % + (time.tm_hour, time.tm_min, time.tm_year, time.tm_mon, time.tm_mday)) print("#include \"less.h\"") print("constant char helpdata[] = {") ch = 0 @@ -16,17 +16,19 @@ if ch == '': break if (ch == "'"): - print("'\\'',", end='') + print("'\\'',", end='') elif (ch == "\\"): print("'\\\\',", end='') elif (ch == "\b"): - print ("'\\b',", end='') + print("'\\b',", end='') elif (ch == "\t"): - print ("'\\t',", end='') + print("'\\t',", end='') elif (ch == "\n"): - if prevch != "\r": print("'\\n',") + if prevch != "\r": + print("'\\n',") elif (ch == "\r"): - if prevch != "\n": print("'\\n',") + if prevch != "\n": + print("'\\n',") else: if ((ord(ch) >= ord(' ')) and (ord(ch) < 0x7f)): print(f"'{ch}',", end='') diff --git a/mkutable.py b/mkutable.py index cb1cf14b..fd76408d 100644 --- a/mkutable.py +++ b/mkutable.py @@ -1,9 +1,8 @@ #! /usr/bin/env python from argparse import ArgumentParser -from sys import exit +import sys from subprocess import run -from fileinput import input import re # Override Unicode tables for certain control chars @@ -27,45 +26,39 @@ def main() -> int: parser = ArgumentParser() parser.add_argument("-n", help="Take non-matching types", - action=store_true) + action='store_true') parser.add_argument("-f", help="Zero-based type field (default 2)", type=int, default=2) - parser.add_argument("type", nargs='+', type=string, required=True) + parser.add_argument("types", nargs='+', type=str) # parser.add_argument("--") # parser.add_argument("<") - parser.add_argument("UnicodeData.txt", required=True) + + # Kludge: cannot specify alternate var for dest= parameter when the positional + # argument's name contains a dot.' + parser.add_argument("data_file", type=open, metavar="UnicodeData.txt") args = parser.parse_args() global opt_n opt_n = args.n global type_field type_field = args.f - types = {} - for arg in sys.argv: - if arg == "--": - break - types[arg] = 1 - - # while ($arg = shift @ARGV) { - # last if $arg eq '--'; - # $types{$arg} = 1; - # } - # out = { 'types': types } + types = {type: 1 for type in args.types} + out = {'types': types} - force_compose = {} - for lo, hi in force_compose: - force_compose = [1 for ch in range(lo, hi)] + global_list = globals() + force_compose = {ch: 1 for lo, hi in global_list['force_compose'] + for ch in range(lo, hi)} # Only works for Python 3.7+ - date = run("date", capture_output=True, text=True).rstrip('\n') - print(f"/* Generated by \"{sys.argv[0]} {sys.argv[1:]}\" on {date}.") - print(" Python FTW! */") + date = run("date", capture_output=True, text=True).stdout.rstrip('\r\n') + print(f"/* Generated by \"{sys.argv[0]} {' '.join(sys.argv[1:])}\" on {date}.") + print(" Python FTW! */") last_code = 0 - with open(UnicodeData.txt) as file: - for line in file: - line.rstrip('\n') - line = re.sub(r"s/#.*//", line) + with args.data_file as file: + for line in file.read(): + line.rstrip('\r\n') + line = re.sub(r"s/#.*//", '', line) fields = line.split(';') if not fields: continue @@ -76,38 +69,45 @@ def main() -> int: hi_code = int(m.group(2), 16) else: lo_code = hi_code = int(codes, 16) + print(fields) + print(type_field) type = fields[type_field] type = re.match(r"s/\s//g", type) for last_code in range(lo_code, hi_code): output(out, last_code, 'Zs' if force_space[last_code] else 'Mn' - if force_compose[last_code] else type) + if force_compose[last_code] else type) output(out, last_code) return 1 + def output(out: dict, code: int, type: str): - type_ok = (type and ${${$out}{types}}{$type}) - type_ok = not type_ok if opt_n - prev_code = out{prev_code} + type_ok = type and out['types'][type] + if opt_n: + type_ok = not type_ok + prev_code = out["prev_code"] if not type_ok: end_run(out, prev_code) - elif (not out{in_run} or type ne out{run_type} or code != prev_code+1): + elif (not out["in_run"] or type != out["run_type"] or code != prev_code+1): end_run(out, prev_code) start_run(out, code, type) - out{prev_code} = code + out[prev_code] = code + def start_run(out: dict, code: int, type: str): - out[start_code] = code - out[prev_code] = code - out[run_type] = type - out[in_run] = 1 + out["start_code"] = code + out["prev_code"] = code + out["run_type"] = type + out["in_run"] = True + def end_run(out: dict, code: int): - if not out[in_run]: + if not out["in_run"]: return - print("\t{ 0x%04x, 0x%04x }, /* %s */" % $$out{start_code}, code, - out{run_type}) - out[in_run] = 0 + print("\t{ 0x%04x, 0x%04x }, /* %s */" % out["start_code"], code, + out["run_type"]) + out["in_run"] = False + if __name__ == "__main__": exit(0 if main() else 1) From 01ab369c51a5d9fcfeda069a5250232e661556ca Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Sun, 28 Mar 2021 14:50:30 -0400 Subject: [PATCH 09/56] WIP --- mkutable.py | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/mkutable.py b/mkutable.py index fd76408d..446b4c7b 100644 --- a/mkutable.py +++ b/mkutable.py @@ -1,9 +1,12 @@ #! /usr/bin/env python from argparse import ArgumentParser -import sys +from sys import argv from subprocess import run import re +import friendly + +friendly.install() # Override Unicode tables for certain control chars # that are expected to be found in normal text files. @@ -30,11 +33,9 @@ def main() -> int: parser.add_argument("-f", help="Zero-based type field (default 2)", type=int, default=2) parser.add_argument("types", nargs='+', type=str) - # parser.add_argument("--") - # parser.add_argument("<") - # Kludge: cannot specify alternate var for dest= parameter when the positional - # argument's name contains a dot.' + # Kludge: cannot specify alternate var for dest= parameter when the positional + # argument's name contains a dot. parser.add_argument("data_file", type=open, metavar="UnicodeData.txt") args = parser.parse_args() global opt_n @@ -43,7 +44,8 @@ def main() -> int: type_field = args.f types = {type: 1 for type in args.types} - out = {'types': types} + out = {'types': types, 'prev_code': 0, 'in_run': False, 'run_type': "", + 'start_code': 0} global_list = globals() force_compose = {ch: 1 for lo, hi in global_list['force_compose'] @@ -51,12 +53,12 @@ def main() -> int: # Only works for Python 3.7+ date = run("date", capture_output=True, text=True).stdout.rstrip('\r\n') - print(f"/* Generated by \"{sys.argv[0]} {' '.join(sys.argv[1:])}\" on {date}.") + print(f"/* Generated by \"{argv[0]} {' '.join(argv[1:])}\" on {date}.") print(" Python FTW! */") last_code = 0 with args.data_file as file: - for line in file.read(): + for line in file: line.rstrip('\r\n') line = re.sub(r"s/#.*//", '', line) fields = line.split(';') @@ -69,8 +71,6 @@ def main() -> int: hi_code = int(m.group(2), 16) else: lo_code = hi_code = int(codes, 16) - print(fields) - print(type_field) type = fields[type_field] type = re.match(r"s/\s//g", type) for last_code in range(lo_code, hi_code): @@ -80,12 +80,18 @@ def main() -> int: return 1 -def output(out: dict, code: int, type: str): - type_ok = type and out['types'][type] +def output(out: dict, code: int, type: str = None): + type_ok = type is not None and type in out['types'].keys() if opt_n: type_ok = not type_ok - prev_code = out["prev_code"] + prev_code = out['prev_code'] + + print(out['types'].keys()) + import pprint + p = pprint.PrettyPrinter() + p.pprint(out) + print(type) if not type_ok: end_run(out, prev_code) elif (not out["in_run"] or type != out["run_type"] or code != prev_code+1): @@ -108,6 +114,5 @@ def end_run(out: dict, code: int): out["run_type"]) out["in_run"] = False - if __name__ == "__main__": exit(0 if main() else 1) From 03ef74231b2b2ce4d012f7c9a63e4199b32135d8 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Thu, 25 Mar 2021 16:07:20 -0400 Subject: [PATCH 10/56] Commit more Python versions of scripts. WIP --- add_copyright.py | 34 ++++++++++++++ mkutable.py | 113 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+) create mode 100644 add_copyright.py create mode 100644 mkutable.py diff --git a/add_copyright.py b/add_copyright.py new file mode 100644 index 00000000..f61156a8 --- /dev/null +++ b/add_copyright.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 + +from sys import argv, open +from os.path import isadir +import io +import re + +dir = pop(argv) +if (!isadir(dir)): + print(dir + "is not a directory\n") + exit(1) + +copyright = open("copyright", "r") +copyright_line = copyright.read(2) +copyright.close() + +for i in argv: + out = dir + "/" + i + if (!open(i000)): + print("Cannot open {0}\n", i) + pass + if (!open(out)): + print("Cannot create {0}\n", out) + close(i) + pass + while(input): + + close(copyright) + if (exec): + mode = 0555 + else: + mode = 0444 + chmod(out. mode) + diff --git a/mkutable.py b/mkutable.py new file mode 100644 index 00000000..cb1cf14b --- /dev/null +++ b/mkutable.py @@ -0,0 +1,113 @@ +#! /usr/bin/env python + +from argparse import ArgumentParser +from sys import exit +from subprocess import run +from fileinput import input +import re + +# Override Unicode tables for certain control chars +# that are expected to be found in normal text files. +force_space = { + 0x08: 1, # backspace + 0x09: 1, # tab + 0x0a: 1, # newline + 0x0c: 1, # form feed + 0x0d: 1, # carriage return +} + +# Hangul Jamo medial vowels and final consonants should be zero width. +force_compose = [ + [0x1160, 0x11ff], + [0xd7b0, 0xd7c6], + [0xd7cb, 0xd7fb] +] + + +def main() -> int: + parser = ArgumentParser() + parser.add_argument("-n", help="Take non-matching types", + action=store_true) + parser.add_argument("-f", help="Zero-based type field (default 2)", + type=int, default=2) + parser.add_argument("type", nargs='+', type=string, required=True) + # parser.add_argument("--") + # parser.add_argument("<") + parser.add_argument("UnicodeData.txt", required=True) + args = parser.parse_args() + global opt_n + opt_n = args.n + global type_field + type_field = args.f + + types = {} + for arg in sys.argv: + if arg == "--": + break + types[arg] = 1 + + # while ($arg = shift @ARGV) { + # last if $arg eq '--'; + # $types{$arg} = 1; + # } + # out = { 'types': types } + + force_compose = {} + for lo, hi in force_compose: + force_compose = [1 for ch in range(lo, hi)] + + # Only works for Python 3.7+ + date = run("date", capture_output=True, text=True).rstrip('\n') + print(f"/* Generated by \"{sys.argv[0]} {sys.argv[1:]}\" on {date}.") + print(" Python FTW! */") + + last_code = 0 + with open(UnicodeData.txt) as file: + for line in file: + line.rstrip('\n') + line = re.sub(r"s/#.*//", line) + fields = line.split(';') + if not fields: + continue + lo_code = hi_code = 0 + codes = fields[0] + if m := re.match(r"(\w+)\.\.(\w+)", codes): + lo_code = int(m.group(1), 16) + hi_code = int(m.group(2), 16) + else: + lo_code = hi_code = int(codes, 16) + type = fields[type_field] + type = re.match(r"s/\s//g", type) + for last_code in range(lo_code, hi_code): + output(out, last_code, 'Zs' if force_space[last_code] else 'Mn' + if force_compose[last_code] else type) + output(out, last_code) + return 1 + +def output(out: dict, code: int, type: str): + type_ok = (type and ${${$out}{types}}{$type}) + type_ok = not type_ok if opt_n + prev_code = out{prev_code} + + if not type_ok: + end_run(out, prev_code) + elif (not out{in_run} or type ne out{run_type} or code != prev_code+1): + end_run(out, prev_code) + start_run(out, code, type) + out{prev_code} = code + +def start_run(out: dict, code: int, type: str): + out[start_code] = code + out[prev_code] = code + out[run_type] = type + out[in_run] = 1 + +def end_run(out: dict, code: int): + if not out[in_run]: + return + print("\t{ 0x%04x, 0x%04x }, /* %s */" % $$out{start_code}, code, + out{run_type}) + out[in_run] = 0 + +if __name__ == "__main__": + exit(0 if main() else 1) From 816109ee990b10e353fb880fa44b60dd3e48fc79 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Fri, 26 Mar 2021 15:42:33 -0400 Subject: [PATCH 11/56] Add Python version of add_copyright, and adjust Makefile.aut accordingly. --- Makefile.aut | 4 +++- add_copyright.py | 52 ++++++++++++++++++++++-------------------------- 2 files changed, 27 insertions(+), 29 deletions(-) diff --git a/Makefile.aut b/Makefile.aut index 9a3b2d2f..01280dd5 100644 --- a/Makefile.aut +++ b/Makefile.aut @@ -11,9 +11,11 @@ srcdir = . ifeq ($(USE_PYTHON),1) MKHELP = mkhelp.py MKFUNCS = mkfuncs.py + ADD_COPYRIGHT = add_copyright.py else MKHELP = mkhelp.pl MKFUNCS = mkfuncs.pl + ADD_COPYRIGHT = add_copyright endif SRC = \ @@ -130,7 +132,7 @@ dist: ${DISTFILES} echo "Preparing $$REL"; \ rm -rf $$REL; mkdir $$REL; \ for file in ${DISTFILES}; do \ - ./add_copyright $$file $$REL; \ + ${srcdir}/${ADD_COPYRIGHT} $$file $$REL; \ done; \ cd $$REL; chmod -w *; chmod +w ${DISTFILES_W}; chmod +x configure; cd ..; \ echo "Creating release/$$REL/$$REL.tar.gz"; \ diff --git a/add_copyright.py b/add_copyright.py index f61156a8..bc8bc618 100644 --- a/add_copyright.py +++ b/add_copyright.py @@ -1,34 +1,30 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python -from sys import argv, open -from os.path import isadir -import io +import os +import argparse +import pathlib import re -dir = pop(argv) -if (!isadir(dir)): - print(dir + "is not a directory\n") - exit(1) +parser = argparse.ArgumentParser() +parser.add_argument("file", type=str) +parser.add_argument("dir", type=str) +args = parser.parse_args() +out = pathlib.Path(args.dir) / args.file -copyright = open("copyright", "r") -copyright_line = copyright.read(2) -copyright.close() +copyright = copyright_line = "" +with open("copyright") as f: + copyright = f.read() +for line in copyright.splitlines(): + if '(C)' in line: + copyright_line = line.strip("/*/\r\n").rstrip(" ") + break -for i in argv: - out = dir + "/" + i - if (!open(i000)): - print("Cannot open {0}\n", i) - pass - if (!open(out)): - print("Cannot create {0}\n", out) - close(i) - pass - while(input): +with open(args.file) as input, out.open('w') as o: + for line in input: + if re.search(r"\@\@copyright\@\@", line): + o.write(copyright) + continue + out_line = re.sub(r"\@\@copyright_oneline\@\@", copyright_line, line) + o.write(out_line) - close(copyright) - if (exec): - mode = 0555 - else: - mode = 0444 - chmod(out. mode) - + out.chmod(0o555 if os.access(out, os.X_OK) else 0o444) From 229bf662709999300cd4d4220b49860eeaa56b6d Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Sat, 27 Mar 2021 15:46:43 -0400 Subject: [PATCH 12/56] Finalize conversion of Perl build scripts into Python versions. --- mkfuncs.py | 34 ++++++++++++----------- mkhelp.py | 18 ++++++------ mkutable.py | 80 ++++++++++++++++++++++++++--------------------------- 3 files changed, 68 insertions(+), 64 deletions(-) diff --git a/mkfuncs.py b/mkfuncs.py index 7a2da5ad..c397942f 100755 --- a/mkfuncs.py +++ b/mkfuncs.py @@ -8,19 +8,21 @@ params = 0 for line in fileinput.input(): - if test := re.search(r'^\tpublic\s+(.*)', line): - definition = 'public ' + test.group(1) - state = 1 - params = 0 - elif (state == 1) and (test := re.search(r'(\w+)\s*\(', line)): - definition = '{0} LESSPARAMS (('.format(test.group(1)) - state = 2 - elif state == 2: - if re.search(r'^{', line): - if not params: definition += 'VOID_PARAM' - print(f'{definition}));') - state = 0 - elif test := re.search(r'^\s*([^;]*)', line): - if (definition[-1:] != '('): definition += ', ' - definition += test.group(1) - params = 1 + if test := re.search(r'^\tpublic\s+(.*)', line): + definition = 'public ' + test.group(1) + state = 1 + params = 0 + elif (state == 1) and (test := re.search(r'(\w+)\s*\(', line)): + definition = '{0} LESSPARAMS (('.format(test.group(1)) + state = 2 + elif state == 2: + if re.search(r'^{', line): + if not params: + definition += 'VOID_PARAM' + print(f'{definition}));') + state = 0 + elif test := re.search(r'^\s*([^;]*)', line): + if (definition[-1:] != '('): + definition += ', ' + definition += test.group(1) + params = 1 diff --git a/mkhelp.py b/mkhelp.py index 029f1505..4619457f 100755 --- a/mkhelp.py +++ b/mkhelp.py @@ -4,9 +4,9 @@ import sys time = time.gmtime() -print("/* This file was generated by mkhelp.py from less.hlp at "\ - "%d:%02d GMT on %d/%d/%d */\n" % - (time.tm_hour, time.tm_min, time.tm_year, time.tm_mon, time.tm_mday)) +print("/* This file was generated by mkhelp.py from less.hlp at " + "%d:%02d GMT on %d/%d/%d */\n" % + (time.tm_hour, time.tm_min, time.tm_year, time.tm_mon, time.tm_mday)) print("#include \"less.h\"") print("constant char helpdata[] = {") ch = 0 @@ -16,17 +16,19 @@ if ch == '': break if (ch == "'"): - print("'\\'',", end='') + print("'\\'',", end='') elif (ch == "\\"): print("'\\\\',", end='') elif (ch == "\b"): - print ("'\\b',", end='') + print("'\\b',", end='') elif (ch == "\t"): - print ("'\\t',", end='') + print("'\\t',", end='') elif (ch == "\n"): - if prevch != "\r": print("'\\n',") + if prevch != "\r": + print("'\\n',") elif (ch == "\r"): - if prevch != "\n": print("'\\n',") + if prevch != "\n": + print("'\\n',") else: if ((ord(ch) >= ord(' ')) and (ord(ch) < 0x7f)): print(f"'{ch}',", end='') diff --git a/mkutable.py b/mkutable.py index cb1cf14b..fd76408d 100644 --- a/mkutable.py +++ b/mkutable.py @@ -1,9 +1,8 @@ #! /usr/bin/env python from argparse import ArgumentParser -from sys import exit +import sys from subprocess import run -from fileinput import input import re # Override Unicode tables for certain control chars @@ -27,45 +26,39 @@ def main() -> int: parser = ArgumentParser() parser.add_argument("-n", help="Take non-matching types", - action=store_true) + action='store_true') parser.add_argument("-f", help="Zero-based type field (default 2)", type=int, default=2) - parser.add_argument("type", nargs='+', type=string, required=True) + parser.add_argument("types", nargs='+', type=str) # parser.add_argument("--") # parser.add_argument("<") - parser.add_argument("UnicodeData.txt", required=True) + + # Kludge: cannot specify alternate var for dest= parameter when the positional + # argument's name contains a dot.' + parser.add_argument("data_file", type=open, metavar="UnicodeData.txt") args = parser.parse_args() global opt_n opt_n = args.n global type_field type_field = args.f - types = {} - for arg in sys.argv: - if arg == "--": - break - types[arg] = 1 - - # while ($arg = shift @ARGV) { - # last if $arg eq '--'; - # $types{$arg} = 1; - # } - # out = { 'types': types } + types = {type: 1 for type in args.types} + out = {'types': types} - force_compose = {} - for lo, hi in force_compose: - force_compose = [1 for ch in range(lo, hi)] + global_list = globals() + force_compose = {ch: 1 for lo, hi in global_list['force_compose'] + for ch in range(lo, hi)} # Only works for Python 3.7+ - date = run("date", capture_output=True, text=True).rstrip('\n') - print(f"/* Generated by \"{sys.argv[0]} {sys.argv[1:]}\" on {date}.") - print(" Python FTW! */") + date = run("date", capture_output=True, text=True).stdout.rstrip('\r\n') + print(f"/* Generated by \"{sys.argv[0]} {' '.join(sys.argv[1:])}\" on {date}.") + print(" Python FTW! */") last_code = 0 - with open(UnicodeData.txt) as file: - for line in file: - line.rstrip('\n') - line = re.sub(r"s/#.*//", line) + with args.data_file as file: + for line in file.read(): + line.rstrip('\r\n') + line = re.sub(r"s/#.*//", '', line) fields = line.split(';') if not fields: continue @@ -76,38 +69,45 @@ def main() -> int: hi_code = int(m.group(2), 16) else: lo_code = hi_code = int(codes, 16) + print(fields) + print(type_field) type = fields[type_field] type = re.match(r"s/\s//g", type) for last_code in range(lo_code, hi_code): output(out, last_code, 'Zs' if force_space[last_code] else 'Mn' - if force_compose[last_code] else type) + if force_compose[last_code] else type) output(out, last_code) return 1 + def output(out: dict, code: int, type: str): - type_ok = (type and ${${$out}{types}}{$type}) - type_ok = not type_ok if opt_n - prev_code = out{prev_code} + type_ok = type and out['types'][type] + if opt_n: + type_ok = not type_ok + prev_code = out["prev_code"] if not type_ok: end_run(out, prev_code) - elif (not out{in_run} or type ne out{run_type} or code != prev_code+1): + elif (not out["in_run"] or type != out["run_type"] or code != prev_code+1): end_run(out, prev_code) start_run(out, code, type) - out{prev_code} = code + out[prev_code] = code + def start_run(out: dict, code: int, type: str): - out[start_code] = code - out[prev_code] = code - out[run_type] = type - out[in_run] = 1 + out["start_code"] = code + out["prev_code"] = code + out["run_type"] = type + out["in_run"] = True + def end_run(out: dict, code: int): - if not out[in_run]: + if not out["in_run"]: return - print("\t{ 0x%04x, 0x%04x }, /* %s */" % $$out{start_code}, code, - out{run_type}) - out[in_run] = 0 + print("\t{ 0x%04x, 0x%04x }, /* %s */" % out["start_code"], code, + out["run_type"]) + out["in_run"] = False + if __name__ == "__main__": exit(0 if main() else 1) From 2566293076c26230a5c70433832f843d5b0e1ba6 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Sun, 28 Mar 2021 14:50:30 -0400 Subject: [PATCH 13/56] WIP --- mkutable.py | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/mkutable.py b/mkutable.py index fd76408d..446b4c7b 100644 --- a/mkutable.py +++ b/mkutable.py @@ -1,9 +1,12 @@ #! /usr/bin/env python from argparse import ArgumentParser -import sys +from sys import argv from subprocess import run import re +import friendly + +friendly.install() # Override Unicode tables for certain control chars # that are expected to be found in normal text files. @@ -30,11 +33,9 @@ def main() -> int: parser.add_argument("-f", help="Zero-based type field (default 2)", type=int, default=2) parser.add_argument("types", nargs='+', type=str) - # parser.add_argument("--") - # parser.add_argument("<") - # Kludge: cannot specify alternate var for dest= parameter when the positional - # argument's name contains a dot.' + # Kludge: cannot specify alternate var for dest= parameter when the positional + # argument's name contains a dot. parser.add_argument("data_file", type=open, metavar="UnicodeData.txt") args = parser.parse_args() global opt_n @@ -43,7 +44,8 @@ def main() -> int: type_field = args.f types = {type: 1 for type in args.types} - out = {'types': types} + out = {'types': types, 'prev_code': 0, 'in_run': False, 'run_type': "", + 'start_code': 0} global_list = globals() force_compose = {ch: 1 for lo, hi in global_list['force_compose'] @@ -51,12 +53,12 @@ def main() -> int: # Only works for Python 3.7+ date = run("date", capture_output=True, text=True).stdout.rstrip('\r\n') - print(f"/* Generated by \"{sys.argv[0]} {' '.join(sys.argv[1:])}\" on {date}.") + print(f"/* Generated by \"{argv[0]} {' '.join(argv[1:])}\" on {date}.") print(" Python FTW! */") last_code = 0 with args.data_file as file: - for line in file.read(): + for line in file: line.rstrip('\r\n') line = re.sub(r"s/#.*//", '', line) fields = line.split(';') @@ -69,8 +71,6 @@ def main() -> int: hi_code = int(m.group(2), 16) else: lo_code = hi_code = int(codes, 16) - print(fields) - print(type_field) type = fields[type_field] type = re.match(r"s/\s//g", type) for last_code in range(lo_code, hi_code): @@ -80,12 +80,18 @@ def main() -> int: return 1 -def output(out: dict, code: int, type: str): - type_ok = type and out['types'][type] +def output(out: dict, code: int, type: str = None): + type_ok = type is not None and type in out['types'].keys() if opt_n: type_ok = not type_ok - prev_code = out["prev_code"] + prev_code = out['prev_code'] + + print(out['types'].keys()) + import pprint + p = pprint.PrettyPrinter() + p.pprint(out) + print(type) if not type_ok: end_run(out, prev_code) elif (not out["in_run"] or type != out["run_type"] or code != prev_code+1): @@ -108,6 +114,5 @@ def end_run(out: dict, code: int): out["run_type"]) out["in_run"] = False - if __name__ == "__main__": exit(0 if main() else 1) From bb82079c78f56d12ba7a84869618dceb6bd76b20 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Thu, 25 Mar 2021 16:07:20 -0400 Subject: [PATCH 14/56] Commit more Python versions of scripts. WIP --- mkutable.py | 47 +++++++++++++++++------------------------------ 1 file changed, 17 insertions(+), 30 deletions(-) diff --git a/mkutable.py b/mkutable.py index 446b4c7b..c8dcf95d 100644 --- a/mkutable.py +++ b/mkutable.py @@ -1,12 +1,10 @@ #! /usr/bin/env python from argparse import ArgumentParser -from sys import argv +from sys import exit from subprocess import run +from fileinput import input import re -import friendly - -friendly.install() # Override Unicode tables for certain control chars # that are expected to be found in normal text files. @@ -42,7 +40,6 @@ def main() -> int: opt_n = args.n global type_field type_field = args.f - types = {type: 1 for type in args.types} out = {'types': types, 'prev_code': 0, 'in_run': False, 'run_type': "", 'start_code': 0} @@ -75,44 +72,34 @@ def main() -> int: type = re.match(r"s/\s//g", type) for last_code in range(lo_code, hi_code): output(out, last_code, 'Zs' if force_space[last_code] else 'Mn' - if force_compose[last_code] else type) + if force_compose[last_code] else type) output(out, last_code) return 1 +def output(out: dict, code: int, type: str): + type_ok = (type and ${${$out}{types}}{$type}) + type_ok = not type_ok if opt_n + prev_code = out{prev_code} -def output(out: dict, code: int, type: str = None): - type_ok = type is not None and type in out['types'].keys() - if opt_n: - type_ok = not type_ok - prev_code = out['prev_code'] - - print(out['types'].keys()) - import pprint - p = pprint.PrettyPrinter() - p.pprint(out) - - print(type) if not type_ok: end_run(out, prev_code) - elif (not out["in_run"] or type != out["run_type"] or code != prev_code+1): + elif (not out{in_run} or type ne out{run_type} or code != prev_code+1): end_run(out, prev_code) start_run(out, code, type) - out[prev_code] = code - + out{prev_code} = code def start_run(out: dict, code: int, type: str): - out["start_code"] = code - out["prev_code"] = code - out["run_type"] = type - out["in_run"] = True - + out[start_code] = code + out[prev_code] = code + out[run_type] = type + out[in_run] = 1 def end_run(out: dict, code: int): - if not out["in_run"]: + if not out[in_run]: return - print("\t{ 0x%04x, 0x%04x }, /* %s */" % out["start_code"], code, - out["run_type"]) - out["in_run"] = False + print("\t{ 0x%04x, 0x%04x }, /* %s */" % $$out{start_code}, code, + out{run_type}) + out[in_run] = 0 if __name__ == "__main__": exit(0 if main() else 1) From 5568dbf5658309815914bd304c147dfb3f97b507 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Sat, 27 Mar 2021 15:46:43 -0400 Subject: [PATCH 15/56] Finalize conversion of Perl build scripts into Python versions. --- mkutable.py | 45 ++++++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/mkutable.py b/mkutable.py index c8dcf95d..20957294 100644 --- a/mkutable.py +++ b/mkutable.py @@ -1,9 +1,8 @@ #! /usr/bin/env python from argparse import ArgumentParser -from sys import exit +import sys from subprocess import run -from fileinput import input import re # Override Unicode tables for certain control chars @@ -31,7 +30,6 @@ def main() -> int: parser.add_argument("-f", help="Zero-based type field (default 2)", type=int, default=2) parser.add_argument("types", nargs='+', type=str) - # Kludge: cannot specify alternate var for dest= parameter when the positional # argument's name contains a dot. parser.add_argument("data_file", type=open, metavar="UnicodeData.txt") @@ -43,20 +41,18 @@ def main() -> int: types = {type: 1 for type in args.types} out = {'types': types, 'prev_code': 0, 'in_run': False, 'run_type': "", 'start_code': 0} - global_list = globals() force_compose = {ch: 1 for lo, hi in global_list['force_compose'] for ch in range(lo, hi)} # Only works for Python 3.7+ date = run("date", capture_output=True, text=True).stdout.rstrip('\r\n') - print(f"/* Generated by \"{argv[0]} {' '.join(argv[1:])}\" on {date}.") + print(f"/* Generated by \"{sys.argv[0]} {' '.join(sys.argv[1:])}\" on {date}." print(" Python FTW! */") last_code = 0 with args.data_file as file: - for line in file: - line.rstrip('\r\n') + for line in file:line.rstrip('\r\n') line = re.sub(r"s/#.*//", '', line) fields = line.split(';') if not fields: @@ -68,38 +64,45 @@ def main() -> int: hi_code = int(m.group(2), 16) else: lo_code = hi_code = int(codes, 16) + print(fields) + print(type_field) type = fields[type_field] type = re.match(r"s/\s//g", type) for last_code in range(lo_code, hi_code): output(out, last_code, 'Zs' if force_space[last_code] else 'Mn' - if force_compose[last_code] else type) + if force_compose[last_code] else type) output(out, last_code) return 1 + def output(out: dict, code: int, type: str): - type_ok = (type and ${${$out}{types}}{$type}) - type_ok = not type_ok if opt_n - prev_code = out{prev_code} + type_ok = type and out['types'][type] + if opt_n: + type_ok = not type_ok + prev_code = out["prev_code"] if not type_ok: end_run(out, prev_code) - elif (not out{in_run} or type ne out{run_type} or code != prev_code+1): + elif (not out["in_run"] or type != out["run_type"] or code != prev_code+1): end_run(out, prev_code) start_run(out, code, type) - out{prev_code} = code + out[prev_code] = code + def start_run(out: dict, code: int, type: str): - out[start_code] = code - out[prev_code] = code - out[run_type] = type - out[in_run] = 1 + out["start_code"] = code + out["prev_code"] = code + out["run_type"] = type + out["in_run"] = True + def end_run(out: dict, code: int): - if not out[in_run]: + if not out["in_run"]: return - print("\t{ 0x%04x, 0x%04x }, /* %s */" % $$out{start_code}, code, - out{run_type}) - out[in_run] = 0 + print("\t{ 0x%04x, 0x%04x }, /* %s */" % out["start_code"], code, + out["run_type"]) + out["in_run"] = False + if __name__ == "__main__": exit(0 if main() else 1) From a6102a560b10402634f7d6fbcf8ef340a2c557d4 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Sun, 28 Mar 2021 14:50:30 -0400 Subject: [PATCH 16/56] Debug in progress... --- mkutable.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/mkutable.py b/mkutable.py index 20957294..36c67f47 100644 --- a/mkutable.py +++ b/mkutable.py @@ -1,9 +1,12 @@ #! /usr/bin/env python from argparse import ArgumentParser -import sys +from sys import argv from subprocess import run import re +import friendly + +friendly.install() # Override Unicode tables for certain control chars # that are expected to be found in normal text files. @@ -52,7 +55,8 @@ def main() -> int: last_code = 0 with args.data_file as file: - for line in file:line.rstrip('\r\n') + for line in file: + line.rstrip('\r\n') line = re.sub(r"s/#.*//", '', line) fields = line.split(';') if not fields: @@ -64,8 +68,6 @@ def main() -> int: hi_code = int(m.group(2), 16) else: lo_code = hi_code = int(codes, 16) - print(fields) - print(type_field) type = fields[type_field] type = re.match(r"s/\s//g", type) for last_code in range(lo_code, hi_code): @@ -75,12 +77,18 @@ def main() -> int: return 1 -def output(out: dict, code: int, type: str): - type_ok = type and out['types'][type] +def output(out: dict, code: int, type: str = None): + type_ok = type is not None and type in out['types'].keys() if opt_n: type_ok = not type_ok - prev_code = out["prev_code"] + prev_code = out['prev_code'] + + print(out['types'].keys()) + import pprint + p = pprint.PrettyPrinter() + p.pprint(out) + print(type) if not type_ok: end_run(out, prev_code) elif (not out["in_run"] or type != out["run_type"] or code != prev_code+1): From d434e8854554128aa133117b51d6e5edebe0a825 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Thu, 25 Mar 2021 16:07:20 -0400 Subject: [PATCH 17/56] Commit more Python versions of scripts. WIP --- add_copyright.py | 34 ++++++++++++++ mkutable.py | 113 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+) create mode 100644 add_copyright.py create mode 100644 mkutable.py diff --git a/add_copyright.py b/add_copyright.py new file mode 100644 index 00000000..f61156a8 --- /dev/null +++ b/add_copyright.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 + +from sys import argv, open +from os.path import isadir +import io +import re + +dir = pop(argv) +if (!isadir(dir)): + print(dir + "is not a directory\n") + exit(1) + +copyright = open("copyright", "r") +copyright_line = copyright.read(2) +copyright.close() + +for i in argv: + out = dir + "/" + i + if (!open(i000)): + print("Cannot open {0}\n", i) + pass + if (!open(out)): + print("Cannot create {0}\n", out) + close(i) + pass + while(input): + + close(copyright) + if (exec): + mode = 0555 + else: + mode = 0444 + chmod(out. mode) + diff --git a/mkutable.py b/mkutable.py new file mode 100644 index 00000000..cb1cf14b --- /dev/null +++ b/mkutable.py @@ -0,0 +1,113 @@ +#! /usr/bin/env python + +from argparse import ArgumentParser +from sys import exit +from subprocess import run +from fileinput import input +import re + +# Override Unicode tables for certain control chars +# that are expected to be found in normal text files. +force_space = { + 0x08: 1, # backspace + 0x09: 1, # tab + 0x0a: 1, # newline + 0x0c: 1, # form feed + 0x0d: 1, # carriage return +} + +# Hangul Jamo medial vowels and final consonants should be zero width. +force_compose = [ + [0x1160, 0x11ff], + [0xd7b0, 0xd7c6], + [0xd7cb, 0xd7fb] +] + + +def main() -> int: + parser = ArgumentParser() + parser.add_argument("-n", help="Take non-matching types", + action=store_true) + parser.add_argument("-f", help="Zero-based type field (default 2)", + type=int, default=2) + parser.add_argument("type", nargs='+', type=string, required=True) + # parser.add_argument("--") + # parser.add_argument("<") + parser.add_argument("UnicodeData.txt", required=True) + args = parser.parse_args() + global opt_n + opt_n = args.n + global type_field + type_field = args.f + + types = {} + for arg in sys.argv: + if arg == "--": + break + types[arg] = 1 + + # while ($arg = shift @ARGV) { + # last if $arg eq '--'; + # $types{$arg} = 1; + # } + # out = { 'types': types } + + force_compose = {} + for lo, hi in force_compose: + force_compose = [1 for ch in range(lo, hi)] + + # Only works for Python 3.7+ + date = run("date", capture_output=True, text=True).rstrip('\n') + print(f"/* Generated by \"{sys.argv[0]} {sys.argv[1:]}\" on {date}.") + print(" Python FTW! */") + + last_code = 0 + with open(UnicodeData.txt) as file: + for line in file: + line.rstrip('\n') + line = re.sub(r"s/#.*//", line) + fields = line.split(';') + if not fields: + continue + lo_code = hi_code = 0 + codes = fields[0] + if m := re.match(r"(\w+)\.\.(\w+)", codes): + lo_code = int(m.group(1), 16) + hi_code = int(m.group(2), 16) + else: + lo_code = hi_code = int(codes, 16) + type = fields[type_field] + type = re.match(r"s/\s//g", type) + for last_code in range(lo_code, hi_code): + output(out, last_code, 'Zs' if force_space[last_code] else 'Mn' + if force_compose[last_code] else type) + output(out, last_code) + return 1 + +def output(out: dict, code: int, type: str): + type_ok = (type and ${${$out}{types}}{$type}) + type_ok = not type_ok if opt_n + prev_code = out{prev_code} + + if not type_ok: + end_run(out, prev_code) + elif (not out{in_run} or type ne out{run_type} or code != prev_code+1): + end_run(out, prev_code) + start_run(out, code, type) + out{prev_code} = code + +def start_run(out: dict, code: int, type: str): + out[start_code] = code + out[prev_code] = code + out[run_type] = type + out[in_run] = 1 + +def end_run(out: dict, code: int): + if not out[in_run]: + return + print("\t{ 0x%04x, 0x%04x }, /* %s */" % $$out{start_code}, code, + out{run_type}) + out[in_run] = 0 + +if __name__ == "__main__": + exit(0 if main() else 1) From ac48936dad811b821218046e5c457f4fbce6a967 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Fri, 26 Mar 2021 15:42:33 -0400 Subject: [PATCH 18/56] Add Python version of add_copyright, and adjust Makefile.aut accordingly. --- Makefile.aut | 4 +++- add_copyright.py | 52 ++++++++++++++++++++++-------------------------- 2 files changed, 27 insertions(+), 29 deletions(-) diff --git a/Makefile.aut b/Makefile.aut index 8eb9f8af..9b42ee6d 100644 --- a/Makefile.aut +++ b/Makefile.aut @@ -11,9 +11,11 @@ srcdir = . ifeq ($(USE_PYTHON),1) MKHELP = mkhelp.py MKFUNCS = mkfuncs.py + ADD_COPYRIGHT = add_copyright.py else MKHELP = mkhelp.pl MKFUNCS = mkfuncs.pl + ADD_COPYRIGHT = add_copyright endif SRC = \ @@ -130,7 +132,7 @@ dist: ${DISTFILES} echo "Preparing $$REL"; \ rm -rf $$REL; mkdir $$REL; \ for file in ${DISTFILES}; do \ - ./add_copyright $$file $$REL; \ + ${srcdir}/${ADD_COPYRIGHT} $$file $$REL; \ done; \ cd $$REL; chmod -w *; chmod +w ${DISTFILES_W}; chmod +x configure; cd ..; \ echo "Creating release/$$REL/$$REL.tar.gz"; \ diff --git a/add_copyright.py b/add_copyright.py index f61156a8..bc8bc618 100644 --- a/add_copyright.py +++ b/add_copyright.py @@ -1,34 +1,30 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python -from sys import argv, open -from os.path import isadir -import io +import os +import argparse +import pathlib import re -dir = pop(argv) -if (!isadir(dir)): - print(dir + "is not a directory\n") - exit(1) +parser = argparse.ArgumentParser() +parser.add_argument("file", type=str) +parser.add_argument("dir", type=str) +args = parser.parse_args() +out = pathlib.Path(args.dir) / args.file -copyright = open("copyright", "r") -copyright_line = copyright.read(2) -copyright.close() +copyright = copyright_line = "" +with open("copyright") as f: + copyright = f.read() +for line in copyright.splitlines(): + if '(C)' in line: + copyright_line = line.strip("/*/\r\n").rstrip(" ") + break -for i in argv: - out = dir + "/" + i - if (!open(i000)): - print("Cannot open {0}\n", i) - pass - if (!open(out)): - print("Cannot create {0}\n", out) - close(i) - pass - while(input): +with open(args.file) as input, out.open('w') as o: + for line in input: + if re.search(r"\@\@copyright\@\@", line): + o.write(copyright) + continue + out_line = re.sub(r"\@\@copyright_oneline\@\@", copyright_line, line) + o.write(out_line) - close(copyright) - if (exec): - mode = 0555 - else: - mode = 0444 - chmod(out. mode) - + out.chmod(0o555 if os.access(out, os.X_OK) else 0o444) From a16166ad0c7d1ea559123e50e2c93d90700eccf0 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Sat, 27 Mar 2021 15:46:43 -0400 Subject: [PATCH 19/56] Finalize conversion of Perl build scripts into Python versions. --- mkfuncs.py | 34 ++++++++++++----------- mkhelp.py | 18 ++++++------ mkutable.py | 80 ++++++++++++++++++++++++++--------------------------- 3 files changed, 68 insertions(+), 64 deletions(-) diff --git a/mkfuncs.py b/mkfuncs.py index 7a2da5ad..c397942f 100755 --- a/mkfuncs.py +++ b/mkfuncs.py @@ -8,19 +8,21 @@ params = 0 for line in fileinput.input(): - if test := re.search(r'^\tpublic\s+(.*)', line): - definition = 'public ' + test.group(1) - state = 1 - params = 0 - elif (state == 1) and (test := re.search(r'(\w+)\s*\(', line)): - definition = '{0} LESSPARAMS (('.format(test.group(1)) - state = 2 - elif state == 2: - if re.search(r'^{', line): - if not params: definition += 'VOID_PARAM' - print(f'{definition}));') - state = 0 - elif test := re.search(r'^\s*([^;]*)', line): - if (definition[-1:] != '('): definition += ', ' - definition += test.group(1) - params = 1 + if test := re.search(r'^\tpublic\s+(.*)', line): + definition = 'public ' + test.group(1) + state = 1 + params = 0 + elif (state == 1) and (test := re.search(r'(\w+)\s*\(', line)): + definition = '{0} LESSPARAMS (('.format(test.group(1)) + state = 2 + elif state == 2: + if re.search(r'^{', line): + if not params: + definition += 'VOID_PARAM' + print(f'{definition}));') + state = 0 + elif test := re.search(r'^\s*([^;]*)', line): + if (definition[-1:] != '('): + definition += ', ' + definition += test.group(1) + params = 1 diff --git a/mkhelp.py b/mkhelp.py index 029f1505..4619457f 100755 --- a/mkhelp.py +++ b/mkhelp.py @@ -4,9 +4,9 @@ import sys time = time.gmtime() -print("/* This file was generated by mkhelp.py from less.hlp at "\ - "%d:%02d GMT on %d/%d/%d */\n" % - (time.tm_hour, time.tm_min, time.tm_year, time.tm_mon, time.tm_mday)) +print("/* This file was generated by mkhelp.py from less.hlp at " + "%d:%02d GMT on %d/%d/%d */\n" % + (time.tm_hour, time.tm_min, time.tm_year, time.tm_mon, time.tm_mday)) print("#include \"less.h\"") print("constant char helpdata[] = {") ch = 0 @@ -16,17 +16,19 @@ if ch == '': break if (ch == "'"): - print("'\\'',", end='') + print("'\\'',", end='') elif (ch == "\\"): print("'\\\\',", end='') elif (ch == "\b"): - print ("'\\b',", end='') + print("'\\b',", end='') elif (ch == "\t"): - print ("'\\t',", end='') + print("'\\t',", end='') elif (ch == "\n"): - if prevch != "\r": print("'\\n',") + if prevch != "\r": + print("'\\n',") elif (ch == "\r"): - if prevch != "\n": print("'\\n',") + if prevch != "\n": + print("'\\n',") else: if ((ord(ch) >= ord(' ')) and (ord(ch) < 0x7f)): print(f"'{ch}',", end='') diff --git a/mkutable.py b/mkutable.py index cb1cf14b..fd76408d 100644 --- a/mkutable.py +++ b/mkutable.py @@ -1,9 +1,8 @@ #! /usr/bin/env python from argparse import ArgumentParser -from sys import exit +import sys from subprocess import run -from fileinput import input import re # Override Unicode tables for certain control chars @@ -27,45 +26,39 @@ def main() -> int: parser = ArgumentParser() parser.add_argument("-n", help="Take non-matching types", - action=store_true) + action='store_true') parser.add_argument("-f", help="Zero-based type field (default 2)", type=int, default=2) - parser.add_argument("type", nargs='+', type=string, required=True) + parser.add_argument("types", nargs='+', type=str) # parser.add_argument("--") # parser.add_argument("<") - parser.add_argument("UnicodeData.txt", required=True) + + # Kludge: cannot specify alternate var for dest= parameter when the positional + # argument's name contains a dot.' + parser.add_argument("data_file", type=open, metavar="UnicodeData.txt") args = parser.parse_args() global opt_n opt_n = args.n global type_field type_field = args.f - types = {} - for arg in sys.argv: - if arg == "--": - break - types[arg] = 1 - - # while ($arg = shift @ARGV) { - # last if $arg eq '--'; - # $types{$arg} = 1; - # } - # out = { 'types': types } + types = {type: 1 for type in args.types} + out = {'types': types} - force_compose = {} - for lo, hi in force_compose: - force_compose = [1 for ch in range(lo, hi)] + global_list = globals() + force_compose = {ch: 1 for lo, hi in global_list['force_compose'] + for ch in range(lo, hi)} # Only works for Python 3.7+ - date = run("date", capture_output=True, text=True).rstrip('\n') - print(f"/* Generated by \"{sys.argv[0]} {sys.argv[1:]}\" on {date}.") - print(" Python FTW! */") + date = run("date", capture_output=True, text=True).stdout.rstrip('\r\n') + print(f"/* Generated by \"{sys.argv[0]} {' '.join(sys.argv[1:])}\" on {date}.") + print(" Python FTW! */") last_code = 0 - with open(UnicodeData.txt) as file: - for line in file: - line.rstrip('\n') - line = re.sub(r"s/#.*//", line) + with args.data_file as file: + for line in file.read(): + line.rstrip('\r\n') + line = re.sub(r"s/#.*//", '', line) fields = line.split(';') if not fields: continue @@ -76,38 +69,45 @@ def main() -> int: hi_code = int(m.group(2), 16) else: lo_code = hi_code = int(codes, 16) + print(fields) + print(type_field) type = fields[type_field] type = re.match(r"s/\s//g", type) for last_code in range(lo_code, hi_code): output(out, last_code, 'Zs' if force_space[last_code] else 'Mn' - if force_compose[last_code] else type) + if force_compose[last_code] else type) output(out, last_code) return 1 + def output(out: dict, code: int, type: str): - type_ok = (type and ${${$out}{types}}{$type}) - type_ok = not type_ok if opt_n - prev_code = out{prev_code} + type_ok = type and out['types'][type] + if opt_n: + type_ok = not type_ok + prev_code = out["prev_code"] if not type_ok: end_run(out, prev_code) - elif (not out{in_run} or type ne out{run_type} or code != prev_code+1): + elif (not out["in_run"] or type != out["run_type"] or code != prev_code+1): end_run(out, prev_code) start_run(out, code, type) - out{prev_code} = code + out[prev_code] = code + def start_run(out: dict, code: int, type: str): - out[start_code] = code - out[prev_code] = code - out[run_type] = type - out[in_run] = 1 + out["start_code"] = code + out["prev_code"] = code + out["run_type"] = type + out["in_run"] = True + def end_run(out: dict, code: int): - if not out[in_run]: + if not out["in_run"]: return - print("\t{ 0x%04x, 0x%04x }, /* %s */" % $$out{start_code}, code, - out{run_type}) - out[in_run] = 0 + print("\t{ 0x%04x, 0x%04x }, /* %s */" % out["start_code"], code, + out["run_type"]) + out["in_run"] = False + if __name__ == "__main__": exit(0 if main() else 1) From 3bda1d94d465c601255a3338363cf34f8b6c664b Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Sun, 28 Mar 2021 14:50:30 -0400 Subject: [PATCH 20/56] WIP --- mkutable.py | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/mkutable.py b/mkutable.py index fd76408d..446b4c7b 100644 --- a/mkutable.py +++ b/mkutable.py @@ -1,9 +1,12 @@ #! /usr/bin/env python from argparse import ArgumentParser -import sys +from sys import argv from subprocess import run import re +import friendly + +friendly.install() # Override Unicode tables for certain control chars # that are expected to be found in normal text files. @@ -30,11 +33,9 @@ def main() -> int: parser.add_argument("-f", help="Zero-based type field (default 2)", type=int, default=2) parser.add_argument("types", nargs='+', type=str) - # parser.add_argument("--") - # parser.add_argument("<") - # Kludge: cannot specify alternate var for dest= parameter when the positional - # argument's name contains a dot.' + # Kludge: cannot specify alternate var for dest= parameter when the positional + # argument's name contains a dot. parser.add_argument("data_file", type=open, metavar="UnicodeData.txt") args = parser.parse_args() global opt_n @@ -43,7 +44,8 @@ def main() -> int: type_field = args.f types = {type: 1 for type in args.types} - out = {'types': types} + out = {'types': types, 'prev_code': 0, 'in_run': False, 'run_type': "", + 'start_code': 0} global_list = globals() force_compose = {ch: 1 for lo, hi in global_list['force_compose'] @@ -51,12 +53,12 @@ def main() -> int: # Only works for Python 3.7+ date = run("date", capture_output=True, text=True).stdout.rstrip('\r\n') - print(f"/* Generated by \"{sys.argv[0]} {' '.join(sys.argv[1:])}\" on {date}.") + print(f"/* Generated by \"{argv[0]} {' '.join(argv[1:])}\" on {date}.") print(" Python FTW! */") last_code = 0 with args.data_file as file: - for line in file.read(): + for line in file: line.rstrip('\r\n') line = re.sub(r"s/#.*//", '', line) fields = line.split(';') @@ -69,8 +71,6 @@ def main() -> int: hi_code = int(m.group(2), 16) else: lo_code = hi_code = int(codes, 16) - print(fields) - print(type_field) type = fields[type_field] type = re.match(r"s/\s//g", type) for last_code in range(lo_code, hi_code): @@ -80,12 +80,18 @@ def main() -> int: return 1 -def output(out: dict, code: int, type: str): - type_ok = type and out['types'][type] +def output(out: dict, code: int, type: str = None): + type_ok = type is not None and type in out['types'].keys() if opt_n: type_ok = not type_ok - prev_code = out["prev_code"] + prev_code = out['prev_code'] + + print(out['types'].keys()) + import pprint + p = pprint.PrettyPrinter() + p.pprint(out) + print(type) if not type_ok: end_run(out, prev_code) elif (not out["in_run"] or type != out["run_type"] or code != prev_code+1): @@ -108,6 +114,5 @@ def end_run(out: dict, code: int): out["run_type"]) out["in_run"] = False - if __name__ == "__main__": exit(0 if main() else 1) From 38a4000943368788bf2c0721a159bc1b82b98147 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Thu, 25 Mar 2021 16:07:20 -0400 Subject: [PATCH 21/56] Commit more Python versions of scripts. WIP --- mkutable.py | 47 +++++++++++++++++------------------------------ 1 file changed, 17 insertions(+), 30 deletions(-) diff --git a/mkutable.py b/mkutable.py index 446b4c7b..c8dcf95d 100644 --- a/mkutable.py +++ b/mkutable.py @@ -1,12 +1,10 @@ #! /usr/bin/env python from argparse import ArgumentParser -from sys import argv +from sys import exit from subprocess import run +from fileinput import input import re -import friendly - -friendly.install() # Override Unicode tables for certain control chars # that are expected to be found in normal text files. @@ -42,7 +40,6 @@ def main() -> int: opt_n = args.n global type_field type_field = args.f - types = {type: 1 for type in args.types} out = {'types': types, 'prev_code': 0, 'in_run': False, 'run_type': "", 'start_code': 0} @@ -75,44 +72,34 @@ def main() -> int: type = re.match(r"s/\s//g", type) for last_code in range(lo_code, hi_code): output(out, last_code, 'Zs' if force_space[last_code] else 'Mn' - if force_compose[last_code] else type) + if force_compose[last_code] else type) output(out, last_code) return 1 +def output(out: dict, code: int, type: str): + type_ok = (type and ${${$out}{types}}{$type}) + type_ok = not type_ok if opt_n + prev_code = out{prev_code} -def output(out: dict, code: int, type: str = None): - type_ok = type is not None and type in out['types'].keys() - if opt_n: - type_ok = not type_ok - prev_code = out['prev_code'] - - print(out['types'].keys()) - import pprint - p = pprint.PrettyPrinter() - p.pprint(out) - - print(type) if not type_ok: end_run(out, prev_code) - elif (not out["in_run"] or type != out["run_type"] or code != prev_code+1): + elif (not out{in_run} or type ne out{run_type} or code != prev_code+1): end_run(out, prev_code) start_run(out, code, type) - out[prev_code] = code - + out{prev_code} = code def start_run(out: dict, code: int, type: str): - out["start_code"] = code - out["prev_code"] = code - out["run_type"] = type - out["in_run"] = True - + out[start_code] = code + out[prev_code] = code + out[run_type] = type + out[in_run] = 1 def end_run(out: dict, code: int): - if not out["in_run"]: + if not out[in_run]: return - print("\t{ 0x%04x, 0x%04x }, /* %s */" % out["start_code"], code, - out["run_type"]) - out["in_run"] = False + print("\t{ 0x%04x, 0x%04x }, /* %s */" % $$out{start_code}, code, + out{run_type}) + out[in_run] = 0 if __name__ == "__main__": exit(0 if main() else 1) From 37eaafcc83e83a5024a38a43a5e4449e799c48ee Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Sat, 27 Mar 2021 15:46:43 -0400 Subject: [PATCH 22/56] Finalize conversion of Perl build scripts into Python versions. --- mkutable.py | 45 ++++++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/mkutable.py b/mkutable.py index c8dcf95d..20957294 100644 --- a/mkutable.py +++ b/mkutable.py @@ -1,9 +1,8 @@ #! /usr/bin/env python from argparse import ArgumentParser -from sys import exit +import sys from subprocess import run -from fileinput import input import re # Override Unicode tables for certain control chars @@ -31,7 +30,6 @@ def main() -> int: parser.add_argument("-f", help="Zero-based type field (default 2)", type=int, default=2) parser.add_argument("types", nargs='+', type=str) - # Kludge: cannot specify alternate var for dest= parameter when the positional # argument's name contains a dot. parser.add_argument("data_file", type=open, metavar="UnicodeData.txt") @@ -43,20 +41,18 @@ def main() -> int: types = {type: 1 for type in args.types} out = {'types': types, 'prev_code': 0, 'in_run': False, 'run_type': "", 'start_code': 0} - global_list = globals() force_compose = {ch: 1 for lo, hi in global_list['force_compose'] for ch in range(lo, hi)} # Only works for Python 3.7+ date = run("date", capture_output=True, text=True).stdout.rstrip('\r\n') - print(f"/* Generated by \"{argv[0]} {' '.join(argv[1:])}\" on {date}.") + print(f"/* Generated by \"{sys.argv[0]} {' '.join(sys.argv[1:])}\" on {date}." print(" Python FTW! */") last_code = 0 with args.data_file as file: - for line in file: - line.rstrip('\r\n') + for line in file:line.rstrip('\r\n') line = re.sub(r"s/#.*//", '', line) fields = line.split(';') if not fields: @@ -68,38 +64,45 @@ def main() -> int: hi_code = int(m.group(2), 16) else: lo_code = hi_code = int(codes, 16) + print(fields) + print(type_field) type = fields[type_field] type = re.match(r"s/\s//g", type) for last_code in range(lo_code, hi_code): output(out, last_code, 'Zs' if force_space[last_code] else 'Mn' - if force_compose[last_code] else type) + if force_compose[last_code] else type) output(out, last_code) return 1 + def output(out: dict, code: int, type: str): - type_ok = (type and ${${$out}{types}}{$type}) - type_ok = not type_ok if opt_n - prev_code = out{prev_code} + type_ok = type and out['types'][type] + if opt_n: + type_ok = not type_ok + prev_code = out["prev_code"] if not type_ok: end_run(out, prev_code) - elif (not out{in_run} or type ne out{run_type} or code != prev_code+1): + elif (not out["in_run"] or type != out["run_type"] or code != prev_code+1): end_run(out, prev_code) start_run(out, code, type) - out{prev_code} = code + out[prev_code] = code + def start_run(out: dict, code: int, type: str): - out[start_code] = code - out[prev_code] = code - out[run_type] = type - out[in_run] = 1 + out["start_code"] = code + out["prev_code"] = code + out["run_type"] = type + out["in_run"] = True + def end_run(out: dict, code: int): - if not out[in_run]: + if not out["in_run"]: return - print("\t{ 0x%04x, 0x%04x }, /* %s */" % $$out{start_code}, code, - out{run_type}) - out[in_run] = 0 + print("\t{ 0x%04x, 0x%04x }, /* %s */" % out["start_code"], code, + out["run_type"]) + out["in_run"] = False + if __name__ == "__main__": exit(0 if main() else 1) From 3d7b19e5315dd4052121c02a34d76ab156a8083e Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Sun, 28 Mar 2021 14:50:30 -0400 Subject: [PATCH 23/56] Debug in progress... --- mkutable.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/mkutable.py b/mkutable.py index 20957294..36c67f47 100644 --- a/mkutable.py +++ b/mkutable.py @@ -1,9 +1,12 @@ #! /usr/bin/env python from argparse import ArgumentParser -import sys +from sys import argv from subprocess import run import re +import friendly + +friendly.install() # Override Unicode tables for certain control chars # that are expected to be found in normal text files. @@ -52,7 +55,8 @@ def main() -> int: last_code = 0 with args.data_file as file: - for line in file:line.rstrip('\r\n') + for line in file: + line.rstrip('\r\n') line = re.sub(r"s/#.*//", '', line) fields = line.split(';') if not fields: @@ -64,8 +68,6 @@ def main() -> int: hi_code = int(m.group(2), 16) else: lo_code = hi_code = int(codes, 16) - print(fields) - print(type_field) type = fields[type_field] type = re.match(r"s/\s//g", type) for last_code in range(lo_code, hi_code): @@ -75,12 +77,18 @@ def main() -> int: return 1 -def output(out: dict, code: int, type: str): - type_ok = type and out['types'][type] +def output(out: dict, code: int, type: str = None): + type_ok = type is not None and type in out['types'].keys() if opt_n: type_ok = not type_ok - prev_code = out["prev_code"] + prev_code = out['prev_code'] + + print(out['types'].keys()) + import pprint + p = pprint.PrettyPrinter() + p.pprint(out) + print(type) if not type_ok: end_run(out, prev_code) elif (not out["in_run"] or type != out["run_type"] or code != prev_code+1): From 1f8437cff4b5ecfee3f06dda99e5843d6bc57552 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Thu, 20 May 2021 14:37:56 -0400 Subject: [PATCH 24/56] Fourth time? --- mkutable.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/mkutable.py b/mkutable.py index 36c67f47..e469f145 100644 --- a/mkutable.py +++ b/mkutable.py @@ -1,12 +1,11 @@ #! /usr/bin/env python from argparse import ArgumentParser -from sys import argv +from sys import exit from subprocess import run +from fileinput import input import re -import friendly - -friendly.install() +>>>>>>> 27ef7b3 (Commit more Python versions of scripts. WIP) # Override Unicode tables for certain control chars # that are expected to be found in normal text files. @@ -111,6 +110,5 @@ def end_run(out: dict, code: int): out["run_type"]) out["in_run"] = False - if __name__ == "__main__": exit(0 if main() else 1) From c2dc7053562fafcef4bb08c24c415a5981f9217e Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Wed, 26 May 2021 10:26:54 -0400 Subject: [PATCH 25/56] Add Python equivalents to scripts in lesstest. Signed-off-by: Charlie Lin --- lesstest/gen.py | 67 ++++++++++++++++++++++++++++++++++++++++++++++ lesstest/run.py | 71 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 lesstest/gen.py create mode 100644 lesstest/run.py diff --git a/lesstest/gen.py b/lesstest/gen.py new file mode 100644 index 00000000..ea427829 --- /dev/null +++ b/lesstest/gen.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python + +from argparse import ArgumentParser, FileType +from subprocess import run +from sys import exit, stderr, argv +from itertools import count +from pathlib import Path +from re import search +from os import environ + +lesstest = "./lesstest" +outdir = Path("./suite") + + +def main(): + pars = ArgumentParser(description="Generate a lesstest file.", add_help=False) + pars.add_argument("-w", help="width", type=int) + pars.add_argument("-h", help="height", type=int) + pars.add_argument("--lessvar", default=environ.get("LT_LESSVAR")) + pars.add_argument("less", type=Path) + pars.add_argument("lessflags", nargs="?") + pars.add_argument("textfile", type=FileType("r")) + args = pars.parse_args() + + width, height = term_size() + if width > 2: + width -= 2 + if height > 1: + width -= 1 + if args.w is not None: + width = args.w + if args.h is not None: + height = args.h + + if not args.textfile.is_file(): + print(f"cannot open {args.textfile.name}", file=stderr) + return 0 + + base = args.textfile.name + if base: + pass + + outfile = "" + for n in count(1): + outfile = outdir / Path(f"{base}-{n}.lt") + if outfile.stat().st_size == 0: + break + cmd = "" + if width > 0: + cmd = f"LT_COLUMNS={width} {cmd}" + if height > 0: + cmd = f"LT_LINES={height} {cmd}" + cmd = f"{cmd} {lesstest} -s ./lt_screen -o {outfile} -- ".join(" ", argv) + if run(cmd).returncode: + print(f"error running {lesstest}", file=stderr) + return 1 + + +def term_size(): + stty = run(["stty", "-a"], capture_output=True, text=True).output + width = search(r"(rows|lines)\s+(\d+)", stty).group(0) + col = search(r"columns\s+(\d+)", stty).group(0) + return int(width), int(col) + + +if __name__ == "__main__": + exit(0 if main() else 1) diff --git a/lesstest/run.py b/lesstest/run.py new file mode 100644 index 00000000..26ddee7c --- /dev/null +++ b/lesstest/run.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python + +import argparse +import sys +import subprocess +from pathlib import Path + +lesstest = "./lesstest" +less = "../obj/less" +verbose = False + + +def main(): + args = argparse.ArgumentParser(description="Run one or more test files") + args.add_argument('targets', metavar='file.lt | dir', type=Path, nargs='+') + args.add_argument('-v', '--verbose', action="store_true") + args.add_argument('-l', help="less.exe", type=Path) + args.add_argument('-o', help="arguments to be passed to lesstest", + nargs='+') + parsed = args.parse_args() + + if parsed.l: + global less + less = parsed.l + global lesstest_opts + lesstest_opts = parsed.o.strip() + if lesstest_opts and lesstest_opts[0] != '-': + lesstest_opts = '-'+lesstest_opts + global verbose + verbose = parsed.v + + errors = 0 + for file in parsed.targets: + errors += run(file) + if errors > 0: + print(f"ERR {errors} errors", file=sys.stderr) + return 0 + + return 1 + + +def run(file): + if file.is_dir(): + return run_dir(file) + if file.suffix() != '.lt': + print(f'WARNING skipping {file.name}: not .lt file') + return 0 + if not file.is_file(): + print(f"ERR cannot open {file.name}", file=sys.stderr) + return 1 + + cmd = f'{lesstest} -s ./lt_screen -t {file.name} {lesstest_opts} {less}' + if verbose: + print(f"RUN {cmd}", file=sys.stderr) + if subprocess.run(cmd.split()).returncode: + print(f"ERR running {cmd}", file=sys.stderr) + return 1 + return 0 + + +def run_dir(dir): + if not dir.is_dir(): + print(f"ERR cannot directory {dir.name}", file=sys.stderr) + return 1 + errors = 0 + errors += run(file for file in dir.iterdir()) + return errors + + +if __name__ == '__main__': + sys.exit(0 if main() else 1) From ec07e10cfc3427b371ebcd199c4d2fd754bd66aa Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Wed, 26 May 2021 17:56:09 -0400 Subject: [PATCH 26/56] Append Perl scripts with .pl extension, and more. In the README, acknowledge the existence of the USE_PYTHON flag when using Makefile.aut (relevant when checking from Git repo). Signed-off-by: Charlie Lin --- Makefile.aut | 12 +++++++----- README | 3 ++- add_copyright => add_copyright.pl | 0 mkutable => mkutable.pl | 0 4 files changed, 9 insertions(+), 6 deletions(-) rename add_copyright => add_copyright.pl (100%) mode change 100755 => 100644 rename mkutable => mkutable.pl (100%) mode change 100755 => 100644 diff --git a/Makefile.aut b/Makefile.aut index 9b42ee6d..07e215ac 100644 --- a/Makefile.aut +++ b/Makefile.aut @@ -11,11 +11,13 @@ srcdir = . ifeq ($(USE_PYTHON),1) MKHELP = mkhelp.py MKFUNCS = mkfuncs.py + MKUTABLE = mkutable.py ADD_COPYRIGHT = add_copyright.py else MKHELP = mkhelp.pl MKFUNCS = mkfuncs.pl - ADD_COPYRIGHT = add_copyright + MKUTABLE = mkutable.pl + ADD_COPYRIGHT = add_copyright.pl endif SRC = \ @@ -107,13 +109,13 @@ ${srcdir}/lessecho.man: ${srcdir}/lessecho.nro ${NROFF} ${srcdir}/lessecho.nro >${srcdir}/lessecho.man compose.uni: unicode/UnicodeData.txt - ./mkutable -f2 Mn Me -- unicode/UnicodeData.txt > $@ + ${srcdir}/${MKUTABLE} -f2 Mn Me -- unicode/UnicodeData.txt > $@ fmt.uni: unicode/UnicodeData.txt - ./mkutable -f2 Cf -- unicode/UnicodeData.txt > $@ + ${srcdir}/${MKUTABLE} -f2 Cf -- unicode/UnicodeData.txt > $@ ubin.uni: unicode/UnicodeData.txt - ./mkutable -f2 Cc Cs Co Zl Zp -- unicode/UnicodeData.txt > $@ + ${srcdir}/${MKUTABLE} -f2 Cc Cs Co Zl Zp -- unicode/UnicodeData.txt > $@ wide.uni: unicode/EastAsianWidth.txt - ./mkutable -f1 W F -- unicode/EastAsianWidth.txt > $@ + ${srcdir}/${MKUTABLE} -f1 W F -- unicode/EastAsianWidth.txt > $@ unicode/UnicodeData.txt: mkdir -p unicode diff --git a/README b/README index 955bf828..57061ce0 100644 --- a/README +++ b/README @@ -33,7 +33,8 @@ INSTALLATION (Unix & Linux systems only): if you have not already done so. 2. If you are building from a clone of a git repository, - type "make -f Makefile.aut". + type "make -f Makefile.aut". If you would like to use Python versions of + build scripts, define "USE_PYTHON=1" before typing make. If you are building from a numbered release package (a tar or zip file with a name like less-999.tar.gz or less-999.zip downloaded from greenwoodsoftware.com, not from github), you should skip this step. diff --git a/add_copyright b/add_copyright.pl old mode 100755 new mode 100644 similarity index 100% rename from add_copyright rename to add_copyright.pl diff --git a/mkutable b/mkutable.pl old mode 100755 new mode 100644 similarity index 100% rename from mkutable rename to mkutable.pl From ff4d1a0e61e1c0dff04eb75971687a236a47a3ac Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Wed, 18 Aug 2021 23:03:05 -0400 Subject: [PATCH 27/56] Improve Python version of mkhelp. --- mkhelp.py | 64 +++++++++++++++++++++++++------------------------------ 1 file changed, 29 insertions(+), 35 deletions(-) mode change 100755 => 100644 mkhelp.py diff --git a/mkhelp.py b/mkhelp.py old mode 100755 new mode 100644 index 4619457f..3319c8ed --- a/mkhelp.py +++ b/mkhelp.py @@ -1,38 +1,32 @@ #!/usr/bin/env python -import time -import sys +from time import gmtime -time = time.gmtime() -print("/* This file was generated by mkhelp.py from less.hlp at " - "%d:%02d GMT on %d/%d/%d */\n" % - (time.tm_hour, time.tm_min, time.tm_year, time.tm_mon, time.tm_mday)) -print("#include \"less.h\"") -print("constant char helpdata[] = {") -ch = 0 -while True: - prevch = ch - ch = sys.stdin.read(1) - if ch == '': - break - if (ch == "'"): - print("'\\'',", end='') - elif (ch == "\\"): - print("'\\\\',", end='') - elif (ch == "\b"): - print("'\\b',", end='') - elif (ch == "\t"): - print("'\\t',", end='') - elif (ch == "\n"): - if prevch != "\r": - print("'\\n',") - elif (ch == "\r"): - if prevch != "\n": - print("'\\n',") - else: - if ((ord(ch) >= ord(' ')) and (ord(ch) < 0x7f)): - print(f"'{ch}',", end='') - else: - print("0x%02x," % ord(ch), end='') -print(" '\\0' };") -print("constant int size_helpdata = sizeof(helpdata) - 1;") +time = gmtime() +with open('less.c', 'w', newline='') as out, open('less.hlp') as infile: + out.write("/* This file was generated by mkhelp.py from less.hlp at " + "%d:%02d on %d/%d/%d */\n" % + (time.tm_hour, time.tm_min, time.tm_year, time.tm_mon, + time.tm_mday)) + out.write("#include \"less.h\"\n") + out.write("constant char helpdata[] = {\n") + for i, line in enumerate(infile): + for ch in line: + if ch == "'": + out.write("'\\'',") + elif ch == "\\": + out.write("'\\\\',") + elif ch == "\b": + out.write("'\\b',") + elif ch == "\t": + out.write("'\\t',") + elif ch == "\n": + if (ch == line[0] and i == 0) or (len(line) > 1): + out.write("'\\n',\n") + else: + if (ord(ch) >= ord(' ')) and (ord(ch) < 0x7f): + out.write(f"'{ch}',") + else: + out.write("0x%02x," % ord(ch)) + out.write(" 0 };\n") + out.write("constant int size_helpdata = sizeof(helpdata) - 1;\n") From 6b8fe608e9f81129b8b99717edb1d4bb16c95f8b Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Thu, 25 Mar 2021 16:07:20 -0400 Subject: [PATCH 28/56] Commit more Python versions of scripts. WIP --- add_copyright.py | 34 ++++++++++++++ mkutable.py | 113 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+) create mode 100644 add_copyright.py create mode 100644 mkutable.py diff --git a/add_copyright.py b/add_copyright.py new file mode 100644 index 00000000..f61156a8 --- /dev/null +++ b/add_copyright.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 + +from sys import argv, open +from os.path import isadir +import io +import re + +dir = pop(argv) +if (!isadir(dir)): + print(dir + "is not a directory\n") + exit(1) + +copyright = open("copyright", "r") +copyright_line = copyright.read(2) +copyright.close() + +for i in argv: + out = dir + "/" + i + if (!open(i000)): + print("Cannot open {0}\n", i) + pass + if (!open(out)): + print("Cannot create {0}\n", out) + close(i) + pass + while(input): + + close(copyright) + if (exec): + mode = 0555 + else: + mode = 0444 + chmod(out. mode) + diff --git a/mkutable.py b/mkutable.py new file mode 100644 index 00000000..cb1cf14b --- /dev/null +++ b/mkutable.py @@ -0,0 +1,113 @@ +#! /usr/bin/env python + +from argparse import ArgumentParser +from sys import exit +from subprocess import run +from fileinput import input +import re + +# Override Unicode tables for certain control chars +# that are expected to be found in normal text files. +force_space = { + 0x08: 1, # backspace + 0x09: 1, # tab + 0x0a: 1, # newline + 0x0c: 1, # form feed + 0x0d: 1, # carriage return +} + +# Hangul Jamo medial vowels and final consonants should be zero width. +force_compose = [ + [0x1160, 0x11ff], + [0xd7b0, 0xd7c6], + [0xd7cb, 0xd7fb] +] + + +def main() -> int: + parser = ArgumentParser() + parser.add_argument("-n", help="Take non-matching types", + action=store_true) + parser.add_argument("-f", help="Zero-based type field (default 2)", + type=int, default=2) + parser.add_argument("type", nargs='+', type=string, required=True) + # parser.add_argument("--") + # parser.add_argument("<") + parser.add_argument("UnicodeData.txt", required=True) + args = parser.parse_args() + global opt_n + opt_n = args.n + global type_field + type_field = args.f + + types = {} + for arg in sys.argv: + if arg == "--": + break + types[arg] = 1 + + # while ($arg = shift @ARGV) { + # last if $arg eq '--'; + # $types{$arg} = 1; + # } + # out = { 'types': types } + + force_compose = {} + for lo, hi in force_compose: + force_compose = [1 for ch in range(lo, hi)] + + # Only works for Python 3.7+ + date = run("date", capture_output=True, text=True).rstrip('\n') + print(f"/* Generated by \"{sys.argv[0]} {sys.argv[1:]}\" on {date}.") + print(" Python FTW! */") + + last_code = 0 + with open(UnicodeData.txt) as file: + for line in file: + line.rstrip('\n') + line = re.sub(r"s/#.*//", line) + fields = line.split(';') + if not fields: + continue + lo_code = hi_code = 0 + codes = fields[0] + if m := re.match(r"(\w+)\.\.(\w+)", codes): + lo_code = int(m.group(1), 16) + hi_code = int(m.group(2), 16) + else: + lo_code = hi_code = int(codes, 16) + type = fields[type_field] + type = re.match(r"s/\s//g", type) + for last_code in range(lo_code, hi_code): + output(out, last_code, 'Zs' if force_space[last_code] else 'Mn' + if force_compose[last_code] else type) + output(out, last_code) + return 1 + +def output(out: dict, code: int, type: str): + type_ok = (type and ${${$out}{types}}{$type}) + type_ok = not type_ok if opt_n + prev_code = out{prev_code} + + if not type_ok: + end_run(out, prev_code) + elif (not out{in_run} or type ne out{run_type} or code != prev_code+1): + end_run(out, prev_code) + start_run(out, code, type) + out{prev_code} = code + +def start_run(out: dict, code: int, type: str): + out[start_code] = code + out[prev_code] = code + out[run_type] = type + out[in_run] = 1 + +def end_run(out: dict, code: int): + if not out[in_run]: + return + print("\t{ 0x%04x, 0x%04x }, /* %s */" % $$out{start_code}, code, + out{run_type}) + out[in_run] = 0 + +if __name__ == "__main__": + exit(0 if main() else 1) From bd7d4f289bc981037cd9234e4d703e0c78ee487b Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Fri, 26 Mar 2021 15:42:33 -0400 Subject: [PATCH 29/56] Adjust Makefile.aut. --- Makefile.aut | 2 ++ add_copyright.py | 52 ++++++++++++++++++++++-------------------------- 2 files changed, 26 insertions(+), 28 deletions(-) diff --git a/Makefile.aut b/Makefile.aut index f0eb26d4..f4f65fcf 100644 --- a/Makefile.aut +++ b/Makefile.aut @@ -12,9 +12,11 @@ REL := $(shell sed -e '/char version/!d' -e 's/[^0-9.]*\([0-9.]*\).*/\1/' -e q $ ifeq ($(USE_PYTHON),1) MKHELP = mkhelp.py MKFUNCS = mkfuncs.py + ADD_COPYRIGHT = add_copyright.py else MKHELP = mkhelp.pl MKFUNCS = mkfuncs.pl + ADD_COPYRIGHT = add_copyright endif SRC = \ diff --git a/add_copyright.py b/add_copyright.py index f61156a8..bc8bc618 100644 --- a/add_copyright.py +++ b/add_copyright.py @@ -1,34 +1,30 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python -from sys import argv, open -from os.path import isadir -import io +import os +import argparse +import pathlib import re -dir = pop(argv) -if (!isadir(dir)): - print(dir + "is not a directory\n") - exit(1) +parser = argparse.ArgumentParser() +parser.add_argument("file", type=str) +parser.add_argument("dir", type=str) +args = parser.parse_args() +out = pathlib.Path(args.dir) / args.file -copyright = open("copyright", "r") -copyright_line = copyright.read(2) -copyright.close() +copyright = copyright_line = "" +with open("copyright") as f: + copyright = f.read() +for line in copyright.splitlines(): + if '(C)' in line: + copyright_line = line.strip("/*/\r\n").rstrip(" ") + break -for i in argv: - out = dir + "/" + i - if (!open(i000)): - print("Cannot open {0}\n", i) - pass - if (!open(out)): - print("Cannot create {0}\n", out) - close(i) - pass - while(input): +with open(args.file) as input, out.open('w') as o: + for line in input: + if re.search(r"\@\@copyright\@\@", line): + o.write(copyright) + continue + out_line = re.sub(r"\@\@copyright_oneline\@\@", copyright_line, line) + o.write(out_line) - close(copyright) - if (exec): - mode = 0555 - else: - mode = 0444 - chmod(out. mode) - + out.chmod(0o555 if os.access(out, os.X_OK) else 0o444) From e58a5707ac032000c69f8ebd8d64fcc59439bfd9 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Sat, 27 Mar 2021 15:46:43 -0400 Subject: [PATCH 30/56] Finalize conversion of Perl build scripts into Python versions. --- mkfuncs.py | 34 ++++++++++++----------- mkhelp.py | 18 ++++++------ mkutable.py | 80 ++++++++++++++++++++++++++--------------------------- 3 files changed, 68 insertions(+), 64 deletions(-) diff --git a/mkfuncs.py b/mkfuncs.py index 7a2da5ad..c397942f 100755 --- a/mkfuncs.py +++ b/mkfuncs.py @@ -8,19 +8,21 @@ params = 0 for line in fileinput.input(): - if test := re.search(r'^\tpublic\s+(.*)', line): - definition = 'public ' + test.group(1) - state = 1 - params = 0 - elif (state == 1) and (test := re.search(r'(\w+)\s*\(', line)): - definition = '{0} LESSPARAMS (('.format(test.group(1)) - state = 2 - elif state == 2: - if re.search(r'^{', line): - if not params: definition += 'VOID_PARAM' - print(f'{definition}));') - state = 0 - elif test := re.search(r'^\s*([^;]*)', line): - if (definition[-1:] != '('): definition += ', ' - definition += test.group(1) - params = 1 + if test := re.search(r'^\tpublic\s+(.*)', line): + definition = 'public ' + test.group(1) + state = 1 + params = 0 + elif (state == 1) and (test := re.search(r'(\w+)\s*\(', line)): + definition = '{0} LESSPARAMS (('.format(test.group(1)) + state = 2 + elif state == 2: + if re.search(r'^{', line): + if not params: + definition += 'VOID_PARAM' + print(f'{definition}));') + state = 0 + elif test := re.search(r'^\s*([^;]*)', line): + if (definition[-1:] != '('): + definition += ', ' + definition += test.group(1) + params = 1 diff --git a/mkhelp.py b/mkhelp.py index 029f1505..4619457f 100755 --- a/mkhelp.py +++ b/mkhelp.py @@ -4,9 +4,9 @@ import sys time = time.gmtime() -print("/* This file was generated by mkhelp.py from less.hlp at "\ - "%d:%02d GMT on %d/%d/%d */\n" % - (time.tm_hour, time.tm_min, time.tm_year, time.tm_mon, time.tm_mday)) +print("/* This file was generated by mkhelp.py from less.hlp at " + "%d:%02d GMT on %d/%d/%d */\n" % + (time.tm_hour, time.tm_min, time.tm_year, time.tm_mon, time.tm_mday)) print("#include \"less.h\"") print("constant char helpdata[] = {") ch = 0 @@ -16,17 +16,19 @@ if ch == '': break if (ch == "'"): - print("'\\'',", end='') + print("'\\'',", end='') elif (ch == "\\"): print("'\\\\',", end='') elif (ch == "\b"): - print ("'\\b',", end='') + print("'\\b',", end='') elif (ch == "\t"): - print ("'\\t',", end='') + print("'\\t',", end='') elif (ch == "\n"): - if prevch != "\r": print("'\\n',") + if prevch != "\r": + print("'\\n',") elif (ch == "\r"): - if prevch != "\n": print("'\\n',") + if prevch != "\n": + print("'\\n',") else: if ((ord(ch) >= ord(' ')) and (ord(ch) < 0x7f)): print(f"'{ch}',", end='') diff --git a/mkutable.py b/mkutable.py index cb1cf14b..fd76408d 100644 --- a/mkutable.py +++ b/mkutable.py @@ -1,9 +1,8 @@ #! /usr/bin/env python from argparse import ArgumentParser -from sys import exit +import sys from subprocess import run -from fileinput import input import re # Override Unicode tables for certain control chars @@ -27,45 +26,39 @@ def main() -> int: parser = ArgumentParser() parser.add_argument("-n", help="Take non-matching types", - action=store_true) + action='store_true') parser.add_argument("-f", help="Zero-based type field (default 2)", type=int, default=2) - parser.add_argument("type", nargs='+', type=string, required=True) + parser.add_argument("types", nargs='+', type=str) # parser.add_argument("--") # parser.add_argument("<") - parser.add_argument("UnicodeData.txt", required=True) + + # Kludge: cannot specify alternate var for dest= parameter when the positional + # argument's name contains a dot.' + parser.add_argument("data_file", type=open, metavar="UnicodeData.txt") args = parser.parse_args() global opt_n opt_n = args.n global type_field type_field = args.f - types = {} - for arg in sys.argv: - if arg == "--": - break - types[arg] = 1 - - # while ($arg = shift @ARGV) { - # last if $arg eq '--'; - # $types{$arg} = 1; - # } - # out = { 'types': types } + types = {type: 1 for type in args.types} + out = {'types': types} - force_compose = {} - for lo, hi in force_compose: - force_compose = [1 for ch in range(lo, hi)] + global_list = globals() + force_compose = {ch: 1 for lo, hi in global_list['force_compose'] + for ch in range(lo, hi)} # Only works for Python 3.7+ - date = run("date", capture_output=True, text=True).rstrip('\n') - print(f"/* Generated by \"{sys.argv[0]} {sys.argv[1:]}\" on {date}.") - print(" Python FTW! */") + date = run("date", capture_output=True, text=True).stdout.rstrip('\r\n') + print(f"/* Generated by \"{sys.argv[0]} {' '.join(sys.argv[1:])}\" on {date}.") + print(" Python FTW! */") last_code = 0 - with open(UnicodeData.txt) as file: - for line in file: - line.rstrip('\n') - line = re.sub(r"s/#.*//", line) + with args.data_file as file: + for line in file.read(): + line.rstrip('\r\n') + line = re.sub(r"s/#.*//", '', line) fields = line.split(';') if not fields: continue @@ -76,38 +69,45 @@ def main() -> int: hi_code = int(m.group(2), 16) else: lo_code = hi_code = int(codes, 16) + print(fields) + print(type_field) type = fields[type_field] type = re.match(r"s/\s//g", type) for last_code in range(lo_code, hi_code): output(out, last_code, 'Zs' if force_space[last_code] else 'Mn' - if force_compose[last_code] else type) + if force_compose[last_code] else type) output(out, last_code) return 1 + def output(out: dict, code: int, type: str): - type_ok = (type and ${${$out}{types}}{$type}) - type_ok = not type_ok if opt_n - prev_code = out{prev_code} + type_ok = type and out['types'][type] + if opt_n: + type_ok = not type_ok + prev_code = out["prev_code"] if not type_ok: end_run(out, prev_code) - elif (not out{in_run} or type ne out{run_type} or code != prev_code+1): + elif (not out["in_run"] or type != out["run_type"] or code != prev_code+1): end_run(out, prev_code) start_run(out, code, type) - out{prev_code} = code + out[prev_code] = code + def start_run(out: dict, code: int, type: str): - out[start_code] = code - out[prev_code] = code - out[run_type] = type - out[in_run] = 1 + out["start_code"] = code + out["prev_code"] = code + out["run_type"] = type + out["in_run"] = True + def end_run(out: dict, code: int): - if not out[in_run]: + if not out["in_run"]: return - print("\t{ 0x%04x, 0x%04x }, /* %s */" % $$out{start_code}, code, - out{run_type}) - out[in_run] = 0 + print("\t{ 0x%04x, 0x%04x }, /* %s */" % out["start_code"], code, + out["run_type"]) + out["in_run"] = False + if __name__ == "__main__": exit(0 if main() else 1) From e57713c180fabfacffb6665a2fcee012272b9b8c Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Sun, 28 Mar 2021 14:50:30 -0400 Subject: [PATCH 31/56] WIP --- mkutable.py | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/mkutable.py b/mkutable.py index fd76408d..446b4c7b 100644 --- a/mkutable.py +++ b/mkutable.py @@ -1,9 +1,12 @@ #! /usr/bin/env python from argparse import ArgumentParser -import sys +from sys import argv from subprocess import run import re +import friendly + +friendly.install() # Override Unicode tables for certain control chars # that are expected to be found in normal text files. @@ -30,11 +33,9 @@ def main() -> int: parser.add_argument("-f", help="Zero-based type field (default 2)", type=int, default=2) parser.add_argument("types", nargs='+', type=str) - # parser.add_argument("--") - # parser.add_argument("<") - # Kludge: cannot specify alternate var for dest= parameter when the positional - # argument's name contains a dot.' + # Kludge: cannot specify alternate var for dest= parameter when the positional + # argument's name contains a dot. parser.add_argument("data_file", type=open, metavar="UnicodeData.txt") args = parser.parse_args() global opt_n @@ -43,7 +44,8 @@ def main() -> int: type_field = args.f types = {type: 1 for type in args.types} - out = {'types': types} + out = {'types': types, 'prev_code': 0, 'in_run': False, 'run_type': "", + 'start_code': 0} global_list = globals() force_compose = {ch: 1 for lo, hi in global_list['force_compose'] @@ -51,12 +53,12 @@ def main() -> int: # Only works for Python 3.7+ date = run("date", capture_output=True, text=True).stdout.rstrip('\r\n') - print(f"/* Generated by \"{sys.argv[0]} {' '.join(sys.argv[1:])}\" on {date}.") + print(f"/* Generated by \"{argv[0]} {' '.join(argv[1:])}\" on {date}.") print(" Python FTW! */") last_code = 0 with args.data_file as file: - for line in file.read(): + for line in file: line.rstrip('\r\n') line = re.sub(r"s/#.*//", '', line) fields = line.split(';') @@ -69,8 +71,6 @@ def main() -> int: hi_code = int(m.group(2), 16) else: lo_code = hi_code = int(codes, 16) - print(fields) - print(type_field) type = fields[type_field] type = re.match(r"s/\s//g", type) for last_code in range(lo_code, hi_code): @@ -80,12 +80,18 @@ def main() -> int: return 1 -def output(out: dict, code: int, type: str): - type_ok = type and out['types'][type] +def output(out: dict, code: int, type: str = None): + type_ok = type is not None and type in out['types'].keys() if opt_n: type_ok = not type_ok - prev_code = out["prev_code"] + prev_code = out['prev_code'] + + print(out['types'].keys()) + import pprint + p = pprint.PrettyPrinter() + p.pprint(out) + print(type) if not type_ok: end_run(out, prev_code) elif (not out["in_run"] or type != out["run_type"] or code != prev_code+1): @@ -108,6 +114,5 @@ def end_run(out: dict, code: int): out["run_type"]) out["in_run"] = False - if __name__ == "__main__": exit(0 if main() else 1) From 6cde79d586de0aa5e6defbea7716430763fad9c3 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Thu, 25 Mar 2021 16:07:20 -0400 Subject: [PATCH 32/56] Commit more Python versions of scripts. WIP --- mkutable.py | 47 +++++++++++++++++------------------------------ 1 file changed, 17 insertions(+), 30 deletions(-) diff --git a/mkutable.py b/mkutable.py index 446b4c7b..c8dcf95d 100644 --- a/mkutable.py +++ b/mkutable.py @@ -1,12 +1,10 @@ #! /usr/bin/env python from argparse import ArgumentParser -from sys import argv +from sys import exit from subprocess import run +from fileinput import input import re -import friendly - -friendly.install() # Override Unicode tables for certain control chars # that are expected to be found in normal text files. @@ -42,7 +40,6 @@ def main() -> int: opt_n = args.n global type_field type_field = args.f - types = {type: 1 for type in args.types} out = {'types': types, 'prev_code': 0, 'in_run': False, 'run_type': "", 'start_code': 0} @@ -75,44 +72,34 @@ def main() -> int: type = re.match(r"s/\s//g", type) for last_code in range(lo_code, hi_code): output(out, last_code, 'Zs' if force_space[last_code] else 'Mn' - if force_compose[last_code] else type) + if force_compose[last_code] else type) output(out, last_code) return 1 +def output(out: dict, code: int, type: str): + type_ok = (type and ${${$out}{types}}{$type}) + type_ok = not type_ok if opt_n + prev_code = out{prev_code} -def output(out: dict, code: int, type: str = None): - type_ok = type is not None and type in out['types'].keys() - if opt_n: - type_ok = not type_ok - prev_code = out['prev_code'] - - print(out['types'].keys()) - import pprint - p = pprint.PrettyPrinter() - p.pprint(out) - - print(type) if not type_ok: end_run(out, prev_code) - elif (not out["in_run"] or type != out["run_type"] or code != prev_code+1): + elif (not out{in_run} or type ne out{run_type} or code != prev_code+1): end_run(out, prev_code) start_run(out, code, type) - out[prev_code] = code - + out{prev_code} = code def start_run(out: dict, code: int, type: str): - out["start_code"] = code - out["prev_code"] = code - out["run_type"] = type - out["in_run"] = True - + out[start_code] = code + out[prev_code] = code + out[run_type] = type + out[in_run] = 1 def end_run(out: dict, code: int): - if not out["in_run"]: + if not out[in_run]: return - print("\t{ 0x%04x, 0x%04x }, /* %s */" % out["start_code"], code, - out["run_type"]) - out["in_run"] = False + print("\t{ 0x%04x, 0x%04x }, /* %s */" % $$out{start_code}, code, + out{run_type}) + out[in_run] = 0 if __name__ == "__main__": exit(0 if main() else 1) From be8b86d24da5c904cbcc3b4a9bd279e1ec9dea3e Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Sat, 27 Mar 2021 15:46:43 -0400 Subject: [PATCH 33/56] Finalize conversion of Perl build scripts into Python versions. --- mkutable.py | 45 ++++++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/mkutable.py b/mkutable.py index c8dcf95d..20957294 100644 --- a/mkutable.py +++ b/mkutable.py @@ -1,9 +1,8 @@ #! /usr/bin/env python from argparse import ArgumentParser -from sys import exit +import sys from subprocess import run -from fileinput import input import re # Override Unicode tables for certain control chars @@ -31,7 +30,6 @@ def main() -> int: parser.add_argument("-f", help="Zero-based type field (default 2)", type=int, default=2) parser.add_argument("types", nargs='+', type=str) - # Kludge: cannot specify alternate var for dest= parameter when the positional # argument's name contains a dot. parser.add_argument("data_file", type=open, metavar="UnicodeData.txt") @@ -43,20 +41,18 @@ def main() -> int: types = {type: 1 for type in args.types} out = {'types': types, 'prev_code': 0, 'in_run': False, 'run_type': "", 'start_code': 0} - global_list = globals() force_compose = {ch: 1 for lo, hi in global_list['force_compose'] for ch in range(lo, hi)} # Only works for Python 3.7+ date = run("date", capture_output=True, text=True).stdout.rstrip('\r\n') - print(f"/* Generated by \"{argv[0]} {' '.join(argv[1:])}\" on {date}.") + print(f"/* Generated by \"{sys.argv[0]} {' '.join(sys.argv[1:])}\" on {date}." print(" Python FTW! */") last_code = 0 with args.data_file as file: - for line in file: - line.rstrip('\r\n') + for line in file:line.rstrip('\r\n') line = re.sub(r"s/#.*//", '', line) fields = line.split(';') if not fields: @@ -68,38 +64,45 @@ def main() -> int: hi_code = int(m.group(2), 16) else: lo_code = hi_code = int(codes, 16) + print(fields) + print(type_field) type = fields[type_field] type = re.match(r"s/\s//g", type) for last_code in range(lo_code, hi_code): output(out, last_code, 'Zs' if force_space[last_code] else 'Mn' - if force_compose[last_code] else type) + if force_compose[last_code] else type) output(out, last_code) return 1 + def output(out: dict, code: int, type: str): - type_ok = (type and ${${$out}{types}}{$type}) - type_ok = not type_ok if opt_n - prev_code = out{prev_code} + type_ok = type and out['types'][type] + if opt_n: + type_ok = not type_ok + prev_code = out["prev_code"] if not type_ok: end_run(out, prev_code) - elif (not out{in_run} or type ne out{run_type} or code != prev_code+1): + elif (not out["in_run"] or type != out["run_type"] or code != prev_code+1): end_run(out, prev_code) start_run(out, code, type) - out{prev_code} = code + out[prev_code] = code + def start_run(out: dict, code: int, type: str): - out[start_code] = code - out[prev_code] = code - out[run_type] = type - out[in_run] = 1 + out["start_code"] = code + out["prev_code"] = code + out["run_type"] = type + out["in_run"] = True + def end_run(out: dict, code: int): - if not out[in_run]: + if not out["in_run"]: return - print("\t{ 0x%04x, 0x%04x }, /* %s */" % $$out{start_code}, code, - out{run_type}) - out[in_run] = 0 + print("\t{ 0x%04x, 0x%04x }, /* %s */" % out["start_code"], code, + out["run_type"]) + out["in_run"] = False + if __name__ == "__main__": exit(0 if main() else 1) From 76b37afa4dc15b9d4939f0bcf9c731fd2ef3a73f Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Sun, 28 Mar 2021 14:50:30 -0400 Subject: [PATCH 34/56] Debug in progress... --- mkutable.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/mkutable.py b/mkutable.py index 20957294..36c67f47 100644 --- a/mkutable.py +++ b/mkutable.py @@ -1,9 +1,12 @@ #! /usr/bin/env python from argparse import ArgumentParser -import sys +from sys import argv from subprocess import run import re +import friendly + +friendly.install() # Override Unicode tables for certain control chars # that are expected to be found in normal text files. @@ -52,7 +55,8 @@ def main() -> int: last_code = 0 with args.data_file as file: - for line in file:line.rstrip('\r\n') + for line in file: + line.rstrip('\r\n') line = re.sub(r"s/#.*//", '', line) fields = line.split(';') if not fields: @@ -64,8 +68,6 @@ def main() -> int: hi_code = int(m.group(2), 16) else: lo_code = hi_code = int(codes, 16) - print(fields) - print(type_field) type = fields[type_field] type = re.match(r"s/\s//g", type) for last_code in range(lo_code, hi_code): @@ -75,12 +77,18 @@ def main() -> int: return 1 -def output(out: dict, code: int, type: str): - type_ok = type and out['types'][type] +def output(out: dict, code: int, type: str = None): + type_ok = type is not None and type in out['types'].keys() if opt_n: type_ok = not type_ok - prev_code = out["prev_code"] + prev_code = out['prev_code'] + + print(out['types'].keys()) + import pprint + p = pprint.PrettyPrinter() + p.pprint(out) + print(type) if not type_ok: end_run(out, prev_code) elif (not out["in_run"] or type != out["run_type"] or code != prev_code+1): From a25bac30c18811376bde35bdabca441ffe106d32 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Thu, 20 May 2021 14:37:56 -0400 Subject: [PATCH 35/56] Fourth time? --- mkutable.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/mkutable.py b/mkutable.py index 36c67f47..e469f145 100644 --- a/mkutable.py +++ b/mkutable.py @@ -1,12 +1,11 @@ #! /usr/bin/env python from argparse import ArgumentParser -from sys import argv +from sys import exit from subprocess import run +from fileinput import input import re -import friendly - -friendly.install() +>>>>>>> 27ef7b3 (Commit more Python versions of scripts. WIP) # Override Unicode tables for certain control chars # that are expected to be found in normal text files. @@ -111,6 +110,5 @@ def end_run(out: dict, code: int): out["run_type"]) out["in_run"] = False - if __name__ == "__main__": exit(0 if main() else 1) From 7d3b755b6a63dac080891cd2f1ea39224584394e Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Thu, 25 Mar 2021 16:07:20 -0400 Subject: [PATCH 36/56] Commit more Python versions of scripts. WIP --- add_copyright.py | 30 ------------------------------ mkutable.py | 1 - 2 files changed, 31 deletions(-) delete mode 100644 add_copyright.py diff --git a/add_copyright.py b/add_copyright.py deleted file mode 100644 index bc8bc618..00000000 --- a/add_copyright.py +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env python - -import os -import argparse -import pathlib -import re - -parser = argparse.ArgumentParser() -parser.add_argument("file", type=str) -parser.add_argument("dir", type=str) -args = parser.parse_args() -out = pathlib.Path(args.dir) / args.file - -copyright = copyright_line = "" -with open("copyright") as f: - copyright = f.read() -for line in copyright.splitlines(): - if '(C)' in line: - copyright_line = line.strip("/*/\r\n").rstrip(" ") - break - -with open(args.file) as input, out.open('w') as o: - for line in input: - if re.search(r"\@\@copyright\@\@", line): - o.write(copyright) - continue - out_line = re.sub(r"\@\@copyright_oneline\@\@", copyright_line, line) - o.write(out_line) - - out.chmod(0o555 if os.access(out, os.X_OK) else 0o444) diff --git a/mkutable.py b/mkutable.py index e469f145..4cc9bb47 100644 --- a/mkutable.py +++ b/mkutable.py @@ -5,7 +5,6 @@ from subprocess import run from fileinput import input import re ->>>>>>> 27ef7b3 (Commit more Python versions of scripts. WIP) # Override Unicode tables for certain control chars # that are expected to be found in normal text files. From c55f7140ca6dd8fdf9ef216d421351f5ad792f6a Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Sat, 27 Mar 2021 15:46:43 -0400 Subject: [PATCH 37/56] Finalize conversion of Perl build scripts into Python versions. --- mkutable.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mkutable.py b/mkutable.py index 4cc9bb47..03646203 100644 --- a/mkutable.py +++ b/mkutable.py @@ -1,9 +1,8 @@ #! /usr/bin/env python from argparse import ArgumentParser -from sys import exit +import sys from subprocess import run -from fileinput import input import re # Override Unicode tables for certain control chars @@ -66,6 +65,8 @@ def main() -> int: hi_code = int(m.group(2), 16) else: lo_code = hi_code = int(codes, 16) + print(fields) + print(type_field) type = fields[type_field] type = re.match(r"s/\s//g", type) for last_code in range(lo_code, hi_code): @@ -74,7 +75,6 @@ def main() -> int: output(out, last_code) return 1 - def output(out: dict, code: int, type: str = None): type_ok = type is not None and type in out['types'].keys() if opt_n: From d0303aba13ed7df39ae1d7763349b6c0e911e264 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Sun, 28 Mar 2021 14:50:30 -0400 Subject: [PATCH 38/56] WIP --- mkutable.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/mkutable.py b/mkutable.py index 03646203..0e17ec03 100644 --- a/mkutable.py +++ b/mkutable.py @@ -1,9 +1,12 @@ #! /usr/bin/env python from argparse import ArgumentParser -import sys +from sys import argv from subprocess import run import re +import friendly + +friendly.install() # Override Unicode tables for certain control chars # that are expected to be found in normal text files. @@ -30,6 +33,7 @@ def main() -> int: parser.add_argument("-f", help="Zero-based type field (default 2)", type=int, default=2) parser.add_argument("types", nargs='+', type=str) + # Kludge: cannot specify alternate var for dest= parameter when the positional # argument's name contains a dot. parser.add_argument("data_file", type=open, metavar="UnicodeData.txt") @@ -47,7 +51,7 @@ def main() -> int: # Only works for Python 3.7+ date = run("date", capture_output=True, text=True).stdout.rstrip('\r\n') - print(f"/* Generated by \"{sys.argv[0]} {' '.join(sys.argv[1:])}\" on {date}." + print(f"/* Generated by \"{argv[0]} {' '.join(sys.argv[1:])}\" on {date}." print(" Python FTW! */") last_code = 0 @@ -65,8 +69,6 @@ def main() -> int: hi_code = int(m.group(2), 16) else: lo_code = hi_code = int(codes, 16) - print(fields) - print(type_field) type = fields[type_field] type = re.match(r"s/\s//g", type) for last_code in range(lo_code, hi_code): From 692cdd24fbd827ead1a3a53d69f42274a7f86c98 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Sat, 27 Mar 2021 15:46:43 -0400 Subject: [PATCH 39/56] Finalize conversion of Perl build scripts into Python versions. --- mkutable.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mkutable.py b/mkutable.py index 0e17ec03..c1283609 100644 --- a/mkutable.py +++ b/mkutable.py @@ -33,7 +33,6 @@ def main() -> int: parser.add_argument("-f", help="Zero-based type field (default 2)", type=int, default=2) parser.add_argument("types", nargs='+', type=str) - # Kludge: cannot specify alternate var for dest= parameter when the positional # argument's name contains a dot. parser.add_argument("data_file", type=open, metavar="UnicodeData.txt") @@ -56,8 +55,7 @@ def main() -> int: last_code = 0 with args.data_file as file: - for line in file: - line.rstrip('\r\n') + for line in file:line.rstrip('\r\n') line = re.sub(r"s/#.*//", '', line) fields = line.split(';') if not fields: @@ -69,6 +67,8 @@ def main() -> int: hi_code = int(m.group(2), 16) else: lo_code = hi_code = int(codes, 16) + print(fields) + print(type_field) type = fields[type_field] type = re.match(r"s/\s//g", type) for last_code in range(lo_code, hi_code): @@ -77,11 +77,11 @@ def main() -> int: output(out, last_code) return 1 -def output(out: dict, code: int, type: str = None): - type_ok = type is not None and type in out['types'].keys() +def output(out: dict, code: int, type: str): + type_ok = type and out['types'][type] if opt_n: type_ok = not type_ok - prev_code = out['prev_code'] + prev_code = out["prev_code"] print(out['types'].keys()) import pprint From 6ae35b81bbbf85dd4590df3eee9f150abe7e15ce Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Sun, 28 Mar 2021 14:50:30 -0400 Subject: [PATCH 40/56] Debug in progress... --- mkutable.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/mkutable.py b/mkutable.py index c1283609..6dfabdcf 100644 --- a/mkutable.py +++ b/mkutable.py @@ -55,7 +55,8 @@ def main() -> int: last_code = 0 with args.data_file as file: - for line in file:line.rstrip('\r\n') + for line in file: + line.rstrip('\r\n') line = re.sub(r"s/#.*//", '', line) fields = line.split(';') if not fields: @@ -67,8 +68,6 @@ def main() -> int: hi_code = int(m.group(2), 16) else: lo_code = hi_code = int(codes, 16) - print(fields) - print(type_field) type = fields[type_field] type = re.match(r"s/\s//g", type) for last_code in range(lo_code, hi_code): @@ -81,14 +80,12 @@ def output(out: dict, code: int, type: str): type_ok = type and out['types'][type] if opt_n: type_ok = not type_ok - prev_code = out["prev_code"] + prev_code = out['prev_code'] print(out['types'].keys()) import pprint p = pprint.PrettyPrinter() p.pprint(out) - - print(type) if not type_ok: end_run(out, prev_code) elif (not out["in_run"] or type != out["run_type"] or code != prev_code+1): From 223a440a514d4013357c541ddda6ca3d3ec38824 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Thu, 25 Mar 2021 16:07:20 -0400 Subject: [PATCH 41/56] Commit more Python versions of scripts. WIP --- add_copyright.py | 34 ++++++++++++++++++++++++++++++++++ mkutable.py | 48 +++++++++++++++++++++++------------------------- 2 files changed, 57 insertions(+), 25 deletions(-) create mode 100644 add_copyright.py diff --git a/add_copyright.py b/add_copyright.py new file mode 100644 index 00000000..f61156a8 --- /dev/null +++ b/add_copyright.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 + +from sys import argv, open +from os.path import isadir +import io +import re + +dir = pop(argv) +if (!isadir(dir)): + print(dir + "is not a directory\n") + exit(1) + +copyright = open("copyright", "r") +copyright_line = copyright.read(2) +copyright.close() + +for i in argv: + out = dir + "/" + i + if (!open(i000)): + print("Cannot open {0}\n", i) + pass + if (!open(out)): + print("Cannot create {0}\n", out) + close(i) + pass + while(input): + + close(copyright) + if (exec): + mode = 0555 + else: + mode = 0444 + chmod(out. mode) + diff --git a/mkutable.py b/mkutable.py index 6dfabdcf..7f8eedeb 100644 --- a/mkutable.py +++ b/mkutable.py @@ -1,12 +1,10 @@ #! /usr/bin/env python from argparse import ArgumentParser -from sys import argv +from sys import exit from subprocess import run +from fileinput import input import re -import friendly - -friendly.install() # Override Unicode tables for certain control chars # that are expected to be found in normal text files. @@ -53,52 +51,52 @@ def main() -> int: print(f"/* Generated by \"{argv[0]} {' '.join(sys.argv[1:])}\" on {date}." print(" Python FTW! */") - last_code = 0 + last_code=0 with args.data_file as file: for line in file: line.rstrip('\r\n') - line = re.sub(r"s/#.*//", '', line) - fields = line.split(';') + line=re.sub(r"s/#.*//", '', line) + fields=line.split(';') if not fields: continue - lo_code = hi_code = 0 - codes = fields[0] + lo_code=hi_code=0 + codes=fields[0] if m := re.match(r"(\w+)\.\.(\w+)", codes): - lo_code = int(m.group(1), 16) - hi_code = int(m.group(2), 16) + lo_code=int(m.group(1), 16) + hi_code=int(m.group(2), 16) else: - lo_code = hi_code = int(codes, 16) - type = fields[type_field] - type = re.match(r"s/\s//g", type) + lo_code=hi_code=int(codes, 16) + type=fields[type_field] + type=re.match(r"s/\s//g", type) for last_code in range(lo_code, hi_code): output(out, last_code, 'Zs' if force_space[last_code] else 'Mn' - if force_compose[last_code] else type) + if force_compose[last_code] else type) output(out, last_code) return 1 def output(out: dict, code: int, type: str): - type_ok = type and out['types'][type] + type_ok=type and out['types'][type] if opt_n: - type_ok = not type_ok - prev_code = out['prev_code'] + type_ok=not type_ok + prev_code=out['prev_code'] print(out['types'].keys()) import pprint - p = pprint.PrettyPrinter() + p=pprint.PrettyPrinter() p.pprint(out) if not type_ok: end_run(out, prev_code) elif (not out["in_run"] or type != out["run_type"] or code != prev_code+1): end_run(out, prev_code) start_run(out, code, type) - out[prev_code] = code + out[prev_code]=code def start_run(out: dict, code: int, type: str): - out["start_code"] = code - out["prev_code"] = code - out["run_type"] = type - out["in_run"] = True + out["start_code"]=code + out["prev_code"]=code + out["run_type"]=type + out["in_run"]=True def end_run(out: dict, code: int): @@ -106,7 +104,7 @@ def end_run(out: dict, code: int): return print("\t{ 0x%04x, 0x%04x }, /* %s */" % out["start_code"], code, out["run_type"]) - out["in_run"] = False + out["in_run"]=False if __name__ == "__main__": exit(0 if main() else 1) From 14582480ff3bd53c67e05e95026f8e148121069a Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Fri, 26 Mar 2021 15:42:33 -0400 Subject: [PATCH 42/56] Adjust Makefile.aut accordingly. --- add_copyright.py | 34 ---------------------------------- 1 file changed, 34 deletions(-) delete mode 100644 add_copyright.py diff --git a/add_copyright.py b/add_copyright.py deleted file mode 100644 index f61156a8..00000000 --- a/add_copyright.py +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env python3 - -from sys import argv, open -from os.path import isadir -import io -import re - -dir = pop(argv) -if (!isadir(dir)): - print(dir + "is not a directory\n") - exit(1) - -copyright = open("copyright", "r") -copyright_line = copyright.read(2) -copyright.close() - -for i in argv: - out = dir + "/" + i - if (!open(i000)): - print("Cannot open {0}\n", i) - pass - if (!open(out)): - print("Cannot create {0}\n", out) - close(i) - pass - while(input): - - close(copyright) - if (exec): - mode = 0555 - else: - mode = 0444 - chmod(out. mode) - From 146bbeda72295abc1b6bd48357a29b41430dd511 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Sat, 27 Mar 2021 15:46:43 -0400 Subject: [PATCH 43/56] Finalize conversion of Perl build scripts into Python versions. --- mkutable.py | 42 +++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/mkutable.py b/mkutable.py index 7f8eedeb..91922b97 100644 --- a/mkutable.py +++ b/mkutable.py @@ -1,9 +1,8 @@ #! /usr/bin/env python from argparse import ArgumentParser -from sys import exit +import sys from subprocess import run -from fileinput import input import re # Override Unicode tables for certain control chars @@ -39,24 +38,31 @@ def main() -> int: opt_n = args.n global type_field type_field = args.f +<<<<<<< HEAD types = {type: 1 for type in args.types} out = {'types': types, 'prev_code': 0, 'in_run': False, 'run_type': "", 'start_code': 0} +======= + + types = {type: 1 for type in args.types} + out = {'types': types} + +>>>>>>> 40c61ae (Finalize conversion of Perl build scripts into Python versions.) global_list = globals() force_compose = {ch: 1 for lo, hi in global_list['force_compose'] for ch in range(lo, hi)} # Only works for Python 3.7+ date = run("date", capture_output=True, text=True).stdout.rstrip('\r\n') - print(f"/* Generated by \"{argv[0]} {' '.join(sys.argv[1:])}\" on {date}." + print(f"/* Generated by \"{sys.argv[0]} {' '.join(sys.argv[1:])}\" on {date}.") print(" Python FTW! */") - last_code=0 + last_code = 0 with args.data_file as file: - for line in file: + for line in file.read(): line.rstrip('\r\n') - line=re.sub(r"s/#.*//", '', line) - fields=line.split(';') + line = re.sub(r"s/#.*//", '', line) + fields = line.split(';') if not fields: continue lo_code=hi_code=0 @@ -70,15 +76,16 @@ def main() -> int: type=re.match(r"s/\s//g", type) for last_code in range(lo_code, hi_code): output(out, last_code, 'Zs' if force_space[last_code] else 'Mn' - if force_compose[last_code] else type) + if force_compose[last_code] else type) output(out, last_code) return 1 + def output(out: dict, code: int, type: str): - type_ok=type and out['types'][type] + type_ok = type and out['types'][type] if opt_n: - type_ok=not type_ok - prev_code=out['prev_code'] + type_ok = not type_ok + prev_code = out["prev_code"] print(out['types'].keys()) import pprint @@ -89,14 +96,14 @@ def output(out: dict, code: int, type: str): elif (not out["in_run"] or type != out["run_type"] or code != prev_code+1): end_run(out, prev_code) start_run(out, code, type) - out[prev_code]=code + out[prev_code] = code def start_run(out: dict, code: int, type: str): - out["start_code"]=code - out["prev_code"]=code - out["run_type"]=type - out["in_run"]=True + out["start_code"] = code + out["prev_code"] = code + out["run_type"] = type + out["in_run"] = True def end_run(out: dict, code: int): @@ -104,7 +111,8 @@ def end_run(out: dict, code: int): return print("\t{ 0x%04x, 0x%04x }, /* %s */" % out["start_code"], code, out["run_type"]) - out["in_run"]=False + out["in_run"] = False + if __name__ == "__main__": exit(0 if main() else 1) From 07c0751825a7b47af97ccf8f6f02bee863dcbe91 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Sun, 28 Mar 2021 14:50:30 -0400 Subject: [PATCH 44/56] WIP --- mkutable.py | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/mkutable.py b/mkutable.py index 91922b97..f60c9175 100644 --- a/mkutable.py +++ b/mkutable.py @@ -1,9 +1,12 @@ #! /usr/bin/env python from argparse import ArgumentParser -import sys +from sys import argv from subprocess import run import re +import friendly + +friendly.install() # Override Unicode tables for certain control chars # that are expected to be found in normal text files. @@ -38,28 +41,23 @@ def main() -> int: opt_n = args.n global type_field type_field = args.f -<<<<<<< HEAD + types = {type: 1 for type in args.types} out = {'types': types, 'prev_code': 0, 'in_run': False, 'run_type': "", 'start_code': 0} -======= - types = {type: 1 for type in args.types} - out = {'types': types} - ->>>>>>> 40c61ae (Finalize conversion of Perl build scripts into Python versions.) global_list = globals() force_compose = {ch: 1 for lo, hi in global_list['force_compose'] for ch in range(lo, hi)} # Only works for Python 3.7+ date = run("date", capture_output=True, text=True).stdout.rstrip('\r\n') - print(f"/* Generated by \"{sys.argv[0]} {' '.join(sys.argv[1:])}\" on {date}.") + print(f"/* Generated by \"{argv[0]} {' '.join(argv[1:])}\" on {date}.") print(" Python FTW! */") last_code = 0 with args.data_file as file: - for line in file.read(): + for line in file: line.rstrip('\r\n') line = re.sub(r"s/#.*//", '', line) fields = line.split(';') @@ -71,9 +69,9 @@ def main() -> int: lo_code=int(m.group(1), 16) hi_code=int(m.group(2), 16) else: - lo_code=hi_code=int(codes, 16) - type=fields[type_field] - type=re.match(r"s/\s//g", type) + lo_code = hi_code = int(codes, 16) + type = fields[type_field] + type = re.match(r"s/\s//g", type) for last_code in range(lo_code, hi_code): output(out, last_code, 'Zs' if force_space[last_code] else 'Mn' if force_compose[last_code] else type) @@ -81,16 +79,12 @@ def main() -> int: return 1 -def output(out: dict, code: int, type: str): - type_ok = type and out['types'][type] +def output(out: dict, code: int, type: str = None): + type_ok = type is not None and type in out['types'].keys() if opt_n: type_ok = not type_ok - prev_code = out["prev_code"] + prev_code = out['prev_code'] - print(out['types'].keys()) - import pprint - p=pprint.PrettyPrinter() - p.pprint(out) if not type_ok: end_run(out, prev_code) elif (not out["in_run"] or type != out["run_type"] or code != prev_code+1): @@ -113,6 +107,5 @@ def end_run(out: dict, code: int): out["run_type"]) out["in_run"] = False - if __name__ == "__main__": exit(0 if main() else 1) From a08e9ffe6e3cb240fc06681fbceef8189b795363 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Sun, 28 Mar 2021 14:50:30 -0400 Subject: [PATCH 45/56] Debug in progress... --- mkutable.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mkutable.py b/mkutable.py index f60c9175..8c9155e2 100644 --- a/mkutable.py +++ b/mkutable.py @@ -85,6 +85,7 @@ def output(out: dict, code: int, type: str = None): type_ok = not type_ok prev_code = out['prev_code'] + print(type) if not type_ok: end_run(out, prev_code) elif (not out["in_run"] or type != out["run_type"] or code != prev_code+1): From 695a9783ca4b5405232698223a03bdce0e939745 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Thu, 25 Mar 2021 16:07:20 -0400 Subject: [PATCH 46/56] Commit more Python versions of scripts. WIP --- add_copyright.py | 34 ++++++++++++++++++++++++++++++++++ mkutable.py | 7 +++++++ 2 files changed, 41 insertions(+) create mode 100644 add_copyright.py diff --git a/add_copyright.py b/add_copyright.py new file mode 100644 index 00000000..f61156a8 --- /dev/null +++ b/add_copyright.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 + +from sys import argv, open +from os.path import isadir +import io +import re + +dir = pop(argv) +if (!isadir(dir)): + print(dir + "is not a directory\n") + exit(1) + +copyright = open("copyright", "r") +copyright_line = copyright.read(2) +copyright.close() + +for i in argv: + out = dir + "/" + i + if (!open(i000)): + print("Cannot open {0}\n", i) + pass + if (!open(out)): + print("Cannot create {0}\n", out) + close(i) + pass + while(input): + + close(copyright) + if (exec): + mode = 0555 + else: + mode = 0444 + chmod(out. mode) + diff --git a/mkutable.py b/mkutable.py index 8c9155e2..52b2a28e 100644 --- a/mkutable.py +++ b/mkutable.py @@ -1,12 +1,19 @@ #! /usr/bin/env python from argparse import ArgumentParser +<<<<<<< HEAD from sys import argv from subprocess import run import re import friendly friendly.install() +======= +from sys import exit +from subprocess import run +from fileinput import input +import re +>>>>>>> d2f9e16 (Commit more Python versions of scripts. WIP) # Override Unicode tables for certain control chars # that are expected to be found in normal text files. From 2b35fdcdf40f4e9366011e288ae092109c657bc8 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Fri, 26 Mar 2021 15:42:33 -0400 Subject: [PATCH 47/56] Adjust Makefile.aut accordingly. --- add_copyright.py | 52 ++++++++++++++++++++++-------------------------- mkutable.py | 9 --------- 2 files changed, 24 insertions(+), 37 deletions(-) diff --git a/add_copyright.py b/add_copyright.py index f61156a8..bc8bc618 100644 --- a/add_copyright.py +++ b/add_copyright.py @@ -1,34 +1,30 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python -from sys import argv, open -from os.path import isadir -import io +import os +import argparse +import pathlib import re -dir = pop(argv) -if (!isadir(dir)): - print(dir + "is not a directory\n") - exit(1) +parser = argparse.ArgumentParser() +parser.add_argument("file", type=str) +parser.add_argument("dir", type=str) +args = parser.parse_args() +out = pathlib.Path(args.dir) / args.file -copyright = open("copyright", "r") -copyright_line = copyright.read(2) -copyright.close() +copyright = copyright_line = "" +with open("copyright") as f: + copyright = f.read() +for line in copyright.splitlines(): + if '(C)' in line: + copyright_line = line.strip("/*/\r\n").rstrip(" ") + break -for i in argv: - out = dir + "/" + i - if (!open(i000)): - print("Cannot open {0}\n", i) - pass - if (!open(out)): - print("Cannot create {0}\n", out) - close(i) - pass - while(input): +with open(args.file) as input, out.open('w') as o: + for line in input: + if re.search(r"\@\@copyright\@\@", line): + o.write(copyright) + continue + out_line = re.sub(r"\@\@copyright_oneline\@\@", copyright_line, line) + o.write(out_line) - close(copyright) - if (exec): - mode = 0555 - else: - mode = 0444 - chmod(out. mode) - + out.chmod(0o555 if os.access(out, os.X_OK) else 0o444) diff --git a/mkutable.py b/mkutable.py index 52b2a28e..a609a5c3 100644 --- a/mkutable.py +++ b/mkutable.py @@ -1,19 +1,10 @@ #! /usr/bin/env python from argparse import ArgumentParser -<<<<<<< HEAD -from sys import argv -from subprocess import run -import re -import friendly - -friendly.install() -======= from sys import exit from subprocess import run from fileinput import input import re ->>>>>>> d2f9e16 (Commit more Python versions of scripts. WIP) # Override Unicode tables for certain control chars # that are expected to be found in normal text files. From 6d08bad15bee3731a7d5c0fc1fdd788a53dae7e5 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Sat, 27 Mar 2021 15:46:43 -0400 Subject: [PATCH 48/56] Finalize conversion of Perl build scripts into Python versions. --- mkutable.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/mkutable.py b/mkutable.py index a609a5c3..92991566 100644 --- a/mkutable.py +++ b/mkutable.py @@ -1,9 +1,8 @@ #! /usr/bin/env python from argparse import ArgumentParser -from sys import exit +from sys import argv from subprocess import run -from fileinput import input import re # Override Unicode tables for certain control chars @@ -68,6 +67,8 @@ def main() -> int: hi_code=int(m.group(2), 16) else: lo_code = hi_code = int(codes, 16) + print(fields) + print(type_field) type = fields[type_field] type = re.match(r"s/\s//g", type) for last_code in range(lo_code, hi_code): @@ -77,11 +78,11 @@ def main() -> int: return 1 -def output(out: dict, code: int, type: str = None): - type_ok = type is not None and type in out['types'].keys() +def output(out: dict, code: int, type: str): + type_ok = type and out['types'][type] if opt_n: type_ok = not type_ok - prev_code = out['prev_code'] + prev_code = out["prev_code"] print(type) if not type_ok: From 3e0f67128ec836859ab1514bb6cab1718dbf02fb Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Sun, 28 Mar 2021 14:50:30 -0400 Subject: [PATCH 49/56] Debug in progress... --- mkutable.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/mkutable.py b/mkutable.py index 92991566..e5c60ecf 100644 --- a/mkutable.py +++ b/mkutable.py @@ -4,6 +4,9 @@ from sys import argv from subprocess import run import re +import friendly + +friendly.install() # Override Unicode tables for certain control chars # that are expected to be found in normal text files. @@ -67,8 +70,6 @@ def main() -> int: hi_code=int(m.group(2), 16) else: lo_code = hi_code = int(codes, 16) - print(fields) - print(type_field) type = fields[type_field] type = re.match(r"s/\s//g", type) for last_code in range(lo_code, hi_code): @@ -78,11 +79,16 @@ def main() -> int: return 1 -def output(out: dict, code: int, type: str): - type_ok = type and out['types'][type] +def output(out: dict, code: int, type: str = None): + type_ok = type is not None and type in out['types'].keys() if opt_n: type_ok = not type_ok - prev_code = out["prev_code"] + prev_code = out['prev_code'] + + print(out['types'].keys()) + import pprint + p = pprint.PrettyPrinter() + p.pprint(out) print(type) if not type_ok: From 75cef10fe2312740e32ae7120cb6fa3675ae99ef Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Wed, 26 May 2021 10:26:54 -0400 Subject: [PATCH 50/56] Add Python equivalents to scripts in lesstest. Signed-off-by: Charlie Lin --- lesstest/gen.py | 67 ++++++++++++++++++++++++++++++++++++++++++++++ lesstest/run.py | 71 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 lesstest/gen.py create mode 100644 lesstest/run.py diff --git a/lesstest/gen.py b/lesstest/gen.py new file mode 100644 index 00000000..ea427829 --- /dev/null +++ b/lesstest/gen.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python + +from argparse import ArgumentParser, FileType +from subprocess import run +from sys import exit, stderr, argv +from itertools import count +from pathlib import Path +from re import search +from os import environ + +lesstest = "./lesstest" +outdir = Path("./suite") + + +def main(): + pars = ArgumentParser(description="Generate a lesstest file.", add_help=False) + pars.add_argument("-w", help="width", type=int) + pars.add_argument("-h", help="height", type=int) + pars.add_argument("--lessvar", default=environ.get("LT_LESSVAR")) + pars.add_argument("less", type=Path) + pars.add_argument("lessflags", nargs="?") + pars.add_argument("textfile", type=FileType("r")) + args = pars.parse_args() + + width, height = term_size() + if width > 2: + width -= 2 + if height > 1: + width -= 1 + if args.w is not None: + width = args.w + if args.h is not None: + height = args.h + + if not args.textfile.is_file(): + print(f"cannot open {args.textfile.name}", file=stderr) + return 0 + + base = args.textfile.name + if base: + pass + + outfile = "" + for n in count(1): + outfile = outdir / Path(f"{base}-{n}.lt") + if outfile.stat().st_size == 0: + break + cmd = "" + if width > 0: + cmd = f"LT_COLUMNS={width} {cmd}" + if height > 0: + cmd = f"LT_LINES={height} {cmd}" + cmd = f"{cmd} {lesstest} -s ./lt_screen -o {outfile} -- ".join(" ", argv) + if run(cmd).returncode: + print(f"error running {lesstest}", file=stderr) + return 1 + + +def term_size(): + stty = run(["stty", "-a"], capture_output=True, text=True).output + width = search(r"(rows|lines)\s+(\d+)", stty).group(0) + col = search(r"columns\s+(\d+)", stty).group(0) + return int(width), int(col) + + +if __name__ == "__main__": + exit(0 if main() else 1) diff --git a/lesstest/run.py b/lesstest/run.py new file mode 100644 index 00000000..26ddee7c --- /dev/null +++ b/lesstest/run.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python + +import argparse +import sys +import subprocess +from pathlib import Path + +lesstest = "./lesstest" +less = "../obj/less" +verbose = False + + +def main(): + args = argparse.ArgumentParser(description="Run one or more test files") + args.add_argument('targets', metavar='file.lt | dir', type=Path, nargs='+') + args.add_argument('-v', '--verbose', action="store_true") + args.add_argument('-l', help="less.exe", type=Path) + args.add_argument('-o', help="arguments to be passed to lesstest", + nargs='+') + parsed = args.parse_args() + + if parsed.l: + global less + less = parsed.l + global lesstest_opts + lesstest_opts = parsed.o.strip() + if lesstest_opts and lesstest_opts[0] != '-': + lesstest_opts = '-'+lesstest_opts + global verbose + verbose = parsed.v + + errors = 0 + for file in parsed.targets: + errors += run(file) + if errors > 0: + print(f"ERR {errors} errors", file=sys.stderr) + return 0 + + return 1 + + +def run(file): + if file.is_dir(): + return run_dir(file) + if file.suffix() != '.lt': + print(f'WARNING skipping {file.name}: not .lt file') + return 0 + if not file.is_file(): + print(f"ERR cannot open {file.name}", file=sys.stderr) + return 1 + + cmd = f'{lesstest} -s ./lt_screen -t {file.name} {lesstest_opts} {less}' + if verbose: + print(f"RUN {cmd}", file=sys.stderr) + if subprocess.run(cmd.split()).returncode: + print(f"ERR running {cmd}", file=sys.stderr) + return 1 + return 0 + + +def run_dir(dir): + if not dir.is_dir(): + print(f"ERR cannot directory {dir.name}", file=sys.stderr) + return 1 + errors = 0 + errors += run(file for file in dir.iterdir()) + return errors + + +if __name__ == '__main__': + sys.exit(0 if main() else 1) From 8ce3322d48501dcee8918d5a50d5dec0ecb73e70 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Wed, 26 May 2021 17:56:09 -0400 Subject: [PATCH 51/56] Append Perl scripts with .pl extension, and more. In the README, acknowledge the existence of the USE_PYTHON flag when using Makefile.aut (relevant when checking from Git repo). Signed-off-by: Charlie Lin --- Makefile.aut | 12 +++++++----- README | 3 ++- mkutable => mkutable.pl | 0 3 files changed, 9 insertions(+), 6 deletions(-) rename mkutable => mkutable.pl (100%) mode change 100755 => 100644 diff --git a/Makefile.aut b/Makefile.aut index f4f65fcf..19f16399 100644 --- a/Makefile.aut +++ b/Makefile.aut @@ -12,11 +12,13 @@ REL := $(shell sed -e '/char version/!d' -e 's/[^0-9.]*\([0-9.]*\).*/\1/' -e q $ ifeq ($(USE_PYTHON),1) MKHELP = mkhelp.py MKFUNCS = mkfuncs.py + MKUTABLE = mkutable.py ADD_COPYRIGHT = add_copyright.py else MKHELP = mkhelp.pl MKFUNCS = mkfuncs.pl - ADD_COPYRIGHT = add_copyright + MKUTABLE = mkutable.pl + ADD_COPYRIGHT = add_copyright.pl endif SRC = \ @@ -107,13 +109,13 @@ ${srcdir}/lessecho.man: ${srcdir}/lessecho.nro ${NROFF} ${srcdir}/lessecho.nro >${srcdir}/lessecho.man compose.uni: unicode/UnicodeData.txt - ./mkutable -f2 Mn Me -- unicode/UnicodeData.txt > $@ + ${srcdir}/${MKUTABLE} -f2 Mn Me -- unicode/UnicodeData.txt > $@ fmt.uni: unicode/UnicodeData.txt - ./mkutable -f2 Cf -- unicode/UnicodeData.txt > $@ + ${srcdir}/${MKUTABLE} -f2 Cf -- unicode/UnicodeData.txt > $@ ubin.uni: unicode/UnicodeData.txt - ./mkutable -f2 Cc Cs Co Zl Zp -- unicode/UnicodeData.txt > $@ + ${srcdir}/${MKUTABLE} -f2 Cc Cs Co Zl Zp -- unicode/UnicodeData.txt > $@ wide.uni: unicode/EastAsianWidth.txt - ./mkutable -f1 W F -- unicode/EastAsianWidth.txt > $@ + ${srcdir}/${MKUTABLE} -f1 W F -- unicode/EastAsianWidth.txt > $@ unicode/UnicodeData.txt: mkdir -p unicode diff --git a/README b/README index 955bf828..57061ce0 100644 --- a/README +++ b/README @@ -33,7 +33,8 @@ INSTALLATION (Unix & Linux systems only): if you have not already done so. 2. If you are building from a clone of a git repository, - type "make -f Makefile.aut". + type "make -f Makefile.aut". If you would like to use Python versions of + build scripts, define "USE_PYTHON=1" before typing make. If you are building from a numbered release package (a tar or zip file with a name like less-999.tar.gz or less-999.zip downloaded from greenwoodsoftware.com, not from github), you should skip this step. diff --git a/mkutable b/mkutable.pl old mode 100755 new mode 100644 similarity index 100% rename from mkutable rename to mkutable.pl From ead4c8fcbb7ff3ed39aafd7ce70f7a1ff1a74877 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Wed, 18 Aug 2021 23:03:05 -0400 Subject: [PATCH 52/56] Improve Python version of mkhelp. --- mkhelp.py | 64 +++++++++++++++++++++++++------------------------------ 1 file changed, 29 insertions(+), 35 deletions(-) mode change 100755 => 100644 mkhelp.py diff --git a/mkhelp.py b/mkhelp.py old mode 100755 new mode 100644 index 4619457f..3319c8ed --- a/mkhelp.py +++ b/mkhelp.py @@ -1,38 +1,32 @@ #!/usr/bin/env python -import time -import sys +from time import gmtime -time = time.gmtime() -print("/* This file was generated by mkhelp.py from less.hlp at " - "%d:%02d GMT on %d/%d/%d */\n" % - (time.tm_hour, time.tm_min, time.tm_year, time.tm_mon, time.tm_mday)) -print("#include \"less.h\"") -print("constant char helpdata[] = {") -ch = 0 -while True: - prevch = ch - ch = sys.stdin.read(1) - if ch == '': - break - if (ch == "'"): - print("'\\'',", end='') - elif (ch == "\\"): - print("'\\\\',", end='') - elif (ch == "\b"): - print("'\\b',", end='') - elif (ch == "\t"): - print("'\\t',", end='') - elif (ch == "\n"): - if prevch != "\r": - print("'\\n',") - elif (ch == "\r"): - if prevch != "\n": - print("'\\n',") - else: - if ((ord(ch) >= ord(' ')) and (ord(ch) < 0x7f)): - print(f"'{ch}',", end='') - else: - print("0x%02x," % ord(ch), end='') -print(" '\\0' };") -print("constant int size_helpdata = sizeof(helpdata) - 1;") +time = gmtime() +with open('less.c', 'w', newline='') as out, open('less.hlp') as infile: + out.write("/* This file was generated by mkhelp.py from less.hlp at " + "%d:%02d on %d/%d/%d */\n" % + (time.tm_hour, time.tm_min, time.tm_year, time.tm_mon, + time.tm_mday)) + out.write("#include \"less.h\"\n") + out.write("constant char helpdata[] = {\n") + for i, line in enumerate(infile): + for ch in line: + if ch == "'": + out.write("'\\'',") + elif ch == "\\": + out.write("'\\\\',") + elif ch == "\b": + out.write("'\\b',") + elif ch == "\t": + out.write("'\\t',") + elif ch == "\n": + if (ch == line[0] and i == 0) or (len(line) > 1): + out.write("'\\n',\n") + else: + if (ord(ch) >= ord(' ')) and (ord(ch) < 0x7f): + out.write(f"'{ch}',") + else: + out.write("0x%02x," % ord(ch)) + out.write(" 0 };\n") + out.write("constant int size_helpdata = sizeof(helpdata) - 1;\n") From d38a3c6ce9ef3139fca10dc305c56aa191407b79 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Thu, 19 Aug 2021 12:23:36 -0400 Subject: [PATCH 53/56] Touch up a few Python scripts --- mkfuncs.py | 14 +++++++------- mkutable.py | 42 +++++++++++++++++++++--------------------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/mkfuncs.py b/mkfuncs.py index c397942f..21aab894 100755 --- a/mkfuncs.py +++ b/mkfuncs.py @@ -1,28 +1,28 @@ #!/usr/bin/env python import fileinput -import re +from re import search definition = '' state = 0 params = 0 for line in fileinput.input(): - if test := re.search(r'^\tpublic\s+(.*)', line): + if test := search(r'^\tpublic\s+(.*)', line): definition = 'public ' + test.group(1) state = 1 params = 0 - elif (state == 1) and (test := re.search(r'(\w+)\s*\(', line)): - definition = '{0} LESSPARAMS (('.format(test.group(1)) + elif (state == 1) and (test := search(r'(\w+)\s*\(', line)): + definition = f'{test.group(1)} LESSPARAMS ((' state = 2 elif state == 2: - if re.search(r'^{', line): + if search(r'^{', line): if not params: definition += 'VOID_PARAM' print(f'{definition}));') state = 0 - elif test := re.search(r'^\s*([^;]*)', line): - if (definition[-1:] != '('): + elif test := search(r'^\s*([^;]*)', line): + if (definition[-1] != '('): definition += ', ' definition += test.group(1) params = 1 diff --git a/mkutable.py b/mkutable.py index 0fbb1f85..128fa6c9 100644 --- a/mkutable.py +++ b/mkutable.py @@ -51,23 +51,23 @@ def main() -> int: print(f"/* Generated by \"{argv[0]} {' '.join(argv[1:])}\" on {date}.") print(" Python FTW! */") - last_code = 0 + last_code=0 with args.data_file as file: for line in file: line.rstrip('\r\n') - line = re.sub(r"s/#.*//", '', line) - fields = line.split(';') + line=re.sub(r"s/#.*//", '', line) + fields=line.split(';') if not fields: continue - lo_code = hi_code = 0 - codes = fields[0] + lo_code=hi_code=0 + codes=fields[0] if m := re.match(r"(\w+)\.\.(\w+)", codes): - lo_code = int(m.group(1), 16) - hi_code = int(m.group(2), 16) + lo_code=int(m.group(1), 16) + hi_code=int(m.group(2), 16) else: - lo_code = hi_code = int(codes, 16) - type = fields[type_field] - type = re.match(r"s/\s//g", type) + lo_code=hi_code=int(codes, 16) + type=fields[type_field] + type=re.match(r"s/\s//g", type) for last_code in range(lo_code, hi_code): output(out, last_code, 'Zs' if force_space[last_code] else 'Mn' if force_compose[last_code] else type) @@ -75,15 +75,15 @@ def main() -> int: return 1 -def output(out: dict, code: int, type: str = None): - type_ok = type is not None and type in out['types'].keys() +def output(out: dict, code: int, type: str=None): + type_ok=type is not None and type in out['types'].keys() if opt_n: - type_ok = not type_ok - prev_code = out['prev_code'] + type_ok=not type_ok + prev_code=out['prev_code'] print(out['types'].keys()) import pprint - p = pprint.PrettyPrinter() + p=pprint.PrettyPrinter() p.pprint(out) print(type) @@ -92,14 +92,14 @@ def output(out: dict, code: int, type: str = None): elif (not out["in_run"] or type != out["run_type"] or code != prev_code+1): end_run(out, prev_code) start_run(out, code, type) - out[prev_code] = code + out[prev_code]=code def start_run(out: dict, code: int, type: str): - out["start_code"] = code - out["prev_code"] = code - out["run_type"] = type - out["in_run"] = True + out["start_code"]=code + out["prev_code"]=code + out["run_type"]=type + out["in_run"]=True def end_run(out: dict, code: int): @@ -107,7 +107,7 @@ def end_run(out: dict, code: int): return print("\t{ 0x%04x, 0x%04x }, /* %s */" % out["start_code"], code, out["run_type"]) - out["in_run"] = False + out["in_run"]=False if __name__ == "__main__": From f7441da109fdaeed89198b0bc076f71ce2bafe4e Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Thu, 19 Aug 2021 12:26:17 -0400 Subject: [PATCH 54/56] Remove mention of ADD_COPYRIGHT. --- Makefile.aut | 2 -- 1 file changed, 2 deletions(-) diff --git a/Makefile.aut b/Makefile.aut index 19f16399..e50f8a49 100644 --- a/Makefile.aut +++ b/Makefile.aut @@ -13,12 +13,10 @@ ifeq ($(USE_PYTHON),1) MKHELP = mkhelp.py MKFUNCS = mkfuncs.py MKUTABLE = mkutable.py - ADD_COPYRIGHT = add_copyright.py else MKHELP = mkhelp.pl MKFUNCS = mkfuncs.pl MKUTABLE = mkutable.pl - ADD_COPYRIGHT = add_copyright.pl endif SRC = \ From 0cc2c60d5b44b4498a6830aeda55919a2c04a23c Mon Sep 17 00:00:00 2001 From: Mark Nudelman Date: Sun, 15 Aug 2021 17:38:12 -0700 Subject: [PATCH 55/56] parent 0202daa172c6bc08c7043ebc740b2fde2db488b6 author Mark Nudelman 1629074292 -0700 committer Charlie Lin 1629392807 -0400 Open 592 --- Makefile.aut | 12 +++-- README | 3 +- add_copyright.py | 30 +++++++++++ lesstest/gen.py | 67 +++++++++++++++++++++++ lesstest/run.py | 71 +++++++++++++++++++++++++ mkfuncs.py | 36 +++++++------ mkhelp.py | 62 ++++++++++------------ mkutable => mkutable.pl | 0 mkutable.py | 114 ++++++++++++++++++++++++++++++++++++++++ search.c | 6 ++- version.c | 3 +- 11 files changed, 347 insertions(+), 57 deletions(-) create mode 100644 add_copyright.py create mode 100644 lesstest/gen.py create mode 100644 lesstest/run.py mode change 100755 => 100644 mkhelp.py rename mkutable => mkutable.pl (100%) mode change 100755 => 100644 create mode 100644 mkutable.py diff --git a/Makefile.aut b/Makefile.aut index f0eb26d4..19f16399 100644 --- a/Makefile.aut +++ b/Makefile.aut @@ -12,9 +12,13 @@ REL := $(shell sed -e '/char version/!d' -e 's/[^0-9.]*\([0-9.]*\).*/\1/' -e q $ ifeq ($(USE_PYTHON),1) MKHELP = mkhelp.py MKFUNCS = mkfuncs.py + MKUTABLE = mkutable.py + ADD_COPYRIGHT = add_copyright.py else MKHELP = mkhelp.pl MKFUNCS = mkfuncs.pl + MKUTABLE = mkutable.pl + ADD_COPYRIGHT = add_copyright.pl endif SRC = \ @@ -105,13 +109,13 @@ ${srcdir}/lessecho.man: ${srcdir}/lessecho.nro ${NROFF} ${srcdir}/lessecho.nro >${srcdir}/lessecho.man compose.uni: unicode/UnicodeData.txt - ./mkutable -f2 Mn Me -- unicode/UnicodeData.txt > $@ + ${srcdir}/${MKUTABLE} -f2 Mn Me -- unicode/UnicodeData.txt > $@ fmt.uni: unicode/UnicodeData.txt - ./mkutable -f2 Cf -- unicode/UnicodeData.txt > $@ + ${srcdir}/${MKUTABLE} -f2 Cf -- unicode/UnicodeData.txt > $@ ubin.uni: unicode/UnicodeData.txt - ./mkutable -f2 Cc Cs Co Zl Zp -- unicode/UnicodeData.txt > $@ + ${srcdir}/${MKUTABLE} -f2 Cc Cs Co Zl Zp -- unicode/UnicodeData.txt > $@ wide.uni: unicode/EastAsianWidth.txt - ./mkutable -f1 W F -- unicode/EastAsianWidth.txt > $@ + ${srcdir}/${MKUTABLE} -f1 W F -- unicode/EastAsianWidth.txt > $@ unicode/UnicodeData.txt: mkdir -p unicode diff --git a/README b/README index 955bf828..57061ce0 100644 --- a/README +++ b/README @@ -33,7 +33,8 @@ INSTALLATION (Unix & Linux systems only): if you have not already done so. 2. If you are building from a clone of a git repository, - type "make -f Makefile.aut". + type "make -f Makefile.aut". If you would like to use Python versions of + build scripts, define "USE_PYTHON=1" before typing make. If you are building from a numbered release package (a tar or zip file with a name like less-999.tar.gz or less-999.zip downloaded from greenwoodsoftware.com, not from github), you should skip this step. diff --git a/add_copyright.py b/add_copyright.py new file mode 100644 index 00000000..bc8bc618 --- /dev/null +++ b/add_copyright.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python + +import os +import argparse +import pathlib +import re + +parser = argparse.ArgumentParser() +parser.add_argument("file", type=str) +parser.add_argument("dir", type=str) +args = parser.parse_args() +out = pathlib.Path(args.dir) / args.file + +copyright = copyright_line = "" +with open("copyright") as f: + copyright = f.read() +for line in copyright.splitlines(): + if '(C)' in line: + copyright_line = line.strip("/*/\r\n").rstrip(" ") + break + +with open(args.file) as input, out.open('w') as o: + for line in input: + if re.search(r"\@\@copyright\@\@", line): + o.write(copyright) + continue + out_line = re.sub(r"\@\@copyright_oneline\@\@", copyright_line, line) + o.write(out_line) + + out.chmod(0o555 if os.access(out, os.X_OK) else 0o444) diff --git a/lesstest/gen.py b/lesstest/gen.py new file mode 100644 index 00000000..ea427829 --- /dev/null +++ b/lesstest/gen.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python + +from argparse import ArgumentParser, FileType +from subprocess import run +from sys import exit, stderr, argv +from itertools import count +from pathlib import Path +from re import search +from os import environ + +lesstest = "./lesstest" +outdir = Path("./suite") + + +def main(): + pars = ArgumentParser(description="Generate a lesstest file.", add_help=False) + pars.add_argument("-w", help="width", type=int) + pars.add_argument("-h", help="height", type=int) + pars.add_argument("--lessvar", default=environ.get("LT_LESSVAR")) + pars.add_argument("less", type=Path) + pars.add_argument("lessflags", nargs="?") + pars.add_argument("textfile", type=FileType("r")) + args = pars.parse_args() + + width, height = term_size() + if width > 2: + width -= 2 + if height > 1: + width -= 1 + if args.w is not None: + width = args.w + if args.h is not None: + height = args.h + + if not args.textfile.is_file(): + print(f"cannot open {args.textfile.name}", file=stderr) + return 0 + + base = args.textfile.name + if base: + pass + + outfile = "" + for n in count(1): + outfile = outdir / Path(f"{base}-{n}.lt") + if outfile.stat().st_size == 0: + break + cmd = "" + if width > 0: + cmd = f"LT_COLUMNS={width} {cmd}" + if height > 0: + cmd = f"LT_LINES={height} {cmd}" + cmd = f"{cmd} {lesstest} -s ./lt_screen -o {outfile} -- ".join(" ", argv) + if run(cmd).returncode: + print(f"error running {lesstest}", file=stderr) + return 1 + + +def term_size(): + stty = run(["stty", "-a"], capture_output=True, text=True).output + width = search(r"(rows|lines)\s+(\d+)", stty).group(0) + col = search(r"columns\s+(\d+)", stty).group(0) + return int(width), int(col) + + +if __name__ == "__main__": + exit(0 if main() else 1) diff --git a/lesstest/run.py b/lesstest/run.py new file mode 100644 index 00000000..26ddee7c --- /dev/null +++ b/lesstest/run.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python + +import argparse +import sys +import subprocess +from pathlib import Path + +lesstest = "./lesstest" +less = "../obj/less" +verbose = False + + +def main(): + args = argparse.ArgumentParser(description="Run one or more test files") + args.add_argument('targets', metavar='file.lt | dir', type=Path, nargs='+') + args.add_argument('-v', '--verbose', action="store_true") + args.add_argument('-l', help="less.exe", type=Path) + args.add_argument('-o', help="arguments to be passed to lesstest", + nargs='+') + parsed = args.parse_args() + + if parsed.l: + global less + less = parsed.l + global lesstest_opts + lesstest_opts = parsed.o.strip() + if lesstest_opts and lesstest_opts[0] != '-': + lesstest_opts = '-'+lesstest_opts + global verbose + verbose = parsed.v + + errors = 0 + for file in parsed.targets: + errors += run(file) + if errors > 0: + print(f"ERR {errors} errors", file=sys.stderr) + return 0 + + return 1 + + +def run(file): + if file.is_dir(): + return run_dir(file) + if file.suffix() != '.lt': + print(f'WARNING skipping {file.name}: not .lt file') + return 0 + if not file.is_file(): + print(f"ERR cannot open {file.name}", file=sys.stderr) + return 1 + + cmd = f'{lesstest} -s ./lt_screen -t {file.name} {lesstest_opts} {less}' + if verbose: + print(f"RUN {cmd}", file=sys.stderr) + if subprocess.run(cmd.split()).returncode: + print(f"ERR running {cmd}", file=sys.stderr) + return 1 + return 0 + + +def run_dir(dir): + if not dir.is_dir(): + print(f"ERR cannot directory {dir.name}", file=sys.stderr) + return 1 + errors = 0 + errors += run(file for file in dir.iterdir()) + return errors + + +if __name__ == '__main__': + sys.exit(0 if main() else 1) diff --git a/mkfuncs.py b/mkfuncs.py index 7a2da5ad..21aab894 100755 --- a/mkfuncs.py +++ b/mkfuncs.py @@ -1,26 +1,28 @@ #!/usr/bin/env python import fileinput -import re +from re import search definition = '' state = 0 params = 0 for line in fileinput.input(): - if test := re.search(r'^\tpublic\s+(.*)', line): - definition = 'public ' + test.group(1) - state = 1 - params = 0 - elif (state == 1) and (test := re.search(r'(\w+)\s*\(', line)): - definition = '{0} LESSPARAMS (('.format(test.group(1)) - state = 2 - elif state == 2: - if re.search(r'^{', line): - if not params: definition += 'VOID_PARAM' - print(f'{definition}));') - state = 0 - elif test := re.search(r'^\s*([^;]*)', line): - if (definition[-1:] != '('): definition += ', ' - definition += test.group(1) - params = 1 + if test := search(r'^\tpublic\s+(.*)', line): + definition = 'public ' + test.group(1) + state = 1 + params = 0 + elif (state == 1) and (test := search(r'(\w+)\s*\(', line)): + definition = f'{test.group(1)} LESSPARAMS ((' + state = 2 + elif state == 2: + if search(r'^{', line): + if not params: + definition += 'VOID_PARAM' + print(f'{definition}));') + state = 0 + elif test := search(r'^\s*([^;]*)', line): + if (definition[-1] != '('): + definition += ', ' + definition += test.group(1) + params = 1 diff --git a/mkhelp.py b/mkhelp.py old mode 100755 new mode 100644 index 029f1505..3319c8ed --- a/mkhelp.py +++ b/mkhelp.py @@ -1,36 +1,32 @@ #!/usr/bin/env python -import time -import sys +from time import gmtime -time = time.gmtime() -print("/* This file was generated by mkhelp.py from less.hlp at "\ - "%d:%02d GMT on %d/%d/%d */\n" % - (time.tm_hour, time.tm_min, time.tm_year, time.tm_mon, time.tm_mday)) -print("#include \"less.h\"") -print("constant char helpdata[] = {") -ch = 0 -while True: - prevch = ch - ch = sys.stdin.read(1) - if ch == '': - break - if (ch == "'"): - print("'\\'',", end='') - elif (ch == "\\"): - print("'\\\\',", end='') - elif (ch == "\b"): - print ("'\\b',", end='') - elif (ch == "\t"): - print ("'\\t',", end='') - elif (ch == "\n"): - if prevch != "\r": print("'\\n',") - elif (ch == "\r"): - if prevch != "\n": print("'\\n',") - else: - if ((ord(ch) >= ord(' ')) and (ord(ch) < 0x7f)): - print(f"'{ch}',", end='') - else: - print("0x%02x," % ord(ch), end='') -print(" '\\0' };") -print("constant int size_helpdata = sizeof(helpdata) - 1;") +time = gmtime() +with open('less.c', 'w', newline='') as out, open('less.hlp') as infile: + out.write("/* This file was generated by mkhelp.py from less.hlp at " + "%d:%02d on %d/%d/%d */\n" % + (time.tm_hour, time.tm_min, time.tm_year, time.tm_mon, + time.tm_mday)) + out.write("#include \"less.h\"\n") + out.write("constant char helpdata[] = {\n") + for i, line in enumerate(infile): + for ch in line: + if ch == "'": + out.write("'\\'',") + elif ch == "\\": + out.write("'\\\\',") + elif ch == "\b": + out.write("'\\b',") + elif ch == "\t": + out.write("'\\t',") + elif ch == "\n": + if (ch == line[0] and i == 0) or (len(line) > 1): + out.write("'\\n',\n") + else: + if (ord(ch) >= ord(' ')) and (ord(ch) < 0x7f): + out.write(f"'{ch}',") + else: + out.write("0x%02x," % ord(ch)) + out.write(" 0 };\n") + out.write("constant int size_helpdata = sizeof(helpdata) - 1;\n") diff --git a/mkutable b/mkutable.pl old mode 100755 new mode 100644 similarity index 100% rename from mkutable rename to mkutable.pl diff --git a/mkutable.py b/mkutable.py new file mode 100644 index 00000000..b208c032 --- /dev/null +++ b/mkutable.py @@ -0,0 +1,114 @@ +#! /usr/bin/env python + +from argparse import ArgumentParser +from sys import exit +from subprocess import run +from fileinput import input +import re +>>>>>>> 27ef7b3 (Commit more Python versions of scripts. WIP) + +# Override Unicode tables for certain control chars +# that are expected to be found in normal text files. +force_space = { + 0x08: 1, # backspace + 0x09: 1, # tab + 0x0a: 1, # newline + 0x0c: 1, # form feed + 0x0d: 1, # carriage return +} + +# Hangul Jamo medial vowels and final consonants should be zero width. +force_compose = [ + [0x1160, 0x11ff], + [0xd7b0, 0xd7c6], + [0xd7cb, 0xd7fb] +] + + +def main() -> int: + parser = ArgumentParser() + parser.add_argument("-n", help="Take non-matching types", + action='store_true') + parser.add_argument("-f", help="Zero-based type field (default 2)", + type=int, default=2) + parser.add_argument("types", nargs='+', type=str) + # Kludge: cannot specify alternate var for dest= parameter when the positional + # argument's name contains a dot. + parser.add_argument("data_file", type=open, metavar="UnicodeData.txt") + args = parser.parse_args() + global opt_n + opt_n = args.n + global type_field + type_field = args.f + types = {type: 1 for type in args.types} + out = {'types': types, 'prev_code': 0, 'in_run': False, 'run_type': "", + 'start_code': 0} + global_list = globals() + force_compose = {ch: 1 for lo, hi in global_list['force_compose'] + for ch in range(lo, hi)} + + # Only works for Python 3.7+ + date = run("date", capture_output=True, text=True).stdout.rstrip('\r\n') + print(f"/* Generated by \"{sys.argv[0]} {' '.join(sys.argv[1:])}\" on {date}." + print(" Python FTW! */") + + last_code=0 + with args.data_file as file: + for line in file: + line.rstrip('\r\n') + line=re.sub(r"s/#.*//", '', line) + fields=line.split(';') + if not fields: + continue + lo_code=hi_code=0 + codes=fields[0] + if m := re.match(r"(\w+)\.\.(\w+)", codes): + lo_code=int(m.group(1), 16) + hi_code=int(m.group(2), 16) + else: + lo_code=hi_code=int(codes, 16) + type=fields[type_field] + type=re.match(r"s/\s//g", type) + for last_code in range(lo_code, hi_code): + output(out, last_code, 'Zs' if force_space[last_code] else 'Mn' + if force_compose[last_code] else type) + output(out, last_code) + return 1 + + +def output(out: dict, code: int, type: str=None): + type_ok=type is not None and type in out['types'].keys() + if opt_n: + type_ok=not type_ok + prev_code=out['prev_code'] + + print(out['types'].keys()) + import pprint + p=pprint.PrettyPrinter() + p.pprint(out) + + print(type) + if not type_ok: + end_run(out, prev_code) + elif (not out["in_run"] or type != out["run_type"] or code != prev_code+1): + end_run(out, prev_code) + start_run(out, code, type) + out[prev_code]=code + + +def start_run(out: dict, code: int, type: str): + out["start_code"]=code + out["prev_code"]=code + out["run_type"]=type + out["in_run"]=True + + +def end_run(out: dict, code: int): + if not out["in_run"]: + return + print("\t{ 0x%04x, 0x%04x }, /* %s */" % out["start_code"], code, + out["run_type"]) + out["in_run"]=False + +if __name__ == "__main__": + exit(0 if main() else 1) diff --git a/search.c b/search.c index f619fbeb..4cc6832e 100644 --- a/search.c +++ b/search.c @@ -1908,7 +1908,11 @@ set_filter_pattern(pattern, search_type) /* Create a new filter and add it to the filter_infos list. */ filter = ecalloc(1, sizeof(struct pattern_info)); init_pattern(filter); - set_pattern(filter, pattern, search_type, 1); + if (set_pattern(filter, pattern, search_type, 1) < 0) + { + free(filter); + return; + } filter->next = filter_infos; filter_infos = filter; } diff --git a/version.c b/version.c index 4c09f6f2..8ab939f5 100644 --- a/version.c +++ b/version.c @@ -933,6 +933,7 @@ v589 5/29/21 Copyright & build changes. v590 6/3/21 Fix non-autoconf Makefiles. v591 8/8/21 Use \kB for backspace key in lesskey; add more \k codes; handle multibyte chars in prompt. +v592 */ -char version[] = "591"; +char version[] = "592x"; From 705bbe137f9d3aec0dfbf6a866f35c8debd71058 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Thu, 19 Aug 2021 12:26:17 -0400 Subject: [PATCH 56/56] Remove mention of ADD_COPYRIGHT. --- Makefile.aut | 2 -- 1 file changed, 2 deletions(-) diff --git a/Makefile.aut b/Makefile.aut index 19f16399..e50f8a49 100644 --- a/Makefile.aut +++ b/Makefile.aut @@ -13,12 +13,10 @@ ifeq ($(USE_PYTHON),1) MKHELP = mkhelp.py MKFUNCS = mkfuncs.py MKUTABLE = mkutable.py - ADD_COPYRIGHT = add_copyright.py else MKHELP = mkhelp.pl MKFUNCS = mkfuncs.pl MKUTABLE = mkutable.pl - ADD_COPYRIGHT = add_copyright.pl endif SRC = \