From 179ce3a92cea46ed614a51876cbdd5e3f4fd26dc Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 15 May 2019 20:12:04 -0300 Subject: [PATCH 01/35] c++ style single line comments implemented --- bython/parser.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/bython/parser.py b/bython/parser.py index e0de952..c1e3588 100644 --- a/bython/parser.py +++ b/bython/parser.py @@ -119,19 +119,22 @@ def parse_file(filepath, add_true_line, filename_prefix, outputname=None, change for line in infile_str_raw.split("\n"): # Search for comments, and remove for now. Re-add them before writing to # result string - m = re.search(r"[ \t]*(#.*$)", line) + m = re.search(r"[ \t]*([(\/\/)#].*$)", line) # Make sure # sign is not inside quotations. Delete match object if it is if m is not None: - m2 = re.search(r"[\"'].*#.*[\"']", m.group(0)) + m2 = re.search(r"[\"'].*[(\/\/)#].*[\"']", m.group(0)) if m2 is not None: m = None if m is not None: add_comment = m.group(0) - line = re.sub(r"[ \t]*(#.*$)", "", line) + line = re.sub(r"[ \t]*([(\/\/)#].*$)", "", line) else: add_comment = "" + + # replace // with # + add_comment = add_comment.replace("//", "#", 1) # skip empty lines: if line.strip() in ('\n', '\r\n', ''): From 305e8ac17624ad000cf977b22753a8793ce63e75 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 15 May 2019 21:49:00 -0300 Subject: [PATCH 02/35] started the implementation of a recursive parser --- bython/parser.py | 59 +++++++++++++++++++++++++++++++++++++++++++++++ test-code.by | 17 ++++++++++++++ test-recursive.py | 6 +++++ 3 files changed, 82 insertions(+) create mode 100644 test-code.by create mode 100644 test-recursive.py diff --git a/bython/parser.py b/bython/parser.py index c1e3588..ef57edd 100644 --- a/bython/parser.py +++ b/bython/parser.py @@ -180,3 +180,62 @@ def parse_file(filepath, add_true_line, filename_prefix, outputname=None, change infile.close() outfile.close() + + +def parse_file_recursive(filepath, add_true_line=False, filename_prefix="", outputname=None, change_imports=None): + """ + Converts a bython file to a python file recursively and writes it to disk. + + Args: + filename (str): Path to the bython file you want to parse. + add_true_line (boolean): Whether to add a line at the top of the + file, adding support for C-style true/false + in addition to capitalized True/False. + filename_prefix (str): Prefix to resulting file name (if -c or -k + is not present, then the files are prefixed + with a '.'). + outputname (str): Optional. Override name of output file. If + omitted it defaults to substituting '.by' to + '.py' + change_imports (dict): Names of imported bython modules, and their + python alternative. + """ + + # TODO remove defaults for the parameters 'add_true_line' and 'filename_prefix' + # i've put them there for ease of use + + # inner function for parsing recursively + def recursive_parser(code, position=0, scope=""): + + # scope equal to "" means it's on global scope + if scope == "": + print("g", end="") # for debugging + while position < len(code): + + if code[position] == "{": + position = recursive_parser(code, position + 1, "{") + print("g", end="") # for debugging + + else: + position = position + 1 + + elif scope == "{": + print("{", end="") # for debugging + while position < len(code): + + if code[position] == "{": + position = recursive_parser(code, position + 1, "{") + + elif code[position] == "}": + print("}", end="") + return position + 1 + + else: + position = position + 1 + + # open file and read contents + infile = open(filepath, "r") + infile_str = infile.read() + infile.close() + + recursive_parser(infile_str) \ No newline at end of file diff --git a/test-code.by b/test-code.by new file mode 100644 index 0000000..96897b4 --- /dev/null +++ b/test-code.by @@ -0,0 +1,17 @@ +# this file is for testing purposes only +# TODO remove this file later + +def print_hello_world() { + str = "hello"; # hello + str += " world"; // world + if str == "hello world" { + print("its equal") + } + else + { + print("not equal") + } + print(str) /////# let's print it! +} + +print_hello_world(); \ No newline at end of file diff --git a/test-recursive.py b/test-recursive.py new file mode 100644 index 0000000..6bd2576 --- /dev/null +++ b/test-recursive.py @@ -0,0 +1,6 @@ +# this file is for testing purposes only +# TODO remove this file later + +from bython.parser import parse_file_recursive + +parse_file_recursive("test-code.by") \ No newline at end of file From f658382ac3a0ccbe36efa545aaf876c8f6f6cb5a Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 15 May 2019 21:57:48 -0300 Subject: [PATCH 03/35] added '#' scope on recursive parser --- bython/parser.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/bython/parser.py b/bython/parser.py index ef57edd..b52bfc4 100644 --- a/bython/parser.py +++ b/bython/parser.py @@ -216,6 +216,9 @@ def recursive_parser(code, position=0, scope=""): position = recursive_parser(code, position + 1, "{") print("g", end="") # for debugging + if code[position] == "#": + position = recursive_parser(code, position + 1, "#") + else: position = position + 1 @@ -226,6 +229,9 @@ def recursive_parser(code, position=0, scope=""): if code[position] == "{": position = recursive_parser(code, position + 1, "{") + if code[position] == "#": + position = recursive_parser(code, position + 1, "#") + elif code[position] == "}": print("}", end="") return position + 1 @@ -233,6 +239,21 @@ def recursive_parser(code, position=0, scope=""): else: position = position + 1 + elif scope == "#": + print("#", end="") # for debugging + while position < len(code): + + if code[position] == "\n": + print("n", end="") # for debugging + return position + 1 + + else: + position = position + 1 + + else: + raise Exception("invalid scope was reached") + + # open file and read contents infile = open(filepath, "r") infile_str = infile.read() From d64fe83a1c6aed140759ebae667ad67efc657554 Mon Sep 17 00:00:00 2001 From: Filipe Jordan do Nascimento Date: Wed, 15 May 2019 23:02:05 -0300 Subject: [PATCH 04/35] recommited all the changes with my name set on git --- bython/parser.py | 89 +++++++++++++++++++++++++++++++++++++++++++++-- test-code.by | 17 +++++++++ test-recursive.py | 6 ++++ 3 files changed, 109 insertions(+), 3 deletions(-) create mode 100644 test-code.by create mode 100644 test-recursive.py diff --git a/bython/parser.py b/bython/parser.py index e0de952..b52bfc4 100644 --- a/bython/parser.py +++ b/bython/parser.py @@ -119,19 +119,22 @@ def parse_file(filepath, add_true_line, filename_prefix, outputname=None, change for line in infile_str_raw.split("\n"): # Search for comments, and remove for now. Re-add them before writing to # result string - m = re.search(r"[ \t]*(#.*$)", line) + m = re.search(r"[ \t]*([(\/\/)#].*$)", line) # Make sure # sign is not inside quotations. Delete match object if it is if m is not None: - m2 = re.search(r"[\"'].*#.*[\"']", m.group(0)) + m2 = re.search(r"[\"'].*[(\/\/)#].*[\"']", m.group(0)) if m2 is not None: m = None if m is not None: add_comment = m.group(0) - line = re.sub(r"[ \t]*(#.*$)", "", line) + line = re.sub(r"[ \t]*([(\/\/)#].*$)", "", line) else: add_comment = "" + + # replace // with # + add_comment = add_comment.replace("//", "#", 1) # skip empty lines: if line.strip() in ('\n', '\r\n', ''): @@ -177,3 +180,83 @@ def parse_file(filepath, add_true_line, filename_prefix, outputname=None, change infile.close() outfile.close() + + +def parse_file_recursive(filepath, add_true_line=False, filename_prefix="", outputname=None, change_imports=None): + """ + Converts a bython file to a python file recursively and writes it to disk. + + Args: + filename (str): Path to the bython file you want to parse. + add_true_line (boolean): Whether to add a line at the top of the + file, adding support for C-style true/false + in addition to capitalized True/False. + filename_prefix (str): Prefix to resulting file name (if -c or -k + is not present, then the files are prefixed + with a '.'). + outputname (str): Optional. Override name of output file. If + omitted it defaults to substituting '.by' to + '.py' + change_imports (dict): Names of imported bython modules, and their + python alternative. + """ + + # TODO remove defaults for the parameters 'add_true_line' and 'filename_prefix' + # i've put them there for ease of use + + # inner function for parsing recursively + def recursive_parser(code, position=0, scope=""): + + # scope equal to "" means it's on global scope + if scope == "": + print("g", end="") # for debugging + while position < len(code): + + if code[position] == "{": + position = recursive_parser(code, position + 1, "{") + print("g", end="") # for debugging + + if code[position] == "#": + position = recursive_parser(code, position + 1, "#") + + else: + position = position + 1 + + elif scope == "{": + print("{", end="") # for debugging + while position < len(code): + + if code[position] == "{": + position = recursive_parser(code, position + 1, "{") + + if code[position] == "#": + position = recursive_parser(code, position + 1, "#") + + elif code[position] == "}": + print("}", end="") + return position + 1 + + else: + position = position + 1 + + elif scope == "#": + print("#", end="") # for debugging + while position < len(code): + + if code[position] == "\n": + print("n", end="") # for debugging + return position + 1 + + else: + position = position + 1 + + else: + raise Exception("invalid scope was reached") + + + # open file and read contents + infile = open(filepath, "r") + infile_str = infile.read() + infile.close() + + recursive_parser(infile_str) \ No newline at end of file diff --git a/test-code.by b/test-code.by new file mode 100644 index 0000000..96897b4 --- /dev/null +++ b/test-code.by @@ -0,0 +1,17 @@ +# this file is for testing purposes only +# TODO remove this file later + +def print_hello_world() { + str = "hello"; # hello + str += " world"; // world + if str == "hello world" { + print("its equal") + } + else + { + print("not equal") + } + print(str) /////# let's print it! +} + +print_hello_world(); \ No newline at end of file diff --git a/test-recursive.py b/test-recursive.py new file mode 100644 index 0000000..6bd2576 --- /dev/null +++ b/test-recursive.py @@ -0,0 +1,6 @@ +# this file is for testing purposes only +# TODO remove this file later + +from bython.parser import parse_file_recursive + +parse_file_recursive("test-code.by") \ No newline at end of file From 1fab9b0383e4b8aefcfc0aa8d69a74c4bdfd442b Mon Sep 17 00:00:00 2001 From: Filipe Jordan do Nascimento Date: Thu, 16 May 2019 20:12:14 -0300 Subject: [PATCH 05/35] implemented tests for: comments (c, c++ and python styled), strings (single and double quoted) and dicts --- bython/parser.py | 135 +++++++++++++++++++++++++++++++++++++++++++++-- test-code.by | 17 +++++- 2 files changed, 145 insertions(+), 7 deletions(-) diff --git a/bython/parser.py b/bython/parser.py index b52bfc4..74e6835 100644 --- a/bython/parser.py +++ b/bython/parser.py @@ -212,12 +212,33 @@ def recursive_parser(code, position=0, scope=""): print("g", end="") # for debugging while position < len(code): + # check for brace opening if code[position] == "{": position = recursive_parser(code, position + 1, "{") print("g", end="") # for debugging - if code[position] == "#": + # check for python-style comment + elif code[position] == "#": position = recursive_parser(code, position + 1, "#") + + # check for c and cpp-style comment + elif code[position] == "/": + if code[position + 1] == "/": + position = recursive_parser(code, position + 2, "//") + elif code[position + 1] == "*": + position = recursive_parser(code, position + 2, "/*") + + # check for single-quote string start + elif code[position] == "\'": + position = recursive_parser(code, position + 1, "\'") + + # check for double-quote string start + elif code[position] == "\"": + position = recursive_parser(code, position + 1, "\"") + + # check for equals (for python dicts with braces) + elif code[position] == "=": + position = recursive_parser(code, position + 1, "=") else: position = position + 1 @@ -226,16 +247,39 @@ def recursive_parser(code, position=0, scope=""): print("{", end="") # for debugging while position < len(code): + # check for brace opening if code[position] == "{": position = recursive_parser(code, position + 1, "{") - - if code[position] == "#": - position = recursive_parser(code, position + 1, "#") - + + # check for brace closing elif code[position] == "}": print("}", end="") return position + 1 + # check for python-style comment + elif code[position] == "#": + position = recursive_parser(code, position + 1, "#") + + # check for c and cpp-style comment + elif code[position] == "/": + if code[position + 1] == "/": + position = recursive_parser(code, position + 2, "//") + elif code[position + 1] == "*": + position = recursive_parser(code, position + 2, "/*") + + # check for single-quote string start + elif code[position] == "\'": + position = recursive_parser(code, position + 1, "\'") + + # check for double-quote string start + elif code[position] == "\"": + position = recursive_parser(code, position + 1, "\"") + + # check for equals (for python dicts with braces) + elif code[position] == "=": + position = recursive_parser(code, position + 1, "=") + + else: position = position + 1 @@ -250,6 +294,87 @@ def recursive_parser(code, position=0, scope=""): else: position = position + 1 + elif scope == "//": + print("//", end="") # for debugging + while position < len(code): + + if code[position] == "\n": + print("n", end="") # for debugging + return position + 1 + + else: + position = position + 1 + + elif scope == "/*": + print("/*", end="") # for debugging + while position < len(code): + + # check for c-style comment closing + if code[position] == "*": + if code[position + 1] == "/": + print("*/", end="") # for debugging + return position + 2 + + else: + position = position + 1 + + elif scope == "\'": + print("\'^", end="") # for debugging + while position < len(code): + + # check for single-quote string ending + if code[position] == "\'": + # check if its escaped + if code[position - 1] != "\\": + print("$\'", end="") # for debugging + return position + 1 + else: + position = position + 1 + + else: + position = position + 1 + + elif scope == "\"": + print("\"^", end="") # for debugging + while position < len(code): + + # check for single-quote string ending + if code[position] == "\"": + # check if its escaped + if code[position - 1] != "\\": + print("$\'", end="") # for debugging + return position + 1 + else: + position = position + 1 + + else: + position = position + 1 + + elif scope == "=": + print("=", end="") # for debugging + while position < len(code): + + # check for dicts + if code[position] == "{": + print(".dict.", end="") # for debugging + return recursive_parser(code, position + 1, "={") + + # check for non dicts + elif re.search(r"[^\s\n\r]", code[position]): + print("!", end="") # for debugging + return position + + else: + position = position + 1 + + elif scope == "={": + while position < len(code): + if code[position] == "}": + return position + 1 + + else: + position = position + 1 + else: raise Exception("invalid scope was reached") diff --git a/test-code.by b/test-code.by index 96897b4..1ac3dde 100644 --- a/test-code.by +++ b/test-code.by @@ -2,8 +2,21 @@ # TODO remove this file later def print_hello_world() { - str = "hello"; # hello - str += " world"; // world + str = "he\"ll\"o"; # hello + str += ' world'; // world + /* + print("error - not yet implemented"); + while(true) // remove + { + sleep(1000); + + } + */ + mydictionary = { + 'key1': "value1", + 'key2': 2, + 'key3': "hi", + } if str == "hello world" { print("its equal") } From 6af238dd4722db797bb8d2f7d357b0acd7b66f83 Mon Sep 17 00:00:00 2001 From: Filipe Jordan do Nascimento Date: Thu, 16 May 2019 20:27:00 -0300 Subject: [PATCH 06/35] unified the global and brace scopes conditional --- bython/parser.py | 58 +++++++++++++----------------------------------- 1 file changed, 16 insertions(+), 42 deletions(-) diff --git a/bython/parser.py b/bython/parser.py index 74e6835..1f53404 100644 --- a/bython/parser.py +++ b/bython/parser.py @@ -208,58 +208,25 @@ def parse_file_recursive(filepath, add_true_line=False, filename_prefix="", outp def recursive_parser(code, position=0, scope=""): # scope equal to "" means it's on global scope - if scope == "": - print("g", end="") # for debugging - while position < len(code): - - # check for brace opening - if code[position] == "{": - position = recursive_parser(code, position + 1, "{") - print("g", end="") # for debugging + # scope equal to "{" means it's on local scope + if scope == "" or scope == "{": - # check for python-style comment - elif code[position] == "#": - position = recursive_parser(code, position + 1, "#") - - # check for c and cpp-style comment - elif code[position] == "/": - if code[position + 1] == "/": - position = recursive_parser(code, position + 2, "//") - elif code[position + 1] == "*": - position = recursive_parser(code, position + 2, "/*") - - # check for single-quote string start - elif code[position] == "\'": - position = recursive_parser(code, position + 1, "\'") + if scope == "": + print("g", end="") # for debugging - # check for double-quote string start - elif code[position] == "\"": - position = recursive_parser(code, position + 1, "\"") - - # check for equals (for python dicts with braces) - elif code[position] == "=": - position = recursive_parser(code, position + 1, "=") - - else: - position = position + 1 - - elif scope == "{": - print("{", end="") # for debugging while position < len(code): # check for brace opening if code[position] == "{": + print("{", end="") # for debugging position = recursive_parser(code, position + 1, "{") - - # check for brace closing - elif code[position] == "}": - print("}", end="") - return position + 1 + if scope == "": + print("g", end="") # for debugging # check for python-style comment elif code[position] == "#": position = recursive_parser(code, position + 1, "#") - + # check for c and cpp-style comment elif code[position] == "/": if code[position + 1] == "/": @@ -278,7 +245,14 @@ def recursive_parser(code, position=0, scope=""): # check for equals (for python dicts with braces) elif code[position] == "=": position = recursive_parser(code, position + 1, "=") - + + # check for brace closing (when not on global) + elif scope == "{": + if code[position] == "}": + print("}", end="") + return position + 1 + else: + position = position + 1 else: position = position + 1 From 4a12556f3362641c08742844f74bd88b6faa7331 Mon Sep 17 00:00:00 2001 From: Filipe Jordan do Nascimento Date: Thu, 16 May 2019 21:00:05 -0300 Subject: [PATCH 07/35] outputting the .py file for the first time --- bython/parser.py | 56 ++++++++++++++++++++++++++++++++---------------- test-code.py | 27 +++++++++++++++++++++++ 2 files changed, 65 insertions(+), 18 deletions(-) create mode 100644 test-code.py diff --git a/bython/parser.py b/bython/parser.py index 1f53404..9b46dbd 100644 --- a/bython/parser.py +++ b/bython/parser.py @@ -205,7 +205,7 @@ def parse_file_recursive(filepath, add_true_line=False, filename_prefix="", outp # i've put them there for ease of use # inner function for parsing recursively - def recursive_parser(code, position=0, scope=""): + def recursive_parser(code, position, scope, outfile, identation): # scope equal to "" means it's on global scope # scope equal to "{" means it's on local scope @@ -213,38 +213,47 @@ def recursive_parser(code, position=0, scope=""): if scope == "": print("g", end="") # for debugging + else: + identation = identation + 1 while position < len(code): # check for brace opening if code[position] == "{": print("{", end="") # for debugging - position = recursive_parser(code, position + 1, "{") + outfile.write(":") + position = recursive_parser(code, position + 1, "{", outfile, identation) if scope == "": print("g", end="") # for debugging # check for python-style comment elif code[position] == "#": - position = recursive_parser(code, position + 1, "#") + outfile.write(code[position]) + position = recursive_parser(code, position + 1, "#", outfile, identation) # check for c and cpp-style comment elif code[position] == "/": if code[position + 1] == "/": - position = recursive_parser(code, position + 2, "//") + outfile.write("#") + position = recursive_parser(code, position + 2, "//", outfile, identation) elif code[position + 1] == "*": - position = recursive_parser(code, position + 2, "/*") + outfile.write(code[position:position+2]) # TODO implement comment on all lines + position = recursive_parser(code, position + 2, "/*", outfile, identation) # check for single-quote string start elif code[position] == "\'": - position = recursive_parser(code, position + 1, "\'") + outfile.write(code[position]) + position = recursive_parser(code, position + 1, "\'", outfile, identation) # check for double-quote string start elif code[position] == "\"": - position = recursive_parser(code, position + 1, "\"") + outfile.write(code[position]) + position = recursive_parser(code, position + 1, "\"", outfile, identation) # check for equals (for python dicts with braces) elif code[position] == "=": - position = recursive_parser(code, position + 1, "=") + outfile.write(code[position]) + position = recursive_parser(code, position + 1, "=", outfile, identation) # check for brace closing (when not on global) elif scope == "{": @@ -252,15 +261,17 @@ def recursive_parser(code, position=0, scope=""): print("}", end="") return position + 1 else: + outfile.write(code[position]) position = position + 1 else: + outfile.write(code[position]) position = position + 1 elif scope == "#": print("#", end="") # for debugging while position < len(code): - + outfile.write(code[position]) if code[position] == "\n": print("n", end="") # for debugging return position + 1 @@ -271,7 +282,7 @@ def recursive_parser(code, position=0, scope=""): elif scope == "//": print("//", end="") # for debugging while position < len(code): - + outfile.write(code[position]) if code[position] == "\n": print("n", end="") # for debugging return position + 1 @@ -282,10 +293,11 @@ def recursive_parser(code, position=0, scope=""): elif scope == "/*": print("/*", end="") # for debugging while position < len(code): - + outfile.write(code[position]) # check for c-style comment closing if code[position] == "*": if code[position + 1] == "/": + outfile.write(code[position + 1]) # TODO remove print("*/", end="") # for debugging return position + 2 @@ -295,7 +307,7 @@ def recursive_parser(code, position=0, scope=""): elif scope == "\'": print("\'^", end="") # for debugging while position < len(code): - + outfile.write(code[position]) # check for single-quote string ending if code[position] == "\'": # check if its escaped @@ -311,7 +323,7 @@ def recursive_parser(code, position=0, scope=""): elif scope == "\"": print("\"^", end="") # for debugging while position < len(code): - + outfile.write(code[position]) # check for single-quote string ending if code[position] == "\"": # check if its escaped @@ -325,13 +337,13 @@ def recursive_parser(code, position=0, scope=""): position = position + 1 elif scope == "=": + identation = identation + 1 print("=", end="") # for debugging while position < len(code): - # check for dicts if code[position] == "{": print(".dict.", end="") # for debugging - return recursive_parser(code, position + 1, "={") + return recursive_parser(code, position + 1, "={", outfile, identation) # check for non dicts elif re.search(r"[^\s\n\r]", code[position]): @@ -339,9 +351,11 @@ def recursive_parser(code, position=0, scope=""): return position else: + outfile.write(code[position]) position = position + 1 elif scope == "={": + outfile.write(code[position]) while position < len(code): if code[position] == "}": return position + 1 @@ -353,9 +367,15 @@ def recursive_parser(code, position=0, scope=""): raise Exception("invalid scope was reached") - # open file and read contents - infile = open(filepath, "r") + filename = os.path.basename(filepath) + filedir = os.path.dirname(filepath) + + infile = open(filepath, 'r') infile_str = infile.read() infile.close() - recursive_parser(infile_str) \ No newline at end of file + outfile = open(filename_prefix + _change_file_name(filename, outputname), 'w') + + recursive_parser(infile_str, 0, "", outfile, 0) + + outfile.close() \ No newline at end of file diff --git a/test-code.py b/test-code.py new file mode 100644 index 0000000..b69c5b2 --- /dev/null +++ b/test-code.py @@ -0,0 +1,27 @@ +# this file is for testing purposes only +# TODO remove this file later + +def print_hello_world() : + str = "he\"ll\"o"; # hello + str += ' world'; # world + /* + print("error - not yet implemented"); + while(true) // remove + { + sleep(1000); + + } + */ + mydictionary = + + if str == "hello world" : + print("its equal") + + else + : + print("not equal") + + print(str) #///# let's print it! + + +print_hello_world(); \ No newline at end of file From a4d6e9ca73484e3da57e8dff5f082cb59dede213 Mon Sep 17 00:00:00 2001 From: Filipe Jordan do Nascimento Date: Thu, 16 May 2019 21:21:03 -0300 Subject: [PATCH 08/35] implemented a function for removing indentation --- bython/parser.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bython/parser.py b/bython/parser.py index 9b46dbd..1da3a15 100644 --- a/bython/parser.py +++ b/bython/parser.py @@ -182,6 +182,12 @@ def parse_file(filepath, add_true_line, filename_prefix, outputname=None, change outfile.close() +def remove_indentation(code): + code = re.sub(r"^[ \t]*", "", code, 1) # remove spaces a the start of the file + code = re.sub(r"\n\r[ \t]+", "\n\r", code) + code = re.sub(r"\n[ \t]+", "\n", code) + return code + def parse_file_recursive(filepath, add_true_line=False, filename_prefix="", outputname=None, change_imports=None): """ Converts a bython file to a python file recursively and writes it to disk. @@ -376,6 +382,7 @@ def recursive_parser(code, position, scope, outfile, identation): outfile = open(filename_prefix + _change_file_name(filename, outputname), 'w') + infile_str = remove_indentation(infile_str) recursive_parser(infile_str, 0, "", outfile, 0) outfile.close() \ No newline at end of file From 070f19a5e5aa05c4b039f1c35f9e6265c3ae8841 Mon Sep 17 00:00:00 2001 From: Filipe Jordan do Nascimento Date: Thu, 16 May 2019 22:01:27 -0300 Subject: [PATCH 09/35] implemented functions for preparing braces and removing empty lines. also changed the remove_identation function --- bython/parser.py | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/bython/parser.py b/bython/parser.py index 1da3a15..bfa1bcd 100644 --- a/bython/parser.py +++ b/bython/parser.py @@ -183,11 +183,25 @@ def parse_file(filepath, add_true_line, filename_prefix, outputname=None, change def remove_indentation(code): - code = re.sub(r"^[ \t]*", "", code, 1) # remove spaces a the start of the file - code = re.sub(r"\n\r[ \t]+", "\n\r", code) - code = re.sub(r"\n[ \t]+", "\n", code) + code = re.sub(r"^[ \t]*", "", code, 1) + code = re.sub(r"\r?\n[ \t]+", "\n", code) return code + +def prepare_braces(code): + # TODO fix issue with comment on the line before the brace + # TODO fix issue with brance within comments + code = re.sub(r" \{", "{", code) + code = re.sub(r"[ \t]*\r?\n[ \t]*\{", "{", code) + code = re.sub(r"\{", "{\n", code) + return code + + +def remove_empty_lines(code): + code = re.sub(r"\r?\n[ \t]*(\r?\n[ \t]*)+", "\n", code) + return code + + def parse_file_recursive(filepath, add_true_line=False, filename_prefix="", outputname=None, change_imports=None): """ Converts a bython file to a python file recursively and writes it to disk. @@ -383,6 +397,14 @@ def recursive_parser(code, position, scope, outfile, identation): outfile = open(filename_prefix + _change_file_name(filename, outputname), 'w') infile_str = remove_indentation(infile_str) + infile_str = prepare_braces(infile_str) + infile_str = remove_empty_lines(infile_str) + + # TODO remove + filteredfile = open(filename + ".filtered", 'w') + filteredfile.write(infile_str) + filteredfile.close() + recursive_parser(infile_str, 0, "", outfile, 0) outfile.close() \ No newline at end of file From a5ca35916b7f0ca024a44c4e0aa273904fb56919 Mon Sep 17 00:00:00 2001 From: Filipe Jordan do Nascimento Date: Thu, 16 May 2019 22:25:22 -0300 Subject: [PATCH 10/35] implemented indentation --- bython/parser.py | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/bython/parser.py b/bython/parser.py index bfa1bcd..1dabe03 100644 --- a/bython/parser.py +++ b/bython/parser.py @@ -202,6 +202,13 @@ def remove_empty_lines(code): return code +def indent_if_newline(code, outfile, indentation, indentation_str): + if code == "\n": + #print(identation, end="") + for x in range(indentation): + outfile.write(indentation_str) + + def parse_file_recursive(filepath, add_true_line=False, filename_prefix="", outputname=None, change_imports=None): """ Converts a bython file to a python file recursively and writes it to disk. @@ -225,7 +232,7 @@ def parse_file_recursive(filepath, add_true_line=False, filename_prefix="", outp # i've put them there for ease of use # inner function for parsing recursively - def recursive_parser(code, position, scope, outfile, identation): + def recursive_parser(code, position, scope, outfile, indentation, indentation_str=" "): # scope equal to "" means it's on global scope # scope equal to "{" means it's on local scope @@ -234,7 +241,7 @@ def recursive_parser(code, position, scope, outfile, identation): if scope == "": print("g", end="") # for debugging else: - identation = identation + 1 + indentation = indentation + 1 while position < len(code): @@ -242,38 +249,38 @@ def recursive_parser(code, position, scope, outfile, identation): if code[position] == "{": print("{", end="") # for debugging outfile.write(":") - position = recursive_parser(code, position + 1, "{", outfile, identation) + position = recursive_parser(code, position + 1, "{", outfile, indentation + 1, indentation_str) if scope == "": print("g", end="") # for debugging # check for python-style comment elif code[position] == "#": outfile.write(code[position]) - position = recursive_parser(code, position + 1, "#", outfile, identation) + position = recursive_parser(code, position + 1, "#", outfile, indentation, indentation_str) # check for c and cpp-style comment elif code[position] == "/": if code[position + 1] == "/": outfile.write("#") - position = recursive_parser(code, position + 2, "//", outfile, identation) + position = recursive_parser(code, position + 2, "//", outfile, indentation, indentation_str) elif code[position + 1] == "*": outfile.write(code[position:position+2]) # TODO implement comment on all lines - position = recursive_parser(code, position + 2, "/*", outfile, identation) + position = recursive_parser(code, position + 2, "/*", outfile, indentation, indentation_str) # check for single-quote string start elif code[position] == "\'": outfile.write(code[position]) - position = recursive_parser(code, position + 1, "\'", outfile, identation) + position = recursive_parser(code, position + 1, "\'", outfile, indentation, indentation_str) # check for double-quote string start elif code[position] == "\"": outfile.write(code[position]) - position = recursive_parser(code, position + 1, "\"", outfile, identation) + position = recursive_parser(code, position + 1, "\"", outfile, indentation, indentation_str) # check for equals (for python dicts with braces) elif code[position] == "=": outfile.write(code[position]) - position = recursive_parser(code, position + 1, "=", outfile, identation) + position = recursive_parser(code, position + 1, "=", outfile, indentation, indentation_str) # check for brace closing (when not on global) elif scope == "{": @@ -282,16 +289,19 @@ def recursive_parser(code, position, scope, outfile, identation): return position + 1 else: outfile.write(code[position]) + indent_if_newline(code[position], outfile, indentation, indentation_str) position = position + 1 else: outfile.write(code[position]) + indent_if_newline(code[position], outfile, indentation, indentation_str) position = position + 1 elif scope == "#": print("#", end="") # for debugging while position < len(code): outfile.write(code[position]) + indent_if_newline(code[position], outfile, indentation, indentation_str) if code[position] == "\n": print("n", end="") # for debugging return position + 1 @@ -303,6 +313,7 @@ def recursive_parser(code, position, scope, outfile, identation): print("//", end="") # for debugging while position < len(code): outfile.write(code[position]) + indent_if_newline(code[position], outfile, indentation, indentation_str) if code[position] == "\n": print("n", end="") # for debugging return position + 1 @@ -314,6 +325,7 @@ def recursive_parser(code, position, scope, outfile, identation): print("/*", end="") # for debugging while position < len(code): outfile.write(code[position]) + indent_if_newline(code[position], outfile, indentation, indentation_str) # check for c-style comment closing if code[position] == "*": if code[position + 1] == "/": @@ -328,6 +340,7 @@ def recursive_parser(code, position, scope, outfile, identation): print("\'^", end="") # for debugging while position < len(code): outfile.write(code[position]) + indent_if_newline(code[position], outfile, indentation, indentation_str) # check for single-quote string ending if code[position] == "\'": # check if its escaped @@ -344,6 +357,7 @@ def recursive_parser(code, position, scope, outfile, identation): print("\"^", end="") # for debugging while position < len(code): outfile.write(code[position]) + indent_if_newline(code[position], outfile, indentation, indentation_str) # check for single-quote string ending if code[position] == "\"": # check if its escaped @@ -357,13 +371,13 @@ def recursive_parser(code, position, scope, outfile, identation): position = position + 1 elif scope == "=": - identation = identation + 1 + indentation = indentation + 1 print("=", end="") # for debugging while position < len(code): # check for dicts if code[position] == "{": print(".dict.", end="") # for debugging - return recursive_parser(code, position + 1, "={", outfile, identation) + return recursive_parser(code, position + 1, "={", outfile, indentation + 1, indentation_str) # check for non dicts elif re.search(r"[^\s\n\r]", code[position]): @@ -372,10 +386,12 @@ def recursive_parser(code, position, scope, outfile, identation): else: outfile.write(code[position]) + indent_if_newline(code[position], outfile, indentation, indentation_str) position = position + 1 elif scope == "={": outfile.write(code[position]) + indent_if_newline(code[position], outfile, indentation, indentation_str) while position < len(code): if code[position] == "}": return position + 1 @@ -405,6 +421,6 @@ def recursive_parser(code, position, scope, outfile, identation): filteredfile.write(infile_str) filteredfile.close() - recursive_parser(infile_str, 0, "", outfile, 0) + recursive_parser(infile_str, 0, "", outfile, 0, " ") outfile.close() \ No newline at end of file From 2713ce19bc894d81ee3d6e86fbde58da2d831645 Mon Sep 17 00:00:00 2001 From: Filipe Jordan do Nascimento Date: Thu, 16 May 2019 22:33:02 -0300 Subject: [PATCH 11/35] fixed python dicts --- bython/parser.py | 11 +++++++++-- test-code.by | 7 +++++-- test-code.by.filtered | 25 ++++++++++++++++++++++++ test-code.py | 44 +++++++++++++++++++++---------------------- 4 files changed, 60 insertions(+), 27 deletions(-) create mode 100644 test-code.by.filtered diff --git a/bython/parser.py b/bython/parser.py index 1dabe03..e26846d 100644 --- a/bython/parser.py +++ b/bython/parser.py @@ -376,11 +376,15 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st while position < len(code): # check for dicts if code[position] == "{": + outfile.write(code[position]) + indent_if_newline(code[position], outfile, indentation, indentation_str) print(".dict.", end="") # for debugging return recursive_parser(code, position + 1, "={", outfile, indentation + 1, indentation_str) # check for non dicts elif re.search(r"[^\s\n\r]", code[position]): + outfile.write(code[position]) + indent_if_newline(code[position], outfile, indentation, indentation_str) print("!", end="") # for debugging return position @@ -390,13 +394,16 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st position = position + 1 elif scope == "={": - outfile.write(code[position]) - indent_if_newline(code[position], outfile, indentation, indentation_str) while position < len(code): + if code[position] == "}": + outfile.write(code[position]) + indent_if_newline(code[position], outfile, indentation - 1, indentation_str) return position + 1 else: + outfile.write(code[position]) + indent_if_newline(code[position], outfile, indentation, indentation_str) position = position + 1 else: diff --git a/test-code.by b/test-code.by index 1ac3dde..b3dfd2d 100644 --- a/test-code.by +++ b/test-code.by @@ -7,11 +7,14 @@ def print_hello_world() { /* print("error - not yet implemented"); while(true) // remove - { - sleep(1000); + { sleep(1000); } */ + + + + mydictionary = { 'key1': "value1", 'key2': 2, diff --git a/test-code.by.filtered b/test-code.by.filtered new file mode 100644 index 0000000..b776a60 --- /dev/null +++ b/test-code.by.filtered @@ -0,0 +1,25 @@ +# this file is for testing purposes only +# TODO remove this file later +def print_hello_world(){ +str = "he\"ll\"o"; # hello +str += ' world'; // world +/* +print("error - not yet implemented"); +while(true) // remove{ + sleep(1000); +} +*/ +mydictionary ={ +'key1': "value1", +'key2': 2, +'key3': "hi", +} +if str == "hello world"{ +print("its equal") +} +else{ +print("not equal") +} +print(str) /////# let's print it! +} +print_hello_world(); \ No newline at end of file diff --git a/test-code.py b/test-code.py index b69c5b2..8cbf230 100644 --- a/test-code.py +++ b/test-code.py @@ -1,27 +1,25 @@ # this file is for testing purposes only # TODO remove this file later - -def print_hello_world() : - str = "he\"ll\"o"; # hello - str += ' world'; # world - /* - print("error - not yet implemented"); - while(true) // remove - { - sleep(1000); +def print_hello_world(): + str = ""he\"ll\"o"; # hello + str += '' world'; # world + /* + print("error - not yet implemented"); + while(true) // remove{ + sleep(1000); + } + */ + mydictionary ={ + 'key1': "value1", + 'key2': 2, + 'key3': "hi", + } + if str === ""hello world": + print("its equal") + + else: + print("not equal") + + print(str) #///# let's print it! - } - */ - mydictionary = - - if str == "hello world" : - print("its equal") - - else - : - print("not equal") - - print(str) #///# let's print it! - - print_hello_world(); \ No newline at end of file From a349d6c4908b6f8b3f61a16c72cbee3ea53a2684 Mon Sep 17 00:00:00 2001 From: Filipe Jordan do Nascimento Date: Mon, 20 May 2019 16:14:44 -0300 Subject: [PATCH 12/35] fixed error when there's a comment after a call and before the opening brace --- bython/parser.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/bython/parser.py b/bython/parser.py index e26846d..021e9f3 100644 --- a/bython/parser.py +++ b/bython/parser.py @@ -189,10 +189,9 @@ def remove_indentation(code): def prepare_braces(code): - # TODO fix issue with comment on the line before the brace - # TODO fix issue with brance within comments - code = re.sub(r" \{", "{", code) - code = re.sub(r"[ \t]*\r?\n[ \t]*\{", "{", code) + # TODO fix issue with brace within comments + code = re.sub(r"[ \t]*\{", "{", code) + code = re.sub(r"[ \t]*(\/\/.*)?(\#.*)?\r?\n[ \t]*{", "{", code) code = re.sub(r"\{", "{\n", code) return code From daa7e3e1465d2b4fee17267bb7221d7e39a1bd2d Mon Sep 17 00:00:00 2001 From: Filipe Jordan do Nascimento Date: Mon, 20 May 2019 16:26:34 -0300 Subject: [PATCH 13/35] implemented the removal of semicolons --- bython/parser.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/bython/parser.py b/bython/parser.py index 021e9f3..25b3108 100644 --- a/bython/parser.py +++ b/bython/parser.py @@ -196,6 +196,12 @@ def prepare_braces(code): return code +def remove_semicolons(code): + code = re.sub(r";\r?(?=\n)", "", code) + code = re.sub(r";$", "", code) + return code + + def remove_empty_lines(code): code = re.sub(r"\r?\n[ \t]*(\r?\n[ \t]*)+", "\n", code) return code @@ -234,7 +240,7 @@ def parse_file_recursive(filepath, add_true_line=False, filename_prefix="", outp def recursive_parser(code, position, scope, outfile, indentation, indentation_str=" "): # scope equal to "" means it's on global scope - # scope equal to "{" means it's on local scope + # scope equal to "{" means it's on a local scope if scope == "" or scope == "{": if scope == "": @@ -248,7 +254,7 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st if code[position] == "{": print("{", end="") # for debugging outfile.write(":") - position = recursive_parser(code, position + 1, "{", outfile, indentation + 1, indentation_str) + position = recursive_parser(code, position + 1, "{", outfile, indentation, indentation_str) if scope == "": print("g", end="") # for debugging @@ -421,6 +427,7 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st infile_str = remove_indentation(infile_str) infile_str = prepare_braces(infile_str) infile_str = remove_empty_lines(infile_str) + infile_str = remove_semicolons(infile_str) # TODO remove filteredfile = open(filename + ".filtered", 'w') From 83ed356b885423bbff22d9ce8d9dca600e37f794 Mon Sep 17 00:00:00 2001 From: Filipe Jordan do Nascimento Date: Mon, 20 May 2019 16:37:20 -0300 Subject: [PATCH 14/35] fixed double indentation and wrong indentation after dict --- bython/parser.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bython/parser.py b/bython/parser.py index 25b3108..8a2d0a2 100644 --- a/bython/parser.py +++ b/bython/parser.py @@ -376,7 +376,6 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st position = position + 1 elif scope == "=": - indentation = indentation + 1 print("=", end="") # for debugging while position < len(code): # check for dicts @@ -403,12 +402,14 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st if code[position] == "}": outfile.write(code[position]) - indent_if_newline(code[position], outfile, indentation - 1, indentation_str) return position + 1 else: outfile.write(code[position]) - indent_if_newline(code[position], outfile, indentation, indentation_str) + if code[position + 1] == "}": + indent_if_newline(code[position], outfile, indentation - 1, indentation_str) + else: + indent_if_newline(code[position], outfile, indentation, indentation_str) position = position + 1 else: From 682587661bc0fb8bb180427766260f742b4515d9 Mon Sep 17 00:00:00 2001 From: Filipe Jordan do Nascimento Date: Mon, 20 May 2019 17:02:14 -0300 Subject: [PATCH 15/35] implemented multi-line comments --- bython/parser.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/bython/parser.py b/bython/parser.py index 8a2d0a2..970db92 100644 --- a/bython/parser.py +++ b/bython/parser.py @@ -269,17 +269,16 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st outfile.write("#") position = recursive_parser(code, position + 2, "//", outfile, indentation, indentation_str) elif code[position + 1] == "*": - outfile.write(code[position:position+2]) # TODO implement comment on all lines + outfile.write("#") + outfile.write(code[position:position+2]) position = recursive_parser(code, position + 2, "/*", outfile, indentation, indentation_str) # check for single-quote string start elif code[position] == "\'": - outfile.write(code[position]) position = recursive_parser(code, position + 1, "\'", outfile, indentation, indentation_str) # check for double-quote string start elif code[position] == "\"": - outfile.write(code[position]) position = recursive_parser(code, position + 1, "\"", outfile, indentation, indentation_str) # check for equals (for python dicts with braces) @@ -331,13 +330,16 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st while position < len(code): outfile.write(code[position]) indent_if_newline(code[position], outfile, indentation, indentation_str) + if code[position] == "\n": + outfile.write("#") + # check for c-style comment closing if code[position] == "*": if code[position + 1] == "/": - outfile.write(code[position + 1]) # TODO remove print("*/", end="") # for debugging + outfile.write(code[position + 1]) return position + 2 - + else: position = position + 1 From 5041fa2571b3cf6696033d2be263bc8edd45f607 Mon Sep 17 00:00:00 2001 From: Filipe Jordan do Nascimento Date: Mon, 20 May 2019 17:13:03 -0300 Subject: [PATCH 16/35] fixed doubled character after non-dicts --- bython/parser.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/bython/parser.py b/bython/parser.py index 970db92..763c96c 100644 --- a/bython/parser.py +++ b/bython/parser.py @@ -235,7 +235,9 @@ def parse_file_recursive(filepath, add_true_line=False, filename_prefix="", outp # TODO remove defaults for the parameters 'add_true_line' and 'filename_prefix' # i've put them there for ease of use - + # TODO create a class and put all the new function in it + # TODO make the code cleaner + # inner function for parsing recursively def recursive_parser(code, position, scope, outfile, indentation, indentation_str=" "): @@ -275,10 +277,12 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st # check for single-quote string start elif code[position] == "\'": + outfile.write("\'") position = recursive_parser(code, position + 1, "\'", outfile, indentation, indentation_str) # check for double-quote string start elif code[position] == "\"": + outfile.write("\"") position = recursive_parser(code, position + 1, "\"", outfile, indentation, indentation_str) # check for equals (for python dicts with braces) @@ -387,17 +391,17 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st print(".dict.", end="") # for debugging return recursive_parser(code, position + 1, "={", outfile, indentation + 1, indentation_str) - # check for non dicts - elif re.search(r"[^\s\n\r]", code[position]): + # check for whitespaces/newlines + elif re.search(r"[\s\n\r]", code[position]): outfile.write(code[position]) indent_if_newline(code[position], outfile, indentation, indentation_str) - print("!", end="") # for debugging - return position + position = position + 1 + # if it gets here, non-dict was found else: - outfile.write(code[position]) indent_if_newline(code[position], outfile, indentation, indentation_str) - position = position + 1 + print("!", end="") # for debugging + return position elif scope == "={": while position < len(code): From d870bd101b417157bbce60a23da0f71db640942d Mon Sep 17 00:00:00 2001 From: Filipe Jordan do Nascimento Date: Mon, 20 May 2019 17:20:25 -0300 Subject: [PATCH 17/35] disabled the remove_semicolons for now --- bython/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bython/parser.py b/bython/parser.py index 763c96c..aa97268 100644 --- a/bython/parser.py +++ b/bython/parser.py @@ -197,6 +197,7 @@ def prepare_braces(code): def remove_semicolons(code): + # TODO reimplement! not working properly when there are comments after it code = re.sub(r";\r?(?=\n)", "", code) code = re.sub(r";$", "", code) return code @@ -434,7 +435,6 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st infile_str = remove_indentation(infile_str) infile_str = prepare_braces(infile_str) infile_str = remove_empty_lines(infile_str) - infile_str = remove_semicolons(infile_str) # TODO remove filteredfile = open(filename + ".filtered", 'w') From 5e9cd48a1982c7701399a6fed1a8e0c7b99f4cdc Mon Sep 17 00:00:00 2001 From: Filipe Jordan do Nascimento Date: Thu, 23 May 2019 17:49:06 -0300 Subject: [PATCH 18/35] added some comments and minor fixes --- bython/parser.py | 36 +++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/bython/parser.py b/bython/parser.py index aa97268..4f6569c 100644 --- a/bython/parser.py +++ b/bython/parser.py @@ -168,7 +168,7 @@ def parse_file(filepath, add_true_line, filename_prefix, outputname=None, change # Support for extra, non-brace related stuff infile_str_indented = re.sub(r"else\s+if", "elif", infile_str_indented) - infile_str_indented = re.sub(r";\n", "\n", infile_str_indented) + infile_str_indented = re.sub(r";[ \t]+\n", "\n", infile_str_indented) # Change imported names if necessary if change_imports is not None: @@ -198,6 +198,8 @@ def prepare_braces(code): def remove_semicolons(code): # TODO reimplement! not working properly when there are comments after it + # TODO remove the single-line comments for now (?) + code = re.sub(r";\r?(?=\n)", "", code) code = re.sub(r";$", "", code) return code @@ -234,8 +236,6 @@ def parse_file_recursive(filepath, add_true_line=False, filename_prefix="", outp python alternative. """ - # TODO remove defaults for the parameters 'add_true_line' and 'filename_prefix' - # i've put them there for ease of use # TODO create a class and put all the new function in it # TODO make the code cleaner @@ -423,24 +423,50 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st raise Exception("invalid scope was reached") + # get filepath/filename filename = os.path.basename(filepath) - filedir = os.path.dirname(filepath) + # open input file infile = open(filepath, 'r') infile_str = infile.read() infile.close() + # open output file outfile = open(filename_prefix + _change_file_name(filename, outputname), 'w') + # true=True; false=False; + if add_true_line: + outfile.write("true=True\nfalse=False\n") + + # remove indentation infile_str = remove_indentation(infile_str) + + # rearrange braces infile_str = prepare_braces(infile_str) + + # remove empty lines infile_str = remove_empty_lines(infile_str) - # TODO remove + # change 'else if' into 'elif' + infile_str = re.sub(r"else\s+if", "elif", infile_str) + + # remove semicolons + infile_str = remove_semicolons(infile_str) + + # change imported names (if necessary) + if change_imports is not None: + for module in change_imports: + infile_str = re.sub("(?<=import\\s){}".format(module), "{} as {}".format(change_imports[module], module), infile_str) + infile_str = re.sub("(?<=from\\s){}(?=\\s+import)".format(module), change_imports[module], infile_str) + + # output filtered file (for debugging) + # TODO remove later filteredfile = open(filename + ".filtered", 'w') filteredfile.write(infile_str) filteredfile.close() + # start recursive function recursive_parser(infile_str, 0, "", outfile, 0, " ") + # close output file outfile.close() \ No newline at end of file From ca0ce33f7518a9c2ef73460476199c38571c2621 Mon Sep 17 00:00:00 2001 From: Filipe Jordan do Nascimento Date: Thu, 23 May 2019 21:06:07 -0300 Subject: [PATCH 19/35] fixed a few cases where comments would be removed --- bython/parser.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/bython/parser.py b/bython/parser.py index 4f6569c..ed8b3ee 100644 --- a/bython/parser.py +++ b/bython/parser.py @@ -191,16 +191,19 @@ def remove_indentation(code): def prepare_braces(code): # TODO fix issue with brace within comments code = re.sub(r"[ \t]*\{", "{", code) - code = re.sub(r"[ \t]*(\/\/.*)?(\#.*)?\r?\n[ \t]*{", "{", code) - code = re.sub(r"\{", "{\n", code) + code = re.sub(r"[ \t]*(\/\/.*|\#.*)?\r?\n[ \t]*\{", "{ \\1\n", code) return code def remove_semicolons(code): - # TODO reimplement! not working properly when there are comments after it - # TODO remove the single-line comments for now (?) + # remove semicolons, but keep any comments + # TODO fix: if a semicolon is follwed by a comment starter '//' or '#', the semicolon will be removed even inside strings or comments + code = re.sub(r"[ \t]*;[ \t]*(\/\/.*|\#.*)?\r?(?=\n)", " \\1", code) - code = re.sub(r";\r?(?=\n)", "", code) + # remove any extra spaces added at the end of lines + code = re.sub(r"[ \t]\r?\n", "\n", code) + + # remove a semicolon placed right before the EOF code = re.sub(r";$", "", code) return code From 258af8abceb31b68536e48a2c5b301113bbca268 Mon Sep 17 00:00:00 2001 From: Filipe Jordan do Nascimento Date: Thu, 23 May 2019 21:15:49 -0300 Subject: [PATCH 20/35] renamed a function to parse_file_recursively --- bython/parser.py | 2 +- test-recursive.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bython/parser.py b/bython/parser.py index ed8b3ee..9b0680c 100644 --- a/bython/parser.py +++ b/bython/parser.py @@ -220,7 +220,7 @@ def indent_if_newline(code, outfile, indentation, indentation_str): outfile.write(indentation_str) -def parse_file_recursive(filepath, add_true_line=False, filename_prefix="", outputname=None, change_imports=None): +def parse_file_recursively(filepath, add_true_line=False, filename_prefix="", outputname=None, change_imports=None): """ Converts a bython file to a python file recursively and writes it to disk. diff --git a/test-recursive.py b/test-recursive.py index 6bd2576..b7270df 100644 --- a/test-recursive.py +++ b/test-recursive.py @@ -1,6 +1,6 @@ # this file is for testing purposes only # TODO remove this file later -from bython.parser import parse_file_recursive +from bython.parser import parse_file_recursively -parse_file_recursive("test-code.by") \ No newline at end of file +parse_file_recursively("test-code.by") \ No newline at end of file From 69bb5bfb96881c3c3402fd66e460a115c4df19ef Mon Sep 17 00:00:00 2001 From: Filipe Jordan do Nascimento Date: Fri, 24 May 2019 10:32:26 -0300 Subject: [PATCH 21/35] added a debug mode --- bython/parser.py | 57 +++++++++++++++++++++++++++++++----------------- 1 file changed, 37 insertions(+), 20 deletions(-) diff --git a/bython/parser.py b/bython/parser.py index 9b0680c..a736bc6 100644 --- a/bython/parser.py +++ b/bython/parser.py @@ -220,7 +220,7 @@ def indent_if_newline(code, outfile, indentation, indentation_str): outfile.write(indentation_str) -def parse_file_recursively(filepath, add_true_line=False, filename_prefix="", outputname=None, change_imports=None): +def parse_file_recursively(filepath, add_true_line=False, filename_prefix="", outputname=None, change_imports=None, debug_mode=False): """ Converts a bython file to a python file recursively and writes it to disk. @@ -243,14 +243,15 @@ def parse_file_recursively(filepath, add_true_line=False, filename_prefix="", ou # TODO make the code cleaner # inner function for parsing recursively - def recursive_parser(code, position, scope, outfile, indentation, indentation_str=" "): + def recursive_parser(code, position, scope, outfile, indentation, indentation_str=" ", debug_mode=False): # scope equal to "" means it's on global scope # scope equal to "{" means it's on a local scope if scope == "" or scope == "{": if scope == "": - print("g", end="") # for debugging + if debug_mode: + print("g", end="") # for debugging else: indentation = indentation + 1 @@ -258,11 +259,13 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st # check for brace opening if code[position] == "{": - print("{", end="") # for debugging + if debug_mode: + print("{", end="") # for debugging outfile.write(":") position = recursive_parser(code, position + 1, "{", outfile, indentation, indentation_str) if scope == "": - print("g", end="") # for debugging + if debug_mode: + print("g", end="") # for debugging # check for python-style comment elif code[position] == "#": @@ -297,7 +300,8 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st # check for brace closing (when not on global) elif scope == "{": if code[position] == "}": - print("}", end="") + if debug_mode: + print("}", end="") return position + 1 else: outfile.write(code[position]) @@ -310,31 +314,36 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st position = position + 1 elif scope == "#": - print("#", end="") # for debugging + if debug_mode: + print("#", end="") # for debugging while position < len(code): outfile.write(code[position]) indent_if_newline(code[position], outfile, indentation, indentation_str) if code[position] == "\n": - print("n", end="") # for debugging + if debug_mode: + print("n", end="") # for debugging return position + 1 else: position = position + 1 elif scope == "//": - print("//", end="") # for debugging + if debug_mode: + print("//", end="") # for debugging while position < len(code): outfile.write(code[position]) indent_if_newline(code[position], outfile, indentation, indentation_str) if code[position] == "\n": - print("n", end="") # for debugging + if debug_mode: + print("n", end="") # for debugging return position + 1 else: position = position + 1 elif scope == "/*": - print("/*", end="") # for debugging + if debug_mode: + print("/*", end="") # for debugging while position < len(code): outfile.write(code[position]) indent_if_newline(code[position], outfile, indentation, indentation_str) @@ -344,7 +353,8 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st # check for c-style comment closing if code[position] == "*": if code[position + 1] == "/": - print("*/", end="") # for debugging + if debug_mode: + print("*/", end="") # for debugging outfile.write(code[position + 1]) return position + 2 @@ -352,7 +362,8 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st position = position + 1 elif scope == "\'": - print("\'^", end="") # for debugging + if debug_mode: + print("\'^", end="") # for debugging while position < len(code): outfile.write(code[position]) indent_if_newline(code[position], outfile, indentation, indentation_str) @@ -360,7 +371,8 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st if code[position] == "\'": # check if its escaped if code[position - 1] != "\\": - print("$\'", end="") # for debugging + if debug_mode: + print("$\'", end="") # for debugging return position + 1 else: position = position + 1 @@ -369,7 +381,8 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st position = position + 1 elif scope == "\"": - print("\"^", end="") # for debugging + if debug_mode: + print("\"^", end="") # for debugging while position < len(code): outfile.write(code[position]) indent_if_newline(code[position], outfile, indentation, indentation_str) @@ -377,7 +390,8 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st if code[position] == "\"": # check if its escaped if code[position - 1] != "\\": - print("$\'", end="") # for debugging + if debug_mode: + print("$\'", end="") # for debugging return position + 1 else: position = position + 1 @@ -386,13 +400,15 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st position = position + 1 elif scope == "=": - print("=", end="") # for debugging + if debug_mode: + print("=", end="") # for debugging while position < len(code): # check for dicts if code[position] == "{": outfile.write(code[position]) indent_if_newline(code[position], outfile, indentation, indentation_str) - print(".dict.", end="") # for debugging + if debug_mode: + print(".dict.", end="") # for debugging return recursive_parser(code, position + 1, "={", outfile, indentation + 1, indentation_str) # check for whitespaces/newlines @@ -404,7 +420,8 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st # if it gets here, non-dict was found else: indent_if_newline(code[position], outfile, indentation, indentation_str) - print("!", end="") # for debugging + if debug_mode: + print("!", end="") # for debugging return position elif scope == "={": @@ -469,7 +486,7 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st filteredfile.close() # start recursive function - recursive_parser(infile_str, 0, "", outfile, 0, " ") + recursive_parser(infile_str, 0, "", outfile, 0, " ", debug_mode) # close output file outfile.close() \ No newline at end of file From e86c6a76c076fe91e361e587a8bb5579d650e87e Mon Sep 17 00:00:00 2001 From: Filipe Jordan do Nascimento Date: Fri, 24 May 2019 10:33:08 -0300 Subject: [PATCH 22/35] preparing for first release --- bython/__init__.py | 2 +- bython/importing.py | 2 +- scripts/bython | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) mode change 100644 => 100755 scripts/bython diff --git a/bython/__init__.py b/bython/__init__.py index c5a9edc..3d114aa 100644 --- a/bython/__init__.py +++ b/bython/__init__.py @@ -2,4 +2,4 @@ import bython.logger import bython.importing -VERSION_NUMBER = "0.8" +VERSION_NUMBER = "0.9" diff --git a/bython/importing.py b/bython/importing.py index a0f9dd9..3f6bd40 100644 --- a/bython/importing.py +++ b/bython/importing.py @@ -41,7 +41,7 @@ def bython_import(module_name, globals, logger=None): logger.log_info("Parsing %s" % path) try: - bython.parser.parse_file(path, False, os.path.join(sys.path[0], "python_")) + bython.parser.parse_file_recursively(path, False, os.path.join(sys.path[0], "python_")) error_during_parsing = None diff --git a/scripts/bython b/scripts/bython old mode 100644 new mode 100755 index e3d62ac..26376fa --- a/scripts/bython +++ b/scripts/bython @@ -31,7 +31,7 @@ def main(): formatter_class=argparse.RawTextHelpFormatter) argparser.add_argument("-V", "--version", action="version", - version="Bython v%s\nMathias Lohne and Tristan Pepin 2018" % VERSION_NUMBER) + version="Bython v%s\nMathias Lohne, Tristan Pepin and Filipe Nascimento 2019" % VERSION_NUMBER) argparser.add_argument("-v", "--verbose", help="print progress", action="store_true") @@ -133,7 +133,7 @@ def main(): else: outputname = cmd_args.output[0] - parser.parse_file(file, cmd_args.lower_true, path_prefix, outputname, import_translations) + parser.parse_file_recursively(file, cmd_args.lower_true, path_prefix, outputname, import_translations) except (TypeError, FileNotFoundError) as e: logger.log_error("Error while parsing '%s'.\n%s" % (current_file_name, str(e))) From cf452c7bfcfc828db68cb9d54d525159f001375a Mon Sep 17 00:00:00 2001 From: Filipe Jordan do Nascimento Date: Fri, 24 May 2019 14:20:18 -0300 Subject: [PATCH 23/35] disabled one regex substitution and fixed an infinite loop --- bython/parser.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/bython/parser.py b/bython/parser.py index a736bc6..90d5585 100644 --- a/bython/parser.py +++ b/bython/parser.py @@ -190,7 +190,8 @@ def remove_indentation(code): def prepare_braces(code): # TODO fix issue with brace within comments - code = re.sub(r"[ \t]*\{", "{", code) + # TODO fix removing spaces and tabs from within strings + #code = re.sub(r"[ \t]*\{", "{", code) code = re.sub(r"[ \t]*(\/\/.*|\#.*)?\r?\n[ \t]*\{", "{ \\1\n", code) return code @@ -220,7 +221,7 @@ def indent_if_newline(code, outfile, indentation, indentation_str): outfile.write(indentation_str) -def parse_file_recursively(filepath, add_true_line=False, filename_prefix="", outputname=None, change_imports=None, debug_mode=False): +def parse_file_recursively(filepath, add_true_line=False, filename_prefix="", outputname=None, change_imports=None, debug_mode=True): """ Converts a bython file to a python file recursively and writes it to disk. @@ -281,6 +282,9 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st outfile.write("#") outfile.write(code[position:position+2]) position = recursive_parser(code, position + 2, "/*", outfile, indentation, indentation_str) + else: + outfile.write("/") + position = position + 1 # check for single-quote string start elif code[position] == "\'": @@ -395,7 +399,6 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st return position + 1 else: position = position + 1 - else: position = position + 1 @@ -488,5 +491,8 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st # start recursive function recursive_parser(infile_str, 0, "", outfile, 0, " ", debug_mode) + if debug_mode: + print("\n", end="") + # close output file outfile.close() \ No newline at end of file From d1aeeae8b91994d4f08c4cd214a035b2b479b8f3 Mon Sep 17 00:00:00 2001 From: Filipe Jordan do Nascimento Date: Fri, 24 May 2019 14:43:37 -0300 Subject: [PATCH 24/35] implemented a debug option for the cli --- bython/parser.py | 18 +++++++++--------- scripts/bython | 5 ++++- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/bython/parser.py b/bython/parser.py index 90d5585..d044fa1 100644 --- a/bython/parser.py +++ b/bython/parser.py @@ -221,7 +221,7 @@ def indent_if_newline(code, outfile, indentation, indentation_str): outfile.write(indentation_str) -def parse_file_recursively(filepath, add_true_line=False, filename_prefix="", outputname=None, change_imports=None, debug_mode=True): +def parse_file_recursively(filepath, add_true_line=False, filename_prefix="", outputname=None, change_imports=None, debug_mode=False): """ Converts a bython file to a python file recursively and writes it to disk. @@ -263,7 +263,7 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st if debug_mode: print("{", end="") # for debugging outfile.write(":") - position = recursive_parser(code, position + 1, "{", outfile, indentation, indentation_str) + position = recursive_parser(code, position + 1, "{", outfile, indentation, indentation_str, debug_mode) if scope == "": if debug_mode: print("g", end="") # for debugging @@ -271,17 +271,17 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st # check for python-style comment elif code[position] == "#": outfile.write(code[position]) - position = recursive_parser(code, position + 1, "#", outfile, indentation, indentation_str) + position = recursive_parser(code, position + 1, "#", outfile, indentation, indentation_str, debug_mode) # check for c and cpp-style comment elif code[position] == "/": if code[position + 1] == "/": outfile.write("#") - position = recursive_parser(code, position + 2, "//", outfile, indentation, indentation_str) + position = recursive_parser(code, position + 2, "//", outfile, indentation, indentation_str, debug_mode) elif code[position + 1] == "*": outfile.write("#") outfile.write(code[position:position+2]) - position = recursive_parser(code, position + 2, "/*", outfile, indentation, indentation_str) + position = recursive_parser(code, position + 2, "/*", outfile, indentation, indentation_str, debug_mode) else: outfile.write("/") position = position + 1 @@ -289,17 +289,17 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st # check for single-quote string start elif code[position] == "\'": outfile.write("\'") - position = recursive_parser(code, position + 1, "\'", outfile, indentation, indentation_str) + position = recursive_parser(code, position + 1, "\'", outfile, indentation, indentation_str, debug_mode) # check for double-quote string start elif code[position] == "\"": outfile.write("\"") - position = recursive_parser(code, position + 1, "\"", outfile, indentation, indentation_str) + position = recursive_parser(code, position + 1, "\"", outfile, indentation, indentation_str, debug_mode) # check for equals (for python dicts with braces) elif code[position] == "=": outfile.write(code[position]) - position = recursive_parser(code, position + 1, "=", outfile, indentation, indentation_str) + position = recursive_parser(code, position + 1, "=", outfile, indentation, indentation_str, debug_mode) # check for brace closing (when not on global) elif scope == "{": @@ -412,7 +412,7 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st indent_if_newline(code[position], outfile, indentation, indentation_str) if debug_mode: print(".dict.", end="") # for debugging - return recursive_parser(code, position + 1, "={", outfile, indentation + 1, indentation_str) + return recursive_parser(code, position + 1, "={", outfile, indentation + 1, indentation_str, debug_mode) # check for whitespaces/newlines elif re.search(r"[\s\n\r]", code[position]): diff --git a/scripts/bython b/scripts/bython index 26376fa..c62a0b6 100755 --- a/scripts/bython +++ b/scripts/bython @@ -47,6 +47,9 @@ def main(): argparser.add_argument("-2", "--python2", help="use python2 instead of python3 (default)", action="store_true") + argparser.add_argument("-d", "--debug", + help="enable debug mode", + action="store_true") argparser.add_argument("-o", "--output", type=str, help="specify name of output file (if -c is present)", @@ -133,7 +136,7 @@ def main(): else: outputname = cmd_args.output[0] - parser.parse_file_recursively(file, cmd_args.lower_true, path_prefix, outputname, import_translations) + parser.parse_file_recursively(file, cmd_args.lower_true, path_prefix, outputname, import_translations, cmd_args.debug) except (TypeError, FileNotFoundError) as e: logger.log_error("Error while parsing '%s'.\n%s" % (current_file_name, str(e))) From 611c1c2ba6658d9d0f4fa9e6c26da218efae7179 Mon Sep 17 00:00:00 2001 From: Filipe Jordan do Nascimento Date: Sat, 25 May 2019 19:34:42 -0300 Subject: [PATCH 25/35] fixed empty lines bug. filtered file now only appears when debug mode is active --- bython/parser.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/bython/parser.py b/bython/parser.py index d044fa1..3b43eec 100644 --- a/bython/parser.py +++ b/bython/parser.py @@ -216,7 +216,6 @@ def remove_empty_lines(code): def indent_if_newline(code, outfile, indentation, indentation_str): if code == "\n": - #print(identation, end="") for x in range(indentation): outfile.write(indentation_str) @@ -308,8 +307,11 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st print("}", end="") return position + 1 else: - outfile.write(code[position]) - indent_if_newline(code[position], outfile, indentation, indentation_str) + if code[position] == "\n" and code[position + 1] == "}": + pass + else: + outfile.write(code[position]) + indent_if_newline(code[position], outfile, indentation, indentation_str) position = position + 1 else: @@ -484,9 +486,10 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st # output filtered file (for debugging) # TODO remove later - filteredfile = open(filename + ".filtered", 'w') - filteredfile.write(infile_str) - filteredfile.close() + if debug_mode: + filteredfile = open(filename + ".filtered", 'w') + filteredfile.write(infile_str) + filteredfile.close() # start recursive function recursive_parser(infile_str, 0, "", outfile, 0, " ", debug_mode) From 5b69929b0985ed62ebe0726c9f96b2f009353c54 Mon Sep 17 00:00:00 2001 From: Filipe Jordan do Nascimento Date: Sat, 25 May 2019 20:06:11 -0300 Subject: [PATCH 26/35] fixed '} else {' --- bython/parser.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bython/parser.py b/bython/parser.py index 3b43eec..5569c16 100644 --- a/bython/parser.py +++ b/bython/parser.py @@ -191,8 +191,10 @@ def remove_indentation(code): def prepare_braces(code): # TODO fix issue with brace within comments # TODO fix removing spaces and tabs from within strings - #code = re.sub(r"[ \t]*\{", "{", code) code = re.sub(r"[ \t]*(\/\/.*|\#.*)?\r?\n[ \t]*\{", "{ \\1\n", code) + code = re.sub(r"(?<=\})[ \t]+", "", code) + code = re.sub(r"(?<=\n\})([^\n]+?)(?=\n)", "\n\\1", code) + code = re.sub(r"[ \t]+(?=\{\n)", "", code) return code From ffc4a9cdc0cec77b1b743b311a89efd028921e2f Mon Sep 17 00:00:00 2001 From: Filipe Jordan do Nascimento Date: Sat, 25 May 2019 20:19:50 -0300 Subject: [PATCH 27/35] implemented parenthesis scope --- bython/parser.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/bython/parser.py b/bython/parser.py index 5569c16..422f6d8 100644 --- a/bython/parser.py +++ b/bython/parser.py @@ -249,7 +249,7 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st # scope equal to "" means it's on global scope # scope equal to "{" means it's on a local scope - if scope == "" or scope == "{": + if scope == "" or scope == "{" or scope == "(": if scope == "": if debug_mode: @@ -268,6 +268,13 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st if scope == "": if debug_mode: print("g", end="") # for debugging + + # check for parenthesis opening + if code[position] == "(": + if debug_mode: + print("(", end="") # for debugging + outfile.write(code[position]) + position = recursive_parser(code, position + 1, "(", outfile, indentation, indentation_str, debug_mode) # check for python-style comment elif code[position] == "#": @@ -315,6 +322,15 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st outfile.write(code[position]) indent_if_newline(code[position], outfile, indentation, indentation_str) position = position + 1 + + elif scope == "(": + outfile.write(code[position]) + indent_if_newline(code[position], outfile, indentation, indentation_str) + if code[position] == ")": + if debug_mode: + print(")", end="") + return position + 1 + position = position + 1 else: outfile.write(code[position]) From 1ed72b60d9cf3be7dc61948b105752eebcf0ac01 Mon Sep 17 00:00:00 2001 From: Filipe Jordan do Nascimento Date: Sat, 25 May 2019 22:37:51 -0300 Subject: [PATCH 28/35] added function descriptions and more comments --- bython/parser.py | 88 +++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 80 insertions(+), 8 deletions(-) diff --git a/bython/parser.py b/bython/parser.py index 422f6d8..68ed198 100644 --- a/bython/parser.py +++ b/bython/parser.py @@ -183,12 +183,33 @@ def parse_file(filepath, add_true_line, filename_prefix, outputname=None, change def remove_indentation(code): + """ + Removes indentation from string + + Args: + code (str): The string that will be manipulated + + Returns: + str: A string with the changes applied + """ code = re.sub(r"^[ \t]*", "", code, 1) code = re.sub(r"\r?\n[ \t]+", "\n", code) return code def prepare_braces(code): + """ + Moves braces to the optimal position. E.g.: "while x < 10{". + Removes additional spacing after "}". + Ensures comments come after the opening braces. E.g. "while x < 10{ // comment". + + Args: + code (str): The string that will be manipulated + + Returns: + str: A string with the changes applied + """ + # TODO fix issue with brace within comments # TODO fix removing spaces and tabs from within strings code = re.sub(r"[ \t]*(\/\/.*|\#.*)?\r?\n[ \t]*\{", "{ \\1\n", code) @@ -199,6 +220,15 @@ def prepare_braces(code): def remove_semicolons(code): + """ + Find and removes the semicolons placed at the end of lines + + Args: + code (str): The string that will be manipulated + + Returns: + str: A string with the changes applied + """ # remove semicolons, but keep any comments # TODO fix: if a semicolon is follwed by a comment starter '//' or '#', the semicolon will be removed even inside strings or comments code = re.sub(r"[ \t]*;[ \t]*(\/\/.*|\#.*)?\r?(?=\n)", " \\1", code) @@ -212,11 +242,29 @@ def remove_semicolons(code): def remove_empty_lines(code): + """ + Find and removes empty lines + + Args: + code (str): The string that will be manipulated + + Returns: + str: A string with the changes applied + """ code = re.sub(r"\r?\n[ \t]*(\r?\n[ \t]*)+", "\n", code) return code def indent_if_newline(code, outfile, indentation, indentation_str): + """ + Applies indentation. + + Args: + code (str): The string that will be manipulated + outfile (file): The file which will be indented + indentation (int): The desired indentation level + indentation_str (str): The indentation style (usually spaces or tabs) + """ if code == "\n": for x in range(indentation): outfile.write(indentation_str) @@ -239,16 +287,30 @@ def parse_file_recursively(filepath, add_true_line=False, filename_prefix="", ou '.py' change_imports (dict): Names of imported bython modules, and their python alternative. + debug_mode (boolean): Enables debug output (scope detection) """ - - # TODO create a class and put all the new function in it - # TODO make the code cleaner # inner function for parsing recursively def recursive_parser(code, position, scope, outfile, indentation, indentation_str=" ", debug_mode=False): - + """ + Recursive inner function for scope detection and for writing the final .py code to disk + + Args: + code (str): The string that will be interpreted + position (int): Current position on the code string + scope (str): Current scope ("", "{", "(", "#", "//" + "/*", "/'", "/"", "=", "={") + outfile (file) The output file which will be writen to + indentation (int): The current indentation level + indentation_str (str): The indentation style (usually spaces or tabs) + debug_mode (boolean): Enables debug output (scope detection) + + Returns: + int: Next position on the string + """ # scope equal to "" means it's on global scope # scope equal to "{" means it's on a local scope + # scope equal to "(" means it's inside function parameters or tuples if scope == "" or scope == "{" or scope == "(": if scope == "": @@ -257,6 +319,7 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st else: indentation = indentation + 1 + # keep parsing until EOF while position < len(code): # check for brace opening @@ -323,6 +386,7 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st indent_if_newline(code[position], outfile, indentation, indentation_str) position = position + 1 + # check for parenthesis opening elif scope == "(": outfile.write(code[position]) indent_if_newline(code[position], outfile, indentation, indentation_str) @@ -337,6 +401,7 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st indent_if_newline(code[position], outfile, indentation, indentation_str) position = position + 1 + # scope equal to "#" means it's inside a python style comment elif scope == "#": if debug_mode: print("#", end="") # for debugging @@ -351,6 +416,7 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st else: position = position + 1 + # scope equal to "//" means it's inside a c++ style comment elif scope == "//": if debug_mode: print("//", end="") # for debugging @@ -365,6 +431,7 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st else: position = position + 1 + # scope equal to "/*" means it's inside a c style comment elif scope == "/*": if debug_mode: print("/*", end="") # for debugging @@ -385,6 +452,7 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st else: position = position + 1 + # scope equal to "\'" means it's inside a single quote string elif scope == "\'": if debug_mode: print("\'^", end="") # for debugging @@ -404,6 +472,7 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st else: position = position + 1 + # scope equal to "\"" means it's inside a double quote string elif scope == "\"": if debug_mode: print("\"^", end="") # for debugging @@ -422,6 +491,7 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st else: position = position + 1 + # scope equal to "=" means a possible python dictionary elif scope == "=": if debug_mode: print("=", end="") # for debugging @@ -447,6 +517,7 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st print("!", end="") # for debugging return position + # scope equal to "={" means it's inside a python dictionary elif scope == "={": while position < len(code): @@ -462,6 +533,7 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st indent_if_newline(code[position], outfile, indentation, indentation_str) position = position + 1 + # if scope is invalid an exception will be thrown else: raise Exception("invalid scope was reached") @@ -497,17 +569,17 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st infile_str = remove_semicolons(infile_str) # change imported names (if necessary) + # TODO testing if change_imports is not None: for module in change_imports: infile_str = re.sub("(?<=import\\s){}".format(module), "{} as {}".format(change_imports[module], module), infile_str) infile_str = re.sub("(?<=from\\s){}(?=\\s+import)".format(module), change_imports[module], infile_str) # output filtered file (for debugging) - # TODO remove later if debug_mode: - filteredfile = open(filename + ".filtered", 'w') - filteredfile.write(infile_str) - filteredfile.close() + filtered_file = open(filename + ".filtered", 'w') + filtered_file.write(infile_str) + filtered_file.close() # start recursive function recursive_parser(infile_str, 0, "", outfile, 0, " ", debug_mode) From 65d15d2737d5a36778a96714f35124cae3141249 Mon Sep 17 00:00:00 2001 From: Filipe Jordan do Nascimento Date: Sat, 25 May 2019 22:54:58 -0300 Subject: [PATCH 29/35] put my name in the authors --- etc/bython.1 | 4 +++- scripts/bython | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/etc/bython.1 b/etc/bython.1 index 8b059e6..627ecfd 100644 --- a/etc/bython.1 +++ b/etc/bython.1 @@ -38,7 +38,9 @@ Specify the name of output file (valid only if -c is present, throws an error if .SH SEE ALSO py2by(1) .SH AUTHOR -Mathias Lohne and Tristan Pepin 2018 +Mathias Lohne 2018, +Tristan Pepin 2018, +Filipe do Nascimento 2019 .SH LICENSE This program is licensed under the MIT license. See [1] for details. TL;DR: Do whatever you want as long as you include the original copyright and license notice. .TP diff --git a/scripts/bython b/scripts/bython index c62a0b6..399e796 100755 --- a/scripts/bython +++ b/scripts/bython @@ -31,7 +31,7 @@ def main(): formatter_class=argparse.RawTextHelpFormatter) argparser.add_argument("-V", "--version", action="version", - version="Bython v%s\nMathias Lohne, Tristan Pepin and Filipe Nascimento 2019" % VERSION_NUMBER) + version="Bython v%s\nMathias Lohne, Tristan Pepin and Filipe do Nascimento 2019" % VERSION_NUMBER) argparser.add_argument("-v", "--verbose", help="print progress", action="store_true") From 5dcc5cc2687249a29349c5fc6c7ed5b994225353 Mon Sep 17 00:00:00 2001 From: Filipe Jordan do Nascimento Date: Sun, 26 May 2019 14:18:09 -0300 Subject: [PATCH 30/35] fixed infinite loop inside "/*" scope --- bython/parser.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bython/parser.py b/bython/parser.py index 68ed198..2d121df 100644 --- a/bython/parser.py +++ b/bython/parser.py @@ -448,6 +448,8 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st print("*/", end="") # for debugging outfile.write(code[position + 1]) return position + 2 + else: + position = position + 1 else: position = position + 1 From b35ca4cfa024f9429a5e6c19d3a73d8a5d24b1cc Mon Sep 17 00:00:00 2001 From: Filipe Jordan do Nascimento Date: Thu, 30 May 2019 10:42:23 -0300 Subject: [PATCH 31/35] Fixed an exception that happened when the file didn't end with a newline --- bython/parser.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bython/parser.py b/bython/parser.py index 2d121df..b85004e 100644 --- a/bython/parser.py +++ b/bython/parser.py @@ -582,6 +582,9 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st filtered_file = open(filename + ".filtered", 'w') filtered_file.write(infile_str) filtered_file.close() + + # adding a newline at EOF + infile_str += infile_str + "\n" # start recursive function recursive_parser(infile_str, 0, "", outfile, 0, " ", debug_mode) From d3da4c59157ba0de67071c7186c20ac4e75af907 Mon Sep 17 00:00:00 2001 From: Filipe Jordan do Nascimento Date: Thu, 30 May 2019 13:47:30 -0300 Subject: [PATCH 32/35] Fixed an issue where the code was being duplicated --- bython/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bython/parser.py b/bython/parser.py index b85004e..7931636 100644 --- a/bython/parser.py +++ b/bython/parser.py @@ -584,7 +584,7 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st filtered_file.close() # adding a newline at EOF - infile_str += infile_str + "\n" + infile_str += "\n" # start recursive function recursive_parser(infile_str, 0, "", outfile, 0, " ", debug_mode) From 3f38de37ef84f20220cc3e036ba1343ffbfdbf3c Mon Sep 17 00:00:00 2001 From: Filipe Jordan do Nascimento Date: Thu, 30 May 2019 13:55:37 -0300 Subject: [PATCH 33/35] Fixed an issue where spaces and tabs after closing braces were removed incorrectly --- bython/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bython/parser.py b/bython/parser.py index 7931636..31ad5a6 100644 --- a/bython/parser.py +++ b/bython/parser.py @@ -213,7 +213,7 @@ def prepare_braces(code): # TODO fix issue with brace within comments # TODO fix removing spaces and tabs from within strings code = re.sub(r"[ \t]*(\/\/.*|\#.*)?\r?\n[ \t]*\{", "{ \\1\n", code) - code = re.sub(r"(?<=\})[ \t]+", "", code) + code = re.sub(r"(?<=\})[ \t]+\n", "\n", code) code = re.sub(r"(?<=\n\})([^\n]+?)(?=\n)", "\n\\1", code) code = re.sub(r"[ \t]+(?=\{\n)", "", code) return code From 311329ceecb617723539e903e7d98e0a6dced98a Mon Sep 17 00:00:00 2001 From: Filipe Jordan do Nascimento Date: Fri, 7 Jun 2019 13:56:55 -0300 Subject: [PATCH 34/35] else and elif fix --- bython/parser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bython/parser.py b/bython/parser.py index 31ad5a6..69ed006 100644 --- a/bython/parser.py +++ b/bython/parser.py @@ -214,6 +214,7 @@ def prepare_braces(code): # TODO fix removing spaces and tabs from within strings code = re.sub(r"[ \t]*(\/\/.*|\#.*)?\r?\n[ \t]*\{", "{ \\1\n", code) code = re.sub(r"(?<=\})[ \t]+\n", "\n", code) + code = re.sub(r"(?<=\})[ \t]+((else|elif))", "\n\\1", code) code = re.sub(r"(?<=\n\})([^\n]+?)(?=\n)", "\n\\1", code) code = re.sub(r"[ \t]+(?=\{\n)", "", code) return code From 8fdadbff381f374c6aac2019da93e2b17be10640 Mon Sep 17 00:00:00 2001 From: Filipe Jordan do Nascimento Date: Fri, 7 Jun 2019 14:00:10 -0300 Subject: [PATCH 35/35] I just found out that python has a '//' operator. Disabling '//' comments. --- bython/parser.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bython/parser.py b/bython/parser.py index 69ed006..d793aa5 100644 --- a/bython/parser.py +++ b/bython/parser.py @@ -347,10 +347,10 @@ def recursive_parser(code, position, scope, outfile, indentation, indentation_st # check for c and cpp-style comment elif code[position] == "/": - if code[position + 1] == "/": - outfile.write("#") - position = recursive_parser(code, position + 2, "//", outfile, indentation, indentation_str, debug_mode) - elif code[position + 1] == "*": + #if code[position + 1] == "/": + #outfile.write("#") + #position = recursive_parser(code, position + 2, "//", outfile, indentation, indentation_str, debug_mode) + if code[position + 1] == "*": outfile.write("#") outfile.write(code[position:position+2]) position = recursive_parser(code, position + 2, "/*", outfile, indentation, indentation_str, debug_mode)