diff --git a/MANIFEST.in b/MANIFEST.in index 6177c41..d18b373 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,3 +4,4 @@ include INSTALL.md include setup.py include tests/__init__.py include pypreprocessor/__init__.py +include pypreprocessor/__main__.py diff --git a/README.md b/README.md index 281ca09..53aedbf 100644 --- a/README.md +++ b/README.md @@ -55,17 +55,16 @@ ## Syntax -The syntax for pypreprocessor uses a select subset of the stanard c-style preprocessor directives, and then some... +The syntax for pypreprocessor uses a select subset of the standard c-style preprocessor directives, and then some... **Supported directives** -* define non-value constants used by the preprocessor +* defines constants (valued or not) used by the preprocessor ```python -#define constant +#define constant [value] ``` - -* remove a non-value constant from the list of defined constants +* removes a constant from the list of defined constants ```python #undef constant ``` @@ -75,21 +74,46 @@ The syntax for pypreprocessor uses a select subset of the stanard c-style prepro #ifdef constant ``` -* makes the subsequent block of code available if all of the preceding #ifdef statements return false +* makes the subsequent block of code available if the specified constant is not set +```python +#ifndef constant +``` + +* makes the subsequent block of code available if the specified condition returns true +```python +#if boolean_condition +``` + +* makes the subsequent block of code available if all of the preceding #ifdef, #elif, #if statements returns false ```python #else ``` +* makes the subsequent block of code available if all of the preceding #ifdef, #elif, #if statements return false and the specifified condition returns true +```python +#elif boolean_condition +``` + * required to close out an #ifdef/#else block ```python #endif ``` + +* Interrupts execution and returns error when reached +```python +#error +``` + +**Unofficial supported directives** + +Unofficial directives exist to ease writing long files but should not be used in file that could be preprocessed without pypreprocessor + * possibility to close all open blocks ```python #endifall ``` -* exclude the subsequent block of code (conditionals not included). I know it doesn't fit into the standard set of c-style directives but it's too handy to exclude (no pun). +* excludes the subsequent block of code (conditionals not included). I know it doesn't fit into the standard set of c-style directives but it's too handy to exclude (no pun). ```python #exclude ``` @@ -99,6 +123,50 @@ The syntax for pypreprocessor uses a select subset of the stanard c-style prepro #endexclude ``` +* Attempts closing #ifdef/#else blocks +```python +#endif +``` + +* Similar to #ifndef +```python +#ifnotdef constant +``` + +* Similar to #ifndef +```python +#ifdefnot constant +``` + +* Similar to #elif boolean_condition +```python +#elseif boolean_condition +``` + +* Similar to #elif constant +```python +#elseifdef constant +``` + +* Similar to #endif followed by #ifdef constant +```python +#endififdef constant +``` + +**Unsupported directives** + +Unsupported directives are not handled by pypreprocessor and concidered as comment + +* Inserts a particuliar header from another file. This has no use in Python +```python +#include +``` + +* Issues special commands to the compiler, using a standardized method. This has no use in Python +```python +#pragma +``` + **Options** The following options need to be set prior to pypreprocessor.parse() @@ -113,12 +181,16 @@ add defines to the preprocessor programmatically, this allows the source file to pypreprocessor.run = True / False pypreprocessor.resume = True / False pypreprocessor.save = True / False +pypreprocessor.overload = True / False +pypreprocessor.quiet = True / False ``` set the options of the preprocessor: * run: Run the preprocessed code if true. Default is true * resume: Return after a file is preprocessed and can preprocess a next file if true. Default is false * save: Save preprocessed code if true. Default is true +* overload: Any defines added to the preprocessor will overload existing defines. Default is false +* quiet: no warning about not understood directives or missing #indef ```python pypreprocessor.input = 'inputFile.py' diff --git a/pypreprocessor/__init__.py b/pypreprocessor/__init__.py index b0b2a07..4c5371e 100644 --- a/pypreprocessor/__init__.py +++ b/pypreprocessor/__init__.py @@ -2,20 +2,38 @@ # pypreprocessor.py __author__ = 'Evan Plaice' -__coauthor__ = 'Hendi O L, Epikem' -__version__ = '0.7.7' +__coauthor__ = 'Hendi O L, Epikem, Laurent Pinson' +__version__ = '1.0' import sys import os import traceback import imp import io +import collections +import re + +#support for <=0.7.7 +class customDict(dict): + def append(self, var): + self[var]=True + + class preprocessor: - def __init__(self, inFile=sys.argv[0], outFile='', defines=[], \ - removeMeta=False, escapeChar=None, mode=None, escape='#', \ - run=True, resume=False, save=True): + __overloaded = [] + defines = customDict() + + def __init__(self, inFile=sys.argv[0], outFile='', defines={}, removeMeta=False, + escapeChar=None, mode=None, escape='#', run=True, resume=False, + save=True, overload=True, quiet=False): # public variables - self.defines = defines + # support for <=0.7.7 + if isinstance(defines, collections.Sequence): + for x in defines: + self.define(*x.split(':')) + else: + for x,y in defines.items(): + self.define(x,y) self.input = inFile self.output = outFile self.removeMeta = removeMeta @@ -25,26 +43,30 @@ def __init__(self, inFile=sys.argv[0], outFile='', defines=[], \ self.run = run self.resume = resume self.save = save - self.readEncoding = sys.stdin.encoding + self.overload = overload + self.quiet = quiet + self.readEncoding = sys.stdin.encoding self.writeEncoding = sys.stdout.encoding + # private variables - self.__linenum = 0 - self.__excludeblock = False - self.__ifblocks = [] - self.__ifconditions = [] - self.__evalsquelch = True - self.__outputBuffer = '' + self.__reset_internal() def check_deprecation(self): + """ + Deprecation checks for older implementation of this library + + """ def deprecation(message): import warnings warnings.simplefilter('always', DeprecationWarning) warnings.warn(message, DeprecationWarning) warnings.simplefilter('default', DeprecationWarning) + if self.escapeChar != None: deprecation("'pypreprocessor.escapeChar' is deprecated. Use 'escape' instead.") if self.escape == '#': self.escape = self.escapeChar + if self.mode != None: msg = "'pypreprocessor.mode' is deprecated. Use 'run/resume/save' options instead." if self.run != True or self.resume != False or self.save != True: @@ -66,220 +88,314 @@ def deprecation(message): print('Unknown mode : ' + str(self.mode)) deprecation(msg) - # reseting internal things to parse a second file def __reset_internal(self): self.__linenum = 0 self.__excludeblock = False - self.__ifblocks = [] - self.__ifconditions = [] - self.__evalsquelch = True + # contains the evaluated if conditions + # due to the introduction of #elif, elements of __ifblocks are duos of boolean + # the 1st is the evaluation of the current #if or #elif or #else + # the 2nd indicates if at least one #if or #elif was True in the whole #if/#endif block + self.__ifblocks = [] + # contains the if conditions + self.__ifconditions = [] self.__outputBuffer = '' + self.__overloaded = list(self.defines.keys()) if self.overload else [] + + def define(self, name, val=True): + """ + Adds variable definition to the store as expected from a #define directive. + The directive can contains no value as it would be tested with a #ifdef directive or + with a value for an evaluation as in an #if directive. - # the #define directive - def define(self, define): - self.defines.append(define) + Note: if the `name` was part of the initial definition and `overload` was set to + True, this new definition will be skipped + + :params + name (str): definition name + + val (str): definition value when it exists. Default is None + """ + # try conversion for number else evaluate() might fail + try: + val = int(val) + except: + # assume val is string + pass + if name not in self.__overloaded: + self.defines[name]=val - # the #undef directive def undefine(self, define): - # re-map the defines list excluding the define specified in the args - self.defines[:] = [x for x in self.defines if x != define] + """ + Removes variable definition from store as expected from an #undef directive - # search: if define is defined - def search_defines(self, define): + :params + define (str): definition name + + """ if define in self.defines: + self.defines.pop(define) + + def __is_defined(self, define): + """ + Checks variable is defined as used in #ifdef, #ifnotdef & #elseif directives + + :params + define (str): definition name + + """ + return define in self.defines + + def __evaluate_if(self, line): + """ + Evaluate the content of a #if, #elseif, #elif directive + + :params + line (str): definition name + + """ + try: + # replace C-style bool format by Python's + line = line.replace('&&', 'and').replace('||', 'or').replace('!','not ') + return eval(line, self.defines) or False + except BaseException as e: + print(str(e)) + self.exit_error(self.escape + 'if') + + def __validate_ifs(self): + """ + Evaluate if the successive #ifs block are validated for the current position + + :return + ifs (bool): True if all ifs condition are validated + + """ + # no ifs mean we pass else check all ifs are True + return not self.__ifblocks or all(x[0] for x in self.__ifblocks) + + def __is_directive(self, line, directive, *size): + """ + Checks the `line` is a `directive` and , if `size` is provided, checks its number of + elements is amongst the list of allowed `size` + + :params: + line (str): line to check + + directive (str): directive to be found in the `line` + + *size (int): list of allowed number of elements to compose the directive. Can be empty + + """ + if line.startswith(self.escape + directive): + if size and len(line.split()) not in size: + self.exit_error(self.escape + directive) return True - else: - return False + return False + + def __cleanup_line(self, line): + """ + Clean a line of anything that should not impact parsing such as C-style comment + + :params: + line (str): line to check - #returning: validness of #ifdef #else block - def __if(self): - value = bool(self.__ifblocks) - for ib in self.__ifblocks: - value *= ib #* represents and: value = value and ib - return not value #not: because True means removing + :return + line (str): cleaned line + + """ + line= re.sub('\s*/\*.*\*/\s+', '', line) #remove /* */ C-style comment + line= re.sub('\s*//.*', '', line) #remove // C-style comment + return line - # evaluate def lexer(self, line): - # return values are (squelch, metadata) + """ + Analyse the `line`. This method attempts to find a known directive and, when found, to + understand it and to perform appropriate action. + + :params + line (str): line of code to analyse + + :return + exclude (bool): should the line be excluded in the final output? + + metadata (bool): is this line a directive? + + """ + line = line.strip() if not (self.__ifblocks or self.__excludeblock): if 'pypreprocessor.parse()' in line: return True, True - #this block only for faster processing (not necessary) - elif line[:len(self.escape)] != self.escape: - return False, False - # handle #define directives - if line[:len(self.escape) + 6] == self.escape + 'define': - if len(line.split()) != 2: - self.exit_error(self.escape + 'define') - else: - self.define(line.split()[1]) - return False, True - # handle #undef directives - elif line[:len(self.escape) + 5] == self.escape + 'undef': - if len(line.split()) != 2: - self.exit_error(self.escape + 'undef') - else: - self.undefine(line.split()[1]) - return False, True - # handle #exclude directives - elif line[:len(self.escape) + 7] == self.escape + 'exclude': - if len(line.split()) != 1: - self.exit_error(self.escape + 'exclude') - else: - self.__excludeblock = True - return False, True - # handle #endexclude directives - elif line[:len(self.escape) + 10] == self.escape + 'endexclude': - if len(line.split()) != 1: - self.exit_error(self.escape + 'endexclude') - else: - self.__excludeblock = False - return False, True - # handle #ifnotdef directives (is the same as: #ifdef X #else) - elif line[:len(self.escape) + 8] == self.escape + 'ifdefnot': - if len(line.split()) != 2: - self.exit_error(self.escape + 'ifdefnot') - else: - self.__ifblocks.append(not self.search_defines(line.split()[1])) - self.__ifconditions.append(line.split()[1]) - return False, True - # handle #ifdef directives - elif line[:len(self.escape) + 5] == self.escape + 'ifdef': - if len(line.split()) != 2: - self.exit_error(self.escape + 'ifdef') - else: - self.__ifblocks.append(self.search_defines(line.split()[1])) - self.__ifconditions.append(line.split()[1]) - return False, True - # handle #else... - # handle #elseif directives - elif line[:len(self.escape) + 6] == self.escape + 'elseif': - if len(line.split()) != 2: - self.exit_error(self.escape + 'elseif') - else: - self.__ifblocks[-1] = not self.__ifblocks[-1] - #self.search_defines(self.__ifconditions[-1])) - self.__ifblocks.append(self.search_defines(line.split()[1])) - self.__ifconditions.append(line.split()[1]) - return False, True - # handle #else directives - elif line[:len(self.escape) + 4] == self.escape + 'else': - if len(line.split()) != 1: - self.exit_error(self.escape + 'else') - else: - self.__ifblocks[-1] = not self.__ifblocks[-1] - #self.search_defines(self.__ifconditions[-1])) - return False, True - # handle #endif.. - # handle #endififdef - elif line[:len(self.escape) + 10] == self.escape + 'endififdef': - if len(line.split()) != 2: - self.exit_error(self.escape + 'endififdef') + + # put that block here for faster processing + if not line.startswith(self.escape): #No directive --> execute + # exclude=True if we are in an exclude block or the ifs are not validated + return self.__excludeblock or not self.__validate_ifs(), False + + # strip line of any C-style comment + line = self.__cleanup_line(line) + + if self.__is_directive(line, 'define', 2,3): + self.define(*line.split()[1:]) + + elif self.__is_directive(line, 'undef', 2): + self.undefine(line.split()[1]) + + elif self.__is_directive(line, 'exclude', 1): + self.__excludeblock = True + + elif self.__is_directive(line, 'endexclude', 1): + self.__excludeblock = False + + # #ifnotdef sounds better than #ifdefnot.. + elif self.__is_directive(line, 'ifdefnot', 2) or \ + self.__is_directive(line, 'ifnotdef', 2) or \ + self.__is_directive(line, 'ifndef', 2): + _check = not self.__is_defined(line.split()[1]) + self.__ifblocks.append([ _check, _check]) + self.__ifconditions.append(line.split()[1]) + + elif self.__is_directive(line, 'ifdef', 2): + _check = self.__is_defined(line.split()[1]) + self.__ifblocks.append([ _check, _check]) + self.__ifconditions.append(line.split()[1]) + + elif self.__is_directive(line, 'if'): + _check = self.__evaluate_if(' '.join(line.split()[1:])) + self.__ifblocks.append([ _check, _check]) + self.__ifconditions.append(' '.join(line.split()[1:])) + + # since in version <=0.7.7, it didn't handle #if it should be #elseifdef instead. + # kept elseif with 2 elements for retro-compatibility (equivalent to #elseifdef). + elif self.__is_directive(line, 'elseif') or \ + self.__is_directive(line, 'elif'): + _cur, _whole = self.__ifblocks[-1] + if len(line.split()) == 2: + #old behaviour + _check = self.__is_defined(line.split()[1]) else: - if len(self.__ifconditions) >= 1: + #new behaviour + _check = self.__evaluate_if(' '.join(line.split()[1:])) + self.__ifblocks[-1]=[ not _whole and _check, _whole or _check ] + self.__ifconditions[-1]=' '.join(line.split()[1:]) + + elif self.__is_directive(line, 'elseifdef', 2): + _cur, _whole = self.__ifblocks[-1] + _check = self.__is_defined(line.split()[1]) + self.__ifblocks[-1]=[ not _whole and _check, _whole or _check ] + self.__ifconditions[-1]=' '.join(line.split()[1:]) + + elif self.__is_directive(line, 'else', 1): + _cur, _whole = self.__ifblocks[-1] + self.__ifblocks[-1] = [not _whole, not _whole] #opposite of the whole if/elif block + + elif self.__is_directive(line, 'endififdef', 2): + # do endif + if len(self.__ifconditions) >= 1: + self.__ifblocks.pop(-1) + self.__ifconditions.pop(-1) + # do ifdef + self.__ifblocks.append(self.__is_defined(line.split()[1])) + self.__ifconditions.append(line.split()[1]) + + elif self.__is_directive(line, 'endifall', 1): + self.__ifblocks = [] + self.__ifconditions = [] + + # handle #endif and #endif directives + elif self.__is_directive(line, 'endif', 1): + try: + number = int(line[6:]) + except ValueError as VE: + number = 1 + + try: + while number: self.__ifblocks.pop(-1) - self.__ifcondition = self.__ifconditions.pop(-1) - else: - self.__ifblocks = [] - self.__ifconditions = [] - self.__ifblocks.append(self.search_defines(line.split()[1])) - self.__ifconditions.append(line.split()[1]) - return False, True - # handle #endifall directives - elif line[:len(self.escape) + 8] == self.escape + 'endifall': - if len(line.split()) != 1: - self.exit_error(self.escape + 'endifall') - else: - self.__ifblocks = [] - self.__ifconditions = [] - return False, True - # handle #endif and #endif numb directives - elif line[:len(self.escape) + 5] == self.escape + 'endif': - if len(line.split()) != 1: - self.exit_error(self.escape + 'endif number') - else: - try: - number = int(line[6:]) - except ValueError as VE: - #print('ValueError',VE) - #self.exit_error(self.escape + 'endif number') - number = 1 - if len(self.__ifconditions) > number: - for i in range(0, number): - self.__ifblocks.pop(-1) - self.__ifcondition = self.__ifconditions.pop(-1) - elif len(self.__ifconditions) == number: - self.__ifblocks = [] - self.__ifconditions = [] - else: - print('Warning try to remove more blocks than present', \ - self.input, self.__linenum) - self.__ifblocks = [] - self.__ifconditions = [] - return False, True - else: #No directive --> execute - # process the excludeblock - if self.__excludeblock is True: - return True, False - # process the ifblock - elif self.__ifblocks: # is True: - return self.__if(), False - #here can add other stuff for deleting comnments eg - else: - return False, False + self.__ifconditions.pop(-1) + number-=1 + except: + if not self.quiet: + print('Warning trying to remove more blocks than present', + self.input, self.__linenum) + + elif self.__is_directive(line, 'error'): + if self.__validate_ifs(): + print('File: "' + self.input + '", line ' + str(self.__linenum + 1)) + print('Error directive reached') + sys.exit(1) + + else: + # escapechar + space ==> comment + # starts with #!/ ==> shebang + # else print warning + if len(line.split()[0]) > 1 and not line.startswith('#!/') and not self.quiet: + print('Warning unknown directive or comment starting with ', + line.split()[0], self.input, self.__linenum + 1) + + return False, True # error handling def exit_error(self, directive): - print('File: "' + self.input + '", line ' + str(self.__linenum)) + """ + Prints error and interrupts execution + + :params + directive (str): faulty directive + + """ + print('File: "' + self.input + '", line ' + str(self.__linenum + 1)) print('SyntaxError: Invalid ' + directive + ' directive') sys.exit(1) + def rewrite_traceback(self): + """ + Dumps traceback but with the input file name included + + """ trace = traceback.format_exc().splitlines() - index = 0 + trace[-2]=trace[-2].replace("", self.input) for line in trace: - if index == (len(trace) - 2): - print(line.replace("", self.input)) - else: - print(line) - index += 1 + print(line) - # parsing/processing def parse(self): + """ + Main method: + - reset internal counters/values + - check & warn about deprecation + - starts the parsing of the input file + - warn of unclosed #ifdef blocks if any + - trigger post-process activities + + """ self.__reset_internal() self.check_deprecation() # open the input file - input_file = io.open(os.path.join(self.input), 'r', encoding=self.readEncoding) try: - # process the input file - for line in input_file: - self.__linenum += 1 - # to squelch or not to squelch - squelch, metaData = self.lexer(line) - # process and output - if self.removeMeta is True: - if metaData is True or squelch is True: + with io.open(os.path.join(self.input), 'r', encoding=self.readEncoding) as input_file: + for self.__linenum, line in enumerate(input_file): + exclude, metaData = self.lexer(line) + # process and output + if self.removeMeta: + if metaData or exclude: + continue + if exclude: + if metaData: + self.__outputBuffer += self.escape + line + else: + self.__outputBuffer += self.escape[0] + line continue - if squelch is True: - if metaData: - self.__outputBuffer += self.escape + line else: - self.__outputBuffer += self.escape[0] + line - continue - if squelch is False: - self.__outputBuffer += line - continue + self.__outputBuffer += line + continue finally: - input_file.close() #Warnings for unclosed #ifdef blocks - if self.__ifblocks: + if self.__ifblocks and not self.quiet: print('Warning: Number of unclosed Ifdefblocks: ', len(self.__ifblocks)) print('Can cause unwished behaviour in the preprocessed code, preprocessor is safe') try: select = input('Do you want more Information? ') except SyntaxError: select = 'no' - select = select.lower() - if select in ('yes', 'true', 'y', '1'): + if select.lower() in ('yes', 'true', 'y', '1'): print('Name of input and output file: ', self.input, ' ', self.output) for i, item in enumerate(self.__ifconditions): if (item in self.defines) != self.__ifblocks[i]: @@ -289,18 +405,25 @@ def parse(self): print('Block:', item, ' is in condition: ', cond) self.post_process() - # post-processor def post_process(self): + """ + Perform post-parsing activities: + - write output file from parsing. + - override import if requested or attempt execution with its content + - remove output file if no save was requested + - force exit if resume was not requested + + """ try: # set file name - if self.output == '': + if not self.output: self.output = self.input[0:-len(self.input.split('.')[-1])-1]+'_out.'+self.input.split('.')[-1] - # open file for output - output_file = io.open(self.output, 'w', encoding=self.writeEncoding) + # write post-processed code to file - output_file.write(self.__outputBuffer) + with io.open(self.output, 'w', encoding=self.writeEncoding) as output_file: + output_file.write(self.__outputBuffer) finally: - output_file.close() + pass if self.run: # if this module is loaded as a library override the import @@ -308,10 +431,12 @@ def post_process(self): self.override_import() else: self.on_the_fly() + if not self.save: # remove tmp file if os.path.exists(self.output): os.remove(self.output) + if not self.resume: # break execution so python doesn't # run the rest of the pre-processed code @@ -319,6 +444,10 @@ def post_process(self): # postprocessor - override an import def override_import(self): + """ + Override the import of the output of the processed file + + """ try: moduleName = self.input.split('.')[0] tmpModuleName = self.output.split('.')[0] @@ -334,10 +463,13 @@ def override_import(self): # postprocessor - on-the-fly execution def on_the_fly(self): + """ + Execute output of the processed file + + """ try: - f = io.open(self.output, "r", encoding=self.readEncoding) - exec(f.read()) - f.close() + with io.open(self.output, "r", encoding=self.readEncoding) as f: + exec(f.read()) except: self.rewrite_traceback() diff --git a/pypreprocessor/__main__.py b/pypreprocessor/__main__.py new file mode 100644 index 0000000..73235b2 --- /dev/null +++ b/pypreprocessor/__main__.py @@ -0,0 +1,34 @@ +import argparse +from . import preprocessor + +parser = argparse.ArgumentParser( + description="Check https://github.com/interpreters/pypreprocessor for documentation", + epilog="""Examples: + python -m """+__package__+""" somepython.py post-proc.py -m -d DEBUG FILE:4 EXEC + python -m """+__package__+""" table.json --escape #@ -d NUM:4 ID:1 + """, + formatter_class=argparse.RawDescriptionHelpFormatter, + usage=' python -m '+__package__+' input [output] [-h] [-r] [-m] [-e ESCAPE] [-o] [-q] [-d [DEFINE ...]]', +) +parser.add_argument("-r", "--run", help="run on the fly", + action='store_true', default=False) +parser.add_argument("-m", "--removeMeta", help="remove meta lines from the output", + action='store_true', default=False) +parser.add_argument("-e", "--escape", help="define the escape sequence to use. Default is #") +parser.add_argument("-d", "--define", help="list of constants to define", nargs='*', default=[]) +parser.add_argument("-o", "--overload", help="overload variable definition in the file by those \ + provided by --define", action='store_true', default=False) +parser.add_argument("-q", "--quiet", help="No warning on not understood directives and missign ending", + action='store_true', default=False) +parser.add_argument("input", help="input file.") +parser.add_argument("output", nargs='?', help="output file. Default is _out.") +args = parser.parse_args() + +p=preprocessor(inFile=args.input, defines=args.define, mode=None, removeMeta=args.removeMeta, escapeChar=None, + run=args.run, resume=False, save=True, overload=args.overload, quiet=args.quiet) +if args.output: + p.output = args.output +if args.escape: + p.escape = args.escape + +p.parse() \ No newline at end of file