From 62597a9b65bc51759568278561e415717e1e6ed4 Mon Sep 17 00:00:00 2001 From: Victor Lopez Date: Tue, 17 Nov 2020 15:24:14 +0100 Subject: [PATCH] Run pygen script with either python2 or python3 --- Makefile.am | 7 +- configure.ac | 2 + scripts/pygen.py3 | 166 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 174 insertions(+), 1 deletion(-) create mode 100755 scripts/pygen.py3 diff --git a/Makefile.am b/Makefile.am index 7d575e7..b274011 100644 --- a/Makefile.am +++ b/Makefile.am @@ -83,7 +83,11 @@ nodist_include_HEADERS = src/LB_MPI/MPI_interface.h \ #******************************************************************************** # Built Sources #******************************************************************************** -MPI_GENERATOR = $(top_srcdir)/scripts/pygen.py +if PYTHON3 +MPI_GENERATOR = $(PYTHON) $(top_srcdir)/scripts/pygen.py3 +else +MPI_GENERATOR = $(PYTHON) $(top_srcdir)/scripts/pygen.py +endif MPICALLS_DB = $(top_srcdir)/src/LB_MPI/mpicalls.json EXTRA_DIST += src/LB_MPI/mpicalls.json @@ -739,6 +743,7 @@ EXTRA_DIST += \ scripts/run_with_dlb.sh.in \ scripts/dlb.spec \ scripts/pygen.py \ + scripts/pygen.py3 \ scripts/viewer/dlb_cpu_usage.in \ scripts/viewer/dlb_viewer.py.in \ $(DEBIAN_EXTRA) \ diff --git a/configure.ac b/configure.ac index 2200d23..8a587b9 100644 --- a/configure.ac +++ b/configure.ac @@ -235,6 +235,8 @@ AC_SUBST([INSTR_DEBUG_CPPFLAGS]) AM_PATH_PYTHON AX_COMPARE_VERSION([$PYTHON_VERSION], [ge], [2.7], [lit=yes], [lit=no]) AM_CONDITIONAL([LIT_SUPPORT], [test "x$lit" = xyes]) +AX_COMPARE_VERSION([$PYTHON_VERSION], [ge], [3.0], [python3=yes], [python3=no]) +AM_CONDITIONAL([PYTHON3], [test "x$python3" = xyes]) # check for OpenMP availability AC_OPENMP diff --git a/scripts/pygen.py3 b/scripts/pygen.py3 new file mode 100755 index 0000000..9748d96 --- /dev/null +++ b/scripts/pygen.py3 @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 + +import sys +import getopt +import json +import re + +class MPIFile: + _filename = None + _mpi_calls = None + + def __init__(self, inputfile, outputfile, calls=[]): + self._in_filename = inputfile + self._out_filename = outputfile + self._mpi_calls = calls + + def parse(self): + with open(self._in_filename, 'r') as in_f: + with open(self._out_filename, 'w+') as out_f: + pragma_scope = False + pragma_block = '' + condition = 'True' + for line in in_f: + if pragma_scope: + if '#pragma pygen end' in line: + pragma_scope = False + parsed = self._parse_block(pragma_block, condition) + out_f.write(parsed) + else: + pragma_block += line + else: + if '#pragma pygen start' in line: + pragma_scope = True + pragma_block = '' + condition = self._parse_condition(line) + elif line != '': + out_f.write(line) + + def _parse_condition(self, line): + try: + match = re.match(r"#pragma pygen start where\((?P.*)\)", line) + condition_str = match.group('condition') + + # Type: "string" in key + match = re.match(r'(?P"[A-Za-z0-9_\./\\-]+") in (?P[A-Za-z0-9_]+)', condition_str) + if match: + return '{string} in func[\'{key}\']'.format( + string = match.group('string'), + key = match.group('key')) + + # Type: "string" not in key + match = re.match(r'(?P"[A-Za-z0-9_\./\\-]+") not in (?P[A-Za-z0-9_]+)', condition_str) + if match: + return '{string} not in func[\'{key}\']'.format( + string = match.group('string'), + key = match.group('key')) + + except AttributeError: + # If some regexp fail, just ignore everything else and return + pass + + return 'True' + + + def _parse_block(self, block, condition): + if not isinstance(condition, str) or not condition: + condition = 'True' + + parsed = '' + for func in self._mpi_calls: + if func['enabled'] and eval(condition): + parsed += block.format( + MPI_NAME = func['name'], + MPI_LCASE = func['name'].lower(), + C_PARAMS = func['cpar'], + F_PARAMS = func['fpar'], + C_ARG_LIST = func['c_args'], + F_ARG_LIST = func['f_args'], + TAGS = func['tags'], + MPI_KEYNAME = func['name'][4:], + BEFORE_FUNC = func['before'], + AFTER_FUNC = func['after'] + ) + return parsed + + +def enrich(mpi_calls): + last_word = re.compile(r'(\w+)(\[\])?\Z') + for func in mpi_calls: + # C: Parse arg list: "int argc, char *argv[]" -> "argc, argv" + c_args = [] + if func['cpar'] != 'void': + for arg in func['cpar'].split(','): + try: + c_args.append(last_word.search(arg).group(1)) + except AttributeError: + print('Error parsing function ' + func['name']) + raise + func['c_args'] = ', '.join(c_args) + # Fortran: Parse arg list: "MPI_Fint *comm, MPI_Fint *ierror" -> "comm, ierror" + f_args = [] + if func['fpar'] != '': + for arg in func['fpar'].split(','): + try: + f_args.append(last_word.search(arg).group(1)) + except AttributeError: + print('Error parsing function ' + func['name']) + raise + func['f_args'] = ', '.join(f_args) + + # Set tag _Unknown if not defined + func.setdefault('tags', '_Unknown') + if func['tags'] == '': + func['tags'] = '_Unknown' + + # Set before and after funtions + if 'MPI_Init' in func['name']: + func['before'] = 'before_init()' + func['after'] = 'after_init()' + elif 'MPI_Finalize' in func['name']: + func['before'] = 'before_finalize()' + func['after'] = 'after_finalize()' + else: + func['before'] = 'before_mpi({0}, 0, 0)'.format(func['name'].replace('MPI_', '')) + func['after'] = 'after_mpi({0})'.format(func['name'].replace('MPI_', '')) + + # Set flag enabled if not defined + func.setdefault('enabled', True) + + func.setdefault('info', '') + +def main(argv): + inputfile = '' + outputfile = '' + jsonfile = '' + try: + opts, args = getopt.getopt(argv[1:],'hi:o:j:',['ifile=','ofile=','json=']) + except getopt.GetoptError: + print(argv[0], ' -i -o -j ') + sys.exit(2) + for opt, arg in opts: + if opt == '-h': + print(argv[0], ' -i -o -j ') + sys.exit() + elif opt in ('-i', '--ifile'): + inputfile = arg + elif opt in ('-o', '--ofile'): + outputfile = arg + elif opt in ('-j', '--json'): + jsonfile = arg + + if inputfile == '' or outputfile == '' or jsonfile == '': + print(argv[0], ' -i -o -j ') + sys.exit() + + # Read JSON file + with open(jsonfile, 'r') as json_data: + mpi_calls = json.load(json_data)['mpi_calls'] + enrich(mpi_calls) + + # Parse input file + mpi_intercept_c = MPIFile(inputfile, outputfile, mpi_calls) + mpi_intercept_c.parse() + +if __name__ == '__main__': + main(sys.argv)