From 05f93a7e596a521e634fafb78e43a8c32e08ee63 Mon Sep 17 00:00:00 2001 From: Randy Eckman Date: Mon, 11 Dec 2023 03:50:56 -0600 Subject: [PATCH 01/22] allow for whitespace in PP directives, better def arg regex --- fortls/regex_patterns.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/fortls/regex_patterns.py b/fortls/regex_patterns.py index 6f590402..a89c7dcd 100644 --- a/fortls/regex_patterns.py +++ b/fortls/regex_patterns.py @@ -124,11 +124,14 @@ class FortranRegularExpressions: FREE_FORMAT_TEST: Pattern = compile(r"[ ]{1,4}[a-z]", I) # Preprocessor matching rules DEFINED: Pattern = compile(r"defined[ ]*\(?[ ]*([a-z_]\w*)[ ]*\)?", I) - PP_REGEX: Pattern = compile(r"#(if |ifdef|ifndef|else|elif|endif)") - PP_DEF: Pattern = compile(r"#(define|undef)[ ]*([\w]+)(\((\w+(,[ ]*)?)+\))?", I) + PP_REGEX: Pattern = compile(r"#[ ]*(if |ifdef|ifndef|else|elif|endif)") + PP_DEF: Pattern = compile( + r"#[ ]*(define|undef)[ ]*([\w]+)[ ]*(\([ ]*(\w+[ ]*,?[ \w,]*\w+)+[ ]*\))?", + I, + ) PP_DEF_TEST: Pattern = compile(r"(![ ]*)?defined[ ]*\([ ]*(\w*)[ ]*\)$", I) - PP_INCLUDE: Pattern = compile(r"#include[ ]*([\"\w\.]*)", I) - PP_ANY: Pattern = compile(r"(^#:?\w+)") + PP_INCLUDE: Pattern = compile(r"#[ ]*include[ ]*([\"\w\.]*)", I) + PP_ANY: Pattern = compile(r"(^#[\w]*:?\w+)") # Context matching rules CALL: Pattern = compile(r"[ ]*CALL[ ]+[\w%]*$", I) INT_STMNT: Pattern = compile(r"^[ ]*[a-z]*$", I) From f297da5864d0700600aa19b4f2a7cb0aa35ac19e Mon Sep 17 00:00:00 2001 From: Randy Eckman Date: Tue, 2 Jan 2024 18:34:26 -0600 Subject: [PATCH 02/22] add draft of unit test --- test/test_preproc.py | 6 +++ test/test_source/pp/include/indent.h | 22 +++++++++++ .../pp/preproc_spacing_arg_defs.F90 | 39 +++++++++++++++++++ 3 files changed, 67 insertions(+) create mode 100644 test/test_source/pp/include/indent.h create mode 100644 test/test_source/pp/preproc_spacing_arg_defs.F90 diff --git a/test/test_preproc.py b/test/test_preproc.py index 50f50607..2cdc39f8 100644 --- a/test/test_preproc.py +++ b/test/test_preproc.py @@ -42,6 +42,9 @@ def check_return(result_array, checks): string += hover_req(file_path, 30, 23) file_path = root_dir / "preproc_if_elif_skip.F90" string += hover_req(file_path, 30, 23) + file_path = root_dir / "preproc_spacing_arg_defs.F90" + string += hover_req(file_path, 11, 20) + string += hover_req(file_path, 19, 25) config = str(root_dir / ".pp_conf.json") errcode, results = run_request(string, ["--config", config]) assert errcode == 0 @@ -68,6 +71,9 @@ def check_return(result_array, checks): "```fortran90\nINTEGER, PARAMETER :: res = 0+1+0+0\n```", "```fortran90\nINTEGER, PARAMETER :: res = 0+0+0+1\n```", "```fortran90\nINTEGER, PARAMETER :: res = 1+0+0+0\n```", + "```fortran90\nINTEGER, PARAMETER :: res = 0+0+0+1\n```", + "```fortran90\nPROCEDURE :: test_type_print_test\n```", + "```fortran90\nINTEGER, PARAMETER :: argtest = me%test_int + 4\n```", ) assert len(ref_results) == len(results) - 1 check_return(results[1:], ref_results) diff --git a/test/test_source/pp/include/indent.h b/test/test_source/pp/include/indent.h new file mode 100644 index 00000000..f477f249 --- /dev/null +++ b/test/test_source/pp/include/indent.h @@ -0,0 +1,22 @@ +!! sample code adapted from json-fortran/json_macros.inc + + # define SPACING_TEST + # define FILE_ENCODING ,encoding='UTF-8' + +# ifdef __GFORTRAN__ +! gfortran uses cpp in old-school compatibility mode so +! the # stringify and ## concatenate operators don't work +! but we can use C/C++ style comment to ensure PROCEDURE is +! correctly tokenized and prepended with 'wrap_' when the +! macro is expanded +# define MAYBEWRAP(PROCEDURE) PROCEDURE , wrap_/**/PROCEDURE +# else +! Intel's fpp does support the more contemporary ## concatenation +! operator, but doesn't treat the C/C++ comments the same way. +! If you use the gfortran approach and pass the -noB switch to +! fpp, the macro will expand, but with a space between wrap_ and +! whatever PROCEDURE expands to +# define MAYBEWRAP(PROCEDURE) PROCEDURE +# endif + +# define MACROARGS( x , y ) x + y diff --git a/test/test_source/pp/preproc_spacing_arg_defs.F90 b/test/test_source/pp/preproc_spacing_arg_defs.F90 new file mode 100644 index 00000000..ce7fd4e9 --- /dev/null +++ b/test/test_source/pp/preproc_spacing_arg_defs.F90 @@ -0,0 +1,39 @@ +program preprocessor_spacing_arg_defs + implicit none + + #include "indent.h" + + type :: test_type + private + integer, public :: test_int + + contains + generic, public :: print_test => MAYBEWRAP(test_type_print_test) + procedure :: MAYBEWRAP(test_type_print_test) + + end type test_type + + type(test_type) :: the_test + + call the_test%print_test() + + integer, parameter :: argtest = MACROARGS(me%test_int, 4) + +contains + subroutine test_type_print_test(me) + implicit none + + class(test_type), intent(inout) :: me + + me%test_int = 3 + end subroutine json_file_load_from_string + + subroutine wrap_test_type_print_test(me) + implicit none + + class(test_type), intent(inout) :: me + + me%test_int = 5 + end subroutine wrap_test_type_print_test + +end program preprocessor_spacing_arg_defs From 37752c8bfd60cc764643d78b2aa0160fbcfb5a2f Mon Sep 17 00:00:00 2001 From: Randy Eckman Date: Tue, 2 Jan 2024 19:47:29 -0600 Subject: [PATCH 03/22] revert changes to try to fix unit tests --- fortls/regex_patterns.py | 18 +++++++++++------- test/test_interface.py | 3 ++- test/test_preproc.py | 11 +++++------ 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/fortls/regex_patterns.py b/fortls/regex_patterns.py index a89c7dcd..244050f0 100644 --- a/fortls/regex_patterns.py +++ b/fortls/regex_patterns.py @@ -124,14 +124,18 @@ class FortranRegularExpressions: FREE_FORMAT_TEST: Pattern = compile(r"[ ]{1,4}[a-z]", I) # Preprocessor matching rules DEFINED: Pattern = compile(r"defined[ ]*\(?[ ]*([a-z_]\w*)[ ]*\)?", I) - PP_REGEX: Pattern = compile(r"#[ ]*(if |ifdef|ifndef|else|elif|endif)") - PP_DEF: Pattern = compile( - r"#[ ]*(define|undef)[ ]*([\w]+)[ ]*(\([ ]*(\w+[ ]*,?[ \w,]*\w+)+[ ]*\))?", - I, - ) + PP_REGEX: Pattern = compile(r"#(if |ifdef|ifndef|else|elif|endif)") + PP_DEF: Pattern = compile(r"#(define|undef)[ ]*([\w]+)(\((\w+(,[ ]*)?)+\))?", I) + # PP_REGEX: Pattern = compile(r"#[ ]*(if |ifdef|ifndef|else|elif|endif)") + # PP_DEF: Pattern = compile( + # r"#[ ]*(define|undef)[ ]*([\w]+)[ ]*(\([ ]*(\w+[ ]*,?[ \w,]*\w+)+[ ]*\))?", + # I, + # ) PP_DEF_TEST: Pattern = compile(r"(![ ]*)?defined[ ]*\([ ]*(\w*)[ ]*\)$", I) - PP_INCLUDE: Pattern = compile(r"#[ ]*include[ ]*([\"\w\.]*)", I) - PP_ANY: Pattern = compile(r"(^#[\w]*:?\w+)") + PP_INCLUDE: Pattern = compile(r"#include[ ]*([\"\w\.]*)", I) + PP_ANY: Pattern = compile(r"(^#:?\w+)") + # PP_INCLUDE: Pattern = compile(r"#[ ]*include[ ]*([\"\w\.]*)", I) + # PP_ANY: Pattern = compile(r"(^#[\w]*:?\w+)") # Context matching rules CALL: Pattern = compile(r"[ ]*CALL[ ]+[\w%]*$", I) INT_STMNT: Pattern = compile(r"^[ ]*[a-z]*$", I) diff --git a/test/test_interface.py b/test/test_interface.py index 0e550e2d..8832b40f 100644 --- a/test/test_interface.py +++ b/test/test_interface.py @@ -179,7 +179,8 @@ def test_version_update_pypi(): s = LangServer(conn=JSONRPC2Connection(ReadWriter(stdin, stdout)), settings=args) s.root_path = (Path(__file__).parent / "test_source").resolve() did_update = s._update_version_pypi(test=True) - assert did_update + isconda = os.path.exists(os.path.join(sys.prefix, "conda-meta")) + assert not did_update if isconda else did_update s.disable_autoupdate = True did_update = s._update_version_pypi() diff --git a/test/test_preproc.py b/test/test_preproc.py index 2cdc39f8..bd378faa 100644 --- a/test/test_preproc.py +++ b/test/test_preproc.py @@ -42,9 +42,9 @@ def check_return(result_array, checks): string += hover_req(file_path, 30, 23) file_path = root_dir / "preproc_if_elif_skip.F90" string += hover_req(file_path, 30, 23) - file_path = root_dir / "preproc_spacing_arg_defs.F90" - string += hover_req(file_path, 11, 20) - string += hover_req(file_path, 19, 25) + # file_path = root_dir / "preproc_spacing_arg_defs.F90" + # string += hover_req(file_path, 11, 20) + # string += hover_req(file_path, 19, 25) config = str(root_dir / ".pp_conf.json") errcode, results = run_request(string, ["--config", config]) assert errcode == 0 @@ -71,9 +71,8 @@ def check_return(result_array, checks): "```fortran90\nINTEGER, PARAMETER :: res = 0+1+0+0\n```", "```fortran90\nINTEGER, PARAMETER :: res = 0+0+0+1\n```", "```fortran90\nINTEGER, PARAMETER :: res = 1+0+0+0\n```", - "```fortran90\nINTEGER, PARAMETER :: res = 0+0+0+1\n```", - "```fortran90\nPROCEDURE :: test_type_print_test\n```", - "```fortran90\nINTEGER, PARAMETER :: argtest = me%test_int + 4\n```", + # "```fortran90\nPROCEDURE :: test_type_print_test\n```", + # "```fortran90\nINTEGER, PARAMETER :: argtest = me%test_int + 4\n```", ) assert len(ref_results) == len(results) - 1 check_return(results[1:], ref_results) From 4367f3200d026220a1820ecfc59d76fa79dcfd1d Mon Sep 17 00:00:00 2001 From: Randy Eckman Date: Tue, 2 Jan 2024 21:29:57 -0600 Subject: [PATCH 04/22] everything working but the def regex --- fortls/regex_patterns.py | 9 +++------ test/test_server.py | 5 +++++ test/test_source/pp/preproc_spacing_arg_defs.F90 | 14 +++++++------- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/fortls/regex_patterns.py b/fortls/regex_patterns.py index 244050f0..f6a9c6ec 100644 --- a/fortls/regex_patterns.py +++ b/fortls/regex_patterns.py @@ -124,18 +124,15 @@ class FortranRegularExpressions: FREE_FORMAT_TEST: Pattern = compile(r"[ ]{1,4}[a-z]", I) # Preprocessor matching rules DEFINED: Pattern = compile(r"defined[ ]*\(?[ ]*([a-z_]\w*)[ ]*\)?", I) - PP_REGEX: Pattern = compile(r"#(if |ifdef|ifndef|else|elif|endif)") + PP_REGEX: Pattern = compile(r"#[ ]*(if |ifdef|ifndef|else|elif|endif)") PP_DEF: Pattern = compile(r"#(define|undef)[ ]*([\w]+)(\((\w+(,[ ]*)?)+\))?", I) - # PP_REGEX: Pattern = compile(r"#[ ]*(if |ifdef|ifndef|else|elif|endif)") # PP_DEF: Pattern = compile( # r"#[ ]*(define|undef)[ ]*([\w]+)[ ]*(\([ ]*(\w+[ ]*,?[ \w,]*\w+)+[ ]*\))?", # I, # ) PP_DEF_TEST: Pattern = compile(r"(![ ]*)?defined[ ]*\([ ]*(\w*)[ ]*\)$", I) - PP_INCLUDE: Pattern = compile(r"#include[ ]*([\"\w\.]*)", I) - PP_ANY: Pattern = compile(r"(^#:?\w+)") - # PP_INCLUDE: Pattern = compile(r"#[ ]*include[ ]*([\"\w\.]*)", I) - # PP_ANY: Pattern = compile(r"(^#[\w]*:?\w+)") + PP_INCLUDE: Pattern = compile(r"#[ ]*include[ ]*([\"\w\.]*)", I) + PP_ANY: Pattern = compile(r"(^[ ]*#[ ]*[\w]*:?\w+)") # Context matching rules CALL: Pattern = compile(r"[ ]*CALL[ ]+[\w%]*$", I) INT_STMNT: Pattern = compile(r"^[ ]*[a-z]*$", I) diff --git a/test/test_server.py b/test/test_server.py index 639ef427..ba812120 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -175,6 +175,7 @@ def test_workspace_symbols(): def check_return(result_array): # Expected objects objs = ( + ["argtest", 13, 19], ["test", 6, 7], ["test_abstract", 2, 0], ["test_associate_block", 2, 0], @@ -196,7 +197,11 @@ def check_return(result_array): ["test_str1", 13, 5], ["test_str2", 13, 5], ["test_sub", 6, 8], + ["test_type", 5, 5], + ["test_type_set_test", 6, 22], ["test_vis_mod", 2, 0], + ["the_test", 13, 15], + ["wrap_test_type_set_test", 6, 30], ) assert len(result_array) == len(objs) for i, obj in enumerate(objs): diff --git a/test/test_source/pp/preproc_spacing_arg_defs.F90 b/test/test_source/pp/preproc_spacing_arg_defs.F90 index ce7fd4e9..f3191063 100644 --- a/test/test_source/pp/preproc_spacing_arg_defs.F90 +++ b/test/test_source/pp/preproc_spacing_arg_defs.F90 @@ -8,32 +8,32 @@ program preprocessor_spacing_arg_defs integer, public :: test_int contains - generic, public :: print_test => MAYBEWRAP(test_type_print_test) - procedure :: MAYBEWRAP(test_type_print_test) + generic, public :: set_test => MAYBEWRAP(test_type_set_test) + procedure :: MAYBEWRAP(test_type_set_test) end type test_type type(test_type) :: the_test - call the_test%print_test() + call the_test%set_test() integer, parameter :: argtest = MACROARGS(me%test_int, 4) contains - subroutine test_type_print_test(me) + subroutine test_type_set_test(me) implicit none class(test_type), intent(inout) :: me me%test_int = 3 - end subroutine json_file_load_from_string + end subroutine test_type_set_test - subroutine wrap_test_type_print_test(me) + subroutine wrap_test_type_set_test(me) implicit none class(test_type), intent(inout) :: me me%test_int = 5 - end subroutine wrap_test_type_print_test + end subroutine wrap_test_type_set_test end program preprocessor_spacing_arg_defs From 1b3735e5c515f1e597f7dc49aef8e747c816a7ff Mon Sep 17 00:00:00 2001 From: Randy Eckman Date: Tue, 2 Jan 2024 22:33:04 -0600 Subject: [PATCH 05/22] working regex that is at least backwards compatible --- fortls/regex_patterns.py | 15 +++++++-------- test/test_preproc.py | 12 ++++++++++++ 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/fortls/regex_patterns.py b/fortls/regex_patterns.py index f6a9c6ec..e64cfab7 100644 --- a/fortls/regex_patterns.py +++ b/fortls/regex_patterns.py @@ -124,15 +124,14 @@ class FortranRegularExpressions: FREE_FORMAT_TEST: Pattern = compile(r"[ ]{1,4}[a-z]", I) # Preprocessor matching rules DEFINED: Pattern = compile(r"defined[ ]*\(?[ ]*([a-z_]\w*)[ ]*\)?", I) - PP_REGEX: Pattern = compile(r"#[ ]*(if |ifdef|ifndef|else|elif|endif)") - PP_DEF: Pattern = compile(r"#(define|undef)[ ]*([\w]+)(\((\w+(,[ ]*)?)+\))?", I) - # PP_DEF: Pattern = compile( - # r"#[ ]*(define|undef)[ ]*([\w]+)[ ]*(\([ ]*(\w+[ ]*,?[ \w,]*\w+)+[ ]*\))?", - # I, - # ) + PP_REGEX: Pattern = compile(r"#\s*(if |ifdef|ifndef|else|elif|endif)") + PP_DEF: Pattern = compile( + r"#\s*(define|undef)\s*(\w+)(\(\s*(\w+\s*,?[\s\w,]*)*\s*\))?", + I, + ) PP_DEF_TEST: Pattern = compile(r"(![ ]*)?defined[ ]*\([ ]*(\w*)[ ]*\)$", I) - PP_INCLUDE: Pattern = compile(r"#[ ]*include[ ]*([\"\w\.]*)", I) - PP_ANY: Pattern = compile(r"(^[ ]*#[ ]*[\w]*:?\w+)") + PP_INCLUDE: Pattern = compile(r"#\s*include\s*([\"\w\.]*)", I) + PP_ANY: Pattern = compile(r"(^\s*#\s*\w*:?\w+)") # Context matching rules CALL: Pattern = compile(r"[ ]*CALL[ ]+[\w%]*$", I) INT_STMNT: Pattern = compile(r"^[ ]*[a-z]*$", I) diff --git a/test/test_preproc.py b/test/test_preproc.py index bd378faa..4148c211 100644 --- a/test/test_preproc.py +++ b/test/test_preproc.py @@ -76,3 +76,15 @@ def check_return(result_array, checks): ) assert len(ref_results) == len(results) - 1 check_return(results[1:], ref_results) + + +def test_FortranFile_pp(): + from fortls import FortranFile + + root_dir = test_dir / "pp" + file_path = root_dir / "preproc.F90" + ff = FortranFile(file_path) + ff.load_from_disk() + include_dirs = set([root_dir / 'include']) + file_ast = ff.parse(include_dirs=include_dirs, debug=True) + assert file_ast From 3ddeb00c203b37f8c053b4d4a6c2ccbd806cd70b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 3 Jan 2024 04:33:20 +0000 Subject: [PATCH 06/22] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- test/test_preproc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_preproc.py b/test/test_preproc.py index 4148c211..9a797dfe 100644 --- a/test/test_preproc.py +++ b/test/test_preproc.py @@ -85,6 +85,6 @@ def test_FortranFile_pp(): file_path = root_dir / "preproc.F90" ff = FortranFile(file_path) ff.load_from_disk() - include_dirs = set([root_dir / 'include']) + include_dirs = {root_dir / "include"} file_ast = ff.parse(include_dirs=include_dirs, debug=True) assert file_ast From 6e63f04fde282ef9303f526cfc006e46b645f7e5 Mon Sep 17 00:00:00 2001 From: Randy Eckman Date: Wed, 3 Jan 2024 01:49:55 -0600 Subject: [PATCH 07/22] working version of macro expansion w/ tests --- fortls/langserver.py | 8 ++- fortls/parsers/internal/parser.py | 60 ++++++++++++++++--- fortls/regex_patterns.py | 6 +- test/test_preproc.py | 23 +++---- test/test_server.py | 6 +- .../pp/preproc_spacing_arg_defs.F90 | 3 +- 6 files changed, 80 insertions(+), 26 deletions(-) diff --git a/fortls/langserver.py b/fortls/langserver.py index 422061d9..2a3b4f6f 100644 --- a/fortls/langserver.py +++ b/fortls/langserver.py @@ -751,11 +751,17 @@ def get_definition( return None # Search in Preprocessor defined variables if def_name in def_file.pp_defs: + def_value = def_file.pp_defs.get(def_name) + def_arg_str = '' + if isinstance(def_value, tuple): + def_arg_str, def_value = def_value + def_arg_str = f'({def_arg_str})' + var = Variable( def_file.ast, def_line + 1, def_name, - f"#define {def_name} {def_file.pp_defs.get(def_name)}", + f"#define {def_name}{def_arg_str} {def_value}", [], ) return var diff --git a/fortls/parsers/internal/parser.py b/fortls/parsers/internal/parser.py index ae5cfa1e..929d2680 100644 --- a/fortls/parsers/internal/parser.py +++ b/fortls/parsers/internal/parser.py @@ -9,9 +9,9 @@ # Python < 3.8 does not have typing.Literals try: - from typing import Literal + from typing import Literal, Tuple except ImportError: - from typing_extensions import Literal + from typing_extensions import Literal, Tuple from re import Match, Pattern @@ -2098,10 +2098,11 @@ def replace_vars(line: str): if def_cont_name is not None: output_file.append("") if line.rstrip()[-1] != "\\": - defs_tmp[def_cont_name] += line.strip() + append_multiline_macro(defs_tmp, def_cont_name, line.strip()) def_cont_name = None else: - defs_tmp[def_cont_name] += line[0:-1].strip() + append_multiline_macro(defs_tmp, def_cont_name, line[0:-1].strip()) + continue # Handle conditional statements match = FRegex.PP_REGEX.match(line) @@ -2206,15 +2207,24 @@ def replace_vars(line: str): # def_name += match.group(3) if (match.group(1) == "define") and (def_name not in defs_tmp): eq_ind = line[match.end(0) :].find(" ") + if eq_ind < 0: + eq_ind = line[match.end(0) :].find("\t") + if eq_ind >= 0: # Handle multiline macros if line.rstrip()[-1] == "\\": - defs_tmp[def_name] = line[match.end(0) + eq_ind : -1].strip() + def_value = line[match.end(0) + eq_ind : -1].strip() def_cont_name = def_name else: - defs_tmp[def_name] = line[match.end(0) + eq_ind :].strip() + def_value = line[match.end(0) + eq_ind :].strip() else: - defs_tmp[def_name] = "True" + def_value = "True" + + # are there arguments to parse? + if match.group(3): + def_value = (match.group(4), def_value) + + defs_tmp[def_name] = def_value elif (match.group(1) == "undef") and (def_name in defs_tmp): defs_tmp.pop(def_name, None) log.debug(f"{line.strip()} !!! Define statement({i + 1})") @@ -2265,8 +2275,16 @@ def replace_vars(line: str): continue def_regex = def_regexes.get(def_tmp) if def_regex is None: - def_regex = re.compile(rf"\b{def_tmp}\b") + if isinstance(value, tuple): + def_regex = expand_def_func_macro(def_tmp, value) + else: + def_regex = re.compile(rf"\b{def_tmp}\b") + def_regexes[def_tmp] = def_regex + + if isinstance(def_regex, tuple): + def_regex, value = def_regex + line_new, nsubs = def_regex.subn(value, line) if nsubs > 0: log.debug( @@ -2275,3 +2293,29 @@ def replace_vars(line: str): line = line_new output_file.append(line) return output_file, pp_skips, pp_defines, defs_tmp + + +def expand_def_func_macro(def_name: str, def_value: Tuple[str, str]): + def_args, sub = def_value + def_args = def_args.split(',') + regex = re.compile(rf"\b{def_name}\s*\({','.join(['(.*)']*len(def_args))}\)") + + for i, arg in enumerate(def_args): + arg = arg.strip() + sub = re.sub(rf"\b({arg})\b", rf"\\{i + 1}", sub) + + return regex, sub + + +def append_multiline_macro(pp_defs: dict, def_name: str, line: str): + def_value = pp_defs[def_name] + def_args = None + if isinstance(def_value, tuple): + def_args, def_value = def_value + + def_value += line + + if def_args is not None: + def_value = (def_args, def_value) + + pp_defs[def_name] = def_value diff --git a/fortls/regex_patterns.py b/fortls/regex_patterns.py index e64cfab7..8dcfbcd8 100644 --- a/fortls/regex_patterns.py +++ b/fortls/regex_patterns.py @@ -124,13 +124,13 @@ class FortranRegularExpressions: FREE_FORMAT_TEST: Pattern = compile(r"[ ]{1,4}[a-z]", I) # Preprocessor matching rules DEFINED: Pattern = compile(r"defined[ ]*\(?[ ]*([a-z_]\w*)[ ]*\)?", I) - PP_REGEX: Pattern = compile(r"#\s*(if |ifdef|ifndef|else|elif|endif)") + PP_REGEX: Pattern = compile(r"\s*#\s*(if |ifdef|ifndef|else|elif|endif)") PP_DEF: Pattern = compile( - r"#\s*(define|undef)\s*(\w+)(\(\s*(\w+\s*,?[\s\w,]*)*\s*\))?", + r"\s*#\s*(define|undef)\s*(\w+)(\(\s*(\w+\s*,?[\s\w,]*)*\s*\))?", I, ) PP_DEF_TEST: Pattern = compile(r"(![ ]*)?defined[ ]*\([ ]*(\w*)[ ]*\)$", I) - PP_INCLUDE: Pattern = compile(r"#\s*include\s*([\"\w\.]*)", I) + PP_INCLUDE: Pattern = compile(r"\s*#\s*include\s*([\"\w\.]*)", I) PP_ANY: Pattern = compile(r"(^\s*#\s*\w*:?\w+)") # Context matching rules CALL: Pattern = compile(r"[ ]*CALL[ ]+[\w%]*$", I) diff --git a/test/test_preproc.py b/test/test_preproc.py index 9a797dfe..cf8e9199 100644 --- a/test/test_preproc.py +++ b/test/test_preproc.py @@ -42,9 +42,10 @@ def check_return(result_array, checks): string += hover_req(file_path, 30, 23) file_path = root_dir / "preproc_if_elif_skip.F90" string += hover_req(file_path, 30, 23) - # file_path = root_dir / "preproc_spacing_arg_defs.F90" - # string += hover_req(file_path, 11, 20) - # string += hover_req(file_path, 19, 25) + file_path = root_dir / "preproc_spacing_arg_defs.F90" + string += hover_req(file_path, 11, 20) + string += hover_req(file_path, 18, 17) + string += hover_req(file_path, 20, 13) config = str(root_dir / ".pp_conf.json") errcode, results = run_request(string, ["--config", config]) assert errcode == 0 @@ -55,12 +56,12 @@ def check_return(result_array, checks): "```fortran90\n#define PETSC_ERR_INT_OVERFLOW 84\n```", "```fortran90\n#define varVar 55\n```", ( - "```fortran90\n#define ewrite if (priority <= 3) write((priority)," - " format)\n```" + "```fortran90\n#define ewrite(priority, format)" + " if (priority <= 3) write((priority), format)\n```" ), ( - "```fortran90\n#define ewrite2 if (priority <= 3) write((priority)," - " format)\n```" + "```fortran90\n#define ewrite2(priority, format)" + " if (priority <= 3) write((priority), format)\n```" ), "```fortran90\n#define SUCCESS .true.\n```", "```fortran90\nREAL, CONTIGUOUS, POINTER, DIMENSION(:) :: var1\n```", @@ -71,8 +72,9 @@ def check_return(result_array, checks): "```fortran90\nINTEGER, PARAMETER :: res = 0+1+0+0\n```", "```fortran90\nINTEGER, PARAMETER :: res = 0+0+0+1\n```", "```fortran90\nINTEGER, PARAMETER :: res = 1+0+0+0\n```", - # "```fortran90\nPROCEDURE :: test_type_print_test\n```", - # "```fortran90\nINTEGER, PARAMETER :: argtest = me%test_int + 4\n```", + "```fortran90\n#define MAYBEWRAP(PROCEDURE) PROCEDURE\n```", + "```fortran90\nSUBROUTINE test_type_set_test()\n```", + "```fortran90\n#define MACROARGS(x , y ) x + y\n```", ) assert len(ref_results) == len(results) - 1 check_return(results[1:], ref_results) @@ -82,7 +84,8 @@ def test_FortranFile_pp(): from fortls import FortranFile root_dir = test_dir / "pp" - file_path = root_dir / "preproc.F90" + # file_path = root_dir / "preproc.F90" + file_path = root_dir / "preproc_spacing_arg_defs.F90" ff = FortranFile(file_path) ff.load_from_disk() include_dirs = {root_dir / "include"} diff --git a/test/test_server.py b/test/test_server.py index ba812120..875d3035 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -175,7 +175,7 @@ def test_workspace_symbols(): def check_return(result_array): # Expected objects objs = ( - ["argtest", 13, 19], + ["argtest", 13, 16], ["test", 6, 7], ["test_abstract", 2, 0], ["test_associate_block", 2, 0], @@ -198,10 +198,10 @@ def check_return(result_array): ["test_str2", 13, 5], ["test_sub", 6, 8], ["test_type", 5, 5], - ["test_type_set_test", 6, 22], + ["test_type_set_test", 6, 23], ["test_vis_mod", 2, 0], ["the_test", 13, 15], - ["wrap_test_type_set_test", 6, 30], + ["wrap_test_type_set_test", 6, 31], ) assert len(result_array) == len(objs) for i, obj in enumerate(objs): diff --git a/test/test_source/pp/preproc_spacing_arg_defs.F90 b/test/test_source/pp/preproc_spacing_arg_defs.F90 index f3191063..72e11917 100644 --- a/test/test_source/pp/preproc_spacing_arg_defs.F90 +++ b/test/test_source/pp/preproc_spacing_arg_defs.F90 @@ -14,10 +14,11 @@ program preprocessor_spacing_arg_defs end type test_type type(test_type) :: the_test + integer :: argtest call the_test%set_test() - integer, parameter :: argtest = MACROARGS(me%test_int, 4) + argtest = MACROARGS(the_test%test_int, 4) contains subroutine test_type_set_test(me) From e1ce5eb96cdc6090f61a514c68aa492f0f60d40c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 3 Jan 2024 07:50:32 +0000 Subject: [PATCH 08/22] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- fortls/langserver.py | 4 ++-- fortls/parsers/internal/parser.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/fortls/langserver.py b/fortls/langserver.py index 2a3b4f6f..444eb2c7 100644 --- a/fortls/langserver.py +++ b/fortls/langserver.py @@ -752,10 +752,10 @@ def get_definition( # Search in Preprocessor defined variables if def_name in def_file.pp_defs: def_value = def_file.pp_defs.get(def_name) - def_arg_str = '' + def_arg_str = "" if isinstance(def_value, tuple): def_arg_str, def_value = def_value - def_arg_str = f'({def_arg_str})' + def_arg_str = f"({def_arg_str})" var = Variable( def_file.ast, diff --git a/fortls/parsers/internal/parser.py b/fortls/parsers/internal/parser.py index 929d2680..2b75010f 100644 --- a/fortls/parsers/internal/parser.py +++ b/fortls/parsers/internal/parser.py @@ -2295,9 +2295,9 @@ def replace_vars(line: str): return output_file, pp_skips, pp_defines, defs_tmp -def expand_def_func_macro(def_name: str, def_value: Tuple[str, str]): +def expand_def_func_macro(def_name: str, def_value: tuple[str, str]): def_args, sub = def_value - def_args = def_args.split(',') + def_args = def_args.split(",") regex = re.compile(rf"\b{def_name}\s*\({','.join(['(.*)']*len(def_args))}\)") for i, arg in enumerate(def_args): From 0b6fbaa9ef8a24e642621bc03564e5bf93eec6fc Mon Sep 17 00:00:00 2001 From: Randy Eckman Date: Wed, 3 Jan 2024 01:56:27 -0600 Subject: [PATCH 09/22] housekeeping --- fortls/parsers/internal/parser.py | 4 ++-- test/test_preproc.py | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/fortls/parsers/internal/parser.py b/fortls/parsers/internal/parser.py index 2b75010f..481ae433 100644 --- a/fortls/parsers/internal/parser.py +++ b/fortls/parsers/internal/parser.py @@ -9,9 +9,9 @@ # Python < 3.8 does not have typing.Literals try: - from typing import Literal, Tuple + from typing import Literal except ImportError: - from typing_extensions import Literal, Tuple + from typing_extensions import Literal from re import Match, Pattern diff --git a/test/test_preproc.py b/test/test_preproc.py index cf8e9199..27a83cf6 100644 --- a/test/test_preproc.py +++ b/test/test_preproc.py @@ -80,14 +80,14 @@ def check_return(result_array, checks): check_return(results[1:], ref_results) -def test_FortranFile_pp(): - from fortls import FortranFile +# def test_FortranFile_pp(): +# from fortls import FortranFile - root_dir = test_dir / "pp" - # file_path = root_dir / "preproc.F90" - file_path = root_dir / "preproc_spacing_arg_defs.F90" - ff = FortranFile(file_path) - ff.load_from_disk() - include_dirs = {root_dir / "include"} - file_ast = ff.parse(include_dirs=include_dirs, debug=True) - assert file_ast +# root_dir = test_dir / "pp" +# # file_path = root_dir / "preproc.F90" +# file_path = root_dir / "preproc_spacing_arg_defs.F90" +# ff = FortranFile(file_path) +# ff.load_from_disk() +# include_dirs = {root_dir / "include"} +# file_ast = ff.parse(include_dirs=include_dirs, debug=True) +# assert file_ast From ccb4d3734a3985438431cbd4d6a4786e4f923623 Mon Sep 17 00:00:00 2001 From: Randy Eckman Date: Wed, 3 Jan 2024 12:59:12 -0600 Subject: [PATCH 10/22] replace \s* with [ ]* --- fortls/regex_patterns.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/fortls/regex_patterns.py b/fortls/regex_patterns.py index 8dcfbcd8..37e10c3e 100644 --- a/fortls/regex_patterns.py +++ b/fortls/regex_patterns.py @@ -124,14 +124,14 @@ class FortranRegularExpressions: FREE_FORMAT_TEST: Pattern = compile(r"[ ]{1,4}[a-z]", I) # Preprocessor matching rules DEFINED: Pattern = compile(r"defined[ ]*\(?[ ]*([a-z_]\w*)[ ]*\)?", I) - PP_REGEX: Pattern = compile(r"\s*#\s*(if |ifdef|ifndef|else|elif|endif)") + PP_REGEX: Pattern = compile(r"[ ]*#[ ]*(if |ifdef|ifndef|else|elif|endif)") PP_DEF: Pattern = compile( - r"\s*#\s*(define|undef)\s*(\w+)(\(\s*(\w+\s*,?[\s\w,]*)*\s*\))?", + r"[ ]*#[ ]*(define|undef)[ ]*(\w+)(\([ ]*(\w+[ ]*,?[ \w,]*)*[ ]*\))?", I, ) PP_DEF_TEST: Pattern = compile(r"(![ ]*)?defined[ ]*\([ ]*(\w*)[ ]*\)$", I) - PP_INCLUDE: Pattern = compile(r"\s*#\s*include\s*([\"\w\.]*)", I) - PP_ANY: Pattern = compile(r"(^\s*#\s*\w*:?\w+)") + PP_INCLUDE: Pattern = compile(r"[ ]*#[ ]*include[ ]*([\"\w\.]*)", I) + PP_ANY: Pattern = compile(r"(^[ ]*#[ ]*\w*:?\w+)") # Context matching rules CALL: Pattern = compile(r"[ ]*CALL[ ]+[\w%]*$", I) INT_STMNT: Pattern = compile(r"^[ ]*[a-z]*$", I) From b7c35725f8c0fc21acce99ff9c10c72fdcd280e4 Mon Sep 17 00:00:00 2001 From: Randy Eckman Date: Thu, 4 Jan 2024 11:48:48 -0600 Subject: [PATCH 11/22] attempt to remedy regex warnings --- fortls/regex_patterns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fortls/regex_patterns.py b/fortls/regex_patterns.py index 37e10c3e..33001c97 100644 --- a/fortls/regex_patterns.py +++ b/fortls/regex_patterns.py @@ -126,11 +126,11 @@ class FortranRegularExpressions: DEFINED: Pattern = compile(r"defined[ ]*\(?[ ]*([a-z_]\w*)[ ]*\)?", I) PP_REGEX: Pattern = compile(r"[ ]*#[ ]*(if |ifdef|ifndef|else|elif|endif)") PP_DEF: Pattern = compile( - r"[ ]*#[ ]*(define|undef)[ ]*(\w+)(\([ ]*(\w+[ ]*,?[ \w,]*)*[ ]*\))?", + r"[ ]*#[ ]*(define|undef)[ ]+(\w+)(\([ ]*(\w+[ ]*,?[ \w,]*)*[ ]*\))?", I, ) PP_DEF_TEST: Pattern = compile(r"(![ ]*)?defined[ ]*\([ ]*(\w*)[ ]*\)$", I) - PP_INCLUDE: Pattern = compile(r"[ ]*#[ ]*include[ ]*([\"\w\.]*)", I) + PP_INCLUDE: Pattern = compile(r"[ ]*#[ ]*include[ ]+([\"\w\.]*)", I) PP_ANY: Pattern = compile(r"(^[ ]*#[ ]*\w*:?\w+)") # Context matching rules CALL: Pattern = compile(r"[ ]*CALL[ ]+[\w%]*$", I) From efed76f05d1a8b585e03408a53c910b2987127d1 Mon Sep 17 00:00:00 2001 From: Randy Eckman Date: Thu, 4 Jan 2024 12:11:29 -0600 Subject: [PATCH 12/22] second attempt to fix regex warnings --- fortls/regex_patterns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fortls/regex_patterns.py b/fortls/regex_patterns.py index 33001c97..93420639 100644 --- a/fortls/regex_patterns.py +++ b/fortls/regex_patterns.py @@ -126,7 +126,7 @@ class FortranRegularExpressions: DEFINED: Pattern = compile(r"defined[ ]*\(?[ ]*([a-z_]\w*)[ ]*\)?", I) PP_REGEX: Pattern = compile(r"[ ]*#[ ]*(if |ifdef|ifndef|else|elif|endif)") PP_DEF: Pattern = compile( - r"[ ]*#[ ]*(define|undef)[ ]+(\w+)(\([ ]*(\w+[ ]*,?[ \w,]*)*[ ]*\))?", + r"[ ]*#[ ]*(define|undef)[ ]+(\w+)(\([ ]*([\w][ \w,]*)*[ ]*\))?", I, ) PP_DEF_TEST: Pattern = compile(r"(![ ]*)?defined[ ]*\([ ]*(\w*)[ ]*\)$", I) From c62ef0ff4d4d5abf618a3e54b4c50c82cef57f8d Mon Sep 17 00:00:00 2001 From: Randy Eckman Date: Thu, 4 Jan 2024 12:58:53 -0600 Subject: [PATCH 13/22] remove ambiguous define arg matching --- fortls/langserver.py | 1 + fortls/regex_patterns.py | 2 +- test/test_preproc.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/fortls/langserver.py b/fortls/langserver.py index 444eb2c7..2b25caa7 100644 --- a/fortls/langserver.py +++ b/fortls/langserver.py @@ -755,6 +755,7 @@ def get_definition( def_arg_str = "" if isinstance(def_value, tuple): def_arg_str, def_value = def_value + def_arg_str = ', '.join([x.strip() for x in def_arg_str.split(',')]) def_arg_str = f"({def_arg_str})" var = Variable( diff --git a/fortls/regex_patterns.py b/fortls/regex_patterns.py index 93420639..19cfef29 100644 --- a/fortls/regex_patterns.py +++ b/fortls/regex_patterns.py @@ -126,7 +126,7 @@ class FortranRegularExpressions: DEFINED: Pattern = compile(r"defined[ ]*\(?[ ]*([a-z_]\w*)[ ]*\)?", I) PP_REGEX: Pattern = compile(r"[ ]*#[ ]*(if |ifdef|ifndef|else|elif|endif)") PP_DEF: Pattern = compile( - r"[ ]*#[ ]*(define|undef)[ ]+(\w+)(\([ ]*([\w][ \w,]*)*[ ]*\))?", + r"[ ]*#[ ]*(define|undef)[ ]+(\w+)(\([ ]*([ \w,]*?)[ ]*\))?", I, ) PP_DEF_TEST: Pattern = compile(r"(![ ]*)?defined[ ]*\([ ]*(\w*)[ ]*\)$", I) diff --git a/test/test_preproc.py b/test/test_preproc.py index 27a83cf6..a00a65f1 100644 --- a/test/test_preproc.py +++ b/test/test_preproc.py @@ -74,7 +74,7 @@ def check_return(result_array, checks): "```fortran90\nINTEGER, PARAMETER :: res = 1+0+0+0\n```", "```fortran90\n#define MAYBEWRAP(PROCEDURE) PROCEDURE\n```", "```fortran90\nSUBROUTINE test_type_set_test()\n```", - "```fortran90\n#define MACROARGS(x , y ) x + y\n```", + "```fortran90\n#define MACROARGS(x, y) x + y\n```", ) assert len(ref_results) == len(results) - 1 check_return(results[1:], ref_results) From 107180d12d3412c789947f4b599711a7d825f5fe Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 4 Jan 2024 18:59:11 +0000 Subject: [PATCH 14/22] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- fortls/langserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fortls/langserver.py b/fortls/langserver.py index 2b25caa7..484ea765 100644 --- a/fortls/langserver.py +++ b/fortls/langserver.py @@ -755,7 +755,7 @@ def get_definition( def_arg_str = "" if isinstance(def_value, tuple): def_arg_str, def_value = def_value - def_arg_str = ', '.join([x.strip() for x in def_arg_str.split(',')]) + def_arg_str = ", ".join([x.strip() for x in def_arg_str.split(",")]) def_arg_str = f"({def_arg_str})" var = Variable( From 0ecc55dcf8b5109b0c7e5e7a29ccaab89b9e6f0c Mon Sep 17 00:00:00 2001 From: Randy Eckman Date: Sat, 6 Jan 2024 18:43:22 -0600 Subject: [PATCH 15/22] additional support for Intel FPP --- docs/options.rst | 8 +++ fortls/__init__.py | 5 +- fortls/fortls.schema.json | 6 +++ fortls/interface.py | 5 ++ fortls/langserver.py | 14 ++++-- fortls/parsers/internal/parser.py | 49 ++++++++++++------- fortls/regex_patterns.py | 12 +++-- test/test_interface.py | 5 +- test/test_preproc.py | 8 +-- test/test_server.py | 4 +- test/test_source/f90_config.json | 1 + test/test_source/pp/.pp_conf.json | 3 +- .../pp/preproc_spacing_arg_defs.F90 | 8 ++- 13 files changed, 96 insertions(+), 32 deletions(-) diff --git a/docs/options.rst b/docs/options.rst index c5327418..faf9c37a 100644 --- a/docs/options.rst +++ b/docs/options.rst @@ -72,6 +72,7 @@ All the ``fortls`` settings with their default arguments can be found below "pp_suffixes": [], "include_dirs": [], "pp_defs": {}, + "pp_parse_intel": false, "symbol_skip_mem": false, @@ -198,6 +199,13 @@ Additional **preprocessor definitions** from what are specified in files found i .. note:: Definitions in ``pp_defs`` will override definitions from ``include_dirs`` +pp_parse_intel +************** + +Parses Intel compiler directive defines and conditionals of the form +``!DEC$``, ``!DIR$``, ``CDIR$``, ``CDEC$``, ``*DIR$``, ``*DEC$``, or ``!MS$``. +Only defines, undefines, and if defined statements are evaluated. + Limitations *********** diff --git a/fortls/__init__.py b/fortls/__init__.py index faebde88..1bfeaa1e 100644 --- a/fortls/__init__.py +++ b/fortls/__init__.py @@ -472,6 +472,7 @@ def locate_config(root: str) -> str | None: pp_suffixes = None pp_defs = {} include_dirs = set() + pp_parse_intel = False if args.debug_rootpath: # Check for config files config_path = locate_config(args.debug_rootpath) @@ -482,6 +483,7 @@ def locate_config(root: str) -> str | None: config_dict = json.load(fhandle) pp_suffixes = config_dict.get("pp_suffixes", None) pp_defs = config_dict.get("pp_defs", {}) + pp_parse_intel = config_dict.get("pp_parse_intel", False) include_dirs = set() for path in config_dict.get("include_dirs", set()): include_dirs.update( @@ -501,7 +503,8 @@ def locate_config(root: str) -> str | None: error_exit(f"Reading file failed: {err_str}") print(f" Detected format: {'fixed' if file_obj.fixed else 'free'}") print("\n=========\nParser Output\n=========\n") - file_ast = file_obj.parse(debug=True, pp_defs=pp_defs, include_dirs=include_dirs) + file_ast = file_obj.parse(debug=True, pp_defs=pp_defs, include_dirs=include_dirs, + pp_parse_intel=pp_parse_intel) print("\n=========\nObject Tree\n=========\n") for obj in file_ast.get_scopes(): print("{}: {}".format(obj.get_type(), obj.FQSN)) diff --git a/fortls/fortls.schema.json b/fortls/fortls.schema.json index f6a8c09a..978d0b37 100644 --- a/fortls/fortls.schema.json +++ b/fortls/fortls.schema.json @@ -171,6 +171,12 @@ "default": {}, "type": "object" }, + "pp_parse_intel": { + "title": "Parse Intel FPP directives", + "description": "Parses Intel compiler directive defines and conditionals.", + "default": false, + "type": "boolean" + }, "symbol_skip_mem": { "title": "Symbol Skip Mem", "description": "Do not include type members in document symbol results", diff --git a/fortls/interface.py b/fortls/interface.py index 05cc8c18..6113cfa6 100644 --- a/fortls/interface.py +++ b/fortls/interface.py @@ -256,6 +256,11 @@ def cli(name: str = "fortls") -> argparse.ArgumentParser: "Preprocessor definitions are normally included via INCLUDE_DIRS" ), ) + group.add_argument( + "--pp_parse_intel", + action="store_true", + help="Parses Intel compiler directive defines and conditionals.", + ) # Symbols options ---------------------------------------------------------- group = parser.add_argument_group("Symbols options") diff --git a/fortls/langserver.py b/fortls/langserver.py index 484ea765..c1025e72 100644 --- a/fortls/langserver.py +++ b/fortls/langserver.py @@ -1323,7 +1323,8 @@ def serve_onChange(self, request: dict): # Update inheritance (currently file only) # tmp_file.ast.resolve_links(self.obj_tree, self.link_version) elif file_obj.preproc: - file_obj.preprocess(pp_defs=self.pp_defs) + file_obj.preprocess(pp_defs=self.pp_defs, + pp_parse_intel=self.pp_parse_intel) self.pp_defs = {**self.pp_defs, **file_obj.pp_defs} def serve_onOpen(self, request: dict): @@ -1396,7 +1397,8 @@ def update_workspace_file( if not file_changed: return False, None ast_new = file_obj.parse( - pp_defs=self.pp_defs, include_dirs=self.include_dirs + pp_defs=self.pp_defs, include_dirs=self.include_dirs, + pp_parse_intel=self.pp_parse_intel ) # Add the included read in pp_defs from to the ones specified in the # configuration file @@ -1429,6 +1431,7 @@ def file_init( pp_suffixes: list[str], include_dirs: set[str], sort: bool, + pp_parse_intel: bool, ): """Initialise a Fortran file @@ -1444,6 +1447,8 @@ def file_init( Preprocessor only include directories, not used by normal parser sort : bool Whether or not keywords should be sorted + pp_parse_intel : bool + Parse Intel FPP directives Returns ------- @@ -1460,7 +1465,8 @@ def file_init( # This is a bypass. # For more see on SO: shorturl.at/hwAG1 set_keyword_ordering(sort) - file_ast = file_obj.parse(pp_defs=pp_defs, include_dirs=include_dirs) + file_ast = file_obj.parse(pp_defs=pp_defs, include_dirs=include_dirs, + pp_parse_intel=pp_parse_intel) except: log.error("Error while parsing file %s", filepath, exc_info=True) return "Error during parsing" @@ -1483,6 +1489,7 @@ def workspace_init(self): self.pp_suffixes, self.include_dirs, self.sort_keywords, + self.pp_parse_intel, ), ) pool.close() @@ -1639,6 +1646,7 @@ def _load_config_file_general(self, config_dict: dict) -> None: def _load_config_file_preproc(self, config_dict: dict) -> None: self.pp_suffixes = config_dict.get("pp_suffixes", None) self.pp_defs = config_dict.get("pp_defs", {}) + self.pp_parse_intel = config_dict.get("pp_parse_intel", False) if isinstance(self.pp_defs, list): self.pp_defs = {key: "" for key in self.pp_defs} diff --git a/fortls/parsers/internal/parser.py b/fortls/parsers/internal/parser.py index 481ae433..7f38e3f8 100644 --- a/fortls/parsers/internal/parser.py +++ b/fortls/parsers/internal/parser.py @@ -1176,7 +1176,8 @@ def find_word_in_code_line( return line_no, word_range def preprocess( - self, pp_defs: dict = None, include_dirs: set = None, debug: bool = False + self, pp_defs: dict = None, include_dirs: set = None, debug: bool = False, + pp_parse_intel: bool = False, ) -> tuple[list, list]: if pp_defs is None: pp_defs = {} @@ -1189,6 +1190,7 @@ def preprocess( pp_defs=pp_defs, include_dirs=include_dirs, debug=debug, + pp_parse_intel=pp_parse_intel, ) return pp_skips, pp_defines @@ -1231,6 +1233,7 @@ def parse( debug: bool = False, pp_defs: dict = None, include_dirs: set = None, + pp_parse_intel: bool = False, ) -> FortranAST: """Parse Fortran file contents of a fortran_file object and build an Abstract Syntax Tree (AST) @@ -1243,6 +1246,8 @@ def parse( Preprocessor definitions and their values, by default None include_dirs : set, optional Preprocessor include directories, by default None + pp_parse_intel : bool, optional + Parse Intel FPP directives, by default False Returns ------- @@ -1265,7 +1270,8 @@ def parse( if self.preproc: log.debug("=== PreProc Pass ===\n") pp_skips, pp_defines = self.preprocess( - pp_defs=pp_defs, include_dirs=include_dirs, debug=debug + pp_defs=pp_defs, include_dirs=include_dirs, debug=debug, + pp_parse_intel=pp_parse_intel, ) for pp_reg in pp_skips: file_ast.start_ppif(pp_reg[0]) @@ -2026,6 +2032,7 @@ def preprocess_file( pp_defs: dict = None, include_dirs: set = None, debug: bool = False, + pp_parse_intel: bool = False, ): # Look for and mark excluded preprocessor paths in file # Initial implementation only looks for "if" and "ifndef" statements. @@ -2106,19 +2113,19 @@ def replace_vars(line: str): continue # Handle conditional statements match = FRegex.PP_REGEX.match(line) - if match: + if match and check_pp_prefix(match.group(1), pp_parse_intel): output_file.append(line) def_name = None if_start = False # Opening conditional statements - if match.group(1) == "if ": - is_path = eval_pp_if(line[match.end(1) :], defs_tmp) + if match.group(2) == "if ": + is_path = eval_pp_if(line[match.end(2) :], defs_tmp) if_start = True - elif match.group(1) == "ifdef": + elif match.group(2) == "ifdef": if_start = True def_name = line[match.end(0) :].strip() is_path = def_name in defs_tmp - elif match.group(1) == "ifndef": + elif match.group(2) == "ifndef": if_start = True def_name = line[match.end(0) :].strip() is_path = not (def_name in defs_tmp) @@ -2136,7 +2143,7 @@ def replace_vars(line: str): inc_start = False exc_start = False exc_continue = False - if match.group(1) == "elif": + if match.group(2) == "elif": if (not pp_stack_group) or (pp_stack_group[-1][0] != len(pp_stack)): # First elif statement for this elif group if pp_stack[-1][0] < 0: @@ -2148,7 +2155,7 @@ def replace_vars(line: str): exc_continue = True if pp_stack[-1][0] < 0: pp_stack[-1][0] = i + 1 - elif eval_pp_if(line[match.end(1) :], defs_tmp): + elif eval_pp_if(line[match.end(2) :], defs_tmp): pp_stack[-1][1] = i + 1 pp_skips.append(pp_stack.pop()) pp_stack_group[-1][1] = True @@ -2156,7 +2163,7 @@ def replace_vars(line: str): inc_start = True else: exc_start = True - elif match.group(1) == "else": + elif match.group(2) == "else": if pp_stack[-1][0] < 0: pp_stack[-1][0] = i + 1 exc_start = True @@ -2172,7 +2179,7 @@ def replace_vars(line: str): pp_skips.append(pp_stack.pop()) pp_stack.append([-1, -1]) inc_start = True - elif match.group(1) == "endif": + elif match.group(2) == "endif": if pp_stack_group and (pp_stack_group[-1][0] == len(pp_stack)): pp_stack_group.pop() if pp_stack[-1][0] < 0: @@ -2193,10 +2200,11 @@ def replace_vars(line: str): continue # Handle variable/macro definitions files match = FRegex.PP_DEF.match(line) - if (match is not None) and ((len(pp_stack) == 0) or (pp_stack[-1][0] < 0)): + if ((match is not None and check_pp_prefix(match.group(1), pp_parse_intel)) + and ((len(pp_stack) == 0) or (pp_stack[-1][0] < 0))): output_file.append(line) pp_defines.append(i + 1) - def_name = match.group(2) + def_name = match.group(3) # If this is an argument list of a function add them to the name # get_definition will only return the function name upon hover # hence if the argument list is appended in the def_name then @@ -2205,7 +2213,7 @@ def replace_vars(line: str): # This also does not allow for multiline argument list definitions. # if match.group(3): # def_name += match.group(3) - if (match.group(1) == "define") and (def_name not in defs_tmp): + if (match.group(2) == "define") and (def_name not in defs_tmp): eq_ind = line[match.end(0) :].find(" ") if eq_ind < 0: eq_ind = line[match.end(0) :].find("\t") @@ -2221,11 +2229,13 @@ def replace_vars(line: str): def_value = "True" # are there arguments to parse? - if match.group(3): - def_value = (match.group(4), def_value) + if match.group(4): + def_value = (match.group(5), def_value) defs_tmp[def_name] = def_value - elif (match.group(1) == "undef") and (def_name in defs_tmp): + elif ((match.group(2) == "undef" + or (pp_parse_intel and match.group(2) == "undefine")) + and (def_name in defs_tmp)): defs_tmp.pop(def_name, None) log.debug(f"{line.strip()} !!! Define statement({i + 1})") continue @@ -2255,6 +2265,7 @@ def replace_vars(line: str): pp_defs=defs_tmp, include_dirs=include_dirs, debug=debug, + pp_parse_intel=pp_parse_intel, ) log.debug("!!! Completed parsing include file\n") @@ -2319,3 +2330,7 @@ def append_multiline_macro(pp_defs: dict, def_name: str, line: str): def_value = (def_args, def_value) pp_defs[def_name] = def_value + + +def check_pp_prefix(prefix: str, pp_parse_intel: bool): + return prefix == '#' or (pp_parse_intel and FRegex.INTEL_FPP_PRE.match(prefix)) diff --git a/fortls/regex_patterns.py b/fortls/regex_patterns.py index 19cfef29..64b4f269 100644 --- a/fortls/regex_patterns.py +++ b/fortls/regex_patterns.py @@ -124,14 +124,20 @@ class FortranRegularExpressions: FREE_FORMAT_TEST: Pattern = compile(r"[ ]{1,4}[a-z]", I) # Preprocessor matching rules DEFINED: Pattern = compile(r"defined[ ]*\(?[ ]*([a-z_]\w*)[ ]*\)?", I) - PP_REGEX: Pattern = compile(r"[ ]*#[ ]*(if |ifdef|ifndef|else|elif|endif)") + INTEL_FPP_PRE_STR = r"[c!*]DEC\$|[c!*]DIR\$|!MS\$" + INTEL_FPP_PRE: Pattern = compile(f"{INTEL_FPP_PRE_STR}", I) + PP_REGEX: Pattern = compile( + rf"[ ]*(#|{INTEL_FPP_PRE_STR})[ ]*(if |ifdef|ifndef|else|elif|endif)", + I, + ) PP_DEF: Pattern = compile( - r"[ ]*#[ ]*(define|undef)[ ]+(\w+)(\([ ]*([ \w,]*?)[ ]*\))?", + rf"[ ]*(#|{INTEL_FPP_PRE_STR})[ ]*(define|undef|undefined)" + r"[ ]+(\w+)(\([ ]*([ \w,]*?)[ ]*\))?", I, ) PP_DEF_TEST: Pattern = compile(r"(![ ]*)?defined[ ]*\([ ]*(\w*)[ ]*\)$", I) PP_INCLUDE: Pattern = compile(r"[ ]*#[ ]*include[ ]+([\"\w\.]*)", I) - PP_ANY: Pattern = compile(r"(^[ ]*#[ ]*\w*:?\w+)") + PP_ANY: Pattern = compile(rf"(^[ ]*(?:#|{INTEL_FPP_PRE_STR})[ ]*\w*:?\w+)") # Context matching rules CALL: Pattern = compile(r"[ ]*CALL[ ]+[\w%]*$", I) INT_STMNT: Pattern = compile(r"^[ ]*[a-z]*$", I) diff --git a/test/test_interface.py b/test/test_interface.py index 8832b40f..03e63211 100644 --- a/test/test_interface.py +++ b/test/test_interface.py @@ -65,12 +65,14 @@ def test_command_line_diagnostic_options(): def test_command_line_preprocessor_options(): args = parser.parse_args( - "--pp_suffixes .h .fh --include_dirs /usr/include/** ./local/incl --pp_defs" + "--pp_suffixes .h .fh --include_dirs /usr/include/** ./local/incl" + " --pp_parse_intel --pp_defs" ' {"HAVE_PETSC":"","HAVE_ZOLTAN":"","Mat":"type(tMat)"}'.split() ) assert args.pp_suffixes == [".h", ".fh"] assert args.include_dirs == {"/usr/include/**", "./local/incl"} assert args.pp_defs == {"HAVE_PETSC": "", "HAVE_ZOLTAN": "", "Mat": "type(tMat)"} + assert args.pp_parse_intel def test_command_line_symbol_options(): @@ -150,6 +152,7 @@ def test_config_file_preprocessor_options(): "HAVE_ZOLTAN": "", "Mat": "type(tMat)", } + assert server.pp_parse_intel def test_config_file_symbols_options(): diff --git a/test/test_preproc.py b/test/test_preproc.py index a00a65f1..c8fc3214 100644 --- a/test/test_preproc.py +++ b/test/test_preproc.py @@ -44,8 +44,9 @@ def check_return(result_array, checks): string += hover_req(file_path, 30, 23) file_path = root_dir / "preproc_spacing_arg_defs.F90" string += hover_req(file_path, 11, 20) - string += hover_req(file_path, 18, 17) - string += hover_req(file_path, 20, 13) + string += hover_req(file_path, 24, 17) + string += hover_req(file_path, 26, 13) + string += hover_req(file_path, 26, 42) config = str(root_dir / ".pp_conf.json") errcode, results = run_request(string, ["--config", config]) assert errcode == 0 @@ -75,6 +76,7 @@ def check_return(result_array, checks): "```fortran90\n#define MAYBEWRAP(PROCEDURE) PROCEDURE\n```", "```fortran90\nSUBROUTINE test_type_set_test()\n```", "```fortran90\n#define MACROARGS(x, y) x + y\n```", + "```fortran90\nINTEGER(KIND=4), PARAMETER :: C_LONG = 4\n```", ) assert len(ref_results) == len(results) - 1 check_return(results[1:], ref_results) @@ -89,5 +91,5 @@ def check_return(result_array, checks): # ff = FortranFile(file_path) # ff.load_from_disk() # include_dirs = {root_dir / "include"} -# file_ast = ff.parse(include_dirs=include_dirs, debug=True) +# file_ast = ff.parse(include_dirs=include_dirs, debug=True, pp_parse_intel=True) # assert file_ast diff --git a/test/test_server.py b/test/test_server.py index 875d3035..fb63377d 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -198,10 +198,10 @@ def check_return(result_array): ["test_str2", 13, 5], ["test_sub", 6, 8], ["test_type", 5, 5], - ["test_type_set_test", 6, 23], + ["test_type_set_test", 6, 29], ["test_vis_mod", 2, 0], ["the_test", 13, 15], - ["wrap_test_type_set_test", 6, 31], + ["wrap_test_type_set_test", 6, 37], ) assert len(result_array) == len(objs) for i, obj in enumerate(objs): diff --git a/test/test_source/f90_config.json b/test/test_source/f90_config.json index 2d4779b7..2466d3ae 100644 --- a/test/test_source/f90_config.json +++ b/test/test_source/f90_config.json @@ -31,6 +31,7 @@ "HAVE_ZOLTAN": "", "Mat": "type(tMat)" }, + "pp_parse_intel": true, "symbol_skip_mem": true, diff --git a/test/test_source/pp/.pp_conf.json b/test/test_source/pp/.pp_conf.json index 0cf75a8a..ac1bd691 100644 --- a/test/test_source/pp/.pp_conf.json +++ b/test/test_source/pp/.pp_conf.json @@ -7,5 +7,6 @@ "pp_suffixes": [".h", ".F90"], "incl_suffixes": [".h"], "include_dirs": ["include"], - "pp_defs": { "HAVE_CONTIGUOUS": "" } + "pp_defs": { "HAVE_CONTIGUOUS": "" }, + "pp_parse_intel": true } diff --git a/test/test_source/pp/preproc_spacing_arg_defs.F90 b/test/test_source/pp/preproc_spacing_arg_defs.F90 index 72e11917..cac7753d 100644 --- a/test/test_source/pp/preproc_spacing_arg_defs.F90 +++ b/test/test_source/pp/preproc_spacing_arg_defs.F90 @@ -16,9 +16,15 @@ program preprocessor_spacing_arg_defs type(test_type) :: the_test integer :: argtest + !DEC$ IF DEFINED(SPACING_TEST) + INTEGER (KIND=4), PARAMETER :: C_LONG = 4 + !DEC$ ELSE + INTEGER (KIND=4), PARAMETER :: C_LONG = 8 + !DEC$ ENDIF + call the_test%set_test() - argtest = MACROARGS(the_test%test_int, 4) + argtest = MACROARGS(the_test%test_int, C_LONG) contains subroutine test_type_set_test(me) From 0de9e0e285e27ecfaffeeb6c3d76c6665385b9a1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 7 Jan 2024 00:43:42 +0000 Subject: [PATCH 16/22] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- fortls/__init__.py | 8 ++++++-- fortls/langserver.py | 17 +++++++++++------ fortls/parsers/internal/parser.py | 23 +++++++++++++++-------- 3 files changed, 32 insertions(+), 16 deletions(-) diff --git a/fortls/__init__.py b/fortls/__init__.py index 1bfeaa1e..ddedcd19 100644 --- a/fortls/__init__.py +++ b/fortls/__init__.py @@ -503,8 +503,12 @@ def locate_config(root: str) -> str | None: error_exit(f"Reading file failed: {err_str}") print(f" Detected format: {'fixed' if file_obj.fixed else 'free'}") print("\n=========\nParser Output\n=========\n") - file_ast = file_obj.parse(debug=True, pp_defs=pp_defs, include_dirs=include_dirs, - pp_parse_intel=pp_parse_intel) + file_ast = file_obj.parse( + debug=True, + pp_defs=pp_defs, + include_dirs=include_dirs, + pp_parse_intel=pp_parse_intel, + ) print("\n=========\nObject Tree\n=========\n") for obj in file_ast.get_scopes(): print("{}: {}".format(obj.get_type(), obj.FQSN)) diff --git a/fortls/langserver.py b/fortls/langserver.py index c1025e72..d41b4017 100644 --- a/fortls/langserver.py +++ b/fortls/langserver.py @@ -1323,8 +1323,9 @@ def serve_onChange(self, request: dict): # Update inheritance (currently file only) # tmp_file.ast.resolve_links(self.obj_tree, self.link_version) elif file_obj.preproc: - file_obj.preprocess(pp_defs=self.pp_defs, - pp_parse_intel=self.pp_parse_intel) + file_obj.preprocess( + pp_defs=self.pp_defs, pp_parse_intel=self.pp_parse_intel + ) self.pp_defs = {**self.pp_defs, **file_obj.pp_defs} def serve_onOpen(self, request: dict): @@ -1397,8 +1398,9 @@ def update_workspace_file( if not file_changed: return False, None ast_new = file_obj.parse( - pp_defs=self.pp_defs, include_dirs=self.include_dirs, - pp_parse_intel=self.pp_parse_intel + pp_defs=self.pp_defs, + include_dirs=self.include_dirs, + pp_parse_intel=self.pp_parse_intel, ) # Add the included read in pp_defs from to the ones specified in the # configuration file @@ -1465,8 +1467,11 @@ def file_init( # This is a bypass. # For more see on SO: shorturl.at/hwAG1 set_keyword_ordering(sort) - file_ast = file_obj.parse(pp_defs=pp_defs, include_dirs=include_dirs, - pp_parse_intel=pp_parse_intel) + file_ast = file_obj.parse( + pp_defs=pp_defs, + include_dirs=include_dirs, + pp_parse_intel=pp_parse_intel, + ) except: log.error("Error while parsing file %s", filepath, exc_info=True) return "Error during parsing" diff --git a/fortls/parsers/internal/parser.py b/fortls/parsers/internal/parser.py index 7f38e3f8..9709f584 100644 --- a/fortls/parsers/internal/parser.py +++ b/fortls/parsers/internal/parser.py @@ -1176,7 +1176,10 @@ def find_word_in_code_line( return line_no, word_range def preprocess( - self, pp_defs: dict = None, include_dirs: set = None, debug: bool = False, + self, + pp_defs: dict = None, + include_dirs: set = None, + debug: bool = False, pp_parse_intel: bool = False, ) -> tuple[list, list]: if pp_defs is None: @@ -1270,7 +1273,9 @@ def parse( if self.preproc: log.debug("=== PreProc Pass ===\n") pp_skips, pp_defines = self.preprocess( - pp_defs=pp_defs, include_dirs=include_dirs, debug=debug, + pp_defs=pp_defs, + include_dirs=include_dirs, + debug=debug, pp_parse_intel=pp_parse_intel, ) for pp_reg in pp_skips: @@ -2200,8 +2205,9 @@ def replace_vars(line: str): continue # Handle variable/macro definitions files match = FRegex.PP_DEF.match(line) - if ((match is not None and check_pp_prefix(match.group(1), pp_parse_intel)) - and ((len(pp_stack) == 0) or (pp_stack[-1][0] < 0))): + if (match is not None and check_pp_prefix(match.group(1), pp_parse_intel)) and ( + (len(pp_stack) == 0) or (pp_stack[-1][0] < 0) + ): output_file.append(line) pp_defines.append(i + 1) def_name = match.group(3) @@ -2233,9 +2239,10 @@ def replace_vars(line: str): def_value = (match.group(5), def_value) defs_tmp[def_name] = def_value - elif ((match.group(2) == "undef" - or (pp_parse_intel and match.group(2) == "undefine")) - and (def_name in defs_tmp)): + elif ( + match.group(2) == "undef" + or (pp_parse_intel and match.group(2) == "undefine") + ) and (def_name in defs_tmp): defs_tmp.pop(def_name, None) log.debug(f"{line.strip()} !!! Define statement({i + 1})") continue @@ -2333,4 +2340,4 @@ def append_multiline_macro(pp_defs: dict, def_name: str, line: str): def check_pp_prefix(prefix: str, pp_parse_intel: bool): - return prefix == '#' or (pp_parse_intel and FRegex.INTEL_FPP_PRE.match(prefix)) + return prefix == "#" or (pp_parse_intel and FRegex.INTEL_FPP_PRE.match(prefix)) From 1ea9700929938332f806b0445f45399641eb535d Mon Sep 17 00:00:00 2001 From: Randy Eckman Date: Sat, 6 Jan 2024 18:48:19 -0600 Subject: [PATCH 17/22] fix issue with json schema --- fortls/fortls.schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fortls/fortls.schema.json b/fortls/fortls.schema.json index 978d0b37..ba78802c 100644 --- a/fortls/fortls.schema.json +++ b/fortls/fortls.schema.json @@ -172,7 +172,7 @@ "type": "object" }, "pp_parse_intel": { - "title": "Parse Intel FPP directives", + "title": "Pp Parse Intel", "description": "Parses Intel compiler directive defines and conditionals.", "default": false, "type": "boolean" From 755370efb7556877f043173b34ea40e59ab556cb Mon Sep 17 00:00:00 2001 From: Randy Eckman Date: Sat, 6 Jan 2024 19:53:50 -0600 Subject: [PATCH 18/22] addl logical operators for intel fpp --- fortls/parsers/internal/parser.py | 50 +++++++++++++++---- .../pp/preproc_spacing_arg_defs.F90 | 2 +- 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/fortls/parsers/internal/parser.py b/fortls/parsers/internal/parser.py index 9709f584..41769ff2 100644 --- a/fortls/parsers/internal/parser.py +++ b/fortls/parsers/internal/parser.py @@ -2043,13 +2043,41 @@ def preprocess_file( # Initial implementation only looks for "if" and "ifndef" statements. # For "if" statements all blocks are excluded except the "else" block if present # For "ifndef" statements all blocks excluding the first block are excluded - def eval_pp_if(text, defs: dict = None): - def replace_ops(expr: str): + def eval_pp_if(text, defs: dict = None, pp_parse_intel: bool = False): + def replace_ops(expr: str, pp_parse_intel: bool): expr = expr.replace("&&", " and ") expr = expr.replace("||", " or ") expr = expr.replace("!=", " <> ") expr = expr.replace("!", " not ") expr = expr.replace(" <> ", " != ") + + if pp_parse_intel: + expr = expr.replace("/=", " != ") + expr = expr.replace(".AND.", " and ") + expr = expr.replace(".LT.", " < ") + expr = expr.replace(".GT.", " > ") + expr = expr.replace(".EQ.", " == ") + expr = expr.replace(".LE.", " <= ") + expr = expr.replace(".GE.", " >= ") + expr = expr.replace(".NE.", " != ") + expr = expr.replace(".EQV.", " == ") + expr = expr.replace(".NEQV.", " != ") + expr = expr.replace(".NOT.", " not ") + expr = expr.replace(".OR.", " or ") + expr = expr.replace(".XOR.", " != ") # admittedly a hack... + expr = expr.replace(".and.", " and ") + expr = expr.replace(".lt.", " < ") + expr = expr.replace(".gt.", " > ") + expr = expr.replace(".eq.", " == ") + expr = expr.replace(".le.", " <= ") + expr = expr.replace(".ge.", " >= ") + expr = expr.replace(".ne.", " != ") + expr = expr.replace(".eqv.", " == ") + expr = expr.replace(".neqv.", " != ") + expr = expr.replace(".not.", " not ") + expr = expr.replace(".or.", " or ") + expr = expr.replace(".xor.", " != ") # admittedly a hack... + return expr def replace_defined(line: str): @@ -2085,7 +2113,7 @@ def replace_vars(line: str): out_line = replace_defined(text) out_line = replace_vars(out_line) try: - line_res = eval(replace_ops(out_line)) + line_res = eval(replace_ops(out_line, pp_parse_intel)) except: return False else: @@ -2123,14 +2151,14 @@ def replace_vars(line: str): def_name = None if_start = False # Opening conditional statements - if match.group(2) == "if ": - is_path = eval_pp_if(line[match.end(2) :], defs_tmp) + if match.group(2).lower() == "if ": + is_path = eval_pp_if(line[match.end(2) :], defs_tmp, pp_parse_intel) if_start = True - elif match.group(2) == "ifdef": + elif match.group(2).lower() == "ifdef": if_start = True def_name = line[match.end(0) :].strip() is_path = def_name in defs_tmp - elif match.group(2) == "ifndef": + elif match.group(2).lower() == "ifndef": if_start = True def_name = line[match.end(0) :].strip() is_path = not (def_name in defs_tmp) @@ -2148,7 +2176,7 @@ def replace_vars(line: str): inc_start = False exc_start = False exc_continue = False - if match.group(2) == "elif": + if match.group(2).lower() == "elif": if (not pp_stack_group) or (pp_stack_group[-1][0] != len(pp_stack)): # First elif statement for this elif group if pp_stack[-1][0] < 0: @@ -2160,7 +2188,7 @@ def replace_vars(line: str): exc_continue = True if pp_stack[-1][0] < 0: pp_stack[-1][0] = i + 1 - elif eval_pp_if(line[match.end(2) :], defs_tmp): + elif eval_pp_if(line[match.end(2) :], defs_tmp, pp_parse_intel): pp_stack[-1][1] = i + 1 pp_skips.append(pp_stack.pop()) pp_stack_group[-1][1] = True @@ -2168,7 +2196,7 @@ def replace_vars(line: str): inc_start = True else: exc_start = True - elif match.group(2) == "else": + elif match.group(2).lower() == "else": if pp_stack[-1][0] < 0: pp_stack[-1][0] = i + 1 exc_start = True @@ -2184,7 +2212,7 @@ def replace_vars(line: str): pp_skips.append(pp_stack.pop()) pp_stack.append([-1, -1]) inc_start = True - elif match.group(2) == "endif": + elif match.group(2).lower() == "endif": if pp_stack_group and (pp_stack_group[-1][0] == len(pp_stack)): pp_stack_group.pop() if pp_stack[-1][0] < 0: diff --git a/test/test_source/pp/preproc_spacing_arg_defs.F90 b/test/test_source/pp/preproc_spacing_arg_defs.F90 index cac7753d..ced397ab 100644 --- a/test/test_source/pp/preproc_spacing_arg_defs.F90 +++ b/test/test_source/pp/preproc_spacing_arg_defs.F90 @@ -16,7 +16,7 @@ program preprocessor_spacing_arg_defs type(test_type) :: the_test integer :: argtest - !DEC$ IF DEFINED(SPACING_TEST) + !DEC$ IF DEFINED(SPACING_TEST).AND.DEFINED(MACROARGS) INTEGER (KIND=4), PARAMETER :: C_LONG = 4 !DEC$ ELSE INTEGER (KIND=4), PARAMETER :: C_LONG = 8 From 9b67c67505dfda5d651a714dd30c7ce48a9d99f8 Mon Sep 17 00:00:00 2001 From: Randy Eckman Date: Sat, 6 Jan 2024 20:31:12 -0600 Subject: [PATCH 19/22] parse intel fpp operators separately --- fortls/parsers/internal/parser.py | 65 +++++++++++++++++-------------- 1 file changed, 36 insertions(+), 29 deletions(-) diff --git a/fortls/parsers/internal/parser.py b/fortls/parsers/internal/parser.py index 41769ff2..0889bad7 100644 --- a/fortls/parsers/internal/parser.py +++ b/fortls/parsers/internal/parser.py @@ -2044,39 +2044,41 @@ def preprocess_file( # For "if" statements all blocks are excluded except the "else" block if present # For "ifndef" statements all blocks excluding the first block are excluded def eval_pp_if(text, defs: dict = None, pp_parse_intel: bool = False): - def replace_ops(expr: str, pp_parse_intel: bool): + def replace_ops(expr: str): expr = expr.replace("&&", " and ") expr = expr.replace("||", " or ") expr = expr.replace("!=", " <> ") expr = expr.replace("!", " not ") expr = expr.replace(" <> ", " != ") - if pp_parse_intel: - expr = expr.replace("/=", " != ") - expr = expr.replace(".AND.", " and ") - expr = expr.replace(".LT.", " < ") - expr = expr.replace(".GT.", " > ") - expr = expr.replace(".EQ.", " == ") - expr = expr.replace(".LE.", " <= ") - expr = expr.replace(".GE.", " >= ") - expr = expr.replace(".NE.", " != ") - expr = expr.replace(".EQV.", " == ") - expr = expr.replace(".NEQV.", " != ") - expr = expr.replace(".NOT.", " not ") - expr = expr.replace(".OR.", " or ") - expr = expr.replace(".XOR.", " != ") # admittedly a hack... - expr = expr.replace(".and.", " and ") - expr = expr.replace(".lt.", " < ") - expr = expr.replace(".gt.", " > ") - expr = expr.replace(".eq.", " == ") - expr = expr.replace(".le.", " <= ") - expr = expr.replace(".ge.", " >= ") - expr = expr.replace(".ne.", " != ") - expr = expr.replace(".eqv.", " == ") - expr = expr.replace(".neqv.", " != ") - expr = expr.replace(".not.", " not ") - expr = expr.replace(".or.", " or ") - expr = expr.replace(".xor.", " != ") # admittedly a hack... + return expr + + def replace_intel_ops(expr: str): + expr = expr.replace("/=", " != ") + expr = expr.replace(".AND.", " && ") + expr = expr.replace(".LT.", " < ") + expr = expr.replace(".GT.", " > ") + expr = expr.replace(".EQ.", " == ") + expr = expr.replace(".LE.", " <= ") + expr = expr.replace(".GE.", " >= ") + expr = expr.replace(".NE.", " != ") + expr = expr.replace(".EQV.", " == ") + expr = expr.replace(".NEQV.", " != ") + expr = expr.replace(".NOT.", "!") + expr = expr.replace(".OR.", " || ") + expr = expr.replace(".XOR.", " != ") # admittedly a hack... + expr = expr.replace(".and.", " && ") + expr = expr.replace(".lt.", " < ") + expr = expr.replace(".gt.", " > ") + expr = expr.replace(".eq.", " == ") + expr = expr.replace(".le.", " <= ") + expr = expr.replace(".ge.", " >= ") + expr = expr.replace(".ne.", " != ") + expr = expr.replace(".eqv.", " == ") + expr = expr.replace(".neqv.", " != ") + expr = expr.replace(".not.", "!") + expr = expr.replace(".or.", " || ") + expr = expr.replace(".xor.", " != ") # admittedly a hack... return expr @@ -2110,10 +2112,15 @@ def replace_vars(line: str): if defs is None: defs = {} - out_line = replace_defined(text) + + out_line = text + if pp_parse_intel: + out_line = replace_intel_ops(out_line) + + out_line = replace_defined(out_line) out_line = replace_vars(out_line) try: - line_res = eval(replace_ops(out_line, pp_parse_intel)) + line_res = eval(replace_ops(out_line)) except: return False else: From ad4de4cc963c5929e00280d3c718678046182185 Mon Sep 17 00:00:00 2001 From: Randy Eckman Date: Sat, 20 Jan 2024 21:17:43 -0600 Subject: [PATCH 20/22] init commit of cherry-picked changes from #341 --- fortls/__init__.py | 6 +- fortls/langserver.py | 21 +++- fortls/parsers/internal/parser.py | 104 ++++++++++++++---- fortls/regex_patterns.py | 15 ++- test/test_preproc.py | 15 ++- test/test_server.py | 5 + test/test_source/pp/.pp_conf.json | 2 +- test/test_source/pp/include/indent.h | 22 ++++ .../pp/preproc_spacing_arg_defs.F90 | 42 +++++++ 9 files changed, 196 insertions(+), 36 deletions(-) create mode 100644 test/test_source/pp/include/indent.h create mode 100644 test/test_source/pp/preproc_spacing_arg_defs.F90 diff --git a/fortls/__init__.py b/fortls/__init__.py index faebde88..a4fc3209 100644 --- a/fortls/__init__.py +++ b/fortls/__init__.py @@ -501,7 +501,11 @@ def locate_config(root: str) -> str | None: error_exit(f"Reading file failed: {err_str}") print(f" Detected format: {'fixed' if file_obj.fixed else 'free'}") print("\n=========\nParser Output\n=========\n") - file_ast = file_obj.parse(debug=True, pp_defs=pp_defs, include_dirs=include_dirs) + file_ast = file_obj.parse( + debug=True, + pp_defs=pp_defs, + include_dirs=include_dirs, + ) print("\n=========\nObject Tree\n=========\n") for obj in file_ast.get_scopes(): print("{}: {}".format(obj.get_type(), obj.FQSN)) diff --git a/fortls/langserver.py b/fortls/langserver.py index 422061d9..ce79ec3b 100644 --- a/fortls/langserver.py +++ b/fortls/langserver.py @@ -751,11 +751,18 @@ def get_definition( return None # Search in Preprocessor defined variables if def_name in def_file.pp_defs: + def_value = def_file.pp_defs.get(def_name) + def_arg_str = "" + if isinstance(def_value, tuple): + def_arg_str, def_value = def_value + def_arg_str = ", ".join([x.strip() for x in def_arg_str.split(",")]) + def_arg_str = f"({def_arg_str})" + var = Variable( def_file.ast, def_line + 1, def_name, - f"#define {def_name} {def_file.pp_defs.get(def_name)}", + f"#define {def_name}{def_arg_str} {def_value}", [], ) return var @@ -1316,7 +1323,9 @@ def serve_onChange(self, request: dict): # Update inheritance (currently file only) # tmp_file.ast.resolve_links(self.obj_tree, self.link_version) elif file_obj.preproc: - file_obj.preprocess(pp_defs=self.pp_defs) + file_obj.preprocess( + pp_defs=self.pp_defs, + ) self.pp_defs = {**self.pp_defs, **file_obj.pp_defs} def serve_onOpen(self, request: dict): @@ -1389,7 +1398,8 @@ def update_workspace_file( if not file_changed: return False, None ast_new = file_obj.parse( - pp_defs=self.pp_defs, include_dirs=self.include_dirs + pp_defs=self.pp_defs, + include_dirs=self.include_dirs, ) # Add the included read in pp_defs from to the ones specified in the # configuration file @@ -1453,7 +1463,10 @@ def file_init( # This is a bypass. # For more see on SO: shorturl.at/hwAG1 set_keyword_ordering(sort) - file_ast = file_obj.parse(pp_defs=pp_defs, include_dirs=include_dirs) + file_ast = file_obj.parse( + pp_defs=pp_defs, + include_dirs=include_dirs, + ) except: log.error("Error while parsing file %s", filepath, exc_info=True) return "Error during parsing" diff --git a/fortls/parsers/internal/parser.py b/fortls/parsers/internal/parser.py index ae5cfa1e..14052a7a 100644 --- a/fortls/parsers/internal/parser.py +++ b/fortls/parsers/internal/parser.py @@ -1176,7 +1176,10 @@ def find_word_in_code_line( return line_no, word_range def preprocess( - self, pp_defs: dict = None, include_dirs: set = None, debug: bool = False + self, + pp_defs: dict = None, + include_dirs: set = None, + debug: bool = False, ) -> tuple[list, list]: if pp_defs is None: pp_defs = {} @@ -1265,7 +1268,9 @@ def parse( if self.preproc: log.debug("=== PreProc Pass ===\n") pp_skips, pp_defines = self.preprocess( - pp_defs=pp_defs, include_dirs=include_dirs, debug=debug + pp_defs=pp_defs, + include_dirs=include_dirs, + debug=debug, ) for pp_reg in pp_skips: file_ast.start_ppif(pp_reg[0]) @@ -2038,6 +2043,7 @@ def replace_ops(expr: str): expr = expr.replace("!=", " <> ") expr = expr.replace("!", " not ") expr = expr.replace(" <> ", " != ") + return expr def replace_defined(line: str): @@ -2070,7 +2076,9 @@ def replace_vars(line: str): if defs is None: defs = {} - out_line = replace_defined(text) + + out_line = text + out_line = replace_defined(out_line) out_line = replace_vars(out_line) try: line_res = eval(replace_ops(out_line)) @@ -2098,26 +2106,27 @@ def replace_vars(line: str): if def_cont_name is not None: output_file.append("") if line.rstrip()[-1] != "\\": - defs_tmp[def_cont_name] += line.strip() + append_multiline_macro(defs_tmp, def_cont_name, line.strip()) def_cont_name = None else: - defs_tmp[def_cont_name] += line[0:-1].strip() + append_multiline_macro(defs_tmp, def_cont_name, line[0:-1].strip()) + continue # Handle conditional statements match = FRegex.PP_REGEX.match(line) - if match: + if match and check_pp_prefix(match.group(1)): output_file.append(line) def_name = None if_start = False # Opening conditional statements - if match.group(1) == "if ": - is_path = eval_pp_if(line[match.end(1) :], defs_tmp) + if match.group(2).lower() == "if ": + is_path = eval_pp_if(line[match.end(2) :], defs_tmp) if_start = True - elif match.group(1) == "ifdef": + elif match.group(2).lower() == "ifdef": if_start = True def_name = line[match.end(0) :].strip() is_path = def_name in defs_tmp - elif match.group(1) == "ifndef": + elif match.group(2).lower() == "ifndef": if_start = True def_name = line[match.end(0) :].strip() is_path = not (def_name in defs_tmp) @@ -2135,7 +2144,7 @@ def replace_vars(line: str): inc_start = False exc_start = False exc_continue = False - if match.group(1) == "elif": + if match.group(2).lower() == "elif": if (not pp_stack_group) or (pp_stack_group[-1][0] != len(pp_stack)): # First elif statement for this elif group if pp_stack[-1][0] < 0: @@ -2147,7 +2156,7 @@ def replace_vars(line: str): exc_continue = True if pp_stack[-1][0] < 0: pp_stack[-1][0] = i + 1 - elif eval_pp_if(line[match.end(1) :], defs_tmp): + elif eval_pp_if(line[match.end(2) :], defs_tmp): pp_stack[-1][1] = i + 1 pp_skips.append(pp_stack.pop()) pp_stack_group[-1][1] = True @@ -2155,7 +2164,7 @@ def replace_vars(line: str): inc_start = True else: exc_start = True - elif match.group(1) == "else": + elif match.group(2).lower() == "else": if pp_stack[-1][0] < 0: pp_stack[-1][0] = i + 1 exc_start = True @@ -2171,7 +2180,7 @@ def replace_vars(line: str): pp_skips.append(pp_stack.pop()) pp_stack.append([-1, -1]) inc_start = True - elif match.group(1) == "endif": + elif match.group(2).lower() == "endif": if pp_stack_group and (pp_stack_group[-1][0] == len(pp_stack)): pp_stack_group.pop() if pp_stack[-1][0] < 0: @@ -2192,10 +2201,12 @@ def replace_vars(line: str): continue # Handle variable/macro definitions files match = FRegex.PP_DEF.match(line) - if (match is not None) and ((len(pp_stack) == 0) or (pp_stack[-1][0] < 0)): + if (match is not None and check_pp_prefix(match.group(1))) and ( + (len(pp_stack) == 0) or (pp_stack[-1][0] < 0) + ): output_file.append(line) pp_defines.append(i + 1) - def_name = match.group(2) + def_name = match.group(3) # If this is an argument list of a function add them to the name # get_definition will only return the function name upon hover # hence if the argument list is appended in the def_name then @@ -2204,18 +2215,29 @@ def replace_vars(line: str): # This also does not allow for multiline argument list definitions. # if match.group(3): # def_name += match.group(3) - if (match.group(1) == "define") and (def_name not in defs_tmp): + if (match.group(2) == "define") and (def_name not in defs_tmp): eq_ind = line[match.end(0) :].find(" ") + if eq_ind < 0: + eq_ind = line[match.end(0) :].find("\t") + if eq_ind >= 0: # Handle multiline macros if line.rstrip()[-1] == "\\": - defs_tmp[def_name] = line[match.end(0) + eq_ind : -1].strip() + def_value = line[match.end(0) + eq_ind : -1].strip() def_cont_name = def_name else: - defs_tmp[def_name] = line[match.end(0) + eq_ind :].strip() + def_value = line[match.end(0) + eq_ind :].strip() else: - defs_tmp[def_name] = "True" - elif (match.group(1) == "undef") and (def_name in defs_tmp): + def_value = "True" + + # are there arguments to parse? + if match.group(4): + def_value = (match.group(5), def_value) + + defs_tmp[def_name] = def_value + elif ( + match.group(2) == "undef" + ) and (def_name in defs_tmp): defs_tmp.pop(def_name, None) log.debug(f"{line.strip()} !!! Define statement({i + 1})") continue @@ -2265,8 +2287,16 @@ def replace_vars(line: str): continue def_regex = def_regexes.get(def_tmp) if def_regex is None: - def_regex = re.compile(rf"\b{def_tmp}\b") + if isinstance(value, tuple): + def_regex = expand_def_func_macro(def_tmp, value) + else: + def_regex = re.compile(rf"\b{def_tmp}\b") + def_regexes[def_tmp] = def_regex + + if isinstance(def_regex, tuple): + def_regex, value = def_regex + line_new, nsubs = def_regex.subn(value, line) if nsubs > 0: log.debug( @@ -2275,3 +2305,33 @@ def replace_vars(line: str): line = line_new output_file.append(line) return output_file, pp_skips, pp_defines, defs_tmp + + +def expand_def_func_macro(def_name: str, def_value: tuple[str, str]): + def_args, sub = def_value + def_args = def_args.split(",") + regex = re.compile(rf"\b{def_name}\s*\({','.join(['(.*)']*len(def_args))}\)") + + for i, arg in enumerate(def_args): + arg = arg.strip() + sub = re.sub(rf"\b({arg})\b", rf"\\{i + 1}", sub) + + return regex, sub + + +def append_multiline_macro(pp_defs: dict, def_name: str, line: str): + def_value = pp_defs[def_name] + def_args = None + if isinstance(def_value, tuple): + def_args, def_value = def_value + + def_value += line + + if def_args is not None: + def_value = (def_args, def_value) + + pp_defs[def_name] = def_value + + +def check_pp_prefix(prefix: str): + return prefix == "#" diff --git a/fortls/regex_patterns.py b/fortls/regex_patterns.py index 6f590402..d22fd0e9 100644 --- a/fortls/regex_patterns.py +++ b/fortls/regex_patterns.py @@ -124,11 +124,18 @@ class FortranRegularExpressions: FREE_FORMAT_TEST: Pattern = compile(r"[ ]{1,4}[a-z]", I) # Preprocessor matching rules DEFINED: Pattern = compile(r"defined[ ]*\(?[ ]*([a-z_]\w*)[ ]*\)?", I) - PP_REGEX: Pattern = compile(r"#(if |ifdef|ifndef|else|elif|endif)") - PP_DEF: Pattern = compile(r"#(define|undef)[ ]*([\w]+)(\((\w+(,[ ]*)?)+\))?", I) + PP_REGEX: Pattern = compile( + r"[ ]*(#)[ ]*(if |ifdef|ifndef|else|elif|endif)", + I, + ) + PP_DEF: Pattern = compile( + r"[ ]*(#)[ ]*(define|undef|undefined)" + r"[ ]+(\w+)(\([ ]*([ \w,]*?)[ ]*\))?", + I, + ) PP_DEF_TEST: Pattern = compile(r"(![ ]*)?defined[ ]*\([ ]*(\w*)[ ]*\)$", I) - PP_INCLUDE: Pattern = compile(r"#include[ ]*([\"\w\.]*)", I) - PP_ANY: Pattern = compile(r"(^#:?\w+)") + PP_INCLUDE: Pattern = compile(r"[ ]*#[ ]*include[ ]+([\"\w\.]*)", I) + PP_ANY: Pattern = compile(r"(^[ ]*(?:#)[ ]*\w*:?\w+)") # Context matching rules CALL: Pattern = compile(r"[ ]*CALL[ ]+[\w%]*$", I) INT_STMNT: Pattern = compile(r"^[ ]*[a-z]*$", I) diff --git a/test/test_preproc.py b/test/test_preproc.py index 50f50607..7d2e77b9 100644 --- a/test/test_preproc.py +++ b/test/test_preproc.py @@ -42,6 +42,10 @@ def check_return(result_array, checks): string += hover_req(file_path, 30, 23) file_path = root_dir / "preproc_if_elif_skip.F90" string += hover_req(file_path, 30, 23) + file_path = root_dir / "preproc_spacing_arg_defs.F90" + string += hover_req(file_path, 11, 20) + string += hover_req(file_path, 20, 17) + string += hover_req(file_path, 22, 13) config = str(root_dir / ".pp_conf.json") errcode, results = run_request(string, ["--config", config]) assert errcode == 0 @@ -52,12 +56,12 @@ def check_return(result_array, checks): "```fortran90\n#define PETSC_ERR_INT_OVERFLOW 84\n```", "```fortran90\n#define varVar 55\n```", ( - "```fortran90\n#define ewrite if (priority <= 3) write((priority)," - " format)\n```" + "```fortran90\n#define ewrite(priority, format)" + " if (priority <= 3) write((priority), format)\n```" ), ( - "```fortran90\n#define ewrite2 if (priority <= 3) write((priority)," - " format)\n```" + "```fortran90\n#define ewrite2(priority, format)" + " if (priority <= 3) write((priority), format)\n```" ), "```fortran90\n#define SUCCESS .true.\n```", "```fortran90\nREAL, CONTIGUOUS, POINTER, DIMENSION(:) :: var1\n```", @@ -68,6 +72,9 @@ def check_return(result_array, checks): "```fortran90\nINTEGER, PARAMETER :: res = 0+1+0+0\n```", "```fortran90\nINTEGER, PARAMETER :: res = 0+0+0+1\n```", "```fortran90\nINTEGER, PARAMETER :: res = 1+0+0+0\n```", + "```fortran90\n#define MAYBEWRAP(PROCEDURE) PROCEDURE\n```", + "```fortran90\nSUBROUTINE test_type_set_test()\n```", + "```fortran90\n#define MACROARGS(x, y) x + y\n```", ) assert len(ref_results) == len(results) - 1 check_return(results[1:], ref_results) diff --git a/test/test_server.py b/test/test_server.py index 639ef427..c3a35dc6 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -175,6 +175,7 @@ def test_workspace_symbols(): def check_return(result_array): # Expected objects objs = ( + ["argtest", 13, 16], ["test", 6, 7], ["test_abstract", 2, 0], ["test_associate_block", 2, 0], @@ -196,7 +197,11 @@ def check_return(result_array): ["test_str1", 13, 5], ["test_str2", 13, 5], ["test_sub", 6, 8], + ["test_type", 5, 5], + ["test_type_set_test", 6, 25], ["test_vis_mod", 2, 0], + ["the_test", 13, 15], + ["wrap_test_type_set_test", 6, 33], ) assert len(result_array) == len(objs) for i, obj in enumerate(objs): diff --git a/test/test_source/pp/.pp_conf.json b/test/test_source/pp/.pp_conf.json index 0cf75a8a..246c4b53 100644 --- a/test/test_source/pp/.pp_conf.json +++ b/test/test_source/pp/.pp_conf.json @@ -7,5 +7,5 @@ "pp_suffixes": [".h", ".F90"], "incl_suffixes": [".h"], "include_dirs": ["include"], - "pp_defs": { "HAVE_CONTIGUOUS": "" } + "pp_defs": { "HAVE_CONTIGUOUS": "" }, } diff --git a/test/test_source/pp/include/indent.h b/test/test_source/pp/include/indent.h new file mode 100644 index 00000000..f477f249 --- /dev/null +++ b/test/test_source/pp/include/indent.h @@ -0,0 +1,22 @@ +!! sample code adapted from json-fortran/json_macros.inc + + # define SPACING_TEST + # define FILE_ENCODING ,encoding='UTF-8' + +# ifdef __GFORTRAN__ +! gfortran uses cpp in old-school compatibility mode so +! the # stringify and ## concatenate operators don't work +! but we can use C/C++ style comment to ensure PROCEDURE is +! correctly tokenized and prepended with 'wrap_' when the +! macro is expanded +# define MAYBEWRAP(PROCEDURE) PROCEDURE , wrap_/**/PROCEDURE +# else +! Intel's fpp does support the more contemporary ## concatenation +! operator, but doesn't treat the C/C++ comments the same way. +! If you use the gfortran approach and pass the -noB switch to +! fpp, the macro will expand, but with a space between wrap_ and +! whatever PROCEDURE expands to +# define MAYBEWRAP(PROCEDURE) PROCEDURE +# endif + +# define MACROARGS( x , y ) x + y diff --git a/test/test_source/pp/preproc_spacing_arg_defs.F90 b/test/test_source/pp/preproc_spacing_arg_defs.F90 new file mode 100644 index 00000000..0ecc0f02 --- /dev/null +++ b/test/test_source/pp/preproc_spacing_arg_defs.F90 @@ -0,0 +1,42 @@ +program preprocessor_spacing_arg_defs + implicit none + + #include "indent.h" + + type :: test_type + private + integer, public :: test_int + + contains + generic, public :: set_test => MAYBEWRAP(test_type_set_test) + procedure :: MAYBEWRAP(test_type_set_test) + + end type test_type + + type(test_type) :: the_test + integer :: argtest + + INTEGER (KIND=4), PARAMETER :: C_LONG = 4 + + call the_test%set_test() + + argtest = MACROARGS(the_test%test_int, C_LONG) + +contains + subroutine test_type_set_test(me) + implicit none + + class(test_type), intent(inout) :: me + + me%test_int = 3 + end subroutine test_type_set_test + + subroutine wrap_test_type_set_test(me) + implicit none + + class(test_type), intent(inout) :: me + + me%test_int = 5 + end subroutine wrap_test_type_set_test + +end program preprocessor_spacing_arg_defs From 5cf36e35235f1d40336dfe01a6ae908e7b052cde Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 21 Jan 2024 03:26:31 +0000 Subject: [PATCH 21/22] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- fortls/parsers/internal/parser.py | 4 +--- fortls/regex_patterns.py | 3 +-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/fortls/parsers/internal/parser.py b/fortls/parsers/internal/parser.py index 14052a7a..9e09840f 100644 --- a/fortls/parsers/internal/parser.py +++ b/fortls/parsers/internal/parser.py @@ -2235,9 +2235,7 @@ def replace_vars(line: str): def_value = (match.group(5), def_value) defs_tmp[def_name] = def_value - elif ( - match.group(2) == "undef" - ) and (def_name in defs_tmp): + elif (match.group(2) == "undef") and (def_name in defs_tmp): defs_tmp.pop(def_name, None) log.debug(f"{line.strip()} !!! Define statement({i + 1})") continue diff --git a/fortls/regex_patterns.py b/fortls/regex_patterns.py index d22fd0e9..2c3cbee5 100644 --- a/fortls/regex_patterns.py +++ b/fortls/regex_patterns.py @@ -129,8 +129,7 @@ class FortranRegularExpressions: I, ) PP_DEF: Pattern = compile( - r"[ ]*(#)[ ]*(define|undef|undefined)" - r"[ ]+(\w+)(\([ ]*([ \w,]*?)[ ]*\))?", + r"[ ]*(#)[ ]*(define|undef|undefined)" r"[ ]+(\w+)(\([ ]*([ \w,]*?)[ ]*\))?", I, ) PP_DEF_TEST: Pattern = compile(r"(![ ]*)?defined[ ]*\([ ]*(\w*)[ ]*\)$", I) From c97500626e12e7aac9b29362837fb70be662870e Mon Sep 17 00:00:00 2001 From: Randy Eckman Date: Sat, 20 Jan 2024 21:50:15 -0600 Subject: [PATCH 22/22] rebase intel-specific changes on top of PR #350 --- fortls/__init__.py | 1 + fortls/langserver.py | 4 +- fortls/parsers/internal/parser.py | 40 +++++++++++++++++-- fortls/regex_patterns.py | 9 +++-- test/test_preproc.py | 6 ++- test/test_server.py | 4 +- test/test_source/pp/.pp_conf.json | 1 + .../pp/preproc_spacing_arg_defs.F90 | 6 ++- 8 files changed, 59 insertions(+), 12 deletions(-) diff --git a/fortls/__init__.py b/fortls/__init__.py index cfdba5bc..ddedcd19 100644 --- a/fortls/__init__.py +++ b/fortls/__init__.py @@ -507,6 +507,7 @@ def locate_config(root: str) -> str | None: debug=True, pp_defs=pp_defs, include_dirs=include_dirs, + pp_parse_intel=pp_parse_intel, ) print("\n=========\nObject Tree\n=========\n") for obj in file_ast.get_scopes(): diff --git a/fortls/langserver.py b/fortls/langserver.py index 92fb887f..d41b4017 100644 --- a/fortls/langserver.py +++ b/fortls/langserver.py @@ -1324,7 +1324,7 @@ def serve_onChange(self, request: dict): # tmp_file.ast.resolve_links(self.obj_tree, self.link_version) elif file_obj.preproc: file_obj.preprocess( - pp_defs=self.pp_defs, + pp_defs=self.pp_defs, pp_parse_intel=self.pp_parse_intel ) self.pp_defs = {**self.pp_defs, **file_obj.pp_defs} @@ -1400,6 +1400,7 @@ def update_workspace_file( ast_new = file_obj.parse( pp_defs=self.pp_defs, include_dirs=self.include_dirs, + pp_parse_intel=self.pp_parse_intel, ) # Add the included read in pp_defs from to the ones specified in the # configuration file @@ -1469,6 +1470,7 @@ def file_init( file_ast = file_obj.parse( pp_defs=pp_defs, include_dirs=include_dirs, + pp_parse_intel=pp_parse_intel, ) except: log.error("Error while parsing file %s", filepath, exc_info=True) diff --git a/fortls/parsers/internal/parser.py b/fortls/parsers/internal/parser.py index 3a25518f..0889bad7 100644 --- a/fortls/parsers/internal/parser.py +++ b/fortls/parsers/internal/parser.py @@ -1180,6 +1180,7 @@ def preprocess( pp_defs: dict = None, include_dirs: set = None, debug: bool = False, + pp_parse_intel: bool = False, ) -> tuple[list, list]: if pp_defs is None: pp_defs = {} @@ -1275,6 +1276,7 @@ def parse( pp_defs=pp_defs, include_dirs=include_dirs, debug=debug, + pp_parse_intel=pp_parse_intel, ) for pp_reg in pp_skips: file_ast.start_ppif(pp_reg[0]) @@ -2051,6 +2053,35 @@ def replace_ops(expr: str): return expr + def replace_intel_ops(expr: str): + expr = expr.replace("/=", " != ") + expr = expr.replace(".AND.", " && ") + expr = expr.replace(".LT.", " < ") + expr = expr.replace(".GT.", " > ") + expr = expr.replace(".EQ.", " == ") + expr = expr.replace(".LE.", " <= ") + expr = expr.replace(".GE.", " >= ") + expr = expr.replace(".NE.", " != ") + expr = expr.replace(".EQV.", " == ") + expr = expr.replace(".NEQV.", " != ") + expr = expr.replace(".NOT.", "!") + expr = expr.replace(".OR.", " || ") + expr = expr.replace(".XOR.", " != ") # admittedly a hack... + expr = expr.replace(".and.", " && ") + expr = expr.replace(".lt.", " < ") + expr = expr.replace(".gt.", " > ") + expr = expr.replace(".eq.", " == ") + expr = expr.replace(".le.", " <= ") + expr = expr.replace(".ge.", " >= ") + expr = expr.replace(".ne.", " != ") + expr = expr.replace(".eqv.", " == ") + expr = expr.replace(".neqv.", " != ") + expr = expr.replace(".not.", "!") + expr = expr.replace(".or.", " || ") + expr = expr.replace(".xor.", " != ") # admittedly a hack... + + return expr + def replace_defined(line: str): i0 = 0 out_line = "" @@ -2083,6 +2114,9 @@ def replace_vars(line: str): defs = {} out_line = text + if pp_parse_intel: + out_line = replace_intel_ops(out_line) + out_line = replace_defined(out_line) out_line = replace_vars(out_line) try: @@ -2119,13 +2153,13 @@ def replace_vars(line: str): continue # Handle conditional statements match = FRegex.PP_REGEX.match(line) - if match and check_pp_prefix(match.group(1)): + if match and check_pp_prefix(match.group(1), pp_parse_intel): output_file.append(line) def_name = None if_start = False # Opening conditional statements if match.group(2).lower() == "if ": - is_path = eval_pp_if(line[match.end(2) :], defs_tmp) + is_path = eval_pp_if(line[match.end(2) :], defs_tmp, pp_parse_intel) if_start = True elif match.group(2).lower() == "ifdef": if_start = True @@ -2161,7 +2195,7 @@ def replace_vars(line: str): exc_continue = True if pp_stack[-1][0] < 0: pp_stack[-1][0] = i + 1 - elif eval_pp_if(line[match.end(2) :], defs_tmp): + elif eval_pp_if(line[match.end(2) :], defs_tmp, pp_parse_intel): pp_stack[-1][1] = i + 1 pp_skips.append(pp_stack.pop()) pp_stack_group[-1][1] = True diff --git a/fortls/regex_patterns.py b/fortls/regex_patterns.py index 2c3cbee5..64b4f269 100644 --- a/fortls/regex_patterns.py +++ b/fortls/regex_patterns.py @@ -124,17 +124,20 @@ class FortranRegularExpressions: FREE_FORMAT_TEST: Pattern = compile(r"[ ]{1,4}[a-z]", I) # Preprocessor matching rules DEFINED: Pattern = compile(r"defined[ ]*\(?[ ]*([a-z_]\w*)[ ]*\)?", I) + INTEL_FPP_PRE_STR = r"[c!*]DEC\$|[c!*]DIR\$|!MS\$" + INTEL_FPP_PRE: Pattern = compile(f"{INTEL_FPP_PRE_STR}", I) PP_REGEX: Pattern = compile( - r"[ ]*(#)[ ]*(if |ifdef|ifndef|else|elif|endif)", + rf"[ ]*(#|{INTEL_FPP_PRE_STR})[ ]*(if |ifdef|ifndef|else|elif|endif)", I, ) PP_DEF: Pattern = compile( - r"[ ]*(#)[ ]*(define|undef|undefined)" r"[ ]+(\w+)(\([ ]*([ \w,]*?)[ ]*\))?", + rf"[ ]*(#|{INTEL_FPP_PRE_STR})[ ]*(define|undef|undefined)" + r"[ ]+(\w+)(\([ ]*([ \w,]*?)[ ]*\))?", I, ) PP_DEF_TEST: Pattern = compile(r"(![ ]*)?defined[ ]*\([ ]*(\w*)[ ]*\)$", I) PP_INCLUDE: Pattern = compile(r"[ ]*#[ ]*include[ ]+([\"\w\.]*)", I) - PP_ANY: Pattern = compile(r"(^[ ]*(?:#)[ ]*\w*:?\w+)") + PP_ANY: Pattern = compile(rf"(^[ ]*(?:#|{INTEL_FPP_PRE_STR})[ ]*\w*:?\w+)") # Context matching rules CALL: Pattern = compile(r"[ ]*CALL[ ]+[\w%]*$", I) INT_STMNT: Pattern = compile(r"^[ ]*[a-z]*$", I) diff --git a/test/test_preproc.py b/test/test_preproc.py index 9bc40837..c8fc3214 100644 --- a/test/test_preproc.py +++ b/test/test_preproc.py @@ -44,8 +44,9 @@ def check_return(result_array, checks): string += hover_req(file_path, 30, 23) file_path = root_dir / "preproc_spacing_arg_defs.F90" string += hover_req(file_path, 11, 20) - string += hover_req(file_path, 20, 17) - string += hover_req(file_path, 22, 13) + string += hover_req(file_path, 24, 17) + string += hover_req(file_path, 26, 13) + string += hover_req(file_path, 26, 42) config = str(root_dir / ".pp_conf.json") errcode, results = run_request(string, ["--config", config]) assert errcode == 0 @@ -75,6 +76,7 @@ def check_return(result_array, checks): "```fortran90\n#define MAYBEWRAP(PROCEDURE) PROCEDURE\n```", "```fortran90\nSUBROUTINE test_type_set_test()\n```", "```fortran90\n#define MACROARGS(x, y) x + y\n```", + "```fortran90\nINTEGER(KIND=4), PARAMETER :: C_LONG = 4\n```", ) assert len(ref_results) == len(results) - 1 check_return(results[1:], ref_results) diff --git a/test/test_server.py b/test/test_server.py index c3a35dc6..fb63377d 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -198,10 +198,10 @@ def check_return(result_array): ["test_str2", 13, 5], ["test_sub", 6, 8], ["test_type", 5, 5], - ["test_type_set_test", 6, 25], + ["test_type_set_test", 6, 29], ["test_vis_mod", 2, 0], ["the_test", 13, 15], - ["wrap_test_type_set_test", 6, 33], + ["wrap_test_type_set_test", 6, 37], ) assert len(result_array) == len(objs) for i, obj in enumerate(objs): diff --git a/test/test_source/pp/.pp_conf.json b/test/test_source/pp/.pp_conf.json index 246c4b53..ac1bd691 100644 --- a/test/test_source/pp/.pp_conf.json +++ b/test/test_source/pp/.pp_conf.json @@ -8,4 +8,5 @@ "incl_suffixes": [".h"], "include_dirs": ["include"], "pp_defs": { "HAVE_CONTIGUOUS": "" }, + "pp_parse_intel": true } diff --git a/test/test_source/pp/preproc_spacing_arg_defs.F90 b/test/test_source/pp/preproc_spacing_arg_defs.F90 index 0ecc0f02..ced397ab 100644 --- a/test/test_source/pp/preproc_spacing_arg_defs.F90 +++ b/test/test_source/pp/preproc_spacing_arg_defs.F90 @@ -16,7 +16,11 @@ program preprocessor_spacing_arg_defs type(test_type) :: the_test integer :: argtest - INTEGER (KIND=4), PARAMETER :: C_LONG = 4 + !DEC$ IF DEFINED(SPACING_TEST).AND.DEFINED(MACROARGS) + INTEGER (KIND=4), PARAMETER :: C_LONG = 4 + !DEC$ ELSE + INTEGER (KIND=4), PARAMETER :: C_LONG = 8 + !DEC$ ENDIF call the_test%set_test()