Reformat code and enable run with command line arguments

Wyrdix · Wyrdix · commit 65465fd3d207 · 2024-07-29T12:10:02.000+02:00
diff --git a/README.md b/README.md
@@ -0,0 +1,94 @@
+# GitXApiAnalysis
+
+This project is used to analyse git trace.
+
+To do so git project are converted to xApi files, then the program tries to classify commits.
+
+## Running
+
+### Generate files
+
+`python main.py path/to/repo -g`
+Output location can be customized with `-o`
+
+### Run analysis
+
+`python main.py path/to/xapi`
+Output location can be customized with `-o`
+
+## Output Format
+
+The output consists mainly of one file named "'repo'_processed.json" which consists of an xapi files with a list of statements.
+Each statements may be one of the original commit or generated by the program in an effort to try to split commits into atomic tasks.
+
+Classified statements have a 'task' field in the 'context' part of the statement.
+
+Full list of current tasks :
+
+- ADD_RETURN_VALUE
+- ADD_VARIABLE_VALUE
+- AnnotationAdd
+- AnnotationRemove
+- ASSIGN_VARIABLE
+- BlockAdd
+- BlockRemove
+- CHANGE_METHOD_INVOCATED
+- CHANGE_VARIABLE_USED
+- CHANGE_VARIABLE_VALUE
+- CHANGE_LITTERAL_VALUE
+- LITTERAL_TO_VARIABLE
+- VARIABLE_TO_LITTERAL
+- ClassAdd
+- ClassRemove
+- COMMENT_ADDITION
+- COMMENT_DELETION
+- COMMENT_EDITION
+- COMMENT_MOVED
+- CONDITION
+- ADD_FUNCTION_PARAMETER
+- REMOVE_FUNCTION_PARAMETER
+- EDIT_FUNCTION_PARAMETERS
+- EDIT_RETURN_VALUE
+- CHANGE_LITTERAL_VALUE
+- CutPaste
+- DeclarationStatementAdd
+- DeclarationStatementRemove
+- EmptyCommit
+- EmptyLine
+- ForAdd
+- ForRemove
+- IfAdd
+- IfRemove
+- SyntaxTypoAdd
+- SyntaxTypoRemove
+- FunctionAdd
+- FunctionRemove
+- ImportStatementAdd
+- ImportStatementRemove
+- InvocationStatementAdd
+- InvocationStatementRemove
+- LineBreakChange
+- LineSpacingChange
+- ModifyVariableValue
+- NotSource
+- PackageStatementAdd
+- PackageStatementRemove
+- RefactoringMiner
+- RENAME_VARIABLE
+- RENAME_FUNCTION
+- REPLACE_FUNCTION
+- REPLACE_METHOD
+- ReturnStatementAdd
+- ReturnStatementRemove
+- STRING_EDITION
+- MODIFY_VARIABLE_VALUE
+- TrimEditionContent
+- TRUE_TO_FALSE
+- FALSE_TO_TRUE
+- TYPO_ADD
+- TYPO_DEL
+- UNASSIGN_VARIABLE
+- USE_NEW_VARIABLE
+- USE_OLD_VARIABLE
+- VariableAssignStatementAdd
+- VariableAssignStatementRemove
diff --git a/identifier/tasks_identifier/code_task/RefactoringMinerTask.py b/identifier/tasks_identifier/code_task/RefactoringMinerTask.py
@@ -8,8 +8,8 @@
 
 class RefactoringMinerTask(CodeTaskIdentifier):
 
-    def __init__(self) -> None:
-        with open("refactoring.json") as f:
+    def __init__(self, file) -> None:
+        with open(file) as f:
             self.json = dict(
                 [
                     (c["sha1"], c["refactorings"])
diff --git a/main.py b/main.py
@@ -1,4 +1,5 @@
 import json
+import GitToXApi.utils
 from tincan import Statement
 import GitToXApi.differential
 from GitToXApi.utils import *
@@ -9,8 +10,9 @@
 import debug
 import shutil
 import os
-import pm4py
-from pm4py.objects.log.obj import EventLog, Trace, Event
+from xes_file import *
+import argparse
+import subprocess
 
 
 def format_statement(st: Statement) -> Statement:
@@ -45,9 +47,12 @@ def exec_modifier(statements: list[Statement], identifier: CodeModifier):
 
 
 def dump(
-    name: str, statements: list[Statement], filter: Callable[[Statement], bool]
+    name: str,
+    out: str,
+    statements: list[Statement],
+    filter: Callable[[Statement], bool],
 ) -> None:
-    with open("out/" + name + ".json", "w") as f:
+    with open(out + name + ".json", "w") as f:
         f.write(
             json.dumps(
                 [stmt.as_version() for stmt in statements if filter(stmt)],
@@ -56,18 +61,44 @@ def dump(
         )
 
 
-if __name__ == "__main__":
+def generate_files(repo: Repo, out_folder, repo_name: str):
+    repo_path = repo.git_dir[: repo.git_dir.rindex("/")]
+    print("Getting files for", repo_path)
+    stmts = GitToXApi.utils.generate_xapi(repo, {})
+
+    dest_xapi = out_folder + repo_name + ".json"
+    if not os.path.exists(dest_xapi):
+        print("Generation of xapi files for", repo_path)
+        with open(dest_xapi, "w") as f:
+            f.write(GitToXApi.utils.serialize_statements(stmts, indent=2))
+
+    dest_refactoring = out_folder + repo_name + "_refactoring.json"
+    if not os.path.exists(dest_refactoring):
+        print("Generation of refactoring files for", repo_path)
+        subprocess.run(
+            "RefactoringMiner -a " + repo_path + " -json " + dest_refactoring,
+            shell=True,
+            capture_output=True,
+        )
+
+    print("Files where collected for", repo_path)
+
+
+def process_file(path: str, out: str):
+
+    refactoring_file = path[: path.rfind(".")] + "_refactoring.json"
 
     initial_statements = None
-    with open("original.json") as f:
+    with open(path) as f:
         initial_statements = deserialize_statements(f)
+    initial_statements = [format_statement(s) for s in initial_statements]
     initial_total = len(initial_statements)
     statements = copy.deepcopy(initial_statements)
 
     code_modifiers = [
         PreciseVerbModifier(),
         NotSourceTask(),
-        RefactoringMinerTask(),
+        RefactoringMinerTask(refactoring_file),
         TrimEditionContentModifier(),
         LineBreakAndSpacingChangeTask(),
         CutPasteTask(),
@@ -97,9 +128,6 @@ def dump(
     for modif in code_modifiers:
         statements = exec_modifier(statements, modif)
 
-    shutil.rmtree("out", ignore_errors=True)
-    os.mkdir("./out")
-
     scores = {"UNKNOWN": 0}
 
     for st in statements:
@@ -117,16 +145,20 @@ def dump(
         safe_name: str = k.lower().replace(" ", "_")
         dump(
             safe_name,
+            out,
             statements,
             lambda x: "task" in x.context.extensions
             and x.context.extensions["task"]["id"] == k,
         )
     dump(
         "unknown",
+        out,
         statements,
         lambda x: not "task" in x.context.extensions,
     )
 
+    dump(path[: path.rfind(".")] + "_processed.json", out, statements, lambda x: True)
+
     scores = [(k, scores[k]) for k in scores]
     scores.sort(key=lambda v: -v[1])
 
@@ -136,54 +168,45 @@ def dump(
     ]
 
     if debug.GENERATE_XES_FROM_INITIAL:
-        keys = [st.object.id for st in initial_statements]
-
-        event_log = EventLog()
-        trace = Trace()
-
-        for key in keys:
-            event = Event()
-            classes = set()
-            for st in statements:
-                task = TaskIdentifier.get_task(st)
-                if task == None or not "origins" in st.context.extensions:
-                    continue
-                if not key in st.context.extensions["origins"]:
-                    continue
-                classes.add(task[0])
-            if len(classes) == 0:
-                classes.add("UNKNOWN")
-            classes = list(classes)
-            classes.sort()
-            event["concept:name"] = "/".join(classes)
-            event["org:resource"] = st.object.definition.description["en-US"]
-            event["time:timestamp"] = st.timestamp
-            trace.append(event)
-
-        event_log.append(trace)
-
-        pm4py.write_xes(event_log, "out/initial.xes")
+        generate_xes_from_initial(
+            initial_statements, statements, out=out + "initial.xes"
+        )
 
     if debug.GENERATE_XES_FROM_CREATED:
+        generate_xes_from_created(statements, out=out + "created.xes")
 
-        event_log = EventLog()
-        trace = Trace()
 
-        for st in statements:
-            event = Event()
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        prog="GitXApiAnalysis",
+        description="Program used to analyze git xapi files",
+    )
+
+    parser.add_argument("filename")
+    parser.add_argument(
+        "--generate", "-g", action=argparse.BooleanOptionalAction, default=False
+    )
+    parser.add_argument("-o", "--out", default="")
+
+    args = parser.parse_args()
 
-            clazz = "UNKNOWN"
-            task = TaskIdentifier.get_task(st)
-            if task != None:
-                clazz = task[0]
+    filename = args.filename
+    generate = args.generate
+    out_folder = args.out
 
-            if clazz == "EmptyCommit":
-                continue
+    if out_folder != "" and out_folder[-1] != "/":
+        out_folder += "/"
+    if filename[-1] == "/":
+        filename = filename[:-1]
 
-            event["concept:name"] = clazz
-            event["time:timestamp"] = st.timestamp
-            trace.append(event)
+    if out_folder != "" and not os.path.exists(out_folder):
+        os.makedirs(out_folder)
 
-        event_log.append(trace)
+    if not os.path.exists(filename):
+        raise FileNotFoundError(filename)
 
-        pm4py.write_xes(event_log, "out/artificial.xes")
+    if generate:
+        repo = Repo(filename)
+        generate_files(repo, out_folder, filename)
+    else:
+        process_file(filename, out_folder)
diff --git a/xes_file.py b/xes_file.py
@@ -0,0 +1,62 @@
+from GitToXApi import *
+from pm4py.objects.log.obj import EventLog, Trace, Event
+from pm4py import write_xes
+from identifier.tasks_identifier import TaskIdentifier
+
+
+def generate_xes_from_initial(
+    initial: list[Statement], created: list[Statement], out="out/initial.xes"
+):
+    keys = [st.object.id for st in initial]
+
+    event_log = EventLog()
+    trace = Trace()
+
+    for key in keys:
+        event = Event()
+        classes = set()
+        for st in created:
+            task = TaskIdentifier.get_task(st)
+            if task == None or not "origins" in st.context.extensions:
+                continue
+            if not key in st.context.extensions["origins"]:
+                continue
+            classes.add(task[0])
+        if len(classes) == 0:
+            classes.add("UNKNOWN")
+        classes = list(classes)
+        classes.sort()
+        event["concept:name"] = "/".join(classes)
+        event["org:resource"] = st.object.definition.description["en-US"]
+        event["time:timestamp"] = st.timestamp
+        trace.append(event)
+
+    event_log.append(trace)
+
+    write_xes(event_log, out)
+
+
+def generate_xes_from_created(
+    created: list[Statement], mask: list = ["EmptyCommit"], out="out/initial.xes"
+):
+    event_log = EventLog()
+    trace = Trace()
+
+    for st in created:
+        event = Event()
+
+        clazz = "UNKNOWN"
+        task = TaskIdentifier.get_task(st)
+        if task != None:
+            clazz = task[0]
+
+        if clazz in mask:
+            continue
+
+        event["concept:name"] = clazz
+        event["time:timestamp"] = st.timestamp
+        trace.append(event)
+
+    event_log.append(trace)
+
+    write_xes(event_log, out)

Original file line number	Diff line number	Diff line change
`@@ -8,8 +8,8 @@`
`8`	`8`
`9`	`9`	`class RefactoringMinerTask(CodeTaskIdentifier):`
`10`	`10`
`11`		`- def __init__(self) -> None:`
`12`		`- with open("refactoring.json") as f:`
	`11`	`+ def __init__(self, file) -> None:`
	`12`	`+ with open(file) as f:`
`13`	`13`	`self.json = dict(`
`14`	`14`	`[`
`15`	`15`	`(c["sha1"], c["refactorings"])`