322 Notebook Executables

DESCRIPTION

@@ -2,7 +2,7 @@ Package: kaiaulu
 Type: Package
 Title: Kaiaulu
 Version: 0.0.0.9700
-Description: Kaiaulu is an R package and common interface that helps with understanding evolving software development communities, and the artifacts (gitlog, mailing list, files, etc.) which developers collaborate and communicate about. See Paradis et al., (2012) <doi:10.1007/978-3-031-15116-3_6>. 
+Description: Kaiaulu is an R package and common interface that helps with understanding evolving software development communities, and the artifacts (gitlog, mailing list, files, etc.) which developers collaborate and communicate about. See Paradis et al., (2012) <doi:10.1007/978-3-031-15116-3_6>.
 Authors@R: c(
     person('Carlos', 'Paradis', role = c('aut', 'cre'),
       email = '[email protected]',
@@ -21,6 +21,7 @@ Authors@R: c(
     person('Anthony', 'Lau', role = c('ctb')),
     person('Sean', 'Sunoo', role = c('ctb')),
     person('Ian Jaymes', 'Iwata', role= c('ctb')),
+    person('Raven', 'Quiddaoen', role= c('ctb')),
     person('Nicholas', 'Beydler', role = c('ctb')),
     person('Mark', 'Burgess', role = c('ctb'))
     )

NAMESPACE

@@ -3,6 +3,7 @@
 export(annotate_src_text)
 export(assign_exact_identity)
 export(bipartite_graph_projection)
+export(build_understand_project)
 export(commit_message_id_coverage)
 export(community_oslom)
 export(convert_pipermail_to_mbox)
@@ -189,6 +190,7 @@ export(parse_r_dependencies)
 export(parse_r_function_definition)
 export(parse_r_function_dependencies)
 export(parse_rfile_ast)
+export(parse_understand_dependencies)
 export(query_src_text)
 export(query_src_text_class_names)
 export(query_src_text_namespace)
@@ -214,6 +216,7 @@ export(transform_gitlog_to_temporal_network)
 export(transform_r_dependencies_to_network)
 export(transform_reply_to_bipartite_network)
 export(transform_temporal_gitlog_to_adsmj)
+export(transform_understand_dependencies_to_network)
 export(weight_scheme_count_deleted_nodes)
 export(weight_scheme_cum_temporal)
 export(weight_scheme_pairwise_cum_temporal)

NEWS.md

@@ -3,6 +3,7 @@ __kaiaulu 0.0.0.9700 (in development)__
 
 ### NEW FEATURES
 
+ * `build_understand_project (project_path , language, output_dir)`, `parse_understand_dependencies(output_dir, parse_type)`, and `transform_understand_dependencies_to_network(parsed, weights)` have been added. These functions handle creating tables from xml data generated from Scitool's Understand. [#308](https://github.com/sailuh/kaiaulu/issues/308)
  * The GitHUB API has been expanded to use refresh, along with other functions. `github_api_project_issue_search` has been added that makes the search/issues endpoint API calls. `github_api_project_issue_or_pr_comments_by_date` and `github_api_project_issue_by_date` have been added to download issue data and comments by date ranges. `github_parse_search_issues_refresh` has been added that parses the issue data downloaded from the search endpoint in the refresh_issues folder. `github_api_project_issue_refresh` and `github_api_project_issue_or_pr_comment_refresh` were added to download issue data or comments respectively that have not already been downloaded. `format_created_at_from_file` was added to retrieve the greatest date from a JSON file. See the Reference Docs on GitHub section for more details. [#282](https://github.com/sailuh/kaiaulu/issues/282)
  * `config.R` now contains a set of getter functions used to centralize the gathering of configuration data and these getter functions are used to refactor configuration file information gathering. For example, loading configuration file information with variable assignment is as follows `git_repo_path <- config_file[["version_control"]][["log"]]` but refactoring with a config.R getter function becomes `git_repo_path <- get_git_repo_path(config_file)`.  [#230](https://github.com/sailuh/kaiaulu/issues/230)
  * `refresh_jira_issues()` had been added. It is a wrapper function for the previous downloader and downloads only issues greater than the greatest key already downloaded. [#275](https://github.com/sailuh/kaiaulu/issues/275)

R/src.R

@@ -4,8 +4,141 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
+############## Understand Project Builder ##############
+#' Build Understand Analysis Folder
+#'
+#' @description This function builds the data files for Understand from the project_path folder, reading from files that are written in the target language into output_dir
+#'
+#' @param project_path path to the project folder to analyze
+#' @param language the primary language of the project (language must be supported by Understand)
+#' @param output_dir path to output directory (formatted output_path/)
+#' @export
+#' @family parsers
+build_understand_project <- function(project_path, language, output_dir){
+  # Create variables for command line
+  command <- "und"
+  project_path <- shQuote(project_path) # Quoting the project path
+  db_dir <- file.path(output_dir, "Understand.und")
+  args <- c("create", "-db", db_dir, "-languages", language)
+
+  # Build the Understand project by parsing through using Understand's und command
+  # Derived from pg. 352 in https://documentation.scitools.com/pdf/understand.pdf Sept. 2024 Edition
+  system2(command, args)
+  args <- c("-db", db_dir, "add", project_path)
+  system2(command, args)
+  args <- c("analyze", db_dir)
+  system2(command, args)
+
+}
+
 ############## Parsers ##############
+#' Parse Built Folder to Network
+#'
+#' @description This function parses the data in the Understand build folder to export the parse_type dependencies into a network
+#'
+#' @param understand_dir path to the built Understand project folder used in \code{\link{build_understand_project}}
+#' @param parse_type Type of dependencies to generate into xml (either "file" or "class")
+#' @export
+#' @family parsers
+parse_understand_dependencies <- function(understand_dir, parse_type = c("file", "class")) {
+  # Before running, check if parse_type is correct
+  parse_type <- match.arg(parse_type)
+
+  # Create the variables used in command lines
+  db_dir <- file.path(understand_dir, "Understand.und")
+  file_name <- paste0(parse_type, "Dependencies.xml")
+  xml_dir <- file.path(db_dir, file_name)
+
+  # Generate the XML file
+  # Derived from pg. 352 in https://documentation.scitools.com/pdf/understand.pdf Sept. 2024 Edition
+  args <- c("export", "-dependencies", parse_type, "cytoscape", xml_dir, db_dir)
+  system2("und", args)
+
+  # Generated XML file is assumed to be in this approximate format (regardless of parse_type) using Understand Build 1202
+  # <graph ...>
+  #   ... [Irrelevant graph attributes and rdf grandchildren]
+  #   <node id="67" label="ObjectMapper id:67">
+  #     <att type="string" name="node.shape" value="rect"/>
+  #     <att type="string" name="node.fontSize" value="5"/>
+  #     <att type="string" name="node.label" value="ObjectMapper"/>
+  #     <att type="string" name="longName" value="com.fasterxml.jackson.databind.ObjectMapper"/>
+  #     <att type="string" name="kind" value="Unknown Class"/>
+  #     <graphics type="RECTANGLE" h="35" w="35" x="0" y="0" fill="#ffffff" width="1" outline="#000000" cy:nodeTransparency="1.0" cy:nodeLabelFont="Default-0-8" cy:borderLineType="solid"/>
+  #   </node>
+  #   ... [Other nodes sharing the format]
+  #   <edge source="2" target="9" label="App(Depends On)CalculatorUI">
+  #     <att type="string" name="edge.targetArrowShape" value="ARROW"/>
+  #     <att type="string" name="edge.color" value="#0000FF"/>
+  #     <att type="string" name="canonicalName" value="App(Depends On)CalculatorUI"/>
+  #     <att type="string" name="interaction" value="Depends On"/>
+  #     <att type="string" name="dependency kind" value="Call, Create"/>
+  #   </edge>
+  #   ... [Other edges sharing the format]
+
+
+  # Parse the XML file
+  xml_data <- xmlParse(xml_dir)  # Creates pointer to file
+  xml_nodes <- xmlRoot(xml_data)  # Finds the head: graph
+  xml_nodes <- xmlChildren(xml_nodes)
+  # xml_nodes now contains the nodes and edges (which were children of graph) and also graph's atts
+
+  # From child nodes- filter for those with name "node"
+  # Create a list by iterating through all the children in xml_nodes
+  node_elements <- lapply(xml_nodes, function(child) {
+    if (xmlName(child) == "node") {  # We're searching for nodes, not att or edges
+      id <- xmlGetAttr(child, "id")  # Extract the id from the node line
+      att_nodes <- xmlChildren(child)  # To access the atts of the node
+      node_label <- xmlGetAttr(att_nodes[[3]], "value")  # Relevant att is the 3rd line
+      long_name <- xmlGetAttr(att_nodes[[4]], "value")  # Relevant att is the 4th line
+      return(data.table(node_label = node_label, id = id, long_name = long_name))  # Returns the table containing the filtered node data
+    } else {
+      return(NULL) # Return NULL for the entry to be filtered out later
+    }
+  })
+
+  # Remove NULLs and combine the results from the node_elements list
+  node_list <- rbindlist(node_elements[!sapply(node_elements, is.null)], use.names = TRUE, fill = TRUE)
+
+  # From child nodes- filter for those with name "edge"
+  # Create a list by iterating through all the children in xml_nodes
+  edge_elements <- lapply(xml_nodes, function(child) {
+    if (xmlName(child) == "edge") {  # We're searching for edges, not att or nodes
+      # Extract the id_from and id_to from the edge line
+      id_from <- xmlGetAttr(child, "source")
+      id_to <- xmlGetAttr(child, "target")
+      att_nodes <- xmlChildren(child)  # To access the atts of the edge
+      dependency_kind <- xmlGetAttr(att_nodes[[5]], "value")  # Relevant att is the 5th line
+      # Error handling for empty and NULL dependency_kind (this is necessary as errors do occur even in the formatted style)
+      # Code correctly handles all the edges, however produces error if error handling is not included... so...
+      if (!is.null(dependency_kind) && dependency_kind != "") {
+        dependency_kind <- unlist(stri_split(dependency_kind, regex = ",\\s*"))  # Separates the string into a vector
+        return(data.table(id_from = id_from, id_to = id_to, dependency_kind = dependency_kind)) # Returns the table containing the filtered node data
+      } else {
+        return(NULL) # Return NULL for the entry to be filtered out later
+      }
+    } else {
+      return(NULL) # Return NULL for the entry to be filtered out later
+    }
+  })
+
+  # Remove NULLs and combine the results from the edge_elements list
+  edge_list <- rbindlist(edge_elements[!sapply(edge_elements, is.null)], use.names = TRUE, fill = TRUE)
+
+  # Merge edges with nodes to get label_from
+  edge_list <- merge(edge_list, node_list[, .(id, node_label)], by.x = "id_from", by.y = "id", all.x = TRUE)
+  setnames(edge_list, "node_label", "label_from")
 
+  # Merge again to get label_to
+  edge_list <- merge(edge_list, node_list[, .(id, node_label)], by.x = "id_to", by.y = "id", all.x = TRUE)
+  setnames(edge_list, "node_label", "label_to")
+
+  # Reorder columns to have label_from and label_to on the left
+  edge_list <- edge_list[, .(label_from, label_to, id_from, id_to, dependency_kind)]
+
+  # Create a list of the network to return
+  graph <- list(node_list = node_list, edge_list = edge_list)
+  return(graph)
+}
 
 #' Parse dependencies from Depends
 #'
@@ -214,6 +347,33 @@ parse_r_dependencies <- function(folder_path){
 }
 
 ############## Network Transform ##############
+#' Transform Understand Dependencies
+#'
+#' @description This function subsets a parsed table from parse_understand_dependencies
+#'
+#' @param parsed Parsed table from \code{\link{parse_understand_dependencies}}
+#' @param weight_types The weight types as defined in Depends. Accepts single string and vector input
+#' @export
+#' @family edgelists
+transform_understand_dependencies_to_network <- function(parsed, weight_types) {
+
+  nodes <- parsed[["node_list"]]
+  edges <- parsed[["edge_list"]]
+
+  # Filter out by weights if vector provided
+  if (length(weight_types) > 0) {
+    edges <- edges[dependency_kind %in% weight_types]
+  }
+
+  # If filter removed all edges:
+  if (nrow(edges) == 0) {
+    stop("Error: No edges found under weight_types.")
+  }
+
+  # Create a list to return
+  graph <- list(node_list = nodes, edge_list = edges)
+  return(graph)
+}
 
 #' Transform parsed dependencies into a network
 #'

exec/depends.R

@@ -0,0 +1,71 @@
+#!/usr/local/bin/Rscript
+
+require(kaiaulu, quietly = TRUE)
+require(data.table, quietly = TRUE)
+require(stringi, quietly = TRUE)
+require(gh, quietly = TRUE)
+require(yaml, quietly = TRUE)
+require(magrittr, quietly = TRUE)
+require(knitr, quietly = TRUE)
+require(cli, quietly = TRUE)
+
+doc <- "
+USAGE:
+  depends.R (-h | -help)
+  depends.R --version
+  depends.R parse help
+  depends.R parse <config_filepath.yml> <tool_filepath.yml> <project_github_url> <save_path>
+
+DESCRIPTION:
+  Parses file dependencies using the Depends tool.
+
+OPTIONS:
+  -h --help     Show this screen.
+  --version     Show version.
+"
+
+arguments <- docopt::docopt(doc, version = 'Kaiaulu 0.0.0.9700')
+
+# Determine which function to run and save output
+if (arguments[["parse"]] & arguments[["help"]]) {
+  cli::cli_alert_info("Parses a given GitHub project supplied by <project_github_url> and  saves two tables at the specified folder <save_path> using Depends")
+} else if (arguments[["parse"]]) {
+  tool <- parse_config(arguments[["<tool_filepath.yml>"]])
+  conf <- parse_config(arguments[["<config_filepath.yml>"]])
+  project_github_url <- arguments[["<project_github_url>"]]
+  git_repo_path <- get_git_repo_path(conf)
+
+  # Depends parameters
+  depends_jar_path <- get_tool_project("depends", tool)
+  language <- get_depends_code_language(conf)
+  keep_dependencies_type <- get_depends_keep_dependencies_type(conf)
+
+  # Filters
+  file_extensions <- get_file_extensions(conf)
+  substring_filepath <- get_substring_filepath(conf)
+
+  # Output
+  save_path <- arguments[["<save_path>"]]
+  node_file <- paste0(save_path, "/node.csv")
+  edge_file <- paste0(save_path, "/edge.csv")
+
+  # Construct File Network
+  project_github_url <- stri_split_regex(project_github_url,pattern="/")[[1]]
+  owner <- project_github_url[length(project_github_url)-1]
+  repo <- project_github_url[length(project_github_url)]
+  language_distribution_byte <- unlist(gh("GET /repos/:owner/:repo/languages",owner=owner,repo=repo))
+  language_distribution_byte <- language_distribution_byte/sum(language_distribution_byte)
+  format(round(language_distribution_byte, 2), nsmall = 2)
+
+  result <- parse_dependencies(depends_jar_path,git_repo_path,language=language) # Parse Dependencies
+
+  data.table::fwrite(result$nodes, node_file)
+  data.table::fwrite(result$edgelist, edge_file)
+  cli::cli_alert_success(paste0("Dependencies table was saved at: ", save_path))
+} else if (arguments[["-h"]] || arguments[["--help"]]) {
+  cli::cli_alert_info(doc)
+} else if (arguments[["--version"]]) {
+  cli::cli_alert_info('Kaiaulu 0.0.0.9700')
+} else {
+  stop("No/invalid option(s) provided.")
+}

exec/linemetrics.R

@@ -0,0 +1,55 @@
+#!/usr/local/bin/Rscript
+
+require(kaiaulu, quietly = TRUE)
+require(data.table, quietly = TRUE)
+require(magrittr, quietly = TRUE)
+require(knitr, quietly = TRUE)
+require(cli, quietly = TRUE)
+
+doc <- "
+USAGE:
+  linemetrics.R (-h | -help)
+  linemetrics.R --version
+  linemetrics.R analyze help
+  linemetrics.R analyze <config_filepath.yml> <tool_filepath.yml> <save_path>
+
+DESCRIPTION:
+  Analyze line metrics using the SCC tool.
+
+OPTIONS:
+  -h --help     Show this screen.
+  --version     Show version.
+"
+
+arguments <- docopt::docopt(doc, version = 'Kaiaulu 0.0.0.9700')
+
+# Determine which function to run and save output
+if (arguments[["analyze"]] & arguments[["help"]]) {
+  cli::cli_alert_info("Analyzes a project specified in the <config_filepath.yml> config file and returns a csv file designated by <save_path> using SCC")
+} else if (arguments[["parse"]]) {
+  tool <- parse_config(arguments[["<tool_filepath.yml>"]])
+  conf <- parse_config(arguments[["<config_filepath.yml>"]])
+  scc_path <- get_tool_project("scc", tool)
+
+  git_repo_path <- get_git_repo_path(conf)
+  git_branch <- get_git_branches(conf)[1]
+
+  # Filters
+  file_extensions <- get_file_extensions(conf)
+  substring_filepath <- get_substring_filepath(conf)
+
+  # Output
+  save_path <- arguments[["<save_path>"]]
+
+  git_checkout(git_branch,git_repo_path)
+  result <- parse_line_metrics(scc_path,git_repo_path) # Parse Dependencies
+
+  data.table::fwrite(result, save_path)
+  cli::cli_alert_success(paste0("Line metrics table was saved at: ", save_path))
+} else if (arguments[["-h"]] || arguments[["--help"]]) {
+  cli::cli_alert_info(doc)
+} else if (arguments[["--version"]]) {
+  cli::cli_alert_info('Kaiaulu 0.0.0.9700')
+} else {
+  stop("No/invalid option(s) provided.")
+}