ENH : improvement csv processing

rocketpy/mathutils/function.py

@@ -10,7 +10,9 @@
 
 import matplotlib.pyplot as plt
 import numpy as np
+import csv
 from scipy import integrate, linalg, optimize
+from ..tools import data_preprocessing
 
 try:
     from functools import cached_property
@@ -206,10 +208,12 @@ def set_source(self, source):
         if isinstance(source, (str, Path)):
             with open(source, "r") as file:
                 try:
-                    source = np.loadtxt(file, delimiter=",", dtype=float)
+                    source = np.loadtxt(file, delimiter=",")
                 except ValueError:
                     # If an error occurs, headers are present
-                    source = np.loadtxt(source, delimiter=",", dtype=float, skiprows=1)
+                    source = np.loadtxt(
+                        data_preprocessing(source), delimiter=",", dtype=np.float64
+                    )
                 except Exception as e:
                     raise ValueError(
                         "The source file is not a valid csv or txt file."
@@ -2911,10 +2915,12 @@ def _check_user_input(
             if isinstance(source, (str, Path)):
                 # Convert to numpy array
                 try:
-                    source = np.loadtxt(source, delimiter=",", dtype=float)
+                    source = np.loadtxt(source, delimiter=",", dtype=np.float64)
                 except ValueError:
-                    # Skip header
-                    source = np.loadtxt(source, delimiter=",", dtype=float, skiprows=1)
+                    # If an error occurs, there is a header
+                    source = np.loadtxt(
+                        data_preprocessing(source), delimiter=",", dtype=np.float64
+                    )
                 except Exception as e:
                     raise ValueError(
                         "The source file is not a valid csv or txt file."

rocketpy/tools.py

@@ -1,6 +1,7 @@
 import importlib
 import importlib.metadata
 import re
+import csv
 from bisect import bisect_left
 
 import pytz
@@ -381,6 +382,105 @@ def check_requirement_version(module_name, version):
     return True
 
 
+def is_float(element):
+    """
+    Returns a boolean indicating us if an element is convertible to a float or not.
+    True : the element is convertible to a float
+    False : the element is not convertible to a float
+
+    Parameters
+    ----------
+    element : any
+        This is the element to test.
+
+    Returns
+    -------
+    result : boolean
+        The element is convertible or not.
+    """
+    try:
+        float(element)
+        return True
+    except (ValueError, TypeError):
+        return False
+
+
+def return_first_data(source):
+    """
+    Returns the first data of a CSV file.
+
+    Parameters
+    ----------
+    source : string
+        This is the file path to the csv.
+
+    Returns
+    -------
+    result : any
+        The first data of the CSV file.
+    """
+    with open(source, "r") as native_data:
+        for row in native_data:
+            for value in row:
+                return value
+
+
+def if_header(source):
+    """
+    Returns if a CSV file has a header or not.
+    True : The CSV file has a header
+    False : The CSV file has no header
+
+    Parameters
+    ----------
+    source : string
+        This is the file path to the csv.
+
+    Returns
+    -------
+    result : boolean
+        The result of the CSV file containing a header or not.
+    """
+    return not is_float(return_first_data(source))
+
+
+def data_preprocessing(source):
+    """Clear data (in particular NaN objects) and returns a CSV file without header and its name.
+
+    Parameters
+    ----------
+    source : string
+        The file path to the CSV file.
+
+    Returns
+    -------
+    Function
+        The function with the incoming cleared CSV
+    """
+    output_path = "cleaned_data.csv"
+
+    with open(source, "r") as file:
+        reader = csv.reader(file)
+        header = next(reader)  # Read the header
+
+        data = [row for row in reader]
+
+    # Create a new list without the headers
+    data_no_headers = []
+
+    for row in data[1:]:
+        # Check if the row is not empty and if all values in the row can be converted to float
+        if row and all(is_float(value) for value in row):
+            data_no_headers.append(row)
+
+    # Save the processed data to a new CSV file
+    with open(output_path, "w", encoding="utf-8") as output_file:
+        writer = csv.writer(output_file, delimiter=",")
+        writer.writerows(data_no_headers)
+
+    return output_path
+
+
 if __name__ == "__main__":
     import doctest