diff --git a/rocketpy/mathutils/function.py b/rocketpy/mathutils/function.py index 251d58199..914841af3 100644 --- a/rocketpy/mathutils/function.py +++ b/rocketpy/mathutils/function.py @@ -10,7 +10,9 @@ import matplotlib.pyplot as plt import numpy as np +import csv from scipy import integrate, linalg, optimize +from ..tools import data_preprocessing try: from functools import cached_property @@ -206,10 +208,12 @@ def set_source(self, source): if isinstance(source, (str, Path)): with open(source, "r") as file: try: - source = np.loadtxt(file, delimiter=",", dtype=float) + source = np.loadtxt(file, delimiter=",") except ValueError: # If an error occurs, headers are present - source = np.loadtxt(source, delimiter=",", dtype=float, skiprows=1) + source = np.loadtxt( + data_preprocessing(source), delimiter=",", dtype=np.float64 + ) except Exception as e: raise ValueError( "The source file is not a valid csv or txt file." @@ -2911,10 +2915,12 @@ def _check_user_input( if isinstance(source, (str, Path)): # Convert to numpy array try: - source = np.loadtxt(source, delimiter=",", dtype=float) + source = np.loadtxt(source, delimiter=",", dtype=np.float64) except ValueError: - # Skip header - source = np.loadtxt(source, delimiter=",", dtype=float, skiprows=1) + # If an error occurs, there is a header + source = np.loadtxt( + data_preprocessing(source), delimiter=",", dtype=np.float64 + ) except Exception as e: raise ValueError( "The source file is not a valid csv or txt file." diff --git a/rocketpy/tools.py b/rocketpy/tools.py index 6e765ca7a..18eb0f37d 100644 --- a/rocketpy/tools.py +++ b/rocketpy/tools.py @@ -1,6 +1,7 @@ import importlib import importlib.metadata import re +import csv from bisect import bisect_left import pytz @@ -381,6 +382,105 @@ def check_requirement_version(module_name, version): return True +def is_float(element): + """ + Returns a boolean indicating us if an element is convertible to a float or not. + True : the element is convertible to a float + False : the element is not convertible to a float + + Parameters + ---------- + element : any + This is the element to test. + + Returns + ------- + result : boolean + The element is convertible or not. + """ + try: + float(element) + return True + except (ValueError, TypeError): + return False + + +def return_first_data(source): + """ + Returns the first data of a CSV file. + + Parameters + ---------- + source : string + This is the file path to the csv. + + Returns + ------- + result : any + The first data of the CSV file. + """ + with open(source, "r") as native_data: + for row in native_data: + for value in row: + return value + + +def if_header(source): + """ + Returns if a CSV file has a header or not. + True : The CSV file has a header + False : The CSV file has no header + + Parameters + ---------- + source : string + This is the file path to the csv. + + Returns + ------- + result : boolean + The result of the CSV file containing a header or not. + """ + return not is_float(return_first_data(source)) + + +def data_preprocessing(source): + """Clear data (in particular NaN objects) and returns a CSV file without header and its name. + + Parameters + ---------- + source : string + The file path to the CSV file. + + Returns + ------- + Function + The function with the incoming cleared CSV + """ + output_path = "cleaned_data.csv" + + with open(source, "r") as file: + reader = csv.reader(file) + header = next(reader) # Read the header + + data = [row for row in reader] + + # Create a new list without the headers + data_no_headers = [] + + for row in data[1:]: + # Check if the row is not empty and if all values in the row can be converted to float + if row and all(is_float(value) for value in row): + data_no_headers.append(row) + + # Save the processed data to a new CSV file + with open(output_path, "w", encoding="utf-8") as output_file: + writer = csv.writer(output_file, delimiter=",") + writer.writerows(data_no_headers) + + return output_path + + if __name__ == "__main__": import doctest