Fix mixed CRLF / CR (#8238)

* Remove "% Encoding: UTF8"

* Introduce BibWriter

* Fixes for the results of the block functionality

* The written `.bib` file keeps the newline separator of the loaded `.bib` file

* Fix checkstyle

* Fix architecture test --> BibDatabase uses Sytem directly

* NEWLINE does not need to be altered anymore

* Fix obosolte language key

* Fix checkstyle

* Increase LOOKAHEAD

Author: koppor

CHANGELOG.md

@@ -42,6 +42,10 @@ Note that this project **does not** adhere to [Semantic Versioning](http://semve
 - We improved the Drag and Drop behavior in the "Customize Entry Types" Dialog [#6338](https://github.com/JabRef/jabref/issues/6338)
 - When determining the URL of an ArXiV eprint, the URL now points to the version [#8149](https://github.com/JabRef/jabref/pull/8149)
 - We Included all standard fields with citation key when exporting to Old OpenOffice/LibreOffice Calc Format [#8176](https://github.com/JabRef/jabref/pull/8176)
+- In case the database is encoded with `UTF8`, the `% Encoding` marker is not written anymore
+- The written `.bib` file has the same line endings [#390](https://github.com/koppor/jabref/issues/390)
+- The written `.bib` file always has a final line break
+- The written `.bib` file keeps the newline separator of the loaded `.bib` file
 - We present options to manually enter an article or return to the New Entry menu when the fetcher DOI fails to find an entry for an ID [#7870](https://github.com/JabRef/jabref/issues/7870)
 - We trim white space and non-ASCII characters from DOI [#8127](https://github.com/JabRef/jabref/issues/8127)
 - The duplicate checker now inspects other fields in case no difference in the required and optional fields are found.

src/jmh/java/org/jabref/benchmarks/Benchmarks.java

@@ -9,6 +9,7 @@
 import java.util.stream.Collectors;
 
 import org.jabref.gui.Globals;
+import org.jabref.logic.exporter.BibWriter;
 import org.jabref.logic.exporter.BibtexDatabaseWriter;
 import org.jabref.logic.exporter.SavePreferences;
 import org.jabref.logic.formatter.bibtexfields.HtmlToLatexFormatter;
@@ -17,6 +18,7 @@
 import org.jabref.logic.layout.format.HTMLChars;
 import org.jabref.logic.layout.format.LatexToUnicodeFormatter;
 import org.jabref.logic.search.SearchQuery;
+import org.jabref.logic.util.OS;
 import org.jabref.model.database.BibDatabase;
 import org.jabref.model.database.BibDatabaseContext;
 import org.jabref.model.database.BibDatabaseMode;
@@ -77,7 +79,8 @@ public void init() throws Exception {
 
     private StringWriter getOutputWriter() throws IOException {
         StringWriter outputWriter = new StringWriter();
-        BibtexDatabaseWriter databaseWriter = new BibtexDatabaseWriter(outputWriter, mock(GeneralPreferences.class), mock(SavePreferences.class), new BibEntryTypesManager());
+        BibWriter bibWriter = new BibWriter(outputWriter, OS.NEWLINE);
+        BibtexDatabaseWriter databaseWriter = new BibtexDatabaseWriter(bibWriter, mock(GeneralPreferences.class), mock(SavePreferences.class), new BibEntryTypesManager());
         databaseWriter.savePartOfDatabase(new BibDatabaseContext(database, new MetaData()), database.getEntries());
         return outputWriter;
     }

src/main/java/org/jabref/cli/ArgumentProcessor.java

@@ -22,6 +22,7 @@
 import org.jabref.logic.citationkeypattern.CitationKeyGenerator;
 import org.jabref.logic.exporter.AtomicFileWriter;
 import org.jabref.logic.exporter.BibDatabaseWriter;
+import org.jabref.logic.exporter.BibWriter;
 import org.jabref.logic.exporter.BibtexDatabaseWriter;
 import org.jabref.logic.exporter.EmbeddedBibFilePdfExporter;
 import org.jabref.logic.exporter.Exporter;
@@ -531,7 +532,8 @@ private void saveDatabase(BibDatabase newBase, String subName) {
             GeneralPreferences generalPreferences = preferencesService.getGeneralPreferences();
             SavePreferences savePreferences = preferencesService.getSavePreferences();
             AtomicFileWriter fileWriter = new AtomicFileWriter(Path.of(subName), generalPreferences.getDefaultEncoding());
-            BibDatabaseWriter databaseWriter = new BibtexDatabaseWriter(fileWriter, generalPreferences, savePreferences, Globals.entryTypesManager);
+            BibWriter bibWriter = new BibWriter(fileWriter, OS.NEWLINE);
+            BibDatabaseWriter databaseWriter = new BibtexDatabaseWriter(bibWriter, generalPreferences, savePreferences, Globals.entryTypesManager);
             databaseWriter.saveDatabase(new BibDatabaseContext(newBase));
 
             // Show just a warning message if encoding did not work for all characters:

src/main/java/org/jabref/gui/JabRefMain.java

@@ -25,7 +25,6 @@
 import org.jabref.logic.protectedterms.ProtectedTermsLoader;
 import org.jabref.logic.remote.RemotePreferences;
 import org.jabref.logic.remote.client.RemoteClient;
-import org.jabref.logic.util.OS;
 import org.jabref.migrations.PreferencesMigrations;
 import org.jabref.model.database.BibDatabaseContext;
 import org.jabref.model.database.BibDatabaseMode;
@@ -141,10 +140,6 @@ private static void applyPreferences(PreferencesService preferences) {
 
         // Initialize protected terms loader
         Globals.protectedTermsLoader = new ProtectedTermsLoader(preferences.getProtectedTermsPreferences());
-
-        // Override used newline character with the one stored in the preferences.
-        // The preferences return the system newline character sequence as default.
-        OS.NEWLINE = preferences.getNewLineSeparator().toString();
     }
 
     private static void configureProxy(ProxyPreferences proxyPreferences) {

src/main/java/org/jabref/gui/SendAsEMailAction.java

@@ -15,7 +15,9 @@
 import org.jabref.gui.util.BackgroundTask;
 import org.jabref.logic.bibtex.BibEntryWriter;
 import org.jabref.logic.bibtex.FieldWriter;
+import org.jabref.logic.exporter.BibWriter;
 import org.jabref.logic.l10n.Localization;
+import org.jabref.logic.util.OS;
 import org.jabref.logic.util.io.FileUtil;
 import org.jabref.model.database.BibDatabaseContext;
 import org.jabref.model.entry.BibEntry;
@@ -72,6 +74,7 @@ private String sendEmail() throws Exception {
         }
 
         StringWriter rawEntries = new StringWriter();
+        BibWriter bibWriter = new BibWriter(rawEntries, OS.NEWLINE);
         BibDatabaseContext databaseContext = stateManager.getActiveDatabase().get();
         List<BibEntry> entries = stateManager.getSelectedEntries();
 
@@ -80,7 +83,7 @@ private String sendEmail() throws Exception {
 
         for (BibEntry entry : entries) {
             try {
-                bibtexEntryWriter.write(entry, rawEntries, databaseContext.getMode());
+                bibtexEntryWriter.write(entry, bibWriter, databaseContext.getMode());
             } catch (IOException e) {
                 LOGGER.warn("Problem creating BibTeX file for mailing.", e);
             }
@@ -104,7 +107,7 @@ private String sendEmail() throws Exception {
             }
         }
 
-        String mailTo = "?Body=".concat(rawEntries.getBuffer().toString());
+        String mailTo = "?Body=".concat(rawEntries.toString());
         mailTo = mailTo.concat("&Subject=");
         mailTo = mailTo.concat(preferencesService.getExternalApplicationsPreferences().getEmailSubject());
         for (String path : attachments) {

src/main/java/org/jabref/gui/entryeditor/SourceTab.java

@@ -38,11 +38,13 @@
 import org.jabref.logic.bibtex.FieldWriter;
 import org.jabref.logic.bibtex.FieldWriterPreferences;
 import org.jabref.logic.bibtex.InvalidFieldValueException;
+import org.jabref.logic.exporter.BibWriter;
 import org.jabref.logic.importer.ImportFormatPreferences;
 import org.jabref.logic.importer.ParserResult;
 import org.jabref.logic.importer.fileformat.BibtexParser;
 import org.jabref.logic.l10n.Localization;
 import org.jabref.logic.search.SearchQuery;
+import org.jabref.logic.util.OS;
 import org.jabref.model.database.BibDatabase;
 import org.jabref.model.database.BibDatabaseContext;
 import org.jabref.model.database.BibDatabaseMode;
@@ -128,11 +130,11 @@ private void highlightSearchPattern() {
     }
 
     private String getSourceString(BibEntry entry, BibDatabaseMode type, FieldWriterPreferences fieldWriterPreferences) throws IOException {
-        StringWriter stringWriter = new StringWriter(200);
+        StringWriter writer = new StringWriter();
+        BibWriter bibWriter = new BibWriter(writer, OS.NEWLINE);
         FieldWriter fieldWriter = FieldWriter.buildIgnoreHashes(fieldWriterPreferences);
-        new BibEntryWriter(fieldWriter, Globals.entryTypesManager).writeWithoutPrependedNewlines(entry, stringWriter, type);
-
-        return stringWriter.getBuffer().toString();
+        new BibEntryWriter(fieldWriter, Globals.entryTypesManager).write(entry, bibWriter, type);
+        return writer.toString();
     }
 
     /* Work around for different input methods.

src/main/java/org/jabref/gui/exporter/SaveDatabaseAction.java

@@ -24,6 +24,7 @@
 import org.jabref.logic.autosaveandbackup.AutosaveManager;
 import org.jabref.logic.autosaveandbackup.BackupManager;
 import org.jabref.logic.exporter.AtomicFileWriter;
+import org.jabref.logic.exporter.BibWriter;
 import org.jabref.logic.exporter.BibtexDatabaseWriter;
 import org.jabref.logic.exporter.SaveException;
 import org.jabref.logic.exporter.SavePreferences;
@@ -229,12 +230,14 @@ private boolean saveDatabase(Path file, boolean selectedOnly, Charset encoding,
         SavePreferences savePreferences = this.preferences.getSavePreferences()
                                                       .withSaveType(saveType);
         try (AtomicFileWriter fileWriter = new AtomicFileWriter(file, encoding, savePreferences.shouldMakeBackup())) {
-            BibtexDatabaseWriter databaseWriter = new BibtexDatabaseWriter(fileWriter, generalPreferences, savePreferences, entryTypesManager);
+            BibDatabaseContext bibDatabaseContext = libraryTab.getBibDatabaseContext();
+            BibWriter bibWriter = new BibWriter(fileWriter, bibDatabaseContext.getDatabase().getNewLineSeparator());
+            BibtexDatabaseWriter databaseWriter = new BibtexDatabaseWriter(bibWriter, generalPreferences, savePreferences, entryTypesManager);
 
             if (selectedOnly) {
-                databaseWriter.savePartOfDatabase(libraryTab.getBibDatabaseContext(), libraryTab.getSelectedEntries());
+                databaseWriter.savePartOfDatabase(bibDatabaseContext, libraryTab.getSelectedEntries());
             } else {
-                databaseWriter.saveDatabase(libraryTab.getBibDatabaseContext());
+                databaseWriter.saveDatabase(bibDatabaseContext);
             }
 
             libraryTab.registerUndoableChanges(databaseWriter.getSaveActionsFieldChanges());

src/main/java/org/jabref/gui/preferences/file/FileTab.fxml

@@ -31,10 +31,6 @@
                      toggleGroup="$stringsResolveToggleGroup"/>
         <TextField fx:id="resolveStringsExcept" HBox.hgrow="ALWAYS"/>
     </HBox>
-    <HBox alignment="CENTER_LEFT" spacing="10.0">
-        <Label alignment="TOP_LEFT" text="%Newline separator"/>
-        <ComboBox fx:id="newLineSeparator" prefWidth="120.0"/>
-    </HBox>
     <CheckBox fx:id="alwaysReformatBib" text="%Always reformat BIB file on save and export"/>
 
     <Label styleClass="sectionHeader" text="%Autosave"/>

src/main/java/org/jabref/gui/preferences/file/FileTab.java

@@ -3,7 +3,6 @@
 import javafx.fxml.FXML;
 import javafx.scene.control.Button;
 import javafx.scene.control.CheckBox;
-import javafx.scene.control.ComboBox;
 import javafx.scene.control.RadioButton;
 import javafx.scene.control.TextField;
 
@@ -13,10 +12,8 @@
 import org.jabref.gui.help.HelpAction;
 import org.jabref.gui.preferences.AbstractPreferenceTabView;
 import org.jabref.gui.preferences.PreferencesTab;
-import org.jabref.gui.util.ViewModelListCellFactory;
 import org.jabref.logic.help.HelpFile;
 import org.jabref.logic.l10n.Localization;
-import org.jabref.preferences.NewLineSeparator;
 
 import com.airhacks.afterburner.views.ViewLoader;
 
@@ -27,7 +24,6 @@ public class FileTab extends AbstractPreferenceTabView<FileTabViewModel> impleme
     @FXML private RadioButton resolveStringsBibTex;
     @FXML private RadioButton resolveStringsAll;
     @FXML private TextField resolveStringsExcept;
-    @FXML private ComboBox<NewLineSeparator> newLineSeparator;
     @FXML private CheckBox alwaysReformatBib;
 
     @FXML private CheckBox autosaveLocalLibraries;
@@ -47,11 +43,6 @@ public void initialize() {
         resolveStringsAll.selectedProperty().bindBidirectional(viewModel.resolveStringsAllProperty());
         resolveStringsExcept.textProperty().bindBidirectional(viewModel.resolveStringsExceptProperty());
         resolveStringsExcept.disableProperty().bind(resolveStringsAll.selectedProperty().not());
-        new ViewModelListCellFactory<NewLineSeparator>()
-                .withText(NewLineSeparator::getDisplayName)
-                .install(newLineSeparator);
-        newLineSeparator.itemsProperty().bind(viewModel.newLineSeparatorListProperty());
-        newLineSeparator.valueProperty().bindBidirectional(viewModel.selectedNewLineSeparatorProperty());
         alwaysReformatBib.selectedProperty().bindBidirectional(viewModel.alwaysReformatBibProperty());
 
         autosaveLocalLibraries.selectedProperty().bindBidirectional(viewModel.autosaveLocalLibrariesProperty());

src/main/java/org/jabref/gui/preferences/file/FileTabViewModel.java

@@ -1,18 +1,12 @@
 package org.jabref.gui.preferences.file;
 
 import javafx.beans.property.BooleanProperty;
-import javafx.beans.property.ListProperty;
-import javafx.beans.property.ObjectProperty;
 import javafx.beans.property.SimpleBooleanProperty;
-import javafx.beans.property.SimpleListProperty;
-import javafx.beans.property.SimpleObjectProperty;
 import javafx.beans.property.SimpleStringProperty;
 import javafx.beans.property.StringProperty;
-import javafx.collections.FXCollections;
 
 import org.jabref.gui.preferences.PreferenceTabViewModel;
 import org.jabref.preferences.ImportExportPreferences;
-import org.jabref.preferences.NewLineSeparator;
 import org.jabref.preferences.PreferencesService;
 
 public class FileTabViewModel implements PreferenceTabViewModel {
@@ -22,8 +16,6 @@ public class FileTabViewModel implements PreferenceTabViewModel {
     private final BooleanProperty resolveStringsBibTexProperty = new SimpleBooleanProperty();
     private final BooleanProperty resolveStringsAllProperty = new SimpleBooleanProperty();
     private final StringProperty resolveStringsExceptProperty = new SimpleStringProperty("");
-    private final ListProperty<NewLineSeparator> newLineSeparatorListProperty = new SimpleListProperty<>();
-    private final ObjectProperty<NewLineSeparator> selectedNewLineSeparatorProperty = new SimpleObjectProperty<>();
     private final BooleanProperty alwaysReformatBibProperty = new SimpleBooleanProperty();
     private final BooleanProperty autosaveLocalLibraries = new SimpleBooleanProperty();
 
@@ -43,8 +35,6 @@ public void setValues() {
         resolveStringsAllProperty.setValue(importExportPreferences.shouldResolveStringsForAllStrings()); // Flipped around
         resolveStringsBibTexProperty.setValue(importExportPreferences.shouldResolveStringsForStandardBibtexFields());
         resolveStringsExceptProperty.setValue(importExportPreferences.getNonResolvableFields());
-        newLineSeparatorListProperty.setValue(FXCollections.observableArrayList(NewLineSeparator.values()));
-        selectedNewLineSeparatorProperty.setValue(importExportPreferences.getNewLineSeparator());
 
         alwaysReformatBibProperty.setValue(importExportPreferences.shouldAlwaysReformatOnSave());
 
@@ -59,7 +49,6 @@ public void storeSettings() {
         importExportPreferences.setResolveStringsForStandardBibtexFields(resolveStringsBibTexProperty.getValue());
         importExportPreferences.setResolveStringsForAllStrings(resolveStringsAllProperty.getValue());
         importExportPreferences.setNonResolvableFields(resolveStringsExceptProperty.getValue().trim());
-        importExportPreferences.setNewLineSeparator(selectedNewLineSeparatorProperty.getValue());
         importExportPreferences.setAlwaysReformatOnSave(alwaysReformatBibProperty.getValue());
 
         preferences.storeShouldAutosave(autosaveLocalLibraries.getValue());
@@ -89,14 +78,6 @@ public StringProperty resolveStringsExceptProperty() {
         return resolveStringsExceptProperty;
     }
 
-    public ListProperty<NewLineSeparator> newLineSeparatorListProperty() {
-        return newLineSeparatorListProperty;
-    }
-
-    public ObjectProperty<NewLineSeparator> selectedNewLineSeparatorProperty() {
-        return selectedNewLineSeparatorProperty;
-    }
-
     public BooleanProperty alwaysReformatBibProperty() {
         return alwaysReformatBibProperty;
     }

src/main/java/org/jabref/logic/autosaveandbackup/BackupManager.java

@@ -1,6 +1,7 @@
 package org.jabref.logic.autosaveandbackup;
 
 import java.io.IOException;
+import java.io.Writer;
 import java.nio.charset.Charset;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -13,10 +14,12 @@
 
 import org.jabref.logic.bibtex.InvalidFieldValueException;
 import org.jabref.logic.exporter.AtomicFileWriter;
+import org.jabref.logic.exporter.BibWriter;
 import org.jabref.logic.exporter.BibtexDatabaseWriter;
 import org.jabref.logic.exporter.SavePreferences;
 import org.jabref.logic.util.CoarseChangeFilter;
 import org.jabref.logic.util.DelayTaskThrottler;
+import org.jabref.logic.util.OS;
 import org.jabref.logic.util.io.FileUtil;
 import org.jabref.model.database.BibDatabaseContext;
 import org.jabref.model.database.event.BibDatabaseContextChangedEvent;
@@ -135,7 +138,9 @@ private void performBackup(Path backupPath) {
             GeneralPreferences generalPreferences = preferences.getGeneralPreferences();
             SavePreferences savePreferences = preferences.getSavePreferences()
                                                          .withMakeBackup(false);
-            new BibtexDatabaseWriter(new AtomicFileWriter(backupPath, charset), generalPreferences, savePreferences, entryTypesManager)
+            Writer writer = new AtomicFileWriter(backupPath, charset);
+            BibWriter bibWriter = new BibWriter(writer, OS.NEWLINE);
+            new BibtexDatabaseWriter(bibWriter, generalPreferences, savePreferences, entryTypesManager)
                     .saveDatabase(bibDatabaseContext);
         } catch (IOException e) {
             logIfCritical(backupPath, e);