Skip to content

Commit 58ea24c

Browse files
author
Stepan Kamenik
committed
feat(FgForrest#3): add chatgpt translator a configurations
1 parent 769da9a commit 58ea24c

12 files changed

+343
-110
lines changed

‎pom.xml

+5-1
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,11 @@
8282
</dependencyManagement>
8383

8484
<dependencies>
85-
85+
<dependency>
86+
<groupId>com.theokanning.openai-gpt3-java</groupId>
87+
<artifactId>service</artifactId>
88+
<version>0.18.2</version>
89+
</dependency>
8690
<dependency>
8791
<groupId>com.google.cloud</groupId>
8892
<artifactId>google-cloud-translate</artifactId>

‎src/main/java/one/edee/babylon/MainService.java

+3-8
Original file line numberDiff line numberDiff line change
@@ -29,20 +29,15 @@ public MainService(Exporter exporter,
2929
this.importProcessor = importProcessor;
3030
}
3131

32-
public void startTranslation(Action action, TranslationConfiguration configuration, String spreadsheetId, boolean combineSheets, String translatorApiKey) throws IOException, GeneralSecurityException, InterruptedException {
32+
public void startTranslation(Action action, TranslationConfiguration configuration, String spreadsheetId, boolean combineSheets) throws IOException, GeneralSecurityException, InterruptedException {
3333
long stTime = System.currentTimeMillis();
3434
switch (action) {
3535
case EXPORT:
3636
log.info("Babylon starting...");
3737
exporter.walkPathsAndWriteSheets(
38-
configuration.getPath(),
39-
configuration.getMutations(),
38+
configuration,
4039
spreadsheetId,
41-
configuration.getSnapshotPath(),
42-
configuration.getLockedCellEditors(),
43-
combineSheets,
44-
translatorApiKey,
45-
configuration.getDefaultLang());
40+
combineSheets);
4641
break;
4742
case IMPORT:
4843
importProcessor.doImport(spreadsheetId);

‎src/main/java/one/edee/babylon/SpringBootConsoleApplication.java

+3-12
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ public void run(String... args) {
4444
try {
4545
log.info("Loading config file: '" + arguments.getConfigFileName() + "'");
4646
TranslationConfiguration configuration = configurationReader.readAndCheckConfiguration(arguments.getConfigFileName());
47-
mainService.startTranslation(arguments.getAction(), configuration, arguments.getGoogleSheetId(), arguments.isCombineSheets(), arguments.getTranslatorApiKey());
47+
mainService.startTranslation(arguments.getAction(), configuration, arguments.getGoogleSheetId(), arguments.isCombineSheets());
4848
} catch (Exception e) {
4949
log.error("BABYLON ERROR: ", e);
5050
System.exit(-1);
@@ -75,10 +75,7 @@ public static Arguments parseArguments(String... args) {
7575
arguments.setConfigFileName(args[1]);
7676
arguments.setGoogleSheetId(args[2]);
7777
if (args.length > 3){
78-
arguments.setTranslatorApiKey(args[3]);
79-
if (args.length > 4){
80-
arguments.setCombineSheets(Boolean.parseBoolean(args[4]));
81-
}
78+
arguments.setCombineSheets(Boolean.parseBoolean(args[3]));
8279
}
8380
return arguments;
8481
}
@@ -87,8 +84,7 @@ private static void printRequiredArguments() {
8784
log.info("1 - expected action (import, export)");
8885
log.info("2 - path to translator-config.json file");
8986
log.info("3 - ID of the google sheet (e.g. 1xhnBAOpy8-9KWhl8NP0ZIy6mhlgXKnKcLJwKcIeyjPc)");
90-
log.info("4 - arg to specify translator api key");
91-
log.info("5 - arg to specify combineSheets mode");
87+
log.info("4 - arg to specify combineSheets mode");
9288
}
9389

9490
/**
@@ -118,11 +114,6 @@ public static class Arguments {
118114
* This mode is useful to correct duplicates, etc.
119115
*/
120116
private boolean combineSheets = false;
121-
122-
/**
123-
* Translator api key (Google/Deepl).
124-
*/
125-
private String translatorApiKey;
126117
}
127118

128119
}

‎src/main/java/one/edee/babylon/config/SupportedTranslators.java

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
package one.edee.babylon.config;
2+
3+
/**
4+
* I apologize in advance for the lack of documentation in this code.
5+
* I had every intention of providing clear and concise explanations
6+
* for every line of code, but then I got distracted by a squirrel outside
7+
* my window and the next thing I knew it was three weeks later.
8+
* <p>
9+
* So instead, I've included some helpful comments here and there.
10+
* They might not make sense, but hey, at least they're something.
11+
*
12+
* @author Štěpán Kameník ([email protected]), FG Forrest a.s. (c) 2024
13+
**/
14+
public enum SupportedTranslators {
15+
16+
/**
17+
* @see one.edee.babylon.export.translator.GoogleTranslator
18+
*/
19+
GOOGLE,
20+
21+
/**
22+
* @see one.edee.babylon.export.translator.DeeplTranslator
23+
*/
24+
DEEPL,
25+
26+
/**
27+
* @see one.edee.babylon.export.translator.OpenAiTranslator
28+
*/
29+
OPENAI
30+
}

‎src/main/java/one/edee/babylon/config/TranslationConfiguration.java

+8-1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import lombok.NoArgsConstructor;
66
import lombok.NonNull;
77
import lombok.RequiredArgsConstructor;
8+
import org.jetbrains.annotations.Nullable;
89

910
import java.io.Serializable;
1011
import java.nio.file.Path;
@@ -52,7 +53,13 @@ public class TranslationConfiguration implements Serializable {
5253
/**
5354
* Default language of project properties.
5455
*/
55-
@NonNull
56+
@Nullable
57+
private SupportedTranslators translator;
58+
59+
/**
60+
* Default language of project properties.
61+
*/
62+
@Nullable
5663
private String translatorApiKey;
5764

5865
@JsonIgnore

‎src/main/java/one/edee/babylon/export/Exporter.java

+45-77
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,21 @@
11
package one.edee.babylon.export;
22

3-
import com.deepl.api.DeepLException;
4-
import com.deepl.api.Translator;
5-
import com.google.api.client.http.HttpRequestInitializer;
6-
import com.google.cloud.translate.Translate;
7-
import com.google.cloud.translate.TranslateOptions;
3+
import lombok.RequiredArgsConstructor;
84
import lombok.extern.apachecommons.CommonsLog;
5+
import one.edee.babylon.config.SupportedTranslators;
6+
import one.edee.babylon.config.TranslationConfiguration;
97
import one.edee.babylon.db.SnapshotUtils;
108
import one.edee.babylon.export.dto.ExportResult;
119
import one.edee.babylon.export.dto.TranslationSheet;
10+
import one.edee.babylon.export.translator.Translator;
1211
import one.edee.babylon.sheets.SheetsException;
1312
import one.edee.babylon.sheets.gsheets.model.ASheet;
1413
import one.edee.babylon.snapshot.TranslationSnapshotWriteContract;
1514
import one.edee.babylon.util.AntPathResourceLoader;
1615
import one.edee.babylon.util.PathUtils;
1716
import org.jetbrains.annotations.NotNull;
17+
import org.springframework.context.ApplicationContext;
18+
import org.springframework.util.Assert;
1819
import org.springframework.util.StringUtils;
1920

2021
import java.io.File;
@@ -25,66 +26,35 @@
2526
import java.util.function.Function;
2627
import java.util.stream.Collectors;
2728

28-
import static com.google.cloud.translate.Translate.TranslateOption.sourceLanguage;
29-
import static com.google.cloud.translate.Translate.TranslateOption.targetLanguage;
29+
import static java.util.Optional.ofNullable;
3030

3131
/**
3232
* Performs the export phase that generates translation sheets.
3333
*/
3434
@CommonsLog
35+
@RequiredArgsConstructor
3536
public class Exporter {
3637
private static final String COMBINING_SHEET_NAME = "ALL";
3738

3839

40+
private final ApplicationContext applicationContext;
3941
private final TranslationCollector translationCollector;
4042
private final TranslationSnapshotWriteContract snapshot;
4143
private final SheetContract gsc;
4244
private final AntPathResourceLoader resourceLoader;
43-
private final PathUtils pu;
44-
45-
public Exporter(TranslationCollector translationCollector, TranslationSnapshotWriteContract snapshot, SheetContract gsc, AntPathResourceLoader resourceLoader) {
46-
this.translationCollector = translationCollector;
47-
this.snapshot = snapshot;
48-
this.gsc = gsc;
49-
this.resourceLoader = resourceLoader;
50-
this.pu = new PathUtils();
51-
}
45+
private final PathUtils pu = new PathUtils();
5246

53-
/**
54-
* Walks message file paths, gathering messages and translations, producing translation sheets in given GSheet spreadsheet.
55-
*
56-
* @param patternPaths paths of message files to export
57-
* @param translationLangs languages to translate messages to
58-
* @param spreadsheetId id of GSheets spreadsheet, must be empty
59-
* @param snapshotPath path to the translation snapshot file
60-
*/
61-
public void walkPathsAndWriteSheets(List<String> patternPaths,
62-
List<String> translationLangs,
63-
String spreadsheetId,
64-
Path snapshotPath,
65-
boolean combineSheets,
66-
String translatorApiKey) {
67-
walkPathsAndWriteSheets(patternPaths, translationLangs, spreadsheetId, snapshotPath, Collections.emptyList(), combineSheets, translatorApiKey, null);
68-
}
6947

7048
/**
7149
* Walks message file paths, gathering messages and translations, producing translation sheets in given GSheet spreadsheet.
7250
*
73-
* @param patternPaths paths of message files to export
74-
* @param translationLangs languages to translate messages to
51+
* @param configuration configuration of translation run
7552
* @param spreadsheetId id of GSheets spreadsheet, must be empty
76-
* @param snapshotPath path to the translation snapshot file
77-
* @param lockedCellEditors list of Google account emails, these account will have the permission to edit locked cells
78-
* @param translatorApiKey
7953
*/
80-
public void walkPathsAndWriteSheets(List<String> patternPaths,
81-
List<String> translationLangs,
54+
public void walkPathsAndWriteSheets(TranslationConfiguration configuration,
8255
String spreadsheetId,
83-
Path snapshotPath,
84-
List<String> lockedCellEditors,
85-
boolean combineSheets,
86-
String translatorApiKey,
87-
String defaultLang) {
56+
boolean combineSheets) {
57+
List<String> patternPaths = configuration.getPath();
8858
warnDuplicatePaths(patternPaths);
8959

9060
List<ASheet> prevSheets = listAllSheets(spreadsheetId);
@@ -95,7 +65,7 @@ public void walkPathsAndWriteSheets(List<String> patternPaths,
9565
throw new IllegalArgumentException("Please fix the message file paths in the configuration file.");
9666
}
9767

98-
ExportResult result = translationCollector.walkPathsAndCollectTranslationSheets(allUniquePaths, translationLangs);
68+
ExportResult result = translationCollector.walkPathsAndCollectTranslationSheets(allUniquePaths, configuration.getMutations());
9969

10070
if (combineSheets) {
10171
// only for translation debugging
@@ -116,31 +86,43 @@ public void walkPathsAndWriteSheets(List<String> patternPaths,
11686
original.add(new TranslationSheet(COMBINING_SHEET_NAME,combine));
11787
}
11888

119-
Map<String, List<String>> changed = translateTextsByExternalTool(translatorApiKey, defaultLang, result);
89+
Map<String, List<String>> changed = translateTextsByExternalTool(configuration, result);
12090

121-
uploadTranslations(result, spreadsheetId, lockedCellEditors, changed);
91+
uploadTranslations(result, spreadsheetId, configuration.getLockedCellEditors(), changed);
12292

123-
updateSnapshotAndWriteToDisk(this.snapshot, result, snapshotPath);
93+
updateSnapshotAndWriteToDisk(this.snapshot, result, configuration.getSnapshotPath());
12494

12595
List<Integer> prevSheetIds = prevSheets.stream().map(ASheet::getId).collect(Collectors.toList());
12696
deleteOldSheets(prevSheetIds, spreadsheetId);
12797
}
12898

12999
@NotNull
130-
private static Map<String, List<String>> translateTextsByExternalTool(String translatorApiKey, String defaultLang, ExportResult result) {
100+
private Map<String, List<String>> translateTextsByExternalTool(TranslationConfiguration configuration, ExportResult result) {
131101
Map<String, List<String>> changed = new HashMap<>();
132102

133-
if (translatorApiKey != null) {
103+
if (configuration.getTranslatorApiKey() != null) {
104+
SupportedTranslators translatorType = ofNullable(configuration.getTranslator()).orElse(SupportedTranslators.GOOGLE);
105+
106+
Translator translator = applicationContext.getBeansOfType(Translator.class)
107+
.values()
108+
.stream()
109+
.filter(i -> i.getSupportedTranslator().equals(translatorType))
110+
.findFirst()
111+
.orElseThrow(() -> new IllegalArgumentException("Cannot find translator bean for type" + translatorType));
112+
translator.init(configuration.getTranslatorApiKey());
113+
134114
try {
135-
// Translator translator = new Translator(translatorApiKey);
136-
//noinspection deprecation
137-
Translate translate = TranslateOptions.newBuilder().setApiKey(translatorApiKey).build().getService();
138115
for (TranslationSheet sheet : result.getSheets()) {
139116
log.info("Translating sheet " + sheet.getSheetName());
140117

141118
List<List<String>> rows = sheet.getRows();
142119
List<String> header = rows.get(0);
120+
List<String> originals = rows.stream().map(i->i.get(1)).map(i->StringUtils.hasText(i) ? i : "____DUMMY").collect(Collectors.toList());
121+
Map<String, List<String>> translations = new HashMap<>();
143122

123+
for (String lang : header.stream().skip(2).collect(Collectors.toList())) {
124+
translations.put(lang, translator.translate(configuration.getDefaultLang(), originals, lang));
125+
}
144126

145127
for (int i = 1; i < rows.size(); i++) {
146128
Map<Integer, String> toChange = new HashMap<>();
@@ -152,17 +134,17 @@ private static Map<String, List<String>> translateTextsByExternalTool(String tra
152134

153135
String lang = header.get(l);
154136

155-
if (lang.equals("en")) {
156-
lang = "en-GB";
157-
}
158-
159137
if (StringUtils.hasText(original)) {
160-
String translatedText = getTranslatedTextByGoogle(defaultLang, translate, original, lang);
161-
toChange.put(l, translatedText);
162-
163-
changed
164-
.computeIfAbsent(sheet.getSheetName(), key -> new LinkedList<>())
165-
.add(i + "_" + l);
138+
String transOriginal = originals.get(i);
139+
if (!Objects.equals(original, "____DUMMY")){
140+
Assert.isTrue(Objects.equals(transOriginal, original), "Originals does not equals!");
141+
String translatedText = translations.get(lang).get(i);
142+
toChange.put(l, translatedText);
143+
144+
changed
145+
.computeIfAbsent(sheet.getSheetName(), key -> new LinkedList<>())
146+
.add(i + "_" + l);
147+
}
166148
}
167149
}
168150
}
@@ -181,20 +163,6 @@ private static Map<String, List<String>> translateTextsByExternalTool(String tra
181163
return changed;
182164
}
183165

184-
private static String getTranslatedTextByDeepl(String defaultLang, Translator translator, String original, String lang) throws DeepLException, InterruptedException {
185-
return translator.translateText(original, defaultLang, lang).getText();
186-
}
187-
188-
private static String getTranslatedTextByGoogle(String defaultLang, Translate translate, String original, String lang) {
189-
190-
return translate.translate(
191-
original,
192-
sourceLanguage(defaultLang),
193-
targetLanguage(lang))
194-
.getTranslatedText();
195-
}
196-
197-
198166
private void warnDuplicatePaths(List<String> patternPaths) {
199167
List<String> duplicatePaths = detectDuplicatePatternPaths(patternPaths);
200168
if (!duplicatePaths.isEmpty()) {

0 commit comments

Comments
 (0)