Skip to content

Commit d8a2185

Browse files
frankpaganfrankpagan
committed
feat: enhance file processing by adding support for include/exclude filters and implementing dynamic data retrieval for various file types
1 parent c03b649 commit d8a2185

File tree

1 file changed

+188
-85
lines changed

1 file changed

+188
-85
lines changed

‎src/server.js‎

Lines changed: 188 additions & 85 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ const fs = require("fs");
44
const realpathAsync = fs.promises.realpath;
55

66
const path = require("path");
7+
const { pathToFileURL } = require("url");
78
const mimeTypes = {
89
".aac": "audio/aac",
910
".abw": "application/x-abiword",
@@ -174,46 +175,60 @@ module.exports = async function file(
174175
async function runDirectories() {
175176
for (const directory of directories) {
176177
const entry = directory.entry;
177-
const exclude = directory.exclude || [];
178-
await runFiles(directory, entry, exclude);
178+
await runFiles(directory, entry);
179179
}
180180
return;
181181
}
182182

183-
async function runFiles(directory, entry, exclude, Path, directoryName) {
183+
async function runFiles(directory, entry, Path, directoryName) {
184184
const entryPath = path.resolve(configDirectoryPath, entry);
185185
let files = fs.readdirSync(entryPath);
186-
186+
let exclude = directory.exclude || [];
187+
let include = directory.include || [];
187188
for (let file of files) {
188-
let skip = false;
189-
for (let i = 0; i < exclude.length; i++) {
190-
if (file.includes(exclude[i])) {
191-
skip = true;
192-
break;
193-
}
194-
}
195-
if (skip) continue;
189+
const filePath = path.resolve(entryPath, file);
196190

197191
let isDirectory;
198192
let isSymlink = fs
199-
.lstatSync(`${entryPath}/${file}`)
193+
.lstatSync(filePath)
200194
.isSymbolicLink();
201195
if (isSymlink) {
202-
let symlinkPath = await realpathAsync(`${entryPath}/${file}`);
196+
let symlinkPath = await realpathAsync(filePath);
203197
isDirectory =
204198
fs.existsSync(symlinkPath) &&
205199
fs.lstatSync(symlinkPath).isDirectory();
206200
} else
207201
isDirectory =
208-
fs.existsSync(`${entryPath}/${file}`) &&
209-
fs.lstatSync(`${entryPath}/${file}`).isDirectory();
202+
fs.existsSync(filePath) &&
203+
fs.lstatSync(filePath).isDirectory();
204+
205+
let skip = false;
206+
for (let i = 0; i < exclude.length; i++) {
207+
if (filePath.includes(exclude[i])) {
208+
skip = true;
209+
break;
210+
}
211+
}
212+
213+
for (let i = 0; i < include.length; i++) {
214+
if (filePath.includes(include[i])) {
215+
skip = false;
216+
break;
217+
} else if (isDirectory) {
218+
skip = "directory";
219+
break;
220+
} else {
221+
skip = true;
222+
}
223+
}
224+
225+
if (skip === true) continue;
210226

211227
let name = file;
212228
let source = "";
213229

214230
for (let i = 0; i < match.length; i++) {
215231
skip = true;
216-
const filePath = path.resolve(entryPath, file);
217232
if (filePath.startsWith(match[i])) {
218233
skip = false;
219234
break;
@@ -237,7 +252,7 @@ module.exports = async function file(
237252
} else directoryName = "/";
238253
}
239254

240-
if (exclude && exclude.includes(directoryName)) continue;
255+
// if (exclude && exclude.includes(directoryName)) continue;
241256

242257
if (!Path) {
243258
if (directoryName === "/") Path = directoryName;
@@ -248,99 +263,136 @@ module.exports = async function file(
248263
if (Path === "/") pathname = Path + name;
249264
else pathname = Path + "/" + name;
250265

251-
if (isDirectory) mimeType = "text/directory";
252-
else
266+
if (isDirectory) {
267+
mimeType = "text/directory";
268+
} else {
253269
source = await getSource(
254270
`${entryPath}/${file}`,
255271
mimeType,
256272
isSymlink
257273
);
274+
}
258275

259276
let values = {
260277
"{{name}}": name || "",
261278
"{{source}}": Buffer.isBuffer(source) ? `data:${mimeType};base64,${source.toString('base64')}` : source || "",
262279
"{{directory}}": directoryName || "",
263280
"{{path}}": Path || "",
264281
"{{pathname}}": pathname,
265-
"{{content-type}}": mimeType || ""
282+
"{{content-type}}": mimeType || "",
266283
};
267284

268-
let object = { ...directory.object };
269-
if (!object.name) object.name = "{{name}}";
270-
if (!object.src) object.src = "{{source}}";
271-
if (!object.directory) object.directory = "{{directory}}";
272-
if (!object.path) object.path = "{{path}}";
273-
if (!object.pathname) object.pathname = "{{pathname}}";
274-
if (!object["content-type"])
275-
object["content-type"] = "{{content-type}}";
276-
if (
277-
!object.public &&
278-
object.public != false &&
279-
object.public != "false"
280-
)
281-
object.public = "true";
282-
283-
let newObject = {
284-
array: directory.array || "files",
285-
object
285+
let data = {
286+
array: directory.array || "files"
286287
};
287288

288-
if (
289-
options.translate &&
290-
mimeType === "text/html" &&
291-
Array.isArray(directory.languages) &&
292-
!object.translations
293-
) {
294-
try {
295-
// Call your AI translation service
296-
const translations = await options.translate(
297-
Buffer.isBuffer(source) ? source.toString('utf-8') : source,
298-
directory.languages
289+
let isData = false;
290+
if ( typeof directory.$data === "string") {
291+
if (isDirectory) {
292+
skip = "directory";
293+
} else {
294+
isData = true;
295+
data = await getData(
296+
`${entryPath}/${file}`,
297+
mimeType,
298+
isSymlink
299+
);
300+
if (!data) continue
301+
}
302+
} else if ( typeof directory.object === "string") {
303+
if (isDirectory) {
304+
skip = "directory";
305+
} else {
306+
isData = true;
307+
data.object = await getData(
308+
`${entryPath}/${file}`,
309+
mimeType,
310+
isSymlink
299311
);
300-
newObject.object.translations = translations;
301-
} catch (err) {
302-
console.error("Translation error:", err);
303-
// Continue without translations
312+
if (!data.object) continue
313+
}
314+
} else if (typeof directory.object === "object" && directory.object !== null) {
315+
data.array = directory.array || "files";
316+
let object = { ...directory.object };
317+
if (!object.name) object.name = "{{name}}";
318+
if (!object.src) object.src = "{{source}}";
319+
if (!object.directory) object.directory = "{{directory}}";
320+
if (!object.path) object.path = "{{path}}";
321+
if (!object.pathname) object.pathname = "{{pathname}}";
322+
if (!object["content-type"])
323+
object["content-type"] = "{{content-type}}";
324+
if (
325+
!object.public &&
326+
object.public != false &&
327+
object.public != "false"
328+
)
329+
object.public = "true";
330+
331+
data.object = object;
332+
333+
if (!data.object._id) {
334+
data.$filter = {
335+
query: {
336+
pathname
337+
}
338+
};
304339
}
305-
}
306340

307-
if (directory.storage) newObject.storage = directory.storage;
308-
if (directory.database) newObject.database = directory.database;
309-
if (directory.array) newObject.array = directory.array || "files";
341+
if (
342+
options.translate &&
343+
mimeType === "text/html" &&
344+
Array.isArray(directory.languages) &&
345+
!object.translations
346+
) {
347+
try {
348+
// Call your AI translation service
349+
const translations = await options.translate(
350+
Buffer.isBuffer(source) ? source.toString('utf-8') : source,
351+
directory.languages
352+
);
353+
data.object.translations = translations;
354+
} catch (err) {
355+
console.error("Translation error:", err);
356+
// Continue without translations
357+
}
358+
}
310359

311-
for (const key of Object.keys(directory.object)) {
312-
if (typeof directory.object[key] == "string") {
313-
let variables = directory.object[key].match(
314-
/{{([A-Za-z0-9_.,\[\]\-\/ ]*)}}/g
315-
);
316-
if (variables) {
317-
for (let variable of variables) {
318-
let replacement = values[variable];
319-
if (key === 'src' && variable === '{{source}}' && Buffer.isBuffer(source)) {
320-
replacement = `data:${mimeType};base64,${source.toString('base64')}`;
360+
if (directory.storage) data.storage = directory.storage;
361+
if (directory.database) data.database = directory.database;
362+
if (directory.array) data.array = directory.array || "files";
363+
364+
for (const key of Object.keys(directory.object)) {
365+
if (typeof directory.object[key] == "string") {
366+
let variables = directory.object[key].match(
367+
/{{([A-Za-z0-9_.,\[\]\-\/ ]*)}}/g
368+
);
369+
if (variables) {
370+
for (let variable of variables) {
371+
let replacement = values[variable];
372+
if (key === 'src' && variable === '{{source}}' && Buffer.isBuffer(source)) {
373+
replacement = `data:${mimeType};base64,${source.toString('base64')}`;
374+
}
375+
data.object[key] = data.object[
376+
key
377+
].replace(variable, replacement);
321378
}
322-
newObject.object[key] = newObject.object[
323-
key
324-
].replace(variable, replacement);
325379
}
326380
}
327381
}
328382
}
329383

330384
if (skip !== "directory") {
331-
if (!newObject.object._id)
332-
newObject.$filter = {
333-
query: {
334-
pathname
335-
}
336-
};
337-
338-
response = await runStore(newObject);
339-
console.log(
340-
`Uploaded: ${entryPath}/${file}`,
341-
`To: ${pathname}`
342-
);
343-
385+
response = await runStore(data);
386+
if (isData) {
387+
console.log(
388+
`Saved: ${entryPath}/${file}`
389+
);
390+
} else {
391+
console.log(
392+
`Uploaded: ${entryPath}/${file}`,
393+
`To: ${pathname}`
394+
);
395+
}
344396
if (response.error) errorLog.push(response.error);
345397
}
346398

@@ -349,7 +401,7 @@ module.exports = async function file(
349401
if (entry.endsWith("/")) newEntry = entry + name;
350402
else newEntry = entry + "/" + name;
351403

352-
await runFiles(directory, newEntry, exclude, pathname, name);
404+
await runFiles(directory, newEntry, pathname, name);
353405
}
354406
}
355407
// if (errorLog.length)
@@ -389,6 +441,57 @@ module.exports = async function file(
389441
}
390442
}
391443

444+
async function getData(filePath, mimeType, isSymlink) {
445+
let resolvedPath = filePath;
446+
if (isSymlink) {
447+
resolvedPath = await realpathAsync(filePath);
448+
}
449+
450+
try {
451+
const fileMimeType = mimeTypes[path.extname(resolvedPath)] || "text/plain";
452+
if (fileMimeType === "application/json") {
453+
// Parse JSON files
454+
return JSON.parse(fs.readFileSync(resolvedPath, "utf8"));
455+
} else if (
456+
fileMimeType === "application/javascript" ||
457+
fileMimeType === "text/javascript"
458+
) {
459+
// Try CommonJS require first (fast path)
460+
try {
461+
// clear require cache to ensure fresh load
462+
delete require.cache[require.resolve(resolvedPath)];
463+
return require(resolvedPath);
464+
} catch (err) {
465+
// If require fails due to ESM syntax (export / import), fall back to dynamic import
466+
const isESMSyntaxError =
467+
err instanceof SyntaxError ||
468+
/Unexpected token 'export'/.test(err.message) ||
469+
/Cannot use import statement outside a module/.test(err.message) ||
470+
/Unexpected token 'import'/.test(err.message);
471+
472+
if (isESMSyntaxError) {
473+
try {
474+
const module = await import(pathToFileURL(resolvedPath).href);
475+
// return default export when present otherwise return full module
476+
return module && module.default ? module.default : module;
477+
} catch (impErr) {
478+
console.error(`Failed to dynamic-import module: ${resolvedPath}`, impErr);
479+
throw impErr;
480+
}
481+
}
482+
// rethrow original require error if it's not an ESM issue
483+
throw err;
484+
}
485+
} else {
486+
return fs.readFileSync(resolvedPath, "utf8");
487+
}
488+
} catch (error) {
489+
console.error(`Failed to process file: ${resolvedPath}`, error);
490+
return "";
491+
}
492+
493+
}
494+
392495
/**
393496
* Store files by config sources
394497
**/

0 commit comments

Comments
 (0)