Skip to content

Commit 4455206

Browse files
authored
Fix to many files open (#35)
* add semaphore to limit number of files open * made request async instead of sync
1 parent 72ce764 commit 4455206

File tree

5 files changed

+70
-34
lines changed

5 files changed

+70
-34
lines changed

deno.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
"@cloudydeno/kubernetes-apis": "jsr:@cloudydeno/[email protected]",
2626
"@cloudydeno/kubernetes-client": "jsr:@cloudydeno/[email protected]",
2727
"@deno-library/compress": "jsr:@deno-library/compress@^0.5.6",
28+
"@henrygd/semaphore": "jsr:@henrygd/semaphore@^0.0.2",
2829
"@std/yaml": "jsr:@std/[email protected]"
2930
}
3031
}

deno.lock

Lines changed: 5 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

src/logic/core.js

Lines changed: 46 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,33 @@
11
import { stringify as toYaml } from '@std/yaml';
22
import { tgz } from '@deno-library/compress';
33
import { getPodLogs } from './k8s.js';
4+
import { getSemaphore } from '@henrygd/semaphore';
45

5-
export function writeYaml(data, name, dirPath) {
6-
Deno.mkdirSync(dirPath, { recursive: true });
7-
const filePath = `${dirPath}/${name}.yaml`;
8-
Deno.writeTextFileSync(filePath, toYaml(data, { skipInvalid: true }));
6+
const semaphore = getSemaphore('supportPackageSemaphore', 10);
7+
8+
export async function writeYaml(data, name, dirPath) {
9+
await semaphore.acquire();
10+
try {
11+
await Deno.mkdir(dirPath, { recursive: true });
12+
const filePath = `${dirPath}/${name}.yaml`;
13+
await Deno.writeTextFile(filePath, toYaml(data, { skipInvalid: true }));
14+
} finally {
15+
semaphore.release();
16+
}
917
}
1018

1119
export async function preparePackage(dirPath) {
12-
const supportPackageZip = `${dirPath}.tar.gz`;
13-
console.log('Preparing the Support Package');
14-
await tgz.compress(dirPath, supportPackageZip);
15-
console.log('Cleaning up temp directory');
16-
Deno.removeSync(dirPath, { recursive: true });
17-
console.log(`\nPlease attach ${supportPackageZip} to your support ticket.`);
20+
await semaphore.acquire();
21+
try {
22+
const supportPackageZip = `${dirPath}.tar.gz`;
23+
console.log('Preparing the Support Package');
24+
await tgz.compress(dirPath, supportPackageZip);
25+
console.log('Cleaning up temp directory');
26+
await Deno.remove(dirPath, { recursive: true });
27+
console.log(`\nPlease attach ${supportPackageZip} to your support ticket.`);
28+
} finally {
29+
semaphore.release();
30+
}
1831
}
1932

2033
export async function processData(dirPath, k8sResources) {
@@ -29,16 +42,21 @@ export async function processData(dirPath, k8sResources) {
2942

3043
if (k8sType == 'pods') {
3144
for (const pod of resources.items) {
32-
delete pod.metadata.managedFields;
45+
await semaphore.acquire();
46+
try {
47+
delete pod.metadata.managedFields;
3348

34-
writeYaml(pod, `spec_${pod.metadata.name}`, `${dirPath}/${k8sType}/${pod.metadata.name}`);
49+
await writeYaml(pod, `spec_${pod.metadata.name}`, `${dirPath}/${k8sType}/${pod.metadata.name}`);
3550

36-
const logs = await getPodLogs(pod);
37-
for (const [containerName, logData] of Object.entries(logs)) {
38-
Deno.writeTextFileSync(
39-
`${dirPath}/${k8sType}/${pod.metadata.name}/log_${containerName}.log`,
40-
logData,
41-
);
51+
const logs = await getPodLogs(pod);
52+
for (const [containerName, logData] of Object.entries(logs)) {
53+
await Deno.writeTextFile(
54+
`${dirPath}/${k8sType}/${pod.metadata.name}/log_${containerName}.log`,
55+
logData,
56+
);
57+
}
58+
} finally {
59+
semaphore.release();
4260
}
4361
}
4462
continue;
@@ -60,14 +78,19 @@ export async function processData(dirPath, k8sResources) {
6078
const header = 'LAST SEEN\tTYPE\tREASON\tOBJECT\tMESSAGE\n';
6179
const content = header + formattedEvents.join('\n');
6280

63-
Deno.writeTextFileSync(`${dirPath}/${k8sType}.csv`, content);
81+
await Deno.writeTextFile(`${dirPath}/${k8sType}.csv`, content);
6482

6583
continue;
6684
}
6785

68-
resources.items.map((data) => {
69-
delete data.metadata.managedFields;
70-
writeYaml(data, `${data.metadata.name}_get`, `${dirPath}/${k8sType}`);
71-
});
86+
await Promise.all(resources.items.map(async (data) => {
87+
await semaphore.acquire();
88+
try {
89+
delete data.metadata.managedFields;
90+
await writeYaml(data, `${data.metadata.name}_get`, `${dirPath}/${k8sType}`);
91+
} finally {
92+
semaphore.release();
93+
}
94+
}));
7295
}
7396
}

src/onprem.js

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -27,14 +27,21 @@ export async function onprem(namespace) {
2727
}
2828

2929
if (cfCreds) {
30-
const accounts = await getAllAccounts(cfCreds);
31-
writeYaml(accounts, 'OnPrem_Accounts', dirPath);
32-
const runtimes = await getAllRuntimes(cfCreds);
33-
writeYaml(runtimes, 'OnPrem_Runtimes', dirPath);
34-
const featureFlags = await getSystemFeatureFlags(cfCreds);
35-
writeYaml(featureFlags, 'OnPrem_Feature_Flags', dirPath);
36-
const totalUsers = await getTotalUsers(cfCreds);
37-
writeYaml(totalUsers, 'OnPrem_Total_Users', dirPath);
30+
const dataFetchers = [
31+
{ name: 'OnPrem_Accounts', fetcher: getAllAccounts },
32+
{ name: 'OnPrem_Runtimes', fetcher: getAllRuntimes },
33+
{ name: 'OnPrem_Feature_Flags', fetcher: getSystemFeatureFlags },
34+
{ name: 'OnPrem_Total_Users', fetcher: getTotalUsers },
35+
];
36+
37+
for (const { name, fetcher } of dataFetchers) {
38+
try {
39+
const data = await fetcher(cfCreds);
40+
await writeYaml(data, name, dirPath);
41+
} catch (error) {
42+
console.error(`Failed to fetch or write ${name}:`, error.message);
43+
}
44+
}
3845
}
3946

4047
console.log(`Gathering data in the '${namespace}' namespace for Codefresh OnPrem`);

src/pipelines.js

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,11 +30,11 @@ export async function pipelines(namespace, runtime) {
3030
} while (isNaN(selection) || selection < 1 || selection > runtimes.length);
3131

3232
const reSpec = runtimes[selection - 1];
33-
writeYaml(reSpec, 'Runtime_Spec', dirPath);
33+
await writeYaml(reSpec, 'Runtime_Spec', dirPath);
3434
}
3535
} else {
36-
const reSpec = getRuntimeSpec(cfCreds, runtime);
37-
writeYaml(reSpec, 'Runtime_Spec', dirPath);
36+
const reSpec = await getRuntimeSpec(cfCreds, runtime);
37+
await writeYaml(reSpec, 'Runtime_Spec', dirPath);
3838
}
3939
}
4040

0 commit comments

Comments
 (0)