smaller image size, verbose flag, and lib search

mrfrase3 · mrfrase3 · commit bb51bb96ae71 · 2021-02-01T11:08:38.000+08:00
diff --git a/.dockerignore b/.dockerignore
@@ -1,4 +1,5 @@
-base-all
+base-all/data/arduino
+base-all/data/arduino-downloads
 Arduino
 node_modules
 .env
diff --git a/Dockerfile b/Dockerfile
@@ -1,16 +1,52 @@
-FROM duinoapp/arduino-cli-base:latest
+FROM node:10-buster
+
+USER root
+RUN useradd duino
+
+COPY setup/index.js /home/duino/setup/index.js
+COPY setup/arduino-cli.yml /home/duino/data/arduino-cli.yml
+# COPY Arduino /home/duino/Arduino
+
+# RUN mkdir /home/duino
+RUN chown duino:duino /home/duino -R
+RUN apt-get update && apt-get install build-essential -y
+
+WORKDIR /home/duino
+USER duino
+
+RUN wget https://github.com/arduino/arduino-cli/releases/download/0.13.0/arduino-cli_0.13.0_Linux_64bit.tar.gz -O - | tar -xz
+
+RUN ls
+
+
+ENV CLI_ARGS="--config-file /home/duino/data/arduino-cli.yml --format json"
+RUN ./arduino-cli core update-index ${CLI_ARGS}
+RUN ./arduino-cli core search "" ${CLI_ARGS} > /home/duino/data/cores.json
+
+RUN ./arduino-cli lib update-index ${CLI_ARGS}
+RUN ./arduino-cli lib search "" ${CLI_ARGS} > /home/duino/data/libs.json
+
+RUN node setup/ cores
+RUN node setup/ boards
 
 USER root
 COPY package*.json /home/duino/
 RUN chown duino:duino /home/duino/package*.json
+RUN apt-get install nano
 USER duino
 
 RUN npm ci
 
 USER root
 COPY src /home/duino/src
 RUN chown duino:duino /home/duino/src -R
+RUN mkdir /data
+RUN chmod 0777 /data
+COPY entrypoint.sh /home/duino/
+COPY load-libs.sh /home/duino/
+RUN chmod +x ./*.sh
+RUN chown duino:duino ./*.sh
 USER duino
 
 EXPOSE 3030
-CMD [ "node", "src/index.js" ]
+CMD [ "./entrypoint.sh" ]
diff --git a/base-all/data/.dockerignore b/base-all/data/.dockerignore
diff --git a/base-all/data/Dockerfile b/base-all/data/Dockerfile
diff --git a/base-all/data/arduino-downloads/.gitkeep b/base-all/data/arduino-downloads/.gitkeep
diff --git a/base-all/data/arduino/.gitkeep b/base-all/data/arduino/.gitkeep
diff --git a/deployment/docker-compose.yml b/deployment/docker-compose.yml
@@ -25,11 +25,15 @@ services:
     image: duinoapp/duinoapp-server:latest
     container_name: duinoapp
     restart: unless-stopped
+    volumes:
+      - duinoapp-lib-downloads:/data
     environment: # update these, server info displayed to users
       - SERVER_INFO_NAME=Example Server
       - SERVER_INFO_LOCATION=Australia/Sydney
       # must be a two letter country code
       - SERVER_INFO_COUNTRY=AU
       - SERVER_INFO_OWNER=Jane Smith
       - SERVER_INFO_WEBSITE=site.example.com
-      - SERVER_INFO_DESCRIPTION=Just Another Example Server
+      - SERVER_INFO_DESCRIPTION=Just Another Example Server
+volumes:
+  duinoapp-lib-downloads:
diff --git a/entrypoint.sh b/entrypoint.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+./load-libs.sh
+
+node src/index.js
diff --git a/load-libs.sh b/load-libs.sh
@@ -0,0 +1,6 @@
+file="/data/arduino.tar.gz"
+uri="http://duino-dump.s3-website.us-east-2.amazonaws.com/arduino.tar.gz"
+
+curl -o "$file" -z "$file" "$uri"
+rm -rf /home/duino/Arduino
+tar -xzf $file
diff --git a/setup/arduino-cli.yml b/setup/arduino-cli.yml
@@ -1,5 +1,6 @@
 board_manager:
   additional_urls:
     - http://arduino.esp8266.com/stable/package_esp8266com_index.json
+    - https://raw.githubusercontent.com/espressif/arduino-esp32/gh-pages/package_esp32_index.json
 arduino_data: '/home/duino/data/arduino/'
 downloads_dir: '/home/duino/data/arduino-downloads/'
diff --git a/setup/index.js b/setup/index.js
diff --git a/src/actions/info.js b/src/actions/info.js
@@ -5,6 +5,8 @@ const cores = require('../../data/cores.json');
 const libs = require('../../data/libs.json');
 const boards = require('../../data/boards.json');
 
+const searchRegex = (search = '') => new RegExp(`(${search.replace(/[^\w-\s]/g, '.').split(/\s+/g).join(')|(')})`, 'i');
+
 module.exports.server = (socket, done) => done && done({
   name: process.env.SERVER_INFO_NAME || 'Test Compile Server',
   location: process.env.SERVER_INFO_LOCATION || 'Unknown',
@@ -45,7 +47,13 @@ let processedBoards = boards;
 processedCores.forEach((core) => {
   const [pack, arch] = core.id.split(':');
   const path = `/home/duino/.arduino15/packages/${pack}/hardware/${arch}/${core.latest}/boards.txt`;
-  const props = processFile(path);
+  let props;
+  try {
+    props = processFile(path);
+  } catch (err) {
+    if (!err.message.includes('no such file or directory')) throw err;
+    props = processFile(path.replace('arduino-beta', 'arduino'));
+  }
   processedBoards = processedBoards.map((board) => (!board.fqbn.includes(`${core.id}:`) ? board : {
     ...board,
     properties: props[board.fqbn.split(':')[2]],
@@ -72,6 +80,23 @@ processedBoards
     }
   });
 
+module.exports.librariesSearch = (data, socket, done) => {
+  if (!done) return;
+  const limit = Math.min(Math.max(data.limit || 10, 1), 100);
+  const skip = Math.max(data.skip || 0, 0);
+  const reg = searchRegex(data.search);
+  const { sortBy = 'name', sortDesc = false } = data;
+  const res = processedLibs.filter((lib) => reg.test(lib.name)).sort((a, b) => {
+    const ai = `${a[sortBy]}`.toLowerCase();
+    const bi = `${b[sortBy]}`.toLowerCase();
+    if (ai === bi) return 0;
+    return (ai < bi ? -1 : 1) * (sortDesc ? -1 : 1);
+  });
+  done({
+    limit, skip, total: res.length, data: res.slice(skip * limit, (skip + 1) * limit),
+  });
+};
+
 module.exports.libraries = (socket, done) => done && done(processedLibs);
 
 module.exports.cores = (socket, done) => done && done(processedCores);
diff --git a/src/actions/program.js b/src/actions/program.js
@@ -19,26 +19,29 @@ const program = {
     return {};
   },
 
-  compile: async ({ fqbn, files, noHex = false }, socket, done) => {
+  compile: async ({
+    fqbn, files, noHex = false, flags = {},
+  }, socket, done) => {
     await tmpFiles.loadTempFiles(files, socket);
     const res = await cli('compile', [
-      '-v',
+      ...(flags.verbose ? ['-v'] : []),
       '--warnings', 'all',
       '--fqbn', fqbn,
-      ...(!noHex ? ['--output', `${socket.sketchPath}/output`] : []),
+      ...(!noHex ? ['--output-dir', `${socket.sketchPath}/output`] : []),
       socket.sketchPath,
-    ], socket);
+    ], socket, { noJson: true });
     const response = program.getError(res);
     if (!response.error && !noHex) {
-      const hex = await fs.readFile(`${socket.sketchPath}/output.hex`, 'base64');
+      const ref = socket.sketchPath.split('/').pop();
+      const hex = await fs.readFile(`${socket.sketchPath}/output/${ref}.ino.hex`, 'base64');
       response.hex = hex;
     }
     // tmpFiles.cleanup(socket);
     if (done) done(response);
     return response;
   },
 
-  upload: async ({ id = program.randId(), fqbn }, socket, done) => {
+  upload: async ({ id = program.randId(), fqbn, flags = {} }, socket, done) => {
     if (socket.uploading) return { Message: 'Already uploading, please hold.', Cause: 'Upload already initiated.' };
     socket.uploading = id;
     socket.emit('upload.id', id);
@@ -58,7 +61,13 @@ const program = {
     const port = 4000 + Math.floor(Math.random() * 1000);
     sock.listen(port);
 
-    const res = await cli('upload', ['-p', `net:localhost:${port}`, '-v', '--fqbn', fqbn, socket.sketchPath], socket);
+    const res = await cli('upload', [
+      '-p', `net:localhost:${port}`,
+      ...(flags.verbose ? ['-v'] : []),
+      ...(flags.programmer ? ['-P', flags.programmer] : []),
+      '--fqbn', fqbn,
+      socket.sketchPath,
+    ], socket, { noJson: true });
 
     sock.close();
     socket.uploading = null;
@@ -78,10 +87,10 @@ const program = {
       'all',
       '--fqbn',
       fqbn,
-      '--output',
+      '--output-dir',
       `${session.sketchPath}/legacy`,
       session.sketchPath,
-    ]);
+    ], null, { noJson: true });
     const response = program.getError(res);
     if (response.error) {
       return {
@@ -92,7 +101,8 @@ const program = {
         stderr: res,
       };
     }
-    const hex = await fs.readFile(`${session.sketchPath}/legacy.hex`, 'base64');
+    const ref = session.sketchPath.split('/').pop();
+    const hex = await fs.readFile(`${session.sketchPath}/legacy/${ref}.ino.hex`, 'base64');
     tmpFiles.cleanup(session);
     return {
       success: true,
diff --git a/src/index.js b/src/index.js
@@ -36,6 +36,7 @@ io.on('connection', async (socket) => {
   socket.on('info.server', (done) => info.server(socket, done));
   socket.on('info.cores', (done) => info.cores(socket, done));
   socket.on('info.libraries', (done) => info.libraries(socket, done));
+  socket.on('info.librariesSearch', (data, done) => info.librariesSearch(data, socket, done));
   socket.on('info.boards', (done) => info.boards(socket, done));
 
   socket.on('compile.start', (data, done) => program.compile(data, socket, done));
diff --git a/src/utils/arduino-exec.js b/src/utils/arduino-exec.js
@@ -11,11 +11,13 @@ module.exports = (commands, args, socket, options) => new Promise((resolve) => {
     // console.log(data.toString('utf-8'));
     if (opts.emit) socket.emit('console.log', data.toString('utf-8'));
   };
+  let cliArgs = process.env.CLI_ARGS || `--config-file ${__dirname}/../../data/arduino-cli.yml --format json`;
+  if (opts.noJson) cliArgs = cliArgs.replace(' --format json', '');
 
   const exec = spawn(`${__dirname}/../../arduino-cli`, [
     ...(Array.isArray(commands) ? commands : commands.split('.')),
     ...(Array.isArray(args) ? args : [args]).map((arg) => `${`${arg}`.replace(/"/g, '')}`),
-    ...(process.env.CLI_ARGS || `--config-file ${__dirname}/../../data/arduino-cli.yml --format json`).split(' '),
+    ...(cliArgs).split(' '),
   ], { cwd: socket ? socket.tmpDir.path : `${__dirname}/../../` });
   exec.stdout.on('data', (data) => log(data));
   exec.stderr.on('data', (data) => log(data));
