Add github actions workflow for uploading to IPFS and fix uploader

Author: lucas-dydx

.github/workflows/upload-ipfs.yml

@@ -0,0 +1,42 @@
+name: Upload IPFS
+
+on:
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  upload-to-ipfs: 
+    name: dip-uploader
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v2
+
+      - name: Rename dip to upper case
+        run: cd content/dips && bash toUpper.sh || true && cd ../..
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v1
+        with:
+          node-version: 12.x
+
+      - name: Install dependencies
+        run: npm install
+      
+      - name: Parse Markdown
+        run: npm run parse-markdown
+
+      - name: Upload to Pinata and the Graph IPFS
+        env:
+          PINATA_KEY: ${{ secrets.PINATA_KEY }}
+          PINATA_SECRET: ${{ secrets.PINATA_SECRET }}
+        run: npm run upload:pinata-thegraph
+
+      - name: Commit changes
+        uses: EndBug/add-and-commit@v6
+        with:
+          author_name: GitHub Actions
+          author_email: 41898282+github-actions[bot]@users.noreply.github.com
+          message: '[PIPELINE] Committed IPFS uploads'
+          add: '.'

.gitignore

@@ -0,0 +1 @@
+/node_modules

dip-uploader.js

@@ -2,63 +2,91 @@
 const bs58 = require('bs58');
 const fs = require('fs')
 const fetch = require('node-fetch')
-const { exec, spawn } = require('child_process');
+const { execSync } = require('child_process');
+const { exit } = require('process');
 
-const { PINATA_KEY, PINATA_SECRET} = process.env;
+const { PINATA_KEY, PINATA_SECRET } = process.env;
 
 const pinataEndpoint = 'https://api.pinata.cloud/pinning/pinJSONToIPFS'
 
+async function delay(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
 
-jsonAips = JSON.parse(fs.readFileSync('./content/ipfs-aips/all-aips.json').toString());
-Object.keys(jsonAips).forEach((id) => {
-  delete Object.assign(jsonAips[id], {'description': jsonAips[id]['content'] })['content'];
-  fetch(pinataEndpoint, {
-    method: 'POST',
-    body: JSON.stringify({
-      pinataOptions: { cidVersion: 0 },
-      pinataContent: jsonAips[id],
-    }),
-    headers: {
-      'Content-Type': 'application/json',
-      pinata_api_key: PINATA_KEY,
-      pinata_secret_api_key: PINATA_SECRET,
-    },
-  })
-  .then(res => {
-    return res.json()
-  }).then(result => {
-    if (result.error) throw { message: result.error }
-    const hash = result.IpfsHash
-    const encodedHash = `0x${
-      bs58
-      .decode(hash)
-      .slice(2)
-      .toString('hex')
-    }`
-    jsonAips[id].ipfsHash = hash
-    jsonAips[id].encodeIpfsHash = encodedHash
-    console.log('✅ Success!')
-    console.log(` IPFS hash: ${hash}`)
-    console.log(` Encoded IPFS hash (for proposal creation): ${encodedHash}`)
-    console.log(` See the file here: https://gateway.pinata.cloud/ipfs/${hash}`)
-    fs.writeFileSync(
-      `./content/ipfs-aips/${id}-Ipfs-hashes.json`,
-      JSON.stringify({aip: id, hash, encodedHash}, null, 2)
-    )
-    fs.writeFileSync(
-      `./content/ipfs-aips/all-aips.json`,
-      JSON.stringify(jsonAips, null, 2)
-    )
-    exec(`curl https://gateway.pinata.cloud/ipfs/${hash} > tmp && curl -sF file='@./tmp' https://api.thegraph.com/ipfs/api/v0/add`, (err, stdout, stderr) => {
-      if(err) {
-        throw new Error(stderr)
+const rawJsonDips = JSON.parse(fs.readFileSync('./content/ipfs-dips/all-dips.json').toString());
+const jsonDips = Object.values(rawJsonDips);
+
+async function main() {
+  const dipIds = Object.keys(jsonDips).sort(((a, b) => a.split('-')[1] - b.split('-')[1]));
+
+  for (let x = 0; x < dipIds.length; x++) {
+    const id = dipIds[x];
+
+    delete Object.assign(jsonDips[id], { 'description': jsonDips[id]['content'] })['content'];
+    
+    try {
+      const res = await fetch(pinataEndpoint, {
+        method: 'POST',
+        body: JSON.stringify({
+          pinataOptions: { cidVersion: 0 },
+          pinataContent: jsonDips[id],
+        }),
+        headers: {
+          'Content-Type': 'application/json',
+          pinata_api_key: PINATA_KEY,
+          pinata_secret_api_key: PINATA_SECRET,
+        },
+      })
+
+      if (!res.ok) {
+        throw Error(await res.text());
       }
-      console.log(stdout)
-    });
-  }).catch((err) => {
-    console.log('error:', err);
-  })
-})
+
+      const result = await res.json()
+
+      if (result.error) throw { message: result.error }
+
+      const hash = result.IpfsHash
+      const encodedHash = `0x${bs58
+        .decode(hash)
+        .slice(2)
+        .toString('hex')
+        }`
+      jsonDips[id].ipfsHash = hash
+      jsonDips[id].encodeIpfsHash = encodedHash
+      console.log(`${jsonDips[id].title}: ✅ Success!`)
+      console.log(` IPFS hash: ${hash}`)
+      console.log(` Encoded IPFS hash (for proposal creation): ${encodedHash}`)
+      console.log(` See the file here: https://gateway.pinata.cloud/ipfs/${hash}`)
+      fs.writeFileSync(
+        `./content/ipfs-dips/${jsonDips[id].basename}-Ipfs-hashes.json`,
+        JSON.stringify({ dip: id, hash, encodedHash }, null, 2)
+      )
+      fs.writeFileSync(
+        `./content/ipfs-dips/all-dips.json`,
+        JSON.stringify(jsonDips, null, 2)
+      )
+      await delay(250);
+      execSync(`curl -s https://gateway.pinata.cloud/ipfs/${hash} > tmp && curl -sF file='@./tmp' https://api.thegraph.com/ipfs/api/v0/add`, (err, stdout, stderr) => {
+        if (err) {
+          throw new Error(stderr)
+        }
+        console.log(stdout)
+      });
+      await delay(250);
+    } catch (error) {
+      console.error(`Error during main loop: ${error}`)
+      throw error;
+    }
+  }
+}
 
 
-  
+(async () => {
+  try {
+    await main();
+  } catch (e) {
+    console.error(`Exiting [dip-uploader] process due next error: \n ${error}`)
+    exit(1)
+  }
+})();

package-lock.json

@@ -49,25 +49,18 @@
     "gh-pages": "^2.2.0",
     "prettier": "2.0.4"
   },
-  "homepage": "https://github.com/dYdX/dip",
+  "homepage": "https://github.com/dydxfoundation/dip",
   "keywords": [
     "gatsby"
   ],
   "main": "n/a",
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/dYdX/dip"
+    "url": "git+https://github.com/dydxfoundation/dip"
   },
   "scripts": {
-    "build": "gatsby build --prefix-paths",
-    "develop": "gatsby develop",
     "parse-markdown": "m2j ./content/dips/*.md -c -o ./content/ipfs-dips/all-dips.json",
     "upload:pinata-thegraph": "node dip-uploader.js && rm tmp",
-    "format": "prettier --write \"**/*.{js,jsx,json,md}\"",
-    "start": "npm run develop",
-    "serve": "gatsby serve --prefix-paths",
-    "clean": "gatsby clean",
-    "test": "echo \"Write tests! -> https://gatsby.dev/unit-testing\" && exit 1",
-    "deploy": "gatsby build --prefix-paths && gh-pages -d public"
+    "format": "prettier --write \"**/*.{js,jsx,json,md}\""
   }
 }