all the data

nice-registry · Jan 15, 2018 · 5d1bf7c · 5d1bf7c
1 parent fc3b4a3
commit 5d1bf7c
Show file tree

Hide file tree

Showing 44 changed files with 532 additions and 10 deletions.
diff --git a/check.js b/check.js
@@ -0,0 +1,18 @@
+
+const db = require('level')('./db')
+const fs = require('fs')
+const path = require('path')
+const urlencode = require('urlencode')
+let good = 0
+let bad = 0 
+
+db.createReadStream()
+  .on('data', async ({key, value}) => {
+    value = JSON.parse(value)
+    if (value && Object.keys(value).length) {
+      good++
+    } else {
+      bad++
+    }
+    console.log(good, bad)
+  })
diff --git a/db/.lev_history b/db/.lev_history
diff --git a/db/000005.ldb b/db/000005.ldb
diff --git a/db/000028.ldb b/db/000028.ldb
diff --git a/db/000029.ldb b/db/000029.ldb
diff --git a/db/000030.ldb b/db/000030.ldb
diff --git a/db/000031.ldb b/db/000031.ldb
diff --git a/db/000032.ldb b/db/000032.ldb
diff --git a/db/000045.ldb b/db/000045.ldb
diff --git a/db/000046.ldb b/db/000046.ldb
diff --git a/db/000091.ldb b/db/000091.ldb
diff --git a/db/000101.ldb b/db/000101.ldb
diff --git a/db/000112.ldb b/db/000112.ldb
diff --git a/db/000123.ldb b/db/000123.ldb
diff --git a/db/000134.ldb b/db/000134.ldb
diff --git a/db/000138.ldb b/db/000138.ldb
diff --git a/db/000139.ldb b/db/000139.ldb
diff --git a/db/000140.ldb b/db/000140.ldb
diff --git a/db/000141.ldb b/db/000141.ldb
diff --git a/db/000142.ldb b/db/000142.ldb
diff --git a/db/000176.ldb b/db/000176.ldb
diff --git a/db/000180.ldb b/db/000180.ldb
diff --git a/db/000181.ldb b/db/000181.ldb
diff --git a/db/000182.ldb b/db/000182.ldb
diff --git a/db/000183.ldb b/db/000183.ldb
diff --git a/db/000184.ldb b/db/000184.ldb
diff --git a/db/000218.ldb b/db/000218.ldb
diff --git a/db/000225.ldb b/db/000225.ldb
diff --git a/db/000226.ldb b/db/000226.ldb
diff --git a/db/000227.ldb b/db/000227.ldb
diff --git a/db/000228.ldb b/db/000228.ldb
diff --git a/db/000229.ldb b/db/000229.ldb
diff --git a/db/000230.ldb b/db/000230.ldb
diff --git a/db/000231.ldb b/db/000231.ldb
diff --git a/db/000233.ldb b/db/000233.ldb
diff --git a/db/000027.log → db/000234.log b/db/000027.log → db/000234.log
diff --git a/db/CURRENT b/db/CURRENT
@@ -1 +1 @@
-MANIFEST-000025
+MANIFEST-000232
diff --git a/db/LOG b/db/LOG
@@ -0,0 +1,12 @@
+2018/01/15-12:39:49.619498 70000a01a000 Recovering log #223
+2018/01/15-12:39:49.627377 70000a01a000 Level-0 table #233: started
+2018/01/15-12:39:49.634932 70000a01a000 Level-0 table #233: 401226 bytes OK
+2018/01/15-12:39:49.636255 70000a01a000 Delete type=2 #5
+2018/01/15-12:39:49.636502 70000a01a000 Delete type=2 #16
+2018/01/15-12:39:49.637256 70000a01a000 Delete type=2 #17
+2018/01/15-12:39:49.637585 70000a01a000 Delete type=2 #18
+2018/01/15-12:39:49.638321 70000a01a000 Delete type=2 #19
+2018/01/15-12:39:49.638411 70000a01a000 Delete type=2 #21
+2018/01/15-12:39:49.638969 70000a01a000 Delete type=2 #23
+2018/01/15-12:39:49.639074 70000a01a000 Delete type=0 #223
+2018/01/15-12:39:49.639347 70000a01a000 Delete type=3 #22
diff --git a/db/LOG.old b/db/LOG.old
diff --git a/db/MANIFEST-000025 b/db/MANIFEST-000025
diff --git a/db/MANIFEST-000232 b/db/MANIFEST-000232
diff --git a/package.json b/package.json
@@ -6,6 +6,7 @@
   "license": "MIT",
   "devDependencies": {
     "all-the-package-names": "^1.3440.0",
-    "level": "^2.1.1"
+    "level": "^2.1.1",
+    "urlencode": "^1.1.0"
   }
-}
+}
diff --git a/populate.js b/populate.js
@@ -0,0 +1,27 @@
+
+const db = require('level')('./db')
+const fs = require('fs')
+const path = require('path')
+const urlencode = require('urlencode')
+
+let extant = 0
+let nonexistent = 0
+db.createKeyStream()
+  .on('data', async (name) => {
+    const dataFile = path.join(__dirname, '../sourceranks/data', urlencode(name) + '.json')
+    if (fs.existsSync(dataFile)) {
+      await saveRank(name, dataFile)
+      console.log(dataFile, 'exists')
+      extant++
+    } else {
+      nonexistent++
+      console.log(dataFile, 'DOES NOT EXIST')
+    }
+  })
+  .on('end', () => {
+    console.log('done', extant, nonexistent)
+  })
+
+async function saveRank (name, dataFile) {
+  db.put(name, JSON.stringify(require(dataFile), null, 2))
+}
diff --git a/seed.js b/seed.js
@@ -8,5 +8,5 @@ for (i = 0; i < names.length; i++) {
 }
 
 async function saveName (name) {
-  await db.put(name, {})
+  await db.put(name, JSON.stringify({}))
 }