passing tests

db/CURRENT

@@ -1 +1 @@
-MANIFEST-000232
+MANIFEST-000296

db/LOG

@@ -1,12 +1,3 @@
-2018/01/15-12:39:49.619498 70000a01a000 Recovering log #223
-2018/01/15-12:39:49.627377 70000a01a000 Level-0 table #233: started
-2018/01/15-12:39:49.634932 70000a01a000 Level-0 table #233: 401226 bytes OK
-2018/01/15-12:39:49.636255 70000a01a000 Delete type=2 #5
-2018/01/15-12:39:49.636502 70000a01a000 Delete type=2 #16
-2018/01/15-12:39:49.637256 70000a01a000 Delete type=2 #17
-2018/01/15-12:39:49.637585 70000a01a000 Delete type=2 #18
-2018/01/15-12:39:49.638321 70000a01a000 Delete type=2 #19
-2018/01/15-12:39:49.638411 70000a01a000 Delete type=2 #21
-2018/01/15-12:39:49.638969 70000a01a000 Delete type=2 #23
-2018/01/15-12:39:49.639074 70000a01a000 Delete type=0 #223
-2018/01/15-12:39:49.639347 70000a01a000 Delete type=3 #22
+2018/01/15-21:07:11.409573 700005565000 Recovering log #295
+2018/01/15-21:07:11.412481 700005565000 Delete type=0 #295
+2018/01/15-21:07:11.412591 700005565000 Delete type=3 #294

index.js

@@ -0,0 +1,16 @@
+const path = require('path')
+const db = require('level')(path.join(__dirname, 'db'))
+
+module.exports = {
+  get: async (key) => {
+    const data = await db.get(key)
+    if (!data) return null
+    return JSON.parse(data)
+  },
+
+  set: async (key, value) => {
+    await db.set(key, JSON.stringify(value, null, 2))
+  },
+
+  db: db
+}

package.json

@@ -6,7 +6,15 @@
   "license": "MIT",
   "devDependencies": {
     "all-the-package-names": "^1.3440.0",
+    "chai": "^4.1.2",
+    "chalk": "^2.3.0",
+    "is-number": "^4.0.0",
     "level": "^2.1.1",
+    "lodash": "^4.17.4",
+    "mocha": "^4.1.0",
     "urlencode": "^1.1.0"
+  },
+  "scripts": {
+    "test": "mocha"
   }
 }

populate.js

@@ -1,27 +1,29 @@
-
 const db = require('level')('./db')
 const fs = require('fs')
 const path = require('path')
 const urlencode = require('urlencode')
+const {gray, yellow} = require('chalk')
+const {sum} = require('lodash')
 
-let extant = 0
-let nonexistent = 0
 db.createKeyStream()
   .on('data', async (name) => {
     const dataFile = path.join(__dirname, '../sourceranks/data', urlencode(name) + '.json')
-    if (fs.existsSync(dataFile)) {
-      await saveRank(name, dataFile)
-      console.log(dataFile, 'exists')
-      extant++
+    let data = {}
+    if (fs.existsSync(dataFile)) { 
+      const scores = require(dataFile)
+      data = Object.assign({}, data, {
+        scores: scores,
+        total: sum(Object.values(scores))
+      })
+      process.stdout.write(gray('x'))
     } else {
-      nonexistent++
-      console.log(dataFile, 'DOES NOT EXIST')
+      data.notFound = true
+      process.stdout.write(yellow('x'))
     }
+
+    data.updatedAt = new Date()
+    await db.put(name, JSON.stringify(data, null, 2))
   })
   .on('end', () => {
-    console.log('done', extant, nonexistent)
-  })
-
-async function saveRank (name, dataFile) {
-  db.put(name, JSON.stringify(require(dataFile), null, 2))
-}
+    console.log('done')
+  })

seed.js

@@ -1,5 +1,11 @@
 const db = require('level')('./db')
-const names = require('all-the-package-names')
+const {chain} = require('lodash')
+const isNumber = require('is-number')
+const names = chain(require('all-the-package-names'))
+  .map(name => name.trim())
+  .filter(name => !isNumber(name))
+  .uniq()
+  .value()
 
 for (i = 0; i < names.length; i++) {
   const name = names[i]

test/index.js

@@ -0,0 +1,34 @@
+require('chai').should()
+
+const {describe, it} = require('mocha')
+const test = it
+const sourceranks = require('..')
+
+describe('sourcerank-data', function () {
+  this.timeout(120 * 1000)
+
+  test('.get()', async () => {
+    const express = await sourceranks.get('express')
+    express.should.be.an('object')
+    express.total.should.be.above(25)
+  })
+
+  it('exposes a streamable leveldb instance', (done) => {
+    let bad = 0
+    let good = 0
+    var stream = sourceranks.db.createReadStream()
+      .on('data', ({key, value}) => {
+        value = JSON.parse(value)
+        if (value.scores && value.total) {
+          good++
+        } else {
+          bad++
+        }
+      })
+      .on('end', () => {
+        good.should.be.above(500 * 1000)
+        bad.should.be.above(10 * 1000).and.below(20 * 1000)
+        done()
+      })
+  })
+})