Initial commit

Author: karmatr0n

.gitignore

@@ -0,0 +1,9 @@
+db/*
+!db/.gitkeep
+nvd_data/*
+!nvd_data/.gitkeep
+log/*
+!log/.gitkeep
+tmp/*
+!tmp/.gitkeep
+coverage

.rspec

@@ -0,0 +1,3 @@
+--colour
+--backtrace
+--format documentation

.ruby-gemset

@@ -0,0 +1 @@
+cve_server

.ruby-version

@@ -0,0 +1 @@
+ruby-2.2.3

Capfile

@@ -0,0 +1,8 @@
+require 'capistrano/setup'
+require 'capistrano/deploy'
+require 'capistrano/puma'
+require 'capistrano/puma'
+require 'capistrano/rvm'
+require 'capistrano/bundler'
+
+Dir.glob('lib/capistrano/tasks/*.rake').each { |r| import r }

Gemfile

@@ -0,0 +1,21 @@
+source 'https://rubygems.org'
+
+gem 'sinatra', '~> 1.4.6'
+gem 'sinatra-json', '~> 0.1.0'
+gem 'nokogiri', '~> 1.6.6.2'
+gem 'mongo', '~> 2.1.0'
+gem 'puma', '~> 2.13.4'
+
+group :production do
+  gem 'capistrano', '~> 3.4.0'
+  gem 'capistrano-bundler', '~> 1.1.4'
+  gem 'capistrano-rvm', '~> 0.1.2'
+  gem 'capistrano3-puma', '~> 1.2.1'
+end
+
+group :development, :test do
+  gem 'pry', '~> 0.10.1'
+  gem 'rspec', '~> 3.3.0'
+  gem 'rack-test', '~> 0.6.3'
+  gem 'simplecov', '~> 0.10.0', :require => false
+end

Gemfile.lock

@@ -0,0 +1,96 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    bson (3.2.4)
+    capistrano (3.4.0)
+      i18n
+      rake (>= 10.0.0)
+      sshkit (~> 1.3)
+    capistrano-bundler (1.1.4)
+      capistrano (~> 3.1)
+      sshkit (~> 1.2)
+    capistrano-rvm (0.1.2)
+      capistrano (~> 3.0)
+      sshkit (~> 1.2)
+    capistrano3-puma (1.2.1)
+      capistrano (~> 3.0)
+      puma (>= 2.6)
+    coderay (1.1.0)
+    colorize (0.7.7)
+    diff-lcs (1.2.5)
+    docile (1.1.5)
+    i18n (0.7.0)
+    json (1.8.3)
+    method_source (0.8.2)
+    mini_portile (0.6.2)
+    mongo (2.1.0)
+      bson (~> 3.0)
+    multi_json (1.11.1)
+    net-scp (1.2.1)
+      net-ssh (>= 2.6.5)
+    net-ssh (2.9.2)
+    nokogiri (1.6.6.2)
+      mini_portile (~> 0.6.0)
+    pry (0.10.1)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    puma (2.13.4)
+    rack (1.6.2)
+    rack-protection (1.5.3)
+      rack
+    rack-test (0.6.3)
+      rack (>= 1.0)
+    rake (10.4.2)
+    rspec (3.3.0)
+      rspec-core (~> 3.3.0)
+      rspec-expectations (~> 3.3.0)
+      rspec-mocks (~> 3.3.0)
+    rspec-core (3.3.1)
+      rspec-support (~> 3.3.0)
+    rspec-expectations (3.3.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.3.0)
+    rspec-mocks (3.3.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.3.0)
+    rspec-support (3.3.0)
+    simplecov (0.10.0)
+      docile (~> 1.1.0)
+      json (~> 1.8)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.0)
+    sinatra (1.4.6)
+      rack (~> 1.4)
+      rack-protection (~> 1.4)
+      tilt (>= 1.3, < 3)
+    sinatra-json (0.1.0)
+      multi_json (~> 1.0)
+      sinatra (~> 1.0)
+    slop (3.6.0)
+    sshkit (1.7.1)
+      colorize (>= 0.7.0)
+      net-scp (>= 1.1.2)
+      net-ssh (>= 2.8.0)
+    tilt (2.0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  capistrano (~> 3.4.0)
+  capistrano-bundler (~> 1.1.4)
+  capistrano-rvm (~> 0.1.2)
+  capistrano3-puma (~> 1.2.1)
+  mongo (~> 2.1.0)
+  nokogiri (~> 1.6.6.2)
+  pry (~> 0.10.1)
+  puma (~> 2.13.4)
+  rack-test (~> 0.6.3)
+  rspec (~> 3.3.0)
+  simplecov (~> 0.10.0)
+  sinatra (~> 1.4.6)
+  sinatra-json (~> 0.1.0)
+
+BUNDLED WITH
+   1.10.6

README.md

@@ -0,0 +1,55 @@
+# CVEServer
+
+Simple REST-style web service for the CVE searching
+
+# Requirements
+
+  * Ruby 2.x.x
+  * Mongo
+  * Ruby bundler
+
+# Installation
+
+  * Clone our repository
+
+    $ git clone https://github.com/SpiderLabs/cve_server.git
+
+  * Install the ruby dependencies
+
+    $ bundle install
+
+  * Download the raw data from the National Vulnerability Database
+
+    $ ./bin/nvd_downloader
+
+  * Configure your database
+
+    $ vi config/database.yml
+
+  * Create and populate the database for you environment
+
+    $ RACK_ENV=development ./bin/seed
+
+  * Create and populate the database
+
+    $ RACK_ENV=development ./bin/seed
+
+  * Start the server
+
+    $ RACK_ENV=development puma
+
+# Using the API
+
+  * Search for an specific CVE using its ID
+
+    http://localhost:port/v1/cve/CVE-ID
+
+  * Search for CVEs related to any cpe
+
+    http://localhost:port/v1/cpe/php:php
+
+  * List all the available CPEs
+
+    http://localhost:port/v1/cpe/
+
+    http://localhost:port/v1/cpe/microsoft:windows

bin/nvd_downloader

@@ -0,0 +1,72 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift File.expand_path(File.join('..', '..', '/lib'), __FILE__)
+require 'nokogiri'
+require 'net/http'
+require 'cve_server'
+
+def fetch_page(url)
+  uri = URI.parse(url)
+  use_ssl = uri.scheme == 'https'
+  Net::HTTP.start(uri.host, uri.port, use_ssl: use_ssl) do |http|
+    response = http.request_get(uri)
+    case response
+    when Net::HTTPSuccess then
+      response
+    when Net::HTTPRedirection then
+      location = response['location']
+      fail "redirected to #{location}"
+    else
+      response.value
+    end
+  end
+end
+
+def download_file(url, dest_path)
+  uri = URI.parse(url)
+  use_ssl = uri.scheme == 'https'
+  Net::HTTP.start(uri.host, uri.port, use_ssl: use_ssl) do |http|
+    http.request_get(uri.path) do |response|
+      case response
+      when Net::HTTPSuccess then
+        f = File.open(dest_path, 'w')
+        response.read_body do |seg|
+          f << seg
+          sleep 0.005
+        end
+        f.close
+      when Net::HTTPRedirection then
+        location = response['location']
+        fail "redirected to #{location}"
+      else
+        fail "Unable to download #{url}"
+      end
+    end
+  end
+end
+
+def dest_path(link)
+  filename = link.split('/').last
+  File.join(CVEServer::Boot.config.raw_data_path, filename)
+end
+
+response = fetch_page('https://nvd.nist.gov/download.cfm')
+
+if response.is_a?(Net::HTTPSuccess)
+  @doc = Nokogiri::HTML(response.body)
+  xml_file_path = '//td[@class="xml-file-type file-20"]'
+  @doc.xpath('//html').xpath(xml_file_path).each do |td|
+    link = td.xpath('a').first['href']
+    next unless link =~ /.gz$/
+
+    dest_path = dest_path(link)
+    downloaded_path = [dest_path, '.download'].join
+
+    puts "Downloading file from #{link}.."
+    download_file(link, downloaded_path)
+
+    if File.exist?(downloaded_path) && File.size?(downloaded_path)
+      File.delete(dest_path) if File.exist?(dest_path)
+      File.rename(downloaded_path, dest_path)
+    end
+  end
+end

bin/seed

@@ -0,0 +1,18 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift File.expand_path(File.join('..', '..', '/lib'), __FILE__)
+require 'zlib'
+require 'cve_server'
+require 'cve_server/nvd/reader'
+
+files = File.join(CVEServer::Boot.config.raw_data_path, '*.xml.gz')
+
+CVEServer::Cve.drop_all
+Dir.glob(files).sort.each do |infile|
+  puts "Uncompressing #{infile}"
+  input = Zlib::GzipReader.open(infile).read
+  @doc = CVEServer::NVD::Reader.new(input)
+  puts 'Exporting data into the CVE collection'
+  CVEServer::Cve.bulk_create(@doc.all_cve)
+end
+puts "Reducing the cpe list"
+CVEServer::Cve.reduce_cpes

config.ru

@@ -0,0 +1,4 @@
+$LOAD_PATH.unshift File.expand_path('../lib', __FILE__)
+require 'cve_server/app'
+
+run CVEServer::App

config/database.yml

@@ -0,0 +1,17 @@
+production:
+  database: cves_production
+  adapter: mongo
+  host: 127.0.0.1
+  port: 27017
+
+development:
+  database: cves_development
+  adapter: mongo
+  host: 127.0.0.1
+  port: 27017
+
+test:
+  database: cves_test
+  adapter: mongo
+  host: 127.0.0.1
+  port: 27017

config/deploy.rb

@@ -0,0 +1,72 @@
+lock '3.4.0'
+set :application, 'cve_server'
+set :repo_url, '[email protected]:SpiderLabs/cve_server.git'
+set :branch, 'master' # Default branch is :master
+set :deploy_to, '/home/deployer/cve_server'
+set :stage, :production
+set :pty, false
+set :linked_dirs, fetch(:linked_dirs, []).push('nvd_data', 'log', 'tmp')
+
+set :puma_rackup, -> { File.join(current_path, 'config.ru') }
+set :puma_state, "#{shared_path}/tmp/pids/puma.state"
+set :puma_pid, "#{shared_path}/tmp/pids/puma.pid"
+set :puma_bind, "unix://#{shared_path}/tmp/sockets/puma.sock"    #accept array for multi-bind
+set :puma_default_control_app, "unix://#{shared_path}/tmp/sockets/pumactl.sock"
+set :puma_conf, "#{shared_path}/puma.rb"
+set :puma_access_log, "#{shared_path}/log/puma_access.log"
+set :puma_error_log, "#{shared_path}/log/puma_error.log"
+set :puma_role, :app
+set :puma_env, fetch(:rack_env, fetch(:rails_env, 'production'))
+set :puma_threads, [0, 16]
+set :puma_workers, 0
+set :puma_worker_timeout, nil
+set :puma_init_active_record, false
+set :puma_preload_app, true
+
+namespace :deploy do
+
+  namespace :symlink do
+    desc 'Symlink linked directories'
+    task :linked_dirs do
+      next unless any? :linked_dirs
+      on release_roles :all do
+        execute :mkdir, '-pv', linked_dir_parents(shared_path)
+        execute :mkdir, '-pv', shared_path.join('tmp/sockets')
+        execute :mkdir, '-pv', shared_path.join('tmp/pids')
+
+        fetch(:linked_dirs).each do |dir|
+          target = release_path.join(dir)
+          source = shared_path.join(dir)
+          unless test "[ -L #{target} ]"
+            if Dir.exist?(target)
+              execute :rm, '-rf', target
+            end
+            execute :ln, '-s', source, target
+          end
+        end
+      end
+    end
+  end
+
+  desc 'download the nvd reports'
+  task :download_nvd_reports do
+    on fetch(:bundle_servers) do
+      within release_path do
+        with fetch(:bundle_env_variables, {}) do
+          execute :bundle, 'exec', './bin/nvd_downloader'
+        end
+      end
+    end
+  end
+
+  desc 'reload the database with seed data'
+  task :seed do
+    on fetch(:bundle_servers) do
+      within release_path do
+        with fetch(:bundle_env_variables, {}) do
+          execute :bundle, 'exec', "./bin/seed RACK_ENV=#{fetch(:rack_env,{})}"
+        end
+      end
+    end
+  end
+end

config/deploy/production.rb

@@ -0,0 +1,2 @@
+server 'YourIP', user: 'deployer', roles: %w{app}
+set :rack_env, 'production'