Fix the cve import2.1.17

The cve feeds are using json instead of xml now.
The import task has been migrated to read the json
feeds now.

Signed-off-by: Max Magorsch <arzano@gentoo.org>

1 files changed, 251 insertions, 171 deletions

diff --git a/lib/tasks/cve.rake b/lib/tasks/cve.rake
index dbe06cf..4fccf47 100644
--- a/lib/tasks/cve.rake
+++ b/lib/tasks/cve.rake
@@ -1,5 +1,6 @@
 # ===GLSAMaker v2
 #  Copyright (C) 2010–15 Alex Legler <a3li@gentoo.org>
+#  Copyright (C) 2020 Max Magorsch <arzano@gentoo.org>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -7,8 +8,7 @@
 # (at your option) any later version.
 #
 # For more information, see the LICENSE file.
-
-libs = %w( nokogiri zlib stringio )
+libs = %w( json nokogiri zlib stringio )
 libs << File.join(File.dirname(__FILE__), '..', 'bugzilla')
 libs << File.join(File.dirname(__FILE__), '..', 'glsamaker')
 libs << File.join(File.dirname(__FILE__), 'utils')
@@ -18,7 +18,7 @@ libs.each { |lib| require lib }
 TMPDIR = File.join(File.dirname(__FILE__), '..', '..', 'tmp')
 # What year are the first CVEs from?
 YEAR = (ENV['START_YEAR'] || 2004).to_i
-BASEURL = 'https://nvd.nist.gov/feeds/xml/cve/nvdcve-2.0-%s.xml.gz'
+BASEURL = 'https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-%s.json.gz'
 
 DEBUG   = ENV.key? 'DEBUG'
 VERBOSE = (ENV.key?('VERBOSE') || DEBUG)
@@ -27,6 +27,23 @@ QUIET   = ENV.key? 'QUIET'
 fail "I can't be quiet and verbose at the same time..." if QUIET && VERBOSE
 
 namespace :cve do
+
+  desc 'Destroy all CVEs and CPEs'
+  task destroy_all: [:environment, 'db:load_config'] do
+    CveReference.destroy_all
+    Cve.destroy_all
+    Cpe.destroy_all
+  end
+
+
+  desc 'Print all CVEs'
+  task print_all: [:environment, 'db:load_config'] do
+    Cve.find_each do |cve|
+      puts cve.cve_id + ",\"" + cve.summary + "\"," + (cve.cvss || "")  + "," + cve.state + "," + cve.published_at.to_formatted_s(:iso8601) + "," + cve.last_changed_at.to_formatted_s(:iso8601) + "," + cve.created_at.to_formatted_s(:iso8601)  + "," + cve.updated_at.to_formatted_s(:iso8601)
+    end
+  end
+
+
   desc 'Full CVE data import'
   task full_import: [:environment, 'db:load_config'] do
     start_ts = Time.now
@@ -34,43 +51,16 @@ namespace :cve do
     (YEAR..Date.today.year).each do |year|
       info 'Processing CVEs from '.bold + year.to_s.purple
 
-      xmldata = status 'Downloading' do
+      jsondata = status 'Downloading' do
         gunzip_str Glsamaker::HTTP.get(BASEURL % year)
       end
 
-      xml = status 'Loading XML' do
-        Nokogiri::XML(xmldata)
+      json = status 'Loading JSON' do
+        JSON.parse(jsondata)
       end
 
-      namespace = { 'cve' => 'http://scap.nist.gov/schema/feed/vulnerability/2.0' }
-      processed_cves = 0
-      cpe_cache = {}
-
-      cves = xml.root.xpath('cve:entry', namespace)
-      info "#{cves.size.to_s.purple} CVEs (one dot equals 100, purple dot equals 500)"
-
-      cves.each do |cve|
-        unless VERBOSE || QUIET
-
-          if processed_cves % 500 == 0
-            print '.'.purple
-          else
-            print '.' if processed_cves % 100 == 0
-          end
-        end
-
-        puts cve['id'].to_s.purple if VERBOSE
-
-        begin
-          create_cve(cve)
-        rescue ActiveRecord::StatementInvalid => e
-          # Ignore dupes, ONLY do that for the full db update!
-          raise e unless e.message =~ /Duplicate entry/
-        end
-
-        processed_cves += 1
-        STDOUT.flush
-      end
+      cves = json["CVE_Items"]
+      create_cves(cves)
 
       puts
     end
@@ -79,139 +69,273 @@ namespace :cve do
     info "(#{Time.now - start_ts} seconds)"
   end
 
-  desc 'Incremental CVE data update'
+
+  desc 'Incrementally update last CVEs'
   task update: :environment do
-    start_ts = Time.now
+
     info 'Running incremental CVE data update...'
 
-    xmldata = status 'Downloading' do
+    jsondata = status 'Downloading' do
       gunzip_str Glsamaker::HTTP.get(BASEURL % 'modified')
     end
 
-    xml = status 'Loading XML' do
-      Nokogiri::XML(xmldata)
+    json = status 'Loading JSON' do
+      JSON.parse(jsondata)
     end
 
-    namespace = { 'cve' => 'http://scap.nist.gov/schema/feed/vulnerability/2.0' }
-    processed_cves = created_cves = updated_cves = 0
-    cpe_cache = {}
+    cves = json["CVE_Items"]
+    update_cves(cves)
+  end
+
+
+  desc 'Update all CVEs'
+  task update_all: :environment do
 
-    cves = xml.root.xpath('cve:entry', namespace)
-    info "#{cves.size.to_s.purple} CVEs (one dot equals 100, purple dot equals 500)"
+    info 'Running complete CVE data update...'
+
+    (YEAR..Date.today.year).each do |year|
+      info 'Processing CVEs from '.bold + year.to_s.purple
+
+      jsondata = status 'Downloading' do
+        gunzip_str Glsamaker::HTTP.get(BASEURL % year)
+      end
+
+      json = status 'Loading JSON' do
+        JSON.parse(jsondata)
+      end
 
-    cves.each do |cve|
-      unless VERBOSE || QUIET
-        if processed_cves % 500 == 0
-          print '.'.purple
-        else
-          print '.' if processed_cves % 100 == 0
+      cves = json["CVE_Items"]
+      update_cves(cves)
+    end
+  end
+
+end
+
+
+#
+# Update the given cve
+#
+def create_cve(cve)
+  summary = cve.dig('cve', 'description', 'description_data', 0, 'value')
+  _cve = Cve.create(
+    :cve_id => cve.dig('cve', 'CVE_data_meta', 'ID'),
+    :summary => summary,
+    :cvss => cve.dig('impact', 'baseMetricV2', 'cvssV2', 'vectorString'),
+    :published_at => DateTime.parse(cve['publishedDate']),
+    :last_changed_at => DateTime.parse(cve['lastModifiedDate']),
+    :state => (summary =~ /^\*\* REJECT \*\*/ ? 'REJECTED' : 'NEW')
+  )
+
+  if cve.dig('references', 'reference_data')
+    cve.dig('references', 'reference_data').each do |ref|
+      CveReference.create(
+        :cve => _cve,
+        :source => ref['refsource'],
+        :title => ref['name'],
+        :uri => ref['url']
+      )
+    end
+  end
+
+  cve.dig('configurations', 'nodes').each do |node|
+    if node.key?('cpe_match')
+      node['cpe_match'].each do |cpe_match|
+        cpe_str = cpe_match['cpe23Uri']
+
+        cpe = Cpe.where(:cpe => cpe_str).first
+        cpe ||= Cpe.create(:cpe => cpe_str)
+
+        _cve.cpes << cpe
+      end
+    elsif node.key?('children')
+      node['children'].each do |child|
+        if child.key?('cpe_match')
+          child['cpe_match'].each do |cpe_match|
+            cpe_str = cpe_match['cpe23Uri']
+
+            cpe = Cpe.where(:cpe => cpe_str).first
+            cpe ||= Cpe.create(:cpe => cpe_str)
+
+            _cve.cpes << cpe
+          end
         end
       end
+    end
+  end
+end
 
-      puts cve['id'].to_s.purple if VERBOSE
 
-      c = Cve.find_by_cve_id cve['id']
+#
+# Create the all given cves
+#
+def create_cves(cves)
+  processed_cves = 0
+  info "#{cves.size.to_s.purple} CVEs (one dot equals 100, purple dot equals 500)"
+
+  cves.each do |cve|
+    unless VERBOSE || QUIET
 
-      if c.nil?
-        debug 'Creating CVE.'
-        create_cve(cve)
-        created_cves += 1
+      if processed_cves % 500 == 0
+        print '.'.purple
       else
-        last_changed_at = Time.parse(cve.xpath('vuln:last-modified-datetime').first.content).utc
-        db_lca = c.last_changed_at
-
-        if last_changed_at.to_i > c.last_changed_at.to_i
-          debug 'Updating CVE. Timestamp changed.'
-          summary = cve.xpath('vuln:summary').first.content
-          c.attributes = {
-            cve_id: cve['id'],
+        print '.' if processed_cves % 100 == 0
+      end
+    end
+
+    puts cve.dig('cve', 'CVE_data_meta', 'ID').to_s.purple if VERBOSE
+
+    begin
+      create_cve(cve)
+    rescue ActiveRecord::StatementInvalid => e
+      # Ignore dupes, ONLY do that for the full db update!
+      raise e unless e.message =~ /Duplicate entry/
+    end
+
+    processed_cves += 1
+    STDOUT.flush
+  end
+end
+
+
+#
+# Update the all given cves
+#
+def update_cves(cves)
+  start_ts = Time.now
+  processed_cves = created_cves = updated_cves = 0
+
+  info "#{cves.size.to_s.purple} CVEs (one dot equals 100, purple dot equals 500)"
+
+  cves.each do |cve|
+    unless VERBOSE || QUIET
+      if processed_cves % 500 == 0
+        print '.'.purple
+      else
+        print '.' if processed_cves % 100 == 0
+      end
+    end
+
+    puts cve.dig('cve', 'CVE_data_meta', 'ID').to_s.purple if VERBOSE
+
+    c = Cve.find_by_cve_id cve.dig('cve', 'CVE_data_meta', 'ID')
+
+    if c.nil?
+      debug 'Creating CVE.'
+      create_cve(cve)
+      created_cves += 1
+    else
+      last_changed_at = Time.parse(cve['lastModifiedDate']).utc
+
+      if last_changed_at.to_i > c.last_changed_at.to_i
+        debug 'Updating CVE. Timestamp changed.'
+        summary = cve.dig('cve', 'description', 'description_data', 0, 'value')
+        c.attributes = {
+            cve_id: cve.dig('cve','CVE_data_meta','ID'),
             summary: summary,
-            cvss: cvss_xml2str(cve.xpath('vuln:cvss')),
-            published_at: DateTime.parse(cve.xpath('vuln:published-datetime').first.content),
-            last_changed_at: DateTime.parse(cve.xpath('vuln:last-modified-datetime').first.content)
-          }
+            cvss: cve.dig('impact', 'baseMetricV2', 'cvssV2', 'vectorString'),
+            published_at: DateTime.parse(cve['publishedDate']),
+            last_changed_at: DateTime.parse(cve['lastModifiedDate']),
+        }
 
-          c.state = 'REJECTED' if summary =~ /^\*\* REJECT \*\*/
-          c.save!
+        c.state = 'REJECTED' if summary =~ /^\*\* REJECT \*\*/
+        c.save!
 
-          db_references = []
-          xml_references = []
-          cve.xpath('vuln:references').each do |ref|
+        db_references = []
+        xml_references = []
+
+        if cve.dig 'references', 'reference_data'
+          cve.dig('references', 'reference_data').each do |ref|
             xml_references << [
-              ref.xpath('vuln:source').first.content,
-              ref.xpath('vuln:reference').first.content,
-              ref.xpath('vuln:reference').first['href']
+                :source => ref['refsource'],
+                :title => ref['name'],
+                :uri => ref['url']
             ]
           end
+        end
 
-          c.references.each do |ref|
-            db_references << [ref.source, ref.title, ref.uri]
-          end
+        c.references.each do |ref|
+          db_references << [ref.source, ref.title, ref.uri]
+        end
 
-          rem = db_references - xml_references
-          debug "Removing references: #{rem.inspect}"
+        rem = db_references - xml_references
+        debug "Removing references: #{rem.inspect}"
 
-          rem.each do |item|
-            ref = c.references.where(['source = ? AND title = ? AND uri = ?', *item]).first
-            debug ref
-            c.references.delete(ref)
-            ref.destroy
-          end
+        rem.each do |item|
+          ref = c.references.where(['source = ? AND title = ? AND uri = ?', *item]).first
+          debug ref
+          c.references.delete(ref)
+          ref.destroy
+        end
 
-          add = xml_references - db_references
-          debug "Ading references:    #{add.inspect}"
+        add = xml_references - db_references
+        debug "Ading references:    #{add.inspect}"
 
-          add.each do |item|
-            c.references.create(
+        add.each do |item|
+          c.references.create(
               source: item[0],
               title:  item[1],
               uri:    item[2]
-            )
-          end
-
-          db_cpes = []
-          xml_cpes = []
-          cve.xpath('vuln:vulnerable-software-list/vuln:product').each do |prod|
-            xml_cpes << prod.content
-          end
+          )
+        end
 
-          c.cpes.each do |prod|
-            db_cpes << prod.cpe
+        db_cpes = []
+        xml_cpes = []
+
+        cve.dig('configurations', 'nodes').each do |node|
+          if node.key?('cpe_match')
+            node['cpe_match'].each do |cpe_match|
+              xml_cpes << cpe_match['cpe23Uri']
+            end
+          elsif node.key?('children')
+            node['children'].each do |child|
+              if child.key?('cpe_match')
+                child['cpe_match'].each do |cpe_match|
+                  xml_cpes << cpe_match['cpe23Uri']
+                end
+              end
+            end
           end
+        end
 
-          rem = db_cpes - xml_cpes
-          debug "Removing CPEs: #{rem.inspect}"
+        c.cpes.each do |prod|
+          db_cpes << prod.cpe
+        end
 
-          rem.each do |item|
-            c.cpes.delete(Cpe.find_by_cpe(item))
-          end
+        rem = db_cpes - xml_cpes
+        debug "Removing CPEs: #{rem.inspect}"
 
-          add = xml_cpes - db_cpes
-          debug "Ading CPEs:    #{add.inspect}"
+        rem.each do |item|
+          c.cpes.delete(Cpe.find_by_cpe(item))
+        end
 
-          add.each do |item|
-            cpe = Cpe.where(cpe: item).first
-            cpe ||= Cpe.create(cpe: item)
+        add = xml_cpes - db_cpes
+        debug "Ading CPEs:    #{add.inspect}"
 
-            c.cpes << cpe
-          end
+        add.each do |item|
+          cpe = Cpe.where(cpe: item).first
+          cpe ||= Cpe.create(cpe: item)
 
-          c.save!
-          updated_cves += 1
+          c.cpes << cpe
         end
-      end
 
-      processed_cves += 1
-      STDOUT.flush
+        c.save!
+        updated_cves += 1
+      end
     end
 
-    info ''
-
-    info "(#{Time.now - start_ts} seconds, #{created_cves} new CVE entries, #{updated_cves} updated CVE entries)"
+    processed_cves += 1
+    STDOUT.flush
   end
+
+  info ''
+
+  info "(#{Time.now - start_ts} seconds, #{created_cves} new CVE entries, #{updated_cves} updated CVE entries)"
 end
 
+
+#
 # Run something, and display a status info around it
+#
 def status(message)
   fail ArgumentError, 'I want a block :(' unless block_given?
 
@@ -223,63 +347,19 @@ def status(message)
   stuff
 end
 
+
 def debug(msg)
   $stderr.puts msg if DEBUG
 end
 
+
 def info(msg)
   puts msg unless QUIET
 end
 
-# 7.2/AV:L/AC:L/Au:N/C:C/I:C/A:C
-def cvss_xml2str(data)
-  def get_content(x, y)
-    x.xpath(y).first.content
-  end
-
-  return nil if data.size == 0
-
-  str  = "#{get_content(data, 'cvss:base_metrics/cvss:score')}/"
-  str += "AV:#{get_content(data, 'cvss:base_metrics/cvss:access-vector')[0, 1]}/"
-  str += "AC:#{get_content(data, 'cvss:base_metrics/cvss:access-complexity')[0, 1]}/"
-  str += "Au:#{get_content(data, 'cvss:base_metrics/cvss:authentication')[0, 1]}/"
-  str += "C:#{get_content(data, 'cvss:base_metrics/cvss:confidentiality-impact')[0, 1]}/"
-  str += "I:#{get_content(data, 'cvss:base_metrics/cvss:integrity-impact')[0, 1]}/"
-  str += "A:#{get_content(data, 'cvss:base_metrics/cvss:availability-impact')[0, 1]}"
-
-  str
-end
-
-def create_cve(cve)
-  summary = cve.xpath('vuln:summary').first.content
-  _cve = Cve.create(
-    :cve_id => cve['id'],
-    :summary => summary,
-    :cvss => cvss_xml2str(cve.xpath('vuln:cvss')),
-    :published_at => DateTime.parse(cve.xpath('vuln:published-datetime').first.content),
-    :last_changed_at => DateTime.parse(cve.xpath('vuln:last-modified-datetime').first.content),
-    :state => (summary =~ /^\*\* REJECT \*\*/ ? 'REJECTED' : 'NEW')
-  )
-
-  cve.xpath('vuln:references').each do |ref|
-    CveReference.create(
-      :cve => _cve,
-      :source => ref.xpath('vuln:source').first.content,
-      :title => ref.xpath('vuln:reference').first.content,
-      :uri => ref.xpath('vuln:reference').first['href']
-    )
-  end
-
-  cve.xpath('vuln:vulnerable-software-list/vuln:product').each do |prod|
-    cpe_str = prod.content
-
-    cpe = Cpe.where(:cpe => cpe_str).first
-    cpe ||= Cpe.create(:cpe => cpe_str)
-
-    _cve.cpes << cpe
-  end
-end
-
+#
+# unzip the given string
+#
 def gunzip_str(str)
   Zlib::GzipReader.new(StringIO.new(str)).read
 end