diff options
Diffstat (limited to 'lib/tasks/cve.rake')
| -rw-r--r-- | lib/tasks/cve.rake | 366 |
1 files changed, 0 insertions, 366 deletions
diff --git a/lib/tasks/cve.rake b/lib/tasks/cve.rake deleted file mode 100644 index dbdc7a8..0000000 --- a/lib/tasks/cve.rake +++ /dev/null @@ -1,366 +0,0 @@ -# ===GLSAMaker v2 -# Copyright (C) 2010–15 Alex Legler <a3li@gentoo.org> -# Copyright (C) 2020 Max Magorsch <arzano@gentoo.org> -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# For more information, see the LICENSE file. -libs = %w( json nokogiri zlib stringio ) -libs << File.join(File.dirname(__FILE__), '..', 'bugzilla') -libs << File.join(File.dirname(__FILE__), '..', 'glsamaker') -libs << File.join(File.dirname(__FILE__), 'utils') - -libs.each { |lib| require lib } - -TMPDIR = File.join(File.dirname(__FILE__), '..', '..', 'tmp') -# What year are the first CVEs from? -YEAR = (ENV['START_YEAR'] || 2004).to_i -BASEURL = 'https://nvd.nist.gov/feeds/json/cve/1.1/nvdcve-1.1-%s.json.gz' - -DEBUG = ENV.key? 'DEBUG' -VERBOSE = (ENV.key?('VERBOSE') || DEBUG) -QUIET = ENV.key? 'QUIET' - -fail "I can't be quiet and verbose at the same time..." if QUIET && VERBOSE - -namespace :cve do - - desc 'Destroy all CVEs and CPEs' - task destroy_all: [:environment, 'db:load_config'] do - CveReference.destroy_all - Cve.destroy_all - Cpe.destroy_all - end - - - desc 'Print all CVEs' - task print_all: [:environment, 'db:load_config'] do - Cve.find_each do |cve| - puts cve.cve_id + ",\"" + cve.summary + "\"," + (cve.cvss || "") + "," + cve.state + "," + cve.published_at.to_formatted_s(:iso8601) + "," + cve.last_changed_at.to_formatted_s(:iso8601) + "," + cve.created_at.to_formatted_s(:iso8601) + "," + cve.updated_at.to_formatted_s(:iso8601) - end - end - - - desc 'Full CVE data import' - task full_import: [:environment, 'db:load_config'] do - Rails.cache.write(CVE_CACHE_LAST_IMPORT, Time.now.utc) - start_ts = Time.now - - (YEAR..Date.today.year).each do |year| - info 'Processing CVEs from '.bold + year.to_s.purple - - jsondata = status 'Downloading' do - gunzip_str Glsamaker::HTTP.get(BASEURL % year) - end - - json = status 'Loading JSON' do - JSON.parse(jsondata) - end - - cves = json["CVE_Items"] - create_cves(cves) - - puts - end - - info 'done'.green - info "(#{Time.now - start_ts} seconds)" - end - - - desc 'Incrementally update last CVEs' - task update: :environment do - Rails.cache.write(CVE_CACHE_LAST_IMPORT, Time.now.utc) - info 'Running incremental CVE data update...' - - jsondata = status 'Downloading' do - gunzip_str Glsamaker::HTTP.get(BASEURL % 'modified') - end - - json = status 'Loading JSON' do - JSON.parse(jsondata) - end - - cves = json["CVE_Items"] - update_cves(cves) - end - - - desc 'Update all CVEs' - task update_all: :environment do - Rails.cache.write(CVE_CACHE_LAST_IMPORT, Time.now.utc) - info 'Running complete CVE data update...' - - (YEAR..Date.today.year).each do |year| - info 'Processing CVEs from '.bold + year.to_s.purple - - jsondata = status 'Downloading' do - gunzip_str Glsamaker::HTTP.get(BASEURL % year) - end - - json = status 'Loading JSON' do - JSON.parse(jsondata) - end - - cves = json["CVE_Items"] - update_cves(cves) - end - end - -end - - -# -# Update the given cve -# -def create_cve(cve) - summary = cve.dig('cve', 'description', 'description_data', 0, 'value') - _cve = Cve.create( - :cve_id => cve.dig('cve', 'CVE_data_meta', 'ID'), - :summary => summary, - :cvss => cve.dig('impact', 'baseMetricV2', 'cvssV2', 'vectorString'), - :published_at => DateTime.parse(cve['publishedDate']), - :last_changed_at => DateTime.parse(cve['lastModifiedDate']), - :state => (summary =~ /^\*\* REJECT \*\*/ ? 'REJECTED' : 'NEW') - ) - - if cve.dig('references', 'reference_data') - cve.dig('references', 'reference_data').each do |ref| - CveReference.create( - :cve => _cve, - :source => ref['refsource'], - :title => ref['name'], - :uri => ref['url'] - ) - end - end - - cve.dig('configurations', 'nodes').each do |node| - if node.key?('cpe_match') - node['cpe_match'].each do |cpe_match| - cpe_str = cpe_match['cpe23Uri'] - - cpe = Cpe.where(:cpe => cpe_str).first - cpe ||= Cpe.create(:cpe => cpe_str) - - _cve.cpes << cpe - end - elsif node.key?('children') - node['children'].each do |child| - if child.key?('cpe_match') - child['cpe_match'].each do |cpe_match| - cpe_str = cpe_match['cpe23Uri'] - - cpe = Cpe.where(:cpe => cpe_str).first - cpe ||= Cpe.create(:cpe => cpe_str) - - _cve.cpes << cpe - end - end - end - end - end -end - - -# -# Create the all given cves -# -def create_cves(cves) - processed_cves = 0 - info "#{cves.size.to_s.purple} CVEs (one dot equals 100, purple dot equals 500)" - - cves.each do |cve| - unless VERBOSE || QUIET - - if processed_cves % 500 == 0 - print '.'.purple - else - print '.' if processed_cves % 100 == 0 - end - end - - puts cve.dig('cve', 'CVE_data_meta', 'ID').to_s.purple if VERBOSE - - begin - create_cve(cve) - rescue ActiveRecord::StatementInvalid => e - # Ignore dupes, ONLY do that for the full db update! - raise e unless e.message =~ /Duplicate entry/ - end - - processed_cves += 1 - STDOUT.flush - end -end - - -# -# Update the all given cves -# -def update_cves(cves) - start_ts = Time.now - processed_cves = created_cves = updated_cves = 0 - - info "#{cves.size.to_s.purple} CVEs (one dot equals 100, purple dot equals 500)" - - cves.each do |cve| - unless VERBOSE || QUIET - if processed_cves % 500 == 0 - print '.'.purple - else - print '.' if processed_cves % 100 == 0 - end - end - - puts cve.dig('cve', 'CVE_data_meta', 'ID').to_s.purple if VERBOSE - - c = Cve.find_by_cve_id cve.dig('cve', 'CVE_data_meta', 'ID') - - if c.nil? - debug 'Creating CVE.' - create_cve(cve) - created_cves += 1 - else - last_changed_at = Time.parse(cve['lastModifiedDate']).utc - - if last_changed_at.to_i > c.last_changed_at.to_i - debug 'Updating CVE. Timestamp changed.' - summary = cve.dig('cve', 'description', 'description_data', 0, 'value') - c.attributes = { - cve_id: cve.dig('cve','CVE_data_meta','ID'), - summary: summary, - cvss: cve.dig('impact', 'baseMetricV2', 'cvssV2', 'vectorString'), - published_at: DateTime.parse(cve['publishedDate']), - last_changed_at: DateTime.parse(cve['lastModifiedDate']), - } - - c.state = 'REJECTED' if summary =~ /^\*\* REJECT \*\*/ - c.save! - - db_references = [] - xml_references = [] - - if cve.dig 'references', 'reference_data' - cve.dig('references', 'reference_data').each do |ref| - xml_references << [ - :source => ref['refsource'], - :title => ref['name'], - :uri => ref['url'] - ] - end - end - - c.references.each do |ref| - db_references << [ref.source, ref.title, ref.uri] - end - - rem = db_references - xml_references - debug "Removing references: #{rem.inspect}" - - rem.each do |item| - ref = c.references.where(['source = ? AND title = ? AND uri = ?', *item]).first - debug ref - c.references.delete(ref) - ref.destroy - end - - add = xml_references - db_references - debug "Ading references: #{add.inspect}" - - add.each do |item| - c.references.create( - source: item[0], - title: item[1], - uri: item[2] - ) - end - - db_cpes = [] - xml_cpes = [] - - cve.dig('configurations', 'nodes').each do |node| - if node.key?('cpe_match') - node['cpe_match'].each do |cpe_match| - xml_cpes << cpe_match['cpe23Uri'] - end - elsif node.key?('children') - node['children'].each do |child| - if child.key?('cpe_match') - child['cpe_match'].each do |cpe_match| - xml_cpes << cpe_match['cpe23Uri'] - end - end - end - end - end - - c.cpes.each do |prod| - db_cpes << prod.cpe - end - - rem = db_cpes - xml_cpes - debug "Removing CPEs: #{rem.inspect}" - - rem.each do |item| - c.cpes.delete(Cpe.find_by_cpe(item)) - end - - add = xml_cpes - db_cpes - debug "Ading CPEs: #{add.inspect}" - - add.each do |item| - cpe = Cpe.where(cpe: item).first - cpe ||= Cpe.create(cpe: item) - - c.cpes << cpe - end - - c.save! - updated_cves += 1 - end - end - - processed_cves += 1 - STDOUT.flush - end - - info '' - - info "(#{Time.now - start_ts} seconds, #{created_cves} new CVE entries, #{updated_cves} updated CVE entries)" -end - - -# -# Run something, and display a status info around it -# -def status(message) - fail ArgumentError, 'I want a block :(' unless block_given? - - print "#{message}....." unless QUIET - STDOUT.flush - - stuff = yield - print "\b" * 5 + ' done.'.green + "\n" unless QUIET - stuff -end - - -def debug(msg) - $stderr.puts msg if DEBUG -end - - -def info(msg) - puts msg unless QUIET -end - -# -# unzip the given string -# -def gunzip_str(str) - Zlib::GzipReader.new(StringIO.new(str)).read -end |