diff options
author | 2009-10-31 12:52:36 +0100 | |
---|---|---|
committer | 2009-10-31 12:52:36 +0100 | |
commit | 63d063d03127573225e1c43d0d9e03e5c7884d21 (patch) | |
tree | f1eb27680c808e5e69f520a545d1f1a64d7e8fe6 | |
parent | Code cleanup (diff) | |
download | kernel-check-63d063d03127573225e1c43d0d9e03e5c7884d21.tar.gz kernel-check-63d063d03127573225e1c43d0d9e03e5c7884d21.tar.bz2 kernel-check-63d063d03127573225e1c43d0d9e03e5c7884d21.zip |
Large restructuration
-rw-r--r-- | .gitignore | 3 | ||||
-rw-r--r-- | TODO | 10 | ||||
-rwxr-xr-x | collector.py | 123 | ||||
-rwxr-xr-x | kernel-check.py | 16 | ||||
-rw-r--r-- | lib/__init__.py | 0 | ||||
-rwxr-xr-x | lib/guidexml.py (renamed from guidexml.py) | 0 | ||||
-rwxr-xr-x | lib/kernellib.py (renamed from kernellib.py) | 474 | ||||
-rwxr-xr-x | tools/cron.py | 405 | ||||
-rwxr-xr-x | tools/findcommit.sh (renamed from findcommit.sh) | 8 | ||||
-rwxr-xr-x | tools/testsuite.py (renamed from testsuite.py) | 0 |
10 files changed, 448 insertions, 591 deletions
@@ -1,5 +1,4 @@ *pyc *~ *# -tmp -out +tools/tmp @@ -1,18 +1,19 @@ Implementation ============== - Implement Report +- Implement GUI +- Implement kernel testing framwork - Handle "best kernel not found" - Add further error handling -- Implement find_cve() (return bugid) -- Create a GENERIC-MAP-NOMATCH entry in the cve xml files - Implement hardend/xen intervall Cleanup and Rework ================== +- Rework cron.py - Remove unused code and find better ways - Check lookaround of 'grp_all' -- Rework interval class and all interval functions (expand?) -- Rework cves.refs and find a solution for <ref source url/> +- Rework interval class +- Rework cves.refs Dokumentation ============= @@ -23,7 +24,6 @@ Dokumentation Whiteboard changes ================== - Move arch into whiteboard e.g. {x86, amd64} -- [kernel +<version] -> [+kernel <version] Summary changes =============== diff --git a/collector.py b/collector.py deleted file mode 100755 index 30f3272..0000000 --- a/collector.py +++ /dev/null @@ -1,123 +0,0 @@ -#!/usr/bin/env python -# kernel-check -- Kernel security information -# Copyright 2009-2009 Gentoo Foundation -# Distributed under the terms of the GNU General Public License v2 - -import getopt -import os -import sys -import time - -import kernellib as lib - - -def main(argv): - 'Main function' - - try: - opts, args = getopt.getopt(argv, 'd:fh:sv', - ['delay=', 'force', 'help', 'skip', 'verbose']) - except getopt.GetoptError: - usage() - return - - for opt, arg in opts: - if opt in ('-d', '--delay'): - if arg.isdigit(): - lib.DELAY = int(arg) - elif opt in ('-f', '--force'): - lib.FORCE = True - elif opt in ('-h', '--help'): - usage() - return - elif opt in ('-s', '--skip'): - lib.SKIP = True - elif opt in ('-v', '--verbose'): - lib.VERBOSE = True - - for directory in lib.DIR: - if not os.path.isdir(lib.DIR[directory]): - os.makedirs(lib.DIR[directory]) - - print 'Reading available genpatches...' - try: - read_patches = lib.read_genpatch_file(lib.DIR['out']) - except IOError: - read_patches = list() - - print 'Parsing genpatches from portage...' - found_patches = lib.parse_genpatch_list(lib.PORTDIR) - - new_patches = 0 - for item in found_patches: - if item not in read_patches: - read_patches.append(item) - new_patches += 1 - - if (new_patches): - lib.write_genpatch_file(lib.DIR['out'], read_patches) - print 'Added %i new genpatches!' % new_patches - - print '\nReceiving the latest xml file from the nvd...' - lib.receive_nvd_recent(lib.DIR['nvd']) - - if not lib.SKIP: - print 'Receiving earlier xml files from the nvd...' - lib.receive_nvd_all(lib.DIR['nvd']) - - print 'Creating the nvd dictionary...' - nvd_dict = lib.parse_nvd_dict(lib.DIR['nvd']) - - print 'Receiving the kernel vulnerability list from bugzilla...' - lib.receive_bugzilla_list(lib.DIR['tmp']) - - buglist = lib.parse_bugzilla_list(lib.DIR['tmp']) - print 'Found %i kernel vulnerabilities!' % len(buglist) - - print '\nCreating the xml files...' - - created_files = 0 - for item in buglist: - try: - lib.receive_bugzilla_bug(lib.DIR['bug'], item) - vul = lib.parse_bugzilla_dict(lib.DIR['bug'], item) - vul = lib.search_nvd_dict(nvd_dict, vul) - lib.write_cve_file(lib.DIR['out'], vul) - - created_files += 1 - time.sleep(lib.DELAY) - - except lib.InvalidWhiteboardError, e: - print '\n[%s] Invalid whiteboard' % item - print '%s' % e.value - - except lib.InvalidCveError, e: - print '\n[%s] Invalid CVE' % item - print '%s' % e.value - - except lib.NvdEntryError, e: - print '\n[%s] No Nvd Entry' % item - print '%s' % e.value - - except lib.CveDuplicateError, e: - print '\n[%s] CVE Duplicate' % item - print '%s' % e.value - - print '\nCreated %i xml files!' % created_files - - -def usage(): - 'Prints the usage screen' - - print 'Usage: %s [OPTION]...' % sys.argv[0][:-3] - print 'Kernel security information %s\r\n' % lib.VERSION - print ' -d, --delay [ticks] add delay to xml file creation' - print ' -f, --force force update of xml files' - print ' -h, --help display help information' - print ' -s, --skip skip update of prior nvd files' - print ' -v, --verbose display additional information' - - -if __name__ == '__main__': - main(sys.argv[1:]) - diff --git a/kernel-check.py b/kernel-check.py index 9c7e563..a6d6051 100755 --- a/kernel-check.py +++ b/kernel-check.py @@ -9,7 +9,7 @@ import sys import textwrap import os -import kernellib as lib +import lib.kernellib as lib info = portage.output.EOutput().einfo warn = portage.output.EOutput().ewarn @@ -61,8 +61,10 @@ def main(argv): (kernel.version, kernel.revision)))) info('Kernel source : %s' % color('GOOD', kernel.source)) - kernel.genpatch = lib.get_genpatch(lib.read_genpatch_file(lib.DIR['out']), - kernel) + + genpatches = lib.parse_genpatch_list(lib.PORTDIR) + + kernel.genpatch = lib.get_genpatch(genpatches, kernel) if kernel.genpatch is not None: info('Gen(too)patch : %s' % color('GOOD', '%s %s' % @@ -79,6 +81,7 @@ def main(argv): print '\n>>> Reading all kernel vulnerabilities' + """ supported = list() for item in lib.SUPPORTED: best = (lib.all_version(item)) @@ -88,6 +91,7 @@ def main(argv): i.genpatch = lib.get_genpatch(lib.read_genpatch_file( lib.DIR['out']), i) supported.append(i) + """ kernel_eval = lib.eval_cve_files(lib.DIR['out'], kernel, arch) if not kernel_eval: @@ -109,6 +113,7 @@ def main(argv): color('BAD', str(len(kernel_eval.affected)))) print_summary(kernel_eval.affected) + """ info('You have the following choices: ') print '' @@ -148,6 +153,7 @@ def main(argv): color('BAD', str(len(kernel_eval.affected))), color('BAD', str(len(comparison.new))))) print '' + """ print_information() print_beta() @@ -244,17 +250,15 @@ def print_beta(): 'Prints a beta warning message' print('') - error('%s You are using a very early version of kernel-check.' % + error('%s You are using a early version of kernel-check.' % color('BAD', 'IMPORTANT')) error('Please note that this tool might not operate as expected.') - error('Moreover the given information are most likely incorrect.') def print_information(): 'Prints an information message' info('To print more information about a vulnerability try:') - info('') info(' $ %s -s [bugid|cve]' % sys.argv[0]) diff --git a/lib/__init__.py b/lib/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/lib/__init__.py diff --git a/guidexml.py b/lib/guidexml.py index d6babb8..d6babb8 100755 --- a/guidexml.py +++ b/lib/guidexml.py diff --git a/kernellib.py b/lib/kernellib.py index fe63a3e..d320c20 100755 --- a/kernellib.py +++ b/lib/kernellib.py @@ -27,19 +27,8 @@ BUGORDER = ['bugid', 'reporter', 'reported', 'status', 'arch', 'affected'] CVEORDER = ['cve', 'published', 'desc', 'severity', 'vector', 'score', 'refs'] REGEX = { - 'bugzilla' : re.compile(r'(?<=bug.cgi\?id=)\d*'), 'gp_version' : re.compile(r'(?<=K_GENPATCHES_VER\=\").+(?=\")'), 'gp_want' : re.compile(r'(?<=K_WANT_GENPATCHES\=\").+(?=\")'), - 'grp_all' : re.compile(r'(?<=\()[ (]*CVE-(\d{4})' \ - r'([-,(){}|, \d]+)(?=\))'), - 'grp_split' : re.compile(r'(?<=\D)(\d{4})(?=\D|$)'), - 'm_nomatch' : re.compile(r'.*GENERIC-MAP-NOMATCH.*'), - 'wb_match' : re.compile(r'\s*\[\s*([^ +<=>]+)\s*(\+?)' \ - r'\s*([<=>]{1,2})\s*([^ <=>\]' \ - r']+)\s*(?:([<=>]{1,2})\s*([^' \ - r' \]]+))?\s*\]\s*(.*)'), - 'wb_version' : re.compile(r'^(?:\d{1,2}\.){0,3}\d{1,2}' \ - r'(?:[-_](?:r|rc)?\d{1,2})*$'), 'k_version' : re.compile(r'^((?:\d{1,2}\.){0,4}\d{1,2})(-.*)?$'), 'rc_kernel' : re.compile(r'^rc\d{1,3}$'), 'git_kernel' : re.compile(r'^git(\d{1,3})$'), @@ -61,23 +50,23 @@ KERNEL_TYPES = [ 'vserver', 'win4lin', 'wolk-dev', 'wolk', 'xbox', 'xen', 'xfs' ] -VERSION = '0.3.8' +VERSION = '0.3.9' NOCVE = 'GENERIC-MAP-NOMATCH' NOCVEDESC = 'This GENERIC identifier is not specific to any vulnerability. '\ 'GENERIC-MAP-NOMATCH is used by products, databases, and ' \ 'services to specify when a particular vulnerability element ' \ 'does not map to a corresponding CVE entry.' CVES = dict() -PORTDIR = portage.settings['PORTDIR'] DEBUG = False VERBOSE = False FORCE = False SKIP = False DELAY = 0 FILEPATH = os.path.dirname(os.path.realpath(__file__)) +PORTDIR = portage.settings['PORTDIR'] DIR = { 'tmp' : os.path.join(FILEPATH, 'tmp'), - 'out' : os.path.join(FILEPATH, 'out'), + 'out' : os.path.join(PORTDIR, 'metadata', 'kernel'), 'bug' : os.path.join(FILEPATH, 'tmp', 'bug'), 'nvd' : os.path.join(FILEPATH, 'tmp', 'nvd') } @@ -87,22 +76,6 @@ def BUG_ON(msg): print 'DEBUG line %s in %s(): %s' % (inspect.stack()[1][2], inspect.stack()[1][3], msg) -class InvalidWhiteboardError(Exception): - def __init__(self, value): - self.value = value - -class InvalidCveError(Exception): - def __init__(self, value): - self.value = value - -class CveDuplicateError(Exception): - def __init__(self, value): - self.value = value - -class NvdEntryError(Exception): - def __init__(self, value): - self.value = value - class Evaluation: """Evaluation class @@ -311,21 +284,6 @@ class Interval: return interval -def interval_to_xml(interval, root): - 'Formats an interval for xml output' - - intnode = et.Element('interval') - intnode.set('source', interval.name) - - root.append(intnode) - - for item in ('lower', 'upper'): - if getattr(interval, item): - node = et.SubElement(intnode, item) - node.text = getattr(interval, item) - node.set('inclusive', str(getattr(interval, item + '_i')).lower()) - - def interval_from_xml(root): 'Returns an interval from xml' @@ -395,33 +353,7 @@ def is_in_interval(interval, kernel, bugid=None): return True - -def extract_genpatch(ebuild, directory, sources): - 'Returns a genpatch from an ebuild' - - pkg = portage.versions.catpkgsplit('sys-kernel/%s' % ebuild[:-7]) - - with open(os.path.join(directory, sources, ebuild), 'r') as ebuild_file: - content = ebuild_file.read() - - try: - genpatch_v = REGEX['gp_version'].findall(content)[0] - genpatch_w = REGEX['gp_want'].findall(content)[0] - except: - return None - - kernel = Kernel(pkg[1].replace('-sources', '')) - kernel.version = pkg[2] - kernel.revision = pkg[3] - - genpatch = Genpatch(pkg[2] + '-' + genpatch_v) - genpatch.kernel = kernel - genpatch.base = ('base' in genpatch_w) - genpatch.extras = ('extras' in genpatch_w) - - return genpatch - - +#TODO Add inline get_genpatch def parse_genpatch_list(directory): 'Returns a list containing all genpatches from portage' @@ -440,55 +372,6 @@ def parse_genpatch_list(directory): return patches -def read_genpatch_file(directory): - 'Read the genpatch file created by collector' - - patches = list() - filename = os.path.join(directory, 'genpatches.xml') - - try: - with open(filename, 'r+') as xml_data: - memory_map = mmap.mmap(xml_data.fileno(), 0) - root = et.parse(memory_map).getroot() - - except SyntaxError: - return list() - - except IOError: - return list() - - for tree in root: - kernel = extract_version(tree.get('kernel')) - - if kernel is None: - continue - - genpatch = Genpatch(tree.get('version')) - genpatch.kernel = kernel - genpatch.base = (tree.get('base') == 'true') - genpatch.extras = (tree.get('extras') == 'true') - - patches.append(genpatch) - - return patches - - -def write_genpatch_file(directory, patches): - 'Write the genpatch file with all genpatches' - - filename = os.path.join(directory, 'genpatches.xml') - root = et.Element('patches') - - for item in patches: - genpatch = et.SubElement(root, 'genpatch') - genpatch.set('kernel', repr(item.kernel)) - genpatch.set('version', item.version) - genpatch.set('base', str(item.base).lower()) - genpatch.set('extras', str(item.extras).lower()) - - write_xml(root, filename) - - def get_genpatch(patches, kernel): 'Returns the genpatch for a specific kernel' @@ -499,142 +382,30 @@ def get_genpatch(patches, kernel): return None -def parse_bugzilla_list(directory): - 'Returns a list containing all bugzilla kernel bugs' - - filename = os.path.join(directory, 'bugzilla.xml') - - with open(filename, 'r+') as buglist_file: - memory_map = mmap.mmap(buglist_file.fileno(), 0) - - buglist = REGEX['bugzilla'].findall(memory_map.read(-1)) - - return buglist - - -def parse_bugzilla_dict(directory, bugid): - 'Returns a vulnerability class containing information about a bug' - - filename = os.path.join(directory, bugid) - - try: - with open(filename, 'r+') as xml_data: - memory_map = mmap.mmap(xml_data.fileno(), 0) - root = et.parse(memory_map).getroot()[0] - - except IOError: #FIXME Handle Exception - return - - vul = Vulnerability(bugid) - - try: - vul.cvelist = extract_cves(root.find('short_desc').text) - if not vul.cvelist: - raise InvalidCveError(root.find('short_desc').text) - - for item in vul.cvelist: - if item != NOCVE: - if item not in CVES: - CVES[item] = vul.bugid - else: - raise CveDuplicateError(CVES[item]) +def extract_genpatch(ebuild, directory, sources): + 'Returns a genpatch from an ebuild' - vul.arch = root.find('rep_platform').text.lower() - vul.reported = root.find('creation_ts').text - vul.reporter = root.find('reporter').text.lower() - vul.status = root.find('bug_status').text.lower() + pkg = portage.versions.catpkgsplit('sys-kernel/%s' % ebuild[:-7]) - except AttributeError: - #TODO Error - pass + with open(os.path.join(directory, sources, ebuild), 'r') as ebuild_file: + content = ebuild_file.read() try: - vul.affected = interval_from_whiteboard( - root.find('status_whiteboard').text) - - except AttributeError: - raise InvalidWhiteboardError('Empty') - - return vul - - -def search_nvd_dict(nvd, vul): - 'Adds all matching cves found in the nvd dicitonay to vul' - - cves = list() - for item in vul.cvelist: - if item == NOCVE: - vul.cves = list() - return vul - - try: - cves.append(nvd[item]) - except KeyError: - raise NvdEntryError(item) - - vul.cves = cves - - return vul - - -def parse_nvd_dict(directory): - 'Returns a dictionary from the National Vulnerability Database' - - nvd = dict() - - for nvdfile in os.listdir(directory): - filename = os.path.join(directory, nvdfile) - - with open(filename, 'r+') as xml_data: - memory_map = mmap.mmap(xml_data.fileno(), 0) - root = et.parse(memory_map).getroot() - - namespace = root.tag[:-3] - - for tree in root: - cve = Cve(tree.get('name')) - cve.published = tree.get('published') - cve.severity = tree.get('severity') - cve.vector = tree.get('CVSS_vector') - cve.score = tree.get('CVSS_score') - - #FIXME - desc = tree.find('%sdesc/%sdescript/' % (namespace, namespace)) - if desc is not None: - cve.desc = desc.text - - #TODO Rework! - reftree = tree.find(namespace + 'refs') - reftree.tag = reftree.tag.replace(namespace, '') - for elem in reftree.findall('.//*'): - elem.tag = elem.tag.replace(namespace, '') - bugref = et.SubElement(reftree, 'ref') - bugref.set('source', 'GENTOO') - bugref.set('url', 'https://bugs.gentoo.org/show_bug.cgi?id=%s' % - cve.cve) - bugref.text = 'Gentoo %s' % cve.cve - - cve.refs = reftree - - nvd[cve.cve] = cve - - return nvd - - -def extract_cves(string): - 'Returns a list containing all CVEs of a particular string' - - cves = list() - string = string.replace('CAN', 'CVE') + genpatch_v = REGEX['gp_version'].findall(content)[0] + genpatch_w = REGEX['gp_want'].findall(content)[0] + except: + return None - if string in REGEX['m_nomatch'].findall(string): - return [NOCVE] + kernel = Kernel(pkg[1].replace('-sources', '')) + kernel.version = pkg[2] + kernel.revision = pkg[3] - for (year, split_cves) in REGEX['grp_all'].findall(string): - for cve in REGEX['grp_split'].findall(split_cves): - cves.append('CVE-%s-%s' % (year, cve)) + genpatch = Genpatch(pkg[2] + '-' + genpatch_v) + genpatch.kernel = kernel + genpatch.base = ('base' in genpatch_w) + genpatch.extras = ('extras' in genpatch_w) - return cves + return genpatch def parse_cve_files(directory): @@ -695,15 +466,14 @@ def eval_cve_files(directory, kernel, arch): return evaluation - -def is_affected(interval_list, kernel, item): #TODO Remove item +#TODO Remove item +def is_affected(interval_list, kernel, item): 'Returns true if a kernel is affected' kernel_gentoo = (kernel.source == 'gentoo' and kernel.genpatch is not None) kernel_affected = False kernel_linux_affected = False kernel_gp_affected = False - #TODO kernel_gp_exp_affected = False linux_interval = False gentoo_interval = False @@ -810,125 +580,6 @@ def read_cve_file(directory, bugid): return vul - -def write_cve_file(directory, vul): - 'Write a bug file containing all important information for kernel-check' - - filename = os.path.join(directory, vul.bugid + '.xml') - - root = et.Element('vulnerability') - bugroot = et.SubElement(root, 'bug') - - for element in BUGORDER: - if element == 'affected': - affectedroot = et.SubElement(bugroot, 'affected') - for item in vul.affected: - interval_to_xml(item, affectedroot) - else: - node = et.SubElement(bugroot, element) - node.text = getattr(vul, element) - - for cve in vul.cves: - cveroot = et.SubElement(root, 'cve') - for element in CVEORDER: - if element == 'refs': - cveroot.append(cve.refs) - else: - node = et.SubElement(cveroot, element) - node.text = getattr(cve, element) - - write_xml(root, filename) - - -def write_xml(root, filename): - 'Write root to a xml file' - - with open(filename, 'w') as xmlout: - __indent__(root) - doc = et.ElementTree(root) - doc.write(xmlout, encoding='utf-8') - - -def __indent__(node, level=0): - 'Indents xml layout for printing' - - i = '\n' + level * ' ' * 4 - if len(node): - if not node.text or not node.text.strip(): - node.text = i + ' ' * 4 - if not node.tail or not node.tail.strip(): - node.tail = i - for node in node: - __indent__(node, level + 1) - if not node.tail or not node.tail.strip(): - node.tail = i - else: - if level and (not node.tail or not node.tail.strip()): - node.tail = i - - -def interval_from_whiteboard(whiteboard): - 'Returns a list of intervals within a whiteboard string' - - expand = False #TODO - upper_inc = None - upper = None - lower_inc = None - lower = None - - affected = list() - - while len(whiteboard.strip()) > 0: - match = REGEX['wb_match'].match(whiteboard) - if not match: - raise InvalidWhiteboardError(whiteboard) - - name = match.group(1) - exp = match.group(2) - comp1 = match.group(3) - vers1 = match.group(4) - comp2 = match.group(5) - vers2 = match.group(6) - - if exp == '+': - expand = True - - if comp1 == '=' or comp1 == '==': - lower_inc = True - upper_inc = True - lower = vers1 - upper = vers1 - - if not REGEX['wb_version'].match(vers1): - raise InvalidWhiteboardError(whiteboard) - else: - for (char, version) in ((comp1, vers1), (comp2, vers2)): - if char == '<': - upper_inc = False - upper = version - elif char == '<=' or char == '=<': - upper_inc = True - upper = version - elif char == '>': - lower_inc = False - lower = version - elif char == '>=' or char == '=>': - lower_inc = True - lower = version - elif char: - raise InvalidWhiteboardError(whiteboard) - - if version and not REGEX['wb_version'].match(version): - raise InvalidWhiteboardError(whiteboard) - - affected.append(Interval(name, lower, upper, lower_inc, - upper_inc, expand)) - - whiteboard = match.group(7) - - return affected - - #TODO Use Exceptions def extract_version(release): 'Extracts revision, source and version out of a release tag' @@ -987,82 +638,3 @@ def all_version(source): return versions - -#TODO Remove BUG_ON; use Exceptions -def receive_file(directory, path, xml_file, - max_age = datetime.timedelta(0, 59*60)): - 'Generic download function' - - filename = os.path.join(directory, xml_file) - - if not FORCE: - if os.path.exists(filename): - age = datetime.datetime.now() - \ - datetime.datetime.fromtimestamp(os.path.getmtime(filename)) - if age < max_age: - BUG_ON('File %s - %sKB is recent enough [%s]' % - (filename, os.path.getsize(filename)/1024, - str(age)[:-7])) - return - try: - with closing(cStringIO.StringIO()) as data: - with closing(urllib.urlopen(path + xml_file)) as resource: - data.write(resource.read()) - - with open(filename, 'w') as output: - output.write(data.getvalue()) - - except IOError: - BUG_ON('Download failed!') #FIXME Handle exception - - BUG_ON('File %s - %sKB received' % - (filename, os.path.getsize(filename)/1024)) - - -def receive_nvd_recent(directory): - 'Download the latest CVEs file from the National Vulnerability Database' - - url = 'http://nvd.nist.gov/download/' - - receive_file(directory, url, 'nvdcve-recent.xml') - - -def receive_nvd_all(directory): - 'Download all earlier CVEs files from the National Vulnerability Database' - - url = 'http://nvd.nist.gov/download/' - year = datetime.datetime.now().year - - if year < 2002 or year > 2020: - year = 2020 - - for i in xrange(2002, year + 1): - receive_file(directory, url, 'nvdcve-%s.xml' % str(i), - max_age = datetime.timedelta(1)) - - -def receive_bugzilla_list(directory): - 'Download a list containing all Bugzilla kernel bugs' - - status = ['NEW', 'ASSIGNED', 'REOPENED', 'RESOLVED', 'VERIFIED', 'CLOSED'] - resolution = ['FIXED', 'LATER', 'TEST-REQUEST', 'UPSTREAM', '---'] - - url = ['https://bugs.gentoo.org/buglist.cgi?', - 'query_format=advanced&component=Kernel'] - - for i in status: - url.append('&bug_status=' + i) - for i in resolution: - url.append('&resolution=' + i) - url.append('#') - - receive_file(directory, ''.join(url), 'bugzilla.xml') - - -def receive_bugzilla_bug(directory, bugid): - 'Download the xml file of a particular Bugzilla kernel bug' - - url = 'https://bugs.gentoo.org/show_bug.cgi?ctype=xml&id=' - - receive_file(directory, url, bugid) - diff --git a/tools/cron.py b/tools/cron.py new file mode 100755 index 0000000..01e04af --- /dev/null +++ b/tools/cron.py @@ -0,0 +1,405 @@ +#!/usr/bin/env python +# kernel-check -- Kernel security information +# Copyright 2009-2009 Gentoo Foundation +# Distributed under the terms of the GNU General Public License v2 + +from contextlib import closing +import xml.etree.cElementTree as et +import cStringIO +import datetime +import logging +import mmap +import os +import portage +import re +import sys +import time +import urllib + + +class CronError(Exception): + def __init__(self, value): + self.value = value + +NOCVE = 'GENERIC-MAP-NOMATCH' +NOCVEDESC = 'This GENERIC identifier is not specific to any vulnerability. '\ + 'GENERIC-MAP-NOMATCH is used by products, databases, and ' \ + 'services to specify when a particular vulnerability element ' \ + 'does not map to a corresponding CVE entry.' +DELAY = 0.2 +SKIP = True +MINYEAR = 2002 +MAXYEAR = 2020 +NVDURL = 'http://nvd.nist.gov/' +BZURL = 'https://bugs.gentoo.org/' +STATE = ['NEW', 'ASSIGNED', 'REOPENED', 'RESOLVED', 'VERIFIED', 'CLOSED'] +RESOLUTION = ['FIXED', 'LATER', 'TEST-REQUEST', 'UPSTREAM', '---'] +BUGORDER = ['bugid', 'reporter', 'reported', 'status', 'arch', 'affected'] +CVEORDER = ['cve', 'published', 'desc', 'severity', 'vector', 'score', 'refs'] +FILEPATH = os.path.dirname(os.path.realpath(__file__)) +PORTDIR = portage.settings['PORTDIR'] +LOGFILE = None #os.path.join(FILEPATH, 'cron.log') +DIR = { + 'tmp' : os.path.join(FILEPATH, 'tmp'), + 'out' : os.path.join(PORTDIR, 'metadata', 'kernel'), + 'bug' : os.path.join(FILEPATH, 'tmp', 'bug'), + 'nvd' : os.path.join(FILEPATH, 'tmp', 'nvd') +} +REGEX = { + 'bugzilla' : re.compile(r'(?<=bug.cgi\?id=)\d*'), + 'grp_all' : re.compile(r'(?<=\()[ (]*CVE-(\d{4})' \ + r'([-,(){}|, \d]+)(?=\))'), + 'm_nomatch' : re.compile(r'.*GENERIC-MAP-NOMATCH.*'), + 'grp_split' : re.compile(r'(?<=\D)(\d{4})(?=\D|$)'), + 'wb_match' : re.compile(r'\s*\[\s*([^ +<=>]+)\s*([' \ + r'<=>]{1,2})\s*([^ <=>\]]+' \ + r')\s*(?:([<=>]{1,2})\s*([' \ + r'^ \]]+))?\s*\]\s*(.*)'), + 'wb_version' : re.compile(r'^(?:\d{1,2}\.){0,3}\d{1,2}' \ + r'(?:[-_](?:r|rc)?\d{1,2})*$') +} +CVES = dict() +logging.basicConfig(format='[%(asctime)s] %(levelname)-6s : %(message)s', + datefmt='%H:%M:%S', filename=LOGFILE, level=logging.DEBUG) + + +def main(argv): + 'Main function' + + logging.info('Running cron...') + + current_year = datetime.datetime.now().year + if current_year < MINYEAR or current_year > MAXYEAR: + current_year = MAXYEAR + + for directory in DIR: + if not os.path.isdir(DIR[directory]): + os.makedirs(DIR[directory]) + + logging.info('Receiving the latest xml file from the nvd') + + receive_file(DIR['nvd'], [NVDURL, 'download/'],'nvdcve-recent.xml') + + if not SKIP: + logging.info('Receiving earlier xml files from the nvd') + + for year in xrange(MINYEAR, current_year + 1): + receive_file(DIR['nvd'], [NVDURL, 'download/'], + 'nvdcve-%s.xml' % str(year)) + + logging.info('Receiving the kernel vulnerability list from bugzilla') + + url = [BZURL, 'buglist.cgi?query_format=advanced&component=Kernel'] + + for item in STATE: + url.append('&bug_status=' + item) + for item in RESOLUTION: + url.append('&resolution=' + item) + url.append('#') + + receive_file(DIR['tmp'], url, 'bugzilla.xml') + + filename = os.path.join(DIR['tmp'], 'bugzilla.xml') + with open(filename, 'r+') as buglist_file: + memory_map = mmap.mmap(buglist_file.fileno(), 0) + buglist = REGEX['bugzilla'].findall(memory_map.read(-1)) + + logging.info('Found %i kernel vulnerabilities' % len(buglist)) + + logging.info('Creating the nvd dictionary') + nvd_dict = parse_nvd_dict(DIR['nvd']) + + logging.info('Creating the xml files') + + created_files = 0 + for item in buglist: + try: + receive_file(DIR['bug'], [BZURL, 'show_bug.cgi?ctype=xml&id='], + item) + + vul = parse_bugzilla_dict(DIR['bug'], item) + + for cve in vul['cvelist']: + if cve == NOCVE: + vul['cves'] = [NOCVE] + break; #TODO + else: + try: + vul['cves'].append(nvd_dict[cve]) + except KeyError: + raise CronError('No Nvd entry: ' + cve) + + write_cve_file(DIR['out'], vul) + created_files += 1 + time.sleep(DELAY) + + except CronError, e: + logging.error('[%s] %s' % (item, e.value)) + + logging.info('Created %i xml files' % created_files) + + +def receive_file(directory, url, xml_file): + 'Generic download function' + + filename = os.path.join(directory, xml_file) + url.append(xml_file) + + try: + with closing(cStringIO.StringIO()) as data: + with closing(urllib.urlopen(''.join(url))) as resource: + data.write(resource.read()) + + with open(filename, 'w') as output: + output.write(data.getvalue()) + + except IOError: + logging.error('File %s - Download failed!' % filename) + + logging.debug('File %s - %sKB received' % + (filename, os.path.getsize(filename)/1024)) + + +def parse_nvd_dict(directory): + 'Returns a dictionary from the National Vulnerability Database' + + nvd = dict() + + for nvdfile in os.listdir(directory): + filename = os.path.join(directory, nvdfile) + try: + with open(filename, 'r+') as xml_data: + memory_map = mmap.mmap(xml_data.fileno(), 0) + + except SyntaxError: + continue + + root = et.parse(memory_map).getroot() + namespace = root.tag[:-3] + + for tree in root: + cve = { + 'cve' : tree.get('name'), + 'published' : tree.get('published'), + 'severity' : tree.get('severity'), + 'vector' : tree.get('CVSS_vector'), + 'score' : tree.get('CVSS_score') + } + + desc = tree.find('%sdesc/%sdescript/' % (namespace, namespace)) + if desc is not None: + cve['desc'] = desc.text + + reftree = tree.find(namespace + 'refs') + reftree.tag = reftree.tag.replace(namespace, '') + for elem in reftree.findall('.//*'): + elem.tag = elem.tag.replace(namespace, '') + + bugref = et.SubElement(reftree, 'ref') + bugref.set('source', 'GENTOO') + bugref.set('url', '%sshow_bug.cgi?id=%s' % (BZURL, cve['cve'])) + bugref.text = 'Gentoo %s' % cve['cve'] + + cve['refs'] = reftree + + nvd[cve['cve']] = cve + + return nvd + +def parse_bugzilla_dict(directory, bugid): + 'Returns a vulnerability class containing information about a bug' + + filename = os.path.join(directory, bugid) + + try: + with open(filename, 'r+') as xml_data: + memory_map = mmap.mmap(xml_data.fileno(), 0) + root = et.parse(memory_map).getroot()[0] + + except IOError: + return + + string = str() + + try: + string = root.find('short_desc').text + except AttributeError: + CronError('No Cve') + + try: + cvelist = list() + string = string.replace('CAN', 'CVE') + + if string in REGEX['m_nomatch'].findall(string): + cvelist = [NOCVE] + + for (year, split_cves) in REGEX['grp_all'].findall(string): + for cve in REGEX['grp_split'].findall(split_cves): + cvelist.append('CVE-%s-%s' % (year, cve)) + + vul = { + 'bugid' : bugid, + 'cvelist' : cvelist, + 'cves' : list(), + 'arch' : root.find('rep_platform').text.lower(), + 'reporter' : root.find('reporter').text.lower(), + 'reported' : root.find('creation_ts').text, + 'status' : root.find('bug_status').text.lower(), + } + + for item in vul['cvelist']: + if item != NOCVE: + if item not in CVES: + CVES[item] = vul.bugid + else: + raise CronError('Duplicate: ' + CVES[item]) + + except AttributeError: + pass + + try: + wb = root.find('status_whiteboard').text; + vul['affected'] = interval_from_whiteboard(wb) + + if vul['affected'] == None: + raise CronError('Invalid whiteboard: ' + wb) + + except AttributeError: + raise CronError('Empty whiteboard') + + return vul + + +def interval_from_whiteboard(whiteboard): + 'Returns a list of intervals within a whiteboard string' + + upper_inc = None + upper = None + lower_inc = None + lower = None + + affected = list() + + while len(whiteboard.strip()) > 0: + match = REGEX['wb_match'].match(whiteboard) + if not match: + return None + + name = match.group(1) + comp1 = match.group(2) + vers1 = match.group(3) + comp2 = match.group(4) + vers2 = match.group(5) + whiteboard = match.group(6) + + if comp1 == '=' or comp1 == '==': + lower_inc = True + upper_inc = True + lower = vers1 + upper = vers1 + + if not REGEX['wb_version'].match(vers1): + return None + else: + for (char, version) in ((comp1, vers1), (comp2, vers2)): + + if char == '<': + upper_inc = False + upper = version + elif char == '<=' or char == '=<': + upper_inc = True + upper = version + elif char == '>': + lower_inc = False + lower = version + elif char == '>=' or char == '=>': + lower_inc = True + lower = version + elif char: + return None + + if version and not REGEX['wb_version'].match(version): + return None + + interval = { + 'name' : name, + 'lower' : lower, + 'upper' : upper, + 'lower_inc' : lower_inc, + 'upper_inc' : upper_inc + } + + affected.append(interval) + + return affected + + +def write_cve_file(directory, vul): + 'Write a bug file containing all important information for kernel-check' + + filename = os.path.join(directory, vul['bugid'] + '.xml') + + root = et.Element('vulnerability') + bugroot = et.SubElement(root, 'bug') + + for element in BUGORDER: + if element == 'affected': + affectedroot = et.SubElement(bugroot, 'affected') + for item in vul['affected']: + intnode = et.Element('interval') + intnode.set('source', item['name']) + + affectedroot.append(intnode) + + for i in ('lower', 'upper'): + if item[i]: + node = et.SubElement(intnode, i) + node.text = item[i] + node.set('inclusive', + str(item[i + '_inc']).lower()) + else: + node = et.SubElement(bugroot, element) + node.text = vul[element] + + for cve in vul['cves']: + cveroot = et.SubElement(root, 'cve') + if cve == NOCVE: + node = et.SubElement(cveroot, 'cve') + node.text = NOCVE + node = et.SubElement(cveroot, 'desc') + node.text = NOCVEDESC + else: + for element in CVEORDER: + if element == 'refs': + cveroot.append(cve[element]) + else: + node = et.SubElement(cveroot, element) + node.text = cve[element] + + with open(filename, 'w') as xmlout: + __indent__(root) + doc = et.ElementTree(root) + doc.write(xmlout, encoding='utf-8') + + +def __indent__(node, level=0): + 'Indents xml layout for printing' + + i = '\n' + level * ' ' * 4 + if len(node): + if not node.text or not node.text.strip(): + node.text = i + ' ' * 4 + if not node.tail or not node.tail.strip(): + node.tail = i + for node in node: + __indent__(node, level + 1) + if not node.tail or not node.tail.strip(): + node.tail = i + else: + if level and (not node.tail or not node.tail.strip()): + node.tail = i + + +if __name__ == '__main__': + main(sys.argv[1:]) + diff --git a/findcommit.sh b/tools/findcommit.sh index 88c5857..770078b 100755 --- a/findcommit.sh +++ b/tools/findcommit.sh @@ -47,11 +47,11 @@ for B in $BRANCHES ; do WHITEBOARD="$WHITEBOARD [linux <${VERSION}]" else if [ "${LASTFIXEDBRANCH}" == "${VERSION/-rc*/}" ] ; then - WHITEBOARD="${WHITEBOARD} ${BAD}[linux >= - ${LASTFIXEDBRANCH} <${VERSION}]${NORMAL}" + WHITEBOARD="${WHITEBOARD} ${BAD}[linux >=" \ + "${LASTFIXEDBRANCH} <${VERSION}]${NORMAL}" else - WHITEBOARD="${WHITEBOARD} [linux >= - ${LASTFIXEDBRANCH} <${VERSION}]" + WHITEBOARD="${WHITEBOARD} [linux >=" \ + "${LASTFIXEDBRANCH} <${VERSION}]" fi fi LASTFIXEDBRANCH=$NEXTBRANCH diff --git a/testsuite.py b/tools/testsuite.py index 1a9287c..1a9287c 100755 --- a/testsuite.py +++ b/tools/testsuite.py |