Bump Buildroot to 2021.02-rc3 (#1260)

* Rebase patches to Buildroot 2021.02-rc3

* Update Buildroot to 2021.02-rc3

* Declare Kernel headers to be Linux version 5.10 (since they are, and new Buildroot knows about 5.10)
This commit is contained in:
Stefan Agner
2021-03-04 00:50:33 +01:00
committed by GitHub
parent b77d633382
commit f358f322da
2130 changed files with 23612 additions and 21038 deletions

View File

@@ -18,6 +18,10 @@ def main():
if not sys.argv[1].startswith('qemu_'):
sys.exit(0)
if not os.path.exists('output/images/start-qemu.sh'):
print('qemu-start.sh is missing, cannot test.')
sys.exit(0)
qemu_start = os.path.join(os.getcwd(), 'output/images/start-qemu.sh')
child = pexpect.spawn(qemu_start, ['serial-only'],
@@ -32,7 +36,7 @@ def main():
time.sleep(1)
try:
child.expect(["buildroot login:", pexpect.TIMEOUT], timeout=60)
child.expect(["buildroot login:"], timeout=60)
except pexpect.EOF as e:
# Some emulations require a fork of qemu-system, which may be
# missing on the system, and is not provided by Buildroot.
@@ -54,7 +58,7 @@ def main():
child.sendline("root\r")
try:
child.expect(["# ", pexpect.TIMEOUT], timeout=60)
child.expect(["# "], timeout=60)
except pexpect.EOF:
print("Cannot connect to shell")
sys.exit(1)
@@ -65,7 +69,7 @@ def main():
child.sendline("poweroff\r")
try:
child.expect(["System halted", pexpect.TIMEOUT], timeout=60)
child.expect(["System halted"], timeout=60)
child.expect(pexpect.EOF)
except pexpect.EOF:
pass

View File

@@ -0,0 +1,174 @@
#!/usr/bin/env python3
import xml.etree.ElementTree as ET
from xml.etree.ElementTree import Element, SubElement
import gzip
import os
import requests
import time
from xml.dom import minidom
VALID_REFS = ['VENDOR', 'VERSION', 'CHANGE_LOG', 'PRODUCT', 'PROJECT', 'ADVISORY']
CPEDB_URL = "https://static.nvd.nist.gov/feeds/xml/cpe/dictionary/official-cpe-dictionary_v2.3.xml.gz"
ns = {
'': 'http://cpe.mitre.org/dictionary/2.0',
'cpe-23': 'http://scap.nist.gov/schema/cpe-extension/2.3',
'xml': 'http://www.w3.org/XML/1998/namespace'
}
class CPE:
def __init__(self, cpe_str, titles, refs):
self.cpe_str = cpe_str
self.titles = titles
self.references = refs
self.cpe_cur_ver = "".join(self.cpe_str.split(":")[5:6])
def update_xml_dict(self):
ET.register_namespace('', 'http://cpe.mitre.org/dictionary/2.0')
cpes = Element('cpe-list')
cpes.set('xmlns:cpe-23', "http://scap.nist.gov/schema/cpe-extension/2.3")
cpes.set('xmlns:ns6', "http://scap.nist.gov/schema/scap-core/0.1")
cpes.set('xmlns:scap-core', "http://scap.nist.gov/schema/scap-core/0.3")
cpes.set('xmlns:config', "http://scap.nist.gov/schema/configuration/0.1")
cpes.set('xmlns:xsi', "http://www.w3.org/2001/XMLSchema-instance")
cpes.set('xmlns:meta', "http://scap.nist.gov/schema/cpe-dictionary-metadata/0.2")
cpes.set('xsi:schemaLocation', " ".join(["http://scap.nist.gov/schema/cpe-extension/2.3",
"https://scap.nist.gov/schema/cpe/2.3/cpe-dictionary-extension_2.3.xsd",
"http://cpe.mitre.org/dictionary/2.0",
"https://scap.nist.gov/schema/cpe/2.3/cpe-dictionary_2.3.xsd",
"http://scap.nist.gov/schema/cpe-dictionary-metadata/0.2",
"https://scap.nist.gov/schema/cpe/2.1/cpe-dictionary-metadata_0.2.xsd",
"http://scap.nist.gov/schema/scap-core/0.3",
"https://scap.nist.gov/schema/nvd/scap-core_0.3.xsd",
"http://scap.nist.gov/schema/configuration/0.1",
"https://scap.nist.gov/schema/nvd/configuration_0.1.xsd",
"http://scap.nist.gov/schema/scap-core/0.1",
"https://scap.nist.gov/schema/nvd/scap-core_0.1.xsd"]))
item = SubElement(cpes, 'cpe-item')
cpe_short_name = CPE.short_name(self.cpe_str)
cpe_new_ver = CPE.version_update(self.cpe_str)
item.set('name', 'cpe:/' + cpe_short_name)
self.titles[0].text.replace(self.cpe_cur_ver, cpe_new_ver)
for title in self.titles:
item.append(title)
if self.references:
item.append(self.references)
cpe23item = SubElement(item, 'cpe-23:cpe23-item')
cpe23item.set('name', self.cpe_str)
# Generate the XML as a string
xmlstr = ET.tostring(cpes)
# And use minidom to pretty print the XML
return minidom.parseString(xmlstr).toprettyxml(encoding="utf-8").decode("utf-8")
@staticmethod
def version(cpe):
return cpe.split(":")[5]
@staticmethod
def product(cpe):
return cpe.split(":")[4]
@staticmethod
def short_name(cpe):
return ":".join(cpe.split(":")[2:6])
@staticmethod
def version_update(cpe):
return ":".join(cpe.split(":")[5:6])
@staticmethod
def no_version(cpe):
return ":".join(cpe.split(":")[:5])
class CPEDB:
def __init__(self, nvd_path):
self.all_cpes = dict()
self.all_cpes_no_version = dict()
self.nvd_path = nvd_path
def get_xml_dict(self):
print("CPE: Setting up NIST dictionary")
if not os.path.exists(os.path.join(self.nvd_path, "cpe")):
os.makedirs(os.path.join(self.nvd_path, "cpe"))
cpe_dict_local = os.path.join(self.nvd_path, "cpe", os.path.basename(CPEDB_URL))
if not os.path.exists(cpe_dict_local) or os.stat(cpe_dict_local).st_mtime < time.time() - 86400:
print("CPE: Fetching xml manifest from [" + CPEDB_URL + "]")
cpe_dict = requests.get(CPEDB_URL)
open(cpe_dict_local, "wb").write(cpe_dict.content)
print("CPE: Unzipping xml manifest...")
nist_cpe_file = gzip.GzipFile(fileobj=open(cpe_dict_local, 'rb'))
print("CPE: Converting xml manifest to dict...")
tree = ET.parse(nist_cpe_file)
all_cpedb = tree.getroot()
self.parse_dict(all_cpedb)
def parse_dict(self, all_cpedb):
# Cycle through the dict and build two dict to be used for custom
# lookups of partial and complete CPE objects
# The objects are then used to create new proposed XML updates if
# if is determined one is required
# Out of the different language titles, select English
for cpe in all_cpedb.findall(".//{http://cpe.mitre.org/dictionary/2.0}cpe-item"):
cpe_titles = []
for title in cpe.findall('.//{http://cpe.mitre.org/dictionary/2.0}title[@xml:lang="en-US"]', ns):
title.tail = None
cpe_titles.append(title)
# Some older CPE don't include references, if they do, make
# sure we handle the case of one ref needing to be packed
# in a list
cpe_ref = cpe.find(".//{http://cpe.mitre.org/dictionary/2.0}references")
if cpe_ref:
for ref in cpe_ref.findall(".//{http://cpe.mitre.org/dictionary/2.0}reference"):
ref.tail = None
ref.text = ref.text.upper()
if ref.text not in VALID_REFS:
ref.text = ref.text + "-- UPDATE this entry, here are some examples and just one word should be used -- " + ' '.join(VALID_REFS) # noqa E501
cpe_ref.tail = None
cpe_ref.text = None
cpe_str = cpe.find(".//{http://scap.nist.gov/schema/cpe-extension/2.3}cpe23-item").get('name')
item = CPE(cpe_str, cpe_titles, cpe_ref)
cpe_str_no_version = CPE.no_version(cpe_str)
# This dict must have a unique key for every CPE version
# which allows matching to the specific obj data of that
# NIST dict entry
self.all_cpes.update({cpe_str: item})
# This dict has one entry for every CPE (w/o version) to allow
# partial match (no valid version) check (the obj is saved and
# used as seed for suggested xml updates. By updating the same
# non-version'd entry, it assumes the last update here is the
# latest version in the NIST dict)
self.all_cpes_no_version.update({cpe_str_no_version: item})
def find_partial(self, cpe_str):
cpe_str_no_version = CPE.no_version(cpe_str)
if cpe_str_no_version in self.all_cpes_no_version:
return cpe_str_no_version
def find_partial_obj(self, cpe_str):
cpe_str_no_version = CPE.no_version(cpe_str)
if cpe_str_no_version in self.all_cpes_no_version:
return self.all_cpes_no_version[cpe_str_no_version]
def find_partial_latest_version(self, cpe_str_partial):
cpe_obj = self.find_partial_obj(cpe_str_partial)
return cpe_obj.cpe_cur_ver
def find(self, cpe_str):
if self.find_partial(cpe_str):
if cpe_str in self.all_cpes:
return cpe_str
def gen_update_xml(self, cpe_str):
cpe = self.find_partial_obj(cpe_str)
return cpe.update_xml_dict()

View File

@@ -47,6 +47,24 @@ ops = {
}
# Check if two CPE IDs match each other
def cpe_matches(cpe1, cpe2):
cpe1_elems = cpe1.split(":")
cpe2_elems = cpe2.split(":")
remains = filter(lambda x: x[0] not in ["*", "-"] and x[1] not in ["*", "-"] and x[0] != x[1],
zip(cpe1_elems, cpe2_elems))
return len(list(remains)) == 0
def cpe_product(cpe):
return cpe.split(':')[4]
def cpe_version(cpe):
return cpe.split(':')[5]
class CVE:
"""An accessor class for CVE Items in NVD files"""
CVE_AFFECTS = 1
@@ -134,7 +152,11 @@ class CVE:
for cpe in node.get('cpe_match', ()):
if not cpe['vulnerable']:
return
vendor, product, version = cpe['cpe23Uri'].split(':')[3:6]
product = cpe_product(cpe['cpe23Uri'])
version = cpe_version(cpe['cpe23Uri'])
# ignore when product is '-', which means N/A
if product == '-':
return
op_start = ''
op_end = ''
v_start = ''
@@ -144,10 +166,6 @@ class CVE:
# Version is defined, this is a '=' match
op_start = '='
v_start = version
elif version == '-':
# no version information is available
op_start = '='
v_start = version
else:
# Parse start version, end version and operators
if 'versionStartIncluding' in cpe:
@@ -167,8 +185,7 @@ class CVE:
v_end = cpe['versionEndExcluding']
yield {
'vendor': vendor,
'product': product,
'id': cpe['cpe23Uri'],
'v_start': v_start,
'op_start': op_start,
'v_end': v_end,
@@ -186,11 +203,11 @@ class CVE:
return self.nvd_cve['cve']['CVE_data_meta']['ID']
@property
def pkg_names(self):
"""The set of package names referred by this CVE definition"""
return set(p['product'] for p in self.each_cpe())
def affected_products(self):
"""The set of CPE products referred by this CVE definition"""
return set(cpe_product(p['id']) for p in self.each_cpe())
def affects(self, name, version, cve_ignore_list):
def affects(self, name, version, cve_ignore_list, cpeid=None):
"""
True if the Buildroot Package object passed as argument is affected
by this CVE.
@@ -203,14 +220,15 @@ class CVE:
print("Cannot parse package '%s' version '%s'" % (name, version))
pkg_version = None
# if we don't have a cpeid, build one based on name and version
if not cpeid:
cpeid = "cpe:2.3:*:*:%s:%s:*:*:*:*:*:*:*" % (name, version)
for cpe in self.each_cpe():
if cpe['product'] != name:
if not cpe_matches(cpe['id'], cpeid):
continue
if cpe['v_start'] == '-':
return self.CVE_AFFECTS
if not cpe['v_start'] and not cpe['v_end']:
print("No CVE affected version")
continue
return self.CVE_AFFECTS
if not pkg_version:
continue

View File

@@ -263,8 +263,6 @@ class Toolchain:
# glibc doesn't support static only configuration
depends.append("!BR2_STATIC_LIBS")
selects.append("BR2_TOOLCHAIN_EXTERNAL_GLIBC")
# all glibc toolchains have RPC support
selects.append("BR2_TOOLCHAIN_HAS_NATIVE_RPC")
elif frag.startswith("BR2_TOOLCHAIN_EXTERNAL_CUSTOM_MUSL"):
# musl needs mmu support
depends.append("BR2_USE_MMU")

View File

@@ -32,7 +32,7 @@ brpath = os.path.normpath(os.path.join(os.path.dirname(__file__), "..", ".."))
sys.path.append(os.path.join(brpath, "utils"))
from getdeveloperlib import parse_developers # noqa: E402
from cpedb import CPEDB # noqa: E402
INFRA_RE = re.compile(r"\$\(eval \$\(([a-z-]*)-package\)\)")
URL_RE = re.compile(r"\s*https?://\S*\s*$")
@@ -77,6 +77,7 @@ class Package:
all_license_files = list()
all_versions = dict()
all_ignored_cves = dict()
all_cpeids = dict()
# This is the list of all possible checks. Add new checks to this list so
# a tool that post-processeds the json output knows the checks before
# iterating over the packages.
@@ -97,7 +98,9 @@ class Package:
self.current_version = None
self.url = None
self.url_worker = None
self.cpeid = None
self.cves = list()
self.ignored_cves = list()
self.latest_version = {'status': RM_API_STATUS_ERROR, 'version': None, 'id': None}
self.status = {}
@@ -142,8 +145,8 @@ class Package:
self.infras = list()
with open(os.path.join(brpath, self.path), 'r') as f:
lines = f.readlines()
for l in lines:
match = INFRA_RE.match(l)
for line in lines:
match = INFRA_RE.match(line)
if not match:
continue
infra = match.group(1)
@@ -212,6 +215,22 @@ class Package:
if var in self.all_versions:
self.current_version = self.all_versions[var]
def set_cpeid(self):
"""
Fills in the .cpeid field
"""
var = self.pkgvar()
if not self.has_valid_infra:
self.status['cpe'] = ("na", "no valid package infra")
return
if var in self.all_cpeids:
self.cpeid = self.all_cpeids[var]
# Set a preliminary status, it might be overridden by check_package_cpes()
self.status['cpe'] = ("warning", "not checked against CPE dictionnary")
else:
self.status['cpe'] = ("error", "no verified CPE identifier")
def set_check_package_warnings(self):
"""
Fills in the .warnings and .status['pkg-check'] fields
@@ -235,12 +254,11 @@ class Package:
self.status['pkg-check'] = ("error", "{} warnings".format(self.warnings))
return
@property
def ignored_cves(self):
def set_ignored_cves(self):
"""
Give the list of CVEs ignored by the package
"""
return list(self.all_ignored_cves.get(self.pkgvar(), []))
self.ignored_cves = list(self.all_ignored_cves.get(self.pkgvar(), []))
def set_developers(self, developers):
"""
@@ -258,7 +276,13 @@ class Package:
self.status['developers'] = ("warning", "no developers")
def is_status_ok(self, name):
return self.status[name][0] == 'ok'
return name in self.status and self.status[name][0] == 'ok'
def is_status_error(self, name):
return name in self.status and self.status[name][0] == 'error'
def is_status_na(self, name):
return name in self.status and self.status[name][0] == 'na'
def __eq__(self, other):
return self.path == other.path
@@ -335,13 +359,13 @@ def get_pkglist(npackages, package_list):
def get_config_packages():
cmd = ["make", "--no-print-directory", "show-info"]
js = json.loads(subprocess.check_output(cmd))
return js.keys()
return set([v["name"] for v in js.values()])
def package_init_make_info():
# Fetch all variables at once
variables = subprocess.check_output(["make", "BR2_HAVE_DOT_CONFIG=y", "-s", "printvars",
"VARS=%_LICENSE %_LICENSE_FILES %_VERSION %_IGNORE_CVES"])
"VARS=%_LICENSE %_LICENSE_FILES %_VERSION %_IGNORE_CVES %_CPE_ID"])
variable_list = variables.decode().splitlines()
# We process first the host package VERSION, and then the target
@@ -379,6 +403,10 @@ def package_init_make_info():
pkgvar = pkgvar[:-12]
Package.all_ignored_cves[pkgvar] = value.split()
elif pkgvar.endswith("_CPE_ID"):
pkgvar = pkgvar[:-7]
Package.all_cpeids[pkgvar] = value
check_url_count = 0
@@ -535,16 +563,54 @@ async def check_package_latest_version(packages):
await asyncio.wait(tasks)
def check_package_cve_affects(cve, cpe_product_pkgs):
for product in cve.affected_products:
if product not in cpe_product_pkgs:
continue
for pkg in cpe_product_pkgs[product]:
if cve.affects(pkg.name, pkg.current_version, pkg.ignored_cves, pkg.cpeid) == cve.CVE_AFFECTS:
pkg.cves.append(cve.identifier)
def check_package_cves(nvd_path, packages):
if not os.path.isdir(nvd_path):
os.makedirs(nvd_path)
cpe_product_pkgs = defaultdict(list)
for pkg in packages:
if not pkg.has_valid_infra:
pkg.status['cve'] = ("na", "no valid package infra")
continue
if not pkg.current_version:
pkg.status['cve'] = ("na", "no version information available")
continue
if pkg.cpeid:
cpe_product = cvecheck.cpe_product(pkg.cpeid)
cpe_product_pkgs[cpe_product].append(pkg)
else:
cpe_product_pkgs[pkg.name].append(pkg)
for cve in cvecheck.CVE.read_nvd_dir(nvd_path):
for pkg_name in cve.pkg_names:
if pkg_name in packages:
pkg = packages[pkg_name]
if cve.affects(pkg.name, pkg.current_version, pkg.ignored_cves) == cve.CVE_AFFECTS:
pkg.cves.append(cve.identifier)
check_package_cve_affects(cve, cpe_product_pkgs)
for pkg in packages:
if 'cve' not in pkg.status:
if pkg.cves:
pkg.status['cve'] = ("error", "affected by CVEs")
else:
pkg.status['cve'] = ("ok", "not affected by CVEs")
def check_package_cpes(nvd_path, packages):
cpedb = CPEDB(nvd_path)
cpedb.get_xml_dict()
for p in packages:
if not p.cpeid:
continue
if cpedb.find(p.cpeid):
p.status['cpe'] = ("ok", "verified CPE identifier")
else:
p.status['cpe'] = ("error", "CPE identifier unknown in CPE database")
def calculate_stats(packages):
@@ -586,6 +652,10 @@ def calculate_stats(packages):
stats["total-cves"] += len(pkg.cves)
if len(pkg.cves) != 0:
stats["pkg-cves"] += 1
if pkg.cpeid:
stats["cpe-id"] += 1
else:
stats["no-cpe-id"] += 1
return stats
@@ -641,6 +711,30 @@ td.version-error {
background: #ccc;
}
td.cpe-ok {
background: #d2ffc4;
}
td.cpe-nok {
background: #ff9a69;
}
td.cpe-unknown {
background: #ffd870;
}
td.cve-ok {
background: #d2ffc4;
}
td.cve-nok {
background: #ff9a69;
}
td.cve-unknown {
background: #ffd870;
}
</style>
<title>Statistics of Buildroot packages</title>
</head>
@@ -799,13 +893,35 @@ def dump_html_pkg(f, pkg):
# CVEs
td_class = ["centered"]
if len(pkg.cves) == 0:
td_class.append("correct")
if pkg.is_status_ok("cve"):
td_class.append("cve-ok")
elif pkg.is_status_error("cve"):
td_class.append("cve-nok")
else:
td_class.append("wrong")
td_class.append("cve-unknown")
f.write(" <td class=\"%s\">\n" % " ".join(td_class))
for cve in pkg.cves:
f.write(" <a href=\"https://security-tracker.debian.org/tracker/%s\">%s<br/>\n" % (cve, cve))
if pkg.is_status_error("cve"):
for cve in pkg.cves:
f.write(" <a href=\"https://security-tracker.debian.org/tracker/%s\">%s<br/>\n" % (cve, cve))
elif pkg.is_status_na("cve"):
f.write(" %s" % pkg.status['cve'][1])
else:
f.write(" N/A\n")
f.write(" </td>\n")
# CPE ID
td_class = ["left"]
if pkg.is_status_ok("cpe"):
td_class.append("cpe-ok")
elif pkg.is_status_error("cpe"):
td_class.append("cpe-nok")
else:
td_class.append("cpe-unknown")
f.write(" <td class=\"%s\">\n" % " ".join(td_class))
if pkg.cpeid:
f.write(" <code>%s</code>\n" % pkg.cpeid)
if not pkg.is_status_ok("cpe"):
f.write(" %s%s\n" % ("<br/>" if pkg.cpeid else "", pkg.status['cpe'][1]))
f.write(" </td>\n")
f.write(" </tr>\n")
@@ -826,6 +942,7 @@ def dump_html_all_pkgs(f, packages):
<td class=\"centered\">Warnings</td>
<td class=\"centered\">Upstream URL</td>
<td class=\"centered\">CVEs</td>
<td class=\"centered\">CPE ID</td>
</tr>
""")
for pkg in sorted(packages):
@@ -868,6 +985,10 @@ def dump_html_stats(f, stats):
stats["pkg-cves"])
f.write("<tr><td>Total number of CVEs affecting all packages</td><td>%s</td></tr>\n" %
stats["total-cves"])
f.write("<tr><td>Packages with CPE ID</td><td>%s</td></tr>\n" %
stats["cpe-id"])
f.write("<tr><td>Packages without CPE ID</td><td>%s</td></tr>\n" %
stats["no-cpe-id"])
f.write("</table>\n")
@@ -984,7 +1105,9 @@ def __main__():
pkg.set_patch_count()
pkg.set_check_package_warnings()
pkg.set_current_version()
pkg.set_cpeid()
pkg.set_url()
pkg.set_ignored_cves()
pkg.set_developers(developers)
print("Checking URL status")
loop = asyncio.get_event_loop()
@@ -994,7 +1117,8 @@ def __main__():
loop.run_until_complete(check_package_latest_version(packages))
if args.nvd_path:
print("Checking packages CVEs")
check_package_cves(args.nvd_path, {p.name: p for p in packages})
check_package_cves(args.nvd_path, packages)
check_package_cpes(args.nvd_path, packages)
print("Calculate stats")
stats = calculate_stats(packages)
if args.html: