d-two/operating-system
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Bump Buildroot to 2021.02-rc3 (#1260)

Browse Source 
* Rebase patches to Buildroot 2021.02-rc3

* Update Buildroot to 2021.02-rc3

* Declare Kernel headers to be Linux version 5.10 (since they are, and new Buildroot knows about 5.10)
This commit is contained in:
Stefan Agner
2021-03-04 00:50:33 +01:00
committed by GitHub
parent b77d633382
commit f358f322da
2130 changed files with 23612 additions and 21038 deletions
Download Patch File Download Diff File Expand all files Collapse all files

164
buildroot/support/scripts/pkg-stats
View File

@@ -32,7 +32,7 @@ brpath = os.path.normpath(os.path.join(os.path.dirname(__file__), "..", ".."))
sys.path.append(os.path.join(brpath, "utils"))
from getdeveloperlib import parse_developers # noqa: E402
from cpedb import CPEDB # noqa: E402
INFRA_RE = re.compile(r"\$\(eval \$\(([a-z-]*)-package\)\)")
URL_RE = re.compile(r"\s*https?://\S*\s*$")
@@ -77,6 +77,7 @@ class Package:
all_license_files = list()
all_versions = dict()
all_ignored_cves = dict()
all_cpeids = dict()
# This is the list of all possible checks. Add new checks to this list so
# a tool that post-processeds the json output knows the checks before
# iterating over the packages.
@@ -97,7 +98,9 @@ class Package:
self.current_version = None
self.url = None
self.url_worker = None
self.cpeid = None
self.cves = list()
self.ignored_cves = list()
self.latest_version = {'status': RM_API_STATUS_ERROR, 'version': None, 'id': None}
self.status = {}
@@ -142,8 +145,8 @@ class Package:
self.infras = list()
with open(os.path.join(brpath, self.path), 'r') as f:
lines = f.readlines()
for l in lines:
match = INFRA_RE.match(l)
for line in lines:
match = INFRA_RE.match(line)
if not match:
continue
infra = match.group(1)
@@ -212,6 +215,22 @@ class Package:
if var in self.all_versions:
self.current_version = self.all_versions[var]
def set_cpeid(self):
"""
Fills in the .cpeid field
"""
var = self.pkgvar()
if not self.has_valid_infra:
self.status['cpe'] = ("na", "no valid package infra")
return
if var in self.all_cpeids:
self.cpeid = self.all_cpeids[var]
# Set a preliminary status, it might be overridden by check_package_cpes()
self.status['cpe'] = ("warning", "not checked against CPE dictionnary")
else:
self.status['cpe'] = ("error", "no verified CPE identifier")
def set_check_package_warnings(self):
"""
Fills in the .warnings and .status['pkg-check'] fields
@@ -235,12 +254,11 @@ class Package:
self.status['pkg-check'] = ("error", "{} warnings".format(self.warnings))
return
@property
def ignored_cves(self):
def set_ignored_cves(self):
"""
Give the list of CVEs ignored by the package
"""
return list(self.all_ignored_cves.get(self.pkgvar(), []))
self.ignored_cves = list(self.all_ignored_cves.get(self.pkgvar(), []))
def set_developers(self, developers):
"""
@@ -258,7 +276,13 @@ class Package:
self.status['developers'] = ("warning", "no developers")
def is_status_ok(self, name):
return self.status[name][0] == 'ok'
return name in self.status and self.status[name][0] == 'ok'
def is_status_error(self, name):
return name in self.status and self.status[name][0] == 'error'
def is_status_na(self, name):
return name in self.status and self.status[name][0] == 'na'
def __eq__(self, other):
return self.path == other.path
@@ -335,13 +359,13 @@ def get_pkglist(npackages, package_list):
def get_config_packages():
cmd = ["make", "--no-print-directory", "show-info"]
js = json.loads(subprocess.check_output(cmd))
return js.keys()
return set([v["name"] for v in js.values()])
def package_init_make_info():
# Fetch all variables at once
variables = subprocess.check_output(["make", "BR2_HAVE_DOT_CONFIG=y", "-s", "printvars",
"VARS=%_LICENSE %_LICENSE_FILES %_VERSION %_IGNORE_CVES"])
"VARS=%_LICENSE %_LICENSE_FILES %_VERSION %_IGNORE_CVES %_CPE_ID"])
variable_list = variables.decode().splitlines()
# We process first the host package VERSION, and then the target
@@ -379,6 +403,10 @@ def package_init_make_info():
pkgvar = pkgvar[:-12]
Package.all_ignored_cves[pkgvar] = value.split()
elif pkgvar.endswith("_CPE_ID"):
pkgvar = pkgvar[:-7]
Package.all_cpeids[pkgvar] = value
check_url_count = 0
@@ -535,16 +563,54 @@ async def check_package_latest_version(packages):
await asyncio.wait(tasks)
def check_package_cve_affects(cve, cpe_product_pkgs):
for product in cve.affected_products:
if product not in cpe_product_pkgs:
continue
for pkg in cpe_product_pkgs[product]:
if cve.affects(pkg.name, pkg.current_version, pkg.ignored_cves, pkg.cpeid) == cve.CVE_AFFECTS:
pkg.cves.append(cve.identifier)
def check_package_cves(nvd_path, packages):
if not os.path.isdir(nvd_path):
os.makedirs(nvd_path)
cpe_product_pkgs = defaultdict(list)
for pkg in packages:
if not pkg.has_valid_infra:
pkg.status['cve'] = ("na", "no valid package infra")
continue
if not pkg.current_version:
pkg.status['cve'] = ("na", "no version information available")
continue
if pkg.cpeid:
cpe_product = cvecheck.cpe_product(pkg.cpeid)
cpe_product_pkgs[cpe_product].append(pkg)
else:
cpe_product_pkgs[pkg.name].append(pkg)
for cve in cvecheck.CVE.read_nvd_dir(nvd_path):
for pkg_name in cve.pkg_names:
if pkg_name in packages:
pkg = packages[pkg_name]
if cve.affects(pkg.name, pkg.current_version, pkg.ignored_cves) == cve.CVE_AFFECTS:
pkg.cves.append(cve.identifier)
check_package_cve_affects(cve, cpe_product_pkgs)
for pkg in packages:
if 'cve' not in pkg.status:
if pkg.cves:
pkg.status['cve'] = ("error", "affected by CVEs")
else:
pkg.status['cve'] = ("ok", "not affected by CVEs")
def check_package_cpes(nvd_path, packages):
cpedb = CPEDB(nvd_path)
cpedb.get_xml_dict()
for p in packages:
if not p.cpeid:
continue
if cpedb.find(p.cpeid):
p.status['cpe'] = ("ok", "verified CPE identifier")
else:
p.status['cpe'] = ("error", "CPE identifier unknown in CPE database")
def calculate_stats(packages):
@@ -586,6 +652,10 @@ def calculate_stats(packages):
stats["total-cves"] += len(pkg.cves)
if len(pkg.cves) != 0:
stats["pkg-cves"] += 1
if pkg.cpeid:
stats["cpe-id"] += 1
else:
stats["no-cpe-id"] += 1
return stats
@@ -641,6 +711,30 @@ td.version-error {
background: #ccc;
}
td.cpe-ok {
background: #d2ffc4;
}
td.cpe-nok {
background: #ff9a69;
}
td.cpe-unknown {
background: #ffd870;
}
td.cve-ok {
background: #d2ffc4;
}
td.cve-nok {
background: #ff9a69;
}
td.cve-unknown {
background: #ffd870;
}
</style>
<title>Statistics of Buildroot packages</title>
</head>
@@ -799,13 +893,35 @@ def dump_html_pkg(f, pkg):
# CVEs
td_class = ["centered"]
if len(pkg.cves) == 0:
td_class.append("correct")
if pkg.is_status_ok("cve"):
td_class.append("cve-ok")
elif pkg.is_status_error("cve"):
td_class.append("cve-nok")
else:
td_class.append("wrong")
td_class.append("cve-unknown")
f.write(" <td class=\"%s\">\n" % " ".join(td_class))
for cve in pkg.cves:
f.write(" <a href=\"https://security-tracker.debian.org/tracker/%s\">%s<br/>\n" % (cve, cve))
if pkg.is_status_error("cve"):
for cve in pkg.cves:
f.write(" <a href=\"https://security-tracker.debian.org/tracker/%s\">%s<br/>\n" % (cve, cve))
elif pkg.is_status_na("cve"):
f.write(" %s" % pkg.status['cve'][1])
else:
f.write(" N/A\n")
f.write(" </td>\n")
# CPE ID
td_class = ["left"]
if pkg.is_status_ok("cpe"):
td_class.append("cpe-ok")
elif pkg.is_status_error("cpe"):
td_class.append("cpe-nok")
else:
td_class.append("cpe-unknown")
f.write(" <td class=\"%s\">\n" % " ".join(td_class))
if pkg.cpeid:
f.write(" <code>%s</code>\n" % pkg.cpeid)
if not pkg.is_status_ok("cpe"):
f.write(" %s%s\n" % ("<br/>" if pkg.cpeid else "", pkg.status['cpe'][1]))
f.write(" </td>\n")
f.write(" </tr>\n")
@@ -826,6 +942,7 @@ def dump_html_all_pkgs(f, packages):
<td class=\"centered\">Warnings</td>
<td class=\"centered\">Upstream URL</td>
<td class=\"centered\">CVEs</td>
<td class=\"centered\">CPE ID</td>
</tr>
""")
for pkg in sorted(packages):
@@ -868,6 +985,10 @@ def dump_html_stats(f, stats):
stats["pkg-cves"])
f.write("<tr><td>Total number of CVEs affecting all packages</td><td>%s</td></tr>\n" %
stats["total-cves"])
f.write("<tr><td>Packages with CPE ID</td><td>%s</td></tr>\n" %
stats["cpe-id"])
f.write("<tr><td>Packages without CPE ID</td><td>%s</td></tr>\n" %
stats["no-cpe-id"])
f.write("</table>\n")
@@ -984,7 +1105,9 @@ def __main__():
pkg.set_patch_count()
pkg.set_check_package_warnings()
pkg.set_current_version()
pkg.set_cpeid()
pkg.set_url()
pkg.set_ignored_cves()
pkg.set_developers(developers)
print("Checking URL status")
loop = asyncio.get_event_loop()
@@ -994,7 +1117,8 @@ def __main__():
loop.run_until_complete(check_package_latest_version(packages))
if args.nvd_path:
print("Checking packages CVEs")
check_package_cves(args.nvd_path, {p.name: p for p in packages})
check_package_cves(args.nvd_path, packages)
check_package_cpes(args.nvd_path, packages)
print("Calculate stats")
stats = calculate_stats(packages)
if args.html: