d-two/operating-system
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update Buildroot to 2019.02.3 (#415)

Browse Source 
* Update Buildroot to 2019-02.3

* Fix enter script

* Update ova_defconfig

* Fix network manager

* Remove runc patches

* Use same docker version

* Fix build

* Fix vmtools

* Fix depens

* Fix handling with tempfiles

* Fix permission handling

* Fix cp

* Cleanup

* Fix mounts
This commit is contained in:
Pascal Vizeli
2019-06-27 11:58:50 +02:00
committed by GitHub
parent bb201fb842
commit 41d3f59002
2416 changed files with 36288 additions and 21885 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
buildroot/utils/check-package
View File

@@ -6,6 +6,7 @@ import argparse
import inspect
import os
import re
import six
import sys
import checkpackagelib.lib_config
@@ -127,10 +128,19 @@ def check_file_using_lib(fname):
for cf in objects:
nwarnings += print_warnings(cf.before())
for lineno, text in enumerate(open(fname, "r").readlines()):
if six.PY3:
f = open(fname, "r", errors="surrogateescape")
else:
f = open(fname, "r")
lastline = ""
for lineno, text in enumerate(f.readlines()):
nlines += 1
for cf in objects:
if cf.disable.search(lastline):
continue
nwarnings += print_warnings(cf.check_line(lineno + 1, text))
lastline = text
f.close()
for cf in objects:
nwarnings += print_warnings(cf.after())

2
buildroot/utils/checkpackagelib/base.py
View File

@@ -1,10 +1,12 @@
# See utils/checkpackagelib/readme.txt before editing this file.
import re
class _CheckFunction(object):
def __init__(self, filename, url_to_manual):
self.filename = filename
self.url_to_manual = url_to_manual
self.disable = re.compile(r"^\s*# check-package .*\b{}\b".format(self.__class__.__name__))
def before(self):
pass

2
buildroot/utils/checkpackagelib/lib.py
View File

@@ -1,6 +1,6 @@
# See utils/checkpackagelib/readme.txt before editing this file.
from base import _CheckFunction
from checkpackagelib.base import _CheckFunction
class ConsecutiveEmptyLines(_CheckFunction):

10
buildroot/utils/checkpackagelib/lib_config.py
View File

@@ -5,11 +5,11 @@
import re
from base import _CheckFunction
from lib import ConsecutiveEmptyLines # noqa: F401
from lib import EmptyLastLine # noqa: F401
from lib import NewlineAtEof # noqa: F401
from lib import TrailingSpace # noqa: F401
from checkpackagelib.base import _CheckFunction
from checkpackagelib.lib import ConsecutiveEmptyLines # noqa: F401
from checkpackagelib.lib import EmptyLastLine # noqa: F401
from checkpackagelib.lib import NewlineAtEof # noqa: F401
from checkpackagelib.lib import TrailingSpace # noqa: F401
def _empty_or_comment(text):

10
buildroot/utils/checkpackagelib/lib_hash.py
View File

@@ -5,11 +5,11 @@
import re
from base import _CheckFunction
from lib import ConsecutiveEmptyLines # noqa: F401
from lib import EmptyLastLine # noqa: F401
from lib import NewlineAtEof # noqa: F401
from lib import TrailingSpace # noqa: F401
from checkpackagelib.base import _CheckFunction
from checkpackagelib.lib import ConsecutiveEmptyLines # noqa: F401
from checkpackagelib.lib import EmptyLastLine # noqa: F401
from checkpackagelib.lib import NewlineAtEof # noqa: F401
from checkpackagelib.lib import TrailingSpace # noqa: F401
def _empty_line_or_comment(text):

80
buildroot/utils/checkpackagelib/lib_mk.py
View File

@@ -6,16 +6,20 @@
import re
from base import _CheckFunction
from lib import ConsecutiveEmptyLines # noqa: F401
from lib import EmptyLastLine # noqa: F401
from lib import NewlineAtEof # noqa: F401
from lib import TrailingSpace # noqa: F401
from checkpackagelib.base import _CheckFunction
from checkpackagelib.lib import ConsecutiveEmptyLines # noqa: F401
from checkpackagelib.lib import EmptyLastLine # noqa: F401
from checkpackagelib.lib import NewlineAtEof # noqa: F401
from checkpackagelib.lib import TrailingSpace # noqa: F401
# used in more than one check
start_conditional = ["ifdef", "ifeq", "ifndef", "ifneq"]
end_conditional = ["endif"]
class Indent(_CheckFunction):
COMMENT = re.compile("^\s*#")
CONDITIONAL = re.compile("^\s*(ifeq|ifneq|endif)\s")
CONDITIONAL = re.compile("^\s*({})\s".format("|".join(start_conditional + end_conditional)))
ENDS_WITH_BACKSLASH = re.compile(r"^[^#].*\\$")
END_DEFINE = re.compile("^\s*endef\s")
MAKEFILE_TARGET = re.compile("^[^# \t]+:\s")
@@ -69,6 +73,66 @@ class Indent(_CheckFunction):
text]
class OverriddenVariable(_CheckFunction):
CONCATENATING = re.compile("^([A-Z0-9_]+)\s*(\+|:|)=\s*\$\(\\1\)")
END_CONDITIONAL = re.compile("^\s*({})".format("|".join(end_conditional)))
OVERRIDING_ASSIGNMENTS = [':=', "="]
START_CONDITIONAL = re.compile("^\s*({})".format("|".join(start_conditional)))
VARIABLE = re.compile("^([A-Z0-9_]+)\s*((\+|:|)=)")
USUALLY_OVERRIDDEN = re.compile("^[A-Z0-9_]+({})".format("|".join([
"_ARCH\s*=\s*",
"_CPU\s*=\s*",
"_SITE\s*=\s*",
"_SOURCE\s*=\s*",
"_VERSION\s*=\s*"])))
def before(self):
self.conditional = 0
self.unconditionally_set = []
self.conditionally_set = []
def check_line(self, lineno, text):
if self.START_CONDITIONAL.search(text):
self.conditional += 1
return
if self.END_CONDITIONAL.search(text):
self.conditional -= 1
return
m = self.VARIABLE.search(text)
if m is None:
return
variable, assignment = m.group(1, 2)
if self.conditional == 0:
if variable in self.conditionally_set:
self.unconditionally_set.append(variable)
if assignment in self.OVERRIDING_ASSIGNMENTS:
return ["{}:{}: unconditional override of variable {} previously conditionally set"
.format(self.filename, lineno, variable),
text]
if variable not in self.unconditionally_set:
self.unconditionally_set.append(variable)
return
if assignment in self.OVERRIDING_ASSIGNMENTS:
return ["{}:{}: unconditional override of variable {}"
.format(self.filename, lineno, variable),
text]
else:
if variable not in self.unconditionally_set:
self.conditionally_set.append(variable)
return
if self.CONCATENATING.search(text):
return
if self.USUALLY_OVERRIDDEN.search(text):
return
if assignment in self.OVERRIDING_ASSIGNMENTS:
return ["{}:{}: conditional override of variable {}"
.format(self.filename, lineno, variable),
text]
class PackageHeader(_CheckFunction):
def before(self):
self.skip = False
@@ -222,8 +286,8 @@ class UselessFlag(_CheckFunction):
"_INSTALL_REDISTRIBUTE\s*=\s*YES",
"_INSTALL_STAGING\s*=\s*NO",
"_INSTALL_TARGET\s*=\s*YES"])))
END_CONDITIONAL = re.compile("^\s*(endif)")
START_CONDITIONAL = re.compile("^\s*(ifeq|ifneq)")
END_CONDITIONAL = re.compile("^\s*({})".format("|".join(end_conditional)))
START_CONDITIONAL = re.compile("^\s*({})".format("|".join(start_conditional)))
def before(self):
self.conditional = 0

4
buildroot/utils/checkpackagelib/lib_patch.py
View File

@@ -5,8 +5,8 @@
import re
from base import _CheckFunction
from lib import NewlineAtEof # noqa: F401
from checkpackagelib.base import _CheckFunction
from checkpackagelib.lib import NewlineAtEof # noqa: F401
class ApplyOrder(_CheckFunction):

10
buildroot/utils/genrandconfig
View File

@@ -310,17 +310,9 @@ def fixup_config(configfile):
'BR2_PACKAGE_QT5BASE_GUI=y\n' in configlines:
return False
# The cs nios2 2017.02 toolchain is affected by binutils PR19405
if 'BR2_TOOLCHAIN_EXTERNAL_CODESOURCERY_NIOSII=y\n' in configlines and \
'BR2_PACKAGE_QT_GUI_MODULE=y\n' in configlines:
return False
# The cs nios2 2017.02 toolchain is affected by binutils PR19405
if 'BR2_TOOLCHAIN_EXTERNAL_CODESOURCERY_NIOSII=y\n' in configlines and \
'BR2_PACKAGE_FLANN=y\n' in configlines:
return False
# or1k affected by binutils PR21464
if 'BR2_or1k=y\n' in configlines and \
'BR2_PACKAGE_QT_GUI_MODULE=y\n' in configlines:
return False
with open(configfile, "w+") as configf:
configf.writelines(configlines)
@@ -366,6 +358,8 @@ def gen_config(args):
configlines.append("BR2_STATIC_LIBS=y\n")
if randint(0, 20) == 0:
configlines.append("BR2_PACKAGE_PYTHON_PY_ONLY=y\n")
if randint(0, 5) == 0:
configlines.append("BR2_OPTIMIZE_2=y\n")
# Write out the configuration file
if not os.path.exists(args.outputdir):

15
buildroot/utils/get-developers
View File

@@ -24,11 +24,6 @@ def parse_args():
def __main__():
# DEVELOPERS is one level up from here
devs_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..')
devs = getdeveloperlib.parse_developers(devs_dir)
if devs is None:
sys.exit(1)
args = parse_args()
# Check that only one action is given
@@ -50,9 +45,17 @@ def __main__():
print("No action specified")
return
# getdeveloperlib expects to be executed from the toplevel buildroot
# directory, which is one level up from this script
os.chdir(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..'))
devs = getdeveloperlib.parse_developers()
if devs is None:
sys.exit(1)
# Handle the check action
if args.check:
files = getdeveloperlib.check_developers(devs, devs_dir)
files = getdeveloperlib.check_developers(devs)
for f in files:
print(f)

13
buildroot/utils/getdeveloperlib.py
View File

@@ -1,7 +1,9 @@
from __future__ import print_function
import os
import re
import glob
import subprocess
import sys
#
# Patch parsing functions
@@ -158,8 +160,6 @@ def parse_developers(basepath=None):
linen = 0
if basepath is None:
basepath = os.getcwd()
else:
basepath = os.path.abspath(basepath)
with open(os.path.join(basepath, "DEVELOPERS"), "r") as f:
files = []
name = None
@@ -169,13 +169,15 @@ def parse_developers(basepath=None):
continue
elif line.startswith("N:"):
if name is not None or len(files) != 0:
print("Syntax error in DEVELOPERS file, line %d" % linen)
print("Syntax error in DEVELOPERS file, line %d" % linen,
file=sys.stderr)
name = line[2:].strip()
elif line.startswith("F:"):
fname = line[2:].strip()
dev_files = glob.glob(os.path.join(basepath, fname))
if len(dev_files) == 0:
print("WARNING: '%s' doesn't match any file" % fname)
print("WARNING: '%s' doesn't match any file" % fname,
file=sys.stderr)
files += dev_files
elif line == "":
if not name:
@@ -184,7 +186,8 @@ def parse_developers(basepath=None):
files = []
name = None
else:
print("Syntax error in DEVELOPERS file, line %d: '%s'" % (linen, line))
print("Syntax error in DEVELOPERS file, line %d: '%s'" % (linen, line),
file=sys.stderr)
return None
linen += 1
# handle last developer

83
buildroot/utils/scancpan
View File

@@ -487,9 +487,9 @@ use Digest::SHA qw(sha256_hex);
use Text::Wrap;
$Text::Wrap::columns = 62;
# Below, 5.026 should be aligned with the version of perl actually
# Below, 5.028 should be aligned with the version of perl actually
# bundled in Buildroot:
die <<"MSG" if $] < 5.026;
die <<"MSG" if $] < 5.028;
This script needs a host perl with the same major version as Buildroot target perl.
Your current host perl is:
@@ -497,7 +497,7 @@ Your current host perl is:
version $]
You may install a local one by running:
perlbrew install perl-5.26.0
perlbrew install perl-5.28.0
MSG
my ($help, $man, $quiet, $force, $recommend, $test, $host);
@@ -519,6 +519,7 @@ my %dist; # name -> metacpan data
my %need_target; # name -> 1 if target package is needed
my %need_host; # name -> 1 if host package is needed
my %need_dlopen; # name -> 1 if requires dynamic library
my %is_xs; # name -> 1 if XS module
my %deps_build; # name -> list of host dependencies
my %deps_runtime; # name -> list of target dependencies
my %license_files; # name -> hash of license files
@@ -575,6 +576,32 @@ sub find_license_files {
return @license_files;
}
sub want_test {
my ($distname) = @_;
return 1 if $need_dlopen{$distname} && scalar @{$deps_runtime{$distname}} > 0;
}
sub get_dependencies {
my ($distname) = @_;
my %dep = map { $_ => 1 } @{$deps_runtime{$distname}};
for my $direct (@{$deps_runtime{$distname}}) {
for (get_dependencies( $direct )) {
$dep{$_} = 1;
}
}
return keys %dep;
}
sub get_indirect_dependencies {
my ($distname) = @_;
my %indirect;
my %direct = map { $_ => 1 } @{$deps_runtime{$distname}};
for my $dep (get_dependencies( $distname )) {
$indirect{$dep} = 1 unless exists $direct{$dep};
}
return keys %indirect;
}
sub fetch {
my ($name, $need_target, $need_host, $top) = @_;
$need_target{$name} = $need_target if $need_target;
@@ -588,7 +615,7 @@ sub fetch {
my $author = $result->{author};
my $release = $name . q{-} . $result->{version};
my $manifest = $mcpan->source( author => $author, release => $release, path => 'MANIFEST' );
$need_dlopen{$name} = is_xs( $manifest );
$need_dlopen{$name} = $is_xs{$name} = is_xs( $manifest );
foreach my $fname (find_license_files( $manifest )) {
my $license = $mcpan->source( author => $author, release => $release, path => $fname );
$license_files{$name}->{$fname} = sha256_hex( $license );
@@ -688,6 +715,7 @@ while (my ($distname, $dist) = each %dist) {
my $mkname = $dirname . q{/} . $fsname . q{.mk};
my $hashname = $dirname . q{/} . $fsname . q{.hash};
my $brname = brname( $fsname );
my $testname = q{support/testing/tests/package/test_} . lc $brname . q{.py};
unless (-d $dirname) {
mkdir $dirname;
$new_pkgs = 1;
@@ -779,6 +807,47 @@ while (my ($distname, $dist) = each %dist) {
}
close $fh;
}
if (want_test( $distname ) && ($force || !-f $testname)) {
my $classname = $distname;
$classname =~ s|-||g;
my $modname = $distname;
$modname =~ s|-|::|g;
my $mark = $is_xs{$distname} ? q{ XS} : q{};
my @indirect = (get_indirect_dependencies( $distname ));
say qq{write ${testname}} unless $quiet;
open my $fh, q{>}, $testname;
say {$fh} qq{from tests.package.test_perl import TestPerlBase};
say {$fh} qq{};
say {$fh} qq{};
say {$fh} qq{class TestPerl${classname}(TestPerlBase):};
say {$fh} qq{ """};
say {$fh} qq{ package:};
say {$fh} qq{ ${distname}${mark}};
say {$fh} qq{ direct dependencies:};
foreach my $dep (sort @{$deps_runtime{$distname}}) {
$mark = $is_xs{$dep} ? q{ XS} : q{};
say {$fh} qq{ ${dep}${mark}};
}
if (scalar @indirect > 0) {
say {$fh} qq{ indirect dependencies:};
foreach my $dep (sort @indirect) {
$mark = $is_xs{$dep} ? q{ XS} : q{};
say {$fh} qq{ ${dep}${mark}};
}
}
say {$fh} qq{ """};
say {$fh} qq{};
say {$fh} qq{ config = TestPerlBase.config + \\};
say {$fh} qq{ """};
say {$fh} qq{ BR2_PACKAGE_PERL=y};
say {$fh} qq{ BR2_PACKAGE_${brname}=y};
say {$fh} qq{ """};
say {$fh} qq{};
say {$fh} qq{ def test_run(self):};
say {$fh} qq{ self.login()};
say {$fh} qq{ self.module_test("${modname}")};
close $fh;
}
}
if ($new_pkgs) {
@@ -812,7 +881,7 @@ utils/scancpan Try-Tiny Moo
=head1 SYNOPSIS
supports/scripts/scancpan [options] [distname ...]
utils/scancpan [options] [distname ...]
Options:
-help
@@ -901,9 +970,9 @@ This script is a part of Buildroot.
This script requires the module C<MetaCPAN::API::Tiny> (version 1.131730)
which was included at the beginning of this file by the tool C<fatpack>.
See L<http://search.cpan.org/~nperez/MetaCPAN-API-Tiny-1.131730/>.
See L<https://metacpan.org/release/NPEREZ/MetaCPAN-API-Tiny-1.131730>.
See L<http://search.cpan.org/search?query=App-FatPacker&mode=dist>.
See L<https://metacpan.org/release/App-FatPacker>.
These both libraries are free software and may be distributed under the same
terms as perl itself.

46
buildroot/utils/scanpypi
View File

@@ -10,9 +10,6 @@ from __future__ import print_function
from __future__ import absolute_import
import argparse
import json
import six.moves.urllib.request
import six.moves.urllib.error
import six.moves.urllib.parse
import sys
import os
import shutil
@@ -25,6 +22,9 @@ import textwrap
import tempfile
import imp
from functools import wraps
import six.moves.urllib.request
import six.moves.urllib.error
import six.moves.urllib.parse
from six.moves import map
from six.moves import zip
from six.moves import input
@@ -97,6 +97,7 @@ def pkg_buildroot_name(pkg_name):
pkg_name -- String to rename
"""
name = re.sub('[^\w-]', '', pkg_name.lower())
name = name.replace('_', '-')
prefix = 'python-'
pattern = re.compile('^(?!' + prefix + ')(.+?)$')
name = pattern.sub(r'python-\1', name)
@@ -225,6 +226,22 @@ class BuildrootPackage():
self.filename = self.used_url['filename']
self.url = self.used_url['url']
def check_archive(self, members):
"""
Check archive content before extracting
Keyword arguments:
members -- list of archive members
"""
# Protect against https://github.com/snyk/zip-slip-vulnerability
# Older python versions do not validate that the extracted files are
# inside the target directory. Detect and error out on evil paths
evil = [e for e in members if os.path.relpath(e).startswith(('/', '..'))]
if evil:
print('ERROR: Refusing to extract {} with suspicious members {}'.format(
self.filename, evil))
sys.exit(1)
def extract_package(self, tmp_path):
"""
Extract the package contents into a directrory
@@ -249,6 +266,7 @@ class BuildrootPackage():
print('Removing {pkg}...'.format(pkg=tmp_pkg))
shutil.rmtree(tmp_pkg)
os.makedirs(tmp_pkg)
self.check_archive(as_zipfile.namelist())
as_zipfile.extractall(tmp_pkg)
pkg_filename = self.filename.split(".zip")[0]
else:
@@ -264,6 +282,7 @@ class BuildrootPackage():
print('Removing {pkg}...'.format(pkg=tmp_pkg))
shutil.rmtree(tmp_pkg)
os.makedirs(tmp_pkg)
self.check_archive(as_tarfile.getnames())
as_tarfile.extractall(tmp_pkg)
pkg_filename = self.filename.split(".tar")[0]
@@ -281,6 +300,12 @@ class BuildrootPackage():
sys.path.append(self.tmp_extract)
s_file, s_path, s_desc = imp.find_module('setup', [self.tmp_extract])
setup = imp.load_module('setup', s_file, s_path, s_desc)
if self.metadata_name in self.setup_args:
pass
elif self.metadata_name.replace('_', '-') in self.setup_args:
self.metadata_name = self.metadata_name.replace('_', '-')
elif self.metadata_name.replace('-', '_') in self.setup_args:
self.metadata_name = self.metadata_name.replace('-', '_')
try:
self.setup_metadata = self.setup_args[self.metadata_name]
except KeyError:
@@ -354,13 +379,14 @@ class BuildrootPackage():
version=self.version)
lines.append(version_line)
targz = self.filename.replace(
self.version,
'$({name}_VERSION)'.format(name=self.mk_name))
targz_line = '{name}_SOURCE = {filename}\n'.format(
name=self.mk_name,
filename=targz)
lines.append(targz_line)
if self.buildroot_name != self.real_name:
targz = self.filename.replace(
self.version,
'$({name}_VERSION)'.format(name=self.mk_name))
targz_line = '{name}_SOURCE = {filename}\n'.format(
name=self.mk_name,
filename=targz)
lines.append(targz_line)
if self.filename not in self.url:
# Sometimes the filename is in the url, sometimes it's not

21
buildroot/utils/test-pkg
View File

@@ -2,12 +2,20 @@
set -e
TOOLCHAINS_CSV='support/config-fragments/autobuild/toolchain-configs.csv'
TEMP_CONF=""
do_clean() {
if [ ! -z "${TEMP_CONF}" ]; then
rm -f "${TEMP_CONF}"
fi
}
main() {
local o O opts
local cfg dir pkg random toolchains_dir toolchain all number mode
local ret nb nb_skip nb_fail nb_legal nb_tc build_dir
local -a toolchains
local pkg_br_name
o='hac:d:n:p:r:t:'
O='help,config-snippet:build-dir:package:,random:,toolchains-dir:'
@@ -50,8 +58,15 @@ main() {
;;
esac
done
trap do_clean INT TERM HUP EXIT
if [ -z "${cfg}" ]; then
printf "error: no config snippet specified\n" >&2; exit 1
pkg_br_name="${pkg//-/_}"
pkg_br_name="BR2_PACKAGE_${pkg_br_name^^}"
TEMP_CONF=$(mktemp /tmp/test-${pkg}-config.XXXXXX)
echo "${pkg_br_name}=y" > ${TEMP_CONF}
cfg="${TEMP_CONF}"
fi
if [ ! -e "${cfg}" ]; then
printf "error: %s: no such file\n" "${cfg}" >&2; exit 1
@@ -129,7 +144,7 @@ build_one() {
mkdir -p "${dir}"
support/kconfig/merge_config.sh -O "${dir}" \
CONFIG_= support/kconfig/merge_config.sh -O "${dir}" \
"${toolchainconfig}" "support/config-fragments/minimal.config" "${cfg}" \
>> "${dir}/logfile" 2>&1
# We want all the options from the snippet to be present as-is (set
@@ -152,7 +167,7 @@ build_one() {
fi
# shellcheck disable=SC2086
if ! make O="${dir}" ${pkg} >> "${dir}/logfile" 2>&1; then
if ! BR_FORCE_CHECK_DEPENDENCIES=YES make O="${dir}" ${pkg} >> "${dir}/logfile" 2>&1; then
return 2
fi