OUTPUT_FILE = 'boards.cfg'
CONFIG_DIR = 'configs'
SLEEP_TIME = 0.03
-COMMENT_BLOCK = '''#
+COMMENT_BLOCK = f'''#
# List of boards
-# Automatically generated by %s: don't edit
+# Automatically generated by {__file__}: don't edit
#
# Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers
-''' % __file__
+'''
-def try_remove(f):
- """Remove a file ignoring 'No such file or directory' error."""
+def try_remove(fname):
+ """Remove a file ignoring 'No such file or directory' error.
+
+ Args:
+ fname (str): Filename to remove
+
+ Raises:
+ OSError: output file exists but could not be removed
+ """
try:
- os.remove(f)
+ os.remove(fname)
except OSError as exception:
# Ignore 'No such file or directory' error
if exception.errno != errno.ENOENT:
def output_is_new(output):
"""Check if the output file is up to date.
+ Looks at defconfig and Kconfig files to make sure none is newer than the
+ output file. Also ensures that the boards.cfg does not mention any removed
+ boards.
+
+ Args:
+ output (str): Filename to check
+
Returns:
- True if the given output file exists and is newer than any of
- *_defconfig, MAINTAINERS and Kconfig*. False otherwise.
+ True if the given output file exists and is newer than any of
+ *_defconfig, MAINTAINERS and Kconfig*. False otherwise.
+
+ Raises:
+ OSError: output file exists but could not be opened
"""
+ # pylint: disable=too-many-branches
try:
ctime = os.path.getctime(output)
except OSError as exception:
if exception.errno == errno.ENOENT:
# return False on 'No such file or directory' error
return False
- else:
- raise
+ raise
- for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
+ for (dirpath, _, filenames) in os.walk(CONFIG_DIR):
for filename in fnmatch.filter(filenames, '*_defconfig'):
if fnmatch.fnmatch(filename, '.*'):
continue
if ctime < os.path.getctime(filepath):
return False
- for (dirpath, dirnames, filenames) in os.walk('.'):
+ for (dirpath, _, filenames) in os.walk('.'):
for filename in filenames:
if (fnmatch.fnmatch(filename, '*~') or
not fnmatch.fnmatch(filename, 'Kconfig*') and
# Detect a board that has been removed since the current board database
# was generated
- with open(output, encoding="utf-8") as f:
- for line in f:
+ with open(output, encoding="utf-8") as inf:
+ for line in inf:
if line[0] == '#' or line == '\n':
continue
defconfig = line.split()[6] + '_defconfig'
"""Set up a new Expr object.
Args:
- expr: String cotaining regular expression to store
+ expr (str): String cotaining regular expression to store
"""
self._expr = expr
self._re = re.compile(expr)
"""Check if any of the properties match the regular expression.
Args:
- props: List of properties to check
+ props (list of str): List of properties to check
Returns:
True if any of the properties match the regular expression
"""
"""Add an Expr object to the list to check.
Args:
- expr: New Expr object to add to the list of those that must
+ expr (Expr): New Expr object to add to the list of those that must
match for a board to be built.
"""
self._expr_list.append(Expr(expr))
Each of the expressions in the term is checked. All must match.
Args:
- props: List of properties to check
+ props (list of str): List of properties to check
Returns:
True if all of the expressions in the Term match, else False
"""
os.environ['srctree'] = os.getcwd()
os.environ['UBOOTVERSION'] = 'dummy'
os.environ['KCONFIG_OBJDIR'] = ''
+ self._tmpfile = None
self._conf = kconfiglib.Kconfig(warn=False)
def __del__(self):
the temporary file might be left over. In that case, it should be
deleted in this destructor.
"""
- if hasattr(self, '_tmpfile') and self._tmpfile:
+ if self._tmpfile:
try_remove(self._tmpfile)
def scan(self, defconfig):
"""Load a defconfig file to obtain board parameters.
- Arguments:
- defconfig: path to the defconfig file to be processed
+ Args:
+ defconfig (str): path to the defconfig file to be processed
Returns:
- A dictionary of board parameters. It has a form of:
- {
- 'arch': <arch_name>,
- 'cpu': <cpu_name>,
- 'soc': <soc_name>,
- 'vendor': <vendor_name>,
- 'board': <board_name>,
- 'target': <target_name>,
- 'config': <config_header_name>,
- 'options': <extra_options>
- }
+ Dictionary of board parameters. It has a form:
+ {
+ 'cpu': <cpu_name>,
+ 'soc': <soc_name>,
+ 'vendor': <vendor_name>,
+ 'board': <board_name>,
+ 'target': <target_name>,
+ 'config': <config_header_name>,
+ 'options': <extra_options>
+ }
"""
# strip special prefixes and save it in a temporary file
- fd, self._tmpfile = tempfile.mkstemp()
- with os.fdopen(fd, 'w') as f:
- for line in open(defconfig):
- colon = line.find(':CONFIG_')
- if colon == -1:
- f.write(line)
- else:
- f.write(line[colon + 1:])
+ outfd, self._tmpfile = tempfile.mkstemp()
+ with os.fdopen(outfd, 'w') as outf:
+ with open(defconfig, encoding='utf-8') as inf:
+ for line in inf:
+ colon = line.find(':CONFIG_')
+ if colon == -1:
+ outf.write(line)
+ else:
+ outf.write(line[colon + 1:])
self._conf.load_config(self._tmpfile)
try_remove(self._tmpfile)
defconfig = os.path.basename(defconfig)
params['target'], match, rear = defconfig.partition('_defconfig')
- assert match and not rear, '%s : invalid defconfig' % defconfig
+ assert match and not rear, f'{defconfig} : invalid defconfig'
# fix-up for aarch64
if params['arch'] == 'arm' and params['cpu'] == 'armv8':
class MaintainersDatabase:
- """The database of board status and maintainers."""
+ """The database of board status and maintainers.
+
+ Properties:
+ database: dict:
+ key: Board-target name (e.g. 'snow')
+ value: tuple:
+ str: Board status (e.g. 'Active')
+ str: List of maintainers, separated by :
+ """
def __init__(self):
"""Create an empty database."""
Display a warning message and return '-' if status information
is not found.
+ Args:
+ target (str): Build-target name
+
Returns:
- 'Active', 'Orphan' or '-'.
+ str: 'Active', 'Orphan' or '-'.
"""
if not target in self.database:
- print("WARNING: no status info for '%s'" % target, file=sys.stderr)
+ print(f"WARNING: no status info for '{target}'", file=sys.stderr)
return '-'
tmp = self.database[target][0]
if tmp.startswith('Maintained'):
return 'Active'
- elif tmp.startswith('Supported'):
+ if tmp.startswith('Supported'):
return 'Active'
- elif tmp.startswith('Orphan'):
+ if tmp.startswith('Orphan'):
return 'Orphan'
- else:
- print(("WARNING: %s: unknown status for '%s'" %
- (tmp, target)), file=sys.stderr)
- return '-'
+ print(f"WARNING: {tmp}: unknown status for '{target}'", file=sys.stderr)
+ return '-'
def get_maintainers(self, target):
"""Return the maintainers of the given board.
+ Args:
+ target (str): Build-target name
+
Returns:
- Maintainers of the board. If the board has two or more maintainers,
- they are separated with colons.
+ str: Maintainers of the board. If the board has two or more
+ maintainers, they are separated with colons.
"""
if not target in self.database:
- print("WARNING: no maintainers for '%s'" % target, file=sys.stderr)
+ print(f"WARNING: no maintainers for '{target}'", file=sys.stderr)
return ''
return ':'.join(self.database[target][1])
- def parse_file(self, file):
+ def parse_file(self, fname):
"""Parse a MAINTAINERS file.
- Parse a MAINTAINERS file and accumulates board status and
- maintainers information.
+ Parse a MAINTAINERS file and accumulate board status and maintainers
+ information in the self.database dict.
- Arguments:
- file: MAINTAINERS file to be parsed
+ Args:
+ fname (str): MAINTAINERS file to be parsed
"""
targets = []
maintainers = []
status = '-'
- for line in open(file, encoding="utf-8"):
- # Check also commented maintainers
- if line[:3] == '#M:':
- line = line[1:]
- tag, rest = line[:2], line[2:].strip()
- if tag == 'M:':
- maintainers.append(rest)
- elif tag == 'F:':
- # expand wildcard and filter by 'configs/*_defconfig'
- for f in glob.glob(rest):
- front, match, rear = f.partition('configs/')
- if not front and match:
- front, match, rear = rear.rpartition('_defconfig')
- if match and not rear:
- targets.append(front)
- elif tag == 'S:':
- status = rest
- elif line == '\n':
- for target in targets:
- self.database[target] = (status, maintainers)
- targets = []
- maintainers = []
- status = '-'
+ with open(fname, encoding="utf-8") as inf:
+ for line in inf:
+ # Check also commented maintainers
+ if line[:3] == '#M:':
+ line = line[1:]
+ tag, rest = line[:2], line[2:].strip()
+ if tag == 'M:':
+ maintainers.append(rest)
+ elif tag == 'F:':
+ # expand wildcard and filter by 'configs/*_defconfig'
+ for item in glob.glob(rest):
+ front, match, rear = item.partition('configs/')
+ if not front and match:
+ front, match, rear = rear.rpartition('_defconfig')
+ if match and not rear:
+ targets.append(front)
+ elif tag == 'S:':
+ status = rest
+ elif line == '\n':
+ for target in targets:
+ self.database[target] = (status, maintainers)
+ targets = []
+ maintainers = []
+ status = '-'
if targets:
for target in targets:
self.database[target] = (status, maintainers)
The board's target member must not already exist in the board list.
Args:
- brd: board to add
+ brd (Board): board to add
"""
self._boards.append(brd)
Create a Board object for each and add it to our _boards list.
Args:
- fname: Filename of boards.cfg file
+ fname (str): Filename of boards.cfg file
"""
with open(fname, 'r', encoding='utf-8') as inf:
for line in inf:
a board to be selected.
Args:
- args: List of command line arguments
+ args (list of str): List of command line arguments
+
Returns:
- A list of Term objects
+ list of Term: A list of Term objects
"""
syms = []
for arg in args:
If brds and args are both empty, all boards are selected.
Args:
- args: List of strings specifying boards to include, either named,
- or by their target, architecture, cpu, vendor or soc. If
- empty, all boards are selected.
- exclude: List of boards to exclude, regardless of 'args'
- brds: List of boards to build
+ args (list of str): List of strings specifying boards to include,
+ either named, or by their target, architecture, cpu, vendor or
+ soc. If empty, all boards are selected.
+ exclude (list of str): List of boards to exclude, regardless of
+ 'args', or None for none
+ brds (list of Board): List of boards to build, or None/[] for all
Returns:
Tuple
due to each argument, arranged by argument.
List of errors found
"""
- result = OrderedDict()
- warnings = []
- terms = self._build_terms(args)
+ def _check_board(brd):
+ """Check whether to include or exclude a board
- result['all'] = []
- for term in terms:
- result[str(term)] = []
+ Checks the various terms and decide whether to build it or not (the
+ 'build_it' variable).
- exclude_list = []
- if exclude:
- for expr in exclude:
- exclude_list.append(Expr(expr))
+ If it is built, add the board to the result[term] list so we know
+ which term caused it to be built. Add it to result['all'] also.
- found = []
- for brd in self._boards:
+ Keep a list of boards we found in 'found', so we can report boards
+ which appear in self._boards but not in brds.
+
+ Args:
+ brd (Board): Board to check
+ """
matching_term = None
build_it = False
if terms:
result[matching_term].append(brd.target)
result['all'].append(brd.target)
+ result = OrderedDict()
+ warnings = []
+ terms = self._build_terms(args)
+
+ result['all'] = []
+ for term in terms:
+ result[str(term)] = []
+
+ exclude_list = []
+ if exclude:
+ for expr in exclude:
+ exclude_list.append(Expr(expr))
+
+ found = []
+ for brd in self._boards:
+ _check_board(brd)
+
if brds:
remaining = set(brds) - set(found)
if remaining:
return result, warnings
- def scan_defconfigs_for_multiprocess(self, queue, defconfigs):
+ @classmethod
+ def scan_defconfigs_for_multiprocess(cls, queue, defconfigs):
"""Scan defconfig files and queue their board parameters
- This function is intended to be passed to
- multiprocessing.Process() constructor.
+ This function is intended to be passed to multiprocessing.Process()
+ constructor.
- Arguments:
- queue: An instance of multiprocessing.Queue().
- The resulting board parameters are written into it.
- defconfigs: A sequence of defconfig files to be scanned.
+ Args:
+ queue (multiprocessing.Queue): The resulting board parameters are
+ written into this.
+ defconfigs (sequence of str): A sequence of defconfig files to be
+ scanned.
"""
kconf_scanner = KconfigScanner()
for defconfig in defconfigs:
queue.put(kconf_scanner.scan(defconfig))
- def read_queues(self, queues, params_list):
+ @classmethod
+ def read_queues(cls, queues, params_list):
"""Read the queues and append the data to the paramers list"""
- for q in queues:
- while not q.empty():
- params_list.append(q.get())
+ for que in queues:
+ while not que.empty():
+ params_list.append(que.get())
def scan_defconfigs(self, jobs=1):
"""Collect board parameters for all defconfig files.
This function invokes multiple processes for faster processing.
- Arguments:
- jobs: The number of jobs to run simultaneously
+ Args:
+ jobs (int): The number of jobs to run simultaneously
"""
all_defconfigs = []
- for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
+ for (dirpath, _, filenames) in os.walk(CONFIG_DIR):
for filename in fnmatch.filter(filenames, '*_defconfig'):
if fnmatch.fnmatch(filename, '.*'):
continue
for i in range(jobs):
defconfigs = all_defconfigs[total_boards * i // jobs :
total_boards * (i + 1) // jobs]
- q = multiprocessing.Queue(maxsize=-1)
- p = multiprocessing.Process(
+ que = multiprocessing.Queue(maxsize=-1)
+ proc = multiprocessing.Process(
target=self.scan_defconfigs_for_multiprocess,
- args=(q, defconfigs))
- p.start()
- processes.append(p)
- queues.append(q)
+ args=(que, defconfigs))
+ proc.start()
+ processes.append(proc)
+ queues.append(que)
# The resulting data should be accumulated to this list
params_list = []
# Data in the queues should be retrieved preriodically.
# Otherwise, the queues would become full and subprocesses would get stuck.
- while any([p.is_alive() for p in processes]):
+ while any(p.is_alive() for p in processes):
self.read_queues(queues, params_list)
# sleep for a while until the queues are filled
time.sleep(SLEEP_TIME)
# Joining subprocesses just in case
# (All subprocesses should already have been finished)
- for p in processes:
- p.join()
+ for proc in processes:
+ proc.join()
# retrieve leftover data
self.read_queues(queues, params_list)
return params_list
- def insert_maintainers_info(self, params_list):
+ @classmethod
+ def insert_maintainers_info(cls, params_list):
"""Add Status and Maintainers information to the board parameters list.
- Arguments:
- params_list: A list of the board parameters
+ Args:
+ params_list (list of dict): A list of the board parameters
"""
database = MaintainersDatabase()
- for (dirpath, dirnames, filenames) in os.walk('.'):
+ for (dirpath, _, filenames) in os.walk('.'):
if 'MAINTAINERS' in filenames:
database.parse_file(os.path.join(dirpath, 'MAINTAINERS'))
params['maintainers'] = database.get_maintainers(target)
params_list[i] = params
- def format_and_output(self, params_list, output):
+ @classmethod
+ def format_and_output(cls, params_list, output):
"""Write board parameters into a file.
Columnate the board parameters, sort lines alphabetically,
and then write them to a file.
- Arguments:
- params_list: The list of board parameters
- output: The path to the output file
+ Args:
+ params_list (list of dict): The list of board parameters
+ output (str): The path to the output file
"""
- FIELDS = ('status', 'arch', 'cpu', 'soc', 'vendor', 'board', 'target',
- 'options', 'maintainers')
+ fields = ('status', 'arch', 'cpu', 'soc', 'vendor', 'board', 'target',
+ 'options', 'maintainers')
# First, decide the width of each column
- max_length = dict([ (f, 0) for f in FIELDS])
+ max_length = {f: 0 for f in fields}
for params in params_list:
- for f in FIELDS:
- max_length[f] = max(max_length[f], len(params[f]))
+ for field in fields:
+ max_length[field] = max(max_length[field], len(params[field]))
output_lines = []
for params in params_list:
line = ''
- for f in FIELDS:
+ for field in fields:
# insert two spaces between fields like column -t would
- line += ' ' + params[f].ljust(max_length[f])
+ line += ' ' + params[field].ljust(max_length[field])
output_lines.append(line.strip())
# ignore case when sorting
output_lines.sort(key=str.lower)
- with open(output, 'w', encoding="utf-8") as f:
- f.write(COMMENT_BLOCK + '\n'.join(output_lines) + '\n')
+ with open(output, 'w', encoding="utf-8") as outf:
+ outf.write(COMMENT_BLOCK + '\n'.join(output_lines) + '\n')
def ensure_board_list(self, output, jobs=1, force=False, quiet=False):
"""Generate a board database file if needed.
- Arguments:
- output: The name of the output file
- jobs: The number of jobs to run simultaneously
- force: Force to generate the output even if it is new
- quiet: True to avoid printing a message if nothing needs doing
+ Args:
+ output (str): The name of the output file
+ jobs (int): The number of jobs to run simultaneously
+ force (bool): Force to generate the output even if it is new
+ quiet (bool): True to avoid printing a message if nothing needs doing
"""
if not force and output_is_new(output):
if not quiet:
- print("%s is up to date. Nothing to do." % output)
+ print(f'{output} is up to date. Nothing to do.')
return
params_list = self.scan_defconfigs(jobs)
self.insert_maintainers_info(params_list)
projects / u-boot.git / commitdiff
