console: file should always be non-negative

We use the parameter file in console functions to choose from an array
after checking against MAX_FILES but we never check if the value of file
is negative.

Running ./u-boot -T -l and issuing the poweroff command has resulted in
crashes because os_exit() results in std::ostream::flush() calling U-Boot's
fflush with file being a pointer which when converted to int may be
represented by a negative number.

This shows that checking against MAX_FILES is not enough. We have to ensure
that the file argument is always positive.

Signed-off-by: Heinrich Schuchardt <heinrich.schuchardt@canonical.com>
Reviewed-by: Simon Glass <sjg@chromium.org>

diff --git a/common/console.c b/common/console.c
index 0c9bf66c3f6d554e45ca0b6e8146957a20ffaf41..10ab361d006928016ed12b309f6284c9462a9319 100644 (file)
--- a/common/console.c
+++ b/common/console.c
@@ -497,7 +497,7 @@ int serial_printf(const char *fmt, ...)
 
 int fgetc(int file)
 {
-       if (file < MAX_FILES) {
+       if ((unsigned int)file < MAX_FILES) {
                /*
                 * Effectively poll for input wherever it may be available.
                 */
@@ -530,7 +530,7 @@ int fgetc(int file)
 
 int ftstc(int file)
 {
-       if (file < MAX_FILES)
+       if ((unsigned int)file < MAX_FILES)
                return console_tstc(file);
 
        return -1;
@@ -538,20 +538,20 @@ int ftstc(int file)
 
 void fputc(int file, const char c)
 {
-       if (file < MAX_FILES)
+       if ((unsigned int)file < MAX_FILES)
                console_putc(file, c);
 }
 
 void fputs(int file, const char *s)
 {
-       if (file < MAX_FILES)
+       if ((unsigned int)file < MAX_FILES)
                console_puts(file, s);
 }
 
 #ifdef CONFIG_CONSOLE_FLUSH_SUPPORT
 void fflush(int file)
 {
-       if (file < MAX_FILES)
+       if ((unsigned int)file < MAX_FILES)
                console_flush(file);
 }
 #endif