buildman: Be more selective about which directories to remove

At present buildman removes any directory it doesn't intend to write
output into. This is overly expansive since if the output directory
happens to be somewhere with existing files, they may be removed. Using
an existing directory for buildman is not a good practice, but since the
result might be catastrophic, it is best to guard against it.

A previous commit[1] fixed this by refusing to write to a subdirectory
of the current directory, assumed to have U-Boot source code. But we can
do better by only removing directories that look like the ones buildman
creates.

Update the code to do this and add a test.

Signed-off-by: Simon Glass <sjg@chromium.org>
[1] 409fc029c40 tools: buildman: Don't use the working dir as build dir

diff --git a/tools/buildman/builder.py b/tools/buildman/builder.py
index a41d0b316e67bd3d94c98431900446c8b2ceda37..30ec4254f85e295034b46d083d776cb147b6b1ac 100644 (file)
--- a/tools/buildman/builder.py
+++ b/tools/buildman/builder.py
@@ -485,6 +485,7 @@ class Builder:
         if self.commits:
             commit = self.commits[commit_upto]
             subject = commit.subject.translate(trans_valid_chars)
+            # See _GetOutputSpaceRemovals() which parses this name
             commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1,
                     self.commit_count, commit.hash, subject[:20]))
         elif not self.no_subdirs:
@@ -1525,12 +1526,15 @@ class Builder:
         for thread in range(max_threads):
             self._PrepareThread(thread, setup_git)
 
-    def _PrepareOutputSpace(self):
+    def _GetOutputSpaceRemovals(self):
         """Get the output directories ready to receive files.
 
-        We delete any output directories which look like ones we need to
-        create. Having left over directories is confusing when the user wants
-        to check the output manually.
+        Figure out what needs to be deleted in the output directory before it
+        can be used. We only delete old buildman directories which have the
+        expected name pattern. See _GetOutputDir().
+
+        Returns:
+            List of full paths of directories to remove
         """
         if not self.commits:
             return
@@ -1541,7 +1545,20 @@ class Builder:
         to_remove = []
         for dirname in glob.glob(os.path.join(self.base_dir, '*')):
             if dirname not in dir_list:
-                to_remove.append(dirname)
+                leaf = dirname[len(self.base_dir) + 1:]
+                m =  re.match('[0-9]+_of_[0-9]+_g[0-9a-f]+_.*', leaf)
+                if m:
+                    to_remove.append(dirname)
+        return to_remove
+
+    def _PrepareOutputSpace(self):
+        """Get the output directories ready to receive files.
+
+        We delete any output directories which look like ones we need to
+        create. Having left over directories is confusing when the user wants
+        to check the output manually.
+        """
+        to_remove = self._GetOutputSpaceRemovals()
         if to_remove:
             Print('Removing %d old build directories' % len(to_remove),
                   newline=False)

diff --git a/tools/buildman/test.py b/tools/buildman/test.py
index acd862b3b0fcd8dbe518dafc93e2eb244ef54312..2aaedf44ac7f0bcfd8eacfacabe69904a71507de 100644 (file)
--- a/tools/buildman/test.py
+++ b/tools/buildman/test.py
@@ -22,6 +22,7 @@ import commit
 import terminal
 import test_util
 import toolchain
+import tools
 
 use_network = True
 
@@ -469,6 +470,25 @@ class TestBuild(unittest.TestCase):
         self.assertEqual('HOSTCC=clang CC=clang',
                          tc.GetEnvArgs(toolchain.VAR_MAKE_ARGS))
 
+    def testPrepareOutputSpace(self):
+        def _Touch(fname):
+            tools.WriteFile(os.path.join(base_dir, fname), b'')
+
+        base_dir = tempfile.mkdtemp()
+
+        # Add various files that we want removed and left alone
+        to_remove = ['01_of_22_g0982734987_title', '102_of_222_g92bf_title',
+                     '01_of_22_g2938abd8_title']
+        to_leave = ['something_else', '01-something.patch', '01_of_22_another']
+        for name in to_remove + to_leave:
+            _Touch(name)
+
+        build = builder.Builder(self.toolchains, base_dir, None, 1, 2)
+        build.commits = self.commits
+        build.commit_count = len(commits)
+        result = set(build._GetOutputSpaceRemovals())
+        expected = set([os.path.join(base_dir, f) for f in to_remove])
+        self.assertEqual(expected, result)
 
 if __name__ == "__main__":
     unittest.main()