more parallel ci

tools/build_family.py

@@ -3,7 +3,7 @@ import glob
 import sys
 import subprocess
 import time
-from multiprocessing import Process
+from multiprocessing import Pool
 
 import build_utils
 
@@ -11,13 +11,6 @@ SUCCEEDED = "\033[32msucceeded\033[0m"
 FAILED = "\033[31mfailed\033[0m"
 SKIPPED = "\033[33mskipped\033[0m"
 
-success_count = 0
-fail_count = 0
-skip_count = 0
-exit_status = 0
-
-total_time = time.monotonic()
-
 build_format = '| {:29} | {:30} | {:18} | {:7} | {:6} | {:6} |'
 build_separator = '-' * 106
 
@@ -54,46 +47,45 @@ def build_family(example, family):
     filter_with_input(all_boards)
     all_boards.sort()
 
-    plist = []
+    with Pool(processes=len(all_boards)) as pool:
-    for board in all_boards:
+        pool_args = list((map(lambda b, e=example: [e, b], all_boards)))
-        p = Process(target=build_board, args=(example, board))
+        result = pool.starmap(build_board, pool_args)
-        plist.append(p)
+        return list(map(sum, list(zip(*result))))
-        p.start()
-
-    for p in plist:
-        p.join()
     
 def build_board(example, board):
-    global success_count, fail_count, skip_count, exit_status
     start_time = time.monotonic()
     flash_size = "-"
     sram_size = "-"
 
+    # succeeded, failed, skipped
+    ret = [0, 0, 0]
+
     # Check if board is skipped
     if build_utils.skip_example(example, board):
-        success = SKIPPED
+        status = SKIPPED
-        skip_count += 1
+        ret[2] = 1
-        print(build_format.format(example, board, success, '-', flash_size, sram_size))
+        print(build_format.format(example, board, status, '-', flash_size, sram_size))
     else:   
         #subprocess.run("make -C examples/{} BOARD={} clean".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
         build_result = subprocess.run("make -j -C examples/{} BOARD={} all".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 
         if build_result.returncode == 0:
-            success = SUCCEEDED
+            status = SUCCEEDED
-            success_count += 1
+            ret[0] = 1
             (flash_size, sram_size) = build_size(example, board)
             subprocess.run("make -j -C examples/{} BOARD={} copy-artifact".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
         else:
-            exit_status = build_result.returncode
+            status = FAILED
-            success = FAILED
+            ret[1] = 1
-            fail_count += 1
 
         build_duration = time.monotonic() - start_time
-        print(build_format.format(example, board, success, "{:.2f}s".format(build_duration), flash_size, sram_size))
+        print(build_format.format(example, board, status, "{:.2f}s".format(build_duration), flash_size, sram_size))
 
         if build_result.returncode != 0:
             print(build_result.stdout.decode("utf-8"))
 
+    return ret
+
 def build_size(example, board):
     elf_file = 'examples/{}/_build/{}/*.elf'.format(example, board)
     size_output = subprocess.run('size {}'.format(elf_file), shell=True, stdout=subprocess.PIPE).stdout.decode("utf-8")
@@ -103,17 +95,22 @@ def build_size(example, board):
     return (flash_size, sram_size)
 
 if __name__ == '__main__':
+    # succeeded, failed, skipped
+    total_result = [0, 0, 0]
+
     print(build_separator)
     print(build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM'))
+    total_time = time.monotonic()
 
     for example in all_examples:
         print(build_separator)
         for family in all_families:
-            build_family(example, family)
+            fret = build_family(example, family)
+            total_result = list(map(lambda x, y: x+y, total_result, fret))
 
     total_time = time.monotonic() - total_time
     print(build_separator)
-    print("Build Summary: {} {}, {} {}, {} {} and took {:.2f}s".format(success_count, SUCCEEDED, fail_count, FAILED, skip_count, SKIPPED, total_time))
+    print("Build Summary: {} {}, {} {}, {} {} and took {:.2f}s".format(total_result[0], SUCCEEDED, total_result[1], FAILED, total_result[2], SKIPPED, total_time))
     print(build_separator)
 
-    sys.exit(exit_status)
+    sys.exit(total_result[1])