diff --git a/.github/workflows/build_aarch64.yml b/.github/workflows/build_aarch64.yml index 1720ba592..b4ea8a0eb 100644 --- a/.github/workflows/build_aarch64.yml +++ b/.github/workflows/build_aarch64.yml @@ -55,6 +55,9 @@ jobs: - name: Set Toolchain Path run: echo >> $GITHUB_PATH `echo ~/cache/toolchain/*/bin` + - name: Get Dependencies + run: python3 tools/get_dependencies.py ${{ matrix.family }} + - name: Build run: python3 tools/build_family.py ${{ matrix.family }} diff --git a/.github/workflows/build_arm.yml b/.github/workflows/build_arm.yml index 27cb9f1f7..4297ba895 100644 --- a/.github/workflows/build_arm.yml +++ b/.github/workflows/build_arm.yml @@ -67,6 +67,11 @@ jobs: - name: Setup Python uses: actions/setup-python@v3 + - name: Install ARM GCC + uses: carlosperate/arm-none-eabi-gcc-action@v1 + with: + release: '11.2-2022.02' + - name: Checkout TinyUSB uses: actions/checkout@v3 @@ -84,12 +89,9 @@ jobs: run: | git clone --depth 1 -b develop https://github.com/raspberrypi/pico-sdk ~/pico-sdk echo >> $GITHUB_ENV PICO_SDK_PATH=~/pico-sdk - git submodule update --init hw/mcu/raspberry_pi/Pico-PIO-USB - - name: Install ARM GCC - uses: carlosperate/arm-none-eabi-gcc-action@v1 - with: - release: '11.2-2022.02' + - name: Get Dependencies + run: python3 tools/get_dependencies.py ${{ matrix.family }} - name: Build run: python3 tools/build_family.py ${{ matrix.family }} @@ -122,16 +124,16 @@ jobs: - name: Setup Python uses: actions/setup-python@v3 + - name: Install ARM GCC + uses: carlosperate/arm-none-eabi-gcc-action@v1 + with: + release: '11.2-2022.02' + - name: Checkout TinyUSB uses: actions/checkout@v3 - name: Checkout common submodules in lib run: git submodule update --init lib/FreeRTOS-Kernel lib/lwip - - name: Install ARM GCC - uses: carlosperate/arm-none-eabi-gcc-action@v1 - with: - release: '11.2-2022.02' - - name: Build run: python3 tools/build_board.py ${{ matrix.example }} diff --git a/.github/workflows/build_msp430.yml b/.github/workflows/build_msp430.yml index 6a468ab04..ea93f09a0 100644 --- a/.github/workflows/build_msp430.yml +++ b/.github/workflows/build_msp430.yml @@ -52,6 +52,9 @@ jobs: - name: Set Toolchain Path run: echo >> $GITHUB_PATH `echo ~/cache/toolchain/*/bin` + - name: Get Dependencies + run: python3 tools/get_dependencies.py ${{ matrix.family }} + - name: Build run: python3 tools/build_family.py ${{ matrix.family }} diff --git a/.github/workflows/build_renesas.yml b/.github/workflows/build_renesas.yml index 50618ff6b..2563d3549 100644 --- a/.github/workflows/build_renesas.yml +++ b/.github/workflows/build_renesas.yml @@ -53,6 +53,9 @@ jobs: - name: Set Toolchain Path run: echo >> $GITHUB_PATH `echo ~/cache/toolchain/*/bin` + - name: Get Dependencies + run: python3 tools/get_dependencies.py ${{ matrix.family }} + - name: Build run: python3 tools/build_family.py ${{ matrix.family }} diff --git a/.github/workflows/build_riscv.yml b/.github/workflows/build_riscv.yml index 2d670138d..90dc35206 100644 --- a/.github/workflows/build_riscv.yml +++ b/.github/workflows/build_riscv.yml @@ -53,6 +53,9 @@ jobs: - name: Set Toolchain Path run: echo >> $GITHUB_PATH `echo ~/cache/toolchain/*/bin` + - name: Get Dependencies + run: python3 tools/get_dependencies.py ${{ matrix.family }} + - name: Build run: python3 tools/build_family.py ${{ matrix.family }} diff --git a/docs/reference/getting_started.rst b/docs/reference/getting_started.rst index fa5c51eff..1088d4700 100644 --- a/docs/reference/getting_started.rst +++ b/docs/reference/getting_started.rst @@ -50,7 +50,12 @@ Some TinyUSB examples also requires external submodule libraries in ``/lib`` suc $ git submodule update --init lib -In addition, MCU driver submodule is also needed to provide low-level MCU peripheral's driver. Luckily, it will be fetched if needed when you run the ``make`` to build your board. +In addition, MCU driver submodule is also needed to provide low-level MCU peripheral's driver. To download these depencies for your board, run the ``get-dpes`` as follow. + +.. code-block:: + + $ make BOARD=feather_nrf52840_express get-deps + Some modules will also require a module-specific SDK (e.g. RP2040) or binary (e.g. Sony Spresense) to build examples. diff --git a/examples/make.mk b/examples/make.mk index 27e1db38c..47520d660 100644 --- a/examples/make.mk +++ b/examples/make.mk @@ -45,11 +45,6 @@ else SRC_C += $(subst $(TOP)/,,$(wildcard $(TOP)/$(FAMILY_PATH)/*.c)) endif -# Fetch submodules depended by family -fetch_submodule_if_empty = $(if $(wildcard $(TOP)/$1/*),,$(info $(shell git -C $(TOP) submodule update --init $1))) -ifdef DEPS_SUBMODULES - $(foreach s,$(DEPS_SUBMODULES),$(call fetch_submodule_if_empty,$(s))) -endif #-------------- Cross Compiler ------------ # Can be set by board, default to ARM GCC diff --git a/examples/rules.mk b/examples/rules.mk index 538dffbcf..c3134056a 100644 --- a/examples/rules.mk +++ b/examples/rules.mk @@ -5,6 +5,7 @@ # Set all as default goal .DEFAULT_GOAL := all +# ---------------- GNU Make Start ----------------------- # ESP32-Sx and RP2040 has its own CMake build system ifeq (,$(findstring $(FAMILY),esp32s2 esp32s3 rp2040)) @@ -141,7 +142,23 @@ $(BUILD)/obj/%_asm.o: %.S @echo AS $(notdir $@) @$(CC) -x assembler-with-cpp $(ASFLAGS) -c -o $@ $< -endif # GNU Make +endif + +.PHONY: clean +clean: +ifeq ($(CMDEXE),1) + rd /S /Q $(subst /,\,$(BUILD)) +else + $(RM) -rf $(BUILD) +endif +# ---------------- GNU Make End ----------------------- + +# get depenecies +.PHONY: get-deps +get-deps: + ifdef DEPS_SUBMODULES + git -C $(TOP) submodule update --init $(DEPS_SUBMODULES) + endif size: $(BUILD)/$(PROJECT).elf -@echo '' @@ -152,14 +169,6 @@ size: $(BUILD)/$(PROJECT).elf linkermap: $(BUILD)/$(PROJECT).elf @linkermap -v $<.map -.PHONY: clean -clean: -ifeq ($(CMDEXE),1) - rd /S /Q $(subst /,\,$(BUILD)) -else - $(RM) -rf $(BUILD) -endif - # --------------------------------------- # Flash Targets # --------------------------------------- diff --git a/hw/bsp/rp2040/family.mk b/hw/bsp/rp2040/family.mk index 5db784b14..cf6b53793 100644 --- a/hw/bsp/rp2040/family.mk +++ b/hw/bsp/rp2040/family.mk @@ -1,6 +1,8 @@ JLINK_DEVICE = rp2040_m0_0 PYOCD_TARGET = rp2040 +DEPS_SUBMODULES += hw/mcu/raspberry_pi/Pico-PIO-USB + ifeq ($(DEBUG), 1) CMAKE_DEFSYM += -DCMAKE_BUILD_TYPE=Debug endif diff --git a/tools/build_board.py b/tools/build_board.py index 4d895329a..8d10ef820 100644 --- a/tools/build_board.py +++ b/tools/build_board.py @@ -1,8 +1,8 @@ import os -import glob import sys -import subprocess import time +import subprocess +from multiprocessing import Pool import build_utils @@ -10,90 +10,60 @@ SUCCEEDED = "\033[32msucceeded\033[0m" FAILED = "\033[31mfailed\033[0m" SKIPPED = "\033[33mskipped\033[0m" -success_count = 0 -fail_count = 0 -skip_count = 0 -exit_status = 0 - -total_time = time.monotonic() - -build_format = '| {:29} | {:30} | {:18} | {:7} | {:6} | {:6} |' build_separator = '-' * 106 + def filter_with_input(mylist): if len(sys.argv) > 1: input_args = list(set(mylist).intersection(sys.argv)) if len(input_args) > 0: mylist[:] = input_args -# If examples are not specified in arguments, build all -all_examples = [] -for dir1 in os.scandir("examples"): - if dir1.is_dir(): - for entry in os.scandir(dir1.path): - if entry.is_dir(): - all_examples.append(dir1.name + '/' + entry.name) -filter_with_input(all_examples) -all_examples.sort() -# If boards are not specified in arguments, build all -all_boards = [] -for entry in os.scandir("hw/bsp"): - if entry.is_dir() and os.path.exists(entry.path + "/board.mk"): - all_boards.append(entry.name) -filter_with_input(all_boards) -all_boards.sort() +if __name__ == '__main__': + # If examples are not specified in arguments, build all + all_examples = [] + for dir1 in os.scandir("examples"): + if dir1.is_dir(): + for entry in os.scandir(dir1.path): + if entry.is_dir(): + all_examples.append(dir1.name + '/' + entry.name) + filter_with_input(all_examples) + all_examples.sort() -def build_board(example, board): - global success_count, fail_count, skip_count, exit_status - start_time = time.monotonic() - flash_size = "-" - sram_size = "-" + # If boards are not specified in arguments, build all + all_boards = [] + for entry in os.scandir("hw/bsp"): + if entry.is_dir() and os.path.exists(entry.path + "/board.mk"): + all_boards.append(entry.name) + filter_with_input(all_boards) + all_boards.sort() - # Check if board is skipped - if build_utils.skip_example(example, board): - success = SKIPPED - skip_count += 1 - print(build_format.format(example, board, success, '-', flash_size, sram_size)) - else: - subprocess.run("make -C examples/{} BOARD={} clean".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - build_result = subprocess.run("make -j -C examples/{} BOARD={} all".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + # Get dependencies + for b in all_boards: + subprocess.run("make -C examples/device/board_test BOARD={} get-deps".format(b), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - if build_result.returncode == 0: - success = SUCCEEDED - success_count += 1 - (flash_size, sram_size) = build_size(example, board) - else: - exit_status = build_result.returncode - success = FAILED - fail_count += 1 - - build_duration = time.monotonic() - start_time - print(build_format.format(example, board, success, "{:.2f}s".format(build_duration), flash_size, sram_size)) - - if build_result.returncode != 0: - print(build_result.stdout.decode("utf-8")) - -def build_size(example, board): - #elf_file = 'examples/device/{}/_build/{}/{}-firmware.elf'.format(example, board, board) - elf_file = 'examples/{}/_build/{}/*.elf'.format(example, board) - size_output = subprocess.run('size {}'.format(elf_file), shell=True, stdout=subprocess.PIPE).stdout.decode("utf-8") - size_list = size_output.split('\n')[1].split('\t') - flash_size = int(size_list[0]) - sram_size = int(size_list[1]) + int(size_list[2]) - return (flash_size, sram_size) - -print(build_separator) -print(build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM')) - -for example in all_examples: print(build_separator) - for board in all_boards: - build_board(example, board) + print(build_utils.build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM')) + total_time = time.monotonic() -total_time = time.monotonic() - total_time -print(build_separator) -print("Build Summary: {} {}, {} {}, {} {} and took {:.2f}s".format(success_count, SUCCEEDED, fail_count, FAILED, skip_count, SKIPPED, total_time)) -print(build_separator) + # succeeded, failed, skipped + total_result = [0, 0, 0] + for example in all_examples: + print(build_separator) + with Pool(processes=os.cpu_count()) as pool: + pool_args = list((map(lambda b, e=example: [e, b], all_boards))) + result = pool.starmap(build_utils.build_example, pool_args) + # sum all element of same index (column sum) + result = list(map(sum, list(zip(*result)))) + + # add to total result + total_result = list(map(lambda x, y: x + y, total_result, result)) -sys.exit(exit_status) + total_time = time.monotonic() - total_time + print(build_separator) + print("Build Summary: {} {}, {} {}, {} {} and took {:.2f}s".format(total_result[0], SUCCEEDED, total_result[1], + FAILED, total_result[2], SKIPPED, total_time)) + print(build_separator) + + sys.exit(total_result[1]) diff --git a/tools/build_family.py b/tools/build_family.py index 680411eab..c6c64d2b3 100644 --- a/tools/build_family.py +++ b/tools/build_family.py @@ -1,8 +1,7 @@ import os -import glob import sys -import subprocess import time +from multiprocessing import Pool import build_utils @@ -10,40 +9,15 @@ SUCCEEDED = "\033[32msucceeded\033[0m" FAILED = "\033[31mfailed\033[0m" SKIPPED = "\033[33mskipped\033[0m" -success_count = 0 -fail_count = 0 -skip_count = 0 -exit_status = 0 - -total_time = time.monotonic() - -build_format = '| {:29} | {:30} | {:18} | {:7} | {:6} | {:6} |' build_separator = '-' * 106 + def filter_with_input(mylist): if len(sys.argv) > 1: input_args = list(set(mylist).intersection(sys.argv)) if len(input_args) > 0: mylist[:] = input_args -# If examples are not specified in arguments, build all -all_examples = [] -for dir1 in os.scandir("examples"): - if dir1.is_dir(): - for entry in os.scandir(dir1.path): - if entry.is_dir(): - all_examples.append(dir1.name + '/' + entry.name) -filter_with_input(all_examples) -all_examples.sort() - -# If family are not specified in arguments, build all -all_families = [] -for entry in os.scandir("hw/bsp"): - if entry.is_dir() and os.path.isdir(entry.path + "/boards") and entry.name not in ("esp32s2", "esp32s3"): - all_families.append(entry.name) - -filter_with_input(all_families) -all_families.sort() def build_family(example, family): all_boards = [] @@ -52,61 +26,49 @@ def build_family(example, family): all_boards.append(entry.name) filter_with_input(all_boards) all_boards.sort() - - for board in all_boards: - build_board(example, board) - -def build_board(example, board): - global success_count, fail_count, skip_count, exit_status - start_time = time.monotonic() - flash_size = "-" - sram_size = "-" - # Check if board is skipped - if build_utils.skip_example(example, board): - success = SKIPPED - skip_count += 1 - print(build_format.format(example, board, success, '-', flash_size, sram_size)) - else: - #subprocess.run("make -C examples/{} BOARD={} clean".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - build_result = subprocess.run("make -j -C examples/{} BOARD={} all".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + with Pool(processes=os.cpu_count()) as pool: + pool_args = list((map(lambda b, e=example: [e, b], all_boards))) + result = pool.starmap(build_utils.build_example, pool_args) + # sum all element of same index (column sum) + return list(map(sum, list(zip(*result)))) - if build_result.returncode == 0: - success = SUCCEEDED - success_count += 1 - (flash_size, sram_size) = build_size(example, board) - subprocess.run("make -j -C examples/{} BOARD={} copy-artifact".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - else: - exit_status = build_result.returncode - success = FAILED - fail_count += 1 - build_duration = time.monotonic() - start_time - print(build_format.format(example, board, success, "{:.2f}s".format(build_duration), flash_size, sram_size)) +if __name__ == '__main__': + # If examples are not specified in arguments, build all + all_examples = [] + for dir1 in os.scandir("examples"): + if dir1.is_dir(): + for entry in os.scandir(dir1.path): + if entry.is_dir(): + all_examples.append(dir1.name + '/' + entry.name) + filter_with_input(all_examples) + all_examples.sort() - if build_result.returncode != 0: - print(build_result.stdout.decode("utf-8")) + # If family are not specified in arguments, build all + all_families = [] + for entry in os.scandir("hw/bsp"): + if entry.is_dir() and os.path.isdir(entry.path + "/boards") and entry.name not in ("esp32s2", "esp32s3"): + all_families.append(entry.name) + filter_with_input(all_families) + all_families.sort() -def build_size(example, board): - #elf_file = 'examples/device/{}/_build/{}/{}-firmware.elf'.format(example, board, board) - elf_file = 'examples/{}/_build/{}/*.elf'.format(example, board) - size_output = subprocess.run('size {}'.format(elf_file), shell=True, stdout=subprocess.PIPE).stdout.decode("utf-8") - size_list = size_output.split('\n')[1].split('\t') - flash_size = int(size_list[0]) - sram_size = int(size_list[1]) + int(size_list[2]) - return (flash_size, sram_size) - -print(build_separator) -print(build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM')) - -for example in all_examples: print(build_separator) - for family in all_families: - build_family(example, family) + print(build_utils.build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM')) + total_time = time.monotonic() -total_time = time.monotonic() - total_time -print(build_separator) -print("Build Summary: {} {}, {} {}, {} {} and took {:.2f}s".format(success_count, SUCCEEDED, fail_count, FAILED, skip_count, SKIPPED, total_time)) -print(build_separator) + # succeeded, failed, skipped + total_result = [0, 0, 0] + for example in all_examples: + print(build_separator) + for family in all_families: + fret = build_family(example, family) + total_result = list(map(lambda x, y: x + y, total_result, fret)) -sys.exit(exit_status) + total_time = time.monotonic() - total_time + print(build_separator) + print("Build Summary: {} {}, {} {}, {} {} and took {:.2f}s".format(total_result[0], SUCCEEDED, total_result[1], + FAILED, total_result[2], SKIPPED, total_time)) + print(build_separator) + + sys.exit(total_result[1]) diff --git a/tools/build_utils.py b/tools/build_utils.py index d570d20e3..f457c7986 100644 --- a/tools/build_utils.py +++ b/tools/build_utils.py @@ -1,9 +1,18 @@ +import subprocess import pathlib +import time + +build_format = '| {:29} | {:30} | {:18} | {:7} | {:6} | {:6} |' + +SUCCEEDED = "\033[32msucceeded\033[0m" +FAILED = "\033[31mfailed\033[0m" +SKIPPED = "\033[33mskipped\033[0m" + def skip_example(example, board): ex_dir = pathlib.Path('examples/') / example bsp = pathlib.Path("hw/bsp") - + if (bsp / board / "board.mk").exists(): # board without family board_dir = bsp / board @@ -15,19 +24,19 @@ def skip_example(example, board): if not board_dir: # Skip unknown boards return True - + board_dir = list(board_dir)[0] - + family_dir = board_dir.parent.parent family = family_dir.name - + # family CMake family_mk = family_dir / "family.cmake" - + # family.mk if not family_mk.exists(): family_mk = family_dir / "family.mk" - + mk_contents = family_mk.read_text() # Find the mcu, first in family mk then board mk @@ -66,3 +75,48 @@ def skip_example(example, board): "family:" + family in onlys) return False + + +def build_example(example, board): + start_time = time.monotonic() + flash_size = "-" + sram_size = "-" + + # succeeded, failed, skipped + ret = [0, 0, 0] + + # Check if board is skipped + if skip_example(example, board): + status = SKIPPED + ret[2] = 1 + print(build_format.format(example, board, status, '-', flash_size, sram_size)) + else: + build_result = subprocess.run("make -j -C examples/{} BOARD={} all".format(example, board), shell=True, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + + if build_result.returncode == 0: + status = SUCCEEDED + ret[0] = 1 + (flash_size, sram_size) = build_size(example, board) + subprocess.run("make -j -C examples/{} BOARD={} copy-artifact".format(example, board), shell=True, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + else: + status = FAILED + ret[1] = 1 + + build_duration = time.monotonic() - start_time + print(build_format.format(example, board, status, "{:.2f}s".format(build_duration), flash_size, sram_size)) + + if build_result.returncode != 0: + print(build_result.stdout.decode("utf-8")) + + return ret + + +def build_size(example, board): + elf_file = 'examples/{}/_build/{}/*.elf'.format(example, board) + size_output = subprocess.run('size {}'.format(elf_file), shell=True, stdout=subprocess.PIPE).stdout.decode("utf-8") + size_list = size_output.split('\n')[1].split('\t') + flash_size = int(size_list[0]) + sram_size = int(size_list[1]) + int(size_list[2]) + return (flash_size, sram_size) diff --git a/tools/get_dependencies.py b/tools/get_dependencies.py new file mode 100644 index 000000000..e7d3e0a76 --- /dev/null +++ b/tools/get_dependencies.py @@ -0,0 +1,25 @@ +import os +import sys +import subprocess + + +# dependency lookup (ABC sorted) +# deps = { +# 'LPC11UXX' : [ [] ] +# } + + +def get_family_dep(family): + for entry in os.scandir("hw/bsp/{}/boards".format(family)): + if entry.is_dir(): + result = subprocess.run("make -C examples/device/board_test BOARD={} get-deps".format(entry.name), + shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + print(result.stdout.decode("utf-8")) + return result.returncode + +status = 0 +all_family = sys.argv[1:] +for f in all_family: + status += get_family_dep(f) + +sys.exit(status) \ No newline at end of file