From 6f47746e5ff6e33b25e12742d62bcbf69f8a27a0 Mon Sep 17 00:00:00 2001 From: Ha Thach Date: Mon, 13 May 2024 16:41:11 +0700 Subject: [PATCH] more ci update (#2642) * Circi use small docker * caching espressif docker image * only run make job on pull request or push to master * hw test run on pull request only, rename build_cmake to build.yml * enable all ci build, cmake(clang) and make(*) only run with pull_request or push to master --- .circleci/config.yml | 2 + .github/actions/setup_toolchain/action.yml | 11 +++-- .../setup_toolchain/download/action.yml | 21 ++++++---- .../setup_toolchain/espressif/action.yml | 42 +++++++++++++++++++ .../workflows/{build_cmake.yml => build.yml} | 11 ++++- .github/workflows/build_iar.yml | 9 ---- .github/workflows/hil_test.yml | 10 ----- tools/get_deps.py | 13 +++++- 8 files changed, 83 insertions(+), 36 deletions(-) create mode 100644 .github/actions/setup_toolchain/espressif/action.yml rename .github/workflows/{build_cmake.yml => build.yml} (90%) diff --git a/.circleci/config.yml b/.circleci/config.yml index d5d401c51..d91de8419 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,6 +13,8 @@ jobs: # See: https://circleci.com/developer/images/image/cimg/base - image: cimg/base:current + resource_class: small + # Add steps to the job # See: https://circleci.com/docs/jobs-steps/#steps-overview & https://circleci.com/docs/configuration-reference/#steps steps: diff --git a/.github/actions/setup_toolchain/action.yml b/.github/actions/setup_toolchain/action.yml index e6c79e7dd..34bcbc68b 100644 --- a/.github/actions/setup_toolchain/action.yml +++ b/.github/actions/setup_toolchain/action.yml @@ -2,11 +2,11 @@ name: Setup Toolchain inputs: toolchain: + description: 'Toolchain name' required: true - type: string toolchain_url: + description: 'Toolchain URL or version' required: false - type: string outputs: build_option: @@ -24,8 +24,10 @@ runs: - name: Pull ESP-IDF docker if: inputs.toolchain == 'esp-idf' - run: docker pull espressif/idf:${{ inputs.toolchain_url }} - shell: bash + uses: ./.github/actions/setup_toolchain/espressif + with: + toolchain: ${{ inputs.toolchain }} + toolchain_url: ${{ inputs.toolchain_url }} - name: Download Toolchain if: >- @@ -33,6 +35,7 @@ runs: inputs.toolchain != 'esp-idf' uses: ./.github/actions/setup_toolchain/download with: + toolchain: ${{ inputs.toolchain }} toolchain_url: ${{ inputs.toolchain_url }} - name: Set toolchain option diff --git a/.github/actions/setup_toolchain/download/action.yml b/.github/actions/setup_toolchain/download/action.yml index db85e9027..0b35bddb1 100644 --- a/.github/actions/setup_toolchain/download/action.yml +++ b/.github/actions/setup_toolchain/download/action.yml @@ -1,29 +1,32 @@ name: Download Toolchain inputs: - toolchain_url: + toolchain: + description: 'Toolchain name' + required: true + toolchain_url: + description: 'Toolchain URL' required: true - type: string runs: using: "composite" steps: - name: Cache Toolchain uses: actions/cache@v4 - id: cache-toolchain + id: cache-toolchain-download with: - path: ~/cache/toolchain - key: ${{ runner.os }}-${{ inputs.toolchain_url }} + path: ~/cache/${{ inputs.toolchain }} + key: ${{ runner.os }}-${{ inputs.toolchain }}-${{ inputs.toolchain_url }} - name: Install Toolchain - if: steps.cache-toolchain.outputs.cache-hit != 'true' + if: steps.cache-toolchain-download.outputs.cache-hit != 'true' run: | - mkdir -p ~/cache/toolchain + mkdir -p ~/cache/${{ inputs.toolchain }} wget --progress=dot:mega ${{ inputs.toolchain_url }} -O toolchain.tar.gz - tar -C ~/cache/toolchain -xaf toolchain.tar.gz + tar -C ~/cache/${{ inputs.toolchain }} -xaf toolchain.tar.gz shell: bash - name: Set Toolchain Path run: | - echo >> $GITHUB_PATH `echo ~/cache/toolchain/*/bin` + echo >> $GITHUB_PATH `echo ~/cache/${{ inputs.toolchain }}/*/bin` shell: bash diff --git a/.github/actions/setup_toolchain/espressif/action.yml b/.github/actions/setup_toolchain/espressif/action.yml new file mode 100644 index 000000000..a4c6127c6 --- /dev/null +++ b/.github/actions/setup_toolchain/espressif/action.yml @@ -0,0 +1,42 @@ +name: Setup ESP-IDF Toolchain + +inputs: + toolchain: + description: 'Toolchain name' + required: true + toolchain_url: + description: 'Toolchain URL or version' + required: true + +runs: + using: "composite" + steps: + - id: set-docker-image + run: | + DOCKER_IMAGE=$HOME/cache/${{ inputs.toolchain }}/docker_image.tar + echo "DOCKER_IMAGE=$DOCKER_IMAGE" >> $GITHUB_ENV + echo "DOCKER_IMAGE=$DOCKER_IMAGE" >> $GITHUB_OUTPUT + shell: bash + + - name: Cache Docker Image + uses: actions/cache@v4 + id: cache-toolchain-espressif + with: + path: ${{ steps.set-docker-image.outputs.DOCKER_IMAGE }} + key: ${{ runner.os }}-${{ inputs.toolchain }}-${{ inputs.toolchain_url }} + + - name: Pull and Save Docker Image + if: steps.cache-toolchain-espressif.outputs.cache-hit != 'true' + run: | + docker pull espressif/idf:${{ inputs.toolchain_url }} + mkdir -p ~/cache/${{ inputs.toolchain }} + docker save -o $DOCKER_IMAGE espressif/idf:${{ inputs.toolchain_url }} + du -sh $DOCKER_IMAGE + shell: bash + + - name: Load Docker Image + if: steps.cache-toolchain-espressif.outputs.cache-hit == 'true' + run: | + du -sh $DOCKER_IMAGE + docker load --input $DOCKER_IMAGE + shell: bash diff --git a/.github/workflows/build_cmake.yml b/.github/workflows/build.yml similarity index 90% rename from .github/workflows/build_cmake.yml rename to .github/workflows/build.yml index 4723cf8d9..d51e1c92a 100644 --- a/.github/workflows/build_cmake.yml +++ b/.github/workflows/build.yml @@ -11,7 +11,7 @@ on: - 'tools/get_deps.py' - 'tools/build.py' - '.github/actions/**' - - '.github/workflows/build_cmake.yml' + - '.github/workflows/build.yml' - '.github/workflows/build_util.yml' - '.github/workflows/ci_set_matrix.py' pull_request: @@ -24,7 +24,7 @@ on: - 'tools/get_deps.py' - 'tools/build.py' - '.github/actions/**' - - '.github/workflows/build_cmake.yml' + - '.github/workflows/build.yml' - '.github/workflows/build_util.yml' - '.github/workflows/ci_set_matrix.py' concurrency: @@ -67,6 +67,10 @@ jobs: - 'arm-gcc' - 'msp430-gcc' - 'riscv-gcc' + if: >- + matrix.toolchain != 'arm-clang' || + github.event_name == 'pull_request' || + (github.event_name == 'push' && github.ref == 'refs/heads/master') with: build-system: 'cmake' toolchain: ${{ matrix.toolchain }} @@ -77,6 +81,9 @@ jobs: # Build Make # --------------------------------------- make: + if: >- + github.event_name == 'pull_request' || + (github.event_name == 'push' && github.ref == 'refs/heads/master') needs: set-matrix uses: ./.github/workflows/build_util.yml strategy: diff --git a/.github/workflows/build_iar.yml b/.github/workflows/build_iar.yml index 34dbda192..c8a095c18 100644 --- a/.github/workflows/build_iar.yml +++ b/.github/workflows/build_iar.yml @@ -2,15 +2,6 @@ name: Build IAR on: workflow_dispatch: - push: - paths: - - 'src/**' - - 'examples/**' - - 'lib/**' - - 'hw/**' - - 'tools/get_deps.py' - - 'test/hil/**' - - '.github/workflows/build_iar.yml' pull_request: branches: [ master ] paths: diff --git a/.github/workflows/hil_test.yml b/.github/workflows/hil_test.yml index 796ff32dc..eb3f2cbab 100644 --- a/.github/workflows/hil_test.yml +++ b/.github/workflows/hil_test.yml @@ -2,16 +2,6 @@ name: Hardware Test on: workflow_dispatch: - push: - paths: - - 'src/**' - - 'examples/**' - - 'lib/**' - - 'hw/**' - - 'test/hil/**' - - 'tools/get_deps.py' - - '.github/actions/**' - - '.github/workflows/hil_test.yml' pull_request: branches: [ master ] paths: diff --git a/tools/get_deps.py b/tools/get_deps.py index 20cbe64c7..2c2f97b4b 100644 --- a/tools/get_deps.py +++ b/tools/get_deps.py @@ -242,10 +242,12 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument('families', nargs='*', default=[], help='Families to fetch') parser.add_argument('-b', '--board', action='append', default=[], help='Boards to fetch') + parser.add_argument('--print', action='store_true', help='Print commit hash only') args = parser.parse_args() families = args.families boards = args.board + print_only = args.print if len(families) == 0 and len(boards) == 0: print("Warning: family and board are not specified, only fetching mandatory dependencies.") @@ -268,8 +270,15 @@ def main(): if f in deps_optional[d][2]: deps.append(d) - with Pool() as pool: - status = sum(pool.map(get_a_dep, deps)) + if print_only: + pvalue = {} + for d in deps: + commit = deps_all[d][1] + pvalue[d] = commit + print(pvalue) + else: + with Pool() as pool: + status = sum(pool.map(get_a_dep, deps)) return status