From 9c82cd9f439d7f38f1384037518ce51a5202958a Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 17 Nov 2021 19:13:34 +0100 Subject: [PATCH 01/13] Declare which Python packages we use Add pip requirements files. We'll have separate requirements files for different target audiences. Each file can use `-r` lines to include other files. This commit adds two requirement files: one with everything that's needed to pass the CI, and one with additional tools that are suggested for Mbed TLS maintainers to install locally. Signed-off-by: Gilles Peskine --- scripts/ci.requirements.txt | 10 ++++++++++ scripts/maintainer.requirements.txt | 6 ++++++ 2 files changed, 16 insertions(+) create mode 100644 scripts/ci.requirements.txt create mode 100644 scripts/maintainer.requirements.txt diff --git a/scripts/ci.requirements.txt b/scripts/ci.requirements.txt new file mode 100644 index 0000000000..18b40ec173 --- /dev/null +++ b/scripts/ci.requirements.txt @@ -0,0 +1,10 @@ +# Python package requirements for Mbed TLS testing. + +# Use a known version of Pylint, because new versions tend to add warnings +# that could start rejecting our code. +# 2.4.4 is the version in Ubuntu 20.04. It supports Python >=3.5. +pylint == 2.4.4 + +# Use the earliest version of mypy that works with our code base. +# See https://github.com/ARMmbed/mbedtls/pull/3953 . +mypy >= 0.780 diff --git a/scripts/maintainer.requirements.txt b/scripts/maintainer.requirements.txt new file mode 100644 index 0000000000..670617ba7b --- /dev/null +++ b/scripts/maintainer.requirements.txt @@ -0,0 +1,6 @@ +# Python packages that are only useful to Mbed TLS maintainers. + +-r ci.requirements.txt + +# For source code analyses +clang From 87485a3f2825f6a644e860db748ca2616797d92d Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 17 Nov 2021 19:17:03 +0100 Subject: [PATCH 02/13] Add requirement on Jinja to integrate drivers Driver implementers need to regenerate wrappers. This will use Jinja2 as discussed in https://github.com/ARMmbed/mbedtls/pull/5067#discussion_r738794607 On the development branch, driver integration is always needed to generate the driver wrapper and thus to build the library, so this requirement applies to everyone, not just driver implementers. In releases, we plan to include a default driver wrapper with support for basic use cases only, meaning that the line `-r driver.requirements.txt` should be removed from `basic.requirements.txt` in releases. Signed-off-by: Gilles Peskine --- README.md | 5 ++++- scripts/basic.requirements.txt | 5 +++++ scripts/ci.requirements.txt | 2 ++ scripts/driver.requirements.txt | 9 +++++++++ 4 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 scripts/basic.requirements.txt create mode 100644 scripts/driver.requirements.txt diff --git a/README.md b/README.md index c8d94500e2..ea1d7a37b7 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,10 @@ The source code of Mbed TLS includes some files that are automatically generated The following tools are required: * Perl, for some library source files and for Visual Studio build files. -* Python 3, for some sample programs and test data. +* Python 3 and some Python packages, for some library source files, sample programs and test data. To install the necessary packages, run + ``` + python -m pip install -r scripts/basic.requirements.txt + ``` * A C compiler for the host platform, for some test data. If you are cross-compiling, you must set the `CC` environment variable to a C compiler for the host platform when generating the configuration-independent files. diff --git a/scripts/basic.requirements.txt b/scripts/basic.requirements.txt new file mode 100644 index 0000000000..1be3d0c235 --- /dev/null +++ b/scripts/basic.requirements.txt @@ -0,0 +1,5 @@ +# Python modules required to build Mbed TLS in ordinary conditions. + +# Required to (re-)generate source files. Not needed if the generated source +# files are already present and up-to-date. +-r driver.requirements.txt diff --git a/scripts/ci.requirements.txt b/scripts/ci.requirements.txt index 18b40ec173..209ae3d8ff 100644 --- a/scripts/ci.requirements.txt +++ b/scripts/ci.requirements.txt @@ -1,5 +1,7 @@ # Python package requirements for Mbed TLS testing. +-r driver.requirements.txt + # Use a known version of Pylint, because new versions tend to add warnings # that could start rejecting our code. # 2.4.4 is the version in Ubuntu 20.04. It supports Python >=3.5. diff --git a/scripts/driver.requirements.txt b/scripts/driver.requirements.txt new file mode 100644 index 0000000000..5a3e76a0ec --- /dev/null +++ b/scripts/driver.requirements.txt @@ -0,0 +1,9 @@ +# Python package requirements for driver implementers. + +# Use the version of Jinja that's in Ubuntu 20.04. +# See https://github.com/ARMmbed/mbedtls/pull/5067#discussion_r738794607 . +# Note that Jinja2 3.0 drops support for Python 3.5, so we need 2.x. +Jinja2 >= 2.10.1 +# Jinja2 >=2.10, <<3.0 needs a separate package for type annotations +types-Jinja2 + From 9172c9c073fa98e86bcdd73c5f28025b55c3b320 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 17 Nov 2021 19:25:43 +0100 Subject: [PATCH 03/13] Script to install minimum versions of the requirements Wherever we have a requirement on foo>=N, install foo==N. This is for testing, to ensure that we don't accidentally depend on features that are not present in the minimum version we declare support for. Signed-off-by: Gilles Peskine --- scripts/min_requirements.py | 106 ++++++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100755 scripts/min_requirements.py diff --git a/scripts/min_requirements.py b/scripts/min_requirements.py new file mode 100755 index 0000000000..3b156a36ab --- /dev/null +++ b/scripts/min_requirements.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""Install all the required Python packages, with the minimum Python version. +""" + +# Copyright The Mbed TLS Contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import re +import sys +import typing + +from typing import List +from mbedtls_dev import typing_util + +def pylint_doesn_t_notice_that_certain_types_are_used_in_annotations( + _list: List[typing.Any], +) -> None: + pass + + +class Requirements: + """Collect and massage Python requirements.""" + + def __init__(self) -> None: + self.requirements = [] #type: List[str] + + def adjust_requirement(self, req: str) -> str: + """Adjust a requirement to the minimum specified version.""" + # allow inheritance #pylint: disable=no-self-use + # If a requirement specifies a minimum version, impose that version. + req = re.sub(r'>=|~=', r'==', req) + return req + + def add_file(self, filename: str) -> None: + """Add requirements from the specified file. + + This method supports a subset of pip's requirement file syntax: + * One requirement specifier per line, which is passed to + `adjust_requirement`. + * Comments (``#`` at the beginning of the line or after whitespace). + * ``-r FILENAME`` to include another file. + """ + for line in open(filename): + line = line.strip() + line = re.sub(r'(\A|\s+)#.*', r'', line) + if not line: + continue + m = re.match(r'-r\s+', line) + if m: + nested_file = os.path.join(os.path.dirname(filename), + line[m.end(0):]) + self.add_file(nested_file) + continue + self.requirements.append(self.adjust_requirement(line)) + + def write(self, out: typing_util.Writable) -> None: + """List the gathered requirements.""" + for req in self.requirements: + out.write(req + '\n') + + def install(self) -> None: + """Call pip to install the requirements.""" + if not self.requirements: + return + ret = os.spawnl(os.P_WAIT, sys.executable, 'python', '-m', 'pip', + 'install', *self.requirements) + if ret != 0: + sys.exit(ret) + + +def main() -> None: + """Command line entry point.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--no-act', '-n', + action='store_true', + help="Don't act, just print what will be done") + parser.add_argument('files', nargs='*', metavar='FILE', + help="Requirement files" + "(default: requirements.txt in the script's directory)") + options = parser.parse_args() + if not options.files: + options.files = [os.path.join(os.path.dirname(__file__), + 'ci.requirements.txt')] + reqs = Requirements() + for filename in options.files: + reqs.add_file(filename) + reqs.write(sys.stdout) + if not options.no_act: + reqs.install() + +if __name__ == '__main__': + main() From 8cbb7b995f305d194c27d6b34e718db53be3b275 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 17 Nov 2021 19:48:21 +0100 Subject: [PATCH 04/13] Docker: Python requirements are now managed in-tree Neither mbed-host-tests nor mock are currently used. Signed-off-by: Gilles Peskine --- tests/docker/bionic/Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/docker/bionic/Dockerfile b/tests/docker/bionic/Dockerfile index 41789c677c..3da14a2745 100644 --- a/tests/docker/bionic/Dockerfile +++ b/tests/docker/bionic/Dockerfile @@ -161,6 +161,4 @@ RUN cd /tmp \ ENV GNUTLS_NEXT_CLI=/usr/local/gnutls-3.7.2/bin/gnutls-cli ENV GNUTLS_NEXT_SERV=/usr/local/gnutls-3.7.2/bin/gnutls-serv -RUN pip3 install --no-cache-dir \ - mbed-host-tests \ - mock +RUN scripts/min_requirements.py From d9d5c7856f9ec935a6d4f0940990b516277c7108 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 17 Nov 2021 19:29:38 +0100 Subject: [PATCH 05/13] Travis: use the in-tree Python package requirements Signed-off-by: Gilles Peskine --- .travis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 39ae19c190..f910ace98b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,8 +18,6 @@ jobs: - libc6-dev-armel-cross language: python # Needed to get pip for Python 3 python: 3.5 # version from Ubuntu 16.04 - install: - - pip install mypy==0.780 pylint==2.4.4 script: - tests/scripts/all.sh -k 'check_*' - tests/scripts/all.sh -k test_default_out_of_box @@ -53,6 +51,9 @@ env: - SEED=1 - secure: "FrI5d2s+ckckC17T66c8jm2jV6i2DkBPU5nyWzwbedjmEBeocREfQLd/x8yKpPzLDz7ghOvr+/GQvsPPn0dVkGlNzm3Q+hGHc/ujnASuUtGrcuMM+0ALnJ3k4rFr9xEvjJeWb4SmhJO5UCAZYvTItW4k7+bj9L+R6lt3TzQbXzg=" +install: + - scripts/min_requirements.py + addons: apt: packages: From d80cf54e10cc0b2e76e86d7b23a2bcc35cd5fe06 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Wed, 17 Nov 2021 21:13:01 +0100 Subject: [PATCH 06/13] Declare all jobs as Python This way we get our chosen Python version everywhere, and pip is available. Travis doesn't support the python job type on Windows, however, so keep installing Python manually there. Signed-off-by: Gilles Peskine --- .travis.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index f910ace98b..08d7dfb703 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,7 @@ -language: c -compiler: gcc +# Declare python as our language. This way we get our chosen Python version, +# and pip is available. Gcc and clang are available anyway. +language: python +python: 3.5 sudo: false cache: ccache @@ -16,8 +18,6 @@ jobs: - libnewlib-arm-none-eabi - gcc-arm-linux-gnueabi - libc6-dev-armel-cross - language: python # Needed to get pip for Python 3 - python: 3.5 # version from Ubuntu 16.04 script: - tests/scripts/all.sh -k 'check_*' - tests/scripts/all.sh -k test_default_out_of_box @@ -30,6 +30,10 @@ jobs: - name: Windows os: windows + # The language 'python' is currently unsupported on the + # Windows Build Environment. And 'generic' causes the job to get stuck + # on "Booting virtual machine". + language: c before_install: - choco install python --version=3.5.4 env: From dd386697036f54f6663f8e9d9dfa215e67643d66 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 18 Nov 2021 17:35:01 +0100 Subject: [PATCH 07/13] Reconcile python interpreter names Our shebangs use `python3`, which is the desired name on Linux (where `python` is still Python 2). But on Windows, Choco's Python only provides a `python3.exe` executable. Our build scripts deal with this, but we need to cope when invoking a Python script from Travis itself. Signed-off-by: Gilles Peskine --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 08d7dfb703..cdb68ac42f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,6 +39,7 @@ jobs: env: # Add the directory where the Choco packages go - PATH=/c/Python35:/c/Python35/Scripts:$PATH + - PYTHON=python.exe script: - type perl; perl --version - type python; python --version @@ -56,7 +57,7 @@ env: - secure: "FrI5d2s+ckckC17T66c8jm2jV6i2DkBPU5nyWzwbedjmEBeocREfQLd/x8yKpPzLDz7ghOvr+/GQvsPPn0dVkGlNzm3Q+hGHc/ujnASuUtGrcuMM+0ALnJ3k4rFr9xEvjJeWb4SmhJO5UCAZYvTItW4k7+bj9L+R6lt3TzQbXzg=" install: - - scripts/min_requirements.py + - $PYTHON scripts/min_requirements.py addons: apt: From 8f63f6dcc623404b61256605d0084f7a74b5f265 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 18 Nov 2021 18:18:35 +0100 Subject: [PATCH 08/13] Use a method to invoke pip that works on Windows Passing arguments on the command line apparently didn't work due to quoting issues. Use a temporary file instead. Signed-off-by: Gilles Peskine --- scripts/min_requirements.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/scripts/min_requirements.py b/scripts/min_requirements.py index 3b156a36ab..ef6c42bf61 100755 --- a/scripts/min_requirements.py +++ b/scripts/min_requirements.py @@ -20,7 +20,9 @@ import argparse import os import re +import subprocess import sys +import tempfile import typing from typing import List @@ -74,12 +76,18 @@ class Requirements: def install(self) -> None: """Call pip to install the requirements.""" - if not self.requirements: - return - ret = os.spawnl(os.P_WAIT, sys.executable, 'python', '-m', 'pip', - 'install', *self.requirements) - if ret != 0: - sys.exit(ret) + with tempfile.TemporaryDirectory() as temp_dir: + # This is more complicated than it needs to be for the sake + # of Windows. Use a temporary file rather than the command line + # to avoid quoting issues. Use a temporary directory rather + # than NamedTemporaryFile because with a NamedTemporaryFile on + # Windows, the subprocess can't open the file because this process + # has an exclusive lock on it. + req_file_name = os.path.join(temp_dir, 'requirements.txt') + with open(req_file_name, 'w') as req_file: + self.write(req_file) + subprocess.check_call([sys.executable, '-m', 'pip', + 'install', '-r', req_file_name]) def main() -> None: From 7f29ea6d3966512e33d95700e5c050b3fa6453f2 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Mon, 22 Nov 2021 12:52:24 +0000 Subject: [PATCH 09/13] Allow passing options to pip Signed-off-by: Gilles Peskine --- scripts/min_requirements.py | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/scripts/min_requirements.py b/scripts/min_requirements.py index ef6c42bf61..db301e8913 100755 --- a/scripts/min_requirements.py +++ b/scripts/min_requirements.py @@ -25,7 +25,7 @@ import sys import tempfile import typing -from typing import List +from typing import List, Optional from mbedtls_dev import typing_util def pylint_doesn_t_notice_that_certain_types_are_used_in_annotations( @@ -74,8 +74,16 @@ class Requirements: for req in self.requirements: out.write(req + '\n') - def install(self) -> None: + def install( + self, + pip_general_options: Optional[List[str]] = None, + pip_install_options: Optional[List[str]] = None, + ) -> None: """Call pip to install the requirements.""" + if pip_general_options is None: + pip_general_options = [] + if pip_install_options is None: + pip_install_options = [] with tempfile.TemporaryDirectory() as temp_dir: # This is more complicated than it needs to be for the sake # of Windows. Use a temporary file rather than the command line @@ -86,8 +94,10 @@ class Requirements: req_file_name = os.path.join(temp_dir, 'requirements.txt') with open(req_file_name, 'w') as req_file: self.write(req_file) - subprocess.check_call([sys.executable, '-m', 'pip', - 'install', '-r', req_file_name]) + subprocess.check_call([sys.executable, '-m', 'pip'] + + pip_general_options + + ['install'] + pip_install_options + + ['-r', req_file_name]) def main() -> None: @@ -96,9 +106,20 @@ def main() -> None: parser.add_argument('--no-act', '-n', action='store_true', help="Don't act, just print what will be done") + parser.add_argument('--pip-install-option', + action='append', dest='pip_install_options', + help="Pass this option to pip install") + parser.add_argument('--pip-option', + action='append', dest='pip_general_options', + help="Pass this general option to pip") + parser.add_argument('--user', + action='append_const', dest='pip_install_options', + const='--user', + help="Install to the Python user install directory" + " (short for --pip-install-option --user)") parser.add_argument('files', nargs='*', metavar='FILE', help="Requirement files" - "(default: requirements.txt in the script's directory)") + " (default: requirements.txt in the script's directory)") options = parser.parse_args() if not options.files: options.files = [os.path.join(os.path.dirname(__file__), @@ -108,7 +129,8 @@ def main() -> None: reqs.add_file(filename) reqs.write(sys.stdout) if not options.no_act: - reqs.install() + reqs.install(pip_general_options=options.pip_general_options, + pip_install_options=options.pip_install_options) if __name__ == '__main__': main() From 2673aa58124579122983cb9e0a1375f29ed7ae00 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 2 Dec 2021 12:44:50 +0100 Subject: [PATCH 10/13] Remove accidental requirement on the worktree content This made the build impossible since mbedtls isn't available when building the container. Signed-off-by: Gilles Peskine --- tests/docker/bionic/Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/docker/bionic/Dockerfile b/tests/docker/bionic/Dockerfile index 3da14a2745..50f5a7fba8 100644 --- a/tests/docker/bionic/Dockerfile +++ b/tests/docker/bionic/Dockerfile @@ -160,5 +160,3 @@ RUN cd /tmp \ ENV GNUTLS_NEXT_CLI=/usr/local/gnutls-3.7.2/bin/gnutls-cli ENV GNUTLS_NEXT_SERV=/usr/local/gnutls-3.7.2/bin/gnutls-serv - -RUN scripts/min_requirements.py From b3e5340b1f8b3edb85ae3bb9aa527fa047e52081 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 2 Dec 2021 12:48:50 +0100 Subject: [PATCH 11/13] Clarify comment Signed-off-by: Gilles Peskine --- scripts/driver.requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/driver.requirements.txt b/scripts/driver.requirements.txt index 5a3e76a0ec..17569bb170 100644 --- a/scripts/driver.requirements.txt +++ b/scripts/driver.requirements.txt @@ -2,7 +2,8 @@ # Use the version of Jinja that's in Ubuntu 20.04. # See https://github.com/ARMmbed/mbedtls/pull/5067#discussion_r738794607 . -# Note that Jinja2 3.0 drops support for Python 3.5, so we need 2.x. +# Note that Jinja 3.0 drops support for Python 3.5, so we need to support +# Jinja 2.x as long as we're still using Python 3.5 anywhere. Jinja2 >= 2.10.1 # Jinja2 >=2.10, <<3.0 needs a separate package for type annotations types-Jinja2 From 26f60b38de4ffddd1bdb63d1f417afa5578e2a00 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Thu, 2 Dec 2021 12:50:06 +0100 Subject: [PATCH 12/13] Add Cryptodome to maintainer requirements See e.g. https://github.com/ARMmbed/mbedtls/pull/5218 Signed-off-by: Gilles Peskine --- scripts/maintainer.requirements.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/maintainer.requirements.txt b/scripts/maintainer.requirements.txt index 670617ba7b..b149921a24 100644 --- a/scripts/maintainer.requirements.txt +++ b/scripts/maintainer.requirements.txt @@ -4,3 +4,7 @@ # For source code analyses clang + +# For building some test vectors +pycryptodomex +pycryptodome-test-vectors From 3d57afe2b0006c41074a3ebd0c6e1f6510789f29 Mon Sep 17 00:00:00 2001 From: Gilles Peskine Date: Fri, 3 Dec 2021 13:32:10 +0100 Subject: [PATCH 13/13] Correct default requirements file name in help Signed-off-by: Gilles Peskine --- scripts/min_requirements.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/min_requirements.py b/scripts/min_requirements.py index db301e8913..eecab1c1e1 100755 --- a/scripts/min_requirements.py +++ b/scripts/min_requirements.py @@ -99,6 +99,7 @@ class Requirements: ['install'] + pip_install_options + ['-r', req_file_name]) +DEFAULT_REQUIREMENTS_FILE = 'ci.requirements.txt' def main() -> None: """Command line entry point.""" @@ -119,11 +120,12 @@ def main() -> None: " (short for --pip-install-option --user)") parser.add_argument('files', nargs='*', metavar='FILE', help="Requirement files" - " (default: requirements.txt in the script's directory)") + " (default: {} in the script's directory)" \ + .format(DEFAULT_REQUIREMENTS_FILE)) options = parser.parse_args() if not options.files: options.files = [os.path.join(os.path.dirname(__file__), - 'ci.requirements.txt')] + DEFAULT_REQUIREMENTS_FILE)] reqs = Requirements() for filename in options.files: reqs.add_file(filename)