From 2120d5d0f0b9c2aeec8e7d040bf042a3d57cbcd2 Mon Sep 17 00:00:00 2001 From: Chris Lalancette Date: Sun, 28 Jan 2024 09:17:27 -0500 Subject: [PATCH] Pin all versions of python dependencies. This allows us to be in control of our destiny, and not have random python upstream package updates destroy the entire buildfarm. We accomplish this through 3 different mechanisms, depending on the platform: 1. Ubuntu - we install everything from debian packages, and nothing from pip. Because Ubuntu promises to not to breaking changes within a distribution, this guarantees we won't break. This also serves as our canonical version of what should work. 2. RHEL - For RHEL-9, we install everything from RPM packages, and nothing from pip. For RHEL-8, we install most everything from RPM packages, with the exception of mypy, pytest, and pytest-rerunfailures. Since RHEL/EPEL typically doesn't update major versions within a release, we typically won't break. 3. Windows - We install all python packages via pip, and we constrain the versions we install to those in Ubuntu. This means that we'll always match what works on Ubuntu and we should always work. Signed-off-by: Chris Lalancette --- .gitmodules | 2 +- ros2_batch_job/__main__.py | 174 ++++++------------------ windows_docker_resources/ros2-cookbooks | 2 +- 3 files changed, 44 insertions(+), 134 deletions(-) diff --git a/.gitmodules b/.gitmodules index d13fd6b59..04fafb9aa 100644 --- a/.gitmodules +++ b/.gitmodules @@ -20,4 +20,4 @@ [submodule "windows_docker_resources/ros2-cookbooks"] path = windows_docker_resources/ros2-cookbooks url = git@github.com:ros-infrastructure/ros2-cookbooks - branch = latest + branch = clalancette/windows-pip-install diff --git a/ros2_batch_job/__main__.py b/ros2_batch_job/__main__.py index b9039ba90..21621a0e8 100644 --- a/ros2_batch_job/__main__.py +++ b/ros2_batch_job/__main__.py @@ -47,66 +47,6 @@ sys.stdout = UnbufferedIO(sys.stdout) sys.stderr = UnbufferedIO(sys.stderr) -# One of the maintainers of pyparsing suggests pinning to 2.4.7 for now; -# see https://github.com/pyparsing/pyparsing/issues/323 -pip_dependencies = [ - 'EmPy', - 'coverage', - 'catkin_pkg', - 'flake8', - 'flake8-blind-except==0.1.1', - 'flake8-builtins', - 'flake8-class-newline', - 'flake8-comprehensions', - 'flake8-deprecated', - 'flake8-docstrings', - 'flake8-import-order', - 'flake8-quotes', - 'importlib-metadata', - 'mock', - 'nose', - 'pep8', - 'pydocstyle', - 'pyflakes', - 'pyparsing==2.4.7', - 'pytest', - 'pytest-cov', - 'pytest-mock', - 'pytest-repeat', - 'pytest-rerunfailures', - 'pytest-runner', - 'pyyaml', - 'vcstool', - 'yamllint', -] -# https://github.com/pyca/cryptography/issues/5433 -pip_cryptography_version = '==3.0' - -colcon_packages = [ - 'colcon-core', - 'colcon-defaults', - 'colcon-library-path', - 'colcon-metadata', - 'colcon-mixin', - 'colcon-output', - 'colcon-package-information', - 'colcon-package-selection', - 'colcon-parallel-executor', - 'colcon-pkg-config', - 'colcon-powershell', - 'colcon-python-setup-py', - 'colcon-recursive-crawl', - 'colcon-test-result', - 'colcon-cmake', - 'colcon-ros', - 'colcon-ros-domain-id-coordinator', -] -if sys.platform != 'win32': - colcon_packages += [ - 'colcon-bash', - 'colcon-zsh', - ] - gcov_flags = '--coverage' colcon_space_defaults = { @@ -440,91 +380,61 @@ def run(args, build_function, blacklisted_package_names=None): # Now inside of the workspace... with change_directory(args.workspace): - def need_package_from_pipy(pkg_name): - try: - importlib.import_module(pkg_name) - except ModuleNotFoundError: - return True - - return False - print('# BEGIN SUBSECTION: install Python packages') # Print setuptools version job.run(['"%s"' % job.python, '-c', '"import setuptools; print(setuptools.__version__)"'], shell=True) # Print the pip version job.run(['"%s"' % job.python, '-m', 'pip', '--version'], shell=True) - # Install pip dependencies - pip_packages = list(pip_dependencies) - - # We prefer to get mypy from the distribution if it exists. If not we install it via pip. - if need_package_from_pipy("mypy"): - pip_packages += ["mypy==0.931"] - - # We prefer to get pytest-timeout from the distribution if it exists. If not we install it via pip. - if need_package_from_pipy("pytest_timeout"): - pip_packages += ["pytest-timeout==2.1.0"] - # We prefer to get lark from the distribution if it exists. If not we install it via pip. - if need_package_from_pipy("lark"): - pip_packages += ["lark==1.1.1"] - - if sys.platform == 'win32': - # Install fork of pyreadline containing fix for deprecation warnings - # TODO(jacobperron): Until upstream issue is resolved https://github.com/pyreadline/pyreadline/issues/65 - pip_packages += ['git+https://github.com/osrf/pyreadline'] - - # Setuptools > 61 somehow have broken Windows Debug. Pin it to 59.6.0 here which - # matches Ubuntu Jammy, and wait until upstream setuptools settles down. - pip_packages += ["setuptools==59.6.0"] - - if args.cmake_build_type == 'Debug': - pip_packages += [ - 'https://github.com/ros2/ros2/releases/download/cryptography-archives/cffi-1.14.0-cp38-cp38d-win_amd64.whl', # required by cryptography - 'https://github.com/ros2/ros2/releases/download/cryptography-archives/cryptography-2.9.2-cp38-cp38d-win_amd64.whl', - 'https://github.com/ros2/ros2/releases/download/lxml-archives/lxml-4.5.1-cp38-cp38d-win_amd64.whl', - 'https://github.com/ros2/ros2/releases/download/numpy-archives/numpy-1.18.4-cp38-cp38d-win_amd64.whl', - 'https://github.com/ros2/ros2/releases/download/psutil-archives/psutil-5.9.5-cp38-cp38d-win_amd64.whl', - ] - if args.ros_distro in ('humble', 'iron'): - pip_packages.append('https://github.com/ros2/ros2/releases/download/netifaces-archives/netifaces-0.10.9-cp38-cp38d-win_amd64.whl') - else: - pip_packages += [ - f'cryptography{pip_cryptography_version}', - 'lxml', - 'numpy', - ] - if args.ros_distro in ('humble', 'iron'): - pip_packages.append('netifaces') - if not args.colcon_branch: - pip_packages += colcon_packages - if sys.platform == 'win32': - job.run( - ['"%s"' % job.python, '-m', 'pip', 'uninstall', '-y'] + - colcon_packages, shell=True) + if sys.platform == 'win32' and args.cmake_build_type == 'Debug': + pip_packages = [ + 'https://github.com/ros2/ros2/releases/download/cryptography-archives/cffi-1.14.0-cp38-cp38d-win_amd64.whl', # required by cryptography + 'https://github.com/ros2/ros2/releases/download/cryptography-archives/cryptography-2.9.2-cp38-cp38d-win_amd64.whl', + 'https://github.com/ros2/ros2/releases/download/lxml-archives/lxml-4.5.1-cp38-cp38d-win_amd64.whl', + 'https://github.com/ros2/ros2/releases/download/numpy-archives/numpy-1.18.4-cp38-cp38d-win_amd64.whl', + 'https://github.com/ros2/ros2/releases/download/psutil-archives/psutil-5.9.5-cp38-cp38d-win_amd64.whl', + ] + if args.ros_distro in ('humble', 'iron'): + pip_packages.append('https://github.com/ros2/ros2/releases/download/netifaces-archives/netifaces-0.10.9-cp38-cp38d-win_amd64.whl') # to ensure that the build type specific package is installed job.run( ['"%s"' % job.python, '-m', 'pip', 'uninstall', '-y'] + - [f'cryptography{pip_cryptography_version}', 'lxml', 'numpy'], shell=True) - with open('constraints.txt', 'w') as outfp: - outfp.write('empy < 4\n') - outfp.write('flake8 < 5.0.0\n') - outfp.write('setuptools==59.6.0\n') - outfp.write('pytest==6.2.5\n') - - pip_cmd = ['"%s"' % job.python, '-m', 'pip', 'install', '-c', 'constraints.txt', '-U'] - if sys.platform == 'win32': - # Force reinstall so all dependencies are in virtual environment - # On Windows since we switch between the debug and non-debug - # interpreter all packages need to be reinstalled too - pip_cmd.append('--force-reinstall') - job.run( - pip_cmd + pip_packages, - shell=True) + [f'cryptography', 'lxml', 'numpy'], shell=True) + + pip_cmd = ['"%s"' % job.python, '-m', 'pip', 'install', '-c', 'constraints.txt', '-U', '--force-reinstall'] + job.run(pip_cmd + pip_packages, shell=True) - vcs_cmd = ['vcs'] + vcs_cmd = [which('vcs')] if args.colcon_branch: + colcon_packages = [ + 'colcon-argcomplete', + 'colcon-cd', + 'colcon-cmake', + 'colcon-core', + 'colcon-defaults', # Not in colcon-common-extensions + 'colcon-devtools', + 'colcon-library-path', + 'colcon-metadata', + 'colcon-mixin', # Not in colcon-common-extensions + 'colcon-output', + 'colcon-package-information', + 'colcon-package-selection', + 'colcon-parallel-executor', + 'colcon-pkg-config', # Not in colcon-common-extensions + 'colcon-powershell', + 'colcon-python-setup-py', + 'colcon-recursive-crawl', + 'colcon-ros', + 'colcon-test-result', + ] + if sys.platform != 'win32': + colcon_packages += [ + 'colcon-bash', + 'colcon-zsh', + ] + # create .repos file for colcon repositories os.makedirs('colcon', exist_ok=True) with open('colcon/colcon.repos', 'w') as h: diff --git a/windows_docker_resources/ros2-cookbooks b/windows_docker_resources/ros2-cookbooks index 8ac2ddf1c..e9e03ad6b 160000 --- a/windows_docker_resources/ros2-cookbooks +++ b/windows_docker_resources/ros2-cookbooks @@ -1 +1 @@ -Subproject commit 8ac2ddf1ceec864211971478bf3d515fbe71298e +Subproject commit e9e03ad6b7ff8f4b4b759cba708f41c7b4016b5c