Skip to content

Commit

Permalink
Pin all versions of python dependencies.
Browse files Browse the repository at this point in the history 
This allows us to be in control of our destiny,
and not have random python upstream package updates
destroy the entire buildfarm.

We accomplish this through 3 different mechanisms,
depending on the platform:

1.  Ubuntu - we install everything from debian packages,
             and nothing from pip.  Because Ubuntu promises
             to not to breaking changes within a distribution,
             this guarantees we won't break.  This also
             serves as our canonical version of what should work.
2.  RHEL - For RHEL-9, we install everything from RPM packages,
           and nothing from pip.  For RHEL-8, we install most
           everything from RPM packages, with the exception of
           mypy, pytest, and pytest-rerunfailures.  Since RHEL/EPEL
           typically doesn't update major versions within a release,
           we typically won't break.
3.  Windows - We install all python packages via pip, and we constrain
              the versions we install to those in Ubuntu.  This means
              that we'll always match what works on Ubuntu and we should
              always work.

Signed-off-by: Chris Lalancette <[email protected]>
  • Loading branch information
@clalancette
clalancette committed Mar 11, 2024
1 parent 272db18 commit 2120d5d
Show file tree
Hide file tree
Showing 3 changed files with 44 additions and 134 deletions.
2 changes: 1 addition & 1 deletion .gitmodules
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,4 +20,4 @@
[submodule "windows_docker_resources/ros2-cookbooks"]
path = windows_docker_resources/ros2-cookbooks
url = [email protected]:ros-infrastructure/ros2-cookbooks
branch = latest
branch = clalancette/windows-pip-install
174 changes: 42 additions & 132 deletions ros2_batch_job/__main__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,66 +47,6 @@
sys.stdout = UnbufferedIO(sys.stdout)
sys.stderr = UnbufferedIO(sys.stderr)

# One of the maintainers of pyparsing suggests pinning to 2.4.7 for now;
# see https://github.com/pyparsing/pyparsing/issues/323
pip_dependencies = [
'EmPy',
'coverage',
'catkin_pkg',
'flake8',
'flake8-blind-except==0.1.1',
'flake8-builtins',
'flake8-class-newline',
'flake8-comprehensions',
'flake8-deprecated',
'flake8-docstrings',
'flake8-import-order',
'flake8-quotes',
'importlib-metadata',
'mock',
'nose',
'pep8',
'pydocstyle',
'pyflakes',
'pyparsing==2.4.7',
'pytest',
'pytest-cov',
'pytest-mock',
'pytest-repeat',
'pytest-rerunfailures',
'pytest-runner',
'pyyaml',
'vcstool',
'yamllint',
]
# https://github.com/pyca/cryptography/issues/5433
pip_cryptography_version = '==3.0'

colcon_packages = [
'colcon-core',
'colcon-defaults',
'colcon-library-path',
'colcon-metadata',
'colcon-mixin',
'colcon-output',
'colcon-package-information',
'colcon-package-selection',
'colcon-parallel-executor',
'colcon-pkg-config',
'colcon-powershell',
'colcon-python-setup-py',
'colcon-recursive-crawl',
'colcon-test-result',
'colcon-cmake',
'colcon-ros',
'colcon-ros-domain-id-coordinator',
]
if sys.platform != 'win32':
colcon_packages += [
'colcon-bash',
'colcon-zsh',
]

gcov_flags = '--coverage'

colcon_space_defaults = {
Expand Down Expand Up @@ -440,91 +380,61 @@ def run(args, build_function, blacklisted_package_names=None):

# Now inside of the workspace...
with change_directory(args.workspace):
def need_package_from_pipy(pkg_name):
try:
importlib.import_module(pkg_name)
except ModuleNotFoundError:
return True

return False

print('# BEGIN SUBSECTION: install Python packages')
# Print setuptools version
job.run(['"%s"' % job.python, '-c', '"import setuptools; print(setuptools.__version__)"'],
shell=True)
# Print the pip version
job.run(['"%s"' % job.python, '-m', 'pip', '--version'], shell=True)
# Install pip dependencies
pip_packages = list(pip_dependencies)

# We prefer to get mypy from the distribution if it exists. If not we install it via pip.
if need_package_from_pipy("mypy"):
pip_packages += ["mypy==0.931"]

# We prefer to get pytest-timeout from the distribution if it exists. If not we install it via pip.
if need_package_from_pipy("pytest_timeout"):
pip_packages += ["pytest-timeout==2.1.0"]

# We prefer to get lark from the distribution if it exists. If not we install it via pip.
if need_package_from_pipy("lark"):
pip_packages += ["lark==1.1.1"]

if sys.platform == 'win32':
# Install fork of pyreadline containing fix for deprecation warnings
# TODO(jacobperron): Until upstream issue is resolved https://github.com/pyreadline/pyreadline/issues/65
pip_packages += ['git+https://github.com/osrf/pyreadline']

# Setuptools > 61 somehow have broken Windows Debug. Pin it to 59.6.0 here which
# matches Ubuntu Jammy, and wait until upstream setuptools settles down.
pip_packages += ["setuptools==59.6.0"]

if args.cmake_build_type == 'Debug':
pip_packages += [
'https://github.com/ros2/ros2/releases/download/cryptography-archives/cffi-1.14.0-cp38-cp38d-win_amd64.whl', # required by cryptography
'https://github.com/ros2/ros2/releases/download/cryptography-archives/cryptography-2.9.2-cp38-cp38d-win_amd64.whl',
'https://github.com/ros2/ros2/releases/download/lxml-archives/lxml-4.5.1-cp38-cp38d-win_amd64.whl',
'https://github.com/ros2/ros2/releases/download/numpy-archives/numpy-1.18.4-cp38-cp38d-win_amd64.whl',
'https://github.com/ros2/ros2/releases/download/psutil-archives/psutil-5.9.5-cp38-cp38d-win_amd64.whl',
]
if args.ros_distro in ('humble', 'iron'):
pip_packages.append('https://github.com/ros2/ros2/releases/download/netifaces-archives/netifaces-0.10.9-cp38-cp38d-win_amd64.whl')
else:
pip_packages += [
f'cryptography{pip_cryptography_version}',
'lxml',
'numpy',
]
if args.ros_distro in ('humble', 'iron'):
pip_packages.append('netifaces')
if not args.colcon_branch:
pip_packages += colcon_packages
if sys.platform == 'win32':
job.run(
['"%s"' % job.python, '-m', 'pip', 'uninstall', '-y'] +
colcon_packages, shell=True)
if sys.platform == 'win32' and args.cmake_build_type == 'Debug':
pip_packages = [
'https://github.com/ros2/ros2/releases/download/cryptography-archives/cffi-1.14.0-cp38-cp38d-win_amd64.whl', # required by cryptography
'https://github.com/ros2/ros2/releases/download/cryptography-archives/cryptography-2.9.2-cp38-cp38d-win_amd64.whl',
'https://github.com/ros2/ros2/releases/download/lxml-archives/lxml-4.5.1-cp38-cp38d-win_amd64.whl',
'https://github.com/ros2/ros2/releases/download/numpy-archives/numpy-1.18.4-cp38-cp38d-win_amd64.whl',
'https://github.com/ros2/ros2/releases/download/psutil-archives/psutil-5.9.5-cp38-cp38d-win_amd64.whl',
]
if args.ros_distro in ('humble', 'iron'):
pip_packages.append('https://github.com/ros2/ros2/releases/download/netifaces-archives/netifaces-0.10.9-cp38-cp38d-win_amd64.whl')
# to ensure that the build type specific package is installed
job.run(
['"%s"' % job.python, '-m', 'pip', 'uninstall', '-y'] +
[f'cryptography{pip_cryptography_version}', 'lxml', 'numpy'], shell=True)
with open('constraints.txt', 'w') as outfp:
outfp.write('empy < 4\n')
outfp.write('flake8 < 5.0.0\n')
outfp.write('setuptools==59.6.0\n')
outfp.write('pytest==6.2.5\n')

pip_cmd = ['"%s"' % job.python, '-m', 'pip', 'install', '-c', 'constraints.txt', '-U']
if sys.platform == 'win32':
# Force reinstall so all dependencies are in virtual environment
# On Windows since we switch between the debug and non-debug
# interpreter all packages need to be reinstalled too
pip_cmd.append('--force-reinstall')
job.run(
pip_cmd + pip_packages,
shell=True)
[f'cryptography', 'lxml', 'numpy'], shell=True)

pip_cmd = ['"%s"' % job.python, '-m', 'pip', 'install', '-c', 'constraints.txt', '-U', '--force-reinstall']
job.run(pip_cmd + pip_packages, shell=True)

vcs_cmd = ['vcs']
vcs_cmd = [which('vcs')]

if args.colcon_branch:
colcon_packages = [
'colcon-argcomplete',
'colcon-cd',
'colcon-cmake',
'colcon-core',
'colcon-defaults', # Not in colcon-common-extensions
'colcon-devtools',
'colcon-library-path',
'colcon-metadata',
'colcon-mixin', # Not in colcon-common-extensions
'colcon-output',
'colcon-package-information',
'colcon-package-selection',
'colcon-parallel-executor',
'colcon-pkg-config', # Not in colcon-common-extensions
'colcon-powershell',
'colcon-python-setup-py',
'colcon-recursive-crawl',
'colcon-ros',
'colcon-test-result',
]
if sys.platform != 'win32':
colcon_packages += [
'colcon-bash',
'colcon-zsh',
]

# create .repos file for colcon repositories
os.makedirs('colcon', exist_ok=True)
with open('colcon/colcon.repos', 'w') as h:
Expand Down
2 changes: 1 addition & 1 deletion windows_docker_resources/ros2-cookbooks
Submodule ros2-cookbooks updated 1 files
+149 −58 cookbooks/ros2_windows/recipes/pip_installs.rb

0 comments on commit 2120d5d

Please sign in to comment.