Merge pull request #208 from mvdbeek/fix_ci

Fix CI tests

.github/workflows/ci.yaml

@@ -1,27 +1,25 @@
 name: Python CI
-on: [pull_request]
+on: [pull_request, push]
 jobs:
   test:
     runs-on: ubuntu-latest
     timeout-minutes: 90
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.7']
+        python-version: ['3.8']
         tox-action:
           - lint
           - pytest
-          - integration
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 1
-      - uses: actions/setup-python@v1
+      - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
           pip install tox coveralls
       - name: Test with tox
         run: |
@@ -31,11 +29,11 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 1
           submodules: true
-      - uses: actions/setup-python@v1
+      - uses: actions/setup-python@v4
         with:
           python-version: '3.7'
       - name: Install dependencies

dev-requirements.txt

@@ -3,7 +3,7 @@ tox
 coverage
 pytest
 pytest-cov
-docker
+galaxy-test-driver
 
 #Building Docs
 sphinx_rtd_theme

setup.cfg

@@ -1,15 +1,6 @@
 [wheel]
 universal = 1
 
-[nosetests]
-verbosity=1
-detailed-errors=1
-with-doctest=1
-with-coverage=1
-logging-filter=ephemeris
-logging-level=INFO
-
-
 [flake8]
 max-line-length = 150
 max-complexity = 30

src/ephemeris/common_parser.py

@@ -2,6 +2,8 @@
 
 import argparse
 
+DEFAULT_JOB_SLEEP = 3
+
 
 class HideUnderscoresHelpFormatter(argparse.HelpFormatter):
     def add_arguments(self, actions):

src/ephemeris/generate_tool_list_from_ga_workflow_files.py

@@ -18,14 +18,6 @@
 INSTALL_RESOLVER_DEPENDENCIES = "install_resolver_dependencies: True"
 
 
-def _parse_cli_options():
-    """
-    Parse command line options, returning `parse_args` from `ArgumentParser`.
-    """
-    parser = _parser()
-    return parser.parse_args()
-
-
 def _parser():
     parser = ArgumentParser(
         formatter_class=RawDescriptionHideUnderscoresHelpFormatter,
@@ -164,8 +156,8 @@ def generate_tool_list_from_workflow(
     print_yaml_tool_list(convert_dict, output_file)
 
 
-def main():
-    options = _parse_cli_options()
+def main(argv=None):
+    options = _parser().parse_args(argv)
     generate_tool_list_from_workflow(
         options.workflow_files, options.panel_label, options.output_file
     )

src/ephemeris/get_tool_list_from_galaxy.py

@@ -311,22 +311,14 @@ def get_repo_from_tool(tool):
     return repo
 
 
-def _parse_cli_options():
-    """
-    Parse command line options, returning `parse_args` from `ArgumentParser`.
-    """
-    parser = _parser()
-    return parser.parse_args()
-
-
 def check_galaxy_version(gi):
     version = gi.config.get_version()
     if StrictVersion(version["version_major"]) < StrictVersion("16.04"):
         raise Exception("This script needs galaxy version 16.04 or newer")
 
 
-def main():
-    options = _parse_cli_options()
+def main(argv=None):
+    options = _parser().parse_args(argv)
     gi = get_galaxy_connection(options, login_required=False)
     check_galaxy_version(gi)
     gi_to_tool_yaml = GiToToolYaml(

src/ephemeris/install_tool_deps.py

@@ -44,11 +44,11 @@ def _install(tool_client, tool_id):
             raise
 
 
-def main():
+def main(argv=None):
     """
     This script uses bioblend to trigger dependencies installations for the provided tools
     """
-    args = _parser().parse_args()
+    args = _parser().parse_args(argv)
     gi = get_galaxy_connection(args)
     tool_client = ToolClient(gi)
 

src/ephemeris/run_data_managers.py

@@ -38,6 +38,7 @@
     load_yaml_file,
 )
 from .common_parser import (
+    DEFAULT_JOB_SLEEP,
     get_common_args,
     HideUnderscoresHelpFormatter,
 )
@@ -94,7 +95,7 @@ def wait(gi, job_list, log):
             job_list.remove(finished_job)
         # only sleep if job_list is not empty yet.
         if bool(job_list):
-            time.sleep(30)
+            time.sleep(DEFAULT_JOB_SLEEP)
     return successful_jobs, failed_jobs
 
 
@@ -346,10 +347,10 @@ def _parser():
     return parser
 
 
-def main():
+def main(argv=None):
     disable_external_library_logging()
     parser = _parser()
-    args = parser.parse_args()
+    args = parser.parse_args(argv)
     log = setup_global_logger(name=__name__, log_file=args.log_file)
     if args.verbose:
         log.setLevel(logging.DEBUG)

src/ephemeris/set_library_permissions.py

@@ -103,8 +103,8 @@ def _parser():
     return parser
 
 
-def main():
-    args = _parser().parse_args()
+def main(argv=None):
+    args = _parser().parse_args(argv)
     if args.user and args.password:
         gi = galaxy.GalaxyInstance(
             url=args.galaxy, email=args.user, password=args.password

src/ephemeris/setup_data_libraries.py

@@ -9,6 +9,7 @@
 from bioblend import galaxy
 
 from .common_parser import (
+    DEFAULT_JOB_SLEEP,
     get_common_args,
     HideUnderscoresHelpFormatter,
 )
@@ -205,7 +206,7 @@ def normalize_items(has_items):
 
             if all(job_states):
                 break
-            time.sleep(3)
+            time.sleep(DEFAULT_JOB_SLEEP)
 
         log.info("Finished importing test data.")
 
@@ -234,8 +235,8 @@ def _parser():
     return parser
 
 
-def main():
-    args = _parser().parse_args()
+def main(argv=None):
+    args = _parser().parse_args(argv)
     if args.user and args.password:
         gi = galaxy.GalaxyInstance(
             url=args.galaxy, email=args.user, password=args.password

src/ephemeris/shed_tools.py

@@ -741,9 +741,9 @@ def args_to_repos(args) -> List[InstallRepoDict]:
     return repos
 
 
-def main():
+def main(argv=None):
     disable_external_library_logging()
-    args = parser().parse_args()
+    args = parser().parse_args(argv)
     log = setup_global_logger(name=__name__, log_file=args.log_file)
     gi = get_galaxy_connection(
         args, file=args.tool_list_file, log=log, login_required=True

src/ephemeris/sleep.py

@@ -54,14 +54,6 @@ def _parser():
     return parser
 
 
-def _parse_cli_options():
-    """
-    Parse command line options, returning `parse_args` from `ArgumentParser`.
-    """
-    parser = _parser()
-    return parser.parse_args()
-
-
 class SleepCondition(object):
     def __init__(self):
         self.sleep = True
@@ -171,11 +163,11 @@ def galaxy_wait(
     return True
 
 
-def main():
+def main(argv=None):
     """
     Main function
     """
-    options = _parse_cli_options()
+    options = _parser().parse_args(argv)
 
     galaxy_alive = galaxy_wait(
         galaxy_url=options.galaxy,

src/ephemeris/workflow_install.py

@@ -45,11 +45,11 @@ def _parser():
     return parser
 
 
-def main():
+def main(argv=None):
     """
     This script uses bioblend to import .ga workflow files into a running instance of Galaxy
     """
-    args = _parser().parse_args()
+    args = _parser().parse_args(argv)
     gi = get_galaxy_connection(args)
 
     if os.path.isdir(args.workflow_path):

tests/conftest.py

@@ -1,65 +1,58 @@
 from collections import namedtuple
 
-import docker
 import pytest
 from bioblend.galaxy import GalaxyInstance
-
-from ephemeris.sleep import galaxy_wait
-
-# It needs to work well with dev. Alternatively we can pin this to 'master' or another stable branch.
-# Preferably a branch that updates with each stable release
-GALAXY_IMAGE = "bgruening/galaxy-stable:20.05"
-GALAXY_ADMIN_KEY = "fakekey"
-GALAXY_ADMIN_PASSWORD = "password"
-GALAXY_ADMIN_USER = "[email protected]"
-
-client = docker.from_env()
+from galaxy.tool_util.verify.interactor import GalaxyInteractorApi
+from galaxy_test.driver.driver_util import GalaxyTestDriver
 
 GalaxyContainer = namedtuple(
-    "GalaxyContainer", ["url", "container", "attributes", "gi"]
+    "GalaxyContainer", ["url", "gi", "password", "username", "api_key"]
 )
 
 
-# Class scope is chosen here so we can group tests on the same galaxy in a class.
 @pytest.fixture(scope="class")
-def start_container(**kwargs):
-    """Starts a docker container with the galaxy image. Returns a named tuple with the url, a GalaxyInstance object,
-    the container attributes, and the container itself."""
-    # We start a container from the galaxy image. We detach it. Port 80 is exposed to the host at a random port.
-    # The random port is because we need mac compatibility. On GNU/linux a better option would be not to expose it
-    # and use the internal ip address instead.
-    # But alas, the trappings of a proprietary BSD kernel compel us to do ugly workarounds.
-    key = kwargs.get("api_key", GALAXY_ADMIN_KEY)
-    ensure_admin = kwargs.get("ensure_admin", True)
-
-    container = client.containers.run(
-        GALAXY_IMAGE, detach=True, ports={"80/tcp": None}, **kwargs
-    )
-    container_id = container.attrs.get("Id")
-    print(container_id)
-
-    # This seems weird as we also can just get container.attrs but for some reason
-    # the network settings are not loaded in container.attrs. With the get request
-    # these attributes are loaded
-    container_attributes = client.containers.get(container_id).attrs
-
-    # Venturing into deep nested dictionaries.
-    exposed_port = (
-        container_attributes.get("NetworkSettings")
-        .get("Ports")
-        .get("80/tcp")[0]
-        .get("HostPort")
-    )
-
-    container_url = "http://localhost:{0}".format(exposed_port)
-    assert key
-    ready = galaxy_wait(
-        container_url, timeout=180, api_key=key, ensure_admin=ensure_admin
-    )
-    if not ready:
-        raise Exception("Failed to wait on Galaxy to start.")
-    gi = GalaxyInstance(container_url, key=key)
-    yield GalaxyContainer(
-        url=container_url, container=container, attributes=container_attributes, gi=gi
+def start_container(tmpdir_factory):
+    config_dir = tmpdir_factory.mktemp("config")
+    database = config_dir.join("universe.sqlite")
+    test_driver = GalaxyTestDriver()
+    test_driver.galaxy_config = {
+        "job_config": {
+            "runners": {
+                "local": {
+                    "load": "galaxy.jobs.runners.local:LocalJobRunner",
+                    "workers": 1,
+                }
+            },
+            "execution": {
+                "default": "local_docker",
+                "environments": {
+                    "local_docker": {"runner": "local", "docker_enabled": True},
+                },
+            },
+        },
+        "admin_users": "[email protected]",
+        "bootstrap_admin_api_key": "123456789",
+        "conda_auto_init": False,
+        "config_dir": str(config_dir),
+        "database_connection": f"sqlite:///{database}?isolation_level=IMMEDIATE",
+    }
+    test_driver.setup()
+    server_wrapper = test_driver.server_wrappers[0]
+    host = server_wrapper.host
+    port = server_wrapper.port
+    prefix = server_wrapper.prefix or ""
+    url = f"http://{host}:{port}{prefix.rstrip('/')}/"
+    interactor = GalaxyInteractorApi(
+        galaxy_url=url, master_api_key="123456789", test_user="[email protected]"
     )
-    container.remove(force=True)
+    gi = GalaxyInstance(url, key=interactor.api_key, password="testpass")
+    try:
+        yield GalaxyContainer(
+            url=url,
+            gi=gi,
+            password="testpass",
+            username="[email protected]",
+            api_key=interactor.api_key,
+        )
+    finally:
+        test_driver.stop_servers()