From 92b5488cc0de57e0528be43d9f6f4ceea7b3d9af Mon Sep 17 00:00:00 2001 From: Lumir Balhar Date: Mon, 30 Nov 2020 09:08:19 +0100 Subject: [PATCH] Update from the upstream Github repository --- README.md | 108 +++++++++++-- test/check_imagestreams.py | 73 +++++++++ test/from-dockerfile/Dockerfile.tpl | 14 ++ test/imagestreams/python-centos.json | 153 ++++++++++++++++++ test/imagestreams/python-rhel-aarch64.json | 93 +++++++++++ test/imagestreams/python-rhel.json | 173 +++++++++++++++++++++ test/npm-virtualenv-uwsgi-test-app/app.sh | 2 +- test/run | 22 +++ test/run-openshift | 12 ++ test/test-lib-openshift.sh | 80 ++++++---- test/test-lib-python.sh | 2 +- test/test-lib.sh | 132 +++++++++++++++- 12 files changed, 809 insertions(+), 55 deletions(-) create mode 100755 test/check_imagestreams.py create mode 100644 test/from-dockerfile/Dockerfile.tpl create mode 100644 test/imagestreams/python-centos.json create mode 100644 test/imagestreams/python-rhel-aarch64.json create mode 100644 test/imagestreams/python-rhel.json diff --git a/README.md b/README.md index 14b66a7..16fffe3 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ Python 3.8 container image -=================== +========================= This container image includes Python 3.8 as a [S2I](https://github.com/openshift/source-to-image) base image for your Python 3.8 applications. Users can choose between RHEL and CentOS based builder images. The RHEL images are available in the [Red Hat Container Catalog](https://access.redhat.com/containers/), -the CentOS images are available on [Docker Hub](https://hub.docker.com/r/centos/), +the CentOS images are available on [Quay.io](https://quay.io/organization/centos7), and the Fedora images are available in [Fedora Registry](https://registry.fedoraproject.org/). The resulting image can be run using [podman](https://github.com/containers/libpod) or [docker](http://docker.io). @@ -29,28 +29,104 @@ modules for their web applications. There is no guarantee for any specific npm o version, that is included in the image; those versions can be changed anytime and the nodejs itself is included just to make the npm work. -Usage ---------------------- +Usage in Openshift +------------------ For this, we will assume that you are using the supported image, available via `python:3.8` imagestream tag in Openshift. -Building a simple [python-sample-app](https://github.com/sclorg/s2i-python-container/tree/master/3.8/test/setup-test-app) application +Building a simple [python-sample-app](https://github.com/sclorg/django-ex.git) application in Openshift can be achieved with the following step: ``` - oc new-app python:3.8~https://github.com/sclorg/s2i-python-container.git --context-dir=3.8/test/setup-test-app/ + oc new-app python:3.8~https://github.com/sclorg/django-ex.git ``` -The same application can also be built using the standalone [S2I](https://github.com/openshift/source-to-image) application on systems that have it available: - - ``` - $ s2i build https://github.com/sclorg/s2i-python-container.git --context-dir=3.8/test/setup-test-app/ python-sample-app - ``` - -Where `` is the s2i-python image you [downloaded from RHEL, Centos or Fedora registry](../README.md#Download) or [built](../README.md#Build) from these sources. For example ubi8/python-36, centos/python-36-centos7 or f31/python3. - **Accessing the application:** ``` -$ curl 127.0.0.1:8080 +$ oc get pods +$ oc exec -- curl 127.0.0.1:8080 +``` + +Source-to-Image framework and scripts +------------------------------------- +This image supports the [Source-to-Image](https://docs.openshift.com/container-platform/3.11/creating_images/s2i.html) +(S2I) strategy in OpenShift. The Source-to-Image is an OpenShift framework +which makes it easy to write images that take application source code as +an input, use a builder image like this Python container image, and produce +a new image that runs the assembled application as an output. + +To support the Source-to-Image framework, important scripts are included in the builder image: + +* The `/usr/libexec/s2i/assemble` script inside the image is run to produce a new image with the application artifacts. +The script takes sources of a given application and places them into appropriate directories inside the image. +It utilizes some common patterns in Perl application development (see the **Environment variables** section below). +* The `/usr/libexec/s2i/run` script is set as the default command in the resulting container image (the new image with the application artifacts). +It runs your application according to settings in `APP_MODULE`, `APP_FILE` or `APP_SCRIPT` environment variables or it tries to detect the best +way automatically. + +Building an application using a Dockerfile +------------------------------------------ +Compared to the Source-to-Image strategy, using a Dockerfile is a more +flexible way to build a Python container image with an application. +Use a Dockerfile when Source-to-Image is not sufficiently flexible for you or +when you build the image outside of the OpenShift environment. + +To use the Python image in a Dockerfile, follow these steps: +#### 1. Pull a base builder image to build on + +``` +podman pull registry.access.redhat.com/ubi8/python-38 +``` + +#### 2. Pull and application code + +An example application available at https://github.com/sclorg/django-ex.git is used here. Feel free to clone the repository for further experiments. +You can also take a look at code examples in s2i-python-container repository: https://github.com/sclorg/s2i-python-container/tree/master/examples + +``` +git clone https://github.com/sclorg/django-ex.git app-src +``` + +#### 3. Prepare an application inside a container + +This step usually consists of at least these parts: + +* putting the application source into the container +* installing the dependencies +* setting the default command in the resulting image + +For all these three parts, users can either setup all manually and use commands `python` and `pip` explicitly in the Dockerfile, +or users can use the Source-to-Image scripts inside the image. + +The manual way comes with the highest level of flexibility but requires you to know how to work +with modules or software collections manually, how to setup virtual environment with the right version +of Python and many more. On the other hand, using Source-to-Image scripts makes your Dockerfile +prepared for a future flawless switch to a newer or different platform. + +To use the Source-to-Image scripts and build an image using a Dockerfile, create a Dockerfile with this content: + +``` +FROM registry.access.redhat.com/ubi8/python-38 +# Add application sources to a directory that the assemble script expects them +# and set permissions so that the container runs without root access +USER 0 +ADD app-src /tmp/src +RUN chown -R 1001:0 /tmp/src +USER 1001 +# Install the dependencies +RUN /usr/libexec/s2i/assemble +# Set the default command for the resulting image +CMD /usr/libexec/s2i/run +``` +#### 4. Build a new image from a Dockerfile prepared in the previous step + +``` +podman build -t python-app . +``` + +#### 5. Run the resulting image with final application + +``` +podman run -d python-app ``` Environment variables @@ -244,7 +320,7 @@ following ways, in precedence order: application. Hot deploy ---------------------- +---------- If you are using Django, hot deploy will work out of the box. diff --git a/test/check_imagestreams.py b/test/check_imagestreams.py new file mode 100755 index 0000000..6da0f63 --- /dev/null +++ b/test/check_imagestreams.py @@ -0,0 +1,73 @@ +#!/bin/env python3 + +import sys +import json +import logging +import os + +from pathlib import Path +from typing import Dict + +IMAGESTREAMS_DIR: str = "imagestreams" + + +class ImageStreamChecker(object): + version: str = "" + results: Dict = {} + + def __init__(self, version: str): + self.version = version + + def load_json_file(self, filename: Path): + with open(str(filename)) as f: + return json.load(f) + + def check_version(self, json_dict: Dict): + res = [] + for tags in json_dict["spec"]["tags"]: + # The name can be"" or "-elX" or "-ubiX" + if (tags["name"] == self.version or + tags["name"].startswith(self.version + '-')): + res.append(tags) + return res + + def check_latest_tag(self, json_dict: Dict): + latest_tag_correct: bool = False + for tags in json_dict["spec"]["tags"]: + if tags["name"] != "latest": + continue + # The latest can link to either "" or "-elX" or "-ubiX" + if tags["from"]["name"] == self.version or tags["from"]["name"].startswith(self.version + '-'): + latest_tag_correct = True + return latest_tag_correct + + def check_imagestreams(self): + p = Path(".") + json_files = p.glob(f"{IMAGESTREAMS_DIR}/*.json") + if not json_files: + print(f"No json files present in {IMAGESTREAMS_DIR}.") + return 0 + for f in json_files: + if os.environ.get("TARGET") in ("rhel7", "centos7") and "aarch64" in str(f): + print("Imagestream aarch64 is not supported on rhel7") + continue + print(f"Checking file {str(f)}.") + json_dict = self.load_json_file(f) + if not (self.check_version(json_dict) and self.check_latest_tag(json_dict)): + print(f"The latest version is not present in {str(f)} or in latest tag.") + self.results[f] = False + if self.results: + return 1 + print("Imagestreams contains the latest version.") + return 0 + + +if __name__ == "__main__": + if len(sys.argv) != 2: + logging.fatal("%s: %s", sys.argv[0], "VERSION as an argument was not provided") + sys.exit(1) + + print(f"Version to check is {sys.argv[1]}.") + isc = ImageStreamChecker(version=sys.argv[1]) + sys.exit(isc.check_imagestreams()) + diff --git a/test/from-dockerfile/Dockerfile.tpl b/test/from-dockerfile/Dockerfile.tpl new file mode 100644 index 0000000..2a12b8c --- /dev/null +++ b/test/from-dockerfile/Dockerfile.tpl @@ -0,0 +1,14 @@ +FROM #IMAGE_NAME# # Replaced by sed in tests, see test_from_dockerfile in test/run + +# Add application sources to a directory that the assemble script expects them +# and set permissions so that the container runs without root access +USER 0 +ADD app-src /tmp/src +RUN chown -R 1001:0 /tmp/src +USER 1001 + +# Install the dependencies +RUN /usr/libexec/s2i/assemble + +# Set the default command for the resulting image +CMD /usr/libexec/s2i/run diff --git a/test/imagestreams/python-centos.json b/test/imagestreams/python-centos.json new file mode 100644 index 0000000..0f6aeae --- /dev/null +++ b/test/imagestreams/python-centos.json @@ -0,0 +1,153 @@ +{ + "kind": "ImageStream", + "apiVersion": "v1", + "metadata": { + "name": "python", + "annotations": { + "openshift.io/display-name": "Python" + } + }, + "spec": { + "tags": [ + { + "name": "latest", + "annotations": { + "openshift.io/display-name": "Python (Latest)", + "openshift.io/provider-display-name": "Red Hat, Inc.", + "description": "Build and run Python applications on UBI. For more information about using this builder image, including OpenShift considerations, see https://github.com/sclorg/s2i-python-container/blob/master/3.8/README.md.\n\nWARNING: By selecting this tag, your application will automatically update to use the latest version of Python available on OpenShift, including major version updates.", + "iconClass": "icon-python", + "tags": "builder,python", + "supports":"python", + "sampleRepo": "https://github.com/sclorg/django-ex.git" + }, + "from": { + "kind": "ImageStreamTag", + "name": "3.8-ubi8" + }, + "referencePolicy": { + "type": "Local" + } + }, + { + "name": "3.8-ubi8", + "annotations": { + "openshift.io/display-name": "Python 3.8 (UBI 8)", + "openshift.io/provider-display-name": "Red Hat, Inc.", + "description": "Build and run Python 3.8 applications on UBI 8. For more information about using this builder image, including OpenShift considerations, see https://github.com/sclorg/s2i-python-container/blob/master/3.8/README.md.", + "iconClass": "icon-python", + "tags": "builder,python", + "supports":"python:3.8,python", + "version": "3.8", + "sampleRepo": "https://github.com/sclorg/django-ex.git" + }, + "from": { + "kind": "DockerImage", + "name": "registry.access.redhat.com/ubi8/python-38:latest" + }, + "referencePolicy": { + "type": "Local" + } + }, + { + "name": "3.8-ubi7", + "annotations": { + "openshift.io/display-name": "Python 3.8 (UBI 7)", + "openshift.io/provider-display-name": "Red Hat, Inc.", + "description": "Build and run Python 3.8 applications on UBI 7. For more information about using this builder image, including OpenShift considerations, see https://github.com/sclorg/s2i-python-container/blob/master/3.8/README.md.", + "iconClass": "icon-python", + "tags": "builder,python", + "supports":"python:3.8,python", + "version": "3.8", + "sampleRepo": "https://github.com/sclorg/django-ex.git" + }, + "from": { + "kind": "DockerImage", + "name": "registry.access.redhat.com/ubi7/python-38:latest" + }, + "referencePolicy": { + "type": "Local" + } + }, + { + "name": "3.6-ubi8", + "annotations": { + "openshift.io/display-name": "Python 3.6 (UBI 8)", + "openshift.io/provider-display-name": "Red Hat, Inc.", + "description": "Build and run Python 3.6 applications on UBI 8. For more information about using this builder image, including OpenShift considerations, see https://github.com/sclorg/s2i-python-container/blob/master/3.6/README.md.", + "iconClass": "icon-python", + "tags": "builder,python", + "supports":"python:3.6,python", + "version": "3.6", + "sampleRepo": "https://github.com/sclorg/django-ex.git" + }, + "from": { + "kind": "DockerImage", + "name": "registry.access.redhat.com/ubi8/python-36:latest" + }, + "referencePolicy": { + "type": "Local" + } + }, + { + "name": "2.7-ubi8", + "annotations": { + "openshift.io/display-name": "Python 2.7 (UBI 8)", + "openshift.io/provider-display-name": "Red Hat, Inc.", + "description": "Build and run Python 2.7 applications on UBI 8. For more information about using this builder image, including OpenShift considerations, see https://github.com/sclorg/s2i-python-container/blob/master/2.7/README.md.", + "iconClass": "icon-python", + "tags": "builder,python", + "supports":"python:2.7,python", + "version": "2.7", + "sampleRepo": "https://github.com/sclorg/django-ex.git" + }, + "from": { + "kind": "DockerImage", + "name": "registry.access.redhat.com/ubi8/python-27:latest" + }, + "referencePolicy": { + "type": "Local" + } + }, + { + "name": "2.7-ubi7", + "annotations": { + "openshift.io/display-name": "Python 2.7 (UBI 7)", + "openshift.io/provider-display-name": "Red Hat, Inc.", + "description": "Build and run Python 2.7 applications on UBI 7. For more information about using this builder image, including OpenShift considerations, see https://github.com/sclorg/s2i-python-container/blob/master/2.7/README.md.", + "iconClass": "icon-python", + "tags": "builder,python", + "supports":"python:2.7,python", + "version": "2.7", + "sampleRepo": "https://github.com/sclorg/django-ex.git" + }, + "from": { + "kind": "DockerImage", + "name": "registry.access.redhat.com/ubi7/python-27:latest" + }, + "referencePolicy": { + "type": "Local" + } + }, + { + "name": "2.7", + "annotations": { + "openshift.io/display-name": "Python 2.7", + "openshift.io/provider-display-name": "Red Hat, Inc.", + "description": "Build and run Python 2.7 applications on CentOS 7. For more information about using this builder image, including OpenShift considerations, see https://github.com/sclorg/s2i-python-container/blob/master/2.7/README.md.", + "iconClass": "icon-python", + "tags": "builder,python,hidden", + "supports":"python:2.7,python", + "version": "2.7", + "sampleRepo": "https://github.com/sclorg/django-ex.git" + }, + "from": { + "kind": "DockerImage", + "name": "docker.io/centos/python-27-centos7:latest" + }, + "referencePolicy": { + "type": "Local" + } + } + ] + } +} diff --git a/test/imagestreams/python-rhel-aarch64.json b/test/imagestreams/python-rhel-aarch64.json new file mode 100644 index 0000000..1c2ed27 --- /dev/null +++ b/test/imagestreams/python-rhel-aarch64.json @@ -0,0 +1,93 @@ +{ + "kind": "ImageStream", + "apiVersion": "v1", + "metadata": { + "name": "python", + "annotations": { + "openshift.io/display-name": "Python" + } + }, + "spec": { + "tags": [ + { + "name": "latest", + "annotations": { + "openshift.io/display-name": "Python (Latest)", + "openshift.io/provider-display-name": "Red Hat, Inc.", + "description": "Build and run Python applications on UBI. For more information about using this builder image, including OpenShift considerations, see https://github.com/sclorg/s2i-python-container/blob/master/3.8/README.md.\n\nWARNING: By selecting this tag, your application will automatically update to use the latest version of Python available on OpenShift, including major version updates.", + "iconClass": "icon-python", + "tags": "builder,python", + "supports":"python", + "sampleRepo": "https://github.com/sclorg/django-ex.git" + }, + "from": { + "kind": "ImageStreamTag", + "name": "3.8-ubi8" + }, + "referencePolicy": { + "type": "Local" + } + }, + { + "name": "3.8-ubi8", + "annotations": { + "openshift.io/display-name": "Python 3.8 (UBI 8)", + "openshift.io/provider-display-name": "Red Hat, Inc.", + "description": "Build and run Python 3.8 applications on UBI 8. For more information about using this builder image, including OpenShift considerations, see https://github.com/sclorg/s2i-python-container/blob/master/3.8/README.md.", + "iconClass": "icon-python", + "tags": "builder,python", + "supports":"python:3.8,python", + "version": "3.8", + "sampleRepo": "https://github.com/sclorg/django-ex.git" + }, + "from": { + "kind": "DockerImage", + "name": "registry.redhat.io/ubi8/python-38:latest" + }, + "referencePolicy": { + "type": "Local" + } + }, + { + "name": "3.6-ubi8", + "annotations": { + "openshift.io/display-name": "Python 3.6 (UBI 8)", + "openshift.io/provider-display-name": "Red Hat, Inc.", + "description": "Build and run Python 3.6 applications on UBI 8. For more information about using this builder image, including OpenShift considerations, see https://github.com/sclorg/s2i-python-container/blob/master/3.6/README.md.", + "iconClass": "icon-python", + "tags": "builder,python", + "supports":"python:3.6,python", + "version": "3.6", + "sampleRepo": "https://github.com/sclorg/django-ex.git" + }, + "from": { + "kind": "DockerImage", + "name": "registry.redhat.io/ubi8/python-36:latest" + }, + "referencePolicy": { + "type": "Local" + } + }, + { + "name": "2.7-ubi8", + "annotations": { + "openshift.io/display-name": "Python 2.7 (UBI 8)", + "openshift.io/provider-display-name": "Red Hat, Inc.", + "description": "Build and run Python 2.7 applications on UBI 8. For more information about using this builder image, including OpenShift considerations, see https://github.com/sclorg/s2i-python-container/blob/master/2.7/README.md.", + "iconClass": "icon-python", + "tags": "builder,python", + "supports":"python:2.7,python", + "version": "2.7", + "sampleRepo": "https://github.com/sclorg/django-ex.git" + }, + "from": { + "kind": "DockerImage", + "name": "registry.redhat.io/ubi8/python-27:latest" + }, + "referencePolicy": { + "type": "Local" + } + } + ] + } +} diff --git a/test/imagestreams/python-rhel.json b/test/imagestreams/python-rhel.json new file mode 100644 index 0000000..ce2e080 --- /dev/null +++ b/test/imagestreams/python-rhel.json @@ -0,0 +1,173 @@ +{ + "kind": "ImageStream", + "apiVersion": "v1", + "metadata": { + "name": "python", + "annotations": { + "openshift.io/display-name": "Python" + } + }, + "spec": { + "tags": [ + { + "name": "latest", + "annotations": { + "openshift.io/display-name": "Python (Latest)", + "openshift.io/provider-display-name": "Red Hat, Inc.", + "description": "Build and run Python applications on UBI. For more information about using this builder image, including OpenShift considerations, see https://github.com/sclorg/s2i-python-container/blob/master/3.8/README.md.\n\nWARNING: By selecting this tag, your application will automatically update to use the latest version of Python available on OpenShift, including major version updates.", + "iconClass": "icon-python", + "tags": "builder,python", + "supports":"python", + "sampleRepo": "https://github.com/sclorg/django-ex.git" + }, + "from": { + "kind": "ImageStreamTag", + "name": "3.8-ubi8" + }, + "referencePolicy": { + "type": "Local" + } + }, + { + "name": "3.8-ubi8", + "annotations": { + "openshift.io/display-name": "Python 3.8 (UBI 8)", + "openshift.io/provider-display-name": "Red Hat, Inc.", + "description": "Build and run Python 3.8 applications on UBI 8. For more information about using this builder image, including OpenShift considerations, see https://github.com/sclorg/s2i-python-container/blob/master/3.8/README.md.", + "iconClass": "icon-python", + "tags": "builder,python", + "supports":"python:3.8,python", + "version": "3.8", + "sampleRepo": "https://github.com/sclorg/django-ex.git" + }, + "from": { + "kind": "DockerImage", + "name": "registry.redhat.io/ubi8/python-38:latest" + }, + "referencePolicy": { + "type": "Local" + } + }, + { + "name": "3.8-ubi7", + "annotations": { + "openshift.io/display-name": "Python 3.8 (UBI 7)", + "openshift.io/provider-display-name": "Red Hat, Inc.", + "description": "Build and run Python 3.8 applications on UBI 7. For more information about using this builder image, including OpenShift considerations, see https://github.com/sclorg/s2i-python-container/blob/master/3.8/README.md.", + "iconClass": "icon-python", + "tags": "builder,python", + "supports":"python:3.8,python", + "version": "3.8", + "sampleRepo": "https://github.com/sclorg/django-ex.git" + }, + "from": { + "kind": "DockerImage", + "name": "registry.redhat.io/ubi7/python-38:latest" + }, + "referencePolicy": { + "type": "Local" + } + }, + { + "name": "3.8", + "annotations": { + "openshift.io/display-name": "Python 3.8", + "openshift.io/provider-display-name": "Red Hat, Inc.", + "description": "Build and run Python 3.8 applications on RHEL 7. For more information about using this builder image, including OpenShift considerations, see https://github.com/sclorg/s2i-python-container/blob/master/3.8/README.md.", + "iconClass": "icon-python", + "tags": "builder,python,hidden", + "supports":"python:3.8,python", + "version": "3.8", + "sampleRepo": "https://github.com/sclorg/django-ex.git" + }, + "from": { + "kind": "DockerImage", + "name": "registry.redhat.io/rhscl/python-38-rhel7:latest" + }, + "referencePolicy": { + "type": "Local" + } + }, + { + "name": "3.6-ubi8", + "annotations": { + "openshift.io/display-name": "Python 3.6 (UBI 8)", + "openshift.io/provider-display-name": "Red Hat, Inc.", + "description": "Build and run Python 3.6 applications on UBI 8. For more information about using this builder image, including OpenShift considerations, see https://github.com/sclorg/s2i-python-container/blob/master/3.6/README.md.", + "iconClass": "icon-python", + "tags": "builder,python", + "supports":"python:3.6,python", + "version": "3.6", + "sampleRepo": "https://github.com/sclorg/django-ex.git" + }, + "from": { + "kind": "DockerImage", + "name": "registry.redhat.io/ubi8/python-36:latest" + }, + "referencePolicy": { + "type": "Local" + } + }, + { + "name": "2.7-ubi8", + "annotations": { + "openshift.io/display-name": "Python 2.7 (UBI 8)", + "openshift.io/provider-display-name": "Red Hat, Inc.", + "description": "Build and run Python 2.7 applications on UBI 8. For more information about using this builder image, including OpenShift considerations, see https://github.com/sclorg/s2i-python-container/blob/master/2.7/README.md.", + "iconClass": "icon-python", + "tags": "builder,python", + "supports":"python:2.7,python", + "version": "2.7", + "sampleRepo": "https://github.com/sclorg/django-ex.git" + }, + "from": { + "kind": "DockerImage", + "name": "registry.redhat.io/ubi8/python-27:latest" + }, + "referencePolicy": { + "type": "Local" + } + }, + { + "name": "2.7-ubi7", + "annotations": { + "openshift.io/display-name": "Python 2.7 (UBI 7)", + "openshift.io/provider-display-name": "Red Hat, Inc.", + "description": "Build and run Python 2.7 applications on UBI 7. For more information about using this builder image, including OpenShift considerations, see https://github.com/sclorg/s2i-python-container/blob/master/2.7/README.md.", + "iconClass": "icon-python", + "tags": "builder,python", + "supports":"python:2.7,python", + "version": "2.7", + "sampleRepo": "https://github.com/sclorg/django-ex.git" + }, + "from": { + "kind": "DockerImage", + "name": "registry.redhat.io/ubi7/python-27:latest" + }, + "referencePolicy": { + "type": "Local" + } + }, + { + "name": "2.7", + "annotations": { + "openshift.io/display-name": "Python 2.7", + "openshift.io/provider-display-name": "Red Hat, Inc.", + "description": "Build and run Python 2.7 applications on RHEL 7. For more information about using this builder image, including OpenShift considerations, see https://github.com/sclorg/s2i-python-container/blob/master/2.7/README.md.", + "iconClass": "icon-python", + "tags": "builder,python,hidden", + "supports":"python:2.7,python", + "version": "2.7", + "sampleRepo": "https://github.com/sclorg/django-ex.git" + }, + "from": { + "kind": "DockerImage", + "name": "registry.redhat.io/rhscl/python-27-rhel7:latest" + }, + "referencePolicy": { + "type": "Local" + } + } + ] + } +} diff --git a/test/npm-virtualenv-uwsgi-test-app/app.sh b/test/npm-virtualenv-uwsgi-test-app/app.sh index 2659e06..f34357a 100755 --- a/test/npm-virtualenv-uwsgi-test-app/app.sh +++ b/test/npm-virtualenv-uwsgi-test-app/app.sh @@ -13,7 +13,7 @@ packages=("pip" "setuptools" "wheel") for pkg in ${packages[@]}; do # grep returns exit code 1 if the output contains only one line starting # with "Requirement already …" which means that the package is updated - python -m pip install -U --no-python-version-warning $pkg 2>&1 | grep -v "^Requirement already up-to-date: " + python -m pip install -U --no-deps --no-python-version-warning $pkg 2>&1 | grep -v "^Requirement already up-to-date: " if [ $? -eq 0 ]; then echo "ERROR: Failed to upgrade '$pkg' to the latest version." exit 1 diff --git a/test/run b/test/run index d3829de..42a52d5 100755 --- a/test/run +++ b/test/run @@ -187,6 +187,12 @@ test_application() { cleanup_app } +test_from_dockerfile(){ + info "Test from Dockerfile" + sed "s@#IMAGE_NAME#@${IMAGE_NAME}@" $test_dir/from-dockerfile/Dockerfile.tpl > $test_dir/from-dockerfile/Dockerfile + ct_test_app_dockerfile $test_dir/from-dockerfile/Dockerfile 'https://github.com/sclorg/django-ex.git' 'Welcome to your Django application on OpenShift' app-src + check_result $? +} # Since we built the candidate image locally, we don't want S2I attempt to pull # it from Docker hub @@ -228,4 +234,20 @@ for app in ${@:-${WEB_APPS[@]}}; do cleanup ${app} done +if [ "$OS" == "rhel7" ] || [ "$OS" == "centos7" ]; then + # autocleanup only enabled here as only the following tests so far use it + CID_FILE_DIR=$(mktemp -d) + ct_enable_cleanup + + info "Testing variable presence during \`docker exec\`" + ct_check_exec_env_vars + check_result $? + + info "Checking if all scl variables are defined in Dockerfile" + ct_check_scl_enable_vars + check_result $? +fi + +test_from_dockerfile + info "All tests finished successfully." diff --git a/test/run-openshift b/test/run-openshift index 4a09812..35d3b7d 100755 --- a/test/run-openshift +++ b/test/run-openshift @@ -7,6 +7,7 @@ # The image has to be available before this script is executed. THISDIR=$(dirname ${BASH_SOURCE[0]}) +test_dir="$(readlink -f $(dirname ${BASH_SOURCE[0]}))" source "${THISDIR}/test-lib.sh" source "${THISDIR}/test-lib-openshift.sh" @@ -16,6 +17,14 @@ set -eo nounset trap ct_os_cleanup EXIT SIGINT +test_latest_imagestreams() { + info "Testing the latest version in imagestreams" + # Switch to root directory of a container + pushd "${test_dir}/../.." >/dev/null + ct_check_latest_imagestreams + popd >/dev/null +} + ct_os_check_compulsory_vars ct_os_enable_print_logs @@ -44,6 +53,9 @@ done # Check the imagestream test_python_imagestream +# check if latest imagestream version is correct +test_latest_imagestreams + OS_TESTSUITE_RESULT=0 ct_os_cluster_down diff --git a/test/test-lib-openshift.sh b/test/test-lib-openshift.sh index b92e950..c11a025 100644 --- a/test/test-lib-openshift.sh +++ b/test/test-lib-openshift.sh @@ -145,7 +145,7 @@ function ct_os_get_pod_status() { # Arguments: pod_prefix - prefix or whole ID of the pod function ct_os_get_build_pod_status() { local pod_prefix="${1}" ; shift - local query="custom-columns=NAME:.metadata.name,Ready:status.containerStatuses[0].state.terminated.reason" + local query="custom-columns=NAME:.metadata.name,Ready:status.phase" oc get pods -o "$query" | grep -e "${pod_prefix}" | grep -E "\-build\s" \ | sort -u | awk '{print $2}' | tail -n 1 } @@ -193,7 +193,7 @@ function ct_os_wait_pod_ready() { if ct_os_get_all_pods_name | grep -E "${pod_prefix}.*-build"; then SECONDS=0 echo -n "Waiting for ${pod_prefix} build pod to finish ..." - while ! [ "$(ct_os_get_build_pod_status "${pod_prefix}")" == "Completed" ] ; do + while ! [ "$(ct_os_get_build_pod_status "${pod_prefix}")" == "Succeeded" ] ; do echo -n "." [ "${SECONDS}" -gt "${timeout}0" ] && echo " FAIL" && return 1 sleep 3 @@ -219,7 +219,7 @@ function ct_os_wait_rc_ready() { local pod_prefix="${1}" ; shift local timeout="${1}" ; shift SECONDS=0 - echo -n "Waiting for ${pod_prefix} pod becoming ready ..." + echo -n "Waiting for ${pod_prefix} having desired numbers of replicas ..." while ! test "$( (oc get --no-headers statefulsets; oc get --no-headers rc) 2>/dev/null \ | grep "^${pod_prefix}" | awk '$2==$3 {print "ready"}')" == "ready" ; do echo -n "." @@ -413,6 +413,7 @@ function ct_os_install_in_centos() { bash-completion origin-clients docker origin-clients } + # ct_os_cluster_up [DIR, IS_PUBLIC, CLUSTER_VERSION] # -------------------- # Runs the local OpenShift cluster using 'oc cluster up' and logs in as developer. @@ -604,8 +605,7 @@ function ct_os_test_s2i_app_func() { local check_command=${4} local oc_args=${5:-} local image_name_no_namespace=${image_name##*/} - local service_name="${image_name_no_namespace}-testing" - local image_tagged="${image_name_no_namespace}:${VERSION}" + local service_name="${image_name_no_namespace%%:*}-testing" local namespace if [ $# -lt 4 ] || [ -z "${1}" ] || [ -z "${2}" ] || [ -z "${3}" ] || [ -z "${4}" ]; then @@ -618,16 +618,20 @@ function ct_os_test_s2i_app_func() { namespace=${CT_NAMESPACE:-"$(oc project -q)"} - # Create a specific imagestream tag for the image so that oc cannot use anything else - if [ "${CT_SKIP_UPLOAD_IMAGE:-false}" == 'true' ] ; then - echo "Importing image ${image_name} as ${namespace}/${image_tagged}" - # Use --reference-policy=local to pull remote image content to the cluster - # Works around the issue of builder pods not having access to registry.redhat.io - oc tag --source=docker "${image_name}" "${namespace}/${image_tagged}" --insecure=true --reference-policy=local - ct_os_wait_stream_ready "${image_tagged}" "${namespace}" + if [ "${CT_EXTERNAL_REGISTRY:-false}" == 'true' ] ; then + ct_os_import_image_ocp4 "${image_name}" "${image_name_no_namespace}" else - echo "Uploading image ${image_name} as ${image_tagged}" - ct_os_upload_image "${image_name}" "${image_tagged}" + # Create a specific imagestream tag for the image so that oc cannot use anything else + if [ "${CT_SKIP_UPLOAD_IMAGE:-false}" == 'true' ] ; then + echo "Importing image ${image_name} as ${namespace}/${image_name_no_namespace}" + # Use --reference-policy=local to pull remote image content to the cluster + # Works around the issue of builder pods not having access to registry.redhat.io + oc tag --source=docker "${image_name}" "${namespace}/${image_name_no_namespace}" --insecure=true --reference-policy=local + ct_os_wait_stream_ready "${image_name_no_namespace}" "${namespace}" + else + echo "Uploading image ${image_name} as ${image_name_no_namespace}" + ct_os_upload_image "${image_name}" "${image_name_no_namespace}" + fi fi local app_param="${app}" @@ -638,7 +642,7 @@ function ct_os_test_s2i_app_func() { fi # shellcheck disable=SC2086 - ct_os_deploy_s2i_image "${image_tagged}" "${app_param}" \ + ct_os_deploy_s2i_image "${image_name_no_namespace}" "${app_param}" \ --context-dir="${context_dir}" \ --name "${service_name}" \ ${oc_args} @@ -752,27 +756,35 @@ function ct_os_test_template_app_func() { ct_os_new_project namespace=${CT_NAMESPACE:-"$(oc project -q)"} - # Create a specific imagestream tag for the image so that oc cannot use anything else - if [ "${CT_SKIP_UPLOAD_IMAGE:-false}" == 'true' ] ; then - echo "Importing image ${image_name} as ${image_tagged}" - # Use --reference-policy=local to pull remote image content to the cluster - # Works around the issue of builder pods not having access to registry.redhat.io - oc tag --source=docker "${image_name}" "${namespace}/${image_tagged}" --insecure=true --reference-policy=local - ct_os_wait_stream_ready "${image_tagged}" "${namespace}" + if [ "${CT_EXTERNAL_REGISTRY:-false}" == 'true' ] ; then + ct_os_import_image_ocp4 "${image_name}" "${image_tagged}" else - echo "Uploading image ${image_name} as ${image_tagged}" - ct_os_upload_image "${image_name}" "${image_tagged}" - - # upload also other images, that template might need (list of pairs in the format | - local image_tag_a - local i_t - for i_t in ${other_images} ; do - echo "${i_t}" - IFS='|' read -ra image_tag_a <<< "${i_t}" - docker pull "${image_tag_a[0]}" - ct_os_upload_image "${image_tag_a[0]}" "${image_tag_a[1]}" - done + if [ "${CT_SKIP_UPLOAD_IMAGE:-false}" == 'true' ] ; then + echo "Importing image ${image_name} as ${image_tagged}" + # Use --reference-policy=local to pull remote image content to the cluster + # Works around the issue of builder pods not having access to registry.redhat.io + oc tag --source=docker "${image_name}" "${namespace}/${image_tagged}" --insecure=true --reference-policy=local + ct_os_wait_stream_ready "${image_tagged}" "${namespace}" + else + echo "Uploading image ${image_name} as ${image_tagged}" + ct_os_upload_image "${image_name}" "${image_tagged}" + fi + fi + if [ "${CT_SKIP_UPLOAD_IMAGE:-false}" == 'false' ] ; then + # upload also other images, that template might need (list of pairs in the format | + local image_tag_a + local i_t + for i_t in ${other_images} ; do + echo "${i_t}" + IFS='|' read -ra image_tag_a <<< "${i_t}" + docker pull "${image_tag_a[0]}" + if [ "${CT_EXTERNAL_REGISTRY:-false}" == 'true' ] ; then + ct_os_import_image_ocp4 "${image_tag_a[0]}" "${image_tag_a[1]}" + else + ct_os_upload_image "${image_tag_a[0]}" "${image_tag_a[1]}" + fi + done fi # get the template file from remote or local location; if not found, it is diff --git a/test/test-lib-python.sh b/test/test-lib-python.sh index c86056a..6cf2db3 100644 --- a/test/test-lib-python.sh +++ b/test/test-lib-python.sh @@ -18,7 +18,7 @@ function test_python_imagestream() { *) echo "Imagestream testing not supported for $OS environment." ; return 0 ;; esac - ct_os_test_image_stream_quickstart "${THISDIR}/imagestreams/python-${OS}.json" \ + ct_os_test_image_stream_quickstart "${THISDIR}/imagestreams/python-${OS%[0-9]*}.json" \ 'https://raw.githubusercontent.com/sclorg/django-ex/master/openshift/templates/django-postgresql.json' \ "${IMAGE_NAME}" \ 'python' \ diff --git a/test/test-lib.sh b/test/test-lib.sh index 52bee15..b2e092e 100644 --- a/test/test-lib.sh +++ b/test/test-lib.sh @@ -25,7 +25,9 @@ EXPECTED_EXIT_CODE=0 # Uses: $CID_FILE_DIR - path to directory containing cid_files # Uses: $EXPECTED_EXIT_CODE - expected container exit code function ct_cleanup() { + ct_show_resources for cid_file in "$CID_FILE_DIR"/* ; do + [ -f "$cid_file" ] || continue local container container=$(cat "$cid_file") @@ -68,6 +70,7 @@ function ct_check_envs_set { loop_envs=$1; shift env_format=${1:-"*VALUE*"} while read -r variable; do + [ -z "$variable" ] && continue var_name=$(echo "$variable" | awk -F= '{ print $1 }') stripped=$(echo "$variable" | awk -F= '{ print $2 }') filtered_envs=$(echo "$check_envs" | grep "^$var_name=") @@ -339,7 +342,7 @@ function ct_binary_found_from_df() { # Create Dockerfile that looks for the binary cat <"$tmpdir/Dockerfile" FROM $IMAGE_NAME -RUN which $binary | grep "$binary_path" +RUN command -v $binary | grep "$binary_path" EOF # Build an image, looking for expected path in the output if ! docker build -f "$tmpdir/Dockerfile" --no-cache "$tmpdir"; then @@ -557,7 +560,7 @@ ct_registry_from_os() { registry=registry.redhat.io ;; *) - registry=docker.io + registry=quay.io ;; esac echo "$registry" @@ -583,7 +586,9 @@ ct_get_public_image_name() { elif [ "x$os" == "xrhel8" ]; then public_image_name=$registry/rhel8/$base_image_name-${version//./} elif [ "x$os" == "xcentos7" ]; then - public_image_name=$registry/centos/$base_image_name-${version//./}-centos7 + public_image_name=$registry/centos7/$base_image_name-${version//./}-centos7 + elif [ "x$os" == "xcentos8" ]; then + public_image_name=$registry/centos8/$base_image_name-${version//./}-centos8 fi echo "$public_image_name" @@ -773,4 +778,125 @@ ct_check_image_availability() { fi } +# ct_check_latest_imagestreams +# ----------------------------- +# Check if the latest version present in Makefile in the variable VERSIONS +# is present in all imagestreams. +# Also the latest tag in the imagestreams has to contain the latest version +ct_check_latest_imagestreams() { + local latest_version= + local test_lib_dir= + + # We only maintain imagestreams for RHEL and CentOS (Community) + if [[ "$OS" =~ ^fedora.* ]] ; then + echo "Imagestreams for Fedora are not maintained, skipping ct_check_latest_imagestreams" + return 0 + fi + + # Check only lines which starts with VERSIONS + latest_version=$(grep '^VERSIONS' Makefile | rev | cut -d ' ' -f 1 | rev ) + # Fall back to previous version if the latest is excluded for this OS + [ -f "$latest_version/.exclude-$OS" ] && latest_version=$(grep '^VERSIONS' Makefile | rev | cut -d ' ' -f 2 | rev ) + # Only test the imagestream once, when the version matches + # ignore the SC warning, $VERSION is always available + # shellcheck disable=SC2153 + if [ "$latest_version" == "$VERSION" ]; then + test_lib_dir=$(dirname "$(readlink -f "$0")") + python3 "${test_lib_dir}/check_imagestreams.py" "$latest_version" + else + echo "Image version $VERSION is not latest, skipping ct_check_latest_imagestreams" + fi +} + +# ct_show_resources +# ---------------- +# Prints the available resources +ct_show_resources() +{ + echo "Resources info:" + echo "Memory:" + free -h + echo "Storage:" + df -h + echo "CPU" + lscpu +} + +# ct_test_app_dockerfile +# ----------------------------- +# Argument: dockerfile - path to a Dockerfile that will be used for building an image +# (must work with an application directory called 'app-src') +# Argument: app_url - git URI with a testing application +# Argument: body_regexp - PCRE regular expression that must match the response body +# Argument: app_dir - name of the application directory that is used in the Dockerfile +# Argument: port - Optional port number (default: 8080) +ct_test_app_dockerfile() { + local dockerfile=$1 + local app_url=$2 + local expected_text=$3 + local app_dir=$4 # this is a directory that must match with the name in the Dockerfile + local port=${5:-8080} + local app_image_name=myapp + local ret + local cname=app_dockerfile + + if [ -z "$app_dir" ] ; then + echo "ERROR: Option app_dir not set. Terminating the Dockerfile build." + return 1 + fi + + if ! [ -r "${dockerfile}" ] || ! [ -s "${dockerfile}" ] ; then + echo "ERROR: Dockerfile ${dockerfile} does not exist or is empty." + echo "Terminating the Dockerfile build." + return 1 + fi + + CID_FILE_DIR=${CID_FILE_DIR:-$(mktemp -d)} + local dockerfile_abs + dockerfile_abs=$(readlink -f "${dockerfile}") + tmpdir=$(mktemp -d) + pushd "$tmpdir" >/dev/null + cp "${dockerfile_abs}" Dockerfile + + # Rewrite the source image to what we test + sed -i -e "s|^FROM.*$|FROM $IMAGE_NAME|" Dockerfile + # a bit more verbose, but should help debugging failures + echo "Using this Dockerfile:" + cat Dockerfile + + if ! git clone "${app_url}" "${app_dir}" ; then + echo "ERROR: Git repository ${app_url} cannot be cloned into ${app_dir}." + echo "Terminating the Dockerfile build." + return 1 + fi + + echo "Building '${app_image_name}' image using docker build" + if ! docker build --no-cache=true -t "${app_image_name}" . ; then + echo "ERROR: The image cannot be built from ${dockerfile} and application ${app_url}." + echo "Terminating the Dockerfile build." + return 1 + fi + + if ! docker run -d --cidfile="${CID_FILE_DIR}/app_dockerfile" --rm "${app_image_name}" ; then + echo "ERROR: The image ${app_image_name} cannot be run for ${dockerfile} and application ${app_url}." + echo "Terminating the Dockerfile build." + return 1 + fi + echo "Waiting for ${app_image_name} to start" + ct_wait_for_cid "${CID_FILE_DIR}/app_dockerfile" + + ip="$(ct_get_cip "${cname}")" + ct_test_response "http://$ip:${port}" 200 "${expected_text}" + ret=$? + + # cleanup + docker kill "$(ct_get_cid "${cname}")" + sleep 2 + docker rmi "${app_image_name}" + popd >/dev/null + rm -rf "${tmpdir}" + rm -f "${CID_FILE_DIR}/${cname}" + return $ret +} + # vim: set tabstop=2:shiftwidth=2:expandtab: