diff --git a/README.md b/README.md index 9c40082..0e4a781 100644 --- a/README.md +++ b/README.md @@ -14,12 +14,12 @@ Note: while the examples in this README are calling `podman`, you can replace an Description ----------- -Python 3.7 available as container is a base platform for -building and running various Python 3.7 applications and frameworks. -Python is an easy to learn, powerful programming language. It has efficient high-level -data structures and a simple but effective approach to object-oriented programming. -Python's elegant syntax and dynamic typing, together with its interpreted nature, -make it an ideal language for scripting and rapid application development in many areas +Python 3.7 available as container is a base platform for +building and running various Python 3.7 applications and frameworks. +Python is an easy to learn, powerful programming language. It has efficient high-level +data structures and a simple but effective approach to object-oriented programming. +Python's elegant syntax and dynamic typing, together with its interpreted nature, +make it an ideal language for scripting and rapid application development in many areas on most platforms. This container image includes an npm utility @@ -123,10 +123,16 @@ file inside your source code repository. * **ENABLE_PIPENV** - Set this variable to use [Pipenv](https://github.com/kennethreitz/pipenv), + Set this variable to use [Pipenv](https://github.com/pypa/pipenv), the higher-level Python packaging tool, to manage dependencies of the application. This should be used only if your project contains properly formated Pipfile and Pipfile.lock. +* **PIN_PIPENV_VERSION** + + Set this variable together with `ENABLE_PIPENV` to use a specific version of Pipenv. + If not set, the latest stable version from PyPI is installed. + For example `PIN_PIPENV_VERSION=2018.11.26` installs `pipenv==2018.11.26`. + * **ENABLE_MICROPIPENV** Set this variable to use [micropipenv](https://github.com/thoth-station/micropipenv), @@ -145,9 +151,12 @@ file inside your source code repository. * **PIP_INDEX_URL** - Set this variable to use a custom index URL or mirror to download required packages - during build process. This only affects packages listed in requirements.txt. - Pipenv ignores this variable. + Set this variable to use a custom index URL or mirror to download required + packages during build process. This affects packages listed in + requirements.txt. It also affects the installation of pipenv and + micropipenv and the update of pip in the container, though if not found in + the custom index, the container will try to install/update them from + upstream PyPI afterwards. * **UPGRADE_PIP_TO_LATEST** diff --git a/s2i/bin/assemble b/s2i/bin/assemble index a42dd53..3aaf9cc 100755 --- a/s2i/bin/assemble +++ b/s2i/bin/assemble @@ -23,7 +23,15 @@ function install_tool() { echo "---> Installing $1 packaging tool ..." VENV_DIR=$HOME/.local/venvs/$1 virtualenv_bin "$VENV_DIR" - $VENV_DIR/bin/pip --isolated install -U $1$2 # Combines package name with [extras] if [extras] is defined as $2 + # First, try to install the tool without --isolated which means that if you + # have your own PyPI mirror, it will take it from there. If this try fails, try it + # again with --isolated which ignores external pip settings (env vars, config file) + # and installs the tool from PyPI (needs internet connetion). + # $1$2 combines package name with [extras] or version specifier if is defined as $2``` + if ! $VENV_DIR/bin/pip install -U $1$2; then + echo "WARNING: Installation of $1 failed, trying again from official PyPI with pip --isolated install" + $VENV_DIR/bin/pip install --isolated -U $1$2 # Combines package name with [extras] or version specifier if is defined as $2``` + fi mkdir -p $HOME/.local/bin ln -s $VENV_DIR/bin/$1 $HOME/.local/bin/$1 } @@ -50,15 +58,25 @@ fix-permissions /opt/app-root -P # * pip < 19.3 does not support manylinux2014 wheels. Only manylinux2014 wheels # support platforms like ppc64le, aarch64 or armv7 echo "---> Upgrading pip to version 19.3.1 ..." -pip install -U "pip==19.3.1" +if ! pip install -U "pip==19.3.1"; then + echo "WARNING: Installation of 'pip==19.3.1' failed, trying again from official PyPI with pip --isolated install" + pip install --isolated -U "pip==19.3.1" +fi if [[ ! -z "$UPGRADE_PIP_TO_LATEST" ]]; then echo "---> Upgrading pip to latest version ..." - pip install -U pip setuptools wheel + if ! pip install -U pip setuptools wheel; then + echo "WARNING: Installation of the latest pip,setuptools and wheel failed, trying again from official PyPI with pip --isolated install" + pip install --isolated -U pip setuptools wheel + fi fi if [[ ! -z "$ENABLE_PIPENV" ]]; then - install_tool "pipenv" "==2018.11.26" + if [[ ! -z "$PIN_PIPENV_VERSION" ]]; then + # Add == as a prefix to pipenv version, if defined + PIN_PIPENV_VERSION="==$PIN_PIPENV_VERSION" + fi + install_tool "pipenv" "$PIN_PIPENV_VERSION" echo "---> Installing dependencies via pipenv ..." if [[ -f Pipfile ]]; then pipenv install --deploy diff --git a/test/django-test-app/requirements.txt b/test/django-test-app/requirements.txt index 1f45a73..30eb621 100644 --- a/test/django-test-app/requirements.txt +++ b/test/django-test-app/requirements.txt @@ -1 +1 @@ -Django==1.11.28 +Django==1.11.29 diff --git a/test/imagestreams/python-centos7.json b/test/imagestreams/python-centos7.json new file mode 100644 index 0000000..bc129c0 --- /dev/null +++ b/test/imagestreams/python-centos7.json @@ -0,0 +1,73 @@ +{ + "kind": "ImageStream", + "apiVersion": "v1", + "metadata": { + "name": "python", + "annotations": { + "openshift.io/display-name": "Python" + } + }, + "spec": { + "tags": [ + { + "name": "latest", + "annotations": { + "openshift.io/display-name": "Python (Latest)", + "openshift.io/provider-display-name": "Red Hat, Inc.", + "description": "Build and run Python applications on CentOS 7. For more information about using this builder image, including OpenShift considerations, see https://github.com/sclorg/s2i-python-container/blob/master/3.6/README.md.\n\nWARNING: By selecting this tag, your application will automatically update to use the latest version of Python available on OpenShift, including major version updates.", + "iconClass": "icon-python", + "tags": "builder,python", + "supports":"python", + "sampleRepo": "https://github.com/sclorg/django-ex.git" + }, + "from": { + "kind": "ImageStreamTag", + "name": "3.6" + }, + "referencePolicy": { + "type": "Local" + } + }, + { + "name": "3.6", + "annotations": { + "openshift.io/display-name": "Python 3.6", + "openshift.io/provider-display-name": "Red Hat, Inc.", + "description": "Build and run Python 3.6 applications on CentOS 7. For more information about using this builder image, including OpenShift considerations, see https://github.com/sclorg/s2i-python-container/blob/master/3.6/README.md.", + "iconClass": "icon-python", + "tags": "builder,python", + "supports":"python:3.6,python", + "version": "3.6", + "sampleRepo": "https://github.com/sclorg/django-ex.git" + }, + "from": { + "kind": "DockerImage", + "name": "docker.io/centos/python-36-centos7:latest" + }, + "referencePolicy": { + "type": "Local" + } + }, + { + "name": "2.7", + "annotations": { + "openshift.io/display-name": "Python 2.7", + "openshift.io/provider-display-name": "Red Hat, Inc.", + "description": "Build and run Python 2.7 applications on CentOS 7. For more information about using this builder image, including OpenShift considerations, see https://github.com/sclorg/s2i-python-container/blob/master/2.7/README.md.", + "iconClass": "icon-python", + "tags": "builder,python", + "supports":"python:2.7,python", + "version": "2.7", + "sampleRepo": "https://github.com/sclorg/django-ex.git" + }, + "from": { + "kind": "DockerImage", + "name": "docker.io/centos/python-27-centos7:latest" + }, + "referencePolicy": { + "type": "Local" + } + } + ] + } +} diff --git a/test/imagestreams/python-rhel7.json b/test/imagestreams/python-rhel7.json new file mode 100644 index 0000000..7de8ec1 --- /dev/null +++ b/test/imagestreams/python-rhel7.json @@ -0,0 +1,93 @@ +{ + "kind": "ImageStream", + "apiVersion": "v1", + "metadata": { + "name": "python", + "annotations": { + "openshift.io/display-name": "Python" + } + }, + "spec": { + "tags": [ + { + "name": "latest", + "annotations": { + "openshift.io/display-name": "Python (Latest)", + "openshift.io/provider-display-name": "Red Hat, Inc.", + "description": "Build and run Python applications on RHEL 7. For more information about using this builder image, including OpenShift considerations, see https://github.com/sclorg/s2i-python-container/blob/master/3.8/README.md.\n\nWARNING: By selecting this tag, your application will automatically update to use the latest version of Python available on OpenShift, including major version updates.", + "iconClass": "icon-python", + "tags": "builder,python", + "supports":"python", + "sampleRepo": "https://github.com/sclorg/django-ex.git" + }, + "from": { + "kind": "ImageStreamTag", + "name": "3.8" + }, + "referencePolicy": { + "type": "Local" + } + }, + { + "name": "3.8", + "annotations": { + "openshift.io/display-name": "Python 3.8", + "openshift.io/provider-display-name": "Red Hat, Inc.", + "description": "Build and run Python 3.8 applications on RHEL 7. For more information about using this builder image, including OpenShift considerations, see https://github.com/sclorg/s2i-python-container/blob/master/3.8/README.md.", + "iconClass": "icon-python", + "tags": "builder,python", + "supports":"python:3.8,python", + "version": "3.8", + "sampleRepo": "https://github.com/sclorg/django-ex.git" + }, + "from": { + "kind": "DockerImage", + "name": "registry.redhat.io/rhscl/python-38-rhel7:latest" + }, + "referencePolicy": { + "type": "Local" + } + }, + { + "name": "3.6", + "annotations": { + "openshift.io/display-name": "Python 3.6", + "openshift.io/provider-display-name": "Red Hat, Inc.", + "description": "Build and run Python 3.6 applications on RHEL 7. For more information about using this builder image, including OpenShift considerations, see https://github.com/sclorg/s2i-python-container/blob/master/3.6/README.md.", + "iconClass": "icon-python", + "tags": "builder,python", + "supports":"python:3.6,python", + "version": "3.6", + "sampleRepo": "https://github.com/sclorg/django-ex.git" + }, + "from": { + "kind": "DockerImage", + "name": "registry.redhat.io/rhscl/python-36-rhel7:latest" + }, + "referencePolicy": { + "type": "Local" + } + }, + { + "name": "2.7", + "annotations": { + "openshift.io/display-name": "Python 2.7", + "openshift.io/provider-display-name": "Red Hat, Inc.", + "description": "Build and run Python 2.7 applications on RHEL 7. For more information about using this builder image, including OpenShift considerations, see https://github.com/sclorg/s2i-python-container/blob/master/2.7/README.md.", + "iconClass": "icon-python", + "tags": "builder,python", + "supports":"python:2.7,python", + "version": "2.7", + "sampleRepo": "https://github.com/sclorg/django-ex.git" + }, + "from": { + "kind": "DockerImage", + "name": "registry.redhat.io/rhscl/python-27-rhel7:latest" + }, + "referencePolicy": { + "type": "Local" + } + } + ] + } +} diff --git a/test/imagestreams/python-ubi8.json b/test/imagestreams/python-ubi8.json new file mode 100644 index 0000000..b075cda --- /dev/null +++ b/test/imagestreams/python-ubi8.json @@ -0,0 +1,93 @@ +{ + "kind": "ImageStream", + "apiVersion": "v1", + "metadata": { + "name": "python", + "annotations": { + "openshift.io/display-name": "Python" + } + }, + "spec": { + "tags": [ + { + "name": "latest", + "annotations": { + "openshift.io/display-name": "Python (Latest)", + "openshift.io/provider-display-name": "Red Hat, Inc.", + "description": "Build and run Python applications on UBI 8. For more information about using this builder image, including OpenShift considerations, see https://github.com/sclorg/s2i-python-container/blob/master/3.8/README.md.\n\nWARNING: By selecting this tag, your application will automatically update to use the latest version of Python available on OpenShift, including major version updates.", + "iconClass": "icon-python", + "tags": "builder,python", + "supports":"python", + "sampleRepo": "https://github.com/sclorg/django-ex.git" + }, + "from": { + "kind": "ImageStreamTag", + "name": "3.8" + }, + "referencePolicy": { + "type": "Local" + } + }, + { + "name": "3.8", + "annotations": { + "openshift.io/display-name": "Python 3.8", + "openshift.io/provider-display-name": "Red Hat, Inc.", + "description": "Build and run Python 3.8 applications on UBI 8. For more information about using this builder image, including OpenShift considerations, see https://github.com/sclorg/s2i-python-container/blob/master/3.8/README.md.", + "iconClass": "icon-python", + "tags": "builder,python", + "supports":"python:3.8,python", + "version": "3.8", + "sampleRepo": "https://github.com/sclorg/django-ex.git" + }, + "from": { + "kind": "DockerImage", + "name": "registry.redhat.io/ubi8/python-38:latest" + }, + "referencePolicy": { + "type": "Local" + } + }, + { + "name": "3.6", + "annotations": { + "openshift.io/display-name": "Python 3.6", + "openshift.io/provider-display-name": "Red Hat, Inc.", + "description": "Build and run Python 3.6 applications on UBI 8. For more information about using this builder image, including OpenShift considerations, see https://github.com/sclorg/s2i-python-container/blob/master/3.6/README.md.", + "iconClass": "icon-python", + "tags": "builder,python", + "supports":"python:3.6,python", + "version": "3.6", + "sampleRepo": "https://github.com/sclorg/django-ex.git" + }, + "from": { + "kind": "DockerImage", + "name": "registry.redhat.io/ubi8/python-36:latest" + }, + "referencePolicy": { + "type": "Local" + } + }, + { + "name": "2.7", + "annotations": { + "openshift.io/display-name": "Python 2.7", + "openshift.io/provider-display-name": "Red Hat, Inc.", + "description": "Build and run Python 2.7 applications on UBI 8. For more information about using this builder image, including OpenShift considerations, see https://github.com/sclorg/s2i-python-container/blob/master/2.7/README.md.", + "iconClass": "icon-python", + "tags": "builder,python", + "supports":"python:2.7,python", + "version": "2.7", + "sampleRepo": "https://github.com/sclorg/django-ex.git" + }, + "from": { + "kind": "DockerImage", + "name": "registry.redhat.io/ubi8/python-27:latest" + }, + "referencePolicy": { + "type": "Local" + } + } + ] + } +} diff --git a/test/micropipenv-test-app/.s2i/environment b/test/micropipenv-test-app/.s2i/environment index 2d40b15..29efed4 100644 --- a/test/micropipenv-test-app/.s2i/environment +++ b/test/micropipenv-test-app/.s2i/environment @@ -1,2 +1,5 @@ ENABLE_MICROPIPENV=true DISABLE_SETUP_PY_PROCESSING=true +# This tests second try to install micropipenv with --isolated +# because the first one won't work with following setting +PIP_INDEX_URL=https://example.com/ diff --git a/test/pin-pipenv-version-test-app/.s2i/environment b/test/pin-pipenv-version-test-app/.s2i/environment new file mode 100644 index 0000000..6440a42 --- /dev/null +++ b/test/pin-pipenv-version-test-app/.s2i/environment @@ -0,0 +1,2 @@ +ENABLE_PIPENV=1 +PIN_PIPENV_VERSION=2018.11.26 diff --git a/test/pin-pipenv-version-test-app/app.sh b/test/pin-pipenv-version-test-app/app.sh new file mode 100755 index 0000000..b6d71a0 --- /dev/null +++ b/test/pin-pipenv-version-test-app/app.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +echo "Testing PIN_PIPENV_VERSION (set in .s2i/environment) ..." +pipenv_version=`pipenv --version` +expected="pipenv, version 2018.11.26" +if [ "$pipenv_version" != "$expected" ]; then + echo "ERROR: pipenv version is different than expected." + echo "Expected: ${expected}" + echo "Actual: ${pipenv_version}" + exit 1 +fi + +# Test the uwsgi server +exec uwsgi \ + --http-socket :8080 \ + --die-on-term \ + --master \ + --single-interpreter \ + --enable-threads \ + --threads=5 \ + --thunder-lock \ + --module wsgi diff --git a/test/pin-pipenv-version-test-app/requirements.txt b/test/pin-pipenv-version-test-app/requirements.txt new file mode 100644 index 0000000..66d4205 --- /dev/null +++ b/test/pin-pipenv-version-test-app/requirements.txt @@ -0,0 +1,2 @@ +uWSGI +Flask diff --git a/test/pin-pipenv-version-test-app/wsgi.py b/test/pin-pipenv-version-test-app/wsgi.py new file mode 100644 index 0000000..96e1a61 --- /dev/null +++ b/test/pin-pipenv-version-test-app/wsgi.py @@ -0,0 +1,9 @@ +from flask import Flask +application = Flask(__name__) + +@application.route('/') +def hello(): + return b'Hello World from uWSGI hosted WSGI application!' + +if __name__ == '__main__': + application.run() diff --git a/test/run b/test/run index 5b0efcb..d3829de 100755 --- a/test/run +++ b/test/run @@ -6,7 +6,7 @@ # IMAGE_NAME specifies a name of the candidate image used for testing. # The image has to be available before this script is executed. # -declare -a WEB_APPS=({standalone,setup,setup-requirements,django,numpy,app-home,npm-virtualenv-uwsgi,locale,mod-wsgi,pipenv,pipenv-and-micropipenv-should-fail,micropipenv,micropipenv-requirements}-test-app) +declare -a WEB_APPS=({standalone,setup,setup-requirements,django,numpy,app-home,npm-virtualenv-uwsgi,locale,mod-wsgi,pipenv,pipenv-and-micropipenv-should-fail,pin-pipenv-version,micropipenv,micropipenv-requirements}-test-app) # TODO: Make command compatible for Mac users test_dir="$(readlink -zf $(dirname "${BASH_SOURCE[0]}"))" @@ -86,7 +86,7 @@ check_result() { # positive test & non-zero exit status = ERROR # negative test & zero exit status = ERROR local result="$1" - local test=${2:-positive} # if not defined, we expect possitive test type (zero exit code) + local type=${2:-positive} # if not defined, we expect possitive test type (zero exit code) if [[ "$type" == "positive" && "$result" != "0" ]]; then info "TEST FAILED (${type}), EXPECTED:0 GOT:${result}" cleanup @@ -102,13 +102,13 @@ wait_for_cid() { local max_attempts=10 local sleep_time=1 local attempt=1 - local result=1 info "Waiting for application container to start $CONTAINER_ARGS ..." while [ $attempt -le $max_attempts ]; do - [ -f $cid_file ] && [ -s $cid_file ] && break + [ -f $cid_file ] && [ -s $cid_file ] && return 0 attempt=$(( $attempt + 1 )) sleep $sleep_time done + return 1 } test_s2i_usage() { @@ -172,6 +172,9 @@ test_application() { # Wait for the container to write it's CID file wait_for_cid + # Some test apps have tests in their startup code so we have to check + # that the container starts at all + check_result $? test_scl_usage "python --version" "Python $VERSION." "${cid_file}" check_result $? diff --git a/test/run-openshift b/test/run-openshift index 87ac440..4a09812 100755 --- a/test/run-openshift +++ b/test/run-openshift @@ -10,11 +10,15 @@ THISDIR=$(dirname ${BASH_SOURCE[0]}) source "${THISDIR}/test-lib.sh" source "${THISDIR}/test-lib-openshift.sh" +source "${THISDIR}/test-lib-python.sh" -set -exo nounset +set -eo nounset -test -n "${IMAGE_NAME-}" || false 'make sure $IMAGE_NAME is defined' -test -n "${VERSION-}" || false 'make sure $VERSION is defined' +trap ct_os_cleanup EXIT SIGINT + +ct_os_check_compulsory_vars + +ct_os_enable_print_logs if [ -z "${EPHEMERAL_TEMPLATES:-}" ]; then EPHEMERAL_TEMPLATES=" @@ -36,3 +40,13 @@ for template in $EPHEMERAL_TEMPLATES; do 8080 http 200 "-p SOURCE_REPOSITORY_REF=master -p PYTHON_VERSION=${VERSION} -p POSTGRESQL_VERSION=9.6 -p NAME=python-testing" \ "centos/postgresql-96-centos7|postgresql:9.6" done + +# Check the imagestream +test_python_imagestream + +OS_TESTSUITE_RESULT=0 + +ct_os_cluster_down + +# vim: set tabstop=2:shiftwidth=2:expandtab: + diff --git a/test/test-lib-openshift.sh b/test/test-lib-openshift.sh index 89fc1cf..b92e950 100644 --- a/test/test-lib-openshift.sh +++ b/test/test-lib-openshift.sh @@ -139,6 +139,17 @@ function ct_os_get_pod_status() { | awk '{print $1}' | head -n 1 } +# ct_os_get_build_pod_status POD_PREFIX +# -------------------- +# Returns status of the build pod specified by prefix [pod_prefix]. +# Arguments: pod_prefix - prefix or whole ID of the pod +function ct_os_get_build_pod_status() { + local pod_prefix="${1}" ; shift + local query="custom-columns=NAME:.metadata.name,Ready:status.containerStatuses[0].state.terminated.reason" + oc get pods -o "$query" | grep -e "${pod_prefix}" | grep -E "\-build\s" \ + | sort -u | awk '{print $2}' | tail -n 1 +} + # ct_os_get_pod_name POD_PREFIX # -------------------- # Returns the full name of pods specified by prefix [pod_prefix]. @@ -177,6 +188,18 @@ function ct_os_check_pod_readiness() { function ct_os_wait_pod_ready() { local pod_prefix="${1}" ; shift local timeout="${1}" ; shift + # If there is a build pod - wait for it to finish first + sleep 3 + if ct_os_get_all_pods_name | grep -E "${pod_prefix}.*-build"; then + SECONDS=0 + echo -n "Waiting for ${pod_prefix} build pod to finish ..." + while ! [ "$(ct_os_get_build_pod_status "${pod_prefix}")" == "Completed" ] ; do + echo -n "." + [ "${SECONDS}" -gt "${timeout}0" ] && echo " FAIL" && return 1 + sleep 3 + done + echo " DONE" + fi SECONDS=0 echo -n "Waiting for ${pod_prefix} pod becoming ready ..." while ! ct_os_check_pod_readiness "${pod_prefix}" "true" ; do @@ -231,7 +254,7 @@ function ct_os_deploy_s2i_image() { local image="${1}" ; shift local app="${1}" ; shift # ignore error exit code, because oc new-app returns error when image exists - oc new-app "${image}~${app}" "$@" || : + oc new-app "${image}~${app}" --strategy=source "$@" || : # let openshift cluster to sync to avoid some race condition errors sleep 3 @@ -580,10 +603,10 @@ function ct_os_test_s2i_app_func() { local context_dir=${3} local check_command=${4} local oc_args=${5:-} - local import_image=${6:-} local image_name_no_namespace=${image_name##*/} local service_name="${image_name_no_namespace}-testing" local image_tagged="${image_name_no_namespace}:${VERSION}" + local namespace if [ $# -lt 4 ] || [ -z "${1}" ] || [ -z "${2}" ] || [ -z "${3}" ] || [ -z "${4}" ]; then echo "ERROR: ct_os_test_s2i_app_func() requires at least 4 arguments that cannot be emtpy." >&2 @@ -592,18 +615,19 @@ function ct_os_test_s2i_app_func() { # shellcheck disable=SC2119 ct_os_new_project + + namespace=${CT_NAMESPACE:-"$(oc project -q)"} + # Create a specific imagestream tag for the image so that oc cannot use anything else if [ "${CT_SKIP_UPLOAD_IMAGE:-false}" == 'true' ] ; then - if [ -n "${import_image}" ] ; then - echo "Importing image ${import_image} as ${image_name}:${VERSION}" - # Use --reference-policy=local to pull remote image content to the cluster - # Works around the issue of builder pods not having access to registry.redhat.io - oc import-image "${image_name}":"${VERSION}" --from "${import_image}" --confirm --reference-policy=local - else - echo "Uploading and importing image skipped." - fi + echo "Importing image ${image_name} as ${namespace}/${image_tagged}" + # Use --reference-policy=local to pull remote image content to the cluster + # Works around the issue of builder pods not having access to registry.redhat.io + oc tag --source=docker "${image_name}" "${namespace}/${image_tagged}" --insecure=true --reference-policy=local + ct_os_wait_stream_ready "${image_tagged}" "${namespace}" else - ct_os_upload_image "${import_image:-$image_name}" "${image_tagged}" + echo "Uploading image ${image_name} as ${image_tagged}" + ct_os_upload_image "${image_name}" "${image_tagged}" fi local app_param="${app}" @@ -630,15 +654,21 @@ function ct_os_test_s2i_app_func() { local ip local check_command_exp + local image_id + + # get image ID from the deployment config + image_id=$(oc get "deploymentconfig.apps.openshift.io/${service_name}" -o custom-columns=IMAGE:.spec.template.spec.containers[*].image | tail -n 1) ip=$(ct_os_get_service_ip "${service_name}") # shellcheck disable=SC2001 - check_command_exp=$(echo "$check_command" | sed -e "s//$ip/g") + check_command_exp=$(echo "$check_command" | sed -e "s//$ip/g" -e "s||${image_id}|g") echo " Checking APP using $check_command_exp ..." local result=0 eval "$check_command_exp" || result=1 + ct_os_service_image_info "${service_name}" + if [ $result -eq 0 ] ; then echo " Check passed." else @@ -672,7 +702,6 @@ function ct_os_test_s2i_app() { local protocol=${6:-http} local response_code=${7:-200} local oc_args=${8:-} - local import_image=${9:-} if [ $# -lt 4 ] || [ -z "${1}" ] || [ -z "${2}" ] || [ -z "${3}" ] || [ -z "${4}" ]; then echo "ERROR: ct_os_test_s2i_app() requires at least 4 arguments that cannot be emtpy." >&2 @@ -683,7 +712,7 @@ function ct_os_test_s2i_app() { "${app}" \ "${context_dir}" \ "ct_os_test_response_internal '${protocol}://:${port}' '${response_code}' '${expected_output}'" \ - "${oc_args}" "${import_image}" + "${oc_args}" } # ct_os_test_template_app_func IMAGE APP IMAGE_IN_TEMPLATE CHECK_CMD [OC_ARGS] @@ -709,7 +738,6 @@ function ct_os_test_template_app_func() { local check_command=${4} local oc_args=${5:-} local other_images=${6:-} - local import_image=${7:-} if [ $# -lt 4 ] || [ -z "${1}" ] || [ -z "${2}" ] || [ -z "${3}" ] || [ -z "${4}" ]; then echo "ERROR: ct_os_test_template_app_func() requires at least 4 arguments that cannot be emtpy." >&2 @@ -718,22 +746,23 @@ function ct_os_test_template_app_func() { local service_name="${name_in_template}-testing" local image_tagged="${name_in_template}:${VERSION}" + local namespace # shellcheck disable=SC2119 ct_os_new_project + namespace=${CT_NAMESPACE:-"$(oc project -q)"} + # Create a specific imagestream tag for the image so that oc cannot use anything else if [ "${CT_SKIP_UPLOAD_IMAGE:-false}" == 'true' ] ; then - if [ -n "${import_image}" ] ; then - echo "Importing image ${import_image} as ${image_name}:${VERSION}" - # Use --reference-policy=local to pull remote image content to the cluster - # Works around the issue of builder pods not having access to registry.redhat.io - oc import-image "${image_name}":"${VERSION}" --from "${import_image}" --confirm --reference-policy=local - else - echo "Uploading and importing image skipped." - fi + echo "Importing image ${image_name} as ${image_tagged}" + # Use --reference-policy=local to pull remote image content to the cluster + # Works around the issue of builder pods not having access to registry.redhat.io + oc tag --source=docker "${image_name}" "${namespace}/${image_tagged}" --insecure=true --reference-policy=local + ct_os_wait_stream_ready "${image_tagged}" "${namespace}" else - ct_os_upload_image "${import_image:-$image_name}" "${image_tagged}" + echo "Uploading image ${image_name} as ${image_tagged}" + ct_os_upload_image "${image_name}" "${image_tagged}" # upload also other images, that template might need (list of pairs in the format | local image_tag_a @@ -750,9 +779,6 @@ function ct_os_test_template_app_func() { # considered an internal template name, like 'mysql', so use the name # explicitly local local_template - local namespace - - namespace=${CT_NAMESPACE:-$(oc project -q)} local_template=$(ct_obtain_input "${template}" 2>/dev/null || echo "--template=${template}") # shellcheck disable=SC2086 @@ -765,15 +791,21 @@ function ct_os_test_template_app_func() { local ip local check_command_exp + local image_id + + # get image ID from the deployment config + image_id=$(oc get "deploymentconfig.apps.openshift.io/${service_name}" -o custom-columns=IMAGE:.spec.template.spec.containers[*].image | tail -n 1) ip=$(ct_os_get_service_ip "${service_name}") # shellcheck disable=SC2001 - check_command_exp=$(echo "$check_command" | sed -e "s//$ip/g") + check_command_exp=$(echo "$check_command" | sed -e "s//$ip/g" -e "s||${image_id}|g") echo " Checking APP using $check_command_exp ..." local result=0 eval "$check_command_exp" || result=1 + ct_os_service_image_info "${service_name}" + if [ $result -eq 0 ] ; then echo " Check passed." else @@ -813,7 +845,6 @@ function ct_os_test_template_app() { local response_code=${7:-200} local oc_args=${8:-} local other_images=${9:-} - local import_image=${10:-} if [ $# -lt 4 ] || [ -z "${1}" ] || [ -z "${2}" ] || [ -z "${3}" ] || [ -z "${4}" ]; then echo "ERROR: ct_os_test_template_app() requires at least 4 arguments that cannot be emtpy." >&2 @@ -825,8 +856,7 @@ function ct_os_test_template_app() { "${name_in_template}" \ "ct_os_test_response_internal '${protocol}://:${port}' '${response_code}' '${expected_output}'" \ "${oc_args}" \ - "${other_images}" \ - "${import_image}" + "${other_images}" } # ct_os_test_image_update IMAGE_NAME OLD_IMAGE ISTAG CHECK_FUNCTION OC_ARGS @@ -1017,7 +1047,7 @@ function ct_os_check_cmd_internal() { # shellcheck disable=SC2001 check_command_exp=$(echo "$check_command" | sed -e "s//$ip/g") - ct_os_deploy_cmd_image "$(ct_os_get_image_from_pod "${util_image_name##*/}" | head -n 1)" + ct_os_deploy_cmd_image "${util_image_name}" SECONDS=0 echo -n "Waiting for ${service_name} service becoming ready ..." @@ -1037,4 +1067,200 @@ function ct_os_check_cmd_internal() { return 1 } +# ct_os_test_image_stream_template IMAGE_STREAM_FILE TEMPLATE_FILE SERVICE NAME [TEMPLATE_PARAMS] +# ------------------------ +# Creates an image stream and deploys a specified template. Then checks that a pod runs. +# Argument: image_stream_file - local or remote file with the image stream definition +# Argument: template_file - local file name with a template +# Argument: service_name - how the pod will be named (prefix) +# Argument: template_params (optional) - parameters for the template, like image stream version +function ct_os_test_image_stream_template() { + local image_stream_file=${1} + local template_file=${2} + local service_name=${3} + local template_params=${4:-} + local local_image_stream_file + local local_template_file + + if [ $# -lt 3 ] || [ -z "${1}" ] || [ -z "${2}" ] || [ -z "${3}" ]; then + echo "ERROR: ct_os_test_image_stream() requires at least 3 arguments that cannot be empty." >&2 + return 1 + fi + + echo "Running image stream test for stream ${image_stream_file} and template ${template_file}" + # shellcheck disable=SC2119 + ct_os_new_project + + local_image_stream_file=$(ct_obtain_input "${image_stream_file}") + local_template_file=$(ct_obtain_input "${template_file}") + oc create -f "${local_image_stream_file}" + + # shellcheck disable=SC2086 + if ! ct_os_deploy_template_image "${local_template_file}" -p NAMESPACE="${CT_NAMESPACE:-$(oc project -q)}" ${template_params} ; then + echo "ERROR: ${template_file} could not be loaded" + return 1 + # Deliberately not runnig ct_os_delete_project here because user either + # might want to investigate or the cleanup is done with the cleanup trap. + # Most functions depend on the set -e anyway at this point. + fi + ct_os_wait_pod_ready "${service_name}" 120 + + # shellcheck disable=SC2119 + ct_os_delete_project +} + +# ct_os_wait_stream_ready IMAGE_STREAM_FILE NAMESPACE [ TIMEOUT ] +# ------------------------ +# Waits max timeout seconds till a [stream] is available in the [namespace]. +# Arguments: image_stream - stream name (usuallly :) +# Arguments: namespace - namespace name +# Arguments: timeout - how many seconds to wait +function ct_os_wait_stream_ready() { + local image_stream=${1} + local namespace=${2} + local timeout=${3:-60} + # It takes some time for the first time before the image is pulled in + SECONDS=0 + echo -n "Waiting for ${namespace}/${image_stream} to become available ..." + while ! oc get -n "${namespace}" istag "${image_stream}" &>/dev/null; do + if [ "$SECONDS" -gt "${timeout}" ] ; then + echo "FAIL: ${namespace}/${image_stream} not available after ${timeout}s:" + echo "oc get -n ${namespace} istag ${image_stream}" + oc get -n "${namespace}" istag "${image_stream}" + return 1 + fi + sleep 3 + echo -n . + done + echo " DONE" +} + +# ct_os_test_image_stream_s2i IMAGE_STREAM_FILE IMAGE_NAME APP CONTEXT_DIR EXPECTED_OUTPUT [PORT, PROTOCOL, RESPONSE_CODE, OC_ARGS, ... ] +# -------------------- +# Check the imagestream with an s2i app check. First it imports the given image stream, then +# it runs [image] and [app] in the openshift and optionally specifies env_params +# as environment variables to the image. Then check the http response. +# Argument: image_stream_file - local or remote file with the image stream definition +# Argument: image_name - container image we test (or name of the existing image stream in : format) +# Argument: app - url or local path to git repo with the application sources (compulsory) +# Argument: context_dir - sub-directory inside the repository with the application sources (compulsory) +# Argument: expected_output - PCRE regular expression that must match the response body (compulsory) +# Argument: port - which port to use (optional; default: 8080) +# Argument: protocol - which protocol to use (optional; default: http) +# Argument: response_code - what http response code to expect (optional; default: 200) +# Argument: oc_args - all other arguments are used as additional parameters for the `oc new-app` +# command, typically environment variables (optional) +function ct_os_test_image_stream_s2i() { + local image_stream_file=${1} + local image_name=${2} + local app=${3} + local context_dir=${4} + local expected_output=${5} + local port=${6:-8080} + local protocol=${7:-http} + local response_code=${8:-200} + local oc_args=${9:-} + local result + local local_image_stream_file + + echo "Running image stream test for stream ${image_stream_file} and application ${app} with context ${context_dir}" + + # shellcheck disable=SC2119 + ct_os_new_project + + local_image_stream_file=$(ct_obtain_input "${image_stream_file}") + oc create -f "${local_image_stream_file}" + + # ct_os_test_s2i_app creates a new project, but we already need + # it before for the image stream import, so tell it to skip this time + CT_SKIP_NEW_PROJECT=true \ + ct_os_test_s2i_app "${IMAGE_NAME}" "${app}" "${context_dir}" "${expected_output}" \ + "${port}" "${protocol}" "${response_code}" "${oc_args}" + result=$? + + # shellcheck disable=SC2119 + ct_os_delete_project + + return $result +} + +# ct_os_test_image_stream_quickstart IMAGE_STREAM_FILE TEMPLATE IMAGE_NAME NAME_IN_TEMPLATE EXPECTED_OUTPUT [PORT, PROTOCOL, RESPONSE_CODE, OC_ARGS, OTHER_IMAGES ] +# -------------------- +# Check the imagestream with an s2i app check. First it imports the given image stream, then +# it runs [image] and [app] in the openshift and optionally specifies env_params +# as environment variables to the image. Then check the http response. +# Argument: image_stream_file - local or remote file with the image stream definition +# Argument: template_file - local file name with a template +# Argument: image_name - container image we test (or name of the existing image stream in : format) +# Argument: name_in_template - image name used in the template +# Argument: expected_output - PCRE regular expression that must match the response body (compulsory) +# Argument: port - which port to use (optional; default: 8080) +# Argument: protocol - which protocol to use (optional; default: http) +# Argument: response_code - what http response code to expect (optional; default: 200) +# Argument: oc_args - all other arguments are used as additional parameters for the `oc new-app` +# command, typically environment variables (optional) +# Argument: other_images - some templates need other image to be pushed into the OpenShift registry, +# specify them in this parameter as "|", where "" is a full image name +# (including registry if needed) and "" is a tag under which the image should be available +# in the OpenShift registry. +function ct_os_test_image_stream_quickstart() { + local image_stream_file=${1} + local template_file=${2} + local image_name=${3} + local name_in_template=${4} + local expected_output=${5} + local port=${6:-8080} + local protocol=${7:-http} + local response_code=${8:-200} + local oc_args=${9:-} + local other_images=${10:-} + local result + local local_image_stream_file + local local_template_file + + echo "Running image stream test for stream ${image_stream_file} and quickstart template ${template_file}" + + # shellcheck disable=SC2119 + ct_os_new_project + + local_image_stream_file=$(ct_obtain_input "${image_stream_file}") + local_template_file=$(ct_obtain_input "${template_file}") + oc create -f "${local_image_stream_file}" + + # ct_os_test_template_app creates a new project, but we already need + # it before for the image stream import, so tell it to skip this time + CT_SKIP_NEW_PROJECT=true \ + ct_os_test_template_app "${image_name}" \ + "${local_template_file}" \ + "${name_in_template}" \ + "${expected_output}" \ + "${port}" "${protocol}" "${response_code}" "${oc_args}" "${other_images}" + + result=$? + + # shellcheck disable=SC2119 + ct_os_delete_project + + return $result +} + +# ct_os_service_image_info SERVICE_NAME +# -------------------- +# Shows information about the image used by a specified service. +# Argument: service_name - Service name (uesd for deployment config) +function ct_os_service_image_info() { + local service_name=$1 + local image_id + local namespace + + # get image ID from the deployment config + image_id=$(oc get "deploymentconfig.apps.openshift.io/${service_name}" -o custom-columns=IMAGE:.spec.template.spec.containers[*].image | tail -n 1) + namespace=${CT_NAMESPACE:-"$(oc project -q)"} + + echo " Information about the image we work with:" + oc get deploymentconfig.apps.openshift.io/"${service_name}" -o yaml | grep lastTriggeredImage + # for s2i builds, the resulting image is actually in the current namespace, + # so if the specified namespace does not succeed, try the current namespace + oc get isimage -n "${namespace}" "${image_id##*/}" -o yaml || oc get isimage "${image_id##*/}" -o yaml +} # vim: set tabstop=2:shiftwidth=2:expandtab: diff --git a/test/test-lib-python.sh b/test/test-lib-python.sh new file mode 100644 index 0000000..c86056a --- /dev/null +++ b/test/test-lib-python.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# +# Functions for tests for the Python image in OpenShift. +# +# IMAGE_NAME specifies a name of the candidate image used for testing. +# The image has to be available before this script is executed. +# + +THISDIR=$(dirname ${BASH_SOURCE[0]}) + +source "${THISDIR}/test-lib.sh" +source "${THISDIR}/test-lib-openshift.sh" + +# Check the imagestream +function test_python_imagestream() { + case ${OS} in + rhel7|centos7) ;; + *) echo "Imagestream testing not supported for $OS environment." ; return 0 ;; + esac + + ct_os_test_image_stream_quickstart "${THISDIR}/imagestreams/python-${OS}.json" \ + 'https://raw.githubusercontent.com/sclorg/django-ex/master/openshift/templates/django-postgresql.json' \ + "${IMAGE_NAME}" \ + 'python' \ + 'Welcome to your Django application on OpenShift' \ + 8080 http 200 "-p SOURCE_REPOSITORY_REF=master -p PYTHON_VERSION=${VERSION} -p POSTGRESQL_VERSION=9.6 -p NAME=python-testing" \ + "centos/postgresql-96-centos7|postgresql:9.6" +} + +# vim: set tabstop=2:shiftwidth=2:expandtab: + diff --git a/test/test-lib.sh b/test/test-lib.sh index b959c34..52bee15 100644 --- a/test/test-lib.sh +++ b/test/test-lib.sh @@ -50,6 +50,44 @@ function ct_enable_cleanup() { trap ct_cleanup EXIT SIGINT } +# ct_check_envs_set env_filter check_envs loop_envs [env_format] +# -------------------- +# Compares values from one list of environment variable definitions against such list, +# checking if the values are present and have a specific format. +# Argument: env_filter - optional string passed to grep used for +# choosing which variables to filter out in env var lists. +# Argument: check_envs - list of env var definitions to check values against +# Argument: loop_envs - list of env var definitions to check values for +# Argument: env_format (optional) - format string for bash substring deletion used +# for checking whether the value is contained in check_envs. +# Defaults to: "*VALUE*", VALUE string gets replaced by actual value from loop_envs +function ct_check_envs_set { + local env_filter check_envs env_format + env_filter=$1; shift + check_envs=$1; shift + loop_envs=$1; shift + env_format=${1:-"*VALUE*"} + while read -r variable; do + var_name=$(echo "$variable" | awk -F= '{ print $1 }') + stripped=$(echo "$variable" | awk -F= '{ print $2 }') + filtered_envs=$(echo "$check_envs" | grep "^$var_name=") + [ -z "$filtered_envs" ] && { echo "$var_name not found during \` docker exec\`"; return 1; } + old_IFS=$IFS + # For each such variable compare its content with the `docker exec` result, use `:` as delimiter + IFS=: + for value in $stripped; do + # If the falue checked does not go through env_filter we do not care about it + echo "$value" | grep -q "$env_filter" || continue + if [ -n "${filtered_envs##${env_format//VALUE/$value}}" ]; then + echo " Value $value is missing from variable $var_name" + echo "$filtered_envs" + return 1 + fi + done + done <<< "$(echo "$loop_envs" | grep "$env_filter" | grep -v "^PWD=")" + IFS=$old_IFS +} + # ct_get_cid [name] # -------------------- # Prints container id from cid_file based on the name of the file. @@ -285,6 +323,91 @@ function ct_npm_works() { : " Success!" } +# ct_binary_found_from_df binary [path] +# -------------------- +# Checks if a binary can be found in PATH during Dockerfile build +# Argument: binary - name of the binary to test accessibility for +# Argument: path - optional path in which the binary should reside in +# /opt/rh by default +function ct_binary_found_from_df() { + local tmpdir + local binary=$1; shift + local binary_path=${1:-"^/opt/rh"} + tmpdir=$(mktemp -d) + : " Testing $binary in build from Dockerfile" + + # Create Dockerfile that looks for the binary + cat <"$tmpdir/Dockerfile" +FROM $IMAGE_NAME +RUN which $binary | grep "$binary_path" +EOF + # Build an image, looking for expected path in the output + if ! docker build -f "$tmpdir/Dockerfile" --no-cache "$tmpdir"; then + echo " ERROR: Failed to find $binary in Dockerfile!" >&2 + return 1 + fi + : " Success!" +} + +# ct_check_exec_env_vars [env_filter] +# -------------------- +# Checks if all relevant environment variables from `docker run` +# can be found in `docker exec` as well. +# Argument: env_filter - optional string passed to grep used for +# choosing which variables to check in the test case. +# Defaults to X_SCLS and variables containing /opt/app-root, /opt/rh +# Uses: $CID_FILE_DIR - path to directory containing cid_files +# Uses: $IMAGE_NAME - name of the image being tested +function ct_check_exec_env_vars() { + local tmpdir exec_envs cid old_IFS env_filter + local var_name stripped filtered_envs run_envs + env_filter=${1:-"^X_SCLS=\|/opt/rh\|/opt/app-root"} + tmpdir=$(mktemp -d) + CID_FILE_DIR=${CID_FILE_DIR:-$(mktemp -d)} + # Get environment variables from `docker run` + run_envs=$(docker run --rm "$IMAGE_NAME" /bin/bash -c "env") + # Get environment variables from `docker exec` + ct_create_container "test_exec_envs" bash -c "sleep 1000" >/dev/null + cid=$(ct_get_cid "test_exec_envs") + exec_envs=$(docker exec "$cid" env) + # Filter out variables we are not interested in + # Always check X_SCLS, ignore PWD + # Check variables from `docker run` that have alternative paths inside (/opt/rh, /opt/app-root) + ct_check_envs_set "$env_filter" "$exec_envs" "$run_envs" "*VALUE*" || return 1 + echo " All values present in \`docker exec\`" + return 0 +} + +# ct_check_scl_enable_vars [env_filter] +# -------------------- +# Checks if all relevant environment variables from `docker run` +# are set twice after a second call of `scl enable $SCLS`. +# Argument: env_filter - optional string passed to grep used for +# choosing which variables to check in the test case. +# Defaults to paths containing enabled SCLS in the image +# Uses: $IMAGE_NAME - name of the image being tested +function ct_check_scl_enable_vars() { + local tmpdir exec_envs cid old_IFS env_filter enabled_scls + local var_name stripped filtered_envs loop_envs + env_filter=$1 + tmpdir=$(mktemp -d) + enabled_scls=$(docker run --rm "$IMAGE_NAME" /bin/bash -c "echo \$X_SCLS") + if [ -z "$env_filter" ]; then + for scl in $enabled_scls; do + [ -z "$env_filter" ] && env_filter="/$scl" && continue + # env_filter not empty, append to the existing list + env_filter="$env_filter|/$scl" + done + fi + # Get environment variables from `docker run` + loop_envs=$(docker run --rm "$IMAGE_NAME" /bin/bash -c "env") + run_envs=$(docker run --rm "$IMAGE_NAME" /bin/bash -c "X_SCLS= scl enable $enabled_scls env") + # Check if the values are set twice in the second set of envs + ct_check_envs_set "$env_filter" "$run_envs" "$loop_envs" "*VALUE*VALUE*" || return 1 + echo " All scl_enable values present" + return 0 +} + # ct_path_append PATH_VARNAME DIRECTORY # ------------------------------------- # Append DIRECTORY to VARIABLE of name PATH_VARNAME, the VARIABLE must consist