container/python3
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #485 from phracek/support_push_minimal_container_to_quay

Browse source 
Build and push python minimal image to quay.io/sclorg
This commit is contained in:
phracek 2021-12-13 12:37:35 +00:00
commit 9a3b04f941
7 changed files with 280 additions and 69 deletions
Download patch file Download diff file Expand all files Collapse all files

30
test/check_imagestreams.py
View file

@ -28,42 +28,47 @@ import logging
import os
from pathlib import Path
from typing import Dict
from typing import Dict, List, Any
IMAGESTREAMS_DIR: str = "imagestreams"
class ImageStreamChecker(object):
version: str = ""
results: Dict = {}
def __init__(self, version: str):
self.version = version
self.results: Dict[Any, Any] = {}
def load_json_file(self, filename: Path):
def load_json_file(self, filename: Path) -> Any:
with open(str(filename)) as f:
return json.load(f)
data = json.load(f)
isinstance(data, Dict)
return data
def check_version(self, json_dict: Dict):
def check_version(self, json_dict: Dict[Any, Any]) -> List[str]:
res = []
for tags in json_dict["spec"]["tags"]:
# The name can be"<stream>" or "<stream>-elX" or "<stream>-ubiX"
if (tags["name"] == self.version or
tags["name"].startswith(self.version + '-')):
if tags["name"] == self.version or tags["name"].startswith(
self.version + "-"
):
res.append(tags)
return res
def check_latest_tag(self, json_dict: Dict):
def check_latest_tag(self, json_dict: Dict[Any, Any]) -> bool:
latest_tag_correct: bool = False
for tags in json_dict["spec"]["tags"]:
if tags["name"] != "latest":
continue
# The latest can link to either "<stream>" or "<stream>-elX" or "<stream>-ubiX"
if tags["from"]["name"] == self.version or tags["from"]["name"].startswith(self.version + '-'):
if tags["from"]["name"] == self.version or tags["from"]["name"].startswith(
self.version + "-"
):
latest_tag_correct = True
return latest_tag_correct
def check_imagestreams(self):
def check_imagestreams(self) -> int:
p = Path(".")
json_files = p.glob(f"{IMAGESTREAMS_DIR}/*.json")
if not json_files:
@ -76,7 +81,9 @@ class ImageStreamChecker(object):
print(f"Checking file {str(f)}.")
json_dict = self.load_json_file(f)
if not (self.check_version(json_dict) and self.check_latest_tag(json_dict)):
print(f"The latest version is not present in {str(f)} or in latest tag.")
print(
f"The latest version is not present in {str(f)} or in latest tag."
)
self.results[f] = False
if self.results:
return 1
@ -92,4 +99,3 @@ if __name__ == "__main__":
print(f"Version to check is {sys.argv[1]}.")
isc = ImageStreamChecker(version=sys.argv[1])
sys.exit(isc.check_imagestreams())

6
test/setup-cfg-test-app/pyproject.toml Normal file
View file

@ -0,0 +1,6 @@
[build-system]
requires = [
"setuptools>=42",
"wheel"
]
build-backend = "setuptools.build_meta"

14
test/setup-cfg-test-app/setup.cfg Normal file
View file

@ -0,0 +1,14 @@
[metadata]
name = testapp
version = 0.1
[options]
install_requires =
gunicorn<20.0.0; python_version < '3.5'
gunicorn>=20.0.0; python_version >= '3.5'
package_dir =
= .
packages = find:
[options.packages.find]
where = .

4
test/setup-cfg-test-app/wsgi.py Normal file
View file

@ -0,0 +1,4 @@
def application(environ, start_response):
start_response('200 OK', [('Content-Type','text/plain')])
return [b"Hello from gunicorn WSGI application!"]

77
test/test-lib-openshift.sh
View file

@ -342,6 +342,10 @@ function _ct_os_get_uniq_project_name() {
# to authenticate to image registries.
# shellcheck disable=SC2120
function ct_os_new_project() {
if [ "${CVP:-0}" -eq "1" ]; then
echo "Testing in CVP environment. No need to create OpenShift project. This is done by CVP pipeline"
return
fi
if [ "${CT_SKIP_NEW_PROJECT:-false}" == 'true' ] ; then
echo "Creating project skipped."
return
@ -364,7 +368,7 @@ function ct_os_new_project() {
# Arguments: project - project name, uses the current project if omitted
# shellcheck disable=SC2120
function ct_os_delete_project() {
if [ "${CT_SKIP_NEW_PROJECT:-false}" == 'true' ] ; then
if [ "${CT_SKIP_NEW_PROJECT:-false}" == 'true' ] || [ "${CVP:-0}" -eq "1" ]; then
echo "Deleting project skipped, cleaning objects only."
# when not having enough privileges (remote cluster), it might fail and
# it is not a big problem, so ignore failure in this case
@ -526,6 +530,8 @@ function ct_os_cluster_up() {
# Shuts down the local OpenShift cluster using 'oc cluster down'
function ct_os_cluster_down() {
if [ ${OS_CLUSTER_STARTED_BY_TEST:-0} -eq 1 ] ; then
echo "Switching user to system:admin before cluster is going down."
oc login -u system:admin
echo "Cluster started by the test, shutting down."
oc cluster down
else
@ -665,20 +671,24 @@ function ct_os_test_s2i_app_func() {
namespace=${CT_NAMESPACE:-"$(oc project -q)"}
local image_tagged="${image_name_no_namespace%:*}:${VERSION}"
if [ "${CT_EXTERNAL_REGISTRY:-false}" == 'true' ] ; then
ct_os_import_image_ocp4 "${image_name}" "${image_tagged}"
else
# Create a specific imagestream tag for the image so that oc cannot use anything else
if [ "${CT_SKIP_UPLOAD_IMAGE:-false}" == 'true' ] ; then
echo "Importing image ${image_name} as ${namespace}/${image_tagged}"
# Use --reference-policy=local to pull remote image content to the cluster
# Works around the issue of builder pods not having access to registry.redhat.io
oc tag --source=docker "${image_name}" "${namespace}/${image_tagged}" --insecure=true --reference-policy=local
ct_os_wait_stream_ready "${image_tagged}" "${namespace}"
if [ "${CVP:-0}" -eq "0" ]; then
if [ "${CT_EXTERNAL_REGISTRY:-false}" == 'true' ] ; then
ct_os_import_image_ocp4 "${image_name}" "${image_tagged}"
else
echo "Uploading image ${image_name} as ${image_tagged}"
ct_os_upload_image "${image_name}" "${image_tagged}"
# Create a specific imagestream tag for the image so that oc cannot use anything else
if [ "${CT_SKIP_UPLOAD_IMAGE:-false}" == 'true' ] ; then
echo "Importing image ${image_name} as ${namespace}/${image_tagged}"
# Use --reference-policy=local to pull remote image content to the cluster
# Works around the issue of builder pods not having access to registry.redhat.io
oc tag --source=docker "${image_name}" "${namespace}/${image_tagged}" --insecure=true --reference-policy=local
ct_os_wait_stream_ready "${image_tagged}" "${namespace}"
else
echo "Uploading image ${image_name} as ${image_tagged}"
ct_os_upload_image "${image_name}" "${image_tagged}"
fi
fi
else
echo "Testing image ${image_name} in CVP pipeline."
fi
local app_param="${app}"
@ -803,21 +813,27 @@ function ct_os_test_template_app_func() {
ct_os_new_project
namespace=${CT_NAMESPACE:-"$(oc project -q)"}
# Create a specific imagestream tag for the image so that oc cannot use anything else
if [ "${CT_EXTERNAL_REGISTRY:-false}" == 'true' ] ; then
ct_os_import_image_ocp4 "${image_name}" "${image_tagged}"
else
if [ "${CT_SKIP_UPLOAD_IMAGE:-false}" == 'true' ] ; then
echo "Importing image ${image_name} as ${image_tagged}"
# Use --reference-policy=local to pull remote image content to the cluster
# Works around the issue of builder pods not having access to registry.redhat.io
oc tag --source=docker "${image_name}" "${namespace}/${image_tagged}" --insecure=true --reference-policy=local
ct_os_wait_stream_ready "${image_tagged}" "${namespace}"
# Upload main image is already done by CVP pipeline. No need to do it twice.
if [ "${CVP:-0}" -eq "0" ]; then
# Create a specific imagestream tag for the image so that oc cannot use anything else
if [ "${CT_EXTERNAL_REGISTRY:-false}" == 'true' ] ; then
ct_os_import_image_ocp4 "${image_name}" "${image_tagged}"
else
echo "Uploading image ${image_name} as ${image_tagged}"
ct_os_upload_image "${image_name}" "${image_tagged}"
if [ "${CT_SKIP_UPLOAD_IMAGE:-false}" == 'true' ] ; then
echo "Importing image ${image_name} as ${image_tagged}"
# Use --reference-policy=local to pull remote image content to the cluster
# Works around the issue of builder pods not having access to registry.redhat.io
oc tag --source=docker "${image_name}" "${namespace}/${image_tagged}" --insecure=true --reference-policy=local
ct_os_wait_stream_ready "${image_tagged}" "${namespace}"
else
echo "Uploading image ${image_name} as ${image_tagged}"
ct_os_upload_image "${image_name}" "${image_tagged}"
fi
fi
else
echo "Import is already done by CVP pipeline."
fi
# Other images are not uploaded by CVP pipeline. We need to do it.
if [ "${CT_SKIP_UPLOAD_IMAGE:-false}" == 'false' ] ; then
# upload also other images, that template might need (list of pairs in the format <image>|<tag>
local image_tag_a
@ -825,7 +841,14 @@ function ct_os_test_template_app_func() {
for i_t in ${other_images} ; do
echo "${i_t}"
IFS='|' read -ra image_tag_a <<< "${i_t}"
docker pull "${image_tag_a[0]}"
if [[ "$(docker images -q "$image_name" 2>/dev/null)" == "" ]]; then
echo "ERROR: Image $image_name is not pulled yet."
docker images
echo "Add to the beginning of scripts run-openshift-remote-cluster and run-openshift row"
echo "'ct_pull_image $image_name true'."
exit 1
fi
if [ "${CT_EXTERNAL_REGISTRY:-false}" == 'true' ] ; then
ct_os_import_image_ocp4 "${image_tag_a[0]}" "${image_tag_a[1]}"
else
@ -1043,7 +1066,7 @@ ct_os_test_response_internal() {
local status
local response_code
local response_file
local util_image_name='ubi7/ubi'
local util_image_name='registry.access.redhat.com/ubi7/ubi'
response_file=$(mktemp /tmp/ct_test_response_XXXXXX)
ct_os_deploy_cmd_image "${util_image_name}"

4
test/test-lib-remote-openshift.sh
View file

@ -47,6 +47,10 @@ function ct_os_set_path_oc_4() {
#
#
function ct_os_set_ocp4() {
if [ "${CVP:-0}" -eq "1" ]; then
echo "Testing in CVP environment. No need to login to OpenShift cluster. This is already done by CVP pipeline."
return
fi
local login
OS_OC_CLIENT_VERSION=${OS_OC_CLIENT_VERSION:-4.4}
ct_os_set_path_oc_4 "${OS_OC_CLIENT_VERSION}"

214
test/test-lib.sh
View file

@ -2,11 +2,11 @@
#
# Test a container image.
#
# Always use sourced from a specific container testfile
# Always use sourced from a specific container testfile
#
# reguires definition of CID_FILE_DIR
# CID_FILE_DIR=$(mktemp --suffix=<container>_test_cidfiles -d)
# reguires definition of TEST_LIST
# reguires definition of TEST_LIST
# TEST_LIST="\
# ctest_container_creation
# ctest_doc_content"
@ -52,6 +52,50 @@ function ct_enable_cleanup() {
trap ct_cleanup EXIT SIGINT
}
# ct_pull_image
# -------------
# Function pull an image before tests execution
# Argument: image_name - string containing the public name of the image to pull
# Argument: exit - in case "true" is defined and pull failed, then script has to exit with 1 and no tests are executed
# Argument: loops - how many times to pull image in case of failure
# Function returns either 0 in case of pull was successful
# Or the test suite exit with 1 in case of pull error
function ct_pull_image() {
local image_name="$1"; shift
local exit=${1:-"false"}; shift
local loops=${1:-10}; shift
local loop=0
# Let's try to pull image.
echo "-> Pulling image $image_name ..."
# Sometimes in Fedora case it fails with HTTP 50X
# Check if the image is available locally and try to pull it if it is not
if [[ "$(docker images -q "$image_name" 2>/dev/null)" != "" ]]; then
echo "The image $image_name is already pulled."
return 0
fi
# Try pulling the image to see if it is accessible
# WORKAROUND: Since Fedora registry sometimes fails randomly, let's try it more times
while ! docker pull "$image_name"; do
((loop++)) || :
echo "Pulling image $image_name failed."
if [ "$loop" -gt "$loops" ]; then
echo "Pulling of image $image_name failed $loops times in a row. Giving up."
echo "!!! ERROR with pulling image $image_name !!!!"
# shellcheck disable=SC2268
if [[ x"$exit" == x"false" ]]; then
return 1
else
exit 1
fi
fi
echo "Let's wait $((loop*5)) seconds and try again."
sleep "$((loop*5))"
done
}
# ct_check_envs_set env_filter check_envs loop_envs [env_format]
# --------------------
# Compares values from one list of environment variable definitions against such list,
@ -185,7 +229,7 @@ function ct_assert_container_creation_fails() {
function ct_create_container() {
local cid_file="$CID_FILE_DIR/$1" ; shift
# create container with a cidfile in a directory for cleanup
# shellcheck disable=SC2086
# shellcheck disable=SC2086,SC2153
docker run --cidfile="$cid_file" -d ${CONTAINER_ARGS:-} "$IMAGE_NAME" "$@"
ct_wait_for_cid "$cid_file" || return 1
: "Created container $(cat "$cid_file")"
@ -310,6 +354,7 @@ function ct_npm_works() {
if ! docker exec "$(cat "$cid_file")" /bin/bash -c "npm --verbose install jquery && test -f node_modules/jquery/src/jquery.js" >"${tmpdir}/jquery" 2>&1 ; then
echo "ERROR: npm could not install jquery inside the image ${IMAGE_NAME}." >&2
cat "${tmpdir}/jquery"
return 1
fi
@ -582,13 +627,13 @@ ct_get_public_image_name() {
local registry
registry=$(ct_registry_from_os "$os")
if [ "x$os" == "xrhel7" ]; then
if [ "$os" == "rhel7" ]; then
public_image_name=$registry/rhscl/$base_image_name-${version//./}-rhel7
elif [ "x$os" == "xrhel8" ]; then
elif [ "$os" == "rhel8" ]; then
public_image_name=$registry/rhel8/$base_image_name-${version//./}
elif [ "x$os" == "xcentos7" ]; then
elif [ "$os" == "centos7" ]; then
public_image_name=$registry/centos7/$base_image_name-${version//./}-centos7
elif [ "x$os" == "xcentos8" ]; then
elif [ "$os" == "centos8" ]; then
public_image_name=$registry/centos8/$base_image_name-${version//./}-centos8
fi
@ -670,7 +715,7 @@ ct_s2i_build_as_df()
local df_name=
local tmpdir=
local incremental=false
local mount_options=""
local mount_options=()
# Run the entire thing inside a subshell so that we do not leak shell options outside of the function
(
@ -687,14 +732,11 @@ ct_s2i_build_as_df()
# Default to root if no user is set by the image
user=${user:-0}
# run the user through the image in case it is non-numeric or does not exist
# NOTE: The '-eq' test is used to check if $user is numeric as it will fail if $user is not an integer
if ! [ "$user" -eq "$user" ] 2>/dev/null && ! user_id=$(docker run --rm "$src_image" bash -c "id -u $user 2>/dev/null"); then
echo "ERROR: id of user $user not found inside image $src_image."
if ! user_id=$(ct_get_uid_from_image "$user" "$src_image"); then
echo "Terminating s2i build."
return 1
else
user_id=${user_id:-$user}
fi
echo "$s2i_args" | grep -q "\-\-incremental" && incremental=true
if $incremental; then
inc_tmp=$(mktemp -d --tmpdir incremental.XXXX)
@ -757,14 +799,93 @@ EOF
fi
# Check if -v parameter is present in s2i_args and add it into docker build command
mount_options=$(echo "$s2i_args" | grep -o -e '\(-v\)[[:space:]]\.*\S*' || true)
read -ra mount_options <<< "$(echo "$s2i_args" | grep -o -e '\(-v\)[[:space:]]\.*\S*' || true)"
# Run the build and tag the result
# shellcheck disable=SC2086
docker build $mount_options -f "$df_name" --no-cache=true -t "$dst_image" .
docker build ${mount_options[@]+"${mount_options[@]}"} -f "$df_name" --no-cache=true -t "$dst_image" .
)
}
# ct_s2i_multistage_build APP_PATH SRC_IMAGE DST_IMAGE SEC_IMAGE [S2I_ARGS]
# ----------------------------
# Create a new s2i app image from local sources in a similar way as source-to-image would have used.
# Argument: APP_PATH - local path to the app sources to be used in the test
# Argument: SRC_IMAGE - image to be used as a base for the s2i build process
# Argument: SEC_IMAGE - image to be used as the base for the result of the build process
# Argument: DST_IMAGE - image name to be used during the tagging of the s2i build result
# Argument: S2I_ARGS - Additional list of source-to-image arguments.
# Only used to check for environment variable definitions.
ct_s2i_multistage_build() {
local app_path=$1; shift
local src_image=$1; shift
local sec_image=$1; shift
local dst_image=$1; shift
local s2i_args=$*;
local local_app="app-src"
local user_id=
local mount_options=()
# Run the entire thing inside a subshell so that we do not leak shell options outside of the function
(
# Error out if any part of the build fails
set -e
user=$(docker inspect -f "{{.Config.User}}" "$src_image")
# Default to root if no user is set by the image
user=${user:-0}
# run the user through the image in case it is non-numeric or does not exist
if ! user_id=$(ct_get_uid_from_image "$user" "$src_image"); then
echo "Terminating s2i build."
return 1
fi
# Use /tmp to not pollute cwd
tmpdir=$(mktemp -d)
df_name=$(mktemp -p "$tmpdir" Dockerfile.XXXX)
cd "$tmpdir"
# If the path exists on the local host, copy it into the directory for the build
# Otherwise handle it as a link to a git repository
if [ -e "${app_path/file:\/\//}/." ] ; then
mkdir -p "$local_app"
# Strip file:// from APP_PATH and copy its contents into current context
cp -r "${app_path/file:\/\//}/." "$local_app"
else
ct_clone_git_repository "$app_path" "$local_app"
fi
cat <<EOF >"$df_name"
# First stage builds the application
FROM $src_image as builder
# Add application sources to a directory that the assemble script expects them
# and set permissions so that the container runs without root access
USER 0
ADD app-src /tmp/src
RUN chown -R 1001:0 /tmp/src
$(echo "$s2i_args" | grep -o -e '\(-e\|--env\)[[:space:]=]\S*=\S*' | sed -e 's/-e /ENV /' -e 's/--env[ =]/ENV /')
# Check if CA autority is present on host and add it into Dockerfile
$([ -f "$(full_ca_file_path)" ] && echo "RUN cd /etc/pki/ca-trust/source/anchors && update-ca-trust extract")
USER $user_id
# Install the dependencies
RUN /usr/libexec/s2i/assemble
# Second stage copies the application to the minimal image
FROM $sec_image
# Copy the application source and build artifacts from the builder image to this one
COPY --from=builder \$HOME \$HOME
# Set the default command for the resulting image
CMD /usr/libexec/s2i/run
EOF
# Check if -v parameter is present in s2i_args and add it into docker build command
read -ra mount_options <<< "$(echo "$s2i_args" | grep -o -e '\(-v\)[[:space:]]\.*\S*' || true)"
docker build ${mount_options[@]+"${mount_options[@]}"} -f "$df_name" --no-cache=true -t "$dst_image" .
)
}
# ct_check_image_availability PUBLIC_IMAGE_NAME
# ----------------------------
# Pull an image from the public repositories to see if the image is already available.
@ -773,7 +894,7 @@ ct_check_image_availability() {
local public_image_name=$1;
# Try pulling the image to see if it is accessible
if ! docker pull "$public_image_name" &>/dev/null; then
if ! ct_pull_image "$public_image_name" &>/dev/null; then
echo "$public_image_name could not be downloaded via 'docker'"
return 1
fi
@ -823,6 +944,50 @@ ct_show_resources()
lscpu
}
# ct_clone_git_repository
# -----------------------------
# Argument: app_url - git URI pointing to a repository, supports "@" to indicate a different branch
# Argument: app_dir (optional) - name of the directory to clone the repository into
ct_clone_git_repository()
{
local app_url=$1; shift
local app_dir=$1
# If app_url contains @, the string after @ is considered
# as a name of a branch to clone instead of the main/master branch
IFS='@' read -ra git_url_parts <<< "${app_url}"
if [ -n "${git_url_parts[1]}" ]; then
git_clone_cmd="git clone --branch ${git_url_parts[1]} ${git_url_parts[0]} ${app_dir}"
else
git_clone_cmd="git clone ${app_url} ${app_dir}"
fi
if ! $git_clone_cmd ; then
echo "ERROR: Git repository ${app_url} cannot be cloned into ${app_dir}."
return 1
fi
}
# ct_get_uid_from_image
# -----------------------------
# Argument: user - user to get uid for inside the image
# Argument: src_image - image to use for user information
ct_get_uid_from_image()
{
local user=$1; shift
local src_image=$1
local user_id=
# NOTE: The '-eq' test is used to check if $user is numeric as it will fail if $user is not an integer
if ! [ "$user" -eq "$user" ] 2>/dev/null && ! user_id=$(docker run --rm "$src_image" bash -c "id -u $user 2>/dev/null"); then
echo "ERROR: id of user $user not found inside image $src_image."
return 1
else
echo "${user_id:-$user}"
fi
}
# ct_test_app_dockerfile
# -----------------------------
# Argument: dockerfile - path to a Dockerfile that will be used for building an image
@ -867,20 +1032,9 @@ ct_test_app_dockerfile() {
if [ -d "$app_url" ] ; then
echo "Copying local folder: $app_url -> $app_dir."
cp -Lr $app_url $app_dir
cp -Lr "$app_url" "$app_dir"
else
# If app_url contains @, the string after @ is considered
# as a name of a branch to clone instead of the main/master branch
IFS='@' read -ra git_url_parts <<< "${app_url}"
if [ -n "${git_url_parts[1]}" ]; then
git_clone_cmd="git clone --branch ${git_url_parts[1]} ${git_url_parts[0]} ${app_dir}"
else
git_clone_cmd="git clone ${app_url} ${app_dir}"
fi
if ! $git_clone_cmd ; then
echo "ERROR: Git repository ${app_url} cannot be cloned into ${app_dir}."
if ! ct_clone_git_repository "$app_url" "$app_dir" ; then
echo "Terminating the Dockerfile build."
return 1
fi