Merge pull request #119 from sclorg/use_internal_openshift_register
Use internal openshift register instead of external
This commit is contained in:
phracek
parent
5b8f63e1ef
commit
1d016557d1
11 changed files with 444 additions and 164 deletions
|
|
@ -1,4 +1,4 @@
|
|||
FROM registry.fedoraproject.org/f34/s2i-core:latest
|
||||
FROM quay.io/fedora/s2i-core:latest
|
||||
|
||||
# Redis image based on Software Collections packages
|
||||
#
|
||||
|
|
@ -8,7 +8,6 @@ FROM registry.fedoraproject.org/f34/s2i-core:latest
|
|||
# * $REDIS_PASSWORD - Database password
|
||||
|
||||
ENV NAME=redis \
|
||||
RELEASE=1 \
|
||||
VERSION=6
|
||||
|
||||
ENV REDIS_VERSION=$VERSION \
|
||||
|
|
@ -30,9 +29,9 @@ LABEL summary="$SUMMARY" \
|
|||
io.openshift.expose-services="6379:redis" \
|
||||
io.openshift.tags="database,redis,redis6" \
|
||||
com.redhat.component="$NAME" \
|
||||
name="$FGC/$NAME" \
|
||||
name="fedora/$NAME-$VERSION" \
|
||||
version="$VERSION" \
|
||||
usage="docker run -d --name redis_database -p 6379:6379 $FGC/$NAME" \
|
||||
usage="docker run -d --name redis_database -p 6379:6379 quay.io/fedora/$NAME-$VERSION" \
|
||||
maintainer="SoftwareCollections.org <sclorg@redhat.com>"
|
||||
|
||||
EXPOSE 6379
|
||||
|
|
|
|||
76
Dockerfile.c8s
Normal file
76
Dockerfile.c8s
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
FROM quay.io/sclorg/s2i-core-c8s:c8s
|
||||
|
||||
# Redis image based on Software Collections packages
|
||||
#
|
||||
# Volumes:
|
||||
# * /var/lib/redis/data - Datastore for Redis
|
||||
# Environment:
|
||||
# * $REDIS_PASSWORD - Database password
|
||||
|
||||
ENV REDIS_VERSION=6 \
|
||||
HOME=/var/lib/redis
|
||||
|
||||
ENV SUMMARY="Redis in-memory data structure store, used as database, cache and message broker" \
|
||||
DESCRIPTION="Redis $REDIS_VERSION available as container, is an advanced key-value store. \
|
||||
It is often referred to as a data structure server since keys can contain strings, hashes, lists, \
|
||||
sets and sorted sets. You can run atomic operations on these types, like appending to a string; \
|
||||
incrementing the value in a hash; pushing to a list; computing set intersection, union and difference; \
|
||||
or getting the member with highest ranking in a sorted set. In order to achieve its outstanding \
|
||||
performance, Redis works with an in-memory dataset. Depending on your use case, you can persist \
|
||||
it either by dumping the dataset to disk every once in a while, or by appending each command to a log."
|
||||
|
||||
LABEL summary="$SUMMARY" \
|
||||
description="$DESCRIPTION" \
|
||||
io.k8s.description="$DESCRIPTION" \
|
||||
io.k8s.display-name="Redis $REDIS_VERSION" \
|
||||
io.openshift.expose-services="6379:redis" \
|
||||
io.openshift.tags="database,redis,redis$REDIS_VERSION,redis-$REDIS_VERSION" \
|
||||
com.redhat.component="redis-$REDIS_VERSION-container" \
|
||||
name="sclorg/redis-$REDIS_VERSION-c8s" \
|
||||
version="1" \
|
||||
usage="podman run -d --name redis_database -p 6379:6379 sclorg/redis-$REDIS_VERSION-c8s" \
|
||||
maintainer="SoftwareCollections.org <sclorg@redhat.com>"
|
||||
|
||||
EXPOSE 6379
|
||||
|
||||
# Create user for redis that has known UID
|
||||
# We need to do this before installing the RPMs which would create user with random UID
|
||||
# The UID is the one used by the default user from the parent layer (1001),
|
||||
# and since the user exists already, do not create a new one, but only rename
|
||||
# the existing
|
||||
# This image must forever use UID 1001 for redis user so our volumes are
|
||||
# safe in the future. This should *never* change, the last test is there
|
||||
# to make sure of that.
|
||||
RUN getent group redis &> /dev/null || groupadd -r redis &> /dev/null && \
|
||||
usermod -l redis -aG redis -c 'Redis Server' default &> /dev/null && \
|
||||
# Install gettext for envsubst command
|
||||
yum -y module enable redis:$REDIS_VERSION && \
|
||||
INSTALL_PKGS="policycoreutils redis" && \
|
||||
yum install -y --setopt=tsflags=nodocs $INSTALL_PKGS && \
|
||||
rpm -V $INSTALL_PKGS && \
|
||||
yum -y clean all --enablerepo='*' && \
|
||||
mkdir -p /var/lib/redis/data && chown -R redis.0 /var/lib/redis && \
|
||||
[[ "$(id redis)" == "uid=1001(redis)"* ]]
|
||||
|
||||
# Get prefix path and path to scripts rather than hard-code them in scripts
|
||||
ENV CONTAINER_SCRIPTS_PATH=/usr/share/container-scripts/redis \
|
||||
REDIS_PREFIX=/usr \
|
||||
REDIS_CONF=/etc/redis.conf
|
||||
|
||||
COPY root /
|
||||
|
||||
# this is needed due to issues with squash
|
||||
# when this directory gets rm'd by the container-setup
|
||||
# script.
|
||||
RUN /usr/libexec/container-setup
|
||||
|
||||
VOLUME ["/var/lib/redis/data"]
|
||||
|
||||
# Using a numeric value because of a comment in [1]:
|
||||
# If your S2I image does not include a USER declaration with a numeric user,
|
||||
# your builds will fail by default.
|
||||
# [1] https://docs.openshift.com/container-platform/4.4/openshift_images/create-images.html#images-create-guide-openshift_create-images
|
||||
USER 1001
|
||||
|
||||
ENTRYPOINT ["container-entrypoint"]
|
||||
CMD ["run-redis"]
|
||||
76
Dockerfile.c9s
Normal file
76
Dockerfile.c9s
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
FROM quay.io/sclorg/s2i-core-c9s:c9s
|
||||
|
||||
# Redis image based on Software Collections packages
|
||||
#
|
||||
# Volumes:
|
||||
# * /var/lib/redis/data - Datastore for Redis
|
||||
# Environment:
|
||||
# * $REDIS_PASSWORD - Database password
|
||||
|
||||
ENV REDIS_VERSION=6 \
|
||||
HOME=/var/lib/redis
|
||||
|
||||
ENV SUMMARY="Redis in-memory data structure store, used as database, cache and message broker" \
|
||||
DESCRIPTION="Redis $REDIS_VERSION available as container, is an advanced key-value store. \
|
||||
It is often referred to as a data structure server since keys can contain strings, hashes, lists, \
|
||||
sets and sorted sets. You can run atomic operations on these types, like appending to a string; \
|
||||
incrementing the value in a hash; pushing to a list; computing set intersection, union and difference; \
|
||||
or getting the member with highest ranking in a sorted set. In order to achieve its outstanding \
|
||||
performance, Redis works with an in-memory dataset. Depending on your use case, you can persist \
|
||||
it either by dumping the dataset to disk every once in a while, or by appending each command to a log."
|
||||
|
||||
LABEL summary="$SUMMARY" \
|
||||
description="$DESCRIPTION" \
|
||||
io.k8s.description="$DESCRIPTION" \
|
||||
io.k8s.display-name="Redis $REDIS_VERSION" \
|
||||
io.openshift.expose-services="6379:redis" \
|
||||
io.openshift.tags="database,redis,redis$REDIS_VERSION,redis-$REDIS_VERSION" \
|
||||
com.redhat.component="redis-$REDIS_VERSION-container" \
|
||||
name="sclorg/redis-$REDIS_VERSION-c9s" \
|
||||
version="1" \
|
||||
com.redhat.license_terms="https://www.redhat.com/en/about/red-hat-end-user-license-agreements#rhel" \
|
||||
usage="podman run -d --name redis_database -p 6379:6379 quay.io/sclorg/redis-$REDIS_VERSION-c9s" \
|
||||
maintainer="SoftwareCollections.org <sclorg@redhat.com>"
|
||||
|
||||
EXPOSE 6379
|
||||
|
||||
# Create user for redis that has known UID
|
||||
# We need to do this before installing the RPMs which would create user with random UID
|
||||
# The UID is the one used by the default user from the parent layer (1001),
|
||||
# and since the user exists already, do not create a new one, but only rename
|
||||
# the existing
|
||||
# This image must forever use UID 1001 for redis user so our volumes are
|
||||
# safe in the future. This should *never* change, the last test is there
|
||||
# to make sure of that.
|
||||
RUN getent group redis &> /dev/null || groupadd -r redis &> /dev/null && \
|
||||
usermod -l redis -aG redis -c 'Redis Server' default &> /dev/null && \
|
||||
# Install gettext for envsubst command
|
||||
INSTALL_PKGS="policycoreutils redis" && \
|
||||
yum install -y --setopt=tsflags=nodocs $INSTALL_PKGS && \
|
||||
rpm -V $INSTALL_PKGS && \
|
||||
yum -y clean all --enablerepo='*' && \
|
||||
mkdir -p /var/lib/redis/data && chown -R redis.0 /var/lib/redis && \
|
||||
[[ "$(id redis)" == "uid=1001(redis)"* ]]
|
||||
|
||||
# Get prefix path and path to scripts rather than hard-code them in scripts
|
||||
ENV CONTAINER_SCRIPTS_PATH=/usr/share/container-scripts/redis \
|
||||
REDIS_PREFIX=/usr \
|
||||
REDIS_CONF=/etc/redis/redis.conf
|
||||
|
||||
COPY root /
|
||||
|
||||
# this is needed due to issues with squash
|
||||
# when this directory gets rm'd by the container-setup
|
||||
# script.
|
||||
RUN /usr/libexec/container-setup
|
||||
|
||||
VOLUME ["/var/lib/redis/data"]
|
||||
|
||||
# Using a numeric value because of a comment in [1]:
|
||||
# If your S2I image does not include a USER declaration with a numeric user,
|
||||
# your builds will fail by default.
|
||||
# [1] https://docs.openshift.com/container-platform/4.4/openshift_images/create-images.html#images-create-guide-openshift_create-images
|
||||
USER 1001
|
||||
|
||||
ENTRYPOINT ["container-entrypoint"]
|
||||
CMD ["run-redis"]
|
||||
|
|
@ -82,4 +82,5 @@ Dockerfile and other sources for this container image are available on
|
|||
https://github.com/sclorg/redis-container.
|
||||
In that repository you also can find another versions of Python environment Dockerfiles.
|
||||
Dockerfile for CentOS is called `Dockerfile`, Dockerfile for RHEL7 is called `Dockerfile.rhel7`,
|
||||
for RHEL8 it's `Dockerfile.rhel8` and the Fedora Dockerfile is called Dockerfile.fedora.
|
||||
for RHEL8 it's `Dockerfile.rhel8`, for CentOS Stream 8 it's `Dockerfile.c8s`,
|
||||
for CentOS Stream 9 it's `Dockerfile.c9s` and the Fedora Dockerfile is called Dockerfile.fedora.
|
||||
|
|
|
|||
|
|
@ -26,6 +26,24 @@
|
|||
},
|
||||
"name": "latest"
|
||||
},
|
||||
{
|
||||
"annotations": {
|
||||
"description": "Provides a Redis 6 database on RHEL 9. For more information about using this database image, including OpenShift considerations, see https://github.com/sclorg/redis-container/tree/master/6/README.md.",
|
||||
"iconClass": "icon-redis",
|
||||
"openshift.io/display-name": "Redis 6 (RHEL 9)",
|
||||
"openshift.io/provider-display-name": "Red Hat, Inc.",
|
||||
"tags": "redis",
|
||||
"version": "6"
|
||||
},
|
||||
"from": {
|
||||
"kind": "DockerImage",
|
||||
"name": "registry.redhat.io/rhel9/redis-6:latest"
|
||||
},
|
||||
"referencePolicy": {
|
||||
"type": "Local"
|
||||
},
|
||||
"name": "6-el9"
|
||||
},
|
||||
{
|
||||
"annotations": {
|
||||
"description": "Provides a Redis 6 database on RHEL 8. For more information about using this database image, including OpenShift considerations, see https://github.com/sclorg/redis-container/tree/master/6/README.md.",
|
||||
|
|
@ -43,24 +61,6 @@
|
|||
"type": "Local"
|
||||
},
|
||||
"name": "6-el8"
|
||||
},
|
||||
{
|
||||
"annotations": {
|
||||
"description": "Provides a Redis 5 database on RHEL 8. For more information about using this database image, including OpenShift considerations, see https://github.com/sclorg/redis-container/tree/master/5/README.md.",
|
||||
"iconClass": "icon-redis",
|
||||
"openshift.io/display-name": "Redis 5 (RHEL 8)",
|
||||
"openshift.io/provider-display-name": "Red Hat, Inc.",
|
||||
"tags": "redis",
|
||||
"version": "5"
|
||||
},
|
||||
"from": {
|
||||
"kind": "DockerImage",
|
||||
"name": "registry.redhat.io/rhel8/redis-5:latest"
|
||||
},
|
||||
"referencePolicy": {
|
||||
"type": "Local"
|
||||
},
|
||||
"name": "5-el8"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,6 +26,24 @@
|
|||
},
|
||||
"name": "latest"
|
||||
},
|
||||
{
|
||||
"annotations": {
|
||||
"description": "Provides a Redis 6 database on RHEL 9. For more information about using this database image, including OpenShift considerations, see https://github.com/sclorg/redis-container/tree/master/6/README.md.",
|
||||
"iconClass": "icon-redis",
|
||||
"openshift.io/display-name": "Redis 6 (RHEL 9)",
|
||||
"openshift.io/provider-display-name": "Red Hat, Inc.",
|
||||
"tags": "redis",
|
||||
"version": "6"
|
||||
},
|
||||
"from": {
|
||||
"kind": "DockerImage",
|
||||
"name": "registry.redhat.io/rhel9/redis-6:latest"
|
||||
},
|
||||
"referencePolicy": {
|
||||
"type": "Local"
|
||||
},
|
||||
"name": "6-el9"
|
||||
},
|
||||
{
|
||||
"annotations": {
|
||||
"description": "Provides a Redis 6 database on RHEL 8. For more information about using this database image, including OpenShift considerations, see https://github.com/sclorg/redis-container/tree/master/6/README.md.",
|
||||
|
|
@ -61,60 +79,6 @@
|
|||
"type": "Local"
|
||||
},
|
||||
"name": "6-el7"
|
||||
},
|
||||
{
|
||||
"annotations": {
|
||||
"description": "Provides a Redis 5 database on RHEL 8. For more information about using this database image, including OpenShift considerations, see https://github.com/sclorg/redis-container/tree/master/5/README.md.",
|
||||
"iconClass": "icon-redis",
|
||||
"openshift.io/display-name": "Redis 5 (RHEL 8)",
|
||||
"openshift.io/provider-display-name": "Red Hat, Inc.",
|
||||
"tags": "redis",
|
||||
"version": "5"
|
||||
},
|
||||
"from": {
|
||||
"kind": "DockerImage",
|
||||
"name": "registry.redhat.io/rhel8/redis-5:latest"
|
||||
},
|
||||
"referencePolicy": {
|
||||
"type": "Local"
|
||||
},
|
||||
"name": "5-el8"
|
||||
},
|
||||
{
|
||||
"annotations": {
|
||||
"description": "Provides a Redis 5 database on RHEL 7. For more information about using this database image, including OpenShift considerations, see https://github.com/sclorg/redis-container/tree/master/5/README.md.",
|
||||
"iconClass": "icon-redis",
|
||||
"openshift.io/display-name": "Redis 5 (RHEL 7)",
|
||||
"openshift.io/provider-display-name": "Red Hat, Inc.",
|
||||
"tags": "redis",
|
||||
"version": "5"
|
||||
},
|
||||
"from": {
|
||||
"kind": "DockerImage",
|
||||
"name": "registry.redhat.io/rhscl/redis-5-rhel7:latest"
|
||||
},
|
||||
"referencePolicy": {
|
||||
"type": "Local"
|
||||
},
|
||||
"name": "5-el7"
|
||||
},
|
||||
{
|
||||
"annotations": {
|
||||
"description": "Provides a Redis 5 database on RHEL 7. For more information about using this database image, including OpenShift considerations, see https://github.com/sclorg/redis-container/tree/master/5/README.md.",
|
||||
"iconClass": "icon-redis",
|
||||
"openshift.io/display-name": "Redis 5",
|
||||
"openshift.io/provider-display-name": "Red Hat, Inc.",
|
||||
"tags": "redis,hidden",
|
||||
"version": "5"
|
||||
},
|
||||
"from": {
|
||||
"kind": "DockerImage",
|
||||
"name": "registry.redhat.io/rhscl/redis-5-rhel7:latest"
|
||||
},
|
||||
"referencePolicy": {
|
||||
"type": "Local"
|
||||
},
|
||||
"name": "5"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,7 +12,10 @@ THISDIR=$(dirname ${BASH_SOURCE[0]})
|
|||
|
||||
source ${THISDIR}/test-lib-redis.sh
|
||||
|
||||
set -eo nounset
|
||||
TEST_LIST="\
|
||||
test_redis_integration
|
||||
test_redis_imagestream
|
||||
"
|
||||
|
||||
trap ct_os_cleanup EXIT SIGINT
|
||||
|
||||
|
|
@ -20,18 +23,15 @@ ct_os_set_ocp4
|
|||
|
||||
ct_os_check_compulsory_vars
|
||||
|
||||
oc status || false "It looks like oc is not properly logged in."
|
||||
ct_os_check_login || exit 1
|
||||
|
||||
# For testing on OpenShift 4 we use external registry
|
||||
export CT_EXTERNAL_REGISTRY=true
|
||||
set -u
|
||||
|
||||
# Check the template
|
||||
test_redis_integration "${IMAGE_NAME}"
|
||||
# For testing on OpenShift 4 we use internal registry
|
||||
export CT_OCP4_TEST=true
|
||||
|
||||
# Check the imagestream
|
||||
test_redis_imagestream
|
||||
|
||||
OS_TESTSUITE_RESULT=0
|
||||
TEST_SUMMARY=''
|
||||
TEST_SET=${TESTS:-$TEST_LIST} ct_run_tests_from_testset "openshift-remote-cluster"
|
||||
|
||||
# vim: set tabstop=2:shiftwidth=2:expandtab:
|
||||
|
||||
|
|
|
|||
|
|
@ -5,21 +5,23 @@ source "$(dirname "${BASH_SOURCE[0]}")"/test-lib.sh
|
|||
|
||||
# Set of functions for testing docker images in OpenShift using 'oc' command
|
||||
|
||||
# A variable containing the overall test result; must be changed to 0 in the end
|
||||
# of the testing script:
|
||||
# OS_TESTSUITE_RESULT=0
|
||||
# A variable containing the overall test result
|
||||
# TESTSUITE_RESULT=0
|
||||
# And the following trap must be set, in the beginning of the test script:
|
||||
# trap ct_os_cleanup EXIT SIGINT
|
||||
OS_TESTSUITE_RESULT=1
|
||||
TESTSUITE_RESULT=0
|
||||
OS_CLUSTER_STARTED_BY_TEST=0
|
||||
|
||||
function ct_os_cleanup() {
|
||||
if [ $OS_TESTSUITE_RESULT -eq 0 ] ; then
|
||||
echo "${TEST_SUMMARY:-}"
|
||||
if [ $TESTSUITE_RESULT -eq 0 ] ; then
|
||||
# shellcheck disable=SC2153
|
||||
echo "OpenShift tests for ${IMAGE_NAME} succeeded."
|
||||
exit 0
|
||||
else
|
||||
# shellcheck disable=SC2153
|
||||
echo "OpenShift tests for ${IMAGE_NAME} failed."
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
|
|
@ -44,6 +46,7 @@ function ct_os_check_compulsory_vars() {
|
|||
function ct_os_get_status() {
|
||||
oc get all
|
||||
oc status
|
||||
oc status --suggest
|
||||
}
|
||||
|
||||
# ct_os_print_logs
|
||||
|
|
@ -397,13 +400,13 @@ function ct_delete_all_objects() {
|
|||
sleep 10
|
||||
}
|
||||
|
||||
# ct_os_docker_login
|
||||
# ct_os_docker_login_v3
|
||||
# --------------------
|
||||
# Logs in into docker daemon
|
||||
# Uses global REGISRTY_ADDRESS environment variable for arbitrary registry address.
|
||||
# Does not do anything if REGISTRY_ADDRESS is set.
|
||||
function ct_os_docker_login() {
|
||||
[ -n "${REGISTRY_ADDRESS:-}" ] && "REGISTRY_ADDRESS set, not trying to docker login." && return 0
|
||||
function ct_os_docker_login_v3() {
|
||||
[ -n "${REGISTRY_ADDRESS:-}" ] && echo "REGISTRY_ADDRESS set, not trying to docker login." && return 0
|
||||
# docker login fails with "404 page not found" error sometimes, just try it more times
|
||||
# shellcheck disable=SC2034
|
||||
for i in $(seq 12) ; do
|
||||
|
|
@ -414,6 +417,26 @@ function ct_os_docker_login() {
|
|||
return 1
|
||||
}
|
||||
|
||||
# ct_os_docker_login_v4
|
||||
# --------------------
|
||||
# Logs in into docker daemon
|
||||
# Uses global REGISRTY_ADDRESS environment variable for arbitrary registry address.
|
||||
# Does not do anything if REGISTRY_ADDRESS is set.
|
||||
function ct_os_docker_login_v4() {
|
||||
OCP4_REGISTER=$(oc get route default-route -n openshift-image-registry --template='{{ .spec.host }}')
|
||||
echo "OCP4 loging address is $OCP4_REGISTER."
|
||||
if [ -z "${OCP4_REGISTER}" ]; then
|
||||
echo "!!!OpenShift 4 registry address not found. This is an error. Check OpenShift 4 cluster!!!"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if docker login -u kubeadmin -p "$(oc whoami -t)" "${OCP4_REGISTER}"; then
|
||||
echo "Login to $OCP4_REGISTER was successfully."
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
# ct_os_upload_image IMAGE [IMAGESTREAM]
|
||||
# --------------------
|
||||
# Uploads image from local registry to the OpenShift internal registry.
|
||||
|
|
@ -422,15 +445,36 @@ function ct_os_docker_login() {
|
|||
# In the format of name:tag ($image_name:latest by default)
|
||||
# Uses global REGISRTY_ADDRESS environment variable for arbitrary registry address.
|
||||
function ct_os_upload_image() {
|
||||
local os_version="${1}" ; shift
|
||||
local input_name="${1}" ; shift
|
||||
local image_name=${input_name##*/}
|
||||
local imagestream=${1:-$image_name:latest}
|
||||
local image_name=${1}
|
||||
local output_name
|
||||
local source_name
|
||||
local docker_options=""
|
||||
|
||||
output_name="${REGISRTY_ADDRESS:-172.30.1.1:5000}/$(oc project -q)/$imagestream"
|
||||
if [ "${os_version}" != "v3" ] && [ "${os_version}" != "v4" ]; then
|
||||
echo "You have to specify OpenShift version to upload an image."
|
||||
echo "Either 'v3' or 'v4' is allowed"
|
||||
return 1
|
||||
fi
|
||||
|
||||
ct_os_docker_login
|
||||
docker tag "${input_name}" "${output_name}"
|
||||
source_name="${input_name}"
|
||||
if [ "${os_version}" == "v3" ]; then
|
||||
output_name="${REGISRTY_ADDRESS:-172.30.1.1:5000}/$(oc project -q)/$image_name"
|
||||
|
||||
if ! ct_os_docker_login_v3; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
fi
|
||||
if [ "${os_version}" == "v4" ]; then
|
||||
# Variable OCP4_REGISTER is set in function ct_os_docker_login_v4
|
||||
if ! ct_os_docker_login_v4; then
|
||||
return 1
|
||||
fi
|
||||
output_name="$OCP4_REGISTER/$namespace/$image_name"
|
||||
fi
|
||||
docker tag "${source_name}" "${output_name}"
|
||||
docker push "${output_name}"
|
||||
}
|
||||
|
||||
|
|
@ -672,8 +716,9 @@ function ct_os_test_s2i_app_func() {
|
|||
local image_tagged="${image_name_no_namespace%:*}:${VERSION}"
|
||||
|
||||
if [ "${CVP:-0}" -eq "0" ]; then
|
||||
if [ "${CT_EXTERNAL_REGISTRY:-false}" == 'true' ] ; then
|
||||
ct_os_import_image_ocp4 "${image_name}" "${image_tagged}"
|
||||
if [ "${CT_OCP4_TEST:-false}" == 'true' ] ; then
|
||||
echo "Uploading image ${image_name} as ${image_tagged} into OpenShift internal registry."
|
||||
ct_os_upload_image "v4" "${image_name}" "${image_tagged}"
|
||||
else
|
||||
# Create a specific imagestream tag for the image so that oc cannot use anything else
|
||||
if [ "${CT_SKIP_UPLOAD_IMAGE:-false}" == 'true' ] ; then
|
||||
|
|
@ -684,7 +729,7 @@ function ct_os_test_s2i_app_func() {
|
|||
ct_os_wait_stream_ready "${image_tagged}" "${namespace}"
|
||||
else
|
||||
echo "Uploading image ${image_name} as ${image_tagged}"
|
||||
ct_os_upload_image "${image_name}" "${image_tagged}"
|
||||
ct_os_upload_image "v3" "${image_name}" "${image_tagged}"
|
||||
fi
|
||||
fi
|
||||
else
|
||||
|
|
@ -816,8 +861,9 @@ function ct_os_test_template_app_func() {
|
|||
# Upload main image is already done by CVP pipeline. No need to do it twice.
|
||||
if [ "${CVP:-0}" -eq "0" ]; then
|
||||
# Create a specific imagestream tag for the image so that oc cannot use anything else
|
||||
if [ "${CT_EXTERNAL_REGISTRY:-false}" == 'true' ] ; then
|
||||
ct_os_import_image_ocp4 "${image_name}" "${image_tagged}"
|
||||
if [ "${CT_OCP4_TEST:-false}" == 'true' ] ; then
|
||||
echo "Uploading image ${image_name} as ${image_tagged} into OpenShift internal registry."
|
||||
ct_os_upload_image "v4" "${image_name}" "${image_tagged}"
|
||||
else
|
||||
if [ "${CT_SKIP_UPLOAD_IMAGE:-false}" == 'true' ] ; then
|
||||
echo "Importing image ${image_name} as ${image_tagged}"
|
||||
|
|
@ -827,7 +873,7 @@ function ct_os_test_template_app_func() {
|
|||
ct_os_wait_stream_ready "${image_tagged}" "${namespace}"
|
||||
else
|
||||
echo "Uploading image ${image_name} as ${image_tagged}"
|
||||
ct_os_upload_image "${image_name}" "${image_tagged}"
|
||||
ct_os_upload_image "v3" "${image_name}" "${image_tagged}"
|
||||
fi
|
||||
fi
|
||||
else
|
||||
|
|
@ -849,10 +895,11 @@ function ct_os_test_template_app_func() {
|
|||
exit 1
|
||||
fi
|
||||
|
||||
if [ "${CT_EXTERNAL_REGISTRY:-false}" == 'true' ] ; then
|
||||
ct_os_import_image_ocp4 "${image_tag_a[0]}" "${image_tag_a[1]}"
|
||||
if [ "${CT_OCP4_TEST:-false}" == 'true' ] ; then
|
||||
echo "Uploading image ${image_tag_a[0]} as ${image_tag_a[1]} into OpenShift internal registry."
|
||||
ct_os_upload_image "v4" "${image_name}" "${image_tag_a[1]}"
|
||||
else
|
||||
ct_os_upload_image "${image_tag_a[0]}" "${image_tag_a[1]}"
|
||||
ct_os_upload_image "v3" "${image_tag_a[0]}" "${image_tag_a[1]}"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
|
@ -957,8 +1004,9 @@ ct_os_test_image_update() {
|
|||
local old_image=$1; shift
|
||||
local istag=$1; shift
|
||||
local check_function=$1; shift
|
||||
local service_name=${image_name##*/}
|
||||
local ip="" check_command_exp=""
|
||||
local image_name_no_namespace=${image_name##*/}
|
||||
local service_name="${image_name_no_namespace%%:*}-testing"
|
||||
|
||||
echo "Running image update test for: $image_name"
|
||||
# shellcheck disable=SC2119
|
||||
|
|
@ -966,7 +1014,7 @@ ct_os_test_image_update() {
|
|||
|
||||
# Get current image from repository and create an imagestream
|
||||
docker pull "$old_image:latest" 2>/dev/null
|
||||
ct_os_upload_image "$old_image" "$istag"
|
||||
ct_os_upload_image "v3" "$old_image" "$istag"
|
||||
|
||||
# Setup example application with curent image
|
||||
oc new-app "$@" --name "$service_name"
|
||||
|
|
@ -978,7 +1026,7 @@ ct_os_test_image_update() {
|
|||
ct_assert_cmd_success "$check_command_exp"
|
||||
|
||||
# Tag built image into the imagestream and wait for rebuild
|
||||
ct_os_upload_image "$image_name" "$istag"
|
||||
ct_os_upload_image "v3" "$image_name" "$istag"
|
||||
ct_os_wait_pod_ready "${service_name}-2" 60
|
||||
|
||||
# Check application output
|
||||
|
|
|
|||
|
|
@ -13,9 +13,8 @@ source ${THISDIR}/test-lib-openshift.sh
|
|||
source ${THISDIR}/test-lib-remote-openshift.sh
|
||||
|
||||
function test_redis_integration() {
|
||||
local image_name=$1
|
||||
local service_name=redis
|
||||
ct_os_test_template_app_func "${image_name}" \
|
||||
ct_os_test_template_app_func "${IMAGE_NAME}" \
|
||||
"https://raw.githubusercontent.com/openshift/origin/master/examples/db-templates/redis-ephemeral-template.json" \
|
||||
"${service_name}" \
|
||||
"ct_os_check_cmd_internal '<SAME_IMAGE>' '${service_name}-testing' 'timeout 15 redis-cli -h <IP> -a testp ping' 'PONG'" \
|
||||
|
|
|
|||
|
|
@ -5,9 +5,8 @@ source "$(dirname "${BASH_SOURCE[0]}")"/test-lib.sh
|
|||
|
||||
# Set of functions for testing docker images in OpenShift using 'oc' command
|
||||
|
||||
# A variable containing the overall test result; must be changed to 0 in the end
|
||||
# of the testing script:
|
||||
# OS_TESTSUITE_RESULT=0
|
||||
# A variable containing the overall test result
|
||||
# TESTSUITE_RESULT=0
|
||||
# And the following trap must be set, in the beginning of the test script:
|
||||
# trap ct_os_cleanup EXIT SIGINT
|
||||
|
||||
|
|
@ -32,13 +31,6 @@ function ct_os_set_path_oc_4() {
|
|||
return 1
|
||||
fi
|
||||
export PATH="${oc_path}:${PATH}"
|
||||
oc version
|
||||
if ! oc version | grep -q "Client Version: ${oc_version}." ; then
|
||||
echo "ERROR: something went wrong, oc located at ${oc_path}, but oc of version ${oc_version} not found in PATH ($PATH)" >&1
|
||||
return 1
|
||||
else
|
||||
echo "PATH set correctly, binary oc found in version ${oc_version}: $(command -v oc)"
|
||||
fi
|
||||
}
|
||||
|
||||
# ct_os_prepare_ocp4
|
||||
|
|
@ -55,10 +47,15 @@ function ct_os_set_ocp4() {
|
|||
OS_OC_CLIENT_VERSION=${OS_OC_CLIENT_VERSION:-4.4}
|
||||
ct_os_set_path_oc_4 "${OS_OC_CLIENT_VERSION}"
|
||||
|
||||
oc version
|
||||
|
||||
login=$(cat "$KUBEPASSWORD")
|
||||
oc login -u kubeadmin -p "$login"
|
||||
oc version
|
||||
if ! oc version | grep -q "Client Version: ${OS_OC_CLIENT_VERSION}." ; then
|
||||
echo "ERROR: something went wrong, oc located at ${oc_path}, but oc of version ${OS_OC_CLIENT_VERSION} not found in PATH ($PATH)" >&1
|
||||
return 1
|
||||
else
|
||||
echo "PATH set correctly, binary oc found in version ${OS_OC_CLIENT_VERSION}: $(command -v oc)"
|
||||
fi
|
||||
echo "Login to OpenShift ${OS_OC_CLIENT_VERSION} is DONE"
|
||||
# let openshift cluster to sync to avoid some race condition errors
|
||||
sleep 3
|
||||
|
|
@ -80,37 +77,28 @@ function ct_os_upload_image_external_registry() {
|
|||
}
|
||||
|
||||
|
||||
function ct_os_login_external_registry() {
|
||||
local docker_token
|
||||
# docker login fails with "404 page not found" error sometimes, just try it more times
|
||||
# shellcheck disable=SC2034
|
||||
echo "loging"
|
||||
[ -z "${INTERNAL_DOCKER_REGISTRY:-}" ] && "INTERNAL_DOCKER_REGISTRY has to be set for working with Internal registry" && return 1
|
||||
# shellcheck disable=SC2034
|
||||
for i in $(seq 12) ; do
|
||||
# shellcheck disable=SC2015
|
||||
docker_token=$(cat "$DOCKER_UPSHIFT_TOKEN")
|
||||
# shellcheck disable=SC2015
|
||||
docker login -u rhscl-ci-testing -p "$docker_token" "${INTERNAL_DOCKER_REGISTRY}" && return 0 || :
|
||||
sleep 5
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
function ct_os_import_image_ocp4() {
|
||||
local image_name="${1}"; shift
|
||||
local imagestream=${1:-$image_name:latest}
|
||||
local namespace
|
||||
|
||||
namespace=${CT_NAMESPACE:-"$(oc project -q)"}
|
||||
deploy_image_name="${INTERNAL_DOCKER_REGISTRY}/rhscl-ci-testing/${imagestream}"
|
||||
echo "Uploading image ${image_name} as ${deploy_image_name} , ${imagestream} into external registry."
|
||||
ct_os_upload_image_external_registry "${image_name}" "${imagestream}"
|
||||
if [ "${CT_TAG_IMAGE:-false}" == 'true' ]; then
|
||||
echo "Tag ${deploy_image_name} to ${namespace}/${imagestream}"
|
||||
oc tag --source=docker "${deploy_image_name}" "${namespace}/${imagestream}" --insecure=true --reference-policy=local
|
||||
else
|
||||
echo "Import image into OpenShift 4 environment ${namespace}/${imagestream} from ${deploy_image_name}"
|
||||
oc import-image "${namespace}/${imagestream}" --from="${deploy_image_name}" --confirm --reference-policy=local
|
||||
fi
|
||||
echo "Uploading image ${image_name} as ${imagestream} into OpenShift internal registry."
|
||||
ct_os_upload_image_v4 "${image_name}" "${imagestream}"
|
||||
|
||||
}
|
||||
|
||||
# ct_os_check_login
|
||||
# ---------------
|
||||
# function checks if the login to openshift was successful
|
||||
# if successful returns 0
|
||||
# if not, write error message, sets test result to 1
|
||||
# and exits with non-zero
|
||||
# Uses: $TESTSUITE_RESULT - overall result of all tests
|
||||
function ct_os_check_login() {
|
||||
oc status || {
|
||||
echo "-------------------------------------------"
|
||||
echo "It looks like oc is not properly logged in."
|
||||
# shellcheck disable=SC2034
|
||||
TESTSUITE_RESULT=1
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
|
|
|||
175
test/test-lib.sh
175
test/test-lib.sh
|
|
@ -16,6 +16,11 @@
|
|||
|
||||
# may be redefined in the specific container testfile
|
||||
EXPECTED_EXIT_CODE=0
|
||||
export TESTSUITE_RESULT=0
|
||||
|
||||
# define UNSTABLE_TESTS if not already defined, as this variable
|
||||
# is not mandatory for containers
|
||||
UNSTABLE_TESTS="${UNSTABLE_TESTS:-""}"
|
||||
|
||||
# ct_cleanup
|
||||
# --------------------
|
||||
|
|
@ -24,6 +29,7 @@ EXPECTED_EXIT_CODE=0
|
|||
# unexpectedly. Removes the cid_files and CID_FILE_DIR as well.
|
||||
# Uses: $CID_FILE_DIR - path to directory containing cid_files
|
||||
# Uses: $EXPECTED_EXIT_CODE - expected container exit code
|
||||
# Uses: $TESTSUITE_RESULT - overall result of all tests
|
||||
function ct_cleanup() {
|
||||
ct_show_resources
|
||||
for cid_file in "$CID_FILE_DIR"/* ; do
|
||||
|
|
@ -33,16 +39,47 @@ function ct_cleanup() {
|
|||
|
||||
: "Stopping and removing container $container..."
|
||||
docker stop "$container"
|
||||
exit_status=$(docker inspect -f '{{.State.ExitCode}}' "$container")
|
||||
if [ "$exit_status" != "$EXPECTED_EXIT_CODE" ]; then
|
||||
: "Dumping logs for $container"
|
||||
docker logs "$container"
|
||||
|
||||
# Container has not been removed by `docker stop` and still exists
|
||||
if [ "$( docker ps -a -f "id=$container" | wc -l )" -eq 2 ]; then
|
||||
exit_status=$(docker inspect -f '{{.State.ExitCode}}' "$container")
|
||||
if [ "$exit_status" != "$EXPECTED_EXIT_CODE" ]; then
|
||||
: "Dumping logs for $container"
|
||||
docker logs "$container"
|
||||
fi
|
||||
docker rm -v "$container"
|
||||
fi
|
||||
docker rm -v "$container"
|
||||
|
||||
rm "$cid_file"
|
||||
done
|
||||
rmdir "$CID_FILE_DIR"
|
||||
: "Done."
|
||||
|
||||
ct_show_results
|
||||
exit "${TESTSUITE_RESULT:-0}"
|
||||
}
|
||||
|
||||
# ct_show_results
|
||||
# ---------------
|
||||
# Prints results of all test cases that are stored into TEST_SUMMARY variable.
|
||||
# Uses: $TEST_SUMMARY - text info about test-cases
|
||||
# Uses: $TESTSUITE_RESULT - overall result of all tests
|
||||
function ct_show_results() {
|
||||
echo
|
||||
echo "==============================================="
|
||||
echo "Test cases results:"
|
||||
echo
|
||||
echo "${TEST_SUMMARY:-}"
|
||||
|
||||
if [ -n "${TESTSUITE_RESULT:-}" ] ; then
|
||||
if [ "$TESTSUITE_RESULT" -eq 0 ] ; then
|
||||
# shellcheck disable=SC2153
|
||||
echo "Tests for ${IMAGE_NAME} succeeded."
|
||||
else
|
||||
# shellcheck disable=SC2153
|
||||
echo "Tests for ${IMAGE_NAME} failed."
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# ct_enable_cleanup
|
||||
|
|
@ -125,6 +162,7 @@ function ct_check_envs_set {
|
|||
for value in $stripped; do
|
||||
# If the falue checked does not go through env_filter we do not care about it
|
||||
echo "$value" | grep -q "$env_filter" || continue
|
||||
# shellcheck disable=SC2295
|
||||
if [ -n "${filtered_envs##${env_format//VALUE/$value}}" ]; then
|
||||
echo " Value $value is missing from variable $var_name"
|
||||
echo "$filtered_envs"
|
||||
|
|
@ -284,7 +322,7 @@ function ct_doc_content_old() {
|
|||
docker run --rm "${IMAGE_NAME}" /bin/bash -c "cat /${f}" >"${tmpdir}/$(basename "${f}")"
|
||||
# Check whether the files contain some important information
|
||||
for term in "$@" ; do
|
||||
if ! grep -F -q -e "${term}" "${tmpdir}/$(basename "${f}")" ; then
|
||||
if ! grep -E -q -e "${term}" "${tmpdir}/$(basename "${f}")" ; then
|
||||
echo "ERROR: File /${f} does not include '${term}'." >&2
|
||||
return 1
|
||||
fi
|
||||
|
|
@ -489,21 +527,6 @@ ct_path_foreach ()
|
|||
}
|
||||
|
||||
|
||||
# ct_run_test_list
|
||||
# --------------------
|
||||
# Execute the tests specified by TEST_LIST
|
||||
# Uses: $TEST_LIST - list of test names
|
||||
function ct_run_test_list() {
|
||||
for test_case in $TEST_LIST; do
|
||||
: "Running test $test_case"
|
||||
# shellcheck source=/dev/null
|
||||
[ -f "test/$test_case" ] && source "test/$test_case"
|
||||
# shellcheck source=/dev/null
|
||||
[ -f "../test/$test_case" ] && source "../test/$test_case"
|
||||
$test_case
|
||||
done;
|
||||
}
|
||||
|
||||
# ct_gen_self_signed_cert_pem
|
||||
# ---------------------------
|
||||
# Generates a self-signed PEM certificate pair into specified directory.
|
||||
|
|
@ -631,10 +654,14 @@ ct_get_public_image_name() {
|
|||
public_image_name=$registry/rhscl/$base_image_name-${version//./}-rhel7
|
||||
elif [ "$os" == "rhel8" ]; then
|
||||
public_image_name=$registry/rhel8/$base_image_name-${version//./}
|
||||
elif [ "$os" == "rhel9" ]; then
|
||||
public_image_name=$registry/rhel9/$base_image_name-${version//./}
|
||||
elif [ "$os" == "centos7" ]; then
|
||||
public_image_name=$registry/centos7/$base_image_name-${version//./}-centos7
|
||||
elif [ "$os" == "centos8" ]; then
|
||||
public_image_name=$registry/centos8/$base_image_name-${version//./}-centos8
|
||||
elif [ "$os" == "c8s" ]; then
|
||||
public_image_name=$registry/sclorg/$base_image_name-${version//./}-c8s
|
||||
elif [ "$os" == "c9s" ]; then
|
||||
public_image_name=$registry/sclorg/$base_image_name-${version//./}-c9s
|
||||
fi
|
||||
|
||||
echo "$public_image_name"
|
||||
|
|
@ -646,6 +673,7 @@ ct_get_public_image_name() {
|
|||
# Argument: CMD - Command to be run
|
||||
function ct_assert_cmd_success() {
|
||||
echo "Checking '$*' for success ..."
|
||||
# shellcheck disable=SC2294
|
||||
if ! eval "$@" &>/dev/null; then
|
||||
echo " FAIL"
|
||||
return 1
|
||||
|
|
@ -660,6 +688,7 @@ function ct_assert_cmd_success() {
|
|||
# Argument: CMD - Command to be run
|
||||
function ct_assert_cmd_failure() {
|
||||
echo "Checking '$*' for failure ..."
|
||||
# shellcheck disable=SC2294
|
||||
if eval "$@" &>/dev/null; then
|
||||
echo " FAIL"
|
||||
return 1
|
||||
|
|
@ -935,6 +964,7 @@ ct_check_latest_imagestreams() {
|
|||
# Prints the available resources
|
||||
ct_show_resources()
|
||||
{
|
||||
echo
|
||||
echo "Resources info:"
|
||||
echo "Memory:"
|
||||
free -h
|
||||
|
|
@ -1059,6 +1089,8 @@ ct_test_app_dockerfile() {
|
|||
ct_test_response "http://$ip:${port}" 200 "${expected_text}"
|
||||
ret=$?
|
||||
|
||||
[[ $ret -eq 0 ]] || docker logs "$(ct_get_cid "${cname}")"
|
||||
|
||||
# cleanup
|
||||
docker kill "$(ct_get_cid "${cname}")"
|
||||
sleep 2
|
||||
|
|
@ -1069,4 +1101,101 @@ ct_test_app_dockerfile() {
|
|||
return $ret
|
||||
}
|
||||
|
||||
# ct_check_testcase_result
|
||||
# -----------------------------
|
||||
# Check if testcase ended in success or error
|
||||
# Argument: result - testcase result value
|
||||
# Uses: $TESTCASE_RESULT - result of the testcase
|
||||
# Uses: $IMAGE_NAME - name of the image being tested
|
||||
ct_check_testcase_result() {
|
||||
local result="$1"
|
||||
if [[ "$result" != "0" ]]; then
|
||||
echo "Test for image '${IMAGE_NAME}' FAILED (exit code: ${result})"
|
||||
TESTCASE_RESULT=1
|
||||
fi
|
||||
return "$result"
|
||||
}
|
||||
|
||||
# ct_run_tests_from_testset
|
||||
# -----------------------------
|
||||
# Runs all tests in $TEST_SET, prints result to
|
||||
# the $TEST_SUMMARY variable
|
||||
# Argument: app_name - application name to log
|
||||
# Uses: $TEST_SET - set of test cases to run
|
||||
# Uses: $TEST_SUMMARY - variable for storing test results
|
||||
# Uses: $IMAGE_NAME - name of the image being tested
|
||||
# Uses: $UNSTABLE_TESTS - set of tests, whose result can be ignored
|
||||
# Uses: $IGNORE_UNSTABLE_TESTS - flag to ignore unstable tests
|
||||
ct_run_tests_from_testset() {
|
||||
local app_name="${1:-appnamenotset}"
|
||||
local time_beg_pretty
|
||||
local time_beg
|
||||
local time_end
|
||||
local time_diff
|
||||
local test_msg
|
||||
local is_unstable
|
||||
|
||||
# Let's store in the log what change do we test
|
||||
echo
|
||||
git show -s
|
||||
echo
|
||||
|
||||
for test_case in $TEST_SET; do
|
||||
TESTCASE_RESULT=0
|
||||
# shellcheck disable=SC2076
|
||||
if [[ " ${UNSTABLE_TESTS[*]} " =~ " ${app_name} " ]]; then
|
||||
is_unstable=1
|
||||
else
|
||||
is_unstable=0
|
||||
fi
|
||||
time_beg_pretty=$(ct_timestamp_pretty)
|
||||
time_beg=$(ct_timestamp_s)
|
||||
echo "-----------------------------------------------"
|
||||
echo "Running test $test_case (starting at $time_beg_pretty) ... "
|
||||
echo "-----------------------------------------------"
|
||||
$test_case
|
||||
ct_check_testcase_result $?
|
||||
time_end=$(ct_timestamp_s)
|
||||
if [ $TESTCASE_RESULT -eq 0 ]; then
|
||||
test_msg="[PASSED]"
|
||||
else
|
||||
if [ -n "${IGNORE_UNSTABLE_TESTS:-""}" ] && [ $is_unstable -eq 1 ]; then
|
||||
test_msg="[FAILED][UNSTABLE-IGNORED]"
|
||||
else
|
||||
test_msg="[FAILED]"
|
||||
TESTSUITE_RESULT=1
|
||||
fi
|
||||
fi
|
||||
time_diff=$(ct_timestamp_diff "$time_beg" "$time_end")
|
||||
printf -v TEST_SUMMARY "%s %s for '%s' %s (%s)\n" "${TEST_SUMMARY:-}" "${test_msg}" "${app_name}" "$test_case" "$time_diff"
|
||||
[ -n "${FAIL_QUICKLY:-}" ] && return 1
|
||||
done;
|
||||
}
|
||||
|
||||
# ct_timestamp_s
|
||||
# --------------
|
||||
# Returns timestamp in seconds since unix era -- a large integer
|
||||
function ct_timestamp_s() {
|
||||
date '+%s'
|
||||
}
|
||||
|
||||
# ct_timestamp_pretty
|
||||
# -----------------
|
||||
# Returns timestamp readable to a human, like 2022-05-18 10:52:44+02:00
|
||||
function ct_timestamp_pretty() {
|
||||
date --rfc-3339=seconds
|
||||
}
|
||||
|
||||
# ct_timestamp_diff
|
||||
# -----------------
|
||||
# Computes a time diff between two timestamps
|
||||
# Argument: start_date - Beginning (in seconds since unix era -- a large integer)
|
||||
# Argument: final_date - End (in seconds since unix era -- a large integer)
|
||||
# Returns: Time difference in format HH:MM:SS
|
||||
function ct_timestamp_diff() {
|
||||
local start_date=$1
|
||||
local final_date=$2
|
||||
date -u -d "0 $final_date seconds - $start_date seconds" +"%H:%M:%S"
|
||||
}
|
||||
|
||||
# vim: set tabstop=2:shiftwidth=2:expandtab:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue