From d27be39ae71f81a4d97b09d38cf58f19b35a9620 Mon Sep 17 00:00:00 2001 From: Honza Horak Date: Fri, 24 Feb 2017 21:24:23 +0100 Subject: [PATCH 01/32] Initial commit --- Dockerfile | 64 +++ README.md | 1 + root/etc/my.cnf | 12 + root/help.1 | 332 +++++++++++++ root/usr/bin/cgroup-limits | 92 ++++ root/usr/bin/container-entrypoint | 2 + root/usr/bin/mysqld-master | 1 + root/usr/bin/mysqld-slave | 1 + root/usr/bin/run-mysqld | 35 ++ root/usr/bin/run-mysqld-master | 50 ++ root/usr/bin/run-mysqld-slave | 60 +++ root/usr/libexec/container-setup | 58 +++ .../share/container-scripts/mysql/README.md | 135 ++++++ .../share/container-scripts/mysql/common.sh | 164 +++++++ .../share/container-scripts/mysql/helpers.sh | 24 + .../mysql/my-base.cnf.template | 5 + .../mysql/my-master.cnf.template | 7 + .../mysql/my-paas.cnf.template | 26 ++ .../mysql/my-repl-gtid.cnf.template | 4 + .../mysql/my-slave.cnf.template | 7 + .../mysql/my-tuning.cnf.template | 28 ++ .../container-scripts/mysql/passwd-change.sh | 23 + .../container-scripts/mysql/post-init.sh | 0 .../share/container-scripts/mysql/scl_enable | 3 + .../mysql/validate-replication-variables.sh | 18 + .../mysql/validate-variables.sh | 78 ++++ test/run | 437 ++++++++++++++++++ 27 files changed, 1667 insertions(+) create mode 100644 Dockerfile create mode 120000 README.md create mode 100644 root/etc/my.cnf create mode 100644 root/help.1 create mode 100755 root/usr/bin/cgroup-limits create mode 100755 root/usr/bin/container-entrypoint create mode 120000 root/usr/bin/mysqld-master create mode 120000 root/usr/bin/mysqld-slave create mode 100755 root/usr/bin/run-mysqld create mode 100755 root/usr/bin/run-mysqld-master create mode 100755 root/usr/bin/run-mysqld-slave create mode 100755 root/usr/libexec/container-setup create mode 100644 root/usr/share/container-scripts/mysql/README.md create mode 100644 root/usr/share/container-scripts/mysql/common.sh create mode 100644 root/usr/share/container-scripts/mysql/helpers.sh create mode 100644 root/usr/share/container-scripts/mysql/my-base.cnf.template create mode 100644 root/usr/share/container-scripts/mysql/my-master.cnf.template create mode 100644 root/usr/share/container-scripts/mysql/my-paas.cnf.template create mode 100644 root/usr/share/container-scripts/mysql/my-repl-gtid.cnf.template create mode 100644 root/usr/share/container-scripts/mysql/my-slave.cnf.template create mode 100644 root/usr/share/container-scripts/mysql/my-tuning.cnf.template create mode 100644 root/usr/share/container-scripts/mysql/passwd-change.sh create mode 100644 root/usr/share/container-scripts/mysql/post-init.sh create mode 100644 root/usr/share/container-scripts/mysql/scl_enable create mode 100644 root/usr/share/container-scripts/mysql/validate-replication-variables.sh create mode 100644 root/usr/share/container-scripts/mysql/validate-variables.sh create mode 100755 test/run diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..3bf46f8 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,64 @@ +FROM fedora:25 + +LABEL MAINTAINER "Honza Horak" + +# MariaDB image for OpenShift. +# +# Volumes: +# * /var/lib/mysql/data - Datastore for MariaDB +# Environment: +# * $MYSQL_USER - Database user name +# * $MYSQL_PASSWORD - User's password +# * $MYSQL_DATABASE - Name of the database to create +# * $MYSQL_ROOT_PASSWORD (Optional) - Password for the 'root' MySQL account + +ENV MYSQL_VERSION=10.1 \ + HOME=/var/lib/mysql + +LABEL summary="MariaDB is a multi-user, multi-threaded SQL database server" \ + io.k8s.description="MariaDB is a multi-user, multi-threaded SQL database server" \ + io.k8s.display-name="MariaDB 10.1" \ + io.openshift.expose-services="3306:mysql" \ + io.openshift.tags="database,mysql,mariadb,mariadb101,galera" + +ENV NAME=mariadb VERSION=10.1 RELEASE=1 ARCH=x86_64 +LABEL BZComponent="$NAME" \ + Name="$FGC/$NAME" \ + Version="$VERSION" \ + Release="$RELEASE.$DISTTAG" \ + Architecture="$ARCH" + +EXPOSE 3306 + +# This image must forever use UID 27 for mysql user so our volumes are +# safe in the future. This should *never* change, the last test is there +# to make sure of that. +RUN INSTALL_PKGS="rsync tar gettext hostname bind-utils mariadb-server policycoreutils" && \ + dnf install -y --setopt=tsflags=nodocs $INSTALL_PKGS && \ + rpm -V $INSTALL_PKGS && \ + dnf clean all && \ + mkdir -p /var/lib/mysql/data && chown -R mysql.0 /var/lib/mysql && \ + test "$(id mysql)" = "uid=27(mysql) gid=27(mysql) groups=27(mysql)" + +# On Fedora, we fake missing python binary. In case user installs the python2 +# in the container, this hack will be removed by installing /usr/bin/python from RPM. +RUN ln -s /usr/bin/python3 /usr/bin/python + +# Get prefix path and path to scripts rather than hard-code them in scripts +ENV CONTAINER_SCRIPTS_PATH=/usr/share/container-scripts/mysql \ + MYSQL_PREFIX=/usr + +ADD root / + +# this is needed due to issues with squash +# when this directory gets rm'd by the container-setup +# script. +RUN rm -rf /etc/my.cnf.d/* +RUN /usr/libexec/container-setup + +VOLUME ["/var/lib/mysql/data"] + +USER 27 + +ENTRYPOINT ["container-entrypoint"] +CMD ["run-mysqld"] diff --git a/README.md b/README.md new file mode 120000 index 0000000..cc942f0 --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +root/usr/share/container-scripts/mysql/README.md \ No newline at end of file diff --git a/root/etc/my.cnf b/root/etc/my.cnf new file mode 100644 index 0000000..0844075 --- /dev/null +++ b/root/etc/my.cnf @@ -0,0 +1,12 @@ +[mysqld] + +# Disabling symbolic-links is recommended to prevent assorted security risks +symbolic-links = 0 + +# http://www.percona.com/blog/2008/05/31/dns-achilles-heel-mysql-installation/ +skip_name_resolve + +# http://www.chriscalender.com/ignoring-the-lostfound-directory-in-your-datadir/ +ignore-db-dir=lost+found + +!includedir /etc/my.cnf.d diff --git a/root/help.1 b/root/help.1 new file mode 100644 index 0000000..59c37b8 --- /dev/null +++ b/root/help.1 @@ -0,0 +1,332 @@ +.\"t +.\" WARNING: Do not edit this file manually, it is generated from README.md automatically. +.\" +.\"t +.\" Automatically generated by Pandoc 1.16.0.2 +.\" +.TH "MARIADB\-101\-RHEL7" "1" "February 22, 2017" "Container Image Pages" "" +.hy +.SH MariaDB Docker image +.PP +This container image includes MariaDB server 10.1 for OpenShift and +general usage. +Users can choose between RHEL and CentOS based images. +.PP +Dockerfile for CentOS is called Dockerfile, Dockerfile for RHEL is +called Dockerfile.rhel7. +.SS Environment variables and volumes +.PP +The image recognizes the following environment variables that you can +set during initialization by passing \f[C]\-e\ VAR=VALUE\f[] to the +Docker run command. +.PP +.TS +tab(@); +l l. +T{ +Variable name +T}@T{ +Description +T} +_ +T{ +\f[C]MYSQL_USER\f[] +T}@T{ +User name for MySQL account to be created +T} +T{ +\f[C]MYSQL_PASSWORD\f[] +T}@T{ +Password for the user account +T} +T{ +\f[C]MYSQL_DATABASE\f[] +T}@T{ +Database name +T} +T{ +\f[C]MYSQL_ROOT_PASSWORD\f[] +T}@T{ +Password for the root user (optional) +T} +.TE +.PP +The following environment variables influence the MySQL configuration +file. +They are all optional. +.PP +.TS +tab(@); +lw(17.2n) lw(35.5n) lw(17.2n). +T{ +Variable name +T}@T{ +Description +T}@T{ +Default +T} +_ +T{ +\f[C]MYSQL_LOWER_CASE_TABLE_NAMES\f[] +T}@T{ +Sets how the table names are stored and compared +T}@T{ +0 +T} +T{ +\f[C]MYSQL_MAX_CONNECTIONS\f[] +T}@T{ +The maximum permitted number of simultaneous client connections +T}@T{ +151 +T} +T{ +\f[C]MYSQL_MAX_ALLOWED_PACKET\f[] +T}@T{ +The maximum size of one packet or any generated/intermediate string +T}@T{ +200M +T} +T{ +\f[C]MYSQL_FT_MIN_WORD_LEN\f[] +T}@T{ +The minimum length of the word to be included in a FULLTEXT index +T}@T{ +4 +T} +T{ +\f[C]MYSQL_FT_MAX_WORD_LEN\f[] +T}@T{ +The maximum length of the word to be included in a FULLTEXT index +T}@T{ +20 +T} +T{ +\f[C]MYSQL_AIO\f[] +T}@T{ +Controls the \f[C]innodb_use_native_aio\f[] setting value in case the +native AIO is broken. +See http://help.directadmin.com/item.php?id=529 +T}@T{ +1 +T} +T{ +\f[C]MYSQL_TABLE_OPEN_CACHE\f[] +T}@T{ +The number of open tables for all threads +T}@T{ +400 +T} +T{ +\f[C]MYSQL_KEY_BUFFER_SIZE\f[] +T}@T{ +The size of the buffer used for index blocks +T}@T{ +32M (or 10% of available memory) +T} +T{ +\f[C]MYSQL_SORT_BUFFER_SIZE\f[] +T}@T{ +The size of the buffer used for sorting +T}@T{ +256K +T} +T{ +\f[C]MYSQL_READ_BUFFER_SIZE\f[] +T}@T{ +The size of the buffer used for a sequential scan +T}@T{ +8M (or 5% of available memory) +T} +T{ +\f[C]MYSQL_INNODB_BUFFER_POOL_SIZE\f[] +T}@T{ +The size of the buffer pool where InnoDB caches table and index data +T}@T{ +32M (or 50% of available memory) +T} +T{ +\f[C]MYSQL_INNODB_LOG_FILE_SIZE\f[] +T}@T{ +The size of each log file in a log group +T}@T{ +8M (or 15% of available available) +T} +T{ +\f[C]MYSQL_INNODB_LOG_BUFFER_SIZE\f[] +T}@T{ +The size of the buffer that InnoDB uses to write to the log files on +disk +T}@T{ +8M (or 15% of available memory) +T} +T{ +\f[C]MYSQL_DEFAULTS_FILE\f[] +T}@T{ +Point to an alternative configuration file +T}@T{ +/etc/my.cnf +T} +T{ +\f[C]MYSQL_BINLOG_FORMAT\f[] +T}@T{ +Set sets the binlog format, supported values are \f[C]row\f[] and +\f[C]statement\f[] +T}@T{ +statement +T} +.TE +.PP +You can also set the following mount points by passing the +\f[C]\-v\ /host:/container\f[] flag to Docker. +.PP +.TS +tab(@); +l l. +T{ +Volume mount point +T}@T{ +Description +T} +_ +T{ +\f[C]/var/lib/mysql/data\f[] +T}@T{ +MySQL data directory +T} +.TE +.PP +\f[B]Notice: When mouting a directory from the host into the container, +ensure that the mounted directory has the appropriate permissions and +that the owner and group of the directory matches the user UID or name +which is running inside the container.\f[] +.SS Usage +.PP +For this, we will assume that you are using the +\f[C]rhscl/mariadb\-100\-rhel7\f[] image. +If you want to set only the mandatory environment variables and not +store the database in a host directory, execute the following command: +.IP +.nf +\f[C] +$\ docker\ run\ \-d\ \-\-name\ mariadb_database\ \-e\ MYSQL_USER=user\ \-e\ MYSQL_PASSWORD=pass\ \-e\ MYSQL_DATABASE=db\ \-p\ 3306:3306\ rhscl/mariadb\-100\-rhel7 +\f[] +.fi +.PP +This will create a container named \f[C]mariadb_database\f[] running +MySQL with database \f[C]db\f[] and user with credentials +\f[C]user:pass\f[]. +Port 3306 will be exposed and mapped to the host. +If you want your database to be persistent across container executions, +also add a \f[C]\-v\ /host/db/path:/var/lib/mysql/data\f[] argument. +This will be the MySQL data directory. +.PP +If the database directory is not initialized, the entrypoint script will +first run +\f[C]mysql_install_db\f[] (https://dev.mysql.com/doc/refman/5.6/en/mysql-install-db.html) +and setup necessary database users and passwords. +After the database is initialized, or if it was already present, +\f[C]mysqld\f[] is executed and will run as PID 1. +You can stop the detached container by running +\f[C]docker\ stop\ mariadb_database\f[]. +.SS MariaDB auto\-tuning +.PP +When the MySQL image is run with the \f[C]\-\-memory\f[] parameter set +and you didn\[aq]t specify value for some parameters, their values will +be automatically calculated based on the available memory. +.PP +.TS +tab(@); +l l l. +T{ +Variable name +T}@T{ +Configuration parameter +T}@T{ +Relative value +T} +_ +T{ +\f[C]MYSQL_KEY_BUFFER_SIZE\f[] +T}@T{ +\f[C]key_buffer_size\f[] +T}@T{ +10% +T} +T{ +\f[C]MYSQL_READ_BUFFER_SIZE\f[] +T}@T{ +\f[C]read_buffer_size\f[] +T}@T{ +5% +T} +T{ +\f[C]MYSQL_INNODB_BUFFER_POOL_SIZE\f[] +T}@T{ +\f[C]innodb_buffer_pool_size\f[] +T}@T{ +50% +T} +T{ +\f[C]MYSQL_INNODB_LOG_FILE_SIZE\f[] +T}@T{ +\f[C]innodb_log_file_size\f[] +T}@T{ +15% +T} +T{ +\f[C]MYSQL_INNODB_LOG_BUFFER_SIZE\f[] +T}@T{ +\f[C]innodb_log_buffer_size\f[] +T}@T{ +15% +T} +.TE +.SS MySQL root user +.PP +The root user has no password set by default, only allowing local +connections. +You can set it by setting the \f[C]MYSQL_ROOT_PASSWORD\f[] environment +variable. +This will allow you to login to the root account remotely. +Local connections will still not require a password. +.PP +To disable remote root access, simply unset \f[C]MYSQL_ROOT_PASSWORD\f[] +and restart the container. +.SS Changing passwords +.PP +Since passwords are part of the image configuration, the only supported +method to change passwords for the database user (\f[C]MYSQL_USER\f[]) +and root user is by changing the environment variables +\f[C]MYSQL_PASSWORD\f[] and \f[C]MYSQL_ROOT_PASSWORD\f[], respectively. +.PP +Changing database passwords through SQL statements or any way other than +through the environment variables aforementioned will cause a mismatch +between the values stored in the variables and the actual passwords. +Whenever a database container starts it will reset the passwords to the +values stored in the environment variables. +.SS Default my.cnf file +.PP +With environment variables we are able to customize a lot of different +parameters or configurations for the mysql bootstrap configurations. +If you\[aq]d prefer to use your own configuration file, you can override +the \f[C]MYSQL_DEFAULTS_FILE\f[] env variable with the full path of the +file you wish to use. +For example, the default location is \f[C]/etc/my.cnf\f[] but you can +change it to \f[C]/etc/mysql/my.cnf\f[] by setting +\f[C]MYSQL_DEFAULTS_FILE=/etc/mysql/my.cnf\f[] +.SS Changing the replication binlog_format +.PP +Some applications may wish to use \f[C]row\f[] binlog_formats (for +example, those built with change\-data\-capture in mind). +The default replication/binlog format is \f[C]statement\f[] but to +change it you can set the \f[C]MYSQL_BINLOG_FORMAT\f[] environment +variable. +For example \f[C]MYSQL_BINLOG_FORMAT=row\f[]. +Now when you run the database with \f[C]master\f[] replication turned on +(ie, set the Docker/container \f[C]cmd\f[] to be +\f[C]run\-mysqld\-master\f[]) the binlog will emit the actual data for +the rows that change as opposed to the statements (ie, DML like +insert...) that caused the change. +.SH AUTHORS +Red Hat. diff --git a/root/usr/bin/cgroup-limits b/root/usr/bin/cgroup-limits new file mode 100755 index 0000000..b9d4edc --- /dev/null +++ b/root/usr/bin/cgroup-limits @@ -0,0 +1,92 @@ +#!/usr/bin/python + +""" +Script for parsing cgroup information + +This script will read some limits from the cgroup system and parse +them, printing out "VARIABLE=VALUE" on each line for every limit that is +successfully read. Output of this script can be directly fed into +bash's export command. Recommended usage from a bash script: + + set -o errexit + export_vars=$(cgroup-limits) ; export $export_vars + +Variables currently supported: + MAX_MEMORY_LIMIT_IN_BYTES + Maximum possible limit MEMORY_LIMIT_IN_BYTES can have. This is + currently constant value of 9223372036854775807. + MEMORY_LIMIT_IN_BYTES + Maximum amount of user memory in bytes. If this value is set + to the same value as MAX_MEMORY_LIMIT_IN_BYTES, it means that + there is no limit set. The value is taken from + /sys/fs/cgroup/memory/memory.limit_in_bytes + NUMBER_OF_CORES + Number of detected CPU cores that can be used. This value is + calculated from /sys/fs/cgroup/cpuset/cpuset.cpus + NO_MEMORY_LIMIT + Set to "true" if MEMORY_LIMIT_IN_BYTES is so high that the caller + can act as if no memory limit was set. Undefined otherwise. +""" + +from __future__ import print_function +import sys + + +def _read_file(path): + try: + with open(path, 'r') as f: + return f.read().strip() + except IOError: + return None + + +def get_memory_limit(): + """ + Read memory limit, in bytes. + """ + + limit = _read_file('/sys/fs/cgroup/memory/memory.limit_in_bytes') + if limit is None or not limit.isdigit(): + print("Warning: Can't detect memory limit from cgroups", + file=sys.stderr) + return None + return int(limit) + + +def get_number_of_cores(): + """ + Read number of CPU cores. + """ + + core_count = 0 + + line = _read_file('/sys/fs/cgroup/cpuset/cpuset.cpus') + if line is None: + print("Warning: Can't detect number of CPU cores from cgroups", + file=sys.stderr) + return None + + for group in line.split(','): + core_ids = list(map(int, group.split('-'))) + if len(core_ids) == 2: + core_count += core_ids[1] - core_ids[0] + 1 + else: + core_count += 1 + + return core_count + + +if __name__ == "__main__": + env_vars = { + "MAX_MEMORY_LIMIT_IN_BYTES": 9223372036854775807, + "MEMORY_LIMIT_IN_BYTES": get_memory_limit(), + "NUMBER_OF_CORES": get_number_of_cores() + } + + env_vars = {k: v for k, v in env_vars.items() if v is not None} + + if env_vars.get("MEMORY_LIMIT_IN_BYTES", 0) >= 92233720368547: + env_vars["NO_MEMORY_LIMIT"] = "true" + + for key, value in env_vars.items(): + print("{0}={1}".format(key, value)) diff --git a/root/usr/bin/container-entrypoint b/root/usr/bin/container-entrypoint new file mode 100755 index 0000000..9d8ad4d --- /dev/null +++ b/root/usr/bin/container-entrypoint @@ -0,0 +1,2 @@ +#!/bin/bash +exec "$@" diff --git a/root/usr/bin/mysqld-master b/root/usr/bin/mysqld-master new file mode 120000 index 0000000..8a0786e --- /dev/null +++ b/root/usr/bin/mysqld-master @@ -0,0 +1 @@ +run-mysqld-master \ No newline at end of file diff --git a/root/usr/bin/mysqld-slave b/root/usr/bin/mysqld-slave new file mode 120000 index 0000000..dc0f58b --- /dev/null +++ b/root/usr/bin/mysqld-slave @@ -0,0 +1 @@ +run-mysqld-slave \ No newline at end of file diff --git a/root/usr/bin/run-mysqld b/root/usr/bin/run-mysqld new file mode 100755 index 0000000..cd899a7 --- /dev/null +++ b/root/usr/bin/run-mysqld @@ -0,0 +1,35 @@ +#!/bin/bash + +export_vars=$(cgroup-limits); export $export_vars +source ${CONTAINER_SCRIPTS_PATH}/common.sh +set -eu + +[ -f ${CONTAINER_SCRIPTS_PATH}/validate-variables.sh ] && source ${CONTAINER_SCRIPTS_PATH}/validate-variables.sh + +# Process the MySQL configuration files +log_info 'Processing MySQL configuration files ...' +envsubst < ${CONTAINER_SCRIPTS_PATH}/my-base.cnf.template > /etc/my.cnf.d/base.cnf +envsubst < ${CONTAINER_SCRIPTS_PATH}/my-paas.cnf.template > /etc/my.cnf.d/paas.cnf +envsubst < ${CONTAINER_SCRIPTS_PATH}/my-tuning.cnf.template > /etc/my.cnf.d/tuning.cnf + +if [ ! -d "$MYSQL_DATADIR/mysql" ]; then + initialize_database "$@" +else + start_local_mysql "$@" +fi + +if [ -f ${CONTAINER_SCRIPTS_PATH}/passwd-change.sh ]; then + log_info 'Setting passwords ...' + source ${CONTAINER_SCRIPTS_PATH}/passwd-change.sh +fi +if [ -f ${CONTAINER_SCRIPTS_PATH}/post-init.sh ]; then + log_info 'Sourcing post-init.sh ...' + source ${CONTAINER_SCRIPTS_PATH}/post-init.sh +fi + +# Restart the MySQL server with public IP bindings +shutdown_local_mysql +unset_env_vars +log_volume_info $MYSQL_DATADIR +log_info 'Running final exec -- Only MySQL server logs after this point' +exec ${MYSQL_PREFIX}/libexec/mysqld --defaults-file=$MYSQL_DEFAULTS_FILE "$@" 2>&1 diff --git a/root/usr/bin/run-mysqld-master b/root/usr/bin/run-mysqld-master new file mode 100755 index 0000000..054889e --- /dev/null +++ b/root/usr/bin/run-mysqld-master @@ -0,0 +1,50 @@ +#!/bin/bash +# +# This is an entrypoint that runs the MySQL server in the 'master' mode. +# +export_vars=$(cgroup-limits); export $export_vars +source ${CONTAINER_SCRIPTS_PATH}/common.sh +set -eu + +export MYSQL_RUNNING_AS_MASTER=1 + +[ -f ${CONTAINER_SCRIPTS_PATH}/validate_replication_variables.sh ] && source ${CONTAINER_SCRIPTS_PATH}/validate_replication_variables.sh +[ -f ${CONTAINER_SCRIPTS_PATH}/validate_variables.sh ] && source ${CONTAINER_SCRIPTS_PATH}/validate_variables.sh + +# The 'server-id' for master needs to be constant +export MYSQL_SERVER_ID=1 +log_info "The 'master' server-id is ${MYSQL_SERVER_ID}" + +# Process the MySQL configuration files +log_info 'Processing MySQL configuration files ...' +envsubst < ${CONTAINER_SCRIPTS_PATH}/my-base.cnf.template > /etc/my.cnf.d/base.cnf +envsubst < ${CONTAINER_SCRIPTS_PATH}/my-paas.cnf.template > /etc/my.cnf.d/paas.cnf +envsubst < ${CONTAINER_SCRIPTS_PATH}/my-master.cnf.template > /etc/my.cnf.d/master.cnf +envsubst < ${CONTAINER_SCRIPTS_PATH}/my-repl-gtid.cnf.template > /etc/my.cnf.d/repl-gtid.cnf +envsubst < ${CONTAINER_SCRIPTS_PATH}/my-tuning.cnf.template > /etc/my.cnf.d/tuning.cnf + +if [ ! -d "$MYSQL_DATADIR/mysql" ]; then + initialize_database "$@" +else + start_local_mysql "$@" +fi + +log_info 'Setting passwords ...' +[ -f ${CONTAINER_SCRIPTS_PATH}/passwd-change.sh ] && source ${CONTAINER_SCRIPTS_PATH}/passwd-change.sh + +# Setup the 'master' replication on the MySQL server +mysql $mysql_flags <&1 diff --git a/root/usr/bin/run-mysqld-slave b/root/usr/bin/run-mysqld-slave new file mode 100755 index 0000000..51acce5 --- /dev/null +++ b/root/usr/bin/run-mysqld-slave @@ -0,0 +1,60 @@ +#!/bin/bash +# +# This is an entrypoint that runs the MySQL server in the 'slave' mode. +# +export_vars=$(cgroup-limits); export $export_vars +source ${CONTAINER_SCRIPTS_PATH}/common.sh +set -eu + +# Just run normal server if the data directory is already initialized +if [ -d "${MYSQL_DATADIR}/mysql" ]; then + exec /usr/bin/run-mysqld "$@" +fi + +export MYSQL_RUNNING_AS_SLAVE=1 + +[ -f ${CONTAINER_SCRIPTS_PATH}/validate_replication_variables.sh ] && source ${CONTAINER_SCRIPTS_PATH}/validate_replication_variables.sh + +# Generate the unique 'server-id' for this master +export MYSQL_SERVER_ID=$(server_id) +log_info "The 'slave' server-id is ${MYSQL_SERVER_ID}" + +# Process the MySQL configuration files +envsubst < ${CONTAINER_SCRIPTS_PATH}/my-base.cnf.template > /etc/my.cnf.d/base.cnf +envsubst < ${CONTAINER_SCRIPTS_PATH}/my-paas.cnf.template > /etc/my.cnf.d/paas.cnf +envsubst < ${CONTAINER_SCRIPTS_PATH}/my-slave.cnf.template > /etc/my.cnf.d/slave.cnf +envsubst < ${CONTAINER_SCRIPTS_PATH}/my-repl-gtid.cnf.template > /etc/my.cnf.d/repl-gtid.cnf +envsubst < ${CONTAINER_SCRIPTS_PATH}/my-tuning.cnf.template > /etc/my.cnf.d/tuning.cnf + +# Initialize MySQL database and wait for the MySQL master to accept +# connections. +initialize_database "$@" +wait_for_mysql_master + +# Get binlog file and position from master +STATUS_INFO=$(mysql --host "$MYSQL_MASTER_SERVICE_NAME" "-u${MYSQL_MASTER_USER}" "-p${MYSQL_MASTER_PASSWORD}" replication -e 'SELECT gtid from replication limit 1\G') +GTID_VALUE=$(echo "$STATUS_INFO" | grep 'gtid:' | head -n 1 | sed -e 's/^\s*gtid: //') + +# checking STATUS_INFO here because empty GTID_VALUE is valid value +if [ -z "${STATUS_INFO}" ] ; then + echo "Could not read GTID value from master" + exit 1 +fi + +mysql $mysql_flags <&1 diff --git a/root/usr/libexec/container-setup b/root/usr/libexec/container-setup new file mode 100755 index 0000000..29c6ed2 --- /dev/null +++ b/root/usr/libexec/container-setup @@ -0,0 +1,58 @@ +#!/bin/bash + +# This function returns all config files that daemon uses and their path +# includes /opt. It is used to get correct path to the config file. +mysql_get_config_files_scl() { + scl enable ${ENABLED_COLLECTIONS} -- my_print_defaults --help --verbose | \ + grep --after=1 '^Default options' | \ + tail -n 1 | \ + grep -o '[^ ]*opt[^ ]*my.cnf' +} + +# This function picks the main config file that deamon uses and we ship in rpm +mysql_get_correct_config() { + # we use the same config in non-SCL packages, not necessary to guess + [ -z "${ENABLED_COLLECTIONS}" ] && echo -n "/etc/my.cnf" && return + + # from all config files read by daemon, pick the first that exists + for f in `mysql_get_config_files_scl` ; do + [ -f "$f" ] && echo "$f" + done | head -n 1 +} + +export MYSQL_CONFIG_FILE=$(mysql_get_correct_config) + +[ -z "$MYSQL_CONFIG_FILE" ] && echo "MYSQL_CONFIG_FILE is empty" && exit 1 + +unset -f mysql_get_correct_config mysql_get_config_files_scl + +# we provide own config files for the container, so clean what rpm ships here +mkdir -p ${MYSQL_CONFIG_FILE}.d +rm -f ${MYSQL_CONFIG_FILE}.d/* + +# we may add options during service init, so we need to have this dir writable by daemon user +chown -R mysql:0 ${MYSQL_CONFIG_FILE}.d ${MYSQL_CONFIG_FILE} +restorecon -R ${MYSQL_CONFIG_FILE}.d ${MYSQL_CONFIG_FILE} + +# API of the container are standard paths /etc/my.cnf and /etc/my.cnf.d +# we already include own /etc/my.cnf for container, but for cases the +# actually used config file is not on standard path /etc/my.cnf, we +# need to move it to the location daemon expects it and create symlinks +if [ "$MYSQL_CONFIG_FILE" != "/etc/my.cnf" ] ; then + rm -rf /etc/my.cnf.d + mv /etc/my.cnf ${MYSQL_CONFIG_FILE} + ln -s ${MYSQL_CONFIG_FILE} /etc/my.cnf + ln -s ${MYSQL_CONFIG_FILE}.d /etc/my.cnf.d +fi + +# setup directory for data +mkdir -p /var/lib/mysql/data +chown -R mysql:0 /var/lib/mysql +restorecon -R /var/lib/mysql + +# Loosen permission bits for group to avoid problems running container with +# arbitrary UID +# When only specifying user, group is 0, that's why /var/lib/mysql must have +# owner mysql.0; that allows to avoid a+rwx for this dir +chmod g+w -R /var/lib/mysql ${MYSQL_CONFIG_FILE}.d + diff --git a/root/usr/share/container-scripts/mysql/README.md b/root/usr/share/container-scripts/mysql/README.md new file mode 100644 index 0000000..656dbd9 --- /dev/null +++ b/root/usr/share/container-scripts/mysql/README.md @@ -0,0 +1,135 @@ +MariaDB Docker image +==================== + +This container image includes MariaDB server 10.1 for OpenShift and general usage. +Users can choose between RHEL and CentOS based images. + +Dockerfile for CentOS is called Dockerfile, Dockerfile for RHEL is called +Dockerfile.rhel7. + +Environment variables and volumes +---------------------------------- + +The image recognizes the following environment variables that you can set during +initialization by passing `-e VAR=VALUE` to the Docker run command. + +| Variable name | Description | +| :--------------------- | ----------------------------------------- | +| `MYSQL_USER` | User name for MySQL account to be created | +| `MYSQL_PASSWORD` | Password for the user account | +| `MYSQL_DATABASE` | Database name | +| `MYSQL_ROOT_PASSWORD` | Password for the root user (optional) | + +The following environment variables influence the MySQL configuration file. They are all optional. + +| Variable name | Description | Default +| :------------------------------ | ----------------------------------------------------------------- | ------------------------------- +| `MYSQL_LOWER_CASE_TABLE_NAMES` | Sets how the table names are stored and compared | 0 +| `MYSQL_MAX_CONNECTIONS` | The maximum permitted number of simultaneous client connections | 151 +| `MYSQL_MAX_ALLOWED_PACKET` | The maximum size of one packet or any generated/intermediate string | 200M +| `MYSQL_FT_MIN_WORD_LEN` | The minimum length of the word to be included in a FULLTEXT index | 4 +| `MYSQL_FT_MAX_WORD_LEN` | The maximum length of the word to be included in a FULLTEXT index | 20 +| `MYSQL_AIO` | Controls the `innodb_use_native_aio` setting value in case the native AIO is broken. See http://help.directadmin.com/item.php?id=529 | 1 +| `MYSQL_TABLE_OPEN_CACHE` | The number of open tables for all threads | 400 +| `MYSQL_KEY_BUFFER_SIZE` | The size of the buffer used for index blocks | 32M (or 10% of available memory) +| `MYSQL_SORT_BUFFER_SIZE` | The size of the buffer used for sorting | 256K +| `MYSQL_READ_BUFFER_SIZE` | The size of the buffer used for a sequential scan | 8M (or 5% of available memory) +| `MYSQL_INNODB_BUFFER_POOL_SIZE`| The size of the buffer pool where InnoDB caches table and index data | 32M (or 50% of available memory) +| `MYSQL_INNODB_LOG_FILE_SIZE` | The size of each log file in a log group | 8M (or 15% of available available) +| `MYSQL_INNODB_LOG_BUFFER_SIZE` | The size of the buffer that InnoDB uses to write to the log files on disk | 8M (or 15% of available memory) +| `MYSQL_DEFAULTS_FILE` | Point to an alternative configuration file | /etc/my.cnf +| `MYSQL_BINLOG_FORMAT` | Set sets the binlog format, supported values are `row` and `statement` | statement + +You can also set the following mount points by passing the `-v /host:/container` flag to Docker. + +| Volume mount point | Description | +| :----------------------- | -------------------- | +| `/var/lib/mysql/data` | MySQL data directory | + +**Notice: When mouting a directory from the host into the container, ensure that the mounted +directory has the appropriate permissions and that the owner and group of the directory +matches the user UID or name which is running inside the container.** + +Usage +--------------------------------- + +For this, we will assume that you are using the `rhscl/mariadb-100-rhel7` image. +If you want to set only the mandatory environment variables and not store +the database in a host directory, execute the following command: + +``` +$ docker run -d --name mariadb_database -e MYSQL_USER=user -e MYSQL_PASSWORD=pass -e MYSQL_DATABASE=db -p 3306:3306 rhscl/mariadb-100-rhel7 +``` + +This will create a container named `mariadb_database` running MySQL with database +`db` and user with credentials `user:pass`. Port 3306 will be exposed and mapped +to the host. If you want your database to be persistent across container executions, +also add a `-v /host/db/path:/var/lib/mysql/data` argument. This will be the MySQL +data directory. + +If the database directory is not initialized, the entrypoint script will first +run [`mysql_install_db`](https://dev.mysql.com/doc/refman/5.6/en/mysql-install-db.html) +and setup necessary database users and passwords. After the database is initialized, +or if it was already present, `mysqld` is executed and will run as PID 1. You can + stop the detached container by running `docker stop mariadb_database`. + + +MariaDB auto-tuning +------------------- + +When the MySQL image is run with the `--memory` parameter set and you didn't +specify value for some parameters, their values will be automatically +calculated based on the available memory. + +| Variable name | Configuration parameter | Relative value +| :-------------------------------| ------------------------- | -------------- +| `MYSQL_KEY_BUFFER_SIZE` | `key_buffer_size` | 10% +| `MYSQL_READ_BUFFER_SIZE` | `read_buffer_size` | 5% +| `MYSQL_INNODB_BUFFER_POOL_SIZE` | `innodb_buffer_pool_size` | 50% +| `MYSQL_INNODB_LOG_FILE_SIZE` | `innodb_log_file_size` | 15% +| `MYSQL_INNODB_LOG_BUFFER_SIZE` | `innodb_log_buffer_size` | 15% + + +MySQL root user +--------------------------------- +The root user has no password set by default, only allowing local connections. +You can set it by setting the `MYSQL_ROOT_PASSWORD` environment variable. This +will allow you to login to the root account remotely. Local connections will +still not require a password. + +To disable remote root access, simply unset `MYSQL_ROOT_PASSWORD` and restart +the container. + + +Changing passwords +------------------ + +Since passwords are part of the image configuration, the only supported method +to change passwords for the database user (`MYSQL_USER`) and root user is by +changing the environment variables `MYSQL_PASSWORD` and `MYSQL_ROOT_PASSWORD`, +respectively. + +Changing database passwords through SQL statements or any way other than through +the environment variables aforementioned will cause a mismatch between the +values stored in the variables and the actual passwords. Whenever a database +container starts it will reset the passwords to the values stored in the +environment variables. + +Default my.cnf file +------------------- +With environment variables we are able to customize a lot of different parameters +or configurations for the mysql bootstrap configurations. If you'd prefer to use +your own configuration file, you can override the `MYSQL_DEFAULTS_FILE` env +variable with the full path of the file you wish to use. For example, the default +location is `/etc/my.cnf` but you can change it to `/etc/mysql/my.cnf` by setting + `MYSQL_DEFAULTS_FILE=/etc/mysql/my.cnf` + +Changing the replication binlog_format +-------------------------------------- +Some applications may wish to use `row` binlog_formats (for example, those built + with change-data-capture in mind). The default replication/binlog format is + `statement` but to change it you can set the `MYSQL_BINLOG_FORMAT` environment + variable. For example `MYSQL_BINLOG_FORMAT=row`. Now when you run the database + with `master` replication turned on (ie, set the Docker/container `cmd` to be +`run-mysqld-master`) the binlog will emit the actual data for the rows that change +as opposed to the statements (ie, DML like insert...) that caused the change. diff --git a/root/usr/share/container-scripts/mysql/common.sh b/root/usr/share/container-scripts/mysql/common.sh new file mode 100644 index 0000000..e63a750 --- /dev/null +++ b/root/usr/share/container-scripts/mysql/common.sh @@ -0,0 +1,164 @@ +#!/bin/bash + +source ${CONTAINER_SCRIPTS_PATH}/helpers.sh + +# Data directory where MySQL database files live. The data subdirectory is here +# because .bashrc and my.cnf both live in /var/lib/mysql/ and we don't want a +# volume to override it. +export MYSQL_DATADIR=/var/lib/mysql/data + +# Configuration settings. +export MYSQL_DEFAULTS_FILE=${MYSQL_DEFAULTS_FILE:-/etc/my.cnf} +export MYSQL_BINLOG_FORMAT=${MYSQL_BINLOG_FORMAT:-STATEMENT} +export MYSQL_LOWER_CASE_TABLE_NAMES=${MYSQL_LOWER_CASE_TABLE_NAMES:-0} +export MYSQL_MAX_CONNECTIONS=${MYSQL_MAX_CONNECTIONS:-151} +export MYSQL_FT_MIN_WORD_LEN=${MYSQL_FT_MIN_WORD_LEN:-4} +export MYSQL_FT_MAX_WORD_LEN=${MYSQL_FT_MAX_WORD_LEN:-20} +export MYSQL_AIO=${MYSQL_AIO:-1} +export MYSQL_MAX_ALLOWED_PACKET=${MYSQL_MAX_ALLOWED_PACKET:-200M} +export MYSQL_TABLE_OPEN_CACHE=${MYSQL_TABLE_OPEN_CACHE:-400} +export MYSQL_SORT_BUFFER_SIZE=${MYSQL_SORT_BUFFER_SIZE:-256K} + +if [ -n "${NO_MEMORY_LIMIT:-}" -o -z "${MEMORY_LIMIT_IN_BYTES:-}" ]; then + key_buffer_size='32M' + read_buffer_size='8M' + innodb_buffer_pool_size='32M' + innodb_log_file_size='8M' + innodb_log_buffer_size='8M' +else + key_buffer_size="$(python -c "print(int((${MEMORY_LIMIT_IN_BYTES}/(1024*1024))*0.1))")M" + read_buffer_size="$(python -c "print(int((${MEMORY_LIMIT_IN_BYTES}/(1024*1024))*0.05))")M" + innodb_buffer_pool_size="$(python -c "print(int((${MEMORY_LIMIT_IN_BYTES}/(1024*1024))*0.5))")M" + innodb_log_file_size="$(python -c "print(int((${MEMORY_LIMIT_IN_BYTES}/(1024*1024))*0.15))")M" + innodb_log_buffer_size="$(python -c "print(int((${MEMORY_LIMIT_IN_BYTES}/(1024*1024))*0.15))")M" +fi +export MYSQL_KEY_BUFFER_SIZE=${MYSQL_KEY_BUFFER_SIZE:-$key_buffer_size} +export MYSQL_READ_BUFFER_SIZE=${MYSQL_READ_BUFFER_SIZE:-$read_buffer_size} +export MYSQL_INNODB_BUFFER_POOL_SIZE=${MYSQL_INNODB_BUFFER_POOL_SIZE:-$innodb_buffer_pool_size} +export MYSQL_INNODB_LOG_FILE_SIZE=${MYSQL_INNODB_LOG_FILE_SIZE:-$innodb_log_file_size} +export MYSQL_INNODB_LOG_BUFFER_SIZE=${MYSQL_INNODB_LOG_BUFFER_SIZE:-$innodb_log_buffer_size} + +# Be paranoid and stricter than we should be. +# https://dev.mysql.com/doc/refman/en/identifiers.html +mysql_identifier_regex='^[a-zA-Z0-9_]+$' +mysql_password_regex='^[a-zA-Z0-9_~!@#$%^&*()-=<>,.?;:|]+$' + +# Variables that are used to connect to local mysql during initialization +mysql_flags="-u root --socket=/tmp/mysql.sock" +admin_flags="--defaults-file=$MYSQL_DEFAULTS_FILE $mysql_flags" + +# Make sure env variables don't propagate to mysqld process. +function unset_env_vars() { + log_info 'Cleaning up environment variables MYSQL_USER, MYSQL_PASSWORD, MYSQL_DATABASE and MYSQL_ROOT_PASSWORD ...' + unset MYSQL_USER MYSQL_PASSWORD MYSQL_DATABASE MYSQL_ROOT_PASSWORD +} + +# Poll until MySQL responds to our ping. +function wait_for_mysql() { + pid=$1 ; shift + + while [ true ]; do + if [ -d "/proc/$pid" ]; then + mysqladmin --socket=/tmp/mysql.sock ping &>/dev/null && log_info "MySQL started successfully" && return 0 + else + return 1 + fi + log_info "Waiting for MySQL to start ..." + sleep 1 + done +} + +# Start local MySQL server with a defaults file +function start_local_mysql() { + log_info 'Starting MySQL server with disabled networking ...' + ${MYSQL_PREFIX}/libexec/mysqld \ + --defaults-file=$MYSQL_DEFAULTS_FILE \ + --skip-networking --socket=/tmp/mysql.sock "$@" & + mysql_pid=$! + wait_for_mysql $mysql_pid +} + +# Shutdown mysql flushing privileges +function shutdown_local_mysql() { + log_info 'Shutting down MySQL ...' + mysqladmin $admin_flags flush-privileges shutdown +} + +# Initialize the MySQL database (create user accounts and the initial database) +function initialize_database() { + log_info 'Initializing database ...' + log_info 'Running mysql_install_db ...' + # Using --rpm since we need mysql_install_db behaves as in RPM + # Using empty --basedir to work-around https://bugzilla.redhat.com/show_bug.cgi?id=1406391 + mysql_install_db --rpm --datadir=$MYSQL_DATADIR --basedir='' + start_local_mysql "$@" + + if [ -v MYSQL_RUNNING_AS_SLAVE ]; then + log_info 'Initialization finished' + return 0 + fi + + if [ -v MYSQL_RUNNING_AS_MASTER ]; then + # Save master status into a separate database. + STATUS_INFO=$(mysql $admin_flags -e 'SHOW MASTER STATUS\G') + BINLOG_POSITION=$(echo "$STATUS_INFO" | grep 'Position:' | head -n 1 | sed -e 's/^\s*Position: //') + BINLOG_FILE=$(echo "$STATUS_INFO" | grep 'File:' | head -n 1 | sed -e 's/^\s*File: //') + GTID_INFO=$(mysql $admin_flags -e "SELECT BINLOG_GTID_POS('$BINLOG_FILE', '$BINLOG_POSITION') AS gtid_value \G") + GTID_VALUE=$(echo "$GTID_INFO" | grep 'gtid_value:' | head -n 1 | sed -e 's/^\s*gtid_value: //') + + mysqladmin $admin_flags create replication + mysql $admin_flags </dev/null && log_info "MySQL master is ready" && return 0 + sleep 1 + done +} diff --git a/root/usr/share/container-scripts/mysql/helpers.sh b/root/usr/share/container-scripts/mysql/helpers.sh new file mode 100644 index 0000000..4e832fc --- /dev/null +++ b/root/usr/share/container-scripts/mysql/helpers.sh @@ -0,0 +1,24 @@ +function log_info { + echo "---> `date +%T` $@" +} + +function log_and_run { + log_info "Running $@" + "$@" +} + +function log_volume_info { + CONTAINER_DEBUG=${CONTAINER_DEBUG:-} + if [[ "${CONTAINER_DEBUG,,}" != "true" ]]; then + return + fi + + log_info "Volume info for $@:" + set +e + log_and_run mount + while [ $# -gt 0 ]; do + log_and_run ls -alZ $1 + shift + done + set -e +} diff --git a/root/usr/share/container-scripts/mysql/my-base.cnf.template b/root/usr/share/container-scripts/mysql/my-base.cnf.template new file mode 100644 index 0000000..c654f7f --- /dev/null +++ b/root/usr/share/container-scripts/mysql/my-base.cnf.template @@ -0,0 +1,5 @@ +[mysqld] +datadir = ${MYSQL_DATADIR} +basedir = ${MYSQL_PREFIX} +plugin-dir = ${MYSQL_PREFIX}/lib64/mysql/plugin + diff --git a/root/usr/share/container-scripts/mysql/my-master.cnf.template b/root/usr/share/container-scripts/mysql/my-master.cnf.template new file mode 100644 index 0000000..f434885 --- /dev/null +++ b/root/usr/share/container-scripts/mysql/my-master.cnf.template @@ -0,0 +1,7 @@ +[mysqld] + +server-id = ${MYSQL_SERVER_ID} +log_bin = ${MYSQL_DATADIR}/mysql-bin.log +binlog_do_db = mysql +binlog_do_db = ${MYSQL_DATABASE} +binlog_format = ${MYSQL_BINLOG_FORMAT} diff --git a/root/usr/share/container-scripts/mysql/my-paas.cnf.template b/root/usr/share/container-scripts/mysql/my-paas.cnf.template new file mode 100644 index 0000000..11ddd1f --- /dev/null +++ b/root/usr/share/container-scripts/mysql/my-paas.cnf.template @@ -0,0 +1,26 @@ +[mysqld] +# +# Settings configured by the user +# + +# Sets how the table names are stored and compared. Default: 0 +lower_case_table_names = ${MYSQL_LOWER_CASE_TABLE_NAMES} + +# The maximum permitted number of simultaneous client connections. Default: 151 +max_connections = ${MYSQL_MAX_CONNECTIONS} + +# The minimum/maximum lengths of the word to be included in a FULLTEXT index. Default: 4/20 +ft_min_word_len = ${MYSQL_FT_MIN_WORD_LEN} +ft_max_word_len = ${MYSQL_FT_MAX_WORD_LEN} + +# In case the native AIO is broken. Default: 1 +# See http://help.directadmin.com/item.php?id=529 +innodb_use_native_aio = ${MYSQL_AIO} + +[myisamchk] +# The minimum/maximum lengths of the word to be included in a FULLTEXT index. Default: 4/20 +# +# To ensure that myisamchk and the server use the same values for full-text +# parameters, we placed them in both sections. +ft_min_word_len = ${MYSQL_FT_MIN_WORD_LEN} +ft_max_word_len = ${MYSQL_FT_MAX_WORD_LEN} diff --git a/root/usr/share/container-scripts/mysql/my-repl-gtid.cnf.template b/root/usr/share/container-scripts/mysql/my-repl-gtid.cnf.template new file mode 100644 index 0000000..a74a74c --- /dev/null +++ b/root/usr/share/container-scripts/mysql/my-repl-gtid.cnf.template @@ -0,0 +1,4 @@ +[mysqld] + +log-slave-updates = ON + diff --git a/root/usr/share/container-scripts/mysql/my-slave.cnf.template b/root/usr/share/container-scripts/mysql/my-slave.cnf.template new file mode 100644 index 0000000..5bdf109 --- /dev/null +++ b/root/usr/share/container-scripts/mysql/my-slave.cnf.template @@ -0,0 +1,7 @@ +[mysqld] + +server-id = ${MYSQL_SERVER_ID} +log_bin = ${MYSQL_DATADIR}/mysql-bin.log +relay-log = ${MYSQL_DATADIR}/mysql-relay-bin.log +binlog_do_db = mysql +binlog_do_db = ${MYSQL_DATABASE} diff --git a/root/usr/share/container-scripts/mysql/my-tuning.cnf.template b/root/usr/share/container-scripts/mysql/my-tuning.cnf.template new file mode 100644 index 0000000..e90b69a --- /dev/null +++ b/root/usr/share/container-scripts/mysql/my-tuning.cnf.template @@ -0,0 +1,28 @@ +[mysqld] +key_buffer_size = ${MYSQL_KEY_BUFFER_SIZE} +max_allowed_packet = ${MYSQL_MAX_ALLOWED_PACKET} +table_open_cache = ${MYSQL_TABLE_OPEN_CACHE} +sort_buffer_size = ${MYSQL_SORT_BUFFER_SIZE} +read_buffer_size = ${MYSQL_READ_BUFFER_SIZE} +read_rnd_buffer_size = 256K +net_buffer_length = 2K +thread_stack = 256K +myisam_sort_buffer_size = 2M + +# It is recommended that innodb_buffer_pool_size is configured to 50 to 75 percent of system memory. +innodb_buffer_pool_size = ${MYSQL_INNODB_BUFFER_POOL_SIZE} +innodb_additional_mem_pool_size = 2M +# Set .._log_file_size to 25 % of buffer pool size +innodb_log_file_size = ${MYSQL_INNODB_LOG_FILE_SIZE} +innodb_log_buffer_size = ${MYSQL_INNODB_LOG_BUFFER_SIZE} + +[mysqldump] +quick +max_allowed_packet = 16M + +[mysql] +no-auto-rehash + +[myisamchk] +key_buffer_size = 8M +sort_buffer_size = 8M diff --git a/root/usr/share/container-scripts/mysql/passwd-change.sh b/root/usr/share/container-scripts/mysql/passwd-change.sh new file mode 100644 index 0000000..ce06f6a --- /dev/null +++ b/root/usr/share/container-scripts/mysql/passwd-change.sh @@ -0,0 +1,23 @@ +# Set the password for MySQL user and root everytime this container is started. +# This allows to change the password by editing the deployment configuration. +if [[ -v MYSQL_USER && -v MYSQL_PASSWORD ]]; then + mysql $mysql_flags </dev/null + local exit_status + exit_status=$(docker inspect -f '{{.State.ExitCode}}' $CONTAINER) + if [ "$exit_status" != "0" ]; then + echo "Inspecting container $CONTAINER" + docker inspect $CONTAINER + echo "Dumping logs for $CONTAINER" + docker logs $CONTAINER + fi + docker rm -v $CONTAINER >/dev/null + rm $cidfile + echo "Done." + done + rmdir $CIDFILE_DIR +} +trap cleanup EXIT SIGINT + +function get_cid() { + local id="$1" ; shift || return 1 + echo $(cat "$CIDFILE_DIR/$id") +} + +function get_container_ip() { + local id="$1" ; shift + docker inspect --format='{{.NetworkSettings.IPAddress}}' $(get_cid "$id") +} + +function mysql_cmd() { + local container_ip="$1"; shift + local login="$1"; shift + local password="$1"; shift + docker run --rm "$IMAGE_NAME" mysql --host "$container_ip" -u"$login" -p"$password" "$@" db +} + +function test_connection() { + local name=$1 ; shift + local login=$1 ; shift + local password=$1 ; shift + local ip + ip=$(get_container_ip $name) + echo " Testing MySQL connection to $ip..." + local max_attempts=20 + local sleep_time=2 + local i + for i in $(seq $max_attempts); do + echo " Trying to connect..." + if mysql_cmd "$ip" "$login" "$password" <<< 'SELECT 1;'; then + echo " Success!" + return 0 + fi + sleep $sleep_time + done + echo " Giving up: Failed to connect. Logs:" + docker logs $(get_cid $name) + return 1 +} + +function test_mysql() { + local container_ip="$1" + local login="$2" + local password="$3" + + echo " Testing MySQL" + mysql_cmd "$container_ip" "$login" "$password" <<< 'CREATE TABLE tbl (col1 VARCHAR(20), col2 VARCHAR(20));' + mysql_cmd "$container_ip" "$login" "$password" <<< 'INSERT INTO tbl VALUES ("foo1", "bar1");' + mysql_cmd "$container_ip" "$login" "$password" <<< 'INSERT INTO tbl VALUES ("foo2", "bar2");' + mysql_cmd "$container_ip" "$login" "$password" <<< 'INSERT INTO tbl VALUES ("foo3", "bar3");' + mysql_cmd "$container_ip" "$login" "$password" <<< 'SELECT * FROM tbl;' + mysql_cmd "$container_ip" "$login" "$password" <<< 'DROP TABLE tbl;' + echo " Success!" +} + +function create_container() { + local name=$1 ; shift + cidfile="$CIDFILE_DIR/$name" + # create container with a cidfile in a directory for cleanup + local container_id + container_id="$(docker run ${DOCKER_ARGS:-} --cidfile $cidfile -d "$@" $IMAGE_NAME ${CONTAINER_ARGS:-})" + echo "Created container $container_id" +} + +function run_change_password_test() { + local tmpdir=$(mktemp -d) + mkdir "${tmpdir}/data" && chmod -R a+rwx "${tmpdir}" + + # Create MySQL container with persistent volume and set the initial password + create_container "testpass1" -e MYSQL_USER=user -e MYSQL_PASSWORD=foo \ + -e MYSQL_DATABASE=db -v ${tmpdir}:/var/lib/mysql/data:Z + test_connection testpass1 user foo + docker stop $(get_cid testpass1) >/dev/null + + # Create second container with changed password + create_container "testpass2" -e MYSQL_USER=user -e MYSQL_PASSWORD=bar \ + -e MYSQL_DATABASE=db -v ${tmpdir}:/var/lib/mysql/data:Z + test_connection testpass2 user bar + + # The old password should not work anymore + if mysql_cmd "$(get_container_ip testpass2)" user foo -e 'SELECT 1;'; then + return 1 + fi +} + +function run_replication_test() { + local cluster_args="-e MYSQL_MASTER_USER=master -e MYSQL_MASTER_PASSWORD=master -e MYSQL_DATABASE=db" + local max_attempts=30 + + # Run the MySQL master + docker run $cluster_args -e MYSQL_USER=user -e MYSQL_PASSWORD=foo \ + -e MYSQL_ROOT_PASSWORD=root \ + -e MYSQL_INNODB_BUFFER_POOL_SIZE=5M \ + -d --cidfile ${CIDFILE_DIR}/master.cid $IMAGE_NAME mysqld-master >/dev/null + local master_ip + master_ip=$(get_container_ip master.cid) + + # Run the MySQL slave + docker run $cluster_args -e MYSQL_MASTER_SERVICE_NAME=${master_ip} \ + -e MYSQL_INNODB_BUFFER_POOL_SIZE=5M \ + -d --cidfile ${CIDFILE_DIR}/slave.cid $IMAGE_NAME mysqld-slave >/dev/null + local slave_ip + slave_ip=$(get_container_ip slave.cid) + + # Now wait till the MASTER will see the SLAVE + local i + for i in $(seq $max_attempts); do + result="$(mysql_cmd "$master_ip" root root -e 'SHOW SLAVE HOSTS;' | grep "$slave_ip" || true)" + if [[ -n "${result}" ]]; then + echo "${slave_ip} successfully registered as SLAVE for ${master_ip}" + break + fi + if [[ "${i}" == "${max_attempts}" ]]; then + echo "The ${slave_ip} failed to register in MASTER" + echo "Dumping logs for $(get_cid slave.cid)" + docker logs $(get_cid slave.cid) + return 1 + fi + sleep 1 + done + + # do some real work to test replication in practice + mysql_cmd "$master_ip" root root -e "CREATE TABLE t1 (a INT); INSERT INTO t1 VALUES (24);" + + # read value from slave and check whether it is expectd + for i in $(seq $max_attempts); do + set +e + result="$(mysql_cmd "${slave_ip}" root root -e "select * from t1 \G" | grep -e ^a | grep 24)" + set -e + if [[ ! -z "${result}" ]]; then + echo "${slave_ip} successfully got value from MASTER ${master_ip}" + break + fi + if [[ "${i}" == "${max_attempts}" ]]; then + echo "The ${slave_ip} failed to see value added on MASTER" + echo "Dumping logs for $(get_cid slave.cid)" + docker logs $(get_cid slave.cid) + return 1 + fi + sleep 1 + done +} + +function assert_login_access() { + local container_ip=$1; shift + local USER=$1 ; shift + local PASS=$1 ; shift + local success=$1 ; shift + + if mysql_cmd "$container_ip" "$USER" "$PASS" <<< 'SELECT 1;' ; then + if $success ; then + echo " $USER($PASS) access granted as expected" + return + fi + else + if ! $success ; then + echo " $USER($PASS) access denied as expected" + return + fi + fi + echo " $USER($PASS) login assertion failed" + exit 1 +} + +function assert_local_access() { + local id="$1" ; shift + docker exec $(get_cid "$id") bash -c 'mysql <<< "SELECT 1;"' +} + +# Make sure the invocation of docker run fails. +function assert_container_creation_fails() { + + # Time the docker run command. It should fail. If it doesn't fail, + # mysqld will keep running so we kill it with SIGKILL to make sure + # timeout returns a non-zero value. + local ret=0 + timeout -s 9 --preserve-status 60s docker run --rm "$@" $IMAGE_NAME >/dev/null || ret=$? + + # Timeout will exit with a high number. + if [ $ret -gt 30 ]; then + return 1 + fi +} + +function try_image_invalid_combinations() { + assert_container_creation_fails -e MYSQL_USER=user -e MYSQL_DATABASE=db "$@" + assert_container_creation_fails -e MYSQL_PASSWORD=pass -e MYSQL_DATABASE=db "$@" +} + +function run_container_creation_tests() { + echo " Testing image entrypoint usage" + assert_container_creation_fails + try_image_invalid_combinations + try_image_invalid_combinations -e MYSQL_ROOT_PASSWORD=root_pass + + local VERY_LONG_DB_NAME="very_long_database_name_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + assert_container_creation_fails -e MYSQL_USER=user -e MYSQL_PASSWORD=pass + assert_container_creation_fails -e MYSQL_USER=\$invalid -e MYSQL_PASSWORD=pass -e MYSQL_DATABASE=db -e MYSQL_ROOT_PASSWORD=root_pass + assert_container_creation_fails -e MYSQL_USER=very_long_username -e MYSQL_PASSWORD=pass -e MYSQL_DATABASE=db -e MYSQL_ROOT_PASSWORD=root_pass + assert_container_creation_fails -e MYSQL_USER=user -e MYSQL_PASSWORD="\"" -e MYSQL_DATABASE=db -e MYSQL_ROOT_PASSWORD=root_pass + assert_container_creation_fails -e MYSQL_USER=user -e MYSQL_PASSWORD=pass -e MYSQL_DATABASE=\$invalid -e MYSQL_ROOT_PASSWORD=root_pass + assert_container_creation_fails -e MYSQL_USER=user -e MYSQL_PASSWORD=pass -e MYSQL_DATABASE=$VERY_LONG_DB_NAME -e MYSQL_ROOT_PASSWORD=root_pass + assert_container_creation_fails -e MYSQL_USER=user -e MYSQL_PASSWORD=pass -e MYSQL_DATABASE=db -e MYSQL_ROOT_PASSWORD="\"" + assert_container_creation_fails -e MYSQL_USER=root -e MYSQL_PASSWORD=pass -e MYSQL_DATABASE=db -e MYSQL_ROOT_PASSWORD=pass + echo " Success!" +} + +function test_config_option() { + local container_name="$1" + local configuration="$2" + local option_name="$3" + local option_value="$4" + + if ! echo "$configuration" | grep -qx "$option_name[[:space:]]*=[[:space:]]*$option_value"; then + local configs="$(docker exec -t "$(get_cid $container_name)" bash -c 'set +f; shopt -s nullglob; echo /etc/my.cnf /etc/my.cnf.d/* /opt/rh/mysql*/root/etc/my.cnf /opt/rh/mysql*/root/etc/my.cnf.d/* | paste -s')" + echo >&2 "FAIL: option '$option_name' should have value '$option_value', but it wasn't found in any of the configuration files ($configs):" + echo >&2 + echo >&2 "$configuration" + echo >&2 + return 1 + fi + + return 0 +} + +function run_configuration_tests() { + echo " Testing image configuration settings" + + local container_name=config_test + + create_container \ + "$container_name" \ + --env MYSQL_USER=config_test_user \ + --env MYSQL_PASSWORD=config_test \ + --env MYSQL_DATABASE=db \ + --env MYSQL_LOWER_CASE_TABLE_NAMES=1 \ + --env MYSQL_MAX_CONNECTIONS=1337 \ + --env MYSQL_FT_MIN_WORD_LEN=8 \ + --env MYSQL_FT_MAX_WORD_LEN=15 \ + --env MYSQL_MAX_ALLOWED_PACKET=10M \ + --env MYSQL_TABLE_OPEN_CACHE=100 \ + --env MYSQL_SORT_BUFFER_SIZE=256K \ + --env MYSQL_KEY_BUFFER_SIZE=16M \ + --env MYSQL_READ_BUFFER_SIZE=16M \ + --env MYSQL_INNODB_BUFFER_POOL_SIZE=16M \ + --env MYSQL_INNODB_LOG_FILE_SIZE=4M \ + --env MYSQL_INNODB_LOG_BUFFER_SIZE=4M \ + --env WORKAROUND_DOCKER_BUG_14203= + # + + test_connection "$container_name" config_test_user config_test + + # TODO: this check is far from perfect and could be improved: + # - we should look for an option in the desired config, not in all of them + # - we should respect section of the config (now we have duplicated options from a different sections) + local configuration + configuration="$(docker exec -t "$(get_cid $container_name)" bash -c 'set +f; shopt -s nullglob; egrep -hv "^(#|\!|\[|$)" /etc/my.cnf /etc/my.cnf.d/* /opt/rh/mysql*/root/etc/my.cnf /opt/rh/mysql*/root/etc/my.cnf.d/*' | sed 's,\(^[[:space:]]\+\|[[:space:]]\+$\),,' | sort -u)" + + test_config_option "$container_name" "$configuration" lower_case_table_names 1 + test_config_option "$container_name" "$configuration" max_connections 1337 + test_config_option "$container_name" "$configuration" ft_min_word_len 8 + test_config_option "$container_name" "$configuration" ft_max_word_len 15 + test_config_option "$container_name" "$configuration" max_allowed_packet 10M + test_config_option "$container_name" "$configuration" table_open_cache 100 + test_config_option "$container_name" "$configuration" sort_buffer_size 256K + test_config_option "$container_name" "$configuration" key_buffer_size 16M + test_config_option "$container_name" "$configuration" read_buffer_size 16M + test_config_option "$container_name" "$configuration" innodb_buffer_pool_size 16M + test_config_option "$container_name" "$configuration" innodb_log_file_size 4M + test_config_option "$container_name" "$configuration" innodb_log_buffer_size 4M + + docker stop "$(get_cid $container_name)" >/dev/null + + echo " Success!" + echo " Testing image auto-calculated configuration settings" + + container_name=dynamic_config_test + + DOCKER_ARGS='--memory=256m' create_container \ + "$container_name" \ + --env MYSQL_USER=config_test_user \ + --env MYSQL_PASSWORD=config_test \ + --env MYSQL_DATABASE=db + + test_connection "$container_name" config_test_user config_test + + configuration="$(docker exec -t "$(get_cid $container_name)" bash -c 'set +f; shopt -s nullglob; egrep -hv "^(#|\!|\[|$)" /etc/my.cnf /etc/my.cnf.d/* /opt/rh/mysql*/root/etc/my.cnf /opt/rh/mysql*/root/etc/my.cnf.d/*' | sed 's,\(^[[:space:]]\+\|[[:space:]]\+$\),,' | sort -u)" + + test_config_option "$container_name" "$configuration" key_buffer_size 25M + test_config_option "$container_name" "$configuration" read_buffer_size 12M + test_config_option "$container_name" "$configuration" innodb_buffer_pool_size 128M + test_config_option "$container_name" "$configuration" innodb_log_file_size 38M + test_config_option "$container_name" "$configuration" innodb_log_buffer_size 38M + + docker stop "$(get_cid $container_name)" >/dev/null + + echo " Success!" +} + +test_scl_usage() { + local name="$1" + local run_cmd="$2" + local expected="$3" + + echo " Testing the image SCL enable" + local out + out=$(docker run --rm ${IMAGE_NAME} /bin/bash -c "${run_cmd}") + if ! echo "${out}" | grep -q "${expected}"; then + echo "ERROR[/bin/bash -c "${run_cmd}"] Expected '${expected}', got '${out}'" + return 1 + fi + out=$(docker exec $(get_cid $name) /bin/bash -c "${run_cmd}" 2>&1) + if ! echo "${out}" | grep -q "${expected}"; then + echo "ERROR[exec /bin/bash -c "${run_cmd}"] Expected '${expected}', got '${out}'" + return 1 + fi + out=$(docker exec $(get_cid $name) /bin/sh -ic "${run_cmd}" 2>&1) + if ! echo "${out}" | grep -q "${expected}"; then + echo "ERROR[exec /bin/sh -ic "${run_cmd}"] Expected '${expected}', got '${out}'" + return 1 + fi +} + +function run_tests() { + local name=$1 ; shift + envs="-e MYSQL_USER=$USER -e MYSQL_PASSWORD=$PASS -e MYSQL_DATABASE=db" + if [ -v ROOT_PASS ]; then + envs="$envs -e MYSQL_ROOT_PASSWORD=$ROOT_PASS" + fi + create_container $name $envs + test_connection "$name" "$USER" "$PASS" + echo " Testing scl usage" + test_scl_usage $name 'mysql --version' '10.1' + echo " Testing login accesses" + local container_ip + container_ip=$(get_container_ip $name) + assert_login_access "$container_ip" "$USER" "$PASS" true + assert_login_access "$container_ip" "$USER" "${PASS}_foo" false + if [ -v ROOT_PASS ]; then + assert_login_access "$container_ip" root "$ROOT_PASS" true + assert_login_access "$container_ip" root "${ROOT_PASS}_foo" false + else + assert_login_access "$container_ip" root 'foo' false + assert_login_access "$container_ip" root '' false + fi + assert_local_access "$name" + echo " Success!" + test_mysql "$container_ip" "$USER" "$PASS" +} + +run_doc_test() { + local tmpdir=$(mktemp -d) + local f + echo " Testing documentation in the container image" + # Extract the help files from the container + for f in /usr/share/container-scripts/mysql/README.md help.1 ; do + docker run --rm ${IMAGE_NAME} /bin/bash -c "cat /${f}" >${tmpdir}/$(basename ${f}) + # Check whether the files include some important information + for term in MYSQL_ROOT_PASSWORD volume 3306 ; do + if ! cat ${tmpdir}/$(basename ${f}) | grep -q -e "${term}" ; then + echo "ERROR: File /${f} does not include '${term}'." + return 1 + fi + done + done + # Check whether the files use the correct format + if ! file ${tmpdir}/help.1 | grep -q roff ; then + echo "ERROR: /help.1 is not in troff or groff format" + return 1 + fi + echo " Success!" + echo +} + +# Tests. + +run_container_creation_tests + +run_configuration_tests + +# Set lower buffer pool size to avoid running out of memory. +export CONTAINER_ARGS="run-mysqld --innodb_buffer_pool_size=5242880" + +# Normal tests +USER=user PASS=pass run_tests no_root +USER=user1 PASS=pass1 ROOT_PASS=r00t run_tests root +# Test with arbitrary uid for the container +DOCKER_ARGS="-u 12345" USER=user PASS=pass run_tests no_root_altuid +DOCKER_ARGS="-u 12345" USER=user1 PASS=pass1 ROOT_PASS=r00t run_tests root_altuid + +# Test the password change +run_change_password_test + +# Replication tests +run_replication_test + +run_doc_test From d96f7f579031c99dcf522050a82c0480f8f7de62 Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Thu, 2 Mar 2017 20:21:35 -0600 Subject: [PATCH 02/32] Bump RELEASE for automatic rebuild --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3bf46f8..ecc7d15 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ LABEL summary="MariaDB is a multi-user, multi-threaded SQL database server" \ io.openshift.expose-services="3306:mysql" \ io.openshift.tags="database,mysql,mariadb,mariadb101,galera" -ENV NAME=mariadb VERSION=10.1 RELEASE=1 ARCH=x86_64 +ENV NAME=mariadb VERSION=10.1 RELEASE=2 ARCH=x86_64 LABEL BZComponent="$NAME" \ Name="$FGC/$NAME" \ Version="$VERSION" \ From f53d258cdf1d3ae4914ba0bb548faf6dba746ca4 Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Wed, 15 Mar 2017 16:46:18 -0500 Subject: [PATCH 03/32] Bump RELEASE for automatic rebuild --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ecc7d15..17a977e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ LABEL summary="MariaDB is a multi-user, multi-threaded SQL database server" \ io.openshift.expose-services="3306:mysql" \ io.openshift.tags="database,mysql,mariadb,mariadb101,galera" -ENV NAME=mariadb VERSION=10.1 RELEASE=2 ARCH=x86_64 +ENV NAME=mariadb VERSION=10.1 RELEASE=3 ARCH=x86_64 LABEL BZComponent="$NAME" \ Name="$FGC/$NAME" \ Version="$VERSION" \ From 920d97dada1537007e741852ad583e2a2a6c7465 Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Wed, 15 Mar 2017 16:49:21 -0500 Subject: [PATCH 04/32] Bump RELEASE for automatic rebuild --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 17a977e..2d4593f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ LABEL summary="MariaDB is a multi-user, multi-threaded SQL database server" \ io.openshift.expose-services="3306:mysql" \ io.openshift.tags="database,mysql,mariadb,mariadb101,galera" -ENV NAME=mariadb VERSION=10.1 RELEASE=3 ARCH=x86_64 +ENV NAME=mariadb VERSION=10.1 RELEASE=4 ARCH=x86_64 LABEL BZComponent="$NAME" \ Name="$FGC/$NAME" \ Version="$VERSION" \ From 562fb9c3bb949e7dc3ecd13241ccf5f0f53b233d Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Wed, 15 Mar 2017 17:23:52 -0500 Subject: [PATCH 05/32] Bump RELEASE for automatic rebuild --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2d4593f..bab37c1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ LABEL summary="MariaDB is a multi-user, multi-threaded SQL database server" \ io.openshift.expose-services="3306:mysql" \ io.openshift.tags="database,mysql,mariadb,mariadb101,galera" -ENV NAME=mariadb VERSION=10.1 RELEASE=4 ARCH=x86_64 +ENV NAME=mariadb VERSION=10.1 RELEASE=5 ARCH=x86_64 LABEL BZComponent="$NAME" \ Name="$FGC/$NAME" \ Version="$VERSION" \ From f04f11d512f447a4afdb963de74e508b80c4fc13 Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Wed, 29 Mar 2017 15:16:58 -0500 Subject: [PATCH 06/32] Bump RELEASE for automatic rebuild --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index bab37c1..f81b17b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ LABEL summary="MariaDB is a multi-user, multi-threaded SQL database server" \ io.openshift.expose-services="3306:mysql" \ io.openshift.tags="database,mysql,mariadb,mariadb101,galera" -ENV NAME=mariadb VERSION=10.1 RELEASE=5 ARCH=x86_64 +ENV NAME=mariadb VERSION=10.1 RELEASE=6 ARCH=x86_64 LABEL BZComponent="$NAME" \ Name="$FGC/$NAME" \ Version="$VERSION" \ From 0d4c8accced17eba922fbdadd21a345466ad866c Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Wed, 29 Mar 2017 22:38:26 -0500 Subject: [PATCH 07/32] Bump RELEASE for automatic rebuild --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f81b17b..a12348b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ LABEL summary="MariaDB is a multi-user, multi-threaded SQL database server" \ io.openshift.expose-services="3306:mysql" \ io.openshift.tags="database,mysql,mariadb,mariadb101,galera" -ENV NAME=mariadb VERSION=10.1 RELEASE=6 ARCH=x86_64 +ENV NAME=mariadb VERSION=10.1 RELEASE=7 ARCH=x86_64 LABEL BZComponent="$NAME" \ Name="$FGC/$NAME" \ Version="$VERSION" \ From 49bdaa1b66983b66224ab9cbf631f1f725f1e23b Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Thu, 20 Apr 2017 21:31:42 -0500 Subject: [PATCH 08/32] Bump RELEASE for automatic rebuild --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a12348b..1ddbcf5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ LABEL summary="MariaDB is a multi-user, multi-threaded SQL database server" \ io.openshift.expose-services="3306:mysql" \ io.openshift.tags="database,mysql,mariadb,mariadb101,galera" -ENV NAME=mariadb VERSION=10.1 RELEASE=7 ARCH=x86_64 +ENV NAME=mariadb VERSION=10.1 RELEASE=8 ARCH=x86_64 LABEL BZComponent="$NAME" \ Name="$FGC/$NAME" \ Version="$VERSION" \ From 06e630188cb74257426e4c28a7c3f96e2d64581e Mon Sep 17 00:00:00 2001 From: Adam Miller Date: Thu, 20 Apr 2017 21:33:13 -0500 Subject: [PATCH 09/32] Bump RELEASE for automatic rebuild --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1ddbcf5..881217c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ LABEL summary="MariaDB is a multi-user, multi-threaded SQL database server" \ io.openshift.expose-services="3306:mysql" \ io.openshift.tags="database,mysql,mariadb,mariadb101,galera" -ENV NAME=mariadb VERSION=10.1 RELEASE=8 ARCH=x86_64 +ENV NAME=mariadb VERSION=10.1 RELEASE=9 ARCH=x86_64 LABEL BZComponent="$NAME" \ Name="$FGC/$NAME" \ Version="$VERSION" \ From 050cc62c7db0cab13a83e8474e148ead9f16eba0 Mon Sep 17 00:00:00 2001 From: Honza Horak Date: Tue, 16 May 2017 11:44:20 +0200 Subject: [PATCH 10/32] Change version in FROM to 26 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 881217c..8eaa941 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM fedora:25 +FROM fedora:26 LABEL MAINTAINER "Honza Horak" From cc92471aa3a4cf3c1771d42dd2b2b83b8adc9bf2 Mon Sep 17 00:00:00 2001 From: Honza Horak Date: Tue, 16 May 2017 13:24:10 +0200 Subject: [PATCH 11/32] Change version in FROM to rawhide --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8eaa941..f100910 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM fedora:26 +FROM fedora:rawhide LABEL MAINTAINER "Honza Horak" From d07505b9c15d59b190dd842bfdee0d3b69925e68 Mon Sep 17 00:00:00 2001 From: Mohan Boddu Date: Tue, 9 Jan 2018 17:00:23 +0000 Subject: [PATCH 12/32] "Bump RELEASE for automatic rebuild" --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f100910..6360f8d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ LABEL summary="MariaDB is a multi-user, multi-threaded SQL database server" \ io.openshift.expose-services="3306:mysql" \ io.openshift.tags="database,mysql,mariadb,mariadb101,galera" -ENV NAME=mariadb VERSION=10.1 RELEASE=9 ARCH=x86_64 +ENV NAME=mariadb VERSION=10.1 RELEASE=10 ARCH=x86_64 LABEL BZComponent="$NAME" \ Name="$FGC/$NAME" \ Version="$VERSION" \ From d64887152406181a9d5cb13713fcd926da5845c7 Mon Sep 17 00:00:00 2001 From: Mohan Boddu Date: Tue, 9 Jan 2018 17:06:53 +0000 Subject: [PATCH 13/32] "Bump RELEASE for automatic rebuild" --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6360f8d..40bdc59 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ LABEL summary="MariaDB is a multi-user, multi-threaded SQL database server" \ io.openshift.expose-services="3306:mysql" \ io.openshift.tags="database,mysql,mariadb,mariadb101,galera" -ENV NAME=mariadb VERSION=10.1 RELEASE=10 ARCH=x86_64 +ENV NAME=mariadb VERSION=10.1 RELEASE=11 ARCH=x86_64 LABEL BZComponent="$NAME" \ Name="$FGC/$NAME" \ Version="$VERSION" \ From c499da4502f5294120ab35b216af155b5605f1b6 Mon Sep 17 00:00:00 2001 From: Mohan Boddu Date: Tue, 9 Jan 2018 22:37:54 +0000 Subject: [PATCH 14/32] "Bump RELEASE for automatic rebuild" --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 40bdc59..b6b3455 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ LABEL summary="MariaDB is a multi-user, multi-threaded SQL database server" \ io.openshift.expose-services="3306:mysql" \ io.openshift.tags="database,mysql,mariadb,mariadb101,galera" -ENV NAME=mariadb VERSION=10.1 RELEASE=11 ARCH=x86_64 +ENV NAME=mariadb VERSION=10.1 RELEASE=12 ARCH=x86_64 LABEL BZComponent="$NAME" \ Name="$FGC/$NAME" \ Version="$VERSION" \ From 919f612738d99c57ad2c9f2521066499415c685b Mon Sep 17 00:00:00 2001 From: Patrick Uiterwijk Date: Wed, 10 Jan 2018 01:03:48 +0100 Subject: [PATCH 15/32] Build 27 from f27 Signed-off-by: Patrick Uiterwijk --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b6b3455..88d26b0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM fedora:rawhide +FROM fedora:27 LABEL MAINTAINER "Honza Horak" From 1a981e0751d42094a2da2b4d3f5fa58a08ed0159 Mon Sep 17 00:00:00 2001 From: Mohan Boddu Date: Wed, 10 Jan 2018 00:05:13 +0000 Subject: [PATCH 16/32] "Bump RELEASE for automatic rebuild" --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 88d26b0..7158382 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ LABEL summary="MariaDB is a multi-user, multi-threaded SQL database server" \ io.openshift.expose-services="3306:mysql" \ io.openshift.tags="database,mysql,mariadb,mariadb101,galera" -ENV NAME=mariadb VERSION=10.1 RELEASE=12 ARCH=x86_64 +ENV NAME=mariadb VERSION=10.1 RELEASE=13 ARCH=x86_64 LABEL BZComponent="$NAME" \ Name="$FGC/$NAME" \ Version="$VERSION" \ From 98fc7d674bb406539107f6663f991fa1b4d83637 Mon Sep 17 00:00:00 2001 From: Mohan Boddu Date: Thu, 18 Jan 2018 22:35:35 +0000 Subject: [PATCH 17/32] "Bump RELEASE for automatic rebuild" --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 7158382..15c6adf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ LABEL summary="MariaDB is a multi-user, multi-threaded SQL database server" \ io.openshift.expose-services="3306:mysql" \ io.openshift.tags="database,mysql,mariadb,mariadb101,galera" -ENV NAME=mariadb VERSION=10.1 RELEASE=13 ARCH=x86_64 +ENV NAME=mariadb VERSION=10.1 RELEASE=14 ARCH=x86_64 LABEL BZComponent="$NAME" \ Name="$FGC/$NAME" \ Version="$VERSION" \ From 3421bdfac02ca878552ba52e519791e727691ddd Mon Sep 17 00:00:00 2001 From: Mohan Boddu Date: Thu, 18 Jan 2018 23:04:29 +0000 Subject: [PATCH 18/32] "Bump RELEASE for automatic rebuild" --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 15c6adf..794f8fc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ LABEL summary="MariaDB is a multi-user, multi-threaded SQL database server" \ io.openshift.expose-services="3306:mysql" \ io.openshift.tags="database,mysql,mariadb,mariadb101,galera" -ENV NAME=mariadb VERSION=10.1 RELEASE=14 ARCH=x86_64 +ENV NAME=mariadb VERSION=10.1 RELEASE=15 ARCH=x86_64 LABEL BZComponent="$NAME" \ Name="$FGC/$NAME" \ Version="$VERSION" \ From 27d5125f221d92469054e57ea04a0501cb903418 Mon Sep 17 00:00:00 2001 From: Mohan Boddu Date: Thu, 18 Jan 2018 23:18:28 +0000 Subject: [PATCH 19/32] "Bump RELEASE for automatic rebuild" --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 794f8fc..b247ad8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ LABEL summary="MariaDB is a multi-user, multi-threaded SQL database server" \ io.openshift.expose-services="3306:mysql" \ io.openshift.tags="database,mysql,mariadb,mariadb101,galera" -ENV NAME=mariadb VERSION=10.1 RELEASE=15 ARCH=x86_64 +ENV NAME=mariadb VERSION=10.1 RELEASE=16 ARCH=x86_64 LABEL BZComponent="$NAME" \ Name="$FGC/$NAME" \ Version="$VERSION" \ From dead8b119549190b6c9a47080331a92e0b21c759 Mon Sep 17 00:00:00 2001 From: Mohan Boddu Date: Thu, 18 Jan 2018 23:25:02 +0000 Subject: [PATCH 20/32] "Bump RELEASE for automatic rebuild" --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b247ad8..8548779 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ LABEL summary="MariaDB is a multi-user, multi-threaded SQL database server" \ io.openshift.expose-services="3306:mysql" \ io.openshift.tags="database,mysql,mariadb,mariadb101,galera" -ENV NAME=mariadb VERSION=10.1 RELEASE=16 ARCH=x86_64 +ENV NAME=mariadb VERSION=10.1 RELEASE=17 ARCH=x86_64 LABEL BZComponent="$NAME" \ Name="$FGC/$NAME" \ Version="$VERSION" \ From 4a9a5eb80a906e95590d546d086255290935662e Mon Sep 17 00:00:00 2001 From: Mohan Boddu Date: Fri, 16 Feb 2018 14:58:39 +0000 Subject: [PATCH 21/32] "Bump RELEASE for automatic rebuild" --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8548779..3892ecd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ LABEL summary="MariaDB is a multi-user, multi-threaded SQL database server" \ io.openshift.expose-services="3306:mysql" \ io.openshift.tags="database,mysql,mariadb,mariadb101,galera" -ENV NAME=mariadb VERSION=10.1 RELEASE=17 ARCH=x86_64 +ENV NAME=mariadb VERSION=10.1 RELEASE=18 ARCH=x86_64 LABEL BZComponent="$NAME" \ Name="$FGC/$NAME" \ Version="$VERSION" \ From 88366003222d641389d9e274c9693c6312ba650f Mon Sep 17 00:00:00 2001 From: Honza Horak Date: Thu, 1 Mar 2018 10:45:09 +0100 Subject: [PATCH 22/32] Do not verify ghost files, it's fine when they're missing --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3892ecd..8c0661a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,7 +35,7 @@ EXPOSE 3306 # to make sure of that. RUN INSTALL_PKGS="rsync tar gettext hostname bind-utils mariadb-server policycoreutils" && \ dnf install -y --setopt=tsflags=nodocs $INSTALL_PKGS && \ - rpm -V $INSTALL_PKGS && \ + rpm -V --noghost $INSTALL_PKGS && \ dnf clean all && \ mkdir -p /var/lib/mysql/data && chown -R mysql.0 /var/lib/mysql && \ test "$(id mysql)" = "uid=27(mysql) gid=27(mysql) groups=27(mysql)" From 40a99dae73d2c90d56f420b0e6dd86cbee0915c2 Mon Sep 17 00:00:00 2001 From: Mohan Boddu Date: Thu, 1 Mar 2018 17:01:07 +0000 Subject: [PATCH 23/32] "Bump RELEASE for automatic rebuild" --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8c0661a..f544a25 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ LABEL summary="MariaDB is a multi-user, multi-threaded SQL database server" \ io.openshift.expose-services="3306:mysql" \ io.openshift.tags="database,mysql,mariadb,mariadb101,galera" -ENV NAME=mariadb VERSION=10.1 RELEASE=18 ARCH=x86_64 +ENV NAME=mariadb VERSION=10.1 RELEASE=19 ARCH=x86_64 LABEL BZComponent="$NAME" \ Name="$FGC/$NAME" \ Version="$VERSION" \ From d617ef5998f77cacee4dee6d6510f154e1704ef9 Mon Sep 17 00:00:00 2001 From: Mohan Boddu Date: Thu, 1 Mar 2018 18:31:35 +0000 Subject: [PATCH 24/32] "Bump RELEASE for automatic rebuild" --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f544a25..509f056 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ LABEL summary="MariaDB is a multi-user, multi-threaded SQL database server" \ io.openshift.expose-services="3306:mysql" \ io.openshift.tags="database,mysql,mariadb,mariadb101,galera" -ENV NAME=mariadb VERSION=10.1 RELEASE=19 ARCH=x86_64 +ENV NAME=mariadb VERSION=10.1 RELEASE=20 ARCH=x86_64 LABEL BZComponent="$NAME" \ Name="$FGC/$NAME" \ Version="$VERSION" \ From b2b8007f1ea2986580b95158215a512a09e87ec0 Mon Sep 17 00:00:00 2001 From: Mohan Boddu Date: Thu, 15 Mar 2018 15:30:51 +0000 Subject: [PATCH 25/32] "Bump RELEASE for automatic rebuild" --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 509f056..51f6538 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ LABEL summary="MariaDB is a multi-user, multi-threaded SQL database server" \ io.openshift.expose-services="3306:mysql" \ io.openshift.tags="database,mysql,mariadb,mariadb101,galera" -ENV NAME=mariadb VERSION=10.1 RELEASE=20 ARCH=x86_64 +ENV NAME=mariadb VERSION=10.1 RELEASE=21 ARCH=x86_64 LABEL BZComponent="$NAME" \ Name="$FGC/$NAME" \ Version="$VERSION" \ From c05f2e329074dc2027c376ccf76aa89fcfc74026 Mon Sep 17 00:00:00 2001 From: Mohan Boddu Date: Thu, 29 Mar 2018 14:11:53 +0000 Subject: [PATCH 26/32] "Bump RELEASE for automatic rebuild" --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 51f6538..4e0b761 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,7 +21,7 @@ LABEL summary="MariaDB is a multi-user, multi-threaded SQL database server" \ io.openshift.expose-services="3306:mysql" \ io.openshift.tags="database,mysql,mariadb,mariadb101,galera" -ENV NAME=mariadb VERSION=10.1 RELEASE=21 ARCH=x86_64 +ENV NAME=mariadb VERSION=10.1 RELEASE=22 ARCH=x86_64 LABEL BZComponent="$NAME" \ Name="$FGC/$NAME" \ Version="$VERSION" \ From 27698bebfde38383193e2295e476291cf372a193 Mon Sep 17 00:00:00 2001 From: Michal Schorm Date: Wed, 4 Apr 2018 16:07:12 +0200 Subject: [PATCH 27/32] Update metadata about the version of MariaDB --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index f100910..ae4656a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,16 +12,16 @@ LABEL MAINTAINER "Honza Horak" # * $MYSQL_DATABASE - Name of the database to create # * $MYSQL_ROOT_PASSWORD (Optional) - Password for the 'root' MySQL account -ENV MYSQL_VERSION=10.1 \ +ENV MYSQL_VERSION=10.2 \ HOME=/var/lib/mysql LABEL summary="MariaDB is a multi-user, multi-threaded SQL database server" \ io.k8s.description="MariaDB is a multi-user, multi-threaded SQL database server" \ - io.k8s.display-name="MariaDB 10.1" \ + io.k8s.display-name="MariaDB 10.2" \ io.openshift.expose-services="3306:mysql" \ io.openshift.tags="database,mysql,mariadb,mariadb101,galera" -ENV NAME=mariadb VERSION=10.1 RELEASE=9 ARCH=x86_64 +ENV NAME=mariadb VERSION=10.2 RELEASE=14 ARCH=x86_64 LABEL BZComponent="$NAME" \ Name="$FGC/$NAME" \ Version="$VERSION" \ From fc4e117f96679f15874a302ba0afb36b64234076 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Mon, 27 Aug 2018 09:27:30 +0200 Subject: [PATCH 28/32] Drop Release label in favor of OSBS release_bump plugin. OSBS can automatically bump the release number, for that we just need to drop the label from the Dockerfile See https://pagure.io/ContainerSIG/container-sig/issue/1 Signed-off-by: Clement Verna --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 723fece..8ac0ead 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ -FROM fedora:27 +FROM FROM registry.fedoraproject.org/f30/s2i-core:latest + LABEL MAINTAINER "Honza Horak" @@ -21,11 +22,10 @@ LABEL summary="MariaDB is a multi-user, multi-threaded SQL database server" \ io.openshift.expose-services="3306:mysql" \ io.openshift.tags="database,mysql,mariadb,mariadb101,galera" -ENV NAME=mariadb VERSION=10.2 RELEASE=14 ARCH=x86_64 +ENV NAME=mariadb VERSION=10.2 ARCH=x86_64 LABEL BZComponent="$NAME" \ Name="$FGC/$NAME" \ Version="$VERSION" \ - Release="$RELEASE.$DISTTAG" \ Architecture="$ARCH" EXPOSE 3306 From 69f9b2cbd2cfd6b5609b1b9400b22a4e72f181d9 Mon Sep 17 00:00:00 2001 From: Clement Verna Date: Wed, 29 Aug 2018 16:16:46 +0200 Subject: [PATCH 29/32] Fix the FROM line in the Dockerfile Signed-off-by: Clement Verna --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8ac0ead..cebc5a3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM FROM registry.fedoraproject.org/f30/s2i-core:latest +FROM registry.fedoraproject.org/f30/s2i-core:latest LABEL MAINTAINER "Honza Horak" From d988f448d2e17cc929295e0e22fcce4bb6e63723 Mon Sep 17 00:00:00 2001 From: Michal Schorm Date: Tue, 3 Mar 2020 16:23:10 +0100 Subject: [PATCH 30/32] Pull changes from upstream and rebase for: rebuild for latest MariaDB 10.3 created from upstream commit: 9a5d82c444b6f70f822d0280eebd6c73a3d64e31 --- 10.3 | 1 + Dockerfile | 56 +- Dockerfile.fedora | 1 + help.md | 1 + {root => root-common}/etc/my.cnf | 3 + .../usr/bin/container-entrypoint | 0 {root => root-common}/usr/bin/mysqld-master | 0 {root => root-common}/usr/bin/mysqld-slave | 0 root-common/usr/bin/run-mysqld | 31 + .../usr/bin/run-mysqld-master | 24 +- root-common/usr/bin/run-mysqld-slave | 60 + root-common/usr/bin/usage | 4 + .../usr/libexec/container-setup | 3 +- root-common/usr/libexec/fix-permissions | 6 + .../container-scripts/mysql/cnf/40-paas.cnf | 4 + .../mysql/cnf/50-my-tuning.cnf | 1 - .../share/container-scripts/mysql/common.sh | 285 +++++ .../share/container-scripts/mysql/helpers.sh | 7 + .../mysql/init/40-datadir-action.sh | 112 ++ .../mysql/init/50-passwd-change.sh | 49 + .../container-scripts/mysql/post-init.sh | 0 .../mysql/pre-init/20-validate-variables.sh | 7 +- .../25-validate-replication-variables.sh | 5 +- .../mysql/pre-init/30-base-config.sh | 3 + .../mysql/pre-init/60-replication-config.sh | 17 + .../mysql/pre-init/70-s2i-config.sh | 6 + .../mysql/pre-init}/my-base.cnf.template | 0 .../mysql/pre-init}/my-master.cnf.template | 0 .../mysql/pre-init}/my-repl-gtid.cnf.template | 0 .../mysql/pre-init}/my-slave.cnf.template | 0 .../share/container-scripts/mysql/scl_enable | 0 root/help.1 | 773 +++++++----- root/usr/bin/cgroup-limits | 92 -- root/usr/bin/run-mysqld | 35 - root/usr/bin/run-mysqld-slave | 60 - .../share/container-scripts/mysql/README.md | 338 +++++- .../share/container-scripts/mysql/common.sh | 164 --- .../container-scripts/mysql/passwd-change.sh | 23 - s2i-common/bin/assemble | 13 + s2i-common/bin/run | 1 + s2i-common/bin/usage | 8 + sources | 0 test/mariadb-ephemeral-template.json | 254 ++++ test/run | 428 +++++-- test/run-openshift | 1 + test/run-openshift-local-cluster | 54 + test/run-openshift-remote-cluster | 30 + test/test-app/mysql-cfg/myconfig.cnf | 3 + test/test-app/mysql-data/init.sql | 4 + .../mysql-init/80-add-arbitrary-users.sh | 17 + test/test-app/mysql-init/90-init-db.sh | 12 + .../80-check-arbitrary-users.sh | 10 + test/test-lib-mysql.sh | 131 +++ test/test-lib-openshift.sh | 1043 +++++++++++++++++ test/test-lib.sh | 653 +++++++++++ 55 files changed, 3929 insertions(+), 904 deletions(-) create mode 120000 10.3 create mode 120000 Dockerfile.fedora create mode 120000 help.md rename {root => root-common}/etc/my.cnf (84%) rename {root => root-common}/usr/bin/container-entrypoint (100%) rename {root => root-common}/usr/bin/mysqld-master (100%) rename {root => root-common}/usr/bin/mysqld-slave (100%) create mode 100755 root-common/usr/bin/run-mysqld rename {root => root-common}/usr/bin/run-mysqld-master (56%) create mode 100755 root-common/usr/bin/run-mysqld-slave create mode 100755 root-common/usr/bin/usage rename {root => root-common}/usr/libexec/container-setup (95%) create mode 100755 root-common/usr/libexec/fix-permissions rename root/usr/share/container-scripts/mysql/my-paas.cnf.template => root-common/usr/share/container-scripts/mysql/cnf/40-paas.cnf (86%) rename root/usr/share/container-scripts/mysql/my-tuning.cnf.template => root-common/usr/share/container-scripts/mysql/cnf/50-my-tuning.cnf (95%) create mode 100644 root-common/usr/share/container-scripts/mysql/common.sh rename {root => root-common}/usr/share/container-scripts/mysql/helpers.sh (75%) create mode 100644 root-common/usr/share/container-scripts/mysql/init/40-datadir-action.sh create mode 100644 root-common/usr/share/container-scripts/mysql/init/50-passwd-change.sh rename {root => root-common}/usr/share/container-scripts/mysql/post-init.sh (100%) rename root/usr/share/container-scripts/mysql/validate-variables.sh => root-common/usr/share/container-scripts/mysql/pre-init/20-validate-variables.sh (93%) rename root/usr/share/container-scripts/mysql/validate-replication-variables.sh => root-common/usr/share/container-scripts/mysql/pre-init/25-validate-replication-variables.sh (87%) create mode 100644 root-common/usr/share/container-scripts/mysql/pre-init/30-base-config.sh create mode 100644 root-common/usr/share/container-scripts/mysql/pre-init/60-replication-config.sh create mode 100644 root-common/usr/share/container-scripts/mysql/pre-init/70-s2i-config.sh rename {root/usr/share/container-scripts/mysql => root-common/usr/share/container-scripts/mysql/pre-init}/my-base.cnf.template (100%) rename {root/usr/share/container-scripts/mysql => root-common/usr/share/container-scripts/mysql/pre-init}/my-master.cnf.template (100%) rename {root/usr/share/container-scripts/mysql => root-common/usr/share/container-scripts/mysql/pre-init}/my-repl-gtid.cnf.template (100%) rename {root/usr/share/container-scripts/mysql => root-common/usr/share/container-scripts/mysql/pre-init}/my-slave.cnf.template (100%) rename {root => root-common}/usr/share/container-scripts/mysql/scl_enable (100%) delete mode 100755 root/usr/bin/cgroup-limits delete mode 100755 root/usr/bin/run-mysqld delete mode 100755 root/usr/bin/run-mysqld-slave delete mode 100644 root/usr/share/container-scripts/mysql/common.sh delete mode 100644 root/usr/share/container-scripts/mysql/passwd-change.sh create mode 100755 s2i-common/bin/assemble create mode 120000 s2i-common/bin/run create mode 100755 s2i-common/bin/usage delete mode 100644 sources create mode 100644 test/mariadb-ephemeral-template.json create mode 120000 test/run-openshift create mode 100755 test/run-openshift-local-cluster create mode 100755 test/run-openshift-remote-cluster create mode 100644 test/test-app/mysql-cfg/myconfig.cnf create mode 100644 test/test-app/mysql-data/init.sql create mode 100644 test/test-app/mysql-init/80-add-arbitrary-users.sh create mode 100644 test/test-app/mysql-init/90-init-db.sh create mode 100644 test/test-app/mysql-pre-init/80-check-arbitrary-users.sh create mode 100755 test/test-lib-mysql.sh create mode 100644 test/test-lib-openshift.sh create mode 100644 test/test-lib.sh diff --git a/10.3 b/10.3 new file mode 120000 index 0000000..945c9b4 --- /dev/null +++ b/10.3 @@ -0,0 +1 @@ +. \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index cebc5a3..01f4a92 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,5 @@ FROM registry.fedoraproject.org/f30/s2i-core:latest - -LABEL MAINTAINER "Honza Horak" - # MariaDB image for OpenShift. # # Volumes: @@ -13,50 +10,61 @@ LABEL MAINTAINER "Honza Horak" # * $MYSQL_DATABASE - Name of the database to create # * $MYSQL_ROOT_PASSWORD (Optional) - Password for the 'root' MySQL account -ENV MYSQL_VERSION=10.2 \ - HOME=/var/lib/mysql +ENV MYSQL_VERSION=10.3 \ + APP_DATA=/opt/app-root/src \ + HOME=/var/lib/mysql \ + NAME=mariadb \ + VERSION=10.3 \ + ARCH=x86_64 \ + SUMMARY="MariaDB 10.3 SQL database server" \ + DESCRIPTION="MariaDB is a multi-user, multi-threaded SQL database server. The container \ +image provides a containerized packaging of the MariaDB mysqld daemon and client application. \ +The mysqld server daemon accepts connections from clients and provides access to content from \ +MariaDB databases on behalf of the clients." -LABEL summary="MariaDB is a multi-user, multi-threaded SQL database server" \ +LABEL summary="$SUMMARY" \ + description="$DESCRIPTION" \ io.k8s.description="MariaDB is a multi-user, multi-threaded SQL database server" \ - io.k8s.display-name="MariaDB 10.2" \ + io.k8s.display-name="MariaDB 10.3" \ io.openshift.expose-services="3306:mysql" \ - io.openshift.tags="database,mysql,mariadb,mariadb101,galera" - -ENV NAME=mariadb VERSION=10.2 ARCH=x86_64 -LABEL BZComponent="$NAME" \ - Name="$FGC/$NAME" \ - Version="$VERSION" \ - Architecture="$ARCH" + io.openshift.tags="database,mysql,mariadb,mariadb103,galera" \ + com.redhat.component="$NAME" \ + name="$FGC/$NAME" \ + version="$VERSION" \ + usage="docker run -d -e MYSQL_USER=user -e MYSQL_PASSWORD=pass -e MYSQL_DATABASE=db -p 3306:3306 $FGC/$NAME" \ + maintainer="SoftwareCollections.org " EXPOSE 3306 # This image must forever use UID 27 for mysql user so our volumes are # safe in the future. This should *never* change, the last test is there # to make sure of that. -RUN INSTALL_PKGS="rsync tar gettext hostname bind-utils mariadb-server policycoreutils" && \ +RUN INSTALL_PKGS="rsync tar gettext hostname bind-utils groff-base mariadb mariadb-server policycoreutils" && \ dnf install -y --setopt=tsflags=nodocs $INSTALL_PKGS && \ - rpm -V --noghost $INSTALL_PKGS && \ + rpm -V $INSTALL_PKGS && \ dnf clean all && \ mkdir -p /var/lib/mysql/data && chown -R mysql.0 /var/lib/mysql && \ test "$(id mysql)" = "uid=27(mysql) gid=27(mysql) groups=27(mysql)" -# On Fedora, we fake missing python binary. In case user installs the python2 -# in the container, this hack will be removed by installing /usr/bin/python from RPM. -RUN ln -s /usr/bin/python3 /usr/bin/python - # Get prefix path and path to scripts rather than hard-code them in scripts ENV CONTAINER_SCRIPTS_PATH=/usr/share/container-scripts/mysql \ MYSQL_PREFIX=/usr -ADD root / +COPY 10.3/root-common / +COPY 10.3/s2i-common/bin/ $STI_SCRIPTS_PATH +COPY 10.3/root / # this is needed due to issues with squash # when this directory gets rm'd by the container-setup # script. -RUN rm -rf /etc/my.cnf.d/* -RUN /usr/libexec/container-setup +# Also reset permissions of filesystem to default values +RUN rm -rf /etc/my.cnf.d/* && \ + /usr/libexec/container-setup && \ + rpm-file-permissions -VOLUME ["/var/lib/mysql/data"] +# Not using VOLUME statement since it's not working in OpenShift Online: +# https://github.com/sclorg/httpd-container/issues/30 +# VOLUME ["/var/lib/mysql/data"] USER 27 diff --git a/Dockerfile.fedora b/Dockerfile.fedora new file mode 120000 index 0000000..1d1fe94 --- /dev/null +++ b/Dockerfile.fedora @@ -0,0 +1 @@ +Dockerfile \ No newline at end of file diff --git a/help.md b/help.md new file mode 120000 index 0000000..42061c0 --- /dev/null +++ b/help.md @@ -0,0 +1 @@ +README.md \ No newline at end of file diff --git a/root/etc/my.cnf b/root-common/etc/my.cnf similarity index 84% rename from root/etc/my.cnf rename to root-common/etc/my.cnf index 0844075..bfdfbe9 100644 --- a/root/etc/my.cnf +++ b/root-common/etc/my.cnf @@ -9,4 +9,7 @@ skip_name_resolve # http://www.chriscalender.com/ignoring-the-lostfound-directory-in-your-datadir/ ignore-db-dir=lost+found +# GlusterFS equivalent of 'lost+found' +ignore-db-dir=.trashcan + !includedir /etc/my.cnf.d diff --git a/root/usr/bin/container-entrypoint b/root-common/usr/bin/container-entrypoint similarity index 100% rename from root/usr/bin/container-entrypoint rename to root-common/usr/bin/container-entrypoint diff --git a/root/usr/bin/mysqld-master b/root-common/usr/bin/mysqld-master similarity index 100% rename from root/usr/bin/mysqld-master rename to root-common/usr/bin/mysqld-master diff --git a/root/usr/bin/mysqld-slave b/root-common/usr/bin/mysqld-slave similarity index 100% rename from root/usr/bin/mysqld-slave rename to root-common/usr/bin/mysqld-slave diff --git a/root-common/usr/bin/run-mysqld b/root-common/usr/bin/run-mysqld new file mode 100755 index 0000000..2aa2fa3 --- /dev/null +++ b/root-common/usr/bin/run-mysqld @@ -0,0 +1,31 @@ +#!/bin/bash + +export_vars=$(cgroup-limits); export $export_vars +source ${CONTAINER_SCRIPTS_PATH}/common.sh +set -eu +if [[ -v DEBUG_IGNORE_SCRIPT_FAILURES ]]; then + set +e +fi + +export_setting_variables + +log_volume_info $MYSQL_DATADIR + +# pre-init files +process_extending_files ${APP_DATA}/mysql-pre-init/ ${CONTAINER_SCRIPTS_PATH}/pre-init/ + +if [ ! -d "$MYSQL_DATADIR/mysql" ]; then + initialize_database "$@" +else + start_local_mysql "$@" +fi + +# init files +process_extending_files ${APP_DATA}/mysql-init/ ${CONTAINER_SCRIPTS_PATH}/init/ + +# Restart the MySQL server with public IP bindings +shutdown_local_mysql +unset_env_vars +log_volume_info $MYSQL_DATADIR +log_info 'Running final exec -- Only MySQL server logs after this point' +exec ${MYSQL_PREFIX}/libexec/mysqld --defaults-file=$MYSQL_DEFAULTS_FILE "$@" 2>&1 diff --git a/root/usr/bin/run-mysqld-master b/root-common/usr/bin/run-mysqld-master similarity index 56% rename from root/usr/bin/run-mysqld-master rename to root-common/usr/bin/run-mysqld-master index 054889e..5550cb7 100755 --- a/root/usr/bin/run-mysqld-master +++ b/root-common/usr/bin/run-mysqld-master @@ -2,26 +2,26 @@ # # This is an entrypoint that runs the MySQL server in the 'master' mode. # + export_vars=$(cgroup-limits); export $export_vars source ${CONTAINER_SCRIPTS_PATH}/common.sh set -eu +if [[ -v DEBUG_IGNORE_SCRIPT_FAILURES ]]; then + set +e +fi + +export_setting_variables + +log_volume_info $MYSQL_DATADIR export MYSQL_RUNNING_AS_MASTER=1 -[ -f ${CONTAINER_SCRIPTS_PATH}/validate_replication_variables.sh ] && source ${CONTAINER_SCRIPTS_PATH}/validate_replication_variables.sh -[ -f ${CONTAINER_SCRIPTS_PATH}/validate_variables.sh ] && source ${CONTAINER_SCRIPTS_PATH}/validate_variables.sh - # The 'server-id' for master needs to be constant export MYSQL_SERVER_ID=1 log_info "The 'master' server-id is ${MYSQL_SERVER_ID}" -# Process the MySQL configuration files -log_info 'Processing MySQL configuration files ...' -envsubst < ${CONTAINER_SCRIPTS_PATH}/my-base.cnf.template > /etc/my.cnf.d/base.cnf -envsubst < ${CONTAINER_SCRIPTS_PATH}/my-paas.cnf.template > /etc/my.cnf.d/paas.cnf -envsubst < ${CONTAINER_SCRIPTS_PATH}/my-master.cnf.template > /etc/my.cnf.d/master.cnf -envsubst < ${CONTAINER_SCRIPTS_PATH}/my-repl-gtid.cnf.template > /etc/my.cnf.d/repl-gtid.cnf -envsubst < ${CONTAINER_SCRIPTS_PATH}/my-tuning.cnf.template > /etc/my.cnf.d/tuning.cnf +# pre-init files +process_extending_files ${APP_DATA}/mysql-pre-init/ ${CONTAINER_SCRIPTS_PATH}/pre-init/ if [ ! -d "$MYSQL_DATADIR/mysql" ]; then initialize_database "$@" @@ -39,8 +39,8 @@ mysql $mysql_flags <&1 diff --git a/root-common/usr/bin/usage b/root-common/usr/bin/usage new file mode 100755 index 0000000..feafb93 --- /dev/null +++ b/root-common/usr/bin/usage @@ -0,0 +1,4 @@ +#!/bin/bash + +cat /usr/share/container-scripts/mysql/README.md + diff --git a/root/usr/libexec/container-setup b/root-common/usr/libexec/container-setup similarity index 95% rename from root/usr/libexec/container-setup rename to root-common/usr/libexec/container-setup index 29c6ed2..6160d4e 100755 --- a/root/usr/libexec/container-setup +++ b/root-common/usr/libexec/container-setup @@ -54,5 +54,6 @@ restorecon -R /var/lib/mysql # arbitrary UID # When only specifying user, group is 0, that's why /var/lib/mysql must have # owner mysql.0; that allows to avoid a+rwx for this dir -chmod g+w -R /var/lib/mysql ${MYSQL_CONFIG_FILE}.d +/usr/libexec/fix-permissions /var/lib/mysql ${MYSQL_CONFIG_FILE}.d ${APP_DATA}/.. +usermod -a -G root mysql diff --git a/root-common/usr/libexec/fix-permissions b/root-common/usr/libexec/fix-permissions new file mode 100755 index 0000000..820e718 --- /dev/null +++ b/root-common/usr/libexec/fix-permissions @@ -0,0 +1,6 @@ +#!/bin/sh +# Fix permissions on the given directory to allow group read/write of +# regular files and execute of directories. +find $@ -exec chown mysql:0 {} \; +find $@ -exec chmod g+rw {} \; +find $@ -type d -exec chmod g+x {} + diff --git a/root/usr/share/container-scripts/mysql/my-paas.cnf.template b/root-common/usr/share/container-scripts/mysql/cnf/40-paas.cnf similarity index 86% rename from root/usr/share/container-scripts/mysql/my-paas.cnf.template rename to root-common/usr/share/container-scripts/mysql/cnf/40-paas.cnf index 11ddd1f..e79f2c5 100644 --- a/root/usr/share/container-scripts/mysql/my-paas.cnf.template +++ b/root-common/usr/share/container-scripts/mysql/cnf/40-paas.cnf @@ -6,6 +6,10 @@ # Sets how the table names are stored and compared. Default: 0 lower_case_table_names = ${MYSQL_LOWER_CASE_TABLE_NAMES} +# Sets whether queries should be logged +general_log = ${MYSQL_LOG_QUERIES_ENABLED} +general_log_file = ${MYSQL_DATADIR}/mysql-query.log + # The maximum permitted number of simultaneous client connections. Default: 151 max_connections = ${MYSQL_MAX_CONNECTIONS} diff --git a/root/usr/share/container-scripts/mysql/my-tuning.cnf.template b/root-common/usr/share/container-scripts/mysql/cnf/50-my-tuning.cnf similarity index 95% rename from root/usr/share/container-scripts/mysql/my-tuning.cnf.template rename to root-common/usr/share/container-scripts/mysql/cnf/50-my-tuning.cnf index e90b69a..e6b33f4 100644 --- a/root/usr/share/container-scripts/mysql/my-tuning.cnf.template +++ b/root-common/usr/share/container-scripts/mysql/cnf/50-my-tuning.cnf @@ -11,7 +11,6 @@ myisam_sort_buffer_size = 2M # It is recommended that innodb_buffer_pool_size is configured to 50 to 75 percent of system memory. innodb_buffer_pool_size = ${MYSQL_INNODB_BUFFER_POOL_SIZE} -innodb_additional_mem_pool_size = 2M # Set .._log_file_size to 25 % of buffer pool size innodb_log_file_size = ${MYSQL_INNODB_LOG_FILE_SIZE} innodb_log_buffer_size = ${MYSQL_INNODB_LOG_BUFFER_SIZE} diff --git a/root-common/usr/share/container-scripts/mysql/common.sh b/root-common/usr/share/container-scripts/mysql/common.sh new file mode 100644 index 0000000..f214017 --- /dev/null +++ b/root-common/usr/share/container-scripts/mysql/common.sh @@ -0,0 +1,285 @@ +#!/bin/bash + +source ${CONTAINER_SCRIPTS_PATH}/helpers.sh + +# Data directory where MySQL database files live. The data subdirectory is here +# because .bashrc and my.cnf both live in /var/lib/mysql/ and we don't want a +# volume to override it. +export MYSQL_DATADIR=/var/lib/mysql/data + +# Configuration settings. +export MYSQL_DEFAULTS_FILE=${MYSQL_DEFAULTS_FILE:-/etc/my.cnf} + +function export_setting_variables() { + export MYSQL_BINLOG_FORMAT=${MYSQL_BINLOG_FORMAT:-STATEMENT} + export MYSQL_LOWER_CASE_TABLE_NAMES=${MYSQL_LOWER_CASE_TABLE_NAMES:-0} + export MYSQL_LOG_QUERIES_ENABLED=${MYSQL_LOG_QUERIES_ENABLED:-0} + export MYSQL_MAX_CONNECTIONS=${MYSQL_MAX_CONNECTIONS:-151} + export MYSQL_FT_MIN_WORD_LEN=${MYSQL_FT_MIN_WORD_LEN:-4} + export MYSQL_FT_MAX_WORD_LEN=${MYSQL_FT_MAX_WORD_LEN:-20} + export MYSQL_AIO=${MYSQL_AIO:-1} + export MYSQL_MAX_ALLOWED_PACKET=${MYSQL_MAX_ALLOWED_PACKET:-200M} + export MYSQL_TABLE_OPEN_CACHE=${MYSQL_TABLE_OPEN_CACHE:-400} + export MYSQL_SORT_BUFFER_SIZE=${MYSQL_SORT_BUFFER_SIZE:-256K} + + # Export memory limit variables and calculate limits + local export_vars=$(cgroup-limits) && export $export_vars || exit 1 + if [ -n "${NO_MEMORY_LIMIT:-}" -o -z "${MEMORY_LIMIT_IN_BYTES:-}" ]; then + export MYSQL_KEY_BUFFER_SIZE=${MYSQL_KEY_BUFFER_SIZE:-32M} + export MYSQL_READ_BUFFER_SIZE=${MYSQL_READ_BUFFER_SIZE:-8M} + export MYSQL_INNODB_BUFFER_POOL_SIZE=${MYSQL_INNODB_BUFFER_POOL_SIZE:-32M} + export MYSQL_INNODB_LOG_FILE_SIZE=${MYSQL_INNODB_LOG_FILE_SIZE:-8M} + export MYSQL_INNODB_LOG_BUFFER_SIZE=${MYSQL_INNODB_LOG_BUFFER_SIZE:-8M} + else + export MYSQL_KEY_BUFFER_SIZE=${MYSQL_KEY_BUFFER_SIZE:-$((MEMORY_LIMIT_IN_BYTES/1024/1024/10))M} + export MYSQL_READ_BUFFER_SIZE=${MYSQL_READ_BUFFER_SIZE:-$((MEMORY_LIMIT_IN_BYTES/1024/1024/20))M} + export MYSQL_INNODB_BUFFER_POOL_SIZE=${MYSQL_INNODB_BUFFER_POOL_SIZE:-$((MEMORY_LIMIT_IN_BYTES/1024/1024/2))M} + # We are multiplying by 15 first and dividing by 100 later so we get as much + # precision as possible with whole numbers. Result is 15% of memory. + export MYSQL_INNODB_LOG_FILE_SIZE=${MYSQL_INNODB_LOG_FILE_SIZE:-$((MEMORY_LIMIT_IN_BYTES*15/1024/1024/100))M} + export MYSQL_INNODB_LOG_BUFFER_SIZE=${MYSQL_INNODB_LOG_BUFFER_SIZE:-$((MEMORY_LIMIT_IN_BYTES*15/1024/1024/100))M} + fi + export MYSQL_DATADIR_ACTION=${MYSQL_DATADIR_ACTION:-upgrade-warn} +} + +# this stores whether the database was initialized from empty datadir +export MYSQL_DATADIR_FIRST_INIT=false + +# Be paranoid and stricter than we should be. +# https://dev.mysql.com/doc/refman/en/identifiers.html +mysql_identifier_regex='^[a-zA-Z0-9_]+$' +mysql_password_regex='^[a-zA-Z0-9_~!@#$%^&*()-=<>,.?;:|]+$' + +# Variables that are used to connect to local mysql during initialization +mysql_flags="-u root --socket=/tmp/mysql.sock" +admin_flags="--defaults-file=$MYSQL_DEFAULTS_FILE $mysql_flags" + +# Make sure env variables don't propagate to mysqld process. +function unset_env_vars() { + log_info 'Cleaning up environment variables MYSQL_USER, MYSQL_PASSWORD, MYSQL_DATABASE and MYSQL_ROOT_PASSWORD ...' + unset MYSQL_USER MYSQL_PASSWORD MYSQL_DATABASE MYSQL_ROOT_PASSWORD +} + +# Poll until MySQL responds to our ping. +function wait_for_mysql() { + pid=$1 ; shift + + while true; do + if [ -d "/proc/$pid" ]; then + mysqladmin $admin_flags ping &>/dev/null && log_info "MySQL started successfully" && return 0 + else + return 1 + fi + log_info "Waiting for MySQL to start ..." + sleep 1 + done +} + +# Start local MySQL server with a defaults file +function start_local_mysql() { + log_info 'Starting MySQL server with disabled networking ...' + ${MYSQL_PREFIX}/libexec/mysqld \ + --defaults-file=$MYSQL_DEFAULTS_FILE \ + --skip-networking --socket=/tmp/mysql.sock "$@" & + mysql_pid=$! + wait_for_mysql $mysql_pid +} + +# Shutdown mysql flushing privileges +function shutdown_local_mysql() { + log_info 'Shutting down MySQL ...' + mysqladmin $admin_flags flush-privileges shutdown +} + +# Initialize the MySQL database (create user accounts and the initial database) +function initialize_database() { + log_info 'Initializing database ...' + log_info 'Running mysql_install_db ...' + # Using --rpm since we need mysql_install_db behaves as in RPM + mysql_install_db --rpm --datadir=$MYSQL_DATADIR + start_local_mysql "$@" + + # Running mysql_upgrade creates the mysql_upgrade_info file in the data dir, + # which is necessary to detect which version of the mysqld daemon created the data. + # Checking empty file should not take longer than a second and one extra check should not harm. + mysql_upgrade ${admin_flags} + + if [ -v MYSQL_RUNNING_AS_SLAVE ]; then + log_info 'Initialization finished' + return 0 + fi + + if [ -v MYSQL_RUNNING_AS_MASTER ]; then + # Save master status into a separate database. + STATUS_INFO=$(mysql $admin_flags -e 'SHOW MASTER STATUS\G') + BINLOG_POSITION=$(echo "$STATUS_INFO" | grep 'Position:' | head -n 1 | sed -e 's/^\s*Position: //') + BINLOG_FILE=$(echo "$STATUS_INFO" | grep 'File:' | head -n 1 | sed -e 's/^\s*File: //') + GTID_INFO=$(mysql $admin_flags -e "SELECT BINLOG_GTID_POS('$BINLOG_FILE', '$BINLOG_POSITION') AS gtid_value \G") + GTID_VALUE=$(echo "$GTID_INFO" | grep 'gtid_value:' | head -n 1 | sed -e 's/^\s*gtid_value: //') + + mysqladmin $admin_flags create replication + mysql $admin_flags < "10.0" ] ; then +mysql $mysql_flags </dev/null && log_info "MySQL master is ready" && return 0 + sleep 1 + done +} + +# get_matched_files finds file for image extending +function get_matched_files() { + local custom_dir default_dir + custom_dir="$1" + default_dir="$2" + files_matched="$3" + find "$default_dir" -maxdepth 1 -type f -name "$files_matched" -printf "%f\n" + [ -d "$custom_dir" ] && find "$custom_dir" -maxdepth 1 -type f -name "$files_matched" -printf "%f\n" +} + +# process_extending_files process extending files in $1 and $2 directories +# - source all *.sh files +# (if there are files with same name source only file from $1) +function process_extending_files() { + local custom_dir default_dir + custom_dir=$1 + default_dir=$2 + + while read filename ; do + echo "=> sourcing $filename ..." + # Custom file is prefered + if [ -f $custom_dir/$filename ]; then + source $custom_dir/$filename + else + source $default_dir/$filename + fi + done <<<"$(get_matched_files "$custom_dir" "$default_dir" '*.sh' | sort -u)" +} + +# process extending config files in $1 and $2 directories +# - expand variables in *.cnf and copy the files into /etc/my.cnf.d directory +# (if there are files with same name source only file from $1) +function process_extending_config_files() { + local custom_dir default_dir + custom_dir=$1 + default_dir=$2 + + while read filename ; do + echo "=> sourcing $filename ..." + # Custom file is prefered + if [ -f $custom_dir/$filename ]; then + envsubst < $custom_dir/$filename > /etc/my.cnf.d/$filename + else + envsubst < $default_dir/$filename > /etc/my.cnf.d/$filename + fi + done <<<"$(get_matched_files "$custom_dir" "$default_dir" '*.cnf' | sort -u)" +} + +# Converts string version to the integer format (5.5.33 is converted to 505, +# 10.1.23-MariaDB is converted into 1001, etc. +function version2number() { + local version_major=$(echo "$1" | grep -o -e '^[0-9]*\.[0-9]*') + printf %d%02d ${version_major%%.*} ${version_major##*.} +} + +# Converts the version in format of an integer into major.minor +function number2version() { + local numver=${1} + echo $((numver / 100)).$((numver % 100)) +} + +# Prints version of the mysqld that is currently available (string) +function mysqld_version() { + ${MYSQL_PREFIX}/libexec/mysqld -V | awk '{print $3}' +} + +# Returns version from the daemon in integer format +function mysqld_compat_version() { + version2number $(mysqld_version) +} + +# Returns version from the datadir in the integer format +function get_datadir_version() { + local datadir="$1" + local upgrade_info_file=$(get_mysql_upgrade_info_file "$datadir") + [ -r "$upgrade_info_file" ] || return + local version_text=$(cat "$upgrade_info_file" | head -n 1) + version2number "${version_text}" +} + +# Returns name of the file in the datadir that holds version information about the data +function get_mysql_upgrade_info_file() { + local datadir="$1" + echo "$datadir/mysql_upgrade_info" +} + +# Writes version string of the daemon into mysql_upgrade_info file +# (should be only used when the file is missing and only during limited time; +# once most deployments include this version file, we should leave it on +# scripts to generate the file right after initialization or when upgrading) +function write_mysql_upgrade_info_file() { + local datadir="$1" + local version=$(mysqld_version) + local upgrade_info_file=$(get_mysql_upgrade_info_file "$datadir") + if [ -f "$datadir/mysql_upgrade_info" ] ; then + echo "File ${upgrade_info_file} exists, nothing is done." + else + log_info "Storing version '${version}' information into the data dir '${upgrade_info_file}'" + echo "${version}" > "${upgrade_info_file}" + mysqld_version >"$datadir/mysql_upgrade_info" + fi +} diff --git a/root/usr/share/container-scripts/mysql/helpers.sh b/root-common/usr/share/container-scripts/mysql/helpers.sh similarity index 75% rename from root/usr/share/container-scripts/mysql/helpers.sh rename to root-common/usr/share/container-scripts/mysql/helpers.sh index 4e832fc..22db289 100644 --- a/root/usr/share/container-scripts/mysql/helpers.sh +++ b/root-common/usr/share/container-scripts/mysql/helpers.sh @@ -2,6 +2,10 @@ function log_info { echo "---> `date +%T` $@" } +function log_warn { + echo "---> `date +%T` Warning: $@" +} + function log_and_run { log_info "Running $@" "$@" @@ -21,4 +25,7 @@ function log_volume_info { shift done set -e + if [[ -v DEBUG_IGNORE_SCRIPT_FAILURES ]]; then + set +e + fi } diff --git a/root-common/usr/share/container-scripts/mysql/init/40-datadir-action.sh b/root-common/usr/share/container-scripts/mysql/init/40-datadir-action.sh new file mode 100644 index 0000000..6198367 --- /dev/null +++ b/root-common/usr/share/container-scripts/mysql/init/40-datadir-action.sh @@ -0,0 +1,112 @@ +upstream_upgrade_info() { + echo -n "For upstream documentation about upgrading, see: " + case ${MYSQL_VERSION} in + 10.0) echo "https://mariadb.com/kb/en/library/upgrading-from-mariadb-55-to-mariadb-100/" ;; + 10.1) echo "https://mariadb.com/kb/en/library/upgrading-from-mariadb-100-to-mariadb-101/" ;; + 10.2) echo "https://mariadb.com/kb/en/library/upgrading-from-mariadb-101-to-mariadb-102/" ;; + 10.3) echo "https://mariadb.com/kb/en/library/upgrading-from-mariadb-102-to-mariadb-103/" ;; + 5.6) echo "https://dev.mysql.com/doc/refman/5.6/en/upgrading-from-previous-series.html" ;; + 5.7) echo "https://dev.mysql.com/doc/refman/5.7/en/upgrading-from-previous-series.html" ;; + *) echo "Non expected version '${MYSQL_VERSION}'" ; return 1 ;; + esac +} + +check_datadir_version() { + local datadir="$1" + local datadir_version=$(get_datadir_version "$datadir") + local mysqld_version=$(mysqld_compat_version) + local datadir_version_dot=$(number2version "${datadir_version}") + local mysqld_version_dot=$(number2version "${mysqld_version}") + + for datadir_action in ${MYSQL_DATADIR_ACTION//,/ } ; do + log_info "Running datadir action: ${datadir_action}" + case ${datadir_action} in + upgrade-auto|upgrade-warn) + if [ -z "${datadir_version}" ] || [ "${datadir_version}" -eq 0 ] ; then + # Writing the info file, since historically it was not written + log_warn "Version of the data could not be determined."\ + "It is because the file mysql_upgrade_info is missing in the data directory, which"\ + "is most probably because it was not created when initialization of data directory."\ + "In order to allow seamless updates to the next higher version in the future,"\ + "the file mysql_upgrade_info will be created."\ + "If the data directory was created with a different version than ${mysqld_version_dot},"\ + "it is required to run this container with the MYSQL_DATADIR_ACTION environment variable"\ + "set to 'force', or run 'mysql_upgrade' utility manually; the mysql_upgrade tool"\ + "checks the tables and creates such a file as well. $(upstream_upgrade_info)" + write_mysql_upgrade_info_file "${MYSQL_DATADIR}" + continue + # This is currently a dead-code, but should be enabled after the mysql_upgrade_info + # file gets to the deployments (after few months most of the deployments should already have the file) + log_warn "Version of the data could not be determined."\ + "Running such a container is risky."\ + "The current daemon version is ${mysqld_version_dot}."\ + "If you are not sure whether the data directory is compatible with the current"\ + "version ${mysqld_version_dot}, restore the data from a back-up."\ + "If restoring from a back-up is not possible, create a file 'mysql_upgrade_info'"\ + "that includes version information (${mysqld_version_dot} in this case) in the root"\ + "of the data directory."\ + "In order to create the 'mysql_upgrade_info' file, either run this container with"\ + "the MYSQL_DATADIR_ACTION environment variable set to 'force', or run 'mysql_upgrade' utility"\ + "manually; the mysql_upgrade tool checks the tables and creates such a file as well."\ + "That will enable correct upgrade check in the future. $(upstream_upgrade_info)" + fi + + if [ "${datadir_version}" -eq "${mysqld_version}" ] ; then + log_info "MySQL server version check passed, both server and data directory"\ + "are version ${mysqld_version_dot}." + continue + fi + + if [ $(( ${datadir_version} + 1 )) -eq "${mysqld_version}" -o "${datadir_version}" -eq 505 -a "${mysqld_version}" -eq 1000 ] ; then + log_warn "MySQL server is version ${mysqld_version_dot} and datadir is version"\ + "${datadir_version_dot}, which is a compatible combination." + if [ "${MYSQL_DATADIR_ACTION}" == 'upgrade-auto' ] ; then + log_info "The data directory will be upgraded automatically from ${datadir_version_dot}"\ + "to version ${mysqld_version_dot}. $(upstream_upgrade_info)" + log_and_run mysql_upgrade ${mysql_flags} + else + log_warn "Automatic upgrade is not turned on, proceed with the upgrade."\ + "In order to upgrade the data directory, run this container with the MYSQL_DATADIR_ACTION"\ + "environment variable set to 'upgrade-auto' or run mysql_upgrade manually. $(upstream_upgrade_info)" + fi + else + log_warn "MySQL server is version ${mysqld_version_dot} and datadir is version"\ + "${datadir_version_dot}, which are incompatible. Remember, that upgrade is only supported"\ + "by upstream from previous version and it is not allowed to skip versions. $(upstream_upgrade_info)" + if [ "${datadir_version}" -gt "${mysqld_version}" ] ; then + log_warn "Downgrading to the lower version is not supported. Consider"\ + "dumping data and load them again into a fresh instance. $(upstream_upgrade_info)" + fi + log_warn "Consider restoring the database from a back-up. To ignore this"\ + "warning, set 'MYSQL_DATADIR_ACTION' variable to 'upgrade-force', but this may result in data corruption. $(upstream_upgrade_info)" + return 1 + fi + ;; + + upgrade-force) + log_and_run mysql_upgrade ${mysql_flags} --force + ;; + + optimize) + log_and_run mysqlcheck ${mysql_flags} --optimize --all-databases --force + ;; + + analyze) + log_and_run mysqlcheck ${mysql_flags} --analyze --all-databases --force + ;; + + disable) + log_info "Nothing is done about the data directory." + ;; + *) + log_warn "Unknown value of MYSQL_DATADIR_ACTION variable: '${MYSQL_DATADIR_ACTION}', ignoring." + ;; + esac + done +} + +check_datadir_version "${MYSQL_DATADIR}" + +unset -f check_datadir_version upstream_upgrade_info + + diff --git a/root-common/usr/share/container-scripts/mysql/init/50-passwd-change.sh b/root-common/usr/share/container-scripts/mysql/init/50-passwd-change.sh new file mode 100644 index 0000000..9fa0018 --- /dev/null +++ b/root-common/usr/share/container-scripts/mysql/init/50-passwd-change.sh @@ -0,0 +1,49 @@ +password_change() { + log_info 'Setting passwords ...' + + # Set the password for MySQL user and root everytime this container is started. + # This allows to change the password by editing the deployment configuration. + if [[ -v MYSQL_USER && -v MYSQL_PASSWORD ]]; then +mysql $mysql_flags < "10.0" ] ; then +mysql $mysql_flags < "10.0" ] ; then +mysql $mysql_flags < /etc/my.cnf.d/base.cnf + diff --git a/root-common/usr/share/container-scripts/mysql/pre-init/60-replication-config.sh b/root-common/usr/share/container-scripts/mysql/pre-init/60-replication-config.sh new file mode 100644 index 0000000..a923476 --- /dev/null +++ b/root-common/usr/share/container-scripts/mysql/pre-init/60-replication-config.sh @@ -0,0 +1,17 @@ +# mysqld configuration for replication scenarios + +if [ -v MYSQL_RUNNING_AS_MASTER ] || [ -v MYSQL_RUNNING_AS_SLAVE ] ; then + log_info 'Processing basic MySQL configuration for replication (master and slave) files ...' + envsubst < ${CONTAINER_SCRIPTS_PATH}/pre-init/my-repl-gtid.cnf.template > /etc/my.cnf.d/repl-gtid.cnf +fi + +if [ -v MYSQL_RUNNING_AS_MASTER ] ; then + log_info 'Processing basic MySQL configuration for replication (master only) files ...' + envsubst < ${CONTAINER_SCRIPTS_PATH}/pre-init/my-master.cnf.template > /etc/my.cnf.d/master.cnf +fi + +if [ -v MYSQL_RUNNING_AS_SLAVE ] ; then + log_info 'Processing basic MySQL configuration for replication (slave only) files ...' + envsubst < ${CONTAINER_SCRIPTS_PATH}/pre-init/my-slave.cnf.template > /etc/my.cnf.d/slave.cnf +fi + diff --git a/root-common/usr/share/container-scripts/mysql/pre-init/70-s2i-config.sh b/root-common/usr/share/container-scripts/mysql/pre-init/70-s2i-config.sh new file mode 100644 index 0000000..7a8ae5a --- /dev/null +++ b/root-common/usr/share/container-scripts/mysql/pre-init/70-s2i-config.sh @@ -0,0 +1,6 @@ +# additional arbitrary mysqld configuration provided by user using s2i + +log_info 'Processing additional arbitrary MySQL configuration provided by s2i ...' + +process_extending_config_files ${APP_DATA}/mysql-cfg/ ${CONTAINER_SCRIPTS_PATH}/cnf/ + diff --git a/root/usr/share/container-scripts/mysql/my-base.cnf.template b/root-common/usr/share/container-scripts/mysql/pre-init/my-base.cnf.template similarity index 100% rename from root/usr/share/container-scripts/mysql/my-base.cnf.template rename to root-common/usr/share/container-scripts/mysql/pre-init/my-base.cnf.template diff --git a/root/usr/share/container-scripts/mysql/my-master.cnf.template b/root-common/usr/share/container-scripts/mysql/pre-init/my-master.cnf.template similarity index 100% rename from root/usr/share/container-scripts/mysql/my-master.cnf.template rename to root-common/usr/share/container-scripts/mysql/pre-init/my-master.cnf.template diff --git a/root/usr/share/container-scripts/mysql/my-repl-gtid.cnf.template b/root-common/usr/share/container-scripts/mysql/pre-init/my-repl-gtid.cnf.template similarity index 100% rename from root/usr/share/container-scripts/mysql/my-repl-gtid.cnf.template rename to root-common/usr/share/container-scripts/mysql/pre-init/my-repl-gtid.cnf.template diff --git a/root/usr/share/container-scripts/mysql/my-slave.cnf.template b/root-common/usr/share/container-scripts/mysql/pre-init/my-slave.cnf.template similarity index 100% rename from root/usr/share/container-scripts/mysql/my-slave.cnf.template rename to root-common/usr/share/container-scripts/mysql/pre-init/my-slave.cnf.template diff --git a/root/usr/share/container-scripts/mysql/scl_enable b/root-common/usr/share/container-scripts/mysql/scl_enable similarity index 100% rename from root/usr/share/container-scripts/mysql/scl_enable rename to root-common/usr/share/container-scripts/mysql/scl_enable diff --git a/root/help.1 b/root/help.1 index 59c37b8..6524577 100644 --- a/root/help.1 +++ b/root/help.1 @@ -1,332 +1,475 @@ -.\"t -.\" WARNING: Do not edit this file manually, it is generated from README.md automatically. -.\" -.\"t -.\" Automatically generated by Pandoc 1.16.0.2 -.\" -.TH "MARIADB\-101\-RHEL7" "1" "February 22, 2017" "Container Image Pages" "" -.hy -.SH MariaDB Docker image +.TH MariaDB 10.3 SQL Database Server Docker image .PP -This container image includes MariaDB server 10.1 for OpenShift and -general usage. -Users can choose between RHEL and CentOS based images. +This container image includes MariaDB 10.3 SQL database server for OpenShift and general usage. +Users can choose between RHEL, CentOS and Fedora based images. +The RHEL images are available in the Red Hat Container Catalog +\[la]https://access.redhat.com/containers/\[ra], +the CentOS images are available on Docker Hub +\[la]https://hub.docker.com/r/centos/\[ra], +and the Fedora images are available in Fedora Registry +\[la]https://registry.fedoraproject.org/\[ra]\&. +The resulting image can be run using podman +\[la]https://github.com/containers/libpod\[ra]\&. + .PP -Dockerfile for CentOS is called Dockerfile, Dockerfile for RHEL is -called Dockerfile.rhel7. -.SS Environment variables and volumes +Note: while the examples in this README are calling \fB\fCpodman\fR, you can replace any such calls by \fB\fCdocker\fR with the same arguments + +.SH Description .PP -The image recognizes the following environment variables that you can -set during initialization by passing \f[C]\-e\ VAR=VALUE\f[] to the -Docker run command. +This container image provides a containerized packaging of the MariaDB mysqld daemon +and client application. The mysqld server daemon accepts connections from clients +and provides access to content from MySQL databases on behalf of the clients. +You can find more information on the MariaDB project from the project Web site +( +\[la]https://mariadb.org/\[ra]). + +.SH Usage .PP -.TS -tab(@); -l l. -T{ -Variable name -T}@T{ -Description -T} -_ -T{ -\f[C]MYSQL_USER\f[] -T}@T{ -User name for MySQL account to be created -T} -T{ -\f[C]MYSQL_PASSWORD\f[] -T}@T{ -Password for the user account -T} -T{ -\f[C]MYSQL_DATABASE\f[] -T}@T{ -Database name -T} -T{ -\f[C]MYSQL_ROOT_PASSWORD\f[] -T}@T{ -Password for the root user (optional) -T} -.TE +For this, we will assume that you are using the MariaDB 10.3 container image from the +Red Hat Container Catalog called \fB\fCrhel8/mariadb\-103\fR\&. +If you want to set only the mandatory environment variables and not store +the database in a host directory, execute the following command: + .PP -The following environment variables influence the MySQL configuration -file. -They are all optional. -.PP -.TS -tab(@); -lw(17.2n) lw(35.5n) lw(17.2n). -T{ -Variable name -T}@T{ -Description -T}@T{ -Default -T} -_ -T{ -\f[C]MYSQL_LOWER_CASE_TABLE_NAMES\f[] -T}@T{ -Sets how the table names are stored and compared -T}@T{ -0 -T} -T{ -\f[C]MYSQL_MAX_CONNECTIONS\f[] -T}@T{ -The maximum permitted number of simultaneous client connections -T}@T{ -151 -T} -T{ -\f[C]MYSQL_MAX_ALLOWED_PACKET\f[] -T}@T{ -The maximum size of one packet or any generated/intermediate string -T}@T{ -200M -T} -T{ -\f[C]MYSQL_FT_MIN_WORD_LEN\f[] -T}@T{ -The minimum length of the word to be included in a FULLTEXT index -T}@T{ -4 -T} -T{ -\f[C]MYSQL_FT_MAX_WORD_LEN\f[] -T}@T{ -The maximum length of the word to be included in a FULLTEXT index -T}@T{ -20 -T} -T{ -\f[C]MYSQL_AIO\f[] -T}@T{ -Controls the \f[C]innodb_use_native_aio\f[] setting value in case the -native AIO is broken. -See http://help.directadmin.com/item.php?id=529 -T}@T{ -1 -T} -T{ -\f[C]MYSQL_TABLE_OPEN_CACHE\f[] -T}@T{ -The number of open tables for all threads -T}@T{ -400 -T} -T{ -\f[C]MYSQL_KEY_BUFFER_SIZE\f[] -T}@T{ -The size of the buffer used for index blocks -T}@T{ -32M (or 10% of available memory) -T} -T{ -\f[C]MYSQL_SORT_BUFFER_SIZE\f[] -T}@T{ -The size of the buffer used for sorting -T}@T{ -256K -T} -T{ -\f[C]MYSQL_READ_BUFFER_SIZE\f[] -T}@T{ -The size of the buffer used for a sequential scan -T}@T{ -8M (or 5% of available memory) -T} -T{ -\f[C]MYSQL_INNODB_BUFFER_POOL_SIZE\f[] -T}@T{ -The size of the buffer pool where InnoDB caches table and index data -T}@T{ -32M (or 50% of available memory) -T} -T{ -\f[C]MYSQL_INNODB_LOG_FILE_SIZE\f[] -T}@T{ -The size of each log file in a log group -T}@T{ -8M (or 15% of available available) -T} -T{ -\f[C]MYSQL_INNODB_LOG_BUFFER_SIZE\f[] -T}@T{ -The size of the buffer that InnoDB uses to write to the log files on -disk -T}@T{ -8M (or 15% of available memory) -T} -T{ -\f[C]MYSQL_DEFAULTS_FILE\f[] -T}@T{ -Point to an alternative configuration file -T}@T{ -/etc/my.cnf -T} -T{ -\f[C]MYSQL_BINLOG_FORMAT\f[] -T}@T{ -Set sets the binlog format, supported values are \f[C]row\f[] and -\f[C]statement\f[] -T}@T{ -statement -T} -.TE -.PP -You can also set the following mount points by passing the -\f[C]\-v\ /host:/container\f[] flag to Docker. -.PP -.TS -tab(@); -l l. -T{ -Volume mount point -T}@T{ -Description -T} -_ -T{ -\f[C]/var/lib/mysql/data\f[] -T}@T{ -MySQL data directory -T} -.TE -.PP -\f[B]Notice: When mouting a directory from the host into the container, -ensure that the mounted directory has the appropriate permissions and -that the owner and group of the directory matches the user UID or name -which is running inside the container.\f[] -.SS Usage -.PP -For this, we will assume that you are using the -\f[C]rhscl/mariadb\-100\-rhel7\f[] image. -If you want to set only the mandatory environment variables and not -store the database in a host directory, execute the following command: -.IP +.RS + .nf -\f[C] -$\ docker\ run\ \-d\ \-\-name\ mariadb_database\ \-e\ MYSQL_USER=user\ \-e\ MYSQL_PASSWORD=pass\ \-e\ MYSQL_DATABASE=db\ \-p\ 3306:3306\ rhscl/mariadb\-100\-rhel7 -\f[] +$ podman run \-d \-\-name mariadb\_database \-e MYSQL\_USER=user \-e MYSQL\_PASSWORD=pass \-e MYSQL\_DATABASE=db \-p 3306:3306 rhel8/mariadb\-103 + .fi +.RE + .PP -This will create a container named \f[C]mariadb_database\f[] running -MySQL with database \f[C]db\f[] and user with credentials -\f[C]user:pass\f[]. -Port 3306 will be exposed and mapped to the host. -If you want your database to be persistent across container executions, -also add a \f[C]\-v\ /host/db/path:/var/lib/mysql/data\f[] argument. -This will be the MySQL data directory. +This will create a container named \fB\fCmariadb\_database\fR running MySQL with database +\fB\fCdb\fR and user with credentials \fB\fCuser:pass\fR\&. Port 3306 will be exposed and mapped +to the host. If you want your database to be persistent across container executions, +also add a \fB\fC\-v /host/db/path:/var/lib/mysql/data\fR argument. This will be the MySQL +data directory. + .PP -If the database directory is not initialized, the entrypoint script will -first run -\f[C]mysql_install_db\f[] (https://dev.mysql.com/doc/refman/5.6/en/mysql-install-db.html) -and setup necessary database users and passwords. -After the database is initialized, or if it was already present, -\f[C]mysqld\f[] is executed and will run as PID 1. -You can stop the detached container by running -\f[C]docker\ stop\ mariadb_database\f[]. -.SS MariaDB auto\-tuning +If the database directory is not initialized, the entrypoint script will first +run \fB\fCmysql\_install\_db\fR +\[la]https://dev.mysql.com/doc/refman/5.6/en/mysql-install-db.html\[ra] +and setup necessary database users and passwords. After the database is initialized, +or if it was already present, \fB\fCmysqld\fR is executed and will run as PID 1. You can + stop the detached container by running \fB\fCpodman stop mariadb\_database\fR\&. + +.SH Environment variables and volumes .PP -When the MySQL image is run with the \f[C]\-\-memory\f[] parameter set -and you didn\[aq]t specify value for some parameters, their values will -be automatically calculated based on the available memory. +The image recognizes the following environment variables that you can set during +initialization by passing \fB\fC\-e VAR=VALUE\fR to the Docker run command. + .PP -.TS -tab(@); -l l l. -T{ -Variable name -T}@T{ -Configuration parameter -T}@T{ -Relative value -T} -_ -T{ -\f[C]MYSQL_KEY_BUFFER_SIZE\f[] -T}@T{ -\f[C]key_buffer_size\f[] -T}@T{ -10% -T} -T{ -\f[C]MYSQL_READ_BUFFER_SIZE\f[] -T}@T{ -\f[C]read_buffer_size\f[] -T}@T{ -5% -T} -T{ -\f[C]MYSQL_INNODB_BUFFER_POOL_SIZE\f[] -T}@T{ -\f[C]innodb_buffer_pool_size\f[] -T}@T{ -50% -T} -T{ -\f[C]MYSQL_INNODB_LOG_FILE_SIZE\f[] -T}@T{ -\f[C]innodb_log_file_size\f[] -T}@T{ -15% -T} -T{ -\f[C]MYSQL_INNODB_LOG_BUFFER_SIZE\f[] -T}@T{ -\f[C]innodb_log_buffer_size\f[] -T}@T{ -15% -T} -.TE -.SS MySQL root user +\fB\fB\fCMYSQL\_USER\fR\fP +.br + User name for MySQL account to be created + .PP -The root user has no password set by default, only allowing local -connections. -You can set it by setting the \f[C]MYSQL_ROOT_PASSWORD\f[] environment -variable. -This will allow you to login to the root account remotely. -Local connections will still not require a password. +\fB\fB\fCMYSQL\_PASSWORD\fR\fP +.br + Password for the user account + .PP -To disable remote root access, simply unset \f[C]MYSQL_ROOT_PASSWORD\f[] -and restart the container. -.SS Changing passwords +\fB\fB\fCMYSQL\_DATABASE\fR\fP +.br + Database name + .PP -Since passwords are part of the image configuration, the only supported -method to change passwords for the database user (\f[C]MYSQL_USER\f[]) -and root user is by changing the environment variables -\f[C]MYSQL_PASSWORD\f[] and \f[C]MYSQL_ROOT_PASSWORD\f[], respectively. +\fB\fB\fCMYSQL\_ROOT\_PASSWORD\fR\fP +.br + Password for the root user (optional) + .PP -Changing database passwords through SQL statements or any way other than -through the environment variables aforementioned will cause a mismatch -between the values stored in the variables and the actual passwords. -Whenever a database container starts it will reset the passwords to the -values stored in the environment variables. -.SS Default my.cnf file +The following environment variables influence the MySQL configuration file. They are all optional. + .PP -With environment variables we are able to customize a lot of different -parameters or configurations for the mysql bootstrap configurations. -If you\[aq]d prefer to use your own configuration file, you can override -the \f[C]MYSQL_DEFAULTS_FILE\f[] env variable with the full path of the -file you wish to use. -For example, the default location is \f[C]/etc/my.cnf\f[] but you can -change it to \f[C]/etc/mysql/my.cnf\f[] by setting -\f[C]MYSQL_DEFAULTS_FILE=/etc/mysql/my.cnf\f[] -.SS Changing the replication binlog_format +\fB\fB\fCMYSQL\_LOWER\_CASE\_TABLE\_NAMES (default: 0)\fR\fP +.br + Sets how the table names are stored and compared + .PP -Some applications may wish to use \f[C]row\f[] binlog_formats (for -example, those built with change\-data\-capture in mind). -The default replication/binlog format is \f[C]statement\f[] but to -change it you can set the \f[C]MYSQL_BINLOG_FORMAT\f[] environment -variable. -For example \f[C]MYSQL_BINLOG_FORMAT=row\f[]. -Now when you run the database with \f[C]master\f[] replication turned on -(ie, set the Docker/container \f[C]cmd\f[] to be -\f[C]run\-mysqld\-master\f[]) the binlog will emit the actual data for -the rows that change as opposed to the statements (ie, DML like -insert...) that caused the change. -.SH AUTHORS -Red Hat. +\fB\fB\fCMYSQL\_MAX\_CONNECTIONS (default: 151)\fR\fP +.br + The maximum permitted number of simultaneous client connections + +.PP +\fB\fB\fCMYSQL\_MAX\_ALLOWED\_PACKET (default: 200M)\fR\fP +.br + The maximum size of one packet or any generated/intermediate string + +.PP +\fB\fB\fCMYSQL\_FT\_MIN\_WORD\_LEN (default: 4)\fR\fP +.br + The minimum length of the word to be included in a FULLTEXT index + +.PP +\fB\fB\fCMYSQL\_FT\_MAX\_WORD\_LEN (default: 20)\fR\fP +.br + The maximum length of the word to be included in a FULLTEXT index + +.PP +\fB\fB\fCMYSQL\_AIO (default: 1)\fR\fP +.br + Controls the \fB\fCinnodb\_use\_native\_aio\fR setting value in case the native AIO is broken. See +\[la]http://help.directadmin.com/item.php?id=529\[ra] + +.PP +\fB\fB\fCMYSQL\_TABLE\_OPEN\_CACHE (default: 400)\fR\fP +.br + The number of open tables for all threads + +.PP +\fB\fB\fCMYSQL\_KEY\_BUFFER\_SIZE (default: 32M or 10% of available memory)\fR\fP +.br + The size of the buffer used for index blocks + +.PP +\fB\fB\fCMYSQL\_SORT\_BUFFER\_SIZE (default: 256K)\fR\fP +.br + The size of the buffer used for sorting + +.PP +\fB\fB\fCMYSQL\_READ\_BUFFER\_SIZE (default: 8M or 5% of available memory)\fR\fP +.br + The size of the buffer used for a sequential scan + +.PP +\fB\fB\fCMYSQL\_INNODB\_BUFFER\_POOL\_SIZE (default: 32M or 50% of available memory)\fR\fP +.br + The size of the buffer pool where InnoDB caches table and index data + +.PP +\fB\fB\fCMYSQL\_INNODB\_LOG\_FILE\_SIZE (default: 8M or 15% of available memory)\fR\fP +.br + The size of each log file in a log group + +.PP +\fB\fB\fCMYSQL\_INNODB\_LOG\_BUFFER\_SIZE (default: 8M or 15% of available memory)\fR\fP +.br + The size of the buffer that InnoDB uses to write to the log files on disk + +.PP +\fB\fB\fCMYSQL\_DEFAULTS\_FILE (default: /etc/my.cnf)\fR\fP +.br + Point to an alternative configuration file + +.PP +\fB\fB\fCMYSQL\_BINLOG\_FORMAT (default: statement)\fR\fP +.br + Set sets the binlog format, supported values are \fB\fCrow\fR and \fB\fCstatement\fR + +.PP +\fB\fB\fCMYSQL\_LOG\_QUERIES\_ENABLED (default: 0)\fR\fP +.br + To enable query logging set this to \fB\fC1\fR + +.PP +You can also set the following mount points by passing the \fB\fC\-v /host:/container\fR flag to Docker. + +.PP +\fB\fB\fC/var/lib/mysql/data\fR\fP +.br + MySQL data directory + +.PP +\fBNotice: When mouting a directory from the host into the container, ensure that the mounted +directory has the appropriate permissions and that the owner and group of the directory +matches the user UID or name which is running inside the container.\fP + +.SH MariaDB auto\-tuning +.PP +When the MySQL image is run with the \fB\fC\-\-memory\fR parameter set and you didn't +specify value for some parameters, their values will be automatically +calculated based on the available memory. + +.PP +\fB\fB\fCMYSQL\_KEY\_BUFFER\_SIZE (default: 10%)\fR\fP +.br + \fB\fCkey\_buffer\_size\fR + +.PP +\fB\fB\fCMYSQL\_READ\_BUFFER\_SIZE (default: 5%)\fR\fP +.br + \fB\fCread\_buffer\_size\fR + +.PP +\fB\fB\fCMYSQL\_INNODB\_BUFFER\_POOL\_SIZE (default: 50%)\fR\fP +.br + \fB\fCinnodb\_buffer\_pool\_size\fR + +.PP +\fB\fB\fCMYSQL\_INNODB\_LOG\_FILE\_SIZE (default: 15%)\fR\fP +.br + \fB\fCinnodb\_log\_file\_size\fR + +.PP +\fB\fB\fCMYSQL\_INNODB\_LOG\_BUFFER\_SIZE (default: 15%)\fR\fP +.br + \fB\fCinnodb\_log\_buffer\_size\fR + +.SH MySQL root user +.PP +The root user has no password set by default, only allowing local connections. +You can set it by setting the \fB\fCMYSQL\_ROOT\_PASSWORD\fR environment variable. This +will allow you to login to the root account remotely. Local connections will +still not require a password. + +.PP +To disable remote root access, simply unset \fB\fCMYSQL\_ROOT\_PASSWORD\fR and restart +the container. + +.SH Changing passwords +.PP +Since passwords are part of the image configuration, the only supported method +to change passwords for the database user (\fB\fCMYSQL\_USER\fR) and root user is by +changing the environment variables \fB\fCMYSQL\_PASSWORD\fR and \fB\fCMYSQL\_ROOT\_PASSWORD\fR, +respectively. + +.PP +Changing database passwords through SQL statements or any way other than through +the environment variables aforementioned will cause a mismatch between the +values stored in the variables and the actual passwords. Whenever a database +container starts it will reset the passwords to the values stored in the +environment variables. + +.SH Default my.cnf file +.PP +With environment variables we are able to customize a lot of different parameters +or configurations for the mysql bootstrap configurations. If you'd prefer to use +your own configuration file, you can override the \fB\fCMYSQL\_DEFAULTS\_FILE\fR env +variable with the full path of the file you wish to use. For example, the default +location is \fB\fC/etc/my.cnf\fR but you can change it to \fB\fC/etc/mysql/my.cnf\fR by setting + \fB\fCMYSQL\_DEFAULTS\_FILE=/etc/mysql/my.cnf\fR + +.SH Extending image +.PP +This image can be extended in Openshift using the \fB\fCSource\fR build strategy or via the standalone +source\-to\-image +\[la]https://github.com/openshift/source-to-image\[ra] application (where available). +For this, we will assume that you are using the \fB\fCrhscl/mariadb\-103\-rhel7\fR image, +available via \fB\fCmariadb:10.3\fR imagestream tag in Openshift. + +.PP +For example, to build a customized MariaDB database image \fB\fCmy\-mariadb\-rhel7\fR +with a configuration from \fB\fChttps://github.com/sclorg/mariadb\-container/tree/master/examples/extend\-image\fR run: + +.PP +.RS + +.nf +$ oc new\-app mariadb:10.3\~https://github.com/sclorg/mariadb\-container.git \\ + \-\-name my\-mariadb\-rhel7 \\ + \-\-context\-dir=examples/extend\-image \\ + \-\-env MYSQL\_OPERATIONS\_USER=opuser \\ + \-\-env MYSQL\_OPERATIONS\_PASSWORD=oppass \\ + \-\-env MYSQL\_DATABASE=opdb \\ + \-\-env MYSQL\_USER=user \\ + \-\-env MYSQL\_PASSWORD=pass + +.fi +.RE + +.PP +or via s2i: + +.PP +.RS + +.nf +$ s2i build \-\-context\-dir=examples/extend\-image https://github.com/sclorg/mariadb\-container.git rhscl/mariadb\-103\-rhel7 my\-mariadb\-rhel7 + +.fi +.RE + +.PP +The directory passed to Openshift can contain these directories: + +.PP +\fB\fCmysql\-cfg/\fR + When starting the container, files from this directory will be used as + a configuration for the \fB\fCmysqld\fR daemon. + \fB\fCenvsubst\fR command is run on this file to still allow customization of + the image using environmental variables + +.PP +\fB\fCmysql\-pre\-init/\fR + Shell scripts (\fB\fC*.sh\fR) available in this directory are sourced before + \fB\fCmysqld\fR daemon is started. + +.PP +\fB\fCmysql\-init/\fR + Shell scripts (\fB\fC*.sh\fR) available in this directory are sourced when + \fB\fCmysqld\fR daemon is started locally. In this phase, use \fB\fC${mysql\_flags}\fR + to connect to the locally running daemon, for example \fB\fCmysql $mysql\_flags < dump.sql\fR + +.PP +Variables that can be used in the scripts provided to s2i: + +.PP +\fB\fC$mysql\_flags\fR + arguments for the \fB\fCmysql\fR tool that will connect to the locally running \fB\fCmysqld\fR during initialization + +.PP +\fB\fC$MYSQL\_RUNNING\_AS\_MASTER\fR + variable defined when the container is run with \fB\fCrun\-mysqld\-master\fR command + +.PP +\fB\fC$MYSQL\_RUNNING\_AS\_SLAVE\fR + variable defined when the container is run with \fB\fCrun\-mysqld\-slave\fR command + +.PP +\fB\fC$MYSQL\_DATADIR\_FIRST\_INIT\fR + variable defined when the container was initialized from the empty data dir + +.PP +During the s2i build all provided files are copied into \fB\fC/opt/app\-root/src\fR +directory into the resulting image. If some configuration files are present +in the destination directory, files with the same name are overwritten. +Also only one file with the same name can be used for customization and user +provided files are preferred over default files in +\fB\fC/usr/share/container\-scripts/mysql/\fR\- so it is possible to overwrite them. + +.PP +Same configuration directory structure can be used to customize the image +every time the image is started using \fB\fCpodman run\fR\&. The directory has to be +mounted into \fB\fC/opt/app\-root/src/\fR in the image +(\fB\fC\-v ./image\-configuration/:/opt/app\-root/src/\fR). +This overwrites customization built into the image. + +.SH Securing the connection with SSL +.PP +In order to secure the connection with SSL, use the extending feature described +above. In particular, put the SSL certificates into a separate directory: + +.PP +.RS + +.nf +sslapp/mysql\-certs/server\-cert\-selfsigned.pem +sslapp/mysql\-certs/server\-key.pem + +.fi +.RE + +.PP +And then put a separate configuration file into mysql\-cfg: + +.PP +.RS + +.nf +$> cat sslapp/mysql\-cfg/ssl.cnf +[mysqld] +ssl\-key=${APP\_DATA}/mysql\-certs/server\-key.pem +ssl\-cert=${APP\_DATA}/mysql\-certs/server\-cert\-selfsigned.pem + +.fi +.RE + +.PP +Such a directory \fB\fCsslapp\fR can then be mounted into the container with \-v, +or a new container image can be built using s2i. + +.SH Upgrading and data directory version checking +.PP +MySQL and MariaDB use versions that consist of three numbers X.Y.Z (e.g. 5.6.23). +For version changes in Z part, the server's binary data format stays compatible and thus no +special upgrade procedure is needed. For upgrades from X.Y to X.Y+1, consider doing manual +steps as described at + +\[la]https://mariadb.com/kb/en/library/upgrading-from-mariadb-102-to-mariadb-103/\[ra] + +.PP +Skipping versions like from X.Y to X.Y+2 or downgrading to lower version is not supported; +the only exception is ugrading from MariaDB 5.5 to MariaDB 10.0. + +.PP +\fBImportant\fP: Upgrading to a new version is always risky and users are expected to make a full +back\-up of all data before. + +.PP +A safer solution to upgrade is to dump all data using \fB\fCmysqldump\fR or \fB\fCmysqldbexport\fR and then +load the data using \fB\fCmysql\fR or \fB\fCmysqldbimport\fR into an empty (freshly initialized) database. + +.PP +Another way of proceeding with the upgrade is starting the new version of the \fB\fCmysqld\fR daemon +and run \fB\fCmysql\_upgrade\fR right after the start. This so called in\-place upgrade is generally +faster for large data directory, but only possible if upgrading from the very previous version, +so skipping versions is not supported. + +.PP +This container detects whether the data needs to be upgraded using \fB\fCmysql\_upgrade\fR and +we can control it by setting \fB\fCMYSQL\_DATADIR\_ACTION\fR variable, which can have one or more of the following values: + +.RS +.IP \(bu 2 +\fB\fCupgrade\-warn\fR \-\- If the data version can be determined and the data come from a different version +of the daemon, a warning is printed but the container starts. This is the default value. +Since historically the version file \fB\fCmysql\_upgrade\_info\fR was not created, when using this option, +the version file is created if not exist, but no \fB\fCmysql\_upgrade\fR will be called. +However, this automatic creation will be removed after few months, since the version should be +created on most deployments at that point. +.IP \(bu 2 +\fB\fCupgrade\-auto\fR \-\- \fB\fCmysql\_upgrade\fR is run at the beginning of the container start, when the local +daemon is running, but only if the data version can be determined and the data come +with the very previous version. A warning is printed if the data come from even older +or newer version. This value effectively enables automatic upgrades, +but it is always risky and users should still back\-up all the data before starting the newer container. +Set this option only if you have very good back\-ups at any moment and you are fine to fail\-over +from the back\-up. +.IP \(bu 2 +\fB\fCupgrade\-force\fR \-\- \fB\fCmysql\_upgrade \-\-force\fR is run at the beginning of the container start, when the local +daemon is running, no matter what version of the daemon the data come from. +This is also the way to create the missing version file \fB\fCmysql\_upgrade\_info\fR if not present +in the root of the data directory; this file holds information about the version of the data. + +.RE + +.PP +There are also some other actions that you may want to run at the beginning of the container start, +when the local daemon is running, no matter what version of the data is detected: + +.RS +.IP \(bu 2 +\fB\fCoptimize\fR \-\- runs \fB\fCmysqlcheck \-\-optimize\fR\&. It optimizes all the tables. +.IP \(bu 2 +\fB\fCanalyze\fR \-\- runs \fB\fCmysqlcheck \-\-analyze\fR\&. It analyzes all the tables. +.IP \(bu 2 +\fB\fCdisable\fR \-\- nothing is done regarding data directory version. + +.RE + +.PP +Multiple values are separated by comma and run in\-order, e.g. \fB\fCMYSQL\_DATADIR\_ACTION="optimize,analyze"\fR\&. + +.SH Changing the replication binlog\_format +.PP +Some applications may wish to use \fB\fCrow\fR binlog\_formats (for example, those built + with change\-data\-capture in mind). The default replication/binlog format is + \fB\fCstatement\fR but to change it you can set the \fB\fCMYSQL\_BINLOG\_FORMAT\fR environment + variable. For example \fB\fCMYSQL\_BINLOG\_FORMAT=row\fR\&. Now when you run the database + with \fB\fCmaster\fR replication turned on (ie, set the Docker/container \fB\fCcmd\fR to be +\fB\fCrun\-mysqld\-master\fR) the binlog will emit the actual data for the rows that change +as opposed to the statements (ie, DML like insert...) that caused the change. + +.SH Troubleshooting +.PP +The mysqld deamon in the container logs to the standard output, so the log is available in the container log. The log can be examined by running: + +.PP +.RS + +.nf +podman logs + +.fi +.RE + +.SH See also +.PP +Dockerfile and other sources for this container image are available on + +\[la]https://github.com/sclorg/mariadb-container\[ra]\&. +In that repository, the Dockerfile for CentOS is called Dockerfile, the Dockerfile +for RHEL7 is called Dockerfile.rhel7, the Dockerfile for RHEL8 is called Dockerfile.rhel8, +and the Dockerfile for Fedora is called Dockerfile.fedora. diff --git a/root/usr/bin/cgroup-limits b/root/usr/bin/cgroup-limits deleted file mode 100755 index b9d4edc..0000000 --- a/root/usr/bin/cgroup-limits +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/python - -""" -Script for parsing cgroup information - -This script will read some limits from the cgroup system and parse -them, printing out "VARIABLE=VALUE" on each line for every limit that is -successfully read. Output of this script can be directly fed into -bash's export command. Recommended usage from a bash script: - - set -o errexit - export_vars=$(cgroup-limits) ; export $export_vars - -Variables currently supported: - MAX_MEMORY_LIMIT_IN_BYTES - Maximum possible limit MEMORY_LIMIT_IN_BYTES can have. This is - currently constant value of 9223372036854775807. - MEMORY_LIMIT_IN_BYTES - Maximum amount of user memory in bytes. If this value is set - to the same value as MAX_MEMORY_LIMIT_IN_BYTES, it means that - there is no limit set. The value is taken from - /sys/fs/cgroup/memory/memory.limit_in_bytes - NUMBER_OF_CORES - Number of detected CPU cores that can be used. This value is - calculated from /sys/fs/cgroup/cpuset/cpuset.cpus - NO_MEMORY_LIMIT - Set to "true" if MEMORY_LIMIT_IN_BYTES is so high that the caller - can act as if no memory limit was set. Undefined otherwise. -""" - -from __future__ import print_function -import sys - - -def _read_file(path): - try: - with open(path, 'r') as f: - return f.read().strip() - except IOError: - return None - - -def get_memory_limit(): - """ - Read memory limit, in bytes. - """ - - limit = _read_file('/sys/fs/cgroup/memory/memory.limit_in_bytes') - if limit is None or not limit.isdigit(): - print("Warning: Can't detect memory limit from cgroups", - file=sys.stderr) - return None - return int(limit) - - -def get_number_of_cores(): - """ - Read number of CPU cores. - """ - - core_count = 0 - - line = _read_file('/sys/fs/cgroup/cpuset/cpuset.cpus') - if line is None: - print("Warning: Can't detect number of CPU cores from cgroups", - file=sys.stderr) - return None - - for group in line.split(','): - core_ids = list(map(int, group.split('-'))) - if len(core_ids) == 2: - core_count += core_ids[1] - core_ids[0] + 1 - else: - core_count += 1 - - return core_count - - -if __name__ == "__main__": - env_vars = { - "MAX_MEMORY_LIMIT_IN_BYTES": 9223372036854775807, - "MEMORY_LIMIT_IN_BYTES": get_memory_limit(), - "NUMBER_OF_CORES": get_number_of_cores() - } - - env_vars = {k: v for k, v in env_vars.items() if v is not None} - - if env_vars.get("MEMORY_LIMIT_IN_BYTES", 0) >= 92233720368547: - env_vars["NO_MEMORY_LIMIT"] = "true" - - for key, value in env_vars.items(): - print("{0}={1}".format(key, value)) diff --git a/root/usr/bin/run-mysqld b/root/usr/bin/run-mysqld deleted file mode 100755 index cd899a7..0000000 --- a/root/usr/bin/run-mysqld +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -export_vars=$(cgroup-limits); export $export_vars -source ${CONTAINER_SCRIPTS_PATH}/common.sh -set -eu - -[ -f ${CONTAINER_SCRIPTS_PATH}/validate-variables.sh ] && source ${CONTAINER_SCRIPTS_PATH}/validate-variables.sh - -# Process the MySQL configuration files -log_info 'Processing MySQL configuration files ...' -envsubst < ${CONTAINER_SCRIPTS_PATH}/my-base.cnf.template > /etc/my.cnf.d/base.cnf -envsubst < ${CONTAINER_SCRIPTS_PATH}/my-paas.cnf.template > /etc/my.cnf.d/paas.cnf -envsubst < ${CONTAINER_SCRIPTS_PATH}/my-tuning.cnf.template > /etc/my.cnf.d/tuning.cnf - -if [ ! -d "$MYSQL_DATADIR/mysql" ]; then - initialize_database "$@" -else - start_local_mysql "$@" -fi - -if [ -f ${CONTAINER_SCRIPTS_PATH}/passwd-change.sh ]; then - log_info 'Setting passwords ...' - source ${CONTAINER_SCRIPTS_PATH}/passwd-change.sh -fi -if [ -f ${CONTAINER_SCRIPTS_PATH}/post-init.sh ]; then - log_info 'Sourcing post-init.sh ...' - source ${CONTAINER_SCRIPTS_PATH}/post-init.sh -fi - -# Restart the MySQL server with public IP bindings -shutdown_local_mysql -unset_env_vars -log_volume_info $MYSQL_DATADIR -log_info 'Running final exec -- Only MySQL server logs after this point' -exec ${MYSQL_PREFIX}/libexec/mysqld --defaults-file=$MYSQL_DEFAULTS_FILE "$@" 2>&1 diff --git a/root/usr/bin/run-mysqld-slave b/root/usr/bin/run-mysqld-slave deleted file mode 100755 index 51acce5..0000000 --- a/root/usr/bin/run-mysqld-slave +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash -# -# This is an entrypoint that runs the MySQL server in the 'slave' mode. -# -export_vars=$(cgroup-limits); export $export_vars -source ${CONTAINER_SCRIPTS_PATH}/common.sh -set -eu - -# Just run normal server if the data directory is already initialized -if [ -d "${MYSQL_DATADIR}/mysql" ]; then - exec /usr/bin/run-mysqld "$@" -fi - -export MYSQL_RUNNING_AS_SLAVE=1 - -[ -f ${CONTAINER_SCRIPTS_PATH}/validate_replication_variables.sh ] && source ${CONTAINER_SCRIPTS_PATH}/validate_replication_variables.sh - -# Generate the unique 'server-id' for this master -export MYSQL_SERVER_ID=$(server_id) -log_info "The 'slave' server-id is ${MYSQL_SERVER_ID}" - -# Process the MySQL configuration files -envsubst < ${CONTAINER_SCRIPTS_PATH}/my-base.cnf.template > /etc/my.cnf.d/base.cnf -envsubst < ${CONTAINER_SCRIPTS_PATH}/my-paas.cnf.template > /etc/my.cnf.d/paas.cnf -envsubst < ${CONTAINER_SCRIPTS_PATH}/my-slave.cnf.template > /etc/my.cnf.d/slave.cnf -envsubst < ${CONTAINER_SCRIPTS_PATH}/my-repl-gtid.cnf.template > /etc/my.cnf.d/repl-gtid.cnf -envsubst < ${CONTAINER_SCRIPTS_PATH}/my-tuning.cnf.template > /etc/my.cnf.d/tuning.cnf - -# Initialize MySQL database and wait for the MySQL master to accept -# connections. -initialize_database "$@" -wait_for_mysql_master - -# Get binlog file and position from master -STATUS_INFO=$(mysql --host "$MYSQL_MASTER_SERVICE_NAME" "-u${MYSQL_MASTER_USER}" "-p${MYSQL_MASTER_PASSWORD}" replication -e 'SELECT gtid from replication limit 1\G') -GTID_VALUE=$(echo "$STATUS_INFO" | grep 'gtid:' | head -n 1 | sed -e 's/^\s*gtid: //') - -# checking STATUS_INFO here because empty GTID_VALUE is valid value -if [ -z "${STATUS_INFO}" ] ; then - echo "Could not read GTID value from master" - exit 1 -fi - -mysql $mysql_flags <&1 diff --git a/root/usr/share/container-scripts/mysql/README.md b/root/usr/share/container-scripts/mysql/README.md index 656dbd9..7e35c93 100644 --- a/root/usr/share/container-scripts/mysql/README.md +++ b/root/usr/share/container-scripts/mysql/README.md @@ -1,64 +1,35 @@ -MariaDB Docker image -==================== +MariaDB 10.3 SQL Database Server Docker image +============================================= -This container image includes MariaDB server 10.1 for OpenShift and general usage. -Users can choose between RHEL and CentOS based images. +This container image includes MariaDB 10.3 SQL database server for OpenShift and general usage. +Users can choose between RHEL, CentOS and Fedora based images. +The RHEL images are available in the [Red Hat Container Catalog](https://access.redhat.com/containers/), +the CentOS images are available on [Docker Hub](https://hub.docker.com/r/centos/), +and the Fedora images are available in [Fedora Registry](https://registry.fedoraproject.org/). +The resulting image can be run using [podman](https://github.com/containers/libpod). -Dockerfile for CentOS is called Dockerfile, Dockerfile for RHEL is called -Dockerfile.rhel7. +Note: while the examples in this README are calling `podman`, you can replace any such calls by `docker` with the same arguments -Environment variables and volumes ----------------------------------- +Description +----------- -The image recognizes the following environment variables that you can set during -initialization by passing `-e VAR=VALUE` to the Docker run command. +This container image provides a containerized packaging of the MariaDB mysqld daemon +and client application. The mysqld server daemon accepts connections from clients +and provides access to content from MySQL databases on behalf of the clients. +You can find more information on the MariaDB project from the project Web site +(https://mariadb.org/). -| Variable name | Description | -| :--------------------- | ----------------------------------------- | -| `MYSQL_USER` | User name for MySQL account to be created | -| `MYSQL_PASSWORD` | Password for the user account | -| `MYSQL_DATABASE` | Database name | -| `MYSQL_ROOT_PASSWORD` | Password for the root user (optional) | - -The following environment variables influence the MySQL configuration file. They are all optional. - -| Variable name | Description | Default -| :------------------------------ | ----------------------------------------------------------------- | ------------------------------- -| `MYSQL_LOWER_CASE_TABLE_NAMES` | Sets how the table names are stored and compared | 0 -| `MYSQL_MAX_CONNECTIONS` | The maximum permitted number of simultaneous client connections | 151 -| `MYSQL_MAX_ALLOWED_PACKET` | The maximum size of one packet or any generated/intermediate string | 200M -| `MYSQL_FT_MIN_WORD_LEN` | The minimum length of the word to be included in a FULLTEXT index | 4 -| `MYSQL_FT_MAX_WORD_LEN` | The maximum length of the word to be included in a FULLTEXT index | 20 -| `MYSQL_AIO` | Controls the `innodb_use_native_aio` setting value in case the native AIO is broken. See http://help.directadmin.com/item.php?id=529 | 1 -| `MYSQL_TABLE_OPEN_CACHE` | The number of open tables for all threads | 400 -| `MYSQL_KEY_BUFFER_SIZE` | The size of the buffer used for index blocks | 32M (or 10% of available memory) -| `MYSQL_SORT_BUFFER_SIZE` | The size of the buffer used for sorting | 256K -| `MYSQL_READ_BUFFER_SIZE` | The size of the buffer used for a sequential scan | 8M (or 5% of available memory) -| `MYSQL_INNODB_BUFFER_POOL_SIZE`| The size of the buffer pool where InnoDB caches table and index data | 32M (or 50% of available memory) -| `MYSQL_INNODB_LOG_FILE_SIZE` | The size of each log file in a log group | 8M (or 15% of available available) -| `MYSQL_INNODB_LOG_BUFFER_SIZE` | The size of the buffer that InnoDB uses to write to the log files on disk | 8M (or 15% of available memory) -| `MYSQL_DEFAULTS_FILE` | Point to an alternative configuration file | /etc/my.cnf -| `MYSQL_BINLOG_FORMAT` | Set sets the binlog format, supported values are `row` and `statement` | statement - -You can also set the following mount points by passing the `-v /host:/container` flag to Docker. - -| Volume mount point | Description | -| :----------------------- | -------------------- | -| `/var/lib/mysql/data` | MySQL data directory | - -**Notice: When mouting a directory from the host into the container, ensure that the mounted -directory has the appropriate permissions and that the owner and group of the directory -matches the user UID or name which is running inside the container.** Usage ---------------------------------- +----- -For this, we will assume that you are using the `rhscl/mariadb-100-rhel7` image. +For this, we will assume that you are using the MariaDB 10.3 container image from the +Red Hat Container Catalog called `rhel8/mariadb-103`. If you want to set only the mandatory environment variables and not store the database in a host directory, execute the following command: ``` -$ docker run -d --name mariadb_database -e MYSQL_USER=user -e MYSQL_PASSWORD=pass -e MYSQL_DATABASE=db -p 3306:3306 rhscl/mariadb-100-rhel7 +$ podman run -d --name mariadb_database -e MYSQL_USER=user -e MYSQL_PASSWORD=pass -e MYSQL_DATABASE=db -p 3306:3306 rhel8/mariadb-103 ``` This will create a container named `mariadb_database` running MySQL with database @@ -71,7 +42,88 @@ If the database directory is not initialized, the entrypoint script will first run [`mysql_install_db`](https://dev.mysql.com/doc/refman/5.6/en/mysql-install-db.html) and setup necessary database users and passwords. After the database is initialized, or if it was already present, `mysqld` is executed and will run as PID 1. You can - stop the detached container by running `docker stop mariadb_database`. + stop the detached container by running `podman stop mariadb_database`. + + +Environment variables and volumes +--------------------------------- + +The image recognizes the following environment variables that you can set during +initialization by passing `-e VAR=VALUE` to the Docker run command. + +**`MYSQL_USER`** + User name for MySQL account to be created + +**`MYSQL_PASSWORD`** + Password for the user account + +**`MYSQL_DATABASE`** + Database name + +**`MYSQL_ROOT_PASSWORD`** + Password for the root user (optional) + + +The following environment variables influence the MySQL configuration file. They are all optional. + +**`MYSQL_LOWER_CASE_TABLE_NAMES (default: 0)`** + Sets how the table names are stored and compared + +**`MYSQL_MAX_CONNECTIONS (default: 151)`** + The maximum permitted number of simultaneous client connections + +**`MYSQL_MAX_ALLOWED_PACKET (default: 200M)`** + The maximum size of one packet or any generated/intermediate string + +**`MYSQL_FT_MIN_WORD_LEN (default: 4)`** + The minimum length of the word to be included in a FULLTEXT index + +**`MYSQL_FT_MAX_WORD_LEN (default: 20)`** + The maximum length of the word to be included in a FULLTEXT index + +**`MYSQL_AIO (default: 1)`** + Controls the `innodb_use_native_aio` setting value in case the native AIO is broken. See http://help.directadmin.com/item.php?id=529 + +**`MYSQL_TABLE_OPEN_CACHE (default: 400)`** + The number of open tables for all threads + +**`MYSQL_KEY_BUFFER_SIZE (default: 32M or 10% of available memory)`** + The size of the buffer used for index blocks + +**`MYSQL_SORT_BUFFER_SIZE (default: 256K)`** + The size of the buffer used for sorting + +**`MYSQL_READ_BUFFER_SIZE (default: 8M or 5% of available memory)`** + The size of the buffer used for a sequential scan + +**`MYSQL_INNODB_BUFFER_POOL_SIZE (default: 32M or 50% of available memory)`** + The size of the buffer pool where InnoDB caches table and index data + +**`MYSQL_INNODB_LOG_FILE_SIZE (default: 8M or 15% of available memory)`** + The size of each log file in a log group + +**`MYSQL_INNODB_LOG_BUFFER_SIZE (default: 8M or 15% of available memory)`** + The size of the buffer that InnoDB uses to write to the log files on disk + +**`MYSQL_DEFAULTS_FILE (default: /etc/my.cnf)`** + Point to an alternative configuration file + +**`MYSQL_BINLOG_FORMAT (default: statement)`** + Set sets the binlog format, supported values are `row` and `statement` + +**`MYSQL_LOG_QUERIES_ENABLED (default: 0)`** + To enable query logging set this to `1` + + +You can also set the following mount points by passing the `-v /host:/container` flag to Docker. + +**`/var/lib/mysql/data`** + MySQL data directory + + +**Notice: When mouting a directory from the host into the container, ensure that the mounted +directory has the appropriate permissions and that the owner and group of the directory +matches the user UID or name which is running inside the container.** MariaDB auto-tuning @@ -81,13 +133,21 @@ When the MySQL image is run with the `--memory` parameter set and you didn't specify value for some parameters, their values will be automatically calculated based on the available memory. -| Variable name | Configuration parameter | Relative value -| :-------------------------------| ------------------------- | -------------- -| `MYSQL_KEY_BUFFER_SIZE` | `key_buffer_size` | 10% -| `MYSQL_READ_BUFFER_SIZE` | `read_buffer_size` | 5% -| `MYSQL_INNODB_BUFFER_POOL_SIZE` | `innodb_buffer_pool_size` | 50% -| `MYSQL_INNODB_LOG_FILE_SIZE` | `innodb_log_file_size` | 15% -| `MYSQL_INNODB_LOG_BUFFER_SIZE` | `innodb_log_buffer_size` | 15% +**`MYSQL_KEY_BUFFER_SIZE (default: 10%)`** + `key_buffer_size` + +**`MYSQL_READ_BUFFER_SIZE (default: 5%)`** + `read_buffer_size` + +**`MYSQL_INNODB_BUFFER_POOL_SIZE (default: 50%)`** + `innodb_buffer_pool_size` + +**`MYSQL_INNODB_LOG_FILE_SIZE (default: 15%)`** + `innodb_log_file_size` + +**`MYSQL_INNODB_LOG_BUFFER_SIZE (default: 15%)`** + `innodb_log_buffer_size` + MySQL root user @@ -115,6 +175,7 @@ values stored in the variables and the actual passwords. Whenever a database container starts it will reset the passwords to the values stored in the environment variables. + Default my.cnf file ------------------- With environment variables we are able to customize a lot of different parameters @@ -124,6 +185,153 @@ variable with the full path of the file you wish to use. For example, the defaul location is `/etc/my.cnf` but you can change it to `/etc/mysql/my.cnf` by setting `MYSQL_DEFAULTS_FILE=/etc/mysql/my.cnf` + +Extending image +--------------- +This image can be extended in Openshift using the `Source` build strategy or via the standalone +[source-to-image](https://github.com/openshift/source-to-image) application (where available). +For this, we will assume that you are using the `rhscl/mariadb-103-rhel7` image, +available via `mariadb:10.3` imagestream tag in Openshift. + + +For example, to build a customized MariaDB database image `my-mariadb-rhel7` +with a configuration from `https://github.com/sclorg/mariadb-container/tree/master/examples/extend-image` run: + +``` +$ oc new-app mariadb:10.3~https://github.com/sclorg/mariadb-container.git \ + --name my-mariadb-rhel7 \ + --context-dir=examples/extend-image \ + --env MYSQL_OPERATIONS_USER=opuser \ + --env MYSQL_OPERATIONS_PASSWORD=oppass \ + --env MYSQL_DATABASE=opdb \ + --env MYSQL_USER=user \ + --env MYSQL_PASSWORD=pass +``` + +or via s2i: + +``` +$ s2i build --context-dir=examples/extend-image https://github.com/sclorg/mariadb-container.git rhscl/mariadb-103-rhel7 my-mariadb-rhel7 +``` + +The directory passed to Openshift can contain these directories: + +`mysql-cfg/` + When starting the container, files from this directory will be used as + a configuration for the `mysqld` daemon. + `envsubst` command is run on this file to still allow customization of + the image using environmental variables + +`mysql-pre-init/` + Shell scripts (`*.sh`) available in this directory are sourced before + `mysqld` daemon is started. + +`mysql-init/` + Shell scripts (`*.sh`) available in this directory are sourced when + `mysqld` daemon is started locally. In this phase, use `${mysql_flags}` + to connect to the locally running daemon, for example `mysql $mysql_flags < dump.sql` + +Variables that can be used in the scripts provided to s2i: + +`$mysql_flags` + arguments for the `mysql` tool that will connect to the locally running `mysqld` during initialization + +`$MYSQL_RUNNING_AS_MASTER` + variable defined when the container is run with `run-mysqld-master` command + +`$MYSQL_RUNNING_AS_SLAVE` + variable defined when the container is run with `run-mysqld-slave` command + +`$MYSQL_DATADIR_FIRST_INIT` + variable defined when the container was initialized from the empty data dir + +During the s2i build all provided files are copied into `/opt/app-root/src` +directory into the resulting image. If some configuration files are present +in the destination directory, files with the same name are overwritten. +Also only one file with the same name can be used for customization and user +provided files are preferred over default files in +`/usr/share/container-scripts/mysql/`- so it is possible to overwrite them. + +Same configuration directory structure can be used to customize the image +every time the image is started using `podman run`. The directory has to be +mounted into `/opt/app-root/src/` in the image +(`-v ./image-configuration/:/opt/app-root/src/`). +This overwrites customization built into the image. + + +Securing the connection with SSL +-------------------------------- +In order to secure the connection with SSL, use the extending feature described +above. In particular, put the SSL certificates into a separate directory: + + sslapp/mysql-certs/server-cert-selfsigned.pem + sslapp/mysql-certs/server-key.pem + +And then put a separate configuration file into mysql-cfg: + + $> cat sslapp/mysql-cfg/ssl.cnf + [mysqld] + ssl-key=${APP_DATA}/mysql-certs/server-key.pem + ssl-cert=${APP_DATA}/mysql-certs/server-cert-selfsigned.pem + +Such a directory `sslapp` can then be mounted into the container with -v, +or a new container image can be built using s2i. + + +Upgrading and data directory version checking +--------------------------------------------- + +MySQL and MariaDB use versions that consist of three numbers X.Y.Z (e.g. 5.6.23). +For version changes in Z part, the server's binary data format stays compatible and thus no +special upgrade procedure is needed. For upgrades from X.Y to X.Y+1, consider doing manual +steps as described at +https://mariadb.com/kb/en/library/upgrading-from-mariadb-102-to-mariadb-103/ + +Skipping versions like from X.Y to X.Y+2 or downgrading to lower version is not supported; +the only exception is ugrading from MariaDB 5.5 to MariaDB 10.0. + +**Important**: Upgrading to a new version is always risky and users are expected to make a full +back-up of all data before. + +A safer solution to upgrade is to dump all data using `mysqldump` or `mysqldbexport` and then +load the data using `mysql` or `mysqldbimport` into an empty (freshly initialized) database. + +Another way of proceeding with the upgrade is starting the new version of the `mysqld` daemon +and run `mysql_upgrade` right after the start. This so called in-place upgrade is generally +faster for large data directory, but only possible if upgrading from the very previous version, +so skipping versions is not supported. + +This container detects whether the data needs to be upgraded using `mysql_upgrade` and +we can control it by setting `MYSQL_DATADIR_ACTION` variable, which can have one or more of the following values: + + * `upgrade-warn` -- If the data version can be determined and the data come from a different version + of the daemon, a warning is printed but the container starts. This is the default value. + Since historically the version file `mysql_upgrade_info` was not created, when using this option, + the version file is created if not exist, but no `mysql_upgrade` will be called. + However, this automatic creation will be removed after few months, since the version should be + created on most deployments at that point. + * `upgrade-auto` -- `mysql_upgrade` is run at the beginning of the container start, when the local + daemon is running, but only if the data version can be determined and the data come + with the very previous version. A warning is printed if the data come from even older + or newer version. This value effectively enables automatic upgrades, + but it is always risky and users should still back-up all the data before starting the newer container. + Set this option only if you have very good back-ups at any moment and you are fine to fail-over + from the back-up. + * `upgrade-force` -- `mysql_upgrade --force` is run at the beginning of the container start, when the local + daemon is running, no matter what version of the daemon the data come from. + This is also the way to create the missing version file `mysql_upgrade_info` if not present + in the root of the data directory; this file holds information about the version of the data. + +There are also some other actions that you may want to run at the beginning of the container start, +when the local daemon is running, no matter what version of the data is detected: + + * `optimize` -- runs `mysqlcheck --optimize`. It optimizes all the tables. + * `analyze` -- runs `mysqlcheck --analyze`. It analyzes all the tables. + * `disable` -- nothing is done regarding data directory version. + +Multiple values are separated by comma and run in-order, e.g. `MYSQL_DATADIR_ACTION="optimize,analyze"`. + + Changing the replication binlog_format -------------------------------------- Some applications may wish to use `row` binlog_formats (for example, those built @@ -133,3 +341,19 @@ Some applications may wish to use `row` binlog_formats (for example, those built with `master` replication turned on (ie, set the Docker/container `cmd` to be `run-mysqld-master`) the binlog will emit the actual data for the rows that change as opposed to the statements (ie, DML like insert...) that caused the change. + + +Troubleshooting +--------------- +The mysqld deamon in the container logs to the standard output, so the log is available in the container log. The log can be examined by running: + + podman logs + + +See also +-------- +Dockerfile and other sources for this container image are available on +https://github.com/sclorg/mariadb-container. +In that repository, the Dockerfile for CentOS is called Dockerfile, the Dockerfile +for RHEL7 is called Dockerfile.rhel7, the Dockerfile for RHEL8 is called Dockerfile.rhel8, +and the Dockerfile for Fedora is called Dockerfile.fedora. diff --git a/root/usr/share/container-scripts/mysql/common.sh b/root/usr/share/container-scripts/mysql/common.sh deleted file mode 100644 index e63a750..0000000 --- a/root/usr/share/container-scripts/mysql/common.sh +++ /dev/null @@ -1,164 +0,0 @@ -#!/bin/bash - -source ${CONTAINER_SCRIPTS_PATH}/helpers.sh - -# Data directory where MySQL database files live. The data subdirectory is here -# because .bashrc and my.cnf both live in /var/lib/mysql/ and we don't want a -# volume to override it. -export MYSQL_DATADIR=/var/lib/mysql/data - -# Configuration settings. -export MYSQL_DEFAULTS_FILE=${MYSQL_DEFAULTS_FILE:-/etc/my.cnf} -export MYSQL_BINLOG_FORMAT=${MYSQL_BINLOG_FORMAT:-STATEMENT} -export MYSQL_LOWER_CASE_TABLE_NAMES=${MYSQL_LOWER_CASE_TABLE_NAMES:-0} -export MYSQL_MAX_CONNECTIONS=${MYSQL_MAX_CONNECTIONS:-151} -export MYSQL_FT_MIN_WORD_LEN=${MYSQL_FT_MIN_WORD_LEN:-4} -export MYSQL_FT_MAX_WORD_LEN=${MYSQL_FT_MAX_WORD_LEN:-20} -export MYSQL_AIO=${MYSQL_AIO:-1} -export MYSQL_MAX_ALLOWED_PACKET=${MYSQL_MAX_ALLOWED_PACKET:-200M} -export MYSQL_TABLE_OPEN_CACHE=${MYSQL_TABLE_OPEN_CACHE:-400} -export MYSQL_SORT_BUFFER_SIZE=${MYSQL_SORT_BUFFER_SIZE:-256K} - -if [ -n "${NO_MEMORY_LIMIT:-}" -o -z "${MEMORY_LIMIT_IN_BYTES:-}" ]; then - key_buffer_size='32M' - read_buffer_size='8M' - innodb_buffer_pool_size='32M' - innodb_log_file_size='8M' - innodb_log_buffer_size='8M' -else - key_buffer_size="$(python -c "print(int((${MEMORY_LIMIT_IN_BYTES}/(1024*1024))*0.1))")M" - read_buffer_size="$(python -c "print(int((${MEMORY_LIMIT_IN_BYTES}/(1024*1024))*0.05))")M" - innodb_buffer_pool_size="$(python -c "print(int((${MEMORY_LIMIT_IN_BYTES}/(1024*1024))*0.5))")M" - innodb_log_file_size="$(python -c "print(int((${MEMORY_LIMIT_IN_BYTES}/(1024*1024))*0.15))")M" - innodb_log_buffer_size="$(python -c "print(int((${MEMORY_LIMIT_IN_BYTES}/(1024*1024))*0.15))")M" -fi -export MYSQL_KEY_BUFFER_SIZE=${MYSQL_KEY_BUFFER_SIZE:-$key_buffer_size} -export MYSQL_READ_BUFFER_SIZE=${MYSQL_READ_BUFFER_SIZE:-$read_buffer_size} -export MYSQL_INNODB_BUFFER_POOL_SIZE=${MYSQL_INNODB_BUFFER_POOL_SIZE:-$innodb_buffer_pool_size} -export MYSQL_INNODB_LOG_FILE_SIZE=${MYSQL_INNODB_LOG_FILE_SIZE:-$innodb_log_file_size} -export MYSQL_INNODB_LOG_BUFFER_SIZE=${MYSQL_INNODB_LOG_BUFFER_SIZE:-$innodb_log_buffer_size} - -# Be paranoid and stricter than we should be. -# https://dev.mysql.com/doc/refman/en/identifiers.html -mysql_identifier_regex='^[a-zA-Z0-9_]+$' -mysql_password_regex='^[a-zA-Z0-9_~!@#$%^&*()-=<>,.?;:|]+$' - -# Variables that are used to connect to local mysql during initialization -mysql_flags="-u root --socket=/tmp/mysql.sock" -admin_flags="--defaults-file=$MYSQL_DEFAULTS_FILE $mysql_flags" - -# Make sure env variables don't propagate to mysqld process. -function unset_env_vars() { - log_info 'Cleaning up environment variables MYSQL_USER, MYSQL_PASSWORD, MYSQL_DATABASE and MYSQL_ROOT_PASSWORD ...' - unset MYSQL_USER MYSQL_PASSWORD MYSQL_DATABASE MYSQL_ROOT_PASSWORD -} - -# Poll until MySQL responds to our ping. -function wait_for_mysql() { - pid=$1 ; shift - - while [ true ]; do - if [ -d "/proc/$pid" ]; then - mysqladmin --socket=/tmp/mysql.sock ping &>/dev/null && log_info "MySQL started successfully" && return 0 - else - return 1 - fi - log_info "Waiting for MySQL to start ..." - sleep 1 - done -} - -# Start local MySQL server with a defaults file -function start_local_mysql() { - log_info 'Starting MySQL server with disabled networking ...' - ${MYSQL_PREFIX}/libexec/mysqld \ - --defaults-file=$MYSQL_DEFAULTS_FILE \ - --skip-networking --socket=/tmp/mysql.sock "$@" & - mysql_pid=$! - wait_for_mysql $mysql_pid -} - -# Shutdown mysql flushing privileges -function shutdown_local_mysql() { - log_info 'Shutting down MySQL ...' - mysqladmin $admin_flags flush-privileges shutdown -} - -# Initialize the MySQL database (create user accounts and the initial database) -function initialize_database() { - log_info 'Initializing database ...' - log_info 'Running mysql_install_db ...' - # Using --rpm since we need mysql_install_db behaves as in RPM - # Using empty --basedir to work-around https://bugzilla.redhat.com/show_bug.cgi?id=1406391 - mysql_install_db --rpm --datadir=$MYSQL_DATADIR --basedir='' - start_local_mysql "$@" - - if [ -v MYSQL_RUNNING_AS_SLAVE ]; then - log_info 'Initialization finished' - return 0 - fi - - if [ -v MYSQL_RUNNING_AS_MASTER ]; then - # Save master status into a separate database. - STATUS_INFO=$(mysql $admin_flags -e 'SHOW MASTER STATUS\G') - BINLOG_POSITION=$(echo "$STATUS_INFO" | grep 'Position:' | head -n 1 | sed -e 's/^\s*Position: //') - BINLOG_FILE=$(echo "$STATUS_INFO" | grep 'File:' | head -n 1 | sed -e 's/^\s*File: //') - GTID_INFO=$(mysql $admin_flags -e "SELECT BINLOG_GTID_POS('$BINLOG_FILE', '$BINLOG_POSITION') AS gtid_value \G") - GTID_VALUE=$(echo "$GTID_INFO" | grep 'gtid_value:' | head -n 1 | sed -e 's/^\s*gtid_value: //') - - mysqladmin $admin_flags create replication - mysql $admin_flags </dev/null && log_info "MySQL master is ready" && return 0 - sleep 1 - done -} diff --git a/root/usr/share/container-scripts/mysql/passwd-change.sh b/root/usr/share/container-scripts/mysql/passwd-change.sh deleted file mode 100644 index ce06f6a..0000000 --- a/root/usr/share/container-scripts/mysql/passwd-change.sh +++ /dev/null @@ -1,23 +0,0 @@ -# Set the password for MySQL user and root everytime this container is started. -# This allows to change the password by editing the deployment configuration. -if [[ -v MYSQL_USER && -v MYSQL_PASSWORD ]]; then - mysql $mysql_flags < Installing application source ..." +mv /tmp/src/* ./ 2>/dev/null || true + +# Fix source directory permissions +/usr/libexec/fix-permissions ./ + diff --git a/s2i-common/bin/run b/s2i-common/bin/run new file mode 120000 index 0000000..4b21ab5 --- /dev/null +++ b/s2i-common/bin/run @@ -0,0 +1 @@ +/bin/run-mysqld \ No newline at end of file diff --git a/s2i-common/bin/usage b/s2i-common/bin/usage new file mode 100755 index 0000000..d6a3b9a --- /dev/null +++ b/s2i-common/bin/usage @@ -0,0 +1,8 @@ +#!/bin/sh + +set -o errexit +set -o nounset +set -o pipefail + +groff -t -man -ETascii /help.1 + diff --git a/sources b/sources deleted file mode 100644 index e69de29..0000000 diff --git a/test/mariadb-ephemeral-template.json b/test/mariadb-ephemeral-template.json new file mode 100644 index 0000000..27db3fd --- /dev/null +++ b/test/mariadb-ephemeral-template.json @@ -0,0 +1,254 @@ +{ + "kind": "Template", + "apiVersion": "v1", + "metadata": { + "name": "mariadb-ephemeral", + "annotations": { + "openshift.io/display-name": "MariaDB (Ephemeral)", + "description": "MariaDB database service, without persistent storage. For more information about using this template, including OpenShift considerations, see https://github.com/sclorg/mariadb-container/blob/master/10.2/root/usr/share/container-scripts/mysql/README.md.\n\nWARNING: Any data stored will be lost upon pod destruction. Only use this template for testing", + "iconClass": "icon-mariadb", + "tags": "database,mariadb", + "openshift.io/long-description": "This template provides a standalone MariaDB server with a database created. The database is not stored on persistent storage, so any restart of the service will result in all data being lost. The database name, username, and password are chosen via parameters when provisioning this service.", + "openshift.io/provider-display-name": "Red Hat, Inc.", + "openshift.io/documentation-url": "https://github.com/sclorg/mariadb-container/blob/master/10.2/root/usr/share/container-scripts/mysql/README.md", + "openshift.io/support-url": "https://access.redhat.com" + } + }, + "message": "The following service(s) have been created in your project: ${DATABASE_SERVICE_NAME}.\n\n Username: ${MYSQL_USER}\n Password: ${MYSQL_PASSWORD}\n Database Name: ${MYSQL_DATABASE}\n Connection URL: mysql://${DATABASE_SERVICE_NAME}:3306/\n\nFor more information about using this template, including OpenShift considerations, see https://github.com/sclorg/mariadb-container/blob/master/10.2/root/usr/share/container-scripts/mysql/README.md.", + "labels": { + "template": "mariadb-ephemeral-template" + }, + "objects": [ + { + "kind": "Secret", + "apiVersion": "v1", + "metadata": { + "name": "${DATABASE_SERVICE_NAME}", + "annotations": { + "template.openshift.io/expose-username": "{.data['database-user']}", + "template.openshift.io/expose-password": "{.data['database-password']}", + "template.openshift.io/expose-root_password": "{.data['database-root-password']}", + "template.openshift.io/expose-database_name": "{.data['database-name']}" + } + }, + "stringData" : { + "database-user" : "${MYSQL_USER}", + "database-password" : "${MYSQL_PASSWORD}", + "database-root-password" : "${MYSQL_ROOT_PASSWORD}", + "database-name" : "${MYSQL_DATABASE}" + } + }, + { + "kind": "Service", + "apiVersion": "v1", + "metadata": { + "name": "${DATABASE_SERVICE_NAME}", + "annotations": { + "template.openshift.io/expose-uri": "mysql://{.spec.clusterIP}:{.spec.ports[?(.name==\"mariadb\")].port}" + } + }, + "spec": { + "ports": [ + { + "name": "mariadb", + "port": 3306 + } + ], + "selector": { + "name": "${DATABASE_SERVICE_NAME}" + } + } + }, + { + "kind": "DeploymentConfig", + "apiVersion": "v1", + "metadata": { + "name": "${DATABASE_SERVICE_NAME}", + "annotations": { + "template.alpha.openshift.io/wait-for-ready": "true" + } + }, + "spec": { + "strategy": { + "type": "Recreate" + }, + "triggers": [ + { + "type": "ImageChange", + "imageChangeParams": { + "automatic": true, + "containerNames": [ + "mariadb" + ], + "from": { + "kind": "ImageStreamTag", + "name": "mariadb:${MARIADB_VERSION}", + "namespace": "${NAMESPACE}" + } + } + }, + { + "type": "ConfigChange" + } + ], + "replicas": 1, + "selector": { + "name": "${DATABASE_SERVICE_NAME}" + }, + "template": { + "metadata": { + "labels": { + "name": "${DATABASE_SERVICE_NAME}" + } + }, + "spec": { + "containers": [ + { + "name": "mariadb", + "image": " ", + "ports": [ + { + "containerPort": 3306 + } + ], + "readinessProbe": { + "timeoutSeconds": 1, + "initialDelaySeconds": 5, + "exec": { + "command": [ "/bin/sh", "-i", "-c", + "MYSQL_PWD=\"$MYSQL_PASSWORD\" mysql -h 127.0.0.1 -u $MYSQL_USER -D $MYSQL_DATABASE -e 'SELECT 1'"] + } + }, + "livenessProbe": { + "timeoutSeconds": 1, + "initialDelaySeconds": 30, + "tcpSocket": { + "port": 3306 + } + }, + "env": [ + { + "name": "MYSQL_USER", + "valueFrom": { + "secretKeyRef" : { + "name" : "${DATABASE_SERVICE_NAME}", + "key" : "database-user" + } + } + }, + { + "name": "MYSQL_PASSWORD", + "valueFrom": { + "secretKeyRef" : { + "name" : "${DATABASE_SERVICE_NAME}", + "key" : "database-password" + } + } + }, + { + "name": "MYSQL_ROOT_PASSWORD", + "valueFrom": { + "secretKeyRef" : { + "name" : "${DATABASE_SERVICE_NAME}", + "key" : "database-root-password" + } + } + }, + { + "name": "MYSQL_DATABASE", + "valueFrom": { + "secretKeyRef" : { + "name" : "${DATABASE_SERVICE_NAME}", + "key" : "database-name" + } + } + } + ], + "resources": { + "limits": { + "memory": "${MEMORY_LIMIT}" + } + }, + "volumeMounts": [ + { + "name": "${DATABASE_SERVICE_NAME}-data", + "mountPath": "/var/lib/mysql/data" + } + ], + "imagePullPolicy": "IfNotPresent" + } + ], + "volumes": [ + { + "name": "${DATABASE_SERVICE_NAME}-data", + "emptyDir": { + "medium": "" + } + } + ] + } + } + } + } + ], + "parameters": [ + { + "name": "MEMORY_LIMIT", + "displayName": "Memory Limit", + "description": "Maximum amount of memory the container can use.", + "value": "512Mi", + "required": true + }, + { + "name": "NAMESPACE", + "displayName": "Namespace", + "description": "The OpenShift Namespace where the ImageStream resides.", + "value": "openshift" + }, + { + "name": "DATABASE_SERVICE_NAME", + "displayName": "Database Service Name", + "description": "The name of the OpenShift Service exposed for the database.", + "value": "mariadb", + "required": true + }, + { + "name": "MYSQL_USER", + "displayName": "MariaDB Connection Username", + "description": "Username for MariaDB user that will be used for accessing the database.", + "generate": "expression", + "from": "user[A-Z0-9]{3}", + "required": true + }, + { + "name": "MYSQL_PASSWORD", + "displayName": "MariaDB Connection Password", + "description": "Password for the MariaDB connection user.", + "generate": "expression", + "from": "[a-zA-Z0-9]{16}", + "required": true + }, + { + "name": "MYSQL_ROOT_PASSWORD", + "displayName": "MariaDB root Password", + "description": "Password for the MariaDB root user.", + "generate": "expression", + "from": "[a-zA-Z0-9]{16}", + "required": true + }, + { + "name": "MYSQL_DATABASE", + "displayName": "MariaDB Database Name", + "description": "Name of the MariaDB database accessed.", + "value": "sampledb", + "required": true + }, + { + "name": "MARIADB_VERSION", + "displayName": "Version of MariaDB Image", + "description": "Version of MariaDB image to be used (10.2 or latest).", + "value": "10.2", + "required": true + } + ] +} diff --git a/test/run b/test/run index df15715..9d9bb0d 100755 --- a/test/run +++ b/test/run @@ -10,49 +10,48 @@ set -o errexit set -o nounset shopt -s nullglob -IMAGE_NAME=${IMAGE_NAME-centos/mariadb-101-centos7-candidate} +THISDIR=$(dirname ${BASH_SOURCE[0]}) +source ${THISDIR}/test-lib.sh -CIDFILE_DIR=$(mktemp --suffix=mysql_test_cidfiles -d) +TEST_LIST="\ +run_container_creation_tests +run_configuration_tests +run_general_tests +run_change_password_test +run_replication_test +run_doc_test +run_s2i_test +run_ssl_test +run_upgrade_test +" + +if [ -e "${IMAGE_NAME:-}" ] ; then + echo "Error: IMAGE_NAME must be specified" + exit 1 +fi + +CID_FILE_DIR=$(mktemp --suffix=mysql_test_cidfiles -d) +TESTSUITE_RESULT=1 +test_dir="$(readlink -f $(dirname "${BASH_SOURCE[0]}"))" + +s2i_args="--pull-policy=never " function cleanup() { - local cidfile - for cidfile in $CIDFILE_DIR/* ; do - local CONTAINER - CONTAINER=$(cat $cidfile) + ct_cleanup - echo "Stopping and removing container $CONTAINER..." - docker stop $CONTAINER >/dev/null - local exit_status - exit_status=$(docker inspect -f '{{.State.ExitCode}}' $CONTAINER) - if [ "$exit_status" != "0" ]; then - echo "Inspecting container $CONTAINER" - docker inspect $CONTAINER - echo "Dumping logs for $CONTAINER" - docker logs $CONTAINER - fi - docker rm -v $CONTAINER >/dev/null - rm $cidfile - echo "Done." - done - rmdir $CIDFILE_DIR + if [ $TESTSUITE_RESULT -eq 0 ] ; then + echo "Tests for ${IMAGE_NAME} succeeded." + else + echo "Tests for ${IMAGE_NAME} failed." + fi } trap cleanup EXIT SIGINT -function get_cid() { - local id="$1" ; shift || return 1 - echo $(cat "$CIDFILE_DIR/$id") -} - -function get_container_ip() { - local id="$1" ; shift - docker inspect --format='{{.NetworkSettings.IPAddress}}' $(get_cid "$id") -} - function mysql_cmd() { local container_ip="$1"; shift local login="$1"; shift local password="$1"; shift - docker run --rm "$IMAGE_NAME" mysql --host "$container_ip" -u"$login" -p"$password" "$@" db + docker run --rm ${CONTAINER_EXTRA_ARGS:-} "$IMAGE_NAME" mysql --host "$container_ip" -u"$login" -p"$password" "$@" db } function test_connection() { @@ -60,21 +59,36 @@ function test_connection() { local login=$1 ; shift local password=$1 ; shift local ip - ip=$(get_container_ip $name) + ip=$(ct_get_cip $name) echo " Testing MySQL connection to $ip..." local max_attempts=20 local sleep_time=2 local i + local status='' + echo -n " Trying to connect..." for i in $(seq $max_attempts); do - echo " Trying to connect..." - if mysql_cmd "$ip" "$login" "$password" <<< 'SELECT 1;'; then + local status=$(docker inspect -f '{{.State.Status}}' $(ct_get_cid "${name}")) + if [ "${status}" != 'running' ] ; then + break; + fi + echo -n "." + if mysql_cmd "$ip" "$login" "$password" &>/dev/null <<< 'SELECT 1;'; then + echo " OK" echo " Success!" return 0 fi sleep $sleep_time done - echo " Giving up: Failed to connect. Logs:" - docker logs $(get_cid $name) + echo " FAIL" + echo " Giving up: Failed to connect." + if [ "${status}" == 'running' ] ; then + echo " Container is still running." + else + local exit_status=$(docker inspect -f '{{.State.ExitCode}}' ${name}) + echo " Container finised with exit code ${exit_status}." + fi + echo "Logs:" + docker logs $(ct_get_cid $name) return 1 } @@ -95,22 +109,22 @@ function test_mysql() { function create_container() { local name=$1 ; shift - cidfile="$CIDFILE_DIR/$name" + cidfile="$CID_FILE_DIR/$name" # create container with a cidfile in a directory for cleanup local container_id container_id="$(docker run ${DOCKER_ARGS:-} --cidfile $cidfile -d "$@" $IMAGE_NAME ${CONTAINER_ARGS:-})" - echo "Created container $container_id" + echo " Created container $container_id" } function run_change_password_test() { local tmpdir=$(mktemp -d) - mkdir "${tmpdir}/data" && chmod -R a+rwx "${tmpdir}" + chmod -R a+rwx "${tmpdir}" # Create MySQL container with persistent volume and set the initial password create_container "testpass1" -e MYSQL_USER=user -e MYSQL_PASSWORD=foo \ -e MYSQL_DATABASE=db -v ${tmpdir}:/var/lib/mysql/data:Z test_connection testpass1 user foo - docker stop $(get_cid testpass1) >/dev/null + docker stop $(ct_get_cid testpass1) >/dev/null # Create second container with changed password create_container "testpass2" -e MYSQL_USER=user -e MYSQL_PASSWORD=bar \ @@ -118,7 +132,7 @@ function run_change_password_test() { test_connection testpass2 user bar # The old password should not work anymore - if mysql_cmd "$(get_container_ip testpass2)" user foo -e 'SELECT 1;'; then + if mysql_cmd "$(ct_get_cip testpass2)" user foo -e 'SELECT 1;'; then return 1 fi } @@ -131,16 +145,16 @@ function run_replication_test() { docker run $cluster_args -e MYSQL_USER=user -e MYSQL_PASSWORD=foo \ -e MYSQL_ROOT_PASSWORD=root \ -e MYSQL_INNODB_BUFFER_POOL_SIZE=5M \ - -d --cidfile ${CIDFILE_DIR}/master.cid $IMAGE_NAME mysqld-master >/dev/null + -d --cidfile ${CID_FILE_DIR}/master.cid $IMAGE_NAME mysqld-master >/dev/null local master_ip - master_ip=$(get_container_ip master.cid) + master_ip=$(ct_get_cip master.cid) # Run the MySQL slave docker run $cluster_args -e MYSQL_MASTER_SERVICE_NAME=${master_ip} \ -e MYSQL_INNODB_BUFFER_POOL_SIZE=5M \ - -d --cidfile ${CIDFILE_DIR}/slave.cid $IMAGE_NAME mysqld-slave >/dev/null + -d --cidfile ${CID_FILE_DIR}/slave.cid $IMAGE_NAME mysqld-slave >/dev/null local slave_ip - slave_ip=$(get_container_ip slave.cid) + slave_ip=$(ct_get_cip slave.cid) # Now wait till the MASTER will see the SLAVE local i @@ -152,8 +166,8 @@ function run_replication_test() { fi if [[ "${i}" == "${max_attempts}" ]]; then echo "The ${slave_ip} failed to register in MASTER" - echo "Dumping logs for $(get_cid slave.cid)" - docker logs $(get_cid slave.cid) + echo "Dumping logs for $(ct_get_cid slave.cid)" + docker logs $(ct_get_cid slave.cid) return 1 fi sleep 1 @@ -173,8 +187,8 @@ function run_replication_test() { fi if [[ "${i}" == "${max_attempts}" ]]; then echo "The ${slave_ip} failed to see value added on MASTER" - echo "Dumping logs for $(get_cid slave.cid)" - docker logs $(get_cid slave.cid) + echo "Dumping logs for $(ct_get_cid slave.cid)" + docker logs $(ct_get_cid slave.cid) return 1 fi sleep 1 @@ -204,7 +218,12 @@ function assert_login_access() { function assert_local_access() { local id="$1" ; shift - docker exec $(get_cid "$id") bash -c 'mysql <<< "SELECT 1;"' + if docker exec $(ct_get_cid "$id") bash -c 'mysql -uroot <<< "SELECT 1;"' ; then + echo " local access granted as expected" + return + fi + echo " local access assertion failed" + return 1 } # Make sure the invocation of docker run fails. @@ -220,6 +239,7 @@ function assert_container_creation_fails() { if [ $ret -gt 30 ]; then return 1 fi + echo " Success!" } function try_image_invalid_combinations() { @@ -234,9 +254,10 @@ function run_container_creation_tests() { try_image_invalid_combinations -e MYSQL_ROOT_PASSWORD=root_pass local VERY_LONG_DB_NAME="very_long_database_name_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" + local VERY_LONG_USER_NAME="very_long_user_name_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" assert_container_creation_fails -e MYSQL_USER=user -e MYSQL_PASSWORD=pass assert_container_creation_fails -e MYSQL_USER=\$invalid -e MYSQL_PASSWORD=pass -e MYSQL_DATABASE=db -e MYSQL_ROOT_PASSWORD=root_pass - assert_container_creation_fails -e MYSQL_USER=very_long_username -e MYSQL_PASSWORD=pass -e MYSQL_DATABASE=db -e MYSQL_ROOT_PASSWORD=root_pass + assert_container_creation_fails -e MYSQL_USER=$VERY_LONG_USER_NAME -e MYSQL_PASSWORD=pass -e MYSQL_DATABASE=db -e MYSQL_ROOT_PASSWORD=root_pass assert_container_creation_fails -e MYSQL_USER=user -e MYSQL_PASSWORD="\"" -e MYSQL_DATABASE=db -e MYSQL_ROOT_PASSWORD=root_pass assert_container_creation_fails -e MYSQL_USER=user -e MYSQL_PASSWORD=pass -e MYSQL_DATABASE=\$invalid -e MYSQL_ROOT_PASSWORD=root_pass assert_container_creation_fails -e MYSQL_USER=user -e MYSQL_PASSWORD=pass -e MYSQL_DATABASE=$VERY_LONG_DB_NAME -e MYSQL_ROOT_PASSWORD=root_pass @@ -252,7 +273,7 @@ function test_config_option() { local option_value="$4" if ! echo "$configuration" | grep -qx "$option_name[[:space:]]*=[[:space:]]*$option_value"; then - local configs="$(docker exec -t "$(get_cid $container_name)" bash -c 'set +f; shopt -s nullglob; echo /etc/my.cnf /etc/my.cnf.d/* /opt/rh/mysql*/root/etc/my.cnf /opt/rh/mysql*/root/etc/my.cnf.d/* | paste -s')" + local configs="$(docker exec "$(ct_get_cid $container_name)" bash -c 'set +f; shopt -s nullglob; echo /etc/my.cnf /etc/my.cnf.d/* /opt/rh/mysql*/root/etc/my.cnf /opt/rh/mysql*/root/etc/my.cnf.d/* | paste -s')" echo >&2 "FAIL: option '$option_name' should have value '$option_value', but it wasn't found in any of the configuration files ($configs):" echo >&2 echo >&2 "$configuration" @@ -274,6 +295,7 @@ function run_configuration_tests() { --env MYSQL_PASSWORD=config_test \ --env MYSQL_DATABASE=db \ --env MYSQL_LOWER_CASE_TABLE_NAMES=1 \ + --env MYSQL_LOG_QUERIES_ENABLED=1 \ --env MYSQL_MAX_CONNECTIONS=1337 \ --env MYSQL_FT_MIN_WORD_LEN=8 \ --env MYSQL_FT_MAX_WORD_LEN=15 \ @@ -294,9 +316,10 @@ function run_configuration_tests() { # - we should look for an option in the desired config, not in all of them # - we should respect section of the config (now we have duplicated options from a different sections) local configuration - configuration="$(docker exec -t "$(get_cid $container_name)" bash -c 'set +f; shopt -s nullglob; egrep -hv "^(#|\!|\[|$)" /etc/my.cnf /etc/my.cnf.d/* /opt/rh/mysql*/root/etc/my.cnf /opt/rh/mysql*/root/etc/my.cnf.d/*' | sed 's,\(^[[:space:]]\+\|[[:space:]]\+$\),,' | sort -u)" + configuration="$(docker exec "$(ct_get_cid $container_name)" bash -c 'set +f; shopt -s nullglob; egrep -hv "^(#|\!|\[|$)" /etc/my.cnf /etc/my.cnf.d/* /opt/rh/mysql*/root/etc/my.cnf /opt/rh/mysql*/root/etc/my.cnf.d/*' | sed 's,\(^[[:space:]]\+\|[[:space:]]\+$\),,' | sort -u)" test_config_option "$container_name" "$configuration" lower_case_table_names 1 + test_config_option "$container_name" "$configuration" general_log 1 test_config_option "$container_name" "$configuration" max_connections 1337 test_config_option "$container_name" "$configuration" ft_min_word_len 8 test_config_option "$container_name" "$configuration" ft_max_word_len 15 @@ -309,7 +332,7 @@ function run_configuration_tests() { test_config_option "$container_name" "$configuration" innodb_log_file_size 4M test_config_option "$container_name" "$configuration" innodb_log_buffer_size 4M - docker stop "$(get_cid $container_name)" >/dev/null + docker stop "$(ct_get_cid $container_name)" >/dev/null echo " Success!" echo " Testing image auto-calculated configuration settings" @@ -324,7 +347,7 @@ function run_configuration_tests() { test_connection "$container_name" config_test_user config_test - configuration="$(docker exec -t "$(get_cid $container_name)" bash -c 'set +f; shopt -s nullglob; egrep -hv "^(#|\!|\[|$)" /etc/my.cnf /etc/my.cnf.d/* /opt/rh/mysql*/root/etc/my.cnf /opt/rh/mysql*/root/etc/my.cnf.d/*' | sed 's,\(^[[:space:]]\+\|[[:space:]]\+$\),,' | sort -u)" + configuration="$(docker exec "$(ct_get_cid $container_name)" bash -c 'set +f; shopt -s nullglob; egrep -hv "^(#|\!|\[|$)" /etc/my.cnf /etc/my.cnf.d/* /opt/rh/mysql*/root/etc/my.cnf /opt/rh/mysql*/root/etc/my.cnf.d/*' | sed 's,\(^[[:space:]]\+\|[[:space:]]\+$\),,' | sort -u)" test_config_option "$container_name" "$configuration" key_buffer_size 25M test_config_option "$container_name" "$configuration" read_buffer_size 12M @@ -332,35 +355,11 @@ function run_configuration_tests() { test_config_option "$container_name" "$configuration" innodb_log_file_size 38M test_config_option "$container_name" "$configuration" innodb_log_buffer_size 38M - docker stop "$(get_cid $container_name)" >/dev/null + docker stop "$(ct_get_cid $container_name)" >/dev/null echo " Success!" } -test_scl_usage() { - local name="$1" - local run_cmd="$2" - local expected="$3" - - echo " Testing the image SCL enable" - local out - out=$(docker run --rm ${IMAGE_NAME} /bin/bash -c "${run_cmd}") - if ! echo "${out}" | grep -q "${expected}"; then - echo "ERROR[/bin/bash -c "${run_cmd}"] Expected '${expected}', got '${out}'" - return 1 - fi - out=$(docker exec $(get_cid $name) /bin/bash -c "${run_cmd}" 2>&1) - if ! echo "${out}" | grep -q "${expected}"; then - echo "ERROR[exec /bin/bash -c "${run_cmd}"] Expected '${expected}', got '${out}'" - return 1 - fi - out=$(docker exec $(get_cid $name) /bin/sh -ic "${run_cmd}" 2>&1) - if ! echo "${out}" | grep -q "${expected}"; then - echo "ERROR[exec /bin/sh -ic "${run_cmd}"] Expected '${expected}', got '${out}'" - return 1 - fi -} - function run_tests() { local name=$1 ; shift envs="-e MYSQL_USER=$USER -e MYSQL_PASSWORD=$PASS -e MYSQL_DATABASE=db" @@ -370,10 +369,10 @@ function run_tests() { create_container $name $envs test_connection "$name" "$USER" "$PASS" echo " Testing scl usage" - test_scl_usage $name 'mysql --version' '10.1' + ct_scl_usage_old $name 'mysql --version' "$VERSION" echo " Testing login accesses" local container_ip - container_ip=$(get_container_ip $name) + container_ip=$(ct_get_cip $name) assert_login_access "$container_ip" "$USER" "$PASS" true assert_login_access "$container_ip" "$USER" "${PASS}_foo" false if [ -v ROOT_PASS ]; then @@ -389,49 +388,244 @@ function run_tests() { } run_doc_test() { - local tmpdir=$(mktemp -d) - local f echo " Testing documentation in the container image" - # Extract the help files from the container - for f in /usr/share/container-scripts/mysql/README.md help.1 ; do - docker run --rm ${IMAGE_NAME} /bin/bash -c "cat /${f}" >${tmpdir}/$(basename ${f}) - # Check whether the files include some important information - for term in MYSQL_ROOT_PASSWORD volume 3306 ; do - if ! cat ${tmpdir}/$(basename ${f}) | grep -q -e "${term}" ; then - echo "ERROR: File /${f} does not include '${term}'." - return 1 - fi - done - done - # Check whether the files use the correct format - if ! file ${tmpdir}/help.1 | grep -q roff ; then - echo "ERROR: /help.1 is not in troff or groff format" - return 1 - fi + ct_doc_content_old "MYSQL\_ROOT\_PASSWORD" volume 3306 echo " Success!" echo } -# Tests. +_s2i_test_image() { + local container_name="$1" + local mount_opts="$2" + echo " Testing s2i app image with invalid configuration" + assert_container_creation_fails -e MYSQL_USER=root -e MYSQL_PASSWORD=pass -e MYSQL_DATABASE=db -e MYSQL_ROOT_PASSWORD=pass + echo " Testing s2i app image with correct configuration" + create_container \ + "$container_name" \ + --env MYSQL_USER=config_test_user \ + --env MYSQL_PASSWORD=config_test \ + --env MYSQL_DATABASE=db \ + --env MYSQL_OPERATIONS_USER=operations_user \ + --env MYSQL_OPERATIONS_PASSWORD=operations_pass \ + ${mount_opts} -run_container_creation_tests + test_connection "$container_name" operations_user operations_pass -run_configuration_tests + configuration="$(docker exec "$(ct_get_cid $container_name)" bash -c 'set +f; shopt -s nullglob; egrep -hv "^(#|\!|\[|$)" /etc/my.cnf /etc/my.cnf.d/* /opt/rh/mysql*/root/etc/my.cnf /opt/rh/mysql*/root/etc/my.cnf.d/*' | sed 's,\(^[[:space:]]\+\|[[:space:]]\+$\),,' | sort -u)" -# Set lower buffer pool size to avoid running out of memory. -export CONTAINER_ARGS="run-mysqld --innodb_buffer_pool_size=5242880" + docker stop "$(ct_get_cid $container_name)" >/dev/null +} -# Normal tests -USER=user PASS=pass run_tests no_root -USER=user1 PASS=pass1 ROOT_PASS=r00t run_tests root -# Test with arbitrary uid for the container -DOCKER_ARGS="-u 12345" USER=user PASS=pass run_tests no_root_altuid -DOCKER_ARGS="-u 12345" USER=user1 PASS=pass1 ROOT_PASS=r00t run_tests root_altuid +run_s2i_test() { + echo " Testing s2i usage" + ct_s2i_usage ${IMAGE_NAME} ${s2i_args} &>/dev/null -# Test the password change -run_change_password_test + echo " Testing s2i build" + ct_s2i_build_as_df file://${test_dir}/test-app ${IMAGE_NAME} ${IMAGE_NAME}-testapp + local image_name_backup=${IMAGE_NAME} + export IMAGE_NAME=${IMAGE_NAME}-testapp -# Replication tests -run_replication_test + local container_name=s2i_config_build + _s2i_test_image "s2i_config_build" "" + + # return back original value for IMAGE_NAME + export IMAGE_NAME=${image_name_backup} + + echo " Testing s2i mount" + test_app_dir=$(mktemp -d) + cp -Lr ${test_dir}/test-app ${test_app_dir}/ + chown -R 27:27 ${test_app_dir} + _s2i_test_image "_s2i_test_mount" "-v ${test_app_dir}/test-app:/opt/app-root/src/:z" + rm -rf ${test_app_dir} + echo " Success!" +} + +gen_self_signed_cert() { + local output_dir=$1 ; shift + local base_name=$1 ; shift + mkdir -p ${output_dir} + openssl req -newkey rsa:2048 -nodes -keyout ${output_dir}/${base_name}-key.pem -subj '/C=GB/ST=Berkshire/L=Newbury/O=My Server Company' > ${output_dir}/${base_name}-req.pem + openssl req -new -x509 -nodes -key ${output_dir}/${base_name}-key.pem -batch > ${output_dir}/${base_name}-cert-selfsigned.pem +} + +run_ssl_test() { + echo " Testing ssl usage" + test_app_dir=$(mktemp -d) + mkdir -p ${test_app_dir}/{mysql-certs,mysql-cfg} + gen_self_signed_cert ${test_app_dir}/mysql-certs server + echo "[mysqld] +ssl-key=\${APP_DATA}/mysql-certs/server-key.pem +ssl-cert=\${APP_DATA}/mysql-certs/server-cert-selfsigned.pem +" >${test_app_dir}/mysql-cfg/ssl.cnf + chown -R 27:27 ${test_app_dir} + local ca_cert_path="/opt/app-root/src/mysql-certs/server-cert-selfsigned.pem" + + create_container \ + "_s2i_test_ssl" \ + --env MYSQL_USER=ssl_test_user \ + --env MYSQL_PASSWORD=ssl_test \ + --env MYSQL_DATABASE=db \ + -v ${test_app_dir}:/opt/app-root/src/:z + + test_connection "_s2i_test_ssl" ssl_test_user ssl_test + ip=$(ct_get_cip _s2i_test_ssl) + + # At least MySQL 5.6 requires ssl-ca option on client side, otherwise the ssl is not used + CONTAINER_EXTRA_ARGS="-v ${test_app_dir}:/opt/app-root/src/:z" + + # MySQL requires --ssl-mode to be set in order to require SSL + case ${VERSION} in + 5*) ssl_mode_opt='--ssl-mode=REQUIRED' + esac + + if mysql_cmd "$ip" "ssl_test_user" "ssl_test" ${ssl_mode_opt:-} --ssl-ca=${ca_cert_path} -e 'show status like "Ssl_cipher" \G' | grep 'Value: [A-Z][A-Z0-9-]*' ; then + echo " Success!" + rm -rf ${test_app_dir} + else + echo " FAIL!" + mysql_cmd "$ip" "ssl_test_user" "ssl_test" --ssl-ca=${ca_cert_path} -e 'show status like "%ssl%" \G' + return 1 + fi + # Clear the global variable content after we are done using it + CONTAINER_EXTRA_ARGS="" +} + +function run_general_tests() { + # Set lower buffer pool size to avoid running out of memory. + export CONTAINER_ARGS="run-mysqld --innodb_buffer_pool_size=5242880" + + # Normal tests + USER=user PASS=pass run_tests no_root + USER=user1 PASS=pass1 ROOT_PASS=r00t run_tests root + # Test with arbitrary uid for the container + DOCKER_ARGS="--user 12345" USER=user PASS=pass run_tests no_root_altuid + DOCKER_ARGS="--user 12345" USER=user1 PASS=pass1 ROOT_PASS=r00t run_tests root_altuid +} + +function get_previous_major_version() { + case "${1}" in + 5.5) echo "5.1" ;; + 5.6) echo "5.5" ;; + 5.7) echo "5.6" ;; + 8.0) echo "5.7" ;; + 10.0) echo "5.5" ;; + 10.1) echo "10.0" ;; + 10.2) echo "10.1" ;; + 10.3) echo "10.2" ;; + *) echo "Non expected version '${1}'" ; return 1 ;; + esac +} + +function run_upgrade_test() { + local tmpdir=$(mktemp -d) + echo " Testing upgrade of the container image" + mkdir "${tmpdir}/data" && chmod -R a+rwx "${tmpdir}" + + # Create MySQL container with persistent volume and set the version from too old version + local datadir=${tmpdir}/data + create_container "testupg1" -e MYSQL_USER=user -e MYSQL_PASSWORD=foo \ + -e MYSQL_DATABASE=db -v ${datadir}:/var/lib/mysql/data:Z + test_connection testupg1 user foo + docker stop $(ct_get_cid testupg1) >/dev/null + + # Simulate datadir without version information + rm -f ${datadir}/mysql_upgrade_info + echo " Testing upgrade from data without version" + # This should work, but warning should be printed + create_container "testupg2" -e MYSQL_USER=user -e MYSQL_PASSWORD=foo \ + -e MYSQL_DATABASE=db -v ${datadir}:/var/lib/mysql/data:Z + test_connection testupg2 user foo + docker stop $(ct_get_cid testupg2) >/dev/null + # Check whether some information is provided + if ! docker logs $(ct_get_cid testupg2) 2>&1 | grep -e 'Version of the data could not be determined' &>/dev/null ; then + echo "Information about missing version file is not available in the logs" + return 1 + fi + # Check whether upgrade did not happen + if docker logs $(ct_get_cid testupg2) 2>&1 | grep -e 'Running mysql_upgrade' &>/dev/null ; then + echo "Upgrade should not be run when information about version is missing" + return 1 + fi + + # Create version file that is too old + echo " Testing upgrade from too old data" + echo "5.0.12" >${datadir}/mysql_upgrade_info + # Create another container with same data and upgrade set to 'upgrade-auto' + assert_container_creation_fails -e MYSQL_USER=user -e MYSQL_PASSWORD=foo \ + -e MYSQL_DATABASE=db -v ${datadir}:/var/lib/mysql/data:Z -e MYSQL_DATADIR_ACTION=upgrade-auto 2>/dev/null + + # Create version file that we can upgrade from + echo " Testing upgrade from previous version" + echo "$(get_previous_major_version ${VERSION}).12" >${datadir}/mysql_upgrade_info + # Create another container with same data and upgrade set to 'upgrade-aauto' + create_container "testupg3" -e MYSQL_USER=user -e MYSQL_PASSWORD=foo \ + -e MYSQL_DATABASE=db -v ${datadir}:/var/lib/mysql/data:Z -e MYSQL_DATADIR_ACTION=upgrade-auto + test_connection testupg3 user foo + docker stop $(ct_get_cid testupg3) >/dev/null + # Check whether some upgrade happened + if ! docker logs $(ct_get_cid testupg3) 2>&1 | grep -qe 'Running mysql_upgrade' ; then + echo "Upgrade did not happen but it should when upgrading from previous version" + docker logs $(ct_get_cid testupg3) + return 1 + fi + + # Create version file that we don't need to upgrade from + echo " Testing upgrade from the same version" + echo "${VERSION}.12" >${datadir}/mysql_upgrade_info + # Create another container with same data and upgrade set to 'upgrade-aauto' + create_container "testupg4" -e MYSQL_USER=user -e MYSQL_PASSWORD=foo \ + -e MYSQL_DATABASE=db -v ${datadir}:/var/lib/mysql/data:Z -e MYSQL_DATADIR_ACTION=upgrade-auto + test_connection testupg4 user foo + docker stop $(ct_get_cid testupg4) >/dev/null + # Check whether some upgrade happened + if docker logs $(ct_get_cid testupg4) 2>&1 | grep -e 'Running mysql_upgrade' &>/dev/null ; then + echo "Upgrade happened but it should not when upgrading from current version" + return 1 + fi + + # Create second container with same data and upgrade set to 'analyze' + echo " Testing running --analyze" + create_container "testupg5" -e MYSQL_USER=user -e MYSQL_PASSWORD=foo \ + -e MYSQL_DATABASE=db -v ${datadir}:/var/lib/mysql/data:Z -e MYSQL_DATADIR_ACTION=analyze + test_connection testupg5 user foo + docker stop $(ct_get_cid testupg5) >/dev/null + # Check whether analyze happened + if ! docker logs $(ct_get_cid testupg5) 2>&1 | grep -e '--analyze --all-databases' &>/dev/null ; then + echo "Analyze did not happen but it should" + return 1 + fi + + # Create another container with same data and upgrade set to 'optimize' + echo " Testing running --optimize" + create_container "testupg6" -e MYSQL_USER=user -e MYSQL_PASSWORD=foo \ + -e MYSQL_DATABASE=db -v ${datadir}:/var/lib/mysql/data:Z -e MYSQL_DATADIR_ACTION=optimize + test_connection testupg6 user foo + docker stop $(ct_get_cid testupg6) >/dev/null + # Check whether optimize happened + if ! docker logs $(ct_get_cid testupg6) 2>&1 | grep -e '--optimize --all-databases' &>/dev/null ; then + echo "Optimize did not happen but it should" + return 1 + fi + + # Create version file that we cannot upgrade from + echo " Testing upgrade from the future version" + echo "20.1.12" >${datadir}/mysql_upgrade_info + assert_container_creation_fails -e MYSQL_USER=user -e MYSQL_PASSWORD=foo \ + -e MYSQL_DATABASE=db -v ${datadir}:/var/lib/mysql/data:Z -e MYSQL_DATADIR_ACTION=upgrade-auto 2>/dev/null + + echo " Upgrade tests succeeded!" + echo +} + +function run_all_tests() { + for test_case in $TEST_LIST; do + echo "Running test $test_case for ${IMAGE_NAME}" + $test_case + done; +} + +# Run the chosen tests +TEST_LIST=${@:-$TEST_LIST} run_all_tests + +TESTSUITE_RESULT=0 -run_doc_test diff --git a/test/run-openshift b/test/run-openshift new file mode 120000 index 0000000..d84575f --- /dev/null +++ b/test/run-openshift @@ -0,0 +1 @@ +run-openshift-local-cluster \ No newline at end of file diff --git a/test/run-openshift-local-cluster b/test/run-openshift-local-cluster new file mode 100755 index 0000000..8a39a40 --- /dev/null +++ b/test/run-openshift-local-cluster @@ -0,0 +1,54 @@ +#!/bin/bash +# +# Test the MariaDB image in OpenShift (local cluster) +# +# IMAGE_NAME specifies a name of the candidate image used for testing. +# The image has to be available before this script is executed. +# VERSION specifies the major version of the MariaDB in format of X.Y +# OS specifies RHEL version (e.g. OS=rhel7) +# + +THISDIR=$(dirname ${BASH_SOURCE[0]}) + +source ${THISDIR}/test-lib-mysql.sh + +set -eo nounset + +trap ct_os_cleanup EXIT SIGINT + +ct_os_check_compulsory_vars + +ct_os_cluster_up + +test_mysql_pure_image "${IMAGE_NAME}" + +test_mysql_template "${IMAGE_NAME}" + +# TODO: Can we make the build against examples inside the same PR? +test_mysql_s2i "${IMAGE_NAME}" "https://github.com/sclorg/mariadb-container.git" test/test-app + +test_mariadb_integration "${IMAGE_NAME}" "${VERSION}" mariadb + +# test with a released image and an integrated template +# ignore possible failure of this test for centos images +fail_not_released=true +if [ "${OS}" == "rhel7" ] ; then + PUBLIC_IMAGE_NAME=${PUBLIC_IMAGE_NAME:-${REGISTRY:-registry.redhat.io/}rhscl/${BASE_IMAGE_NAME}-${VERSION//./}-rhel7} +else + PUBLIC_IMAGE_NAME=${PUBLIC_IMAGE_NAME:-${REGISTRY:-}centos/${BASE_IMAGE_NAME}-${VERSION//./}-centos7} + fail_not_released=false +fi + +export CT_SKIP_UPLOAD_IMAGE=true +# Try pulling the image first to see if it is accessible +if docker pull "${PUBLIC_IMAGE_NAME}"; then + test_mariadb_integration mariadb "${VERSION}" "${PUBLIC_IMAGE_NAME}" +else + echo "Warning: ${PUBLIC_IMAGE_NAME} could not be downloaded via 'docker'" + ! $fail_not_released || false "ERROR: Failed to pull image" +fi + +OS_TESTSUITE_RESULT=0 + +ct_os_cluster_down + diff --git a/test/run-openshift-remote-cluster b/test/run-openshift-remote-cluster new file mode 100755 index 0000000..f52c670 --- /dev/null +++ b/test/run-openshift-remote-cluster @@ -0,0 +1,30 @@ +#!/bin/bash +# +# Test the MariaDB image in OpenShift (remote cluster) +# +# IMAGE_NAME specifies a name of the candidate image used for testing. +# The image has to be available before this script is executed. +# VERSION specifies the major version of the MariaDB in format of X.Y +# OS specifies RHEL version (e.g. OS=rhel7) +# + +THISDIR=$(dirname ${BASH_SOURCE[0]}) + +source ${THISDIR}/test-lib-mysql.sh + +set -eo nounset + +trap ct_os_cleanup EXIT SIGINT + +ct_os_check_compulsory_vars + +oc status || false "It looks like oc is not properly logged in." + +export CT_SKIP_NEW_PROJECT=true +export CT_SKIP_UPLOAD_IMAGE=true +export CT_NAMESPACE=openshift + +test_mariadb_integration mariadb ${VERSION} "${IMAGE_NAME}" + +OS_TESTSUITE_RESULT=0 + diff --git a/test/test-app/mysql-cfg/myconfig.cnf b/test/test-app/mysql-cfg/myconfig.cnf new file mode 100644 index 0000000..7764adf --- /dev/null +++ b/test/test-app/mysql-cfg/myconfig.cnf @@ -0,0 +1,3 @@ +[mysqld] +query-cache-limit=262144 + diff --git a/test/test-app/mysql-data/init.sql b/test/test-app/mysql-data/init.sql new file mode 100644 index 0000000..3159982 --- /dev/null +++ b/test/test-app/mysql-data/init.sql @@ -0,0 +1,4 @@ +CREATE TABLE products (id INTEGER, name VARCHAR(256), price FLOAT, variant INTEGER); +CREATE TABLE products_variant (id INTEGER, name VARCHAR(256)); +INSERT INTO products_variant (id, name) VALUES ('1', 'blue'), ('2', 'green'); + diff --git a/test/test-app/mysql-init/80-add-arbitrary-users.sh b/test/test-app/mysql-init/80-add-arbitrary-users.sh new file mode 100644 index 0000000..55ae2d2 --- /dev/null +++ b/test/test-app/mysql-init/80-add-arbitrary-users.sh @@ -0,0 +1,17 @@ +create_arbitrary_users() { + # Do not care what option is compulsory here, just create what is specified + log_info "Creating user specified by MYSQL_OPERATIONS_USER (${MYSQL_OPERATIONS_USER}) ..." +mysql $mysql_flags < testdb -utestu -ptestp\" '^42' 120" \ + "-p MARIADB_VERSION=${VERSION} \ + -p DATABASE_SERVICE_NAME="${service_name}-testing" \ + -p MYSQL_USER=testu \ + -p MYSQL_PASSWORD=testp \ + -p MYSQL_DATABASE=testdb" "" "${import_image}" +} + +# vim: set tabstop=2:shiftwidth=2:expandtab: diff --git a/test/test-lib-openshift.sh b/test/test-lib-openshift.sh new file mode 100644 index 0000000..f62eab6 --- /dev/null +++ b/test/test-lib-openshift.sh @@ -0,0 +1,1043 @@ +# shellcheck shell=bash +# some functions are used from test-lib.sh, that is usually in the same dir +# shellcheck source=/dev/null +source "$(dirname "${BASH_SOURCE[0]}")"/test-lib.sh + +# Set of functions for testing docker images in OpenShift using 'oc' command + +# A variable containing the overall test result; must be changed to 0 in the end +# of the testing script: +# OS_TESTSUITE_RESULT=0 +# And the following trap must be set, in the beginning of the test script: +# trap ct_os_cleanup EXIT SIGINT +OS_TESTSUITE_RESULT=1 +OS_CLUSTER_STARTED_BY_TEST=0 + +function ct_os_cleanup() { + if [ $OS_TESTSUITE_RESULT -eq 0 ] ; then + # shellcheck disable=SC2153 + echo "OpenShift tests for ${IMAGE_NAME} succeeded." + else + # shellcheck disable=SC2153 + echo "OpenShift tests for ${IMAGE_NAME} failed." + fi +} + +# ct_os_check_compulsory_vars +# --------------------------- +# Check the compulsory variables: +# * IMAGE_NAME specifies a name of the candidate image used for testing. +# * VERSION specifies the major version of the MariaDB in format of X.Y +# * OS specifies RHEL version (e.g. OS=rhel7) +function ct_os_check_compulsory_vars() { + # shellcheck disable=SC2016 + test -n "${IMAGE_NAME-}" || ( echo 'make sure $IMAGE_NAME is defined' >&2 ; exit 1) + # shellcheck disable=SC2016 + test -n "${VERSION-}" || ( echo 'make sure $VERSION is defined' >&2 ; exit 1) + # shellcheck disable=SC2016 + test -n "${OS-}" || ( echo 'make sure $OS is defined' >&2 ; exit 1) +} + +# ct_os_get_status +# -------------------- +# Returns status of all objects to make debugging easier. +function ct_os_get_status() { + oc get all + oc status +} + +# ct_os_print_logs +# -------------------- +# Returns status of all objects and logs from all pods. +function ct_os_print_logs() { + ct_os_get_status + while read -r pod_name; do + echo "INFO: printing logs for pod ${pod_name}" + oc logs "${pod_name}" + done < <(oc get pods --no-headers=true -o custom-columns=NAME:.metadata.name) +} + +# ct_os_enable_print_logs +# -------------------- +# Enables automatic printing of pod logs on ERR. +function ct_os_enable_print_logs() { + set -E + trap ct_os_print_logs ERR +} + +# ct_get_public_ip +# -------------------- +# Returns best guess for the IP that the node is accessible from other computers. +# This is a bit funny heuristic, simply goes through all IPv4 addresses that +# hostname -I returns and de-prioritizes IP addresses commonly used for local +# addressing. The rest of addresses are taken as public with higher probability. +function ct_get_public_ip() { + local hostnames + local public_ip='' + local found_ip + hostnames=$(hostname -I) + for guess_exp in '127\.0\.0\.1' '192\.168\.[0-9\.]*' '172\.[0-9\.]*' \ + '10\.[0-9\.]*' '[0-9\.]*' ; do + found_ip=$(echo "${hostnames}" | grep -oe "${guess_exp}") + if [ -n "${found_ip}" ] ; then + # shellcheck disable=SC2001 + hostnames=$(echo "${hostnames}" | sed -e "s/${found_ip}//") + public_ip="${found_ip}" + fi + done + if [ -z "${public_ip}" ] ; then + echo "ERROR: public IP could not be guessed." >&2 + return 1 + fi + echo "${public_ip}" +} + +# ct_os_run_in_pod POD_NAME CMD +# -------------------- +# Runs [cmd] in the pod specified by prefix [pod_prefix]. +# Arguments: pod_name - full name of the pod +# Arguments: cmd - command to be run in the pod +function ct_os_run_in_pod() { + local pod_name="$1" ; shift + + oc exec "$pod_name" -- "$@" +} + +# ct_os_get_service_ip SERVICE_NAME +# -------------------- +# Returns IP of the service specified by [service_name]. +# Arguments: service_name - name of the service +function ct_os_get_service_ip() { + local service_name="${1}" ; shift + oc get "svc/${service_name}" -o yaml | grep clusterIP | \ + cut -d':' -f2 | grep -oe '172\.30\.[0-9\.]*' +} + + +# ct_os_get_all_pods_status +# -------------------- +# Returns status of all pods. +function ct_os_get_all_pods_status() { + oc get pods -o custom-columns=Ready:status.containerStatuses[0].ready,NAME:.metadata.name +} + +# ct_os_get_all_pods_name +# -------------------- +# Returns the full name of all pods. +function ct_os_get_all_pods_name() { + oc get pods --no-headers -o custom-columns=NAME:.metadata.name +} + +# ct_os_get_pod_status POD_PREFIX +# -------------------- +# Returns status of the pod specified by prefix [pod_prefix]. +# Note: Ignores -build and -deploy pods +# Arguments: pod_prefix - prefix or whole ID of the pod +function ct_os_get_pod_status() { + local pod_prefix="${1}" ; shift + ct_os_get_all_pods_status | grep -e "${pod_prefix}" | grep -Ev "(build|deploy)$" \ + | awk '{print $1}' | head -n 1 +} + +# ct_os_get_pod_name POD_PREFIX +# -------------------- +# Returns the full name of pods specified by prefix [pod_prefix]. +# Note: Ignores -build and -deploy pods +# Arguments: pod_prefix - prefix or whole ID of the pod +function ct_os_get_pod_name() { + local pod_prefix="${1}" ; shift + ct_os_get_all_pods_name | grep -e "^${pod_prefix}" | grep -Ev "(build|deploy)$" +} + +# ct_os_get_pod_ip POD_NAME +# -------------------- +# Returns the ip of the pod specified by [pod_name]. +# Arguments: pod_name - full name of the pod +function ct_os_get_pod_ip() { + local pod_name="${1}" + oc get pod "$pod_name" --no-headers -o custom-columns=IP:status.podIP +} + +# ct_os_check_pod_readiness POD_PREFIX STATUS +# -------------------- +# Checks whether the pod is ready. +# Arguments: pod_prefix - prefix or whole ID of the pod +# Arguments: status - expected status (true, false) +function ct_os_check_pod_readiness() { + local pod_prefix="${1}" ; shift + local status="${1}" ; shift + test "$(ct_os_get_pod_status "${pod_prefix}")" == "${status}" +} + +# ct_os_wait_pod_ready POD_PREFIX TIMEOUT +# -------------------- +# Wait maximum [timeout] for the pod becomming ready. +# Arguments: pod_prefix - prefix or whole ID of the pod +# Arguments: timeout - how many seconds to wait seconds +function ct_os_wait_pod_ready() { + local pod_prefix="${1}" ; shift + local timeout="${1}" ; shift + SECONDS=0 + echo -n "Waiting for ${pod_prefix} pod becoming ready ..." + while ! ct_os_check_pod_readiness "${pod_prefix}" "true" ; do + echo -n "." + [ "${SECONDS}" -gt "${timeout}" ] && echo " FAIL" && return 1 + sleep 3 + done + echo " DONE" +} + +# ct_os_wait_rc_ready POD_PREFIX TIMEOUT +# -------------------- +# Wait maximum [timeout] for the rc having desired number of replicas ready. +# Arguments: pod_prefix - prefix of the replication controller +# Arguments: timeout - how many seconds to wait seconds +function ct_os_wait_rc_ready() { + local pod_prefix="${1}" ; shift + local timeout="${1}" ; shift + SECONDS=0 + echo -n "Waiting for ${pod_prefix} pod becoming ready ..." + while ! test "$( (oc get --no-headers statefulsets; oc get --no-headers rc) 2>/dev/null \ + | grep "^${pod_prefix}" | awk '$2==$3 {print "ready"}')" == "ready" ; do + echo -n "." + [ "${SECONDS}" -gt "${timeout}" ] && echo " FAIL" && return 1 + sleep 3 + done + echo " DONE" +} + +# ct_os_deploy_pure_image IMAGE [ENV_PARAMS, ...] +# -------------------- +# Runs [image] in the openshift and optionally specifies env_params +# as environment variables to the image. +# Arguments: image - prefix or whole ID of the pod to run the cmd in +# Arguments: env_params - environment variables parameters for the images. +function ct_os_deploy_pure_image() { + local image="${1}" ; shift + # ignore error exit code, because oc new-app returns error when image exists + oc new-app "${image}" "$@" || : + # let openshift cluster to sync to avoid some race condition errors + sleep 3 +} + +# ct_os_deploy_s2i_image IMAGE APP [ENV_PARAMS, ... ] +# -------------------- +# Runs [image] and [app] in the openshift and optionally specifies env_params +# as environment variables to the image. +# Arguments: image - prefix or whole ID of the pod to run the cmd in +# Arguments: app - url or local path to git repo with the application sources. +# Arguments: env_params - environment variables parameters for the images. +function ct_os_deploy_s2i_image() { + local image="${1}" ; shift + local app="${1}" ; shift + # ignore error exit code, because oc new-app returns error when image exists + oc new-app "${image}~${app}" "$@" || : + + # let openshift cluster to sync to avoid some race condition errors + sleep 3 +} + +# ct_os_deploy_template_image TEMPLATE [ENV_PARAMS, ...] +# -------------------- +# Runs template in the openshift and optionally gives env_params to use +# specific values in the template. +# Arguments: template - prefix or whole ID of the pod to run the cmd in +# Arguments: env_params - environment variables parameters for the template. +# Example usage: ct_os_deploy_template_image mariadb-ephemeral-template.yaml \ +# DATABASE_SERVICE_NAME=mysql-57-centos7 \ +# DATABASE_IMAGE=mysql-57-centos7 \ +# MYSQL_USER=testu \ +# MYSQL_PASSWORD=testp \ +# MYSQL_DATABASE=testdb +function ct_os_deploy_template_image() { + local template="${1}" ; shift + oc process -f "${template}" "$@" | oc create -f - + # let openshift cluster to sync to avoid some race condition errors + sleep 3 +} + +# _ct_os_get_uniq_project_name +# -------------------- +# Returns a uniq name of the OpenShift project. +function _ct_os_get_uniq_project_name() { + local r + while true ; do + r=${RANDOM} + mkdir /var/tmp/sclorg-test-${r} &>/dev/null && echo sclorg-test-${r} && break + done +} + +# ct_os_new_project [PROJECT] +# -------------------- +# Creates a new project in the openshfit using 'os' command. +# Arguments: project - project name, uses a new random name if omitted +# Expects 'os' command that is properly logged in to the OpenShift cluster. +# Not using mktemp, because we cannot use uppercase characters. +# The OPENSHIFT_CLUSTER_PULLSECRET_PATH environment variable can be set +# to contain a path to a k8s secret definition which will be used +# to authenticate to image registries. +# shellcheck disable=SC2120 +function ct_os_new_project() { + if [ "${CT_SKIP_NEW_PROJECT:-false}" == 'true' ] ; then + echo "Creating project skipped." + return + fi + local project_name="${1:-$(_ct_os_get_uniq_project_name)}" ; shift || : + oc new-project "${project_name}" + # let openshift cluster to sync to avoid some race condition errors + sleep 3 + if test -n "${OPENSHIFT_CLUSTER_PULLSECRET_PATH:-}" -a -e "${OPENSHIFT_CLUSTER_PULLSECRET_PATH:-}"; then + oc create -f "$OPENSHIFT_CLUSTER_PULLSECRET_PATH" + # add registry pullsecret to the serviceaccount if provided + secret_name=$(grep '^\s*name:' "$OPENSHIFT_CLUSTER_PULLSECRET_PATH" | awk '{ print $2 }') + secret_json='{"imagePullSecrets": [{"name": "'${secret_name}'"}]}' + oc patch serviceaccount default -p "$secret_json" + fi +} + +# ct_os_delete_project [PROJECT] +# -------------------- +# Deletes the specified project in the openshfit +# Arguments: project - project name, uses the current project if omitted +# shellcheck disable=SC2120 +function ct_os_delete_project() { + if [ "${CT_SKIP_NEW_PROJECT:-false}" == 'true' ] ; then + echo "Deleting project skipped, cleaning objects only." + # when not having enough privileges (remote cluster), it might fail and + # it is not a big problem, so ignore failure in this case + ct_delete_all_objects || : + return + fi + local project_name="${1:-$(oc project -q)}" ; shift || : + oc delete project "${project_name}" +} + +# ct_delete_all_objects +# ----------------- +# Deletes all objects within the project. +# Handy when we have one project and want to run more tests. +function ct_delete_all_objects() { + for x in bc builds dc is isimage istag po pv pvc rc routes secrets svc ; do + oc delete "$x" --all + done + # for some objects it takes longer to be really deleted, so a dummy sleep + # to avoid some races when other test can see not-yet-deleted objects and can fail + sleep 10 +} + +# ct_os_docker_login +# -------------------- +# Logs in into docker daemon +# Uses global REGISRTY_ADDRESS environment variable for arbitrary registry address. +# Does not do anything if REGISTRY_ADDRESS is set. +function ct_os_docker_login() { + [ -n "${REGISTRY_ADDRESS:-}" ] && "REGISTRY_ADDRESS set, not trying to docker login." && return 0 + # docker login fails with "404 page not found" error sometimes, just try it more times + # shellcheck disable=SC2034 + for i in $(seq 12) ; do + # shellcheck disable=SC2015 + docker login -u developer -p "$(oc whoami -t)" "${REGISRTY_ADDRESS:-172.30.1.1:5000}" && return 0 || : + sleep 5 + done + return 1 +} + +# ct_os_upload_image IMAGE [IMAGESTREAM] +# -------------------- +# Uploads image from local registry to the OpenShift internal registry. +# Arguments: image - image name to upload +# Arguments: imagestream - name and tag to use for the internal registry. +# In the format of name:tag ($image_name:latest by default) +# Uses global REGISRTY_ADDRESS environment variable for arbitrary registry address. +function ct_os_upload_image() { + local input_name="${1}" ; shift + local image_name=${input_name##*/} + local imagestream=${1:-$image_name:latest} + local output_name + + output_name="${REGISRTY_ADDRESS:-172.30.1.1:5000}/$(oc project -q)/$imagestream" + + ct_os_docker_login + docker tag "${input_name}" "${output_name}" + docker push "${output_name}" +} + +# ct_os_is_tag_exists IS_NAME TAG +# -------------------- +# Checks whether the specified tag exists for an image stream +# Arguments: is_name - name of the image stream +# Arguments: tag - name of the tag (usually version) +function ct_os_is_tag_exists() { + local is_name=$1 ; shift + local tag=$1 ; shift + oc get is "${is_name}" -n openshift -o=jsonpath='{.spec.tags[*].name}' | grep -qw "${tag}" +} + +# ct_os_template_exists T_NAME +# -------------------- +# Checks whether the specified template exists for an image stream +# Arguments: t_name - template name of the image stream +function ct_os_template_exists() { + local t_name=$1 ; shift + oc get templates -n openshift | grep -q "^${t_name}\s" +} + +# ct_os_install_in_centos +# -------------------- +# Installs os cluster in CentOS +function ct_os_install_in_centos() { + yum install -y centos-release-openshift-origin + yum install -y wget git net-tools bind-utils iptables-services bridge-utils\ + bash-completion origin-clients docker origin-clients +} + +# ct_os_cluster_up [DIR, IS_PUBLIC, CLUSTER_VERSION] +# -------------------- +# Runs the local OpenShift cluster using 'oc cluster up' and logs in as developer. +# Arguments: dir - directory to keep configuration data in, random if omitted +# Arguments: is_public - sets either private or public hostname for web-UI, +# use "true" for allow remote access to the web-UI, +# "false" is default +# Arguments: cluster_version - version of the OpenShift cluster to use, empty +# means default version of `oc`; example value: 3.7; +# also can be specified outside by OC_CLUSTER_VERSION +function ct_os_cluster_up() { + ct_os_cluster_running && echo "Cluster already running. Nothing is done." && return 0 + ct_os_logged_in && echo "Already logged in to a cluster. Nothing is done." && return 0 + + mkdir -p /var/tmp/openshift + local dir="${1:-$(mktemp -d /var/tmp/openshift/os-data-XXXXXX)}" ; shift || : + local is_public="${1:-'false'}" ; shift || : + local default_cluster_version=${OC_CLUSTER_VERSION:-} + local cluster_version=${1:-${default_cluster_version}} ; shift || : + if ! grep -qe '--insecure-registry.*172\.30\.0\.0' /etc/sysconfig/docker ; then + sed -i "s|OPTIONS='|OPTIONS='--insecure-registry 172.30.0.0/16 |" /etc/sysconfig/docker + fi + + systemctl stop firewalld || : + setenforce 0 + iptables -F + + systemctl restart docker + local cluster_ip="127.0.0.1" + [ "${is_public}" == "true" ] && cluster_ip=$(ct_get_public_ip) + + if [ -n "${cluster_version}" ] ; then + # if $cluster_version is not set, we simply use oc that is available + ct_os_set_path_oc "${cluster_version}" + fi + + mkdir -p "${dir}"/{config,data,pv} + case $(oc version| head -n 1) in + "oc v3.1"?.*) + oc cluster up --base-dir="${dir}/data" --public-hostname="${cluster_ip}" + ;; + "oc v3."*) + oc cluster up --host-data-dir="${dir}/data" --host-config-dir="${dir}/config" \ + --host-pv-dir="${dir}/pv" --use-existing-config --public-hostname="${cluster_ip}" + ;; + *) + echo "ERROR: Unexpected oc version." >&2 + return 1 + ;; + esac + oc version + oc login -u system:admin + oc project default + ct_os_wait_rc_ready docker-registry 180 + ct_os_wait_rc_ready router 30 + oc login -u developer -p developer + OS_CLUSTER_STARTED_BY_TEST=1 + # let openshift cluster to sync to avoid some race condition errors + sleep 3 +} + +# ct_os_cluster_down +# -------------------- +# Shuts down the local OpenShift cluster using 'oc cluster down' +function ct_os_cluster_down() { + if [ ${OS_CLUSTER_STARTED_BY_TEST:-0} -eq 1 ] ; then + echo "Cluster started by the test, shutting down." + oc cluster down + else + echo "Cluster not started by the test, shutting down skipped." + fi +} + +# ct_os_cluster_running +# -------------------- +# Returns 0 if oc cluster is running +function ct_os_cluster_running() { + oc cluster status &>/dev/null +} + +# ct_os_logged_in +# --------------- +# Returns 0 if logged in to a cluster (remote or local) +function ct_os_logged_in() { + oc whoami >/dev/null +} + +# ct_os_set_path_oc OC_VERSION +# -------------------- +# This is a trick that helps using correct version of the `oc`: +# The input is version of the openshift in format v3.6.0 etc. +# If the currently available version of oc is not of this version, +# it first takes a look into /usr/local/oc-/bin directory, +# and if not found there it downloads the community release from github. +# In the end the PATH variable is changed, so the other tests can still use just 'oc'. +# Arguments: oc_version - X.Y part of the version of OSE (e.g. 3.9) +function ct_os_set_path_oc() { + local oc_version + local oc_path + + oc_version=$(ct_os_get_latest_ver "$1") + + if oc version | grep -q "oc ${oc_version%.*}." ; then + echo "Binary oc found already available in version ${oc_version}: $(command -v oc) Doing noting." + return 0 + fi + + # first check whether we already have oc available in /usr/local + local installed_oc_path="/usr/local/oc-${oc_version%.*}/bin" + + if [ -x "${installed_oc_path}/oc" ] ; then + oc_path="${installed_oc_path}" + echo "Binary oc found in ${installed_oc_path}" >&2 + else + # oc not available in /usr/local, try to download it from github (community release) + oc_path="/tmp/oc-${oc_version}-bin" + ct_os_download_upstream_oc "${oc_version}" "${oc_path}" + fi + if [ -z "${oc_path}" ] ; then + echo "ERROR: oc not found installed, nor downloaded" >&1 + return 1 + fi + export PATH="${oc_path}:${PATH}" + if ! oc version | grep -q "oc ${oc_version%.*}." ; then + echo "ERROR: something went wrong, oc located at ${oc_path}, but oc of version ${oc_version} not found in PATH ($PATH)" >&1 + return 1 + else + echo "PATH set correctly, binary oc found in version ${oc_version}: $(command -v oc)" + fi +} + +# ct_os_get_latest_ver VERSION_PART_X +# -------------------- +# Returns full version (vX.Y.Z) from part of the version (X.Y) +# Arguments: vxy - X.Y part of the version +# Returns vX.Y.Z variant of the version +function ct_os_get_latest_ver(){ + local vxy="v$1" + for vz in {3..0} ; do + curl -sif "https://github.com/openshift/origin/releases/tag/${vxy}.${vz}" >/dev/null && echo "${vxy}.${vz}" && return 0 + done + echo "ERROR: version ${vxy} not found in https://github.com/openshift/origin/tags" >&2 + return 1 +} + +# ct_os_download_upstream_oc OC_VERSION OUTPUT_DIR +# -------------------- +# Downloads a particular version of openshift-origin-client-tools from +# github into specified output directory +# Arguments: oc_version - version of OSE (e.g. v3.7.2) +# Arguments: output_dir - output directory +function ct_os_download_upstream_oc() { + local oc_version=$1 + local output_dir=$2 + + # check whether we already have the binary in place + [ -x "${output_dir}/oc" ] && return 0 + + mkdir -p "${output_dir}" + # using html output instead of https://api.github.com/repos/openshift/origin/releases/tags/${oc_version}, + # because API is limited for number of queries if not authenticated + tarball=$(curl -si "https://github.com/openshift/origin/releases/tag/${oc_version}" | grep -o -e "openshift-origin-client-tools-${oc_version}-[a-f0-9]*-linux-64bit.tar.gz" | head -n 1) + + # download, unpack the binaries and then put them into output directory + echo "Downloading https://github.com/openshift/origin/releases/download/${oc_version}/${tarball} into ${output_dir}/" >&2 + curl -sL https://github.com/openshift/origin/releases/download/"${oc_version}"/"${tarball}" | tar -C "${output_dir}" -xz + mv -f "${output_dir}"/"${tarball%.tar.gz}"/* "${output_dir}/" + + rmdir "${output_dir}"/"${tarball%.tar.gz}" +} + + +# ct_os_test_s2i_app_func IMAGE APP CONTEXT_DIR CHECK_CMD [OC_ARGS] +# -------------------- +# Runs [image] and [app] in the openshift and optionally specifies env_params +# as environment variables to the image. Then check the container by arbitrary +# function given as argument (such an argument may include string, +# that will be replaced with actual IP). +# Arguments: image - prefix or whole ID of the pod to run the cmd in (compulsory) +# Arguments: app - url or local path to git repo with the application sources (compulsory) +# Arguments: context_dir - sub-directory inside the repository with the application sources (compulsory) +# Arguments: check_command - CMD line that checks whether the container works (compulsory; '' will be replaced with actual IP) +# Arguments: oc_args - all other arguments are used as additional parameters for the `oc new-app` +# command, typically environment variables (optional) +function ct_os_test_s2i_app_func() { + local image_name=${1} + local app=${2} + local context_dir=${3} + local check_command=${4} + local oc_args=${5:-} + local import_image=${6:-} + local image_name_no_namespace=${image_name##*/} + local service_name="${image_name_no_namespace}-testing" + local image_tagged="${image_name_no_namespace}:${VERSION}" + + if [ $# -lt 4 ] || [ -z "${1}" ] || [ -z "${2}" ] || [ -z "${3}" ] || [ -z "${4}" ]; then + echo "ERROR: ct_os_test_s2i_app_func() requires at least 4 arguments that cannot be emtpy." >&2 + return 1 + fi + + # shellcheck disable=SC2119 + ct_os_new_project + # Create a specific imagestream tag for the image so that oc cannot use anything else + if [ "${CT_SKIP_UPLOAD_IMAGE:-false}" == 'true' ] ; then + if [ -n "${import_image}" ] ; then + echo "Importing image ${import_image} as ${image_name}:${VERSION}" + oc import-image "${image_name}":"${VERSION}" --from "${import_image}" --confirm + else + echo "Uploading and importing image skipped." + fi + else + if [ -n "${import_image}" ] ; then + echo "Warning: Import image ${import_image} requested, but uploading image ${image_name} instead." + fi + ct_os_upload_image "${image_name}" "${image_tagged}" + fi + + local app_param="${app}" + if [ -d "${app}" ] ; then + # for local directory, we need to copy the content, otherwise too smart os command + # pulls the git remote repository instead + app_param=$(ct_obtain_input "${app}") + fi + + # shellcheck disable=SC2086 + ct_os_deploy_s2i_image "${image_tagged}" "${app_param}" \ + --context-dir="${context_dir}" \ + --name "${service_name}" \ + ${oc_args} + + if [ -d "${app}" ] ; then + # in order to avoid weird race seen sometimes, let's wait shortly + # before starting the build explicitly + sleep 5 + oc start-build "${service_name}" --from-dir="${app_param}" + fi + + ct_os_wait_pod_ready "${service_name}" 300 + + local ip + local check_command_exp + + ip=$(ct_os_get_service_ip "${service_name}") + # shellcheck disable=SC2001 + check_command_exp=$(echo "$check_command" | sed -e "s//$ip/g") + + echo " Checking APP using $check_command_exp ..." + local result=0 + eval "$check_command_exp" || result=1 + + if [ $result -eq 0 ] ; then + echo " Check passed." + else + echo " Check failed." + fi + + # shellcheck disable=SC2119 + ct_os_delete_project + return $result +} + +# ct_os_test_s2i_app IMAGE APP CONTEXT_DIR EXPECTED_OUTPUT [PORT, PROTOCOL, RESPONSE_CODE, OC_ARGS, ... ] +# -------------------- +# Runs [image] and [app] in the openshift and optionally specifies env_params +# as environment variables to the image. Then check the http response. +# Arguments: image - prefix or whole ID of the pod to run the cmd in (compulsory) +# Arguments: app - url or local path to git repo with the application sources (compulsory) +# Arguments: context_dir - sub-directory inside the repository with the application sources (compulsory) +# Arguments: expected_output - PCRE regular expression that must match the response body (compulsory) +# Arguments: port - which port to use (optional; default: 8080) +# Arguments: protocol - which protocol to use (optional; default: http) +# Arguments: response_code - what http response code to expect (optional; default: 200) +# Arguments: oc_args - all other arguments are used as additional parameters for the `oc new-app` +# command, typically environment variables (optional) +function ct_os_test_s2i_app() { + local image_name=${1} + local app=${2} + local context_dir=${3} + local expected_output=${4} + local port=${5:-8080} + local protocol=${6:-http} + local response_code=${7:-200} + local oc_args=${8:-} + local import_image=${9:-} + + if [ $# -lt 4 ] || [ -z "${1}" ] || [ -z "${2}" ] || [ -z "${3}" ] || [ -z "${4}" ]; then + echo "ERROR: ct_os_test_s2i_app() requires at least 4 arguments that cannot be emtpy." >&2 + return 1 + fi + + ct_os_test_s2i_app_func "${image_name}" \ + "${app}" \ + "${context_dir}" \ + "ct_os_test_response_internal '${protocol}://:${port}' '${response_code}' '${expected_output}'" \ + "${oc_args}" "${import_image}" +} + +# ct_os_test_template_app_func IMAGE APP IMAGE_IN_TEMPLATE CHECK_CMD [OC_ARGS] +# -------------------- +# Runs [image] and [app] in the openshift and optionally specifies env_params +# as environment variables to the image. Then check the container by arbitrary +# function given as argument (such an argument may include string, +# that will be replaced with actual IP). +# Arguments: image_name - prefix or whole ID of the pod to run the cmd in (compulsory) +# Arguments: template - url or local path to a template to use (compulsory) +# Arguments: name_in_template - image name used in the template +# Arguments: check_command - CMD line that checks whether the container works (compulsory; '' will be replaced with actual IP) +# Arguments: oc_args - all other arguments are used as additional parameters for the `oc new-app` +# command, typically environment variables (optional) +# Arguments: other_images - some templates need other image to be pushed into the OpenShift registry, +# specify them in this parameter as "|", where "" is a full image name +# (including registry if needed) and "" is a tag under which the image should be available +# in the OpenShift registry. +function ct_os_test_template_app_func() { + local image_name=${1} + local template=${2} + local name_in_template=${3} + local check_command=${4} + local oc_args=${5:-} + local other_images=${6:-} + local import_image=${7:-} + + if [ $# -lt 4 ] || [ -z "${1}" ] || [ -z "${2}" ] || [ -z "${3}" ] || [ -z "${4}" ]; then + echo "ERROR: ct_os_test_template_app_func() requires at least 4 arguments that cannot be emtpy." >&2 + return 1 + fi + + local service_name="${name_in_template}-testing" + local image_tagged="${name_in_template}:${VERSION}" + + # shellcheck disable=SC2119 + ct_os_new_project + + # Create a specific imagestream tag for the image so that oc cannot use anything else + if [ "${CT_SKIP_UPLOAD_IMAGE:-false}" == 'true' ] ; then + if [ -n "${import_image}" ] ; then + echo "Importing image ${import_image} as ${image_name}:${VERSION}" + oc import-image "${image_name}":"${VERSION}" --from "${import_image}" --confirm + else + echo "Uploading and importing image skipped." + fi + else + if [ -n "${import_image}" ] ; then + echo "Warning: Import image ${import_image} requested, but uploading image ${image_name} instead." + fi + ct_os_upload_image "${image_name}" "${image_tagged}" + + # upload also other images, that template might need (list of pairs in the format | + local image_tag_a + local i_t + for i_t in ${other_images} ; do + echo "${i_t}" + IFS='|' read -ra image_tag_a <<< "${i_t}" + docker pull "${image_tag_a[0]}" + ct_os_upload_image "${image_tag_a[0]}" "${image_tag_a[1]}" + done + fi + + # get the template file from remote or local location; if not found, it is + # considered an internal template name, like 'mysql', so use the name + # explicitly + local local_template + local namespace + + namespace=${CT_NAMESPACE:-$(oc project -q)} + + local_template=$(ct_obtain_input "${template}" 2>/dev/null || echo "--template=${template}") + # shellcheck disable=SC2086 + oc new-app "${local_template}" \ + --name "${name_in_template}" \ + -p NAMESPACE="${namespace}" \ + ${oc_args} + + ct_os_wait_pod_ready "${service_name}" 300 + + local ip + local check_command_exp + + ip=$(ct_os_get_service_ip "${service_name}") + # shellcheck disable=SC2001 + check_command_exp=$(echo "$check_command" | sed -e "s//$ip/g") + + echo " Checking APP using $check_command_exp ..." + local result=0 + eval "$check_command_exp" || result=1 + + if [ $result -eq 0 ] ; then + echo " Check passed." + else + echo " Check failed." + fi + + # shellcheck disable=SC2119 + ct_os_delete_project + return $result +} + +# params: +# ct_os_test_template_app IMAGE APP IMAGE_IN_TEMPLATE EXPECTED_OUTPUT [PORT, PROTOCOL, RESPONSE_CODE, OC_ARGS, ... ] +# -------------------- +# Runs [image] and [app] in the openshift and optionally specifies env_params +# as environment variables to the image. Then check the http response. +# Arguments: image_name - prefix or whole ID of the pod to run the cmd in (compulsory) +# Arguments: template - url or local path to a template to use (compulsory) +# Arguments: name_in_template - image name used in the template +# Arguments: expected_output - PCRE regular expression that must match the response body (compulsory) +# Arguments: port - which port to use (optional; default: 8080) +# Arguments: protocol - which protocol to use (optional; default: http) +# Arguments: response_code - what http response code to expect (optional; default: 200) +# Arguments: oc_args - all other arguments are used as additional parameters for the `oc new-app` +# command, typically environment variables (optional) +# Arguments: other_images - some templates need other image to be pushed into the OpenShift registry, +# specify them in this parameter as "|", where "" is a full image name +# (including registry if needed) and "" is a tag under which the image should be available +# in the OpenShift registry. +function ct_os_test_template_app() { + local image_name=${1} + local template=${2} + local name_in_template=${3} + local expected_output=${4} + local port=${5:-8080} + local protocol=${6:-http} + local response_code=${7:-200} + local oc_args=${8:-} + local other_images=${9:-} + local import_image=${10:-} + + if [ $# -lt 4 ] || [ -z "${1}" ] || [ -z "${2}" ] || [ -z "${3}" ] || [ -z "${4}" ]; then + echo "ERROR: ct_os_test_template_app() requires at least 4 arguments that cannot be emtpy." >&2 + return 1 + fi + + ct_os_test_template_app_func "${image_name}" \ + "${template}" \ + "${name_in_template}" \ + "ct_os_test_response_internal '${protocol}://:${port}' '${response_code}' '${expected_output}'" \ + "${oc_args}" \ + "${other_images}" \ + "${import_image}" +} + +# ct_os_test_image_update IMAGE_NAME OLD_IMAGE ISTAG CHECK_FUNCTION OC_ARGS +# -------------------- +# Runs an image update test with [image] uploaded to [is] imagestream +# and checks the services using an arbitrary function provided in [check_function]. +# Arguments: image_name - prefix or whole ID of the pod to run the cmd in (compulsory) +# Arguments: old_image - valid name of the image from the registry +# Arguments: istag - imagestream to upload the images into (compulsory) +# Arguments: check_function - command to be run to check functionality of created services (compulsory) +# Arguments: oc_args - arguments to use during oc new-app (compulsory) +ct_os_test_image_update() { + local image_name=$1; shift + local old_image=$1; shift + local istag=$1; shift + local check_function=$1; shift + local service_name=${image_name##*/} + local ip="" check_command_exp="" + + echo "Running image update test for: $image_name" + # shellcheck disable=SC2119 + ct_os_new_project + + # Get current image from repository and create an imagestream + docker pull "$old_image:latest" 2>/dev/null + ct_os_upload_image "$old_image" "$istag" + + # Setup example application with curent image + oc new-app "$@" --name "$service_name" + ct_os_wait_pod_ready "$service_name" 60 + + # Check application output + ip=$(ct_os_get_service_ip "$service_name") + check_command_exp=${check_function///$ip} + ct_assert_cmd_success "$check_command_exp" + + # Tag built image into the imagestream and wait for rebuild + ct_os_upload_image "$image_name" "$istag" + ct_os_wait_pod_ready "${service_name}-2" 60 + + # Check application output + ip=$(ct_os_get_service_ip "$service_name") + check_command_exp=${check_function///$ip} + ct_assert_cmd_success "$check_command_exp" + + # shellcheck disable=SC2119 + ct_os_delete_project +} + +# ct_os_deploy_cmd_image IMAGE_NAME +# -------------------- +# Runs a special command pod, a pod that does nothing, but includes utilities for testing. +# A typical usage is a mysql pod that includes mysql commandline, that we need for testing. +# Running commands inside this command pod is done via ct_os_cmd_image_run function. +# The pod is not run again if already running. +# Arguments: image_name - image to be used as a command pod +function ct_os_deploy_cmd_image() { + local image_name=${1} + oc get pod command-app &>/dev/null && echo "command POD already running" && return 0 + echo "command POD not running yet, will start one called command-app" + oc create -f - <" + local sleep_time=3 + local attempt=1 + local result=1 + local status + local response_code + local response_file + local util_image_name='python:3.6' + + response_file=$(mktemp /tmp/ct_test_response_XXXXXX) + ct_os_deploy_cmd_image "${util_image_name}" + + while [ "${attempt}" -le "${max_attempts}" ]; do + ct_os_cmd_image_run "curl --connect-timeout 10 -s -w '%{http_code}' '${url}'" >"${response_file}" && status=0 || status=1 + if [ "${status}" -eq 0 ]; then + response_code=$(tail -c 3 "${response_file}") + if [ "${response_code}" -eq "${expected_code}" ]; then + result=0 + fi + grep -qP -e "${body_regexp}" "${response_file}" || result=1; + # Some services return 40x code until they are ready, so let's give them + # some chance and not end with failure right away + # Do not wait if we already have expected outcome though + if [ "${result}" -eq 0 ] || [ "${attempt}" -gt "${ignore_error_attempts}" ] || [ "${attempt}" -eq "${max_attempts}" ] ; then + break + fi + fi + attempt=$(( attempt + 1 )) + sleep "${sleep_time}" + done + rm -f "${response_file}" + return "${result}" +} + +# ct_os_get_image_from_pod +# ------------------------ +# Print image identifier from an existing pod to stdout +# Argument: pod_prefix - prefix or full name of the pod to get image from +ct_os_get_image_from_pod() { + local pod_prefix=$1 ; shift + local pod_name + pod_name=$(ct_os_get_pod_name "$pod_prefix") + oc get "po/${pod_name}" -o yaml | sed -ne 's/^\s*image:\s*\(.*\)\s*$/\1/ p' | head -1 +} + +# ct_os_check_cmd_internal +# ---------------- +# Runs a specified command, checks exit code and compares the output with expected regexp. +# That all is done inside an image in the cluster, so the function is used +# typically in clusters that are not accessible outside. +# The check is repeated until timeout. +# Argument: util_image_name - name of the image in the cluster that is used for running the cmd +# Argument: service_name - kubernetes' service name to work with (IP address is taken from this one) +# Argument: check_command - command that is run within the util_image_name container +# Argument: expected_content_match - regexp that must be in the output (use .* to ignore check) +# Argument: timeout - number of seconds to wait till the check succeeds +function ct_os_check_cmd_internal() { + local util_image_name=$1 ; shift + local service_name=$1 ; shift + local check_command=$1 ; shift + local expected_content_match=${1:-.*} ; shift + local timeout=${1:-60} ; shift || : + + : " Service ${service_name} check ..." + + local output + local ret + local ip + local check_command_exp + + ip=$(ct_os_get_service_ip "${service_name}") + # shellcheck disable=SC2001 + check_command_exp=$(echo "$check_command" | sed -e "s//$ip/g") + + ct_os_deploy_cmd_image "$(ct_os_get_image_from_pod "${util_image_name##*/}" | head -n 1)" + SECONDS=0 + + echo -n "Waiting for ${service_name} service becoming ready ..." + while true ; do + output=$(ct_os_cmd_image_run "$check_command_exp") + ret=$? + echo "${output}" | grep -qe "${expected_content_match}" || ret=1 + if [ ${ret} -eq 0 ] ; then + echo " PASS" + return 0 + fi + echo -n "." + [ ${SECONDS} -gt "${timeout}" ] && break + sleep 3 + done + echo " FAIL" + return 1 +} + +# vim: set tabstop=2:shiftwidth=2:expandtab: diff --git a/test/test-lib.sh b/test/test-lib.sh new file mode 100644 index 0000000..b959c34 --- /dev/null +++ b/test/test-lib.sh @@ -0,0 +1,653 @@ +# shellcheck shell=bash +# +# Test a container image. +# +# Always use sourced from a specific container testfile +# +# reguires definition of CID_FILE_DIR +# CID_FILE_DIR=$(mktemp --suffix=_test_cidfiles -d) +# reguires definition of TEST_LIST +# TEST_LIST="\ +# ctest_container_creation +# ctest_doc_content" + +# Container CI tests +# abbreviated as "ct" + +# may be redefined in the specific container testfile +EXPECTED_EXIT_CODE=0 + +# ct_cleanup +# -------------------- +# Cleans up containers used during tests. Stops and removes all containers +# referenced by cid_files in CID_FILE_DIR. Dumps logs if a container exited +# unexpectedly. Removes the cid_files and CID_FILE_DIR as well. +# Uses: $CID_FILE_DIR - path to directory containing cid_files +# Uses: $EXPECTED_EXIT_CODE - expected container exit code +function ct_cleanup() { + for cid_file in "$CID_FILE_DIR"/* ; do + local container + container=$(cat "$cid_file") + + : "Stopping and removing container $container..." + docker stop "$container" + exit_status=$(docker inspect -f '{{.State.ExitCode}}' "$container") + if [ "$exit_status" != "$EXPECTED_EXIT_CODE" ]; then + : "Dumping logs for $container" + docker logs "$container" + fi + docker rm -v "$container" + rm "$cid_file" + done + rmdir "$CID_FILE_DIR" + : "Done." +} + +# ct_enable_cleanup +# -------------------- +# Enables automatic container cleanup after tests. +function ct_enable_cleanup() { + trap ct_cleanup EXIT SIGINT +} + +# ct_get_cid [name] +# -------------------- +# Prints container id from cid_file based on the name of the file. +# Argument: name - name of cid_file where the container id will be stored +# Uses: $CID_FILE_DIR - path to directory containing cid_files +function ct_get_cid() { + local name="$1" ; shift || return 1 + cat "$CID_FILE_DIR/$name" +} + +# ct_get_cip [id] +# -------------------- +# Prints container ip address based on the container id. +# Argument: id - container id +function ct_get_cip() { + local id="$1" ; shift + docker inspect --format='{{.NetworkSettings.IPAddress}}' "$(ct_get_cid "$id")" +} + +# ct_wait_for_cid [cid_file] +# -------------------- +# Holds the execution until the cid_file is created. Usually run after container +# creation. +# Argument: cid_file - name of the cid_file that should be created +function ct_wait_for_cid() { + local cid_file=$1 + local max_attempts=10 + local sleep_time=1 + local attempt=1 + local result=1 + while [ $attempt -le $max_attempts ]; do + [ -f "$cid_file" ] && [ -s "$cid_file" ] && return 0 + : "Waiting for container start..." + attempt=$(( attempt + 1 )) + sleep $sleep_time + done + return 1 +} + +# ct_assert_container_creation_fails [container_args] +# -------------------- +# The invocation of docker run should fail based on invalid container_args +# passed to the function. Returns 0 when container fails to start properly. +# Argument: container_args - all arguments are passed directly to dokcer run +# Uses: $CID_FILE_DIR - path to directory containing cid_files +function ct_assert_container_creation_fails() { + local ret=0 + local max_attempts=10 + local attempt=1 + local cid_file=assert + set +e + local old_container_args="${CONTAINER_ARGS-}" + # we really work with CONTAINER_ARGS as with a string + # shellcheck disable=SC2124 + CONTAINER_ARGS="$@" + if ct_create_container "$cid_file" ; then + local cid + cid=$(ct_get_cid "$cid_file") + + while [ "$(docker inspect -f '{{.State.Running}}' "$cid")" == "true" ] ; do + sleep 2 + attempt=$(( attempt + 1 )) + if [ "$attempt" -gt "$max_attempts" ]; then + docker stop "$cid" + ret=1 + break + fi + done + exit_status=$(docker inspect -f '{{.State.ExitCode}}' "$cid") + if [ "$exit_status" == "0" ]; then + ret=1 + fi + docker rm -v "$cid" + rm "$CID_FILE_DIR/$cid_file" + fi + [ -n "$old_container_args" ] && CONTAINER_ARGS="$old_container_args" + set -e + return "$ret" +} + +# ct_create_container [name, command] +# -------------------- +# Creates a container using the IMAGE_NAME and CONTAINER_ARGS variables. Also +# stores the container id to a cid_file located in the CID_FILE_DIR, and waits +# for the creation of the file. +# Argument: name - name of cid_file where the container id will be stored +# Argument: command - optional command to be executed in the container +# Uses: $CID_FILE_DIR - path to directory containing cid_files +# Uses: $CONTAINER_ARGS - optional arguments passed directly to docker run +# Uses: $IMAGE_NAME - name of the image being tested +function ct_create_container() { + local cid_file="$CID_FILE_DIR/$1" ; shift + # create container with a cidfile in a directory for cleanup + # shellcheck disable=SC2086 + docker run --cidfile="$cid_file" -d ${CONTAINER_ARGS:-} "$IMAGE_NAME" "$@" + ct_wait_for_cid "$cid_file" || return 1 + : "Created container $(cat "$cid_file")" +} + +# ct_scl_usage_old [name, command, expected] +# -------------------- +# Tests three ways of running the SCL, by looking for an expected string +# in the output of the command +# Argument: name - name of cid_file where the container id will be stored +# Argument: command - executed inside the container +# Argument: expected - string that is expected to be in the command output +# Uses: $CID_FILE_DIR - path to directory containing cid_files +# Uses: $IMAGE_NAME - name of the image being tested +function ct_scl_usage_old() { + local name="$1" + local command="$2" + local expected="$3" + local out="" + : " Testing the image SCL enable" + out=$(docker run --rm "${IMAGE_NAME}" /bin/bash -c "${command}") + if ! echo "${out}" | grep -q "${expected}"; then + echo "ERROR[/bin/bash -c \"${command}\"] Expected '${expected}', got '${out}'" >&2 + return 1 + fi + out=$(docker exec "$(ct_get_cid "$name")" /bin/bash -c "${command}" 2>&1) + if ! echo "${out}" | grep -q "${expected}"; then + echo "ERROR[exec /bin/bash -c \"${command}\"] Expected '${expected}', got '${out}'" >&2 + return 1 + fi + out=$(docker exec "$(ct_get_cid "$name")" /bin/sh -ic "${command}" 2>&1) + if ! echo "${out}" | grep -q "${expected}"; then + echo "ERROR[exec /bin/sh -ic \"${command}\"] Expected '${expected}', got '${out}'" >&2 + return 1 + fi +} + +# ct_doc_content_old [strings] +# -------------------- +# Looks for occurence of stirngs in the documentation files and checks +# the format of the files. Files examined: help.1 +# Argument: strings - strings expected to appear in the documentation +# Uses: $IMAGE_NAME - name of the image being tested +function ct_doc_content_old() { + local tmpdir + tmpdir=$(mktemp -d) + local f + : " Testing documentation in the container image" + # Extract the help files from the container + # shellcheck disable=SC2043 + for f in help.1 ; do + docker run --rm "${IMAGE_NAME}" /bin/bash -c "cat /${f}" >"${tmpdir}/$(basename "${f}")" + # Check whether the files contain some important information + for term in "$@" ; do + if ! grep -F -q -e "${term}" "${tmpdir}/$(basename "${f}")" ; then + echo "ERROR: File /${f} does not include '${term}'." >&2 + return 1 + fi + done + # Check whether the files use the correct format + for term in TH PP SH ; do + if ! grep -q "^\.${term}" "${tmpdir}/help.1" ; then + echo "ERROR: /help.1 is probably not in troff or groff format, since '${term}' is missing." >&2 + return 1 + fi + done + done + : " Success!" +} + +# full_ca_file_path +# Return string for full path to CA file +function full_ca_file_path() +{ + echo "/etc/pki/ca-trust/source/anchors/RH-IT-Root-CA.crt" +} +# ct_mount_ca_file +# ------------------ +# Check if /etc/pki/certs/RH-IT-Root-CA.crt file exists +# return mount string for containers or empty string +function ct_mount_ca_file() +{ + # mount CA file only if NPM_REGISTRY variable is present. + local mount_parameter="" + if [ -n "$NPM_REGISTRY" ] && [ -f "$(full_ca_file_path)" ]; then + mount_parameter="-v $(full_ca_file_path):$(full_ca_file_path):Z" + fi + echo "$mount_parameter" +} + +# ct_build_s2i_npm_variables URL_TO_NPM_JS_SERVER +# ------------------------------------------ +# Function returns -e NPM_MIRROR and -v MOUNT_POINT_FOR_CAFILE +# or empty string +function ct_build_s2i_npm_variables() +{ + npm_variables="" + if [ -n "$NPM_REGISTRY" ] && [ -f "$(full_ca_file_path)" ]; then + npm_variables="-e NPM_MIRROR=$NPM_REGISTRY $(ct_mount_ca_file)" + fi + echo "$npm_variables" +} + +# ct_npm_works +# -------------------- +# Checks existance of the npm tool and runs it. +function ct_npm_works() { + local tmpdir + tmpdir=$(mktemp -d) + : " Testing npm in the container image" + local cid_file="${tmpdir}/cid" + if ! docker run --rm "${IMAGE_NAME}" /bin/bash -c "npm --version" >"${tmpdir}/version" ; then + echo "ERROR: 'npm --version' does not work inside the image ${IMAGE_NAME}." >&2 + return 1 + fi + + # shellcheck disable=SC2046 + docker run -d $(ct_mount_ca_file) --rm --cidfile="$cid_file" "${IMAGE_NAME}-testapp" + + # Wait for the container to write it's CID file + ct_wait_for_cid "$cid_file" || return 1 + + if ! docker exec "$(cat "$cid_file")" /bin/bash -c "npm --verbose install jquery && test -f node_modules/jquery/src/jquery.js" >"${tmpdir}/jquery" 2>&1 ; then + echo "ERROR: npm could not install jquery inside the image ${IMAGE_NAME}." >&2 + return 1 + fi + + if [ -n "$NPM_REGISTRY" ] && [ -f "$(full_ca_file_path)" ]; then + if ! grep -qo "$NPM_REGISTRY" "${tmpdir}/jquery"; then + echo "ERROR: Internal repository is NOT set. Even it is requested." + return 1 + fi + fi + + if [ -f "$cid_file" ]; then + docker stop "$(cat "$cid_file")" + rm "$cid_file" + fi + : " Success!" +} + +# ct_path_append PATH_VARNAME DIRECTORY +# ------------------------------------- +# Append DIRECTORY to VARIABLE of name PATH_VARNAME, the VARIABLE must consist +# of colon-separated list of directories. +ct_path_append () +{ + if eval "test -n \"\${$1-}\""; then + eval "$1=\$2:\$$1" + else + eval "$1=\$2" + fi +} + + +# ct_path_foreach PATH ACTION [ARGS ...] +# -------------------------------------- +# For each DIR in PATH execute ACTION (path is colon separated list of +# directories). The particular calls to ACTION will look like +# '$ ACTION directory [ARGS ...]' +ct_path_foreach () +{ + local dir dirlist action save_IFS + save_IFS=$IFS + IFS=: + dirlist=$1 + action=$2 + shift 2 + for dir in $dirlist; do "$action" "$dir" "$@" ; done + IFS=$save_IFS +} + + +# ct_run_test_list +# -------------------- +# Execute the tests specified by TEST_LIST +# Uses: $TEST_LIST - list of test names +function ct_run_test_list() { + for test_case in $TEST_LIST; do + : "Running test $test_case" + # shellcheck source=/dev/null + [ -f "test/$test_case" ] && source "test/$test_case" + # shellcheck source=/dev/null + [ -f "../test/$test_case" ] && source "../test/$test_case" + $test_case + done; +} + +# ct_gen_self_signed_cert_pem +# --------------------------- +# Generates a self-signed PEM certificate pair into specified directory. +# Argument: output_dir - output directory path +# Argument: base_name - base name of the certificate files +# Resulted files will be those: +# /-cert-selfsigned.pem -- public PEM cert +# /-key.pem -- PEM private key +ct_gen_self_signed_cert_pem() { + local output_dir=$1 ; shift + local base_name=$1 ; shift + mkdir -p "${output_dir}" + openssl req -newkey rsa:2048 -nodes -keyout "${output_dir}"/"${base_name}"-key.pem -subj '/C=GB/ST=Berkshire/L=Newbury/O=My Server Company' > "${base_name}"-req.pem + openssl req -new -x509 -nodes -key "${output_dir}"/"${base_name}"-key.pem -batch > "${output_dir}"/"${base_name}"-cert-selfsigned.pem +} + +# ct_obtain_input FILE|DIR|URL +# -------------------- +# Either copies a file or a directory to a tmp location for local copies, or +# downloads the file from remote location. +# Resulted file path is printed, so it can be later used by calling function. +# Arguments: input - local file, directory or remote URL +function ct_obtain_input() { + local input=$1 + local extension="${input##*.}" + + # Try to use same extension for the temporary file if possible + [[ "${extension}" =~ ^[a-z0-9]*$ ]] && extension=".${extension}" || extension="" + + local output + output=$(mktemp "/var/tmp/test-input-XXXXXX$extension") + if [ -f "${input}" ] ; then + cp -f "${input}" "${output}" + elif [ -d "${input}" ] ; then + rm -f "${output}" + cp -r -LH "${input}" "${output}" + elif echo "${input}" | grep -qe '^http\(s\)\?://' ; then + curl "${input}" > "${output}" + else + echo "ERROR: file type not known: ${input}" >&2 + return 1 + fi + echo "${output}" +} + +# ct_test_response +# ---------------- +# Perform GET request to the application container, checks output with +# a reg-exp and HTTP response code. +# Argument: url - request URL path +# Argument: expected_code - expected HTTP response code +# Argument: body_regexp - PCRE regular expression that must match the response body +# Argument: max_attempts - Optional number of attempts (default: 20), three seconds sleep between +# Argument: ignore_error_attempts - Optional number of attempts when we ignore error output (default: 10) +ct_test_response() { + local url="$1" + local expected_code="$2" + local body_regexp="$3" + local max_attempts=${4:-20} + local ignore_error_attempts=${5:-10} + + : " Testing the HTTP(S) response for <${url}>" + local sleep_time=3 + local attempt=1 + local result=1 + local status + local response_code + local response_file + response_file=$(mktemp /tmp/ct_test_response_XXXXXX) + while [ "${attempt}" -le "${max_attempts}" ]; do + curl --connect-timeout 10 -s -w '%{http_code}' "${url}" >"${response_file}" && status=0 || status=1 + if [ "${status}" -eq 0 ]; then + response_code=$(tail -c 3 "${response_file}") + if [ "${response_code}" -eq "${expected_code}" ]; then + result=0 + fi + grep -qP -e "${body_regexp}" "${response_file}" || result=1; + # Some services return 40x code until they are ready, so let's give them + # some chance and not end with failure right away + # Do not wait if we already have expected outcome though + if [ "${result}" -eq 0 ] || [ "${attempt}" -gt "${ignore_error_attempts}" ] || [ "${attempt}" -eq "${max_attempts}" ] ; then + break + fi + fi + attempt=$(( attempt + 1 )) + sleep "${sleep_time}" + done + rm -f "${response_file}" + return "${result}" +} + +# ct_registry_from_os OS +# ---------------- +# Transform operating system string [os] into registry url +# Argument: OS - string containing the os version +ct_registry_from_os() { + local registry="" + case $1 in + rhel*) + registry=registry.redhat.io + ;; + *) + registry=docker.io + ;; + esac + echo "$registry" +} + + # ct_get_public_image_name OS BASE_IMAGE_NAME VERSION +# ---------------- +# Transform the arguments into public image name +# Argument: OS - string containing the os version +# Argument: BASE_IMAGE_NAME - string containing the base name of the image as defined in the Makefile +# Argument: VERSION - string containing the version of the image as defined in the Makefile +ct_get_public_image_name() { + local os=$1; shift + local base_image_name=$1; shift + local version=$1; shift + + local public_image_name + local registry + + registry=$(ct_registry_from_os "$os") + if [ "x$os" == "xrhel7" ]; then + public_image_name=$registry/rhscl/$base_image_name-${version//./}-rhel7 + elif [ "x$os" == "xrhel8" ]; then + public_image_name=$registry/rhel8/$base_image_name-${version//./} + elif [ "x$os" == "xcentos7" ]; then + public_image_name=$registry/centos/$base_image_name-${version//./}-centos7 + fi + + echo "$public_image_name" +} + +# ct_assert_cmd_success CMD +# ---------------- +# Evaluates [cmd] and fails if it does not succeed. +# Argument: CMD - Command to be run +function ct_assert_cmd_success() { + echo "Checking '$*' for success ..." + if ! eval "$@" &>/dev/null; then + echo " FAIL" + return 1 + fi + echo " PASS" + return 0 +} + +# ct_assert_cmd_failure CMD +# ---------------- +# Evaluates [cmd] and fails if it succeeds. +# Argument: CMD - Command to be run +function ct_assert_cmd_failure() { + echo "Checking '$*' for failure ..." + if eval "$@" &>/dev/null; then + echo " FAIL" + return 1 + fi + echo " PASS" + return 0 +} + + +# ct_random_string [LENGTH=10] +# ---------------------------- +# Generate pseudorandom alphanumeric string of LENGTH bytes, the +# default length is 10. The string is printed on stdout. +ct_random_string() +( + export LC_ALL=C + dd if=/dev/urandom count=1 bs=10k 2>/dev/null \ + | tr -dc 'a-z0-9' \ + | fold -w "${1-10}" \ + | head -n 1 +) + +# ct_s2i_usage IMG_NAME [S2I_ARGS] +# ---------------------------- +# Create a container and run the usage script inside +# Argument: IMG_NAME - name of the image to be used for the container run +# Argument: S2I_ARGS - Additional list of source-to-image arguments, currently unused. +ct_s2i_usage() +{ + local img_name=$1; shift + local s2i_args="$*"; + local usage_command="/usr/libexec/s2i/usage" + docker run --rm "$img_name" bash -c "$usage_command" +} + +# ct_s2i_build_as_df APP_PATH SRC_IMAGE DST_IMAGE [S2I_ARGS] +# ---------------------------- +# Create a new s2i app image from local sources in a similar way as source-to-image would have used. +# Argument: APP_PATH - local path to the app sources to be used in the test +# Argument: SRC_IMAGE - image to be used as a base for the s2i build +# Argument: DST_IMAGE - image name to be used during the tagging of the s2i build result +# Argument: S2I_ARGS - Additional list of source-to-image arguments. +# Only used to check for pull-policy=never and environment variable definitions. +ct_s2i_build_as_df() +{ + local app_path=$1; shift + local src_image=$1; shift + local dst_image=$1; shift + local s2i_args="$*"; + local local_app=upload/src/ + local local_scripts=upload/scripts/ + local user_id= + local df_name= + local tmpdir= + local incremental=false + local mount_options="" + + # Run the entire thing inside a subshell so that we do not leak shell options outside of the function + ( + # Error out if any part of the build fails + set -e + + # Use /tmp to not pollute cwd + tmpdir=$(mktemp -d) + df_name=$(mktemp -p "$tmpdir" Dockerfile.XXXX) + cd "$tmpdir" + # Check if the image is available locally and try to pull it if it is not + docker images "$src_image" &>/dev/null || echo "$s2i_args" | grep -q "pull-policy=never" || docker pull "$src_image" + user=$(docker inspect -f "{{.Config.User}}" "$src_image") + # Default to root if no user is set by the image + user=${user:-0} + # run the user through the image in case it is non-numeric or does not exist + # NOTE: The '-eq' test is used to check if $user is numeric as it will fail if $user is not an integer + if ! [ "$user" -eq "$user" ] 2>/dev/null && ! user_id=$(docker run --rm "$src_image" bash -c "id -u $user 2>/dev/null"); then + echo "ERROR: id of user $user not found inside image $src_image." + echo "Terminating s2i build." + return 1 + else + user_id=${user_id:-$user} + fi + echo "$s2i_args" | grep -q "\-\-incremental" && incremental=true + if $incremental; then + inc_tmp=$(mktemp -d --tmpdir incremental.XXXX) + setfacl -m "u:$user_id:rwx" "$inc_tmp" + # Check if the image exists, build should fail (for testing use case) if it does not + docker images "$dst_image" &>/dev/null || (echo "Image $dst_image not found."; false) + # Run the original image with a mounted in volume and get the artifacts out of it + cmd="if [ -s /usr/libexec/s2i/save-artifacts ]; then /usr/libexec/s2i/save-artifacts > \"$inc_tmp/artifacts.tar\"; else touch \"$inc_tmp/artifacts.tar\"; fi" + docker run --rm -v "$inc_tmp:$inc_tmp:Z" "$dst_image" bash -c "$cmd" + # Move the created content into the $tmpdir for the build to pick it up + mv "$inc_tmp/artifacts.tar" "$tmpdir/" + fi + # Strip file:// from APP_PATH and copy its contents into current context + mkdir -p "$local_app" + cp -r "${app_path/file:\/\//}/." "$local_app" + [ -d "$local_app/.s2i/bin/" ] && mv "$local_app/.s2i/bin" "$local_scripts" + # Create a Dockerfile named df_name and fill it with proper content + #FIXME: Some commands could be combined into a single layer but not sure if worth the trouble for testing purposes + cat <"$df_name" +FROM $src_image +LABEL "io.openshift.s2i.build.image"="$src_image" \\ + "io.openshift.s2i.build.source-location"="$app_path" +USER root +COPY $local_app /tmp/src +EOF + [ -d "$local_scripts" ] && echo "COPY $local_scripts /tmp/scripts" >> "$df_name" && + echo "RUN chown -R $user_id:0 /tmp/scripts" >>"$df_name" + echo "RUN chown -R $user_id:0 /tmp/src" >>"$df_name" + # Check for custom environment variables inside .s2i/ folder + if [ -e "$local_app/.s2i/environment" ]; then + # Remove any comments and add the contents as ENV commands to the Dockerfile + sed '/^\s*#.*$/d' "$local_app/.s2i/environment" | while read -r line; do + echo "ENV $line" >>"$df_name" + done + fi + # Filter out env var definitions from $s2i_args and create Dockerfile ENV commands out of them + echo "$s2i_args" | grep -o -e '\(-e\|--env\)[[:space:]=]\S*=\S*' | sed -e 's/-e /ENV /' -e 's/--env[ =]/ENV /' >>"$df_name" + # Check if CA autority is present on host and add it into Dockerfile + [ -f "$(full_ca_file_path)" ] && echo "RUN cd /etc/pki/ca-trust/source/anchors && update-ca-trust extract" >>"$df_name" + + # Add in artifacts if doing an incremental build + if $incremental; then + { echo "RUN mkdir /tmp/artifacts" + echo "ADD artifacts.tar /tmp/artifacts" + echo "RUN chown -R $user_id:0 /tmp/artifacts" ; } >>"$df_name" + fi + + echo "USER $user_id" >>"$df_name" + # If exists, run the custom assemble script, else default to /usr/libexec/s2i/assemble + if [ -x "$local_scripts/assemble" ]; then + echo "RUN /tmp/scripts/assemble" >>"$df_name" + else + echo "RUN /usr/libexec/s2i/assemble" >>"$df_name" + fi + # If exists, set the custom run script as CMD, else default to /usr/libexec/s2i/run + if [ -x "$local_scripts/run" ]; then + echo "CMD /tmp/scripts/run" >>"$df_name" + else + echo "CMD /usr/libexec/s2i/run" >>"$df_name" + fi + + # Check if -v parameter is present in s2i_args and add it into docker build command + mount_options=$(echo "$s2i_args" | grep -o -e '\(-v\)[[:space:]]\.*\S*' || true) + + # Run the build and tag the result + # shellcheck disable=SC2086 + docker build $mount_options -f "$df_name" --no-cache=true -t "$dst_image" . + ) +} + +# ct_check_image_availability PUBLIC_IMAGE_NAME +# ---------------------------- +# Pull an image from the public repositories to see if the image is already available. +# Argument: PUBLIC_IMAGE_NAME - string containing the public name of the image to pull +ct_check_image_availability() { + local public_image_name=$1; + + # Try pulling the image to see if it is accessible + if ! docker pull "$public_image_name" &>/dev/null; then + echo "$public_image_name could not be downloaded via 'docker'" + return 1 + fi +} + +# vim: set tabstop=2:shiftwidth=2:expandtab: From 7cc8795d0cb19bc2dac2969a16d4e9257594db0d Mon Sep 17 00:00:00 2001 From: Michal Schorm Date: Tue, 10 Mar 2020 10:29:11 +0100 Subject: [PATCH 31/32] Update for F31 base image --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 01f4a92..75c259c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM registry.fedoraproject.org/f30/s2i-core:latest +FROM registry.fedoraproject.org/f31/s2i-core:latest # MariaDB image for OpenShift. # From 958f1b7b2d38ea6181f23dfe951796e48583d877 Mon Sep 17 00:00:00 2001 From: Honza Horak Date: Tue, 8 Jul 2025 17:28:06 +0200 Subject: [PATCH 32/32] sources not used anymore for building fedora containers --- .gitignore | 0 10.3 | 1 - Dockerfile | 72 -- Dockerfile.fedora | 1 - README.md | 1 - dead.package | 1 + help.md | 1 - root-common/etc/my.cnf | 15 - root-common/usr/bin/container-entrypoint | 2 - root-common/usr/bin/mysqld-master | 1 - root-common/usr/bin/mysqld-slave | 1 - root-common/usr/bin/run-mysqld | 31 - root-common/usr/bin/run-mysqld-master | 50 - root-common/usr/bin/run-mysqld-slave | 60 - root-common/usr/bin/usage | 4 - root-common/usr/libexec/container-setup | 59 - root-common/usr/libexec/fix-permissions | 6 - .../container-scripts/mysql/cnf/40-paas.cnf | 30 - .../mysql/cnf/50-my-tuning.cnf | 27 - .../share/container-scripts/mysql/common.sh | 285 ----- .../share/container-scripts/mysql/helpers.sh | 31 - .../mysql/init/40-datadir-action.sh | 112 -- .../mysql/init/50-passwd-change.sh | 49 - .../container-scripts/mysql/post-init.sh | 0 .../mysql/pre-init/20-validate-variables.sh | 81 -- .../25-validate-replication-variables.sh | 21 - .../mysql/pre-init/30-base-config.sh | 3 - .../mysql/pre-init/60-replication-config.sh | 17 - .../mysql/pre-init/70-s2i-config.sh | 6 - .../mysql/pre-init/my-base.cnf.template | 5 - .../mysql/pre-init/my-master.cnf.template | 7 - .../mysql/pre-init/my-repl-gtid.cnf.template | 4 - .../mysql/pre-init/my-slave.cnf.template | 7 - .../share/container-scripts/mysql/scl_enable | 3 - root/help.1 | 475 -------- .../share/container-scripts/mysql/README.md | 359 ------ s2i-common/bin/assemble | 13 - s2i-common/bin/run | 1 - s2i-common/bin/usage | 8 - test/mariadb-ephemeral-template.json | 254 ---- test/run | 631 ---------- test/run-openshift | 1 - test/run-openshift-local-cluster | 54 - test/run-openshift-remote-cluster | 30 - test/test-app/mysql-cfg/myconfig.cnf | 3 - test/test-app/mysql-data/init.sql | 4 - .../mysql-init/80-add-arbitrary-users.sh | 17 - test/test-app/mysql-init/90-init-db.sh | 12 - .../80-check-arbitrary-users.sh | 10 - test/test-lib-mysql.sh | 131 --- test/test-lib-openshift.sh | 1043 ----------------- test/test-lib.sh | 653 ----------- 52 files changed, 1 insertion(+), 4692 deletions(-) delete mode 100644 .gitignore delete mode 120000 10.3 delete mode 100644 Dockerfile delete mode 120000 Dockerfile.fedora delete mode 120000 README.md create mode 100644 dead.package delete mode 120000 help.md delete mode 100644 root-common/etc/my.cnf delete mode 100755 root-common/usr/bin/container-entrypoint delete mode 120000 root-common/usr/bin/mysqld-master delete mode 120000 root-common/usr/bin/mysqld-slave delete mode 100755 root-common/usr/bin/run-mysqld delete mode 100755 root-common/usr/bin/run-mysqld-master delete mode 100755 root-common/usr/bin/run-mysqld-slave delete mode 100755 root-common/usr/bin/usage delete mode 100755 root-common/usr/libexec/container-setup delete mode 100755 root-common/usr/libexec/fix-permissions delete mode 100644 root-common/usr/share/container-scripts/mysql/cnf/40-paas.cnf delete mode 100644 root-common/usr/share/container-scripts/mysql/cnf/50-my-tuning.cnf delete mode 100644 root-common/usr/share/container-scripts/mysql/common.sh delete mode 100644 root-common/usr/share/container-scripts/mysql/helpers.sh delete mode 100644 root-common/usr/share/container-scripts/mysql/init/40-datadir-action.sh delete mode 100644 root-common/usr/share/container-scripts/mysql/init/50-passwd-change.sh delete mode 100644 root-common/usr/share/container-scripts/mysql/post-init.sh delete mode 100644 root-common/usr/share/container-scripts/mysql/pre-init/20-validate-variables.sh delete mode 100644 root-common/usr/share/container-scripts/mysql/pre-init/25-validate-replication-variables.sh delete mode 100644 root-common/usr/share/container-scripts/mysql/pre-init/30-base-config.sh delete mode 100644 root-common/usr/share/container-scripts/mysql/pre-init/60-replication-config.sh delete mode 100644 root-common/usr/share/container-scripts/mysql/pre-init/70-s2i-config.sh delete mode 100644 root-common/usr/share/container-scripts/mysql/pre-init/my-base.cnf.template delete mode 100644 root-common/usr/share/container-scripts/mysql/pre-init/my-master.cnf.template delete mode 100644 root-common/usr/share/container-scripts/mysql/pre-init/my-repl-gtid.cnf.template delete mode 100644 root-common/usr/share/container-scripts/mysql/pre-init/my-slave.cnf.template delete mode 100644 root-common/usr/share/container-scripts/mysql/scl_enable delete mode 100644 root/help.1 delete mode 100644 root/usr/share/container-scripts/mysql/README.md delete mode 100755 s2i-common/bin/assemble delete mode 120000 s2i-common/bin/run delete mode 100755 s2i-common/bin/usage delete mode 100644 test/mariadb-ephemeral-template.json delete mode 100755 test/run delete mode 120000 test/run-openshift delete mode 100755 test/run-openshift-local-cluster delete mode 100755 test/run-openshift-remote-cluster delete mode 100644 test/test-app/mysql-cfg/myconfig.cnf delete mode 100644 test/test-app/mysql-data/init.sql delete mode 100644 test/test-app/mysql-init/80-add-arbitrary-users.sh delete mode 100644 test/test-app/mysql-init/90-init-db.sh delete mode 100644 test/test-app/mysql-pre-init/80-check-arbitrary-users.sh delete mode 100755 test/test-lib-mysql.sh delete mode 100644 test/test-lib-openshift.sh delete mode 100644 test/test-lib.sh diff --git a/.gitignore b/.gitignore deleted file mode 100644 index e69de29..0000000 diff --git a/10.3 b/10.3 deleted file mode 120000 index 945c9b4..0000000 --- a/10.3 +++ /dev/null @@ -1 +0,0 @@ -. \ No newline at end of file diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 75c259c..0000000 --- a/Dockerfile +++ /dev/null @@ -1,72 +0,0 @@ -FROM registry.fedoraproject.org/f31/s2i-core:latest - -# MariaDB image for OpenShift. -# -# Volumes: -# * /var/lib/mysql/data - Datastore for MariaDB -# Environment: -# * $MYSQL_USER - Database user name -# * $MYSQL_PASSWORD - User's password -# * $MYSQL_DATABASE - Name of the database to create -# * $MYSQL_ROOT_PASSWORD (Optional) - Password for the 'root' MySQL account - -ENV MYSQL_VERSION=10.3 \ - APP_DATA=/opt/app-root/src \ - HOME=/var/lib/mysql \ - NAME=mariadb \ - VERSION=10.3 \ - ARCH=x86_64 \ - SUMMARY="MariaDB 10.3 SQL database server" \ - DESCRIPTION="MariaDB is a multi-user, multi-threaded SQL database server. The container \ -image provides a containerized packaging of the MariaDB mysqld daemon and client application. \ -The mysqld server daemon accepts connections from clients and provides access to content from \ -MariaDB databases on behalf of the clients." - -LABEL summary="$SUMMARY" \ - description="$DESCRIPTION" \ - io.k8s.description="MariaDB is a multi-user, multi-threaded SQL database server" \ - io.k8s.display-name="MariaDB 10.3" \ - io.openshift.expose-services="3306:mysql" \ - io.openshift.tags="database,mysql,mariadb,mariadb103,galera" \ - com.redhat.component="$NAME" \ - name="$FGC/$NAME" \ - version="$VERSION" \ - usage="docker run -d -e MYSQL_USER=user -e MYSQL_PASSWORD=pass -e MYSQL_DATABASE=db -p 3306:3306 $FGC/$NAME" \ - maintainer="SoftwareCollections.org " - -EXPOSE 3306 - -# This image must forever use UID 27 for mysql user so our volumes are -# safe in the future. This should *never* change, the last test is there -# to make sure of that. -RUN INSTALL_PKGS="rsync tar gettext hostname bind-utils groff-base mariadb mariadb-server policycoreutils" && \ - dnf install -y --setopt=tsflags=nodocs $INSTALL_PKGS && \ - rpm -V $INSTALL_PKGS && \ - dnf clean all && \ - mkdir -p /var/lib/mysql/data && chown -R mysql.0 /var/lib/mysql && \ - test "$(id mysql)" = "uid=27(mysql) gid=27(mysql) groups=27(mysql)" - -# Get prefix path and path to scripts rather than hard-code them in scripts -ENV CONTAINER_SCRIPTS_PATH=/usr/share/container-scripts/mysql \ - MYSQL_PREFIX=/usr - -COPY 10.3/root-common / -COPY 10.3/s2i-common/bin/ $STI_SCRIPTS_PATH -COPY 10.3/root / - -# this is needed due to issues with squash -# when this directory gets rm'd by the container-setup -# script. -# Also reset permissions of filesystem to default values -RUN rm -rf /etc/my.cnf.d/* && \ - /usr/libexec/container-setup && \ - rpm-file-permissions - -# Not using VOLUME statement since it's not working in OpenShift Online: -# https://github.com/sclorg/httpd-container/issues/30 -# VOLUME ["/var/lib/mysql/data"] - -USER 27 - -ENTRYPOINT ["container-entrypoint"] -CMD ["run-mysqld"] diff --git a/Dockerfile.fedora b/Dockerfile.fedora deleted file mode 120000 index 1d1fe94..0000000 --- a/Dockerfile.fedora +++ /dev/null @@ -1 +0,0 @@ -Dockerfile \ No newline at end of file diff --git a/README.md b/README.md deleted file mode 120000 index cc942f0..0000000 --- a/README.md +++ /dev/null @@ -1 +0,0 @@ -root/usr/share/container-scripts/mysql/README.md \ No newline at end of file diff --git a/dead.package b/dead.package new file mode 100644 index 0000000..785cb0a --- /dev/null +++ b/dead.package @@ -0,0 +1 @@ +sources not used anymore for building fedora containers diff --git a/help.md b/help.md deleted file mode 120000 index 42061c0..0000000 --- a/help.md +++ /dev/null @@ -1 +0,0 @@ -README.md \ No newline at end of file diff --git a/root-common/etc/my.cnf b/root-common/etc/my.cnf deleted file mode 100644 index bfdfbe9..0000000 --- a/root-common/etc/my.cnf +++ /dev/null @@ -1,15 +0,0 @@ -[mysqld] - -# Disabling symbolic-links is recommended to prevent assorted security risks -symbolic-links = 0 - -# http://www.percona.com/blog/2008/05/31/dns-achilles-heel-mysql-installation/ -skip_name_resolve - -# http://www.chriscalender.com/ignoring-the-lostfound-directory-in-your-datadir/ -ignore-db-dir=lost+found - -# GlusterFS equivalent of 'lost+found' -ignore-db-dir=.trashcan - -!includedir /etc/my.cnf.d diff --git a/root-common/usr/bin/container-entrypoint b/root-common/usr/bin/container-entrypoint deleted file mode 100755 index 9d8ad4d..0000000 --- a/root-common/usr/bin/container-entrypoint +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -exec "$@" diff --git a/root-common/usr/bin/mysqld-master b/root-common/usr/bin/mysqld-master deleted file mode 120000 index 8a0786e..0000000 --- a/root-common/usr/bin/mysqld-master +++ /dev/null @@ -1 +0,0 @@ -run-mysqld-master \ No newline at end of file diff --git a/root-common/usr/bin/mysqld-slave b/root-common/usr/bin/mysqld-slave deleted file mode 120000 index dc0f58b..0000000 --- a/root-common/usr/bin/mysqld-slave +++ /dev/null @@ -1 +0,0 @@ -run-mysqld-slave \ No newline at end of file diff --git a/root-common/usr/bin/run-mysqld b/root-common/usr/bin/run-mysqld deleted file mode 100755 index 2aa2fa3..0000000 --- a/root-common/usr/bin/run-mysqld +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -export_vars=$(cgroup-limits); export $export_vars -source ${CONTAINER_SCRIPTS_PATH}/common.sh -set -eu -if [[ -v DEBUG_IGNORE_SCRIPT_FAILURES ]]; then - set +e -fi - -export_setting_variables - -log_volume_info $MYSQL_DATADIR - -# pre-init files -process_extending_files ${APP_DATA}/mysql-pre-init/ ${CONTAINER_SCRIPTS_PATH}/pre-init/ - -if [ ! -d "$MYSQL_DATADIR/mysql" ]; then - initialize_database "$@" -else - start_local_mysql "$@" -fi - -# init files -process_extending_files ${APP_DATA}/mysql-init/ ${CONTAINER_SCRIPTS_PATH}/init/ - -# Restart the MySQL server with public IP bindings -shutdown_local_mysql -unset_env_vars -log_volume_info $MYSQL_DATADIR -log_info 'Running final exec -- Only MySQL server logs after this point' -exec ${MYSQL_PREFIX}/libexec/mysqld --defaults-file=$MYSQL_DEFAULTS_FILE "$@" 2>&1 diff --git a/root-common/usr/bin/run-mysqld-master b/root-common/usr/bin/run-mysqld-master deleted file mode 100755 index 5550cb7..0000000 --- a/root-common/usr/bin/run-mysqld-master +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash -# -# This is an entrypoint that runs the MySQL server in the 'master' mode. -# - -export_vars=$(cgroup-limits); export $export_vars -source ${CONTAINER_SCRIPTS_PATH}/common.sh -set -eu -if [[ -v DEBUG_IGNORE_SCRIPT_FAILURES ]]; then - set +e -fi - -export_setting_variables - -log_volume_info $MYSQL_DATADIR - -export MYSQL_RUNNING_AS_MASTER=1 - -# The 'server-id' for master needs to be constant -export MYSQL_SERVER_ID=1 -log_info "The 'master' server-id is ${MYSQL_SERVER_ID}" - -# pre-init files -process_extending_files ${APP_DATA}/mysql-pre-init/ ${CONTAINER_SCRIPTS_PATH}/pre-init/ - -if [ ! -d "$MYSQL_DATADIR/mysql" ]; then - initialize_database "$@" -else - start_local_mysql "$@" -fi - -log_info 'Setting passwords ...' -[ -f ${CONTAINER_SCRIPTS_PATH}/passwd-change.sh ] && source ${CONTAINER_SCRIPTS_PATH}/passwd-change.sh - -# Setup the 'master' replication on the MySQL server -mysql $mysql_flags <&1 diff --git a/root-common/usr/bin/run-mysqld-slave b/root-common/usr/bin/run-mysqld-slave deleted file mode 100755 index 8a710ba..0000000 --- a/root-common/usr/bin/run-mysqld-slave +++ /dev/null @@ -1,60 +0,0 @@ -#!/bin/bash -# -# This is an entrypoint that runs the MySQL server in the 'slave' mode. -# - -export_vars=$(cgroup-limits); export $export_vars -source ${CONTAINER_SCRIPTS_PATH}/common.sh -set -eu -if [[ -v DEBUG_IGNORE_SCRIPT_FAILURES ]]; then - set +e -fi - -export_setting_variables - -log_volume_info $MYSQL_DATADIR - -export MYSQL_RUNNING_AS_SLAVE=1 - -# Generate the unique 'server-id' for this master -export MYSQL_SERVER_ID=$(server_id) -log_info "The 'slave' server-id is ${MYSQL_SERVER_ID}" - -# pre-init files -process_extending_files ${APP_DATA}/mysql-pre-init/ ${CONTAINER_SCRIPTS_PATH}/pre-init/ - -if [ ! -e "${MYSQL_DATADIR}/mysql" ]; then - # Initialize MySQL database and wait for the MySQL master to accept - # connections. - initialize_database "$@" - wait_for_mysql_master - - # Get binlog file and position from master - STATUS_INFO=$(mysql --host "$MYSQL_MASTER_SERVICE_NAME" "-u${MYSQL_MASTER_USER}" "-p${MYSQL_MASTER_PASSWORD}" replication -e 'SELECT gtid from replication limit 1\G') - GTID_VALUE=$(echo "$STATUS_INFO" | grep 'gtid:' | head -n 1 | sed -e 's/^\s*gtid: //') - - # checking STATUS_INFO here because empty GTID_VALUE is valid value - if [ -z "${STATUS_INFO}" ] ; then - echo "Could not read GTID value from master" - exit 1 - fi - - mysql $mysql_flags <&1 diff --git a/root-common/usr/bin/usage b/root-common/usr/bin/usage deleted file mode 100755 index feafb93..0000000 --- a/root-common/usr/bin/usage +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -cat /usr/share/container-scripts/mysql/README.md - diff --git a/root-common/usr/libexec/container-setup b/root-common/usr/libexec/container-setup deleted file mode 100755 index 6160d4e..0000000 --- a/root-common/usr/libexec/container-setup +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash - -# This function returns all config files that daemon uses and their path -# includes /opt. It is used to get correct path to the config file. -mysql_get_config_files_scl() { - scl enable ${ENABLED_COLLECTIONS} -- my_print_defaults --help --verbose | \ - grep --after=1 '^Default options' | \ - tail -n 1 | \ - grep -o '[^ ]*opt[^ ]*my.cnf' -} - -# This function picks the main config file that deamon uses and we ship in rpm -mysql_get_correct_config() { - # we use the same config in non-SCL packages, not necessary to guess - [ -z "${ENABLED_COLLECTIONS}" ] && echo -n "/etc/my.cnf" && return - - # from all config files read by daemon, pick the first that exists - for f in `mysql_get_config_files_scl` ; do - [ -f "$f" ] && echo "$f" - done | head -n 1 -} - -export MYSQL_CONFIG_FILE=$(mysql_get_correct_config) - -[ -z "$MYSQL_CONFIG_FILE" ] && echo "MYSQL_CONFIG_FILE is empty" && exit 1 - -unset -f mysql_get_correct_config mysql_get_config_files_scl - -# we provide own config files for the container, so clean what rpm ships here -mkdir -p ${MYSQL_CONFIG_FILE}.d -rm -f ${MYSQL_CONFIG_FILE}.d/* - -# we may add options during service init, so we need to have this dir writable by daemon user -chown -R mysql:0 ${MYSQL_CONFIG_FILE}.d ${MYSQL_CONFIG_FILE} -restorecon -R ${MYSQL_CONFIG_FILE}.d ${MYSQL_CONFIG_FILE} - -# API of the container are standard paths /etc/my.cnf and /etc/my.cnf.d -# we already include own /etc/my.cnf for container, but for cases the -# actually used config file is not on standard path /etc/my.cnf, we -# need to move it to the location daemon expects it and create symlinks -if [ "$MYSQL_CONFIG_FILE" != "/etc/my.cnf" ] ; then - rm -rf /etc/my.cnf.d - mv /etc/my.cnf ${MYSQL_CONFIG_FILE} - ln -s ${MYSQL_CONFIG_FILE} /etc/my.cnf - ln -s ${MYSQL_CONFIG_FILE}.d /etc/my.cnf.d -fi - -# setup directory for data -mkdir -p /var/lib/mysql/data -chown -R mysql:0 /var/lib/mysql -restorecon -R /var/lib/mysql - -# Loosen permission bits for group to avoid problems running container with -# arbitrary UID -# When only specifying user, group is 0, that's why /var/lib/mysql must have -# owner mysql.0; that allows to avoid a+rwx for this dir -/usr/libexec/fix-permissions /var/lib/mysql ${MYSQL_CONFIG_FILE}.d ${APP_DATA}/.. -usermod -a -G root mysql - diff --git a/root-common/usr/libexec/fix-permissions b/root-common/usr/libexec/fix-permissions deleted file mode 100755 index 820e718..0000000 --- a/root-common/usr/libexec/fix-permissions +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh -# Fix permissions on the given directory to allow group read/write of -# regular files and execute of directories. -find $@ -exec chown mysql:0 {} \; -find $@ -exec chmod g+rw {} \; -find $@ -type d -exec chmod g+x {} + diff --git a/root-common/usr/share/container-scripts/mysql/cnf/40-paas.cnf b/root-common/usr/share/container-scripts/mysql/cnf/40-paas.cnf deleted file mode 100644 index e79f2c5..0000000 --- a/root-common/usr/share/container-scripts/mysql/cnf/40-paas.cnf +++ /dev/null @@ -1,30 +0,0 @@ -[mysqld] -# -# Settings configured by the user -# - -# Sets how the table names are stored and compared. Default: 0 -lower_case_table_names = ${MYSQL_LOWER_CASE_TABLE_NAMES} - -# Sets whether queries should be logged -general_log = ${MYSQL_LOG_QUERIES_ENABLED} -general_log_file = ${MYSQL_DATADIR}/mysql-query.log - -# The maximum permitted number of simultaneous client connections. Default: 151 -max_connections = ${MYSQL_MAX_CONNECTIONS} - -# The minimum/maximum lengths of the word to be included in a FULLTEXT index. Default: 4/20 -ft_min_word_len = ${MYSQL_FT_MIN_WORD_LEN} -ft_max_word_len = ${MYSQL_FT_MAX_WORD_LEN} - -# In case the native AIO is broken. Default: 1 -# See http://help.directadmin.com/item.php?id=529 -innodb_use_native_aio = ${MYSQL_AIO} - -[myisamchk] -# The minimum/maximum lengths of the word to be included in a FULLTEXT index. Default: 4/20 -# -# To ensure that myisamchk and the server use the same values for full-text -# parameters, we placed them in both sections. -ft_min_word_len = ${MYSQL_FT_MIN_WORD_LEN} -ft_max_word_len = ${MYSQL_FT_MAX_WORD_LEN} diff --git a/root-common/usr/share/container-scripts/mysql/cnf/50-my-tuning.cnf b/root-common/usr/share/container-scripts/mysql/cnf/50-my-tuning.cnf deleted file mode 100644 index e6b33f4..0000000 --- a/root-common/usr/share/container-scripts/mysql/cnf/50-my-tuning.cnf +++ /dev/null @@ -1,27 +0,0 @@ -[mysqld] -key_buffer_size = ${MYSQL_KEY_BUFFER_SIZE} -max_allowed_packet = ${MYSQL_MAX_ALLOWED_PACKET} -table_open_cache = ${MYSQL_TABLE_OPEN_CACHE} -sort_buffer_size = ${MYSQL_SORT_BUFFER_SIZE} -read_buffer_size = ${MYSQL_READ_BUFFER_SIZE} -read_rnd_buffer_size = 256K -net_buffer_length = 2K -thread_stack = 256K -myisam_sort_buffer_size = 2M - -# It is recommended that innodb_buffer_pool_size is configured to 50 to 75 percent of system memory. -innodb_buffer_pool_size = ${MYSQL_INNODB_BUFFER_POOL_SIZE} -# Set .._log_file_size to 25 % of buffer pool size -innodb_log_file_size = ${MYSQL_INNODB_LOG_FILE_SIZE} -innodb_log_buffer_size = ${MYSQL_INNODB_LOG_BUFFER_SIZE} - -[mysqldump] -quick -max_allowed_packet = 16M - -[mysql] -no-auto-rehash - -[myisamchk] -key_buffer_size = 8M -sort_buffer_size = 8M diff --git a/root-common/usr/share/container-scripts/mysql/common.sh b/root-common/usr/share/container-scripts/mysql/common.sh deleted file mode 100644 index f214017..0000000 --- a/root-common/usr/share/container-scripts/mysql/common.sh +++ /dev/null @@ -1,285 +0,0 @@ -#!/bin/bash - -source ${CONTAINER_SCRIPTS_PATH}/helpers.sh - -# Data directory where MySQL database files live. The data subdirectory is here -# because .bashrc and my.cnf both live in /var/lib/mysql/ and we don't want a -# volume to override it. -export MYSQL_DATADIR=/var/lib/mysql/data - -# Configuration settings. -export MYSQL_DEFAULTS_FILE=${MYSQL_DEFAULTS_FILE:-/etc/my.cnf} - -function export_setting_variables() { - export MYSQL_BINLOG_FORMAT=${MYSQL_BINLOG_FORMAT:-STATEMENT} - export MYSQL_LOWER_CASE_TABLE_NAMES=${MYSQL_LOWER_CASE_TABLE_NAMES:-0} - export MYSQL_LOG_QUERIES_ENABLED=${MYSQL_LOG_QUERIES_ENABLED:-0} - export MYSQL_MAX_CONNECTIONS=${MYSQL_MAX_CONNECTIONS:-151} - export MYSQL_FT_MIN_WORD_LEN=${MYSQL_FT_MIN_WORD_LEN:-4} - export MYSQL_FT_MAX_WORD_LEN=${MYSQL_FT_MAX_WORD_LEN:-20} - export MYSQL_AIO=${MYSQL_AIO:-1} - export MYSQL_MAX_ALLOWED_PACKET=${MYSQL_MAX_ALLOWED_PACKET:-200M} - export MYSQL_TABLE_OPEN_CACHE=${MYSQL_TABLE_OPEN_CACHE:-400} - export MYSQL_SORT_BUFFER_SIZE=${MYSQL_SORT_BUFFER_SIZE:-256K} - - # Export memory limit variables and calculate limits - local export_vars=$(cgroup-limits) && export $export_vars || exit 1 - if [ -n "${NO_MEMORY_LIMIT:-}" -o -z "${MEMORY_LIMIT_IN_BYTES:-}" ]; then - export MYSQL_KEY_BUFFER_SIZE=${MYSQL_KEY_BUFFER_SIZE:-32M} - export MYSQL_READ_BUFFER_SIZE=${MYSQL_READ_BUFFER_SIZE:-8M} - export MYSQL_INNODB_BUFFER_POOL_SIZE=${MYSQL_INNODB_BUFFER_POOL_SIZE:-32M} - export MYSQL_INNODB_LOG_FILE_SIZE=${MYSQL_INNODB_LOG_FILE_SIZE:-8M} - export MYSQL_INNODB_LOG_BUFFER_SIZE=${MYSQL_INNODB_LOG_BUFFER_SIZE:-8M} - else - export MYSQL_KEY_BUFFER_SIZE=${MYSQL_KEY_BUFFER_SIZE:-$((MEMORY_LIMIT_IN_BYTES/1024/1024/10))M} - export MYSQL_READ_BUFFER_SIZE=${MYSQL_READ_BUFFER_SIZE:-$((MEMORY_LIMIT_IN_BYTES/1024/1024/20))M} - export MYSQL_INNODB_BUFFER_POOL_SIZE=${MYSQL_INNODB_BUFFER_POOL_SIZE:-$((MEMORY_LIMIT_IN_BYTES/1024/1024/2))M} - # We are multiplying by 15 first and dividing by 100 later so we get as much - # precision as possible with whole numbers. Result is 15% of memory. - export MYSQL_INNODB_LOG_FILE_SIZE=${MYSQL_INNODB_LOG_FILE_SIZE:-$((MEMORY_LIMIT_IN_BYTES*15/1024/1024/100))M} - export MYSQL_INNODB_LOG_BUFFER_SIZE=${MYSQL_INNODB_LOG_BUFFER_SIZE:-$((MEMORY_LIMIT_IN_BYTES*15/1024/1024/100))M} - fi - export MYSQL_DATADIR_ACTION=${MYSQL_DATADIR_ACTION:-upgrade-warn} -} - -# this stores whether the database was initialized from empty datadir -export MYSQL_DATADIR_FIRST_INIT=false - -# Be paranoid and stricter than we should be. -# https://dev.mysql.com/doc/refman/en/identifiers.html -mysql_identifier_regex='^[a-zA-Z0-9_]+$' -mysql_password_regex='^[a-zA-Z0-9_~!@#$%^&*()-=<>,.?;:|]+$' - -# Variables that are used to connect to local mysql during initialization -mysql_flags="-u root --socket=/tmp/mysql.sock" -admin_flags="--defaults-file=$MYSQL_DEFAULTS_FILE $mysql_flags" - -# Make sure env variables don't propagate to mysqld process. -function unset_env_vars() { - log_info 'Cleaning up environment variables MYSQL_USER, MYSQL_PASSWORD, MYSQL_DATABASE and MYSQL_ROOT_PASSWORD ...' - unset MYSQL_USER MYSQL_PASSWORD MYSQL_DATABASE MYSQL_ROOT_PASSWORD -} - -# Poll until MySQL responds to our ping. -function wait_for_mysql() { - pid=$1 ; shift - - while true; do - if [ -d "/proc/$pid" ]; then - mysqladmin $admin_flags ping &>/dev/null && log_info "MySQL started successfully" && return 0 - else - return 1 - fi - log_info "Waiting for MySQL to start ..." - sleep 1 - done -} - -# Start local MySQL server with a defaults file -function start_local_mysql() { - log_info 'Starting MySQL server with disabled networking ...' - ${MYSQL_PREFIX}/libexec/mysqld \ - --defaults-file=$MYSQL_DEFAULTS_FILE \ - --skip-networking --socket=/tmp/mysql.sock "$@" & - mysql_pid=$! - wait_for_mysql $mysql_pid -} - -# Shutdown mysql flushing privileges -function shutdown_local_mysql() { - log_info 'Shutting down MySQL ...' - mysqladmin $admin_flags flush-privileges shutdown -} - -# Initialize the MySQL database (create user accounts and the initial database) -function initialize_database() { - log_info 'Initializing database ...' - log_info 'Running mysql_install_db ...' - # Using --rpm since we need mysql_install_db behaves as in RPM - mysql_install_db --rpm --datadir=$MYSQL_DATADIR - start_local_mysql "$@" - - # Running mysql_upgrade creates the mysql_upgrade_info file in the data dir, - # which is necessary to detect which version of the mysqld daemon created the data. - # Checking empty file should not take longer than a second and one extra check should not harm. - mysql_upgrade ${admin_flags} - - if [ -v MYSQL_RUNNING_AS_SLAVE ]; then - log_info 'Initialization finished' - return 0 - fi - - if [ -v MYSQL_RUNNING_AS_MASTER ]; then - # Save master status into a separate database. - STATUS_INFO=$(mysql $admin_flags -e 'SHOW MASTER STATUS\G') - BINLOG_POSITION=$(echo "$STATUS_INFO" | grep 'Position:' | head -n 1 | sed -e 's/^\s*Position: //') - BINLOG_FILE=$(echo "$STATUS_INFO" | grep 'File:' | head -n 1 | sed -e 's/^\s*File: //') - GTID_INFO=$(mysql $admin_flags -e "SELECT BINLOG_GTID_POS('$BINLOG_FILE', '$BINLOG_POSITION') AS gtid_value \G") - GTID_VALUE=$(echo "$GTID_INFO" | grep 'gtid_value:' | head -n 1 | sed -e 's/^\s*gtid_value: //') - - mysqladmin $admin_flags create replication - mysql $admin_flags < "10.0" ] ; then -mysql $mysql_flags </dev/null && log_info "MySQL master is ready" && return 0 - sleep 1 - done -} - -# get_matched_files finds file for image extending -function get_matched_files() { - local custom_dir default_dir - custom_dir="$1" - default_dir="$2" - files_matched="$3" - find "$default_dir" -maxdepth 1 -type f -name "$files_matched" -printf "%f\n" - [ -d "$custom_dir" ] && find "$custom_dir" -maxdepth 1 -type f -name "$files_matched" -printf "%f\n" -} - -# process_extending_files process extending files in $1 and $2 directories -# - source all *.sh files -# (if there are files with same name source only file from $1) -function process_extending_files() { - local custom_dir default_dir - custom_dir=$1 - default_dir=$2 - - while read filename ; do - echo "=> sourcing $filename ..." - # Custom file is prefered - if [ -f $custom_dir/$filename ]; then - source $custom_dir/$filename - else - source $default_dir/$filename - fi - done <<<"$(get_matched_files "$custom_dir" "$default_dir" '*.sh' | sort -u)" -} - -# process extending config files in $1 and $2 directories -# - expand variables in *.cnf and copy the files into /etc/my.cnf.d directory -# (if there are files with same name source only file from $1) -function process_extending_config_files() { - local custom_dir default_dir - custom_dir=$1 - default_dir=$2 - - while read filename ; do - echo "=> sourcing $filename ..." - # Custom file is prefered - if [ -f $custom_dir/$filename ]; then - envsubst < $custom_dir/$filename > /etc/my.cnf.d/$filename - else - envsubst < $default_dir/$filename > /etc/my.cnf.d/$filename - fi - done <<<"$(get_matched_files "$custom_dir" "$default_dir" '*.cnf' | sort -u)" -} - -# Converts string version to the integer format (5.5.33 is converted to 505, -# 10.1.23-MariaDB is converted into 1001, etc. -function version2number() { - local version_major=$(echo "$1" | grep -o -e '^[0-9]*\.[0-9]*') - printf %d%02d ${version_major%%.*} ${version_major##*.} -} - -# Converts the version in format of an integer into major.minor -function number2version() { - local numver=${1} - echo $((numver / 100)).$((numver % 100)) -} - -# Prints version of the mysqld that is currently available (string) -function mysqld_version() { - ${MYSQL_PREFIX}/libexec/mysqld -V | awk '{print $3}' -} - -# Returns version from the daemon in integer format -function mysqld_compat_version() { - version2number $(mysqld_version) -} - -# Returns version from the datadir in the integer format -function get_datadir_version() { - local datadir="$1" - local upgrade_info_file=$(get_mysql_upgrade_info_file "$datadir") - [ -r "$upgrade_info_file" ] || return - local version_text=$(cat "$upgrade_info_file" | head -n 1) - version2number "${version_text}" -} - -# Returns name of the file in the datadir that holds version information about the data -function get_mysql_upgrade_info_file() { - local datadir="$1" - echo "$datadir/mysql_upgrade_info" -} - -# Writes version string of the daemon into mysql_upgrade_info file -# (should be only used when the file is missing and only during limited time; -# once most deployments include this version file, we should leave it on -# scripts to generate the file right after initialization or when upgrading) -function write_mysql_upgrade_info_file() { - local datadir="$1" - local version=$(mysqld_version) - local upgrade_info_file=$(get_mysql_upgrade_info_file "$datadir") - if [ -f "$datadir/mysql_upgrade_info" ] ; then - echo "File ${upgrade_info_file} exists, nothing is done." - else - log_info "Storing version '${version}' information into the data dir '${upgrade_info_file}'" - echo "${version}" > "${upgrade_info_file}" - mysqld_version >"$datadir/mysql_upgrade_info" - fi -} diff --git a/root-common/usr/share/container-scripts/mysql/helpers.sh b/root-common/usr/share/container-scripts/mysql/helpers.sh deleted file mode 100644 index 22db289..0000000 --- a/root-common/usr/share/container-scripts/mysql/helpers.sh +++ /dev/null @@ -1,31 +0,0 @@ -function log_info { - echo "---> `date +%T` $@" -} - -function log_warn { - echo "---> `date +%T` Warning: $@" -} - -function log_and_run { - log_info "Running $@" - "$@" -} - -function log_volume_info { - CONTAINER_DEBUG=${CONTAINER_DEBUG:-} - if [[ "${CONTAINER_DEBUG,,}" != "true" ]]; then - return - fi - - log_info "Volume info for $@:" - set +e - log_and_run mount - while [ $# -gt 0 ]; do - log_and_run ls -alZ $1 - shift - done - set -e - if [[ -v DEBUG_IGNORE_SCRIPT_FAILURES ]]; then - set +e - fi -} diff --git a/root-common/usr/share/container-scripts/mysql/init/40-datadir-action.sh b/root-common/usr/share/container-scripts/mysql/init/40-datadir-action.sh deleted file mode 100644 index 6198367..0000000 --- a/root-common/usr/share/container-scripts/mysql/init/40-datadir-action.sh +++ /dev/null @@ -1,112 +0,0 @@ -upstream_upgrade_info() { - echo -n "For upstream documentation about upgrading, see: " - case ${MYSQL_VERSION} in - 10.0) echo "https://mariadb.com/kb/en/library/upgrading-from-mariadb-55-to-mariadb-100/" ;; - 10.1) echo "https://mariadb.com/kb/en/library/upgrading-from-mariadb-100-to-mariadb-101/" ;; - 10.2) echo "https://mariadb.com/kb/en/library/upgrading-from-mariadb-101-to-mariadb-102/" ;; - 10.3) echo "https://mariadb.com/kb/en/library/upgrading-from-mariadb-102-to-mariadb-103/" ;; - 5.6) echo "https://dev.mysql.com/doc/refman/5.6/en/upgrading-from-previous-series.html" ;; - 5.7) echo "https://dev.mysql.com/doc/refman/5.7/en/upgrading-from-previous-series.html" ;; - *) echo "Non expected version '${MYSQL_VERSION}'" ; return 1 ;; - esac -} - -check_datadir_version() { - local datadir="$1" - local datadir_version=$(get_datadir_version "$datadir") - local mysqld_version=$(mysqld_compat_version) - local datadir_version_dot=$(number2version "${datadir_version}") - local mysqld_version_dot=$(number2version "${mysqld_version}") - - for datadir_action in ${MYSQL_DATADIR_ACTION//,/ } ; do - log_info "Running datadir action: ${datadir_action}" - case ${datadir_action} in - upgrade-auto|upgrade-warn) - if [ -z "${datadir_version}" ] || [ "${datadir_version}" -eq 0 ] ; then - # Writing the info file, since historically it was not written - log_warn "Version of the data could not be determined."\ - "It is because the file mysql_upgrade_info is missing in the data directory, which"\ - "is most probably because it was not created when initialization of data directory."\ - "In order to allow seamless updates to the next higher version in the future,"\ - "the file mysql_upgrade_info will be created."\ - "If the data directory was created with a different version than ${mysqld_version_dot},"\ - "it is required to run this container with the MYSQL_DATADIR_ACTION environment variable"\ - "set to 'force', or run 'mysql_upgrade' utility manually; the mysql_upgrade tool"\ - "checks the tables and creates such a file as well. $(upstream_upgrade_info)" - write_mysql_upgrade_info_file "${MYSQL_DATADIR}" - continue - # This is currently a dead-code, but should be enabled after the mysql_upgrade_info - # file gets to the deployments (after few months most of the deployments should already have the file) - log_warn "Version of the data could not be determined."\ - "Running such a container is risky."\ - "The current daemon version is ${mysqld_version_dot}."\ - "If you are not sure whether the data directory is compatible with the current"\ - "version ${mysqld_version_dot}, restore the data from a back-up."\ - "If restoring from a back-up is not possible, create a file 'mysql_upgrade_info'"\ - "that includes version information (${mysqld_version_dot} in this case) in the root"\ - "of the data directory."\ - "In order to create the 'mysql_upgrade_info' file, either run this container with"\ - "the MYSQL_DATADIR_ACTION environment variable set to 'force', or run 'mysql_upgrade' utility"\ - "manually; the mysql_upgrade tool checks the tables and creates such a file as well."\ - "That will enable correct upgrade check in the future. $(upstream_upgrade_info)" - fi - - if [ "${datadir_version}" -eq "${mysqld_version}" ] ; then - log_info "MySQL server version check passed, both server and data directory"\ - "are version ${mysqld_version_dot}." - continue - fi - - if [ $(( ${datadir_version} + 1 )) -eq "${mysqld_version}" -o "${datadir_version}" -eq 505 -a "${mysqld_version}" -eq 1000 ] ; then - log_warn "MySQL server is version ${mysqld_version_dot} and datadir is version"\ - "${datadir_version_dot}, which is a compatible combination." - if [ "${MYSQL_DATADIR_ACTION}" == 'upgrade-auto' ] ; then - log_info "The data directory will be upgraded automatically from ${datadir_version_dot}"\ - "to version ${mysqld_version_dot}. $(upstream_upgrade_info)" - log_and_run mysql_upgrade ${mysql_flags} - else - log_warn "Automatic upgrade is not turned on, proceed with the upgrade."\ - "In order to upgrade the data directory, run this container with the MYSQL_DATADIR_ACTION"\ - "environment variable set to 'upgrade-auto' or run mysql_upgrade manually. $(upstream_upgrade_info)" - fi - else - log_warn "MySQL server is version ${mysqld_version_dot} and datadir is version"\ - "${datadir_version_dot}, which are incompatible. Remember, that upgrade is only supported"\ - "by upstream from previous version and it is not allowed to skip versions. $(upstream_upgrade_info)" - if [ "${datadir_version}" -gt "${mysqld_version}" ] ; then - log_warn "Downgrading to the lower version is not supported. Consider"\ - "dumping data and load them again into a fresh instance. $(upstream_upgrade_info)" - fi - log_warn "Consider restoring the database from a back-up. To ignore this"\ - "warning, set 'MYSQL_DATADIR_ACTION' variable to 'upgrade-force', but this may result in data corruption. $(upstream_upgrade_info)" - return 1 - fi - ;; - - upgrade-force) - log_and_run mysql_upgrade ${mysql_flags} --force - ;; - - optimize) - log_and_run mysqlcheck ${mysql_flags} --optimize --all-databases --force - ;; - - analyze) - log_and_run mysqlcheck ${mysql_flags} --analyze --all-databases --force - ;; - - disable) - log_info "Nothing is done about the data directory." - ;; - *) - log_warn "Unknown value of MYSQL_DATADIR_ACTION variable: '${MYSQL_DATADIR_ACTION}', ignoring." - ;; - esac - done -} - -check_datadir_version "${MYSQL_DATADIR}" - -unset -f check_datadir_version upstream_upgrade_info - - diff --git a/root-common/usr/share/container-scripts/mysql/init/50-passwd-change.sh b/root-common/usr/share/container-scripts/mysql/init/50-passwd-change.sh deleted file mode 100644 index 9fa0018..0000000 --- a/root-common/usr/share/container-scripts/mysql/init/50-passwd-change.sh +++ /dev/null @@ -1,49 +0,0 @@ -password_change() { - log_info 'Setting passwords ...' - - # Set the password for MySQL user and root everytime this container is started. - # This allows to change the password by editing the deployment configuration. - if [[ -v MYSQL_USER && -v MYSQL_PASSWORD ]]; then -mysql $mysql_flags < "10.0" ] ; then -mysql $mysql_flags < "10.0" ] ; then -mysql $mysql_flags < /etc/my.cnf.d/base.cnf - diff --git a/root-common/usr/share/container-scripts/mysql/pre-init/60-replication-config.sh b/root-common/usr/share/container-scripts/mysql/pre-init/60-replication-config.sh deleted file mode 100644 index a923476..0000000 --- a/root-common/usr/share/container-scripts/mysql/pre-init/60-replication-config.sh +++ /dev/null @@ -1,17 +0,0 @@ -# mysqld configuration for replication scenarios - -if [ -v MYSQL_RUNNING_AS_MASTER ] || [ -v MYSQL_RUNNING_AS_SLAVE ] ; then - log_info 'Processing basic MySQL configuration for replication (master and slave) files ...' - envsubst < ${CONTAINER_SCRIPTS_PATH}/pre-init/my-repl-gtid.cnf.template > /etc/my.cnf.d/repl-gtid.cnf -fi - -if [ -v MYSQL_RUNNING_AS_MASTER ] ; then - log_info 'Processing basic MySQL configuration for replication (master only) files ...' - envsubst < ${CONTAINER_SCRIPTS_PATH}/pre-init/my-master.cnf.template > /etc/my.cnf.d/master.cnf -fi - -if [ -v MYSQL_RUNNING_AS_SLAVE ] ; then - log_info 'Processing basic MySQL configuration for replication (slave only) files ...' - envsubst < ${CONTAINER_SCRIPTS_PATH}/pre-init/my-slave.cnf.template > /etc/my.cnf.d/slave.cnf -fi - diff --git a/root-common/usr/share/container-scripts/mysql/pre-init/70-s2i-config.sh b/root-common/usr/share/container-scripts/mysql/pre-init/70-s2i-config.sh deleted file mode 100644 index 7a8ae5a..0000000 --- a/root-common/usr/share/container-scripts/mysql/pre-init/70-s2i-config.sh +++ /dev/null @@ -1,6 +0,0 @@ -# additional arbitrary mysqld configuration provided by user using s2i - -log_info 'Processing additional arbitrary MySQL configuration provided by s2i ...' - -process_extending_config_files ${APP_DATA}/mysql-cfg/ ${CONTAINER_SCRIPTS_PATH}/cnf/ - diff --git a/root-common/usr/share/container-scripts/mysql/pre-init/my-base.cnf.template b/root-common/usr/share/container-scripts/mysql/pre-init/my-base.cnf.template deleted file mode 100644 index c654f7f..0000000 --- a/root-common/usr/share/container-scripts/mysql/pre-init/my-base.cnf.template +++ /dev/null @@ -1,5 +0,0 @@ -[mysqld] -datadir = ${MYSQL_DATADIR} -basedir = ${MYSQL_PREFIX} -plugin-dir = ${MYSQL_PREFIX}/lib64/mysql/plugin - diff --git a/root-common/usr/share/container-scripts/mysql/pre-init/my-master.cnf.template b/root-common/usr/share/container-scripts/mysql/pre-init/my-master.cnf.template deleted file mode 100644 index f434885..0000000 --- a/root-common/usr/share/container-scripts/mysql/pre-init/my-master.cnf.template +++ /dev/null @@ -1,7 +0,0 @@ -[mysqld] - -server-id = ${MYSQL_SERVER_ID} -log_bin = ${MYSQL_DATADIR}/mysql-bin.log -binlog_do_db = mysql -binlog_do_db = ${MYSQL_DATABASE} -binlog_format = ${MYSQL_BINLOG_FORMAT} diff --git a/root-common/usr/share/container-scripts/mysql/pre-init/my-repl-gtid.cnf.template b/root-common/usr/share/container-scripts/mysql/pre-init/my-repl-gtid.cnf.template deleted file mode 100644 index a74a74c..0000000 --- a/root-common/usr/share/container-scripts/mysql/pre-init/my-repl-gtid.cnf.template +++ /dev/null @@ -1,4 +0,0 @@ -[mysqld] - -log-slave-updates = ON - diff --git a/root-common/usr/share/container-scripts/mysql/pre-init/my-slave.cnf.template b/root-common/usr/share/container-scripts/mysql/pre-init/my-slave.cnf.template deleted file mode 100644 index 5bdf109..0000000 --- a/root-common/usr/share/container-scripts/mysql/pre-init/my-slave.cnf.template +++ /dev/null @@ -1,7 +0,0 @@ -[mysqld] - -server-id = ${MYSQL_SERVER_ID} -log_bin = ${MYSQL_DATADIR}/mysql-bin.log -relay-log = ${MYSQL_DATADIR}/mysql-relay-bin.log -binlog_do_db = mysql -binlog_do_db = ${MYSQL_DATABASE} diff --git a/root-common/usr/share/container-scripts/mysql/scl_enable b/root-common/usr/share/container-scripts/mysql/scl_enable deleted file mode 100644 index 5a25432..0000000 --- a/root-common/usr/share/container-scripts/mysql/scl_enable +++ /dev/null @@ -1,3 +0,0 @@ -# This will make scl collection binaries work out of box. -unset BASH_ENV PROMPT_COMMAND ENV -source scl_source enable ${ENABLED_COLLECTIONS} diff --git a/root/help.1 b/root/help.1 deleted file mode 100644 index 6524577..0000000 --- a/root/help.1 +++ /dev/null @@ -1,475 +0,0 @@ -.TH MariaDB 10.3 SQL Database Server Docker image -.PP -This container image includes MariaDB 10.3 SQL database server for OpenShift and general usage. -Users can choose between RHEL, CentOS and Fedora based images. -The RHEL images are available in the Red Hat Container Catalog -\[la]https://access.redhat.com/containers/\[ra], -the CentOS images are available on Docker Hub -\[la]https://hub.docker.com/r/centos/\[ra], -and the Fedora images are available in Fedora Registry -\[la]https://registry.fedoraproject.org/\[ra]\&. -The resulting image can be run using podman -\[la]https://github.com/containers/libpod\[ra]\&. - -.PP -Note: while the examples in this README are calling \fB\fCpodman\fR, you can replace any such calls by \fB\fCdocker\fR with the same arguments - -.SH Description -.PP -This container image provides a containerized packaging of the MariaDB mysqld daemon -and client application. The mysqld server daemon accepts connections from clients -and provides access to content from MySQL databases on behalf of the clients. -You can find more information on the MariaDB project from the project Web site -( -\[la]https://mariadb.org/\[ra]). - -.SH Usage -.PP -For this, we will assume that you are using the MariaDB 10.3 container image from the -Red Hat Container Catalog called \fB\fCrhel8/mariadb\-103\fR\&. -If you want to set only the mandatory environment variables and not store -the database in a host directory, execute the following command: - -.PP -.RS - -.nf -$ podman run \-d \-\-name mariadb\_database \-e MYSQL\_USER=user \-e MYSQL\_PASSWORD=pass \-e MYSQL\_DATABASE=db \-p 3306:3306 rhel8/mariadb\-103 - -.fi -.RE - -.PP -This will create a container named \fB\fCmariadb\_database\fR running MySQL with database -\fB\fCdb\fR and user with credentials \fB\fCuser:pass\fR\&. Port 3306 will be exposed and mapped -to the host. If you want your database to be persistent across container executions, -also add a \fB\fC\-v /host/db/path:/var/lib/mysql/data\fR argument. This will be the MySQL -data directory. - -.PP -If the database directory is not initialized, the entrypoint script will first -run \fB\fCmysql\_install\_db\fR -\[la]https://dev.mysql.com/doc/refman/5.6/en/mysql-install-db.html\[ra] -and setup necessary database users and passwords. After the database is initialized, -or if it was already present, \fB\fCmysqld\fR is executed and will run as PID 1. You can - stop the detached container by running \fB\fCpodman stop mariadb\_database\fR\&. - -.SH Environment variables and volumes -.PP -The image recognizes the following environment variables that you can set during -initialization by passing \fB\fC\-e VAR=VALUE\fR to the Docker run command. - -.PP -\fB\fB\fCMYSQL\_USER\fR\fP -.br - User name for MySQL account to be created - -.PP -\fB\fB\fCMYSQL\_PASSWORD\fR\fP -.br - Password for the user account - -.PP -\fB\fB\fCMYSQL\_DATABASE\fR\fP -.br - Database name - -.PP -\fB\fB\fCMYSQL\_ROOT\_PASSWORD\fR\fP -.br - Password for the root user (optional) - -.PP -The following environment variables influence the MySQL configuration file. They are all optional. - -.PP -\fB\fB\fCMYSQL\_LOWER\_CASE\_TABLE\_NAMES (default: 0)\fR\fP -.br - Sets how the table names are stored and compared - -.PP -\fB\fB\fCMYSQL\_MAX\_CONNECTIONS (default: 151)\fR\fP -.br - The maximum permitted number of simultaneous client connections - -.PP -\fB\fB\fCMYSQL\_MAX\_ALLOWED\_PACKET (default: 200M)\fR\fP -.br - The maximum size of one packet or any generated/intermediate string - -.PP -\fB\fB\fCMYSQL\_FT\_MIN\_WORD\_LEN (default: 4)\fR\fP -.br - The minimum length of the word to be included in a FULLTEXT index - -.PP -\fB\fB\fCMYSQL\_FT\_MAX\_WORD\_LEN (default: 20)\fR\fP -.br - The maximum length of the word to be included in a FULLTEXT index - -.PP -\fB\fB\fCMYSQL\_AIO (default: 1)\fR\fP -.br - Controls the \fB\fCinnodb\_use\_native\_aio\fR setting value in case the native AIO is broken. See -\[la]http://help.directadmin.com/item.php?id=529\[ra] - -.PP -\fB\fB\fCMYSQL\_TABLE\_OPEN\_CACHE (default: 400)\fR\fP -.br - The number of open tables for all threads - -.PP -\fB\fB\fCMYSQL\_KEY\_BUFFER\_SIZE (default: 32M or 10% of available memory)\fR\fP -.br - The size of the buffer used for index blocks - -.PP -\fB\fB\fCMYSQL\_SORT\_BUFFER\_SIZE (default: 256K)\fR\fP -.br - The size of the buffer used for sorting - -.PP -\fB\fB\fCMYSQL\_READ\_BUFFER\_SIZE (default: 8M or 5% of available memory)\fR\fP -.br - The size of the buffer used for a sequential scan - -.PP -\fB\fB\fCMYSQL\_INNODB\_BUFFER\_POOL\_SIZE (default: 32M or 50% of available memory)\fR\fP -.br - The size of the buffer pool where InnoDB caches table and index data - -.PP -\fB\fB\fCMYSQL\_INNODB\_LOG\_FILE\_SIZE (default: 8M or 15% of available memory)\fR\fP -.br - The size of each log file in a log group - -.PP -\fB\fB\fCMYSQL\_INNODB\_LOG\_BUFFER\_SIZE (default: 8M or 15% of available memory)\fR\fP -.br - The size of the buffer that InnoDB uses to write to the log files on disk - -.PP -\fB\fB\fCMYSQL\_DEFAULTS\_FILE (default: /etc/my.cnf)\fR\fP -.br - Point to an alternative configuration file - -.PP -\fB\fB\fCMYSQL\_BINLOG\_FORMAT (default: statement)\fR\fP -.br - Set sets the binlog format, supported values are \fB\fCrow\fR and \fB\fCstatement\fR - -.PP -\fB\fB\fCMYSQL\_LOG\_QUERIES\_ENABLED (default: 0)\fR\fP -.br - To enable query logging set this to \fB\fC1\fR - -.PP -You can also set the following mount points by passing the \fB\fC\-v /host:/container\fR flag to Docker. - -.PP -\fB\fB\fC/var/lib/mysql/data\fR\fP -.br - MySQL data directory - -.PP -\fBNotice: When mouting a directory from the host into the container, ensure that the mounted -directory has the appropriate permissions and that the owner and group of the directory -matches the user UID or name which is running inside the container.\fP - -.SH MariaDB auto\-tuning -.PP -When the MySQL image is run with the \fB\fC\-\-memory\fR parameter set and you didn't -specify value for some parameters, their values will be automatically -calculated based on the available memory. - -.PP -\fB\fB\fCMYSQL\_KEY\_BUFFER\_SIZE (default: 10%)\fR\fP -.br - \fB\fCkey\_buffer\_size\fR - -.PP -\fB\fB\fCMYSQL\_READ\_BUFFER\_SIZE (default: 5%)\fR\fP -.br - \fB\fCread\_buffer\_size\fR - -.PP -\fB\fB\fCMYSQL\_INNODB\_BUFFER\_POOL\_SIZE (default: 50%)\fR\fP -.br - \fB\fCinnodb\_buffer\_pool\_size\fR - -.PP -\fB\fB\fCMYSQL\_INNODB\_LOG\_FILE\_SIZE (default: 15%)\fR\fP -.br - \fB\fCinnodb\_log\_file\_size\fR - -.PP -\fB\fB\fCMYSQL\_INNODB\_LOG\_BUFFER\_SIZE (default: 15%)\fR\fP -.br - \fB\fCinnodb\_log\_buffer\_size\fR - -.SH MySQL root user -.PP -The root user has no password set by default, only allowing local connections. -You can set it by setting the \fB\fCMYSQL\_ROOT\_PASSWORD\fR environment variable. This -will allow you to login to the root account remotely. Local connections will -still not require a password. - -.PP -To disable remote root access, simply unset \fB\fCMYSQL\_ROOT\_PASSWORD\fR and restart -the container. - -.SH Changing passwords -.PP -Since passwords are part of the image configuration, the only supported method -to change passwords for the database user (\fB\fCMYSQL\_USER\fR) and root user is by -changing the environment variables \fB\fCMYSQL\_PASSWORD\fR and \fB\fCMYSQL\_ROOT\_PASSWORD\fR, -respectively. - -.PP -Changing database passwords through SQL statements or any way other than through -the environment variables aforementioned will cause a mismatch between the -values stored in the variables and the actual passwords. Whenever a database -container starts it will reset the passwords to the values stored in the -environment variables. - -.SH Default my.cnf file -.PP -With environment variables we are able to customize a lot of different parameters -or configurations for the mysql bootstrap configurations. If you'd prefer to use -your own configuration file, you can override the \fB\fCMYSQL\_DEFAULTS\_FILE\fR env -variable with the full path of the file you wish to use. For example, the default -location is \fB\fC/etc/my.cnf\fR but you can change it to \fB\fC/etc/mysql/my.cnf\fR by setting - \fB\fCMYSQL\_DEFAULTS\_FILE=/etc/mysql/my.cnf\fR - -.SH Extending image -.PP -This image can be extended in Openshift using the \fB\fCSource\fR build strategy or via the standalone -source\-to\-image -\[la]https://github.com/openshift/source-to-image\[ra] application (where available). -For this, we will assume that you are using the \fB\fCrhscl/mariadb\-103\-rhel7\fR image, -available via \fB\fCmariadb:10.3\fR imagestream tag in Openshift. - -.PP -For example, to build a customized MariaDB database image \fB\fCmy\-mariadb\-rhel7\fR -with a configuration from \fB\fChttps://github.com/sclorg/mariadb\-container/tree/master/examples/extend\-image\fR run: - -.PP -.RS - -.nf -$ oc new\-app mariadb:10.3\~https://github.com/sclorg/mariadb\-container.git \\ - \-\-name my\-mariadb\-rhel7 \\ - \-\-context\-dir=examples/extend\-image \\ - \-\-env MYSQL\_OPERATIONS\_USER=opuser \\ - \-\-env MYSQL\_OPERATIONS\_PASSWORD=oppass \\ - \-\-env MYSQL\_DATABASE=opdb \\ - \-\-env MYSQL\_USER=user \\ - \-\-env MYSQL\_PASSWORD=pass - -.fi -.RE - -.PP -or via s2i: - -.PP -.RS - -.nf -$ s2i build \-\-context\-dir=examples/extend\-image https://github.com/sclorg/mariadb\-container.git rhscl/mariadb\-103\-rhel7 my\-mariadb\-rhel7 - -.fi -.RE - -.PP -The directory passed to Openshift can contain these directories: - -.PP -\fB\fCmysql\-cfg/\fR - When starting the container, files from this directory will be used as - a configuration for the \fB\fCmysqld\fR daemon. - \fB\fCenvsubst\fR command is run on this file to still allow customization of - the image using environmental variables - -.PP -\fB\fCmysql\-pre\-init/\fR - Shell scripts (\fB\fC*.sh\fR) available in this directory are sourced before - \fB\fCmysqld\fR daemon is started. - -.PP -\fB\fCmysql\-init/\fR - Shell scripts (\fB\fC*.sh\fR) available in this directory are sourced when - \fB\fCmysqld\fR daemon is started locally. In this phase, use \fB\fC${mysql\_flags}\fR - to connect to the locally running daemon, for example \fB\fCmysql $mysql\_flags < dump.sql\fR - -.PP -Variables that can be used in the scripts provided to s2i: - -.PP -\fB\fC$mysql\_flags\fR - arguments for the \fB\fCmysql\fR tool that will connect to the locally running \fB\fCmysqld\fR during initialization - -.PP -\fB\fC$MYSQL\_RUNNING\_AS\_MASTER\fR - variable defined when the container is run with \fB\fCrun\-mysqld\-master\fR command - -.PP -\fB\fC$MYSQL\_RUNNING\_AS\_SLAVE\fR - variable defined when the container is run with \fB\fCrun\-mysqld\-slave\fR command - -.PP -\fB\fC$MYSQL\_DATADIR\_FIRST\_INIT\fR - variable defined when the container was initialized from the empty data dir - -.PP -During the s2i build all provided files are copied into \fB\fC/opt/app\-root/src\fR -directory into the resulting image. If some configuration files are present -in the destination directory, files with the same name are overwritten. -Also only one file with the same name can be used for customization and user -provided files are preferred over default files in -\fB\fC/usr/share/container\-scripts/mysql/\fR\- so it is possible to overwrite them. - -.PP -Same configuration directory structure can be used to customize the image -every time the image is started using \fB\fCpodman run\fR\&. The directory has to be -mounted into \fB\fC/opt/app\-root/src/\fR in the image -(\fB\fC\-v ./image\-configuration/:/opt/app\-root/src/\fR). -This overwrites customization built into the image. - -.SH Securing the connection with SSL -.PP -In order to secure the connection with SSL, use the extending feature described -above. In particular, put the SSL certificates into a separate directory: - -.PP -.RS - -.nf -sslapp/mysql\-certs/server\-cert\-selfsigned.pem -sslapp/mysql\-certs/server\-key.pem - -.fi -.RE - -.PP -And then put a separate configuration file into mysql\-cfg: - -.PP -.RS - -.nf -$> cat sslapp/mysql\-cfg/ssl.cnf -[mysqld] -ssl\-key=${APP\_DATA}/mysql\-certs/server\-key.pem -ssl\-cert=${APP\_DATA}/mysql\-certs/server\-cert\-selfsigned.pem - -.fi -.RE - -.PP -Such a directory \fB\fCsslapp\fR can then be mounted into the container with \-v, -or a new container image can be built using s2i. - -.SH Upgrading and data directory version checking -.PP -MySQL and MariaDB use versions that consist of three numbers X.Y.Z (e.g. 5.6.23). -For version changes in Z part, the server's binary data format stays compatible and thus no -special upgrade procedure is needed. For upgrades from X.Y to X.Y+1, consider doing manual -steps as described at - -\[la]https://mariadb.com/kb/en/library/upgrading-from-mariadb-102-to-mariadb-103/\[ra] - -.PP -Skipping versions like from X.Y to X.Y+2 or downgrading to lower version is not supported; -the only exception is ugrading from MariaDB 5.5 to MariaDB 10.0. - -.PP -\fBImportant\fP: Upgrading to a new version is always risky and users are expected to make a full -back\-up of all data before. - -.PP -A safer solution to upgrade is to dump all data using \fB\fCmysqldump\fR or \fB\fCmysqldbexport\fR and then -load the data using \fB\fCmysql\fR or \fB\fCmysqldbimport\fR into an empty (freshly initialized) database. - -.PP -Another way of proceeding with the upgrade is starting the new version of the \fB\fCmysqld\fR daemon -and run \fB\fCmysql\_upgrade\fR right after the start. This so called in\-place upgrade is generally -faster for large data directory, but only possible if upgrading from the very previous version, -so skipping versions is not supported. - -.PP -This container detects whether the data needs to be upgraded using \fB\fCmysql\_upgrade\fR and -we can control it by setting \fB\fCMYSQL\_DATADIR\_ACTION\fR variable, which can have one or more of the following values: - -.RS -.IP \(bu 2 -\fB\fCupgrade\-warn\fR \-\- If the data version can be determined and the data come from a different version -of the daemon, a warning is printed but the container starts. This is the default value. -Since historically the version file \fB\fCmysql\_upgrade\_info\fR was not created, when using this option, -the version file is created if not exist, but no \fB\fCmysql\_upgrade\fR will be called. -However, this automatic creation will be removed after few months, since the version should be -created on most deployments at that point. -.IP \(bu 2 -\fB\fCupgrade\-auto\fR \-\- \fB\fCmysql\_upgrade\fR is run at the beginning of the container start, when the local -daemon is running, but only if the data version can be determined and the data come -with the very previous version. A warning is printed if the data come from even older -or newer version. This value effectively enables automatic upgrades, -but it is always risky and users should still back\-up all the data before starting the newer container. -Set this option only if you have very good back\-ups at any moment and you are fine to fail\-over -from the back\-up. -.IP \(bu 2 -\fB\fCupgrade\-force\fR \-\- \fB\fCmysql\_upgrade \-\-force\fR is run at the beginning of the container start, when the local -daemon is running, no matter what version of the daemon the data come from. -This is also the way to create the missing version file \fB\fCmysql\_upgrade\_info\fR if not present -in the root of the data directory; this file holds information about the version of the data. - -.RE - -.PP -There are also some other actions that you may want to run at the beginning of the container start, -when the local daemon is running, no matter what version of the data is detected: - -.RS -.IP \(bu 2 -\fB\fCoptimize\fR \-\- runs \fB\fCmysqlcheck \-\-optimize\fR\&. It optimizes all the tables. -.IP \(bu 2 -\fB\fCanalyze\fR \-\- runs \fB\fCmysqlcheck \-\-analyze\fR\&. It analyzes all the tables. -.IP \(bu 2 -\fB\fCdisable\fR \-\- nothing is done regarding data directory version. - -.RE - -.PP -Multiple values are separated by comma and run in\-order, e.g. \fB\fCMYSQL\_DATADIR\_ACTION="optimize,analyze"\fR\&. - -.SH Changing the replication binlog\_format -.PP -Some applications may wish to use \fB\fCrow\fR binlog\_formats (for example, those built - with change\-data\-capture in mind). The default replication/binlog format is - \fB\fCstatement\fR but to change it you can set the \fB\fCMYSQL\_BINLOG\_FORMAT\fR environment - variable. For example \fB\fCMYSQL\_BINLOG\_FORMAT=row\fR\&. Now when you run the database - with \fB\fCmaster\fR replication turned on (ie, set the Docker/container \fB\fCcmd\fR to be -\fB\fCrun\-mysqld\-master\fR) the binlog will emit the actual data for the rows that change -as opposed to the statements (ie, DML like insert...) that caused the change. - -.SH Troubleshooting -.PP -The mysqld deamon in the container logs to the standard output, so the log is available in the container log. The log can be examined by running: - -.PP -.RS - -.nf -podman logs - -.fi -.RE - -.SH See also -.PP -Dockerfile and other sources for this container image are available on - -\[la]https://github.com/sclorg/mariadb-container\[ra]\&. -In that repository, the Dockerfile for CentOS is called Dockerfile, the Dockerfile -for RHEL7 is called Dockerfile.rhel7, the Dockerfile for RHEL8 is called Dockerfile.rhel8, -and the Dockerfile for Fedora is called Dockerfile.fedora. diff --git a/root/usr/share/container-scripts/mysql/README.md b/root/usr/share/container-scripts/mysql/README.md deleted file mode 100644 index 7e35c93..0000000 --- a/root/usr/share/container-scripts/mysql/README.md +++ /dev/null @@ -1,359 +0,0 @@ -MariaDB 10.3 SQL Database Server Docker image -============================================= - -This container image includes MariaDB 10.3 SQL database server for OpenShift and general usage. -Users can choose between RHEL, CentOS and Fedora based images. -The RHEL images are available in the [Red Hat Container Catalog](https://access.redhat.com/containers/), -the CentOS images are available on [Docker Hub](https://hub.docker.com/r/centos/), -and the Fedora images are available in [Fedora Registry](https://registry.fedoraproject.org/). -The resulting image can be run using [podman](https://github.com/containers/libpod). - -Note: while the examples in this README are calling `podman`, you can replace any such calls by `docker` with the same arguments - -Description ------------ - -This container image provides a containerized packaging of the MariaDB mysqld daemon -and client application. The mysqld server daemon accepts connections from clients -and provides access to content from MySQL databases on behalf of the clients. -You can find more information on the MariaDB project from the project Web site -(https://mariadb.org/). - - -Usage ------ - -For this, we will assume that you are using the MariaDB 10.3 container image from the -Red Hat Container Catalog called `rhel8/mariadb-103`. -If you want to set only the mandatory environment variables and not store -the database in a host directory, execute the following command: - -``` -$ podman run -d --name mariadb_database -e MYSQL_USER=user -e MYSQL_PASSWORD=pass -e MYSQL_DATABASE=db -p 3306:3306 rhel8/mariadb-103 -``` - -This will create a container named `mariadb_database` running MySQL with database -`db` and user with credentials `user:pass`. Port 3306 will be exposed and mapped -to the host. If you want your database to be persistent across container executions, -also add a `-v /host/db/path:/var/lib/mysql/data` argument. This will be the MySQL -data directory. - -If the database directory is not initialized, the entrypoint script will first -run [`mysql_install_db`](https://dev.mysql.com/doc/refman/5.6/en/mysql-install-db.html) -and setup necessary database users and passwords. After the database is initialized, -or if it was already present, `mysqld` is executed and will run as PID 1. You can - stop the detached container by running `podman stop mariadb_database`. - - -Environment variables and volumes ---------------------------------- - -The image recognizes the following environment variables that you can set during -initialization by passing `-e VAR=VALUE` to the Docker run command. - -**`MYSQL_USER`** - User name for MySQL account to be created - -**`MYSQL_PASSWORD`** - Password for the user account - -**`MYSQL_DATABASE`** - Database name - -**`MYSQL_ROOT_PASSWORD`** - Password for the root user (optional) - - -The following environment variables influence the MySQL configuration file. They are all optional. - -**`MYSQL_LOWER_CASE_TABLE_NAMES (default: 0)`** - Sets how the table names are stored and compared - -**`MYSQL_MAX_CONNECTIONS (default: 151)`** - The maximum permitted number of simultaneous client connections - -**`MYSQL_MAX_ALLOWED_PACKET (default: 200M)`** - The maximum size of one packet or any generated/intermediate string - -**`MYSQL_FT_MIN_WORD_LEN (default: 4)`** - The minimum length of the word to be included in a FULLTEXT index - -**`MYSQL_FT_MAX_WORD_LEN (default: 20)`** - The maximum length of the word to be included in a FULLTEXT index - -**`MYSQL_AIO (default: 1)`** - Controls the `innodb_use_native_aio` setting value in case the native AIO is broken. See http://help.directadmin.com/item.php?id=529 - -**`MYSQL_TABLE_OPEN_CACHE (default: 400)`** - The number of open tables for all threads - -**`MYSQL_KEY_BUFFER_SIZE (default: 32M or 10% of available memory)`** - The size of the buffer used for index blocks - -**`MYSQL_SORT_BUFFER_SIZE (default: 256K)`** - The size of the buffer used for sorting - -**`MYSQL_READ_BUFFER_SIZE (default: 8M or 5% of available memory)`** - The size of the buffer used for a sequential scan - -**`MYSQL_INNODB_BUFFER_POOL_SIZE (default: 32M or 50% of available memory)`** - The size of the buffer pool where InnoDB caches table and index data - -**`MYSQL_INNODB_LOG_FILE_SIZE (default: 8M or 15% of available memory)`** - The size of each log file in a log group - -**`MYSQL_INNODB_LOG_BUFFER_SIZE (default: 8M or 15% of available memory)`** - The size of the buffer that InnoDB uses to write to the log files on disk - -**`MYSQL_DEFAULTS_FILE (default: /etc/my.cnf)`** - Point to an alternative configuration file - -**`MYSQL_BINLOG_FORMAT (default: statement)`** - Set sets the binlog format, supported values are `row` and `statement` - -**`MYSQL_LOG_QUERIES_ENABLED (default: 0)`** - To enable query logging set this to `1` - - -You can also set the following mount points by passing the `-v /host:/container` flag to Docker. - -**`/var/lib/mysql/data`** - MySQL data directory - - -**Notice: When mouting a directory from the host into the container, ensure that the mounted -directory has the appropriate permissions and that the owner and group of the directory -matches the user UID or name which is running inside the container.** - - -MariaDB auto-tuning -------------------- - -When the MySQL image is run with the `--memory` parameter set and you didn't -specify value for some parameters, their values will be automatically -calculated based on the available memory. - -**`MYSQL_KEY_BUFFER_SIZE (default: 10%)`** - `key_buffer_size` - -**`MYSQL_READ_BUFFER_SIZE (default: 5%)`** - `read_buffer_size` - -**`MYSQL_INNODB_BUFFER_POOL_SIZE (default: 50%)`** - `innodb_buffer_pool_size` - -**`MYSQL_INNODB_LOG_FILE_SIZE (default: 15%)`** - `innodb_log_file_size` - -**`MYSQL_INNODB_LOG_BUFFER_SIZE (default: 15%)`** - `innodb_log_buffer_size` - - - -MySQL root user ---------------------------------- -The root user has no password set by default, only allowing local connections. -You can set it by setting the `MYSQL_ROOT_PASSWORD` environment variable. This -will allow you to login to the root account remotely. Local connections will -still not require a password. - -To disable remote root access, simply unset `MYSQL_ROOT_PASSWORD` and restart -the container. - - -Changing passwords ------------------- - -Since passwords are part of the image configuration, the only supported method -to change passwords for the database user (`MYSQL_USER`) and root user is by -changing the environment variables `MYSQL_PASSWORD` and `MYSQL_ROOT_PASSWORD`, -respectively. - -Changing database passwords through SQL statements or any way other than through -the environment variables aforementioned will cause a mismatch between the -values stored in the variables and the actual passwords. Whenever a database -container starts it will reset the passwords to the values stored in the -environment variables. - - -Default my.cnf file -------------------- -With environment variables we are able to customize a lot of different parameters -or configurations for the mysql bootstrap configurations. If you'd prefer to use -your own configuration file, you can override the `MYSQL_DEFAULTS_FILE` env -variable with the full path of the file you wish to use. For example, the default -location is `/etc/my.cnf` but you can change it to `/etc/mysql/my.cnf` by setting - `MYSQL_DEFAULTS_FILE=/etc/mysql/my.cnf` - - -Extending image ---------------- -This image can be extended in Openshift using the `Source` build strategy or via the standalone -[source-to-image](https://github.com/openshift/source-to-image) application (where available). -For this, we will assume that you are using the `rhscl/mariadb-103-rhel7` image, -available via `mariadb:10.3` imagestream tag in Openshift. - - -For example, to build a customized MariaDB database image `my-mariadb-rhel7` -with a configuration from `https://github.com/sclorg/mariadb-container/tree/master/examples/extend-image` run: - -``` -$ oc new-app mariadb:10.3~https://github.com/sclorg/mariadb-container.git \ - --name my-mariadb-rhel7 \ - --context-dir=examples/extend-image \ - --env MYSQL_OPERATIONS_USER=opuser \ - --env MYSQL_OPERATIONS_PASSWORD=oppass \ - --env MYSQL_DATABASE=opdb \ - --env MYSQL_USER=user \ - --env MYSQL_PASSWORD=pass -``` - -or via s2i: - -``` -$ s2i build --context-dir=examples/extend-image https://github.com/sclorg/mariadb-container.git rhscl/mariadb-103-rhel7 my-mariadb-rhel7 -``` - -The directory passed to Openshift can contain these directories: - -`mysql-cfg/` - When starting the container, files from this directory will be used as - a configuration for the `mysqld` daemon. - `envsubst` command is run on this file to still allow customization of - the image using environmental variables - -`mysql-pre-init/` - Shell scripts (`*.sh`) available in this directory are sourced before - `mysqld` daemon is started. - -`mysql-init/` - Shell scripts (`*.sh`) available in this directory are sourced when - `mysqld` daemon is started locally. In this phase, use `${mysql_flags}` - to connect to the locally running daemon, for example `mysql $mysql_flags < dump.sql` - -Variables that can be used in the scripts provided to s2i: - -`$mysql_flags` - arguments for the `mysql` tool that will connect to the locally running `mysqld` during initialization - -`$MYSQL_RUNNING_AS_MASTER` - variable defined when the container is run with `run-mysqld-master` command - -`$MYSQL_RUNNING_AS_SLAVE` - variable defined when the container is run with `run-mysqld-slave` command - -`$MYSQL_DATADIR_FIRST_INIT` - variable defined when the container was initialized from the empty data dir - -During the s2i build all provided files are copied into `/opt/app-root/src` -directory into the resulting image. If some configuration files are present -in the destination directory, files with the same name are overwritten. -Also only one file with the same name can be used for customization and user -provided files are preferred over default files in -`/usr/share/container-scripts/mysql/`- so it is possible to overwrite them. - -Same configuration directory structure can be used to customize the image -every time the image is started using `podman run`. The directory has to be -mounted into `/opt/app-root/src/` in the image -(`-v ./image-configuration/:/opt/app-root/src/`). -This overwrites customization built into the image. - - -Securing the connection with SSL --------------------------------- -In order to secure the connection with SSL, use the extending feature described -above. In particular, put the SSL certificates into a separate directory: - - sslapp/mysql-certs/server-cert-selfsigned.pem - sslapp/mysql-certs/server-key.pem - -And then put a separate configuration file into mysql-cfg: - - $> cat sslapp/mysql-cfg/ssl.cnf - [mysqld] - ssl-key=${APP_DATA}/mysql-certs/server-key.pem - ssl-cert=${APP_DATA}/mysql-certs/server-cert-selfsigned.pem - -Such a directory `sslapp` can then be mounted into the container with -v, -or a new container image can be built using s2i. - - -Upgrading and data directory version checking ---------------------------------------------- - -MySQL and MariaDB use versions that consist of three numbers X.Y.Z (e.g. 5.6.23). -For version changes in Z part, the server's binary data format stays compatible and thus no -special upgrade procedure is needed. For upgrades from X.Y to X.Y+1, consider doing manual -steps as described at -https://mariadb.com/kb/en/library/upgrading-from-mariadb-102-to-mariadb-103/ - -Skipping versions like from X.Y to X.Y+2 or downgrading to lower version is not supported; -the only exception is ugrading from MariaDB 5.5 to MariaDB 10.0. - -**Important**: Upgrading to a new version is always risky and users are expected to make a full -back-up of all data before. - -A safer solution to upgrade is to dump all data using `mysqldump` or `mysqldbexport` and then -load the data using `mysql` or `mysqldbimport` into an empty (freshly initialized) database. - -Another way of proceeding with the upgrade is starting the new version of the `mysqld` daemon -and run `mysql_upgrade` right after the start. This so called in-place upgrade is generally -faster for large data directory, but only possible if upgrading from the very previous version, -so skipping versions is not supported. - -This container detects whether the data needs to be upgraded using `mysql_upgrade` and -we can control it by setting `MYSQL_DATADIR_ACTION` variable, which can have one or more of the following values: - - * `upgrade-warn` -- If the data version can be determined and the data come from a different version - of the daemon, a warning is printed but the container starts. This is the default value. - Since historically the version file `mysql_upgrade_info` was not created, when using this option, - the version file is created if not exist, but no `mysql_upgrade` will be called. - However, this automatic creation will be removed after few months, since the version should be - created on most deployments at that point. - * `upgrade-auto` -- `mysql_upgrade` is run at the beginning of the container start, when the local - daemon is running, but only if the data version can be determined and the data come - with the very previous version. A warning is printed if the data come from even older - or newer version. This value effectively enables automatic upgrades, - but it is always risky and users should still back-up all the data before starting the newer container. - Set this option only if you have very good back-ups at any moment and you are fine to fail-over - from the back-up. - * `upgrade-force` -- `mysql_upgrade --force` is run at the beginning of the container start, when the local - daemon is running, no matter what version of the daemon the data come from. - This is also the way to create the missing version file `mysql_upgrade_info` if not present - in the root of the data directory; this file holds information about the version of the data. - -There are also some other actions that you may want to run at the beginning of the container start, -when the local daemon is running, no matter what version of the data is detected: - - * `optimize` -- runs `mysqlcheck --optimize`. It optimizes all the tables. - * `analyze` -- runs `mysqlcheck --analyze`. It analyzes all the tables. - * `disable` -- nothing is done regarding data directory version. - -Multiple values are separated by comma and run in-order, e.g. `MYSQL_DATADIR_ACTION="optimize,analyze"`. - - -Changing the replication binlog_format --------------------------------------- -Some applications may wish to use `row` binlog_formats (for example, those built - with change-data-capture in mind). The default replication/binlog format is - `statement` but to change it you can set the `MYSQL_BINLOG_FORMAT` environment - variable. For example `MYSQL_BINLOG_FORMAT=row`. Now when you run the database - with `master` replication turned on (ie, set the Docker/container `cmd` to be -`run-mysqld-master`) the binlog will emit the actual data for the rows that change -as opposed to the statements (ie, DML like insert...) that caused the change. - - -Troubleshooting ---------------- -The mysqld deamon in the container logs to the standard output, so the log is available in the container log. The log can be examined by running: - - podman logs - - -See also --------- -Dockerfile and other sources for this container image are available on -https://github.com/sclorg/mariadb-container. -In that repository, the Dockerfile for CentOS is called Dockerfile, the Dockerfile -for RHEL7 is called Dockerfile.rhel7, the Dockerfile for RHEL8 is called Dockerfile.rhel8, -and the Dockerfile for Fedora is called Dockerfile.fedora. diff --git a/s2i-common/bin/assemble b/s2i-common/bin/assemble deleted file mode 100755 index d65b7e0..0000000 --- a/s2i-common/bin/assemble +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -set -o errexit -set -o nounset -set -o pipefail - -shopt -s dotglob -echo "---> Installing application source ..." -mv /tmp/src/* ./ 2>/dev/null || true - -# Fix source directory permissions -/usr/libexec/fix-permissions ./ - diff --git a/s2i-common/bin/run b/s2i-common/bin/run deleted file mode 120000 index 4b21ab5..0000000 --- a/s2i-common/bin/run +++ /dev/null @@ -1 +0,0 @@ -/bin/run-mysqld \ No newline at end of file diff --git a/s2i-common/bin/usage b/s2i-common/bin/usage deleted file mode 100755 index d6a3b9a..0000000 --- a/s2i-common/bin/usage +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -set -o errexit -set -o nounset -set -o pipefail - -groff -t -man -ETascii /help.1 - diff --git a/test/mariadb-ephemeral-template.json b/test/mariadb-ephemeral-template.json deleted file mode 100644 index 27db3fd..0000000 --- a/test/mariadb-ephemeral-template.json +++ /dev/null @@ -1,254 +0,0 @@ -{ - "kind": "Template", - "apiVersion": "v1", - "metadata": { - "name": "mariadb-ephemeral", - "annotations": { - "openshift.io/display-name": "MariaDB (Ephemeral)", - "description": "MariaDB database service, without persistent storage. For more information about using this template, including OpenShift considerations, see https://github.com/sclorg/mariadb-container/blob/master/10.2/root/usr/share/container-scripts/mysql/README.md.\n\nWARNING: Any data stored will be lost upon pod destruction. Only use this template for testing", - "iconClass": "icon-mariadb", - "tags": "database,mariadb", - "openshift.io/long-description": "This template provides a standalone MariaDB server with a database created. The database is not stored on persistent storage, so any restart of the service will result in all data being lost. The database name, username, and password are chosen via parameters when provisioning this service.", - "openshift.io/provider-display-name": "Red Hat, Inc.", - "openshift.io/documentation-url": "https://github.com/sclorg/mariadb-container/blob/master/10.2/root/usr/share/container-scripts/mysql/README.md", - "openshift.io/support-url": "https://access.redhat.com" - } - }, - "message": "The following service(s) have been created in your project: ${DATABASE_SERVICE_NAME}.\n\n Username: ${MYSQL_USER}\n Password: ${MYSQL_PASSWORD}\n Database Name: ${MYSQL_DATABASE}\n Connection URL: mysql://${DATABASE_SERVICE_NAME}:3306/\n\nFor more information about using this template, including OpenShift considerations, see https://github.com/sclorg/mariadb-container/blob/master/10.2/root/usr/share/container-scripts/mysql/README.md.", - "labels": { - "template": "mariadb-ephemeral-template" - }, - "objects": [ - { - "kind": "Secret", - "apiVersion": "v1", - "metadata": { - "name": "${DATABASE_SERVICE_NAME}", - "annotations": { - "template.openshift.io/expose-username": "{.data['database-user']}", - "template.openshift.io/expose-password": "{.data['database-password']}", - "template.openshift.io/expose-root_password": "{.data['database-root-password']}", - "template.openshift.io/expose-database_name": "{.data['database-name']}" - } - }, - "stringData" : { - "database-user" : "${MYSQL_USER}", - "database-password" : "${MYSQL_PASSWORD}", - "database-root-password" : "${MYSQL_ROOT_PASSWORD}", - "database-name" : "${MYSQL_DATABASE}" - } - }, - { - "kind": "Service", - "apiVersion": "v1", - "metadata": { - "name": "${DATABASE_SERVICE_NAME}", - "annotations": { - "template.openshift.io/expose-uri": "mysql://{.spec.clusterIP}:{.spec.ports[?(.name==\"mariadb\")].port}" - } - }, - "spec": { - "ports": [ - { - "name": "mariadb", - "port": 3306 - } - ], - "selector": { - "name": "${DATABASE_SERVICE_NAME}" - } - } - }, - { - "kind": "DeploymentConfig", - "apiVersion": "v1", - "metadata": { - "name": "${DATABASE_SERVICE_NAME}", - "annotations": { - "template.alpha.openshift.io/wait-for-ready": "true" - } - }, - "spec": { - "strategy": { - "type": "Recreate" - }, - "triggers": [ - { - "type": "ImageChange", - "imageChangeParams": { - "automatic": true, - "containerNames": [ - "mariadb" - ], - "from": { - "kind": "ImageStreamTag", - "name": "mariadb:${MARIADB_VERSION}", - "namespace": "${NAMESPACE}" - } - } - }, - { - "type": "ConfigChange" - } - ], - "replicas": 1, - "selector": { - "name": "${DATABASE_SERVICE_NAME}" - }, - "template": { - "metadata": { - "labels": { - "name": "${DATABASE_SERVICE_NAME}" - } - }, - "spec": { - "containers": [ - { - "name": "mariadb", - "image": " ", - "ports": [ - { - "containerPort": 3306 - } - ], - "readinessProbe": { - "timeoutSeconds": 1, - "initialDelaySeconds": 5, - "exec": { - "command": [ "/bin/sh", "-i", "-c", - "MYSQL_PWD=\"$MYSQL_PASSWORD\" mysql -h 127.0.0.1 -u $MYSQL_USER -D $MYSQL_DATABASE -e 'SELECT 1'"] - } - }, - "livenessProbe": { - "timeoutSeconds": 1, - "initialDelaySeconds": 30, - "tcpSocket": { - "port": 3306 - } - }, - "env": [ - { - "name": "MYSQL_USER", - "valueFrom": { - "secretKeyRef" : { - "name" : "${DATABASE_SERVICE_NAME}", - "key" : "database-user" - } - } - }, - { - "name": "MYSQL_PASSWORD", - "valueFrom": { - "secretKeyRef" : { - "name" : "${DATABASE_SERVICE_NAME}", - "key" : "database-password" - } - } - }, - { - "name": "MYSQL_ROOT_PASSWORD", - "valueFrom": { - "secretKeyRef" : { - "name" : "${DATABASE_SERVICE_NAME}", - "key" : "database-root-password" - } - } - }, - { - "name": "MYSQL_DATABASE", - "valueFrom": { - "secretKeyRef" : { - "name" : "${DATABASE_SERVICE_NAME}", - "key" : "database-name" - } - } - } - ], - "resources": { - "limits": { - "memory": "${MEMORY_LIMIT}" - } - }, - "volumeMounts": [ - { - "name": "${DATABASE_SERVICE_NAME}-data", - "mountPath": "/var/lib/mysql/data" - } - ], - "imagePullPolicy": "IfNotPresent" - } - ], - "volumes": [ - { - "name": "${DATABASE_SERVICE_NAME}-data", - "emptyDir": { - "medium": "" - } - } - ] - } - } - } - } - ], - "parameters": [ - { - "name": "MEMORY_LIMIT", - "displayName": "Memory Limit", - "description": "Maximum amount of memory the container can use.", - "value": "512Mi", - "required": true - }, - { - "name": "NAMESPACE", - "displayName": "Namespace", - "description": "The OpenShift Namespace where the ImageStream resides.", - "value": "openshift" - }, - { - "name": "DATABASE_SERVICE_NAME", - "displayName": "Database Service Name", - "description": "The name of the OpenShift Service exposed for the database.", - "value": "mariadb", - "required": true - }, - { - "name": "MYSQL_USER", - "displayName": "MariaDB Connection Username", - "description": "Username for MariaDB user that will be used for accessing the database.", - "generate": "expression", - "from": "user[A-Z0-9]{3}", - "required": true - }, - { - "name": "MYSQL_PASSWORD", - "displayName": "MariaDB Connection Password", - "description": "Password for the MariaDB connection user.", - "generate": "expression", - "from": "[a-zA-Z0-9]{16}", - "required": true - }, - { - "name": "MYSQL_ROOT_PASSWORD", - "displayName": "MariaDB root Password", - "description": "Password for the MariaDB root user.", - "generate": "expression", - "from": "[a-zA-Z0-9]{16}", - "required": true - }, - { - "name": "MYSQL_DATABASE", - "displayName": "MariaDB Database Name", - "description": "Name of the MariaDB database accessed.", - "value": "sampledb", - "required": true - }, - { - "name": "MARIADB_VERSION", - "displayName": "Version of MariaDB Image", - "description": "Version of MariaDB image to be used (10.2 or latest).", - "value": "10.2", - "required": true - } - ] -} diff --git a/test/run b/test/run deleted file mode 100755 index 9d9bb0d..0000000 --- a/test/run +++ /dev/null @@ -1,631 +0,0 @@ -#!/bin/bash -# -# Test the MySQL image. -# -# IMAGE_NAME specifies the name of the candidate image used for testing. -# The image has to be available before this script is executed. -# - -set -o errexit -set -o nounset -shopt -s nullglob - -THISDIR=$(dirname ${BASH_SOURCE[0]}) -source ${THISDIR}/test-lib.sh - -TEST_LIST="\ -run_container_creation_tests -run_configuration_tests -run_general_tests -run_change_password_test -run_replication_test -run_doc_test -run_s2i_test -run_ssl_test -run_upgrade_test -" - -if [ -e "${IMAGE_NAME:-}" ] ; then - echo "Error: IMAGE_NAME must be specified" - exit 1 -fi - -CID_FILE_DIR=$(mktemp --suffix=mysql_test_cidfiles -d) -TESTSUITE_RESULT=1 -test_dir="$(readlink -f $(dirname "${BASH_SOURCE[0]}"))" - -s2i_args="--pull-policy=never " - -function cleanup() { - ct_cleanup - - if [ $TESTSUITE_RESULT -eq 0 ] ; then - echo "Tests for ${IMAGE_NAME} succeeded." - else - echo "Tests for ${IMAGE_NAME} failed." - fi -} -trap cleanup EXIT SIGINT - -function mysql_cmd() { - local container_ip="$1"; shift - local login="$1"; shift - local password="$1"; shift - docker run --rm ${CONTAINER_EXTRA_ARGS:-} "$IMAGE_NAME" mysql --host "$container_ip" -u"$login" -p"$password" "$@" db -} - -function test_connection() { - local name=$1 ; shift - local login=$1 ; shift - local password=$1 ; shift - local ip - ip=$(ct_get_cip $name) - echo " Testing MySQL connection to $ip..." - local max_attempts=20 - local sleep_time=2 - local i - local status='' - echo -n " Trying to connect..." - for i in $(seq $max_attempts); do - local status=$(docker inspect -f '{{.State.Status}}' $(ct_get_cid "${name}")) - if [ "${status}" != 'running' ] ; then - break; - fi - echo -n "." - if mysql_cmd "$ip" "$login" "$password" &>/dev/null <<< 'SELECT 1;'; then - echo " OK" - echo " Success!" - return 0 - fi - sleep $sleep_time - done - echo " FAIL" - echo " Giving up: Failed to connect." - if [ "${status}" == 'running' ] ; then - echo " Container is still running." - else - local exit_status=$(docker inspect -f '{{.State.ExitCode}}' ${name}) - echo " Container finised with exit code ${exit_status}." - fi - echo "Logs:" - docker logs $(ct_get_cid $name) - return 1 -} - -function test_mysql() { - local container_ip="$1" - local login="$2" - local password="$3" - - echo " Testing MySQL" - mysql_cmd "$container_ip" "$login" "$password" <<< 'CREATE TABLE tbl (col1 VARCHAR(20), col2 VARCHAR(20));' - mysql_cmd "$container_ip" "$login" "$password" <<< 'INSERT INTO tbl VALUES ("foo1", "bar1");' - mysql_cmd "$container_ip" "$login" "$password" <<< 'INSERT INTO tbl VALUES ("foo2", "bar2");' - mysql_cmd "$container_ip" "$login" "$password" <<< 'INSERT INTO tbl VALUES ("foo3", "bar3");' - mysql_cmd "$container_ip" "$login" "$password" <<< 'SELECT * FROM tbl;' - mysql_cmd "$container_ip" "$login" "$password" <<< 'DROP TABLE tbl;' - echo " Success!" -} - -function create_container() { - local name=$1 ; shift - cidfile="$CID_FILE_DIR/$name" - # create container with a cidfile in a directory for cleanup - local container_id - container_id="$(docker run ${DOCKER_ARGS:-} --cidfile $cidfile -d "$@" $IMAGE_NAME ${CONTAINER_ARGS:-})" - echo " Created container $container_id" -} - -function run_change_password_test() { - local tmpdir=$(mktemp -d) - chmod -R a+rwx "${tmpdir}" - - # Create MySQL container with persistent volume and set the initial password - create_container "testpass1" -e MYSQL_USER=user -e MYSQL_PASSWORD=foo \ - -e MYSQL_DATABASE=db -v ${tmpdir}:/var/lib/mysql/data:Z - test_connection testpass1 user foo - docker stop $(ct_get_cid testpass1) >/dev/null - - # Create second container with changed password - create_container "testpass2" -e MYSQL_USER=user -e MYSQL_PASSWORD=bar \ - -e MYSQL_DATABASE=db -v ${tmpdir}:/var/lib/mysql/data:Z - test_connection testpass2 user bar - - # The old password should not work anymore - if mysql_cmd "$(ct_get_cip testpass2)" user foo -e 'SELECT 1;'; then - return 1 - fi -} - -function run_replication_test() { - local cluster_args="-e MYSQL_MASTER_USER=master -e MYSQL_MASTER_PASSWORD=master -e MYSQL_DATABASE=db" - local max_attempts=30 - - # Run the MySQL master - docker run $cluster_args -e MYSQL_USER=user -e MYSQL_PASSWORD=foo \ - -e MYSQL_ROOT_PASSWORD=root \ - -e MYSQL_INNODB_BUFFER_POOL_SIZE=5M \ - -d --cidfile ${CID_FILE_DIR}/master.cid $IMAGE_NAME mysqld-master >/dev/null - local master_ip - master_ip=$(ct_get_cip master.cid) - - # Run the MySQL slave - docker run $cluster_args -e MYSQL_MASTER_SERVICE_NAME=${master_ip} \ - -e MYSQL_INNODB_BUFFER_POOL_SIZE=5M \ - -d --cidfile ${CID_FILE_DIR}/slave.cid $IMAGE_NAME mysqld-slave >/dev/null - local slave_ip - slave_ip=$(ct_get_cip slave.cid) - - # Now wait till the MASTER will see the SLAVE - local i - for i in $(seq $max_attempts); do - result="$(mysql_cmd "$master_ip" root root -e 'SHOW SLAVE HOSTS;' | grep "$slave_ip" || true)" - if [[ -n "${result}" ]]; then - echo "${slave_ip} successfully registered as SLAVE for ${master_ip}" - break - fi - if [[ "${i}" == "${max_attempts}" ]]; then - echo "The ${slave_ip} failed to register in MASTER" - echo "Dumping logs for $(ct_get_cid slave.cid)" - docker logs $(ct_get_cid slave.cid) - return 1 - fi - sleep 1 - done - - # do some real work to test replication in practice - mysql_cmd "$master_ip" root root -e "CREATE TABLE t1 (a INT); INSERT INTO t1 VALUES (24);" - - # read value from slave and check whether it is expectd - for i in $(seq $max_attempts); do - set +e - result="$(mysql_cmd "${slave_ip}" root root -e "select * from t1 \G" | grep -e ^a | grep 24)" - set -e - if [[ ! -z "${result}" ]]; then - echo "${slave_ip} successfully got value from MASTER ${master_ip}" - break - fi - if [[ "${i}" == "${max_attempts}" ]]; then - echo "The ${slave_ip} failed to see value added on MASTER" - echo "Dumping logs for $(ct_get_cid slave.cid)" - docker logs $(ct_get_cid slave.cid) - return 1 - fi - sleep 1 - done -} - -function assert_login_access() { - local container_ip=$1; shift - local USER=$1 ; shift - local PASS=$1 ; shift - local success=$1 ; shift - - if mysql_cmd "$container_ip" "$USER" "$PASS" <<< 'SELECT 1;' ; then - if $success ; then - echo " $USER($PASS) access granted as expected" - return - fi - else - if ! $success ; then - echo " $USER($PASS) access denied as expected" - return - fi - fi - echo " $USER($PASS) login assertion failed" - exit 1 -} - -function assert_local_access() { - local id="$1" ; shift - if docker exec $(ct_get_cid "$id") bash -c 'mysql -uroot <<< "SELECT 1;"' ; then - echo " local access granted as expected" - return - fi - echo " local access assertion failed" - return 1 -} - -# Make sure the invocation of docker run fails. -function assert_container_creation_fails() { - - # Time the docker run command. It should fail. If it doesn't fail, - # mysqld will keep running so we kill it with SIGKILL to make sure - # timeout returns a non-zero value. - local ret=0 - timeout -s 9 --preserve-status 60s docker run --rm "$@" $IMAGE_NAME >/dev/null || ret=$? - - # Timeout will exit with a high number. - if [ $ret -gt 30 ]; then - return 1 - fi - echo " Success!" -} - -function try_image_invalid_combinations() { - assert_container_creation_fails -e MYSQL_USER=user -e MYSQL_DATABASE=db "$@" - assert_container_creation_fails -e MYSQL_PASSWORD=pass -e MYSQL_DATABASE=db "$@" -} - -function run_container_creation_tests() { - echo " Testing image entrypoint usage" - assert_container_creation_fails - try_image_invalid_combinations - try_image_invalid_combinations -e MYSQL_ROOT_PASSWORD=root_pass - - local VERY_LONG_DB_NAME="very_long_database_name_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - local VERY_LONG_USER_NAME="very_long_user_name_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" - assert_container_creation_fails -e MYSQL_USER=user -e MYSQL_PASSWORD=pass - assert_container_creation_fails -e MYSQL_USER=\$invalid -e MYSQL_PASSWORD=pass -e MYSQL_DATABASE=db -e MYSQL_ROOT_PASSWORD=root_pass - assert_container_creation_fails -e MYSQL_USER=$VERY_LONG_USER_NAME -e MYSQL_PASSWORD=pass -e MYSQL_DATABASE=db -e MYSQL_ROOT_PASSWORD=root_pass - assert_container_creation_fails -e MYSQL_USER=user -e MYSQL_PASSWORD="\"" -e MYSQL_DATABASE=db -e MYSQL_ROOT_PASSWORD=root_pass - assert_container_creation_fails -e MYSQL_USER=user -e MYSQL_PASSWORD=pass -e MYSQL_DATABASE=\$invalid -e MYSQL_ROOT_PASSWORD=root_pass - assert_container_creation_fails -e MYSQL_USER=user -e MYSQL_PASSWORD=pass -e MYSQL_DATABASE=$VERY_LONG_DB_NAME -e MYSQL_ROOT_PASSWORD=root_pass - assert_container_creation_fails -e MYSQL_USER=user -e MYSQL_PASSWORD=pass -e MYSQL_DATABASE=db -e MYSQL_ROOT_PASSWORD="\"" - assert_container_creation_fails -e MYSQL_USER=root -e MYSQL_PASSWORD=pass -e MYSQL_DATABASE=db -e MYSQL_ROOT_PASSWORD=pass - echo " Success!" -} - -function test_config_option() { - local container_name="$1" - local configuration="$2" - local option_name="$3" - local option_value="$4" - - if ! echo "$configuration" | grep -qx "$option_name[[:space:]]*=[[:space:]]*$option_value"; then - local configs="$(docker exec "$(ct_get_cid $container_name)" bash -c 'set +f; shopt -s nullglob; echo /etc/my.cnf /etc/my.cnf.d/* /opt/rh/mysql*/root/etc/my.cnf /opt/rh/mysql*/root/etc/my.cnf.d/* | paste -s')" - echo >&2 "FAIL: option '$option_name' should have value '$option_value', but it wasn't found in any of the configuration files ($configs):" - echo >&2 - echo >&2 "$configuration" - echo >&2 - return 1 - fi - - return 0 -} - -function run_configuration_tests() { - echo " Testing image configuration settings" - - local container_name=config_test - - create_container \ - "$container_name" \ - --env MYSQL_USER=config_test_user \ - --env MYSQL_PASSWORD=config_test \ - --env MYSQL_DATABASE=db \ - --env MYSQL_LOWER_CASE_TABLE_NAMES=1 \ - --env MYSQL_LOG_QUERIES_ENABLED=1 \ - --env MYSQL_MAX_CONNECTIONS=1337 \ - --env MYSQL_FT_MIN_WORD_LEN=8 \ - --env MYSQL_FT_MAX_WORD_LEN=15 \ - --env MYSQL_MAX_ALLOWED_PACKET=10M \ - --env MYSQL_TABLE_OPEN_CACHE=100 \ - --env MYSQL_SORT_BUFFER_SIZE=256K \ - --env MYSQL_KEY_BUFFER_SIZE=16M \ - --env MYSQL_READ_BUFFER_SIZE=16M \ - --env MYSQL_INNODB_BUFFER_POOL_SIZE=16M \ - --env MYSQL_INNODB_LOG_FILE_SIZE=4M \ - --env MYSQL_INNODB_LOG_BUFFER_SIZE=4M \ - --env WORKAROUND_DOCKER_BUG_14203= - # - - test_connection "$container_name" config_test_user config_test - - # TODO: this check is far from perfect and could be improved: - # - we should look for an option in the desired config, not in all of them - # - we should respect section of the config (now we have duplicated options from a different sections) - local configuration - configuration="$(docker exec "$(ct_get_cid $container_name)" bash -c 'set +f; shopt -s nullglob; egrep -hv "^(#|\!|\[|$)" /etc/my.cnf /etc/my.cnf.d/* /opt/rh/mysql*/root/etc/my.cnf /opt/rh/mysql*/root/etc/my.cnf.d/*' | sed 's,\(^[[:space:]]\+\|[[:space:]]\+$\),,' | sort -u)" - - test_config_option "$container_name" "$configuration" lower_case_table_names 1 - test_config_option "$container_name" "$configuration" general_log 1 - test_config_option "$container_name" "$configuration" max_connections 1337 - test_config_option "$container_name" "$configuration" ft_min_word_len 8 - test_config_option "$container_name" "$configuration" ft_max_word_len 15 - test_config_option "$container_name" "$configuration" max_allowed_packet 10M - test_config_option "$container_name" "$configuration" table_open_cache 100 - test_config_option "$container_name" "$configuration" sort_buffer_size 256K - test_config_option "$container_name" "$configuration" key_buffer_size 16M - test_config_option "$container_name" "$configuration" read_buffer_size 16M - test_config_option "$container_name" "$configuration" innodb_buffer_pool_size 16M - test_config_option "$container_name" "$configuration" innodb_log_file_size 4M - test_config_option "$container_name" "$configuration" innodb_log_buffer_size 4M - - docker stop "$(ct_get_cid $container_name)" >/dev/null - - echo " Success!" - echo " Testing image auto-calculated configuration settings" - - container_name=dynamic_config_test - - DOCKER_ARGS='--memory=256m' create_container \ - "$container_name" \ - --env MYSQL_USER=config_test_user \ - --env MYSQL_PASSWORD=config_test \ - --env MYSQL_DATABASE=db - - test_connection "$container_name" config_test_user config_test - - configuration="$(docker exec "$(ct_get_cid $container_name)" bash -c 'set +f; shopt -s nullglob; egrep -hv "^(#|\!|\[|$)" /etc/my.cnf /etc/my.cnf.d/* /opt/rh/mysql*/root/etc/my.cnf /opt/rh/mysql*/root/etc/my.cnf.d/*' | sed 's,\(^[[:space:]]\+\|[[:space:]]\+$\),,' | sort -u)" - - test_config_option "$container_name" "$configuration" key_buffer_size 25M - test_config_option "$container_name" "$configuration" read_buffer_size 12M - test_config_option "$container_name" "$configuration" innodb_buffer_pool_size 128M - test_config_option "$container_name" "$configuration" innodb_log_file_size 38M - test_config_option "$container_name" "$configuration" innodb_log_buffer_size 38M - - docker stop "$(ct_get_cid $container_name)" >/dev/null - - echo " Success!" -} - -function run_tests() { - local name=$1 ; shift - envs="-e MYSQL_USER=$USER -e MYSQL_PASSWORD=$PASS -e MYSQL_DATABASE=db" - if [ -v ROOT_PASS ]; then - envs="$envs -e MYSQL_ROOT_PASSWORD=$ROOT_PASS" - fi - create_container $name $envs - test_connection "$name" "$USER" "$PASS" - echo " Testing scl usage" - ct_scl_usage_old $name 'mysql --version' "$VERSION" - echo " Testing login accesses" - local container_ip - container_ip=$(ct_get_cip $name) - assert_login_access "$container_ip" "$USER" "$PASS" true - assert_login_access "$container_ip" "$USER" "${PASS}_foo" false - if [ -v ROOT_PASS ]; then - assert_login_access "$container_ip" root "$ROOT_PASS" true - assert_login_access "$container_ip" root "${ROOT_PASS}_foo" false - else - assert_login_access "$container_ip" root 'foo' false - assert_login_access "$container_ip" root '' false - fi - assert_local_access "$name" - echo " Success!" - test_mysql "$container_ip" "$USER" "$PASS" -} - -run_doc_test() { - echo " Testing documentation in the container image" - ct_doc_content_old "MYSQL\_ROOT\_PASSWORD" volume 3306 - echo " Success!" - echo -} - -_s2i_test_image() { - local container_name="$1" - local mount_opts="$2" - echo " Testing s2i app image with invalid configuration" - assert_container_creation_fails -e MYSQL_USER=root -e MYSQL_PASSWORD=pass -e MYSQL_DATABASE=db -e MYSQL_ROOT_PASSWORD=pass - echo " Testing s2i app image with correct configuration" - create_container \ - "$container_name" \ - --env MYSQL_USER=config_test_user \ - --env MYSQL_PASSWORD=config_test \ - --env MYSQL_DATABASE=db \ - --env MYSQL_OPERATIONS_USER=operations_user \ - --env MYSQL_OPERATIONS_PASSWORD=operations_pass \ - ${mount_opts} - - test_connection "$container_name" operations_user operations_pass - - configuration="$(docker exec "$(ct_get_cid $container_name)" bash -c 'set +f; shopt -s nullglob; egrep -hv "^(#|\!|\[|$)" /etc/my.cnf /etc/my.cnf.d/* /opt/rh/mysql*/root/etc/my.cnf /opt/rh/mysql*/root/etc/my.cnf.d/*' | sed 's,\(^[[:space:]]\+\|[[:space:]]\+$\),,' | sort -u)" - - docker stop "$(ct_get_cid $container_name)" >/dev/null -} - -run_s2i_test() { - echo " Testing s2i usage" - ct_s2i_usage ${IMAGE_NAME} ${s2i_args} &>/dev/null - - echo " Testing s2i build" - ct_s2i_build_as_df file://${test_dir}/test-app ${IMAGE_NAME} ${IMAGE_NAME}-testapp - local image_name_backup=${IMAGE_NAME} - export IMAGE_NAME=${IMAGE_NAME}-testapp - - local container_name=s2i_config_build - _s2i_test_image "s2i_config_build" "" - - # return back original value for IMAGE_NAME - export IMAGE_NAME=${image_name_backup} - - echo " Testing s2i mount" - test_app_dir=$(mktemp -d) - cp -Lr ${test_dir}/test-app ${test_app_dir}/ - chown -R 27:27 ${test_app_dir} - _s2i_test_image "_s2i_test_mount" "-v ${test_app_dir}/test-app:/opt/app-root/src/:z" - rm -rf ${test_app_dir} - echo " Success!" -} - -gen_self_signed_cert() { - local output_dir=$1 ; shift - local base_name=$1 ; shift - mkdir -p ${output_dir} - openssl req -newkey rsa:2048 -nodes -keyout ${output_dir}/${base_name}-key.pem -subj '/C=GB/ST=Berkshire/L=Newbury/O=My Server Company' > ${output_dir}/${base_name}-req.pem - openssl req -new -x509 -nodes -key ${output_dir}/${base_name}-key.pem -batch > ${output_dir}/${base_name}-cert-selfsigned.pem -} - -run_ssl_test() { - echo " Testing ssl usage" - test_app_dir=$(mktemp -d) - mkdir -p ${test_app_dir}/{mysql-certs,mysql-cfg} - gen_self_signed_cert ${test_app_dir}/mysql-certs server - echo "[mysqld] -ssl-key=\${APP_DATA}/mysql-certs/server-key.pem -ssl-cert=\${APP_DATA}/mysql-certs/server-cert-selfsigned.pem -" >${test_app_dir}/mysql-cfg/ssl.cnf - chown -R 27:27 ${test_app_dir} - local ca_cert_path="/opt/app-root/src/mysql-certs/server-cert-selfsigned.pem" - - create_container \ - "_s2i_test_ssl" \ - --env MYSQL_USER=ssl_test_user \ - --env MYSQL_PASSWORD=ssl_test \ - --env MYSQL_DATABASE=db \ - -v ${test_app_dir}:/opt/app-root/src/:z - - test_connection "_s2i_test_ssl" ssl_test_user ssl_test - ip=$(ct_get_cip _s2i_test_ssl) - - # At least MySQL 5.6 requires ssl-ca option on client side, otherwise the ssl is not used - CONTAINER_EXTRA_ARGS="-v ${test_app_dir}:/opt/app-root/src/:z" - - # MySQL requires --ssl-mode to be set in order to require SSL - case ${VERSION} in - 5*) ssl_mode_opt='--ssl-mode=REQUIRED' - esac - - if mysql_cmd "$ip" "ssl_test_user" "ssl_test" ${ssl_mode_opt:-} --ssl-ca=${ca_cert_path} -e 'show status like "Ssl_cipher" \G' | grep 'Value: [A-Z][A-Z0-9-]*' ; then - echo " Success!" - rm -rf ${test_app_dir} - else - echo " FAIL!" - mysql_cmd "$ip" "ssl_test_user" "ssl_test" --ssl-ca=${ca_cert_path} -e 'show status like "%ssl%" \G' - return 1 - fi - # Clear the global variable content after we are done using it - CONTAINER_EXTRA_ARGS="" -} - -function run_general_tests() { - # Set lower buffer pool size to avoid running out of memory. - export CONTAINER_ARGS="run-mysqld --innodb_buffer_pool_size=5242880" - - # Normal tests - USER=user PASS=pass run_tests no_root - USER=user1 PASS=pass1 ROOT_PASS=r00t run_tests root - # Test with arbitrary uid for the container - DOCKER_ARGS="--user 12345" USER=user PASS=pass run_tests no_root_altuid - DOCKER_ARGS="--user 12345" USER=user1 PASS=pass1 ROOT_PASS=r00t run_tests root_altuid -} - -function get_previous_major_version() { - case "${1}" in - 5.5) echo "5.1" ;; - 5.6) echo "5.5" ;; - 5.7) echo "5.6" ;; - 8.0) echo "5.7" ;; - 10.0) echo "5.5" ;; - 10.1) echo "10.0" ;; - 10.2) echo "10.1" ;; - 10.3) echo "10.2" ;; - *) echo "Non expected version '${1}'" ; return 1 ;; - esac -} - -function run_upgrade_test() { - local tmpdir=$(mktemp -d) - echo " Testing upgrade of the container image" - mkdir "${tmpdir}/data" && chmod -R a+rwx "${tmpdir}" - - # Create MySQL container with persistent volume and set the version from too old version - local datadir=${tmpdir}/data - create_container "testupg1" -e MYSQL_USER=user -e MYSQL_PASSWORD=foo \ - -e MYSQL_DATABASE=db -v ${datadir}:/var/lib/mysql/data:Z - test_connection testupg1 user foo - docker stop $(ct_get_cid testupg1) >/dev/null - - # Simulate datadir without version information - rm -f ${datadir}/mysql_upgrade_info - echo " Testing upgrade from data without version" - # This should work, but warning should be printed - create_container "testupg2" -e MYSQL_USER=user -e MYSQL_PASSWORD=foo \ - -e MYSQL_DATABASE=db -v ${datadir}:/var/lib/mysql/data:Z - test_connection testupg2 user foo - docker stop $(ct_get_cid testupg2) >/dev/null - # Check whether some information is provided - if ! docker logs $(ct_get_cid testupg2) 2>&1 | grep -e 'Version of the data could not be determined' &>/dev/null ; then - echo "Information about missing version file is not available in the logs" - return 1 - fi - # Check whether upgrade did not happen - if docker logs $(ct_get_cid testupg2) 2>&1 | grep -e 'Running mysql_upgrade' &>/dev/null ; then - echo "Upgrade should not be run when information about version is missing" - return 1 - fi - - # Create version file that is too old - echo " Testing upgrade from too old data" - echo "5.0.12" >${datadir}/mysql_upgrade_info - # Create another container with same data and upgrade set to 'upgrade-auto' - assert_container_creation_fails -e MYSQL_USER=user -e MYSQL_PASSWORD=foo \ - -e MYSQL_DATABASE=db -v ${datadir}:/var/lib/mysql/data:Z -e MYSQL_DATADIR_ACTION=upgrade-auto 2>/dev/null - - # Create version file that we can upgrade from - echo " Testing upgrade from previous version" - echo "$(get_previous_major_version ${VERSION}).12" >${datadir}/mysql_upgrade_info - # Create another container with same data and upgrade set to 'upgrade-aauto' - create_container "testupg3" -e MYSQL_USER=user -e MYSQL_PASSWORD=foo \ - -e MYSQL_DATABASE=db -v ${datadir}:/var/lib/mysql/data:Z -e MYSQL_DATADIR_ACTION=upgrade-auto - test_connection testupg3 user foo - docker stop $(ct_get_cid testupg3) >/dev/null - # Check whether some upgrade happened - if ! docker logs $(ct_get_cid testupg3) 2>&1 | grep -qe 'Running mysql_upgrade' ; then - echo "Upgrade did not happen but it should when upgrading from previous version" - docker logs $(ct_get_cid testupg3) - return 1 - fi - - # Create version file that we don't need to upgrade from - echo " Testing upgrade from the same version" - echo "${VERSION}.12" >${datadir}/mysql_upgrade_info - # Create another container with same data and upgrade set to 'upgrade-aauto' - create_container "testupg4" -e MYSQL_USER=user -e MYSQL_PASSWORD=foo \ - -e MYSQL_DATABASE=db -v ${datadir}:/var/lib/mysql/data:Z -e MYSQL_DATADIR_ACTION=upgrade-auto - test_connection testupg4 user foo - docker stop $(ct_get_cid testupg4) >/dev/null - # Check whether some upgrade happened - if docker logs $(ct_get_cid testupg4) 2>&1 | grep -e 'Running mysql_upgrade' &>/dev/null ; then - echo "Upgrade happened but it should not when upgrading from current version" - return 1 - fi - - # Create second container with same data and upgrade set to 'analyze' - echo " Testing running --analyze" - create_container "testupg5" -e MYSQL_USER=user -e MYSQL_PASSWORD=foo \ - -e MYSQL_DATABASE=db -v ${datadir}:/var/lib/mysql/data:Z -e MYSQL_DATADIR_ACTION=analyze - test_connection testupg5 user foo - docker stop $(ct_get_cid testupg5) >/dev/null - # Check whether analyze happened - if ! docker logs $(ct_get_cid testupg5) 2>&1 | grep -e '--analyze --all-databases' &>/dev/null ; then - echo "Analyze did not happen but it should" - return 1 - fi - - # Create another container with same data and upgrade set to 'optimize' - echo " Testing running --optimize" - create_container "testupg6" -e MYSQL_USER=user -e MYSQL_PASSWORD=foo \ - -e MYSQL_DATABASE=db -v ${datadir}:/var/lib/mysql/data:Z -e MYSQL_DATADIR_ACTION=optimize - test_connection testupg6 user foo - docker stop $(ct_get_cid testupg6) >/dev/null - # Check whether optimize happened - if ! docker logs $(ct_get_cid testupg6) 2>&1 | grep -e '--optimize --all-databases' &>/dev/null ; then - echo "Optimize did not happen but it should" - return 1 - fi - - # Create version file that we cannot upgrade from - echo " Testing upgrade from the future version" - echo "20.1.12" >${datadir}/mysql_upgrade_info - assert_container_creation_fails -e MYSQL_USER=user -e MYSQL_PASSWORD=foo \ - -e MYSQL_DATABASE=db -v ${datadir}:/var/lib/mysql/data:Z -e MYSQL_DATADIR_ACTION=upgrade-auto 2>/dev/null - - echo " Upgrade tests succeeded!" - echo -} - -function run_all_tests() { - for test_case in $TEST_LIST; do - echo "Running test $test_case for ${IMAGE_NAME}" - $test_case - done; -} - -# Run the chosen tests -TEST_LIST=${@:-$TEST_LIST} run_all_tests - -TESTSUITE_RESULT=0 - diff --git a/test/run-openshift b/test/run-openshift deleted file mode 120000 index d84575f..0000000 --- a/test/run-openshift +++ /dev/null @@ -1 +0,0 @@ -run-openshift-local-cluster \ No newline at end of file diff --git a/test/run-openshift-local-cluster b/test/run-openshift-local-cluster deleted file mode 100755 index 8a39a40..0000000 --- a/test/run-openshift-local-cluster +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash -# -# Test the MariaDB image in OpenShift (local cluster) -# -# IMAGE_NAME specifies a name of the candidate image used for testing. -# The image has to be available before this script is executed. -# VERSION specifies the major version of the MariaDB in format of X.Y -# OS specifies RHEL version (e.g. OS=rhel7) -# - -THISDIR=$(dirname ${BASH_SOURCE[0]}) - -source ${THISDIR}/test-lib-mysql.sh - -set -eo nounset - -trap ct_os_cleanup EXIT SIGINT - -ct_os_check_compulsory_vars - -ct_os_cluster_up - -test_mysql_pure_image "${IMAGE_NAME}" - -test_mysql_template "${IMAGE_NAME}" - -# TODO: Can we make the build against examples inside the same PR? -test_mysql_s2i "${IMAGE_NAME}" "https://github.com/sclorg/mariadb-container.git" test/test-app - -test_mariadb_integration "${IMAGE_NAME}" "${VERSION}" mariadb - -# test with a released image and an integrated template -# ignore possible failure of this test for centos images -fail_not_released=true -if [ "${OS}" == "rhel7" ] ; then - PUBLIC_IMAGE_NAME=${PUBLIC_IMAGE_NAME:-${REGISTRY:-registry.redhat.io/}rhscl/${BASE_IMAGE_NAME}-${VERSION//./}-rhel7} -else - PUBLIC_IMAGE_NAME=${PUBLIC_IMAGE_NAME:-${REGISTRY:-}centos/${BASE_IMAGE_NAME}-${VERSION//./}-centos7} - fail_not_released=false -fi - -export CT_SKIP_UPLOAD_IMAGE=true -# Try pulling the image first to see if it is accessible -if docker pull "${PUBLIC_IMAGE_NAME}"; then - test_mariadb_integration mariadb "${VERSION}" "${PUBLIC_IMAGE_NAME}" -else - echo "Warning: ${PUBLIC_IMAGE_NAME} could not be downloaded via 'docker'" - ! $fail_not_released || false "ERROR: Failed to pull image" -fi - -OS_TESTSUITE_RESULT=0 - -ct_os_cluster_down - diff --git a/test/run-openshift-remote-cluster b/test/run-openshift-remote-cluster deleted file mode 100755 index f52c670..0000000 --- a/test/run-openshift-remote-cluster +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -# -# Test the MariaDB image in OpenShift (remote cluster) -# -# IMAGE_NAME specifies a name of the candidate image used for testing. -# The image has to be available before this script is executed. -# VERSION specifies the major version of the MariaDB in format of X.Y -# OS specifies RHEL version (e.g. OS=rhel7) -# - -THISDIR=$(dirname ${BASH_SOURCE[0]}) - -source ${THISDIR}/test-lib-mysql.sh - -set -eo nounset - -trap ct_os_cleanup EXIT SIGINT - -ct_os_check_compulsory_vars - -oc status || false "It looks like oc is not properly logged in." - -export CT_SKIP_NEW_PROJECT=true -export CT_SKIP_UPLOAD_IMAGE=true -export CT_NAMESPACE=openshift - -test_mariadb_integration mariadb ${VERSION} "${IMAGE_NAME}" - -OS_TESTSUITE_RESULT=0 - diff --git a/test/test-app/mysql-cfg/myconfig.cnf b/test/test-app/mysql-cfg/myconfig.cnf deleted file mode 100644 index 7764adf..0000000 --- a/test/test-app/mysql-cfg/myconfig.cnf +++ /dev/null @@ -1,3 +0,0 @@ -[mysqld] -query-cache-limit=262144 - diff --git a/test/test-app/mysql-data/init.sql b/test/test-app/mysql-data/init.sql deleted file mode 100644 index 3159982..0000000 --- a/test/test-app/mysql-data/init.sql +++ /dev/null @@ -1,4 +0,0 @@ -CREATE TABLE products (id INTEGER, name VARCHAR(256), price FLOAT, variant INTEGER); -CREATE TABLE products_variant (id INTEGER, name VARCHAR(256)); -INSERT INTO products_variant (id, name) VALUES ('1', 'blue'), ('2', 'green'); - diff --git a/test/test-app/mysql-init/80-add-arbitrary-users.sh b/test/test-app/mysql-init/80-add-arbitrary-users.sh deleted file mode 100644 index 55ae2d2..0000000 --- a/test/test-app/mysql-init/80-add-arbitrary-users.sh +++ /dev/null @@ -1,17 +0,0 @@ -create_arbitrary_users() { - # Do not care what option is compulsory here, just create what is specified - log_info "Creating user specified by MYSQL_OPERATIONS_USER (${MYSQL_OPERATIONS_USER}) ..." -mysql $mysql_flags < testdb -utestu -ptestp\" '^42' 120" \ - "-p MARIADB_VERSION=${VERSION} \ - -p DATABASE_SERVICE_NAME="${service_name}-testing" \ - -p MYSQL_USER=testu \ - -p MYSQL_PASSWORD=testp \ - -p MYSQL_DATABASE=testdb" "" "${import_image}" -} - -# vim: set tabstop=2:shiftwidth=2:expandtab: diff --git a/test/test-lib-openshift.sh b/test/test-lib-openshift.sh deleted file mode 100644 index f62eab6..0000000 --- a/test/test-lib-openshift.sh +++ /dev/null @@ -1,1043 +0,0 @@ -# shellcheck shell=bash -# some functions are used from test-lib.sh, that is usually in the same dir -# shellcheck source=/dev/null -source "$(dirname "${BASH_SOURCE[0]}")"/test-lib.sh - -# Set of functions for testing docker images in OpenShift using 'oc' command - -# A variable containing the overall test result; must be changed to 0 in the end -# of the testing script: -# OS_TESTSUITE_RESULT=0 -# And the following trap must be set, in the beginning of the test script: -# trap ct_os_cleanup EXIT SIGINT -OS_TESTSUITE_RESULT=1 -OS_CLUSTER_STARTED_BY_TEST=0 - -function ct_os_cleanup() { - if [ $OS_TESTSUITE_RESULT -eq 0 ] ; then - # shellcheck disable=SC2153 - echo "OpenShift tests for ${IMAGE_NAME} succeeded." - else - # shellcheck disable=SC2153 - echo "OpenShift tests for ${IMAGE_NAME} failed." - fi -} - -# ct_os_check_compulsory_vars -# --------------------------- -# Check the compulsory variables: -# * IMAGE_NAME specifies a name of the candidate image used for testing. -# * VERSION specifies the major version of the MariaDB in format of X.Y -# * OS specifies RHEL version (e.g. OS=rhel7) -function ct_os_check_compulsory_vars() { - # shellcheck disable=SC2016 - test -n "${IMAGE_NAME-}" || ( echo 'make sure $IMAGE_NAME is defined' >&2 ; exit 1) - # shellcheck disable=SC2016 - test -n "${VERSION-}" || ( echo 'make sure $VERSION is defined' >&2 ; exit 1) - # shellcheck disable=SC2016 - test -n "${OS-}" || ( echo 'make sure $OS is defined' >&2 ; exit 1) -} - -# ct_os_get_status -# -------------------- -# Returns status of all objects to make debugging easier. -function ct_os_get_status() { - oc get all - oc status -} - -# ct_os_print_logs -# -------------------- -# Returns status of all objects and logs from all pods. -function ct_os_print_logs() { - ct_os_get_status - while read -r pod_name; do - echo "INFO: printing logs for pod ${pod_name}" - oc logs "${pod_name}" - done < <(oc get pods --no-headers=true -o custom-columns=NAME:.metadata.name) -} - -# ct_os_enable_print_logs -# -------------------- -# Enables automatic printing of pod logs on ERR. -function ct_os_enable_print_logs() { - set -E - trap ct_os_print_logs ERR -} - -# ct_get_public_ip -# -------------------- -# Returns best guess for the IP that the node is accessible from other computers. -# This is a bit funny heuristic, simply goes through all IPv4 addresses that -# hostname -I returns and de-prioritizes IP addresses commonly used for local -# addressing. The rest of addresses are taken as public with higher probability. -function ct_get_public_ip() { - local hostnames - local public_ip='' - local found_ip - hostnames=$(hostname -I) - for guess_exp in '127\.0\.0\.1' '192\.168\.[0-9\.]*' '172\.[0-9\.]*' \ - '10\.[0-9\.]*' '[0-9\.]*' ; do - found_ip=$(echo "${hostnames}" | grep -oe "${guess_exp}") - if [ -n "${found_ip}" ] ; then - # shellcheck disable=SC2001 - hostnames=$(echo "${hostnames}" | sed -e "s/${found_ip}//") - public_ip="${found_ip}" - fi - done - if [ -z "${public_ip}" ] ; then - echo "ERROR: public IP could not be guessed." >&2 - return 1 - fi - echo "${public_ip}" -} - -# ct_os_run_in_pod POD_NAME CMD -# -------------------- -# Runs [cmd] in the pod specified by prefix [pod_prefix]. -# Arguments: pod_name - full name of the pod -# Arguments: cmd - command to be run in the pod -function ct_os_run_in_pod() { - local pod_name="$1" ; shift - - oc exec "$pod_name" -- "$@" -} - -# ct_os_get_service_ip SERVICE_NAME -# -------------------- -# Returns IP of the service specified by [service_name]. -# Arguments: service_name - name of the service -function ct_os_get_service_ip() { - local service_name="${1}" ; shift - oc get "svc/${service_name}" -o yaml | grep clusterIP | \ - cut -d':' -f2 | grep -oe '172\.30\.[0-9\.]*' -} - - -# ct_os_get_all_pods_status -# -------------------- -# Returns status of all pods. -function ct_os_get_all_pods_status() { - oc get pods -o custom-columns=Ready:status.containerStatuses[0].ready,NAME:.metadata.name -} - -# ct_os_get_all_pods_name -# -------------------- -# Returns the full name of all pods. -function ct_os_get_all_pods_name() { - oc get pods --no-headers -o custom-columns=NAME:.metadata.name -} - -# ct_os_get_pod_status POD_PREFIX -# -------------------- -# Returns status of the pod specified by prefix [pod_prefix]. -# Note: Ignores -build and -deploy pods -# Arguments: pod_prefix - prefix or whole ID of the pod -function ct_os_get_pod_status() { - local pod_prefix="${1}" ; shift - ct_os_get_all_pods_status | grep -e "${pod_prefix}" | grep -Ev "(build|deploy)$" \ - | awk '{print $1}' | head -n 1 -} - -# ct_os_get_pod_name POD_PREFIX -# -------------------- -# Returns the full name of pods specified by prefix [pod_prefix]. -# Note: Ignores -build and -deploy pods -# Arguments: pod_prefix - prefix or whole ID of the pod -function ct_os_get_pod_name() { - local pod_prefix="${1}" ; shift - ct_os_get_all_pods_name | grep -e "^${pod_prefix}" | grep -Ev "(build|deploy)$" -} - -# ct_os_get_pod_ip POD_NAME -# -------------------- -# Returns the ip of the pod specified by [pod_name]. -# Arguments: pod_name - full name of the pod -function ct_os_get_pod_ip() { - local pod_name="${1}" - oc get pod "$pod_name" --no-headers -o custom-columns=IP:status.podIP -} - -# ct_os_check_pod_readiness POD_PREFIX STATUS -# -------------------- -# Checks whether the pod is ready. -# Arguments: pod_prefix - prefix or whole ID of the pod -# Arguments: status - expected status (true, false) -function ct_os_check_pod_readiness() { - local pod_prefix="${1}" ; shift - local status="${1}" ; shift - test "$(ct_os_get_pod_status "${pod_prefix}")" == "${status}" -} - -# ct_os_wait_pod_ready POD_PREFIX TIMEOUT -# -------------------- -# Wait maximum [timeout] for the pod becomming ready. -# Arguments: pod_prefix - prefix or whole ID of the pod -# Arguments: timeout - how many seconds to wait seconds -function ct_os_wait_pod_ready() { - local pod_prefix="${1}" ; shift - local timeout="${1}" ; shift - SECONDS=0 - echo -n "Waiting for ${pod_prefix} pod becoming ready ..." - while ! ct_os_check_pod_readiness "${pod_prefix}" "true" ; do - echo -n "." - [ "${SECONDS}" -gt "${timeout}" ] && echo " FAIL" && return 1 - sleep 3 - done - echo " DONE" -} - -# ct_os_wait_rc_ready POD_PREFIX TIMEOUT -# -------------------- -# Wait maximum [timeout] for the rc having desired number of replicas ready. -# Arguments: pod_prefix - prefix of the replication controller -# Arguments: timeout - how many seconds to wait seconds -function ct_os_wait_rc_ready() { - local pod_prefix="${1}" ; shift - local timeout="${1}" ; shift - SECONDS=0 - echo -n "Waiting for ${pod_prefix} pod becoming ready ..." - while ! test "$( (oc get --no-headers statefulsets; oc get --no-headers rc) 2>/dev/null \ - | grep "^${pod_prefix}" | awk '$2==$3 {print "ready"}')" == "ready" ; do - echo -n "." - [ "${SECONDS}" -gt "${timeout}" ] && echo " FAIL" && return 1 - sleep 3 - done - echo " DONE" -} - -# ct_os_deploy_pure_image IMAGE [ENV_PARAMS, ...] -# -------------------- -# Runs [image] in the openshift and optionally specifies env_params -# as environment variables to the image. -# Arguments: image - prefix or whole ID of the pod to run the cmd in -# Arguments: env_params - environment variables parameters for the images. -function ct_os_deploy_pure_image() { - local image="${1}" ; shift - # ignore error exit code, because oc new-app returns error when image exists - oc new-app "${image}" "$@" || : - # let openshift cluster to sync to avoid some race condition errors - sleep 3 -} - -# ct_os_deploy_s2i_image IMAGE APP [ENV_PARAMS, ... ] -# -------------------- -# Runs [image] and [app] in the openshift and optionally specifies env_params -# as environment variables to the image. -# Arguments: image - prefix or whole ID of the pod to run the cmd in -# Arguments: app - url or local path to git repo with the application sources. -# Arguments: env_params - environment variables parameters for the images. -function ct_os_deploy_s2i_image() { - local image="${1}" ; shift - local app="${1}" ; shift - # ignore error exit code, because oc new-app returns error when image exists - oc new-app "${image}~${app}" "$@" || : - - # let openshift cluster to sync to avoid some race condition errors - sleep 3 -} - -# ct_os_deploy_template_image TEMPLATE [ENV_PARAMS, ...] -# -------------------- -# Runs template in the openshift and optionally gives env_params to use -# specific values in the template. -# Arguments: template - prefix or whole ID of the pod to run the cmd in -# Arguments: env_params - environment variables parameters for the template. -# Example usage: ct_os_deploy_template_image mariadb-ephemeral-template.yaml \ -# DATABASE_SERVICE_NAME=mysql-57-centos7 \ -# DATABASE_IMAGE=mysql-57-centos7 \ -# MYSQL_USER=testu \ -# MYSQL_PASSWORD=testp \ -# MYSQL_DATABASE=testdb -function ct_os_deploy_template_image() { - local template="${1}" ; shift - oc process -f "${template}" "$@" | oc create -f - - # let openshift cluster to sync to avoid some race condition errors - sleep 3 -} - -# _ct_os_get_uniq_project_name -# -------------------- -# Returns a uniq name of the OpenShift project. -function _ct_os_get_uniq_project_name() { - local r - while true ; do - r=${RANDOM} - mkdir /var/tmp/sclorg-test-${r} &>/dev/null && echo sclorg-test-${r} && break - done -} - -# ct_os_new_project [PROJECT] -# -------------------- -# Creates a new project in the openshfit using 'os' command. -# Arguments: project - project name, uses a new random name if omitted -# Expects 'os' command that is properly logged in to the OpenShift cluster. -# Not using mktemp, because we cannot use uppercase characters. -# The OPENSHIFT_CLUSTER_PULLSECRET_PATH environment variable can be set -# to contain a path to a k8s secret definition which will be used -# to authenticate to image registries. -# shellcheck disable=SC2120 -function ct_os_new_project() { - if [ "${CT_SKIP_NEW_PROJECT:-false}" == 'true' ] ; then - echo "Creating project skipped." - return - fi - local project_name="${1:-$(_ct_os_get_uniq_project_name)}" ; shift || : - oc new-project "${project_name}" - # let openshift cluster to sync to avoid some race condition errors - sleep 3 - if test -n "${OPENSHIFT_CLUSTER_PULLSECRET_PATH:-}" -a -e "${OPENSHIFT_CLUSTER_PULLSECRET_PATH:-}"; then - oc create -f "$OPENSHIFT_CLUSTER_PULLSECRET_PATH" - # add registry pullsecret to the serviceaccount if provided - secret_name=$(grep '^\s*name:' "$OPENSHIFT_CLUSTER_PULLSECRET_PATH" | awk '{ print $2 }') - secret_json='{"imagePullSecrets": [{"name": "'${secret_name}'"}]}' - oc patch serviceaccount default -p "$secret_json" - fi -} - -# ct_os_delete_project [PROJECT] -# -------------------- -# Deletes the specified project in the openshfit -# Arguments: project - project name, uses the current project if omitted -# shellcheck disable=SC2120 -function ct_os_delete_project() { - if [ "${CT_SKIP_NEW_PROJECT:-false}" == 'true' ] ; then - echo "Deleting project skipped, cleaning objects only." - # when not having enough privileges (remote cluster), it might fail and - # it is not a big problem, so ignore failure in this case - ct_delete_all_objects || : - return - fi - local project_name="${1:-$(oc project -q)}" ; shift || : - oc delete project "${project_name}" -} - -# ct_delete_all_objects -# ----------------- -# Deletes all objects within the project. -# Handy when we have one project and want to run more tests. -function ct_delete_all_objects() { - for x in bc builds dc is isimage istag po pv pvc rc routes secrets svc ; do - oc delete "$x" --all - done - # for some objects it takes longer to be really deleted, so a dummy sleep - # to avoid some races when other test can see not-yet-deleted objects and can fail - sleep 10 -} - -# ct_os_docker_login -# -------------------- -# Logs in into docker daemon -# Uses global REGISRTY_ADDRESS environment variable for arbitrary registry address. -# Does not do anything if REGISTRY_ADDRESS is set. -function ct_os_docker_login() { - [ -n "${REGISTRY_ADDRESS:-}" ] && "REGISTRY_ADDRESS set, not trying to docker login." && return 0 - # docker login fails with "404 page not found" error sometimes, just try it more times - # shellcheck disable=SC2034 - for i in $(seq 12) ; do - # shellcheck disable=SC2015 - docker login -u developer -p "$(oc whoami -t)" "${REGISRTY_ADDRESS:-172.30.1.1:5000}" && return 0 || : - sleep 5 - done - return 1 -} - -# ct_os_upload_image IMAGE [IMAGESTREAM] -# -------------------- -# Uploads image from local registry to the OpenShift internal registry. -# Arguments: image - image name to upload -# Arguments: imagestream - name and tag to use for the internal registry. -# In the format of name:tag ($image_name:latest by default) -# Uses global REGISRTY_ADDRESS environment variable for arbitrary registry address. -function ct_os_upload_image() { - local input_name="${1}" ; shift - local image_name=${input_name##*/} - local imagestream=${1:-$image_name:latest} - local output_name - - output_name="${REGISRTY_ADDRESS:-172.30.1.1:5000}/$(oc project -q)/$imagestream" - - ct_os_docker_login - docker tag "${input_name}" "${output_name}" - docker push "${output_name}" -} - -# ct_os_is_tag_exists IS_NAME TAG -# -------------------- -# Checks whether the specified tag exists for an image stream -# Arguments: is_name - name of the image stream -# Arguments: tag - name of the tag (usually version) -function ct_os_is_tag_exists() { - local is_name=$1 ; shift - local tag=$1 ; shift - oc get is "${is_name}" -n openshift -o=jsonpath='{.spec.tags[*].name}' | grep -qw "${tag}" -} - -# ct_os_template_exists T_NAME -# -------------------- -# Checks whether the specified template exists for an image stream -# Arguments: t_name - template name of the image stream -function ct_os_template_exists() { - local t_name=$1 ; shift - oc get templates -n openshift | grep -q "^${t_name}\s" -} - -# ct_os_install_in_centos -# -------------------- -# Installs os cluster in CentOS -function ct_os_install_in_centos() { - yum install -y centos-release-openshift-origin - yum install -y wget git net-tools bind-utils iptables-services bridge-utils\ - bash-completion origin-clients docker origin-clients -} - -# ct_os_cluster_up [DIR, IS_PUBLIC, CLUSTER_VERSION] -# -------------------- -# Runs the local OpenShift cluster using 'oc cluster up' and logs in as developer. -# Arguments: dir - directory to keep configuration data in, random if omitted -# Arguments: is_public - sets either private or public hostname for web-UI, -# use "true" for allow remote access to the web-UI, -# "false" is default -# Arguments: cluster_version - version of the OpenShift cluster to use, empty -# means default version of `oc`; example value: 3.7; -# also can be specified outside by OC_CLUSTER_VERSION -function ct_os_cluster_up() { - ct_os_cluster_running && echo "Cluster already running. Nothing is done." && return 0 - ct_os_logged_in && echo "Already logged in to a cluster. Nothing is done." && return 0 - - mkdir -p /var/tmp/openshift - local dir="${1:-$(mktemp -d /var/tmp/openshift/os-data-XXXXXX)}" ; shift || : - local is_public="${1:-'false'}" ; shift || : - local default_cluster_version=${OC_CLUSTER_VERSION:-} - local cluster_version=${1:-${default_cluster_version}} ; shift || : - if ! grep -qe '--insecure-registry.*172\.30\.0\.0' /etc/sysconfig/docker ; then - sed -i "s|OPTIONS='|OPTIONS='--insecure-registry 172.30.0.0/16 |" /etc/sysconfig/docker - fi - - systemctl stop firewalld || : - setenforce 0 - iptables -F - - systemctl restart docker - local cluster_ip="127.0.0.1" - [ "${is_public}" == "true" ] && cluster_ip=$(ct_get_public_ip) - - if [ -n "${cluster_version}" ] ; then - # if $cluster_version is not set, we simply use oc that is available - ct_os_set_path_oc "${cluster_version}" - fi - - mkdir -p "${dir}"/{config,data,pv} - case $(oc version| head -n 1) in - "oc v3.1"?.*) - oc cluster up --base-dir="${dir}/data" --public-hostname="${cluster_ip}" - ;; - "oc v3."*) - oc cluster up --host-data-dir="${dir}/data" --host-config-dir="${dir}/config" \ - --host-pv-dir="${dir}/pv" --use-existing-config --public-hostname="${cluster_ip}" - ;; - *) - echo "ERROR: Unexpected oc version." >&2 - return 1 - ;; - esac - oc version - oc login -u system:admin - oc project default - ct_os_wait_rc_ready docker-registry 180 - ct_os_wait_rc_ready router 30 - oc login -u developer -p developer - OS_CLUSTER_STARTED_BY_TEST=1 - # let openshift cluster to sync to avoid some race condition errors - sleep 3 -} - -# ct_os_cluster_down -# -------------------- -# Shuts down the local OpenShift cluster using 'oc cluster down' -function ct_os_cluster_down() { - if [ ${OS_CLUSTER_STARTED_BY_TEST:-0} -eq 1 ] ; then - echo "Cluster started by the test, shutting down." - oc cluster down - else - echo "Cluster not started by the test, shutting down skipped." - fi -} - -# ct_os_cluster_running -# -------------------- -# Returns 0 if oc cluster is running -function ct_os_cluster_running() { - oc cluster status &>/dev/null -} - -# ct_os_logged_in -# --------------- -# Returns 0 if logged in to a cluster (remote or local) -function ct_os_logged_in() { - oc whoami >/dev/null -} - -# ct_os_set_path_oc OC_VERSION -# -------------------- -# This is a trick that helps using correct version of the `oc`: -# The input is version of the openshift in format v3.6.0 etc. -# If the currently available version of oc is not of this version, -# it first takes a look into /usr/local/oc-/bin directory, -# and if not found there it downloads the community release from github. -# In the end the PATH variable is changed, so the other tests can still use just 'oc'. -# Arguments: oc_version - X.Y part of the version of OSE (e.g. 3.9) -function ct_os_set_path_oc() { - local oc_version - local oc_path - - oc_version=$(ct_os_get_latest_ver "$1") - - if oc version | grep -q "oc ${oc_version%.*}." ; then - echo "Binary oc found already available in version ${oc_version}: $(command -v oc) Doing noting." - return 0 - fi - - # first check whether we already have oc available in /usr/local - local installed_oc_path="/usr/local/oc-${oc_version%.*}/bin" - - if [ -x "${installed_oc_path}/oc" ] ; then - oc_path="${installed_oc_path}" - echo "Binary oc found in ${installed_oc_path}" >&2 - else - # oc not available in /usr/local, try to download it from github (community release) - oc_path="/tmp/oc-${oc_version}-bin" - ct_os_download_upstream_oc "${oc_version}" "${oc_path}" - fi - if [ -z "${oc_path}" ] ; then - echo "ERROR: oc not found installed, nor downloaded" >&1 - return 1 - fi - export PATH="${oc_path}:${PATH}" - if ! oc version | grep -q "oc ${oc_version%.*}." ; then - echo "ERROR: something went wrong, oc located at ${oc_path}, but oc of version ${oc_version} not found in PATH ($PATH)" >&1 - return 1 - else - echo "PATH set correctly, binary oc found in version ${oc_version}: $(command -v oc)" - fi -} - -# ct_os_get_latest_ver VERSION_PART_X -# -------------------- -# Returns full version (vX.Y.Z) from part of the version (X.Y) -# Arguments: vxy - X.Y part of the version -# Returns vX.Y.Z variant of the version -function ct_os_get_latest_ver(){ - local vxy="v$1" - for vz in {3..0} ; do - curl -sif "https://github.com/openshift/origin/releases/tag/${vxy}.${vz}" >/dev/null && echo "${vxy}.${vz}" && return 0 - done - echo "ERROR: version ${vxy} not found in https://github.com/openshift/origin/tags" >&2 - return 1 -} - -# ct_os_download_upstream_oc OC_VERSION OUTPUT_DIR -# -------------------- -# Downloads a particular version of openshift-origin-client-tools from -# github into specified output directory -# Arguments: oc_version - version of OSE (e.g. v3.7.2) -# Arguments: output_dir - output directory -function ct_os_download_upstream_oc() { - local oc_version=$1 - local output_dir=$2 - - # check whether we already have the binary in place - [ -x "${output_dir}/oc" ] && return 0 - - mkdir -p "${output_dir}" - # using html output instead of https://api.github.com/repos/openshift/origin/releases/tags/${oc_version}, - # because API is limited for number of queries if not authenticated - tarball=$(curl -si "https://github.com/openshift/origin/releases/tag/${oc_version}" | grep -o -e "openshift-origin-client-tools-${oc_version}-[a-f0-9]*-linux-64bit.tar.gz" | head -n 1) - - # download, unpack the binaries and then put them into output directory - echo "Downloading https://github.com/openshift/origin/releases/download/${oc_version}/${tarball} into ${output_dir}/" >&2 - curl -sL https://github.com/openshift/origin/releases/download/"${oc_version}"/"${tarball}" | tar -C "${output_dir}" -xz - mv -f "${output_dir}"/"${tarball%.tar.gz}"/* "${output_dir}/" - - rmdir "${output_dir}"/"${tarball%.tar.gz}" -} - - -# ct_os_test_s2i_app_func IMAGE APP CONTEXT_DIR CHECK_CMD [OC_ARGS] -# -------------------- -# Runs [image] and [app] in the openshift and optionally specifies env_params -# as environment variables to the image. Then check the container by arbitrary -# function given as argument (such an argument may include string, -# that will be replaced with actual IP). -# Arguments: image - prefix or whole ID of the pod to run the cmd in (compulsory) -# Arguments: app - url or local path to git repo with the application sources (compulsory) -# Arguments: context_dir - sub-directory inside the repository with the application sources (compulsory) -# Arguments: check_command - CMD line that checks whether the container works (compulsory; '' will be replaced with actual IP) -# Arguments: oc_args - all other arguments are used as additional parameters for the `oc new-app` -# command, typically environment variables (optional) -function ct_os_test_s2i_app_func() { - local image_name=${1} - local app=${2} - local context_dir=${3} - local check_command=${4} - local oc_args=${5:-} - local import_image=${6:-} - local image_name_no_namespace=${image_name##*/} - local service_name="${image_name_no_namespace}-testing" - local image_tagged="${image_name_no_namespace}:${VERSION}" - - if [ $# -lt 4 ] || [ -z "${1}" ] || [ -z "${2}" ] || [ -z "${3}" ] || [ -z "${4}" ]; then - echo "ERROR: ct_os_test_s2i_app_func() requires at least 4 arguments that cannot be emtpy." >&2 - return 1 - fi - - # shellcheck disable=SC2119 - ct_os_new_project - # Create a specific imagestream tag for the image so that oc cannot use anything else - if [ "${CT_SKIP_UPLOAD_IMAGE:-false}" == 'true' ] ; then - if [ -n "${import_image}" ] ; then - echo "Importing image ${import_image} as ${image_name}:${VERSION}" - oc import-image "${image_name}":"${VERSION}" --from "${import_image}" --confirm - else - echo "Uploading and importing image skipped." - fi - else - if [ -n "${import_image}" ] ; then - echo "Warning: Import image ${import_image} requested, but uploading image ${image_name} instead." - fi - ct_os_upload_image "${image_name}" "${image_tagged}" - fi - - local app_param="${app}" - if [ -d "${app}" ] ; then - # for local directory, we need to copy the content, otherwise too smart os command - # pulls the git remote repository instead - app_param=$(ct_obtain_input "${app}") - fi - - # shellcheck disable=SC2086 - ct_os_deploy_s2i_image "${image_tagged}" "${app_param}" \ - --context-dir="${context_dir}" \ - --name "${service_name}" \ - ${oc_args} - - if [ -d "${app}" ] ; then - # in order to avoid weird race seen sometimes, let's wait shortly - # before starting the build explicitly - sleep 5 - oc start-build "${service_name}" --from-dir="${app_param}" - fi - - ct_os_wait_pod_ready "${service_name}" 300 - - local ip - local check_command_exp - - ip=$(ct_os_get_service_ip "${service_name}") - # shellcheck disable=SC2001 - check_command_exp=$(echo "$check_command" | sed -e "s//$ip/g") - - echo " Checking APP using $check_command_exp ..." - local result=0 - eval "$check_command_exp" || result=1 - - if [ $result -eq 0 ] ; then - echo " Check passed." - else - echo " Check failed." - fi - - # shellcheck disable=SC2119 - ct_os_delete_project - return $result -} - -# ct_os_test_s2i_app IMAGE APP CONTEXT_DIR EXPECTED_OUTPUT [PORT, PROTOCOL, RESPONSE_CODE, OC_ARGS, ... ] -# -------------------- -# Runs [image] and [app] in the openshift and optionally specifies env_params -# as environment variables to the image. Then check the http response. -# Arguments: image - prefix or whole ID of the pod to run the cmd in (compulsory) -# Arguments: app - url or local path to git repo with the application sources (compulsory) -# Arguments: context_dir - sub-directory inside the repository with the application sources (compulsory) -# Arguments: expected_output - PCRE regular expression that must match the response body (compulsory) -# Arguments: port - which port to use (optional; default: 8080) -# Arguments: protocol - which protocol to use (optional; default: http) -# Arguments: response_code - what http response code to expect (optional; default: 200) -# Arguments: oc_args - all other arguments are used as additional parameters for the `oc new-app` -# command, typically environment variables (optional) -function ct_os_test_s2i_app() { - local image_name=${1} - local app=${2} - local context_dir=${3} - local expected_output=${4} - local port=${5:-8080} - local protocol=${6:-http} - local response_code=${7:-200} - local oc_args=${8:-} - local import_image=${9:-} - - if [ $# -lt 4 ] || [ -z "${1}" ] || [ -z "${2}" ] || [ -z "${3}" ] || [ -z "${4}" ]; then - echo "ERROR: ct_os_test_s2i_app() requires at least 4 arguments that cannot be emtpy." >&2 - return 1 - fi - - ct_os_test_s2i_app_func "${image_name}" \ - "${app}" \ - "${context_dir}" \ - "ct_os_test_response_internal '${protocol}://:${port}' '${response_code}' '${expected_output}'" \ - "${oc_args}" "${import_image}" -} - -# ct_os_test_template_app_func IMAGE APP IMAGE_IN_TEMPLATE CHECK_CMD [OC_ARGS] -# -------------------- -# Runs [image] and [app] in the openshift and optionally specifies env_params -# as environment variables to the image. Then check the container by arbitrary -# function given as argument (such an argument may include string, -# that will be replaced with actual IP). -# Arguments: image_name - prefix or whole ID of the pod to run the cmd in (compulsory) -# Arguments: template - url or local path to a template to use (compulsory) -# Arguments: name_in_template - image name used in the template -# Arguments: check_command - CMD line that checks whether the container works (compulsory; '' will be replaced with actual IP) -# Arguments: oc_args - all other arguments are used as additional parameters for the `oc new-app` -# command, typically environment variables (optional) -# Arguments: other_images - some templates need other image to be pushed into the OpenShift registry, -# specify them in this parameter as "|", where "" is a full image name -# (including registry if needed) and "" is a tag under which the image should be available -# in the OpenShift registry. -function ct_os_test_template_app_func() { - local image_name=${1} - local template=${2} - local name_in_template=${3} - local check_command=${4} - local oc_args=${5:-} - local other_images=${6:-} - local import_image=${7:-} - - if [ $# -lt 4 ] || [ -z "${1}" ] || [ -z "${2}" ] || [ -z "${3}" ] || [ -z "${4}" ]; then - echo "ERROR: ct_os_test_template_app_func() requires at least 4 arguments that cannot be emtpy." >&2 - return 1 - fi - - local service_name="${name_in_template}-testing" - local image_tagged="${name_in_template}:${VERSION}" - - # shellcheck disable=SC2119 - ct_os_new_project - - # Create a specific imagestream tag for the image so that oc cannot use anything else - if [ "${CT_SKIP_UPLOAD_IMAGE:-false}" == 'true' ] ; then - if [ -n "${import_image}" ] ; then - echo "Importing image ${import_image} as ${image_name}:${VERSION}" - oc import-image "${image_name}":"${VERSION}" --from "${import_image}" --confirm - else - echo "Uploading and importing image skipped." - fi - else - if [ -n "${import_image}" ] ; then - echo "Warning: Import image ${import_image} requested, but uploading image ${image_name} instead." - fi - ct_os_upload_image "${image_name}" "${image_tagged}" - - # upload also other images, that template might need (list of pairs in the format | - local image_tag_a - local i_t - for i_t in ${other_images} ; do - echo "${i_t}" - IFS='|' read -ra image_tag_a <<< "${i_t}" - docker pull "${image_tag_a[0]}" - ct_os_upload_image "${image_tag_a[0]}" "${image_tag_a[1]}" - done - fi - - # get the template file from remote or local location; if not found, it is - # considered an internal template name, like 'mysql', so use the name - # explicitly - local local_template - local namespace - - namespace=${CT_NAMESPACE:-$(oc project -q)} - - local_template=$(ct_obtain_input "${template}" 2>/dev/null || echo "--template=${template}") - # shellcheck disable=SC2086 - oc new-app "${local_template}" \ - --name "${name_in_template}" \ - -p NAMESPACE="${namespace}" \ - ${oc_args} - - ct_os_wait_pod_ready "${service_name}" 300 - - local ip - local check_command_exp - - ip=$(ct_os_get_service_ip "${service_name}") - # shellcheck disable=SC2001 - check_command_exp=$(echo "$check_command" | sed -e "s//$ip/g") - - echo " Checking APP using $check_command_exp ..." - local result=0 - eval "$check_command_exp" || result=1 - - if [ $result -eq 0 ] ; then - echo " Check passed." - else - echo " Check failed." - fi - - # shellcheck disable=SC2119 - ct_os_delete_project - return $result -} - -# params: -# ct_os_test_template_app IMAGE APP IMAGE_IN_TEMPLATE EXPECTED_OUTPUT [PORT, PROTOCOL, RESPONSE_CODE, OC_ARGS, ... ] -# -------------------- -# Runs [image] and [app] in the openshift and optionally specifies env_params -# as environment variables to the image. Then check the http response. -# Arguments: image_name - prefix or whole ID of the pod to run the cmd in (compulsory) -# Arguments: template - url or local path to a template to use (compulsory) -# Arguments: name_in_template - image name used in the template -# Arguments: expected_output - PCRE regular expression that must match the response body (compulsory) -# Arguments: port - which port to use (optional; default: 8080) -# Arguments: protocol - which protocol to use (optional; default: http) -# Arguments: response_code - what http response code to expect (optional; default: 200) -# Arguments: oc_args - all other arguments are used as additional parameters for the `oc new-app` -# command, typically environment variables (optional) -# Arguments: other_images - some templates need other image to be pushed into the OpenShift registry, -# specify them in this parameter as "|", where "" is a full image name -# (including registry if needed) and "" is a tag under which the image should be available -# in the OpenShift registry. -function ct_os_test_template_app() { - local image_name=${1} - local template=${2} - local name_in_template=${3} - local expected_output=${4} - local port=${5:-8080} - local protocol=${6:-http} - local response_code=${7:-200} - local oc_args=${8:-} - local other_images=${9:-} - local import_image=${10:-} - - if [ $# -lt 4 ] || [ -z "${1}" ] || [ -z "${2}" ] || [ -z "${3}" ] || [ -z "${4}" ]; then - echo "ERROR: ct_os_test_template_app() requires at least 4 arguments that cannot be emtpy." >&2 - return 1 - fi - - ct_os_test_template_app_func "${image_name}" \ - "${template}" \ - "${name_in_template}" \ - "ct_os_test_response_internal '${protocol}://:${port}' '${response_code}' '${expected_output}'" \ - "${oc_args}" \ - "${other_images}" \ - "${import_image}" -} - -# ct_os_test_image_update IMAGE_NAME OLD_IMAGE ISTAG CHECK_FUNCTION OC_ARGS -# -------------------- -# Runs an image update test with [image] uploaded to [is] imagestream -# and checks the services using an arbitrary function provided in [check_function]. -# Arguments: image_name - prefix or whole ID of the pod to run the cmd in (compulsory) -# Arguments: old_image - valid name of the image from the registry -# Arguments: istag - imagestream to upload the images into (compulsory) -# Arguments: check_function - command to be run to check functionality of created services (compulsory) -# Arguments: oc_args - arguments to use during oc new-app (compulsory) -ct_os_test_image_update() { - local image_name=$1; shift - local old_image=$1; shift - local istag=$1; shift - local check_function=$1; shift - local service_name=${image_name##*/} - local ip="" check_command_exp="" - - echo "Running image update test for: $image_name" - # shellcheck disable=SC2119 - ct_os_new_project - - # Get current image from repository and create an imagestream - docker pull "$old_image:latest" 2>/dev/null - ct_os_upload_image "$old_image" "$istag" - - # Setup example application with curent image - oc new-app "$@" --name "$service_name" - ct_os_wait_pod_ready "$service_name" 60 - - # Check application output - ip=$(ct_os_get_service_ip "$service_name") - check_command_exp=${check_function///$ip} - ct_assert_cmd_success "$check_command_exp" - - # Tag built image into the imagestream and wait for rebuild - ct_os_upload_image "$image_name" "$istag" - ct_os_wait_pod_ready "${service_name}-2" 60 - - # Check application output - ip=$(ct_os_get_service_ip "$service_name") - check_command_exp=${check_function///$ip} - ct_assert_cmd_success "$check_command_exp" - - # shellcheck disable=SC2119 - ct_os_delete_project -} - -# ct_os_deploy_cmd_image IMAGE_NAME -# -------------------- -# Runs a special command pod, a pod that does nothing, but includes utilities for testing. -# A typical usage is a mysql pod that includes mysql commandline, that we need for testing. -# Running commands inside this command pod is done via ct_os_cmd_image_run function. -# The pod is not run again if already running. -# Arguments: image_name - image to be used as a command pod -function ct_os_deploy_cmd_image() { - local image_name=${1} - oc get pod command-app &>/dev/null && echo "command POD already running" && return 0 - echo "command POD not running yet, will start one called command-app" - oc create -f - <" - local sleep_time=3 - local attempt=1 - local result=1 - local status - local response_code - local response_file - local util_image_name='python:3.6' - - response_file=$(mktemp /tmp/ct_test_response_XXXXXX) - ct_os_deploy_cmd_image "${util_image_name}" - - while [ "${attempt}" -le "${max_attempts}" ]; do - ct_os_cmd_image_run "curl --connect-timeout 10 -s -w '%{http_code}' '${url}'" >"${response_file}" && status=0 || status=1 - if [ "${status}" -eq 0 ]; then - response_code=$(tail -c 3 "${response_file}") - if [ "${response_code}" -eq "${expected_code}" ]; then - result=0 - fi - grep -qP -e "${body_regexp}" "${response_file}" || result=1; - # Some services return 40x code until they are ready, so let's give them - # some chance and not end with failure right away - # Do not wait if we already have expected outcome though - if [ "${result}" -eq 0 ] || [ "${attempt}" -gt "${ignore_error_attempts}" ] || [ "${attempt}" -eq "${max_attempts}" ] ; then - break - fi - fi - attempt=$(( attempt + 1 )) - sleep "${sleep_time}" - done - rm -f "${response_file}" - return "${result}" -} - -# ct_os_get_image_from_pod -# ------------------------ -# Print image identifier from an existing pod to stdout -# Argument: pod_prefix - prefix or full name of the pod to get image from -ct_os_get_image_from_pod() { - local pod_prefix=$1 ; shift - local pod_name - pod_name=$(ct_os_get_pod_name "$pod_prefix") - oc get "po/${pod_name}" -o yaml | sed -ne 's/^\s*image:\s*\(.*\)\s*$/\1/ p' | head -1 -} - -# ct_os_check_cmd_internal -# ---------------- -# Runs a specified command, checks exit code and compares the output with expected regexp. -# That all is done inside an image in the cluster, so the function is used -# typically in clusters that are not accessible outside. -# The check is repeated until timeout. -# Argument: util_image_name - name of the image in the cluster that is used for running the cmd -# Argument: service_name - kubernetes' service name to work with (IP address is taken from this one) -# Argument: check_command - command that is run within the util_image_name container -# Argument: expected_content_match - regexp that must be in the output (use .* to ignore check) -# Argument: timeout - number of seconds to wait till the check succeeds -function ct_os_check_cmd_internal() { - local util_image_name=$1 ; shift - local service_name=$1 ; shift - local check_command=$1 ; shift - local expected_content_match=${1:-.*} ; shift - local timeout=${1:-60} ; shift || : - - : " Service ${service_name} check ..." - - local output - local ret - local ip - local check_command_exp - - ip=$(ct_os_get_service_ip "${service_name}") - # shellcheck disable=SC2001 - check_command_exp=$(echo "$check_command" | sed -e "s//$ip/g") - - ct_os_deploy_cmd_image "$(ct_os_get_image_from_pod "${util_image_name##*/}" | head -n 1)" - SECONDS=0 - - echo -n "Waiting for ${service_name} service becoming ready ..." - while true ; do - output=$(ct_os_cmd_image_run "$check_command_exp") - ret=$? - echo "${output}" | grep -qe "${expected_content_match}" || ret=1 - if [ ${ret} -eq 0 ] ; then - echo " PASS" - return 0 - fi - echo -n "." - [ ${SECONDS} -gt "${timeout}" ] && break - sleep 3 - done - echo " FAIL" - return 1 -} - -# vim: set tabstop=2:shiftwidth=2:expandtab: diff --git a/test/test-lib.sh b/test/test-lib.sh deleted file mode 100644 index b959c34..0000000 --- a/test/test-lib.sh +++ /dev/null @@ -1,653 +0,0 @@ -# shellcheck shell=bash -# -# Test a container image. -# -# Always use sourced from a specific container testfile -# -# reguires definition of CID_FILE_DIR -# CID_FILE_DIR=$(mktemp --suffix=_test_cidfiles -d) -# reguires definition of TEST_LIST -# TEST_LIST="\ -# ctest_container_creation -# ctest_doc_content" - -# Container CI tests -# abbreviated as "ct" - -# may be redefined in the specific container testfile -EXPECTED_EXIT_CODE=0 - -# ct_cleanup -# -------------------- -# Cleans up containers used during tests. Stops and removes all containers -# referenced by cid_files in CID_FILE_DIR. Dumps logs if a container exited -# unexpectedly. Removes the cid_files and CID_FILE_DIR as well. -# Uses: $CID_FILE_DIR - path to directory containing cid_files -# Uses: $EXPECTED_EXIT_CODE - expected container exit code -function ct_cleanup() { - for cid_file in "$CID_FILE_DIR"/* ; do - local container - container=$(cat "$cid_file") - - : "Stopping and removing container $container..." - docker stop "$container" - exit_status=$(docker inspect -f '{{.State.ExitCode}}' "$container") - if [ "$exit_status" != "$EXPECTED_EXIT_CODE" ]; then - : "Dumping logs for $container" - docker logs "$container" - fi - docker rm -v "$container" - rm "$cid_file" - done - rmdir "$CID_FILE_DIR" - : "Done." -} - -# ct_enable_cleanup -# -------------------- -# Enables automatic container cleanup after tests. -function ct_enable_cleanup() { - trap ct_cleanup EXIT SIGINT -} - -# ct_get_cid [name] -# -------------------- -# Prints container id from cid_file based on the name of the file. -# Argument: name - name of cid_file where the container id will be stored -# Uses: $CID_FILE_DIR - path to directory containing cid_files -function ct_get_cid() { - local name="$1" ; shift || return 1 - cat "$CID_FILE_DIR/$name" -} - -# ct_get_cip [id] -# -------------------- -# Prints container ip address based on the container id. -# Argument: id - container id -function ct_get_cip() { - local id="$1" ; shift - docker inspect --format='{{.NetworkSettings.IPAddress}}' "$(ct_get_cid "$id")" -} - -# ct_wait_for_cid [cid_file] -# -------------------- -# Holds the execution until the cid_file is created. Usually run after container -# creation. -# Argument: cid_file - name of the cid_file that should be created -function ct_wait_for_cid() { - local cid_file=$1 - local max_attempts=10 - local sleep_time=1 - local attempt=1 - local result=1 - while [ $attempt -le $max_attempts ]; do - [ -f "$cid_file" ] && [ -s "$cid_file" ] && return 0 - : "Waiting for container start..." - attempt=$(( attempt + 1 )) - sleep $sleep_time - done - return 1 -} - -# ct_assert_container_creation_fails [container_args] -# -------------------- -# The invocation of docker run should fail based on invalid container_args -# passed to the function. Returns 0 when container fails to start properly. -# Argument: container_args - all arguments are passed directly to dokcer run -# Uses: $CID_FILE_DIR - path to directory containing cid_files -function ct_assert_container_creation_fails() { - local ret=0 - local max_attempts=10 - local attempt=1 - local cid_file=assert - set +e - local old_container_args="${CONTAINER_ARGS-}" - # we really work with CONTAINER_ARGS as with a string - # shellcheck disable=SC2124 - CONTAINER_ARGS="$@" - if ct_create_container "$cid_file" ; then - local cid - cid=$(ct_get_cid "$cid_file") - - while [ "$(docker inspect -f '{{.State.Running}}' "$cid")" == "true" ] ; do - sleep 2 - attempt=$(( attempt + 1 )) - if [ "$attempt" -gt "$max_attempts" ]; then - docker stop "$cid" - ret=1 - break - fi - done - exit_status=$(docker inspect -f '{{.State.ExitCode}}' "$cid") - if [ "$exit_status" == "0" ]; then - ret=1 - fi - docker rm -v "$cid" - rm "$CID_FILE_DIR/$cid_file" - fi - [ -n "$old_container_args" ] && CONTAINER_ARGS="$old_container_args" - set -e - return "$ret" -} - -# ct_create_container [name, command] -# -------------------- -# Creates a container using the IMAGE_NAME and CONTAINER_ARGS variables. Also -# stores the container id to a cid_file located in the CID_FILE_DIR, and waits -# for the creation of the file. -# Argument: name - name of cid_file where the container id will be stored -# Argument: command - optional command to be executed in the container -# Uses: $CID_FILE_DIR - path to directory containing cid_files -# Uses: $CONTAINER_ARGS - optional arguments passed directly to docker run -# Uses: $IMAGE_NAME - name of the image being tested -function ct_create_container() { - local cid_file="$CID_FILE_DIR/$1" ; shift - # create container with a cidfile in a directory for cleanup - # shellcheck disable=SC2086 - docker run --cidfile="$cid_file" -d ${CONTAINER_ARGS:-} "$IMAGE_NAME" "$@" - ct_wait_for_cid "$cid_file" || return 1 - : "Created container $(cat "$cid_file")" -} - -# ct_scl_usage_old [name, command, expected] -# -------------------- -# Tests three ways of running the SCL, by looking for an expected string -# in the output of the command -# Argument: name - name of cid_file where the container id will be stored -# Argument: command - executed inside the container -# Argument: expected - string that is expected to be in the command output -# Uses: $CID_FILE_DIR - path to directory containing cid_files -# Uses: $IMAGE_NAME - name of the image being tested -function ct_scl_usage_old() { - local name="$1" - local command="$2" - local expected="$3" - local out="" - : " Testing the image SCL enable" - out=$(docker run --rm "${IMAGE_NAME}" /bin/bash -c "${command}") - if ! echo "${out}" | grep -q "${expected}"; then - echo "ERROR[/bin/bash -c \"${command}\"] Expected '${expected}', got '${out}'" >&2 - return 1 - fi - out=$(docker exec "$(ct_get_cid "$name")" /bin/bash -c "${command}" 2>&1) - if ! echo "${out}" | grep -q "${expected}"; then - echo "ERROR[exec /bin/bash -c \"${command}\"] Expected '${expected}', got '${out}'" >&2 - return 1 - fi - out=$(docker exec "$(ct_get_cid "$name")" /bin/sh -ic "${command}" 2>&1) - if ! echo "${out}" | grep -q "${expected}"; then - echo "ERROR[exec /bin/sh -ic \"${command}\"] Expected '${expected}', got '${out}'" >&2 - return 1 - fi -} - -# ct_doc_content_old [strings] -# -------------------- -# Looks for occurence of stirngs in the documentation files and checks -# the format of the files. Files examined: help.1 -# Argument: strings - strings expected to appear in the documentation -# Uses: $IMAGE_NAME - name of the image being tested -function ct_doc_content_old() { - local tmpdir - tmpdir=$(mktemp -d) - local f - : " Testing documentation in the container image" - # Extract the help files from the container - # shellcheck disable=SC2043 - for f in help.1 ; do - docker run --rm "${IMAGE_NAME}" /bin/bash -c "cat /${f}" >"${tmpdir}/$(basename "${f}")" - # Check whether the files contain some important information - for term in "$@" ; do - if ! grep -F -q -e "${term}" "${tmpdir}/$(basename "${f}")" ; then - echo "ERROR: File /${f} does not include '${term}'." >&2 - return 1 - fi - done - # Check whether the files use the correct format - for term in TH PP SH ; do - if ! grep -q "^\.${term}" "${tmpdir}/help.1" ; then - echo "ERROR: /help.1 is probably not in troff or groff format, since '${term}' is missing." >&2 - return 1 - fi - done - done - : " Success!" -} - -# full_ca_file_path -# Return string for full path to CA file -function full_ca_file_path() -{ - echo "/etc/pki/ca-trust/source/anchors/RH-IT-Root-CA.crt" -} -# ct_mount_ca_file -# ------------------ -# Check if /etc/pki/certs/RH-IT-Root-CA.crt file exists -# return mount string for containers or empty string -function ct_mount_ca_file() -{ - # mount CA file only if NPM_REGISTRY variable is present. - local mount_parameter="" - if [ -n "$NPM_REGISTRY" ] && [ -f "$(full_ca_file_path)" ]; then - mount_parameter="-v $(full_ca_file_path):$(full_ca_file_path):Z" - fi - echo "$mount_parameter" -} - -# ct_build_s2i_npm_variables URL_TO_NPM_JS_SERVER -# ------------------------------------------ -# Function returns -e NPM_MIRROR and -v MOUNT_POINT_FOR_CAFILE -# or empty string -function ct_build_s2i_npm_variables() -{ - npm_variables="" - if [ -n "$NPM_REGISTRY" ] && [ -f "$(full_ca_file_path)" ]; then - npm_variables="-e NPM_MIRROR=$NPM_REGISTRY $(ct_mount_ca_file)" - fi - echo "$npm_variables" -} - -# ct_npm_works -# -------------------- -# Checks existance of the npm tool and runs it. -function ct_npm_works() { - local tmpdir - tmpdir=$(mktemp -d) - : " Testing npm in the container image" - local cid_file="${tmpdir}/cid" - if ! docker run --rm "${IMAGE_NAME}" /bin/bash -c "npm --version" >"${tmpdir}/version" ; then - echo "ERROR: 'npm --version' does not work inside the image ${IMAGE_NAME}." >&2 - return 1 - fi - - # shellcheck disable=SC2046 - docker run -d $(ct_mount_ca_file) --rm --cidfile="$cid_file" "${IMAGE_NAME}-testapp" - - # Wait for the container to write it's CID file - ct_wait_for_cid "$cid_file" || return 1 - - if ! docker exec "$(cat "$cid_file")" /bin/bash -c "npm --verbose install jquery && test -f node_modules/jquery/src/jquery.js" >"${tmpdir}/jquery" 2>&1 ; then - echo "ERROR: npm could not install jquery inside the image ${IMAGE_NAME}." >&2 - return 1 - fi - - if [ -n "$NPM_REGISTRY" ] && [ -f "$(full_ca_file_path)" ]; then - if ! grep -qo "$NPM_REGISTRY" "${tmpdir}/jquery"; then - echo "ERROR: Internal repository is NOT set. Even it is requested." - return 1 - fi - fi - - if [ -f "$cid_file" ]; then - docker stop "$(cat "$cid_file")" - rm "$cid_file" - fi - : " Success!" -} - -# ct_path_append PATH_VARNAME DIRECTORY -# ------------------------------------- -# Append DIRECTORY to VARIABLE of name PATH_VARNAME, the VARIABLE must consist -# of colon-separated list of directories. -ct_path_append () -{ - if eval "test -n \"\${$1-}\""; then - eval "$1=\$2:\$$1" - else - eval "$1=\$2" - fi -} - - -# ct_path_foreach PATH ACTION [ARGS ...] -# -------------------------------------- -# For each DIR in PATH execute ACTION (path is colon separated list of -# directories). The particular calls to ACTION will look like -# '$ ACTION directory [ARGS ...]' -ct_path_foreach () -{ - local dir dirlist action save_IFS - save_IFS=$IFS - IFS=: - dirlist=$1 - action=$2 - shift 2 - for dir in $dirlist; do "$action" "$dir" "$@" ; done - IFS=$save_IFS -} - - -# ct_run_test_list -# -------------------- -# Execute the tests specified by TEST_LIST -# Uses: $TEST_LIST - list of test names -function ct_run_test_list() { - for test_case in $TEST_LIST; do - : "Running test $test_case" - # shellcheck source=/dev/null - [ -f "test/$test_case" ] && source "test/$test_case" - # shellcheck source=/dev/null - [ -f "../test/$test_case" ] && source "../test/$test_case" - $test_case - done; -} - -# ct_gen_self_signed_cert_pem -# --------------------------- -# Generates a self-signed PEM certificate pair into specified directory. -# Argument: output_dir - output directory path -# Argument: base_name - base name of the certificate files -# Resulted files will be those: -# /-cert-selfsigned.pem -- public PEM cert -# /-key.pem -- PEM private key -ct_gen_self_signed_cert_pem() { - local output_dir=$1 ; shift - local base_name=$1 ; shift - mkdir -p "${output_dir}" - openssl req -newkey rsa:2048 -nodes -keyout "${output_dir}"/"${base_name}"-key.pem -subj '/C=GB/ST=Berkshire/L=Newbury/O=My Server Company' > "${base_name}"-req.pem - openssl req -new -x509 -nodes -key "${output_dir}"/"${base_name}"-key.pem -batch > "${output_dir}"/"${base_name}"-cert-selfsigned.pem -} - -# ct_obtain_input FILE|DIR|URL -# -------------------- -# Either copies a file or a directory to a tmp location for local copies, or -# downloads the file from remote location. -# Resulted file path is printed, so it can be later used by calling function. -# Arguments: input - local file, directory or remote URL -function ct_obtain_input() { - local input=$1 - local extension="${input##*.}" - - # Try to use same extension for the temporary file if possible - [[ "${extension}" =~ ^[a-z0-9]*$ ]] && extension=".${extension}" || extension="" - - local output - output=$(mktemp "/var/tmp/test-input-XXXXXX$extension") - if [ -f "${input}" ] ; then - cp -f "${input}" "${output}" - elif [ -d "${input}" ] ; then - rm -f "${output}" - cp -r -LH "${input}" "${output}" - elif echo "${input}" | grep -qe '^http\(s\)\?://' ; then - curl "${input}" > "${output}" - else - echo "ERROR: file type not known: ${input}" >&2 - return 1 - fi - echo "${output}" -} - -# ct_test_response -# ---------------- -# Perform GET request to the application container, checks output with -# a reg-exp and HTTP response code. -# Argument: url - request URL path -# Argument: expected_code - expected HTTP response code -# Argument: body_regexp - PCRE regular expression that must match the response body -# Argument: max_attempts - Optional number of attempts (default: 20), three seconds sleep between -# Argument: ignore_error_attempts - Optional number of attempts when we ignore error output (default: 10) -ct_test_response() { - local url="$1" - local expected_code="$2" - local body_regexp="$3" - local max_attempts=${4:-20} - local ignore_error_attempts=${5:-10} - - : " Testing the HTTP(S) response for <${url}>" - local sleep_time=3 - local attempt=1 - local result=1 - local status - local response_code - local response_file - response_file=$(mktemp /tmp/ct_test_response_XXXXXX) - while [ "${attempt}" -le "${max_attempts}" ]; do - curl --connect-timeout 10 -s -w '%{http_code}' "${url}" >"${response_file}" && status=0 || status=1 - if [ "${status}" -eq 0 ]; then - response_code=$(tail -c 3 "${response_file}") - if [ "${response_code}" -eq "${expected_code}" ]; then - result=0 - fi - grep -qP -e "${body_regexp}" "${response_file}" || result=1; - # Some services return 40x code until they are ready, so let's give them - # some chance and not end with failure right away - # Do not wait if we already have expected outcome though - if [ "${result}" -eq 0 ] || [ "${attempt}" -gt "${ignore_error_attempts}" ] || [ "${attempt}" -eq "${max_attempts}" ] ; then - break - fi - fi - attempt=$(( attempt + 1 )) - sleep "${sleep_time}" - done - rm -f "${response_file}" - return "${result}" -} - -# ct_registry_from_os OS -# ---------------- -# Transform operating system string [os] into registry url -# Argument: OS - string containing the os version -ct_registry_from_os() { - local registry="" - case $1 in - rhel*) - registry=registry.redhat.io - ;; - *) - registry=docker.io - ;; - esac - echo "$registry" -} - - # ct_get_public_image_name OS BASE_IMAGE_NAME VERSION -# ---------------- -# Transform the arguments into public image name -# Argument: OS - string containing the os version -# Argument: BASE_IMAGE_NAME - string containing the base name of the image as defined in the Makefile -# Argument: VERSION - string containing the version of the image as defined in the Makefile -ct_get_public_image_name() { - local os=$1; shift - local base_image_name=$1; shift - local version=$1; shift - - local public_image_name - local registry - - registry=$(ct_registry_from_os "$os") - if [ "x$os" == "xrhel7" ]; then - public_image_name=$registry/rhscl/$base_image_name-${version//./}-rhel7 - elif [ "x$os" == "xrhel8" ]; then - public_image_name=$registry/rhel8/$base_image_name-${version//./} - elif [ "x$os" == "xcentos7" ]; then - public_image_name=$registry/centos/$base_image_name-${version//./}-centos7 - fi - - echo "$public_image_name" -} - -# ct_assert_cmd_success CMD -# ---------------- -# Evaluates [cmd] and fails if it does not succeed. -# Argument: CMD - Command to be run -function ct_assert_cmd_success() { - echo "Checking '$*' for success ..." - if ! eval "$@" &>/dev/null; then - echo " FAIL" - return 1 - fi - echo " PASS" - return 0 -} - -# ct_assert_cmd_failure CMD -# ---------------- -# Evaluates [cmd] and fails if it succeeds. -# Argument: CMD - Command to be run -function ct_assert_cmd_failure() { - echo "Checking '$*' for failure ..." - if eval "$@" &>/dev/null; then - echo " FAIL" - return 1 - fi - echo " PASS" - return 0 -} - - -# ct_random_string [LENGTH=10] -# ---------------------------- -# Generate pseudorandom alphanumeric string of LENGTH bytes, the -# default length is 10. The string is printed on stdout. -ct_random_string() -( - export LC_ALL=C - dd if=/dev/urandom count=1 bs=10k 2>/dev/null \ - | tr -dc 'a-z0-9' \ - | fold -w "${1-10}" \ - | head -n 1 -) - -# ct_s2i_usage IMG_NAME [S2I_ARGS] -# ---------------------------- -# Create a container and run the usage script inside -# Argument: IMG_NAME - name of the image to be used for the container run -# Argument: S2I_ARGS - Additional list of source-to-image arguments, currently unused. -ct_s2i_usage() -{ - local img_name=$1; shift - local s2i_args="$*"; - local usage_command="/usr/libexec/s2i/usage" - docker run --rm "$img_name" bash -c "$usage_command" -} - -# ct_s2i_build_as_df APP_PATH SRC_IMAGE DST_IMAGE [S2I_ARGS] -# ---------------------------- -# Create a new s2i app image from local sources in a similar way as source-to-image would have used. -# Argument: APP_PATH - local path to the app sources to be used in the test -# Argument: SRC_IMAGE - image to be used as a base for the s2i build -# Argument: DST_IMAGE - image name to be used during the tagging of the s2i build result -# Argument: S2I_ARGS - Additional list of source-to-image arguments. -# Only used to check for pull-policy=never and environment variable definitions. -ct_s2i_build_as_df() -{ - local app_path=$1; shift - local src_image=$1; shift - local dst_image=$1; shift - local s2i_args="$*"; - local local_app=upload/src/ - local local_scripts=upload/scripts/ - local user_id= - local df_name= - local tmpdir= - local incremental=false - local mount_options="" - - # Run the entire thing inside a subshell so that we do not leak shell options outside of the function - ( - # Error out if any part of the build fails - set -e - - # Use /tmp to not pollute cwd - tmpdir=$(mktemp -d) - df_name=$(mktemp -p "$tmpdir" Dockerfile.XXXX) - cd "$tmpdir" - # Check if the image is available locally and try to pull it if it is not - docker images "$src_image" &>/dev/null || echo "$s2i_args" | grep -q "pull-policy=never" || docker pull "$src_image" - user=$(docker inspect -f "{{.Config.User}}" "$src_image") - # Default to root if no user is set by the image - user=${user:-0} - # run the user through the image in case it is non-numeric or does not exist - # NOTE: The '-eq' test is used to check if $user is numeric as it will fail if $user is not an integer - if ! [ "$user" -eq "$user" ] 2>/dev/null && ! user_id=$(docker run --rm "$src_image" bash -c "id -u $user 2>/dev/null"); then - echo "ERROR: id of user $user not found inside image $src_image." - echo "Terminating s2i build." - return 1 - else - user_id=${user_id:-$user} - fi - echo "$s2i_args" | grep -q "\-\-incremental" && incremental=true - if $incremental; then - inc_tmp=$(mktemp -d --tmpdir incremental.XXXX) - setfacl -m "u:$user_id:rwx" "$inc_tmp" - # Check if the image exists, build should fail (for testing use case) if it does not - docker images "$dst_image" &>/dev/null || (echo "Image $dst_image not found."; false) - # Run the original image with a mounted in volume and get the artifacts out of it - cmd="if [ -s /usr/libexec/s2i/save-artifacts ]; then /usr/libexec/s2i/save-artifacts > \"$inc_tmp/artifacts.tar\"; else touch \"$inc_tmp/artifacts.tar\"; fi" - docker run --rm -v "$inc_tmp:$inc_tmp:Z" "$dst_image" bash -c "$cmd" - # Move the created content into the $tmpdir for the build to pick it up - mv "$inc_tmp/artifacts.tar" "$tmpdir/" - fi - # Strip file:// from APP_PATH and copy its contents into current context - mkdir -p "$local_app" - cp -r "${app_path/file:\/\//}/." "$local_app" - [ -d "$local_app/.s2i/bin/" ] && mv "$local_app/.s2i/bin" "$local_scripts" - # Create a Dockerfile named df_name and fill it with proper content - #FIXME: Some commands could be combined into a single layer but not sure if worth the trouble for testing purposes - cat <"$df_name" -FROM $src_image -LABEL "io.openshift.s2i.build.image"="$src_image" \\ - "io.openshift.s2i.build.source-location"="$app_path" -USER root -COPY $local_app /tmp/src -EOF - [ -d "$local_scripts" ] && echo "COPY $local_scripts /tmp/scripts" >> "$df_name" && - echo "RUN chown -R $user_id:0 /tmp/scripts" >>"$df_name" - echo "RUN chown -R $user_id:0 /tmp/src" >>"$df_name" - # Check for custom environment variables inside .s2i/ folder - if [ -e "$local_app/.s2i/environment" ]; then - # Remove any comments and add the contents as ENV commands to the Dockerfile - sed '/^\s*#.*$/d' "$local_app/.s2i/environment" | while read -r line; do - echo "ENV $line" >>"$df_name" - done - fi - # Filter out env var definitions from $s2i_args and create Dockerfile ENV commands out of them - echo "$s2i_args" | grep -o -e '\(-e\|--env\)[[:space:]=]\S*=\S*' | sed -e 's/-e /ENV /' -e 's/--env[ =]/ENV /' >>"$df_name" - # Check if CA autority is present on host and add it into Dockerfile - [ -f "$(full_ca_file_path)" ] && echo "RUN cd /etc/pki/ca-trust/source/anchors && update-ca-trust extract" >>"$df_name" - - # Add in artifacts if doing an incremental build - if $incremental; then - { echo "RUN mkdir /tmp/artifacts" - echo "ADD artifacts.tar /tmp/artifacts" - echo "RUN chown -R $user_id:0 /tmp/artifacts" ; } >>"$df_name" - fi - - echo "USER $user_id" >>"$df_name" - # If exists, run the custom assemble script, else default to /usr/libexec/s2i/assemble - if [ -x "$local_scripts/assemble" ]; then - echo "RUN /tmp/scripts/assemble" >>"$df_name" - else - echo "RUN /usr/libexec/s2i/assemble" >>"$df_name" - fi - # If exists, set the custom run script as CMD, else default to /usr/libexec/s2i/run - if [ -x "$local_scripts/run" ]; then - echo "CMD /tmp/scripts/run" >>"$df_name" - else - echo "CMD /usr/libexec/s2i/run" >>"$df_name" - fi - - # Check if -v parameter is present in s2i_args and add it into docker build command - mount_options=$(echo "$s2i_args" | grep -o -e '\(-v\)[[:space:]]\.*\S*' || true) - - # Run the build and tag the result - # shellcheck disable=SC2086 - docker build $mount_options -f "$df_name" --no-cache=true -t "$dst_image" . - ) -} - -# ct_check_image_availability PUBLIC_IMAGE_NAME -# ---------------------------- -# Pull an image from the public repositories to see if the image is already available. -# Argument: PUBLIC_IMAGE_NAME - string containing the public name of the image to pull -ct_check_image_availability() { - local public_image_name=$1; - - # Try pulling the image to see if it is accessible - if ! docker pull "$public_image_name" &>/dev/null; then - echo "$public_image_name could not be downloaded via 'docker'" - return 1 - fi -} - -# vim: set tabstop=2:shiftwidth=2:expandtab: