From fa7e7e5dec0b064aceb1cffb4c5394f64b992511 Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: Thu, 11 May 2017 11:56:47 +0200 Subject: [PATCH] add files for building containter artifacts Lifted in large parts from version 9.5 of the SCL postgresql container. https://github.com/sclorg/postgresql-container --- Dockerfile | 100 ++++++++ module-postgresql.repo.in | 11 + root/help.1 | 190 +++++++++++++++ root/usr/bin/cgroup-limits | 92 ++++++++ root/usr/bin/container-entrypoint | 3 + root/usr/bin/run-postgresql | 28 +++ root/usr/bin/run-postgresql-master | 5 + root/usr/bin/run-postgresql-slave | 36 +++ root/usr/libexec/fix-permissions | 7 + .../container-scripts/postgresql/README.md | 98 ++++++++ .../container-scripts/postgresql/common.sh | 220 ++++++++++++++++++ ...ustom-postgresql-replication.conf.template | 7 + .../openshift-custom-postgresql.conf.template | 21 ++ .../openshift-custom-recovery.conf.template | 9 + .../container-scripts/postgresql/scl_enable | 3 + 15 files changed, 830 insertions(+) create mode 100644 Dockerfile create mode 100644 module-postgresql.repo.in create mode 100644 root/help.1 create mode 100755 root/usr/bin/cgroup-limits create mode 100755 root/usr/bin/container-entrypoint create mode 100755 root/usr/bin/run-postgresql create mode 100755 root/usr/bin/run-postgresql-master create mode 100755 root/usr/bin/run-postgresql-slave create mode 100755 root/usr/libexec/fix-permissions create mode 100644 root/usr/share/container-scripts/postgresql/README.md create mode 100644 root/usr/share/container-scripts/postgresql/common.sh create mode 100644 root/usr/share/container-scripts/postgresql/openshift-custom-postgresql-replication.conf.template create mode 100644 root/usr/share/container-scripts/postgresql/openshift-custom-postgresql.conf.template create mode 100644 root/usr/share/container-scripts/postgresql/openshift-custom-recovery.conf.template create mode 100644 root/usr/share/container-scripts/postgresql/scl_enable diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..3722091 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,100 @@ +# Container artifact for the PostgreSQL module + +# This should probably rather be something like...: +# FROM registry.fedoraproject.org/module-base-runtime:26 +# ...and probably would: +# - contain microdnf rather than dnf +# - not have any repositories configured +# - need a shared-userspace module repo enabled +FROM registry.fedoraproject.org/fedora:26 + +# PostgreSQL container image +# Exposed ports: +# * 5432/tcp - postgres +# Volumes: +# * /var/lib/psql/data - Database cluster for PostgreSQL + + +ENV NAME=postgresql \ + VERSION=0 \ + RELEASE=1 \ + ARCH=x86_64 \ + POSTGRESQL_VERSION=9.6 \ + LANG=en_US.UTF-8 \ + LC_ALL=en_US.UTF-8 \ + HOME=/var/lib/pgsql \ + PGUSER=postgres \ + POSTGRESQL_MODULE_HASH=5a0a295c9673c2a1 \ + SHARED_USERSPACE_MODULE_HASH=e67c1e728d6aa7be + +LABEL summary = "PostgreSQL is an object-relational DBMS." \ + name = "$FGC/$NAME" \ + version = "$VERSION" \ + release="$RELEASE.$DISTTAG" \ + architecture = "$ARCH" \ + maintainer = "Nils Philippsen " \ + description = "PostgreSQL is an advanced Object-Relational database management system (DBMS). This container contains the programs needed to create and run a PostgreSQL server, which will in turn allow you to create and maintain PostgreSQL databases." \ + vendor="Fedora Project" \ + com.redhat.component="$NAME" \ + org.fedoraproject.component="postgresql" \ + authoritative-source-url="registry.fedoraproject.org" \ + usage="docker run -v :/var/lib/pgsql:Z -p 5432:5432 -e POSTGRESQL_USER= -e POSTGRESQL_PASSWORD= -e POSTGRESQL_DATABASE= modularitycontainers/postgresql" \ + io.k8s.description = "PostgreSQL is an advanced Object-Relational database management system (DBMS). This container contains the programs needed to create and run a PostgreSQL server, which will in turn allow you to create and maintain PostgreSQL databases." \ + io.k8s.display-name="PostgreSQL ${POSTGRESQL_VERSION}" \ + io.openshift.tags="database,postgresql,postgresql96" \ + io.openshift.expose-services="5432/tcp:postgres" + +EXPOSE 5432 + +ADD root / + +# Update packages so everything available is current +RUN dnf update -y --setopt=tsflags=nodocs + +# Need to have charset files, disable normal repo(s), enable module repo(s) +RUN dnf install -y --setopt=tsflags=nodocs 'dnf-command(config-manager)' +RUN dnf install -y glibc-locale-source + +# ORCHESTRATION: +# Newer versions of the shared-userspace module have this, but they're not +# available yet +RUN dnf install -y findutils + +# /usr/bin/envsubst +RUN dnf install -y gettext +# nss_wrapper.so +RUN dnf install -y nss_wrapper + +# No working Python module yet +RUN dnf install -y /usr/bin/python +RUN dnf config-manager --set-disabled fedora +RUN dnf config-manager --set-disabled updates +RUN dnf config-manager --set-disabled rawhide +COPY module-postgresql.repo.in /tmp/module-postgresql.repo.in +RUN sed 's|@POSTGRESQL_MODULE_HASH@|'${POSTGRESQL_MODULE_HASH}'|g; s|@SHARED_USERSPACE_MODULE_HASH@|'${SHARED_USERSPACE_MODULE_HASH}'|g' < /tmp/module-postgresql.repo.in > /etc/yum.repos.d/module-postgresql.repo && rm -f /tmp/module-postgresql.repo.in + +# Install the postgresql server component. +# +# This image must forever use UID 26 for postgres user so our volumes are +# safe in the future. This should *never* change, the last test is there +# to make sure of that. +RUN INSTALL_PKGS="postgresql postgresql-server" && \ + dnf install -y --setopt=tsflags=nodocs $INSTALL_PKGS && \ + rpm -V $INSTALL_PKGS && \ + dnf --enablerepo=\* -y clean all && \ + localedef -f UTF-8 -i en_US en_US.UTF-8 && \ + test "$(id -u postgres)" = "26" && \ + test "$(id -g postgres)" = "26" && \ + mkdir -p /var/lib/pgsql/data && \ + /usr/libexec/fix-permissions /var/lib/pgsql && \ + /usr/libexec/fix-permissions /var/run/postgresql + +# Get prefix path and path to scripts rather than hard-code them in scripts +ENV CONTAINER_SCRIPTS_PATH=/usr/share/container-scripts/postgresql + +VOLUME ["/var/lib/pgsql/data"] + +USER 26 + +ENTRYPOINT ["container-entrypoint"] +CMD ["run-postgresql"] diff --git a/module-postgresql.repo.in b/module-postgresql.repo.in new file mode 100644 index 0000000..f93e3dc --- /dev/null +++ b/module-postgresql.repo.in @@ -0,0 +1,11 @@ +[module-shared-userspace] +name=Repository for the Shared Userspace module +baseurl=https://kojipkgs.fedoraproject.org/repos/module-@SHARED_USERSPACE_MODULE_HASH@/latest/${basearch}/ +enabled=1 +gpgcheck=0 + +[module-postgresql] +name=Repository for the PostgreSQL module +baseurl=https://kojipkgs.fedoraproject.org/repos/module-@POSTGRESQL_MODULE_HASH@/latest/${basearch}/ +enabled=1 +gpgcheck=0 diff --git a/root/help.1 b/root/help.1 new file mode 100644 index 0000000..89c84d9 --- /dev/null +++ b/root/help.1 @@ -0,0 +1,190 @@ +.\"t +.\" WARNING: Do not edit this file manually, it is generated from README.md automatically. +.\" +.\"t +.\" Automatically generated by Pandoc 1.16.0.2 +.\" +.TH "POSTGRESQL\-95\-RHEL7" "1" "February 22, 2017" "Container Image Pages" "" +.hy +.SH PostgreSQL Docker image +.PP +This repository contains Dockerfiles for PostgreSQL images for general +usage and OpenShift. +Users can choose between RHEL and CentOS based images. +.SS Environment variables and volumes +.PP +The image recognizes the following environment variables that you can +set during initialization by passing \f[C]\-e\ VAR=VALUE\f[] to the +Docker run command. +.PP +.TS +tab(@); +lw(26.7n) lw(43.3n). +T{ +Variable name +T}@T{ +Description +T} +_ +T{ +\f[C]POSTGRESQL_USER\f[] +T}@T{ +User name for PostgreSQL account to be created +T} +T{ +\f[C]POSTGRESQL_PASSWORD\f[] +T}@T{ +Password for the user account +T} +T{ +\f[C]POSTGRESQL_DATABASE\f[] +T}@T{ +Database name +T} +T{ +\f[C]POSTGRESQL_ADMIN_PASSWORD\f[] +T}@T{ +Password for the \f[C]postgres\f[] admin account (optional) +T} +.TE +.PP +The following environment variables influence the PostgreSQL +configuration file. +They are all optional. +.PP +.TS +tab(@); +lw(15.7n) lw(37.6n) lw(16.7n). +T{ +Variable name +T}@T{ +Description +T}@T{ +Default +T} +_ +T{ +\f[C]POSTGRESQL_MAX_CONNECTIONS\f[] +T}@T{ +The maximum number of client connections allowed +T}@T{ +100 +T} +T{ +\f[C]POSTGRESQL_MAX_PREPARED_TRANSACTIONS\f[] +T}@T{ +Sets the maximum number of transactions that can be in the "prepared" +state. +If you are using prepared transactions, you will probably want this to +be at least as large as max_connections +T}@T{ +0 +T} +T{ +\f[C]POSTGRESQL_SHARED_BUFFERS\f[] +T}@T{ +Sets how much memory is dedicated to PostgreSQL to use for caching data +T}@T{ +32M +T} +T{ +\f[C]POSTGRESQL_EFFECTIVE_CACHE_SIZE\f[] +T}@T{ +Set to an estimate of how much memory is available for disk caching by +the operating system and within the database itself +T}@T{ +128M +T} +.TE +.PP +You can also set the following mount points by passing the +\f[C]\-v\ /host:/container\f[] flag to Docker. +.PP +.TS +tab(@); +l l. +T{ +Volume mount point +T}@T{ +Description +T} +_ +T{ +\f[C]/var/lib/pgsql/data\f[] +T}@T{ +PostgreSQL database cluster directory +T} +.TE +.PP +\f[B]Notice: When mouting a directory from the host into the container, +ensure that the mounted directory has the appropriate permissions and +that the owner and group of the directory matches the user UID or name +which is running inside the container.\f[] +.SS Usage +.PP +For this, we will assume that you are using the +\f[C]openshift/postgresql\-92\-centos7\f[] image. +If you want to set only the mandatory environment variables and not +store the database in a host directory, execute the following command: +.IP +.nf +\f[C] +$\ docker\ run\ \-d\ \-\-name\ postgresql_database\ \-e\ POSTGRESQL_USER=user\ \-e\ POSTGRESQL_PASSWORD=pass\ \-e\ POSTGRESQL_DATABASE=db\ \-p\ 5432:5432\ openshift/postgresql\-92\-centos7 +\f[] +.fi +.PP +This will create a container named \f[C]postgresql_database\f[] running +PostgreSQL with database \f[C]db\f[] and user with credentials +\f[C]user:pass\f[]. +Port 5432 will be exposed and mapped to the host. +If you want your database to be persistent across container executions, +also add a \f[C]\-v\ /host/db/path:/var/lib/pgsql/data\f[] argument. +This will be the PostgreSQL database cluster directory. +.PP +If the database cluster directory is not initialized, the entrypoint +script will first run +\f[C]initdb\f[] (http://www.postgresql.org/docs/9.2/static/app-initdb.html) +and setup necessary database users and passwords. +After the database is initialized, or if it was already present, +\f[C]postgres\f[] (http://www.postgresql.org/docs/9.2/static/app-postgres.html) +is executed and will run as PID 1. +You can stop the detached container by running +\f[C]docker\ stop\ postgresql_database\f[]. +.SS PostgreSQL auto\-tuning +.PP +When the PostgreSQL image is run with the \f[C]\-\-memory\f[] parameter +set and if there are no values provided for +\f[C]POSTGRESQL_SHARED_BUFFERS\f[] and +\f[C]POSTGRESQL_EFFECTIVE_CACHE_SIZE\f[] those values are automatically +calculated based on the value provided in the \f[C]\-\-memory\f[] +parameter. +.PP +The values are calculated based on the +upstream (https://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server) +formulas. +For the \f[C]shared_buffers\f[] we use 1/4 of given memory and for the +\f[C]effective_cache_size\f[] we set the value to 1/2 of the given +memory. +.SS PostgreSQL admin account +.PP +The admin account \f[C]postgres\f[] has no password set by default, only +allowing local connections. +You can set it by setting the \f[C]POSTGRESQL_ADMIN_PASSWORD\f[] +environment variable when initializing your container. +This will allow you to login to the \f[C]postgres\f[] account remotely. +Local connections will still not require a password. +.SS Changing passwords +.PP +Since passwords are part of the image configuration, the only supported +method to change passwords for the database user +(\f[C]POSTGRESQL_USER\f[]) and \f[C]postgres\f[] admin user is by +changing the environment variables \f[C]POSTGRESQL_PASSWORD\f[] and +\f[C]POSTGRESQL_ADMIN_PASSWORD\f[], respectively. +.PP +Changing database passwords through SQL statements or any way other than +through the environment variables aforementioned will cause a mismatch +between the values stored in the variables and the actual passwords. +Whenever a database container starts it will reset the passwords to the +values stored in the environment variables. +.SH AUTHORS +Red Hat. diff --git a/root/usr/bin/cgroup-limits b/root/usr/bin/cgroup-limits new file mode 100755 index 0000000..b9d4edc --- /dev/null +++ b/root/usr/bin/cgroup-limits @@ -0,0 +1,92 @@ +#!/usr/bin/python + +""" +Script for parsing cgroup information + +This script will read some limits from the cgroup system and parse +them, printing out "VARIABLE=VALUE" on each line for every limit that is +successfully read. Output of this script can be directly fed into +bash's export command. Recommended usage from a bash script: + + set -o errexit + export_vars=$(cgroup-limits) ; export $export_vars + +Variables currently supported: + MAX_MEMORY_LIMIT_IN_BYTES + Maximum possible limit MEMORY_LIMIT_IN_BYTES can have. This is + currently constant value of 9223372036854775807. + MEMORY_LIMIT_IN_BYTES + Maximum amount of user memory in bytes. If this value is set + to the same value as MAX_MEMORY_LIMIT_IN_BYTES, it means that + there is no limit set. The value is taken from + /sys/fs/cgroup/memory/memory.limit_in_bytes + NUMBER_OF_CORES + Number of detected CPU cores that can be used. This value is + calculated from /sys/fs/cgroup/cpuset/cpuset.cpus + NO_MEMORY_LIMIT + Set to "true" if MEMORY_LIMIT_IN_BYTES is so high that the caller + can act as if no memory limit was set. Undefined otherwise. +""" + +from __future__ import print_function +import sys + + +def _read_file(path): + try: + with open(path, 'r') as f: + return f.read().strip() + except IOError: + return None + + +def get_memory_limit(): + """ + Read memory limit, in bytes. + """ + + limit = _read_file('/sys/fs/cgroup/memory/memory.limit_in_bytes') + if limit is None or not limit.isdigit(): + print("Warning: Can't detect memory limit from cgroups", + file=sys.stderr) + return None + return int(limit) + + +def get_number_of_cores(): + """ + Read number of CPU cores. + """ + + core_count = 0 + + line = _read_file('/sys/fs/cgroup/cpuset/cpuset.cpus') + if line is None: + print("Warning: Can't detect number of CPU cores from cgroups", + file=sys.stderr) + return None + + for group in line.split(','): + core_ids = list(map(int, group.split('-'))) + if len(core_ids) == 2: + core_count += core_ids[1] - core_ids[0] + 1 + else: + core_count += 1 + + return core_count + + +if __name__ == "__main__": + env_vars = { + "MAX_MEMORY_LIMIT_IN_BYTES": 9223372036854775807, + "MEMORY_LIMIT_IN_BYTES": get_memory_limit(), + "NUMBER_OF_CORES": get_number_of_cores() + } + + env_vars = {k: v for k, v in env_vars.items() if v is not None} + + if env_vars.get("MEMORY_LIMIT_IN_BYTES", 0) >= 92233720368547: + env_vars["NO_MEMORY_LIMIT"] = "true" + + for key, value in env_vars.items(): + print("{0}={1}".format(key, value)) diff --git a/root/usr/bin/container-entrypoint b/root/usr/bin/container-entrypoint new file mode 100755 index 0000000..5fc4448 --- /dev/null +++ b/root/usr/bin/container-entrypoint @@ -0,0 +1,3 @@ +#!/bin/bash + +exec "$@" diff --git a/root/usr/bin/run-postgresql b/root/usr/bin/run-postgresql new file mode 100755 index 0000000..1c8de78 --- /dev/null +++ b/root/usr/bin/run-postgresql @@ -0,0 +1,28 @@ +#!/bin/bash + +export ENABLE_REPLICATION=${ENABLE_REPLICATION:-false} + +set -eu +export_vars=$(cgroup-limits) ; export $export_vars + +source "${CONTAINER_SCRIPTS_PATH}/common.sh" + +set_pgdata +check_env_vars +generate_passwd_file +generate_postgresql_config + +if [ ! -f "$PGDATA/postgresql.conf" ]; then + initialize_database + NEED_TO_CREATE_USERS=yes +fi + +pg_ctl -w start -o "-h ''" +if [ "${NEED_TO_CREATE_USERS:-}" == "yes" ]; then + create_users +fi +set_passwords +pg_ctl stop + +unset_env_vars +exec postgres "$@" diff --git a/root/usr/bin/run-postgresql-master b/root/usr/bin/run-postgresql-master new file mode 100755 index 0000000..79e7cc2 --- /dev/null +++ b/root/usr/bin/run-postgresql-master @@ -0,0 +1,5 @@ +#!/bin/bash + +export ENABLE_REPLICATION=true + +exec run-postgresql "$@" diff --git a/root/usr/bin/run-postgresql-slave b/root/usr/bin/run-postgresql-slave new file mode 100755 index 0000000..5d42d0d --- /dev/null +++ b/root/usr/bin/run-postgresql-slave @@ -0,0 +1,36 @@ +#!/bin/bash + +export ENABLE_REPLICATION=true + +set -eu +export_vars=$(cgroup-limits) ; export $export_vars + +source "$CONTAINER_SCRIPTS_PATH"/common.sh + +set_pgdata + +function initialize_replica() { + echo "Initializing PostgreSQL slave ..." + # TODO: Validate and reuse existing data? + rm -rf $PGDATA + PGPASSWORD="${POSTGRESQL_MASTER_PASSWORD}" pg_basebackup -x --no-password --pgdata ${PGDATA} --host=${MASTER_FQDN} --port=5432 -U "${POSTGRESQL_MASTER_USER}" + + # PostgreSQL recovery configuration. + generate_postgresql_recovery_config + cat >> "$PGDATA/recovery.conf" <&2 "error: $1" + fi + + cat >&2 </dev/null) + # FIXME: This is for debugging (docker run) + if [ -v POSTGRESQL_MASTER_IP ]; then + endpoints=${POSTGRESQL_MASTER_IP:-} + fi + if [ -z "$endpoints" ]; then + >&2 echo "Failed to resolve PostgreSQL master IP address" + exit 3 + fi + echo -n "$(echo $endpoints | cut -d ' ' -f 1)" +} + +# New config is generated every time a container is created. It only contains +# additional custom settings and is included from $PGDATA/postgresql.conf. +function generate_postgresql_config() { + envsubst \ + < "${CONTAINER_SCRIPTS_PATH}/openshift-custom-postgresql.conf.template" \ + > "${POSTGRESQL_CONFIG_FILE}" + + if [ "${ENABLE_REPLICATION}" == "true" ]; then + envsubst \ + < "${CONTAINER_SCRIPTS_PATH}/openshift-custom-postgresql-replication.conf.template" \ + >> "${POSTGRESQL_CONFIG_FILE}" + fi +} + +function generate_postgresql_recovery_config() { + envsubst \ + < "${CONTAINER_SCRIPTS_PATH}/openshift-custom-recovery.conf.template" \ + > "${POSTGRESQL_RECOVERY_FILE}" +} + +# Generate passwd file based on current uid +function generate_passwd_file() { + export USER_ID=$(id -u) + export GROUP_ID=$(id -g) + grep -v ^postgres /etc/passwd > "$HOME/passwd" + echo "postgres:x:${USER_ID}:${GROUP_ID}:PostgreSQL Server:${HOME}:/bin/bash" >> "$HOME/passwd" + export LD_PRELOAD=libnss_wrapper.so + export NSS_WRAPPER_PASSWD=${HOME}/passwd + export NSS_WRAPPER_GROUP=/etc/group +} + +function initialize_database() { + # Initialize the database cluster with utf8 support enabled by default. + # This might affect performance, see: + # http://www.postgresql.org/docs/9.5/static/locale.html + LANG=${LANG:-en_US.utf8} initdb + + # PostgreSQL configuration. + cat >> "$PGDATA/postgresql.conf" <> "$PGDATA/pg_hba.conf" <