From 21ae3f4ea023117e4c0b837bf091888ab7e8331a Mon Sep 17 00:00:00 2001 From: "Petr \"Stone\" Hracek" Date: Thu, 20 Apr 2017 12:49:17 +0200 Subject: [PATCH 01/47] Using f26 tags Signed-off-by: Petr "Stone" Hracek --- postgresql.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/postgresql.yaml b/postgresql.yaml index 0e98fea..2351873 100644 --- a/postgresql.yaml +++ b/postgresql.yaml @@ -18,11 +18,11 @@ data: - PostgreSQL dependencies: buildrequires: - base-runtime: master - common-build-dependencies: master - perl: master + base-runtime: f26 + common-build-dependencies: f26 + perl: f26 requires: - base-runtime: master + base-runtime: f26 api: rpms: - postgresql From 14f7b4ae610327a7a0541341c3ed6ccd759f8765 Mon Sep 17 00:00:00 2001 From: "Petr \"Stone\" Hracek" Date: Thu, 20 Apr 2017 12:55:32 +0200 Subject: [PATCH 02/47] Fix perl dependencies Signed-off-by: Petr "Stone" Hracek --- postgresql.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/postgresql.yaml b/postgresql.yaml index 0e98fea..4bcb6cb 100644 --- a/postgresql.yaml +++ b/postgresql.yaml @@ -23,6 +23,7 @@ data: perl: master requires: base-runtime: master + perl: master api: rpms: - postgresql @@ -38,6 +39,17 @@ data: rpms: # part of common-build-dependencies: # multilib-rpm-config, gettext + postgresql: + rationale: 'Build dependency (for postgresql-contrib).' + ref: f26 + buildorder: 10 + autoconf: + rationale: 'Build dependency (for postgresql-contrib).' + ref: f26 uuid: rationale: 'Build dependency (for postgresql-contrib).' ref: f26 + buildorder: 1 + perl-Data-UUID: + rationale: 'Build dependency (for postgresql-contrib).' + ref: f26 From f96469031557b754a5637eac79b6fdc32b5ff767 Mon Sep 17 00:00:00 2001 From: "Petr \"Stone\" Hracek" Date: Fri, 21 Apr 2017 13:22:38 +0200 Subject: [PATCH 03/47] Add common-build-dependencies-bootstrap Signed-off-by: Petr "Stone" Hracek --- postgresql.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/postgresql.yaml b/postgresql.yaml index 4bcb6cb..4a3fef1 100644 --- a/postgresql.yaml +++ b/postgresql.yaml @@ -19,7 +19,9 @@ data: dependencies: buildrequires: base-runtime: master - common-build-dependencies: master + common-build-dependencies: f26 + common-build-dependencies-bootstrap: f26 + shared-userspace: f26 perl: master requires: base-runtime: master @@ -43,9 +45,6 @@ data: rationale: 'Build dependency (for postgresql-contrib).' ref: f26 buildorder: 10 - autoconf: - rationale: 'Build dependency (for postgresql-contrib).' - ref: f26 uuid: rationale: 'Build dependency (for postgresql-contrib).' ref: f26 From 398481abedba0ee1a2cb1876aac267822544afa1 Mon Sep 17 00:00:00 2001 From: "Petr \"Stone\" Hracek" Date: Mon, 24 Apr 2017 15:58:23 +0200 Subject: [PATCH 04/47] Add elinks to private-f27-modules Signed-off-by: Petr "Stone" Hracek --- postgresql.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/postgresql.yaml b/postgresql.yaml index 4a3fef1..73d78c5 100644 --- a/postgresql.yaml +++ b/postgresql.yaml @@ -52,3 +52,7 @@ data: perl-Data-UUID: rationale: 'Build dependency (for postgresql-contrib).' ref: f26 + elinks: + rationale: 'Build dependency (for postgresql-contrib).' + ref: private-f27-modules + buildorder: 3 From 9e73316a6c0ea29b599346187f63a9e0d9be903f Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: Fri, 21 Apr 2017 16:23:11 +0200 Subject: [PATCH 05/47] add perl-Data-UUID as component uuid needs it for building --- postgresql.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/postgresql.yaml b/postgresql.yaml index 2351873..64446fb 100644 --- a/postgresql.yaml +++ b/postgresql.yaml @@ -38,6 +38,9 @@ data: rpms: # part of common-build-dependencies: # multilib-rpm-config, gettext + perl-Data-UUID: + rationale: 'Build dependency (for uuid)' + ref: f26 uuid: rationale: 'Build dependency (for postgresql-contrib).' ref: f26 From 0c4a8d07cafb5247bb46d0e5bff21a20de1e09f8 Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: Fri, 21 Apr 2017 17:05:36 +0200 Subject: [PATCH 06/47] make uuid find perl-Data-UUID Tack a buildorder on uuid and sort the components by build order. --- postgresql.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/postgresql.yaml b/postgresql.yaml index 64446fb..11070b3 100644 --- a/postgresql.yaml +++ b/postgresql.yaml @@ -42,5 +42,6 @@ data: rationale: 'Build dependency (for uuid)' ref: f26 uuid: + buildorder: 1 rationale: 'Build dependency (for postgresql-contrib).' ref: f26 From 036b9515cb21a551b8c25246882c5c2739780888 Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: Mon, 24 Apr 2017 12:03:42 +0200 Subject: [PATCH 07/47] require c-b-d-bootstrap module for building --- postgresql.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/postgresql.yaml b/postgresql.yaml index 11070b3..b2cf4f9 100644 --- a/postgresql.yaml +++ b/postgresql.yaml @@ -20,6 +20,7 @@ data: buildrequires: base-runtime: f26 common-build-dependencies: f26 + common-build-dependencies-bootstrap: f26 perl: f26 requires: base-runtime: f26 @@ -38,6 +39,8 @@ data: rpms: # part of common-build-dependencies: # multilib-rpm-config, gettext + # part of common-build-dependencies-bootstrap: + # autoconf(uuid) perl-Data-UUID: rationale: 'Build dependency (for uuid)' ref: f26 From 966765906bd448f1188f694863774f54aaa4f32a Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: Mon, 24 Apr 2017 13:26:18 +0200 Subject: [PATCH 08/47] actually add the postgresql component, d'oh --- postgresql.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/postgresql.yaml b/postgresql.yaml index b2cf4f9..a194298 100644 --- a/postgresql.yaml +++ b/postgresql.yaml @@ -48,3 +48,7 @@ data: buildorder: 1 rationale: 'Build dependency (for postgresql-contrib).' ref: f26 + postgresql: + buildorder: 2 + rationale: 'The API we want to provide.' + ref: f26 From b7e34e62a945f17d3d21fa834ad68b2f37acf172 Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: Mon, 24 Apr 2017 14:03:27 +0200 Subject: [PATCH 09/47] mention source package in rationale --- postgresql.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgresql.yaml b/postgresql.yaml index a194298..a797e1a 100644 --- a/postgresql.yaml +++ b/postgresql.yaml @@ -46,7 +46,7 @@ data: ref: f26 uuid: buildorder: 1 - rationale: 'Build dependency (for postgresql-contrib).' + rationale: 'Build dependency (for postgresql).' ref: f26 postgresql: buildorder: 2 From 9fe61a6e45fc835f37bd8b3d267ac73da35cee40 Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: Mon, 24 Apr 2017 16:34:57 +0200 Subject: [PATCH 10/47] use modularity-tainted branch of postgresql It doesn't need systemtap for building, which pulls in lots of other crud. --- postgresql.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/postgresql.yaml b/postgresql.yaml index a797e1a..460d99b 100644 --- a/postgresql.yaml +++ b/postgresql.yaml @@ -51,4 +51,6 @@ data: postgresql: buildorder: 2 rationale: 'The API we want to provide.' - ref: f26 + # This branch doesn't BR: systemtap(-devel) which pulls in loads of + # other junk + ref: private-f27-module From 7575406d3ab77ed05c235d8199347376e5d20496 Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: Mon, 24 Apr 2017 16:39:29 +0200 Subject: [PATCH 11/47] use shared-userspace module for python2 dep --- postgresql.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/postgresql.yaml b/postgresql.yaml index 460d99b..6797408 100644 --- a/postgresql.yaml +++ b/postgresql.yaml @@ -21,6 +21,7 @@ data: base-runtime: f26 common-build-dependencies: f26 common-build-dependencies-bootstrap: f26 + shared-userspace: f26 perl: f26 requires: base-runtime: f26 @@ -41,6 +42,8 @@ data: # multilib-rpm-config, gettext # part of common-build-dependencies-bootstrap: # autoconf(uuid) + # part of shared-userspace: + # python2 perl-Data-UUID: rationale: 'Build dependency (for uuid)' ref: f26 From c3172dfc8fb44a3bb327f36859abae538242c5cd Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: Mon, 24 Apr 2017 14:02:30 +0200 Subject: [PATCH 12/47] drill down the dependency rabbit hole --- postgresql.yaml | 116 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 114 insertions(+), 2 deletions(-) diff --git a/postgresql.yaml b/postgresql.yaml index 6797408..0b643af 100644 --- a/postgresql.yaml +++ b/postgresql.yaml @@ -44,16 +44,128 @@ data: # autoconf(uuid) # part of shared-userspace: # python2 + #chrpath: + # rationale: 'Build dependency (for expect)' + # ref: f26 + docbook-dtds: + rationale: >- + Build dependency (for docbook-utils), + runtime dependency (for docbook-style-dsssl) + ref: f26 + docbook-style-dsssl: + rationale: 'Build dependency (for docbook-utils)' + ref: f26 + #libedit: + # rationale: 'Build dependency (for js)' + # ref: f26 + libxslt: + rationale: 'Build dependency (for postgresql)' + ref: f26 + oniguruma: + rationale: 'Build dependency (for slang)' + ref: f26 + opensp: + rationale: 'Build dependency (for openjade)' + ref: f26 perl-Data-UUID: rationale: 'Build dependency (for uuid)' ref: f26 + #perl-Class-Inspector: + # rationale: 'Build dependency (for perl-File-ShareDir)' + # ref: f26 + perl-SGMLSpm: + rationale: 'Build dependency (for docbook-utils)' + ref: f26 + rsh: + rationale: 'Build dependency (for lynx)' + ref: f26 + telnet: + rationale: 'Build dependency (for lynx)' + ref: f26 + tcl: + rationale: 'Build dependency (for postgresql)' + ref: f26 + #expect: + # buildorder: 1 + # rationale: 'Build dependency (for dejagnu)' + # ref: f26 + openjade: + buildorder: 1 + rationale: 'Build dependency (for docbook-utils)' + ref: f26 + #perl-File-ShareDir: + # buildorder: 1 + # rationale: 'Build dependency (for perl-libintl-perl)' + # ref: f26 + slang: + buildorder: 1 + rationale: 'Build dependency (for lynx)' + ref: f26 uuid: buildorder: 1 rationale: 'Build dependency (for postgresql).' ref: f26 - postgresql: + docbook-utils: buildorder: 2 + rationale: 'Build dependency (for postgresql)' + ref: f26 + #perl-libintl-perl: + # buildorder: 2 + # rationale: >- + # Build dependency (for gpm, screen), runtime dependency (for texinfo) + # ref: f26 + #perl-Module-Package: + # buildorder: 2 + # rationale: >- + # Build dependency (for perl-Module-Package-Au, perl-Unicode-EastAsianWidth) + # ref: f26 + #perl-Module-Package-Au: + # buildorder: 2 + # rationale: 'Build dependency (for perl-Unicode-EastAsianWidth)' + # ref: f26 + #perl-Pod-Markdown: + # buildorder: 2 + # rationale: 'Build dependency (for perl-Unicode-EastAsianWidth)' + # ref: f26 + #perl-Unicode-EastAsianWidth: + # buildorder: 3 + # rationale: >- + # Build dependency (for gpm, screen), runtime dependency (for texinfo) + # ref: f26 + #perl-Text-Unidecode: + # buildorder: 3 + # rationale: >- + # Build dependency (for gpm, screen), runtime dependency (for texinfo) + # ref: f26 + #gpm: + # buildorder: 4 + # rationale: 'Build dependency (for elinks)' + # ref: f26 + #screen: + # buildorder: 4 + # rationale: 'Build dependency (for dejagnu)' + # ref: f26 + #dejagnu: + # buildorder: 5 + # rationale: 'Build dependency (for autoconf213)' + # ref: f26 + #autoconf213: + # buildorder: 6 + # rationale: 'Build dependency (for js)' + # ref: f26 + #js: + # buildorder: 7 + # rationale: 'Build dependency (for elinks)' + # ref: f26 + lynx: + buildorder: 8 + rationale: >- + Build dependency (for postgresql), + runtime dependency (for docbook-utils) + ref: f26 + postgresql: + buildorder: 9 rationale: 'The API we want to provide.' # This branch doesn't BR: systemtap(-devel) which pulls in loads of # other junk - ref: private-f27-module + ref: private-f27-modules From e7715dc4eaacc85b9a670e451bd09e79a9f52d3b Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: Thu, 27 Apr 2017 10:40:06 +0200 Subject: [PATCH 13/47] remove elinks-induced cruft --- postgresql.yaml | 65 ------------------------------------------------- 1 file changed, 65 deletions(-) diff --git a/postgresql.yaml b/postgresql.yaml index 0b643af..952f326 100644 --- a/postgresql.yaml +++ b/postgresql.yaml @@ -44,9 +44,6 @@ data: # autoconf(uuid) # part of shared-userspace: # python2 - #chrpath: - # rationale: 'Build dependency (for expect)' - # ref: f26 docbook-dtds: rationale: >- Build dependency (for docbook-utils), @@ -55,9 +52,6 @@ data: docbook-style-dsssl: rationale: 'Build dependency (for docbook-utils)' ref: f26 - #libedit: - # rationale: 'Build dependency (for js)' - # ref: f26 libxslt: rationale: 'Build dependency (for postgresql)' ref: f26 @@ -70,9 +64,6 @@ data: perl-Data-UUID: rationale: 'Build dependency (for uuid)' ref: f26 - #perl-Class-Inspector: - # rationale: 'Build dependency (for perl-File-ShareDir)' - # ref: f26 perl-SGMLSpm: rationale: 'Build dependency (for docbook-utils)' ref: f26 @@ -85,18 +76,10 @@ data: tcl: rationale: 'Build dependency (for postgresql)' ref: f26 - #expect: - # buildorder: 1 - # rationale: 'Build dependency (for dejagnu)' - # ref: f26 openjade: buildorder: 1 rationale: 'Build dependency (for docbook-utils)' ref: f26 - #perl-File-ShareDir: - # buildorder: 1 - # rationale: 'Build dependency (for perl-libintl-perl)' - # ref: f26 slang: buildorder: 1 rationale: 'Build dependency (for lynx)' @@ -109,54 +92,6 @@ data: buildorder: 2 rationale: 'Build dependency (for postgresql)' ref: f26 - #perl-libintl-perl: - # buildorder: 2 - # rationale: >- - # Build dependency (for gpm, screen), runtime dependency (for texinfo) - # ref: f26 - #perl-Module-Package: - # buildorder: 2 - # rationale: >- - # Build dependency (for perl-Module-Package-Au, perl-Unicode-EastAsianWidth) - # ref: f26 - #perl-Module-Package-Au: - # buildorder: 2 - # rationale: 'Build dependency (for perl-Unicode-EastAsianWidth)' - # ref: f26 - #perl-Pod-Markdown: - # buildorder: 2 - # rationale: 'Build dependency (for perl-Unicode-EastAsianWidth)' - # ref: f26 - #perl-Unicode-EastAsianWidth: - # buildorder: 3 - # rationale: >- - # Build dependency (for gpm, screen), runtime dependency (for texinfo) - # ref: f26 - #perl-Text-Unidecode: - # buildorder: 3 - # rationale: >- - # Build dependency (for gpm, screen), runtime dependency (for texinfo) - # ref: f26 - #gpm: - # buildorder: 4 - # rationale: 'Build dependency (for elinks)' - # ref: f26 - #screen: - # buildorder: 4 - # rationale: 'Build dependency (for dejagnu)' - # ref: f26 - #dejagnu: - # buildorder: 5 - # rationale: 'Build dependency (for autoconf213)' - # ref: f26 - #autoconf213: - # buildorder: 6 - # rationale: 'Build dependency (for js)' - # ref: f26 - #js: - # buildorder: 7 - # rationale: 'Build dependency (for elinks)' - # ref: f26 lynx: buildorder: 8 rationale: >- From c0f73c53af1a0090e26da5032d84d957268fe0b9 Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: Thu, 27 Apr 2017 10:40:42 +0200 Subject: [PATCH 14/47] tidy up build order --- postgresql.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/postgresql.yaml b/postgresql.yaml index 952f326..e526a60 100644 --- a/postgresql.yaml +++ b/postgresql.yaml @@ -93,13 +93,13 @@ data: rationale: 'Build dependency (for postgresql)' ref: f26 lynx: - buildorder: 8 + buildorder: 2 rationale: >- Build dependency (for postgresql), runtime dependency (for docbook-utils) ref: f26 postgresql: - buildorder: 9 + buildorder: 3 rationale: 'The API we want to provide.' # This branch doesn't BR: systemtap(-devel) which pulls in loads of # other junk From 1e402a48e3a0457a35885b9e741dcd9e4bd5f021 Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: Thu, 27 Apr 2017 11:18:52 +0200 Subject: [PATCH 15/47] amend postgresql private branch comment --- postgresql.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/postgresql.yaml b/postgresql.yaml index e526a60..662ff4c 100644 --- a/postgresql.yaml +++ b/postgresql.yaml @@ -101,6 +101,6 @@ data: postgresql: buildorder: 3 rationale: 'The API we want to provide.' - # This branch doesn't BR: systemtap(-devel) which pulls in loads of - # other junk + # This branch doesn't BR: systemtap(-devel), elinks which pulls in + # loads of other junk ref: private-f27-modules From 9362e1ab0345e4dcb307527a232a7653d963101c Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: Thu, 27 Apr 2017 15:29:20 +0200 Subject: [PATCH 16/47] use perl:master until we have a working f26 module The perl module built fine in f26, but isn't flagged as 'active' in PDC, i.e. not available for subsequent builds. This needs to be reverted and requires a rebuild when PDC is fixed. --- postgresql.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgresql.yaml b/postgresql.yaml index 662ff4c..374d099 100644 --- a/postgresql.yaml +++ b/postgresql.yaml @@ -22,7 +22,7 @@ data: common-build-dependencies: f26 common-build-dependencies-bootstrap: f26 shared-userspace: f26 - perl: f26 + perl: master requires: base-runtime: f26 api: From 81611552803c914c353ec762ae4a730c29f1ce33 Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: Thu, 27 Apr 2017 18:11:34 +0200 Subject: [PATCH 17/47] use perl:f26 again --- postgresql.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgresql.yaml b/postgresql.yaml index 374d099..662ff4c 100644 --- a/postgresql.yaml +++ b/postgresql.yaml @@ -22,7 +22,7 @@ data: common-build-dependencies: f26 common-build-dependencies-bootstrap: f26 shared-userspace: f26 - perl: master + perl: f26 requires: base-runtime: f26 api: From ba9d082c942d17210553725349920c0fc4e4149b Mon Sep 17 00:00:00 2001 From: Radoslav Pitonak Date: Thu, 4 May 2017 10:50:54 +0200 Subject: [PATCH 18/47] Dockerfile based on BRT, test suite for module --- Dockerfile | 51 ++++ Makefile | 15 ++ repos/fedora.repo | 5 + repos/postgresql.repo | 4 + root/help.1 | 190 +++++++++++++++ root/usr/bin/cgroup-limits | 92 ++++++++ root/usr/bin/container-entrypoint | 3 + root/usr/bin/run-postgresql | 28 +++ root/usr/bin/run-postgresql-master | 5 + root/usr/bin/run-postgresql-slave | 36 +++ root/usr/libexec/fix-permissions | 7 + .../container-scripts/postgresql/README.md | 98 ++++++++ .../container-scripts/postgresql/common.sh | 220 ++++++++++++++++++ ...ustom-postgresql-replication.conf.template | 7 + .../openshift-custom-postgresql.conf.template | 21 ++ .../openshift-custom-recovery.conf.template | 9 + tests/Makefile | 6 + tests/config.yaml | 14 ++ tests/rpmvalidator.py | 56 +++++ tests/sanity1.py | 46 ++++ tests/sanity2.py | 47 ++++ tests/simpleTest.py | 47 ++++ 22 files changed, 1007 insertions(+) create mode 100644 Dockerfile create mode 100644 Makefile create mode 100644 repos/fedora.repo create mode 100644 repos/postgresql.repo create mode 100644 root/help.1 create mode 100755 root/usr/bin/cgroup-limits create mode 100755 root/usr/bin/container-entrypoint create mode 100755 root/usr/bin/run-postgresql create mode 100755 root/usr/bin/run-postgresql-master create mode 100755 root/usr/bin/run-postgresql-slave create mode 100755 root/usr/libexec/fix-permissions create mode 100644 root/usr/share/container-scripts/postgresql/README.md create mode 100644 root/usr/share/container-scripts/postgresql/common.sh create mode 100644 root/usr/share/container-scripts/postgresql/openshift-custom-postgresql-replication.conf.template create mode 100644 root/usr/share/container-scripts/postgresql/openshift-custom-postgresql.conf.template create mode 100644 root/usr/share/container-scripts/postgresql/openshift-custom-recovery.conf.template create mode 100644 tests/Makefile create mode 100644 tests/config.yaml create mode 100644 tests/rpmvalidator.py create mode 100644 tests/sanity1.py create mode 100644 tests/sanity2.py create mode 100644 tests/simpleTest.py diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ae334f6 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,51 @@ +FROM baseruntime/baseruntime:latest + +# PostgreSQL image for OpenShift. +# Volumes: +# * /var/lib/psql/data - Database cluster for PostgreSQL +# Environment: +# * $POSTGRESQL_USER - Database user name +# * $POSTGRESQL_PASSWORD - User's password +# * $POSTGRESQL_DATABASE - Name of the database to create +# * $POSTGRESQL_ADMIN_PASSWORD (Optional) - Password for the 'postgres' +# PostgreSQL administrative account + +ENV POSTGRESQL_VERSION=9.5 \ + HOME=/var/lib/pgsql \ + PGUSER=postgres + +LABEL io.k8s.description="PostgreSQL is an advanced Object-Relational database management system" \ + io.k8s.display-name="PostgreSQL 9.5" \ + io.openshift.expose-services="5432:postgresql" \ + io.openshift.tags="database,postgresql,postgresql95" \ + com.redhat.component="postgresql" \ + maintainer="Pavel Raiskup " \ + name="$FCG/postgresql" \ + version="0" \ + release="1.$DISTTAG" \ + architecture="x86_64" \ + usage="Run without arguments to get usage info." \ + help="/help.1" + +EXPOSE 5432 + +ADD root / + +COPY repos/* /etc/yum.repos.d/ + +RUN microdnf --nodocs --enablerepo postgresql install -y postgresql postgresql-server postgresql-contrib && \ + microdnf --nodocs --enablerepo fedora install -y rsync tar gettext bind-utils nss_wrapper findutils python && \ + microdnf clean all && \ + test "$(id postgres)" = "uid=26(postgres) gid=26(postgres) groups=26(postgres)" && \ + mkdir -p /var/lib/pgsql/data && \ + /usr/libexec/fix-permissions /var/lib/pgsql && \ + /usr/libexec/fix-permissions /var/run/postgresql + +# Get prefix path and path to scripts rather than hard-code them in scripts +ENV CONTAINER_SCRIPTS_PATH=/usr/share/container-scripts/postgresql + +VOLUME ["/var/lib/pgsql/data"] + +USER 26 + +CMD ["run-postgresql"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..c635e5e --- /dev/null +++ b/Makefile @@ -0,0 +1,15 @@ +IMAGE_NAME = postgresql + +MODULEMDURL=file://postgresql.yaml + +default: run + +build: + docker build --tag=$(IMAGE_NAME) . + +run: build + docker run -d $(IMAGE_NAME) + +test: build + cd tests; MODULE=docker MODULEMD=$(MODULEMDURL) URL="docker=$(IMAGE_NAME)" make all + cd tests; MODULE=rpm MODULEMD=$(MODULEMDURL) URL="docker=$(IMAGE_NAME)" make all diff --git a/repos/fedora.repo b/repos/fedora.repo new file mode 100644 index 0000000..f86d71c --- /dev/null +++ b/repos/fedora.repo @@ -0,0 +1,5 @@ +[fedora] +name=fedora +baseurl=https://mirrors.nic.cz/fedora/linux/development/26/Everything/x86_64/os/ +enabled=0 + diff --git a/repos/postgresql.repo b/repos/postgresql.repo new file mode 100644 index 0000000..584b510 --- /dev/null +++ b/repos/postgresql.repo @@ -0,0 +1,4 @@ +[postgresql] +name=postgresql +baseurl=https://kojipkgs.fedoraproject.org/repos/module-5a0a295c9673c2a1/latest/x86_64/ +enabled=0 diff --git a/root/help.1 b/root/help.1 new file mode 100644 index 0000000..89c84d9 --- /dev/null +++ b/root/help.1 @@ -0,0 +1,190 @@ +.\"t +.\" WARNING: Do not edit this file manually, it is generated from README.md automatically. +.\" +.\"t +.\" Automatically generated by Pandoc 1.16.0.2 +.\" +.TH "POSTGRESQL\-95\-RHEL7" "1" "February 22, 2017" "Container Image Pages" "" +.hy +.SH PostgreSQL Docker image +.PP +This repository contains Dockerfiles for PostgreSQL images for general +usage and OpenShift. +Users can choose between RHEL and CentOS based images. +.SS Environment variables and volumes +.PP +The image recognizes the following environment variables that you can +set during initialization by passing \f[C]\-e\ VAR=VALUE\f[] to the +Docker run command. +.PP +.TS +tab(@); +lw(26.7n) lw(43.3n). +T{ +Variable name +T}@T{ +Description +T} +_ +T{ +\f[C]POSTGRESQL_USER\f[] +T}@T{ +User name for PostgreSQL account to be created +T} +T{ +\f[C]POSTGRESQL_PASSWORD\f[] +T}@T{ +Password for the user account +T} +T{ +\f[C]POSTGRESQL_DATABASE\f[] +T}@T{ +Database name +T} +T{ +\f[C]POSTGRESQL_ADMIN_PASSWORD\f[] +T}@T{ +Password for the \f[C]postgres\f[] admin account (optional) +T} +.TE +.PP +The following environment variables influence the PostgreSQL +configuration file. +They are all optional. +.PP +.TS +tab(@); +lw(15.7n) lw(37.6n) lw(16.7n). +T{ +Variable name +T}@T{ +Description +T}@T{ +Default +T} +_ +T{ +\f[C]POSTGRESQL_MAX_CONNECTIONS\f[] +T}@T{ +The maximum number of client connections allowed +T}@T{ +100 +T} +T{ +\f[C]POSTGRESQL_MAX_PREPARED_TRANSACTIONS\f[] +T}@T{ +Sets the maximum number of transactions that can be in the "prepared" +state. +If you are using prepared transactions, you will probably want this to +be at least as large as max_connections +T}@T{ +0 +T} +T{ +\f[C]POSTGRESQL_SHARED_BUFFERS\f[] +T}@T{ +Sets how much memory is dedicated to PostgreSQL to use for caching data +T}@T{ +32M +T} +T{ +\f[C]POSTGRESQL_EFFECTIVE_CACHE_SIZE\f[] +T}@T{ +Set to an estimate of how much memory is available for disk caching by +the operating system and within the database itself +T}@T{ +128M +T} +.TE +.PP +You can also set the following mount points by passing the +\f[C]\-v\ /host:/container\f[] flag to Docker. +.PP +.TS +tab(@); +l l. +T{ +Volume mount point +T}@T{ +Description +T} +_ +T{ +\f[C]/var/lib/pgsql/data\f[] +T}@T{ +PostgreSQL database cluster directory +T} +.TE +.PP +\f[B]Notice: When mouting a directory from the host into the container, +ensure that the mounted directory has the appropriate permissions and +that the owner and group of the directory matches the user UID or name +which is running inside the container.\f[] +.SS Usage +.PP +For this, we will assume that you are using the +\f[C]openshift/postgresql\-92\-centos7\f[] image. +If you want to set only the mandatory environment variables and not +store the database in a host directory, execute the following command: +.IP +.nf +\f[C] +$\ docker\ run\ \-d\ \-\-name\ postgresql_database\ \-e\ POSTGRESQL_USER=user\ \-e\ POSTGRESQL_PASSWORD=pass\ \-e\ POSTGRESQL_DATABASE=db\ \-p\ 5432:5432\ openshift/postgresql\-92\-centos7 +\f[] +.fi +.PP +This will create a container named \f[C]postgresql_database\f[] running +PostgreSQL with database \f[C]db\f[] and user with credentials +\f[C]user:pass\f[]. +Port 5432 will be exposed and mapped to the host. +If you want your database to be persistent across container executions, +also add a \f[C]\-v\ /host/db/path:/var/lib/pgsql/data\f[] argument. +This will be the PostgreSQL database cluster directory. +.PP +If the database cluster directory is not initialized, the entrypoint +script will first run +\f[C]initdb\f[] (http://www.postgresql.org/docs/9.2/static/app-initdb.html) +and setup necessary database users and passwords. +After the database is initialized, or if it was already present, +\f[C]postgres\f[] (http://www.postgresql.org/docs/9.2/static/app-postgres.html) +is executed and will run as PID 1. +You can stop the detached container by running +\f[C]docker\ stop\ postgresql_database\f[]. +.SS PostgreSQL auto\-tuning +.PP +When the PostgreSQL image is run with the \f[C]\-\-memory\f[] parameter +set and if there are no values provided for +\f[C]POSTGRESQL_SHARED_BUFFERS\f[] and +\f[C]POSTGRESQL_EFFECTIVE_CACHE_SIZE\f[] those values are automatically +calculated based on the value provided in the \f[C]\-\-memory\f[] +parameter. +.PP +The values are calculated based on the +upstream (https://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server) +formulas. +For the \f[C]shared_buffers\f[] we use 1/4 of given memory and for the +\f[C]effective_cache_size\f[] we set the value to 1/2 of the given +memory. +.SS PostgreSQL admin account +.PP +The admin account \f[C]postgres\f[] has no password set by default, only +allowing local connections. +You can set it by setting the \f[C]POSTGRESQL_ADMIN_PASSWORD\f[] +environment variable when initializing your container. +This will allow you to login to the \f[C]postgres\f[] account remotely. +Local connections will still not require a password. +.SS Changing passwords +.PP +Since passwords are part of the image configuration, the only supported +method to change passwords for the database user +(\f[C]POSTGRESQL_USER\f[]) and \f[C]postgres\f[] admin user is by +changing the environment variables \f[C]POSTGRESQL_PASSWORD\f[] and +\f[C]POSTGRESQL_ADMIN_PASSWORD\f[], respectively. +.PP +Changing database passwords through SQL statements or any way other than +through the environment variables aforementioned will cause a mismatch +between the values stored in the variables and the actual passwords. +Whenever a database container starts it will reset the passwords to the +values stored in the environment variables. +.SH AUTHORS +Red Hat. diff --git a/root/usr/bin/cgroup-limits b/root/usr/bin/cgroup-limits new file mode 100755 index 0000000..b9d4edc --- /dev/null +++ b/root/usr/bin/cgroup-limits @@ -0,0 +1,92 @@ +#!/usr/bin/python + +""" +Script for parsing cgroup information + +This script will read some limits from the cgroup system and parse +them, printing out "VARIABLE=VALUE" on each line for every limit that is +successfully read. Output of this script can be directly fed into +bash's export command. Recommended usage from a bash script: + + set -o errexit + export_vars=$(cgroup-limits) ; export $export_vars + +Variables currently supported: + MAX_MEMORY_LIMIT_IN_BYTES + Maximum possible limit MEMORY_LIMIT_IN_BYTES can have. This is + currently constant value of 9223372036854775807. + MEMORY_LIMIT_IN_BYTES + Maximum amount of user memory in bytes. If this value is set + to the same value as MAX_MEMORY_LIMIT_IN_BYTES, it means that + there is no limit set. The value is taken from + /sys/fs/cgroup/memory/memory.limit_in_bytes + NUMBER_OF_CORES + Number of detected CPU cores that can be used. This value is + calculated from /sys/fs/cgroup/cpuset/cpuset.cpus + NO_MEMORY_LIMIT + Set to "true" if MEMORY_LIMIT_IN_BYTES is so high that the caller + can act as if no memory limit was set. Undefined otherwise. +""" + +from __future__ import print_function +import sys + + +def _read_file(path): + try: + with open(path, 'r') as f: + return f.read().strip() + except IOError: + return None + + +def get_memory_limit(): + """ + Read memory limit, in bytes. + """ + + limit = _read_file('/sys/fs/cgroup/memory/memory.limit_in_bytes') + if limit is None or not limit.isdigit(): + print("Warning: Can't detect memory limit from cgroups", + file=sys.stderr) + return None + return int(limit) + + +def get_number_of_cores(): + """ + Read number of CPU cores. + """ + + core_count = 0 + + line = _read_file('/sys/fs/cgroup/cpuset/cpuset.cpus') + if line is None: + print("Warning: Can't detect number of CPU cores from cgroups", + file=sys.stderr) + return None + + for group in line.split(','): + core_ids = list(map(int, group.split('-'))) + if len(core_ids) == 2: + core_count += core_ids[1] - core_ids[0] + 1 + else: + core_count += 1 + + return core_count + + +if __name__ == "__main__": + env_vars = { + "MAX_MEMORY_LIMIT_IN_BYTES": 9223372036854775807, + "MEMORY_LIMIT_IN_BYTES": get_memory_limit(), + "NUMBER_OF_CORES": get_number_of_cores() + } + + env_vars = {k: v for k, v in env_vars.items() if v is not None} + + if env_vars.get("MEMORY_LIMIT_IN_BYTES", 0) >= 92233720368547: + env_vars["NO_MEMORY_LIMIT"] = "true" + + for key, value in env_vars.items(): + print("{0}={1}".format(key, value)) diff --git a/root/usr/bin/container-entrypoint b/root/usr/bin/container-entrypoint new file mode 100755 index 0000000..5fc4448 --- /dev/null +++ b/root/usr/bin/container-entrypoint @@ -0,0 +1,3 @@ +#!/bin/bash + +exec "$@" diff --git a/root/usr/bin/run-postgresql b/root/usr/bin/run-postgresql new file mode 100755 index 0000000..1c8de78 --- /dev/null +++ b/root/usr/bin/run-postgresql @@ -0,0 +1,28 @@ +#!/bin/bash + +export ENABLE_REPLICATION=${ENABLE_REPLICATION:-false} + +set -eu +export_vars=$(cgroup-limits) ; export $export_vars + +source "${CONTAINER_SCRIPTS_PATH}/common.sh" + +set_pgdata +check_env_vars +generate_passwd_file +generate_postgresql_config + +if [ ! -f "$PGDATA/postgresql.conf" ]; then + initialize_database + NEED_TO_CREATE_USERS=yes +fi + +pg_ctl -w start -o "-h ''" +if [ "${NEED_TO_CREATE_USERS:-}" == "yes" ]; then + create_users +fi +set_passwords +pg_ctl stop + +unset_env_vars +exec postgres "$@" diff --git a/root/usr/bin/run-postgresql-master b/root/usr/bin/run-postgresql-master new file mode 100755 index 0000000..79e7cc2 --- /dev/null +++ b/root/usr/bin/run-postgresql-master @@ -0,0 +1,5 @@ +#!/bin/bash + +export ENABLE_REPLICATION=true + +exec run-postgresql "$@" diff --git a/root/usr/bin/run-postgresql-slave b/root/usr/bin/run-postgresql-slave new file mode 100755 index 0000000..5d42d0d --- /dev/null +++ b/root/usr/bin/run-postgresql-slave @@ -0,0 +1,36 @@ +#!/bin/bash + +export ENABLE_REPLICATION=true + +set -eu +export_vars=$(cgroup-limits) ; export $export_vars + +source "$CONTAINER_SCRIPTS_PATH"/common.sh + +set_pgdata + +function initialize_replica() { + echo "Initializing PostgreSQL slave ..." + # TODO: Validate and reuse existing data? + rm -rf $PGDATA + PGPASSWORD="${POSTGRESQL_MASTER_PASSWORD}" pg_basebackup -x --no-password --pgdata ${PGDATA} --host=${MASTER_FQDN} --port=5432 -U "${POSTGRESQL_MASTER_USER}" + + # PostgreSQL recovery configuration. + generate_postgresql_recovery_config + cat >> "$PGDATA/recovery.conf" <&2 "error: $1" + fi + + cat >&2 </dev/null) + # FIXME: This is for debugging (docker run) + if [ -v POSTGRESQL_MASTER_IP ]; then + endpoints=${POSTGRESQL_MASTER_IP:-} + fi + if [ -z "$endpoints" ]; then + >&2 echo "Failed to resolve PostgreSQL master IP address" + exit 3 + fi + echo -n "$(echo $endpoints | cut -d ' ' -f 1)" +} + +# New config is generated every time a container is created. It only contains +# additional custom settings and is included from $PGDATA/postgresql.conf. +function generate_postgresql_config() { + envsubst \ + < "${CONTAINER_SCRIPTS_PATH}/openshift-custom-postgresql.conf.template" \ + > "${POSTGRESQL_CONFIG_FILE}" + + if [ "${ENABLE_REPLICATION}" == "true" ]; then + envsubst \ + < "${CONTAINER_SCRIPTS_PATH}/openshift-custom-postgresql-replication.conf.template" \ + >> "${POSTGRESQL_CONFIG_FILE}" + fi +} + +function generate_postgresql_recovery_config() { + envsubst \ + < "${CONTAINER_SCRIPTS_PATH}/openshift-custom-recovery.conf.template" \ + > "${POSTGRESQL_RECOVERY_FILE}" +} + +# Generate passwd file based on current uid +function generate_passwd_file() { + export USER_ID=$(id -u) + export GROUP_ID=$(id -g) + grep -v ^postgres /etc/passwd > "$HOME/passwd" + echo "postgres:x:${USER_ID}:${GROUP_ID}:PostgreSQL Server:${HOME}:/bin/bash" >> "$HOME/passwd" + export LD_PRELOAD=libnss_wrapper.so + export NSS_WRAPPER_PASSWD=${HOME}/passwd + export NSS_WRAPPER_GROUP=/etc/group +} + +function initialize_database() { + # Initialize the database cluster with utf8 support enabled by default. + # This might affect performance, see: + # http://www.postgresql.org/docs/9.5/static/locale.html + LANG=${LANG:-en_US.utf8} initdb + + # PostgreSQL configuration. + cat >> "$PGDATA/postgresql.conf" <> "$PGDATA/pg_hba.conf" < +# + +from avocado import main +from moduleframework import module_framework + + +class ExampleRpmValidation(module_framework.AvocadoTest): + """ + :avocado: enable + """ + fhs_base_paths = [ + '/bin', + '/boot', + '/dev', + '/etc', + '/home', + '/lib', + '/lib64', + '/media', + '/mnt', + '/opt', + '/proc', + '/root', + '/run', + '/sbin', + '/sys', + '/srv', + '/tmp', + '/usr/bin' + ] + + def testPaths(self): + self.start() + for directory in self.fhs_base_paths: + self.run("test -d %s" % directory) diff --git a/tests/sanity1.py b/tests/sanity1.py new file mode 100644 index 0000000..677abff --- /dev/null +++ b/tests/sanity1.py @@ -0,0 +1,46 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# This Modularity Testing Framework helps you to write tests for modules +# Copyright (C) 2017 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# he Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Authors: Rado Pitonak +# + +from avocado import main +from avocado.core import exceptions +from moduleframework import module_framework + + +class SanityCheck1(module_framework.AvocadoTest): + """ + :avocado: enable + """ + + def test1(self): + self.start() + self.run("ls / | grep bin") + + # check if perl is installed in the right version. + def test2PostgresVersion(self): + postgresVersion = "9.6" + self.start() + self.run("postgres --version | grep " + postgresVersion) + + +if __name__ == '__main__': + main() diff --git a/tests/sanity2.py b/tests/sanity2.py new file mode 100644 index 0000000..a6bee42 --- /dev/null +++ b/tests/sanity2.py @@ -0,0 +1,47 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# This Modularity Testing Framework helps you to write tests for modules +# Copyright (C) 2017 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# he Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Authors: Rado Pitonak +# + +from avocado import main +from avocado.core import exceptions +from moduleframework import module_framework +import time + + +class SanityCheck2(module_framework.AvocadoTest): + """ + :avocado: enable + """ + + def testConnectToDB(self): + self.start() + time.sleep(5) + self.run("psql -U $POSTGRESQL_USER -d $POSTGRESQL_DATABASE -c 'SELECT 1'") + + def testCreateTable(self): + self.start() + time.sleep(5) + self.run("psql -U $POSTGRESQL_USER -d $POSTGRESQL_DATABASE -c 'CREATE TABLE mytable(ID INT PRIMARY KEY NOT NULL);'") + + +if __name__ == '__main__': + main() diff --git a/tests/simpleTest.py b/tests/simpleTest.py new file mode 100644 index 0000000..780e387 --- /dev/null +++ b/tests/simpleTest.py @@ -0,0 +1,47 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# This Modularity Testing Framework helps you to write tests for modules +# Copyright (C) 2017 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# he Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Authors: Jan Scotka +# + +from moduleframework import module_framework +import os + + +class simpleTests(module_framework.AvocadoTest): + """ + :avocado: enable + """ + + def testPath(self): + print ">>>>>>>>>>>>>> ", module_framework.__file__ + print ">>>>>>>>>>>>>> ", __file__ + + def testAssertIn(self): + self.start() + self.assertIn("sbin", self.run("ls /").stdout) + + def testInsideModule(self): + self.start() + self.assertEqual("a", self.run("echo a").stdout.strip()) + + def testCommandOnHost(self): + self.start() + self.assertEqual("a", self.runHost("echo a").stdout.strip()) From fa7e7e5dec0b064aceb1cffb4c5394f64b992511 Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: Thu, 11 May 2017 11:56:47 +0200 Subject: [PATCH 19/47] add files for building containter artifacts Lifted in large parts from version 9.5 of the SCL postgresql container. https://github.com/sclorg/postgresql-container --- Dockerfile | 100 ++++++++ module-postgresql.repo.in | 11 + root/help.1 | 190 +++++++++++++++ root/usr/bin/cgroup-limits | 92 ++++++++ root/usr/bin/container-entrypoint | 3 + root/usr/bin/run-postgresql | 28 +++ root/usr/bin/run-postgresql-master | 5 + root/usr/bin/run-postgresql-slave | 36 +++ root/usr/libexec/fix-permissions | 7 + .../container-scripts/postgresql/README.md | 98 ++++++++ .../container-scripts/postgresql/common.sh | 220 ++++++++++++++++++ ...ustom-postgresql-replication.conf.template | 7 + .../openshift-custom-postgresql.conf.template | 21 ++ .../openshift-custom-recovery.conf.template | 9 + .../container-scripts/postgresql/scl_enable | 3 + 15 files changed, 830 insertions(+) create mode 100644 Dockerfile create mode 100644 module-postgresql.repo.in create mode 100644 root/help.1 create mode 100755 root/usr/bin/cgroup-limits create mode 100755 root/usr/bin/container-entrypoint create mode 100755 root/usr/bin/run-postgresql create mode 100755 root/usr/bin/run-postgresql-master create mode 100755 root/usr/bin/run-postgresql-slave create mode 100755 root/usr/libexec/fix-permissions create mode 100644 root/usr/share/container-scripts/postgresql/README.md create mode 100644 root/usr/share/container-scripts/postgresql/common.sh create mode 100644 root/usr/share/container-scripts/postgresql/openshift-custom-postgresql-replication.conf.template create mode 100644 root/usr/share/container-scripts/postgresql/openshift-custom-postgresql.conf.template create mode 100644 root/usr/share/container-scripts/postgresql/openshift-custom-recovery.conf.template create mode 100644 root/usr/share/container-scripts/postgresql/scl_enable diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..3722091 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,100 @@ +# Container artifact for the PostgreSQL module + +# This should probably rather be something like...: +# FROM registry.fedoraproject.org/module-base-runtime:26 +# ...and probably would: +# - contain microdnf rather than dnf +# - not have any repositories configured +# - need a shared-userspace module repo enabled +FROM registry.fedoraproject.org/fedora:26 + +# PostgreSQL container image +# Exposed ports: +# * 5432/tcp - postgres +# Volumes: +# * /var/lib/psql/data - Database cluster for PostgreSQL + + +ENV NAME=postgresql \ + VERSION=0 \ + RELEASE=1 \ + ARCH=x86_64 \ + POSTGRESQL_VERSION=9.6 \ + LANG=en_US.UTF-8 \ + LC_ALL=en_US.UTF-8 \ + HOME=/var/lib/pgsql \ + PGUSER=postgres \ + POSTGRESQL_MODULE_HASH=5a0a295c9673c2a1 \ + SHARED_USERSPACE_MODULE_HASH=e67c1e728d6aa7be + +LABEL summary = "PostgreSQL is an object-relational DBMS." \ + name = "$FGC/$NAME" \ + version = "$VERSION" \ + release="$RELEASE.$DISTTAG" \ + architecture = "$ARCH" \ + maintainer = "Nils Philippsen " \ + description = "PostgreSQL is an advanced Object-Relational database management system (DBMS). This container contains the programs needed to create and run a PostgreSQL server, which will in turn allow you to create and maintain PostgreSQL databases." \ + vendor="Fedora Project" \ + com.redhat.component="$NAME" \ + org.fedoraproject.component="postgresql" \ + authoritative-source-url="registry.fedoraproject.org" \ + usage="docker run -v :/var/lib/pgsql:Z -p 5432:5432 -e POSTGRESQL_USER= -e POSTGRESQL_PASSWORD= -e POSTGRESQL_DATABASE= modularitycontainers/postgresql" \ + io.k8s.description = "PostgreSQL is an advanced Object-Relational database management system (DBMS). This container contains the programs needed to create and run a PostgreSQL server, which will in turn allow you to create and maintain PostgreSQL databases." \ + io.k8s.display-name="PostgreSQL ${POSTGRESQL_VERSION}" \ + io.openshift.tags="database,postgresql,postgresql96" \ + io.openshift.expose-services="5432/tcp:postgres" + +EXPOSE 5432 + +ADD root / + +# Update packages so everything available is current +RUN dnf update -y --setopt=tsflags=nodocs + +# Need to have charset files, disable normal repo(s), enable module repo(s) +RUN dnf install -y --setopt=tsflags=nodocs 'dnf-command(config-manager)' +RUN dnf install -y glibc-locale-source + +# ORCHESTRATION: +# Newer versions of the shared-userspace module have this, but they're not +# available yet +RUN dnf install -y findutils + +# /usr/bin/envsubst +RUN dnf install -y gettext +# nss_wrapper.so +RUN dnf install -y nss_wrapper + +# No working Python module yet +RUN dnf install -y /usr/bin/python +RUN dnf config-manager --set-disabled fedora +RUN dnf config-manager --set-disabled updates +RUN dnf config-manager --set-disabled rawhide +COPY module-postgresql.repo.in /tmp/module-postgresql.repo.in +RUN sed 's|@POSTGRESQL_MODULE_HASH@|'${POSTGRESQL_MODULE_HASH}'|g; s|@SHARED_USERSPACE_MODULE_HASH@|'${SHARED_USERSPACE_MODULE_HASH}'|g' < /tmp/module-postgresql.repo.in > /etc/yum.repos.d/module-postgresql.repo && rm -f /tmp/module-postgresql.repo.in + +# Install the postgresql server component. +# +# This image must forever use UID 26 for postgres user so our volumes are +# safe in the future. This should *never* change, the last test is there +# to make sure of that. +RUN INSTALL_PKGS="postgresql postgresql-server" && \ + dnf install -y --setopt=tsflags=nodocs $INSTALL_PKGS && \ + rpm -V $INSTALL_PKGS && \ + dnf --enablerepo=\* -y clean all && \ + localedef -f UTF-8 -i en_US en_US.UTF-8 && \ + test "$(id -u postgres)" = "26" && \ + test "$(id -g postgres)" = "26" && \ + mkdir -p /var/lib/pgsql/data && \ + /usr/libexec/fix-permissions /var/lib/pgsql && \ + /usr/libexec/fix-permissions /var/run/postgresql + +# Get prefix path and path to scripts rather than hard-code them in scripts +ENV CONTAINER_SCRIPTS_PATH=/usr/share/container-scripts/postgresql + +VOLUME ["/var/lib/pgsql/data"] + +USER 26 + +ENTRYPOINT ["container-entrypoint"] +CMD ["run-postgresql"] diff --git a/module-postgresql.repo.in b/module-postgresql.repo.in new file mode 100644 index 0000000..f93e3dc --- /dev/null +++ b/module-postgresql.repo.in @@ -0,0 +1,11 @@ +[module-shared-userspace] +name=Repository for the Shared Userspace module +baseurl=https://kojipkgs.fedoraproject.org/repos/module-@SHARED_USERSPACE_MODULE_HASH@/latest/${basearch}/ +enabled=1 +gpgcheck=0 + +[module-postgresql] +name=Repository for the PostgreSQL module +baseurl=https://kojipkgs.fedoraproject.org/repos/module-@POSTGRESQL_MODULE_HASH@/latest/${basearch}/ +enabled=1 +gpgcheck=0 diff --git a/root/help.1 b/root/help.1 new file mode 100644 index 0000000..89c84d9 --- /dev/null +++ b/root/help.1 @@ -0,0 +1,190 @@ +.\"t +.\" WARNING: Do not edit this file manually, it is generated from README.md automatically. +.\" +.\"t +.\" Automatically generated by Pandoc 1.16.0.2 +.\" +.TH "POSTGRESQL\-95\-RHEL7" "1" "February 22, 2017" "Container Image Pages" "" +.hy +.SH PostgreSQL Docker image +.PP +This repository contains Dockerfiles for PostgreSQL images for general +usage and OpenShift. +Users can choose between RHEL and CentOS based images. +.SS Environment variables and volumes +.PP +The image recognizes the following environment variables that you can +set during initialization by passing \f[C]\-e\ VAR=VALUE\f[] to the +Docker run command. +.PP +.TS +tab(@); +lw(26.7n) lw(43.3n). +T{ +Variable name +T}@T{ +Description +T} +_ +T{ +\f[C]POSTGRESQL_USER\f[] +T}@T{ +User name for PostgreSQL account to be created +T} +T{ +\f[C]POSTGRESQL_PASSWORD\f[] +T}@T{ +Password for the user account +T} +T{ +\f[C]POSTGRESQL_DATABASE\f[] +T}@T{ +Database name +T} +T{ +\f[C]POSTGRESQL_ADMIN_PASSWORD\f[] +T}@T{ +Password for the \f[C]postgres\f[] admin account (optional) +T} +.TE +.PP +The following environment variables influence the PostgreSQL +configuration file. +They are all optional. +.PP +.TS +tab(@); +lw(15.7n) lw(37.6n) lw(16.7n). +T{ +Variable name +T}@T{ +Description +T}@T{ +Default +T} +_ +T{ +\f[C]POSTGRESQL_MAX_CONNECTIONS\f[] +T}@T{ +The maximum number of client connections allowed +T}@T{ +100 +T} +T{ +\f[C]POSTGRESQL_MAX_PREPARED_TRANSACTIONS\f[] +T}@T{ +Sets the maximum number of transactions that can be in the "prepared" +state. +If you are using prepared transactions, you will probably want this to +be at least as large as max_connections +T}@T{ +0 +T} +T{ +\f[C]POSTGRESQL_SHARED_BUFFERS\f[] +T}@T{ +Sets how much memory is dedicated to PostgreSQL to use for caching data +T}@T{ +32M +T} +T{ +\f[C]POSTGRESQL_EFFECTIVE_CACHE_SIZE\f[] +T}@T{ +Set to an estimate of how much memory is available for disk caching by +the operating system and within the database itself +T}@T{ +128M +T} +.TE +.PP +You can also set the following mount points by passing the +\f[C]\-v\ /host:/container\f[] flag to Docker. +.PP +.TS +tab(@); +l l. +T{ +Volume mount point +T}@T{ +Description +T} +_ +T{ +\f[C]/var/lib/pgsql/data\f[] +T}@T{ +PostgreSQL database cluster directory +T} +.TE +.PP +\f[B]Notice: When mouting a directory from the host into the container, +ensure that the mounted directory has the appropriate permissions and +that the owner and group of the directory matches the user UID or name +which is running inside the container.\f[] +.SS Usage +.PP +For this, we will assume that you are using the +\f[C]openshift/postgresql\-92\-centos7\f[] image. +If you want to set only the mandatory environment variables and not +store the database in a host directory, execute the following command: +.IP +.nf +\f[C] +$\ docker\ run\ \-d\ \-\-name\ postgresql_database\ \-e\ POSTGRESQL_USER=user\ \-e\ POSTGRESQL_PASSWORD=pass\ \-e\ POSTGRESQL_DATABASE=db\ \-p\ 5432:5432\ openshift/postgresql\-92\-centos7 +\f[] +.fi +.PP +This will create a container named \f[C]postgresql_database\f[] running +PostgreSQL with database \f[C]db\f[] and user with credentials +\f[C]user:pass\f[]. +Port 5432 will be exposed and mapped to the host. +If you want your database to be persistent across container executions, +also add a \f[C]\-v\ /host/db/path:/var/lib/pgsql/data\f[] argument. +This will be the PostgreSQL database cluster directory. +.PP +If the database cluster directory is not initialized, the entrypoint +script will first run +\f[C]initdb\f[] (http://www.postgresql.org/docs/9.2/static/app-initdb.html) +and setup necessary database users and passwords. +After the database is initialized, or if it was already present, +\f[C]postgres\f[] (http://www.postgresql.org/docs/9.2/static/app-postgres.html) +is executed and will run as PID 1. +You can stop the detached container by running +\f[C]docker\ stop\ postgresql_database\f[]. +.SS PostgreSQL auto\-tuning +.PP +When the PostgreSQL image is run with the \f[C]\-\-memory\f[] parameter +set and if there are no values provided for +\f[C]POSTGRESQL_SHARED_BUFFERS\f[] and +\f[C]POSTGRESQL_EFFECTIVE_CACHE_SIZE\f[] those values are automatically +calculated based on the value provided in the \f[C]\-\-memory\f[] +parameter. +.PP +The values are calculated based on the +upstream (https://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server) +formulas. +For the \f[C]shared_buffers\f[] we use 1/4 of given memory and for the +\f[C]effective_cache_size\f[] we set the value to 1/2 of the given +memory. +.SS PostgreSQL admin account +.PP +The admin account \f[C]postgres\f[] has no password set by default, only +allowing local connections. +You can set it by setting the \f[C]POSTGRESQL_ADMIN_PASSWORD\f[] +environment variable when initializing your container. +This will allow you to login to the \f[C]postgres\f[] account remotely. +Local connections will still not require a password. +.SS Changing passwords +.PP +Since passwords are part of the image configuration, the only supported +method to change passwords for the database user +(\f[C]POSTGRESQL_USER\f[]) and \f[C]postgres\f[] admin user is by +changing the environment variables \f[C]POSTGRESQL_PASSWORD\f[] and +\f[C]POSTGRESQL_ADMIN_PASSWORD\f[], respectively. +.PP +Changing database passwords through SQL statements or any way other than +through the environment variables aforementioned will cause a mismatch +between the values stored in the variables and the actual passwords. +Whenever a database container starts it will reset the passwords to the +values stored in the environment variables. +.SH AUTHORS +Red Hat. diff --git a/root/usr/bin/cgroup-limits b/root/usr/bin/cgroup-limits new file mode 100755 index 0000000..b9d4edc --- /dev/null +++ b/root/usr/bin/cgroup-limits @@ -0,0 +1,92 @@ +#!/usr/bin/python + +""" +Script for parsing cgroup information + +This script will read some limits from the cgroup system and parse +them, printing out "VARIABLE=VALUE" on each line for every limit that is +successfully read. Output of this script can be directly fed into +bash's export command. Recommended usage from a bash script: + + set -o errexit + export_vars=$(cgroup-limits) ; export $export_vars + +Variables currently supported: + MAX_MEMORY_LIMIT_IN_BYTES + Maximum possible limit MEMORY_LIMIT_IN_BYTES can have. This is + currently constant value of 9223372036854775807. + MEMORY_LIMIT_IN_BYTES + Maximum amount of user memory in bytes. If this value is set + to the same value as MAX_MEMORY_LIMIT_IN_BYTES, it means that + there is no limit set. The value is taken from + /sys/fs/cgroup/memory/memory.limit_in_bytes + NUMBER_OF_CORES + Number of detected CPU cores that can be used. This value is + calculated from /sys/fs/cgroup/cpuset/cpuset.cpus + NO_MEMORY_LIMIT + Set to "true" if MEMORY_LIMIT_IN_BYTES is so high that the caller + can act as if no memory limit was set. Undefined otherwise. +""" + +from __future__ import print_function +import sys + + +def _read_file(path): + try: + with open(path, 'r') as f: + return f.read().strip() + except IOError: + return None + + +def get_memory_limit(): + """ + Read memory limit, in bytes. + """ + + limit = _read_file('/sys/fs/cgroup/memory/memory.limit_in_bytes') + if limit is None or not limit.isdigit(): + print("Warning: Can't detect memory limit from cgroups", + file=sys.stderr) + return None + return int(limit) + + +def get_number_of_cores(): + """ + Read number of CPU cores. + """ + + core_count = 0 + + line = _read_file('/sys/fs/cgroup/cpuset/cpuset.cpus') + if line is None: + print("Warning: Can't detect number of CPU cores from cgroups", + file=sys.stderr) + return None + + for group in line.split(','): + core_ids = list(map(int, group.split('-'))) + if len(core_ids) == 2: + core_count += core_ids[1] - core_ids[0] + 1 + else: + core_count += 1 + + return core_count + + +if __name__ == "__main__": + env_vars = { + "MAX_MEMORY_LIMIT_IN_BYTES": 9223372036854775807, + "MEMORY_LIMIT_IN_BYTES": get_memory_limit(), + "NUMBER_OF_CORES": get_number_of_cores() + } + + env_vars = {k: v for k, v in env_vars.items() if v is not None} + + if env_vars.get("MEMORY_LIMIT_IN_BYTES", 0) >= 92233720368547: + env_vars["NO_MEMORY_LIMIT"] = "true" + + for key, value in env_vars.items(): + print("{0}={1}".format(key, value)) diff --git a/root/usr/bin/container-entrypoint b/root/usr/bin/container-entrypoint new file mode 100755 index 0000000..5fc4448 --- /dev/null +++ b/root/usr/bin/container-entrypoint @@ -0,0 +1,3 @@ +#!/bin/bash + +exec "$@" diff --git a/root/usr/bin/run-postgresql b/root/usr/bin/run-postgresql new file mode 100755 index 0000000..1c8de78 --- /dev/null +++ b/root/usr/bin/run-postgresql @@ -0,0 +1,28 @@ +#!/bin/bash + +export ENABLE_REPLICATION=${ENABLE_REPLICATION:-false} + +set -eu +export_vars=$(cgroup-limits) ; export $export_vars + +source "${CONTAINER_SCRIPTS_PATH}/common.sh" + +set_pgdata +check_env_vars +generate_passwd_file +generate_postgresql_config + +if [ ! -f "$PGDATA/postgresql.conf" ]; then + initialize_database + NEED_TO_CREATE_USERS=yes +fi + +pg_ctl -w start -o "-h ''" +if [ "${NEED_TO_CREATE_USERS:-}" == "yes" ]; then + create_users +fi +set_passwords +pg_ctl stop + +unset_env_vars +exec postgres "$@" diff --git a/root/usr/bin/run-postgresql-master b/root/usr/bin/run-postgresql-master new file mode 100755 index 0000000..79e7cc2 --- /dev/null +++ b/root/usr/bin/run-postgresql-master @@ -0,0 +1,5 @@ +#!/bin/bash + +export ENABLE_REPLICATION=true + +exec run-postgresql "$@" diff --git a/root/usr/bin/run-postgresql-slave b/root/usr/bin/run-postgresql-slave new file mode 100755 index 0000000..5d42d0d --- /dev/null +++ b/root/usr/bin/run-postgresql-slave @@ -0,0 +1,36 @@ +#!/bin/bash + +export ENABLE_REPLICATION=true + +set -eu +export_vars=$(cgroup-limits) ; export $export_vars + +source "$CONTAINER_SCRIPTS_PATH"/common.sh + +set_pgdata + +function initialize_replica() { + echo "Initializing PostgreSQL slave ..." + # TODO: Validate and reuse existing data? + rm -rf $PGDATA + PGPASSWORD="${POSTGRESQL_MASTER_PASSWORD}" pg_basebackup -x --no-password --pgdata ${PGDATA} --host=${MASTER_FQDN} --port=5432 -U "${POSTGRESQL_MASTER_USER}" + + # PostgreSQL recovery configuration. + generate_postgresql_recovery_config + cat >> "$PGDATA/recovery.conf" <&2 "error: $1" + fi + + cat >&2 </dev/null) + # FIXME: This is for debugging (docker run) + if [ -v POSTGRESQL_MASTER_IP ]; then + endpoints=${POSTGRESQL_MASTER_IP:-} + fi + if [ -z "$endpoints" ]; then + >&2 echo "Failed to resolve PostgreSQL master IP address" + exit 3 + fi + echo -n "$(echo $endpoints | cut -d ' ' -f 1)" +} + +# New config is generated every time a container is created. It only contains +# additional custom settings and is included from $PGDATA/postgresql.conf. +function generate_postgresql_config() { + envsubst \ + < "${CONTAINER_SCRIPTS_PATH}/openshift-custom-postgresql.conf.template" \ + > "${POSTGRESQL_CONFIG_FILE}" + + if [ "${ENABLE_REPLICATION}" == "true" ]; then + envsubst \ + < "${CONTAINER_SCRIPTS_PATH}/openshift-custom-postgresql-replication.conf.template" \ + >> "${POSTGRESQL_CONFIG_FILE}" + fi +} + +function generate_postgresql_recovery_config() { + envsubst \ + < "${CONTAINER_SCRIPTS_PATH}/openshift-custom-recovery.conf.template" \ + > "${POSTGRESQL_RECOVERY_FILE}" +} + +# Generate passwd file based on current uid +function generate_passwd_file() { + export USER_ID=$(id -u) + export GROUP_ID=$(id -g) + grep -v ^postgres /etc/passwd > "$HOME/passwd" + echo "postgres:x:${USER_ID}:${GROUP_ID}:PostgreSQL Server:${HOME}:/bin/bash" >> "$HOME/passwd" + export LD_PRELOAD=libnss_wrapper.so + export NSS_WRAPPER_PASSWD=${HOME}/passwd + export NSS_WRAPPER_GROUP=/etc/group +} + +function initialize_database() { + # Initialize the database cluster with utf8 support enabled by default. + # This might affect performance, see: + # http://www.postgresql.org/docs/9.5/static/locale.html + LANG=${LANG:-en_US.utf8} initdb + + # PostgreSQL configuration. + cat >> "$PGDATA/postgresql.conf" <> "$PGDATA/pg_hba.conf" < Date: Thu, 18 May 2017 11:37:07 +0200 Subject: [PATCH 20/47] weed out unnecessary components Several components are carried by modules lower in the stack now (shared-userspace, common-build-dependencies, base-runtime). --- postgresql.yaml | 25 ++++--------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/postgresql.yaml b/postgresql.yaml index 662ff4c..bc14f87 100644 --- a/postgresql.yaml +++ b/postgresql.yaml @@ -38,23 +38,12 @@ data: filter: {} components: rpms: + # part of base-runtime: + # gettext # part of common-build-dependencies: - # multilib-rpm-config, gettext - # part of common-build-dependencies-bootstrap: - # autoconf(uuid) + # autoconf(uuid), multilib-rpm-config # part of shared-userspace: - # python2 - docbook-dtds: - rationale: >- - Build dependency (for docbook-utils), - runtime dependency (for docbook-style-dsssl) - ref: f26 - docbook-style-dsssl: - rationale: 'Build dependency (for docbook-utils)' - ref: f26 - libxslt: - rationale: 'Build dependency (for postgresql)' - ref: f26 + # docbook-dtds, docbook-style-dsssl, libxslt, perl-SGMLSpm, python2, tcl oniguruma: rationale: 'Build dependency (for slang)' ref: f26 @@ -64,18 +53,12 @@ data: perl-Data-UUID: rationale: 'Build dependency (for uuid)' ref: f26 - perl-SGMLSpm: - rationale: 'Build dependency (for docbook-utils)' - ref: f26 rsh: rationale: 'Build dependency (for lynx)' ref: f26 telnet: rationale: 'Build dependency (for lynx)' ref: f26 - tcl: - rationale: 'Build dependency (for postgresql)' - ref: f26 openjade: buildorder: 1 rationale: 'Build dependency (for docbook-utils)' From 1ae18a54f9d58d0963042c4b65fe34b1a8d74b1c Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: Fri, 19 May 2017 11:46:10 +0200 Subject: [PATCH 21/47] bring in line with github.com/container-images - use baseruntime:latest as base image - reimplement cgroup-limits in shell - take real memory size into account for configuration - don't use nss_wrapper.so - don't reference separate shared-userspace repo - fix usage label - get rid of /var/lib/psql <-> .../data distinction - drop a few unused files from root/ --- Dockerfile | 71 ++++--- README.md | 41 ++++ module-postgresql.repo.in | 8 +- root/help.1 | 190 ------------------ root/usr/bin/cgroup-limits | 171 ++++++++-------- .../container-scripts/postgresql/common.sh | 13 +- .../container-scripts/postgresql/scl_enable | 3 - 7 files changed, 175 insertions(+), 322 deletions(-) create mode 100644 README.md delete mode 100644 root/help.1 delete mode 100644 root/usr/share/container-scripts/postgresql/scl_enable diff --git a/Dockerfile b/Dockerfile index 3722091..98cd566 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,10 +3,9 @@ # This should probably rather be something like...: # FROM registry.fedoraproject.org/module-base-runtime:26 # ...and probably would: -# - contain microdnf rather than dnf # - not have any repositories configured # - need a shared-userspace module repo enabled -FROM registry.fedoraproject.org/fedora:26 +FROM baseruntime/baseruntime:latest # PostgreSQL container image # Exposed ports: @@ -19,13 +18,12 @@ ENV NAME=postgresql \ VERSION=0 \ RELEASE=1 \ ARCH=x86_64 \ + LANG=C.UTF-8 \ + LC_ALL=C.UTF-8 \ POSTGRESQL_VERSION=9.6 \ - LANG=en_US.UTF-8 \ - LC_ALL=en_US.UTF-8 \ HOME=/var/lib/pgsql \ PGUSER=postgres \ - POSTGRESQL_MODULE_HASH=5a0a295c9673c2a1 \ - SHARED_USERSPACE_MODULE_HASH=e67c1e728d6aa7be + POSTGRESQL_MODULE_HASH=d87b0b15567e47f3 LABEL summary = "PostgreSQL is an object-relational DBMS." \ name = "$FGC/$NAME" \ @@ -48,43 +46,44 @@ EXPOSE 5432 ADD root / -# Update packages so everything available is current -RUN dnf update -y --setopt=tsflags=nodocs - -# Need to have charset files, disable normal repo(s), enable module repo(s) -RUN dnf install -y --setopt=tsflags=nodocs 'dnf-command(config-manager)' -RUN dnf install -y glibc-locale-source - -# ORCHESTRATION: -# Newer versions of the shared-userspace module have this, but they're not -# available yet -RUN dnf install -y findutils - -# /usr/bin/envsubst -RUN dnf install -y gettext -# nss_wrapper.so -RUN dnf install -y nss_wrapper - -# No working Python module yet -RUN dnf install -y /usr/bin/python -RUN dnf config-manager --set-disabled fedora -RUN dnf config-manager --set-disabled updates -RUN dnf config-manager --set-disabled rawhide COPY module-postgresql.repo.in /tmp/module-postgresql.repo.in -RUN sed 's|@POSTGRESQL_MODULE_HASH@|'${POSTGRESQL_MODULE_HASH}'|g; s|@SHARED_USERSPACE_MODULE_HASH@|'${SHARED_USERSPACE_MODULE_HASH}'|g' < /tmp/module-postgresql.repo.in > /etc/yum.repos.d/module-postgresql.repo && rm -f /tmp/module-postgresql.repo.in +COPY run_tests.sh /usr/bin/run_tests.sh # Install the postgresql server component. # # This image must forever use UID 26 for postgres user so our volumes are # safe in the future. This should *never* change, the last test is there # to make sure of that. -RUN INSTALL_PKGS="postgresql postgresql-server" && \ - dnf install -y --setopt=tsflags=nodocs $INSTALL_PKGS && \ + +# Notes about the below: +# - first sed cmd: don't ask +# - rpm -e ... microdnf install systemd: workaround to missing +# microdnf downgrade/distro-sync +# - Update packages so everything available is current +# - Need to have charset files, disable normal repo(s), enable module repo(s) +# ORCHESTRATION: +# - Newer versions of the shared-userspace module are supposed to have +# findutils, not there yet +# - gettext: /usr/bin/envsubst +# - no working python module yet + +RUN \ + sed -i 's|/jkaluza/|/ralph/|g' /etc/yum.repos.d/build.repo && \ + rpm -e --justdb --nodeps systemd-libs && \ + microdnf install -y --setopt=tsflags=nodocs systemd && \ + # microdnf update -y --setopt=tsflags=nodocs && \ + # microdnf install -y glibc-locale-source && \ + microdnf install -y findutils && \ + microdnf install -y gettext && \ + # microdnf install -y nss_wrapper && \ + # microdnf install -y /usr/bin/python && \ + sed 's|@POSTGRESQL_MODULE_HASH@|'${POSTGRESQL_MODULE_HASH}'|g; s|${basearch}|'${ARCH}'|g' < /tmp/module-postgresql.repo.in > /etc/yum.repos.d/module-postgresql.repo && rm -f /tmp/module-postgresql.repo.in && \ + INSTALL_PKGS="postgresql postgresql-server" && \ + microdnf install -y --setopt=tsflags=nodocs $INSTALL_PKGS && \ rpm -V $INSTALL_PKGS && \ - dnf --enablerepo=\* -y clean all && \ - localedef -f UTF-8 -i en_US en_US.UTF-8 && \ - test "$(id -u postgres)" = "26" && \ - test "$(id -g postgres)" = "26" && \ + microdnf -y clean all && \ + # localedef -f UTF-8 -i en_US en_US.UTF-8 && \ + test "$(id postgres)" = "uid=26(postgres) gid=26(postgres) groups=26(postgres)" && \ mkdir -p /var/lib/pgsql/data && \ /usr/libexec/fix-permissions /var/lib/pgsql && \ /usr/libexec/fix-permissions /var/run/postgresql @@ -92,7 +91,7 @@ RUN INSTALL_PKGS="postgresql postgresql-server" && \ # Get prefix path and path to scripts rather than hard-code them in scripts ENV CONTAINER_SCRIPTS_PATH=/usr/share/container-scripts/postgresql -VOLUME ["/var/lib/pgsql/data"] +VOLUME ["/var/lib/pgsql"] USER 26 diff --git a/README.md b/README.md new file mode 100644 index 0000000..0d365a5 --- /dev/null +++ b/README.md @@ -0,0 +1,41 @@ +# PostgreSQL + +PostgreSQL is an object-relational Database Management System. + +This is based on the SCL container image for version 9.5 at: + +https://github.com/sclorg/postgresql-container + +## Configuration + +You must either specify the following environment variables: + +* POSTGRESQL_USER +* POSTGRESQL_PASSWORD +* POSTGRESQL_DATABASE + +Or the following environment variable: + +* POSTGRESQL_ADMIN_PASSWORD + +Or both. + +Optional settings: + +* POSTGRESQL_MAX_CONNECTIONS (default: 100) +* POSTGRESQL_MAX_PREPARED_TRANSACTIONS (default: 0) +* POSTGRESQL_SHARED_BUFFERS (default: 32MB) + +## Running in docker + +``` +docker run -v :/var/lib/pgsql:Z -p 5432:5432 -e POSTGRESQL_USER= -e POSTGRESQL_PASSWORD= -e POSTGRESQL_DATABASE= modularitycontainers/postgresql +``` + +Substitute these placeholders with real values: + +* `dbroot_path` - path to the database root on the host which should be mounted + on `/var/lib/pgsql` in the container. Note that the `postgres` user (uid 26) + needs to be able to read and write in that directory. +* `user`, `password`, `database` - details with which a database and user + should be created if the database cluster directory isn't initialized yet. diff --git a/module-postgresql.repo.in b/module-postgresql.repo.in index f93e3dc..96cf8b2 100644 --- a/module-postgresql.repo.in +++ b/module-postgresql.repo.in @@ -1,11 +1,5 @@ -[module-shared-userspace] -name=Repository for the Shared Userspace module -baseurl=https://kojipkgs.fedoraproject.org/repos/module-@SHARED_USERSPACE_MODULE_HASH@/latest/${basearch}/ -enabled=1 -gpgcheck=0 - [module-postgresql] name=Repository for the PostgreSQL module -baseurl=https://kojipkgs.fedoraproject.org/repos/module-@POSTGRESQL_MODULE_HASH@/latest/${basearch}/ +baseurl=https://kojipkgs.fedoraproject.org/repos/module-@POSTGRESQL_MODULE_HASH@-build/latest/${basearch}/ enabled=1 gpgcheck=0 diff --git a/root/help.1 b/root/help.1 deleted file mode 100644 index 89c84d9..0000000 --- a/root/help.1 +++ /dev/null @@ -1,190 +0,0 @@ -.\"t -.\" WARNING: Do not edit this file manually, it is generated from README.md automatically. -.\" -.\"t -.\" Automatically generated by Pandoc 1.16.0.2 -.\" -.TH "POSTGRESQL\-95\-RHEL7" "1" "February 22, 2017" "Container Image Pages" "" -.hy -.SH PostgreSQL Docker image -.PP -This repository contains Dockerfiles for PostgreSQL images for general -usage and OpenShift. -Users can choose between RHEL and CentOS based images. -.SS Environment variables and volumes -.PP -The image recognizes the following environment variables that you can -set during initialization by passing \f[C]\-e\ VAR=VALUE\f[] to the -Docker run command. -.PP -.TS -tab(@); -lw(26.7n) lw(43.3n). -T{ -Variable name -T}@T{ -Description -T} -_ -T{ -\f[C]POSTGRESQL_USER\f[] -T}@T{ -User name for PostgreSQL account to be created -T} -T{ -\f[C]POSTGRESQL_PASSWORD\f[] -T}@T{ -Password for the user account -T} -T{ -\f[C]POSTGRESQL_DATABASE\f[] -T}@T{ -Database name -T} -T{ -\f[C]POSTGRESQL_ADMIN_PASSWORD\f[] -T}@T{ -Password for the \f[C]postgres\f[] admin account (optional) -T} -.TE -.PP -The following environment variables influence the PostgreSQL -configuration file. -They are all optional. -.PP -.TS -tab(@); -lw(15.7n) lw(37.6n) lw(16.7n). -T{ -Variable name -T}@T{ -Description -T}@T{ -Default -T} -_ -T{ -\f[C]POSTGRESQL_MAX_CONNECTIONS\f[] -T}@T{ -The maximum number of client connections allowed -T}@T{ -100 -T} -T{ -\f[C]POSTGRESQL_MAX_PREPARED_TRANSACTIONS\f[] -T}@T{ -Sets the maximum number of transactions that can be in the "prepared" -state. -If you are using prepared transactions, you will probably want this to -be at least as large as max_connections -T}@T{ -0 -T} -T{ -\f[C]POSTGRESQL_SHARED_BUFFERS\f[] -T}@T{ -Sets how much memory is dedicated to PostgreSQL to use for caching data -T}@T{ -32M -T} -T{ -\f[C]POSTGRESQL_EFFECTIVE_CACHE_SIZE\f[] -T}@T{ -Set to an estimate of how much memory is available for disk caching by -the operating system and within the database itself -T}@T{ -128M -T} -.TE -.PP -You can also set the following mount points by passing the -\f[C]\-v\ /host:/container\f[] flag to Docker. -.PP -.TS -tab(@); -l l. -T{ -Volume mount point -T}@T{ -Description -T} -_ -T{ -\f[C]/var/lib/pgsql/data\f[] -T}@T{ -PostgreSQL database cluster directory -T} -.TE -.PP -\f[B]Notice: When mouting a directory from the host into the container, -ensure that the mounted directory has the appropriate permissions and -that the owner and group of the directory matches the user UID or name -which is running inside the container.\f[] -.SS Usage -.PP -For this, we will assume that you are using the -\f[C]openshift/postgresql\-92\-centos7\f[] image. -If you want to set only the mandatory environment variables and not -store the database in a host directory, execute the following command: -.IP -.nf -\f[C] -$\ docker\ run\ \-d\ \-\-name\ postgresql_database\ \-e\ POSTGRESQL_USER=user\ \-e\ POSTGRESQL_PASSWORD=pass\ \-e\ POSTGRESQL_DATABASE=db\ \-p\ 5432:5432\ openshift/postgresql\-92\-centos7 -\f[] -.fi -.PP -This will create a container named \f[C]postgresql_database\f[] running -PostgreSQL with database \f[C]db\f[] and user with credentials -\f[C]user:pass\f[]. -Port 5432 will be exposed and mapped to the host. -If you want your database to be persistent across container executions, -also add a \f[C]\-v\ /host/db/path:/var/lib/pgsql/data\f[] argument. -This will be the PostgreSQL database cluster directory. -.PP -If the database cluster directory is not initialized, the entrypoint -script will first run -\f[C]initdb\f[] (http://www.postgresql.org/docs/9.2/static/app-initdb.html) -and setup necessary database users and passwords. -After the database is initialized, or if it was already present, -\f[C]postgres\f[] (http://www.postgresql.org/docs/9.2/static/app-postgres.html) -is executed and will run as PID 1. -You can stop the detached container by running -\f[C]docker\ stop\ postgresql_database\f[]. -.SS PostgreSQL auto\-tuning -.PP -When the PostgreSQL image is run with the \f[C]\-\-memory\f[] parameter -set and if there are no values provided for -\f[C]POSTGRESQL_SHARED_BUFFERS\f[] and -\f[C]POSTGRESQL_EFFECTIVE_CACHE_SIZE\f[] those values are automatically -calculated based on the value provided in the \f[C]\-\-memory\f[] -parameter. -.PP -The values are calculated based on the -upstream (https://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server) -formulas. -For the \f[C]shared_buffers\f[] we use 1/4 of given memory and for the -\f[C]effective_cache_size\f[] we set the value to 1/2 of the given -memory. -.SS PostgreSQL admin account -.PP -The admin account \f[C]postgres\f[] has no password set by default, only -allowing local connections. -You can set it by setting the \f[C]POSTGRESQL_ADMIN_PASSWORD\f[] -environment variable when initializing your container. -This will allow you to login to the \f[C]postgres\f[] account remotely. -Local connections will still not require a password. -.SS Changing passwords -.PP -Since passwords are part of the image configuration, the only supported -method to change passwords for the database user -(\f[C]POSTGRESQL_USER\f[]) and \f[C]postgres\f[] admin user is by -changing the environment variables \f[C]POSTGRESQL_PASSWORD\f[] and -\f[C]POSTGRESQL_ADMIN_PASSWORD\f[], respectively. -.PP -Changing database passwords through SQL statements or any way other than -through the environment variables aforementioned will cause a mismatch -between the values stored in the variables and the actual passwords. -Whenever a database container starts it will reset the passwords to the -values stored in the environment variables. -.SH AUTHORS -Red Hat. diff --git a/root/usr/bin/cgroup-limits b/root/usr/bin/cgroup-limits index b9d4edc..7ad9527 100755 --- a/root/usr/bin/cgroup-limits +++ b/root/usr/bin/cgroup-limits @@ -1,92 +1,99 @@ -#!/usr/bin/python +#!/bin/bash -""" -Script for parsing cgroup information +# Script for parsing cgroup information +# +# This script will read some limits from the cgroup system and parse +# them, printing out "VARIABLE=VALUE" on each line for every limit that is +# successfully read. Output of this script can be directly fed into +# bash's export command. Recommended usage from a bash script: +# +# set -o errexit +# export_vars=$(cgroup-limits) ; export $export_vars +# +# Variables currently supported: +# MAX_MEMORY_LIMIT_IN_BYTES +# Maximum possible limit MEMORY_LIMIT_IN_BYTES can have. This is +# currently a constant value of 9223372036854775807. +# MEMORY_LIMIT_IN_BYTES +# Maximum amount of user memory in bytes. If this value is set +# to the same value as MAX_MEMORY_LIMIT_IN_BYTES, it means that +# there is no limit set. The value is taken from +# /sys/fs/cgroup/memory/memory.limit_in_bytes +# RAM_SIZE_IN_BYTES +# Available memory in bytes. The value is taken from /proc/meminfo. +# NUMBER_OF_CORES +# Number of detected CPU cores that can be used. This value is +# calculated from /sys/fs/cgroup/cpuset/cpuset.cpus +# NO_MEMORY_LIMIT +# Set to "true" if MEMORY_LIMIT_IN_BYTES is so high that the caller +# can act as if no memory limit was set. Undefined otherwise. -This script will read some limits from the cgroup system and parse -them, printing out "VARIABLE=VALUE" on each line for every limit that is -successfully read. Output of this script can be directly fed into -bash's export command. Recommended usage from a bash script: +get_memory_limit() { + local limit="$(cat /sys/fs/cgroup/memory/memory.limit_in_bytes)" + if [ $? -eq 0 ]; then + echo "$limit" + else + echo "Warning: Can't detect memory limit from cgroups" >&2 + fi +} - set -o errexit - export_vars=$(cgroup-limits) ; export $export_vars +get_meminfo_ram_size() { + local limit="$(cat /proc/meminfo | while read key val unit rest; do if [ "$key" = "MemTotal:" ]; then echo "$val $unit"; break; fi; done)" + if [ $? -eq 0 ]; then + local val="${limit/ *}" + local unit="${limit/* }" + case "$unit" in + kB) + val=$(($val * 1024)) + ;; + MB) + val=$(($val * 1024 * 1024)) + ;; + GB) + val=$(($val * 1024 * 1024 * 1024)) + ;; + esac + echo "$val" + else + echo "Warning: Can't detect RAM site from /proc/meminfo." >&2 + fi +} -Variables currently supported: - MAX_MEMORY_LIMIT_IN_BYTES - Maximum possible limit MEMORY_LIMIT_IN_BYTES can have. This is - currently constant value of 9223372036854775807. - MEMORY_LIMIT_IN_BYTES - Maximum amount of user memory in bytes. If this value is set - to the same value as MAX_MEMORY_LIMIT_IN_BYTES, it means that - there is no limit set. The value is taken from - /sys/fs/cgroup/memory/memory.limit_in_bytes - NUMBER_OF_CORES - Number of detected CPU cores that can be used. This value is - calculated from /sys/fs/cgroup/cpuset/cpuset.cpus - NO_MEMORY_LIMIT - Set to "true" if MEMORY_LIMIT_IN_BYTES is so high that the caller - can act as if no memory limit was set. Undefined otherwise. -""" +get_number_of_cores() { + local core_count=0 + local group + local sysfs_cpus="$(cat /sys/fs/cgroup/cpuset/cpuset.cpus)" -from __future__ import print_function -import sys + if [ $? -eq 0 ]; then + for group in ${sysfs_cpus//,/ }; do + local from="${group%-*}" + local to="${group#*-}" + core_count=$(($core_count + $to - $from + 1)) + done + echo "$core_count" + else + echo "Warning: Can't detect number of CPU cores from cgroups" >&2 + fi +} +MAX_MEMORY_LIMIT_IN_BYTES=9223372036854775807 +MEMORY_LIMIT_IN_BYTES="$(get_memory_limit)" +RAM_SIZE_IN_BYTES="$(get_meminfo_ram_size)" +NUMBER_OF_CORES="$(get_number_of_cores)" -def _read_file(path): - try: - with open(path, 'r') as f: - return f.read().strip() - except IOError: - return None +echo "MAX_MEMORY_LIMIT_IN_BYTES=$MAX_MEMORY_LIMIT_IN_BYTES" +if [ -n "$MEMORY_LIMIT_IN_BYTES" ]; then + echo "MEMORY_LIMIT_IN_BYTES=$MEMORY_LIMIT_IN_BYTES" + if [ "$MEMORY_LIMIT_IN_BYTES" -ge "$MAX_MEMORY_LIMIT_IN_BYTES" ]; then + echo "NO_MEMORY_LIMIT=true" + fi +fi -def get_memory_limit(): - """ - Read memory limit, in bytes. - """ +if [ -n "$RAM_SIZE_IN_BYTES" ]; then + echo "RAM_SIZE_IN_BYTES=$RAM_SIZE_IN_BYTES" +fi - limit = _read_file('/sys/fs/cgroup/memory/memory.limit_in_bytes') - if limit is None or not limit.isdigit(): - print("Warning: Can't detect memory limit from cgroups", - file=sys.stderr) - return None - return int(limit) - - -def get_number_of_cores(): - """ - Read number of CPU cores. - """ - - core_count = 0 - - line = _read_file('/sys/fs/cgroup/cpuset/cpuset.cpus') - if line is None: - print("Warning: Can't detect number of CPU cores from cgroups", - file=sys.stderr) - return None - - for group in line.split(','): - core_ids = list(map(int, group.split('-'))) - if len(core_ids) == 2: - core_count += core_ids[1] - core_ids[0] + 1 - else: - core_count += 1 - - return core_count - - -if __name__ == "__main__": - env_vars = { - "MAX_MEMORY_LIMIT_IN_BYTES": 9223372036854775807, - "MEMORY_LIMIT_IN_BYTES": get_memory_limit(), - "NUMBER_OF_CORES": get_number_of_cores() - } - - env_vars = {k: v for k, v in env_vars.items() if v is not None} - - if env_vars.get("MEMORY_LIMIT_IN_BYTES", 0) >= 92233720368547: - env_vars["NO_MEMORY_LIMIT"] = "true" - - for key, value in env_vars.items(): - print("{0}={1}".format(key, value)) +if [ -n "$NUMBER_OF_CORES" ]; then + echo "NUMBER_OF_CORES=$NUMBER_OF_CORES" +fi diff --git a/root/usr/share/container-scripts/postgresql/common.sh b/root/usr/share/container-scripts/postgresql/common.sh index ca58a0b..60fe4cd 100644 --- a/root/usr/share/container-scripts/postgresql/common.sh +++ b/root/usr/share/container-scripts/postgresql/common.sh @@ -6,14 +6,19 @@ export POSTGRESQL_MAX_PREPARED_TRANSACTIONS=${POSTGRESQL_MAX_PREPARED_TRANSACTIO # limits are set). # Users can still override this by setting the POSTGRESQL_SHARED_BUFFERS # and POSTGRESQL_EFFECTIVE_CACHE_SIZE variables. -if [[ "${NO_MEMORY_LIMIT:-}" == "true" || -z "${MEMORY_LIMIT_IN_BYTES:-}" ]]; then +if [[ "${NO_MEMORY_LIMIT:-}" == "true" || -z "${MEMORY_LIMIT_IN_BYTES:-}" || -z "${RAM_SIZE_IN_BYTES:-}" ]]; then export POSTGRESQL_SHARED_BUFFERS=${POSTGRESQL_SHARED_BUFFERS:-32MB} export POSTGRESQL_EFFECTIVE_CACHE_SIZE=${POSTGRESQL_EFFECTIVE_CACHE_SIZE:-128MB} else + if [ "$MEMORY_LIMIT_IN_BYTES" -gt "$RAM_SIZE_IN_BYTES" ]; then + mem_limit="$RAM_SIZE_IN_BYTES" + else + mem_limit="$MEMORY_LIMIT_IN_BYTES" + fi # Use 1/4 of given memory for shared buffers - shared_buffers_computed="$(($MEMORY_LIMIT_IN_BYTES/1024/1024/4))MB" + shared_buffers_computed="$(($mem_limit/1024/1024/4))MB" # Setting effective_cache_size to 1/2 of total memory would be a normal conservative setting, - effective_cache="$(($MEMORY_LIMIT_IN_BYTES/1024/1024/2))MB" + effective_cache="$(($mem_limit/1024/1024/2))MB" export POSTGRESQL_SHARED_BUFFERS=${POSTGRESQL_SHARED_BUFFERS:-$shared_buffers_computed} export POSTGRESQL_EFFECTIVE_CACHE_SIZE=${POSTGRESQL_EFFECTIVE_CACHE_SIZE:-$effective_cache} fi @@ -126,7 +131,7 @@ function generate_passwd_file() { export GROUP_ID=$(id -g) grep -v ^postgres /etc/passwd > "$HOME/passwd" echo "postgres:x:${USER_ID}:${GROUP_ID}:PostgreSQL Server:${HOME}:/bin/bash" >> "$HOME/passwd" - export LD_PRELOAD=libnss_wrapper.so + #export LD_PRELOAD=libnss_wrapper.so export NSS_WRAPPER_PASSWD=${HOME}/passwd export NSS_WRAPPER_GROUP=/etc/group } diff --git a/root/usr/share/container-scripts/postgresql/scl_enable b/root/usr/share/container-scripts/postgresql/scl_enable deleted file mode 100644 index 1d967f9..0000000 --- a/root/usr/share/container-scripts/postgresql/scl_enable +++ /dev/null @@ -1,3 +0,0 @@ -# This will make scl collection binaries work out of box. -unset BASH_ENV PROMPT_COMMAND ENV -source scl_source enable $ENABLED_COLLECTIONS From 9c069c414173a6c93edefe13531b7ebea54ace60 Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: Fri, 19 May 2017 19:31:05 +0200 Subject: [PATCH 22/47] add files for modularity-testing-framework based tests Currently there seems to be an issue with passing the test command line to the shell in the container, though. --- tests/config.yaml | 39 +++++++++++++++++++++++++++++++++++++++ tests/test.sql | 5 +++++ 2 files changed, 44 insertions(+) create mode 100644 tests/config.yaml create mode 100644 tests/test.sql diff --git a/tests/config.yaml b/tests/config.yaml new file mode 100644 index 0000000..8780ba2 --- /dev/null +++ b/tests/config.yaml @@ -0,0 +1,39 @@ +document: modularity-testing +version: 1 +name: postgresql +modulemd-url: https://src.fedoraproject.org/cgit/modules/postgresql.git/tree/postgresql.yaml?h=f26 +service: + port: 5432 +packages: + rpms: + - postgresql-server +testdependecies: + rpms: + - postgresql + - systemd +module: + docker: + start: "docker run -it -p 5432:5432 -e POSTGRESQL_USER=mtftest -e POSTGRESQL_PASSWORD=mtftest -e POSTGRESQL_DATABASE=mtftest" + labels: + description: "PostgreSQL is an advanced Object-Relational database management system (DBMS)." + io.k8s.description: "PostgreSQL is an advanced Object-Relational database management system (DBMS)." + source: https://github.com/container-images/postgresql.git + container: docker.io/modularitycontainers/postgresql + rpm: + start: systemctl start postgresql + stop: systemctl stop postgresql + status: systemctl status memcached + repos: + - https://kojipkgs.fedoraproject.org/repos/module-d87b0b15567e47f3-build/latest/x86_64/ +test: + processrunning: + - 'ls /proc/*/exe -alh | grep postgres' +testhost: + selfcheck: + - 'env PGPASSWORD=mtftest psql -h localhost -U mtftest -f test.sql | egrep "anything|something"' + #- 'echo errr | nc localhost 11211' + #- 'echo set AAA 0 4 2 | nc localhost 11211' + #- 'echo get AAA | nc localhost 11211' + #selcheckError: + #- 'echo errr | nc localhost 11211 |grep ERROR' + diff --git a/tests/test.sql b/tests/test.sql new file mode 100644 index 0000000..5ecc838 --- /dev/null +++ b/tests/test.sql @@ -0,0 +1,5 @@ +CREATE TABLE mtftest (id SERIAL NOT NULL, anything VARCHAR(255) NOT NULL); + +INSERT INTO mtftest (anything) VALUES ('something'); + +SELECT anything FROM mtftest; From 14eb0cd14c41d816e3bbe216c25b66aa7c4c04df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Petr=20=C5=A0abata?= Date: Wed, 24 May 2017 10:56:04 +0200 Subject: [PATCH 23/47] Rebuilding for f26 signing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Petr Šabata From b4193a89b71ac3dba892f3be9efabfd2fe9f190d Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: Wed, 24 May 2017 16:30:16 +0200 Subject: [PATCH 24/47] tests/rpm: check status of correct service --- tests/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/config.yaml b/tests/config.yaml index 8780ba2..463f924 100644 --- a/tests/config.yaml +++ b/tests/config.yaml @@ -22,7 +22,7 @@ module: rpm: start: systemctl start postgresql stop: systemctl stop postgresql - status: systemctl status memcached + status: systemctl status postgresql repos: - https://kojipkgs.fedoraproject.org/repos/module-d87b0b15567e47f3-build/latest/x86_64/ test: From f3e435ec35755af32915684c75868caab400b7c9 Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: Wed, 24 May 2017 16:31:03 +0200 Subject: [PATCH 25/47] tests: check for up to 20s if postgresql is running Grep for the whole path of the binary because its basename is the same as the name of the user all this is running under. --- tests/config.yaml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/config.yaml b/tests/config.yaml index 463f924..1cbd39a 100644 --- a/tests/config.yaml +++ b/tests/config.yaml @@ -7,6 +7,8 @@ service: packages: rpms: - postgresql-server + # seq + - coreutils testdependecies: rpms: - postgresql @@ -26,8 +28,20 @@ module: repos: - https://kojipkgs.fedoraproject.org/repos/module-d87b0b15567e47f3-build/latest/x86_64/ test: - processrunning: - - 'ls /proc/*/exe -alh | grep postgres' + process_running: + - | + up=0 + for i in `seq 20`; do + echo "Checking for /usr/bin/postgres process... ($i)" + if ls /proc/*/exe -alh 2>/dev/null | grep /usr/bin/postgres; then + echo "Found." + up=1 + break + else + sleep 1 + fi + done + test "$up" -ne 0 testhost: selfcheck: - 'env PGPASSWORD=mtftest psql -h localhost -U mtftest -f test.sql | egrep "anything|something"' From fcb7466f35f06f76f0e59ba25acededca711af37 Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: Wed, 24 May 2017 16:32:56 +0200 Subject: [PATCH 26/47] tests: hold off until daemon is available The postgresql container initializes DB files e.a. in these tests, this can take a couple of seconds. Try contacting it for up to 20 seconds, and only then run the tests: create a table, store something in it and retrieve it again. --- tests/config.yaml | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/tests/config.yaml b/tests/config.yaml index 1cbd39a..556a512 100644 --- a/tests/config.yaml +++ b/tests/config.yaml @@ -43,11 +43,23 @@ test: done test "$up" -ne 0 testhost: - selfcheck: - - 'env PGPASSWORD=mtftest psql -h localhost -U mtftest -f test.sql | egrep "anything|something"' - #- 'echo errr | nc localhost 11211' - #- 'echo set AAA 0 4 2 | nc localhost 11211' - #- 'echo get AAA | nc localhost 11211' - #selcheckError: - #- 'echo errr | nc localhost 11211 |grep ERROR' - + db_usable: + - | + export PGPASSWORD=mtftest + PSQL="psql -h localhost -U mtftest -q" + up=0 + for i in `seq 20`; do + echo "Pinging PostgreSQL ($i)" + if $PSQL -c "" >&/dev/null; then + up=1 + break + else + sleep 1 + fi + done + if [ "$up" -eq 1 ]; then + $PSQL -f test.sql | egrep "anything|something" + else + echo "Unable to connect to database." >&2 + false + fi From 5157eada715d2070b5316fec930cfe6a840299e5 Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: Fri, 26 May 2017 15:32:57 +0200 Subject: [PATCH 27/47] add Makefiles --- Makefile | 15 +++++++++++++++ tests/Makefile | 6 ++++++ 2 files changed, 21 insertions(+) create mode 100644 Makefile create mode 100644 tests/Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..80729a8 --- /dev/null +++ b/Makefile @@ -0,0 +1,15 @@ +IMAGE_NAME = modularitycontainers/postgresql + +MODULEMDURL=file://postgresql.yaml + +default: run + +build: + docker build --tag=$(IMAGE_NAME) . + +run: build + docker run -it -p 80:80 $(IMAGE_NAME) + +test: build + cd tests; MODULE=docker MODULEMD=$(MODULEMDURL) URL="docker=$(IMAGE_NAME)" make all + cd tests; MODULE=rpm MODULEMD=$(MODULEMDURL) URL="docker=$(IMAGE_NAME)" make all diff --git a/tests/Makefile b/tests/Makefile new file mode 100644 index 0000000..8fd8ae7 --- /dev/null +++ b/tests/Makefile @@ -0,0 +1,6 @@ +MODULE_LINT=/usr/share/moduleframework/tools/modulelint.py +CMD=python -m avocado run --filter-by-tags=-WIP $(MODULE_LINT) *.py + +# +all: + $(CMD) From e0dc694125dce49f3a754a9280ad02ff2a044555 Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: Tue, 30 May 2017 11:06:36 +0200 Subject: [PATCH 28/47] add tcl-pgtcl:f26 (runtime dep for postgresql-pltcl) --- postgresql.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/postgresql.yaml b/postgresql.yaml index bc14f87..d5aad92 100644 --- a/postgresql.yaml +++ b/postgresql.yaml @@ -87,3 +87,7 @@ data: # This branch doesn't BR: systemtap(-devel), elinks which pulls in # loads of other junk ref: private-f27-modules + tcl-pgtcl: + buildorder: 4 + rationale: 'Runtime dependency (for postgresql-pltcl)' + ref: f26 From a2eef8062baa3eda8cda58425df746beb559ea00 Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: Wed, 31 May 2017 15:43:35 +0200 Subject: [PATCH 29/47] tests/rpm: initialize database first --- tests/config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/config.yaml b/tests/config.yaml index 556a512..06702cf 100644 --- a/tests/config.yaml +++ b/tests/config.yaml @@ -22,6 +22,7 @@ module: source: https://github.com/container-images/postgresql.git container: docker.io/modularitycontainers/postgresql rpm: + setup: postgresql-setup initdb start: systemctl start postgresql stop: systemctl stop postgresql status: systemctl status postgresql From a2bd98103225005c7001701fd296b2ed8caee762 Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: Wed, 31 May 2017 15:43:54 +0200 Subject: [PATCH 30/47] tests/rpm: use Fedora 26 Boltron Server repo --- tests/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/config.yaml b/tests/config.yaml index 06702cf..47875fc 100644 --- a/tests/config.yaml +++ b/tests/config.yaml @@ -27,7 +27,7 @@ module: stop: systemctl stop postgresql status: systemctl status postgresql repos: - - https://kojipkgs.fedoraproject.org/repos/module-d87b0b15567e47f3-build/latest/x86_64/ + - https://kojipkgs.fedoraproject.org/compose/latest-Fedora-Modular-26/compose/Server/x86_64/os/ test: process_running: - | From 0b7e8de26ca8aaa8e507735175d43f25ebec8072 Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: Tue, 20 Jun 2017 17:00:27 +0200 Subject: [PATCH 31/47] remove docbook-utils build dependency This and related packages have been moved to common-build-dependencies. closes #1 --- postgresql.yaml | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/postgresql.yaml b/postgresql.yaml index d5aad92..f8fceda 100644 --- a/postgresql.yaml +++ b/postgresql.yaml @@ -41,15 +41,12 @@ data: # part of base-runtime: # gettext # part of common-build-dependencies: - # autoconf(uuid), multilib-rpm-config + # autoconf(uuid), multilib-rpm-config, docbook-utils # part of shared-userspace: # docbook-dtds, docbook-style-dsssl, libxslt, perl-SGMLSpm, python2, tcl oniguruma: rationale: 'Build dependency (for slang)' ref: f26 - opensp: - rationale: 'Build dependency (for openjade)' - ref: f26 perl-Data-UUID: rationale: 'Build dependency (for uuid)' ref: f26 @@ -59,10 +56,6 @@ data: telnet: rationale: 'Build dependency (for lynx)' ref: f26 - openjade: - buildorder: 1 - rationale: 'Build dependency (for docbook-utils)' - ref: f26 slang: buildorder: 1 rationale: 'Build dependency (for lynx)' @@ -71,15 +64,10 @@ data: buildorder: 1 rationale: 'Build dependency (for postgresql).' ref: f26 - docbook-utils: - buildorder: 2 - rationale: 'Build dependency (for postgresql)' - ref: f26 lynx: buildorder: 2 rationale: >- Build dependency (for postgresql), - runtime dependency (for docbook-utils) ref: f26 postgresql: buildorder: 3 From 0603455e67820227129fe5fa8db4f2ff32904ca5 Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: Fri, 25 Aug 2017 10:34:51 +0200 Subject: [PATCH 32/47] import modulemd generated by dependency-report --- postgresql.yaml | 77 ++++++++++++++----------------------------------- 1 file changed, 22 insertions(+), 55 deletions(-) diff --git a/postgresql.yaml b/postgresql.yaml index 73d78c5..36e1375 100644 --- a/postgresql.yaml +++ b/postgresql.yaml @@ -1,58 +1,25 @@ document: modulemd version: 1 data: - # name, stream, version are filled in by the build system - #name: "" - #stream: "" - #version: 0 - summary: 'The programs needed to create and run a PostgreSQL server' - description: |- - PostgreSQL is an advanced Object-Relational database management system - (DBMS). The postgresql-server package contains the programs needed to - create and run a PostgreSQL server, which will in turn allow you to create - and maintain PostgreSQL databases. - license: - module: - - MIT - content: - - PostgreSQL - dependencies: - buildrequires: - base-runtime: master - common-build-dependencies: f26 - common-build-dependencies-bootstrap: f26 - shared-userspace: f26 - perl: master - requires: - base-runtime: master - perl: master - api: - rpms: - - postgresql - profiles: - default: - rpms: - - postgresql-server - client: - rpms: - - postgresql - filter: {} - components: - rpms: - # part of common-build-dependencies: - # multilib-rpm-config, gettext - postgresql: - rationale: 'Build dependency (for postgresql-contrib).' - ref: f26 - buildorder: 10 - uuid: - rationale: 'Build dependency (for postgresql-contrib).' - ref: f26 - buildorder: 1 - perl-Data-UUID: - rationale: 'Build dependency (for postgresql-contrib).' - ref: f26 - elinks: - rationale: 'Build dependency (for postgresql-contrib).' - ref: private-f27-modules - buildorder: 3 + summary: postgresql module + description: This postgresql has been generated + license: + module: [ MIT ] + dependencies: + buildrequires: + bootstrap: master + requires: + platform: master + platform-placeholder: master + references: + community: https://docs.pagure.org/modularity/ + documentation: https://github.com/modularity-modules/postgresql + tracker: https://github.com/modularity-modules/postgresql + components: + rpms: + postgresql: + rationale: Generated. + ref: master. + uuid: + rationale: Generated. + ref: master. From 6cdab777aa5ae36faab6b9f74f4b2deb3c4b315a Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: Fri, 25 Aug 2017 10:40:55 +0200 Subject: [PATCH 33/47] reindent --- postgresql.yaml | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/postgresql.yaml b/postgresql.yaml index 36e1375..674facc 100644 --- a/postgresql.yaml +++ b/postgresql.yaml @@ -1,25 +1,25 @@ document: modulemd version: 1 data: - summary: postgresql module - description: This postgresql has been generated - license: - module: [ MIT ] - dependencies: - buildrequires: - bootstrap: master - requires: - platform: master - platform-placeholder: master - references: - community: https://docs.pagure.org/modularity/ - documentation: https://github.com/modularity-modules/postgresql - tracker: https://github.com/modularity-modules/postgresql - components: - rpms: - postgresql: - rationale: Generated. - ref: master. - uuid: - rationale: Generated. - ref: master. + summary: postgresql module + description: This postgresql has been generated + license: + module: [ MIT ] + dependencies: + buildrequires: + bootstrap: master + requires: + platform: master + platform-placeholder: master + references: + community: https://docs.pagure.org/modularity/ + documentation: https://github.com/modularity-modules/postgresql + tracker: https://github.com/modularity-modules/postgresql + components: + rpms: + postgresql: + rationale: Generated. + ref: master. + uuid: + rationale: Generated. + ref: master. From e090c6f0fe81f337460a81e1350b4083fcecbbb5 Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: Fri, 25 Aug 2017 10:41:03 +0200 Subject: [PATCH 34/47] re-add api, profiles --- postgresql.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/postgresql.yaml b/postgresql.yaml index 674facc..4f3a479 100644 --- a/postgresql.yaml +++ b/postgresql.yaml @@ -15,6 +15,17 @@ data: community: https://docs.pagure.org/modularity/ documentation: https://github.com/modularity-modules/postgresql tracker: https://github.com/modularity-modules/postgresql + api: + rpms: + - postgresql + profiles: + default: + rpms: + - postgresql-server + client: + rpms: + - postgresql + filter: {} components: rpms: postgresql: From 83452c25e6c4dfaeccbbc02d7dca1713bdb95272 Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: Fri, 25 Aug 2017 10:46:24 +0200 Subject: [PATCH 35/47] reindent again (grrr, tabs) --- postgresql.yaml | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/postgresql.yaml b/postgresql.yaml index 4f3a479..23dea4d 100644 --- a/postgresql.yaml +++ b/postgresql.yaml @@ -1,20 +1,20 @@ document: modulemd version: 1 data: - summary: postgresql module - description: This postgresql has been generated - license: - module: [ MIT ] - dependencies: - buildrequires: - bootstrap: master - requires: - platform: master - platform-placeholder: master - references: - community: https://docs.pagure.org/modularity/ - documentation: https://github.com/modularity-modules/postgresql - tracker: https://github.com/modularity-modules/postgresql + summary: postgresql module + description: This postgresql has been generated + license: + module: [ MIT ] + dependencies: + buildrequires: + bootstrap: master + requires: + platform: master + platform-placeholder: master + references: + community: https://docs.pagure.org/modularity/ + documentation: https://github.com/modularity-modules/postgresql + tracker: https://github.com/modularity-modules/postgresql api: rpms: - postgresql @@ -28,9 +28,9 @@ data: filter: {} components: rpms: - postgresql: - rationale: Generated. - ref: master. - uuid: - rationale: Generated. - ref: master. + postgresql: + rationale: Generated. + ref: master. + uuid: + rationale: Generated. + ref: master. From a61016ef1a60e8108169be42ce07e614bb7b974d Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: Fri, 25 Aug 2017 10:53:50 +0200 Subject: [PATCH 36/47] make the description a proper sentence --- postgresql.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgresql.yaml b/postgresql.yaml index 23dea4d..6f7e490 100644 --- a/postgresql.yaml +++ b/postgresql.yaml @@ -2,7 +2,7 @@ document: modulemd version: 1 data: summary: postgresql module - description: This postgresql has been generated + description: This postgresql module has been generated. license: module: [ MIT ] dependencies: From e26462750c73da621183a1e345a237063ef150ee Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: Fri, 25 Aug 2017 10:54:54 +0200 Subject: [PATCH 37/47] pull component packages from f27 --- postgresql.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/postgresql.yaml b/postgresql.yaml index 6f7e490..df1dc5d 100644 --- a/postgresql.yaml +++ b/postgresql.yaml @@ -30,7 +30,7 @@ data: rpms: postgresql: rationale: Generated. - ref: master. + ref: f27 uuid: rationale: Generated. - ref: master. + ref: f27 From 74b97eac49c324896948d188cf7790f2aa2da82a Mon Sep 17 00:00:00 2001 From: Nils Philippsen Date: Fri, 25 Aug 2017 12:47:05 +0200 Subject: [PATCH 38/47] remove bogus platform-placeholder dependency --- postgresql.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/postgresql.yaml b/postgresql.yaml index df1dc5d..48a8832 100644 --- a/postgresql.yaml +++ b/postgresql.yaml @@ -10,7 +10,6 @@ data: bootstrap: master requires: platform: master - platform-placeholder: master references: community: https://docs.pagure.org/modularity/ documentation: https://github.com/modularity-modules/postgresql From 71e7d7e4174d189a351299a87291b2e79f4e1cfc Mon Sep 17 00:00:00 2001 From: Karsten Hopp Date: Mon, 9 Oct 2017 16:07:25 +0200 Subject: [PATCH 39/47] provide server profile as requested by sct --- postgresql.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/postgresql.yaml b/postgresql.yaml index 48a8832..29792f2 100644 --- a/postgresql.yaml +++ b/postgresql.yaml @@ -21,6 +21,9 @@ data: default: rpms: - postgresql-server + server: + rpms: + - postgresql-server client: rpms: - postgresql From fce1b50a7ad4a741d65b5989abbd9c3fd1cf64d6 Mon Sep 17 00:00:00 2001 From: Karsten Hopp Date: Thu, 12 Oct 2017 11:16:44 +0200 Subject: [PATCH 40/47] bump and rebuild From de62aeaeb5bd4902f53a2aa8d13d0342a63687ba Mon Sep 17 00:00:00 2001 From: Karsten Hopp Date: Thu, 12 Oct 2017 11:50:28 +0200 Subject: [PATCH 41/47] bump From 5d9488e1439f5e5b4f272961ceed4b4965f8bde2 Mon Sep 17 00:00:00 2001 From: Karsten Hopp Date: Thu, 12 Oct 2017 11:55:55 +0200 Subject: [PATCH 42/47] use older hast for mbs/pdc sync tests --- postgresql.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgresql.yaml b/postgresql.yaml index 29792f2..653cd24 100644 --- a/postgresql.yaml +++ b/postgresql.yaml @@ -32,7 +32,7 @@ data: rpms: postgresql: rationale: Generated. - ref: f27 + ref: 1fa73e0a7b04e79c7ba7c4fddeee422d27634884 uuid: rationale: Generated. ref: f27 From 8adcc7910fd09c315dde251242010be57d59c395 Mon Sep 17 00:00:00 2001 From: Karsten Hopp Date: Thu, 12 Oct 2017 12:00:54 +0200 Subject: [PATCH 43/47] use an even older hash for mbs/pdc sync tests --- postgresql.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgresql.yaml b/postgresql.yaml index 653cd24..c4516dd 100644 --- a/postgresql.yaml +++ b/postgresql.yaml @@ -32,7 +32,7 @@ data: rpms: postgresql: rationale: Generated. - ref: 1fa73e0a7b04e79c7ba7c4fddeee422d27634884 + ref: 61feab0edf0880151e44d388400caea6522067cf uuid: rationale: Generated. ref: f27 From e733195a45298ed48abc58a805e4be97390030e3 Mon Sep 17 00:00:00 2001 From: Karsten Hopp Date: Thu, 12 Oct 2017 13:13:27 +0200 Subject: [PATCH 44/47] Revert "use an even older hash for mbs/pdc sync tests" This reverts commit 8adcc7910fd09c315dde251242010be57d59c395. --- postgresql.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgresql.yaml b/postgresql.yaml index c4516dd..653cd24 100644 --- a/postgresql.yaml +++ b/postgresql.yaml @@ -32,7 +32,7 @@ data: rpms: postgresql: rationale: Generated. - ref: 61feab0edf0880151e44d388400caea6522067cf + ref: 1fa73e0a7b04e79c7ba7c4fddeee422d27634884 uuid: rationale: Generated. ref: f27 From 7541816442f847c2f69c19cb40416aa31d62208d Mon Sep 17 00:00:00 2001 From: Karsten Hopp Date: Thu, 12 Oct 2017 13:14:15 +0200 Subject: [PATCH 45/47] Revert "use older hast for mbs/pdc sync tests" This reverts commit 5d9488e1439f5e5b4f272961ceed4b4965f8bde2. --- postgresql.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgresql.yaml b/postgresql.yaml index 653cd24..29792f2 100644 --- a/postgresql.yaml +++ b/postgresql.yaml @@ -32,7 +32,7 @@ data: rpms: postgresql: rationale: Generated. - ref: 1fa73e0a7b04e79c7ba7c4fddeee422d27634884 + ref: f27 uuid: rationale: Generated. ref: f27 From b8b9571938ed41b303d600f2eb7df9f5219faf06 Mon Sep 17 00:00:00 2001 From: Karsten Hopp Date: Wed, 18 Oct 2017 10:35:30 +0200 Subject: [PATCH 46/47] switch to f27 stream requirements --- postgresql.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/postgresql.yaml b/postgresql.yaml index 29792f2..0dfdb12 100644 --- a/postgresql.yaml +++ b/postgresql.yaml @@ -7,9 +7,9 @@ data: module: [ MIT ] dependencies: buildrequires: - bootstrap: master + bootstrap: f27 requires: - platform: master + platform: f27 references: community: https://docs.pagure.org/modularity/ documentation: https://github.com/modularity-modules/postgresql From 0afe31547e9e4c15269286d79038d8bcfee4a460 Mon Sep 17 00:00:00 2001 From: Karsten Hopp Date: Fri, 20 Oct 2017 12:44:21 +0200 Subject: [PATCH 47/47] update master hashes --- postgresql.yaml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/postgresql.yaml b/postgresql.yaml index 0dfdb12..fa4ff16 100644 --- a/postgresql.yaml +++ b/postgresql.yaml @@ -7,9 +7,9 @@ data: module: [ MIT ] dependencies: buildrequires: - bootstrap: f27 + bootstrap: master requires: - platform: f27 + platform: master references: community: https://docs.pagure.org/modularity/ documentation: https://github.com/modularity-modules/postgresql @@ -30,9 +30,12 @@ data: filter: {} components: rpms: + # postgresql-10.0-3.fc28 postgresql: rationale: Generated. - ref: f27 + ref: 07894e0da4b994092abd0084fba982e075ccf3f3 + # uuid-1.6.2-39.fc27 uuid: rationale: Generated. - ref: f27 + ref: 43c916caff937bdf8a97ded1abff3e01df4895f4 +