diff --git a/build-and-update-all-branches b/build-and-update-all-branches new file mode 100755 index 0000000..9d7b6da --- /dev/null +++ b/build-and-update-all-branches @@ -0,0 +1,37 @@ +#! /bin/sh + +# Prepare rawhide branch, review the list of branches below and then execute this +# script. + +main=main +branches="$main epel8 epel9 f41 f42" + +exit_handler () +{ + git checkout $main +} + +trap exit_handler EXIT + +set -e +set -x + +koji hello + +tasks= +for branch in $branches; do + if test $branch != $main; then + git checkout "$branch" + git merge $main + fi + git push + tasks="${tasks}`fedpkg build --nowait | grep 'Created task' | cut -d: -f2`" +done + +if test -n "$tasks"; then + koji watch-task $tasks +fi + +set -- $branches +git checkout $1 +fedpkg update diff --git a/resalloc-agent-spawner.service b/resalloc-agent-spawner.service new file mode 100644 index 0000000..b4bb2c6 --- /dev/null +++ b/resalloc-agent-spawner.service @@ -0,0 +1,15 @@ +[Unit] +Description=Start self-stending agent-like resources using Resalloc +After=syslog.target network.target auditd.service + +[Service] +Type=simple +User=resalloc +Group=resalloc +ExecStart=/usr/bin/resalloc-agent-spawner +# we don't want to kill background action processors (daemoncontext) +KillMode=process +Restart=on-failure + +[Install] +WantedBy=multi-user.target diff --git a/resalloc.service b/resalloc.service index 5099103..e29ae38 100644 --- a/resalloc.service +++ b/resalloc.service @@ -4,7 +4,8 @@ After=network.target [Service] Type=simple - +Restart=always +RestartSec=5 User=resalloc Group=resalloc diff --git a/resalloc.spec b/resalloc.spec index 5b44294..7093616 100644 --- a/resalloc.spec +++ b/resalloc.spec @@ -2,9 +2,20 @@ %global sysuser resalloc %global sysgroup %sysuser -%global _logdir %_var/log/%{name}server %global _homedir %_sharedstatedir/%{name}server +%global agent_user resalloc-agent-spawner +%global agent_group %agent_user + +%global create_user_group() \ +getent group "%1" >/dev/null || groupadd -r "%1" \ +getent passwd "%1" >/dev/null || \\\ +useradd -r -g "%2" -G "%2" -s "%3" \\\ + -c "%1 service user" "%1" \\\ + -d "%4" + +%global _logdir %_var/log/%{name}server + %global sum Resource allocator for expensive resources %global desc \ The resalloc project aims to help with taking care of dynamically \ @@ -14,7 +25,7 @@ the purposes of CI/CD tasks. %bcond_without check -%if 0%{?fedora} || 0%{?rhel} > 7 +%if 0%{?fedora} || 0%{?rhel} > 7 || 0%{?is_opensuse} %bcond_with python2 %bcond_without python3 %else @@ -22,14 +33,26 @@ the purposes of CI/CD tasks. %bcond_with python3 %endif +# Modern distributions (using RPM v4.19+; for example, Fedora 39+) do not +# require the %%pre scriptlet for creating users/groups because the sysusers +# feature is now built directly into RPM. Simply including the sysusers +# `mock.conf` file in a package payload is sufficient to leverage this feature. +# However, for older distributions that lack this capability, we still define +# the %%pre scriptlet. +%if (0%{?rhel} && 0%{?rhel} < 10) || (0%{?mageia} && 0%{?mageia} < 10) || (0%{?suse_version} && 0%{?suse_version} < 1660) +%bcond_without sysusers_compat +%else +%bcond_with sysusers_compat +%endif + %global default_python %{?with_python3:python3}%{!?with_python3:python2} %global default_sitelib %{?with_python3:%python3_sitelib}%{!?with_python3:%python_sitelib} Name: %srcname Summary: %sum - client tooling -Version: 4.3 -Release: 1%{?dist} -License: GPLv2+ +Version: 5.11 +Release: 4%{?dist} +License: GPL-2.0-or-later URL: https://github.com/praiskup/resalloc BuildArch: noarch @@ -39,6 +62,7 @@ BuildRequires: postgresql-server %if %{with python3} BuildRequires: python3-alembic +BuildRequires: python3-argparse-manpage BuildRequires: python3-devel BuildRequires: python3-psycopg2 BuildRequires: python3-pytest @@ -56,6 +80,7 @@ BuildRequires: python3-yaml %if %{with python2} BuildRequires: python-alembic +BuildRequires: python2-argparse-manpage BuildRequires: python2-devel BuildRequires: python-psycopg2 BuildRequires: python2-mock @@ -69,11 +94,18 @@ BuildRequires: python-yaml Requires: %default_python-%srcname = %version-%release +%if %{with sysusers_compat} +Requires(pre): shadow-utils +%endif + Source0: https://github.com/praiskup/%name/releases/download/v%version/%name-%version.tar.gz Source1: resalloc.service +Source5: resalloc-agent-spawner.service Source2: logrotate Source3: merge-hook-logs Source4: cron.hourly +# GPL-2.0-or-later too +Source6: https://raw.githubusercontent.com/praiskup/wait-for-ssh/main/wait-for-ssh %description %desc @@ -87,6 +119,7 @@ Summary: %sum - server part Requires: crontabs Requires: logrotate Requires: %default_python-%srcname = %version-%release +Requires: %srcname-helpers = %version-%release %if %{with python3} Requires: python3-alembic Requires: python3-six @@ -99,13 +132,23 @@ Requires: python-sqlalchemy Requires: python-yaml %endif -Requires(pre): /usr/sbin/useradd %description server %desc The %name-server package provides the resalloc server, and some tooling for resalloc administrators. + +%package helpers +Summary: %sum - helper/library scripts + +%description helpers +%desc + +Helper and library-like scripts for external Resalloc plugins like resalloc-aws, +resalloc-openstack, etc. + + %if %{with python3} %package webui Summary: %sum - webui part @@ -114,6 +157,7 @@ Summary: %sum - webui part Requires: %default_python-%srcname = %version-%release Requires: %name-server Requires: python3-flask +Recommends: %name-selinux %endif %description webui @@ -123,6 +167,27 @@ The %name-webui package provides the resalloc webui, it shows page with information about resalloc resources. %endif +%if %{with python3} +%package agent-spawner +Summary: %sum - daemon starting agent-like resources + +Requires: python3-copr-common >= 0.23 +Requires: python3-daemon +Requires: python3-redis +Requires: python3-resalloc = %version-%release +Requires: python3-setproctitle + +%description agent-spawner +%desc + +Agent Spawner maintains sets resources (agents) of certain kind and in certain +number, according to given configuration. Typical Resalloc resource is +completely dummy, fully controlled from the outside. With agent-like resources +this is different — such resources are self-standing, they take care of +themselves, perhaps interacting/competing with each other. The only thing that +agent-spawner needs to do is to control the ideal number of them. +%endif + %if %{with python3} %package -n python3-%srcname Summary: %sum - Python 3 client library @@ -147,16 +212,47 @@ to the resalloc server. %endif +%package selinux +Summary: SELinux module for %{name} +# Requires(post): policycoreutils-python +BuildRequires: selinux-policy-devel +%{?selinux_requires} + +%description selinux +%desc + +%post selinux +semanage fcontext -a -t httpd_sys_script_exec_t \ + %_var/www/cgi-%{name} 2>/dev/null || : +restorecon -R %_var/www/cgi-%{name} || : + + %prep %autosetup -p1 -n %name-%version +%if %{without python3} +rm -r resalloc_agent_spawner +%endif + +# Create sysusers.d config files +cat >resalloc.sysusers.conf <resalloc-agent-spawner.sysusers.conf < %{name}-wait-for-ssh %install @@ -165,14 +261,20 @@ to the resalloc server. rm -r %buildroot%python2_sitelib/%{name}webui %else %py3_install -install -d -m 700 %buildroot%_datadir/%{name}webui +install -d -m 755 %buildroot%_datadir/%{name}webui cp -r %{name}webui/templates %buildroot%_datadir/%{name}webui/ cp -r %{name}webui/static %buildroot%_datadir/%{name}webui/ + +install -d -m 755 %buildroot%_var/www/ +install -p -m 755 %{name}webui/cgi-resalloc %buildroot%_var/www/cgi-%{name} %endif mkdir -p %buildroot%_unitdir mkdir -p %buildroot%_logdir install -p -m 644 %SOURCE1 %buildroot%_unitdir +%if %{with python3} +install -p -m 644 %SOURCE5 %buildroot%_unitdir +%endif install -d -m 700 %buildroot%_homedir install -d -m 700 %buildroot%_sysconfdir/logrotate.d install -p -m 644 %SOURCE2 %buildroot%_sysconfdir/logrotate.d/resalloc-server @@ -181,6 +283,15 @@ install -d -m 755 %buildroot/%_libexecdir install -p -m 755 %SOURCE3 %buildroot/%_libexecdir/%name-merge-hook-logs install -d %buildroot%_sysconfdir/cron.hourly install -p -m 755 %SOURCE4 %buildroot%_sysconfdir/cron.hourly/resalloc +install -p -m 755 %name-wait-for-ssh %buildroot%_bindir/%name-wait-for-ssh + +%if %{without python3} +rm %buildroot%_bindir/%name-agent-* +rm %buildroot%_sysconfdir/resalloc-agent-spawner/config.yaml +%endif + +install -m0644 -D resalloc.sysusers.conf %{buildroot}%{_sysusersdir}/resalloc.conf +install -m0644 -D resalloc-agent-spawner.sysusers.conf %{buildroot}%{_sysusersdir}/resalloc-agent-spawner.conf %if %{with check} @@ -197,15 +308,10 @@ make check TEST_PYTHONS="python3" ln -s "%{default_sitelib}/%{name}server" %buildroot%_homedir/project +%if %{with sysusers_compat} %pre server -user=%sysuser -group=%sysgroup -getent group "$user" >/dev/null || groupadd -r "$group" -getent passwd "$user" >/dev/null || \ -useradd -r -g "$group" -G "$group" -s /bin/bash \ - -c "resalloc server's user" "$user" \ - -d "%_homedir" - +%create_user_group %sysuser %sysgroup /bin/bash %_homedir +%endif %post server %systemd_post resalloc.service @@ -214,7 +320,21 @@ useradd -r -g "$group" -G "$group" -s /bin/bash \ %systemd_postun_with_restart resalloc.service -%global doc_files NEWS README +%if %{with python3} +%if %{with sysusers_compat} +%pre agent-spawner +%create_user_group %agent_user %agent_group /bin/false / +%endif + +%post agent-spawner +%systemd_post resalloc-agent-spawner.service + +%postun agent-spawner +%systemd_postun_with_restart resalloc-agent-spawner.service +%endif + + +%global doc_files NEWS README.md %files %doc %doc_files @@ -247,8 +367,7 @@ useradd -r -g "$group" -G "$group" -s /bin/bash \ %{default_sitelib}/%{name}server %{_bindir}/%{name}-server %{_bindir}/%{name}-maint -%{_bindir}/%{name}-check-vm-ip -%attr(0700, %sysuser, %sysgroup) %dir %{_sysconfdir}/%{name}server +%attr(0750, %sysuser, %sysgroup) %dir %{_sysconfdir}/%{name}server %config(noreplace) %{_sysconfdir}/%{name}server/* %_unitdir/resalloc.service %attr(0700, %sysuser, %sysgroup) %dir %_logdir @@ -258,16 +377,142 @@ useradd -r -g "$group" -G "$group" -s /bin/bash \ %config %_sysconfdir/logrotate.d/resalloc-server %_libexecdir/resalloc-merge-hook-logs %config %attr(0755, root, root) %{_sysconfdir}/cron.hourly/resalloc +%{_sysusersdir}/resalloc.conf + + +%files helpers +%doc %doc_files +%license COPYING +%{_bindir}/%{name}-check-vm-ip +%{_bindir}/%{name}-wait-for-ssh + %if %{with python3} +%files agent-spawner +%_bindir/resalloc-agent* +%{default_sitelib}/%{name}_agent_spawner +%_unitdir/resalloc-agent-spawner.service +%config(noreplace) %_sysconfdir/resalloc-agent-spawner +%{_sysusersdir}/resalloc-agent-spawner.conf + %files webui %doc %doc_files %license COPYING %{default_sitelib}/%{name}webui/ %_datadir/%{name}webui/ +%_var/www/cgi-%{name} %endif +%files selinux + + %changelog +* Fri Sep 19 2025 Python Maint - 5.11-4 +- Rebuilt for Python 3.14.0rc3 bytecode + +* Fri Aug 15 2025 Python Maint - 5.11-3 +- Rebuilt for Python 3.14.0rc2 bytecode + +* Fri Jul 25 2025 Fedora Release Engineering - 5.11-2 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_43_Mass_Rebuild + +* Mon Jul 21 2025 Pavel Raiskup - 5.11-1 +- new upstream release, don't keep cleanup processes indefinitely: + https://github.com/praiskup/resalloc/releases/tag/v5.11 + +* Thu Jun 12 2025 Pavel Raiskup - 5.10-1 +- new upstream release, packages use RPM built-in sysusers support: + https://github.com/praiskup/resalloc/releases/tag/v5.10 + +* Thu Jun 05 2025 Python Maint - 5.9-2 +- Rebuilt for Python 3.14 + +* Sat Jan 18 2025 Pavel Raiskup - 5.9-1 +- New upstream release https://github.com/praiskup/resalloc/releases/tag/v5.9 + +* Thu Jan 16 2025 Jakub Kadlcik - 5.8-1 +- New upstream release https://github.com/praiskup/resalloc/releases/tag/v5.8 + +* Fri Jan 10 2025 Jakub Kadlcik - 5.7-1 +- New upstream release https://github.com/praiskup/resalloc/releases/tag/v5.7 + +* Mon Oct 07 2024 Pavel Raiskup - 5.6-1 +- New upstream release https://github.com/praiskup/resalloc/releases/tag/v5.6 + +* Fri Jul 19 2024 Fedora Release Engineering - 5.5-3 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_41_Mass_Rebuild + +* Sun Jun 09 2024 Python Maint - 5.5-2 +- Rebuilt for Python 3.13 + +* Wed Apr 24 2024 Pavel Raiskup - 5.5-1 +- New upstream release https://github.com/praiskup/resalloc/releases/tag/v5.5 + +* Fri Mar 15 2024 Pavel Raiskup - 5.4-1 +- New upstream release https://github.com/praiskup/resalloc/releases/tag/v5.4 + +* Wed Feb 28 2024 Pavel Raiskup - 5.3-1 +- New upstream release https://github.com/praiskup/resalloc/releases/tag/v5.3 + +* Wed Feb 28 2024 Pavel Raiskup - 5.2-1 +- New upstream release https://github.com/praiskup/resalloc/releases/tag/v5.2 + +* Fri Jan 26 2024 Fedora Release Engineering - 5.1-3 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_40_Mass_Rebuild + +* Mon Jan 22 2024 Fedora Release Engineering - 5.1-2 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_40_Mass_Rebuild + +* Mon Nov 06 2023 Pavel Raiskup - 5.1-1 +- new upstream release https://github.com/praiskup/resalloc/releases/tag/v5.1 + +* Fri Aug 11 2023 Pavel Raiskup - 5.0-1 +- new upstream release https://github.com/praiskup/resalloc/releases/tag/v5.0 + +* Fri Jul 21 2023 Fedora Release Engineering - 4.9-4 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_39_Mass_Rebuild + +* Tue Jul 04 2023 Python Maint - 4.9-3 +- Rebuilt for Python 3.12 + +* Mon Jan 30 2023 Miro Hrončok - 4.9-2 +- Rebuilt to change Python shebangs to /usr/bin/python3.6 on EPEL 8 + +* Mon Jan 23 2023 Pavel Raiskup - 4.9-1 +- new upstream release https://github.com/praiskup/resalloc/releases/tag/v4.9 + +* Fri Jan 20 2023 Fedora Release Engineering - 4.8-2 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_38_Mass_Rebuild + +* Fri Sep 23 2022 Pavel Raiskup - 4.8-1 +- new upstream release: + https://github.com/praiskup/resalloc/releases/tag/v4.8 + +* Tue Sep 20 2022 Pavel Raiskup - 4.7-1 +- new upstream release: + https://github.com/praiskup/resalloc/releases/tag/v4.7 + +* Sat Jul 23 2022 Fedora Release Engineering - 4.6-3 +- Rebuilt for https://fedoraproject.org/wiki/Fedora_37_Mass_Rebuild + +* Wed Jun 29 2022 Jakub Kadlcik - 4.6-2 +- Add resalloc-selinux subpackage + +* Wed Jun 29 2022 Jakub Kadlcik - 4.6-1 +- New upstream version: + https://github.com/praiskup/resalloc/releases/tag/v4.6 + +* Thu Jun 23 2022 Pavel Raiskup - 4.5-1 +- New upstream version: + https://github.com/praiskup/resalloc/releases/tag/v4.5 + +* Wed Jun 22 2022 Jakub Kadlcik - 4.4-1 +- New upstream version: + https://github.com/praiskup/resalloc/releases/tag/v4.4 + +* Tue Jun 14 2022 Python Maint - 4.3-2 +- Rebuilt for Python 3.11 + * Thu Jan 20 2022 Pavel Raiskup - 4.3-1 - new upstream release: https://github.com/praiskup/resalloc/releases/tag/v4.3 diff --git a/sources b/sources index 4009b04..f458908 100644 --- a/sources +++ b/sources @@ -1 +1 @@ -SHA512 (resalloc-4.3.tar.gz) = e48dc3ccdb1c62ebabc01ce609fbe11f7a832fec054c78494e831c98983f3c290523bcfd0e855ddbac8f80b1e906badc9aedea95e870608c76a14615e734dc4d +SHA512 (resalloc-5.11.tar.gz) = 73becebe671a59912006eb4b227c10f6ac7be91a91e94cff470ac6bb32eb4447ec905b1d151c5eeb54fdc6c2a6179f9b7d606bcf25c3f9caeb63bde171759ac3 diff --git a/wait-for-ssh b/wait-for-ssh new file mode 100644 index 0000000..d302c77 --- /dev/null +++ b/wait-for-ssh @@ -0,0 +1,161 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright (C) 2017 Pavel Raiskup +# +# This program accepts one argument IP or HOSTNAME. First try to connect to the +# HOSTNAME as 'root' user. If cloud-init scripts instruct us to use different +# user than 'root', switch to that user and check again. In the end, print the +# successful username on stdout. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +from re import compile as re_compile +import sys +from os import devnull +from threading import Thread, Event +from argparse import ArgumentParser +from subprocess import Popen, PIPE +import logging + +handler = logging.StreamHandler() +log = logging.getLogger() +log.setLevel(logging.INFO) +log.addHandler(handler) + +# create console handler and set level to debug + +ssh = [ + 'ssh', + '-o', 'StrictHostKeyChecking=no', + '-o', 'UserKnownHostsFile=/dev/null', + '-o', 'PasswordAuthentication=no', + '-o', 'ConnectTimeout=10', +] + +expected_output = 'foobar' +inner_cmd = 'echo ' + expected_output + + +class Checker(Thread): + user = 'root' + daemon = True + user_re = '[a-zA-Z0-9_.][a-zA-Z0-9_.-]*[$]?' + re_clouduser = re_compile('Please login as the user "({0})"'.format(user_re)) + event = Event() + + def loop(self): + cmd = ssh + [ + '{0}@{1}'.format(self.user, self.args.host), + inner_cmd, + ] + + with open(devnull, 'w') as drop: + log.debug('executing: ' + ' '.join(cmd)) + self.child = Popen(cmd, stdout=PIPE, stderr=drop) + (stdout, _) = self.child.communicate() + + exp = (expected_output + '\n').encode('ascii') + if self.child.returncode == 0 and stdout == exp: + if self.args.print_user: + print(self.user) + return True + + if self.args.cloud_user: + match = self.re_clouduser.search(str(stdout)) + if match: + self.user = match.group(1) + log.info('cloud user switched to ' + self.user) + return False + + def run(self): + while True: + if self.loop(): + # Success! + break + + if self.event.wait(1): + log.debug("stopping per kill event") + break + + def kill(self): + self.event.set() + + # Best effort kill. + try: + self.child.kill() + except: + pass + self.join() + + +parser = ArgumentParser( + description="Wait till the host's ssh becomes responsive.") +parser.add_argument('host', help='hostname or IP') +parser.add_argument('--timeout', + help='seconds to wait before failure, default=indefinitely', + default=None, type=float) +parser.add_argument('--check-cloud-user', action='store_true', default=False, + dest='cloud_user', + help='if cloud-init disallows "root" login, try to detect the cloud ' \ + +'user and use that') +parser.add_argument('--print-user', action='store_true', default=False, + dest='print_user', + help='print the username which succeeded to connect on stdout') +parser.add_argument('--log', default=False, + dest='log_verbosity', + help='set the threshold for logging, e.g. debug, info, error, ...') + + +def main(): + sleep_period = 1.0 + args = parser.parse_args() + + if args.log_verbosity: + log.setLevel(logging.getLevelName(args.log_verbosity.upper())) + + def timeouted(): + if args.timeout is None: + return False + log.debug("wait {0}s, remains {1}s".format(sleep_period, args.timeout)) + args.timeout -= sleep_period + return args.timeout <= 0 + + checker = Checker() + checker.args = args + checker.start() + + try: + # threading.join() is not Ctrl-C interruptable :( in python2, so we need + # this ugly infinite loop. + # https://stackoverflow.com/questions/25676835/signal-handling-in-multi-threaded-python + while True: + checker.join(sleep_period) + if not checker.is_alive(): + # Success! + return 0 + + if timeouted(): + log.error("timeout!") + checker.kill() + return 1 + + except KeyboardInterrupt: + log.error("interrupt by user") + checker.kill() + return 1 + +if __name__ == "__main__": + sys.exit(main())