From 2bd951b3db9747e78bcde5fb48ec3aeb07840417 Mon Sep 17 00:00:00 2001
From: Daniel Juarez <daniel.juarez.gonzalez@cern.ch>
Date: Fri, 11 Feb 2022 12:43:22 +0100
Subject: [PATCH 1/5] Add extra foolprof instructions for koji upgrade

---
 docs/koji/upgrading.md | 79 +++++++++++++++++++++++++++++++++++++++---
 1 file changed, 74 insertions(+), 5 deletions(-)

diff --git a/docs/koji/upgrading.md b/docs/koji/upgrading.md
index cd4ed51..5ee0967 100644
--- a/docs/koji/upgrading.md
+++ b/docs/koji/upgrading.md
@@ -13,7 +13,8 @@ The standard process to upgrade koji is:
     The below instructions are tailored for kojitest - adaption will be required to execute against koji production
 
 !!! Note ""
-    Upgrading koji production will require the usual ITSSB notifications
+    Upgrading koji production will require the usual ITSSB notifications. Take [OTG0069160](https://cern.service-now.com/service-portal?id=outage&n=OTG0069160)
+    as an example. Try to schedule them with at least one week in advance.
 
 ## Build new rpm and tag appropriately
 see [https://gitlab.cern.ch/linuxsupport/rpms/koji](https://gitlab.cern.ch/linuxsupport/rpms/koji)
@@ -21,12 +22,23 @@ see [https://gitlab.cern.ch/linuxsupport/rpms/koji](https://gitlab.cern.ch/linux
 !!! Note ""
     Don't forget to tag the build as needed **before** shutting down Koji!
 
+# Test it
+
+The following process applies for both test and prod nodes, so please first do it on <kojitest.cern.ch> involved nodes, adapting the commands when necessary.
+Once you have tested a few builds, tags, image-builds or else, you will have validated that the built rpm works as it should and you can announce the OTG.
+
+You may want to install the test rpm from <linuxsoft.cern.ch/internal/repos/linuxsupport8s-qa/> (or the corresponding one for other distros) locally to do this.Please be aware of your `/etc/koji.conf` content, take it from `aiadm.cern.ch` if you need to adjust it.
+
 ## Disable alerts
 
 Make sure Roger knows something is going on:
 
 ```
-roger update --all_alarms false --message OTG0062870 --duration 2h koji{10,11,20,21,22,30,31,32,33,34,35,40,41}
+# roger status changes must be ran with sudo even when ssh'ing as root
+# test
+wassh -l root -c lsb/test2 'sudo roger update --all_alarms false --message OTGXXXXXX --duration 2h'
+# prod
+wassh -l root -c lsb 'sudo roger update --all_alarms false --message OTGXXXXXX --duration 2h'
 ```
 
 (If you change the appstate, the machines would be removed from the LB alias and the intervention will take longer)
@@ -36,30 +48,70 @@ roger update --all_alarms false --message OTG0062870 --duration 2h koji{10,11,20
 *puppet (disable)*
 
 ```
+# test
 mco puppet disable "koji upgrade OTGXXXXXX" --dm puppetdb -T lsb -F 'hostgroup_1=test2'
+# prod
+mco puppet disable "koji upgrade OTGXXXXXX" --dm puppetdb -T lsb
+
+# If mco does not work for you:
+# test
+wassh -l root -c lsb/test2 'puppet agent --disable'
+# prod
+wassh -l root -c lsb 'puppet agent --disable'
 ```
 
 *builders*
 
 ```
+# test
 mco service stop kojid --dm puppetdb -T lsb -F 'hostgroup_1=test2' -F 'hostgroup_2=builder'
+# prod
+mco service stop kojid --dm puppetdb -T lsb -F 'hostgroup_1=builder'
+
+# If mco does not work for you:
+# test
+wassh -l root -c lsb/test2/builder 'service kojid stop'
+# prod
+wassh -l root -c lsb/builder 'service kojid stop'
 ```
 
 *hub*
 
 ```
+# test
 mco service stop kojira --dm puppetdb -T lsb -F 'hostgroup_1=test2' -F 'hostgroup_2=hub'
 mco service stop httpd --dm puppetdb -T lsb -F 'hostgroup_1=test2' -F 'hostgroup_2=hub'
+# prod
+mco service stop kojira --dm puppetdb -T lsb -F 'hostgroup_1=hub'
+mco service stop httpd --dm puppetdb -T lsb -F 'hostgroup_1=hub'
+
+# If mco does not work for you:
+# test
+wassh -l root -c lsb/test2/hub 'service kojira stop'
+wassh -l root -c lsb/test2/hub 'service httpd stop'
+# prod
+wassh -l root -c lsb/hub 'service kojira stop'
+wassh -l root -c lsb/hub 'service httpd stop'
 ```
 
 *web*
 
 ```
+# test
 mco service stop httpd --dm puppetdb -T lsb -F 'hostgroup_1=test2' -F 'hostgroup_2=web'
+# prod
+mco service stop httpd --dm puppetdb -T lsb -F 'hostgroup_1=web'
+
+# If mco does not work for you:
+# test
+wassh -l root -c lsb/test2/web 'service httpd stop'
+# prod
+wassh -l root -c lsb/web 'service httpd stop'
 ```
 
 ## Backup and run migration script
-*Extract migration script*
+
+*Extract migration script **if there is one***
 
 ```
 yumdownloader koji
@@ -67,6 +119,7 @@ rpm2cpio koji-1.18.1-1.el7.cern.noarch.rpm | cpio -idv ./usr/share/doc/koji-1.18
 scp usr/share/doc/koji-1.18.1/docs/schema-upgrade-1.17-1.18.sql aiadm:
 ```
 
+
 *Initialise postgresql 9.6 environment*
 ```
 ssh aiadm
@@ -84,10 +137,19 @@ psql -h $dbod.cern.ch -p $port -d $database -U $username < schema-upgrade-1.17-1
 !!! Note ""
     You can retrieve the credentials from /etc/koji-hub/hub.conf
 
-##Upgrade Koji RPMs
+## Upgrade Koji RPMs
 
 ```
+# test
 mco shell run '/usr/bin/yum clean all && /usr/local/sbin/distro_sync.sh' --dm puppetdb -T lsb -F 'hostgroup_1=test2'
+# prod
+mco shell run '/usr/bin/yum clean all && /usr/local/sbin/distro_sync.sh' --dm puppetdb -T lsb
+
+# If mco does not work for you:
+# test
+wassh -l root -c lsb/test2 '/usr/bin/yum clean all && /usr/local/sbin/distro_sync.sh'
+# prod
+wassh -l root -c lsb '/usr/bin/yum clean all && /usr/local/sbin/distro_sync.sh'
 ```
 
 ## Restart Koji
@@ -103,5 +165,12 @@ mco puppet runonce --dm puppetdb -T lsb -F 'hostgroup_1=test2'
 
 ```
 mco puppet enable --dm puppetdb -T lsb -F 'hostgroup_1=test2'
-roger update --all_alarms true koji{10,11,20,21,22,30,31,32,33,34,35,40,41}
+
+# test
+wassh -l root -c lsb/test2 'puppet agent --enable'
+wassh -l root -c lsb/test2 'sudo roger update --all_alarms true'
+
+# prod
+wassh -l root -c lsb 'puppet agent --enable'
+wassh -l root -c lsb 'sudo roger update --all_alarms true'
 ```
-- 
GitLab


From d888d061141d305a96df7d9ebbb70ceb85cc1fce Mon Sep 17 00:00:00 2001
From: Daniel Juarez <daniel.juarez.gonzalez@cern.ch>
Date: Fri, 11 Feb 2022 13:56:32 +0100
Subject: [PATCH 2/5] koji nodes must all be on production env before tagging

---
 docs/koji/upgrading.md | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/docs/koji/upgrading.md b/docs/koji/upgrading.md
index 5ee0967..b27e1cd 100644
--- a/docs/koji/upgrading.md
+++ b/docs/koji/upgrading.md
@@ -17,7 +17,12 @@ The standard process to upgrade koji is:
     as an example. Try to schedule them with at least one week in advance.
 
 ## Build new rpm and tag appropriately
-see [https://gitlab.cern.ch/linuxsupport/rpms/koji](https://gitlab.cern.ch/linuxsupport/rpms/koji)
+
+See [https://gitlab.cern.ch/linuxsupport/rpms/koji](https://gitlab.cern.ch/linuxsupport/rpms/koji)
+
+**Before you tag the rpm to qa make sure all koji.cern.ch nodes are on the production environment and all kojitest.cern.ch nodes are on the qa environment.**
+
+If you do not do so, you may have prod nodes with qa repos which will make versions inconsistent across the LSB nodes.
 
 !!! Note ""
     Don't forget to tag the build as needed **before** shutting down Koji!
-- 
GitLab


From 91e818fe858444b1f4dd898d1e90c992b7944067 Mon Sep 17 00:00:00 2001
From: Daniel Juarez <daniel.juarez.gonzalez@cern.ch>
Date: Mon, 14 Feb 2022 16:04:11 +0100
Subject: [PATCH 3/5] Koji DB tbag instructions

---
 docs/koji/upgrading.md | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/docs/koji/upgrading.md b/docs/koji/upgrading.md
index b27e1cd..2ae2c47 100644
--- a/docs/koji/upgrading.md
+++ b/docs/koji/upgrading.md
@@ -20,7 +20,7 @@ The standard process to upgrade koji is:
 
 See [https://gitlab.cern.ch/linuxsupport/rpms/koji](https://gitlab.cern.ch/linuxsupport/rpms/koji)
 
-**Before you tag the rpm to qa make sure all koji.cern.ch nodes are on the production environment and all kojitest.cern.ch nodes are on the qa environment.**
+**Before you tag the rpm to `qa` make sure all koji.cern.ch nodes are on the production environment and all kojitest.cern.ch nodes are on the qa environment.**
 
 If you do not do so, you may have prod nodes with qa repos which will make versions inconsistent across the LSB nodes.
 
@@ -139,8 +139,14 @@ pg_dump -h $dbod.cern.ch -p $port -d $database -U $username > kojitest_1.17-`dat
 psql -h $dbod.cern.ch -p $port -d $database -U $username < schema-upgrade-1.17-1.18.sql
 ```
 
-!!! Note ""
-    You can retrieve the credentials from /etc/koji-hub/hub.conf
+**Note**: You can retrieve the credentials from `/etc/koji-hub/hub.conf` or with `tbag`:
+
+```
+# test
+tbag show koji_db_password --hg lsb/test2
+# prod
+tbag show koji_db_password --hg lsb
+```
 
 ## Upgrade Koji RPMs
 
-- 
GitLab


From 39109c1b026ca1f74646349d2d8492b33e1e0dbb Mon Sep 17 00:00:00 2001
From: Alex Iribarren <alex.iribarren@cern.ch>
Date: Mon, 14 Feb 2022 17:26:17 +0100
Subject: [PATCH 4/5] Apply 1 suggestion(s) to 1 file(s)

---
 docs/koji/upgrading.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/koji/upgrading.md b/docs/koji/upgrading.md
index 2ae2c47..7034b55 100644
--- a/docs/koji/upgrading.md
+++ b/docs/koji/upgrading.md
@@ -32,7 +32,7 @@ If you do not do so, you may have prod nodes with qa repos which will make versi
 The following process applies for both test and prod nodes, so please first do it on <kojitest.cern.ch> involved nodes, adapting the commands when necessary.
 Once you have tested a few builds, tags, image-builds or else, you will have validated that the built rpm works as it should and you can announce the OTG.
 
-You may want to install the test rpm from <linuxsoft.cern.ch/internal/repos/linuxsupport8s-qa/> (or the corresponding one for other distros) locally to do this.Please be aware of your `/etc/koji.conf` content, take it from `aiadm.cern.ch` if you need to adjust it.
+You may want to install the test rpm from <linuxsoft.cern.ch/internal/repos/linuxsupport8s-qa/> (or the corresponding one for other distros) locally to do this. Please be aware of your `/etc/koji.conf` content, take it from `aiadm.cern.ch` if you need to adjust it.
 
 ## Disable alerts
 
-- 
GitLab


From bda20d6fdb47f2e1cd22233c0290b475bccb78f2 Mon Sep 17 00:00:00 2001
From: Daniel Juarez <daniel.juarez.gonzalez@cern.ch>
Date: Tue, 15 Feb 2022 16:53:15 +0100
Subject: [PATCH 5/5] Restrict commands to subhostgroups

---
 docs/koji/upgrading.md | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/docs/koji/upgrading.md b/docs/koji/upgrading.md
index 7034b55..f49a0ca 100644
--- a/docs/koji/upgrading.md
+++ b/docs/koji/upgrading.md
@@ -42,8 +42,10 @@ Make sure Roger knows something is going on:
 # roger status changes must be ran with sudo even when ssh'ing as root
 # test
 wassh -l root -c lsb/test2 'sudo roger update --all_alarms false --message OTGXXXXXX --duration 2h'
-# prod
-wassh -l root -c lsb 'sudo roger update --all_alarms false --message OTGXXXXXX --duration 2h'
+# prod (limit it to prod nodes, not the whole hostgroup)
+wassh -l root -c lsb/hub 'sudo roger update --all_alarms false --message OTGXXXXXX --duration 2h'
+wassh -l root -c lsb/web 'sudo roger update --all_alarms false --message OTGXXXXXX --duration 2h'
+wassh -l root -c lsb/builder 'sudo roger update --all_alarms false --message OTGXXXXXX --duration 2h'
 ```
 
 (If you change the appstate, the machines would be removed from the LB alias and the intervention will take longer)
@@ -55,8 +57,10 @@ wassh -l root -c lsb 'sudo roger update --all_alarms false --message OTGXXXXXX -
 ```
 # test
 mco puppet disable "koji upgrade OTGXXXXXX" --dm puppetdb -T lsb -F 'hostgroup_1=test2'
-# prod
-mco puppet disable "koji upgrade OTGXXXXXX" --dm puppetdb -T lsb
+# prod (limit it to prod nodes, not the whole hostgroup)
+mco puppet disable "koji upgrade OTGXXXXXX" --dm puppetdb -T lsb -F 'hostgroup_1=hub'
+mco puppet disable "koji upgrade OTGXXXXXX" --dm puppetdb -T lsb -F 'hostgroup_1=web'
+mco puppet disable "koji upgrade OTGXXXXXX" --dm puppetdb -T lsb -F 'hostgroup_1=builder'
 
 # If mco does not work for you:
 # test
-- 
GitLab