From 61a2eb80ff7e22652d7afcc3c3afcacfabe7508b Mon Sep 17 00:00:00 2001
From: Ben Morrice <ben.morrice@cern.ch>
Date: Mon, 3 May 2021 07:54:27 +0000
Subject: [PATCH] add rhncheck

- this script will check if the certs defined for cdn.redhat.com
  repositories are valid
- if the certs are not valid, the script will download new certs
  from redhat and perform another check against the newly downloaded certs
- an email is sent containing all repositories that have issues, including
  an attachment of the new cert should one exist
- .gitlab-ci, generatejobs have been 'tweaked' to support this 'non reposync'
  workflow
---
 .gitlab-ci.yml          |   2 +
 dev.repos.d/rhncheck    |   2 +
 dev.repos.yaml          |   7 +++
 dev.variables.sh        |   2 +
 generateJobs.py         |   6 ++
 prod.repos.yaml         |   9 ++-
 prod.variables.sh       |   2 +
 reposync.nomad.tpl      |   4 ++
 reposync/Dockerfile     |   4 +-
 reposync/rhncheck.py    | 124 ++++++++++++++++++++++++++++++++++++++++
 reposync/runreposync.sh |   7 ++-
 11 files changed, 166 insertions(+), 3 deletions(-)
 create mode 100644 dev.repos.d/rhncheck
 create mode 100755 reposync/rhncheck.py

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index a7eb7d0..5da0259 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -5,9 +5,11 @@ prepare_dirs:
   stage: prepare
   script:
     - cp -r gpgkeys/ reposync/gpgkeys/
+    - cp -r prod.repos.d/ reposync/prod.repos.d/
   artifacts:
     paths:
       - reposync/gpgkeys/
+      - reposync/prod.repos.d/
     expire_in: 1 day
 
 generate_jobs:
diff --git a/dev.repos.d/rhncheck b/dev.repos.d/rhncheck
new file mode 100644
index 0000000..99e9b70
--- /dev/null
+++ b/dev.repos.d/rhncheck
@@ -0,0 +1,2 @@
+[rhncheck]
+baseurl=None
diff --git a/dev.repos.yaml b/dev.repos.yaml
index 9d9c868..da60a81 100644
--- a/dev.repos.yaml
+++ b/dev.repos.yaml
@@ -15,7 +15,14 @@ defaults:
   pathcut: '//'
   run_reposync: true
   run_createrepo: true
+  run_rhncheck: false
 
+### specific configuration for rhn_check script ###
+rhncheck:
+  schedule: '0 5 * * *'
+  run_reposync: false
+  run_createrepo: false
+  run_rhncheck: true
 
 ### Repos go here ###
 
diff --git a/dev.variables.sh b/dev.variables.sh
index 290cd8f..d528627 100644
--- a/dev.variables.sh
+++ b/dev.variables.sh
@@ -1,3 +1,5 @@
 MOUNTPOINT="/mnt/data2/test/repos"
 REPOS="dev.repos.d"
 REPOSCONF="dev.repos.yaml"
+RHNCHECK_RHN_USER="cernlinux"
+RHNCHECK_ADMIN_EMAIL="ben.morrice@cern.ch"
diff --git a/generateJobs.py b/generateJobs.py
index e137663..58498b6 100755
--- a/generateJobs.py
+++ b/generateJobs.py
@@ -47,6 +47,7 @@ try:
   def_pathcut    = config['defaults']['pathcut']
   def_reposync   = config['defaults']['run_reposync']
   def_createrepo = config['defaults']['run_createrepo']
+  def_rhncheck   = config['defaults']['run_rhncheck']
 except IndexError:
   print('Missing configuration options')
   sys.exit(2)
@@ -93,6 +94,10 @@ for repofile in os.listdir(yumdir_name):
       RUN_CREATEREPO = config[repofile]['run_createrepo']
     except KeyError:
       RUN_CREATEREPO = def_reposync
+    try:
+      RUN_RHNCHECK = config[repofile]['run_rhncheck']
+    except KeyError:
+      RUN_RHNCHECK = def_rhncheck
 
     url = repoconfig[rid]['baseurl']
     path = url.split(PATHCUT)[-1].lstrip('/')
@@ -110,6 +115,7 @@ for repofile in os.listdir(yumdir_name):
       'SCHEDULE'      : SCHEDULE,
       'RUN_REPOSYNC'  : 1 if RUN_REPOSYNC else 0,
       'RUN_CREATEREPO': 1 if RUN_CREATEREPO else 0,
+      'RUN_RHNCHECK': 1 if RUN_RHNCHECK else 0,
     }
 
     jobfile = '{0}_reposync_{1}.nomad'.format(os.getenv('PREFIX', 'dev'), rid)
diff --git a/prod.repos.yaml b/prod.repos.yaml
index d772596..cbce835 100644
--- a/prod.repos.yaml
+++ b/prod.repos.yaml
@@ -15,7 +15,14 @@ defaults:
   pathcut: '//'
   run_reposync: true
   run_createrepo: true
-
+  run_rhncheck: false
+
+### specific configuration for rhn_check script ###
+rhncheck:
+  schedule: '0 5 * * *'
+  run_reposync: false
+  run_createrepo: false
+  run_rhncheck: true
 
 ### Repos go here ###
 
diff --git a/prod.variables.sh b/prod.variables.sh
index 38161c7..fa974a7 100644
--- a/prod.variables.sh
+++ b/prod.variables.sh
@@ -1,3 +1,5 @@
 MOUNTPOINT="/mnt/data1/dist"
 REPOS="prod.repos.d"
 REPOSCONF="prod.repos.yaml"
+RHNCHECK_RHN_USER="cernlinux"
+RHNCHECK_ADMIN_EMAIL="lxsoft-admins@cern.ch"
diff --git a/reposync.nomad.tpl b/reposync.nomad.tpl
index 8f44799..84b7623 100644
--- a/reposync.nomad.tpl
+++ b/reposync.nomad.tpl
@@ -53,6 +53,10 @@ job "${PREFIX}_reposync_${REPOID}" {
       CHECKSUM = "$CHECKSUM"
       RUN_REPOSYNC = "$RUN_REPOSYNC"
       RUN_CREATEREPO = "$RUN_CREATEREPO"
+      RUN_RHNCHECK = "$RUN_RHNCHECK"
+      RHNCHECK_ADMIN_EMAIL = "$RHNCHECK_ADMIN_EMAIL"
+      RHNCHECK_RHN_USER = "$RHNCHECK_RHN_USER"
+      RHNCHECK_RHN_PASSWORD = "$RHNCHECK_RHN_PASSWORD"
     }
 
     resources {
diff --git a/reposync/Dockerfile b/reposync/Dockerfile
index 0eed39d..c97eccd 100644
--- a/reposync/Dockerfile
+++ b/reposync/Dockerfile
@@ -1,12 +1,14 @@
 FROM gitlab-registry.cern.ch/linuxsupport/cc7-base:latest
 
-RUN yum install -y jq createrepo patch \
+RUN yum install -y jq createrepo patch python-requests \
   && yum clean all
 
 RUN rm -rf /etc/yum.repos.d/*
 COPY gpgkeys/ /etc/pki/rpm-gpg/
 RUN find /etc/pki/rpm-gpg/ -type f -exec rpm --import {} \;
 COPY runreposync.sh /root/
+COPY rhncheck.py /root/
+COPY prod.repos.d/ /root/prod.repos.d/
 
 # Temporary hack, hopefully. Waiting on
 # https://github.com/rpm-software-management/yum-utils/pull/49
diff --git a/reposync/rhncheck.py b/reposync/rhncheck.py
new file mode 100755
index 0000000..5b07793
--- /dev/null
+++ b/reposync/rhncheck.py
@@ -0,0 +1,124 @@
+#!/usr/bin/env python
+
+import sys
+import json
+import requests
+import base64
+import ssl
+import glob
+import os
+import ConfigParser
+import tempfile
+import smtplib
+
+from email.MIMEMultipart import MIMEMultipart
+from email.MIMEBase import MIMEBase
+from email import Encoders
+from email.mime.text import MIMEText
+
+requests.packages.urllib3.disable_warnings(requests.packages.urllib3.exceptions.InsecureRequestWarning)
+
+email_to = os.getenv('RHNCHECK_ADMIN_EMAIL')
+login = os.getenv('RHNCHECK_RHN_USER')
+password = os.getenv('RHNCHECK_RHN_PASSWORD')
+if email_to is None or login is None or password is None:
+  print('Cannot determine email_to, login or password variables. Exiting')
+  sys.exit(1)
+
+
+# taken from access.redhat.com for the 'linuxsoft-mirror' host
+linuxsoft_uuid="b4ec8c2d-3eae-4ae0-b8fa-ec6d8a08ce9f"
+email_from='linux.support@cern.ch'
+
+problems = set()
+pending_repo=None
+pending_baseurl=None
+pending_sslclientcert=None
+
+tmpdir = tempfile.mkdtemp(dir = '/tmp')
+
+def call_https_rhsm(url):
+  base64string = base64.encodestring('%s:%s' % (login, password)).strip()
+  headers = {'Authorization': 'Basic %s' % base64string}
+  result = requests.get(url, headers=headers, verify=False)
+  return json.loads(result.content)
+
+def call_https_cdn(url, cert):
+  try:
+    request = requests.get(url, cert=(cert), verify=False)
+    if request.status_code == 200:
+      return True
+    else:
+      return False
+  except:
+    return False
+
+def get_entitlements(directory):
+  if glob.glob(directory + '/*pem'):
+    print('rhn certs have already been downloaded this run, not doing that again')
+    return True
+  key = call_https_rhsm('https://subscription.rhsm.redhat.com/subscription/consumers/' + linuxsoft_uuid)['idCert']['key']
+  certificates = call_https_rhsm('https://subscription.rhsm.redhat.com/subscription/consumers/' + linuxsoft_uuid + "/certificates")
+  for cert in certificates:
+    if not cert['serial']['revoked']:
+      print('Downloading rhn cert: %s' % str(cert['serial']['id']) + '.pem')
+      f = open(directory + '/' + str(cert['serial']['id']) + '.pem', 'w')
+      f.write(cert['cert'])
+      f.write(key)
+      f.close()
+
+email_certs = []
+for repofile in glob.glob("prod.repos.d/*repo"):
+  f = open(repofile, 'r')
+  config = ConfigParser.RawConfigParser()
+  config.read(repofile)
+  for repo in config.sections():
+    if 'cdn.redhat.com' in config.get(repo, 'baseurl'):
+      baseurl = config.get(repo, 'baseurl')
+      sslclientcert = config.get(repo, 'sslclientcert')
+      print('Checking repo: %s' % repo)
+      if not os.path.exists(sslclientcert):
+        problems.add('%s: %s does not exist, however is defined' % (repo, sslclientcert))
+      else:
+        if not call_https_cdn(baseurl, sslclientcert):
+          print('Installed cert does not auth, trying with new certs from rhn')
+          get_entitlements(tmpdir)
+          valid_certs = []
+          for cert in glob.glob(tmpdir + '/*pem'):
+            if call_https_cdn(baseurl, cert):
+              valid_certs.append(os.path.split(cert)[1])
+              email_certs.append(cert)
+            if len(valid_certs) is 0:
+              problems.add('%s: %s failed to auth to %s. A replacement cert was not found.' % (repo, sslclientcert, baseurl))
+            else:
+              new_certs = ','.join(valid_certs)
+              problems.add('%s: %s failed to auth to %s. One of the following certs would be a valid replacement: %s (attached)' % (repo, sslclientcert, baseurl, new_certs))
+
+email_max_problems = 60
+
+if len(problems) > 0:
+  msg = MIMEMultipart()
+  msg['Subject'] = 'reposync: cdn.redhat.com sync problems detected'
+  msg['From'] = email_from
+  msg['To'] = email_to
+  if len(problems) > email_max_problems:
+    problems_string = 'Too many problems to list via email!\n\nPlease check elastic search logs to see what is going on...'
+  else:
+    problems_string = '\n'.join(list(map(str,problems)))
+
+  body=MIMEText('Dear admins,\n\nI thought you might be interested to know that we had a few sync errors against cdn.redhat.com repositories.\n\nThere are currently ' + str(len(problems)) + ' problems that need addressing.\n\n' + problems_string + '\n\n\nBest regards,\nCERN Linux Droid\n(on behalf of the friendly humans of Linux Support)',_subtype='plain')
+  msg.attach(body)
+
+  if len(email_certs) > 0 and len(problems) < email_max_problems:
+    for cert in list(set(email_certs)):
+      part = MIMEBase('application', "octet-stream")
+      part.set_payload(open(cert, "rb").read())
+      Encoders.encode_base64(part)
+      part.add_header('Content-Disposition', 'attachment; filename="' + os.path.split(cert)[1] + '"')
+      msg.attach(part)
+
+  server = smtplib.SMTP('cernmx.cern.ch')
+  server.sendmail(email_from, email_to, msg.as_string())
+
+else:
+  print("No problems today! :)")
diff --git a/reposync/runreposync.sh b/reposync/runreposync.sh
index fb52d63..071fdb4 100755
--- a/reposync/runreposync.sh
+++ b/reposync/runreposync.sh
@@ -19,7 +19,12 @@ error () {
 EOF
 }
 
-echo "Inputs: REPOID=\"$REPOID\" REPOFILE=\"$REPOFILE\" REPOPATH=\"$REPOPATH\" CHECKSUM=\"$CHECKSUM\" RUN_REPOSYNC=\"$RUN_REPOSYNC\" RUN_CREATEREPO=\"$RUN_CREATEREPO\""
+echo "Inputs: REPOID=\"$REPOID\" REPOFILE=\"$REPOFILE\" REPOPATH=\"$REPOPATH\" CHECKSUM=\"$CHECKSUM\" RUN_REPOSYNC=\"$RUN_REPOSYNC\" RUN_CREATEREPO=\"$RUN_CREATEREPO\" RUN_RHNCHECK=\"$RUN_RHNCHECK\""
+
+if [[ $RUN_RHNCHECK -eq 1 ]]; then
+  python /root/rhncheck.py
+  exit
+fi
 
 echo "[${REPOID}]" > /etc/yum.repos.d/sync.repo
 echo $REPOFILE | base64 -d >> /etc/yum.repos.d/sync.repo
-- 
GitLab