From 40d46d3b5f2da4176cb54678544f77a5756ccc48 Mon Sep 17 00:00:00 2001 From: Georgios Argyriou Date: Tue, 28 Jun 2022 17:44:38 +0200 Subject: [PATCH 01/15] Support handling of RHEL qemu images. --- dev.variables.sh | 1 + download_rhel_iso.nomad | 11 +- download_rhel_iso/Dockerfile | 14 +- download_rhel_iso/convertimage.sh | 78 +++ download_rhel_iso/download_rhel_iso.py | 602 ++++++++++++++++------ download_rhel_iso/make_qemu_prod.sh | 23 + download_rhel_iso/openstack-upstream.repo | 6 + download_rhel_iso/os_auth.sh | 4 + download_rhel_iso/run_test_os_img.sh | 113 ++++ download_rhel_iso/share_test_img.sh | 10 + download_rhel_iso/upload2openstack.sh | 196 +++++++ prod.variables.sh | 1 + 12 files changed, 902 insertions(+), 157 deletions(-) create mode 100755 download_rhel_iso/convertimage.sh create mode 100755 download_rhel_iso/make_qemu_prod.sh create mode 100755 download_rhel_iso/openstack-upstream.repo create mode 100755 download_rhel_iso/os_auth.sh create mode 100755 download_rhel_iso/run_test_os_img.sh create mode 100755 download_rhel_iso/share_test_img.sh create mode 100755 download_rhel_iso/upload2openstack.sh diff --git a/dev.variables.sh b/dev.variables.sh index 4f6ff7d..1c8eb9d 100755 --- a/dev.variables.sh +++ b/dev.variables.sh @@ -2,4 +2,5 @@ SCHEDULE="0 8 * * *" ADMIN_EMAIL="georgios.argyriou@cern.ch" USER_EMAIL="gioargyr@gmail.com" RHEL_MOUNT="/mnt/data2/test/rhel_iso" +OS_PROJECT_NAME="IT Linux Support - Test VMs" CSETS="rhel-7-server-isos, rhel-8-for-x86_64-baseos-isos, rhel-8-for-aarch64-baseos-isos, rhel-8-for-ppc64le-baseos-isos, rhel-9-for-x86_64-baseos-isos, rhel-9-for-aarch64-baseos-isos, rhel-9-for-ppc64le-baseos-isos" \ No newline at end of file diff --git a/download_rhel_iso.nomad b/download_rhel_iso.nomad index 03289aa..32e9679 100644 --- a/download_rhel_iso.nomad +++ b/download_rhel_iso.nomad @@ -21,16 +21,25 @@ job "${PREFIX}_download_rhel_iso" { tag = "${PREFIX}_download_rhel_iso" } } - volumes = [ "$RHEL_MOUNT:/rhel" ] + volumes = [ + "$RHEL_MOUNT:/rhel", + "/etc/nomad/linuxci_api_token:/secrets/linuxci_api_token" + ] } env { RHSM_OFFLINE_TOKEN = "$RHSM_OFFLINE_TOKEN" + IMAGECI_USER = "$IMAGECI_USER" + IMAGECI_PWD = "$IMAGECI_PWD" LINUXCI_USER = "$LINUXCI_USER" LINUXCI_PWD = "$LINUXCI_PWD" + LINUXCI_APITOKEN = "$LINUXCI_APITOKEN" NOMAD_ADDR = "$NOMAD_ADDR" ADMIN_EMAIL = "$ADMIN_EMAIL" USER_EMAIL = "$USER_EMAIL" CSETS = "$CSETS" + OS_PROJECT_NAME = "$OS_PROJECT_NAME" + OS_USERNAME="$IMAGECI_USER" + OS_PASSWORD="$IMAGECI_PWD" } resources { diff --git a/download_rhel_iso/Dockerfile b/download_rhel_iso/Dockerfile index 0066f9b..82ef3d1 100644 --- a/download_rhel_iso/Dockerfile +++ b/download_rhel_iso/Dockerfile @@ -1,9 +1,19 @@ FROM gitlab-registry.cern.ch/linuxsupport/cs9-base:latest -COPY linuxsupport9-stable.repo /etc/yum.repos.d/ +COPY *repo /etc/yum.repos.d/ RUN dnf install -y epel-release \ - && dnf install -y python3 python3-requests python3-pycdlib aims2client + && dnf install -y python3 python3-requests python3-pycdlib aims2client python3-requests-kerberos python3-openstackclient python3-glanceclient jq curl \ + && yum install -y libguestfs-tools-c COPY *py *sh /root/ +ENV OS_AUTH_URL=https://keystone.cern.ch/v3 +ENV OS_PROJECT_DOMAIN_ID=default +ENV OS_IDENTITY_PROVIDER=sssd +ENV OS_PROTOCOL=kerberos +ENV OS_MUTUAL_AUTH=disabled +ENV OS_USER_DOMAIN_NAME=default +ENV PROJECT_PHYSICAL="IT Linux Support - CI Physical" +ENV PROJECT_PHYSICAL_ID="c18a256b-ee55-4c81-b52b-47684c19f2fe" + CMD ["python3", "-u", "/root/download_rhel_iso.py"] diff --git a/download_rhel_iso/convertimage.sh b/download_rhel_iso/convertimage.sh new file mode 100755 index 0000000..70337ec --- /dev/null +++ b/download_rhel_iso/convertimage.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +# This script is used to apply neccessary fixes to upstream +# RedHat images, for use at CERN + +function usage { + echo "`basename $0` image" +} + +[ -z $1 ] && usage && exit 1 +IMGFILE=$1 +echo "Converting $IMGFILE ..." +IMGFILEOUT=${IMGFILE/qcow2/raw} + +TMPIMAGE=$(mktemp -d) +LIBGUESTFS_BACKEND=direct virt-copy-out -a $IMGFILE /var/lib/rpm $TMPIMAGE +IMAGE_VER=$(rpm -q krb5-libs --qf="%{version}-%{release}" --dbpath $TMPIMAGE/rpm) +if [[ $IMAGE_VER == *"el7"* ]]; then + OS=7 +elif [[ $IMAGE_VER == *"el8"* ]]; then + OS=8 +elif [[ $IMAGE_VER == *"el9"* ]]; then + OS=9 +else + echo "Unsupported OS release ($IMAGE_VER)" + exit 1 +fi +echo "Image is for RHEL ${OS}." + +# Apparently this is no longer needed? +# echo "Checking if bootstrap repo needs updating" +# REPO="http://linuxsoft.cern.ch/internal/bootstrap/rhel${OS}/x86_64" +# REPO_VER=$(repoquery --repoid bootstrap --repofrompath=bootstrap,$REPO --qf="%{version}-%{release}" --latest-limit 1 krb5-libs) +# if [ "$IMAGE_VER" != "$REPO_VER" ]; then +# echo "Error: Bootstrap repo needs updating before continuing." +# echo "Without proceeding with this step, puppet managed RHEL hosts will be unable to bootstrap" +# echo "The image contains krb5-libs version $IMAGE_VER, however the the packages krb5-workstation, krb5-libs and libkadm5 exist in $REPO with version $REPO_VER. These packages need to be updated to $IMAGE_VER" +# exit 1 +# fi + +echo "Ensuring that root can ssh (in lieu of 'cloud-user')" +LIBGUESTFS_BACKEND=direct virt-copy-out -a $IMGFILE /etc/cloud/cloud.cfg $TMPIMAGE +sed -i 's|^disable_root: 1|disable_root: 0|' $TMPIMAGE/cloud.cfg +sed -i 's|name: cloud-user|name: root|' $TMPIMAGE/cloud.cfg +sed -i '/gecos:/d' $TMPIMAGE/cloud.cfg $TMPIMAGE/cloud.cfg +sed -i '/groups:/d' $TMPIMAGE/cloud.cfg $TMPIMAGE/cloud.cfg +sed -i '/sudo:/d' $TMPIMAGE/cloud.cfg $TMPIMAGE/cloud.cfg +LIBGUESTFS_BACKEND=direct virt-copy-in -a $IMGFILE $TMPIMAGE/cloud.cfg /etc/cloud + +echo "Defining .repo configuration for linuxsoft mirror" +REPOFILE="https://linux.web.cern.ch/rhel/repofiles/rhel${OS}.repo" +if [[ $OS -eq 7 ]]; then + CERNURL="http://linuxsoft.cern.ch/cern/centos/${OS}/cern/x86_64/" +else + CERNURL="http://linuxsoft.cern.ch/cern/centos/s${OS}/CERN/x86_64/" +fi +curl -o $TMPIMAGE/rhel.repo $REPOFILE +cat > $TMPIMAGE/CERN.repo < 0: + qemu_handling(qemus_of_interest, auth_token, success_qemus, failed_qemus, flt_os_imgs_names) + if len(isos_of_interest) > 0: + iso_handling(isos_of_interest, auth_token, success_isos, failed_isos) + + ## Download is completed for all images. Procceding with the rest of the tasks + email_managing(failed_csets, success_isos, failed_isos) + email_managing(failed_csets, success_qemus, failed_qemus) - else: - arch_rel_dir = os.path.join(major_rel_dir, item_release, arch) - downl_status = download_iso(auth_token, item, arch_rel_dir) - if downl_status[0]: - if extract_iso(downl_status[1], arch_rel_dir): - if arch == "x86_64" or arch == "aarch64": - rhel_iso_to_aims(item) - else: - print("ppc64le architecture is not allowed in AIMS!") - success_isos.append(item) - else: - failed_isos[item['filename']] = "Extraction error." - else: - failed_isos[item['filename']] = downl_status[1] + sys.exit(0) - if len(success_isos) == 0: - if len(failed_csets) == 0 and len(failed_isos) == 0: + +""" + email_managing() + ARG: ??? + RETURN: ??? +""" +def email_managing(failed_csets, success_imgs, failed_imgs): + + if len(success_imgs) == 0: + if len(failed_csets) == 0 and len(failed_imgs) == 0: dt = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") print("download_rhel_iso.py ran succesfully on %s, but no new RHEL images found." % dt) else: - fail_email(failed_csets, failed_isos) + fail_email(failed_csets, failed_imgs) else: - success_email(success_isos) - if len(failed_csets) != 0 or len(failed_isos) != 0: - fail_email(failed_csets, failed_isos) - - sys.exit(0) + success_email(success_imgs) + if len(failed_csets) != 0 or len(failed_imgs) != 0: + fail_email(failed_csets, failed_imgs) """ - get_auth() get authorization based on OFFLINE_TOKEN - It generates an access token from the 'offline_token' - ARG: offline_token - RETURN: token + qemu_handling() gets all qemu images of interest based on what is available from Red Hat, + compares them with what is available at OpenStack in CERN and initiates all processes + for ???, ???. It returns lists/dicts of succesfull downloads and failed downloads. + ARG: a list of available qemus from Red Hat that we are interested in (e.g. qcow2 images) + RETURN: a tuple of (success_qemus, failed_qemus) """ -def get_auth(offline_token): +def qemu_handling(qemus_of_interest, auth_token, success_qemus, failed_qemus, qemus_in_os): + + ## Pre-Stage: Find the latest available upstream qemu + latest_qemu_release = '6.0' + latest_qemu = {} + for qemu in qemus_of_interest: + qemu_release = re.findall("\d+\.\d+", qemu['filename'])[0] + if qemu_release > latest_qemu_release: + latest_qemu_release = qemu_release + latest_qemu = qemu + major = latest_qemu_release.split(".")[0] + operatingsystem = "RHEL" + major + print("latest_qemu\t" + str(latest_qemu)) + release_date = datetime.datetime.strptime(latest_qemu['datePublished'].split('T')[0], '%Y-%m-%d') + + # Check if the latest available upstream qemu exists in OpenStack + for qemu_in_os in qemus_in_os: + print("Comparing " + operatingsystem + " released on " + str(release_date.strftime('%Y-%m-%d')) + "\tVS\t" + qemu_in_os) + if operatingsystem in qemu_in_os and release_date.strftime('%Y-%m-%d') in qemu_in_os: + print('A production %s image with the release date of %s already exists in OpenStack.' % (operatingsystem, release_date.strftime('%Y-%m-%d'))) + #failed_qemus[latest_qemu['filename']] = "Fail simulation" + return + + ## Download stage (aka download_upstream_rhelX) + print("\tDownload stage") + downl_dir = os.path.join(os.getcwd(), operatingsystem) + downl_status = download_rhel_img(auth_token, latest_qemu, downl_dir) + + ## Build stage (aka build_rhelX) + if downl_status[0]: + date_on_filename = release_date.strftime('%Y%m%d') + print("\tBuild stage.\tcurrent_fp:\t" + downl_status[1]) + raw_fp = exec_build_stage(latest_qemu_release, date_on_filename, downl_status[1]) + else: + failed_qemus[latest_qemu['filename']] = downl_status[1] + return + + ## Upload stage (aka upload_rhelX_test) + print("\tUpload stage.\tRaw file is:\t" + raw_fp) + upload_status = exec_upload_stage(operatingsystem, "test", date_on_filename, raw_fp) + + # Variable(s) for next stages + if upload_status[0]: + upload_json_fp = upload_status[1] + #print("upload.json is:\t" + upload_json_fp) + upload_json_dict = {} + with open(upload_json_fp) as upload_json: + upload_json_dict = json.load(upload_json) + # print("upload_json_dict:\t" + str(upload_json_dict)) + # print("exec_share_stage argument:\t" + str(upload_json_dict['id'])) + else: + failed_qemus[latest_qemu['filename']] = upload_status[1] + return - url = 'https://sso.redhat.com/auth/realms/redhat-external/protocol/openid-connect/token' - data = { 'grant_type': 'refresh_token', 'client_id': 'rhsm-api', 'refresh_token': offline_token } + ## Share_image stage (aka share_rhelX_test) + print("\tShare_image stage.") + share_status = exec_share_stage(upload_json_dict['id']) - retry = 0 - max_retry = 3 - while retry < max_retry: - attempts = retry + 1 - result = requests.post(url, data=data) - if result.status_code == 200: - auth_token = json.loads(result.content)['access_token'] - print("Succesfull authorization after %s attempt(s)." % attempts) - break - else: - retry += 1 - sleep = 5 - print("Auth failed, sleeping for %s sec and then retry." % sleep) - time.sleep(sleep) + ## Tests stage (aka share_rhelX_test) + if share_status[0]: + print("\tTest stage") + tests_status = exec_tests_stage(upload_json_dict) + else: + failed_qemus[latest_qemu['filename']] = share_status[1] + return + + ## Prod stage (aka share_rhelX_test) + if tests_status[0]: + print("\tProd stage") + prod_status = exec_prod_stage(upload_json_dict) + else: + failed_qemus[latest_qemu['filename']] = tests_status[1] - if retry == max_retry: - subject = "Auth error" - body = "ERROR: Unable to auth after %s attempts, exiting" % attempts - send_email(ADMIN_EMAIL, subject, body) - print(body) - sys.exit(0) + if prod_status[0]: + success_qemus.append(latest_qemu) + print("The following qemu image was handled succesfully!\n" + str(latest_qemu) + "\n") + else: + failed_qemus[latest_qemu['filename']] = tests_status[1] + return - return auth_token + +""" + iso_handling() gets all isos of interest based on what is available from Red Hat, + compares them with what is available at Ceph FS in CERN and initiates all processes + for download, extraction, adding to AIMS. It returns lists and dicts of + succesfull downloads and failed downloads. + ARG: a list of available isos from Red Hat that we are interested in (e.g. dvd.iso images) + RETURN: a tuple of (success_isos, failed_isos) +""" +def iso_handling(isos_of_interest, auth_token, success_isos, failed_isos): + + # Initiates all the processes (download, extract, put-to-AIMS) if an iso of interest doesn't exist in CephFS + def iso_managing(auth_token, item, arch_rel_dir): + + downl_status = download_rhel_img(auth_token, item, arch_rel_dir) + if downl_status[0]: + if extract_iso(downl_status[1], arch_rel_dir): + if arch == "x86_64" or arch == "aarch64": + rhel_iso_to_aims(item) + else: + print("ppc64le architecture is not allowed in aims!") + success_isos.append(item) + else: + failed_isos[item['filename']] = "No extraction." + else: + failed_isos[item['filename']] = downl_status[1] + + + major = 0 + arch = "" + for item in isos_of_interest: + item_release = re.findall("\d+\.\d+", item['filename'])[0] + major = item_release.split(".")[0] + arch = item['arch'] + major_rel_dir = os.path.join("/rhel", major) + if os.path.exists(major_rel_dir): + minor_rel_dir = os.path.join(major_rel_dir, item_release) + if os.path.exists(minor_rel_dir): + arch_rel_dir = os.path.join(minor_rel_dir, arch) + if os.path.exists(arch_rel_dir): + print("%s is already downloaded." % item['filename']) + else: + iso_managing(auth_token, item, arch_rel_dir) + else: + arch_rel_dir = os.path.join(minor_rel_dir, arch) + iso_managing(auth_token, item, arch_rel_dir) + else: + arch_rel_dir = os.path.join(major_rel_dir, item_release, arch) + iso_managing(auth_token, item, arch_rel_dir) """ - download_iso() downloads an iso file from Red Hat + download_rhel_img() downloads an image from Red Hat (e.g. iso, qcow) and verifies it through checksum. ARG: the whole dictionary with info of the iso to be downloaded, auth token and path for downloading RETURN: tuple = (Bool: True for success, False for fail, String: ISO filepath for success, Fail report for fail) """ -def download_iso(auth_token, iso_dict, downl_dirpath): +def download_rhel_img(auth_token, iso_dict, downl_dirpath): filename = iso_dict['filename'] release_date = datetime.datetime.strptime(iso_dict['datePublished'].split('T')[0], '%Y-%m-%d') url = iso_dict['downloadHref'] local_auth_token = auth_token - ## Download iso request + ## Download img request retry = 0 max_retry = 3 fail_reason = {} @@ -223,12 +340,12 @@ def download_iso(auth_token, iso_dict, downl_dirpath): print("ERROR: %s, on attempt no. %s to download %s, which is the last attempt. Procceding to the next image." % (fail_reason[attempts], attempts, filename)) return (False, str(fail_reason)) else: - print("ERROR: %s, on attempt no. %s to download %s. Sleep for 5 sec and then retry after new auth request." % (fail_reason[attempts], attempts, filename)) + print("ERROR: %s, on attempt no. %s to download %s. Sleep for 5 sec and then retry with new auth." % (fail_reason[attempts], attempts, filename)) retry += 1 time.sleep(5) local_auth_token = get_auth(OFFLINE_TOKEN) - ## Downloading iso + ## Downloading img os.makedirs(downl_dirpath) retry = 0 max_retry = 3 @@ -244,7 +361,7 @@ def download_iso(auth_token, iso_dict, downl_dirpath): f.write(chunk) except exceptions.StreamConsumedError: f.close() - fail_reason[attempts] = "StreamConsumedError" + fail_reason[attempts] = "StreamConsumedError error" end = time.time() downl_time = round(((end - start)/60), 2) print("Downloading procedure for %s lasted %s min." % (filename, downl_time)) @@ -359,6 +476,222 @@ def extract_iso(iso_fp, extract_dir): return verify +""" + rhel_iso_to_aims() runs a shell script through subprocess. + The shell script is running aims2client command to add the freshly downloaded image in AIMS. + ARGS: The dictionary of the iso that defines the downloaded image. + RETURN: - +""" +def rhel_iso_to_aims(iso_dict): + + arch = iso_dict['arch'] + + item_release = re.findall("\d+\.\d+", iso_dict['filename'])[0] + name = "RHEL_" + item_release.replace(".", "_") + "_" + arch.upper() + + major = item_release.split(".")[0] + minor = item_release.split(".")[1] + description = "RHEL " + major + " SERVER UPDATE " + minor + " FOR " + arch.upper() + + pxe_path = os.path.join("/rhel", major, item_release, arch, "images/pxeboot") + + nomad_task = os.getenv('NOMAD_TASK_NAME') + project_status = nomad_task.split("_")[0] + if project_status == "prod": + aims_dest = "" + else: + aims_dest = "--testserver" + + result = subprocess.run(["/root/rheliso_to_aims.sh", name, arch, description, pxe_path, aims_dest], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + if result.returncode == 0: + print("%s added in AIMS as %s" % (iso_dict['filename'], name)) + else: + subject = "Shell script to AIMS error" + body = "ERROR: rheliso_to_aims.sh did NOT run succesfully due to:\n%s" % result.stderr + send_email(ADMIN_EMAIL, subject, body) + print(body) + sys.exit(0) + + +""" + exec_build_stage() runs convertimage.sh through subprocess. + This script is used to apply neccessary fixes to upstream RedHat images, for use at CERN + ARGS: The dictionary of the iso that defines the downloaded image. + RETURN: - +""" +def exec_build_stage(latest_qemu_release, date_on_filename, current_fp): + + revision = "1" + new_filename = 'rhel%s-cloud-%s-%s.x86_64.qcow2' % (latest_qemu_release, date_on_filename, revision) + #print("new_filename:\t" + new_filename) + new_fp = os.path.join(os.path.dirname(current_fp), new_filename) + #print("new_fp:\t" + new_fp) + #print('Renaming %s to %s ...' % (current_fp, new_fp)) + os.rename(current_fp, new_fp) + + result = subprocess.run(["/root/convertimage.sh", new_fp], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + if result.returncode == 0: + #print("Succesfull rebuild") + #print("This is the shell script's stdout:\n" + str(result.stdout)) + raw_fp = new_fp.replace("qcow2", "raw") + if os.path.exists(raw_fp): + #print("Raw file exists. Procceding to the next stage.") + return raw_fp + else: + subject = "Rebuild error" + body = "ERROR: convertimage.sh did NOT run succesfully due to:\n%s" % result.stderr + send_email(ADMIN_EMAIL, subject, body) + print(body) + sys.exit(0) + + +""" + exec_upload_stage() runs upload2openstack.sh through subprocess. + All the arguments needed for the script, are given as arguments to the Python function. + ARGS: The dictionary of the iso that defines the downloaded image. + RETURN: - +""" +def exec_upload_stage(op_sys, img_type, date_on_filename, raw_fp): + + # print("Arguments to run upload2openstack.sh.") + # print("op_sys =\t\t" + op_sys) + # print("img_type =\t\t" + img_type) + # print("date_on_filename =\t" + date_on_filename) + revision = "1" + #print("revision =\t\t" + revision) + cwd = os.path.dirname(raw_fp) + #print("cwd =\t\t\t" + cwd) + result = subprocess.run(["/root/upload2openstack.sh", op_sys, img_type, date_on_filename, revision, cwd], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + if result.returncode == 0: + print("Succesfull upload") + #print("This is the shell script's stdout:\n" + str(result.stdout)) + upload_json_fp = os.path.join(cwd, "upload.json") + if os.path.exists(upload_json_fp): + #print("upload.json file exists. Procceding to the next stage.") + return (True, upload_json_fp) + else: + upload_fail_report = "ERROR: upload2openstack.sh did NOT run succesfully due to:\n%s" % result.stderr + return (False, upload_fail_report) + + + +""" + exec_share_stage() runs share_test_img.sh through subprocess. + This script is used to apply neccessary fixes to upstream RedHat images, for use at CERN + ARGS: The dictionary of the iso that defines the downloaded image. + RETURN: - +""" +def exec_share_stage(os_img_id): + + #print("SHARE TEST IMAGE WITH ID = " + os_img_id) + result = subprocess.run(["/root/share_test_img.sh", os_img_id], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + if result.returncode == 0: + print("Succesfull share") + #print("This is the shell script's stdout:\n" + str(result.stdout)) + return (True, "Dummy string") + else: + share_fail_report = "ERROR: share_test_img.sh did NOT run succesfully due to:\n%s" % result.stderr + return (False, share_fail_report) + + +""" + exec_tests_stage() runs os_auth.sh through subprocess. + ARGS: The dictionary of the iso that defines the downloaded image. + RETURN: - +""" +def exec_tests_stage(upload_json_dict): + + print("os_img_id =\t" + upload_json_dict['id']) + trigger_job = "TEST_OSRH" + upload_json_dict["properties"]["os_distro_major"] + print("trigger_job =\t" + trigger_job) + result = subprocess.run(["/root/run_test_os_img.sh", upload_json_dict['id'], trigger_job], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + if result.returncode == 0: + print("\tShell script ran succesfully. For success/fail check its output!") + print("This is the shell script's stdout:\n" + str(result.stdout)) + return (True, "Dummy string") + else: + tests_fail_report = "ERROR: run_test_os_img.sh did NOT run succesfully due to:\n%s" % result.stderr + return (False, tests_fail_report) + + +""" + exec_prod_stage() runs os_auth.sh through subprocess. + ARGS: The dictionary of the iso that defines the downloaded image. + RETURN: - +""" +def exec_prod_stage(upload_json_dict): + + # print("os_img_id =\t" + upload_json_dict["id"]) + # print("os_img_name =\t" + upload_json_dict["name"]) + # print("os_img_distro =\t" + upload_json_dict["properties"]["os_distro"]) + # print("os_img_distro_major =\t" + upload_json_dict["properties"]["os_distro_major"]) + # print("os_img_arch =\t" + upload_json_dict["properties"]["architecture"]) + result = subprocess.run(["/root/make_qemu_prod.sh", upload_json_dict["id"], upload_json_dict["name"], upload_json_dict["properties"]["os_distro"], upload_json_dict["properties"]["os_distro_major"], upload_json_dict["properties"]["architecture"]], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + if result.returncode == 0: + print("\tOS image is succesfully on prod!") + #print("This is the shell script's stdout:\n" + str(result.stdout)) + return (True, "Dummy string") + else: + prod_fail_report = "ERROR: make_qemu_prod.sh did NOT run succesfully due to:\n%s" % result.stderr + return (False, prod_fail_report) + + +""" + get_auth() gets authorization based on OFFLINE_TOKEN + It generates an access token from the 'offline_token' + ARG: offline_token + RETURN: token +""" +def get_auth(offline_token): + + url = 'https://sso.redhat.com/auth/realms/redhat-external/protocol/openid-connect/token' + data = { 'grant_type': 'refresh_token', 'client_id': 'rhsm-api', 'refresh_token': offline_token } + + retry = 0 + max_retry = 3 + while retry < max_retry: + attempts = retry + 1 + result = requests.post(url, data=data) + if result.status_code == 200: + auth_token = json.loads(result.content)['access_token'] + print("Succesfull RHEL authorization after %s attempt(s)." % attempts) + break + else: + retry += 1 + sleep = 5 + print("Auth failed, sleeping for %s sec." % sleep) + time.sleep(sleep) + print("Retry auth...") + + if retry == max_retry: + subject = "Auth error" + body = "ERROR: Unable to auth after %s attempts, exiting" % attempts + send_email(ADMIN_EMAIL, subject, body) + print(body) + sys.exit(0) + + return auth_token + + +""" + os_auth() runs os_auth.sh through subprocess. + ARGS: The dictionary of the iso that defines the downloaded image. + RETURN: - +""" +def os_auth(): + + result = subprocess.run(["/root/os_auth.sh"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + if result.returncode == 0: + print("Succesfull OS authorization") + #print("This is the shell script's stdout:\n" + str(result.stdout)) + else: + subject = "AUTH to OS error" + body = "ERROR: os_auth.sh did NOT run succesfully due to:\n%s" % result.stderr + send_email(ADMIN_EMAIL, subject, body) + print(body) + sys.exit(0) + + """ send_email() sends email(s) ARGS: receiver of the email @@ -386,23 +719,21 @@ def send_email(email_to, subject, body, email_from='linux.support@cern.ch'): """ - fail_email() sends email to the ADMIN to inform what CSET content and/or what isos could not be downloaded - ARGS: dictionary of the release as returned by CSET + fail_email() sends email to the ADMIN to inform what cset content and/or what isos could not be downloaded + ARGS: dictionary of the release as returned by cset RETURN: - """ def fail_email(failed_csets, failed_isos): - #body = "download_rhel_iso ran and the following error(s) happened:\n" - if len(failed_csets) > 0: - subject1 = "RHEL iso image: error on CSET content request" + subject1 = "CSET(s) error " body1 = "ERROR: Check if the following CSET(s) have a problem as the(ir) content was not retrieved:\n" for fcset in failed_csets: body1 += fcset + "\n" print(body1) send_email(ADMIN_EMAIL, subject1, body1) if len(failed_isos) > 0: - subject2 = "RHEL iso image: error while downloading image" + subject2 = "Download(s) error" body2 = "ERROR: Downloading the following image(s) was unsuccesful after all possible attempts:\n" for fiso in failed_isos: body2 += fiso + "\t" + failed_isos[fiso] + "\n" body2 += "\nWARNING: Failed download attempts may create directories in CEPH FS with unusable RHEL iso content.\n" @@ -414,7 +745,7 @@ def fail_email(failed_csets, failed_isos): """ success_email() sends email to the USER to inform that a new RHEL release is available at CERN. - ARGS: dictionary of the release as returned by CSET + ARGS: dictionary of the release as returned by cset RETURN: - """ def success_email(success_isos): @@ -454,7 +785,7 @@ def success_email(success_isos): subject = "RHEL %s %s is now available" % (release, archs_to_email) - body = "Dear Red Hat Linux users,\n\n" + body = "Hello %s,\n\n" % USER_EMAIL lxsoft_release_path = os.path.join(lxsoft_parent_path, major, item_release) body += "Today RHEL %s %s was released and is now available for use at CERN: %s .\n" % (release, archs_to_email, lxsoft_release_path) if len(aims_names) != 0: @@ -463,47 +794,10 @@ def success_email(success_isos): body += an + "\n" body += "\nAdditional information about this release can be found from the release notes which are also mirrored at " body += "https://linux.web.cern.ch/rhel/#red-hat-enterprise-linux-server-rhel\n\n" - body += "---\nBest regards,\nCERN Linux Droid\n(on behalf of the friendly humans of Linux Support)\n" + body += "Regards,\nCERN Linux team" send_email(USER_EMAIL, subject, body) -""" - rhel_iso_to_aims() runs a shell script through subprocess. - The shell script is running aims2client command to add the freshly downloaded image in AIMS. - ARGS: The dictionary of the iso that defines the downloaded image. - RETURN: - -""" -def rhel_iso_to_aims(iso_dict): - - arch = iso_dict['arch'] - - item_release = re.findall("\d+\.\d+", iso_dict['filename'])[0] - name = "RHEL_" + item_release.replace(".", "_") + "_" + arch.upper() - - major = item_release.split(".")[0] - minor = item_release.split(".")[1] - description = "RHEL " + major + " SERVER UPDATE " + minor + " FOR " + arch.upper() - - pxe_path = os.path.join("/rhel", major, item_release, arch, "images/pxeboot") - - nomad_task = os.getenv('NOMAD_TASK_NAME') - project_status = nomad_task.split("_")[0] - if project_status == "prod": - aims_dest = "" - else: - aims_dest = "--testserver" - - result = subprocess.run(["/root/rheliso_to_aims.sh", name, arch, description, pxe_path, aims_dest], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) - if result.returncode == 0: - print("%s added in AIMS as %s" % (iso_dict['filename'], name)) - else: - subject = "Shell script to AIMS error" - body = "ERROR: rheliso_to_aims.sh did NOT run succesfully due to:\n%s" % result.stderr - send_email(ADMIN_EMAIL, subject, body) - print(body) - sys.exit(0) - - - if __name__ == '__main__': - main() + time.sleep(36000) + #main() diff --git a/download_rhel_iso/make_qemu_prod.sh b/download_rhel_iso/make_qemu_prod.sh new file mode 100755 index 0000000..249bc2a --- /dev/null +++ b/download_rhel_iso/make_qemu_prod.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# export id=`jq -r '.id' upload.json` +# export name=`jq -r '.name' upload.json` +# export os_distro=`jq -r '.properties.os_distro' upload.json` +# export os_distro_major=`jq -r '.properties.os_distro_major' upload.json` +# export architecture=`jq -r '.properties.architecture' upload.json` + +OS_IMG_ID=$1 +OS_IMG_NAME=$2 +OS_IMG_DISTRO=$3 +OS_IMG_DISTRO_MAJOR=$4 +OS_IMG_ARCH=$5 + +echo $OS_IMG_ID +echo $OS_IMG_NAME +echo $OS_IMG_DISTRO +echo $OS_IMG_DISTRO_MAJOR +echo $OS_IMG_ARCH + +openstack image set ${OS_IMG_ID} --name "${OS_IMG_NAME/TEST /}" --property custom_name="${OS_IMG_NAME/TEST /}" --property os_edition="Base" +echo "This echo command is just a workaround. If you delete it, the next command will fail !!!" +openstack image set ${OS_IMG_ID} --community diff --git a/download_rhel_iso/openstack-upstream.repo b/download_rhel_iso/openstack-upstream.repo new file mode 100755 index 0000000..6376907 --- /dev/null +++ b/download_rhel_iso/openstack-upstream.repo @@ -0,0 +1,6 @@ +[centos-cloud-openstack-train] +name=Openstack RDO +baseurl=http://linuxsoft.cern.ch/cern/centos/s9/cloud/$basearch/openstackclient-xena +enabled=1 +priority=1 +gpgcheck=0 diff --git a/download_rhel_iso/os_auth.sh b/download_rhel_iso/os_auth.sh new file mode 100755 index 0000000..72ae5e4 --- /dev/null +++ b/download_rhel_iso/os_auth.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +echo $IMAGECI_PWD | kinit $IMAGECI_USER@CERN.CH +export OS_PROJECT_NAME="IT Linux Support - Test VMs" diff --git a/download_rhel_iso/run_test_os_img.sh b/download_rhel_iso/run_test_os_img.sh new file mode 100755 index 0000000..f6ee829 --- /dev/null +++ b/download_rhel_iso/run_test_os_img.sh @@ -0,0 +1,113 @@ +#!/bin/bash + +TRIGGER_REPO="linuxsupport/testing/image-ci" +LINUXCI_API_TOKEN=$(cat /secrets/linuxci_api_token) +OS_IMG_ID=$1 +echo ${OS_IMG_ID} +TRIGGER_JOB=$2 +echo ${TRIGGER_JOB} + +waitFor() { + MAX_ATTEMPTS=$1 + shift + POLLING_INTERVAL=$1 + shift + ATTEMPT_COUNTER=0 + + while [[ $ATTEMPT_COUNTER -lt $MAX_ATTEMPTS ]]; do + ATTEMPT_COUNTER=$(($ATTEMPT_COUNTER+1)) + # Run the user's command + $@ + RET=$? + + case $RET in + 0) + # 0 for success + return 0 + ;; + 2) + # 2 for canceled + return 2 + ;; + 3) + # 3 for failed + return 3 + ;; + 4) + # 4 for pending/still running + ;; + esac + + sleep $POLLING_INTERVAL + done + + echo "Max attempts reached." + # 1 for timeout + return 1 +} + +checkPipeline() { + # Use https://docs.gitlab.com/ee/api/README.html#namespaced-path-encoding + PROJECT=$1 + PROJECT_PATH_ENC=$(echo $PROJECT | sed 's#/#%2F#g' ) + PIPELINE_ID=$2 + + # Wait for pipeline to finish + POLLING_STATUS="$(curl -sL --max-time 10 --header "PRIVATE-TOKEN: ${LINUXCI_API_TOKEN}" \ + https://gitlab.cern.ch/api/v4/projects/${PROJECT_PATH_ENC}/pipelines/${PIPELINE_ID} | jq -r .status)" + + case $POLLING_STATUS in + "created"|"running"|"pending") + echo "Still running, check https://gitlab.cern.ch/${PROJECT}/pipelines/${PIPELINE_ID}" + return 4 + ;; + "success") + return 0 + ;; + "canceled") + return 2 + ;; + "failed") + return 3 + ;; + *) + echo "Unknown state: ${POLLING_STATUS}" + return 3 + ;; + esac +} + +PIPELINE_ID=$(curl -sL --request POST \ + --header "PRIVATE-TOKEN: $LINUXCI_API_TOKEN" \ + --header "Content-Type: application/json" \ + --data "{ \"ref\": \"master\", \"variables\": [ {\"key\": \"IMAGE\", \"value\": \"${OS_IMG_ID}\"}, {\"key\": \"PROJECT_VIRTUAL\", \"value\": \"${OS_PROJECT_NAME}\"}, {\"key\": \"TEST_VIRTUAL\", \"value\": \"True\"},{\"key\": \"TEST_PHYSICAL\", \"value\": \"False\"}, {\"key\": \"TEST_UNMANAGED\", \"value\": \"True\"}, {\"key\": \"TEST_PUPPET\", \"value\": \"True\"}, {\"key\": \"${TRIGGER_JOB}\", \"value\": \"True\"}, {\"key\": \"TEST_OS7\", \"value\": \"False\"}, {\"key\": \"TEST_OS8s\", \"value\": \"False\"}, {\"key\": \"TEST_OS9\", \"value\": \"False\"} ] }" \ + "https://gitlab.cern.ch/api/v4/projects/linuxsupport%2Ftesting%2Fimage-ci/pipeline" | jq -r .id) + +if [ "$PIPELINE_ID" == "null" ]; then + echo "Could not create a pipeline for https://gitlab.cern.ch/${TRIGGER_REPO} :(" + exit 0 +fi + +# 360 attempts and 60 sec poll interval equals to 6 hours. +# Usual pipelines are about 4 hours long: https://gitlab.cern.ch/linuxsupport/rpms/openafs/pipelines +# Adding 2 extra hours for possible waiting times in CI and Koji +waitFor 360 60 checkPipeline $TRIGGER_REPO $PIPELINE_ID +RETURN_CODE=$? +if [ $RETURN_CODE -ne 0 ]; then + case $RETURN_CODE in + 1) + echo "Trigger OpenAFS kmod builds on kernel updates, pipeline reached the timeout: https://gitlab.cern.ch/${TRIGGER_REPO}/pipelines/${PIPELINE_ID}" + ;; + 2) + echo "Pipeline https://gitlab.cern.ch/${TRIGGER_REPO}/pipelines/${PIPELINE_ID} was canceled." + ;; + 3) + echo "Pipeline https://gitlab.cern.ch/${TRIGGER_REPO}/pipelines/${PIPELINE_ID} was failed." + ;; + 4) + echo "Pipeline https://gitlab.cern.ch/${TRIGGER_REPO}/pipelines/${PIPELINE_ID} got an unknown status." + ;; + esac +fi + + diff --git a/download_rhel_iso/share_test_img.sh b/download_rhel_iso/share_test_img.sh new file mode 100755 index 0000000..747f69f --- /dev/null +++ b/download_rhel_iso/share_test_img.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +id=$1 + +#export id=`jq -r '.id' ${UPLOAD_JSON_FP}` +openstack image set --shared "${id}" +openstack image add project "${id}" "${PROJECT_PHYSICAL_ID}" +export OS_PROJECT_NAME="${PROJECT_PHYSICAL}" +openstack image set --accept "${id}" + diff --git a/download_rhel_iso/upload2openstack.sh b/download_rhel_iso/upload2openstack.sh new file mode 100755 index 0000000..24e1118 --- /dev/null +++ b/download_rhel_iso/upload2openstack.sh @@ -0,0 +1,196 @@ +#!/bin/bash + +#temp script: TBD better. + +cd $5 + +function usage { + echo "`basename $0` major test {date} {rel}" + echo " major = 7,8,9,rhel7,rhel8,rhel9" + echo " test = test/prod" + echo " date = YYYYMMDD" + echo " rel = 1" +} + +[ -z $1 ] && usage && exit 1 +[ -z $2 ] && usage && exit 1 + +VER=${1,,} +ARCHS="" +if [[ "${VER: -1}" == "a" ]]; then + ARCHS="aarch64" + VER="${VER:0:-1}" +fi +if [ x$2 == "xprod" ]; then + os_edition='Base' + OSEDITION="" +else + os_edition='Test' + OSEDITION="TEST" +fi +[ -z $3 ] && KOJIIMGDATE=`date "+%Y%m%d"` || KOJIIMGDATE=$3 +[ -z $4 ] && KOJIIMGREL=1 || KOJIIMGREL=$4 + +UPLDAYDATE=`echo ${KOJIIMGDATE} | cut -c 1-4 | tr -d '\n' && echo -n "-"` +UPLDAYDATE="${UPLDAYDATE}`echo ${KOJIIMGDATE} | cut -c 5-6 | tr -d '\n' && echo -n \"-\"`" +UPLDAYDATE="${UPLDAYDATE}`echo ${KOJIIMGDATE} | cut -c 7-8 | tr -d '\n'`" +FORMAT="raw" +hw_firmware_type="bios" +hw_machine_type="pc" + +case $VER in + 7) + [ x$ARCHS == "x" ] && ARCHS="x86_64" + os_distro="CC" + centos_test_cleanup="true" + IMGPREFIX="cc7-cloud" + ;; + 8) + [ x$ARCHS == "x" ] && ARCHS="x86_64" + os_distro="C" + centos_test_cleanup="true" + IMGPREFIX="c8-cloud" + ;; + 8s) + [ x$ARCHS == "x" ] && ARCHS="x86_64" + os_distro="CS" + centos_test_cleanup="true" + IMGPREFIX="cs8-cloud" + hw_firmware_type="uefi" + hw_machine_type="q35" + ;; + 9s) + [ x$ARCHS == "x" ] && ARCHS="x86_64" + os_distro="CS" + centos_test_cleanup="true" + IMGPREFIX="cs9-cloud" + hw_firmware_type="uefi" + hw_machine_type="q35" + ;; + rhel6) + [ x$ARCHS == "x" ] && ARCHS="x86_64" + os_distro="RHEL" + # PROD RHEL images must not be taken into account for cleanup, they are never public and are long-lived + if [ "${OSEDITION}" == "TEST" ]; then + centos_test_cleanup="true" + else + centos_test_cleanup="false" + fi + IMGPREFIX="rhel6-cloud" + ;; + rhel7 | rhel8 | rhel9) + [ x$ARCHS == "x" ] && ARCHS="x86_64" + os_distro="RHEL" + # PROD RHEL images must not be taken into account for cleanup, they are never public and are long-lived + if [ "${OSEDITION}" == "TEST" ]; then + centos_test_cleanup="true" + else + centos_test_cleanup="false" + fi + IMGPREFIX=`ls *raw | cut -d\- -f1,2` + ;; +esac + +[ -x /usr/bin/ai-rc ] && eval `ai-rc 'IT Linux Support - CI VMs'` + +if [[ -n "${CI_SERVER_URL}" ]]; then + KSFILE="--property ks_file=${CI_SERVER_URL}/${CI_PROJECT_PATH}/-/blob/${CI_COMMIT_SHORT_SHA}/${IMGPREFIX}.ks" +fi + +for ARCH in ${ARCHS}; do + daydate=$UPLDAYDATE + release_date="${daydate}T13:13:13" + upstream_provider="linux.support@cern.ch" + [ $ARCH == "i686" ] && FARCH="i386" || FARCH=$ARCH + img="${IMGPREFIX}-${KOJIIMGDATE}-${KOJIIMGREL}.${FARCH}.${FORMAT}" + echo "Inspecting ${img}:" + # If we have an oz log, the image was built at CERN + if [ -f oz-$ARCH.log ]; then + if grep -q "packaging:666:centos-release-" oz-$ARCH.log; then + # CC7 oz logs have a different format + version=`sed '/packaging:666:centos-release-/!d; s/.*release-\(\w-\w\).*/\1/' oz-$ARCH.log` + version="${version/-/.}" + else + version=`sed '/INFO.*Installed: centos\-\(linux\-\|stream\-\)\?release-/!d; s/.*666:\(\w\.\w\)-.*/\1/' oz-$ARCH.log` + fi + # Upstream images from Red Hat will not have a oz log file, but the file is conveniently named with this information :) + else + version=`ls *raw |cut -d\- -f1 | sed 's/rhel//'` + fi + if [[ -z "$version" ]]; then + echo "Unable to figure out the version, something is very wrong." + exit 1 + fi + os_distro_major=${version:0:1} + if [ ${#version} -gt 1 ]; then + os_distro_minor=`echo $version | cut -d. -f2` + else + os_distro_minor=0 + fi + if [ X${OSEDITION} == "X" ]; then + image_name="$os_distro$os_distro_major - ${ARCH} [$daydate]" + else + image_name="$os_distro$os_distro_major ${OSEDITION} - ${ARCH} [$daydate]" + fi + + # Show what we're working with, which might be useful for debugging + # Protip: if you need to inspect the image, you can use this command: + # LIBGUESTFS_BACKEND=direct guestfish --ro -a ${img} + # in the resulting shell, run "run" to start the VM, then "mount /dev/sda1" + # (or whatever) to mount the filesystem and then you can "cat" whatever you want. + PARTITIONS=$(LIBGUESTFS_BACKEND=direct virt-filesystems -a ${img} --long --uuid --no-title) + echo "Partition table:" + echo "Name Type VFS Label Size Parent UUID" + printf "$PARTITIONS\n" + echo "-----------------------------------------------" + + if [[ $(printf "$PARTITIONS\n" | wc -l) -eq 1 ]]; then + # If there's only one partition, it's got to be the root + rootfs_uuid=$(printf "$PARTITIONS\n" | awk '{print $7}') + else + # RHEL images use 'root', we define the label to be 'ROOT'. Use a case insensitive grep + rootfs_uuid=$(printf "$PARTITIONS\n" | grep -i ROOT | awk '{print $7}') + fi + if [[ -z "$rootfs_uuid" ]]; then + echo "Unable to find rootfs UUID, something is very wrong." + exit 1 + fi + echo "rootfs_uuid=${rootfs_uuid}" + + echo "os_distro=$os_distro" + echo "os_distro_major=$os_distro_major" + echo "os_distro_minor=$os_distro_minor" + echo "release_date=$release_date" + echo "os_edition=$os_edition" + echo "centos_test_cleanup=$centos_test_cleanup" + echo "architecture=${ARCH}" + echo "custom_name=$image_name" + echo "upstream_provider=$upstream_provider" + echo "name=$image_name" + echo "rootfs_uuid=$rootfs_uuid" + echo "hw_firmware_type=$hw_firmware_type" + echo "hw_machine_type=$hw_machine_type" + echo "ksfiel=$KSFILE" + echo "img=$img" + echo "image_name=$image_name" + openstack image create -f json --container-format bare --disk-format ${FORMAT} \ + --property os="LINUX" \ + --property hypervisor_type="qemu" \ + --property os_distro="$os_distro" \ + --property os_distro_major="$os_distro_major" \ + --property os_distro_minor="$os_distro_minor" \ + --property release_date="$release_date" \ + --property os_edition="$os_edition" \ + --property gitops="enable" \ + --property centos_test_cleanup="$centos_test_cleanup" \ + --property architecture="${ARCH}" \ + --property custom_name="$image_name" \ + --property upstream_provider="$upstream_provider" \ + --property name="$image_name" \ + --property rootfs_uuid="$rootfs_uuid" \ + --property hw_firmware_type="$hw_firmware_type" \ + --property hw_machine_type="$hw_machine_type" \ + $KSFILE \ + --file $img \ + "$image_name" | tee upload.json +done diff --git a/prod.variables.sh b/prod.variables.sh index 9ff483a..2e4b38e 100644 --- a/prod.variables.sh +++ b/prod.variables.sh @@ -2,4 +2,5 @@ SCHEDULE="0 8 * * *" ADMIN_EMAIL="lxsoft-admins@cern.ch" USER_EMAIL="linux-announce-rhel@cern.ch" RHEL_MOUNT="/mnt/data1/dist/enterprise/rhel/server" +OS_PROJECT_NAME="IT Linux Support - CI VMs" CSETS="rhel-7-server-isos, rhel-8-for-x86_64-baseos-isos, rhel-8-for-aarch64-baseos-isos, rhel-8-for-ppc64le-baseos-isos, rhel-9-for-x86_64-baseos-isos, rhel-9-for-aarch64-baseos-isos, rhel-9-for-ppc64le-baseos-isos" -- GitLab From 330cbb465b84224eeeb2698883ccf0e63053799f Mon Sep 17 00:00:00 2001 From: geargyri Date: Wed, 29 Jun 2022 12:14:16 +0200 Subject: [PATCH 02/15] Improvements on supporting qemu images - A few bugs fixed. - Verbosity is lowered. --- download_rhel_iso/download_rhel_iso.py | 106 +++++++++---------------- download_rhel_iso/make_qemu_prod.sh | 1 + 2 files changed, 39 insertions(+), 68 deletions(-) diff --git a/download_rhel_iso/download_rhel_iso.py b/download_rhel_iso/download_rhel_iso.py index 7460aa6..fe43ba4 100755 --- a/download_rhel_iso/download_rhel_iso.py +++ b/download_rhel_iso/download_rhel_iso.py @@ -52,14 +52,6 @@ def main(): os_auth() os_imgs_as_dict = [] - print("\tConnecting to OS...") - print(os.environ.get('OS_AUTH_URL')) - print(os.environ.get('OS_PROJECT_NAME')) - print(os.environ.get('OS_PROJECT_DOMAIN_ID')) - print(os.environ.get('OS_IDENTITY_PROVIDER')) - print(os.environ.get('OS_PROTOCOL')) - print(os.environ.get('OS_MUTUAL_AUTH')) - auth = MappedKerberos(auth_url=os.environ.get('OS_AUTH_URL'), project_name=os.environ.get('OS_PROJECT_NAME'), project_domain_id=os.environ.get('OS_PROJECT_DOMAIN_ID'), @@ -79,9 +71,7 @@ def main(): if "TEST" not in img["custom_name"] and ("RHEL7" in img["custom_name"] or "RHEL8" in img["custom_name"] or "RHEL9" in img["custom_name"]): flt_os_imgs_names.append(img["custom_name"]) - # print("All images are:\t" + str(len(os_imgs_as_dict))) - # for name in flt_os_imgs_names: - # print(name) + print("Succesfully connected to OS project %s. All available images are: %s" % (os.environ.get('OS_PROJECT_NAME'), str(len(os_imgs_as_dict)))) ## Getting list of available RHEL images according to cset failed_csets = [] @@ -133,16 +123,14 @@ def main(): if 'qcow2' in item['filename'] and 'x86_64' in item['filename']: qemus_of_interest.append(item) - print("The amount of qemus_of_interest from this cset is:\t" + str(len(qemus_of_interest))) - if len(qemus_of_interest) > 0: qemu_handling(qemus_of_interest, auth_token, success_qemus, failed_qemus, flt_os_imgs_names) if len(isos_of_interest) > 0: iso_handling(isos_of_interest, auth_token, success_isos, failed_isos) ## Download is completed for all images. Procceding with the rest of the tasks - email_managing(failed_csets, success_isos, failed_isos) - email_managing(failed_csets, success_qemus, failed_qemus) + email_managing(failed_csets, success_isos, failed_isos, "iso") + email_managing(failed_csets, success_qemus, failed_qemus, "qemu") sys.exit(0) @@ -152,12 +140,12 @@ def main(): ARG: ??? RETURN: ??? """ -def email_managing(failed_csets, success_imgs, failed_imgs): +def email_managing(failed_csets, success_imgs, failed_imgs, img_type): if len(success_imgs) == 0: if len(failed_csets) == 0 and len(failed_imgs) == 0: dt = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") - print("download_rhel_iso.py ran succesfully on %s, but no new RHEL images found." % dt) + print("download_rhel_iso.py ran succesfully on %s, but no new RHEL %s images found." % (dt, img_type)) else: fail_email(failed_csets, failed_imgs) else: @@ -185,7 +173,6 @@ def qemu_handling(qemus_of_interest, auth_token, success_qemus, failed_qemus, qe latest_qemu = qemu major = latest_qemu_release.split(".")[0] operatingsystem = "RHEL" + major - print("latest_qemu\t" + str(latest_qemu)) release_date = datetime.datetime.strptime(latest_qemu['datePublished'].split('T')[0], '%Y-%m-%d') # Check if the latest available upstream qemu exists in OpenStack @@ -193,7 +180,6 @@ def qemu_handling(qemus_of_interest, auth_token, success_qemus, failed_qemus, qe print("Comparing " + operatingsystem + " released on " + str(release_date.strftime('%Y-%m-%d')) + "\tVS\t" + qemu_in_os) if operatingsystem in qemu_in_os and release_date.strftime('%Y-%m-%d') in qemu_in_os: print('A production %s image with the release date of %s already exists in OpenStack.' % (operatingsystem, release_date.strftime('%Y-%m-%d'))) - #failed_qemus[latest_qemu['filename']] = "Fail simulation" return ## Download stage (aka download_upstream_rhelX) @@ -205,24 +191,24 @@ def qemu_handling(qemus_of_interest, auth_token, success_qemus, failed_qemus, qe if downl_status[0]: date_on_filename = release_date.strftime('%Y%m%d') print("\tBuild stage.\tcurrent_fp:\t" + downl_status[1]) - raw_fp = exec_build_stage(latest_qemu_release, date_on_filename, downl_status[1]) + build_status = exec_build_stage(latest_qemu_release, date_on_filename, downl_status[1]) else: failed_qemus[latest_qemu['filename']] = downl_status[1] return ## Upload stage (aka upload_rhelX_test) - print("\tUpload stage.\tRaw file is:\t" + raw_fp) - upload_status = exec_upload_stage(operatingsystem, "test", date_on_filename, raw_fp) + if build_status[0]: + print("\tUpload stage.\tRaw file is:\t" + build_status[1]) + upload_status = exec_upload_stage(operatingsystem, "test", date_on_filename, build_status[1]) + else: + failed_qemus[latest_qemu['filename']] = build_status[1] + return # Variable(s) for next stages if upload_status[0]: - upload_json_fp = upload_status[1] - #print("upload.json is:\t" + upload_json_fp) upload_json_dict = {} - with open(upload_json_fp) as upload_json: + with open(upload_status[1]) as upload_json: upload_json_dict = json.load(upload_json) - # print("upload_json_dict:\t" + str(upload_json_dict)) - # print("exec_share_stage argument:\t" + str(upload_json_dict['id'])) else: failed_qemus[latest_qemu['filename']] = upload_status[1] return @@ -233,7 +219,7 @@ def qemu_handling(qemus_of_interest, auth_token, success_qemus, failed_qemus, qe ## Tests stage (aka share_rhelX_test) if share_status[0]: - print("\tTest stage") + print("\tTests stage") tests_status = exec_tests_stage(upload_json_dict) else: failed_qemus[latest_qemu['filename']] = share_status[1] @@ -248,9 +234,9 @@ def qemu_handling(qemus_of_interest, auth_token, success_qemus, failed_qemus, qe if prod_status[0]: success_qemus.append(latest_qemu) - print("The following qemu image was handled succesfully!\n" + str(latest_qemu) + "\n") + print("QEMU image %s was handled succesfully!" % latest_qemu['filename']) else: - failed_qemus[latest_qemu['filename']] = tests_status[1] + failed_qemus[latest_qemu['filename']] = prod_status[1] return @@ -523,26 +509,23 @@ def exec_build_stage(latest_qemu_release, date_on_filename, current_fp): revision = "1" new_filename = 'rhel%s-cloud-%s-%s.x86_64.qcow2' % (latest_qemu_release, date_on_filename, revision) - #print("new_filename:\t" + new_filename) new_fp = os.path.join(os.path.dirname(current_fp), new_filename) - #print("new_fp:\t" + new_fp) - #print('Renaming %s to %s ...' % (current_fp, new_fp)) os.rename(current_fp, new_fp) result = subprocess.run(["/root/convertimage.sh", new_fp], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) if result.returncode == 0: - #print("Succesfull rebuild") - #print("This is the shell script's stdout:\n" + str(result.stdout)) raw_fp = new_fp.replace("qcow2", "raw") if os.path.exists(raw_fp): - #print("Raw file exists. Procceding to the next stage.") - return raw_fp + print("Build stage executed succesfully. Output file is:\t%s" % raw_fp) + return (True, raw_fp) + else: + build_fail_report = "ERROR: convertimage.sh did run succesfully, BUT the produced file does NOT exist. Logs:\n" % result.stdout + print(build_fail_report) + return (False, build_fail_report) else: - subject = "Rebuild error" - body = "ERROR: convertimage.sh did NOT run succesfully due to:\n%s" % result.stderr - send_email(ADMIN_EMAIL, subject, body) - print(body) - sys.exit(0) + build_fail_report = "ERROR: convertimage.sh did NOT run succesfully due to:\n%s" % result.stderr + print(build_fail_report) + return (False, build_fail_report) """ @@ -553,28 +536,24 @@ def exec_build_stage(latest_qemu_release, date_on_filename, current_fp): """ def exec_upload_stage(op_sys, img_type, date_on_filename, raw_fp): - # print("Arguments to run upload2openstack.sh.") - # print("op_sys =\t\t" + op_sys) - # print("img_type =\t\t" + img_type) - # print("date_on_filename =\t" + date_on_filename) revision = "1" - #print("revision =\t\t" + revision) cwd = os.path.dirname(raw_fp) - #print("cwd =\t\t\t" + cwd) result = subprocess.run(["/root/upload2openstack.sh", op_sys, img_type, date_on_filename, revision, cwd], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) if result.returncode == 0: - print("Succesfull upload") - #print("This is the shell script's stdout:\n" + str(result.stdout)) upload_json_fp = os.path.join(cwd, "upload.json") if os.path.exists(upload_json_fp): - #print("upload.json file exists. Procceding to the next stage.") + print("Upload stage executed succesfully. Output file is:\t%s" % upload_json_fp) return (True, upload_json_fp) + else: + upload_fail_report = "ERROR: upload2openstack.sh did run succesfully, BUT the produced file does NOT exist. Logs:\n" % result.stdout + print(upload_fail_report) + return (False, upload_fail_report) else: upload_fail_report = "ERROR: upload2openstack.sh did NOT run succesfully due to:\n%s" % result.stderr + print(upload_fail_report) return (False, upload_fail_report) - """ exec_share_stage() runs share_test_img.sh through subprocess. This script is used to apply neccessary fixes to upstream RedHat images, for use at CERN @@ -583,14 +562,13 @@ def exec_upload_stage(op_sys, img_type, date_on_filename, raw_fp): """ def exec_share_stage(os_img_id): - #print("SHARE TEST IMAGE WITH ID = " + os_img_id) result = subprocess.run(["/root/share_test_img.sh", os_img_id], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) if result.returncode == 0: - print("Succesfull share") - #print("This is the shell script's stdout:\n" + str(result.stdout)) + print("Share stage executed succesfully.") return (True, "Dummy string") else: share_fail_report = "ERROR: share_test_img.sh did NOT run succesfully due to:\n%s" % result.stderr + print(share_fail_report) return (False, share_fail_report) @@ -601,16 +579,14 @@ def exec_share_stage(os_img_id): """ def exec_tests_stage(upload_json_dict): - print("os_img_id =\t" + upload_json_dict['id']) trigger_job = "TEST_OSRH" + upload_json_dict["properties"]["os_distro_major"] - print("trigger_job =\t" + trigger_job) result = subprocess.run(["/root/run_test_os_img.sh", upload_json_dict['id'], trigger_job], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) if result.returncode == 0: - print("\tShell script ran succesfully. For success/fail check its output!") - print("This is the shell script's stdout:\n" + str(result.stdout)) + print("Tests stage executed succesfully.") return (True, "Dummy string") else: tests_fail_report = "ERROR: run_test_os_img.sh did NOT run succesfully due to:\n%s" % result.stderr + print(tests_fail_report) return (False, tests_fail_report) @@ -621,18 +597,13 @@ def exec_tests_stage(upload_json_dict): """ def exec_prod_stage(upload_json_dict): - # print("os_img_id =\t" + upload_json_dict["id"]) - # print("os_img_name =\t" + upload_json_dict["name"]) - # print("os_img_distro =\t" + upload_json_dict["properties"]["os_distro"]) - # print("os_img_distro_major =\t" + upload_json_dict["properties"]["os_distro_major"]) - # print("os_img_arch =\t" + upload_json_dict["properties"]["architecture"]) result = subprocess.run(["/root/make_qemu_prod.sh", upload_json_dict["id"], upload_json_dict["name"], upload_json_dict["properties"]["os_distro"], upload_json_dict["properties"]["os_distro_major"], upload_json_dict["properties"]["architecture"]], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) if result.returncode == 0: - print("\tOS image is succesfully on prod!") - #print("This is the shell script's stdout:\n" + str(result.stdout)) + print("Prod stage executed succesfully.") return (True, "Dummy string") else: prod_fail_report = "ERROR: make_qemu_prod.sh did NOT run succesfully due to:\n%s" % result.stderr + print(prod_fail_report) return (False, prod_fail_report) @@ -799,5 +770,4 @@ def success_email(success_isos): if __name__ == '__main__': - time.sleep(36000) - #main() + main() diff --git a/download_rhel_iso/make_qemu_prod.sh b/download_rhel_iso/make_qemu_prod.sh index 249bc2a..a56a141 100755 --- a/download_rhel_iso/make_qemu_prod.sh +++ b/download_rhel_iso/make_qemu_prod.sh @@ -21,3 +21,4 @@ echo $OS_IMG_ARCH openstack image set ${OS_IMG_ID} --name "${OS_IMG_NAME/TEST /}" --property custom_name="${OS_IMG_NAME/TEST /}" --property os_edition="Base" echo "This echo command is just a workaround. If you delete it, the next command will fail !!!" openstack image set ${OS_IMG_ID} --community +echo "Moi aussi!" -- GitLab From 8b7fb97e6063550b42cc85dff02e3e9a44b235b2 Mon Sep 17 00:00:00 2001 From: geargyri Date: Wed, 29 Jun 2022 17:39:40 +0200 Subject: [PATCH 03/15] Emails for qemu and iso differentiated. --- download_rhel_iso.nomad | 5 +- download_rhel_iso/download_rhel_iso.py | 66 +++++++++++++++++++------- download_rhel_iso/run_test_os_img.sh | 3 +- 3 files changed, 51 insertions(+), 23 deletions(-) diff --git a/download_rhel_iso.nomad b/download_rhel_iso.nomad index 32e9679..1f23a26 100644 --- a/download_rhel_iso.nomad +++ b/download_rhel_iso.nomad @@ -21,10 +21,7 @@ job "${PREFIX}_download_rhel_iso" { tag = "${PREFIX}_download_rhel_iso" } } - volumes = [ - "$RHEL_MOUNT:/rhel", - "/etc/nomad/linuxci_api_token:/secrets/linuxci_api_token" - ] + volumes = [ "$RHEL_MOUNT:/rhel" ] } env { RHSM_OFFLINE_TOKEN = "$RHSM_OFFLINE_TOKEN" diff --git a/download_rhel_iso/download_rhel_iso.py b/download_rhel_iso/download_rhel_iso.py index fe43ba4..be9758a 100755 --- a/download_rhel_iso/download_rhel_iso.py +++ b/download_rhel_iso/download_rhel_iso.py @@ -147,11 +147,14 @@ def email_managing(failed_csets, success_imgs, failed_imgs, img_type): dt = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") print("download_rhel_iso.py ran succesfully on %s, but no new RHEL %s images found." % (dt, img_type)) else: - fail_email(failed_csets, failed_imgs) + fail_email(failed_csets, failed_imgs, img_type) else: - success_email(success_imgs) + if img_type == "iso": + success_iso_email(success_imgs) + if img_type == "qemu": + success_qemu_email(success_imgs) if len(failed_csets) != 0 or len(failed_imgs) != 0: - fail_email(failed_csets, failed_imgs) + fail_email(failed_csets, failed_imgs, img_type) """ @@ -213,17 +216,13 @@ def qemu_handling(qemus_of_interest, auth_token, success_qemus, failed_qemus, qe failed_qemus[latest_qemu['filename']] = upload_status[1] return - ## Share_image stage (aka share_rhelX_test) - print("\tShare_image stage.") - share_status = exec_share_stage(upload_json_dict['id']) + # ## Share_image stage (aka share_rhelX_test) + # print("\tShare_image stage.") + # share_status = exec_share_stage(upload_json_dict['id']) ## Tests stage (aka share_rhelX_test) - if share_status[0]: - print("\tTests stage") - tests_status = exec_tests_stage(upload_json_dict) - else: - failed_qemus[latest_qemu['filename']] = share_status[1] - return + print("\tTests stage") + tests_status = exec_tests_stage(upload_json_dict) ## Prod stage (aka share_rhelX_test) if tests_status[0]: @@ -233,6 +232,7 @@ def qemu_handling(qemus_of_interest, auth_token, success_qemus, failed_qemus, qe failed_qemus[latest_qemu['filename']] = tests_status[1] if prod_status[0]: + latest_qemu['upload_info'] = upload_json_dict success_qemus.append(latest_qemu) print("QEMU image %s was handled succesfully!" % latest_qemu['filename']) else: @@ -694,7 +694,7 @@ def send_email(email_to, subject, body, email_from='linux.support@cern.ch'): ARGS: dictionary of the release as returned by cset RETURN: - """ -def fail_email(failed_csets, failed_isos): +def fail_email(failed_csets, failed_imgs, img_type): if len(failed_csets) > 0: subject1 = "CSET(s) error " @@ -703,23 +703,32 @@ def fail_email(failed_csets, failed_isos): print(body1) send_email(ADMIN_EMAIL, subject1, body1) - if len(failed_isos) > 0: + if len(failed_imgs) > 0 and img_type == "iso": subject2 = "Download(s) error" body2 = "ERROR: Downloading the following image(s) was unsuccesful after all possible attempts:\n" - for fiso in failed_isos: body2 += fiso + "\t" + failed_isos[fiso] + "\n" + for fimg in failed_imgs: body2 += fimg + "\t" + failed_imgs[fimg] + "\n" body2 += "\nWARNING: Failed download attempts may create directories in CEPH FS with unusable RHEL iso content.\n" body2 += "ACTIONS:\n\t- Inspect these directories to verify the unsuccessful downloads." body2 += "\n\t- Delete these directories, if you want the script to try download the corresponding iso images in the next run." print(body2) send_email(ADMIN_EMAIL, subject2, body2) + if len(failed_imgs) > 0 and img_type == "qemu": + subject3 = "QEMU image(s) handling error" + body3 = "ERROR: Handling the following image(s) was unsuccesful:\n" + for fimg in failed_imgs: body3 += fimg + "\t" + failed_imgs[fimg] + "\n" + body3 += "\nWARNING: Fails during hadndling qemu image(s) may add image(s) in OpenStack project %s that are not usable.\n" % os.environ.get('OS_PROJECT_NAME') + body3 += "ACTIONS:\n\t- Inspect any unsuccesful handling and delete the possibly dangerous image(s)." + print(body3) + send_email(ADMIN_EMAIL, subject3, body3) + """ - success_email() sends email to the USER to inform that a new RHEL release is available at CERN. + success_iso_email() sends email to the USER to inform that a new RHEL release is available at CERN. ARGS: dictionary of the release as returned by cset RETURN: - """ -def success_email(success_isos): +def success_iso_email(success_isos): isos_by_release = {} @@ -754,7 +763,6 @@ def success_email(success_isos): else: aims_to_email += arch - subject = "RHEL %s %s is now available" % (release, archs_to_email) body = "Hello %s,\n\n" % USER_EMAIL lxsoft_release_path = os.path.join(lxsoft_parent_path, major, item_release) @@ -769,5 +777,27 @@ def success_email(success_isos): send_email(USER_EMAIL, subject, body) +""" + success_qemu_email() sends email to the USER to inform that a new RHEL release is available at CERN. + ARGS: dictionary of the release as returned by cset + RETURN: - +""" +def success_qemu_email(success_imgs): + + for img in success_imgs: + major = img['upload_info']['properties']['os_distro_major'] + minor = img['upload_info']['properties']['os_distro_minor'] + name = img['upload_info']['name'].replace('TEST ', '') + img_id = img['upload_info']['id'] + subject = "New RHEL image available" + body = "Dear RHEL users,\n\n" + body += "Today a new RHEL%s.%s image (%s) is available. The image uuid is: %s\n\n" % (major, minor, name, img_id) + body += "You can use this image with:\n\nopenstack server create --image %s...\n\n" % img_id + body += "or alternatively from aiadm.cern.ch with\n\nai-bs -i %s ...\n\n" % img_id + body += "Best regards,\nCERN Linux Droid\n(on behalf of the friendly humans of Linux Support)" + send_email(USER_EMAIL, subject, body) + + + if __name__ == '__main__': main() diff --git a/download_rhel_iso/run_test_os_img.sh b/download_rhel_iso/run_test_os_img.sh index f6ee829..0fdf812 100755 --- a/download_rhel_iso/run_test_os_img.sh +++ b/download_rhel_iso/run_test_os_img.sh @@ -1,7 +1,8 @@ #!/bin/bash TRIGGER_REPO="linuxsupport/testing/image-ci" -LINUXCI_API_TOKEN=$(cat /secrets/linuxci_api_token) +#LINUXCI_API_TOKEN=$(cat /secrets/linuxci_api_token) +LINUXCI_API_TOKEN=${LINUXCI_APITOKEN} OS_IMG_ID=$1 echo ${OS_IMG_ID} TRIGGER_JOB=$2 -- GitLab From 3b62cd803b13dbb1a059379f2a8d23afbd7d0789 Mon Sep 17 00:00:00 2001 From: Georgios Argyriou Date: Thu, 30 Jun 2022 12:00:54 +0200 Subject: [PATCH 04/15] Improvements on managing RHEL qemu images. --- download_rhel_iso.nomad | 0 download_rhel_iso/Dockerfile | 12 +- download_rhel_iso/download_rhel_iso.py | 273 ++++++++++---------- download_rhel_iso/linuxsupport9-stable.repo | 0 download_rhel_iso/make_qemu_prod.sh | 6 - download_rhel_iso/run_download_rhel_iso.sh | 5 + download_rhel_iso/run_test_os_img.sh | 2 - download_rhel_iso/upload2openstack.sh | 32 +-- prod.variables.sh | 0 9 files changed, 163 insertions(+), 167 deletions(-) mode change 100644 => 100755 download_rhel_iso.nomad mode change 100644 => 100755 download_rhel_iso/Dockerfile mode change 100644 => 100755 download_rhel_iso/linuxsupport9-stable.repo create mode 100755 download_rhel_iso/run_download_rhel_iso.sh mode change 100644 => 100755 prod.variables.sh diff --git a/download_rhel_iso.nomad b/download_rhel_iso.nomad old mode 100644 new mode 100755 diff --git a/download_rhel_iso/Dockerfile b/download_rhel_iso/Dockerfile old mode 100644 new mode 100755 index 82ef3d1..6d11aa8 --- a/download_rhel_iso/Dockerfile +++ b/download_rhel_iso/Dockerfile @@ -7,13 +7,5 @@ RUN dnf install -y epel-release \ && yum install -y libguestfs-tools-c COPY *py *sh /root/ -ENV OS_AUTH_URL=https://keystone.cern.ch/v3 -ENV OS_PROJECT_DOMAIN_ID=default -ENV OS_IDENTITY_PROVIDER=sssd -ENV OS_PROTOCOL=kerberos -ENV OS_MUTUAL_AUTH=disabled -ENV OS_USER_DOMAIN_NAME=default -ENV PROJECT_PHYSICAL="IT Linux Support - CI Physical" -ENV PROJECT_PHYSICAL_ID="c18a256b-ee55-4c81-b52b-47684c19f2fe" - -CMD ["python3", "-u", "/root/download_rhel_iso.py"] +#CMD ["python3", "-u", "/root/download_rhel_iso.py"] +CMD ["/root/run_download_rhel_iso.sh"] diff --git a/download_rhel_iso/download_rhel_iso.py b/download_rhel_iso/download_rhel_iso.py index be9758a..c3dc916 100755 --- a/download_rhel_iso/download_rhel_iso.py +++ b/download_rhel_iso/download_rhel_iso.py @@ -29,13 +29,19 @@ USER_EMAIL = os.getenv('USER_EMAIL') OFFLINE_TOKEN = os.getenv('RHSM_OFFLINE_TOKEN') """ - Main method for downloading and extracting RHEL iso images. + Main method for downloading and managing RHEL images. Errors are send to ADMIN_EMAIL. - Succesfull download and extraction is sent to USER_EMAIL. + Succesfull handling is sent to USER_EMAIL. """ def main(): ## Main variables + OS_AUTH_URL = "https://keystone.cern.ch/v3" + OS_PROTOCOL = "kerberos" + OS_MUTUAL_AUTH = "disabled" + OS_IDENTITY_PROVIDER = "sssd" + OS_PROJECT_DOMAIN_ID = "default" + OS_PROJECT_NAME = os.environ.get('OS_PROJECT_NAME') CSETS = os.getenv('CSETS') ## Connecting to RHEL @@ -45,19 +51,20 @@ def main(): send_email(ADMIN_EMAIL, subject, body) print(body) sys.exit(0) - auth_token = get_auth(OFFLINE_TOKEN) ## Connecting to OpenStack - os_auth() + # For connecting to OpenStack you must first get kerberos credentials (It is done through CLI) + #os_auth() os_imgs_as_dict = [] - auth = MappedKerberos(auth_url=os.environ.get('OS_AUTH_URL'), - project_name=os.environ.get('OS_PROJECT_NAME'), - project_domain_id=os.environ.get('OS_PROJECT_DOMAIN_ID'), - identity_provider=os.environ.get('OS_IDENTITY_PROVIDER'), - protocol=os.environ.get('OS_PROTOCOL'), - mutual_auth=os.environ.get('OS_MUTUAL_AUTH')) + auth = MappedKerberos(auth_url=OS_AUTH_URL, + protocol=OS_PROTOCOL, + mutual_auth=OS_MUTUAL_AUTH, + identity_provider=OS_IDENTITY_PROVIDER, + project_domain_id=OS_PROJECT_DOMAIN_ID, + project_name=OS_PROJECT_NAME + ) sess = session.Session(auth=auth) glance = Client('2', session=sess) @@ -71,7 +78,7 @@ def main(): if "TEST" not in img["custom_name"] and ("RHEL7" in img["custom_name"] or "RHEL8" in img["custom_name"] or "RHEL9" in img["custom_name"]): flt_os_imgs_names.append(img["custom_name"]) - print("Succesfully connected to OS project %s. All available images are: %s" % (os.environ.get('OS_PROJECT_NAME'), str(len(os_imgs_as_dict)))) + print("Succesfully connected to OS project '%s'. All available images are: %s" % (OS_PROJECT_NAME, str(len(os_imgs_as_dict)))) ## Getting list of available RHEL images according to cset failed_csets = [] @@ -122,49 +129,77 @@ def main(): # Getting all .qcow2 (qemu images) items if 'qcow2' in item['filename'] and 'x86_64' in item['filename']: qemus_of_interest.append(item) - - if len(qemus_of_interest) > 0: - qemu_handling(qemus_of_interest, auth_token, success_qemus, failed_qemus, flt_os_imgs_names) + if len(isos_of_interest) > 0: - iso_handling(isos_of_interest, auth_token, success_isos, failed_isos) + iso_managing(isos_of_interest, auth_token, success_isos, failed_isos) + if len(qemus_of_interest) > 0: + qemu_managing(qemus_of_interest, auth_token, success_qemus, failed_qemus, flt_os_imgs_names) ## Download is completed for all images. Procceding with the rest of the tasks - email_managing(failed_csets, success_isos, failed_isos, "iso") - email_managing(failed_csets, success_qemus, failed_qemus, "qemu") + manage_email(failed_csets, success_isos, failed_isos, "iso") + manage_email(failed_csets, success_qemus, failed_qemus, "qemu") sys.exit(0) """ - email_managing() - ARG: ??? - RETURN: ??? + iso_managing() gets all isos of interest based on what is available from Red Hat, + compares them with what is available at Ceph FS in CERN and initiates all processes + for download, extraction, adding to AIMS. It returns lists and dicts of + succesfull downloads and failed downloads. + ARG: a list of available isos from Red Hat that we are interested in (e.g. dvd.iso images) + RETURN: a tuple of (success_isos, failed_isos) """ -def email_managing(failed_csets, success_imgs, failed_imgs, img_type): +def iso_managing(isos_of_interest, auth_token, success_isos, failed_isos): - if len(success_imgs) == 0: - if len(failed_csets) == 0 and len(failed_imgs) == 0: - dt = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") - print("download_rhel_iso.py ran succesfully on %s, but no new RHEL %s images found." % (dt, img_type)) + # iso_handling() Initiates all the processes (download, extract, put-to-AIMS) if an iso of interest doesn't exist in CephFS + def iso_handling(auth_token, item, arch_rel_dir): + + downl_status = download_rhel_img(auth_token, item, arch_rel_dir) + if downl_status[0]: + if extract_iso(downl_status[1], arch_rel_dir): + if arch == "x86_64" or arch == "aarch64": + rhel_iso_to_aims(item) + else: + print("ppc64le architecture is not allowed in aims!") + success_isos.append(item) + else: + failed_isos[item['filename']] = "No extraction." else: - fail_email(failed_csets, failed_imgs, img_type) - else: - if img_type == "iso": - success_iso_email(success_imgs) - if img_type == "qemu": - success_qemu_email(success_imgs) - if len(failed_csets) != 0 or len(failed_imgs) != 0: - fail_email(failed_csets, failed_imgs, img_type) + failed_isos[item['filename']] = downl_status[1] + + + major = 0 + arch = "" + for item in isos_of_interest: + item_release = re.findall("\d+\.\d+", item['filename'])[0] + major = item_release.split(".")[0] + arch = item['arch'] + major_rel_dir = os.path.join("/rhel", major) + if os.path.exists(major_rel_dir): + minor_rel_dir = os.path.join(major_rel_dir, item_release) + if os.path.exists(minor_rel_dir): + arch_rel_dir = os.path.join(minor_rel_dir, arch) + if os.path.exists(arch_rel_dir): + print("%s is already downloaded." % item['filename']) + else: + iso_handling(auth_token, item, arch_rel_dir) + else: + arch_rel_dir = os.path.join(minor_rel_dir, arch) + iso_handling(auth_token, item, arch_rel_dir) + else: + arch_rel_dir = os.path.join(major_rel_dir, item_release, arch) + iso_handling(auth_token, item, arch_rel_dir) """ - qemu_handling() gets all qemu images of interest based on what is available from Red Hat, + qemu_managing() gets all qemu images of interest based on what is available from Red Hat, compares them with what is available at OpenStack in CERN and initiates all processes for ???, ???. It returns lists/dicts of succesfull downloads and failed downloads. ARG: a list of available qemus from Red Hat that we are interested in (e.g. qcow2 images) RETURN: a tuple of (success_qemus, failed_qemus) """ -def qemu_handling(qemus_of_interest, auth_token, success_qemus, failed_qemus, qemus_in_os): +def qemu_managing(qemus_of_interest, auth_token, success_qemus, failed_qemus, qemus_in_os): ## Pre-Stage: Find the latest available upstream qemu latest_qemu_release = '6.0' @@ -240,56 +275,6 @@ def qemu_handling(qemus_of_interest, auth_token, success_qemus, failed_qemus, qe return -""" - iso_handling() gets all isos of interest based on what is available from Red Hat, - compares them with what is available at Ceph FS in CERN and initiates all processes - for download, extraction, adding to AIMS. It returns lists and dicts of - succesfull downloads and failed downloads. - ARG: a list of available isos from Red Hat that we are interested in (e.g. dvd.iso images) - RETURN: a tuple of (success_isos, failed_isos) -""" -def iso_handling(isos_of_interest, auth_token, success_isos, failed_isos): - - # Initiates all the processes (download, extract, put-to-AIMS) if an iso of interest doesn't exist in CephFS - def iso_managing(auth_token, item, arch_rel_dir): - - downl_status = download_rhel_img(auth_token, item, arch_rel_dir) - if downl_status[0]: - if extract_iso(downl_status[1], arch_rel_dir): - if arch == "x86_64" or arch == "aarch64": - rhel_iso_to_aims(item) - else: - print("ppc64le architecture is not allowed in aims!") - success_isos.append(item) - else: - failed_isos[item['filename']] = "No extraction." - else: - failed_isos[item['filename']] = downl_status[1] - - - major = 0 - arch = "" - for item in isos_of_interest: - item_release = re.findall("\d+\.\d+", item['filename'])[0] - major = item_release.split(".")[0] - arch = item['arch'] - major_rel_dir = os.path.join("/rhel", major) - if os.path.exists(major_rel_dir): - minor_rel_dir = os.path.join(major_rel_dir, item_release) - if os.path.exists(minor_rel_dir): - arch_rel_dir = os.path.join(minor_rel_dir, arch) - if os.path.exists(arch_rel_dir): - print("%s is already downloaded." % item['filename']) - else: - iso_managing(auth_token, item, arch_rel_dir) - else: - arch_rel_dir = os.path.join(minor_rel_dir, arch) - iso_managing(auth_token, item, arch_rel_dir) - else: - arch_rel_dir = os.path.join(major_rel_dir, item_release, arch) - iso_managing(auth_token, item, arch_rel_dir) - - """ download_rhel_img() downloads an image from Red Hat (e.g. iso, qcow) and verifies it through checksum. @@ -583,7 +568,7 @@ def exec_tests_stage(upload_json_dict): result = subprocess.run(["/root/run_test_os_img.sh", upload_json_dict['id'], trigger_job], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) if result.returncode == 0: print("Tests stage executed succesfully.") - return (True, "Dummy string") + return (True, "Dummy string: run_test_os_img.sh ran succesfully.") else: tests_fail_report = "ERROR: run_test_os_img.sh did NOT run succesfully due to:\n%s" % result.stderr print(tests_fail_report) @@ -600,7 +585,7 @@ def exec_prod_stage(upload_json_dict): result = subprocess.run(["/root/make_qemu_prod.sh", upload_json_dict["id"], upload_json_dict["name"], upload_json_dict["properties"]["os_distro"], upload_json_dict["properties"]["os_distro_major"], upload_json_dict["properties"]["architecture"]], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) if result.returncode == 0: print("Prod stage executed succesfully.") - return (True, "Dummy string") + return (True, "Dummy string: make_qemu_prod.sh ran succesfully.") else: prod_fail_report = "ERROR: make_qemu_prod.sh did NOT run succesfully due to:\n%s" % result.stderr print(prod_fail_report) @@ -644,49 +629,45 @@ def get_auth(offline_token): return auth_token -""" - os_auth() runs os_auth.sh through subprocess. - ARGS: The dictionary of the iso that defines the downloaded image. - RETURN: - -""" -def os_auth(): +# """ +# os_auth() runs os_auth.sh through subprocess. +# ARGS: The dictionary of the iso that defines the downloaded image. +# RETURN: - +# """ +# def os_auth(): - result = subprocess.run(["/root/os_auth.sh"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) - if result.returncode == 0: - print("Succesfull OS authorization") - #print("This is the shell script's stdout:\n" + str(result.stdout)) - else: - subject = "AUTH to OS error" - body = "ERROR: os_auth.sh did NOT run succesfully due to:\n%s" % result.stderr - send_email(ADMIN_EMAIL, subject, body) - print(body) - sys.exit(0) +# result = subprocess.run(["/root/os_auth.sh"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) +# if result.returncode == 0: +# print("Succesfull OS authorization") +# #print("This is the shell script's stdout:\n" + str(result.stdout)) +# else: +# subject = "AUTH to OS error" +# body = "ERROR: os_auth.sh did NOT run succesfully due to:\n%s" % result.stderr +# send_email(ADMIN_EMAIL, subject, body) +# print(body) +# sys.exit(0) """ - send_email() sends email(s) - ARGS: receiver of the email - the subject of the email - the body of the email - email sender - RETURN: - + manage_email() + ARG: ??? + RETURN: ??? """ -def send_email(email_to, subject, body, email_from='linux.support@cern.ch'): +def manage_email(failed_csets, success_imgs, failed_imgs, img_type): - server = smtplib.SMTP('cernmx.cern.ch') - msg = MIMEMultipart() - msg['Subject'] = subject - msg['From'] = email_from - msg['To'] = email_to - msg.add_header('reply-to', 'noreply.Linux.Support@cern.ch') - body = MIMEText(f"{body}", _subtype='plain') - msg.attach(body) - - try: - server.sendmail(email_from, email_to, msg.as_string()) - time.sleep(2) - except: - print("failed to send email to %s, continuing..." % email_to) + if len(success_imgs) == 0: + if len(failed_csets) == 0 and len(failed_imgs) == 0: + dt = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + print("download_rhel_iso.py ran succesfully on %s, but no new RHEL %s images found." % (dt, img_type)) + else: + fail_email(failed_csets, failed_imgs, img_type) + else: + if img_type == "iso": + success_iso_email(success_imgs) + if img_type == "qemu": + success_qemu_email(success_imgs) + if len(failed_csets) != 0 or len(failed_imgs) != 0: + fail_email(failed_csets, failed_imgs, img_type) """ @@ -782,22 +763,48 @@ def success_iso_email(success_isos): ARGS: dictionary of the release as returned by cset RETURN: - """ -def success_qemu_email(success_imgs): +def success_qemu_email(success_qemus): - for img in success_imgs: - major = img['upload_info']['properties']['os_distro_major'] - minor = img['upload_info']['properties']['os_distro_minor'] - name = img['upload_info']['name'].replace('TEST ', '') - img_id = img['upload_info']['id'] + for qemu in success_qemus: + major = qemu['upload_info']['properties']['os_distro_major'] + minor = qemu['upload_info']['properties']['os_distro_minor'] + name = qemu['upload_info']['name'].replace('TEST ', '') + qemu_id = qemu['upload_info']['id'] subject = "New RHEL image available" body = "Dear RHEL users,\n\n" - body += "Today a new RHEL%s.%s image (%s) is available. The image uuid is: %s\n\n" % (major, minor, name, img_id) - body += "You can use this image with:\n\nopenstack server create --image %s...\n\n" % img_id - body += "or alternatively from aiadm.cern.ch with\n\nai-bs -i %s ...\n\n" % img_id + body += "Today a new RHEL%s.%s image (%s) is available. The image uuid is: %s\n\n" % (major, minor, name, qemu_id) + body += "You can use this image with:\n\nopenstack server create --image %s...\n\n" % qemu_id + body += "or alternatively from aiadm.cern.ch with\n\nai-bs -i %s ...\n\n" % qemu_id body += "Best regards,\nCERN Linux Droid\n(on behalf of the friendly humans of Linux Support)" send_email(USER_EMAIL, subject, body) +""" + send_email() sends email(s) + ARGS: receiver of the email + the subject of the email + the body of the email + email sender + RETURN: - +""" +def send_email(email_to, subject, body, email_from='linux.support@cern.ch'): + + server = smtplib.SMTP('cernmx.cern.ch') + msg = MIMEMultipart() + msg['Subject'] = subject + msg['From'] = email_from + msg['To'] = email_to + msg.add_header('reply-to', 'noreply.Linux.Support@cern.ch') + body = MIMEText(f"{body}", _subtype='plain') + msg.attach(body) + + try: + server.sendmail(email_from, email_to, msg.as_string()) + time.sleep(2) + except: + print("failed to send email to %s, continuing..." % email_to) + + if __name__ == '__main__': main() diff --git a/download_rhel_iso/linuxsupport9-stable.repo b/download_rhel_iso/linuxsupport9-stable.repo old mode 100644 new mode 100755 diff --git a/download_rhel_iso/make_qemu_prod.sh b/download_rhel_iso/make_qemu_prod.sh index a56a141..4809c10 100755 --- a/download_rhel_iso/make_qemu_prod.sh +++ b/download_rhel_iso/make_qemu_prod.sh @@ -12,12 +12,6 @@ OS_IMG_DISTRO=$3 OS_IMG_DISTRO_MAJOR=$4 OS_IMG_ARCH=$5 -echo $OS_IMG_ID -echo $OS_IMG_NAME -echo $OS_IMG_DISTRO -echo $OS_IMG_DISTRO_MAJOR -echo $OS_IMG_ARCH - openstack image set ${OS_IMG_ID} --name "${OS_IMG_NAME/TEST /}" --property custom_name="${OS_IMG_NAME/TEST /}" --property os_edition="Base" echo "This echo command is just a workaround. If you delete it, the next command will fail !!!" openstack image set ${OS_IMG_ID} --community diff --git a/download_rhel_iso/run_download_rhel_iso.sh b/download_rhel_iso/run_download_rhel_iso.sh new file mode 100755 index 0000000..6011dee --- /dev/null +++ b/download_rhel_iso/run_download_rhel_iso.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +echo $IMAGECI_PWD | kinit $IMAGECI_USER@CERN.CH + +python3 -u /root/download_rhel_iso.py diff --git a/download_rhel_iso/run_test_os_img.sh b/download_rhel_iso/run_test_os_img.sh index 0fdf812..c890f29 100755 --- a/download_rhel_iso/run_test_os_img.sh +++ b/download_rhel_iso/run_test_os_img.sh @@ -4,9 +4,7 @@ TRIGGER_REPO="linuxsupport/testing/image-ci" #LINUXCI_API_TOKEN=$(cat /secrets/linuxci_api_token) LINUXCI_API_TOKEN=${LINUXCI_APITOKEN} OS_IMG_ID=$1 -echo ${OS_IMG_ID} TRIGGER_JOB=$2 -echo ${TRIGGER_JOB} waitFor() { MAX_ATTEMPTS=$1 diff --git a/download_rhel_iso/upload2openstack.sh b/download_rhel_iso/upload2openstack.sh index 24e1118..912cd87 100755 --- a/download_rhel_iso/upload2openstack.sh +++ b/download_rhel_iso/upload2openstack.sh @@ -157,22 +157,22 @@ for ARCH in ${ARCHS}; do fi echo "rootfs_uuid=${rootfs_uuid}" - echo "os_distro=$os_distro" - echo "os_distro_major=$os_distro_major" - echo "os_distro_minor=$os_distro_minor" - echo "release_date=$release_date" - echo "os_edition=$os_edition" - echo "centos_test_cleanup=$centos_test_cleanup" - echo "architecture=${ARCH}" - echo "custom_name=$image_name" - echo "upstream_provider=$upstream_provider" - echo "name=$image_name" - echo "rootfs_uuid=$rootfs_uuid" - echo "hw_firmware_type=$hw_firmware_type" - echo "hw_machine_type=$hw_machine_type" - echo "ksfiel=$KSFILE" - echo "img=$img" - echo "image_name=$image_name" + # echo "os_distro=$os_distro" + # echo "os_distro_major=$os_distro_major" + # echo "os_distro_minor=$os_distro_minor" + # echo "release_date=$release_date" + # echo "os_edition=$os_edition" + # echo "centos_test_cleanup=$centos_test_cleanup" + # echo "architecture=${ARCH}" + # echo "custom_name=$image_name" + # echo "upstream_provider=$upstream_provider" + # echo "name=$image_name" + # echo "rootfs_uuid=$rootfs_uuid" + # echo "hw_firmware_type=$hw_firmware_type" + # echo "hw_machine_type=$hw_machine_type" + # echo "ksfiel=$KSFILE" + # echo "img=$img" + # echo "image_name=$image_name" openstack image create -f json --container-format bare --disk-format ${FORMAT} \ --property os="LINUX" \ --property hypervisor_type="qemu" \ diff --git a/prod.variables.sh b/prod.variables.sh old mode 100644 new mode 100755 -- GitLab From 01f0474c9c3bf0618bf981dea04b7713370aa718 Mon Sep 17 00:00:00 2001 From: geargyri Date: Thu, 30 Jun 2022 15:22:06 +0200 Subject: [PATCH 05/15] Testing mode on. --- download_rhel_iso/download_rhel_iso.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/download_rhel_iso/download_rhel_iso.py b/download_rhel_iso/download_rhel_iso.py index c3dc916..53a4d62 100755 --- a/download_rhel_iso/download_rhel_iso.py +++ b/download_rhel_iso/download_rhel_iso.py @@ -100,7 +100,7 @@ def main(): result = requests.get(url, headers=headers) if result.status_code == 200: content = json.loads(result.content) - print("Got cset content of %s succesfully after %s attempt(s)." % (cset, attempts)) + print("%s CSET content retrieved succesfully from Red Hat after %s attempt(s)." % (cset, attempts)) break else: if attempts == max_retry: @@ -807,4 +807,5 @@ def send_email(email_to, subject, body, email_from='linux.support@cern.ch'): if __name__ == '__main__': + time.sleep(36000) main() -- GitLab From fb666bf3d5bc8fffa290c292fc57c619db06446e Mon Sep 17 00:00:00 2001 From: geargyri Date: Fri, 1 Jul 2022 15:40:17 +0200 Subject: [PATCH 06/15] All updates applied. In testing mode for final test. --- download_rhel_iso/Dockerfile | 1 - download_rhel_iso/download_rhel_iso.py | 115 ++++++++----------------- download_rhel_iso/os_auth.sh | 4 - download_rhel_iso/share_test_img.sh | 10 --- download_rhel_iso/upload2openstack.sh | 16 ---- 5 files changed, 38 insertions(+), 108 deletions(-) delete mode 100755 download_rhel_iso/os_auth.sh delete mode 100755 download_rhel_iso/share_test_img.sh diff --git a/download_rhel_iso/Dockerfile b/download_rhel_iso/Dockerfile index 6d11aa8..a850ea5 100755 --- a/download_rhel_iso/Dockerfile +++ b/download_rhel_iso/Dockerfile @@ -7,5 +7,4 @@ RUN dnf install -y epel-release \ && yum install -y libguestfs-tools-c COPY *py *sh /root/ -#CMD ["python3", "-u", "/root/download_rhel_iso.py"] CMD ["/root/run_download_rhel_iso.sh"] diff --git a/download_rhel_iso/download_rhel_iso.py b/download_rhel_iso/download_rhel_iso.py index 53a4d62..2218f48 100755 --- a/download_rhel_iso/download_rhel_iso.py +++ b/download_rhel_iso/download_rhel_iso.py @@ -41,9 +41,15 @@ def main(): OS_MUTUAL_AUTH = "disabled" OS_IDENTITY_PROVIDER = "sssd" OS_PROJECT_DOMAIN_ID = "default" - OS_PROJECT_NAME = os.environ.get('OS_PROJECT_NAME') + OS_USER_DOMAIN_NAME = "default" + OS_PROJECT_NAME = os.getenv('OS_PROJECT_NAME') CSETS = os.getenv('CSETS') + ## Setting ENVs, needed by any shell script (executed through this Python script) runs command `openstack` + os.environ["OS_AUTH_URL"] = OS_AUTH_URL + os.environ["OS_PROJECT_DOMAIN_ID"] = OS_PROJECT_DOMAIN_ID + os.environ["OS_USER_DOMAIN_NAME"] = OS_USER_DOMAIN_NAME + ## Connecting to RHEL if OFFLINE_TOKEN is None: subject = "Token error" @@ -55,29 +61,23 @@ def main(): ## Connecting to OpenStack # For connecting to OpenStack you must first get kerberos credentials (It is done through CLI) - #os_auth() - os_imgs_as_dict = [] auth = MappedKerberos(auth_url=OS_AUTH_URL, protocol=OS_PROTOCOL, mutual_auth=OS_MUTUAL_AUTH, identity_provider=OS_IDENTITY_PROVIDER, project_domain_id=OS_PROJECT_DOMAIN_ID, - project_name=OS_PROJECT_NAME - ) + project_name=OS_PROJECT_NAME) sess = session.Session(auth=auth) glance = Client('2', session=sess) - os_images = glance.images.list() for img in os_images: os_imgs_as_dict.append(ast.literal_eval(str(img).strip())) - flt_os_imgs_names = [] for img in os_imgs_as_dict: if "custom_name" in img.keys(): if "TEST" not in img["custom_name"] and ("RHEL7" in img["custom_name"] or "RHEL8" in img["custom_name"] or "RHEL9" in img["custom_name"]): flt_os_imgs_names.append(img["custom_name"]) - print("Succesfully connected to OS project '%s'. All available images are: %s" % (OS_PROJECT_NAME, str(len(os_imgs_as_dict)))) ## Getting list of available RHEL images according to cset @@ -100,7 +100,7 @@ def main(): result = requests.get(url, headers=headers) if result.status_code == 200: content = json.loads(result.content) - print("%s CSET content retrieved succesfully from Red Hat after %s attempt(s)." % (cset, attempts)) + print("'%s' CSET content retrieved succesfully from Red Hat after %s attempt(s)." % (cset, attempts)) break else: if attempts == max_retry: @@ -135,9 +135,8 @@ def main(): if len(qemus_of_interest) > 0: qemu_managing(qemus_of_interest, auth_token, success_qemus, failed_qemus, flt_os_imgs_names) - ## Download is completed for all images. Procceding with the rest of the tasks - manage_email(failed_csets, success_isos, failed_isos, "iso") - manage_email(failed_csets, success_qemus, failed_qemus, "qemu") + ## Management of RHEL images is complete. Logging and sending report(s) through email(s). + manage_email(failed_csets, success_isos, failed_isos, success_qemus, failed_qemus) sys.exit(0) @@ -251,10 +250,6 @@ def qemu_managing(qemus_of_interest, auth_token, success_qemus, failed_qemus, qe failed_qemus[latest_qemu['filename']] = upload_status[1] return - # ## Share_image stage (aka share_rhelX_test) - # print("\tShare_image stage.") - # share_status = exec_share_stage(upload_json_dict['id']) - ## Tests stage (aka share_rhelX_test) print("\tTests stage") tests_status = exec_tests_stage(upload_json_dict) @@ -363,7 +358,7 @@ def download_rhel_img(auth_token, iso_dict, downl_dirpath): print("ERROR: %s, on attempt no. %s to download %s. Sleep for 5 sec and then retry." % (fail_reason[attempts], attempts, filename)) time.sleep(5) retry += 1 - + return (True, iso_fp) @@ -523,6 +518,7 @@ def exec_upload_stage(op_sys, img_type, date_on_filename, raw_fp): revision = "1" cwd = os.path.dirname(raw_fp) + result = subprocess.run(["/root/upload2openstack.sh", op_sys, img_type, date_on_filename, revision, cwd], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) if result.returncode == 0: upload_json_fp = os.path.join(cwd, "upload.json") @@ -539,24 +535,6 @@ def exec_upload_stage(op_sys, img_type, date_on_filename, raw_fp): return (False, upload_fail_report) -""" - exec_share_stage() runs share_test_img.sh through subprocess. - This script is used to apply neccessary fixes to upstream RedHat images, for use at CERN - ARGS: The dictionary of the iso that defines the downloaded image. - RETURN: - -""" -def exec_share_stage(os_img_id): - - result = subprocess.run(["/root/share_test_img.sh", os_img_id], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) - if result.returncode == 0: - print("Share stage executed succesfully.") - return (True, "Dummy string") - else: - share_fail_report = "ERROR: share_test_img.sh did NOT run succesfully due to:\n%s" % result.stderr - print(share_fail_report) - return (False, share_fail_report) - - """ exec_tests_stage() runs os_auth.sh through subprocess. ARGS: The dictionary of the iso that defines the downloaded image. @@ -565,10 +543,11 @@ def exec_share_stage(os_img_id): def exec_tests_stage(upload_json_dict): trigger_job = "TEST_OSRH" + upload_json_dict["properties"]["os_distro_major"] + result = subprocess.run(["/root/run_test_os_img.sh", upload_json_dict['id'], trigger_job], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) if result.returncode == 0: print("Tests stage executed succesfully.") - return (True, "Dummy string: run_test_os_img.sh ran succesfully.") + return (True, result.stdout) else: tests_fail_report = "ERROR: run_test_os_img.sh did NOT run succesfully due to:\n%s" % result.stderr print(tests_fail_report) @@ -585,7 +564,7 @@ def exec_prod_stage(upload_json_dict): result = subprocess.run(["/root/make_qemu_prod.sh", upload_json_dict["id"], upload_json_dict["name"], upload_json_dict["properties"]["os_distro"], upload_json_dict["properties"]["os_distro_major"], upload_json_dict["properties"]["architecture"]], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) if result.returncode == 0: print("Prod stage executed succesfully.") - return (True, "Dummy string: make_qemu_prod.sh ran succesfully.") + return (True, result.stdout) else: prod_fail_report = "ERROR: make_qemu_prod.sh did NOT run succesfully due to:\n%s" % result.stderr print(prod_fail_report) @@ -629,45 +608,27 @@ def get_auth(offline_token): return auth_token -# """ -# os_auth() runs os_auth.sh through subprocess. -# ARGS: The dictionary of the iso that defines the downloaded image. -# RETURN: - -# """ -# def os_auth(): - -# result = subprocess.run(["/root/os_auth.sh"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) -# if result.returncode == 0: -# print("Succesfull OS authorization") -# #print("This is the shell script's stdout:\n" + str(result.stdout)) -# else: -# subject = "AUTH to OS error" -# body = "ERROR: os_auth.sh did NOT run succesfully due to:\n%s" % result.stderr -# send_email(ADMIN_EMAIL, subject, body) -# print(body) -# sys.exit(0) - - """ manage_email() ARG: ??? RETURN: ??? """ -def manage_email(failed_csets, success_imgs, failed_imgs, img_type): +def manage_email(failed_csets, success_isos, failed_isos, success_qemus, failed_qemus): - if len(success_imgs) == 0: - if len(failed_csets) == 0 and len(failed_imgs) == 0: - dt = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") - print("download_rhel_iso.py ran succesfully on %s, but no new RHEL %s images found." % (dt, img_type)) - else: - fail_email(failed_csets, failed_imgs, img_type) + dt = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + if len(failed_csets) != 0 or len(failed_isos) != 0 or len(failed_qemus) != 0: + fail_email(failed_csets, failed_isos, failed_qemus) else: - if img_type == "iso": - success_iso_email(success_imgs) - if img_type == "qemu": - success_qemu_email(success_imgs) - if len(failed_csets) != 0 or len(failed_imgs) != 0: - fail_email(failed_csets, failed_imgs, img_type) + print("download_rhel_iso.py ran succesfully on %s" % dt) + if len(success_isos) == 0: + print("Νo new RHEL ISO images found.") + if len(success_qemus) == 0: + print("Νo new RHEL QEMU images found.") + if len(success_isos) > 0: + success_iso_email(success_isos) + if len(success_qemus) > 0: + success_qemu_email(success_qemus) """ @@ -675,29 +636,29 @@ def manage_email(failed_csets, success_imgs, failed_imgs, img_type): ARGS: dictionary of the release as returned by cset RETURN: - """ -def fail_email(failed_csets, failed_imgs, img_type): +def fail_email(failed_csets, failed_isos, failed_qemus): if len(failed_csets) > 0: - subject1 = "CSET(s) error " + subject1 = "RHEL images CSET(s) error" body1 = "ERROR: Check if the following CSET(s) have a problem as the(ir) content was not retrieved:\n" for fcset in failed_csets: body1 += fcset + "\n" print(body1) send_email(ADMIN_EMAIL, subject1, body1) - if len(failed_imgs) > 0 and img_type == "iso": - subject2 = "Download(s) error" + if len(failed_isos) > 0: + subject2 = "Download RHEL ISO image(s) error" body2 = "ERROR: Downloading the following image(s) was unsuccesful after all possible attempts:\n" - for fimg in failed_imgs: body2 += fimg + "\t" + failed_imgs[fimg] + "\n" + for fiso in failed_isos: body2 += fiso + "\t" + failed_isos[fiso] + "\n" body2 += "\nWARNING: Failed download attempts may create directories in CEPH FS with unusable RHEL iso content.\n" body2 += "ACTIONS:\n\t- Inspect these directories to verify the unsuccessful downloads." body2 += "\n\t- Delete these directories, if you want the script to try download the corresponding iso images in the next run." print(body2) send_email(ADMIN_EMAIL, subject2, body2) - if len(failed_imgs) > 0 and img_type == "qemu": - subject3 = "QEMU image(s) handling error" + if len(failed_qemus) > 0: + subject3 = "RHEL QEMU image(s) managing error" body3 = "ERROR: Handling the following image(s) was unsuccesful:\n" - for fimg in failed_imgs: body3 += fimg + "\t" + failed_imgs[fimg] + "\n" + for fqemu in failed_qemus: body3 += fqemu + "\t" + failed_qemus[fqemu] + "\n" body3 += "\nWARNING: Fails during hadndling qemu image(s) may add image(s) in OpenStack project %s that are not usable.\n" % os.environ.get('OS_PROJECT_NAME') body3 += "ACTIONS:\n\t- Inspect any unsuccesful handling and delete the possibly dangerous image(s)." print(body3) diff --git a/download_rhel_iso/os_auth.sh b/download_rhel_iso/os_auth.sh deleted file mode 100755 index 72ae5e4..0000000 --- a/download_rhel_iso/os_auth.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -echo $IMAGECI_PWD | kinit $IMAGECI_USER@CERN.CH -export OS_PROJECT_NAME="IT Linux Support - Test VMs" diff --git a/download_rhel_iso/share_test_img.sh b/download_rhel_iso/share_test_img.sh deleted file mode 100755 index 747f69f..0000000 --- a/download_rhel_iso/share_test_img.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -id=$1 - -#export id=`jq -r '.id' ${UPLOAD_JSON_FP}` -openstack image set --shared "${id}" -openstack image add project "${id}" "${PROJECT_PHYSICAL_ID}" -export OS_PROJECT_NAME="${PROJECT_PHYSICAL}" -openstack image set --accept "${id}" - diff --git a/download_rhel_iso/upload2openstack.sh b/download_rhel_iso/upload2openstack.sh index 912cd87..c71b200 100755 --- a/download_rhel_iso/upload2openstack.sh +++ b/download_rhel_iso/upload2openstack.sh @@ -157,22 +157,6 @@ for ARCH in ${ARCHS}; do fi echo "rootfs_uuid=${rootfs_uuid}" - # echo "os_distro=$os_distro" - # echo "os_distro_major=$os_distro_major" - # echo "os_distro_minor=$os_distro_minor" - # echo "release_date=$release_date" - # echo "os_edition=$os_edition" - # echo "centos_test_cleanup=$centos_test_cleanup" - # echo "architecture=${ARCH}" - # echo "custom_name=$image_name" - # echo "upstream_provider=$upstream_provider" - # echo "name=$image_name" - # echo "rootfs_uuid=$rootfs_uuid" - # echo "hw_firmware_type=$hw_firmware_type" - # echo "hw_machine_type=$hw_machine_type" - # echo "ksfiel=$KSFILE" - # echo "img=$img" - # echo "image_name=$image_name" openstack image create -f json --container-format bare --disk-format ${FORMAT} \ --property os="LINUX" \ --property hypervisor_type="qemu" \ -- GitLab From 1c9481ce7dbf2648a9e008b737a277c949009777 Mon Sep 17 00:00:00 2001 From: geargyri Date: Fri, 1 Jul 2022 15:58:33 +0200 Subject: [PATCH 07/15] No testing mode. --- download_rhel_iso/download_rhel_iso.py | 1 - 1 file changed, 1 deletion(-) diff --git a/download_rhel_iso/download_rhel_iso.py b/download_rhel_iso/download_rhel_iso.py index 2218f48..38f305d 100755 --- a/download_rhel_iso/download_rhel_iso.py +++ b/download_rhel_iso/download_rhel_iso.py @@ -768,5 +768,4 @@ def send_email(email_to, subject, body, email_from='linux.support@cern.ch'): if __name__ == '__main__': - time.sleep(36000) main() -- GitLab From 4cc31c20720490af7ed2f415bdbdb0cdf110ad60 Mon Sep 17 00:00:00 2001 From: geargyri Date: Wed, 6 Jul 2022 20:43:45 +0200 Subject: [PATCH 08/15] Major commit - Make_prod stage fully python-ized. Cleared from old code. - qemu_upload2openstack.sh updated on exit codes. - qemu_test_os_img.sh updated on exit codes. - The corresponding functions updated to better handle shell scripts' outpout. --- download_rhel_iso/download_rhel_iso.py | 69 +++++++------------ ...{rheliso_to_aims.sh => iso_add_to_aims.sh} | 0 download_rhel_iso/make_qemu_prod.sh | 18 ----- .../{convertimage.sh => qemu_convertimage.sh} | 2 +- ...run_test_os_img.sh => qemu_test_os_img.sh} | 24 ++++--- ...2openstack.sh => qemu_upload2openstack.sh} | 8 +++ 6 files changed, 51 insertions(+), 70 deletions(-) rename download_rhel_iso/{rheliso_to_aims.sh => iso_add_to_aims.sh} (100%) delete mode 100755 download_rhel_iso/make_qemu_prod.sh rename download_rhel_iso/{convertimage.sh => qemu_convertimage.sh} (97%) rename download_rhel_iso/{run_test_os_img.sh => qemu_test_os_img.sh} (83%) rename download_rhel_iso/{upload2openstack.sh => qemu_upload2openstack.sh} (95%) diff --git a/download_rhel_iso/download_rhel_iso.py b/download_rhel_iso/download_rhel_iso.py index 38f305d..c45e25c 100755 --- a/download_rhel_iso/download_rhel_iso.py +++ b/download_rhel_iso/download_rhel_iso.py @@ -18,7 +18,6 @@ import smtplib from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText -## qemu needs import ast from keystoneauth1 import session from keystoneauth1.extras.kerberos import MappedKerberos @@ -115,7 +114,6 @@ def main(): time.sleep(sleep) print("Now I will ask for a new auth_token") auth_token = get_auth(OFFLINE_TOKEN) - print("Current auth token:\t" + str(auth_token[0:10])) if retry == max_retry: continue @@ -133,7 +131,7 @@ def main(): if len(isos_of_interest) > 0: iso_managing(isos_of_interest, auth_token, success_isos, failed_isos) if len(qemus_of_interest) > 0: - qemu_managing(qemus_of_interest, auth_token, success_qemus, failed_qemus, flt_os_imgs_names) + qemu_managing(glance, qemus_of_interest, auth_token, success_qemus, failed_qemus, flt_os_imgs_names) ## Management of RHEL images is complete. Logging and sending report(s) through email(s). manage_email(failed_csets, success_isos, failed_isos, success_qemus, failed_qemus) @@ -198,7 +196,7 @@ def iso_managing(isos_of_interest, auth_token, success_isos, failed_isos): ARG: a list of available qemus from Red Hat that we are interested in (e.g. qcow2 images) RETURN: a tuple of (success_qemus, failed_qemus) """ -def qemu_managing(qemus_of_interest, auth_token, success_qemus, failed_qemus, qemus_in_os): +def qemu_managing(glance_client, qemus_of_interest, auth_token, success_qemus, failed_qemus, qemus_in_os): ## Pre-Stage: Find the latest available upstream qemu latest_qemu_release = '6.0' @@ -241,7 +239,7 @@ def qemu_managing(qemus_of_interest, auth_token, success_qemus, failed_qemus, qe failed_qemus[latest_qemu['filename']] = build_status[1] return - # Variable(s) for next stages + # intermediate stage: Variable(s) for next stages if upload_status[0]: upload_json_dict = {} with open(upload_status[1]) as upload_json: @@ -254,19 +252,17 @@ def qemu_managing(qemus_of_interest, auth_token, success_qemus, failed_qemus, qe print("\tTests stage") tests_status = exec_tests_stage(upload_json_dict) - ## Prod stage (aka share_rhelX_test) + ## Prod stage (python-ized) if tests_status[0]: print("\tProd stage") - prod_status = exec_prod_stage(upload_json_dict) - else: - failed_qemus[latest_qemu['filename']] = tests_status[1] - - if prod_status[0]: + new_img_name = upload_json_dict["name"].replace("TEST ", "") + new_img_custom_name = upload_json_dict["properties"]["custom_name"].replace("TEST ", "") + glance_client.images.update(upload_json_dict['id'], name=new_img_name, custom_name=new_img_custom_name, os_edition="Base", visibility="community") latest_qemu['upload_info'] = upload_json_dict success_qemus.append(latest_qemu) print("QEMU image %s was handled succesfully!" % latest_qemu['filename']) else: - failed_qemus[latest_qemu['filename']] = prod_status[1] + failed_qemus[latest_qemu['filename']] = tests_status[1] return @@ -468,19 +464,19 @@ def rhel_iso_to_aims(iso_dict): else: aims_dest = "--testserver" - result = subprocess.run(["/root/rheliso_to_aims.sh", name, arch, description, pxe_path, aims_dest], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + result = subprocess.run(["/root/iso_add_to_aims.sh", name, arch, description, pxe_path, aims_dest], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) if result.returncode == 0: print("%s added in AIMS as %s" % (iso_dict['filename'], name)) else: subject = "Shell script to AIMS error" - body = "ERROR: rheliso_to_aims.sh did NOT run succesfully due to:\n%s" % result.stderr + body = "ERROR: iso_add_to_aims.sh did NOT run succesfully due to:\n%s" % result.stderr send_email(ADMIN_EMAIL, subject, body) print(body) sys.exit(0) """ - exec_build_stage() runs convertimage.sh through subprocess. + exec_build_stage() runs qemu_convertimage.sh through subprocess. This script is used to apply neccessary fixes to upstream RedHat images, for use at CERN ARGS: The dictionary of the iso that defines the downloaded image. RETURN: - @@ -492,24 +488,24 @@ def exec_build_stage(latest_qemu_release, date_on_filename, current_fp): new_fp = os.path.join(os.path.dirname(current_fp), new_filename) os.rename(current_fp, new_fp) - result = subprocess.run(["/root/convertimage.sh", new_fp], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + result = subprocess.run(["/root/qemu_convertimage.sh", new_fp], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) if result.returncode == 0: raw_fp = new_fp.replace("qcow2", "raw") if os.path.exists(raw_fp): print("Build stage executed succesfully. Output file is:\t%s" % raw_fp) return (True, raw_fp) else: - build_fail_report = "ERROR: convertimage.sh did run succesfully, BUT the produced file does NOT exist. Logs:\n" % result.stdout + build_fail_report = "ERROR: qemu_convertimage.sh did run succesfully, BUT the produced file does NOT exist. Logs:\n" % result.stdout print(build_fail_report) return (False, build_fail_report) else: - build_fail_report = "ERROR: convertimage.sh did NOT run succesfully due to:\n%s" % result.stderr + build_fail_report = "ERROR: qemu_convertimage.sh did NOT run succesfully due to:\n%s" % result.stderr print(build_fail_report) return (False, build_fail_report) """ - exec_upload_stage() runs upload2openstack.sh through subprocess. + exec_upload_stage() runs qemu_upload2openstack.sh through subprocess. All the arguments needed for the script, are given as arguments to the Python function. ARGS: The dictionary of the iso that defines the downloaded image. RETURN: - @@ -519,18 +515,18 @@ def exec_upload_stage(op_sys, img_type, date_on_filename, raw_fp): revision = "1" cwd = os.path.dirname(raw_fp) - result = subprocess.run(["/root/upload2openstack.sh", op_sys, img_type, date_on_filename, revision, cwd], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + result = subprocess.run(["/root/qemu_upload2openstack.sh", op_sys, img_type, date_on_filename, revision, cwd], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) if result.returncode == 0: upload_json_fp = os.path.join(cwd, "upload.json") if os.path.exists(upload_json_fp): print("Upload stage executed succesfully. Output file is:\t%s" % upload_json_fp) return (True, upload_json_fp) else: - upload_fail_report = "ERROR: upload2openstack.sh did run succesfully, BUT the produced file does NOT exist. Logs:\n" % result.stdout + upload_fail_report = "ERROR: qemu_upload2openstack.sh did run succesfully, BUT the produced file does NOT exist. Logs:\n" % result.stdout print(upload_fail_report) return (False, upload_fail_report) else: - upload_fail_report = "ERROR: upload2openstack.sh did NOT run succesfully due to:\n%s" % result.stderr + upload_fail_report = "ERROR: qemu_upload2openstack.sh did NOT run succesfully due to:\n%s" % result.stderr print(upload_fail_report) return (False, upload_fail_report) @@ -544,33 +540,20 @@ def exec_tests_stage(upload_json_dict): trigger_job = "TEST_OSRH" + upload_json_dict["properties"]["os_distro_major"] - result = subprocess.run(["/root/run_test_os_img.sh", upload_json_dict['id'], trigger_job], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + result = subprocess.run(["/root/qemu_test_os_img.sh", upload_json_dict['id'], trigger_job], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) if result.returncode == 0: - print("Tests stage executed succesfully.") + stdout_split = result.stdout.split(" ") + pipeline_url = "PIPELINE URL NOT RETRIEVED" + if len(stdout_split) > 3: + pipeline_url = stdout_split[3] + print("Tests stage executed succesfully. Check the image-ci pipeline here:\t%s" % pipeline_url) return (True, result.stdout) else: - tests_fail_report = "ERROR: run_test_os_img.sh did NOT run succesfully due to:\n%s" % result.stderr + tests_fail_report = "ERROR: qemu_test_os_img.sh did NOT run succesfully due to:\n%s" % result.stderr print(tests_fail_report) return (False, tests_fail_report) -""" - exec_prod_stage() runs os_auth.sh through subprocess. - ARGS: The dictionary of the iso that defines the downloaded image. - RETURN: - -""" -def exec_prod_stage(upload_json_dict): - - result = subprocess.run(["/root/make_qemu_prod.sh", upload_json_dict["id"], upload_json_dict["name"], upload_json_dict["properties"]["os_distro"], upload_json_dict["properties"]["os_distro_major"], upload_json_dict["properties"]["architecture"]], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) - if result.returncode == 0: - print("Prod stage executed succesfully.") - return (True, result.stdout) - else: - prod_fail_report = "ERROR: make_qemu_prod.sh did NOT run succesfully due to:\n%s" % result.stderr - print(prod_fail_report) - return (False, prod_fail_report) - - """ get_auth() gets authorization based on OFFLINE_TOKEN It generates an access token from the 'offline_token' @@ -657,7 +640,7 @@ def fail_email(failed_csets, failed_isos, failed_qemus): if len(failed_qemus) > 0: subject3 = "RHEL QEMU image(s) managing error" - body3 = "ERROR: Handling the following image(s) was unsuccesful:\n" + body3 = "ERROR: Handling the following image(s) was unsuccessful:\n" for fqemu in failed_qemus: body3 += fqemu + "\t" + failed_qemus[fqemu] + "\n" body3 += "\nWARNING: Fails during hadndling qemu image(s) may add image(s) in OpenStack project %s that are not usable.\n" % os.environ.get('OS_PROJECT_NAME') body3 += "ACTIONS:\n\t- Inspect any unsuccesful handling and delete the possibly dangerous image(s)." diff --git a/download_rhel_iso/rheliso_to_aims.sh b/download_rhel_iso/iso_add_to_aims.sh similarity index 100% rename from download_rhel_iso/rheliso_to_aims.sh rename to download_rhel_iso/iso_add_to_aims.sh diff --git a/download_rhel_iso/make_qemu_prod.sh b/download_rhel_iso/make_qemu_prod.sh deleted file mode 100755 index 4809c10..0000000 --- a/download_rhel_iso/make_qemu_prod.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -# export id=`jq -r '.id' upload.json` -# export name=`jq -r '.name' upload.json` -# export os_distro=`jq -r '.properties.os_distro' upload.json` -# export os_distro_major=`jq -r '.properties.os_distro_major' upload.json` -# export architecture=`jq -r '.properties.architecture' upload.json` - -OS_IMG_ID=$1 -OS_IMG_NAME=$2 -OS_IMG_DISTRO=$3 -OS_IMG_DISTRO_MAJOR=$4 -OS_IMG_ARCH=$5 - -openstack image set ${OS_IMG_ID} --name "${OS_IMG_NAME/TEST /}" --property custom_name="${OS_IMG_NAME/TEST /}" --property os_edition="Base" -echo "This echo command is just a workaround. If you delete it, the next command will fail !!!" -openstack image set ${OS_IMG_ID} --community -echo "Moi aussi!" diff --git a/download_rhel_iso/convertimage.sh b/download_rhel_iso/qemu_convertimage.sh similarity index 97% rename from download_rhel_iso/convertimage.sh rename to download_rhel_iso/qemu_convertimage.sh index 70337ec..20d5fe8 100755 --- a/download_rhel_iso/convertimage.sh +++ b/download_rhel_iso/qemu_convertimage.sh @@ -40,7 +40,7 @@ echo "Image is for RHEL ${OS}." echo "Ensuring that root can ssh (in lieu of 'cloud-user')" LIBGUESTFS_BACKEND=direct virt-copy-out -a $IMGFILE /etc/cloud/cloud.cfg $TMPIMAGE -sed -i 's|^disable_root: 1|disable_root: 0|' $TMPIMAGE/cloud.cfg +sed -i 's|^disable_root: .*|disable_root: 0|' $TMPIMAGE/cloud.cfg sed -i 's|name: cloud-user|name: root|' $TMPIMAGE/cloud.cfg sed -i '/gecos:/d' $TMPIMAGE/cloud.cfg $TMPIMAGE/cloud.cfg sed -i '/groups:/d' $TMPIMAGE/cloud.cfg $TMPIMAGE/cloud.cfg diff --git a/download_rhel_iso/run_test_os_img.sh b/download_rhel_iso/qemu_test_os_img.sh similarity index 83% rename from download_rhel_iso/run_test_os_img.sh rename to download_rhel_iso/qemu_test_os_img.sh index c890f29..57519b4 100755 --- a/download_rhel_iso/run_test_os_img.sh +++ b/download_rhel_iso/qemu_test_os_img.sh @@ -82,11 +82,20 @@ PIPELINE_ID=$(curl -sL --request POST \ --data "{ \"ref\": \"master\", \"variables\": [ {\"key\": \"IMAGE\", \"value\": \"${OS_IMG_ID}\"}, {\"key\": \"PROJECT_VIRTUAL\", \"value\": \"${OS_PROJECT_NAME}\"}, {\"key\": \"TEST_VIRTUAL\", \"value\": \"True\"},{\"key\": \"TEST_PHYSICAL\", \"value\": \"False\"}, {\"key\": \"TEST_UNMANAGED\", \"value\": \"True\"}, {\"key\": \"TEST_PUPPET\", \"value\": \"True\"}, {\"key\": \"${TRIGGER_JOB}\", \"value\": \"True\"}, {\"key\": \"TEST_OS7\", \"value\": \"False\"}, {\"key\": \"TEST_OS8s\", \"value\": \"False\"}, {\"key\": \"TEST_OS9\", \"value\": \"False\"} ] }" \ "https://gitlab.cern.ch/api/v4/projects/linuxsupport%2Ftesting%2Fimage-ci/pipeline" | jq -r .id) +### The original script doesn't fail: +#if [ "$PIPELINE_ID" == "null" ]; then +# echo "Could not create a pipeline for https://gitlab.cern.ch/${TRIGGER_REPO} :(" +# exit 0 +#fi + +### In case of any failure, return code is != 0 and message is logged to stderr. Now the runner(Python script) can catch the error! if [ "$PIPELINE_ID" == "null" ]; then - echo "Could not create a pipeline for https://gitlab.cern.ch/${TRIGGER_REPO} :(" - exit 0 + echo "Could not create a pipeline for https://gitlab.cern.ch/${TRIGGER_REPO} :(" >&2 + exit 1 fi +### In case of any failure, return code is != 0 and message is logged to stderr. Now the runner(Python script) can catch the error! + # 360 attempts and 60 sec poll interval equals to 6 hours. # Usual pipelines are about 4 hours long: https://gitlab.cern.ch/linuxsupport/rpms/openafs/pipelines # Adding 2 extra hours for possible waiting times in CI and Koji @@ -95,18 +104,17 @@ RETURN_CODE=$? if [ $RETURN_CODE -ne 0 ]; then case $RETURN_CODE in 1) - echo "Trigger OpenAFS kmod builds on kernel updates, pipeline reached the timeout: https://gitlab.cern.ch/${TRIGGER_REPO}/pipelines/${PIPELINE_ID}" + echo "Trigger OpenAFS kmod builds on kernel updates, pipeline reached the timeout: https://gitlab.cern.ch/${TRIGGER_REPO}/pipelines/${PIPELINE_ID}" >&2 ;; 2) - echo "Pipeline https://gitlab.cern.ch/${TRIGGER_REPO}/pipelines/${PIPELINE_ID} was canceled." + echo "Pipeline https://gitlab.cern.ch/${TRIGGER_REPO}/pipelines/${PIPELINE_ID} was canceled." >&2 ;; 3) - echo "Pipeline https://gitlab.cern.ch/${TRIGGER_REPO}/pipelines/${PIPELINE_ID} was failed." + echo "Pipeline https://gitlab.cern.ch/${TRIGGER_REPO}/pipelines/${PIPELINE_ID} was failed." >&2 ;; 4) - echo "Pipeline https://gitlab.cern.ch/${TRIGGER_REPO}/pipelines/${PIPELINE_ID} got an unknown status." + echo "Pipeline https://gitlab.cern.ch/${TRIGGER_REPO}/pipelines/${PIPELINE_ID} got an unknown status." >&2 ;; esac + exit 1 fi - - diff --git a/download_rhel_iso/upload2openstack.sh b/download_rhel_iso/qemu_upload2openstack.sh similarity index 95% rename from download_rhel_iso/upload2openstack.sh rename to download_rhel_iso/qemu_upload2openstack.sh index c71b200..30c0e1a 100755 --- a/download_rhel_iso/upload2openstack.sh +++ b/download_rhel_iso/qemu_upload2openstack.sh @@ -2,6 +2,7 @@ #temp script: TBD better. +# working directory defined from argument $5 cd $5 function usage { @@ -177,4 +178,11 @@ for ARCH in ${ARCHS}; do $KSFILE \ --file $img \ "$image_name" | tee upload.json + ## Checking the openstack command's exit code (PIPESTATUS[0]), + OS_EXIT_CODE=${PIPESTATUS[0]} + echo "PIPESTATUS[0] is: " ${OS_EXIT_CODE} + if [ ${OS_EXIT_CODE} -ne 0 ]; then + echo "ERROR: openstack command failed. Check stderr." + exit 1 + fi done -- GitLab From f708c67c24190eeb71df2247ed84b9061b22d941 Mon Sep 17 00:00:00 2001 From: Georgios Argyriou Date: Thu, 7 Jul 2022 17:11:55 +0200 Subject: [PATCH 09/15] Renaming and polishing - Project name renamed along with several files and definitions. - Several functions renamed. - All comments and descriptions for functions updated. --- ...rhel_iso.nomad => rhel_manage_images.nomad | 8 +- .../Dockerfile | 2 +- .../iso_add_to_aims.sh | 0 .../linuxsupport9-stable.repo | 0 .../openstack-upstream.repo | 0 .../qemu_convertimage.sh | 0 .../qemu_test_os_img.sh | 0 .../qemu_upload2openstack.sh | 0 .../rhel_manage_images.py | 132 +++++++++--------- .../run_rhel_manage_images.sh | 2 +- 10 files changed, 71 insertions(+), 73 deletions(-) rename download_rhel_iso.nomad => rhel_manage_images.nomad (83%) rename {download_rhel_iso => rhel_manage_images}/Dockerfile (89%) rename {download_rhel_iso => rhel_manage_images}/iso_add_to_aims.sh (100%) rename {download_rhel_iso => rhel_manage_images}/linuxsupport9-stable.repo (100%) rename {download_rhel_iso => rhel_manage_images}/openstack-upstream.repo (100%) rename {download_rhel_iso => rhel_manage_images}/qemu_convertimage.sh (100%) rename {download_rhel_iso => rhel_manage_images}/qemu_test_os_img.sh (100%) rename {download_rhel_iso => rhel_manage_images}/qemu_upload2openstack.sh (100%) rename download_rhel_iso/download_rhel_iso.py => rhel_manage_images/rhel_manage_images.py (83%) rename download_rhel_iso/run_download_rhel_iso.sh => rhel_manage_images/run_rhel_manage_images.sh (61%) diff --git a/download_rhel_iso.nomad b/rhel_manage_images.nomad similarity index 83% rename from download_rhel_iso.nomad rename to rhel_manage_images.nomad index 1f23a26..ac891f1 100755 --- a/download_rhel_iso.nomad +++ b/rhel_manage_images.nomad @@ -1,4 +1,4 @@ -job "${PREFIX}_download_rhel_iso" { +job "${PREFIX}_rhel_manage_images" { datacenters = ["meyrin"] type = "batch" @@ -9,16 +9,16 @@ job "${PREFIX}_download_rhel_iso" { prohibit_overlap = true } - task "${PREFIX}_download_rhel_iso" { + task "${PREFIX}_rhel_manage_images" { driver = "docker" config { - image = "https://gitlab-registry.cern.ch/linuxsupport/cronjobs/download_rhel_iso/download_rhel_iso:${CI_COMMIT_REF_NAME}" + image = "https://gitlab-registry.cern.ch/linuxsupport/cronjobs/rhel_manage_images/rhel_manage_images:${CI_COMMIT_REF_NAME}" force_pull = ${FORCE_PULL} logging { config { - tag = "${PREFIX}_download_rhel_iso" + tag = "${PREFIX}_rhel_manage_images" } } volumes = [ "$RHEL_MOUNT:/rhel" ] diff --git a/download_rhel_iso/Dockerfile b/rhel_manage_images/Dockerfile similarity index 89% rename from download_rhel_iso/Dockerfile rename to rhel_manage_images/Dockerfile index a850ea5..2ee6afb 100755 --- a/download_rhel_iso/Dockerfile +++ b/rhel_manage_images/Dockerfile @@ -7,4 +7,4 @@ RUN dnf install -y epel-release \ && yum install -y libguestfs-tools-c COPY *py *sh /root/ -CMD ["/root/run_download_rhel_iso.sh"] +CMD ["/root/run_rhel_manage_images.sh"] diff --git a/download_rhel_iso/iso_add_to_aims.sh b/rhel_manage_images/iso_add_to_aims.sh similarity index 100% rename from download_rhel_iso/iso_add_to_aims.sh rename to rhel_manage_images/iso_add_to_aims.sh diff --git a/download_rhel_iso/linuxsupport9-stable.repo b/rhel_manage_images/linuxsupport9-stable.repo similarity index 100% rename from download_rhel_iso/linuxsupport9-stable.repo rename to rhel_manage_images/linuxsupport9-stable.repo diff --git a/download_rhel_iso/openstack-upstream.repo b/rhel_manage_images/openstack-upstream.repo similarity index 100% rename from download_rhel_iso/openstack-upstream.repo rename to rhel_manage_images/openstack-upstream.repo diff --git a/download_rhel_iso/qemu_convertimage.sh b/rhel_manage_images/qemu_convertimage.sh similarity index 100% rename from download_rhel_iso/qemu_convertimage.sh rename to rhel_manage_images/qemu_convertimage.sh diff --git a/download_rhel_iso/qemu_test_os_img.sh b/rhel_manage_images/qemu_test_os_img.sh similarity index 100% rename from download_rhel_iso/qemu_test_os_img.sh rename to rhel_manage_images/qemu_test_os_img.sh diff --git a/download_rhel_iso/qemu_upload2openstack.sh b/rhel_manage_images/qemu_upload2openstack.sh similarity index 100% rename from download_rhel_iso/qemu_upload2openstack.sh rename to rhel_manage_images/qemu_upload2openstack.sh diff --git a/download_rhel_iso/download_rhel_iso.py b/rhel_manage_images/rhel_manage_images.py similarity index 83% rename from download_rhel_iso/download_rhel_iso.py rename to rhel_manage_images/rhel_manage_images.py index c45e25c..9fb1752 100755 --- a/download_rhel_iso/download_rhel_iso.py +++ b/rhel_manage_images/rhel_manage_images.py @@ -28,9 +28,9 @@ USER_EMAIL = os.getenv('USER_EMAIL') OFFLINE_TOKEN = os.getenv('RHSM_OFFLINE_TOKEN') """ - Main method for downloading and managing RHEL images. + Main method for managing RHEL images (ISO and QEMU). Errors are send to ADMIN_EMAIL. - Succesfull handling is sent to USER_EMAIL. + Succesfull managing is sent to USER_EMAIL. """ def main(): @@ -51,12 +51,12 @@ def main(): ## Connecting to RHEL if OFFLINE_TOKEN is None: - subject = "Token error" + subject = "RHEL manage images - Token error" body = "ERROR: Offline_token variable needs to be passed, exiting." send_email(ADMIN_EMAIL, subject, body) print(body) sys.exit(0) - auth_token = get_auth(OFFLINE_TOKEN) + auth_token = get_rhel_auth(OFFLINE_TOKEN) ## Connecting to OpenStack # For connecting to OpenStack you must first get kerberos credentials (It is done through CLI) @@ -113,7 +113,7 @@ def main(): print("ERROR: Getting cset content of %s failed on attempt no. %s. Sleep for %s sec and then request new auth_token." % (cset, attempts, sleep)) time.sleep(sleep) print("Now I will ask for a new auth_token") - auth_token = get_auth(OFFLINE_TOKEN) + auth_token = get_rhel_auth(OFFLINE_TOKEN) if retry == max_retry: continue @@ -134,18 +134,18 @@ def main(): qemu_managing(glance, qemus_of_interest, auth_token, success_qemus, failed_qemus, flt_os_imgs_names) ## Management of RHEL images is complete. Logging and sending report(s) through email(s). - manage_email(failed_csets, success_isos, failed_isos, success_qemus, failed_qemus) + manage_emails(failed_csets, success_isos, failed_isos, success_qemus, failed_qemus) sys.exit(0) """ - iso_managing() gets all isos of interest based on what is available from Red Hat, + iso_managing() retrieves all ISO images of interest based on what is available from Red Hat, compares them with what is available at Ceph FS in CERN and initiates all processes - for download, extraction, adding to AIMS. It returns lists and dicts of - succesfull downloads and failed downloads. - ARG: a list of available isos from Red Hat that we are interested in (e.g. dvd.iso images) - RETURN: a tuple of (success_isos, failed_isos) + for download, extraction and addition to AIMS (if applicable). + It adds items to a list of succesfully managed ISO image(s) a dictionary of unsuccesfully managed one(s). + ARG: a list of available isos, RHEL authorization token, lists of succesfull and failed isos + RETURN: - (items are added to success_isos and failed_isos. These lists are used as class variables) """ def iso_managing(isos_of_interest, auth_token, success_isos, failed_isos): @@ -156,7 +156,7 @@ def iso_managing(isos_of_interest, auth_token, success_isos, failed_isos): if downl_status[0]: if extract_iso(downl_status[1], arch_rel_dir): if arch == "x86_64" or arch == "aarch64": - rhel_iso_to_aims(item) + exec_add_to_aims_stage(item) else: print("ppc64le architecture is not allowed in aims!") success_isos.append(item) @@ -190,11 +190,12 @@ def iso_managing(isos_of_interest, auth_token, success_isos, failed_isos): """ - qemu_managing() gets all qemu images of interest based on what is available from Red Hat, + qemu_managing() retrieves all QEMU images of interest based on what is available from Red Hat, compares them with what is available at OpenStack in CERN and initiates all processes - for ???, ???. It returns lists/dicts of succesfull downloads and failed downloads. - ARG: a list of available qemus from Red Hat that we are interested in (e.g. qcow2 images) - RETURN: a tuple of (success_qemus, failed_qemus) + for downloading, re-building, upload2openstack, testing and ma(r)king as production OS images. + It adds items to a list of succesfully managed QEMU image(s) a dictionary of unsuccesfully managed one(s). + ARG: a list of available isos, RHEL authorization token, lists of succesfull and failed isos + RETURN: - (items are added to success_qemus and failed_qemus. These lists are used as class variables) """ def qemu_managing(glance_client, qemus_of_interest, auth_token, success_qemus, failed_qemus, qemus_in_os): @@ -267,10 +268,8 @@ def qemu_managing(glance_client, qemus_of_interest, auth_token, success_qemus, f """ - download_rhel_img() downloads an image from Red Hat (e.g. iso, qcow) - and verifies it through checksum. - ARG: the whole dictionary with info of the iso to be downloaded, - auth token and path for downloading + download_rhel_img() downloads an image from Red Hat (e.g. iso, qcow) and verifies it through checksum. + ARG: the whole dictionary with info of the iso to be downloaded, auth token and path for downloading RETURN: tuple = (Bool: True for success, False for fail, String: ISO filepath for success, Fail report for fail) """ @@ -305,7 +304,7 @@ def download_rhel_img(auth_token, iso_dict, downl_dirpath): print("ERROR: %s, on attempt no. %s to download %s. Sleep for 5 sec and then retry with new auth." % (fail_reason[attempts], attempts, filename)) retry += 1 time.sleep(5) - local_auth_token = get_auth(OFFLINE_TOKEN) + local_auth_token = get_rhel_auth(OFFLINE_TOKEN) ## Downloading img os.makedirs(downl_dirpath) @@ -360,8 +359,8 @@ def download_rhel_img(auth_token, iso_dict, downl_dirpath): """ extract_iso() extracts the contents of an iso file in a specified directory. - ARGS: The filepath of an iso file and - the filepath of the desirable directory to extract the contents. + ARGS: The filepaths of an iso file and that of the desirable directory to extract the contents. + RETURN: Boolean (True for success, False for fail) """ def extract_iso(iso_fp, extract_dir): @@ -439,12 +438,12 @@ def extract_iso(iso_fp, extract_dir): """ - rhel_iso_to_aims() runs a shell script through subprocess. - The shell script is running aims2client command to add the freshly downloaded image in AIMS. + exec_add_to_aims_stage() prepares the arguments for and runs the shell script iso_add_to_aims.sh. + iso_add_to_aims.sh executes aims2client command to add the recently downloaded image in AIMS. ARGS: The dictionary of the iso that defines the downloaded image. - RETURN: - + RETURN: - (Exits if there is an error... TODO: modify and make it like the rest exec_ functions) """ -def rhel_iso_to_aims(iso_dict): +def exec_add_to_aims_stage(iso_dict): arch = iso_dict['arch'] @@ -468,7 +467,7 @@ def rhel_iso_to_aims(iso_dict): if result.returncode == 0: print("%s added in AIMS as %s" % (iso_dict['filename'], name)) else: - subject = "Shell script to AIMS error" + subject = "RHEL manage images - Shell script to AIMS error" body = "ERROR: iso_add_to_aims.sh did NOT run succesfully due to:\n%s" % result.stderr send_email(ADMIN_EMAIL, subject, body) print(body) @@ -476,10 +475,12 @@ def rhel_iso_to_aims(iso_dict): """ - exec_build_stage() runs qemu_convertimage.sh through subprocess. - This script is used to apply neccessary fixes to upstream RedHat images, for use at CERN - ARGS: The dictionary of the iso that defines the downloaded image. - RETURN: - + exec_build_stage() prepares the arguments for and runs the shell script qemu_convertimage.sh. + qemu_convertimage.sh is used to apply neccessary fixes to upstream RedHat images, for use at CERN. + ARGS: The dictionary of the iso that defines the downloaded image, date in specific format and + filepath of the downloaded QEMU image. + RETURN: tuple = (Bool: True for success, False for fail, + String: filepath of .raw image if success, Fail report if fail) """ def exec_build_stage(latest_qemu_release, date_on_filename, current_fp): @@ -505,10 +506,12 @@ def exec_build_stage(latest_qemu_release, date_on_filename, current_fp): """ - exec_upload_stage() runs qemu_upload2openstack.sh through subprocess. - All the arguments needed for the script, are given as arguments to the Python function. - ARGS: The dictionary of the iso that defines the downloaded image. - RETURN: - + exec_upload_stage() prepares the arguments for and runs the shell script qemu_upload2openstack.sh. + qemu_upload2openstack.sh prepares the .raw image and uploads it to OS (predefined OS_PROJECT) + qemu_upload2openstack.sh produces / returns a report json file called: upload.json + ARGS: operating system, image type, date of specific format, filepath of .raw image file + RETURN: tuple = (Bool: True for success, False for fail, + String: filepath of upload.json if success, Fail report if fail) """ def exec_upload_stage(op_sys, img_type, date_on_filename, raw_fp): @@ -532,9 +535,10 @@ def exec_upload_stage(op_sys, img_type, date_on_filename, raw_fp): """ - exec_tests_stage() runs os_auth.sh through subprocess. - ARGS: The dictionary of the iso that defines the downloaded image. - RETURN: - + exec_tests_stage() prepares the arguments for and runs the shell script qemu_test_os_img.sh. + ARGS: The dictionary of the QEMU image that is uploaded to OS. + RETURN: tuple = (Bool: True for success, False for fail, + String: shell script's stdout of if success, Fail report if fail) """ def exec_tests_stage(upload_json_dict): @@ -555,12 +559,12 @@ def exec_tests_stage(upload_json_dict): """ - get_auth() gets authorization based on OFFLINE_TOKEN - It generates an access token from the 'offline_token' - ARG: offline_token - RETURN: token + get_rhel_auth() gets authorization based on OFFLINE_TOKEN and generates an access token + In case of failure (after max_retry attempts) an email is sent to the admin and the Python scripts exits. + ARG: offline token + RETURN: access token """ -def get_auth(offline_token): +def get_rhel_auth(offline_token): url = 'https://sso.redhat.com/auth/realms/redhat-external/protocol/openid-connect/token' data = { 'grant_type': 'refresh_token', 'client_id': 'rhsm-api', 'refresh_token': offline_token } @@ -582,7 +586,7 @@ def get_auth(offline_token): print("Retry auth...") if retry == max_retry: - subject = "Auth error" + subject = "RHEL manage images - Auth error" body = "ERROR: Unable to auth after %s attempts, exiting" % attempts send_email(ADMIN_EMAIL, subject, body) print(body) @@ -592,11 +596,12 @@ def get_auth(offline_token): """ - manage_email() - ARG: ??? - RETURN: ??? + manage_emails() gets all lists and dictionaries with images and csets that failed for any reason and images that succeeded. + It filters them and sends the corresponding emails to admin(s) and user(s) + ARG: all success and failure reports as lists and dictionaries. (CSETS, QEMU and ISO images) + RETURN: - """ -def manage_email(failed_csets, success_isos, failed_isos, success_qemus, failed_qemus): +def manage_emails(failed_csets, success_isos, failed_isos, success_qemus, failed_qemus): dt = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") @@ -615,21 +620,20 @@ def manage_email(failed_csets, success_isos, failed_isos, success_qemus, failed_ """ - fail_email() sends email to the ADMIN to inform what cset content and/or what isos could not be downloaded - ARGS: dictionary of the release as returned by cset - RETURN: - + fail_email() sends email to the ADMIN to inform which CSET, QEMU image(s) and ISO image(s) failed. + ARGS: list with failed cset(s), 2 dictionaries with the corresponding failed ISO and QEMU image(s) """ def fail_email(failed_csets, failed_isos, failed_qemus): if len(failed_csets) > 0: - subject1 = "RHEL images CSET(s) error" + subject1 = "RHEL manage images - CSET(s) error" body1 = "ERROR: Check if the following CSET(s) have a problem as the(ir) content was not retrieved:\n" for fcset in failed_csets: body1 += fcset + "\n" print(body1) send_email(ADMIN_EMAIL, subject1, body1) if len(failed_isos) > 0: - subject2 = "Download RHEL ISO image(s) error" + subject2 = "RHEL manage images - ISO image(s) error" body2 = "ERROR: Downloading the following image(s) was unsuccesful after all possible attempts:\n" for fiso in failed_isos: body2 += fiso + "\t" + failed_isos[fiso] + "\n" body2 += "\nWARNING: Failed download attempts may create directories in CEPH FS with unusable RHEL iso content.\n" @@ -639,7 +643,7 @@ def fail_email(failed_csets, failed_isos, failed_qemus): send_email(ADMIN_EMAIL, subject2, body2) if len(failed_qemus) > 0: - subject3 = "RHEL QEMU image(s) managing error" + subject3 = "RHEL manage images - QEMU image(s) error" body3 = "ERROR: Handling the following image(s) was unsuccessful:\n" for fqemu in failed_qemus: body3 += fqemu + "\t" + failed_qemus[fqemu] + "\n" body3 += "\nWARNING: Fails during hadndling qemu image(s) may add image(s) in OpenStack project %s that are not usable.\n" % os.environ.get('OS_PROJECT_NAME') @@ -649,9 +653,8 @@ def fail_email(failed_csets, failed_isos, failed_qemus): """ - success_iso_email() sends email to the USER to inform that a new RHEL release is available at CERN. - ARGS: dictionary of the release as returned by cset - RETURN: - + success_iso_email() sends email to the USER to inform which ISO image(s) succeeded. + ARGS: Dictionary with the corresponding successful ISO image(s). """ def success_iso_email(success_isos): @@ -703,9 +706,8 @@ def success_iso_email(success_isos): """ - success_qemu_email() sends email to the USER to inform that a new RHEL release is available at CERN. - ARGS: dictionary of the release as returned by cset - RETURN: - + success_qemu_email() sends email to the USER to inform which QEMU image(s) succeeded. + ARGS: Dictionary with the corresponding successful QEMU image(s). """ def success_qemu_email(success_qemus): @@ -724,12 +726,8 @@ def success_qemu_email(success_qemus): """ - send_email() sends email(s) - ARGS: receiver of the email - the subject of the email - the body of the email - email sender - RETURN: - + send_email() works as function-template for sending email(s) + ARGS: receiver of the email, the subject of the email, the body of the email, email sender """ def send_email(email_to, subject, body, email_from='linux.support@cern.ch'): diff --git a/download_rhel_iso/run_download_rhel_iso.sh b/rhel_manage_images/run_rhel_manage_images.sh similarity index 61% rename from download_rhel_iso/run_download_rhel_iso.sh rename to rhel_manage_images/run_rhel_manage_images.sh index 6011dee..1a9202f 100755 --- a/download_rhel_iso/run_download_rhel_iso.sh +++ b/rhel_manage_images/run_rhel_manage_images.sh @@ -2,4 +2,4 @@ echo $IMAGECI_PWD | kinit $IMAGECI_USER@CERN.CH -python3 -u /root/download_rhel_iso.py +python3 -u /root/rhel_manage_images.py -- GitLab From a87b696a6b95f55b41c6f87a1a560b1501d3c8aa Mon Sep 17 00:00:00 2001 From: Georgios Argyriou Date: Thu, 7 Jul 2022 17:21:16 +0200 Subject: [PATCH 10/15] Renaming within gitlab-ci.yaml! --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 86dce55..6d62e86 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,14 +2,14 @@ include: 'https://gitlab.cern.ch/linuxsupport/cronjobs/base/raw/master/gitlab-ci.yml' -build_download_rhel_iso: +build_rhel_manage_images: stage: build tags: - docker-image-build script: "echo" # unused but this line is required by GitLab CI variables: - CONTEXT_DIR: download_rhel_iso - TO: $CI_REGISTRY_IMAGE/download_rhel_iso:$CI_COMMIT_REF_NAME + CONTEXT_DIR: rhel_manage_images + TO: $CI_REGISTRY_IMAGE/rhel_manage_images:$CI_COMMIT_REF_NAME deploy: extends: .nomad -- GitLab From 40d5cf489940f1a0caa87c8adc1148a888bffda9 Mon Sep 17 00:00:00 2001 From: Georgios Argyriou Date: Fri, 8 Jul 2022 02:48:02 +0200 Subject: [PATCH 11/15] Test nomad with hybrid image url --- rhel_manage_images.nomad | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rhel_manage_images.nomad b/rhel_manage_images.nomad index ac891f1..2508d17 100755 --- a/rhel_manage_images.nomad +++ b/rhel_manage_images.nomad @@ -13,7 +13,7 @@ job "${PREFIX}_rhel_manage_images" { driver = "docker" config { - image = "https://gitlab-registry.cern.ch/linuxsupport/cronjobs/rhel_manage_images/rhel_manage_images:${CI_COMMIT_REF_NAME}" + image = "https://gitlab-registry.cern.ch/linuxsupport/cronjobs/download_rhel_iso/rhel_manage_images:${CI_COMMIT_REF_NAME}" force_pull = ${FORCE_PULL} logging { -- GitLab From 6123baaf7d91c4d6ba397ab3f96ad69ae252d5fb Mon Sep 17 00:00:00 2001 From: geargyri Date: Fri, 8 Jul 2022 10:18:06 +0200 Subject: [PATCH 12/15] Repo path changed on gitlab. Nomad is adapted accordingly. --- rhel_manage_images.nomad | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rhel_manage_images.nomad b/rhel_manage_images.nomad index 2508d17..ac891f1 100755 --- a/rhel_manage_images.nomad +++ b/rhel_manage_images.nomad @@ -13,7 +13,7 @@ job "${PREFIX}_rhel_manage_images" { driver = "docker" config { - image = "https://gitlab-registry.cern.ch/linuxsupport/cronjobs/download_rhel_iso/rhel_manage_images:${CI_COMMIT_REF_NAME}" + image = "https://gitlab-registry.cern.ch/linuxsupport/cronjobs/rhel_manage_images/rhel_manage_images:${CI_COMMIT_REF_NAME}" force_pull = ${FORCE_PULL} logging { -- GitLab From e02d611028a04bb77640f1d529085d2ab7cd5db9 Mon Sep 17 00:00:00 2001 From: geargyri Date: Fri, 8 Jul 2022 11:35:53 +0200 Subject: [PATCH 13/15] Updates derived from comments during merge request. --- rhel_manage_images/Dockerfile | 3 +-- rhel_manage_images/openstack-upstream.repo | 2 +- rhel_manage_images/qemu_test_os_img.sh | 3 +-- rhel_manage_images/rhel_manage_images.py | 8 ++++---- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/rhel_manage_images/Dockerfile b/rhel_manage_images/Dockerfile index 2ee6afb..6db75c4 100755 --- a/rhel_manage_images/Dockerfile +++ b/rhel_manage_images/Dockerfile @@ -3,8 +3,7 @@ FROM gitlab-registry.cern.ch/linuxsupport/cs9-base:latest COPY *repo /etc/yum.repos.d/ RUN dnf install -y epel-release \ - && dnf install -y python3 python3-requests python3-pycdlib aims2client python3-requests-kerberos python3-openstackclient python3-glanceclient jq curl \ - && yum install -y libguestfs-tools-c + && dnf install -y python3 python3-requests python3-pycdlib aims2client python3-requests-kerberos python3-openstackclient python3-glanceclient jq curl libguestfs-tools-c COPY *py *sh /root/ CMD ["/root/run_rhel_manage_images.sh"] diff --git a/rhel_manage_images/openstack-upstream.repo b/rhel_manage_images/openstack-upstream.repo index 6376907..e85b2b5 100755 --- a/rhel_manage_images/openstack-upstream.repo +++ b/rhel_manage_images/openstack-upstream.repo @@ -1,4 +1,4 @@ -[centos-cloud-openstack-train] +[centos-cloud-openstack-xena] name=Openstack RDO baseurl=http://linuxsoft.cern.ch/cern/centos/s9/cloud/$basearch/openstackclient-xena enabled=1 diff --git a/rhel_manage_images/qemu_test_os_img.sh b/rhel_manage_images/qemu_test_os_img.sh index 57519b4..dd56b83 100755 --- a/rhel_manage_images/qemu_test_os_img.sh +++ b/rhel_manage_images/qemu_test_os_img.sh @@ -1,7 +1,6 @@ #!/bin/bash TRIGGER_REPO="linuxsupport/testing/image-ci" -#LINUXCI_API_TOKEN=$(cat /secrets/linuxci_api_token) LINUXCI_API_TOKEN=${LINUXCI_APITOKEN} OS_IMG_ID=$1 TRIGGER_JOB=$2 @@ -104,7 +103,7 @@ RETURN_CODE=$? if [ $RETURN_CODE -ne 0 ]; then case $RETURN_CODE in 1) - echo "Trigger OpenAFS kmod builds on kernel updates, pipeline reached the timeout: https://gitlab.cern.ch/${TRIGGER_REPO}/pipelines/${PIPELINE_ID}" >&2 + echo "Pipeline reached timeout: https://gitlab.cern.ch/${TRIGGER_REPO}/pipelines/${PIPELINE_ID}" >&2 ;; 2) echo "Pipeline https://gitlab.cern.ch/${TRIGGER_REPO}/pipelines/${PIPELINE_ID} was canceled." >&2 diff --git a/rhel_manage_images/rhel_manage_images.py b/rhel_manage_images/rhel_manage_images.py index 9fb1752..c190548 100755 --- a/rhel_manage_images/rhel_manage_images.py +++ b/rhel_manage_images/rhel_manage_images.py @@ -261,7 +261,7 @@ def qemu_managing(glance_client, qemus_of_interest, auth_token, success_qemus, f glance_client.images.update(upload_json_dict['id'], name=new_img_name, custom_name=new_img_custom_name, os_edition="Base", visibility="community") latest_qemu['upload_info'] = upload_json_dict success_qemus.append(latest_qemu) - print("QEMU image %s was handled succesfully!" % latest_qemu['filename']) + print("QEMU image %s was managed succesfully!" % latest_qemu['filename']) else: failed_qemus[latest_qemu['filename']] = tests_status[1] return @@ -644,10 +644,10 @@ def fail_email(failed_csets, failed_isos, failed_qemus): if len(failed_qemus) > 0: subject3 = "RHEL manage images - QEMU image(s) error" - body3 = "ERROR: Handling the following image(s) was unsuccessful:\n" + body3 = "ERROR: Managing the following image(s) was unsuccessful:\n" for fqemu in failed_qemus: body3 += fqemu + "\t" + failed_qemus[fqemu] + "\n" - body3 += "\nWARNING: Fails during hadndling qemu image(s) may add image(s) in OpenStack project %s that are not usable.\n" % os.environ.get('OS_PROJECT_NAME') - body3 += "ACTIONS:\n\t- Inspect any unsuccesful handling and delete the possibly dangerous image(s)." + body3 += "\nWARNING: Fails during managing qemu image(s) may add image(s) in OpenStack project %s that are not usable.\n" % os.environ.get('OS_PROJECT_NAME') + body3 += "ACTIONS:\n\t- Inspect any unsuccesful management procedures and delete the possibly dangerous image(s)." print(body3) send_email(ADMIN_EMAIL, subject3, body3) -- GitLab From 78a0801963d8a0a2512562069b0cb4631c639823 Mon Sep 17 00:00:00 2001 From: Georgios Argyriou Date: Fri, 8 Jul 2022 14:38:49 +0200 Subject: [PATCH 14/15] README updated. --- README.md | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d25f771..1206ba1 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,29 @@ -# download_rhel_iso +# manage_rhel_images + +This repository holds files and scripts for automating the management of RHEL images (ISO and QEMU). + +The code is scheduled to run every day within a container in Nomad. + +The main Python script [rhel_manage_images.py](https://gitlab.cern.ch/linuxsupport/cronjobs/rhel_manage_images/-/blob/qemu/rhel_manage_images/rhel_manage_images.py) manages 2 kind of RHEL images: ISO and QEMU. + +Workflow +- The script connects with Red Hat and gets updates for the available RHEL images + +- For RHEL ISO images: + - The available RHEL ISO images are compared with what is available at CERN (specific directory in Ceph FS) + - If there is a new ISO image the following actions are: + - Download + - Extract contents + - Add to AIMS (if applicable) + - Inform USER(s) by email + +- For RHEL QEMU images: + - The available RHEL QEMU images are compared with what is available at CERN's OpenStack (project 'IT Linux Support - CI VMs'). + - If there is a new QEMU image the following actions are: + - Download + - Rebuild + - Upload to OpenStack (marked as 'TEST') + - Test the image using [Image CI](https://gitlab.cern.ch/linuxsupport/testing/image-ci/) repository / pipeline. + - Mark the image production ready + -This repo includes code scripts for automating the download RHEL iso images procedure. -- GitLab From 8bb9ac2404742aea9c02ef613762797d40be8971 Mon Sep 17 00:00:00 2001 From: Georgios Argyriou Date: Fri, 8 Jul 2022 14:40:37 +0200 Subject: [PATCH 15/15] README updated again. --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 1206ba1..6bb6f5f 100644 --- a/README.md +++ b/README.md @@ -13,17 +13,17 @@ Workflow - The available RHEL ISO images are compared with what is available at CERN (specific directory in Ceph FS) - If there is a new ISO image the following actions are: - Download - - Extract contents - - Add to AIMS (if applicable) - - Inform USER(s) by email + - Extract contents. + - Add to AIMS (if applicable). + - Inform USER(s) by email. - For RHEL QEMU images: - The available RHEL QEMU images are compared with what is available at CERN's OpenStack (project 'IT Linux Support - CI VMs'). - If there is a new QEMU image the following actions are: - Download - Rebuild - - Upload to OpenStack (marked as 'TEST') + - Upload to OpenStack (marked as 'TEST'). - Test the image using [Image CI](https://gitlab.cern.ch/linuxsupport/testing/image-ci/) repository / pipeline. - - Mark the image production ready - + - Mark the image production ready. + - Inform USER(s) by email. -- GitLab