diff --git a/dev.variables.sh b/dev.variables.sh
index ac83c6aad099d1dc25ce23559aa7ee8d876c15fb..8760a0973e9721b8e23d9c6e643bfcfc51b980da 100644
--- a/dev.variables.sh
+++ b/dev.variables.sh
@@ -1,5 +1,5 @@
SCHEDULE="0 8 * * *"
DB_MOUNTPOINT="/mnt/data2/test/package_alerts"
INTERESTING_PACKAGES="dev.packages.yml"
-FEEDS="rhel-8-for-x86_64-baseos-rpms,rhel-8-for-x86_64-appstream-rpms,rhel-9-for-x86_64-baseos-rpms,rhel-9-for-x86_64-appstream-rpms,almalinux-8-for-x86_64-AppStream-rpms,almalinux-8-for-x86_64-BaseOS-rpms,almalinux-9-for-x86_64-AppStream-rpms,almalinux-9-for-x86_64-BaseOS-rpms"
MATTERMOST_INTEGRATION_URL_PATH="/etc/nomad/mm_hook_test"
+FEEDS="rhel-8-for-x86_64-baseos-rpms,rhel-8-for-x86_64-appstream-rpms,rhel-9-for-x86_64-baseos-rpms,rhel-9-for-x86_64-appstream-rpms,almalinux-8-for-x86_64-AppStream-rpms,almalinux-8-for-x86_64-BaseOS-rpms,almalinux-9-for-x86_64-AppStream-rpms,almalinux-9-for-x86_64-BaseOS-rpms,debian-bookworm-amd64"
diff --git a/package_alerts/Dockerfile b/package_alerts/Dockerfile
index c65f3760a5f0a4840849f9ad00a4f856596ccfd9..22edb84b2ede031ccaf8be8ed58439d2956cb1bf 100644
--- a/package_alerts/Dockerfile
+++ b/package_alerts/Dockerfile
@@ -1,6 +1,7 @@
FROM gitlab-registry.cern.ch/linuxsupport/alma9-base:latest
-RUN dnf -y install python3 python3-dnf python3-requests python3-yaml
+RUN dnf -y install epel-release # epel is needed for dpkg
+RUN dnf -y install dpkg python3 python3-dnf python3-requests python3-yaml
COPY package_alerts *.packages.yml /root/
diff --git a/package_alerts/dev.packages.yml b/package_alerts/dev.packages.yml
index 5daca95130b75cee004f0ef784bd06b00441c75f..00a59749d00d6c5482f6da9054335d94b5e1c5a1 100644
--- a/package_alerts/dev.packages.yml
+++ b/package_alerts/dev.packages.yml
@@ -19,3 +19,7 @@ packages:
glibc:
microcode_ctl:
linux-firmware:
+ linux-image-amd64: # "kernel"
+ feed: debian-bookworm-amd64
+ libc-bin: # "glibc"
+ feed: debian-bookworm-amd64
diff --git a/package_alerts/package_alerts b/package_alerts/package_alerts
index affe0a402314b78f88e22456da6fff0a1853c549..1730ef4e607a1afc8649f6174610bb1eee8366f3 100755
--- a/package_alerts/package_alerts
+++ b/package_alerts/package_alerts
@@ -3,6 +3,7 @@
import os
import glob
+import gzip
import html
import json
import re
@@ -12,6 +13,7 @@ import sys
import tempfile
from collections import ChainMap
from datetime import datetime
+from io import BytesIO
import dnf
import requests
import yaml
@@ -59,14 +61,17 @@ logfilename = f"/work/{datetime.today().strftime('%Y%m%d-%H%M')}.log"
# pylint: disable=consider-using-with
logfile = open(logfilename, "w", encoding='utf-8')
-def generate_temp_file(prefix):
- """Generate a temp file"""
+def generate_temp_file(prefix, directory=False):
+ """Generate a temp file or directory"""
try:
# pylint: disable=consider-using-with
- tfh = tempfile.NamedTemporaryFile(mode="w", prefix=prefix, dir="/tmp")
+ if directory:
+ tfh = tempfile.TemporaryDirectory(prefix=prefix, dir="/tmp")
+ else:
+ tfh = tempfile.NamedTemporaryFile(mode="w", prefix=prefix, dir="/tmp")
filename = tfh.name
except PermissionError:
- print(f"cannot create temporary {prefix} config file")
+ print(f"cannot create temporary {prefix} file")
sys.exit(1)
return filename
@@ -86,9 +91,8 @@ def do_execute(cmd, tdir="/tmp", return_output=False, return_error=False):
return err.decode().strip()
return True
-
def get_changelog(token, payload):
- """get the changelog for a rpm, which requires downloading it"""
+ """get the changelog for a deb/rpm, which requires downloading it"""
# Almalinux
if token is None:
url = payload
@@ -99,8 +103,8 @@ def get_changelog(token, payload):
# changelog which is not very useful
attempts = 0
max_attempts = 3
- rpm_downloaded = False
- temp_file = generate_temp_file("payload")
+ package_downloaded = False
+ temp_file = generate_temp_file("changelog")
while attempts < max_attempts:
attempts += 1
result = request_url(token, url, stream=True)
@@ -110,19 +114,24 @@ def get_changelog(token, payload):
for chunk in result.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
- rpm_downloaded = True
+ package_downloaded = True
except:
print(f"Saving changelog appeared to have failed. Retrying {attempts}/{max_attempts}", flush=True)
f.close()
- if rpm_downloaded:
+ if package_downloaded:
break
else:
- print(f"API call failed, Retrying {attempts}/{max_attempts}", flush=True)
- if not rpm_downloaded:
- print("Changelog failed to download, exiting", flush=True)
+ print(f"Downloading deb/rpm failed, Retrying {attempts}/{max_attempts}", flush=True)
+ if not package_downloaded:
+ print("Package failed to download, exiting", flush=True)
sys.exit(1)
# TODO: maybe have some better logic instead of the head command
- changelog = do_execute(f"rpm -qp {temp_file} --changelog | head -20", return_output=True)
+ if 'debian' in feed:
+ temp_dir = generate_temp_file('extraction', directory=True)
+ do_execute(f"dpkg -X {temp_file} {temp_dir}")
+ changelog = do_execute(f"zcat {temp_dir}/usr/share/doc/*/changelog.Debian.gz | head -20", return_output=True)
+ else:
+ changelog = do_execute(f"rpm -qp {temp_file} --changelog | head -20", return_output=True)
# Even though we are running through a container, let's clean up after ourselves
try:
os.unlink(temp_file)
@@ -164,6 +173,26 @@ def request_url(token, url, params=None, stream=False):
def get_upstream(token, feed):
"""retrieve full list of packages from a cset / package url"""
packages = []
+ if 'debian' in feed:
+ baseurl='https://linuxsoft.cern.ch/debian'
+ url_suffix=f"/dists/{feed.split('-')[1]}/main/binary-{feed.split('-')[2]}/Packages.gz"
+ response = requests.get(f"{baseurl}/{url_suffix}")
+ buffer_data = BytesIO(response.content)
+ f = gzip.GzipFile(fileobj=buffer_data)
+ package = version = filename = None
+ for bytes_line in f.readlines():
+ line = str(bytes_line.decode().strip())
+ if 'Package:' in line:
+ package=line.split(':')[1].strip()
+ if 'Version:' in line:
+ version=line.split(':')[1].strip()
+ if 'Filename:' in line:
+ filename=line.split(':')[1].strip()
+ if all(x is not None for x in [package, version, filename]):
+ packages.append(f"{package}_{version},{baseurl}/{filename}")
+ package = version = filename = None
+ return packages
+ # From here we deal with rpm based distributions
matches = re.match(r"(\w+)-([0-9])-\w+-(\w+)-(\w+)", feed)
arch = matches.group(3)
# almalinux is simplier, so we process it earlier
@@ -252,15 +281,18 @@ def format_release(p, f, token):
newpackage = p.split(',')[0]
payload = p.split(',')[1]
logfile.write(f"{newpackage}\n")
- # check for epoch in name, and if it exists: ignore it
- try:
- name = (
- dnf.subject.Subject(newpackage.split(":")[1])
- .get_nevra_possibilities()[0]
- .name
- )
- except IndexError:
- name = dnf.subject.Subject(newpackage).get_nevra_possibilities()[0].name
+ if 'debian' in payload:
+ name = newpackage.split('_')[0]
+ else:
+ # check for epoch in name, and if it exists: ignore it
+ try:
+ name = (
+ dnf.subject.Subject(newpackage.split(":")[1])
+ .get_nevra_possibilities()[0]
+ .name
+ )
+ except IndexError:
+ name = dnf.subject.Subject(newpackage).get_nevra_possibilities()[0].name
print(f" - {newpackage}", flush=True)
if "i686" not in newpackage and name in interesting_packages:
diff --git a/package_alerts/prod.packages.yml b/package_alerts/prod.packages.yml
index 5daca95130b75cee004f0ef784bd06b00441c75f..00a59749d00d6c5482f6da9054335d94b5e1c5a1 100644
--- a/package_alerts/prod.packages.yml
+++ b/package_alerts/prod.packages.yml
@@ -19,3 +19,7 @@ packages:
glibc:
microcode_ctl:
linux-firmware:
+ linux-image-amd64: # "kernel"
+ feed: debian-bookworm-amd64
+ libc-bin: # "glibc"
+ feed: debian-bookworm-amd64