Add debian support

dev.variables.sh

 SCHEDULE="0 8 * * *"
 DB_MOUNTPOINT="/mnt/data2/test/package_alerts"
 INTERESTING_PACKAGES="dev.packages.yml"
-FEEDS="rhel-8-for-x86_64-baseos-rpms,rhel-8-for-x86_64-appstream-rpms,rhel-9-for-x86_64-baseos-rpms,rhel-9-for-x86_64-appstream-rpms,almalinux-8-for-x86_64-AppStream-rpms,almalinux-8-for-x86_64-BaseOS-rpms,almalinux-9-for-x86_64-AppStream-rpms,almalinux-9-for-x86_64-BaseOS-rpms"
 MATTERMOST_INTEGRATION_URL_PATH="/etc/nomad/mm_hook_test"
+FEEDS="rhel-8-for-x86_64-baseos-rpms,rhel-8-for-x86_64-appstream-rpms,rhel-9-for-x86_64-baseos-rpms,rhel-9-for-x86_64-appstream-rpms,almalinux-8-for-x86_64-AppStream-rpms,almalinux-8-for-x86_64-BaseOS-rpms,almalinux-9-for-x86_64-AppStream-rpms,almalinux-9-for-x86_64-BaseOS-rpms,debian-bookworm-amd64"

package_alerts/Dockerfile

 FROM gitlab-registry.cern.ch/linuxsupport/alma9-base:latest
 
-RUN dnf -y install python3 python3-dnf python3-requests python3-yaml
+RUN dnf -y install epel-release # epel is needed for dpkg
+RUN dnf -y install dpkg python3 python3-dnf python3-requests python3-yaml
 
 COPY package_alerts *.packages.yml /root/
 

package_alerts/dev.packages.yml

@@ -19,3 +19,7 @@ packages:
   glibc:
   microcode_ctl:
   linux-firmware:
+  linux-image-amd64: # "kernel"
+    feed: debian-bookworm-amd64
+  libc-bin: # "glibc"
+    feed: debian-bookworm-amd64

package_alerts/package_alerts

@@ -3,6 +3,7 @@
 
 import os
 import glob
+import gzip
 import html
 import json
 import re
@@ -12,6 +13,7 @@ import sys
 import tempfile
 from collections import ChainMap
 from datetime import datetime
+from io import BytesIO
 import dnf
 import requests
 import yaml
@@ -59,14 +61,17 @@ logfilename = f"/work/{datetime.today().strftime('%Y%m%d-%H%M')}.log"
 # pylint: disable=consider-using-with
 logfile = open(logfilename, "w", encoding='utf-8')
 
-def generate_temp_file(prefix):
-    """Generate a temp file"""
+def generate_temp_file(prefix, directory=False):
+    """Generate a temp file or directory"""
     try:
         # pylint: disable=consider-using-with
-        tfh = tempfile.NamedTemporaryFile(mode="w", prefix=prefix, dir="/tmp")
+        if directory:
+            tfh = tempfile.TemporaryDirectory(prefix=prefix, dir="/tmp")
+        else:
+            tfh = tempfile.NamedTemporaryFile(mode="w", prefix=prefix, dir="/tmp")
         filename = tfh.name
     except PermissionError:
-        print(f"cannot create temporary {prefix} config file")
+        print(f"cannot create temporary {prefix} file")
         sys.exit(1)
     return filename
 
@@ -86,9 +91,8 @@ def do_execute(cmd, tdir="/tmp", return_output=False, return_error=False):
         return err.decode().strip()
     return True
 
-
 def get_changelog(token, payload):
-    """get the changelog for a rpm, which requires downloading it"""
+    """get the changelog for a deb/rpm, which requires downloading it"""
     # Almalinux
     if token is None:
         url = payload
@@ -99,8 +103,8 @@ def get_changelog(token, payload):
     # changelog which is not very useful
     attempts = 0
     max_attempts = 3
-    rpm_downloaded = False
-    temp_file = generate_temp_file("payload")
+    package_downloaded = False
+    temp_file = generate_temp_file("changelog")
     while attempts < max_attempts:
         attempts += 1
         result = request_url(token, url, stream=True)
@@ -110,19 +114,24 @@ def get_changelog(token, payload):
                     for chunk in result.iter_content(chunk_size=1024):
                         if chunk:
                             f.write(chunk)
-                    rpm_downloaded = True
+                    package_downloaded = True
                 except:
                     print(f"Saving changelog appeared to have failed. Retrying {attempts}/{max_attempts}", flush=True)
                     f.close()
-            if rpm_downloaded:
+            if package_downloaded:
                 break
         else:
-            print(f"API call failed, Retrying {attempts}/{max_attempts}", flush=True)
-    if not rpm_downloaded:
-        print("Changelog failed to download, exiting", flush=True)
+            print(f"Downloading deb/rpm failed, Retrying {attempts}/{max_attempts}", flush=True)
+    if not package_downloaded:
+        print("Package failed to download, exiting", flush=True)
         sys.exit(1)
     # TODO: maybe have some better logic instead of the head command
-    changelog = do_execute(f"rpm -qp {temp_file} --changelog | head -20", return_output=True)
+    if 'debian' in feed:
+        temp_dir = generate_temp_file('extraction', directory=True)
+        do_execute(f"dpkg -X {temp_file} {temp_dir}")
+        changelog = do_execute(f"zcat {temp_dir}/usr/share/doc/*/changelog.Debian.gz | head -20", return_output=True)
+    else:
+        changelog = do_execute(f"rpm -qp {temp_file} --changelog | head -20", return_output=True)
     # Even though we are running through a container, let's clean up after ourselves
     try:
         os.unlink(temp_file)
@@ -164,6 +173,26 @@ def request_url(token, url, params=None, stream=False):
 def get_upstream(token, feed):
     """retrieve full list of packages from a cset / package url"""
     packages = []
+    if 'debian' in feed:
+        baseurl='https://linuxsoft.cern.ch/debian'
+        url_suffix=f"/dists/{feed.split('-')[1]}/main/binary-{feed.split('-')[2]}/Packages.gz"
+        response = requests.get(f"{baseurl}/{url_suffix}")
+        buffer_data = BytesIO(response.content)
+        f = gzip.GzipFile(fileobj=buffer_data)
+        package = version = filename = None
+        for bytes_line in f.readlines():
+            line = str(bytes_line.decode().strip())
+            if 'Package:' in line:
+                package=line.split(':')[1].strip()
+            if 'Version:' in line:
+                version=line.split(':')[1].strip()
+            if 'Filename:' in line:
+                filename=line.split(':')[1].strip()
+            if all(x is not None for x in [package, version, filename]):
+                packages.append(f"{package}_{version},{baseurl}/{filename}")
+                package = version = filename = None
+        return packages
+    # From here we deal with rpm based distributions
     matches = re.match(r"(\w+)-([0-9])-\w+-(\w+)-(\w+)", feed)
     arch = matches.group(3)
     # almalinux is simplier, so we process it earlier
@@ -252,15 +281,18 @@ def format_release(p, f, token):
     newpackage = p.split(',')[0]
     payload = p.split(',')[1]
     logfile.write(f"{newpackage}\n")
-    # check for epoch in name, and if it exists: ignore it
-    try:
-        name = (
-            dnf.subject.Subject(newpackage.split(":")[1])
-            .get_nevra_possibilities()[0]
-            .name
-        )
-    except IndexError:
-        name = dnf.subject.Subject(newpackage).get_nevra_possibilities()[0].name
+    if 'debian' in payload:
+        name = newpackage.split('_')[0]
+    else:
+        # check for epoch in name, and if it exists: ignore it
+        try:
+            name = (
+                dnf.subject.Subject(newpackage.split(":")[1])
+                .get_nevra_possibilities()[0]
+                .name
+            )
+        except IndexError:
+            name = dnf.subject.Subject(newpackage).get_nevra_possibilities()[0].name
 
     print(f" - {newpackage}", flush=True)
     if "i686" not in newpackage and name in interesting_packages:

package_alerts/prod.packages.yml

@@ -19,3 +19,7 @@ packages:
   glibc:
   microcode_ctl:
   linux-firmware:
+  linux-image-amd64: # "kernel"
+    feed: debian-bookworm-amd64
+  libc-bin: # "glibc"
+    feed: debian-bookworm-amd64