.github/workflows/build-and-push-aws.yaml

+name: Push Images to AWS ECR
+on:
+  schedule:
+    - cron: "45 15 * * 5"
+
+jobs:
+  build-and-push-docker-images:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Set up Docker
+        uses: docker/setup-buildx-action@v1
+      - name: Build Docker images
+        run: |
+          make all
+      - name: Clone jalien
+        run: |
+          cd ${{ github.workspace }}
+          cd ..
+          git clone https://gitlab.cern.ch/jalien/jalien
+      - name: Create config.sh for bash
+        run: |
+          cd ${{ github.workspace }}
+          cd ..
+          echo "
+          export BASE_DIR=$(pwd)
+          export SCRIPT_DIR=$(pwd)/jalien-setup/bash
+          export SHARED_VOLUME=$(pwd)/SHARED_VOLUME
+          export JALIEN=$(pwd)/jalien
+          export JALIEN_SETUP=$(pwd)/jalien-setup
+          export JALIEN_SOURCE=https://gitlab.cern.ch/jalien/jalien
+          export JALIEN_SETUP_SOURCE=https://github.com/FYP-Jalien/jalien-setup
+          export CE_NAME=shared_volume_JCentral-dev-CE_1
+          export JCENTRAL_NAME=shared_volume_JCentral-dev_1
+          export SCHEDD_NAME=shared_volume_schedd_1
+          export SE_NAME=shared_volume_JCentral-dev-SE_1
+          export WORKER_NAME=shared_volume_worker1_1
+          " >> ${{ github.workspace }}/../jalien-setup/bash/config/config.sh
+      - name: Run jalien-setup bash scripts
+        run: |
+          cd ${{ github.workspace }}/../jalien-setup/bash
+          ./start.sh --shared --local-images
+
+      - name: Clone test-suite
+        run: |
+          cd ${{ github.workspace }}
+          cd ..
+          git clone https://github.com/FYP-Jalien/test-suite
+      - name: Create .env for test-suite
+        run: |
+          cd ${{ github.workspace }}
+          cd ..
+          echo "
+          export SHARED_VOLUME_PATH=$(pwd)/SHARED_VOLUME
+          export JALIEN_SETUP_PATH=$(pwd)/jalien-setup
+          export CONTAINER_NAME_CE=shared_volume_JCentral-dev-CE_1
+          export CONTAINER_NAME_CENTRAL=shared_volume_JCentral-dev_1
+          export CONTAINER_NAME_SCHEDD=shared_volume_schedd_1
+          export CONTAINER_NAME_SE=shared_volume_JCentral-dev-SE_1
+          export CONTAINER_NAME_WORKER=shared_volume_worker1_1
+          export SCRIPT_DIR=$(pwd)/test-suite
+          export ALIENV_PATH=$(pwd)/test-suite/files/alma-alienv
+          export SAMPLE_JDL_PATH=$(pwd)/test-suite/files/sample_test.jdl
+          export TESTSCRIPT_PATH=$(pwd)/test-suite/files/testscript_test.sh
+          " >> ${{ github.workspace }}/../test-suite/.env
+      - name: Run test-suite for container specific tests
+        run: |
+          cd ${{ github.workspace }}/../test-suite
+          ./index.sh --container-only
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-1
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v2
+        with:
+          registry-type: public
+
+      - name: Build, tag, and push jalien-base to Amazon ECR
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          REGISTRY_ALIAS: k7p0k7q2
+          ECR_REPOSITORY: fyp-jalien-base
+          IMAGE_TAG: latest
+        run: |
+          docker tag jalien-base:latest $ECR_REGISTRY/$REGISTRY_ALIAS/$ECR_REPOSITORY:$IMAGE_TAG
+          docker push $ECR_REGISTRY/$REGISTRY_ALIAS/$ECR_REPOSITORY:$IMAGE_TAG
+
+      - name: Build, tag, and push jalien-ce to Amazon ECR
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          REGISTRY_ALIAS: k7p0k7q2
+          ECR_REPOSITORY: fyp-jalien-ce
+          IMAGE_TAG: latest
+        run: |
+          docker tag jalien-ce:latest $ECR_REGISTRY/$REGISTRY_ALIAS/$ECR_REPOSITORY:$IMAGE_TAG
+          docker push $ECR_REGISTRY/$REGISTRY_ALIAS/$ECR_REPOSITORY:$IMAGE_TAG
+
+      - name: Build, tag, and push worker-base to Amazon ECR
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          REGISTRY_ALIAS: k7p0k7q2
+          ECR_REPOSITORY: fyp-worker-base
+          IMAGE_TAG: latest
+        run: |
+          docker tag worker-base:latest $ECR_REGISTRY/$REGISTRY_ALIAS/$ECR_REPOSITORY:$IMAGE_TAG
+          docker push $ECR_REGISTRY/$REGISTRY_ALIAS/$ECR_REPOSITORY:$IMAGE_TAG
+
+      - name: Build, tag, and push xrootd-se to Amazon ECR
+        env:
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          REGISTRY_ALIAS: k7p0k7q2
+          ECR_REPOSITORY: fyp-xrootd-se
+          IMAGE_TAG: latest
+        run: |
+          docker tag xrootd-se:latest $ECR_REGISTRY/$REGISTRY_ALIAS/$ECR_REPOSITORY:$IMAGE_TAG
+          docker push $ECR_REGISTRY/$REGISTRY_ALIAS/$ECR_REPOSITORY:$IMAGE_TAG

.github/workflows/verify-setup.yaml

+name: Verify jalien-setup
+on: 
+  pull_request:
+    branches:
+      - master
+jobs:
+  test-jalien-setup:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Set up Docker
+        uses: docker/setup-buildx-action@v1
+      - name: Clone jalien
+        run: |
+          cd ${{ github.workspace }}
+          cd ..
+          git clone https://gitlab.cern.ch/jalien/jalien
+      - name: Create config.sh for bash
+        run: |
+          cd ${{ github.workspace }}
+          cd ..
+          echo "
+          export BASE_DIR=$(pwd)
+          export SCRIPT_DIR=$(pwd)/jalien-setup/bash
+          export SHARED_VOLUME=$(pwd)/SHARED_VOLUME
+          export JALIEN=$(pwd)/jalien
+          export JALIEN_SETUP=$(pwd)/jalien-setup
+          export JALIEN_SOURCE=https://gitlab.cern.ch/jalien/jalien
+          export JALIEN_SETUP_SOURCE=https://github.com/FYP-Jalien/jalien-setup
+          export CE_NAME=shared_volume_JCentral-dev-CE_1
+          export JCENTRAL_NAME=shared_volume_JCentral-dev_1
+          export SCHEDD_NAME=shared_volume_schedd_1
+          export SE_NAME=shared_volume_JCentral-dev-SE_1
+          export WORKER_NAME=shared_volume_worker1_1
+          " >> ${{ github.workspace }}/../jalien-setup/bash/config/config.sh  
+      - name: Run jalien-setup bash scripts
+        run: |
+          cd ${{ github.workspace }}/../jalien-setup/bash
+          ./start.sh --shared
+      - name: Clone test-suite
+        run: |
+          cd ${{ github.workspace }}
+          cd ..
+          git clone https://github.com/FYP-Jalien/test-suite
+      - name: Create .env for test-suite
+        run: |
+          cd ${{ github.workspace }}
+          cd ..
+          echo "
+          export SHARED_VOLUME_PATH=$(pwd)/SHARED_VOLUME
+          export JALIEN_SETUP_PATH=$(pwd)/jalien-setup
+          export CONTAINER_NAME_CE=shared_volume_JCentral-dev-CE_1
+          export CONTAINER_NAME_CENTRAL=shared_volume_JCentral-dev_1
+          export CONTAINER_NAME_SCHEDD=shared_volume_schedd_1
+          export CONTAINER_NAME_SE=shared_volume_JCentral-dev-SE_1
+          export CONTAINER_NAME_WORKER=shared_volume_worker1_1
+          export SCRIPT_DIR=$(pwd)/test-suite
+          export ALIENV_PATH=$(pwd)/test-suite/files/alma-alienv
+          export SAMPLE_JDL_PATH=$(pwd)/test-suite/files/sample_test.jdl
+          export TESTSCRIPT_PATH=$(pwd)/test-suite/files/testscript_test.sh
+          " >> ${{ github.workspace }}/../test-suite/.env   
+      - name: Run test-suite for container specific tests
+        run: |
+          cd ${{ github.workspace }}/../test-suite
+          ./index.sh --container-only      
\ No newline at end of file

.gitignore

 .#*
+vol/*

.gitlab-ci.yml

-e2e-test:
-  image: registry.gitlab.com/finestructure/pipeline-trigger
+default:
+  image: ubuntu:latest
+
+stages:
+  - build
+  - test
+  - conclude
+  - deploy
+
+variables: 
+  JALIEN_BASE_IMAGE_NAME: jalien-base
+  JALIEN_CE_IMAGE_NAME: jalien-ce
+  WORKER_IMAGE_NAME: worker-base
+  XROOTD_IMAGE_NAME: xrootd-se
+  OWNER: zensanp
+  PIPELINE_REPO: JAliEn-replica-pipeline
+  TEST_TAG: test
+  NEW_TAG: latest
+
+# First the images are built and tagged as test. They are tagged as latest if the test pipeline passes.
+build_base_image:
+    stage: build
+    variables:
+      # To push to a specific docker tag other than latest(the default), amend the --destination parameter, e.g. --destination $CI_REGISTRY_IMAGE:$CI_BUILD_REF_NAME
+      # See https://docs.gitlab.com/ee/ci/variables/predefined_variables.html#variables-reference for available variables
+      IMAGE_DESTINATION: ${CI_REGISTRY_IMAGE}/${JALIEN_BASE_IMAGE_NAME}:${TEST_TAG}
+      CONTEXT_DIR: ${CI_PROJECT_DIR}/base
+      HTCONDOR_PASSWORD: ${CI_PROJECT_DIR}/ce-setup/htcondor-conf/pool_password
+    image: 
+      # The kaniko debug image is recommended because it has a shell, and a shell is required for an image to be used with GitLab CI/CD.
+      name: gcr.io/kaniko-project/executor:debug
+      entrypoint: [""]
+    script: 
+      # Prepare Kaniko configuration file
+      - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json
+      # Build and push the image from the Dockerfile at the root of the project.
+      - /kaniko/executor --context ${CONTEXT_DIR} --dockerfile ${CONTEXT_DIR}/Dockerfile --destination ${IMAGE_DESTINATION} --build-arg htcondor_password=${HTCONDOR_PASSWORD} --tar-path ${JALIEN_BASE_IMAGE_NAME}.tar
+      # Print the full registry path of the pushed image
+      - echo "${JALIEN_BASE_IMAGE_NAME} succesfully pushed."
+    artifacts:
+      paths:
+        - ${JALIEN_BASE_IMAGE_NAME}.tar
+      when: on_success
+
+
+build_ce_image:
+    stage: build
+    variables:
+      IMAGE_DESTINATION: ${CI_REGISTRY_IMAGE}/${JALIEN_CE_IMAGE_NAME}:${TEST_TAG}
+      CONTEXT_DIR: ${CI_PROJECT_DIR}/ce-setup
+    image: 
+        name: gcr.io/kaniko-project/executor:debug
+        entrypoint: [""]
+    script:
+      - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json
+      - /kaniko/executor --context ${CONTEXT_DIR} --dockerfile ${CONTEXT_DIR}/Dockerfile --destination ${IMAGE_DESTINATION} --tar-path ${JALIEN_CE_IMAGE_NAME}.tar
+      - echo "${JALIEN_CE_IMAGE_NAME} succesfully pushed."
+    artifacts:
+      paths:
+        - ${JALIEN_CE_IMAGE_NAME}.tar
+      when: on_success
+
+
+build_worker_image:
+    stage: build
+    variables:
+      IMAGE_DESTINATION: ${CI_REGISTRY_IMAGE}/${WORKER_IMAGE_NAME}:${TEST_TAG}
+      CONTEXT_DIR: ${CI_PROJECT_DIR}/worker
+    image: 
+        name: gcr.io/kaniko-project/executor:debug
+        entrypoint: [""]
+    script:
+      - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json
+      - /kaniko/executor --context ${CONTEXT_DIR} --dockerfile ${CONTEXT_DIR}/Dockerfile --destination ${IMAGE_DESTINATION} --tar-path ${WORKER_IMAGE_NAME}.tar
+      - echo "${WORKER_IMAGE_NAME} succesfully pushed."
+    artifacts:
+      paths:
+        - ${WORKER_IMAGE_NAME}.tar
+      when: on_success
+
+
+build_xrootd_image:
+    stage: build
+    variables:
+      IMAGE_DESTINATION: ${CI_REGISTRY_IMAGE}/${XROOTD_IMAGE_NAME}:${TEST_TAG}
+      CONTEXT_DIR: ${CI_PROJECT_DIR}/xrootd
+    image: 
+        name: gcr.io/kaniko-project/executor:debug
+        entrypoint: [""]
+    script:
+      - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json
+      - /kaniko/executor --context ${CONTEXT_DIR} --dockerfile ${CONTEXT_DIR}/Dockerfile --destination ${IMAGE_DESTINATION} --tar-path ${XROOTD_IMAGE_NAME}.tar
+      - echo "${XROOTD_IMAGE_NAME} succesfully pushed."
+    artifacts:
+      paths:
+        - ${XROOTD_IMAGE_NAME}.tar
+      when: on_success
+
+call_test_pipeline:
+    stage: test
+    before_script:
+        - apt update
+        - apt install -y curl
+    script: |
+        curl -L \
+        -X POST \
+        -H "Accept: application/vnd.github+json" \
+        -H "Authorization: Bearer ${GH_PAT}" \
+        -H "X-GitHub-Api-Version: 2022-11-28" \
+        https://api.github.com/repos/${OWNER}/${PIPELINE_REPO}/actions/workflows/99371901/dispatches \
+        -d '{"ref":"main", "inputs": { "image_tag": "test" }}'
+
+check_test_output:
+    stage: conclude
+    before_script:
+        - apt update
+        - apt install -y curl jq
+    script: |
+        sleep 60 # sleep for 60 seconds to allow github pipeline to start
+        max_iterations=10
+        cur_iteration=0
+
+        while [ $cur_iteration -lt $max_iterations ]; do
+            cur_iteration=$((cur_iteration + 1))
+            workflow_conclusion=$( curl --silent -H "Accept: application/vnd.github+json" \
+                -H "Authorization: Bearer ${GH_PAT}" \
+                -H "X-GitHub-Api-Version: 2022-11-28" \
+                "https://api.github.com/repos/${OWNER}/${PIPELINE_REPO}/actions/runs?per_page=1" | jq '.workflow_runs[0].conclusion' |  sed 's/"//g'
+            )
+
+            if [ "$workflow_conclusion" = "success" ]; then
+                echo "Test pipeline succesful. Moving to retag the images as latest..."
+                break;
+            elif [ "$workflow_conclusion" = "failure" ]; then
+                echo "Test pipeline failed. Exiting the workflow."
+                exit 1
+            else
+                workflow_status=$( curl --silent -H "Accept: application/vnd.github+json" \
+                -H "Authorization: Bearer ${GH_PAT}" \
+                -H "X-GitHub-Api-Version: 2022-11-28" \
+                "https://api.github.com/repos/${OWNER}/${PIPELINE_REPO}/actions/runs?per_page=1" | jq '.workflow_runs[0].status' |  sed 's/"//g'
+                )
+                echo "Attempt $cur_iteration/$max_iterations - Current status of the test pipeline is $workflow_status. Retrying in 2 minutes..."
+                sleep 120
+            fi
+        done
+
+        if [ $cur_iteration -eq $max_iterations ]; then
+            echo "Failed to pass the test suite. Failing this stage."
+            exit 1
+        fi
+
+retag_images:
+  stage: deploy
+  image:
+    name: gcr.io/go-containerregistry/crane:debug
+    entrypoint: [""]
   script:
-    - trigger -h gitlab.cern.ch -a "$API_TOKEN" -p "$CI_JOB_TOKEN" -t dev -e JALIEN_SETUP=1 -e JALIEN_SETUP_REPO=$CI_REPOSITORY_URL -e JALIEN_SETUP_BRANCH=$CI_COMMIT_REF_NAME 46440
+    - crane auth login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+    - crane push ${JALIEN_BASE_IMAGE_NAME}.tar ${CI_REGISTRY_IMAGE}/${JALIEN_BASE_IMAGE_NAME}:${NEW_TAG}
+    - crane push ${JALIEN_CE_IMAGE_NAME}.tar ${CI_REGISTRY_IMAGE}/${JALIEN_CE_IMAGE_NAME}:${NEW_TAG}
+    - crane push ${WORKER_IMAGE_NAME}.tar ${CI_REGISTRY_IMAGE}/${WORKER_IMAGE_NAME}:${NEW_TAG}
+    - crane push ${XROOTD_IMAGE_NAME}.tar ${CI_REGISTRY_IMAGE}/${XROOTD_IMAGE_NAME}:${NEW_TAG}
+

Makefile

 gitlab_base_image=gitlab-registry.cern.ch/jalien/jalien-setup/jalien-base
 gitlab_xrootd_image=gitlab-registry.cern.ch/jalien/jalien-setup/xrootd-se
+htcondor_password=$(shell cat ce-setup/htcondor-conf/pool_password)
 
 base-image:
-	docker build -t jalien-base -f base/Dockerfile base
+	docker build --progress=plain --build-arg htcondor_password=${htcondor_password} -t jalien-base -f base/Dockerfile base
+
+ce-image:
+	docker build -t jalien-ce -f ce-setup/Dockerfile ce-setup
 
 xrootd-image:
 	docker build -t xrootd-se -f xrootd/Dockerfile xrootd
 
+worker-image:
+	docker build -t worker-base -f worker/Dockerfile worker
+
 push-base: base-image xrootd-image
 	docker tag jalien-base ${gitlab_base_image}
 	docker push ${gitlab_base_image}
@@ -24,4 +31,4 @@ retag:
 	docker tag ${gitlab_xrootd_image} xrootd-se
 
 push-all: push-base push-xrootd
-all: base-image xrootd-image
+all: worker-image base-image xrootd-image ce-image

README.md

@@ -19,6 +19,12 @@ For host system
 - MySQL
 - jalien repository clone and or alien-cs.jar
 
+##### Using Automated scripts
+
+Instead of following the classical procedure of setting everything from scratch, one can utilize bash scripts, available at [here](https://github.com/FYP-Jalien/jalien-setup/tree/master/bash), to automate the process. Further details are available in the repository. 
+
+It's important to note that these scripts only handle the setup of the jalien-setup. Users still need to follow necessary steps regarding alien.py.
+
 ##### Building containers
 Build containers using `make all` in `/path/to/repo` on your local system
 

base/Dockerfile

-FROM ubuntu:18.04
+FROM ubuntu/mysql
 COPY docker-setup.sh /
 
 ENV JAVA_HOME='/usr/lib/jvm/java-11-oracle'
+ARG htcondor_password 
+
 RUN /bin/bash docker-setup.sh;

base/docker-setup.sh

-# Fix systemd-resolved problem
-apt update;
+# Exit if errors are encountered
+set -e
 
-# setup MySQL
-apt-get install -y debconf-utils;
-{ \
-echo mysql-community-server mysql-community-server/root-pass password ''; \
-echo mysql-community-server mysql-community-server/re-root-pass password ''; \
-} | debconf-set-selections \
-&& apt install -y mysql-server
+# Fix systemd-resolved problem
+apt-get update;
+apt-get install -y ca-certificates curl gnupg lsb-release
 
 # Install dependencies
 export DEBIAN_FRONTEND=noninteractive
 export LC_ALL=C
-apt install -y openjdk-11-jdk python3 python3-pip git slapd ldap-utils rsync vim tmux entr less cmake zlib1g-dev uuid uuid-dev libssl-dev
+apt-get install -y openjdk-11-jdk python3 python3-pip git slapd ldap-utils rsync vim tmux entr less cmake zlib1g-dev uuid uuid-dev libssl-dev debconf-utils
 
 # Install XRootD
-apt install -y wget
-wget https://xrootd.slac.stanford.edu/download/v4.12.1/xrootd-4.12.1.tar.gz
-tar xvzf xrootd-4.12.1.tar.gz
+apt install -y wget pkg-config
+wget --no-check-certificate https://xrootd.slac.stanford.edu/download/v5.6.0/xrootd-5.6.0.tar.gz
+tar xvzf xrootd-5.6.0.tar.gz
 mkdir /build && cd /build
-cmake /xrootd-4.12.1 -DCMAKE_INSTALL_PREFIX=/usr -DENABLE_PERL=FALSE
+cmake /xrootd-5.6.0 -DCMAKE_INSTALL_PREFIX=/usr -DENABLE_PERL=FALSE
 make && make install
 cd /
 
 #Install HTCondor
-wget -qO - https://research.cs.wisc.edu/htcondor/ubuntu/HTCondor-Release.gpg.key | apt-key add -
-echo "deb http://research.cs.wisc.edu/htcondor/ubuntu/8.8/bionic bionic contrib">> /etc/apt/sources.list
-echo "deb-src http://research.cs.wisc.edu/htcondor/ubuntu/8.8/bionic bionic contrib" >> /etc/apt/sources.list
-apt update
-apt install -y htcondor supervisor environment-modules tcl
\ No newline at end of file
+apt-get update && apt-get install -y curl
+central_manager_name="schedd" 
+curl -fsSL https://get.htcondor.org | GET_HTCONDOR_PASSWORD="$htcondor_password" /bin/bash -s -- --no-dry-run --submit $central_manager_name
+apt-get install -y htcondor supervisor environment-modules tcl
+

bash-setup/CreateConfig.sh

@@ -23,6 +23,8 @@ jAuthZ.pub.key.location = /jalien-dev/globus/authz/AuthZ_pub.pem
 SE.priv.key.location = /jalien-dev/globus/SE/SE_priv.pem
 SE.pub.key.location = /jalien-dev/globus/SE/SE_pub.pem
 
+alien.catalogue.PackageUtils.writeEnable=false
+
 ca.password =
 EoF
 

bash-setup/CreateDB.sh

@@ -32,6 +32,7 @@ my_cnf_content="[mysqld]\n
                 user= ${username}\n
                 datadir=${sql_home}/data\n
                 port= ${sql_port}\n
+                secure-file-priv = ""\n
                 socket= ${sql_socket}\n\n
 
                 [mysqld_safe]\n
@@ -80,12 +81,18 @@ function startDB(){
     mysqld_safe --defaults-file=$my_cnf &>/dev/null &
 }
 
-function fillDatabase(){
+function setPasswordDatabase(){
     cp $sql_templates/mysql_passwd.txt /tmp
     sed -i -e "s:sql_pass:${mysql_pass}:g" -e "s:dataDB:${dataDB}:g" -e "s:userDB:${userDB}:g" /tmp/mysql_passwd.txt
     mysql --verbose -u root -h 127.0.0.1 -P $sql_port -D mysql < /tmp/mysql_passwd.txt
 }
 
+function fillDatabase(){
+    cp $sql_templates/create_functions.txt /tmp
+    sed -i -e "s:sql_pass:${mysql_pass}:g" -e "s:dataDB:${dataDB}:g" -e "s:userDB:${userDB}:g" /tmp/create_functions.txt
+    mysql_apply < /tmp/create_functions.txt
+}
+
 function createCatalogueDB(){
     cp $sql_templates/createCatalogue.txt /tmp
     sed -i -e "s:catDB:${1}:g" /tmp/createCatalogue.txt
@@ -181,7 +188,7 @@ function addSEtoDB(){
 
 
 function addProcesses(){
-    cp $sql_templates/processes.txt $sql_templates/status_codes.txt /tmp
+    cp $sql_templates/processes.txt $sql_templates/status_codes.txt $sql_templates/init_processes.txt /tmp
     mysql_apply < /tmp/processes.txt
     for n in $(cat ${sql_templates}/status_codes.txt); do
         code=$(echo $n | cut -d "," -f 1)
@@ -189,6 +196,14 @@ function addProcesses(){
         sql_cmd="insert into processes.QUEUE_STATUS values ($code, $status);"
         echo $sql_cmd | mysql_apply
     done
+    
+    mysql_apply < /tmp/init_processes.txt
+}
+
+
+function addTag0(){
+    sql_cmd="USE ${dataDB};TAG0 (tagName, path, tableName, user) values ('PackageDef', '/alice/packages/', 'TadminVPackageDef', 'admin');"
+    echo $sql_cmd | mysql -u root -h 127.0.0.1 -p$mysql_pass -P $sql_port -D mysql
 }
 
 function main(){
@@ -206,9 +221,13 @@ function main(){
 
             sleep 6
 
-            fillDatabase
+	    setPasswordDatabase
             createCatalogueDB $dataDB
             createCatalogueDB $userDB
+            fillDatabase
+
+	    # Log all queries
+	    # mysql_apply < $sql_templates/mysql_log.txt 
 
             catalogueInitialDirectories
             #addUserToDB "admin" 1

bash-setup/CreateLDAP.sh

@@ -13,7 +13,7 @@ ldap_pass="pass"
 
 function ldap_apply_ldif () {
     file=$1
-    ldapadd -x -w ${ldap_pass} -h localhost -p ${ldap_port} -D "cn=Manager,dc=localdomain" -f "$file"
+    ldapadd -x -w ${ldap_pass} -H ldap://localhost:${ldap_port} -D "cn=Manager,dc=localdomain" -f "$file"
 }
 
 function createConfig(){
@@ -29,6 +29,18 @@ function startLDAP(){
     nohup slapd -d -1 -s 0 -h ldap://:${ldap_port} -F ${ldap_conf_dir} > ${ldap_log} 2>&1> /dev/null&
 }
 
+function waitLDAP(){
+    while true
+    do
+        if ldapwhoami -x -H ldap://localhost:8389; then
+
+            break
+        else
+            sleep 1
+        fi
+    done
+}
+
 function initializeLDAP(){
     arr=(
         # setup VO
@@ -84,6 +96,6 @@ function initializeLDAP(){
 createConfig
 createSchema
 startLDAP
-sleep 2
+waitLDAP
 initializeLDAP
 echo "CreateLDAP.sh done"

bash-setup/entrypoint.sh

@@ -6,8 +6,12 @@ export JALIEN_DEV=/jalien-dev
 export LOGS=$JALIEN_DEV/logs
 export PATH=$PATH:$JALIEN_SETUP/bash-setup
 
-# NOTE: LDAP config still depends on ~/.j/testVO paths
-target="/root/.j/testVO"
+
+apt update
+apt install -y xrootd-server xrootd-client
+
+# NOTE: LDAP config still depends on ~/.alien/testVO paths
+target="/root/.alien/testVO"
 mkdir -p $target
 CreateLDAP.sh $target/slapd/slapd.d &>>$LOGS/setup_log.txt &
 tail --pid $! -f $LOGS/setup_log.txt
@@ -21,5 +25,5 @@ cp $JALIEN_DEV/trusts/alien.p12 ~/.globus
 JCENTRAL_CMD="java -cp $JALIEN_DEV/alien-cs.jar -Duserid=$(id -u) -DAliEnConfig=/jalien-dev/config/JCentral alien.JCentral $(pwd)"
 
 
-ls $JALIEN_DEV/*.jar | entr -rcs "$JCENTRAL_CMD &>$LOGS/jcentral_stdout.txt" &
+ls $JALIEN_DEV/*.jar | entr -n -rcs "$JCENTRAL_CMD &>$LOGS/jcentral_stdout.txt" &
 tail --pid $! -f $LOGS/jcentral_stdout.txt

bash-setup/optimiser.sh

@@ -2,7 +2,17 @@
                     
 echo "Ceci n'est pas un Optimizer"
 MYSQLCALL="mysql --verbose --host=127.0.0.1 --port=3307 --password=pass --user=root -D processes -e"
-$MYSQLCALL 'update HOSTS set maxJobs=3000, maxqueued=300 where 1=1;'
+$MYSQLCALL 'update HOSTS set maxJobs=3000, maxqueued=300 where 1=1;UPDATE SITEQUEUES SET blocked="open";'
+$MYSQLCALL 'insert into PRIORITY(userId,maxUnfinishedJobs,maxTotalCpuCost,maxTotalRunningTime) values (1235890, 10000,10000,10000);'
+$MYSQLCALL 'insert into SITEQUEUES(siteId,site) values (-1,"unassigned::site");'
+
+#Log all queries
+#$MYSQLCALL 'set global general_log = 1;'
+#$MYSQLCALL 'set global log_output = "table";'
+
+#Create jalien user if not created
+$MYSQLCALL 'insert into QUEUE_USER(userId,user) values (1235890, "jalien");'
+
 while :
 do
         echo \"Optimizing\"
@@ -12,7 +22,12 @@ do
         #$MYSQLCALL "insert into JOBAGENT(entryId,priority,noce,fileBroker,revision,price,partition,disk,ttl,oldestQueueId,ce,userId,packages,site,counter)values('1','100','','0','0','1','%','50000000','80000','0',',ALICE::JALIEN::CEJALIEN,','6','%',',JALIEN,',1)" 
         #$MYSQLCALL "insert into JOBAGENT (entryId,priority,noce,fileBroker,revision,price,partition,disk,ttl,oldestQueueId,ce,userId,site,counter) values (1,100,NULL,0,0,1.0,NULL,50000000,80000,0,'LOCALHOST::JTESTSITE::FIRSTSE',6,'JTESTSITE',1);"
         #$MYSQLCALL "insert into JOBAGENT (entryId,priority,noce,fileBroker,revision,price,disk,ttl,oldestQueueId,ce,userId,packages,site,counter) values (1,100,'',0,0,1,0,0,0,'LOCALHOST::JTESTSITE::FIRSTSE',1235890,'%','JTESTSITE',1);"
-        $MYSQLCALL 'insert into JOBAGENT (entryId,priority,noce,fileBroker,`partition`,disk,ttl,ce,userId,packages,site,counter,price,oldestQueueId,revision) values (1,100,"",0,",,",0,0,",ALICE::JTestSite::firstce,",1235890,"%",",JTestSite,",1,1,0,0);'
+
+	$MYSQLCALL 'insert into JOBAGENT (entryId,priority,noce,fileBroker,`partition`,disk,cpucores,ttl,ce,userId,packages,site,counter,price,oldestQueueId,revision) values (1,100,"",0,"%",0,1,0,",ALICE::JTestSite::firstce,",1235890,"%",",JTestSite,",1,1,0,0);'
+
+	# Uncomment this for testing SITECONSTRAINTS
+	# $MYSQLCALL 'insert into JOBAGENT (entryId,priority,noce,fileBroker,`partition`,disk,cpucores,ttl,ce,userId,packages,site,counter,price,oldestQueueId,revision,OS_NAME,CPU_flags,CGROUPSv2_AVAILABLE) values (1,100,"",0,",,",0,1,0,",ALICE::JTestSite::firstce,",1235890,"%",",JTestSite,",1,1,0,0,"CentOS Linux","%bc%", TRUE);'
+        ##$MYSQLCALL 'insert into JOBAGENT (entryId,priority,noce,fileBroker,`partition`,disk,cpucores,ttl,ce,userId,packages,site,counter,price,oldestQueueId,revision,OS_NAME,CPU_flags,CGROUPSv2_AVAILABLE) values (2,100,"",0,",,",0,1,0,",ALICE::JTestSite::firstce,",1235891,"%",",JTestSite,",1,1,0,0,"CentOS Linux","avx", FALSE);'
         #Register jobs transitioned to "Waiting" with the JOBAGENT entry
         $MYSQLCALL  "update QUEUE set agentId=1 where statusId=5;"
         #Cleanup killed jobs

bash-setup/templates/docker-compose.yml

 version: "3.3"
 services:
   JCentral-dev:
-    image: jalien-base
+    image: gitlab-registry.cern.ch/jalien/jalien-setup/jalien-base:latest 
     command: /jalien-setup/bash-setup/entrypoint.sh
     hostname: JCentral-dev
     ports:
@@ -12,14 +12,24 @@ services:
     volumes:
       - $SHARED_VOLUME:/jalien-dev
       - $JALIEN_SETUP_DIR:/jalien-setup:ro
+    healthcheck:
+      test:
+        - CMD-SHELL
+        - >
+          mysql --verbose --host=127.0.0.1 --port=3307 --password=pass --user=root --execute "SHOW DATABASES;" &&
+          ldapsearch -x -b "o=localhost,dc=localdomain" -H ldap://localhost:8389
+      start_period: "180s"
     environment:
       SE_HOST: JCentral-dev-SE
 
   JCentral-dev-SE:
-    image: xrootd-se
+    image: gitlab-registry.cern.ch/jalien/jalien-setup/xrootd-se:latest 
     entrypoint: bash -c
     command: xrootd -c /etc/xrootd/xrootd-standalone.cfg
     hostname: JCentral-dev-SE
+    depends_on:
+      JCentral-dev:
+        condition: service_healthy
     ports:
       - 1094:1094 # XRootD
     volumes:
@@ -27,34 +37,41 @@ services:
       - jsite-storage:/shared-volume
 
   schedd:
-    image: "htcondor/cm:8.9.6-el7"
+    image: "htcondor/cm:23.2.0-el8"
     environment:
       USE_POOL_PASSWORD: "yes"
     volumes:
       - $JALIEN_SETUP_DIR/ce-setup/htcondor-conf/pool_password:/root/secrets/pool_password
 
-  worker:
-    image: "htcondor/execute:8.9.6-el7"
-    command: bash /ce-setup/cvmfs-checker.sh
+  worker1:
+    image: gitlab-registry.cern.ch/jalien/jalien-setup/worker-base:latest 
+    command: bash /ce-setup/htcondor-conf/start.sh
+    hostname: "worker1"
     environment:
       CONDOR_HOST: "schedd"
       USE_POOL_PASSWORD: "yes"
     volumes:
-      - /cvmfs/alice.cern.ch:/cvmfs/alice.cern.ch
       - $JALIEN_SETUP_DIR/ce-setup:/ce-setup:ro
       - $SHARED_VOLUME:/jalien-dev
       - $JALIEN_SETUP_DIR/ce-setup/htcondor-conf/pool_password:/root/secrets/pool_password
-
+  
   JCentral-dev-CE:
-    image: "jalien-base"
+    image: gitlab-registry.cern.ch/jalien/jalien-setup/jalien-ce:latest 
     command: bash /ce-setup/ce-entrypoint.sh
     hostname: "localhost.localdomain"
     environment:
       CONDOR_HOST: "schedd"
       USE_POOL_PASSWORD: "yes"
-      LD_LIBRARY_PATH: "/tmp"
+    depends_on:
+      JCentral-dev:
+        condition: service_healthy
+    healthcheck:
+      test:
+        - CMD-SHELL
+        - >
+          condor_q
+      start_period: "30s"
     volumes:
-      - /cvmfs/alice.cern.ch:/cvmfs/alice.cern.ch
       - $SHARED_VOLUME:/jalien-dev
       - $JALIEN_SETUP_DIR/ce-setup:/ce-setup:ro
       - $JALIEN_SETUP_DIR/ce-setup/htcondor-conf/pool_password:/root/secrets/pool_password

bash-setup/templates/ldap/config/cn=config/cn=module{0}.ldif

@@ -2,7 +2,7 @@ dn: cn=module{0}
 objectClass: olcModuleList
 cn: module{0}
 olcModulePath: /usr/lib/ldap
-olcModuleLoad: {0}back_hdb
+olcModuleLoad: {0}back_mdb
 structuralObjectClass: olcModuleList
 entryUUID: 95a9a684-7941-1030-99f2-f301eb8bc9b9
 creatorsName: cn=config

bash-setup/templates/ldap/config/cn=config/olcBackend={0}hdb.ldif → bash-setup/templates/ldap/config/cn=config/olcBackend={0}mdb.ldif

-dn: olcBackend={0}hdb
+dn: olcBackend={0}mdb
 objectClass: olcBackendConfig
-olcBackend: {0}hdb
+olcBackend: {0}mdb
 structuralObjectClass: olcBackendConfig
 entryUUID: 95a9d4e2-7941-1030-99f3-f301eb8bc9b9
 creatorsName: cn=config

bash-setup/templates/ldap/config/cn=config/olcDatabase={1}hdb.ldif → bash-setup/templates/ldap/config/cn=config/olcDatabase={1}mdb.ldif

-dn: olcDatabase={1}hdb
+dn: olcDatabase={1}mdb
 objectClass: olcDatabaseConfig
-objectClass: olcHdbConfig
-olcDatabase: {1}hdb
-olcDbDirectory: /root/.j/testVO/slapd
+objectClass: olcMdbConfig
+olcDatabase: {1}mdb
+olcDbDirectory: /root/.alien/testVO/slapd
 olcSuffix: dc=localdomain
 olcAccess: {0}to attrs=userPassword,shadowLastChange by self write by anonymous auth by dn="cn=Manager,dc=localdomain" write by * none
 olcAccess: {1}to dn.base="" by * read
@@ -16,7 +16,7 @@ olcDbConfig: {1}set_lk_max_objects 1500
 olcDbConfig: {2}set_lk_max_locks 1500
 olcDbConfig: {3}set_lk_max_lockers 1500
 olcDbIndex: objectClass eq
-structuralObjectClass: olcHdbConfig
+structuralObjectClass: olcMdbConfig
 entryUUID: 95a9db0e-7941-1030-99f4-f301eb8bc9b9
 creatorsName: cn=config
 createTimestamp: 20110922083534Z

bash-setup/templates/ldap/ldif/add_SE_firstse.ldif

@@ -7,7 +7,7 @@ objectClass: AliEnSE
 objectClass: AliEnMSS
 objectClass: AliEnSOAPServer
 QoS: disk
-savedir: /root/.j/testVO/SE_storage/firstse
+savedir: /root/.alien/testVO/SE_storage/firstse
 mss: File
 ioDaemons: file:host=localhost:port=${port}
 host: localhost

bash-setup/templates/ldap/ldif/add_SE_secondse.ldif

@@ -7,7 +7,7 @@ objectClass: AliEnSE
 objectClass: AliEnMSS
 objectClass: AliEnSOAPServer
 QoS: disk
-savedir: /root/.j/testVO/SE_storage/firstse
+savedir: /root/.alien/testVO/SE_storage/firstse
 mss: File
 ioDaemons: file:host=localhost:port=${port}
-host: localhost
\ No newline at end of file
+host: localhost

bash-setup/templates/sql/createCatalogue.txt

@@ -63,7 +63,11 @@ DROP TABLE IF EXISTS `INDEXTABLE`;
 
 CREATE TABLE `INDEXTABLE` (  `indexId` int(11) NOT NULL AUTO_INCREMENT,  `hostIndex` int(11) NOT NULL,  `tableName` int(11) NOT NULL,
               `lfn` varchar(255) COLLATE latin1_general_cs DEFAULT NULL,  PRIMARY KEY (`indexId`),  UNIQUE KEY `lfn` (`lfn`)
-            ) ENGINE=MyISAM AUTO_INCREMENT=2 DEFAULT CHARSET=latin1 COLLATE=latin1_general_cs;
+            ) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=latin1 COLLATE=latin1_general_cs;
+
+DROP TABLE IF EXISTS `INDEXTABLE_UPDATE`;
+
+CREATE TABLE `INDEXTABLE_UPDATE` (  `entryId` ENUM("1") NOT NULL PRIMARY KEY, `last_updated` TIMESTAMP);
 
 DROP TABLE IF EXISTS `L0L`;
 
@@ -74,7 +78,7 @@ CREATE TABLE `L0L` (  `entryId` bigint(11) NOT NULL AUTO_INCREMENT,  `owner` var
              `size` bigint(20) NOT NULL DEFAULT '0',  `dir` bigint(11) DEFAULT NULL,  `gowner` varchar(20) COLLATE latin1_general_cs NOT NULL,
              `type` char(1) COLLATE latin1_general_cs NOT NULL DEFAULT 'f',  `guid` binary(16) DEFAULT NULL,  `md5` varchar(32) COLLATE latin1_general_cs DEFAULT NULL,
              `perm` char(3) COLLATE latin1_general_cs NOT NULL,  PRIMARY KEY (`entryId`),  UNIQUE KEY `lfn` (`lfn`),  KEY `dir` (`dir`),  KEY `guid` (`guid`),
-             KEY `type` (`type`),  KEY `ctime` (`ctime`),  KEY `guidtime` (`guidtime`)
+             KEY `type` (`type`),  KEY `ctime` (`ctime`),  KEY `guidtime` (`guidtime`), KEY `jobid` (`jobid`)
             ) ENGINE=MyISAM AUTO_INCREMENT=42 DEFAULT CHARSET=latin1 COLLATE=latin1_general_cs;
 
 DROP TABLE IF EXISTS `L0L_QUOTA`;
@@ -116,10 +120,68 @@ CREATE TABLE `LL_STATS` (  `tableNumber` int(11) NOT NULL,  `max_time` char(20)
 
 DROP TABLE IF EXISTS `PACKAGES`;
 
-CREATE TABLE `PACKAGES` (  `fullPackageName` varchar(255) COLLATE latin1_general_cs DEFAULT NULL,  `lfn` varchar(255) COLLATE latin1_general_cs DEFAULT NULL,
-           `packageName` varchar(255) COLLATE latin1_general_cs DEFAULT NULL,  `username` varchar(20) COLLATE latin1_general_cs DEFAULT NULL,
-             `size` bigint(20) DEFAULT NULL,  `platform` varchar(255) COLLATE latin1_general_cs DEFAULT NULL,
-             `packageVersion` varchar(255) COLLATE latin1_general_cs DEFAULT NULL) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_general_cs;
+CREATE TABLE `PACKAGES` (
+  `fullPackageName` varchar(255) COLLATE latin1_general_cs DEFAULT NULL,
+  `packageVersion` varchar(255) COLLATE latin1_general_cs DEFAULT NULL,
+  `packageName` varchar(255) COLLATE latin1_general_cs DEFAULT NULL,
+  `username` varchar(10) COLLATE latin1_general_cs DEFAULT NULL,
+  `platform` varchar(255) COLLATE latin1_general_cs DEFAULT NULL,
+  `lfn` varchar(255) COLLATE latin1_general_cs DEFAULT NULL,
+  `size` bigint DEFAULT NULL,
+  `metadataFile` varchar(255) COLLATE latin1_general_cs DEFAULT NULL,
+  `packageComment` varchar(255) COLLATE latin1_general_cs DEFAULT NULL,
+  `cvmfsRevision` int DEFAULT NULL,
+  UNIQUE KEY `PACKAGES_pkey` (`lfn`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_general_cs;
+
+DROP TABLE IF EXISTS `OPTIMIZERS`;
+
+CREATE TABLE `OPTIMIZERS` (
+  `class` varchar(100) COLLATE latin1_general_cs NOT NULL,
+  `lastUpdate` bigint NOT NULL,
+  `frequency` int NOT NULL,
+  `lastUpdatedLog` text CHARACTER SET latin1 COLLATE latin1_general_ci,
+  `server` varchar(100) COLLATE latin1_general_cs DEFAULT NULL,
+  PRIMARY KEY (`class`)
+) ENGINE=MyISAM DEFAULT CHARSET=latin1 COLLATE=latin1_general_cs;
+
+DROP TABLE IF EXISTS `TAG0`;
+CREATE TABLE `TAG0` (
+  `tagName` char(20) COLLATE latin1_general_cs DEFAULT NULL,
+  `entryId` int NOT NULL AUTO_INCREMENT,
+  `path` char(255) COLLATE latin1_general_cs DEFAULT NULL,
+  `tableName` char(40) COLLATE latin1_general_cs DEFAULT NULL,
+  `user` varchar(20) COLLATE latin1_general_cs DEFAULT NULL,
+  PRIMARY KEY (`entryId`),
+  KEY `path` (`path`)
+) ENGINE=InnoDB AUTO_INCREMENT=25700 DEFAULT CHARSET=latin1 COLLATE=latin1_general_cs;
+
+DROP TABLE IF EXISTS `TadminVPackageDef`;
+
+CREATE TABLE `TadminVPackageDef` (
+  `file` char(255) COLLATE latin1_general_cs DEFAULT NULL,
+  `offset` int DEFAULT NULL,
+  `entryId` int NOT NULL AUTO_INCREMENT,
+  `dependencies` varchar(4590) COLLATE latin1_general_cs DEFAULT NULL,
+  `executable` varchar(255) COLLATE latin1_general_cs DEFAULT NULL,
+  `description` varchar(255) COLLATE latin1_general_cs DEFAULT NULL,
+  `size` int DEFAULT NULL,
+  `md5sum` int DEFAULT NULL,
+  `setup` varchar(255) COLLATE latin1_general_cs DEFAULT NULL,
+  `unpack` varchar(255) COLLATE latin1_general_cs DEFAULT NULL,
+  `compile` varchar(255) COLLATE latin1_general_cs DEFAULT NULL,
+  `install` varchar(255) COLLATE latin1_general_cs DEFAULT NULL,
+  `pre_install` varchar(255) COLLATE latin1_general_cs DEFAULT NULL,
+  `post_install` varchar(255) COLLATE latin1_general_cs DEFAULT NULL,
+  `pre_rm` varchar(255) COLLATE latin1_general_cs DEFAULT NULL,
+  `post_rm` varchar(255) COLLATE latin1_general_cs DEFAULT NULL,
+  `config` varchar(255) COLLATE latin1_general_cs DEFAULT NULL,
+  `path` varchar(255) COLLATE latin1_general_cs DEFAULT NULL,
+  `shared` int DEFAULT NULL,
+  KEY `entryId` (`entryId`),
+  KEY `file` (`file`)
+) ENGINE=InnoDB AUTO_INCREMENT=44043 DEFAULT CHARSET=latin1 COLLATE=latin1_general_cs;
+
 
 DROP TABLE IF EXISTS `SE`;
 
@@ -150,3 +212,30 @@ DROP TABLE IF EXISTS `HOSTS`;
 CREATE TABLE `HOSTS` (  `hostIndex` int(11) NOT NULL AUTO_INCREMENT,  `address` char(50) COLLATE latin1_general_cs,  `db` char(40) COLLATE latin1_general_cs,
               `driver` char(10) COLLATE latin1_general_cs,  `organisation` char(40) COLLATE latin1_general_cs,  PRIMARY KEY (`hostIndex`)
             ) ENGINE=MyISAM AUTO_INCREMENT=2 DEFAULT CHARSET=latin1 COLLATE=latin1_general_cs;
+
+DROP TABLE IF EXISTS `orphan_pfns`;
+
+CREATE TABLE `orphan_pfns` ( `guid` binary(16) DEFAULT NULL,
+  `se` int(11) DEFAULT NULL,
+  `fail_count` int(11) DEFAULT '0',
+  `md5sum` char(32) DEFAULT NULL,
+  `size` bigint(20) DEFAULT NULL,
+  `pfn` varchar(1024) DEFAULT NULL,
+  `flags` int(11) DEFAULT NULL,
+  UNIQUE KEY `orphan_pfns_uids` (`guid`,`se`),
+  KEY `orphan_pfns_se` (`se`,`fail_count`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+
+DROP TABLE IF EXISTS `orphan_pfns_0`;
+
+CREATE TABLE `orphan_pfns_0` (
+  `guid` binary(16) DEFAULT NULL,
+  `se` int(11) DEFAULT NULL,
+  `fail_count` int(11) DEFAULT '0',
+  `md5sum` char(32) DEFAULT NULL,
+  `size` bigint(20) DEFAULT NULL,
+  `pfn` varchar(1024) DEFAULT NULL,
+  `flags` int(11) DEFAULT NULL,
+  UNIQUE KEY `orphan_pfns_uids` (`guid`),
+  KEY `orphan_pfns_size_fail_count_idx` (`fail_count`,`size`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;