Commit 67383f96 authored by Vineet Reddy Rajula's avatar Vineet Reddy Rajula
Browse files

Add more logic

parent f133af3a
Pipeline #3315100 passed with stages
in 4 minutes and 8 seconds
......@@ -281,11 +281,11 @@ func (r *DrupalSiteReconciler) Reconcile(ctx context.Context, req ctrl.Request)
// In situations where there are no db updates, but 'DBUpdatesPending' is set without a 'DBUpdatesFailed' status, remove the 'DBUpdatesPending'
if drupalSite.ConditionTrue("DBUpdatesPending") && !drupalSite.ConditionTrue("DBUpdatesFailed") {
updateStatus, err := r.checkForDBUpdates(ctx, drupalSite, log)
updateStatus, updatesPending, err := r.checkForDBUpdates(ctx, drupalSite, log)
if err != nil {
return r.updateCRStatusOrFailReconcile(ctx, log, drupalSite)
}
if updateStatus {
if updateStatus && !updatesPending {
update = drupalSite.Status.Conditions.RemoveCondition("DBUpdatesPending") || update
}
}
......@@ -309,7 +309,7 @@ func (r *DrupalSiteReconciler) Reconcile(ctx context.Context, req ctrl.Request)
return r.updateCRStatusOrFailReconcile(ctx, log, drupalSite)
}
// Condition `UpdateNeeded` <- either image not matching `releaseID` or `drush updb` needed
// Condition `UpdateNeeded` <- when serving image in the deployment not matching `releaseID`
updateNeeded, reconcileErr := r.updateNeeded(ctx, drupalSite)
_, isUpdateAnnotationSet := drupalSite.Annotations["updateInProgress"]
if !isUpdateAnnotationSet && !drupalSite.ConditionTrue("CodeUpdateFailed") && !drupalSite.ConditionTrue("DBUpdatesFailed") {
......@@ -475,6 +475,7 @@ func (r *DrupalSiteReconciler) isDrupalSiteReady(ctx context.Context, d *webserv
// isDrupalSiteInstalled checks if the drupal site is initialized by running drush status command in the PHP pod
func (r *DrupalSiteReconciler) isDrupalSiteInstalled(ctx context.Context, d *webservicesv1a1.DrupalSite) bool {
if r.isDrupalSiteReady(ctx, d) {
// Figure something else here instead of moving exec into a job. Maybe it is a bit of over load.
if _, err := r.execToServerPodErrOnStderr(ctx, d, "php-fpm", nil, checkIfSiteIsInstalled()...); err != nil {
return false
}
......@@ -722,11 +723,12 @@ func (r *DrupalSiteReconciler) updateDrupalVersion(ctx context.Context, d *webse
// 5. If there is a permanent unrecoverable error, restore the DB using the backup and set 'DBUpdateFailed' status
// 6. If no error, remove the 'DBUpdatesPending' status and continue
func (r *DrupalSiteReconciler) updateDBSchema(ctx context.Context, d *webservicesv1a1.DrupalSite, log logr.Logger) (update bool) {
sout, err := r.execToServerPodErrOnStderr(ctx, d, "php-fpm", nil, checkUpdbStatus()...)
_, updatesPending, err := r.checkForDBUpdates(ctx, d, log)
if err != nil {
// return r.updateCRStatusOrFailReconcile(ctx, log, d)
return true
}
if sout != "" {
if updatesPending {
// Set DBUpdatesPending status
if setDBUpdatesPending(d) {
return true
......@@ -741,14 +743,17 @@ func (r *DrupalSiteReconciler) updateDBSchema(ctx context.Context, d *webservice
// Run updb
// The updb scripts, puts the site in maintenance mode, runs updb and removes the site from maintenance mode
_, err = r.execToServerPodErrOnStderr(ctx, d, "php-fpm", nil, runUpDBCommand()...)
_, err = r.runUpdateDB(ctx, d, log)
if err != nil {
if err.Temporary() {
return true
}
err = r.rollBackDBUpdate(ctx, d, backupFileName)
if err != nil {
setConditionStatus(d, "DBUpdatesFailed", true, newApplicationError(err, ErrDBUpdateFailed), false)
return true
}
setConditionStatus(d, "DBUpdatesFailed", true, newApplicationError(err, ErrDBUpdateFailed), false)
// setConditionStatus(d, "DBUpdatesFailed", true, newApplicationError(err, ErrDBUpdateFailed), false)
return true
}
}
......@@ -812,13 +817,111 @@ func (r *DrupalSiteReconciler) addGitlabWebhookToStatus(ctx context.Context, d *
}
// checkForDBUpdates checkes if DBUpdates are required for the given site by creating a job and checking it's status.
func (r *DrupalSiteReconciler) checkForDBUpdates(ctx context.Context, d *webservicesv1a1.DrupalSite, log logr.Logger) (update bool, updatesPending bool, err reconcileError) {
// Try to fetch the job
job := &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "site-operation-" + d.Name, Namespace: d.Namespace}}
if err := r.Get(ctx, types.NamespacedName{Name: job.Name, Namespace: job.Namespace}, job); err != nil {
switch {
case k8sapierrors.IsNotFound(err):
// Create the job if it doesn't exist
if transientErr := r.ensureDrupalOperationsJob(ctx, d, log, "CheckDBUpdates", checkUpdbStatus()...); transientErr != nil {
return false, false, transientErr
}
default:
return false, false, newApplicationError(err, ErrClientK8s)
}
}
pod, err := r.getOperationsJobPod(ctx, d)
if err != nil {
return false, false, err
}
// Check if job has completed
if job.Status.CompletionTime == nil {
log.Error(err, "job not completed", "Kind", job.TypeMeta.Kind, "Resource.Namespace", job.Namespace, "Resource.Name", job.Name)
return false, false, newApplicationError(errors.New("job not completed"), ErrTemporary)
}
// if pod.Status.Phase == corev1.PodRunning {
// log.Error(err, "waiting for job to complete", "Kind", job.TypeMeta.Kind, "Resource.Namespace", job.Namespace, "Resource.Name", job.Name)
// return false, false, newApplicationError(errors.New("waiting for job to complete"), ErrTemporary)
// }
if len(pod.Status.ContainerStatuses) > 0 {
if pod.Status.ContainerStatuses[0].State.Terminated.ExitCode == 0 {
if len(pod.Status.ContainerStatuses[0].State.Terminated.Message) == 0 {
d.Status.Conditions.RemoveCondition("DBUpdatesPending")
// Delete the job
if err := r.ensureNoDrupalOperationsJob(ctx, d, log); err != nil {
return false, false, err
}
return true, false, nil
}
// Delete the job
if err := r.ensureNoDrupalOperationsJob(ctx, d, log); err != nil {
return false, false, err
}
return false, true, nil
}
err = newApplicationError(errors.New("error while checking for db updates"), ErrDBUpdateFailed)
d.Status.Conditions.RemoveCondition("DBUpdatesPending")
setConditionStatus(d, "DBUpdatesFailed", true, err, false)
return true, false, err
}
// Delete the job
if err := r.ensureNoDrupalOperationsJob(ctx, d, log); err != nil {
return false, false, err
}
d.Status.Conditions.RemoveCondition("DBUpdatesPending")
return false, false, nil
}
// runUpdateDB updates the tables that are required for the given site by creating a job and checking it's status.
// The function also deletes the job after it checks
func (r *DrupalSiteReconciler) runUpdateDB(ctx context.Context, d *webservicesv1a1.DrupalSite, log logr.Logger) (update bool, err reconcileError) {
job := &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "site-operation-" + d.Name, Namespace: d.Namespace}}
if err := r.Get(ctx, types.NamespacedName{Name: job.Name, Namespace: job.Namespace}, job); err != nil {
switch {
case k8sapierrors.IsNotFound(err):
if transientErr := r.ensureDrupalOperationsJob(ctx, d, log, "UpdateDB", runUpDBCommand()...); transientErr != nil {
return false, transientErr
}
default:
return false, newApplicationError(err, ErrClientK8s)
}
}
pod, err := r.getOperationsJobPod(ctx, d)
if err != nil {
return false, err
}
// Check if job has completed
if job.Status.CompletionTime != nil {
log.Error(err, "job not completed", "Kind", job.TypeMeta.Kind, "Resource.Namespace", job.Namespace, "Resource.Name", job.Name)
return false, newApplicationError(errors.New("job not completed"), ErrTemporary)
}
if len(pod.Status.ContainerStatuses) > 0 {
if pod.Status.ContainerStatuses[0].State.Terminated.ExitCode == 0 {
if len(pod.Status.ContainerStatuses[0].State.Terminated.Message) == 0 {
d.Status.Conditions.RemoveCondition("DBUpdatesPending")
return true, nil
}
} else {
err = newApplicationError(errors.New("error while updating db tables"), ErrDBUpdateFailed)
setConditionStatus(d, "DBUpdatesFailed", true, err, false)
return true, err
}
}
if err := r.ensureNoDrupalOperationsJob(ctx, d, log); err != nil {
return false, err
}
return false, nil
}
// restoreDatabase restores the database for the given site by creating a job and checking it's status.
// The function also deletes the job after it checks
func (r *DrupalSiteReconciler) checkForDBUpdates(ctx context.Context, d *webservicesv1a1.DrupalSite, log logr.Logger) (update bool, err reconcileError) {
func (r *DrupalSiteReconciler) restoreDatabase(ctx context.Context, d *webservicesv1a1.DrupalSite, log logr.Logger) (update bool, err reconcileError) {
job := &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "site-operation-" + d.Name, Namespace: d.Namespace}}
if err := r.Get(ctx, types.NamespacedName{Name: job.Name, Namespace: job.Namespace}, job); err != nil {
switch {
case k8sapierrors.IsNotFound(err):
if transientErr := r.ensureDrupalOperationsJob(ctx, d, "CheckDBUpdates", log); transientErr != nil {
if transientErr := r.ensureDrupalOperationsJob(ctx, d, log, "restoreDB", restoreBackup("dfw")...); transientErr != nil {
return false, transientErr
}
default:
......@@ -829,6 +932,11 @@ func (r *DrupalSiteReconciler) checkForDBUpdates(ctx context.Context, d *webserv
if err != nil {
return false, err
}
// Check if job has completed
if job.Status.CompletionTime != nil {
log.Error(err, "job not completed", "Kind", job.TypeMeta.Kind, "Resource.Namespace", job.Namespace, "Resource.Name", job.Name)
return false, newApplicationError(errors.New("job not completed"), ErrTemporary)
}
if len(pod.Status.ContainerStatuses) > 0 {
if pod.Status.ContainerStatuses[0].State.Terminated.ExitCode == 0 {
if len(pod.Status.ContainerStatuses[0].State.Terminated.Message) == 0 {
......@@ -836,7 +944,7 @@ func (r *DrupalSiteReconciler) checkForDBUpdates(ctx context.Context, d *webserv
return true, nil
}
} else {
err = newApplicationError(errors.New("error while check for db updates"), ErrDBUpdateFailed)
err = newApplicationError(errors.New("error while updating db tables"), ErrDBUpdateFailed)
setConditionStatus(d, "DBUpdatesFailed", true, err, false)
return true, err
}
......
......@@ -584,12 +584,12 @@ func (r *DrupalSiteReconciler) ensureDrupalDeployment(ctx context.Context, d *we
}
// ensureDrupalOperationsJob is similar to ensureResourceX, but for the Drupal operations job, which requires extra information like the operations script
func (r *DrupalSiteReconciler) ensureDrupalOperationsJob(ctx context.Context, d *webservicesv1a1.DrupalSite, operation string, log logr.Logger) (transientErr reconcileError) {
func (r *DrupalSiteReconciler) ensureDrupalOperationsJob(ctx context.Context, d *webservicesv1a1.DrupalSite, log logr.Logger, extraLabel string, operation ...string) (transientErr reconcileError) {
if databaseSecret := databaseSecretName(d); len(databaseSecret) != 0 {
job := &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "site-operation-" + d.Name, Namespace: d.Namespace}}
_, err := controllerruntime.CreateOrUpdate(ctx, r.Client, job, func() error {
log.V(3).Info("Ensuring Resource", "Kind", job.TypeMeta.Kind, "Resource.Namespace", job.Namespace, "Resource.Name", job.Name)
return jobForDrupalSiteOperations(job, databaseSecret, d, operation)
return jobForDrupalSiteOperations(job, databaseSecret, d, extraLabel, operation...)
})
if err != nil {
log.Error(err, "Failed to ensure Resource", "Kind", job.TypeMeta.Kind, "Resource.Namespace", job.Namespace, "Resource.Name", job.Name)
......@@ -600,6 +600,7 @@ func (r *DrupalSiteReconciler) ensureDrupalOperationsJob(ctx context.Context, d
}
// ensureNoDrupalOperationsJob ensures the Drupal operations job is deleted
// TODO: Make sure to delete the pods created by the job as well
func (r *DrupalSiteReconciler) ensureNoDrupalOperationsJob(ctx context.Context, d *webservicesv1a1.DrupalSite, log logr.Logger) (transientErr reconcileError) {
job := &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "site-operation-" + d.Name, Namespace: d.Namespace}}
if err := r.Get(ctx, types.NamespacedName{Name: job.Name, Namespace: job.Namespace}, job); err != nil {
......@@ -610,7 +611,9 @@ func (r *DrupalSiteReconciler) ensureNoDrupalOperationsJob(ctx context.Context,
return newApplicationError(err, ErrClientK8s)
}
}
if err := r.Delete(ctx, job); err != nil {
// PropagationPolicy needs to be passed when deleting job, for the child pods to be deleted as well
propagationPolicy := metav1.DeletePropagationBackground
if err := r.Delete(ctx, job, &client.DeleteOptions{PropagationPolicy: &propagationPolicy}); err != nil {
return newApplicationError(err, ErrClientK8s)
}
return nil
......@@ -1894,7 +1897,7 @@ func secretForS2iGitlabTrigger(currentobject *corev1.Secret, d *webservicesv1a1.
}
// jobForDrupalSiteOperations returns a job object thats runs a given script on the drupalSite
func jobForDrupalSiteOperations(currentobject *batchv1.Job, databaseSecret string, d *webservicesv1a1.DrupalSite, operation string) error {
func jobForDrupalSiteOperations(currentobject *batchv1.Job, databaseSecret string, d *webservicesv1a1.DrupalSite, extraLabel string, operation ...string) error {
ls := labelsForDrupalSite(d.Name)
if currentobject.CreationTimestamp.IsZero() {
addOwnerRefToObject(currentobject, asOwner(d))
......@@ -1907,6 +1910,7 @@ func jobForDrupalSiteOperations(currentobject *batchv1.Job, databaseSecret strin
Containers: []corev1.Container{{
Image: sitebuilderImageRefToUse(d, releaseID(d)).Name,
Name: "site-operation",
Command: operation,
ImagePullPolicy: "Always",
Env: []corev1.EnvVar{
{
......@@ -1931,10 +1935,6 @@ func jobForDrupalSiteOperations(currentobject *batchv1.Job, databaseSecret strin
},
},
VolumeMounts: []corev1.VolumeMount{
{
Name: "drupal-directory-" + string(d.Spec.Configuration.CloneFrom),
MountPath: "/drupal-data-source",
},
{
Name: "drupal-directory-" + d.Name,
MountPath: "/drupal-data",
......@@ -1962,14 +1962,6 @@ func jobForDrupalSiteOperations(currentobject *batchv1.Job, databaseSecret strin
},
},
},
{
Name: "drupal-directory-" + string(d.Spec.Configuration.CloneFrom),
VolumeSource: corev1.VolumeSource{
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
ClaimName: "pv-claim-" + string(d.Spec.Configuration.CloneFrom),
},
},
},
{
Name: "site-settings-php",
VolumeSource: corev1.VolumeSource{
......@@ -1992,14 +1984,8 @@ func jobForDrupalSiteOperations(currentobject *batchv1.Job, databaseSecret strin
},
},
}
switch {
case operation == "CheckDBUpdates":
currentobject.Spec.Template.Spec.Containers[0].Command = checkUpdbStatus()
case operation == "UpdateDB":
currentobject.Spec.Template.Spec.Containers[0].Command = runUpDBCommand()
}
ls["app"] = "site-operation"
ls["operation"] = operation
ls["operation"] = extraLabel
for k, v := range ls {
currentobject.Labels[k] = v
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment