Skip to content

GitLab

Commit 8f9a4b6e authored by Domenico Giordano's avatar Domenico Giordano
Browse files

Qa

parent 4c92320e
Expand all Hide whitespace changes
Inline Side-by-side
.gitlab-ci.yml
View file @ 8f9a4b6e
stages:
- build-tox-image
- build-base-image
- test-components
- build-qa-images
- test-qa
......@@ -34,8 +34,8 @@ stages:
#------------------------------------
# Build image to be used in tox tests
job_build_tox_image:
stage: build-tox-image
build_tox_image:
stage: build-base-image
before_script:
- export DOCKERFILE=$CI_PROJECT_DIR/docker-images/tox/Dockerfile
- export CONTEXT=$CI_PROJECT_DIR/docker-images/tox
......@@ -48,6 +48,21 @@ job_build_tox_image:
changes:
- docker-images/tox/*
#------------------------------------
# Build image to be used in tox tests
build_sparknotebook_base_image:
stage: build-base-image
before_script:
- export DOCKERFILE=$CI_PROJECT_DIR/docker-images/sparknotebook/Dockerfile-swan-base
- export CONTEXT=$CI_PROJECT_DIR/docker-images/sparknotebook
- export IMAGE_NAME=sparknotebook_base
- export IMAGE_TAG=v1.0
<<: *template_build_image
only:
refs:
- qa
changes:
- docker-images/sparknotebook/Dockerfile-swan-base
############################################################
##### BUILD QA IMAGES #####
......
docker-images/sparknotebook/Dockerfile
View file @ 8f9a4b6e
# This part if rebuilt does not work
#FROM gitlab-registry.cern.ch/swan/docker-images/notebook:v4.0.1
# NB newer versions of this notebook suffer of a _bz2 issue
# and would require recompiling python
# as documented in https://github.com/freqtrade/freqtrade/issues/2245
# This images is the copy of the image previously available at
# FROM gitlab-registry.cern.ch/pkothuri/spark-pipelines/sparknotebook:v1.0
# and now removed
FROM gitlab-registry.cern.ch/cloud-infrastructure/data-analytics/sparknotebook_base:v1.0
# Install HEP_OSlibs - which is required to use software from CVMFS
#RUN yum -y install HEP_OSlibs-7.2.7-1.el7.cern \
# yum clean all && rm -r /var/cache/yum/*
# Install papermill - required to execute notebook
#RUN pip3 install papermill
FROM gitlab-registry.cern.ch/pkothuri/spark-pipelines/sparknotebook:v1.0
# INSTALL SQLLITE
# https://github.com/freqtrade/freqtrade/issues/2245
RUN yum -y install sqlite
# INSTALL PYTHON AND COMPILE
RUN yum install gcc openssl-devel bzip2-devel libffi-devel wget -y
RUN ls -la
RUN wget https://www.python.org/ftp/python/3.6.12/Python-3.6.12.tgz; ls -la; tar -xzf Python-3.6.12.tgz; ls -la; cd Python-3.6.12/; ./configure --enable-optimizations; make altinstall
# INSTALL OTHER STUFF
# INSTALL this package
COPY . /to_be_installed
RUN cd /to_be_installed ; pip3 install .
CMD []
\ No newline at end of file
docker-images/sparknotebook/Dockerfile-swan-base 0 → 100644
View file @ 8f9a4b6e
##-----------------------------------------------------------------
# For documentation: this is the original Dockerfile that generates
# gitlab-registry.cern.ch/pkothuri/spark-pipelines/sparknotebook:v1.0
#FROM gitlab-registry.cern.ch/swan/docker-images/notebook:v4.0.1
#MAINTAINER Prasanth Kothuri <prasanth.kothuri@cern.ch>
## Install HEP_OSlibs - which is required to use software from CVMFS
#RUN yum -y install HEP_OSlibs-7.2.7-1.el7.cern \
# yum clean all && rm -r /var/cache/yum/*
## Install papermill - required to execute notebook
#RUN pip3 install papermill
##-----------------------------------------------------------------
# This images is the copy of the image previously available at
# FROM gitlab-registry.cern.ch/pkothuri/spark-pipelines/sparknotebook:v1.0
# and now removed.
FROM gitlab-registry.cern.ch/cloud-infrastructure/data-analytics/sparknotebook_pkothuri:v1.0
# INSTALL SQLLITE
# https://github.com/freqtrade/freqtrade/issues/2245
RUN yum -y install sqlite
# INSTALL PYTHON AND COMPILE
RUN yum install -y gcc openssl-devel \
bzip2-devel libffi-devel wget && \
yum clean all
# NB newer versions of this notebook suffer of a _bz2 issue
# and would require recompiling python
# as documented in https://github.com/freqtrade/freqtrade/issues/2245
RUN wget https://www.python.org/ftp/python/3.6.12/Python-3.6.12.tgz; ls -la; tar -xzf Python-3.6.12.tgz; ls -la; cd Python-3.6.12/; ./configure --enable-optimizations; make altinstall
grafana_patch/README.md → grafana_extension/README.md
View file @ 8f9a4b6e
This folder contains the patchs to the Grafana Client JS needed to extend the Grafana Annotation functionalities,
This folder contains the extension to the Grafana Client JS needed to extend the Grafana Annotation functionalities,
allowing the automatic annotation of Anomalies.
In addition, all the templated variables are saved in the grafana annotation as tags.
The code in this folder is under the same license of Grafana
Apache License 2.0 https://github.com/grafana/grafana/blob/v7.1.3/LICENSE
Steps to install advanced annotation capabilities
### Build the extension
Procedure to extend the js code, when a new version of monit-grafana is released.
The js code to be extended can be found via browser developers tools in the folder `monit-grafana.cern.ch/public/build/app.*js`
1. Download it in this folder to extend it
```
curl -v -o app.grafana_7.3.3_extend_annotations.js https://monit-grafana.cern.ch/public/build/app.68bcddb017f3fbc49566.js
```
1. Commit the code for reference
1. Replace in the js file the Annotation input form with what reported in [new_annotation_buttons.js_diff](new_annotation_buttons.js_diff)
1. Add in the js file the Annotation button functions with what reported in [new_annotation_buttons_functionality.js_diff](new_annotation_buttons_functionality.js_diff)
1. Expose the js file in some accessible url (see next section)
### Install advanced grafana annotation capabilities in the browser
- Install "Resource override" plugin for substituting the default javascript with our empowered version
- CHROME: https://chrome.google.com/webstore/detail/resource-override/pkoacgokdfckfpndoffpifphamojphii
- FIREFORX: https://addons.mozilla.org/en-US/firefox/addon/resourceoverride/
- Add this rule:
- From: https://monit-grafana.cern.ch/public/build/app.*.js
- To: <some_url_where_the_patch_file_is_exported>
- To: <some_url_where_the_new_js_file_is_exported>
- NB: you cannot serve this js directly from gitlab in raw format for security reasons (see https://gitlab.com/gitlab-org/gitlab-foss/-/issues/17234)
grafana_extension/app.grafana_7.3.3_extend_annotations.js 0 → 100644
View file @ 8f9a4b6e
This diff is collapsed.
grafana_extension/new_annotation_buttons.js_diff 0 → 100644
View file @ 8f9a4b6e
--- a/grafana_extension/app.grafana_7.3.3_extend_annotations.js
+++ b/grafana_extension/app.grafana_7.3.3_extend_annotations.js
@@ -105901,7 +105901,7 @@
vk65: function(e, t) {
var n = "public/app/features/annotations/partials/event_editor.html";
window.angular.module("ng").run(["$templateCache", function(e) {
- e.put(n, '\n<div class=graph-annotation>\n\t<div class=graph-annotation__header>\n\t\t<div class=graph-annotation__user bs-tooltip="\'Created by {{ctrl.login}}\'">\n\t\t</div>\n\n\t\t<div class=graph-annotation__title>\n\t\t\t<span ng-if=!ctrl.event.id>Add Annotation</span>\n\t\t\t<span ng-if=ctrl.event.id>Edit Annotation</span>\n\t\t</div>\n\n <div class=graph-annotation__time>{{ctrl.timeFormated}}</div>\n\t</div>\n\n\t<form name=ctrl.form class="graph-annotation__body text-center">\n\t\t<div style=display:inline-block>\n\t\t\t<div class="gf-form gf-form--v-stretch">\n\t\t\t\t<span class="gf-form-label width-7">Description</span>\n\t\t\t\t<textarea class="gf-form-input width-20" rows=2 ng-model=ctrl.event.text placeholder=Description></textarea>\n\t\t\t</div>\n\n\t\t\t<div class=gf-form>\n\t\t\t\t<span class="gf-form-label width-7">Tags</span>\n\t\t\t\t<bootstrap-tagsinput ng-model=ctrl.event.tags tagclass="label label-tag" placeholder="add tags">\n\t\t\t\t</bootstrap-tagsinput>\n\t\t\t</div>\n\n\t\t\t<div class=gf-form-button-row>\n\t\t\t\t<button type=submit class="btn btn-primary" ng-click=ctrl.save()>Save</button>\n\t\t\t\t<button ng-if=ctrl.event.id type=submit class="btn btn-danger" ng-click=ctrl.delete()>Delete</button>\n\t\t\t\t<a class=btn-text ng-click=ctrl.close();>Cancel</a>\n\t\t\t</div>\n\t\t</div>\n\t</form>\n</div>\n')
+ e.put(n, '\n<div class=graph-annotation>\n\t<div class=graph-annotation__header>\n\t\t<div class=graph-annotation__user bs-tooltip="\'Created by {{ctrl.login}}\'">\n\t\t</div>\n\n\t\t<div class=graph-annotation__title>\n\t\t\t<span ng-if=!ctrl.event.id>Add Annotation</span>\n\t\t\t<span ng-if=ctrl.event.id>Edit Annotation</span>\n\t\t</div>\n\n <div class=graph-annotation__time>{{ctrl.timeFormated}}</div>\n\t</div>\n\n\t<form name=ctrl.form class="graph-annotation__body text-center">\n\t\t<div style=display:inline-block>\n\t\t\t<div class="gf-form gf-form--v-stretch">\n\t\t\t\t<span class="gf-form-label width-7">Description</span>\n\t\t\t\t<textarea class="gf-form-input width-20" rows=2 ng-model=ctrl.event.text placeholder=Description></textarea>\n\t\t\t</div>\n\n\t\t\t<div class=gf-form>\n\t\t\t\t<span class="gf-form-label width-7">Tags</span>\n\t\t\t\t<bootstrap-tagsinput ng-model=ctrl.event.tags tagclass="label label-tag" placeholder="add tags">\n\t\t\t\t</bootstrap-tagsinput>\n\t\t\t</div>\n\n\t\t\t<div class=gf-form-button-row>\n\t\t\t\t<button type=submit class="btn btn-primary" ng-click=ctrl.save_anomaly_with_vars()>Anomaly</button> <button type=submit class="btn btn-primary" ng-click=ctrl.save_normal_with_vars()>Normal</button><button type=submit class="btn btn-primary" ng-click=ctrl.save()>Save</button>\n\t\t\t\t<button ng-if=ctrl.event.id type=submit class="btn btn-danger" ng-click=ctrl.delete()>Delete</button>\n\t\t\t\t<a class=btn-text ng-click=ctrl.close();>Cancel</a>\n\t\t\t</div>\n\t\t</div>\n\t</form>\n</div>\n')
}]), e.exports = n
},
vkM9: function(e, t, n) {
grafana_extension/new_annotation_buttons_functionality.js_diff 0 → 100644
View file @ 8f9a4b6e
@@ -108889,6 +108889,74 @@
}
}
}, {
+ key: "save_normal_with_vars",
+ value: function() {
+ var e = this;
+ if (this.form.$valid) {
+ var t = u.a.cloneDeep(this.event);
+ //Extension: Include automatically templated values as tags
+ t.tags.push("normal");
+ // Get list of templated variables from the DOM
+ var mylist = document.querySelectorAll('[aria-label="Dashboard template variables submenu item"]');
+ //console.log(mylist);
+ for (var i = 0; i < mylist.length; i++) {
+ var mylabel = mylist[i].childNodes[0].childNodes[0].innerText;
+ var myvalue = mylist[i].childNodes[0].childNodes[1].innerText;
+
+ console.log("templated var label: " + mylabel);
+ console.log("templated var value: " + myvalue);
+ var mytag = mylabel + ":" + myvalue;
+ console.log(mytag);
+ //t.tags.push(myvalue);
+ t.tags.push(mytag);
+ }
+ //End Extension
+ t.time = t.time.valueOf(), t.timeEnd = 0, t.isRegion && (t.timeEnd = this.event.timeEnd.valueOf(), t.timeEnd < t.time) ? console.log("invalid time") : t.id ? this.anno
+ e.panelCtrl.refresh(), e.close()
+ })).catch((function() {
+ e.panelCtrl.refresh(), e.close()
+ })) : this.annotationsSrv.saveAnnotationEvent(t).then((function() {
+ e.panelCtrl.refresh(), e.close()
+ })).catch((function() {
+ e.panelCtrl.refresh(), e.close()
+ }))
+ }
+ }
+ },{
+ key: "save_anomaly_with_vars",
+ value: function() {
+ var e = this;
+ if (this.form.$valid) {
+ var t = u.a.cloneDeep(this.event);
+ //Extension: Include automatically templated values as tags
+ t.tags.push("anomaly");
+ // Get list of templated variables from the DOM
+ var mylist = document.querySelectorAll('[aria-label="Dashboard template variables submenu item"]');
+ //console.log(mylist);
+ for (var i = 0; i < mylist.length; i++) {
+ var mylabel = mylist[i].childNodes[0].childNodes[0].innerText;
+ var myvalue = mylist[i].childNodes[0].childNodes[1].innerText;
+
+ console.log("templated var label: " + mylabel);
+ console.log("templated var value: " + myvalue);
+ var mytag = mylabel + ":" + myvalue;
+ console.log(mytag);
+ //t.tags.push(myvalue);
+ t.tags.push(mytag);
+ }
+ //End Extension
+ t.time = t.time.valueOf(), t.timeEnd = 0, t.isRegion && (t.timeEnd = this.event.timeEnd.valueOf(), t.timeEnd < t.time) ? console.log("invalid time") : t.id ? this.anno
+ e.panelCtrl.refresh(), e.close()
+ })).catch((function() {
+ e.panelCtrl.refresh(), e.close()
+ })) : this.annotationsSrv.saveAnnotationEvent(t).then((function() {
+ e.panelCtrl.refresh(), e.close()
+ })).catch((function() {
+ e.panelCtrl.refresh(), e.close()
+ }))
+ }
+ }
+ },{
key: "delete",
value: function() {
var e = this;
tests/spark_etl/ci_test_script.sh
View file @ 8f9a4b6e
......@@ -32,6 +32,7 @@ export CI_COMMIT_BRANCH=${CI_COMMIT_BRANCH:-qa}
export IMAGE_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_BRANCH}
docker-compose -f docker-compose.yml down --remove-orphans --volumes
docker-compose pull
docker-compose -f docker-compose.yml -p spark_etl up --remove-orphans --renew-anon-volumes --abort-on-container-exit --exit-code-from srv_spark_etl
#docker-compose logs -f 2>&1 >> compose.log &
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment