From 0089ee82a3a23853e8c1cfa6ed42e9984e17d5fd Mon Sep 17 00:00:00 2001 From: wjurasz <wiktor.jan.jurasz@cern.ch> Date: Fri, 2 Feb 2018 10:26:03 +0100 Subject: [PATCH 01/24] NXCALS-1468 initial commit --- .../build.gradle | 8 +- accsoft-nxcals-data-access-python3/setup.py | 107 ++++++++++++++++++ 2 files changed, 109 insertions(+), 6 deletions(-) create mode 100644 accsoft-nxcals-data-access-python3/setup.py diff --git a/accsoft-nxcals-data-access-python/build.gradle b/accsoft-nxcals-data-access-python/build.gradle index c909fb8a0f..3672b8bad6 100644 --- a/accsoft-nxcals-data-access-python/build.gradle +++ b/accsoft-nxcals-data-access-python/build.gradle @@ -30,12 +30,8 @@ distributions { python { testDir = file('src/test') srcDir = file('src/main') - details { - //virtualEnvPrompt = "(${project.name})" - //activateLink = project.file("activate") // File you can source to activate the virtual env - pythonVersion = '2.7' // Sets the version of python to use, will search your PATH to get the location - systemPythonInterpreter = file("/usr/bin/python2.7") // used to force an interpreter to be used to build the venv - } + details.setSystemPythonInterpreter('/user/bdisoft/operational/bin/Python/anaconda_py35/bin/python') + details.prependExecutableDirectory(new File('/user/bdisoft/operational/bin/Python/anaconda_py35/lib')) } test.dependsOn(pytest) diff --git a/accsoft-nxcals-data-access-python3/setup.py b/accsoft-nxcals-data-access-python3/setup.py new file mode 100644 index 0000000000..698f9be64a --- /dev/null +++ b/accsoft-nxcals-data-access-python3/setup.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# import io +import os +import sys +from shutil import rmtree + +from setuptools import find_packages, setup, Command + +# Package meta-data. +NAME = 'accsoft-nxcals-data-access-python3' +# DESCRIPTION = 'My short description for my project.' +# URL = 'https://github.com/me/myproject' +EMAIL = 'acclog@cern.ch' +# AUTHOR = 'BE-CE' + +# What packages are required for this module to be executed? +REQUIRED = [ + # 'requests', 'maya', 'records', +] + +here = os.path.abspath(os.path.dirname(__file__)) + +# Import the README and use it as the long-description. +# Note: this will only work if 'README.rst' is present in your MANIFEST.in file! +# with io.open(os.path.join(here, 'README.rst'), encoding='utf-8') as f: +# long_description = '\n' + f.read() + +# Load the package's __version__.py module as a dictionary. +# about = {} +# with open(os.path.join(here, NAME, '__version__.py')) as f: +# exec(f.read(), about) + + +class UploadCommand(Command): + """Support setup.py upload.""" + + description = 'Build and publish the package.' + user_options = [] + + @staticmethod + def status(s): + """Prints things in bold.""" + print('\033[1m{0}\033[0m'.format(s)) + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + try: + self.status('Removing previous builds…') + rmtree(os.path.join(here, 'dist')) + except OSError: + pass + + self.status('Building Source and Wheel (universal) distribution…') + os.system('{0} setup.py sdist bdist_wheel --universal'.format(sys.executable)) + + # self.status('Uploading the package to PyPi via Twine…') + # os.system('twine upload dist/*') + + sys.exit() + + +# Where the magic happens: +setup( + name=NAME, + # version=about['__version__'], + # description=DESCRIPTION, + # long_description=long_description, + # author=AUTHOR, + author_email=EMAIL, + # url=URL, + packages=find_packages(exclude=('tests',)), + # If your package is a single module, use this instead of 'packages': + # py_modules=['mypackage'], + + # entry_points={ + # 'console_scripts': ['mycli=mymodule:cli'], + # }, + install_requires=REQUIRED, + include_package_data=True, + # license='MIT', + classifiers=[ + # Trove classifiers + # Full list: https://pypi.python.org/pypi?%3Aaction=list_classifiers + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: Implementation :: CPython', + 'Programming Language :: Python :: Implementation :: PyPy' + ], + # $ setup.py publish support. + cmdclass={ + 'upload': UploadCommand, + }, +) -- GitLab From 28629fc85ee73cf9a5c5497fb9651be5e5afb03f Mon Sep 17 00:00:00 2001 From: wjurasz <wiktor.jan.jurasz@cern.ch> Date: Tue, 6 Feb 2018 09:44:59 +0100 Subject: [PATCH 02/24] NXCALS-14681468146814681468146814681468146814681468146814681468 Import tests. --- .../cern/__init__.py | 0 .../cern/accsoft/__init__.py | 0 .../cern/accsoft/nxcals/__init__.py | 0 .../cern/accsoft/nxcals/pyquery/__init__.py | 0 .../nxcals/pyquery/builders/__init__.py | 1 + .../accsoft/nxcals/pyquery/builders/test1.py | 2 + .../accsoft/nxcals/pyquery/builders/test2.py | 1 + accsoft-nxcals-data-access-python3/setup.py | 68 +------------------ 8 files changed, 6 insertions(+), 66 deletions(-) create mode 100644 accsoft-nxcals-data-access-python3/cern/__init__.py create mode 100644 accsoft-nxcals-data-access-python3/cern/accsoft/__init__.py create mode 100644 accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/__init__.py create mode 100644 accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/__init__.py create mode 100644 accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/__init__.py create mode 100644 accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/test1.py create mode 100644 accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/test2.py diff --git a/accsoft-nxcals-data-access-python3/cern/__init__.py b/accsoft-nxcals-data-access-python3/cern/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/accsoft-nxcals-data-access-python3/cern/accsoft/__init__.py b/accsoft-nxcals-data-access-python3/cern/accsoft/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/__init__.py b/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/__init__.py b/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/__init__.py b/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/__init__.py new file mode 100644 index 0000000000..98a96b6785 --- /dev/null +++ b/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/__init__.py @@ -0,0 +1 @@ +from cern.accsoft.nxcals.pyquery.builders.test1 import foo \ No newline at end of file diff --git a/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/test1.py b/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/test1.py new file mode 100644 index 0000000000..f71e5e37df --- /dev/null +++ b/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/test1.py @@ -0,0 +1,2 @@ +def foo(a): + return a + 1 diff --git a/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/test2.py b/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/test2.py new file mode 100644 index 0000000000..ae1d007f69 --- /dev/null +++ b/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/test2.py @@ -0,0 +1 @@ +print(foo(5)) \ No newline at end of file diff --git a/accsoft-nxcals-data-access-python3/setup.py b/accsoft-nxcals-data-access-python3/setup.py index 698f9be64a..93cc9c9926 100644 --- a/accsoft-nxcals-data-access-python3/setup.py +++ b/accsoft-nxcals-data-access-python3/setup.py @@ -1,27 +1,16 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- - -# import io -import os -import sys -from shutil import rmtree - -from setuptools import find_packages, setup, Command +from setuptools import find_packages, setup # Package meta-data. NAME = 'accsoft-nxcals-data-access-python3' -# DESCRIPTION = 'My short description for my project.' -# URL = 'https://github.com/me/myproject' EMAIL = 'acclog@cern.ch' -# AUTHOR = 'BE-CE' # What packages are required for this module to be executed? REQUIRED = [ # 'requests', 'maya', 'records', ] -here = os.path.abspath(os.path.dirname(__file__)) - # Import the README and use it as the long-description. # Note: this will only work if 'README.rst' is present in your MANIFEST.in file! # with io.open(os.path.join(here, 'README.rst'), encoding='utf-8') as f: @@ -33,75 +22,22 @@ here = os.path.abspath(os.path.dirname(__file__)) # exec(f.read(), about) -class UploadCommand(Command): - """Support setup.py upload.""" - - description = 'Build and publish the package.' - user_options = [] - - @staticmethod - def status(s): - """Prints things in bold.""" - print('\033[1m{0}\033[0m'.format(s)) - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - try: - self.status('Removing previous builds…') - rmtree(os.path.join(here, 'dist')) - except OSError: - pass - - self.status('Building Source and Wheel (universal) distribution…') - os.system('{0} setup.py sdist bdist_wheel --universal'.format(sys.executable)) - - # self.status('Uploading the package to PyPi via Twine…') - # os.system('twine upload dist/*') - - sys.exit() - - # Where the magic happens: setup( name=NAME, - # version=about['__version__'], - # description=DESCRIPTION, - # long_description=long_description, - # author=AUTHOR, author_email=EMAIL, - # url=URL, packages=find_packages(exclude=('tests',)), - # If your package is a single module, use this instead of 'packages': - # py_modules=['mypackage'], - - # entry_points={ - # 'console_scripts': ['mycli=mymodule:cli'], - # }, install_requires=REQUIRED, include_package_data=True, - # license='MIT', classifiers=[ # Trove classifiers # Full list: https://pypi.python.org/pypi?%3Aaction=list_classifiers 'License :: OSI Approved :: MIT License', 'Programming Language :: Python', - 'Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: Implementation :: CPython', - 'Programming Language :: Python :: Implementation :: PyPy' + 'Programming Language :: Python :: 3.6' ], - # $ setup.py publish support. - cmdclass={ - 'upload': UploadCommand, - }, ) -- GitLab From 1d00fd386d998ad5fd547d12a95e40a4dd2f9363 Mon Sep 17 00:00:00 2001 From: wjurasz <wiktor.jan.jurasz@cern.ch> Date: Fri, 9 Feb 2018 09:26:33 +0100 Subject: [PATCH 03/24] NXCALS-1468 gradle setup.py integration done --- .gitignore | 3 + .../build.gradle | 2 + .../build.gradle | 46 +++++++ .../builders/DataAccessQueryBuilder.py | 111 +++++++++++++++++ .../builders/DevicePropertyQueryBuilder.py | 43 +++++++ .../builders/DevicePropertyQueryData.py | 55 +++++++++ .../nxcals/pyquery/builders/QueryData.py | 112 ++++++++++++++++++ .../nxcals/pyquery/builders/TimeUtils.py | 26 ++++ .../pyquery/builders/VariableQueryBuilder.py | 50 ++++++++ .../pyquery/builders/VariableQueryData.py | 41 +++++++ .../nxcals/pyquery/builders/__init__.py | 1 - .../accsoft/nxcals/pyquery/builders/test1.py | 2 - .../accsoft/nxcals/pyquery/builders/test2.py | 1 - accsoft-nxcals-data-access-python3/setup.py | 29 +++-- settings.gradle | 2 + 15 files changed, 505 insertions(+), 19 deletions(-) create mode 100644 accsoft-nxcals-data-access-python3/build.gradle create mode 100644 accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/DataAccessQueryBuilder.py create mode 100644 accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/DevicePropertyQueryBuilder.py create mode 100644 accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/DevicePropertyQueryData.py create mode 100644 accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/QueryData.py create mode 100644 accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/TimeUtils.py create mode 100644 accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/VariableQueryBuilder.py create mode 100644 accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/VariableQueryData.py delete mode 100644 accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/test1.py delete mode 100644 accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/test2.py diff --git a/.gitignore b/.gitignore index a88b3f2e34..d2bb2347d8 100644 --- a/.gitignore +++ b/.gitignore @@ -56,8 +56,11 @@ accsoft-nxcals-data-access-python/**/*.pyc accsoft-nxcals-data-access-python/**/dist/ # Python egg metadata, regenerated from source files by setuptools. +accsoft-nxcals-data-access-python3/*.egg-info +accsoft-nxcals-data-access-python3/**/*.egg-info accsoft-nxcals-data-access-python/**/*.egg-info accsoft-nxcals-data-access-python/**/*.egg +*.whl #PyGradle modules accsoft-nxcals-data-access-python/.cache diff --git a/accsoft-nxcals-data-access-python/build.gradle b/accsoft-nxcals-data-access-python/build.gradle index 3672b8bad6..129e635afb 100644 --- a/accsoft-nxcals-data-access-python/build.gradle +++ b/accsoft-nxcals-data-access-python/build.gradle @@ -71,6 +71,8 @@ task cleanGenerated() { } + + distTar.dependsOn(tasks.generateSetup) distTar.finalizedBy tasks.cleanGenerated diff --git a/accsoft-nxcals-data-access-python3/build.gradle b/accsoft-nxcals-data-access-python3/build.gradle new file mode 100644 index 0000000000..660ace35b4 --- /dev/null +++ b/accsoft-nxcals-data-access-python3/build.gradle @@ -0,0 +1,46 @@ +println "Module=" + name +println "Version=" + version + +apply plugin: 'org.sonarqube' + +def pathToPythonInterpreter = '/user/bdisoft/operational/bin/Python/anaconda_py35/bin/python' + +task testPythonVersion() { + def stdout = new ByteArrayOutputStream() + exec { + commandLine 'python', '--version' + errorOutput = stdout + } + + if (stdout.toString().startsWith('Python 3')) { + pathToPythonInterpreter = 'python' + } else if (project.hasProperty('pythonPath')) { + pathToPythonInterpreter = pythonPath + } //else check if default python exists and if no the error message that client has to provide python3 interpreter +} + + +task distWheel(type: Exec) { + commandLine pathToPythonInterpreter, 'setup.py', 'bdist_wheel', "--nxcals-version=$currentVersion" +} + + +task cleanWheel(type: Delete) { + exec { + commandLine pathToPythonInterpreter, 'setup.py', 'clean', '--all', "--nxcals-version=$currentVersion" + } + delete 'dist', 'accsoft_nxcals_data_access_python3.egg-info' +} + +sonarqube { + properties { + property "sonar.python.coverage.reportPath", "$projectDir/coverage.xml" + property "sonar.sources", "$projectDir/src/main" + } +} + + +build.dependsOn(testPythonVersion) +build.dependsOn(distWheel) +clean.dependsOn(testPythonVersion) +clean.dependsOn(cleanWheel) \ No newline at end of file diff --git a/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/DataAccessQueryBuilder.py b/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/DataAccessQueryBuilder.py new file mode 100644 index 0000000000..d778a1e365 --- /dev/null +++ b/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/DataAccessQueryBuilder.py @@ -0,0 +1,111 @@ +import json + +from .QueryData import * +from .TimeUtils import getNanosFromDateTime, convertToDatetime, getEndTimeInNanos + +class DataAccessQueryBuilder(object): + + def __init__(self): + self._system = None + self._device = None + self._key_values = {} + self._start_time = None + self._end_time = None + self._fields = None + self._alias_fields = {} + + def system(self, system): + if system is None: + raise AssertionError("System cannot be None !") + self._system = system + return DataAccessQueryBuilder._KeyValues(self) + + class _KeyValues(object): + def __init__(self, builder): + self._builder = builder + + def key_values(self, **key_values): + if not key_values or None in key_values.values(): + raise AssertionError("Method key_Value() should be called only when specifying given fields !") + self._builder._key_values.update(key_values) + return DataAccessQueryBuilder._StartTime(self._builder) + + def key_value(self, key, value): + return DataAccessQueryBuilder._KeyValue(self._builder).key_value(key, value) + + class _StartTime(object): + def __init__(self, builder): + self._builder = builder + + def start_time(self, start_time): + self._builder._start_time = getNanosFromDateTime(convertToDatetime(start_time)) + return DataAccessQueryBuilder._EndTime(self._builder) + + def at_time(self, at_time): + self._builder._start_time = getNanosFromDateTime(convertToDatetime(at_time)) + return DataAccessQueryBuilder._EndTime(self._builder).end_time(at_time) + + class _KeyValue(_StartTime): + def __init__(self, builder): + self._builder = builder + + def key_value(self, key, value): + if key is None or not isinstance(key, basestring) or value is None or not isinstance(value, basestring): + raise AssertionError("Key and Value must not be None and must be a basestring") + self._builder._key_values.update({key: value}) + return self + + class _EndTime(object): + def __init__(self, builder): + self._builder = builder + + def end_time(self, end_time): + self._builder._end_time = getNanosFromDateTime(convertToDatetime(end_time)) + return DataAccessQueryBuilder._Fields(self._builder) + + def duration(self, duration): + self._builder._end_time = getEndTimeInNanos(self._builder._start_time, duration) + return DataAccessQueryBuilder._Fields(self._builder) + + class _Fields(object): + def __init__(self, builder): + self._builder = builder + + def fields(self, *fields): + if None in fields: + raise AssertionError("Method fields() cannot get None as an attribute !") + if fields: + self._builder._fields = str.join(",", fields) + return self + + def alias_fields(self, alias, *fields): + if alias is None or None in fields: + raise AssertionError("Method alias_fields cannot get None as an attribute !") + if fields: + self._builder._alias_fields[alias] = fields + return self + + def build(self): + query = dict() + query["system"] = self._builder._system + query["start_time"] = self._builder._start_time + query["end_time"] = self._builder._end_time + + if self._builder._key_values or self._builder._key_values is not None: + query['key_values'] = json.dumps(self._builder._key_values) + + if self._builder._fields or self._builder._fields is not None: + query["fields_key"] = self._builder._fields + + if self._builder._alias_fields or self._builder._alias_fields is not None: + query["aliases_key"] = json.dumps(self._builder._alias_fields) + return QueryData(query) + + def buildDataset(self, session): + imported_py_sql_session = __import__('pyspark.sql.session', globals(), locals(), ['SparkSession'], 0) + if type(session) is not imported_py_sql_session.SparkSession: + raise AssertionError("Session must be a SparkSession instance!") + return session.read\ + .options(**self.build().get_map())\ + .format("cern.accsoft.nxcals.data.access.api")\ + .load() \ No newline at end of file diff --git a/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/DevicePropertyQueryBuilder.py b/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/DevicePropertyQueryBuilder.py new file mode 100644 index 0000000000..d3ee3cb8ea --- /dev/null +++ b/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/DevicePropertyQueryBuilder.py @@ -0,0 +1,43 @@ +from .DataAccessQueryBuilder import * + + +class DevicePropertyQueryBuilder(DataAccessQueryBuilder): + def system(self, system): + if system is None: + raise AssertionError("System cannot be None !") + else: + self._system = system + return _Device(self) + + +class _Device(object): + def __init__(self, builder): + self._builder = builder + + def device(self, device): + if device is None: + raise AssertionError("Device cannot be None !") + self._builder._key_values.update(device=device) + return _Property(self._builder) + + def parameter(self, parameter): + if parameter is None: + raise AssertionError("Parameter cannot be None !") + + __param = str(parameter).split("/") + if len(__param) != 2: + raise AssertionError("Parameter must contain device/property value!") + self._builder._key_values.update(device=__param[0]) + self._builder._key_values.update(property=__param[1]) + return DataAccessQueryBuilder._StartTime(self._builder) + + +class _Property(object): + def __init__(self, builder): + self._builder = builder + + def property(self, property): + if property is None: + raise AssertionError("Property cannot be None !") + self._builder._key_values.update(property=property) + return DataAccessQueryBuilder._StartTime(self._builder) \ No newline at end of file diff --git a/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/DevicePropertyQueryData.py b/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/DevicePropertyQueryData.py new file mode 100644 index 0000000000..49c3c2cc33 --- /dev/null +++ b/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/DevicePropertyQueryData.py @@ -0,0 +1,55 @@ +from .QueryData import QueryData +import json + +class DevicePropertyQueryData(QueryData): + def __init__(self, query): + self._query = query + + def get_map(self): + return self._query + + @staticmethod + def modify_query(query_data): + return DevicePropertyQueryData._StageDevicePropertyQueryData(query_data.get_map()) + + class _StageDevicePropertyQueryData(QueryData._StageQueryData): + def __init__(self, query): + self._query = query.copy() + + def system(self, system): + if system is None: + raise AssertionError("System cannot be None !") + self._query.update({"system":system}) + return self + + def device(self, device): + if device is None: + raise AssertionError("Device cannot be None !") + temporaryDict = json.loads(self._query['key_values']) + temporaryDict['device'] = device + self._query['key_values'] = json.dumps(temporaryDict) + return self + + def parameter(self, parameter): + if parameter is None: + raise AssertionError("Parameter cannot be None !") + + __param = str(parameter).split("/") + if len(__param) != 2: + raise AssertionError("Parameter must contain device/property value!") + temporaryDict = json.loads(self._query['key_values']) + temporaryDict['device'] = __param[0] + temporaryDict['property'] = __param[1] + self._query['key_values'] = json.dumps(temporaryDict) + return self + + def property(self, property): + if property is None: + raise AssertionError("Property cannot be None !") + temporaryDict = json.loads(self._query['key_values']) + temporaryDict['property'] = property + self._query['key_values'] = json.dumps(temporaryDict) + return self + + def build(self): + return QueryData(self._query) diff --git a/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/QueryData.py b/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/QueryData.py new file mode 100644 index 0000000000..c4d1b11ff8 --- /dev/null +++ b/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/QueryData.py @@ -0,0 +1,112 @@ +from .TimeUtils import getNanosFromDateTime, convertToDatetime +from datetime import datetime +import json + + +class QueryData(object): + def __init__(self, query): + self._query = query + + def get_map(self): + return self._query + + @staticmethod + def modify_query(query_data): + return QueryData._StageQueryData(query_data.get_map()) + + class _StageQueryData(object): + def __init__(self, query): + self._query = query.copy() + + def system(self, system): + if system is None: + raise AssertionError("System cannot be None !") + self._query.update({"system":system}) + return self + + def key_values(self, **key_values): + if not key_values or None in key_values.values(): + raise AssertionError("Method key_Value() should be called only when specifying given fields !") + temporaryDict = json.loads(self._query['key_values']) + for key, value in key_values.iteritems(): + temporaryDict.update({key:value}) + self._query['key_values'] = json.dumps(temporaryDict) + return self + + def remove_keys(self, *keys): + if None in keys: + raise AssertionError("The key cannot be None !") + temporaryDict = json.loads(self._query['key_values']) + for key in keys: + del temporaryDict[key] + self._query['key_values'] = json.dumps(temporaryDict) + return self + + def start_time(self, start_time): + start_time = convertToDatetime(start_time) + if type(start_time) is not datetime or start_time is None: + raise AssertionError("Start time must be a datatime object !") + self._query.update({"start_time":getNanosFromDateTime(start_time)}) + return self + + def end_time(self, end_time): + end_time = convertToDatetime(end_time) + if type(end_time) is not datetime or end_time is None: + raise AssertionError("End time must be a datatime object !") + self._query.update({"end_time":getNanosFromDateTime(end_time)}) + return self + + def replace_all_fields(self, *fields): + if None in fields: + raise AssertionError("Method replace_all_fields() cannot get None as an attribute !") + if fields: + self._query.update({"fields_key":str.join(",", fields)}) + return self + + def add_fields(self, *fields): + if None in fields: + raise AssertionError("Method include_fields() cannot get None as an attribute !") + if fields: + query_fields = self._query.get("fields_key").split(",") + new_fields = str.join(",", fields).split(",") + new_fields_set = set(query_fields + new_fields) + self._query.update({"fields_key":str.join(",", new_fields_set)}) + return self + + def remove_fields(self, *fields): + if None in fields: + raise AssertionError("Method exclude_fields() cannot get None as an attribute !") + if fields: + query_fields = self._query.get("fields_key").split(",") + for field in fields: + query_fields.remove(field) + self._query.update({"fields_key": str.join(",", query_fields)}) + return self + + def include_all_fields(self): + self._query.pop("fields_key") + return self + + def remove_all_aliases(self): + self._query.pop("aliases_key") + return self + + def add_alias(self, alias, *fields): + if alias is None or None in fields: + raise AssertionError("Method add_alias cannot get None as an attribute !") + if fields: + alias_fields = json.loads(self._query.get("aliases_key")) + alias_fields[alias] = fields + self._query.update({"aliases_key": json.dumps(alias_fields)}) + return self + + def remove_alias(self, alias): + if alias is None: + raise AssertionError("Method remove_alias cannot get None as an attribute !") + alias_fields = json.loads(self._query.get("aliases_key")) + alias_fields.pop(alias) + self._query.update({"aliases_key": json.dumps(alias_fields)}) + return self + + def build(self): + return QueryData(self._query) diff --git a/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/TimeUtils.py b/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/TimeUtils.py new file mode 100644 index 0000000000..ed96319972 --- /dev/null +++ b/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/TimeUtils.py @@ -0,0 +1,26 @@ +import time +from datetime import datetime, timedelta +from decimal import Decimal + +def getNanosFromDateTime(t): + if t is None or type(t) is not datetime: + raise AssertionError("The provided value is not datetime !") + return long((Decimal(time.mktime(t.timetuple())) + Decimal(t.microsecond)/1000000)*1000000000) + + +def convertToDatetime(t): + if t is None: + raise AssertionError("Datetime value cannot be None !") + if type(t) is datetime: + return t + try: + converted_time = datetime.strptime(t, '%Y-%m-%d %H:%M:%S.%f') + return converted_time + except BaseException: + raise AssertionError("The used datetime format doesn't respect pattern %Y-%m-%d %H:%M:%S.%f") + +def getEndTimeInNanos(start_time, duration): + if type(duration) is not timedelta or type(start_time) is not long: + raise AssertionError("Duration must be datetime.timedelta instance and start_time must be long") + duration_nanos = (duration.days*86400000 + duration.seconds*1000 + duration.microseconds/1000) * 1000 + return start_time + duration_nanos diff --git a/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/VariableQueryBuilder.py b/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/VariableQueryBuilder.py new file mode 100644 index 0000000000..c7f0ca511f --- /dev/null +++ b/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/VariableQueryBuilder.py @@ -0,0 +1,50 @@ +from .QueryData import * +from .TimeUtils import getNanosFromDateTime, convertToDatetime +from datetime import datetime + + +class VariableQueryBuilder(object): + def __init__(self): + self._variable = None + self._start_time = None + self._end_time = None + + def variable(self, variable): + if variable is None: + raise AssertionError("Variable cannot be None !") + self._variable = variable + return VariableQueryBuilder._StartTime(self) + + class _StartTime(object): + def __init__(self, builder): + self._builder = builder + + def start_time(self, start_time): + start_time = convertToDatetime(start_time) + if type(start_time) is not datetime or start_time is None: + raise AssertionError("Start time must be a datatime object !") + self._builder._start_time = getNanosFromDateTime(start_time) + return VariableQueryBuilder._EndTime(self._builder) + + class _EndTime(object): + def __init__(self, builder): + self._builder = builder + + def end_time(self, end_time): + end_time = convertToDatetime(end_time) + if type(end_time) is not datetime or end_time is None: + raise AssertionError("End time must be a datatime object !") + self._builder._end_time = getNanosFromDateTime(end_time) + return VariableQueryBuilder._BuildQuery(self._builder) + + class _BuildQuery(object): + def __init__(self, builder): + self._builder = builder + + def build(self): + query = dict() + query["system"] = 'VARIABLE' + query["variable"] = self._builder._variable + query["start_time"] = self._builder._start_time + query["end_time"] = self._builder._end_time + return QueryData(query) \ No newline at end of file diff --git a/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/VariableQueryData.py b/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/VariableQueryData.py new file mode 100644 index 0000000000..fc44a045c6 --- /dev/null +++ b/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/VariableQueryData.py @@ -0,0 +1,41 @@ +from .TimeUtils import getNanosFromDateTime, convertToDatetime +from datetime import datetime + + +class VariableQueryData(object): + def __init__(self, query): + self._query = query + + def get_map(self): + return self._query + + @staticmethod + def modify_query(query_data): + return VariableQueryData._StageVariablesQueryData(query_data.get_map()) + + class _StageVariablesQueryData(object): + def __init__(self, query): + self._query = query.copy() + + def variable(self, variable): + if variable is None: + raise AssertionError("Variable cannot be None !") + self._query.update({"variable":variable}) + return self + + def start_time(self, start_time): + start_time = convertToDatetime(start_time) + if type(start_time) is not datetime or start_time is None: + raise AssertionError("Start time must be a datatime object !") + self._query.update({"start_time":getNanosFromDateTime(start_time)}) + return self + + def end_time(self, end_time): + end_time = convertToDatetime(end_time) + if type(end_time) is not datetime or end_time is None: + raise AssertionError("End time must be a datatime object !") + self._query.update({"end_time":getNanosFromDateTime(end_time)}) + return self + + def build(self): + return VariableQueryData(self._query) \ No newline at end of file diff --git a/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/__init__.py b/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/__init__.py index 98a96b6785..e69de29bb2 100644 --- a/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/__init__.py +++ b/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/__init__.py @@ -1 +0,0 @@ -from cern.accsoft.nxcals.pyquery.builders.test1 import foo \ No newline at end of file diff --git a/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/test1.py b/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/test1.py deleted file mode 100644 index f71e5e37df..0000000000 --- a/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/test1.py +++ /dev/null @@ -1,2 +0,0 @@ -def foo(a): - return a + 1 diff --git a/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/test2.py b/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/test2.py deleted file mode 100644 index ae1d007f69..0000000000 --- a/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/test2.py +++ /dev/null @@ -1 +0,0 @@ -print(foo(5)) \ No newline at end of file diff --git a/accsoft-nxcals-data-access-python3/setup.py b/accsoft-nxcals-data-access-python3/setup.py index 93cc9c9926..7b6d4295ec 100644 --- a/accsoft-nxcals-data-access-python3/setup.py +++ b/accsoft-nxcals-data-access-python3/setup.py @@ -1,6 +1,17 @@ -#!/usr/bin/env python +#!/opt/wjurasz/python3/installation/bin/python3 # -*- coding: utf-8 -*- from setuptools import find_packages, setup +import sys +import argparse + +version_argument = '--nxcals-version' + +parser = argparse.ArgumentParser() +parser.add_argument(version_argument, required=True, help='Version which will be visible in .whl file') +args, _ = parser.parse_known_args() + +# This is because setup() also parses command line and complains on unrecognized options +sys.argv.remove(version_argument + '=' + args.nxcals_version) # Package meta-data. NAME = 'accsoft-nxcals-data-access-python3' @@ -11,28 +22,16 @@ REQUIRED = [ # 'requests', 'maya', 'records', ] -# Import the README and use it as the long-description. -# Note: this will only work if 'README.rst' is present in your MANIFEST.in file! -# with io.open(os.path.join(here, 'README.rst'), encoding='utf-8') as f: -# long_description = '\n' + f.read() - -# Load the package's __version__.py module as a dictionary. -# about = {} -# with open(os.path.join(here, NAME, '__version__.py')) as f: -# exec(f.read(), about) - - # Where the magic happens: setup( name=NAME, + version=args.nxcals_version, + description='Python data extraction API for NXCALS system', author_email=EMAIL, packages=find_packages(exclude=('tests',)), install_requires=REQUIRED, include_package_data=True, classifiers=[ - # Trove classifiers - # Full list: https://pypi.python.org/pypi?%3Aaction=list_classifiers - 'License :: OSI Approved :: MIT License', 'Programming Language :: Python', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.3', diff --git a/settings.gradle b/settings.gradle index 639a7fe783..ca5b387367 100644 --- a/settings.gradle +++ b/settings.gradle @@ -27,3 +27,5 @@ include 'accsoft-nxcals-monitoring-grok' include 'accsoft-nxcals-data-access-demo' include 'accsoft-nxcals-integration-tests' include 'accsoft-nxcals-service-db' + +include 'accsoft-nxcals-data-access-python3' -- GitLab From 2875aced38543248064bc388fa6bfc7381499704 Mon Sep 17 00:00:00 2001 From: wjurasz <wiktor.jan.jurasz@cern.ch> Date: Fri, 9 Feb 2018 13:41:24 +0100 Subject: [PATCH 04/24] NXCALS-1468 reverted python2 changes --- accsoft-nxcals-data-access-python/build.gradle | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/accsoft-nxcals-data-access-python/build.gradle b/accsoft-nxcals-data-access-python/build.gradle index 129e635afb..c909fb8a0f 100644 --- a/accsoft-nxcals-data-access-python/build.gradle +++ b/accsoft-nxcals-data-access-python/build.gradle @@ -30,8 +30,12 @@ distributions { python { testDir = file('src/test') srcDir = file('src/main') - details.setSystemPythonInterpreter('/user/bdisoft/operational/bin/Python/anaconda_py35/bin/python') - details.prependExecutableDirectory(new File('/user/bdisoft/operational/bin/Python/anaconda_py35/lib')) + details { + //virtualEnvPrompt = "(${project.name})" + //activateLink = project.file("activate") // File you can source to activate the virtual env + pythonVersion = '2.7' // Sets the version of python to use, will search your PATH to get the location + systemPythonInterpreter = file("/usr/bin/python2.7") // used to force an interpreter to be used to build the venv + } } test.dependsOn(pytest) @@ -71,8 +75,6 @@ task cleanGenerated() { } - - distTar.dependsOn(tasks.generateSetup) distTar.finalizedBy tasks.cleanGenerated -- GitLab From 6b67039c4f1ad899202ea51adee82f41be316f85 Mon Sep 17 00:00:00 2001 From: wjurasz <wiktor.jan.jurasz@cern.ch> Date: Fri, 9 Feb 2018 15:02:24 +0100 Subject: [PATCH 05/24] NXCALS-1468 fixed sonar path --- accsoft-nxcals-data-access-python3/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/accsoft-nxcals-data-access-python3/build.gradle b/accsoft-nxcals-data-access-python3/build.gradle index 660ace35b4..9c86b17e8d 100644 --- a/accsoft-nxcals-data-access-python3/build.gradle +++ b/accsoft-nxcals-data-access-python3/build.gradle @@ -35,7 +35,7 @@ task cleanWheel(type: Delete) { sonarqube { properties { property "sonar.python.coverage.reportPath", "$projectDir/coverage.xml" - property "sonar.sources", "$projectDir/src/main" + property "sonar.sources", "$projectDir/cern" } } -- GitLab From 1f2b131f68d281286ac6354a751d75ca3c7b70ef Mon Sep 17 00:00:00 2001 From: wjurasz <wiktor.jan.jurasz@cern.ch> Date: Fri, 9 Feb 2018 17:04:01 +0100 Subject: [PATCH 06/24] NXCALS-1468 improved build.gradle --- .../build.gradle | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/accsoft-nxcals-data-access-python3/build.gradle b/accsoft-nxcals-data-access-python3/build.gradle index 9c86b17e8d..4e2ba8967d 100644 --- a/accsoft-nxcals-data-access-python3/build.gradle +++ b/accsoft-nxcals-data-access-python3/build.gradle @@ -5,6 +5,11 @@ apply plugin: 'org.sonarqube' def pathToPythonInterpreter = '/user/bdisoft/operational/bin/Python/anaconda_py35/bin/python' +def pythonProjectName = name.replace('-', '_') +def pythonProjectVersion = version.replace('-', '_') +def wheelArtifactName = pythonProjectName + '-' + pythonProjectVersion + '-py3-none-any.whl' +def pythonPathVariableName = 'pythonPath' + task testPythonVersion() { def stdout = new ByteArrayOutputStream() exec { @@ -14,9 +19,15 @@ task testPythonVersion() { if (stdout.toString().startsWith('Python 3')) { pathToPythonInterpreter = 'python' - } else if (project.hasProperty('pythonPath')) { + } else if (project.hasProperty(pythonPathVariableName)) { pathToPythonInterpreter = pythonPath - } //else check if default python exists and if no the error message that client has to provide python3 interpreter + } else { + if (!file(pathToPythonInterpreter).exists()) { + throw new GradleException("To proceed python3 interpreter has to be provided, either as default intepreter (ran by \"python\" command) or by specifying -P$pythonPathVariableName=/path/to/python3 gradle option") + } + } + + //else check if default python exists and if no the error message that client has to provide python3 interpreter } @@ -29,7 +40,7 @@ task cleanWheel(type: Delete) { exec { commandLine pathToPythonInterpreter, 'setup.py', 'clean', '--all', "--nxcals-version=$currentVersion" } - delete 'dist', 'accsoft_nxcals_data_access_python3.egg-info' + delete 'dist', pythonProjectName + '.egg-info' } sonarqube { @@ -39,7 +50,12 @@ sonarqube { } } +artifacts { + archives file('dist/' + wheelArtifactName) +} +install.dependsOn(testPythonVersion) +install.dependsOn(distWheel) build.dependsOn(testPythonVersion) build.dependsOn(distWheel) clean.dependsOn(testPythonVersion) -- GitLab From dbb72b1a8018a44825e11dc34759841604b839c1 Mon Sep 17 00:00:00 2001 From: wjurasz <wiktor.jan.jurasz@cern.ch> Date: Mon, 12 Feb 2018 10:33:19 +0100 Subject: [PATCH 07/24] NXCALS-1468 temorary changed project groupid --- build.gradle | 2 +- settings.gradle | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index b18a802bc0..e8f955c13a 100644 --- a/build.gradle +++ b/build.gradle @@ -27,7 +27,7 @@ subprojects { apply plugin: 'maven' apply plugin: 'jacoco' - group 'cern.accsoft.nxcals' + group 'python3test.cern.accsoft.nxcals' version = currentVersion repositories { diff --git a/settings.gradle b/settings.gradle index ca5b387367..72561f7f9e 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,5 +1,8 @@ import org.gradle.internal.os.OperatingSystem + +rootProject.name='python3test' + include 'accsoft-nxcals-ansible' include 'accsoft-nxcals-common' include 'accsoft-nxcals-common-spark' -- GitLab From 9d327dc6702535b6c222b2d7883b19d6a39205cd Mon Sep 17 00:00:00 2001 From: wjurasz <wiktor.jan.jurasz@cern.ch> Date: Mon, 12 Feb 2018 11:11:21 +0100 Subject: [PATCH 08/24] NXCALS-1468 gradle.build modified --- accsoft-nxcals-data-access-python3/build.gradle | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/accsoft-nxcals-data-access-python3/build.gradle b/accsoft-nxcals-data-access-python3/build.gradle index 4e2ba8967d..4190a5a472 100644 --- a/accsoft-nxcals-data-access-python3/build.gradle +++ b/accsoft-nxcals-data-access-python3/build.gradle @@ -54,9 +54,7 @@ artifacts { archives file('dist/' + wheelArtifactName) } -install.dependsOn(testPythonVersion) +distWheel.dependsOn(testPythonVersion) install.dependsOn(distWheel) -build.dependsOn(testPythonVersion) build.dependsOn(distWheel) -clean.dependsOn(testPythonVersion) clean.dependsOn(cleanWheel) \ No newline at end of file -- GitLab From 5d6f7606f31170b2a36e4d5e2beadf96593c89bb Mon Sep 17 00:00:00 2001 From: wjurasz <wiktor.jan.jurasz@cern.ch> Date: Mon, 12 Feb 2018 11:50:28 +0100 Subject: [PATCH 09/24] NXCALS-1468 build scripts done --- accsoft-nxcals-data-access-python3/build.gradle | 1 - accsoft-nxcals-data-access-python3/setup.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/accsoft-nxcals-data-access-python3/build.gradle b/accsoft-nxcals-data-access-python3/build.gradle index 4190a5a472..e0adb38bf2 100644 --- a/accsoft-nxcals-data-access-python3/build.gradle +++ b/accsoft-nxcals-data-access-python3/build.gradle @@ -27,7 +27,6 @@ task testPythonVersion() { } } - //else check if default python exists and if no the error message that client has to provide python3 interpreter } diff --git a/accsoft-nxcals-data-access-python3/setup.py b/accsoft-nxcals-data-access-python3/setup.py index 7b6d4295ec..5a68aea0e7 100644 --- a/accsoft-nxcals-data-access-python3/setup.py +++ b/accsoft-nxcals-data-access-python3/setup.py @@ -10,7 +10,7 @@ parser = argparse.ArgumentParser() parser.add_argument(version_argument, required=True, help='Version which will be visible in .whl file') args, _ = parser.parse_known_args() -# This is because setup() also parses command line and complains on unrecognized options +# This is because setup() also parses command line and complains about unrecognized options sys.argv.remove(version_argument + '=' + args.nxcals_version) # Package meta-data. -- GitLab From 9d9fe8a9c52483536fdaea7d694540b0982a80a4 Mon Sep 17 00:00:00 2001 From: wjurasz <wiktor.jan.jurasz@cern.ch> Date: Mon, 12 Feb 2018 14:30:31 +0100 Subject: [PATCH 10/24] NXCALS-1468 removed temporary name --- build.gradle | 2 +- settings.gradle | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/build.gradle b/build.gradle index e8f955c13a..b18a802bc0 100644 --- a/build.gradle +++ b/build.gradle @@ -27,7 +27,7 @@ subprojects { apply plugin: 'maven' apply plugin: 'jacoco' - group 'python3test.cern.accsoft.nxcals' + group 'cern.accsoft.nxcals' version = currentVersion repositories { diff --git a/settings.gradle b/settings.gradle index 72561f7f9e..639a7fe783 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,8 +1,5 @@ import org.gradle.internal.os.OperatingSystem - -rootProject.name='python3test' - include 'accsoft-nxcals-ansible' include 'accsoft-nxcals-common' include 'accsoft-nxcals-common-spark' @@ -30,5 +27,3 @@ include 'accsoft-nxcals-monitoring-grok' include 'accsoft-nxcals-data-access-demo' include 'accsoft-nxcals-integration-tests' include 'accsoft-nxcals-service-db' - -include 'accsoft-nxcals-data-access-python3' -- GitLab From 5b4f47c44b23dc40614f775dd24decfe1003ad91 Mon Sep 17 00:00:00 2001 From: wjurasz <wiktor.jan.jurasz@cern.ch> Date: Mon, 12 Feb 2018 14:31:02 +0100 Subject: [PATCH 11/24] NXCALS-1468 add temporary name --- build.gradle | 2 +- settings.gradle | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index b18a802bc0..e8f955c13a 100644 --- a/build.gradle +++ b/build.gradle @@ -27,7 +27,7 @@ subprojects { apply plugin: 'maven' apply plugin: 'jacoco' - group 'cern.accsoft.nxcals' + group 'python3test.cern.accsoft.nxcals' version = currentVersion repositories { diff --git a/settings.gradle b/settings.gradle index 639a7fe783..72561f7f9e 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,5 +1,8 @@ import org.gradle.internal.os.OperatingSystem + +rootProject.name='python3test' + include 'accsoft-nxcals-ansible' include 'accsoft-nxcals-common' include 'accsoft-nxcals-common-spark' @@ -27,3 +30,5 @@ include 'accsoft-nxcals-monitoring-grok' include 'accsoft-nxcals-data-access-demo' include 'accsoft-nxcals-integration-tests' include 'accsoft-nxcals-service-db' + +include 'accsoft-nxcals-data-access-python3' -- GitLab From 680ce1b85d5538c38573f33d6befe6500538b1da Mon Sep 17 00:00:00 2001 From: wjurasz <wiktor.jan.jurasz@cern.ch> Date: Mon, 12 Feb 2018 15:09:19 +0100 Subject: [PATCH 12/24] NXCALS-1468 added python3 install script to ansible --- accsoft-nxcals-ansible/group_vars/all/all.yml | 4 ++++ .../roles/openstack/tasks/install-python3.yml | 18 ++++++++++++++++++ .../roles/openstack/tasks/main.yml | 4 ++++ 3 files changed, 26 insertions(+) create mode 100644 accsoft-nxcals-ansible/roles/openstack/tasks/install-python3.yml diff --git a/accsoft-nxcals-ansible/group_vars/all/all.yml b/accsoft-nxcals-ansible/group_vars/all/all.yml index 6f9d85a8ab..e718ea4057 100644 --- a/accsoft-nxcals-ansible/group_vars/all/all.yml +++ b/accsoft-nxcals-ansible/group_vars/all/all.yml @@ -73,6 +73,10 @@ java_path_to_alias: '{{jdk_install_dir}}/jdk' extended_security_download_url: 'http://download.oracle.com/otn-pub/java/jce/8/jce_policy-8.zip' +#Python +python3_version_dotted: '3.4' +python3_version: '{{python3_version_dotted | regex_replace("\.")}}' + #Java security ################################################################################# trust_store_pem_certs: [ diff --git a/accsoft-nxcals-ansible/roles/openstack/tasks/install-python3.yml b/accsoft-nxcals-ansible/roles/openstack/tasks/install-python3.yml new file mode 100644 index 0000000000..534f8af120 --- /dev/null +++ b/accsoft-nxcals-ansible/roles/openstack/tasks/install-python3.yml @@ -0,0 +1,18 @@ +- name: install python{{python3_version}} package + yum: + name: python{{python3_version}} + state: present + update_cache: true + +- name: install python{{python3_version}}-setuptools package + yum: + name: python{{python3_version}}-setuptools + state: present + update_cache: true + +- name: install pip3 + shell: easy_install-{{python3_version_dotted}} pip + +- name: install wheel + shell: pip3 install wheel + diff --git a/accsoft-nxcals-ansible/roles/openstack/tasks/main.yml b/accsoft-nxcals-ansible/roles/openstack/tasks/main.yml index 12170ad48a..c9fa35a1bb 100644 --- a/accsoft-nxcals-ansible/roles/openstack/tasks/main.yml +++ b/accsoft-nxcals-ansible/roles/openstack/tasks/main.yml @@ -12,6 +12,10 @@ become: true tags: java +- include: install-python3.yml + become: true + tags: python + - include: yum-install.yml become: true tags: install -- GitLab From 47b8489eebfa243c75adfd17c0c7018280d05405 Mon Sep 17 00:00:00 2001 From: wjurasz <wiktor.jan.jurasz@cern.ch> Date: Mon, 12 Feb 2018 15:40:11 +0100 Subject: [PATCH 13/24] NXCALS-1468 naming fixed --- accsoft-nxcals-data-access-python3/build.gradle | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/accsoft-nxcals-data-access-python3/build.gradle b/accsoft-nxcals-data-access-python3/build.gradle index e0adb38bf2..3faae979fa 100644 --- a/accsoft-nxcals-data-access-python3/build.gradle +++ b/accsoft-nxcals-data-access-python3/build.gradle @@ -7,7 +7,11 @@ def pathToPythonInterpreter = '/user/bdisoft/operational/bin/Python/anaconda_py3 def pythonProjectName = name.replace('-', '_') def pythonProjectVersion = version.replace('-', '_') -def wheelArtifactName = pythonProjectName + '-' + pythonProjectVersion + '-py3-none-any.whl' +def artifactSufix = '-py3-none-any.whl' +def distDirectoryPath = 'dist/' +def wheelArtifactPath = distDirectoryPath + pythonProjectName + '-' + pythonProjectVersion + artifactSufix +def finaArtifactPath = distDirectoryPath + project.name + '-' + version + artifactSufix + def pythonPathVariableName = 'pythonPath' task testPythonVersion() { @@ -32,6 +36,9 @@ task testPythonVersion() { task distWheel(type: Exec) { commandLine pathToPythonInterpreter, 'setup.py', 'bdist_wheel', "--nxcals-version=$currentVersion" + doLast { + file(wheelArtifactPath).renameTo(file(finaArtifactPath)) + } } @@ -48,9 +55,9 @@ sonarqube { property "sonar.sources", "$projectDir/cern" } } - +\ artifacts { - archives file('dist/' + wheelArtifactName) + archives file(finaArtifactPath) } distWheel.dependsOn(testPythonVersion) -- GitLab From 8294cfe34898a4def966d4944f8d3400e26e7bd6 Mon Sep 17 00:00:00 2001 From: wjurasz <wiktor.jan.jurasz@cern.ch> Date: Mon, 12 Feb 2018 16:07:39 +0100 Subject: [PATCH 14/24] NXCALS-1468 removed temporary name --- accsoft-nxcals-data-access-python3/build.gradle | 2 +- build.gradle | 2 +- settings.gradle | 5 ----- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/accsoft-nxcals-data-access-python3/build.gradle b/accsoft-nxcals-data-access-python3/build.gradle index 3faae979fa..118b0717bb 100644 --- a/accsoft-nxcals-data-access-python3/build.gradle +++ b/accsoft-nxcals-data-access-python3/build.gradle @@ -46,7 +46,7 @@ task cleanWheel(type: Delete) { exec { commandLine pathToPythonInterpreter, 'setup.py', 'clean', '--all', "--nxcals-version=$currentVersion" } - delete 'dist', pythonProjectName + '.egg-info' + delete distDirectoryPath, pythonProjectName + '.egg-info' } sonarqube { diff --git a/build.gradle b/build.gradle index e8f955c13a..b18a802bc0 100644 --- a/build.gradle +++ b/build.gradle @@ -27,7 +27,7 @@ subprojects { apply plugin: 'maven' apply plugin: 'jacoco' - group 'python3test.cern.accsoft.nxcals' + group 'cern.accsoft.nxcals' version = currentVersion repositories { diff --git a/settings.gradle b/settings.gradle index 72561f7f9e..639a7fe783 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,8 +1,5 @@ import org.gradle.internal.os.OperatingSystem - -rootProject.name='python3test' - include 'accsoft-nxcals-ansible' include 'accsoft-nxcals-common' include 'accsoft-nxcals-common-spark' @@ -30,5 +27,3 @@ include 'accsoft-nxcals-monitoring-grok' include 'accsoft-nxcals-data-access-demo' include 'accsoft-nxcals-integration-tests' include 'accsoft-nxcals-service-db' - -include 'accsoft-nxcals-data-access-python3' -- GitLab From 91037dc35e288599d687f55112ff473f15bd575d Mon Sep 17 00:00:00 2001 From: wjurasz <wiktor.jan.jurasz@cern.ch> Date: Tue, 13 Feb 2018 08:40:44 +0100 Subject: [PATCH 15/24] NXCALS-1468 added python3 to build.gradle --- settings.gradle | 2 ++ 1 file changed, 2 insertions(+) diff --git a/settings.gradle b/settings.gradle index 639a7fe783..ca5b387367 100644 --- a/settings.gradle +++ b/settings.gradle @@ -27,3 +27,5 @@ include 'accsoft-nxcals-monitoring-grok' include 'accsoft-nxcals-data-access-demo' include 'accsoft-nxcals-integration-tests' include 'accsoft-nxcals-service-db' + +include 'accsoft-nxcals-data-access-python3' -- GitLab From 2d5a4ad088bfc057ef18372023db2c2a5ec8ef4f Mon Sep 17 00:00:00 2001 From: wjurasz <wiktor.jan.jurasz@cern.ch> Date: Tue, 13 Feb 2018 10:32:42 +0100 Subject: [PATCH 16/24] NXCALS-1468 upgraded build.gradle to search for python3 variable --- accsoft-nxcals-data-access-python3/build.gradle | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/accsoft-nxcals-data-access-python3/build.gradle b/accsoft-nxcals-data-access-python3/build.gradle index 118b0717bb..27fea3fe18 100644 --- a/accsoft-nxcals-data-access-python3/build.gradle +++ b/accsoft-nxcals-data-access-python3/build.gradle @@ -20,14 +20,15 @@ task testPythonVersion() { commandLine 'python', '--version' errorOutput = stdout } - - if (stdout.toString().startsWith('Python 3')) { - pathToPythonInterpreter = 'python' - } else if (project.hasProperty(pythonPathVariableName)) { + if (project.hasProperty(pythonPathVariableName)) { pathToPythonInterpreter = pythonPath + } else if (stdout.toString().startsWith('Python 3')) { + pathToPythonInterpreter = 'python' + } else if (exec { commandLine 'which', 'python3'; ignoreExitValue = true }.getExitValue() == 0) { + pathToPythonInterpreter = 'python3' } else { if (!file(pathToPythonInterpreter).exists()) { - throw new GradleException("To proceed python3 interpreter has to be provided, either as default intepreter (ran by \"python\" command) or by specifying -P$pythonPathVariableName=/path/to/python3 gradle option") + throw new GradleException("To proceed python3 interpreter has to be provided, either as default intepreter (ran by \"python\" or \"python3\" command) or by specifying -P$pythonPathVariableName=/path/to/python3 gradle option") } } @@ -46,7 +47,7 @@ task cleanWheel(type: Delete) { exec { commandLine pathToPythonInterpreter, 'setup.py', 'clean', '--all', "--nxcals-version=$currentVersion" } - delete distDirectoryPath, pythonProjectName + '.egg-info' + delete distDirectoryPath, pythonProjectName + '.egg-info' } sonarqube { @@ -56,7 +57,7 @@ sonarqube { } } \ -artifacts { + artifacts { archives file(finaArtifactPath) } -- GitLab From 02f49770c8f5a40f1721efca8e47658fbf9c04c4 Mon Sep 17 00:00:00 2001 From: wjurasz <wiktor.jan.jurasz@cern.ch> Date: Tue, 13 Feb 2018 10:36:58 +0100 Subject: [PATCH 17/24] NXCALS-1468 added tests --- .../tests/DataAccessQueryBuilder_test.py | 164 +++++++++++ .../TestDevicePropertyQueryBuilder_test.py | 91 ++++++ .../tests/TestDevicePropertyQueryData_test.py | 259 ++++++++++++++++++ .../tests/TestQueryData_test.py | 233 ++++++++++++++++ .../tests/TestTimeUtils_test.py | 56 ++++ 5 files changed, 803 insertions(+) create mode 100644 accsoft-nxcals-data-access-python3/tests/DataAccessQueryBuilder_test.py create mode 100644 accsoft-nxcals-data-access-python3/tests/TestDevicePropertyQueryBuilder_test.py create mode 100644 accsoft-nxcals-data-access-python3/tests/TestDevicePropertyQueryData_test.py create mode 100644 accsoft-nxcals-data-access-python3/tests/TestQueryData_test.py create mode 100644 accsoft-nxcals-data-access-python3/tests/TestTimeUtils_test.py diff --git a/accsoft-nxcals-data-access-python3/tests/DataAccessQueryBuilder_test.py b/accsoft-nxcals-data-access-python3/tests/DataAccessQueryBuilder_test.py new file mode 100644 index 0000000000..b42e7fd5ec --- /dev/null +++ b/accsoft-nxcals-data-access-python3/tests/DataAccessQueryBuilder_test.py @@ -0,0 +1,164 @@ +from cern.accsoft.nxcals.pyquery.builders import DataAccessQueryBuilder +from cern.accsoft.nxcals.pyquery.builders.TimeUtils import getNanosFromDateTime, convertToDatetime, getEndTimeInNanos +from datetime import datetime, timedelta +from unittest import TestCase +import json + + +class ShouldBuildQueryDataWithoutFields(TestCase): + def runTest(self): + # given + start_time = datetime.now() + end_time = datetime.now() + + # when + query = DataAccessQueryBuilder().system("TEST").key_value("key","value").start_time(start_time) \ + .end_time(end_time).build() + + # then + self.assertIsNotNone(query, "Query data should not be None !") + self.assertEquals("TEST", query.get_map().get("system")) + self.assertEquals(json.dumps({"key": "value"}), query.get_map().get("key_values")) + self.assertEquals(getNanosFromDateTime(start_time), query.get_map().get("start_time")) + self.assertEquals(getNanosFromDateTime(end_time), query.get_map().get("end_time")) + + +class ShouldBuildQueryDataWithAtTime(TestCase): + def runTest(self): + # given + at_time = datetime.now() + + # when + query = DataAccessQueryBuilder().system("TEST").key_values(key="value").at_time(at_time).build() + + # then + self.assertIsNotNone(query, "Query data should not be None !") + self.assertEquals("TEST", query.get_map().get("system")) + self.assertEquals(json.dumps({"key": "value"}), query.get_map().get("key_values")) + self.assertEquals(getNanosFromDateTime(at_time), query.get_map().get("start_time")) + self.assertEquals(getNanosFromDateTime(at_time), query.get_map().get("end_time")) + +class ShouldBuildQueryDataWithAtTimeAsString(TestCase): + def runTest(self): + # given + at_time = "1998-04-18 16:48:37.53" + + # when + query = DataAccessQueryBuilder().system("TEST").key_values(key="value").at_time(at_time).build() + + # then + self.assertIsNotNone(query, "Query data should not be None !") + self.assertEquals("TEST", query.get_map().get("system")) + self.assertEquals(json.dumps({"key": "value"}), query.get_map().get("key_values")) + self.assertEquals(getNanosFromDateTime(convertToDatetime(at_time)), query.get_map().get("start_time")) + self.assertEquals(getNanosFromDateTime(convertToDatetime(at_time)), query.get_map().get("end_time")) + + +class ShouldBuildQueryDataWithTimeDelta(TestCase): + def runTest(self): + # given + start_time = datetime.now() + delta = timedelta(days=1) + + # when + query = DataAccessQueryBuilder().system("TEST").key_values(key="value").start_time(start_time).duration(delta).build() + + # then + self.assertIsNotNone(query, "Query data should not be None !") + self.assertEquals("TEST", query.get_map().get("system")) + self.assertEquals(json.dumps({"key": "value"}), query.get_map().get("key_values")) + self.assertEquals(getNanosFromDateTime(start_time), query.get_map().get("start_time")) + self.assertEquals(getEndTimeInNanos(getNanosFromDateTime(start_time), delta), query.get_map().get("end_time")) + + +class ShouldBuildQueryDataWithFields(TestCase): + def runTest(self): + # given + start_time = datetime.now() - timedelta(days=1) + end_time = datetime.now() + + # when + query = DataAccessQueryBuilder().system("TEST").key_values(device="device_name", property="property_name") \ + .start_time(start_time).end_time(end_time).fields("field1", "field2", "field3").build() + + # then + self.assertIsNotNone(query, "Query data should not be None !") + self.assertEquals("TEST", query.get_map().get("system")) + self.assertEquals(json.dumps({"device": "device_name", "property": "property_name"}), query.get_map().get("key_values")) + self.assertEquals(getNanosFromDateTime(start_time), query.get_map().get("start_time")) + self.assertEquals(getNanosFromDateTime(end_time), query.get_map().get("end_time")) + self.assertEquals("field1,field2,field3", query.get_map().get("fields_key")) + + +class ShouldBuildQueryDataWithFieldsAsString(TestCase): + def runTest(self): + # given + start_time = "1998-04-18 16:48:37.53" + end_time = "1998-04-19 16:48:37.53" + + # when + query = DataAccessQueryBuilder().system("TEST").key_values(device="device_name", property="property_name") \ + .start_time(start_time).end_time(end_time).fields("field1", "field2", "field3").build() + + # then + self.assertIsNotNone(query, "Query data should not be None !") + self.assertEquals("TEST", query.get_map().get("system")) + self.assertEquals(json.dumps({"device": "device_name", "property": "property_name"}), query.get_map().get("key_values")) + self.assertEquals(getNanosFromDateTime(convertToDatetime(start_time)), query.get_map().get("start_time")) + self.assertEquals(getNanosFromDateTime(convertToDatetime(end_time)), query.get_map().get("end_time")) + self.assertEquals("field1,field2,field3", query.get_map().get("fields_key")) + + +class ShouldNotBuildQueryWithMissingSystem(TestCase): + def runTest(self): + with self.assertRaises(AssertionError): + DataAccessQueryBuilder().system(None) + + +class ShouldNotBuildQueryWithMissingKeyValues(TestCase): + def runTest(self): + with self.assertRaises(AssertionError): + DataAccessQueryBuilder().system("TEST").key_values() + + +class ShouldNotBuildQueryWithNoneKeyValue(TestCase): + def runTest(self): + with self.assertRaises(AssertionError): + DataAccessQueryBuilder().system("TEST").key_values(key=None) + + +class ShouldNotBuildQueryWithNoneStartTime(TestCase): + def runTest(self): + with self.assertRaises(AssertionError): + DataAccessQueryBuilder().system("TEST").key_values(key="value").start_time(None) + + +class ShouldNotBuildQueryWithNoneAt(TestCase): + def runTest(self): + with self.assertRaises(AssertionError): + DataAccessQueryBuilder().system("TEST").key_values(key="value").at_time(None) + + +class ShouldNotBuildQueryWithIllegalStartTime(TestCase): + def runTest(self): + with self.assertRaises(AssertionError): + DataAccessQueryBuilder().system("TEST").key_values(key="value").start_time("time") + + +class ShouldNotBuildQueryWithNoneEndTime(TestCase): + def runTest(self): + with self.assertRaises(AssertionError): + DataAccessQueryBuilder().system("TEST").key_values(key="value").start_time(datetime.now()).end_time(None) + + +class ShouldNotBuildQueryWithIllegalEndTime(TestCase): + def runTest(self): + with self.assertRaises(AssertionError): + DataAccessQueryBuilder().system("TEST").key_values(key="value").start_time(datetime.now()).end_time("test") + + +class ShouldNotBuildQueryWithNoneFields(TestCase): + def runTest(self): + with self.assertRaises(AssertionError): + DataAccessQueryBuilder().system("TEST").key_values(key="value").start_time(datetime.now()) \ + .end_time(datetime.now()).fields(None) diff --git a/accsoft-nxcals-data-access-python3/tests/TestDevicePropertyQueryBuilder_test.py b/accsoft-nxcals-data-access-python3/tests/TestDevicePropertyQueryBuilder_test.py new file mode 100644 index 0000000000..1b6d5f2fab --- /dev/null +++ b/accsoft-nxcals-data-access-python3/tests/TestDevicePropertyQueryBuilder_test.py @@ -0,0 +1,91 @@ +from datetime import datetime, timedelta +from unittest import TestCase +import json + +from cern.accsoft.nxcals.pyquery.builders.DevicePropertyQueryBuilder import DevicePropertyQueryBuilder, getNanosFromDateTime + + +class ShouldBuildDevicePropertyQuery(TestCase): + def runTest(self): + # given + start_time = datetime.now() - timedelta(days=1) + end_time = datetime.now() + + # when + query = DevicePropertyQueryBuilder().system("TEST").device("device_name").property("property_name") \ + .start_time(start_time).end_time(end_time).build() + + # then + self.assertIsNotNone(query,"Query data should not be None !") + self.assertEquals("TEST", query.get_map().get("system")) + self.assertEquals(json.dumps({"device": "device_name", "property": "property_name"}), query.get_map().get("key_values")) + self.assertEquals(getNanosFromDateTime(start_time), query.get_map().get("start_time")) + self.assertEquals(getNanosFromDateTime(end_time), query.get_map().get("end_time")) + + +class ShouldBuildDevicePropertyQueryWithFields(TestCase): + def runTest(self): + # given + start_time = datetime.now() - timedelta(days=1) + end_time = datetime.now() + + # when + query = DevicePropertyQueryBuilder().system("TEST").device("device_name").property("property_name") \ + .start_time(start_time).end_time(end_time).fields("field1","field2","field3").build() + + # then + self.assertIsNotNone(query,"Query data should not be None !") + self.assertEquals("TEST", query.get_map().get("system")) + self.assertEquals(json.dumps({"device": "device_name", "property": "property_name"}), query.get_map().get("key_values")) + self.assertEquals(getNanosFromDateTime(start_time), query.get_map().get("start_time")) + self.assertEquals(getNanosFromDateTime(end_time), query.get_map().get("end_time")) + self.assertEquals("field1,field2,field3", query.get_map().get("fields_key")) + + +class ShouldBuildDevicePropertyQueryWithParameter(TestCase): + def runTest(self): + # given + start_time = datetime.now() - timedelta(days=1) + end_time = datetime.now() + + # when + query = DevicePropertyQueryBuilder().system("TEST").parameter("device_name/property_name") \ + .start_time(start_time).end_time(end_time).fields("field1","field2","field3").build() + + # then + self.assertIsNotNone(query,"Query data should not be None !") + self.assertEquals("TEST", query.get_map().get("system")) + self.assertEquals(json.dumps({"device": "device_name", "property": "property_name"}), query.get_map().get("key_values")) + self.assertEquals(getNanosFromDateTime(start_time), query.get_map().get("start_time")) + self.assertEquals(getNanosFromDateTime(end_time), query.get_map().get("end_time")) + self.assertEquals("field1,field2,field3", query.get_map().get("fields_key")) + + +class ShouldNotBuildQueryWithMissingSystem(TestCase): + def runTest(self): + with self.assertRaises(TypeError): + DevicePropertyQueryBuilder().system() + + +class ShouldNotBuildQueryWithNoneDevice(TestCase): + def runTest(self): + with self.assertRaises(AssertionError): + DevicePropertyQueryBuilder().system("TEST").device(None) + + +class ShouldNotBuildQueryWithNoneProperty(TestCase): + def runTest(self): + with self.assertRaises(AssertionError): + DevicePropertyQueryBuilder().system("TEST").device("device_name").property(None) + + +class ShouldNotBuildQueryWithIllegalParameter(TestCase): + def runTest(self): + with self.assertRaises(AssertionError): + DevicePropertyQueryBuilder().system("TEST").parameter("device_name:property_name") + + +class ShouldNotBuildQueryWithNoneParameter(TestCase): + def runTest(self): + with self.assertRaises(AssertionError): + DevicePropertyQueryBuilder().system("TEST").parameter(None) \ No newline at end of file diff --git a/accsoft-nxcals-data-access-python3/tests/TestDevicePropertyQueryData_test.py b/accsoft-nxcals-data-access-python3/tests/TestDevicePropertyQueryData_test.py new file mode 100644 index 0000000000..5ace72a803 --- /dev/null +++ b/accsoft-nxcals-data-access-python3/tests/TestDevicePropertyQueryData_test.py @@ -0,0 +1,259 @@ +from datetime import datetime, timedelta +from unittest import TestCase + +from cern.accsoft.nxcals.pyquery.builders.DevicePropertyQueryBuilder import DevicePropertyQueryBuilder +from cern.accsoft.nxcals.pyquery.builders.DevicePropertyQueryData import DevicePropertyQueryData +from cern.accsoft.nxcals.pyquery.builders.TimeUtils import getNanosFromDateTime + +import json + +class TestDevicePropertyQueryData(TestCase): + def setUp(self): + # given + self.start_time = datetime.now() - timedelta(days=1) + self.end_time = datetime.now() + self.query = DevicePropertyQueryBuilder().system("TEST").parameter("device_name/property_name").start_time(self.start_time) \ + .end_time(self.end_time).fields("field1","field2","field3").build() + + +class ShouldCreateNewQueryDataFromExisting(TestDevicePropertyQueryData): + def runTest(self): + # when + new_query = DevicePropertyQueryData.modify_query(self.query).build() + + # then + self.assertIsNotNone(new_query) + self.assertNotEquals(new_query, self.query) + self.assertEqual("TEST", self.query.get_map().get("system")) + self.assertEqual("TEST", new_query.get_map().get("system")) + self.assertEquals(json.dumps({"device": "device_name", "property": "property_name"}), new_query.get_map().get("key_values")) + self.assertEqual(getNanosFromDateTime(self.start_time), new_query.get_map().get("start_time")) + self.assertEqual(getNanosFromDateTime(self.end_time), new_query.get_map().get("end_time")) + self.assertEqual("field1,field2,field3", new_query.get_map().get("fields_key")) + + +class ShouldModifySystem(TestDevicePropertyQueryData): + def runTest(self): + # when + new_query = DevicePropertyQueryData.modify_query(self.query).system("SYSTEM1").build() + + # then + self.assertIsNotNone(new_query) + self.assertNotEqual(new_query, self.query) + self.assertEqual("TEST", self.query.get_map().get("system")) + self.assertEqual("SYSTEM1", new_query.get_map().get("system")) + self.assertEquals(json.dumps({"device": "device_name", "property": "property_name"}), new_query.get_map().get("key_values")) + self.assertEqual(getNanosFromDateTime(self.start_time), new_query.get_map().get("start_time")) + self.assertEqual(getNanosFromDateTime(self.end_time), new_query.get_map().get("end_time")) + self.assertEqual("field1,field2,field3", new_query.get_map().get("fields_key")) + + +class ShouldUpdateDevice(TestDevicePropertyQueryData): + def runTest(self): + # when + new_query = DevicePropertyQueryData.modify_query(self.query).device("new_device_name").build() + + # then + self.assertIsNotNone(new_query) + self.assertNotEqual(new_query, self.query) + self.assertEqual("TEST", new_query.get_map().get("system")) + self.assertEquals(json.dumps({"device": "device_name", "property": "property_name"}), self.query.get_map().get("key_values")) + self.assertEquals(json.dumps({"device": "new_device_name", "property": "property_name"}), new_query.get_map().get("key_values")) + self.assertEqual(getNanosFromDateTime(self.start_time), new_query.get_map().get("start_time")) + self.assertEqual(getNanosFromDateTime(self.end_time), new_query.get_map().get("end_time")) + self.assertEqual("field1,field2,field3", new_query.get_map().get("fields_key")) + + +class ShouldUpdateProperty(TestDevicePropertyQueryData): + def runTest(self): + # when + new_query = DevicePropertyQueryData.modify_query(self.query).property("new_property_name").build() + + # then + self.assertIsNotNone(new_query) + self.assertNotEqual(new_query, self.query) + self.assertEqual("TEST", new_query.get_map().get("system")) + self.assertEquals(json.dumps({"device": "device_name", "property": "property_name"}), self.query.get_map().get("key_values")) + self.assertEquals(json.dumps({"device": "device_name", "property": "new_property_name"}), new_query.get_map().get("key_values")) + self.assertEqual(getNanosFromDateTime(self.start_time), new_query.get_map().get("start_time")) + self.assertEqual(getNanosFromDateTime(self.end_time), new_query.get_map().get("end_time")) + self.assertEqual("field1,field2,field3", new_query.get_map().get("fields_key")) + + +class ShouldUpdateParameter(TestDevicePropertyQueryData): + def runTest(self): + # when + new_query = DevicePropertyQueryData.modify_query(self.query).parameter("new_device_name/new_property_name").build() + + # then + self.assertIsNotNone(new_query) + self.assertNotEqual(new_query, self.query) + self.assertEqual("TEST", new_query.get_map().get("system")) + self.assertEquals(json.dumps({"device": "device_name", "property": "property_name"}), self.query.get_map().get("key_values")) + self.assertEquals(json.dumps({"device": "new_device_name", "property": "new_property_name"}), new_query.get_map().get("key_values")) + self.assertEqual(getNanosFromDateTime(self.start_time), new_query.get_map().get("start_time")) + self.assertEqual(getNanosFromDateTime(self.end_time), new_query.get_map().get("end_time")) + self.assertEqual("field1,field2,field3", new_query.get_map().get("fields_key")) + + +class ShouldModifyStartTime(TestDevicePropertyQueryData): + def runTest(self): + # given + new_start_time = datetime.now() - timedelta(hours=1) + + # when + new_query = DevicePropertyQueryData.modify_query(self.query).start_time(new_start_time).build() + + # then + self.assertIsNotNone(new_query) + self.assertNotEqual(new_query, self.query) + self.assertEqual("TEST", new_query.get_map().get("system")) + self.assertEquals(json.dumps({"device": "device_name", "property": "property_name"}), new_query.get_map().get("key_values")) + self.assertEqual(getNanosFromDateTime(self.start_time),self.query.get_map().get("start_time")) + self.assertEqual(getNanosFromDateTime(new_start_time), new_query.get_map().get("start_time")) + self.assertEqual(getNanosFromDateTime(self.end_time), new_query.get_map().get("end_time")) + self.assertEqual("field1,field2,field3", new_query.get_map().get("fields_key")) + + +class ShouldReplaceAllFields(TestDevicePropertyQueryData): + def runTest(self): + # when + new_query = DevicePropertyQueryData.modify_query(self.query).replace_all_fields("new_field1", "new_field2", "field3").build() + + # then + self.assertIsNotNone(new_query) + self.assertNotEqual(new_query, self.query) + self.assertEqual("TEST", new_query.get_map().get("system")) + self.assertEquals(json.dumps({"device": "device_name", "property": "property_name"}), new_query.get_map().get("key_values")) + self.assertEqual(getNanosFromDateTime(self.start_time), new_query.get_map().get("start_time")) + self.assertEqual(getNanosFromDateTime(self.end_time), new_query.get_map().get("end_time")) + self.assertEqual("field1,field2,field3", self.query.get_map().get("fields_key")) + self.assertEqual("new_field1,new_field2,field3", new_query.get_map().get("fields_key")) + + +class ShouldAddFields(TestDevicePropertyQueryData): + def runTest(self): + # when + new_query = DevicePropertyQueryData.modify_query(self.query).add_fields("new_field1", "new_field2", "field3").build() + + # then + self.assertIsNotNone(new_query) + self.assertNotEqual(new_query, self.query) + self.assertEqual("TEST", new_query.get_map().get("system")) + self.assertEquals(json.dumps({"device": "device_name", "property": "property_name"}), new_query.get_map().get("key_values")) + self.assertEqual(getNanosFromDateTime(self.start_time), new_query.get_map().get("start_time")) + self.assertEqual(getNanosFromDateTime(self.end_time), new_query.get_map().get("end_time")) + self.assertEqual("field1,field2,field3", self.query.get_map().get("fields_key")) + self.assertEqual(set("field1,field2,field3,new_field1,new_field2".split(",")), + set(new_query.get_map().get("fields_key").split(","))) + + +class ShouldRemoveFields(TestDevicePropertyQueryData): + def runTest(self): + # when + new_query = DevicePropertyQueryData.modify_query(self.query).remove_fields("field2","field1").build() + + # then + self.assertIsNotNone(new_query) + self.assertNotEqual(new_query, self.query) + self.assertEqual("TEST", new_query.get_map().get("system")) + self.assertEquals(json.dumps({"device": "device_name", "property": "property_name"}), new_query.get_map().get("key_values")) + self.assertEqual(getNanosFromDateTime(self.start_time), new_query.get_map().get("start_time")) + self.assertEqual(getNanosFromDateTime(self.end_time), new_query.get_map().get("end_time")) + self.assertEqual("field1,field2,field3", self.query.get_map().get("fields_key")) + self.assertEqual("field3", new_query.get_map().get("fields_key")) + + +# To include all fields in Query means to remove "fields_key" key internaly +class ShouldIncludeAllFieldsInQuery(TestDevicePropertyQueryData): + def runTest(self): + # when + new_query = DevicePropertyQueryData.modify_query(self.query).include_all_fields().build() + + # then + self.assertIsNotNone(new_query) + self.assertNotEqual(new_query, self.query) + self.assertEqual("TEST", new_query.get_map().get("system")) + self.assertEquals(json.dumps({"device": "device_name", "property": "property_name"}), new_query.get_map().get("key_values")) + self.assertEqual(getNanosFromDateTime(self.start_time), + new_query.get_map().get("start_time")) + self.assertEqual(getNanosFromDateTime(self.end_time), + new_query.get_map().get("end_time")) + self.assertEqual("field1,field2,field3", self.query.get_map().get("fields_key")) + self.assertIsNone(new_query.get_map().get("fields_key")) + self.assertEqual(len(self.query.get_map()) - 1, len(new_query.get_map())) + + +class ShouldNotAcceptNoneSystem(TestDevicePropertyQueryData): + def runTest(self): + with self.assertRaises(AssertionError): + DevicePropertyQueryData.modify_query(self.query).system(None) + + +class ShouldNotAcceptNoneDevice(TestDevicePropertyQueryData): + def runTest(self): + with self.assertRaises(AssertionError): + DevicePropertyQueryData.modify_query(self.query).system("TEST").device(None) + + +class ShouldNotAcceptNoneProperty(TestDevicePropertyQueryData): + def runTest(self): + with self.assertRaises(AssertionError): + DevicePropertyQueryData.modify_query(self.query).system("TEST").device("device_name").property(None) + + +class ShouldNotAcceptIllegalParameter(TestDevicePropertyQueryData): + def runTest(self): + with self.assertRaises(AssertionError): + DevicePropertyQueryData.modify_query(self.query).system("TEST").parameter("device_name:property_name") + + +class ShouldNotAcceptNoneParameter(TestDevicePropertyQueryData): + def runTest(self): + with self.assertRaises(AssertionError): + DevicePropertyQueryData.modify_query(self.query).system("TEST").parameter(None) + + +class ShouldNotAcceptNoneStartTime(TestDevicePropertyQueryData): + def runTest(self): + with self.assertRaises(AssertionError): + DevicePropertyQueryData.modify_query(self.query).system("TEST").parameter("device_name/property_name").start_time(None) + + +class ShouldNotAcceptIllegalStartTime(TestDevicePropertyQueryData): + def runTest(self): + with self.assertRaises(AssertionError): + DevicePropertyQueryData.modify_query(self.query).system("TEST").parameter("device_name/property_name").start_time("time") + + +class ShouldNotAcceptNoneEndTime(TestDevicePropertyQueryData): + def runTest(self): + with self.assertRaises(AssertionError): + DevicePropertyQueryData.modify_query(self.query).system("TEST").parameter("device_name/property_name").start_time(datetime.now()).end_time(None) + + +class ShouldNotAcceptIllegalEndTime(TestDevicePropertyQueryData): + def runTest(self): + with self.assertRaises(AssertionError): + DevicePropertyQueryData.modify_query(self.query).system("TEST").parameter("device_name/property_name").start_time(datetime.now()).end_time("test") + + +class ShouldNotAcceptNoneFields(TestDevicePropertyQueryData): + def runTest(self): + with self.assertRaises(AssertionError): + DevicePropertyQueryData.modify_query(self.query).system("TEST").parameter("device_name/property_name").start_time(datetime.now()) \ + .end_time(datetime.now()).replace_all_fields(None) + + +class ShouldNotRemoveNoneFields(TestDevicePropertyQueryData): + def runTest(self): + with self.assertRaises(AssertionError): + DevicePropertyQueryData.modify_query(self.query).system("TEST").parameter("device_name/property_name").start_time(datetime.now()) \ + .end_time(datetime.now()).remove_fields(None) + + +class ShouldNotAddNoneFields(TestDevicePropertyQueryData): + def runTest(self): + with self.assertRaises(AssertionError): + DevicePropertyQueryData.modify_query(self.query).system("TEST").parameter("device_name/property_name").start_time(datetime.now()) \ + .end_time(datetime.now()).add_fields(None) diff --git a/accsoft-nxcals-data-access-python3/tests/TestQueryData_test.py b/accsoft-nxcals-data-access-python3/tests/TestQueryData_test.py new file mode 100644 index 0000000000..596d9dfc58 --- /dev/null +++ b/accsoft-nxcals-data-access-python3/tests/TestQueryData_test.py @@ -0,0 +1,233 @@ +from datetime import datetime, timedelta +from unittest import TestCase + +from cern.accsoft.nxcals.pyquery.builders.TimeUtils import getNanosFromDateTime +from cern.accsoft.nxcals.pyquery.builders.DataAccessQueryBuilder import DataAccessQueryBuilder +from cern.accsoft.nxcals.pyquery.builders.QueryData import QueryData + +import json + +class TestQueryData(TestCase): + def setUp(self): + # given + self.start_time = datetime.now() - timedelta(days=1) + self.end_time = datetime.now() + self.query = DataAccessQueryBuilder().system("TEST").key_values(key="value").start_time(self.start_time) \ + .end_time(self.end_time).fields("field1","field2","field3").build() + + +class ShouldCreateNewQueryDataFromExisting(TestQueryData): + def runTest(self): + # when + new_query = QueryData.modify_query(self.query).build() + + # then + self.assertIsNotNone(new_query) + self.assertNotEquals(new_query, self.query) + self.assertEqual("TEST", self.query.get_map().get("system")) + self.assertEqual("TEST", new_query.get_map().get("system")) + self.assertEquals(json.dumps({"key": "value"}), new_query.get_map().get("key_values")) + self.assertEqual(getNanosFromDateTime(self.start_time), new_query.get_map().get("start_time")) + self.assertEqual(getNanosFromDateTime(self.end_time), new_query.get_map().get("end_time")) + self.assertEqual("field1,field2,field3", new_query.get_map().get("fields_key")) + + +class ShouldModifySystem(TestQueryData): + def runTest(self): + # when + new_query = QueryData.modify_query(self.query).system("SYSTEM1").build() + + # then + self.assertIsNotNone(new_query) + self.assertNotEqual(new_query, self.query) + self.assertEqual("TEST", self.query.get_map().get("system")) + self.assertEqual("SYSTEM1", new_query.get_map().get("system")) + self.assertEquals(json.dumps({"key": "value"}), new_query.get_map().get("key_values")) + self.assertEqual(getNanosFromDateTime(self.start_time), new_query.get_map().get("start_time")) + self.assertEqual(getNanosFromDateTime(self.end_time), new_query.get_map().get("end_time")) + self.assertEqual("field1,field2,field3", new_query.get_map().get("fields_key")) + + +class ShouldUpdateKeyValuesAndAddNewKeys(TestQueryData): + def runTest(self): + # when + new_query = QueryData.modify_query(self.query).key_values(key="value1", key2="value2").build() + + # then + self.assertIsNotNone(new_query) + self.assertNotEqual(new_query, self.query) + self.assertEqual("TEST", new_query.get_map().get("system")) + self.assertEquals(json.dumps({"key": "value"}), self.query.get_map().get("key_values")) + self.assertEquals(json.dumps({"key": "value1", "key2": "value2"}), new_query.get_map().get("key_values")) + self.assertEqual(getNanosFromDateTime(self.start_time), new_query.get_map().get("start_time")) + self.assertEqual(getNanosFromDateTime(self.end_time), new_query.get_map().get("end_time")) + self.assertEqual("field1,field2,field3", new_query.get_map().get("fields_key")) + + +class ShouldRemoveKeyValues(TestQueryData): + def runTest(self): + self.assertEqual("1", "1") + # when + new_query = QueryData.modify_query(self.query).remove_keys("key").build() + + # then + self.assertIsNotNone(new_query) + self.assertNotEqual(new_query, self.query) + self.assertEqual("TEST", new_query.get_map().get("system")) + self.assertEquals(json.dumps({"key": "value"}), self.query.get_map().get("key_values")) + self.assertEquals(json.dumps({}), new_query.get_map().get("key_values")) + self.assertEqual(len(self.query.get_map()), len(new_query.get_map())) + self.assertEqual(getNanosFromDateTime(self.start_time), new_query.get_map().get("start_time")) + self.assertEqual(getNanosFromDateTime(self.end_time), new_query.get_map().get("end_time")) + self.assertEqual("field1,field2,field3", new_query.get_map().get("fields_key")) + + +class ShouldModifyStartTime(TestQueryData): + def runTest(self): + # given + new_start_time = datetime.now() - timedelta(hours=1) + + # when + new_query = QueryData.modify_query(self.query).start_time(new_start_time).build() + + # then + self.assertIsNotNone(new_query) + self.assertNotEqual(new_query, self.query) + self.assertEqual("TEST", new_query.get_map().get("system")) + self.assertEquals(json.dumps({"key": "value"}), self.query.get_map().get("key_values")) + self.assertEqual(getNanosFromDateTime(self.start_time),self.query.get_map().get("start_time")) + self.assertEqual(getNanosFromDateTime(new_start_time), new_query.get_map().get("start_time")) + self.assertEqual(getNanosFromDateTime(self.end_time), new_query.get_map().get("end_time")) + self.assertEqual("field1,field2,field3", new_query.get_map().get("fields_key")) + + +class ShouldReplaceAllFields(TestQueryData): + def runTest(self): + # when + new_query = QueryData.modify_query(self.query).replace_all_fields("new_field1", "new_field2", "field3").build() + + # then + self.assertIsNotNone(new_query) + self.assertNotEqual(new_query, self.query) + self.assertEqual("TEST", new_query.get_map().get("system")) + self.assertEquals(json.dumps({"key": "value"}), self.query.get_map().get("key_values")) + self.assertEqual(getNanosFromDateTime(self.start_time), new_query.get_map().get("start_time")) + self.assertEqual(getNanosFromDateTime(self.end_time), new_query.get_map().get("end_time")) + self.assertEqual("field1,field2,field3", self.query.get_map().get("fields_key")) + self.assertEqual("new_field1,new_field2,field3", new_query.get_map().get("fields_key")) + + +class ShouldAddFields(TestQueryData): + def runTest(self): + # when + new_query = QueryData.modify_query(self.query).add_fields("new_field1", "new_field2", "field3").build() + + # then + self.assertIsNotNone(new_query) + self.assertNotEqual(new_query, self.query) + self.assertEqual("TEST", new_query.get_map().get("system")) + self.assertEquals(json.dumps({"key": "value"}), self.query.get_map().get("key_values")) + self.assertEqual(getNanosFromDateTime(self.start_time), new_query.get_map().get("start_time")) + self.assertEqual(getNanosFromDateTime(self.end_time), new_query.get_map().get("end_time")) + self.assertEqual("field1,field2,field3", self.query.get_map().get("fields_key")) + self.assertEqual(set("field1,field2,field3,new_field1,new_field2".split(",")), + set(new_query.get_map().get("fields_key").split(","))) + + +class ShouldRemoveFields(TestQueryData): + def runTest(self): + # when + new_query = QueryData.modify_query(self.query).remove_fields("field2","field1").build() + + # then + self.assertIsNotNone(new_query) + self.assertNotEqual(new_query, self.query) + self.assertEqual("TEST", new_query.get_map().get("system")) + self.assertEquals(json.dumps({"key": "value"}), self.query.get_map().get("key_values")) + self.assertEqual(getNanosFromDateTime(self.start_time), new_query.get_map().get("start_time")) + self.assertEqual(getNanosFromDateTime(self.end_time), new_query.get_map().get("end_time")) + self.assertEqual("field1,field2,field3", self.query.get_map().get("fields_key")) + self.assertEqual("field3", new_query.get_map().get("fields_key")) + + +# To include all fields in Query means to remove "fields_key" key internaly +class ShouldIncludeAllFieldsInQuery(TestQueryData): + def runTest(self): + # when + new_query = QueryData.modify_query(self.query).include_all_fields().build() + + # then + self.assertIsNotNone(new_query) + self.assertNotEqual(new_query, self.query) + self.assertEqual("TEST", new_query.get_map().get("system")) + self.assertEquals(json.dumps({"key": "value"}), self.query.get_map().get("key_values")) + self.assertEqual(getNanosFromDateTime(self.start_time), + new_query.get_map().get("start_time")) + self.assertEqual(getNanosFromDateTime(self.end_time), + new_query.get_map().get("end_time")) + self.assertEqual("field1,field2,field3", self.query.get_map().get("fields_key")) + self.assertIsNone(new_query.get_map().get("fields_key")) + self.assertEqual(len(self.query.get_map()) - 1, len(new_query.get_map())) + + +class ShouldNotAcceptNoneSystem(TestQueryData): + def runTest(self): + with self.assertRaises(AssertionError): + QueryData.modify_query(self.query).system(None) + + +class ShouldNotAcceptEmptyKeyValues(TestQueryData): + def runTest(self): + with self.assertRaises(AssertionError): + QueryData.modify_query(self.query).system("TEST").key_values() + + +class ShouldNotAcceptNoneKeyValue(TestQueryData): + def runTest(self): + with self.assertRaises(AssertionError): + QueryData.modify_query(self.query).system("TEST").key_values(key=None) + + +class ShouldNotAcceptNoneStartTime(TestQueryData): + def runTest(self): + with self.assertRaises(AssertionError): + QueryData.modify_query(self.query).system("TEST").key_values(key="value").start_time(None) + + +class ShouldNotAcceptIllegalStartTime(TestQueryData): + def runTest(self): + with self.assertRaises(AssertionError): + QueryData.modify_query(self.query).system("TEST").key_values(key="value").start_time("time") + + +class ShouldNotAcceptNoneEndTime(TestQueryData): + def runTest(self): + with self.assertRaises(AssertionError): + QueryData.modify_query(self.query).system("TEST").key_values(key="value").start_time(datetime.now()).end_time(None) + + +class ShouldNotAcceptIllegalEndTime(TestQueryData): + def runTest(self): + with self.assertRaises(AssertionError): + QueryData.modify_query(self.query).system("TEST").key_values(key="value").start_time(datetime.now()).end_time("test") + + +class ShouldNotAcceptNoneFields(TestQueryData): + def runTest(self): + with self.assertRaises(AssertionError): + QueryData.modify_query(self.query).system("TEST").key_values(key="value").start_time(datetime.now()) \ + .end_time(datetime.now()).replace_all_fields(None) + + +class ShouldNotRemoveNoneFields(TestQueryData): + def runTest(self): + with self.assertRaises(AssertionError): + QueryData.modify_query(self.query).system("TEST").key_values(key="value").start_time(datetime.now()) \ + .end_time(datetime.now()).remove_fields(None) + + +class ShouldNotAddNoneFields(TestQueryData): + def runTest(self): + with self.assertRaises(AssertionError): + QueryData.modify_query(self.query).system("TEST").key_values(key="value").start_time(datetime.now()) \ + .end_time(datetime.now()).add_fields(None) diff --git a/accsoft-nxcals-data-access-python3/tests/TestTimeUtils_test.py b/accsoft-nxcals-data-access-python3/tests/TestTimeUtils_test.py new file mode 100644 index 0000000000..034734ef35 --- /dev/null +++ b/accsoft-nxcals-data-access-python3/tests/TestTimeUtils_test.py @@ -0,0 +1,56 @@ +from datetime import datetime +from unittest import TestCase + +from cern.accsoft.nxcals.pyquery.builders.TimeUtils import getNanosFromDateTime, convertToDatetime + + +class TestShouldConvertToNanos(TestCase): + def test_1(self): + # given + time_in_nanos = 1487772729000001000 + time = convertToDatetime('2017-02-22 15:12:09.000001') + + # when + converted_time = getNanosFromDateTime(time) + + # then + self.assertIsNotNone(converted_time) + self.assertEqual(time_in_nanos,converted_time) + + +class ShouldConvertStringToDatetime(TestCase): + def runTest(self): + # given + time_string = '2017-02-22 15:12:09.000001' + + # when + time = convertToDatetime(time_string) + + # then + self.assertIsNotNone(time) + self.assertTrue(type(time) is datetime) + self.assertTrue(time_string,str(time)) + + +class ShouldFailToGetNanosWithWrongFormat(TestCase): + def runTest(self): + with self.assertRaises(AssertionError): + getNanosFromDateTime("test") + + +class ShouldFailToGetNanosWithNoneValue(TestCase): + def runTest(self): + with self.assertRaises(AssertionError): + getNanosFromDateTime(None) + + +class ShouldFailToConvertWithWrongFormat(TestCase): + def runTest(self): + with self.assertRaises(AssertionError): + convertToDatetime("test") + + +class ShouldFailToConvertWithNoneValue(TestCase): + def runTest(self): + with self.assertRaises(AssertionError): + convertToDatetime(None) \ No newline at end of file -- GitLab From 12bd3f348181df24bffac0520aa30d7e3f78119b Mon Sep 17 00:00:00 2001 From: wjurasz <wiktor.jan.jurasz@cern.ch> Date: Tue, 13 Feb 2018 12:05:56 +0100 Subject: [PATCH 18/24] NXCALS-1468 tests fixed --- .../src/test/__init__.py | 0 .../build.gradle | 28 +++++++++++-------- .../builders/DataAccessQueryBuilder.py | 4 +-- .../nxcals/pyquery/builders/QueryData.py | 2 +- .../nxcals/pyquery/builders/TimeUtils.py | 6 ++-- accsoft-nxcals-data-access-python3/setup.py | 1 + .../tests/DataAccessQueryBuilder_test.py | 2 +- .../tests/__init__.py | 0 8 files changed, 25 insertions(+), 18 deletions(-) create mode 100644 accsoft-nxcals-data-access-python/src/test/__init__.py create mode 100644 accsoft-nxcals-data-access-python3/tests/__init__.py diff --git a/accsoft-nxcals-data-access-python/src/test/__init__.py b/accsoft-nxcals-data-access-python/src/test/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/accsoft-nxcals-data-access-python3/build.gradle b/accsoft-nxcals-data-access-python3/build.gradle index 27fea3fe18..89aa635e6c 100644 --- a/accsoft-nxcals-data-access-python3/build.gradle +++ b/accsoft-nxcals-data-access-python3/build.gradle @@ -34,6 +34,17 @@ task testPythonVersion() { } +task cleanWheel(type: Delete) { + exec { + commandLine pathToPythonInterpreter, 'setup.py', 'clean', '--all', "--nxcals-version=$currentVersion" + } + delete distDirectoryPath, pythonProjectName + '.egg-info', fileTree('.') { include '**/*.pyc'} +} + + +task testWheel(type: Exec) { + commandLine pathToPythonInterpreter, 'setup.py', 'test', "--nxcals-version=$currentVersion" +} task distWheel(type: Exec) { commandLine pathToPythonInterpreter, 'setup.py', 'bdist_wheel', "--nxcals-version=$currentVersion" @@ -42,14 +53,6 @@ task distWheel(type: Exec) { } } - -task cleanWheel(type: Delete) { - exec { - commandLine pathToPythonInterpreter, 'setup.py', 'clean', '--all', "--nxcals-version=$currentVersion" - } - delete distDirectoryPath, pythonProjectName + '.egg-info' -} - sonarqube { properties { property "sonar.python.coverage.reportPath", "$projectDir/coverage.xml" @@ -57,11 +60,14 @@ sonarqube { } } \ - artifacts { + artifacts { archives file(finaArtifactPath) } +testWheel.dependsOn(testPythonVersion) distWheel.dependsOn(testPythonVersion) -install.dependsOn(distWheel) + +clean.dependsOn(cleanWheel) +test.dependsOn(testWheel) build.dependsOn(distWheel) -clean.dependsOn(cleanWheel) \ No newline at end of file +install.dependsOn(distWheel) diff --git a/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/DataAccessQueryBuilder.py b/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/DataAccessQueryBuilder.py index d778a1e365..9207750ff1 100644 --- a/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/DataAccessQueryBuilder.py +++ b/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/DataAccessQueryBuilder.py @@ -50,8 +50,8 @@ class DataAccessQueryBuilder(object): self._builder = builder def key_value(self, key, value): - if key is None or not isinstance(key, basestring) or value is None or not isinstance(value, basestring): - raise AssertionError("Key and Value must not be None and must be a basestring") + if key is None or not isinstance(key, str) or value is None or not isinstance(value, str): + raise AssertionError("Key and Value must not be None and must be a string") self._builder._key_values.update({key: value}) return self diff --git a/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/QueryData.py b/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/QueryData.py index c4d1b11ff8..23c7e18b4e 100644 --- a/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/QueryData.py +++ b/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/QueryData.py @@ -28,7 +28,7 @@ class QueryData(object): if not key_values or None in key_values.values(): raise AssertionError("Method key_Value() should be called only when specifying given fields !") temporaryDict = json.loads(self._query['key_values']) - for key, value in key_values.iteritems(): + for key, value in key_values.items(): temporaryDict.update({key:value}) self._query['key_values'] = json.dumps(temporaryDict) return self diff --git a/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/TimeUtils.py b/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/TimeUtils.py index ed96319972..855c7bff3a 100644 --- a/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/TimeUtils.py +++ b/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/TimeUtils.py @@ -5,7 +5,7 @@ from decimal import Decimal def getNanosFromDateTime(t): if t is None or type(t) is not datetime: raise AssertionError("The provided value is not datetime !") - return long((Decimal(time.mktime(t.timetuple())) + Decimal(t.microsecond)/1000000)*1000000000) + return int((Decimal(time.mktime(t.timetuple())) + Decimal(t.microsecond)/1000000)*1000000000) def convertToDatetime(t): @@ -20,7 +20,7 @@ def convertToDatetime(t): raise AssertionError("The used datetime format doesn't respect pattern %Y-%m-%d %H:%M:%S.%f") def getEndTimeInNanos(start_time, duration): - if type(duration) is not timedelta or type(start_time) is not long: - raise AssertionError("Duration must be datetime.timedelta instance and start_time must be long") + if type(duration) is not timedelta or type(start_time) is not int: + raise AssertionError("Duration must be datetime.timedelta instance and start_time must be int") duration_nanos = (duration.days*86400000 + duration.seconds*1000 + duration.microseconds/1000) * 1000 return start_time + duration_nanos diff --git a/accsoft-nxcals-data-access-python3/setup.py b/accsoft-nxcals-data-access-python3/setup.py index 5a68aea0e7..fb834d7bd9 100644 --- a/accsoft-nxcals-data-access-python3/setup.py +++ b/accsoft-nxcals-data-access-python3/setup.py @@ -31,6 +31,7 @@ setup( packages=find_packages(exclude=('tests',)), install_requires=REQUIRED, include_package_data=True, + test_suite="tests", classifiers=[ 'Programming Language :: Python', 'Programming Language :: Python :: 3', diff --git a/accsoft-nxcals-data-access-python3/tests/DataAccessQueryBuilder_test.py b/accsoft-nxcals-data-access-python3/tests/DataAccessQueryBuilder_test.py index b42e7fd5ec..4dfbdbdece 100644 --- a/accsoft-nxcals-data-access-python3/tests/DataAccessQueryBuilder_test.py +++ b/accsoft-nxcals-data-access-python3/tests/DataAccessQueryBuilder_test.py @@ -1,4 +1,4 @@ -from cern.accsoft.nxcals.pyquery.builders import DataAccessQueryBuilder +from cern.accsoft.nxcals.pyquery.builders.DataAccessQueryBuilder import DataAccessQueryBuilder from cern.accsoft.nxcals.pyquery.builders.TimeUtils import getNanosFromDateTime, convertToDatetime, getEndTimeInNanos from datetime import datetime, timedelta from unittest import TestCase diff --git a/accsoft-nxcals-data-access-python3/tests/__init__.py b/accsoft-nxcals-data-access-python3/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 -- GitLab From 379ee813b9f4aa3c7ad452dc7f8e546dc00f2176 Mon Sep 17 00:00:00 2001 From: wjurasz <wiktor.jan.jurasz@cern.ch> Date: Tue, 13 Feb 2018 12:11:25 +0100 Subject: [PATCH 19/24] NXCALS-1468 naming fixed --- .gitignore | 1 + .../builders/DataAccessQueryBuilder.py | 12 ++--- .../nxcals/pyquery/builders/QueryData.py | 10 ++--- .../nxcals/pyquery/builders/TimeUtils.py | 6 +-- .../pyquery/builders/VariableQueryBuilder.py | 10 ++--- .../pyquery/builders/VariableQueryData.py | 10 ++--- .../tests/DataAccessQueryBuilder_test.py | 26 +++++------ .../TestDevicePropertyQueryBuilder_test.py | 14 +++--- .../tests/TestDevicePropertyQueryData_test.py | 44 +++++++++---------- .../tests/TestQueryData_test.py | 40 ++++++++--------- .../tests/TestTimeUtils_test.py | 16 +++---- 11 files changed, 95 insertions(+), 94 deletions(-) diff --git a/.gitignore b/.gitignore index d2bb2347d8..85456b05e8 100644 --- a/.gitignore +++ b/.gitignore @@ -61,6 +61,7 @@ accsoft-nxcals-data-access-python3/**/*.egg-info accsoft-nxcals-data-access-python/**/*.egg-info accsoft-nxcals-data-access-python/**/*.egg *.whl +*.pyc #PyGradle modules accsoft-nxcals-data-access-python/.cache diff --git a/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/DataAccessQueryBuilder.py b/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/DataAccessQueryBuilder.py index 9207750ff1..aae0f8eacf 100644 --- a/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/DataAccessQueryBuilder.py +++ b/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/DataAccessQueryBuilder.py @@ -1,7 +1,7 @@ import json from .QueryData import * -from .TimeUtils import getNanosFromDateTime, convertToDatetime, getEndTimeInNanos +from .TimeUtils import get_nanos_from_date_time, convert_to_datetime, get_end_time_in_nanos class DataAccessQueryBuilder(object): @@ -38,11 +38,11 @@ class DataAccessQueryBuilder(object): self._builder = builder def start_time(self, start_time): - self._builder._start_time = getNanosFromDateTime(convertToDatetime(start_time)) + self._builder._start_time = get_nanos_from_date_time(convert_to_datetime(start_time)) return DataAccessQueryBuilder._EndTime(self._builder) def at_time(self, at_time): - self._builder._start_time = getNanosFromDateTime(convertToDatetime(at_time)) + self._builder._start_time = get_nanos_from_date_time(convert_to_datetime(at_time)) return DataAccessQueryBuilder._EndTime(self._builder).end_time(at_time) class _KeyValue(_StartTime): @@ -60,11 +60,11 @@ class DataAccessQueryBuilder(object): self._builder = builder def end_time(self, end_time): - self._builder._end_time = getNanosFromDateTime(convertToDatetime(end_time)) + self._builder._end_time = get_nanos_from_date_time(convert_to_datetime(end_time)) return DataAccessQueryBuilder._Fields(self._builder) def duration(self, duration): - self._builder._end_time = getEndTimeInNanos(self._builder._start_time, duration) + self._builder._end_time = get_end_time_in_nanos(self._builder._start_time, duration) return DataAccessQueryBuilder._Fields(self._builder) class _Fields(object): @@ -101,7 +101,7 @@ class DataAccessQueryBuilder(object): query["aliases_key"] = json.dumps(self._builder._alias_fields) return QueryData(query) - def buildDataset(self, session): + def build_dataset(self, session): imported_py_sql_session = __import__('pyspark.sql.session', globals(), locals(), ['SparkSession'], 0) if type(session) is not imported_py_sql_session.SparkSession: raise AssertionError("Session must be a SparkSession instance!") diff --git a/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/QueryData.py b/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/QueryData.py index 23c7e18b4e..97830946e1 100644 --- a/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/QueryData.py +++ b/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/QueryData.py @@ -1,4 +1,4 @@ -from .TimeUtils import getNanosFromDateTime, convertToDatetime +from .TimeUtils import get_nanos_from_date_time, convert_to_datetime from datetime import datetime import json @@ -43,17 +43,17 @@ class QueryData(object): return self def start_time(self, start_time): - start_time = convertToDatetime(start_time) + start_time = convert_to_datetime(start_time) if type(start_time) is not datetime or start_time is None: raise AssertionError("Start time must be a datatime object !") - self._query.update({"start_time":getNanosFromDateTime(start_time)}) + self._query.update({"start_time":get_nanos_from_date_time(start_time)}) return self def end_time(self, end_time): - end_time = convertToDatetime(end_time) + end_time = convert_to_datetime(end_time) if type(end_time) is not datetime or end_time is None: raise AssertionError("End time must be a datatime object !") - self._query.update({"end_time":getNanosFromDateTime(end_time)}) + self._query.update({"end_time":get_nanos_from_date_time(end_time)}) return self def replace_all_fields(self, *fields): diff --git a/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/TimeUtils.py b/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/TimeUtils.py index 855c7bff3a..b3904f5ca8 100644 --- a/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/TimeUtils.py +++ b/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/TimeUtils.py @@ -2,13 +2,13 @@ import time from datetime import datetime, timedelta from decimal import Decimal -def getNanosFromDateTime(t): +def get_nanos_from_date_time(t): if t is None or type(t) is not datetime: raise AssertionError("The provided value is not datetime !") return int((Decimal(time.mktime(t.timetuple())) + Decimal(t.microsecond)/1000000)*1000000000) -def convertToDatetime(t): +def convert_to_datetime(t): if t is None: raise AssertionError("Datetime value cannot be None !") if type(t) is datetime: @@ -19,7 +19,7 @@ def convertToDatetime(t): except BaseException: raise AssertionError("The used datetime format doesn't respect pattern %Y-%m-%d %H:%M:%S.%f") -def getEndTimeInNanos(start_time, duration): +def get_end_time_in_nanos(start_time, duration): if type(duration) is not timedelta or type(start_time) is not int: raise AssertionError("Duration must be datetime.timedelta instance and start_time must be int") duration_nanos = (duration.days*86400000 + duration.seconds*1000 + duration.microseconds/1000) * 1000 diff --git a/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/VariableQueryBuilder.py b/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/VariableQueryBuilder.py index c7f0ca511f..1fb54c5786 100644 --- a/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/VariableQueryBuilder.py +++ b/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/VariableQueryBuilder.py @@ -1,5 +1,5 @@ from .QueryData import * -from .TimeUtils import getNanosFromDateTime, convertToDatetime +from .TimeUtils import get_nanos_from_date_time, convert_to_datetime from datetime import datetime @@ -20,10 +20,10 @@ class VariableQueryBuilder(object): self._builder = builder def start_time(self, start_time): - start_time = convertToDatetime(start_time) + start_time = convert_to_datetime(start_time) if type(start_time) is not datetime or start_time is None: raise AssertionError("Start time must be a datatime object !") - self._builder._start_time = getNanosFromDateTime(start_time) + self._builder._start_time = get_nanos_from_date_time(start_time) return VariableQueryBuilder._EndTime(self._builder) class _EndTime(object): @@ -31,10 +31,10 @@ class VariableQueryBuilder(object): self._builder = builder def end_time(self, end_time): - end_time = convertToDatetime(end_time) + end_time = convert_to_datetime(end_time) if type(end_time) is not datetime or end_time is None: raise AssertionError("End time must be a datatime object !") - self._builder._end_time = getNanosFromDateTime(end_time) + self._builder._end_time = get_nanos_from_date_time(end_time) return VariableQueryBuilder._BuildQuery(self._builder) class _BuildQuery(object): diff --git a/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/VariableQueryData.py b/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/VariableQueryData.py index fc44a045c6..87c618cd6c 100644 --- a/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/VariableQueryData.py +++ b/accsoft-nxcals-data-access-python3/cern/accsoft/nxcals/pyquery/builders/VariableQueryData.py @@ -1,4 +1,4 @@ -from .TimeUtils import getNanosFromDateTime, convertToDatetime +from .TimeUtils import get_nanos_from_date_time, convert_to_datetime from datetime import datetime @@ -24,17 +24,17 @@ class VariableQueryData(object): return self def start_time(self, start_time): - start_time = convertToDatetime(start_time) + start_time = convert_to_datetime(start_time) if type(start_time) is not datetime or start_time is None: raise AssertionError("Start time must be a datatime object !") - self._query.update({"start_time":getNanosFromDateTime(start_time)}) + self._query.update({"start_time":get_nanos_from_date_time(start_time)}) return self def end_time(self, end_time): - end_time = convertToDatetime(end_time) + end_time = convert_to_datetime(end_time) if type(end_time) is not datetime or end_time is None: raise AssertionError("End time must be a datatime object !") - self._query.update({"end_time":getNanosFromDateTime(end_time)}) + self._query.update({"end_time":get_nanos_from_date_time(end_time)}) return self def build(self): diff --git a/accsoft-nxcals-data-access-python3/tests/DataAccessQueryBuilder_test.py b/accsoft-nxcals-data-access-python3/tests/DataAccessQueryBuilder_test.py index 4dfbdbdece..76194749f7 100644 --- a/accsoft-nxcals-data-access-python3/tests/DataAccessQueryBuilder_test.py +++ b/accsoft-nxcals-data-access-python3/tests/DataAccessQueryBuilder_test.py @@ -1,5 +1,5 @@ from cern.accsoft.nxcals.pyquery.builders.DataAccessQueryBuilder import DataAccessQueryBuilder -from cern.accsoft.nxcals.pyquery.builders.TimeUtils import getNanosFromDateTime, convertToDatetime, getEndTimeInNanos +from cern.accsoft.nxcals.pyquery.builders.TimeUtils import get_nanos_from_date_time, convert_to_datetime, get_end_time_in_nanos from datetime import datetime, timedelta from unittest import TestCase import json @@ -19,8 +19,8 @@ class ShouldBuildQueryDataWithoutFields(TestCase): self.assertIsNotNone(query, "Query data should not be None !") self.assertEquals("TEST", query.get_map().get("system")) self.assertEquals(json.dumps({"key": "value"}), query.get_map().get("key_values")) - self.assertEquals(getNanosFromDateTime(start_time), query.get_map().get("start_time")) - self.assertEquals(getNanosFromDateTime(end_time), query.get_map().get("end_time")) + self.assertEquals(get_nanos_from_date_time(start_time), query.get_map().get("start_time")) + self.assertEquals(get_nanos_from_date_time(end_time), query.get_map().get("end_time")) class ShouldBuildQueryDataWithAtTime(TestCase): @@ -35,8 +35,8 @@ class ShouldBuildQueryDataWithAtTime(TestCase): self.assertIsNotNone(query, "Query data should not be None !") self.assertEquals("TEST", query.get_map().get("system")) self.assertEquals(json.dumps({"key": "value"}), query.get_map().get("key_values")) - self.assertEquals(getNanosFromDateTime(at_time), query.get_map().get("start_time")) - self.assertEquals(getNanosFromDateTime(at_time), query.get_map().get("end_time")) + self.assertEquals(get_nanos_from_date_time(at_time), query.get_map().get("start_time")) + self.assertEquals(get_nanos_from_date_time(at_time), query.get_map().get("end_time")) class ShouldBuildQueryDataWithAtTimeAsString(TestCase): def runTest(self): @@ -50,8 +50,8 @@ class ShouldBuildQueryDataWithAtTimeAsString(TestCase): self.assertIsNotNone(query, "Query data should not be None !") self.assertEquals("TEST", query.get_map().get("system")) self.assertEquals(json.dumps({"key": "value"}), query.get_map().get("key_values")) - self.assertEquals(getNanosFromDateTime(convertToDatetime(at_time)), query.get_map().get("start_time")) - self.assertEquals(getNanosFromDateTime(convertToDatetime(at_time)), query.get_map().get("end_time")) + self.assertEquals(get_nanos_from_date_time(convert_to_datetime(at_time)), query.get_map().get("start_time")) + self.assertEquals(get_nanos_from_date_time(convert_to_datetime(at_time)), query.get_map().get("end_time")) class ShouldBuildQueryDataWithTimeDelta(TestCase): @@ -67,8 +67,8 @@ class ShouldBuildQueryDataWithTimeDelta(TestCase): self.assertIsNotNone(query, "Query data should not be None !") self.assertEquals("TEST", query.get_map().get("system")) self.assertEquals(json.dumps({"key": "value"}), query.get_map().get("key_values")) - self.assertEquals(getNanosFromDateTime(start_time), query.get_map().get("start_time")) - self.assertEquals(getEndTimeInNanos(getNanosFromDateTime(start_time), delta), query.get_map().get("end_time")) + self.assertEquals(get_nanos_from_date_time(start_time), query.get_map().get("start_time")) + self.assertEquals(get_end_time_in_nanos(get_nanos_from_date_time(start_time), delta), query.get_map().get("end_time")) class ShouldBuildQueryDataWithFields(TestCase): @@ -85,8 +85,8 @@ class ShouldBuildQueryDataWithFields(TestCase): self.assertIsNotNone(query, "Query data should not be None !") self.assertEquals("TEST", query.get_map().get("system")) self.assertEquals(json.dumps({"device": "device_name", "property": "property_name"}), query.get_map().get("key_values")) - self.assertEquals(getNanosFromDateTime(start_time), query.get_map().get("start_time")) - self.assertEquals(getNanosFromDateTime(end_time), query.get_map().get("end_time")) + self.assertEquals(get_nanos_from_date_time(start_time), query.get_map().get("start_time")) + self.assertEquals(get_nanos_from_date_time(end_time), query.get_map().get("end_time")) self.assertEquals("field1,field2,field3", query.get_map().get("fields_key")) @@ -104,8 +104,8 @@ class ShouldBuildQueryDataWithFieldsAsString(TestCase): self.assertIsNotNone(query, "Query data should not be None !") self.assertEquals("TEST", query.get_map().get("system")) self.assertEquals(json.dumps({"device": "device_name", "property": "property_name"}), query.get_map().get("key_values")) - self.assertEquals(getNanosFromDateTime(convertToDatetime(start_time)), query.get_map().get("start_time")) - self.assertEquals(getNanosFromDateTime(convertToDatetime(end_time)), query.get_map().get("end_time")) + self.assertEquals(get_nanos_from_date_time(convert_to_datetime(start_time)), query.get_map().get("start_time")) + self.assertEquals(get_nanos_from_date_time(convert_to_datetime(end_time)), query.get_map().get("end_time")) self.assertEquals("field1,field2,field3", query.get_map().get("fields_key")) diff --git a/accsoft-nxcals-data-access-python3/tests/TestDevicePropertyQueryBuilder_test.py b/accsoft-nxcals-data-access-python3/tests/TestDevicePropertyQueryBuilder_test.py index 1b6d5f2fab..992c67eaba 100644 --- a/accsoft-nxcals-data-access-python3/tests/TestDevicePropertyQueryBuilder_test.py +++ b/accsoft-nxcals-data-access-python3/tests/TestDevicePropertyQueryBuilder_test.py @@ -2,7 +2,7 @@ from datetime import datetime, timedelta from unittest import TestCase import json -from cern.accsoft.nxcals.pyquery.builders.DevicePropertyQueryBuilder import DevicePropertyQueryBuilder, getNanosFromDateTime +from cern.accsoft.nxcals.pyquery.builders.DevicePropertyQueryBuilder import DevicePropertyQueryBuilder, get_nanos_from_date_time class ShouldBuildDevicePropertyQuery(TestCase): @@ -19,8 +19,8 @@ class ShouldBuildDevicePropertyQuery(TestCase): self.assertIsNotNone(query,"Query data should not be None !") self.assertEquals("TEST", query.get_map().get("system")) self.assertEquals(json.dumps({"device": "device_name", "property": "property_name"}), query.get_map().get("key_values")) - self.assertEquals(getNanosFromDateTime(start_time), query.get_map().get("start_time")) - self.assertEquals(getNanosFromDateTime(end_time), query.get_map().get("end_time")) + self.assertEquals(get_nanos_from_date_time(start_time), query.get_map().get("start_time")) + self.assertEquals(get_nanos_from_date_time(end_time), query.get_map().get("end_time")) class ShouldBuildDevicePropertyQueryWithFields(TestCase): @@ -37,8 +37,8 @@ class ShouldBuildDevicePropertyQueryWithFields(TestCase): self.assertIsNotNone(query,"Query data should not be None !") self.assertEquals("TEST", query.get_map().get("system")) self.assertEquals(json.dumps({"device": "device_name", "property": "property_name"}), query.get_map().get("key_values")) - self.assertEquals(getNanosFromDateTime(start_time), query.get_map().get("start_time")) - self.assertEquals(getNanosFromDateTime(end_time), query.get_map().get("end_time")) + self.assertEquals(get_nanos_from_date_time(start_time), query.get_map().get("start_time")) + self.assertEquals(get_nanos_from_date_time(end_time), query.get_map().get("end_time")) self.assertEquals("field1,field2,field3", query.get_map().get("fields_key")) @@ -56,8 +56,8 @@ class ShouldBuildDevicePropertyQueryWithParameter(TestCase): self.assertIsNotNone(query,"Query data should not be None !") self.assertEquals("TEST", query.get_map().get("system")) self.assertEquals(json.dumps({"device": "device_name", "property": "property_name"}), query.get_map().get("key_values")) - self.assertEquals(getNanosFromDateTime(start_time), query.get_map().get("start_time")) - self.assertEquals(getNanosFromDateTime(end_time), query.get_map().get("end_time")) + self.assertEquals(get_nanos_from_date_time(start_time), query.get_map().get("start_time")) + self.assertEquals(get_nanos_from_date_time(end_time), query.get_map().get("end_time")) self.assertEquals("field1,field2,field3", query.get_map().get("fields_key")) diff --git a/accsoft-nxcals-data-access-python3/tests/TestDevicePropertyQueryData_test.py b/accsoft-nxcals-data-access-python3/tests/TestDevicePropertyQueryData_test.py index 5ace72a803..9d966b89a5 100644 --- a/accsoft-nxcals-data-access-python3/tests/TestDevicePropertyQueryData_test.py +++ b/accsoft-nxcals-data-access-python3/tests/TestDevicePropertyQueryData_test.py @@ -3,7 +3,7 @@ from unittest import TestCase from cern.accsoft.nxcals.pyquery.builders.DevicePropertyQueryBuilder import DevicePropertyQueryBuilder from cern.accsoft.nxcals.pyquery.builders.DevicePropertyQueryData import DevicePropertyQueryData -from cern.accsoft.nxcals.pyquery.builders.TimeUtils import getNanosFromDateTime +from cern.accsoft.nxcals.pyquery.builders.TimeUtils import get_nanos_from_date_time import json @@ -27,8 +27,8 @@ class ShouldCreateNewQueryDataFromExisting(TestDevicePropertyQueryData): self.assertEqual("TEST", self.query.get_map().get("system")) self.assertEqual("TEST", new_query.get_map().get("system")) self.assertEquals(json.dumps({"device": "device_name", "property": "property_name"}), new_query.get_map().get("key_values")) - self.assertEqual(getNanosFromDateTime(self.start_time), new_query.get_map().get("start_time")) - self.assertEqual(getNanosFromDateTime(self.end_time), new_query.get_map().get("end_time")) + self.assertEqual(get_nanos_from_date_time(self.start_time), new_query.get_map().get("start_time")) + self.assertEqual(get_nanos_from_date_time(self.end_time), new_query.get_map().get("end_time")) self.assertEqual("field1,field2,field3", new_query.get_map().get("fields_key")) @@ -43,8 +43,8 @@ class ShouldModifySystem(TestDevicePropertyQueryData): self.assertEqual("TEST", self.query.get_map().get("system")) self.assertEqual("SYSTEM1", new_query.get_map().get("system")) self.assertEquals(json.dumps({"device": "device_name", "property": "property_name"}), new_query.get_map().get("key_values")) - self.assertEqual(getNanosFromDateTime(self.start_time), new_query.get_map().get("start_time")) - self.assertEqual(getNanosFromDateTime(self.end_time), new_query.get_map().get("end_time")) + self.assertEqual(get_nanos_from_date_time(self.start_time), new_query.get_map().get("start_time")) + self.assertEqual(get_nanos_from_date_time(self.end_time), new_query.get_map().get("end_time")) self.assertEqual("field1,field2,field3", new_query.get_map().get("fields_key")) @@ -59,8 +59,8 @@ class ShouldUpdateDevice(TestDevicePropertyQueryData): self.assertEqual("TEST", new_query.get_map().get("system")) self.assertEquals(json.dumps({"device": "device_name", "property": "property_name"}), self.query.get_map().get("key_values")) self.assertEquals(json.dumps({"device": "new_device_name", "property": "property_name"}), new_query.get_map().get("key_values")) - self.assertEqual(getNanosFromDateTime(self.start_time), new_query.get_map().get("start_time")) - self.assertEqual(getNanosFromDateTime(self.end_time), new_query.get_map().get("end_time")) + self.assertEqual(get_nanos_from_date_time(self.start_time), new_query.get_map().get("start_time")) + self.assertEqual(get_nanos_from_date_time(self.end_time), new_query.get_map().get("end_time")) self.assertEqual("field1,field2,field3", new_query.get_map().get("fields_key")) @@ -75,8 +75,8 @@ class ShouldUpdateProperty(TestDevicePropertyQueryData): self.assertEqual("TEST", new_query.get_map().get("system")) self.assertEquals(json.dumps({"device": "device_name", "property": "property_name"}), self.query.get_map().get("key_values")) self.assertEquals(json.dumps({"device": "device_name", "property": "new_property_name"}), new_query.get_map().get("key_values")) - self.assertEqual(getNanosFromDateTime(self.start_time), new_query.get_map().get("start_time")) - self.assertEqual(getNanosFromDateTime(self.end_time), new_query.get_map().get("end_time")) + self.assertEqual(get_nanos_from_date_time(self.start_time), new_query.get_map().get("start_time")) + self.assertEqual(get_nanos_from_date_time(self.end_time), new_query.get_map().get("end_time")) self.assertEqual("field1,field2,field3", new_query.get_map().get("fields_key")) @@ -91,8 +91,8 @@ class ShouldUpdateParameter(TestDevicePropertyQueryData): self.assertEqual("TEST", new_query.get_map().get("system")) self.assertEquals(json.dumps({"device": "device_name", "property": "property_name"}), self.query.get_map().get("key_values")) self.assertEquals(json.dumps({"device": "new_device_name", "property": "new_property_name"}), new_query.get_map().get("key_values")) - self.assertEqual(getNanosFromDateTime(self.start_time), new_query.get_map().get("start_time")) - self.assertEqual(getNanosFromDateTime(self.end_time), new_query.get_map().get("end_time")) + self.assertEqual(get_nanos_from_date_time(self.start_time), new_query.get_map().get("start_time")) + self.assertEqual(get_nanos_from_date_time(self.end_time), new_query.get_map().get("end_time")) self.assertEqual("field1,field2,field3", new_query.get_map().get("fields_key")) @@ -109,9 +109,9 @@ class ShouldModifyStartTime(TestDevicePropertyQueryData): self.assertNotEqual(new_query, self.query) self.assertEqual("TEST", new_query.get_map().get("system")) self.assertEquals(json.dumps({"device": "device_name", "property": "property_name"}), new_query.get_map().get("key_values")) - self.assertEqual(getNanosFromDateTime(self.start_time),self.query.get_map().get("start_time")) - self.assertEqual(getNanosFromDateTime(new_start_time), new_query.get_map().get("start_time")) - self.assertEqual(getNanosFromDateTime(self.end_time), new_query.get_map().get("end_time")) + self.assertEqual(get_nanos_from_date_time(self.start_time), self.query.get_map().get("start_time")) + self.assertEqual(get_nanos_from_date_time(new_start_time), new_query.get_map().get("start_time")) + self.assertEqual(get_nanos_from_date_time(self.end_time), new_query.get_map().get("end_time")) self.assertEqual("field1,field2,field3", new_query.get_map().get("fields_key")) @@ -125,8 +125,8 @@ class ShouldReplaceAllFields(TestDevicePropertyQueryData): self.assertNotEqual(new_query, self.query) self.assertEqual("TEST", new_query.get_map().get("system")) self.assertEquals(json.dumps({"device": "device_name", "property": "property_name"}), new_query.get_map().get("key_values")) - self.assertEqual(getNanosFromDateTime(self.start_time), new_query.get_map().get("start_time")) - self.assertEqual(getNanosFromDateTime(self.end_time), new_query.get_map().get("end_time")) + self.assertEqual(get_nanos_from_date_time(self.start_time), new_query.get_map().get("start_time")) + self.assertEqual(get_nanos_from_date_time(self.end_time), new_query.get_map().get("end_time")) self.assertEqual("field1,field2,field3", self.query.get_map().get("fields_key")) self.assertEqual("new_field1,new_field2,field3", new_query.get_map().get("fields_key")) @@ -141,8 +141,8 @@ class ShouldAddFields(TestDevicePropertyQueryData): self.assertNotEqual(new_query, self.query) self.assertEqual("TEST", new_query.get_map().get("system")) self.assertEquals(json.dumps({"device": "device_name", "property": "property_name"}), new_query.get_map().get("key_values")) - self.assertEqual(getNanosFromDateTime(self.start_time), new_query.get_map().get("start_time")) - self.assertEqual(getNanosFromDateTime(self.end_time), new_query.get_map().get("end_time")) + self.assertEqual(get_nanos_from_date_time(self.start_time), new_query.get_map().get("start_time")) + self.assertEqual(get_nanos_from_date_time(self.end_time), new_query.get_map().get("end_time")) self.assertEqual("field1,field2,field3", self.query.get_map().get("fields_key")) self.assertEqual(set("field1,field2,field3,new_field1,new_field2".split(",")), set(new_query.get_map().get("fields_key").split(","))) @@ -158,8 +158,8 @@ class ShouldRemoveFields(TestDevicePropertyQueryData): self.assertNotEqual(new_query, self.query) self.assertEqual("TEST", new_query.get_map().get("system")) self.assertEquals(json.dumps({"device": "device_name", "property": "property_name"}), new_query.get_map().get("key_values")) - self.assertEqual(getNanosFromDateTime(self.start_time), new_query.get_map().get("start_time")) - self.assertEqual(getNanosFromDateTime(self.end_time), new_query.get_map().get("end_time")) + self.assertEqual(get_nanos_from_date_time(self.start_time), new_query.get_map().get("start_time")) + self.assertEqual(get_nanos_from_date_time(self.end_time), new_query.get_map().get("end_time")) self.assertEqual("field1,field2,field3", self.query.get_map().get("fields_key")) self.assertEqual("field3", new_query.get_map().get("fields_key")) @@ -175,9 +175,9 @@ class ShouldIncludeAllFieldsInQuery(TestDevicePropertyQueryData): self.assertNotEqual(new_query, self.query) self.assertEqual("TEST", new_query.get_map().get("system")) self.assertEquals(json.dumps({"device": "device_name", "property": "property_name"}), new_query.get_map().get("key_values")) - self.assertEqual(getNanosFromDateTime(self.start_time), + self.assertEqual(get_nanos_from_date_time(self.start_time), new_query.get_map().get("start_time")) - self.assertEqual(getNanosFromDateTime(self.end_time), + self.assertEqual(get_nanos_from_date_time(self.end_time), new_query.get_map().get("end_time")) self.assertEqual("field1,field2,field3", self.query.get_map().get("fields_key")) self.assertIsNone(new_query.get_map().get("fields_key")) diff --git a/accsoft-nxcals-data-access-python3/tests/TestQueryData_test.py b/accsoft-nxcals-data-access-python3/tests/TestQueryData_test.py index 596d9dfc58..1cb23474ba 100644 --- a/accsoft-nxcals-data-access-python3/tests/TestQueryData_test.py +++ b/accsoft-nxcals-data-access-python3/tests/TestQueryData_test.py @@ -1,7 +1,7 @@ from datetime import datetime, timedelta from unittest import TestCase -from cern.accsoft.nxcals.pyquery.builders.TimeUtils import getNanosFromDateTime +from cern.accsoft.nxcals.pyquery.builders.TimeUtils import get_nanos_from_date_time from cern.accsoft.nxcals.pyquery.builders.DataAccessQueryBuilder import DataAccessQueryBuilder from cern.accsoft.nxcals.pyquery.builders.QueryData import QueryData @@ -27,8 +27,8 @@ class ShouldCreateNewQueryDataFromExisting(TestQueryData): self.assertEqual("TEST", self.query.get_map().get("system")) self.assertEqual("TEST", new_query.get_map().get("system")) self.assertEquals(json.dumps({"key": "value"}), new_query.get_map().get("key_values")) - self.assertEqual(getNanosFromDateTime(self.start_time), new_query.get_map().get("start_time")) - self.assertEqual(getNanosFromDateTime(self.end_time), new_query.get_map().get("end_time")) + self.assertEqual(get_nanos_from_date_time(self.start_time), new_query.get_map().get("start_time")) + self.assertEqual(get_nanos_from_date_time(self.end_time), new_query.get_map().get("end_time")) self.assertEqual("field1,field2,field3", new_query.get_map().get("fields_key")) @@ -43,8 +43,8 @@ class ShouldModifySystem(TestQueryData): self.assertEqual("TEST", self.query.get_map().get("system")) self.assertEqual("SYSTEM1", new_query.get_map().get("system")) self.assertEquals(json.dumps({"key": "value"}), new_query.get_map().get("key_values")) - self.assertEqual(getNanosFromDateTime(self.start_time), new_query.get_map().get("start_time")) - self.assertEqual(getNanosFromDateTime(self.end_time), new_query.get_map().get("end_time")) + self.assertEqual(get_nanos_from_date_time(self.start_time), new_query.get_map().get("start_time")) + self.assertEqual(get_nanos_from_date_time(self.end_time), new_query.get_map().get("end_time")) self.assertEqual("field1,field2,field3", new_query.get_map().get("fields_key")) @@ -59,8 +59,8 @@ class ShouldUpdateKeyValuesAndAddNewKeys(TestQueryData): self.assertEqual("TEST", new_query.get_map().get("system")) self.assertEquals(json.dumps({"key": "value"}), self.query.get_map().get("key_values")) self.assertEquals(json.dumps({"key": "value1", "key2": "value2"}), new_query.get_map().get("key_values")) - self.assertEqual(getNanosFromDateTime(self.start_time), new_query.get_map().get("start_time")) - self.assertEqual(getNanosFromDateTime(self.end_time), new_query.get_map().get("end_time")) + self.assertEqual(get_nanos_from_date_time(self.start_time), new_query.get_map().get("start_time")) + self.assertEqual(get_nanos_from_date_time(self.end_time), new_query.get_map().get("end_time")) self.assertEqual("field1,field2,field3", new_query.get_map().get("fields_key")) @@ -77,8 +77,8 @@ class ShouldRemoveKeyValues(TestQueryData): self.assertEquals(json.dumps({"key": "value"}), self.query.get_map().get("key_values")) self.assertEquals(json.dumps({}), new_query.get_map().get("key_values")) self.assertEqual(len(self.query.get_map()), len(new_query.get_map())) - self.assertEqual(getNanosFromDateTime(self.start_time), new_query.get_map().get("start_time")) - self.assertEqual(getNanosFromDateTime(self.end_time), new_query.get_map().get("end_time")) + self.assertEqual(get_nanos_from_date_time(self.start_time), new_query.get_map().get("start_time")) + self.assertEqual(get_nanos_from_date_time(self.end_time), new_query.get_map().get("end_time")) self.assertEqual("field1,field2,field3", new_query.get_map().get("fields_key")) @@ -95,9 +95,9 @@ class ShouldModifyStartTime(TestQueryData): self.assertNotEqual(new_query, self.query) self.assertEqual("TEST", new_query.get_map().get("system")) self.assertEquals(json.dumps({"key": "value"}), self.query.get_map().get("key_values")) - self.assertEqual(getNanosFromDateTime(self.start_time),self.query.get_map().get("start_time")) - self.assertEqual(getNanosFromDateTime(new_start_time), new_query.get_map().get("start_time")) - self.assertEqual(getNanosFromDateTime(self.end_time), new_query.get_map().get("end_time")) + self.assertEqual(get_nanos_from_date_time(self.start_time), self.query.get_map().get("start_time")) + self.assertEqual(get_nanos_from_date_time(new_start_time), new_query.get_map().get("start_time")) + self.assertEqual(get_nanos_from_date_time(self.end_time), new_query.get_map().get("end_time")) self.assertEqual("field1,field2,field3", new_query.get_map().get("fields_key")) @@ -111,8 +111,8 @@ class ShouldReplaceAllFields(TestQueryData): self.assertNotEqual(new_query, self.query) self.assertEqual("TEST", new_query.get_map().get("system")) self.assertEquals(json.dumps({"key": "value"}), self.query.get_map().get("key_values")) - self.assertEqual(getNanosFromDateTime(self.start_time), new_query.get_map().get("start_time")) - self.assertEqual(getNanosFromDateTime(self.end_time), new_query.get_map().get("end_time")) + self.assertEqual(get_nanos_from_date_time(self.start_time), new_query.get_map().get("start_time")) + self.assertEqual(get_nanos_from_date_time(self.end_time), new_query.get_map().get("end_time")) self.assertEqual("field1,field2,field3", self.query.get_map().get("fields_key")) self.assertEqual("new_field1,new_field2,field3", new_query.get_map().get("fields_key")) @@ -127,8 +127,8 @@ class ShouldAddFields(TestQueryData): self.assertNotEqual(new_query, self.query) self.assertEqual("TEST", new_query.get_map().get("system")) self.assertEquals(json.dumps({"key": "value"}), self.query.get_map().get("key_values")) - self.assertEqual(getNanosFromDateTime(self.start_time), new_query.get_map().get("start_time")) - self.assertEqual(getNanosFromDateTime(self.end_time), new_query.get_map().get("end_time")) + self.assertEqual(get_nanos_from_date_time(self.start_time), new_query.get_map().get("start_time")) + self.assertEqual(get_nanos_from_date_time(self.end_time), new_query.get_map().get("end_time")) self.assertEqual("field1,field2,field3", self.query.get_map().get("fields_key")) self.assertEqual(set("field1,field2,field3,new_field1,new_field2".split(",")), set(new_query.get_map().get("fields_key").split(","))) @@ -144,8 +144,8 @@ class ShouldRemoveFields(TestQueryData): self.assertNotEqual(new_query, self.query) self.assertEqual("TEST", new_query.get_map().get("system")) self.assertEquals(json.dumps({"key": "value"}), self.query.get_map().get("key_values")) - self.assertEqual(getNanosFromDateTime(self.start_time), new_query.get_map().get("start_time")) - self.assertEqual(getNanosFromDateTime(self.end_time), new_query.get_map().get("end_time")) + self.assertEqual(get_nanos_from_date_time(self.start_time), new_query.get_map().get("start_time")) + self.assertEqual(get_nanos_from_date_time(self.end_time), new_query.get_map().get("end_time")) self.assertEqual("field1,field2,field3", self.query.get_map().get("fields_key")) self.assertEqual("field3", new_query.get_map().get("fields_key")) @@ -161,9 +161,9 @@ class ShouldIncludeAllFieldsInQuery(TestQueryData): self.assertNotEqual(new_query, self.query) self.assertEqual("TEST", new_query.get_map().get("system")) self.assertEquals(json.dumps({"key": "value"}), self.query.get_map().get("key_values")) - self.assertEqual(getNanosFromDateTime(self.start_time), + self.assertEqual(get_nanos_from_date_time(self.start_time), new_query.get_map().get("start_time")) - self.assertEqual(getNanosFromDateTime(self.end_time), + self.assertEqual(get_nanos_from_date_time(self.end_time), new_query.get_map().get("end_time")) self.assertEqual("field1,field2,field3", self.query.get_map().get("fields_key")) self.assertIsNone(new_query.get_map().get("fields_key")) diff --git a/accsoft-nxcals-data-access-python3/tests/TestTimeUtils_test.py b/accsoft-nxcals-data-access-python3/tests/TestTimeUtils_test.py index 034734ef35..8da325a461 100644 --- a/accsoft-nxcals-data-access-python3/tests/TestTimeUtils_test.py +++ b/accsoft-nxcals-data-access-python3/tests/TestTimeUtils_test.py @@ -1,17 +1,17 @@ from datetime import datetime from unittest import TestCase -from cern.accsoft.nxcals.pyquery.builders.TimeUtils import getNanosFromDateTime, convertToDatetime +from cern.accsoft.nxcals.pyquery.builders.TimeUtils import get_nanos_from_date_time, convert_to_datetime class TestShouldConvertToNanos(TestCase): def test_1(self): # given time_in_nanos = 1487772729000001000 - time = convertToDatetime('2017-02-22 15:12:09.000001') + time = convert_to_datetime('2017-02-22 15:12:09.000001') # when - converted_time = getNanosFromDateTime(time) + converted_time = get_nanos_from_date_time(time) # then self.assertIsNotNone(converted_time) @@ -24,7 +24,7 @@ class ShouldConvertStringToDatetime(TestCase): time_string = '2017-02-22 15:12:09.000001' # when - time = convertToDatetime(time_string) + time = convert_to_datetime(time_string) # then self.assertIsNotNone(time) @@ -35,22 +35,22 @@ class ShouldConvertStringToDatetime(TestCase): class ShouldFailToGetNanosWithWrongFormat(TestCase): def runTest(self): with self.assertRaises(AssertionError): - getNanosFromDateTime("test") + get_nanos_from_date_time("test") class ShouldFailToGetNanosWithNoneValue(TestCase): def runTest(self): with self.assertRaises(AssertionError): - getNanosFromDateTime(None) + get_nanos_from_date_time(None) class ShouldFailToConvertWithWrongFormat(TestCase): def runTest(self): with self.assertRaises(AssertionError): - convertToDatetime("test") + convert_to_datetime("test") class ShouldFailToConvertWithNoneValue(TestCase): def runTest(self): with self.assertRaises(AssertionError): - convertToDatetime(None) \ No newline at end of file + convert_to_datetime(None) \ No newline at end of file -- GitLab From 1d21fb5c869635b7a40860b88363e69c34dab755 Mon Sep 17 00:00:00 2001 From: wjurasz <wiktor.jan.jurasz@cern.ch> Date: Wed, 14 Feb 2018 12:11:13 +0100 Subject: [PATCH 20/24] NXCALS-1468 added zip to artifacts --- .gitignore | 2 ++ .../build.gradle | 22 ++++++++++++++----- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 85456b05e8..49c4a34625 100644 --- a/.gitignore +++ b/.gitignore @@ -62,6 +62,8 @@ accsoft-nxcals-data-access-python/**/*.egg-info accsoft-nxcals-data-access-python/**/*.egg *.whl *.pyc +accsoft-nxcals-data-access-python3/dist/*.zip +accsoft-nxcals-data-access-python3/dist/**/*.zip #PyGradle modules accsoft-nxcals-data-access-python/.cache diff --git a/accsoft-nxcals-data-access-python3/build.gradle b/accsoft-nxcals-data-access-python3/build.gradle index 89aa635e6c..f79ab4241f 100644 --- a/accsoft-nxcals-data-access-python3/build.gradle +++ b/accsoft-nxcals-data-access-python3/build.gradle @@ -9,8 +9,14 @@ def pythonProjectName = name.replace('-', '_') def pythonProjectVersion = version.replace('-', '_') def artifactSufix = '-py3-none-any.whl' def distDirectoryPath = 'dist/' + def wheelArtifactPath = distDirectoryPath + pythonProjectName + '-' + pythonProjectVersion + artifactSufix -def finaArtifactPath = distDirectoryPath + project.name + '-' + version + artifactSufix + +def finalArtifactName = project.name + '-' + version + artifactSufix; +def finalArtifactPath = distDirectoryPath + finalArtifactName + +def finaArtifactNameZip = finalArtifactName.replace('.whl', '.zip') +def finaArtifactPathZip = distDirectoryPath + finaArtifactNameZip def pythonPathVariableName = 'pythonPath' @@ -49,7 +55,13 @@ task testWheel(type: Exec) { task distWheel(type: Exec) { commandLine pathToPythonInterpreter, 'setup.py', 'bdist_wheel', "--nxcals-version=$currentVersion" doLast { - file(wheelArtifactPath).renameTo(file(finaArtifactPath)) + file(wheelArtifactPath).renameTo(file(finalArtifactPath)) + //This is for pyspark which cannot consume whl file, but works perfectly fine with whl file renamed to zip... + copy { + from finalArtifactPath + into distDirectoryPath + rename finalArtifactName, finaArtifactNameZip + } } } @@ -59,9 +71,9 @@ sonarqube { property "sonar.sources", "$projectDir/cern" } } -\ - artifacts { - archives file(finaArtifactPath) + +artifacts { + archives file(finalArtifactPath), file(finaArtifactPathZip) } testWheel.dependsOn(testPythonVersion) -- GitLab From f557b36336a99df2b8236902023ad28367e25ca7 Mon Sep 17 00:00:00 2001 From: wjurasz <wiktor.jan.jurasz@cern.ch> Date: Wed, 14 Feb 2018 14:43:01 +0100 Subject: [PATCH 21/24] NXCALS-1468 applied review comments --- accsoft-nxcals-data-access-python3/build.gradle | 3 --- accsoft-nxcals-data-access-python3/setup.py | 9 +-------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/accsoft-nxcals-data-access-python3/build.gradle b/accsoft-nxcals-data-access-python3/build.gradle index f79ab4241f..8045725cee 100644 --- a/accsoft-nxcals-data-access-python3/build.gradle +++ b/accsoft-nxcals-data-access-python3/build.gradle @@ -1,6 +1,3 @@ -println "Module=" + name -println "Version=" + version - apply plugin: 'org.sonarqube' def pathToPythonInterpreter = '/user/bdisoft/operational/bin/Python/anaconda_py35/bin/python' diff --git a/accsoft-nxcals-data-access-python3/setup.py b/accsoft-nxcals-data-access-python3/setup.py index fb834d7bd9..4dcfb8d83f 100644 --- a/accsoft-nxcals-data-access-python3/setup.py +++ b/accsoft-nxcals-data-access-python3/setup.py @@ -1,5 +1,3 @@ -#!/opt/wjurasz/python3/installation/bin/python3 -# -*- coding: utf-8 -*- from setuptools import find_packages, setup import sys import argparse @@ -17,19 +15,14 @@ sys.argv.remove(version_argument + '=' + args.nxcals_version) NAME = 'accsoft-nxcals-data-access-python3' EMAIL = 'acclog@cern.ch' -# What packages are required for this module to be executed? -REQUIRED = [ - # 'requests', 'maya', 'records', -] -# Where the magic happens: + setup( name=NAME, version=args.nxcals_version, description='Python data extraction API for NXCALS system', author_email=EMAIL, packages=find_packages(exclude=('tests',)), - install_requires=REQUIRED, include_package_data=True, test_suite="tests", classifiers=[ -- GitLab From d1cd93aceeacec0adc3df0b6b5f3e54d38ec9d33 Mon Sep 17 00:00:00 2001 From: wjurasz <wiktor.jan.jurasz@cern.ch> Date: Tue, 20 Feb 2018 10:41:03 +0100 Subject: [PATCH 22/24] NXCALS-1468 Fixed order of python interpreters in gradle --- accsoft-nxcals-data-access-python3/build.gradle | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/accsoft-nxcals-data-access-python3/build.gradle b/accsoft-nxcals-data-access-python3/build.gradle index 8045725cee..8615cc2078 100644 --- a/accsoft-nxcals-data-access-python3/build.gradle +++ b/accsoft-nxcals-data-access-python3/build.gradle @@ -1,6 +1,8 @@ apply plugin: 'org.sonarqube' -def pathToPythonInterpreter = '/user/bdisoft/operational/bin/Python/anaconda_py35/bin/python' +def defaultPythonInterpreter = '/user/bdisoft/operational/bin/Python/anaconda_py35/bin/python' + +def pathToPythonInterpreter = null def pythonProjectName = name.replace('-', '_') def pythonProjectVersion = version.replace('-', '_') @@ -23,16 +25,17 @@ task testPythonVersion() { commandLine 'python', '--version' errorOutput = stdout } + if (project.hasProperty(pythonPathVariableName)) { pathToPythonInterpreter = pythonPath + } else if (file(defaultPythonInterpreter).exists()) { + pathToPythonInterpreter = defaultPythonInterpreter } else if (stdout.toString().startsWith('Python 3')) { pathToPythonInterpreter = 'python' } else if (exec { commandLine 'which', 'python3'; ignoreExitValue = true }.getExitValue() == 0) { pathToPythonInterpreter = 'python3' } else { - if (!file(pathToPythonInterpreter).exists()) { - throw new GradleException("To proceed python3 interpreter has to be provided, either as default intepreter (ran by \"python\" or \"python3\" command) or by specifying -P$pythonPathVariableName=/path/to/python3 gradle option") - } + throw new GradleException("To proceed python3 interpreter has to be provided, either as default intepreter (ran by \"python\" or \"python3\" command) or by specifying -P$pythonPathVariableName=/path/to/python3 gradle option") } } @@ -41,7 +44,7 @@ task cleanWheel(type: Delete) { exec { commandLine pathToPythonInterpreter, 'setup.py', 'clean', '--all', "--nxcals-version=$currentVersion" } - delete distDirectoryPath, pythonProjectName + '.egg-info', fileTree('.') { include '**/*.pyc'} + delete distDirectoryPath, pythonProjectName + '.egg-info', fileTree('.') { include '**/*.pyc' } } -- GitLab From e6d7f048f4d62de08564d48b99136b6b4c9a2c23 Mon Sep 17 00:00:00 2001 From: wjurasz <wiktor.jan.jurasz@cern.ch> Date: Tue, 20 Feb 2018 10:46:38 +0100 Subject: [PATCH 23/24] NXCALS-1468 Added comment --- accsoft-nxcals-data-access-python3/build.gradle | 2 ++ 1 file changed, 2 insertions(+) diff --git a/accsoft-nxcals-data-access-python3/build.gradle b/accsoft-nxcals-data-access-python3/build.gradle index 8615cc2078..f2782a9fd7 100644 --- a/accsoft-nxcals-data-access-python3/build.gradle +++ b/accsoft-nxcals-data-access-python3/build.gradle @@ -19,6 +19,8 @@ def finaArtifactPathZip = distDirectoryPath + finaArtifactNameZip def pythonPathVariableName = 'pythonPath' +//Beside appropriate version following packages have to be present in python env: +//"setuptools" and "wheel" task testPythonVersion() { def stdout = new ByteArrayOutputStream() exec { -- GitLab From 56bc2441fa7f882e5b0563231946d1ba7948dac4 Mon Sep 17 00:00:00 2001 From: wjurasz <wiktor.jan.jurasz@cern.ch> Date: Tue, 20 Feb 2018 10:49:07 +0100 Subject: [PATCH 24/24] NXCALS-1468 Improved error message --- accsoft-nxcals-data-access-python3/build.gradle | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/accsoft-nxcals-data-access-python3/build.gradle b/accsoft-nxcals-data-access-python3/build.gradle index f2782a9fd7..dac2823132 100644 --- a/accsoft-nxcals-data-access-python3/build.gradle +++ b/accsoft-nxcals-data-access-python3/build.gradle @@ -37,7 +37,9 @@ task testPythonVersion() { } else if (exec { commandLine 'which', 'python3'; ignoreExitValue = true }.getExitValue() == 0) { pathToPythonInterpreter = 'python3' } else { - throw new GradleException("To proceed python3 interpreter has to be provided, either as default intepreter (ran by \"python\" or \"python3\" command) or by specifying -P$pythonPathVariableName=/path/to/python3 gradle option") + throw new GradleException('To proceed python3 interpreter has to be provided, either as default intepreter (ran by \"python\" or \"python3\" command) ' + + 'or by specifying -P$pythonPathVariableName=/path/to/python3 gradle option.' + + 'Also following packages have to be present in python env: setooptools and wheel') } } -- GitLab