From 4d0a8058d06391fcf822768b1629f6309fa17ade Mon Sep 17 00:00:00 2001
From: Joao Lopes <joao.pedro.batista.lopes@cern.ch>
Date: Tue, 14 Jun 2022 23:00:49 +0200
Subject: [PATCH] Start Tape REST API funtional tests
- Introduces TestCaseBase(..) as a base class for all the tests
- Starts functional tests for bring online operations
- Implements a GfalWrapper(..) class that extends some functionalities of the gfal2 python bindings
---
tape-rest-api/libs/config.py | 11 ++
tape-rest-api/libs/gfal_helper.py | 47 +++++++++
tape-rest-api/test_bringonline.py | 162 ++++++++++++++++++++++++++++++
tape-rest-api/test_case_base.py | 39 +++++++
4 files changed, 259 insertions(+)
create mode 100644 tape-rest-api/libs/config.py
create mode 100644 tape-rest-api/libs/gfal_helper.py
create mode 100644 tape-rest-api/test_bringonline.py
create mode 100644 tape-rest-api/test_case_base.py
diff --git a/tape-rest-api/libs/config.py b/tape-rest-api/libs/config.py
new file mode 100644
index 0000000..4d14899
--- /dev/null
+++ b/tape-rest-api/libs/config.py
@@ -0,0 +1,11 @@
+# Tape REST API tests configuration
+import os
+
+TapeBaseDirectory = os.environ.get('TAPE_ENDPOINT', "replacethis")
+SourceFile = os.environ.get('TEST_FILE', 'file:///etc/hosts')
+
+# Size of bulk requests
+BulkSize = 10
+
+# Max Polling interval in seconds
+MaxPollInterval = 10
diff --git a/tape-rest-api/libs/gfal_helper.py b/tape-rest-api/libs/gfal_helper.py
new file mode 100644
index 0000000..dd643ab
--- /dev/null
+++ b/tape-rest-api/libs/gfal_helper.py
@@ -0,0 +1,47 @@
+import gfal2
+import uuid
+import errno
+
+
+def generate_random_url(root, filename):
+ return root + filename + '_' + str(uuid.uuid4())
+
+
+class GfalWrapper:
+
+ def __init__(self):
+ self.context = gfal2.creat_context()
+
+ def copy_file(self, source, destination, timeout, overwrite=False):
+ params = self.context.transfer_parameters()
+ params.timeout = timeout
+ params.overwrite = overwrite
+ return self.context.filecopy(params, source, destination)
+
+ def release_list(self, files, token):
+ return [self.context.release(file, token) for file in files]
+
+ def rm_list(self, files):
+ for file in files:
+ self.context.unlink(file)
+
+ def bring_online_poll_list(self, urls, token):
+ errors = self.context.bring_online_poll(urls, token)
+ error_count = 0
+ online_count = 0
+
+ for error in errors:
+ if error is None:
+ online_count += 1
+ elif error.code != errno.EAGAIN:
+ error_count += 1
+ if error_count == len(errors):
+ return -1, errors
+ elif online_count == len(errors):
+ return 1, errors
+ elif (online_count + error_count) == len(errors):
+ return 2, errors
+
+ # Request still not finished
+ return 0, errors
+
diff --git a/tape-rest-api/test_bringonline.py b/tape-rest-api/test_bringonline.py
new file mode 100644
index 0000000..5d3b141
--- /dev/null
+++ b/tape-rest-api/test_bringonline.py
@@ -0,0 +1,162 @@
+#!/usr/bin/python3
+
+import time
+from test_case_base import *
+
+
+class TestBringOnlineSingle(TestCaseBase):
+
+ # Stage one single file
+ def test_single_file(self):
+ (status, token) = self.handle.context.bring_online(self.url, 60, 60, False)
+ self.assertEqual(status, 0)
+ while status == 0:
+ status = self.handle.context.bring_online_poll(self.url, token)
+ self.assertEqual(status, 1)
+
+ # Stage a file that does not exist
+ def test_single_file_enoent(self):
+ file = generate_random_url(self.root, "tape_rest_api_enoent")
+ (status, token) = self.handle.context.bring_online(file, 60, 60, False)
+ self.assertEqual(status, 0)
+ try:
+ status = self.handle.context.bring_online_poll(file, token)
+ self.assertEqual(status, -1)
+ except Exception as e:
+ self.assertEqual(e.code, errno.ENOMSG)
+
+ # Stage bulk request
+ def test_bulk_request(self):
+ self._upload_and_register_files(self.bulk_size)
+ (errors, token) = self.handle.context.bring_online(self.remote_files, 60, 60, False)
+ sleep = 1
+ status = 0
+ while status == 0:
+ print("Polling")
+ (status, errors) = self.handle.bring_online_poll_list(self.remote_files, token)
+ time.sleep(sleep)
+ sleep *= 2
+ sleep = min(sleep, self.max_poll_interval)
+
+ self.assertEqual(status, 1)
+ self.assertAllNone(errors)
+
+ # Stage bulk request (some files in the request do not exist)
+ def test_bulk_request_enoent(self):
+ files_enoent = [generate_random_url(self.root, "tape_rest_api_enoent") for _ in range(self.bulk_size)]
+ self._upload_and_register_files(self.bulk_size)
+ urls = [el for pair in zip(self.remote_files, files_enoent) for el in pair]
+ (errors, token) = self.handle.context.bring_online(urls, 60, 60, False)
+
+ sleep = 1
+ status = 0
+ while status == 0:
+ print("Polling")
+ (status, errors) = self.handle.bring_online_poll_list(urls, token)
+ time.sleep(sleep)
+ sleep *= 2
+ sleep = min(sleep, 10)
+
+ self.assertEqual(status, 2)
+ self.assertAllNone(errors[0::2])
+ self.assertAllEqual([error.code for error in errors[1::2]], errno.ENOMSG)
+
+ # Stage a list of files with duplicated entries
+ def test_duplicates(self):
+ random_url = generate_random_url(self.root, "tape_rest_api_enoent")
+ files = [self.url, random_url] * 10
+ (errors, token) = self.handle.context.bring_online(files, 60, 60, False)
+
+ sleep = 1
+ status = 0
+ while status == 0:
+ (status, errors) = self.handle.bring_online_poll_list(files, token)
+ time.sleep(sleep)
+ sleep *= 2
+ sleep = min(sleep, 30)
+
+ self.assertEqual(status, 2)
+ self.assertAllNone(errors[0::2])
+ self.assertAllEqual([error.code for error in errors[1::2]], errno.ENOMSG)
+
+ # Poll with an invalid token
+ def test_invalid_token(self):
+ (status, token) = self.handle.context.bring_online(self.url, 60, 60, False)
+ self.assertEqual(status, 0)
+ try:
+ status = self.handle.context.bring_online_poll(self.url, "abcde-12345")
+ self.assertEqual(status, -1)
+ except Exception as e:
+ self.assertEqual(e.code, errno.EINVAL)
+
+ # Release a file
+ def test_release_file(self):
+ # Stage file to disk
+ (status, token) = self.handle.context.bring_online(self.url, 60, 60, False)
+ self.assertEqual(status, 0)
+ while status == 0:
+ status = self.handle.context.bring_online_poll(self.url, token)
+ self.assertEqual(status, 1)
+ # Release file
+ status = self.handle.context.release(self.url, token)
+ self.assertEqual(status, 0)
+
+ # Release a file with invalid token
+ def test_release_file_invalid_token(self):
+ # Stage file to disk
+ (status, token) = self.handle.context.bring_online(self.url, 60, 60, False)
+ self.assertEqual(status, 0)
+ # Release file with wrong token is not an error
+ status = self.handle.context.release(self.url, "abcde-12345")
+ self.assertEqual(status, 0)
+
+ # Release a wrong file
+ def test_release_wrong_file(self):
+ # Stage file to disk
+ (status, token) = self.handle.context.bring_online(self.url, 60, 60, False)
+ self.assertEqual(status, 0)
+ # Release file that does not belong to the request ID
+ try:
+ random_url = generate_random_url(self.root, "tape_rest_api_enoent")
+ status = self.handle.context.abort_bring_online(random_url, token)
+ self.assertEqual(status, -1)
+ except Exception as e:
+ self.assertEqual(e.code, errno.EINVAL)
+
+ # Abort a staging request
+ def test_abort_request(self):
+ # Stage file to disk
+ (status, token) = self.handle.context.bring_online(self.url, 60, 60, False)
+ self.assertEqual(status, 0)
+ # Abort request
+ status = self.handle.context.abort_bring_online(self.url, token)
+ self.assertEqual(status, 0)
+
+ # Abort a staging request with wrong token
+ def test_abort_request_invalid_token(self):
+ # Stage file to disk
+ (status, token) = self.handle.context.bring_online(self.url, 60, 60, False)
+ self.assertEqual(status, 0)
+ # Abort request with wrong token
+ try:
+ status = self.handle.context.abort_bring_online(self.url, "abcde-12345")
+ self.assertEqual(status, -1)
+ except Exception as e:
+ self.assertEqual(e.code, errno.EINVAL)
+
+ # Abort a staging request with wrong file
+ def test_abort_request_wrong_file(self):
+ # Stage file to disk
+ (status, token) = self.handle.context.bring_online(self.url, 60, 60, False)
+ self.assertEqual(status, 0)
+ # Abort request with wrong file
+ try:
+ random_url = generate_random_url(self.root, "tape_rest_api_enoent")
+ status = self.handle.context.abort_bring_online(random_url, token)
+ self.assertEqual(status, -1)
+ except Exception as e:
+ self.assertEqual(e.code, errno.EINVAL)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/tape-rest-api/test_case_base.py b/tape-rest-api/test_case_base.py
new file mode 100644
index 0000000..239772b
--- /dev/null
+++ b/tape-rest-api/test_case_base.py
@@ -0,0 +1,39 @@
+import unittest
+from libs.gfal_helper import *
+from libs import config
+
+
+class TestCaseBase(unittest.TestCase):
+
+ def setUp(self):
+ self.handle = GfalWrapper()
+ self.source = config.SourceFile
+ self.root = config.TapeBaseDirectory
+ self.bulk_size = config.BulkSize
+ self.max_poll_interval = config.MaxPollInterval
+ self.url = generate_random_url(self.root, "tape_rest_api")
+ status = self.handle.copy_file(self.source, self.url, 60, False)
+ self.assertEqual(status, 0)
+ self.remote_files = [self.url, ]
+
+ def tearDown(self):
+ # Remove all files uploaded to the remote host
+ self.handle.rm_list(self.remote_files)
+
+ def _upload_and_register_files(self, n_files=1):
+ src = [self.source]*n_files
+ dst = []
+ for i in range(n_files):
+ dst.append(generate_random_url(self.root, "tape_rest_api"))
+ errors = self.handle.copy_file(src, dst, 60, False)
+ for err in errors:
+ self.assertIsNone(err)
+ self.remote_files += dst
+
+ def assertAllEqual(self, seq, key):
+ for el in seq:
+ self.assertEqual(el, key)
+
+ def assertAllNone(self, seq):
+ for el in seq:
+ self.assertIsNone(el)
--
GitLab