Optimisation of JAR downloading
Merge request related to issue #6 (closed) proposed by @pelson
As far cmmnbuild_dep_manager was cleaning lib directory during resolve step regardless present jars were useful or not. After the change Manager will only remove jars that are not included in resolved dependencies and later download only missing files.
Edited by Pawel Budzynski
Merge request reports
Activity
187 187 )) 188 188 return classes 189 189 190 def _get_jar_basename(self, jar): changed this line in version 9 of the diff
634 637 )) 635 638 shutil.rmtree(old_jars_dir) 636 639 637 # Remove existing jars 638 old_jars = glob.glob(os.path.join(self.jar_path(), '*.jar')) 640 # Remove existing unnecessary jars 641 existing_jars = glob.glob(os.path.join(self.jar_path(), '*.jar')) 642 requited_jars = [dep['uri'].split('/')[-1] for dep in resolver.resolved_deps] changed this line in version 9 of the diff
644 jars_to_delete = [jar for jar in existing_jars 645 if self._get_jar_basename(jar) not in requited_jars] 646 639 647 self.log.info('removing {0} jars from {1}'.format( 640 len(old_jars), self.jar_path() 648 len(jars_to_delete), self.jar_path() 641 649 )) 642 for jar in old_jars: 650 for jar in jars_to_delete: 643 651 os.remove(jar) 644 652 653 # Check which of required jars are already present and skip them 654 # while downloading 655 required_jars_present = [self._get_jar_basename(jar) for jar in existing_jars 656 if self._get_jar_basename(jar) in requited_jars] 657 self.log.info("Already have {num}/{total} of required jars. " changed this line in version 9 of the diff
650 for jar in jars_to_delete: 643 651 os.remove(jar) 644 652 653 # Check which of required jars are already present and skip them 654 # while downloading 655 required_jars_present = [self._get_jar_basename(jar) for jar in existing_jars 656 if self._get_jar_basename(jar) in requited_jars] 657 self.log.info("Already have {num}/{total} of required jars. " 658 "Will skip them while download: {jars}" 659 .format(num=len(required_jars_present), 660 total=len(requited_jars), 661 jars=required_jars_present)) 662 645 663 # Deploy new jars 646 resolver.save_jars(self.jar_path()) 664 resolver.save_jars(self.jar_path(), to_skip=required_jars_present) changed this line in version 12 of the diff
59 59 sys.modules.pop(name) 60 60 61 61 62 def create_mock_jar_file(dir, filename): changed this line in version 12 of the diff
288 289 290 def test_download_remove_unnecessary_jars(mocked_mgr): 291 mgr, resolver = mocked_mgr 292 # pre-defined result from resolver 293 resolver().resolved_deps = [ 294 {'uri': 'www.some.where/group/atr/todownload.jar'}, 295 {'uri': 'www.some.where/group/atr/todownload2.jar'}] 296 297 create_mock_jar_file(mgr.jar_path(), "existing.jar") 298 299 with tmp_mod('fakemod3', version='1.2.3', dependencies=['dep', ]): 300 mgr.install('fakemod3') 301 302 resolver().save_jars.assert_called_once_with(mgr.jar_path(), to_skip=[]) 303 assert "existing.jar" not in [f.name for f in mgr.jar_path().iterdir()] changed this line in version 12 of the diff
91 97 92 98 93 99 @contextmanager 94 100 def no_pre_installed_modules(): changed this line in version 10 of the diff
634 634 )) 635 635 shutil.rmtree(old_jars_dir) 636 636 637 # Remove existing jars 638 old_jars = glob.glob(os.path.join(self.jar_path(), '*.jar')) 637 # Remove existing unnecessary jars 638 existing_jars = glob.glob(os.path.join(self.jar_path(), '*.jar')) 639 required_jars = [dep['uri'].split('/')[-1] for dep in resolver.resolved_deps] changed this line in version 12 of the diff
added 1 commit
- 8ccc888b - tests don't produce random modules.json content anymore
10 9 11 10 import pytest 12 11 13 from cmmnbuild_dep_manager.cmmnbuild_dep_manager import PKG_MODULES_JSON 12 import cmmnbuild_dep_manager as cbdm 14 13 15 14 16 TMP_PKG_MODULES_JSON = ( 17 PKG_MODULES_JSON.parent / ("_test_" + PKG_MODULES_JSON.name)) 18 LIB_DIR = PKG_MODULES_JSON.parent / 'lib' 19 TMP_LIB_DIR = PKG_MODULES_JSON.parent / '_test_lib' 20 21 22 @pytest.fixture(autouse=True, scope="session") 20 21 22 @pytest.fixture(autouse=True, scope="session") 23 def autoclean_modules_json(): 24 # NOTE: Runs one-startup and one teardown (scope=session). 25 # Each test should still make sure that it tidies up properly for the 26 # sake of test consistency - this fixture simply ensures that the pre-test 27 # state is preserved after testing. 28 29 if LIB_DIR.exists(): 30 if TMP_LIB_DIR.exists(): 31 raise RuntimeError( 32 f'Testing already in progress - {TMP_LIB_DIR} exists already') 33 LIB_DIR.rename(TMP_LIB_DIR) 15 @pytest.fixture(scope='function') 16 def manager(tmp_lib_dir): changed this line in version 11 of the diff
Please register or sign in to reply