Skip to content
Snippets Groups Projects
Commit ddc68751 authored by Guido Sterbini's avatar Guido Sterbini
Browse files

cosmetics

parent 07f5ab18
No related branches found
No related tags found
No related merge requests found
Pipeline #2734111 passed
Stage: test
Hide whitespace changes
Inline Side-by-side
examples/001_example/001.ipynb
+ 27
− 27
View file @ ddc68751
%% Cell type:markdown id:709283b4 tags: %% Cell type:markdown id:8114a275 tags:
Let aussume that we need to make this computation Let aussume that we need to make this computation
$\sqrt{|(a+b)\times c|}$ $\sqrt{|(a+b)\times c|}$
and we want to compute the standard deviation of the result assuming that a, b and c are normal distributed independent variables. Clearly the problem is quite naive but we want to address is as if we will need a cluster to solve it. and we want to compute the standard deviation of the result assuming that a, b and c are normal distributed independent variables. Clearly the problem is quite naive but we want to address is as if we will need a cluster to solve it.
We can partition the problem in a three conscutive operations We can partition the problem in a three conscutive operations
1. A sum: $(a+b)$ 1. A sum: $(a+b)$
2. A multiplication of the result 1 with c: $(a+b)\times c$ 2. A multiplication of the result 1 with c: $(a+b)\times c$
3. A sqrt of the result of 2: $\sqrt{|(a+b)\times c|}$ 3. A sqrt of the result of 2: $\sqrt{|(a+b)\times c|}$
See https://gitlab.cern.ch/abpcomputing/sandbox/tree_maker for the installation. See https://gitlab.cern.ch/abpcomputing/sandbox/tree_maker for the installation.
Documentation (only started, you need to be on GPN) can be found at https://acc-py.web.cern.ch/gitlab/abpcomputing/sandbox/tree_maker/docs/master/. Documentation (only started, you need to be on GPN) can be found at https://acc-py.web.cern.ch/gitlab/abpcomputing/sandbox/tree_maker/docs/master/.
%% Cell type:code id:ff6ab8d0 tags: %% Cell type:code id:2263dcb4 tags:
``` python ``` python
import tree_maker import tree_maker
from tree_maker import NodeJob from tree_maker import NodeJob
``` ```
%% Cell type:code id:71eb56f4 tags: %% Cell type:code id:06366b27 tags:
``` python ``` python
# Clearly for this easy task on can do all in the very same python kernel # Clearly for this easy task on can do all in the very same python kernel
# BUT here we want to mimic the typical flow # BUT here we want to mimic the typical flow
# 1. MADX for optics matching/error seeding # 1. MADX for optics matching/error seeding
# 2. Tracking for FMA and or DA studies # 2. Tracking for FMA and or DA studies
# 3. simulation baby-sitting and # 3. simulation baby-sitting and
# 4. postprocessing # 4. postprocessing
import numpy as np import numpy as np
a=np.random.randn(4) a=np.random.randn(4)
b=np.random.randn(4) b=np.random.randn(4)
c=np.random.randn(2) c=np.random.randn(2)
my_list_original=[] my_list_original=[]
for ii in c: for ii in c:
my_list_original+=list(np.sqrt(np.abs((a+b)*ii))) my_list_original+=list(np.sqrt(np.abs((a+b)*ii)))
my_list_original=sorted(my_list_original) my_list_original=sorted(my_list_original)
``` ```
%% Cell type:code id:e58f9d94 tags: %% Cell type:code id:b3538e96 tags:
``` python ``` python
#root #root
root = NodeJob(name='root', parent=None) root = NodeJob(name='root', parent=None)
# to be modified accordingly # to be modified accordingly
root.path = '/home/jovyan/local_host_home/CERNBox/2021/tree_maker/examples/001_example/study_000' root.path = '/home/jovyan/local_host_home/CERNBox/2021/tree_maker/examples/001_example/study_000'
root.dictionary = {'log_file': f"{root.path}/log.yaml"} root.dictionary = {'log_file': f"{root.path}/log.yaml"}
root.clean_log() root.clean_log()
``` ```
%% Cell type:code id:83d64c0e tags: %% Cell type:code id:6f68ad18 tags:
``` python ``` python
#first generation #first generation
for node in root.leaves: for node in root.leaves:
node.children=[NodeJob(name=f"{child:03}", node.children=[NodeJob(name=f"{child:03}",
parent=node, parent=node,
path = f"{node.path}/{child:03}", path = f"{node.path}/{child:03}",
template_path = root.path+'/../templates/sum_it', template_path = root.path+'/../templates/sum_it',
submit_command = f'python run.py', submit_command = f'python run.py',
dictionary={'a':float(a[child]), dictionary={'a':float(a[child]),
'b':float(b[child]), 'b':float(b[child]),
'log_file': f"{node.path}/{child:03}/log.yaml" 'log_file': f"{node.path}/{child:03}/log.yaml"
}) })
for child in range(len(a))] for child in range(len(a))]
# To combine different lists one can use the product or the zip functions # To combine different lists one can use the product or the zip functions
#import itertools #import itertools
#[[i, j, z] for i, j, z in itertools.product(['a','b'],['c','d'],[1,2,3])] #[[i, j, z] for i, j, z in itertools.product(['a','b'],['c','d'],[1,2,3])]
#[[i, j, z] for i, j, z in zip(['a','b'],['c','d'],[1,2,3])] #[[i, j, z] for i, j, z in zip(['a','b'],['c','d'],[1,2,3])]
root.print_it() root.print_it()
``` ```
%% Output %% Output
root root
├── 000 ├── 000
├── 001 ├── 001
├── 002 ├── 002
╰── 003 ╰── 003
%% Cell type:code id:8a64d9ce tags: %% Cell type:code id:346457b6 tags:
``` python ``` python
#second generation #second generation
for node in root.leaves: for node in root.leaves:
node.children=[NodeJob(name=f"{child:03}", node.children=[NodeJob(name=f"{child:03}",
parent=node, parent=node,
path = f"{node.path}/{child:03}", path = f"{node.path}/{child:03}",
template_path = root.path+'/../templates/multiply_it', template_path = root.path+'/../templates/multiply_it',
submit_command = f'python run.py', submit_command = f'python run.py',
dictionary={'parent':f'{node.path}', dictionary={'parent':f'{node.path}',
'c': float(c[child]), 'c': float(c[child]),
'log_file': f'{node.path}/{child:03}/log.yaml', 'log_file': f'{node.path}/{child:03}/log.yaml',
}) })
for child in range(len(c))] for child in range(len(c))]
root.print_it() root.print_it()
``` ```
%% Output %% Output
root root
├── 000 ├── 000
│ ├── 000 │ ├── 000
│ ╰── 001 │ ╰── 001
├── 001 ├── 001
│ ├── 000 │ ├── 000
│ ╰── 001 │ ╰── 001
├── 002 ├── 002
│ ├── 000 │ ├── 000
│ ╰── 001 │ ╰── 001
╰── 003 ╰── 003
├── 000 ├── 000
╰── 001 ╰── 001
%% Cell type:code id:7b08541c tags: %% Cell type:code id:a88d1b31 tags:
``` python ``` python
#third generation #third generation
for node in root.leaves: for node in root.leaves:
node.children=[NodeJob(name=f"{child:03}", node.children=[NodeJob(name=f"{child:03}",
parent=node, parent=node,
path = f"{node.path}/{child:03}", path = f"{node.path}/{child:03}",
template_path = root.path+'/../templates/square_root_it', template_path = root.path+'/../templates/square_root_it',
submit_command = f'python run.py', submit_command = f'python run.py',
dictionary={'parent':f'{node.path}', dictionary={'parent':f'{node.path}',
'log_file': f"{node.path}/{child:03}/log.yaml", 'log_file': f"{node.path}/{child:03}/log.yaml",
'test': {'is an example':4} 'test': {'is an example':4}
}) })
for child in range(1)] for child in range(1)]
root.print_it() root.print_it()
``` ```
%% Output %% Output
root root
├── 000 ├── 000
│ ├── 000 │ ├── 000
│ │ ╰── 000 │ │ ╰── 000
│ ╰── 001 │ ╰── 001
│ ╰── 000 │ ╰── 000
├── 001 ├── 001
│ ├── 000 │ ├── 000
│ │ ╰── 000 │ │ ╰── 000
│ ╰── 001 │ ╰── 001
│ ╰── 000 │ ╰── 000
├── 002 ├── 002
│ ├── 000 │ ├── 000
│ │ ╰── 000 │ │ ╰── 000
│ ╰── 001 │ ╰── 001
│ ╰── 000 │ ╰── 000
╰── 003 ╰── 003
├── 000 ├── 000
│ ╰── 000 │ ╰── 000
╰── 001 ╰── 001
╰── 000 ╰── 000
%% Cell type:code id:f4a65e8e tags: %% Cell type:code id:b7e9a224 tags:
``` python ``` python
# we can also modify the submit command # we can also modify the submit command
if False: if False:
for i, node in enumerate(root.leaves): for i, node in enumerate(root.leaves):
if i>3: if i>3:
print(i) print(i)
node.submit_command = f'condor_submit run.sub -batch-name square_root' node.submit_command = f'condor_submit run.sub -batch-name square_root'
``` ```
%% Cell type:code id:c5218b5d tags: %% Cell type:code id:4e0373c0 tags:
``` python ``` python
# we can inspect the data structure # we can inspect the data structure
root.children[3].children[1].children[0].submit_command root.children[3].children[1].children[0].submit_command
``` ```
%% Output %% Output
'python run.py' 'python run.py'
%% Cell type:code id:42ee35ce tags: %% Cell type:code id:16e2e0b7 tags:
``` python ``` python
root.to_yaml() root.to_yaml()
``` ```
%% Cell type:code id:0a14d92a tags: %% Cell type:code id:ebffa95a tags:
``` python ``` python
# important concept of generation # important concept of generation
root.generation(2) root.generation(2)
``` ```
%% Output %% Output
[<tree_maker.NodeJob.NodeJob at 0x7f4a12257290>, [<tree_maker.NodeJob.NodeJob at 0x7f4a12257290>,
<tree_maker.NodeJob.NodeJob at 0x7f4a12257310>, <tree_maker.NodeJob.NodeJob at 0x7f4a12257310>,
<tree_maker.NodeJob.NodeJob at 0x7f4a12257390>, <tree_maker.NodeJob.NodeJob at 0x7f4a12257390>,
<tree_maker.NodeJob.NodeJob at 0x7f4a12257410>, <tree_maker.NodeJob.NodeJob at 0x7f4a12257410>,
<tree_maker.NodeJob.NodeJob at 0x7f4a12257490>, <tree_maker.NodeJob.NodeJob at 0x7f4a12257490>,
<tree_maker.NodeJob.NodeJob at 0x7f4a12257510>, <tree_maker.NodeJob.NodeJob at 0x7f4a12257510>,
<tree_maker.NodeJob.NodeJob at 0x7f4a12257590>, <tree_maker.NodeJob.NodeJob at 0x7f4a12257590>,
<tree_maker.NodeJob.NodeJob at 0x7f4a12257610>] <tree_maker.NodeJob.NodeJob at 0x7f4a12257610>]
%% Cell type:code id:eea5d4a7 tags: %% Cell type:code id:1dc15a4f tags:
``` python ``` python
# We map the pythonic tree in a >folder< tree # We map the pythonic tree in a >folder< tree
root.clean_log() root.clean_log()
root.rm_children_folders() root.rm_children_folders()
for depth in range(root.height): for depth in range(root.height):
[x.clone_children() for x in root.generation(depth)] [x.clone_children() for x in root.generation(depth)]
root.tag_as('cloned') root.tag_as('cloned')
``` ```
%% Cell type:code id:256c5c69 tags: %% Cell type:code id:686da9ab tags:
``` python ``` python
root.tag_as('launched') root.tag_as('launched')
for node in root.generation(1): for node in root.generation(1):
node.clean_log() node.clean_log()
node.tag_as('mutated') node.tag_as('mutated')
node.mutate() node.mutate()
node.tag_as('submitted') node.tag_as('submitted')
node.submit() node.submit()
``` ```
%% Cell type:code id:91dff0cb tags: %% Cell type:code id:9d9db8e5 tags:
``` python ``` python
def cleanlog_mutate_submit(self): def cleanlog_mutate_submit(self):
self.clean_log() self.clean_log()
self.tag_as('mutated') self.tag_as('mutated')
self.mutate() self.mutate()
self.tag_as('submitted') self.tag_as('submitted')
self.submit() self.submit()
NodeJob.cleanlog_mutate_submit=cleanlog_mutate_submit NodeJob.cleanlog_mutate_submit=cleanlog_mutate_submit
``` ```
%% Cell type:code id:2a39303e tags: %% Cell type:code id:93e71fc5 tags:
``` python ``` python
for node in root.generation(2): for node in root.generation(2):
node.cleanlog_mutate_submit() node.cleanlog_mutate_submit()
``` ```
%% Cell type:code id:b2e57e4b tags: %% Cell type:code id:230e71ee tags:
``` python ``` python
for node in root.generation(3): for node in root.generation(3):
node.cleanlog_mutate_submit() node.cleanlog_mutate_submit()
``` ```
%% Cell type:code id:734792fd tags: %% Cell type:code id:78d67569 tags:
``` python ``` python
# check if all root descendants are completed # check if all root descendants are completed
if all([descendant.has_been('completed') for descendant in root.descendants]): if all([descendant.has_been('completed') for descendant in root.descendants]):
root.tag_as('completed') root.tag_as('completed')
``` ```
%% Cell type:code id:8e41a894 tags: %% Cell type:code id:1db375fa tags:
``` python ``` python
# retrieve the output # retrieve the output
my_list=[] my_list=[]
for node in root.leaves: for node in root.leaves:
output = tree_maker.from_yaml(node.path+'/output.yaml') output = tree_maker.from_yaml(node.path+'/output.yaml')
my_list.append(output['result']) my_list.append(output['result'])
``` ```
%% Cell type:code id:2276f255 tags: %% Cell type:code id:03bd9fee tags:
``` python ``` python
# sanity check # sanity check
assert any(np.array(sorted(my_list))-np.array(my_list_original))==0 assert any(np.array(sorted(my_list))-np.array(my_list_original))==0
``` ```
%% Cell type:code id:03dcbf0b tags: %% Cell type:code id:1c4e1436 tags:
``` python ``` python
root=tree_maker.tree_from_yaml(f'/home/jovyan/local_host_home/CERNBox/2021/tree_maker/examples/001_example/study_000/tree.yaml') root=tree_maker.tree_from_yaml(f'/home/jovyan/local_host_home/CERNBox/2021/tree_maker/examples/001_example/study_000/tree.yaml')
``` ```
%% Cell type:code id:99cce676 tags: %% Cell type:code id:8a89e5f6 tags:
``` python ``` python
for node in root.find(filter_= for node in root.find(filter_=
lambda node: node.is_leaf and lambda node: node.is_leaf and
node.has_been('completed')): node.has_been('completed')):
print(node.path) print(node.path)
``` ```
%% Output %% Output
/home/jovyan/local_host_home/CERNBox/2021/tree_maker/examples/001_example/study_000/000/000/000 /home/jovyan/local_host_home/CERNBox/2021/tree_maker/examples/001_example/study_000/000/000/000
/home/jovyan/local_host_home/CERNBox/2021/tree_maker/examples/001_example/study_000/000/001/000 /home/jovyan/local_host_home/CERNBox/2021/tree_maker/examples/001_example/study_000/000/001/000
/home/jovyan/local_host_home/CERNBox/2021/tree_maker/examples/001_example/study_000/001/000/000 /home/jovyan/local_host_home/CERNBox/2021/tree_maker/examples/001_example/study_000/001/000/000
/home/jovyan/local_host_home/CERNBox/2021/tree_maker/examples/001_example/study_000/001/001/000 /home/jovyan/local_host_home/CERNBox/2021/tree_maker/examples/001_example/study_000/001/001/000
/home/jovyan/local_host_home/CERNBox/2021/tree_maker/examples/001_example/study_000/002/000/000 /home/jovyan/local_host_home/CERNBox/2021/tree_maker/examples/001_example/study_000/002/000/000
/home/jovyan/local_host_home/CERNBox/2021/tree_maker/examples/001_example/study_000/002/001/000 /home/jovyan/local_host_home/CERNBox/2021/tree_maker/examples/001_example/study_000/002/001/000
/home/jovyan/local_host_home/CERNBox/2021/tree_maker/examples/001_example/study_000/003/000/000 /home/jovyan/local_host_home/CERNBox/2021/tree_maker/examples/001_example/study_000/003/000/000
/home/jovyan/local_host_home/CERNBox/2021/tree_maker/examples/001_example/study_000/003/001/000 /home/jovyan/local_host_home/CERNBox/2021/tree_maker/examples/001_example/study_000/003/001/000
%% Cell type:code id:2a88a22b tags: %% Cell type:code id:461b8fb9 tags:
``` python ``` python
def my_test(node): def my_test(node):
output = tree_maker.from_yaml(node.path+'/output.yaml') output = tree_maker.from_yaml(node.path+'/output.yaml')
return output['result']<.3 return output['result']<.3
for node in root.find(filter_=lambda node: node.is_leaf and for node in root.find(filter_=lambda node: node.is_leaf and
node.has_been('completed') and node.has_been('completed') and
my_test(node)): my_test(node)):
print(node.path) print(node.path)
output = tree_maker.from_yaml(node.path+'/output.yaml') output = tree_maker.from_yaml(node.path+'/output.yaml')
print(f'{output["result"]}'+'\n') print(f'{output["result"]}'+'\n')
``` ```
%% Output %% Output
/home/jovyan/local_host_home/CERNBox/2021/tree_maker/examples/001_example/study_000/000/000/000 /home/jovyan/local_host_home/CERNBox/2021/tree_maker/examples/001_example/study_000/000/000/000
0.22973667009077783 0.22973667009077783
/home/jovyan/local_host_home/CERNBox/2021/tree_maker/examples/001_example/study_000/000/001/000 /home/jovyan/local_host_home/CERNBox/2021/tree_maker/examples/001_example/study_000/000/001/000
0.2055935932288798 0.2055935932288798
/home/jovyan/local_host_home/CERNBox/2021/tree_maker/examples/001_example/study_000/002/000/000 /home/jovyan/local_host_home/CERNBox/2021/tree_maker/examples/001_example/study_000/002/000/000
0.22105104624872 0.22105104624872
/home/jovyan/local_host_home/CERNBox/2021/tree_maker/examples/001_example/study_000/002/001/000 /home/jovyan/local_host_home/CERNBox/2021/tree_maker/examples/001_example/study_000/002/001/000
0.19782074349436635 0.19782074349436635
/home/jovyan/local_host_home/CERNBox/2021/tree_maker/examples/001_example/study_000/003/000/000 /home/jovyan/local_host_home/CERNBox/2021/tree_maker/examples/001_example/study_000/003/000/000
0.2646190341828162 0.2646190341828162
/home/jovyan/local_host_home/CERNBox/2021/tree_maker/examples/001_example/study_000/003/001/000 /home/jovyan/local_host_home/CERNBox/2021/tree_maker/examples/001_example/study_000/003/001/000
0.23681016205599148 0.23681016205599148
%% Cell type:code id:fa51e5c2 tags: %% Cell type:code id:7c2971ee tags:
``` python ``` python
``` ```
......
examples/001_example/001_chronjob.py
+ 1
− 1
View file @ ddc68751
...@@ -22,7 +22,7 @@ from tree_maker import NodeJob ...@@ -22,7 +22,7 @@ from tree_maker import NodeJob
try: try:
root=tree_maker.tree_from_yaml( root=tree_maker.tree_from_yaml(
f'/home/jovyan/local_host_home/CERNBox/2021/tree_maker/examples/001_example/study_000/tree.yaml') f'/home/jovyan/local_host_home/CERNBox/2021/tree_maker/examples/001_example/study_000/tree.yaml')
except exception as e: except Exception as e:
print(e) print(e)
print('Probably you forgot to edit the address of you yaml file...') print('Probably you forgot to edit the address of you yaml file...')
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment