etf_plugin_cms.py 6.97 KB
Newer Older
Andrea Sciaba's avatar
Andrea Sciaba committed
1
import logging
2
import json
Andrea Sciaba's avatar
Andrea Sciaba committed
3

Andrea Sciaba's avatar
Andrea Sciaba committed
4
from ncgx.inventory import Hosts, Checks, Groups
Andrea Sciaba's avatar
Andrea Sciaba committed
5
6
7
8
from vofeed.api import VOFeed

log = logging.getLogger('ncgx')

9
10
11
12
13
14
FLAVOR_MAP = {'CREAM-CE': 'cream',
              'ARC-CE': 'arc',
              'HTCONDOR-CE': 'condor',
              'GLOBUS': 'gt',
              'OSG-CE': 'gt'}

Andrea Sciaba's avatar
Andrea Sciaba committed
15
CE_STATE_METRICS = [
16
    'org.sam.CONDOR-JobState-/cms/Role=lcgadmin']
Andrea Sciaba's avatar
Andrea Sciaba committed
17
18

CE_METRICS = (
19
    'org.sam.CONDOR-JobSubmit-/cms/Role=lcgadmin',)
Andrea Sciaba's avatar
Andrea Sciaba committed
20
21

SE_METRICS = (
22
23
24
25
26
27
28
29
    'org.cms.SRM-AllCMS-/cms/Role=production',
    'org.cms.SRM-GetPFNFromTFC-/cms/Role=production',
    'org.cms.SRM-VODel-/cms/Role=production',
    'org.cms.SRM-VOGet-/cms/Role=production',
    'org.cms.SRM-VOGetTURLs-/cms/Role=production',
    'org.cms.SRM-VOLs-/cms/Role=production',
    'org.cms.SRM-VOLsDir-/cms/Role=production',
    'org.cms.SRM-VOPut-/cms/Role=production')
Andrea Sciaba's avatar
Andrea Sciaba committed
30

31
XROOT_METRICS = (
32
33
34
    'org.cms.SE-xrootd-contain',
    'org.cms.SE-xrootd-connection',
    'org.cms.SE-xrootd-version',
35
)
Andrea Sciaba's avatar
Andrea Sciaba committed
36

37
WN_METRICS = {
Marian Babik's avatar
Marian Babik committed
38
    'WN-cvmfs': 'org.cms.WN-cvmfs-/cms/Role=lcgadmin',
39
40
41
42
43
44
45
46
47
48
49
    'CE-cms-analysis.sing': 'org.cms.WN-analysis-/cms/Role=lcgadmin',
    'CE-cms-singularity': 'org.cms.WN-isolation-/cms/Role=lcgadmin',
    'CE-cms-basic': 'org.cms.WN-basic-/cms/Role=lcgadmin',
    'CE-cms-env': 'org.cms.WN-env-/cms/Role=lcgadmin',
    'CE-cms-frontier.sing': 'org.cms.WN-frontier-/cms/Role=lcgadmin',
    'CE-cms-mc.sing': 'org.cms.WN-mc-/cms/Role=lcgadmin',
    'CE-cms-squid.sing': 'org.cms.WN-squid-/cms/Role=lcgadmin',
    'CE-cms-xrootd-access.sing': 'org.cms.WN-xrootd-access-/cms/Role=lcgadmin',
    'CE-cms-xrootd-fallback.sing': 'org.cms.WN-xrootd-fallback-/cms/Role=lcgadmin'
}

Marian Babik's avatar
Marian Babik committed
50
51
# legacy SFT/SAME metrics which are not nagios compliant
WN_METRICS_LEGACY = [x for x in WN_METRICS.keys() if 'CE' in x]
Marian Babik's avatar
Marian Babik committed
52

53
54

def run(url, ipv6=False):
Andrea Sciaba's avatar
Andrea Sciaba committed
55
56
57
58
    log.info("Processing vo feed: %s" % url)

    # Get services from the VO feed, i.e 
    # list of tuples (hostname, flavor, endpoint)
Andrea Sciaba's avatar
Andrea Sciaba committed
59
60
    feed = VOFeed(url)
    services = feed.get_services()
Andrea Sciaba's avatar
Andrea Sciaba committed
61
62
63
64
65
66
67
68
    
    # Add hosts, each tagged with corresponding flavors
    # creates /etc/ncgx/conf.d/generated_hosts.cfg
    h = Hosts()
    for service in services:
        h.add(service[0], tags=[service[1]])
    h.serialize()

69
70
71
    # Add host groups
    sites = feed.get_groups("CMS_Site")
    hg = Groups("host_groups")
Marian Babik's avatar
Marian Babik committed
72
    for site, hosts in sites.items():
73
74
75
76
        for host in hosts:
            hg.add(site, host)
    hg.serialize()

Andrea Sciaba's avatar
Andrea Sciaba committed
77
78
79
    # Add corresponding metrics to tags
    # creates /etc/ncgx/conf.d/generated_checks.cfg
    c = Checks()
80
81
    c.add_all(CE_METRICS, tags=["CREAM-CE", "ARC-CE", "HTCONDOR-CE"])
    c.add_all(WN_METRICS.values(), tags=["CREAM-CE", "ARC-CE", "HTCONDOR-CE"])
82
    c.add_all(SE_METRICS, tags=["SRM"])
83
    c.add_all(XROOT_METRICS, tags=["XROOTD"])
84
    # IPv6
85
86
    c.add("org.cms.DNS-IPv6", tags=["SRM"], params={'extends': 'check_dig'})
    # Special proxy
87
88
    c.add("org.globus.GridProxy-Get-/cms/Role=production", hosts=("localhost",))
    c.add("org.globus.GridProxy-Valid-/cms/Role=production", hosts=("localhost",))
Andrea Sciaba's avatar
Andrea Sciaba committed
89

90
91
92
93
    # XRoot
    for service in services:
        flavor = service[1]
        if flavor not in ["XROOTD"]:
Marian Babik's avatar
Marian Babik committed
94
            continue
95
96
97
98
99
100
101
102
103
        host = service[0]
        endpoint = service[2]
        site = hg.exact_match(host)
        if not site:
            log.warning("Unable to find site for host %s, skipping" % host)
            continue
        if len(site) > 1:
            log.warning("Host assigned to multiple sites: %s, %s picking first one" % (host, site))
        if ipv6:
Marian Babik's avatar
Marian Babik committed
104
105
106
            c.add("org.cms.SE-xrootd-read", hosts=(host,), params={'args': {'--site': site.pop(),
                                                                            '--endpoint': endpoint, '-6': ''},
                                                                   '_tags': 'XROOTD'})
107
        else:
Marian Babik's avatar
Marian Babik committed
108
109
110
            c.add("org.cms.SE-xrootd-read", hosts=(host,), params={'args': {'--site': site.pop(),
                                                                            '--endpoint': endpoint, '-4': ''},
                                                                   '_tags': 'XROOTD'})
111

112
113
114
    # ETF env - environment variables to export on the worker node (global for all sites), such as:
    # ETF_TESTS - points to a list of WN tests to execute (stored in WN_METRICS)
    # ETF_LEGACY should be a subset of ETF_TESTS that identifies SFT tests (those that are not nagios compliant)
Marian Babik's avatar
Marian Babik committed
115
    # SAME* environment needed by the legacy/SFT tests
116
    with open('/tmp/etf-env.sh', 'w') as etf_env:
Marian Babik's avatar
Marian Babik committed
117
118
        etf_env.write('ETF_TESTS={}\n'.format(
                            ','.join(['etf/probes/org.cms/testjob/tests/'+m for m in WN_METRICS.keys()])))
Marian Babik's avatar
Marian Babik committed
119
        etf_env.write('ETF_LEGACY={}\n'.format(
Marian Babik's avatar
Marian Babik committed
120
121
122
                            ','.join(['etf/probes/org.cms/testjob/tests/' + m for m in WN_METRICS_LEGACY])))
        # for code, value in SAME_CODES.items():
        #    etf_env.write('SAME_{}={}\n'.format(code, value))
123
        etf_env.write('SAME_VO=cms\n')
Marian Babik's avatar
Marian Babik committed
124
125
        etf_env.write('SAME_TEST_DIRNAME=$ETFROOT/probes/org.cms/testjob/tests\n')
        etf_env.write('SAME_SENSOR_HOME=$ETFROOT/probes/org.cms/testjob\n')
126

Marian Babik's avatar
Marian Babik committed
127
    # ETF WN-qFM config - maps WN scripts back to metrics (WN-cvmfs -> org.lhcb.WN-cvmfs-/lhcb/Role=production)
128
129
130
    with open('/tmp/etf_wnfm.json', 'w') as etf_wnfm:
        json.dump({'wn_metric_map': WN_METRICS, 'counter_enabled': True}, etf_wnfm)

131
    # Queues
132
133
134
135
    for service in services:
        host = service[0]
        flavor = service[1]
        if flavor not in ["CREAM-CE", "ARC-CE", "GLOBUS", "HTCONDOR-CE"]:
Andrea Sciaba's avatar
Andrea Sciaba committed
136
            continue
137
138
        if flavor == 'HTCONDOR-CE':
            # special handling for HTCONDOR-CE, no queues
139
140
141
142
            for m in CE_STATE_METRICS:
                c.add(m, hosts=(service[0],), params={'args': {'--resource': 'htcondor://%s' % service[0],
                                                               '--jdl-ads': '\'+maxMemory=2000\''}})
            continue
143
144
145
146
147
148
149
150
151
152
153
        ce_resources = feed.get_ce_resources(host, flavor)
        if ce_resources:
            batch = ce_resources[0][0]
            queue = ce_resources[0][1]
            if not batch:
                batch = "nopbs"
            if flavor not in FLAVOR_MAP.keys():
                log.warning("Unable to determine type for flavour %s" % flavor)
                continue
            res = "%s://%s/%s/%s/%s" % (FLAVOR_MAP[flavor], host, 'nosched', batch, queue)
            for m in CE_STATE_METRICS:
154
155
156
157
158
159
                if flavor == 'CREAM-CE':
                    c.add(m, hosts=(service[0],),
                          params={'args': {'--resource': '%s' % res,
                                  '--jdl-ads': '\'cream_attributes=CERequirements=\"other.GlueHostMainMemoryRAMSize==2000\"\''}})
                else:
                    c.add(m, hosts=(service[0],), params={'args': {'--resource': '%s' % res}})
160
        else:
161
162
            res = "%s://%s/%s/%s/%s" % (FLAVOR_MAP[flavor], host, 'nosched', 'nobatch', 'noqueue')
            for m in CE_STATE_METRICS:
Marian Babik's avatar
Marian Babik committed
163
                c.add(m, hosts=(service[0],), params={'args': {'--resource': '%s' % res}})
164
            
Andrea Sciaba's avatar
Andrea Sciaba committed
165
    c.serialize()