etf_plugin_cms.py 3.56 KB
Newer Older
Andrea Sciaba's avatar
Andrea Sciaba committed
1
2
3
4
5
6
import logging
import itertools
import requests
import urlparse
import xml.etree.ElementTree as ET

Andrea Sciaba's avatar
Andrea Sciaba committed
7
from ncgx.inventory import Hosts, Checks, Groups
Andrea Sciaba's avatar
Andrea Sciaba committed
8
9
10
11
from vofeed.api import VOFeed

log = logging.getLogger('ncgx')

12
13
14
15
16
17
FLAVOR_MAP = {'CREAM-CE': 'cream',
              'ARC-CE': 'arc',
              'HTCONDOR-CE': 'condor',
              'GLOBUS': 'gt',
              'OSG-CE': 'gt'}

Andrea Sciaba's avatar
Andrea Sciaba committed
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
CE_STATE_METRICS = (
'org.sam.CONDOR-JobState-/cms/Role=lcgadmin',
'org.sam.CONDOR-JobState-/cms/Role=pilot',
'org.sam.CONDOR-JobState-/cms/Role=production',)

CE_METRICS = (
'org.sam.CONDOR-JobSubmit-/cms/Role=lcgadmin',
'org.sam.CONDOR-JobSubmit-/cms/Role=pilot',
'org.sam.CONDOR-JobSubmit-/cms/Role=production',
'org.cms.glexec.WN-gLExec-/cms/Role=pilot',
'org.cms.WN-analysis-/cms/Role=lcgadmin',
'org.cms.WN-basic-/cms/Role=lcgadmin',
'org.cms.WN-cvmfs-/cms/Role=lcgadmin',
'org.cms.WN-env-/cms/Role=lcgadmin',
'org.cms.WN-frontier-/cms/Role=lcgadmin',
'org.cms.WN-mc-/cms/Role=production',
'org.cms.WN-remotestageout-/cms/Role=lcgadmin',
'org.cms.WN-squid-/cms/Role=lcgadmin',
'org.cms.WN-swinst-/cms/Role=lcgadmin',
Andrea Sciaba's avatar
Andrea Sciaba committed
37
'org.cms.WN-singularity-/cms/Role=lcgadmin',
Andrea Sciaba's avatar
Andrea Sciaba committed
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
'org.cms.WN-xrootd-access-/cms/Role=lcgadmin',
'org.cms.WN-xrootd-fallback-/cms/Role=lcgadmin') 

SE_METRICS = (
'org.cms.SRM-AllCMS-/cms/Role=production',
'org.cms.SRM-GetPFNFromTFC-/cms/Role=production',
'org.cms.SRM-VODel-/cms/Role=production',
'org.cms.SRM-VOGet-/cms/Role=production',
'org.cms.SRM-VOGetTURLs-/cms/Role=production',
'org.cms.SRM-VOLs-/cms/Role=production',
'org.cms.SRM-VOLsDir-/cms/Role=production',
'org.cms.SRM-VOPut-/cms/Role=production')


def run(url):
    log.info("Processing vo feed: %s" % url)

    # Get services from the VO feed, i.e 
    # list of tuples (hostname, flavor, endpoint)
Andrea Sciaba's avatar
Andrea Sciaba committed
57
58
    feed = VOFeed(url)
    services = feed.get_services()
Andrea Sciaba's avatar
Andrea Sciaba committed
59
60
61
62
63
64
65
66
67
68
69
    
    # Add hosts, each tagged with corresponding flavors
    # creates /etc/ncgx/conf.d/generated_hosts.cfg
    h = Hosts()
    for service in services:
        h.add(service[0], tags=[service[1]])
    h.serialize()

    # Add corresponding metrics to tags
    # creates /etc/ncgx/conf.d/generated_checks.cfg
    c = Checks()
70
71
    c.add_all(CE_METRICS, tags=["CREAM-CE", "ARC-CE", "GLOBUS", "HTCONDOR-CE"])
    c.add_all(SE_METRICS, tags=["SRM"])
Andrea Sciaba's avatar
Andrea Sciaba committed
72
73


74
75
76
77
    for service in services:
        host = service[0]
        flavor = service[1]
        if flavor not in ["CREAM-CE", "ARC-CE", "GLOBUS", "HTCONDOR-CE"]:
Andrea Sciaba's avatar
Andrea Sciaba committed
78
            continue
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
        if flavor == 'HTCONDOR-CE':
            # special handling for HTCONDOR-CE, no queues
           for m in CE_STATE_METRICS:
               c.add(m, hosts=(service[0],), params={ 'args': { '--resource': 'htcondor://%s' % service[0] }})
           continue
        ce_resources = feed.get_ce_resources(host, flavor)
        if ce_resources:
            batch = ce_resources[0][0]
            queue = ce_resources[0][1]
            if not batch:
                batch = "nopbs"
            if flavor not in FLAVOR_MAP.keys():
                log.warning("Unable to determine type for flavour %s" % flavor)
                continue
            res = "%s://%s/%s/%s/%s" % (FLAVOR_MAP[flavor], host, 'nosched', batch, queue)
            for m in CE_STATE_METRICS:
               c.add(m, hosts=(service[0],), params={ 'args': { '--resource': '%s' % res }})
        else:
            log.warning("No ce_resources found for host %s, BDII will used" % host)
Andrea Sciaba's avatar
Andrea Sciaba committed
98
    c.serialize()
Andrea Sciaba's avatar
Andrea Sciaba committed
99
100
101
102
103
104
105
106
107

    # Add host groups
    sites = feed.get_groups("CMS_Site") 
    hg = Groups("host_groups")
    for site, hosts in sites.iteritems():
        for host in hosts:
            hg.add(site, host)
    hg.serialize()