From 7feb7edb4a8179877a6b98fb48799996f2e1a857 Mon Sep 17 00:00:00 2001
From: Emmanuel Ormancey <emmanuel.ormancey@cern.ch>
Date: Mon, 20 Dec 2021 11:40:01 +0100
Subject: [PATCH] [#5] Expand egroups migration script to cover egroup system
 change ourselves

---
 Python/egroup-to-channel.Python/README.md     |  29 ++++-
 .../api_library/__init__.py                   |   0
 .../api_library/channel.py                    |  77 ++++++++++++
 .../{ => api_library}/egroup_from_ldap.py     |   2 +-
 .../api_library/egroups_api.py                |  85 ++++++++++++++
 .../{ => api_library}/get_api_token.py        |   0
 .../api_library/grappa_api.py                 |  37 ++++++
 .../api_library/grappa_get_api_token.py       |  64 ++++++++++
 .../egroup_to_channel.py                      | 111 +++++-------------
 9 files changed, 319 insertions(+), 86 deletions(-)
 create mode 100644 Python/egroup-to-channel.Python/api_library/__init__.py
 create mode 100644 Python/egroup-to-channel.Python/api_library/channel.py
 rename Python/egroup-to-channel.Python/{ => api_library}/egroup_from_ldap.py (91%)
 create mode 100644 Python/egroup-to-channel.Python/api_library/egroups_api.py
 rename Python/egroup-to-channel.Python/{ => api_library}/get_api_token.py (100%)
 create mode 100644 Python/egroup-to-channel.Python/api_library/grappa_api.py
 create mode 100644 Python/egroup-to-channel.Python/api_library/grappa_get_api_token.py

diff --git a/Python/egroup-to-channel.Python/README.md b/Python/egroup-to-channel.Python/README.md
index 52f3503..ba9585f 100644
--- a/Python/egroup-to-channel.Python/README.md
+++ b/Python/egroup-to-channel.Python/README.md
@@ -2,6 +2,19 @@
 
 Tool to migrate egroups to Notification service Channels.
 
+## Prerequisites for LXPLUS usage
+Install a local auth-get-sso-cookie compatible with python3
+```
+git clone https://gitlab.cern.ch/authzsvc/tools/auth-get-sso-cookie.git
+cd auth-get-sso-cookie
+python3 setup.py install --user
+```
+
+Install a local suds module
+```
+pip3 install --user suds-py3
+```
+
 ## Create a Channel for the specified Egroup
 - With the corresponding grappa group as channel member
 - With email posting permissions for egroup
@@ -15,8 +28,22 @@ Samples:
 - Migrate one egroup retrieving information from xldap: ```./egroup_to_channel.py -g it-dep-cda-wf -ldap```
 - Migrate one egroup specifying information: ```./egroup_to_channel.py -g it-dep-cda-wf -a it-dep-cda-wf-admins -o awagner -d "All members of IT-CDA-WF"```
 
+## TEMPORARY IMPLEMENTATION
+Command line: ```--removeSync```
+
+Script will for now complete migration by:
+- Disable sync between Grappa group and Egroup
+- Remove all EGroup members
+- Add channel email to the EGroup members
+
+As a result: all mails sent to the egroup will then be relayued to the channel which will handled delivery to the Grappa group members.
+
+**Note: the egroup will be empty, so authorizations based on it will not work anymore.**
+
+## OR Disable Egroup posting to Exchange
+**Currently only running manually by Mail team, only for groups with no recursive membership**
+The optimum migration should then be complete by:
 
-## Disable Egroup posting to Exchange 
 Run Mail team (sympa) egroup migration script to:
 - Stop sending mails to all egroup members via Exchange
 - Relay the egroup mails to the Notification Channel email address
\ No newline at end of file
diff --git a/Python/egroup-to-channel.Python/api_library/__init__.py b/Python/egroup-to-channel.Python/api_library/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/Python/egroup-to-channel.Python/api_library/channel.py b/Python/egroup-to-channel.Python/api_library/channel.py
new file mode 100644
index 0000000..763fdf2
--- /dev/null
+++ b/Python/egroup-to-channel.Python/api_library/channel.py
@@ -0,0 +1,77 @@
+import requests
+import re, sys
+from api_library.get_api_token import get_api_token
+
+BACKEND_URL='https://api-notifications-dev.app.cern.ch'
+#BACKEND_URL='https://localhost:8080'
+ACCESS_TOKEN=get_api_token()
+HEADER={'Authorization': 'Bearer ' + ACCESS_TOKEN}
+VERIFY=False # Verify SSL certificate for requests
+
+# Create new Channel
+def create_channel(egroup, admingroup, description):
+    print('Creating Channel')
+    data = {'channel': {
+        'name': egroup,
+        'slug': re.sub('[^0-9a-z-_]', '-', egroup.lower()),
+        'description': description + ' - Migrated from Egroups',
+        'adminGroup': { 'groupIdentifier': admingroup },
+        'visibility': 'RESTRICTED',
+        'submissionByForm': [ 'ADMINISTRATORS' ],
+        'submissionByEmail': [ 'EGROUP' ],
+        'incomingEgroup': egroup + '@cern.ch',
+    }}
+    #print(data)
+    r = requests.post(BACKEND_URL + '/channels/', json=data, headers=HEADER, verify=VERIFY)
+    if r.status_code != requests.codes.ok:
+        print('error creating channel', r.json())
+        sys.exit(2)
+    new_channel = r.json()
+    #print(new_channel)
+
+    return new_channel['id'], new_channel['slug']
+
+# Add egroup as Channel Member
+def add_egroup_to_channel(channel_id, egroup):
+    print('Adding group to Channel members', egroup)
+    data = { 'group': { 'groupIdentifier': egroup } }
+    r = requests.put(BACKEND_URL + '/channels/' + channel_id + '/groups', json=data, headers=HEADER, verify=VERIFY)
+    if r.status_code != requests.codes.ok:
+        print('error updating channel', r.json())
+        sys.exit(2)
+    updated_channel = r.json()
+
+    return updated_channel['id']
+
+# Remove ME from Members
+def remove_me_from_channel(channel_id):
+    print('Removing ME from Channel members')
+    r = requests.get(BACKEND_URL + '/usersettings', headers=HEADER, verify=VERIFY)
+    if r.status_code != requests.codes.ok:
+        print('error removing ME from channel', r.json())
+        sys.exit(2)
+    me = r.json()
+    if not me['userId']:
+        print('error retrieving ME', me)
+        sys.exit(2)
+
+    data = { 'userId': me['userId'] }
+    r = requests.delete(BACKEND_URL + '/channels/' + channel_id + '/members', json=data, headers=HEADER, verify=VERIFY)
+    if r.status_code != requests.codes.ok:
+        print('error removing ME from channel members', r.json())
+        sys.exit(2)
+    updated_channel = r.json()
+
+    return updated_channel['id']
+
+# Change Channel owner
+def set_channel_owner(channel_id, username):
+    print('Setting Channel owner to', username)
+    data = { 'username': username }
+    r = requests.put(BACKEND_URL + '/channels/' + channel_id + '/owner', json=data, headers=HEADER, verify=VERIFY)
+    if r.status_code != requests.codes.ok:
+        print('error setting channel owner', r.json())
+        sys.exit(2)
+    updated_channel = r.json()
+
+    return updated_channel['id']
diff --git a/Python/egroup-to-channel.Python/egroup_from_ldap.py b/Python/egroup-to-channel.Python/api_library/egroup_from_ldap.py
similarity index 91%
rename from Python/egroup-to-channel.Python/egroup_from_ldap.py
rename to Python/egroup-to-channel.Python/api_library/egroup_from_ldap.py
index 2a279f9..33f0b95 100644
--- a/Python/egroup-to-channel.Python/egroup_from_ldap.py
+++ b/Python/egroup-to-channel.Python/api_library/egroup_from_ldap.py
@@ -20,7 +20,7 @@ def egroup_from_ldap(egroup):
         return {
             'name': str(results[0]['cn'][0], 'utf-8'),
             'description': str(results[0]['description'][0], 'utf-8'),
-            'adminGroup': str(results[0]['extensionAttribute6'][0], 'utf-8'),
+            'adminGroup': str(results[0]['extensionAttribute6'][0], 'utf-8') if results[0].get('extensionAttribute6') else None,
             'owner': str(results[0]['managedBy'][0], 'utf-8').replace(',OU=Users,OU=Organic Units,DC=cern,DC=ch', '').replace('CN=', ''),
         }
 
diff --git a/Python/egroup-to-channel.Python/api_library/egroups_api.py b/Python/egroup-to-channel.Python/api_library/egroups_api.py
new file mode 100644
index 0000000..5a47e52
--- /dev/null
+++ b/Python/egroup-to-channel.Python/api_library/egroups_api.py
@@ -0,0 +1,85 @@
+#
+# script to create a new eGroup from a template using python's "suds" package
+# (https://fedorahosted.org/suds/) I've been using 0.4.1(beta) successfully.
+# The package needs to be installed and available in your PYTHONPATH or you need
+# to set the path explicitly (see below)
+#
+# Author: Andreas Pfeiffer (andreas.pfeiffer@cern.ch) May 11, 2014
+#
+# latest update: Aug 27, 2014 : handle case where template group does not have
+#                any members yet (code kindly provided by Joel.Closier@cern.ch)
+#                works also with Python 2.7
+#
+import os, sys
+#import logging
+#logging.basicConfig(level=logging.INFO, filename='./suds.log')
+#logging.getLogger('suds.client').setLevel(logging.DEBUG)
+#logging.getLogger('suds.transport').setLevel(logging.DEBUG)
+# for (much) more debugging uncomment the next two logging lines
+# ---------------------------------------------------------------------
+# CAREFUL: enabling these lines will show your connection details
+#          in clear text on the screen (including your password) !!!!
+# ---------------------------------------------------------------------
+# logging.getLogger('suds.wsdl').setLevel(logging.DEBUG)
+# logging.getLogger('suds.xsd.schema').setLevel(logging.DEBUG)
+# add the path where "suds" is installed (~/python/ in my case)
+# if you have it installed in your system (and available in your PYTHONPATH,
+# you can comment the next two lines:
+sudsPath = os.path.join(os.environ['HOME'],'python')
+sys.path.append(sudsPath)
+from suds.client import Client #  pip3 install suds-py3
+from suds.transport.http import HttpAuthenticated
+from suds import WebFault
+if sys.version_info < (3, 0):
+    import urllib2
+else:
+    import urllib.request as urllib2
+from getpass import getpass
+
+# helper functions
+def checkOK( replyIn ):
+    reply = str( replyIn )
+    if "ErrorType" in reply : return False
+     
+    return True
+    
+def findGroup(groupname = None):
+    print("Enter credentials for egroups API access")
+    login = input("Login: ") 
+    pwd = getpass('Password: ')
+
+    url = 'https://foundservices.cern.ch/ws/egroups/v1/EgroupsWebService/EgroupsWebService.wsdl'
+    client = Client(url, username=login, password=pwd)
+    group = client.service.FindEgroupByName(groupname).result
+    # for more debugging:
+    # print "client.service.FindEgroupByName returned:"
+    # print groupTmpl
+    # print "="*80
+    return client, group
+    
+# add a member to an eGroup
+def addMemberEmail(eGroupName, eMailAddress):
+    client, group = findGroup(eGroupName)
+    print("Adding member ", eMailAddress, 'to', eGroupName)
+
+    members = []
+    member = client.factory.create('ns0:MemberType')
+    member.Email = eMailAddress
+    member.Type = 'External'
+    members.append(member)
+
+    overWriteMembers = True    # or True, if you want to reset the list
+    ret = client.service.AddEgroupMembers( eGroupName, overWriteMembers, members )
+    if not checkOK( ret ):
+        print("ERROR could not add user", eMailAddress, 'to group', eGroupName)
+        print("      reason given by server:", ret)
+
+
+# if __name__ == "__main__":
+#    # for simplicity, just take the first argument as name of the new group:
+#    newGroupName = sys.argv[1].lower()
+#    # create the new group based on the template (see above)
+#    #newGroup( newGroupName )
+    
+#    # now we have the group, so we can add a new member.  
+#    addMemberEmail( newGroupName, 'notifications+another-test+NORMAL@dovecotmta.cern.ch' )
\ No newline at end of file
diff --git a/Python/egroup-to-channel.Python/get_api_token.py b/Python/egroup-to-channel.Python/api_library/get_api_token.py
similarity index 100%
rename from Python/egroup-to-channel.Python/get_api_token.py
rename to Python/egroup-to-channel.Python/api_library/get_api_token.py
diff --git a/Python/egroup-to-channel.Python/api_library/grappa_api.py b/Python/egroup-to-channel.Python/api_library/grappa_api.py
new file mode 100644
index 0000000..e8b3fdb
--- /dev/null
+++ b/Python/egroup-to-channel.Python/api_library/grappa_api.py
@@ -0,0 +1,37 @@
+from api_library.grappa_get_api_token import grappa_get_api_token
+import requests
+import sys
+
+def update_sync_type(group_id, sync_type="NoSync"):
+    """
+    Call the AuthZSvc API to update the sync type
+    """
+    GRAPPA_API_URL = "https://authorization-service-api.web.cern.ch/api/v1.0"
+    SYNC_TYPE_KEY = "syncType"
+    #sync_types = ["Slave", "SlaveWithPlaceholders", "Master", "NoSync"]
+    ACCESS_TOKEN=grappa_get_api_token()
+    HEADER={'Authorization': 'Bearer ' + ACCESS_TOKEN}
+    VERIFY=False # Verify SSL certificate for requests
+
+    r = requests.get(GRAPPA_API_URL + '/Group/' + group_id, headers=HEADER, verify=VERIFY)
+    if r.status_code != requests.codes.ok:
+        print('error retrieving group', r.json())
+        sys.exit(2)
+    group = r.json()
+    #print('retrieved group', group)
+    if not group['data']:
+        print('error retrieving group', group)
+        sys.exit(2)
+
+    group = group['data']
+    group[SYNC_TYPE_KEY] = sync_type
+
+    r = requests.put(GRAPPA_API_URL + '/Group/' + group_id, json=group, headers=HEADER, verify=VERIFY)
+    if r.status_code != requests.codes.ok:
+        print('error updating group', r.json())
+        sys.exit(2)
+    updated_group = r.json()
+    #print('updated_group', updated_group)
+
+    return updated_group['data'][SYNC_TYPE_KEY] 
+
diff --git a/Python/egroup-to-channel.Python/api_library/grappa_get_api_token.py b/Python/egroup-to-channel.Python/api_library/grappa_get_api_token.py
new file mode 100644
index 0000000..a05da83
--- /dev/null
+++ b/Python/egroup-to-channel.Python/api_library/grappa_get_api_token.py
@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+from auth_get_sso_cookie import cern_sso
+import subprocess
+import requests
+
+AUTH_HOSTNAME = "auth.cern.ch"
+AUTH_REALM = "cern"
+
+################# CONFIGURATION ##################
+
+# The Client application (application portal, option my app cannot keep a secret)
+clientapp_name = "python-user-scripts"
+
+# Standard localhost uri for this virtual app
+clientapp_uri = "http://localhost"
+
+# The target application (the backend API), with granted permissions to client application for token exchange
+audience = "authorization-service-api"
+
+##################################################
+
+#if __name__ == "__main__":
+def grappa_get_api_token():
+    # Get Token for the clientscript application
+    # Using https://gitlab.cern.ch/authzsvc/tools/auth-get-sso-cookie/
+    # Run with parameters for the clientscript application 
+    # clientapi.py -u https://localhost -c tmp-push-notifications-clientscript
+    #token = command_line_tools.auth_get_sso_token()
+    # proc = subprocess.Popen(
+    #     ["auth-get-sso-token", "-u", clientapp_uri, "-c", clientapp_name], 
+    #     stdout=subprocess.PIPE, 
+    #     stderr=subprocess.STDOUT)
+    # token = proc.communicate()[0].rstrip()
+    token = cern_sso.get_sso_token(clientapp_uri, clientapp_name, True, AUTH_HOSTNAME, AUTH_REALM)
+    #print("TOKEN to exchange retrieved")
+    #print(token)
+
+    # Do Token Exchange for the Backend API application
+    # https://auth.docs.cern.ch/user-documentation/oidc/exchange-for-api/
+    r = requests.post(
+            "https://auth.cern.ch/auth/realms/cern/protocol/openid-connect/token",
+            data={
+                "client_id": clientapp_name,
+                "grant_type": "urn:ietf:params:oauth:grant-type:token-exchange",
+                "subject_token": token,
+                "requested_token_type": "urn:ietf:params:oauth:token-type:refresh_token",
+                "audience": audience,
+            },
+        )
+    if not r.ok:
+        print(
+            "The token response was not successful: {}".format(r.json()))
+        r.raise_for_status()
+
+    token_response = r.json()
+    access_token = token_response["access_token"]
+    #print("access_token retrieved")
+    #print(access_token)
+    return access_token
+
+    # Then calls to the backend can be performed with this access token
+    # ACCESS_TOKEN=$(python get-api-token.py)
+    # curl -X GET "https://api-notifications-dev.app.cern.ch/channels/" -H  "authorization: Bearer $ACCESS_TOKEN"
+
diff --git a/Python/egroup-to-channel.Python/egroup_to_channel.py b/Python/egroup-to-channel.Python/egroup_to_channel.py
index 6e1ba90..1f0a769 100644
--- a/Python/egroup-to-channel.Python/egroup_to_channel.py
+++ b/Python/egroup-to-channel.Python/egroup_to_channel.py
@@ -1,90 +1,16 @@
 #!/usr/bin/python
 
-import requests
-import json, os, re
 import sys, getopt
-from get_api_token import get_api_token
-from egroup_from_ldap import egroup_from_ldap
-
-
-BACKEND_URL='https://api-notifications-dev.app.cern.ch'
-#BACKEND_URL='https://localhost:8080'
-ACCESS_TOKEN=get_api_token()
-HEADER={'Authorization': 'Bearer ' + ACCESS_TOKEN}
-VERIFY=False # Verify SSL certificate for requests
-
-# Create new Channel
-def create_channel(egroup, admingroup, description):
-    print('Creating Channel')
-    data = {'channel': {
-        'name': egroup,
-        'slug': re.sub('[^0-9a-z-_]', '-', egroup.lower()),
-        'description': description + ' - Migrated from Egroups',
-        'adminGroup': { 'groupIdentifier': admingroup },
-        'visibility': 'RESTRICTED',
-        'submissionByForm': [ 'ADMINISTRATORS' ],
-        'submissionByEmail': [ 'EGROUP' ],
-        'incomingEgroup': egroup + '@cern.ch',
-    }}
-    #print(data)
-    r = requests.post(BACKEND_URL + '/channels/', json=data, headers=HEADER, verify=VERIFY)
-    if r.status_code != requests.codes.ok:
-        print('error creating channel', r.json())
-        sys.exit(2)
-    new_channel = r.json()
-    #print(new_channel)
-
-    return new_channel['id']
-
-# Add egroup as Channel Member
-def add_egroup_to_channel(channel_id, egroup):
-    print('Adding group to Channel members', egroup)
-    data = { 'group': { 'groupIdentifier': egroup } }
-    r = requests.put(BACKEND_URL + '/channels/' + channel_id + '/groups', json=data, headers=HEADER, verify=VERIFY)
-    if r.status_code != requests.codes.ok:
-        print('error updating channel', r.json())
-        sys.exit(2)
-    updated_channel = r.json()
-
-    return updated_channel['id']
-
-# Remove ME from Members
-def remove_me_from_channel(channel_id):
-    print('Removing ME from Channel members')
-    r = requests.get(BACKEND_URL + '/me', headers=HEADER, verify=VERIFY)
-    if r.status_code != requests.codes.ok:
-        print('error removing ME from channel', r.json())
-        sys.exit(2)
-    me = r.json()
-    if not me['userId']:
-        print('error retrieving ME', me)
-        sys.exit(2)
-
-    data = { 'userId': me['userId'] }
-    r = requests.delete(BACKEND_URL + '/channels/' + channel_id + '/members', json=data, headers=HEADER, verify=VERIFY)
-    if r.status_code != requests.codes.ok:
-        print('error removing ME from channel members', r.json())
-        sys.exit(2)
-    updated_channel = r.json()
-
-    return updated_channel['id']
-
-# Change Channel owner
-def set_channel_owner(channel_id, username):
-    print('Setting Channel owner to', username)
-    data = { 'username': username }
-    r = requests.put(BACKEND_URL + '/channels/' + channel_id + '/owner', json=data, headers=HEADER, verify=VERIFY)
-    if r.status_code != requests.codes.ok:
-        print('error setting channel owner', r.json())
-        sys.exit(2)
-    updated_channel = r.json()
-
-    return updated_channel['id']
+from api_library.egroup_from_ldap import egroup_from_ldap
+from api_library.egroups_api import addMemberEmail
+from api_library.grappa_api import update_sync_type
+from api_library.channel import create_channel, remove_me_from_channel, set_channel_owner, add_egroup_to_channel
 
 def usage():
-    print('egroup-to-channel.py -g <egroupname> [-a <admingroup> -o <owner> -d <description> -ldap]')
+    print('egroup-to-channel.py -g <egroupname> [-a <admingroup> -o <owner> -d <description> --ldap --removeSync]')
     print('\t-g <egroupname> is required')
-    print('\t-ldap will search in ldap for admingroup, owner and description information')
+    print('\t--ldap will search in ldap for admingroup, owner and description information')
+    print('\t--removeSync: stop sync between EGroup and Grappa, clear EGroup members. Either this or ask Mail Team to manually change the EGroup mail target.')
 
 # Main
 def main(argv):
@@ -93,8 +19,9 @@ def main(argv):
     owner = ''
     description = ''
     enableldap = False
+    removeSync = False
     try:
-        opts, args = getopt.getopt(argv, "lhg:a:d:o:", ["group=", "admingroup=", "description=", "owner=", "ldap"])
+        opts, args = getopt.getopt(argv, "lhg:a:d:o:", ["group=", "admingroup=", "description=", "owner=", "ldap", "removeSync"])
     except getopt.GetoptError:
         usage()
         sys.exit(2)
@@ -112,6 +39,8 @@ def main(argv):
             description = arg
         elif opt in ("-l", "--ldap"):
             enableldap = True
+        elif opt in ("--removeSync"):
+            removeSync = True
     if not egroup:
         usage()
         sys.exit(2)
@@ -124,7 +53,8 @@ def main(argv):
         ret = egroup_from_ldap(egroup)
         if ret:
             owner = ret['owner']
-            adminGroup = ret['adminGroup']
+            if ret['adminGroup']:
+                adminGroup = ret['adminGroup']
             description = ret['description']
 
     print('Creating Channel from Egroup')
@@ -143,7 +73,7 @@ def main(argv):
     # HEADER={'Authorization': 'Bearer ' + ACCESS_TOKEN}
 
     # Create Channel
-    channel_id = create_channel(egroup, adminGroup, description)
+    channel_id, channel_slug = create_channel(egroup, adminGroup, description)
     if not channel_id:
         print('Error creating channel ', egroup)
         sys.exit()
@@ -154,5 +84,18 @@ def main(argv):
     # Change owner to egroup owner
     set_channel_owner(channel_id, owner)
 
+    channel_email = "notifications+" + channel_slug + "+NORMAL@dovecotmta.cern.ch"
+
+    if removeSync:
+        print("Migrating with removeSync option: egroups will be cleaned and grappa sync disabled.")
+        # Remove Egroup <-> Grappa sync for this Grappa group
+        update_sync_type(egroup)
+        # Clear Egroup members and set only the channel email as member
+        print("Channel email:", channel_email)
+        addMemberEmail(egroup, channel_email)
+        print("Egroup was cleaned, kept only 1 member:", channel_email)
+    else:
+        print("Egroup mailing needs to be disabled by Mail Team, and forwarded to:", channel_email)
+
 if __name__ == "__main__":
    main(sys.argv[1:])
-- 
GitLab