diff --git a/atlas/check_map_voms_roles b/atlas/check_map_voms_roles index 091539a..6b58de1 100755 --- a/atlas/check_map_voms_roles +++ b/atlas/check_map_voms_roles @@ -22,7 +22,7 @@ import requests.auth from rucio.client import Client from rucio.common.config import config_get -from rucio.common.exception import Duplicate, IdentityError +from rucio.common.exception import Duplicate, IdentityError, AccountNotFound UNKNOWN = 3 CRITICAL = 2 @@ -46,6 +46,7 @@ def get_access_token(token_url, client_id, client_secret): def get_userid2certs_mapping(scim_url, access_token): ret = {} + userid2email = {} with requests.Session() as session: session.headers = { @@ -62,11 +63,13 @@ def get_userid2certs_mapping(scim_url, access_token): d = response.json() if d['itemsPerPage'] == 0: break for user in d['Resources']: + if not user['active']: continue + userid2email[user['id']] = user['emails'][0]['value'] for certificate in user.get('urn:indigo-dc:scim:schemas:IndigoUser', {}).get('certificates', []): ret.setdefault(user['id'], []).append((certificate['subjectDn'], certificate['issuerDn'], user['emails'][0]['value'])) start += page_size - return ret + return ret, userid2email def get_group2id_mapping(scim_url, access_token): ret = {} @@ -119,10 +122,13 @@ def get_groupid_members(scim_url, groupid, access_token): def get_account_identities(account): ret = [] client = Client() - for identity in client.list_identities(account): - if identity['type'] != 'X509': - continue - ret.append(identity['identity']) + try: + for identity in client.list_identities(account): + if identity['type'] not in ['X509', 'OIDC']: + continue + ret.append(identity['identity']) + except AccountNotFound: + return None return ret @@ -142,8 +148,9 @@ if __name__ == '__main__': except Exception as error: print("Failed to get also client certificate and key from rucio.cfg") sys.exit(CRITICAL) - TOKEN_URL = 'https://atlas-auth.cern.ch/token' - SCIM_URL = 'https://atlas-auth.cern.ch/scim' + ISSUER = 'https://atlas-auth.cern.ch/' + TOKEN_URL = '{}/token'.format(ISSUER) + SCIM_URL = '{}/scim'.format(ISSUER) try: CLIENT_ID = os.environ['CLIENT_ID'] CLIENT_SECRET = os.environ['CLIENT_SECRET'] @@ -156,7 +163,7 @@ if __name__ == '__main__': CLIENT = Client() access_token = get_access_token(TOKEN_URL, CLIENT_ID, CLIENT_SECRET) - userid2certs = get_userid2certs_mapping(SCIM_URL, access_token) + userid2certs, userid2email = get_userid2certs_mapping(SCIM_URL, access_token) access_token = get_access_token(TOKEN_URL, CLIENT_ID, CLIENT_SECRET) group2id = get_group2id_mapping(SCIM_URL, access_token) @@ -169,8 +176,29 @@ if __name__ == '__main__': dns = [] groupid = group2id["atlas/{}".format(account)] account_identities = get_account_identities(ACCOUNT_MAP[account]) + if account_identities == None: + print("missing rucio account for {}".format(ACCOUNT_MAP[account])) + continue access_token = get_access_token(TOKEN_URL, CLIENT_ID, CLIENT_SECRET) for userid in get_groupid_members(SCIM_URL, groupid, access_token): + if userid not in userid2email: + print("missing or disabled account for userid {}".format(userid)) + continue + dns.append(userid) + if userid not in account_identities: + NBUSERS += 1 + try: + oidc_identity = "SUB={}, ISS={}".format(userid, ISSUER) + if not TEST: + CLIENT.add_identity(account=ACCOUNT_MAP[account], identity=oidc_identity, authtype='OIDC', email=userid2email[userid], default=True) + else: + print("CMD: rucio-admin identity add --account {0} --type OIDC --id '{1}' --email '{2}'".format(ACCOUNT_MAP[account], oidc_identity, userid2email[userid])) + print('Identity {0} added to {1}'.format(userid, ACCOUNT_MAP[account])) + except Duplicate: + pass + except Exception as error: + print('ERROR {0} not added to {1}: {2}'.format(userid, ACCOUNT_MAP[account], error)) + STATUS = WARNING if userid not in userid2certs: print("missing user details for userid {}".format(userid)) continue @@ -194,12 +222,13 @@ if __name__ == '__main__': for dn in account_identities: if dn in dns: continue + itype = 'X509' if '=' in dn else 'OIDC' NDUSERS += 1 try: if not TEST: CLIENT.del_identity(account=ACCOUNT_MAP[account], identity=dn, authtype='X509') else: - print("CMD: rucio-admin identity delete --account {0} --type X509 --id '{1}'".format(ACCOUNT_MAP[account], dn)) + print("CMD: rucio-admin identity delete --account {0} --type {1} --id '{2}'".format(ACCOUNT_MAP[account], itype, dn)) print('Identity {0} removed from {1}'.format(dn, ACCOUNT_MAP[account])) except IdentityError: pass @@ -225,8 +254,29 @@ if __name__ == '__main__': dns = [] groupid = group2id["atlas/{}".format(account)] account_identities = get_account_identities(account) + if account_identities == None: + print("missing rucio account for {}".format(account)) + continue access_token = get_access_token(TOKEN_URL, CLIENT_ID, CLIENT_SECRET) for userid in get_groupid_members(SCIM_URL, groupid, access_token): + if userid not in userid2email: + print("missing or disabled account for userid {}".format(userid)) + continue + dns.append(userid) + if userid not in account_identities: + NBUSERS += 1 + try: + oidc_identity = "SUB={}, ISS={}".format(userid, ISSUER) + if not TEST: + CLIENT.add_identity(account=account, identity=oidc_identity, authtype='OIDC', email=userid2email[userid], default=True) + else: + print("CMD: rucio-admin identity add --account {0} --type OIDC --id '{1}' --email '{2}'".format(account, oidc_identity, userid2email[userid])) + print('Identity {0} added to {1}'.format(userid, account)) + except Duplicate: + pass + except Exception as error: + print('ERROR {0} not added to {1}: {2}'.format(userid, ACCOUNT_MAP[account], error)) + STATUS = WARNING if userid not in userid2certs: print("missing user details for userid {}".format(userid)) continue @@ -250,12 +300,13 @@ if __name__ == '__main__': for dn in account_identities: if dn in dns: continue + itype = 'OIDC' if 'ISS=' in dn and 'SUB=' in dn else 'X509' NDUSERS += 1 try: if not TEST: - CLIENT.del_identity(account=account, identity=dn, authtype='X509') + CLIENT.del_identity(account=account, identity=dn, authtype=itype) else: - print("CMD: rucio-admin identity delete --account {0} --type X509 --id '{1}'".format(account, dn)) + print("CMD: rucio-admin identity delete --account {0} --type {1} --id '{2}'".format(account, itype, dn)) print('Identity {0} removed from {1}'.format(dn, account)) except IdentityError: pass diff --git a/atlas/check_voms b/atlas/check_voms index 191b7cb..82f6823 100755 --- a/atlas/check_voms +++ b/atlas/check_voms @@ -54,30 +54,36 @@ LDAP_PAGE_SIZE = 1000 def get_accounts_identities_fast(): from rucio.db.sqla.session import get_session session = get_session() - query = '''select b.identity, a.account, a.email, a.account_type from atlas_rucio.accounts a, atlas_rucio.account_map b where a.account=b.account and identity_type='X509' ''' + query = '''select b.identity, a.account, a.email, a.account_type, b.identity_type from atlas_rucio.accounts a, atlas_rucio.account_map b where a.account=b.account''' dns = {} + account2identities = {} try: result = session.execute(query) - for dn, account, email, atype in result: - dns.setdefault(atype, {})[dn] = (account, email) - return dns + for identity, account, email, atype, itype in result: + account2identities.setdefault(account, []).append((itype, identity)) + if itype not in ['X509', 'OIDC']: + continue + dns.setdefault(atype, {})[identity] = (account, email) + return dns, account2identities except Exception as error: print(error) def get_accounts_identities_slow(): dns = {} + account2identities = {} client = Client() for account in client.list_accounts(): #if account['type'] != 'USER': # continue for identity in client.list_identities(account['account']): - if identity['type'] != 'X509': + account2identities.setdefault(account['account'], []).append((identity['type'], identity['identity'])) + if identity['type'] not in ['X509', 'OIDC']: continue - if identity['identity'] in dns: - print("Duplicate identity for users {} and {}: {}".format(dns[identity['identity']][0], account['account'], identity['identity'])) + if account['type'] == 'USER' and identity['identity'] in dns.get(account['type'], {}): + print("Duplicate identity for users {} and {}: {}".format(dns[account['type']][identity['identity']][0], account['account'], identity['identity'])) dns.setdefault(account['type'], {})[identity['identity']] = (account['account'], account['email']) - return dns + return dns, account2identities get_accounts_identities = get_accounts_identities_slow @@ -189,11 +195,12 @@ def get_users(scim_url, access_token): d = response.json() if d['itemsPerPage'] == 0: break for user in d['Resources']: + if not user['active']: continue certs = [] for certificate in user.get('urn:indigo-dc:scim:schemas:IndigoUser', {}).get('certificates', []): certs.append(certificate['subjectDn']) groups = [x['display'] for x in user.get('groups', [])] - ret.append((user['userName'], user['active'], user['emails'][0]['value'], certs, groups)) + ret.append((user['id'], user['userName'], user['active'], user['emails'][0]['value'], certs, groups)) start += page_size return ret @@ -214,8 +221,9 @@ if __name__ == '__main__': except Exception as error: print("Failed to get also client certificate and key from rucio.cfg") sys.exit(CRITICAL) - TOKEN_URL = 'https://atlas-auth.cern.ch/token' - SCIM_URL = 'https://atlas-auth.cern.ch/scim' + ISSUER = 'https://atlas-auth.cern.ch/' + TOKEN_URL = '{}/token'.format(ISSUER) + SCIM_URL = '{}/scim'.format(ISSUER) try: CLIENT_ID = os.environ['CLIENT_ID'] CLIENT_SECRET = os.environ['CLIENT_SECRET'] @@ -229,13 +237,13 @@ if __name__ == '__main__': client = Client() accounts = {account['account']: account for account in client.list_accounts()} scopes = [_ for _ in client.list_scopes()] - dns = get_accounts_identities() + dns, account2identities = get_accounts_identities() ldap_accounts = get_ldap_identities() # synchronize accounts, identities and scopes synced_accounts = [] access_token = get_access_token(TOKEN_URL, CLIENT_ID, CLIENT_SECRET) - for nickname, active, email, certs, groups in get_users(SCIM_URL, access_token): + for userid, nickname, active, email, certs, groups in get_users(SCIM_URL, access_token): if 'atlas' not in groups: print("Skipping user {0} because it is not member of ATLAS groups".format(nickname)) continue @@ -296,10 +304,22 @@ if __name__ == '__main__': print('Identity {0} added to {2} account {1}'.format(dn, nickname, atype)) except Duplicate: pass + # c) add OIDC identity + oidc_identity = "SUB={}, ISS={}".format(userid, ISSUER) + if oidc_identity not in dns.get(atype, []): + try: + if not TEST: + client.add_identity(account=nickname, identity=oidc_identity, authtype='OIDC', email=email, default=True) + else: + print("CMD: rucio-admin identity add --account {0} --type OIDC --id '{1}' --email '{2}'".format(nickname, oidc_identity, email)) + nbusers += 1 + print('Identity {0} added to {2} account {1}'.format(userid, nickname, atype)) + except Duplicate: + pass if atype == 'USER': scope = 'user.' + nickname - elif active and nickname in ldap_accounts and len(certs) > 0: + elif active and nickname in ldap_accounts: # no rucio account exists for this nickname if email.lower() not in [mail.lower() for mail in ldap_accounts[nickname]['mails']]: print('Account %s (%s) without matching email in LDAP does not exist. To create it : rucio-admin account add --type USER --email %s %s' % (nickname, ldap_accounts[nickname]['type'], email, nickname)) @@ -318,6 +338,13 @@ if __name__ == '__main__': print("CMD: rucio-admin identity add --account {0} --type X509 --id '{1}' --email '{2}'".format(nickname, dn, email)) nbusers += 1 print('Identity {0} added to USER account {1}'.format(dn, nickname)) + oidc_identity = "SUB={}, ISS={}".format(userid, ISSUER) + if not TEST: + client.add_identity(account=nickname, identity=oidc_identity, authtype='OIDC', email=email, default=True) + else: + print("CMD: rucio-admin identity add --account {0} --type OIDC --id '{1}' --email '{2}'".format(nickname, oidc_identity, email)) + nbusers += 1 + print('Identity {0} added to {2} account {1}'.format(userid, nickname, atype)) scope = 'user.' + nickname except Exception: print('Failed to add new account %s (%s)' % (nickname, ldap_accounts[nickname]['type'])) @@ -343,8 +370,9 @@ if __name__ == '__main__': aname = account['account'] if aname not in synced_accounts: delusers += 1 + identities = account2identities.get(aname, []) if TEST: - print('User account {0} is no longer member VO, details {1}'.format(aname, account)) + print('User account {0} is no longer member VO, details {1}, identities({2}) {3}'.format(aname, account, len(identities), identities)) print('%i users extracted from VOMS, %i users updated, %i users not in VO' % (syncusers, nbusers, delusers)) diff --git a/atlas/check_voms_admin b/atlas/check_voms_admin index a5d03bb..cef18b6 100755 --- a/atlas/check_voms_admin +++ b/atlas/check_voms_admin @@ -80,6 +80,7 @@ def get_userid2certs_mapping(scim_url, access_token): d = response.json() if d['itemsPerPage'] == 0: break for user in d['Resources']: + if not user['active']: continue for certificate in user.get('urn:indigo-dc:scim:schemas:IndigoUser', {}).get('certificates', []): #ret.setdefault(user['id'], []).append((certificate['subjectDn'], certificate['issuerDn'], user['emails'][0]['value'])) ret.setdefault(user['id'], []).append(certificate['subjectDn']) @@ -153,8 +154,9 @@ if __name__ == '__main__': except Exception as error: print("Failed to get also client certificate and key from rucio.cfg") sys.exit(CRITICAL) - TOKEN_URL = 'https://atlas-auth.cern.ch/token' - SCIM_URL = 'https://atlas-auth.cern.ch/scim' + ISSUER = 'https://atlas-auth.cern.ch/' + TOKEN_URL = '{}/token'.format(ISSUER) + SCIM_URL = '{}/scim'.format(ISSUER) try: CLIENT_ID = os.environ['CLIENT_ID'] CLIENT_SECRET = os.environ['CLIENT_SECRET'] @@ -177,7 +179,7 @@ if __name__ == '__main__': account2attrs[account['account']] = {} if account['type'] != 'USER': continue for identity in client.list_identities(account['account']): - if identity['type'] != 'X509': continue + if identity['type'] not in ['X509', 'OIDC']: continue dn2useraccount[identity['identity']] = account['account'] access_token = get_access_token(TOKEN_URL, CLIENT_ID, CLIENT_SECRET) @@ -186,6 +188,9 @@ if __name__ == '__main__': userid2accounts = {} for userid, certs in userid2certs.items(): accounts = set() + oidc_identity = "SUB={}, ISS={}".format(userid, ISSUER) + if oidc_identity in dn2useraccount: + accounts.add(dn2useraccount[oidc_identity]) for cert in certs: if cert in dn2useraccount: accounts.add(dn2useraccount[cert]) @@ -230,7 +235,7 @@ if __name__ == '__main__': account2attributes.setdefault(account, {}).update(attributes) # cloud admins from ldap egroups - for cloud in ['ca', 'de', 'es', 'fr', 'it', 'ng', 'nl', 'ru', 't0', 'tw', 'uk', 'us']: + for cloud in ['ca', 'de', 'es', 'fr', 'it', 'ng', 'nl', 'ru', 't0', 'uk', 'us']: egroup = "atlas-support-cloud-{}".format(cloud) if cloud == 'ru': egroup = 'atlas-adc-cloud-ru'