TASK-5688 - Allow study administrators to synchronise groups from LDAP/Azure

opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpenCgaCompleter.java

@@ -69,7 +69,7 @@ public abstract class OpenCgaCompleter implements Completer {
             .map(Candidate::new)
             .collect(toList());
 
-    private List<Candidate> usersList = asList( "anonymous","create","login","password","search","info","configs","configs-update","filters","password-reset","update")
+    private List<Candidate> usersList = asList( "anonymous","create","login","password","search","sync","info","configs","configs-update","filters","password-reset","update")
             .stream()
             .map(Candidate::new)
             .collect(toList());

opencga-app/src/main/java/org/opencb/opencga/app/cli/main/OpencgaCliOptionsParser.java

@@ -215,6 +215,7 @@ public OpencgaCliOptionsParser() {
         usersSubCommands.addCommand("login", usersCommandOptions.loginCommandOptions);
         usersSubCommands.addCommand("password", usersCommandOptions.passwordCommandOptions);
         usersSubCommands.addCommand("search", usersCommandOptions.searchCommandOptions);
+        usersSubCommands.addCommand("sync", usersCommandOptions.syncCommandOptions);
         usersSubCommands.addCommand("info", usersCommandOptions.infoCommandOptions);
         usersSubCommands.addCommand("configs", usersCommandOptions.configsCommandOptions);
         usersSubCommands.addCommand("configs-update", usersCommandOptions.updateConfigsCommandOptions);

opencga-app/src/main/java/org/opencb/opencga/app/cli/main/executors/UsersCommandExecutor.java

@@ -18,7 +18,9 @@
 import org.opencb.opencga.catalog.utils.ParamUtils.AddRemoveAction;
 import org.opencb.opencga.client.exceptions.ClientException;
 import org.opencb.opencga.core.common.JacksonUtils;
+import org.opencb.opencga.core.models.admin.GroupSyncParams;
 import org.opencb.opencga.core.models.common.Enums;
+import org.opencb.opencga.core.models.study.Group;
 import org.opencb.opencga.core.models.user.AuthenticationResponse;
 import org.opencb.opencga.core.models.user.ConfigUpdateParams;
 import org.opencb.opencga.core.models.user.FilterUpdateParams;
@@ -80,6 +82,9 @@ public void execute() throws Exception {
             case "search":
                 queryResponse = search();
                 break;
+            case "sync":
+                queryResponse = sync();
+                break;
             case "info":
                 queryResponse = info();
                 break;
@@ -209,6 +214,36 @@ private RestResponse<User> search() throws Exception {
         return openCGAClient.getUserClient().search(queryParams);
     }
 
+    private RestResponse<Group> sync() throws Exception {
+        logger.debug("Executing sync in Users command line");
+
+        UsersCommandOptions.SyncCommandOptions commandOptions = usersCommandOptions.syncCommandOptions;
+
+        GroupSyncParams groupSyncParams = null;
+        if (commandOptions.jsonDataModel) {
+            RestResponse<Group> res = new RestResponse<>();
+            res.setType(QueryType.VOID);
+            PrintUtils.println(getObjectAsJSON(categoryName,"/{apiVersion}/users/sync"));
+            return res;
+        } else if (commandOptions.jsonFile != null) {
+            groupSyncParams = JacksonUtils.getDefaultObjectMapper()
+                    .readValue(new java.io.File(commandOptions.jsonFile), GroupSyncParams.class);
+        } else {
+            ObjectMap beanParams = new ObjectMap();
+            putNestedIfNotEmpty(beanParams, "authenticationOriginId", commandOptions.authenticationOriginId, true);
+            putNestedIfNotEmpty(beanParams, "from", commandOptions.from, true);
+            putNestedIfNotEmpty(beanParams, "to", commandOptions.to, true);
+            putNestedIfNotEmpty(beanParams, "study", commandOptions.study, true);
+            putNestedIfNotNull(beanParams, "syncAll", commandOptions.syncAll, true);
+            putNestedIfNotNull(beanParams, "force", commandOptions.force, true);
+
+            groupSyncParams = JacksonUtils.getDefaultObjectMapper().copy()
+                    .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, true)
+                    .readValue(beanParams.toJson(), GroupSyncParams.class);
+        }
+        return openCGAClient.getUserClient().sync(groupSyncParams);
+    }
+
     private RestResponse<User> info() throws Exception {
         logger.debug("Executing info in Users command line");
 

opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/AdminCommandOptions.java

@@ -248,7 +248,7 @@ public class SearchUsersCommandOptions {
 
     }
 
-    @Parameters(commandNames = {"users-sync"}, commandDescription ="Synchronise a group of users from an authentication origin with a group in a study from catalog")
+    @Parameters(commandNames = {"users-sync"}, commandDescription ="[DEPRECATED] Moved to /users/sync")
     public class SyncUsersCommandOptions {
 
         @ParametersDelegate

opencga-app/src/main/java/org/opencb/opencga/app/cli/main/options/UsersCommandOptions.java

@@ -38,6 +38,7 @@ public class UsersCommandOptions extends CustomUsersCommandOptions {
         public LoginCommandOptions loginCommandOptions;
         public PasswordCommandOptions passwordCommandOptions;
         public SearchCommandOptions searchCommandOptions;
+        public SyncCommandOptions syncCommandOptions;
         public InfoCommandOptions infoCommandOptions;
         public ConfigsCommandOptions configsCommandOptions;
         public UpdateConfigsCommandOptions updateConfigsCommandOptions;
@@ -54,6 +55,7 @@ public UsersCommandOptions(CommonCommandOptions commonCommandOptions, JCommander
         this.loginCommandOptions = new LoginCommandOptions();
         this.passwordCommandOptions = new PasswordCommandOptions();
         this.searchCommandOptions = new SearchCommandOptions();
+        this.syncCommandOptions = new SyncCommandOptions();
         this.infoCommandOptions = new InfoCommandOptions();
         this.configsCommandOptions = new ConfigsCommandOptions();
         this.updateConfigsCommandOptions = new UpdateConfigsCommandOptions();
@@ -170,6 +172,38 @@ public class SearchCommandOptions {
 
     }
 
+    @Parameters(commandNames = {"sync"}, commandDescription ="Synchronise a group of users from an authentication origin with a group in a study from catalog")
+    public class SyncCommandOptions {
+
+        @ParametersDelegate
+        public CommonCommandOptions commonOptions = commonCommandOptions;
+
+        @Parameter(names = {"--json-file"}, description = "File with the body data in JSON format. Note, that using this parameter will ignore all the other parameters.", required = false, arity = 1)
+        public String jsonFile;
+
+        @Parameter(names = {"--json-data-model"}, description = "Show example of file structure for body data.", help = true, arity = 0)
+        public Boolean jsonDataModel = false;
+
+        @Parameter(names = {"--authentication-origin-id"}, description = "The body web service authenticationOriginId parameter", required = false, arity = 1)
+        public String authenticationOriginId;
+
+        @Parameter(names = {"--from"}, description = "The body web service from parameter", required = false, arity = 1)
+        public String from;
+
+        @Parameter(names = {"--to"}, description = "The body web service to parameter", required = false, arity = 1)
+        public String to;
+
+        @Parameter(names = {"--study", "-s"}, description = "The body web service study parameter", required = false, arity = 1)
+        public String study;
+
+        @Parameter(names = {"--sync-all"}, description = "The body web service syncAll parameter", required = false, help = true, arity = 0)
+        public boolean syncAll = false;
+
+        @Parameter(names = {"--force"}, description = "The body web service force parameter", required = false, help = true, arity = 0)
+        public boolean force = false;
+
+    }
+
     @Parameters(commandNames = {"info"}, commandDescription ="Return the user information including its projects and studies")
     public class InfoCommandOptions {
 

opencga-catalog/src/main/java/org/opencb/opencga/catalog/managers/UserManager.java

@@ -46,6 +46,7 @@
 import org.opencb.opencga.core.models.organizations.Organization;
 import org.opencb.opencga.core.models.study.Group;
 import org.opencb.opencga.core.models.study.GroupUpdateParams;
+import org.opencb.opencga.core.models.study.Study;
 import org.opencb.opencga.core.models.user.*;
 import org.opencb.opencga.core.response.OpenCGAResult;
 import org.slf4j.Logger;
@@ -302,6 +303,7 @@ public JwtPayload validateToken(String token) throws CatalogException {
         return jwtPayload;
     }
 
+    @Deprecated
     public void syncAllUsersOfExternalGroup(String organizationId, String study, String authOrigin, String token) throws CatalogException {
         if (!OPENCGA.equals(authenticationFactory.getUserId(organizationId, authOrigin, token))) {
             throw new CatalogAuthorizationException("Only the root user can perform this action");
@@ -365,19 +367,74 @@ public void syncAllUsersOfExternalGroup(String organizationId, String study, Str
         }
     }
 
-    /**
-     * Register all the users belonging to a remote group. If internalGroup and study are not null, it will also associate the remote group
-     * to the internalGroup defined.
-     *
-     * @param organizationId Organization id.
-     * @param authOrigin     Authentication origin.
-     * @param remoteGroup    Group name of the remote authentication origin.
-     * @param internalGroup  Group name in Catalog that will be associated to the remote group.
-     * @param study          Study where the internal group will be associated.
-     * @param sync           Boolean indicating whether the remote group will be synced with the internal group or not.
-     * @param token          JWT token. The token should belong to the root user.
-     * @throws CatalogException If any of the parameters is wrong or there is any internal error.
-     */
+    public void syncAllUsersOfExternalGroup(String studyStr, String authOrigin, String token) throws CatalogException {
+        JwtPayload tokenPayload = validateToken(token);
+        CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload);
+        Study study = catalogManager.getStudyManager().resolveId(studyFqn, null, tokenPayload);
+        String organizationId = studyFqn.getOrganizationId();
+        String userId = tokenPayload.getUserId(organizationId);
+
+        authorizationManager.checkIsAtLeastStudyAdministrator(organizationId, study.getUid(), userId);
+
+        OpenCGAResult<Group> allGroups = catalogManager.getStudyManager().getGroup(studyStr, null, token);
+
+        boolean foundAny = false;
+        for (Group group : allGroups.getResults()) {
+            if (group.getSyncedFrom() != null && group.getSyncedFrom().getAuthOrigin().equals(authOrigin)) {
+                logger.info("Fetching users of group '{}' from authentication origin '{}'", group.getSyncedFrom().getRemoteGroup(),
+                        group.getSyncedFrom().getAuthOrigin());
+                foundAny = true;
+
+                List<User> userList;
+                try {
+                    userList = authenticationFactory.getUsersFromRemoteGroup(organizationId, group.getSyncedFrom().getAuthOrigin(),
+                            group.getSyncedFrom().getRemoteGroup());
+                } catch (CatalogException e) {
+                    // There was some kind of issue for which we could not retrieve the group information.
+                    logger.info("Removing all users from group '{}' belonging to group '{}' in the external authentication origin",
+                            group.getId(), group.getSyncedFrom().getAuthOrigin());
+                    logger.info("Please, manually remove group '{}' if external group '{}' was removed from the authentication origin",
+                            group.getId(), group.getSyncedFrom().getAuthOrigin());
+                    catalogManager.getStudyManager().updateGroup(studyStr, group.getId(), ParamUtils.BasicUpdateAction.SET,
+                            new GroupUpdateParams(Collections.emptyList()), token);
+                    continue;
+                }
+                Iterator<User> iterator = userList.iterator();
+                while (iterator.hasNext()) {
+                    User user = iterator.next();
+                    user.setOrganization(organizationId);
+                    try {
+                        create(user, null, token);
+                        logger.info("User '{}' ({}) successfully created", user.getId(), user.getName());
+                    } catch (CatalogParameterException e) {
+                        logger.warn("Could not create user '{}' ({}). {}", user.getId(), user.getName(), e.getMessage());
+                        iterator.remove();
+                    } catch (CatalogException e) {
+                        if (!e.getMessage().contains("already exists")) {
+                            logger.warn("Could not create user '{}' ({}). {}", user.getId(), user.getName(), e.getMessage());
+                            iterator.remove();
+                        }
+                    }
+                }
+
+                GroupUpdateParams updateParams;
+                if (ListUtils.isEmpty(userList)) {
+                    logger.info("No members associated to the external group");
+                    updateParams = new GroupUpdateParams(Collections.emptyList());
+                } else {
+                    logger.info("Associating members to the internal OpenCGA group");
+                    updateParams = new GroupUpdateParams(new ArrayList<>(userList.stream().map(User::getId).collect(Collectors.toSet())));
+                }
+                catalogManager.getStudyManager().updateGroup(studyStr, group.getId(), ParamUtils.BasicUpdateAction.SET,
+                        updateParams, token);
+            }
+        }
+        if (!foundAny) {
+            logger.info("No synced groups found in study '{}' from authentication origin '{}'", studyStr, authOrigin);
+        }
+    }
+
+    @Deprecated
     public void importRemoteGroupOfUsers(String organizationId, String authOrigin, String remoteGroup, @Nullable String internalGroup,
                                          @Nullable String study, boolean sync, String token) throws CatalogException {
         JwtPayload payload = validateToken(token);
@@ -454,6 +511,107 @@ public void importRemoteGroupOfUsers(String organizationId, String authOrigin, S
         }
     }
 
+    /**
+     * Register all the users belonging to a remote group. If internalGroup and study are not null, it will also associate the remote group
+     * to the internalGroup defined.
+     *
+     * @param authOrigin     Authentication origin.
+     * @param remoteGroup    Group name of the remote authentication origin.
+     * @param internalGroup  Group name in Catalog that will be associated to the remote group.
+     * @param studyStr       Study where the internal group will be associated.
+     * @param sync           Boolean indicating whether the remote group will be synced with the internal group or not.
+     * @param token          JWT token. The token should belong to the root user.
+     * @throws CatalogException If any of the parameters is wrong or there is any internal error.
+     */
+    public void importRemoteGroupOfUsers(String authOrigin, String remoteGroup, @Nullable String internalGroup,
+                                         @Nullable String studyStr, boolean sync, String token) throws CatalogException {
+        JwtPayload tokenPayload = validateToken(token);
+        String organizationId;
+        String userId;
+        Study study = null;
+        if (StringUtils.isNotEmpty(studyStr)) {
+            CatalogFqn studyFqn = CatalogFqn.extractFqnFromStudy(studyStr, tokenPayload);
+            study = catalogManager.getStudyManager().resolveId(studyFqn, null, tokenPayload);
+            organizationId = studyFqn.getOrganizationId();
+            userId = tokenPayload.getUserId(organizationId);
+        } else {
+            organizationId = tokenPayload.getOrganization();
+            userId = tokenPayload.getUserId();
+        }
+
+        ObjectMap auditParams = new ObjectMap()
+                .append("organizationId", organizationId)
+                .append("authOrigin", authOrigin)
+                .append("remoteGroup", remoteGroup)
+                .append("internalGroup", internalGroup)
+                .append("study", studyStr)
+                .append("sync", sync)
+                .append("token", token);
+        try {
+            if (studyStr != null) {
+                authorizationManager.checkIsAtLeastStudyAdministrator(organizationId, study.getUid(), userId);
+            } else {
+                authorizationManager.checkIsAtLeastOrganizationOwnerOrAdmin(organizationId, userId);
+            }
+
+            ParamUtils.checkParameter(authOrigin, "Authentication origin");
+            ParamUtils.checkParameter(remoteGroup, "Remote group");
+
+            List<User> userList;
+            if (sync) {
+                // We don't create any user as they will be automatically populated during login
+                userList = Collections.emptyList();
+            } else {
+                logger.info("Fetching users from authentication origin '{}'", authOrigin);
+
+                // Register the users
+                userList = authenticationFactory.getUsersFromRemoteGroup(organizationId, authOrigin, remoteGroup);
+                for (User user : userList) {
+                    user.setOrganization(organizationId);
+                    try {
+                        create(user, null, token);
+                        logger.info("User '{}' successfully created", user.getId());
+                    } catch (CatalogException e) {
+                        logger.warn("{}", e.getMessage());
+                    }
+                }
+            }
+
+            if (StringUtils.isNotEmpty(internalGroup) && StringUtils.isNotEmpty(studyStr)) {
+                // Check if the group already exists
+                OpenCGAResult<Group> groupResult = catalogManager.getStudyManager().getGroup(studyStr, internalGroup, token);
+                if (groupResult.getNumResults() == 1) {
+                    logger.error("Cannot synchronise with group {}. The group already exists and is already in use.", internalGroup);
+                    throw new CatalogException("Cannot synchronise with group " + internalGroup
+                            + ". The group already exists and is already in use.");
+                }
+
+                // Create new group associating it to the remote group
+                try {
+                    logger.info("Attempting to register group '{}' in study '{}'", internalGroup, studyStr);
+                    Group.Sync groupSync = null;
+                    if (sync) {
+                        groupSync = new Group.Sync(authOrigin, remoteGroup);
+                    }
+                    Group group = new Group(internalGroup, userList.stream().map(User::getId).collect(Collectors.toList()))
+                            .setSyncedFrom(groupSync);
+                    catalogManager.getStudyManager().createGroup(studyStr, group, token);
+                    logger.info("Group '{}' created and synchronised with external group", internalGroup);
+                    auditManager.audit(organizationId, userId, Enums.Action.IMPORT_EXTERNAL_GROUP_OF_USERS, Enums.Resource.USER,
+                            group.getId(), "", studyStr, "", auditParams, new AuditRecord.Status(AuditRecord.Status.Result.SUCCESS));
+                } catch (CatalogException e) {
+                    logger.error("Could not register group '{}' in study '{}'\n{}", internalGroup, studyStr, e.getMessage(), e);
+                    throw new CatalogException("Could not register group '" + internalGroup + "' in study '" + studyStr + "': "
+                            + e.getMessage(), e);
+                }
+            }
+        } catch (CatalogException e) {
+            auditManager.audit(organizationId, userId, Enums.Action.IMPORT_EXTERNAL_GROUP_OF_USERS, Enums.Resource.USER, "", "", "", "",
+                    auditParams, new AuditRecord.Status(AuditRecord.Status.Result.ERROR, e.getError()));
+            throw e;
+        }
+    }
+
     /**
      * Register all the ids. If internalGroup and study are not null, it will also associate the users to the internalGroup defined.
      *