diff --git a/pagure/__init__.py b/pagure/__init__.py
index 64dd781..713bba1 100644
--- a/pagure/__init__.py
+++ b/pagure/__init__.py
@@ -120,7 +120,12 @@ if APP.config.get('PAGURE_AUTH', None) in ['fas', 'openid']:
 
             # If groups are managed outside pagure, set up the user at login
             if not APP.config.get('ENABLE_GROUP_MNGT', False):
-                for group in flask.g.fas_user.groups:
+                user = pagure.lib.search_user(
+                    SESSION, username=flask.g.fas_user.username)
+                groups = set(user.groups)
+                fas_groups = set(flask.g.fas_user.groups)
+                # Add the new groups
+                for group in fas_groups - groups:
                     group = pagure.lib.search_groups(
                         SESSION, group_name=group)
                     if not group:
@@ -135,6 +140,19 @@ if APP.config.get('PAGURE_AUTH', None) in ['fas', 'openid']:
                         )
                     except pagure.exceptions.PagureException:
                         pass
+                # Remove the old groups
+                for group in groups - fas_groups:
+                    try:
+                        pagure.lib.delete_user_of_group(
+                            session=SESSION,
+                            username=flask.g.fas_user.username,
+                            groupname=group,
+                            user=flask.g.fas_user.username,
+                            is_admin=is_admin(),
+                            force=True,
+                        )
+                    except pagure.exceptions.PagureException:
+                        pass
 
             SESSION.commit()
         except SQLAlchemyError as err:
diff --git a/pagure/lib/__init__.py b/pagure/lib/__init__.py
index 9ab1b95..e5e0557 100644
--- a/pagure/lib/__init__.py
+++ b/pagure/lib/__init__.py
@@ -2472,7 +2472,8 @@ def add_user_to_group(session, username, group, user, is_admin):
         new_user.username, group.group_name)
 
 
-def delete_user_of_group(session, username, groupname, user, is_admin):
+def delete_user_of_group(session, username, groupname, user, is_admin,
+                         force=False):
     ''' Removes the specified user from the given group.
     '''
     group_obj = search_groups(session, group_name=groupname)
@@ -2496,7 +2497,7 @@ def delete_user_of_group(session, username, groupname, user, is_admin):
         raise pagure.exceptions.PagureException(
             'You are not allowed to remove user from this group')
 
-    if drop_user.username == group_obj.creator.username:
+    if drop_user.username == group_obj.creator.username and not force:
         raise pagure.exceptions.PagureException(
             'The creator of a group cannot be removed')