diff --git a/pagure/lib/__init__.py b/pagure/lib/__init__.py
index b6b1c2e..29b8264 100644
--- a/pagure/lib/__init__.py
+++ b/pagure/lib/__init__.py
@@ -700,9 +700,9 @@ def remove_tags_obj(session, obj, tags, ticketfolder, user):
         tags = [tags]
 
     removed_tags = []
-    for objtag in obj.tags:
-        if objtag.tag in tags:
-            tag = objtag.tag
+    for objtag in obj.tags_issues_colored:
+        if objtag.tag.tag in tags:
+            tag = objtag.tag.tag
             removed_tags.append(tag)
             session.delete(objtag)
 
diff --git a/pagure/lib/model.py b/pagure/lib/model.py
index ad01277..9a831a8 100644
--- a/pagure/lib/model.py
+++ b/pagure/lib/model.py
@@ -1105,6 +1105,7 @@ class TagColored(BASE):
 
     project = relation(
         'Project', foreign_keys=[project_id], remote_side=[Project.id],
+        backref="tags_colored",
     )
 
     def __repr__(self):
@@ -1137,6 +1138,7 @@ class TagIssueColored(BASE):
 
     issue = relation(
         'Issue', foreign_keys=[issue_uid], remote_side=[Issue.uid],
+        backref="tags_issues_colored"
     )
     tag = relation(
         'TagColored', foreign_keys=[tag_id], remote_side=[TagColored.id],
diff --git a/tests/test_pagure_lib.py b/tests/test_pagure_lib.py
index 6aa9fa4..e3d1114 100644
--- a/tests/test_pagure_lib.py
+++ b/tests/test_pagure_lib.py
@@ -1934,11 +1934,22 @@ class PagureLibtests(tests.Modeltests):
         issue = pagure.lib.search_issues(self.session, repo, issueid=1)
 
         # before
+        self.assertEqual(repo.tags_colored, [])
         self.assertEqual(issue.tags_text, [])
 
         messages = pagure.lib.update_tags(
             self.session, issue, 'tag', 'pingou', ticketfolder=None)
         self.assertEqual(messages, ['Tag added: tag'])
+
+        # after
+        repo = pagure.lib.get_project(self.session, 'test')
+        issue = pagure.lib.search_issues(self.session, repo, issueid=1)
+
+        self.assertEqual(
+            [t.tag for t in repo.tags_colored], ['tag'])
+        self.assertEqual(issue.tags_text, ['tag'])
+
+        # Replace the tag by two others
         messages = pagure.lib.update_tags(
             self.session, issue, ['tag2', 'tag3'], 'pingou',
             ticketfolder=None)
@@ -1946,8 +1957,15 @@ class PagureLibtests(tests.Modeltests):
             messages, ['Tag added: tag2, tag3', 'Removed tag: tag'])
 
         # after
+        repo = pagure.lib.get_project(self.session, 'test')
+        issue = pagure.lib.search_issues(self.session, repo, issueid=1)
+
+        self.assertEqual(
+            sorted([t.tag for t in repo.tags_colored]),
+            ['tag', 'tag2', 'tag3'])
         self.assertEqual(issue.tags_text, ['tag2', 'tag3'])
 
+
     @patch('pagure.lib.git.update_git')
     @patch('pagure.lib.notify.send_email')
     def test_update_dependency_issue(self, p_send_email, p_ugt):