diff --git a/pagure/lib/model.py b/pagure/lib/model.py
index aaae389..4785d14 100644
--- a/pagure/lib/model.py
+++ b/pagure/lib/model.py
@@ -2036,19 +2036,18 @@ class PullRequest(BASE):
 
         An user can only give one +1 and one -1.
         """
-        positive = set()
-        negative = set()
+        votes = {}
         for comment in self.discussion:
             for word in ["+1", ":thumbsup:"]:
                 if word in comment.comment:
-                    positive.add(comment.user_id)
+                    votes[comment.user_id] = 1
                     break
             for word in ["-1", ":thumbsdown:"]:
                 if word in comment.comment:
-                    negative.add(comment.user_id)
+                    votes[comment.user_id] = -1
                     break
 
-        return len(positive) - len(negative)
+        return sum(votes.values())
 
     @property
     def remote(self):
diff --git a/tests/test_pagure_lib.py b/tests/test_pagure_lib.py
index cdb2033..e9b7507 100644
--- a/tests/test_pagure_lib.py
+++ b/tests/test_pagure_lib.py
@@ -3106,7 +3106,7 @@ class PagureLibtests(tests.Modeltests):
         self.assertEqual(msg, 'Comment added')
 
         self.assertEqual(len(request.discussion), 3)
-        self.assertEqual(request.score, 1)
+        self.assertEqual(request.score, 2)
 
     def test_add_group(self):
         """ Test the add_group method of pagure.lib.query. """