diff --git a/app/src/main/java/com/owncloud/android/ui/adapter/OCFileListAdapter.java b/app/src/main/java/com/owncloud/android/ui/adapter/OCFileListAdapter.java
index 3fcb5aaa16eb..d63b783b95c8 100644
--- a/app/src/main/java/com/owncloud/android/ui/adapter/OCFileListAdapter.java
+++ b/app/src/main/java/com/owncloud/android/ui/adapter/OCFileListAdapter.java
@@ -221,13 +221,15 @@ public int getItemPosition(@NonNull OCFile file) {
         return position;
     }
 
+    @SuppressLint("NotifyDataSetChanged")
     public void setFavoriteAttributeForItemID(String remotePath, boolean favorite, boolean removeFromList) {
+        List<OCFile> filesToDelete = new ArrayList<>();
         for (OCFile file : mFiles) {
             if (file.getRemotePath().equals(remotePath)) {
                 file.setFavorite(favorite);
 
                 if (removeFromList) {
-                    mFiles.remove(file);
+                    filesToDelete.add(file);
                 }
 
                 break;
@@ -241,7 +243,7 @@ public void setFavoriteAttributeForItemID(String remotePath, boolean favorite, b
                 mStorageManager.saveFile(file);
 
                 if (removeFromList) {
-                    mFiles.remove(file);
+                    filesToDelete.add(file);
                 }
 
                 break;
@@ -257,7 +259,10 @@ public void setFavoriteAttributeForItemID(String remotePath, boolean favorite, b
             mFiles = sortOrder.sortCloudFiles(mFiles);
         }
 
-        new Handler(Looper.getMainLooper()).post(this::notifyDataSetChanged);
+        new Handler(Looper.getMainLooper()).post(() -> {
+            mFiles.removeAll(filesToDelete);
+            notifyDataSetChanged();
+        });
     }
 
     public void refreshCommentsCount(String fileId) {