Skip to content

perf(file-list): db pagination #17505

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
alperozturk96 wants to merge 2 commits into
base: master
Choose a base branch
from
Draft

perf(file-list): db pagination #17505

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0d83796
fix(file-list): db pagination
alperozturk96 Aug 19, 2026
ef1a647
wip
alperozturk96 Sep 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 28 additions & 8 deletions app/src/main/java/com/nextcloud/client/database/dao/FileDao.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,11 +41,31 @@ interface FileDao {
@Query("SELECT * FROM filelist WHERE remote_id = :remoteId LIMIT 1")
suspend fun getFileByRemoteId(remoteId: String): FileEntity?

@Query("SELECT * FROM filelist WHERE parent = :parentId ORDER BY ${ProviderTableMeta.FILE_DEFAULT_SORT_ORDER}")
fun getFolderContent(parentId: Long): List<FileEntity>
@Query(
"""
SELECT ${ProviderTableMeta._ID}
FROM filelist
WHERE parent = :parentId
ORDER BY ${ProviderTableMeta.FILE_DEFAULT_SORT_ORDER}
"""
)
fun getFolderContentIds(parentId: Long): List<Long>

@Query("SELECT * FROM filelist WHERE parent = :parentId ORDER BY ${ProviderTableMeta.FILE_DEFAULT_SORT_ORDER}")
suspend fun getFolderContentSuspended(parentId: Long): List<FileEntity>
@Query(
"""
SELECT ${ProviderTableMeta._ID}
FROM filelist
WHERE parent = :parentId
ORDER BY ${ProviderTableMeta.FILE_DEFAULT_SORT_ORDER}
"""
)
suspend fun getFolderContentIdsSuspended(parentId: Long): List<Long>

@Query("SELECT * FROM filelist WHERE ${ProviderTableMeta._ID} IN (:ids)")
fun getFilesByIds(ids: List<Long>): List<FileEntity>

@Query("SELECT * FROM filelist WHERE ${ProviderTableMeta._ID} IN (:ids)")
suspend fun getFilesByIdsSuspended(ids: List<Long>): List<FileEntity>

@Query(
"SELECT * FROM filelist WHERE modified >= :startDate" +
Expand Down Expand Up @@ -145,7 +165,7 @@ interface FileDao {

@Query(
"""
SELECT *
SELECT ${ProviderTableMeta._ID}
FROM filelist
WHERE file_owner = :accountName
AND (
Expand All @@ -156,18 +176,18 @@ interface FileDao {
ORDER BY ${ProviderTableMeta.FILE_DEFAULT_SORT_ORDER}
"""
)
suspend fun getSharedFiles(accountName: String): List<FileEntity>
suspend fun getSharedFileIds(accountName: String): List<Long>

@Query(
"""
SELECT *
SELECT ${ProviderTableMeta._ID}
FROM filelist
WHERE file_owner = :fileOwner
AND favorite = 1
ORDER BY ${ProviderTableMeta.FILE_DEFAULT_SORT_ORDER}
"""
)
suspend fun getFavoriteFiles(fileOwner: String): List<FileEntity>
suspend fun getFavoriteFileIds(fileOwner: String): List<Long>

@Query(
"""
Expand Down
26 changes: 26 additions & 0 deletions app/src/main/java/com/nextcloud/utils/extensions/FileDataStorageManagerExtensions.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
package com.nextcloud.utils.extensions

import com.nextcloud.client.database.dao.FileDao
import com.nextcloud.client.database.entity.FileEntity
import com.nextcloud.client.database.entity.model.ShareeKey
import com.nextcloud.client.database.entity.toOCCapability
import com.owncloud.android.datamodel.FileDataStorageManager
Expand Down Expand Up @@ -249,3 +250,28 @@ private fun FileDao.moveFilesInDb(
updateAll(updated)
return originalMediaPaths
}

private const val FILE_ID_CHUNK_SIZE = 100

private fun List<Long>.toEntitiesInOrder(loadChunk: (List<Long>) -> List<FileEntity>): List<FileEntity> {
val byId = chunked(FILE_ID_CHUNK_SIZE).flatMap(loadChunk).associateBy { it.id }
return mapNotNull { byId[it] }
}

fun FileDataStorageManager.getFolderContentEntities(parentId: Long): List<FileEntity> =
fileDao.getFolderContentIds(parentId).toEntitiesInOrder(fileDao::getFilesByIds)

suspend fun FileDataStorageManager.getFolderContentEntitiesSuspended(parentId: Long): List<FileEntity> =
fileDao.getFolderContentIdsSuspended(parentId).toEntitiesInOrderSuspended(this)

suspend fun FileDataStorageManager.getSharedFileEntities(accountName: String): List<FileEntity> =
fileDao.getSharedFileIds(accountName).toEntitiesInOrderSuspended(this)

suspend fun FileDataStorageManager.getFavoriteFileEntities(accountName: String): List<FileEntity> =

@alperozturk96 alperozturk96 Sep 9, 2026 •
edited
Loading

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

On master we only show 100 at root level. Is this expected @tobiasKaminsky ?

Shared tab needs to be checked as well.

@daniele-verducci daniele-verducci Sep 9, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also, this bug is happening both on master and of course here:

  • Log in
  • Let it load the root folder: you have x files
  • Add from another source 10k files (e.g. create them from terminal and then occ files:scan ...)
  • Pull to refresh: the same files as before are shown
  • Uninstall and reinstall the app, log in
  • All the files, including the new ones, are shown

fileDao.getFavoriteFileIds(accountName).toEntitiesInOrderSuspended(this)

private suspend fun List<Long>.toEntitiesInOrderSuspended(storageManager: FileDataStorageManager): List<FileEntity> {
val entities = chunked(FILE_ID_CHUNK_SIZE).flatMap { storageManager.fileDao.getFilesByIdsSuspended(it) }
val byId = entities.associateBy { it.id }
return mapNotNull { byId[it] }
}
2 changes: 1 addition & 1 deletion app/src/main/java/com/owncloud/android/datamodel/FileDataStorageManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1216,7 +1216,7 @@ private List<OCFile> getFolderContent(long parentId, boolean onlyOnDevice) {
Log_OC.d(TAG, "getFolderContent - start");
List<OCFile> folderContent = new ArrayList<>();

List<FileEntity> files = fileDao.getFolderContent(parentId);
List<FileEntity> files = FileDataStorageManagerExtensionsKt.getFolderContentEntities(this, parentId);
for (FileEntity fileEntity : files) {
OCFile child = createFileInstance(fileEntity);
if (!onlyOnDevice || child.existsOnDevice()) {
Expand Down
3 changes: 2 additions & 1 deletion app/src/main/java/com/owncloud/android/datamodel/OCFileListAdapterDataProviderImpl.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
package com.owncloud.android.datamodel

import com.nextcloud.client.database.entity.FileEntity
import com.nextcloud.utils.extensions.getFolderContentEntitiesSuspended
import com.owncloud.android.ui.adapter.helper.OCFileListAdapterDataProvider

@Suppress("ReturnCount")
Expand All @@ -17,7 +18,7 @@ class OCFileListAdapterDataProviderImpl(private val storageManager: FileDataStor
storageManager.offlineOperationsRepository.convertToOCFiles(id)

override suspend fun getFolderContent(id: Long): List<FileEntity> =
storageManager.fileDao.getFolderContentSuspended(id)
storageManager.getFolderContentEntitiesSuspended(id)

override fun createFileInstance(entity: FileEntity): OCFile = storageManager.createFileInstance(entity)
}
6 changes: 4 additions & 2 deletions app/src/main/java/com/owncloud/android/ui/fragment/OCFileListSearchTask.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ import android.content.ContentValues
import androidx.lifecycle.lifecycleScope
import com.nextcloud.client.account.User
import com.nextcloud.client.preferences.AppPreferences
import com.nextcloud.utils.extensions.getFavoriteFileEntities
import com.nextcloud.utils.extensions.getSharedFileEntities
import com.owncloud.android.R
import com.owncloud.android.datamodel.FileDataStorageManager
import com.owncloud.android.datamodel.OCFile
Expand Down Expand Up @@ -113,9 +115,9 @@ class OCFileListSearchTask(
fragment: OCFileListFragment
): List<OCFile> {
val files = if (searchType == SearchRemoteOperation.SearchType.SHARED_FILTER) {
storageManager.fileDao.getSharedFiles(currentUser.accountName)
storageManager.getSharedFileEntities(currentUser.accountName)
} else {
storageManager.fileDao.getFavoriteFiles(currentUser.accountName)
storageManager.getFavoriteFileEntities(currentUser.accountName)
}.mapNotNull { storageManager.createFileInstance(it) }

return sortSearchData(files, fragmentSearchType, fragment)
Expand Down
Loading