Skip to content

Commit

Permalink
fix: workaround LIMIT limitation in background scanner
Browse files Browse the repository at this point in the history
Signed-off-by: Robin Appelman <[email protected]>
  • Loading branch information
icewind1991 committed Aug 15, 2024
1 parent 719f623 commit bb37e40
Showing 1 changed file with 58 additions and 10 deletions.
68 changes: 58 additions & 10 deletions apps/files/lib/BackgroundJob/ScanFiles.php
Original file line number Diff line number Diff line change
Expand Up @@ -70,17 +70,65 @@ protected function runScanner(string $user): void {
* @return string|false
*/
private function getUserToScan() {
if ($this->connection->getShardDefinition("filecache")) {
// for sharded filecache, the "LIMIT" from the normal query doesn't work

// first we try it with a "LEFT JOIN" on mounts, this is fast, but might return a storage that isn't mounted.
// we also ask for up to 10 results from different storages to increase the odds of finding a result that is mounted
$query = $this->connection->getQueryBuilder();
$query->select('m.user_id')
->from('filecache', 'f')
->leftJoin('f', 'mounts', 'm', $query->expr()->eq('m.storage_id', 'f.storage'))
->where($query->expr()->lt('f.size', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT)))
->andWhere($query->expr()->gt('f.parent', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT)))
->setMaxResults(10)
->groupBy("f.storage")
->runAcrossAllShards();

$result = $query->executeQuery();
while ($res = $result->fetch()) {
if ($res['user_id']) {
return $res['user_id'];
}
}
$userId = $res ? $res['user_id'] : false;
if ($userId) {
return $userId;
}

// as a fallback, we try a slower approach where we find all mounted storages first
// this is essentially doing the inner join manually
$storages = $this->getAllMountedStorages();

$query = $this->connection->getQueryBuilder();
$query->select('m.user_id')
->from('filecache', 'f')
->leftJoin('f', 'mounts', 'm', $query->expr()->eq('m.storage_id', 'f.storage'))
->where($query->expr()->lt('f.size', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT)))
->andWhere($query->expr()->gt('f.parent', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT)))
->andWhere($query->expr()->in('f.storage', $query->createNamedParameter($storages, IQueryBuilder::PARAM_INT_ARRAY)))
->setMaxResults(1)
->runAcrossAllShards();
return $query->executeQuery()->fetchOne();
} else {
$query = $this->connection->getQueryBuilder();
$query->select('m.user_id')
->from('filecache', 'f')
->innerJoin('f', 'mounts', 'm', $query->expr()->eq('m.storage_id', 'f.storage'))
->where($query->expr()->lt('f.size', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT)))
->andWhere($query->expr()->gt('f.parent', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT)))
->setMaxResults(1)
->runAcrossAllShards();

return $query->executeQuery()->fetchOne();
}
}

private function getAllMountedStorages(): array {
$query = $this->connection->getQueryBuilder();
$query->select('m.user_id')
->from('filecache', 'f')
->innerJoin('f', 'mounts', 'm', $query->expr()->eq('m.storage_id', 'f.storage'))
->where($query->expr()->lt('f.size', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT)))
->andWhere($query->expr()->gt('f.parent', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT)))
->setMaxResults(1)
->runAcrossAllShards();

$res = $query->executeQuery()->fetch();
return $res ? $res['user_id'] : false;
$query->selectDistinct('storage_id')
->from('mounts');
return $query->executeQuery()->fetchAll(\PDO::FETCH_COLUMN);
}

/**
Expand Down

0 comments on commit bb37e40

Please sign in to comment.