Cache: show upgrade progress

apps/files/ajax/upgrade.php

+<?php
+set_time_limit(0); //scanning can take ages
+session_write_close();
+
+$user = OC_User::getUser();
+$eventSource = new OC_EventSource();
+$listener = new UpgradeListener($eventSource);
+$legacy = new \OC\Files\Cache\Legacy($user);
+
+if ($legacy->hasItems()) {
+	OC_Hook::connect('\OC\Files\Cache\Upgrade', 'migrate_path', $listener, 'upgradePath');
+
+	$upgrade = new \OC\Files\Cache\Upgrade($legacy);
+	$count = $legacy->getCount();
+	$eventSource->send('total', $count);
+	$upgrade->upgradePath('/' . $user . '/files');
+}
+\OC\Files\Cache\Upgrade::upgradeDone($user);
+$eventSource->send('done', true);
+$eventSource->close();
+
+class UpgradeListener {
+	/**
+	 * @var OC_EventSource $eventSource
+	 */
+	private $eventSource;
+
+	private $count = 0;
+	private $lastSend = 0;
+
+	public function __construct($eventSource) {
+		$this->eventSource = $eventSource;
+	}
+
+	public function upgradePath($path) {
+		$this->count++;
+		if ($this->count > ($this->lastSend + 5)) {
+			$this->lastSend = $this->count;
+			$this->eventSource->send('count', $this->count);
+		}
+	}
+}

apps/files/appinfo/app.php

@@ -10,13 +10,3 @@ OCP\App::addNavigationEntry( array( "id" => "files_index",
 									"name" => $l->t("Files") ));
 
 OC_Search::registerProvider('OC_Search_Provider_File');
-
-if (OC_User::isLoggedIn()) {
-	// update OC4.5 filecache to OC5 filecache, can't do this in update.php since it needs to happen for each user individually
-	$cacheVersion = (int)OCP\Config::getUserValue(OC_User::getUser(), 'files', 'cache_version', 4);
-	if ($cacheVersion < 5) {
-		\OC_Log::write('files', 'updating filecache to 5.0 for user ' . OC_User::getUser(), \OC_Log::INFO);
-		\OC\Files\Cache\Upgrade::upgrade();
-		OCP\Config::setUserValue(OC_User::getUser(), 'files', 'cache_version', 5);
-	}
-}

apps/files/css/files.css

@@ -122,3 +122,12 @@ a.action>img { max-height:16px; max-width:16px; vertical-align:text-bottom; }
 #scanning-message{ top:40%; left:40%; position:absolute; display:none; }
 
 div.crumb a{ padding:0.9em 0 0.7em 0; }
+
+#upgrade {
+	width: 400px;
+	position: absolute;
+	top: 200px;
+	left: 50%;
+	text-align: center;
+	margin-left: -200px;
+}

apps/files/index.php

@@ -28,7 +28,6 @@ OCP\User::checkLoggedIn();
 OCP\Util::addStyle('files', 'files');
 OCP\Util::addscript('files', 'jquery.iframe-transport');
 OCP\Util::addscript('files', 'jquery.fileupload');
-OCP\Util::addscript('files', 'files');
 OCP\Util::addscript('files', 'filelist');
 OCP\Util::addscript('files', 'fileactions');
 OCP\Util::addscript('files', 'keyboardshortcuts');
@@ -37,8 +36,8 @@ OCP\App::setActiveNavigationEntry('files_index');
 // Load the files
 $dir = isset($_GET['dir']) ? stripslashes($_GET['dir']) : '';
 // Redirect if directory does not exist
-if(!\OC\Files\Filesystem::is_dir($dir . '/')) {
-	header('Location: '.$_SERVER['SCRIPT_NAME'].'');
+if (!\OC\Files\Filesystem::is_dir($dir . '/')) {
+	header('Location: ' . $_SERVER['SCRIPT_NAME'] . '');
 	exit();
 }
 
@@ -53,16 +52,25 @@ function fileCmp($a, $b) {
 }
 
 $files = array();
-foreach( \OC\Files\Filesystem::getDirectoryContent( $dir ) as $i ) {
-	$i['date'] = OCP\Util::formatDate($i['mtime'] );
-	if($i['type'] == 'file') {
+$user = OC_User::getUser();
+if (\OC\Files\Cache\Upgrade::needUpgrade($user)) { //dont load anything if we need to upgrade the cache
+	$content = array();
+	$needUpgrade = true;
+	$freeSpace = 0;
+} else {
+	$content = \OC\Files\Filesystem::getDirectoryContent($dir);
+	$freeSpace = \OC\Files\Filesystem::free_space($dir);
+	$needUpgrade = false;
+}
+foreach ($content as $i) {
+	$i['date'] = OCP\Util::formatDate($i['mtime']);
+	if ($i['type'] == 'file') {
 		$fileinfo = pathinfo($i['name']);
 		$i['basename'] = $fileinfo['filename'];
 		if (!empty($fileinfo['extension'])) {
-			$i['extension']='.' . $fileinfo['extension'];
-		}
-		else {
-			$i['extension']='';
+			$i['extension'] = '.' . $fileinfo['extension'];
+		} else {
+			$i['extension'] = '';
 		}
 	}
 	$i['directory'] = $dir;
@@ -74,10 +82,10 @@ usort($files, "fileCmp");
 // Make breadcrumb
 $breadcrumb = array();
 $pathtohere = '';
-foreach( explode( '/', $dir ) as $i ) {
-	if( $i != '' ) {
+foreach (explode('/', $dir) as $i) {
+	if ($i != '') {
 		$pathtohere .= '/' . $i;
-		$breadcrumb[] = array( 'dir' => $pathtohere, 'name' => $i );
+		$breadcrumb[] = array('dir' => $pathtohere, 'name' => $i);
 	}
 }
 
@@ -94,29 +102,35 @@ $upload_max_filesize = OCP\Util::computerFileSize(ini_get('upload_max_filesize')
 $post_max_size = OCP\Util::computerFileSize(ini_get('post_max_size'));
 $maxUploadFilesize = min($upload_max_filesize, $post_max_size);
 
-$freeSpace = \OC\Files\Filesystem::free_space($dir);
-$freeSpace = max($freeSpace,0);
+$freeSpace = max($freeSpace, 0);
 $maxUploadFilesize = min($maxUploadFilesize, $freeSpace);
 
 $permissions = OCP\PERMISSION_READ;
 if (\OC\Files\Filesystem::isUpdatable($dir . '/')) {
-    $permissions |= OCP\PERMISSION_UPDATE;
+	$permissions |= OCP\PERMISSION_UPDATE;
 }
 if (\OC\Files\Filesystem::isDeletable($dir . '/')) {
-    $permissions |= OCP\PERMISSION_DELETE;
+	$permissions |= OCP\PERMISSION_DELETE;
 }
 if (\OC\Files\Filesystem::isSharable($dir . '/')) {
-    $permissions |= OCP\PERMISSION_SHARE;
+	$permissions |= OCP\PERMISSION_SHARE;
 }
 
-$tmpl = new OCP\Template( 'files', 'index', 'user' );
-$tmpl->assign( 'fileList', $list->fetchPage(), false );
-$tmpl->assign( 'breadcrumb', $breadcrumbNav->fetchPage(), false );
-$tmpl->assign( 'dir', \OC\Files\Filesystem::normalizePath($dir));
-$tmpl->assign( 'isCreatable', \OC\Files\Filesystem::isCreatable($dir . '/'));
-$tmpl->assign('permissions', $permissions);
-$tmpl->assign('files', $files);
-$tmpl->assign('uploadMaxFilesize', $maxUploadFilesize);
-$tmpl->assign('uploadMaxHumanFilesize', OCP\Util::humanFileSize($maxUploadFilesize));
-$tmpl->assign('allowZipDownload', intval(OCP\Config::getSystemValue('allowZipDownload', true)));
-$tmpl->printPage();
+if ($needUpgrade) {
+	OCP\Util::addscript('files', 'upgrade');
+	$tmpl = new OCP\Template('files', 'upgrade', 'user');
+	$tmpl->printPage();
+} else {
+	OCP\Util::addscript('files', 'files');
+	$tmpl = new OCP\Template('files', 'index', 'user');
+	$tmpl->assign('fileList', $list->fetchPage(), false);
+	$tmpl->assign('breadcrumb', $breadcrumbNav->fetchPage(), false);
+	$tmpl->assign('dir', \OC\Files\Filesystem::normalizePath($dir));
+	$tmpl->assign('isCreatable', \OC\Files\Filesystem::isCreatable($dir . '/'));
+	$tmpl->assign('permissions', $permissions);
+	$tmpl->assign('files', $files);
+	$tmpl->assign('uploadMaxFilesize', $maxUploadFilesize);
+	$tmpl->assign('uploadMaxHumanFilesize', OCP\Util::humanFileSize($maxUploadFilesize));
+	$tmpl->assign('allowZipDownload', intval(OCP\Config::getSystemValue('allowZipDownload', true)));
+	$tmpl->printPage();
+}

apps/files/js/upgrade.js

+$(document).ready(function () {
+	var eventSource, total, bar = $('#progressbar');
+	console.log('start');
+	bar.progressbar({value: 0});
+	eventSource = new OC.EventSource(OC.filePath('files', 'ajax', 'upgrade.php'));
+	eventSource.listen('total', function (count) {
+		total = count;
+		console.log(count + ' files needed to be migrated');
+	});
+	eventSource.listen('count', function (count) {
+		bar.progressbar({value: (count / total) * 100});
+		console.log(count);
+	});
+	eventSource.listen('done', function () {
+		document.location.reload();
+	});
+});

apps/files/templates/upgrade.php

+<div id="upgrade">
+	<?php echo $l->t('Upgrading filesystem cache...');?>
+	<div id="progressbar" />
+</div>

lib/files/cache/legacy.php

+<?php
+/**
+ * Copyright (c) 2012 Robin Appelman <icewind@owncloud.com>
+ * This file is licensed under the Affero General Public License version 3 or
+ * later.
+ * See the COPYING-README file.
+ */
+
+namespace OC\Files\Cache;
+
+/**
+ * Provide read only support for the old filecache
+ */
+class Legacy {
+	private $user;
+
+	public function __construct($user) {
+		$this->user = $user;
+	}
+
+	function getCount() {
+		$query = \OC_DB::prepare('SELECT COUNT(`id`) AS `count` FROM `*PREFIX*fscache` WHERE `user` = ?');
+		$result = $query->execute(array($this->user));
+		if ($row = $result->fetchRow()) {
+			return $row['count'];
+		} else {
+			return 0;
+		}
+	}
+
+	/**
+	 * check if a legacy cache is present and holds items
+	 *
+	 * @return bool
+	 */
+	function hasItems() {
+		try {
+			$query = \OC_DB::prepare('SELECT `id` FROM `*PREFIX*fscache` WHERE `user` = ? LIMIT 1');
+		} catch (\Exception $e) {
+			return false;
+		}
+		try {
+			$result = $query->execute(array($this->user));
+		} catch (\Exception $e) {
+			return false;
+		}
+		return (bool)$result->fetchRow();
+	}
+
+	/**
+	 * @param string|int $path
+	 * @return array
+	 */
+	function get($path) {
+		if (is_numeric($path)) {
+			$query = \OC_DB::prepare('SELECT * FROM `*PREFIX*fscache` WHERE `id` = ?');
+		} else {
+			$query = \OC_DB::prepare('SELECT * FROM `*PREFIX*fscache` WHERE `path` = ?');
+		}
+		$result = $query->execute(array($path));
+		return $result->fetchRow();
+	}
+
+	/**
+	 * @param int $id
+	 * @return array
+	 */
+	function getChildren($id) {
+		$query = \OC_DB::prepare('SELECT * FROM `*PREFIX*fscache` WHERE `parent` = ?');
+		$result = $query->execute(array($id));
+		return $result->fetchAll();
+	}
+}

lib/files/cache/upgrade.php

@@ -9,62 +9,102 @@
 namespace OC\Files\Cache;
 
 class Upgrade {
-	static $permissionsCaches = array();
+	/**
+	 * @var Legacy $legacy
+	 */
+	private $legacy;
 
-	static $numericIds = array();
+	private $permissionsCaches = array();
 
-	static function upgrade() {
-		$insertQuery = \OC_DB::prepare('INSERT INTO `*PREFIX*filecache`( `fileid`, `storage`, `path`, `path_hash`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted` )
-			VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)');
+	private $numericIds = array();
 
-		try {
-			$oldEntriesQuery = \OC_DB::prepare('SELECT * FROM `*PREFIX*fscache` ORDER BY `id` ASC'); //sort ascending to ensure the parent gets inserted before a child
-		} catch (\Exception $e) {
-			return;
-		}
-		try {
-			$oldEntriesResult = $oldEntriesQuery->execute();
-		} catch (\Exception $e) {
-			return;
-		}
-		if (!$oldEntriesResult) {
+	private $mimeTypeIds = array();
+
+	/**
+	 * @param Legacy $legacy
+	 */
+	public function __construct($legacy) {
+		$this->legacy = $legacy;
+	}
+
+	/**
+	 * Preform a shallow upgrade
+	 *
+	 * @param string $path
+	 * @param int $mode
+	 */
+	function upgradePath($path, $mode = Scanner::SCAN_RECURSIVE) {
+		if (!$this->legacy->hasItems()) {
 			return;
 		}
+		\OC_Hook::emit('\OC\Files\Cache\Upgrade', 'migrate_path', $path);
 
-		$checkExistingQuery = \OC_DB::prepare('SELECT `fileid` FROM `*PREFIX*filecache` WHERE `fileid` = ?');
+		if ($row = $this->legacy->get($path)) {
+			$data = $this->getNewData($row);
+			$this->insert($data);
 
-		while ($row = $oldEntriesResult->fetchRow()) {
-			if ($checkExistingQuery->execute(array($row['id']))->fetchRow()) {
-				continue;
+			$children = $this->legacy->getChildren($data['id']);
+			foreach ($children as $child) {
+				if ($mode == Scanner::SCAN_SHALLOW) {
+					$childData = $this->getNewData($child);
+					\OC_Hook::emit('\OC\Files\Cache\Upgrade', 'migrate_path', $child['path']);
+					$this->insert($childData);
+				} else {
+					$this->upgradePath($child['path']);
+				}
 			}
+		}
+	}
 
-			list($storage, $internalPath) = \OC\Files\Filesystem::resolvePath($row['path']);
-			/**
-			 * @var \OC\Files\Storage\Storage $storage
-			 * @var string $internalPath;
-			 */
-			$pathHash = md5($internalPath);
-			$storageId = self::getNumericId($storage);
-			$parentId = ($internalPath === '') ? -1 : $row['parent'];
+	/**
+	 * @param array $data the data for the new cache
+	 */
+	function insert($data) {
+		$insertQuery = \OC_DB::prepare('INSERT INTO `*PREFIX*filecache`
+				( `fileid`, `storage`, `path`, `path_hash`, `parent`, `name`, `mimetype`, `mimepart`, `size`, `mtime`, `encrypted` )
+				VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)');
 
-			$insertQuery->execute(array($row['id'], $storageId, $internalPath, $pathHash, $parentId, $row['name'], $row['mimetype'], $row['mimepart'], $row['size'], $row['mtime'], $row['encrypted']));
+		$insertQuery->execute(array($data['id'], $data['storage'], $data['path'], $data['path_hash'], $data['parent'], $data['name'],
+			$data['mimetype'], $data['mimepart'], $data['size'], $data['mtime'], $data['encrypted']));
 
-			$permissions = ($row['writable']) ? \OCP\PERMISSION_ALL : \OCP\PERMISSION_READ;
-			$permissionsCache = self::getPermissionsCache($storage);
-			$permissionsCache->set($row['id'], $row['user'], $permissions);
-		}
+		$permissionsCache = $this->getPermissionsCache($data['storage_object']);
+		$permissionsCache->set($data['id'], $data['user'], $data['permissions']);
+	}
+
+	/**
+	 * get the new data array from the old one
+	 *
+	 * @param array $data the data from the old cache
+	 * @return array
+	 */
+	function getNewData($data) {
+		$newData = $data;
+		list($storage, $internalPath) = \OC\Files\Filesystem::resolvePath($data['path']);
+		/**
+		 * @var \OC\Files\Storage\Storage $storage
+		 * @var string $internalPath;
+		 */
+		$newData['path_hash'] = md5($internalPath);
+		$newData['path'] = $internalPath;
+		$newData['storage'] = $this->getNumericId($storage);
+		$newData['parent'] = ($internalPath === '') ? -1 : $data['parent'];
+		$newData['permissions'] = ($data['writable']) ? \OCP\PERMISSION_ALL : \OCP\PERMISSION_READ;
+		$newData['storage_object'] = $storage;
+		$newData['mimetype'] = $this->getMimetypeId($newData['mimetype'], $storage);
+		$newData['mimepart'] = $this->getMimetypeId($newData['mimepart'], $storage);
+		return $newData;
 	}
 
 	/**
 	 * @param \OC\Files\Storage\Storage $storage
 	 * @return Permissions
 	 */
-	static function getPermissionsCache($storage) {
+	function getPermissionsCache($storage) {
 		$storageId = $storage->getId();
-		if (!isset(self::$permissionsCaches[$storageId])) {
-			self::$permissionsCaches[$storageId] = $storage->getPermissionsCache();
+		if (!isset($this->permissionsCaches[$storageId])) {
+			$this->permissionsCaches[$storageId] = $storage->getPermissionsCache();
 		}
-		return self::$permissionsCaches[$storageId];
+		return $this->permissionsCaches[$storageId];
 	}
 
 	/**
@@ -73,12 +113,45 @@ class Upgrade {
 	 * @param \OC\Files\Storage\Storage $storage
 	 * @return int
 	 */
-	static function getNumericId($storage) {
+	function getNumericId($storage) {
 		$storageId = $storage->getId();
-		if (!isset(self::$numericIds[$storageId])) {
+		if (!isset($this->numericIds[$storageId])) {
+			$cache = $storage->getCache();
+			$this->numericIds[$storageId] = $cache->getNumericStorageId();
+		}
+		return $this->numericIds[$storageId];
+	}
+
+	/**
+	 * @param string $mimetype
+	 * @param \OC\Files\Storage\Storage $storage
+	 * @return int
+	 */
+	function getMimetypeId($mimetype, $storage) {
+		if (!isset($this->mimeTypeIds[$mimetype])) {
 			$cache = new Cache($storage);
-			self::$numericIds[$storageId] = $cache->getNumericStorageId();
+			$this->mimeTypeIds[$mimetype] = $cache->getMimetypeId($mimetype);
 		}
-		return self::$numericIds[$storageId];
+		return $this->mimeTypeIds[$mimetype];
+	}
+
+	/**
+	 * check if a cache upgrade is required for $user
+	 *
+	 * @param string $user
+	 * @return bool
+	 */
+	static function needUpgrade($user) {
+		$cacheVersion = (int)\OCP\Config::getUserValue($user, 'files', 'cache_version', 4);
+		return $cacheVersion < 5;
+	}
+
+	/**
+	 * mark the filecache as upgrade
+	 *
+	 * @param string $user
+	 */
+	static function upgradeDone($user) {
+		\OCP\Config::setUserValue($user, 'files', 'cache_version', 5);
 	}
 }