amazons3.php 15 KB
Newer Older
1
2
3
<?php

/**
4
5
6
 * ownCloud
 *
 * @author Michael Gapczynski
7
 * @author Christian Berendt
8
 * @copyright 2012 Michael Gapczynski mtgap@owncloud.com
9
 * @copyright 2013 Christian Berendt berendt@b1-systems.de
10
11
12
13
14
15
16
17
18
19
20
21
22
23
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU AFFERO GENERAL PUBLIC LICENSE
 * License as published by the Free Software Foundation; either
 * version 3 of the License, or any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU AFFERO GENERAL PUBLIC LICENSE for more details.
 *
 * You should have received a copy of the GNU Affero General Public
 * License along with this library.  If not, see <http://www.gnu.org/licenses/>.
 */
24

25
namespace OC\Files\Storage;
26

27
set_include_path(get_include_path() . PATH_SEPARATOR .
Robin McCorkell's avatar
Robin McCorkell committed
28
	\OC_App::getAppPath('files_external') . '/3rdparty/aws-sdk-php');
29
require 'aws-autoloader.php';
30
31
32

use Aws\S3\S3Client;
use Aws\S3\Exception\S3Exception;
33

34
class AmazonS3 extends \OC\Files\Storage\Common {
35

Christian Berendt's avatar
Christian Berendt committed
36
37
38
	/**
	 * @var \Aws\S3\S3Client
	 */
39
	private $connection;
Christian Berendt's avatar
Christian Berendt committed
40
41
42
	/**
	 * @var string
	 */
43
	private $bucket;
Christian Berendt's avatar
Christian Berendt committed
44
45
46
	/**
	 * @var array
	 */
47
	private static $tmpFiles = array();
Vincent Petry's avatar
Vincent Petry committed
48
49
50
51
	/**
	 * @var array
	 */
	private $params;
Christian Berendt's avatar
Christian Berendt committed
52
53
54
	/**
	 * @var bool
	 */
55
	private $test = false;
Christian Berendt's avatar
Christian Berendt committed
56
57
58
	/**
	 * @var int
	 */
59
	private $timeout = 15;
Jörn Friedrich Dreyer's avatar
Jörn Friedrich Dreyer committed
60
61
62
63
	/**
	 * @var int in seconds
	 */
	private $rescanDelay = 10;
64

65
66
	/**
	 * @param string $path
67
	 * @return string correctly encoded path
68
	 */
69
	private function normalizePath($path) {
Christian Berendt's avatar
Christian Berendt committed
70
		$path = trim($path, '/');
71

Christian Berendt's avatar
Christian Berendt committed
72
		if (!$path) {
73
74
75
76
77
			$path = '.';
		}

		return $path;
	}
78

79
80
81
	/**
	 * when running the tests wait to let the buckets catch up
	 */
82
83
84
85
86
	private function testTimeout() {
		if ($this->test) {
			sleep($this->timeout);
		}
	}
87

Jörn Friedrich Dreyer's avatar
Jörn Friedrich Dreyer committed
88
89
90
91
	private function isRoot($path) {
		return $path === '.';
	}

92
	private function cleanKey($path) {
Jörn Friedrich Dreyer's avatar
Jörn Friedrich Dreyer committed
93
		if ($this->isRoot($path)) {
94
95
96
97
			return '/';
		}
		return $path;
	}
98

99
	public function __construct($params) {
Jörn Friedrich Dreyer's avatar
Jörn Friedrich Dreyer committed
100
		if (empty($params['key']) || empty($params['secret']) || empty($params['bucket'])) {
Christian Berendt's avatar
Christian Berendt committed
101
			throw new \Exception("Access Key, Secret and Bucket have to be configured.");
102
		}
103

Jörn Friedrich Dreyer's avatar
Jörn Friedrich Dreyer committed
104
105
		$this->id = 'amazon::' . $params['bucket'];
		$this->updateLegacyId($params);
106

107
		$this->bucket = $params['bucket'];
108
		$this->test = isset($params['test']);
109
		$this->timeout = (!isset($params['timeout'])) ? 15 : $params['timeout'];
Jörn Friedrich Dreyer's avatar
Jörn Friedrich Dreyer committed
110
111
112
		$this->rescanDelay = (!isset($params['rescanDelay'])) ? 10 : $params['rescanDelay'];
		$params['region'] = empty($params['region']) ? 'eu-west-1' : $params['region'];
		$params['hostname'] = empty($params['hostname']) ? 's3.amazonaws.com' : $params['hostname'];
113
		if (!isset($params['port']) || $params['port'] === '') {
114
			$params['port'] = ($params['use_ssl'] === 'false') ? 80 : 443;
115
		}
Vincent Petry's avatar
Vincent Petry committed
116
		$this->params = $params;
Jörn Friedrich Dreyer's avatar
Jörn Friedrich Dreyer committed
117
118
119
120
121
122
123
124
125
	}

	/**
	 * Updates old storage ids (v0.2.1 and older) that are based on key and secret to new ones based on the bucket name.
	 * TODO Do this in an update.php. requires iterating over all users and loading the mount.json from their home
	 *
	 * @param array $params
	 */
	public function updateLegacyId (array $params) {
126
127
128
		$oldId = 'amazon::' . $params['key'] . md5($params['secret']);

		// find by old id or bucket
Jörn Friedrich Dreyer's avatar
Jörn Friedrich Dreyer committed
129
		$stmt = \OC::$server->getDatabaseConnection()->prepare(
130
			'SELECT `numeric_id`, `id` FROM `*PREFIX*storages` WHERE `id` IN (?, ?)'
Jörn Friedrich Dreyer's avatar
Jörn Friedrich Dreyer committed
131
		);
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
		$stmt->execute(array($oldId, $this->id));
		while ($row = $stmt->fetch()) {
			$storages[$row['id']] = $row['numeric_id'];
		}

		if (isset($storages[$this->id]) && isset($storages[$oldId])) {
			// if both ids exist, delete the old storage and corresponding filecache entries
			\OC\Files\Cache\Storage::remove($oldId);
		} else if (isset($storages[$oldId])) {
			// if only the old id exists do an update
			$stmt = \OC::$server->getDatabaseConnection()->prepare(
				'UPDATE `*PREFIX*storages` SET `id` = ? WHERE `id` = ?'
			);
			$stmt->execute(array($this->id, $oldId));
		}
		// only the bucket based id may exist, do nothing
Jörn Friedrich Dreyer's avatar
Jörn Friedrich Dreyer committed
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
	}

	/**
	 * Remove a file or folder
	 *
	 * @param string $path
	 * @return bool
	 */
	protected function remove($path) {
		// remember fileType to reduce http calls
		$fileType = $this->filetype($path);
		if ($fileType === 'dir') {
			return $this->rmdir($path);
		} else if ($fileType === 'file') {
			return $this->unlink($path);
		} else {
			return false;
165
		}
166
167
	}

168
	public function mkdir($path) {
169
170
		$path = $this->normalizePath($path);

Christian Berendt's avatar
Christian Berendt committed
171
		if ($this->is_dir($path)) {
172
173
174
175
			return false;
		}

		try {
176
			$this->getConnection()->putObject(array(
177
				'Bucket' => $this->bucket,
178
				'Key' => $path . '/',
179
				'Body' => '',
180
				'ContentType' => 'httpd/unix-directory'
181
			));
182
			$this->testTimeout();
183
		} catch (S3Exception $e) {
Jörn Friedrich Dreyer's avatar
Jörn Friedrich Dreyer committed
184
			\OCP\Util::logException('files_external', $e);
185
186
187
188
189
190
191
			return false;
		}

		return true;
	}

	public function file_exists($path) {
Jörn Friedrich Dreyer's avatar
Jörn Friedrich Dreyer committed
192
		return $this->filetype($path) !== false;
193
194
	}

195

196
	public function rmdir($path) {
197
198
		$path = $this->normalizePath($path);

Jörn Friedrich Dreyer's avatar
Jörn Friedrich Dreyer committed
199
		if ($this->isRoot($path)) {
200
201
202
			return $this->clearBucket();
		}

Christian Berendt's avatar
Christian Berendt committed
203
		if (!$this->file_exists($path)) {
204
205
206
			return false;
		}

207
208
209
210
		return $this->batchDelete($path);
	}

	protected function clearBucket() {
211
		try {
212
			$this->getConnection()->clearBucket($this->bucket);
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
			return true;
			// clearBucket() is not working with Ceph, so if it fails we try the slower approach
		} catch (\Exception $e) {
			return $this->batchDelete();
		}
		return false;
	}

	private function batchDelete ($path = null) {
		$params = array(
			'Bucket' => $this->bucket
		);
		if ($path !== null) {
			$params['Prefix'] = $path . '/';
		}
		try {
			// Since there are no real directories on S3, we need
			// to delete all objects prefixed with the path.
			do {
				// instead of the iterator, manually loop over the list ...
233
				$objects = $this->getConnection()->listObjects($params);
234
				// ... so we can delete the files in batches
235
				$this->getConnection()->deleteObjects(array(
Jörn Friedrich Dreyer's avatar
Jörn Friedrich Dreyer committed
236
237
238
239
					'Bucket' => $this->bucket,
					'Objects' => $objects['Contents']
				));
				$this->testTimeout();
240
				// we reached the end when the list is no longer truncated
Jörn Friedrich Dreyer's avatar
Jörn Friedrich Dreyer committed
241
			} while ($objects['IsTruncated']);
242
		} catch (S3Exception $e) {
Jörn Friedrich Dreyer's avatar
Jörn Friedrich Dreyer committed
243
			\OCP\Util::logException('files_external', $e);
244
			return false;
245
		}
246
		return true;
247
248
249
	}

	public function opendir($path) {
250
251
		$path = $this->normalizePath($path);

Jörn Friedrich Dreyer's avatar
Jörn Friedrich Dreyer committed
252
		if ($this->isRoot($path)) {
253
			$path = '';
Jörn Friedrich Dreyer's avatar
Jörn Friedrich Dreyer committed
254
		} else {
255
			$path .= '/';
256
		}
257
258

		try {
259
			$files = array();
260
			$result = $this->getConnection()->getIterator('ListObjects', array(
261
262
263
264
265
266
267
268
269
270
				'Bucket' => $this->bucket,
				'Delimiter' => '/',
				'Prefix' => $path
			), array('return_prefixes' => true));

			foreach ($result as $object) {
				$file = basename(
					isset($object['Key']) ? $object['Key'] : $object['Prefix']
				);

Christian Berendt's avatar
Christian Berendt committed
271
				if ($file != basename($path)) {
272
					$files[] = $file;
273
274
				}
			}
275

276
			\OC\Files\Stream\Dir::register('amazons3' . $path, $files);
277

278
			return opendir('fakedir://amazons3' . $path);
279
		} catch (S3Exception $e) {
Jörn Friedrich Dreyer's avatar
Jörn Friedrich Dreyer committed
280
			\OCP\Util::logException('files_external', $e);
281
			return false;
282
283
284
285
		}
	}

	public function stat($path) {
286
287
288
289
		$path = $this->normalizePath($path);

		try {
			$stat = array();
Jörn Friedrich Dreyer's avatar
Jörn Friedrich Dreyer committed
290
291
292
293
			if ($this->is_dir($path)) {
				//folders don't really exist
				$stat['size'] = -1; //unknown
				$stat['mtime'] = time() - $this->rescanDelay * 1000;
294
			} else {
295
				$result = $this->getConnection()->headObject(array(
Jörn Friedrich Dreyer's avatar
Jörn Friedrich Dreyer committed
296
297
298
299
300
301
302
303
304
305
					'Bucket' => $this->bucket,
					'Key' => $path
				));

				$stat['size'] = $result['ContentLength'] ? $result['ContentLength'] : 0;
				if ($result['Metadata']['lastmodified']) {
					$stat['mtime'] = strtotime($result['Metadata']['lastmodified']);
				} else {
					$stat['mtime'] = strtotime($result['LastModified']);
				}
306
			}
307
			$stat['atime'] = time();
308

309
			return $stat;
Jörn Friedrich Dreyer's avatar
Jörn Friedrich Dreyer committed
310
311
		} catch(S3Exception $e) {
			\OCP\Util::logException('files_external', $e);
312
			return false;
313
314
315
316
		}
	}

	public function filetype($path) {
317
318
		$path = $this->normalizePath($path);

Jörn Friedrich Dreyer's avatar
Jörn Friedrich Dreyer committed
319
320
321
322
		if ($this->isRoot($path)) {
			return 'dir';
		}

323
		try {
324
			if ($this->getConnection()->doesObjectExist($this->bucket, $path)) {
325
326
				return 'file';
			}
327
			if ($this->getConnection()->doesObjectExist($this->bucket, $path.'/')) {
328
329
330
				return 'dir';
			}
		} catch (S3Exception $e) {
Jörn Friedrich Dreyer's avatar
Jörn Friedrich Dreyer committed
331
			\OCP\Util::logException('files_external', $e);
332
			return false;
333
		}
334

335
336
337
		return false;
	}

338
339
340
	public function unlink($path) {
		$path = $this->normalizePath($path);

341
		if ($this->is_dir($path)) {
342
343
344
			return $this->rmdir($path);
		}

345
		try {
346
			$this->getConnection()->deleteObject(array(
347
				'Bucket' => $this->bucket,
Jörn Friedrich Dreyer's avatar
Jörn Friedrich Dreyer committed
348
				'Key' => $path
349
			));
350
			$this->testTimeout();
351
		} catch (S3Exception $e) {
Jörn Friedrich Dreyer's avatar
Jörn Friedrich Dreyer committed
352
			\OCP\Util::logException('files_external', $e);
353
354
355
356
			return false;
		}

		return true;
357
358
359
	}

	public function fopen($path, $mode) {
360
361
		$path = $this->normalizePath($path);

362
363
364
		switch ($mode) {
			case 'r':
			case 'rb':
365
				$tmpFile = \OC_Helper::tmpFile();
366
367
368
				self::$tmpFiles[$tmpFile] = $path;

				try {
369
					$this->getConnection()->getObject(array(
370
						'Bucket' => $this->bucket,
Jörn Friedrich Dreyer's avatar
Jörn Friedrich Dreyer committed
371
						'Key' => $path,
372
373
374
						'SaveAs' => $tmpFile
					));
				} catch (S3Exception $e) {
Jörn Friedrich Dreyer's avatar
Jörn Friedrich Dreyer committed
375
					\OCP\Util::logException('files_external', $e);
376
					return false;
377
				}
378
379

				return fopen($tmpFile, 'r');
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
			case 'w':
			case 'wb':
			case 'a':
			case 'ab':
			case 'r+':
			case 'w+':
			case 'wb+':
			case 'a+':
			case 'x':
			case 'x+':
			case 'c':
			case 'c+':
				if (strrpos($path, '.') !== false) {
					$ext = substr($path, strrpos($path, '.'));
				} else {
					$ext = '';
				}
397
				$tmpFile = \OC_Helper::tmpFile($ext);
398
				\OC\Files\Stream\Close::registerCallback($tmpFile, array($this, 'writeBack'));
399
400
401
402
				if ($this->file_exists($path)) {
					$source = $this->fopen($path, 'r');
					file_put_contents($tmpFile, $source);
				}
403
404
				self::$tmpFiles[$tmpFile] = $path;

405
				return fopen('close://' . $tmpFile, $mode);
406
407
408
409
410
		}
		return false;
	}

	public function getMimeType($path) {
411
412
413
		$path = $this->normalizePath($path);

		if ($this->is_dir($path)) {
414
			return 'httpd/unix-directory';
415
416
		} else if ($this->file_exists($path)) {
			try {
417
				$result = $this->getConnection()->headObject(array(
418
					'Bucket' => $this->bucket,
Jörn Friedrich Dreyer's avatar
Jörn Friedrich Dreyer committed
419
					'Key' => $path
420
421
				));
			} catch (S3Exception $e) {
Jörn Friedrich Dreyer's avatar
Jörn Friedrich Dreyer committed
422
				\OCP\Util::logException('files_external', $e);
423
				return false;
424
			}
425
426

			return $result['ContentType'];
427
428
429
430
431
		}
		return false;
	}

	public function touch($path, $mtime = null) {
432
433
434
		$path = $this->normalizePath($path);

		$metadata = array();
Christian Berendt's avatar
Christian Berendt committed
435
		if (!is_null($mtime)) {
436
			$metadata = array('lastmodified' => $mtime);
437
		}
438

Jörn Friedrich Dreyer's avatar
Jörn Friedrich Dreyer committed
439
		$fileType = $this->filetype($path);
440
		try {
Jörn Friedrich Dreyer's avatar
Jörn Friedrich Dreyer committed
441
442
			if ($fileType !== false) {
				if ($fileType === 'dir' && ! $this->isRoot($path)) {
443
444
					$path .= '/';
				}
445
				$this->getConnection()->copyObject(array(
446
					'Bucket' => $this->bucket,
447
					'Key' => $this->cleanKey($path),
448
449
450
					'Metadata' => $metadata,
					'CopySource' => $this->bucket . '/' . $path
				));
451
				$this->testTimeout();
452
			} else {
453
				$mimeType = \OC_Helper::getMimetypeDetector()->detectPath($path);
454
				$this->getConnection()->putObject(array(
455
					'Bucket' => $this->bucket,
456
					'Key' => $this->cleanKey($path),
457
					'Metadata' => $metadata,
458
459
					'Body' => '',
					'ContentType' => $mimeType
460
				));
461
				$this->testTimeout();
462
463
			}
		} catch (S3Exception $e) {
Jörn Friedrich Dreyer's avatar
Jörn Friedrich Dreyer committed
464
			\OCP\Util::logException('files_external', $e);
465
466
467
468
469
470
471
472
473
474
475
476
			return false;
		}

		return true;
	}

	public function copy($path1, $path2) {
		$path1 = $this->normalizePath($path1);
		$path2 = $this->normalizePath($path2);

		if ($this->is_file($path1)) {
			try {
477
				$this->getConnection()->copyObject(array(
478
					'Bucket' => $this->bucket,
479
					'Key' => $this->cleanKey($path2),
480
					'CopySource' => S3Client::encodeKey($this->bucket . '/' . $path1)
481
				));
482
				$this->testTimeout();
483
			} catch (S3Exception $e) {
Jörn Friedrich Dreyer's avatar
Jörn Friedrich Dreyer committed
484
				\OCP\Util::logException('files_external', $e);
485
486
487
				return false;
			}
		} else {
Jörn Friedrich Dreyer's avatar
Jörn Friedrich Dreyer committed
488
			$this->remove($path2);
489
490

			try {
491
				$this->getConnection()->copyObject(array(
492
493
					'Bucket' => $this->bucket,
					'Key' => $path2 . '/',
494
					'CopySource' => S3Client::encodeKey($this->bucket . '/' . $path1 . '/')
495
				));
496
				$this->testTimeout();
497
			} catch (S3Exception $e) {
Jörn Friedrich Dreyer's avatar
Jörn Friedrich Dreyer committed
498
				\OCP\Util::logException('files_external', $e);
499
500
501
502
				return false;
			}

			$dh = $this->opendir($path1);
503
			if (is_resource($dh)) {
504
505
506
507
508
509
510
511
				while (($file = readdir($dh)) !== false) {
					if ($file === '.' || $file === '..') {
						continue;
					}

					$source = $path1 . '/' . $file;
					$target = $path2 . '/' . $file;
					$this->copy($source, $target);
512
				}
513
			}
514
515
516
517
518
519
520
521
522
523
		}

		return true;
	}

	public function rename($path1, $path2) {
		$path1 = $this->normalizePath($path1);
		$path2 = $this->normalizePath($path2);

		if ($this->is_file($path1)) {
524

Christian Berendt's avatar
Christian Berendt committed
525
			if ($this->copy($path1, $path2) === false) {
526
527
528
				return false;
			}

Christian Berendt's avatar
Christian Berendt committed
529
			if ($this->unlink($path1) === false) {
530
531
532
533
534
				$this->unlink($path2);
				return false;
			}
		} else {

Christian Berendt's avatar
Christian Berendt committed
535
			if ($this->copy($path1, $path2) === false) {
536
537
538
				return false;
			}

Christian Berendt's avatar
Christian Berendt committed
539
			if ($this->rmdir($path1) === false) {
540
541
542
				$this->rmdir($path2);
				return false;
			}
543
		}
544
545

		return true;
546
547
	}

548
	public function test() {
549
		$test = $this->getConnection()->getBucketAcl(array(
Vincent Petry's avatar
Vincent Petry committed
550
551
552
			'Bucket' => $this->bucket,
		));
		if (isset($test) && !is_null($test->getPath('Owner/ID'))) {
553
554
555
556
557
			return true;
		}
		return false;
	}

558
559
560
561
	public function getId() {
		return $this->id;
	}

562
563
564
565
566
567
	/**
	 * Returns the connection
	 *
	 * @return S3Client connected client
	 * @throws \Exception if connection could not be made
	 */
568
	public function getConnection() {
569
570
571
572
		if (!is_null($this->connection)) {
			return $this->connection;
		}

Vincent Petry's avatar
Vincent Petry committed
573
574
575
		$scheme = ($this->params['use_ssl'] === 'false') ? 'http' : 'https';
		$base_url = $scheme . '://' . $this->params['hostname'] . ':' . $this->params['port'] . '/';

576
		$this->connection = S3Client::factory(array(
Vincent Petry's avatar
Vincent Petry committed
577
578
			'key' => $this->params['key'],
			'secret' => $this->params['secret'],
579
			'base_url' => $base_url,
Vincent Petry's avatar
Vincent Petry committed
580
			'region' => $this->params['region']
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
		));

		if (!$this->connection->isValidBucketName($this->bucket)) {
			throw new \Exception("The configured bucket name is invalid.");
		}

		if (!$this->connection->doesBucketExist($this->bucket)) {
			try {
				$this->connection->createBucket(array(
					'Bucket' => $this->bucket
				));
				$this->connection->waitUntilBucketExists(array(
					'Bucket' => $this->bucket,
					'waiter.interval' => 1,
					'waiter.max_attempts' => 15
				));
				$this->testTimeout();
			} catch (S3Exception $e) {
				\OCP\Util::logException('files_external', $e);
				throw new \Exception('Creation of bucket failed. '.$e->getMessage());
			}
		}

604
605
606
607
		return $this->connection;
	}

	public function writeBack($tmpFile) {
Christian Berendt's avatar
Christian Berendt committed
608
		if (!isset(self::$tmpFiles[$tmpFile])) {
609
610
611
612
			return false;
		}

		try {
613
			$this->getConnection()->putObject(array(
614
				'Bucket' => $this->bucket,
615
				'Key' => $this->cleanKey(self::$tmpFiles[$tmpFile]),
616
617
618
619
				'SourceFile' => $tmpFile,
				'ContentType' => \OC_Helper::getMimeType($tmpFile),
				'ContentLength' => filesize($tmpFile)
			));
620
			$this->testTimeout();
621
622
623

			unlink($tmpFile);
		} catch (S3Exception $e) {
Jörn Friedrich Dreyer's avatar
Jörn Friedrich Dreyer committed
624
			\OCP\Util::logException('files_external', $e);
625
626
627
			return false;
		}
	}
628
629
630
631
632
633
634
635
636
637
638
639

	/**
	 * check if curl is installed
	 */
	public static function checkDependencies() {
		if (function_exists('curl_init')) {
			return true;
		} else {
			return array('curl');
		}
	}

640
}