Merge pull request #14389 from annando/expire-media

Delete unused media attachments

database.sql

@@ -1,6 +1,6 @@
 -- ------------------------------------------
 -- Friendica 2024.09-dev (Yellow Archangel)
--- DB_UPDATE_VERSION 1572
+-- DB_UPDATE_VERSION 1573
 -- ------------------------------------------
 
 
@@ -1424,6 +1424,7 @@ CREATE TABLE IF NOT EXISTS `post-media` (
 	`uri-id` int unsigned NOT NULL COMMENT 'Id of the item-uri table entry that contains the item uri',
 	`url` varbinary(1024) NOT NULL COMMENT 'Media URL',
 	`media-uri-id` int unsigned COMMENT 'Id of the item-uri table entry that contains the activities uri-id',
+	`attach-id` int unsigned COMMENT 'In case of a local attachment, this field is filled with the id in the attach table',
 	`type` tinyint unsigned NOT NULL DEFAULT 0 COMMENT 'Media type',
 	`mimetype` varchar(60) COMMENT '',
 	`height` smallint unsigned COMMENT 'Height of the media',
@@ -1445,8 +1446,10 @@ CREATE TABLE IF NOT EXISTS `post-media` (
 	 UNIQUE INDEX `uri-id-url` (`uri-id`,`url`(512)),
 	 INDEX `uri-id-id` (`uri-id`,`id`),
 	 INDEX `media-uri-id` (`media-uri-id`),
+	 INDEX `attach-id` (`attach-id`),
 	FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE,
-	FOREIGN KEY (`media-uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE
+	FOREIGN KEY (`media-uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE,
+	FOREIGN KEY (`attach-id`) REFERENCES `attach` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE
 ) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Attached media';
 
 --

doc/database/db_post-media.md

@@ -7,11 +7,12 @@ Fields
 ------
 
 | Field           | Description                                                                         | Type              | Null | Key | Default | Extra          |
-| --------------- | ------------------------------------------------------------------ | ----------------- | ---- | --- | ------- | -------------- |
+| --------------- | ----------------------------------------------------------------------------------- | ----------------- | ---- | --- | ------- | -------------- |
 | id              | sequential ID                                                                       | int unsigned      | NO   | PRI | NULL    | auto_increment |
 | uri-id          | Id of the item-uri table entry that contains the item uri                           | int unsigned      | NO   |     | NULL    |                |
 | url             | Media URL                                                                           | varbinary(1024)   | NO   |     | NULL    |                |
 | media-uri-id    | Id of the item-uri table entry that contains the activities uri-id                  | int unsigned      | YES  |     | NULL    |                |
+| attach-id       | In case of a local attachment, this field is filled with the id in the attach table | int unsigned      | YES  |     | NULL    |                |
 | type            | Media type                                                                          | tinyint unsigned  | NO   |     | 0       |                |
 | mimetype        |                                                                                     | varchar(60)       | YES  |     | NULL    |                |
 | height          | Height of the media                                                                 | smallint unsigned | YES  |     | NULL    |                |
@@ -39,6 +40,7 @@ Indexes
 | uri-id-url   | UNIQUE, uri-id, url(512) |
 | uri-id-id    | uri-id, id               |
 | media-uri-id | media-uri-id             |
+| attach-id    | attach-id                |
 
 Foreign Keys
 ------------
@@ -47,5 +49,6 @@ Foreign Keys
 |-------|--------------|--------------|
 | uri-id | [item-uri](help/database/db_item-uri) | id |
 | media-uri-id | [item-uri](help/database/db_item-uri) | id |
+| attach-id | [attach](help/database/db_attach) | id |
 
 Return to [database documentation](help/database)

src/Model/Item.php

@@ -380,14 +380,6 @@ class Item
 			Event::delete($item['event-id']);
 		}
 
-		// If item has attachments, drop them
-		$attachments = Post\Media::getByURIId($item['uri-id'], [Post\Media::DOCUMENT]);
-		foreach ($attachments as $attachment) {
-			if (preg_match('|attach/(\d+)|', $attachment['url'], $matches)) {
-				Attach::delete(['id' => $matches[1], 'uid' => $item['uid']]);
-			}
-		}
-
 		// Set the item to "deleted"
 		$item_fields = ['deleted' => true, 'edited' => DateTimeFormat::utcNow(), 'changed' => DateTimeFormat::utcNow()];
 		Post::update($item_fields, ['id' => $item['id']]);

src/Model/Post/Media.php

@@ -384,8 +384,9 @@ class Media
 	private static function fetchLocalData(array $media): array
 	{
 		if (preg_match('|.*?/attach/(\d+)|', $media['url'], $matches)) {
-			$attachment = Attach::selectFirst(['filename', 'filetype', 'filesize'], ['id' => $matches[1]]);
+			$attachment = Attach::selectFirst(['id', 'filename', 'filetype', 'filesize'], ['id' => $matches[1]]);
 			if (!empty($attachment)) {
+				$media['attach-id'] = $attachment['id'];
 				$media['name']      = $attachment['filename'];
 				$media['mimetype']  = $attachment['filetype'];
 				$media['size']      = $attachment['filesize'];
@@ -396,7 +397,7 @@ class Media
 		if (!preg_match('|.*?/photo/(.*[a-fA-F0-9])\-(.*[0-9])\..*[\w]|', $media['url'], $matches)) {
 			return $media;
 		}
-		$photo = Photo::selectFirst([], ['resource-id' => $matches[1], 'scale' => $matches[2]]);
+		$photo = Photo::selectFirst(['type', 'datasize', 'width', 'height', 'blurhash'], ['resource-id' => $matches[1], 'scale' => $matches[2]]);
 		if (!empty($photo)) {
 			$media['mimetype'] = $photo['type'];
 			$media['size'] = $photo['datasize'];
@@ -408,7 +409,7 @@ class Media
 		if (!preg_match('|.*?/photo/(.*[a-fA-F0-9])\-(.*[0-9])\..*[\w]|', $media['preview'] ?? '', $matches)) {
 			return $media;
 		}
-		$photo = Photo::selectFirst([], ['resource-id' => $matches[1], 'scale' => $matches[2]]);
+		$photo = Photo::selectFirst(['width', 'height'], ['resource-id' => $matches[1], 'scale' => $matches[2]]);
 		if (!empty($photo)) {
 			$media['preview-width'] = $photo['width'];
 			$media['preview-height'] = $photo['height'];

src/Module/Admin/Site.php

@@ -53,6 +53,7 @@ class Site extends BaseAdmin
 		$language         = (!empty($_POST['language'])         ? trim($_POST['language'])      : '');
 		$theme            = (!empty($_POST['theme'])            ? trim($_POST['theme'])         : '');
 		$theme_mobile     = (!empty($_POST['theme_mobile'])     ? trim($_POST['theme_mobile'])  : '');
+		$maxfilesize      = (!empty($_POST['maxfilesize'])      ? trim($_POST['maxfilesize'])                        : 0);
 		$maximagesize     = (!empty($_POST['maximagesize'])     ? trim($_POST['maximagesize'])                       : 0);
 		$maximagelength   = (!empty($_POST['maximagelength'])   ? intval(trim($_POST['maximagelength']))             : -1);
 		$jpegimagequality = (!empty($_POST['jpegimagequality']) ? intval(trim($_POST['jpegimagequality']))           : 100);
@@ -224,6 +225,11 @@ class Site extends BaseAdmin
 		} else {
 			$transactionConfig->set('system', 'singleuser', $singleuser);
 		}
+		if (preg_match('/\d+(?:\s*[kmg])?/i', $maxfilesize)) {
+			$transactionConfig->set('system', 'maxfilesize', $maxfilesize);
+		} else {
+			DI::sysmsg()->addNotice(DI::l10n()->t('%s is no valid input for maximum media size', $maxfilesize));
+		}
 		if (preg_match('/\d+(?:\s*[kmg])?/i', $maximagesize)) {
 			$transactionConfig->set('system', 'maximagesize', $maximagesize);
 		} else {
@@ -484,6 +490,10 @@ class Site extends BaseAdmin
 													'', 'pattern="\d+(?:\s*[kmg])?"'],
 			'$maximagelength'   => ['maximagelength', DI::l10n()->t('Maximum image length'), DI::config()->get('system', 'max_image_length'), DI::l10n()->t('Maximum length in pixels of the longest side of uploaded images. Default is -1, which means no limits.')],
 			'$jpegimagequality' => ['jpegimagequality', DI::l10n()->t('JPEG image quality'), DI::config()->get('system', 'jpeg_quality'), DI::l10n()->t('Uploaded JPEGS will be saved at this quality setting [0-100]. Default is 100, which is full quality.')],
+			'$maxfilesize'      => ['maxfilesize', DI::l10n()->t('Maximum media file size'), DI::config()->get('system', 'maxfilesize'), DI::l10n()->t('Maximum size in bytes of uploaded media files. Default is 0, which means no limits. You can put k, m, or g behind the desired value for KiB, MiB, GiB, respectively.
+													The value of <code>upload_max_filesize</code> in your <code>PHP.ini</code> needs be set to at least the desired limit.
+													Currently <code>upload_max_filesize</code> is set to %s (%s byte)', Strings::formatBytes(Strings::getBytesFromShorthand(ini_get('upload_max_filesize'))), Strings::getBytesFromShorthand(ini_get('upload_max_filesize'))),
+													'', 'pattern="\d+(?:\s*[kmg])?"'],
 
 			'$register_policy'        => ['register_policy', DI::l10n()->t('Register policy'), DI::config()->get('config', 'register_policy'), '', $register_choices],
 			'$max_registered_users'   => ['max_registered_users', DI::l10n()->t('Maximum Users'), DI::config()->get('config', 'max_registered_users'), DI::l10n()->t('If defined, the register policy is automatically closed when the given number of users is reached and reopens the registry when the number drops below the limit. It only works when the policy is set to open or close, but not when the policy is set to approval.')],

src/Module/Api/Mastodon/Instance.php

@@ -79,9 +79,14 @@ class Instance extends BaseApi
 			$image_matrix_limit = 33177600; // 5760^2
 		}
 
+		$media_size_limit = Strings::getBytesFromShorthand($this->config->get('system', 'maxfilesize'));
+		if (empty($media_size_limit)) {
+			$media_size_limit = Strings::getBytesFromShorthand(ini_get('upload_max_filesize'));
+		}
+
 		return new InstanceV2Entity\Configuration(
 			$statuses_config,
-			new InstanceV2Entity\MediaAttachmentsConfig(Images::supportedMimeTypes(), $image_size_limit, $image_matrix_limit),
+			new InstanceV2Entity\MediaAttachmentsConfig(Images::supportedMimeTypes(), $image_size_limit, $image_matrix_limit, $media_size_limit),
 			new InstanceV2Entity\Polls(),
 			new InstanceV2Entity\Accounts(),
 		);

src/Module/Api/Mastodon/InstanceV2.php

@@ -115,9 +115,14 @@ class InstanceV2 extends BaseApi
 			$image_matrix_limit = 33177600; // 5760^2
 		}
 
+		$media_size_limit = Strings::getBytesFromShorthand($this->config->get('system', 'maxfilesize'));
+		if (empty($media_size_limit)) {
+			$media_size_limit = Strings::getBytesFromShorthand(ini_get('upload_max_filesize'));
+		}
+
 		return new InstanceEntity\Configuration(
 			$statuses_config,
-			new InstanceEntity\MediaAttachmentsConfig($this->supportedMimeTypes(), $image_size_limit, $image_matrix_limit),
+			new InstanceEntity\MediaAttachmentsConfig($this->supportedMimeTypes(), $image_size_limit, $image_matrix_limit, $media_size_limit),
 			new InstanceEntity\Polls(),
 			new InstanceEntity\Accounts(),
 		);

src/Module/Api/Mastodon/Media.php

@@ -14,6 +14,7 @@ use Friendica\Model\Contact;
 use Friendica\Model\Photo;
 use Friendica\Model\Post;
 use Friendica\Module\BaseApi;
+use Friendica\Util\Strings;
 
 /**
  * @see https://docs.joinmastodon.org/methods/statuses/media/
@@ -26,7 +27,7 @@ class Media extends BaseApi
 		$uid = self::getCurrentUserID();
 
 		$request = $this->getRequest([
-			'file'        => [], // The file to be attached, using multipart form data.
+			'file'        => $_FILES['file'] ?? [], // The file to be attached, using multipart form data.
 			'thumbnail'   => [], // The custom thumbnail of the media to be attached, using multipart form data.
 			'description' => '', // A plain-text description of the media, for accessibility purposes.
 			'focus'       => '', // Two floating points (x,y), comma-delimited ranging from -1.0 to 1.0
@@ -34,14 +35,15 @@ class Media extends BaseApi
 
 		Logger::info('Photo post', ['request' => $request, 'files' => $_FILES]);
 
-		if (empty($_FILES['file'])) {
+		if (empty($request['file'])) {
+			Logger::notice('Upload is invalid', ['request' => $request]);
 			$this->logAndJsonError(422, $this->errorFactory->UnprocessableEntity());
 		}
 
-		$type = Post\Media::getType($_FILES['file']['type']);
+		$type = Post\Media::getType($request['file']['type']);
 
 		if (in_array($type, [Post\Media::IMAGE, Post\Media::UNKNOWN])) {
-			$media = Photo::upload($uid, $_FILES['file'], '', null, null, '', '', $request['description']);
+			$media = Photo::upload($uid, $request['file'], '', null, null, '', '', $request['description']);
 			if (empty($media)) {
 				$this->logAndJsonError(422, $this->errorFactory->UnprocessableEntity());
 			}
@@ -50,22 +52,24 @@ class Media extends BaseApi
 
 			$this->jsonExit(DI::mstdnAttachment()->createFromPhoto($media['id']));
 		} else {
-			$tempFileName = $_FILES['file']['tmp_name'];
+			$tempFileName = $request['file']['tmp_name'];
-			$fileName     = basename($_FILES['file']['name']);
+			$fileName     = basename($request['file']['name']);
-			$fileSize     = intval($_FILES['file']['size']);
+			$fileSize     = intval($request['file']['size']);
-			$maxFileSize  = DI::config()->get('system', 'maxfilesize');
+			$maxFileSize  = Strings::getBytesFromShorthand(DI::config()->get('system', 'maxfilesize'));
 
 			if ($fileSize <= 0) {
+				Logger::notice('Filesize is invalid', ['size' => $fileSize, 'request' => $request]);
 				@unlink($tempFileName);
 				$this->logAndJsonError(422, $this->errorFactory->UnprocessableEntity());
 			}
 
 			if ($maxFileSize && $fileSize > $maxFileSize) {
+				Logger::notice('Filesize is too large', ['size' => $fileSize, 'max' => $maxFileSize, 'request' => $request]);
 				@unlink($tempFileName);
 				$this->logAndJsonError(422, $this->errorFactory->UnprocessableEntity());
 			}
 
-			$id = Attach::storeFile($tempFileName, self::getCurrentUserID(), $fileName, $_FILES['file']['type'], '<' . Contact::getPublicIdByUserId(self::getCurrentUserID()) . '>');
+			$id = Attach::storeFile($tempFileName, self::getCurrentUserID(), $fileName, $request['file']['type'], '<' . Contact::getPublicIdByUserId(self::getCurrentUserID()) . '>');
 			@unlink($tempFileName);
 			Logger::info('Uploaded media', ['id' => $id]);
 			$this->jsonExit(DI::mstdnAttachment()->createFromAttach($id));

src/Module/Media/Attachment/Upload.php

@@ -71,7 +71,7 @@ class Upload extends \Friendica\BaseModule
 		$tempFileName = $_FILES['userfile']['tmp_name'];
 		$fileName     = basename($_FILES['userfile']['name']);
 		$fileSize     = intval($_FILES['userfile']['size']);
-		$maxFileSize  = $this->config->get('system', 'maxfilesize');
+		$maxFileSize  = Strings::getBytesFromShorthand($this->config->get('system', 'maxfilesize'));
 
 		/*
 		 * Found html code written in text field of form, when trying to upload a

src/Object/Api/Mastodon/InstanceV2/MediaAttachmentsConfig.php

@@ -32,12 +32,12 @@ class MediaAttachmentsConfig extends BaseDataTransferObject
 	/**
 	 * @param array $supported_mime_types
 	 */
-	public function __construct(array $supported_mime_types, int $image_size_limit, int $image_matrix_limit)
+	public function __construct(array $supported_mime_types, int $image_size_limit, int $image_matrix_limit, int $media_size_limit)
 	{
 		$this->supported_mime_types = $supported_mime_types;
 		$this->image_size_limit     = $image_size_limit;
 		$this->image_matrix_limit   = $image_matrix_limit;
-		$this->video_size_limit     = $image_size_limit;
+		$this->video_size_limit     = $media_size_limit;
 		$this->video_matrix_limit   = $image_matrix_limit;
 	}
 }

src/Protocol/ActivityPub/Processor.php

@@ -979,10 +979,12 @@ class Processor
 			}
 		}
 
+		if (!empty($item['author-id'])) {
 			$author = Contact::selectFirstAccount(['ap-posting-restricted'], ['id' => $item['author-id']]);
 			if (!empty($author['ap-posting-restricted'])) {
 				$item['restrictions'] = $item['restrictions'] | Item::CANT_REPLY;
 			}
+		}
 
 		$item['location'] = $activity['location'];
 

src/Worker/ExpirePosts.php

@@ -13,6 +13,7 @@ use Friendica\Database\Database;
 use Friendica\Database\DBA;
 use Friendica\Database\DBStructure;
 use Friendica\DI;
+use Friendica\Model\Attach;
 use Friendica\Model\Item;
 use Friendica\Model\Post;
 use Friendica\Util\DateTimeFormat;
@@ -43,6 +44,8 @@ class ExpirePosts
 			self::addMissingEntries();
 		}
 
+		self::deleteUnusedAttachments();
+
 		DBA::releaseOptimizeLock();
 
 		// Set the expiry for origin posts
@@ -304,4 +307,17 @@ class ExpirePosts
 			} while ($affected_count);
 		}
 	}
+
+	/**
+	 * Delete media attachments (excluding photos) that aren't linked to any post
+	 *
+	 * @return void
+	 */
+	private static function deleteUnusedAttachments()
+	{
+		$postmedia = DBA::select('attach', ['id'], ["`id` NOT IN (SELECT `attach-id` FROM `post-media`)"]);
+		while ($media = DBA::fetch($postmedia)) {
+			Attach::delete(['id' => $media['id']]);
+		}
+	}
 }

static/dbstructure.config.php

@@ -44,7 +44,7 @@ use Friendica\Database\DBA;
 
 // This file is required several times during the test in DbaDefinition which justifies this condition
 if (!defined('DB_UPDATE_VERSION')) {
-	define('DB_UPDATE_VERSION', 1572);
+	define('DB_UPDATE_VERSION', 1573);
 }
 
 return [
@@ -1431,6 +1431,7 @@ return [
 			"uri-id" => ["type" => "int unsigned", "not null" => "1", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"],
 			"url" => ["type" => "varbinary(1024)", "not null" => "1", "comment" => "Media URL"],
 			"media-uri-id" => ["type" => "int unsigned", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the activities uri-id"],
+			"attach-id" => ["type" => "int unsigned", "foreign" => ["attach" => "id"], "comment" => "In case of a local attachment, this field is filled with the id in the attach table"],
 			"type" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "comment" => "Media type"],
 			"mimetype" => ["type" => "varchar(60)", "comment" => ""],
 			"height" => ["type" => "smallint unsigned", "comment" => "Height of the media"],
@@ -1454,6 +1455,7 @@ return [
 			"uri-id-url" => ["UNIQUE", "uri-id", "url(512)"],
 			"uri-id-id" => ["uri-id", "id"],
 			"media-uri-id" => ["media-uri-id"],
+			"attach-id" => ["attach-id"],
 		]
 	],
 	"post-origin" => [

update.php

@@ -40,6 +40,7 @@ use Friendica\Database\Database;
 use Friendica\Database\DBA;
 use Friendica\Database\DBStructure;
 use Friendica\DI;
+use Friendica\Model\Attach;
 use Friendica\Model\Contact;
 use Friendica\Model\Item;
 use Friendica\Model\ItemURI;
@@ -1502,3 +1503,27 @@ function update_1571()
 
 	return Update::SUCCESS;
 }
+
+function update_1573()
+{
+	$postmedia = DBA::select('post-media', ['id', 'url'], ["`url` LIKE ?", '%/attach/%']);
+	while ($media = DBA::fetch($postmedia)) {
+		if (!DI::baseUrl()->isLocalUrl($media['url'])) {
+			continue;
+		}
+		if (preg_match('|.*?/attach/(\d+)|', $media['url'], $matches)) {
+			$attachment = Attach::selectFirst(['id', 'filename', 'filetype', 'filesize'], ['id' => $matches[1]]);
+			if (!empty($attachment)) {
+				$fields = [
+					'attach-id' => $attachment['id'],
+					'name'      => $attachment['filename'],
+					'mimetype'  => $attachment['filetype'],
+					'size'      => $attachment['filesize'],
+				];
+				DBA::update('post-media', $fields, ['id' => $media['id']]);
+			}
+		}
+	}
+	DBA::close($media);
+	return Update::SUCCESS;
+}

view/templates/admin/site.tpl

@@ -47,6 +47,7 @@
 		{{include file="field_input.tpl" field=$maximagesize}}
 		{{include file="field_input.tpl" field=$maximagelength}}
 		{{include file="field_input.tpl" field=$jpegimagequality}}
+		{{include file="field_input.tpl" field=$maxfilesize}}
 		<div class="submit"><input type="submit" name="page_site" value="{{$submit}}"/></div>
 
 		<h2>{{$corporate}}</h2>

view/theme/frio/templates/admin/site.tpl

@@ -106,6 +106,7 @@
 						{{include file="field_input.tpl" field=$maximagesize}}
 						{{include file="field_input.tpl" field=$maximagelength}}
 						{{include file="field_input.tpl" field=$jpegimagequality}}
+						{{include file="field_input.tpl" field=$maxfilesize}}
 					</div>
 					<div class="panel-footer">
 						<input type="submit" name="page_site" class="btn btn-primary" value="{{$submit}}"/>