Merge pull request #14389 from annando/expire-media

Delete unused media attachments
pull/14394/head
Tobias Diekershoff 2024-08-24 19:51:32 +02:00 committed by GitHub
commit 1b3aa74b87
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
17 changed files with 459 additions and 372 deletions

View File

@ -1,6 +1,6 @@
-- ------------------------------------------
-- Friendica 2024.09-dev (Yellow Archangel)
-- DB_UPDATE_VERSION 1572
-- DB_UPDATE_VERSION 1573
-- ------------------------------------------
@ -1424,6 +1424,7 @@ CREATE TABLE IF NOT EXISTS `post-media` (
`uri-id` int unsigned NOT NULL COMMENT 'Id of the item-uri table entry that contains the item uri',
`url` varbinary(1024) NOT NULL COMMENT 'Media URL',
`media-uri-id` int unsigned COMMENT 'Id of the item-uri table entry that contains the activities uri-id',
`attach-id` int unsigned COMMENT 'In case of a local attachment, this field is filled with the id in the attach table',
`type` tinyint unsigned NOT NULL DEFAULT 0 COMMENT 'Media type',
`mimetype` varchar(60) COMMENT '',
`height` smallint unsigned COMMENT 'Height of the media',
@ -1445,8 +1446,10 @@ CREATE TABLE IF NOT EXISTS `post-media` (
UNIQUE INDEX `uri-id-url` (`uri-id`,`url`(512)),
INDEX `uri-id-id` (`uri-id`,`id`),
INDEX `media-uri-id` (`media-uri-id`),
INDEX `attach-id` (`attach-id`),
FOREIGN KEY (`uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE,
FOREIGN KEY (`media-uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE
FOREIGN KEY (`media-uri-id`) REFERENCES `item-uri` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE,
FOREIGN KEY (`attach-id`) REFERENCES `attach` (`id`) ON UPDATE RESTRICT ON DELETE CASCADE
) DEFAULT COLLATE utf8mb4_general_ci COMMENT='Attached media';
--

View File

@ -6,29 +6,30 @@ Attached media
Fields
------
| Field | Description | Type | Null | Key | Default | Extra |
| --------------- | ------------------------------------------------------------------ | ----------------- | ---- | --- | ------- | -------------- |
| id | sequential ID | int unsigned | NO | PRI | NULL | auto_increment |
| uri-id | Id of the item-uri table entry that contains the item uri | int unsigned | NO | | NULL | |
| url | Media URL | varbinary(1024) | NO | | NULL | |
| media-uri-id | Id of the item-uri table entry that contains the activities uri-id | int unsigned | YES | | NULL | |
| type | Media type | tinyint unsigned | NO | | 0 | |
| mimetype | | varchar(60) | YES | | NULL | |
| height | Height of the media | smallint unsigned | YES | | NULL | |
| width | Width of the media | smallint unsigned | YES | | NULL | |
| size | Media size | bigint unsigned | YES | | NULL | |
| blurhash | BlurHash representation of the image | varbinary(255) | YES | | NULL | |
| preview | Preview URL | varbinary(512) | YES | | NULL | |
| preview-height | Height of the preview picture | smallint unsigned | YES | | NULL | |
| preview-width | Width of the preview picture | smallint unsigned | YES | | NULL | |
| description | | text | YES | | NULL | |
| name | Name of the media | varchar(255) | YES | | NULL | |
| author-url | URL of the author of the media | varbinary(383) | YES | | NULL | |
| author-name | Name of the author of the media | varchar(255) | YES | | NULL | |
| author-image | Image of the author of the media | varbinary(383) | YES | | NULL | |
| publisher-url | URL of the publisher of the media | varbinary(383) | YES | | NULL | |
| publisher-name | Name of the publisher of the media | varchar(255) | YES | | NULL | |
| publisher-image | Image of the publisher of the media | varbinary(383) | YES | | NULL | |
| Field | Description | Type | Null | Key | Default | Extra |
| --------------- | ----------------------------------------------------------------------------------- | ----------------- | ---- | --- | ------- | -------------- |
| id | sequential ID | int unsigned | NO | PRI | NULL | auto_increment |
| uri-id | Id of the item-uri table entry that contains the item uri | int unsigned | NO | | NULL | |
| url | Media URL | varbinary(1024) | NO | | NULL | |
| media-uri-id | Id of the item-uri table entry that contains the activities uri-id | int unsigned | YES | | NULL | |
| attach-id | In case of a local attachment, this field is filled with the id in the attach table | int unsigned | YES | | NULL | |
| type | Media type | tinyint unsigned | NO | | 0 | |
| mimetype | | varchar(60) | YES | | NULL | |
| height | Height of the media | smallint unsigned | YES | | NULL | |
| width | Width of the media | smallint unsigned | YES | | NULL | |
| size | Media size | bigint unsigned | YES | | NULL | |
| blurhash | BlurHash representation of the image | varbinary(255) | YES | | NULL | |
| preview | Preview URL | varbinary(512) | YES | | NULL | |
| preview-height | Height of the preview picture | smallint unsigned | YES | | NULL | |
| preview-width | Width of the preview picture | smallint unsigned | YES | | NULL | |
| description | | text | YES | | NULL | |
| name | Name of the media | varchar(255) | YES | | NULL | |
| author-url | URL of the author of the media | varbinary(383) | YES | | NULL | |
| author-name | Name of the author of the media | varchar(255) | YES | | NULL | |
| author-image | Image of the author of the media | varbinary(383) | YES | | NULL | |
| publisher-url | URL of the publisher of the media | varbinary(383) | YES | | NULL | |
| publisher-name | Name of the publisher of the media | varchar(255) | YES | | NULL | |
| publisher-image | Image of the publisher of the media | varbinary(383) | YES | | NULL | |
Indexes
------------
@ -39,6 +40,7 @@ Indexes
| uri-id-url | UNIQUE, uri-id, url(512) |
| uri-id-id | uri-id, id |
| media-uri-id | media-uri-id |
| attach-id | attach-id |
Foreign Keys
------------
@ -47,5 +49,6 @@ Foreign Keys
|-------|--------------|--------------|
| uri-id | [item-uri](help/database/db_item-uri) | id |
| media-uri-id | [item-uri](help/database/db_item-uri) | id |
| attach-id | [attach](help/database/db_attach) | id |
Return to [database documentation](help/database)

View File

@ -380,14 +380,6 @@ class Item
Event::delete($item['event-id']);
}
// If item has attachments, drop them
$attachments = Post\Media::getByURIId($item['uri-id'], [Post\Media::DOCUMENT]);
foreach ($attachments as $attachment) {
if (preg_match('|attach/(\d+)|', $attachment['url'], $matches)) {
Attach::delete(['id' => $matches[1], 'uid' => $item['uid']]);
}
}
// Set the item to "deleted"
$item_fields = ['deleted' => true, 'edited' => DateTimeFormat::utcNow(), 'changed' => DateTimeFormat::utcNow()];
Post::update($item_fields, ['id' => $item['id']]);

View File

@ -384,11 +384,12 @@ class Media
private static function fetchLocalData(array $media): array
{
if (preg_match('|.*?/attach/(\d+)|', $media['url'], $matches)) {
$attachment = Attach::selectFirst(['filename', 'filetype', 'filesize'], ['id' => $matches[1]]);
$attachment = Attach::selectFirst(['id', 'filename', 'filetype', 'filesize'], ['id' => $matches[1]]);
if (!empty($attachment)) {
$media['name'] = $attachment['filename'];
$media['mimetype'] = $attachment['filetype'];
$media['size'] = $attachment['filesize'];
$media['attach-id'] = $attachment['id'];
$media['name'] = $attachment['filename'];
$media['mimetype'] = $attachment['filetype'];
$media['size'] = $attachment['filesize'];
}
return $media;
}
@ -396,7 +397,7 @@ class Media
if (!preg_match('|.*?/photo/(.*[a-fA-F0-9])\-(.*[0-9])\..*[\w]|', $media['url'], $matches)) {
return $media;
}
$photo = Photo::selectFirst([], ['resource-id' => $matches[1], 'scale' => $matches[2]]);
$photo = Photo::selectFirst(['type', 'datasize', 'width', 'height', 'blurhash'], ['resource-id' => $matches[1], 'scale' => $matches[2]]);
if (!empty($photo)) {
$media['mimetype'] = $photo['type'];
$media['size'] = $photo['datasize'];
@ -408,7 +409,7 @@ class Media
if (!preg_match('|.*?/photo/(.*[a-fA-F0-9])\-(.*[0-9])\..*[\w]|', $media['preview'] ?? '', $matches)) {
return $media;
}
$photo = Photo::selectFirst([], ['resource-id' => $matches[1], 'scale' => $matches[2]]);
$photo = Photo::selectFirst(['width', 'height'], ['resource-id' => $matches[1], 'scale' => $matches[2]]);
if (!empty($photo)) {
$media['preview-width'] = $photo['width'];
$media['preview-height'] = $photo['height'];

View File

@ -53,7 +53,8 @@ class Site extends BaseAdmin
$language = (!empty($_POST['language']) ? trim($_POST['language']) : '');
$theme = (!empty($_POST['theme']) ? trim($_POST['theme']) : '');
$theme_mobile = (!empty($_POST['theme_mobile']) ? trim($_POST['theme_mobile']) : '');
$maximagesize = (!empty($_POST['maximagesize']) ? trim($_POST['maximagesize']) : 0);
$maxfilesize = (!empty($_POST['maxfilesize']) ? trim($_POST['maxfilesize']) : 0);
$maximagesize = (!empty($_POST['maximagesize']) ? trim($_POST['maximagesize']) : 0);
$maximagelength = (!empty($_POST['maximagelength']) ? intval(trim($_POST['maximagelength'])) : -1);
$jpegimagequality = (!empty($_POST['jpegimagequality']) ? intval(trim($_POST['jpegimagequality'])) : 100);
@ -224,6 +225,11 @@ class Site extends BaseAdmin
} else {
$transactionConfig->set('system', 'singleuser', $singleuser);
}
if (preg_match('/\d+(?:\s*[kmg])?/i', $maxfilesize)) {
$transactionConfig->set('system', 'maxfilesize', $maxfilesize);
} else {
DI::sysmsg()->addNotice(DI::l10n()->t('%s is no valid input for maximum media size', $maxfilesize));
}
if (preg_match('/\d+(?:\s*[kmg])?/i', $maximagesize)) {
$transactionConfig->set('system', 'maximagesize', $maximagesize);
} else {
@ -484,6 +490,10 @@ class Site extends BaseAdmin
'', 'pattern="\d+(?:\s*[kmg])?"'],
'$maximagelength' => ['maximagelength', DI::l10n()->t('Maximum image length'), DI::config()->get('system', 'max_image_length'), DI::l10n()->t('Maximum length in pixels of the longest side of uploaded images. Default is -1, which means no limits.')],
'$jpegimagequality' => ['jpegimagequality', DI::l10n()->t('JPEG image quality'), DI::config()->get('system', 'jpeg_quality'), DI::l10n()->t('Uploaded JPEGS will be saved at this quality setting [0-100]. Default is 100, which is full quality.')],
'$maxfilesize' => ['maxfilesize', DI::l10n()->t('Maximum media file size'), DI::config()->get('system', 'maxfilesize'), DI::l10n()->t('Maximum size in bytes of uploaded media files. Default is 0, which means no limits. You can put k, m, or g behind the desired value for KiB, MiB, GiB, respectively.
The value of <code>upload_max_filesize</code> in your <code>PHP.ini</code> needs be set to at least the desired limit.
Currently <code>upload_max_filesize</code> is set to %s (%s byte)', Strings::formatBytes(Strings::getBytesFromShorthand(ini_get('upload_max_filesize'))), Strings::getBytesFromShorthand(ini_get('upload_max_filesize'))),
'', 'pattern="\d+(?:\s*[kmg])?"'],
'$register_policy' => ['register_policy', DI::l10n()->t('Register policy'), DI::config()->get('config', 'register_policy'), '', $register_choices],
'$max_registered_users' => ['max_registered_users', DI::l10n()->t('Maximum Users'), DI::config()->get('config', 'max_registered_users'), DI::l10n()->t('If defined, the register policy is automatically closed when the given number of users is reached and reopens the registry when the number drops below the limit. It only works when the policy is set to open or close, but not when the policy is set to approval.')],

View File

@ -79,9 +79,14 @@ class Instance extends BaseApi
$image_matrix_limit = 33177600; // 5760^2
}
$media_size_limit = Strings::getBytesFromShorthand($this->config->get('system', 'maxfilesize'));
if (empty($media_size_limit)) {
$media_size_limit = Strings::getBytesFromShorthand(ini_get('upload_max_filesize'));
}
return new InstanceV2Entity\Configuration(
$statuses_config,
new InstanceV2Entity\MediaAttachmentsConfig(Images::supportedMimeTypes(), $image_size_limit, $image_matrix_limit),
new InstanceV2Entity\MediaAttachmentsConfig(Images::supportedMimeTypes(), $image_size_limit, $image_matrix_limit, $media_size_limit),
new InstanceV2Entity\Polls(),
new InstanceV2Entity\Accounts(),
);

View File

@ -115,9 +115,14 @@ class InstanceV2 extends BaseApi
$image_matrix_limit = 33177600; // 5760^2
}
$media_size_limit = Strings::getBytesFromShorthand($this->config->get('system', 'maxfilesize'));
if (empty($media_size_limit)) {
$media_size_limit = Strings::getBytesFromShorthand(ini_get('upload_max_filesize'));
}
return new InstanceEntity\Configuration(
$statuses_config,
new InstanceEntity\MediaAttachmentsConfig($this->supportedMimeTypes(), $image_size_limit, $image_matrix_limit),
new InstanceEntity\MediaAttachmentsConfig($this->supportedMimeTypes(), $image_size_limit, $image_matrix_limit, $media_size_limit),
new InstanceEntity\Polls(),
new InstanceEntity\Accounts(),
);

View File

@ -14,6 +14,7 @@ use Friendica\Model\Contact;
use Friendica\Model\Photo;
use Friendica\Model\Post;
use Friendica\Module\BaseApi;
use Friendica\Util\Strings;
/**
* @see https://docs.joinmastodon.org/methods/statuses/media/
@ -26,7 +27,7 @@ class Media extends BaseApi
$uid = self::getCurrentUserID();
$request = $this->getRequest([
'file' => [], // The file to be attached, using multipart form data.
'file' => $_FILES['file'] ?? [], // The file to be attached, using multipart form data.
'thumbnail' => [], // The custom thumbnail of the media to be attached, using multipart form data.
'description' => '', // A plain-text description of the media, for accessibility purposes.
'focus' => '', // Two floating points (x,y), comma-delimited ranging from -1.0 to 1.0
@ -34,14 +35,15 @@ class Media extends BaseApi
Logger::info('Photo post', ['request' => $request, 'files' => $_FILES]);
if (empty($_FILES['file'])) {
if (empty($request['file'])) {
Logger::notice('Upload is invalid', ['request' => $request]);
$this->logAndJsonError(422, $this->errorFactory->UnprocessableEntity());
}
$type = Post\Media::getType($_FILES['file']['type']);
$type = Post\Media::getType($request['file']['type']);
if (in_array($type, [Post\Media::IMAGE, Post\Media::UNKNOWN])) {
$media = Photo::upload($uid, $_FILES['file'], '', null, null, '', '', $request['description']);
$media = Photo::upload($uid, $request['file'], '', null, null, '', '', $request['description']);
if (empty($media)) {
$this->logAndJsonError(422, $this->errorFactory->UnprocessableEntity());
}
@ -50,22 +52,24 @@ class Media extends BaseApi
$this->jsonExit(DI::mstdnAttachment()->createFromPhoto($media['id']));
} else {
$tempFileName = $_FILES['file']['tmp_name'];
$fileName = basename($_FILES['file']['name']);
$fileSize = intval($_FILES['file']['size']);
$maxFileSize = DI::config()->get('system', 'maxfilesize');
$tempFileName = $request['file']['tmp_name'];
$fileName = basename($request['file']['name']);
$fileSize = intval($request['file']['size']);
$maxFileSize = Strings::getBytesFromShorthand(DI::config()->get('system', 'maxfilesize'));
if ($fileSize <= 0) {
Logger::notice('Filesize is invalid', ['size' => $fileSize, 'request' => $request]);
@unlink($tempFileName);
$this->logAndJsonError(422, $this->errorFactory->UnprocessableEntity());
}
if ($maxFileSize && $fileSize > $maxFileSize) {
Logger::notice('Filesize is too large', ['size' => $fileSize, 'max' => $maxFileSize, 'request' => $request]);
@unlink($tempFileName);
$this->logAndJsonError(422, $this->errorFactory->UnprocessableEntity());
}
$id = Attach::storeFile($tempFileName, self::getCurrentUserID(), $fileName, $_FILES['file']['type'], '<' . Contact::getPublicIdByUserId(self::getCurrentUserID()) . '>');
$id = Attach::storeFile($tempFileName, self::getCurrentUserID(), $fileName, $request['file']['type'], '<' . Contact::getPublicIdByUserId(self::getCurrentUserID()) . '>');
@unlink($tempFileName);
Logger::info('Uploaded media', ['id' => $id]);
$this->jsonExit(DI::mstdnAttachment()->createFromAttach($id));

View File

@ -71,7 +71,7 @@ class Upload extends \Friendica\BaseModule
$tempFileName = $_FILES['userfile']['tmp_name'];
$fileName = basename($_FILES['userfile']['name']);
$fileSize = intval($_FILES['userfile']['size']);
$maxFileSize = $this->config->get('system', 'maxfilesize');
$maxFileSize = Strings::getBytesFromShorthand($this->config->get('system', 'maxfilesize'));
/*
* Found html code written in text field of form, when trying to upload a

View File

@ -32,12 +32,12 @@ class MediaAttachmentsConfig extends BaseDataTransferObject
/**
* @param array $supported_mime_types
*/
public function __construct(array $supported_mime_types, int $image_size_limit, int $image_matrix_limit)
public function __construct(array $supported_mime_types, int $image_size_limit, int $image_matrix_limit, int $media_size_limit)
{
$this->supported_mime_types = $supported_mime_types;
$this->image_size_limit = $image_size_limit;
$this->image_matrix_limit = $image_matrix_limit;
$this->video_size_limit = $image_size_limit;
$this->video_size_limit = $media_size_limit;
$this->video_matrix_limit = $image_matrix_limit;
}
}

View File

@ -979,9 +979,11 @@ class Processor
}
}
$author = Contact::selectFirstAccount(['ap-posting-restricted'], ['id' => $item['author-id']]);
if (!empty($author['ap-posting-restricted'])) {
$item['restrictions'] = $item['restrictions'] | Item::CANT_REPLY;
if (!empty($item['author-id'])) {
$author = Contact::selectFirstAccount(['ap-posting-restricted'], ['id' => $item['author-id']]);
if (!empty($author['ap-posting-restricted'])) {
$item['restrictions'] = $item['restrictions'] | Item::CANT_REPLY;
}
}
$item['location'] = $activity['location'];

View File

@ -13,6 +13,7 @@ use Friendica\Database\Database;
use Friendica\Database\DBA;
use Friendica\Database\DBStructure;
use Friendica\DI;
use Friendica\Model\Attach;
use Friendica\Model\Item;
use Friendica\Model\Post;
use Friendica\Util\DateTimeFormat;
@ -43,6 +44,8 @@ class ExpirePosts
self::addMissingEntries();
}
self::deleteUnusedAttachments();
DBA::releaseOptimizeLock();
// Set the expiry for origin posts
@ -304,4 +307,17 @@ class ExpirePosts
} while ($affected_count);
}
}
/**
* Delete media attachments (excluding photos) that aren't linked to any post
*
* @return void
*/
private static function deleteUnusedAttachments()
{
$postmedia = DBA::select('attach', ['id'], ["`id` NOT IN (SELECT `attach-id` FROM `post-media`)"]);
while ($media = DBA::fetch($postmedia)) {
Attach::delete(['id' => $media['id']]);
}
}
}

View File

@ -44,7 +44,7 @@ use Friendica\Database\DBA;
// This file is required several times during the test in DbaDefinition which justifies this condition
if (!defined('DB_UPDATE_VERSION')) {
define('DB_UPDATE_VERSION', 1572);
define('DB_UPDATE_VERSION', 1573);
}
return [
@ -1431,6 +1431,7 @@ return [
"uri-id" => ["type" => "int unsigned", "not null" => "1", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the item uri"],
"url" => ["type" => "varbinary(1024)", "not null" => "1", "comment" => "Media URL"],
"media-uri-id" => ["type" => "int unsigned", "foreign" => ["item-uri" => "id"], "comment" => "Id of the item-uri table entry that contains the activities uri-id"],
"attach-id" => ["type" => "int unsigned", "foreign" => ["attach" => "id"], "comment" => "In case of a local attachment, this field is filled with the id in the attach table"],
"type" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "comment" => "Media type"],
"mimetype" => ["type" => "varchar(60)", "comment" => ""],
"height" => ["type" => "smallint unsigned", "comment" => "Height of the media"],
@ -1454,6 +1455,7 @@ return [
"uri-id-url" => ["UNIQUE", "uri-id", "url(512)"],
"uri-id-id" => ["uri-id", "id"],
"media-uri-id" => ["media-uri-id"],
"attach-id" => ["attach-id"],
]
],
"post-origin" => [

View File

@ -40,6 +40,7 @@ use Friendica\Database\Database;
use Friendica\Database\DBA;
use Friendica\Database\DBStructure;
use Friendica\DI;
use Friendica\Model\Attach;
use Friendica\Model\Contact;
use Friendica\Model\Item;
use Friendica\Model\ItemURI;
@ -1502,3 +1503,27 @@ function update_1571()
return Update::SUCCESS;
}
function update_1573()
{
$postmedia = DBA::select('post-media', ['id', 'url'], ["`url` LIKE ?", '%/attach/%']);
while ($media = DBA::fetch($postmedia)) {
if (!DI::baseUrl()->isLocalUrl($media['url'])) {
continue;
}
if (preg_match('|.*?/attach/(\d+)|', $media['url'], $matches)) {
$attachment = Attach::selectFirst(['id', 'filename', 'filetype', 'filesize'], ['id' => $matches[1]]);
if (!empty($attachment)) {
$fields = [
'attach-id' => $attachment['id'],
'name' => $attachment['filename'],
'mimetype' => $attachment['filetype'],
'size' => $attachment['filesize'],
];
DBA::update('post-media', $fields, ['id' => $media['id']]);
}
}
}
DBA::close($media);
return Update::SUCCESS;
}

File diff suppressed because it is too large Load Diff

View File

@ -47,6 +47,7 @@
{{include file="field_input.tpl" field=$maximagesize}}
{{include file="field_input.tpl" field=$maximagelength}}
{{include file="field_input.tpl" field=$jpegimagequality}}
{{include file="field_input.tpl" field=$maxfilesize}}
<div class="submit"><input type="submit" name="page_site" value="{{$submit}}"/></div>
<h2>{{$corporate}}</h2>

View File

@ -106,6 +106,7 @@
{{include file="field_input.tpl" field=$maximagesize}}
{{include file="field_input.tpl" field=$maximagelength}}
{{include file="field_input.tpl" field=$jpegimagequality}}
{{include file="field_input.tpl" field=$maxfilesize}}
</div>
<div class="panel-footer">
<input type="submit" name="page_site" class="btn btn-primary" value="{{$submit}}"/>