From a5be5b27e3193c1134855126eab258c810505d93 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 4 Dec 2022 13:29:21 +0000 Subject: [PATCH] Support Blurhash --- composer.json | 3 +- composer.lock | 46 +++++++++++++- database.sql | 4 +- doc/database/db_photo.md | 1 + doc/database/db_post-media.md | 1 + src/Factory/Api/Mastodon/Attachment.php | 3 +- src/Factory/Api/Mastodon/Card.php | 1 + src/Model/Contact/Relation.php | 2 +- src/Model/Photo.php | 1 + src/Model/Post/Media.php | 7 ++- src/Object/Api/Mastodon/Attachment.php | 3 + src/Object/Api/Mastodon/Card.php | 3 + src/Object/Image.php | 83 ++++++++++++++++++++++--- src/Util/Images.php | 6 +- src/Util/ParseUrl.php | 1 + static/dbstructure.config.php | 4 +- 16 files changed, 152 insertions(+), 17 deletions(-) diff --git a/composer.json b/composer.json index d0fc5619b9..0e3e42c721 100644 --- a/composer.json +++ b/composer.json @@ -70,7 +70,8 @@ "npm-asset/moment": "^2.24", "npm-asset/perfect-scrollbar": "0.6.16", "npm-asset/textcomplete": "^0.18.2", - "npm-asset/typeahead.js": "^0.11.1" + "npm-asset/typeahead.js": "^0.11.1", + "kornrunner/blurhash": "^1.2" }, "repositories": [ { diff --git a/composer.lock b/composer.lock index 1805c9d21a..9a4854f15c 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "2e082bac083ca61cc0c22f7055d690bf", + "content-hash": "f8e7baec685d20e6aee56978c275d64c", "packages": [ { "name": "asika/simple-console", @@ -1116,6 +1116,50 @@ ], "time": "2022-06-20T21:43:03+00:00" }, + { + "name": "kornrunner/blurhash", + "version": "v1.2.2", + "source": { + "type": "git", + "url": "https://github.com/kornrunner/php-blurhash.git", + "reference": "bc8a4596cb0a49874f0158696a382ab3933fefe4" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/kornrunner/php-blurhash/zipball/bc8a4596cb0a49874f0158696a382ab3933fefe4", + "reference": "bc8a4596cb0a49874f0158696a382ab3933fefe4", + "shasum": "" + }, + "require": { + "php": "^7.3|^8.0" + }, + "require-dev": { + "ext-gd": "*", + "ocramius/package-versions": "^1.4|^2.0", + "phpstan/phpstan": "^0.12", + "phpunit/phpunit": "^9", + "vimeo/psalm": "^4.3" + }, + "type": "library", + "autoload": { + "psr-4": { + "kornrunner\\Blurhash\\": "src" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Boris Momčilović", + "email": "boris.momcilovic@gmail.com" + } + ], + "description": "Pure PHP implementation of Blurhash", + "homepage": "https://github.com/kornrunner/php-blurhash", + "time": "2022-07-13T19:38:39+00:00" + }, { "name": "league/html-to-markdown", "version": "4.10.0", diff --git a/database.sql b/database.sql index f41defe428..290f8bed3f 100644 --- a/database.sql +++ b/database.sql @@ -1,6 +1,6 @@ -- ------------------------------------------ -- Friendica 2022.12-dev (Giant Rhubarb) --- DB_UPDATE_VERSION 1496 +-- DB_UPDATE_VERSION 1497 -- ------------------------------------------ @@ -1088,6 +1088,7 @@ CREATE TABLE IF NOT EXISTS `photo` ( `height` smallint unsigned NOT NULL DEFAULT 0 COMMENT '', `width` smallint unsigned NOT NULL DEFAULT 0 COMMENT '', `datasize` int unsigned NOT NULL DEFAULT 0 COMMENT '', + `blurhash` varbinary(255) COMMENT 'BlurHash representation of the photo', `data` mediumblob NOT NULL COMMENT '', `scale` tinyint unsigned NOT NULL DEFAULT 0 COMMENT '', `profile` boolean NOT NULL DEFAULT '0' COMMENT '', @@ -1313,6 +1314,7 @@ CREATE TABLE IF NOT EXISTS `post-media` ( `height` smallint unsigned COMMENT 'Height of the media', `width` smallint unsigned COMMENT 'Width of the media', `size` bigint unsigned COMMENT 'Media size', + `blurhash` varbinary(255) COMMENT 'BlurHash representation of the image', `preview` varbinary(512) COMMENT 'Preview URL', `preview-height` smallint unsigned COMMENT 'Height of the preview picture', `preview-width` smallint unsigned COMMENT 'Width of the preview picture', diff --git a/doc/database/db_photo.md b/doc/database/db_photo.md index a93cf65f91..ed417f6147 100644 --- a/doc/database/db_photo.md +++ b/doc/database/db_photo.md @@ -25,6 +25,7 @@ Fields | height | | smallint unsigned | NO | | 0 | | | width | | smallint unsigned | NO | | 0 | | | datasize | | int unsigned | NO | | 0 | | +| blurhash | BlurHash representation of the photo | varbinary(255) | YES | | NULL | | | data | | mediumblob | NO | | NULL | | | scale | | tinyint unsigned | NO | | 0 | | | profile | | boolean | NO | | 0 | | diff --git a/doc/database/db_post-media.md b/doc/database/db_post-media.md index d6e3a703c6..2d73a39cfc 100644 --- a/doc/database/db_post-media.md +++ b/doc/database/db_post-media.md @@ -17,6 +17,7 @@ Fields | height | Height of the media | smallint unsigned | YES | | NULL | | | width | Width of the media | smallint unsigned | YES | | NULL | | | size | Media size | bigint unsigned | YES | | NULL | | +| blurhash | BlurHash representation of the image | varbinary(255) | YES | | NULL | | | preview | Preview URL | varbinary(512) | YES | | NULL | | | preview-height | Height of the preview picture | smallint unsigned | YES | | NULL | | | preview-width | Width of the preview picture | smallint unsigned | YES | | NULL | | diff --git a/src/Factory/Api/Mastodon/Attachment.php b/src/Factory/Api/Mastodon/Attachment.php index accebe3433..197548b8a6 100644 --- a/src/Factory/Api/Mastodon/Attachment.php +++ b/src/Factory/Api/Mastodon/Attachment.php @@ -94,7 +94,7 @@ class Attachment extends BaseFactory */ public function createFromPhoto(int $id): array { - $photo = Photo::selectFirst(['resource-id', 'uid', 'id', 'title', 'type', 'width', 'height'], ['id' => $id]); + $photo = Photo::selectFirst(['resource-id', 'uid', 'id', 'title', 'type', 'width', 'height', 'blurhash'], ['id' => $id]); if (empty($photo)) { return []; } @@ -104,6 +104,7 @@ class Attachment extends BaseFactory 'description' => $photo['title'], 'width' => $photo['width'], 'height' => $photo['height'], + 'blurhash' => $photo['blurhash'], ]; $photoTypes = Images::supportedTypes(); diff --git a/src/Factory/Api/Mastodon/Card.php b/src/Factory/Api/Mastodon/Card.php index ac50841847..3efc625171 100644 --- a/src/Factory/Api/Mastodon/Card.php +++ b/src/Factory/Api/Mastodon/Card.php @@ -74,6 +74,7 @@ class Card extends BaseFactory $data['image'] = $attached['preview']; $data['width'] = $attached['preview-width']; $data['height'] = $attached['preview-height']; + $data['blurhash'] = $attached['blurhash']; } } diff --git a/src/Model/Contact/Relation.php b/src/Model/Contact/Relation.php index 3db882bb94..defb984f24 100644 --- a/src/Model/Contact/Relation.php +++ b/src/Model/Contact/Relation.php @@ -76,7 +76,7 @@ class Relation */ public static function discoverByUser(int $uid) { - $contact = Contact::selectFirst(['id', 'url'], ['uid' => $uid, 'self' => true]); + $contact = Contact::selectFirst(['id', 'url', 'network'], ['uid' => $uid, 'self' => true]); if (empty($contact)) { Logger::warning('Self contact for user not found', ['uid' => $uid]); return; diff --git a/src/Model/Photo.php b/src/Model/Photo.php index 126bc152b4..213551b115 100644 --- a/src/Model/Photo.php +++ b/src/Model/Photo.php @@ -436,6 +436,7 @@ class Photo 'height' => $image->getHeight(), 'width' => $image->getWidth(), 'datasize' => strlen($image->asString()), + 'blurhash' => $image->getBlurHash(), 'data' => $data, 'scale' => $scale, 'photo-type' => $type, diff --git a/src/Model/Post/Media.php b/src/Model/Post/Media.php index 854e5d8f91..62590594be 100644 --- a/src/Model/Post/Media.php +++ b/src/Model/Post/Media.php @@ -117,7 +117,7 @@ class Media */ private static function unsetEmptyFields(array $media): array { - $fields = ['mimetype', 'height', 'width', 'size', 'preview', 'preview-height', 'preview-width', 'description']; + $fields = ['mimetype', 'height', 'width', 'size', 'preview', 'preview-height', 'preview-width', 'blurhash', 'description']; foreach ($fields as $field) { if (empty($media[$field])) { unset($media[$field]); @@ -203,6 +203,7 @@ class Media $media['size'] = $imagedata['size']; $media['width'] = $imagedata[0]; $media['height'] = $imagedata[1]; + $media['blurhash'] = $imagedata['blurhash'] ?? null; } else { Logger::notice('No image data', ['media' => $media]); } @@ -232,6 +233,7 @@ class Media $media['preview'] = $data['images'][0]['src'] ?? null; $media['preview-height'] = $data['images'][0]['height'] ?? null; $media['preview-width'] = $data['images'][0]['width'] ?? null; + $media['blurhash'] = $data['images'][0]['blurhash'] ?? null; $media['description'] = $data['text'] ?? null; $media['name'] = $data['title'] ?? null; $media['author-url'] = $data['author_url'] ?? null; @@ -287,6 +289,7 @@ class Media $media['preview'] = null; $media['preview-height'] = null; $media['preview-width'] = null; + $media['blurhash'] = null; $media['description'] = $item['body']; $media['name'] = $item['title']; $media['author-url'] = $item['author-link']; @@ -328,6 +331,7 @@ class Media $media['preview'] = null; $media['preview-height'] = null; $media['preview-width'] = null; + $media['blurhash'] = null; $media['description'] = $contact['about']; $media['name'] = $contact['name']; $media['author-url'] = $contact['url']; @@ -357,6 +361,7 @@ class Media $media['size'] = $photo['datasize']; $media['width'] = $photo['width']; $media['height'] = $photo['height']; + $media['blurhash'] = $photo['blurhash']; } if (!preg_match('|.*?/photo/(.*[a-fA-F0-9])\-(.*[0-9])\..*[\w]|', $media['preview'] ?? '', $matches)) { diff --git a/src/Object/Api/Mastodon/Attachment.php b/src/Object/Api/Mastodon/Attachment.php index 3f890bf744..da467fc641 100644 --- a/src/Object/Api/Mastodon/Attachment.php +++ b/src/Object/Api/Mastodon/Attachment.php @@ -46,6 +46,8 @@ class Attachment extends BaseDataTransferObject protected $text_url; /** @var string */ protected $description; + /** @var string */ + protected $blurhash; /** @var array */ protected $meta; @@ -68,6 +70,7 @@ class Attachment extends BaseDataTransferObject $this->remote_url = $remote; $this->text_url = $this->remote_url ?? $this->url; $this->description = $attachment['description']; + $this->blurhash = $attachment['blurhash']; if ($type === 'image') { if ((int) $attachment['width'] > 0 && (int) $attachment['height'] > 0) { $this->meta['original']['width'] = (int) $attachment['width']; diff --git a/src/Object/Api/Mastodon/Card.php b/src/Object/Api/Mastodon/Card.php index bf87617190..aa5d8913a4 100644 --- a/src/Object/Api/Mastodon/Card.php +++ b/src/Object/Api/Mastodon/Card.php @@ -52,6 +52,8 @@ class Card extends BaseDataTransferObject protected $height; /** @var string */ protected $image; + /** @var string */ + protected $blurhash; /** * Creates a card record from an attachment array. @@ -72,6 +74,7 @@ class Card extends BaseDataTransferObject $this->width = $attachment['width'] ?? 0; $this->height = $attachment['height'] ?? 0; $this->image = $attachment['image'] ?? ''; + $this->blurhash = $attachment['blurhash'] ?? ''; $this->history = $history; } diff --git a/src/Object/Image.php b/src/Object/Image.php index 866ac268fa..c2cfa9e2f1 100644 --- a/src/Object/Image.php +++ b/src/Object/Image.php @@ -26,13 +26,15 @@ use Friendica\DI; use Friendica\Util\Images; use Imagick; use ImagickPixel; +use GDImage; +use kornrunner\Blurhash\Blurhash; /** * Class to handle images */ class Image { - /** @var Imagick|resource */ + /** @var GDImage|Imagick|resource */ private $image; /* @@ -695,14 +697,13 @@ class Image try { /* Clean it */ $this->image = $this->image->deconstructImages(); - $string = $this->image->getImagesBlob(); - return $string; + return $this->image->getImagesBlob(); } catch (Exception $e) { return false; } } - ob_start(); + $stream = fopen('php://memory','r+'); // Enable interlacing imageinterlace($this->image, true); @@ -710,18 +711,82 @@ class Image switch ($this->getType()) { case 'image/png': $quality = DI::config()->get('system', 'png_quality'); - imagepng($this->image, null, $quality); + imagepng($this->image, $stream, $quality); break; case 'image/jpeg': case 'image/jpg': $quality = DI::config()->get('system', 'jpeg_quality'); - imagejpeg($this->image, null, $quality); + imagejpeg($this->image, $stream, $quality); break; } - $string = ob_get_contents(); - ob_end_clean(); + rewind($stream); + return stream_get_contents($stream); + } - return $string; + /** + * Create a blurhash out of a given image string + * + * @param string $img_str + * @return string + */ + public function getBlurHash(): string + { + if ($this->isImagick()) { + // Imagick is not supported + return ''; + } + + $width = $this->getWidth(); + $height = $this->getHeight(); + + if (max($width, $height) > 90) { + $this->scaleDown(90); + $width = $this->getWidth(); + $height = $this->getHeight(); + } + + $pixels = []; + for ($y = 0; $y < $height; ++$y) { + $row = []; + for ($x = 0; $x < $width; ++$x) { + $index = imagecolorat($this->image, $x, $y); + $colors = imagecolorsforindex($this->image, $index); + + $row[] = [$colors['red'], $colors['green'], $colors['blue']]; + } + $pixels[] = $row; + } + + // The components define the amount of details (1 to 9). + $components_x = 9; + $components_y = 9; + + return Blurhash::encode($pixels, $components_x, $components_y); + } + + /** + * Create an image out of a blurhash + * + * @param string $blurhash + * @param integer $width + * @param integer $height + * @return void + */ + public function getFromBlurHash(string $blurhash, int $width, int $height) + { + if ($this->isImagick()) { + // Imagick is not supported + return; + } + + $pixels = Blurhash::decode($blurhash, $width, $height); + $this->image = imagecreatetruecolor($width, $height); + for ($y = 0; $y < $height; ++$y) { + for ($x = 0; $x < $width; ++$x) { + [$r, $g, $b] = $pixels[$y][$x]; + imagesetpixel($this->image, $x, $y, imagecolorallocate($this->image, $r, $g, $b)); + } + } } } diff --git a/src/Util/Images.php b/src/Util/Images.php index eb1e8c4375..533feec844 100644 --- a/src/Util/Images.php +++ b/src/Util/Images.php @@ -25,6 +25,7 @@ use Friendica\Core\Logger; use Friendica\DI; use Friendica\Model\Photo; use Friendica\Network\HTTPClient\Client\HttpClientAccept; +use Friendica\Object\Image; /** * Image utilities @@ -244,7 +245,10 @@ class Images } if ($data) { - $data['size'] = $filesize; + $image = new Image($img_str); + + $data['blurhash'] = $image->getBlurHash(); + $data['size'] = $filesize; } return is_array($data) ? $data : []; diff --git a/src/Util/ParseUrl.php b/src/Util/ParseUrl.php index f0afa1ec2b..5b0bf6a5ca 100644 --- a/src/Util/ParseUrl.php +++ b/src/Util/ParseUrl.php @@ -567,6 +567,7 @@ class ParseUrl $image['width'] = $photodata[0]; $image['height'] = $photodata[1]; $image['contenttype'] = $photodata['mime']; + $image['blurhash'] = $photodata['blurhash'] ?? null; unset($image['url']); ksort($image); } else { diff --git a/static/dbstructure.config.php b/static/dbstructure.config.php index 2979ca80d0..f8e01e51ee 100644 --- a/static/dbstructure.config.php +++ b/static/dbstructure.config.php @@ -55,7 +55,7 @@ use Friendica\Database\DBA; if (!defined('DB_UPDATE_VERSION')) { - define('DB_UPDATE_VERSION', 1496); + define('DB_UPDATE_VERSION', 1497); } return [ @@ -1129,6 +1129,7 @@ return [ "height" => ["type" => "smallint unsigned", "not null" => "1", "default" => "0", "comment" => ""], "width" => ["type" => "smallint unsigned", "not null" => "1", "default" => "0", "comment" => ""], "datasize" => ["type" => "int unsigned", "not null" => "1", "default" => "0", "comment" => ""], + "blurhash" => ["type" => "varbinary(255)", "comment" => "BlurHash representation of the photo"], "data" => ["type" => "mediumblob", "not null" => "1", "comment" => ""], "scale" => ["type" => "tinyint unsigned", "not null" => "1", "default" => "0", "comment" => ""], "profile" => ["type" => "boolean", "not null" => "1", "default" => "0", "comment" => ""], @@ -1340,6 +1341,7 @@ return [ "height" => ["type" => "smallint unsigned", "comment" => "Height of the media"], "width" => ["type" => "smallint unsigned", "comment" => "Width of the media"], "size" => ["type" => "bigint unsigned", "comment" => "Media size"], + "blurhash" => ["type" => "varbinary(255)", "comment" => "BlurHash representation of the image"], "preview" => ["type" => "varbinary(512)", "comment" => "Preview URL"], "preview-height" => ["type" => "smallint unsigned", "comment" => "Height of the preview picture"], "preview-width" => ["type" => "smallint unsigned", "comment" => "Width of the preview picture"],