From 6c849cf9d6b2b16356ed7ad2e865f7ff265148a9 Mon Sep 17 00:00:00 2001 From: Khokan Sardar Date: Thu, 6 Aug 2026 23:13:43 +0530 Subject: [PATCH 1/2] General: Replace core's remaining `utf8_encode()` calls with `wp_scrub_utf8()`. `wxr_cdata()` and `wp_read_image_metadata()` hold the last three calls to `utf8_encode()` in core. That function was deprecated in PHP 8.2, is removed in PHP 9.0, and core itself polyfilled it with a deprecation notice in [60950], so these sites emit a deprecation notice on every affected export and image upload. All three used the `if ( ! wp_is_valid_utf8( $text ) ) { utf8_encode( $text ); }` pattern, which assumes any text failing UTF-8 validation is ISO-8859-1 and re-encodes the raw bytes on that assumption. That guess is wrong for every other single-byte encoding and silently produces mojibake. Replacing the invalid spans with the Unicode replacement character neutralizes the corruption without inventing an encoding, using the UTF-8 pipeline core added in 6.9. Adds regression tests covering the WXR export and IPTC metadata that carry invalid UTF-8. See #65828, #55603. --- src/wp-admin/includes/export.php | 2 +- src/wp-admin/includes/image.php | 4 +- tests/phpunit/tests/admin/exportWp.php | 62 +++++++++++++++++++++ tests/phpunit/tests/image/meta.php | 75 ++++++++++++++++++++++++++ 4 files changed, 140 insertions(+), 3 deletions(-) diff --git a/src/wp-admin/includes/export.php b/src/wp-admin/includes/export.php index a77cb804f0780..3be747e8fc24c 100644 --- a/src/wp-admin/includes/export.php +++ b/src/wp-admin/includes/export.php @@ -246,7 +246,7 @@ function wxr_cdata( $str ) { $str = (string) $str; if ( ! wp_is_valid_utf8( $str ) ) { - $str = utf8_encode( $str ); + $str = wp_scrub_utf8( $str ); } // $str = ent2ncr(esc_html($str)); $str = '', ']]]]>', $str ) . ']]>'; diff --git a/src/wp-admin/includes/image.php b/src/wp-admin/includes/image.php index 935c613d561e9..f0542e8f32346 100644 --- a/src/wp-admin/includes/image.php +++ b/src/wp-admin/includes/image.php @@ -1049,13 +1049,13 @@ function wp_read_image_metadata( $file ) { foreach ( array( 'title', 'caption', 'credit', 'copyright', 'camera', 'iso' ) as $key ) { if ( $meta[ $key ] && ! wp_is_valid_utf8( $meta[ $key ] ) ) { - $meta[ $key ] = utf8_encode( $meta[ $key ] ); + $meta[ $key ] = wp_scrub_utf8( $meta[ $key ] ); } } foreach ( $meta['keywords'] as $key => $keyword ) { if ( ! wp_is_valid_utf8( $keyword ) ) { - $meta['keywords'][ $key ] = utf8_encode( $keyword ); + $meta['keywords'][ $key ] = wp_scrub_utf8( $keyword ); } } diff --git a/tests/phpunit/tests/admin/exportWp.php b/tests/phpunit/tests/admin/exportWp.php index f17ef0d4ad343..7a6e8b990d83b 100644 --- a/tests/phpunit/tests/admin/exportWp.php +++ b/tests/phpunit/tests/admin/exportWp.php @@ -475,6 +475,68 @@ public function test_export_with_null_term_meta_values() { $this->assertGreaterThan( 0, count( $xml->channel->item ), 'Export should contain items' ); } + /** + * Ensures the WXR export neutralizes invalid UTF-8 instead of reinterpreting it as ISO-8859-1. + * + * `wxr_cdata()` previously called the deprecated `utf8_encode()`, which assumed any + * string that failed UTF-8 validation was ISO-8859-1 and re-encoded the raw bytes on + * that assumption. That guess is wrong for every other single-byte encoding, so the + * invalid spans are now replaced with the Unicode replacement character instead. + * + * @ticket 65828 + * + * @dataProvider data_invalid_utf8_strings + * + * @param string $input Bytes which are not valid UTF-8. + * @param string $expected Expected CDATA contents. + */ + public function test_wxr_cdata_scrubs_invalid_utf8( $input, $expected ) { + // Running an export defines the nested WXR helper functions. + $this->get_the_export( array( 'content' => 'post' ) ); + + $actual = wxr_cdata( $input ); + $inner = substr( $actual, strlen( '' ) ); + + $this->assertTrue( + wp_is_valid_utf8( $inner ), + 'The exported CDATA section should always contain valid UTF-8.' + ); + $this->assertSame( + $expected, + $inner, + 'Invalid bytes should be replaced, not reinterpreted as ISO-8859-1.' + ); + } + + /** + * Data provider. + * + * @return array[] + */ + public function data_invalid_utf8_strings() { + return array( + 'Lone high byte' => array( "Caf\xE9", "Caf\u{FFFD}" ), + 'Never-valid byte' => array( "a\xC0b", "a\u{FFFD}b" ), + 'Truncated sequence' => array( "a\xE2\x9Cb", "a\u{FFFD}b" ), + 'Overlong sequence' => array( "a\xC1\xBFb", "a\u{FFFD}\u{FFFD}b" ), + 'Surrogate half' => array( "a\xED\xA0\x80b", "a\u{FFFD}\u{FFFD}\u{FFFD}b" ), + ); + } + + /** + * Ensures valid UTF-8, including multibyte text, survives the export untouched. + * + * @ticket 65828 + */ + public function test_wxr_cdata_preserves_valid_utf8() { + $this->get_the_export( array( 'content' => 'post' ) ); + + $valid = 'Это комментарий. / Βλέπετε ένα σχόλιο. / 🅰'; + $inner = substr( wxr_cdata( $valid ), strlen( '' ) ); + + $this->assertSame( $valid, $inner, 'Valid UTF-8 should pass through unchanged.' ); + } + /** * Ensure that posts types with 'can_export' set to false are not included in the export. * diff --git a/tests/phpunit/tests/image/meta.php b/tests/phpunit/tests/image/meta.php index b6a1849fae9ae..a3325bc788c40 100644 --- a/tests/phpunit/tests/image/meta.php +++ b/tests/phpunit/tests/image/meta.php @@ -149,6 +149,81 @@ public function test_utf8_iptc_tags() { $this->assertSame( 'This is a comment. / Это комментарий. / Βλέπετε ένα σχόλιο.', $out['caption'] ); } + /** + * Ensures invalid UTF-8 in IPTC fields is neutralized rather than assumed to be ISO-8859-1. + * + * `wp_read_image_metadata()` previously passed any field that failed UTF-8 validation + * through the deprecated `utf8_encode()`, which reinterpreted the raw bytes as + * ISO-8859-1. IPTC carries no reliable encoding declaration, so the invalid spans are + * now replaced with the Unicode replacement character instead of being guessed at. + * + * @ticket 65828 + */ + public function test_iptc_invalid_utf8_is_scrubbed() { + if ( ! is_callable( 'iptcembed' ) || ! is_callable( 'iptcparse' ) ) { + $this->markTestSkipped( 'The iptcembed() and iptcparse() functions are required.' ); + } + + $block = $this->build_iptc_block( + array( + 105 => "Headline \xE9", + 120 => "Caf\xE9 by the r\xEDver", + 110 => "Credit \xC0", + 116 => "Copyright \xE9", + 25 => array( 'valid keyword', "sunset\xC0" ), + ) + ); + + $file = wp_tempnam( 'iptc-invalid-utf8.jpg' ); + file_put_contents( $file, iptcembed( $block, DIR_TESTDATA . '/images/test-image.jpg' ) ); + + $out = wp_read_image_metadata( $file ); + + unlink( $file ); + + $this->assertIsArray( $out, 'Metadata should have been read from the image.' ); + + foreach ( array( 'title', 'caption', 'credit', 'copyright' ) as $key ) { + $this->assertTrue( + wp_is_valid_utf8( $out[ $key ] ), + "The '{$key}' field should always be valid UTF-8." + ); + } + + $this->assertSame( + "Caf\u{FFFD} by the r\u{FFFD}ver", + $out['caption'], + 'Invalid bytes should be replaced, not reinterpreted as ISO-8859-1.' + ); + + $this->assertSame( + array( 'valid keyword', "sunset\u{FFFD}" ), + $out['keywords'], + 'Keywords should be scrubbed individually while valid entries are left alone.' + ); + } + + /** + * Builds a raw IPTC APP13 block for the given record 2 datasets. + * + * @param array $tags Map of dataset number to a string value or list of string values. + * @return string Binary IPTC block suitable for iptcembed(). + */ + private function build_iptc_block( array $tags ) { + $block = ''; + + foreach ( $tags as $dataset => $values ) { + foreach ( (array) $values as $value ) { + $length = strlen( $value ); + $block .= chr( 0x1C ) . chr( 2 ) . chr( $dataset ) + . chr( ( $length >> 8 ) & 0xFF ) . chr( $length & 0xFF ) + . $value; + } + } + + return $block; + } + /** * wp_read_image_metadata() should return false if the image file doesn't exist. */ From e2b220b68d2ef5ef10fd6978c752123cd5cb407e Mon Sep 17 00:00:00 2001 From: Khokan Sardar Date: Fri, 7 Aug 2026 15:05:40 +0530 Subject: [PATCH 2/2] Address review: drop redundant guards, test real encodings. - `wp_scrub_utf8()` validates internally, so the preceding `wp_is_valid_utf8()` checks were redundant. The `$meta[ $key ]` truthiness check stays, as `iso` defaults to int `0`. - Test docblocks describe current behaviour rather than narrating the code that was removed. - Adds data-provider cases for text in ISO-8859-1, ISO-8859-2, Windows-1251 and Windows-1252, which exercise the actual behavioural change; the previous cases only covered malformed UTF-8. See #65828. --- src/wp-admin/includes/export.php | 5 +--- src/wp-admin/includes/image.php | 8 ++---- tests/phpunit/tests/admin/exportWp.php | 40 +++++++++++++++++++------- tests/phpunit/tests/image/meta.php | 24 ++++++++++------ 4 files changed, 47 insertions(+), 30 deletions(-) diff --git a/src/wp-admin/includes/export.php b/src/wp-admin/includes/export.php index 3be747e8fc24c..30c5f0fb14b4d 100644 --- a/src/wp-admin/includes/export.php +++ b/src/wp-admin/includes/export.php @@ -243,11 +243,8 @@ function export_wp( $args = array() ) { * @return string */ function wxr_cdata( $str ) { - $str = (string) $str; + $str = wp_scrub_utf8( (string) $str ); - if ( ! wp_is_valid_utf8( $str ) ) { - $str = wp_scrub_utf8( $str ); - } // $str = ent2ncr(esc_html($str)); $str = '', ']]]]>', $str ) . ']]>'; diff --git a/src/wp-admin/includes/image.php b/src/wp-admin/includes/image.php index f0542e8f32346..925e9b6534764 100644 --- a/src/wp-admin/includes/image.php +++ b/src/wp-admin/includes/image.php @@ -1048,16 +1048,12 @@ function wp_read_image_metadata( $file ) { } foreach ( array( 'title', 'caption', 'credit', 'copyright', 'camera', 'iso' ) as $key ) { - if ( $meta[ $key ] && ! wp_is_valid_utf8( $meta[ $key ] ) ) { + if ( $meta[ $key ] ) { $meta[ $key ] = wp_scrub_utf8( $meta[ $key ] ); } } - foreach ( $meta['keywords'] as $key => $keyword ) { - if ( ! wp_is_valid_utf8( $keyword ) ) { - $meta['keywords'][ $key ] = wp_scrub_utf8( $keyword ); - } - } + $meta['keywords'] = array_map( 'wp_scrub_utf8', $meta['keywords'] ); $meta = wp_kses_post_deep( $meta ); diff --git a/tests/phpunit/tests/admin/exportWp.php b/tests/phpunit/tests/admin/exportWp.php index 7a6e8b990d83b..cf196629a2319 100644 --- a/tests/phpunit/tests/admin/exportWp.php +++ b/tests/phpunit/tests/admin/exportWp.php @@ -476,12 +476,12 @@ public function test_export_with_null_term_meta_values() { } /** - * Ensures the WXR export neutralizes invalid UTF-8 instead of reinterpreting it as ISO-8859-1. + * Ensures the WXR export always emits valid UTF-8. * - * `wxr_cdata()` previously called the deprecated `utf8_encode()`, which assumed any - * string that failed UTF-8 validation was ISO-8859-1 and re-encoded the raw bytes on - * that assumption. That guess is wrong for every other single-byte encoding, so the - * invalid spans are now replaced with the Unicode replacement character instead. + * Byte spans which cannot decode as UTF-8 are replaced with the Unicode replacement + * character. This covers both malformed UTF-8 and text supplied in some other + * encoding entirely; core has no way to know which, so the bytes are neutralized + * rather than reinterpreted as any particular encoding. * * @ticket 65828 * @@ -504,7 +504,7 @@ public function test_wxr_cdata_scrubs_invalid_utf8( $input, $expected ) { $this->assertSame( $expected, $inner, - 'Invalid bytes should be replaced, not reinterpreted as ISO-8859-1.' + 'Byte spans which cannot decode as UTF-8 should be replaced.' ); } @@ -515,11 +515,29 @@ public function test_wxr_cdata_scrubs_invalid_utf8( $input, $expected ) { */ public function data_invalid_utf8_strings() { return array( - 'Lone high byte' => array( "Caf\xE9", "Caf\u{FFFD}" ), - 'Never-valid byte' => array( "a\xC0b", "a\u{FFFD}b" ), - 'Truncated sequence' => array( "a\xE2\x9Cb", "a\u{FFFD}b" ), - 'Overlong sequence' => array( "a\xC1\xBFb", "a\u{FFFD}\u{FFFD}b" ), - 'Surrogate half' => array( "a\xED\xA0\x80b", "a\u{FFFD}\u{FFFD}\u{FFFD}b" ), + // Malformed UTF-8. + 'Never-valid byte' => array( "a\xC0b", "a\u{FFFD}b" ), + 'Truncated sequence' => array( "a\xE2\x9Cb", "a\u{FFFD}b" ), + 'Overlong sequence' => array( "a\xC1\xBFb", "a\u{FFFD}\u{FFFD}b" ), + 'Surrogate half' => array( "a\xED\xA0\x80b", "a\u{FFFD}\u{FFFD}\u{FFFD}b" ), + + // Text which is well-formed, but in some encoding other than UTF-8. + 'ISO-8859-1 text' => array( + mb_convert_encoding( 'Café', 'ISO-8859-1', 'UTF-8' ), + "Caf\u{FFFD}", + ), + 'ISO-8859-2 text' => array( + mb_convert_encoding( 'wyróżnij', 'ISO-8859-2', 'UTF-8' ), + "wyr\u{FFFD}nij", + ), + 'Windows-1251 text' => array( + mb_convert_encoding( 'Привет', 'Windows-1251', 'UTF-8' ), + str_repeat( "\u{FFFD}", 6 ), + ), + 'Windows-1252 quotations' => array( + mb_convert_encoding( '“quoted”', 'Windows-1252', 'UTF-8' ), + "\u{FFFD}quoted\u{FFFD}", + ), ); } diff --git a/tests/phpunit/tests/image/meta.php b/tests/phpunit/tests/image/meta.php index a3325bc788c40..fa192ce79ed4a 100644 --- a/tests/phpunit/tests/image/meta.php +++ b/tests/phpunit/tests/image/meta.php @@ -150,12 +150,12 @@ public function test_utf8_iptc_tags() { } /** - * Ensures invalid UTF-8 in IPTC fields is neutralized rather than assumed to be ISO-8859-1. + * Ensures image metadata is always returned as valid UTF-8. * - * `wp_read_image_metadata()` previously passed any field that failed UTF-8 validation - * through the deprecated `utf8_encode()`, which reinterpreted the raw bytes as - * ISO-8859-1. IPTC carries no reliable encoding declaration, so the invalid spans are - * now replaced with the Unicode replacement character instead of being guessed at. + * Core does not read the IPTC coded character set, so the encoding of these fields is + * unknown. Byte spans which cannot decode as UTF-8 are replaced with the Unicode + * replacement character rather than reinterpreted as any particular encoding. Both + * malformed UTF-8 and text in another encoding are covered here. * * @ticket 65828 */ @@ -166,8 +166,8 @@ public function test_iptc_invalid_utf8_is_scrubbed() { $block = $this->build_iptc_block( array( - 105 => "Headline \xE9", - 120 => "Caf\xE9 by the r\xEDver", + 105 => mb_convert_encoding( 'wyróżnij', 'ISO-8859-2', 'UTF-8' ), + 120 => mb_convert_encoding( 'Café', 'ISO-8859-1', 'UTF-8' ), 110 => "Credit \xC0", 116 => "Copyright \xE9", 25 => array( 'valid keyword', "sunset\xC0" ), @@ -191,9 +191,15 @@ public function test_iptc_invalid_utf8_is_scrubbed() { } $this->assertSame( - "Caf\u{FFFD} by the r\u{FFFD}ver", + "Caf\u{FFFD}", $out['caption'], - 'Invalid bytes should be replaced, not reinterpreted as ISO-8859-1.' + 'ISO-8859-1 text should be neutralized, not reinterpreted.' + ); + + $this->assertSame( + "wyr\u{FFFD}nij", + $out['title'], + 'ISO-8859-2 text should be neutralized, not reinterpreted.' ); $this->assertSame(