diff --git a/src/wp-admin/includes/export.php b/src/wp-admin/includes/export.php index a77cb804f0780..30c5f0fb14b4d 100644 --- a/src/wp-admin/includes/export.php +++ b/src/wp-admin/includes/export.php @@ -243,11 +243,8 @@ function export_wp( $args = array() ) { * @return string */ function wxr_cdata( $str ) { - $str = (string) $str; + $str = wp_scrub_utf8( (string) $str ); - if ( ! wp_is_valid_utf8( $str ) ) { - $str = utf8_encode( $str ); - } // $str = ent2ncr(esc_html($str)); $str = '', ']]]]>', $str ) . ']]>'; diff --git a/src/wp-admin/includes/image.php b/src/wp-admin/includes/image.php index 935c613d561e9..925e9b6534764 100644 --- a/src/wp-admin/includes/image.php +++ b/src/wp-admin/includes/image.php @@ -1048,16 +1048,12 @@ function wp_read_image_metadata( $file ) { } foreach ( array( 'title', 'caption', 'credit', 'copyright', 'camera', 'iso' ) as $key ) { - if ( $meta[ $key ] && ! wp_is_valid_utf8( $meta[ $key ] ) ) { - $meta[ $key ] = utf8_encode( $meta[ $key ] ); + if ( $meta[ $key ] ) { + $meta[ $key ] = wp_scrub_utf8( $meta[ $key ] ); } } - foreach ( $meta['keywords'] as $key => $keyword ) { - if ( ! wp_is_valid_utf8( $keyword ) ) { - $meta['keywords'][ $key ] = utf8_encode( $keyword ); - } - } + $meta['keywords'] = array_map( 'wp_scrub_utf8', $meta['keywords'] ); $meta = wp_kses_post_deep( $meta ); diff --git a/tests/phpunit/tests/admin/exportWp.php b/tests/phpunit/tests/admin/exportWp.php index f17ef0d4ad343..cf196629a2319 100644 --- a/tests/phpunit/tests/admin/exportWp.php +++ b/tests/phpunit/tests/admin/exportWp.php @@ -475,6 +475,86 @@ public function test_export_with_null_term_meta_values() { $this->assertGreaterThan( 0, count( $xml->channel->item ), 'Export should contain items' ); } + /** + * Ensures the WXR export always emits valid UTF-8. + * + * Byte spans which cannot decode as UTF-8 are replaced with the Unicode replacement + * character. This covers both malformed UTF-8 and text supplied in some other + * encoding entirely; core has no way to know which, so the bytes are neutralized + * rather than reinterpreted as any particular encoding. + * + * @ticket 65828 + * + * @dataProvider data_invalid_utf8_strings + * + * @param string $input Bytes which are not valid UTF-8. + * @param string $expected Expected CDATA contents. + */ + public function test_wxr_cdata_scrubs_invalid_utf8( $input, $expected ) { + // Running an export defines the nested WXR helper functions. + $this->get_the_export( array( 'content' => 'post' ) ); + + $actual = wxr_cdata( $input ); + $inner = substr( $actual, strlen( '' ) ); + + $this->assertTrue( + wp_is_valid_utf8( $inner ), + 'The exported CDATA section should always contain valid UTF-8.' + ); + $this->assertSame( + $expected, + $inner, + 'Byte spans which cannot decode as UTF-8 should be replaced.' + ); + } + + /** + * Data provider. + * + * @return array[] + */ + public function data_invalid_utf8_strings() { + return array( + // Malformed UTF-8. + 'Never-valid byte' => array( "a\xC0b", "a\u{FFFD}b" ), + 'Truncated sequence' => array( "a\xE2\x9Cb", "a\u{FFFD}b" ), + 'Overlong sequence' => array( "a\xC1\xBFb", "a\u{FFFD}\u{FFFD}b" ), + 'Surrogate half' => array( "a\xED\xA0\x80b", "a\u{FFFD}\u{FFFD}\u{FFFD}b" ), + + // Text which is well-formed, but in some encoding other than UTF-8. + 'ISO-8859-1 text' => array( + mb_convert_encoding( 'Café', 'ISO-8859-1', 'UTF-8' ), + "Caf\u{FFFD}", + ), + 'ISO-8859-2 text' => array( + mb_convert_encoding( 'wyróżnij', 'ISO-8859-2', 'UTF-8' ), + "wyr\u{FFFD}nij", + ), + 'Windows-1251 text' => array( + mb_convert_encoding( 'Привет', 'Windows-1251', 'UTF-8' ), + str_repeat( "\u{FFFD}", 6 ), + ), + 'Windows-1252 quotations' => array( + mb_convert_encoding( '“quoted”', 'Windows-1252', 'UTF-8' ), + "\u{FFFD}quoted\u{FFFD}", + ), + ); + } + + /** + * Ensures valid UTF-8, including multibyte text, survives the export untouched. + * + * @ticket 65828 + */ + public function test_wxr_cdata_preserves_valid_utf8() { + $this->get_the_export( array( 'content' => 'post' ) ); + + $valid = 'Это комментарий. / Βλέπετε ένα σχόλιο. / 🅰'; + $inner = substr( wxr_cdata( $valid ), strlen( '' ) ); + + $this->assertSame( $valid, $inner, 'Valid UTF-8 should pass through unchanged.' ); + } + /** * Ensure that posts types with 'can_export' set to false are not included in the export. * diff --git a/tests/phpunit/tests/image/meta.php b/tests/phpunit/tests/image/meta.php index b6a1849fae9ae..fa192ce79ed4a 100644 --- a/tests/phpunit/tests/image/meta.php +++ b/tests/phpunit/tests/image/meta.php @@ -149,6 +149,87 @@ public function test_utf8_iptc_tags() { $this->assertSame( 'This is a comment. / Это комментарий. / Βλέπετε ένα σχόλιο.', $out['caption'] ); } + /** + * Ensures image metadata is always returned as valid UTF-8. + * + * Core does not read the IPTC coded character set, so the encoding of these fields is + * unknown. Byte spans which cannot decode as UTF-8 are replaced with the Unicode + * replacement character rather than reinterpreted as any particular encoding. Both + * malformed UTF-8 and text in another encoding are covered here. + * + * @ticket 65828 + */ + public function test_iptc_invalid_utf8_is_scrubbed() { + if ( ! is_callable( 'iptcembed' ) || ! is_callable( 'iptcparse' ) ) { + $this->markTestSkipped( 'The iptcembed() and iptcparse() functions are required.' ); + } + + $block = $this->build_iptc_block( + array( + 105 => mb_convert_encoding( 'wyróżnij', 'ISO-8859-2', 'UTF-8' ), + 120 => mb_convert_encoding( 'Café', 'ISO-8859-1', 'UTF-8' ), + 110 => "Credit \xC0", + 116 => "Copyright \xE9", + 25 => array( 'valid keyword', "sunset\xC0" ), + ) + ); + + $file = wp_tempnam( 'iptc-invalid-utf8.jpg' ); + file_put_contents( $file, iptcembed( $block, DIR_TESTDATA . '/images/test-image.jpg' ) ); + + $out = wp_read_image_metadata( $file ); + + unlink( $file ); + + $this->assertIsArray( $out, 'Metadata should have been read from the image.' ); + + foreach ( array( 'title', 'caption', 'credit', 'copyright' ) as $key ) { + $this->assertTrue( + wp_is_valid_utf8( $out[ $key ] ), + "The '{$key}' field should always be valid UTF-8." + ); + } + + $this->assertSame( + "Caf\u{FFFD}", + $out['caption'], + 'ISO-8859-1 text should be neutralized, not reinterpreted.' + ); + + $this->assertSame( + "wyr\u{FFFD}nij", + $out['title'], + 'ISO-8859-2 text should be neutralized, not reinterpreted.' + ); + + $this->assertSame( + array( 'valid keyword', "sunset\u{FFFD}" ), + $out['keywords'], + 'Keywords should be scrubbed individually while valid entries are left alone.' + ); + } + + /** + * Builds a raw IPTC APP13 block for the given record 2 datasets. + * + * @param array $tags Map of dataset number to a string value or list of string values. + * @return string Binary IPTC block suitable for iptcembed(). + */ + private function build_iptc_block( array $tags ) { + $block = ''; + + foreach ( $tags as $dataset => $values ) { + foreach ( (array) $values as $value ) { + $length = strlen( $value ); + $block .= chr( 0x1C ) . chr( 2 ) . chr( $dataset ) + . chr( ( $length >> 8 ) & 0xFF ) . chr( $length & 0xFF ) + . $value; + } + } + + return $block; + } + /** * wp_read_image_metadata() should return false if the image file doesn't exist. */