General: Replace core's remaining utf8_encode() calls with wp_scrub_utf8()

src/wp-admin/includes/export.php

@@ -243,11 +243,8 @@ function export_wp( $args = array() ) {
 	 * @return string
 	 */
 	function wxr_cdata( $str ) {
-		$str = (string) $str;
+		$str = wp_scrub_utf8( (string) $str );
 
-		if ( ! wp_is_valid_utf8( $str ) ) {
-			$str = utf8_encode( $str );
-		}
 		// $str = ent2ncr(esc_html($str));
 		$str = '<![CDATA[' . str_replace( ']]>', ']]]]><![CDATA[>', $str ) . ']]>';
 

src/wp-admin/includes/image.php

@@ -1048,16 +1048,12 @@ function wp_read_image_metadata( $file ) {
 	}
 
 	foreach ( array( 'title', 'caption', 'credit', 'copyright', 'camera', 'iso' ) as $key ) {
-		if ( $meta[ $key ] && ! wp_is_valid_utf8( $meta[ $key ] ) ) {
-			$meta[ $key ] = utf8_encode( $meta[ $key ] );
+		if ( $meta[ $key ] ) {
+			$meta[ $key ] = wp_scrub_utf8( $meta[ $key ] );
 		}
 	}
 
-	foreach ( $meta['keywords'] as $key => $keyword ) {
-		if ( ! wp_is_valid_utf8( $keyword ) ) {
-			$meta['keywords'][ $key ] = utf8_encode( $keyword );
-		}
-	}
+	$meta['keywords'] = array_map( 'wp_scrub_utf8', $meta['keywords'] );
 
 	$meta = wp_kses_post_deep( $meta );
 

tests/phpunit/tests/admin/exportWp.php

@@ -475,6 +475,86 @@ public function test_export_with_null_term_meta_values() {
 		$this->assertGreaterThan( 0, count( $xml->channel->item ), 'Export should contain items' );
 	}
 
+	/**
+	 * Ensures the WXR export always emits valid UTF-8.
+	 *
+	 * Byte spans which cannot decode as UTF-8 are replaced with the Unicode replacement
+	 * character. This covers both malformed UTF-8 and text supplied in some other
+	 * encoding entirely; core has no way to know which, so the bytes are neutralized
+	 * rather than reinterpreted as any particular encoding.
+	 *
+	 * @ticket 65828
+	 *
+	 * @dataProvider data_invalid_utf8_strings
+	 *
+	 * @param string $input    Bytes which are not valid UTF-8.
+	 * @param string $expected Expected CDATA contents.
+	 */
+	public function test_wxr_cdata_scrubs_invalid_utf8( $input, $expected ) {
+		// Running an export defines the nested WXR helper functions.
+		$this->get_the_export( array( 'content' => 'post' ) );
+
+		$actual = wxr_cdata( $input );
+		$inner  = substr( $actual, strlen( '<![CDATA[' ), -strlen( ']]>' ) );
+
+		$this->assertTrue(
+			wp_is_valid_utf8( $inner ),
+			'The exported CDATA section should always contain valid UTF-8.'
+		);
+		$this->assertSame(
+			$expected,
+			$inner,
+			'Byte spans which cannot decode as UTF-8 should be replaced.'
+		);
+	}
+
+	/**
+	 * Data provider.
+	 *
+	 * @return array[]
+	 */
+	public function data_invalid_utf8_strings() {
+		return array(
+			// Malformed UTF-8.
+			'Never-valid byte'        => array( "a\xC0b", "a\u{FFFD}b" ),
+			'Truncated sequence'      => array( "a\xE2\x9Cb", "a\u{FFFD}b" ),
+			'Overlong sequence'       => array( "a\xC1\xBFb", "a\u{FFFD}\u{FFFD}b" ),
+			'Surrogate half'          => array( "a\xED\xA0\x80b", "a\u{FFFD}\u{FFFD}\u{FFFD}b" ),
+
+			// Text which is well-formed, but in some encoding other than UTF-8.
+			'ISO-8859-1 text'         => array(
+				mb_convert_encoding( 'Café', 'ISO-8859-1', 'UTF-8' ),
+				"Caf\u{FFFD}",
+			),
+			'ISO-8859-2 text'         => array(
+				mb_convert_encoding( 'wyróżnij', 'ISO-8859-2', 'UTF-8' ),
+				"wyr\u{FFFD}nij",
+			),
+			'Windows-1251 text'       => array(
+				mb_convert_encoding( 'Привет', 'Windows-1251', 'UTF-8' ),
+				str_repeat( "\u{FFFD}", 6 ),
+			),
+			'Windows-1252 quotations' => array(
+				mb_convert_encoding( '“quoted”', 'Windows-1252', 'UTF-8' ),
+				"\u{FFFD}quoted\u{FFFD}",
+			),
+		);
+	}
+
+	/**
+	 * Ensures valid UTF-8, including multibyte text, survives the export untouched.
+	 *
+	 * @ticket 65828
+	 */
+	public function test_wxr_cdata_preserves_valid_utf8() {
+		$this->get_the_export( array( 'content' => 'post' ) );
+
+		$valid = 'Это комментарий. / Βλέπετε ένα σχόλιο. / 🅰';
+		$inner = substr( wxr_cdata( $valid ), strlen( '<![CDATA[' ), -strlen( ']]>' ) );
+
+		$this->assertSame( $valid, $inner, 'Valid UTF-8 should pass through unchanged.' );
+	}
+
 	/**
 	 * Ensure that posts types with 'can_export' set to false are not included in the export.
 	 *

tests/phpunit/tests/image/meta.php

@@ -149,6 +149,87 @@ public function test_utf8_iptc_tags() {
 		$this->assertSame( 'This is a comment. / Это комментарий. / Βλέπετε ένα σχόλιο.', $out['caption'] );
 	}
 
+	/**
+	 * Ensures image metadata is always returned as valid UTF-8.
+	 *
+	 * Core does not read the IPTC coded character set, so the encoding of these fields is
+	 * unknown. Byte spans which cannot decode as UTF-8 are replaced with the Unicode
+	 * replacement character rather than reinterpreted as any particular encoding. Both
+	 * malformed UTF-8 and text in another encoding are covered here.
+	 *
+	 * @ticket 65828
+	 */
+	public function test_iptc_invalid_utf8_is_scrubbed() {
+		if ( ! is_callable( 'iptcembed' ) || ! is_callable( 'iptcparse' ) ) {
+			$this->markTestSkipped( 'The iptcembed() and iptcparse() functions are required.' );
+		}
+
+		$block = $this->build_iptc_block(
+			array(
+				105 => mb_convert_encoding( 'wyróżnij', 'ISO-8859-2', 'UTF-8' ),
+				120 => mb_convert_encoding( 'Café', 'ISO-8859-1', 'UTF-8' ),
+				110 => "Credit \xC0",
+				116 => "Copyright \xE9",
+				25  => array( 'valid keyword', "sunset\xC0" ),
+			)
+		);
+
+		$file = wp_tempnam( 'iptc-invalid-utf8.jpg' );
+		file_put_contents( $file, iptcembed( $block, DIR_TESTDATA . '/images/test-image.jpg' ) );
+
+		$out = wp_read_image_metadata( $file );
+
+		unlink( $file );
+
+		$this->assertIsArray( $out, 'Metadata should have been read from the image.' );
+
+		foreach ( array( 'title', 'caption', 'credit', 'copyright' ) as $key ) {
+			$this->assertTrue(
+				wp_is_valid_utf8( $out[ $key ] ),
+				"The '{$key}' field should always be valid UTF-8."
+			);
+		}
+
+		$this->assertSame(
+			"Caf\u{FFFD}",
+			$out['caption'],
+			'ISO-8859-1 text should be neutralized, not reinterpreted.'
+		);
+
+		$this->assertSame(
+			"wyr\u{FFFD}nij",
+			$out['title'],
+			'ISO-8859-2 text should be neutralized, not reinterpreted.'
+		);
+
+		$this->assertSame(
+			array( 'valid keyword', "sunset\u{FFFD}" ),
+			$out['keywords'],
+			'Keywords should be scrubbed individually while valid entries are left alone.'
+		);
+	}
+
+	/**
+	 * Builds a raw IPTC APP13 block for the given record 2 datasets.
+	 *
+	 * @param array $tags Map of dataset number to a string value or list of string values.
+	 * @return string Binary IPTC block suitable for iptcembed().
+	 */
+	private function build_iptc_block( array $tags ) {
+		$block = '';
+
+		foreach ( $tags as $dataset => $values ) {
+			foreach ( (array) $values as $value ) {
+				$length = strlen( $value );
+				$block .= chr( 0x1C ) . chr( 2 ) . chr( $dataset )
+					. chr( ( $length >> 8 ) & 0xFF ) . chr( $length & 0xFF )
+					. $value;
+			}
+		}
+
+		return $block;
+	}
+
 	/**
 	 * wp_read_image_metadata() should return false if the image file doesn't exist.
 	 */