From d744f6b3cee82785965d053c0da265f292c725f7 Mon Sep 17 00:00:00 2001 From: Andreas Stefl Date: Sun, 6 Sep 2026 14:32:05 +0200 Subject: [PATCH] feat(bindings): mirror TextEncoding The enum never left C++. Java, python and objc each built a charset() string from TextFile::encoding() and stopped there, so nothing outside C++ could name an encoding it had not already been handed, or ask whether one is decodable. All four bindings now carry the enum and the five lookups, and a text file reports encoding() beside the charset() it keeps. wasm derives its table from all_text_encodings() rather than restating ordinals, the way FileType already does. Nothing is removed. CsvOptions carries a TextEncoding, so this is what the decode options need before they can cross a binding. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_016hxDa2rev11eLUEJZJ5nmz --- CHANGELOG.md | 4 + apple/include/OdrCoreObjC/ODRFile.h | 50 +++++++++ apple/include/OdrCoreObjC/ODROdr.h | 17 +++ apple/src/ODRFile.mm | 9 ++ apple/src/ODROdr.mm | 54 +++++++++ apple/tests/OdrCoreTests.swift | 33 ++++++ jni/CMakeLists.txt | 2 + .../app/opendocument/core/TextEncoding.java | 105 ++++++++++++++++++ jni/java/app/opendocument/core/TextFile.java | 17 ++- jni/src/jni_core.cpp | 48 ++++++++ jni/src/jni_file.cpp | 13 +-- .../opendocument/core/TextEncodingTest.java | 65 +++++++++++ python/src/bind_core.cpp | 27 +++++ python/src/bind_file.cpp | 48 ++++++++ python/tests/test_meta.py | 27 +++++ wasm/js/index.d.ts | 1 + wasm/src/wasm_core.cpp | 14 ++- wasm/tests/enums.test.mjs | 8 ++ 18 files changed, 528 insertions(+), 14 deletions(-) create mode 100644 jni/java/app/opendocument/core/TextEncoding.java create mode 100644 jni/tests/app/opendocument/core/TextEncodingTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 3056203ae..37db6f222 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,10 @@ The release run heads these entries with the version and opens a fresh ## Unreleased +- `TextEncoding` and its five lookups now reach the java, python, objc and wasm + bindings, and a text file reports `encoding()` beside the `charset()` it + keeps. wasm gets it as `Odr.enums.TextEncoding`. Nothing is removed. + - **Breaking**: `odr::open` is the only way to decode a file. The `DecodedFile` and `DocumentFile` constructors and `DocumentFile::from_disk`/`::from_memory` are gone, in every binding; narrow with `open(...).as_document_file()`. diff --git a/apple/include/OdrCoreObjC/ODRFile.h b/apple/include/OdrCoreObjC/ODRFile.h index 137ad8bea..a521c637b 100644 --- a/apple/include/OdrCoreObjC/ODRFile.h +++ b/apple/include/OdrCoreObjC/ODRFile.h @@ -115,6 +115,52 @@ typedef NS_ENUM(NSInteger, ODREncryptionState) { ODREncryptionStateDecrypted, } NS_SWIFT_NAME(EncryptionState); +/// A text encoding. Only some can be decoded — see +/// `Odr.isDecodable(textEncoding:)`; the rest can only be named. +typedef NS_ENUM(NSInteger, ODRTextEncoding) { + ODRTextEncodingUnknown = 0, + ODRTextEncodingUtf8, + ODRTextEncodingUtf16le, + ODRTextEncodingUtf16be, + ODRTextEncodingUtf32le, + ODRTextEncodingUtf32be, + ODRTextEncodingIbm866, + ODRTextEncodingIso88591, + ODRTextEncodingIso88592, + ODRTextEncodingIso88593, + ODRTextEncodingIso88594, + ODRTextEncodingIso88595, + ODRTextEncodingIso88596, + ODRTextEncodingIso88597, + ODRTextEncodingIso88598, + ODRTextEncodingIso885910, + ODRTextEncodingIso885913, + ODRTextEncodingIso885914, + ODRTextEncodingIso885915, + ODRTextEncodingIso885916, + ODRTextEncodingKoi8R, + ODRTextEncodingKoi8U, + ODRTextEncodingMacintosh, + ODRTextEncodingWindows874, + ODRTextEncodingWindows1250, + ODRTextEncodingWindows1251, + ODRTextEncodingWindows1252, + ODRTextEncodingWindows1253, + ODRTextEncodingWindows1254, + ODRTextEncodingWindows1255, + ODRTextEncodingWindows1256, + ODRTextEncodingWindows1257, + ODRTextEncodingWindows1258, + ODRTextEncodingXMacCyrillic, + ODRTextEncodingBig5, + ODRTextEncodingEucJp, + ODRTextEncodingEucKr, + ODRTextEncodingGb18030, + ODRTextEncodingIso2022Jp, + ODRTextEncodingIso2022Kr, + ODRTextEncodingShiftJis, +} NS_SWIFT_NAME(TextEncoding); + typedef NS_ENUM(NSInteger, ODRDocumentType) { ODRDocumentTypeUnknown = 0, ODRDocumentTypeText, @@ -298,7 +344,11 @@ NS_SWIFT_NAME(DecodedFile) /// A decoded text file — `odr::TextFile`. NS_SWIFT_NAME(TextFile) @interface ODRTextFile : ODRDecodedFile +/// The encoding the bytes were detected as, or decoded with. +@property(nonatomic, readonly) ODRTextEncoding encoding; /// The detected charset, `nil` if it could not be determined. +/// +/// Deprecated: read `encoding` and ask `Odr.string(textEncoding:)` for a name. @property(nonatomic, readonly, nullable, copy) NSString *charset; /// The decoded text. - (nullable NSString *)textWithError:(NSError **)error NS_SWIFT_NAME(text()); diff --git a/apple/include/OdrCoreObjC/ODROdr.h b/apple/include/OdrCoreObjC/ODROdr.h index ca342301d..60086800c 100644 --- a/apple/include/OdrCoreObjC/ODROdr.h +++ b/apple/include/OdrCoreObjC/ODROdr.h @@ -63,6 +63,23 @@ NS_SWIFT_NAME(Odr) + (ODRFileTypeCapabilities *)capabilitiesForFileType:(ODRFileType)type NS_SWIFT_NAME(capabilities(fileType:)); +/// Every text encoding the library knows about, excluding `unknown`. +@property(class, nonatomic, readonly) NSArray *allTextEncodings; +/// The canonical name, a label a browser accepts. `nil` for `unknown`. ++ (nullable NSString *)stringForTextEncoding:(ODRTextEncoding)encoding + NS_SWIFT_NAME(string(textEncoding:)); +/// The encoding for a name, `unknown` if none. Case and any `-`, `_` or space +/// are ignored. ++ (ODRTextEncoding)textEncodingForName:(NSString *)name + NS_SWIFT_NAME(textEncoding(name:)); +/// Every accepted name, canonical first. ++ (NSArray *)namesForTextEncoding:(ODRTextEncoding)encoding + NS_SWIFT_NAME(names(textEncoding:)); +/// Whether the library can decode this encoding, as opposed to merely naming +/// it. ++ (BOOL)isDecodableTextEncoding:(ODRTextEncoding)encoding + NS_SWIFT_NAME(isDecodable(textEncoding:)); + + (NSString *)stringForFileType:(ODRFileType)type NS_SWIFT_NAME(string(fileType:)); + (NSString *)stringForFileCategory:(ODRFileCategory)category diff --git a/apple/src/ODRFile.mm b/apple/src/ODRFile.mm index 918c54739..9f4c2c4f7 100644 --- a/apple/src/ODRFile.mm +++ b/apple/src/ODRFile.mm @@ -522,6 +522,15 @@ - (nullable ODRFontFile *)asFontFileWithError:(NSError **)error { @implementation ODRTextFile +- (ODRTextEncoding)encoding { + return guarded_value( + [&] { + return static_cast( + self.handle.as_text_file().encoding()); + }, + ODRTextEncodingUnknown); +} + - (nullable NSString *)charset { return guarded_value( [&]() -> NSString * { diff --git a/apple/src/ODROdr.mm b/apple/src/ODROdr.mm index 084474edb..b97a821a9 100644 --- a/apple/src/ODROdr.mm +++ b/apple/src/ODROdr.mm @@ -146,6 +146,60 @@ + (ODRFileTypeCapabilities *)capabilitiesForFileType:(ODRFileType)type { nil); } ++ (NSArray *)allTextEncodings { + return guarded_value( + [&]() -> NSArray * { + const std::vector encodings = + odr::all_text_encodings(); + NSMutableArray *const result = + [NSMutableArray arrayWithCapacity:encodings.size()]; + for (const odr::TextEncoding encoding : encodings) { + [result addObject:@(static_cast(encoding))]; + } + return result; + }, + @[]); +} + ++ (nullable NSString *)stringForTextEncoding:(ODRTextEncoding)encoding { + // throws for `unknown`, which has no name; nil says so without an NSError + return guarded_value( + [&]() -> NSString * { + return to_nsstring(std::string(odr::text_encoding_to_string( + static_cast(encoding)))); + }, + nil); +} + ++ (ODRTextEncoding)textEncodingForName:(NSString *)name { + return guarded_value( + [&] { + return static_cast( + odr::text_encoding_by_name(to_string(name))); + }, + ODRTextEncodingUnknown); +} + ++ (NSArray *)namesForTextEncoding:(ODRTextEncoding)encoding { + return guarded_value( + [&] { + return to_nsarray( + odr::text_encoding_names(static_cast(encoding))); + }, + @[]); +} + ++ (BOOL)isDecodableTextEncoding:(ODRTextEncoding)encoding { + return guarded_value( + [&] { + return odr::text_encoding_is_decodable( + static_cast(encoding)) + ? YES + : NO; + }, + NO); +} + + (NSString *)stringForFileType:(ODRFileType)type { return guarded_value( [&] { diff --git a/apple/tests/OdrCoreTests.swift b/apple/tests/OdrCoreTests.swift index e5f503fff..267318c79 100644 --- a/apple/tests/OdrCoreTests.swift +++ b/apple/tests/OdrCoreTests.swift @@ -47,6 +47,39 @@ final class FileTypeTableTests: XCTestCase { } } +final class TextEncodingTests: XCTestCase { + func testNamesResolveBothWays() { + XCTAssertEqual(Odr.string(textEncoding: .windows1252), "windows-1252") + // case and separators are ignored, so an alias resolves + XCTAssertEqual(Odr.textEncoding(name: "CP1252"), .windows1252) + XCTAssertEqual(Odr.textEncoding(name: "definitely-not-an-encoding"), .unknown) + XCTAssertTrue(Odr.names(textEncoding: .windows1252).contains("windows-1252")) + } + + /// `unknown` is the one with no name, and says so rather than inventing one. + func testUnknownHasNoName() { + XCTAssertNil(Odr.string(textEncoding: .unknown)) + } + + func testAllLeavesOutUnknown() { + let all = Odr.allTextEncodings.map { TextEncoding(rawValue: $0.intValue) } + XCTAssertTrue(all.contains(.utf8)) + XCTAssertFalse(all.contains(.unknown)) + } + + func testDecodableIsNarrowerThanNamed() { + XCTAssertTrue(Odr.isDecodable(textEncoding: .utf8)) + // named so a caller can say what a file is, but not decoded here + XCTAssertFalse(Odr.isDecodable(textEncoding: .shiftJis)) + } + + func testATextFileReportsItsEncoding() throws { + let path = try write("hello", as: "plain.txt") + let decoded = try DecodedFile.decode(path: path) + XCTAssertEqual((decoded as! TextFile).encoding, .utf8) + } +} + final class DecodeTests: XCTestCase { /// The non-BMP character is the point of the payload: `to_nsstring` converts /// UTF-8 to UTF-16, and a 😀 is a surrogate pair on the way out. diff --git a/jni/CMakeLists.txt b/jni/CMakeLists.txt index 7f528d273..225f3148c 100644 --- a/jni/CMakeLists.txt +++ b/jni/CMakeLists.txt @@ -149,6 +149,7 @@ add_jar(odr_java "java/app/opendocument/core/TableColumnStyle.java" "java/app/opendocument/core/TableDimensions.java" "java/app/opendocument/core/TablePosition.java" + "java/app/opendocument/core/TextEncoding.java" "java/app/opendocument/core/TableRow.java" "java/app/opendocument/core/TableRowStyle.java" "java/app/opendocument/core/TableStyle.java" @@ -196,6 +197,7 @@ if (ODR_TEST AND NOT ANDROID) "tests/app/opendocument/core/HttpServerTest.java" "tests/app/opendocument/core/LoggerTest.java" "tests/app/opendocument/core/MetaTest.java" + "tests/app/opendocument/core/TextEncodingTest.java" # shared with the instrumented suite of the AAR, see `android/` "testfixtures/app/opendocument/core/TestFiles.java" # `RESOURCES NAMESPACE` rather than listing the file under SOURCES: diff --git a/jni/java/app/opendocument/core/TextEncoding.java b/jni/java/app/opendocument/core/TextEncoding.java new file mode 100644 index 000000000..5259a0888 --- /dev/null +++ b/jni/java/app/opendocument/core/TextEncoding.java @@ -0,0 +1,105 @@ +package app.opendocument.core; + +/** Mirrors {@code odr::TextEncoding}; constant order must match the C++ declaration. */ +public enum TextEncoding { + UNKNOWN, + UTF8, + UTF16LE, + UTF16BE, + UTF32LE, + UTF32BE, + IBM866, + ISO_8859_1, + ISO_8859_2, + ISO_8859_3, + ISO_8859_4, + ISO_8859_5, + ISO_8859_6, + ISO_8859_7, + ISO_8859_8, + ISO_8859_10, + ISO_8859_13, + ISO_8859_14, + ISO_8859_15, + ISO_8859_16, + KOI8_R, + KOI8_U, + MACINTOSH, + WINDOWS_874, + WINDOWS_1250, + WINDOWS_1251, + WINDOWS_1252, + WINDOWS_1253, + WINDOWS_1254, + WINDOWS_1255, + WINDOWS_1256, + WINDOWS_1257, + WINDOWS_1258, + X_MAC_CYRILLIC, + BIG5, + EUC_JP, + EUC_KR, + GB18030, + ISO_2022_JP, + ISO_2022_KR, + SHIFT_JIS; + + static TextEncoding fromNative(int code) { + return code < 0 ? null : values()[code]; + } + + int toNative() { + return ordinal(); + } + + /** + * The canonical name, a label a browser accepts. + * + * @throws OdrException for {@link #UNKNOWN}, which has no name + */ + public String canonicalName() { + return toStringNative(toNative()); + } + + /** Every accepted name, canonical first. */ + public String[] names() { + return namesNative(toNative()); + } + + /** Whether the library can decode this encoding, as opposed to merely naming it. */ + public boolean isDecodable() { + return isDecodableNative(toNative()); + } + + /** + * The encoding for a name, {@link #UNKNOWN} if none. Case and any {@code -}, {@code _} or + * space are ignored. + */ + public static TextEncoding byName(String name) { + return fromNative(byNameNative(name)); + } + + /** Every encoding the library knows about, excluding {@link #UNKNOWN}. */ + public static TextEncoding[] all() { + int[] codes = allNative(); + TextEncoding[] result = new TextEncoding[codes.length]; + for (int i = 0; i < codes.length; i++) { + result[i] = fromNative(codes[i]); + } + return result; + } + + static { + NativeLibrary.load(); + } + + private static native String toStringNative(int encoding); + + private static native String[] namesNative(int encoding); + + private static native boolean isDecodableNative(int encoding); + + private static native int byNameNative(String name); + + private static native int[] allNative(); +} diff --git a/jni/java/app/opendocument/core/TextFile.java b/jni/java/app/opendocument/core/TextFile.java index 9b72fe0e0..3dc25b2b4 100644 --- a/jni/java/app/opendocument/core/TextFile.java +++ b/jni/java/app/opendocument/core/TextFile.java @@ -6,16 +6,27 @@ public final class TextFile extends DecodedFile { super(handle); } - /** Detected character set; {@code null} if unknown. */ + /** The encoding the bytes were detected as, or decoded with. */ + public TextEncoding encoding() { + return TextEncoding.fromNative(encodingNative(handle())); + } + + /** + * Detected character set; {@code null} if unknown. + * + * @deprecated use {@link #encoding()} + */ + @Deprecated public String charset() { - return charsetNative(handle()); + TextEncoding encoding = encoding(); + return encoding == TextEncoding.UNKNOWN ? null : encoding.canonicalName(); } public String text() { return textNative(handle()); } - private native String charsetNative(long handle); + private native int encodingNative(long handle); private native String textNative(long handle); } diff --git a/jni/src/jni_core.cpp b/jni/src/jni_core.cpp index 08470776e..30c06508d 100644 --- a/jni/src/jni_core.cpp +++ b/jni/src/jni_core.cpp @@ -266,6 +266,54 @@ Java_app_opendocument_core_Odr_openWithPreferenceNative( }); } +// app.opendocument.core.TextEncoding + +extern "C" JNIEXPORT jstring JNICALL +Java_app_opendocument_core_TextEncoding_toStringNative(JNIEnv *env, jclass, + jint encoding) { + return guarded(env, [&] { + return to_jstring(env, odr::text_encoding_to_string( + static_cast(encoding))); + }); +} + +extern "C" JNIEXPORT jobjectArray JNICALL +Java_app_opendocument_core_TextEncoding_namesNative(JNIEnv *env, jclass, + jint encoding) { + return guarded(env, [&] { + return to_jstring_array(env, odr::text_encoding_names( + static_cast(encoding))); + }); +} + +extern "C" JNIEXPORT jboolean JNICALL +Java_app_opendocument_core_TextEncoding_isDecodableNative(JNIEnv *env, jclass, + jint encoding) { + return guarded(env, [&] { + return static_cast(odr::text_encoding_is_decodable( + static_cast(encoding))); + }); +} + +extern "C" JNIEXPORT jint JNICALL +Java_app_opendocument_core_TextEncoding_byNameNative(JNIEnv *env, jclass, + jstring name) { + return guarded(env, [&] { + return static_cast(odr::text_encoding_by_name(to_string(env, name))); + }); +} + +extern "C" JNIEXPORT jintArray JNICALL +Java_app_opendocument_core_TextEncoding_allNative(JNIEnv *env, jclass) { + return guarded(env, [&] { + std::vector codes; + for (const odr::TextEncoding encoding : odr::all_text_encodings()) { + codes.push_back(static_cast(encoding)); + } + return to_jint_array(env, codes); + }); +} + // app.opendocument.core.TablePosition extern "C" JNIEXPORT jint JNICALL diff --git a/jni/src/jni_file.cpp b/jni/src/jni_file.cpp index 8e836122a..5477025e3 100644 --- a/jni/src/jni_file.cpp +++ b/jni/src/jni_file.cpp @@ -277,16 +277,11 @@ Java_app_opendocument_core_DecodedFile_asFontFileNative(JNIEnv *env, jobject, // app.opendocument.core.TextFile -extern "C" JNIEXPORT jstring JNICALL -Java_app_opendocument_core_TextFile_charsetNative(JNIEnv *env, jobject, - jlong handle) { +extern "C" JNIEXPORT jint JNICALL +Java_app_opendocument_core_TextFile_encodingNative(JNIEnv *env, jobject, + jlong handle) { return guarded(env, [&] { - const odr::TextEncoding encoding = - decoded(handle).as_text_file().encoding(); - return odr_jni::make_string_opt( - env, encoding == odr::TextEncoding::unknown - ? std::optional{} - : std::string(odr::text_encoding_to_string(encoding))); + return static_cast(decoded(handle).as_text_file().encoding()); }); } diff --git a/jni/tests/app/opendocument/core/TextEncodingTest.java b/jni/tests/app/opendocument/core/TextEncodingTest.java new file mode 100644 index 000000000..d29d38c8c --- /dev/null +++ b/jni/tests/app/opendocument/core/TextEncodingTest.java @@ -0,0 +1,65 @@ +package app.opendocument.core; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class TextEncodingTest { + @TempDir Path tempDir; + + @Test + void ordinalsMatchTheNativeEnum() { + // the mapping is by ordinal, so the two ends have to agree on the first + assertEquals(0, TextEncoding.UNKNOWN.ordinal()); + assertEquals("UTF-8", TextEncoding.UTF8.canonicalName()); + } + + @Test + void namesResolveBothWays() { + assertEquals("windows-1252", TextEncoding.WINDOWS_1252.canonicalName()); + // case and separators are ignored, so an alias resolves + assertEquals(TextEncoding.WINDOWS_1252, TextEncoding.byName("CP1252")); + assertEquals(TextEncoding.UNKNOWN, TextEncoding.byName("nope")); + assertTrue(Arrays.asList(TextEncoding.WINDOWS_1252.names()).contains("windows-1252")); + } + + @Test + void unknownHasNoName() { + assertThrows(OdrException.class, TextEncoding.UNKNOWN::canonicalName); + } + + @Test + void decodableIsNarrowerThanNamed() { + assertTrue(TextEncoding.UTF8.isDecodable()); + // named so a caller can say what a file is, but not decoded here + assertFalse(TextEncoding.SHIFT_JIS.isDecodable()); + } + + @Test + void allLeavesOutUnknown() { + TextEncoding[] all = TextEncoding.all(); + assertTrue(Arrays.asList(all).contains(TextEncoding.UTF8)); + assertFalse(Arrays.asList(all).contains(TextEncoding.UNKNOWN)); + } + + @Test + void aTextFileReportsItsEncoding() throws IOException { + Path path = Files.writeString(tempDir.resolve("plain.txt"), "hello"); + + try (DecodedFile decoded = Odr.open(path.toString())) { + TextFile file = decoded.asTextFile(); + assertEquals(TextEncoding.UTF8, file.encoding()); + assertEquals("UTF-8", file.charset()); + } + } +} diff --git a/python/src/bind_core.cpp b/python/src/bind_core.cpp index b7eff768d..e9d369f59 100644 --- a/python/src/bind_core.cpp +++ b/python/src/bind_core.cpp @@ -58,6 +58,33 @@ void odr_python::bind_core(py::module_ &m) { void odr_python::bind_functions(py::module_ &m) { m.def("all_file_types", &odr::all_file_types, "Every file type this library knows about."); + m.def("all_text_encodings", &odr::all_text_encodings, + "Every text encoding the library knows about, excluding `unknown`."); + m.def( + "text_encoding_to_string", + [](const odr::TextEncoding encoding) { + return std::string(odr::text_encoding_to_string(encoding)); + }, + py::arg("encoding"), + "The canonical name, a label a browser accepts. Raises for `unknown`."); + m.def("text_encoding_by_name", &odr::text_encoding_by_name, py::arg("name"), + "The encoding for a name, `unknown` if none. Case and any `-`, `_` or " + "space are ignored."); + m.def( + "text_encoding_names", + [](const odr::TextEncoding encoding) { + std::vector result; + for (const std::string_view name : odr::text_encoding_names(encoding)) { + result.emplace_back(name); + } + return result; + }, + py::arg("encoding"), "Every accepted name, canonical first."); + m.def("text_encoding_is_decodable", &odr::text_encoding_is_decodable, + py::arg("encoding"), + "Whether the library can decode this encoding, as opposed to merely " + "naming it."); + m.def("file_type_by_file_extension", &odr::file_type_by_file_extension, py::arg("extension")); m.def( diff --git a/python/src/bind_file.cpp b/python/src/bind_file.cpp index af713efb4..199a18946 100644 --- a/python/src/bind_file.cpp +++ b/python/src/bind_file.cpp @@ -100,6 +100,52 @@ void odr_python::bind_file(py::module_ &m) { .value("audio", odr::FileCategory::audio) .value("video", odr::FileCategory::video); + py::enum_( + m, "TextEncoding", + "A text encoding. Only some can be decoded - see\n" + "`text_encoding_is_decodable`.") + .value("unknown", odr::TextEncoding::unknown) + .value("utf8", odr::TextEncoding::utf8) + .value("utf16le", odr::TextEncoding::utf16le) + .value("utf16be", odr::TextEncoding::utf16be) + .value("utf32le", odr::TextEncoding::utf32le) + .value("utf32be", odr::TextEncoding::utf32be) + .value("ibm866", odr::TextEncoding::ibm866) + .value("iso_8859_1", odr::TextEncoding::iso_8859_1) + .value("iso_8859_2", odr::TextEncoding::iso_8859_2) + .value("iso_8859_3", odr::TextEncoding::iso_8859_3) + .value("iso_8859_4", odr::TextEncoding::iso_8859_4) + .value("iso_8859_5", odr::TextEncoding::iso_8859_5) + .value("iso_8859_6", odr::TextEncoding::iso_8859_6) + .value("iso_8859_7", odr::TextEncoding::iso_8859_7) + .value("iso_8859_8", odr::TextEncoding::iso_8859_8) + .value("iso_8859_10", odr::TextEncoding::iso_8859_10) + .value("iso_8859_13", odr::TextEncoding::iso_8859_13) + .value("iso_8859_14", odr::TextEncoding::iso_8859_14) + .value("iso_8859_15", odr::TextEncoding::iso_8859_15) + .value("iso_8859_16", odr::TextEncoding::iso_8859_16) + .value("koi8_r", odr::TextEncoding::koi8_r) + .value("koi8_u", odr::TextEncoding::koi8_u) + .value("macintosh", odr::TextEncoding::macintosh) + .value("windows_874", odr::TextEncoding::windows_874) + .value("windows_1250", odr::TextEncoding::windows_1250) + .value("windows_1251", odr::TextEncoding::windows_1251) + .value("windows_1252", odr::TextEncoding::windows_1252) + .value("windows_1253", odr::TextEncoding::windows_1253) + .value("windows_1254", odr::TextEncoding::windows_1254) + .value("windows_1255", odr::TextEncoding::windows_1255) + .value("windows_1256", odr::TextEncoding::windows_1256) + .value("windows_1257", odr::TextEncoding::windows_1257) + .value("windows_1258", odr::TextEncoding::windows_1258) + .value("x_mac_cyrillic", odr::TextEncoding::x_mac_cyrillic) + .value("big5", odr::TextEncoding::big5) + .value("euc_jp", odr::TextEncoding::euc_jp) + .value("euc_kr", odr::TextEncoding::euc_kr) + .value("gb18030", odr::TextEncoding::gb18030) + .value("iso_2022_jp", odr::TextEncoding::iso_2022_jp) + .value("iso_2022_kr", odr::TextEncoding::iso_2022_kr) + .value("shift_jis", odr::TextEncoding::shift_jis); + py::enum_(m, "FileLocation") .value("unknown", odr::FileLocation::unknown) .value("memory", odr::FileLocation::memory) @@ -213,6 +259,8 @@ void odr_python::bind_file(py::module_ &m) { .def("as_font_file", &odr::DecodedFile::as_font_file); py::class_(m, "TextFile") + .def("encoding", &odr::TextFile::encoding, + "The encoding the bytes were detected as, or decoded with.") .def("charset", [](const odr::TextFile &file) -> std::optional { const odr::TextEncoding encoding = file.encoding(); diff --git a/python/tests/test_meta.py b/python/tests/test_meta.py index 6421481ed..d28d0f7ba 100644 --- a/python/tests/test_meta.py +++ b/python/tests/test_meta.py @@ -128,3 +128,30 @@ def test_mimetype_names_what_is_inside_the_container(odt_path): # An ODF file is a ZIP, and the answer worth having is the one from inside # it. Detection opens the container to get there. assert pyodr.mimetype(str(odt_path)) == "application/vnd.oasis.opendocument.text" + + +def test_text_encoding_lookups(): + assert pyodr.TextEncoding.utf8 in pyodr.all_text_encodings() + # `unknown` is the one with no name, and is left out + assert pyodr.TextEncoding.unknown not in pyodr.all_text_encodings() + + assert ( + pyodr.text_encoding_to_string(pyodr.TextEncoding.windows_1252) == "windows-1252" + ) + # case and separators are ignored, so an alias resolves + assert pyodr.text_encoding_by_name("CP1252") == pyodr.TextEncoding.windows_1252 + assert pyodr.text_encoding_by_name("nope") == pyodr.TextEncoding.unknown + assert "windows-1252" in pyodr.text_encoding_names(pyodr.TextEncoding.windows_1252) + + assert pyodr.text_encoding_is_decodable(pyodr.TextEncoding.utf8) + # named so a caller can say what a file is, but not decoded here + assert not pyodr.text_encoding_is_decodable(pyodr.TextEncoding.shift_jis) + + +def test_text_file_reports_its_encoding(tmp_path): + path = tmp_path / "plain.txt" + path.write_text("hello", encoding="utf-8") + + text_file = pyodr.open(str(path)).as_text_file() + assert text_file.encoding() != pyodr.TextEncoding.unknown + assert isinstance(text_file.charset(), str) diff --git a/wasm/js/index.d.ts b/wasm/js/index.d.ts index b73b7c01e..b2bc94035 100644 --- a/wasm/js/index.d.ts +++ b/wasm/js/index.d.ts @@ -5,6 +5,7 @@ export interface EnumTables { FileType: Record; FileCategory: Record; + TextEncoding: Record; DocumentType: Record; HtmlResourceType: Record; HtmlColorScheme: Record; diff --git a/wasm/src/wasm_core.cpp b/wasm/src/wasm_core.cpp index 63aee1f79..437678fbd 100644 --- a/wasm/src/wasm_core.cpp +++ b/wasm/src/wasm_core.cpp @@ -52,8 +52,9 @@ emscripten::val file_types() { } /// Enum name to ordinal, so the JS side never restates an ordinal by hand. -/// `FileType`, `FileCategory` and `DocumentType` are derived from the library's -/// tables and cannot drift; the rest have no runtime table and are listed here, +/// `FileType`, `FileCategory`, `DocumentType` and `TextEncoding` are derived +/// from the library's tables and cannot drift; the rest have no runtime table +/// and are listed here, /// pinned by `tests/enums.test.mjs`. emscripten::val enum_tables() { const auto table = [](const auto &...entries) { @@ -87,8 +88,17 @@ emscripten::val enum_tables() { static_cast(type)); } + // `all_text_encodings` leaves `unknown` out, and it is the one with no name + emscripten::val text_encoding = emscripten::val::object(); + text_encoding.set("unknown", static_cast(TextEncoding::unknown)); + for (const TextEncoding encoding : odr::all_text_encodings()) { + text_encoding.set(std::string(odr::text_encoding_to_string(encoding)), + static_cast(encoding)); + } + emscripten::val result = emscripten::val::object(); result.set("FileType", file_type); + result.set("TextEncoding", text_encoding); result.set("FileCategory", file_category); result.set("DocumentType", document_type); result.set("HtmlResourceType", diff --git a/wasm/tests/enums.test.mjs b/wasm/tests/enums.test.mjs index 060b448cc..b6ad45c21 100644 --- a/wasm/tests/enums.test.mjs +++ b/wasm/tests/enums.test.mjs @@ -62,6 +62,14 @@ describe('enums', () => { }); } + it('derives TextEncoding from the library, unknown first', () => { + assert.equal(enums.TextEncoding.unknown, 0); + // keyed by the canonical name, as `FileType` is keyed by `odt` + assert.equal(enums.TextEncoding['UTF-8'], 1); + // a named-but-not-decodable one is still in the table + assert.equal(typeof enums.TextEncoding['Shift_JIS'], 'number'); + }); + it('derives FileType from the library, unknown first', () => { assert.equal(enums.FileType.unknown, 0); assert.equal(typeof enums.FileType.odt, 'number');