feat(bindings): mirror TextEncoding

CHANGELOG.md

@@ -16,6 +16,10 @@ The release run heads these entries with the version and opens a fresh
 
 ## Unreleased
 
+- `TextEncoding` and its five lookups now reach the java, python, objc and wasm
+  bindings, and a text file reports `encoding()` beside the `charset()` it
+  keeps. wasm gets it as `Odr.enums.TextEncoding`. Nothing is removed.
+
 - **Breaking**: `odr::open` is the only way to decode a file. The `DecodedFile`
   and `DocumentFile` constructors and `DocumentFile::from_disk`/`::from_memory`
   are gone, in every binding; narrow with `open(...).as_document_file()`.

apple/include/OdrCoreObjC/ODRFile.h

@@ -115,6 +115,52 @@ typedef NS_ENUM(NSInteger, ODREncryptionState) {
   ODREncryptionStateDecrypted,
 } NS_SWIFT_NAME(EncryptionState);
 
+/// A text encoding. Only some can be decoded — see
+/// `Odr.isDecodable(textEncoding:)`; the rest can only be named.
+typedef NS_ENUM(NSInteger, ODRTextEncoding) {
+  ODRTextEncodingUnknown = 0,
+  ODRTextEncodingUtf8,
+  ODRTextEncodingUtf16le,
+  ODRTextEncodingUtf16be,
+  ODRTextEncodingUtf32le,
+  ODRTextEncodingUtf32be,
+  ODRTextEncodingIbm866,
+  ODRTextEncodingIso88591,
+  ODRTextEncodingIso88592,
+  ODRTextEncodingIso88593,
+  ODRTextEncodingIso88594,
+  ODRTextEncodingIso88595,
+  ODRTextEncodingIso88596,
+  ODRTextEncodingIso88597,
+  ODRTextEncodingIso88598,
+  ODRTextEncodingIso885910,
+  ODRTextEncodingIso885913,
+  ODRTextEncodingIso885914,
+  ODRTextEncodingIso885915,
+  ODRTextEncodingIso885916,
+  ODRTextEncodingKoi8R,
+  ODRTextEncodingKoi8U,
+  ODRTextEncodingMacintosh,
+  ODRTextEncodingWindows874,
+  ODRTextEncodingWindows1250,
+  ODRTextEncodingWindows1251,
+  ODRTextEncodingWindows1252,
+  ODRTextEncodingWindows1253,
+  ODRTextEncodingWindows1254,
+  ODRTextEncodingWindows1255,
+  ODRTextEncodingWindows1256,
+  ODRTextEncodingWindows1257,
+  ODRTextEncodingWindows1258,
+  ODRTextEncodingXMacCyrillic,
+  ODRTextEncodingBig5,
+  ODRTextEncodingEucJp,
+  ODRTextEncodingEucKr,
+  ODRTextEncodingGb18030,
+  ODRTextEncodingIso2022Jp,
+  ODRTextEncodingIso2022Kr,
+  ODRTextEncodingShiftJis,
+} NS_SWIFT_NAME(TextEncoding);
+
 typedef NS_ENUM(NSInteger, ODRDocumentType) {
   ODRDocumentTypeUnknown = 0,
   ODRDocumentTypeText,
@@ -298,7 +344,11 @@ NS_SWIFT_NAME(DecodedFile)
 /// A decoded text file — `odr::TextFile`.
 NS_SWIFT_NAME(TextFile)
 @interface ODRTextFile : ODRDecodedFile
+/// The encoding the bytes were detected as, or decoded with.
+@property(nonatomic, readonly) ODRTextEncoding encoding;
 /// The detected charset, `nil` if it could not be determined.
+///
+/// Deprecated: read `encoding` and ask `Odr.string(textEncoding:)` for a name.
 @property(nonatomic, readonly, nullable, copy) NSString *charset;
 /// The decoded text.
 - (nullable NSString *)textWithError:(NSError **)error NS_SWIFT_NAME(text());

apple/include/OdrCoreObjC/ODROdr.h

@@ -63,6 +63,23 @@ NS_SWIFT_NAME(Odr)
 + (ODRFileTypeCapabilities *)capabilitiesForFileType:(ODRFileType)type
     NS_SWIFT_NAME(capabilities(fileType:));
 
+/// Every text encoding the library knows about, excluding `unknown`.
+@property(class, nonatomic, readonly) NSArray<NSNumber *> *allTextEncodings;
+/// The canonical name, a label a browser accepts. `nil` for `unknown`.
++ (nullable NSString *)stringForTextEncoding:(ODRTextEncoding)encoding
+    NS_SWIFT_NAME(string(textEncoding:));
+/// The encoding for a name, `unknown` if none. Case and any `-`, `_` or space
+/// are ignored.
++ (ODRTextEncoding)textEncodingForName:(NSString *)name
+    NS_SWIFT_NAME(textEncoding(name:));
+/// Every accepted name, canonical first.
++ (NSArray<NSString *> *)namesForTextEncoding:(ODRTextEncoding)encoding
+    NS_SWIFT_NAME(names(textEncoding:));
+/// Whether the library can decode this encoding, as opposed to merely naming
+/// it.
++ (BOOL)isDecodableTextEncoding:(ODRTextEncoding)encoding
+    NS_SWIFT_NAME(isDecodable(textEncoding:));
+
 + (NSString *)stringForFileType:(ODRFileType)type
     NS_SWIFT_NAME(string(fileType:));
 + (NSString *)stringForFileCategory:(ODRFileCategory)category

apple/src/ODRFile.mm

@@ -522,6 +522,15 @@ - (nullable ODRFontFile *)asFontFileWithError:(NSError **)error {
 
 @implementation ODRTextFile
 
+- (ODRTextEncoding)encoding {
+  return guarded_value(
+      [&] {
+        return static_cast<ODRTextEncoding>(
+            self.handle.as_text_file().encoding());
+      },
+      ODRTextEncodingUnknown);
+}
+
 - (nullable NSString *)charset {
   return guarded_value(
       [&]() -> NSString * {

apple/src/ODROdr.mm

@@ -146,6 +146,60 @@ + (ODRFileTypeCapabilities *)capabilitiesForFileType:(ODRFileType)type {
       nil);
 }
 
++ (NSArray<NSNumber *> *)allTextEncodings {
+  return guarded_value(
+      [&]() -> NSArray<NSNumber *> * {
+        const std::vector<odr::TextEncoding> encodings =
+            odr::all_text_encodings();
+        NSMutableArray<NSNumber *> *const result =
+            [NSMutableArray arrayWithCapacity:encodings.size()];
+        for (const odr::TextEncoding encoding : encodings) {
+          [result addObject:@(static_cast<NSInteger>(encoding))];
+        }
+        return result;
+      },
+      @[]);
+}
+
++ (nullable NSString *)stringForTextEncoding:(ODRTextEncoding)encoding {
+  // throws for `unknown`, which has no name; nil says so without an NSError
+  return guarded_value(
+      [&]() -> NSString * {
+        return to_nsstring(std::string(odr::text_encoding_to_string(
+            static_cast<odr::TextEncoding>(encoding))));
+      },
+      nil);
+}
+
++ (ODRTextEncoding)textEncodingForName:(NSString *)name {
+  return guarded_value(
+      [&] {
+        return static_cast<ODRTextEncoding>(
+            odr::text_encoding_by_name(to_string(name)));
+      },
+      ODRTextEncodingUnknown);
+}
+
++ (NSArray<NSString *> *)namesForTextEncoding:(ODRTextEncoding)encoding {
+  return guarded_value(
+      [&] {
+        return to_nsarray(
+            odr::text_encoding_names(static_cast<odr::TextEncoding>(encoding)));
+      },
+      @[]);
+}
+
++ (BOOL)isDecodableTextEncoding:(ODRTextEncoding)encoding {
+  return guarded_value(
+      [&] {
+        return odr::text_encoding_is_decodable(
+                   static_cast<odr::TextEncoding>(encoding))
+                   ? YES
+                   : NO;
+      },
+      NO);
+}
+
 + (NSString *)stringForFileType:(ODRFileType)type {
   return guarded_value(
       [&] {

apple/tests/OdrCoreTests.swift

@@ -47,6 +47,39 @@ final class FileTypeTableTests: XCTestCase {
   }
 }
 
+final class TextEncodingTests: XCTestCase {
+  func testNamesResolveBothWays() {
+    XCTAssertEqual(Odr.string(textEncoding: .windows1252), "windows-1252")
+    // case and separators are ignored, so an alias resolves
+    XCTAssertEqual(Odr.textEncoding(name: "CP1252"), .windows1252)
+    XCTAssertEqual(Odr.textEncoding(name: "definitely-not-an-encoding"), .unknown)
+    XCTAssertTrue(Odr.names(textEncoding: .windows1252).contains("windows-1252"))
+  }
+
+  /// `unknown` is the one with no name, and says so rather than inventing one.
+  func testUnknownHasNoName() {
+    XCTAssertNil(Odr.string(textEncoding: .unknown))
+  }
+
+  func testAllLeavesOutUnknown() {
+    let all = Odr.allTextEncodings.map { TextEncoding(rawValue: $0.intValue) }
+    XCTAssertTrue(all.contains(.utf8))
+    XCTAssertFalse(all.contains(.unknown))
+  }
+
+  func testDecodableIsNarrowerThanNamed() {
+    XCTAssertTrue(Odr.isDecodable(textEncoding: .utf8))
+    // named so a caller can say what a file is, but not decoded here
+    XCTAssertFalse(Odr.isDecodable(textEncoding: .shiftJis))
+  }
+
+  func testATextFileReportsItsEncoding() throws {
+    let path = try write("hello", as: "plain.txt")
+    let decoded = try DecodedFile.decode(path: path)
+    XCTAssertEqual((decoded as! TextFile).encoding, .utf8)
+  }
+}
+
 final class DecodeTests: XCTestCase {
   /// The non-BMP character is the point of the payload: `to_nsstring` converts
   /// UTF-8 to UTF-16, and a 😀 is a surrogate pair on the way out.

jni/CMakeLists.txt

@@ -149,6 +149,7 @@ add_jar(odr_java
         "java/app/opendocument/core/TableColumnStyle.java"
         "java/app/opendocument/core/TableDimensions.java"
         "java/app/opendocument/core/TablePosition.java"
+        "java/app/opendocument/core/TextEncoding.java"
         "java/app/opendocument/core/TableRow.java"
         "java/app/opendocument/core/TableRowStyle.java"
         "java/app/opendocument/core/TableStyle.java"
@@ -196,6 +197,7 @@ if (ODR_TEST AND NOT ANDROID)
             "tests/app/opendocument/core/HttpServerTest.java"
             "tests/app/opendocument/core/LoggerTest.java"
             "tests/app/opendocument/core/MetaTest.java"
+            "tests/app/opendocument/core/TextEncodingTest.java"
             # shared with the instrumented suite of the AAR, see `android/`
             "testfixtures/app/opendocument/core/TestFiles.java"
             # `RESOURCES NAMESPACE` rather than listing the file under SOURCES:

jni/java/app/opendocument/core/TextEncoding.java

@@ -0,0 +1,105 @@
+package app.opendocument.core;
+
+/** Mirrors {@code odr::TextEncoding}; constant order must match the C++ declaration. */
+public enum TextEncoding {
+  UNKNOWN,
+  UTF8,
+  UTF16LE,
+  UTF16BE,
+  UTF32LE,
+  UTF32BE,
+  IBM866,
+  ISO_8859_1,
+  ISO_8859_2,
+  ISO_8859_3,
+  ISO_8859_4,
+  ISO_8859_5,
+  ISO_8859_6,
+  ISO_8859_7,
+  ISO_8859_8,
+  ISO_8859_10,
+  ISO_8859_13,
+  ISO_8859_14,
+  ISO_8859_15,
+  ISO_8859_16,
+  KOI8_R,
+  KOI8_U,
+  MACINTOSH,
+  WINDOWS_874,
+  WINDOWS_1250,
+  WINDOWS_1251,
+  WINDOWS_1252,
+  WINDOWS_1253,
+  WINDOWS_1254,
+  WINDOWS_1255,
+  WINDOWS_1256,
+  WINDOWS_1257,
+  WINDOWS_1258,
+  X_MAC_CYRILLIC,
+  BIG5,
+  EUC_JP,
+  EUC_KR,
+  GB18030,
+  ISO_2022_JP,
+  ISO_2022_KR,
+  SHIFT_JIS;
+
+  static TextEncoding fromNative(int code) {
+    return code < 0 ? null : values()[code];
+  }
+
+  int toNative() {
+    return ordinal();
+  }
+
+  /**
+   * The canonical name, a label a browser accepts.
+   *
+   * @throws OdrException for {@link #UNKNOWN}, which has no name
+   */
+  public String canonicalName() {
+    return toStringNative(toNative());
+  }
+
+  /** Every accepted name, canonical first. */
+  public String[] names() {
+    return namesNative(toNative());
+  }
+
+  /** Whether the library can decode this encoding, as opposed to merely naming it. */
+  public boolean isDecodable() {
+    return isDecodableNative(toNative());
+  }
+
+  /**
+   * The encoding for a name, {@link #UNKNOWN} if none. Case and any {@code -}, {@code _} or
+   * space are ignored.
+   */
+  public static TextEncoding byName(String name) {
+    return fromNative(byNameNative(name));
+  }
+
+  /** Every encoding the library knows about, excluding {@link #UNKNOWN}. */
+  public static TextEncoding[] all() {
+    int[] codes = allNative();
+    TextEncoding[] result = new TextEncoding[codes.length];
+    for (int i = 0; i < codes.length; i++) {
+      result[i] = fromNative(codes[i]);
+    }
+    return result;
+  }
+
+  static {
+    NativeLibrary.load();
+  }
+
+  private static native String toStringNative(int encoding);
+
+  private static native String[] namesNative(int encoding);
+
+  private static native boolean isDecodableNative(int encoding);
+
+  private static native int byNameNative(String name);
+
+  private static native int[] allNative();
+}

jni/java/app/opendocument/core/TextFile.java

@@ -6,16 +6,27 @@ public final class TextFile extends DecodedFile {
     super(handle);
   }
 
-  /** Detected character set; {@code null} if unknown. */
+  /** The encoding the bytes were detected as, or decoded with. */
+  public TextEncoding encoding() {
+    return TextEncoding.fromNative(encodingNative(handle()));
+  }
+
+  /**
+   * Detected character set; {@code null} if unknown.
+   *
+   * @deprecated use {@link #encoding()}
+   */
+  @Deprecated
   public String charset() {
-    return charsetNative(handle());
+    TextEncoding encoding = encoding();
+    return encoding == TextEncoding.UNKNOWN ? null : encoding.canonicalName();
   }
 
   public String text() {
     return textNative(handle());
   }
 
-  private native String charsetNative(long handle);
+  private native int encodingNative(long handle);
 
   private native String textNative(long handle);
 }