From dd32cb00e0e7921cde0d8da2f15a4308a7fa692c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Henrik=20O=2E=20S=C3=B8rensen?= Date: Fri, 7 Aug 2026 11:46:15 +0200 Subject: [PATCH] fix: Marshal spacing as the output buffer it is liblouis's spacing parameter is in/out: it answers in the same buffer it reads the request from. It was declared as a string, so the answer landed in the marshaller's temporary and was freed unread - no caller could ever receive spacing information - and the buffer was sized to the input while liblouis writes per output cell. Sizing is max(inputLength, outputLength) + 1, not the "at least inlen elements" the header promises, because the two directions disagree. Forward writes inlen + 1 bytes (lou_translateString.c:1385); back-translation opens with memset(spacing, '*', *outlen) (lou_backTranslateString.c:229) and then writes per output cell. An inlen-sized buffer overran by outlen - inlen bytes backwards - 80 bytes for a 27-char input with an outlen of 108, not the one byte the header implies. It is also a char buffer rather than widechar, and liblouis indexes it in widechars, so the caller's per-char request collapses to one entry per character on the way in. The answer surfaces as TranslatedString.OutputSpacing, following OutputDots78: the typeform write-back has the same shape, and the same reason it cannot go back into the caller's own argument. All four public signatures are unchanged. Co-Authored-By: Claude Opus 5 --- LibLouis.NET.Test/SpacingTests.cs | 164 ++++++++++++++++++++++++++++++ LibLouis.NET/LibLouis.cs | 131 ++++++++++++++++++++++-- LibLouis.NET/NativeMethod.cs | 30 ++++-- LibLouis.NET/TranslatedString.cs | 20 ++++ 4 files changed, 329 insertions(+), 16 deletions(-) create mode 100644 LibLouis.NET.Test/SpacingTests.cs diff --git a/LibLouis.NET.Test/SpacingTests.cs b/LibLouis.NET.Test/SpacingTests.cs new file mode 100644 index 0000000..5c61ad0 --- /dev/null +++ b/LibLouis.NET.Test/SpacingTests.cs @@ -0,0 +1,164 @@ +using System; +using System.IO; +using System.Linq; + +using Xunit; + +namespace LibLouis.NET.Test; + +/// +/// The spacing parameter is in/out: liblouis answers in the same buffer it reads the request from. +/// It used to be declared as a string, so the answer landed in the marshaller's temporary and was +/// freed unread, and the buffer was sized to the input while liblouis writes per output cell. +/// +public class SpacingTests +{ + private static readonly string[] Tables = + new[] { "da-dk-braillo.dis", "da-dk-g26.ctb" } + .Select(t => Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "tables", t)) + .ToArray(); + + private static TranslatedString Translate(string input, string? spacing, int outputLength) + { + return LibLouis.Instance.Translate( + Tables, input, outputLength, null, spacing, + new int[input.Length], new int[outputLength], -1, TranslationMode.Regular); + } + + private static TranslatedString BackTranslate(string input, string? spacing, int outputLength) + { + return LibLouis.Instance.BackTranslate( + Tables, input, outputLength, null, spacing, + new int[input.Length], new int[outputLength], -1, TranslationMode.Regular); + } + + [Fact] + public void Spacing_IsNotReported_WhenNotRequested() + { + TranslatedString translated = Translate("Anden linje, med kursiveret tekst.", null, 200); + + Assert.Null(translated.OutputSpacing); + } + + [Fact] + public void Spacing_ReportsTheRequestedDigits_AgainstTheOutputCellsTheyProduced() + { + const string input = "Anden linje, med kursiveret tekst."; + + // Mark the three characters of "lin" in "linje". + char[] request = new string('0', input.Length).ToCharArray(); + request[6] = request[7] = request[8] = '3'; + + TranslatedString translated = Translate(input, new string(request), 200); + + Assert.Equal("@anç linje, m kursi#rò ükz.", translated.Output); + Assert.Equal("**0003330000000000000000000", translated.OutputSpacing); + + // One entry per output cell: '*' where liblouis reported nothing for a cell, otherwise the + // digit belonging to the input character that produced it. + Assert.Equal(translated.Output.Length, translated.OutputSpacing!.Length); + + for (int cell = 0; cell < translated.OutputSpacing.Length; cell++) + { + if (translated.OutputSpacing[cell] == '3') + { + Assert.Contains(translated.InputPosition[cell], new[] { 6, 7, 8 }); + } + } + } + + [Fact] + public void Spacing_IsClippedToTheInputLength_WhenTheOutputGrows() + { + // Forward translation copies its answer back over only the first inlen bytes of the buffer, + // so a translation that grows past the input reports nothing for the cells beyond it. The + // marker table expands "yes" into a foreign-language run. + const string input = "Han sagde yes."; + + char[] request = new string('7', input.Length).ToCharArray(); + + string[] markerTables = + new[] { "da-dk-braillo.dis", "da-dk-g16-markers.ctb" } + .Select(t => Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "tables", t)) + .ToArray(); + + TypeForm[] formtype = new TypeForm[input.Length]; + Array.Fill(formtype, TypeForm.Italic); + + TranslatedString translated = LibLouis.Instance.Translate( + markerTables, input, 200, formtype, new string(request), + new int[input.Length], new int[200], -1, TranslationMode.Regular); + + Assert.True(translated.Output.Length > input.Length, "expected the marker table to grow the text"); + Assert.Equal(input.Length, translated.OutputSpacing!.Length); + } + + [Fact] + public void Spacing_LeadingX_DisablesTheComputation() + { + const string input = "Anden linje, med kursiveret tekst."; + + // liblouis treats a leading 'X' as "do not compute", so there is no answer to report. + string request = 'X' + new string('0', input.Length - 1); + + TranslatedString translated = Translate(input, request, 200); + + Assert.Null(translated.OutputSpacing); + } + + [Fact] + public void Spacing_BackTranslation_CoversTheWholeOutput() + { + // Back-translation opens with memset(spacing, '*', outlen) and then writes per output cell, + // so the buffer has to be sized to the output capacity, not to the input. Here the capacity + // is four times the input's length: sizing to the input overran the buffer by 80 bytes. + const string braille = "@anç linje, m kursi#rò ükz."; + + int outputLength = braille.Length * 4; + + TranslatedString translated = BackTranslate(braille, new string('0', braille.Length), outputLength); + + Assert.True(translated.Output.Length > braille.Length, "expected back-translation to grow the text"); + Assert.Equal(translated.Output.Length, translated.OutputSpacing!.Length); + Assert.All(translated.OutputSpacing, c => Assert.True(c == '*' || char.IsAsciiDigit(c), $"unexpected spacing value '{c}'")); + } + + [Fact] + public void Spacing_CollapsesSurrogatePairs_ToOneEntryPerCharacter() + { + // liblouis indexes the spacing buffer in widechars, so the caller's per-char request has to + // collapse a surrogate pair into a single entry on the way in. Counting in UTF-16 units + // would describe a longer buffer than the one allocated. + const string input = "a\U0001F600b"; + + Assert.Equal(4, input.Length); + + TranslatedString translated = Translate(input, new string('5', input.Length), 40); + + Assert.Equal(3, translated.OutputSpacing!.Length); + } + + [Fact] + public void Spacing_MustMatchTheInputLength() + { + Assert.Throws(() => Translate("Anden linje.", "00", 200)); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void Spacing_OnTheOverloadsWithoutPositions_IsComputedAndDiscarded(bool forward) + { + // These overloads return a bare string and have nowhere to report the answer, but liblouis + // still writes into the buffer, so it has to be sized the same way. + const string input = "Anden linje, med kursiveret tekst."; + + string request = new string('0', input.Length); + + string output = forward + ? LibLouis.Instance.Translate(Tables, input, input.Length * 4, null, request, TranslationMode.Regular) + : LibLouis.Instance.BackTranslate(Tables, input, input.Length * 4, null, request, TranslationMode.Regular); + + Assert.NotEmpty(output); + } +} diff --git a/LibLouis.NET/LibLouis.cs b/LibLouis.NET/LibLouis.cs index 6d8e661..5c4eb0a 100644 --- a/LibLouis.NET/LibLouis.cs +++ b/LibLouis.NET/LibLouis.cs @@ -318,7 +318,14 @@ public string CharactersToDots(IEnumerable tableList, string input) /// String to translate. /// Maximum output length. /// The typeform parameter is used to indicate italic type, boldface type, computer braille, etc. It is an array of formtype with the same length as the input buffer pointed to by input. Each element indicates the typeform of the corresponding character in the input buffer. - /// The spacing parameter is used to indicate differences in spacing between the input string and the translated output string. It is also of the same length as the input string. If this parameter is NULL, no spacing information is computed. + /// + /// The spacing information to compute, one char per char of , or + /// to skip the computation. An ASCII digit marks a character whose + /// spacing should be carried into the output; liblouis ignores anything else, except that a + /// leading 'X' disables the computation. The answer arrives in + /// rather than in this string, which liblouis has + /// no way to write to. + /// /// /// /// @@ -366,6 +373,11 @@ public TranslatedString Translate( byte[] inputBuffer = PrepareUCSInputBuffer(input); byte[] outputBuffer = PrepareUCSOutputBuffer(outputBufferLength); TypeForm[]? typeFormBuffer = PrepareTypeFormBuffer(formtype, inputLength, outputBufferLength); + byte[]? spacingBuffer = PrepareSpacingBuffer(spacing, input, inputLength, outputBufferLength); + + // liblouis overwrites inlen with the number of characters it consumed, so hold on to the + // length that went in: it bounds how much of the spacing buffer liblouis fills. + int spacingInputLength = inputLength; // The cursor arrives as a .NET string index and liblouis wants a widechar index. int[] inputOffsets = Utf16OffsetOfWidechar(input); @@ -377,7 +389,7 @@ public TranslatedString Translate( lock (NativeLock) { ThrowIfShutDown(); - success = NativeMethods.lou_translate(tables, inputBuffer, ref inputLength, outputBuffer, ref outputLength, typeFormBuffer, spacing, outputPosition, inputPosition, ref widecharCursor, mode) > 0; + success = NativeMethods.lou_translate(tables, inputBuffer, ref inputLength, outputBuffer, ref outputLength, typeFormBuffer, spacingBuffer, outputPosition, inputPosition, ref widecharCursor, mode) > 0; } if (!success) @@ -397,6 +409,11 @@ public TranslatedString Translate( InputPosition = mappedInputPosition, OutputPosition = mappedOutputPosition, OutputDots78 = ExtractOutputDots78(typeFormBuffer, outputLength), + + // Forward translation copies its answer back over only the first inlen bytes of the + // buffer (lou_translateString.c:1384), so cells past the input's length hold no answer + // however long the output grew. + OutputSpacing = ExtractOutputSpacing(spacingBuffer, Math.Min(outputLength, spacingInputLength)), }; } @@ -409,7 +426,13 @@ public TranslatedString Translate( /// String to translate. /// Maximum output length. /// The typeform parameter is used to indicate italic type, boldface type, computer braille, etc. It is an array of formtype with the same length as the input buffer pointed to by input. Each element indicates the typeform of the corresponding character in the input buffer. - /// The spacing parameter is used to indicate differences in spacing between the input string and the translated output string. It is also of the same length as the input string. If this parameter is NULL, no spacing information is computed. + /// + /// The spacing information to compute, one char per char of , or + /// to skip the computation. This overload has nowhere to report the + /// answer, since liblouis returns it in a buffer and an immutable string cannot receive it, so + /// it is computed and discarded. Use the overload returning and + /// read if you need it. + /// /// The mode parameter specifies how the translation should be done. They are all powers of 2, so that a combined mode can be specified by adding up different values. /// /// @@ -435,6 +458,7 @@ public string Translate(IEnumerable tableList, string input, int outputL byte[] inputBuffer = PrepareUCSInputBuffer(input); byte[] outputBuffer = PrepareUCSOutputBuffer(outputBufferLength); TypeForm[]? typeFormBuffer = PrepareTypeFormBuffer(formtype, inputLength, outputBufferLength); + byte[]? spacingBuffer = PrepareSpacingBuffer(spacing, input, inputLength, outputBufferLength); string tables = string.Join(',', tableList); bool success; @@ -442,7 +466,7 @@ public string Translate(IEnumerable tableList, string input, int outputL lock (NativeLock) { ThrowIfShutDown(); - success = NativeMethods.lou_translateString(tables, inputBuffer, ref inputLength, outputBuffer, ref outputLength, typeFormBuffer, spacing, mode) > 0; + success = NativeMethods.lou_translateString(tables, inputBuffer, ref inputLength, outputBuffer, ref outputLength, typeFormBuffer, spacingBuffer, mode) > 0; } if (!success) @@ -463,7 +487,14 @@ public string Translate(IEnumerable tableList, string input, int outputL /// String to translate. /// Maximum output length. /// The typeform parameter is used to indicate italic type, boldface type, computer braille, etc. It is an array of formtype with the same length as the input buffer pointed to by input. Each element indicates the typeform of the corresponding character in the input buffer. - /// The spacing parameter is used to indicate differences in spacing between the input string and the translated output string. It is also of the same length as the input string. If this parameter is NULL, no spacing information is computed. + /// + /// The spacing information to compute, one char per char of , or + /// to skip the computation. An ASCII digit marks a character whose + /// spacing should be carried into the output; liblouis ignores anything else, except that a + /// leading 'X' disables the computation. The answer arrives in + /// rather than in this string, which liblouis has + /// no way to write to. + /// /// /// /// @@ -511,6 +542,7 @@ public TranslatedString BackTranslate( byte[] inputBuffer = PrepareUCSInputBuffer(input); byte[] outputBuffer = PrepareUCSOutputBuffer(outputBufferLength); TypeForm[]? typeFormBuffer = PrepareTypeFormBuffer(formtype, inputLength, outputBufferLength); + byte[]? spacingBuffer = PrepareSpacingBuffer(spacing, input, inputLength, outputBufferLength); // The cursor arrives as a .NET string index and liblouis wants a widechar index. int[] inputOffsets = Utf16OffsetOfWidechar(input); @@ -522,7 +554,7 @@ public TranslatedString BackTranslate( lock (NativeLock) { ThrowIfShutDown(); - success = NativeMethods.lou_backTranslate(tables, inputBuffer, ref inputLength, outputBuffer, ref outputLength, typeFormBuffer, spacing, outputPosition, inputPosition, ref widecharCursor, mode) > 0; + success = NativeMethods.lou_backTranslate(tables, inputBuffer, ref inputLength, outputBuffer, ref outputLength, typeFormBuffer, spacingBuffer, outputPosition, inputPosition, ref widecharCursor, mode) > 0; } if (!success) @@ -541,6 +573,11 @@ public TranslatedString BackTranslate( CursorPosition = mappedCursor, InputPosition = mappedInputPosition, OutputPosition = mappedOutputPosition, + + // Back-translation writes straight into the caller's buffer, one entry per output cell + // across the whole output, so unlike the forward direction it is not clipped to the + // input's length. + OutputSpacing = ExtractOutputSpacing(spacingBuffer, outputLength), }; } @@ -551,7 +588,13 @@ public TranslatedString BackTranslate( /// String to translate. /// Maximum output length. /// The typeform parameter is used to indicate italic type, boldface type, computer braille, etc. It is an array of formtype with the same length as the input buffer pointed to by input. Each element indicates the typeform of the corresponding character in the input buffer. - /// The spacing parameter is used to indicate differences in spacing between the input string and the translated output string. It is also of the same length as the input string. If this parameter is NULL, no spacing information is computed. + /// + /// The spacing information to compute, one char per char of , or + /// to skip the computation. This overload has nowhere to report the + /// answer, since liblouis returns it in a buffer and an immutable string cannot receive it, so + /// it is computed and discarded. Use the overload returning and + /// read if you need it. + /// /// The mode parameter specifies how the translation should be done. They are all powers of 2, so that a combined mode can be specified by adding up different values. /// /// @@ -577,6 +620,7 @@ public string BackTranslate(IEnumerable tableList, string input, int out byte[] inputBuffer = PrepareUCSInputBuffer(input); byte[] outputBuffer = PrepareUCSOutputBuffer(outputBufferLength); TypeForm[]? typeFormBuffer = PrepareTypeFormBuffer(formtype, inputLength, outputBufferLength); + byte[]? spacingBuffer = PrepareSpacingBuffer(spacing, input, inputLength, outputBufferLength); string tables = string.Join(',', tableList); bool success; @@ -584,7 +628,7 @@ public string BackTranslate(IEnumerable tableList, string input, int out lock (NativeLock) { ThrowIfShutDown(); - success = NativeMethods.lou_backTranslateString(tables, inputBuffer, ref inputLength, outputBuffer, ref outputLength, typeFormBuffer, spacing, mode) > 0; + success = NativeMethods.lou_backTranslateString(tables, inputBuffer, ref inputLength, outputBuffer, ref outputLength, typeFormBuffer, spacingBuffer, mode) > 0; } if (!success) @@ -708,6 +752,77 @@ public string Hyphenate(IEnumerable tableList, string input, Translation return dots; } + /// + /// Copy the caller's spacing request into a buffer that is safe to hand to liblouis. + /// + /// + /// The spacing parameter is in/out - liblouis writes its answer back over the same buffer it + /// read the request from - but it used to be declared as a string. That meant the answer landed + /// in the marshaller's temporary, which is freed after the call and could never be copied back + /// into an immutable string, so no caller could ever receive spacing information. The trailing + /// NUL liblouis appends at spacing[inlen] (lou_translateString.c:1385) also fell one byte + /// past that temporary. + /// + /// Sizing is max(inputLength, outputLength) + 1, not the "at least inlen elements" the + /// header promises, because the two directions disagree: forward translation writes + /// inlen + 1 bytes, while back-translation opens with + /// memset(spacing, '*', outlen) (lou_backTranslateString.c:229) and then writes per + /// output cell. An inlen-sized buffer is a heap overrun in the back-translation direction + /// whenever the output is longer than the input. + /// + /// + /// It is a char buffer rather than widechar, and liblouis indexes it in widechars, so the + /// caller's one-entry-per-char string is collapsed to one entry per character on the way in. + /// Only ASCII digits mean anything to liblouis; everything else is ignored, except that a first + /// byte of 'X' disables the computation outright (lou_translateString.c:1203). + /// + /// + private byte[]? PrepareSpacingBuffer(string? spacing, string input, int inputLength, int outputLength) + { + if (spacing is null) + { + return null; + } + + byte[] buffer = new byte[Math.Max(inputLength, outputLength) + 1]; + int[] offsets = Utf16OffsetOfWidechar(input); + + for (int k = 0; k < inputLength; k++) + { + char c = spacing[offsets[k]]; + buffer[k] = c < 0x80 ? (byte)c : (byte)'*'; + } + + return buffer; + } + + /// + /// Reads the spacing information liblouis wrote back into the scratch spacing buffer. + /// + /// + /// The write-back half of . Values are indexed by *output* + /// cell in both directions: '*' where liblouis reported nothing for a cell, an ASCII digit + /// carried over from the input character that produced it, or '1' where back-translation + /// inserted a space. differs by direction and is the caller's to + /// decide - see the call sites. + /// + private static string? ExtractOutputSpacing(byte[]? spacingBuffer, int count) + { + if (spacingBuffer is null) + { + return null; + } + + // A request whose first byte is 'X' tells forward translation to skip the computation, so + // the buffer still holds the request and there is no answer to report. + if (spacingBuffer[0] == (byte)'X') + { + return null; + } + + return Encoding.ASCII.GetString(spacingBuffer, 0, Math.Clamp(count, 0, spacingBuffer.Length - 1)); + } + /// /// Converts a cursor given as a .NET string index into the widechar index liblouis expects. /// diff --git a/LibLouis.NET/NativeMethod.cs b/LibLouis.NET/NativeMethod.cs index 95f55df..6a02483 100644 --- a/LibLouis.NET/NativeMethod.cs +++ b/LibLouis.NET/NativeMethod.cs @@ -26,7 +26,11 @@ public static partial class NativeMethods /// Buffer for output. /// Length of buffer for output (make sure to allow for additional characters). /// Formtype is not used. - /// Spacing is not used. + /// + /// In/out buffer of single bytes, not widechars, or NULL. Must be at least + /// max(inlen, outlen) + 1 bytes long; see for why + /// the header's "at least inlen" is not enough. + /// /// Array of original-to-braille positions. /// Array of braille-to-original positions. /// Cursor position is not used. @@ -41,7 +45,7 @@ internal static partial int lou_translate( byte[] outbuf, ref int outlen, TypeForm[]? formtype, - string? spacing, + byte[]? spacing, int[] outputPos, int[] inputPos, ref int cursorPos, @@ -56,7 +60,10 @@ internal static partial int lou_translate( /// Buffer for output. /// Length of buffer for output (make sure to allow for additional characters). /// Formtype is not used. - /// Spacing is not used. + /// + /// In/out buffer of single bytes, not widechars, or NULL. Must be at least + /// max(inlen, outlen) + 1 bytes long; back-translation memsets outlen of them before it starts. + /// /// Array of original-to-braille positions. /// Array of braille-to-original positions. /// Cursor position is not used. @@ -71,7 +78,7 @@ internal static partial int lou_backTranslate( byte[] outbuf, ref int outlen, TypeForm[]? formtype, - string? spacing, + byte[]? spacing, int[] outputPos, int[] inputPos, ref int cursorPos, @@ -86,7 +93,11 @@ internal static partial int lou_backTranslate( /// Buffer for output. /// Length of buffer for output (make sure to allow for additional characters). /// Formtype is not used. - /// The spacing parameter is used to indicate differences in spacing between the input string and the translated output string. It is also of the same length as the string pointed to by *inbuf. If this parameter is NULL, no spacing information is computed.. + /// + /// In/out buffer of single bytes, not widechars, or NULL. Must be at least + /// max(inlen, outlen) + 1 bytes long; see for why + /// the header's "at least inlen" is not enough. + /// /// Specifies how the translation should be done. They are all powers of 2, so that a combined mode can be specified by adding up different values. /// 0 if error, 1 if success. [DefaultDllImportSearchPaths(DllImportSearchPath.SafeDirectories)] @@ -98,7 +109,7 @@ internal static partial int lou_translateString( byte[] outbuf, ref int outlen, TypeForm[]? formtype, - string? spacing, + byte[]? spacing, TranslationMode mode); /// @@ -110,7 +121,10 @@ internal static partial int lou_translateString( /// Buffer for output. /// Length of buffer for output (make sure to allow for additional characters). /// Formtype is not used. - /// Spacing is not used. + /// + /// In/out buffer of single bytes, not widechars, or NULL. Must be at least + /// max(inlen, outlen) + 1 bytes long; back-translation memsets outlen of them before it starts. + /// /// Mode is not used. /// 0 if error, 1 if success. [DefaultDllImportSearchPaths(DllImportSearchPath.SafeDirectories)] @@ -122,7 +136,7 @@ internal static partial int lou_backTranslateString( byte[] outbuf, ref int outlen, TypeForm[]? formtype, - string? spacing, + byte[]? spacing, TranslationMode mode); /// diff --git a/LibLouis.NET/TranslatedString.cs b/LibLouis.NET/TranslatedString.cs index 4e82eab..4a00d53 100644 --- a/LibLouis.NET/TranslatedString.cs +++ b/LibLouis.NET/TranslatedString.cs @@ -44,4 +44,24 @@ public class TranslatedString /// translation only: back-translation zero-fills the buffer and reports nothing. /// public bool[]? OutputDots78 { get; set; } + + /// + /// Per output cell, the spacing information liblouis reported: '*' where it reported nothing, + /// an ASCII digit carried over from the input character that produced the cell, or '1' where + /// back-translation inserted a space. when the translation ran without a + /// spacing argument, because liblouis only computes this when one is supplied. + /// + /// + /// This is the write-back half of the native spacing parameter, which is in/out: liblouis + /// answers in the same buffer it reads the request from. It cannot be reported through the + /// caller's spacing string - a .NET string is immutable, which is why the parameter + /// silently did nothing before - and it is indexed per output cell rather than per input + /// character, so it would not fit there anyway. + /// + /// Shorter than when a forward translation grew the text: that direction + /// only copies its answer back over as many bytes as the input was long, so the cells past that + /// point have no answer. Back-translation reports the full output. + /// + /// + public string? OutputSpacing { get; set; } }