diff --git a/api/src/org/labkey/api/util/FileType.java b/api/src/org/labkey/api/util/FileType.java index e96330741d2..fcac2b38769 100644 --- a/api/src/org/labkey/api/util/FileType.java +++ b/api/src/org/labkey/api/util/FileType.java @@ -16,6 +16,7 @@ package org.labkey.api.util; import org.apache.commons.io.IOCase; +import org.apache.logging.log4j.Logger; import org.apache.tika.detect.DefaultDetector; import org.apache.tika.detect.Detector; import org.apache.tika.io.TikaInputStream; @@ -27,6 +28,7 @@ import org.junit.Assert; import org.junit.Test; import org.labkey.api.pipeline.file.FileAnalysisJobSupport; +import org.labkey.api.util.logging.LogHelper; import org.labkey.vfs.FileLike; import java.io.File; @@ -38,20 +40,31 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Locale; import java.util.Objects; /** - * FileType + * Captures a file naming convention via an ordered list of suffixes (usually extensions, but any name-ending works). + * One is the canonical suffix used when creating new files. Optional constraints on MIME content type, directory-ness, + * and file header contents. * - * @author brendanx + * Matching ignores case unless {@link #setCaseSensitiveOnCaseSensitiveFileSystems} is set. + * + * Because suffixes match by name-ending, a broad type swallows names belonging to a more specific one — pepXML's ".xml" + * also matches protXML's ".pep-prot.xml". Register the specific type via {@link #addAntiFileType} to exclude it. + * + * Subclasses distinguish types that share an extension (".txt", ".xml") by overriding {@link #isHeaderMatch}; the base + * implementation never matches on header alone. */ public class FileType implements Serializable { private static final Detector DETECTOR = new DefaultDetector(MimeTypes.getDefaultMimeTypes()); + private static final Logger LOG = LogHelper.getLogger(FileType.class, "file suffix matching"); // For serialization protected FileType() {} + /** Asks the job for a file under each suffix in priority order, falling back to {@link #getDefaultName} when none exist. */ public FileLike findInputFile(FileAnalysisJobSupport support, String baseName) { if (_suffixes.size() > 1) @@ -406,12 +419,13 @@ private String toLowerIfCaseInsensitive(String s) { return s; } - return s.toLowerCase(); + // ROOT, not the default locale: in Turkish 'I' lowercases to dotless 'ı', so ".MZID" would stop matching ".mzid" + return s.toLowerCase(Locale.ROOT); } /** - * Finds the best suffix based on priority order, strips it off, and returns the remainder. If there is no matching - * suffix, returns the original file name. + * Strips the longest matching suffix and returns the remainder, or the original file name if nothing matches. Longest + * rather than first-in-list, so ".msprefix.mzXML" isn't reduced to "foo.msprefix". */ public String getBaseName(File file) { @@ -444,7 +458,7 @@ public String getBaseName(@NotNull java.nio.file.Path file) else if (_supportGZ.booleanValue()) // TPP treats .xml.gz as a native read format { String sgz = s+".gz"; - if (fileName.endsWith(sgz)) + if (toLowerIfCaseInsensitive(fileName).endsWith(toLowerIfCaseInsensitive(sgz))) { if ((null==suffix) || (sgz.length()>suffix.length())) { @@ -453,7 +467,14 @@ else if (_supportGZ.booleanValue()) // TPP treats .xml.gz as a native read forma } } } - assert suffix != null : "Could not find matching suffix even though types match"; + if (suffix == null) + { + // Unreachable unless isType() and this loop disagree; warn so it stays visible with assertions disabled + String message = "Could not find matching suffix for " + fileName + " even though types match: " + this + ", supportGZ: " + _supportGZ; + assert false : message; + LOG.warn(message); + return fileName; + } return fileName.substring(0, fileName.length() - suffix.length()); } @@ -508,7 +529,9 @@ public boolean isType(String filePath) } /** - * Checks if the path matches any of the suffixes and the file header if provided. + * Matches in order: reject if an anti-type matches, accept on content type (detected from the header via Tika when not + * supplied), accept on suffix (with the header, if given) and finally accept on header alone, that last only when + * the caller supplied no content type, since a caller-supplied type that didn't match is authoritative. */ public boolean isType(@Nullable String filePath, @Nullable String contentType, byte @Nullable[] header) { @@ -529,7 +552,7 @@ public boolean isType(@Nullable String filePath, @Nullable String contentType, b if (contentType != null) { - contentType = contentType.toLowerCase().trim(); + contentType = contentType.toLowerCase(Locale.ROOT).trim(); if (_contentTypes.contains(contentType)) return true; } @@ -578,16 +601,19 @@ protected static String detectContentType(String fileName, byte[] header) } } + /** Whether the name is exactly basename plus one of the suffixes, unlike {@link #isType}, which accepts any name ending in a suffix. */ public boolean isMatch(String name, String basename) { + String normalizedName = toLowerIfCaseInsensitive(name); for (String suffix : _suffixes) { - if (name.equalsIgnoreCase(basename + suffix)) + String normalizedBase = toLowerIfCaseInsensitive(basename + suffix); + if (normalizedName.equals(normalizedBase)) { return true; } // TPP treats .xml.gz as a native format - if (_supportGZ.booleanValue() && name.equals(basename + suffix+".gz")) + if (_supportGZ.booleanValue() && normalizedName.equals(normalizedBase + ".gz")) { return true; } @@ -620,7 +646,7 @@ public boolean equals(Object o) if (!Objects.equals(_defaultSuffix, fileType._defaultSuffix)) return false; if (!Objects.equals(_antiTypes, fileType._antiTypes)) return false; - return !(!Objects.equals(_suffixes, fileType._suffixes)); + return Objects.equals(_suffixes, fileType._suffixes); } public String getDefaultSuffix() @@ -649,6 +675,7 @@ public String toString() return (_dir == null || !_dir.booleanValue() ? _suffixes.toString() : _suffixes + "/"); } + /** The subset of types that at least one of the files matches, in the order given by {@code types}. */ @NotNull public static List findTypes(@NotNull List types, @NotNull List files) { @@ -689,6 +716,7 @@ public void setExtensionsMutuallyExclusive(boolean extensionsMutuallyExclusive) _extensionsMutuallyExclusive = extensionsMutuallyExclusive; } + /** The default suffix with everything through the first dot removed (".pep.xml" yields "pep.xml"); used as a pipeline input/output role name. */ public String getDefaultRole() { if (_defaultSuffix.contains(".")) @@ -703,6 +731,7 @@ public boolean isCaseSensitiveOnCaseSensitiveFileSystems() return _caseSensitiveOnCaseSensitiveFileSystems; } + /** Opt out of the default case-insensitive suffix matching, deferring to the file system: on a case-insensitive one (Windows, default macOS) matching stays case-insensitive. */ public void setCaseSensitiveOnCaseSensitiveFileSystems(boolean caseSensitiveOnCaseSensitiveFileSystems) { _caseSensitiveOnCaseSensitiveFileSystems = caseSensitiveOnCaseSensitiveFileSystems; @@ -754,7 +783,51 @@ public void test() assertFalse(ftt.isType("test.foo.bar")); assertTrue(ftt.isType("test.foo")); assertTrue(ftt.isType("test.bar")); + } + @Test + public void testCaseInsensitiveBaseName() + { + FileType ft = new FileType(Arrays.asList(".foo", ".bar"), ".foo", gzSupportLevel.SUPPORT_GZ); + + // getBaseName() must handle everything isType() accepts, including mixed case with .gz + assertEquals("test", ft.getBaseName(Path.of("test.foo"))); + assertEquals("test", ft.getBaseName(Path.of("test.FOO"))); + assertEquals("test", ft.getBaseName(Path.of("test.foo.gz"))); + assertEquals("test", ft.getBaseName(Path.of("test.FOO.gz"))); + assertEquals("test", ft.getBaseName(Path.of("test.bAr.gZ"))); + + // strongest match wins regardless of case + FileType ftPrefix = new FileType(Arrays.asList(".mzxml", ".msprefix.mzxml"), ".mzxml", gzSupportLevel.SUPPORT_GZ); + assertEquals("test", ftPrefix.getBaseName(Path.of("test.MSPREFIX.mzXML"))); + assertEquals("test", ftPrefix.getBaseName(Path.of("test.MSPREFIX.mzXML.GZ"))); + + // the suffix itself may be mixed case, as massSpecDataFileType's ".mzXML" is + FileType ftMixedSuffix = new FileType(".mzXML", gzSupportLevel.SUPPORT_GZ); + assertEquals("test", ftMixedSuffix.getBaseName(Path.of("test.mzXML"))); + assertEquals("test", ftMixedSuffix.getBaseName(Path.of("test.mzxml"))); + assertEquals("test", ftMixedSuffix.getBaseName(Path.of("test.mzxml.gz"))); + assertEquals("test", ftMixedSuffix.getBaseName(Path.of("test.MZXML.GZ"))); + assertTrue(ftMixedSuffix.isMatch("test.mzxml.gz", "test")); + + // no match at all still returns the original file name + assertEquals("test.unrelated", ft.getBaseName(Path.of("test.unrelated"))); + + // isMatch() follows the same case rules as isType(), for both the plain and the .gz forms + assertTrue(ft.isMatch("test.FOO", "test")); + assertTrue(ft.isMatch("test.FOO.gz", "test")); + assertTrue(ft.isMatch("test.bAr.gZ", "test")); + assertFalse(ft.isMatch("test.unrelated", "test")); + + // A type that opts into case sensitivity still agrees with itself on either kind of file system: + // isType(), getBaseName() and isMatch() all accept the file, or none of them do + FileType ftCase = new FileType(".foo", gzSupportLevel.SUPPORT_GZ); + ftCase.setCaseSensitiveOnCaseSensitiveFileSystems(true); + String mixedCase = "test.FOO.gz"; + boolean accepted = ftCase.isType(mixedCase); + assertEquals(accepted ? "test" : mixedCase, ftCase.getBaseName(Path.of(mixedCase))); + assertEquals(accepted, ftCase.isMatch(mixedCase, "test")); + assertTrue(ftCase.isMatch("test.foo.gz", "test")); } } }