diff --git a/api/src/org/labkey/api/util/FileType.java b/api/src/org/labkey/api/util/FileType.java
index e96330741d2..fcac2b38769 100644
--- a/api/src/org/labkey/api/util/FileType.java
+++ b/api/src/org/labkey/api/util/FileType.java
@@ -16,6 +16,7 @@
package org.labkey.api.util;
import org.apache.commons.io.IOCase;
+import org.apache.logging.log4j.Logger;
import org.apache.tika.detect.DefaultDetector;
import org.apache.tika.detect.Detector;
import org.apache.tika.io.TikaInputStream;
@@ -27,6 +28,7 @@
import org.junit.Assert;
import org.junit.Test;
import org.labkey.api.pipeline.file.FileAnalysisJobSupport;
+import org.labkey.api.util.logging.LogHelper;
import org.labkey.vfs.FileLike;
import java.io.File;
@@ -38,20 +40,31 @@
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import java.util.Locale;
import java.util.Objects;
/**
- * FileType
+ * Captures a file naming convention via an ordered list of suffixes (usually extensions, but any name-ending works).
+ * One is the canonical suffix used when creating new files. Optional constraints on MIME content type, directory-ness,
+ * and file header contents.
*
- * @author brendanx
+ * Matching ignores case unless {@link #setCaseSensitiveOnCaseSensitiveFileSystems} is set.
+ *
+ * Because suffixes match by name-ending, a broad type swallows names belonging to a more specific one — pepXML's ".xml"
+ * also matches protXML's ".pep-prot.xml". Register the specific type via {@link #addAntiFileType} to exclude it.
+ *
+ * Subclasses distinguish types that share an extension (".txt", ".xml") by overriding {@link #isHeaderMatch}; the base
+ * implementation never matches on header alone.
*/
public class FileType implements Serializable
{
private static final Detector DETECTOR = new DefaultDetector(MimeTypes.getDefaultMimeTypes());
+ private static final Logger LOG = LogHelper.getLogger(FileType.class, "file suffix matching");
// For serialization
protected FileType() {}
+ /** Asks the job for a file under each suffix in priority order, falling back to {@link #getDefaultName} when none exist. */
public FileLike findInputFile(FileAnalysisJobSupport support, String baseName)
{
if (_suffixes.size() > 1)
@@ -406,12 +419,13 @@ private String toLowerIfCaseInsensitive(String s)
{
return s;
}
- return s.toLowerCase();
+ // ROOT, not the default locale: in Turkish 'I' lowercases to dotless 'ı', so ".MZID" would stop matching ".mzid"
+ return s.toLowerCase(Locale.ROOT);
}
/**
- * Finds the best suffix based on priority order, strips it off, and returns the remainder. If there is no matching
- * suffix, returns the original file name.
+ * Strips the longest matching suffix and returns the remainder, or the original file name if nothing matches. Longest
+ * rather than first-in-list, so ".msprefix.mzXML" isn't reduced to "foo.msprefix".
*/
public String getBaseName(File file)
{
@@ -444,7 +458,7 @@ public String getBaseName(@NotNull java.nio.file.Path file)
else if (_supportGZ.booleanValue()) // TPP treats .xml.gz as a native read format
{
String sgz = s+".gz";
- if (fileName.endsWith(sgz))
+ if (toLowerIfCaseInsensitive(fileName).endsWith(toLowerIfCaseInsensitive(sgz)))
{
if ((null==suffix) || (sgz.length()>suffix.length()))
{
@@ -453,7 +467,14 @@ else if (_supportGZ.booleanValue()) // TPP treats .xml.gz as a native read forma
}
}
}
- assert suffix != null : "Could not find matching suffix even though types match";
+ if (suffix == null)
+ {
+ // Unreachable unless isType() and this loop disagree; warn so it stays visible with assertions disabled
+ String message = "Could not find matching suffix for " + fileName + " even though types match: " + this + ", supportGZ: " + _supportGZ;
+ assert false : message;
+ LOG.warn(message);
+ return fileName;
+ }
return fileName.substring(0, fileName.length() - suffix.length());
}
@@ -508,7 +529,9 @@ public boolean isType(String filePath)
}
/**
- * Checks if the path matches any of the suffixes and the file header if provided.
+ * Matches in order: reject if an anti-type matches, accept on content type (detected from the header via Tika when not
+ * supplied), accept on suffix (with the header, if given) and finally accept on header alone, that last only when
+ * the caller supplied no content type, since a caller-supplied type that didn't match is authoritative.
*/
public boolean isType(@Nullable String filePath, @Nullable String contentType, byte @Nullable[] header)
{
@@ -529,7 +552,7 @@ public boolean isType(@Nullable String filePath, @Nullable String contentType, b
if (contentType != null)
{
- contentType = contentType.toLowerCase().trim();
+ contentType = contentType.toLowerCase(Locale.ROOT).trim();
if (_contentTypes.contains(contentType))
return true;
}
@@ -578,16 +601,19 @@ protected static String detectContentType(String fileName, byte[] header)
}
}
+ /** Whether the name is exactly basename plus one of the suffixes, unlike {@link #isType}, which accepts any name ending in a suffix. */
public boolean isMatch(String name, String basename)
{
+ String normalizedName = toLowerIfCaseInsensitive(name);
for (String suffix : _suffixes)
{
- if (name.equalsIgnoreCase(basename + suffix))
+ String normalizedBase = toLowerIfCaseInsensitive(basename + suffix);
+ if (normalizedName.equals(normalizedBase))
{
return true;
}
// TPP treats .xml.gz as a native format
- if (_supportGZ.booleanValue() && name.equals(basename + suffix+".gz"))
+ if (_supportGZ.booleanValue() && normalizedName.equals(normalizedBase + ".gz"))
{
return true;
}
@@ -620,7 +646,7 @@ public boolean equals(Object o)
if (!Objects.equals(_defaultSuffix, fileType._defaultSuffix))
return false;
if (!Objects.equals(_antiTypes, fileType._antiTypes)) return false;
- return !(!Objects.equals(_suffixes, fileType._suffixes));
+ return Objects.equals(_suffixes, fileType._suffixes);
}
public String getDefaultSuffix()
@@ -649,6 +675,7 @@ public String toString()
return (_dir == null || !_dir.booleanValue() ? _suffixes.toString() : _suffixes + "/");
}
+ /** The subset of types that at least one of the files matches, in the order given by {@code types}. */
@NotNull
public static List findTypes(@NotNull List types, @NotNull List files)
{
@@ -689,6 +716,7 @@ public void setExtensionsMutuallyExclusive(boolean extensionsMutuallyExclusive)
_extensionsMutuallyExclusive = extensionsMutuallyExclusive;
}
+ /** The default suffix with everything through the first dot removed (".pep.xml" yields "pep.xml"); used as a pipeline input/output role name. */
public String getDefaultRole()
{
if (_defaultSuffix.contains("."))
@@ -703,6 +731,7 @@ public boolean isCaseSensitiveOnCaseSensitiveFileSystems()
return _caseSensitiveOnCaseSensitiveFileSystems;
}
+ /** Opt out of the default case-insensitive suffix matching, deferring to the file system: on a case-insensitive one (Windows, default macOS) matching stays case-insensitive. */
public void setCaseSensitiveOnCaseSensitiveFileSystems(boolean caseSensitiveOnCaseSensitiveFileSystems)
{
_caseSensitiveOnCaseSensitiveFileSystems = caseSensitiveOnCaseSensitiveFileSystems;
@@ -754,7 +783,51 @@ public void test()
assertFalse(ftt.isType("test.foo.bar"));
assertTrue(ftt.isType("test.foo"));
assertTrue(ftt.isType("test.bar"));
+ }
+ @Test
+ public void testCaseInsensitiveBaseName()
+ {
+ FileType ft = new FileType(Arrays.asList(".foo", ".bar"), ".foo", gzSupportLevel.SUPPORT_GZ);
+
+ // getBaseName() must handle everything isType() accepts, including mixed case with .gz
+ assertEquals("test", ft.getBaseName(Path.of("test.foo")));
+ assertEquals("test", ft.getBaseName(Path.of("test.FOO")));
+ assertEquals("test", ft.getBaseName(Path.of("test.foo.gz")));
+ assertEquals("test", ft.getBaseName(Path.of("test.FOO.gz")));
+ assertEquals("test", ft.getBaseName(Path.of("test.bAr.gZ")));
+
+ // strongest match wins regardless of case
+ FileType ftPrefix = new FileType(Arrays.asList(".mzxml", ".msprefix.mzxml"), ".mzxml", gzSupportLevel.SUPPORT_GZ);
+ assertEquals("test", ftPrefix.getBaseName(Path.of("test.MSPREFIX.mzXML")));
+ assertEquals("test", ftPrefix.getBaseName(Path.of("test.MSPREFIX.mzXML.GZ")));
+
+ // the suffix itself may be mixed case, as massSpecDataFileType's ".mzXML" is
+ FileType ftMixedSuffix = new FileType(".mzXML", gzSupportLevel.SUPPORT_GZ);
+ assertEquals("test", ftMixedSuffix.getBaseName(Path.of("test.mzXML")));
+ assertEquals("test", ftMixedSuffix.getBaseName(Path.of("test.mzxml")));
+ assertEquals("test", ftMixedSuffix.getBaseName(Path.of("test.mzxml.gz")));
+ assertEquals("test", ftMixedSuffix.getBaseName(Path.of("test.MZXML.GZ")));
+ assertTrue(ftMixedSuffix.isMatch("test.mzxml.gz", "test"));
+
+ // no match at all still returns the original file name
+ assertEquals("test.unrelated", ft.getBaseName(Path.of("test.unrelated")));
+
+ // isMatch() follows the same case rules as isType(), for both the plain and the .gz forms
+ assertTrue(ft.isMatch("test.FOO", "test"));
+ assertTrue(ft.isMatch("test.FOO.gz", "test"));
+ assertTrue(ft.isMatch("test.bAr.gZ", "test"));
+ assertFalse(ft.isMatch("test.unrelated", "test"));
+
+ // A type that opts into case sensitivity still agrees with itself on either kind of file system:
+ // isType(), getBaseName() and isMatch() all accept the file, or none of them do
+ FileType ftCase = new FileType(".foo", gzSupportLevel.SUPPORT_GZ);
+ ftCase.setCaseSensitiveOnCaseSensitiveFileSystems(true);
+ String mixedCase = "test.FOO.gz";
+ boolean accepted = ftCase.isType(mixedCase);
+ assertEquals(accepted ? "test" : mixedCase, ftCase.getBaseName(Path.of(mixedCase)));
+ assertEquals(accepted, ftCase.isMatch(mixedCase, "test"));
+ assertTrue(ftCase.isMatch("test.foo.gz", "test"));
}
}
}