diff --git a/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockExistenceVerifier.java b/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockExistenceVerifier.java index dde79989d02..5cae3453321 100644 --- a/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockExistenceVerifier.java +++ b/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockExistenceVerifier.java @@ -24,6 +24,7 @@ import org.apache.hadoop.hdds.scm.XceiverClientManager; import org.apache.hadoop.hdds.scm.XceiverClientSpi; import org.apache.hadoop.hdds.scm.cli.ContainerOperationClient; +import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; @@ -67,6 +68,11 @@ public BlockVerificationResult verifyBlock(DatanodeDetails datanode, OmKeyLocati } else { return BlockVerificationResult.failCheck("Block does not exist on this replica"); } + } catch (StorageContainerException e) { + if (e.getResult() == ContainerProtos.Result.NO_SUCH_BLOCK) { + return BlockVerificationResult.failCheckAndRefreshKeyLocation(e.getMessage()); + } + return BlockVerificationResult.failIncomplete(e.getMessage()); } catch (IOException e) { return BlockVerificationResult.failIncomplete(e.getMessage()); } finally { diff --git a/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockVerificationResult.java b/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockVerificationResult.java index c73635ba475..e94e45e2497 100644 --- a/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockVerificationResult.java +++ b/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/replicas/BlockVerificationResult.java @@ -28,11 +28,18 @@ public class BlockVerificationResult { private final boolean completed; private final boolean pass; private final List failures; + private final boolean refreshKeyLocation; public BlockVerificationResult(boolean completed, boolean pass, List failures) { + this(completed, pass, failures, false); + } + + public BlockVerificationResult(boolean completed, boolean pass, List failures, + boolean refreshKeyLocation) { this.completed = completed; this.pass = pass; this.failures = failures; + this.refreshKeyLocation = refreshKeyLocation; } public static BlockVerificationResult pass() { @@ -43,6 +50,10 @@ public static BlockVerificationResult failCheck(String message) { return new BlockVerificationResult(true, false, Collections.singletonList(message)); } + public static BlockVerificationResult failCheckAndRefreshKeyLocation(String message) { + return new BlockVerificationResult(true, false, Collections.singletonList(message), true); + } + public static BlockVerificationResult failIncomplete(String message) { return new BlockVerificationResult(false, false, Collections.singletonList(message)); } @@ -59,4 +70,8 @@ public List getFailures() { return failures; } + public boolean shouldRefreshKeyLocation() { + return refreshKeyLocation; + } + } diff --git a/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/replicas/ChecksumVerifier.java b/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/replicas/ChecksumVerifier.java index 96f8218526d..2acc514481a 100644 --- a/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/replicas/ChecksumVerifier.java +++ b/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/replicas/ChecksumVerifier.java @@ -23,9 +23,11 @@ import org.apache.commons.io.output.NullOutputStream; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.scm.OzoneClientConfig; import org.apache.hadoop.hdds.scm.XceiverClientManager; import org.apache.hadoop.hdds.scm.cli.ContainerOperationClient; +import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException; import org.apache.hadoop.hdds.scm.pipeline.Pipeline; import org.apache.hadoop.ozone.client.io.BlockInputStreamFactoryImpl; import org.apache.hadoop.ozone.common.OzoneChecksumException; @@ -68,6 +70,11 @@ public BlockVerificationResult verifyBlock(DatanodeDetails datanode, OmKeyLocati return BlockVerificationResult.pass(); } catch (IOException e) { Throwable cause = e.getCause() != null ? e.getCause() : e; + StorageContainerException storageContainerException = findStorageContainerException(e); + if (storageContainerException != null + && storageContainerException.getResult() == ContainerProtos.Result.NO_SUCH_BLOCK) { + return BlockVerificationResult.failCheckAndRefreshKeyLocation(storageContainerException.getMessage()); + } if (cause instanceof OzoneChecksumException) { return BlockVerificationResult.failCheck(cause.getMessage()); } else { @@ -75,4 +82,15 @@ public BlockVerificationResult verifyBlock(DatanodeDetails datanode, OmKeyLocati } } } + + private static StorageContainerException findStorageContainerException(Throwable throwable) { + Throwable current = throwable; + while (current != null) { + if (current instanceof StorageContainerException) { + return (StorageContainerException) current; + } + current = current.getCause(); + } + return null; + } } diff --git a/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicasVerify.java b/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicasVerify.java index e886f945c34..784e3fdc918 100644 --- a/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicasVerify.java +++ b/hadoop-ozone/cli-debug/src/main/java/org/apache/hadoop/ozone/debug/replicas/ReplicasVerify.java @@ -338,14 +338,69 @@ void checkBucket(OzoneClient ozoneClient, OzoneBucket bucket, ArrayNode keysArra void processKey(OzoneClient ozoneClient, String volumeName, String bucketName, String keyName, ArrayNode keysArray, AtomicBoolean allKeysPassed) throws IOException { keysProcessed.incrementAndGet(); - OmKeyInfo keyInfo = ozoneClient.getProxy().getKeyInfo( - volumeName, bucketName, keyName, refreshContainerLocationsFromScm); + OmKeyInfo keyInfo; + try { + keyInfo = ozoneClient.getProxy().getKeyInfo( + volumeName, bucketName, keyName, refreshContainerLocationsFromScm); + } catch (IOException e) { + LOG.warn("Unable to fetch key info from OM for key {}/{}/{}; marking verification incomplete.", + volumeName, bucketName, keyName, e); + markKeyFetchFailure(volumeName, bucketName, keyName, e.getMessage(), keysArray, allKeysPassed); + return; + } // Check if key should be processed based on replication config if (!shouldProcessKeyByReplicationType(keyInfo)) { return; } + KeyVerificationResult keyVerificationResult = verifyKey(keyInfo, volumeName, bucketName, keyName); + if (keyVerificationResult.shouldRefreshKeyLocation()) { + try { + OmKeyInfo refreshedKeyInfo = ozoneClient.getProxy().getKeyInfo( + volumeName, bucketName, keyName, true); + keyVerificationResult = verifyKey(refreshedKeyInfo, volumeName, bucketName, keyName); + } catch (IOException e) { + LOG.warn("Unable to refresh key location from OM for key {}/{}/{}; marking verification incomplete.", + volumeName, bucketName, keyName, e); + keyVerificationResult.markRefreshFailure(e.getMessage()); + } + } + + keyVerificationResult.getKeyNode().put("pass", keyVerificationResult.passed()); + if (keyVerificationResult.passed()) { + keysPassed.incrementAndGet(); + } else { + keysFailed.incrementAndGet(); + allKeysPassed.set(false); + keyVerificationResult.getFailedVerificationTypes().forEach(failedType -> failuresByType + .computeIfAbsent(failedType, k -> new AtomicInteger(0)) + .incrementAndGet() + ); + } + + if (!keyVerificationResult.passed() || allResults) { + keysArray.add(keyVerificationResult.getKeyNode()); + } + } + + private void markKeyFetchFailure(String volumeName, String bucketName, String keyName, String message, + ArrayNode keysArray, AtomicBoolean allKeysPassed) { + ObjectNode keyNode = JsonUtils.createObjectNode(null); + keyNode.put("volumeName", volumeName); + keyNode.put("bucketName", bucketName); + keyNode.put("name", keyName); + keyNode.putArray("blocks"); + keyNode.put("completed", false); + keyNode.put("pass", false); + keyNode.putArray("failures").addObject().put("message", "Failed to fetch key info from OM: " + message); + keysFailed.incrementAndGet(); + allKeysPassed.set(false); + keysArray.add(keyNode); + } + + private KeyVerificationResult verifyKey( + OmKeyInfo keyInfo, String volumeName, String bucketName, String keyName) { ObjectNode keyNode = JsonUtils.createObjectNode(null); keyNode.put("volumeName", volumeName); keyNode.put("bucketName", bucketName); @@ -353,7 +408,9 @@ void processKey(OzoneClient ozoneClient, String volumeName, String bucketName, S ArrayNode blocksArray = keyNode.putArray("blocks"); boolean keyPass = true; + boolean shouldRefreshKeyLocation = false; Set failedVerificationTypes = new HashSet<>(); + List refreshChecks = new ArrayList<>(); for (OmKeyLocationInfo keyLocation : keyInfo.getLatestVersionLocations().getBlocksLatestVersionOnly()) { long containerID = keyLocation.getContainerID(); @@ -383,6 +440,10 @@ void processKey(OzoneClient ozoneClient, String volumeName, String bucketName, S checkNode.put("type", verifier.getType()); checkNode.put("completed", result.isCompleted()); checkNode.put("pass", result.passed()); + if (result.shouldRefreshKeyLocation()) { + shouldRefreshKeyLocation = true; + refreshChecks.add(checkNode); + } ArrayNode failuresArray = checkNode.putArray("failures"); for (String failure : result.getFailures()) { @@ -406,20 +467,48 @@ void processKey(OzoneClient ozoneClient, String volumeName, String bucketName, S } } - keyNode.put("pass", keyPass); - if (keyPass) { - keysPassed.incrementAndGet(); - } else { - keysFailed.incrementAndGet(); - allKeysPassed.set(false); - failedVerificationTypes.forEach(failedType -> failuresByType - .computeIfAbsent(failedType, k -> new AtomicInteger(0)) - .incrementAndGet() - ); + return new KeyVerificationResult( + keyNode, keyPass, failedVerificationTypes, shouldRefreshKeyLocation, refreshChecks); + } + + private static final class KeyVerificationResult { + private final ObjectNode keyNode; + private final boolean pass; + private final Set failedVerificationTypes; + private final boolean refreshKeyLocation; + private final List refreshChecks; + + private KeyVerificationResult(ObjectNode keyNode, boolean pass, Set failedVerificationTypes, + boolean refreshKeyLocation, List refreshChecks) { + this.keyNode = keyNode; + this.pass = pass; + this.failedVerificationTypes = failedVerificationTypes; + this.refreshKeyLocation = refreshKeyLocation; + this.refreshChecks = refreshChecks; + } + + private ObjectNode getKeyNode() { + return keyNode; + } + + private boolean passed() { + return pass; } - if (!keyPass || allResults) { - keysArray.add(keyNode); + private Set getFailedVerificationTypes() { + return failedVerificationTypes; + } + + private boolean shouldRefreshKeyLocation() { + return refreshKeyLocation; + } + + private void markRefreshFailure(String message) { + String refreshFailure = "Failed to refresh key location from OM: " + message; + for (ObjectNode checkNode : refreshChecks) { + checkNode.put("completed", false); + checkNode.withArray("failures").addObject().put("message", refreshFailure); + } } } diff --git a/hadoop-ozone/cli-debug/src/test/java/org/apache/hadoop/ozone/debug/replicas/TestReplicasVerify.java b/hadoop-ozone/cli-debug/src/test/java/org/apache/hadoop/ozone/debug/replicas/TestReplicasVerify.java index b42a0cf87b9..a70d7bc7baf 100644 --- a/hadoop-ozone/cli-debug/src/test/java/org/apache/hadoop/ozone/debug/replicas/TestReplicasVerify.java +++ b/hadoop-ozone/cli-debug/src/test/java/org/apache/hadoop/ozone/debug/replicas/TestReplicasVerify.java @@ -18,15 +18,45 @@ package org.apache.hadoop.ozone.debug.replicas; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import java.io.IOException; import java.lang.reflect.Field; +import java.util.Arrays; +import java.util.Collections; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.hadoop.hdds.client.BlockID; +import org.apache.hadoop.hdds.client.StandaloneReplicationConfig; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.MockDatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.scm.pipeline.Pipeline; +import org.apache.hadoop.hdds.scm.pipeline.PipelineID; +import org.apache.hadoop.hdds.server.JsonUtils; +import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.client.protocol.ClientProtocol; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; +import org.apache.hadoop.ozone.shell.ShellReplicationOptions; import org.junit.jupiter.api.Test; import picocli.CommandLine; /** - * Unit tests for replicas verify command option parsing. + * Unit tests for replicas verify command. */ -public class TestReplicasVerify { +class TestReplicasVerify { @Test void testRefreshContainerLocationsFromScmOption() throws Exception { @@ -46,9 +76,200 @@ void testRefreshContainerLocationsFromScmDefault() throws Exception { assertThat(isRefreshContainerLocationsFromScmEnabled(command)).isFalse(); } - private boolean isRefreshContainerLocationsFromScmEnabled(ReplicasVerify command) throws Exception { + @Test + void testRetriesWithForcedContainerCacheRefreshOnBlockNotFound() throws Exception { + ReplicasVerify replicasVerify = new ReplicasVerify(); + ReplicaVerifier blockExistenceVerifier = mock(ReplicaVerifier.class); + ReplicaVerifier checksumVerifier = mock(ReplicaVerifier.class); + + when(blockExistenceVerifier.getType()).thenReturn("blockExistence"); + when(checksumVerifier.getType()).thenReturn("checksum"); + when(blockExistenceVerifier.verifyBlock(any(), any())) + .thenReturn(BlockVerificationResult.failCheckAndRefreshKeyLocation("Block not found")) + .thenReturn(BlockVerificationResult.pass()); + when(checksumVerifier.verifyBlock(any(), any())) + .thenReturn(BlockVerificationResult.pass()) + .thenReturn(BlockVerificationResult.pass()); + setField(replicasVerify, "replicaVerifiers", Arrays.asList(blockExistenceVerifier, checksumVerifier)); + setField(replicasVerify, "replication", emptyReplicationFilter()); + + OzoneClient ozoneClient = mock(OzoneClient.class); + ClientProtocol proxy = mock(ClientProtocol.class); + when(ozoneClient.getProxy()).thenReturn(proxy); + + OmKeyLocationInfo firstLocation = createKeyLocationInfo(1L, 11L); + OmKeyLocationInfo refreshedLocation = createKeyLocationInfo(2L, 22L); + when(proxy.getKeyInfo("vol1", "bucket1", "key1", false)) + .thenReturn(createKeyInfo(firstLocation)); + when(proxy.getKeyInfo("vol1", "bucket1", "key1", true)) + .thenReturn(createKeyInfo(refreshedLocation)); + + ObjectNode root = JsonUtils.createObjectNode(null); + ArrayNode keysArray = root.putArray("keys"); + AtomicBoolean allKeysPassed = new AtomicBoolean(true); + + replicasVerify.processKey(ozoneClient, "vol1", "bucket1", "key1", keysArray, allKeysPassed); + + verify(proxy).getKeyInfo("vol1", "bucket1", "key1", false); + verify(proxy).getKeyInfo("vol1", "bucket1", "key1", true); + verify(blockExistenceVerifier, times(2)).verifyBlock(any(), any()); + verify(checksumVerifier, times(2)).verifyBlock(any(), any()); + assertTrue(allKeysPassed.get()); + assertEquals(0, keysArray.size()); + } + + @Test + void testDoesNotRefreshOnNonRefreshableFailure() throws Exception { + ReplicasVerify replicasVerify = new ReplicasVerify(); + ReplicaVerifier blockExistenceVerifier = mock(ReplicaVerifier.class); + ReplicaVerifier checksumVerifier = mock(ReplicaVerifier.class); + + when(blockExistenceVerifier.getType()).thenReturn("blockExistence"); + when(checksumVerifier.getType()).thenReturn("checksum"); + when(blockExistenceVerifier.verifyBlock(any(), any())) + .thenReturn(BlockVerificationResult.failCheck("Block does not exist on this replica")); + when(checksumVerifier.verifyBlock(any(), any())) + .thenReturn(BlockVerificationResult.pass()); + setField(replicasVerify, "replicaVerifiers", Arrays.asList(blockExistenceVerifier, checksumVerifier)); + setField(replicasVerify, "replication", emptyReplicationFilter()); + + OzoneClient ozoneClient = mock(OzoneClient.class); + ClientProtocol proxy = mock(ClientProtocol.class); + when(ozoneClient.getProxy()).thenReturn(proxy); + when(proxy.getKeyInfo("vol1", "bucket1", "key1", false)) + .thenReturn(createKeyInfo(createKeyLocationInfo(1L, 11L))); + + ObjectNode root = JsonUtils.createObjectNode(null); + ArrayNode keysArray = root.putArray("keys"); + AtomicBoolean allKeysPassed = new AtomicBoolean(true); + + replicasVerify.processKey(ozoneClient, "vol1", "bucket1", "key1", keysArray, allKeysPassed); + + verify(proxy).getKeyInfo("vol1", "bucket1", "key1", false); + verify(proxy, never()).getKeyInfo("vol1", "bucket1", "key1", true); + verify(blockExistenceVerifier, times(1)).verifyBlock(any(), any()); + verify(checksumVerifier, times(1)).verifyBlock(any(), any()); + assertFalse(allKeysPassed.get()); + assertEquals(1, keysArray.size()); + assertFalse(keysArray.get(0).get("pass").asBoolean()); + } + + @Test + void testRefreshFailureMarksKeyIncompleteAndDoesNotAbortRun() throws Exception { + ReplicasVerify replicasVerify = new ReplicasVerify(); + ReplicaVerifier blockExistenceVerifier = mock(ReplicaVerifier.class); + + when(blockExistenceVerifier.getType()).thenReturn("blockExistence"); + when(blockExistenceVerifier.verifyBlock(any(), any())) + .thenReturn(BlockVerificationResult.failCheckAndRefreshKeyLocation("Block not found")); + setField(replicasVerify, "replicaVerifiers", Collections.singletonList(blockExistenceVerifier)); + setField(replicasVerify, "replication", emptyReplicationFilter()); + + OzoneClient ozoneClient = mock(OzoneClient.class); + ClientProtocol proxy = mock(ClientProtocol.class); + when(ozoneClient.getProxy()).thenReturn(proxy); + when(proxy.getKeyInfo("vol1", "bucket1", "key1", false)) + .thenReturn(createKeyInfo(createKeyLocationInfo(1L, 11L))); + when(proxy.getKeyInfo("vol1", "bucket1", "key1", true)) + .thenThrow(new IOException("OM refresh failed")); + + ObjectNode root = JsonUtils.createObjectNode(null); + ArrayNode keysArray = root.putArray("keys"); + AtomicBoolean allKeysPassed = new AtomicBoolean(true); + + replicasVerify.processKey(ozoneClient, "vol1", "bucket1", "key1", keysArray, allKeysPassed); + + verify(proxy).getKeyInfo("vol1", "bucket1", "key1", false); + verify(proxy).getKeyInfo("vol1", "bucket1", "key1", true); + assertFalse(allKeysPassed.get()); + assertEquals(1, keysArray.size()); + assertFalse(keysArray.get(0).get("pass").asBoolean()); + assertFalse(keysArray.get(0).get("blocks").get(0) + .get("replicas").get(0).get("checks").get(0).get("completed").asBoolean()); + assertThat(keysArray.get(0).get("blocks").get(0) + .get("replicas").get(0).get("checks").get(0).get("failures").toString()) + .contains("Failed to refresh key location from OM: OM refresh failed"); + } + + @Test + void testInitialGetKeyInfoFailureMarksKeyIncompleteAndDoesNotAbortRun() throws Exception { + ReplicasVerify replicasVerify = new ReplicasVerify(); + OzoneClient ozoneClient = mock(OzoneClient.class); + ClientProtocol proxy = mock(ClientProtocol.class); + when(ozoneClient.getProxy()).thenReturn(proxy); + when(proxy.getKeyInfo("vol1", "bucket1", "key1", false)) + .thenThrow(new IOException("Key not found")); + + ObjectNode root = JsonUtils.createObjectNode(null); + ArrayNode keysArray = root.putArray("keys"); + AtomicBoolean allKeysPassed = new AtomicBoolean(true); + + replicasVerify.processKey(ozoneClient, "vol1", "bucket1", "key1", keysArray, allKeysPassed); + + verify(proxy).getKeyInfo("vol1", "bucket1", "key1", false); + verify(proxy, never()).getKeyInfo("vol1", "bucket1", "key1", true); + assertFalse(allKeysPassed.get()); + assertEquals(1, keysArray.size()); + assertFalse(keysArray.get(0).get("completed").asBoolean()); + assertFalse(keysArray.get(0).get("pass").asBoolean()); + assertThat(keysArray.get(0).get("failures").toString()) + .contains("Failed to fetch key info from OM: Key not found"); + } + + private static boolean isRefreshContainerLocationsFromScmEnabled(ReplicasVerify command) throws Exception { Field field = ReplicasVerify.class.getDeclaredField("refreshContainerLocationsFromScm"); field.setAccessible(true); return field.getBoolean(command); } + + private static OmKeyInfo createKeyInfo(OmKeyLocationInfo keyLocationInfo) { + OmKeyLocationInfoGroup latestVersionLocations = + new OmKeyLocationInfoGroup(0, Collections.singletonList(keyLocationInfo)); + return new OmKeyInfo.Builder() + .setVolumeName("vol1") + .setBucketName("bucket1") + .setKeyName("key1") + .setReplicationConfig(StandaloneReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE)) + .setOmKeyLocationInfos(Collections.singletonList(latestVersionLocations)) + .build(); + } + + private static OmKeyLocationInfo createKeyLocationInfo(long containerId, long localId) { + DatanodeDetails datanode = MockDatanodeDetails.randomDatanodeDetails(); + Pipeline pipeline = Pipeline.newBuilder() + .setId(PipelineID.randomId()) + .setReplicationConfig(StandaloneReplicationConfig.getInstance(HddsProtos.ReplicationFactor.ONE)) + .setState(Pipeline.PipelineState.OPEN) + .setNodes(Collections.singletonList(datanode)) + .build(); + + return new OmKeyLocationInfo.Builder() + .setBlockID(new BlockID(containerId, localId)) + .setPipeline(pipeline) + .setLength(1) + .setOffset(0) + .setCreateVersion(0) + .build(); + } + + private static ShellReplicationOptions emptyReplicationFilter() { + ShellReplicationOptions replication = mock(ShellReplicationOptions.class); + when(replication.fromParams(any())).thenReturn(Optional.empty()); + return replication; + } + + private static void setField(Object target, String fieldName, Object value) throws Exception { + Class type = target.getClass(); + while (type != null) { + try { + Field field = type.getDeclaredField(fieldName); + field.setAccessible(true); + field.set(target, value); + return; + } catch (NoSuchFieldException ignored) { + type = type.getSuperclass(); + } + } + throw new IOException("Unable to set field '" + fieldName + "' on " + target.getClass().getName()); + } }