use of org.apache.hadoop.ozone.protocol.commands.DeleteBlocksCommand in project ozone by apache.
the class HeartbeatEndpointTask method processResponse.
/**
* Add this command to command processing Queue.
*
* @param response - SCMHeartbeat response.
*/
private void processResponse(SCMHeartbeatResponseProto response, final DatanodeDetailsProto datanodeDetails) {
Preconditions.checkState(response.getDatanodeUUID().equalsIgnoreCase(datanodeDetails.getUuid()), "Unexpected datanode ID in the response.");
// Verify the response is indeed for this datanode.
for (SCMCommandProto commandResponseProto : response.getCommandsList()) {
switch(commandResponseProto.getCommandType()) {
case reregisterCommand:
if (rpcEndpoint.getState() == EndPointStates.HEARTBEAT) {
if (LOG.isDebugEnabled()) {
LOG.debug("Received SCM notification to register." + " Interrupt HEARTBEAT and transit to REGISTER state.");
}
rpcEndpoint.setState(EndPointStates.REGISTER);
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("Illegal state {} found, expecting {}.", rpcEndpoint.getState().name(), EndPointStates.HEARTBEAT);
}
}
break;
case deleteBlocksCommand:
DeleteBlocksCommand deleteBlocksCommand = DeleteBlocksCommand.getFromProtobuf(commandResponseProto.getDeleteBlocksCommandProto());
if (commandResponseProto.hasTerm()) {
deleteBlocksCommand.setTerm(commandResponseProto.getTerm());
}
if (!deleteBlocksCommand.blocksTobeDeleted().isEmpty()) {
if (LOG.isDebugEnabled()) {
LOG.debug(DeletedContainerBlocksSummary.getFrom(deleteBlocksCommand.blocksTobeDeleted()).toString());
}
this.context.addCommand(deleteBlocksCommand);
}
break;
case closeContainerCommand:
CloseContainerCommand closeContainer = CloseContainerCommand.getFromProtobuf(commandResponseProto.getCloseContainerCommandProto());
if (commandResponseProto.hasTerm()) {
closeContainer.setTerm(commandResponseProto.getTerm());
}
if (commandResponseProto.hasEncodedToken()) {
closeContainer.setEncodedToken(commandResponseProto.getEncodedToken());
}
if (LOG.isDebugEnabled()) {
LOG.debug("Received SCM container close request for container {}", closeContainer.getContainerID());
}
this.context.addCommand(closeContainer);
break;
case replicateContainerCommand:
ReplicateContainerCommand replicateContainerCommand = ReplicateContainerCommand.getFromProtobuf(commandResponseProto.getReplicateContainerCommandProto());
if (commandResponseProto.hasTerm()) {
replicateContainerCommand.setTerm(commandResponseProto.getTerm());
}
if (LOG.isDebugEnabled()) {
LOG.debug("Received SCM container replicate request for container {}", replicateContainerCommand.getContainerID());
}
this.context.addCommand(replicateContainerCommand);
break;
case deleteContainerCommand:
DeleteContainerCommand deleteContainerCommand = DeleteContainerCommand.getFromProtobuf(commandResponseProto.getDeleteContainerCommandProto());
if (commandResponseProto.hasTerm()) {
deleteContainerCommand.setTerm(commandResponseProto.getTerm());
}
if (LOG.isDebugEnabled()) {
LOG.debug("Received SCM delete container request for container {}", deleteContainerCommand.getContainerID());
}
this.context.addCommand(deleteContainerCommand);
break;
case createPipelineCommand:
CreatePipelineCommand createPipelineCommand = CreatePipelineCommand.getFromProtobuf(commandResponseProto.getCreatePipelineCommandProto());
if (commandResponseProto.hasTerm()) {
createPipelineCommand.setTerm(commandResponseProto.getTerm());
}
if (LOG.isDebugEnabled()) {
LOG.debug("Received SCM create pipeline request {}", createPipelineCommand.getPipelineID());
}
this.context.addCommand(createPipelineCommand);
break;
case closePipelineCommand:
ClosePipelineCommand closePipelineCommand = ClosePipelineCommand.getFromProtobuf(commandResponseProto.getClosePipelineCommandProto());
if (commandResponseProto.hasTerm()) {
closePipelineCommand.setTerm(commandResponseProto.getTerm());
}
if (LOG.isDebugEnabled()) {
LOG.debug("Received SCM close pipeline request {}", closePipelineCommand.getPipelineID());
}
this.context.addCommand(closePipelineCommand);
break;
case setNodeOperationalStateCommand:
SetNodeOperationalStateCommand setNodeOperationalStateCommand = SetNodeOperationalStateCommand.getFromProtobuf(commandResponseProto.getSetNodeOperationalStateCommandProto());
if (commandResponseProto.hasTerm()) {
setNodeOperationalStateCommand.setTerm(commandResponseProto.getTerm());
}
if (LOG.isDebugEnabled()) {
LOG.debug("Received SCM set operational state command. State: {} " + "Expiry: {}", setNodeOperationalStateCommand.getOpState(), setNodeOperationalStateCommand.getStateExpiryEpochSeconds());
}
this.context.addCommand(setNodeOperationalStateCommand);
break;
case finalizeNewLayoutVersionCommand:
FinalizeNewLayoutVersionCommand finalizeNewLayoutVersionCommand = FinalizeNewLayoutVersionCommand.getFromProtobuf(commandResponseProto.getFinalizeNewLayoutVersionCommandProto());
if (commandResponseProto.hasTerm()) {
finalizeNewLayoutVersionCommand.setTerm(commandResponseProto.getTerm());
}
if (LOG.isDebugEnabled()) {
LOG.debug("Received SCM finalize command {}", finalizeNewLayoutVersionCommand.getId());
}
this.context.addCommand(finalizeNewLayoutVersionCommand);
break;
case refreshVolumeUsageInfo:
RefreshVolumeUsageCommand refreshVolumeUsageCommand = RefreshVolumeUsageCommand.getFromProtobuf(commandResponseProto.getRefreshVolumeUsageCommandProto());
if (commandResponseProto.hasTerm()) {
refreshVolumeUsageCommand.setTerm(commandResponseProto.getTerm());
}
this.context.addCommand(refreshVolumeUsageCommand);
break;
default:
throw new IllegalArgumentException("Unknown response : " + commandResponseProto.getCommandType().name());
}
}
}
use of org.apache.hadoop.ozone.protocol.commands.DeleteBlocksCommand in project ozone by apache.
the class TestStorageContainerManager method testBlockDeletingThrottling.
@Test
public void testBlockDeletingThrottling() throws Exception {
int numKeys = 15;
OzoneConfiguration conf = new OzoneConfiguration();
conf.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL, 1, TimeUnit.SECONDS);
conf.setInt(ScmConfigKeys.OZONE_SCM_BLOCK_DELETION_MAX_RETRY, 5);
conf.setTimeDuration(OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 100, TimeUnit.MILLISECONDS);
ScmConfig scmConfig = conf.getObject(ScmConfig.class);
scmConfig.setBlockDeletionInterval(Duration.ofMillis(100));
conf.setFromObject(scmConfig);
conf.setInt(ScmConfigKeys.OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT, numKeys);
conf.setBoolean(HDDS_SCM_SAFEMODE_PIPELINE_CREATION, false);
MiniOzoneCluster cluster = MiniOzoneCluster.newBuilder(conf).setHbInterval(1000).setHbProcessorInterval(3000).setNumDatanodes(1).build();
cluster.waitForClusterToBeReady();
cluster.waitForPipelineTobeReady(HddsProtos.ReplicationFactor.ONE, 30000);
try {
DeletedBlockLog delLog = cluster.getStorageContainerManager().getScmBlockManager().getDeletedBlockLog();
Assert.assertEquals(0, delLog.getNumOfValidTransactions());
int limitSize = 1;
// Reset limit value to 1, so that we only allow one TX is dealt per
// datanode.
SCMBlockDeletingService delService = cluster.getStorageContainerManager().getScmBlockManager().getSCMBlockDeletingService();
delService.setBlockDeleteTXNum(limitSize);
// Create {numKeys} random names keys.
TestStorageContainerManagerHelper helper = new TestStorageContainerManagerHelper(cluster, conf);
Map<String, OmKeyInfo> keyLocations = helper.createKeys(numKeys, 4096);
// Wait for container report
Thread.sleep(5000);
for (OmKeyInfo keyInfo : keyLocations.values()) {
OzoneTestUtils.closeContainers(keyInfo.getKeyLocationVersions(), cluster.getStorageContainerManager());
}
createDeleteTXLog(cluster.getStorageContainerManager(), delLog, keyLocations, helper);
// Verify a few TX gets created in the TX log.
Assert.assertTrue(delLog.getNumOfValidTransactions() > 0);
// Verify the size in delete commands is expected.
GenericTestUtils.waitFor(() -> {
NodeManager nodeManager = cluster.getStorageContainerManager().getScmNodeManager();
LayoutVersionManager versionManager = nodeManager.getLayoutVersionManager();
StorageContainerDatanodeProtocolProtos.LayoutVersionProto layoutInfo = StorageContainerDatanodeProtocolProtos.LayoutVersionProto.newBuilder().setSoftwareLayoutVersion(versionManager.getSoftwareLayoutVersion()).setMetadataLayoutVersion(versionManager.getMetadataLayoutVersion()).build();
List<SCMCommand> commands = nodeManager.processHeartbeat(nodeManager.getNodes(NodeStatus.inServiceHealthy()).get(0), layoutInfo);
if (commands != null) {
for (SCMCommand cmd : commands) {
if (cmd.getType() == SCMCommandProto.Type.deleteBlocksCommand) {
List<DeletedBlocksTransaction> deletedTXs = ((DeleteBlocksCommand) cmd).blocksTobeDeleted();
return deletedTXs != null && deletedTXs.size() == limitSize;
}
}
}
return false;
}, 500, 10000);
} finally {
cluster.shutdown();
}
}
Aggregations