use of org.apache.hadoop.hdfs.tools.DFSAdmin in project hadoop by apache.
the class TestGenericRefresh method testMultipleReturnCodeMerging.
@Test
public void testMultipleReturnCodeMerging() throws Exception {
// Two handlers which return two non-zero values
RefreshHandler handlerOne = Mockito.mock(RefreshHandler.class);
Mockito.stub(handlerOne.handleRefresh(Mockito.anyString(), Mockito.any(String[].class))).toReturn(new RefreshResponse(23, "Twenty Three"));
RefreshHandler handlerTwo = Mockito.mock(RefreshHandler.class);
Mockito.stub(handlerTwo.handleRefresh(Mockito.anyString(), Mockito.any(String[].class))).toReturn(new RefreshResponse(10, "Ten"));
// Then registered to the same ID
RefreshRegistry.defaultRegistry().register("shared", handlerOne);
RefreshRegistry.defaultRegistry().register("shared", handlerTwo);
// We refresh both
DFSAdmin admin = new DFSAdmin(config);
String[] args = new String[] { "-refresh", "localhost:" + cluster.getNameNodePort(), "shared" };
int exitCode = admin.run(args);
// We get -1 because of our logic for melding non-zero return codes
assertEquals(-1, exitCode);
// Verify we called both
Mockito.verify(handlerOne).handleRefresh("shared", new String[] {});
Mockito.verify(handlerTwo).handleRefresh("shared", new String[] {});
RefreshRegistry.defaultRegistry().unregisterAll("shared");
}
use of org.apache.hadoop.hdfs.tools.DFSAdmin in project hadoop by apache.
the class TestClientProtocolForPipelineRecovery method testPipelineRecoveryOnOOB.
/**
* Test recovery on restart OOB message. It also tests the delivery of
* OOB ack originating from the primary datanode. Since there is only
* one node in the cluster, failure of restart-recovery will fail the
* test.
*/
@Test
public void testPipelineRecoveryOnOOB() throws Exception {
Configuration conf = new HdfsConfiguration();
conf.set(HdfsClientConfigKeys.DFS_CLIENT_DATANODE_RESTART_TIMEOUT_KEY, "15");
MiniDFSCluster cluster = null;
try {
int numDataNodes = 1;
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDataNodes).build();
cluster.waitActive();
FileSystem fileSys = cluster.getFileSystem();
Path file = new Path("dataprotocol2.dat");
DFSTestUtil.createFile(fileSys, file, 10240L, (short) 1, 0L);
DFSOutputStream out = (DFSOutputStream) (fileSys.append(file).getWrappedStream());
out.write(1);
out.hflush();
DFSAdmin dfsadmin = new DFSAdmin(conf);
DataNode dn = cluster.getDataNodes().get(0);
final String dnAddr = dn.getDatanodeId().getIpcAddr(false);
// issue shutdown to the datanode.
final String[] args1 = { "-shutdownDatanode", dnAddr, "upgrade" };
Assert.assertEquals(0, dfsadmin.run(args1));
// Wait long enough to receive an OOB ack before closing the file.
GenericTestUtils.waitForThreadTermination("Async datanode shutdown thread", 100, 10000);
// Retart the datanode
cluster.restartDataNode(0, true);
// The following forces a data packet and end of block packets to be sent.
out.close();
} finally {
if (cluster != null) {
cluster.shutdown();
}
}
}
use of org.apache.hadoop.hdfs.tools.DFSAdmin in project hadoop by apache.
the class TestClientProtocolForPipelineRecovery method testPipelineRecoveryOnRestartFailure.
/** Test restart timeout */
@Test
public void testPipelineRecoveryOnRestartFailure() throws Exception {
Configuration conf = new HdfsConfiguration();
conf.set(HdfsClientConfigKeys.DFS_CLIENT_DATANODE_RESTART_TIMEOUT_KEY, "5");
MiniDFSCluster cluster = null;
try {
int numDataNodes = 2;
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDataNodes).build();
cluster.waitActive();
FileSystem fileSys = cluster.getFileSystem();
Path file = new Path("dataprotocol3.dat");
DFSTestUtil.createFile(fileSys, file, 10240L, (short) 2, 0L);
DFSOutputStream out = (DFSOutputStream) (fileSys.append(file).getWrappedStream());
out.write(1);
out.hflush();
DFSAdmin dfsadmin = new DFSAdmin(conf);
DataNode dn = cluster.getDataNodes().get(0);
final String dnAddr1 = dn.getDatanodeId().getIpcAddr(false);
// issue shutdown to the datanode.
final String[] args1 = { "-shutdownDatanode", dnAddr1, "upgrade" };
Assert.assertEquals(0, dfsadmin.run(args1));
GenericTestUtils.waitForThreadTermination("Async datanode shutdown thread", 100, 10000);
// This should succeed without restarting the node. The restart will
// expire and regular pipeline recovery will kick in.
out.close();
// At this point there is only one node in the cluster.
out = (DFSOutputStream) (fileSys.append(file).getWrappedStream());
out.write(1);
out.hflush();
dn = cluster.getDataNodes().get(1);
final String dnAddr2 = dn.getDatanodeId().getIpcAddr(false);
// issue shutdown to the datanode.
final String[] args2 = { "-shutdownDatanode", dnAddr2, "upgrade" };
Assert.assertEquals(0, dfsadmin.run(args2));
GenericTestUtils.waitForThreadTermination("Async datanode shutdown thread", 100, 10000);
try {
// close should fail
out.close();
assert false;
} catch (IOException ioe) {
}
} finally {
if (cluster != null) {
cluster.shutdown();
}
}
}
use of org.apache.hadoop.hdfs.tools.DFSAdmin in project hadoop by apache.
the class TestBalancerBandwidth method testBalancerBandwidth.
@Test
public void testBalancerBandwidth() throws Exception {
/* Set bandwidthPerSec to a low value of 1M bps. */
conf.setLong(DFSConfigKeys.DFS_DATANODE_BALANCE_BANDWIDTHPERSEC_KEY, DEFAULT_BANDWIDTH);
/* Create and start cluster */
try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(NUM_OF_DATANODES).build()) {
cluster.waitActive();
DistributedFileSystem fs = cluster.getFileSystem();
ArrayList<DataNode> datanodes = cluster.getDataNodes();
// Ensure value from the configuration is reflected in the datanodes.
assertEquals(DEFAULT_BANDWIDTH, (long) datanodes.get(0).getBalancerBandwidth());
assertEquals(DEFAULT_BANDWIDTH, (long) datanodes.get(1).getBalancerBandwidth());
DFSAdmin admin = new DFSAdmin(conf);
String dn1Address = datanodes.get(0).ipcServer.getListenerAddress().getHostName() + ":" + datanodes.get(0).getIpcPort();
String dn2Address = datanodes.get(1).ipcServer.getListenerAddress().getHostName() + ":" + datanodes.get(1).getIpcPort();
// verifies the dfsadmin command execution
String[] args = new String[] { "-getBalancerBandwidth", dn1Address };
runGetBalancerBandwidthCmd(admin, args, DEFAULT_BANDWIDTH);
args = new String[] { "-getBalancerBandwidth", dn2Address };
runGetBalancerBandwidthCmd(admin, args, DEFAULT_BANDWIDTH);
// Dynamically change balancer bandwidth and ensure the updated value
// is reflected on the datanodes.
// 12M bps
long newBandwidth = 12 * DEFAULT_BANDWIDTH;
fs.setBalancerBandwidth(newBandwidth);
verifyBalancerBandwidth(datanodes, newBandwidth);
// verifies the dfsadmin command execution
args = new String[] { "-getBalancerBandwidth", dn1Address };
runGetBalancerBandwidthCmd(admin, args, newBandwidth);
args = new String[] { "-getBalancerBandwidth", dn2Address };
runGetBalancerBandwidthCmd(admin, args, newBandwidth);
// Dynamically change balancer bandwidth to 0. Balancer bandwidth on the
// datanodes should remain as it was.
fs.setBalancerBandwidth(0);
verifyBalancerBandwidth(datanodes, newBandwidth);
// verifies the dfsadmin command execution
args = new String[] { "-getBalancerBandwidth", dn1Address };
runGetBalancerBandwidthCmd(admin, args, newBandwidth);
args = new String[] { "-getBalancerBandwidth", dn2Address };
runGetBalancerBandwidthCmd(admin, args, newBandwidth);
}
}
use of org.apache.hadoop.hdfs.tools.DFSAdmin in project hadoop by apache.
the class TestDFSShell method testInvalidShell.
/**
* default setting is file:// which is not a DFS
* so DFSAdmin should throw and catch InvalidArgumentException
* and return -1 exit code.
* @throws Exception
*/
@Test(timeout = 30000)
public void testInvalidShell() throws Exception {
// default FS (non-DFS)
Configuration conf = new Configuration();
DFSAdmin admin = new DFSAdmin();
admin.setConf(conf);
int res = admin.run(new String[] { "-refreshNodes" });
assertEquals("expected to fail -1", res, -1);
}
Aggregations