Search in sources :

Example 46 with MiniDFSCluster

use of org.apache.hadoop.hdfs.MiniDFSCluster in project hadoop by apache.

the class TestWebHDFSForHA method testRetryWhileNNStartup.

/**
   * Make sure the WebHdfsFileSystem will retry based on RetriableException when
   * rpcServer is null in NamenodeWebHdfsMethods while NameNode starts up.
   */
@Test(timeout = 120000)
public void testRetryWhileNNStartup() throws Exception {
    final Configuration conf = DFSTestUtil.newHAConfiguration(LOGICAL_NAME);
    MiniDFSCluster cluster = null;
    final Map<String, Boolean> resultMap = new HashMap<String, Boolean>();
    try {
        cluster = new MiniDFSCluster.Builder(conf).nnTopology(topo).numDataNodes(0).build();
        HATestUtil.setFailoverConfigurations(cluster, conf, LOGICAL_NAME);
        cluster.waitActive();
        cluster.transitionToActive(0);
        final NameNode namenode = cluster.getNameNode(0);
        final NamenodeProtocols rpcServer = namenode.getRpcServer();
        Whitebox.setInternalState(namenode, "rpcServer", null);
        new Thread() {

            @Override
            public void run() {
                boolean result = false;
                FileSystem fs = null;
                try {
                    fs = FileSystem.get(WEBHDFS_URI, conf);
                    final Path dir = new Path("/test");
                    result = fs.mkdirs(dir);
                } catch (IOException e) {
                    result = false;
                } finally {
                    IOUtils.cleanup(null, fs);
                }
                synchronized (TestWebHDFSForHA.this) {
                    resultMap.put("mkdirs", result);
                    TestWebHDFSForHA.this.notifyAll();
                }
            }
        }.start();
        Thread.sleep(1000);
        Whitebox.setInternalState(namenode, "rpcServer", rpcServer);
        synchronized (this) {
            while (!resultMap.containsKey("mkdirs")) {
                this.wait();
            }
            Assert.assertTrue(resultMap.get("mkdirs"));
        }
    } finally {
        if (cluster != null) {
            cluster.shutdown();
        }
    }
}
Also used : Path(org.apache.hadoop.fs.Path) NameNode(org.apache.hadoop.hdfs.server.namenode.NameNode) NamenodeProtocols(org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols) MiniDFSCluster(org.apache.hadoop.hdfs.MiniDFSCluster) Configuration(org.apache.hadoop.conf.Configuration) HashMap(java.util.HashMap) IOException(java.io.IOException) FileSystem(org.apache.hadoop.fs.FileSystem) Test(org.junit.Test)

Example 47 with MiniDFSCluster

use of org.apache.hadoop.hdfs.MiniDFSCluster in project hadoop by apache.

the class TestWebHdfsTokens method testSetTokenServiceAndKind.

@Test
public void testSetTokenServiceAndKind() throws Exception {
    MiniDFSCluster cluster = null;
    try {
        final Configuration clusterConf = new HdfsConfiguration(conf);
        SecurityUtil.setAuthenticationMethod(SIMPLE, clusterConf);
        clusterConf.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
        // trick the NN into thinking s[ecurity is enabled w/o it trying
        // to login from a keytab
        UserGroupInformation.setConfiguration(clusterConf);
        cluster = new MiniDFSCluster.Builder(clusterConf).numDataNodes(0).build();
        cluster.waitActive();
        SecurityUtil.setAuthenticationMethod(KERBEROS, clusterConf);
        final WebHdfsFileSystem fs = WebHdfsTestUtil.getWebHdfsFileSystem(clusterConf, "webhdfs");
        Whitebox.setInternalState(fs, "canRefreshDelegationToken", true);
        URLConnectionFactory factory = new URLConnectionFactory(new ConnectionConfigurator() {

            @Override
            public HttpURLConnection configure(HttpURLConnection conn) throws IOException {
                return conn;
            }
        }) {

            @Override
            public URLConnection openConnection(URL url) throws IOException {
                return super.openConnection(new URL(url + "&service=foo&kind=bar"));
            }
        };
        Whitebox.setInternalState(fs, "connectionFactory", factory);
        Token<?> token1 = fs.getDelegationToken();
        Assert.assertEquals(new Text("bar"), token1.getKind());
        final HttpOpParam.Op op = GetOpParam.Op.GETDELEGATIONTOKEN;
        Token<DelegationTokenIdentifier> token2 = fs.new FsPathResponseRunner<Token<DelegationTokenIdentifier>>(op, null, new RenewerParam(null)) {

            @Override
            Token<DelegationTokenIdentifier> decodeResponse(Map<?, ?> json) throws IOException {
                return JsonUtilClient.toDelegationToken(json);
            }
        }.run();
        Assert.assertEquals(new Text("bar"), token2.getKind());
        Assert.assertEquals(new Text("foo"), token2.getService());
    } finally {
        if (cluster != null) {
            cluster.shutdown();
        }
    }
}
Also used : ConnectionConfigurator(org.apache.hadoop.security.authentication.client.ConnectionConfigurator) MiniDFSCluster(org.apache.hadoop.hdfs.MiniDFSCluster) Configuration(org.apache.hadoop.conf.Configuration) HdfsConfiguration(org.apache.hadoop.hdfs.HdfsConfiguration) DelegationTokenIdentifier(org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier) Text(org.apache.hadoop.io.Text) InvalidToken(org.apache.hadoop.security.token.SecretManager.InvalidToken) Token(org.apache.hadoop.security.token.Token) IOException(java.io.IOException) HdfsConfiguration(org.apache.hadoop.hdfs.HdfsConfiguration) URL(java.net.URL) HttpURLConnection(java.net.HttpURLConnection) Test(org.junit.Test)

Example 48 with MiniDFSCluster

use of org.apache.hadoop.hdfs.MiniDFSCluster in project hadoop by apache.

the class TestBlockToken method testBlockTokenInLastLocatedBlock.

/**
   * This test writes a file and gets the block locations without closing the
   * file, and tests the block token in the last block. Block token is verified
   * by ensuring it is of correct kind.
   *
   * @throws IOException
   * @throws InterruptedException
   */
private void testBlockTokenInLastLocatedBlock(boolean enableProtobuf) throws IOException, InterruptedException {
    Configuration conf = new HdfsConfiguration();
    conf.setBoolean(DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, true);
    conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 512);
    conf.setBoolean(DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_PROTOBUF_ENABLE, enableProtobuf);
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
    cluster.waitActive();
    try {
        FileSystem fs = cluster.getFileSystem();
        String fileName = "/testBlockTokenInLastLocatedBlock";
        Path filePath = new Path(fileName);
        FSDataOutputStream out = fs.create(filePath, (short) 1);
        out.write(new byte[1000]);
        // ensure that the first block is written out (see FSOutputSummer#flush)
        out.flush();
        LocatedBlocks locatedBlocks = cluster.getNameNodeRpc().getBlockLocations(fileName, 0, 1000);
        while (locatedBlocks.getLastLocatedBlock() == null) {
            Thread.sleep(100);
            locatedBlocks = cluster.getNameNodeRpc().getBlockLocations(fileName, 0, 1000);
        }
        Token<BlockTokenIdentifier> token = locatedBlocks.getLastLocatedBlock().getBlockToken();
        Assert.assertEquals(BlockTokenIdentifier.KIND_NAME, token.getKind());
        out.close();
    } finally {
        cluster.shutdown();
    }
}
Also used : Path(org.apache.hadoop.fs.Path) MiniDFSCluster(org.apache.hadoop.hdfs.MiniDFSCluster) Configuration(org.apache.hadoop.conf.Configuration) HdfsConfiguration(org.apache.hadoop.hdfs.HdfsConfiguration) FileSystem(org.apache.hadoop.fs.FileSystem) LocatedBlocks(org.apache.hadoop.hdfs.protocol.LocatedBlocks) FSDataOutputStream(org.apache.hadoop.fs.FSDataOutputStream) HdfsConfiguration(org.apache.hadoop.hdfs.HdfsConfiguration)

Example 49 with MiniDFSCluster

use of org.apache.hadoop.hdfs.MiniDFSCluster in project hadoop by apache.

the class TestNNWithQJM method testMismatchedNNIsRejected.

@Test(timeout = 30000)
public void testMismatchedNNIsRejected() throws Exception {
    conf.set(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY, MiniDFSCluster.getBaseDirectory() + "/TestNNWithQJM/image");
    String defaultEditsDir = conf.get(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY);
    conf.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, mjc.getQuorumJournalURI("myjournal").toString());
    // Start a NN, so the storage is formatted -- both on-disk
    // and QJM.
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).manageNameDfsDirs(false).build();
    cluster.shutdown();
    // Reformat just the on-disk portion
    Configuration onDiskOnly = new Configuration(conf);
    onDiskOnly.set(DFSConfigKeys.DFS_NAMENODE_EDITS_DIR_KEY, defaultEditsDir);
    NameNode.format(onDiskOnly);
    // with the old namespace ID.
    try {
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).manageNameDfsDirs(false).format(false).build();
        fail("New NN with different namespace should have been rejected");
    } catch (IOException ioe) {
        GenericTestUtils.assertExceptionContains("Unable to start log segment 1: too few journals", ioe);
    }
}
Also used : MiniDFSCluster(org.apache.hadoop.hdfs.MiniDFSCluster) HdfsConfiguration(org.apache.hadoop.hdfs.HdfsConfiguration) Configuration(org.apache.hadoop.conf.Configuration) IOException(java.io.IOException) Test(org.junit.Test)

Example 50 with MiniDFSCluster

use of org.apache.hadoop.hdfs.MiniDFSCluster in project hadoop by apache.

the class TestWriteToReplica method testReplicaMapAfterDatanodeRestart.

/**
   * This is a test to check the replica map before and after the datanode 
   * quick restart (less than 5 minutes)
   * @throws Exception
   */
@Test
public void testReplicaMapAfterDatanodeRestart() throws Exception {
    Configuration conf = new HdfsConfiguration();
    MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).nnTopology(MiniDFSNNTopology.simpleFederatedTopology(2)).build();
    try {
        cluster.waitActive();
        NameNode nn1 = cluster.getNameNode(0);
        NameNode nn2 = cluster.getNameNode(1);
        assertNotNull("cannot create nn1", nn1);
        assertNotNull("cannot create nn2", nn2);
        // check number of volumes in fsdataset
        DataNode dn = cluster.getDataNodes().get(0);
        FsDatasetImpl dataSet = (FsDatasetImpl) DataNodeTestUtils.getFSDataset(dn);
        List<FsVolumeSpi> volumes = null;
        try (FsDatasetSpi.FsVolumeReferences referredVols = dataSet.getFsVolumeReferences()) {
            // number of volumes should be 2 - [data1, data2]
            assertEquals("number of volumes is wrong", 2, referredVols.size());
            volumes = new ArrayList<>(referredVols.size());
            for (FsVolumeSpi vol : referredVols) {
                volumes.add(vol);
            }
        }
        ArrayList<String> bpList = new ArrayList<>(Arrays.asList(cluster.getNamesystem(0).getBlockPoolId(), cluster.getNamesystem(1).getBlockPoolId()));
        Assert.assertTrue("Cluster should have 2 block pools", bpList.size() == 2);
        createReplicas(bpList, volumes, cluster.getFsDatasetTestUtils(dn));
        ReplicaMap oldReplicaMap = new ReplicaMap(new AutoCloseableLock());
        oldReplicaMap.addAll(dataSet.volumeMap);
        cluster.restartDataNode(0);
        cluster.waitActive();
        dn = cluster.getDataNodes().get(0);
        dataSet = (FsDatasetImpl) dn.getFSDataset();
        testEqualityOfReplicaMap(oldReplicaMap, dataSet.volumeMap, bpList);
    } finally {
        cluster.shutdown();
    }
}
Also used : NameNode(org.apache.hadoop.hdfs.server.namenode.NameNode) MiniDFSCluster(org.apache.hadoop.hdfs.MiniDFSCluster) Configuration(org.apache.hadoop.conf.Configuration) HdfsConfiguration(org.apache.hadoop.hdfs.HdfsConfiguration) FsDatasetSpi(org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi) ArrayList(java.util.ArrayList) HdfsConfiguration(org.apache.hadoop.hdfs.HdfsConfiguration) DataNode(org.apache.hadoop.hdfs.server.datanode.DataNode) AutoCloseableLock(org.apache.hadoop.util.AutoCloseableLock) FsVolumeSpi(org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi) Test(org.junit.Test)

Aggregations

MiniDFSCluster (org.apache.hadoop.hdfs.MiniDFSCluster)507 Test (org.junit.Test)429 Configuration (org.apache.hadoop.conf.Configuration)403 HdfsConfiguration (org.apache.hadoop.hdfs.HdfsConfiguration)312 Path (org.apache.hadoop.fs.Path)290 FileSystem (org.apache.hadoop.fs.FileSystem)211 DistributedFileSystem (org.apache.hadoop.hdfs.DistributedFileSystem)183 IOException (java.io.IOException)107 File (java.io.File)83 FSDataOutputStream (org.apache.hadoop.fs.FSDataOutputStream)64 ExtendedBlock (org.apache.hadoop.hdfs.protocol.ExtendedBlock)53 DataNode (org.apache.hadoop.hdfs.server.datanode.DataNode)35 RandomAccessFile (java.io.RandomAccessFile)33 MetricsRecordBuilder (org.apache.hadoop.metrics2.MetricsRecordBuilder)33 URI (java.net.URI)31 ArrayList (java.util.ArrayList)29 LocatedBlock (org.apache.hadoop.hdfs.protocol.LocatedBlock)28 FSNamesystem (org.apache.hadoop.hdfs.server.namenode.FSNamesystem)26 FsPermission (org.apache.hadoop.fs.permission.FsPermission)25 HttpServerFunctionalTest (org.apache.hadoop.http.HttpServerFunctionalTest)24