Examples with NamenodeProtocols org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols used on opensource projects

Example 6 with NamenodeProtocols

use of org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols in project hadoop by apache.

the class TestRequestHedgingProxyProvider method testHedgingWhenOneIsSlow.

@Test
public void testHedgingWhenOneIsSlow() throws Exception {
    final NamenodeProtocols goodMock = Mockito.mock(NamenodeProtocols.class);
    Mockito.when(goodMock.getStats()).thenAnswer(new Answer<long[]>() {

        @Override
        public long[] answer(InvocationOnMock invocation) throws Throwable {
            Thread.sleep(1000);
            return new long[] { 1 };
        }
    });
    final NamenodeProtocols badMock = Mockito.mock(NamenodeProtocols.class);
    Mockito.when(badMock.getStats()).thenThrow(new IOException("Bad mock !!"));
    RequestHedgingProxyProvider<NamenodeProtocols> provider = new RequestHedgingProxyProvider<>(conf, nnUri, NamenodeProtocols.class, createFactory(goodMock, badMock));
    long[] stats = provider.getProxy().proxy.getStats();
    Assert.assertTrue(stats.length == 1);
    Assert.assertEquals(1, stats[0]);
    Mockito.verify(badMock).getStats();
    Mockito.verify(goodMock).getStats();
}

Example 7 with NamenodeProtocols

use of org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols in project hadoop by apache.

the class TestRequestHedgingProxyProvider method testPerformFailover.

@Test
public void testPerformFailover() throws Exception {
    final AtomicInteger counter = new AtomicInteger(0);
    final int[] isGood = { 1 };
    final NamenodeProtocols goodMock = Mockito.mock(NamenodeProtocols.class);
    Mockito.when(goodMock.getStats()).thenAnswer(new Answer<long[]>() {

        @Override
        public long[] answer(InvocationOnMock invocation) throws Throwable {
            counter.incrementAndGet();
            if (isGood[0] == 1) {
                Thread.sleep(1000);
                return new long[] { 1 };
            }
            throw new IOException("Was Good mock !!");
        }
    });
    final NamenodeProtocols badMock = Mockito.mock(NamenodeProtocols.class);
    Mockito.when(badMock.getStats()).thenAnswer(new Answer<long[]>() {

        @Override
        public long[] answer(InvocationOnMock invocation) throws Throwable {
            counter.incrementAndGet();
            if (isGood[0] == 2) {
                Thread.sleep(1000);
                return new long[] { 2 };
            }
            throw new IOException("Bad mock !!");
        }
    });
    RequestHedgingProxyProvider<NamenodeProtocols> provider = new RequestHedgingProxyProvider<>(conf, nnUri, NamenodeProtocols.class, createFactory(goodMock, badMock));
    long[] stats = provider.getProxy().proxy.getStats();
    Assert.assertTrue(stats.length == 1);
    Assert.assertEquals(1, stats[0]);
    Assert.assertEquals(2, counter.get());
    Mockito.verify(badMock).getStats();
    Mockito.verify(goodMock).getStats();
    stats = provider.getProxy().proxy.getStats();
    Assert.assertTrue(stats.length == 1);
    Assert.assertEquals(1, stats[0]);
    // Ensure only the previous successful one is invoked
    Mockito.verifyNoMoreInteractions(badMock);
    Assert.assertEquals(3, counter.get());
    // Flip to standby.. so now this should fail
    isGood[0] = 2;
    try {
        provider.getProxy().proxy.getStats();
        Assert.fail("Should fail since previously successful proxy now fails ");
    } catch (Exception ex) {
        Assert.assertTrue(ex instanceof IOException);
    }
    Assert.assertEquals(4, counter.get());
    provider.performFailover(provider.getProxy().proxy);
    stats = provider.getProxy().proxy.getStats();
    Assert.assertTrue(stats.length == 1);
    Assert.assertEquals(2, stats[0]);
    // Counter should update only once
    Assert.assertEquals(5, counter.get());
    stats = provider.getProxy().proxy.getStats();
    Assert.assertTrue(stats.length == 1);
    Assert.assertEquals(2, stats[0]);
    // Counter updates only once now
    Assert.assertEquals(6, counter.get());
    // Flip back to old active.. so now this should fail
    isGood[0] = 1;
    try {
        provider.getProxy().proxy.getStats();
        Assert.fail("Should fail since previously successful proxy now fails ");
    } catch (Exception ex) {
        Assert.assertTrue(ex instanceof IOException);
    }
    Assert.assertEquals(7, counter.get());
    provider.performFailover(provider.getProxy().proxy);
    stats = provider.getProxy().proxy.getStats();
    Assert.assertTrue(stats.length == 1);
    // Ensure correct proxy was called
    Assert.assertEquals(1, stats[0]);
}

Example 8 with NamenodeProtocols

use of org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols in project hadoop by apache.

the class TestRequestHedgingProxyProvider method testHedgingWhenFileNotFoundException.

@Test
public void testHedgingWhenFileNotFoundException() throws Exception {
    NamenodeProtocols active = Mockito.mock(NamenodeProtocols.class);
    Mockito.when(active.getBlockLocations(Matchers.anyString(), Matchers.anyLong(), Matchers.anyLong())).thenThrow(new RemoteException("java.io.FileNotFoundException", "File does not exist!"));
    NamenodeProtocols standby = Mockito.mock(NamenodeProtocols.class);
    Mockito.when(standby.getBlockLocations(Matchers.anyString(), Matchers.anyLong(), Matchers.anyLong())).thenThrow(new RemoteException("org.apache.hadoop.ipc.StandbyException", "Standby NameNode"));
    RequestHedgingProxyProvider<NamenodeProtocols> provider = new RequestHedgingProxyProvider<>(conf, nnUri, NamenodeProtocols.class, createFactory(active, standby));
    try {
        provider.getProxy().proxy.getBlockLocations("/tmp/test.file", 0L, 20L);
        Assert.fail("Should fail since the active namenode throws" + " FileNotFoundException!");
    } catch (MultiException me) {
        for (Exception ex : me.getExceptions().values()) {
            Exception rEx = ((RemoteException) ex).unwrapRemoteException();
            if (rEx instanceof StandbyException) {
                continue;
            }
            Assert.assertTrue(rEx instanceof FileNotFoundException);
        }
    }
    Mockito.verify(active).getBlockLocations(Matchers.anyString(), Matchers.anyLong(), Matchers.anyLong());
    Mockito.verify(standby).getBlockLocations(Matchers.anyString(), Matchers.anyLong(), Matchers.anyLong());
}

Example 9 with NamenodeProtocols

use of org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols in project hadoop by apache.

the class TestWebHDFS method testRaceWhileNNStartup.

/**
   * Make sure a RetriableException is thrown when rpcServer is null in
   * NamenodeWebHdfsMethods.
   */
@Test
public void testRaceWhileNNStartup() throws Exception {
    MiniDFSCluster cluster = null;
    final Configuration conf = WebHdfsTestUtil.createConf();
    try {
        cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
        cluster.waitActive();
        final NameNode namenode = cluster.getNameNode();
        final NamenodeProtocols rpcServer = namenode.getRpcServer();
        Whitebox.setInternalState(namenode, "rpcServer", null);
        final Path foo = new Path("/foo");
        final FileSystem webHdfs = WebHdfsTestUtil.getWebHdfsFileSystem(conf, WebHdfsConstants.WEBHDFS_SCHEME);
        try {
            webHdfs.mkdirs(foo);
            fail("Expected RetriableException");
        } catch (RetriableException e) {
            GenericTestUtils.assertExceptionContains("Namenode is in startup mode", e);
        }
        Whitebox.setInternalState(namenode, "rpcServer", rpcServer);
    } finally {
        if (cluster != null) {
            cluster.shutdown();
        }
    }
}

Example 10 with NamenodeProtocols

use of org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols in project hadoop by apache.

the class TestWebHDFSForHA method testRetryWhileNNStartup.

/**
   * Make sure the WebHdfsFileSystem will retry based on RetriableException when
   * rpcServer is null in NamenodeWebHdfsMethods while NameNode starts up.
   */
@Test(timeout = 120000)
public void testRetryWhileNNStartup() throws Exception {
    final Configuration conf = DFSTestUtil.newHAConfiguration(LOGICAL_NAME);
    MiniDFSCluster cluster = null;
    final Map<String, Boolean> resultMap = new HashMap<String, Boolean>();
    try {
        cluster = new MiniDFSCluster.Builder(conf).nnTopology(topo).numDataNodes(0).build();
        HATestUtil.setFailoverConfigurations(cluster, conf, LOGICAL_NAME);
        cluster.waitActive();
        cluster.transitionToActive(0);
        final NameNode namenode = cluster.getNameNode(0);
        final NamenodeProtocols rpcServer = namenode.getRpcServer();
        Whitebox.setInternalState(namenode, "rpcServer", null);
        new Thread() {

            @Override
            public void run() {
                boolean result = false;
                FileSystem fs = null;
                try {
                    fs = FileSystem.get(WEBHDFS_URI, conf);
                    final Path dir = new Path("/test");
                    result = fs.mkdirs(dir);
                } catch (IOException e) {
                    result = false;
                } finally {
                    IOUtils.cleanup(null, fs);
                }
                synchronized (TestWebHDFSForHA.this) {
                    resultMap.put("mkdirs", result);
                    TestWebHDFSForHA.this.notifyAll();
                }
            }
        }.start();
        Thread.sleep(1000);
        Whitebox.setInternalState(namenode, "rpcServer", rpcServer);
        synchronized (this) {
            while (!resultMap.containsKey("mkdirs")) {
                this.wait();
            }
            Assert.assertTrue(resultMap.get("mkdirs"));
        }
    } finally {
        if (cluster != null) {
            cluster.shutdown();
        }
    }
}