Examples with BrokenBarrierException java.util.concurrent.BrokenBarrierException used on opensource projects

Example 1 with BrokenBarrierException

use of java.util.concurrent.BrokenBarrierException in project hadoop by apache.

the class TestDataNodeHotSwapVolumes method testRemoveVolumeBeingWrittenForDatanode.

/**
   * Test the case that remove a data volume on a particular DataNode when the
   * volume is actively being written.
   * @param dataNodeIdx the index of the DataNode to remove a volume.
   */
private void testRemoveVolumeBeingWrittenForDatanode(int dataNodeIdx) throws IOException, ReconfigurationException, TimeoutException, InterruptedException, BrokenBarrierException {
    // Starts DFS cluster with 3 DataNodes to form a pipeline.
    startDFSCluster(1, 3);
    final short REPLICATION = 3;
    final DataNode dn = cluster.getDataNodes().get(dataNodeIdx);
    final FileSystem fs = cluster.getFileSystem();
    final Path testFile = new Path("/test");
    FSDataOutputStream out = fs.create(testFile, REPLICATION);
    Random rb = new Random(0);
    // half of the block.
    byte[] writeBuf = new byte[BLOCK_SIZE / 2];
    rb.nextBytes(writeBuf);
    out.write(writeBuf);
    out.hflush();
    // Make FsDatasetSpi#finalizeBlock a time-consuming operation. So if the
    // BlockReceiver releases volume reference before finalizeBlock(), the blocks
    // on the volume will be removed, and finalizeBlock() throws IOE.
    final FsDatasetSpi<? extends FsVolumeSpi> data = dn.data;
    dn.data = Mockito.spy(data);
    doAnswer(new Answer<Object>() {

        public Object answer(InvocationOnMock invocation) throws IOException, InterruptedException {
            Thread.sleep(1000);
            // Bypass the argument to FsDatasetImpl#finalizeBlock to verify that
            // the block is not removed, since the volume reference should not
            // be released at this point.
            data.finalizeBlock((ExtendedBlock) invocation.getArguments()[0]);
            return null;
        }
    }).when(dn.data).finalizeBlock(any(ExtendedBlock.class));
    final CyclicBarrier barrier = new CyclicBarrier(2);
    List<String> oldDirs = getDataDirs(dn);
    // Remove the first volume.
    final String newDirs = oldDirs.get(1);
    final List<Exception> exceptions = new ArrayList<>();
    Thread reconfigThread = new Thread() {

        public void run() {
            try {
                barrier.await();
                assertThat("DN did not update its own config", dn.reconfigurePropertyImpl(DFS_DATANODE_DATA_DIR_KEY, newDirs), is(dn.getConf().get(DFS_DATANODE_DATA_DIR_KEY)));
            } catch (ReconfigurationException | InterruptedException | BrokenBarrierException e) {
                exceptions.add(e);
            }
        }
    };
    reconfigThread.start();
    barrier.await();
    rb.nextBytes(writeBuf);
    out.write(writeBuf);
    out.hflush();
    out.close();
    reconfigThread.join();
    // Verify if the data directory reconfigure was successful
    FsDatasetSpi<? extends FsVolumeSpi> fsDatasetSpi = dn.getFSDataset();
    try (FsDatasetSpi.FsVolumeReferences fsVolumeReferences = fsDatasetSpi.getFsVolumeReferences()) {
        for (int i = 0; i < fsVolumeReferences.size(); i++) {
            System.out.println("Vol: " + fsVolumeReferences.get(i).getBaseURI().toString());
        }
        assertEquals("Volume remove wasn't successful.", 1, fsVolumeReferences.size());
    }
    // Verify the file has sufficient replications.
    DFSTestUtil.waitReplication(fs, testFile, REPLICATION);
    // Read the content back
    byte[] content = DFSTestUtil.readFileBuffer(fs, testFile);
    assertEquals(BLOCK_SIZE, content.length);
    if (!exceptions.isEmpty()) {
        throw new IOException(exceptions.get(0).getCause());
    }
}

Example 2 with BrokenBarrierException

use of java.util.concurrent.BrokenBarrierException in project hadoop by apache.

the class TestResourceLocalizationService method testFailedPublicResource.

@Test(timeout = 20000)
// mocked generics
@SuppressWarnings("unchecked")
public void testFailedPublicResource() throws Exception {
    List<Path> localDirs = new ArrayList<Path>();
    String[] sDirs = new String[4];
    for (int i = 0; i < 4; ++i) {
        localDirs.add(lfs.makeQualified(new Path(basedir, i + "")));
        sDirs[i] = localDirs.get(i).toString();
    }
    conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);
    DrainDispatcher dispatcher = new DrainDispatcher();
    EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
    dispatcher.register(ApplicationEventType.class, applicationBus);
    EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
    dispatcher.register(ContainerEventType.class, containerBus);
    ContainerExecutor exec = mock(ContainerExecutor.class);
    DeletionService delService = mock(DeletionService.class);
    LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
    dirsHandler.init(conf);
    dispatcher.init(conf);
    dispatcher.start();
    try {
        ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, dirsHandler, nmContext);
        ResourceLocalizationService spyService = spy(rawService);
        doReturn(mockServer).when(spyService).createServer();
        doReturn(lfs).when(spyService).getLocalFileContext(isA(Configuration.class));
        spyService.init(conf);
        spyService.start();
        final String user = "user0";
        // init application
        final Application app = mock(Application.class);
        final ApplicationId appId = BuilderUtils.newApplicationId(314159265358979L, 3);
        when(app.getUser()).thenReturn(user);
        when(app.getAppId()).thenReturn(appId);
        spyService.handle(new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
        dispatcher.await();
        // init container.
        final Container c = getMockContainer(appId, 42, user);
        // init resources
        Random r = new Random();
        long seed = r.nextLong();
        System.out.println("SEED: " + seed);
        r.setSeed(seed);
        // cause chmod to fail after a delay
        final CyclicBarrier barrier = new CyclicBarrier(2);
        doAnswer(new Answer<Void>() {

            public Void answer(InvocationOnMock invocation) throws IOException {
                try {
                    barrier.await();
                } catch (InterruptedException e) {
                } catch (BrokenBarrierException e) {
                }
                throw new IOException("forced failure");
            }
        }).when(spylfs).setPermission(isA(Path.class), isA(FsPermission.class));
        // Queue up two localization requests for the same public resource
        final LocalResource pubResource = getPublicMockedResource(r);
        final LocalResourceRequest pubReq = new LocalResourceRequest(pubResource);
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        req.put(LocalResourceVisibility.PUBLIC, Collections.singletonList(pubReq));
        Set<LocalResourceRequest> pubRsrcs = new HashSet<LocalResourceRequest>();
        pubRsrcs.add(pubReq);
        spyService.handle(new ContainerLocalizationRequestEvent(c, req));
        spyService.handle(new ContainerLocalizationRequestEvent(c, req));
        dispatcher.await();
        // allow the chmod to fail now that both requests have been queued
        barrier.await();
        verify(containerBus, timeout(5000).times(2)).handle(isA(ContainerResourceFailedEvent.class));
    } finally {
        dispatcher.stop();
    }
}

Example 3 with BrokenBarrierException

use of java.util.concurrent.BrokenBarrierException in project hadoop by apache.

the class TestNodeManagerResync method testBlockNewContainerRequestsOnStartAndResync.

// This test tests new container requests are blocked when NM starts from
// scratch until it register with RM AND while NM is resyncing with RM
@SuppressWarnings("unchecked")
@Test(timeout = 60000)
public void testBlockNewContainerRequestsOnStartAndResync() throws IOException, InterruptedException, YarnException {
    NodeManager nm = new TestNodeManager2();
    int port = ServerSocketUtil.getPort(49154, 10);
    YarnConfiguration conf = createNMConfig(port);
    conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, false);
    nm.init(conf);
    nm.start();
    // Start the container in running state
    ContainerId cId = TestNodeManagerShutdown.createContainerId();
    TestNodeManagerShutdown.startContainer(nm, cId, localFS, tmpDir, processStartFile, port);
    nm.getNMDispatcher().getEventHandler().handle(new NodeManagerEvent(NodeManagerEventType.RESYNC));
    try {
        syncBarrier.await();
    } catch (BrokenBarrierException e) {
    }
    Assert.assertFalse(assertionFailedInThread.get());
    nm.stop();
}

Example 4 with BrokenBarrierException

use of java.util.concurrent.BrokenBarrierException in project hadoop by apache.

the class TestNodeManagerResync method testNMSentContainerStatusOnResync.

// This is to test when NM gets the resync response from last heart beat, it
// should be able to send the already-sent-via-last-heart-beat container
// statuses again when it re-register with RM.
@Test
public void testNMSentContainerStatusOnResync() throws Exception {
    final ContainerStatus testCompleteContainer = TestNodeStatusUpdater.createContainerStatus(2, ContainerState.COMPLETE);
    final Container container = TestNodeStatusUpdater.getMockContainer(testCompleteContainer);
    NMContainerStatus report = createNMContainerStatus(2, ContainerState.COMPLETE);
    when(container.getNMContainerStatus()).thenReturn(report);
    NodeManager nm = new NodeManager() {

        int registerCount = 0;

        @Override
        protected NodeStatusUpdater createNodeStatusUpdater(Context context, Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
            return new TestNodeStatusUpdaterResync(context, dispatcher, healthChecker, metrics) {

                @Override
                protected ResourceTracker createResourceTracker() {
                    return new MockResourceTracker() {

                        @Override
                        public RegisterNodeManagerResponse registerNodeManager(RegisterNodeManagerRequest request) throws YarnException, IOException {
                            if (registerCount == 0) {
                                // first register, no containers info.
                                try {
                                    Assert.assertEquals(0, request.getNMContainerStatuses().size());
                                } catch (AssertionError error) {
                                    error.printStackTrace();
                                    assertionFailedInThread.set(true);
                                }
                                // put the completed container into the context
                                getNMContext().getContainers().put(testCompleteContainer.getContainerId(), container);
                                getNMContext().getApplications().put(testCompleteContainer.getContainerId().getApplicationAttemptId().getApplicationId(), mock(Application.class));
                            } else {
                                // second register contains the completed container info.
                                List<NMContainerStatus> statuses = request.getNMContainerStatuses();
                                try {
                                    Assert.assertEquals(1, statuses.size());
                                    Assert.assertEquals(testCompleteContainer.getContainerId(), statuses.get(0).getContainerId());
                                } catch (AssertionError error) {
                                    error.printStackTrace();
                                    assertionFailedInThread.set(true);
                                }
                            }
                            registerCount++;
                            return super.registerNodeManager(request);
                        }

                        @Override
                        public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request) {
                            // first heartBeat contains the completed container info
                            List<ContainerStatus> statuses = request.getNodeStatus().getContainersStatuses();
                            try {
                                Assert.assertEquals(1, statuses.size());
                                Assert.assertEquals(testCompleteContainer.getContainerId(), statuses.get(0).getContainerId());
                            } catch (AssertionError error) {
                                error.printStackTrace();
                                assertionFailedInThread.set(true);
                            }
                            // notify RESYNC on first heartbeat.
                            return YarnServerBuilderUtils.newNodeHeartbeatResponse(1, NodeAction.RESYNC, null, null, null, null, 1000L);
                        }
                    };
                }
            };
        }
    };
    YarnConfiguration conf = createNMConfig();
    nm.init(conf);
    nm.start();
    try {
        syncBarrier.await();
    } catch (BrokenBarrierException e) {
    }
    Assert.assertFalse(assertionFailedInThread.get());
    nm.stop();
}

Example 5 with BrokenBarrierException

use of java.util.concurrent.BrokenBarrierException in project hive by apache.

the class TestSessionManagerMetrics method testActiveSessionMetrics.

@Test
public void testActiveSessionMetrics() throws Exception {
    final CyclicBarrier ready = new CyclicBarrier(2);
    CyclicBarrier completed = new CyclicBarrier(2);
    String json = metrics.dumpJson();
    MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.GAUGE, MetricsConstant.HS2_ACTIVE_SESSIONS, 0);
    SessionHandle handle = sm.openSession(TProtocolVersion.HIVE_CLI_SERVICE_PROTOCOL_V9, "user", "passw", "127.0.0.1", new HashMap<String, String>());
    final HiveSession session = sm.getSession(handle);
    OperationManager operationManager = mock(OperationManager.class);
    when(operationManager.newGetTablesOperation(session, "catalog", "schema", "table", null)).thenReturn(new BlockingOperation(session, OperationType.GET_TABLES, ready, completed));
    session.setOperationManager(operationManager);
    new Thread(new Runnable() {

        @Override
        public void run() {
            try {
                OperationHandle handle = session.getTables("catalog", "schema", "table", null);
                session.closeOperation(handle);
            } catch (Exception e) {
                throw new RuntimeException(e);
            } finally {
                try {
                    ready.await();
                } catch (InterruptedException | BrokenBarrierException e) {
                // ignore
                }
            }
        }
    }).start();
    ready.await(2, TimeUnit.SECONDS);
    ready.reset();
    json = metrics.dumpJson();
    MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.GAUGE, MetricsConstant.HS2_ACTIVE_SESSIONS, 1);
    completed.await(2, TimeUnit.SECONDS);
    ready.await(2, TimeUnit.SECONDS);
    json = metrics.dumpJson();
    MetricsTestUtils.verifyMetricsJson(json, MetricsTestUtils.GAUGE, MetricsConstant.HS2_ACTIVE_SESSIONS, 0);
}