Examples with LocalState org.apache.storm.utils.LocalState used on opensource projects

Example 21 with LocalState

use of org.apache.storm.utils.LocalState in project storm by apache.

the class SlotTest method testResourcesChangedFiltered.

@Test
public void testResourcesChangedFiltered() throws Exception {
    try (SimulatedTime t = new SimulatedTime(1010)) {
        int port = 8080;
        String cTopoId = "CURRENT";
        List<ExecutorInfo> cExecList = mkExecutorInfoList(1, 2, 3, 4, 5);
        LocalAssignment cAssignment = mkLocalAssignment(cTopoId, cExecList, mkWorkerResources(100.0, 100.0, 100.0));
        String otherTopoId = "OTHER";
        LocalAssignment otherAssignment = mkLocalAssignment(otherTopoId, cExecList, mkWorkerResources(100.0, 100.0, 100.0));
        BlobChangingCallback cb = mock(BlobChangingCallback.class);
        Container cContainer = mock(Container.class);
        LSWorkerHeartbeat chb = mkWorkerHB(cTopoId, port, cExecList, Time.currentTimeSecs());
        when(cContainer.readHeartbeat()).thenReturn(chb);
        when(cContainer.areAllProcessesDead()).thenReturn(false, false, true);
        AsyncLocalizer localizer = mock(AsyncLocalizer.class);
        Container nContainer = mock(Container.class);
        LocalState state = mock(LocalState.class);
        ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
        when(containerLauncher.launchContainer(port, cAssignment, state)).thenReturn(nContainer);
        when(nContainer.readHeartbeat()).thenReturn(chb, chb);
        ISupervisor iSuper = mock(ISupervisor.class);
        long heartbeatTimeoutMs = 5000;
        StaticState staticState = new StaticState(localizer, heartbeatTimeoutMs, 120_000, 1000, 1000, containerLauncher, "localhost", port, iSuper, state, cb, null, null, new SlotMetrics(new StormMetricsRegistry()));
        Set<Slot.BlobChanging> changing = new HashSet<>();
        LocallyCachedBlob stormJar = mock(LocallyCachedBlob.class);
        GoodToGo.GoodToGoLatch stormJarLatch = mock(GoodToGo.GoodToGoLatch.class);
        CompletableFuture<Void> stormJarLatchFuture = mock(CompletableFuture.class);
        when(stormJarLatch.countDown()).thenReturn(stormJarLatchFuture);
        changing.add(new Slot.BlobChanging(cAssignment, stormJar, stormJarLatch));
        Set<Slot.BlobChanging> desired = new HashSet<>(changing);
        LocallyCachedBlob otherJar = mock(LocallyCachedBlob.class);
        GoodToGo.GoodToGoLatch otherJarLatch = mock(GoodToGo.GoodToGoLatch.class);
        changing.add(new Slot.BlobChanging(otherAssignment, otherJar, otherJarLatch));
        SlotMetrics slotMetrics = new SlotMetrics(new StormMetricsRegistry());
        DynamicState dynamicState = new DynamicState(cAssignment, cContainer, cAssignment, slotMetrics).withChangingBlobs(changing);
        DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState);
        assertEquals(MachineState.KILL_BLOB_UPDATE, nextState.state);
        verify(iSuper).killedWorker(port);
        verify(cContainer).kill();
        verify(localizer, never()).requestDownloadTopologyBlobs(any(), anyInt(), any());
        verify(stormJarLatch, never()).countDown();
        verify(otherJarLatch, times(1)).countDown();
        assertNull(nextState.pendingDownload);
        assertNull(nextState.pendingLocalization);
        assertEquals(desired, nextState.changingBlobs);
        assertTrue(nextState.pendingChangingBlobs.isEmpty());
        assertNull(nextState.pendingChangingBlobsAssignment);
        assertThat(Time.currentTimeMillis(), greaterThan(1000L));
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.KILL_BLOB_UPDATE, nextState.state);
        verify(cContainer).forceKill();
        assertNull(nextState.pendingDownload);
        assertNull(nextState.pendingLocalization);
        assertEquals(desired, nextState.changingBlobs);
        assertTrue(nextState.pendingChangingBlobs.isEmpty());
        assertNull(nextState.pendingChangingBlobsAssignment);
        assertThat(Time.currentTimeMillis(), greaterThan(2000L));
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.WAITING_FOR_BLOB_UPDATE, nextState.state);
        verify(cContainer).cleanUp();
        assertThat(Time.currentTimeMillis(), greaterThan(2000L));
        nextState = Slot.stateMachineStep(nextState, staticState);
        verify(stormJarLatchFuture).get(anyLong(), any());
        verify(containerLauncher).launchContainer(port, cAssignment, state);
        assertEquals(MachineState.WAITING_FOR_WORKER_START, nextState.state);
        assertNull(nextState.pendingChangingBlobsAssignment);
        assertTrue(nextState.pendingChangingBlobs.isEmpty());
        assertSame(cAssignment, nextState.currentAssignment);
        assertSame(nContainer, nextState.container);
        assertThat(Time.currentTimeMillis(), greaterThan(2000L));
        assertThat(Time.currentTimeMillis(), lessThan(heartbeatTimeoutMs));
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.RUNNING, nextState.state);
        assertNull(nextState.pendingChangingBlobsAssignment);
        assertTrue(nextState.pendingChangingBlobs.isEmpty());
        assertSame(cAssignment, nextState.currentAssignment);
        assertSame(nContainer, nextState.container);
        assertTrue(Time.currentTimeMillis() > 2000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.RUNNING, nextState.state);
        assertNull(nextState.pendingChangingBlobsAssignment);
        assertTrue(nextState.pendingChangingBlobs.isEmpty());
        assertSame(cAssignment, nextState.currentAssignment);
        assertSame(nContainer, nextState.container);
        assertTrue(Time.currentTimeMillis() > 3000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.RUNNING, nextState.state);
        assertNull(nextState.pendingChangingBlobsAssignment);
        assertTrue(nextState.pendingChangingBlobs.isEmpty());
        assertSame(cAssignment, nextState.currentAssignment);
        assertSame(nContainer, nextState.container);
        assertTrue(Time.currentTimeMillis() > 4000);
    }
}

Example 22 with LocalState

use of org.apache.storm.utils.LocalState in project storm by apache.

the class SlotTest method testErrorHandlingWhenLocalizationFails.

@Test
public void testErrorHandlingWhenLocalizationFails() throws Exception {
    try (SimulatedTime t = new SimulatedTime(1010)) {
        int port = 8080;
        String topoId = "NEW";
        List<ExecutorInfo> execList = mkExecutorInfoList(1, 2, 3, 4, 5);
        LocalAssignment newAssignment = mkLocalAssignment(topoId, execList, mkWorkerResources(100.0, 100.0, 100.0));
        AsyncLocalizer localizer = mock(AsyncLocalizer.class);
        BlobChangingCallback cb = mock(BlobChangingCallback.class);
        Container container = mock(Container.class);
        LocalState state = mock(LocalState.class);
        ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
        when(containerLauncher.launchContainer(port, newAssignment, state)).thenReturn(container);
        LSWorkerHeartbeat hb = mkWorkerHB(topoId, port, execList, Time.currentTimeSecs());
        when(container.readHeartbeat()).thenReturn(hb, hb);
        @SuppressWarnings("unchecked") CompletableFuture<Void> blobFuture = mock(CompletableFuture.class);
        CompletableFuture<Void> secondBlobFuture = mock(CompletableFuture.class);
        when(secondBlobFuture.get(anyLong(), any())).thenThrow(new ExecutionException(new RuntimeException("Localization failure")));
        CompletableFuture<Void> thirdBlobFuture = mock(CompletableFuture.class);
        when(localizer.requestDownloadTopologyBlobs(newAssignment, port, cb)).thenReturn(blobFuture).thenReturn(secondBlobFuture).thenReturn(thirdBlobFuture);
        ISupervisor iSuper = mock(ISupervisor.class);
        SlotMetrics slotMetrics = new SlotMetrics(new StormMetricsRegistry());
        StaticState staticState = new StaticState(localizer, 5000, 120000, 1000, 1000, containerLauncher, "localhost", port, iSuper, state, cb, null, null, slotMetrics);
        DynamicState dynamicState = new DynamicState(null, null, null, slotMetrics).withNewAssignment(newAssignment);
        DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState);
        verify(localizer).requestDownloadTopologyBlobs(newAssignment, port, cb);
        assertEquals(MachineState.WAITING_FOR_BLOB_LOCALIZATION, nextState.state);
        assertSame("pendingDownload not set properly", blobFuture, nextState.pendingDownload);
        assertEquals(newAssignment, nextState.pendingLocalization);
        assertEquals(0, Time.currentTimeMillis());
        // Assignment has changed
        nextState = Slot.stateMachineStep(nextState.withNewAssignment(null), staticState);
        assertThat(nextState.state, is(MachineState.EMPTY));
        assertThat(nextState.pendingChangingBlobs, is(Collections.emptySet()));
        assertThat(nextState.pendingChangingBlobsAssignment, nullValue());
        assertThat(nextState.pendingLocalization, nullValue());
        assertThat(nextState.pendingDownload, nullValue());
        clearInvocations(localizer);
        nextState = Slot.stateMachineStep(dynamicState.withNewAssignment(newAssignment), staticState);
        verify(localizer).requestDownloadTopologyBlobs(newAssignment, port, cb);
        assertEquals(MachineState.WAITING_FOR_BLOB_LOCALIZATION, nextState.state);
        assertSame("pendingDownload not set properly", secondBlobFuture, nextState.pendingDownload);
        assertEquals(newAssignment, nextState.pendingLocalization);
        // Error occurs, but assignment has not changed
        clearInvocations(localizer);
        nextState = Slot.stateMachineStep(nextState, staticState);
        verify(localizer).requestDownloadTopologyBlobs(newAssignment, port, cb);
        assertEquals(MachineState.WAITING_FOR_BLOB_LOCALIZATION, nextState.state);
        assertSame("pendingDownload not set properly", thirdBlobFuture, nextState.pendingDownload);
        assertEquals(newAssignment, nextState.pendingLocalization);
        assertThat(Time.currentTimeMillis(), greaterThan(3L));
        nextState = Slot.stateMachineStep(nextState, staticState);
        verify(thirdBlobFuture).get(1000, TimeUnit.MILLISECONDS);
        verify(containerLauncher).launchContainer(port, newAssignment, state);
        assertEquals(MachineState.WAITING_FOR_WORKER_START, nextState.state);
        assertSame("pendingDownload is not null", null, nextState.pendingDownload);
        assertSame(null, nextState.pendingLocalization);
        assertSame(newAssignment, nextState.currentAssignment);
        assertSame(container, nextState.container);
    }
}

Example 23 with LocalState

use of org.apache.storm.utils.LocalState in project storm by apache.

the class SlotTest method testRelaunch.

@Test
public void testRelaunch() throws Exception {
    try (SimulatedTime t = new SimulatedTime(1010)) {
        int port = 8080;
        String topoId = "CURRENT";
        List<ExecutorInfo> execList = mkExecutorInfoList(1, 2, 3, 4, 5);
        LocalAssignment assignment = mkLocalAssignment(topoId, execList, mkWorkerResources(100.0, 100.0, 100.0));
        AsyncLocalizer localizer = mock(AsyncLocalizer.class);
        BlobChangingCallback cb = mock(BlobChangingCallback.class);
        Container container = mock(Container.class);
        ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
        LSWorkerHeartbeat oldhb = mkWorkerHB(topoId, port, execList, Time.currentTimeSecs() - 10);
        LSWorkerHeartbeat goodhb = mkWorkerHB(topoId, port, execList, Time.currentTimeSecs());
        when(container.readHeartbeat()).thenReturn(oldhb, oldhb, goodhb, goodhb);
        when(container.areAllProcessesDead()).thenReturn(false, false, true);
        ISupervisor iSuper = mock(ISupervisor.class);
        LocalState state = mock(LocalState.class);
        SlotMetrics slotMetrics = new SlotMetrics(new StormMetricsRegistry());
        StaticState staticState = new StaticState(localizer, 5000, 120000, 1000, 1000, containerLauncher, "localhost", port, iSuper, state, cb, null, null, slotMetrics);
        DynamicState dynamicState = new DynamicState(assignment, container, assignment, slotMetrics);
        DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState);
        assertEquals(MachineState.KILL_AND_RELAUNCH, nextState.state);
        verify(container).kill();
        assertTrue(Time.currentTimeMillis() > 1000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.KILL_AND_RELAUNCH, nextState.state);
        verify(container).forceKill();
        assertTrue(Time.currentTimeMillis() > 2000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.WAITING_FOR_WORKER_START, nextState.state);
        verify(container).relaunch();
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.WAITING_FOR_WORKER_START, nextState.state);
        assertTrue(Time.currentTimeMillis() > 3000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.RUNNING, nextState.state);
    }
}

Example 24 with LocalState

use of org.apache.storm.utils.LocalState in project storm by apache.

the class Worker method doHeartBeat.

public void doHeartBeat() throws IOException {
    LocalState state = ConfigUtils.workerState(workerState.conf, workerState.workerId);
    LSWorkerHeartbeat lsWorkerHeartbeat = new LSWorkerHeartbeat(Time.currentTimeSecs(), workerState.topologyId, workerState.localExecutors.stream().map(executor -> new ExecutorInfo(executor.get(0).intValue(), executor.get(1).intValue())).collect(Collectors.toList()), workerState.port);
    state.setWorkerHeartBeat(lsWorkerHeartbeat);
    // this is just in case supervisor is down so that disk doesn't fill up.
    state.cleanup(60);
    // it shouldn't take supervisor 120 seconds between listing dir and reading it
    if (!workerState.stormClusterState.isPacemakerStateStore()) {
        LOG.debug("The pacemaker is not used, send heartbeat to master.");
        heartbeatToMasterIfLocalbeatFail(lsWorkerHeartbeat);
    }
    this.heatbeatMeter.mark();
}

Example 25 with LocalState

use of org.apache.storm.utils.LocalState in project storm by apache.

the class StandaloneSupervisor method prepare.

@Override
public void prepare(Map<String, Object> topoConf, String schedulerLocalDir) {
    try {
        LocalState localState = new LocalState(schedulerLocalDir, true);
        String supervisorId = localState.getSupervisorId();
        if (supervisorId == null) {
            supervisorId = generateSupervisorId();
            localState.setSupervisorId(supervisorId);
        }
        this.conf = topoConf;
        this.supervisorId = supervisorId;
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}