Examples with StormMetricsRegistry org.apache.storm.metric.StormMetricsRegistry used on opensource projects

Example 61 with StormMetricsRegistry

use of org.apache.storm.metric.StormMetricsRegistry in project storm by apache.

the class SlotTest method testReschedule.

@Test
public void testReschedule() throws Exception {
    try (SimulatedTime t = new SimulatedTime(1010)) {
        int port = 8080;
        String cTopoId = "CURRENT";
        List<ExecutorInfo> cExecList = mkExecutorInfoList(1, 2, 3, 4, 5);
        LocalAssignment cAssignment = mkLocalAssignment(cTopoId, cExecList, mkWorkerResources(100.0, 100.0, 100.0));
        BlobChangingCallback cb = mock(BlobChangingCallback.class);
        Container cContainer = mock(Container.class);
        LSWorkerHeartbeat chb = mkWorkerHB(cTopoId, port, cExecList, Time.currentTimeSecs());
        when(cContainer.readHeartbeat()).thenReturn(chb);
        when(cContainer.areAllProcessesDead()).thenReturn(false, false, true);
        String nTopoId = "NEW";
        List<ExecutorInfo> nExecList = mkExecutorInfoList(1, 2, 3, 4, 5);
        LocalAssignment nAssignment = mkLocalAssignment(nTopoId, nExecList, mkWorkerResources(100.0, 100.0, 100.0));
        AsyncLocalizer localizer = mock(AsyncLocalizer.class);
        Container nContainer = mock(Container.class);
        LocalState state = mock(LocalState.class);
        ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
        when(containerLauncher.launchContainer(port, nAssignment, state)).thenReturn(nContainer);
        LSWorkerHeartbeat nhb = mkWorkerHB(nTopoId, 100, nExecList, Time.currentTimeSecs());
        when(nContainer.readHeartbeat()).thenReturn(nhb, nhb);
        @SuppressWarnings("unchecked") CompletableFuture<Void> blobFuture = mock(CompletableFuture.class);
        when(localizer.requestDownloadTopologyBlobs(nAssignment, port, cb)).thenReturn(blobFuture);
        ISupervisor iSuper = mock(ISupervisor.class);
        SlotMetrics slotMetrics = new SlotMetrics(new StormMetricsRegistry());
        StaticState staticState = new StaticState(localizer, 5000, 120000, 1000, 1000, containerLauncher, "localhost", port, iSuper, state, cb, null, null, slotMetrics);
        DynamicState dynamicState = new DynamicState(cAssignment, cContainer, nAssignment, slotMetrics);
        DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState);
        assertEquals(MachineState.KILL, nextState.state);
        verify(cContainer).kill();
        verify(localizer).requestDownloadTopologyBlobs(nAssignment, port, cb);
        assertSame("pendingDownload not set properly", blobFuture, nextState.pendingDownload);
        assertEquals(nAssignment, nextState.pendingLocalization);
        assertTrue(Time.currentTimeMillis() > 1000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.KILL, nextState.state);
        verify(cContainer).forceKill();
        assertSame("pendingDownload not set properly", blobFuture, nextState.pendingDownload);
        assertEquals(nAssignment, nextState.pendingLocalization);
        assertTrue(Time.currentTimeMillis() > 2000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.WAITING_FOR_BLOB_LOCALIZATION, nextState.state);
        verify(cContainer).cleanUp();
        verify(localizer).releaseSlotFor(cAssignment, port);
        assertTrue(Time.currentTimeMillis() > 2000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        verify(blobFuture).get(1000, TimeUnit.MILLISECONDS);
        verify(containerLauncher).launchContainer(port, nAssignment, state);
        assertEquals(MachineState.WAITING_FOR_WORKER_START, nextState.state);
        assertSame("pendingDownload is not null", null, nextState.pendingDownload);
        assertSame(null, nextState.pendingLocalization);
        assertSame(nAssignment, nextState.currentAssignment);
        assertSame(nContainer, nextState.container);
        assertTrue(Time.currentTimeMillis() > 2000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.RUNNING, nextState.state);
        assertSame("pendingDownload is not null", null, nextState.pendingDownload);
        assertSame(null, nextState.pendingLocalization);
        assertSame(nAssignment, nextState.currentAssignment);
        assertSame(nContainer, nextState.container);
        assertTrue(Time.currentTimeMillis() > 2000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.RUNNING, nextState.state);
        assertSame("pendingDownload is not null", null, nextState.pendingDownload);
        assertSame(null, nextState.pendingLocalization);
        assertSame(nAssignment, nextState.currentAssignment);
        assertSame(nContainer, nextState.container);
        assertTrue(Time.currentTimeMillis() > 3000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.RUNNING, nextState.state);
        assertSame("pendingDownload is not null", null, nextState.pendingDownload);
        assertSame(null, nextState.pendingLocalization);
        assertSame(nAssignment, nextState.currentAssignment);
        assertSame(nContainer, nextState.container);
        assertTrue(Time.currentTimeMillis() > 4000);
    }
}

Example 62 with StormMetricsRegistry

use of org.apache.storm.metric.StormMetricsRegistry in project storm by apache.

the class SlotTest method testRunningToEmpty.

@Test
public void testRunningToEmpty() throws Exception {
    try (SimulatedTime t = new SimulatedTime(1010)) {
        int port = 8080;
        String cTopoId = "CURRENT";
        List<ExecutorInfo> cExecList = mkExecutorInfoList(1, 2, 3, 4, 5);
        LocalAssignment cAssignment = mkLocalAssignment(cTopoId, cExecList, mkWorkerResources(100.0, 100.0, 100.0));
        Container cContainer = mock(Container.class);
        LSWorkerHeartbeat chb = mkWorkerHB(cTopoId, port, cExecList, Time.currentTimeSecs());
        when(cContainer.readHeartbeat()).thenReturn(chb);
        when(cContainer.areAllProcessesDead()).thenReturn(false, false, true);
        AsyncLocalizer localizer = mock(AsyncLocalizer.class);
        BlobChangingCallback cb = mock(BlobChangingCallback.class);
        ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
        ISupervisor iSuper = mock(ISupervisor.class);
        LocalState state = mock(LocalState.class);
        SlotMetrics slotMetrics = new SlotMetrics(new StormMetricsRegistry());
        StaticState staticState = new StaticState(localizer, 5000, 120000, 1000, 1000, containerLauncher, "localhost", port, iSuper, state, cb, null, null, slotMetrics);
        DynamicState dynamicState = new DynamicState(cAssignment, cContainer, null, slotMetrics);
        DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState);
        assertEquals(MachineState.KILL, nextState.state);
        verify(cContainer).kill();
        verify(localizer, never()).requestDownloadTopologyBlobs(null, port, cb);
        assertSame("pendingDownload not set properly", null, nextState.pendingDownload);
        assertEquals(null, nextState.pendingLocalization);
        assertTrue(Time.currentTimeMillis() > 1000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.KILL, nextState.state);
        verify(cContainer).forceKill();
        assertSame("pendingDownload not set properly", null, nextState.pendingDownload);
        assertEquals(null, nextState.pendingLocalization);
        assertTrue(Time.currentTimeMillis() > 2000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.EMPTY, nextState.state);
        verify(cContainer).cleanUp();
        verify(localizer).releaseSlotFor(cAssignment, port);
        assertEquals(null, nextState.container);
        assertEquals(null, nextState.currentAssignment);
        assertTrue(Time.currentTimeMillis() > 2000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.EMPTY, nextState.state);
        assertEquals(null, nextState.container);
        assertEquals(null, nextState.currentAssignment);
        assertTrue(Time.currentTimeMillis() > 3000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.EMPTY, nextState.state);
        assertEquals(null, nextState.container);
        assertEquals(null, nextState.currentAssignment);
        assertTrue(Time.currentTimeMillis() > 3000);
    }
}

Example 63 with StormMetricsRegistry

use of org.apache.storm.metric.StormMetricsRegistry in project storm by apache.

the class SlotTest method testRunWithProfileActions.

@Test
public void testRunWithProfileActions() throws Exception {
    try (SimulatedTime t = new SimulatedTime(1010)) {
        int port = 8080;
        String cTopoId = "CURRENT";
        List<ExecutorInfo> cExecList = mkExecutorInfoList(1, 2, 3, 4, 5);
        LocalAssignment cAssignment = mkLocalAssignment(cTopoId, cExecList, mkWorkerResources(100.0, 100.0, 100.0));
        Container cContainer = mock(Container.class);
        // NOT going to timeout for a while
        LSWorkerHeartbeat chb = mkWorkerHB(cTopoId, port, cExecList, Time.currentTimeSecs() + 100);
        when(cContainer.readHeartbeat()).thenReturn(chb, chb, chb, chb, chb, chb);
        when(cContainer.runProfiling(any(ProfileRequest.class), anyBoolean())).thenReturn(true);
        AsyncLocalizer localizer = mock(AsyncLocalizer.class);
        BlobChangingCallback cb = mock(BlobChangingCallback.class);
        ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
        ISupervisor iSuper = mock(ISupervisor.class);
        LocalState state = mock(LocalState.class);
        StaticState staticState = new StaticState(localizer, 5000, 120000, 1000, 1000, containerLauncher, "localhost", port, iSuper, state, cb, null, null, new SlotMetrics(new StormMetricsRegistry()));
        Set<TopoProfileAction> profileActions = new HashSet<>();
        ProfileRequest request = new ProfileRequest();
        request.set_action(ProfileAction.JPROFILE_STOP);
        NodeInfo info = new NodeInfo();
        info.set_node("localhost");
        info.add_to_port(port);
        request.set_nodeInfo(info);
        // 3 seconds from now
        request.set_time_stamp(Time.currentTimeMillis() + 3000);
        TopoProfileAction profile = new TopoProfileAction(cTopoId, request);
        profileActions.add(profile);
        Set<TopoProfileAction> expectedPending = new HashSet<>();
        expectedPending.add(profile);
        SlotMetrics slotMetrics = new SlotMetrics(new StormMetricsRegistry());
        DynamicState dynamicState = new DynamicState(cAssignment, cContainer, cAssignment, slotMetrics).withProfileActions(profileActions, Collections.<TopoProfileAction>emptySet());
        DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState);
        assertEquals(MachineState.RUNNING, nextState.state);
        verify(cContainer).runProfiling(request, false);
        assertEquals(expectedPending, nextState.pendingStopProfileActions);
        assertEquals(expectedPending, nextState.profileActions);
        assertTrue(Time.currentTimeMillis() > 1000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.RUNNING, nextState.state);
        assertEquals(expectedPending, nextState.pendingStopProfileActions);
        assertEquals(expectedPending, nextState.profileActions);
        assertTrue(Time.currentTimeMillis() > 2000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.RUNNING, nextState.state);
        assertEquals(expectedPending, nextState.pendingStopProfileActions);
        assertEquals(expectedPending, nextState.profileActions);
        assertTrue(Time.currentTimeMillis() > 3000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.RUNNING, nextState.state);
        verify(cContainer).runProfiling(request, true);
        assertEquals(Collections.<TopoProfileAction>emptySet(), nextState.pendingStopProfileActions);
        assertEquals(Collections.<TopoProfileAction>emptySet(), nextState.profileActions);
        assertTrue(Time.currentTimeMillis() > 4000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.RUNNING, nextState.state);
        assertEquals(Collections.<TopoProfileAction>emptySet(), nextState.pendingStopProfileActions);
        assertEquals(Collections.<TopoProfileAction>emptySet(), nextState.profileActions);
        assertTrue(Time.currentTimeMillis() > 5000);
    }
}

Example 64 with StormMetricsRegistry

use of org.apache.storm.metric.StormMetricsRegistry in project storm by apache.

the class LocalizedResourceRetentionSetTest method testCleanup.

@Test
public void testCleanup() throws Exception {
    ClientBlobStore mockBlobstore = mock(ClientBlobStore.class);
    when(mockBlobstore.getBlobMeta(any())).thenReturn(new ReadableBlobMeta(new SettableBlobMeta(), 1));
    PortAndAssignment pna1 = new PortAndAssignmentImpl(1, new LocalAssignment("topo1", Collections.emptyList()));
    String user = "user";
    Map<String, Object> conf = new HashMap<>();
    IAdvancedFSOps ops = mock(IAdvancedFSOps.class);
    LocalizedResourceRetentionSet lrretset = spy(new LocalizedResourceRetentionSet(10));
    ConcurrentMap<String, LocalizedResource> lrFiles = new ConcurrentHashMap<>();
    ConcurrentMap<String, LocalizedResource> lrArchives = new ConcurrentHashMap<>();
    StormMetricsRegistry metricsRegistry = new StormMetricsRegistry();
    // no reference to key1
    LocalizedResource localresource1 = new LocalizedResource("key1", Paths.get("./target/TESTING/testfile1"), false, ops, conf, user, metricsRegistry);
    localresource1.setSize(10);
    // no reference to archive1
    LocalizedResource archiveresource1 = new LocalizedResource("archive1", Paths.get("./target/TESTING/testarchive1"), true, ops, conf, user, metricsRegistry);
    archiveresource1.setSize(20);
    // reference to key2
    LocalizedResource localresource2 = new LocalizedResource("key2", Paths.get("./target/TESTING/testfile2"), false, ops, conf, user, metricsRegistry);
    localresource2.addReference(pna1, null);
    // check adding reference to local resource with topology of same name
    localresource2.addReference(pna1, null);
    localresource2.setSize(10);
    lrFiles.put("key1", localresource1);
    lrFiles.put("key2", localresource2);
    lrArchives.put("archive1", archiveresource1);
    lrretset.addResources(lrFiles);
    lrretset.addResources(lrArchives);
    assertEquals("number to clean is not 2", 2, lrretset.getSizeWithNoReferences());
    lrretset.cleanup(mockBlobstore);
    assertEquals("resource not cleaned up", 0, lrretset.getSizeWithNoReferences());
}

Example 65 with StormMetricsRegistry

use of org.apache.storm.metric.StormMetricsRegistry in project storm by apache.

the class LocalizedResourceRetentionSetTest method testAddResources.

@Test
public void testAddResources() throws Exception {
    PortAndAssignment pna1 = new PortAndAssignmentImpl(1, new LocalAssignment("topo1", Collections.emptyList()));
    PortAndAssignment pna2 = new PortAndAssignmentImpl(1, new LocalAssignment("topo2", Collections.emptyList()));
    String user = "user";
    Map<String, Object> conf = new HashMap<>();
    IAdvancedFSOps ops = mock(IAdvancedFSOps.class);
    LocalizedResourceRetentionSet lrretset = new LocalizedResourceRetentionSet(10);
    ConcurrentMap<String, LocalizedResource> lrset = new ConcurrentHashMap<>();
    StormMetricsRegistry metricsRegistry = new StormMetricsRegistry();
    LocalizedResource localresource1 = new LocalizedResource("key1", Paths.get("testfile1"), false, ops, conf, user, metricsRegistry);
    localresource1.addReference(pna1, null);
    LocalizedResource localresource2 = new LocalizedResource("key2", Paths.get("testfile2"), false, ops, conf, user, metricsRegistry);
    localresource2.addReference(pna1, null);
    // check adding reference to local resource with topology of same name
    localresource2.addReference(pna2, null);
    lrset.put("key1", localresource1);
    lrset.put("key2", localresource2);
    lrretset.addResources(lrset);
    assertEquals("number to clean is not 0 " + lrretset.noReferences, 0, lrretset.getSizeWithNoReferences());
    assertTrue(localresource1.removeReference(pna1));
    lrretset = new LocalizedResourceRetentionSet(10);
    lrretset.addResources(lrset);
    assertEquals("number to clean is not 1 " + lrretset.noReferences, 1, lrretset.getSizeWithNoReferences());
    assertTrue(localresource2.removeReference(pna1));
    lrretset = new LocalizedResourceRetentionSet(10);
    lrretset.addResources(lrset);
    assertEquals("number to clean is not 1  " + lrretset.noReferences, 1, lrretset.getSizeWithNoReferences());
    assertTrue(localresource2.removeReference(pna2));
    lrretset = new LocalizedResourceRetentionSet(10);
    lrretset.addResources(lrset);
    assertEquals("number to clean is not 2 " + lrretset.noReferences, 2, lrretset.getSizeWithNoReferences());
}