Examples with StormMetricsRegistry org.apache.storm.metric.StormMetricsRegistry used on opensource projects

Example 56 with StormMetricsRegistry

use of org.apache.storm.metric.StormMetricsRegistry in project storm by apache.

the class ContainerTest method testCleanup.

@Test
public void testCleanup() throws Exception {
    final int supervisorPort = 6628;
    final int port = 8080;
    final String topoId = "test_topology";
    final String workerId = "worker_id";
    final String user = "me";
    final String stormLocal = asAbsPath("tmp", "testing");
    final File workerArtifacts = asAbsFile(stormLocal, topoId, String.valueOf(port));
    final File logMetadataFile = new File(workerArtifacts, "worker.yaml");
    final File workerUserFile = asAbsFile(stormLocal, "workers-users", workerId);
    final File workerRoot = asAbsFile(stormLocal, "workers", workerId);
    final Map<String, Object> topoConf = new HashMap<>();
    final Map<String, Object> superConf = new HashMap<>();
    superConf.put(Config.STORM_LOCAL_DIR, stormLocal);
    superConf.put(Config.STORM_WORKERS_ARTIFACTS_DIR, stormLocal);
    final StringWriter yamlDump = new StringWriter();
    AdvancedFSOps ops = mock(AdvancedFSOps.class);
    when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
    when(ops.fileExists(workerArtifacts)).thenReturn(true);
    when(ops.fileExists(workerRoot)).thenReturn(true);
    when(ops.getWriter(logMetadataFile)).thenReturn(yamlDump);
    ResourceIsolationInterface iso = mock(ResourceIsolationInterface.class);
    when(iso.isResourceManaged()).thenReturn(true);
    LocalAssignment la = new LocalAssignment();
    la.set_owner(user);
    la.set_topology_id(topoId);
    MockContainer mc = new MockContainer(ContainerType.LAUNCH, superConf, "SUPERVISOR", supervisorPort, port, la, iso, workerId, topoConf, ops, new StormMetricsRegistry());
    mc.cleanUp();
    verify(iso).cleanup(user, workerId, port);
    verify(ops).deleteIfExists(eq(new File(workerRoot, "pids")), eq(user), any(String.class));
    verify(ops).deleteIfExists(eq(new File(workerRoot, "tmp")), eq(user), any(String.class));
    verify(ops).deleteIfExists(eq(new File(workerRoot, "heartbeats")), eq(user), any(String.class));
    verify(ops).deleteIfExists(eq(workerRoot), eq(user), any(String.class));
    verify(ops).deleteIfExists(workerUserFile);
}

Example 57 with StormMetricsRegistry

use of org.apache.storm.metric.StormMetricsRegistry in project storm by apache.

the class ContainerTest method testKill.

@Test
public void testKill() throws Exception {
    final String topoId = "test_topology";
    final Map<String, Object> superConf = new HashMap<>();
    AdvancedFSOps ops = mock(AdvancedFSOps.class);
    when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
    LocalAssignment la = new LocalAssignment();
    la.set_topology_id(topoId);
    MockResourceIsolationManager iso = new MockResourceIsolationManager();
    String workerId = "worker-id";
    MockContainer mc = new MockContainer(ContainerType.LAUNCH, superConf, "SUPERVISOR", 6628, 8080, la, iso, workerId, new HashMap<>(), ops, new StormMetricsRegistry());
    iso.allWorkerIds.add(workerId);
    assertEquals(Collections.EMPTY_LIST, iso.killedWorkerIds);
    assertEquals(Collections.EMPTY_LIST, iso.forceKilledWorkerIds);
    mc.kill();
    assertEquals(iso.allWorkerIds, iso.killedWorkerIds);
    assertEquals(Collections.EMPTY_LIST, iso.forceKilledWorkerIds);
    iso.killedWorkerIds.clear();
    mc.forceKill();
    assertEquals(Collections.EMPTY_LIST, iso.killedWorkerIds);
    assertEquals(iso.allWorkerIds, iso.forceKilledWorkerIds);
}

Example 58 with StormMetricsRegistry

use of org.apache.storm.metric.StormMetricsRegistry in project storm by apache.

the class ContainerTest method testSetup.

@SuppressWarnings("unchecked")
@Test
public void testSetup() throws Exception {
    final int port = 8080;
    final String topoId = "test_topology";
    final String workerId = "worker_id";
    final String user = "me";
    final String stormLocal = asAbsPath("tmp", "testing");
    final File workerArtifacts = asAbsFile(stormLocal, topoId, String.valueOf(port));
    final File logMetadataFile = new File(workerArtifacts, "worker.yaml");
    final File workerUserFile = asAbsFile(stormLocal, "workers-users", workerId);
    final File workerRoot = asAbsFile(stormLocal, "workers", workerId);
    final File distRoot = asAbsFile(stormLocal, "supervisor", "stormdist", topoId);
    final Map<String, Object> topoConf = new HashMap<>();
    final List<String> topoUsers = Arrays.asList("t-user-a", "t-user-b");
    final List<String> logUsers = Arrays.asList("l-user-a", "l-user-b");
    final List<String> topoGroups = Arrays.asList("t-group-a", "t-group-b");
    final List<String> logGroups = Arrays.asList("l-group-a", "l-group-b");
    topoConf.put(DaemonConfig.LOGS_GROUPS, logGroups);
    topoConf.put(Config.TOPOLOGY_GROUPS, topoGroups);
    topoConf.put(DaemonConfig.LOGS_USERS, logUsers);
    topoConf.put(Config.TOPOLOGY_USERS, topoUsers);
    final Map<String, Object> superConf = new HashMap<>();
    superConf.put(Config.STORM_LOCAL_DIR, stormLocal);
    superConf.put(Config.STORM_WORKERS_ARTIFACTS_DIR, stormLocal);
    final StringWriter yamlDump = new StringWriter();
    AdvancedFSOps ops = mock(AdvancedFSOps.class);
    when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
    when(ops.fileExists(workerArtifacts)).thenReturn(true);
    when(ops.fileExists(workerRoot)).thenReturn(true);
    when(ops.getWriter(logMetadataFile)).thenReturn(yamlDump);
    LocalAssignment la = new LocalAssignment();
    la.set_topology_id(topoId);
    la.set_owner(user);
    ResourceIsolationInterface iso = mock(ResourceIsolationInterface.class);
    MockContainer mc = new MockContainer(ContainerType.LAUNCH, superConf, "SUPERVISOR", 6628, 8080, la, iso, workerId, topoConf, ops, new StormMetricsRegistry());
    mc.setup();
    // Initial Setup
    verify(ops).forceMkdir(new File(workerRoot, "pids"));
    verify(ops).forceMkdir(new File(workerRoot, "tmp"));
    verify(ops).forceMkdir(new File(workerRoot, "heartbeats"));
    verify(ops).fileExists(workerArtifacts);
    // Log file permissions
    verify(ops).getWriter(logMetadataFile);
    String yamlResult = yamlDump.toString();
    Yaml yaml = new Yaml();
    Map<String, Object> result = yaml.load(yamlResult);
    assertEquals(workerId, result.get("worker-id"));
    assertEquals(user, result.get(Config.TOPOLOGY_SUBMITTER_USER));
    HashSet<String> allowedUsers = new HashSet<>(topoUsers);
    allowedUsers.addAll(logUsers);
    assertEquals(allowedUsers, new HashSet<>(ObjectReader.getStrings(result.get(DaemonConfig.LOGS_USERS))));
    HashSet<String> allowedGroups = new HashSet<>(topoGroups);
    allowedGroups.addAll(logGroups);
    assertEquals(allowedGroups, new HashSet<>(ObjectReader.getStrings(result.get(DaemonConfig.LOGS_GROUPS))));
    // Save the current user to help with recovery
    verify(ops).dump(workerUserFile, user);
    // Create links to artifacts dir
    verify(ops).createSymlink(new File(workerRoot, "artifacts"), workerArtifacts);
    // Create links to blobs
    verify(ops, never()).createSymlink(new File(workerRoot, "resources"), new File(distRoot, "resources"));
}

Example 59 with StormMetricsRegistry

use of org.apache.storm.metric.StormMetricsRegistry in project storm by apache.

the class SlotTest method testEmptyToEmpty.

@Test
public void testEmptyToEmpty() throws Exception {
    try (SimulatedTime t = new SimulatedTime(1010)) {
        AsyncLocalizer localizer = mock(AsyncLocalizer.class);
        LocalState state = mock(LocalState.class);
        BlobChangingCallback cb = mock(BlobChangingCallback.class);
        ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
        ISupervisor iSuper = mock(ISupervisor.class);
        SlotMetrics slotMetrics = new SlotMetrics(new StormMetricsRegistry());
        StaticState staticState = new StaticState(localizer, 1000, 1000, 1000, 1000, containerLauncher, "localhost", 8080, iSuper, state, cb, null, null, slotMetrics);
        DynamicState dynamicState = new DynamicState(null, null, null, slotMetrics);
        DynamicState nextState = Slot.handleEmpty(dynamicState, staticState);
        assertEquals(MachineState.EMPTY, nextState.state);
        assertTrue(Time.currentTimeMillis() > 1000);
    }
}

Example 60 with StormMetricsRegistry

use of org.apache.storm.metric.StormMetricsRegistry in project storm by apache.

the class SlotTest method testLaunchContainerFromEmpty.

@Test
public void testLaunchContainerFromEmpty() throws Exception {
    try (SimulatedTime t = new SimulatedTime(1010)) {
        int port = 8080;
        String topoId = "NEW";
        List<ExecutorInfo> execList = mkExecutorInfoList(1, 2, 3, 4, 5);
        LocalAssignment newAssignment = mkLocalAssignment(topoId, execList, mkWorkerResources(100.0, 100.0, 100.0));
        AsyncLocalizer localizer = mock(AsyncLocalizer.class);
        BlobChangingCallback cb = mock(BlobChangingCallback.class);
        Container container = mock(Container.class);
        LocalState state = mock(LocalState.class);
        ContainerLauncher containerLauncher = mock(ContainerLauncher.class);
        when(containerLauncher.launchContainer(port, newAssignment, state)).thenReturn(container);
        LSWorkerHeartbeat hb = mkWorkerHB(topoId, port, execList, Time.currentTimeSecs());
        when(container.readHeartbeat()).thenReturn(hb, hb);
        @SuppressWarnings("unchecked") CompletableFuture<Void> blobFuture = mock(CompletableFuture.class);
        when(localizer.requestDownloadTopologyBlobs(newAssignment, port, cb)).thenReturn(blobFuture);
        ISupervisor iSuper = mock(ISupervisor.class);
        SlotMetrics slotMetrics = new SlotMetrics(new StormMetricsRegistry());
        StaticState staticState = new StaticState(localizer, 5000, 120000, 1000, 1000, containerLauncher, "localhost", port, iSuper, state, cb, null, null, slotMetrics);
        DynamicState dynamicState = new DynamicState(null, null, null, slotMetrics).withNewAssignment(newAssignment);
        DynamicState nextState = Slot.stateMachineStep(dynamicState, staticState);
        verify(localizer).requestDownloadTopologyBlobs(newAssignment, port, cb);
        assertEquals(MachineState.WAITING_FOR_BLOB_LOCALIZATION, nextState.state);
        assertSame("pendingDownload not set properly", blobFuture, nextState.pendingDownload);
        assertEquals(newAssignment, nextState.pendingLocalization);
        assertEquals(0, Time.currentTimeMillis());
        nextState = Slot.stateMachineStep(nextState, staticState);
        verify(blobFuture).get(1000, TimeUnit.MILLISECONDS);
        verify(containerLauncher).launchContainer(port, newAssignment, state);
        assertEquals(MachineState.WAITING_FOR_WORKER_START, nextState.state);
        assertSame("pendingDownload is not null", null, nextState.pendingDownload);
        assertSame(null, nextState.pendingLocalization);
        assertSame(newAssignment, nextState.currentAssignment);
        assertSame(container, nextState.container);
        assertEquals(0, Time.currentTimeMillis());
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.RUNNING, nextState.state);
        assertSame("pendingDownload is not null", null, nextState.pendingDownload);
        assertSame(null, nextState.pendingLocalization);
        assertSame(newAssignment, nextState.currentAssignment);
        assertSame(container, nextState.container);
        assertEquals(0, Time.currentTimeMillis());
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.RUNNING, nextState.state);
        assertSame("pendingDownload is not null", null, nextState.pendingDownload);
        assertSame(null, nextState.pendingLocalization);
        assertSame(newAssignment, nextState.currentAssignment);
        assertSame(container, nextState.container);
        assertTrue(Time.currentTimeMillis() > 1000);
        nextState = Slot.stateMachineStep(nextState, staticState);
        assertEquals(MachineState.RUNNING, nextState.state);
        assertSame("pendingDownload is not null", null, nextState.pendingDownload);
        assertSame(null, nextState.pendingLocalization);
        assertSame(newAssignment, nextState.currentAssignment);
        assertSame(container, nextState.container);
        assertTrue(Time.currentTimeMillis() > 2000);
    }
}