Examples with StormMetricsRegistry org.apache.storm.metric.StormMetricsRegistry used on opensource projects

Example 66 with StormMetricsRegistry

use of org.apache.storm.metric.StormMetricsRegistry in project storm by apache.

the class LocalizedResourceRetentionSetTest method testRemoveEquivalent.

@Test
public void testRemoveEquivalent() {
    ExecutorInfo a = new ExecutorInfo(1, 1);
    ExecutorInfo b = new ExecutorInfo(2, 2);
    ExecutorInfo c = new ExecutorInfo(3, 3);
    List<ExecutorInfo> origList = new ArrayList<>();
    origList.add(a);
    origList.add(b);
    List<ExecutorInfo> equivList = new ArrayList<>();
    equivList.add(b);
    equivList.add(a);
    List<ExecutorInfo> differentList = new ArrayList<>();
    differentList.add(a);
    differentList.add(c);
    LocalAssignment laOrig = new LocalAssignment("topo1", origList);
    LocalAssignment laEquiv = new LocalAssignment("topo1", equivList);
    LocalAssignment laDifferent = new LocalAssignment("topo1", differentList);
    assertTrue(EquivalenceUtils.areLocalAssignmentsEquivalent(laOrig, laEquiv));
    assertFalse(EquivalenceUtils.areLocalAssignmentsEquivalent(laOrig, laDifferent));
    PortAndAssignment pnaOrig = new PortAndAssignmentImpl(1, laOrig);
    PortAndAssignment pnaEquiv = new PortAndAssignmentImpl(1, laEquiv);
    PortAndAssignment pnaDifferent = new PortAndAssignmentImpl(1, laDifferent);
    assertTrue(pnaOrig.isEquivalentTo(pnaEquiv));
    assertFalse(pnaOrig.isEquivalentTo(pnaDifferent));
    LocalizedResource localresource = new LocalizedResource("key1", Paths.get("testfile1"), false, mock(IAdvancedFSOps.class), new HashMap<>(), "user", new StormMetricsRegistry());
    localresource.addReference(pnaOrig, null);
    assertFalse(localresource.removeReference(pnaDifferent));
    assertTrue(localresource.removeReference(pnaEquiv));
}

Example 67 with StormMetricsRegistry

use of org.apache.storm.metric.StormMetricsRegistry in project storm by apache.

the class DRPCTest method testDequeueAfterTimeout.

@Test
public void testDequeueAfterTimeout() throws Exception {
    long timeout = 1000;
    try (DRPC server = new DRPC(new StormMetricsRegistry(), null, timeout)) {
        long start = Time.currentTimeMillis();
        try {
            server.executeBlocking("testing", "test");
            fail("Should have timed out....");
        } catch (DRPCExecutionException e) {
            long spent = Time.currentTimeMillis() - start;
            assertTrue(spent < timeout * 2);
            assertTrue(spent >= timeout);
            assertEquals(DRPCExceptionType.SERVER_TIMEOUT, e.get_type());
        }
        DRPCRequest request = server.fetchRequest("testing");
        assertNotNull(request);
        assertEquals("", request.get_request_id());
        assertEquals("", request.get_func_args());
    }
}

Example 68 with StormMetricsRegistry

use of org.apache.storm.metric.StormMetricsRegistry in project storm by apache.

the class BasicContainerTest method testLaunchStorm0version.

@Test
public void testLaunchStorm0version() throws Exception {
    final String topoId = "test_topology_storm_0.x";
    final int supervisorPort = 6628;
    final int port = 8080;
    final String stormHome = ContainerTest.asAbsPath("tmp", "storm-home");
    final String stormLogDir = ContainerTest.asFile(".", "target").getCanonicalPath();
    final String workerId = "worker-id";
    final String stormLocal = ContainerTest.asAbsPath("tmp", "storm-local");
    final String distRoot = ContainerTest.asAbsPath(stormLocal, "supervisor", "stormdist", topoId);
    final File stormcode = new File(distRoot, "stormcode.ser");
    final File stormjar = new File(distRoot, "stormjar.jar");
    final String log4jdir = ContainerTest.asAbsPath(stormHome, "conf");
    final String workerConf = ContainerTest.asAbsPath(log4jdir, "worker.xml");
    final String workerRoot = ContainerTest.asAbsPath(stormLocal, "workers", workerId);
    final String workerTmpDir = ContainerTest.asAbsPath(workerRoot, "tmp");
    final StormTopology st = new StormTopology();
    st.set_spouts(new HashMap<>());
    st.set_bolts(new HashMap<>());
    st.set_state_spouts(new HashMap<>());
    // minimum 0.x version of supporting STORM-2448 would be 0.10.3
    st.set_storm_version("0.10.3");
    byte[] serializedState = Utils.gzip(Utils.thriftSerialize(st));
    final Map<String, Object> superConf = new HashMap<>();
    superConf.put(Config.STORM_LOCAL_DIR, stormLocal);
    superConf.put(Config.STORM_WORKERS_ARTIFACTS_DIR, stormLocal);
    superConf.put(DaemonConfig.STORM_LOG4J2_CONF_DIR, log4jdir);
    superConf.put(Config.WORKER_CHILDOPTS, " -Dtesting=true");
    LocalAssignment la = new LocalAssignment();
    la.set_topology_id(topoId);
    AdvancedFSOps ops = mock(AdvancedFSOps.class);
    when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
    when(ops.slurp(stormcode)).thenReturn(serializedState);
    LocalState ls = mock(LocalState.class);
    MockResourceIsolationManager iso = new MockResourceIsolationManager();
    checkpoint(() -> {
        MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf, "SUPERVISOR", supervisorPort, port, la, iso, ls, workerId, new StormMetricsRegistry(), new HashMap<>(), ops, "profile");
        mc.launch();
        assertEquals(1, iso.workerCmds.size());
        CommandRun cmd = iso.workerCmds.get(0);
        iso.workerCmds.clear();
        assertListEquals(Arrays.asList("java", "-cp", "FRAMEWORK_CP:" + stormjar.getAbsolutePath(), "-Dlogging.sensitivity=S3", "-Dlogfile.name=worker.log", "-Dstorm.home=" + stormHome, "-Dworkers.artifacts=" + stormLocal, "-Dstorm.id=" + topoId, "-Dworker.id=" + workerId, "-Dworker.port=" + port, "-Dstorm.log.dir=" + stormLogDir, "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector", "-Dstorm.local.dir=" + stormLocal, "-Dworker.memory_limit_mb=768", "-Dlog4j.configurationFile=" + workerConf, "backtype.storm.LogWriter", "java", "-server", "-Dlogging.sensitivity=S3", "-Dlogfile.name=worker.log", "-Dstorm.home=" + stormHome, "-Dworkers.artifacts=" + stormLocal, "-Dstorm.id=" + topoId, "-Dworker.id=" + workerId, "-Dworker.port=" + port, "-Dstorm.log.dir=" + stormLogDir, "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector", "-Dstorm.local.dir=" + stormLocal, "-Dworker.memory_limit_mb=768", "-Dlog4j.configurationFile=" + workerConf, "-Dtesting=true", "-Djava.library.path=JLP", "-Dstorm.conf.file=", "-Dstorm.options=", "-Djava.io.tmpdir=" + workerTmpDir, "-cp", "FRAMEWORK_CP:" + stormjar.getAbsolutePath(), "backtype.storm.daemon.worker", topoId, "SUPERVISOR", String.valueOf(port), workerId), cmd.cmd);
        assertEquals(new File(workerRoot), cmd.pwd);
    }, ConfigUtils.STORM_HOME, stormHome, "storm.log.dir", stormLogDir);
}

Example 69 with StormMetricsRegistry

use of org.apache.storm.metric.StormMetricsRegistry in project storm by apache.

the class BasicContainerTest method testRecoveryMiss.

@Test
public void testRecoveryMiss() throws Exception {
    final String topoId = "test_topology";
    final int supervisorPort = 6628;
    final int port = 8080;
    LocalAssignment la = new LocalAssignment();
    la.set_topology_id(topoId);
    Map<String, Integer> workerState = new HashMap<String, Integer>();
    workerState.put("somethingelse", port + 1);
    LocalState ls = mock(LocalState.class);
    when(ls.getApprovedWorkers()).thenReturn(workerState);
    try {
        new MockBasicContainer(ContainerType.RECOVER_FULL, new HashMap<String, Object>(), "SUPERVISOR", supervisorPort, port, la, null, ls, null, new StormMetricsRegistry(), new HashMap<>(), null, "profile");
        fail("Container recovered worker incorrectly");
    } catch (ContainerRecoveryException e) {
    // Expected
    }
}

Example 70 with StormMetricsRegistry

use of org.apache.storm.metric.StormMetricsRegistry in project storm by apache.

the class BasicContainerTest method testLaunch.

@Test
public void testLaunch() throws Exception {
    final String topoId = "test_topology_current";
    final int supervisorPort = 6628;
    final int port = 8080;
    final String stormHome = ContainerTest.asAbsPath("tmp", "storm-home");
    final String stormLogDir = ContainerTest.asFile(".", "target").getCanonicalPath();
    final String workerId = "worker-id";
    final String stormLocal = ContainerTest.asAbsPath("tmp", "storm-local");
    final String distRoot = ContainerTest.asAbsPath(stormLocal, "supervisor", "stormdist", topoId);
    final File stormcode = new File(distRoot, "stormcode.ser");
    final File stormjar = new File(distRoot, "stormjar.jar");
    final String log4jdir = ContainerTest.asAbsPath(stormHome, "conf");
    final String workerConf = ContainerTest.asAbsPath(log4jdir, "worker.xml");
    final String workerRoot = ContainerTest.asAbsPath(stormLocal, "workers", workerId);
    final String workerTmpDir = ContainerTest.asAbsPath(workerRoot, "tmp");
    final StormTopology st = new StormTopology();
    st.set_spouts(new HashMap<>());
    st.set_bolts(new HashMap<>());
    st.set_state_spouts(new HashMap<>());
    byte[] serializedState = Utils.gzip(Utils.thriftSerialize(st));
    final Map<String, Object> superConf = new HashMap<>();
    superConf.put(Config.STORM_LOCAL_DIR, stormLocal);
    superConf.put(Config.STORM_WORKERS_ARTIFACTS_DIR, stormLocal);
    superConf.put(DaemonConfig.STORM_LOG4J2_CONF_DIR, log4jdir);
    superConf.put(Config.WORKER_CHILDOPTS, " -Dtesting=true");
    LocalAssignment la = new LocalAssignment();
    la.set_topology_id(topoId);
    AdvancedFSOps ops = mock(AdvancedFSOps.class);
    when(ops.doRequiredTopoFilesExist(superConf, topoId)).thenReturn(true);
    when(ops.slurp(stormcode)).thenReturn(serializedState);
    LocalState ls = mock(LocalState.class);
    MockResourceIsolationManager iso = new MockResourceIsolationManager();
    checkpoint(() -> {
        MockBasicContainer mc = new MockBasicContainer(ContainerType.LAUNCH, superConf, "SUPERVISOR", supervisorPort, port, la, iso, ls, workerId, new StormMetricsRegistry(), new HashMap<>(), ops, "profile");
        mc.launch();
        assertEquals(1, iso.workerCmds.size());
        CommandRun cmd = iso.workerCmds.get(0);
        iso.workerCmds.clear();
        assertListEquals(Arrays.asList("java", "-cp", "FRAMEWORK_CP:" + stormjar.getAbsolutePath(), "-Dlogging.sensitivity=S3", "-Dlogfile.name=worker.log", "-Dstorm.home=" + stormHome, "-Dworkers.artifacts=" + stormLocal, "-Dstorm.id=" + topoId, "-Dworker.id=" + workerId, "-Dworker.port=" + port, "-Dstorm.log.dir=" + stormLogDir, "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector", "-Dstorm.local.dir=" + stormLocal, "-Dworker.memory_limit_mb=768", "-Dlog4j.configurationFile=" + workerConf, "org.apache.storm.LogWriter", "java", "-server", "-Dlogging.sensitivity=S3", "-Dlogfile.name=worker.log", "-Dstorm.home=" + stormHome, "-Dworkers.artifacts=" + stormLocal, "-Dstorm.id=" + topoId, "-Dworker.id=" + workerId, "-Dworker.port=" + port, "-Dstorm.log.dir=" + stormLogDir, "-DLog4jContextSelector=org.apache.logging.log4j.core.selector.BasicContextSelector", "-Dstorm.local.dir=" + stormLocal, "-Dworker.memory_limit_mb=768", "-Dlog4j.configurationFile=" + workerConf, "-Dtesting=true", "-Djava.library.path=JLP", "-Dstorm.conf.file=", "-Dstorm.options=", "-Djava.io.tmpdir=" + workerTmpDir, "-cp", "FRAMEWORK_CP:" + stormjar.getAbsolutePath(), "org.apache.storm.daemon.worker.Worker", topoId, "SUPERVISOR", String.valueOf(supervisorPort), String.valueOf(port), workerId), cmd.cmd);
        assertEquals(new File(workerRoot), cmd.pwd);
    }, ConfigUtils.STORM_HOME, stormHome, "storm.log.dir", stormLogDir);
}