Examples with StormMetricsRegistry org.apache.storm.metric.StormMetricsRegistry used on opensource projects

Example 41 with StormMetricsRegistry

use of org.apache.storm.metric.StormMetricsRegistry in project storm by apache.

the class AsyncLocalizerTest method testRequestDownloadTopologyBlobs.

@Test
public void testRequestDownloadTopologyBlobs() throws Exception {
    ConfigUtils mockedConfigUtils = mock(ConfigUtils.class);
    ConfigUtils previousConfigUtils = ConfigUtils.setInstance(mockedConfigUtils);
    AsyncLocalizer victim = null;
    try (TmpPath stormLocal = new TmpPath();
        TmpPath localizerRoot = new TmpPath()) {
        Map<String, Object> conf = new HashMap<>();
        conf.put(Config.STORM_LOCAL_DIR, stormLocal.getPath());
        AdvancedFSOps ops = AdvancedFSOps.make(conf);
        StormMetricsRegistry metricsRegistry = new StormMetricsRegistry();
        victim = spy(new AsyncLocalizer(conf, ops, localizerRoot.getPath(), metricsRegistry));
        final String topoId = "TOPO-12345";
        final String user = "user";
        final Path userDir = Paths.get(stormLocal.getPath(), user);
        final Path topologyDirRoot = Paths.get(stormLocal.getPath(), topoId);
        final String simpleLocalName = "simple.txt";
        final String simpleKey = "simple";
        Map<String, Map<String, Object>> topoBlobMap = new HashMap<>();
        Map<String, Object> simple = new HashMap<>();
        simple.put("localname", simpleLocalName);
        simple.put("uncompress", false);
        topoBlobMap.put(simpleKey, simple);
        final int port = 8080;
        Map<String, Object> topoConf = new HashMap<>(conf);
        topoConf.put(Config.TOPOLOGY_BLOBSTORE_MAP, topoBlobMap);
        topoConf.put(Config.TOPOLOGY_NAME, "TOPO");
        List<LocalizedResource> localizedList = new ArrayList<>();
        LocalizedResource simpleLocal = new LocalizedResource(simpleKey, localizerRoot.getFile().toPath(), false, ops, conf, user, metricsRegistry);
        localizedList.add(simpleLocal);
        when(mockedConfigUtils.supervisorStormDistRootImpl(conf, topoId)).thenReturn(topologyDirRoot.toString());
        when(mockedConfigUtils.readSupervisorStormConfImpl(conf, topoId)).thenReturn(topoConf);
        when(mockedConfigUtils.readSupervisorTopologyImpl(conf, topoId, ops)).thenReturn(constructEmptyStormTopology());
        // Write the mocking backwards so the actual method is not called on the spy object
        doReturn(CompletableFuture.supplyAsync(() -> null)).when(victim).requestDownloadBaseTopologyBlobs(any(), eq(null));
        Files.createDirectories(topologyDirRoot);
        doReturn(userDir.toFile()).when(victim).getLocalUserFileCacheDir(user);
        doReturn(localizedList).when(victim).getBlobs(any(List.class), any(), any());
        Future<Void> f = victim.requestDownloadTopologyBlobs(constructLocalAssignment(topoId, user), port, null);
        f.get(20, TimeUnit.SECONDS);
        // We should be done now...
        verify(victim).getLocalUserFileCacheDir(user);
        assertTrue(ops.fileExists(userDir));
        verify(victim).getBlobs(any(List.class), any(), any());
        // symlink was created
        assertTrue(Files.isSymbolicLink(topologyDirRoot.resolve(simpleLocalName)));
    } finally {
        ConfigUtils.setInstance(previousConfigUtils);
        if (victim != null) {
            victim.close();
        }
    }
}

Example 42 with StormMetricsRegistry

use of org.apache.storm.metric.StormMetricsRegistry in project storm by apache.

the class AsyncLocalizerTest method testRequestDownloadTopologyBlobsLocalMode.

@Test
public void testRequestDownloadTopologyBlobsLocalMode() throws Exception {
    // tests download of topology blobs in local mode on a topology without resources folder
    ConfigUtils mockedConfigUtils = mock(ConfigUtils.class);
    ServerUtils mockedServerUtils = mock(ServerUtils.class);
    ConfigUtils previousConfigUtils = ConfigUtils.setInstance(mockedConfigUtils);
    ServerUtils previousServerUtils = ServerUtils.setInstance(mockedServerUtils);
    AsyncLocalizer victim = null;
    try (TmpPath stormLocal = new TmpPath();
        TmpPath localizerRoot = new TmpPath()) {
        Map<String, Object> conf = new HashMap<>();
        conf.put(Config.STORM_LOCAL_DIR, stormLocal.getPath());
        conf.put(Config.STORM_CLUSTER_MODE, "local");
        StormMetricsRegistry metricsRegistry = new StormMetricsRegistry();
        AdvancedFSOps ops = AdvancedFSOps.make(conf);
        victim = spy(new AsyncLocalizer(conf, ops, localizerRoot.getPath(), metricsRegistry));
        final String topoId = "TOPO-12345";
        final String user = "user";
        final int port = 8080;
        final Path userDir = Paths.get(stormLocal.getPath(), user);
        final Path stormRoot = Paths.get(stormLocal.getPath(), topoId);
        final String simpleLocalName = "simple.txt";
        final String simpleKey = "simple";
        Map<String, Map<String, Object>> topoBlobMap = new HashMap<>();
        Map<String, Object> simple = new HashMap<>();
        simple.put("localname", simpleLocalName);
        simple.put("uncompress", false);
        topoBlobMap.put(simpleKey, simple);
        Map<String, Object> topoConf = new HashMap<>(conf);
        topoConf.put(Config.TOPOLOGY_BLOBSTORE_MAP, topoBlobMap);
        topoConf.put(Config.TOPOLOGY_NAME, "TOPO");
        List<LocalizedResource> localizedList = new ArrayList<>();
        LocalizedResource simpleLocal = new LocalizedResource(simpleKey, localizerRoot.getFile().toPath(), false, ops, conf, user, metricsRegistry);
        localizedList.add(simpleLocal);
        when(mockedConfigUtils.supervisorStormDistRootImpl(conf, topoId)).thenReturn(stormRoot.toString());
        when(mockedConfigUtils.readSupervisorStormConfImpl(conf, topoId)).thenReturn(topoConf);
        when(mockedConfigUtils.readSupervisorTopologyImpl(conf, topoId, ops)).thenReturn(constructEmptyStormTopology());
        doReturn(mockBlobStore).when(victim).getClientBlobStore();
        doReturn(userDir.toFile()).when(victim).getLocalUserFileCacheDir(user);
        doReturn(localizedList).when(victim).getBlobs(any(List.class), any(), any());
        ReadableBlobMeta blobMeta = new ReadableBlobMeta();
        blobMeta.set_version(1);
        doReturn(blobMeta).when(mockBlobStore).getBlobMeta(any());
        when(mockBlobStore.getBlob(any())).thenAnswer(invocation -> new TestInputStreamWithMeta(LOCAL_MODE_JAR_VERSION));
        Future<Void> f = victim.requestDownloadTopologyBlobs(constructLocalAssignment(topoId, user), port, null);
        f.get(20, TimeUnit.SECONDS);
        verify(victim).getLocalUserFileCacheDir(user);
        assertTrue(ops.fileExists(userDir));
        verify(victim).getBlobs(any(List.class), any(), any());
        // make sure resources directory after blob version commit is created.
        Path extractionDir = stormRoot.resolve(LocallyCachedTopologyBlob.TopologyBlobType.TOPO_JAR.getExtractionDir());
        assertTrue(ops.fileExists(extractionDir));
    } finally {
        ConfigUtils.setInstance(previousConfigUtils);
        ServerUtils.setInstance(previousServerUtils);
        if (victim != null) {
            victim.close();
        }
    }
}

Example 43 with StormMetricsRegistry

use of org.apache.storm.metric.StormMetricsRegistry in project storm by apache.

the class LocallyCachedBlobTest method testOutOfDate.

@Test
public void testOutOfDate() throws KeyNotFoundException, AuthorizationException {
    TestableBlob blob = new TestableBlob("key", Paths.get("/bogus"), false, AdvancedFSOps.make(conf), conf, "user1", new StormMetricsRegistry());
    blob.addReference(pna, null);
    Assert.assertTrue(blob.isUsed());
    Assert.assertTrue(blob.isFullyDownloaded());
    // validate blob needs update due to version mismatch
    Assert.assertTrue(blob.requiresUpdate(blobStore, -1L));
    // when blob update time matches remote blobstore update time, validate blob
    // will skip looking at remote version and assume it's up to date
    blob.localUpdateTime = 101L;
    Assert.assertFalse(blob.requiresUpdate(blobStore, 101L));
    // now when the update time on the remote blobstore differs, we should again see that the
    // blob version differs from the remote blobstore
    Assert.assertTrue(blob.requiresUpdate(blobStore, 102L));
    // now validate we don't need any update as versions match, regardless of remote blobstore update time
    blob.localVersion = blob.getRemoteVersion(blobStore);
    Assert.assertFalse(blob.requiresUpdate(blobStore, -1L));
    Assert.assertFalse(blob.requiresUpdate(blobStore, 101L));
    Assert.assertFalse(blob.requiresUpdate(blobStore, 102L));
}

Example 44 with StormMetricsRegistry

use of org.apache.storm.metric.StormMetricsRegistry in project storm by apache.

the class TestDefaultEvictionStrategy method testEviction.

/**
 * The resources in the cluster are limited. In the first round of scheduling, all resources in the cluster is used.
 * User jerry submits another topology.  Since user jerry has his resource guarantees satisfied, and user bobby
 * has exceeded his resource guarantee, topo-3 from user bobby should be evicted.
 */
@Test
public void testEviction() {
    INimbus iNimbus = new INimbusTest();
    Map<String, SupervisorDetails> supMap = genSupervisors(4, 4, 100, 1000);
    Map<String, Map<String, Number>> resourceUserPool = userResourcePool(userRes("jerry", 200, 2000), userRes("bobby", 100, 1000), userRes("derek", 200, 2000));
    Config config = createClusterConfig(100, 500, 500, resourceUserPool);
    Topologies topologies = new Topologies(genTopology("topo-1", config, 1, 0, 1, 0, currentTime - 2, 10, "jerry"), genTopology("topo-2", config, 1, 0, 1, 0, currentTime - 2, 10, "bobby"), genTopology("topo-3", config, 1, 0, 1, 0, currentTime - 2, 20, "bobby"), genTopology("topo-4", config, 1, 0, 1, 0, currentTime - 2, 29, "derek"));
    Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config);
    scheduler = new ResourceAwareScheduler();
    scheduler.prepare(config, new StormMetricsRegistry());
    scheduler.schedule(topologies, cluster);
    assertTopologiesFullyScheduled(cluster, "topo-1", "topo-2", "topo-3", "topo-4");
    // user jerry submits another topology
    topologies = addTopologies(topologies, genTopology("topo-6", config, 1, 0, 1, 0, currentTime - 2, 20, "jerry"));
    cluster = new Cluster(cluster, topologies);
    scheduler.schedule(topologies, cluster);
    // topo-3 evicted (lowest priority)
    assertTopologiesFullyScheduled(cluster, "topo-1", "topo-2", "topo-4", "topo-6");
    assertTopologiesNotScheduled(cluster, "topo-3");
}

Example 45 with StormMetricsRegistry

use of org.apache.storm.metric.StormMetricsRegistry in project storm by apache.

the class TestDefaultEvictionStrategy method testOverGuaranteeEviction.

/**
 * If users are above his or her guarantee, check if topology eviction works correctly
 */
@Test
public void testOverGuaranteeEviction() {
    INimbus iNimbus = new INimbusTest();
    Map<String, SupervisorDetails> supMap = genSupervisors(4, 4, 100, 1000);
    Map<String, Map<String, Number>> resourceUserPool = userResourcePool(userRes("jerry", 70, 700), userRes("bobby", 100, 1000), userRes("derek", 25, 250));
    Config config = createClusterConfig(100, 500, 500, resourceUserPool);
    Topologies topologies = new Topologies(genTopology("topo-1", config, 1, 0, 1, 0, currentTime - 2, 20, "jerry"), genTopology("topo-3", config, 1, 0, 1, 0, currentTime - 2, 10, "bobby"), genTopology("topo-4", config, 1, 0, 1, 0, currentTime - 2, 10, "bobby"), genTopology("topo-5", config, 1, 0, 1, 0, currentTime - 2, 29, "derek"));
    Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config);
    scheduler = new ResourceAwareScheduler();
    scheduler.prepare(config, new StormMetricsRegistry());
    LOG.info("\n\n\t\tScheduling topos 1,3,4,5");
    scheduler.schedule(topologies, cluster);
    LOG.info("\n\n\t\tDone scheduling...");
    assertTopologiesFullyScheduled(cluster, "topo-1", "topo-3", "topo-4", "topo-5");
    // user derek submits another topology into a full cluster
    // topo6 should not be able to scheduled initially, but since topo6 has higher priority than topo5
    // topo5 will be evicted so that topo6 can be scheduled
    topologies = addTopologies(topologies, genTopology("topo-6", config, 1, 0, 1, 0, currentTime - 2, 10, "derek"));
    cluster = new Cluster(cluster, topologies);
    LOG.info("\n\n\t\tScheduling topos 1,3,4,5,6");
    scheduler.schedule(topologies, cluster);
    LOG.info("\n\n\t\tDone scheduling...");
    assertTopologiesFullyScheduled(cluster, "topo-1", "topo-3", "topo-4", "topo-6");
    assertTopologiesNotScheduled(cluster, "topo-5");
    // user jerry submits topo2
    topologies = addTopologies(topologies, genTopology("topo-2", config, 1, 0, 1, 0, currentTime - 2, 20, "jerry"));
    cluster = new Cluster(cluster, topologies);
    LOG.info("\n\n\t\tScheduling topos 1-6");
    scheduler.schedule(topologies, cluster);
    LOG.info("\n\n\t\tDone scheduling...");
    assertTopologiesFullyScheduled(cluster, "topo-1", "topo-3", "topo-4", "topo-6");
    assertTopologiesNotScheduled(cluster, "topo-2", "topo-5");
}