Examples with ListeningExecutorService com.google.common.util.concurrent.ListeningExecutorService used on opensource projects

Example 76 with ListeningExecutorService

use of com.google.common.util.concurrent.ListeningExecutorService in project druid by druid-io.

the class DruidAvaticaHandlerTest method testConcurrentQueries.

@Test(timeout = 90_000L)
public void testConcurrentQueries() throws Exception {
    final List<ListenableFuture<Integer>> futures = new ArrayList<>();
    final ListeningExecutorService exec = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(AVATICA_CONFIG.getMaxStatementsPerConnection()));
    for (int i = 0; i < 2000; i++) {
        final String query = StringUtils.format("SELECT COUNT(*) + %s AS ci FROM foo", i);
        futures.add(exec.submit(() -> {
            try (final Statement statement = client.createStatement();
                final ResultSet resultSet = statement.executeQuery(query)) {
                final List<Map<String, Object>> rows = getRows(resultSet);
                return ((Number) Iterables.getOnlyElement(rows).get("ci")).intValue();
            } catch (SQLException e) {
                throw new RuntimeException(e);
            }
        }));
    }
    final List<Integer> integers = Futures.allAsList(futures).get();
    for (int i = 0; i < 2000; i++) {
        Assert.assertEquals(i + 6, (int) integers.get(i));
    }
}

Example 77 with ListeningExecutorService

use of com.google.common.util.concurrent.ListeningExecutorService in project druid by druid-io.

the class RunRulesTest method testRulesRunOnNonOvershadowedSegmentsOnly.

@Test
public void testRulesRunOnNonOvershadowedSegmentsOnly() {
    Set<DataSegment> usedSegments = new HashSet<>();
    DataSegment v1 = new DataSegment("test", Intervals.of("2012-01-01/2012-01-02"), "1", new HashMap<>(), new ArrayList<>(), new ArrayList<>(), NoneShardSpec.instance(), IndexIO.CURRENT_VERSION_ID, 1);
    DataSegment v2 = new DataSegment("test", Intervals.of("2012-01-01/2012-01-02"), "2", new HashMap<>(), new ArrayList<>(), new ArrayList<>(), NoneShardSpec.instance(), IndexIO.CURRENT_VERSION_ID, 1);
    usedSegments.add(v1);
    usedSegments.add(v2);
    mockCoordinator();
    mockPeon.loadSegment(EasyMock.eq(v2), EasyMock.anyObject());
    EasyMock.expectLastCall().once();
    mockEmptyPeon();
    EasyMock.expect(databaseRuleManager.getRulesWithDefault(EasyMock.anyObject())).andReturn(Collections.singletonList(new ForeverLoadRule(ImmutableMap.of(DruidServer.DEFAULT_TIER, 1)))).atLeastOnce();
    EasyMock.replay(databaseRuleManager);
    DruidCluster druidCluster = DruidClusterBuilder.newBuilder().addTier(DruidServer.DEFAULT_TIER, new ServerHolder(new DruidServer("serverHot", "hostHot", null, 1000, ServerType.HISTORICAL, DruidServer.DEFAULT_TIER, 0).toImmutableDruidServer(), mockPeon)).build();
    ListeningExecutorService exec = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(1));
    BalancerStrategy balancerStrategy = new CostBalancerStrategyFactory().createBalancerStrategy(exec);
    DruidCoordinatorRuntimeParams params = CoordinatorRuntimeParamsTestHelpers.newBuilder().withDruidCluster(druidCluster).withUsedSegmentsInTest(usedSegments).withDatabaseRuleManager(databaseRuleManager).withSegmentReplicantLookup(SegmentReplicantLookup.make(new DruidCluster(), false)).withBalancerStrategy(balancerStrategy).withDynamicConfigs(CoordinatorDynamicConfig.builder().withMaxSegmentsToMove(5).build()).build();
    DruidCoordinatorRuntimeParams afterParams = ruleRunner.run(params);
    CoordinatorStats stats = afterParams.getCoordinatorStats();
    Assert.assertEquals(1, stats.getTiers("assignedCount").size());
    Assert.assertEquals(1, stats.getTieredStat("assignedCount", "_default_tier"));
    Assert.assertTrue(stats.getTiers("unassignedCount").isEmpty());
    Assert.assertTrue(stats.getTiers("unassignedSize").isEmpty());
    Assert.assertEquals(2, usedSegments.size());
    Assert.assertEquals(usedSegments, params.getUsedSegments());
    Assert.assertEquals(usedSegments, afterParams.getUsedSegments());
    EasyMock.verify(mockPeon);
    exec.shutdown();
}

Example 78 with ListeningExecutorService

use of com.google.common.util.concurrent.ListeningExecutorService in project druid by druid-io.

the class RunRulesTest method testTwoTiersTwoReplicantsWithStrictReplicantLimit.

/**
 * Nodes:
 * normal - 2 replicants
 * hot - 2 replicants
 * maxNonPrimaryReplicantsToLoad - 48
 * Expect only 72 segments to be loaded despite there being 96 primary + non-primary replicants to load!
 */
@Test
public void testTwoTiersTwoReplicantsWithStrictReplicantLimit() {
    mockCoordinator();
    mockPeon.loadSegment(EasyMock.anyObject(), EasyMock.anyObject());
    EasyMock.expectLastCall().atLeastOnce();
    mockEmptyPeon();
    EasyMock.expect(databaseRuleManager.getRulesWithDefault(EasyMock.anyObject())).andReturn(Collections.singletonList(new IntervalLoadRule(Intervals.of("2012-01-01T00:00:00.000Z/2012-01-02T00:00:00.000Z"), ImmutableMap.of("hot", 2, "normal", 2)))).atLeastOnce();
    EasyMock.replay(databaseRuleManager);
    DruidCluster druidCluster = DruidClusterBuilder.newBuilder().addTier("hot", new ServerHolder(new DruidServer("serverHot", "hostHot", null, 1000, ServerType.HISTORICAL, "hot", 0).toImmutableDruidServer(), mockPeon), new ServerHolder(new DruidServer("serverHot2", "hostHot2", null, 1000, ServerType.HISTORICAL, "hot", 0).toImmutableDruidServer(), mockPeon)).addTier("normal", new ServerHolder(new DruidServer("serverNorm", "hostNorm", null, 1000, ServerType.HISTORICAL, "normal", 0).toImmutableDruidServer(), mockPeon), new ServerHolder(new DruidServer("serverNorm2", "hostNorm2", null, 1000, ServerType.HISTORICAL, "normal", 0).toImmutableDruidServer(), mockPeon)).build();
    ListeningExecutorService exec = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(1));
    BalancerStrategy balancerStrategy = new CostBalancerStrategyFactory().createBalancerStrategy(exec);
    DruidCoordinatorRuntimeParams params = makeCoordinatorRuntimeParams(druidCluster, balancerStrategy).withDynamicConfigs(CoordinatorDynamicConfig.builder().withMaxSegmentsToMove(5).withMaxNonPrimaryReplicantsToLoad(48).build()).build();
    DruidCoordinatorRuntimeParams afterParams = ruleRunner.run(params);
    CoordinatorStats stats = afterParams.getCoordinatorStats();
    Assert.assertEquals(72L, stats.getTieredStat("assignedCount", "hot") + stats.getTieredStat("assignedCount", "normal"));
    Assert.assertEquals(48L, stats.getGlobalStat("totalNonPrimaryReplicantsLoaded"));
    exec.shutdown();
    EasyMock.verify(mockPeon);
}

Example 79 with ListeningExecutorService

use of com.google.common.util.concurrent.ListeningExecutorService in project druid by druid-io.

the class RunRulesTest method testDropServerActuallyServesSegment.

@Test
public void testDropServerActuallyServesSegment() {
    mockCoordinator();
    EasyMock.expect(databaseRuleManager.getRulesWithDefault(EasyMock.anyObject())).andReturn(Collections.singletonList(new IntervalLoadRule(Intervals.of("2012-01-01T00:00:00.000Z/2012-01-01T01:00:00.000Z"), ImmutableMap.of("normal", 0)))).atLeastOnce();
    EasyMock.replay(databaseRuleManager);
    DruidServer server1 = new DruidServer("server1", "host1", null, 1000, ServerType.HISTORICAL, "normal", 0);
    server1.addDataSegment(usedSegments.get(0));
    DruidServer server2 = new DruidServer("serverNorm2", "hostNorm2", null, 1000, ServerType.HISTORICAL, "normal", 0);
    server2.addDataSegment(usedSegments.get(1));
    DruidServer server3 = new DruidServer("serverNorm3", "hostNorm3", null, 1000, ServerType.HISTORICAL, "normal", 0);
    server3.addDataSegment(usedSegments.get(1));
    server3.addDataSegment(usedSegments.get(2));
    mockPeon.dropSegment(EasyMock.anyObject(), EasyMock.anyObject());
    EasyMock.expectLastCall().atLeastOnce();
    mockEmptyPeon();
    LoadQueuePeon anotherMockPeon = EasyMock.createMock(LoadQueuePeon.class);
    EasyMock.expect(anotherMockPeon.getLoadQueueSize()).andReturn(10L).atLeastOnce();
    EasyMock.expect(anotherMockPeon.getSegmentsToLoad()).andReturn(new HashSet<>()).anyTimes();
    EasyMock.replay(anotherMockPeon);
    DruidCluster druidCluster = DruidClusterBuilder.newBuilder().addTier("normal", new ServerHolder(server1.toImmutableDruidServer(), mockPeon, false), new ServerHolder(server2.toImmutableDruidServer(), anotherMockPeon, false), new ServerHolder(server3.toImmutableDruidServer(), anotherMockPeon, false)).build();
    SegmentReplicantLookup segmentReplicantLookup = SegmentReplicantLookup.make(druidCluster, false);
    ListeningExecutorService exec = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(1));
    BalancerStrategy balancerStrategy = new CostBalancerStrategyFactory().createBalancerStrategy(exec);
    DruidCoordinatorRuntimeParams params = CoordinatorRuntimeParamsTestHelpers.newBuilder().withDruidCluster(druidCluster).withDynamicConfigs(COORDINATOR_CONFIG_WITH_ZERO_LEADING_TIME_BEFORE_CAN_MARK_AS_UNUSED_OVERSHADOWED_SEGMENTS).withUsedSegmentsInTest(usedSegments).withDatabaseRuleManager(databaseRuleManager).withSegmentReplicantLookup(segmentReplicantLookup).withBalancerStrategy(balancerStrategy).build();
    DruidCoordinatorRuntimeParams afterParams = ruleRunner.run(params);
    CoordinatorStats stats = afterParams.getCoordinatorStats();
    Assert.assertEquals(1L, stats.getTieredStat("droppedCount", "normal"));
    exec.shutdown();
    EasyMock.verify(mockPeon);
    EasyMock.verify(anotherMockPeon);
}

Example 80 with ListeningExecutorService

use of com.google.common.util.concurrent.ListeningExecutorService in project druid by druid-io.

the class RunRulesTest method testDontDropInDifferentTiers.

@Test
public void testDontDropInDifferentTiers() {
    mockCoordinator();
    mockPeon.loadSegment(EasyMock.anyObject(), EasyMock.anyObject());
    EasyMock.expectLastCall().atLeastOnce();
    mockEmptyPeon();
    EasyMock.expect(databaseRuleManager.getRulesWithDefault(EasyMock.anyObject())).andReturn(Lists.newArrayList(new IntervalLoadRule(Intervals.of("2012-01-01T00:00:00.000Z/2012-01-01T12:00:00.000Z"), ImmutableMap.of("hot", 1)), new IntervalDropRule(Intervals.of("2012-01-01T00:00:00.000Z/2012-01-02T00:00:00.000Z")))).atLeastOnce();
    EasyMock.replay(databaseRuleManager);
    DruidServer server1 = new DruidServer("server1", "host1", null, 1000, ServerType.HISTORICAL, "hot", 0);
    DruidServer server2 = new DruidServer("serverNorm2", "hostNorm2", null, 1000, ServerType.HISTORICAL, "normal", 0);
    for (DataSegment segment : usedSegments) {
        server2.addDataSegment(segment);
    }
    DruidCluster druidCluster = DruidClusterBuilder.newBuilder().addTier("hot", new ServerHolder(server1.toImmutableDruidServer(), mockPeon)).addTier("normal", new ServerHolder(server2.toImmutableDruidServer(), mockPeon)).build();
    SegmentReplicantLookup segmentReplicantLookup = SegmentReplicantLookup.make(druidCluster, false);
    ListeningExecutorService exec = MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(1));
    BalancerStrategy balancerStrategy = new CostBalancerStrategyFactory().createBalancerStrategy(exec);
    DruidCoordinatorRuntimeParams params = createCoordinatorRuntimeParams(druidCluster).withDynamicConfigs(COORDINATOR_CONFIG_WITH_ZERO_LEADING_TIME_BEFORE_CAN_MARK_AS_UNUSED_OVERSHADOWED_SEGMENTS).withSegmentReplicantLookup(segmentReplicantLookup).withBalancerStrategy(balancerStrategy).build();
    DruidCoordinatorRuntimeParams afterParams = ruleRunner.run(params);
    CoordinatorStats stats = afterParams.getCoordinatorStats();
    Assert.assertTrue(stats.getTiers("droppedCount").isEmpty());
    Assert.assertEquals(12L, stats.getGlobalStat("deletedCount"));
    exec.shutdown();
    EasyMock.verify(mockPeon);
}