Examples with HostAndPortWithScheme org.apache.druid.server.http.HostAndPortWithScheme used on opensource projects

Example 1 with HostAndPortWithScheme

use of org.apache.druid.server.http.HostAndPortWithScheme in project druid by druid-io.

the class LookupCoordinatorManager method lookupManagementLoop.

@VisibleForTesting
void lookupManagementLoop() {
    // Sanity check for if we are shutting down
    if (Thread.currentThread().isInterrupted() || !lifecycleLock.awaitStarted(15, TimeUnit.SECONDS)) {
        LOG.info("Not updating lookups because process was interrupted or not finished starting yet.");
        return;
    }
    final Map<String, Map<String, LookupExtractorFactoryMapContainer>> allLookupTiers = lookupMapConfigRef.get();
    if (allLookupTiers == null) {
        LOG.info("Not updating lookups because no data exists");
        return;
    }
    LOG.debug("Starting lookup sync for on all nodes.");
    try {
        List<ListenableFuture<Map.Entry>> futures = new ArrayList<>();
        Set<String> discoveredLookupTiers = lookupNodeDiscovery.getAllTiers();
        // Check and Log warnings about lookups configured by user in DB but no nodes discovered to load those.
        for (String tierInDB : allLookupTiers.keySet()) {
            if (!discoveredLookupTiers.contains(tierInDB) && !allLookupTiers.getOrDefault(tierInDB, ImmutableMap.of()).isEmpty()) {
                LOG.warn("Found lookups for tier [%s] in DB, but no nodes discovered for it", tierInDB);
            }
        }
        for (String tier : discoveredLookupTiers) {
            LOG.debug("Starting lookup mgmt for tier [%s].", tier);
            final Map<String, LookupExtractorFactoryMapContainer> tierLookups = allLookupTiers.getOrDefault(tier, ImmutableMap.of());
            for (final HostAndPortWithScheme node : lookupNodeDiscovery.getNodesInTier(tier)) {
                LOG.debug("Starting lookup mgmt for tier [%s] and host [%s:%s:%s].", tier, node.getScheme(), node.getHostText(), node.getPort());
                futures.add(executorService.submit(() -> {
                    try {
                        return new AbstractMap.SimpleImmutableEntry<>(node.getHostAndPort(), doLookupManagementOnNode(node, tierLookups));
                    } catch (InterruptedException ex) {
                        LOG.warn(ex, "lookup management on node [%s:%s:%s] interrupted.", node.getScheme(), node.getHostText(), node.getPort());
                        return null;
                    } catch (Exception ex) {
                        LOG.makeAlert(ex, "Failed to finish lookup management on node [%s:%s:%s]", node.getScheme(), node.getHostText(), node.getPort()).emit();
                        return null;
                    }
                }));
            }
        }
        final ListenableFuture<List<Map.Entry>> allFuture = Futures.allAsList(futures);
        try {
            ImmutableMap.Builder<HostAndPort, LookupsState<LookupExtractorFactoryMapContainer>> stateBuilder = ImmutableMap.builder();
            allFuture.get(lookupCoordinatorManagerConfig.getAllHostTimeout().getMillis(), TimeUnit.MILLISECONDS).stream().filter(Objects::nonNull).forEach(stateBuilder::put);
            knownOldState.set(stateBuilder.build());
        } catch (InterruptedException ex) {
            allFuture.cancel(true);
            Thread.currentThread().interrupt();
            throw ex;
        } catch (Exception ex) {
            allFuture.cancel(true);
            throw ex;
        }
    } catch (Exception ex) {
        LOG.makeAlert(ex, "Failed to finish lookup management loop.").emit();
    }
    LOG.debug("Finished lookup sync for on all nodes.");
}

Example 2 with HostAndPortWithScheme

use of org.apache.druid.server.http.HostAndPortWithScheme in project druid by druid-io.

the class LookupCoordinatorManagerTest method testLookupManagementLoop.

@Test(timeout = 60_000L)
public void testLookupManagementLoop() throws Exception {
    Map<String, LookupExtractorFactoryMapContainer> lookup1 = ImmutableMap.of("lookup1", new LookupExtractorFactoryMapContainer("v1", ImmutableMap.of("k1", "v1")));
    Map<String, Map<String, LookupExtractorFactoryMapContainer>> configuredLookups = ImmutableMap.of("tier1", lookup1);
    EasyMock.reset(configManager);
    EasyMock.expect(configManager.watch(EasyMock.eq(LookupCoordinatorManager.LOOKUP_CONFIG_KEY), EasyMock.<TypeReference>anyObject(), EasyMock.<AtomicReference>isNull())).andReturn(new AtomicReference<>(configuredLookups)).once();
    EasyMock.replay(configManager);
    HostAndPortWithScheme host1 = HostAndPortWithScheme.fromParts("http", "host1", 1234);
    HostAndPortWithScheme host2 = HostAndPortWithScheme.fromParts("http", "host2", 3456);
    EasyMock.reset(lookupNodeDiscovery);
    EasyMock.expect(lookupNodeDiscovery.getAllTiers()).andReturn(ImmutableSet.of("tier1")).once();
    EasyMock.expect(lookupNodeDiscovery.getNodesInTier("tier1")).andReturn(ImmutableList.of(host1, host2)).anyTimes();
    EasyMock.replay(lookupNodeDiscovery);
    LookupCoordinatorManager.LookupsCommunicator lookupsCommunicator = EasyMock.createMock(LookupCoordinatorManager.LookupsCommunicator.class);
    EasyMock.expect(lookupsCommunicator.getLookupStateForNode(host1)).andReturn(new LookupsState<>(ImmutableMap.of("lookup0", new LookupExtractorFactoryMapContainer("v1", ImmutableMap.of("k0", "v0"))), null, null)).once();
    LookupsState<LookupExtractorFactoryMapContainer> host1UpdatedState = new LookupsState<>(lookup1, null, null);
    EasyMock.expect(lookupsCommunicator.updateNode(host1, new LookupsState<>(null, lookup1, ImmutableSet.of("lookup0")))).andReturn(host1UpdatedState).once();
    EasyMock.expect(lookupsCommunicator.getLookupStateForNode(host2)).andReturn(new LookupsState<>(ImmutableMap.of("lookup3", new LookupExtractorFactoryMapContainer("v1", ImmutableMap.of("k0", "v0")), "lookup1", new LookupExtractorFactoryMapContainer("v0", ImmutableMap.of("k0", "v0"))), null, null)).once();
    LookupsState<LookupExtractorFactoryMapContainer> host2UpdatedState = new LookupsState<>(null, lookup1, null);
    EasyMock.expect(lookupsCommunicator.updateNode(host2, new LookupsState<>(null, lookup1, ImmutableSet.of("lookup3")))).andReturn(host2UpdatedState).once();
    EasyMock.replay(lookupsCommunicator);
    LookupCoordinatorManagerConfig lookupCoordinatorManagerConfig = new LookupCoordinatorManagerConfig() {

        @Override
        public long getInitialDelay() {
            return 1;
        }

        @Override
        public int getThreadPoolSize() {
            return 2;
        }
    };
    final LookupCoordinatorManager manager = new LookupCoordinatorManager(druidNodeDiscoveryProvider, configManager, lookupCoordinatorManagerConfig, lookupsCommunicator, lookupNodeDiscovery);
    Assert.assertTrue(manager.knownOldState.get().isEmpty());
    manager.start();
    Map<HostAndPort, LookupsState<LookupExtractorFactoryMapContainer>> expectedKnownState = ImmutableMap.of(host1.getHostAndPort(), host1UpdatedState, host2.getHostAndPort(), host2UpdatedState);
    while (!expectedKnownState.equals(manager.knownOldState.get())) {
        Thread.sleep(100);
    }
    EasyMock.verify(lookupNodeDiscovery, configManager, lookupsCommunicator);
}