Examples with LookupsState org.apache.druid.query.lookup.LookupsState used on opensource projects

Example 1 with LookupsState

use of org.apache.druid.query.lookup.LookupsState in project druid by druid-io.

the class LookupCoordinatorResource method getDetailedAllNodeStatus.

/**
 * Build 'detailed' lookup cluster status, broken down by tier, host, and then the full {@link LookupsState} with
 * complete {@link LookupExtractorFactoryMapContainer} information.
 */
private Map<String, Map<HostAndPort, LookupsState<LookupExtractorFactoryMapContainer>>> getDetailedAllNodeStatus(Map<HostAndPort, LookupsState<LookupExtractorFactoryMapContainer>> lookupsStateOnHosts, Collection<String> tiers) {
    Map<String, Map<HostAndPort, LookupsState<LookupExtractorFactoryMapContainer>>> result = new HashMap<>();
    for (String tier : tiers) {
        Map<HostAndPort, LookupsState<LookupExtractorFactoryMapContainer>> tierNodesStatus = new HashMap<>();
        result.put(tier, tierNodesStatus);
        Collection<HostAndPort> nodes = lookupCoordinatorManager.discoverNodesInTier(tier);
        for (HostAndPort node : nodes) {
            LookupsState<LookupExtractorFactoryMapContainer> lookupsState = lookupsStateOnHosts.get(node);
            if (lookupsState == null) {
                tierNodesStatus.put(node, new LookupsState<>(null, null, null));
            } else {
                tierNodesStatus.put(node, lookupsState);
            }
        }
    }
    return result;
}

Example 2 with LookupsState

use of org.apache.druid.query.lookup.LookupsState in project druid by druid-io.

the class LookupCoordinatorManager method lookupManagementLoop.

@VisibleForTesting
void lookupManagementLoop() {
    // Sanity check for if we are shutting down
    if (Thread.currentThread().isInterrupted() || !lifecycleLock.awaitStarted(15, TimeUnit.SECONDS)) {
        LOG.info("Not updating lookups because process was interrupted or not finished starting yet.");
        return;
    }
    final Map<String, Map<String, LookupExtractorFactoryMapContainer>> allLookupTiers = lookupMapConfigRef.get();
    if (allLookupTiers == null) {
        LOG.info("Not updating lookups because no data exists");
        return;
    }
    LOG.debug("Starting lookup sync for on all nodes.");
    try {
        List<ListenableFuture<Map.Entry>> futures = new ArrayList<>();
        Set<String> discoveredLookupTiers = lookupNodeDiscovery.getAllTiers();
        // Check and Log warnings about lookups configured by user in DB but no nodes discovered to load those.
        for (String tierInDB : allLookupTiers.keySet()) {
            if (!discoveredLookupTiers.contains(tierInDB) && !allLookupTiers.getOrDefault(tierInDB, ImmutableMap.of()).isEmpty()) {
                LOG.warn("Found lookups for tier [%s] in DB, but no nodes discovered for it", tierInDB);
            }
        }
        for (String tier : discoveredLookupTiers) {
            LOG.debug("Starting lookup mgmt for tier [%s].", tier);
            final Map<String, LookupExtractorFactoryMapContainer> tierLookups = allLookupTiers.getOrDefault(tier, ImmutableMap.of());
            for (final HostAndPortWithScheme node : lookupNodeDiscovery.getNodesInTier(tier)) {
                LOG.debug("Starting lookup mgmt for tier [%s] and host [%s:%s:%s].", tier, node.getScheme(), node.getHostText(), node.getPort());
                futures.add(executorService.submit(() -> {
                    try {
                        return new AbstractMap.SimpleImmutableEntry<>(node.getHostAndPort(), doLookupManagementOnNode(node, tierLookups));
                    } catch (InterruptedException ex) {
                        LOG.warn(ex, "lookup management on node [%s:%s:%s] interrupted.", node.getScheme(), node.getHostText(), node.getPort());
                        return null;
                    } catch (Exception ex) {
                        LOG.makeAlert(ex, "Failed to finish lookup management on node [%s:%s:%s]", node.getScheme(), node.getHostText(), node.getPort()).emit();
                        return null;
                    }
                }));
            }
        }
        final ListenableFuture<List<Map.Entry>> allFuture = Futures.allAsList(futures);
        try {
            ImmutableMap.Builder<HostAndPort, LookupsState<LookupExtractorFactoryMapContainer>> stateBuilder = ImmutableMap.builder();
            allFuture.get(lookupCoordinatorManagerConfig.getAllHostTimeout().getMillis(), TimeUnit.MILLISECONDS).stream().filter(Objects::nonNull).forEach(stateBuilder::put);
            knownOldState.set(stateBuilder.build());
        } catch (InterruptedException ex) {
            allFuture.cancel(true);
            Thread.currentThread().interrupt();
            throw ex;
        } catch (Exception ex) {
            allFuture.cancel(true);
            throw ex;
        }
    } catch (Exception ex) {
        LOG.makeAlert(ex, "Failed to finish lookup management loop.").emit();
    }
    LOG.debug("Finished lookup sync for on all nodes.");
}

Example 3 with LookupsState

use of org.apache.druid.query.lookup.LookupsState in project druid by druid-io.

the class LookupCoordinatorResource method getSimpleAllNodeStatus.

/**
 * Build 'simple' lookup cluster status, broken down by tier, host, and then the {@link LookupsState} with
 * the lookup name and version ({@link LookupExtractorFactoryMapContainer#version})
 */
private Map<String, Map<HostAndPort, LookupsState<String>>> getSimpleAllNodeStatus(Map<HostAndPort, LookupsState<LookupExtractorFactoryMapContainer>> lookupsStateOnHosts, Collection<String> tiers) {
    Map<String, Map<HostAndPort, LookupsState<String>>> results = new HashMap<>();
    for (String tier : tiers) {
        Map<HostAndPort, LookupsState<String>> tierNodesStatus = new HashMap<>();
        results.put(tier, tierNodesStatus);
        Collection<HostAndPort> nodes = lookupCoordinatorManager.discoverNodesInTier(tier);
        for (HostAndPort node : nodes) {
            LookupsState<LookupExtractorFactoryMapContainer> lookupsState = lookupsStateOnHosts.get(node);
            if (lookupsState == null) {
                tierNodesStatus.put(node, new LookupsState<>(null, null, null));
            } else {
                Map<String, String> current = lookupsState.getCurrent().entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().getVersion()));
                Map<String, String> toLoad = lookupsState.getToLoad().entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().getVersion()));
                tierNodesStatus.put(node, new LookupsState<>(current, toLoad, lookupsState.getToDrop()));
            }
        }
    }
    return results;
}

Example 4 with LookupsState

use of org.apache.druid.query.lookup.LookupsState in project druid by druid-io.

the class LookupCoordinatorResource method getAllNodesStatus.

@GET
@Produces({ MediaType.APPLICATION_JSON })
@Path("/nodeStatus")
public Response getAllNodesStatus(@QueryParam("discover") boolean discover, @QueryParam("detailed") @Nullable Boolean detailed) {
    boolean full = detailed == null || detailed;
    try {
        Collection<String> tiers;
        if (discover) {
            tiers = lookupCoordinatorManager.discoverTiers();
        } else {
            Map<String, Map<String, LookupExtractorFactoryMapContainer>> configuredLookups = lookupCoordinatorManager.getKnownLookups();
            if (configuredLookups == null) {
                return Response.status(Response.Status.NOT_FOUND).entity(ServletResourceUtils.jsonize("No lookups configured.")).build();
            }
            tiers = configuredLookups.keySet();
        }
        Map<HostAndPort, LookupsState<LookupExtractorFactoryMapContainer>> lookupsStateOnHosts = lookupCoordinatorManager.getLastKnownLookupsStateOnNodes();
        final Map result;
        if (full) {
            result = getDetailedAllNodeStatus(lookupsStateOnHosts, tiers);
        } else {
            // lookups to load per host by version
            result = getSimpleAllNodeStatus(lookupsStateOnHosts, tiers);
        }
        return Response.ok(result).build();
    } catch (Exception ex) {
        LOG.error(ex, "Error getting node status.");
        return Response.serverError().entity(ServletResourceUtils.sanitizeException(ex)).build();
    }
}

Example 5 with LookupsState

use of org.apache.druid.query.lookup.LookupsState in project druid by druid-io.

the class LookupCoordinatorResource method getNodesStatusInTier.

@GET
@Produces({ MediaType.APPLICATION_JSON })
@Path("/nodeStatus/{tier}")
public Response getNodesStatusInTier(@PathParam("tier") String tier) {
    try {
        Map<HostAndPort, LookupsState<LookupExtractorFactoryMapContainer>> lookupsStateOnHosts = lookupCoordinatorManager.getLastKnownLookupsStateOnNodes();
        Map<HostAndPort, LookupsState<LookupExtractorFactoryMapContainer>> tierNodesStatus = new HashMap<>();
        Collection<HostAndPort> nodes = lookupCoordinatorManager.discoverNodesInTier(tier);
        for (HostAndPort node : nodes) {
            LookupsState<LookupExtractorFactoryMapContainer> lookupsState = lookupsStateOnHosts.get(node);
            if (lookupsState == null) {
                tierNodesStatus.put(node, new LookupsState<>(null, null, null));
            } else {
                tierNodesStatus.put(node, lookupsState);
            }
        }
        return Response.ok(tierNodesStatus).build();
    } catch (Exception ex) {
        LOG.error(ex, "Error getting node status for tier [%s].", tier);
        return Response.serverError().entity(ServletResourceUtils.sanitizeException(ex)).build();
    }
}