Examples with RE org.apache.druid.java.util.common.RE used on opensource projects

Example 1 with RE

use of org.apache.druid.java.util.common.RE in project druid by druid-io.

the class LookupCoordinatorResource method getSpecificTier.

@GET
@Produces({ MediaType.APPLICATION_JSON, SmileMediaTypes.APPLICATION_JACKSON_SMILE })
@Path("/config/{tier}")
public Response getSpecificTier(@PathParam("tier") String tier, @DefaultValue("false") @QueryParam("detailed") boolean detailed) {
    try {
        if (Strings.isNullOrEmpty(tier)) {
            return Response.status(Response.Status.BAD_REQUEST).entity(ServletResourceUtils.sanitizeException(new NullPointerException("`tier` required"))).build();
        }
        final Map<String, Map<String, LookupExtractorFactoryMapContainer>> map = lookupCoordinatorManager.getKnownLookups();
        if (map == null) {
            return Response.status(Response.Status.NOT_FOUND).entity(ServletResourceUtils.sanitizeException(new RE("No lookups found"))).build();
        }
        final Map<String, LookupExtractorFactoryMapContainer> tierLookups = map.get(tier);
        if (tierLookups == null) {
            return Response.status(Response.Status.NOT_FOUND).entity(ServletResourceUtils.sanitizeException(new RE("Tier [%s] not found", tier))).build();
        }
        if (detailed) {
            return Response.ok().entity(tierLookups).build();
        } else {
            return Response.ok().entity(tierLookups.keySet()).build();
        }
    } catch (Exception e) {
        LOG.error(e, "Error getting tier [%s]", tier);
        return Response.serverError().entity(ServletResourceUtils.sanitizeException(e)).build();
    }
}

Example 2 with RE

use of org.apache.druid.java.util.common.RE in project druid by druid-io.

the class JettyServerModule method makeAndInitializeServer.

static Server makeAndInitializeServer(Injector injector, Lifecycle lifecycle, DruidNode node, ServerConfig config, TLSServerConfig tlsServerConfig, Binding<SslContextFactory.Server> sslContextFactoryBinding, TLSCertificateChecker certificateChecker) {
    // adjusting to make config.getNumThreads() mean, "number of threads
    // that concurrently handle the requests".
    int numServerThreads = config.getNumThreads() + getMaxJettyAcceptorsSelectorsNum(node);
    final QueuedThreadPool threadPool;
    if (config.getQueueSize() == Integer.MAX_VALUE) {
        threadPool = new QueuedThreadPool();
        threadPool.setMinThreads(numServerThreads);
        threadPool.setMaxThreads(numServerThreads);
    } else {
        threadPool = new QueuedThreadPool(numServerThreads, numServerThreads, // same default is used in other case when threadPool = new QueuedThreadPool()
        60000, new LinkedBlockingQueue<>(config.getQueueSize()));
    }
    threadPool.setDaemon(true);
    jettyServerThreadPool = threadPool;
    final Server server = new Server(threadPool);
    // Without this bean set, the default ScheduledExecutorScheduler runs as non-daemon, causing lifecycle hooks to fail
    // to fire on main exit. Related bug: https://github.com/apache/druid/pull/1627
    server.addBean(new ScheduledExecutorScheduler("JettyScheduler", true), true);
    final List<ServerConnector> serverConnectors = new ArrayList<>();
    if (node.isEnablePlaintextPort()) {
        log.info("Creating http connector with port [%d]", node.getPlaintextPort());
        HttpConfiguration httpConfiguration = new HttpConfiguration();
        if (config.isEnableForwardedRequestCustomizer()) {
            httpConfiguration.addCustomizer(new ForwardedRequestCustomizer());
        }
        httpConfiguration.setRequestHeaderSize(config.getMaxRequestHeaderSize());
        httpConfiguration.setSendServerVersion(false);
        final ServerConnector connector = new ServerConnector(server, new HttpConnectionFactory(httpConfiguration));
        if (node.isBindOnHost()) {
            connector.setHost(node.getHost());
        }
        connector.setPort(node.getPlaintextPort());
        serverConnectors.add(connector);
    }
    final SslContextFactory.Server sslContextFactory;
    if (node.isEnableTlsPort()) {
        log.info("Creating https connector with port [%d]", node.getTlsPort());
        if (sslContextFactoryBinding == null) {
            // Never trust all certificates by default
            sslContextFactory = new IdentityCheckOverrideSslContextFactory(tlsServerConfig, certificateChecker);
            sslContextFactory.setKeyStorePath(tlsServerConfig.getKeyStorePath());
            sslContextFactory.setKeyStoreType(tlsServerConfig.getKeyStoreType());
            sslContextFactory.setKeyStorePassword(tlsServerConfig.getKeyStorePasswordProvider().getPassword());
            sslContextFactory.setCertAlias(tlsServerConfig.getCertAlias());
            sslContextFactory.setKeyManagerFactoryAlgorithm(tlsServerConfig.getKeyManagerFactoryAlgorithm() == null ? KeyManagerFactory.getDefaultAlgorithm() : tlsServerConfig.getKeyManagerFactoryAlgorithm());
            sslContextFactory.setKeyManagerPassword(tlsServerConfig.getKeyManagerPasswordProvider() == null ? null : tlsServerConfig.getKeyManagerPasswordProvider().getPassword());
            if (tlsServerConfig.getIncludeCipherSuites() != null) {
                sslContextFactory.setIncludeCipherSuites(tlsServerConfig.getIncludeCipherSuites().toArray(new String[0]));
            }
            if (tlsServerConfig.getExcludeCipherSuites() != null) {
                sslContextFactory.setExcludeCipherSuites(tlsServerConfig.getExcludeCipherSuites().toArray(new String[0]));
            }
            if (tlsServerConfig.getIncludeProtocols() != null) {
                sslContextFactory.setIncludeProtocols(tlsServerConfig.getIncludeProtocols().toArray(new String[0]));
            }
            if (tlsServerConfig.getExcludeProtocols() != null) {
                sslContextFactory.setExcludeProtocols(tlsServerConfig.getExcludeProtocols().toArray(new String[0]));
            }
            sslContextFactory.setNeedClientAuth(tlsServerConfig.isRequireClientCertificate());
            sslContextFactory.setWantClientAuth(tlsServerConfig.isRequestClientCertificate());
            if (tlsServerConfig.isRequireClientCertificate() || tlsServerConfig.isRequestClientCertificate()) {
                if (tlsServerConfig.getCrlPath() != null) {
                    // setValidatePeerCerts is used just to enable revocation checking using a static CRL file.
                    // Certificate validation is always performed when client certificates are required.
                    sslContextFactory.setValidatePeerCerts(true);
                    sslContextFactory.setCrlPath(tlsServerConfig.getCrlPath());
                }
                if (tlsServerConfig.isValidateHostnames()) {
                    sslContextFactory.setEndpointIdentificationAlgorithm("HTTPS");
                }
                if (tlsServerConfig.getTrustStorePath() != null) {
                    sslContextFactory.setTrustStorePath(tlsServerConfig.getTrustStorePath());
                    sslContextFactory.setTrustStoreType(tlsServerConfig.getTrustStoreType() == null ? KeyStore.getDefaultType() : tlsServerConfig.getTrustStoreType());
                    sslContextFactory.setTrustManagerFactoryAlgorithm(tlsServerConfig.getTrustStoreAlgorithm() == null ? TrustManagerFactory.getDefaultAlgorithm() : tlsServerConfig.getTrustStoreAlgorithm());
                    sslContextFactory.setTrustStorePassword(tlsServerConfig.getTrustStorePasswordProvider() == null ? null : tlsServerConfig.getTrustStorePasswordProvider().getPassword());
                }
            }
        } else {
            sslContextFactory = sslContextFactoryBinding.getProvider().get();
        }
        final HttpConfiguration httpsConfiguration = new HttpConfiguration();
        if (config.isEnableForwardedRequestCustomizer()) {
            httpsConfiguration.addCustomizer(new ForwardedRequestCustomizer());
        }
        httpsConfiguration.setSecureScheme("https");
        httpsConfiguration.setSecurePort(node.getTlsPort());
        httpsConfiguration.addCustomizer(new SecureRequestCustomizer());
        httpsConfiguration.setRequestHeaderSize(config.getMaxRequestHeaderSize());
        httpsConfiguration.setSendServerVersion(false);
        final ServerConnector connector = new ServerConnector(server, new SslConnectionFactory(sslContextFactory, HTTP_1_1_STRING), new HttpConnectionFactory(httpsConfiguration));
        if (node.isBindOnHost()) {
            connector.setHost(node.getHost());
        }
        connector.setPort(node.getTlsPort());
        serverConnectors.add(connector);
    } else {
        sslContextFactory = null;
    }
    final ServerConnector[] connectors = new ServerConnector[serverConnectors.size()];
    int index = 0;
    for (ServerConnector connector : serverConnectors) {
        connectors[index++] = connector;
        connector.setIdleTimeout(Ints.checkedCast(config.getMaxIdleTime().toStandardDuration().getMillis()));
        // workaround suggested in -
        // https://bugs.eclipse.org/bugs/show_bug.cgi?id=435322#c66 for jetty half open connection issues during failovers
        connector.setAcceptorPriorityDelta(-1);
        List<ConnectionFactory> monitoredConnFactories = new ArrayList<>();
        for (ConnectionFactory cf : connector.getConnectionFactories()) {
            // connection factories (in this case HTTP/1.1 after the connection is unencrypted for SSL)
            if (cf.getProtocol().equals(connector.getDefaultProtocol())) {
                monitoredConnFactories.add(new JettyMonitoringConnectionFactory(cf, ACTIVE_CONNECTIONS));
            } else {
                monitoredConnFactories.add(cf);
            }
        }
        connector.setConnectionFactories(monitoredConnFactories);
    }
    server.setConnectors(connectors);
    final long gracefulStop = config.getGracefulShutdownTimeout().toStandardDuration().getMillis();
    if (gracefulStop > 0) {
        server.setStopTimeout(gracefulStop);
    }
    server.addLifeCycleListener(new LifeCycle.Listener() {

        @Override
        public void lifeCycleStarting(LifeCycle event) {
            log.debug("Jetty lifecycle starting [%s]", event.getClass());
        }

        @Override
        public void lifeCycleStarted(LifeCycle event) {
            log.debug("Jetty lifeycle started [%s]", event.getClass());
        }

        @Override
        public void lifeCycleFailure(LifeCycle event, Throwable cause) {
            log.error(cause, "Jetty lifecycle event failed [%s]", event.getClass());
        }

        @Override
        public void lifeCycleStopping(LifeCycle event) {
            log.debug("Jetty lifecycle stopping [%s]", event.getClass());
        }

        @Override
        public void lifeCycleStopped(LifeCycle event) {
            log.debug("Jetty lifecycle stopped [%s]", event.getClass());
        }
    });
    // initialize server
    JettyServerInitializer initializer = injector.getInstance(JettyServerInitializer.class);
    try {
        initializer.initialize(server, injector);
    } catch (Exception e) {
        throw new RE(e, "server initialization exception");
    }
    lifecycle.addHandler(new Lifecycle.Handler() {

        @Override
        public void start() throws Exception {
            log.debug("Starting Jetty Server...");
            server.start();
            if (node.isEnableTlsPort()) {
                // Perform validation
                Preconditions.checkNotNull(sslContextFactory);
                final SSLEngine sslEngine = sslContextFactory.newSSLEngine();
                if (sslEngine.getEnabledCipherSuites() == null || sslEngine.getEnabledCipherSuites().length == 0) {
                    throw new ISE("No supported cipher suites found, supported suites [%s], configured suites include list: [%s] exclude list: [%s]", Arrays.toString(sslEngine.getSupportedCipherSuites()), tlsServerConfig.getIncludeCipherSuites(), tlsServerConfig.getExcludeCipherSuites());
                }
                if (sslEngine.getEnabledProtocols() == null || sslEngine.getEnabledProtocols().length == 0) {
                    throw new ISE("No supported protocols found, supported protocols [%s], configured protocols include list: [%s] exclude list: [%s]", Arrays.toString(sslEngine.getSupportedProtocols()), tlsServerConfig.getIncludeProtocols(), tlsServerConfig.getExcludeProtocols());
                }
            }
        }

        @Override
        public void stop() {
            try {
                final long unannounceDelay = config.getUnannouncePropagationDelay().toStandardDuration().getMillis();
                if (unannounceDelay > 0) {
                    log.info("Sleeping %s ms for unannouncement to propagate.", unannounceDelay);
                    Thread.sleep(unannounceDelay);
                } else {
                    log.debug("Skipping unannounce wait.");
                }
                log.debug("Stopping Jetty Server...");
                server.stop();
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                throw new RE(e, "Interrupted waiting for jetty shutdown.");
            } catch (Exception e) {
                log.warn(e, "Unable to stop Jetty server.");
            }
        }
    }, Lifecycle.Stage.SERVER);
    if (!config.isShowDetailedJettyErrors()) {
        server.setErrorHandler(new ErrorHandler() {

            @Override
            public boolean isShowServlet() {
                return false;
            }

            @Override
            public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException {
                request.setAttribute(RequestDispatcher.ERROR_EXCEPTION, null);
                super.handle(target, baseRequest, request, response);
            }
        });
    }
    return server;
}

Example 3 with RE

use of org.apache.druid.java.util.common.RE in project druid by druid-io.

the class SystemSchema method getThingsFromLeaderNode.

public static <T> JsonParserIterator<T> getThingsFromLeaderNode(String query, TypeReference<T> typeRef, DruidLeaderClient leaderClient, ObjectMapper jsonMapper) {
    Request request;
    InputStreamFullResponseHolder responseHolder;
    try {
        request = leaderClient.makeRequest(HttpMethod.GET, query);
        responseHolder = leaderClient.go(request, new InputStreamFullResponseHandler());
        if (responseHolder.getStatus().getCode() != HttpServletResponse.SC_OK) {
            throw new RE("Failed to talk to leader node at [%s]. Error code[%d], description[%s].", query, responseHolder.getStatus().getCode(), responseHolder.getStatus().getReasonPhrase());
        }
    } catch (IOException | InterruptedException e) {
        throw new RuntimeException(e);
    }
    final JavaType javaType = jsonMapper.getTypeFactory().constructType(typeRef);
    return new JsonParserIterator<>(javaType, Futures.immediateFuture(responseHolder.getContent()), request.getUrl().toString(), null, request.getUrl().getHost(), jsonMapper);
}

Example 4 with RE

use of org.apache.druid.java.util.common.RE in project druid by druid-io.

the class HttpLoadQueuePeon method doSegmentManagement.

private void doSegmentManagement() {
    if (stopped || !mainLoopInProgress.compareAndSet(false, true)) {
        log.trace("[%s]Ignoring tick. Either in-progress already or stopped.", serverId);
        return;
    }
    final int batchSize = config.getHttpLoadQueuePeonBatchSize();
    final List<DataSegmentChangeRequest> newRequests = new ArrayList<>(batchSize);
    synchronized (lock) {
        Iterator<Map.Entry<DataSegment, SegmentHolder>> iter = Iterators.concat(segmentsToDrop.entrySet().iterator(), segmentsToLoad.entrySet().iterator());
        while (newRequests.size() < batchSize && iter.hasNext()) {
            Map.Entry<DataSegment, SegmentHolder> entry = iter.next();
            if (entry.getValue().hasTimedOut()) {
                entry.getValue().requestFailed("timed out");
                iter.remove();
            } else {
                newRequests.add(entry.getValue().getChangeRequest());
            }
        }
    }
    if (newRequests.size() == 0) {
        log.trace("[%s]Found no load/drop requests. SegmentsToLoad[%d], SegmentsToDrop[%d], batchSize[%d].", serverId, segmentsToLoad.size(), segmentsToDrop.size(), config.getHttpLoadQueuePeonBatchSize());
        mainLoopInProgress.set(false);
        return;
    }
    try {
        log.trace("Sending [%d] load/drop requests to Server[%s].", newRequests.size(), serverId);
        BytesAccumulatingResponseHandler responseHandler = new BytesAccumulatingResponseHandler();
        ListenableFuture<InputStream> future = httpClient.go(new Request(HttpMethod.POST, changeRequestURL).addHeader(HttpHeaders.Names.ACCEPT, MediaType.APPLICATION_JSON).addHeader(HttpHeaders.Names.CONTENT_TYPE, MediaType.APPLICATION_JSON).setContent(requestBodyWriter.writeValueAsBytes(newRequests)), responseHandler, new Duration(config.getHttpLoadQueuePeonHostTimeout().getMillis() + 5000));
        Futures.addCallback(future, new FutureCallback<InputStream>() {

            @Override
            public void onSuccess(InputStream result) {
                boolean scheduleNextRunImmediately = true;
                try {
                    if (responseHandler.getStatus() == HttpServletResponse.SC_NO_CONTENT) {
                        log.trace("Received NO CONTENT reseponse from [%s]", serverId);
                    } else if (HttpServletResponse.SC_OK == responseHandler.getStatus()) {
                        try {
                            List<SegmentLoadDropHandler.DataSegmentChangeRequestAndStatus> statuses = jsonMapper.readValue(result, RESPONSE_ENTITY_TYPE_REF);
                            log.trace("Server[%s] returned status response [%s].", serverId, statuses);
                            synchronized (lock) {
                                if (stopped) {
                                    log.trace("Ignoring response from Server[%s]. We are already stopped.", serverId);
                                    scheduleNextRunImmediately = false;
                                    return;
                                }
                                for (SegmentLoadDropHandler.DataSegmentChangeRequestAndStatus e : statuses) {
                                    switch(e.getStatus().getState()) {
                                        case SUCCESS:
                                        case FAILED:
                                            handleResponseStatus(e.getRequest(), e.getStatus());
                                            break;
                                        case PENDING:
                                            log.trace("Request[%s] is still pending on server[%s].", e.getRequest(), serverId);
                                            break;
                                        default:
                                            scheduleNextRunImmediately = false;
                                            log.error("Server[%s] returned unknown state in status[%s].", serverId, e.getStatus());
                                    }
                                }
                            }
                        } catch (Exception ex) {
                            scheduleNextRunImmediately = false;
                            logRequestFailure(ex);
                        }
                    } else {
                        scheduleNextRunImmediately = false;
                        logRequestFailure(new RE("Unexpected Response Status."));
                    }
                } finally {
                    mainLoopInProgress.set(false);
                    if (scheduleNextRunImmediately) {
                        processingExecutor.execute(HttpLoadQueuePeon.this::doSegmentManagement);
                    }
                }
            }

            @Override
            public void onFailure(Throwable t) {
                try {
                    logRequestFailure(t);
                } finally {
                    mainLoopInProgress.set(false);
                }
            }

            private void logRequestFailure(Throwable t) {
                log.error(t, "Request[%s] Failed with status[%s]. Reason[%s].", changeRequestURL, responseHandler.getStatus(), responseHandler.getDescription());
            }
        }, processingExecutor);
    } catch (Throwable th) {
        log.error(th, "Error sending load/drop request to [%s].", serverId);
        mainLoopInProgress.set(false);
    }
}

Example 5 with RE

use of org.apache.druid.java.util.common.RE in project druid by druid-io.

the class ChangeRequestHttpSyncer method sync.

private void sync() {
    if (!startStopLock.awaitStarted(1, TimeUnit.MILLISECONDS)) {
        log.info("Skipping sync() call for server[%s].", logIdentity);
        return;
    }
    lastSyncTime = System.currentTimeMillis();
    try {
        final String req = getRequestString();
        BytesAccumulatingResponseHandler responseHandler = new BytesAccumulatingResponseHandler();
        log.debug("Sending sync request to server[%s]", logIdentity);
        ListenableFuture<InputStream> syncRequestFuture = httpClient.go(new Request(HttpMethod.GET, new URL(baseServerURL, req)).addHeader(HttpHeaders.Names.ACCEPT, SmileMediaTypes.APPLICATION_JACKSON_SMILE).addHeader(HttpHeaders.Names.CONTENT_TYPE, SmileMediaTypes.APPLICATION_JACKSON_SMILE), responseHandler, Duration.millis(serverHttpTimeout));
        log.debug("Sent sync request to [%s]", logIdentity);
        Futures.addCallback(syncRequestFuture, new FutureCallback<InputStream>() {

            @Override
            public void onSuccess(InputStream stream) {
                synchronized (startStopLock) {
                    if (!startStopLock.awaitStarted(1, TimeUnit.MILLISECONDS)) {
                        log.info("Skipping sync() success for server[%s].", logIdentity);
                        return;
                    }
                    try {
                        if (responseHandler.getStatus() == HttpServletResponse.SC_NO_CONTENT) {
                            log.debug("Received NO CONTENT from server[%s]", logIdentity);
                            lastSuccessfulSyncTime = System.currentTimeMillis();
                            return;
                        } else if (responseHandler.getStatus() != HttpServletResponse.SC_OK) {
                            handleFailure(new RE("Bad Sync Response."));
                            return;
                        }
                        log.debug("Received sync response from [%s]", logIdentity);
                        ChangeRequestsSnapshot<T> changes = smileMapper.readValue(stream, responseTypeReferences);
                        log.debug("Finished reading sync response from [%s]", logIdentity);
                        if (changes.isResetCounter()) {
                            log.info("[%s] requested resetCounter for reason [%s].", logIdentity, changes.getResetCause());
                            counter = null;
                            return;
                        }
                        if (counter == null) {
                            listener.fullSync(changes.getRequests());
                        } else {
                            listener.deltaSync(changes.getRequests());
                        }
                        counter = changes.getCounter();
                        if (initializationLatch.getCount() > 0) {
                            initializationLatch.countDown();
                            log.info("[%s] synced successfully for the first time.", logIdentity);
                        }
                        if (consecutiveFailedAttemptCount > 0) {
                            consecutiveFailedAttemptCount = 0;
                            log.info("[%s] synced successfully.", logIdentity);
                        }
                        lastSuccessfulSyncTime = System.currentTimeMillis();
                    } catch (Exception ex) {
                        String logMsg = StringUtils.nonStrictFormat("Error processing sync response from [%s]. Reason [%s]", logIdentity, ex.getMessage());
                        if (incrementFailedAttemptAndCheckUnstabilityTimeout()) {
                            log.error(ex, logMsg);
                        } else {
                            log.info("Temporary Failure. %s", logMsg);
                            log.debug(ex, logMsg);
                        }
                    } finally {
                        addNextSyncToWorkQueue();
                    }
                }
            }

            @Override
            public void onFailure(Throwable t) {
                synchronized (startStopLock) {
                    if (!startStopLock.awaitStarted(1, TimeUnit.MILLISECONDS)) {
                        log.info("Skipping sync() failure for URL[%s].", logIdentity);
                        return;
                    }
                    try {
                        handleFailure(t);
                    } finally {
                        addNextSyncToWorkQueue();
                    }
                }
            }

            private void handleFailure(Throwable t) {
                String logMsg = StringUtils.nonStrictFormat("failed to get sync response from [%s]. Return code [%s], Reason: [%s]", logIdentity, responseHandler.getStatus(), responseHandler.getDescription());
                if (incrementFailedAttemptAndCheckUnstabilityTimeout()) {
                    log.error(t, logMsg);
                } else {
                    log.info("Temporary Failure. %s", logMsg);
                    log.debug(t, logMsg);
                }
            }
        }, executor);
    } catch (Throwable th) {
        try {
            String logMsg = StringUtils.nonStrictFormat("Fatal error while fetching segment list from [%s].", logIdentity);
            if (incrementFailedAttemptAndCheckUnstabilityTimeout()) {
                log.makeAlert(th, logMsg).emit();
            } else {
                log.info("Temporary Failure. %s", logMsg);
                log.debug(th, logMsg);
            }
        } finally {
            addNextSyncToWorkQueue();
        }
    }
}