Examples with ISE org.apache.druid.java.util.common.ISE used on opensource projects

Example 11 with ISE

use of org.apache.druid.java.util.common.ISE in project druid by druid-io.

the class MetricHolder method fromByteBuffer.

public static MetricHolder fromByteBuffer(ByteBuffer buf) {
    final byte ver = buf.get();
    if (VERSION[0] != ver) {
        throw new ISE("Unknown version[%s] of MetricHolder", ver);
    }
    final String metricName = SERIALIZER_UTILS.readString(buf);
    final String typeName = SERIALIZER_UTILS.readString(buf);
    MetricHolder holder = new MetricHolder(metricName, typeName);
    switch(holder.type) {
        case FLOAT:
            holder.floatType = CompressedColumnarFloatsSupplier.fromByteBuffer(buf, ByteOrder.nativeOrder());
            break;
        case COMPLEX:
            final ComplexMetricSerde serdeForType = ComplexMetrics.getSerdeForType(holder.getTypeName());
            if (serdeForType == null) {
                throw new ISE("Unknown type[%s], cannot load.", holder.getTypeName());
            }
            holder.complexType = read(buf, serdeForType);
            break;
        case LONG:
        case DOUBLE:
            throw new ISE("Unsupported type[%s]", holder.type);
    }
    return holder;
}

Example 12 with ISE

use of org.apache.druid.java.util.common.ISE in project druid by druid-io.

the class TopNQueryRunnerTest method testTopNBySegment.

@Test
public void testTopNBySegment() {
    final HashMap<String, Object> specialContext = new HashMap<String, Object>();
    specialContext.put(QueryContexts.BY_SEGMENT_KEY, "true");
    TopNQuery query = new TopNQueryBuilder().dataSource(QueryRunnerTestHelper.DATA_SOURCE).granularity(QueryRunnerTestHelper.ALL_GRAN).dimension(QueryRunnerTestHelper.MARKET_DIMENSION).metric(QueryRunnerTestHelper.INDEX_METRIC).threshold(4).intervals(QueryRunnerTestHelper.FIRST_TO_THIRD).aggregators(commonAggregators).postAggregators(QueryRunnerTestHelper.ADD_ROWS_INDEX_CONSTANT).context(specialContext).build();
    List<Result<TopNResultValue>> expectedResults = Collections.singletonList(new Result<>(DateTimes.of("2011-04-01T00:00:00.000Z"), new TopNResultValue(Arrays.<Map<String, Object>>asList(ImmutableMap.of("addRowsIndexConstant", 5356.814783D, "index", 5351.814783D, QueryRunnerTestHelper.MARKET_DIMENSION, "total_market", "uniques", QueryRunnerTestHelper.UNIQUES_2, "rows", 4L), ImmutableMap.of("addRowsIndexConstant", 4880.669692D, "index", 4875.669692D, QueryRunnerTestHelper.MARKET_DIMENSION, "upfront", "uniques", QueryRunnerTestHelper.UNIQUES_2, "rows", 4L), ImmutableMap.of("addRowsIndexConstant", 2250.876812D, "index", 2231.876812D, QueryRunnerTestHelper.MARKET_DIMENSION, "spot", "uniques", QueryRunnerTestHelper.UNIQUES_9, "rows", 18L)))));
    Sequence<Result<TopNResultValue>> results = runWithMerge(query);
    List<Result<BySegmentTopNResultValue>> resultList = results.map((Result<TopNResultValue> input) -> {
        // Stupid type erasure
        Object val = input.getValue();
        if (val instanceof BySegmentResultValue) {
            BySegmentResultValue bySegVal = (BySegmentResultValue) val;
            return new Result<>(input.getTimestamp(), new BySegmentTopNResultValue(Lists.transform(bySegVal.getResults(), res -> {
                if (Preconditions.checkNotNull(res) instanceof Result) {
                    Result theResult = (Result) res;
                    Object resVal = theResult.getValue();
                    if (resVal instanceof TopNResultValue) {
                        return new Result<>(theResult.getTimestamp(), (TopNResultValue) resVal);
                    }
                }
                throw new IAE("Bad input: [%s]", res);
            }), bySegVal.getSegmentId(), bySegVal.getInterval()));
        }
        throw new ISE("Bad type");
    }).toList();
    Result<BySegmentTopNResultValue> result = resultList.get(0);
    TestHelper.assertExpectedResults(expectedResults, result.getValue().getResults());
}

Example 13 with ISE

use of org.apache.druid.java.util.common.ISE in project druid by druid-io.

the class JettyServerModule method makeAndInitializeServer.

static Server makeAndInitializeServer(Injector injector, Lifecycle lifecycle, DruidNode node, ServerConfig config, TLSServerConfig tlsServerConfig, Binding<SslContextFactory.Server> sslContextFactoryBinding, TLSCertificateChecker certificateChecker) {
    // adjusting to make config.getNumThreads() mean, "number of threads
    // that concurrently handle the requests".
    int numServerThreads = config.getNumThreads() + getMaxJettyAcceptorsSelectorsNum(node);
    final QueuedThreadPool threadPool;
    if (config.getQueueSize() == Integer.MAX_VALUE) {
        threadPool = new QueuedThreadPool();
        threadPool.setMinThreads(numServerThreads);
        threadPool.setMaxThreads(numServerThreads);
    } else {
        threadPool = new QueuedThreadPool(numServerThreads, numServerThreads, // same default is used in other case when threadPool = new QueuedThreadPool()
        60000, new LinkedBlockingQueue<>(config.getQueueSize()));
    }
    threadPool.setDaemon(true);
    jettyServerThreadPool = threadPool;
    final Server server = new Server(threadPool);
    // Without this bean set, the default ScheduledExecutorScheduler runs as non-daemon, causing lifecycle hooks to fail
    // to fire on main exit. Related bug: https://github.com/apache/druid/pull/1627
    server.addBean(new ScheduledExecutorScheduler("JettyScheduler", true), true);
    final List<ServerConnector> serverConnectors = new ArrayList<>();
    if (node.isEnablePlaintextPort()) {
        log.info("Creating http connector with port [%d]", node.getPlaintextPort());
        HttpConfiguration httpConfiguration = new HttpConfiguration();
        if (config.isEnableForwardedRequestCustomizer()) {
            httpConfiguration.addCustomizer(new ForwardedRequestCustomizer());
        }
        httpConfiguration.setRequestHeaderSize(config.getMaxRequestHeaderSize());
        httpConfiguration.setSendServerVersion(false);
        final ServerConnector connector = new ServerConnector(server, new HttpConnectionFactory(httpConfiguration));
        if (node.isBindOnHost()) {
            connector.setHost(node.getHost());
        }
        connector.setPort(node.getPlaintextPort());
        serverConnectors.add(connector);
    }
    final SslContextFactory.Server sslContextFactory;
    if (node.isEnableTlsPort()) {
        log.info("Creating https connector with port [%d]", node.getTlsPort());
        if (sslContextFactoryBinding == null) {
            // Never trust all certificates by default
            sslContextFactory = new IdentityCheckOverrideSslContextFactory(tlsServerConfig, certificateChecker);
            sslContextFactory.setKeyStorePath(tlsServerConfig.getKeyStorePath());
            sslContextFactory.setKeyStoreType(tlsServerConfig.getKeyStoreType());
            sslContextFactory.setKeyStorePassword(tlsServerConfig.getKeyStorePasswordProvider().getPassword());
            sslContextFactory.setCertAlias(tlsServerConfig.getCertAlias());
            sslContextFactory.setKeyManagerFactoryAlgorithm(tlsServerConfig.getKeyManagerFactoryAlgorithm() == null ? KeyManagerFactory.getDefaultAlgorithm() : tlsServerConfig.getKeyManagerFactoryAlgorithm());
            sslContextFactory.setKeyManagerPassword(tlsServerConfig.getKeyManagerPasswordProvider() == null ? null : tlsServerConfig.getKeyManagerPasswordProvider().getPassword());
            if (tlsServerConfig.getIncludeCipherSuites() != null) {
                sslContextFactory.setIncludeCipherSuites(tlsServerConfig.getIncludeCipherSuites().toArray(new String[0]));
            }
            if (tlsServerConfig.getExcludeCipherSuites() != null) {
                sslContextFactory.setExcludeCipherSuites(tlsServerConfig.getExcludeCipherSuites().toArray(new String[0]));
            }
            if (tlsServerConfig.getIncludeProtocols() != null) {
                sslContextFactory.setIncludeProtocols(tlsServerConfig.getIncludeProtocols().toArray(new String[0]));
            }
            if (tlsServerConfig.getExcludeProtocols() != null) {
                sslContextFactory.setExcludeProtocols(tlsServerConfig.getExcludeProtocols().toArray(new String[0]));
            }
            sslContextFactory.setNeedClientAuth(tlsServerConfig.isRequireClientCertificate());
            sslContextFactory.setWantClientAuth(tlsServerConfig.isRequestClientCertificate());
            if (tlsServerConfig.isRequireClientCertificate() || tlsServerConfig.isRequestClientCertificate()) {
                if (tlsServerConfig.getCrlPath() != null) {
                    // setValidatePeerCerts is used just to enable revocation checking using a static CRL file.
                    // Certificate validation is always performed when client certificates are required.
                    sslContextFactory.setValidatePeerCerts(true);
                    sslContextFactory.setCrlPath(tlsServerConfig.getCrlPath());
                }
                if (tlsServerConfig.isValidateHostnames()) {
                    sslContextFactory.setEndpointIdentificationAlgorithm("HTTPS");
                }
                if (tlsServerConfig.getTrustStorePath() != null) {
                    sslContextFactory.setTrustStorePath(tlsServerConfig.getTrustStorePath());
                    sslContextFactory.setTrustStoreType(tlsServerConfig.getTrustStoreType() == null ? KeyStore.getDefaultType() : tlsServerConfig.getTrustStoreType());
                    sslContextFactory.setTrustManagerFactoryAlgorithm(tlsServerConfig.getTrustStoreAlgorithm() == null ? TrustManagerFactory.getDefaultAlgorithm() : tlsServerConfig.getTrustStoreAlgorithm());
                    sslContextFactory.setTrustStorePassword(tlsServerConfig.getTrustStorePasswordProvider() == null ? null : tlsServerConfig.getTrustStorePasswordProvider().getPassword());
                }
            }
        } else {
            sslContextFactory = sslContextFactoryBinding.getProvider().get();
        }
        final HttpConfiguration httpsConfiguration = new HttpConfiguration();
        if (config.isEnableForwardedRequestCustomizer()) {
            httpsConfiguration.addCustomizer(new ForwardedRequestCustomizer());
        }
        httpsConfiguration.setSecureScheme("https");
        httpsConfiguration.setSecurePort(node.getTlsPort());
        httpsConfiguration.addCustomizer(new SecureRequestCustomizer());
        httpsConfiguration.setRequestHeaderSize(config.getMaxRequestHeaderSize());
        httpsConfiguration.setSendServerVersion(false);
        final ServerConnector connector = new ServerConnector(server, new SslConnectionFactory(sslContextFactory, HTTP_1_1_STRING), new HttpConnectionFactory(httpsConfiguration));
        if (node.isBindOnHost()) {
            connector.setHost(node.getHost());
        }
        connector.setPort(node.getTlsPort());
        serverConnectors.add(connector);
    } else {
        sslContextFactory = null;
    }
    final ServerConnector[] connectors = new ServerConnector[serverConnectors.size()];
    int index = 0;
    for (ServerConnector connector : serverConnectors) {
        connectors[index++] = connector;
        connector.setIdleTimeout(Ints.checkedCast(config.getMaxIdleTime().toStandardDuration().getMillis()));
        // workaround suggested in -
        // https://bugs.eclipse.org/bugs/show_bug.cgi?id=435322#c66 for jetty half open connection issues during failovers
        connector.setAcceptorPriorityDelta(-1);
        List<ConnectionFactory> monitoredConnFactories = new ArrayList<>();
        for (ConnectionFactory cf : connector.getConnectionFactories()) {
            // connection factories (in this case HTTP/1.1 after the connection is unencrypted for SSL)
            if (cf.getProtocol().equals(connector.getDefaultProtocol())) {
                monitoredConnFactories.add(new JettyMonitoringConnectionFactory(cf, ACTIVE_CONNECTIONS));
            } else {
                monitoredConnFactories.add(cf);
            }
        }
        connector.setConnectionFactories(monitoredConnFactories);
    }
    server.setConnectors(connectors);
    final long gracefulStop = config.getGracefulShutdownTimeout().toStandardDuration().getMillis();
    if (gracefulStop > 0) {
        server.setStopTimeout(gracefulStop);
    }
    server.addLifeCycleListener(new LifeCycle.Listener() {

        @Override
        public void lifeCycleStarting(LifeCycle event) {
            log.debug("Jetty lifecycle starting [%s]", event.getClass());
        }

        @Override
        public void lifeCycleStarted(LifeCycle event) {
            log.debug("Jetty lifeycle started [%s]", event.getClass());
        }

        @Override
        public void lifeCycleFailure(LifeCycle event, Throwable cause) {
            log.error(cause, "Jetty lifecycle event failed [%s]", event.getClass());
        }

        @Override
        public void lifeCycleStopping(LifeCycle event) {
            log.debug("Jetty lifecycle stopping [%s]", event.getClass());
        }

        @Override
        public void lifeCycleStopped(LifeCycle event) {
            log.debug("Jetty lifecycle stopped [%s]", event.getClass());
        }
    });
    // initialize server
    JettyServerInitializer initializer = injector.getInstance(JettyServerInitializer.class);
    try {
        initializer.initialize(server, injector);
    } catch (Exception e) {
        throw new RE(e, "server initialization exception");
    }
    lifecycle.addHandler(new Lifecycle.Handler() {

        @Override
        public void start() throws Exception {
            log.debug("Starting Jetty Server...");
            server.start();
            if (node.isEnableTlsPort()) {
                // Perform validation
                Preconditions.checkNotNull(sslContextFactory);
                final SSLEngine sslEngine = sslContextFactory.newSSLEngine();
                if (sslEngine.getEnabledCipherSuites() == null || sslEngine.getEnabledCipherSuites().length == 0) {
                    throw new ISE("No supported cipher suites found, supported suites [%s], configured suites include list: [%s] exclude list: [%s]", Arrays.toString(sslEngine.getSupportedCipherSuites()), tlsServerConfig.getIncludeCipherSuites(), tlsServerConfig.getExcludeCipherSuites());
                }
                if (sslEngine.getEnabledProtocols() == null || sslEngine.getEnabledProtocols().length == 0) {
                    throw new ISE("No supported protocols found, supported protocols [%s], configured protocols include list: [%s] exclude list: [%s]", Arrays.toString(sslEngine.getSupportedProtocols()), tlsServerConfig.getIncludeProtocols(), tlsServerConfig.getExcludeProtocols());
                }
            }
        }

        @Override
        public void stop() {
            try {
                final long unannounceDelay = config.getUnannouncePropagationDelay().toStandardDuration().getMillis();
                if (unannounceDelay > 0) {
                    log.info("Sleeping %s ms for unannouncement to propagate.", unannounceDelay);
                    Thread.sleep(unannounceDelay);
                } else {
                    log.debug("Skipping unannounce wait.");
                }
                log.debug("Stopping Jetty Server...");
                server.stop();
            } catch (InterruptedException e) {
                Thread.currentThread().interrupt();
                throw new RE(e, "Interrupted waiting for jetty shutdown.");
            } catch (Exception e) {
                log.warn(e, "Unable to stop Jetty server.");
            }
        }
    }, Lifecycle.Stage.SERVER);
    if (!config.isShowDetailedJettyErrors()) {
        server.setErrorHandler(new ErrorHandler() {

            @Override
            public boolean isShowServlet() {
                return false;
            }

            @Override
            public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException {
                request.setAttribute(RequestDispatcher.ERROR_EXCEPTION, null);
                super.handle(target, baseRequest, request, response);
            }
        });
    }
    return server;
}

Example 14 with ISE

use of org.apache.druid.java.util.common.ISE in project druid by druid-io.

the class LookupCoordinatorManager method start.

// start() and stop() are synchronized so that they never run in parallel in case of ZK acting funny or druid bug and
// coordinator becomes leader and drops leadership in quick succession.
public void start() {
    synchronized (lifecycleLock) {
        if (!lifecycleLock.canStart()) {
            throw new ISE("LookupCoordinatorManager can't start.");
        }
        try {
            LOG.debug("Starting.");
            if (lookupNodeDiscovery == null) {
                lookupNodeDiscovery = new LookupNodeDiscovery(druidNodeDiscoveryProvider);
            }
            // so that we don't have multiple live executorService instances lying around doing lookup management.
            if (executorService != null && !executorService.awaitTermination(lookupCoordinatorManagerConfig.getHostTimeout().getMillis() * 10, TimeUnit.MILLISECONDS)) {
                throw new ISE("LookupCoordinatorManager executor from last start() hasn't finished. Failed to Start.");
            }
            executorService = MoreExecutors.listeningDecorator(Executors.newScheduledThreadPool(lookupCoordinatorManagerConfig.getThreadPoolSize(), Execs.makeThreadFactory("LookupCoordinatorManager--%s")));
            initializeLookupsConfigWatcher();
            this.backgroundManagerExitedLatch = new CountDownLatch(1);
            this.backgroundManagerFuture = executorService.scheduleWithFixedDelay(this::lookupManagementLoop, lookupCoordinatorManagerConfig.getInitialDelay(), lookupCoordinatorManagerConfig.getPeriod(), TimeUnit.MILLISECONDS);
            Futures.addCallback(backgroundManagerFuture, new FutureCallback<Object>() {

                @Override
                public void onSuccess(@Nullable Object result) {
                    backgroundManagerExitedLatch.countDown();
                    LOG.debug("Exited background lookup manager");
                }

                @Override
                public void onFailure(Throwable t) {
                    backgroundManagerExitedLatch.countDown();
                    if (backgroundManagerFuture.isCancelled()) {
                        LOG.debug("Exited background lookup manager due to cancellation.");
                    } else {
                        LOG.makeAlert(t, "Background lookup manager exited with error!").emit();
                    }
                }
            });
            LOG.debug("Started");
        } catch (Exception ex) {
            LOG.makeAlert(ex, "Got Exception while start()").emit();
        } finally {
            // so that subsequent stop() would happen, even if start() failed with exception
            lifecycleLock.started();
            lifecycleLock.exitStart();
        }
    }
}

Example 15 with ISE

use of org.apache.druid.java.util.common.ISE in project druid by druid-io.

the class JoinableFactoryWrapper method convertJoinsToFilters.

/**
 * Converts any join clauses to filters that can be converted, and returns the rest as-is.
 *
 * See {@link #convertJoinToFilter} for details on the logic.
 */
@VisibleForTesting
static Pair<List<Filter>, List<JoinableClause>> convertJoinsToFilters(final List<JoinableClause> clauses, final Set<String> requiredColumns, final int maxNumFilterValues) {
    final List<Filter> filterList = new ArrayList<>();
    final List<JoinableClause> clausesToUse = new ArrayList<>();
    // Join clauses may depend on other, earlier join clauses.
    // We track that using a Multiset, because we'll need to remove required columns one by one as we convert clauses,
    // and multiple clauses may refer to the same column.
    final Multiset<String> columnsRequiredByJoinClauses = HashMultiset.create();
    for (JoinableClause clause : clauses) {
        for (String column : clause.getCondition().getRequiredColumns()) {
            columnsRequiredByJoinClauses.add(column, 1);
        }
    }
    // Walk through the list of clauses, picking off any from the start of the list that can be converted to filters.
    boolean atStart = true;
    for (JoinableClause clause : clauses) {
        if (atStart) {
            // Remove this clause from columnsRequiredByJoinClauses. It's ok if it relies on itself.
            for (String column : clause.getCondition().getRequiredColumns()) {
                columnsRequiredByJoinClauses.remove(column, 1);
            }
            final Optional<Filter> filter = convertJoinToFilter(clause, Sets.union(requiredColumns, columnsRequiredByJoinClauses.elementSet()), maxNumFilterValues);
            if (filter.isPresent()) {
                filterList.add(filter.get());
            } else {
                clausesToUse.add(clause);
                atStart = false;
            }
        } else {
            clausesToUse.add(clause);
        }
    }
    // Sanity check. If this exception is ever thrown, it's a bug.
    if (filterList.size() + clausesToUse.size() != clauses.size()) {
        throw new ISE("Lost a join clause during planning");
    }
    return Pair.of(filterList, clausesToUse);
}