Examples with IAE io.druid.java.util.common.IAE used on opensource projects

Example 1 with IAE

use of io.druid.java.util.common.IAE in project druid by druid-io.

the class KafkaLookupExtractorFactory method start.

@Override
public boolean start() {
    synchronized (started) {
        if (started.get()) {
            LOG.warn("Already started, not starting again");
            return started.get();
        }
        if (executorService.isShutdown()) {
            LOG.warn("Already shut down, not starting again");
            return false;
        }
        final Properties kafkaProperties = new Properties();
        kafkaProperties.putAll(getKafkaProperties());
        if (kafkaProperties.containsKey("group.id")) {
            throw new IAE("Cannot set kafka property [group.id]. Property is randomly generated for you. Found [%s]", kafkaProperties.getProperty("group.id"));
        }
        if (kafkaProperties.containsKey("auto.offset.reset")) {
            throw new IAE("Cannot set kafka property [auto.offset.reset]. Property will be forced to [smallest]. Found [%s]", kafkaProperties.getProperty("auto.offset.reset"));
        }
        Preconditions.checkNotNull(kafkaProperties.getProperty("zookeeper.connect"), "zookeeper.connect required property");
        kafkaProperties.setProperty("group.id", factoryId);
        final String topic = getKafkaTopic();
        LOG.debug("About to listen to topic [%s] with group.id [%s]", topic, factoryId);
        cacheHandler = cacheManager.createCache();
        final Map<String, String> map = cacheHandler.getCache();
        mapRef.set(map);
        // Enable publish-subscribe
        kafkaProperties.setProperty("auto.offset.reset", "smallest");
        final CountDownLatch startingReads = new CountDownLatch(1);
        final ListenableFuture<?> future = executorService.submit(new Runnable() {

            @Override
            public void run() {
                while (!executorService.isShutdown()) {
                    consumerConnector = buildConnector(kafkaProperties);
                    try {
                        if (executorService.isShutdown()) {
                            break;
                        }
                        final List<KafkaStream<String, String>> streams = consumerConnector.createMessageStreamsByFilter(new Whitelist(Pattern.quote(topic)), 1, DEFAULT_STRING_DECODER, DEFAULT_STRING_DECODER);
                        if (streams == null || streams.isEmpty()) {
                            throw new IAE("Topic [%s] had no streams", topic);
                        }
                        if (streams.size() > 1) {
                            throw new ISE("Topic [%s] has %d streams! expected 1", topic, streams.size());
                        }
                        final KafkaStream<String, String> kafkaStream = streams.get(0);
                        startingReads.countDown();
                        for (final MessageAndMetadata<String, String> messageAndMetadata : kafkaStream) {
                            final String key = messageAndMetadata.key();
                            final String message = messageAndMetadata.message();
                            if (key == null || message == null) {
                                LOG.error("Bad key/message from topic [%s]: [%s]", topic, messageAndMetadata);
                                continue;
                            }
                            doubleEventCount.incrementAndGet();
                            map.put(key, message);
                            doubleEventCount.incrementAndGet();
                            LOG.trace("Placed key[%s] val[%s]", key, message);
                        }
                    } catch (Exception e) {
                        LOG.error(e, "Error reading stream for topic [%s]", topic);
                    } finally {
                        consumerConnector.shutdown();
                    }
                }
            }
        });
        Futures.addCallback(future, new FutureCallback<Object>() {

            @Override
            public void onSuccess(Object result) {
                LOG.debug("Success listening to [%s]", topic);
            }

            @Override
            public void onFailure(Throwable t) {
                if (t instanceof CancellationException) {
                    LOG.debug("Topic [%s] cancelled", topic);
                } else {
                    LOG.error(t, "Error in listening to [%s]", topic);
                }
            }
        }, MoreExecutors.sameThreadExecutor());
        this.future = future;
        final Stopwatch stopwatch = Stopwatch.createStarted();
        try {
            while (!startingReads.await(100, TimeUnit.MILLISECONDS) && connectTimeout > 0L) {
                // Don't return until we have actually connected
                if (future.isDone()) {
                    future.get();
                } else {
                    if (stopwatch.elapsed(TimeUnit.MILLISECONDS) > connectTimeout) {
                        throw new TimeoutException("Failed to connect to kafka in sufficient time");
                    }
                }
            }
        } catch (InterruptedException | ExecutionException | TimeoutException e) {
            executorService.shutdown();
            if (!future.isDone() && !future.cancel(false)) {
                LOG.warn("Could not cancel kafka listening thread");
            }
            LOG.error(e, "Failed to start kafka extraction factory");
            cacheHandler.close();
            return false;
        }
        started.set(true);
        return true;
    }
}

Example 2 with IAE

use of io.druid.java.util.common.IAE in project druid by druid-io.

the class ApproximateHistogramFoldingAggregatorFactory method factorize.

@Override
public Aggregator factorize(ColumnSelectorFactory metricFactory) {
    ObjectColumnSelector selector = metricFactory.makeObjectColumnSelector(fieldName);
    if (selector == null) {
        // gracefully handle undefined metrics
        selector = new ObjectColumnSelector<ApproximateHistogram>() {

            @Override
            public Class<ApproximateHistogram> classOfObject() {
                return ApproximateHistogram.class;
            }

            @Override
            public ApproximateHistogram get() {
                return new ApproximateHistogram(0);
            }
        };
    }
    final Class cls = selector.classOfObject();
    if (cls.equals(Object.class) || ApproximateHistogram.class.isAssignableFrom(cls)) {
        return new ApproximateHistogramFoldingAggregator(selector, resolution, lowerLimit, upperLimit);
    }
    throw new IAE("Incompatible type for metric[%s], expected a ApproximateHistogram, got a %s", fieldName, cls);
}

Example 3 with IAE

use of io.druid.java.util.common.IAE in project druid by druid-io.

the class S3DataSegmentPuller method getSegmentFiles.

public FileUtils.FileCopyResult getSegmentFiles(final S3Coords s3Coords, final File outDir) throws SegmentLoadingException {
    log.info("Pulling index at path[%s] to outDir[%s]", s3Coords, outDir);
    if (!isObjectInBucket(s3Coords)) {
        throw new SegmentLoadingException("IndexFile[%s] does not exist.", s3Coords);
    }
    try {
        org.apache.commons.io.FileUtils.forceMkdir(outDir);
        final URI uri = URI.create(String.format("s3://%s/%s", s3Coords.bucket, s3Coords.path));
        final ByteSource byteSource = new ByteSource() {

            @Override
            public InputStream openStream() throws IOException {
                try {
                    return buildFileObject(uri, s3Client).openInputStream();
                } catch (ServiceException e) {
                    if (e.getCause() != null) {
                        if (S3Utils.S3RETRY.apply(e)) {
                            throw new IOException("Recoverable exception", e);
                        }
                    }
                    throw Throwables.propagate(e);
                }
            }
        };
        if (CompressionUtils.isZip(s3Coords.path)) {
            final FileUtils.FileCopyResult result = CompressionUtils.unzip(byteSource, outDir, S3Utils.S3RETRY, true);
            log.info("Loaded %d bytes from [%s] to [%s]", result.size(), s3Coords.toString(), outDir.getAbsolutePath());
            return result;
        }
        if (CompressionUtils.isGz(s3Coords.path)) {
            final String fname = Files.getNameWithoutExtension(uri.getPath());
            final File outFile = new File(outDir, fname);
            final FileUtils.FileCopyResult result = CompressionUtils.gunzip(byteSource, outFile, S3Utils.S3RETRY);
            log.info("Loaded %d bytes from [%s] to [%s]", result.size(), s3Coords.toString(), outFile.getAbsolutePath());
            return result;
        }
        throw new IAE("Do not know how to load file type at [%s]", uri.toString());
    } catch (Exception e) {
        try {
            org.apache.commons.io.FileUtils.deleteDirectory(outDir);
        } catch (IOException ioe) {
            log.warn(ioe, "Failed to remove output directory [%s] for segment pulled from [%s]", outDir.getAbsolutePath(), s3Coords.toString());
        }
        throw new SegmentLoadingException(e, e.getMessage());
    }
}

Example 4 with IAE

use of io.druid.java.util.common.IAE in project druid by druid-io.

the class HyperLogLogCollector method add.

public void add(byte[] hashedValue) {
    if (hashedValue.length < minBytesRequired) {
        throw new IAE("Insufficient bytes, need[%d] got [%d]", minBytesRequired, hashedValue.length);
    }
    estimatedCardinality = null;
    final ByteBuffer buffer = ByteBuffer.wrap(hashedValue);
    short bucket = (short) (buffer.getShort(hashedValue.length - 2) & bucketMask);
    byte positionOf1 = 0;
    for (int i = 0; i < 8; ++i) {
        byte lookupVal = ByteBitLookup.lookup[UnsignedBytes.toInt(hashedValue[i])];
        switch(lookupVal) {
            case 0:
                positionOf1 += (byte) 8;
                continue;
            default:
                positionOf1 += lookupVal;
                i = 8;
                break;
        }
    }
    add(bucket, positionOf1);
}

Example 5 with IAE

use of io.druid.java.util.common.IAE in project druid by druid-io.

the class Granularity method getDateValues.

// Used by the toDate implementations.
final Integer[] getDateValues(String filePath, Formatter formatter) {
    Pattern pattern = defaultPathPattern;
    switch(formatter) {
        case DEFAULT:
        case LOWER_DEFAULT:
            break;
        case HIVE:
            pattern = hivePathPattern;
            break;
        default:
            throw new IAE("Format %s not supported", formatter);
    }
    Matcher matcher = pattern.matcher(filePath);
    // The size is "7" b/c this array contains standard
    // datetime field values namely:
    // year, monthOfYear, dayOfMonth, hourOfDay, minuteOfHour, secondOfMinute,
    // and index 0 is unused.
    Integer[] vals = new Integer[7];
    if (matcher.matches()) {
        for (int i = 1; i <= matcher.groupCount(); i++) {
            vals[i] = (matcher.group(i) != null) ? Integer.parseInt(matcher.group(i)) : null;
        }
    }
    return vals;
}