Examples with IAE io.druid.java.util.common.IAE used on opensource projects

Example 26 with IAE

use of io.druid.java.util.common.IAE in project druid by druid-io.

the class SearchQueryQueryToolChest method getCacheStrategy.

@Override
public CacheStrategy<Result<SearchResultValue>, Object, SearchQuery> getCacheStrategy(final SearchQuery query) {
    return new CacheStrategy<Result<SearchResultValue>, Object, SearchQuery>() {

        private final List<DimensionSpec> dimensionSpecs = query.getDimensions() != null ? query.getDimensions() : Collections.<DimensionSpec>emptyList();

        private final List<String> dimOutputNames = dimensionSpecs.size() > 0 ? Lists.transform(dimensionSpecs, new Function<DimensionSpec, String>() {

            @Override
            public String apply(DimensionSpec input) {
                return input.getOutputName();
            }
        }) : Collections.<String>emptyList();

        @Override
        public boolean isCacheable(SearchQuery query, boolean willMergeRunners) {
            return true;
        }

        @Override
        public byte[] computeCacheKey(SearchQuery query) {
            final DimFilter dimFilter = query.getDimensionsFilter();
            final byte[] filterBytes = dimFilter == null ? new byte[] {} : dimFilter.getCacheKey();
            final byte[] querySpecBytes = query.getQuery().getCacheKey();
            final byte[] granularityBytes = query.getGranularity().getCacheKey();
            final List<DimensionSpec> dimensionSpecs = query.getDimensions() != null ? query.getDimensions() : Collections.<DimensionSpec>emptyList();
            final byte[][] dimensionsBytes = new byte[dimensionSpecs.size()][];
            int dimensionsBytesSize = 0;
            int index = 0;
            for (DimensionSpec dimensionSpec : dimensionSpecs) {
                dimensionsBytes[index] = dimensionSpec.getCacheKey();
                dimensionsBytesSize += dimensionsBytes[index].length;
                ++index;
            }
            final byte[] sortSpecBytes = query.getSort().getCacheKey();
            final ByteBuffer queryCacheKey = ByteBuffer.allocate(1 + 4 + granularityBytes.length + filterBytes.length + querySpecBytes.length + dimensionsBytesSize + sortSpecBytes.length).put(SEARCH_QUERY).put(Ints.toByteArray(query.getLimit())).put(granularityBytes).put(filterBytes).put(querySpecBytes).put(sortSpecBytes);
            for (byte[] bytes : dimensionsBytes) {
                queryCacheKey.put(bytes);
            }
            return queryCacheKey.array();
        }

        @Override
        public TypeReference<Object> getCacheObjectClazz() {
            return OBJECT_TYPE_REFERENCE;
        }

        @Override
        public Function<Result<SearchResultValue>, Object> prepareForCache() {
            return new Function<Result<SearchResultValue>, Object>() {

                @Override
                public Object apply(Result<SearchResultValue> input) {
                    return dimensionSpecs.size() > 0 ? Lists.newArrayList(input.getTimestamp().getMillis(), input.getValue(), dimOutputNames) : Lists.newArrayList(input.getTimestamp().getMillis(), input.getValue());
                }
            };
        }

        @Override
        public Function<Object, Result<SearchResultValue>> pullFromCache() {
            return new Function<Object, Result<SearchResultValue>>() {

                @Override
                @SuppressWarnings("unchecked")
                public Result<SearchResultValue> apply(Object input) {
                    List<Object> result = (List<Object>) input;
                    boolean needsRename = false;
                    final Map<String, String> outputNameMap = Maps.newHashMap();
                    if (hasOutputName(result)) {
                        List<String> cachedOutputNames = (List) result.get(2);
                        Preconditions.checkArgument(cachedOutputNames.size() == dimOutputNames.size(), "cache hit, but number of dimensions mismatch");
                        needsRename = false;
                        for (int idx = 0; idx < cachedOutputNames.size(); idx++) {
                            String cachedOutputName = cachedOutputNames.get(idx);
                            String outputName = dimOutputNames.get(idx);
                            if (!cachedOutputName.equals(outputName)) {
                                needsRename = true;
                            }
                            outputNameMap.put(cachedOutputName, outputName);
                        }
                    }
                    return !needsRename ? new Result<>(new DateTime(((Number) result.get(0)).longValue()), new SearchResultValue(Lists.transform((List) result.get(1), new Function<Object, SearchHit>() {

                        @Override
                        public SearchHit apply(@Nullable Object input) {
                            if (input instanceof Map) {
                                return new SearchHit((String) ((Map) input).get("dimension"), (String) ((Map) input).get("value"), (Integer) ((Map) input).get("count"));
                            } else if (input instanceof SearchHit) {
                                return (SearchHit) input;
                            } else {
                                throw new IAE("Unknown format [%s]", input.getClass());
                            }
                        }
                    }))) : new Result<>(new DateTime(((Number) result.get(0)).longValue()), new SearchResultValue(Lists.transform((List) result.get(1), new Function<Object, SearchHit>() {

                        @Override
                        public SearchHit apply(@Nullable Object input) {
                            String dim = null;
                            String val = null;
                            Integer cnt = null;
                            if (input instanceof Map) {
                                dim = outputNameMap.get((String) ((Map) input).get("dimension"));
                                val = (String) ((Map) input).get("value");
                                cnt = (Integer) ((Map) input).get("count");
                            } else if (input instanceof SearchHit) {
                                SearchHit cached = (SearchHit) input;
                                dim = outputNameMap.get(cached.getDimension());
                                val = cached.getValue();
                                cnt = cached.getCount();
                            } else {
                                throw new IAE("Unknown format [%s]", input.getClass());
                            }
                            return new SearchHit(dim, val, cnt);
                        }
                    })));
                }
            };
        }

        private boolean hasOutputName(List<Object> cachedEntry) {
            /*
         * cached entry is list of two or three objects
         *  1. timestamp
         *  2. SearchResultValue
         *  3. outputName of each dimension (optional)
         *
         * if a cached entry has three objects, dimension name of SearchResultValue should be check if rename is needed
         */
            return cachedEntry.size() == 3;
        }
    };
}

Example 27 with IAE

use of io.druid.java.util.common.IAE in project druid by druid-io.

the class IndexMerger method persist.

public File persist(final IncrementalIndex index, final Interval dataInterval, File outDir, IndexSpec indexSpec, ProgressIndicator progress) throws IOException {
    if (index.isEmpty()) {
        throw new IAE("Trying to persist an empty index!");
    }
    final long firstTimestamp = index.getMinTime().getMillis();
    final long lastTimestamp = index.getMaxTime().getMillis();
    if (!(dataInterval.contains(firstTimestamp) && dataInterval.contains(lastTimestamp))) {
        throw new IAE("interval[%s] does not encapsulate the full range of timestamps[%s, %s]", dataInterval, new DateTime(firstTimestamp), new DateTime(lastTimestamp));
    }
    FileUtils.forceMkdir(outDir);
    log.info("Starting persist for interval[%s], rows[%,d]", dataInterval, index.size());
    return merge(Arrays.<IndexableAdapter>asList(new IncrementalIndexAdapter(dataInterval, index, indexSpec.getBitmapSerdeFactory().getBitmapFactory())), //                     while merging a single iterable
    false, index.getMetricAggs(), outDir, indexSpec, progress);
}

Example 28 with IAE

use of io.druid.java.util.common.IAE in project druid by druid-io.

the class IndexMerger method merge.

public File merge(List<IndexableAdapter> indexes, final boolean rollup, final AggregatorFactory[] metricAggs, File outDir, IndexSpec indexSpec, ProgressIndicator progress) throws IOException {
    FileUtils.deleteDirectory(outDir);
    FileUtils.forceMkdir(outDir);
    final List<String> mergedDimensions = getMergedDimensions(indexes);
    final List<String> mergedMetrics = Lists.transform(mergeIndexed(Lists.newArrayList(FunctionalIterable.create(indexes).transform(new Function<IndexableAdapter, Iterable<String>>() {

        @Override
        public Iterable<String> apply(@Nullable IndexableAdapter input) {
            return input.getMetricNames();
        }
    }))), new Function<String, String>() {

        @Override
        public String apply(@Nullable String input) {
            return input;
        }
    });
    final AggregatorFactory[] sortedMetricAggs = new AggregatorFactory[mergedMetrics.size()];
    for (int i = 0; i < metricAggs.length; i++) {
        AggregatorFactory metricAgg = metricAggs[i];
        int metricIndex = mergedMetrics.indexOf(metricAgg.getName());
        /*
        If metricIndex is negative, one of the metricAggs was not present in the union of metrics from the indices
        we are merging
       */
        if (metricIndex > -1) {
            sortedMetricAggs[metricIndex] = metricAgg;
        }
    }
    /*
      If there is nothing at sortedMetricAggs[i], then we did not have a metricAgg whose name matched the name
      of the ith element of mergedMetrics. I.e. There was a metric in the indices to merge that we did not ask for.
     */
    for (int i = 0; i < sortedMetricAggs.length; i++) {
        if (sortedMetricAggs[i] == null) {
            throw new IAE("Indices to merge contained metric[%s], but requested metrics did not", mergedMetrics.get(i));
        }
    }
    for (int i = 0; i < mergedMetrics.size(); i++) {
        if (!sortedMetricAggs[i].getName().equals(mergedMetrics.get(i))) {
            throw new IAE("Metric mismatch, index[%d] [%s] != [%s]", i, sortedMetricAggs[i].getName(), mergedMetrics.get(i));
        }
    }
    Function<ArrayList<Iterable<Rowboat>>, Iterable<Rowboat>> rowMergerFn = new Function<ArrayList<Iterable<Rowboat>>, Iterable<Rowboat>>() {

        @Override
        public Iterable<Rowboat> apply(@Nullable ArrayList<Iterable<Rowboat>> boats) {
            if (rollup) {
                return CombiningIterable.create(new MergeIterable<Rowboat>(Ordering.<Rowboat>natural().nullsFirst(), boats), Ordering.<Rowboat>natural().nullsFirst(), new RowboatMergeFunction(sortedMetricAggs));
            } else {
                return new MergeIterable<Rowboat>(new Ordering<Rowboat>() {

                    @Override
                    public int compare(Rowboat left, Rowboat right) {
                        return Longs.compare(left.getTimestamp(), right.getTimestamp());
                    }
                }.nullsFirst(), boats);
            }
        }
    };
    return makeIndexFiles(indexes, sortedMetricAggs, outDir, progress, mergedDimensions, mergedMetrics, rowMergerFn, indexSpec);
}

Example 29 with IAE

use of io.druid.java.util.common.IAE in project druid by druid-io.

the class CompressedVSizeIndexedSupplier method fromByteBuffer.

public static CompressedVSizeIndexedSupplier fromByteBuffer(ByteBuffer buffer, ByteOrder order, SmooshedFileMapper fileMapper) {
    byte versionFromBuffer = buffer.get();
    if (versionFromBuffer == version) {
        CompressedVSizeIntsIndexedSupplier offsetSupplier = CompressedVSizeIntsIndexedSupplier.fromByteBuffer(buffer, order, fileMapper);
        CompressedVSizeIntsIndexedSupplier valueSupplier = CompressedVSizeIntsIndexedSupplier.fromByteBuffer(buffer, order, fileMapper);
        return new CompressedVSizeIndexedSupplier(offsetSupplier, valueSupplier);
    }
    throw new IAE("Unknown version[%s]", versionFromBuffer);
}

Example 30 with IAE

use of io.druid.java.util.common.IAE in project druid by druid-io.

the class URIExtractionNamespaceCacheFactory method populateCache.

@Override
@Nullable
public CacheScheduler.VersionedCache populateCache(final URIExtractionNamespace extractionNamespace, final CacheScheduler.EntryImpl<URIExtractionNamespace> entryId, @Nullable final String lastVersion, final CacheScheduler scheduler) throws Exception {
    final boolean doSearch = extractionNamespace.getUriPrefix() != null;
    final URI originalUri = doSearch ? extractionNamespace.getUriPrefix() : extractionNamespace.getUri();
    final SearchableVersionedDataFinder<URI> pullerRaw = pullers.get(originalUri.getScheme());
    if (pullerRaw == null) {
        throw new IAE("Unknown loader type[%s].  Known types are %s", originalUri.getScheme(), pullers.keySet());
    }
    if (!(pullerRaw instanceof URIDataPuller)) {
        throw new IAE("Cannot load data from location [%s]. Data pulling from [%s] not supported", originalUri, originalUri.getScheme());
    }
    final URIDataPuller puller = (URIDataPuller) pullerRaw;
    final URI uri;
    if (doSearch) {
        final Pattern versionRegex;
        if (extractionNamespace.getFileRegex() != null) {
            versionRegex = Pattern.compile(extractionNamespace.getFileRegex());
        } else {
            versionRegex = null;
        }
        uri = pullerRaw.getLatestVersion(extractionNamespace.getUriPrefix(), versionRegex);
        if (uri == null) {
            throw new FileNotFoundException(String.format("Could not find match for pattern `%s` in [%s] for %s", versionRegex, originalUri, extractionNamespace));
        }
    } else {
        uri = extractionNamespace.getUri();
    }
    final String uriPath = uri.getPath();
    return RetryUtils.retry(new Callable<CacheScheduler.VersionedCache>() {

        @Override
        public CacheScheduler.VersionedCache call() throws Exception {
            final String version = puller.getVersion(uri);
            try {
                // Important to call equals() against version because lastVersion could be null
                if (version.equals(lastVersion)) {
                    log.debug("URI [%s] for [%s] has the same last modified time [%s] as the last cached. " + "Skipping ", uri.toString(), entryId, version);
                    return null;
                }
            } catch (NumberFormatException ex) {
                log.debug(ex, "Failed to get last modified timestamp. Assuming no timestamp");
            }
            final ByteSource source;
            if (CompressionUtils.isGz(uriPath)) {
                // Simple gzip stream
                log.debug("Loading gz");
                source = new ByteSource() {

                    @Override
                    public InputStream openStream() throws IOException {
                        return CompressionUtils.gzipInputStream(puller.getInputStream(uri));
                    }
                };
            } else {
                source = new ByteSource() {

                    @Override
                    public InputStream openStream() throws IOException {
                        return puller.getInputStream(uri);
                    }
                };
            }
            CacheScheduler.VersionedCache versionedCache = scheduler.createVersionedCache(entryId, version);
            try {
                final MapPopulator.PopulateResult populateResult = new MapPopulator<>(extractionNamespace.getNamespaceParseSpec().getParser()).populate(source, versionedCache.getCache());
                log.info("Finished loading %,d values from %,d lines for [%s]", populateResult.getEntries(), populateResult.getLines(), entryId);
                return versionedCache;
            } catch (Throwable t) {
                try {
                    versionedCache.close();
                } catch (Exception e) {
                    t.addSuppressed(e);
                }
                throw t;
            }
        }
    }, puller.shouldRetryPredicate(), DEFAULT_NUM_RETRIES);
}