Examples with Closeable java.io.Closeable used on opensource projects

Example 11 with Closeable

use of java.io.Closeable in project druid by druid-io.

the class GroupByQueryEngineV2 method process.

public static Sequence<Row> process(final GroupByQuery query, final StorageAdapter storageAdapter, final StupidPool<ByteBuffer> intermediateResultsBufferPool, final GroupByQueryConfig config) {
    if (storageAdapter == null) {
        throw new ISE("Null storage adapter found. Probably trying to issue a query against a segment being memory unmapped.");
    }
    final List<Interval> intervals = query.getQuerySegmentSpec().getIntervals();
    if (intervals.size() != 1) {
        throw new IAE("Should only have one interval, got[%s]", intervals);
    }
    final Sequence<Cursor> cursors = storageAdapter.makeCursors(Filters.toFilter(query.getDimFilter()), intervals.get(0), query.getVirtualColumns(), query.getGranularity(), false);
    final ResourceHolder<ByteBuffer> bufferHolder = intermediateResultsBufferPool.take();
    final String fudgeTimestampString = Strings.emptyToNull(query.getContextValue(GroupByStrategyV2.CTX_KEY_FUDGE_TIMESTAMP, ""));
    final DateTime fudgeTimestamp = fudgeTimestampString == null ? null : new DateTime(Long.parseLong(fudgeTimestampString));
    return Sequences.concat(Sequences.withBaggage(Sequences.map(cursors, new Function<Cursor, Sequence<Row>>() {

        @Override
        public Sequence<Row> apply(final Cursor cursor) {
            return new BaseSequence<>(new BaseSequence.IteratorMaker<Row, GroupByEngineIterator>() {

                @Override
                public GroupByEngineIterator make() {
                    ColumnSelectorPlus<GroupByColumnSelectorStrategy>[] selectorPlus = DimensionHandlerUtils.createColumnSelectorPluses(STRATEGY_FACTORY, query.getDimensions(), cursor);
                    return new GroupByEngineIterator(query, config, cursor, bufferHolder.get(), fudgeTimestamp, createGroupBySelectorPlus(selectorPlus));
                }

                @Override
                public void cleanup(GroupByEngineIterator iterFromMake) {
                    iterFromMake.close();
                }
            });
        }
    }), new Closeable() {

        @Override
        public void close() throws IOException {
            CloseQuietly.close(bufferHolder);
        }
    }));
}

Example 12 with Closeable

use of java.io.Closeable in project druid by druid-io.

the class GroupByRowProcessor method process.

public static Sequence<Row> process(final Query queryParam, final Sequence<Row> rows, final Map<String, ValueType> rowSignature, final GroupByQueryConfig config, final GroupByQueryResource resource, final ObjectMapper spillMapper, final String processingTmpDir) {
    final GroupByQuery query = (GroupByQuery) queryParam;
    final GroupByQueryConfig querySpecificConfig = config.withOverrides(query);
    final AggregatorFactory[] aggregatorFactories = new AggregatorFactory[query.getAggregatorSpecs().size()];
    for (int i = 0; i < query.getAggregatorSpecs().size(); i++) {
        aggregatorFactories[i] = query.getAggregatorSpecs().get(i);
    }
    final File temporaryStorageDirectory = new File(processingTmpDir, String.format("druid-groupBy-%s_%s", UUID.randomUUID(), query.getId()));
    final List<Interval> queryIntervals = query.getIntervals();
    final Filter filter = Filters.convertToCNFFromQueryContext(query, Filters.toFilter(query.getDimFilter()));
    final SettableSupplier<Row> rowSupplier = new SettableSupplier<>();
    final RowBasedColumnSelectorFactory columnSelectorFactory = RowBasedColumnSelectorFactory.create(rowSupplier, rowSignature);
    final ValueMatcher filterMatcher = filter == null ? BooleanValueMatcher.of(true) : filter.makeMatcher(columnSelectorFactory);
    final FilteredSequence<Row> filteredSequence = new FilteredSequence<>(rows, new Predicate<Row>() {

        @Override
        public boolean apply(Row input) {
            boolean inInterval = false;
            DateTime rowTime = input.getTimestamp();
            for (Interval queryInterval : queryIntervals) {
                if (queryInterval.contains(rowTime)) {
                    inInterval = true;
                    break;
                }
            }
            if (!inInterval) {
                return false;
            }
            rowSupplier.set(input);
            return filterMatcher.matches();
        }
    });
    return new BaseSequence<>(new BaseSequence.IteratorMaker<Row, CloseableGrouperIterator<RowBasedKey, Row>>() {

        @Override
        public CloseableGrouperIterator<RowBasedKey, Row> make() {
            // This contains all closeable objects which are closed when the returned iterator iterates all the elements,
            // or an exceptions is thrown. The objects are closed in their reverse order.
            final List<Closeable> closeOnExit = Lists.newArrayList();
            try {
                final LimitedTemporaryStorage temporaryStorage = new LimitedTemporaryStorage(temporaryStorageDirectory, querySpecificConfig.getMaxOnDiskStorage());
                closeOnExit.add(temporaryStorage);
                Pair<Grouper<RowBasedKey>, Accumulator<Grouper<RowBasedKey>, Row>> pair = RowBasedGrouperHelper.createGrouperAccumulatorPair(query, true, rowSignature, querySpecificConfig, new Supplier<ByteBuffer>() {

                    @Override
                    public ByteBuffer get() {
                        final ResourceHolder<ByteBuffer> mergeBufferHolder = resource.getMergeBuffer();
                        closeOnExit.add(mergeBufferHolder);
                        return mergeBufferHolder.get();
                    }
                }, -1, temporaryStorage, spillMapper, aggregatorFactories);
                final Grouper<RowBasedKey> grouper = pair.lhs;
                final Accumulator<Grouper<RowBasedKey>, Row> accumulator = pair.rhs;
                closeOnExit.add(grouper);
                final Grouper<RowBasedKey> retVal = filteredSequence.accumulate(grouper, accumulator);
                if (retVal != grouper) {
                    throw GroupByQueryHelper.throwAccumulationResourceLimitExceededException();
                }
                return RowBasedGrouperHelper.makeGrouperIterator(grouper, query, new Closeable() {

                    @Override
                    public void close() throws IOException {
                        for (Closeable closeable : Lists.reverse(closeOnExit)) {
                            CloseQuietly.close(closeable);
                        }
                    }
                });
            } catch (Throwable e) {
                // Exception caught while setting up the iterator; release resources.
                for (Closeable closeable : Lists.reverse(closeOnExit)) {
                    CloseQuietly.close(closeable);
                }
                throw e;
            }
        }

        @Override
        public void cleanup(CloseableGrouperIterator<RowBasedKey, Row> iterFromMake) {
            iterFromMake.close();
        }
    });
}

Example 13 with Closeable

use of java.io.Closeable in project druid by druid-io.

the class QueryableIndexIndexableAdapter method getRows.

@Override
public Iterable<Rowboat> getRows() {
    return new Iterable<Rowboat>() {

        @Override
        public Iterator<Rowboat> iterator() {
            return new Iterator<Rowboat>() {

                final GenericColumn timestamps = input.getColumn(Column.TIME_COLUMN_NAME).getGenericColumn();

                final Closeable[] metrics;

                final Closeable[] columns;

                final Closer closer = Closer.create();

                final int numMetrics = getMetricNames().size();

                final DimensionHandler[] handlers = new DimensionHandler[availableDimensions.size()];

                Collection<DimensionHandler> handlerSet = input.getDimensionHandlers().values();

                int currRow = 0;

                boolean done = false;

                {
                    closer.register(timestamps);
                    handlerSet.toArray(handlers);
                    this.columns = FluentIterable.from(handlerSet).transform(new Function<DimensionHandler, Closeable>() {

                        @Override
                        public Closeable apply(DimensionHandler handler) {
                            Column column = input.getColumn(handler.getDimensionName());
                            return handler.getSubColumn(column);
                        }
                    }).toArray(Closeable.class);
                    for (Closeable column : columns) {
                        closer.register(column);
                    }
                    final Indexed<String> availableMetrics = getMetricNames();
                    metrics = new Closeable[availableMetrics.size()];
                    for (int i = 0; i < metrics.length; ++i) {
                        final Column column = input.getColumn(availableMetrics.get(i));
                        final ValueType type = column.getCapabilities().getType();
                        switch(type) {
                            case FLOAT:
                            case LONG:
                                metrics[i] = column.getGenericColumn();
                                break;
                            case COMPLEX:
                                metrics[i] = column.getComplexColumn();
                                break;
                            default:
                                throw new ISE("Cannot handle type[%s]", type);
                        }
                    }
                    for (Closeable metricColumn : metrics) {
                        closer.register(metricColumn);
                    }
                }

                @Override
                public boolean hasNext() {
                    final boolean hasNext = currRow < numRows;
                    if (!hasNext && !done) {
                        CloseQuietly.close(closer);
                        done = true;
                    }
                    return hasNext;
                }

                @Override
                public Rowboat next() {
                    if (!hasNext()) {
                        throw new NoSuchElementException();
                    }
                    final Object[] dims = new Object[columns.length];
                    int dimIndex = 0;
                    for (final Closeable column : columns) {
                        dims[dimIndex] = handlers[dimIndex].getEncodedKeyComponentFromColumn(column, currRow);
                        dimIndex++;
                    }
                    Object[] metricArray = new Object[numMetrics];
                    for (int i = 0; i < metricArray.length; ++i) {
                        if (metrics[i] instanceof IndexedFloatsGenericColumn) {
                            metricArray[i] = ((GenericColumn) metrics[i]).getFloatSingleValueRow(currRow);
                        } else if (metrics[i] instanceof IndexedLongsGenericColumn) {
                            metricArray[i] = ((GenericColumn) metrics[i]).getLongSingleValueRow(currRow);
                        } else if (metrics[i] instanceof ComplexColumn) {
                            metricArray[i] = ((ComplexColumn) metrics[i]).getRowValue(currRow);
                        }
                    }
                    final Rowboat retVal = new Rowboat(timestamps.getLongSingleValueRow(currRow), dims, metricArray, currRow, handlers);
                    ++currRow;
                    return retVal;
                }

                @Override
                public void remove() {
                    throw new UnsupportedOperationException();
                }
            };
        }
    };
}

Example 14 with Closeable

use of java.io.Closeable in project druid by druid-io.

the class StringDimensionMergerLegacy method writeIndexes.

@Override
public void writeIndexes(List<IntBuffer> segmentRowNumConversions, Closer closer) throws IOException {
    final SerializerUtils serializerUtils = new SerializerUtils();
    long dimStartTime = System.currentTimeMillis();
    final BitmapSerdeFactory bitmapSerdeFactory = indexSpec.getBitmapSerdeFactory();
    String bmpFilename = String.format("%s.inverted", dimensionName);
    bitmapWriter = new GenericIndexedWriter<>(ioPeon, bmpFilename, bitmapSerdeFactory.getObjectStrategy());
    bitmapWriter.open();
    final MappedByteBuffer dimValsMapped = Files.map(dictionaryFile);
    closer.register(new Closeable() {

        @Override
        public void close() throws IOException {
            ByteBufferUtils.unmap(dimValsMapped);
        }
    });
    if (!dimensionName.equals(serializerUtils.readString(dimValsMapped))) {
        throw new ISE("dimensions[%s] didn't equate!?  This is a major WTF moment.", dimensionName);
    }
    Indexed<String> dimVals = GenericIndexed.read(dimValsMapped, GenericIndexed.STRING_STRATEGY);
    log.info("Starting dimension[%s] with cardinality[%,d]", dimensionName, dimVals.size());
    final BitmapFactory bmpFactory = bitmapSerdeFactory.getBitmapFactory();
    RTree tree = null;
    spatialWriter = null;
    boolean hasSpatial = capabilities.hasSpatialIndexes();
    if (hasSpatial) {
        String spatialFilename = String.format("%s.spatial", dimensionName);
        spatialWriter = new ByteBufferWriter<>(ioPeon, spatialFilename, new IndexedRTree.ImmutableRTreeObjectStrategy(bmpFactory));
        spatialWriter.open();
        tree = new RTree(2, new LinearGutmanSplitStrategy(0, 50, bmpFactory), bmpFactory);
    }
    IndexSeeker[] dictIdSeeker = toIndexSeekers(adapters, dimConversions, dimensionName);
    //Iterate all dim values's dictionary id in ascending order which in line with dim values's compare result.
    for (int dictId = 0; dictId < dimVals.size(); dictId++) {
        progress.progress();
        mergeBitmaps(segmentRowNumConversions, dimVals, bmpFactory, tree, hasSpatial, dictIdSeeker, dictId, adapters, dimensionName, nullRowsBitmap, bitmapWriter);
    }
    log.info("Completed dimension[%s] in %,d millis.", dimensionName, System.currentTimeMillis() - dimStartTime);
    if (hasSpatial) {
        spatialWriter.write(ImmutableRTree.newImmutableFromMutable(tree));
    }
}

Example 15 with Closeable

use of java.io.Closeable in project druid by druid-io.

the class GenericIndexed method fromIterable.

public static <T> GenericIndexed<T> fromIterable(Iterable<T> objectsIterable, ObjectStrategy<T> strategy) {
    Iterator<T> objects = objectsIterable.iterator();
    if (!objects.hasNext()) {
        final ByteBuffer buffer = ByteBuffer.allocate(4).putInt(0);
        buffer.flip();
        return new GenericIndexed<T>(buffer, strategy, true);
    }
    boolean allowReverseLookup = true;
    int count = 0;
    ByteArrayOutputStream headerBytes = new ByteArrayOutputStream();
    ByteArrayOutputStream valueBytes = new ByteArrayOutputStream();
    try {
        int offset = 0;
        T prevVal = null;
        do {
            count++;
            T next = objects.next();
            if (allowReverseLookup && prevVal != null && !(strategy.compare(prevVal, next) < 0)) {
                allowReverseLookup = false;
            }
            final byte[] bytes = strategy.toBytes(next);
            offset += 4 + bytes.length;
            headerBytes.write(Ints.toByteArray(offset));
            valueBytes.write(Ints.toByteArray(bytes.length));
            valueBytes.write(bytes);
            if (prevVal instanceof Closeable) {
                CloseQuietly.close((Closeable) prevVal);
            }
            prevVal = next;
        } while (objects.hasNext());
        if (prevVal instanceof Closeable) {
            CloseQuietly.close((Closeable) prevVal);
        }
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    ByteBuffer theBuffer = ByteBuffer.allocate(Ints.BYTES + headerBytes.size() + valueBytes.size());
    theBuffer.put(Ints.toByteArray(count));
    theBuffer.put(headerBytes.toByteArray());
    theBuffer.put(valueBytes.toByteArray());
    theBuffer.flip();
    return new GenericIndexed<T>(theBuffer.asReadOnlyBuffer(), strategy, allowReverseLookup);
}