Examples with Closer org.apache.druid.java.util.common.io.Closer used on opensource projects

Example 1 with Closer

use of org.apache.druid.java.util.common.io.Closer in project druid by druid-io.

the class AggregationTestHelper method createGroupByQueryAggregationTestHelper.

public static AggregationTestHelper createGroupByQueryAggregationTestHelper(List<? extends Module> jsonModulesToRegister, GroupByQueryConfig config, TemporaryFolder tempFolder) {
    final ObjectMapper mapper = TestHelper.makeJsonMapper();
    final Pair<GroupByQueryRunnerFactory, Closer> factoryAndCloser = GroupByQueryRunnerTest.makeQueryRunnerFactory(mapper, config);
    final GroupByQueryRunnerFactory factory = factoryAndCloser.lhs;
    final Closer closer = factoryAndCloser.rhs;
    IndexIO indexIO = new IndexIO(mapper, new ColumnConfig() {

        @Override
        public int columnCacheSizeBytes() {
            return 0;
        }
    });
    return new AggregationTestHelper(mapper, new IndexMergerV9(mapper, indexIO, OffHeapMemorySegmentWriteOutMediumFactory.instance()), indexIO, factory.getToolchest(), factory, tempFolder, jsonModulesToRegister, closer, Collections.emptyMap());
}

Example 2 with Closer

use of org.apache.druid.java.util.common.io.Closer in project druid by druid-io.

the class CuratorDruidNodeDiscoveryProvider method stop.

@LifecycleStop
public void stop() throws IOException {
    if (!lifecycleLock.canStop()) {
        throw new ISE("can't stop.");
    }
    log.debug("Stopping.");
    Closer closer = Closer.create();
    closer.registerAll(nodeRoleWatchers.values());
    closer.registerAll(nodeDiscoverers);
    CloseableUtils.closeAll(closer, listenerExecutor::shutdownNow);
}

Example 3 with Closer

use of org.apache.druid.java.util.common.io.Closer in project druid by druid-io.

the class IndexedTableJoinable method getCorrelatedColumnValues.

@Override
public Optional<Set<String>> getCorrelatedColumnValues(String searchColumnName, String searchColumnValue, String retrievalColumnName, long maxCorrelationSetSize, boolean allowNonKeyColumnSearch) {
    int filterColumnPosition = table.rowSignature().indexOf(searchColumnName);
    int correlatedColumnPosition = table.rowSignature().indexOf(retrievalColumnName);
    if (filterColumnPosition < 0 || correlatedColumnPosition < 0) {
        return Optional.empty();
    }
    try (final Closer closer = Closer.create()) {
        Set<String> correlatedValues = new HashSet<>();
        if (table.keyColumns().contains(searchColumnName)) {
            IndexedTable.Index index = table.columnIndex(filterColumnPosition);
            IndexedTable.Reader reader = table.columnReader(correlatedColumnPosition);
            closer.register(reader);
            IntList rowIndex = index.find(searchColumnValue);
            for (int i = 0; i < rowIndex.size(); i++) {
                int rowNum = rowIndex.getInt(i);
                String correlatedDimVal = DimensionHandlerUtils.convertObjectToString(reader.read(rowNum));
                correlatedValues.add(correlatedDimVal);
                if (correlatedValues.size() > maxCorrelationSetSize) {
                    return Optional.empty();
                }
            }
            return Optional.of(correlatedValues);
        } else {
            if (!allowNonKeyColumnSearch) {
                return Optional.empty();
            }
            IndexedTable.Reader dimNameReader = table.columnReader(filterColumnPosition);
            IndexedTable.Reader correlatedColumnReader = table.columnReader(correlatedColumnPosition);
            closer.register(dimNameReader);
            closer.register(correlatedColumnReader);
            for (int i = 0; i < table.numRows(); i++) {
                String dimVal = Objects.toString(dimNameReader.read(i), null);
                if (searchColumnValue.equals(dimVal)) {
                    String correlatedDimVal = DimensionHandlerUtils.convertObjectToString(correlatedColumnReader.read(i));
                    correlatedValues.add(correlatedDimVal);
                    if (correlatedValues.size() > maxCorrelationSetSize) {
                        return Optional.empty();
                    }
                }
            }
            return Optional.of(correlatedValues);
        }
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}

Example 4 with Closer

use of org.apache.druid.java.util.common.io.Closer in project druid by druid-io.

the class StreamAppenderator method mergeAndPush.

/**
 * Merge segment, push to deep storage. Should only be used on segments that have been fully persisted. Must only
 * be run in the single-threaded pushExecutor.
 *
 * @param identifier    sink identifier
 * @param sink          sink to push
 * @param useUniquePath true if the segment should be written to a path with a unique identifier
 *
 * @return segment descriptor, or null if the sink is no longer valid
 */
@Nullable
private DataSegment mergeAndPush(final SegmentIdWithShardSpec identifier, final Sink sink, final boolean useUniquePath) {
    // noinspection ObjectEquality
    if (sinks.get(identifier) != sink) {
        log.warn("Sink for segment[%s] no longer valid, bailing out of mergeAndPush.", identifier);
        return null;
    }
    // Use a descriptor file to indicate that pushing has completed.
    final File persistDir = computePersistDir(identifier);
    final File mergedTarget = new File(persistDir, "merged");
    final File descriptorFile = computeDescriptorFile(identifier);
    // Sanity checks
    for (FireHydrant hydrant : sink) {
        if (sink.isWritable()) {
            throw new ISE("Expected sink to be no longer writable before mergeAndPush for segment[%s].", identifier);
        }
        synchronized (hydrant) {
            if (!hydrant.hasSwapped()) {
                throw new ISE("Expected sink to be fully persisted before mergeAndPush for segment[%s].", identifier);
            }
        }
    }
    try {
        if (descriptorFile.exists()) {
            if (useUniquePath) {
                // Don't reuse the descriptor, because the caller asked for a unique path. Leave the old one as-is, since
                // it might serve some unknown purpose.
                log.debug("Segment[%s] already pushed, but we want a unique path, so will push again with a new path.", identifier);
            } else {
                log.info("Segment[%s] already pushed, skipping.", identifier);
                return objectMapper.readValue(descriptorFile, DataSegment.class);
            }
        }
        removeDirectory(mergedTarget);
        if (mergedTarget.exists()) {
            throw new ISE("Merged target[%s] exists after removing?!", mergedTarget);
        }
        final File mergedFile;
        final long mergeFinishTime;
        final long startTime = System.nanoTime();
        List<QueryableIndex> indexes = new ArrayList<>();
        Closer closer = Closer.create();
        try {
            for (FireHydrant fireHydrant : sink) {
                Pair<ReferenceCountingSegment, Closeable> segmentAndCloseable = fireHydrant.getAndIncrementSegment();
                final QueryableIndex queryableIndex = segmentAndCloseable.lhs.asQueryableIndex();
                log.debug("Segment[%s] adding hydrant[%s]", identifier, fireHydrant);
                indexes.add(queryableIndex);
                closer.register(segmentAndCloseable.rhs);
            }
            mergedFile = indexMerger.mergeQueryableIndex(indexes, schema.getGranularitySpec().isRollup(), schema.getAggregators(), schema.getDimensionsSpec(), mergedTarget, tuningConfig.getIndexSpec(), tuningConfig.getIndexSpecForIntermediatePersists(), new BaseProgressIndicator(), tuningConfig.getSegmentWriteOutMediumFactory(), tuningConfig.getMaxColumnsToMerge());
            mergeFinishTime = System.nanoTime();
            log.debug("Segment[%s] built in %,dms.", identifier, (mergeFinishTime - startTime) / 1000000);
        } catch (Throwable t) {
            throw closer.rethrow(t);
        } finally {
            closer.close();
        }
        final DataSegment segmentToPush = sink.getSegment().withDimensions(IndexMerger.getMergedDimensionsFromQueryableIndexes(indexes, schema.getDimensionsSpec()));
        // Retry pushing segments because uploading to deep storage might fail especially for cloud storage types
        final DataSegment segment = RetryUtils.retry(// semantics.
        () -> dataSegmentPusher.push(mergedFile, segmentToPush, useUniquePath), exception -> exception instanceof Exception, 5);
        final long pushFinishTime = System.nanoTime();
        objectMapper.writeValue(descriptorFile, segment);
        log.info("Segment[%s] of %,d bytes " + "built from %d incremental persist(s) in %,dms; " + "pushed to deep storage in %,dms. " + "Load spec is: %s", identifier, segment.getSize(), indexes.size(), (mergeFinishTime - startTime) / 1000000, (pushFinishTime - mergeFinishTime) / 1000000, objectMapper.writeValueAsString(segment.getLoadSpec()));
        return segment;
    } catch (Exception e) {
        metrics.incrementFailedHandoffs();
        log.warn(e, "Failed to push merged index for segment[%s].", identifier);
        throw new RuntimeException(e);
    }
}

Example 5 with Closer

use of org.apache.druid.java.util.common.io.Closer in project druid by druid-io.

the class GroupByTimeseriesQueryRunnerTest method constructorFeeder.

@SuppressWarnings("unchecked")
@Parameterized.Parameters(name = "{0}, vectorize = {1}")
public static Iterable<Object[]> constructorFeeder() {
    GroupByQueryConfig config = new GroupByQueryConfig();
    config.setMaxIntermediateRows(10000);
    final Pair<GroupByQueryRunnerFactory, Closer> factoryAndCloser = GroupByQueryRunnerTest.makeQueryRunnerFactory(config);
    final GroupByQueryRunnerFactory factory = factoryAndCloser.lhs;
    RESOURCE_CLOSER.register(factoryAndCloser.rhs);
    final List<Object[]> constructors = new ArrayList<>();
    for (QueryRunner<ResultRow> runner : QueryRunnerTestHelper.makeQueryRunners(factory)) {
        final QueryRunner modifiedRunner = new QueryRunner() {

            @Override
            public Sequence run(QueryPlus queryPlus, ResponseContext responseContext) {
                TimeseriesQuery tsQuery = (TimeseriesQuery) queryPlus.getQuery();
                QueryRunner<ResultRow> newRunner = factory.mergeRunners(Execs.directExecutor(), ImmutableList.of(runner));
                QueryToolChest toolChest = factory.getToolchest();
                newRunner = new FinalizeResultsQueryRunner<>(toolChest.mergeResults(toolChest.preMergeQueryDecoration(newRunner)), toolChest);
                final String timeDimension = tsQuery.getTimestampResultField();
                final List<VirtualColumn> virtualColumns = new ArrayList<>(Arrays.asList(tsQuery.getVirtualColumns().getVirtualColumns()));
                Map<String, Object> theContext = tsQuery.getContext();
                if (timeDimension != null) {
                    theContext = new HashMap<>(tsQuery.getContext());
                    final PeriodGranularity granularity = (PeriodGranularity) tsQuery.getGranularity();
                    virtualColumns.add(new ExpressionVirtualColumn("v0", StringUtils.format("timestamp_floor(__time, '%s')", granularity.getPeriod()), ColumnType.LONG, TestExprMacroTable.INSTANCE));
                    theContext.put(GroupByQuery.CTX_TIMESTAMP_RESULT_FIELD, timeDimension);
                    theContext.put(GroupByQuery.CTX_TIMESTAMP_RESULT_FIELD_GRANULARITY, granularity);
                    theContext.put(GroupByQuery.CTX_TIMESTAMP_RESULT_FIELD_INDEX, 0);
                }
                GroupByQuery newQuery = GroupByQuery.builder().setDataSource(tsQuery.getDataSource()).setQuerySegmentSpec(tsQuery.getQuerySegmentSpec()).setGranularity(tsQuery.getGranularity()).setDimFilter(tsQuery.getDimensionsFilter()).setDimensions(timeDimension == null ? ImmutableList.of() : ImmutableList.of(new DefaultDimensionSpec("v0", timeDimension, ColumnType.LONG))).setAggregatorSpecs(tsQuery.getAggregatorSpecs()).setPostAggregatorSpecs(tsQuery.getPostAggregatorSpecs()).setVirtualColumns(VirtualColumns.create(virtualColumns)).setContext(theContext).build();
                return Sequences.map(newRunner.run(queryPlus.withQuery(newQuery), responseContext), new Function<ResultRow, Result<TimeseriesResultValue>>() {

                    @Override
                    public Result<TimeseriesResultValue> apply(final ResultRow input) {
                        final MapBasedRow mapBasedRow = input.toMapBasedRow(newQuery);
                        return new Result<>(mapBasedRow.getTimestamp(), new TimeseriesResultValue(mapBasedRow.getEvent()));
                    }
                });
            }

            @Override
            public String toString() {
                return runner.toString();
            }
        };
        for (boolean vectorize : ImmutableList.of(false, true)) {
            // Add vectorization tests for any indexes that support it.
            if (!vectorize || QueryRunnerTestHelper.isTestRunnerVectorizable(runner)) {
                constructors.add(new Object[] { modifiedRunner, vectorize });
            }
        }
    }
    return constructors;
}