Search in sources :

Example 1 with SegmentReference

use of org.apache.druid.segment.SegmentReference in project druid by druid-io.

the class FireHydrantTest method testGetSegmentForQuerySwapped.

@Test
public void testGetSegmentForQuerySwapped() throws IOException {
    ReferenceCountingSegment incrementalSegmentReference = hydrant.getHydrantSegment();
    hydrant.swapSegment(queryableIndexSegment);
    ReferenceCountingSegment queryableSegmentReference = hydrant.getHydrantSegment();
    Assert.assertEquals(0, incrementalSegmentReference.getNumReferences());
    Assert.assertEquals(0, queryableSegmentReference.getNumReferences());
    Optional<Pair<SegmentReference, Closeable>> maybeSegmentAndCloseable = hydrant.getSegmentForQuery(Function.identity());
    Assert.assertTrue(maybeSegmentAndCloseable.isPresent());
    Assert.assertEquals(0, incrementalSegmentReference.getNumReferences());
    Assert.assertEquals(1, queryableSegmentReference.getNumReferences());
    Pair<SegmentReference, Closeable> segmentAndCloseable = maybeSegmentAndCloseable.get();
    segmentAndCloseable.rhs.close();
    Assert.assertEquals(0, incrementalSegmentReference.getNumReferences());
    Assert.assertEquals(0, queryableSegmentReference.getNumReferences());
}
Also used : ReferenceCountingSegment(org.apache.druid.segment.ReferenceCountingSegment) Closeable(java.io.Closeable) SegmentReference(org.apache.druid.segment.SegmentReference) Pair(org.apache.druid.java.util.common.Pair) InitializedNullHandlingTest(org.apache.druid.testing.InitializedNullHandlingTest) Test(org.junit.Test)

Example 2 with SegmentReference

use of org.apache.druid.segment.SegmentReference in project druid by druid-io.

the class FireHydrantTest method testGetSegmentForQuery.

@Test
public void testGetSegmentForQuery() throws IOException {
    ReferenceCountingSegment incrementalSegmentReference = hydrant.getHydrantSegment();
    Assert.assertEquals(0, incrementalSegmentReference.getNumReferences());
    Optional<Pair<SegmentReference, Closeable>> maybeSegmentAndCloseable = hydrant.getSegmentForQuery(Function.identity());
    Assert.assertTrue(maybeSegmentAndCloseable.isPresent());
    Assert.assertEquals(1, incrementalSegmentReference.getNumReferences());
    Pair<SegmentReference, Closeable> segmentAndCloseable = maybeSegmentAndCloseable.get();
    segmentAndCloseable.rhs.close();
    Assert.assertEquals(0, incrementalSegmentReference.getNumReferences());
}
Also used : ReferenceCountingSegment(org.apache.druid.segment.ReferenceCountingSegment) Closeable(java.io.Closeable) SegmentReference(org.apache.druid.segment.SegmentReference) Pair(org.apache.druid.java.util.common.Pair) InitializedNullHandlingTest(org.apache.druid.testing.InitializedNullHandlingTest) Test(org.junit.Test)

Example 3 with SegmentReference

use of org.apache.druid.segment.SegmentReference in project druid by druid-io.

the class HashJoinSegmentTest method setUp.

@Before
public void setUp() throws IOException {
    allReferencesAcquireCount = 0;
    allReferencesCloseCount = 0;
    referencedSegmentAcquireCount = 0;
    referencedSegmentClosedCount = 0;
    indexedTableJoinableReferenceAcquireCount = 0;
    indexedTableJoinableReferenceCloseCount = 0;
    j0Closed = false;
    j1Closed = false;
    baseSegment = new QueryableIndexSegment(JoinTestHelper.createFactIndexBuilder(temporaryFolder.newFolder()).buildMMappedIndex(), SegmentId.dummy("facts"));
    List<JoinableClause> joinableClauses = ImmutableList.of(new JoinableClause("j0.", new IndexedTableJoinable(JoinTestHelper.createCountriesIndexedTable()) {

        @Override
        public Optional<Closeable> acquireReferences() {
            if (!j0Closed) {
                indexedTableJoinableReferenceAcquireCount++;
                Closer closer = Closer.create();
                closer.register(() -> indexedTableJoinableReferenceCloseCount++);
                return Optional.of(closer);
            }
            return Optional.empty();
        }
    }, JoinType.LEFT, JoinConditionAnalysis.forExpression("1", "j0.", ExprMacroTable.nil())), new JoinableClause("j1.", new IndexedTableJoinable(JoinTestHelper.createRegionsIndexedTable()) {

        @Override
        public Optional<Closeable> acquireReferences() {
            if (!j1Closed) {
                indexedTableJoinableReferenceAcquireCount++;
                Closer closer = Closer.create();
                closer.register(() -> indexedTableJoinableReferenceCloseCount++);
                return Optional.of(closer);
            }
            return Optional.empty();
        }
    }, JoinType.LEFT, JoinConditionAnalysis.forExpression("1", "j1.", ExprMacroTable.nil())));
    referencedSegment = ReferenceCountingSegment.wrapRootGenerationSegment(baseSegment);
    SegmentReference testWrapper = new SegmentReference() {

        @Override
        public Optional<Closeable> acquireReferences() {
            Closer closer = Closer.create();
            return referencedSegment.acquireReferences().map(closeable -> {
                referencedSegmentAcquireCount++;
                closer.register(closeable);
                closer.register(() -> referencedSegmentClosedCount++);
                return closer;
            });
        }

        @Override
        public SegmentId getId() {
            return referencedSegment.getId();
        }

        @Override
        public Interval getDataInterval() {
            return referencedSegment.getDataInterval();
        }

        @Nullable
        @Override
        public QueryableIndex asQueryableIndex() {
            return referencedSegment.asQueryableIndex();
        }

        @Override
        public StorageAdapter asStorageAdapter() {
            return referencedSegment.asStorageAdapter();
        }

        @Override
        public void close() {
            referencedSegment.close();
        }
    };
    hashJoinSegment = new HashJoinSegment(testWrapper, null, joinableClauses, null) {

        @Override
        public Optional<Closeable> acquireReferences() {
            Closer closer = Closer.create();
            return super.acquireReferences().map(closeable -> {
                allReferencesAcquireCount++;
                closer.register(closeable);
                closer.register(() -> allReferencesCloseCount++);
                return closer;
            });
        }
    };
}
Also used : QueryableIndexSegment(org.apache.druid.segment.QueryableIndexSegment) Closer(org.apache.druid.java.util.common.io.Closer) CoreMatchers(org.hamcrest.CoreMatchers) Closer(org.apache.druid.java.util.common.io.Closer) IndexedTableJoinable(org.apache.druid.segment.join.table.IndexedTableJoinable) QueryableIndex(org.apache.druid.segment.QueryableIndex) InitializedNullHandlingTest(org.apache.druid.testing.InitializedNullHandlingTest) Test(org.junit.Test) IOException(java.io.IOException) ReferenceCountingSegment(org.apache.druid.segment.ReferenceCountingSegment) StorageAdapter(org.apache.druid.segment.StorageAdapter) ExprMacroTable(org.apache.druid.math.expr.ExprMacroTable) SegmentReference(org.apache.druid.segment.SegmentReference) Interval(org.joda.time.Interval) List(java.util.List) Rule(org.junit.Rule) ImmutableList(com.google.common.collect.ImmutableList) Closeable(java.io.Closeable) Optional(java.util.Optional) SegmentId(org.apache.druid.timeline.SegmentId) Assert(org.junit.Assert) QueryableIndexSegment(org.apache.druid.segment.QueryableIndexSegment) ExpectedException(org.junit.rules.ExpectedException) TemporaryFolder(org.junit.rules.TemporaryFolder) Nullable(javax.annotation.Nullable) Before(org.junit.Before) Optional(java.util.Optional) Closeable(java.io.Closeable) IndexedTableJoinable(org.apache.druid.segment.join.table.IndexedTableJoinable) SegmentReference(org.apache.druid.segment.SegmentReference) Before(org.junit.Before)

Example 4 with SegmentReference

use of org.apache.druid.segment.SegmentReference in project druid by druid-io.

the class JoinableFactoryWrapperTest method test_createSegmentMapFn_usableClause.

@Test
public void test_createSegmentMapFn_usableClause() {
    final LookupDataSource lookupDataSource = new LookupDataSource("lookyloo");
    final JoinConditionAnalysis conditionAnalysis = JoinConditionAnalysis.forExpression("x == \"j.x\"", "j.", ExprMacroTable.nil());
    final PreJoinableClause clause = new PreJoinableClause("j.", lookupDataSource, JoinType.LEFT, conditionAnalysis);
    JoinableFactoryWrapper joinableFactoryWrapper = new JoinableFactoryWrapper(new JoinableFactory() {

        @Override
        public boolean isDirectlyJoinable(DataSource dataSource) {
            return dataSource.equals(lookupDataSource);
        }

        @Override
        public Optional<Joinable> build(DataSource dataSource, JoinConditionAnalysis condition) {
            if (dataSource.equals(lookupDataSource) && condition.equals(conditionAnalysis)) {
                return Optional.of(LookupJoinable.wrap(new MapLookupExtractor(ImmutableMap.of("k", "v"), false)));
            } else {
                return Optional.empty();
            }
        }
    });
    final Function<SegmentReference, SegmentReference> segmentMapFn = joinableFactoryWrapper.createSegmentMapFn(null, ImmutableList.of(clause), new AtomicLong(), new TestQuery(new TableDataSource("test"), new MultipleIntervalSegmentSpec(ImmutableList.of(Intervals.of("0/100"))), false, new HashMap()));
    Assert.assertNotSame(Function.identity(), segmentMapFn);
}
Also used : TestQuery(org.apache.druid.query.TestQuery) PreJoinableClause(org.apache.druid.query.planning.PreJoinableClause) Optional(java.util.Optional) HashMap(java.util.HashMap) SegmentReference(org.apache.druid.segment.SegmentReference) MultipleIntervalSegmentSpec(org.apache.druid.query.spec.MultipleIntervalSegmentSpec) GlobalTableDataSource(org.apache.druid.query.GlobalTableDataSource) LookupDataSource(org.apache.druid.query.LookupDataSource) DataSource(org.apache.druid.query.DataSource) TableDataSource(org.apache.druid.query.TableDataSource) AtomicLong(java.util.concurrent.atomic.AtomicLong) GlobalTableDataSource(org.apache.druid.query.GlobalTableDataSource) TableDataSource(org.apache.druid.query.TableDataSource) LookupDataSource(org.apache.druid.query.LookupDataSource) MapLookupExtractor(org.apache.druid.query.extraction.MapLookupExtractor) NullHandlingTest(org.apache.druid.common.config.NullHandlingTest) Test(org.junit.Test)

Example 5 with SegmentReference

use of org.apache.druid.segment.SegmentReference in project druid by druid-io.

the class ServerManager method buildAndDecorateQueryRunner.

private <T> QueryRunner<T> buildAndDecorateQueryRunner(final QueryRunnerFactory<T, Query<T>> factory, final QueryToolChest<T, Query<T>> toolChest, final SegmentReference segment, final Optional<byte[]> cacheKeyPrefix, final SegmentDescriptor segmentDescriptor, final AtomicLong cpuTimeAccumulator) {
    final SpecificSegmentSpec segmentSpec = new SpecificSegmentSpec(segmentDescriptor);
    final SegmentId segmentId = segment.getId();
    final Interval segmentInterval = segment.getDataInterval();
    // If the segment is closed after this line, ReferenceCountingSegmentQueryRunner will handle and do the right thing.
    if (segmentId == null || segmentInterval == null) {
        return new ReportTimelineMissingSegmentQueryRunner<>(segmentDescriptor);
    }
    String segmentIdString = segmentId.toString();
    MetricsEmittingQueryRunner<T> metricsEmittingQueryRunnerInner = new MetricsEmittingQueryRunner<>(emitter, toolChest, new ReferenceCountingSegmentQueryRunner<>(factory, segment, segmentDescriptor), QueryMetrics::reportSegmentTime, queryMetrics -> queryMetrics.segment(segmentIdString));
    StorageAdapter storageAdapter = segment.asStorageAdapter();
    long segmentMaxTime = storageAdapter.getMaxTime().getMillis();
    long segmentMinTime = storageAdapter.getMinTime().getMillis();
    Interval actualDataInterval = Intervals.utc(segmentMinTime, segmentMaxTime + 1);
    CachingQueryRunner<T> cachingQueryRunner = new CachingQueryRunner<>(segmentIdString, cacheKeyPrefix, segmentDescriptor, actualDataInterval, objectMapper, cache, toolChest, metricsEmittingQueryRunnerInner, cachePopulator, cacheConfig);
    BySegmentQueryRunner<T> bySegmentQueryRunner = new BySegmentQueryRunner<>(segmentId, segmentInterval.getStart(), cachingQueryRunner);
    MetricsEmittingQueryRunner<T> metricsEmittingQueryRunnerOuter = new MetricsEmittingQueryRunner<>(emitter, toolChest, bySegmentQueryRunner, QueryMetrics::reportSegmentAndCacheTime, queryMetrics -> queryMetrics.segment(segmentIdString)).withWaitMeasuredFromNow();
    SpecificSegmentQueryRunner<T> specificSegmentQueryRunner = new SpecificSegmentQueryRunner<>(metricsEmittingQueryRunnerOuter, segmentSpec);
    PerSegmentOptimizingQueryRunner<T> perSegmentOptimizingQueryRunner = new PerSegmentOptimizingQueryRunner<>(specificSegmentQueryRunner, new PerSegmentQueryOptimizationContext(segmentDescriptor));
    return new SetAndVerifyContextQueryRunner<>(serverConfig, CPUTimeMetricQueryRunner.safeBuild(perSegmentOptimizingQueryRunner, toolChest, emitter, cpuTimeAccumulator, false));
}
Also used : SegmentManager(org.apache.druid.server.SegmentManager) Inject(com.google.inject.Inject) Smile(org.apache.druid.guice.annotations.Smile) QueryProcessingPool(org.apache.druid.query.QueryProcessingPool) StorageAdapter(org.apache.druid.segment.StorageAdapter) NoopQueryRunner(org.apache.druid.query.NoopQueryRunner) SegmentReference(org.apache.druid.segment.SegmentReference) SpecificSegmentQueryRunner(org.apache.druid.query.spec.SpecificSegmentQueryRunner) QueryRunner(org.apache.druid.query.QueryRunner) FinalizeResultsQueryRunner(org.apache.druid.query.FinalizeResultsQueryRunner) ReportTimelineMissingSegmentQueryRunner(org.apache.druid.query.ReportTimelineMissingSegmentQueryRunner) CacheConfig(org.apache.druid.client.cache.CacheConfig) StringUtils(org.apache.druid.java.util.common.StringUtils) JoinableFactoryWrapper(org.apache.druid.segment.join.JoinableFactoryWrapper) ISE(org.apache.druid.java.util.common.ISE) SpecificSegmentSpec(org.apache.druid.query.spec.SpecificSegmentSpec) BySegmentQueryRunner(org.apache.druid.query.BySegmentQueryRunner) SetAndVerifyContextQueryRunner(org.apache.druid.server.SetAndVerifyContextQueryRunner) QueryDataSource(org.apache.druid.query.QueryDataSource) PerSegmentQueryOptimizationContext(org.apache.druid.query.PerSegmentQueryOptimizationContext) ServiceEmitter(org.apache.druid.java.util.emitter.service.ServiceEmitter) Optional(java.util.Optional) PerSegmentOptimizingQueryRunner(org.apache.druid.query.PerSegmentOptimizingQueryRunner) FunctionalIterable(org.apache.druid.java.util.common.guava.FunctionalIterable) SegmentId(org.apache.druid.timeline.SegmentId) DataSourceAnalysis(org.apache.druid.query.planning.DataSourceAnalysis) Intervals(org.apache.druid.java.util.common.Intervals) QueryMetrics(org.apache.druid.query.QueryMetrics) CachingQueryRunner(org.apache.druid.client.CachingQueryRunner) JoinableFactory(org.apache.druid.segment.join.JoinableFactory) Function(java.util.function.Function) PartitionChunk(org.apache.druid.timeline.partition.PartitionChunk) Interval(org.joda.time.Interval) Lists(com.google.common.collect.Lists) MetricsEmittingQueryRunner(org.apache.druid.query.MetricsEmittingQueryRunner) Query(org.apache.druid.query.Query) CachePopulator(org.apache.druid.client.cache.CachePopulator) QuerySegmentWalker(org.apache.druid.query.QuerySegmentWalker) EmittingLogger(org.apache.druid.java.util.emitter.EmittingLogger) VersionedIntervalTimeline(org.apache.druid.timeline.VersionedIntervalTimeline) ServerConfig(org.apache.druid.server.initialization.ServerConfig) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) QueryRunnerFactoryConglomerate(org.apache.druid.query.QueryRunnerFactoryConglomerate) QueryToolChest(org.apache.druid.query.QueryToolChest) ReferenceCountingSegment(org.apache.druid.segment.ReferenceCountingSegment) AtomicLong(java.util.concurrent.atomic.AtomicLong) ReferenceCountingSegmentQueryRunner(org.apache.druid.query.ReferenceCountingSegmentQueryRunner) QueryRunnerFactory(org.apache.druid.query.QueryRunnerFactory) SegmentDescriptor(org.apache.druid.query.SegmentDescriptor) Cache(org.apache.druid.client.cache.Cache) Filters(org.apache.druid.segment.filter.Filters) Collections(java.util.Collections) CPUTimeMetricQueryRunner(org.apache.druid.query.CPUTimeMetricQueryRunner) QueryUnsupportedException(org.apache.druid.query.QueryUnsupportedException) SegmentId(org.apache.druid.timeline.SegmentId) StorageAdapter(org.apache.druid.segment.StorageAdapter) BySegmentQueryRunner(org.apache.druid.query.BySegmentQueryRunner) MetricsEmittingQueryRunner(org.apache.druid.query.MetricsEmittingQueryRunner) PerSegmentQueryOptimizationContext(org.apache.druid.query.PerSegmentQueryOptimizationContext) ReportTimelineMissingSegmentQueryRunner(org.apache.druid.query.ReportTimelineMissingSegmentQueryRunner) SpecificSegmentSpec(org.apache.druid.query.spec.SpecificSegmentSpec) SpecificSegmentQueryRunner(org.apache.druid.query.spec.SpecificSegmentQueryRunner) CachingQueryRunner(org.apache.druid.client.CachingQueryRunner) SetAndVerifyContextQueryRunner(org.apache.druid.server.SetAndVerifyContextQueryRunner) QueryMetrics(org.apache.druid.query.QueryMetrics) PerSegmentOptimizingQueryRunner(org.apache.druid.query.PerSegmentOptimizingQueryRunner) Interval(org.joda.time.Interval)

Aggregations

SegmentReference (org.apache.druid.segment.SegmentReference)15 AtomicLong (java.util.concurrent.atomic.AtomicLong)10 ReferenceCountingSegment (org.apache.druid.segment.ReferenceCountingSegment)10 Optional (java.util.Optional)7 Query (org.apache.druid.query.Query)7 Test (org.junit.Test)7 ISE (org.apache.druid.java.util.common.ISE)6 DataSourceAnalysis (org.apache.druid.query.planning.DataSourceAnalysis)6 Filters (org.apache.druid.segment.filter.Filters)6 Closeable (java.io.Closeable)5 Function (java.util.function.Function)5 Pair (org.apache.druid.java.util.common.Pair)5 QueryRunner (org.apache.druid.query.QueryRunner)5 VersionedIntervalTimeline (org.apache.druid.timeline.VersionedIntervalTimeline)5 Interval (org.joda.time.Interval)5 Nullable (javax.annotation.Nullable)4 FinalizeResultsQueryRunner (org.apache.druid.query.FinalizeResultsQueryRunner)4 NoopQueryRunner (org.apache.druid.query.NoopQueryRunner)4 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)3 Preconditions (com.google.common.base.Preconditions)3