use of org.apache.druid.segment.SegmentReference in project druid by druid-io.
the class FireHydrantTest method testGetSegmentForQuerySwapped.
@Test
public void testGetSegmentForQuerySwapped() throws IOException {
ReferenceCountingSegment incrementalSegmentReference = hydrant.getHydrantSegment();
hydrant.swapSegment(queryableIndexSegment);
ReferenceCountingSegment queryableSegmentReference = hydrant.getHydrantSegment();
Assert.assertEquals(0, incrementalSegmentReference.getNumReferences());
Assert.assertEquals(0, queryableSegmentReference.getNumReferences());
Optional<Pair<SegmentReference, Closeable>> maybeSegmentAndCloseable = hydrant.getSegmentForQuery(Function.identity());
Assert.assertTrue(maybeSegmentAndCloseable.isPresent());
Assert.assertEquals(0, incrementalSegmentReference.getNumReferences());
Assert.assertEquals(1, queryableSegmentReference.getNumReferences());
Pair<SegmentReference, Closeable> segmentAndCloseable = maybeSegmentAndCloseable.get();
segmentAndCloseable.rhs.close();
Assert.assertEquals(0, incrementalSegmentReference.getNumReferences());
Assert.assertEquals(0, queryableSegmentReference.getNumReferences());
}
use of org.apache.druid.segment.SegmentReference in project druid by druid-io.
the class FireHydrantTest method testGetSegmentForQuery.
@Test
public void testGetSegmentForQuery() throws IOException {
ReferenceCountingSegment incrementalSegmentReference = hydrant.getHydrantSegment();
Assert.assertEquals(0, incrementalSegmentReference.getNumReferences());
Optional<Pair<SegmentReference, Closeable>> maybeSegmentAndCloseable = hydrant.getSegmentForQuery(Function.identity());
Assert.assertTrue(maybeSegmentAndCloseable.isPresent());
Assert.assertEquals(1, incrementalSegmentReference.getNumReferences());
Pair<SegmentReference, Closeable> segmentAndCloseable = maybeSegmentAndCloseable.get();
segmentAndCloseable.rhs.close();
Assert.assertEquals(0, incrementalSegmentReference.getNumReferences());
}
use of org.apache.druid.segment.SegmentReference in project druid by druid-io.
the class HashJoinSegmentTest method setUp.
@Before
public void setUp() throws IOException {
allReferencesAcquireCount = 0;
allReferencesCloseCount = 0;
referencedSegmentAcquireCount = 0;
referencedSegmentClosedCount = 0;
indexedTableJoinableReferenceAcquireCount = 0;
indexedTableJoinableReferenceCloseCount = 0;
j0Closed = false;
j1Closed = false;
baseSegment = new QueryableIndexSegment(JoinTestHelper.createFactIndexBuilder(temporaryFolder.newFolder()).buildMMappedIndex(), SegmentId.dummy("facts"));
List<JoinableClause> joinableClauses = ImmutableList.of(new JoinableClause("j0.", new IndexedTableJoinable(JoinTestHelper.createCountriesIndexedTable()) {
@Override
public Optional<Closeable> acquireReferences() {
if (!j0Closed) {
indexedTableJoinableReferenceAcquireCount++;
Closer closer = Closer.create();
closer.register(() -> indexedTableJoinableReferenceCloseCount++);
return Optional.of(closer);
}
return Optional.empty();
}
}, JoinType.LEFT, JoinConditionAnalysis.forExpression("1", "j0.", ExprMacroTable.nil())), new JoinableClause("j1.", new IndexedTableJoinable(JoinTestHelper.createRegionsIndexedTable()) {
@Override
public Optional<Closeable> acquireReferences() {
if (!j1Closed) {
indexedTableJoinableReferenceAcquireCount++;
Closer closer = Closer.create();
closer.register(() -> indexedTableJoinableReferenceCloseCount++);
return Optional.of(closer);
}
return Optional.empty();
}
}, JoinType.LEFT, JoinConditionAnalysis.forExpression("1", "j1.", ExprMacroTable.nil())));
referencedSegment = ReferenceCountingSegment.wrapRootGenerationSegment(baseSegment);
SegmentReference testWrapper = new SegmentReference() {
@Override
public Optional<Closeable> acquireReferences() {
Closer closer = Closer.create();
return referencedSegment.acquireReferences().map(closeable -> {
referencedSegmentAcquireCount++;
closer.register(closeable);
closer.register(() -> referencedSegmentClosedCount++);
return closer;
});
}
@Override
public SegmentId getId() {
return referencedSegment.getId();
}
@Override
public Interval getDataInterval() {
return referencedSegment.getDataInterval();
}
@Nullable
@Override
public QueryableIndex asQueryableIndex() {
return referencedSegment.asQueryableIndex();
}
@Override
public StorageAdapter asStorageAdapter() {
return referencedSegment.asStorageAdapter();
}
@Override
public void close() {
referencedSegment.close();
}
};
hashJoinSegment = new HashJoinSegment(testWrapper, null, joinableClauses, null) {
@Override
public Optional<Closeable> acquireReferences() {
Closer closer = Closer.create();
return super.acquireReferences().map(closeable -> {
allReferencesAcquireCount++;
closer.register(closeable);
closer.register(() -> allReferencesCloseCount++);
return closer;
});
}
};
}
use of org.apache.druid.segment.SegmentReference in project druid by druid-io.
the class JoinableFactoryWrapperTest method test_createSegmentMapFn_usableClause.
@Test
public void test_createSegmentMapFn_usableClause() {
final LookupDataSource lookupDataSource = new LookupDataSource("lookyloo");
final JoinConditionAnalysis conditionAnalysis = JoinConditionAnalysis.forExpression("x == \"j.x\"", "j.", ExprMacroTable.nil());
final PreJoinableClause clause = new PreJoinableClause("j.", lookupDataSource, JoinType.LEFT, conditionAnalysis);
JoinableFactoryWrapper joinableFactoryWrapper = new JoinableFactoryWrapper(new JoinableFactory() {
@Override
public boolean isDirectlyJoinable(DataSource dataSource) {
return dataSource.equals(lookupDataSource);
}
@Override
public Optional<Joinable> build(DataSource dataSource, JoinConditionAnalysis condition) {
if (dataSource.equals(lookupDataSource) && condition.equals(conditionAnalysis)) {
return Optional.of(LookupJoinable.wrap(new MapLookupExtractor(ImmutableMap.of("k", "v"), false)));
} else {
return Optional.empty();
}
}
});
final Function<SegmentReference, SegmentReference> segmentMapFn = joinableFactoryWrapper.createSegmentMapFn(null, ImmutableList.of(clause), new AtomicLong(), new TestQuery(new TableDataSource("test"), new MultipleIntervalSegmentSpec(ImmutableList.of(Intervals.of("0/100"))), false, new HashMap()));
Assert.assertNotSame(Function.identity(), segmentMapFn);
}
use of org.apache.druid.segment.SegmentReference in project druid by druid-io.
the class ServerManager method buildAndDecorateQueryRunner.
private <T> QueryRunner<T> buildAndDecorateQueryRunner(final QueryRunnerFactory<T, Query<T>> factory, final QueryToolChest<T, Query<T>> toolChest, final SegmentReference segment, final Optional<byte[]> cacheKeyPrefix, final SegmentDescriptor segmentDescriptor, final AtomicLong cpuTimeAccumulator) {
final SpecificSegmentSpec segmentSpec = new SpecificSegmentSpec(segmentDescriptor);
final SegmentId segmentId = segment.getId();
final Interval segmentInterval = segment.getDataInterval();
// If the segment is closed after this line, ReferenceCountingSegmentQueryRunner will handle and do the right thing.
if (segmentId == null || segmentInterval == null) {
return new ReportTimelineMissingSegmentQueryRunner<>(segmentDescriptor);
}
String segmentIdString = segmentId.toString();
MetricsEmittingQueryRunner<T> metricsEmittingQueryRunnerInner = new MetricsEmittingQueryRunner<>(emitter, toolChest, new ReferenceCountingSegmentQueryRunner<>(factory, segment, segmentDescriptor), QueryMetrics::reportSegmentTime, queryMetrics -> queryMetrics.segment(segmentIdString));
StorageAdapter storageAdapter = segment.asStorageAdapter();
long segmentMaxTime = storageAdapter.getMaxTime().getMillis();
long segmentMinTime = storageAdapter.getMinTime().getMillis();
Interval actualDataInterval = Intervals.utc(segmentMinTime, segmentMaxTime + 1);
CachingQueryRunner<T> cachingQueryRunner = new CachingQueryRunner<>(segmentIdString, cacheKeyPrefix, segmentDescriptor, actualDataInterval, objectMapper, cache, toolChest, metricsEmittingQueryRunnerInner, cachePopulator, cacheConfig);
BySegmentQueryRunner<T> bySegmentQueryRunner = new BySegmentQueryRunner<>(segmentId, segmentInterval.getStart(), cachingQueryRunner);
MetricsEmittingQueryRunner<T> metricsEmittingQueryRunnerOuter = new MetricsEmittingQueryRunner<>(emitter, toolChest, bySegmentQueryRunner, QueryMetrics::reportSegmentAndCacheTime, queryMetrics -> queryMetrics.segment(segmentIdString)).withWaitMeasuredFromNow();
SpecificSegmentQueryRunner<T> specificSegmentQueryRunner = new SpecificSegmentQueryRunner<>(metricsEmittingQueryRunnerOuter, segmentSpec);
PerSegmentOptimizingQueryRunner<T> perSegmentOptimizingQueryRunner = new PerSegmentOptimizingQueryRunner<>(specificSegmentQueryRunner, new PerSegmentQueryOptimizationContext(segmentDescriptor));
return new SetAndVerifyContextQueryRunner<>(serverConfig, CPUTimeMetricQueryRunner.safeBuild(perSegmentOptimizingQueryRunner, toolChest, emitter, cpuTimeAccumulator, false));
}
Aggregations