use of io.prestosql.spi.connector.FixedPageSource in project hetu-core by openlookeng.
the class MemoryPageSourceProvider method createPageSource.
@Override
public ConnectorPageSource createPageSource(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorSplit split, ConnectorTableHandle table, List<ColumnHandle> columns, Optional<DynamicFilterSupplier> dynamicFilterSupplier) {
MemorySplit memorySplit = (MemorySplit) split;
long tableId = memorySplit.getTable();
int logicalPartNumber = memorySplit.getLogicalPartNum();
long expectedRows = memorySplit.getExpectedRows();
MemoryTableHandle memoryTable = (MemoryTableHandle) table;
OptionalDouble sampleRatio = memoryTable.getSampleRatio();
TupleDomain<ColumnHandle> predicate = memoryTable.getPredicate();
// Commenting for Dynamic filter changes
List<Integer> columnIndexes = columns.stream().map(MemoryColumnHandle.class::cast).map(MemoryColumnHandle::getColumnIndex).collect(toList());
List<Page> pages = pagesStore.getPages(tableId, logicalPartNumber, columnIndexes, expectedRows, memorySplit.getLimit(), sampleRatio, predicate);
if (dynamicFilterSupplier.isPresent()) {
return new FixedPageSource(pages.stream().map(page -> applyFilter(page, dynamicFilterSupplier, columns)).collect(toList()));
} else {
return new FixedPageSource(pages);
}
}
use of io.prestosql.spi.connector.FixedPageSource in project hetu-core by openlookeng.
the class TestSystemMemoryBlocking method testTableScanSystemMemoryBlocking.
@Test
public void testTableScanSystemMemoryBlocking() {
PlanNodeId sourceId = new PlanNodeId("source");
final List<Type> types = ImmutableList.of(VARCHAR);
TableScanOperator source = new TableScanOperator(driverContext.addOperatorContext(1, new PlanNodeId("test"), "values"), sourceId, (session, split, table, columns, dynamicFilter) -> new FixedPageSource(rowPagesBuilder(types).addSequencePage(10, 1).addSequencePage(10, 1).addSequencePage(10, 1).addSequencePage(10, 1).addSequencePage(10, 1).build()), TEST_TABLE_HANDLE, ImmutableList.of(), ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_DEFAULT, new UUID(0, 0), types, false, Optional.empty(), 0, 0);
PageConsumerOperator sink = createSinkOperator(types);
Driver driver = Driver.createDriver(driverContext, source, sink);
assertSame(driver.getDriverContext(), driverContext);
assertFalse(driver.isFinished());
Split testSplit = new Split(new CatalogName("test"), new TestSplit(), Lifespan.taskWide());
driver.updateSource(new TaskSource(sourceId, ImmutableSet.of(new ScheduledSplit(0, sourceId, testSplit)), true));
ListenableFuture<?> blocked = driver.processFor(new Duration(1, NANOSECONDS));
// the driver shouldn't block in the first call as it will be able to move a page between source and the sink operator
// but the operator should be blocked
assertTrue(blocked.isDone());
assertFalse(source.getOperatorContext().isWaitingForMemory().isDone());
// and they should stay blocked until more memory becomes available
for (int i = 0; i < 10; i++) {
blocked = driver.processFor(new Duration(1, NANOSECONDS));
assertFalse(blocked.isDone());
assertFalse(source.getOperatorContext().isWaitingForMemory().isDone());
}
// free up some memory
memoryPool.free(QUERY_ID, "test", memoryPool.getReservedBytes());
// the operator should be unblocked
assertTrue(source.getOperatorContext().isWaitingForMemory().isDone());
// the driver shouldn't be blocked
blocked = driver.processFor(new Duration(1, NANOSECONDS));
assertTrue(blocked.isDone());
}
use of io.prestosql.spi.connector.FixedPageSource in project hetu-core by openlookeng.
the class TestDriver method testMemoryRevocationRace.
@Test
public void testMemoryRevocationRace() {
List<Type> types = ImmutableList.of(VARCHAR, BIGINT, BIGINT);
TableScanOperator source = new AlwaysBlockedMemoryRevokingTableScanOperator(driverContext.addOperatorContext(99, new PlanNodeId("test"), "scan"), new PlanNodeId("source"), (session, split, table, columns, dynamicFilter) -> new FixedPageSource(rowPagesBuilder(types).addSequencePage(10, 20, 30, 40).build()), TEST_TABLE_HANDLE, ImmutableList.of());
Driver driver = Driver.createDriver(driverContext, source, createSinkOperator(types));
// the table scan operator will request memory revocation with requestMemoryRevoking()
// while the driver is still not done with the processFor() method and before it moves to
// updateDriverBlockedFuture() method.
assertTrue(driver.processFor(new Duration(100, TimeUnit.MILLISECONDS)).isDone());
}
use of io.prestosql.spi.connector.FixedPageSource in project hetu-core by openlookeng.
the class TestDriver method testBrokenOperatorAddSource.
@Test
public void testBrokenOperatorAddSource() throws Exception {
PlanNodeId sourceId = new PlanNodeId("source");
final List<Type> types = ImmutableList.of(VARCHAR, BIGINT, BIGINT);
// create a table scan operator that does not block, which will cause the driver loop to busy wait
TableScanOperator source = new NotBlockedTableScanOperator(driverContext.addOperatorContext(99, new PlanNodeId("test"), "values"), sourceId, (session, split, table, columns, dynamicFilter) -> new FixedPageSource(rowPagesBuilder(types).addSequencePage(10, 20, 30, 40).build()), TEST_TABLE_HANDLE, ImmutableList.of());
BrokenOperator brokenOperator = new BrokenOperator(driverContext.addOperatorContext(0, new PlanNodeId("test"), "source"));
final Driver driver = Driver.createDriver(driverContext, source, brokenOperator);
// block thread in operator processing
Future<Boolean> driverProcessFor = executor.submit(new Callable<Boolean>() {
@Override
public Boolean call() {
return driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone();
}
});
brokenOperator.waitForLocked();
assertSame(driver.getDriverContext(), driverContext);
assertFalse(driver.isFinished());
// processFor always returns NOT_BLOCKED, because DriveLockResult was not acquired
assertTrue(driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone());
assertFalse(driver.isFinished());
driver.updateSource(new TaskSource(sourceId, ImmutableSet.of(new ScheduledSplit(0, sourceId, newMockSplit())), true));
assertFalse(driver.isFinished());
// processFor always returns NOT_BLOCKED, because DriveLockResult was not acquired
assertTrue(driver.processFor(new Duration(1, TimeUnit.SECONDS)).isDone());
assertFalse(driver.isFinished());
driver.close();
assertTrue(driver.isFinished());
try {
driverProcessFor.get(1, TimeUnit.SECONDS);
fail("Expected InterruptedException");
} catch (ExecutionException e) {
assertDriverInterrupted(e.getCause());
}
}
use of io.prestosql.spi.connector.FixedPageSource in project hetu-core by openlookeng.
the class TestHiveDistributedJoinQueriesWithDynamicFiltering method testIsPartitionFiltered.
@Test
public void testIsPartitionFiltered() throws IOException {
Properties schema = new Properties();
ImmutableList<HivePartitionKey> partitionKeys = ImmutableList.of(new HivePartitionKey("p1", "100"), new HivePartitionKey("p2", "101"), new HivePartitionKey("p3", "__HIVE_DEFAULT_PARTITION__"));
HiveSplitWrapper split = HiveSplitWrapper.wrap(new HiveSplit("db", "table", "partitionId", "path", 0, 50, 50, 0, schema, partitionKeys, ImmutableList.of(), OptionalInt.empty(), false, ImmutableMap.of(), Optional.empty(), false, Optional.empty(), Optional.empty(), false, ImmutableMap.of()));
List<Long> filterValues = ImmutableList.of(1L, 50L, 100L);
HiveColumnHandle testColumnHandle = new HiveColumnHandle("p1", HIVE_INT, parseTypeSignature(StandardTypes.INTEGER), 0, PARTITION_KEY, Optional.empty());
Supplier<List<Map<ColumnHandle, DynamicFilter>>> dynamicFilter = createDynamicFilterSupplier(filterValues, testColumnHandle, "filter1");
Optional<DynamicFilterSupplier> dynamicFilterSupplier = Optional.of(new DynamicFilterSupplier(dynamicFilter, System.currentTimeMillis(), 10000));
HiveColumnHandle testColumnHandle2 = new HiveColumnHandle("p2", HIVE_INT, parseTypeSignature(StandardTypes.INTEGER), 0, PARTITION_KEY, Optional.empty());
Supplier<List<Map<ColumnHandle, DynamicFilter>>> dynamicFilter2 = createDynamicFilterSupplier(filterValues, testColumnHandle2, "filter2");
Optional<DynamicFilterSupplier> dynamicFilterSupplier2 = Optional.of(new DynamicFilterSupplier(dynamicFilter2, System.currentTimeMillis(), 10000));
HiveColumnHandle testColumnHandle3 = new HiveColumnHandle("p3", HIVE_INT, parseTypeSignature(StandardTypes.INTEGER), 0, PARTITION_KEY, Optional.empty());
Supplier<List<Map<ColumnHandle, DynamicFilter>>> dynamicFilter3 = createDynamicFilterSupplier(filterValues, testColumnHandle3, "filter3");
Optional<DynamicFilterSupplier> dynamicFilterSupplier3 = Optional.of(new DynamicFilterSupplier(dynamicFilter3, System.currentTimeMillis(), 10000));
HiveColumnHandle testColumnHandle4 = new HiveColumnHandle("p4", HIVE_INT, parseTypeSignature(StandardTypes.INTEGER), 0, PARTITION_KEY, Optional.empty());
Supplier<List<Map<ColumnHandle, DynamicFilter>>> dynamicFilter4 = createDynamicFilterSupplier(filterValues, testColumnHandle4, "filter3");
Optional<DynamicFilterSupplier> dynamicFilterSupplier4 = Optional.of(new DynamicFilterSupplier(dynamicFilter4, System.currentTimeMillis(), 0));
HiveConfig config = new HiveConfig();
HivePageSourceProvider provider = new HivePageSourceProvider(config, createTestHdfsEnvironment(config), getDefaultHiveRecordCursorProvider(config), getDefaultHiveDataStreamFactories(config), TYPE_MANAGER, getNoOpIndexCache(), getDefaultHiveSelectiveFactories(config));
TestingConnectorSession session = new TestingConnectorSession(new HiveSessionProperties(config, new OrcFileWriterConfig(), new ParquetFileWriterConfig()).getSessionProperties());
ConnectorTableHandle table = new HiveTableHandle("db", "table", ImmutableMap.of(), ImmutableList.of(), Optional.empty());
HiveTransactionHandle transaction = new HiveTransactionHandle();
try {
ConnectorPageSource result = provider.createPageSource(transaction, session, split, table, ImmutableList.of(testColumnHandle), dynamicFilterSupplier);
assertFalse(result instanceof FixedPageSource);
} catch (Exception e) {
assertTrue(e instanceof PrestoException);
}
try {
ConnectorPageSource result = provider.createPageSource(transaction, session, split, table, ImmutableList.of(testColumnHandle2), dynamicFilterSupplier2);
assertTrue(result instanceof FixedPageSource);
} catch (Exception e) {
fail("A FixedPageSource object should have been created");
}
try {
ConnectorPageSource result = provider.createPageSource(transaction, session, split, table, ImmutableList.of(testColumnHandle3), dynamicFilterSupplier3);
assertFalse(result instanceof FixedPageSource);
} catch (Exception e) {
assertTrue(e instanceof PrestoException);
}
try {
ConnectorPageSource result = provider.createPageSource(transaction, session, split, table, ImmutableList.of(testColumnHandle4), dynamicFilterSupplier4);
assertFalse(result instanceof FixedPageSource);
} catch (Exception e) {
assertTrue(e instanceof PrestoException);
}
}
Aggregations