use of io.prestosql.tests.DistributedQueryRunner in project hetu-core by openlookeng.
the class PostgreSqlQueryRunner method createPostgreSqlQueryRunner.
public static QueryRunner createPostgreSqlQueryRunner(TestingPostgreSqlServer server, Map<String, String> inputConnectorProperties, Iterable<TpchTable<?>> tables) throws Exception {
Map<String, String> connectorProperties = inputConnectorProperties;
DistributedQueryRunner queryRunner = null;
try {
queryRunner = new DistributedQueryRunner(createSession(), 3);
queryRunner.installPlugin(new TpchPlugin());
queryRunner.createCatalog("tpch", "tpch");
connectorProperties = new HashMap<>(ImmutableMap.copyOf(connectorProperties));
connectorProperties.putIfAbsent("connection-url", server.getJdbcUrl());
connectorProperties.putIfAbsent("allow-drop-table", "true");
createSchema(server.getJdbcUrl(), "tpch");
queryRunner.installPlugin(new PostgreSqlPlugin());
queryRunner.createCatalog("postgresql", "postgresql", connectorProperties);
copyTpchTables(queryRunner, "tpch", TINY_SCHEMA_NAME, createSession(), tables);
return queryRunner;
} catch (Throwable e) {
closeAllSuppress(e, queryRunner, server);
throw e;
}
}
use of io.prestosql.tests.DistributedQueryRunner in project hetu-core by openlookeng.
the class TestResourceGroupDbIntegration method testMemoryFraction.
@Test
public void testMemoryFraction() throws Exception {
try (DistributedQueryRunner queryRunner = getSimpleQueryRunner()) {
queryRunner.execute("SELECT COUNT(*), clerk FROM orders GROUP BY clerk");
waitForGlobalResourceGroup(queryRunner);
}
}
use of io.prestosql.tests.DistributedQueryRunner in project hetu-core by openlookeng.
the class TestMemoryManager method createQueryRunner.
public static DistributedQueryRunner createQueryRunner(Session session, Map<String, String> properties) throws Exception {
DistributedQueryRunner queryRunner = new DistributedQueryRunner(session, 2, properties);
try {
queryRunner.installPlugin(new TpchPlugin());
queryRunner.createCatalog("tpch", "tpch");
return queryRunner;
} catch (Exception e) {
queryRunner.close();
throw e;
}
}
use of io.prestosql.tests.DistributedQueryRunner in project hetu-core by openlookeng.
the class TestMemoryManager method testClusterPools.
@Test(timeOut = 240_000)
public void testClusterPools() throws Exception {
Map<String, String> properties = ImmutableMap.<String, String>builder().put("task.verbose-stats", "true").put("experimental.reserved-pool-enabled", "true").build();
try (DistributedQueryRunner queryRunner = createQueryRunner(TINY_SESSION, properties)) {
// Reserve all the memory
QueryId fakeQueryId = new QueryId("fake");
for (TestingPrestoServer server : queryRunner.getServers()) {
for (MemoryPool pool : server.getLocalMemoryManager().getPools()) {
assertTrue(pool.tryReserve(fakeQueryId, "test", pool.getMaxBytes()));
}
}
List<Future<?>> queryFutures = new ArrayList<>();
for (int i = 0; i < 2; i++) {
queryFutures.add(executor.submit(() -> queryRunner.execute("SELECT COUNT(*), clerk FROM orders GROUP BY clerk")));
}
ClusterMemoryManager memoryManager = queryRunner.getCoordinator().getClusterMemoryManager();
ClusterMemoryPool reservedPool;
while ((reservedPool = memoryManager.getPools().get(RESERVED_POOL)) == null) {
MILLISECONDS.sleep(10);
}
ClusterMemoryPool generalPool = memoryManager.getPools().get(GENERAL_POOL);
assertNotNull(generalPool);
// Wait for the queries to start running and get assigned to the expected pools
while (generalPool.getAssignedQueries() != 1 || reservedPool.getAssignedQueries() != 1 || generalPool.getBlockedNodes() != 2 || reservedPool.getBlockedNodes() != 2) {
MILLISECONDS.sleep(10);
}
// Make sure the queries are blocked
List<BasicQueryInfo> currentQueryInfos = queryRunner.getCoordinator().getQueryManager().getQueries();
for (BasicQueryInfo info : currentQueryInfos) {
assertFalse(info.getState().isDone());
}
assertEquals(currentQueryInfos.size(), 2);
// Check that the pool information propagated to the query objects
assertNotEquals(currentQueryInfos.get(0).getMemoryPool(), currentQueryInfos.get(1).getMemoryPool());
while (!currentQueryInfos.stream().allMatch(TestMemoryManager::isBlockedWaitingForMemory)) {
MILLISECONDS.sleep(10);
currentQueryInfos = queryRunner.getCoordinator().getQueryManager().getQueries();
for (BasicQueryInfo info : currentQueryInfos) {
assertFalse(info.getState().isDone());
}
}
// Release the memory in the reserved pool
for (TestingPrestoServer server : queryRunner.getServers()) {
Optional<MemoryPool> reserved = server.getLocalMemoryManager().getReservedPool();
assertTrue(reserved.isPresent());
// Free up the entire pool
reserved.get().free(fakeQueryId, "test", reserved.get().getMaxBytes());
assertTrue(reserved.get().getFreeBytes() > 0);
}
// This also checks that the query in the general pool is successfully moved to the reserved pool.
for (Future<?> query : queryFutures) {
query.get();
}
for (BasicQueryInfo info : queryRunner.getCoordinator().getQueryManager().getQueries()) {
assertEquals(info.getState(), FINISHED);
}
// Make sure we didn't leak any memory on the workers
for (TestingPrestoServer worker : queryRunner.getServers()) {
Optional<MemoryPool> reserved = worker.getLocalMemoryManager().getReservedPool();
assertTrue(reserved.isPresent());
assertEquals(reserved.get().getMaxBytes(), reserved.get().getFreeBytes());
MemoryPool general = worker.getLocalMemoryManager().getGeneralPool();
// Free up the memory we reserved earlier
general.free(fakeQueryId, "test", general.getMaxBytes());
assertEquals(general.getMaxBytes(), general.getFreeBytes());
}
}
}
use of io.prestosql.tests.DistributedQueryRunner in project hetu-core by openlookeng.
the class TestMemoryManager method testOutOfMemoryKiller.
@Test(timeOut = 240_000, expectedExceptions = ExecutionException.class, expectedExceptionsMessageRegExp = ".*Query killed because the cluster is out of memory. Please try again in a few minutes.")
public void testOutOfMemoryKiller() throws Exception {
Map<String, String> properties = ImmutableMap.<String, String>builder().put("task.verbose-stats", "true").put("query.low-memory-killer.delay", "5s").put("query.low-memory-killer.policy", "total-reservation").put("experimental.reserved-pool-enabled", "true").build();
try (DistributedQueryRunner queryRunner = createQueryRunner(TINY_SESSION, properties)) {
// Reserve all the memory
QueryId fakeQueryId = new QueryId("fake");
for (TestingPrestoServer server : queryRunner.getServers()) {
for (MemoryPool pool : server.getLocalMemoryManager().getPools()) {
assertTrue(pool.tryReserve(fakeQueryId, "test", pool.getMaxBytes()));
}
}
List<Future<?>> queryFutures = new ArrayList<>();
for (int i = 0; i < 2; i++) {
queryFutures.add(executor.submit(() -> queryRunner.execute("SELECT COUNT(*), clerk FROM orders GROUP BY clerk")));
}
// Wait for one of the queries to die
waitForQueryToBeKilled(queryRunner);
// Release the memory in the reserved pool
for (TestingPrestoServer server : queryRunner.getServers()) {
Optional<MemoryPool> reserved = server.getLocalMemoryManager().getReservedPool();
assertTrue(reserved.isPresent());
// Free up the entire pool
reserved.get().free(fakeQueryId, "test", reserved.get().getMaxBytes());
assertTrue(reserved.get().getFreeBytes() > 0);
}
for (Future<?> query : queryFutures) {
query.get();
}
}
}
Aggregations