Examples with DistributedQueryRunner com.facebook.presto.tests.DistributedQueryRunner used on opensource projects

Example 86 with DistributedQueryRunner

use of com.facebook.presto.tests.DistributedQueryRunner in project presto by prestodb.

the class HiveQueryRunner method main.

public static void main(String[] args) throws Exception {
    // You need to add "--user user" to your CLI for your queries to work
    Logging.initialize();
    Optional<Path> baseDataDir = Optional.empty();
    if (args.length > 0) {
        if (args.length != 1) {
            log.error("usage: HiveQueryRunner [baseDataDir]\n");
            log.error("       [baseDataDir] is a local directory under which you want the hive_data directory to be created.]\n");
            System.exit(1);
        }
        File baseDataDirFile = new File(args[0]);
        if (baseDataDirFile.exists()) {
            if (!baseDataDirFile.isDirectory()) {
                log.error("Error: " + baseDataDirFile.getAbsolutePath() + " is not a directory.");
                System.exit(1);
            } else if (!baseDataDirFile.canRead() || !baseDataDirFile.canWrite()) {
                log.error("Error: " + baseDataDirFile.getAbsolutePath() + " is not readable/writable.");
                System.exit(1);
            }
        } else {
            // be able to create directory for it. e.g. "/aaa/bbb" is not creatable because path "/" is not writable.
            while (!baseDataDirFile.exists()) {
                baseDataDirFile = baseDataDirFile.getParentFile();
            }
            if (!baseDataDirFile.canRead() || !baseDataDirFile.canWrite()) {
                log.error("Error: The ancestor directory " + baseDataDirFile.getAbsolutePath() + " is not readable/writable.");
                System.exit(1);
            }
        }
        baseDataDir = Optional.of(baseDataDirFile.toPath());
    }
    DistributedQueryRunner queryRunner = createQueryRunner(TpchTable.getTables(), ImmutableMap.of("http-server.http.port", "8080"), baseDataDir);
    Thread.sleep(10);
    Logger log = Logger.get(DistributedQueryRunner.class);
    log.info("======== SERVER STARTED ========");
    log.info("\n====\n%s\n====", queryRunner.getCoordinator().getBaseUrl());
}

Example 87 with DistributedQueryRunner

use of com.facebook.presto.tests.DistributedQueryRunner in project presto by prestodb.

the class AccumuloQueryRunner method createAccumuloQueryRunner.

public static synchronized DistributedQueryRunner createAccumuloQueryRunner(Map<String, String> extraProperties) throws Exception {
    DistributedQueryRunner queryRunner = new DistributedQueryRunner(createSession(), 4, extraProperties);
    queryRunner.installPlugin(new TpchPlugin());
    queryRunner.createCatalog("tpch", "tpch");
    queryRunner.installPlugin(new AccumuloPlugin());
    Map<String, String> accumuloProperties = ImmutableMap.<String, String>builder().put(AccumuloConfig.INSTANCE, connector.getInstance().getInstanceName()).put(AccumuloConfig.ZOOKEEPERS, connector.getInstance().getZooKeepers()).put(AccumuloConfig.USERNAME, MAC_USER).put(AccumuloConfig.PASSWORD, MAC_PASSWORD).put(AccumuloConfig.ZOOKEEPER_METADATA_ROOT, "/presto-accumulo-test").build();
    queryRunner.createCatalog("accumulo", "accumulo", accumuloProperties);
    if (!tpchLoaded) {
        copyTpchTables(queryRunner, "tpch", TINY_SCHEMA_NAME, createSession(), TpchTable.getTables());
        connector.tableOperations().addSplits("tpch.orders", ImmutableSortedSet.of(new Text(new LexicoderRowSerializer().encode(BIGINT, 7500L))));
        tpchLoaded = true;
    }
    return queryRunner;
}

Example 88 with DistributedQueryRunner

use of com.facebook.presto.tests.DistributedQueryRunner in project presto by prestodb.

the class GeoQueryRunner method createQueryRunner.

private static DistributedQueryRunner createQueryRunner(Map<String, String> extraProperties) throws Exception {
    DistributedQueryRunner queryRunner = new DistributedQueryRunner(testSessionBuilder().build(), DEFAULT_WORKER_COUNT, extraProperties);
    queryRunner.installPlugin(new GeoPlugin());
    return queryRunner;
}

Example 89 with DistributedQueryRunner

use of com.facebook.presto.tests.DistributedQueryRunner in project presto by prestodb.

the class GeoQueryRunner method main.

public static void main(String[] args) throws Exception {
    Logging.initialize();
    DistributedQueryRunner queryRunner = createQueryRunner(ImmutableMap.of("http-server.http.port", "8080"));
    Logger log = Logger.get(GeoQueryRunner.class);
    log.info("======== SERVER STARTED ========");
    log.info("\n====\n%s\n====", queryRunner.getCoordinator().getBaseUrl());
}

Example 90 with DistributedQueryRunner

use of com.facebook.presto.tests.DistributedQueryRunner in project presto by prestodb.

the class TestHiveLogicalPlanner method testMaterializedTooManyMissingPartitions.

@Test
public void testMaterializedTooManyMissingPartitions() {
    String table = "orders_partitioned_not_materialized";
    String view = "orders_partitioned_view_not_materialized";
    QueryRunner queryRunner = getQueryRunner();
    try {
        queryRunner.execute(format("CREATE TABLE %s WITH (partitioned_by = ARRAY['ds']) AS " + "SELECT orderkey, orderpriority, '2020-01-01' as ds FROM orders WHERE orderkey < 1000 " + "UNION ALL " + "SELECT orderkey, orderpriority, '2019-01-02' as ds FROM orders WHERE orderkey >= 1000 and orderkey < 2000 " + "UNION ALL " + "SELECT orderkey, orderpriority, '2019-02-02' as ds FROM orders WHERE orderkey >= 2000 and orderkey < 3000 " + "UNION ALL " + "SELECT orderkey, orderpriority, '2019-03-02' as ds FROM orders WHERE orderkey >= 3000 and orderkey < 4000 " + "UNION ALL " + "SELECT orderkey, orderpriority, '2019-04-02' as ds FROM orders WHERE orderkey >= 4000 and orderkey < 5000 " + "UNION ALL " + "SELECT orderkey, orderpriority, '2019-05-02' as ds FROM orders WHERE orderkey >= 5000 and orderkey < 6000 " + "UNION ALL " + "SELECT orderkey, orderpriority, '2019-06-02' as ds FROM orders WHERE orderkey >= 6000 and orderkey < 7000 " + "UNION ALL " + "SELECT orderkey, orderpriority, '2019-07-02' as ds FROM orders WHERE orderkey >= 7000 and orderkey < 8000 ", table));
        assertUpdate(format("CREATE MATERIALIZED VIEW %s WITH (partitioned_by = ARRAY['ds']) " + "AS SELECT orderkey, orderpriority, ds FROM %s", view, table));
        assertTrue(getQueryRunner().tableExists(getQueryRunner().getDefaultSession(), view));
        assertUpdate(format("REFRESH MATERIALIZED VIEW %s WHERE ds = '2020-01-01'", view), 255);
        assertUpdate(format("REFRESH MATERIALIZED VIEW %s WHERE ds = '2019-01-02'", view), 248);
        assertUpdate(format("REFRESH MATERIALIZED VIEW %s WHERE ds = '2019-02-02'", view), 248);
        String viewQuery = format("SELECT orderkey from %s where orderkey < 10000 ORDER BY orderkey", view);
        String baseQuery = format("SELECT orderkey from %s where orderkey < 10000 ORDER BY orderkey", table);
        MaterializedResult viewTable = computeActual(viewQuery);
        MaterializedResult baseTable = computeActual(baseQuery);
        assertEquals(viewTable, baseTable);
        // assert that when missing partition count > threshold, fallback to using base table to satisfy view query
        Session session = Session.builder(getQueryRunner().getDefaultSession()).setCatalogSessionProperty(HIVE_CATALOG, MATERIALIZED_VIEW_MISSING_PARTITIONS_THRESHOLD, Integer.toString(2)).build();
        assertPlan(session, viewQuery, anyTree(filter("orderkey < BIGINT'10000'", PlanMatchPattern.constrainedTableScan(table, ImmutableMap.of(), ImmutableMap.of("orderkey", "orderkey")))));
        // assert that when count of missing partition  <= threshold, use available partitions from view
        session = Session.builder(getQueryRunner().getDefaultSession()).setCatalogSessionProperty(HIVE_CATALOG, MATERIALIZED_VIEW_MISSING_PARTITIONS_THRESHOLD, Integer.toString(100)).build();
        assertPlan(session, viewQuery, anyTree(filter("orderkey < BIGINT'10000'", PlanMatchPattern.constrainedTableScan(table, ImmutableMap.of("ds", multipleValues(createVarcharType(10), utf8Slices("2019-03-02", "2019-04-02", "2019-05-02", "2019-06-02", "2019-07-02"))), ImmutableMap.of("orderkey", "orderkey"))), filter("orderkey_17 < BIGINT'10000'", PlanMatchPattern.constrainedTableScan(view, ImmutableMap.of("ds", multipleValues(createVarcharType(10), utf8Slices("2020-01-01", "2019-01-02", "2019-02-02"))), ImmutableMap.of("orderkey_17", "orderkey")))));
        // if there are too many missing partitions, the optimization rewrite should not happen
        session = Session.builder(getQueryRunner().getDefaultSession()).setSystemProperty(QUERY_OPTIMIZATION_WITH_MATERIALIZED_VIEW_ENABLED, "true").setCatalogSessionProperty(HIVE_CATALOG, MATERIALIZED_VIEW_MISSING_PARTITIONS_THRESHOLD, Integer.toString(2)).build();
        setReferencedMaterializedViews((DistributedQueryRunner) queryRunner, table, ImmutableList.of(view));
        assertPlan(session, baseQuery, anyTree(filter("orderkey < BIGINT'10000'", PlanMatchPattern.constrainedTableScan(table, ImmutableMap.of(), ImmutableMap.of("orderkey", "orderkey")))));
    } finally {
        queryRunner.execute("DROP MATERIALIZED VIEW IF EXISTS " + view);
        queryRunner.execute("DROP TABLE IF EXISTS " + table);
    }
}