Examples with TestBuilder org.apache.drill.test.TestBuilder used on opensource projects

Example 81 with TestBuilder

use of org.apache.drill.test.TestBuilder in project drill by apache.

the class TestLocalExchange method testGroupByMultiFields.

@Test
public void testGroupByMultiFields() throws Exception {
    // Test multifield hash generation
    test("ALTER SESSION SET `planner.slice_target`=1");
    test("ALTER SESSION SET `planner.enable_mux_exchange`=" + true);
    test("ALTER SESSION SET `planner.enable_demux_exchange`=" + false);
    final String groupByMultipleQuery = String.format("SELECT dept_id, mng_id, some_id, count(*) as numEmployees FROM dfs.`%s` e GROUP BY dept_id, mng_id, some_id", EMPT_TABLE);
    final String[] groupByMultipleQueryBaselineColumns = new String[] { "dept_id", "mng_id", "some_id", "numEmployees" };
    final int numOccurrances = NUM_EMPLOYEES / NUM_DEPTS;
    final String plan = getPlanInString("EXPLAIN PLAN FOR " + groupByMultipleQuery, JSON_FORMAT);
    jsonExchangeOrderChecker(plan, false, 1, "hash32asdouble\\(.*, hash32asdouble\\(.*, hash32asdouble\\(.*\\) \\) \\) ");
    // Run the query and verify the output
    final TestBuilder testBuilder = testBuilder().sqlQuery(groupByMultipleQuery).unOrdered().baselineColumns(groupByMultipleQueryBaselineColumns);
    for (int i = 0; i < NUM_DEPTS; i++) {
        testBuilder.baselineValues(new Object[] { (long) i, (long) 0, (long) 0, (long) numOccurrances });
    }
    testBuilder.go();
}

Example 82 with TestBuilder

use of org.apache.drill.test.TestBuilder in project drill by apache.

the class TestFlatten method testFlatten_Drill2162_simple.

@Test
public void testFlatten_Drill2162_simple() throws Exception {
    List<Long> inputList = Lists.newArrayList();
    String jsonRecord = "{ \"int_list\" : [";
    final int listSize = 30;
    for (int i = 1; i < listSize; i++) {
        jsonRecord += i + ", ";
        inputList.add((long) i);
    }
    jsonRecord += listSize + "] }";
    inputList.add((long) listSize);
    int numRecords = 3000;
    new TestConstantFolding.SmallFileCreator(pathDir).setRecord(jsonRecord).createFiles(1, numRecords, "json");
    List<JsonStringHashMap<String, Object>> data = Lists.newArrayList(mapOf("int_list", inputList));
    List<JsonStringHashMap<String, Object>> result = flatten(data, "int_list");
    TestBuilder builder = testBuilder().sqlQuery("select flatten(int_list) as int_list from dfs.`%s/bigfile/bigfile.json`", TEST_DIR).unOrdered().baselineColumns("int_list");
    for (int i = 0; i < numRecords; i++) {
        for (JsonStringHashMap<String, Object> record : result) {
            builder.baselineValues(record.get("int_list"));
        }
    }
    builder.go();
}

Example 83 with TestBuilder

use of org.apache.drill.test.TestBuilder in project drill by apache.

the class TestMergeJoinWithSchemaChanges method testNumericTypes.

@Test
public void testNumericTypes() throws Exception {
    // First create data for numeric types.
    // left side int and float vs right side float
    BufferedWriter writer = new BufferedWriter(new FileWriter(new File(leftDir, "l1.json")));
    for (int i = 0; i < 5000; ++i) {
        writer.write(String.format("{ \"kl\" : %d , \"vl\": %d }\n", i, i));
    }
    writer.close();
    writer = new BufferedWriter(new FileWriter(new File(leftDir, "l2.json")));
    for (int i = 1000; i < 6000; ++i) {
        writer.write(String.format("{ \"kl\" : %f , \"vl\": %f }\n", (float) i, (float) i));
    }
    writer.close();
    // right side is int and float
    writer = new BufferedWriter(new FileWriter(new File(rightDir, "r1.json")));
    for (int i = 2000; i < 7000; ++i) {
        writer.write(String.format("{ \"kr\" : %d , \"vr\": %d }\n", i, i));
    }
    writer.close();
    writer = new BufferedWriter(new FileWriter(new File(rightDir, "r2.json")));
    for (int i = 3000; i < 8000; ++i) {
        writer.write(String.format("{ \"kr\" : %f, \"vr\": %f }\n", (float) i, (float) i));
    }
    writer.close();
    // INNER JOIN
    String query = String.format("select * from dfs.`%s` L %s join dfs.`%s` R on L.kl=R.kr", LEFT_DIR, "inner", RIGHT_DIR);
    TestBuilder builder = testBuilder().sqlQuery(query).optionSettingQueriesForTestQuery("alter session set `planner.enable_hashjoin` = false; alter session set `exec.enable_union_type` = true").unOrdered().baselineColumns("kl", "vl", "kr", "vr");
    for (long i = 2000; i < 3000; ++i) {
        builder.baselineValues(i, i, i, i);
        builder.baselineValues((double) i, (double) i, i, i);
    }
    for (long i = 3000; i < 5000; ++i) {
        builder.baselineValues(i, i, i, i);
        builder.baselineValues(i, i, (double) i, (double) i);
        builder.baselineValues((double) i, (double) i, i, i);
        builder.baselineValues((double) i, (double) i, (double) i, (double) i);
    }
    for (long i = 5000; i < 6000; ++i) {
        builder.baselineValues((double) i, (double) i, i, i);
        builder.baselineValues((double) i, (double) i, (double) i, (double) i);
    }
    builder.go();
    // LEFT JOIN
    query = String.format("select * from dfs.`%s` L %s join dfs.`%s` R on L.kl=R.kr", LEFT_DIR, "left", RIGHT_DIR);
    builder = testBuilder().sqlQuery(query).optionSettingQueriesForTestQuery("alter session set `planner.enable_hashjoin` = false; alter session set `exec.enable_union_type` = true").unOrdered().baselineColumns("kl", "vl", "kr", "vr");
    for (long i = 0; i < 2000; ++i) {
        builder.baselineValues(i, i, null, null);
    }
    for (long i = 1000; i < 2000; ++i) {
        builder.baselineValues((double) i, (double) i, null, null);
    }
    for (long i = 2000; i < 3000; ++i) {
        builder.baselineValues(i, i, i, i);
        builder.baselineValues((double) i, (double) i, i, i);
    }
    for (long i = 3000; i < 5000; ++i) {
        builder.baselineValues(i, i, i, i);
        builder.baselineValues(i, i, (double) i, (double) i);
        builder.baselineValues((double) i, (double) i, i, i);
        builder.baselineValues((double) i, (double) i, (double) i, (double) i);
    }
    for (long i = 5000; i < 6000; ++i) {
        builder.baselineValues((double) i, (double) i, i, i);
        builder.baselineValues((double) i, (double) i, (double) i, (double) i);
    }
    builder.go();
}

Example 84 with TestBuilder

use of org.apache.drill.test.TestBuilder in project drill by apache.

the class TestCorruptParquetDateCorrection method testReadPartitionedOnCorrectDates.

/**
 * Test reading a directory full of partitioned parquet files with dates, these files have a drill version
 * number of "1.9.0-SNAPSHOT" and parquet-writer version number of "2" in their footers, so we can be certain
 * they do not have corruption. The option to disable the correction is passed, but it will not change the result
 * in the case where we are certain correction is NOT needed. For more info see DRILL-4203.
 */
@Test
public void testReadPartitionedOnCorrectDates() throws Exception {
    try {
        for (String selection : new String[] { "*", "date_col" }) {
            // for sanity, try reading all partitions without a filter
            TestBuilder builder = testBuilder().sqlQuery("select %s from table(dfs.`%s` (type => 'parquet', autoCorrectCorruptDates => false))", selection, CORRECT_PARTITIONED_DATES_1_9_PATH).unOrdered().baselineColumns("date_col");
            addDateBaselineValues(builder);
            builder.go();
            String query = format("select %s from table(dfs.`%s` (type => 'parquet', autoCorrectCorruptDates => false))" + " where date_col = date '1970-01-01'", selection, CORRECT_PARTITIONED_DATES_1_9_PATH);
            // verify that pruning is actually taking place
            testPlanMatchingPatterns(query, new String[] { "numFiles=1" }, null);
            // read with a filter on the partition column
            testBuilder().sqlQuery(query).unOrdered().baselineColumns("date_col").baselineValues(LocalDate.of(1970, 1, 1)).go();
        }
    } finally {
        resetAllSessionOptions();
    }
}

Example 85 with TestBuilder

use of org.apache.drill.test.TestBuilder in project drill by apache.

the class TestCorruptParquetDateCorrection method readMixedCorruptedAndCorrectDates.

/**
 * Read a directory with parquet files where some have corrupted dates, see DRILL-4203.
 * @throws Exception
 */
private void readMixedCorruptedAndCorrectDates() throws Exception {
    // for bad values) to set the flag that the values are corrupt
    for (String selection : new String[] { "*", "date_col" }) {
        TestBuilder builder = testBuilder().sqlQuery("select %s from dfs.`%s`", selection, MIXED_CORRUPTED_AND_CORRECT_DATES_PATH).unOrdered().baselineColumns("date_col");
        for (int i = 0; i < 4; i++) {
            addDateBaselineValues(builder);
        }
        builder.go();
    }
}