Examples with TestBuilder org.apache.drill.test.TestBuilder used on opensource projects

Example 51 with TestBuilder

use of org.apache.drill.test.TestBuilder in project drill by axbaretto.

the class TestCorruptParquetDateCorrection method testReadNewMetadataCacheFileOverOldAndNewFiles.

@Test
public void testReadNewMetadataCacheFileOverOldAndNewFiles() throws Exception {
    File meta = dirTestWatcher.copyResourceToRoot(PARQUET_4203.resolve("mixed_version_partitioned_metadata.requires_replace.txt"), MIXED_CORRUPTED_AND_CORRECT_PARTITIONED_FOLDER.resolve(Metadata.METADATA_FILENAME));
    dirTestWatcher.replaceMetaDataContents(meta, dirTestWatcher.getRootDir(), null);
    // for sanity, try reading all partitions without a filter
    TestBuilder builder = testBuilder().sqlQuery("select date_col from dfs.`%s`", MIXED_CORRUPTED_AND_CORRECT_PARTITIONED_FOLDER).unOrdered().baselineColumns("date_col");
    addDateBaselineValues(builder);
    addDateBaselineValues(builder);
    addDateBaselineValues(builder);
    builder.go();
    String query = format("select date_col from dfs.`%s` where date_col = date '1970-01-01'", MIXED_CORRUPTED_AND_CORRECT_PARTITIONED_FOLDER);
    // verify that pruning is actually taking place
    testPlanMatchingPatterns(query, new String[] { "numFiles=3", "usedMetadataFile=true" }, null);
    // read with a filter on the partition column
    testBuilder().sqlQuery(query).unOrdered().baselineColumns("date_col").baselineValues(new DateTime(1970, 1, 1, 0, 0)).baselineValues(new DateTime(1970, 1, 1, 0, 0)).baselineValues(new DateTime(1970, 1, 1, 0, 0)).go();
}

Example 52 with TestBuilder

use of org.apache.drill.test.TestBuilder in project drill by axbaretto.

the class TestCorruptParquetDateCorrection method readFilesWithUserDisabledAutoCorrection.

private void readFilesWithUserDisabledAutoCorrection() throws Exception {
    // for bad values) to set the flag that the values are corrupt
    for (String selection : new String[] { "*", "date_col" }) {
        TestBuilder builder = testBuilder().sqlQuery("select %s from table(dfs.`%s` (type => 'parquet', autoCorrectCorruptDates => false))", selection, MIXED_CORRUPTED_AND_CORRECT_DATES_PATH).unOrdered().baselineColumns("date_col");
        addDateBaselineValues(builder);
        addCorruptedDateBaselineValues(builder);
        addCorruptedDateBaselineValues(builder);
        addCorruptedDateBaselineValues(builder);
        builder.go();
    }
}

Example 53 with TestBuilder

use of org.apache.drill.test.TestBuilder in project drill by axbaretto.

the class TestExternalSort method testNewColumns.

private void testNewColumns(boolean testLegacy) throws Exception {
    final int record_count = 10000;
    final String tableDirName = "newColumns";
    {
        final BatchSchema schema = new SchemaBuilder().add("a", TypeProtos.MinorType.INT).add("b", TypeProtos.MinorType.INT).build();
        final RowSetBuilder rowSetBuilder = new RowSetBuilder(allocator, schema);
        for (int i = 0; i <= record_count; i += 2) {
            rowSetBuilder.addRow(i, i);
        }
        final RowSet rowSet = rowSetBuilder.build();
        final File tableFile = createTableFile(tableDirName, "a.json");
        new JsonFileBuilder(rowSet).build(tableFile);
        rowSet.clear();
    }
    {
        final BatchSchema schema = new SchemaBuilder().add("a", TypeProtos.MinorType.INT).add("c", TypeProtos.MinorType.INT).build();
        final RowSetBuilder rowSetBuilder = new RowSetBuilder(allocator, schema);
        for (int i = 1; i <= record_count; i += 2) {
            rowSetBuilder.addRow(i, i);
        }
        final RowSet rowSet = rowSetBuilder.build();
        final File tableFile = createTableFile(tableDirName, "b.json");
        new JsonFileBuilder(rowSet).build(tableFile);
        rowSet.clear();
    }
    // Test framework currently doesn't handle changing schema (i.e. new
    // columns) on the client side
    TestBuilder builder = testBuilder().sqlQuery("select a, b, c from dfs.`%s` order by a desc", tableDirName).ordered().optionSettingQueriesForTestQuery(getOptions(testLegacy)).baselineColumns("a", "b", "c");
    for (int i = record_count; i >= 0; ) {
        builder.baselineValues((long) i, (long) i--, null);
        if (i >= 0) {
            builder.baselineValues((long) i, null, (long) i--);
        }
    }
    builder.go();
    // TODO: Useless test: just dumps to console
    test("select * from dfs.`%s` order by a desc", tableDirName);
}

Example 54 with TestBuilder

use of org.apache.drill.test.TestBuilder in project drill by axbaretto.

the class TestExternalSort method testNumericAndStringTypes.

private void testNumericAndStringTypes(boolean testLegacy) throws Exception {
    final int record_count = 10000;
    final String tableDirName = "numericAndStringTypes";
    {
        final BatchSchema schema = new SchemaBuilder().add("a", Types.required(TypeProtos.MinorType.INT)).build();
        final RowSetBuilder rowSetBuilder = new RowSetBuilder(allocator, schema);
        for (int i = 0; i <= record_count; i += 2) {
            rowSetBuilder.addRow(i);
        }
        final RowSet rowSet = rowSetBuilder.build();
        final File tableFile = createTableFile(tableDirName, "a.json");
        new JsonFileBuilder(rowSet).build(tableFile);
        rowSet.clear();
    }
    {
        final BatchSchema schema = new SchemaBuilder().add("a", Types.required(TypeProtos.MinorType.INT)).build();
        final RowSetBuilder rowSetBuilder = new RowSetBuilder(allocator, schema);
        for (int i = 1; i <= record_count; i += 2) {
            rowSetBuilder.addRow(i);
        }
        final RowSet rowSet = rowSetBuilder.build();
        final File tableFile = createTableFile(tableDirName, "b.json");
        new JsonFileBuilder(rowSet).setCustomFormatter("a", "\"%05d\"").build(tableFile);
        rowSet.clear();
    }
    TestBuilder builder = testBuilder().sqlQuery("select * from dfs.`%s` order by a desc", tableDirName).ordered().optionSettingQueriesForTestQuery(getOptions(testLegacy)).baselineColumns("a");
    // Strings come first because order by is desc
    for (int i = record_count; i >= 0; ) {
        i--;
        if (i >= 0) {
            builder.baselineValues(String.format("%05d", i--));
        }
    }
    for (int i = record_count; i >= 0; ) {
        builder.baselineValues((long) i--);
        i--;
    }
    builder.go();
}

Example 55 with TestBuilder

use of org.apache.drill.test.TestBuilder in project drill by axbaretto.

the class TestFlatten method testFlatten_Drill2162_simple.

@Test
public void testFlatten_Drill2162_simple() throws Exception {
    List<Long> inputList = Lists.newArrayList();
    String jsonRecord = "{ \"int_list\" : [";
    final int listSize = 30;
    for (int i = 1; i < listSize; i++) {
        jsonRecord += i + ", ";
        inputList.add((long) i);
    }
    jsonRecord += listSize + "] }";
    inputList.add((long) listSize);
    int numRecords = 3000;
    new TestConstantFolding.SmallFileCreator(pathDir).setRecord(jsonRecord).createFiles(1, numRecords, "json");
    @SuppressWarnings("unchecked") List<JsonStringHashMap<String, Object>> data = Lists.newArrayList(mapOf("int_list", inputList));
    List<JsonStringHashMap<String, Object>> result = flatten(data, "int_list");
    TestBuilder builder = testBuilder().sqlQuery("select flatten(int_list) as int_list from dfs.`%s/bigfile/bigfile.json`", TEST_DIR).unOrdered().baselineColumns("int_list");
    for (int i = 0; i < numRecords; i++) {
        for (JsonStringHashMap<String, Object> record : result) {
            builder.baselineValues(record.get("int_list"));
        }
    }
    builder.go();
}