Examples with BatchSchemaBuilder org.apache.drill.exec.record.BatchSchemaBuilder used on opensource projects

Example 46 with BatchSchemaBuilder

use of org.apache.drill.exec.record.BatchSchemaBuilder in project drill by apache.

the class TestRepeatedListAccessors method testSchema2DBatch.

/**
 * Test the case of a simple 2D array. Drill represents
 * this as two levels of materialized fields.
 */
@Test
public void testSchema2DBatch() {
    SchemaBuilder schemaBuilder = new SchemaBuilder().add("id", MinorType.INT).addRepeatedList("list2").addArray(MinorType.VARCHAR).resumeSchema();
    BatchSchema schema = new BatchSchemaBuilder().withSchemaBuilder(schemaBuilder).build();
    assertEquals(2, schema.getFieldCount());
    final MaterializedField list = schema.getColumn(1);
    assertEquals("list2", list.getName());
    assertEquals(MinorType.LIST, list.getType().getMinorType());
    assertEquals(DataMode.REPEATED, list.getType().getMode());
    assertEquals(1, list.getChildren().size());
    final MaterializedField inner = list.getChildren().iterator().next();
    assertEquals("list2", inner.getName());
    assertEquals(MinorType.VARCHAR, inner.getType().getMinorType());
    assertEquals(DataMode.REPEATED, inner.getType().getMode());
}

Example 47 with BatchSchemaBuilder

use of org.apache.drill.exec.record.BatchSchemaBuilder in project drill by apache.

the class TestRepeatedListAccessors method testSchema3DBatch.

@Test
public void testSchema3DBatch() {
    SchemaBuilder schemaBuilder = new SchemaBuilder().add("id", MinorType.INT).addRepeatedList("list2").addDimension().addArray(MinorType.VARCHAR).resumeList().resumeSchema();
    BatchSchema schema = new BatchSchemaBuilder().withSchemaBuilder(schemaBuilder).build();
    assertEquals(2, schema.getFieldCount());
    final MaterializedField list = schema.getColumn(1);
    assertEquals("list2", list.getName());
    assertEquals(MinorType.LIST, list.getType().getMinorType());
    assertEquals(DataMode.REPEATED, list.getType().getMode());
    assertEquals(1, list.getChildren().size());
    final MaterializedField child1 = list.getChildren().iterator().next();
    assertEquals("list2", child1.getName());
    assertEquals(MinorType.LIST, child1.getType().getMinorType());
    assertEquals(DataMode.REPEATED, child1.getType().getMode());
    assertEquals(1, child1.getChildren().size());
    final MaterializedField child2 = child1.getChildren().iterator().next();
    assertEquals("list2", child2.getName());
    assertEquals(MinorType.VARCHAR, child2.getType().getMinorType());
    assertEquals(DataMode.REPEATED, child2.getType().getMode());
    assertEquals(0, child2.getChildren().size());
}

Example 48 with BatchSchemaBuilder

use of org.apache.drill.exec.record.BatchSchemaBuilder in project drill by apache.

the class TestRepeatedListAccessors method testSchemaIncompleteBatch.

/**
 * Test the intermediate case in which a repeated list
 * does not yet have child type.
 */
@Test
public void testSchemaIncompleteBatch() {
    SchemaBuilder schemaBuilder = new SchemaBuilder().add("id", MinorType.INT).addRepeatedList("list2").resumeSchema();
    BatchSchema schema = new BatchSchemaBuilder().withSchemaBuilder(schemaBuilder).build();
    assertEquals(2, schema.getFieldCount());
    final MaterializedField list = schema.getColumn(1);
    assertEquals("list2", list.getName());
    assertEquals(MinorType.LIST, list.getType().getMinorType());
    assertEquals(DataMode.REPEATED, list.getType().getMode());
    assertTrue(list.getChildren().isEmpty());
}

Example 49 with BatchSchemaBuilder

use of org.apache.drill.exec.record.BatchSchemaBuilder in project drill by apache.

the class TestMiniPlan method testUnionFilter.

@Test
public void testUnionFilter() throws Exception {
    List<String> leftJsonBatches = Lists.newArrayList("[{\"a\": 5, \"b\" : 1 }]", "[{\"a\": 5, \"b\" : 5},{\"a\": 3, \"b\" : 8}]", "[{\"a\": 40, \"b\" : 3},{\"a\": 13, \"b\" : 100}]");
    List<String> rightJsonBatches = Lists.newArrayList("[{\"a\": 5, \"b\" : 10 }]", "[{\"a\": 50, \"b\" : 100}]");
    RecordBatch batch = new PopBuilder().physicalOperator(// Children list is provided through RecordBatch
    new UnionAll(Collections.<PhysicalOperator>emptyList())).addInputAsChild().physicalOperator(new Filter(null, parseExpr("a=5"), 1.0f)).addJsonScanAsChild().jsonBatches(leftJsonBatches).columnsToRead("a", "b").buildAddAsInput().buildAddAsInput().addInputAsChild().physicalOperator(new Filter(null, parseExpr("a=50"), 1.0f)).addJsonScanAsChild().jsonBatches(rightJsonBatches).columnsToRead("a", "b").buildAddAsInput().buildAddAsInput().build();
    SchemaBuilder schemaBuilder = new SchemaBuilder().addNullable("a", TypeProtos.MinorType.BIGINT).addNullable("b", TypeProtos.MinorType.BIGINT);
    BatchSchema expectedSchema = new BatchSchemaBuilder().withSchemaBuilder(schemaBuilder).withSVMode(BatchSchema.SelectionVectorMode.NONE).build();
    new MiniPlanTestBuilder().root(batch).expectSchema(expectedSchema).baselineValues(5l, 1l).baselineValues(5l, 5l).baselineValues(50l, 100l).go();
}

Example 50 with BatchSchemaBuilder

use of org.apache.drill.exec.record.BatchSchemaBuilder in project drill by apache.

the class TestLoad method testMapSchemaChange.

@Test
public void testMapSchemaChange() throws SchemaChangeException {
    final BufferAllocator allocator = RootAllocatorFactory.newRoot(drillConfig);
    final RecordBatchLoader batchLoader = new RecordBatchLoader(allocator);
    // Initial schema: a: INT, m: MAP{}
    SchemaBuilder schemaBuilder1 = new SchemaBuilder().add("a", MinorType.INT).addMap("m").resumeSchema();
    BatchSchema schema1 = new BatchSchemaBuilder().withSchemaBuilder(schemaBuilder1).build();
    {
        assertTrue(loadBatch(allocator, batchLoader, schema1));
        assertTrue(schema1.isEquivalent(batchLoader.getSchema()));
        batchLoader.getContainer().zeroVectors();
    }
    // Same schema
    // Schema change: No
    {
        assertFalse(loadBatch(allocator, batchLoader, schema1));
        assertTrue(schema1.isEquivalent(batchLoader.getSchema()));
        batchLoader.getContainer().zeroVectors();
    }
    // Add column to map: a: INT, m: MAP{b: VARCHAR}
    // Schema change: Yes
    SchemaBuilder schemaBuilder2 = new SchemaBuilder().add("a", MinorType.INT).addMap("m").add("b", MinorType.VARCHAR).resumeSchema();
    BatchSchema schema2 = new BatchSchemaBuilder().withSchemaBuilder(schemaBuilder2).build();
    {
        assertTrue(loadBatch(allocator, batchLoader, schema2));
        assertTrue(schema2.isEquivalent(batchLoader.getSchema()));
        batchLoader.getContainer().zeroVectors();
    }
    // Same schema
    // Schema change: No
    {
        assertFalse(loadBatch(allocator, batchLoader, schema2));
        assertTrue(schema2.isEquivalent(batchLoader.getSchema()));
        batchLoader.getContainer().zeroVectors();
    }
    // Add column:  a: INT, m: MAP{b: VARCHAR, c: INT}
    // Schema change: Yes
    {
        SchemaBuilder schemaBuilder = new SchemaBuilder().add("a", MinorType.INT).addMap("m").add("b", MinorType.VARCHAR).add("c", MinorType.INT).resumeSchema();
        BatchSchema schema = new BatchSchemaBuilder().withSchemaBuilder(schemaBuilder).build();
        assertTrue(loadBatch(allocator, batchLoader, schema));
        assertTrue(schema.isEquivalent(batchLoader.getSchema()));
        batchLoader.getContainer().zeroVectors();
    }
    // Drop a column:  a: INT, m: MAP{b: VARCHAR}
    // Schema change: Yes
    {
        SchemaBuilder schemaBuilder = new SchemaBuilder().add("a", MinorType.INT).addMap("m").add("b", MinorType.VARCHAR).resumeSchema();
        BatchSchema schema = new BatchSchemaBuilder().withSchemaBuilder(schemaBuilder).build();
        assertTrue(loadBatch(allocator, batchLoader, schema));
        assertTrue(schema.isEquivalent(batchLoader.getSchema()));
        batchLoader.getContainer().zeroVectors();
    }
    // Change type:  a: INT, m: MAP{b: INT}
    // Schema change: Yes
    {
        SchemaBuilder schemaBuilder = new SchemaBuilder().add("a", MinorType.INT).addMap("m").add("b", MinorType.INT).resumeSchema();
        BatchSchema schema = new BatchSchemaBuilder().withSchemaBuilder(schemaBuilder).build();
        assertTrue(loadBatch(allocator, batchLoader, schema));
        assertTrue(schema.isEquivalent(batchLoader.getSchema()));
        batchLoader.getContainer().zeroVectors();
    }
    // Empty map: a: INT, m: MAP{}
    {
        assertTrue(loadBatch(allocator, batchLoader, schema1));
        assertTrue(schema1.isEquivalent(batchLoader.getSchema()));
        batchLoader.getContainer().zeroVectors();
    }
    // Drop map: a: INT
    {
        SchemaBuilder schemaBuilder = new SchemaBuilder().add("a", MinorType.INT);
        BatchSchema schema = new BatchSchemaBuilder().withSchemaBuilder(schemaBuilder).build();
        assertTrue(loadBatch(allocator, batchLoader, schema));
        assertTrue(schema.isEquivalent(batchLoader.getSchema()));
        batchLoader.getContainer().zeroVectors();
    }
    batchLoader.clear();
    allocator.close();
}