Examples with TupleMetadata org.apache.drill.exec.record.metadata.TupleMetadata used on opensource projects

Example 46 with TupleMetadata

use of org.apache.drill.exec.record.metadata.TupleMetadata in project drill by apache.

the class TestDirectConverter method testSpecialConversionType.

/**
 * Test the specialized types: conversation to/from string.
 */
@Test
public void testSpecialConversionType() {
    StandardConversions conversions = StandardConversions.builder().build();
    TupleMetadata schema = new SchemaBuilder().add("time", MinorType.TIME).add("date", MinorType.DATE).add("ts", MinorType.TIMESTAMP).add("interval", MinorType.INTERVAL).add("year", MinorType.INTERVALYEAR).add("day", MinorType.INTERVALDAY).add("int", MinorType.INT).add("bi", MinorType.BIGINT).add("str", MinorType.VARCHAR).buildSchema();
    ColumnMetadata timeCol = schema.metadata("time");
    ColumnMetadata dateCol = schema.metadata("date");
    ColumnMetadata tsCol = schema.metadata("ts");
    ColumnMetadata intervalCol = schema.metadata("interval");
    ColumnMetadata yearCol = schema.metadata("year");
    ColumnMetadata dayCol = schema.metadata("day");
    ColumnMetadata intCol = schema.metadata("int");
    ColumnMetadata bigIntCol = schema.metadata("bi");
    ColumnMetadata stringCol = schema.metadata("str");
    // TIME
    expect(ConversionType.NONE, conversions.analyze(timeCol, timeCol));
    expect(ConversionType.EXPLICIT, conversions.analyze(timeCol, stringCol));
    expect(ConversionType.EXPLICIT, conversions.analyze(stringCol, timeCol));
    expect(ConversionType.IMPLICIT, conversions.analyze(intCol, timeCol));
    expect(ConversionType.IMPLICIT, conversions.analyze(timeCol, intCol));
    // DATE
    expect(ConversionType.NONE, conversions.analyze(dateCol, dateCol));
    expect(ConversionType.EXPLICIT, conversions.analyze(dateCol, stringCol));
    expect(ConversionType.EXPLICIT, conversions.analyze(stringCol, dateCol));
    expect(ConversionType.IMPLICIT, conversions.analyze(bigIntCol, dateCol));
    expect(ConversionType.IMPLICIT, conversions.analyze(dateCol, bigIntCol));
    // TIMESTAMP
    expect(ConversionType.NONE, conversions.analyze(tsCol, tsCol));
    expect(ConversionType.EXPLICIT, conversions.analyze(tsCol, stringCol));
    expect(ConversionType.EXPLICIT, conversions.analyze(stringCol, tsCol));
    expect(ConversionType.IMPLICIT, conversions.analyze(bigIntCol, tsCol));
    expect(ConversionType.IMPLICIT, conversions.analyze(tsCol, bigIntCol));
    // INTERVAL
    expect(ConversionType.NONE, conversions.analyze(intervalCol, intervalCol));
    expect(ConversionType.EXPLICIT, conversions.analyze(intervalCol, stringCol));
    expect(ConversionType.EXPLICIT, conversions.analyze(stringCol, intervalCol));
    // INTERVALYEAR
    expect(ConversionType.NONE, conversions.analyze(yearCol, yearCol));
    expect(ConversionType.EXPLICIT, conversions.analyze(yearCol, stringCol));
    expect(ConversionType.EXPLICIT, conversions.analyze(stringCol, yearCol));
    // INTERVALDAY
    expect(ConversionType.NONE, conversions.analyze(dayCol, dayCol));
    expect(ConversionType.EXPLICIT, conversions.analyze(dayCol, stringCol));
    expect(ConversionType.EXPLICIT, conversions.analyze(stringCol, dayCol));
}

Example 47 with TupleMetadata

use of org.apache.drill.exec.record.metadata.TupleMetadata in project drill by apache.

the class TestDirectConverter method testImplicitConversionIntTruncation.

/**
 * The column accessors provide only int setters. For performance, the int value is
 * assumed to be of the correct range for the target column. If not, truncation of
 * the highest bytes occurs.
 * <p>
 * The assumption is, if the reader or other code expects that overflow might
 * occur, that code should be implemented in the client (or in a type conversion
 * shim), leaving the normal code path to optimize for the 99% of the cases where
 * the value is in the proper range.
 */
@Test
public void testImplicitConversionIntTruncation() {
    TupleMetadata schema = new SchemaBuilder().add("ti", MinorType.TINYINT).add("si", MinorType.SMALLINT).buildSchema();
    // Test allowed implicit conversions.
    RowSet actual = new RowSetBuilder(fixture.allocator(), schema).addRow(Byte.MAX_VALUE + 1, Short.MAX_VALUE + 1).addRow(Byte.MAX_VALUE + 2, Short.MAX_VALUE + 2).build();
    // Build the expected vector without a type converter.
    final SingleRowSet expected = fixture.rowSetBuilder(schema).addRow(Byte.MIN_VALUE, Short.MIN_VALUE).addRow(Byte.MIN_VALUE + 1, Short.MIN_VALUE + 1).build();
    RowSetUtilities.verify(expected, actual);
}

Example 48 with TupleMetadata

use of org.apache.drill.exec.record.metadata.TupleMetadata in project drill by apache.

the class TestDirectConverter method testDecimalOverflow.

@Test
public void testDecimalOverflow() {
    TupleMetadata outputSchema = new SchemaBuilder().add("id", MinorType.INT).add("dec", MinorType.VARDECIMAL, 4, 2).buildSchema();
    TupleMetadata inputSchema = new SchemaBuilder().add("id", MinorType.INT).add("dec", MinorType.VARCHAR).buildSchema();
    ConversionTestFixture testFixture = new ConversionTestFixture(fixture.allocator(), outputSchema);
    testFixture.createConvertersFor(inputSchema);
    try {
        testFixture.addRow(1, "1234567.89");
        fail();
    } catch (UserException e) {
    // Expected
    }
    testFixture.build().clear();
}

Example 49 with TupleMetadata

use of org.apache.drill.exec.record.metadata.TupleMetadata in project drill by apache.

the class TestOperatorRecordBatch method testSchemaChange.

@Test
public void testSchemaChange() {
    TupleMetadata schema = new SchemaBuilder().add("a", MinorType.INT).add("b", MinorType.VARCHAR).buildSchema();
    SingleRowSet rs = fixture.rowSetBuilder(schema).addRow(10, "fred").addRow(20, "wilma").build();
    VectorContainer container = rs.container();
    MockOperatorExec opExec = new MockOperatorExec(container);
    int schemaVersion = opExec.batchAccessor().schemaVersion();
    // Be tidy: start at 1.
    assertEquals(1, schemaVersion);
    // Changing data does not trigger schema change
    container.zeroVectors();
    opExec.batchAccessor.addBatch(container);
    assertEquals(schemaVersion, opExec.batchAccessor().schemaVersion());
    // Different container, same vectors, does not trigger a change
    VectorContainer c2 = new VectorContainer(fixture.allocator());
    for (VectorWrapper<?> vw : container) {
        c2.add(vw.getValueVector());
    }
    c2.buildSchema(SelectionVectorMode.NONE);
    opExec.batchAccessor.addBatch(c2);
    assertEquals(schemaVersion, opExec.batchAccessor().schemaVersion());
    opExec.batchAccessor.addBatch(container);
    assertEquals(schemaVersion, opExec.batchAccessor().schemaVersion());
    // Replacing a vector with another of the same type does trigger
    // a change.
    VectorContainer c3 = new VectorContainer(fixture.allocator());
    c3.add(container.getValueVector(0).getValueVector());
    c3.add(TypeHelper.getNewVector(container.getValueVector(1).getValueVector().getField(), fixture.allocator(), null));
    c3.buildSchema(SelectionVectorMode.NONE);
    opExec.batchAccessor.addBatch(c3);
    assertEquals(schemaVersion + 1, opExec.batchAccessor().schemaVersion());
    schemaVersion = opExec.batchAccessor().schemaVersion();
    // No change if same schema again
    opExec.batchAccessor.addBatch(c3);
    assertEquals(schemaVersion, opExec.batchAccessor().schemaVersion());
    // Adding a vector triggers a change
    MaterializedField c = SchemaBuilder.columnSchema("c", MinorType.INT, DataMode.OPTIONAL);
    c3.add(TypeHelper.getNewVector(c, fixture.allocator(), null));
    c3.buildSchema(SelectionVectorMode.NONE);
    opExec.batchAccessor.addBatch(c3);
    assertEquals(schemaVersion + 1, opExec.batchAccessor().schemaVersion());
    schemaVersion = opExec.batchAccessor().schemaVersion();
    // No change if same schema again
    opExec.batchAccessor.addBatch(c3);
    assertEquals(schemaVersion, opExec.batchAccessor().schemaVersion());
    // Removing a vector triggers a change
    c3.remove(c3.getValueVector(2).getValueVector());
    c3.buildSchema(SelectionVectorMode.NONE);
    assertEquals(2, c3.getNumberOfColumns());
    opExec.batchAccessor.addBatch(c3);
    assertEquals(schemaVersion + 1, opExec.batchAccessor().schemaVersion());
    schemaVersion = opExec.batchAccessor().schemaVersion();
    // Clean up
    opExec.close();
    c2.clear();
    c3.clear();
}

Example 50 with TupleMetadata

use of org.apache.drill.exec.record.metadata.TupleMetadata in project drill by apache.

the class TestFileScanFramework method testLateSchemaFileWildcards.

@Test
public void testLateSchemaFileWildcards() {
    // Create a mock reader, return two batches: one schema-only, another with data.
    MockLateSchemaReader reader = new MockLateSchemaReader();
    reader.batchLimit = 2;
    reader.returnDataOnFirst = false;
    // Create the scan operator
    FileScanFixtureBuilder builder = new FileScanFixtureBuilder();
    builder.projectAllWithAllImplicit(2);
    builder.addReader(reader);
    ScanFixture scanFixture = builder.build();
    ScanOperatorExec scan = scanFixture.scanOp;
    // Standard startup
    assertFalse(reader.openCalled);
    // First batch: build schema. The reader helps: it returns an
    // empty first batch.
    assertTrue(scan.buildSchema());
    assertTrue(reader.openCalled);
    assertEquals(1, reader.batchCount);
    assertEquals(0, scan.batchAccessor().rowCount());
    // Create the expected result.
    TupleMetadata expectedSchema = new SchemaBuilder().add("a", MinorType.INT).addNullable("b", MinorType.VARCHAR, 10).add(ScanTestUtils.FULLY_QUALIFIED_NAME_COL, MinorType.VARCHAR).add(ScanTestUtils.FILE_PATH_COL, MinorType.VARCHAR).add(ScanTestUtils.FILE_NAME_COL, MinorType.VARCHAR).add(ScanTestUtils.SUFFIX_COL, MinorType.VARCHAR).add(ScanTestUtils.LAST_MODIFIED_TIME_COL, MinorType.VARCHAR).addNullable(ScanTestUtils.PROJECT_METADATA_COL, MinorType.VARCHAR).addNullable(ScanTestUtils.partitionColName(0), MinorType.VARCHAR).addNullable(ScanTestUtils.partitionColName(1), MinorType.VARCHAR).addNullable(ScanTestUtils.partitionColName(2), MinorType.VARCHAR).buildSchema();
    SingleRowSet expected = fixture.rowSetBuilder(expectedSchema).addRow(30, "fred", fqn, pathToFile, MOCK_FILE_NAME, MOCK_SUFFIX, lastModifiedTime, null, MOCK_DIR0, MOCK_DIR1, null).addRow(40, "wilma", fqn, pathToFile, MOCK_FILE_NAME, MOCK_SUFFIX, lastModifiedTime, null, MOCK_DIR0, MOCK_DIR1, null).build();
    assertEquals(expected.batchSchema(), scan.batchAccessor().schema());
    // Next call, return with data.
    assertTrue(scan.next());
    RowSetUtilities.verify(expected, fixture.wrap(scan.batchAccessor().container()));
    // EOF
    assertFalse(scan.next());
    assertTrue(reader.closeCalled);
    assertEquals(0, scan.batchAccessor().rowCount());
    scanFixture.close();
}