Examples with BatchSchema org.apache.drill.exec.record.BatchSchema used on opensource projects

Example 16 with BatchSchema

use of org.apache.drill.exec.record.BatchSchema in project drill by apache.

the class RowSetTest method testLongRW.

private void testLongRW() {
    BatchSchema batchSchema = new SchemaBuilder().add("col", MinorType.BIGINT).build();
    SingleRowSet rs = fixture.rowSetBuilder(batchSchema).add(0L).add(Long.MAX_VALUE).add(Long.MIN_VALUE).build();
    RowSetReader reader = rs.reader();
    assertTrue(reader.next());
    assertEquals(0, reader.column(0).getLong());
    assertTrue(reader.next());
    assertEquals(Long.MAX_VALUE, reader.column(0).getLong());
    assertTrue(reader.next());
    assertEquals(Long.MIN_VALUE, reader.column(0).getLong());
    assertFalse(reader.next());
    rs.clear();
}

Example 17 with BatchSchema

use of org.apache.drill.exec.record.BatchSchema in project drill by apache.

the class RowSetTest method testMapSchema.

@Test
public void testMapSchema() {
    BatchSchema batchSchema = new SchemaBuilder().add("c", MinorType.INT).addMap("a").addNullable("b", MinorType.VARCHAR).add("d", MinorType.INT).addMap("e").add("f", MinorType.VARCHAR).buildMap().add("g", MinorType.INT).buildMap().add("h", MinorType.BIGINT).build();
    RowSetSchema schema = new RowSetSchema(batchSchema);
    // Access schema: flattened with maps removed
    FlattenedSchema access = schema.flatAccess();
    assertEquals(6, access.count());
    crossCheck(access, 0, "c", MinorType.INT);
    crossCheck(access, 1, "a.b", MinorType.VARCHAR);
    crossCheck(access, 2, "a.d", MinorType.INT);
    crossCheck(access, 3, "a.e.f", MinorType.VARCHAR);
    crossCheck(access, 4, "a.g", MinorType.INT);
    crossCheck(access, 5, "h", MinorType.BIGINT);
    // Should have two maps.
    assertEquals(2, access.mapCount());
    assertEquals("a", access.map(0).getName());
    assertEquals("e", access.map(1).getName());
    assertEquals(0, access.mapIndex("a"));
    assertEquals(1, access.mapIndex("a.e"));
    // Verify physical schema: should mirror the schema created above.
    PhysicalSchema physical = schema.physical();
    assertEquals(3, physical.count());
    assertEquals("c", physical.column(0).field().getName());
    assertEquals("c", physical.column(0).fullName());
    assertFalse(physical.column(0).isMap());
    assertNull(physical.column(0).mapSchema());
    assertEquals("a", physical.column(1).field().getName());
    assertEquals("a", physical.column(1).fullName());
    assertTrue(physical.column(1).isMap());
    assertNotNull(physical.column(1).mapSchema());
    assertEquals("h", physical.column(2).field().getName());
    assertEquals("h", physical.column(2).fullName());
    assertFalse(physical.column(2).isMap());
    assertNull(physical.column(2).mapSchema());
    PhysicalSchema aSchema = physical.column(1).mapSchema();
    assertEquals(4, aSchema.count());
    assertEquals("b", aSchema.column(0).field().getName());
    assertEquals("a.b", aSchema.column(0).fullName());
    assertEquals("d", aSchema.column(1).field().getName());
    assertEquals("e", aSchema.column(2).field().getName());
    assertEquals("g", aSchema.column(3).field().getName());
    PhysicalSchema eSchema = aSchema.column(2).mapSchema();
    assertEquals(1, eSchema.count());
    assertEquals("f", eSchema.column(0).field().getName());
    assertEquals("a.e.f", eSchema.column(0).fullName());
}

Example 18 with BatchSchema

use of org.apache.drill.exec.record.BatchSchema in project drill by apache.

the class RowSetTest method testSmallIntRW.

private void testSmallIntRW() {
    BatchSchema batchSchema = new SchemaBuilder().add("col", MinorType.SMALLINT).build();
    SingleRowSet rs = fixture.rowSetBuilder(batchSchema).add(0).add(Short.MAX_VALUE).add(Short.MIN_VALUE).build();
    RowSetReader reader = rs.reader();
    assertTrue(reader.next());
    assertEquals(0, reader.column(0).getInt());
    assertTrue(reader.next());
    assertEquals(Short.MAX_VALUE, reader.column(0).getInt());
    assertTrue(reader.next());
    assertEquals(Short.MIN_VALUE, reader.column(0).getInt());
    assertFalse(reader.next());
    rs.clear();
}

Example 19 with BatchSchema

use of org.apache.drill.exec.record.BatchSchema in project drill by apache.

the class PartitionSenderRootExec method sendEmptyBatch.

public void sendEmptyBatch(boolean isLast) {
    BatchSchema schema = incoming.getSchema();
    if (schema == null) {
        // If the incoming batch has no schema (possible when there are no input records),
        // create an empty schema to avoid NPE.
        schema = BatchSchema.newBuilder().build();
    }
    FragmentHandle handle = context.getHandle();
    for (MinorFragmentEndpoint destination : popConfig.getDestinations()) {
        AccountingDataTunnel tunnel = context.getDataTunnel(destination.getEndpoint());
        FragmentWritableBatch writableBatch = FragmentWritableBatch.getEmptyBatchWithSchema(isLast, handle.getQueryId(), handle.getMajorFragmentId(), handle.getMinorFragmentId(), operator.getOppositeMajorFragmentId(), destination.getId(), schema);
        stats.startWait();
        try {
            tunnel.sendRecordBatch(writableBatch);
        } finally {
            stats.stopWait();
        }
    }
    stats.addLongStat(Metric.BATCHES_SENT, 1);
}

Example 20 with BatchSchema

use of org.apache.drill.exec.record.BatchSchema in project drill by apache.

the class IteratorValidatorBatchIterator method next.

@Override
public IterOutcome next() {
    logger.trace("[#{}; on {}]: next() called.", instNum, batchTypeName);
    final IterOutcome prevBatchState = batchState;
    try {
        // Check whether next() should even have been called in current state.
        if (null != exceptionState) {
            throw new IllegalStateException(String.format("next() [on #%d; %s] called again after it threw %s (after" + " returning %s).  Caller should not have called next() again.", instNum, batchTypeName, exceptionState, batchState));
        }
        // (Note:  This could use validationState.)
        if (batchState == NONE || batchState == STOP) {
            throw new IllegalStateException(String.format("next() [on #%d, %s] called again after it returned %s." + "  Caller should not have called next() again.", instNum, batchTypeName, batchState));
        }
        // Now get result from upstream next().
        batchState = incoming.next();
        logger.trace("[#{}; on {}]: incoming next() return: ({} ->) {}", instNum, batchTypeName, prevBatchState, batchState);
        // Check state transition and update high-level state.
        switch(batchState) {
            case OK_NEW_SCHEMA:
                // OK_NEW_SCHEMA is allowed at any time, except if terminated (checked
                // above).
                // OK_NEW_SCHEMA moves to have-seen-schema state.
                validationState = ValidationState.HAVE_SCHEMA;
                validateBatch();
                break;
            case OK:
                // (checked above).
                if (validationState != ValidationState.HAVE_SCHEMA) {
                    throw new IllegalStateException(String.format("next() returned %s without first returning %s [#%d, %s]", batchState, OK_NEW_SCHEMA, instNum, batchTypeName));
                }
                validateBatch();
                // OK doesn't change high-level state.
                break;
            case NONE:
                // already terminated (checked above).
                if (validationState != ValidationState.HAVE_SCHEMA) {
                    throw new IllegalStateException(String.format("next() returned %s without first returning %s [#%d, %s]", batchState, OK_NEW_SCHEMA, instNum, batchTypeName));
                }
                // NONE moves to terminal high-level state.
                validationState = ValidationState.TERMINAL;
                break;
            case STOP:
                // STOP is allowed at any time, except if already terminated (checked
                // above).
                // STOP moves to terminal high-level state.
                validationState = ValidationState.TERMINAL;
                break;
            case NOT_YET:
            case OUT_OF_MEMORY:
                // NOT_YET and OUT_OF_MEMORY OK don't change high-level state.
                break;
            default:
                throw new AssertionError("Unhandled new " + IterOutcome.class.getSimpleName() + " value " + batchState);
        }
        // Validate schema when available.
        if (batchState == OK || batchState == OK_NEW_SCHEMA) {
            final BatchSchema prevLastSchema = lastSchema;
            final BatchSchema prevLastNewSchema = lastNewSchema;
            lastSchema = incoming.getSchema();
            if (batchState == OK_NEW_SCHEMA) {
                lastNewSchema = lastSchema;
            }
            if (logger.isTraceEnabled()) {
                logger.trace("[#{}; on {}]: incoming next() return: #records = {}, " + "\n  schema:" + "\n    {}, " + "\n  prev. new ({}):" + "\n    {}", instNum, batchTypeName, incoming.getRecordCount(), lastSchema, lastSchema.equals(prevLastNewSchema) ? "equal" : "not equal", prevLastNewSchema);
            }
            if (lastSchema == null) {
                throw new IllegalStateException(String.format("Incoming batch [#%d, %s] has a null schema. This is not allowed.", instNum, batchTypeName));
            }
            if (lastSchema.getFieldCount() == 0) {
                throw new IllegalStateException(String.format("Incoming batch [#%d, %s] has an empty schema. This is not allowed.", instNum, batchTypeName));
            }
            if (incoming.getRecordCount() > MAX_BATCH_SIZE) {
                throw new IllegalStateException(String.format("Incoming batch [#%d, %s] has size %d, which is beyond the" + " limit of %d", instNum, batchTypeName, incoming.getRecordCount(), MAX_BATCH_SIZE));
            }
            if (VALIDATE_VECTORS) {
                VectorValidator.validate(incoming);
            }
        }
        return batchState;
    } catch (RuntimeException | Error e) {
        exceptionState = e;
        logger.trace("[#{}, on {}]: incoming next() exception: ({} ->) {}", instNum, batchTypeName, prevBatchState, exceptionState, exceptionState);
        throw e;
    }
}