Examples with MaterializedField org.apache.drill.exec.record.MaterializedField used on opensource projects

Example 56 with MaterializedField

use of org.apache.drill.exec.record.MaterializedField in project drill by apache.

the class VectorUtil method showVectorAccessibleContent.

public static void showVectorAccessibleContent(VectorAccessible va, int[] columnWidths) {
    int width = 0;
    int columnIndex = 0;
    List<String> columns = Lists.newArrayList();
    List<String> formats = Lists.newArrayList();
    for (VectorWrapper<?> vw : va) {
        int columnWidth = getColumnWidth(columnWidths, columnIndex);
        width += columnWidth + 2;
        formats.add("| %-" + columnWidth + "s");
        MaterializedField field = vw.getValueVector().getField();
        columns.add(field.getPath() + "<" + field.getType().getMinorType() + "(" + field.getType().getMode() + ")" + ">");
        columnIndex++;
    }
    int rows = va.getRecordCount();
    System.out.println(rows + " row(s):");
    for (int row = 0; row < rows; row++) {
        // header, every 50 rows.
        if (row % 50 == 0) {
            System.out.println(StringUtils.repeat("-", width + 1));
            columnIndex = 0;
            for (String column : columns) {
                int columnWidth = getColumnWidth(columnWidths, columnIndex);
                System.out.printf(formats.get(columnIndex), column.length() <= columnWidth ? column : column.substring(0, columnWidth - 1));
                columnIndex++;
            }
            System.out.printf("|\n");
            System.out.println(StringUtils.repeat("-", width + 1));
        }
        // column values
        columnIndex = 0;
        for (VectorWrapper<?> vw : va) {
            int columnWidth = getColumnWidth(columnWidths, columnIndex);
            Object o = vw.getValueVector().getAccessor().getObject(row);
            String cellString;
            if (o instanceof byte[]) {
                cellString = DrillStringUtils.toBinaryString((byte[]) o);
            } else {
                cellString = DrillStringUtils.escapeNewLines(String.valueOf(o));
            }
            System.out.printf(formats.get(columnIndex), cellString.length() <= columnWidth ? cellString : cellString.substring(0, columnWidth - 1));
            columnIndex++;
        }
        System.out.printf("|\n");
    }
    if (rows > 0) {
        System.out.println(StringUtils.repeat("-", width + 1));
    }
    for (VectorWrapper<?> vw : va) {
        vw.clear();
    }
}

Example 57 with MaterializedField

use of org.apache.drill.exec.record.MaterializedField in project drill by apache.

the class BaseRepeatedValueVector method addOrGetVector.

@Override
public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor) {
    boolean created = false;
    if (vector == DEFAULT_DATA_VECTOR && descriptor.getType().getMinorType() != TypeProtos.MinorType.LATE) {
        final MaterializedField field = descriptor.withName(DATA_VECTOR_NAME).getField();
        vector = BasicTypeHelper.getNewVector(field, allocator);
        // returned vector must have the same field
        assert field.equals(vector.getField());
        getField().addChild(field);
        created = true;
    }
    final TypeProtos.MajorType actual = vector.getField().getType();
    if (!actual.equals(descriptor.getType())) {
        final String msg = String.format("Inner vector type mismatch. Requested type: [%s], actual type: [%s]", descriptor.getType(), actual);
        throw new SchemaChangeRuntimeException(msg);
    }
    return new AddOrGetResult<>((T) vector, created);
}

Example 58 with MaterializedField

use of org.apache.drill.exec.record.MaterializedField in project drill by apache.

the class TestWriteToDisk method test.

@Test
@SuppressWarnings("static-method")
public void test() throws Exception {
    final List<ValueVector> vectorList = Lists.newArrayList();
    final DrillConfig config = DrillConfig.create();
    try (final RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
        final Drillbit bit = new Drillbit(config, serviceSet)) {
        bit.run();
        final DrillbitContext context = bit.getContext();
        final MaterializedField intField = MaterializedField.create("int", Types.required(TypeProtos.MinorType.INT));
        final MaterializedField binField = MaterializedField.create("binary", Types.required(TypeProtos.MinorType.VARBINARY));
        try (final IntVector intVector = (IntVector) TypeHelper.getNewVector(intField, context.getAllocator());
            final VarBinaryVector binVector = (VarBinaryVector) TypeHelper.getNewVector(binField, context.getAllocator())) {
            AllocationHelper.allocate(intVector, 4, 4);
            AllocationHelper.allocate(binVector, 4, 5);
            vectorList.add(intVector);
            vectorList.add(binVector);
            intVector.getMutator().setSafe(0, 0);
            binVector.getMutator().setSafe(0, "ZERO".getBytes());
            intVector.getMutator().setSafe(1, 1);
            binVector.getMutator().setSafe(1, "ONE".getBytes());
            intVector.getMutator().setSafe(2, 2);
            binVector.getMutator().setSafe(2, "TWO".getBytes());
            intVector.getMutator().setSafe(3, 3);
            binVector.getMutator().setSafe(3, "THREE".getBytes());
            intVector.getMutator().setValueCount(4);
            binVector.getMutator().setValueCount(4);
            VectorContainer container = new VectorContainer();
            container.addCollection(vectorList);
            container.setRecordCount(4);
            WritableBatch batch = WritableBatch.getBatchNoHVWrap(container.getRecordCount(), container, false);
            VectorAccessibleSerializable wrap = new VectorAccessibleSerializable(batch, context.getAllocator());
            Configuration conf = new Configuration();
            conf.set(FileSystem.FS_DEFAULT_NAME_KEY, FileSystem.DEFAULT_FS);
            final VectorAccessibleSerializable newWrap = new VectorAccessibleSerializable(context.getAllocator());
            try (final FileSystem fs = FileSystem.get(conf)) {
                final File tempDir = Files.createTempDir();
                tempDir.deleteOnExit();
                final Path path = new Path(tempDir.getAbsolutePath(), "drillSerializable");
                try (final FSDataOutputStream out = fs.create(path)) {
                    wrap.writeToStream(out);
                    out.close();
                }
                try (final FSDataInputStream in = fs.open(path)) {
                    newWrap.readFromStream(in);
                }
            }
            final VectorAccessible newContainer = newWrap.get();
            for (VectorWrapper<?> w : newContainer) {
                try (ValueVector vv = w.getValueVector()) {
                    int values = vv.getAccessor().getValueCount();
                    for (int i = 0; i < values; i++) {
                        final Object o = vv.getAccessor().getObject(i);
                        if (o instanceof byte[]) {
                            System.out.println(new String((byte[]) o));
                        } else {
                            System.out.println(o);
                        }
                    }
                }
            }
        }
    }
}

Example 59 with MaterializedField

use of org.apache.drill.exec.record.MaterializedField in project drill by apache.

the class DrillColumnMetaDataListTest method setUp.

@Before
public void setUp() throws Exception {
    emptyList = new DrillColumnMetaDataList();
    // Create mock columns
    final MaterializedField exampleIntField = mock(MaterializedField.class);
    MajorType exampleIntType = MajorType.newBuilder().setMinorType(MinorType.INT).build();
    when(exampleIntField.getPath()).thenReturn("/path/to/testInt");
    when(exampleIntField.getType()).thenReturn(exampleIntType);
    when(exampleIntField.getDataMode()).thenReturn(DataMode.OPTIONAL);
    final MaterializedField exampleStringField = mock(MaterializedField.class);
    MajorType exampleStringType = MajorType.newBuilder().setMinorType(MinorType.VARCHAR).build();
    when(exampleStringField.getPath()).thenReturn("/path/to/testString");
    when(exampleStringField.getType()).thenReturn(exampleStringType);
    when(exampleStringField.getDataMode()).thenReturn(DataMode.REQUIRED);
    oneElementList = new DrillColumnMetaDataList();
    BatchSchema oneElementSchema = mock(BatchSchema.class);
    when(oneElementSchema.getFieldCount()).thenReturn(1);
    doAnswer(new Answer<MaterializedField>() {

        @Override
        public MaterializedField answer(InvocationOnMock invocationOnMock) throws Throwable {
            Integer index = (Integer) invocationOnMock.getArguments()[0];
            if (index == 0) {
                return exampleIntField;
            }
            return null;
        }
    }).when(oneElementSchema).getColumn(Mockito.anyInt());
    List<Class<?>> oneClassList = new ArrayList<>();
    oneClassList.add(Integer.class);
    oneElementList.updateColumnMetaData("testCatalog", "testSchema", "testTable", oneElementSchema, oneClassList);
    twoElementList = new DrillColumnMetaDataList();
    BatchSchema twoElementSchema = mock(BatchSchema.class);
    when(twoElementSchema.getFieldCount()).thenReturn(2);
    doAnswer(new Answer<MaterializedField>() {

        @Override
        public MaterializedField answer(InvocationOnMock invocationOnMock) throws Throwable {
            Integer index = (Integer) invocationOnMock.getArguments()[0];
            if (index == 0) {
                return exampleIntField;
            } else if (index == 1) {
                return exampleStringField;
            }
            return null;
        }
    }).when(twoElementSchema).getColumn(Mockito.anyInt());
    List<Class<?>> twoClassList = new ArrayList<>();
    twoClassList.add(Integer.class);
    twoClassList.add(String.class);
    twoElementList.updateColumnMetaData("testCatalog", "testSchema", "testTable", twoElementSchema, twoClassList);
}

Example 60 with MaterializedField

use of org.apache.drill.exec.record.MaterializedField in project drill by apache.

the class TestValueVector method testRepeatedIntVector.

@Test
public void testRepeatedIntVector() {
    final MaterializedField field = MaterializedField.create(EMPTY_SCHEMA_PATH, RepeatedIntHolder.TYPE);
    // Create a new value vector.
    final RepeatedIntVector vector1 = new RepeatedIntVector(field, allocator);
    // Populate the vector.
    // some tricksy primes
    final int[] values = { 2, 3, 5, 7, 11, 13, 17, 19, 23, 27 };
    final int nRecords = 7;
    final int nElements = values.length;
    vector1.allocateNew(nRecords, nRecords * nElements);
    final RepeatedIntVector.Mutator mutator = vector1.getMutator();
    for (int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
        mutator.startNewValue(recordIndex);
        for (int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
            mutator.add(recordIndex, recordIndex * values[elementIndex]);
        }
    }
    mutator.setValueCount(nRecords);
    // Verify the contents.
    final RepeatedIntVector.Accessor accessor1 = vector1.getAccessor();
    assertEquals(nRecords, accessor1.getValueCount());
    for (int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
        for (int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
            final int value = accessor1.get(recordIndex, elementIndex);
            assertEquals(recordIndex * values[elementIndex], value);
        }
    }
    /* TODO(cwestin)
the interface to load has changed
    // Serialize, reify, and verify.
    final DrillBuf[] buffers1 = vector1.getBuffers(false);
    final DrillBuf buffer1 = combineBuffers(allocator, buffers1);
    final RepeatedIntVector vector2 = new RepeatedIntVector(field, allocator);
    vector2.load(nRecords, nRecords * nElements, buffer1);

    final RepeatedIntVector.Accessor accessor2 = vector2.getAccessor();
    for(int recordIndex = 0; recordIndex < nRecords; ++recordIndex) {
      for(int elementIndex = 0; elementIndex < nElements; ++elementIndex) {
        final int value = accessor2.get(recordIndex, elementIndex);
        assertEquals(accessor1.get(recordIndex,  elementIndex), value);
      }
    }
*/
    vector1.close();
/* TODO(cwestin)
    vector2.close();
    buffer1.release();
*/
}