Examples with MinorType org.apache.drill.common.types.TypeProtos.MinorType used on opensource projects

Example 31 with MinorType

use of org.apache.drill.common.types.TypeProtos.MinorType in project drill by apache.

the class MetadataHandlerBatch method getResultSetLoaderForMetadata.

private ResultSetLoader getResultSetLoaderForMetadata(BaseMetadata baseMetadata) {
    SchemaBuilder schemaBuilder = new SchemaBuilder().addNullable(MetastoreAnalyzeConstants.LOCATION_FIELD, MinorType.VARCHAR);
    for (String segmentColumn : popConfig.getContext().segmentColumns()) {
        schemaBuilder.addNullable(segmentColumn, MinorType.VARCHAR);
    }
    baseMetadata.getColumnsStatistics().entrySet().stream().sorted(Comparator.comparing(e -> e.getKey().getRootSegmentPath())).forEach(entry -> {
        for (StatisticsKind<?> statisticsKind : AnalyzeColumnUtils.COLUMN_STATISTICS_FUNCTIONS.keySet()) {
            MinorType type = AnalyzeColumnUtils.COLUMN_STATISTICS_TYPES.get(statisticsKind);
            type = type != null ? type : entry.getValue().getComparatorType();
            schemaBuilder.addNullable(AnalyzeColumnUtils.getColumnStatisticsFieldName(entry.getKey().getRootSegmentPath(), statisticsKind), type);
        }
    });
    for (StatisticsKind<?> statisticsKind : AnalyzeColumnUtils.META_STATISTICS_FUNCTIONS.keySet()) {
        schemaBuilder.addNullable(AnalyzeColumnUtils.getMetadataStatisticsFieldName(statisticsKind), AnalyzeColumnUtils.COLUMN_STATISTICS_TYPES.get(statisticsKind));
    }
    schemaBuilder.addMapArray(MetastoreAnalyzeConstants.COLLECTED_MAP_FIELD).resumeSchema();
    if (metadataType == MetadataType.SEGMENT) {
        schemaBuilder.addArray(MetastoreAnalyzeConstants.LOCATIONS_FIELD, MinorType.VARCHAR);
    }
    if (metadataType == MetadataType.ROW_GROUP) {
        schemaBuilder.addNullable(columnNamesOptions.rowGroupIndex(), MinorType.VARCHAR);
        schemaBuilder.addNullable(columnNamesOptions.rowGroupStart(), MinorType.VARCHAR);
        schemaBuilder.addNullable(columnNamesOptions.rowGroupLength(), MinorType.VARCHAR);
    }
    schemaBuilder.addNullable(MetastoreAnalyzeConstants.SCHEMA_FIELD, MinorType.VARCHAR).addNullable(columnNamesOptions.lastModifiedTime(), MinorType.VARCHAR).add(MetastoreAnalyzeConstants.METADATA_TYPE, MinorType.VARCHAR);
    ResultSetLoaderImpl.ResultSetOptions options = new ResultSetOptionBuilder().readerSchema(schemaBuilder.buildSchema()).build();
    return new ResultSetLoaderImpl(container.getAllocator(), options);
}

Example 32 with MinorType

use of org.apache.drill.common.types.TypeProtos.MinorType in project drill by apache.

the class TestDrillbitResilience method assertDrillbitsOk.

/**
 * Check that all the drillbits are ok.
 * <p/>
 * <p>The current implementation does this by counting the number of drillbits using a query.
 */
private static void assertDrillbitsOk() {
    SingleRowListener listener = new SingleRowListener() {

        private final BufferAllocator bufferAllocator = RootAllocatorFactory.newRoot(cluster.config());

        private final RecordBatchLoader loader = new RecordBatchLoader(bufferAllocator);

        @Override
        public void rowArrived(QueryDataBatch queryResultBatch) {
            // load the single record
            final QueryData queryData = queryResultBatch.getHeader();
            loader.load(queryData.getDef(), queryResultBatch.getData());
            assertEquals(1, loader.getRecordCount());
            // there should only be one column
            final BatchSchema batchSchema = loader.getSchema();
            assertEquals(1, batchSchema.getFieldCount());
            // the column should be an integer
            final MaterializedField countField = batchSchema.getColumn(0);
            final MinorType fieldType = countField.getType().getMinorType();
            assertEquals(MinorType.BIGINT, fieldType);
            // get the column value
            final VectorWrapper<?> vw = loader.iterator().next();
            final Object obj = vw.getValueVector().getAccessor().getObject(0);
            assertTrue(obj instanceof Long);
            final Long countValue = (Long) obj;
            // assume this means all the drillbits are still ok
            assertEquals(cluster.drillbits().size(), countValue.intValue());
            loader.clear();
        }

        @Override
        public void cleanup() {
            loader.clear();
            DrillAutoCloseables.closeNoChecked(bufferAllocator);
        }
    };
    try {
        QueryTestUtil.testWithListener(client.client(), QueryType.SQL, "select count(*) from sys.memory", listener);
        listener.waitForCompletion();
        QueryState state = listener.getQueryState();
        assertSame(state, QueryState.COMPLETED, () -> String.format("QueryState should be COMPLETED (and not %s).", state));
        assertTrue(listener.getErrorList().isEmpty(), "There should not be any errors when checking if Drillbits are OK");
    } catch (final Exception e) {
        throw new RuntimeException("Couldn't query active drillbits", e);
    } finally {
        logger.debug("Cleanup listener");
        listener.cleanup();
    }
    logger.debug("Drillbits are ok.");
}

Example 33 with MinorType

use of org.apache.drill.common.types.TypeProtos.MinorType in project drill by apache.

the class UnionWriterImpl method addMember.

/**
 * Add a column writer to an existing union writer. Used for implementations
 * that support "live" schema evolution: column discovery while writing.
 * The corresponding metadata must already have been added to the schema.
 * Called by the shim's <tt>addMember</tt> to do writer-level tasks.
 *
 * @param writer the column writer to add
 */
protected void addMember(AbstractObjectWriter writer) {
    final MinorType type = writer.schema().type();
    if (!variantSchema().hasType(type)) {
        variantSchema().addType(writer.schema());
    }
    writer.events().bindIndex(index);
    if (state != State.IDLE) {
        writer.events().startWrite();
        if (state == State.IN_ROW) {
            writer.events().startRow();
        }
    }
}

Example 34 with MinorType

use of org.apache.drill.common.types.TypeProtos.MinorType in project drill by apache.

the class ColumnReaderFactory method buildColumnReader.

public static BaseScalarReader buildColumnReader(VectorAccessor va) {
    MajorType major = va.type();
    MinorType type = major.getMinorType();
    switch(type) {
        case GENERIC_OBJECT:
        case LATE:
        case NULL:
        case LIST:
        case MAP:
        case DICT:
            throw new UnsupportedOperationException(type.toString());
        default:
            return newAccessor(type, requiredReaders);
    }
}

Example 35 with MinorType

use of org.apache.drill.common.types.TypeProtos.MinorType in project drill by apache.

the class HDF5BatchReader method projectDataset.

/**
 * Writes one row of data in a metadata query. The number of dimensions here
 * is n+1. So if the actual dataset is a 1D column, it will be written as a list.
 * This is function is only called in metadata queries as the schema is not
 * known in advance. If the datasize is greater than 16MB, the function does
 * not project the dataset
 *
 * @param rowWriter
 *          The rowWriter to which the data will be written
 * @param datapath
 *          The datapath from which the data will be read
 */
private void projectDataset(RowSetLoader rowWriter, String datapath) {
    String fieldName = HDF5Utils.getNameFromPath(datapath);
    Dataset dataset = hdfFile.getDatasetByPath(datapath);
    // If the dataset is larger than 16MB, do not project the dataset
    if (dataset.getSizeInBytes() > MAX_DATASET_SIZE) {
        logger.warn("Dataset {} is greater than 16MB.  Data will be truncated in Metadata view.", datapath);
    }
    int[] dimensions = dataset.getDimensions();
    // Case for single dimensional data
    if (dimensions.length == 1) {
        MinorType currentDataType = HDF5Utils.getDataType(dataset.getDataType());
        Object data;
        try {
            data = dataset.getData();
        } catch (Exception e) {
            logger.debug("Error reading {}", datapath);
            return;
        }
        assert currentDataType != null;
        // Skip null datasets
        if (data == null) {
            return;
        }
        switch(currentDataType) {
            case GENERIC_OBJECT:
                logger.warn("Couldn't read {}", datapath);
                break;
            case VARCHAR:
                String[] stringData = (String[]) data;
                writeStringListColumn(rowWriter, fieldName, stringData);
                break;
            case TIMESTAMP:
                long[] longList = (long[]) data;
                writeTimestampListColumn(rowWriter, fieldName, longList);
                break;
            case INT:
                int[] intList = (int[]) data;
                writeIntListColumn(rowWriter, fieldName, intList);
                break;
            case SMALLINT:
                short[] shortList = (short[]) data;
                writeSmallIntColumn(rowWriter, fieldName, shortList);
                break;
            case TINYINT:
                byte[] byteList = (byte[]) data;
                writeByteListColumn(rowWriter, fieldName, byteList);
                break;
            case FLOAT4:
                float[] tempFloatList = (float[]) data;
                writeFloat4ListColumn(rowWriter, fieldName, tempFloatList);
                break;
            case FLOAT8:
                double[] tempDoubleList = (double[]) data;
                writeFloat8ListColumn(rowWriter, fieldName, tempDoubleList);
                break;
            case BIGINT:
                long[] tempBigIntList = (long[]) data;
                writeLongListColumn(rowWriter, fieldName, tempBigIntList);
                break;
            case MAP:
                try {
                    getAndMapCompoundData(datapath, hdfFile, rowWriter);
                } catch (Exception e) {
                    throw UserException.dataReadError().message("Error writing Compound Field: " + e.getMessage()).addContext(errorContext).build(logger);
                }
                break;
            default:
                // Case for data types that cannot be read
                logger.warn("{} not implemented.", currentDataType.name());
        }
    } else if (dimensions.length == 2) {
        // Case for 2D data sets.  These are projected as lists of lists or maps of maps
        int cols = dimensions[1];
        int rows = dimensions[0];
        // TODO Add Boolean, Small and TinyInt data types
        switch(HDF5Utils.getDataType(dataset.getDataType())) {
            case INT:
                int[][] colData = (int[][]) dataset.getData();
                mapIntMatrixField(colData, cols, rows, rowWriter);
                break;
            case FLOAT4:
                float[][] floatData = (float[][]) dataset.getData();
                mapFloatMatrixField(floatData, cols, rows, rowWriter);
                break;
            case FLOAT8:
                double[][] doubleData = (double[][]) dataset.getData();
                mapDoubleMatrixField(doubleData, cols, rows, rowWriter);
                break;
            case BIGINT:
                long[][] longData = (long[][]) dataset.getData();
                mapBigIntMatrixField(longData, cols, rows, rowWriter);
                break;
            default:
                logger.warn("{} not implemented.", HDF5Utils.getDataType(dataset.getDataType()));
        }
    } else if (dimensions.length > 2) {
        // Case for data sets with dimensions > 2
        int cols = dimensions[1];
        int rows = dimensions[0];
        switch(HDF5Utils.getDataType(dataset.getDataType())) {
            case INT:
                int[][] intMatrix = HDF5Utils.toIntMatrix((Object[]) dataset.getData());
                mapIntMatrixField(intMatrix, cols, rows, rowWriter);
                break;
            case FLOAT4:
                float[][] floatData = HDF5Utils.toFloatMatrix((Object[]) dataset.getData());
                mapFloatMatrixField(floatData, cols, rows, rowWriter);
                break;
            case FLOAT8:
                double[][] doubleData = HDF5Utils.toDoubleMatrix((Object[]) dataset.getData());
                mapDoubleMatrixField(doubleData, cols, rows, rowWriter);
                break;
            case BIGINT:
                long[][] longData = HDF5Utils.toLongMatrix((Object[]) dataset.getData());
                mapBigIntMatrixField(longData, cols, rows, rowWriter);
                break;
            default:
                logger.warn("{} not implemented.", HDF5Utils.getDataType(dataset.getDataType()));
        }
    }
}