Examples with ColumnSchema org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema used on opensource projects

Example 71 with ColumnSchema

use of org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema in project carbondata by apache.

the class BlockletDataMap method init.

@Override
public void init(DataMapModel dataMapModel) throws IOException, MemoryException {
    long startTime = System.currentTimeMillis();
    assert (dataMapModel instanceof BlockletDataMapModel);
    BlockletDataMapModel blockletDataMapInfo = (BlockletDataMapModel) dataMapModel;
    DataFileFooterConverter fileFooterConverter = new DataFileFooterConverter();
    List<DataFileFooter> indexInfo = fileFooterConverter.getIndexInfo(blockletDataMapInfo.getFilePath(), blockletDataMapInfo.getFileData());
    Path path = new Path(blockletDataMapInfo.getFilePath());
    byte[] filePath = path.getParent().toString().getBytes(CarbonCommonConstants.DEFAULT_CHARSET);
    byte[] fileName = path.getName().toString().getBytes(CarbonCommonConstants.DEFAULT_CHARSET);
    byte[] segmentId = blockletDataMapInfo.getSegmentId().getBytes(CarbonCommonConstants.DEFAULT_CHARSET);
    DataMapRowImpl summaryRow = null;
    byte[] schemaBinary = null;
    // below 2 variables will be used for fetching the relative blocklet id. Relative blocklet ID
    // is id assigned to a blocklet within a part file
    String tempFilePath = null;
    int relativeBlockletId = 0;
    for (DataFileFooter fileFooter : indexInfo) {
        if (segmentProperties == null) {
            List<ColumnSchema> columnInTable = fileFooter.getColumnInTable();
            schemaBinary = convertSchemaToBinary(columnInTable);
            columnCardinality = fileFooter.getSegmentInfo().getColumnCardinality();
            segmentProperties = new SegmentProperties(columnInTable, columnCardinality);
            createSchema(segmentProperties);
            createSummarySchema(segmentProperties, schemaBinary, filePath, fileName, segmentId);
        }
        TableBlockInfo blockInfo = fileFooter.getBlockInfo().getTableBlockInfo();
        BlockMetaInfo blockMetaInfo = blockletDataMapInfo.getBlockMetaInfoMap().get(blockInfo.getFilePath());
        // the file exists physically or not
        if (blockMetaInfo != null) {
            if (fileFooter.getBlockletList() == null) {
                // This is old store scenario, here blocklet information is not available in index file so
                // load only block info
                summaryRow = loadToUnsafeBlock(fileFooter, segmentProperties, blockInfo.getFilePath(), summaryRow, blockMetaInfo);
            } else {
                // blocklet ID will start from 0 again only when part file path is changed
                if (null == tempFilePath || !tempFilePath.equals(blockInfo.getFilePath())) {
                    tempFilePath = blockInfo.getFilePath();
                    relativeBlockletId = 0;
                }
                summaryRow = loadToUnsafe(fileFooter, segmentProperties, blockInfo.getFilePath(), summaryRow, blockMetaInfo, relativeBlockletId);
                // this is done because relative blocklet id need to be incremented based on the
                // total number of blocklets
                relativeBlockletId += fileFooter.getBlockletList().size();
            }
        }
    }
    if (unsafeMemoryDMStore != null) {
        unsafeMemoryDMStore.finishWriting();
    }
    if (null != unsafeMemorySummaryDMStore) {
        addTaskSummaryRowToUnsafeMemoryStore(summaryRow, schemaBinary, filePath, fileName, segmentId);
        unsafeMemorySummaryDMStore.finishWriting();
    }
    LOGGER.info("Time taken to load blocklet datamap from file : " + dataMapModel.getFilePath() + "is " + (System.currentTimeMillis() - startTime));
}

Example 72 with ColumnSchema

use of org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema in project carbondata by apache.

the class ThriftWrapperSchemaConverterImpl method fromWrapperToExternalTableSchema.

/* (non-Javadoc)
   * convert from wrapper to external tableschema
   */
@Override
public org.apache.carbondata.format.TableSchema fromWrapperToExternalTableSchema(TableSchema wrapperTableSchema) {
    List<org.apache.carbondata.format.ColumnSchema> thriftColumnSchema = new ArrayList<org.apache.carbondata.format.ColumnSchema>();
    for (ColumnSchema wrapperColumnSchema : wrapperTableSchema.getListOfColumns()) {
        thriftColumnSchema.add(fromWrapperToExternalColumnSchema(wrapperColumnSchema));
    }
    org.apache.carbondata.format.SchemaEvolution schemaEvolution = fromWrapperToExternalSchemaEvolution(wrapperTableSchema.getSchemaEvalution());
    org.apache.carbondata.format.TableSchema externalTableSchema = new org.apache.carbondata.format.TableSchema(wrapperTableSchema.getTableId(), thriftColumnSchema, schemaEvolution);
    externalTableSchema.setTableProperties(wrapperTableSchema.getTableProperties());
    if (wrapperTableSchema.getBucketingInfo() != null) {
        externalTableSchema.setBucketingInfo(fromWrapperToExternalBucketingInfo(wrapperTableSchema.getBucketingInfo()));
    }
    if (wrapperTableSchema.getPartitionInfo() != null) {
        externalTableSchema.setPartitionInfo(fromWrapperToExternalPartitionInfo(wrapperTableSchema.getPartitionInfo()));
    }
    return externalTableSchema;
}

Example 73 with ColumnSchema

use of org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema in project carbondata by apache.

the class ThriftWrapperSchemaConverterImpl method fromExternalToWrapperColumnSchema.

/* (non-Javadoc)
   * convert from external to wrapper columnschema
   */
@Override
public ColumnSchema fromExternalToWrapperColumnSchema(org.apache.carbondata.format.ColumnSchema externalColumnSchema) {
    ColumnSchema wrapperColumnSchema = new ColumnSchema();
    wrapperColumnSchema.setColumnUniqueId(externalColumnSchema.getColumn_id());
    wrapperColumnSchema.setColumnName(externalColumnSchema.getColumn_name());
    wrapperColumnSchema.setColumnar(externalColumnSchema.isColumnar());
    wrapperColumnSchema.setDataType(fromExternalToWrapperDataType(externalColumnSchema.data_type, externalColumnSchema.precision, externalColumnSchema.scale));
    wrapperColumnSchema.setDimensionColumn(externalColumnSchema.isDimension());
    List<Encoding> encoders = new ArrayList<Encoding>();
    for (org.apache.carbondata.format.Encoding encoder : externalColumnSchema.getEncoders()) {
        encoders.add(fromExternalToWrapperEncoding(encoder));
    }
    wrapperColumnSchema.setEncodingList(encoders);
    wrapperColumnSchema.setNumberOfChild(externalColumnSchema.getNum_child());
    wrapperColumnSchema.setPrecision(externalColumnSchema.getPrecision());
    wrapperColumnSchema.setColumnGroup(externalColumnSchema.getColumn_group_id());
    wrapperColumnSchema.setScale(externalColumnSchema.getScale());
    wrapperColumnSchema.setDefaultValue(externalColumnSchema.getDefault_value());
    wrapperColumnSchema.setInvisible(externalColumnSchema.isInvisible());
    wrapperColumnSchema.setColumnReferenceId(externalColumnSchema.getColumnReferenceId());
    wrapperColumnSchema.setSchemaOrdinal(externalColumnSchema.getSchemaOrdinal());
    wrapperColumnSchema.setSortColumn(false);
    Map<String, String> properties = externalColumnSchema.getColumnProperties();
    if (properties != null) {
        String sortColumns = properties.get(CarbonCommonConstants.SORT_COLUMNS);
        if (sortColumns != null) {
            wrapperColumnSchema.setSortColumn(true);
        }
    }
    wrapperColumnSchema.setFunction(externalColumnSchema.getAggregate_function());
    List<org.apache.carbondata.format.ParentColumnTableRelation> parentColumnTableRelation = externalColumnSchema.getParentColumnTableRelations();
    if (null != parentColumnTableRelation) {
        wrapperColumnSchema.setParentColumnTableRelations(fromExtrenalToWrapperParentTableColumnRelations(parentColumnTableRelation));
    }
    return wrapperColumnSchema;
}

Example 74 with ColumnSchema

use of org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema in project carbondata by apache.

the class ThriftWrapperSchemaConverterImpl method fromWrapperToExternalPartitionInfo.

private org.apache.carbondata.format.PartitionInfo fromWrapperToExternalPartitionInfo(PartitionInfo wrapperPartitionInfo) {
    List<org.apache.carbondata.format.ColumnSchema> thriftColumnSchema = new ArrayList<org.apache.carbondata.format.ColumnSchema>();
    for (ColumnSchema wrapperColumnSchema : wrapperPartitionInfo.getColumnSchemaList()) {
        thriftColumnSchema.add(fromWrapperToExternalColumnSchema(wrapperColumnSchema));
    }
    org.apache.carbondata.format.PartitionInfo externalPartitionInfo = new org.apache.carbondata.format.PartitionInfo(thriftColumnSchema, fromWrapperToExternalPartitionType(wrapperPartitionInfo.getPartitionType()));
    externalPartitionInfo.setList_info(wrapperPartitionInfo.getListInfo());
    externalPartitionInfo.setRange_info(wrapperPartitionInfo.getRangeInfo());
    externalPartitionInfo.setNum_partitions(wrapperPartitionInfo.getNumPartitions());
    externalPartitionInfo.setMax_partition(wrapperPartitionInfo.getMaxPartitionId());
    externalPartitionInfo.setPartition_ids(wrapperPartitionInfo.getPartitionIds());
    return externalPartitionInfo;
}

Example 75 with ColumnSchema

use of org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema in project carbondata by apache.

the class BlockletDetailInfo method readColumnSchema.

/**
 * Read column schema from binary
 * @param schemaArray
 * @throws IOException
 */
public void readColumnSchema(byte[] schemaArray) throws IOException {
    // uncompress it.
    schemaArray = Snappy.uncompress(schemaArray);
    ByteArrayInputStream schemaStream = new ByteArrayInputStream(schemaArray);
    DataInput schemaInput = new DataInputStream(schemaStream);
    columnSchemas = new ArrayList<>();
    int size = schemaInput.readShort();
    for (int i = 0; i < size; i++) {
        ColumnSchema columnSchema = new ColumnSchema();
        columnSchema.readFields(schemaInput);
        columnSchemas.add(columnSchema);
    }
}