Examples with UnsafeMemoryDMStore org.apache.carbondata.core.indexstore.UnsafeMemoryDMStore used on opensource projects

Example 1 with UnsafeMemoryDMStore

use of org.apache.carbondata.core.indexstore.UnsafeMemoryDMStore in project carbondata by apache.

the class BlockletDataMap method createSchema.

private void createSchema(SegmentProperties segmentProperties) throws MemoryException {
    List<CarbonRowSchema> indexSchemas = new ArrayList<>();
    // Index key
    indexSchemas.add(new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY));
    getMinMaxSchema(segmentProperties, indexSchemas);
    // for number of rows.
    indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.INT));
    // for table block path
    indexSchemas.add(new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY));
    // for number of pages.
    indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.SHORT));
    // for version number.
    indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.SHORT));
    // for schema updated time.
    indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.LONG));
    // for blocklet info
    indexSchemas.add(new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY));
    // for block footer offset.
    indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.LONG));
    // for locations
    indexSchemas.add(new CarbonRowSchema.VariableCarbonRowSchema(DataTypes.BYTE_ARRAY));
    // for relative blocklet id i.e. blocklet id that belongs to a particular part file.
    indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.SHORT));
    // for storing block length.
    indexSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.LONG));
    unsafeMemoryDMStore = new UnsafeMemoryDMStore(indexSchemas.toArray(new CarbonRowSchema[indexSchemas.size()]));
}

Example 2 with UnsafeMemoryDMStore

use of org.apache.carbondata.core.indexstore.UnsafeMemoryDMStore in project carbondata by apache.

the class BlockletDataMap method createSummarySchema.

/**
 * Creates the schema to store summary information or the information which can be stored only
 * once per datamap. It stores datamap level max/min of each column and partition information of
 * datamap
 * @param segmentProperties
 * @throws MemoryException
 */
private void createSummarySchema(SegmentProperties segmentProperties, byte[] schemaBinary, byte[] filePath, byte[] fileName, byte[] segmentId) throws MemoryException {
    List<CarbonRowSchema> taskMinMaxSchemas = new ArrayList<>();
    getMinMaxSchema(segmentProperties, taskMinMaxSchemas);
    // for storing column schema
    taskMinMaxSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.BYTE_ARRAY, schemaBinary.length));
    // for storing file path
    taskMinMaxSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.BYTE_ARRAY, filePath.length));
    // for storing file name
    taskMinMaxSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.BYTE_ARRAY, fileName.length));
    // for storing segmentid
    taskMinMaxSchemas.add(new CarbonRowSchema.FixedCarbonRowSchema(DataTypes.BYTE_ARRAY, segmentId.length));
    unsafeMemorySummaryDMStore = new UnsafeMemoryDMStore(taskMinMaxSchemas.toArray(new CarbonRowSchema[taskMinMaxSchemas.size()]));
}

Example 3 with UnsafeMemoryDMStore

use of org.apache.carbondata.core.indexstore.UnsafeMemoryDMStore in project carbondata by apache.

the class BlockIndex method convertToUnsafeDMStore.

/**
 * This method will convert safe to unsafe memory DM store
 */
public void convertToUnsafeDMStore() {
    if (memoryDMStore instanceof SafeMemoryDMStore) {
        UnsafeMemoryDMStore unsafeMemoryDMStore = memoryDMStore.convertToUnsafeDMStore(getFileFooterEntrySchema());
        memoryDMStore.freeMemory();
        memoryDMStore = unsafeMemoryDMStore;
    }
    if (taskSummaryDMStore instanceof SafeMemoryDMStore) {
        UnsafeMemoryDMStore unsafeSummaryMemoryDMStore = taskSummaryDMStore.convertToUnsafeDMStore(getTaskSummarySchema());
        taskSummaryDMStore.freeMemory();
        taskSummaryDMStore = unsafeSummaryMemoryDMStore;
    }
    if (memoryDMStore instanceof UnsafeMemoryDMStore) {
        if (memoryDMStore.isSerialized()) {
            memoryDMStore.copyToMemoryBlock();
        }
    }
    if (taskSummaryDMStore instanceof UnsafeMemoryDMStore) {
        if (taskSummaryDMStore.isSerialized()) {
            taskSummaryDMStore.copyToMemoryBlock();
        }
    }
}