Examples with PartitionInfo org.apache.carbondata.core.metadata.schema.PartitionInfo used on opensource projects

Example 1 with PartitionInfo

use of org.apache.carbondata.core.metadata.schema.PartitionInfo in project carbondata by apache.

the class ThriftWrapperSchemaConverterImpl method fromWrapperToExternalPartitionInfo.

private org.apache.carbondata.format.PartitionInfo fromWrapperToExternalPartitionInfo(PartitionInfo wrapperPartitionInfo) {
    List<org.apache.carbondata.format.ColumnSchema> thriftColumnSchema = new ArrayList<org.apache.carbondata.format.ColumnSchema>();
    for (ColumnSchema wrapperColumnSchema : wrapperPartitionInfo.getColumnSchemaList()) {
        thriftColumnSchema.add(fromWrapperToExternalColumnSchema(wrapperColumnSchema));
    }
    org.apache.carbondata.format.PartitionInfo externalPartitionInfo = new org.apache.carbondata.format.PartitionInfo(thriftColumnSchema, fromWrapperToExternalPartitionType(wrapperPartitionInfo.getPartitionType()));
    externalPartitionInfo.setList_info(wrapperPartitionInfo.getListInfo());
    externalPartitionInfo.setRange_info(wrapperPartitionInfo.getRangeInfo());
    externalPartitionInfo.setNum_partitions(wrapperPartitionInfo.getNumPartitions());
    return externalPartitionInfo;
}

Example 2 with PartitionInfo

use of org.apache.carbondata.core.metadata.schema.PartitionInfo in project carbondata by apache.

the class CarbonInputFormat method getSplits.

/**
   * {@inheritDoc}
   * Configurations FileInputFormat.INPUT_DIR
   * are used to get table path to read.
   *
   * @param job
   * @return List<InputSplit> list of CarbonInputSplit
   * @throws IOException
   */
@Override
public List<InputSplit> getSplits(JobContext job) throws IOException {
    AbsoluteTableIdentifier identifier = getAbsoluteTableIdentifier(job.getConfiguration());
    CacheClient cacheClient = new CacheClient(identifier.getStorePath());
    try {
        List<String> invalidSegments = new ArrayList<>();
        List<UpdateVO> invalidTimestampsList = new ArrayList<>();
        // get all valid segments and set them into the configuration
        if (getSegmentsToAccess(job).length == 0) {
            SegmentStatusManager segmentStatusManager = new SegmentStatusManager(identifier);
            SegmentStatusManager.ValidAndInvalidSegmentsInfo segments = segmentStatusManager.getValidAndInvalidSegments();
            SegmentUpdateStatusManager updateStatusManager = new SegmentUpdateStatusManager(identifier);
            setSegmentsToAccess(job.getConfiguration(), segments.getValidSegments());
            if (segments.getValidSegments().size() == 0) {
                return new ArrayList<>(0);
            }
            // remove entry in the segment index if there are invalid segments
            invalidSegments.addAll(segments.getInvalidSegments());
            for (String invalidSegmentId : invalidSegments) {
                invalidTimestampsList.add(updateStatusManager.getInvalidTimestampRange(invalidSegmentId));
            }
            if (invalidSegments.size() > 0) {
                List<TableSegmentUniqueIdentifier> invalidSegmentsIds = new ArrayList<>(invalidSegments.size());
                for (String segId : invalidSegments) {
                    invalidSegmentsIds.add(new TableSegmentUniqueIdentifier(identifier, segId));
                }
                cacheClient.getSegmentAccessClient().invalidateAll(invalidSegmentsIds);
            }
        }
        // process and resolve the expression
        Expression filter = getFilterPredicates(job.getConfiguration());
        CarbonTable carbonTable = getCarbonTable(job.getConfiguration());
        // this will be null in case of corrupt schema file.
        if (null == carbonTable) {
            throw new IOException("Missing/Corrupt schema file for table.");
        }
        CarbonInputFormatUtil.processFilterExpression(filter, carbonTable);
        // prune partitions for filter query on partition table
        BitSet matchedPartitions = null;
        if (null != filter) {
            PartitionInfo partitionInfo = carbonTable.getPartitionInfo(carbonTable.getFactTableName());
            if (null != partitionInfo) {
                Partitioner partitioner = PartitionUtil.getPartitioner(partitionInfo);
                matchedPartitions = new FilterExpressionProcessor().getFilteredPartitions(filter, partitionInfo, partitioner);
                if (matchedPartitions.cardinality() == 0) {
                    // no partition is required
                    return new ArrayList<InputSplit>();
                }
                if (matchedPartitions.cardinality() == partitioner.numPartitions()) {
                    // all partitions are required, no need to prune partitions
                    matchedPartitions = null;
                }
            }
        }
        FilterResolverIntf filterInterface = CarbonInputFormatUtil.resolveFilter(filter, identifier);
        // do block filtering and get split
        List<InputSplit> splits = getSplits(job, filterInterface, matchedPartitions, cacheClient);
        // pass the invalid segment to task side in order to remove index entry in task side
        if (invalidSegments.size() > 0) {
            for (InputSplit split : splits) {
                ((CarbonInputSplit) split).setInvalidSegments(invalidSegments);
                ((CarbonInputSplit) split).setInvalidTimestampRange(invalidTimestampsList);
            }
        }
        return splits;
    } finally {
        // close the cache cache client to clear LRU cache memory
        cacheClient.close();
    }
}

Example 3 with PartitionInfo

use of org.apache.carbondata.core.metadata.schema.PartitionInfo in project carbondata by apache.

the class ThriftWrapperSchemaConverterImpl method fromExternalToWrapperPartitionInfo.

private PartitionInfo fromExternalToWrapperPartitionInfo(org.apache.carbondata.format.PartitionInfo externalPartitionInfo) {
    List<ColumnSchema> wrapperColumnSchema = new ArrayList<ColumnSchema>();
    for (org.apache.carbondata.format.ColumnSchema columnSchema : externalPartitionInfo.getPartition_columns()) {
        wrapperColumnSchema.add(fromExternalToWrapperColumnSchema(columnSchema));
    }
    PartitionInfo wrapperPartitionInfo = new PartitionInfo(wrapperColumnSchema, fromExternalToWrapperPartitionType(externalPartitionInfo.getPartition_type()));
    wrapperPartitionInfo.setListInfo(externalPartitionInfo.getList_info());
    wrapperPartitionInfo.setRangeInfo(externalPartitionInfo.getRange_info());
    wrapperPartitionInfo.setNumPartitions(externalPartitionInfo.getNum_partitions());
    return wrapperPartitionInfo;
}