Examples with BlockletIndexWrapper org.apache.carbondata.core.indexstore.BlockletIndexWrapper used on opensource projects

Example 1 with BlockletIndexWrapper

use of org.apache.carbondata.core.indexstore.BlockletIndexWrapper in project carbondata by apache.

the class BlockletIndexFactory method getIndexes.

@Override
public List<CoarseGrainIndex> getIndexes(Segment segment, Set<Path> partitionLocations) throws IOException {
    List<CoarseGrainIndex> indexes = new ArrayList<>();
    Set<TableBlockIndexUniqueIdentifier> identifiers = getTableBlockIndexUniqueIdentifiers(segment);
    List<TableBlockIndexUniqueIdentifierWrapper> tableBlockIndexUniqueIdentifierWrappers = new ArrayList<>(identifiers.size());
    getTableBlockUniqueIdentifierWrappers(partitionLocations, tableBlockIndexUniqueIdentifierWrappers, identifiers);
    List<BlockletIndexWrapper> blockletIndexWrappers = cache.getAll(tableBlockIndexUniqueIdentifierWrappers);
    for (BlockletIndexWrapper wrapper : blockletIndexWrappers) {
        indexes.addAll(wrapper.getIndexes());
    }
    return indexes;
}

Example 2 with BlockletIndexWrapper

use of org.apache.carbondata.core.indexstore.BlockletIndexWrapper in project carbondata by apache.

the class TestBlockletIndexFactory method addIndexToCache.

@Test
public void addIndexToCache() throws NoSuchMethodException, InvocationTargetException, IllegalAccessException {
    List<BlockIndex> indexes = new ArrayList<>();
    Method method = BlockletIndexFactory.class.getDeclaredMethod("cache", TableBlockIndexUniqueIdentifierWrapper.class, BlockletIndexWrapper.class);
    method.setAccessible(true);
    method.invoke(blockletIndexFactory, tableBlockIndexUniqueIdentifierWrapper, new BlockletIndexWrapper(tableBlockIndexUniqueIdentifier.getSegmentId(), indexes));
    BlockletIndexWrapper result = cache.getIfPresent(tableBlockIndexUniqueIdentifierWrapper);
    assert null != result;
}

Example 3 with BlockletIndexWrapper

use of org.apache.carbondata.core.indexstore.BlockletIndexWrapper in project carbondata by apache.

the class BlockletIndexInputFormat method createRecordReader.

@Override
public RecordReader<TableBlockIndexUniqueIdentifier, BlockletIndexDetailsWithSchema> createRecordReader(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) {
    return new RecordReader<TableBlockIndexUniqueIdentifier, BlockletIndexDetailsWithSchema>() {

        private BlockletIndexWrapper wrapper = null;

        private TableBlockIndexUniqueIdentifier tableBlockIndexUniqueIdentifier = null;

        private TableBlockIndexUniqueIdentifierWrapper tableBlockIndexUniqueIdentifierWrapper;

        Cache<TableBlockIndexUniqueIdentifierWrapper, BlockletIndexWrapper> cache = CacheProvider.getInstance().createCache(CacheType.DRIVER_BLOCKLET_INDEX);

        private Iterator<TableBlockIndexUniqueIdentifier> iterator;

        // Cache to avoid multiple times listing of files
        private Map<String, Map<String, BlockMetaInfo>> segInfoCache = new HashMap<>();

        @Override
        public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
            BlockletIndexInputSplit segmentDistributable = (BlockletIndexInputSplit) inputSplit;
            TableBlockIndexUniqueIdentifier tableSegmentUniqueIdentifier = segmentDistributable.getTableBlockIndexUniqueIdentifier();
            Segment segment = Segment.toSegment(tableSegmentUniqueIdentifier.getSegmentId(), readCommittedScope);
            iterator = BlockletIndexUtil.getTableBlockUniqueIdentifiers(segment).iterator();
        }

        @Override
        public boolean nextKeyValue() {
            if (iterator.hasNext()) {
                TableBlockIndexUniqueIdentifier tableBlockIndexUniqueIdentifier = iterator.next();
                this.tableBlockIndexUniqueIdentifier = tableBlockIndexUniqueIdentifier;
                TableBlockIndexUniqueIdentifierWrapper tableBlockIndexUniqueIdentifierWrapper = new TableBlockIndexUniqueIdentifierWrapper(tableBlockIndexUniqueIdentifier, table, false, true, true);
                this.tableBlockIndexUniqueIdentifierWrapper = tableBlockIndexUniqueIdentifierWrapper;
                wrapper = ((BlockletIndexStore) cache).get(tableBlockIndexUniqueIdentifierWrapper, segInfoCache);
                return true;
            }
            return false;
        }

        @Override
        public TableBlockIndexUniqueIdentifier getCurrentKey() {
            return tableBlockIndexUniqueIdentifier;
        }

        @Override
        public BlockletIndexDetailsWithSchema getCurrentValue() {
            BlockletIndexDetailsWithSchema blockletIndexDetailsWithSchema = new BlockletIndexDetailsWithSchema(wrapper, table.getTableInfo().isSchemaModified());
            return blockletIndexDetailsWithSchema;
        }

        @Override
        public float getProgress() {
            return 0;
        }

        @Override
        public void close() {
            if (null != tableBlockIndexUniqueIdentifierWrapper) {
                if (null != wrapper && null != wrapper.getIndexes() && !wrapper.getIndexes().isEmpty()) {
                    String segmentId = tableBlockIndexUniqueIdentifierWrapper.getTableBlockIndexUniqueIdentifier().getSegmentId();
                    // as segmentId will be same for all the indexes and segmentProperties cache is
                    // maintained at segment level so it need to be called only once for clearing
                    SegmentPropertiesAndSchemaHolder.getInstance().invalidate(segmentId, wrapper.getIndexes().get(0).getSegmentPropertiesWrapper(), tableBlockIndexUniqueIdentifierWrapper.isAddTableBlockToUnsafeAndLRUCache());
                }
            }
        }
    };
}

Example 4 with BlockletIndexWrapper

use of org.apache.carbondata.core.indexstore.BlockletIndexWrapper in project carbondata by apache.

the class BlockletIndexFactory method clear.

@Override
public void clear(String segment) {
    SegmentBlockIndexInfo segmentBlockIndexInfo = segmentMap.remove(segment);
    Set<TableBlockIndexUniqueIdentifier> blockIndexes = null;
    if (null != segmentBlockIndexInfo) {
        blockIndexes = segmentBlockIndexInfo.getTableBlockIndexUniqueIdentifiers();
    }
    if (blockIndexes != null) {
        for (TableBlockIndexUniqueIdentifier blockIndex : blockIndexes) {
            TableBlockIndexUniqueIdentifierWrapper blockIndexWrapper = new TableBlockIndexUniqueIdentifierWrapper(blockIndex, this.getCarbonTable());
            BlockletIndexWrapper wrapper = cache.getIfPresent(blockIndexWrapper);
            if (null != wrapper) {
                List<BlockIndex> indexes = wrapper.getIndexes();
                for (Index index : indexes) {
                    if (index != null) {
                        cache.invalidate(blockIndexWrapper);
                        index.clear();
                    }
                }
            }
        }
    }
}

Example 5 with BlockletIndexWrapper

use of org.apache.carbondata.core.indexstore.BlockletIndexWrapper in project carbondata by apache.

the class BlockletIndexFactory method getExtendedBlocklet.

private ExtendedBlocklet getExtendedBlocklet(Set<TableBlockIndexUniqueIdentifierWrapper> identifiersWrapper, Blocklet blocklet) throws IOException {
    for (TableBlockIndexUniqueIdentifierWrapper identifierWrapper : identifiersWrapper) {
        BlockletIndexWrapper wrapper = cache.get(identifierWrapper);
        List<BlockIndex> indexes = wrapper.getIndexes();
        for (Index index : indexes) {
            if (((BlockIndex) index).getTableTaskInfo(BlockletIndexRowIndexes.SUMMARY_INDEX_FILE_NAME).startsWith(blocklet.getFilePath())) {
                return ((BlockIndex) index).getDetailedBlocklet(blocklet.getBlockletId());
            }
        }
    }
    throw new IOException("Blocklet not found: " + blocklet.toString());
}