Examples with TableBlockIndexUniqueIdentifierWrapper org.apache.carbondata.core.indexstore.TableBlockIndexUniqueIdentifierWrapper used on opensource projects

Example 1 with TableBlockIndexUniqueIdentifierWrapper

use of org.apache.carbondata.core.indexstore.TableBlockIndexUniqueIdentifierWrapper in project carbondata by apache.

the class BlockletIndexFactory method getIndexes.

@Override
public List<CoarseGrainIndex> getIndexes(Segment segment, Set<Path> partitionLocations) throws IOException {
    List<CoarseGrainIndex> indexes = new ArrayList<>();
    Set<TableBlockIndexUniqueIdentifier> identifiers = getTableBlockIndexUniqueIdentifiers(segment);
    List<TableBlockIndexUniqueIdentifierWrapper> tableBlockIndexUniqueIdentifierWrappers = new ArrayList<>(identifiers.size());
    getTableBlockUniqueIdentifierWrappers(partitionLocations, tableBlockIndexUniqueIdentifierWrappers, identifiers);
    List<BlockletIndexWrapper> blockletIndexWrappers = cache.getAll(tableBlockIndexUniqueIdentifierWrappers);
    for (BlockletIndexWrapper wrapper : blockletIndexWrappers) {
        indexes.addAll(wrapper.getIndexes());
    }
    return indexes;
}

Example 2 with TableBlockIndexUniqueIdentifierWrapper

use of org.apache.carbondata.core.indexstore.TableBlockIndexUniqueIdentifierWrapper in project carbondata by apache.

the class BlockletIndexFactory method getTableBlockIndexUniqueIdentifier.

private List<TableBlockIndexUniqueIdentifierWrapper> getTableBlockIndexUniqueIdentifier(IndexInputSplit distributable) throws IOException {
    List<TableBlockIndexUniqueIdentifierWrapper> identifiersWrapper = new ArrayList<>();
    SegmentBlockIndexInfo segmentBlockIndexInfo = segmentMap.get(distributable.getSegment().getSegmentNo());
    Set<TableBlockIndexUniqueIdentifier> tableBlockIndexUniqueIdentifiers = null;
    if (null != segmentBlockIndexInfo) {
        tableBlockIndexUniqueIdentifiers = segmentBlockIndexInfo.getTableBlockIndexUniqueIdentifiers();
    }
    if (tableBlockIndexUniqueIdentifiers == null) {
        tableBlockIndexUniqueIdentifiers = new HashSet<>();
        Set<String> indexFiles = distributable.getSegment().getCommittedIndexFile().keySet();
        for (String indexFile : indexFiles) {
            CarbonFile carbonFile = FileFactory.getCarbonFile(indexFile);
            String indexFileName;
            String mergeIndexName;
            if (indexFile.endsWith(CarbonTablePath.INDEX_FILE_EXT)) {
                indexFileName = carbonFile.getName();
                mergeIndexName = null;
            } else {
                indexFileName = carbonFile.getName();
                mergeIndexName = carbonFile.getName();
            }
            String parentPath = carbonFile.getParentFile().getAbsolutePath();
            TableBlockIndexUniqueIdentifier tableBlockIndexUniqueIdentifier = new TableBlockIndexUniqueIdentifier(parentPath, indexFileName, mergeIndexName, distributable.getSegment().getSegmentNo());
            identifiersWrapper.add(new TableBlockIndexUniqueIdentifierWrapper(tableBlockIndexUniqueIdentifier, this.getCarbonTable()));
            tableBlockIndexUniqueIdentifiers.add(tableBlockIndexUniqueIdentifier);
        }
        segmentMap.put(distributable.getSegment().getSegmentNo(), new SegmentBlockIndexInfo(tableBlockIndexUniqueIdentifiers, distributable.getSegment().getSegmentMetaDataInfo()));
    } else {
        for (TableBlockIndexUniqueIdentifier tableBlockIndexUniqueIdentifier : tableBlockIndexUniqueIdentifiers) {
            identifiersWrapper.add(new TableBlockIndexUniqueIdentifierWrapper(tableBlockIndexUniqueIdentifier, getCarbonTable()));
        }
    }
    return identifiersWrapper;
}

Example 3 with TableBlockIndexUniqueIdentifierWrapper

use of org.apache.carbondata.core.indexstore.TableBlockIndexUniqueIdentifierWrapper in project carbondata by apache.

the class BlockletIndexFactory method getExtendedBlocklet.

@Override
public ExtendedBlocklet getExtendedBlocklet(Blocklet blocklet, Segment segment) throws IOException {
    if (blocklet instanceof ExtendedBlocklet) {
        return (ExtendedBlocklet) blocklet;
    }
    Set<TableBlockIndexUniqueIdentifier> identifiers = getTableBlockIndexUniqueIdentifiers(segment);
    Set<TableBlockIndexUniqueIdentifierWrapper> tableBlockIndexUniqueIdentifierWrappers = new HashSet<>(identifiers.size());
    for (TableBlockIndexUniqueIdentifier tableBlockIndexUniqueIdentifier : identifiers) {
        tableBlockIndexUniqueIdentifierWrappers.add(new TableBlockIndexUniqueIdentifierWrapper(tableBlockIndexUniqueIdentifier, this.getCarbonTable()));
    }
    return getExtendedBlocklet(tableBlockIndexUniqueIdentifierWrappers, blocklet);
}

Example 4 with TableBlockIndexUniqueIdentifierWrapper

use of org.apache.carbondata.core.indexstore.TableBlockIndexUniqueIdentifierWrapper in project carbondata by apache.

the class TestBlockletIndexFactory method setUp.

@Before
public void setUp() throws ClassNotFoundException, IllegalAccessException, InvocationTargetException, InstantiationException {
    tableInfo = new TableInfo();
    factTable = new TableSchema();
    Constructor<?> constructor = Class.forName("org.apache.carbondata.core.metadata.schema.table.CarbonTable").getDeclaredConstructors()[0];
    constructor.setAccessible(true);
    carbonTable = (CarbonTable) constructor.newInstance();
    absoluteTableIdentifier = AbsoluteTableIdentifier.from("/opt/store/default/carbon_table/", "default", "carbon_table", UUID.randomUUID().toString());
    Deencapsulation.setField(tableInfo, "identifier", absoluteTableIdentifier);
    Deencapsulation.setField(tableInfo, "factTable", factTable);
    Deencapsulation.setField(carbonTable, "tableInfo", tableInfo);
    new MockUp<CarbonTable>() {

        @Mock
        public AbsoluteTableIdentifier getAbsoluteTableIdentifier() {
            return absoluteTableIdentifier;
        }
    };
    blockletIndexFactory = new BlockletIndexFactory(carbonTable, new IndexSchema());
    Deencapsulation.setField(blockletIndexFactory, "cache", CacheProvider.getInstance().createCache(CacheType.DRIVER_BLOCKLET_INDEX));
    tableBlockIndexUniqueIdentifier = new TableBlockIndexUniqueIdentifier("/opt/store/default/carbon_table/Fact/Part0/Segment_0", "0_batchno0-0-1521012756709.carbonindex", null, "0");
    tableBlockIndexUniqueIdentifierWrapper = new TableBlockIndexUniqueIdentifierWrapper(tableBlockIndexUniqueIdentifier, carbonTable);
    cache = CacheProvider.getInstance().createCache(CacheType.DRIVER_BLOCKLET_INDEX);
}

Example 5 with TableBlockIndexUniqueIdentifierWrapper

use of org.apache.carbondata.core.indexstore.TableBlockIndexUniqueIdentifierWrapper in project carbondata by apache.

the class BlockletIndexInputFormat method createRecordReader.

@Override
public RecordReader<TableBlockIndexUniqueIdentifier, BlockletIndexDetailsWithSchema> createRecordReader(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) {
    return new RecordReader<TableBlockIndexUniqueIdentifier, BlockletIndexDetailsWithSchema>() {

        private BlockletIndexWrapper wrapper = null;

        private TableBlockIndexUniqueIdentifier tableBlockIndexUniqueIdentifier = null;

        private TableBlockIndexUniqueIdentifierWrapper tableBlockIndexUniqueIdentifierWrapper;

        Cache<TableBlockIndexUniqueIdentifierWrapper, BlockletIndexWrapper> cache = CacheProvider.getInstance().createCache(CacheType.DRIVER_BLOCKLET_INDEX);

        private Iterator<TableBlockIndexUniqueIdentifier> iterator;

        // Cache to avoid multiple times listing of files
        private Map<String, Map<String, BlockMetaInfo>> segInfoCache = new HashMap<>();

        @Override
        public void initialize(InputSplit inputSplit, TaskAttemptContext taskAttemptContext) throws IOException, InterruptedException {
            BlockletIndexInputSplit segmentDistributable = (BlockletIndexInputSplit) inputSplit;
            TableBlockIndexUniqueIdentifier tableSegmentUniqueIdentifier = segmentDistributable.getTableBlockIndexUniqueIdentifier();
            Segment segment = Segment.toSegment(tableSegmentUniqueIdentifier.getSegmentId(), readCommittedScope);
            iterator = BlockletIndexUtil.getTableBlockUniqueIdentifiers(segment).iterator();
        }

        @Override
        public boolean nextKeyValue() {
            if (iterator.hasNext()) {
                TableBlockIndexUniqueIdentifier tableBlockIndexUniqueIdentifier = iterator.next();
                this.tableBlockIndexUniqueIdentifier = tableBlockIndexUniqueIdentifier;
                TableBlockIndexUniqueIdentifierWrapper tableBlockIndexUniqueIdentifierWrapper = new TableBlockIndexUniqueIdentifierWrapper(tableBlockIndexUniqueIdentifier, table, false, true, true);
                this.tableBlockIndexUniqueIdentifierWrapper = tableBlockIndexUniqueIdentifierWrapper;
                wrapper = ((BlockletIndexStore) cache).get(tableBlockIndexUniqueIdentifierWrapper, segInfoCache);
                return true;
            }
            return false;
        }

        @Override
        public TableBlockIndexUniqueIdentifier getCurrentKey() {
            return tableBlockIndexUniqueIdentifier;
        }

        @Override
        public BlockletIndexDetailsWithSchema getCurrentValue() {
            BlockletIndexDetailsWithSchema blockletIndexDetailsWithSchema = new BlockletIndexDetailsWithSchema(wrapper, table.getTableInfo().isSchemaModified());
            return blockletIndexDetailsWithSchema;
        }

        @Override
        public float getProgress() {
            return 0;
        }

        @Override
        public void close() {
            if (null != tableBlockIndexUniqueIdentifierWrapper) {
                if (null != wrapper && null != wrapper.getIndexes() && !wrapper.getIndexes().isEmpty()) {
                    String segmentId = tableBlockIndexUniqueIdentifierWrapper.getTableBlockIndexUniqueIdentifier().getSegmentId();
                    // as segmentId will be same for all the indexes and segmentProperties cache is
                    // maintained at segment level so it need to be called only once for clearing
                    SegmentPropertiesAndSchemaHolder.getInstance().invalidate(segmentId, wrapper.getIndexes().get(0).getSegmentPropertiesWrapper(), tableBlockIndexUniqueIdentifierWrapper.isAddTableBlockToUnsafeAndLRUCache());
                }
            }
        }
    };
}