Examples with DataFileFooter org.apache.carbondata.core.metadata.blocklet.DataFileFooter used on opensource projects

Example 21 with DataFileFooter

use of org.apache.carbondata.core.metadata.blocklet.DataFileFooter in project carbondata by apache.

the class DataFileFooterConverterTest method testGetIndexInfo.

@Test
public void testGetIndexInfo() throws Exception {
    DataFileFooterConverter dataFileFooterConverter = new DataFileFooterConverter();
    final ThriftReader thriftReader = new ThriftReader("file");
    List<Encoding> encoders = new ArrayList<>();
    encoders.add(Encoding.INVERTED_INDEX);
    encoders.add(Encoding.BIT_PACKED);
    encoders.add(Encoding.DELTA);
    encoders.add(Encoding.DICTIONARY);
    encoders.add(Encoding.DIRECT_DICTIONARY);
    encoders.add(Encoding.RLE);
    ColumnSchema columnSchema = new ColumnSchema(DataType.INT, "column", "3", true, encoders, true);
    ColumnSchema columnSchema1 = new ColumnSchema(DataType.ARRAY, "column", "3", true, encoders, true);
    ColumnSchema columnSchema2 = new ColumnSchema(DataType.DECIMAL, "column", "3", true, encoders, true);
    ColumnSchema columnSchema3 = new ColumnSchema(DataType.DOUBLE, "column", "3", true, encoders, true);
    ColumnSchema columnSchema4 = new ColumnSchema(DataType.LONG, "column", "3", true, encoders, true);
    ColumnSchema columnSchema5 = new ColumnSchema(DataType.SHORT, "column", "3", true, encoders, true);
    ColumnSchema columnSchema6 = new ColumnSchema(DataType.STRUCT, "column", "3", true, encoders, true);
    ColumnSchema columnSchema7 = new ColumnSchema(DataType.STRING, "column", "3", true, encoders, true);
    final List<ColumnSchema> columnSchemas = new ArrayList<>();
    columnSchemas.add(columnSchema);
    columnSchemas.add(columnSchema1);
    columnSchemas.add(columnSchema2);
    columnSchemas.add(columnSchema3);
    columnSchemas.add(columnSchema4);
    columnSchemas.add(columnSchema5);
    columnSchemas.add(columnSchema6);
    columnSchemas.add(columnSchema7);
    final BlockIndex blockIndex = new BlockIndex();
    blockIndex.setBlock_index(new org.apache.carbondata.format.BlockletIndex());
    org.apache.carbondata.format.BlockletIndex blockletIndex1 = new org.apache.carbondata.format.BlockletIndex();
    BlockletBTreeIndex blockletBTreeIndex = new BlockletBTreeIndex();
    blockletBTreeIndex.setStart_key("1".getBytes());
    blockletBTreeIndex.setEnd_key("3".getBytes());
    blockletIndex1.setB_tree_index(blockletBTreeIndex);
    BlockletMinMaxIndex blockletMinMaxIndex = new BlockletMinMaxIndex();
    blockletMinMaxIndex.setMax_values(Arrays.asList(ByteBuffer.allocate(1).put((byte) 2)));
    blockletMinMaxIndex.setMin_values(Arrays.asList(ByteBuffer.allocate(1).put((byte) 1)));
    blockletIndex1.setMin_max_index(blockletMinMaxIndex);
    blockIndex.setBlock_index(blockletIndex1);
    List<Integer> column_cardinalities = new ArrayList<>();
    column_cardinalities.add(new Integer("1"));
    final org.apache.carbondata.format.SegmentInfo segmentInfo1 = new org.apache.carbondata.format.SegmentInfo(3, column_cardinalities);
    new MockUp<CarbonIndexFileReader>() {

        boolean mockedHasNextStatus = true;

        @SuppressWarnings("unused")
        @Mock
        public boolean hasNext() throws IOException {
            boolean temp = mockedHasNextStatus;
            mockedHasNextStatus = false;
            return temp;
        }

        @SuppressWarnings("unused")
        @Mock
        public void openThriftReader(String filePath) throws IOException {
            thriftReader.open();
        }

        @SuppressWarnings("unused")
        @Mock
        public IndexHeader readIndexHeader() throws IOException {
            return new IndexHeader(1, columnSchemas, segmentInfo1);
        }

        @SuppressWarnings("unused")
        @Mock
        public BlockIndex readBlockIndexInfo() throws IOException {
            return blockIndex;
        }

        @SuppressWarnings("unused")
        @Mock
        public void closeThriftReader() {
            thriftReader.close();
        }
    };
    new MockUp<IndexHeader>() {

        @SuppressWarnings("unused")
        @Mock
        public List<ColumnSchema> getTable_columns() {
            return columnSchemas;
        }
    };
    ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream("1".getBytes());
    final DataInputStream dataInputStream = new DataInputStream(byteArrayInputStream);
    new MockUp<FileFactory>() {

        @SuppressWarnings("unused")
        @Mock
        public DataInputStream getDataInputStream(String path, FileFactory.FileType fileType, int bufferSize) {
            return dataInputStream;
        }
    };
    String[] arr = { "a", "b", "c" };
    String fileName = "/part-0-0_batchno0-0-1495074251740.carbondata";
    TableBlockInfo tableBlockInfo = new TableBlockInfo(fileName, 3, "id", arr, 3, ColumnarFormatVersion.V1);
    tableBlockInfo.getBlockletInfos().setNoOfBlockLets(3);
    List<TableBlockInfo> tableBlockInfoList = new ArrayList<>();
    tableBlockInfoList.add(tableBlockInfo);
    String idxFileName = "0_batchno0-0-1495074251740.carbonindex";
    List<DataFileFooter> dataFileFooterList = dataFileFooterConverter.getIndexInfo(idxFileName, tableBlockInfoList);
    byte[] exp = dataFileFooterList.get(0).getBlockletIndex().getBtreeIndex().getStartKey();
    byte[] res = "1".getBytes();
    for (int i = 0; i < exp.length; i++) {
        assertEquals(exp[i], res[i]);
    }
}

Example 22 with DataFileFooter

use of org.apache.carbondata.core.metadata.blocklet.DataFileFooter in project carbondata by apache.

the class BTreeBlockFinderTest method getFileFooter.

private DataFileFooter getFileFooter(byte[] startKey, byte[] endKey, byte[] noDictionaryStartKey, byte[] noDictionaryEndKey) {
    DataFileFooter footer = new DataFileFooter();
    BlockletIndex index = new BlockletIndex();
    BlockletBTreeIndex btreeIndex = new BlockletBTreeIndex();
    ByteBuffer buffer = ByteBuffer.allocate(4 + startKey.length + 4 + noDictionaryStartKey.length);
    buffer.putInt(startKey.length);
    buffer.putInt(noDictionaryStartKey.length);
    buffer.put(startKey);
    buffer.put(noDictionaryStartKey);
    buffer.rewind();
    btreeIndex.setStartKey(buffer.array());
    ByteBuffer buffer1 = ByteBuffer.allocate(4 + startKey.length + 4 + noDictionaryEndKey.length);
    buffer1.putInt(endKey.length);
    buffer1.putInt(noDictionaryEndKey.length);
    buffer1.put(endKey);
    buffer1.put(noDictionaryEndKey);
    buffer1.rewind();
    btreeIndex.setEndKey(buffer1.array());
    BlockletMinMaxIndex minMax = new BlockletMinMaxIndex();
    minMax.setMaxValues(new byte[][] { endKey, noDictionaryEndKey });
    minMax.setMinValues(new byte[][] { startKey, noDictionaryStartKey });
    index.setBtreeIndex(btreeIndex);
    index.setMinMaxIndex(minMax);
    footer.setBlockletIndex(index);
    return footer;
}

Example 23 with DataFileFooter

use of org.apache.carbondata.core.metadata.blocklet.DataFileFooter in project carbondata by apache.

the class BTreeBlockFinderTest method getFileFooterListWithOnlyNoDictionaryKey.

private List<DataFileFooter> getFileFooterListWithOnlyNoDictionaryKey() {
    List<DataFileFooter> list = new ArrayList<DataFileFooter>();
    try {
        int[] dimensionBitLength = CarbonUtil.getDimensionBitLength(new int[] { 10000, 10000 }, new int[] { 1, 1 });
        KeyGenerator multiDimKeyVarLengthGenerator = new MultiDimKeyVarLengthGenerator(dimensionBitLength);
        int i = 1;
        while (i < 1001) {
            byte[] startKey = multiDimKeyVarLengthGenerator.generateKey(new int[] { i, i });
            byte[] endKey = multiDimKeyVarLengthGenerator.generateKey(new int[] { i + 10, i + 10 });
            ByteBuffer buffer = ByteBuffer.allocate(2 + 4);
            buffer.rewind();
            buffer.putShort((short) 1);
            buffer.putInt(i);
            buffer.array();
            byte[] noDictionaryStartKey = buffer.array();
            ByteBuffer buffer1 = ByteBuffer.allocate(2 + 4);
            buffer1.rewind();
            buffer1.putShort((short) 2);
            buffer1.putInt(i + 10);
            buffer1.array();
            byte[] noDictionaryEndKey = buffer.array();
            DataFileFooter footer = getFileMatadataWithOnlyNoDictionaryKey(startKey, endKey, noDictionaryStartKey, noDictionaryEndKey);
            list.add(footer);
            i = i + 10;
        }
    } catch (Exception e) {
        LOGGER.error(e);
    }
    return list;
}

Example 24 with DataFileFooter

use of org.apache.carbondata.core.metadata.blocklet.DataFileFooter in project carbondata by apache.

the class BTreeBlockFinderTest method testBtreeBuldingIsPorper.

@Test
public void testBtreeBuldingIsPorper() {
    BtreeBuilder builder = new BlockBTreeBuilder();
    List<DataFileFooter> footerList = getDataFileFooterList();
    BTreeBuilderInfo infos = new BTreeBuilderInfo(footerList, null);
    builder.build(infos);
}

Example 25 with DataFileFooter

use of org.apache.carbondata.core.metadata.blocklet.DataFileFooter in project carbondata by apache.

the class CarbonCompactionUtil method checkIfAnyRestructuredBlockExists.

/**
   * This method will check for any restructured block in the blocks selected for compaction
   *
   * @param segmentMapping
   * @param dataFileMetadataSegMapping
   * @param tableLastUpdatedTime
   * @return
   */
public static boolean checkIfAnyRestructuredBlockExists(Map<String, TaskBlockInfo> segmentMapping, Map<String, List<DataFileFooter>> dataFileMetadataSegMapping, long tableLastUpdatedTime) {
    boolean restructuredBlockExists = false;
    for (Map.Entry<String, TaskBlockInfo> taskMap : segmentMapping.entrySet()) {
        String segmentId = taskMap.getKey();
        List<DataFileFooter> listMetadata = dataFileMetadataSegMapping.get(segmentId);
        for (DataFileFooter dataFileFooter : listMetadata) {
            // it indicates it is a restructured block
            if (tableLastUpdatedTime > dataFileFooter.getSchemaUpdatedTimeStamp()) {
                restructuredBlockExists = true;
                break;
            }
        }
        if (restructuredBlockExists) {
            break;
        }
    }
    return restructuredBlockExists;
}