Examples with TableBlockInfo org.apache.carbondata.core.datastore.block.TableBlockInfo used on opensource projects

Example 26 with TableBlockInfo

use of org.apache.carbondata.core.datastore.block.TableBlockInfo in project carbondata by apache.

the class CarbonLoaderUtilTest method nodeBlockMapping.

/**
   * Test case with 4 blocks and 4 nodes with 3 replication.
   *
   * @throws Exception
   */
@Test
public void nodeBlockMapping() throws Exception {
    Map<TableBlockInfo, List<String>> inputMap = new HashMap<TableBlockInfo, List<String>>(5);
    TableBlockInfo block1 = new TableBlockInfo("path1", 123, "1", new String[] { "1", "2", "3" }, 111);
    TableBlockInfo block2 = new TableBlockInfo("path2", 123, "2", new String[] { "2", "3", "4" }, 111);
    TableBlockInfo block3 = new TableBlockInfo("path3", 123, "3", new String[] { "3", "4", "1" }, 111);
    TableBlockInfo block4 = new TableBlockInfo("path4", 123, "4", new String[] { "1", "2", "4" }, 111);
    inputMap.put(block1, Arrays.asList(new String[] { "1", "2", "3" }));
    inputMap.put(block2, Arrays.asList(new String[] { "2", "3", "4" }));
    inputMap.put(block3, Arrays.asList(new String[] { "3", "4", "1" }));
    inputMap.put(block4, Arrays.asList(new String[] { "1", "2", "4" }));
    List<TableBlockInfo> inputBlocks = new ArrayList(6);
    inputBlocks.add(block1);
    inputBlocks.add(block2);
    inputBlocks.add(block3);
    inputBlocks.add(block4);
    Map<String, List<TableBlockInfo>> outputMap = CarbonLoaderUtil.nodeBlockMapping(inputBlocks, 4);
    Assert.assertTrue(calculateBlockDistribution(inputMap, outputMap, 4, 4));
    Assert.assertTrue(calculateBlockLocality(inputMap, outputMap, 4, 4));
}

Example 27 with TableBlockInfo

use of org.apache.carbondata.core.datastore.block.TableBlockInfo in project carbondata by apache.

the class CarbonLoaderUtilTest method nodeBlockMappingTestWith5blocks3nodes.

/**
   * Test case with 5 blocks and 3 nodes
   *
   * @throws Exception
   */
@Test
public void nodeBlockMappingTestWith5blocks3nodes() throws Exception {
    Map<TableBlockInfo, List<String>> inputMap = new HashMap<TableBlockInfo, List<String>>(5);
    TableBlockInfo block1 = new TableBlockInfo("part-0-0-1462341987000", 123, "1", new String[] { "1", "2", "3" }, 111);
    TableBlockInfo block2 = new TableBlockInfo("part-1-0-1462341987000", 123, "2", new String[] { "1", "2", "3" }, 111);
    TableBlockInfo block3 = new TableBlockInfo("part-2-0-1462341987000", 123, "3", new String[] { "1", "2", "3" }, 111);
    TableBlockInfo block4 = new TableBlockInfo("part-3-0-1462341987000", 123, "4", new String[] { "1", "2", "3" }, 111);
    TableBlockInfo block5 = new TableBlockInfo("part-4-0-1462341987000", 123, "5", new String[] { "1", "2", "3" }, 111);
    inputMap.put(block1, Arrays.asList(new String[] { "1", "2", "3" }));
    inputMap.put(block2, Arrays.asList(new String[] { "1", "2", "3" }));
    inputMap.put(block3, Arrays.asList(new String[] { "1", "2", "3" }));
    inputMap.put(block4, Arrays.asList(new String[] { "1", "2", "3" }));
    inputMap.put(block5, Arrays.asList(new String[] { "1", "2", "3" }));
    List<TableBlockInfo> inputBlocks = new ArrayList(6);
    inputBlocks.add(block1);
    inputBlocks.add(block2);
    inputBlocks.add(block3);
    inputBlocks.add(block4);
    inputBlocks.add(block5);
    Map<String, List<TableBlockInfo>> outputMap = CarbonLoaderUtil.nodeBlockMapping(inputBlocks, 3);
    Assert.assertTrue(calculateBlockDistribution(inputMap, outputMap, 5, 3));
    Assert.assertTrue(calculateBlockLocality(inputMap, outputMap, 5, 3));
}

Example 28 with TableBlockInfo

use of org.apache.carbondata.core.datastore.block.TableBlockInfo in project carbondata by apache.

the class CarbonLoaderUtilTest method nodeBlockMappingTestWith10Blocks4nodes.

/**
   * Test case with 10 blocks and 4 nodes with 10,60,30 % distribution
   *
   * @throws Exception
   */
@Test
public void nodeBlockMappingTestWith10Blocks4nodes() throws Exception {
    Map<TableBlockInfo, List<String>> inputMap = new HashMap<TableBlockInfo, List<String>>(5);
    TableBlockInfo block1 = new TableBlockInfo("part-1-0-1462341987000", 123, "1", new String[] { "2", "4" }, 111);
    TableBlockInfo block2 = new TableBlockInfo("part-2-0-1462341987000", 123, "2", new String[] { "2", "4" }, 111);
    TableBlockInfo block3 = new TableBlockInfo("part-3-0-1462341987000", 123, "3", new String[] { "2", "4" }, 111);
    TableBlockInfo block4 = new TableBlockInfo("part-4-0-1462341987000", 123, "4", new String[] { "2", "4" }, 111);
    TableBlockInfo block5 = new TableBlockInfo("part-5-0-1462341987000", 123, "5", new String[] { "2", "4" }, 111);
    TableBlockInfo block6 = new TableBlockInfo("part-6-0-1462341987000", 123, "6", new String[] { "2", "4" }, 111);
    TableBlockInfo block7 = new TableBlockInfo("part-7-0-1462341987000", 123, "7", new String[] { "3", "4" }, 111);
    TableBlockInfo block8 = new TableBlockInfo("part-8-0-1462341987000", 123, "8", new String[] { "3", "4" }, 111);
    TableBlockInfo block9 = new TableBlockInfo("part-9-0-1462341987000", 123, "9", new String[] { "3", "4" }, 111);
    TableBlockInfo block10 = new TableBlockInfo("part-10-0-1462341987000", 123, "9", new String[] { "1", "4" }, 111);
    inputMap.put(block1, Arrays.asList(new String[] { "2", "4" }));
    inputMap.put(block2, Arrays.asList(new String[] { "2", "4" }));
    inputMap.put(block3, Arrays.asList(new String[] { "2", "4" }));
    inputMap.put(block4, Arrays.asList(new String[] { "2", "4" }));
    inputMap.put(block5, Arrays.asList(new String[] { "2", "4" }));
    inputMap.put(block6, Arrays.asList(new String[] { "2", "4" }));
    inputMap.put(block7, Arrays.asList(new String[] { "3", "4" }));
    inputMap.put(block8, Arrays.asList(new String[] { "3", "4" }));
    inputMap.put(block9, Arrays.asList(new String[] { "3", "4" }));
    inputMap.put(block10, Arrays.asList(new String[] { "1", "4" }));
    List<TableBlockInfo> inputBlocks = new ArrayList(6);
    inputBlocks.add(block1);
    inputBlocks.add(block2);
    inputBlocks.add(block3);
    inputBlocks.add(block4);
    inputBlocks.add(block5);
    inputBlocks.add(block6);
    inputBlocks.add(block7);
    inputBlocks.add(block8);
    inputBlocks.add(block9);
    inputBlocks.add(block10);
    Map<String, List<TableBlockInfo>> outputMap = CarbonLoaderUtil.nodeBlockMapping(inputBlocks, 4);
    Assert.assertTrue(calculateBlockDistribution(inputMap, outputMap, 10, 4));
    Assert.assertTrue(calculateBlockLocality(inputMap, outputMap, 10, 4));
}

Example 29 with TableBlockInfo

use of org.apache.carbondata.core.datastore.block.TableBlockInfo in project carbondata by apache.

the class CarbonLoaderUtilTest method initSet3.

void initSet3() {
    blockInfos = new ArrayList<>();
    activeNode = new ArrayList<>();
    activeNode.add("node-7");
    activeNode.add("node-11");
    String[] location = { "node-7", "node-9", "node-11" };
    blockInfos.add(new TableBlockInfo("node", 1, "1", location, 0));
    blockInfos.add(new TableBlockInfo("node", 2, "1", location, 0));
    blockInfos.add(new TableBlockInfo("node", 3, "1", location, 0));
    blockInfos.add(new TableBlockInfo("node", 4, "1", location, 0));
    blockInfos.add(new TableBlockInfo("node", 5, "1", location, 0));
    blockInfos.add(new TableBlockInfo("node", 6, "1", location, 0));
    expected = new HashMap<>();
    expected.put("node-7", blockInfos.subList(0, 3));
    expected.put("node-11", blockInfos.subList(3, 6));
}

Example 30 with TableBlockInfo

use of org.apache.carbondata.core.datastore.block.TableBlockInfo in project carbondata by apache.

the class CarbonCompactionUtil method createMappingForSegments.

/**
   * To create a mapping of Segment Id and TableBlockInfo.
   *
   * @param tableBlockInfoList
   * @return
   */
public static Map<String, TaskBlockInfo> createMappingForSegments(List<TableBlockInfo> tableBlockInfoList) {
    // stores taskBlockInfo of each segment
    Map<String, TaskBlockInfo> segmentBlockInfoMapping = new HashMap<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    for (TableBlockInfo info : tableBlockInfoList) {
        String segId = info.getSegmentId();
        // check if segId is already present in map
        TaskBlockInfo taskBlockInfoMapping = segmentBlockInfoMapping.get(segId);
        // extract task ID from file Path.
        String taskNo = CarbonTablePath.DataFileUtil.getTaskNo(info.getFilePath());
        // if taskBlockInfo is not there, then create and add
        if (null == taskBlockInfoMapping) {
            taskBlockInfoMapping = new TaskBlockInfo();
            groupCorrespodingInfoBasedOnTask(info, taskBlockInfoMapping, taskNo);
            // put the taskBlockInfo with respective segment id
            segmentBlockInfoMapping.put(segId, taskBlockInfoMapping);
        } else {
            groupCorrespodingInfoBasedOnTask(info, taskBlockInfoMapping, taskNo);
        }
    }
    return segmentBlockInfoMapping;
}