Examples with HgkvFile com.baidu.hugegraph.computer.core.store.file.hgkvfile.HgkvFile used on opensource projects

Example 1 with HgkvFile

use of com.baidu.hugegraph.computer.core.store.file.hgkvfile.HgkvFile in project hugegraph-computer by hugegraph.

the class HgkvFileTest method testOpenFile.

@Test
public void testOpenFile() throws IOException {
    // The keys in the data must be ordered
    List<Integer> data = testData();
    String filePath = StoreTestUtil.availablePathById("1");
    File file = StoreTestUtil.mapToHgkvFile(CONFIG, data, filePath);
    // Open file
    HgkvFile hgkvFile = HgkvFileImpl.open(file.getPath());
    // Assert magic
    Assert.assertEquals(HgkvFileImpl.MAGIC, hgkvFile.magic());
    // Assert version
    String version = HgkvFileImpl.MAJOR_VERSION + "." + HgkvFileImpl.MINOR_VERSION;
    Assert.assertEquals(version, hgkvFile.version());
    // Assert numEntries
    Assert.assertEquals(5, hgkvFile.numEntries());
    // Assert max key
    int maxKey = StoreTestUtil.byteArrayToInt(hgkvFile.max());
    Assert.assertEquals(6, maxKey);
    // Assert min key
    int minKey = StoreTestUtil.byteArrayToInt(hgkvFile.min());
    Assert.assertEquals(2, minKey);
}

Example 2 with HgkvFile

use of com.baidu.hugegraph.computer.core.store.file.hgkvfile.HgkvFile in project hugegraph-computer by hugegraph.

the class DisperseEvenlySelector method selectedByHgkvFile.

@Override
public List<SelectedFiles> selectedByHgkvFile(List<String> inputs, List<String> outputs) throws IOException {
    E.checkArgument(inputs.size() >= outputs.size(), "The inputs size of InputFilesSelector must be >= " + "outputs size, but got %s inputs < %s outputs", inputs.size(), outputs.size());
    List<HgkvDir> inputDirs = new ArrayList<>();
    for (String input : inputs) {
        inputDirs.add(HgkvDirImpl.open(input));
    }
    /*
         * Reverse sort by entries number first
         * will make the disperse result more uniform
         */
    inputDirs = inputDirs.stream().sorted(Comparator.comparingLong(HgkvFile::numEntries).reversed()).collect(Collectors.toList());
    // Init heap data
    List<Node> heapNodes = new ArrayList<>(outputs.size());
    int i = 0;
    for (; i < outputs.size(); i++) {
        HgkvDir inputDir = inputDirs.get(i);
        Node heapNode = new Node(inputDir.numEntries(), Lists.newArrayList(inputDir.path()), outputs.get(i));
        heapNodes.add(heapNode);
    }
    Heap<Node> heap = new Heap<>(heapNodes, Comparator.comparingLong(Node::num));
    // Distribute the remaining input
    for (; i < inputDirs.size(); i++) {
        HgkvDir inputDir = inputDirs.get(i);
        Node topNode = heap.top();
        topNode.addInput(inputDir.path());
        topNode.addNum(inputDir.numEntries());
        heap.adjust(0);
    }
    List<SelectedFiles> results = new ArrayList<>();
    for (Node node : heapNodes) {
        SelectedFiles result = new DefaultSelectedFiles(node.output(), node.inputs());
        results.add(result);
    }
    return results;
}