Examples with KvEntry com.baidu.hugegraph.computer.core.store.entry.KvEntry used on opensource projects

Example 21 with KvEntry

use of com.baidu.hugegraph.computer.core.store.entry.KvEntry in project hugegraph-computer by hugegraph.

the class VertexMessageRecvPartitionTest method checkTenVertexWithMergedProperties.

private static void checkTenVertexWithMergedProperties(PeekableIterator<KvEntry> it) throws IOException {
    for (long i = 0L; i < 10L; i++) {
        // Assert key
        Assert.assertTrue(it.hasNext());
        KvEntry entry = it.next();
        Id id = ReceiverUtil.readId(entry.key());
        Assert.assertEquals(BytesId.of(i), id);
        // Assert value
        Pointer value = entry.value();
        RandomAccessInput input = value.input();
        long position = input.position();
        input.seek(value.offset());
        String label = StreamGraphInput.readLabel(input);
        Assert.assertEquals("", label);
        Properties properties = graphFactory().createProperties();
        properties.read(input);
        input.seek(position);
        Assert.assertEquals(2, properties.size());
        LongValue v1 = properties.get("p1");
        Assert.assertEquals(new LongValue(i), v1);
        LongValue v2 = properties.get("p2");
        Assert.assertEquals(new LongValue(2L * i), v2);
    }
}

Example 22 with KvEntry

use of com.baidu.hugegraph.computer.core.store.entry.KvEntry in project hugegraph-computer by hugegraph.

the class SorterTest method testMergeSubKvFiles.

private void testMergeSubKvFiles(Config config) throws Exception {
    int flushThreshold = config.get(ComputerOptions.INPUT_MAX_EDGES_IN_ONE_VERTEX);
    List<Integer> kv1 = ImmutableList.of(1, 2, 1, 4, 1);
    List<Integer> kv2 = ImmutableList.of(4, 2, 1, 3, 1);
    List<Integer> kv3 = ImmutableList.of(4, 6, 1, 8, 1);
    List<Integer> kv4 = ImmutableList.of(1, 1, 1, 2, 1);
    List<Integer> kv5 = ImmutableList.of(1, 5, 1, 7, 1);
    List<Integer> kv6 = ImmutableList.of(2, 2, 1, 5, 1);
    List<List<Integer>> data1 = ImmutableList.of(kv1, kv2, kv3);
    List<List<Integer>> data2 = ImmutableList.of(kv4, kv5, kv6);
    List<List<Integer>> data3 = ImmutableList.of(kv4, kv1, kv3);
    List<List<List<Integer>>> datas = ImmutableList.of(data1, data2, data3);
    String input1 = StoreTestUtil.availablePathById(1);
    String input2 = StoreTestUtil.availablePathById(2);
    String input3 = StoreTestUtil.availablePathById(3);
    String output = StoreTestUtil.availablePathById(0);
    List<String> inputs = ImmutableList.of(input1, input2, input3);
    List<String> outputs = ImmutableList.of(output);
    boolean useBufferFile = config.get(ComputerOptions.TRANSPORT_RECV_FILE_MODE);
    for (int i = 0; i < inputs.size(); i++) {
        String input = inputs.get(i);
        List<List<Integer>> data = datas.get(i);
        if (useBufferFile) {
            StoreTestUtil.bufferFileFromSubKvMap(data, input);
        } else {
            StoreTestUtil.hgkvDirFromSubKvMap(config, data, input);
        }
    }
    Sorter sorter = SorterTestUtil.createSorter(config);
    PointerCombiner combiner = SorterTestUtil.createPointerCombiner(IntValue::new, new IntValueSumCombiner());
    OuterSortFlusher flusher = new CombineSubKvOuterSortFlusher(combiner, flushThreshold);
    flusher.sources(inputs.size());
    sorter.mergeInputs(inputs, flusher, outputs, true);
    /* Assert result
         * key 1 subKv 1 2 2 4
         * key 1 subKv 4 2 5 1
         * key 1 subKv 7 1
         * key 2 subKv 2 1 5 1
         * key 4 subKv 2 1 3 1
         * key 4 subKv 6 2 8 2
         */
    try (CIter<KvEntry> kvIter = sorter.iterator(outputs, true)) {
        SorterTestUtil.assertSubKvByKv(kvIter.next(), 1, 1, 2, 2, 4);
        SorterTestUtil.assertSubKvByKv(kvIter.next(), 1, 4, 2, 5, 1);
        SorterTestUtil.assertSubKvByKv(kvIter.next(), 1, 7, 1);
        SorterTestUtil.assertSubKvByKv(kvIter.next(), 2, 2, 1, 5, 1);
        SorterTestUtil.assertSubKvByKv(kvIter.next(), 4, 2, 1, 3, 1);
        SorterTestUtil.assertSubKvByKv(kvIter.next(), 4, 6, 2, 8, 2);
    }
    FileUtil.deleteFilesQuietly(inputs);
    FileUtil.deleteFilesQuietly(outputs);
}

Example 23 with KvEntry

use of com.baidu.hugegraph.computer.core.store.entry.KvEntry in project hugegraph-computer by hugegraph.

the class FlusherTest method testKvOuterSortFlusher.

@Test
public void testKvOuterSortFlusher() throws Exception {
    List<Integer> map1 = ImmutableList.of(2, 1, 2, 1, 3, 1, 4, 1);
    List<Integer> map2 = ImmutableList.of(1, 1, 3, 1, 6, 1);
    BytesInput input1 = SorterTestUtil.inputFromKvMap(map1);
    BytesInput input2 = SorterTestUtil.inputFromKvMap(map2);
    List<RandomAccessInput> inputs = ImmutableList.of(input1, input2);
    String resultFile = StoreTestUtil.availablePathById("1");
    Sorter sorter = SorterTestUtil.createSorter(CONFIG);
    sorter.mergeBuffers(inputs, new KvOuterSortFlusher(), resultFile, false);
    ImmutableList<String> outputs = ImmutableList.of(resultFile);
    Iterator<KvEntry> iter = sorter.iterator(outputs, false);
    SorterTestUtil.assertKvEntry(iter.next(), 1, 1);
    SorterTestUtil.assertKvEntry(iter.next(), 2, 1);
    SorterTestUtil.assertKvEntry(iter.next(), 2, 1);
    SorterTestUtil.assertKvEntry(iter.next(), 3, 1);
    SorterTestUtil.assertKvEntry(iter.next(), 3, 1);
    SorterTestUtil.assertKvEntry(iter.next(), 4, 1);
    SorterTestUtil.assertKvEntry(iter.next(), 6, 1);
}

Example 24 with KvEntry

use of com.baidu.hugegraph.computer.core.store.entry.KvEntry in project hugegraph-computer by hugegraph.

the class SortLargeDataTest method assertFileOrder.

private static void assertFileOrder(Sorter sorter, List<String> files) throws Exception {
    KvEntry last = null;
    try (PeekableIterator<KvEntry> iterator = sorter.iterator(files, false)) {
        while (iterator.hasNext()) {
            KvEntry next = iterator.next();
            if (last == null) {
                last = iterator.next();
                continue;
            }
            Assert.assertLte(0, last.key().compareTo(next.key()));
        }
    }
}

Example 25 with KvEntry

use of com.baidu.hugegraph.computer.core.store.entry.KvEntry in project hugegraph-computer by hugegraph.

the class SortLargeDataTest method testDiffNumEntriesFileMerge.

@Test
public void testDiffNumEntriesFileMerge() throws Exception {
    Config config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.HGKV_MERGE_FILES_NUM, "3", ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
    List<Integer> sizeList = ImmutableList.of(200, 500, 20, 50, 300, 250, 10, 33, 900, 89, 20);
    List<String> inputs = new ArrayList<>();
    for (int j = 0; j < sizeList.size(); j++) {
        String file = StoreTestUtil.availablePathById(j + 10);
        inputs.add(file);
        try (KvEntryFileWriter builder = new HgkvDirBuilderImpl(config, file)) {
            for (int i = 0; i < sizeList.get(j); i++) {
                byte[] keyBytes = StoreTestUtil.intToByteArray(i);
                byte[] valueBytes = StoreTestUtil.intToByteArray(1);
                Pointer key = new InlinePointer(keyBytes);
                Pointer value = new InlinePointer(valueBytes);
                KvEntry entry = new DefaultKvEntry(key, value);
                builder.write(entry);
            }
        }
    }
    List<String> outputs = ImmutableList.of(StoreTestUtil.availablePathById(0), StoreTestUtil.availablePathById(1), StoreTestUtil.availablePathById(2), StoreTestUtil.availablePathById(3));
    Sorter sorter = SorterTestUtil.createSorter(config);
    sorter.mergeInputs(inputs, new KvOuterSortFlusher(), outputs, false);
    int total = sizeList.stream().mapToInt(i -> i).sum();
    int mergeTotal = 0;
    for (String output : outputs) {
        mergeTotal += HgkvDirImpl.open(output).numEntries();
    }
    Assert.assertEquals(total, mergeTotal);
}