Examples with InlinePointer com.baidu.hugegraph.computer.core.store.entry.InlinePointer used on opensource projects

Example 1 with InlinePointer

use of com.baidu.hugegraph.computer.core.store.entry.InlinePointer in project hugegraph-computer by hugegraph.

the class VertexValueCombiner method combine.

@Override
public Pointer combine(Pointer v1, Pointer v2) {
    try {
        RandomAccessInput input1 = v1.input();
        RandomAccessInput input2 = v2.input();
        input1.seek(v1.offset());
        input2.seek(v2.offset());
        String label1 = StreamGraphInput.readLabel(input1);
        String label2 = StreamGraphInput.readLabel(input2);
        assert label1.equals(label2);
        this.v1.read(input1);
        this.v2.read(input2);
        this.combiner.combine(this.v1, this.v2, this.result);
        this.output.seek(0L);
        this.output.writeUTF(label1);
        this.result.write(this.output);
        return new InlinePointer(this.output.buffer(), this.output.position());
    } catch (Exception e) {
        throw new ComputerException("Failed to combine pointer1(offset=%s, length=%s) and " + "pointer2(offset=%s, length=%s)'", e, v1.offset(), v1.length(), v2.offset(), v2.length());
    }
}

Example 2 with InlinePointer

use of com.baidu.hugegraph.computer.core.store.entry.InlinePointer in project hugegraph-computer by hugegraph.

the class PointerCombinerTest method testMessageCombiner.

@Test
public void testMessageCombiner() throws IOException {
    Config config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.WORKER_COMBINER_CLASS, DoubleValueSumCombiner.class.getName());
    Combiner<DoubleValue> valueCombiner = config.createObject(ComputerOptions.WORKER_COMBINER_CLASS);
    PointerCombiner combiner = SorterTestUtil.createPointerCombiner(DoubleValue::new, new DoubleValueSumCombiner());
    try (BytesOutput bytesOutput1 = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE);
        BytesOutput bytesOutput2 = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE)) {
        DoubleValue value1 = new DoubleValue(1.0D);
        DoubleValue value2 = new DoubleValue(2.0D);
        value1.write(bytesOutput1);
        value2.write(bytesOutput2);
        Pointer pointer1 = new InlinePointer(bytesOutput1.buffer(), bytesOutput1.position());
        Pointer pointer2 = new InlinePointer(bytesOutput2.buffer(), bytesOutput2.position());
        Pointer pointer = combiner.combine(pointer1, pointer2);
        BytesInput input = IOFactory.createBytesInput(pointer.bytes());
        DoubleValue combinedValue = new DoubleValue();
        combinedValue.read(input);
        Assert.assertEquals(new DoubleValue(3.0D), combinedValue);
    }
}

Example 3 with InlinePointer

use of com.baidu.hugegraph.computer.core.store.entry.InlinePointer in project hugegraph-computer by hugegraph.

the class PointerCombinerTest method testVertexPropertiesCombiner.

@Test
public void testVertexPropertiesCombiner() throws IOException {
    Config config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.WORKER_COMBINER_CLASS, DoubleValueSumCombiner.class.getName(), ComputerOptions.WORKER_VERTEX_PROPERTIES_COMBINER_CLASS, MergeOldPropertiesCombiner.class.getName());
    Combiner<Properties> valueCombiner = config.createObject(ComputerOptions.WORKER_VERTEX_PROPERTIES_COMBINER_CLASS);
    GraphFactory graphFactory = graphFactory();
    PointerCombiner combiner = SorterTestUtil.createPointerCombiner(graphFactory::createProperties, valueCombiner);
    try (BytesOutput bytesOutput1 = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE);
        BytesOutput bytesOutput2 = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE)) {
        Properties value1 = graphFactory.createProperties();
        value1.put("p1", new LongValue(1L));
        Properties value2 = graphFactory.createProperties();
        value2.put("p2", new LongValue(2L));
        value1.write(bytesOutput1);
        value2.write(bytesOutput2);
        Pointer pointer1 = new InlinePointer(bytesOutput1.buffer(), bytesOutput1.position());
        Pointer pointer2 = new InlinePointer(bytesOutput2.buffer(), bytesOutput2.position());
        Pointer pointer = combiner.combine(pointer1, pointer2);
        BytesInput input = IOFactory.createBytesInput(pointer.bytes());
        Properties combinedValue = graphFactory.createProperties();
        combinedValue.read(input);
        Map<String, Value> map = combinedValue.get();
        Assert.assertEquals(2, map.size());
        Assert.assertEquals(new LongValue(1L), map.get("p1"));
        Assert.assertEquals(new LongValue(2L), map.get("p2"));
    }
}

Example 4 with InlinePointer

use of com.baidu.hugegraph.computer.core.store.entry.InlinePointer in project hugegraph-computer by hugegraph.

the class PointerCombinerTest method testCombineEdgePropertiesFail.

@Test
public void testCombineEdgePropertiesFail() throws IOException {
    Config config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.WORKER_COMBINER_CLASS, DoubleValueSumCombiner.class.getName(), ComputerOptions.WORKER_EDGE_PROPERTIES_COMBINER_CLASS, MergeOldPropertiesCombiner.class.getName());
    Combiner<Properties> valueCombiner = config.createObject(ComputerOptions.WORKER_EDGE_PROPERTIES_COMBINER_CLASS);
    GraphFactory graphFactory = graphFactory();
    PointerCombiner combiner = SorterTestUtil.createPointerCombiner(graphFactory::createProperties, valueCombiner);
    try (BytesOutput bytesOutput1 = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE);
        BytesOutput bytesOutput2 = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE)) {
        Properties value1 = graphFactory.createProperties();
        value1.put("p1", new LongValue(1L));
        Properties value2 = graphFactory.createProperties();
        value2.put("p2", new LongValue(2L));
        // Only write count.
        bytesOutput1.writeInt(1);
        value2.write(bytesOutput2);
        Pointer pointer1 = new InlinePointer(bytesOutput1.buffer(), bytesOutput1.position());
        Pointer pointer2 = new InlinePointer(bytesOutput2.buffer(), bytesOutput2.position());
        Assert.assertThrows(ComputerException.class, () -> {
            combiner.combine(pointer1, pointer2);
        }, e -> {
            Assert.assertContains("Failed to combine pointer", e.getMessage());
        });
    }
}

Example 5 with InlinePointer

use of com.baidu.hugegraph.computer.core.store.entry.InlinePointer in project hugegraph-computer by hugegraph.

the class SortLargeDataTest method testDiffNumEntriesFileMerge.

@Test
public void testDiffNumEntriesFileMerge() throws Exception {
    Config config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.HGKV_MERGE_FILES_NUM, "3", ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
    List<Integer> sizeList = ImmutableList.of(200, 500, 20, 50, 300, 250, 10, 33, 900, 89, 20);
    List<String> inputs = new ArrayList<>();
    for (int j = 0; j < sizeList.size(); j++) {
        String file = StoreTestUtil.availablePathById(j + 10);
        inputs.add(file);
        try (KvEntryFileWriter builder = new HgkvDirBuilderImpl(config, file)) {
            for (int i = 0; i < sizeList.get(j); i++) {
                byte[] keyBytes = StoreTestUtil.intToByteArray(i);
                byte[] valueBytes = StoreTestUtil.intToByteArray(1);
                Pointer key = new InlinePointer(keyBytes);
                Pointer value = new InlinePointer(valueBytes);
                KvEntry entry = new DefaultKvEntry(key, value);
                builder.write(entry);
            }
        }
    }
    List<String> outputs = ImmutableList.of(StoreTestUtil.availablePathById(0), StoreTestUtil.availablePathById(1), StoreTestUtil.availablePathById(2), StoreTestUtil.availablePathById(3));
    Sorter sorter = SorterTestUtil.createSorter(config);
    sorter.mergeInputs(inputs, new KvOuterSortFlusher(), outputs, false);
    int total = sizeList.stream().mapToInt(i -> i).sum();
    int mergeTotal = 0;
    for (String output : outputs) {
        mergeTotal += HgkvDirImpl.open(output).numEntries();
    }
    Assert.assertEquals(total, mergeTotal);
}