Examples with Pointer com.baidu.hugegraph.computer.core.store.entry.Pointer used on opensource projects

Example 1 with Pointer

use of com.baidu.hugegraph.computer.core.store.entry.Pointer in project hugegraph-computer by hugegraph.

the class CombinableSorterFlusher method flush.

public void flush(Iterator<KvEntry> entries) throws IOException {
    E.checkArgument(entries.hasNext(), "Parameter entries can't be empty");
    KvEntry last = entries.next();
    Pointer combineValue = last.value();
    while (true) {
        KvEntry current = null;
        if (entries.hasNext()) {
            current = entries.next();
            if (last.compareTo(current) == 0) {
                combineValue = this.combiner.combine(combineValue, current.value());
                continue;
            }
        }
        this.writeKvEntry(new DefaultKvEntry(last.key(), combineValue));
        if (current == null) {
            break;
        }
        last = current;
        combineValue = last.value();
    }
}

Example 2 with Pointer

use of com.baidu.hugegraph.computer.core.store.entry.Pointer in project hugegraph-computer by hugegraph.

the class CombineSubKvInnerSortFlusher method writeSubKvs.

private void writeSubKvs(KvEntry kvEntry, Iterator<KvEntry> subKvIter) throws IOException {
    E.checkArgument(subKvIter.hasNext(), "Parameter subKvs can't be empty");
    kvEntry.key().write(this.output);
    long position = this.output.position();
    // Write value length placeholder
    this.output.writeFixedInt(0);
    // Write subKv count placeholder
    this.output.writeFixedInt(0);
    // Write subKv to output
    KvEntry lastSubKv = subKvIter.next();
    Pointer lastSubValue = lastSubKv.value();
    int writtenCount = 0;
    while (true) {
        // Write subKv
        KvEntry current = null;
        if (subKvIter.hasNext()) {
            current = subKvIter.next();
            if (lastSubKv.compareTo(current) == 0) {
                lastSubValue = this.combiner.combine(lastSubValue, current.value());
                continue;
            }
        }
        lastSubKv.key().write(this.output);
        lastSubValue.write(this.output);
        writtenCount++;
        if (writtenCount == this.subKvFlushThreshold || current == null) {
            // Fill placeholder
            long currentPosition = this.output.position();
            this.output.seek(position);
            // Fill value length placeholder
            this.output.writeFixedInt((int) (currentPosition - position - 4));
            // Fill subKv count placeholder
            this.output.writeFixedInt(writtenCount);
            this.output.seek(currentPosition);
            if (current == null) {
                break;
            }
            // Used for next loop
            kvEntry.key().write(this.output);
            position = this.output.position();
            // Write value length placeholder
            this.output.writeFixedInt(0);
            // Write subKv count placeholder
            this.output.writeFixedInt(0);
            writtenCount = 0;
        }
        lastSubKv = current;
        lastSubValue = current.value();
    }
}

Example 3 with Pointer

use of com.baidu.hugegraph.computer.core.store.entry.Pointer in project hugegraph-computer by hugegraph.

the class PointerCombinerTest method testMessageCombiner.

@Test
public void testMessageCombiner() throws IOException {
    Config config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.WORKER_COMBINER_CLASS, DoubleValueSumCombiner.class.getName());
    Combiner<DoubleValue> valueCombiner = config.createObject(ComputerOptions.WORKER_COMBINER_CLASS);
    PointerCombiner combiner = SorterTestUtil.createPointerCombiner(DoubleValue::new, new DoubleValueSumCombiner());
    try (BytesOutput bytesOutput1 = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE);
        BytesOutput bytesOutput2 = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE)) {
        DoubleValue value1 = new DoubleValue(1.0D);
        DoubleValue value2 = new DoubleValue(2.0D);
        value1.write(bytesOutput1);
        value2.write(bytesOutput2);
        Pointer pointer1 = new InlinePointer(bytesOutput1.buffer(), bytesOutput1.position());
        Pointer pointer2 = new InlinePointer(bytesOutput2.buffer(), bytesOutput2.position());
        Pointer pointer = combiner.combine(pointer1, pointer2);
        BytesInput input = IOFactory.createBytesInput(pointer.bytes());
        DoubleValue combinedValue = new DoubleValue();
        combinedValue.read(input);
        Assert.assertEquals(new DoubleValue(3.0D), combinedValue);
    }
}

Example 4 with Pointer

use of com.baidu.hugegraph.computer.core.store.entry.Pointer in project hugegraph-computer by hugegraph.

the class PointerCombinerTest method testVertexPropertiesCombiner.

@Test
public void testVertexPropertiesCombiner() throws IOException {
    Config config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.WORKER_COMBINER_CLASS, DoubleValueSumCombiner.class.getName(), ComputerOptions.WORKER_VERTEX_PROPERTIES_COMBINER_CLASS, MergeOldPropertiesCombiner.class.getName());
    Combiner<Properties> valueCombiner = config.createObject(ComputerOptions.WORKER_VERTEX_PROPERTIES_COMBINER_CLASS);
    GraphFactory graphFactory = graphFactory();
    PointerCombiner combiner = SorterTestUtil.createPointerCombiner(graphFactory::createProperties, valueCombiner);
    try (BytesOutput bytesOutput1 = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE);
        BytesOutput bytesOutput2 = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE)) {
        Properties value1 = graphFactory.createProperties();
        value1.put("p1", new LongValue(1L));
        Properties value2 = graphFactory.createProperties();
        value2.put("p2", new LongValue(2L));
        value1.write(bytesOutput1);
        value2.write(bytesOutput2);
        Pointer pointer1 = new InlinePointer(bytesOutput1.buffer(), bytesOutput1.position());
        Pointer pointer2 = new InlinePointer(bytesOutput2.buffer(), bytesOutput2.position());
        Pointer pointer = combiner.combine(pointer1, pointer2);
        BytesInput input = IOFactory.createBytesInput(pointer.bytes());
        Properties combinedValue = graphFactory.createProperties();
        combinedValue.read(input);
        Map<String, Value> map = combinedValue.get();
        Assert.assertEquals(2, map.size());
        Assert.assertEquals(new LongValue(1L), map.get("p1"));
        Assert.assertEquals(new LongValue(2L), map.get("p2"));
    }
}

Example 5 with Pointer

use of com.baidu.hugegraph.computer.core.store.entry.Pointer in project hugegraph-computer by hugegraph.

the class PointerCombinerTest method testCombineEdgePropertiesFail.

@Test
public void testCombineEdgePropertiesFail() throws IOException {
    Config config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.WORKER_COMBINER_CLASS, DoubleValueSumCombiner.class.getName(), ComputerOptions.WORKER_EDGE_PROPERTIES_COMBINER_CLASS, MergeOldPropertiesCombiner.class.getName());
    Combiner<Properties> valueCombiner = config.createObject(ComputerOptions.WORKER_EDGE_PROPERTIES_COMBINER_CLASS);
    GraphFactory graphFactory = graphFactory();
    PointerCombiner combiner = SorterTestUtil.createPointerCombiner(graphFactory::createProperties, valueCombiner);
    try (BytesOutput bytesOutput1 = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE);
        BytesOutput bytesOutput2 = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE)) {
        Properties value1 = graphFactory.createProperties();
        value1.put("p1", new LongValue(1L));
        Properties value2 = graphFactory.createProperties();
        value2.put("p2", new LongValue(2L));
        // Only write count.
        bytesOutput1.writeInt(1);
        value2.write(bytesOutput2);
        Pointer pointer1 = new InlinePointer(bytesOutput1.buffer(), bytesOutput1.position());
        Pointer pointer2 = new InlinePointer(bytesOutput2.buffer(), bytesOutput2.position());
        Assert.assertThrows(ComputerException.class, () -> {
            combiner.combine(pointer1, pointer2);
        }, e -> {
            Assert.assertContains("Failed to combine pointer", e.getMessage());
        });
    }
}