Examples with Pointer com.baidu.hugegraph.computer.core.store.entry.Pointer used on opensource projects

Example 6 with Pointer

use of com.baidu.hugegraph.computer.core.store.entry.Pointer in project hugegraph-computer by hugegraph.

the class VertexMessageRecvPartitionTest method checkTenVertexWithMergedProperties.

private static void checkTenVertexWithMergedProperties(PeekableIterator<KvEntry> it) throws IOException {
    for (long i = 0L; i < 10L; i++) {
        // Assert key
        Assert.assertTrue(it.hasNext());
        KvEntry entry = it.next();
        Id id = ReceiverUtil.readId(entry.key());
        Assert.assertEquals(BytesId.of(i), id);
        // Assert value
        Pointer value = entry.value();
        RandomAccessInput input = value.input();
        long position = input.position();
        input.seek(value.offset());
        String label = StreamGraphInput.readLabel(input);
        Assert.assertEquals("", label);
        Properties properties = graphFactory().createProperties();
        properties.read(input);
        input.seek(position);
        Assert.assertEquals(2, properties.size());
        LongValue v1 = properties.get("p1");
        Assert.assertEquals(new LongValue(i), v1);
        LongValue v2 = properties.get("p2");
        Assert.assertEquals(new LongValue(2L * i), v2);
    }
}

Example 7 with Pointer

use of com.baidu.hugegraph.computer.core.store.entry.Pointer in project hugegraph-computer by hugegraph.

the class SortLargeDataTest method testDiffNumEntriesFileMerge.

@Test
public void testDiffNumEntriesFileMerge() throws Exception {
    Config config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.HGKV_MERGE_FILES_NUM, "3", ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
    List<Integer> sizeList = ImmutableList.of(200, 500, 20, 50, 300, 250, 10, 33, 900, 89, 20);
    List<String> inputs = new ArrayList<>();
    for (int j = 0; j < sizeList.size(); j++) {
        String file = StoreTestUtil.availablePathById(j + 10);
        inputs.add(file);
        try (KvEntryFileWriter builder = new HgkvDirBuilderImpl(config, file)) {
            for (int i = 0; i < sizeList.get(j); i++) {
                byte[] keyBytes = StoreTestUtil.intToByteArray(i);
                byte[] valueBytes = StoreTestUtil.intToByteArray(1);
                Pointer key = new InlinePointer(keyBytes);
                Pointer value = new InlinePointer(valueBytes);
                KvEntry entry = new DefaultKvEntry(key, value);
                builder.write(entry);
            }
        }
    }
    List<String> outputs = ImmutableList.of(StoreTestUtil.availablePathById(0), StoreTestUtil.availablePathById(1), StoreTestUtil.availablePathById(2), StoreTestUtil.availablePathById(3));
    Sorter sorter = SorterTestUtil.createSorter(config);
    sorter.mergeInputs(inputs, new KvOuterSortFlusher(), outputs, false);
    int total = sizeList.stream().mapToInt(i -> i).sum();
    int mergeTotal = 0;
    for (String output : outputs) {
        mergeTotal += HgkvDirImpl.open(output).numEntries();
    }
    Assert.assertEquals(total, mergeTotal);
}

Example 8 with Pointer

use of com.baidu.hugegraph.computer.core.store.entry.Pointer in project hugegraph-computer by hugegraph.

the class PointerTest method test.

@Test
public void test() throws IOException {
    byte[] data = new byte[] { 100, 0, 0, 0 };
    byte[] expectedWriteResult = { 4, 0, 0, 0, 100, 0, 0, 0, 4, 0, 0, 0, 100, 0, 0, 0 };
    BytesOutput output = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE);
    output.writeFixedInt(data.length);
    output.write(data);
    output.writeFixedInt(data.length);
    output.write(data);
    BytesInput input = EntriesUtil.inputFromOutput(output);
    KvEntry inlineKvEntry = EntriesUtil.kvEntryFromInput(input, true, false);
    Pointer inlineKey = inlineKvEntry.key();
    Pointer inlineValue = inlineKvEntry.value();
    Assert.assertEquals(0L, inlineKey.offset());
    Assert.assertEquals(4L, inlineKey.length());
    Assert.assertEquals(0, BytesUtil.compare(data, inlineKey.bytes()));
    Assert.assertEquals(0L, inlineValue.offset());
    Assert.assertEquals(4L, inlineValue.length());
    Assert.assertEquals(0, BytesUtil.compare(data, inlineValue.bytes()));
    BytesOutput writeOutput = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE);
    inlineKey.write(writeOutput);
    inlineValue.write(writeOutput);
    int result = BytesUtil.compare(expectedWriteResult, expectedWriteResult.length, writeOutput.buffer(), (int) writeOutput.position());
    Assert.assertEquals(0, result);
    input.seek(0);
    KvEntry cachedKvEntry = EntriesUtil.kvEntryFromInput(input, false, false);
    Pointer cachedKey = cachedKvEntry.key();
    Pointer cachedValue = cachedKvEntry.value();
    Assert.assertEquals(4L, cachedKey.offset());
    Assert.assertEquals(4L, cachedKey.length());
    Assert.assertEquals(0, BytesUtil.compare(data, cachedKey.bytes()));
    Assert.assertEquals(12L, cachedValue.offset());
    Assert.assertEquals(4L, cachedValue.length());
    Assert.assertEquals(0, BytesUtil.compare(data, cachedValue.bytes()));
    writeOutput = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE);
    cachedKey.write(writeOutput);
    cachedValue.write(writeOutput);
    result = BytesUtil.compare(expectedWriteResult, expectedWriteResult.length, writeOutput.buffer(), (int) writeOutput.position());
    Assert.assertEquals(0, result);
}

Example 9 with Pointer

use of com.baidu.hugegraph.computer.core.store.entry.Pointer in project hugegraph-computer by hugegraph.

the class FileGraphPartition method writeEdges.

private void writeEdges(Pointer vid, PeekableIterator<KvEntry> edges, BufferedFileOutput edgeOut) throws IOException {
    byte[] vidBytes = vid.bytes();
    while (edges.hasNext()) {
        KvEntry entry = edges.peek();
        Pointer key = entry.key();
        int matched = vid.compareTo(key);
        if (matched < 0) {
            return;
        }
        edges.next();
        if (matched > 0) {
            // Skip stale edges
            continue;
        }
        assert matched == 0;
        edgeOut.writeFixedInt(vidBytes.length);
        edgeOut.write(vidBytes);
        long valuePosition = edgeOut.position();
        edgeOut.writeFixedInt(0);
        this.edgeCount += entry.numSubEntries();
        edgeOut.writeFixedInt((int) entry.numSubEntries());
        EntryIterator subKvIt = EntriesUtil.subKvIterFromEntry(entry);
        while (subKvIt.hasNext()) {
            KvEntry subEntry = subKvIt.next();
            // Not write sub-key length
            edgeOut.write(subEntry.key().bytes());
            // Not write sub-value length
            edgeOut.write(subEntry.value().bytes());
        }
        long valueLength = edgeOut.position() - valuePosition - Constants.INT_LEN;
        edgeOut.writeFixedInt(valuePosition, (int) valueLength);
    }
}

Example 10 with Pointer

use of com.baidu.hugegraph.computer.core.store.entry.Pointer in project hugegraph-computer by hugegraph.

the class FileGraphPartition method input.

protected PartitionStat input(PeekableIterator<KvEntry> vertices, PeekableIterator<KvEntry> edges) {
    try {
        createFile(this.vertexFile);
        createFile(this.edgeFile);
        BufferedFileOutput vertexOut = new BufferedFileOutput(this.vertexFile);
        BufferedFileOutput edgeOut = new BufferedFileOutput(this.edgeFile);
        while (vertices.hasNext()) {
            KvEntry entry = vertices.next();
            Pointer key = entry.key();
            Pointer value = entry.value();
            this.writeVertex(key, value, vertexOut);
            this.writeEdges(key, edges, edgeOut);
        }
        vertexOut.close();
        edgeOut.close();
    } catch (IOException e) {
        throw new ComputerException("Failed to init FileGraphPartition '%s'", e, this.partition);
    }
    return new PartitionStat(this.partition, this.vertexCount, this.edgeCount, 0L);
}