Examples with KvEntry com.baidu.hugegraph.computer.core.store.entry.KvEntry used on opensource projects

Example 31 with KvEntry

use of com.baidu.hugegraph.computer.core.store.entry.KvEntry in project hugegraph-computer by hugegraph.

the class MessageInputTest method testMessageInput.

@Test
public void testMessageInput() throws IOException {
    MessageRecvManager receiveManager = this.managers.get(MessageRecvManager.NAME);
    receiveManager.onStarted(this.connectionId);
    // Superstep 0
    receiveManager.beforeSuperstep(this.config, 0);
    receiveManager.onStarted(this.connectionId);
    addMessages((NetworkBuffer buffer) -> {
        receiveManager.handle(MessageType.MSG, 0, buffer);
    });
    receiveManager.onFinished(this.connectionId);
    PeekableIterator<KvEntry> it = receiveManager.messagePartitions().get(0);
    MessageInput<IdList> input = new MessageInput<>(context(), it);
    Map<Id, List<IdList>> expectedMessages = expectedMessages();
    checkMessages(expectedMessages, input);
}

Example 32 with KvEntry

use of com.baidu.hugegraph.computer.core.store.entry.KvEntry in project hugegraph-computer by hugegraph.

the class ComputeManager method input.

public WorkerStat input() {
    WorkerStat workerStat = new WorkerStat();
    this.recvManager.waitReceivedAllMessages();
    Map<Integer, PeekableIterator<KvEntry>> vertices = this.recvManager.vertexPartitions();
    Map<Integer, PeekableIterator<KvEntry>> edges = this.recvManager.edgePartitions();
    // TODO: parallel input process
    for (Map.Entry<Integer, PeekableIterator<KvEntry>> entry : vertices.entrySet()) {
        int partition = entry.getKey();
        PeekableIterator<KvEntry> vertexIter = entry.getValue();
        PeekableIterator<KvEntry> edgesIter = edges.getOrDefault(partition, PeekableIterator.emptyIterator());
        FileGraphPartition part = new FileGraphPartition(this.context, this.managers, partition);
        PartitionStat partitionStat = null;
        ComputerException inputException = null;
        try {
            partitionStat = part.input(vertexIter, edgesIter);
        } catch (ComputerException e) {
            inputException = e;
        } finally {
            try {
                vertexIter.close();
                edgesIter.close();
            } catch (Exception e) {
                String message = "Failed to close vertex or edge file " + "iterator";
                ComputerException closeException = new ComputerException(message, e);
                if (inputException != null) {
                    inputException.addSuppressed(closeException);
                } else {
                    throw closeException;
                }
            }
            if (inputException != null) {
                throw inputException;
            }
        }
        workerStat.add(partitionStat);
        this.partitions.put(partition, part);
    }
    return workerStat;
}

Example 33 with KvEntry

use of com.baidu.hugegraph.computer.core.store.entry.KvEntry in project hugegraph-computer by hugegraph.

the class FileGraphPartition method writeEdges.

private void writeEdges(Pointer vid, PeekableIterator<KvEntry> edges, BufferedFileOutput edgeOut) throws IOException {
    byte[] vidBytes = vid.bytes();
    while (edges.hasNext()) {
        KvEntry entry = edges.peek();
        Pointer key = entry.key();
        int matched = vid.compareTo(key);
        if (matched < 0) {
            return;
        }
        edges.next();
        if (matched > 0) {
            // Skip stale edges
            continue;
        }
        assert matched == 0;
        edgeOut.writeFixedInt(vidBytes.length);
        edgeOut.write(vidBytes);
        long valuePosition = edgeOut.position();
        edgeOut.writeFixedInt(0);
        this.edgeCount += entry.numSubEntries();
        edgeOut.writeFixedInt((int) entry.numSubEntries());
        EntryIterator subKvIt = EntriesUtil.subKvIterFromEntry(entry);
        while (subKvIt.hasNext()) {
            KvEntry subEntry = subKvIt.next();
            // Not write sub-key length
            edgeOut.write(subEntry.key().bytes());
            // Not write sub-value length
            edgeOut.write(subEntry.value().bytes());
        }
        long valueLength = edgeOut.position() - valuePosition - Constants.INT_LEN;
        edgeOut.writeFixedInt(valuePosition, (int) valueLength);
    }
}

Example 34 with KvEntry

use of com.baidu.hugegraph.computer.core.store.entry.KvEntry in project hugegraph-computer by hugegraph.

the class FileGraphPartition method input.

protected PartitionStat input(PeekableIterator<KvEntry> vertices, PeekableIterator<KvEntry> edges) {
    try {
        createFile(this.vertexFile);
        createFile(this.edgeFile);
        BufferedFileOutput vertexOut = new BufferedFileOutput(this.vertexFile);
        BufferedFileOutput edgeOut = new BufferedFileOutput(this.edgeFile);
        while (vertices.hasNext()) {
            KvEntry entry = vertices.next();
            Pointer key = entry.key();
            Pointer value = entry.value();
            this.writeVertex(key, value, vertexOut);
            this.writeEdges(key, edges, edgeOut);
        }
        vertexOut.close();
        edgeOut.close();
    } catch (IOException e) {
        throw new ComputerException("Failed to init FileGraphPartition '%s'", e, this.partition);
    }
    return new PartitionStat(this.partition, this.vertexCount, this.edgeCount, 0L);
}

Example 35 with KvEntry

use of com.baidu.hugegraph.computer.core.store.entry.KvEntry in project hugegraph-computer by hugegraph.

the class MessageInput method iterator.

public Iterator<T> iterator(ReusablePointer vidPointer) {
    while (this.messages.hasNext()) {
        KvEntry entry = this.messages.peek();
        Pointer key = entry.key();
        int status = vidPointer.compareTo(key);
        if (status < 0) {
            return Collections.emptyIterator();
        } else if (status == 0) {
            break;
        } else {
            this.messages.next();
        }
    }
    return new MessageIterator(vidPointer);
}