Examples with RandomAccessInput com.baidu.hugegraph.computer.core.io.RandomAccessInput used on opensource projects

Example 1 with RandomAccessInput

use of com.baidu.hugegraph.computer.core.io.RandomAccessInput in project hugegraph-computer by hugegraph.

the class HgkvFileImpl method readFooter.

private void readFooter() throws IOException {
    File file = new File(this.path);
    // The footerLength occupied 4 bytes, versionLength 2 * 2 bytes
    long versionOffset = file.length() - Short.BYTES * 2 - Integer.BYTES;
    try (RandomAccessInput input = IOFactory.createFileInput(file)) {
        input.seek(versionOffset);
        // Read version
        short majorVersion = input.readShort();
        short minorVersion = input.readShort();
        String version = version(majorVersion, minorVersion);
        // Read footerLength
        int footerLength = input.readFixedInt();
        switch(version) {
            case "1.0":
                this.readFooterV1d0(input, file.length() - footerLength);
                break;
            default:
                throw new ComputerException("Illegal HgkvFile version '%s'", version);
        }
    }
}

Example 2 with RandomAccessInput

use of com.baidu.hugegraph.computer.core.io.RandomAccessInput in project hugegraph-computer by hugegraph.

the class CombineSubKvOuterSortFlusher method flush.

@Override
public void flush(EntryIterator entries, HgkvDirBuilder writer) throws IOException {
    E.checkArgument(entries.hasNext(), "Parameter entries can't be empty");
    PeekableIterator<KvEntry> kvEntries = PeekableIteratorAdaptor.of(entries);
    SubKvSorter sorter = new SubKvSorter(kvEntries, this.sources);
    KvEntry currentKv = sorter.currentKv();
    while (true) {
        currentKv.key().write(this.output);
        long position = this.output.position();
        // Write total sub-entry length placeholder
        this.output.writeFixedInt(0);
        // Write sub-entry count placeholder
        this.output.writeFixedInt(0);
        int writtenCount = 0;
        // Iterate subKv of currentKv
        KvEntry lastSubKv = sorter.next();
        Pointer lastSubValue = lastSubKv.value();
        while (true) {
            KvEntry current = null;
            if (sorter.hasNext()) {
                current = sorter.next();
                if (lastSubKv.compareTo(current) == 0) {
                    lastSubValue = this.combiner.combine(lastSubValue, current.value());
                    continue;
                }
            }
            lastSubKv.key().write(this.output);
            lastSubValue.write(this.output);
            writtenCount++;
            /*
                 * Fill placeholder if the number of subkvs with different
                 * keys is equal to the subKvFlushThreshold.
                 */
            if (current == null || writtenCount == this.subKvFlushThreshold) {
                long currentPosition = this.output.position();
                this.output.seek(position);
                this.output.writeFixedInt((int) (currentPosition - position - Integer.BYTES));
                this.output.writeFixedInt(writtenCount);
                this.output.seek(currentPosition);
                // Write kvEntry to file.
                RandomAccessInput input = EntriesUtil.inputFromOutput(this.output);
                writer.write(EntriesUtil.kvEntryFromInput(input, true, true));
                this.output.seek(0);
                if (current == null) {
                    break;
                }
                currentKv.key().write(this.output);
                position = this.output.position();
                // Write value length placeholder
                this.output.writeFixedInt(0);
                // Write subKv count placeholder
                this.output.writeFixedInt(0);
                writtenCount = 0;
            }
            lastSubKv = current;
            lastSubValue = lastSubKv.value();
        }
        sorter.reset();
        // Get next KV
        if ((currentKv = sorter.currentKv()) == null) {
            break;
        }
    }
    writer.finish();
}

Example 3 with RandomAccessInput

use of com.baidu.hugegraph.computer.core.io.RandomAccessInput in project hugegraph-computer by hugegraph.

the class VertexValueCombiner method combine.

@Override
public Pointer combine(Pointer v1, Pointer v2) {
    try {
        RandomAccessInput input1 = v1.input();
        RandomAccessInput input2 = v2.input();
        input1.seek(v1.offset());
        input2.seek(v2.offset());
        String label1 = StreamGraphInput.readLabel(input1);
        String label2 = StreamGraphInput.readLabel(input2);
        assert label1.equals(label2);
        this.v1.read(input1);
        this.v2.read(input2);
        this.combiner.combine(this.v1, this.v2, this.result);
        this.output.seek(0L);
        this.output.writeUTF(label1);
        this.result.write(this.output);
        return new InlinePointer(this.output.buffer(), this.output.position());
    } catch (Exception e) {
        throw new ComputerException("Failed to combine pointer1(offset=%s, length=%s) and " + "pointer2(offset=%s, length=%s)'", e, v1.offset(), v1.length(), v2.offset(), v2.length());
    }
}

Example 4 with RandomAccessInput

use of com.baidu.hugegraph.computer.core.io.RandomAccessInput in project hugegraph-computer by hugegraph.

the class SortManager method sort.

public CompletableFuture<ByteBuffer> sort(MessageType type, WriteBuffers buffer) {
    return CompletableFuture.supplyAsync(() -> {
        RandomAccessInput bufferForRead = buffer.wrapForRead();
        // TODO：This ByteBuffer should be allocated from the off-heap
        BytesOutput output = IOFactory.createBytesOutput(this.capacity);
        InnerSortFlusher flusher = this.createSortFlusher(type, output, this.flushThreshold);
        try {
            this.sorter.sortBuffer(bufferForRead, flusher, type == MessageType.EDGE);
        } catch (Exception e) {
            throw new ComputerException("Failed to sort buffers of %s " + "message", e, type.name());
        }
        return ByteBuffer.wrap(output.buffer(), 0, (int) output.position());
    }, this.sortExecutor);
}

Example 5 with RandomAccessInput

use of com.baidu.hugegraph.computer.core.io.RandomAccessInput in project hugegraph-computer by hugegraph.

the class SortLargeDataTest method testMergeBuffers.

@Test
public void testMergeBuffers() throws Exception {
    StopWatch watcher = new StopWatch();
    // Sort buffers total size 100M, each buffer is 50KB
    final long bufferSize = Bytes.KB * 50;
    final long bufferNum = 2000;
    final int keyRange = 10000000;
    long totalValue = 0L;
    Random random = new Random();
    List<RandomAccessInput> buffers = new ArrayList<>();
    for (int i = 0; i < bufferNum; i++) {
        BytesOutput buffer = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE);
        while (buffer.position() < bufferSize) {
            // Write data
            int key = random.nextInt(keyRange);
            SorterTestUtil.writeData(buffer, key);
            int value = random.nextInt(100);
            SorterTestUtil.writeData(buffer, value);
            totalValue += value;
        }
        buffers.add(EntriesUtil.inputFromOutput(buffer));
    }
    // Sort buffer
    Sorter sorter = SorterTestUtil.createSorter(CONFIG);
    watcher.start();
    List<RandomAccessInput> sortedBuffers = new ArrayList<>();
    for (RandomAccessInput buffer : buffers) {
        RandomAccessInput sortedBuffer = sortBuffer(sorter, buffer);
        sortedBuffers.add(sortedBuffer);
    }
    watcher.stop();
    LOG.info("testMergeBuffers sort buffer cost time: {}", watcher.getTime());
    String resultFile = StoreTestUtil.availablePathById("0");
    // Sort buffers
    watcher.reset();
    watcher.start();
    sorter.mergeBuffers(sortedBuffers, new KvOuterSortFlusher(), resultFile, false);
    watcher.stop();
    LOG.info("testMergeBuffers merge buffers cost time: {}", watcher.getTime());
    // Assert result
    long result = sumOfEntryValue(sorter, ImmutableList.of(resultFile));
    Assert.assertEquals(totalValue, result);
    assertFileOrder(sorter, ImmutableList.of(resultFile));
}