Examples with BytesOutput com.baidu.hugegraph.computer.core.io.BytesOutput used on opensource projects

Example 1 with BytesOutput

use of com.baidu.hugegraph.computer.core.io.BytesOutput in project hugegraph-computer by hugegraph.

the class StoreTestUtil method kvEntriesFromMap.

public static List<KvEntry> kvEntriesFromMap(List<Integer> map) throws IOException {
    BytesOutput data = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE);
    Iterator<Integer> iterator = map.iterator();
    while (iterator.hasNext()) {
        // Write key length
        writeData(data, iterator.next());
        // Write value length
        writeData(data, iterator.next());
    }
    BytesInput input = IOFactory.createBytesInput(data.buffer(), (int) data.position());
    KvEntriesInput iter = new KvEntriesInput(input);
    List<KvEntry> entries = new ArrayList<>();
    while (iter.hasNext()) {
        entries.add(iter.next());
    }
    iter.close();
    return entries;
}

Example 2 with BytesOutput

use of com.baidu.hugegraph.computer.core.io.BytesOutput in project hugegraph-computer by hugegraph.

the class SortManager method sort.

public CompletableFuture<ByteBuffer> sort(MessageType type, WriteBuffers buffer) {
    return CompletableFuture.supplyAsync(() -> {
        RandomAccessInput bufferForRead = buffer.wrapForRead();
        // TODO：This ByteBuffer should be allocated from the off-heap
        BytesOutput output = IOFactory.createBytesOutput(this.capacity);
        InnerSortFlusher flusher = this.createSortFlusher(type, output, this.flushThreshold);
        try {
            this.sorter.sortBuffer(bufferForRead, flusher, type == MessageType.EDGE);
        } catch (Exception e) {
            throw new ComputerException("Failed to sort buffers of %s " + "message", e, type.name());
        }
        return ByteBuffer.wrap(output.buffer(), 0, (int) output.position());
    }, this.sortExecutor);
}

Example 3 with BytesOutput

use of com.baidu.hugegraph.computer.core.io.BytesOutput in project hugegraph-computer by hugegraph.

the class SortLargeDataTest method testMergeBuffers.

@Test
public void testMergeBuffers() throws Exception {
    StopWatch watcher = new StopWatch();
    // Sort buffers total size 100M, each buffer is 50KB
    final long bufferSize = Bytes.KB * 50;
    final long bufferNum = 2000;
    final int keyRange = 10000000;
    long totalValue = 0L;
    Random random = new Random();
    List<RandomAccessInput> buffers = new ArrayList<>();
    for (int i = 0; i < bufferNum; i++) {
        BytesOutput buffer = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE);
        while (buffer.position() < bufferSize) {
            // Write data
            int key = random.nextInt(keyRange);
            SorterTestUtil.writeData(buffer, key);
            int value = random.nextInt(100);
            SorterTestUtil.writeData(buffer, value);
            totalValue += value;
        }
        buffers.add(EntriesUtil.inputFromOutput(buffer));
    }
    // Sort buffer
    Sorter sorter = SorterTestUtil.createSorter(CONFIG);
    watcher.start();
    List<RandomAccessInput> sortedBuffers = new ArrayList<>();
    for (RandomAccessInput buffer : buffers) {
        RandomAccessInput sortedBuffer = sortBuffer(sorter, buffer);
        sortedBuffers.add(sortedBuffer);
    }
    watcher.stop();
    LOG.info("testMergeBuffers sort buffer cost time: {}", watcher.getTime());
    String resultFile = StoreTestUtil.availablePathById("0");
    // Sort buffers
    watcher.reset();
    watcher.start();
    sorter.mergeBuffers(sortedBuffers, new KvOuterSortFlusher(), resultFile, false);
    watcher.stop();
    LOG.info("testMergeBuffers merge buffers cost time: {}", watcher.getTime());
    // Assert result
    long result = sumOfEntryValue(sorter, ImmutableList.of(resultFile));
    Assert.assertEquals(totalValue, result);
    assertFileOrder(sorter, ImmutableList.of(resultFile));
}

Example 4 with BytesOutput

use of com.baidu.hugegraph.computer.core.io.BytesOutput in project hugegraph-computer by hugegraph.

the class SortLargeDataTest method testAllProcess.

@Test
public void testAllProcess() throws Exception {
    StopWatch watcher = new StopWatch();
    final long bufferSize = Bytes.MB;
    final int mergeBufferNum = 300;
    final int dataSize = 1000000;
    long value = 0;
    Random random = new Random();
    BytesOutput output = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE);
    List<RandomAccessInput> buffers = new ArrayList<>(mergeBufferNum);
    List<String> mergeBufferFiles = new ArrayList<>();
    int fileNum = 10;
    Sorter sorter = SorterTestUtil.createSorter(CONFIG);
    watcher.start();
    for (int i = 0; i < dataSize; i++) {
        SorterTestUtil.writeData(output, random.nextInt(dataSize));
        int entryValue = random.nextInt(5);
        SorterTestUtil.writeData(output, entryValue);
        value = value + entryValue;
        // Write data to buffer and sort buffer
        if (output.position() >= bufferSize || (i + 1) == dataSize) {
            BytesInput input = EntriesUtil.inputFromOutput(output);
            buffers.add(sortBuffer(sorter, input));
            output.seek(0);
        }
        // Merge buffers to HgkvDir
        if (buffers.size() >= mergeBufferNum || (i + 1) == dataSize) {
            String outputFile = StoreTestUtil.availablePathById(fileNum++);
            mergeBufferFiles.add(outputFile);
            mergeBuffers(sorter, buffers, outputFile);
            buffers.clear();
        }
    }
    // Merge file
    String resultFile = StoreTestUtil.availablePathById("0");
    mergeFiles(sorter, mergeBufferFiles, Lists.newArrayList(resultFile));
    watcher.stop();
    LOG.info("testAllProcess sort time: {}", watcher.getTime());
    long result = sumOfEntryValue(sorter, ImmutableList.of(resultFile));
    Assert.assertEquals(value, result);
}

Example 5 with BytesOutput

use of com.baidu.hugegraph.computer.core.io.BytesOutput in project hugegraph-computer by hugegraph.

the class SortLargeDataTest method sortBuffer.

private static RandomAccessInput sortBuffer(Sorter sorter, RandomAccessInput input) throws Exception {
    BytesOutput output = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE);
    PointerCombiner combiner = SorterTestUtil.createPointerCombiner(IntValue::new, new IntValueSumCombiner());
    InnerSortFlusher flusher = new CombineKvInnerSortFlusher(output, combiner);
    sorter.sortBuffer(input, flusher, false);
    return EntriesUtil.inputFromOutput(output);
}