Search in sources :

Example 6 with RandomAccessInput

use of com.baidu.hugegraph.computer.core.io.RandomAccessInput in project hugegraph-computer by hugegraph.

the class SortLargeDataTest method testAllProcess.

@Test
public void testAllProcess() throws Exception {
    StopWatch watcher = new StopWatch();
    final long bufferSize = Bytes.MB;
    final int mergeBufferNum = 300;
    final int dataSize = 1000000;
    long value = 0;
    Random random = new Random();
    BytesOutput output = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE);
    List<RandomAccessInput> buffers = new ArrayList<>(mergeBufferNum);
    List<String> mergeBufferFiles = new ArrayList<>();
    int fileNum = 10;
    Sorter sorter = SorterTestUtil.createSorter(CONFIG);
    watcher.start();
    for (int i = 0; i < dataSize; i++) {
        SorterTestUtil.writeData(output, random.nextInt(dataSize));
        int entryValue = random.nextInt(5);
        SorterTestUtil.writeData(output, entryValue);
        value = value + entryValue;
        // Write data to buffer and sort buffer
        if (output.position() >= bufferSize || (i + 1) == dataSize) {
            BytesInput input = EntriesUtil.inputFromOutput(output);
            buffers.add(sortBuffer(sorter, input));
            output.seek(0);
        }
        // Merge buffers to HgkvDir
        if (buffers.size() >= mergeBufferNum || (i + 1) == dataSize) {
            String outputFile = StoreTestUtil.availablePathById(fileNum++);
            mergeBufferFiles.add(outputFile);
            mergeBuffers(sorter, buffers, outputFile);
            buffers.clear();
        }
    }
    // Merge file
    String resultFile = StoreTestUtil.availablePathById("0");
    mergeFiles(sorter, mergeBufferFiles, Lists.newArrayList(resultFile));
    watcher.stop();
    LOG.info("testAllProcess sort time: {}", watcher.getTime());
    long result = sumOfEntryValue(sorter, ImmutableList.of(resultFile));
    Assert.assertEquals(value, result);
}
Also used : RandomAccessInput(com.baidu.hugegraph.computer.core.io.RandomAccessInput) Random(java.util.Random) BytesOutput(com.baidu.hugegraph.computer.core.io.BytesOutput) BytesInput(com.baidu.hugegraph.computer.core.io.BytesInput) ArrayList(java.util.ArrayList) Sorter(com.baidu.hugegraph.computer.core.sort.Sorter) StopWatch(org.apache.commons.lang3.time.StopWatch) Test(org.junit.Test)

Example 7 with RandomAccessInput

use of com.baidu.hugegraph.computer.core.io.RandomAccessInput in project hugegraph-computer by hugegraph.

the class SorterTest method testSortKvBuffers.

@Test
public void testSortKvBuffers() throws Exception {
    Config config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.HGKV_MAX_FILE_SIZE, "32", ComputerOptions.HGKV_DATABLOCK_SIZE, "16", ComputerOptions.HGKV_MERGE_FILES_NUM, "3", ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
    List<Integer> map1 = ImmutableList.of(2, 3, 2, 1, 5, 2, 6, 9, 6, 2);
    List<Integer> map2 = ImmutableList.of(1, 3, 1, 1, 3, 2, 6, 9, 8, 2);
    String path = StoreTestUtil.availablePathById("1");
    // Merge 4 sorted input
    List<RandomAccessInput> inputs = ImmutableList.of(SorterTestUtil.inputFromKvMap(map1), SorterTestUtil.inputFromKvMap(map2), SorterTestUtil.inputFromKvMap(map1), SorterTestUtil.inputFromKvMap(map2));
    Sorter sorter = SorterTestUtil.createSorter(config);
    PointerCombiner combiner = SorterTestUtil.createPointerCombiner(IntValue::new, new IntValueSumCombiner());
    sorter.mergeBuffers(inputs, new CombineKvOuterSortFlusher(combiner), path, false);
    // Assert merge result from target hgkvDir
    KvEntryFileReader reader = new HgkvDirReaderImpl(path, false);
    EntryIterator iter = reader.iterator();
    SorterTestUtil.assertKvEntry(iter.next(), 1, 8);
    SorterTestUtil.assertKvEntry(iter.next(), 2, 8);
    SorterTestUtil.assertKvEntry(iter.next(), 3, 4);
    SorterTestUtil.assertKvEntry(iter.next(), 5, 4);
    SorterTestUtil.assertKvEntry(iter.next(), 6, 40);
    SorterTestUtil.assertKvEntry(iter.next(), 8, 4);
    Assert.assertFalse(iter.hasNext());
}
Also used : IntValueSumCombiner(com.baidu.hugegraph.computer.core.combiner.IntValueSumCombiner) CombineKvOuterSortFlusher(com.baidu.hugegraph.computer.core.sort.flusher.CombineKvOuterSortFlusher) Config(com.baidu.hugegraph.computer.core.config.Config) EntryIterator(com.baidu.hugegraph.computer.core.store.EntryIterator) PointerCombiner(com.baidu.hugegraph.computer.core.combiner.PointerCombiner) KvEntryFileReader(com.baidu.hugegraph.computer.core.store.KvEntryFileReader) RandomAccessInput(com.baidu.hugegraph.computer.core.io.RandomAccessInput) HgkvDirReaderImpl(com.baidu.hugegraph.computer.core.store.file.hgkvfile.reader.HgkvDirReaderImpl) Sorter(com.baidu.hugegraph.computer.core.sort.Sorter) IntValue(com.baidu.hugegraph.computer.core.graph.value.IntValue) Test(org.junit.Test)

Example 8 with RandomAccessInput

use of com.baidu.hugegraph.computer.core.io.RandomAccessInput in project hugegraph-computer by hugegraph.

the class SorterTest method testSortSubKvBuffers.

@Test
public void testSortSubKvBuffers() throws Exception {
    Config config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.INPUT_MAX_EDGES_IN_ONE_VERTEX, "2", ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
    int flushThreshold = config.get(ComputerOptions.INPUT_MAX_EDGES_IN_ONE_VERTEX);
    BytesInput i1 = this.sortedSubKvBuffer(config);
    BytesInput i2 = this.sortedSubKvBuffer(config);
    BytesInput i3 = this.sortedSubKvBuffer(config);
    List<RandomAccessInput> buffers = ImmutableList.of(i1, i2, i3);
    Sorter sorter = SorterTestUtil.createSorter(config);
    PointerCombiner combiner = SorterTestUtil.createPointerCombiner(IntValue::new, new IntValueSumCombiner());
    OuterSortFlusher flusher = new CombineSubKvOuterSortFlusher(combiner, flushThreshold);
    flusher.sources(buffers.size());
    String outputFile = StoreTestUtil.availablePathById("1");
    sorter.mergeBuffers(buffers, flusher, outputFile, true);
    /*
         * Assert result
         * key 1 subKv 3 3, 5 3
         * key 2 subKv 5 3, 8 6
         * key 2 subKv 9 3
         * key 3 subKv 2 6, 3 3
         * key 3 subKv 4 3
         */
    ImmutableList<String> outputs = ImmutableList.of(outputFile);
    Iterator<KvEntry> kvIter = sorter.iterator(outputs, true);
    SorterTestUtil.assertSubKvByKv(kvIter.next(), 1, 3, 3, 5, 3);
    SorterTestUtil.assertSubKvByKv(kvIter.next(), 2, 5, 3, 8, 6);
    SorterTestUtil.assertSubKvByKv(kvIter.next(), 2, 9, 3);
    SorterTestUtil.assertSubKvByKv(kvIter.next(), 3, 2, 6, 3, 3);
    SorterTestUtil.assertSubKvByKv(kvIter.next(), 3, 4, 3);
    // Assert file properties
    HgkvDir dir = HgkvDirImpl.open(outputFile);
    Assert.assertEquals(5, dir.numEntries());
    Assert.assertEquals(8, dir.numSubEntries());
}
Also used : IntValueSumCombiner(com.baidu.hugegraph.computer.core.combiner.IntValueSumCombiner) CombineSubKvOuterSortFlusher(com.baidu.hugegraph.computer.core.sort.flusher.CombineSubKvOuterSortFlusher) BytesInput(com.baidu.hugegraph.computer.core.io.BytesInput) OuterSortFlusher(com.baidu.hugegraph.computer.core.sort.flusher.OuterSortFlusher) CombineSubKvOuterSortFlusher(com.baidu.hugegraph.computer.core.sort.flusher.CombineSubKvOuterSortFlusher) CombineKvOuterSortFlusher(com.baidu.hugegraph.computer.core.sort.flusher.CombineKvOuterSortFlusher) Config(com.baidu.hugegraph.computer.core.config.Config) KvEntry(com.baidu.hugegraph.computer.core.store.entry.KvEntry) PointerCombiner(com.baidu.hugegraph.computer.core.combiner.PointerCombiner) RandomAccessInput(com.baidu.hugegraph.computer.core.io.RandomAccessInput) Sorter(com.baidu.hugegraph.computer.core.sort.Sorter) IntValue(com.baidu.hugegraph.computer.core.graph.value.IntValue) HgkvDir(com.baidu.hugegraph.computer.core.store.file.hgkvfile.HgkvDir) Test(org.junit.Test)

Example 9 with RandomAccessInput

use of com.baidu.hugegraph.computer.core.io.RandomAccessInput in project hugegraph-computer by hugegraph.

the class SortLargeDataTest method testMergeBuffersAllSameKey.

@Test
public void testMergeBuffersAllSameKey() throws Exception {
    List<RandomAccessInput> buffers = new ArrayList<>();
    for (int i = 0; i < 1000; i++) {
        BytesOutput buffer = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE);
        for (int j = 0; j < 100; j++) {
            // Write data
            SorterTestUtil.writeData(buffer, 1);
            SorterTestUtil.writeData(buffer, 1);
        }
        buffers.add(EntriesUtil.inputFromOutput(buffer));
    }
    String resultFile = StoreTestUtil.availablePathById("0");
    Sorter sorter = SorterTestUtil.createSorter(CONFIG);
    mergeBuffers(sorter, buffers, resultFile);
    // Assert result
    long result = sumOfEntryValue(sorter, ImmutableList.of(resultFile));
    Assert.assertEquals(1000 * 100, result);
}
Also used : RandomAccessInput(com.baidu.hugegraph.computer.core.io.RandomAccessInput) BytesOutput(com.baidu.hugegraph.computer.core.io.BytesOutput) ArrayList(java.util.ArrayList) Sorter(com.baidu.hugegraph.computer.core.sort.Sorter) Test(org.junit.Test)

Example 10 with RandomAccessInput

use of com.baidu.hugegraph.computer.core.io.RandomAccessInput in project hugegraph-computer by hugegraph.

the class WriteBuffersTest method wrapForRead.

@Test
public void wrapForRead() throws IOException {
    GraphFactory graphFactory = context().graphFactory();
    WriteBuffers buffers = new WriteBuffers(context(), 10, 20);
    Vertex vertex = graphFactory.createVertex(BytesId.of(1L), new DoubleValue(0.5d));
    buffers.writeVertex(vertex);
    buffers.prepareSorting();
    try (RandomAccessInput input = buffers.wrapForRead()) {
        EntryInput entryInput = new EntryInputImpl(input);
        StreamGraphInput graphInput = new StreamGraphInput(context(), entryInput);
        vertex.value(null);
        Assert.assertEquals(vertex, graphInput.readVertex());
    }
}
Also used : RandomAccessInput(com.baidu.hugegraph.computer.core.io.RandomAccessInput) Vertex(com.baidu.hugegraph.computer.core.graph.vertex.Vertex) GraphFactory(com.baidu.hugegraph.computer.core.graph.GraphFactory) EntryInputImpl(com.baidu.hugegraph.computer.core.store.entry.EntryInputImpl) DoubleValue(com.baidu.hugegraph.computer.core.graph.value.DoubleValue) EntryInput(com.baidu.hugegraph.computer.core.store.entry.EntryInput) StreamGraphInput(com.baidu.hugegraph.computer.core.io.StreamGraphInput) Test(org.junit.Test)

Aggregations

RandomAccessInput (com.baidu.hugegraph.computer.core.io.RandomAccessInput)18 Test (org.junit.Test)7 ComputerException (com.baidu.hugegraph.computer.core.common.exception.ComputerException)6 Sorter (com.baidu.hugegraph.computer.core.sort.Sorter)6 BytesOutput (com.baidu.hugegraph.computer.core.io.BytesOutput)4 KvEntry (com.baidu.hugegraph.computer.core.store.entry.KvEntry)4 BytesInput (com.baidu.hugegraph.computer.core.io.BytesInput)3 CombineKvOuterSortFlusher (com.baidu.hugegraph.computer.core.sort.flusher.CombineKvOuterSortFlusher)3 ArrayList (java.util.ArrayList)3 IntValueSumCombiner (com.baidu.hugegraph.computer.core.combiner.IntValueSumCombiner)2 PointerCombiner (com.baidu.hugegraph.computer.core.combiner.PointerCombiner)2 Config (com.baidu.hugegraph.computer.core.config.Config)2 IntValue (com.baidu.hugegraph.computer.core.graph.value.IntValue)2 KvOuterSortFlusher (com.baidu.hugegraph.computer.core.sort.flusher.KvOuterSortFlusher)2 SubKvSorter (com.baidu.hugegraph.computer.core.sort.sorter.SubKvSorter)2 InlinePointer (com.baidu.hugegraph.computer.core.store.entry.InlinePointer)2 Pointer (com.baidu.hugegraph.computer.core.store.entry.Pointer)2 File (java.io.File)2 IOException (java.io.IOException)2 Random (java.util.Random)2