use of com.baidu.hugegraph.computer.core.io.RandomAccessInput in project hugegraph-computer by hugegraph.
the class SortLargeDataTest method testAllProcess.
@Test
public void testAllProcess() throws Exception {
StopWatch watcher = new StopWatch();
final long bufferSize = Bytes.MB;
final int mergeBufferNum = 300;
final int dataSize = 1000000;
long value = 0;
Random random = new Random();
BytesOutput output = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE);
List<RandomAccessInput> buffers = new ArrayList<>(mergeBufferNum);
List<String> mergeBufferFiles = new ArrayList<>();
int fileNum = 10;
Sorter sorter = SorterTestUtil.createSorter(CONFIG);
watcher.start();
for (int i = 0; i < dataSize; i++) {
SorterTestUtil.writeData(output, random.nextInt(dataSize));
int entryValue = random.nextInt(5);
SorterTestUtil.writeData(output, entryValue);
value = value + entryValue;
// Write data to buffer and sort buffer
if (output.position() >= bufferSize || (i + 1) == dataSize) {
BytesInput input = EntriesUtil.inputFromOutput(output);
buffers.add(sortBuffer(sorter, input));
output.seek(0);
}
// Merge buffers to HgkvDir
if (buffers.size() >= mergeBufferNum || (i + 1) == dataSize) {
String outputFile = StoreTestUtil.availablePathById(fileNum++);
mergeBufferFiles.add(outputFile);
mergeBuffers(sorter, buffers, outputFile);
buffers.clear();
}
}
// Merge file
String resultFile = StoreTestUtil.availablePathById("0");
mergeFiles(sorter, mergeBufferFiles, Lists.newArrayList(resultFile));
watcher.stop();
LOG.info("testAllProcess sort time: {}", watcher.getTime());
long result = sumOfEntryValue(sorter, ImmutableList.of(resultFile));
Assert.assertEquals(value, result);
}
use of com.baidu.hugegraph.computer.core.io.RandomAccessInput in project hugegraph-computer by hugegraph.
the class SorterTest method testSortKvBuffers.
@Test
public void testSortKvBuffers() throws Exception {
Config config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.HGKV_MAX_FILE_SIZE, "32", ComputerOptions.HGKV_DATABLOCK_SIZE, "16", ComputerOptions.HGKV_MERGE_FILES_NUM, "3", ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
List<Integer> map1 = ImmutableList.of(2, 3, 2, 1, 5, 2, 6, 9, 6, 2);
List<Integer> map2 = ImmutableList.of(1, 3, 1, 1, 3, 2, 6, 9, 8, 2);
String path = StoreTestUtil.availablePathById("1");
// Merge 4 sorted input
List<RandomAccessInput> inputs = ImmutableList.of(SorterTestUtil.inputFromKvMap(map1), SorterTestUtil.inputFromKvMap(map2), SorterTestUtil.inputFromKvMap(map1), SorterTestUtil.inputFromKvMap(map2));
Sorter sorter = SorterTestUtil.createSorter(config);
PointerCombiner combiner = SorterTestUtil.createPointerCombiner(IntValue::new, new IntValueSumCombiner());
sorter.mergeBuffers(inputs, new CombineKvOuterSortFlusher(combiner), path, false);
// Assert merge result from target hgkvDir
KvEntryFileReader reader = new HgkvDirReaderImpl(path, false);
EntryIterator iter = reader.iterator();
SorterTestUtil.assertKvEntry(iter.next(), 1, 8);
SorterTestUtil.assertKvEntry(iter.next(), 2, 8);
SorterTestUtil.assertKvEntry(iter.next(), 3, 4);
SorterTestUtil.assertKvEntry(iter.next(), 5, 4);
SorterTestUtil.assertKvEntry(iter.next(), 6, 40);
SorterTestUtil.assertKvEntry(iter.next(), 8, 4);
Assert.assertFalse(iter.hasNext());
}
use of com.baidu.hugegraph.computer.core.io.RandomAccessInput in project hugegraph-computer by hugegraph.
the class SorterTest method testSortSubKvBuffers.
@Test
public void testSortSubKvBuffers() throws Exception {
Config config = UnitTestBase.updateWithRequiredOptions(ComputerOptions.INPUT_MAX_EDGES_IN_ONE_VERTEX, "2", ComputerOptions.TRANSPORT_RECV_FILE_MODE, "false");
int flushThreshold = config.get(ComputerOptions.INPUT_MAX_EDGES_IN_ONE_VERTEX);
BytesInput i1 = this.sortedSubKvBuffer(config);
BytesInput i2 = this.sortedSubKvBuffer(config);
BytesInput i3 = this.sortedSubKvBuffer(config);
List<RandomAccessInput> buffers = ImmutableList.of(i1, i2, i3);
Sorter sorter = SorterTestUtil.createSorter(config);
PointerCombiner combiner = SorterTestUtil.createPointerCombiner(IntValue::new, new IntValueSumCombiner());
OuterSortFlusher flusher = new CombineSubKvOuterSortFlusher(combiner, flushThreshold);
flusher.sources(buffers.size());
String outputFile = StoreTestUtil.availablePathById("1");
sorter.mergeBuffers(buffers, flusher, outputFile, true);
/*
* Assert result
* key 1 subKv 3 3, 5 3
* key 2 subKv 5 3, 8 6
* key 2 subKv 9 3
* key 3 subKv 2 6, 3 3
* key 3 subKv 4 3
*/
ImmutableList<String> outputs = ImmutableList.of(outputFile);
Iterator<KvEntry> kvIter = sorter.iterator(outputs, true);
SorterTestUtil.assertSubKvByKv(kvIter.next(), 1, 3, 3, 5, 3);
SorterTestUtil.assertSubKvByKv(kvIter.next(), 2, 5, 3, 8, 6);
SorterTestUtil.assertSubKvByKv(kvIter.next(), 2, 9, 3);
SorterTestUtil.assertSubKvByKv(kvIter.next(), 3, 2, 6, 3, 3);
SorterTestUtil.assertSubKvByKv(kvIter.next(), 3, 4, 3);
// Assert file properties
HgkvDir dir = HgkvDirImpl.open(outputFile);
Assert.assertEquals(5, dir.numEntries());
Assert.assertEquals(8, dir.numSubEntries());
}
use of com.baidu.hugegraph.computer.core.io.RandomAccessInput in project hugegraph-computer by hugegraph.
the class SortLargeDataTest method testMergeBuffersAllSameKey.
@Test
public void testMergeBuffersAllSameKey() throws Exception {
List<RandomAccessInput> buffers = new ArrayList<>();
for (int i = 0; i < 1000; i++) {
BytesOutput buffer = IOFactory.createBytesOutput(Constants.SMALL_BUF_SIZE);
for (int j = 0; j < 100; j++) {
// Write data
SorterTestUtil.writeData(buffer, 1);
SorterTestUtil.writeData(buffer, 1);
}
buffers.add(EntriesUtil.inputFromOutput(buffer));
}
String resultFile = StoreTestUtil.availablePathById("0");
Sorter sorter = SorterTestUtil.createSorter(CONFIG);
mergeBuffers(sorter, buffers, resultFile);
// Assert result
long result = sumOfEntryValue(sorter, ImmutableList.of(resultFile));
Assert.assertEquals(1000 * 100, result);
}
use of com.baidu.hugegraph.computer.core.io.RandomAccessInput in project hugegraph-computer by hugegraph.
the class WriteBuffersTest method wrapForRead.
@Test
public void wrapForRead() throws IOException {
GraphFactory graphFactory = context().graphFactory();
WriteBuffers buffers = new WriteBuffers(context(), 10, 20);
Vertex vertex = graphFactory.createVertex(BytesId.of(1L), new DoubleValue(0.5d));
buffers.writeVertex(vertex);
buffers.prepareSorting();
try (RandomAccessInput input = buffers.wrapForRead()) {
EntryInput entryInput = new EntryInputImpl(input);
StreamGraphInput graphInput = new StreamGraphInput(context(), entryInput);
vertex.value(null);
Assert.assertEquals(vertex, graphInput.readVertex());
}
}
Aggregations