Examples with KvEntry com.baidu.hugegraph.computer.core.store.entry.KvEntry used on opensource projects

Example 6 with KvEntry

use of com.baidu.hugegraph.computer.core.store.entry.KvEntry in project hugegraph-computer by hugegraph.

the class CombinableSorterFlusher method flush.

public void flush(Iterator<KvEntry> entries) throws IOException {
    E.checkArgument(entries.hasNext(), "Parameter entries can't be empty");
    KvEntry last = entries.next();
    Pointer combineValue = last.value();
    while (true) {
        KvEntry current = null;
        if (entries.hasNext()) {
            current = entries.next();
            if (last.compareTo(current) == 0) {
                combineValue = this.combiner.combine(combineValue, current.value());
                continue;
            }
        }
        this.writeKvEntry(new DefaultKvEntry(last.key(), combineValue));
        if (current == null) {
            break;
        }
        last = current;
        combineValue = last.value();
    }
}

Example 7 with KvEntry

use of com.baidu.hugegraph.computer.core.store.entry.KvEntry in project hugegraph-computer by hugegraph.

the class CombineSubKvInnerSortFlusher method flush.

@Override
public void flush(Iterator<KvEntry> entries) throws IOException {
    E.checkArgument(entries.hasNext(), "Parameter entries can't be empty");
    KvEntry last = entries.next();
    // TODO: use byte buffer store all value pointer to avoid big collection
    List<KvEntry> sameKeyEntries = new ArrayList<>();
    sameKeyEntries.add(last);
    while (true) {
        KvEntry current = null;
        if (entries.hasNext()) {
            current = entries.next();
            if (last.compareTo(current) == 0) {
                sameKeyEntries.add(current);
                continue;
            }
        }
        this.writeSubKvs(last, this.sortedSubKvFromEntries(sameKeyEntries));
        if (current == null) {
            break;
        }
        sameKeyEntries.clear();
        sameKeyEntries.add(current);
        last = current;
    }
}

Example 8 with KvEntry

use of com.baidu.hugegraph.computer.core.store.entry.KvEntry in project hugegraph-computer by hugegraph.

the class CombineSubKvInnerSortFlusher method writeSubKvs.

private void writeSubKvs(KvEntry kvEntry, Iterator<KvEntry> subKvIter) throws IOException {
    E.checkArgument(subKvIter.hasNext(), "Parameter subKvs can't be empty");
    kvEntry.key().write(this.output);
    long position = this.output.position();
    // Write value length placeholder
    this.output.writeFixedInt(0);
    // Write subKv count placeholder
    this.output.writeFixedInt(0);
    // Write subKv to output
    KvEntry lastSubKv = subKvIter.next();
    Pointer lastSubValue = lastSubKv.value();
    int writtenCount = 0;
    while (true) {
        // Write subKv
        KvEntry current = null;
        if (subKvIter.hasNext()) {
            current = subKvIter.next();
            if (lastSubKv.compareTo(current) == 0) {
                lastSubValue = this.combiner.combine(lastSubValue, current.value());
                continue;
            }
        }
        lastSubKv.key().write(this.output);
        lastSubValue.write(this.output);
        writtenCount++;
        if (writtenCount == this.subKvFlushThreshold || current == null) {
            // Fill placeholder
            long currentPosition = this.output.position();
            this.output.seek(position);
            // Fill value length placeholder
            this.output.writeFixedInt((int) (currentPosition - position - 4));
            // Fill subKv count placeholder
            this.output.writeFixedInt(writtenCount);
            this.output.seek(currentPosition);
            if (current == null) {
                break;
            }
            // Used for next loop
            kvEntry.key().write(this.output);
            position = this.output.position();
            // Write value length placeholder
            this.output.writeFixedInt(0);
            // Write subKv count placeholder
            this.output.writeFixedInt(0);
            writtenCount = 0;
        }
        lastSubKv = current;
        lastSubValue = current.value();
    }
}

Example 9 with KvEntry

use of com.baidu.hugegraph.computer.core.store.entry.KvEntry in project hugegraph-computer by hugegraph.

the class KvInnerSortFlusher method flush.

@Override
public void flush(Iterator<KvEntry> entries) throws IOException {
    E.checkArgument(entries.hasNext(), "Parameter entries can't be empty");
    while (entries.hasNext()) {
        KvEntry entry = entries.next();
        entry.key().write(this.output);
        entry.value().write(this.output);
    }
}

Example 10 with KvEntry

use of com.baidu.hugegraph.computer.core.store.entry.KvEntry in project hugegraph-computer by hugegraph.

the class SortLargeDataTest method sumOfEntryValue.

private static long sumOfEntryValue(Sorter sorter, List<String> files) throws Exception {
    long entrySize = 0L;
    for (String file : files) {
        HgkvDir dir = HgkvDirImpl.open(file);
        entrySize += dir.numEntries();
    }
    LOG.info("Finally kvEntry size: {}", entrySize);
    try (PeekableIterator<KvEntry> iterator = sorter.iterator(files, false)) {
        long result = 0;
        while (iterator.hasNext()) {
            KvEntry next = iterator.next();
            result += StoreTestUtil.dataFromPointer(next.value());
        }
        return result;
    }
}