Examples with CheckpointEntry org.corfudb.protocols.logprotocol.CheckpointEntry used on opensource projects

Example 1 with CheckpointEntry

use of org.corfudb.protocols.logprotocol.CheckpointEntry in project CorfuDB by CorfuDB.

the class StreamViewSMRAdapter method dataAndCheckpointMapper.

private List<SMREntry> dataAndCheckpointMapper(ILogData logData) {
    if (logData.hasCheckpointMetadata()) {
        // This is a CHECKPOINT record.  Extract the SMREntries, if any.
        CheckpointEntry cp = (CheckpointEntry) logData.getPayload(runtime);
        if (cp.getSmrEntries() != null && cp.getSmrEntries().getUpdates().size() > 0) {
            cp.getSmrEntries().getUpdates().forEach(e -> {
                e.setRuntime(runtime);
                e.setEntry(logData);
            });
            return cp.getSmrEntries().getUpdates();
        } else {
            return (List<SMREntry>) Collections.EMPTY_LIST;
        }
    } else {
        return ((ISMRConsumable) logData.getPayload(runtime)).getSMRUpdates(streamView.getID());
    }
}

Example 2 with CheckpointEntry

use of org.corfudb.protocols.logprotocol.CheckpointEntry in project CorfuDB by CorfuDB.

the class CheckpointWriter method startCheckpoint.

/** Append a checkpoint START record to this object's stream.
     *
     *  Corfu client transaction management, if desired, is the
     *  caller's responsibility.
     *
     * @return Global log address of the START record.
     */
public long startCheckpoint() {
    startTime = LocalDateTime.now();
    AbstractTransactionalContext context = TransactionalContext.getCurrentContext();
    long txBeginGlobalAddress = context.getSnapshotTimestamp();
    this.mdKV.put(CheckpointEntry.CheckpointDictKey.START_TIME, startTime.toString());
    this.mdKV.put(CheckpointEntry.CheckpointDictKey.START_LOG_ADDRESS, Long.toString(txBeginGlobalAddress));
    ImmutableMap<CheckpointEntry.CheckpointDictKey, String> mdKV = ImmutableMap.copyOf(this.mdKV);
    CheckpointEntry cp = new CheckpointEntry(CheckpointEntry.CheckpointEntryType.START, author, checkpointID, mdKV, null);
    startAddress = sv.append(Collections.singleton(streamID), cp, null);
    postAppendFunc.accept(cp, startAddress);
    return startAddress;
}

Example 3 with CheckpointEntry

use of org.corfudb.protocols.logprotocol.CheckpointEntry in project CorfuDB by CorfuDB.

the class CheckpointWriter method finishCheckpoint.

/** Append a checkpoint END record to this object's stream.
     *
     *  Corfu client transaction management, if desired, is the
     *  caller's responsibility.
     *
     * @return Global log address of the END record.
     */
public long finishCheckpoint() {
    LocalDateTime endTime = LocalDateTime.now();
    mdKV.put(CheckpointEntry.CheckpointDictKey.END_TIME, endTime.toString());
    numEntries++;
    numBytes++;
    mdKV.put(CheckpointEntry.CheckpointDictKey.ENTRY_COUNT, Long.toString(numEntries));
    mdKV.put(CheckpointEntry.CheckpointDictKey.BYTE_COUNT, Long.toString(numBytes));
    CheckpointEntry cp = new CheckpointEntry(CheckpointEntry.CheckpointEntryType.END, author, checkpointID, mdKV, null);
    endAddress = sv.append(Collections.singleton(streamID), cp, null);
    postAppendFunc.accept(cp, endAddress);
    return endAddress;
}

Example 4 with CheckpointEntry

use of org.corfudb.protocols.logprotocol.CheckpointEntry in project CorfuDB by CorfuDB.

the class CheckpointSmokeTest method writeCheckpointRecords.

private void writeCheckpointRecords(UUID streamId, String checkpointAuthor, UUID checkpointId, Object[] objects, Runnable l1, Runnable l2, boolean write1, boolean write2, boolean write3) throws Exception {
    BackpointerStreamView sv = new BackpointerStreamView(r, streamId);
    Map<CheckpointEntry.CheckpointDictKey, String> mdKV = new HashMap<>();
    mdKV.put(CheckpointEntry.CheckpointDictKey.START_TIME, "The perfect time");
    // Write cp #1 of 3
    if (write1) {
        TokenResponse tokResp1 = r.getSequencerView().nextToken(Collections.singleton(streamId), 0);
        long addr1 = tokResp1.getToken().getTokenValue();
        mdKV.put(CheckpointEntry.CheckpointDictKey.START_LOG_ADDRESS, Long.toString(addr1 + 1));
        CheckpointEntry cp1 = new CheckpointEntry(CheckpointEntry.CheckpointEntryType.START, checkpointAuthor, checkpointId, mdKV, null);
        sv.append(cp1, null, null);
    }
    // Interleaving opportunity #1
    l1.run();
    // Write cp #2 of 3
    if (write2) {
        MultiSMREntry smrEntries = new MultiSMREntry();
        if (objects != null) {
            for (int i = 0; i < objects.length; i++) {
                smrEntries.addTo(new SMREntry("put", (Object[]) objects[i], Serializers.JSON));
            }
        }
        CheckpointEntry cp2 = new CheckpointEntry(CheckpointEntry.CheckpointEntryType.CONTINUATION, checkpointAuthor, checkpointId, mdKV, smrEntries);
        sv.append(cp2, null, null);
    }
    // Interleaving opportunity #2
    l2.run();
    // Write cp #3 of 3
    if (write3) {
        CheckpointEntry cp3 = new CheckpointEntry(CheckpointEntry.CheckpointEntryType.END, checkpointAuthor, checkpointId, mdKV, null);
        sv.append(cp3, null, null);
    }
}

Example 5 with CheckpointEntry

use of org.corfudb.protocols.logprotocol.CheckpointEntry in project CorfuDB by CorfuDB.

the class CheckpointWriter method appendObjectState.

/** Append zero or more CONTINUATION records to this
     *  object's stream.  Each will contain a fraction of
     *  the state of the object that we're checkpointing
     *  (up to batchSize items at a time).
     *
     *  Corfu client transaction management, if desired, is the
     *  caller's responsibility.
     *
     *  The Iterators class appears to preserve the laziness
     *  of Stream processing; we don't wish to use more
     *  memory than strictly necessary to generate the
     *  checkpoint.  NOTE: It would be even more useful if
     *  the map had a lazy iterator: the eagerness of
     *  map.keySet().stream() is not ideal, but at least
     *  it should be much smaller than the entire map.
     *
     *  NOTE: The postAppendFunc lambda is executed in the
     *  current thread context, i.e., inside of a Corfu
     *  transaction, and that transaction will be *aborted*
     *  at the end of this function.  Any Corfu data
     *  modifying ops will be undone by the TXAbort().
     *
     * @return Stream of global log addresses of the
     * CONTINUATION records written.
     */
public List<Long> appendObjectState() {
    ImmutableMap<CheckpointEntry.CheckpointDictKey, String> mdKV = ImmutableMap.copyOf(this.mdKV);
    List<Long> continuationAddresses = new ArrayList<>();
    Iterators.partition(map.keySet().stream().map(k -> {
        return new SMREntry("put", new Object[] { keyMutator.apply(k), valueMutator.apply(map.get(k)) }, serializer);
    }).iterator(), batchSize).forEachRemaining(entries -> {
        MultiSMREntry smrEntries = new MultiSMREntry();
        for (int i = 0; i < ((List) entries).size(); i++) {
            smrEntries.addTo((SMREntry) ((List) entries).get(i));
        }
        CheckpointEntry cp = new CheckpointEntry(CheckpointEntry.CheckpointEntryType.CONTINUATION, author, checkpointID, mdKV, smrEntries);
        long pos = sv.append(Collections.singleton(streamID), cp, null);
        postAppendFunc.accept(cp, pos);
        continuationAddresses.add(pos);
        numEntries++;
        numBytes += cp.getSmrEntriesBytes();
    });
    return continuationAddresses;
}