Examples with LogFileValue org.apache.accumulo.tserver.logger.LogFileValue used on opensource projects

Example 1 with LogFileValue

use of org.apache.accumulo.tserver.logger.LogFileValue in project accumulo by apache.

the class AccumuloReplicaSystem method getWalEdits.

protected WalReplication getWalEdits(ReplicationTarget target, DataInputStream wal, Path p, Status status, long sizeLimit, Set<Integer> desiredTids) throws IOException {
    WalEdits edits = new WalEdits();
    edits.edits = new ArrayList<>();
    long size = 0l;
    long entriesConsumed = 0l;
    long numUpdates = 0l;
    LogFileKey key = new LogFileKey();
    LogFileValue value = new LogFileValue();
    while (size < sizeLimit) {
        try {
            key.readFields(wal);
            value.readFields(wal);
        } catch (EOFException e) {
            log.debug("Caught EOFException reading {}", p);
            if (status.getInfiniteEnd() && status.getClosed()) {
                log.debug("{} is closed and has unknown length, assuming entire file has been consumed", p);
                entriesConsumed = Long.MAX_VALUE;
            }
            break;
        }
        entriesConsumed++;
        switch(key.event) {
            case DEFINE_TABLET:
                // For new DEFINE_TABLETs, we also need to record the new tids we see
                if (target.getSourceTableId().equals(key.tablet.getTableId())) {
                    desiredTids.add(key.tid);
                }
                break;
            case MUTATION:
            case MANY_MUTATIONS:
                // Only write out mutations for tids that are for the desired tablet
                if (desiredTids.contains(key.tid)) {
                    ByteArrayOutputStream baos = new ByteArrayOutputStream();
                    DataOutputStream out = new DataOutputStream(baos);
                    key.write(out);
                    // Only write out the mutations that don't have the given ReplicationTarget
                    // as a replicate source (this prevents infinite replication loops: a->b, b->a, repeat)
                    numUpdates += writeValueAvoidingReplicationCycles(out, value, target);
                    out.flush();
                    byte[] data = baos.toByteArray();
                    size += data.length;
                    edits.addToEdits(ByteBuffer.wrap(data));
                }
                break;
            default:
                log.trace("Ignorning WAL entry which doesn't contain mutations, should not have received such entries");
                break;
        }
    }
    return new WalReplication(edits, size, entriesConsumed, numUpdates);
}

Example 2 with LogFileValue

use of org.apache.accumulo.tserver.logger.LogFileValue in project accumulo by apache.

the class AccumuloReplicaSystem method consumeWalPrefix.

protected Set<Integer> consumeWalPrefix(ReplicationTarget target, DataInputStream wal, Path p, Status status, long sizeLimit) throws IOException {
    Set<Integer> tids = new HashSet<>();
    LogFileKey key = new LogFileKey();
    LogFileValue value = new LogFileValue();
    Set<Integer> desiredTids = new HashSet<>();
    // later on might use that tid
    for (long i = 0; i < status.getBegin(); i++) {
        key.readFields(wal);
        value.readFields(wal);
        switch(key.event) {
            case DEFINE_TABLET:
                if (target.getSourceTableId().equals(key.tablet.getTableId())) {
                    desiredTids.add(key.tid);
                }
                break;
            default:
                break;
        }
    }
    return tids;
}

Example 3 with LogFileValue

use of org.apache.accumulo.tserver.logger.LogFileValue in project accumulo by apache.

the class SortedLogRecovery method findLastStartToFinish.

int findLastStartToFinish(MultiReader reader, int fileno, KeyExtent extent, Set<String> tabletFiles, LastStartToFinish lastStartToFinish) throws IOException, EmptyMapFileException, UnusedException {
    HashSet<String> suffixes = new HashSet<>();
    for (String path : tabletFiles) suffixes.add(getPathSuffix(path));
    // Scan for tableId for this extent (should always be in the log)
    LogFileKey key = new LogFileKey();
    LogFileValue value = new LogFileValue();
    int tid = -1;
    if (!reader.next(key, value))
        throw new EmptyMapFileException();
    if (key.event != OPEN)
        throw new RuntimeException("First log entry value is not OPEN");
    if (key.tserverSession.compareTo(lastStartToFinish.tserverSession) != 0) {
        if (lastStartToFinish.compactionStatus == Status.LOOKING_FOR_FINISH)
            throw new RuntimeException("COMPACTION_FINISH (without preceding COMPACTION_START) is not followed by a successful minor compaction.");
        lastStartToFinish.update(key.tserverSession);
    }
    KeyExtent alternative = extent;
    if (extent.isRootTablet()) {
        alternative = RootTable.OLD_EXTENT;
    }
    LogFileKey defineKey = null;
    // for the maximum tablet id, find the minimum sequence #... may be ok to find the max seq, but just want to make the code behave like it used to
    while (reader.next(key, value)) {
        if (key.event != DEFINE_TABLET)
            break;
        if (key.tablet.equals(extent) || key.tablet.equals(alternative)) {
            if (tid != key.tid) {
                tid = key.tid;
                defineKey = key;
                key = new LogFileKey();
            }
        }
    }
    if (tid < 0) {
        throw new UnusedException();
    }
    log.debug("Found tid, seq {} {}", tid, defineKey.seq);
    // Scan start/stop events for this tablet
    key = defineKey;
    key.event = COMPACTION_START;
    reader.seek(key);
    while (reader.next(key, value)) {
        // LogFileEntry.printEntry(entry);
        if (key.tid != tid)
            break;
        if (key.event == COMPACTION_START) {
            if (lastStartToFinish.compactionStatus == Status.INITIAL)
                lastStartToFinish.compactionStatus = Status.COMPLETE;
            if (key.seq <= lastStartToFinish.lastStart)
                throw new RuntimeException("Sequence numbers are not increasing for start/stop events: " + key.seq + " vs " + lastStartToFinish.lastStart);
            lastStartToFinish.update(fileno, key.seq);
            // Tablet server finished the minor compaction, but didn't remove the entry from the METADATA table.
            log.debug("minor compaction into {} finished, but was still in the METADATA", key.filename);
            if (suffixes.contains(getPathSuffix(key.filename)))
                lastStartToFinish.update(-1);
        } else if (key.event == COMPACTION_FINISH) {
            if (key.seq <= lastStartToFinish.lastStart)
                throw new RuntimeException("Sequence numbers are not increasing for start/stop events: " + key.seq + " vs " + lastStartToFinish.lastStart);
            if (lastStartToFinish.compactionStatus == Status.INITIAL)
                lastStartToFinish.compactionStatus = Status.LOOKING_FOR_FINISH;
            else if (lastStartToFinish.lastFinish > lastStartToFinish.lastStart)
                throw new RuntimeException("COMPACTION_FINISH does not have preceding COMPACTION_START event.");
            else
                lastStartToFinish.compactionStatus = Status.COMPLETE;
            lastStartToFinish.update(key.seq);
        } else
            break;
    }
    return tid;
}

Example 4 with LogFileValue

use of org.apache.accumulo.tserver.logger.LogFileValue in project accumulo by apache.

the class AccumuloReplicaSystemTest method mutationsNotReReplicatedToPeers.

@Test
public void mutationsNotReReplicatedToPeers() throws Exception {
    AccumuloReplicaSystem ars = new AccumuloReplicaSystem();
    Map<String, String> confMap = new HashMap<>();
    confMap.put(Property.REPLICATION_NAME.getKey(), "source");
    AccumuloConfiguration conf = new ConfigurationCopy(confMap);
    ars.setConf(conf);
    LogFileValue value = new LogFileValue();
    value.mutations = new ArrayList<>();
    Mutation m = new Mutation("row");
    m.put("", "", new Value(new byte[0]));
    value.mutations.add(m);
    m = new Mutation("row2");
    m.put("", "", new Value(new byte[0]));
    m.addReplicationSource("peer");
    value.mutations.add(m);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    DataOutputStream out = new DataOutputStream(baos);
    // Replicate our 2 mutations to "peer", from tableid 1 to tableid 1
    ars.writeValueAvoidingReplicationCycles(out, value, new ReplicationTarget("peer", "1", Table.ID.of("1")));
    out.close();
    ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
    DataInputStream in = new DataInputStream(bais);
    int numMutations = in.readInt();
    Assert.assertEquals(1, numMutations);
    m = new Mutation();
    m.readFields(in);
    Assert.assertEquals("row", new String(m.getRow()));
    Assert.assertEquals(1, m.getReplicationSources().size());
    Assert.assertTrue("Expected source cluster to be listed in mutation replication source", m.getReplicationSources().contains("source"));
}

Example 5 with LogFileValue

use of org.apache.accumulo.tserver.logger.LogFileValue in project accumulo by apache.

the class AccumuloReplicaSystemTest method restartInFileKnowsAboutPreviousTableDefines.

@Test
public void restartInFileKnowsAboutPreviousTableDefines() throws Exception {
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    DataOutputStream dos = new DataOutputStream(baos);
    LogFileKey key = new LogFileKey();
    LogFileValue value = new LogFileValue();
    // What is seq used for?
    key.seq = 1l;
    /*
     * Disclaimer: the following series of LogFileKey and LogFileValue pairs have *no* bearing whatsoever in reality regarding what these entries would actually
     * look like in a WAL. They are solely for testing that each LogEvents is handled, order is not important.
     */
    key.event = LogEvents.DEFINE_TABLET;
    key.tablet = new KeyExtent(Table.ID.of("1"), null, null);
    key.tid = 1;
    key.write(dos);
    value.write(dos);
    key.tablet = null;
    key.event = LogEvents.MUTATION;
    key.filename = "/accumulo/wals/tserver+port/" + UUID.randomUUID();
    value.mutations = Arrays.asList(new ServerMutation(new Text("row")));
    key.write(dos);
    value.write(dos);
    key.tablet = null;
    key.event = LogEvents.MUTATION;
    key.tid = 1;
    key.filename = "/accumulo/wals/tserver+port/" + UUID.randomUUID();
    value.mutations = Arrays.asList(new ServerMutation(new Text("row")));
    key.write(dos);
    value.write(dos);
    dos.close();
    Map<String, String> confMap = new HashMap<>();
    confMap.put(Property.REPLICATION_NAME.getKey(), "source");
    AccumuloConfiguration conf = new ConfigurationCopy(confMap);
    AccumuloReplicaSystem ars = new AccumuloReplicaSystem();
    ars.setConf(conf);
    Status status = Status.newBuilder().setBegin(0).setEnd(0).setInfiniteEnd(true).setClosed(false).build();
    DataInputStream dis = new DataInputStream(new ByteArrayInputStream(baos.toByteArray()));
    HashSet<Integer> tids = new HashSet<>();
    // Only consume the first mutation, not the second
    WalReplication repl = ars.getWalEdits(new ReplicationTarget("peer", "1", Table.ID.of("1")), dis, new Path("/accumulo/wals/tserver+port/wal"), status, 1l, tids);
    // We stopped because we got to the end of the file
    Assert.assertEquals(2, repl.entriesConsumed);
    Assert.assertEquals(1, repl.walEdits.getEditsSize());
    Assert.assertEquals(1, repl.sizeInRecords);
    Assert.assertNotEquals(0, repl.sizeInBytes);
    status = Status.newBuilder(status).setBegin(2).build();
    // Consume the rest of the mutations
    repl = ars.getWalEdits(new ReplicationTarget("peer", "1", Table.ID.of("1")), dis, new Path("/accumulo/wals/tserver+port/wal"), status, 1l, tids);
    // We stopped because we got to the end of the file
    Assert.assertEquals(1, repl.entriesConsumed);
    Assert.assertEquals(1, repl.walEdits.getEditsSize());
    Assert.assertEquals(1, repl.sizeInRecords);
    Assert.assertNotEquals(0, repl.sizeInBytes);
}