Examples with LogFileKey org.apache.accumulo.tserver.logger.LogFileKey used on opensource projects

Example 11 with LogFileKey

use of org.apache.accumulo.tserver.logger.LogFileKey in project accumulo by apache.

the class DfsLogger method logManyTablets.

public LoggerOperation logManyTablets(List<TabletMutations> mutations) throws IOException {
    Durability durability = Durability.NONE;
    List<Pair<LogFileKey, LogFileValue>> data = new ArrayList<>();
    for (TabletMutations tabletMutations : mutations) {
        LogFileKey key = new LogFileKey();
        key.event = MANY_MUTATIONS;
        key.seq = tabletMutations.getSeq();
        key.tid = tabletMutations.getTid();
        LogFileValue value = new LogFileValue();
        value.mutations = tabletMutations.getMutations();
        data.add(new Pair<>(key, value));
        if (tabletMutations.getDurability().ordinal() > durability.ordinal()) {
            durability = tabletMutations.getDurability();
        }
    }
    return logFileData(data, chooseDurabilityForGroupCommit(mutations));
}

Example 12 with LogFileKey

use of org.apache.accumulo.tserver.logger.LogFileKey in project accumulo by apache.

the class AccumuloReplicaSystemTest method onlyChooseMutationsForDesiredTableWithClosedStatus.

@Test
public void onlyChooseMutationsForDesiredTableWithClosedStatus() throws Exception {
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    DataOutputStream dos = new DataOutputStream(baos);
    LogFileKey key = new LogFileKey();
    LogFileValue value = new LogFileValue();
    // What is seq used for?
    key.seq = 1l;
    /*
     * Disclaimer: the following series of LogFileKey and LogFileValue pairs have *no* bearing whatsoever in reality regarding what these entries would actually
     * look like in a WAL. They are solely for testing that each LogEvents is handled, order is not important.
     */
    key.event = LogEvents.DEFINE_TABLET;
    key.tablet = new KeyExtent(Table.ID.of("1"), null, null);
    key.tid = 1;
    key.write(dos);
    value.write(dos);
    key.tablet = null;
    key.event = LogEvents.MUTATION;
    key.filename = "/accumulo/wals/tserver+port/" + UUID.randomUUID();
    value.mutations = Arrays.asList(new ServerMutation(new Text("row")));
    key.write(dos);
    value.write(dos);
    key.event = LogEvents.DEFINE_TABLET;
    key.tablet = new KeyExtent(Table.ID.of("2"), null, null);
    key.tid = 2;
    value.mutations = Collections.emptyList();
    key.write(dos);
    value.write(dos);
    key.event = LogEvents.OPEN;
    key.tid = LogFileKey.VERSION;
    key.tserverSession = "foobar";
    key.write(dos);
    value.write(dos);
    key.tablet = null;
    key.event = LogEvents.MUTATION;
    key.filename = "/accumulo/wals/tserver+port/" + UUID.randomUUID();
    value.mutations = Arrays.asList(new ServerMutation(new Text("badrow")));
    key.write(dos);
    value.write(dos);
    key.event = LogEvents.COMPACTION_START;
    key.tid = 2;
    key.filename = "/accumulo/tables/1/t-000001/A000001.rf";
    value.mutations = Collections.emptyList();
    key.write(dos);
    value.write(dos);
    key.event = LogEvents.DEFINE_TABLET;
    key.tablet = new KeyExtent(Table.ID.of("1"), null, null);
    key.tid = 3;
    value.mutations = Collections.emptyList();
    key.write(dos);
    value.write(dos);
    key.event = LogEvents.COMPACTION_FINISH;
    key.tid = 6;
    value.mutations = Collections.emptyList();
    key.write(dos);
    value.write(dos);
    key.tablet = null;
    key.event = LogEvents.MUTATION;
    key.tid = 3;
    key.filename = "/accumulo/wals/tserver+port/" + UUID.randomUUID();
    value.mutations = Arrays.asList(new ServerMutation(new Text("row")));
    key.write(dos);
    value.write(dos);
    dos.close();
    Map<String, String> confMap = new HashMap<>();
    confMap.put(Property.REPLICATION_NAME.getKey(), "source");
    AccumuloConfiguration conf = new ConfigurationCopy(confMap);
    AccumuloReplicaSystem ars = new AccumuloReplicaSystem();
    ars.setConf(conf);
    // Setting the file to be closed with the infinite end implies that we need to bump the begin up to Long.MAX_VALUE
    // If it were still open, more data could be appended that we need to process
    Status status = Status.newBuilder().setBegin(0).setEnd(0).setInfiniteEnd(true).setClosed(true).build();
    DataInputStream dis = new DataInputStream(new ByteArrayInputStream(baos.toByteArray()));
    WalReplication repl = ars.getWalEdits(new ReplicationTarget("peer", "1", Table.ID.of("1")), dis, new Path("/accumulo/wals/tserver+port/wal"), status, Long.MAX_VALUE, new HashSet<>());
    // We stopped because we got to the end of the file
    Assert.assertEquals(Long.MAX_VALUE, repl.entriesConsumed);
    Assert.assertEquals(2, repl.walEdits.getEditsSize());
    Assert.assertEquals(2, repl.sizeInRecords);
    Assert.assertNotEquals(0, repl.sizeInBytes);
}

Example 13 with LogFileKey

use of org.apache.accumulo.tserver.logger.LogFileKey in project accumulo by apache.

the class AccumuloReplicaSystemTest method onlyChooseMutationsForDesiredTableWithOpenStatus.

@Test
public void onlyChooseMutationsForDesiredTableWithOpenStatus() throws Exception {
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    DataOutputStream dos = new DataOutputStream(baos);
    LogFileKey key = new LogFileKey();
    LogFileValue value = new LogFileValue();
    // What is seq used for?
    key.seq = 1l;
    /*
     * Disclaimer: the following series of LogFileKey and LogFileValue pairs have *no* bearing whatsoever in reality regarding what these entries would actually
     * look like in a WAL. They are solely for testing that each LogEvents is handled, order is not important.
     */
    key.event = LogEvents.DEFINE_TABLET;
    key.tablet = new KeyExtent(Table.ID.of("1"), null, null);
    key.tid = 1;
    key.write(dos);
    value.write(dos);
    key.tablet = null;
    key.event = LogEvents.MUTATION;
    key.filename = "/accumulo/wals/tserver+port/" + UUID.randomUUID();
    value.mutations = Arrays.asList(new ServerMutation(new Text("row")));
    key.write(dos);
    value.write(dos);
    key.event = LogEvents.DEFINE_TABLET;
    key.tablet = new KeyExtent(Table.ID.of("2"), null, null);
    key.tid = 2;
    value.mutations = Collections.emptyList();
    key.write(dos);
    value.write(dos);
    key.event = LogEvents.OPEN;
    key.tid = LogFileKey.VERSION;
    key.tserverSession = "foobar";
    key.write(dos);
    value.write(dos);
    key.tablet = null;
    key.event = LogEvents.MUTATION;
    key.filename = "/accumulo/wals/tserver+port/" + UUID.randomUUID();
    value.mutations = Arrays.asList(new ServerMutation(new Text("badrow")));
    key.write(dos);
    value.write(dos);
    key.event = LogEvents.COMPACTION_START;
    key.tid = 2;
    key.filename = "/accumulo/tables/1/t-000001/A000001.rf";
    value.mutations = Collections.emptyList();
    key.write(dos);
    value.write(dos);
    key.event = LogEvents.DEFINE_TABLET;
    key.tablet = new KeyExtent(Table.ID.of("1"), null, null);
    key.tid = 3;
    value.mutations = Collections.emptyList();
    key.write(dos);
    value.write(dos);
    key.event = LogEvents.COMPACTION_FINISH;
    key.tid = 6;
    value.mutations = Collections.emptyList();
    key.write(dos);
    value.write(dos);
    key.tablet = null;
    key.event = LogEvents.MUTATION;
    key.tid = 3;
    key.filename = "/accumulo/wals/tserver+port/" + UUID.randomUUID();
    value.mutations = Arrays.asList(new ServerMutation(new Text("row")));
    key.write(dos);
    value.write(dos);
    dos.close();
    Map<String, String> confMap = new HashMap<>();
    confMap.put(Property.REPLICATION_NAME.getKey(), "source");
    AccumuloConfiguration conf = new ConfigurationCopy(confMap);
    AccumuloReplicaSystem ars = new AccumuloReplicaSystem();
    ars.setConf(conf);
    Status status = Status.newBuilder().setBegin(0).setEnd(0).setInfiniteEnd(true).setClosed(false).build();
    DataInputStream dis = new DataInputStream(new ByteArrayInputStream(baos.toByteArray()));
    WalReplication repl = ars.getWalEdits(new ReplicationTarget("peer", "1", Table.ID.of("1")), dis, new Path("/accumulo/wals/tserver+port/wal"), status, Long.MAX_VALUE, new HashSet<>());
    // We stopped because we got to the end of the file
    Assert.assertEquals(9, repl.entriesConsumed);
    Assert.assertEquals(2, repl.walEdits.getEditsSize());
    Assert.assertEquals(2, repl.sizeInRecords);
    Assert.assertNotEquals(0, repl.sizeInBytes);
}

Example 14 with LogFileKey

use of org.apache.accumulo.tserver.logger.LogFileKey in project accumulo by apache.

the class BatchWriterReplicationReplayerTest method systemTimestampsAreSetOnUpdates.

@Test
public void systemTimestampsAreSetOnUpdates() throws Exception {
    final BatchWriterReplicationReplayer replayer = new BatchWriterReplicationReplayer();
    final String tableName = "foo";
    final long systemTimestamp = 1000;
    final BatchWriterConfig bwCfg = new BatchWriterConfig();
    bwCfg.setMaxMemory(1l);
    LogFileKey key = new LogFileKey();
    key.event = LogEvents.MANY_MUTATIONS;
    key.seq = 1;
    key.tid = 1;
    WalEdits edits = new WalEdits();
    // Make a mutation without timestamps
    Mutation m = new Mutation("row");
    m.put("cf", "cq1", "value");
    m.put("cf", "cq2", "value");
    m.put("cf", "cq3", "value");
    m.put("cf", "cq4", "value");
    m.put("cf", "cq5", "value");
    // Make it a TMutation
    TMutation tMutation = m.toThrift();
    // And then make a ServerMutation from the TMutation, adding in our systemTimestamp
    ServerMutation sMutation = new ServerMutation(tMutation);
    sMutation.setSystemTimestamp(systemTimestamp);
    // Serialize the ServerMutation (what AccumuloReplicaSystem will be doing)
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    DataOutputStream out = new DataOutputStream(baos);
    key.write(out);
    out.writeInt(1);
    sMutation.write(out);
    out.close();
    // Add it to our "input" to BatchWriterReplicationReplayer
    edits.addToEdits(ByteBuffer.wrap(baos.toByteArray()));
    Mutation expectedMutation = new Mutation("row");
    expectedMutation.put("cf", "cq1", sMutation.getSystemTimestamp(), "value");
    expectedMutation.put("cf", "cq2", sMutation.getSystemTimestamp(), "value");
    expectedMutation.put("cf", "cq3", sMutation.getSystemTimestamp(), "value");
    expectedMutation.put("cf", "cq4", sMutation.getSystemTimestamp(), "value");
    expectedMutation.put("cf", "cq5", sMutation.getSystemTimestamp(), "value");
    expect(conf.getAsBytes(Property.TSERV_REPLICATION_BW_REPLAYER_MEMORY)).andReturn(bwCfg.getMaxMemory());
    expect(conn.createBatchWriter(tableName, bwCfg)).andReturn(bw);
    bw.addMutations(Lists.newArrayList(expectedMutation));
    expectLastCall().once();
    bw.close();
    expectLastCall().once();
    replay(conn, conf, bw);
    replayer.replicateLog(context, tableName, edits);
}

Example 15 with LogFileKey

use of org.apache.accumulo.tserver.logger.LogFileKey in project accumulo by apache.

the class BatchWriterReplicationReplayerTest method replicationSourcesArePreserved.

@Test
public void replicationSourcesArePreserved() throws Exception {
    final BatchWriterReplicationReplayer replayer = new BatchWriterReplicationReplayer();
    final String tableName = "foo";
    final long systemTimestamp = 1000;
    final String peerName = "peer";
    final BatchWriterConfig bwCfg = new BatchWriterConfig();
    bwCfg.setMaxMemory(1l);
    LogFileKey key = new LogFileKey();
    key.event = LogEvents.MANY_MUTATIONS;
    key.seq = 1;
    key.tid = 1;
    WalEdits edits = new WalEdits();
    // Make a mutation without timestamps
    Mutation m = new Mutation("row");
    m.put("cf", "cq1", "value");
    m.put("cf", "cq2", "value");
    m.put("cf", "cq3", "value");
    m.put("cf", "cq4", "value");
    m.put("cf", "cq5", "value");
    // This Mutation "came" from a system called "peer"
    m.addReplicationSource(peerName);
    // Make it a TMutation
    TMutation tMutation = m.toThrift();
    // And then make a ServerMutation from the TMutation, adding in our systemTimestamp
    ServerMutation sMutation = new ServerMutation(tMutation);
    sMutation.setSystemTimestamp(systemTimestamp);
    // Serialize the ServerMutation (what AccumuloReplicaSystem will be doing)
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    DataOutputStream out = new DataOutputStream(baos);
    key.write(out);
    out.writeInt(1);
    sMutation.write(out);
    out.close();
    // Add it to our "input" to BatchWriterReplicationReplayer
    edits.addToEdits(ByteBuffer.wrap(baos.toByteArray()));
    Mutation expectedMutation = new Mutation("row");
    expectedMutation.put("cf", "cq1", sMutation.getSystemTimestamp(), "value");
    expectedMutation.put("cf", "cq2", sMutation.getSystemTimestamp(), "value");
    expectedMutation.put("cf", "cq3", sMutation.getSystemTimestamp(), "value");
    expectedMutation.put("cf", "cq4", sMutation.getSystemTimestamp(), "value");
    expectedMutation.put("cf", "cq5", sMutation.getSystemTimestamp(), "value");
    // We expect our peer name to be preserved in the mutation that gets written
    expectedMutation.addReplicationSource(peerName);
    expect(conf.getAsBytes(Property.TSERV_REPLICATION_BW_REPLAYER_MEMORY)).andReturn(bwCfg.getMaxMemory());
    expect(conn.createBatchWriter(tableName, bwCfg)).andReturn(bw);
    bw.addMutations(Lists.newArrayList(expectedMutation));
    expectLastCall().once();
    bw.close();
    expectLastCall().once();
    replay(conn, conf, bw);
    replayer.replicateLog(context, tableName, edits);
}