Search in sources :

Example 6 with WalReplication

use of org.apache.accumulo.tserver.replication.AccumuloReplicaSystem.WalReplication in project accumulo by apache.

the class AccumuloReplicaSystemTest method onlyChooseMutationsForDesiredTableWithClosedStatus.

@Test
public void onlyChooseMutationsForDesiredTableWithClosedStatus() throws Exception {
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    DataOutputStream dos = new DataOutputStream(baos);
    LogFileKey key = new LogFileKey();
    LogFileValue value = new LogFileValue();
    // What is seq used for?
    key.seq = 1l;
    /*
     * Disclaimer: the following series of LogFileKey and LogFileValue pairs have *no* bearing whatsoever in reality regarding what these entries would actually
     * look like in a WAL. They are solely for testing that each LogEvents is handled, order is not important.
     */
    key.event = LogEvents.DEFINE_TABLET;
    key.tablet = new KeyExtent(Table.ID.of("1"), null, null);
    key.tid = 1;
    key.write(dos);
    value.write(dos);
    key.tablet = null;
    key.event = LogEvents.MUTATION;
    key.filename = "/accumulo/wals/tserver+port/" + UUID.randomUUID();
    value.mutations = Arrays.asList(new ServerMutation(new Text("row")));
    key.write(dos);
    value.write(dos);
    key.event = LogEvents.DEFINE_TABLET;
    key.tablet = new KeyExtent(Table.ID.of("2"), null, null);
    key.tid = 2;
    value.mutations = Collections.emptyList();
    key.write(dos);
    value.write(dos);
    key.event = LogEvents.OPEN;
    key.tid = LogFileKey.VERSION;
    key.tserverSession = "foobar";
    key.write(dos);
    value.write(dos);
    key.tablet = null;
    key.event = LogEvents.MUTATION;
    key.filename = "/accumulo/wals/tserver+port/" + UUID.randomUUID();
    value.mutations = Arrays.asList(new ServerMutation(new Text("badrow")));
    key.write(dos);
    value.write(dos);
    key.event = LogEvents.COMPACTION_START;
    key.tid = 2;
    key.filename = "/accumulo/tables/1/t-000001/A000001.rf";
    value.mutations = Collections.emptyList();
    key.write(dos);
    value.write(dos);
    key.event = LogEvents.DEFINE_TABLET;
    key.tablet = new KeyExtent(Table.ID.of("1"), null, null);
    key.tid = 3;
    value.mutations = Collections.emptyList();
    key.write(dos);
    value.write(dos);
    key.event = LogEvents.COMPACTION_FINISH;
    key.tid = 6;
    value.mutations = Collections.emptyList();
    key.write(dos);
    value.write(dos);
    key.tablet = null;
    key.event = LogEvents.MUTATION;
    key.tid = 3;
    key.filename = "/accumulo/wals/tserver+port/" + UUID.randomUUID();
    value.mutations = Arrays.asList(new ServerMutation(new Text("row")));
    key.write(dos);
    value.write(dos);
    dos.close();
    Map<String, String> confMap = new HashMap<>();
    confMap.put(Property.REPLICATION_NAME.getKey(), "source");
    AccumuloConfiguration conf = new ConfigurationCopy(confMap);
    AccumuloReplicaSystem ars = new AccumuloReplicaSystem();
    ars.setConf(conf);
    // Setting the file to be closed with the infinite end implies that we need to bump the begin up to Long.MAX_VALUE
    // If it were still open, more data could be appended that we need to process
    Status status = Status.newBuilder().setBegin(0).setEnd(0).setInfiniteEnd(true).setClosed(true).build();
    DataInputStream dis = new DataInputStream(new ByteArrayInputStream(baos.toByteArray()));
    WalReplication repl = ars.getWalEdits(new ReplicationTarget("peer", "1", Table.ID.of("1")), dis, new Path("/accumulo/wals/tserver+port/wal"), status, Long.MAX_VALUE, new HashSet<>());
    // We stopped because we got to the end of the file
    Assert.assertEquals(Long.MAX_VALUE, repl.entriesConsumed);
    Assert.assertEquals(2, repl.walEdits.getEditsSize());
    Assert.assertEquals(2, repl.sizeInRecords);
    Assert.assertNotEquals(0, repl.sizeInBytes);
}
Also used : Status(org.apache.accumulo.server.replication.proto.Replication.Status) Path(org.apache.hadoop.fs.Path) ConfigurationCopy(org.apache.accumulo.core.conf.ConfigurationCopy) HashMap(java.util.HashMap) DataOutputStream(java.io.DataOutputStream) WalReplication(org.apache.accumulo.tserver.replication.AccumuloReplicaSystem.WalReplication) ServerMutation(org.apache.accumulo.server.data.ServerMutation) Text(org.apache.hadoop.io.Text) ByteArrayOutputStream(java.io.ByteArrayOutputStream) LogFileKey(org.apache.accumulo.tserver.logger.LogFileKey) DataInputStream(java.io.DataInputStream) KeyExtent(org.apache.accumulo.core.data.impl.KeyExtent) ReplicationTarget(org.apache.accumulo.core.replication.ReplicationTarget) ByteArrayInputStream(java.io.ByteArrayInputStream) LogFileValue(org.apache.accumulo.tserver.logger.LogFileValue) AccumuloConfiguration(org.apache.accumulo.core.conf.AccumuloConfiguration) Test(org.junit.Test)

Example 7 with WalReplication

use of org.apache.accumulo.tserver.replication.AccumuloReplicaSystem.WalReplication in project accumulo by apache.

the class AccumuloReplicaSystemTest method onlyChooseMutationsForDesiredTableWithOpenStatus.

@Test
public void onlyChooseMutationsForDesiredTableWithOpenStatus() throws Exception {
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    DataOutputStream dos = new DataOutputStream(baos);
    LogFileKey key = new LogFileKey();
    LogFileValue value = new LogFileValue();
    // What is seq used for?
    key.seq = 1l;
    /*
     * Disclaimer: the following series of LogFileKey and LogFileValue pairs have *no* bearing whatsoever in reality regarding what these entries would actually
     * look like in a WAL. They are solely for testing that each LogEvents is handled, order is not important.
     */
    key.event = LogEvents.DEFINE_TABLET;
    key.tablet = new KeyExtent(Table.ID.of("1"), null, null);
    key.tid = 1;
    key.write(dos);
    value.write(dos);
    key.tablet = null;
    key.event = LogEvents.MUTATION;
    key.filename = "/accumulo/wals/tserver+port/" + UUID.randomUUID();
    value.mutations = Arrays.asList(new ServerMutation(new Text("row")));
    key.write(dos);
    value.write(dos);
    key.event = LogEvents.DEFINE_TABLET;
    key.tablet = new KeyExtent(Table.ID.of("2"), null, null);
    key.tid = 2;
    value.mutations = Collections.emptyList();
    key.write(dos);
    value.write(dos);
    key.event = LogEvents.OPEN;
    key.tid = LogFileKey.VERSION;
    key.tserverSession = "foobar";
    key.write(dos);
    value.write(dos);
    key.tablet = null;
    key.event = LogEvents.MUTATION;
    key.filename = "/accumulo/wals/tserver+port/" + UUID.randomUUID();
    value.mutations = Arrays.asList(new ServerMutation(new Text("badrow")));
    key.write(dos);
    value.write(dos);
    key.event = LogEvents.COMPACTION_START;
    key.tid = 2;
    key.filename = "/accumulo/tables/1/t-000001/A000001.rf";
    value.mutations = Collections.emptyList();
    key.write(dos);
    value.write(dos);
    key.event = LogEvents.DEFINE_TABLET;
    key.tablet = new KeyExtent(Table.ID.of("1"), null, null);
    key.tid = 3;
    value.mutations = Collections.emptyList();
    key.write(dos);
    value.write(dos);
    key.event = LogEvents.COMPACTION_FINISH;
    key.tid = 6;
    value.mutations = Collections.emptyList();
    key.write(dos);
    value.write(dos);
    key.tablet = null;
    key.event = LogEvents.MUTATION;
    key.tid = 3;
    key.filename = "/accumulo/wals/tserver+port/" + UUID.randomUUID();
    value.mutations = Arrays.asList(new ServerMutation(new Text("row")));
    key.write(dos);
    value.write(dos);
    dos.close();
    Map<String, String> confMap = new HashMap<>();
    confMap.put(Property.REPLICATION_NAME.getKey(), "source");
    AccumuloConfiguration conf = new ConfigurationCopy(confMap);
    AccumuloReplicaSystem ars = new AccumuloReplicaSystem();
    ars.setConf(conf);
    Status status = Status.newBuilder().setBegin(0).setEnd(0).setInfiniteEnd(true).setClosed(false).build();
    DataInputStream dis = new DataInputStream(new ByteArrayInputStream(baos.toByteArray()));
    WalReplication repl = ars.getWalEdits(new ReplicationTarget("peer", "1", Table.ID.of("1")), dis, new Path("/accumulo/wals/tserver+port/wal"), status, Long.MAX_VALUE, new HashSet<>());
    // We stopped because we got to the end of the file
    Assert.assertEquals(9, repl.entriesConsumed);
    Assert.assertEquals(2, repl.walEdits.getEditsSize());
    Assert.assertEquals(2, repl.sizeInRecords);
    Assert.assertNotEquals(0, repl.sizeInBytes);
}
Also used : Status(org.apache.accumulo.server.replication.proto.Replication.Status) Path(org.apache.hadoop.fs.Path) ConfigurationCopy(org.apache.accumulo.core.conf.ConfigurationCopy) HashMap(java.util.HashMap) DataOutputStream(java.io.DataOutputStream) WalReplication(org.apache.accumulo.tserver.replication.AccumuloReplicaSystem.WalReplication) ServerMutation(org.apache.accumulo.server.data.ServerMutation) Text(org.apache.hadoop.io.Text) ByteArrayOutputStream(java.io.ByteArrayOutputStream) LogFileKey(org.apache.accumulo.tserver.logger.LogFileKey) DataInputStream(java.io.DataInputStream) KeyExtent(org.apache.accumulo.core.data.impl.KeyExtent) ReplicationTarget(org.apache.accumulo.core.replication.ReplicationTarget) ByteArrayInputStream(java.io.ByteArrayInputStream) LogFileValue(org.apache.accumulo.tserver.logger.LogFileValue) AccumuloConfiguration(org.apache.accumulo.core.conf.AccumuloConfiguration) Test(org.junit.Test)

Aggregations

DataInputStream (java.io.DataInputStream)7 ReplicationTarget (org.apache.accumulo.core.replication.ReplicationTarget)7 Status (org.apache.accumulo.server.replication.proto.Replication.Status)7 WalReplication (org.apache.accumulo.tserver.replication.AccumuloReplicaSystem.WalReplication)7 Path (org.apache.hadoop.fs.Path)7 Test (org.junit.Test)7 ByteArrayInputStream (java.io.ByteArrayInputStream)5 HashMap (java.util.HashMap)5 AccumuloConfiguration (org.apache.accumulo.core.conf.AccumuloConfiguration)5 ConfigurationCopy (org.apache.accumulo.core.conf.ConfigurationCopy)5 ByteArrayOutputStream (java.io.ByteArrayOutputStream)3 DataOutputStream (java.io.DataOutputStream)3 HashSet (java.util.HashSet)3 KeyExtent (org.apache.accumulo.core.data.impl.KeyExtent)3 ServerMutation (org.apache.accumulo.server.data.ServerMutation)3 LogFileKey (org.apache.accumulo.tserver.logger.LogFileKey)3 LogFileValue (org.apache.accumulo.tserver.logger.LogFileValue)3 Text (org.apache.hadoop.io.Text)3 Client (org.apache.accumulo.core.replication.thrift.ReplicationServicer.Client)2 WalEdits (org.apache.accumulo.core.replication.thrift.WalEdits)2