Examples with ServerMutation org.apache.accumulo.server.data.ServerMutation used on opensource projects

Example 26 with ServerMutation

use of org.apache.accumulo.server.data.ServerMutation in project accumulo by apache.

the class AccumuloReplicaSystemTest method onlyChooseMutationsForDesiredTableWithClosedStatus.

@Test
public void onlyChooseMutationsForDesiredTableWithClosedStatus() throws Exception {
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    DataOutputStream dos = new DataOutputStream(baos);
    LogFileKey key = new LogFileKey();
    LogFileValue value = new LogFileValue();
    // What is seq used for?
    key.seq = 1l;
    /*
     * Disclaimer: the following series of LogFileKey and LogFileValue pairs have *no* bearing whatsoever in reality regarding what these entries would actually
     * look like in a WAL. They are solely for testing that each LogEvents is handled, order is not important.
     */
    key.event = LogEvents.DEFINE_TABLET;
    key.tablet = new KeyExtent(Table.ID.of("1"), null, null);
    key.tid = 1;
    key.write(dos);
    value.write(dos);
    key.tablet = null;
    key.event = LogEvents.MUTATION;
    key.filename = "/accumulo/wals/tserver+port/" + UUID.randomUUID();
    value.mutations = Arrays.asList(new ServerMutation(new Text("row")));
    key.write(dos);
    value.write(dos);
    key.event = LogEvents.DEFINE_TABLET;
    key.tablet = new KeyExtent(Table.ID.of("2"), null, null);
    key.tid = 2;
    value.mutations = Collections.emptyList();
    key.write(dos);
    value.write(dos);
    key.event = LogEvents.OPEN;
    key.tid = LogFileKey.VERSION;
    key.tserverSession = "foobar";
    key.write(dos);
    value.write(dos);
    key.tablet = null;
    key.event = LogEvents.MUTATION;
    key.filename = "/accumulo/wals/tserver+port/" + UUID.randomUUID();
    value.mutations = Arrays.asList(new ServerMutation(new Text("badrow")));
    key.write(dos);
    value.write(dos);
    key.event = LogEvents.COMPACTION_START;
    key.tid = 2;
    key.filename = "/accumulo/tables/1/t-000001/A000001.rf";
    value.mutations = Collections.emptyList();
    key.write(dos);
    value.write(dos);
    key.event = LogEvents.DEFINE_TABLET;
    key.tablet = new KeyExtent(Table.ID.of("1"), null, null);
    key.tid = 3;
    value.mutations = Collections.emptyList();
    key.write(dos);
    value.write(dos);
    key.event = LogEvents.COMPACTION_FINISH;
    key.tid = 6;
    value.mutations = Collections.emptyList();
    key.write(dos);
    value.write(dos);
    key.tablet = null;
    key.event = LogEvents.MUTATION;
    key.tid = 3;
    key.filename = "/accumulo/wals/tserver+port/" + UUID.randomUUID();
    value.mutations = Arrays.asList(new ServerMutation(new Text("row")));
    key.write(dos);
    value.write(dos);
    dos.close();
    Map<String, String> confMap = new HashMap<>();
    confMap.put(Property.REPLICATION_NAME.getKey(), "source");
    AccumuloConfiguration conf = new ConfigurationCopy(confMap);
    AccumuloReplicaSystem ars = new AccumuloReplicaSystem();
    ars.setConf(conf);
    // Setting the file to be closed with the infinite end implies that we need to bump the begin up to Long.MAX_VALUE
    // If it were still open, more data could be appended that we need to process
    Status status = Status.newBuilder().setBegin(0).setEnd(0).setInfiniteEnd(true).setClosed(true).build();
    DataInputStream dis = new DataInputStream(new ByteArrayInputStream(baos.toByteArray()));
    WalReplication repl = ars.getWalEdits(new ReplicationTarget("peer", "1", Table.ID.of("1")), dis, new Path("/accumulo/wals/tserver+port/wal"), status, Long.MAX_VALUE, new HashSet<>());
    // We stopped because we got to the end of the file
    Assert.assertEquals(Long.MAX_VALUE, repl.entriesConsumed);
    Assert.assertEquals(2, repl.walEdits.getEditsSize());
    Assert.assertEquals(2, repl.sizeInRecords);
    Assert.assertNotEquals(0, repl.sizeInBytes);
}

Example 27 with ServerMutation

use of org.apache.accumulo.server.data.ServerMutation in project accumulo by apache.

the class AccumuloReplicaSystemTest method onlyChooseMutationsForDesiredTableWithOpenStatus.

@Test
public void onlyChooseMutationsForDesiredTableWithOpenStatus() throws Exception {
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    DataOutputStream dos = new DataOutputStream(baos);
    LogFileKey key = new LogFileKey();
    LogFileValue value = new LogFileValue();
    // What is seq used for?
    key.seq = 1l;
    /*
     * Disclaimer: the following series of LogFileKey and LogFileValue pairs have *no* bearing whatsoever in reality regarding what these entries would actually
     * look like in a WAL. They are solely for testing that each LogEvents is handled, order is not important.
     */
    key.event = LogEvents.DEFINE_TABLET;
    key.tablet = new KeyExtent(Table.ID.of("1"), null, null);
    key.tid = 1;
    key.write(dos);
    value.write(dos);
    key.tablet = null;
    key.event = LogEvents.MUTATION;
    key.filename = "/accumulo/wals/tserver+port/" + UUID.randomUUID();
    value.mutations = Arrays.asList(new ServerMutation(new Text("row")));
    key.write(dos);
    value.write(dos);
    key.event = LogEvents.DEFINE_TABLET;
    key.tablet = new KeyExtent(Table.ID.of("2"), null, null);
    key.tid = 2;
    value.mutations = Collections.emptyList();
    key.write(dos);
    value.write(dos);
    key.event = LogEvents.OPEN;
    key.tid = LogFileKey.VERSION;
    key.tserverSession = "foobar";
    key.write(dos);
    value.write(dos);
    key.tablet = null;
    key.event = LogEvents.MUTATION;
    key.filename = "/accumulo/wals/tserver+port/" + UUID.randomUUID();
    value.mutations = Arrays.asList(new ServerMutation(new Text("badrow")));
    key.write(dos);
    value.write(dos);
    key.event = LogEvents.COMPACTION_START;
    key.tid = 2;
    key.filename = "/accumulo/tables/1/t-000001/A000001.rf";
    value.mutations = Collections.emptyList();
    key.write(dos);
    value.write(dos);
    key.event = LogEvents.DEFINE_TABLET;
    key.tablet = new KeyExtent(Table.ID.of("1"), null, null);
    key.tid = 3;
    value.mutations = Collections.emptyList();
    key.write(dos);
    value.write(dos);
    key.event = LogEvents.COMPACTION_FINISH;
    key.tid = 6;
    value.mutations = Collections.emptyList();
    key.write(dos);
    value.write(dos);
    key.tablet = null;
    key.event = LogEvents.MUTATION;
    key.tid = 3;
    key.filename = "/accumulo/wals/tserver+port/" + UUID.randomUUID();
    value.mutations = Arrays.asList(new ServerMutation(new Text("row")));
    key.write(dos);
    value.write(dos);
    dos.close();
    Map<String, String> confMap = new HashMap<>();
    confMap.put(Property.REPLICATION_NAME.getKey(), "source");
    AccumuloConfiguration conf = new ConfigurationCopy(confMap);
    AccumuloReplicaSystem ars = new AccumuloReplicaSystem();
    ars.setConf(conf);
    Status status = Status.newBuilder().setBegin(0).setEnd(0).setInfiniteEnd(true).setClosed(false).build();
    DataInputStream dis = new DataInputStream(new ByteArrayInputStream(baos.toByteArray()));
    WalReplication repl = ars.getWalEdits(new ReplicationTarget("peer", "1", Table.ID.of("1")), dis, new Path("/accumulo/wals/tserver+port/wal"), status, Long.MAX_VALUE, new HashSet<>());
    // We stopped because we got to the end of the file
    Assert.assertEquals(9, repl.entriesConsumed);
    Assert.assertEquals(2, repl.walEdits.getEditsSize());
    Assert.assertEquals(2, repl.sizeInRecords);
    Assert.assertNotEquals(0, repl.sizeInBytes);
}

Example 28 with ServerMutation

use of org.apache.accumulo.server.data.ServerMutation in project accumulo by apache.

the class BatchWriterReplicationReplayerTest method systemTimestampsAreSetOnUpdates.

@Test
public void systemTimestampsAreSetOnUpdates() throws Exception {
    final BatchWriterReplicationReplayer replayer = new BatchWriterReplicationReplayer();
    final String tableName = "foo";
    final long systemTimestamp = 1000;
    final BatchWriterConfig bwCfg = new BatchWriterConfig();
    bwCfg.setMaxMemory(1l);
    LogFileKey key = new LogFileKey();
    key.event = LogEvents.MANY_MUTATIONS;
    key.seq = 1;
    key.tid = 1;
    WalEdits edits = new WalEdits();
    // Make a mutation without timestamps
    Mutation m = new Mutation("row");
    m.put("cf", "cq1", "value");
    m.put("cf", "cq2", "value");
    m.put("cf", "cq3", "value");
    m.put("cf", "cq4", "value");
    m.put("cf", "cq5", "value");
    // Make it a TMutation
    TMutation tMutation = m.toThrift();
    // And then make a ServerMutation from the TMutation, adding in our systemTimestamp
    ServerMutation sMutation = new ServerMutation(tMutation);
    sMutation.setSystemTimestamp(systemTimestamp);
    // Serialize the ServerMutation (what AccumuloReplicaSystem will be doing)
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    DataOutputStream out = new DataOutputStream(baos);
    key.write(out);
    out.writeInt(1);
    sMutation.write(out);
    out.close();
    // Add it to our "input" to BatchWriterReplicationReplayer
    edits.addToEdits(ByteBuffer.wrap(baos.toByteArray()));
    Mutation expectedMutation = new Mutation("row");
    expectedMutation.put("cf", "cq1", sMutation.getSystemTimestamp(), "value");
    expectedMutation.put("cf", "cq2", sMutation.getSystemTimestamp(), "value");
    expectedMutation.put("cf", "cq3", sMutation.getSystemTimestamp(), "value");
    expectedMutation.put("cf", "cq4", sMutation.getSystemTimestamp(), "value");
    expectedMutation.put("cf", "cq5", sMutation.getSystemTimestamp(), "value");
    expect(conf.getAsBytes(Property.TSERV_REPLICATION_BW_REPLAYER_MEMORY)).andReturn(bwCfg.getMaxMemory());
    expect(conn.createBatchWriter(tableName, bwCfg)).andReturn(bw);
    bw.addMutations(Lists.newArrayList(expectedMutation));
    expectLastCall().once();
    bw.close();
    expectLastCall().once();
    replay(conn, conf, bw);
    replayer.replicateLog(context, tableName, edits);
}

Example 29 with ServerMutation

use of org.apache.accumulo.server.data.ServerMutation in project accumulo by apache.

the class BatchWriterReplicationReplayerTest method replicationSourcesArePreserved.

@Test
public void replicationSourcesArePreserved() throws Exception {
    final BatchWriterReplicationReplayer replayer = new BatchWriterReplicationReplayer();
    final String tableName = "foo";
    final long systemTimestamp = 1000;
    final String peerName = "peer";
    final BatchWriterConfig bwCfg = new BatchWriterConfig();
    bwCfg.setMaxMemory(1l);
    LogFileKey key = new LogFileKey();
    key.event = LogEvents.MANY_MUTATIONS;
    key.seq = 1;
    key.tid = 1;
    WalEdits edits = new WalEdits();
    // Make a mutation without timestamps
    Mutation m = new Mutation("row");
    m.put("cf", "cq1", "value");
    m.put("cf", "cq2", "value");
    m.put("cf", "cq3", "value");
    m.put("cf", "cq4", "value");
    m.put("cf", "cq5", "value");
    // This Mutation "came" from a system called "peer"
    m.addReplicationSource(peerName);
    // Make it a TMutation
    TMutation tMutation = m.toThrift();
    // And then make a ServerMutation from the TMutation, adding in our systemTimestamp
    ServerMutation sMutation = new ServerMutation(tMutation);
    sMutation.setSystemTimestamp(systemTimestamp);
    // Serialize the ServerMutation (what AccumuloReplicaSystem will be doing)
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    DataOutputStream out = new DataOutputStream(baos);
    key.write(out);
    out.writeInt(1);
    sMutation.write(out);
    out.close();
    // Add it to our "input" to BatchWriterReplicationReplayer
    edits.addToEdits(ByteBuffer.wrap(baos.toByteArray()));
    Mutation expectedMutation = new Mutation("row");
    expectedMutation.put("cf", "cq1", sMutation.getSystemTimestamp(), "value");
    expectedMutation.put("cf", "cq2", sMutation.getSystemTimestamp(), "value");
    expectedMutation.put("cf", "cq3", sMutation.getSystemTimestamp(), "value");
    expectedMutation.put("cf", "cq4", sMutation.getSystemTimestamp(), "value");
    expectedMutation.put("cf", "cq5", sMutation.getSystemTimestamp(), "value");
    // We expect our peer name to be preserved in the mutation that gets written
    expectedMutation.addReplicationSource(peerName);
    expect(conf.getAsBytes(Property.TSERV_REPLICATION_BW_REPLAYER_MEMORY)).andReturn(bwCfg.getMaxMemory());
    expect(conn.createBatchWriter(tableName, bwCfg)).andReturn(bw);
    bw.addMutations(Lists.newArrayList(expectedMutation));
    expectLastCall().once();
    bw.close();
    expectLastCall().once();
    replay(conn, conf, bw);
    replayer.replicateLog(context, tableName, edits);
}

Example 30 with ServerMutation

use of org.apache.accumulo.server.data.ServerMutation in project accumulo by apache.

the class UnusedWalDoesntCloseReplicationStatusIT method test.

@Test
public void test() throws Exception {
    File accumuloDir = this.getCluster().getConfig().getAccumuloDir();
    final Connector conn = getConnector();
    final String tableName = getUniqueNames(1)[0];
    conn.securityOperations().grantTablePermission("root", MetadataTable.NAME, TablePermission.WRITE);
    conn.tableOperations().create(tableName);
    final Table.ID tableId = Table.ID.of(conn.tableOperations().tableIdMap().get(tableName));
    final int numericTableId = Integer.parseInt(tableId.canonicalID());
    final int fakeTableId = numericTableId + 1;
    Assert.assertNotNull("Did not find table ID", tableId);
    conn.tableOperations().setProperty(tableName, Property.TABLE_REPLICATION.getKey(), "true");
    conn.tableOperations().setProperty(tableName, Property.TABLE_REPLICATION_TARGET.getKey() + "cluster1", "1");
    // just sleep
    conn.instanceOperations().setProperty(Property.REPLICATION_PEERS.getKey() + "cluster1", ReplicaSystemFactory.getPeerConfigurationValue(MockReplicaSystem.class, "50000"));
    FileSystem fs = FileSystem.getLocal(new Configuration());
    File tserverWalDir = new File(accumuloDir, ServerConstants.WAL_DIR + Path.SEPARATOR + "faketserver+port");
    File tserverWal = new File(tserverWalDir, UUID.randomUUID().toString());
    fs.mkdirs(new Path(tserverWalDir.getAbsolutePath()));
    // Make a fake WAL with no data in it for our real table
    FSDataOutputStream out = fs.create(new Path(tserverWal.getAbsolutePath()));
    out.write(DfsLogger.LOG_FILE_HEADER_V3.getBytes(UTF_8));
    DataOutputStream dos = new DataOutputStream(out);
    dos.writeUTF("NullCryptoModule");
    // Fake a single update WAL that has a mutation for another table
    LogFileKey key = new LogFileKey();
    LogFileValue value = new LogFileValue();
    key.event = OPEN;
    key.tserverSession = tserverWal.getAbsolutePath();
    key.filename = tserverWal.getAbsolutePath();
    key.write(out);
    value.write(out);
    key.event = LogEvents.DEFINE_TABLET;
    key.tablet = new KeyExtent(Table.ID.of(Integer.toString(fakeTableId)), null, null);
    key.seq = 1l;
    key.tid = 1;
    key.write(dos);
    value.write(dos);
    key.tablet = null;
    key.event = LogEvents.MUTATION;
    key.filename = tserverWal.getAbsolutePath();
    value.mutations = Arrays.asList(new ServerMutation(new Text("row")));
    key.write(dos);
    value.write(dos);
    key.event = LogEvents.COMPACTION_START;
    key.filename = accumuloDir.getAbsolutePath() + "/tables/" + fakeTableId + "/t-000001/A000001.rf";
    value.mutations = Collections.emptyList();
    key.write(dos);
    value.write(dos);
    key.event = LogEvents.COMPACTION_FINISH;
    value.mutations = Collections.emptyList();
    key.write(dos);
    value.write(dos);
    dos.close();
    BatchWriter bw = conn.createBatchWriter(tableName, new BatchWriterConfig());
    Mutation m = new Mutation("m");
    m.put("m", "m", "M");
    bw.addMutation(m);
    bw.close();
    log.info("State of metadata table after inserting a record");
    try (Scanner s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
        s.setRange(MetadataSchema.TabletsSection.getRange(tableId));
        for (Entry<Key, Value> entry : s) {
            System.out.println(entry.getKey().toStringNoTruncate() + " " + entry.getValue());
        }
    }
    try (Scanner s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
        s.setRange(MetadataSchema.ReplicationSection.getRange());
        for (Entry<Key, Value> entry : s) {
            System.out.println(entry.getKey().toStringNoTruncate() + " " + ProtobufUtil.toString(Status.parseFrom(entry.getValue().get())));
        }
        log.info("Offline'ing table");
        conn.tableOperations().offline(tableName, true);
        // Add our fake WAL to the log column for this table
        String walUri = tserverWal.toURI().toString();
        KeyExtent extent = new KeyExtent(tableId, null, null);
        bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
        m = new Mutation(extent.getMetadataEntry());
        m.put(MetadataSchema.TabletsSection.LogColumnFamily.NAME, new Text("localhost:12345/" + walUri), new Value((walUri + "|1").getBytes(UTF_8)));
        bw.addMutation(m);
        // Add a replication entry for our fake WAL
        m = new Mutation(MetadataSchema.ReplicationSection.getRowPrefix() + new Path(walUri).toString());
        m.put(MetadataSchema.ReplicationSection.COLF, new Text(tableId.getUtf8()), new Value(StatusUtil.fileCreated(System.currentTimeMillis()).toByteArray()));
        bw.addMutation(m);
        bw.close();
        log.info("State of metadata after injecting WAL manually");
    }
    try (Scanner s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
        s.setRange(MetadataSchema.TabletsSection.getRange(tableId));
        for (Entry<Key, Value> entry : s) {
            log.info("{} {}", entry.getKey().toStringNoTruncate(), entry.getValue());
        }
    }
    try (Scanner s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
        s.setRange(MetadataSchema.ReplicationSection.getRange());
        for (Entry<Key, Value> entry : s) {
            log.info("{} {}", entry.getKey().toStringNoTruncate(), ProtobufUtil.toString(Status.parseFrom(entry.getValue().get())));
        }
        log.info("Bringing table online");
        conn.tableOperations().online(tableName, true);
        Assert.assertEquals(1, Iterables.size(conn.createScanner(tableName, Authorizations.EMPTY)));
        log.info("Table has performed recovery, state of metadata:");
    }
    try (Scanner s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
        s.setRange(MetadataSchema.TabletsSection.getRange(tableId));
        for (Entry<Key, Value> entry : s) {
            log.info("{} {}", entry.getKey().toStringNoTruncate(), entry.getValue());
        }
    }
    try (Scanner s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
        s.setRange(MetadataSchema.ReplicationSection.getRange());
        for (Entry<Key, Value> entry : s) {
            Status status = Status.parseFrom(entry.getValue().get());
            log.info("{} {}", entry.getKey().toStringNoTruncate(), ProtobufUtil.toString(status));
            Assert.assertFalse("Status record was closed and it should not be", status.getClosed());
        }
    }
}