use of org.apache.accumulo.core.replication.ReplicationTarget in project accumulo by apache.
the class FinishedWorkUpdater method run.
@Override
public void run() {
log.debug("Looking for finished replication work");
if (!ReplicationTable.isOnline(conn)) {
log.debug("Replication table is not yet online, will retry");
return;
}
BatchScanner bs;
BatchWriter replBw;
try {
bs = ReplicationTable.getBatchScanner(conn, 4);
replBw = ReplicationTable.getBatchWriter(conn);
} catch (ReplicationTableOfflineException e) {
log.debug("Table is no longer online, will retry");
return;
}
IteratorSetting cfg = new IteratorSetting(50, WholeRowIterator.class);
bs.addScanIterator(cfg);
WorkSection.limit(bs);
bs.setRanges(Collections.singleton(new Range()));
try {
for (Entry<Key, Value> serializedRow : bs) {
SortedMap<Key, Value> wholeRow;
try {
wholeRow = WholeRowIterator.decodeRow(serializedRow.getKey(), serializedRow.getValue());
} catch (IOException e) {
log.warn("Could not deserialize whole row with key {}", serializedRow.getKey().toStringNoTruncate(), e);
continue;
}
log.debug("Processing work progress for {} with {} columns", serializedRow.getKey().getRow(), wholeRow.size());
Map<Table.ID, Long> tableIdToProgress = new HashMap<>();
boolean error = false;
Text buffer = new Text();
// We want to determine what the minimum point that all Work entries have replicated to
for (Entry<Key, Value> entry : wholeRow.entrySet()) {
Status status;
try {
status = Status.parseFrom(entry.getValue().get());
} catch (InvalidProtocolBufferException e) {
log.warn("Could not deserialize protobuf for {}", entry.getKey(), e);
error = true;
break;
}
// Get the replication target for the work record
entry.getKey().getColumnQualifier(buffer);
ReplicationTarget target = ReplicationTarget.from(buffer);
// Initialize the value in the map if we don't have one
if (!tableIdToProgress.containsKey(target.getSourceTableId())) {
tableIdToProgress.put(target.getSourceTableId(), Long.MAX_VALUE);
}
// Find the minimum value for begin (everyone has replicated up to this offset in the file)
tableIdToProgress.put(target.getSourceTableId(), Math.min(tableIdToProgress.get(target.getSourceTableId()), status.getBegin()));
}
if (error) {
continue;
}
// Update the replication table for each source table we found work records for
for (Entry<Table.ID, Long> entry : tableIdToProgress.entrySet()) {
// If the progress is 0, then no one has replicated anything, and we don't need to update anything
if (0 == entry.getValue()) {
continue;
}
serializedRow.getKey().getRow(buffer);
log.debug("For {}, source table ID {} has replicated through {}", serializedRow.getKey().getRow(), entry.getKey(), entry.getValue());
Mutation replMutation = new Mutation(buffer);
// Set that we replicated at least this much data, ignoring the other fields
Status updatedStatus = StatusUtil.replicated(entry.getValue());
Value serializedUpdatedStatus = ProtobufUtil.toValue(updatedStatus);
// Pull the sourceTableId into a Text
Table.ID srcTableId = entry.getKey();
// Make the mutation
StatusSection.add(replMutation, srcTableId, serializedUpdatedStatus);
log.debug("Updating replication status entry for {} with {}", serializedRow.getKey().getRow(), ProtobufUtil.toString(updatedStatus));
try {
replBw.addMutation(replMutation);
} catch (MutationsRejectedException e) {
log.error("Error writing mutations to update replication Status messages in StatusSection, will retry", e);
return;
}
}
}
} finally {
log.debug("Finished updating files with completed replication work");
bs.close();
try {
replBw.close();
} catch (MutationsRejectedException e) {
log.error("Error writing mutations to update replication Status messages in StatusSection, will retry", e);
}
}
}
use of org.apache.accumulo.core.replication.ReplicationTarget in project accumulo by apache.
the class ReplicationProcessor method process.
@Override
public void process(String workID, byte[] data) {
ReplicationTarget target = DistributedWorkQueueWorkAssignerHelper.fromQueueKey(workID).getValue();
String file = new String(data, UTF_8);
log.debug("Received replication work for {} to {}", file, target);
ReplicaSystem replica;
try {
replica = getReplicaSystem(target);
} catch (Exception e) {
log.error("Could not instantiate ReplicaSystem for {}, waiting before returning the work", target, e);
try {
// TODO configurable
Thread.sleep(5000);
} catch (InterruptedException ie) {
Thread.currentThread().interrupt();
}
return;
}
Status status;
try {
status = getStatus(file, target);
} catch (ReplicationTableOfflineException | AccumuloException | AccumuloSecurityException e) {
log.error("Could not look for replication record", e);
throw new IllegalStateException("Could not look for replication record", e);
} catch (InvalidProtocolBufferException e) {
log.error("Could not deserialize Status from Work section for {} and ", file, target);
throw new RuntimeException("Could not parse Status for work record", e);
} catch (NoSuchElementException e) {
log.error("Assigned work for {} to {} but could not find work record", file, target);
return;
}
log.debug("Current status for {} replicating to {}: {}", file, target, ProtobufUtil.toString(status));
// We don't need to do anything (shouldn't have gotten this work record in the first place)
if (!StatusUtil.isWorkRequired(status)) {
log.info("Received work request for {} and {}, but it does not need replication. Ignoring...", file, target);
return;
}
// Sanity check that nothing bad happened and our replication source still exists
Path filePath = new Path(file);
try {
if (!doesFileExist(filePath, target)) {
return;
}
} catch (IOException e) {
log.error("Could not determine if file exists {}", filePath, e);
throw new RuntimeException(e);
}
log.debug("Replicating {} to {} using {}", filePath, target, replica.getClass().getName());
Status newStatus = replica.replicate(filePath, status, target, getHelper());
log.debug("Finished replicating {}. Original status: {}, New status: {}", filePath, status, newStatus);
}
use of org.apache.accumulo.core.replication.ReplicationTarget in project accumulo by apache.
the class AccumuloReplicaSystemTest method onlyChooseMutationsForDesiredTableWithClosedStatus.
@Test
public void onlyChooseMutationsForDesiredTableWithClosedStatus() throws Exception {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
DataOutputStream dos = new DataOutputStream(baos);
LogFileKey key = new LogFileKey();
LogFileValue value = new LogFileValue();
// What is seq used for?
key.seq = 1l;
/*
* Disclaimer: the following series of LogFileKey and LogFileValue pairs have *no* bearing whatsoever in reality regarding what these entries would actually
* look like in a WAL. They are solely for testing that each LogEvents is handled, order is not important.
*/
key.event = LogEvents.DEFINE_TABLET;
key.tablet = new KeyExtent(Table.ID.of("1"), null, null);
key.tid = 1;
key.write(dos);
value.write(dos);
key.tablet = null;
key.event = LogEvents.MUTATION;
key.filename = "/accumulo/wals/tserver+port/" + UUID.randomUUID();
value.mutations = Arrays.asList(new ServerMutation(new Text("row")));
key.write(dos);
value.write(dos);
key.event = LogEvents.DEFINE_TABLET;
key.tablet = new KeyExtent(Table.ID.of("2"), null, null);
key.tid = 2;
value.mutations = Collections.emptyList();
key.write(dos);
value.write(dos);
key.event = LogEvents.OPEN;
key.tid = LogFileKey.VERSION;
key.tserverSession = "foobar";
key.write(dos);
value.write(dos);
key.tablet = null;
key.event = LogEvents.MUTATION;
key.filename = "/accumulo/wals/tserver+port/" + UUID.randomUUID();
value.mutations = Arrays.asList(new ServerMutation(new Text("badrow")));
key.write(dos);
value.write(dos);
key.event = LogEvents.COMPACTION_START;
key.tid = 2;
key.filename = "/accumulo/tables/1/t-000001/A000001.rf";
value.mutations = Collections.emptyList();
key.write(dos);
value.write(dos);
key.event = LogEvents.DEFINE_TABLET;
key.tablet = new KeyExtent(Table.ID.of("1"), null, null);
key.tid = 3;
value.mutations = Collections.emptyList();
key.write(dos);
value.write(dos);
key.event = LogEvents.COMPACTION_FINISH;
key.tid = 6;
value.mutations = Collections.emptyList();
key.write(dos);
value.write(dos);
key.tablet = null;
key.event = LogEvents.MUTATION;
key.tid = 3;
key.filename = "/accumulo/wals/tserver+port/" + UUID.randomUUID();
value.mutations = Arrays.asList(new ServerMutation(new Text("row")));
key.write(dos);
value.write(dos);
dos.close();
Map<String, String> confMap = new HashMap<>();
confMap.put(Property.REPLICATION_NAME.getKey(), "source");
AccumuloConfiguration conf = new ConfigurationCopy(confMap);
AccumuloReplicaSystem ars = new AccumuloReplicaSystem();
ars.setConf(conf);
// Setting the file to be closed with the infinite end implies that we need to bump the begin up to Long.MAX_VALUE
// If it were still open, more data could be appended that we need to process
Status status = Status.newBuilder().setBegin(0).setEnd(0).setInfiniteEnd(true).setClosed(true).build();
DataInputStream dis = new DataInputStream(new ByteArrayInputStream(baos.toByteArray()));
WalReplication repl = ars.getWalEdits(new ReplicationTarget("peer", "1", Table.ID.of("1")), dis, new Path("/accumulo/wals/tserver+port/wal"), status, Long.MAX_VALUE, new HashSet<>());
// We stopped because we got to the end of the file
Assert.assertEquals(Long.MAX_VALUE, repl.entriesConsumed);
Assert.assertEquals(2, repl.walEdits.getEditsSize());
Assert.assertEquals(2, repl.sizeInRecords);
Assert.assertNotEquals(0, repl.sizeInBytes);
}
use of org.apache.accumulo.core.replication.ReplicationTarget in project accumulo by apache.
the class AccumuloReplicaSystemTest method testUserPassword.
@Test
public void testUserPassword() throws Exception {
AccumuloReplicaSystem ars = new AccumuloReplicaSystem();
ReplicationTarget target = new ReplicationTarget("peer", "peer_table", Table.ID.of("1"));
String user = "user", password = "password";
Map<String, String> confMap = new HashMap<>();
confMap.put(Property.REPLICATION_PEER_USER.getKey() + target.getPeerName(), user);
confMap.put(Property.REPLICATION_PEER_PASSWORD.getKey() + target.getPeerName(), password);
AccumuloConfiguration conf = new ConfigurationCopy(confMap);
assertEquals(user, ars.getPrincipal(conf, target));
assertEquals(password, ars.getPassword(conf, target));
}
use of org.apache.accumulo.core.replication.ReplicationTarget in project accumulo by apache.
the class AccumuloReplicaSystemTest method onlyChooseMutationsForDesiredTableWithOpenStatus.
@Test
public void onlyChooseMutationsForDesiredTableWithOpenStatus() throws Exception {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
DataOutputStream dos = new DataOutputStream(baos);
LogFileKey key = new LogFileKey();
LogFileValue value = new LogFileValue();
// What is seq used for?
key.seq = 1l;
/*
* Disclaimer: the following series of LogFileKey and LogFileValue pairs have *no* bearing whatsoever in reality regarding what these entries would actually
* look like in a WAL. They are solely for testing that each LogEvents is handled, order is not important.
*/
key.event = LogEvents.DEFINE_TABLET;
key.tablet = new KeyExtent(Table.ID.of("1"), null, null);
key.tid = 1;
key.write(dos);
value.write(dos);
key.tablet = null;
key.event = LogEvents.MUTATION;
key.filename = "/accumulo/wals/tserver+port/" + UUID.randomUUID();
value.mutations = Arrays.asList(new ServerMutation(new Text("row")));
key.write(dos);
value.write(dos);
key.event = LogEvents.DEFINE_TABLET;
key.tablet = new KeyExtent(Table.ID.of("2"), null, null);
key.tid = 2;
value.mutations = Collections.emptyList();
key.write(dos);
value.write(dos);
key.event = LogEvents.OPEN;
key.tid = LogFileKey.VERSION;
key.tserverSession = "foobar";
key.write(dos);
value.write(dos);
key.tablet = null;
key.event = LogEvents.MUTATION;
key.filename = "/accumulo/wals/tserver+port/" + UUID.randomUUID();
value.mutations = Arrays.asList(new ServerMutation(new Text("badrow")));
key.write(dos);
value.write(dos);
key.event = LogEvents.COMPACTION_START;
key.tid = 2;
key.filename = "/accumulo/tables/1/t-000001/A000001.rf";
value.mutations = Collections.emptyList();
key.write(dos);
value.write(dos);
key.event = LogEvents.DEFINE_TABLET;
key.tablet = new KeyExtent(Table.ID.of("1"), null, null);
key.tid = 3;
value.mutations = Collections.emptyList();
key.write(dos);
value.write(dos);
key.event = LogEvents.COMPACTION_FINISH;
key.tid = 6;
value.mutations = Collections.emptyList();
key.write(dos);
value.write(dos);
key.tablet = null;
key.event = LogEvents.MUTATION;
key.tid = 3;
key.filename = "/accumulo/wals/tserver+port/" + UUID.randomUUID();
value.mutations = Arrays.asList(new ServerMutation(new Text("row")));
key.write(dos);
value.write(dos);
dos.close();
Map<String, String> confMap = new HashMap<>();
confMap.put(Property.REPLICATION_NAME.getKey(), "source");
AccumuloConfiguration conf = new ConfigurationCopy(confMap);
AccumuloReplicaSystem ars = new AccumuloReplicaSystem();
ars.setConf(conf);
Status status = Status.newBuilder().setBegin(0).setEnd(0).setInfiniteEnd(true).setClosed(false).build();
DataInputStream dis = new DataInputStream(new ByteArrayInputStream(baos.toByteArray()));
WalReplication repl = ars.getWalEdits(new ReplicationTarget("peer", "1", Table.ID.of("1")), dis, new Path("/accumulo/wals/tserver+port/wal"), status, Long.MAX_VALUE, new HashSet<>());
// We stopped because we got to the end of the file
Assert.assertEquals(9, repl.entriesConsumed);
Assert.assertEquals(2, repl.walEdits.getEditsSize());
Assert.assertEquals(2, repl.sizeInRecords);
Assert.assertNotEquals(0, repl.sizeInBytes);
}
Aggregations