use of org.apache.accumulo.core.client.BatchWriterConfig in project accumulo by apache.
the class ReplicationIT method filesClosedAfterUnused.
@Test
public void filesClosedAfterUnused() throws Exception {
Connector conn = getConnector();
String table = "table";
conn.tableOperations().create(table);
Table.ID tableId = Table.ID.of(conn.tableOperations().tableIdMap().get(table));
Assert.assertNotNull(tableId);
conn.tableOperations().setProperty(table, Property.TABLE_REPLICATION.getKey(), "true");
conn.tableOperations().setProperty(table, Property.TABLE_REPLICATION_TARGET.getKey() + "cluster1", "1");
// just sleep
conn.instanceOperations().setProperty(Property.REPLICATION_PEERS.getKey() + "cluster1", ReplicaSystemFactory.getPeerConfigurationValue(MockReplicaSystem.class, "50000"));
// Write a mutation to make a log file
BatchWriter bw = conn.createBatchWriter(table, new BatchWriterConfig());
Mutation m = new Mutation("one");
m.put("", "", "");
bw.addMutation(m);
bw.close();
// Write another to make sure the logger rolls itself?
bw = conn.createBatchWriter(table, new BatchWriterConfig());
m = new Mutation("three");
m.put("", "", "");
bw.addMutation(m);
bw.close();
try (Scanner s = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
s.fetchColumnFamily(TabletsSection.LogColumnFamily.NAME);
s.setRange(TabletsSection.getRange(tableId));
Set<String> wals = new HashSet<>();
for (Entry<Key, Value> entry : s) {
LogEntry logEntry = LogEntry.fromKeyValue(entry.getKey(), entry.getValue());
wals.add(new Path(logEntry.filename).toString());
}
log.warn("Found wals {}", wals);
bw = conn.createBatchWriter(table, new BatchWriterConfig());
m = new Mutation("three");
byte[] bytes = new byte[1024 * 1024];
m.put("1".getBytes(), new byte[0], bytes);
m.put("2".getBytes(), new byte[0], bytes);
m.put("3".getBytes(), new byte[0], bytes);
m.put("4".getBytes(), new byte[0], bytes);
m.put("5".getBytes(), new byte[0], bytes);
bw.addMutation(m);
bw.close();
conn.tableOperations().flush(table, null, null, true);
while (!ReplicationTable.isOnline(conn)) {
sleepUninterruptibly(MILLIS_BETWEEN_REPLICATION_TABLE_ONLINE_CHECKS, TimeUnit.MILLISECONDS);
}
for (int i = 0; i < 10; i++) {
try (Scanner s2 = conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
s2.fetchColumnFamily(LogColumnFamily.NAME);
s2.setRange(TabletsSection.getRange(tableId));
for (Entry<Key, Value> entry : s2) {
log.info("{}={}", entry.getKey().toStringNoTruncate(), entry.getValue());
}
}
try (Scanner s3 = ReplicationTable.getScanner(conn)) {
StatusSection.limit(s3);
Text buff = new Text();
boolean allReferencedLogsClosed = true;
int recordsFound = 0;
for (Entry<Key, Value> e : s3) {
recordsFound++;
allReferencedLogsClosed = true;
StatusSection.getFile(e.getKey(), buff);
String file = buff.toString();
if (wals.contains(file)) {
Status stat = Status.parseFrom(e.getValue().get());
if (!stat.getClosed()) {
log.info("{} wasn't closed", file);
allReferencedLogsClosed = false;
}
}
}
if (recordsFound > 0 && allReferencedLogsClosed) {
return;
}
Thread.sleep(2000);
} catch (RuntimeException e) {
Throwable cause = e.getCause();
if (cause instanceof AccumuloSecurityException) {
AccumuloSecurityException ase = (AccumuloSecurityException) cause;
switch(ase.getSecurityErrorCode()) {
case PERMISSION_DENIED:
// We tried to read the replication table before the GRANT went through
Thread.sleep(2000);
break;
default:
throw e;
}
}
}
}
Assert.fail("We had a file that was referenced but didn't get closed");
}
}
use of org.apache.accumulo.core.client.BatchWriterConfig in project accumulo by apache.
the class ReplicationOperationsImplIT method waitsUntilEntriesAreReplicated.
@Test
public void waitsUntilEntriesAreReplicated() throws Exception {
conn.tableOperations().create("foo");
Table.ID tableId = Table.ID.of(conn.tableOperations().tableIdMap().get("foo"));
String file1 = "/accumulo/wals/tserver+port/" + UUID.randomUUID(), file2 = "/accumulo/wals/tserver+port/" + UUID.randomUUID();
Status stat = Status.newBuilder().setBegin(0).setEnd(10000).setInfiniteEnd(false).setClosed(false).build();
BatchWriter bw = ReplicationTable.getBatchWriter(conn);
Mutation m = new Mutation(file1);
StatusSection.add(m, tableId, ProtobufUtil.toValue(stat));
bw.addMutation(m);
m = new Mutation(file2);
StatusSection.add(m, tableId, ProtobufUtil.toValue(stat));
bw.addMutation(m);
bw.close();
bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
m = new Mutation(ReplicationSection.getRowPrefix() + file1);
m.put(ReplicationSection.COLF, new Text(tableId.getUtf8()), ProtobufUtil.toValue(stat));
bw.addMutation(m);
m = new Mutation(ReplicationSection.getRowPrefix() + file2);
m.put(ReplicationSection.COLF, new Text(tableId.getUtf8()), ProtobufUtil.toValue(stat));
bw.close();
final AtomicBoolean done = new AtomicBoolean(false);
final AtomicBoolean exception = new AtomicBoolean(false);
final ReplicationOperationsImpl roi = getReplicationOperations();
Thread t = new Thread(new Runnable() {
@Override
public void run() {
try {
roi.drain("foo");
} catch (Exception e) {
log.error("Got error", e);
exception.set(true);
}
done.set(true);
}
});
t.start();
// With the records, we shouldn't be drained
Assert.assertFalse(done.get());
bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
m = new Mutation(ReplicationSection.getRowPrefix() + file1);
m.putDelete(ReplicationSection.COLF, new Text(tableId.getUtf8()));
bw.addMutation(m);
bw.flush();
Assert.assertFalse(done.get());
m = new Mutation(ReplicationSection.getRowPrefix() + file2);
m.putDelete(ReplicationSection.COLF, new Text(tableId.getUtf8()));
bw.addMutation(m);
bw.flush();
bw.close();
// Removing metadata entries doesn't change anything
Assert.assertFalse(done.get());
// Remove the replication entries too
bw = ReplicationTable.getBatchWriter(conn);
m = new Mutation(file1);
m.putDelete(StatusSection.NAME, new Text(tableId.getUtf8()));
bw.addMutation(m);
bw.flush();
Assert.assertFalse(done.get());
m = new Mutation(file2);
m.putDelete(StatusSection.NAME, new Text(tableId.getUtf8()));
bw.addMutation(m);
bw.flush();
try {
t.join(5000);
} catch (InterruptedException e) {
Assert.fail("ReplicationOperations.drain did not complete");
}
// After both metadata and replication
Assert.assertTrue("Drain never finished", done.get());
Assert.assertFalse("Saw unexpectetd exception", exception.get());
}
use of org.apache.accumulo.core.client.BatchWriterConfig in project accumulo by apache.
the class StatusMakerIT method orderRecordsCreatedWithNoCreatedTime.
@Test
public void orderRecordsCreatedWithNoCreatedTime() throws Exception {
String sourceTable = testName.getMethodName();
conn.tableOperations().create(sourceTable);
ReplicationTableUtil.configureMetadataTable(conn, sourceTable);
BatchWriter bw = conn.createBatchWriter(sourceTable, new BatchWriterConfig());
String walPrefix = "hdfs://localhost:8020/accumulo/wals/tserver+port/";
List<String> files = Arrays.asList(walPrefix + UUID.randomUUID(), walPrefix + UUID.randomUUID(), walPrefix + UUID.randomUUID(), walPrefix + UUID.randomUUID());
Map<String, Long> fileToTableId = new HashMap<>();
Status.Builder statBuilder = Status.newBuilder().setBegin(0).setEnd(0).setInfiniteEnd(true).setClosed(true);
Map<String, Long> statuses = new HashMap<>();
long index = 1;
for (String file : files) {
Mutation m = new Mutation(ReplicationSection.getRowPrefix() + file);
m.put(ReplicationSection.COLF, new Text(Long.toString(index)), ProtobufUtil.toValue(statBuilder.build()));
bw.addMutation(m);
fileToTableId.put(file, index);
FileStatus status = EasyMock.mock(FileStatus.class);
EasyMock.expect(status.getModificationTime()).andReturn(index);
EasyMock.replay(status);
statuses.put(file, index);
EasyMock.expect(fs.exists(new Path(file))).andReturn(true);
EasyMock.expect(fs.getFileStatus(new Path(file))).andReturn(status);
index++;
}
EasyMock.replay(fs);
bw.close();
StatusMaker statusMaker = new StatusMaker(conn, fs);
statusMaker.setSourceTableName(sourceTable);
statusMaker.run();
Scanner s = conn.createScanner(sourceTable, Authorizations.EMPTY);
s.setRange(ReplicationSection.getRange());
s.fetchColumnFamily(ReplicationSection.COLF);
Assert.assertEquals(0, Iterables.size(s));
s = ReplicationTable.getScanner(conn);
OrderSection.limit(s);
Iterator<Entry<Key, Value>> iter = s.iterator();
Assert.assertTrue("Found no order records in replication table", iter.hasNext());
Iterator<String> expectedFiles = files.iterator();
Text buff = new Text();
while (expectedFiles.hasNext() && iter.hasNext()) {
String file = expectedFiles.next();
Entry<Key, Value> entry = iter.next();
Assert.assertEquals(file, OrderSection.getFile(entry.getKey(), buff));
OrderSection.getTableId(entry.getKey(), buff);
Assert.assertEquals(fileToTableId.get(file).intValue(), Integer.parseInt(buff.toString()));
Status status = Status.parseFrom(entry.getValue().get());
Assert.assertTrue(status.hasCreatedTime());
Assert.assertEquals((long) statuses.get(file), status.getCreatedTime());
}
Assert.assertFalse("Found more files unexpectedly", expectedFiles.hasNext());
Assert.assertFalse("Found more entries in replication table unexpectedly", iter.hasNext());
s = conn.createScanner(sourceTable, Authorizations.EMPTY);
s.setRange(ReplicationSection.getRange());
s.fetchColumnFamily(ReplicationSection.COLF);
Assert.assertEquals(0, Iterables.size(s));
s = ReplicationTable.getScanner(conn);
s.setRange(ReplicationSection.getRange());
iter = s.iterator();
Assert.assertTrue("Found no stat records in replication table", iter.hasNext());
Collections.sort(files);
expectedFiles = files.iterator();
while (expectedFiles.hasNext() && iter.hasNext()) {
String file = expectedFiles.next();
Entry<Key, Value> entry = iter.next();
Status status = Status.parseFrom(entry.getValue().get());
Assert.assertTrue(status.hasCreatedTime());
Assert.assertEquals((long) statuses.get(file), status.getCreatedTime());
}
Assert.assertFalse("Found more files unexpectedly", expectedFiles.hasNext());
Assert.assertFalse("Found more entries in replication table unexpectedly", iter.hasNext());
}
use of org.apache.accumulo.core.client.BatchWriterConfig in project accumulo by apache.
the class StatusMakerIT method closedMessagesCreateOrderRecords.
@Test
public void closedMessagesCreateOrderRecords() throws Exception {
String sourceTable = testName.getMethodName();
conn.tableOperations().create(sourceTable);
ReplicationTableUtil.configureMetadataTable(conn, sourceTable);
BatchWriter bw = conn.createBatchWriter(sourceTable, new BatchWriterConfig());
String walPrefix = "hdfs://localhost:8020/accumulo/wals/tserver+port/";
List<String> files = Arrays.asList(walPrefix + UUID.randomUUID(), walPrefix + UUID.randomUUID(), walPrefix + UUID.randomUUID(), walPrefix + UUID.randomUUID());
Map<String, Integer> fileToTableId = new HashMap<>();
Status.Builder statBuilder = Status.newBuilder().setBegin(0).setEnd(0).setInfiniteEnd(true).setClosed(true);
int index = 1;
long time = System.currentTimeMillis();
for (String file : files) {
statBuilder.setCreatedTime(time++);
Mutation m = new Mutation(ReplicationSection.getRowPrefix() + file);
m.put(ReplicationSection.COLF, new Text(Integer.toString(index)), ProtobufUtil.toValue(statBuilder.build()));
bw.addMutation(m);
fileToTableId.put(file, index);
index++;
}
bw.close();
StatusMaker statusMaker = new StatusMaker(conn, fs);
statusMaker.setSourceTableName(sourceTable);
statusMaker.run();
Iterator<Entry<Key, Value>> iter;
Iterator<String> expectedFiles;
try (Scanner s = conn.createScanner(sourceTable, Authorizations.EMPTY)) {
s.setRange(ReplicationSection.getRange());
s.fetchColumnFamily(ReplicationSection.COLF);
Assert.assertEquals(0, Iterables.size(s));
}
try (Scanner s = ReplicationTable.getScanner(conn)) {
OrderSection.limit(s);
iter = s.iterator();
Assert.assertTrue("Found no order records in replication table", iter.hasNext());
expectedFiles = files.iterator();
Text buff = new Text();
while (expectedFiles.hasNext() && iter.hasNext()) {
String file = expectedFiles.next();
Entry<Key, Value> entry = iter.next();
Assert.assertEquals(file, OrderSection.getFile(entry.getKey(), buff));
OrderSection.getTableId(entry.getKey(), buff);
Assert.assertEquals(fileToTableId.get(file).intValue(), Integer.parseInt(buff.toString()));
}
}
Assert.assertFalse("Found more files unexpectedly", expectedFiles.hasNext());
Assert.assertFalse("Found more entries in replication table unexpectedly", iter.hasNext());
}
use of org.apache.accumulo.core.client.BatchWriterConfig in project accumulo by apache.
the class UnorderedWorkAssignerReplicationIT method dataReplicatedToCorrectTableWithoutDrain.
@Test
public void dataReplicatedToCorrectTableWithoutDrain() throws Exception {
MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"), ROOT_PASSWORD);
peerCfg.setNumTservers(1);
peerCfg.setInstanceName("peer");
updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
MiniAccumuloClusterImpl peer1Cluster = new MiniAccumuloClusterImpl(peerCfg);
peer1Cluster.start();
try {
Connector connMaster = getConnector();
Connector connPeer = peer1Cluster.getConnector("root", new PasswordToken(ROOT_PASSWORD));
String peerClusterName = "peer";
String peerUserName = "repl";
String peerPassword = "passwd";
// Create a user on the peer for replication to use
connPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
// Configure the credentials we should use to authenticate ourselves to the peer for replication
connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
connMaster.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
// ...peer = AccumuloReplicaSystem,instanceName,zookeepers
connMaster.instanceOperations().setProperty(Property.REPLICATION_PEERS.getKey() + peerClusterName, ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class, AccumuloReplicaSystem.buildConfiguration(peer1Cluster.getInstanceName(), peer1Cluster.getZooKeepers())));
String masterTable1 = "master1", peerTable1 = "peer1", masterTable2 = "master2", peerTable2 = "peer2";
connMaster.tableOperations().create(masterTable1);
String masterTableId1 = connMaster.tableOperations().tableIdMap().get(masterTable1);
Assert.assertNotNull(masterTableId1);
connMaster.tableOperations().create(masterTable2);
String masterTableId2 = connMaster.tableOperations().tableIdMap().get(masterTable2);
Assert.assertNotNull(masterTableId2);
connPeer.tableOperations().create(peerTable1);
String peerTableId1 = connPeer.tableOperations().tableIdMap().get(peerTable1);
Assert.assertNotNull(peerTableId1);
connPeer.tableOperations().create(peerTable2);
String peerTableId2 = connPeer.tableOperations().tableIdMap().get(peerTable2);
Assert.assertNotNull(peerTableId2);
// Give our replication user the ability to write to the tables
connPeer.securityOperations().grantTablePermission(peerUserName, peerTable1, TablePermission.WRITE);
connPeer.securityOperations().grantTablePermission(peerUserName, peerTable2, TablePermission.WRITE);
// Replicate this table to the peerClusterName in a table with the peerTableId table id
connMaster.tableOperations().setProperty(masterTable1, Property.TABLE_REPLICATION.getKey(), "true");
connMaster.tableOperations().setProperty(masterTable1, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId1);
connMaster.tableOperations().setProperty(masterTable2, Property.TABLE_REPLICATION.getKey(), "true");
connMaster.tableOperations().setProperty(masterTable2, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId2);
// Wait for zookeeper updates (configuration) to propagate
sleepUninterruptibly(3, TimeUnit.SECONDS);
// Write some data to table1
BatchWriter bw = connMaster.createBatchWriter(masterTable1, new BatchWriterConfig());
for (int rows = 0; rows < 2500; rows++) {
Mutation m = new Mutation(masterTable1 + rows);
for (int cols = 0; cols < 100; cols++) {
String value = Integer.toString(cols);
m.put(value, "", value);
}
bw.addMutation(m);
}
bw.close();
// Write some data to table2
bw = connMaster.createBatchWriter(masterTable2, new BatchWriterConfig());
for (int rows = 0; rows < 2500; rows++) {
Mutation m = new Mutation(masterTable2 + rows);
for (int cols = 0; cols < 100; cols++) {
String value = Integer.toString(cols);
m.put(value, "", value);
}
bw.addMutation(m);
}
bw.close();
log.info("Wrote all data to master cluster");
while (!ReplicationTable.isOnline(connMaster)) {
Thread.sleep(500);
}
for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
cluster.killProcess(ServerType.TABLET_SERVER, proc);
}
cluster.exec(TabletServer.class);
// Wait until we fully replicated something
boolean fullyReplicated = false;
for (int i = 0; i < 10 && !fullyReplicated; i++) {
sleepUninterruptibly(timeoutFactor * 2, TimeUnit.SECONDS);
try (Scanner s = ReplicationTable.getScanner(connMaster)) {
WorkSection.limit(s);
for (Entry<Key, Value> entry : s) {
Status status = Status.parseFrom(entry.getValue().get());
if (StatusUtil.isFullyReplicated(status)) {
fullyReplicated |= true;
}
}
}
}
Assert.assertNotEquals(0, fullyReplicated);
long countTable = 0l;
// Check a few times
for (int i = 0; i < 10; i++) {
countTable = 0l;
for (Entry<Key, Value> entry : connPeer.createScanner(peerTable1, Authorizations.EMPTY)) {
countTable++;
Assert.assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString().startsWith(masterTable1));
}
log.info("Found {} records in {}", countTable, peerTable1);
if (0 < countTable) {
break;
}
Thread.sleep(2000);
}
Assert.assertTrue("Did not find any records in " + peerTable1 + " on peer", countTable > 0);
for (int i = 0; i < 10; i++) {
countTable = 0l;
for (Entry<Key, Value> entry : connPeer.createScanner(peerTable2, Authorizations.EMPTY)) {
countTable++;
Assert.assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString().startsWith(masterTable2));
}
log.info("Found {} records in {}", countTable, peerTable2);
if (0 < countTable) {
break;
}
Thread.sleep(2000);
}
Assert.assertTrue("Did not find any records in " + peerTable2 + " on peer", countTable > 0);
} finally {
peer1Cluster.stop();
}
}
Aggregations