use of org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl in project accumulo by apache.
the class MultiInstanceReplicationIT method dataReplicatedToCorrectTable.
@Test
public void dataReplicatedToCorrectTable() throws Exception {
MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"), ROOT_PASSWORD);
peerCfg.setNumTservers(1);
peerCfg.setInstanceName("peer");
peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
MiniAccumuloClusterImpl peer1Cluster = new MiniAccumuloClusterImpl(peerCfg);
peer1Cluster.start();
try (AccumuloClient clientManager = Accumulo.newClient().from(getClientProperties()).build();
AccumuloClient clientPeer = peer1Cluster.createAccumuloClient("root", new PasswordToken(ROOT_PASSWORD))) {
String peerClusterName = "peer";
String peerUserName = "peer", peerPassword = "foo";
// Create local user
clientPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
clientManager.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
clientManager.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
// ...peer = AccumuloReplicaSystem,instanceName,zookeepers
clientManager.instanceOperations().setProperty(Property.REPLICATION_PEERS.getKey() + peerClusterName, ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class, AccumuloReplicaSystem.buildConfiguration(peer1Cluster.getInstanceName(), peer1Cluster.getZooKeepers())));
String managerTable1 = "manager1", peerTable1 = "peer1", managerTable2 = "manager2", peerTable2 = "peer2";
// Create tables
clientPeer.tableOperations().create(peerTable1, new NewTableConfiguration());
String peerTableId1 = clientPeer.tableOperations().tableIdMap().get(peerTable1);
assertNotNull(peerTableId1);
clientPeer.tableOperations().create(peerTable2, new NewTableConfiguration());
String peerTableId2 = clientPeer.tableOperations().tableIdMap().get(peerTable2);
assertNotNull(peerTableId2);
Map<String, String> props1 = new HashMap<>();
props1.put(Property.TABLE_REPLICATION.getKey(), "true");
props1.put(Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId1);
clientManager.tableOperations().create(managerTable1, new NewTableConfiguration().setProperties(props1));
String managerTableId1 = clientManager.tableOperations().tableIdMap().get(managerTable1);
assertNotNull(managerTableId1);
Map<String, String> props2 = new HashMap<>();
props2.put(Property.TABLE_REPLICATION.getKey(), "true");
props2.put(Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId2);
clientManager.tableOperations().create(managerTable2, new NewTableConfiguration().setProperties(props2));
String managerTableId2 = clientManager.tableOperations().tableIdMap().get(managerTable2);
assertNotNull(managerTableId2);
// Give our replication user the ability to write to the tables
clientPeer.securityOperations().grantTablePermission(peerUserName, peerTable1, TablePermission.WRITE);
clientPeer.securityOperations().grantTablePermission(peerUserName, peerTable2, TablePermission.WRITE);
// Write some data to table1
long managerTable1Records = 0L;
try (BatchWriter bw = clientManager.createBatchWriter(managerTable1)) {
for (int rows = 0; rows < 2500; rows++) {
Mutation m = new Mutation(managerTable1 + rows);
for (int cols = 0; cols < 100; cols++) {
String value = Integer.toString(cols);
m.put(value, "", value);
managerTable1Records++;
}
bw.addMutation(m);
}
}
// Write some data to table2
long managerTable2Records = 0L;
try (BatchWriter bw = clientManager.createBatchWriter(managerTable2)) {
for (int rows = 0; rows < 2500; rows++) {
Mutation m = new Mutation(managerTable2 + rows);
for (int cols = 0; cols < 100; cols++) {
String value = Integer.toString(cols);
m.put(value, "", value);
managerTable2Records++;
}
bw.addMutation(m);
}
}
log.info("Wrote all data to manager cluster");
Set<String> filesFor1 = clientManager.replicationOperations().referencedFiles(managerTable1), filesFor2 = clientManager.replicationOperations().referencedFiles(managerTable2);
log.info("Files to replicate for table1: " + filesFor1);
log.info("Files to replicate for table2: " + filesFor2);
// Restart the tserver to force a close on the WAL
for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
cluster.killProcess(ServerType.TABLET_SERVER, proc);
}
cluster.exec(TabletServer.class);
log.info("Restarted the tserver");
// Read the data -- the tserver is back up and running
Iterators.size(clientManager.createScanner(managerTable1, Authorizations.EMPTY).iterator());
while (!ReplicationTable.isOnline(clientManager)) {
log.info("Replication table still offline, waiting");
Thread.sleep(5000);
}
// Wait for both tables to be replicated
log.info("Waiting for {} for {}", filesFor1, managerTable1);
clientManager.replicationOperations().drain(managerTable1, filesFor1);
log.info("Waiting for {} for {}", filesFor2, managerTable2);
clientManager.replicationOperations().drain(managerTable2, filesFor2);
long countTable = 0L;
try (var scanner = clientPeer.createScanner(peerTable1, Authorizations.EMPTY)) {
for (Entry<Key, Value> entry : scanner) {
countTable++;
assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString().startsWith(managerTable1));
}
}
log.info("Found {} records in {}", countTable, peerTable1);
assertEquals(managerTable1Records, countTable);
countTable = 0L;
try (var scanner = clientPeer.createScanner(peerTable2, Authorizations.EMPTY)) {
for (Entry<Key, Value> entry : scanner) {
countTable++;
assertTrue("Found unexpected key-value" + entry.getKey().toStringNoTruncate() + " " + entry.getValue(), entry.getKey().getRow().toString().startsWith(managerTable2));
}
}
log.info("Found {} records in {}", countTable, peerTable2);
assertEquals(managerTable2Records, countTable);
} finally {
peer1Cluster.stop();
}
}
use of org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl in project accumulo by apache.
the class MultiInstanceReplicationIT method dataWasReplicatedToThePeerWithoutDrain.
@Test
public void dataWasReplicatedToThePeerWithoutDrain() throws Exception {
MiniAccumuloConfigImpl peerCfg = new MiniAccumuloConfigImpl(createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"), ROOT_PASSWORD);
peerCfg.setNumTservers(1);
peerCfg.setInstanceName("peer");
peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
MiniAccumuloClusterImpl peerCluster = new MiniAccumuloClusterImpl(peerCfg);
peerCluster.start();
try (AccumuloClient clientManager = Accumulo.newClient().from(getClientProperties()).build();
AccumuloClient clientPeer = peerCluster.createAccumuloClient("root", new PasswordToken(ROOT_PASSWORD))) {
String peerUserName = "repl";
String peerPassword = "passwd";
// Create a user on the peer for replication to use
clientPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
String peerClusterName = "peer";
// ...peer = AccumuloReplicaSystem,instanceName,zookeepers
clientManager.instanceOperations().setProperty(Property.REPLICATION_PEERS.getKey() + peerClusterName, ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class, AccumuloReplicaSystem.buildConfiguration(peerCluster.getInstanceName(), peerCluster.getZooKeepers())));
// Configure the credentials we should use to authenticate ourselves to the peer for
// replication
clientManager.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
clientManager.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
String managerTable = "manager", peerTable = "peer";
clientPeer.tableOperations().create(peerTable, new NewTableConfiguration());
String peerTableId = clientPeer.tableOperations().tableIdMap().get(peerTable);
assertNotNull(peerTableId);
// Give our replication user the ability to write to the table
clientPeer.securityOperations().grantTablePermission(peerUserName, peerTable, TablePermission.WRITE);
Map<String, String> props = new HashMap<>();
props.put(Property.TABLE_REPLICATION.getKey(), "true");
// Replicate this table to the peerClusterName in a table with the peerTableId table id
props.put(Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId);
clientManager.tableOperations().create(managerTable, new NewTableConfiguration().setProperties(props));
String managerTableId = clientManager.tableOperations().tableIdMap().get(managerTable);
assertNotNull(managerTableId);
// Write some data to table1
try (BatchWriter bw = clientManager.createBatchWriter(managerTable)) {
for (int rows = 0; rows < 5000; rows++) {
Mutation m = new Mutation(Integer.toString(rows));
for (int cols = 0; cols < 100; cols++) {
String value = Integer.toString(cols);
m.put(value, "", value);
}
bw.addMutation(m);
}
}
log.info("Wrote all data to manager cluster");
Set<String> files = clientManager.replicationOperations().referencedFiles(managerTable);
log.info("Files to replicate:" + files);
for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
cluster.killProcess(ServerType.TABLET_SERVER, proc);
}
cluster.exec(TabletServer.class);
while (!ReplicationTable.isOnline(clientManager)) {
log.info("Replication table still offline, waiting");
Thread.sleep(5000);
}
Iterators.size(clientManager.createScanner(managerTable, Authorizations.EMPTY).iterator());
try (var scanner = ReplicationTable.getScanner(clientManager)) {
for (Entry<Key, Value> kv : scanner) {
log.debug("{} {}", kv.getKey().toStringNoTruncate(), ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
}
}
clientManager.replicationOperations().drain(managerTable, files);
try (Scanner manager = clientManager.createScanner(managerTable, Authorizations.EMPTY);
Scanner peer = clientPeer.createScanner(peerTable, Authorizations.EMPTY)) {
Iterator<Entry<Key, Value>> managerIter = manager.iterator(), peerIter = peer.iterator();
while (managerIter.hasNext() && peerIter.hasNext()) {
Entry<Key, Value> managerEntry = managerIter.next(), peerEntry = peerIter.next();
assertEquals(peerEntry.getKey() + " was not equal to " + peerEntry.getKey(), 0, managerEntry.getKey().compareTo(peerEntry.getKey(), PartialKey.ROW_COLFAM_COLQUAL_COLVIS));
assertEquals(managerEntry.getValue(), peerEntry.getValue());
}
assertFalse("Had more data to read from the manager", managerIter.hasNext());
assertFalse("Had more data to read from the peer", peerIter.hasNext());
}
} finally {
peerCluster.stop();
}
}
use of org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl in project accumulo by apache.
the class CyclicReplicationIT method dataIsNotOverReplicated.
@Test
public void dataIsNotOverReplicated() throws Exception {
File manager1Dir = createTestDir("manager1"), manager2Dir = createTestDir("manager2");
String password = "password";
MiniAccumuloConfigImpl manager1Cfg;
MiniAccumuloClusterImpl manager1Cluster;
while (true) {
manager1Cfg = new MiniAccumuloConfigImpl(manager1Dir, password);
manager1Cfg.setNumTservers(1);
manager1Cfg.setInstanceName("manager1");
// Set up SSL if needed
ConfigurableMacBase.configureForEnvironment(manager1Cfg, ConfigurableMacBase.getSslDir(manager1Dir));
manager1Cfg.setProperty(Property.REPLICATION_NAME, manager1Cfg.getInstanceName());
manager1Cfg.setProperty(Property.TSERV_WAL_MAX_SIZE, "5M");
manager1Cfg.setProperty(Property.REPLICATION_THREADCHECK, "5m");
manager1Cfg.setProperty(Property.REPLICATION_WORK_ASSIGNMENT_SLEEP, "1s");
manager1Cfg.setProperty(Property.MANAGER_REPLICATION_SCAN_INTERVAL, "1s");
manager1Cluster = new MiniAccumuloClusterImpl(manager1Cfg);
setCoreSite(manager1Cluster);
try {
manager1Cluster.start();
break;
} catch (ZooKeeperBindException e) {
log.warn("Failed to start ZooKeeper on {}, will retry", manager1Cfg.getZooKeeperPort());
}
}
MiniAccumuloConfigImpl manager2Cfg;
MiniAccumuloClusterImpl manager2Cluster;
while (true) {
manager2Cfg = new MiniAccumuloConfigImpl(manager2Dir, password);
manager2Cfg.setNumTservers(1);
manager2Cfg.setInstanceName("manager2");
// Set up SSL if needed. Need to share the same SSL truststore as manager1
this.updatePeerConfigFromPrimary(manager1Cfg, manager2Cfg);
manager2Cfg.setProperty(Property.REPLICATION_NAME, manager2Cfg.getInstanceName());
manager2Cfg.setProperty(Property.TSERV_WAL_MAX_SIZE, "5M");
manager2Cfg.setProperty(Property.REPLICATION_THREADCHECK, "5m");
manager2Cfg.setProperty(Property.REPLICATION_WORK_ASSIGNMENT_SLEEP, "1s");
manager2Cfg.setProperty(Property.MANAGER_REPLICATION_SCAN_INTERVAL, "1s");
manager2Cluster = new MiniAccumuloClusterImpl(manager2Cfg);
setCoreSite(manager2Cluster);
try {
manager2Cluster.start();
break;
} catch (ZooKeeperBindException e) {
log.warn("Failed to start ZooKeeper on {}, will retry", manager2Cfg.getZooKeeperPort());
}
}
try {
AccumuloClient clientManager1 = manager1Cluster.createAccumuloClient("root", new PasswordToken(password)), clientManager2 = manager2Cluster.createAccumuloClient("root", new PasswordToken(password));
String manager1UserName = "manager1", manager1Password = "foo";
String manager2UserName = "manager2", manager2Password = "bar";
String manager1Table = manager1Cluster.getInstanceName(), manager2Table = manager2Cluster.getInstanceName();
clientManager1.securityOperations().createLocalUser(manager1UserName, new PasswordToken(manager1Password));
clientManager2.securityOperations().createLocalUser(manager2UserName, new PasswordToken(manager2Password));
// Configure the credentials we should use to authenticate ourselves to the peer for
// replication
clientManager1.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + manager2Cluster.getInstanceName(), manager2UserName);
clientManager1.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + manager2Cluster.getInstanceName(), manager2Password);
clientManager2.instanceOperations().setProperty(Property.REPLICATION_PEER_USER.getKey() + manager1Cluster.getInstanceName(), manager1UserName);
clientManager2.instanceOperations().setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + manager1Cluster.getInstanceName(), manager1Password);
clientManager1.instanceOperations().setProperty(Property.REPLICATION_PEERS.getKey() + manager2Cluster.getInstanceName(), ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class, AccumuloReplicaSystem.buildConfiguration(manager2Cluster.getInstanceName(), manager2Cluster.getZooKeepers())));
clientManager2.instanceOperations().setProperty(Property.REPLICATION_PEERS.getKey() + manager1Cluster.getInstanceName(), ReplicaSystemFactory.getPeerConfigurationValue(AccumuloReplicaSystem.class, AccumuloReplicaSystem.buildConfiguration(manager1Cluster.getInstanceName(), manager1Cluster.getZooKeepers())));
clientManager1.tableOperations().create(manager1Table, new NewTableConfiguration().withoutDefaultIterators());
String manager1TableId = clientManager1.tableOperations().tableIdMap().get(manager1Table);
assertNotNull(manager1TableId);
clientManager2.tableOperations().create(manager2Table, new NewTableConfiguration().withoutDefaultIterators());
String manager2TableId = clientManager2.tableOperations().tableIdMap().get(manager2Table);
assertNotNull(manager2TableId);
// Replicate manager1 in the manager1 cluster to manager2 in the manager2 cluster
clientManager1.tableOperations().setProperty(manager1Table, Property.TABLE_REPLICATION.getKey(), "true");
clientManager1.tableOperations().setProperty(manager1Table, Property.TABLE_REPLICATION_TARGET.getKey() + manager2Cluster.getInstanceName(), manager2TableId);
// Replicate manager2 in the manager2 cluster to manager1 in the manager2 cluster
clientManager2.tableOperations().setProperty(manager2Table, Property.TABLE_REPLICATION.getKey(), "true");
clientManager2.tableOperations().setProperty(manager2Table, Property.TABLE_REPLICATION_TARGET.getKey() + manager1Cluster.getInstanceName(), manager1TableId);
// Give our replication user the ability to write to the respective table
clientManager1.securityOperations().grantTablePermission(manager1UserName, manager1Table, TablePermission.WRITE);
clientManager2.securityOperations().grantTablePermission(manager2UserName, manager2Table, TablePermission.WRITE);
IteratorSetting summingCombiner = new IteratorSetting(50, SummingCombiner.class);
SummingCombiner.setEncodingType(summingCombiner, Type.STRING);
SummingCombiner.setCombineAllColumns(summingCombiner, true);
// Set a combiner on both instances that will sum multiple values
// We can use this to verify that the mutation was not sent multiple times
clientManager1.tableOperations().attachIterator(manager1Table, summingCombiner);
clientManager2.tableOperations().attachIterator(manager2Table, summingCombiner);
// Write a single entry
try (BatchWriter bw = clientManager1.createBatchWriter(manager1Table)) {
Mutation m = new Mutation("row");
m.put("count", "", "1");
bw.addMutation(m);
}
Set<String> files = clientManager1.replicationOperations().referencedFiles(manager1Table);
log.info("Found {} that need replication from manager1", files);
// Kill and restart the tserver to close the WAL on manager1
for (ProcessReference proc : manager1Cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
manager1Cluster.killProcess(ServerType.TABLET_SERVER, proc);
}
manager1Cluster.exec(TabletServer.class);
log.info("Restarted tserver on manager1");
// Try to avoid ACCUMULO-2964
Thread.sleep(1000);
// Sanity check that the element is there on manager1
Entry<Key, Value> entry;
try (Scanner s = clientManager1.createScanner(manager1Table, Authorizations.EMPTY)) {
entry = Iterables.getOnlyElement(s);
assertEquals("1", entry.getValue().toString());
// Wait for this table to replicate
clientManager1.replicationOperations().drain(manager1Table, files);
Thread.sleep(5000);
}
// Check that the element made it to manager2 only once
try (Scanner s = clientManager2.createScanner(manager2Table, Authorizations.EMPTY)) {
entry = Iterables.getOnlyElement(s);
assertEquals("1", entry.getValue().toString());
// Wait for manager2 to finish replicating it back
files = clientManager2.replicationOperations().referencedFiles(manager2Table);
// Kill and restart the tserver to close the WAL on manager2
for (ProcessReference proc : manager2Cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
manager2Cluster.killProcess(ServerType.TABLET_SERVER, proc);
}
manager2Cluster.exec(TabletServer.class);
// Try to avoid ACCUMULO-2964
Thread.sleep(1000);
}
// Check that the element made it to manager2 only once
try (Scanner s = clientManager2.createScanner(manager2Table, Authorizations.EMPTY)) {
entry = Iterables.getOnlyElement(s);
assertEquals("1", entry.getValue().toString());
clientManager2.replicationOperations().drain(manager2Table, files);
Thread.sleep(5000);
}
// Verify that the entry wasn't sent back to manager1
try (Scanner s = clientManager1.createScanner(manager1Table, Authorizations.EMPTY)) {
entry = Iterables.getOnlyElement(s);
assertEquals("1", entry.getValue().toString());
}
} finally {
manager1Cluster.stop();
manager2Cluster.stop();
}
}
use of org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl in project accumulo by apache.
the class ConfigurableMacBase method createMiniAccumulo.
@SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", justification = "path provided by test")
private void createMiniAccumulo() throws Exception {
// createTestDir will give us a empty directory, we don't need to clean it up ourselves
File baseDir = createTestDir(this.getClass().getName() + "_" + this.testName.getMethodName());
MiniAccumuloConfigImpl cfg = new MiniAccumuloConfigImpl(baseDir, ROOT_PASSWORD);
File nativePathInDevTree = NativeMapIT.nativeMapLocation();
File nativePathInMapReduce = new File(System.getProperty("user.dir"));
cfg.setNativeLibPaths(nativePathInDevTree.getAbsolutePath(), nativePathInMapReduce.toString());
Configuration coreSite = new Configuration(false);
cfg.setProperty(Property.TSERV_NATIVEMAP_ENABLED, Boolean.TRUE.toString());
configure(cfg, coreSite);
configureForEnvironment(cfg, getSslDir(baseDir));
if (Boolean.parseBoolean(cfg.getSiteConfig().get(Property.TSERV_NATIVEMAP_ENABLED.getKey()))) {
NativeMapLoader.loadForTest(List.of(nativePathInDevTree, nativePathInMapReduce), () -> {
throw new IllegalStateException("Native maps were configured, but not available");
});
}
cluster = new MiniAccumuloClusterImpl(cfg);
if (coreSite.size() > 0) {
File csFile = new File(cluster.getConfig().getConfDir(), "core-site.xml");
if (csFile.exists()) {
coreSite.addResource(new Path(csFile.getAbsolutePath()));
}
File tmp = new File(csFile.getAbsolutePath() + ".tmp");
OutputStream out = new BufferedOutputStream(new FileOutputStream(tmp));
coreSite.writeXml(out);
out.close();
assertTrue(tmp.renameTo(csFile));
}
beforeClusterStart(cfg);
}
use of org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl in project accumulo by apache.
the class KerberosRenewalIT method startMac.
@Before
public void startMac() throws Exception {
MiniClusterHarness harness = new MiniClusterHarness();
mac = harness.create(this, new PasswordToken("unused"), kdc, new MiniClusterConfigurationCallback() {
@Override
public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration coreSite) {
Map<String, String> site = cfg.getSiteConfig();
site.put(Property.INSTANCE_ZK_TIMEOUT.getKey(), "15s");
// Reduce the period just to make sure we trigger renewal fast
site.put(Property.GENERAL_KERBEROS_RENEWAL_PERIOD.getKey(), "5s");
cfg.setSiteConfig(site);
cfg.setClientProperty(ClientProperty.INSTANCE_ZOOKEEPERS_TIMEOUT, "15s");
}
});
mac.getConfig().setNumTservers(1);
mac.start();
// Enabled kerberos auth
Configuration conf = new Configuration(false);
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
UserGroupInformation.setConfiguration(conf);
}
Aggregations