use of org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterImpl in project accumulo by apache.
the class TableIT method test.
@Test
public void test() throws Exception {
assumeTrue(getClusterType() == ClusterType.MINI);
AccumuloCluster cluster = getCluster();
MiniAccumuloClusterImpl mac = (MiniAccumuloClusterImpl) cluster;
String rootPath = mac.getConfig().getDir().getAbsolutePath();
try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) {
TableOperations to = c.tableOperations();
String tableName = getUniqueNames(1)[0];
to.create(tableName);
VerifyParams params = new VerifyParams(getClientProps(), tableName);
TestIngest.ingest(c, params);
to.flush(tableName, null, null, true);
VerifyIngest.verifyIngest(c, params);
TableId id = TableId.of(to.tableIdMap().get(tableName));
try (Scanner s = c.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
s.setRange(new KeyExtent(id, null, null).toMetaRange());
s.fetchColumnFamily(DataFileColumnFamily.NAME);
assertTrue(Iterators.size(s.iterator()) > 0);
FileSystem fs = getCluster().getFileSystem();
assertTrue(fs.listStatus(new Path(rootPath + "/accumulo/tables/" + id)).length > 0);
to.delete(tableName);
assertEquals(0, Iterators.size(s.iterator()));
try {
assertEquals(0, fs.listStatus(new Path(rootPath + "/accumulo/tables/" + id)).length);
} catch (FileNotFoundException ex) {
// that's fine, too
}
assertNull(to.tableIdMap().get(tableName));
to.create(tableName);
TestIngest.ingest(c, params);
VerifyIngest.verifyIngest(c, params);
to.delete(tableName);
}
}
}
use of org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterImpl in project accumulo by apache.
the class WALSunnyDayIT method test.
@Test
public void test() throws Exception {
MiniAccumuloClusterImpl mac = getCluster();
MiniAccumuloClusterControl control = mac.getClusterControl();
control.stop(GARBAGE_COLLECTOR);
ServerContext context = getServerContext();
try (AccumuloClient c = Accumulo.newClient().from(getClientProperties()).build()) {
String tableName = getUniqueNames(1)[0];
c.tableOperations().create(tableName);
writeSomeData(c, tableName, 1, 1);
// wal markers are added lazily
Map<String, WalState> wals = getWALsAndAssertCount(context, 2);
assertEquals("all WALs should be in use", 2, countInUse(wals.values()));
// roll log, get a new next
writeSomeData(c, tableName, 1001, 50);
Map<String, WalState> walsAfterRoll = getWALsAndAssertCount(context, 3);
assertTrue("new WALs should be a superset of the old WALs", walsAfterRoll.keySet().containsAll(wals.keySet()));
assertEquals("all WALs should be in use", 3, countInUse(walsAfterRoll.values()));
// flush the tables
for (String table : new String[] { tableName, MetadataTable.NAME, RootTable.NAME }) {
c.tableOperations().flush(table, null, null, true);
}
sleepUninterruptibly(1, TimeUnit.SECONDS);
// rolled WAL is no longer in use, but needs to be GC'd
Map<String, WalState> walsAfterflush = getWALsAndAssertCount(context, 3);
assertEquals("inUse should be 2", 2, countInUse(walsAfterflush.values()));
// let the GC run for a little bit
control.start(GARBAGE_COLLECTOR);
sleepUninterruptibly(5, TimeUnit.SECONDS);
// make sure the unused WAL goes away
getWALsAndAssertCount(context, 2);
control.stop(GARBAGE_COLLECTOR);
// restart the tserver, but don't run recovery on all tablets
control.stop(TABLET_SERVER);
// this delays recovery on the normal tables
assertEquals(0, cluster.exec(SetGoalState.class, "SAFE_MODE").getProcess().waitFor());
control.start(TABLET_SERVER);
// wait for the metadata table to go back online
getRecoveryMarkers(c);
// allow a little time for the manager to notice ASSIGNED_TO_DEAD_SERVER tablets
sleepUninterruptibly(5, TimeUnit.SECONDS);
Map<KeyExtent, List<String>> markers = getRecoveryMarkers(c);
// log.debug("markers " + markers);
assertEquals("one tablet should have markers", 1, markers.size());
assertEquals("tableId of the keyExtent should be 1", "1", markers.keySet().iterator().next().tableId().canonical());
// put some data in the WAL
assertEquals(0, cluster.exec(SetGoalState.class, "NORMAL").getProcess().waitFor());
verifySomeData(c, tableName, 1001 * 50 + 1);
writeSomeData(c, tableName, 100, 100);
Map<String, WalState> walsAfterRestart = getWALsAndAssertCount(context, 4);
// log.debug("wals after " + walsAfterRestart);
assertEquals("used WALs after restart should be 4", 4, countInUse(walsAfterRestart.values()));
control.start(GARBAGE_COLLECTOR);
sleepUninterruptibly(5, TimeUnit.SECONDS);
Map<String, WalState> walsAfterRestartAndGC = getWALsAndAssertCount(context, 2);
assertEquals("logs in use should be 2", 2, countInUse(walsAfterRestartAndGC.values()));
}
}
use of org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterImpl in project accumulo by apache.
the class MiniClusterHarness method create.
public MiniAccumuloClusterImpl create(String testClassName, String testMethodName, AuthenticationToken token, MiniClusterConfigurationCallback configCallback, TestingKdc kdc) throws Exception {
requireNonNull(token);
checkArgument(token instanceof PasswordToken || token instanceof KerberosToken, "A PasswordToken or KerberosToken is required");
String rootPasswd;
if (token instanceof PasswordToken) {
rootPasswd = new String(((PasswordToken) token).getPassword(), UTF_8);
} else {
rootPasswd = UUID.randomUUID().toString();
}
File baseDir = AccumuloClusterHarness.createTestDir(testClassName + "_" + testMethodName);
MiniAccumuloConfigImpl cfg = new MiniAccumuloConfigImpl(baseDir, rootPasswd);
// Enable native maps by default
cfg.setNativeLibPaths(NativeMapIT.nativeMapLocation().getAbsolutePath());
cfg.setProperty(Property.TSERV_NATIVEMAP_ENABLED, Boolean.TRUE.toString());
Configuration coreSite = new Configuration(false);
// Setup SSL and credential providers if the properties request such
configureForEnvironment(cfg, AccumuloClusterHarness.getSslDir(baseDir), coreSite, kdc);
// Invoke the callback for tests to configure MAC before it starts
configCallback.configureMiniCluster(cfg, coreSite);
MiniAccumuloClusterImpl miniCluster = new MiniAccumuloClusterImpl(cfg);
// classpath)
if (coreSite.size() > 0) {
File csFile = new File(miniCluster.getConfig().getConfDir(), "core-site.xml");
if (csFile.exists())
throw new RuntimeException(csFile + " already exist");
OutputStream out = new BufferedOutputStream(new FileOutputStream(new File(miniCluster.getConfig().getConfDir(), "core-site.xml")));
coreSite.writeXml(out);
out.close();
}
return miniCluster;
}
use of org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterImpl in project accumulo by apache.
the class CloneTestIT method testDeleteClone.
@Test
public void testDeleteClone() throws Exception {
String[] tableNames = getUniqueNames(3);
String table1 = tableNames[0];
String table2 = tableNames[1];
String table3 = tableNames[2];
try (AccumuloClient c = Accumulo.newClient().from(getClientProps()).build()) {
AccumuloCluster cluster = getCluster();
assumeTrue(cluster instanceof MiniAccumuloClusterImpl);
MiniAccumuloClusterImpl mac = (MiniAccumuloClusterImpl) cluster;
String rootPath = mac.getConfig().getDir().getAbsolutePath();
// verify that deleting a new table removes the files
c.tableOperations().create(table3);
writeData(table3, c).close();
c.tableOperations().flush(table3, null, null, true);
// check for files
FileSystem fs = getCluster().getFileSystem();
String id = c.tableOperations().tableIdMap().get(table3);
FileStatus[] status = fs.listStatus(new Path(rootPath + "/accumulo/tables/" + id));
assertTrue(status.length > 0);
// verify disk usage
List<DiskUsage> diskUsage = c.tableOperations().getDiskUsage(Collections.singleton(table3));
assertEquals(1, diskUsage.size());
assertTrue(diskUsage.get(0).getUsage() > 100);
// delete the table
c.tableOperations().delete(table3);
// verify its gone from the file system
Path tablePath = new Path(rootPath + "/accumulo/tables/" + id);
if (fs.exists(tablePath)) {
status = fs.listStatus(tablePath);
assertTrue(status == null || status.length == 0);
}
c.tableOperations().create(table1);
writeDataAndClone(c, table1, table2);
// delete source table, should not affect clone
c.tableOperations().delete(table1);
checkData(table2, c);
c.tableOperations().compact(table2, null, null, true, true);
checkData(table2, c);
c.tableOperations().delete(table2);
}
}
use of org.apache.accumulo.miniclusterImpl.MiniAccumuloClusterImpl in project accumulo by apache.
the class ExistingMacIT method testExistingInstance.
@Test
public void testExistingInstance() throws Exception {
AccumuloClient client = getCluster().createAccumuloClient("root", new PasswordToken(ROOT_PASSWORD));
client.tableOperations().create("table1");
try (BatchWriter bw = client.createBatchWriter("table1")) {
Mutation m1 = new Mutation("00081");
m1.put("math", "sqroot", "9");
m1.put("math", "sq", "6560");
bw.addMutation(m1);
}
client.tableOperations().flush("table1", null, null, true);
// TODO use constants
client.tableOperations().flush(MetadataTable.NAME, null, null, true);
client.tableOperations().flush(RootTable.NAME, null, null, true);
Set<Entry<ServerType, Collection<ProcessReference>>> procs = getCluster().getProcesses().entrySet();
for (Entry<ServerType, Collection<ProcessReference>> entry : procs) {
if (entry.getKey() == ServerType.ZOOKEEPER)
continue;
for (ProcessReference pr : entry.getValue()) getCluster().killProcess(entry.getKey(), pr);
}
final DefaultConfiguration defaultConfig = DefaultConfiguration.getInstance();
final long zkTimeout = ConfigurationTypeHelper.getTimeInMillis(getCluster().getConfig().getSiteConfig().get(Property.INSTANCE_ZK_TIMEOUT.getKey()));
ZooReaderWriter zrw = new ZooReaderWriter(getCluster().getZooKeepers(), (int) zkTimeout, defaultConfig.get(Property.INSTANCE_SECRET));
final String zInstanceRoot = Constants.ZROOT + "/" + client.instanceOperations().getInstanceId();
while (!AccumuloStatus.isAccumuloOffline(zrw, zInstanceRoot)) {
log.debug("Accumulo services still have their ZK locks held");
Thread.sleep(1000);
}
File hadoopConfDir = createTestDir(ExistingMacIT.class.getSimpleName() + "_hadoop_conf");
FileUtils.deleteQuietly(hadoopConfDir);
assertTrue(hadoopConfDir.mkdirs());
createEmptyConfig(new File(hadoopConfDir, "core-site.xml"));
createEmptyConfig(new File(hadoopConfDir, "hdfs-site.xml"));
File testDir2 = createTestDir(ExistingMacIT.class.getSimpleName() + "_2");
FileUtils.deleteQuietly(testDir2);
MiniAccumuloConfigImpl macConfig2 = new MiniAccumuloConfigImpl(testDir2, "notused");
macConfig2.useExistingInstance(new File(getCluster().getConfig().getConfDir(), "accumulo.properties"), hadoopConfDir);
MiniAccumuloClusterImpl accumulo2 = new MiniAccumuloClusterImpl(macConfig2);
accumulo2.start();
client = accumulo2.createAccumuloClient("root", new PasswordToken(ROOT_PASSWORD));
try (Scanner scanner = client.createScanner("table1", Authorizations.EMPTY)) {
int sum = 0;
for (Entry<Key, Value> entry : scanner) {
sum += Integer.parseInt(entry.getValue().toString());
}
assertEquals(6569, sum);
}
accumulo2.stop();
}
Aggregations