use of org.apache.accumulo.core.client.AccumuloClient in project accumulo by apache.
the class ExternalCompaction_3_IT method testCoordinatorRestartsDuringCompaction.
@Test
public void testCoordinatorRestartsDuringCompaction() throws Exception {
getCluster().getClusterControl().startCoordinator(CompactionCoordinator.class);
getCluster().getClusterControl().startCompactors(ExternalDoNothingCompactor.class, 1, QUEUE2);
String table1 = this.getUniqueNames(1)[0];
try (AccumuloClient client = Accumulo.newClient().from(getCluster().getClientProperties()).build()) {
createTable(client, table1, "cs2", 2);
writeData(client, table1);
compact(client, table1, 2, QUEUE2, false);
TableId tid = getCluster().getServerContext().getTableId(table1);
// Wait for the compaction to start by waiting for 1 external compaction column
Set<ExternalCompactionId> ecids = waitForCompactionStartAndReturnEcids(getCluster().getServerContext(), tid);
// Stop the Coordinator
getCluster().getClusterControl().stop(ServerType.COMPACTION_COORDINATOR);
// Restart the coordinator while the compaction is running
getCluster().getClusterControl().startCoordinator(CompactionCoordinator.class);
// Confirm compaction is still running
int matches = 0;
while (matches == 0) {
TExternalCompactionList running = getRunningCompactions(getCluster().getServerContext());
if (running.getCompactions() != null) {
for (ExternalCompactionId ecid : ecids) {
TExternalCompaction tec = running.getCompactions().get(ecid.canonical());
if (tec != null && tec.getUpdates() != null && !tec.getUpdates().isEmpty()) {
matches++;
assertEquals(TCompactionState.IN_PROGRESS, getLastState(tec));
}
}
}
UtilWaitThread.sleep(250);
}
assertTrue(matches > 0);
// We need to cancel the compaction or delete the table here because we initiate a user
// compaction above in the test. Even though the external compaction was cancelled
// because we split the table, FaTE will continue to queue up a compaction
client.tableOperations().cancelCompaction(table1);
}
}
use of org.apache.accumulo.core.client.AccumuloClient in project accumulo by apache.
the class ExternalCompaction_3_IT method testMergeCancelsExternalCompaction.
@Test
public void testMergeCancelsExternalCompaction() throws Exception {
getCluster().getClusterControl().startCoordinator(CompactionCoordinator.class);
getCluster().getClusterControl().startCompactors(ExternalDoNothingCompactor.class, 1, QUEUE1);
String table1 = this.getUniqueNames(1)[0];
try (AccumuloClient client = Accumulo.newClient().from(getCluster().getClientProperties()).build()) {
createTable(client, table1, "cs1", 2);
// set compaction ratio to 1 so that majc occurs naturally, not user compaction
// user compaction blocks merge
client.tableOperations().setProperty(table1, Property.TABLE_MAJC_RATIO.toString(), "1.0");
// cause multiple rfiles to be created
writeData(client, table1);
writeData(client, table1);
writeData(client, table1);
writeData(client, table1);
TableId tid = getCluster().getServerContext().getTableId(table1);
// Wait for the compaction to start by waiting for 1 external compaction column
Set<ExternalCompactionId> ecids = waitForCompactionStartAndReturnEcids(getCluster().getServerContext(), tid);
var md = new ArrayList<TabletMetadata>();
try (TabletsMetadata tm = getCluster().getServerContext().getAmple().readTablets().forTable(tid).fetch(ColumnType.PREV_ROW).build()) {
tm.forEach(t -> md.add(t));
assertEquals(2, md.size());
}
// Merge - blocking operation
Text start = md.get(0).getPrevEndRow();
Text end = md.get(1).getEndRow();
client.tableOperations().merge(table1, start, end);
confirmCompactionCompleted(getCluster().getServerContext(), ecids, TCompactionState.CANCELLED);
// ensure compaction ids were deleted by merge operation from metadata table
try (TabletsMetadata tm = getCluster().getServerContext().getAmple().readTablets().forTable(tid).fetch(ColumnType.ECOMP).build()) {
Set<ExternalCompactionId> ecids2 = tm.stream().flatMap(t -> t.getExternalCompactions().keySet().stream()).collect(Collectors.toSet());
// keep checking until test times out
while (!Collections.disjoint(ecids, ecids2)) {
UtilWaitThread.sleep(25);
ecids2 = tm.stream().flatMap(t -> t.getExternalCompactions().keySet().stream()).collect(Collectors.toSet());
}
}
// We need to cancel the compaction or delete the table here because we initiate a user
// compaction above in the test. Even though the external compaction was cancelled
// because we split the table, FaTE will continue to queue up a compaction
client.tableOperations().delete(table1);
}
}
use of org.apache.accumulo.core.client.AccumuloClient in project accumulo by apache.
the class ConfigurableMajorCompactionIT method test.
@Test
public void test() throws Exception {
try (AccumuloClient client = Accumulo.newClient().from(getClientProperties()).build()) {
String tableName = getUniqueNames(1)[0];
client.tableOperations().create(tableName);
client.tableOperations().setProperty(tableName, Property.TABLE_COMPACTION_STRATEGY.getKey(), TestCompactionStrategy.class.getName());
writeFile(client, tableName);
writeFile(client, tableName);
writeFile(client, tableName);
writeFile(client, tableName);
UtilWaitThread.sleep(2_000);
assertEquals(4, countFiles(client));
writeFile(client, tableName);
int count = countFiles(client);
assertTrue(count == 1 || count == 5);
while (count != 1) {
UtilWaitThread.sleep(250);
count = countFiles(client);
}
}
}
use of org.apache.accumulo.core.client.AccumuloClient in project accumulo by apache.
the class ExternalCompactionProgressIT method testProgress.
@Test
public void testProgress() throws Exception {
String table1 = this.getUniqueNames(1)[0];
try (AccumuloClient client = Accumulo.newClient().from(getCluster().getClientProperties()).build()) {
createTable(client, table1, "cs1");
writeData(client, table1, ROWS);
cluster.getClusterControl().startCompactors(Compactor.class, 1, QUEUE1);
cluster.getClusterControl().startCoordinator(CompactionCoordinator.class);
Thread checkerThread = startChecker();
checkerThread.start();
IteratorSetting setting = new IteratorSetting(50, "Slow", SlowIterator.class);
SlowIterator.setSleepTime(setting, 1);
client.tableOperations().attachIterator(table1, setting, EnumSet.of(IteratorUtil.IteratorScope.majc));
log.info("Compacting table");
compact(client, table1, 2, "DCQ1", true);
verify(client, table1, 2, ROWS);
log.info("Done Compacting table");
compactionFinished.set(true);
checkerThread.join();
verifyProgress();
}
}
use of org.apache.accumulo.core.client.AccumuloClient in project accumulo by apache.
the class AccumuloClientIT method testGetAccumuloClientFromConnector.
@SuppressWarnings("deprecation")
@Test
public void testGetAccumuloClientFromConnector() throws Exception {
try (AccumuloClient client1 = Accumulo.newClient().from(getClientProps()).build()) {
org.apache.accumulo.core.client.Connector c = org.apache.accumulo.core.client.Connector.from(client1);
String tableName = getUniqueNames(1)[0];
c.tableOperations().create(tableName);
try (AccumuloClient client2 = org.apache.accumulo.core.client.Connector.newClient(c)) {
assertTrue(client2.tableOperations().list().contains(tableName));
}
// closing client2 should not have had an impact on the connector or client1
assertTrue(client1.tableOperations().list().contains(tableName));
assertTrue(c.tableOperations().list().contains(tableName));
}
}
Aggregations