use of org.apache.accumulo.core.metadata.schema.TabletsMetadata in project accumulo by apache.
the class CompactionDriver method isReady.
@Override
public long isReady(long tid, Manager manager) throws Exception {
if (tableId.equals(RootTable.ID)) {
// this codes not properly handle the root table. See #798
return 0;
}
String zCancelID = createCompactionCancellationPath(manager.getInstanceID(), tableId);
ZooReaderWriter zoo = manager.getContext().getZooReaderWriter();
if (Long.parseLong(new String(zoo.getData(zCancelID))) >= compactId) {
// compaction was canceled
throw new AcceptableThriftTableOperationException(tableId.canonical(), null, TableOperation.COMPACT, TableOperationExceptionType.OTHER, TableOperationsImpl.COMPACTION_CANCELED_MSG);
}
String deleteMarkerPath = PreDeleteTable.createDeleteMarkerPath(manager.getInstanceID(), tableId);
if (zoo.exists(deleteMarkerPath)) {
// table is being deleted
throw new AcceptableThriftTableOperationException(tableId.canonical(), null, TableOperation.COMPACT, TableOperationExceptionType.OTHER, TableOperationsImpl.TABLE_DELETED_MSG);
}
MapCounter<TServerInstance> serversToFlush = new MapCounter<>();
long t1 = System.currentTimeMillis();
int tabletsToWaitFor = 0;
int tabletCount = 0;
TabletsMetadata tablets = TabletsMetadata.builder(manager.getContext()).forTable(tableId).overlapping(startRow, endRow).fetch(LOCATION, PREV_ROW, COMPACT_ID).build();
for (TabletMetadata tablet : tablets) {
if (tablet.getCompactId().orElse(-1) < compactId) {
tabletsToWaitFor++;
if (tablet.hasCurrent()) {
serversToFlush.increment(tablet.getLocation(), 1);
}
}
tabletCount++;
}
long scanTime = System.currentTimeMillis() - t1;
manager.getContext().clearTableListCache();
if (tabletCount == 0 && !manager.getContext().tableNodeExists(tableId))
throw new AcceptableThriftTableOperationException(tableId.canonical(), null, TableOperation.COMPACT, TableOperationExceptionType.NOTFOUND, null);
if (serversToFlush.size() == 0 && manager.getContext().getTableState(tableId) == TableState.OFFLINE)
throw new AcceptableThriftTableOperationException(tableId.canonical(), null, TableOperation.COMPACT, TableOperationExceptionType.OFFLINE, null);
if (tabletsToWaitFor == 0)
return 0;
for (TServerInstance tsi : serversToFlush.keySet()) {
try {
final TServerConnection server = manager.getConnection(tsi);
if (server != null)
server.compact(manager.getManagerLock(), tableId.canonical(), startRow, endRow);
} catch (TException ex) {
LoggerFactory.getLogger(CompactionDriver.class).error(ex.toString());
}
}
long sleepTime = 500;
// make wait time depend on the server with the most to compact
if (serversToFlush.size() > 0)
sleepTime = serversToFlush.max() * sleepTime;
sleepTime = Math.max(2 * scanTime, sleepTime);
sleepTime = Math.min(sleepTime, 30000);
return sleepTime;
}
use of org.apache.accumulo.core.metadata.schema.TabletsMetadata in project accumulo by apache.
the class ExternalCompaction_2_IT method testSplitCancelsExternalCompaction.
@Test
public void testSplitCancelsExternalCompaction() throws Exception {
getCluster().getClusterControl().startCoordinator(CompactionCoordinator.class);
getCluster().getClusterControl().startCompactors(ExternalDoNothingCompactor.class, 1, QUEUE1);
String table1 = this.getUniqueNames(1)[0];
try (AccumuloClient client = Accumulo.newClient().from(getCluster().getClientProperties()).build()) {
createTable(client, table1, "cs1");
TableId tid = getCluster().getServerContext().getTableId(table1);
writeData(client, table1);
compact(client, table1, 2, QUEUE1, false);
// Wait for the compaction to start by waiting for 1 external compaction column
Set<ExternalCompactionId> ecids = ExternalCompactionTestUtils.waitForCompactionStartAndReturnEcids(getCluster().getServerContext(), tid);
// Confirm that this ECID shows up in RUNNING set
int matches = ExternalCompactionTestUtils.confirmCompactionRunning(getCluster().getServerContext(), ecids);
assertTrue(matches > 0);
// ExternalDoNothingCompactor will not compact, it will wait, split the table.
SortedSet<Text> splits = new TreeSet<>();
int jump = MAX_DATA / 5;
for (int r = jump; r < MAX_DATA; r += jump) {
splits.add(new Text(row(r)));
}
client.tableOperations().addSplits(table1, splits);
confirmCompactionCompleted(getCluster().getServerContext(), ecids, TCompactionState.CANCELLED);
// ensure compaction ids were deleted by split operation from metadata table
try (TabletsMetadata tm = getCluster().getServerContext().getAmple().readTablets().forTable(tid).fetch(ColumnType.ECOMP).build()) {
Set<ExternalCompactionId> ecids2 = tm.stream().flatMap(t -> t.getExternalCompactions().keySet().stream()).collect(Collectors.toSet());
assertTrue(Collections.disjoint(ecids, ecids2));
}
// We need to cancel the compaction or delete the table here because we initiate a user
// compaction above in the test. Even though the external compaction was cancelled
// because we split the table, FaTE will continue to queue up a compaction
client.tableOperations().cancelCompaction(table1);
}
}
use of org.apache.accumulo.core.metadata.schema.TabletsMetadata in project accumulo by apache.
the class ExternalCompaction_2_IT method testDeleteTableCancelsExternalCompaction.
@Test
public void testDeleteTableCancelsExternalCompaction() throws Exception {
getCluster().getClusterControl().startCoordinator(CompactionCoordinator.class);
getCluster().getClusterControl().startCompactors(ExternalDoNothingCompactor.class, 1, QUEUE5);
String table1 = this.getUniqueNames(1)[0];
try (AccumuloClient client = Accumulo.newClient().from(getCluster().getClientProperties()).build()) {
createTable(client, table1, "cs5");
// set compaction ratio to 1 so that majc occurs naturally, not user compaction
// user compaction blocks delete
client.tableOperations().setProperty(table1, Property.TABLE_MAJC_RATIO.toString(), "1.0");
// cause multiple rfiles to be created
writeData(client, table1);
writeData(client, table1);
writeData(client, table1);
writeData(client, table1);
TableId tid = getCluster().getServerContext().getTableId(table1);
// Wait for the compaction to start by waiting for 1 external compaction column
Set<ExternalCompactionId> ecids = ExternalCompactionTestUtils.waitForCompactionStartAndReturnEcids(getCluster().getServerContext(), tid);
// Confirm that this ECID shows up in RUNNING set
int matches = ExternalCompactionTestUtils.confirmCompactionRunning(getCluster().getServerContext(), ecids);
assertTrue(matches > 0);
client.tableOperations().delete(table1);
confirmCompactionCompleted(getCluster().getServerContext(), ecids, TCompactionState.CANCELLED);
TabletsMetadata tm = getCluster().getServerContext().getAmple().readTablets().forTable(tid).fetch(ColumnType.ECOMP).build();
assertEquals(0, tm.stream().count());
tm.close();
}
}
use of org.apache.accumulo.core.metadata.schema.TabletsMetadata in project accumulo by apache.
the class ExternalCompaction_3_IT method testMergeCancelsExternalCompaction.
@Test
public void testMergeCancelsExternalCompaction() throws Exception {
getCluster().getClusterControl().startCoordinator(CompactionCoordinator.class);
getCluster().getClusterControl().startCompactors(ExternalDoNothingCompactor.class, 1, QUEUE1);
String table1 = this.getUniqueNames(1)[0];
try (AccumuloClient client = Accumulo.newClient().from(getCluster().getClientProperties()).build()) {
createTable(client, table1, "cs1", 2);
// set compaction ratio to 1 so that majc occurs naturally, not user compaction
// user compaction blocks merge
client.tableOperations().setProperty(table1, Property.TABLE_MAJC_RATIO.toString(), "1.0");
// cause multiple rfiles to be created
writeData(client, table1);
writeData(client, table1);
writeData(client, table1);
writeData(client, table1);
TableId tid = getCluster().getServerContext().getTableId(table1);
// Wait for the compaction to start by waiting for 1 external compaction column
Set<ExternalCompactionId> ecids = waitForCompactionStartAndReturnEcids(getCluster().getServerContext(), tid);
var md = new ArrayList<TabletMetadata>();
try (TabletsMetadata tm = getCluster().getServerContext().getAmple().readTablets().forTable(tid).fetch(ColumnType.PREV_ROW).build()) {
tm.forEach(t -> md.add(t));
assertEquals(2, md.size());
}
// Merge - blocking operation
Text start = md.get(0).getPrevEndRow();
Text end = md.get(1).getEndRow();
client.tableOperations().merge(table1, start, end);
confirmCompactionCompleted(getCluster().getServerContext(), ecids, TCompactionState.CANCELLED);
// ensure compaction ids were deleted by merge operation from metadata table
try (TabletsMetadata tm = getCluster().getServerContext().getAmple().readTablets().forTable(tid).fetch(ColumnType.ECOMP).build()) {
Set<ExternalCompactionId> ecids2 = tm.stream().flatMap(t -> t.getExternalCompactions().keySet().stream()).collect(Collectors.toSet());
// keep checking until test times out
while (!Collections.disjoint(ecids, ecids2)) {
UtilWaitThread.sleep(25);
ecids2 = tm.stream().flatMap(t -> t.getExternalCompactions().keySet().stream()).collect(Collectors.toSet());
}
}
// We need to cancel the compaction or delete the table here because we initiate a user
// compaction above in the test. Even though the external compaction was cancelled
// because we split the table, FaTE will continue to queue up a compaction
client.tableOperations().delete(table1);
}
}
use of org.apache.accumulo.core.metadata.schema.TabletsMetadata in project accumulo by apache.
the class ManagerClientServiceHandler method waitForFlush.
@Override
public void waitForFlush(TInfo tinfo, TCredentials c, String tableIdStr, ByteBuffer startRowBB, ByteBuffer endRowBB, long flushID, long maxLoops) throws ThriftSecurityException, ThriftTableOperationException {
TableId tableId = TableId.of(tableIdStr);
NamespaceId namespaceId = getNamespaceIdFromTableId(TableOperation.FLUSH, tableId);
if (!manager.security.canFlush(c, tableId, namespaceId))
throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
Text startRow = ByteBufferUtil.toText(startRowBB);
Text endRow = ByteBufferUtil.toText(endRowBB);
if (endRow != null && startRow != null && startRow.compareTo(endRow) >= 0)
throw new ThriftTableOperationException(tableId.canonical(), null, TableOperation.FLUSH, TableOperationExceptionType.BAD_RANGE, "start row must be less than end row");
Set<TServerInstance> serversToFlush = new HashSet<>(manager.tserverSet.getCurrentServers());
for (long l = 0; l < maxLoops; l++) {
for (TServerInstance instance : serversToFlush) {
try {
final TServerConnection server = manager.tserverSet.getConnection(instance);
if (server != null)
server.flush(manager.managerLock, tableId, ByteBufferUtil.toBytes(startRowBB), ByteBufferUtil.toBytes(endRowBB));
} catch (TException ex) {
Manager.log.error(ex.toString());
}
}
if (tableId.equals(RootTable.ID))
// this code does not properly handle the root tablet. See #798
break;
if (l == maxLoops - 1)
break;
sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
serversToFlush.clear();
try (TabletsMetadata tablets = TabletsMetadata.builder(manager.getContext()).forTable(tableId).overlapping(startRow, endRow).fetch(FLUSH_ID, LOCATION, LOGS, PREV_ROW).build()) {
int tabletsToWaitFor = 0;
int tabletCount = 0;
for (TabletMetadata tablet : tablets) {
int logs = tablet.getLogs().size();
// when tablet is not online and has no logs, there is no reason to wait for it
if ((tablet.hasCurrent() || logs > 0) && tablet.getFlushId().orElse(-1) < flushID) {
tabletsToWaitFor++;
if (tablet.hasCurrent())
serversToFlush.add(tablet.getLocation());
}
tabletCount++;
}
if (tabletsToWaitFor == 0)
break;
if (tabletCount == 0 && !manager.getContext().tableNodeExists(tableId))
throw new ThriftTableOperationException(tableId.canonical(), null, TableOperation.FLUSH, TableOperationExceptionType.NOTFOUND, null);
} catch (TabletDeletedException e) {
Manager.log.debug("Failed to scan {} table to wait for flush {}", MetadataTable.NAME, tableId, e);
}
}
}
Aggregations