Search in sources :

Example 1 with TabletsMetadata

use of org.apache.accumulo.core.metadata.schema.TabletsMetadata in project accumulo by apache.

the class CompactionDriver method isReady.

@Override
public long isReady(long tid, Manager manager) throws Exception {
    if (tableId.equals(RootTable.ID)) {
        // this codes not properly handle the root table. See #798
        return 0;
    }
    String zCancelID = createCompactionCancellationPath(manager.getInstanceID(), tableId);
    ZooReaderWriter zoo = manager.getContext().getZooReaderWriter();
    if (Long.parseLong(new String(zoo.getData(zCancelID))) >= compactId) {
        // compaction was canceled
        throw new AcceptableThriftTableOperationException(tableId.canonical(), null, TableOperation.COMPACT, TableOperationExceptionType.OTHER, TableOperationsImpl.COMPACTION_CANCELED_MSG);
    }
    String deleteMarkerPath = PreDeleteTable.createDeleteMarkerPath(manager.getInstanceID(), tableId);
    if (zoo.exists(deleteMarkerPath)) {
        // table is being deleted
        throw new AcceptableThriftTableOperationException(tableId.canonical(), null, TableOperation.COMPACT, TableOperationExceptionType.OTHER, TableOperationsImpl.TABLE_DELETED_MSG);
    }
    MapCounter<TServerInstance> serversToFlush = new MapCounter<>();
    long t1 = System.currentTimeMillis();
    int tabletsToWaitFor = 0;
    int tabletCount = 0;
    TabletsMetadata tablets = TabletsMetadata.builder(manager.getContext()).forTable(tableId).overlapping(startRow, endRow).fetch(LOCATION, PREV_ROW, COMPACT_ID).build();
    for (TabletMetadata tablet : tablets) {
        if (tablet.getCompactId().orElse(-1) < compactId) {
            tabletsToWaitFor++;
            if (tablet.hasCurrent()) {
                serversToFlush.increment(tablet.getLocation(), 1);
            }
        }
        tabletCount++;
    }
    long scanTime = System.currentTimeMillis() - t1;
    manager.getContext().clearTableListCache();
    if (tabletCount == 0 && !manager.getContext().tableNodeExists(tableId))
        throw new AcceptableThriftTableOperationException(tableId.canonical(), null, TableOperation.COMPACT, TableOperationExceptionType.NOTFOUND, null);
    if (serversToFlush.size() == 0 && manager.getContext().getTableState(tableId) == TableState.OFFLINE)
        throw new AcceptableThriftTableOperationException(tableId.canonical(), null, TableOperation.COMPACT, TableOperationExceptionType.OFFLINE, null);
    if (tabletsToWaitFor == 0)
        return 0;
    for (TServerInstance tsi : serversToFlush.keySet()) {
        try {
            final TServerConnection server = manager.getConnection(tsi);
            if (server != null)
                server.compact(manager.getManagerLock(), tableId.canonical(), startRow, endRow);
        } catch (TException ex) {
            LoggerFactory.getLogger(CompactionDriver.class).error(ex.toString());
        }
    }
    long sleepTime = 500;
    // make wait time depend on the server with the most to compact
    if (serversToFlush.size() > 0)
        sleepTime = serversToFlush.max() * sleepTime;
    sleepTime = Math.max(2 * scanTime, sleepTime);
    sleepTime = Math.min(sleepTime, 30000);
    return sleepTime;
}
Also used : TServerConnection(org.apache.accumulo.server.manager.LiveTServerSet.TServerConnection) TException(org.apache.thrift.TException) TabletsMetadata(org.apache.accumulo.core.metadata.schema.TabletsMetadata) ZooReaderWriter(org.apache.accumulo.fate.zookeeper.ZooReaderWriter) MapCounter(org.apache.accumulo.core.util.MapCounter) TabletMetadata(org.apache.accumulo.core.metadata.schema.TabletMetadata) TServerInstance(org.apache.accumulo.core.metadata.TServerInstance) AcceptableThriftTableOperationException(org.apache.accumulo.core.clientImpl.AcceptableThriftTableOperationException)

Example 2 with TabletsMetadata

use of org.apache.accumulo.core.metadata.schema.TabletsMetadata in project accumulo by apache.

the class ExternalCompaction_2_IT method testSplitCancelsExternalCompaction.

@Test
public void testSplitCancelsExternalCompaction() throws Exception {
    getCluster().getClusterControl().startCoordinator(CompactionCoordinator.class);
    getCluster().getClusterControl().startCompactors(ExternalDoNothingCompactor.class, 1, QUEUE1);
    String table1 = this.getUniqueNames(1)[0];
    try (AccumuloClient client = Accumulo.newClient().from(getCluster().getClientProperties()).build()) {
        createTable(client, table1, "cs1");
        TableId tid = getCluster().getServerContext().getTableId(table1);
        writeData(client, table1);
        compact(client, table1, 2, QUEUE1, false);
        // Wait for the compaction to start by waiting for 1 external compaction column
        Set<ExternalCompactionId> ecids = ExternalCompactionTestUtils.waitForCompactionStartAndReturnEcids(getCluster().getServerContext(), tid);
        // Confirm that this ECID shows up in RUNNING set
        int matches = ExternalCompactionTestUtils.confirmCompactionRunning(getCluster().getServerContext(), ecids);
        assertTrue(matches > 0);
        // ExternalDoNothingCompactor will not compact, it will wait, split the table.
        SortedSet<Text> splits = new TreeSet<>();
        int jump = MAX_DATA / 5;
        for (int r = jump; r < MAX_DATA; r += jump) {
            splits.add(new Text(row(r)));
        }
        client.tableOperations().addSplits(table1, splits);
        confirmCompactionCompleted(getCluster().getServerContext(), ecids, TCompactionState.CANCELLED);
        // ensure compaction ids were deleted by split operation from metadata table
        try (TabletsMetadata tm = getCluster().getServerContext().getAmple().readTablets().forTable(tid).fetch(ColumnType.ECOMP).build()) {
            Set<ExternalCompactionId> ecids2 = tm.stream().flatMap(t -> t.getExternalCompactions().keySet().stream()).collect(Collectors.toSet());
            assertTrue(Collections.disjoint(ecids, ecids2));
        }
        // We need to cancel the compaction or delete the table here because we initiate a user
        // compaction above in the test. Even though the external compaction was cancelled
        // because we split the table, FaTE will continue to queue up a compaction
        client.tableOperations().cancelCompaction(table1);
    }
}
Also used : AccumuloClient(org.apache.accumulo.core.client.AccumuloClient) TableId(org.apache.accumulo.core.data.TableId) QUEUE4(org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.QUEUE4) TableId(org.apache.accumulo.core.data.TableId) QUEUE5(org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.QUEUE5) ExternalCompactionTestUtils.confirmCompactionRunning(org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.confirmCompactionRunning) SortedSet(java.util.SortedSet) BeforeClass(org.junit.BeforeClass) TabletsMetadata(org.apache.accumulo.core.metadata.schema.TabletsMetadata) UtilWaitThread(org.apache.accumulo.fate.util.UtilWaitThread) LoggerFactory(org.slf4j.LoggerFactory) ExternalCompactionTestUtils.compact(org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.compact) ExternalCompactionTestUtils.writeData(org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.writeData) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Text(org.apache.hadoop.io.Text) ExternalCompactionTestUtils.getRunningCompactions(org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.getRunningCompactions) Accumulo(org.apache.accumulo.core.client.Accumulo) QUEUE1(org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.QUEUE1) QUEUE2(org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.QUEUE2) TreeSet(java.util.TreeSet) QUEUE3(org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.QUEUE3) CompactionCoordinator(org.apache.accumulo.coordinator.CompactionCoordinator) TCompactionState(org.apache.accumulo.core.compaction.thrift.TCompactionState) ExternalCompactionId(org.apache.accumulo.core.metadata.schema.ExternalCompactionId) ExternalCompactionTestUtils.row(org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.row) MiniAccumuloConfigImpl(org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl) Configuration(org.apache.hadoop.conf.Configuration) After(org.junit.After) Assert.fail(org.junit.Assert.fail) Compactor(org.apache.accumulo.compactor.Compactor) Property(org.apache.accumulo.core.conf.Property) ServerType(org.apache.accumulo.minicluster.ServerType) Logger(org.slf4j.Logger) ExternalCompactionTestUtils.confirmCompactionCompleted(org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.confirmCompactionCompleted) Assert.assertTrue(org.junit.Assert.assertTrue) Set(java.util.Set) MiniClusterConfigurationCallback(org.apache.accumulo.harness.MiniClusterConfigurationCallback) Test(org.junit.Test) Collectors(java.util.stream.Collectors) AccumuloClient(org.apache.accumulo.core.client.AccumuloClient) SharedMiniClusterBase(org.apache.accumulo.harness.SharedMiniClusterBase) ExternalCompactionTestUtils.getFinalStatesForTable(org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.getFinalStatesForTable) TExternalCompactionList(org.apache.accumulo.core.compaction.thrift.TExternalCompactionList) ExternalCompactionTestUtils.waitForCompactionStartAndReturnEcids(org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.waitForCompactionStartAndReturnEcids) MAX_DATA(org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.MAX_DATA) ExternalCompactionTestUtils.createTable(org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.createTable) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) ColumnType(org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType) ExternalCompactionId(org.apache.accumulo.core.metadata.schema.ExternalCompactionId) TreeSet(java.util.TreeSet) TabletsMetadata(org.apache.accumulo.core.metadata.schema.TabletsMetadata) Text(org.apache.hadoop.io.Text) Test(org.junit.Test)

Example 3 with TabletsMetadata

use of org.apache.accumulo.core.metadata.schema.TabletsMetadata in project accumulo by apache.

the class ExternalCompaction_2_IT method testDeleteTableCancelsExternalCompaction.

@Test
public void testDeleteTableCancelsExternalCompaction() throws Exception {
    getCluster().getClusterControl().startCoordinator(CompactionCoordinator.class);
    getCluster().getClusterControl().startCompactors(ExternalDoNothingCompactor.class, 1, QUEUE5);
    String table1 = this.getUniqueNames(1)[0];
    try (AccumuloClient client = Accumulo.newClient().from(getCluster().getClientProperties()).build()) {
        createTable(client, table1, "cs5");
        // set compaction ratio to 1 so that majc occurs naturally, not user compaction
        // user compaction blocks delete
        client.tableOperations().setProperty(table1, Property.TABLE_MAJC_RATIO.toString(), "1.0");
        // cause multiple rfiles to be created
        writeData(client, table1);
        writeData(client, table1);
        writeData(client, table1);
        writeData(client, table1);
        TableId tid = getCluster().getServerContext().getTableId(table1);
        // Wait for the compaction to start by waiting for 1 external compaction column
        Set<ExternalCompactionId> ecids = ExternalCompactionTestUtils.waitForCompactionStartAndReturnEcids(getCluster().getServerContext(), tid);
        // Confirm that this ECID shows up in RUNNING set
        int matches = ExternalCompactionTestUtils.confirmCompactionRunning(getCluster().getServerContext(), ecids);
        assertTrue(matches > 0);
        client.tableOperations().delete(table1);
        confirmCompactionCompleted(getCluster().getServerContext(), ecids, TCompactionState.CANCELLED);
        TabletsMetadata tm = getCluster().getServerContext().getAmple().readTablets().forTable(tid).fetch(ColumnType.ECOMP).build();
        assertEquals(0, tm.stream().count());
        tm.close();
    }
}
Also used : AccumuloClient(org.apache.accumulo.core.client.AccumuloClient) TableId(org.apache.accumulo.core.data.TableId) ExternalCompactionId(org.apache.accumulo.core.metadata.schema.ExternalCompactionId) TabletsMetadata(org.apache.accumulo.core.metadata.schema.TabletsMetadata) Test(org.junit.Test)

Example 4 with TabletsMetadata

use of org.apache.accumulo.core.metadata.schema.TabletsMetadata in project accumulo by apache.

the class ExternalCompaction_3_IT method testMergeCancelsExternalCompaction.

@Test
public void testMergeCancelsExternalCompaction() throws Exception {
    getCluster().getClusterControl().startCoordinator(CompactionCoordinator.class);
    getCluster().getClusterControl().startCompactors(ExternalDoNothingCompactor.class, 1, QUEUE1);
    String table1 = this.getUniqueNames(1)[0];
    try (AccumuloClient client = Accumulo.newClient().from(getCluster().getClientProperties()).build()) {
        createTable(client, table1, "cs1", 2);
        // set compaction ratio to 1 so that majc occurs naturally, not user compaction
        // user compaction blocks merge
        client.tableOperations().setProperty(table1, Property.TABLE_MAJC_RATIO.toString(), "1.0");
        // cause multiple rfiles to be created
        writeData(client, table1);
        writeData(client, table1);
        writeData(client, table1);
        writeData(client, table1);
        TableId tid = getCluster().getServerContext().getTableId(table1);
        // Wait for the compaction to start by waiting for 1 external compaction column
        Set<ExternalCompactionId> ecids = waitForCompactionStartAndReturnEcids(getCluster().getServerContext(), tid);
        var md = new ArrayList<TabletMetadata>();
        try (TabletsMetadata tm = getCluster().getServerContext().getAmple().readTablets().forTable(tid).fetch(ColumnType.PREV_ROW).build()) {
            tm.forEach(t -> md.add(t));
            assertEquals(2, md.size());
        }
        // Merge - blocking operation
        Text start = md.get(0).getPrevEndRow();
        Text end = md.get(1).getEndRow();
        client.tableOperations().merge(table1, start, end);
        confirmCompactionCompleted(getCluster().getServerContext(), ecids, TCompactionState.CANCELLED);
        // ensure compaction ids were deleted by merge operation from metadata table
        try (TabletsMetadata tm = getCluster().getServerContext().getAmple().readTablets().forTable(tid).fetch(ColumnType.ECOMP).build()) {
            Set<ExternalCompactionId> ecids2 = tm.stream().flatMap(t -> t.getExternalCompactions().keySet().stream()).collect(Collectors.toSet());
            // keep checking until test times out
            while (!Collections.disjoint(ecids, ecids2)) {
                UtilWaitThread.sleep(25);
                ecids2 = tm.stream().flatMap(t -> t.getExternalCompactions().keySet().stream()).collect(Collectors.toSet());
            }
        }
        // We need to cancel the compaction or delete the table here because we initiate a user
        // compaction above in the test. Even though the external compaction was cancelled
        // because we split the table, FaTE will continue to queue up a compaction
        client.tableOperations().delete(table1);
    }
}
Also used : AccumuloClient(org.apache.accumulo.core.client.AccumuloClient) TableId(org.apache.accumulo.core.data.TableId) TableId(org.apache.accumulo.core.data.TableId) BeforeClass(org.junit.BeforeClass) TabletsMetadata(org.apache.accumulo.core.metadata.schema.TabletsMetadata) UtilWaitThread(org.apache.accumulo.fate.util.UtilWaitThread) ExternalCompactionTestUtils.compact(org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.compact) ExternalCompactionTestUtils.writeData(org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.writeData) Text(org.apache.hadoop.io.Text) ExternalCompactionTestUtils.getRunningCompactions(org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.getRunningCompactions) Accumulo(org.apache.accumulo.core.client.Accumulo) QUEUE1(org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.QUEUE1) QUEUE2(org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.QUEUE2) ArrayList(java.util.ArrayList) TabletMetadata(org.apache.accumulo.core.metadata.schema.TabletMetadata) CompactionCoordinator(org.apache.accumulo.coordinator.CompactionCoordinator) TCompactionState(org.apache.accumulo.core.compaction.thrift.TCompactionState) ExternalCompactionId(org.apache.accumulo.core.metadata.schema.ExternalCompactionId) MiniAccumuloConfigImpl(org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl) Configuration(org.apache.hadoop.conf.Configuration) After(org.junit.After) ExternalCompactionTestUtils.getLastState(org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.getLastState) Property(org.apache.accumulo.core.conf.Property) ServerType(org.apache.accumulo.minicluster.ServerType) ExternalCompactionTestUtils.confirmCompactionCompleted(org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.confirmCompactionCompleted) Assert.assertTrue(org.junit.Assert.assertTrue) Set(java.util.Set) TExternalCompaction(org.apache.accumulo.core.compaction.thrift.TExternalCompaction) MiniClusterConfigurationCallback(org.apache.accumulo.harness.MiniClusterConfigurationCallback) Test(org.junit.Test) Collectors(java.util.stream.Collectors) AccumuloClient(org.apache.accumulo.core.client.AccumuloClient) SharedMiniClusterBase(org.apache.accumulo.harness.SharedMiniClusterBase) TExternalCompactionList(org.apache.accumulo.core.compaction.thrift.TExternalCompactionList) ExternalCompactionTestUtils.waitForCompactionStartAndReturnEcids(org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.waitForCompactionStartAndReturnEcids) ExternalCompactionTestUtils.createTable(org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.createTable) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) ColumnType(org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType) ExternalCompactionId(org.apache.accumulo.core.metadata.schema.ExternalCompactionId) TabletsMetadata(org.apache.accumulo.core.metadata.schema.TabletsMetadata) ArrayList(java.util.ArrayList) Text(org.apache.hadoop.io.Text) Test(org.junit.Test)

Example 5 with TabletsMetadata

use of org.apache.accumulo.core.metadata.schema.TabletsMetadata in project accumulo by apache.

the class ManagerClientServiceHandler method waitForFlush.

@Override
public void waitForFlush(TInfo tinfo, TCredentials c, String tableIdStr, ByteBuffer startRowBB, ByteBuffer endRowBB, long flushID, long maxLoops) throws ThriftSecurityException, ThriftTableOperationException {
    TableId tableId = TableId.of(tableIdStr);
    NamespaceId namespaceId = getNamespaceIdFromTableId(TableOperation.FLUSH, tableId);
    if (!manager.security.canFlush(c, tableId, namespaceId))
        throw new ThriftSecurityException(c.getPrincipal(), SecurityErrorCode.PERMISSION_DENIED);
    Text startRow = ByteBufferUtil.toText(startRowBB);
    Text endRow = ByteBufferUtil.toText(endRowBB);
    if (endRow != null && startRow != null && startRow.compareTo(endRow) >= 0)
        throw new ThriftTableOperationException(tableId.canonical(), null, TableOperation.FLUSH, TableOperationExceptionType.BAD_RANGE, "start row must be less than end row");
    Set<TServerInstance> serversToFlush = new HashSet<>(manager.tserverSet.getCurrentServers());
    for (long l = 0; l < maxLoops; l++) {
        for (TServerInstance instance : serversToFlush) {
            try {
                final TServerConnection server = manager.tserverSet.getConnection(instance);
                if (server != null)
                    server.flush(manager.managerLock, tableId, ByteBufferUtil.toBytes(startRowBB), ByteBufferUtil.toBytes(endRowBB));
            } catch (TException ex) {
                Manager.log.error(ex.toString());
            }
        }
        if (tableId.equals(RootTable.ID))
            // this code does not properly handle the root tablet. See #798
            break;
        if (l == maxLoops - 1)
            break;
        sleepUninterruptibly(50, TimeUnit.MILLISECONDS);
        serversToFlush.clear();
        try (TabletsMetadata tablets = TabletsMetadata.builder(manager.getContext()).forTable(tableId).overlapping(startRow, endRow).fetch(FLUSH_ID, LOCATION, LOGS, PREV_ROW).build()) {
            int tabletsToWaitFor = 0;
            int tabletCount = 0;
            for (TabletMetadata tablet : tablets) {
                int logs = tablet.getLogs().size();
                // when tablet is not online and has no logs, there is no reason to wait for it
                if ((tablet.hasCurrent() || logs > 0) && tablet.getFlushId().orElse(-1) < flushID) {
                    tabletsToWaitFor++;
                    if (tablet.hasCurrent())
                        serversToFlush.add(tablet.getLocation());
                }
                tabletCount++;
            }
            if (tabletsToWaitFor == 0)
                break;
            if (tabletCount == 0 && !manager.getContext().tableNodeExists(tableId))
                throw new ThriftTableOperationException(tableId.canonical(), null, TableOperation.FLUSH, TableOperationExceptionType.NOTFOUND, null);
        } catch (TabletDeletedException e) {
            Manager.log.debug("Failed to scan {} table to wait for flush {}", MetadataTable.NAME, tableId, e);
        }
    }
}
Also used : TableId(org.apache.accumulo.core.data.TableId) TException(org.apache.thrift.TException) Text(org.apache.hadoop.io.Text) TabletDeletedException(org.apache.accumulo.core.metadata.schema.TabletDeletedException) ThriftSecurityException(org.apache.accumulo.core.clientImpl.thrift.ThriftSecurityException) TServerInstance(org.apache.accumulo.core.metadata.TServerInstance) TServerConnection(org.apache.accumulo.server.manager.LiveTServerSet.TServerConnection) TabletsMetadata(org.apache.accumulo.core.metadata.schema.TabletsMetadata) TabletMetadata(org.apache.accumulo.core.metadata.schema.TabletMetadata) ThriftTableOperationException(org.apache.accumulo.core.clientImpl.thrift.ThriftTableOperationException) NamespaceId(org.apache.accumulo.core.data.NamespaceId) HashSet(java.util.HashSet)

Aggregations

TabletsMetadata (org.apache.accumulo.core.metadata.schema.TabletsMetadata)14 TabletMetadata (org.apache.accumulo.core.metadata.schema.TabletMetadata)9 AccumuloClient (org.apache.accumulo.core.client.AccumuloClient)7 TableId (org.apache.accumulo.core.data.TableId)7 Text (org.apache.hadoop.io.Text)7 Test (org.junit.Test)7 Property (org.apache.accumulo.core.conf.Property)5 ArrayList (java.util.ArrayList)4 TreeSet (java.util.TreeSet)4 Accumulo (org.apache.accumulo.core.client.Accumulo)4 ExternalCompactionId (org.apache.accumulo.core.metadata.schema.ExternalCompactionId)4 Set (java.util.Set)3 Collectors (java.util.stream.Collectors)3 KeyExtent (org.apache.accumulo.core.dataImpl.KeyExtent)3 MiniClusterConfigurationCallback (org.apache.accumulo.harness.MiniClusterConfigurationCallback)3 SharedMiniClusterBase (org.apache.accumulo.harness.SharedMiniClusterBase)3 ServerType (org.apache.accumulo.minicluster.ServerType)3 MiniAccumuloConfigImpl (org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl)3 Configuration (org.apache.hadoop.conf.Configuration)3 Assert.assertEquals (org.junit.Assert.assertEquals)3