Search in sources :

Example 1 with MAX_DATA

use of org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.MAX_DATA in project accumulo by apache.

the class ExternalCompaction_2_IT method testSplitCancelsExternalCompaction.

@Test
public void testSplitCancelsExternalCompaction() throws Exception {
    getCluster().getClusterControl().startCoordinator(CompactionCoordinator.class);
    getCluster().getClusterControl().startCompactors(ExternalDoNothingCompactor.class, 1, QUEUE1);
    String table1 = this.getUniqueNames(1)[0];
    try (AccumuloClient client = Accumulo.newClient().from(getCluster().getClientProperties()).build()) {
        createTable(client, table1, "cs1");
        TableId tid = getCluster().getServerContext().getTableId(table1);
        writeData(client, table1);
        compact(client, table1, 2, QUEUE1, false);
        // Wait for the compaction to start by waiting for 1 external compaction column
        Set<ExternalCompactionId> ecids = ExternalCompactionTestUtils.waitForCompactionStartAndReturnEcids(getCluster().getServerContext(), tid);
        // Confirm that this ECID shows up in RUNNING set
        int matches = ExternalCompactionTestUtils.confirmCompactionRunning(getCluster().getServerContext(), ecids);
        assertTrue(matches > 0);
        // ExternalDoNothingCompactor will not compact, it will wait, split the table.
        SortedSet<Text> splits = new TreeSet<>();
        int jump = MAX_DATA / 5;
        for (int r = jump; r < MAX_DATA; r += jump) {
            splits.add(new Text(row(r)));
        }
        client.tableOperations().addSplits(table1, splits);
        confirmCompactionCompleted(getCluster().getServerContext(), ecids, TCompactionState.CANCELLED);
        // ensure compaction ids were deleted by split operation from metadata table
        try (TabletsMetadata tm = getCluster().getServerContext().getAmple().readTablets().forTable(tid).fetch(ColumnType.ECOMP).build()) {
            Set<ExternalCompactionId> ecids2 = tm.stream().flatMap(t -> t.getExternalCompactions().keySet().stream()).collect(Collectors.toSet());
            assertTrue(Collections.disjoint(ecids, ecids2));
        }
        // We need to cancel the compaction or delete the table here because we initiate a user
        // compaction above in the test. Even though the external compaction was cancelled
        // because we split the table, FaTE will continue to queue up a compaction
        client.tableOperations().cancelCompaction(table1);
    }
}
Also used : AccumuloClient(org.apache.accumulo.core.client.AccumuloClient) TableId(org.apache.accumulo.core.data.TableId) QUEUE4(org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.QUEUE4) TableId(org.apache.accumulo.core.data.TableId) QUEUE5(org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.QUEUE5) ExternalCompactionTestUtils.confirmCompactionRunning(org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.confirmCompactionRunning) SortedSet(java.util.SortedSet) BeforeClass(org.junit.BeforeClass) TabletsMetadata(org.apache.accumulo.core.metadata.schema.TabletsMetadata) UtilWaitThread(org.apache.accumulo.fate.util.UtilWaitThread) LoggerFactory(org.slf4j.LoggerFactory) ExternalCompactionTestUtils.compact(org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.compact) ExternalCompactionTestUtils.writeData(org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.writeData) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Text(org.apache.hadoop.io.Text) ExternalCompactionTestUtils.getRunningCompactions(org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.getRunningCompactions) Accumulo(org.apache.accumulo.core.client.Accumulo) QUEUE1(org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.QUEUE1) QUEUE2(org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.QUEUE2) TreeSet(java.util.TreeSet) QUEUE3(org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.QUEUE3) CompactionCoordinator(org.apache.accumulo.coordinator.CompactionCoordinator) TCompactionState(org.apache.accumulo.core.compaction.thrift.TCompactionState) ExternalCompactionId(org.apache.accumulo.core.metadata.schema.ExternalCompactionId) ExternalCompactionTestUtils.row(org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.row) MiniAccumuloConfigImpl(org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl) Configuration(org.apache.hadoop.conf.Configuration) After(org.junit.After) Assert.fail(org.junit.Assert.fail) Compactor(org.apache.accumulo.compactor.Compactor) Property(org.apache.accumulo.core.conf.Property) ServerType(org.apache.accumulo.minicluster.ServerType) Logger(org.slf4j.Logger) ExternalCompactionTestUtils.confirmCompactionCompleted(org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.confirmCompactionCompleted) Assert.assertTrue(org.junit.Assert.assertTrue) Set(java.util.Set) MiniClusterConfigurationCallback(org.apache.accumulo.harness.MiniClusterConfigurationCallback) Test(org.junit.Test) Collectors(java.util.stream.Collectors) AccumuloClient(org.apache.accumulo.core.client.AccumuloClient) SharedMiniClusterBase(org.apache.accumulo.harness.SharedMiniClusterBase) ExternalCompactionTestUtils.getFinalStatesForTable(org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.getFinalStatesForTable) TExternalCompactionList(org.apache.accumulo.core.compaction.thrift.TExternalCompactionList) ExternalCompactionTestUtils.waitForCompactionStartAndReturnEcids(org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.waitForCompactionStartAndReturnEcids) MAX_DATA(org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.MAX_DATA) ExternalCompactionTestUtils.createTable(org.apache.accumulo.test.compaction.ExternalCompactionTestUtils.createTable) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) ColumnType(org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType) ExternalCompactionId(org.apache.accumulo.core.metadata.schema.ExternalCompactionId) TreeSet(java.util.TreeSet) TabletsMetadata(org.apache.accumulo.core.metadata.schema.TabletsMetadata) Text(org.apache.hadoop.io.Text) Test(org.junit.Test)

Aggregations

Collections (java.util.Collections)1 Set (java.util.Set)1 SortedSet (java.util.SortedSet)1 TreeSet (java.util.TreeSet)1 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)1 Collectors (java.util.stream.Collectors)1 Compactor (org.apache.accumulo.compactor.Compactor)1 CompactionCoordinator (org.apache.accumulo.coordinator.CompactionCoordinator)1 Accumulo (org.apache.accumulo.core.client.Accumulo)1 AccumuloClient (org.apache.accumulo.core.client.AccumuloClient)1 TCompactionState (org.apache.accumulo.core.compaction.thrift.TCompactionState)1 TExternalCompactionList (org.apache.accumulo.core.compaction.thrift.TExternalCompactionList)1 Property (org.apache.accumulo.core.conf.Property)1 TableId (org.apache.accumulo.core.data.TableId)1 ExternalCompactionId (org.apache.accumulo.core.metadata.schema.ExternalCompactionId)1 ColumnType (org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType)1 TabletsMetadata (org.apache.accumulo.core.metadata.schema.TabletsMetadata)1 UtilWaitThread (org.apache.accumulo.fate.util.UtilWaitThread)1 MiniClusterConfigurationCallback (org.apache.accumulo.harness.MiniClusterConfigurationCallback)1 SharedMiniClusterBase (org.apache.accumulo.harness.SharedMiniClusterBase)1